2023-10-09T10:16:09.374 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-09T10:16:09.374 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --description upgrade:quincy-x/stress-split/{0-distro/centos_8.stream_container_tools 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} --owner scheduled_teuthology@teuthology --name teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi --archive /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417662 -- /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417662/orig.config.yaml 2023-10-09T10:16:09.447 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T10:16:09.539 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417662 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/centos_8.stream_container_tools 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' first_in_suite: false job_id: '7417662' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic cls tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 5305 sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 targets: smithi125.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLvfSjrBnafMJIFS7H9oHfjQHewGNMax+GQ3sxvUIztX+jkFCA9WPSbSLRc3J7q7WQPVpNnSwcAtA2o/q4iewss= smithi155.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP7/b5hy+Pcu9QddzM6XYEy8fuE7d73rSIMRFCY9EzEPXJxbWWrxGt2DebD8swkrGVC9PJaBejVh7lZlsmOEnrM= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-10-07_14:23:11 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-09T10:16:09.540 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa; will attempt to use it 2023-10-09T10:16:09.541 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks 2023-10-09T10:16:09.541 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-09T10:16:09.542 INFO:teuthology.task.internal:Checking packages... 2023-10-09T10:16:09.626 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0' 2023-10-09T10:16:09.626 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T10:16:09.627 INFO:teuthology.packaging:ref: None 2023-10-09T10:16:09.627 INFO:teuthology.packaging:tag: None 2023-10-09T10:16:09.627 INFO:teuthology.packaging:branch: reef 2023-10-09T10:16:09.627 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T10:16:09.627 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-10-09T10:16:09.835 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-610.g397de1b0 2023-10-09T10:16:09.836 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-09T10:16:09.842 INFO:teuthology.task.internal:no buildpackages task found 2023-10-09T10:16:09.843 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-09T10:16:09.891 INFO:teuthology.task.internal:Saving configuration 2023-10-09T10:16:09.912 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-09T10:16:09.918 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T10:16:09.943 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi125.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417662', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 10:11:03.970578', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLvfSjrBnafMJIFS7H9oHfjQHewGNMax+GQ3sxvUIztX+jkFCA9WPSbSLRc3J7q7WQPVpNnSwcAtA2o/q4iewss='} 2023-10-09T10:16:09.966 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi155.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417662', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 10:11:03.972598', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP7/b5hy+Pcu9QddzM6XYEy8fuE7d73rSIMRFCY9EzEPXJxbWWrxGt2DebD8swkrGVC9PJaBejVh7lZlsmOEnrM='} 2023-10-09T10:16:09.966 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-09T10:16:09.972 INFO:teuthology.task.internal:roles: ubuntu@smithi125.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-10-09T10:16:09.972 INFO:teuthology.task.internal:roles: ubuntu@smithi155.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-10-09T10:16:09.972 INFO:teuthology.run_tasks:Running task console_log... 2023-10-09T10:16:10.066 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f8a21b98af0>, signals=[15]) 2023-10-09T10:16:10.067 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-09T10:16:10.075 INFO:teuthology.task.internal:Opening connections... 2023-10-09T10:16:10.075 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi125.front.sepia.ceph.com 2023-10-09T10:16:10.079 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi125.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T10:16:10.148 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi155.front.sepia.ceph.com 2023-10-09T10:16:10.149 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi155.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T10:16:10.215 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-09T10:16:10.221 DEBUG:teuthology.orchestra.run.smithi125:> uname -m 2023-10-09T10:16:10.244 INFO:teuthology.orchestra.run.smithi125.stdout:x86_64 2023-10-09T10:16:10.244 DEBUG:teuthology.orchestra.run.smithi125:> cat /etc/os-release 2023-10-09T10:16:10.306 INFO:teuthology.orchestra.run.smithi125.stdout:NAME="CentOS Stream" 2023-10-09T10:16:10.306 INFO:teuthology.orchestra.run.smithi125.stdout:VERSION="8" 2023-10-09T10:16:10.306 INFO:teuthology.orchestra.run.smithi125.stdout:ID="centos" 2023-10-09T10:16:10.307 INFO:teuthology.orchestra.run.smithi125.stdout:ID_LIKE="rhel fedora" 2023-10-09T10:16:10.307 INFO:teuthology.orchestra.run.smithi125.stdout:VERSION_ID="8" 2023-10-09T10:16:10.307 INFO:teuthology.orchestra.run.smithi125.stdout:PLATFORM_ID="platform:el8" 2023-10-09T10:16:10.307 INFO:teuthology.orchestra.run.smithi125.stdout:PRETTY_NAME="CentOS Stream 8" 2023-10-09T10:16:10.307 INFO:teuthology.orchestra.run.smithi125.stdout:ANSI_COLOR="0;31" 2023-10-09T10:16:10.307 INFO:teuthology.orchestra.run.smithi125.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-10-09T10:16:10.307 INFO:teuthology.orchestra.run.smithi125.stdout:HOME_URL="https://centos.org/" 2023-10-09T10:16:10.307 INFO:teuthology.orchestra.run.smithi125.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-09T10:16:10.307 INFO:teuthology.orchestra.run.smithi125.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-09T10:16:10.308 INFO:teuthology.orchestra.run.smithi125.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-10-09T10:16:10.308 INFO:teuthology.lock.ops:Updating smithi125.front.sepia.ceph.com on lock server 2023-10-09T10:16:10.331 DEBUG:teuthology.orchestra.run.smithi155:> uname -m 2023-10-09T10:16:10.353 INFO:teuthology.orchestra.run.smithi155.stdout:x86_64 2023-10-09T10:16:10.353 DEBUG:teuthology.orchestra.run.smithi155:> cat /etc/os-release 2023-10-09T10:16:10.414 INFO:teuthology.orchestra.run.smithi155.stdout:NAME="CentOS Stream" 2023-10-09T10:16:10.414 INFO:teuthology.orchestra.run.smithi155.stdout:VERSION="8" 2023-10-09T10:16:10.414 INFO:teuthology.orchestra.run.smithi155.stdout:ID="centos" 2023-10-09T10:16:10.414 INFO:teuthology.orchestra.run.smithi155.stdout:ID_LIKE="rhel fedora" 2023-10-09T10:16:10.414 INFO:teuthology.orchestra.run.smithi155.stdout:VERSION_ID="8" 2023-10-09T10:16:10.414 INFO:teuthology.orchestra.run.smithi155.stdout:PLATFORM_ID="platform:el8" 2023-10-09T10:16:10.414 INFO:teuthology.orchestra.run.smithi155.stdout:PRETTY_NAME="CentOS Stream 8" 2023-10-09T10:16:10.414 INFO:teuthology.orchestra.run.smithi155.stdout:ANSI_COLOR="0;31" 2023-10-09T10:16:10.415 INFO:teuthology.orchestra.run.smithi155.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-10-09T10:16:10.415 INFO:teuthology.orchestra.run.smithi155.stdout:HOME_URL="https://centos.org/" 2023-10-09T10:16:10.415 INFO:teuthology.orchestra.run.smithi155.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-09T10:16:10.415 INFO:teuthology.orchestra.run.smithi155.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-09T10:16:10.415 INFO:teuthology.orchestra.run.smithi155.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-10-09T10:16:10.415 INFO:teuthology.lock.ops:Updating smithi155.front.sepia.ceph.com on lock server 2023-10-09T10:16:10.439 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-09T10:16:10.448 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-09T10:16:10.474 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-09T10:16:10.474 DEBUG:teuthology.orchestra.run.smithi125:> test '!' -e /home/ubuntu/cephtest 2023-10-09T10:16:10.477 DEBUG:teuthology.orchestra.run.smithi155:> test '!' -e /home/ubuntu/cephtest 2023-10-09T10:16:10.496 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-09T10:16:10.502 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-09T10:16:10.502 DEBUG:teuthology.orchestra.run.smithi125:> test -z $(ls -A /var/lib/ceph) 2023-10-09T10:16:10.538 DEBUG:teuthology.orchestra.run.smithi155:> test -z $(ls -A /var/lib/ceph) 2023-10-09T10:16:10.566 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-09T10:16:10.623 INFO:teuthology.run_tasks:Running task kernel... 2023-10-09T10:16:10.637 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-09T10:16:10.637 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-10-09T10:16:10.638 DEBUG:teuthology.orchestra.run.smithi125:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T10:16:10.638 DEBUG:teuthology.orchestra.run.smithi155:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T10:16:10.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:16:10.657 DEBUG:teuthology.orchestra.run.smithi155:> uname -r 2023-10-09T10:16:10.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:16:10.659 DEBUG:teuthology.orchestra.run.smithi125:> uname -r 2023-10-09T10:16:10.717 INFO:teuthology.orchestra.run.smithi155.stdout:4.18.0-489.el8.x86_64 2023-10-09T10:16:10.717 INFO:teuthology.task.kernel:Running kernel on smithi155: 4.18.0-489.el8.x86_64 2023-10-09T10:16:10.718 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum install -y kernel 2023-10-09T10:16:10.720 INFO:teuthology.orchestra.run.smithi125.stdout:4.18.0-489.el8.x86_64 2023-10-09T10:16:10.720 INFO:teuthology.task.kernel:Running kernel on smithi125: 4.18.0-489.el8.x86_64 2023-10-09T10:16:10.721 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum install -y kernel 2023-10-09T10:16:15.292 INFO:teuthology.orchestra.run.smithi155.stdout:CentOS Stream 8 - AppStream 16 MB/s | 33 MB 00:02 2023-10-09T10:16:15.575 INFO:teuthology.orchestra.run.smithi125.stdout:CentOS Stream 8 - AppStream 15 MB/s | 33 MB 00:02 2023-10-09T10:16:22.693 INFO:teuthology.orchestra.run.smithi155.stdout:CentOS Stream 8 - BaseOS 36 MB/s | 48 MB 00:01 2023-10-09T10:16:23.408 INFO:teuthology.orchestra.run.smithi125.stdout:CentOS Stream 8 - BaseOS 26 MB/s | 48 MB 00:01 2023-10-09T10:16:30.042 INFO:teuthology.orchestra.run.smithi155.stdout:CentOS Stream 8 - Extras 34 kB/s | 18 kB 00:00 2023-10-09T10:16:30.376 INFO:teuthology.orchestra.run.smithi155.stdout:CentOS Stream 8 - Extras common packages 46 kB/s | 6.9 kB 00:00 2023-10-09T10:16:30.641 INFO:teuthology.orchestra.run.smithi125.stdout:CentOS Stream 8 - Extras 47 kB/s | 18 kB 00:00 2023-10-09T10:16:30.677 INFO:teuthology.orchestra.run.smithi155.stdout:Copr repo for python3-asyncssh owned by ceph 24 kB/s | 3.5 kB 00:00 2023-10-09T10:16:30.967 INFO:teuthology.orchestra.run.smithi125.stdout:CentOS Stream 8 - Extras common packages 47 kB/s | 6.9 kB 00:00 2023-10-09T10:16:31.260 INFO:teuthology.orchestra.run.smithi125.stdout:Copr repo for python3-asyncssh owned by ceph 24 kB/s | 3.5 kB 00:00 2023-10-09T10:16:31.953 INFO:teuthology.orchestra.run.smithi125.stdout:Extra Packages for Enterprise Linux 29 MB/s | 16 MB 00:00 2023-10-09T10:16:32.256 INFO:teuthology.orchestra.run.smithi155.stdout:Extra Packages for Enterprise Linux 11 MB/s | 16 MB 00:01 2023-10-09T10:16:35.702 INFO:teuthology.orchestra.run.smithi125.stdout:lab-extras 551 kB/s | 24 kB 00:00 2023-10-09T10:16:36.095 INFO:teuthology.orchestra.run.smithi155.stdout:lab-extras 398 kB/s | 24 kB 00:00 2023-10-09T10:16:39.110 INFO:teuthology.orchestra.run.smithi125.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-10-09T10:16:39.110 INFO:teuthology.orchestra.run.smithi125.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-10-09T10:16:39.347 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:16:39.348 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:16:39.348 INFO:teuthology.orchestra.run.smithi125.stdout: Package Architecture Version Repository Size 2023-10-09T10:16:39.348 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:16:39.348 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:16:39.349 INFO:teuthology.orchestra.run.smithi125.stdout: kernel x86_64 4.18.0-514.el8 baseos 10 M 2023-10-09T10:16:39.349 INFO:teuthology.orchestra.run.smithi125.stdout:Installing dependencies: 2023-10-09T10:16:39.349 INFO:teuthology.orchestra.run.smithi125.stdout: kernel-core x86_64 4.18.0-514.el8 baseos 43 M 2023-10-09T10:16:39.349 INFO:teuthology.orchestra.run.smithi125.stdout: kernel-modules x86_64 4.18.0-514.el8 baseos 35 M 2023-10-09T10:16:39.349 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:16:39.349 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:16:39.349 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:16:39.349 INFO:teuthology.orchestra.run.smithi125.stdout:Install 3 Packages 2023-10-09T10:16:39.349 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:16:39.350 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 88 M 2023-10-09T10:16:39.350 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 96 M 2023-10-09T10:16:39.350 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:16:39.610 INFO:teuthology.orchestra.run.smithi155.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-10-09T10:16:39.611 INFO:teuthology.orchestra.run.smithi155.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-10-09T10:16:39.874 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:16:39.875 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:16:39.875 INFO:teuthology.orchestra.run.smithi155.stdout: Package Architecture Version Repository Size 2023-10-09T10:16:39.875 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:16:39.875 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:16:39.875 INFO:teuthology.orchestra.run.smithi155.stdout: kernel x86_64 4.18.0-514.el8 baseos 10 M 2023-10-09T10:16:39.875 INFO:teuthology.orchestra.run.smithi155.stdout:Installing dependencies: 2023-10-09T10:16:39.875 INFO:teuthology.orchestra.run.smithi155.stdout: kernel-core x86_64 4.18.0-514.el8 baseos 43 M 2023-10-09T10:16:39.875 INFO:teuthology.orchestra.run.smithi155.stdout: kernel-modules x86_64 4.18.0-514.el8 baseos 35 M 2023-10-09T10:16:39.876 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:16:39.876 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:16:39.876 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:16:39.876 INFO:teuthology.orchestra.run.smithi155.stdout:Install 3 Packages 2023-10-09T10:16:39.876 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:16:39.876 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 88 M 2023-10-09T10:16:39.877 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 96 M 2023-10-09T10:16:39.877 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:16:39.878 INFO:teuthology.orchestra.run.smithi125.stdout:(1/3): kernel-4.18.0-514.el8.x86_64.rpm 21 MB/s | 10 MB 00:00 2023-10-09T10:16:40.629 INFO:teuthology.orchestra.run.smithi125.stdout:(2/3): kernel-modules-4.18.0-514.el8.x86_64.rpm 29 MB/s | 35 MB 00:01 2023-10-09T10:16:41.062 INFO:teuthology.orchestra.run.smithi125.stdout:(3/3): kernel-core-4.18.0-514.el8.x86_64.rpm 25 MB/s | 43 MB 00:01 2023-10-09T10:16:41.063 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:16:41.063 INFO:teuthology.orchestra.run.smithi125.stdout:Total 51 MB/s | 88 MB 00:01 2023-10-09T10:16:41.096 INFO:teuthology.orchestra.run.smithi155.stdout:(1/3): kernel-4.18.0-514.el8.x86_64.rpm 10 MB/s | 10 MB 00:00 2023-10-09T10:16:41.728 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:16:41.879 INFO:teuthology.orchestra.run.smithi155.stdout:(2/3): kernel-modules-4.18.0-514.el8.x86_64.rpm 20 MB/s | 35 MB 00:01 2023-10-09T10:16:42.144 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:16:42.144 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:16:42.305 INFO:teuthology.orchestra.run.smithi155.stdout:(3/3): kernel-core-4.18.0-514.el8.x86_64.rpm 19 MB/s | 43 MB 00:02 2023-10-09T10:16:42.305 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:16:42.305 INFO:teuthology.orchestra.run.smithi155.stdout:Total 36 MB/s | 88 MB 00:02 2023-10-09T10:16:42.977 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:16:43.360 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:16:43.387 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:16:43.442 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:16:43.442 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:16:44.611 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:16:44.648 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:16:46.153 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:16:46.721 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : kernel-core-4.18.0-514.el8.x86_64 1/3 2023-10-09T10:16:47.427 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:16:47.978 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : kernel-core-4.18.0-514.el8.x86_64 1/3 2023-10-09T10:16:50.258 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: kernel-core-4.18.0-514.el8.x86_64 1/3 2023-10-09T10:16:50.679 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : kernel-modules-4.18.0-514.el8.x86_64 2/3 2023-10-09T10:16:51.586 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: kernel-core-4.18.0-514.el8.x86_64 1/3 2023-10-09T10:16:51.977 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : kernel-modules-4.18.0-514.el8.x86_64 2/3 2023-10-09T10:16:54.058 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: kernel-modules-4.18.0-514.el8.x86_64 2/3 2023-10-09T10:16:54.409 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : kernel-4.18.0-514.el8.x86_64 3/3 2023-10-09T10:16:55.359 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: kernel-modules-4.18.0-514.el8.x86_64 2/3 2023-10-09T10:16:55.673 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : kernel-4.18.0-514.el8.x86_64 3/3 2023-10-09T10:17:16.022 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: kernel-core-4.18.0-514.el8.x86_64 3/3 2023-10-09T10:17:16.045 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: kernel-modules-4.18.0-514.el8.x86_64 3/3 2023-10-09T10:17:16.387 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: kernel-4.18.0-514.el8.x86_64 3/3 2023-10-09T10:17:16.387 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : kernel-4.18.0-514.el8.x86_64 1/3 2023-10-09T10:17:16.388 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : kernel-core-4.18.0-514.el8.x86_64 2/3 2023-10-09T10:17:17.136 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : kernel-modules-4.18.0-514.el8.x86_64 3/3 2023-10-09T10:17:17.137 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:17:17.137 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:17:17.137 INFO:teuthology.orchestra.run.smithi125.stdout: kernel-4.18.0-514.el8.x86_64 kernel-core-4.18.0-514.el8.x86_64 2023-10-09T10:17:17.137 INFO:teuthology.orchestra.run.smithi125.stdout: kernel-modules-4.18.0-514.el8.x86_64 2023-10-09T10:17:17.137 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:17:17.137 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:17:17.275 DEBUG:teuthology.orchestra.run.smithi125:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T10:17:17.525 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: kernel-core-4.18.0-514.el8.x86_64 3/3 2023-10-09T10:17:17.533 INFO:teuthology.orchestra.run.smithi125.stdout:kernel-4.18.0-514.el8.x86_64 2023-10-09T10:17:17.533 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-514.el8.x86_64 2023-10-09T10:17:17.533 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-514.el8.x86_64 2023-10-09T10:17:17.533 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-09T10:17:17.533 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-09T10:17:17.533 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-10-09T10:17:17.533 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi125.front.sepia.ceph.com, path=None, version=distro) 2023-10-09T10:17:17.534 DEBUG:teuthology.orchestra.run.smithi125:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T10:17:17.550 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: kernel-modules-4.18.0-514.el8.x86_64 3/3 2023-10-09T10:17:17.742 INFO:teuthology.orchestra.run.smithi125.stdout:kernel-4.18.0-514.el8.x86_64 2023-10-09T10:17:17.742 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-514.el8.x86_64 2023-10-09T10:17:17.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo rpm -qi grub2-tools 2023-10-09T10:17:17.775 INFO:teuthology.orchestra.run.smithi125.stdout:Name : grub2-tools 2023-10-09T10:17:17.776 INFO:teuthology.orchestra.run.smithi125.stdout:Epoch : 1 2023-10-09T10:17:17.776 INFO:teuthology.orchestra.run.smithi125.stdout:Version : 2.02 2023-10-09T10:17:17.776 INFO:teuthology.orchestra.run.smithi125.stdout:Release : 129.el8 2023-10-09T10:17:17.776 INFO:teuthology.orchestra.run.smithi125.stdout:Architecture: x86_64 2023-10-09T10:17:17.776 INFO:teuthology.orchestra.run.smithi125.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-10-09T10:17:17.776 INFO:teuthology.orchestra.run.smithi125.stdout:Group : System Environment/Base 2023-10-09T10:17:17.776 INFO:teuthology.orchestra.run.smithi125.stdout:Size : 9265586 2023-10-09T10:17:17.776 INFO:teuthology.orchestra.run.smithi125.stdout:License : GPLv3+ 2023-10-09T10:17:17.776 INFO:teuthology.orchestra.run.smithi125.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-10-09T10:17:17.776 INFO:teuthology.orchestra.run.smithi125.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-10-09T10:17:17.776 INFO:teuthology.orchestra.run.smithi125.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-10-09T10:17:17.776 INFO:teuthology.orchestra.run.smithi125.stdout:Build Host : kbuilder.bsys.centos.org 2023-10-09T10:17:17.777 INFO:teuthology.orchestra.run.smithi125.stdout:Relocations : (not relocatable) 2023-10-09T10:17:17.777 INFO:teuthology.orchestra.run.smithi125.stdout:Packager : CentOS BuildSystem 2023-10-09T10:17:17.777 INFO:teuthology.orchestra.run.smithi125.stdout:Vendor : CentOS 2023-10-09T10:17:17.777 INFO:teuthology.orchestra.run.smithi125.stdout:URL : http://www.gnu.org/software/grub/ 2023-10-09T10:17:17.777 INFO:teuthology.orchestra.run.smithi125.stdout:Summary : Support tools for GRUB. 2023-10-09T10:17:17.777 INFO:teuthology.orchestra.run.smithi125.stdout:Description : 2023-10-09T10:17:17.777 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:17:17.777 INFO:teuthology.orchestra.run.smithi125.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-10-09T10:17:17.777 INFO:teuthology.orchestra.run.smithi125.stdout:customizable bootloader with modular architecture. It supports a rich 2023-10-09T10:17:17.777 INFO:teuthology.orchestra.run.smithi125.stdout:variety of kernel formats, file systems, computer architectures and 2023-10-09T10:17:17.777 INFO:teuthology.orchestra.run.smithi125.stdout:hardware devices. 2023-10-09T10:17:17.777 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:17:17.777 INFO:teuthology.orchestra.run.smithi125.stdout:This subpackage provides tools for support of all platforms. 2023-10-09T10:17:17.778 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-10-09T10:17:17.778 INFO:teuthology.task.kernel:Updating grub on smithi125 to boot 4.18.0-514.el8.x86_64 2023-10-09T10:17:17.779 DEBUG:teuthology.orchestra.run.smithi125:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-10-09T10:17:17.899 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: kernel-4.18.0-514.el8.x86_64 3/3 2023-10-09T10:17:17.899 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : kernel-4.18.0-514.el8.x86_64 1/3 2023-10-09T10:17:17.899 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : kernel-core-4.18.0-514.el8.x86_64 2/3 2023-10-09T10:17:18.609 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : kernel-modules-4.18.0-514.el8.x86_64 3/3 2023-10-09T10:17:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:17:18.610 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:17:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: kernel-4.18.0-514.el8.x86_64 kernel-core-4.18.0-514.el8.x86_64 2023-10-09T10:17:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: kernel-modules-4.18.0-514.el8.x86_64 2023-10-09T10:17:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:17:18.610 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:17:18.761 DEBUG:teuthology.orchestra.run.smithi155:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T10:17:18.978 INFO:teuthology.orchestra.run.smithi155.stdout:kernel-4.18.0-514.el8.x86_64 2023-10-09T10:17:18.979 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-514.el8.x86_64 2023-10-09T10:17:18.979 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-514.el8.x86_64 2023-10-09T10:17:18.979 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-09T10:17:18.979 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-09T10:17:18.979 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-10-09T10:17:18.979 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi155.front.sepia.ceph.com, path=None, version=distro) 2023-10-09T10:17:18.979 DEBUG:teuthology.orchestra.run.smithi155:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T10:17:19.188 INFO:teuthology.orchestra.run.smithi155.stdout:kernel-4.18.0-514.el8.x86_64 2023-10-09T10:17:19.188 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-514.el8.x86_64 2023-10-09T10:17:19.188 DEBUG:teuthology.orchestra.run.smithi155:> sudo rpm -qi grub2-tools 2023-10-09T10:17:19.222 INFO:teuthology.orchestra.run.smithi155.stdout:Name : grub2-tools 2023-10-09T10:17:19.222 INFO:teuthology.orchestra.run.smithi155.stdout:Epoch : 1 2023-10-09T10:17:19.222 INFO:teuthology.orchestra.run.smithi155.stdout:Version : 2.02 2023-10-09T10:17:19.223 INFO:teuthology.orchestra.run.smithi155.stdout:Release : 129.el8 2023-10-09T10:17:19.223 INFO:teuthology.orchestra.run.smithi155.stdout:Architecture: x86_64 2023-10-09T10:17:19.223 INFO:teuthology.orchestra.run.smithi155.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-10-09T10:17:19.223 INFO:teuthology.orchestra.run.smithi155.stdout:Group : System Environment/Base 2023-10-09T10:17:19.223 INFO:teuthology.orchestra.run.smithi155.stdout:Size : 9265586 2023-10-09T10:17:19.223 INFO:teuthology.orchestra.run.smithi155.stdout:License : GPLv3+ 2023-10-09T10:17:19.223 INFO:teuthology.orchestra.run.smithi155.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-10-09T10:17:19.223 INFO:teuthology.orchestra.run.smithi155.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-10-09T10:17:19.224 INFO:teuthology.orchestra.run.smithi155.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-10-09T10:17:19.224 INFO:teuthology.orchestra.run.smithi155.stdout:Build Host : kbuilder.bsys.centos.org 2023-10-09T10:17:19.224 INFO:teuthology.orchestra.run.smithi155.stdout:Relocations : (not relocatable) 2023-10-09T10:17:19.224 INFO:teuthology.orchestra.run.smithi155.stdout:Packager : CentOS BuildSystem 2023-10-09T10:17:19.224 INFO:teuthology.orchestra.run.smithi155.stdout:Vendor : CentOS 2023-10-09T10:17:19.224 INFO:teuthology.orchestra.run.smithi155.stdout:URL : http://www.gnu.org/software/grub/ 2023-10-09T10:17:19.224 INFO:teuthology.orchestra.run.smithi155.stdout:Summary : Support tools for GRUB. 2023-10-09T10:17:19.224 INFO:teuthology.orchestra.run.smithi155.stdout:Description : 2023-10-09T10:17:19.224 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:17:19.224 INFO:teuthology.orchestra.run.smithi155.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-10-09T10:17:19.224 INFO:teuthology.orchestra.run.smithi155.stdout:customizable bootloader with modular architecture. It supports a rich 2023-10-09T10:17:19.224 INFO:teuthology.orchestra.run.smithi155.stdout:variety of kernel formats, file systems, computer architectures and 2023-10-09T10:17:19.224 INFO:teuthology.orchestra.run.smithi155.stdout:hardware devices. 2023-10-09T10:17:19.225 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:17:19.225 INFO:teuthology.orchestra.run.smithi155.stdout:This subpackage provides tools for support of all platforms. 2023-10-09T10:17:19.225 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-10-09T10:17:19.226 INFO:teuthology.task.kernel:Updating grub on smithi155 to boot 4.18.0-514.el8.x86_64 2023-10-09T10:17:19.226 DEBUG:teuthology.orchestra.run.smithi155:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-10-09T10:17:19.526 INFO:teuthology.orchestra.run.smithi125.stderr:Generating grub configuration file ... 2023-10-09T10:17:20.772 INFO:teuthology.orchestra.run.smithi155.stderr:Generating grub configuration file ... 2023-10-09T10:17:21.766 INFO:teuthology.orchestra.run.smithi125.stderr:done 2023-10-09T10:17:21.767 DEBUG:teuthology.orchestra.run.smithi125:> mktemp 2023-10-09T10:17:21.800 INFO:teuthology.orchestra.run.smithi125.stdout:/tmp/tmp.9N7KK8TXp7 2023-10-09T10:17:21.800 DEBUG:teuthology.orchestra.run.smithi125:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.9N7KK8TXp7 2023-10-09T10:17:21.891 DEBUG:teuthology.orchestra.run.smithi125:> sudo chmod 0666 /tmp/tmp.9N7KK8TXp7 2023-10-09T10:17:22.011 DEBUG:teuthology.orchestra.remote:smithi125:/tmp/tmp.9N7KK8TXp7 is 6KB 2023-10-09T10:17:22.059 DEBUG:teuthology.orchestra.run.smithi125:> rm -fr /tmp/tmp.9N7KK8TXp7 2023-10-09T10:17:22.076 DEBUG:teuthology.orchestra.run.smithi125:> sudo /bin/ls /boot/loader/entries || true 2023-10-09T10:17:22.152 INFO:teuthology.orchestra.run.smithi125.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-10-09T10:17:22.152 INFO:teuthology.orchestra.run.smithi125.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-10-09T10:17:22.152 INFO:teuthology.orchestra.run.smithi125.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-10-09T10:17:22.152 INFO:teuthology.orchestra.run.smithi125.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-514.el8.x86_64.conf 2023-10-09T10:17:22.153 DEBUG:teuthology.orchestra.run.smithi125:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-514.el8.x86_64 2023-10-09T10:17:22.341 DEBUG:teuthology.orchestra.run.smithi125:> sudo shutdown -r now 2023-10-09T10:17:22.995 INFO:teuthology.orchestra.run.smithi155.stderr:done 2023-10-09T10:17:22.996 DEBUG:teuthology.orchestra.run.smithi155:> mktemp 2023-10-09T10:17:23.022 INFO:teuthology.orchestra.run.smithi155.stdout:/tmp/tmp.vjbtOwTyAi 2023-10-09T10:17:23.022 DEBUG:teuthology.orchestra.run.smithi155:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.vjbtOwTyAi 2023-10-09T10:17:23.104 DEBUG:teuthology.orchestra.run.smithi155:> sudo chmod 0666 /tmp/tmp.vjbtOwTyAi 2023-10-09T10:17:23.271 DEBUG:teuthology.orchestra.remote:smithi155:/tmp/tmp.vjbtOwTyAi is 6KB 2023-10-09T10:17:23.319 DEBUG:teuthology.orchestra.run.smithi155:> rm -fr /tmp/tmp.vjbtOwTyAi 2023-10-09T10:17:23.336 DEBUG:teuthology.orchestra.run.smithi155:> sudo /bin/ls /boot/loader/entries || true 2023-10-09T10:17:23.407 INFO:teuthology.orchestra.run.smithi155.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-10-09T10:17:23.407 INFO:teuthology.orchestra.run.smithi155.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-10-09T10:17:23.407 INFO:teuthology.orchestra.run.smithi155.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-10-09T10:17:23.407 INFO:teuthology.orchestra.run.smithi155.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-514.el8.x86_64.conf 2023-10-09T10:17:23.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-514.el8.x86_64 2023-10-09T10:17:23.568 DEBUG:teuthology.orchestra.run.smithi155:> sudo shutdown -r now 2023-10-09T10:17:52.370 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-10-09T10:17:52.371 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi125.front.sepia.ceph.com' 2023-10-09T10:17:52.371 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi125.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T10:17:53.572 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-10-09T10:17:53.572 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi155.front.sepia.ceph.com' 2023-10-09T10:17:53.573 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi155.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T10:18:27.066 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.125 2023-10-09T10:18:28.090 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.155 2023-10-09T10:18:36.074 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi125.front.sepia.ceph.com' 2023-10-09T10:18:36.075 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi125.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T10:18:37.092 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi155.front.sepia.ceph.com' 2023-10-09T10:18:37.293 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi155.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T10:18:54.458 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.125 2023-10-09T10:18:55.738 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.155 2023-10-09T10:19:06.467 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi125.front.sepia.ceph.com' 2023-10-09T10:19:06.468 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi125.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T10:19:06.859 DEBUG:teuthology.orchestra.run.smithi125:> true 2023-10-09T10:19:07.164 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi125.front.sepia.ceph.com' 2023-10-09T10:19:07.164 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-514.el8.x86_64"... 2023-10-09T10:19:07.164 DEBUG:teuthology.orchestra.run.smithi125:> uname -r 2023-10-09T10:19:07.181 INFO:teuthology.orchestra.run.smithi125.stdout:4.18.0-514.el8.x86_64 2023-10-09T10:19:07.181 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-514.el8.x86_64 vs 4.18.0-514.el8.x86_64 2023-10-09T10:19:07.181 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-09T10:19:07.181 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-10-09T10:19:07.739 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi155.front.sepia.ceph.com' 2023-10-09T10:19:07.740 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi155.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T10:19:08.182 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-10-09T10:19:08.182 DEBUG:teuthology.orchestra.run.smithi125:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T10:19:08.267 INFO:teuthology.orchestra.run.smithi125.stdout:ttyS1 2023-10-09T10:19:08.279 DEBUG:teuthology.parallel:result is None 2023-10-09T10:19:09.185 DEBUG:teuthology.orchestra.run.smithi155:> true 2023-10-09T10:19:09.228 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi155.front.sepia.ceph.com' 2023-10-09T10:19:09.228 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-514.el8.x86_64"... 2023-10-09T10:19:09.228 DEBUG:teuthology.orchestra.run.smithi155:> uname -r 2023-10-09T10:19:09.292 INFO:teuthology.orchestra.run.smithi155.stdout:4.18.0-514.el8.x86_64 2023-10-09T10:19:09.292 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-514.el8.x86_64 vs 4.18.0-514.el8.x86_64 2023-10-09T10:19:09.292 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-09T10:19:09.292 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-10-09T10:19:10.293 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-10-09T10:19:10.293 DEBUG:teuthology.orchestra.run.smithi155:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T10:19:10.351 INFO:teuthology.orchestra.run.smithi155.stdout:ttyS1 2023-10-09T10:19:10.366 DEBUG:teuthology.parallel:result is None 2023-10-09T10:19:10.366 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-09T10:19:10.373 INFO:teuthology.task.internal:Creating test directory... 2023-10-09T10:19:10.373 DEBUG:teuthology.orchestra.run.smithi125:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T10:19:10.377 DEBUG:teuthology.orchestra.run.smithi155:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T10:19:10.397 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-09T10:19:10.404 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-09T10:19:10.440 INFO:teuthology.task.internal:Creating archive directory... 2023-10-09T10:19:10.440 DEBUG:teuthology.orchestra.run.smithi125:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T10:19:10.444 DEBUG:teuthology.orchestra.run.smithi155:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T10:19:10.490 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-09T10:19:10.497 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-09T10:19:10.498 DEBUG:teuthology.orchestra.run.smithi125:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T10:19:10.523 DEBUG:teuthology.orchestra.run.smithi155:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T10:19:10.554 INFO:teuthology.orchestra.run.smithi155.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T10:19:10.564 INFO:teuthology.orchestra.run.smithi155.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T10:19:10.794 INFO:teuthology.orchestra.run.smithi125.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T10:19:10.862 INFO:teuthology.orchestra.run.smithi125.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T10:19:10.873 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-09T10:19:10.881 INFO:teuthology.task.internal:Configuring sudo... 2023-10-09T10:19:10.881 DEBUG:teuthology.orchestra.run.smithi125:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T10:19:10.924 DEBUG:teuthology.orchestra.run.smithi155:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T10:19:11.453 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-09T10:19:11.462 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-09T10:19:11.462 DEBUG:teuthology.orchestra.run.smithi125:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T10:19:11.526 DEBUG:teuthology.orchestra.run.smithi155:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T10:19:11.719 DEBUG:teuthology.orchestra.run.smithi125:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T10:19:12.089 DEBUG:teuthology.orchestra.run.smithi125:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T10:19:12.126 DEBUG:teuthology.orchestra.run.smithi125:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T10:19:12.211 DEBUG:teuthology.orchestra.run.smithi125:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T10:19:12.255 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:19:12.255 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T10:19:12.322 DEBUG:teuthology.orchestra.run.smithi155:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T10:19:12.368 DEBUG:teuthology.orchestra.run.smithi155:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T10:19:12.420 DEBUG:teuthology.orchestra.run.smithi155:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T10:19:12.511 DEBUG:teuthology.orchestra.run.smithi155:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T10:19:12.556 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T10:19:12.556 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T10:19:12.629 DEBUG:teuthology.orchestra.run.smithi125:> sudo service rsyslog restart 2023-10-09T10:19:12.631 DEBUG:teuthology.orchestra.run.smithi155:> sudo service rsyslog restart 2023-10-09T10:19:12.752 INFO:teuthology.orchestra.run.smithi155.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T10:19:12.756 INFO:teuthology.orchestra.run.smithi125.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T10:19:13.263 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-09T10:19:13.271 INFO:teuthology.task.internal:Starting timer... 2023-10-09T10:19:13.271 INFO:teuthology.run_tasks:Running task pcp... 2023-10-09T10:19:13.293 INFO:teuthology.run_tasks:Running task selinux... 2023-10-09T10:19:13.325 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-10-09T10:19:13.326 DEBUG:teuthology.orchestra.run.smithi125:> sudo service auditd rotate 2023-10-09T10:19:13.406 INFO:teuthology.orchestra.run.smithi125.stdout:Rotating logs: 2023-10-09T10:19:13.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo service auditd rotate 2023-10-09T10:19:14.203 INFO:teuthology.orchestra.run.smithi155.stdout:Rotating logs: 2023-10-09T10:19:14.226 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-09T10:19:14.227 DEBUG:teuthology.orchestra.run.smithi125:> /usr/sbin/getenforce 2023-10-09T10:19:14.255 INFO:teuthology.orchestra.run.smithi125.stdout:Permissive 2023-10-09T10:19:14.255 DEBUG:teuthology.orchestra.run.smithi155:> /usr/sbin/getenforce 2023-10-09T10:19:14.437 INFO:teuthology.orchestra.run.smithi155.stdout:Permissive 2023-10-09T10:19:14.437 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi125.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi155.front.sepia.ceph.com': 'permissive'} 2023-10-09T10:19:14.438 DEBUG:teuthology.orchestra.run.smithi125:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T10:19:14.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:19:14.465 DEBUG:teuthology.orchestra.run.smithi155:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T10:19:14.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:19:14.732 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-09T10:19:14.733 DEBUG:teuthology.orchestra.run.smithi125:> sudo /usr/sbin/setenforce permissive 2023-10-09T10:19:14.781 DEBUG:teuthology.orchestra.run.smithi155:> sudo /usr/sbin/setenforce permissive 2023-10-09T10:19:14.809 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-09T10:19:14.819 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-10-09T10:19:14.852 INFO:teuthology.repo_utils:Fetching main from origin 2023-10-09T10:19:14.960 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-09T10:19:14.977 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-09T10:19:14.978 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi125.front.sepia.ceph.com,smithi155.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-09T10:26:36.924 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi125.front.sepia.ceph.com'), Remote(name='ubuntu@smithi155.front.sepia.ceph.com')] 2023-10-09T10:26:36.925 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi125.front.sepia.ceph.com' 2023-10-09T10:26:36.926 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi125.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T10:26:36.996 DEBUG:teuthology.orchestra.run.smithi125:> true 2023-10-09T10:26:37.068 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi125.front.sepia.ceph.com' 2023-10-09T10:26:37.069 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi155.front.sepia.ceph.com' 2023-10-09T10:26:37.069 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi155.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T10:26:37.137 DEBUG:teuthology.orchestra.run.smithi155:> true 2023-10-09T10:26:37.208 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi155.front.sepia.ceph.com' 2023-10-09T10:26:37.208 INFO:teuthology.run_tasks:Running task clock... 2023-10-09T10:26:37.218 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-09T10:26:37.219 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T10:26:37.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T10:26:37.221 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T10:26:37.221 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T10:26:37.252 INFO:teuthology.orchestra.run.smithi125.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-09T10:26:37.267 INFO:teuthology.orchestra.run.smithi125.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-09T10:26:37.279 INFO:teuthology.orchestra.run.smithi155.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-09T10:26:37.294 INFO:teuthology.orchestra.run.smithi155.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-09T10:26:37.336 INFO:teuthology.orchestra.run.smithi125.stderr:sudo: ntpd: command not found 2023-10-09T10:26:37.348 INFO:teuthology.orchestra.run.smithi125.stdout:506 Cannot talk to daemon 2023-10-09T10:26:37.362 INFO:teuthology.orchestra.run.smithi155.stderr:sudo: ntpd: command not found 2023-10-09T10:26:37.362 INFO:teuthology.orchestra.run.smithi125.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-09T10:26:37.373 INFO:teuthology.orchestra.run.smithi155.stdout:506 Cannot talk to daemon 2023-10-09T10:26:37.375 INFO:teuthology.orchestra.run.smithi125.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-09T10:26:37.387 INFO:teuthology.orchestra.run.smithi155.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-09T10:26:37.401 INFO:teuthology.orchestra.run.smithi155.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-09T10:26:37.435 INFO:teuthology.orchestra.run.smithi125.stderr:bash: ntpq: command not found 2023-10-09T10:26:37.439 INFO:teuthology.orchestra.run.smithi125.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T10:26:37.439 INFO:teuthology.orchestra.run.smithi125.stdout:=============================================================================== 2023-10-09T10:26:37.439 INFO:teuthology.orchestra.run.smithi125.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T10:26:37.439 INFO:teuthology.orchestra.run.smithi125.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T10:26:37.439 INFO:teuthology.orchestra.run.smithi125.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T10:26:37.440 INFO:teuthology.orchestra.run.smithi125.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T10:26:37.464 INFO:teuthology.orchestra.run.smithi155.stderr:bash: ntpq: command not found 2023-10-09T10:26:37.468 INFO:teuthology.orchestra.run.smithi155.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T10:26:37.468 INFO:teuthology.orchestra.run.smithi155.stdout:=============================================================================== 2023-10-09T10:26:37.468 INFO:teuthology.orchestra.run.smithi155.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T10:26:37.469 INFO:teuthology.orchestra.run.smithi155.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T10:26:37.469 INFO:teuthology.orchestra.run.smithi155.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T10:26:37.469 INFO:teuthology.orchestra.run.smithi155.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T10:26:37.470 INFO:teuthology.run_tasks:Running task pexec... 2023-10-09T10:26:37.481 INFO:teuthology.task.pexec:Executing custom commands... 2023-10-09T10:26:37.481 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi125.front.sepia.ceph.com 2023-10-09T10:26:37.481 DEBUG:teuthology.orchestra.run.smithi125:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-09T10:26:37.482 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi155.front.sepia.ceph.com 2023-10-09T10:26:37.482 DEBUG:teuthology.orchestra.run.smithi155:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-09T10:26:38.362 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:01:34 ago on Mon 09 Oct 2023 10:25:04 AM UTC. 2023-10-09T10:26:38.393 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:01:36 ago on Mon 09 Oct 2023 10:25:02 AM UTC. 2023-10-09T10:26:40.408 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:26:40.409 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:26:40.410 INFO:teuthology.orchestra.run.smithi125.stdout: Package Architecture Version Repository Size 2023-10-09T10:26:40.410 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:26:40.410 INFO:teuthology.orchestra.run.smithi125.stdout:Resetting modules: 2023-10-09T10:26:40.410 INFO:teuthology.orchestra.run.smithi125.stdout: container-tools 2023-10-09T10:26:40.410 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:26:40.410 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:26:40.410 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:26:40.410 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:26:40.433 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:26:40.434 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:26:40.435 INFO:teuthology.orchestra.run.smithi155.stdout: Package Architecture Version Repository Size 2023-10-09T10:26:40.435 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:26:40.435 INFO:teuthology.orchestra.run.smithi155.stdout:Resetting modules: 2023-10-09T10:26:40.435 INFO:teuthology.orchestra.run.smithi155.stdout: container-tools 2023-10-09T10:26:40.435 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:26:40.435 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:26:40.435 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:26:40.435 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:26:40.663 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:26:40.717 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:26:41.649 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:01:37 ago on Mon 09 Oct 2023 10:25:04 AM UTC. 2023-10-09T10:26:41.659 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:01:39 ago on Mon 09 Oct 2023 10:25:02 AM UTC. 2023-10-09T10:26:43.663 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:26:43.700 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:26:43.818 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:26:43.818 INFO:teuthology.orchestra.run.smithi155.stdout: Package Arch Version Repository Size 2023-10-09T10:26:43.818 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:26:43.818 INFO:teuthology.orchestra.run.smithi155.stdout:Installing group/module packages: 2023-10-09T10:26:43.818 INFO:teuthology.orchestra.run.smithi155.stdout: aardvark-dns x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 1.0 M 2023-10-09T10:26:43.818 INFO:teuthology.orchestra.run.smithi155.stdout: buildah x86_64 1:1.31.3-1.module_el8+664+4072b3ae appstream 8.8 M 2023-10-09T10:26:43.818 INFO:teuthology.orchestra.run.smithi155.stdout: cockpit-podman noarch 75-1.module_el8+664+4072b3ae appstream 739 k 2023-10-09T10:26:43.818 INFO:teuthology.orchestra.run.smithi155.stdout: crun x86_64 1.8.7-1.module_el8+661+d1afb926 appstream 239 k 2023-10-09T10:26:43.819 INFO:teuthology.orchestra.run.smithi155.stdout: netavark x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 3.7 M 2023-10-09T10:26:43.819 INFO:teuthology.orchestra.run.smithi155.stdout: python3-podman noarch 4.6.0-1.module_el8+582+a2558ae2 appstream 177 k 2023-10-09T10:26:43.819 INFO:teuthology.orchestra.run.smithi155.stdout: skopeo x86_64 2:1.13.3-1.module_el8+664+4072b3ae appstream 8.1 M 2023-10-09T10:26:43.819 INFO:teuthology.orchestra.run.smithi155.stdout: toolbox x86_64 0.0.99.4-5.module_el8+649+e2deac1d appstream 2.5 M 2023-10-09T10:26:43.819 INFO:teuthology.orchestra.run.smithi155.stdout: udica noarch 0.2.6-20.module_el8+487+8e42a277 appstream 53 k 2023-10-09T10:26:43.819 INFO:teuthology.orchestra.run.smithi155.stdout:Installing dependencies: 2023-10-09T10:26:43.819 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2023-10-09T10:26:43.819 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2023-10-09T10:26:43.819 INFO:teuthology.orchestra.run.smithi155.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2023-10-09T10:26:43.819 INFO:teuthology.orchestra.run.smithi155.stdout:Installing weak dependencies: 2023-10-09T10:26:43.819 INFO:teuthology.orchestra.run.smithi155.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2023-10-09T10:26:43.819 INFO:teuthology.orchestra.run.smithi155.stdout:Installing module profiles: 2023-10-09T10:26:43.820 INFO:teuthology.orchestra.run.smithi155.stdout: container-tools/common 2023-10-09T10:26:43.820 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:26:43.820 INFO:teuthology.orchestra.run.smithi155.stdout:Enabling module streams: 2023-10-09T10:26:43.820 INFO:teuthology.orchestra.run.smithi155.stdout: container-tools rhel8 2023-10-09T10:26:43.820 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:26:43.820 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:26:43.820 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:26:43.820 INFO:teuthology.orchestra.run.smithi155.stdout:Install 13 Packages 2023-10-09T10:26:43.820 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:26:43.821 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 26 M 2023-10-09T10:26:43.821 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 87 M 2023-10-09T10:26:43.821 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:26:43.861 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:26:43.861 INFO:teuthology.orchestra.run.smithi125.stdout: Package Arch Version Repository Size 2023-10-09T10:26:43.861 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:26:43.861 INFO:teuthology.orchestra.run.smithi125.stdout:Installing group/module packages: 2023-10-09T10:26:43.861 INFO:teuthology.orchestra.run.smithi125.stdout: aardvark-dns x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 1.0 M 2023-10-09T10:26:43.861 INFO:teuthology.orchestra.run.smithi125.stdout: buildah x86_64 1:1.31.3-1.module_el8+664+4072b3ae appstream 8.8 M 2023-10-09T10:26:43.861 INFO:teuthology.orchestra.run.smithi125.stdout: cockpit-podman noarch 75-1.module_el8+664+4072b3ae appstream 739 k 2023-10-09T10:26:43.861 INFO:teuthology.orchestra.run.smithi125.stdout: crun x86_64 1.8.7-1.module_el8+661+d1afb926 appstream 239 k 2023-10-09T10:26:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: netavark x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 3.7 M 2023-10-09T10:26:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: python3-podman noarch 4.6.0-1.module_el8+582+a2558ae2 appstream 177 k 2023-10-09T10:26:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: skopeo x86_64 2:1.13.3-1.module_el8+664+4072b3ae appstream 8.1 M 2023-10-09T10:26:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: toolbox x86_64 0.0.99.4-5.module_el8+649+e2deac1d appstream 2.5 M 2023-10-09T10:26:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: udica noarch 0.2.6-20.module_el8+487+8e42a277 appstream 53 k 2023-10-09T10:26:43.862 INFO:teuthology.orchestra.run.smithi125.stdout:Installing dependencies: 2023-10-09T10:26:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2023-10-09T10:26:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2023-10-09T10:26:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2023-10-09T10:26:43.862 INFO:teuthology.orchestra.run.smithi125.stdout:Installing weak dependencies: 2023-10-09T10:26:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2023-10-09T10:26:43.862 INFO:teuthology.orchestra.run.smithi125.stdout:Installing module profiles: 2023-10-09T10:26:43.863 INFO:teuthology.orchestra.run.smithi125.stdout: container-tools/common 2023-10-09T10:26:43.863 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:26:43.863 INFO:teuthology.orchestra.run.smithi125.stdout:Enabling module streams: 2023-10-09T10:26:43.863 INFO:teuthology.orchestra.run.smithi125.stdout: container-tools rhel8 2023-10-09T10:26:43.863 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:26:43.863 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:26:43.863 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:26:43.863 INFO:teuthology.orchestra.run.smithi125.stdout:Install 13 Packages 2023-10-09T10:26:43.863 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:26:43.864 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 26 M 2023-10-09T10:26:43.864 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 87 M 2023-10-09T10:26:43.864 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:26:44.680 INFO:teuthology.orchestra.run.smithi155.stdout:(1/13): cockpit-podman-75-1.module_el8+664+4072 1.1 MB/s | 739 kB 00:00 2023-10-09T10:26:44.722 INFO:teuthology.orchestra.run.smithi155.stdout:(2/13): aardvark-dns-1.7.0-1.module_el8+487+8e4 1.4 MB/s | 1.0 MB 00:00 2023-10-09T10:26:44.726 INFO:teuthology.orchestra.run.smithi125.stdout:(1/13): cockpit-podman-75-1.module_el8+664+4072 1.1 MB/s | 739 kB 00:00 2023-10-09T10:26:44.760 INFO:teuthology.orchestra.run.smithi125.stdout:(2/13): aardvark-dns-1.7.0-1.module_el8+487+8e4 1.4 MB/s | 1.0 MB 00:00 2023-10-09T10:26:44.789 INFO:teuthology.orchestra.run.smithi155.stdout:(3/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 349 kB/s | 38 kB 00:00 2023-10-09T10:26:44.835 INFO:teuthology.orchestra.run.smithi125.stdout:(3/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 349 kB/s | 38 kB 00:00 2023-10-09T10:26:44.839 INFO:teuthology.orchestra.run.smithi155.stdout:(4/13): crun-1.8.7-1.module_el8+661+d1afb926.x8 2.0 MB/s | 239 kB 00:00 2023-10-09T10:26:44.943 INFO:teuthology.orchestra.run.smithi125.stdout:(4/13): crun-1.8.7-1.module_el8+661+d1afb926.x8 1.3 MB/s | 239 kB 00:00 2023-10-09T10:26:44.964 INFO:teuthology.orchestra.run.smithi155.stdout:(5/13): python3-podman-4.6.0-1.module_el8+582+a 1.4 MB/s | 177 kB 00:00 2023-10-09T10:26:45.060 INFO:teuthology.orchestra.run.smithi125.stdout:(5/13): python3-podman-4.6.0-1.module_el8+582+a 1.5 MB/s | 177 kB 00:00 2023-10-09T10:26:45.123 INFO:teuthology.orchestra.run.smithi155.stdout:(6/13): buildah-1.31.3-1.module_el8+664+4072b3a 7.9 MB/s | 8.8 MB 00:01 2023-10-09T10:26:45.157 INFO:teuthology.orchestra.run.smithi155.stdout:(7/13): python3-pytoml-0.1.14-5.git7dea353.el8. 129 kB/s | 25 kB 00:00 2023-10-09T10:26:45.161 INFO:teuthology.orchestra.run.smithi125.stdout:(6/13): buildah-1.31.3-1.module_el8+664+4072b3a 8.0 MB/s | 8.8 MB 00:01 2023-10-09T10:26:45.204 INFO:teuthology.orchestra.run.smithi125.stdout:(7/13): python3-pytoml-0.1.14-5.git7dea353.el8. 172 kB/s | 25 kB 00:00 2023-10-09T10:26:45.224 INFO:teuthology.orchestra.run.smithi155.stdout:(8/13): python3-pyxdg-0.25-16.el8.noarch.rpm 934 kB/s | 94 kB 00:00 2023-10-09T10:26:45.278 INFO:teuthology.orchestra.run.smithi125.stdout:(8/13): netavark-1.7.0-1.module_el8+487+8e42a27 8.5 MB/s | 3.7 MB 00:00 2023-10-09T10:26:45.282 INFO:teuthology.orchestra.run.smithi155.stdout:(9/13): netavark-1.7.0-1.module_el8+487+8e42a27 7.6 MB/s | 3.7 MB 00:00 2023-10-09T10:26:45.303 INFO:teuthology.orchestra.run.smithi125.stdout:(9/13): python3-pyxdg-0.25-16.el8.noarch.rpm 661 kB/s | 94 kB 00:00 2023-10-09T10:26:45.391 INFO:teuthology.orchestra.run.smithi155.stdout:(10/13): toolbox-0.0.99.4-5.module_el8+649+e2de 15 MB/s | 2.5 MB 00:00 2023-10-09T10:26:45.408 INFO:teuthology.orchestra.run.smithi155.stdout:(11/13): udica-0.2.6-20.module_el8+487+8e42a277 420 kB/s | 53 kB 00:00 2023-10-09T10:26:45.420 INFO:teuthology.orchestra.run.smithi125.stdout:(10/13): udica-0.2.6-20.module_el8+487+8e42a277 451 kB/s | 53 kB 00:00 2023-10-09T10:26:45.470 INFO:teuthology.orchestra.run.smithi125.stdout:(11/13): toolbox-0.0.99.4-5.module_el8+649+e2de 13 MB/s | 2.5 MB 00:00 2023-10-09T10:26:45.484 INFO:teuthology.orchestra.run.smithi155.stdout:(12/13): yajl-2.1.0-12.el8.x86_64.rpm 446 kB/s | 41 kB 00:00 2023-10-09T10:26:45.520 INFO:teuthology.orchestra.run.smithi125.stdout:(12/13): yajl-2.1.0-12.el8.x86_64.rpm 410 kB/s | 41 kB 00:00 2023-10-09T10:26:45.734 INFO:teuthology.orchestra.run.smithi155.stdout:(13/13): skopeo-1.13.3-1.module_el8+664+4072b3a 14 MB/s | 8.1 MB 00:00 2023-10-09T10:26:45.735 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:26:45.735 INFO:teuthology.orchestra.run.smithi155.stdout:Total 13 MB/s | 26 MB 00:01 2023-10-09T10:26:45.763 INFO:teuthology.orchestra.run.smithi125.stdout:(13/13): skopeo-1.13.3-1.module_el8+664+4072b3a 15 MB/s | 8.1 MB 00:00 2023-10-09T10:26:45.763 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:26:45.763 INFO:teuthology.orchestra.run.smithi125.stdout:Total 13 MB/s | 26 MB 00:01 2023-10-09T10:26:46.001 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:26:46.022 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:26:46.022 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:26:46.025 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:26:46.046 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:26:46.046 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:26:46.272 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:26:46.279 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:26:46.297 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:26:46.301 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:26:46.977 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:26:47.017 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:26:47.208 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2023-10-09T10:26:47.246 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2023-10-09T10:26:47.394 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 2/13 2023-10-09T10:26:47.441 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 2/13 2023-10-09T10:26:47.581 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/13 2023-10-09T10:26:47.655 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/13 2023-10-09T10:26:47.838 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-10-09T10:26:47.885 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-10-09T10:26:48.388 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 5/13 2023-10-09T10:26:48.427 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 5/13 2023-10-09T10:26:49.316 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-10-09T10:26:49.380 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-10-09T10:26:49.546 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 7/13 2023-10-09T10:26:49.752 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : crun-1.8.7-1.module_el8+661+d1afb926.x86_64 8/13 2023-10-09T10:26:50.007 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 7/13 2023-10-09T10:26:50.015 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 9/13 2023-10-09T10:26:50.229 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : crun-1.8.7-1.module_el8+661+d1afb926.x86_64 8/13 2023-10-09T10:26:50.424 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : udica-0.2.6-20.module_el8+487+8e42a277.noarch 10/13 2023-10-09T10:26:50.495 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 9/13 2023-10-09T10:26:50.911 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : udica-0.2.6-20.module_el8+487+8e42a277.noarch 10/13 2023-10-09T10:26:51.312 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-10-09T10:26:51.526 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 12/13 2023-10-09T10:26:51.701 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 13/13 2023-10-09T10:26:51.821 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-10-09T10:26:52.056 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 12/13 2023-10-09T10:26:52.128 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 13/13 2023-10-09T10:26:52.128 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 1/13 2023-10-09T10:26:52.128 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 2/13 2023-10-09T10:26:52.128 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 3/13 2023-10-09T10:26:52.128 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-10-09T10:26:52.128 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : crun-1.8.7-1.module_el8+661+d1afb926.x86_64 5/13 2023-10-09T10:26:52.128 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-10-09T10:26:52.128 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 7/13 2023-10-09T10:26:52.128 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2023-10-09T10:26:52.129 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2023-10-09T10:26:52.129 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 10/13 2023-10-09T10:26:52.129 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-10-09T10:26:52.129 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : udica-0.2.6-20.module_el8+487+8e42a277.noarch 12/13 2023-10-09T10:26:52.273 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 13/13 2023-10-09T10:26:52.700 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 13/13 2023-10-09T10:26:52.700 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 1/13 2023-10-09T10:26:52.700 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 2/13 2023-10-09T10:26:52.700 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 3/13 2023-10-09T10:26:52.700 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-10-09T10:26:52.700 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : crun-1.8.7-1.module_el8+661+d1afb926.x86_64 5/13 2023-10-09T10:26:52.701 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-10-09T10:26:52.701 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 7/13 2023-10-09T10:26:52.701 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2023-10-09T10:26:52.701 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2023-10-09T10:26:52.701 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 10/13 2023-10-09T10:26:52.701 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-10-09T10:26:52.701 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : udica-0.2.6-20.module_el8+487+8e42a277.noarch 12/13 2023-10-09T10:26:52.989 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2023-10-09T10:26:52.989 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:26:52.989 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:26:52.989 INFO:teuthology.orchestra.run.smithi155.stdout: aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-10-09T10:26:52.989 INFO:teuthology.orchestra.run.smithi155.stdout: buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 2023-10-09T10:26:52.989 INFO:teuthology.orchestra.run.smithi155.stdout: cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 2023-10-09T10:26:52.989 INFO:teuthology.orchestra.run.smithi155.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2023-10-09T10:26:52.989 INFO:teuthology.orchestra.run.smithi155.stdout: crun-1.8.7-1.module_el8+661+d1afb926.x86_64 2023-10-09T10:26:52.989 INFO:teuthology.orchestra.run.smithi155.stdout: netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-10-09T10:26:52.989 INFO:teuthology.orchestra.run.smithi155.stdout: python3-podman-4.6.0-1.module_el8+582+a2558ae2.noarch 2023-10-09T10:26:52.990 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-10-09T10:26:52.990 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-10-09T10:26:52.990 INFO:teuthology.orchestra.run.smithi155.stdout: skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 2023-10-09T10:26:52.990 INFO:teuthology.orchestra.run.smithi155.stdout: toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 2023-10-09T10:26:52.990 INFO:teuthology.orchestra.run.smithi155.stdout: udica-0.2.6-20.module_el8+487+8e42a277.noarch 2023-10-09T10:26:52.990 INFO:teuthology.orchestra.run.smithi155.stdout: yajl-2.1.0-12.el8.x86_64 2023-10-09T10:26:52.990 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:26:52.990 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:26:53.230 DEBUG:teuthology.parallel:result is None 2023-10-09T10:26:53.492 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2023-10-09T10:26:53.492 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:26:53.492 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:26:53.493 INFO:teuthology.orchestra.run.smithi125.stdout: aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-10-09T10:26:53.493 INFO:teuthology.orchestra.run.smithi125.stdout: buildah-1:1.31.3-1.module_el8+664+4072b3ae.x86_64 2023-10-09T10:26:53.493 INFO:teuthology.orchestra.run.smithi125.stdout: cockpit-podman-75-1.module_el8+664+4072b3ae.noarch 2023-10-09T10:26:53.493 INFO:teuthology.orchestra.run.smithi125.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2023-10-09T10:26:53.493 INFO:teuthology.orchestra.run.smithi125.stdout: crun-1.8.7-1.module_el8+661+d1afb926.x86_64 2023-10-09T10:26:53.493 INFO:teuthology.orchestra.run.smithi125.stdout: netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-10-09T10:26:53.493 INFO:teuthology.orchestra.run.smithi125.stdout: python3-podman-4.6.0-1.module_el8+582+a2558ae2.noarch 2023-10-09T10:26:53.493 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-10-09T10:26:53.493 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-10-09T10:26:53.493 INFO:teuthology.orchestra.run.smithi125.stdout: skopeo-2:1.13.3-1.module_el8+664+4072b3ae.x86_64 2023-10-09T10:26:53.493 INFO:teuthology.orchestra.run.smithi125.stdout: toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 2023-10-09T10:26:53.494 INFO:teuthology.orchestra.run.smithi125.stdout: udica-0.2.6-20.module_el8+487+8e42a277.noarch 2023-10-09T10:26:53.494 INFO:teuthology.orchestra.run.smithi125.stdout: yajl-2.1.0-12.el8.x86_64 2023-10-09T10:26:53.494 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:26:53.494 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:26:53.731 DEBUG:teuthology.parallel:result is None 2023-10-09T10:26:53.731 INFO:teuthology.run_tasks:Running task install... 2023-10-09T10:26:53.743 DEBUG:teuthology.task.install:project ceph 2023-10-09T10:26:53.743 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}} 2023-10-09T10:26:53.743 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T10:26:53.743 INFO:teuthology.task.install:Using flavor: default 2023-10-09T10:26:53.753 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-10-09T10:26:53.753 INFO:teuthology.task.install:extra packages: [] 2023-10-09T10:26:53.754 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0', 'tag': None, 'wait_for_package': False} 2023-10-09T10:26:53.754 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T10:26:53.755 INFO:teuthology.packaging:ref: None 2023-10-09T10:26:53.755 INFO:teuthology.packaging:tag: None 2023-10-09T10:26:53.755 INFO:teuthology.packaging:branch: quincy 2023-10-09T10:26:53.755 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T10:26:53.755 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T10:26:53.758 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0', 'tag': None, 'wait_for_package': False} 2023-10-09T10:26:53.758 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T10:26:53.759 INFO:teuthology.packaging:ref: None 2023-10-09T10:26:53.759 INFO:teuthology.packaging:tag: None 2023-10-09T10:26:53.759 INFO:teuthology.packaging:branch: quincy 2023-10-09T10:26:53.759 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T10:26:53.759 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T10:26:54.064 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/ 2023-10-09T10:26:54.065 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1303.gb2903cb0 2023-10-09T10:26:54.128 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/ 2023-10-09T10:26:54.128 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1303.gb2903cb0 2023-10-09T10:26:54.240 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-09T10:26:54.240 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:26:54.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-09T10:26:54.242 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-09T10:26:54.242 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T10:26:54.243 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-09T10:26:54.273 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-09T10:26:54.273 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T10:26:54.273 INFO:teuthology.packaging:ref: None 2023-10-09T10:26:54.273 INFO:teuthology.packaging:tag: None 2023-10-09T10:26:54.273 INFO:teuthology.packaging:branch: quincy 2023-10-09T10:26:54.273 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T10:26:54.274 DEBUG:teuthology.orchestra.run.smithi125:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-09T10:26:54.278 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-09T10:26:54.278 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T10:26:54.278 INFO:teuthology.packaging:ref: None 2023-10-09T10:26:54.278 INFO:teuthology.packaging:tag: None 2023-10-09T10:26:54.278 INFO:teuthology.packaging:branch: quincy 2023-10-09T10:26:54.279 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T10:26:54.279 DEBUG:teuthology.orchestra.run.smithi155:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-09T10:26:54.342 DEBUG:teuthology.orchestra.run.smithi125:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-09T10:26:54.350 DEBUG:teuthology.orchestra.run.smithi155:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-09T10:26:54.420 DEBUG:teuthology.orchestra.run.smithi125:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-09T10:26:54.430 DEBUG:teuthology.orchestra.run.smithi155:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-09T10:26:54.487 INFO:teuthology.orchestra.run.smithi125.stdout:check_obsoletes = 1 2023-10-09T10:26:54.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum clean all 2023-10-09T10:26:54.497 INFO:teuthology.orchestra.run.smithi155.stdout:check_obsoletes = 1 2023-10-09T10:26:54.499 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum clean all 2023-10-09T10:26:54.756 INFO:teuthology.orchestra.run.smithi125.stdout:76 files removed 2023-10-09T10:26:54.784 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install ceph-radosgw 2023-10-09T10:26:54.813 INFO:teuthology.orchestra.run.smithi155.stdout:76 files removed 2023-10-09T10:26:54.843 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install ceph-radosgw 2023-10-09T10:26:55.473 INFO:teuthology.orchestra.run.smithi125.stdout:ceph packages for x86_64 274 kB/s | 78 kB 00:00 2023-10-09T10:26:55.619 INFO:teuthology.orchestra.run.smithi155.stdout:ceph packages for x86_64 213 kB/s | 78 kB 00:00 2023-10-09T10:26:55.821 INFO:teuthology.orchestra.run.smithi125.stdout:ceph noarch packages 29 kB/s | 9.1 kB 00:00 2023-10-09T10:26:55.928 INFO:teuthology.orchestra.run.smithi155.stdout:ceph noarch packages 34 kB/s | 9.1 kB 00:00 2023-10-09T10:26:56.181 INFO:teuthology.orchestra.run.smithi125.stdout:ceph source packages 4.4 kB/s | 1.4 kB 00:00 2023-10-09T10:26:56.229 INFO:teuthology.orchestra.run.smithi155.stdout:ceph source packages 5.3 kB/s | 1.4 kB 00:00 2023-10-09T10:26:58.339 INFO:teuthology.orchestra.run.smithi125.stdout:CentOS Stream 8 - AppStream 16 MB/s | 33 MB 00:02 2023-10-09T10:26:58.413 INFO:teuthology.orchestra.run.smithi155.stdout:CentOS Stream 8 - AppStream 15 MB/s | 33 MB 00:02 2023-10-09T10:27:07.831 INFO:teuthology.orchestra.run.smithi125.stdout:CentOS Stream 8 - BaseOS 15 MB/s | 48 MB 00:03 2023-10-09T10:27:07.980 INFO:teuthology.orchestra.run.smithi155.stdout:CentOS Stream 8 - BaseOS 14 MB/s | 48 MB 00:03 2023-10-09T10:27:15.447 INFO:teuthology.orchestra.run.smithi125.stdout:CentOS Stream 8 - Extras 28 kB/s | 18 kB 00:00 2023-10-09T10:27:15.496 INFO:teuthology.orchestra.run.smithi155.stdout:CentOS Stream 8 - Extras 34 kB/s | 18 kB 00:00 2023-10-09T10:27:15.781 INFO:teuthology.orchestra.run.smithi125.stdout:CentOS Stream 8 - Extras common packages 45 kB/s | 6.9 kB 00:00 2023-10-09T10:27:15.840 INFO:teuthology.orchestra.run.smithi155.stdout:CentOS Stream 8 - Extras common packages 41 kB/s | 6.9 kB 00:00 2023-10-09T10:27:16.349 INFO:teuthology.orchestra.run.smithi125.stdout:CentOS Stream 8 - PowerTools 15 MB/s | 6.5 MB 00:00 2023-10-09T10:27:16.415 INFO:teuthology.orchestra.run.smithi155.stdout:CentOS Stream 8 - PowerTools 15 MB/s | 6.5 MB 00:00 2023-10-09T10:27:17.751 INFO:teuthology.orchestra.run.smithi125.stdout:Copr repo for python3-asyncssh owned by ceph 29 kB/s | 3.5 kB 00:00 2023-10-09T10:27:17.808 INFO:teuthology.orchestra.run.smithi155.stdout:Copr repo for python3-asyncssh owned by ceph 29 kB/s | 3.5 kB 00:00 2023-10-09T10:27:18.726 INFO:teuthology.orchestra.run.smithi125.stdout:Extra Packages for Enterprise Linux 19 MB/s | 16 MB 00:00 2023-10-09T10:27:21.335 INFO:teuthology.orchestra.run.smithi155.stdout:Extra Packages for Enterprise Linux 4.7 MB/s | 16 MB 00:03 2023-10-09T10:27:22.585 INFO:teuthology.orchestra.run.smithi125.stdout:lab-extras 478 kB/s | 24 kB 00:00 2023-10-09T10:27:25.294 INFO:teuthology.orchestra.run.smithi155.stdout:lab-extras 378 kB/s | 24 kB 00:00 2023-10-09T10:27:27.200 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:27:27.203 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:27:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: Package Arch Version Repository Size 2023-10-09T10:27:27.203 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:27:27.203 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:27:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-radosgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T10:27:27.203 INFO:teuthology.orchestra.run.smithi125.stdout:Upgrading: 2023-10-09T10:27:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: librados2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T10:27:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: librbd1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.6 M 2023-10-09T10:27:27.203 INFO:teuthology.orchestra.run.smithi125.stdout:Installing dependencies: 2023-10-09T10:27:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-base x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.2 M 2023-10-09T10:27:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 23 M 2023-10-09T10:27:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-selinux x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 24 k 2023-10-09T10:27:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-09T10:27:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: libcephfs2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 797 k 2023-10-09T10:27:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-09T10:27:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: librabbitmq x86_64 0.9.0-4.el8 baseos 47 k 2023-10-09T10:27:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: libradosstriper1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 509 k 2023-10-09T10:27:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: librdkafka x86_64 0.11.4-3.el8 appstream 354 k 2023-10-09T10:27:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: librgw2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.6 M 2023-10-09T10:27:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-09T10:27:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: lttng-ust x86_64 2.8.1-11.el8 appstream 259 k 2023-10-09T10:27:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: python3-ceph-argparse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 45 k 2023-10-09T10:27:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: python3-ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 115 k 2023-10-09T10:27:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: python3-cephfs x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 190 k 2023-10-09T10:27:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: python3-prettytable noarch 0.7.2-14.el8 appstream 44 k 2023-10-09T10:27:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: python3-rados x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 370 k 2023-10-09T10:27:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: python3-rbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 358 k 2023-10-09T10:27:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: python3-rgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 106 k 2023-10-09T10:27:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-10-09T10:27:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:27:27.205 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:27:27.206 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:27:27.206 INFO:teuthology.orchestra.run.smithi125.stdout:Install 21 Packages 2023-10-09T10:27:27.206 INFO:teuthology.orchestra.run.smithi125.stdout:Upgrade 2 Packages 2023-10-09T10:27:27.206 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:27:27.206 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 63 M 2023-10-09T10:27:27.207 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:27:28.612 INFO:teuthology.orchestra.run.smithi125.stdout:(1/23): ceph-base-17.2.6-1303.gb2903cb0.el8.x86 4.4 MB/s | 5.2 MB 00:01 2023-10-09T10:27:28.712 INFO:teuthology.orchestra.run.smithi125.stdout:(2/23): ceph-selinux-17.2.6-1303.gb2903cb0.el8. 243 kB/s | 24 kB 00:00 2023-10-09T10:27:28.887 INFO:teuthology.orchestra.run.smithi125.stdout:(3/23): libcephfs2-17.2.6-1303.gb2903cb0.el8.x8 4.4 MB/s | 797 kB 00:00 2023-10-09T10:27:29.245 INFO:teuthology.orchestra.run.smithi125.stdout:(4/23): ceph-common-17.2.6-1303.gb2903cb0.el8.x 13 MB/s | 23 MB 00:01 2023-10-09T10:27:29.296 INFO:teuthology.orchestra.run.smithi125.stdout:(5/23): libradosstriper1-17.2.6-1303.gb2903cb0. 1.2 MB/s | 509 kB 00:00 2023-10-09T10:27:29.396 INFO:teuthology.orchestra.run.smithi125.stdout:(6/23): python3-ceph-argparse-17.2.6-1303.gb290 448 kB/s | 45 kB 00:00 2023-10-09T10:27:29.488 INFO:teuthology.orchestra.run.smithi125.stdout:(7/23): python3-ceph-common-17.2.6-1303.gb2903c 1.2 MB/s | 115 kB 00:00 2023-10-09T10:27:29.605 INFO:teuthology.orchestra.run.smithi125.stdout:(8/23): python3-cephfs-17.2.6-1303.gb2903cb0.el 1.6 MB/s | 190 kB 00:00 2023-10-09T10:27:29.627 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:27:29.629 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:27:29.630 INFO:teuthology.orchestra.run.smithi155.stdout: Package Arch Version Repository Size 2023-10-09T10:27:29.630 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:27:29.630 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:27:29.630 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-radosgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T10:27:29.630 INFO:teuthology.orchestra.run.smithi155.stdout:Upgrading: 2023-10-09T10:27:29.630 INFO:teuthology.orchestra.run.smithi155.stdout: librados2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T10:27:29.630 INFO:teuthology.orchestra.run.smithi155.stdout: librbd1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.6 M 2023-10-09T10:27:29.630 INFO:teuthology.orchestra.run.smithi155.stdout:Installing dependencies: 2023-10-09T10:27:29.631 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-base x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.2 M 2023-10-09T10:27:29.631 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 23 M 2023-10-09T10:27:29.631 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-selinux x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 24 k 2023-10-09T10:27:29.631 INFO:teuthology.orchestra.run.smithi155.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-09T10:27:29.631 INFO:teuthology.orchestra.run.smithi155.stdout: libcephfs2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 797 k 2023-10-09T10:27:29.631 INFO:teuthology.orchestra.run.smithi155.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-09T10:27:29.631 INFO:teuthology.orchestra.run.smithi155.stdout: librabbitmq x86_64 0.9.0-4.el8 baseos 47 k 2023-10-09T10:27:29.631 INFO:teuthology.orchestra.run.smithi155.stdout: libradosstriper1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 509 k 2023-10-09T10:27:29.631 INFO:teuthology.orchestra.run.smithi155.stdout: librdkafka x86_64 0.11.4-3.el8 appstream 354 k 2023-10-09T10:27:29.631 INFO:teuthology.orchestra.run.smithi155.stdout: librgw2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.6 M 2023-10-09T10:27:29.632 INFO:teuthology.orchestra.run.smithi155.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-09T10:27:29.632 INFO:teuthology.orchestra.run.smithi155.stdout: lttng-ust x86_64 2.8.1-11.el8 appstream 259 k 2023-10-09T10:27:29.632 INFO:teuthology.orchestra.run.smithi155.stdout: python3-ceph-argparse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 45 k 2023-10-09T10:27:29.632 INFO:teuthology.orchestra.run.smithi155.stdout: python3-ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 115 k 2023-10-09T10:27:29.632 INFO:teuthology.orchestra.run.smithi155.stdout: python3-cephfs x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 190 k 2023-10-09T10:27:29.632 INFO:teuthology.orchestra.run.smithi155.stdout: python3-prettytable noarch 0.7.2-14.el8 appstream 44 k 2023-10-09T10:27:29.632 INFO:teuthology.orchestra.run.smithi155.stdout: python3-rados x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 370 k 2023-10-09T10:27:29.632 INFO:teuthology.orchestra.run.smithi155.stdout: python3-rbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 358 k 2023-10-09T10:27:29.632 INFO:teuthology.orchestra.run.smithi155.stdout: python3-rgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 106 k 2023-10-09T10:27:29.632 INFO:teuthology.orchestra.run.smithi155.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-10-09T10:27:29.633 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:27:29.633 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:27:29.633 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:27:29.633 INFO:teuthology.orchestra.run.smithi155.stdout:Install 21 Packages 2023-10-09T10:27:29.633 INFO:teuthology.orchestra.run.smithi155.stdout:Upgrade 2 Packages 2023-10-09T10:27:29.633 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:27:29.634 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 63 M 2023-10-09T10:27:29.634 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:27:29.680 INFO:teuthology.orchestra.run.smithi125.stdout:(9/23): librgw2-17.2.6-1303.gb2903cb0.el8.x86_6 13 MB/s | 5.6 MB 00:00 2023-10-09T10:27:29.764 INFO:teuthology.orchestra.run.smithi125.stdout:(10/23): python3-rbd-17.2.6-1303.gb2903cb0.el8. 4.2 MB/s | 358 kB 00:00 2023-10-09T10:27:29.789 INFO:teuthology.orchestra.run.smithi125.stdout:(11/23): python3-rados-17.2.6-1303.gb2903cb0.el 2.0 MB/s | 370 kB 00:00 2023-10-09T10:27:29.814 INFO:teuthology.orchestra.run.smithi125.stdout:(12/23): python3-rgw-17.2.6-1303.gb2903cb0.el8. 2.1 MB/s | 106 kB 00:00 2023-10-09T10:27:29.998 INFO:teuthology.orchestra.run.smithi125.stdout:(13/23): ceph-radosgw-17.2.6-1303.gb2903cb0.el8 6.5 MB/s | 17 MB 00:02 2023-10-09T10:27:30.323 INFO:teuthology.orchestra.run.smithi125.stdout:(14/23): python3-prettytable-0.7.2-14.el8.noarc 135 kB/s | 44 kB 00:00 2023-10-09T10:27:30.490 INFO:teuthology.orchestra.run.smithi125.stdout:(15/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 384 kB/s | 259 kB 00:00 2023-10-09T10:27:30.515 INFO:teuthology.orchestra.run.smithi125.stdout:(16/23): librdkafka-0.11.4-3.el8.x86_64.rpm 487 kB/s | 354 kB 00:00 2023-10-09T10:27:30.574 INFO:teuthology.orchestra.run.smithi125.stdout:(17/23): liboath-2.6.2-3.el8.x86_64.rpm 1.0 MB/s | 59 kB 00:00 2023-10-09T10:27:30.608 INFO:teuthology.orchestra.run.smithi125.stdout:(18/23): gperftools-libs-2.7-9.el8.x86_64.rpm 2.6 MB/s | 306 kB 00:00 2023-10-09T10:27:30.633 INFO:teuthology.orchestra.run.smithi125.stdout:(19/23): libunwind-1.3.1-3.el8.x86_64.rpm 1.3 MB/s | 75 kB 00:00 2023-10-09T10:27:30.708 INFO:teuthology.orchestra.run.smithi125.stdout:(20/23): thrift-0.13.0-2.el8.x86_64.rpm 17 MB/s | 1.7 MB 00:00 2023-10-09T10:27:30.867 INFO:teuthology.orchestra.run.smithi125.stdout:(21/23): librabbitmq-0.9.0-4.el8.x86_64.rpm 86 kB/s | 47 kB 00:00 2023-10-09T10:27:31.203 INFO:teuthology.orchestra.run.smithi155.stdout:(1/23): ceph-base-17.2.6-1303.gb2903cb0.el8.x86 3.9 MB/s | 5.2 MB 00:01 2023-10-09T10:27:31.361 INFO:teuthology.orchestra.run.smithi155.stdout:(2/23): ceph-selinux-17.2.6-1303.gb2903cb0.el8. 153 kB/s | 24 kB 00:00 2023-10-09T10:27:31.475 INFO:teuthology.orchestra.run.smithi125.stdout:(22/23): librbd1-17.2.6-1303.gb2903cb0.el8.x86_ 4.7 MB/s | 3.6 MB 00:00 2023-10-09T10:27:31.637 INFO:teuthology.orchestra.run.smithi155.stdout:(3/23): libcephfs2-17.2.6-1303.gb2903cb0.el8.x8 2.8 MB/s | 797 kB 00:00 2023-10-09T10:27:31.742 INFO:teuthology.orchestra.run.smithi125.stdout:(23/23): librados2-17.2.6-1303.gb2903cb0.el8.x8 3.1 MB/s | 3.5 MB 00:01 2023-10-09T10:27:31.744 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:27:31.744 INFO:teuthology.orchestra.run.smithi125.stdout:Total 14 MB/s | 63 MB 00:04 2023-10-09T10:27:31.786 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:27:31.834 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:27:31.835 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:27:31.879 INFO:teuthology.orchestra.run.smithi155.stdout:(4/23): ceph-common-17.2.6-1303.gb2903cb0.el8.x 12 MB/s | 23 MB 00:01 2023-10-09T10:27:32.012 INFO:teuthology.orchestra.run.smithi155.stdout:(5/23): libradosstriper1-17.2.6-1303.gb2903cb0. 1.3 MB/s | 509 kB 00:00 2023-10-09T10:27:32.096 INFO:teuthology.orchestra.run.smithi155.stdout:(6/23): python3-ceph-argparse-17.2.6-1303.gb290 537 kB/s | 45 kB 00:00 2023-10-09T10:27:32.204 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:27:32.207 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:27:32.229 INFO:teuthology.orchestra.run.smithi155.stdout:(7/23): librgw2-17.2.6-1303.gb2903cb0.el8.x86_6 16 MB/s | 5.6 MB 00:00 2023-10-09T10:27:32.255 INFO:teuthology.orchestra.run.smithi155.stdout:(8/23): python3-ceph-common-17.2.6-1303.gb2903c 728 kB/s | 115 kB 00:00 2023-10-09T10:27:32.305 INFO:teuthology.orchestra.run.smithi155.stdout:(9/23): python3-rados-17.2.6-1303.gb2903cb0.el8 7.2 MB/s | 370 kB 00:00 2023-10-09T10:27:32.331 INFO:teuthology.orchestra.run.smithi155.stdout:(10/23): python3-cephfs-17.2.6-1303.gb2903cb0.e 1.8 MB/s | 190 kB 00:00 2023-10-09T10:27:32.388 INFO:teuthology.orchestra.run.smithi155.stdout:(11/23): python3-rbd-17.2.6-1303.gb2903cb0.el8. 4.2 MB/s | 358 kB 00:00 2023-10-09T10:27:32.423 INFO:teuthology.orchestra.run.smithi155.stdout:(12/23): python3-rgw-17.2.6-1303.gb2903cb0.el8. 1.1 MB/s | 106 kB 00:00 2023-10-09T10:27:32.948 INFO:teuthology.orchestra.run.smithi155.stdout:(13/23): ceph-radosgw-17.2.6-1303.gb2903cb0.el8 5.5 MB/s | 17 MB 00:03 2023-10-09T10:27:32.965 INFO:teuthology.orchestra.run.smithi155.stdout:(14/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 479 kB/s | 259 kB 00:00 2023-10-09T10:27:33.082 INFO:teuthology.orchestra.run.smithi155.stdout:(15/23): librdkafka-0.11.4-3.el8.x86_64.rpm 510 kB/s | 354 kB 00:00 2023-10-09T10:27:33.282 INFO:teuthology.orchestra.run.smithi155.stdout:(16/23): python3-prettytable-0.7.2-14.el8.noarc 131 kB/s | 44 kB 00:00 2023-10-09T10:27:33.303 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:27:33.316 INFO:teuthology.orchestra.run.smithi155.stdout:(17/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.3 MB/s | 306 kB 00:00 2023-10-09T10:27:33.366 INFO:teuthology.orchestra.run.smithi155.stdout:(18/23): libunwind-1.3.1-3.el8.x86_64.rpm 1.5 MB/s | 75 kB 00:00 2023-10-09T10:27:33.392 INFO:teuthology.orchestra.run.smithi155.stdout:(19/23): librabbitmq-0.9.0-4.el8.x86_64.rpm 110 kB/s | 47 kB 00:00 2023-10-09T10:27:33.450 INFO:teuthology.orchestra.run.smithi155.stdout:(20/23): liboath-2.6.2-3.el8.x86_64.rpm 351 kB/s | 59 kB 00:00 2023-10-09T10:27:33.534 INFO:teuthology.orchestra.run.smithi155.stdout:(21/23): thrift-0.13.0-2.el8.x86_64.rpm 10 MB/s | 1.7 MB 00:00 2023-10-09T10:27:33.544 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : liboath-2.6.2-3.el8.x86_64 1/25 2023-10-09T10:27:33.725 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-10-09T10:27:33.734 INFO:teuthology.orchestra.run.smithi155.stdout:(22/23): librbd1-17.2.6-1303.gb2903cb0.el8.x86_ 13 MB/s | 3.6 MB 00:00 2023-10-09T10:27:34.141 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-10-09T10:27:34.370 INFO:teuthology.orchestra.run.smithi125.stdout: Upgrading : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T10:27:34.393 INFO:teuthology.orchestra.run.smithi155.stdout:(23/23): librados2-17.2.6-1303.gb2903cb0.el8.x8 3.5 MB/s | 3.5 MB 00:01 2023-10-09T10:27:34.395 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:27:34.396 INFO:teuthology.orchestra.run.smithi155.stdout:Total 13 MB/s | 63 MB 00:04 2023-10-09T10:27:34.439 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:27:34.458 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T10:27:34.486 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:27:34.486 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:27:34.683 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/25 2023-10-09T10:27:34.851 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:27:34.854 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:27:34.858 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T10:27:35.235 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T10:27:35.408 INFO:teuthology.orchestra.run.smithi125.stdout: Upgrading : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T10:27:35.621 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T10:27:35.821 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : thrift-0.13.0-2.el8.x86_64 7/25 2023-10-09T10:27:35.913 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:27:36.014 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : librabbitmq-0.9.0-4.el8.x86_64 8/25 2023-10-09T10:27:36.130 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : liboath-2.6.2-3.el8.x86_64 1/25 2023-10-09T10:27:36.208 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 9/25 2023-10-09T10:27:36.304 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-10-09T10:27:36.687 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-10-09T10:27:36.765 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 9/25 2023-10-09T10:27:36.854 INFO:teuthology.orchestra.run.smithi155.stdout: Upgrading : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T10:27:36.900 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T10:27:36.935 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T10:27:36.969 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T10:27:37.162 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/25 2023-10-09T10:27:37.349 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 11/25 2023-10-09T10:27:37.387 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T10:27:37.562 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T10:27:37.767 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T10:27:37.779 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T10:27:37.946 INFO:teuthology.orchestra.run.smithi155.stdout: Upgrading : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T10:27:37.996 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 14/25 2023-10-09T10:27:38.158 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T10:27:38.201 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T10:27:38.227 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T10:27:38.383 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : thrift-0.13.0-2.el8.x86_64 7/25 2023-10-09T10:27:38.477 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 16/25 2023-10-09T10:27:38.568 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : librabbitmq-0.9.0-4.el8.x86_64 8/25 2023-10-09T10:27:38.658 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 17/25 2023-10-09T10:27:38.745 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 9/25 2023-10-09T10:27:38.870 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 18/25 2023-10-09T10:27:39.117 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 19/25 2023-10-09T10:27:39.298 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 9/25 2023-10-09T10:27:39.487 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T10:27:39.554 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T10:27:39.722 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 11/25 2023-10-09T10:27:39.915 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T10:27:40.109 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T10:27:40.341 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 14/25 2023-10-09T10:27:40.503 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T10:27:40.564 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T10:27:40.780 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 16/25 2023-10-09T10:27:40.970 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 17/25 2023-10-09T10:27:41.146 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 18/25 2023-10-09T10:27:41.337 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 19/25 2023-10-09T10:27:41.854 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T10:27:42.180 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T10:27:42.764 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T10:27:43.062 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T10:27:43.168 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T10:27:43.374 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T10:27:44.057 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T10:27:44.291 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T10:27:44.878 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T10:27:45.169 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T10:27:45.276 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T10:27:45.461 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T10:28:01.661 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T10:28:01.661 INFO:teuthology.orchestra.run.smithi125.stdout:skipping the directory /sys 2023-10-09T10:28:01.661 INFO:teuthology.orchestra.run.smithi125.stdout:skipping the directory /proc 2023-10-09T10:28:01.661 INFO:teuthology.orchestra.run.smithi125.stdout:skipping the directory /dev 2023-10-09T10:28:01.661 INFO:teuthology.orchestra.run.smithi125.stdout:skipping the directory /run 2023-10-09T10:28:01.661 INFO:teuthology.orchestra.run.smithi125.stdout:skipping the directory /mnt 2023-10-09T10:28:01.661 INFO:teuthology.orchestra.run.smithi125.stdout:skipping the directory /var/tmp 2023-10-09T10:28:01.661 INFO:teuthology.orchestra.run.smithi125.stdout:skipping the directory /home 2023-10-09T10:28:01.662 INFO:teuthology.orchestra.run.smithi125.stdout:skipping the directory /tmp 2023-10-09T10:28:01.662 INFO:teuthology.orchestra.run.smithi125.stdout:skipping the directory /dev 2023-10-09T10:28:01.662 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:03.351 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T10:28:03.463 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T10:28:03.463 INFO:teuthology.orchestra.run.smithi125.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T10:28:03.664 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T10:28:03.665 INFO:teuthology.orchestra.run.smithi125.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T10:28:03.846 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T10:28:03.846 INFO:teuthology.orchestra.run.smithi155.stdout:skipping the directory /sys 2023-10-09T10:28:03.846 INFO:teuthology.orchestra.run.smithi155.stdout:skipping the directory /proc 2023-10-09T10:28:03.846 INFO:teuthology.orchestra.run.smithi155.stdout:skipping the directory /dev 2023-10-09T10:28:03.847 INFO:teuthology.orchestra.run.smithi155.stdout:skipping the directory /run 2023-10-09T10:28:03.847 INFO:teuthology.orchestra.run.smithi155.stdout:skipping the directory /mnt 2023-10-09T10:28:03.847 INFO:teuthology.orchestra.run.smithi155.stdout:skipping the directory /var/tmp 2023-10-09T10:28:03.847 INFO:teuthology.orchestra.run.smithi155.stdout:skipping the directory /home 2023-10-09T10:28:03.847 INFO:teuthology.orchestra.run.smithi155.stdout:skipping the directory /tmp 2023-10-09T10:28:03.847 INFO:teuthology.orchestra.run.smithi155.stdout:skipping the directory /dev 2023-10-09T10:28:03.847 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:04.356 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T10:28:04.357 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/25 2023-10-09T10:28:04.357 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T10:28:04.357 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T10:28:04.357 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/25 2023-10-09T10:28:04.357 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T10:28:04.357 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 6/25 2023-10-09T10:28:04.358 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 7/25 2023-10-09T10:28:04.358 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 8/25 2023-10-09T10:28:04.358 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 9/25 2023-10-09T10:28:04.358 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T10:28:04.358 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 11/25 2023-10-09T10:28:04.358 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T10:28:04.358 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T10:28:04.358 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 14/25 2023-10-09T10:28:04.358 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 15/25 2023-10-09T10:28:04.359 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-09T10:28:04.359 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : librabbitmq-0.9.0-4.el8.x86_64 17/25 2023-10-09T10:28:04.359 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-09T10:28:04.359 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 19/25 2023-10-09T10:28:04.359 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 20/25 2023-10-09T10:28:04.360 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 21/25 2023-10-09T10:28:04.360 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T10:28:04.360 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-09T10:28:04.360 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 24/25 2023-10-09T10:28:05.451 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T10:28:05.452 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:05.452 INFO:teuthology.orchestra.run.smithi125.stdout:Upgraded: 2023-10-09T10:28:05.452 INFO:teuthology.orchestra.run.smithi125.stdout: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:05.452 INFO:teuthology.orchestra.run.smithi125.stdout: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:05.453 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:28:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-09T10:28:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-09T10:28:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: librabbitmq-0.9.0-4.el8.x86_64 2023-10-09T10:28:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-09T10:28:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-09T10:28:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-09T10:28:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-09T10:28:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: thrift-0.13.0-2.el8.x86_64 2023-10-09T10:28:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:05.456 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:28:05.557 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T10:28:05.619 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T10:28:05.620 INFO:teuthology.orchestra.run.smithi155.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T10:28:05.774 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install ceph-test 2023-10-09T10:28:05.885 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T10:28:05.885 INFO:teuthology.orchestra.run.smithi155.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T10:28:06.608 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T10:28:06.608 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/25 2023-10-09T10:28:06.608 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T10:28:06.608 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T10:28:06.608 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/25 2023-10-09T10:28:06.608 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T10:28:06.608 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 6/25 2023-10-09T10:28:06.608 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 7/25 2023-10-09T10:28:06.608 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 8/25 2023-10-09T10:28:06.609 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 9/25 2023-10-09T10:28:06.609 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T10:28:06.609 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 11/25 2023-10-09T10:28:06.609 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T10:28:06.609 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T10:28:06.609 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 14/25 2023-10-09T10:28:06.609 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 15/25 2023-10-09T10:28:06.609 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-09T10:28:06.609 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : librabbitmq-0.9.0-4.el8.x86_64 17/25 2023-10-09T10:28:06.609 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-09T10:28:06.610 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 19/25 2023-10-09T10:28:06.610 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 20/25 2023-10-09T10:28:06.610 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 21/25 2023-10-09T10:28:06.610 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T10:28:06.610 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-09T10:28:06.610 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 24/25 2023-10-09T10:28:06.630 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:00:44 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:28:07.756 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T10:28:07.756 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:07.757 INFO:teuthology.orchestra.run.smithi155.stdout:Upgraded: 2023-10-09T10:28:07.757 INFO:teuthology.orchestra.run.smithi155.stdout: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:07.757 INFO:teuthology.orchestra.run.smithi155.stdout: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:07.757 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:28:07.757 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:07.757 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:07.758 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:07.758 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:07.758 INFO:teuthology.orchestra.run.smithi155.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-09T10:28:07.758 INFO:teuthology.orchestra.run.smithi155.stdout: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:07.758 INFO:teuthology.orchestra.run.smithi155.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-09T10:28:07.758 INFO:teuthology.orchestra.run.smithi155.stdout: librabbitmq-0.9.0-4.el8.x86_64 2023-10-09T10:28:07.759 INFO:teuthology.orchestra.run.smithi155.stdout: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:07.759 INFO:teuthology.orchestra.run.smithi155.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-09T10:28:07.759 INFO:teuthology.orchestra.run.smithi155.stdout: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:07.759 INFO:teuthology.orchestra.run.smithi155.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-09T10:28:07.759 INFO:teuthology.orchestra.run.smithi155.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-09T10:28:07.759 INFO:teuthology.orchestra.run.smithi155.stdout: python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:07.759 INFO:teuthology.orchestra.run.smithi155.stdout: python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:07.760 INFO:teuthology.orchestra.run.smithi155.stdout: python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:07.760 INFO:teuthology.orchestra.run.smithi155.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-09T10:28:07.760 INFO:teuthology.orchestra.run.smithi155.stdout: python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:07.760 INFO:teuthology.orchestra.run.smithi155.stdout: python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:07.760 INFO:teuthology.orchestra.run.smithi155.stdout: python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:07.760 INFO:teuthology.orchestra.run.smithi155.stdout: thrift-0.13.0-2.el8.x86_64 2023-10-09T10:28:07.761 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:07.761 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:28:08.098 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install ceph-test 2023-10-09T10:28:08.215 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:28:08.216 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:28:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: Package Arch Version Repository Size 2023-10-09T10:28:08.217 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:28:08.217 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:28:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-test x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 52 M 2023-10-09T10:28:08.217 INFO:teuthology.orchestra.run.smithi125.stdout:Installing dependencies: 2023-10-09T10:28:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: jq x86_64 1.6-7.el8 appstream 202 k 2023-10-09T10:28:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: libcephsqlite x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 175 k 2023-10-09T10:28:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: oniguruma x86_64 6.8.2-2.el8 appstream 187 k 2023-10-09T10:28:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: socat x86_64 1.7.4.1-1.el8 appstream 323 k 2023-10-09T10:28:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: xmlstarlet x86_64 1.6.1-20.el8 appstream 71 k 2023-10-09T10:28:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:08.218 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:28:08.218 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:28:08.218 INFO:teuthology.orchestra.run.smithi125.stdout:Install 6 Packages 2023-10-09T10:28:08.218 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:08.218 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 53 M 2023-10-09T10:28:08.218 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 233 M 2023-10-09T10:28:08.219 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:28:08.957 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:00:43 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:28:09.612 INFO:teuthology.orchestra.run.smithi125.stdout:(1/6): libcephsqlite-17.2.6-1303.gb2903cb0.el8. 919 kB/s | 175 kB 00:00 2023-10-09T10:28:09.928 INFO:teuthology.orchestra.run.smithi125.stdout:(2/6): jq-1.6-7.el8.x86_64.rpm 399 kB/s | 202 kB 00:00 2023-10-09T10:28:10.104 INFO:teuthology.orchestra.run.smithi125.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 381 kB/s | 187 kB 00:00 2023-10-09T10:28:10.138 INFO:teuthology.orchestra.run.smithi125.stdout:(4/6): socat-1.7.4.1-1.el8.x86_64.rpm 1.5 MB/s | 323 kB 00:00 2023-10-09T10:28:10.229 INFO:teuthology.orchestra.run.smithi125.stdout:(5/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 565 kB/s | 71 kB 00:00 2023-10-09T10:28:10.551 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:28:10.552 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:28:10.553 INFO:teuthology.orchestra.run.smithi155.stdout: Package Arch Version Repository Size 2023-10-09T10:28:10.553 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:28:10.553 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:28:10.553 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-test x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 52 M 2023-10-09T10:28:10.553 INFO:teuthology.orchestra.run.smithi155.stdout:Installing dependencies: 2023-10-09T10:28:10.554 INFO:teuthology.orchestra.run.smithi155.stdout: jq x86_64 1.6-7.el8 appstream 202 k 2023-10-09T10:28:10.554 INFO:teuthology.orchestra.run.smithi155.stdout: libcephsqlite x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 175 k 2023-10-09T10:28:10.554 INFO:teuthology.orchestra.run.smithi155.stdout: oniguruma x86_64 6.8.2-2.el8 appstream 187 k 2023-10-09T10:28:10.554 INFO:teuthology.orchestra.run.smithi155.stdout: socat x86_64 1.7.4.1-1.el8 appstream 323 k 2023-10-09T10:28:10.554 INFO:teuthology.orchestra.run.smithi155.stdout: xmlstarlet x86_64 1.6.1-20.el8 appstream 71 k 2023-10-09T10:28:10.555 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:10.555 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:28:10.555 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:28:10.555 INFO:teuthology.orchestra.run.smithi155.stdout:Install 6 Packages 2023-10-09T10:28:10.555 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:10.556 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 53 M 2023-10-09T10:28:10.556 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 233 M 2023-10-09T10:28:10.556 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:28:12.561 INFO:teuthology.orchestra.run.smithi155.stdout:(1/6): jq-1.6-7.el8.x86_64.rpm 396 kB/s | 202 kB 00:00 2023-10-09T10:28:12.586 INFO:teuthology.orchestra.run.smithi155.stdout:(2/6): libcephsqlite-17.2.6-1303.gb2903cb0.el8. 326 kB/s | 175 kB 00:00 2023-10-09T10:28:12.712 INFO:teuthology.orchestra.run.smithi155.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 1.2 MB/s | 187 kB 00:00 2023-10-09T10:28:12.829 INFO:teuthology.orchestra.run.smithi155.stdout:(4/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 605 kB/s | 71 kB 00:00 2023-10-09T10:28:13.088 INFO:teuthology.orchestra.run.smithi125.stdout:(6/6): ceph-test-17.2.6-1303.gb2903cb0.el8.x86_ 14 MB/s | 52 MB 00:03 2023-10-09T10:28:13.089 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:28:13.090 INFO:teuthology.orchestra.run.smithi125.stdout:Total 11 MB/s | 53 MB 00:04 2023-10-09T10:28:13.133 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:28:13.137 INFO:teuthology.orchestra.run.smithi155.stdout:(5/6): socat-1.7.4.1-1.el8.x86_64.rpm 587 kB/s | 323 kB 00:00 2023-10-09T10:28:13.154 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:28:13.154 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:28:13.442 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:28:13.445 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:28:13.959 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:28:14.167 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2023-10-09T10:28:14.379 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2023-10-09T10:28:14.527 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2023-10-09T10:28:14.609 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2023-10-09T10:28:14.809 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : jq-1.6-7.el8.x86_64 4/6 2023-10-09T10:28:15.002 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T10:28:15.396 INFO:teuthology.orchestra.run.smithi155.stdout:(6/6): ceph-test-17.2.6-1303.gb2903cb0.el8.x86_ 15 MB/s | 52 MB 00:03 2023-10-09T10:28:15.397 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:28:15.398 INFO:teuthology.orchestra.run.smithi155.stdout:Total 11 MB/s | 53 MB 00:04 2023-10-09T10:28:15.442 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:28:15.466 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:28:15.466 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:28:15.749 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:28:15.752 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:28:16.230 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:28:16.495 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2023-10-09T10:28:16.708 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2023-10-09T10:28:16.848 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2023-10-09T10:28:16.925 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2023-10-09T10:28:17.146 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : jq-1.6-7.el8.x86_64 4/6 2023-10-09T10:28:17.356 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T10:28:19.986 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T10:28:20.258 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T10:28:20.533 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T10:28:20.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/6 2023-10-09T10:28:20.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/6 2023-10-09T10:28:20.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : jq-1.6-7.el8.x86_64 3/6 2023-10-09T10:28:20.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2023-10-09T10:28:20.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2023-10-09T10:28:21.188 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2023-10-09T10:28:21.188 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:21.188 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:28:21.189 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:21.189 INFO:teuthology.orchestra.run.smithi125.stdout: jq-1.6-7.el8.x86_64 2023-10-09T10:28:21.189 INFO:teuthology.orchestra.run.smithi125.stdout: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:21.189 INFO:teuthology.orchestra.run.smithi125.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-09T10:28:21.189 INFO:teuthology.orchestra.run.smithi125.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-09T10:28:21.189 INFO:teuthology.orchestra.run.smithi125.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-09T10:28:21.189 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:21.190 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:28:21.438 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install ceph 2023-10-09T10:28:22.308 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:01:00 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:28:22.321 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T10:28:22.562 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T10:28:22.848 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T10:28:22.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/6 2023-10-09T10:28:22.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/6 2023-10-09T10:28:22.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : jq-1.6-7.el8.x86_64 3/6 2023-10-09T10:28:22.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2023-10-09T10:28:22.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2023-10-09T10:28:23.529 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2023-10-09T10:28:23.529 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:23.529 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:28:23.529 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:23.529 INFO:teuthology.orchestra.run.smithi155.stdout: jq-1.6-7.el8.x86_64 2023-10-09T10:28:23.529 INFO:teuthology.orchestra.run.smithi155.stdout: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:23.529 INFO:teuthology.orchestra.run.smithi155.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-09T10:28:23.530 INFO:teuthology.orchestra.run.smithi155.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-09T10:28:23.530 INFO:teuthology.orchestra.run.smithi155.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-09T10:28:23.530 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:23.530 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:28:23.740 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install ceph 2023-10-09T10:28:23.891 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:28:23.894 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:28:23.894 INFO:teuthology.orchestra.run.smithi125.stdout: Package Arch Version Repository Size 2023-10-09T10:28:23.894 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:28:23.895 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:28:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: ceph x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 6.4 k 2023-10-09T10:28:23.895 INFO:teuthology.orchestra.run.smithi125.stdout:Installing dependencies: 2023-10-09T10:28:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mds x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 2.3 M 2023-10-09T10:28:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mgr x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 1.6 M 2023-10-09T10:28:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 238 k 2023-10-09T10:28:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mon x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.9 M 2023-10-09T10:28:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-osd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T10:28:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-09T10:28:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-09T10:28:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: python3-cffi x86_64 1.11.5-6.el8 baseos 249 k 2023-10-09T10:28:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-09T10:28:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-09T10:28:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: python3-cryptography x86_64 3.2.1-6.el8 baseos 641 k 2023-10-09T10:28:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-09T10:28:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-09T10:28:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-09T10:28:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: python3-mako noarch 1.0.6-14.el8 appstream 157 k 2023-10-09T10:28:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: python3-markupsafe x86_64 0.23-19.el8 appstream 39 k 2023-10-09T10:28:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-09T10:28:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-09T10:28:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-09T10:28:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 appstream 103 k 2023-10-09T10:28:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pycparser noarch 2.14-14.el8 baseos 109 k 2023-10-09T10:28:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pytz noarch 2017.2-11.el8 appstream 57 k 2023-10-09T10:28:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-09T10:28:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-09T10:28:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-09T10:28:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-09T10:28:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-09T10:28:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-09T10:28:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-09T10:28:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: python3-werkzeug noarch 0.12.2-4.el8 appstream 457 k 2023-10-09T10:28:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-09T10:28:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:23.897 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:28:23.898 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:28:23.898 INFO:teuthology.orchestra.run.smithi125.stdout:Install 32 Packages 2023-10-09T10:28:23.898 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:23.898 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 28 M 2023-10-09T10:28:23.898 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 110 M 2023-10-09T10:28:23.898 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:28:24.432 INFO:teuthology.orchestra.run.smithi125.stdout:(1/32): ceph-17.2.6-1303.gb2903cb0.el8.x86_64.r 50 kB/s | 6.4 kB 00:00 2023-10-09T10:28:24.574 INFO:teuthology.orchestra.run.smithi125.stdout:(2/32): ceph-mgr-17.2.6-1303.gb2903cb0.el8.x86_ 5.8 MB/s | 1.6 MB 00:00 2023-10-09T10:28:24.599 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:00:59 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:28:25.000 INFO:teuthology.orchestra.run.smithi125.stdout:(3/32): ceph-mon-17.2.6-1303.gb2903cb0.el8.x86_ 6.9 MB/s | 3.9 MB 00:00 2023-10-09T10:28:25.083 INFO:teuthology.orchestra.run.smithi125.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1303.gb290 2.8 MB/s | 238 kB 00:00 2023-10-09T10:28:25.401 INFO:teuthology.orchestra.run.smithi125.stdout:(5/32): ceph-mds-17.2.6-1303.gb2903cb0.el8.x86_ 2.1 MB/s | 2.3 MB 00:01 2023-10-09T10:28:25.626 INFO:teuthology.orchestra.run.smithi125.stdout:(6/32): ceph-osd-17.2.6-1303.gb2903cb0.el8.x86_ 16 MB/s | 17 MB 00:01 2023-10-09T10:28:25.651 INFO:teuthology.orchestra.run.smithi125.stdout:(7/32): python3-mako-1.0.6-14.el8.noarch.rpm 277 kB/s | 157 kB 00:00 2023-10-09T10:28:25.776 INFO:teuthology.orchestra.run.smithi125.stdout:(8/32): python3-pytz-2017.2-11.el8.noarch.rpm 456 kB/s | 57 kB 00:00 2023-10-09T10:28:25.852 INFO:teuthology.orchestra.run.smithi125.stdout:(9/32): python3-markupsafe-0.23-19.el8.x86_64.r 86 kB/s | 39 kB 00:00 2023-10-09T10:28:26.052 INFO:teuthology.orchestra.run.smithi125.stdout:(10/32): python3-werkzeug-0.12.2-4.el8.noarch.r 1.6 MB/s | 457 kB 00:00 2023-10-09T10:28:26.077 INFO:teuthology.orchestra.run.smithi125.stdout:(11/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 229 kB/s | 103 kB 00:00 2023-10-09T10:28:26.195 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:28:26.198 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:28:26.198 INFO:teuthology.orchestra.run.smithi155.stdout: Package Arch Version Repository Size 2023-10-09T10:28:26.199 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:28:26.199 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:28:26.199 INFO:teuthology.orchestra.run.smithi155.stdout: ceph x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 6.4 k 2023-10-09T10:28:26.199 INFO:teuthology.orchestra.run.smithi155.stdout:Installing dependencies: 2023-10-09T10:28:26.199 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mds x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 2.3 M 2023-10-09T10:28:26.199 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mgr x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 1.6 M 2023-10-09T10:28:26.199 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 238 k 2023-10-09T10:28:26.199 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mon x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.9 M 2023-10-09T10:28:26.199 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-osd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T10:28:26.200 INFO:teuthology.orchestra.run.smithi155.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-09T10:28:26.200 INFO:teuthology.orchestra.run.smithi155.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-09T10:28:26.200 INFO:teuthology.orchestra.run.smithi155.stdout: python3-cffi x86_64 1.11.5-6.el8 baseos 249 k 2023-10-09T10:28:26.200 INFO:teuthology.orchestra.run.smithi155.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-09T10:28:26.200 INFO:teuthology.orchestra.run.smithi155.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-09T10:28:26.200 INFO:teuthology.orchestra.run.smithi155.stdout: python3-cryptography x86_64 3.2.1-6.el8 baseos 641 k 2023-10-09T10:28:26.200 INFO:teuthology.orchestra.run.smithi155.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-09T10:28:26.200 INFO:teuthology.orchestra.run.smithi155.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-09T10:28:26.200 INFO:teuthology.orchestra.run.smithi155.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-09T10:28:26.201 INFO:teuthology.orchestra.run.smithi155.stdout: python3-mako noarch 1.0.6-14.el8 appstream 157 k 2023-10-09T10:28:26.201 INFO:teuthology.orchestra.run.smithi155.stdout: python3-markupsafe x86_64 0.23-19.el8 appstream 39 k 2023-10-09T10:28:26.201 INFO:teuthology.orchestra.run.smithi155.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-09T10:28:26.201 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-09T10:28:26.201 INFO:teuthology.orchestra.run.smithi155.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-09T10:28:26.201 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 appstream 103 k 2023-10-09T10:28:26.201 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pycparser noarch 2.14-14.el8 baseos 109 k 2023-10-09T10:28:26.201 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pytz noarch 2017.2-11.el8 appstream 57 k 2023-10-09T10:28:26.201 INFO:teuthology.orchestra.run.smithi155.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-09T10:28:26.202 INFO:teuthology.orchestra.run.smithi155.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-09T10:28:26.202 INFO:teuthology.orchestra.run.smithi155.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-09T10:28:26.202 INFO:teuthology.orchestra.run.smithi155.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-09T10:28:26.202 INFO:teuthology.orchestra.run.smithi155.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-09T10:28:26.202 INFO:teuthology.orchestra.run.smithi155.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-09T10:28:26.202 INFO:teuthology.orchestra.run.smithi155.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-09T10:28:26.202 INFO:teuthology.orchestra.run.smithi155.stdout: python3-werkzeug noarch 0.12.2-4.el8 appstream 457 k 2023-10-09T10:28:26.202 INFO:teuthology.orchestra.run.smithi155.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-09T10:28:26.202 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:26.203 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:28:26.203 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:28:26.203 INFO:teuthology.orchestra.run.smithi155.stdout:Install 32 Packages 2023-10-09T10:28:26.203 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:26.204 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 28 M 2023-10-09T10:28:26.204 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 110 M 2023-10-09T10:28:26.204 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:28:26.511 INFO:teuthology.orchestra.run.smithi125.stdout:(12/32): python3-cffi-1.11.5-6.el8.x86_64.rpm 377 kB/s | 249 kB 00:00 2023-10-09T10:28:26.594 INFO:teuthology.orchestra.run.smithi125.stdout:(13/32): python3-pycparser-2.14-14.el8.noarch.r 211 kB/s | 109 kB 00:00 2023-10-09T10:28:26.658 INFO:teuthology.orchestra.run.smithi155.stdout:(1/32): ceph-17.2.6-1303.gb2903cb0.el8.x86_64.r 33 kB/s | 6.4 kB 00:00 2023-10-09T10:28:26.769 INFO:teuthology.orchestra.run.smithi125.stdout:(14/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 169 kB/s | 44 kB 00:00 2023-10-09T10:28:26.783 INFO:teuthology.orchestra.run.smithi155.stdout:(2/32): ceph-mgr-17.2.6-1303.gb2903cb0.el8.x86_ 5.0 MB/s | 1.6 MB 00:00 2023-10-09T10:28:26.945 INFO:teuthology.orchestra.run.smithi125.stdout:(15/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 529 kB/s | 185 kB 00:00 2023-10-09T10:28:26.970 INFO:teuthology.orchestra.run.smithi125.stdout:(16/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 862 kB/s | 173 kB 00:00 2023-10-09T10:28:27.009 INFO:teuthology.orchestra.run.smithi155.stdout:(3/32): ceph-mds-17.2.6-1303.gb2903cb0.el8.x86_ 4.2 MB/s | 2.3 MB 00:00 2023-10-09T10:28:27.020 INFO:teuthology.orchestra.run.smithi125.stdout:(17/32): python3-cryptography-3.2.1-6.el8.x86_6 662 kB/s | 641 kB 00:00 2023-10-09T10:28:27.037 INFO:teuthology.orchestra.run.smithi125.stdout:(18/32): python3-jaraco-6.2-6.el8.noarch.rpm 166 kB/s | 11 kB 00:00 2023-10-09T10:28:27.096 INFO:teuthology.orchestra.run.smithi125.stdout:(19/32): python3-cherrypy-18.4.0-1.el8.noarch.r 2.5 MB/s | 384 kB 00:00 2023-10-09T10:28:27.109 INFO:teuthology.orchestra.run.smithi155.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1303.gb290 2.3 MB/s | 238 kB 00:00 2023-10-09T10:28:27.113 INFO:teuthology.orchestra.run.smithi125.stdout:(20/32): python3-jaraco-functools-2.0-4.el8.noa 191 kB/s | 18 kB 00:00 2023-10-09T10:28:27.171 INFO:teuthology.orchestra.run.smithi125.stdout:(21/32): python3-more-itertools-7.2.0-3.el8.noa 830 kB/s | 59 kB 00:00 2023-10-09T10:28:27.247 INFO:teuthology.orchestra.run.smithi125.stdout:(22/32): python3-pecan-1.3.2-9.el8.noarch.rpm 2.1 MB/s | 283 kB 00:00 2023-10-09T10:28:27.264 INFO:teuthology.orchestra.run.smithi125.stdout:(23/32): python3-portend-2.6-1.el8.noarch.rpm 169 kB/s | 16 kB 00:00 2023-10-09T10:28:27.322 INFO:teuthology.orchestra.run.smithi125.stdout:(24/32): python3-logutils-0.3.5-11.el8.noarch.r 171 kB/s | 49 kB 00:00 2023-10-09T10:28:27.339 INFO:teuthology.orchestra.run.smithi125.stdout:(25/32): python3-simplegeneric-0.8.1-17.el8.noa 203 kB/s | 19 kB 00:00 2023-10-09T10:28:27.356 INFO:teuthology.orchestra.run.smithi125.stdout:(26/32): python3-singledispatch-3.4.0.3-18.el8. 259 kB/s | 24 kB 00:00 2023-10-09T10:28:27.415 INFO:teuthology.orchestra.run.smithi125.stdout:(27/32): python3-tempora-1.14.1-5.el8.noarch.rp 318 kB/s | 29 kB 00:00 2023-10-09T10:28:27.432 INFO:teuthology.orchestra.run.smithi125.stdout:(28/32): python3-trustme-0.6.0-4.el8.noarch.rpm 367 kB/s | 27 kB 00:00 2023-10-09T10:28:27.457 INFO:teuthology.orchestra.run.smithi125.stdout:(29/32): python3-waitress-1.4.3-1.el8.noarch.rp 2.5 MB/s | 255 kB 00:00 2023-10-09T10:28:27.507 INFO:teuthology.orchestra.run.smithi125.stdout:(30/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.1 MB/s | 85 kB 00:00 2023-10-09T10:28:27.524 INFO:teuthology.orchestra.run.smithi125.stdout:(31/32): python3-zc-lockfile-2.0-2.el8.noarch.r 343 kB/s | 23 kB 00:00 2023-10-09T10:28:27.633 INFO:teuthology.orchestra.run.smithi125.stdout:(32/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.1 MB/s | 251 kB 00:00 2023-10-09T10:28:27.637 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:28:27.637 INFO:teuthology.orchestra.run.smithi125.stdout:Total 7.6 MB/s | 28 MB 00:03 2023-10-09T10:28:27.709 INFO:teuthology.orchestra.run.smithi155.stdout:(5/32): ceph-osd-17.2.6-1303.gb2903cb0.el8.x86_ 18 MB/s | 17 MB 00:00 2023-10-09T10:28:27.720 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:28:27.735 INFO:teuthology.orchestra.run.smithi155.stdout:(6/32): python3-mako-1.0.6-14.el8.noarch.rpm 251 kB/s | 157 kB 00:00 2023-10-09T10:28:27.757 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:28:27.757 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:28:27.877 INFO:teuthology.orchestra.run.smithi155.stdout:(7/32): python3-pyOpenSSL-19.0.0-1.el8.noarch.r 731 kB/s | 103 kB 00:00 2023-10-09T10:28:27.935 INFO:teuthology.orchestra.run.smithi155.stdout:(8/32): ceph-mon-17.2.6-1303.gb2903cb0.el8.x86_ 3.1 MB/s | 3.9 MB 00:01 2023-10-09T10:28:28.011 INFO:teuthology.orchestra.run.smithi155.stdout:(9/32): python3-pytz-2017.2-11.el8.noarch.rpm 427 kB/s | 57 kB 00:00 2023-10-09T10:28:28.036 INFO:teuthology.orchestra.run.smithi155.stdout:(10/32): python3-markupsafe-0.23-19.el8.x86_64. 119 kB/s | 39 kB 00:00 2023-10-09T10:28:28.073 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:28:28.076 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:28:28.553 INFO:teuthology.orchestra.run.smithi155.stdout:(11/32): python3-werkzeug-0.12.2-4.el8.noarch.r 741 kB/s | 457 kB 00:00 2023-10-09T10:28:28.703 INFO:teuthology.orchestra.run.smithi155.stdout:(12/32): python3-cffi-1.11.5-6.el8.x86_64.rpm 359 kB/s | 249 kB 00:00 2023-10-09T10:28:28.870 INFO:teuthology.orchestra.run.smithi155.stdout:(13/32): python3-cryptography-3.2.1-6.el8.x86_6 769 kB/s | 641 kB 00:00 2023-10-09T10:28:28.954 INFO:teuthology.orchestra.run.smithi155.stdout:(14/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 174 kB/s | 44 kB 00:00 2023-10-09T10:28:29.098 INFO:teuthology.orchestra.run.smithi155.stdout:(15/32): python3-pycparser-2.14-14.el8.noarch.r 200 kB/s | 109 kB 00:00 2023-10-09T10:28:29.155 INFO:teuthology.orchestra.run.smithi155.stdout:(16/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 861 kB/s | 173 kB 00:00 2023-10-09T10:28:29.230 INFO:teuthology.orchestra.run.smithi155.stdout:(17/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 515 kB/s | 185 kB 00:00 2023-10-09T10:28:29.248 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:28:29.257 INFO:teuthology.orchestra.run.smithi155.stdout:(18/32): python3-jaraco-6.2-6.el8.noarch.rpm 116 kB/s | 11 kB 00:00 2023-10-09T10:28:29.306 INFO:teuthology.orchestra.run.smithi155.stdout:(19/32): python3-jaraco-functools-2.0-4.el8.noa 235 kB/s | 18 kB 00:00 2023-10-09T10:28:29.342 INFO:teuthology.orchestra.run.smithi155.stdout:(20/32): python3-logutils-0.3.5-11.el8.noarch.r 582 kB/s | 49 kB 00:00 2023-10-09T10:28:29.390 INFO:teuthology.orchestra.run.smithi155.stdout:(21/32): python3-more-itertools-7.2.0-3.el8.noa 705 kB/s | 59 kB 00:00 2023-10-09T10:28:29.468 INFO:teuthology.orchestra.run.smithi155.stdout:(22/32): python3-portend-2.6-1.el8.noarch.rpm 208 kB/s | 16 kB 00:00 2023-10-09T10:28:29.493 INFO:teuthology.orchestra.run.smithi155.stdout:(23/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.8 MB/s | 283 kB 00:00 2023-10-09T10:28:29.514 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-09T10:28:29.550 INFO:teuthology.orchestra.run.smithi155.stdout:(24/32): python3-simplegeneric-0.8.1-17.el8.noa 224 kB/s | 19 kB 00:00 2023-10-09T10:28:29.575 INFO:teuthology.orchestra.run.smithi155.stdout:(25/32): python3-cherrypy-18.4.0-1.el8.noarch.r 808 kB/s | 384 kB 00:00 2023-10-09T10:28:29.601 INFO:teuthology.orchestra.run.smithi155.stdout:(26/32): python3-singledispatch-3.4.0.3-18.el8. 220 kB/s | 24 kB 00:00 2023-10-09T10:28:29.625 INFO:teuthology.orchestra.run.smithi155.stdout:(27/32): python3-tempora-1.14.1-5.el8.noarch.rp 388 kB/s | 29 kB 00:00 2023-10-09T10:28:29.681 INFO:teuthology.orchestra.run.smithi155.stdout:(28/32): python3-trustme-0.6.0-4.el8.noarch.rpm 367 kB/s | 27 kB 00:00 2023-10-09T10:28:29.705 INFO:teuthology.orchestra.run.smithi155.stdout:(29/32): python3-waitress-1.4.3-1.el8.noarch.rp 2.5 MB/s | 255 kB 00:00 2023-10-09T10:28:29.751 INFO:teuthology.orchestra.run.smithi155.stdout:(30/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.0 MB/s | 251 kB 00:00 2023-10-09T10:28:29.765 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-09T10:28:29.776 INFO:teuthology.orchestra.run.smithi155.stdout:(31/32): python3-zc-lockfile-2.0-2.el8.noarch.r 304 kB/s | 23 kB 00:00 2023-10-09T10:28:29.820 INFO:teuthology.orchestra.run.smithi155.stdout:(32/32): python3-webtest-2.0.33-1.el8.noarch.rp 681 kB/s | 85 kB 00:00 2023-10-09T10:28:29.820 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:28:29.820 INFO:teuthology.orchestra.run.smithi155.stdout:Total 7.9 MB/s | 28 MB 00:03 2023-10-09T10:28:29.895 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:28:29.930 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:28:29.930 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:28:29.997 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2023-10-09T10:28:30.228 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:28:30.232 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:28:30.264 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 4/32 2023-10-09T10:28:30.495 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2023-10-09T10:28:30.718 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2023-10-09T10:28:30.895 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2023-10-09T10:28:31.114 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2023-10-09T10:28:31.329 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:28:31.363 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2023-10-09T10:28:31.535 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-09T10:28:31.609 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2023-10-09T10:28:31.787 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-09T10:28:31.814 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2023-10-09T10:28:32.031 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2023-10-09T10:28:32.087 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2023-10-09T10:28:32.277 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 4/32 2023-10-09T10:28:32.475 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2023-10-09T10:28:32.510 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-cffi-1.11.5-6.el8.x86_64 13/32 2023-10-09T10:28:32.655 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2023-10-09T10:28:32.814 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-cryptography-3.2.1-6.el8.x86_64 14/32 2023-10-09T10:28:32.857 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2023-10-09T10:28:32.990 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2023-10-09T10:28:33.051 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2023-10-09T10:28:33.245 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2023-10-09T10:28:33.290 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2023-10-09T10:28:33.469 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2023-10-09T10:28:33.533 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-10-09T10:28:33.694 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2023-10-09T10:28:33.865 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2023-10-09T10:28:33.960 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2023-10-09T10:28:34.104 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2023-10-09T10:28:34.317 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-pytz-2017.2-11.el8.noarch 20/32 2023-10-09T10:28:34.386 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-cffi-1.11.5-6.el8.x86_64 13/32 2023-10-09T10:28:34.536 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-09T10:28:34.651 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-cryptography-3.2.1-6.el8.x86_64 14/32 2023-10-09T10:28:34.788 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-09T10:28:34.820 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2023-10-09T10:28:35.046 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-09T10:28:35.078 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2023-10-09T10:28:35.287 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-10-09T10:28:35.300 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2023-10-09T10:28:35.588 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2023-10-09T10:28:35.645 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-mako-1.0.6-14.el8.noarch 25/32 2023-10-09T10:28:35.817 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2023-10-09T10:28:35.961 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-09T10:28:35.964 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-pytz-2017.2-11.el8.noarch 20/32 2023-10-09T10:28:36.141 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-09T10:28:36.349 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 27/32 2023-10-09T10:28:36.393 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-09T10:28:36.642 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-09T10:28:36.847 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2023-10-09T10:28:37.180 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-mako-1.0.6-14.el8.noarch 25/32 2023-10-09T10:28:37.220 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T10:28:37.484 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-09T10:28:37.892 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 27/32 2023-10-09T10:28:38.153 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T10:28:38.708 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T10:28:38.936 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T10:28:39.344 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T10:28:39.565 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T10:28:39.645 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T10:28:39.782 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T10:28:39.819 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T10:28:39.948 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T10:28:39.966 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T10:28:40.131 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T10:28:40.213 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T10:28:40.394 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T10:28:40.533 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T10:28:40.533 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/32 2023-10-09T10:28:40.533 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/32 2023-10-09T10:28:40.533 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/32 2023-10-09T10:28:40.533 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/32 2023-10-09T10:28:40.533 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/32 2023-10-09T10:28:40.533 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 6/32 2023-10-09T10:28:40.533 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-mako-1.0.6-14.el8.noarch 7/32 2023-10-09T10:28:40.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2023-10-09T10:28:40.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2023-10-09T10:28:40.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-pytz-2017.2-11.el8.noarch 10/32 2023-10-09T10:28:40.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2023-10-09T10:28:40.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-cffi-1.11.5-6.el8.x86_64 12/32 2023-10-09T10:28:40.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-cryptography-3.2.1-6.el8.x86_64 13/32 2023-10-09T10:28:40.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2023-10-09T10:28:40.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2023-10-09T10:28:40.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2023-10-09T10:28:40.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-10-09T10:28:40.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2023-10-09T10:28:40.534 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2023-10-09T10:28:40.535 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2023-10-09T10:28:40.535 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2023-10-09T10:28:40.535 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2023-10-09T10:28:40.535 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2023-10-09T10:28:40.535 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2023-10-09T10:28:40.535 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2023-10-09T10:28:40.535 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2023-10-09T10:28:40.535 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2023-10-09T10:28:40.535 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2023-10-09T10:28:40.535 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 29/32 2023-10-09T10:28:40.536 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2023-10-09T10:28:40.536 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2023-10-09T10:28:40.611 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T10:28:40.811 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T10:28:40.829 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T10:28:40.960 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T10:28:41.358 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T10:28:41.358 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/32 2023-10-09T10:28:41.358 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/32 2023-10-09T10:28:41.359 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/32 2023-10-09T10:28:41.359 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/32 2023-10-09T10:28:41.359 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/32 2023-10-09T10:28:41.359 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 6/32 2023-10-09T10:28:41.359 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-mako-1.0.6-14.el8.noarch 7/32 2023-10-09T10:28:41.359 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2023-10-09T10:28:41.359 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2023-10-09T10:28:41.359 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-pytz-2017.2-11.el8.noarch 10/32 2023-10-09T10:28:41.359 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2023-10-09T10:28:41.359 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-cffi-1.11.5-6.el8.x86_64 12/32 2023-10-09T10:28:41.359 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-cryptography-3.2.1-6.el8.x86_64 13/32 2023-10-09T10:28:41.360 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2023-10-09T10:28:41.360 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2023-10-09T10:28:41.360 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2023-10-09T10:28:41.360 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-10-09T10:28:41.360 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2023-10-09T10:28:41.360 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2023-10-09T10:28:41.360 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2023-10-09T10:28:41.360 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2023-10-09T10:28:41.360 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2023-10-09T10:28:41.360 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2023-10-09T10:28:41.360 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2023-10-09T10:28:41.360 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2023-10-09T10:28:41.361 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2023-10-09T10:28:41.361 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2023-10-09T10:28:41.361 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2023-10-09T10:28:41.361 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 29/32 2023-10-09T10:28:41.361 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2023-10-09T10:28:41.361 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2023-10-09T10:28:41.727 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2023-10-09T10:28:41.727 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:41.727 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:28:41.727 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:41.727 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:41.727 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:41.727 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:28:41.727 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:41.728 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:41.728 INFO:teuthology.orchestra.run.smithi125.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-09T10:28:41.728 INFO:teuthology.orchestra.run.smithi125.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-09T10:28:41.728 INFO:teuthology.orchestra.run.smithi125.stdout: python3-cffi-1.11.5-6.el8.x86_64 2023-10-09T10:28:41.728 INFO:teuthology.orchestra.run.smithi125.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-09T10:28:41.728 INFO:teuthology.orchestra.run.smithi125.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-09T10:28:41.728 INFO:teuthology.orchestra.run.smithi125.stdout: python3-cryptography-3.2.1-6.el8.x86_64 2023-10-09T10:28:41.728 INFO:teuthology.orchestra.run.smithi125.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-09T10:28:41.728 INFO:teuthology.orchestra.run.smithi125.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-09T10:28:41.728 INFO:teuthology.orchestra.run.smithi125.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-09T10:28:41.728 INFO:teuthology.orchestra.run.smithi125.stdout: python3-mako-1.0.6-14.el8.noarch 2023-10-09T10:28:41.728 INFO:teuthology.orchestra.run.smithi125.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-09T10:28:41.729 INFO:teuthology.orchestra.run.smithi125.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-09T10:28:41.729 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-09T10:28:41.729 INFO:teuthology.orchestra.run.smithi125.stdout: python3-portend-2.6-1.el8.noarch 2023-10-09T10:28:41.729 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-09T10:28:41.729 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-09T10:28:41.729 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pytz-2017.2-11.el8.noarch 2023-10-09T10:28:41.729 INFO:teuthology.orchestra.run.smithi125.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-09T10:28:41.729 INFO:teuthology.orchestra.run.smithi125.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-09T10:28:41.729 INFO:teuthology.orchestra.run.smithi125.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-09T10:28:41.729 INFO:teuthology.orchestra.run.smithi125.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-09T10:28:41.729 INFO:teuthology.orchestra.run.smithi125.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-09T10:28:41.729 INFO:teuthology.orchestra.run.smithi125.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-09T10:28:41.730 INFO:teuthology.orchestra.run.smithi125.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-09T10:28:41.730 INFO:teuthology.orchestra.run.smithi125.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-09T10:28:41.730 INFO:teuthology.orchestra.run.smithi125.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-09T10:28:41.730 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:41.730 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:28:42.055 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install ceph-base 2023-10-09T10:28:42.420 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2023-10-09T10:28:42.420 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:42.420 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:28:42.421 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:42.421 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:42.421 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:42.421 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:28:42.421 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:42.421 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:42.421 INFO:teuthology.orchestra.run.smithi155.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-09T10:28:42.421 INFO:teuthology.orchestra.run.smithi155.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-09T10:28:42.421 INFO:teuthology.orchestra.run.smithi155.stdout: python3-cffi-1.11.5-6.el8.x86_64 2023-10-09T10:28:42.421 INFO:teuthology.orchestra.run.smithi155.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-09T10:28:42.421 INFO:teuthology.orchestra.run.smithi155.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-09T10:28:42.422 INFO:teuthology.orchestra.run.smithi155.stdout: python3-cryptography-3.2.1-6.el8.x86_64 2023-10-09T10:28:42.422 INFO:teuthology.orchestra.run.smithi155.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-09T10:28:42.422 INFO:teuthology.orchestra.run.smithi155.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-09T10:28:42.422 INFO:teuthology.orchestra.run.smithi155.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-09T10:28:42.422 INFO:teuthology.orchestra.run.smithi155.stdout: python3-mako-1.0.6-14.el8.noarch 2023-10-09T10:28:42.422 INFO:teuthology.orchestra.run.smithi155.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-09T10:28:42.422 INFO:teuthology.orchestra.run.smithi155.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-09T10:28:42.422 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-09T10:28:42.422 INFO:teuthology.orchestra.run.smithi155.stdout: python3-portend-2.6-1.el8.noarch 2023-10-09T10:28:42.422 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-09T10:28:42.422 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-09T10:28:42.422 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pytz-2017.2-11.el8.noarch 2023-10-09T10:28:42.423 INFO:teuthology.orchestra.run.smithi155.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-09T10:28:42.423 INFO:teuthology.orchestra.run.smithi155.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-09T10:28:42.423 INFO:teuthology.orchestra.run.smithi155.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-09T10:28:42.423 INFO:teuthology.orchestra.run.smithi155.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-09T10:28:42.423 INFO:teuthology.orchestra.run.smithi155.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-09T10:28:42.423 INFO:teuthology.orchestra.run.smithi155.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-09T10:28:42.423 INFO:teuthology.orchestra.run.smithi155.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-09T10:28:42.423 INFO:teuthology.orchestra.run.smithi155.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-09T10:28:42.423 INFO:teuthology.orchestra.run.smithi155.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-09T10:28:42.423 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:42.423 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:28:42.675 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install ceph-base 2023-10-09T10:28:42.911 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:01:20 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:28:43.538 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:01:18 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:28:44.303 INFO:teuthology.orchestra.run.smithi125.stdout:Package ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:28:44.496 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:28:44.497 INFO:teuthology.orchestra.run.smithi125.stdout:Nothing to do. 2023-10-09T10:28:44.497 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:28:44.618 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install cephadm 2023-10-09T10:28:44.926 INFO:teuthology.orchestra.run.smithi155.stdout:Package ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:28:45.113 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:28:45.114 INFO:teuthology.orchestra.run.smithi155.stdout:Nothing to do. 2023-10-09T10:28:45.114 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:28:45.218 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install cephadm 2023-10-09T10:28:45.470 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:01:23 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:28:46.079 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:01:21 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:28:47.060 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:28:47.061 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:28:47.061 INFO:teuthology.orchestra.run.smithi125.stdout: Package Arch Version Repository Size 2023-10-09T10:28:47.061 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:28:47.061 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:28:47.061 INFO:teuthology.orchestra.run.smithi125.stdout: cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 80 k 2023-10-09T10:28:47.062 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:47.062 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:28:47.062 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:28:47.062 INFO:teuthology.orchestra.run.smithi125.stdout:Install 1 Package 2023-10-09T10:28:47.062 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:47.062 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 80 k 2023-10-09T10:28:47.062 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 364 k 2023-10-09T10:28:47.062 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:28:47.661 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:28:47.662 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:28:47.662 INFO:teuthology.orchestra.run.smithi155.stdout: Package Arch Version Repository Size 2023-10-09T10:28:47.662 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:28:47.662 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:28:47.662 INFO:teuthology.orchestra.run.smithi155.stdout: cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 80 k 2023-10-09T10:28:47.663 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:47.663 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:28:47.663 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:28:47.663 INFO:teuthology.orchestra.run.smithi155.stdout:Install 1 Package 2023-10-09T10:28:47.663 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:47.663 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 80 k 2023-10-09T10:28:47.663 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 364 k 2023-10-09T10:28:47.663 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:28:47.841 INFO:teuthology.orchestra.run.smithi125.stdout:cephadm-17.2.6-1303.gb2903cb0.el8.noarch.rpm 103 kB/s | 80 kB 00:00 2023-10-09T10:28:47.842 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:28:47.842 INFO:teuthology.orchestra.run.smithi125.stdout:Total 103 kB/s | 80 kB 00:00 2023-10-09T10:28:47.843 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:28:47.851 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:28:47.851 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:28:47.853 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:28:47.857 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:28:47.920 INFO:teuthology.orchestra.run.smithi155.stdout:cephadm-17.2.6-1303.gb2903cb0.el8.noarch.rpm 312 kB/s | 80 kB 00:00 2023-10-09T10:28:47.921 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:28:47.921 INFO:teuthology.orchestra.run.smithi155.stdout:Total 311 kB/s | 80 kB 00:00 2023-10-09T10:28:47.921 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:28:47.929 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:28:47.929 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:28:47.930 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:28:47.934 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:28:47.982 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:28:48.459 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:28:48.489 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T10:28:48.655 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T10:28:48.884 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T10:28:49.382 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T10:28:49.510 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T10:28:49.510 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:49.510 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:28:49.510 INFO:teuthology.orchestra.run.smithi155.stdout: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:28:49.510 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:49.510 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:28:49.581 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T10:28:49.821 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T10:28:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T10:28:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:50.469 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:28:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:28:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:50.470 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:28:50.679 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install ceph-immutable-object-cache 2023-10-09T10:28:51.295 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install ceph-immutable-object-cache 2023-10-09T10:28:51.540 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:01:29 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:28:52.164 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:01:27 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:28:53.130 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:28:53.130 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:28:53.130 INFO:teuthology.orchestra.run.smithi125.stdout: Package Arch Version Repo Size 2023-10-09T10:28:53.131 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:28:53.131 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:28:53.131 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 161 k 2023-10-09T10:28:53.131 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:53.131 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:28:53.131 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:28:53.131 INFO:teuthology.orchestra.run.smithi125.stdout:Install 1 Package 2023-10-09T10:28:53.131 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:53.132 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 161 k 2023-10-09T10:28:53.132 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 490 k 2023-10-09T10:28:53.132 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:28:53.320 INFO:teuthology.orchestra.run.smithi125.stdout:ceph-immutable-object-cache-17.2.6-1303.gb2903c 857 kB/s | 161 kB 00:00 2023-10-09T10:28:53.321 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:28:53.321 INFO:teuthology.orchestra.run.smithi125.stdout:Total 851 kB/s | 161 kB 00:00 2023-10-09T10:28:53.321 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:28:53.332 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:28:53.332 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:28:53.378 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:28:53.381 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:28:53.529 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:28:53.737 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T10:28:53.741 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:28:53.742 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:28:53.743 INFO:teuthology.orchestra.run.smithi155.stdout: Package Arch Version Repo Size 2023-10-09T10:28:53.743 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:28:53.743 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:28:53.743 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 161 k 2023-10-09T10:28:53.743 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:53.743 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:28:53.743 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:28:53.743 INFO:teuthology.orchestra.run.smithi155.stdout:Install 1 Package 2023-10-09T10:28:53.743 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:53.743 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 161 k 2023-10-09T10:28:53.743 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 490 k 2023-10-09T10:28:53.744 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:28:54.036 INFO:teuthology.orchestra.run.smithi155.stdout:ceph-immutable-object-cache-17.2.6-1303.gb2903c 551 kB/s | 161 kB 00:00 2023-10-09T10:28:54.037 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:28:54.037 INFO:teuthology.orchestra.run.smithi155.stdout:Total 549 kB/s | 161 kB 00:00 2023-10-09T10:28:54.037 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:28:54.048 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:28:54.048 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:28:54.094 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:28:54.097 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:28:54.157 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T10:28:54.200 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:28:54.409 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T10:28:54.841 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T10:28:55.084 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T10:28:55.084 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:55.084 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:28:55.084 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:55.084 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:28:55.085 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:28:55.301 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install ceph-mgr 2023-10-09T10:28:55.472 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T10:28:55.473 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:55.473 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:28:55.473 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:28:55.473 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:28:55.473 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:28:55.710 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install ceph-mgr 2023-10-09T10:28:56.158 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:01:34 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:28:56.575 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:01:31 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:28:57.544 INFO:teuthology.orchestra.run.smithi125.stdout:Package ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:28:57.732 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:28:57.733 INFO:teuthology.orchestra.run.smithi125.stdout:Nothing to do. 2023-10-09T10:28:57.733 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:28:57.879 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install ceph-mgr-dashboard 2023-10-09T10:28:57.958 INFO:teuthology.orchestra.run.smithi155.stdout:Package ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:28:58.148 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:28:58.149 INFO:teuthology.orchestra.run.smithi155.stdout:Nothing to do. 2023-10-09T10:28:58.149 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:28:58.254 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install ceph-mgr-dashboard 2023-10-09T10:28:58.735 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:01:36 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:28:59.106 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:01:34 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:29:00.301 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:29:00.302 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:00.303 INFO:teuthology.orchestra.run.smithi125.stdout: Package Arch Version Repository Size 2023-10-09T10:29:00.303 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:00.303 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:29:00.303 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 1.7 M 2023-10-09T10:29:00.303 INFO:teuthology.orchestra.run.smithi125.stdout:Installing dependencies: 2023-10-09T10:29:00.303 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 23 k 2023-10-09T10:29:00.303 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 14 k 2023-10-09T10:29:00.303 INFO:teuthology.orchestra.run.smithi125.stdout: python3-jwt noarch 1.6.1-2.el8 baseos 43 k 2023-10-09T10:29:00.303 INFO:teuthology.orchestra.run.smithi125.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-09T10:29:00.303 INFO:teuthology.orchestra.run.smithi125.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-09T10:29:00.303 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:00.304 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:29:00.304 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:00.304 INFO:teuthology.orchestra.run.smithi125.stdout:Install 6 Packages 2023-10-09T10:29:00.304 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:00.304 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 2.0 M 2023-10-09T10:29:00.304 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 8.0 M 2023-10-09T10:29:00.305 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:29:00.532 INFO:teuthology.orchestra.run.smithi125.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1303.gb29 160 kB/s | 23 kB 00:00 2023-10-09T10:29:00.640 INFO:teuthology.orchestra.run.smithi125.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1303.gb290 57 kB/s | 14 kB 00:00 2023-10-09T10:29:00.693 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:29:00.693 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:00.693 INFO:teuthology.orchestra.run.smithi155.stdout: Package Arch Version Repository Size 2023-10-09T10:29:00.694 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:00.694 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:29:00.694 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 1.7 M 2023-10-09T10:29:00.694 INFO:teuthology.orchestra.run.smithi155.stdout:Installing dependencies: 2023-10-09T10:29:00.694 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 23 k 2023-10-09T10:29:00.694 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 14 k 2023-10-09T10:29:00.694 INFO:teuthology.orchestra.run.smithi155.stdout: python3-jwt noarch 1.6.1-2.el8 baseos 43 k 2023-10-09T10:29:00.694 INFO:teuthology.orchestra.run.smithi155.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-09T10:29:00.695 INFO:teuthology.orchestra.run.smithi155.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-09T10:29:00.695 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:00.695 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:29:00.695 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:00.695 INFO:teuthology.orchestra.run.smithi155.stdout:Install 6 Packages 2023-10-09T10:29:00.695 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:00.695 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 2.0 M 2023-10-09T10:29:00.695 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 8.0 M 2023-10-09T10:29:00.695 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:29:00.884 INFO:teuthology.orchestra.run.smithi125.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1303.gb2903cb0 3.4 MB/s | 1.7 MB 00:00 2023-10-09T10:29:00.908 INFO:teuthology.orchestra.run.smithi125.stdout:(4/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 126 kB/s | 34 kB 00:00 2023-10-09T10:29:00.961 INFO:teuthology.orchestra.run.smithi125.stdout:(5/6): python3-jwt-1.6.1-2.el8.noarch.rpm 100 kB/s | 43 kB 00:00 2023-10-09T10:29:01.035 INFO:teuthology.orchestra.run.smithi155.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1303.gb29 152 kB/s | 23 kB 00:00 2023-10-09T10:29:01.133 INFO:teuthology.orchestra.run.smithi155.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1303.gb290 57 kB/s | 14 kB 00:00 2023-10-09T10:29:01.234 INFO:teuthology.orchestra.run.smithi125.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 561 kB/s | 196 kB 00:00 2023-10-09T10:29:01.238 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:29:01.238 INFO:teuthology.orchestra.run.smithi125.stdout:Total 2.1 MB/s | 2.0 MB 00:00 2023-10-09T10:29:01.247 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:29:01.256 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:29:01.257 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:29:01.258 INFO:teuthology.orchestra.run.smithi155.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 269 kB/s | 34 kB 00:00 2023-10-09T10:29:01.333 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:29:01.338 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:29:01.391 INFO:teuthology.orchestra.run.smithi155.stdout:(4/6): python3-routes-2.4.1-12.el8.noarch.rpm 1.5 MB/s | 196 kB 00:00 2023-10-09T10:29:01.425 INFO:teuthology.orchestra.run.smithi155.stdout:(5/6): ceph-mgr-dashboard-17.2.6-1303.gb2903cb0 3.1 MB/s | 1.7 MB 00:00 2023-10-09T10:29:01.445 INFO:teuthology.orchestra.run.smithi155.stdout:(6/6): python3-jwt-1.6.1-2.el8.noarch.rpm 104 kB/s | 43 kB 00:00 2023-10-09T10:29:01.446 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:29:01.447 INFO:teuthology.orchestra.run.smithi155.stdout:Total 2.6 MB/s | 2.0 MB 00:00 2023-10-09T10:29:01.455 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:29:01.465 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:29:01.465 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:29:01.544 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:29:01.547 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:29:01.603 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:29:01.804 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:29:01.866 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2023-10-09T10:29:02.010 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2023-10-09T10:29:02.120 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2023-10-09T10:29:02.255 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2023-10-09T10:29:02.363 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2023-10-09T10:29:02.475 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 4/6 2023-10-09T10:29:02.512 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2023-10-09T10:29:02.639 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 4/6 2023-10-09T10:29:02.840 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 5/6 2023-10-09T10:29:03.025 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 5/6 2023-10-09T10:29:03.065 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T10:29:03.223 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T10:29:03.365 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T10:29:03.365 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 1/6 2023-10-09T10:29:03.365 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 2/6 2023-10-09T10:29:03.366 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 3/6 2023-10-09T10:29:03.366 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2023-10-09T10:29:03.366 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2023-10-09T10:29:03.510 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T10:29:03.510 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 1/6 2023-10-09T10:29:03.510 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 2/6 2023-10-09T10:29:03.510 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 3/6 2023-10-09T10:29:03.511 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2023-10-09T10:29:03.511 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2023-10-09T10:29:04.018 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2023-10-09T10:29:04.018 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:04.018 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:29:04.018 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:29:04.018 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:29:04.018 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:29:04.018 INFO:teuthology.orchestra.run.smithi125.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-09T10:29:04.019 INFO:teuthology.orchestra.run.smithi125.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-09T10:29:04.019 INFO:teuthology.orchestra.run.smithi125.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-09T10:29:04.019 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:04.019 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:29:04.145 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2023-10-09T10:29:04.145 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:04.145 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:29:04.145 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:29:04.145 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:29:04.145 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:29:04.145 INFO:teuthology.orchestra.run.smithi155.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-09T10:29:04.145 INFO:teuthology.orchestra.run.smithi155.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-09T10:29:04.145 INFO:teuthology.orchestra.run.smithi155.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-09T10:29:04.145 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:04.146 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:29:04.167 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-09T10:29:04.288 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-09T10:29:05.019 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:01:43 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:29:05.157 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:01:40 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:29:06.606 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:29:06.607 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================================ 2023-10-09T10:29:06.607 INFO:teuthology.orchestra.run.smithi125.stdout: Package Arch Version Repository Size 2023-10-09T10:29:06.607 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================================ 2023-10-09T10:29:06.607 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:29:06.607 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 7.4 M 2023-10-09T10:29:06.608 INFO:teuthology.orchestra.run.smithi125.stdout:Installing dependencies: 2023-10-09T10:29:06.608 INFO:teuthology.orchestra.run.smithi125.stdout: libgfortran x86_64 8.5.0-20.el8 baseos 645 k 2023-10-09T10:29:06.608 INFO:teuthology.orchestra.run.smithi125.stdout: libquadmath x86_64 8.5.0-20.el8 baseos 172 k 2023-10-09T10:29:06.608 INFO:teuthology.orchestra.run.smithi125.stdout: openblas x86_64 0.3.15-4.el8 appstream 4.7 M 2023-10-09T10:29:06.608 INFO:teuthology.orchestra.run.smithi125.stdout: openblas-threads x86_64 0.3.15-4.el8 appstream 4.9 M 2023-10-09T10:29:06.608 INFO:teuthology.orchestra.run.smithi125.stdout: python3-numpy x86_64 1:1.14.3-10.el8 appstream 3.7 M 2023-10-09T10:29:06.608 INFO:teuthology.orchestra.run.smithi125.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 appstream 225 k 2023-10-09T10:29:06.608 INFO:teuthology.orchestra.run.smithi125.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 appstream 14 M 2023-10-09T10:29:06.608 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:06.608 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:29:06.608 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================================ 2023-10-09T10:29:06.609 INFO:teuthology.orchestra.run.smithi125.stdout:Install 8 Packages 2023-10-09T10:29:06.609 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:06.609 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 35 M 2023-10-09T10:29:06.609 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 210 M 2023-10-09T10:29:06.609 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:29:06.745 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:29:06.746 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================================ 2023-10-09T10:29:06.746 INFO:teuthology.orchestra.run.smithi155.stdout: Package Arch Version Repository Size 2023-10-09T10:29:06.746 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================================ 2023-10-09T10:29:06.746 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:29:06.746 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 7.4 M 2023-10-09T10:29:06.746 INFO:teuthology.orchestra.run.smithi155.stdout:Installing dependencies: 2023-10-09T10:29:06.746 INFO:teuthology.orchestra.run.smithi155.stdout: libgfortran x86_64 8.5.0-20.el8 baseos 645 k 2023-10-09T10:29:06.746 INFO:teuthology.orchestra.run.smithi155.stdout: libquadmath x86_64 8.5.0-20.el8 baseos 172 k 2023-10-09T10:29:06.747 INFO:teuthology.orchestra.run.smithi155.stdout: openblas x86_64 0.3.15-4.el8 appstream 4.7 M 2023-10-09T10:29:06.747 INFO:teuthology.orchestra.run.smithi155.stdout: openblas-threads x86_64 0.3.15-4.el8 appstream 4.9 M 2023-10-09T10:29:06.747 INFO:teuthology.orchestra.run.smithi155.stdout: python3-numpy x86_64 1:1.14.3-10.el8 appstream 3.7 M 2023-10-09T10:29:06.747 INFO:teuthology.orchestra.run.smithi155.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 appstream 225 k 2023-10-09T10:29:06.747 INFO:teuthology.orchestra.run.smithi155.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 appstream 14 M 2023-10-09T10:29:06.747 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:06.747 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:29:06.747 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================================ 2023-10-09T10:29:06.747 INFO:teuthology.orchestra.run.smithi155.stdout:Install 8 Packages 2023-10-09T10:29:06.747 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:06.748 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 35 M 2023-10-09T10:29:06.748 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 210 M 2023-10-09T10:29:06.748 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:29:07.852 INFO:teuthology.orchestra.run.smithi155.stdout:(1/8): ceph-mgr-diskprediction-local-17.2.6-130 14 MB/s | 7.4 MB 00:00 2023-10-09T10:29:08.269 INFO:teuthology.orchestra.run.smithi155.stdout:(2/8): openblas-0.3.15-4.el8.x86_64.rpm 4.9 MB/s | 4.7 MB 00:00 2023-10-09T10:29:08.344 INFO:teuthology.orchestra.run.smithi155.stdout:(3/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 4.7 MB/s | 4.9 MB 00:01 2023-10-09T10:29:08.370 INFO:teuthology.orchestra.run.smithi155.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 2.2 MB/s | 225 kB 00:00 2023-10-09T10:29:08.624 INFO:teuthology.orchestra.run.smithi125.stdout:(1/8): ceph-mgr-diskprediction-local-17.2.6-130 14 MB/s | 7.4 MB 00:00 2023-10-09T10:29:08.786 INFO:teuthology.orchestra.run.smithi155.stdout:(5/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 4.0 MB/s | 3.7 MB 00:00 2023-10-09T10:29:08.945 INFO:teuthology.orchestra.run.smithi155.stdout:(6/8): python3-scipy-1.0.0-21.module_el8.5.0+77 23 MB/s | 14 MB 00:00 2023-10-09T10:29:09.058 INFO:teuthology.orchestra.run.smithi125.stdout:(2/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 5.0 MB/s | 4.9 MB 00:00 2023-10-09T10:29:09.138 INFO:teuthology.orchestra.run.smithi125.stdout:(3/8): openblas-0.3.15-4.el8.x86_64.rpm 4.5 MB/s | 4.7 MB 00:01 2023-10-09T10:29:09.167 INFO:teuthology.orchestra.run.smithi125.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 2.0 MB/s | 225 kB 00:00 2023-10-09T10:29:09.279 INFO:teuthology.orchestra.run.smithi155.stdout:(7/8): libgfortran-8.5.0-20.el8.x86_64.rpm 710 kB/s | 645 kB 00:00 2023-10-09T10:29:09.462 INFO:teuthology.orchestra.run.smithi155.stdout:(8/8): libquadmath-8.5.0-20.el8.x86_64.rpm 255 kB/s | 172 kB 00:00 2023-10-09T10:29:09.463 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:29:09.463 INFO:teuthology.orchestra.run.smithi155.stdout:Total 13 MB/s | 35 MB 00:02 2023-10-09T10:29:09.592 INFO:teuthology.orchestra.run.smithi125.stdout:(5/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 3.9 MB/s | 3.7 MB 00:00 2023-10-09T10:29:09.700 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:29:09.718 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:29:09.719 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:29:09.746 INFO:teuthology.orchestra.run.smithi125.stdout:(6/8): python3-scipy-1.0.0-21.module_el8.5.0+77 23 MB/s | 14 MB 00:00 2023-10-09T10:29:10.058 INFO:teuthology.orchestra.run.smithi125.stdout:(7/8): libgfortran-8.5.0-20.el8.x86_64.rpm 724 kB/s | 645 kB 00:00 2023-10-09T10:29:10.110 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:29:10.113 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:29:10.268 INFO:teuthology.orchestra.run.smithi125.stdout:(8/8): libquadmath-8.5.0-20.el8.x86_64.rpm 255 kB/s | 172 kB 00:00 2023-10-09T10:29:10.269 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:29:10.269 INFO:teuthology.orchestra.run.smithi125.stdout:Total 9.7 MB/s | 35 MB 00:03 2023-10-09T10:29:10.506 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:29:10.524 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:29:10.524 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:29:10.661 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:29:10.806 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : libquadmath-8.5.0-20.el8.x86_64 1/8 2023-10-09T10:29:10.942 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: libquadmath-8.5.0-20.el8.x86_64 1/8 2023-10-09T10:29:11.098 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : libgfortran-8.5.0-20.el8.x86_64 2/8 2023-10-09T10:29:11.386 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:29:11.389 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:29:11.694 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: libgfortran-8.5.0-20.el8.x86_64 2/8 2023-10-09T10:29:11.881 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2023-10-09T10:29:12.134 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:29:12.317 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : libquadmath-8.5.0-20.el8.x86_64 1/8 2023-10-09T10:29:12.438 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: libquadmath-8.5.0-20.el8.x86_64 1/8 2023-10-09T10:29:12.532 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2023-10-09T10:29:12.617 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : libgfortran-8.5.0-20.el8.x86_64 2/8 2023-10-09T10:29:13.053 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-10-09T10:29:13.215 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: libgfortran-8.5.0-20.el8.x86_64 2/8 2023-10-09T10:29:13.376 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2023-10-09T10:29:14.026 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2023-10-09T10:29:14.167 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-10-09T10:29:14.447 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-10-09T10:29:14.776 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T10:29:14.878 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-10-09T10:29:15.048 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T10:29:15.220 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T10:29:15.508 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T10:29:16.817 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T10:29:17.270 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T10:29:18.090 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2023-10-09T10:29:18.296 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T10:29:18.557 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2023-10-09T10:29:18.637 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T10:29:18.637 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 1/8 2023-10-09T10:29:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2023-10-09T10:29:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2023-10-09T10:29:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2023-10-09T10:29:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T10:29:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2023-10-09T10:29:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : libgfortran-8.5.0-20.el8.x86_64 7/8 2023-10-09T10:29:18.766 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T10:29:19.088 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T10:29:19.088 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 1/8 2023-10-09T10:29:19.088 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2023-10-09T10:29:19.089 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2023-10-09T10:29:19.089 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2023-10-09T10:29:19.089 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T10:29:19.089 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2023-10-09T10:29:19.089 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : libgfortran-8.5.0-20.el8.x86_64 7/8 2023-10-09T10:29:19.300 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : libquadmath-8.5.0-20.el8.x86_64 8/8 2023-10-09T10:29:19.300 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:19.300 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:29:19.300 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:29:19.301 INFO:teuthology.orchestra.run.smithi155.stdout: libgfortran-8.5.0-20.el8.x86_64 2023-10-09T10:29:19.301 INFO:teuthology.orchestra.run.smithi155.stdout: libquadmath-8.5.0-20.el8.x86_64 2023-10-09T10:29:19.301 INFO:teuthology.orchestra.run.smithi155.stdout: openblas-0.3.15-4.el8.x86_64 2023-10-09T10:29:19.301 INFO:teuthology.orchestra.run.smithi155.stdout: openblas-threads-0.3.15-4.el8.x86_64 2023-10-09T10:29:19.301 INFO:teuthology.orchestra.run.smithi155.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-09T10:29:19.301 INFO:teuthology.orchestra.run.smithi155.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-09T10:29:19.301 INFO:teuthology.orchestra.run.smithi155.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2023-10-09T10:29:19.301 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:19.302 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:29:19.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install ceph-mgr-rook 2023-10-09T10:29:19.763 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : libquadmath-8.5.0-20.el8.x86_64 8/8 2023-10-09T10:29:19.763 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi125.stdout: libgfortran-8.5.0-20.el8.x86_64 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi125.stdout: libquadmath-8.5.0-20.el8.x86_64 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi125.stdout: openblas-0.3.15-4.el8.x86_64 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi125.stdout: openblas-threads-0.3.15-4.el8.x86_64 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi125.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-09T10:29:19.764 INFO:teuthology.orchestra.run.smithi125.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-09T10:29:19.765 INFO:teuthology.orchestra.run.smithi125.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2023-10-09T10:29:19.765 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:19.765 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:29:19.901 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install ceph-mgr-rook 2023-10-09T10:29:20.278 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:01:55 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:29:20.757 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:01:58 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:29:21.881 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:29:21.883 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: Package Arch Version Repository Size 2023-10-09T10:29:21.883 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:21.883 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:29:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mgr-rook noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 47 k 2023-10-09T10:29:21.883 INFO:teuthology.orchestra.run.smithi155.stdout:Installing dependencies: 2023-10-09T10:29:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-09T10:29:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-09T10:29:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-09T10:29:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: python3-jsonpatch noarch 1.21-2.el8 appstream 27 k 2023-10-09T10:29:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: python3-jsonpointer noarch 1.10-11.el8 appstream 20 k 2023-10-09T10:29:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-09T10:29:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: python3-oauthlib noarch 2.1.0-1.el8 baseos 155 k 2023-10-09T10:29:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pyasn1 noarch 0.3.7-6.el8 appstream 126 k 2023-10-09T10:29:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 appstream 110 k 2023-10-09T10:29:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 baseos 43 k 2023-10-09T10:29:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-09T10:29:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-09T10:29:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:21.884 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:29:21.885 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:21.885 INFO:teuthology.orchestra.run.smithi155.stdout:Install 13 Packages 2023-10-09T10:29:21.885 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:21.885 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 1.9 M 2023-10-09T10:29:21.886 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 25 M 2023-10-09T10:29:21.886 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:29:22.162 INFO:teuthology.orchestra.run.smithi155.stdout:(1/13): ceph-mgr-rook-17.2.6-1303.gb2903cb0.el8 295 kB/s | 47 kB 00:00 2023-10-09T10:29:22.330 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:29:22.332 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:22.332 INFO:teuthology.orchestra.run.smithi125.stdout: Package Arch Version Repository Size 2023-10-09T10:29:22.332 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:22.332 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:29:22.332 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mgr-rook noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 47 k 2023-10-09T10:29:22.332 INFO:teuthology.orchestra.run.smithi125.stdout:Installing dependencies: 2023-10-09T10:29:22.332 INFO:teuthology.orchestra.run.smithi125.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-09T10:29:22.333 INFO:teuthology.orchestra.run.smithi125.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-09T10:29:22.333 INFO:teuthology.orchestra.run.smithi125.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-09T10:29:22.333 INFO:teuthology.orchestra.run.smithi125.stdout: python3-jsonpatch noarch 1.21-2.el8 appstream 27 k 2023-10-09T10:29:22.333 INFO:teuthology.orchestra.run.smithi125.stdout: python3-jsonpointer noarch 1.10-11.el8 appstream 20 k 2023-10-09T10:29:22.333 INFO:teuthology.orchestra.run.smithi125.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-09T10:29:22.333 INFO:teuthology.orchestra.run.smithi125.stdout: python3-oauthlib noarch 2.1.0-1.el8 baseos 155 k 2023-10-09T10:29:22.333 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pyasn1 noarch 0.3.7-6.el8 appstream 126 k 2023-10-09T10:29:22.333 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 appstream 110 k 2023-10-09T10:29:22.333 INFO:teuthology.orchestra.run.smithi125.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 baseos 43 k 2023-10-09T10:29:22.333 INFO:teuthology.orchestra.run.smithi125.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-09T10:29:22.334 INFO:teuthology.orchestra.run.smithi125.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-09T10:29:22.334 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:22.334 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:29:22.334 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:22.334 INFO:teuthology.orchestra.run.smithi125.stdout:Install 13 Packages 2023-10-09T10:29:22.334 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:22.335 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 1.9 M 2023-10-09T10:29:22.335 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 25 M 2023-10-09T10:29:22.335 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:29:22.413 INFO:teuthology.orchestra.run.smithi155.stdout:(2/13): python3-jsonpointer-1.10-11.el8.noarch. 49 kB/s | 20 kB 00:00 2023-10-09T10:29:22.438 INFO:teuthology.orchestra.run.smithi155.stdout:(3/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 61 kB/s | 27 kB 00:00 2023-10-09T10:29:22.622 INFO:teuthology.orchestra.run.smithi155.stdout:(4/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 275 kB/s | 126 kB 00:00 2023-10-09T10:29:22.680 INFO:teuthology.orchestra.run.smithi155.stdout:(5/13): python3-pyasn1-modules-0.3.7-6.el8.noar 411 kB/s | 110 kB 00:00 2023-10-09T10:29:22.772 INFO:teuthology.orchestra.run.smithi155.stdout:(6/13): python3-cachetools-3.1.1-4.el8.noarch.r 360 kB/s | 33 kB 00:00 2023-10-09T10:29:22.815 INFO:teuthology.orchestra.run.smithi155.stdout:(7/13): python3-certifi-2018.10.15-7.el8.noarch 390 kB/s | 16 kB 00:00 2023-10-09T10:29:22.882 INFO:teuthology.orchestra.run.smithi155.stdout:(8/13): python3-google-auth-1.1.1-10.el8.noarch 1.2 MB/s | 81 kB 00:00 2023-10-09T10:29:22.990 INFO:teuthology.orchestra.run.smithi155.stdout:(9/13): python3-kubernetes-11.0.0-6.el8.noarch. 11 MB/s | 1.2 MB 00:00 2023-10-09T10:29:22.991 INFO:teuthology.orchestra.run.smithi125.stdout:(1/13): ceph-mgr-rook-17.2.6-1303.gb2903cb0.el8 176 kB/s | 47 kB 00:00 2023-10-09T10:29:23.025 INFO:teuthology.orchestra.run.smithi125.stdout:(2/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 88 kB/s | 27 kB 00:00 2023-10-09T10:29:23.032 INFO:teuthology.orchestra.run.smithi155.stdout:(10/13): python3-rsa-4.9-2.el8.noarch.rpm 1.5 MB/s | 64 kB 00:00 2023-10-09T10:29:23.041 INFO:teuthology.orchestra.run.smithi125.stdout:(3/13): python3-jsonpointer-1.10-11.el8.noarch. 62 kB/s | 20 kB 00:00 2023-10-09T10:29:23.066 INFO:teuthology.orchestra.run.smithi155.stdout:(11/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 247 kB/s | 155 kB 00:00 2023-10-09T10:29:23.091 INFO:teuthology.orchestra.run.smithi155.stdout:(12/13): python3-requests-oauthlib-1.0.0-1.el8. 91 kB/s | 43 kB 00:00 2023-10-09T10:29:23.116 INFO:teuthology.orchestra.run.smithi155.stdout:(13/13): python3-websocket-client-0.56.0-5.el8. 731 kB/s | 61 kB 00:00 2023-10-09T10:29:23.116 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:29:23.117 INFO:teuthology.orchestra.run.smithi155.stdout:Total 1.6 MB/s | 1.9 MB 00:01 2023-10-09T10:29:23.178 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:29:23.194 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:29:23.194 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:29:23.282 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:29:23.285 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:29:23.291 INFO:teuthology.orchestra.run.smithi125.stdout:(4/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 420 kB/s | 126 kB 00:00 2023-10-09T10:29:23.317 INFO:teuthology.orchestra.run.smithi125.stdout:(5/13): python3-pyasn1-modules-0.3.7-6.el8.noar 399 kB/s | 110 kB 00:00 2023-10-09T10:29:23.634 INFO:teuthology.orchestra.run.smithi125.stdout:(6/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 261 kB/s | 155 kB 00:00 2023-10-09T10:29:23.709 INFO:teuthology.orchestra.run.smithi125.stdout:(7/13): python3-requests-oauthlib-1.0.0-1.el8.n 102 kB/s | 43 kB 00:00 2023-10-09T10:29:23.776 INFO:teuthology.orchestra.run.smithi125.stdout:(8/13): python3-cachetools-3.1.1-4.el8.noarch.r 72 kB/s | 33 kB 00:00 2023-10-09T10:29:23.843 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:29:24.043 INFO:teuthology.orchestra.run.smithi125.stdout:(9/13): python3-certifi-2018.10.15-7.el8.noarch 40 kB/s | 16 kB 00:00 2023-10-09T10:29:24.107 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-09T10:29:24.268 INFO:teuthology.orchestra.run.smithi125.stdout:(10/13): python3-google-auth-1.1.1-10.el8.noarc 145 kB/s | 81 kB 00:00 2023-10-09T10:29:24.287 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2023-10-09T10:29:24.302 INFO:teuthology.orchestra.run.smithi125.stdout:(11/13): python3-rsa-4.9-2.el8.noarch.rpm 248 kB/s | 64 kB 00:00 2023-10-09T10:29:24.402 INFO:teuthology.orchestra.run.smithi125.stdout:(12/13): python3-kubernetes-11.0.0-6.el8.noarch 1.9 MB/s | 1.2 MB 00:00 2023-10-09T10:29:24.467 INFO:teuthology.orchestra.run.smithi125.stdout:(13/13): python3-websocket-client-0.56.0-5.el8. 309 kB/s | 61 kB 00:00 2023-10-09T10:29:24.470 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:29:24.470 INFO:teuthology.orchestra.run.smithi125.stdout:Total 925 kB/s | 1.9 MB 00:02 2023-10-09T10:29:24.507 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-rsa-4.9-2.el8.noarch 3/13 2023-10-09T10:29:24.531 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:29:24.547 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:29:24.547 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:29:24.633 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:29:24.636 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:29:24.744 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2023-10-09T10:29:24.935 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2023-10-09T10:29:25.168 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2023-10-09T10:29:25.256 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:29:25.494 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2023-10-09T10:29:25.517 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-09T10:29:25.693 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2023-10-09T10:29:25.723 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2023-10-09T10:29:25.967 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-rsa-4.9-2.el8.noarch 3/13 2023-10-09T10:29:26.290 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2023-10-09T10:29:26.495 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2023-10-09T10:29:26.709 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2023-10-09T10:29:26.753 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2023-10-09T10:29:27.002 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2023-10-09T10:29:27.079 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2023-10-09T10:29:27.211 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-10-09T10:29:27.370 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2023-10-09T10:29:27.397 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2023-10-09T10:29:27.546 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T10:29:27.848 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T10:29:27.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 1/13 2023-10-09T10:29:27.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2023-10-09T10:29:27.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2023-10-09T10:29:27.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2023-10-09T10:29:27.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2023-10-09T10:29:27.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-10-09T10:29:27.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-10-09T10:29:27.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2023-10-09T10:29:27.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-09T10:29:27.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2023-10-09T10:29:27.849 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2023-10-09T10:29:27.850 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 12/13 2023-10-09T10:29:28.436 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2023-10-09T10:29:28.534 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2023-10-09T10:29:28.534 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:28.534 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:29:28.534 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:29:28.535 INFO:teuthology.orchestra.run.smithi155.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-09T10:29:28.535 INFO:teuthology.orchestra.run.smithi155.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-09T10:29:28.535 INFO:teuthology.orchestra.run.smithi155.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-09T10:29:28.535 INFO:teuthology.orchestra.run.smithi155.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-09T10:29:28.536 INFO:teuthology.orchestra.run.smithi155.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-09T10:29:28.536 INFO:teuthology.orchestra.run.smithi155.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-09T10:29:28.536 INFO:teuthology.orchestra.run.smithi155.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-09T10:29:28.536 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-09T10:29:28.536 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-09T10:29:28.536 INFO:teuthology.orchestra.run.smithi155.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-09T10:29:28.537 INFO:teuthology.orchestra.run.smithi155.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-09T10:29:28.537 INFO:teuthology.orchestra.run.smithi155.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-09T10:29:28.537 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:28.537 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:29:28.647 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install ceph-mgr-cephadm 2023-10-09T10:29:28.696 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2023-10-09T10:29:28.938 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-10-09T10:29:29.181 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2023-10-09T10:29:29.372 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T10:29:29.510 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:02:04 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:29:29.677 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T10:29:29.677 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 1/13 2023-10-09T10:29:29.678 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2023-10-09T10:29:29.678 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2023-10-09T10:29:29.678 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2023-10-09T10:29:29.678 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2023-10-09T10:29:29.678 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-10-09T10:29:29.678 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-10-09T10:29:29.678 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2023-10-09T10:29:29.678 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-09T10:29:29.680 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2023-10-09T10:29:29.680 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2023-10-09T10:29:29.680 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 12/13 2023-10-09T10:29:30.363 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2023-10-09T10:29:30.363 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:30.363 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:29:30.363 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:29:30.363 INFO:teuthology.orchestra.run.smithi125.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-09T10:29:30.363 INFO:teuthology.orchestra.run.smithi125.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-09T10:29:30.363 INFO:teuthology.orchestra.run.smithi125.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-09T10:29:30.363 INFO:teuthology.orchestra.run.smithi125.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-09T10:29:30.363 INFO:teuthology.orchestra.run.smithi125.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-09T10:29:30.364 INFO:teuthology.orchestra.run.smithi125.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-09T10:29:30.364 INFO:teuthology.orchestra.run.smithi125.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-09T10:29:30.364 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-09T10:29:30.364 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-09T10:29:30.364 INFO:teuthology.orchestra.run.smithi125.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-09T10:29:30.364 INFO:teuthology.orchestra.run.smithi125.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-09T10:29:30.364 INFO:teuthology.orchestra.run.smithi125.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-09T10:29:30.364 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:30.364 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:29:30.478 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install ceph-mgr-cephadm 2023-10-09T10:29:31.117 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:29:31.118 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:31.118 INFO:teuthology.orchestra.run.smithi155.stdout: Package Arch Version Repository Size 2023-10-09T10:29:31.118 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:31.118 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:29:31.118 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 121 k 2023-10-09T10:29:31.118 INFO:teuthology.orchestra.run.smithi155.stdout:Installing dependencies: 2023-10-09T10:29:31.118 INFO:teuthology.orchestra.run.smithi155.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-10-09T10:29:31.118 INFO:teuthology.orchestra.run.smithi155.stdout: python3-babel noarch 2.5.1-7.el8 appstream 4.8 M 2023-10-09T10:29:31.119 INFO:teuthology.orchestra.run.smithi155.stdout: python3-jinja2 noarch 2.10.1-3.el8 appstream 538 k 2023-10-09T10:29:31.119 INFO:teuthology.orchestra.run.smithi155.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-10-09T10:29:31.119 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:31.119 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:29:31.119 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:31.119 INFO:teuthology.orchestra.run.smithi155.stdout:Install 5 Packages 2023-10-09T10:29:31.119 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:31.120 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 6.0 M 2023-10-09T10:29:31.120 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 26 M 2023-10-09T10:29:31.120 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:29:31.334 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:02:09 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:29:31.550 INFO:teuthology.orchestra.run.smithi155.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1303.gb2903cb0.e 658 kB/s | 121 kB 00:00 2023-10-09T10:29:31.976 INFO:teuthology.orchestra.run.smithi155.stdout:(2/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 1.1 MB/s | 476 kB 00:00 2023-10-09T10:29:32.009 INFO:teuthology.orchestra.run.smithi155.stdout:(3/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 836 kB/s | 538 kB 00:00 2023-10-09T10:29:32.060 INFO:teuthology.orchestra.run.smithi155.stdout:(4/5): python3-natsort-7.1.1-2.el8.noarch.rpm 722 kB/s | 60 kB 00:00 2023-10-09T10:29:32.335 INFO:teuthology.orchestra.run.smithi155.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 4.9 MB/s | 4.8 MB 00:00 2023-10-09T10:29:32.337 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:29:32.338 INFO:teuthology.orchestra.run.smithi155.stdout:Total 4.9 MB/s | 6.0 MB 00:01 2023-10-09T10:29:32.393 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:29:32.406 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:29:32.406 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:29:32.509 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:29:32.512 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:29:32.781 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:29:32.918 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:29:32.919 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:32.919 INFO:teuthology.orchestra.run.smithi125.stdout: Package Arch Version Repository Size 2023-10-09T10:29:32.920 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:32.920 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:29:32.920 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 121 k 2023-10-09T10:29:32.920 INFO:teuthology.orchestra.run.smithi125.stdout:Installing dependencies: 2023-10-09T10:29:32.920 INFO:teuthology.orchestra.run.smithi125.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-10-09T10:29:32.920 INFO:teuthology.orchestra.run.smithi125.stdout: python3-babel noarch 2.5.1-7.el8 appstream 4.8 M 2023-10-09T10:29:32.920 INFO:teuthology.orchestra.run.smithi125.stdout: python3-jinja2 noarch 2.10.1-3.el8 appstream 538 k 2023-10-09T10:29:32.920 INFO:teuthology.orchestra.run.smithi125.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-10-09T10:29:32.920 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:32.920 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:29:32.920 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:32.920 INFO:teuthology.orchestra.run.smithi125.stdout:Install 5 Packages 2023-10-09T10:29:32.921 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:32.921 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 6.0 M 2023-10-09T10:29:32.921 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 26 M 2023-10-09T10:29:32.921 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:29:33.082 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 1/5 2023-10-09T10:29:33.344 INFO:teuthology.orchestra.run.smithi125.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1303.gb2903cb0.e 412 kB/s | 121 kB 00:00 2023-10-09T10:29:33.611 INFO:teuthology.orchestra.run.smithi125.stdout:(2/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 1.7 MB/s | 476 kB 00:00 2023-10-09T10:29:33.686 INFO:teuthology.orchestra.run.smithi125.stdout:(3/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 845 kB/s | 538 kB 00:00 2023-10-09T10:29:33.702 INFO:teuthology.orchestra.run.smithi125.stdout:(4/5): python3-natsort-7.1.1-2.el8.noarch.rpm 658 kB/s | 60 kB 00:00 2023-10-09T10:29:33.995 INFO:teuthology.orchestra.run.smithi125.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 5.1 MB/s | 4.8 MB 00:00 2023-10-09T10:29:33.995 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:29:33.996 INFO:teuthology.orchestra.run.smithi125.stdout:Total 5.5 MB/s | 6.0 MB 00:01 2023-10-09T10:29:34.050 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:29:34.062 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:29:34.062 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:29:34.090 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-10-09T10:29:34.175 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:29:34.178 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:29:34.439 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:29:34.548 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 3/5 2023-10-09T10:29:34.753 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 1/5 2023-10-09T10:29:34.835 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-10-09T10:29:35.027 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T10:29:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T10:29:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/5 2023-10-09T10:29:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/5 2023-10-09T10:29:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/5 2023-10-09T10:29:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-10-09T10:29:35.748 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-10-09T10:29:35.986 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 5/5 2023-10-09T10:29:35.986 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:35.986 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:29:35.986 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:29:35.986 INFO:teuthology.orchestra.run.smithi155.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-10-09T10:29:35.986 INFO:teuthology.orchestra.run.smithi155.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-09T10:29:35.986 INFO:teuthology.orchestra.run.smithi155.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-09T10:29:35.986 INFO:teuthology.orchestra.run.smithi155.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-10-09T10:29:35.986 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:35.987 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:29:36.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install ceph-fuse 2023-10-09T10:29:36.134 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 3/5 2023-10-09T10:29:36.427 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-10-09T10:29:36.671 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T10:29:36.953 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T10:29:36.953 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/5 2023-10-09T10:29:36.953 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/5 2023-10-09T10:29:36.953 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/5 2023-10-09T10:29:36.953 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-10-09T10:29:36.966 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:02:11 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:29:37.583 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 5/5 2023-10-09T10:29:37.583 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:37.583 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:29:37.583 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T10:29:37.583 INFO:teuthology.orchestra.run.smithi125.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-10-09T10:29:37.583 INFO:teuthology.orchestra.run.smithi125.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-09T10:29:37.584 INFO:teuthology.orchestra.run.smithi125.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-09T10:29:37.584 INFO:teuthology.orchestra.run.smithi125.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-10-09T10:29:37.584 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:37.584 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:29:37.724 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install ceph-fuse 2023-10-09T10:29:38.557 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:29:38.557 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:38.558 INFO:teuthology.orchestra.run.smithi155.stdout: Package Arch Version Repository Size 2023-10-09T10:29:38.558 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:38.558 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:29:38.558 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-fuse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 876 k 2023-10-09T10:29:38.558 INFO:teuthology.orchestra.run.smithi155.stdout:Installing dependencies: 2023-10-09T10:29:38.558 INFO:teuthology.orchestra.run.smithi155.stdout: fuse x86_64 2.9.7-17.el8 baseos 83 k 2023-10-09T10:29:38.558 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:38.558 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:29:38.558 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:38.558 INFO:teuthology.orchestra.run.smithi155.stdout:Install 2 Packages 2023-10-09T10:29:38.558 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:38.559 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 959 k 2023-10-09T10:29:38.559 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 3.0 M 2023-10-09T10:29:38.559 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:29:38.592 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:02:16 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:29:39.003 INFO:teuthology.orchestra.run.smithi155.stdout:(1/2): ceph-fuse-17.2.6-1303.gb2903cb0.el8.x86_ 3.3 MB/s | 876 kB 00:00 2023-10-09T10:29:39.228 INFO:teuthology.orchestra.run.smithi155.stdout:(2/2): fuse-2.9.7-17.el8.x86_64.rpm 171 kB/s | 83 kB 00:00 2023-10-09T10:29:39.228 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:29:39.229 INFO:teuthology.orchestra.run.smithi155.stdout:Total 1.4 MB/s | 959 kB 00:00 2023-10-09T10:29:39.240 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:29:39.261 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:29:39.261 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:29:39.348 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:29:39.351 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:29:39.521 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:29:39.869 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : fuse-2.9.7-17.el8.x86_64 1/2 2023-10-09T10:29:40.072 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T10:29:40.181 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:29:40.182 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:40.182 INFO:teuthology.orchestra.run.smithi125.stdout: Package Arch Version Repository Size 2023-10-09T10:29:40.182 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:40.182 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:29:40.182 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-fuse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 876 k 2023-10-09T10:29:40.183 INFO:teuthology.orchestra.run.smithi125.stdout:Installing dependencies: 2023-10-09T10:29:40.183 INFO:teuthology.orchestra.run.smithi125.stdout: fuse x86_64 2.9.7-17.el8 baseos 83 k 2023-10-09T10:29:40.183 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:40.183 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:29:40.183 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:40.183 INFO:teuthology.orchestra.run.smithi125.stdout:Install 2 Packages 2023-10-09T10:29:40.183 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:40.184 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 959 k 2023-10-09T10:29:40.184 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 3.0 M 2023-10-09T10:29:40.184 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:29:40.465 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T10:29:40.465 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/2 2023-10-09T10:29:40.795 INFO:teuthology.orchestra.run.smithi125.stdout:(1/2): ceph-fuse-17.2.6-1303.gb2903cb0.el8.x86_ 2.0 MB/s | 876 kB 00:00 2023-10-09T10:29:40.853 INFO:teuthology.orchestra.run.smithi125.stdout:(2/2): fuse-2.9.7-17.el8.x86_64.rpm 174 kB/s | 83 kB 00:00 2023-10-09T10:29:40.854 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:29:40.854 INFO:teuthology.orchestra.run.smithi125.stdout:Total 1.4 MB/s | 959 kB 00:00 2023-10-09T10:29:40.864 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:29:40.885 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:29:40.885 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:29:40.971 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:29:40.974 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:29:41.075 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : fuse-2.9.7-17.el8.x86_64 2/2 2023-10-09T10:29:41.075 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:41.075 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:29:41.075 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 fuse-2.9.7-17.el8.x86_64 2023-10-09T10:29:41.075 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:41.075 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:29:41.150 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:29:41.281 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install librados-devel 2023-10-09T10:29:41.462 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : fuse-2.9.7-17.el8.x86_64 1/2 2023-10-09T10:29:41.714 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T10:29:42.085 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T10:29:42.085 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/2 2023-10-09T10:29:42.144 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:02:17 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:29:42.680 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : fuse-2.9.7-17.el8.x86_64 2/2 2023-10-09T10:29:42.681 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:42.681 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:29:42.681 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 fuse-2.9.7-17.el8.x86_64 2023-10-09T10:29:42.681 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:42.681 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:29:42.910 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install librados-devel 2023-10-09T10:29:43.707 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:29:43.708 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:43.708 INFO:teuthology.orchestra.run.smithi155.stdout: Package Arch Version Repo Size 2023-10-09T10:29:43.708 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:43.708 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:29:43.708 INFO:teuthology.orchestra.run.smithi155.stdout: librados-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 130 k 2023-10-09T10:29:43.708 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:43.708 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:29:43.708 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:43.708 INFO:teuthology.orchestra.run.smithi155.stdout:Install 1 Package 2023-10-09T10:29:43.708 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:43.709 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 130 k 2023-10-09T10:29:43.709 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 464 k 2023-10-09T10:29:43.709 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:29:43.777 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:02:21 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:29:43.917 INFO:teuthology.orchestra.run.smithi155.stdout:librados-devel-17.2.6-1303.gb2903cb0.el8.x86_64 625 kB/s | 130 kB 00:00 2023-10-09T10:29:43.917 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:29:43.917 INFO:teuthology.orchestra.run.smithi155.stdout:Total 621 kB/s | 130 kB 00:00 2023-10-09T10:29:43.918 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:29:43.923 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:29:43.923 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:29:43.973 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:29:43.976 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:29:44.082 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:29:44.350 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:29:44.627 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:29:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:29:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:45.277 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:29:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:29:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:45.277 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:29:45.352 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:29:45.352 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:45.353 INFO:teuthology.orchestra.run.smithi125.stdout: Package Arch Version Repo Size 2023-10-09T10:29:45.353 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:45.353 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:29:45.353 INFO:teuthology.orchestra.run.smithi125.stdout: librados-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 130 k 2023-10-09T10:29:45.353 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:45.353 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:29:45.353 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:45.353 INFO:teuthology.orchestra.run.smithi125.stdout:Install 1 Package 2023-10-09T10:29:45.353 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:45.354 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 130 k 2023-10-09T10:29:45.354 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 464 k 2023-10-09T10:29:45.354 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:29:45.633 INFO:teuthology.orchestra.run.smithi125.stdout:librados-devel-17.2.6-1303.gb2903cb0.el8.x86_64 465 kB/s | 130 kB 00:00 2023-10-09T10:29:45.633 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:29:45.634 INFO:teuthology.orchestra.run.smithi125.stdout:Total 463 kB/s | 130 kB 00:00 2023-10-09T10:29:45.634 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:29:45.639 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:29:45.639 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:29:45.688 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:29:45.691 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:29:45.806 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:29:46.035 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:29:46.358 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:29:46.599 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install libcephfs2 2023-10-09T10:29:47.001 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:29:47.001 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:47.002 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:29:47.002 INFO:teuthology.orchestra.run.smithi125.stdout: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:29:47.002 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:47.002 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:29:47.243 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install libcephfs2 2023-10-09T10:29:47.467 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:02:22 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:29:48.112 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:02:26 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:29:48.891 INFO:teuthology.orchestra.run.smithi155.stdout:Package libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:29:49.079 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:29:49.080 INFO:teuthology.orchestra.run.smithi155.stdout:Nothing to do. 2023-10-09T10:29:49.080 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:29:49.181 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install libcephfs-devel 2023-10-09T10:29:49.520 INFO:teuthology.orchestra.run.smithi125.stdout:Package libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:29:49.712 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:29:49.714 INFO:teuthology.orchestra.run.smithi125.stdout:Nothing to do. 2023-10-09T10:29:49.714 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:29:49.813 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install libcephfs-devel 2023-10-09T10:29:50.037 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:02:25 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:29:50.685 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:02:28 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:29:51.631 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:29:51.632 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:51.632 INFO:teuthology.orchestra.run.smithi155.stdout: Package Arch Version Repo Size 2023-10-09T10:29:51.632 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:51.632 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:29:51.632 INFO:teuthology.orchestra.run.smithi155.stdout: libcephfs-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 25 k 2023-10-09T10:29:51.632 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:51.632 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:29:51.632 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:29:51.632 INFO:teuthology.orchestra.run.smithi155.stdout:Install 1 Package 2023-10-09T10:29:51.632 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:51.633 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 25 k 2023-10-09T10:29:51.633 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 109 k 2023-10-09T10:29:51.633 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:29:51.800 INFO:teuthology.orchestra.run.smithi155.stdout:libcephfs-devel-17.2.6-1303.gb2903cb0.el8.x86_6 149 kB/s | 25 kB 00:00 2023-10-09T10:29:51.801 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:29:51.801 INFO:teuthology.orchestra.run.smithi155.stdout:Total 148 kB/s | 25 kB 00:00 2023-10-09T10:29:51.801 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:29:51.804 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:29:51.804 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:29:51.806 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:29:51.809 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:29:51.864 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:29:52.049 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:29:52.268 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:29:52.269 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:52.269 INFO:teuthology.orchestra.run.smithi125.stdout: Package Arch Version Repo Size 2023-10-09T10:29:52.269 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:52.269 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:29:52.270 INFO:teuthology.orchestra.run.smithi125.stdout: libcephfs-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 25 k 2023-10-09T10:29:52.270 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:52.270 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:29:52.270 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:29:52.270 INFO:teuthology.orchestra.run.smithi125.stdout:Install 1 Package 2023-10-09T10:29:52.270 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:52.271 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 25 k 2023-10-09T10:29:52.271 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 109 k 2023-10-09T10:29:52.272 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:29:52.305 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:29:52.514 INFO:teuthology.orchestra.run.smithi125.stdout:libcephfs-devel-17.2.6-1303.gb2903cb0.el8.x86_6 102 kB/s | 25 kB 00:00 2023-10-09T10:29:52.515 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:29:52.515 INFO:teuthology.orchestra.run.smithi125.stdout:Total 101 kB/s | 25 kB 00:00 2023-10-09T10:29:52.516 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:29:52.519 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:29:52.519 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:29:52.521 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:29:52.524 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:29:52.599 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:29:52.792 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:29:52.888 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:29:52.888 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:52.888 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:29:52.888 INFO:teuthology.orchestra.run.smithi155.stdout: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:29:52.888 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:29:52.888 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:29:53.086 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:29:53.178 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install librados2 2023-10-09T10:29:53.675 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:29:53.675 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:53.675 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:29:53.675 INFO:teuthology.orchestra.run.smithi125.stdout: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:29:53.676 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:29:53.676 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:29:53.795 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install librados2 2023-10-09T10:29:54.038 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:02:29 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:29:54.657 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:02:32 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:29:55.415 INFO:teuthology.orchestra.run.smithi155.stdout:Package librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:29:55.603 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:29:55.604 INFO:teuthology.orchestra.run.smithi155.stdout:Nothing to do. 2023-10-09T10:29:55.604 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:29:55.699 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install librbd1 2023-10-09T10:29:56.040 INFO:teuthology.orchestra.run.smithi125.stdout:Package librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:29:56.229 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:29:56.231 INFO:teuthology.orchestra.run.smithi125.stdout:Nothing to do. 2023-10-09T10:29:56.261 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:29:56.329 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install librbd1 2023-10-09T10:29:56.559 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:02:31 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:29:57.196 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:02:35 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:29:57.969 INFO:teuthology.orchestra.run.smithi155.stdout:Package librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:29:58.159 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:29:58.160 INFO:teuthology.orchestra.run.smithi155.stdout:Nothing to do. 2023-10-09T10:29:58.160 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:29:58.259 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install python3-rados 2023-10-09T10:29:58.597 INFO:teuthology.orchestra.run.smithi125.stdout:Package librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:29:58.786 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:29:58.787 INFO:teuthology.orchestra.run.smithi125.stdout:Nothing to do. 2023-10-09T10:29:58.787 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:29:58.892 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install python3-rados 2023-10-09T10:29:59.112 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:02:34 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:29:59.745 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:02:37 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:30:00.496 INFO:teuthology.orchestra.run.smithi155.stdout:Package python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:30:00.683 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:30:00.686 INFO:teuthology.orchestra.run.smithi155.stdout:Nothing to do. 2023-10-09T10:30:00.686 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:30:00.776 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install python3-rgw 2023-10-09T10:30:01.116 INFO:teuthology.orchestra.run.smithi125.stdout:Package python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:30:01.305 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:30:01.306 INFO:teuthology.orchestra.run.smithi125.stdout:Nothing to do. 2023-10-09T10:30:01.307 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:30:01.417 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install python3-rgw 2023-10-09T10:30:01.649 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:02:36 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:30:02.271 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:02:40 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:30:03.030 INFO:teuthology.orchestra.run.smithi155.stdout:Package python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:30:03.222 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:30:03.223 INFO:teuthology.orchestra.run.smithi155.stdout:Nothing to do. 2023-10-09T10:30:03.223 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:30:03.316 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install python3-cephfs 2023-10-09T10:30:03.664 INFO:teuthology.orchestra.run.smithi125.stdout:Package python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:30:03.852 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:30:03.853 INFO:teuthology.orchestra.run.smithi125.stdout:Nothing to do. 2023-10-09T10:30:03.853 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:30:03.961 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install python3-cephfs 2023-10-09T10:30:04.170 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:02:39 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:30:04.811 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:02:42 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:30:05.583 INFO:teuthology.orchestra.run.smithi155.stdout:Package python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:30:05.772 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:30:05.773 INFO:teuthology.orchestra.run.smithi155.stdout:Nothing to do. 2023-10-09T10:30:05.773 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:30:05.874 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install python3-rbd 2023-10-09T10:30:06.200 INFO:teuthology.orchestra.run.smithi125.stdout:Package python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:30:06.395 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:30:06.396 INFO:teuthology.orchestra.run.smithi125.stdout:Nothing to do. 2023-10-09T10:30:06.397 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:30:06.502 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install python3-rbd 2023-10-09T10:30:06.724 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:02:41 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:30:07.352 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:02:45 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:30:08.121 INFO:teuthology.orchestra.run.smithi155.stdout:Package python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:30:08.309 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:30:08.310 INFO:teuthology.orchestra.run.smithi155.stdout:Nothing to do. 2023-10-09T10:30:08.311 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:30:08.404 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install rbd-fuse 2023-10-09T10:30:08.749 INFO:teuthology.orchestra.run.smithi125.stdout:Package python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T10:30:08.936 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:30:08.937 INFO:teuthology.orchestra.run.smithi125.stdout:Nothing to do. 2023-10-09T10:30:08.937 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:30:09.037 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install rbd-fuse 2023-10-09T10:30:09.269 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:02:44 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:30:09.895 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:02:47 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:30:10.877 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:30:10.877 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:30:10.877 INFO:teuthology.orchestra.run.smithi155.stdout: Package Architecture Version Repository Size 2023-10-09T10:30:10.877 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:30:10.877 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:30:10.877 INFO:teuthology.orchestra.run.smithi155.stdout: rbd-fuse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 90 k 2023-10-09T10:30:10.878 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:30:10.878 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:30:10.878 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:30:10.878 INFO:teuthology.orchestra.run.smithi155.stdout:Install 1 Package 2023-10-09T10:30:10.878 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:30:10.878 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 90 k 2023-10-09T10:30:10.879 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 264 k 2023-10-09T10:30:10.879 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:30:11.040 INFO:teuthology.orchestra.run.smithi155.stdout:rbd-fuse-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 558 kB/s | 90 kB 00:00 2023-10-09T10:30:11.041 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:30:11.041 INFO:teuthology.orchestra.run.smithi155.stdout:Total 554 kB/s | 90 kB 00:00 2023-10-09T10:30:11.041 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:30:11.051 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:30:11.051 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:30:11.101 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:30:11.104 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:30:11.222 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:30:11.438 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:11.473 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:30:11.474 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:30:11.474 INFO:teuthology.orchestra.run.smithi125.stdout: Package Architecture Version Repository Size 2023-10-09T10:30:11.474 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:30:11.474 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:30:11.474 INFO:teuthology.orchestra.run.smithi125.stdout: rbd-fuse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 90 k 2023-10-09T10:30:11.474 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:30:11.474 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:30:11.475 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:30:11.475 INFO:teuthology.orchestra.run.smithi125.stdout:Install 1 Package 2023-10-09T10:30:11.475 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:30:11.475 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 90 k 2023-10-09T10:30:11.475 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 264 k 2023-10-09T10:30:11.475 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:30:11.737 INFO:teuthology.orchestra.run.smithi125.stdout:rbd-fuse-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 345 kB/s | 90 kB 00:00 2023-10-09T10:30:11.738 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:30:11.738 INFO:teuthology.orchestra.run.smithi125.stdout:Total 343 kB/s | 90 kB 00:00 2023-10-09T10:30:11.738 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:30:11.748 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:30:11.748 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:30:11.751 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:11.798 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:30:11.801 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:30:11.918 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:30:12.164 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:12.355 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:12.355 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:30:12.355 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:30:12.355 INFO:teuthology.orchestra.run.smithi155.stdout: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:30:12.355 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:30:12.355 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:30:12.453 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:12.570 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install rbd-mirror 2023-10-09T10:30:13.061 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:13.062 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:30:13.062 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:30:13.062 INFO:teuthology.orchestra.run.smithi125.stdout: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:30:13.062 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:30:13.062 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:30:13.299 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install rbd-mirror 2023-10-09T10:30:13.434 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:02:48 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:30:14.161 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:02:52 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:30:15.024 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:30:15.025 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:30:15.025 INFO:teuthology.orchestra.run.smithi155.stdout: Package Arch Version Repo Size 2023-10-09T10:30:15.025 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:30:15.025 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:30:15.025 INFO:teuthology.orchestra.run.smithi155.stdout: rbd-mirror x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T10:30:15.025 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:30:15.025 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:30:15.025 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:30:15.026 INFO:teuthology.orchestra.run.smithi155.stdout:Install 1 Package 2023-10-09T10:30:15.026 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:30:15.026 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 3.5 M 2023-10-09T10:30:15.026 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 16 M 2023-10-09T10:30:15.026 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:30:15.552 INFO:teuthology.orchestra.run.smithi155.stdout:rbd-mirror-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 6.6 MB/s | 3.5 MB 00:00 2023-10-09T10:30:15.553 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:30:15.553 INFO:teuthology.orchestra.run.smithi155.stdout:Total 6.6 MB/s | 3.5 MB 00:00 2023-10-09T10:30:15.553 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:30:15.565 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:30:15.565 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:30:15.627 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:30:15.630 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:30:15.736 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:30:15.737 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:30:15.737 INFO:teuthology.orchestra.run.smithi125.stdout: Package Arch Version Repo Size 2023-10-09T10:30:15.737 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:30:15.737 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:30:15.737 INFO:teuthology.orchestra.run.smithi125.stdout: rbd-mirror x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T10:30:15.737 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:30:15.738 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:30:15.738 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:30:15.738 INFO:teuthology.orchestra.run.smithi125.stdout:Install 1 Package 2023-10-09T10:30:15.738 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:30:15.738 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 3.5 M 2023-10-09T10:30:15.738 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 16 M 2023-10-09T10:30:15.739 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:30:16.077 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:30:16.259 INFO:teuthology.orchestra.run.smithi125.stdout:rbd-mirror-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 6.7 MB/s | 3.5 MB 00:00 2023-10-09T10:30:16.260 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:30:16.260 INFO:teuthology.orchestra.run.smithi125.stdout:Total 6.6 MB/s | 3.5 MB 00:00 2023-10-09T10:30:16.261 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:30:16.268 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:16.270 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:30:16.270 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:30:16.336 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:30:16.338 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:30:16.695 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:16.774 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:30:16.981 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:17.311 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:17.311 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:30:17.311 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:30:17.312 INFO:teuthology.orchestra.run.smithi155.stdout: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:30:17.312 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:30:17.312 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:30:17.378 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:17.508 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum -y install rbd-nbd 2023-10-09T10:30:18.052 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:18.052 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:30:18.053 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:30:18.053 INFO:teuthology.orchestra.run.smithi125.stdout: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:30:18.053 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:30:18.053 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:30:18.272 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum -y install rbd-nbd 2023-10-09T10:30:18.385 INFO:teuthology.orchestra.run.smithi155.stdout:Last metadata expiration check: 0:02:53 ago on Mon 09 Oct 2023 10:27:25 AM UTC. 2023-10-09T10:30:19.140 INFO:teuthology.orchestra.run.smithi125.stdout:Last metadata expiration check: 0:02:57 ago on Mon 09 Oct 2023 10:27:22 AM UTC. 2023-10-09T10:30:19.982 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2023-10-09T10:30:19.983 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:30:19.983 INFO:teuthology.orchestra.run.smithi155.stdout: Package Architecture Version Repository Size 2023-10-09T10:30:19.983 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:30:19.983 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2023-10-09T10:30:19.983 INFO:teuthology.orchestra.run.smithi155.stdout: rbd-nbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 173 k 2023-10-09T10:30:19.983 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:30:19.984 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2023-10-09T10:30:19.984 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2023-10-09T10:30:19.984 INFO:teuthology.orchestra.run.smithi155.stdout:Install 1 Package 2023-10-09T10:30:19.984 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:30:19.984 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 173 k 2023-10-09T10:30:19.984 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 552 k 2023-10-09T10:30:19.985 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2023-10-09T10:30:20.181 INFO:teuthology.orchestra.run.smithi155.stdout:rbd-nbd-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 880 kB/s | 173 kB 00:00 2023-10-09T10:30:20.181 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:30:20.181 INFO:teuthology.orchestra.run.smithi155.stdout:Total 875 kB/s | 173 kB 00:00 2023-10-09T10:30:20.182 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2023-10-09T10:30:20.192 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2023-10-09T10:30:20.192 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2023-10-09T10:30:20.244 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2023-10-09T10:30:20.247 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2023-10-09T10:30:20.372 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2023-10-09T10:30:20.544 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:20.715 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2023-10-09T10:30:20.716 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:30:20.716 INFO:teuthology.orchestra.run.smithi125.stdout: Package Architecture Version Repository Size 2023-10-09T10:30:20.716 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:30:20.716 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2023-10-09T10:30:20.716 INFO:teuthology.orchestra.run.smithi125.stdout: rbd-nbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 173 k 2023-10-09T10:30:20.716 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:30:20.716 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2023-10-09T10:30:20.716 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2023-10-09T10:30:20.716 INFO:teuthology.orchestra.run.smithi125.stdout:Install 1 Package 2023-10-09T10:30:20.716 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:30:20.717 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 173 k 2023-10-09T10:30:20.717 INFO:teuthology.orchestra.run.smithi125.stdout:Installed size: 552 k 2023-10-09T10:30:20.717 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2023-10-09T10:30:20.839 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:21.027 INFO:teuthology.orchestra.run.smithi125.stdout:rbd-nbd-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 558 kB/s | 173 kB 00:00 2023-10-09T10:30:21.027 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:30:21.028 INFO:teuthology.orchestra.run.smithi125.stdout:Total 555 kB/s | 173 kB 00:00 2023-10-09T10:30:21.028 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2023-10-09T10:30:21.038 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2023-10-09T10:30:21.038 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2023-10-09T10:30:21.088 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2023-10-09T10:30:21.091 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2023-10-09T10:30:21.206 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2023-10-09T10:30:21.412 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:30:21.476 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2023-10-09T10:30:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:30:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:30:21.476 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2023-10-09T10:30:21.686 DEBUG:teuthology.parallel:result is None 2023-10-09T10:30:21.704 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:22.332 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T10:30:22.332 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:30:22.333 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2023-10-09T10:30:22.333 INFO:teuthology.orchestra.run.smithi125.stdout: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T10:30:22.333 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:30:22.333 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2023-10-09T10:30:22.567 DEBUG:teuthology.parallel:result is None 2023-10-09T10:30:22.567 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T10:30:22.567 INFO:teuthology.packaging:ref: None 2023-10-09T10:30:22.567 INFO:teuthology.packaging:tag: None 2023-10-09T10:30:22.567 INFO:teuthology.packaging:branch: quincy 2023-10-09T10:30:22.567 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T10:30:22.568 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T10:30:22.834 DEBUG:teuthology.orchestra.run.smithi125:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-09T10:30:22.857 INFO:teuthology.orchestra.run.smithi125.stdout:17.2.6-1303.gb2903cb0.el8 2023-10-09T10:30:22.858 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1303.gb2903cb0.el8 2023-10-09T10:30:22.858 INFO:teuthology.task.install:The correct ceph version 17.2.6-1303.gb2903cb0 is installed. 2023-10-09T10:30:22.858 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T10:30:22.859 INFO:teuthology.packaging:ref: None 2023-10-09T10:30:22.859 INFO:teuthology.packaging:tag: None 2023-10-09T10:30:22.859 INFO:teuthology.packaging:branch: quincy 2023-10-09T10:30:22.859 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T10:30:22.859 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T10:30:23.230 DEBUG:teuthology.orchestra.run.smithi155:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-09T10:30:23.258 INFO:teuthology.orchestra.run.smithi155.stdout:17.2.6-1303.gb2903cb0.el8 2023-10-09T10:30:23.259 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1303.gb2903cb0.el8 2023-10-09T10:30:23.259 INFO:teuthology.task.install:The correct ceph version 17.2.6-1303.gb2903cb0 is installed. 2023-10-09T10:30:23.259 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-10-09T10:30:23.259 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:30:23.259 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T10:30:23.289 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T10:30:23.289 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T10:30:23.334 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-10-09T10:30:23.334 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:30:23.334 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T10:30:23.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T10:30:23.431 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T10:30:23.431 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T10:30:23.463 DEBUG:teuthology.orchestra.run.smithi155:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T10:30:23.533 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-10-09T10:30:23.533 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:30:23.533 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T10:30:23.561 DEBUG:teuthology.orchestra.run.smithi125:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T10:30:23.628 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T10:30:23.628 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T10:30:23.658 DEBUG:teuthology.orchestra.run.smithi155:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T10:30:23.727 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-10-09T10:30:23.727 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:30:23.727 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T10:30:23.754 DEBUG:teuthology.orchestra.run.smithi125:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T10:30:23.821 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T10:30:23.821 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T10:30:23.850 DEBUG:teuthology.orchestra.run.smithi155:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T10:30:23.919 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-09T10:30:24.003 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T10:30:24.004 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-10-09T10:30:24.004 INFO:tasks.cephadm:Cluster fsid is daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:30:24.004 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-09T10:30:24.004 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.125', 'mon.c': '[v2:172.21.15.125:3301,v1:172.21.15.125:6790]', 'mon.b': '172.21.15.155'} 2023-10-09T10:30:24.005 INFO:tasks.cephadm:First mon is mon.a on smithi125 2023-10-09T10:30:24.005 INFO:tasks.cephadm:First mgr is y 2023-10-09T10:30:24.005 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-09T10:30:24.005 DEBUG:teuthology.orchestra.run.smithi125:> sudo hostname $(hostname -s) 2023-10-09T10:30:24.032 DEBUG:teuthology.orchestra.run.smithi155:> sudo hostname $(hostname -s) 2023-10-09T10:30:24.063 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-10-09T10:30:24.063 DEBUG:teuthology.orchestra.run.smithi125:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T10:30:24.278 INFO:teuthology.orchestra.run.smithi125.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366887 Oct 9 10:30 /home/ubuntu/cephtest/cephadm 2023-10-09T10:30:24.279 DEBUG:teuthology.orchestra.run.smithi155:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T10:30:24.425 INFO:teuthology.orchestra.run.smithi155.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366887 Oct 9 10:30 /home/ubuntu/cephtest/cephadm 2023-10-09T10:30:24.425 DEBUG:teuthology.orchestra.run.smithi125:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T10:30:24.444 DEBUG:teuthology.orchestra.run.smithi155:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T10:30:24.478 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-10-09T10:30:24.479 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-10-09T10:30:24.487 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-10-09T10:30:25.573 INFO:teuthology.orchestra.run.smithi155.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T10:30:25.673 INFO:teuthology.orchestra.run.smithi125.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T10:30:58.344 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:30:58.344 INFO:teuthology.orchestra.run.smithi155.stdout: "ceph_version": "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)", 2023-10-09T10:30:58.344 INFO:teuthology.orchestra.run.smithi155.stdout: "image_id": "5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf", 2023-10-09T10:30:58.344 INFO:teuthology.orchestra.run.smithi155.stdout: "repo_digests": [ 2023-10-09T10:30:58.344 INFO:teuthology.orchestra.run.smithi155.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:67105d45415a7f6878d82dd27e27caaa40ebf0ca6488030995660f1cad5fce9d" 2023-10-09T10:30:58.344 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:30:58.344 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:31:01.478 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:31:01.479 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph_version": "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)", 2023-10-09T10:31:01.479 INFO:teuthology.orchestra.run.smithi125.stdout: "image_id": "5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf", 2023-10-09T10:31:01.479 INFO:teuthology.orchestra.run.smithi125.stdout: "repo_digests": [ 2023-10-09T10:31:01.479 INFO:teuthology.orchestra.run.smithi125.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:67105d45415a7f6878d82dd27e27caaa40ebf0ca6488030995660f1cad5fce9d" 2023-10-09T10:31:01.479 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:31:01.479 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:31:01.497 DEBUG:teuthology.orchestra.run.smithi125:> sudo mkdir -p /etc/ceph 2023-10-09T10:31:01.527 DEBUG:teuthology.orchestra.run.smithi155:> sudo mkdir -p /etc/ceph 2023-10-09T10:31:01.565 DEBUG:teuthology.orchestra.run.smithi125:> sudo chmod 777 /etc/ceph 2023-10-09T10:31:01.595 DEBUG:teuthology.orchestra.run.smithi155:> sudo chmod 777 /etc/ceph 2023-10-09T10:31:01.636 INFO:tasks.cephadm:Writing seed config... 2023-10-09T10:31:01.637 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-10-09T10:31:01.637 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-10-09T10:31:01.638 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-09T10:31:01.638 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-09T10:31:01.638 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-10-09T10:31:01.638 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2023-10-09T10:31:01.638 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-09T10:31:01.638 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-09T10:31:01.638 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-09T10:31:01.638 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-09T10:31:01.638 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-09T10:31:01.639 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:31:01.639 DEBUG:teuthology.orchestra.run.smithi125:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-09T10:31:01.657 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = daf690ae-668e-11ee-8db5-212e2dc638e7 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-09T10:31:01.657 DEBUG:teuthology.orchestra.run.smithi125:mon.a> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-09T10:31:01.699 DEBUG:teuthology.orchestra.run.smithi125:mgr.y> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.y.service 2023-10-09T10:31:01.726 INFO:journalctl@ceph.mon.a.smithi125.stdout:-- Logs begin at Mon 2023-10-09 10:18:49 UTC. -- 2023-10-09T10:31:01.726 INFO:tasks.cephadm:Bootstrapping... 2023-10-09T10:31:01.727 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.125 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-09T10:31:01.753 INFO:journalctl@ceph.mgr.y.smithi125.stdout:-- Logs begin at Mon 2023-10-09 10:18:49 UTC. -- 2023-10-09T10:31:01.928 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2023-10-09T10:31:01.928 INFO:teuthology.orchestra.run.smithi125.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'daf690ae-668e-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.125', '--skip-admin-label'] 2023-10-09T10:31:01.961 INFO:teuthology.orchestra.run.smithi125.stdout:/bin/podman: stdout 4.6.1 2023-10-09T10:31:01.962 INFO:teuthology.orchestra.run.smithi125.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-10-09T10:31:01.962 INFO:teuthology.orchestra.run.smithi125.stdout:Verifying podman|docker is present... 2023-10-09T10:31:01.997 INFO:teuthology.orchestra.run.smithi125.stdout:/bin/podman: stdout 4.6.1 2023-10-09T10:31:01.998 INFO:teuthology.orchestra.run.smithi125.stdout:Verifying lvm2 is present... 2023-10-09T10:31:01.998 INFO:teuthology.orchestra.run.smithi125.stdout:Verifying time synchronization is in place... 2023-10-09T10:31:02.005 INFO:teuthology.orchestra.run.smithi125.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-09T10:31:02.005 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T10:31:02.010 INFO:teuthology.orchestra.run.smithi125.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-09T10:31:02.010 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stdout inactive 2023-10-09T10:31:02.017 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stdout enabled 2023-10-09T10:31:02.022 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stdout active 2023-10-09T10:31:02.023 INFO:teuthology.orchestra.run.smithi125.stdout:Unit chronyd.service is enabled and running 2023-10-09T10:31:02.023 INFO:teuthology.orchestra.run.smithi125.stdout:Repeating the final host check... 2023-10-09T10:31:02.050 INFO:teuthology.orchestra.run.smithi125.stdout:/bin/podman: stdout 4.6.1 2023-10-09T10:31:02.050 INFO:teuthology.orchestra.run.smithi125.stdout:podman (/bin/podman) version 4.6.1 is present 2023-10-09T10:31:02.050 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl is present 2023-10-09T10:31:02.050 INFO:teuthology.orchestra.run.smithi125.stdout:lvcreate is present 2023-10-09T10:31:02.056 INFO:teuthology.orchestra.run.smithi125.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-09T10:31:02.056 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T10:31:02.062 INFO:teuthology.orchestra.run.smithi125.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-09T10:31:02.062 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stdout inactive 2023-10-09T10:31:02.067 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stdout enabled 2023-10-09T10:31:02.073 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stdout active 2023-10-09T10:31:02.074 INFO:teuthology.orchestra.run.smithi125.stdout:Unit chronyd.service is enabled and running 2023-10-09T10:31:02.074 INFO:teuthology.orchestra.run.smithi125.stdout:Host looks OK 2023-10-09T10:31:02.074 INFO:teuthology.orchestra.run.smithi125.stdout:Cluster fsid: daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:31:02.074 INFO:teuthology.orchestra.run.smithi125.stdout:Acquiring lock 140443899079144 on /run/cephadm/daf690ae-668e-11ee-8db5-212e2dc638e7.lock 2023-10-09T10:31:02.074 INFO:teuthology.orchestra.run.smithi125.stdout:Lock 140443899079144 acquired on /run/cephadm/daf690ae-668e-11ee-8db5-212e2dc638e7.lock 2023-10-09T10:31:02.075 INFO:teuthology.orchestra.run.smithi125.stdout:Verifying IP 172.21.15.125 port 3300 ... 2023-10-09T10:31:02.075 INFO:teuthology.orchestra.run.smithi125.stdout:Verifying IP 172.21.15.125 port 6789 ... 2023-10-09T10:31:02.075 INFO:teuthology.orchestra.run.smithi125.stdout:Base mon IP(s) is [172.21.15.125:3300, 172.21.15.125:6789], mon addrv is [v2:172.21.15.125:3300,v1:172.21.15.125:6789] 2023-10-09T10:31:02.079 INFO:teuthology.orchestra.run.smithi125.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.125 metric 100 2023-10-09T10:31:02.079 INFO:teuthology.orchestra.run.smithi125.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.125 metric 100 2023-10-09T10:31:02.081 INFO:teuthology.orchestra.run.smithi125.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-10-09T10:31:02.081 INFO:teuthology.orchestra.run.smithi125.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2023-10-09T10:31:02.081 INFO:teuthology.orchestra.run.smithi125.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1736sec hoplimit 64 pref medium 2023-10-09T10:31:02.084 INFO:teuthology.orchestra.run.smithi125.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-09T10:31:02.084 INFO:teuthology.orchestra.run.smithi125.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-10-09T10:31:02.084 INFO:teuthology.orchestra.run.smithi125.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-09T10:31:02.084 INFO:teuthology.orchestra.run.smithi125.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-10-09T10:31:02.084 INFO:teuthology.orchestra.run.smithi125.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe8f:d739/64 scope link 2023-10-09T10:31:02.084 INFO:teuthology.orchestra.run.smithi125.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-09T10:31:02.085 INFO:teuthology.orchestra.run.smithi125.stdout:Mon IP `172.21.15.125` is in CIDR network `172.21.0.0/20` 2023-10-09T10:31:02.085 INFO:teuthology.orchestra.run.smithi125.stdout:Mon IP `172.21.15.125` is in CIDR network `172.21.0.0/20` 2023-10-09T10:31:02.085 INFO:teuthology.orchestra.run.smithi125.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-10-09T10:31:02.085 INFO:teuthology.orchestra.run.smithi125.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-09T10:31:02.086 INFO:teuthology.orchestra.run.smithi125.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T10:31:02.437 INFO:teuthology.orchestra.run.smithi125.stdout:/bin/podman: stdout 5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf 2023-10-09T10:31:02.437 INFO:teuthology.orchestra.run.smithi125.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T10:31:02.438 INFO:teuthology.orchestra.run.smithi125.stdout:/bin/podman: stderr Getting image source signatures 2023-10-09T10:31:02.438 INFO:teuthology.orchestra.run.smithi125.stdout:/bin/podman: stderr Copying blob sha256:1b55aded3a89dccf5f05a82d583393d22b3476851fe870b413c77ca909f5f760 2023-10-09T10:31:02.438 INFO:teuthology.orchestra.run.smithi125.stdout:/bin/podman: stderr Copying blob sha256:be9ed931e0f930233626f1c4d225977fbcc33e056e1a8fc67c7f63553d9455a2 2023-10-09T10:31:02.438 INFO:teuthology.orchestra.run.smithi125.stdout:/bin/podman: stderr Copying config sha256:5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf 2023-10-09T10:31:02.438 INFO:teuthology.orchestra.run.smithi125.stdout:/bin/podman: stderr Writing manifest to image destination 2023-10-09T10:31:03.097 INFO:teuthology.orchestra.run.smithi125.stdout:ceph: stdout ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable) 2023-10-09T10:31:03.097 INFO:teuthology.orchestra.run.smithi125.stdout:Ceph version: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable) 2023-10-09T10:31:03.097 INFO:teuthology.orchestra.run.smithi125.stdout:Extracting ceph user uid/gid from container image... 2023-10-09T10:31:03.746 INFO:teuthology.orchestra.run.smithi125.stdout:stat: stdout 167 167 2023-10-09T10:31:03.746 INFO:teuthology.orchestra.run.smithi125.stdout:Creating initial keys... 2023-10-09T10:31:04.404 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph-authtool: stdout AQBo1iNlU2MKAxAAcMDCs4gQ52jy/r2ZdPc7qw== 2023-10-09T10:31:05.025 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph-authtool: stdout AQBo1iNlgkyBKBAAvSfi+JF07Ju4Xbk1OYgl+A== 2023-10-09T10:31:05.645 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph-authtool: stdout AQBp1iNlQSMXERAA4diWrqTMVwD6hsaZZG90Bw== 2023-10-09T10:31:05.646 INFO:teuthology.orchestra.run.smithi125.stdout:Creating initial monmap... 2023-10-09T10:31:06.252 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T10:31:06.252 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-10-09T10:31:06.253 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:31:06.253 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T10:31:06.253 INFO:teuthology.orchestra.run.smithi125.stdout:monmaptool for a [v2:172.21.15.125:3300,v1:172.21.15.125:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T10:31:06.253 INFO:teuthology.orchestra.run.smithi125.stdout:setting min_mon_release = octopus 2023-10-09T10:31:06.253 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/monmaptool: set fsid to daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:31:06.253 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T10:31:06.253 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:31:06.253 INFO:teuthology.orchestra.run.smithi125.stdout:Creating mon... 2023-10-09T10:31:06.868 INFO:teuthology.orchestra.run.smithi125.stdout:create mon.a on 2023-10-09T10:31:06.967 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-10-09T10:31:07.056 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-10-09T10:31:07.150 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-daf690ae-668e-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-daf690ae-668e-11ee-8db5-212e2dc638e7.target. 2023-10-09T10:31:07.151 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-daf690ae-668e-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-daf690ae-668e-11ee-8db5-212e2dc638e7.target. 2023-10-09T10:31:07.280 INFO:teuthology.orchestra.run.smithi125.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T10:31:07.281 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stderr Failed to reset failed state of unit ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-09T10:31:07.373 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-daf690ae-668e-11ee-8db5-212e2dc638e7.target.wants/ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@.service. 2023-10-09T10:31:07.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:07 smithi125 systemd[1]: Starting Ceph mon.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:31:07.746 INFO:teuthology.orchestra.run.smithi125.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T10:31:07.747 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stdout disabled 2023-10-09T10:31:07.754 INFO:teuthology.orchestra.run.smithi125.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T10:31:07.754 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stdout inactive 2023-10-09T10:31:07.754 INFO:teuthology.orchestra.run.smithi125.stdout:firewalld.service is not enabled 2023-10-09T10:31:07.755 INFO:teuthology.orchestra.run.smithi125.stdout:Not possible to enable service . firewalld.service is not available 2023-10-09T10:31:07.755 INFO:teuthology.orchestra.run.smithi125.stdout:Waiting for mon to start... 2023-10-09T10:31:07.755 INFO:teuthology.orchestra.run.smithi125.stdout:Waiting for mon... 2023-10-09T10:31:08.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:07 smithi125 bash[112066]: b488be541f3d200ece03c9b9c356b6f305e9ca991ddcd043753f27ca3523bf29 2023-10-09T10:31:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:07 smithi125 systemd[1]: Started Ceph mon.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:31:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:07 smithi125 ceph-mon[112104]: mkfs daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:31:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:07 smithi125 ceph-mon[112104]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T10:31:08.577 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout cluster: 2023-10-09T10:31:08.578 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout id: daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:31:08.579 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-10-09T10:31:08.579 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 2023-10-09T10:31:08.579 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout services: 2023-10-09T10:31:08.579 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.449395s) 2023-10-09T10:31:08.579 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-10-09T10:31:08.579 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-10-09T10:31:08.579 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 2023-10-09T10:31:08.579 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout data: 2023-10-09T10:31:08.579 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-10-09T10:31:08.579 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-10-09T10:31:08.580 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-10-09T10:31:08.580 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout pgs: 2023-10-09T10:31:08.580 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 2023-10-09T10:31:08.580 INFO:teuthology.orchestra.run.smithi125.stdout:mon is available 2023-10-09T10:31:08.580 INFO:teuthology.orchestra.run.smithi125.stdout:Assimilating anything we can from ceph.conf... 2023-10-09T10:31:09.060 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:08 smithi125 ceph-mon[112104]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T10:31:09.060 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:08 smithi125 ceph-mon[112104]: monmap e1: 1 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0]} removed_ranks: {} 2023-10-09T10:31:09.060 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:08 smithi125 ceph-mon[112104]: fsmap 2023-10-09T10:31:09.060 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:08 smithi125 ceph-mon[112104]: osdmap e1: 0 total, 0 up, 0 in 2023-10-09T10:31:09.060 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:08 smithi125 ceph-mon[112104]: mgrmap e1: no daemons active 2023-10-09T10:31:09.061 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:08 smithi125 ceph-mon[112104]: from='client.? 172.21.15.125:0/4144046919' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-09T10:31:09.353 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 2023-10-09T10:31:09.353 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout [global] 2023-10-09T10:31:09.353 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout fsid = daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:31:09.353 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.125:3300,v1:172.21.15.125:6789] 2023-10-09T10:31:09.353 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-09T10:31:09.353 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-09T10:31:09.353 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-09T10:31:09.353 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-09T10:31:09.353 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 2023-10-09T10:31:09.354 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-09T10:31:09.354 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-09T10:31:09.354 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 2023-10-09T10:31:09.354 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout [osd] 2023-10-09T10:31:09.354 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-09T10:31:09.354 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-09T10:31:09.354 INFO:teuthology.orchestra.run.smithi125.stdout:Generating new minimal ceph.conf... 2023-10-09T10:31:10.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:09 smithi125 ceph-mon[112104]: from='client.? 172.21.15.125:0/1236053850' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T10:31:10.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:09 smithi125 ceph-mon[112104]: from='client.? 172.21.15.125:0/1236053850' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-09T10:31:10.702 INFO:teuthology.orchestra.run.smithi125.stdout:Restarting the monitor... 2023-10-09T10:31:11.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:10 smithi125 systemd[1]: Stopping Ceph mon.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:31:11.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:10 smithi125 ceph-mon[112104]: from='client.? 172.21.15.125:0/1378375891' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:31:11.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[112081]: 2023-10-09T10:31:11.152+0000 7f3755999700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:31:11.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[112081]: 2023-10-09T10:31:11.152+0000 7f3755999700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-09T10:31:12.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:12 smithi125 bash[112432]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T10:31:13.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T10:31:13.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 systemd[1]: Stopped Ceph mon.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:31:13.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 systemd[1]: Starting Ceph mon.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:31:13.452 INFO:teuthology.orchestra.run.smithi125.stdout:Setting mon public_network to 172.21.0.0/20 2023-10-09T10:31:13.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T10:31:13.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 7 2023-10-09T10:31:13.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: pidfile_write: ignore empty --pid-file 2023-10-09T10:31:13.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: load: jerasure load: lrc 2023-10-09T10:31:13.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: RocksDB version: 6.15.5 2023-10-09T10:31:13.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T10:31:13.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Compile date Oct 8 2023 2023-10-09T10:31:13.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: DB SUMMARY 2023-10-09T10:31:13.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: DB Session ID: V5Y6H74YBUJRNS7KCWZT 2023-10-09T10:31:13.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: CURRENT file: CURRENT 2023-10-09T10:31:13.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T10:31:13.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-10-09T10:31:13.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-10-09T10:31:13.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83609 ; 2023-10-09T10:31:13.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.error_if_exists: 0 2023-10-09T10:31:13.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.create_if_missing: 0 2023-10-09T10:31:13.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.paranoid_checks: 1 2023-10-09T10:31:13.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T10:31:13.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.env: 0x5628ca133a80 2023-10-09T10:31:13.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.fs: Posix File System 2023-10-09T10:31:13.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.info_log: 0x5628cbac3d80 2023-10-09T10:31:13.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T10:31:13.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.statistics: (nil) 2023-10-09T10:31:13.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.use_fsync: 0 2023-10-09T10:31:13.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_log_file_size: 0 2023-10-09T10:31:13.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T10:31:13.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T10:31:13.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T10:31:13.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T10:31:13.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.allow_fallocate: 1 2023-10-09T10:31:13.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T10:31:13.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T10:31:13.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.use_direct_reads: 0 2023-10-09T10:31:13.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T10:31:13.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T10:31:13.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.db_log_dir: 2023-10-09T10:31:13.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T10:31:13.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T10:31:13.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T10:31:13.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T10:31:13.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T10:31:13.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T10:31:13.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T10:31:13.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T10:31:13.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T10:31:13.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.write_buffer_manager: 0x5628cc7b6900 2023-10-09T10:31:13.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T10:31:13.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T10:31:13.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T10:31:13.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T10:31:13.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T10:31:13.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T10:31:13.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T10:31:13.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T10:31:13.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T10:31:13.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.unordered_write: 0 2023-10-09T10:31:13.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T10:31:13.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T10:31:13.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T10:31:13.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T10:31:13.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.row_cache: None 2023-10-09T10:31:13.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.wal_filter: None 2023-10-09T10:31:13.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T10:31:13.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T10:31:13.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.preserve_deletes: 0 2023-10-09T10:31:13.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.two_write_queues: 0 2023-10-09T10:31:13.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T10:31:13.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.atomic_flush: 0 2023-10-09T10:31:13.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T10:31:13.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T10:31:13.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T10:31:13.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.log_readahead_size: 0 2023-10-09T10:31:13.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T10:31:13.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T10:31:13.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T10:31:13.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T10:31:13.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T10:31:13.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T10:31:13.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_background_jobs: 2 2023-10-09T10:31:13.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_background_compactions: -1 2023-10-09T10:31:13.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_subcompactions: 1 2023-10-09T10:31:13.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T10:31:13.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T10:31:13.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T10:31:13.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T10:31:13.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T10:31:13.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T10:31:13.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T10:31:13.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T10:31:13.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_open_files: -1 2023-10-09T10:31:13.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T10:31:13.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T10:31:13.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T10:31:13.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T10:31:13.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_background_flushes: -1 2023-10-09T10:31:13.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Compression algorithms supported: 2023-10-09T10:31:13.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T10:31:13.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: kZSTD supported: 0 2023-10-09T10:31:13.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: kXpressCompression supported: 0 2023-10-09T10:31:13.832 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T10:31:13.832 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: kZlibCompression supported: 1 2023-10-09T10:31:13.832 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: kSnappyCompression supported: 1 2023-10-09T10:31:13.832 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: kLZ4Compression supported: 1 2023-10-09T10:31:13.832 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: kBZip2Compression supported: 0 2023-10-09T10:31:13.832 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T10:31:13.832 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-10-09T10:31:13.832 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T10:31:13.832 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T10:31:13.832 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.merge_operator: 2023-10-09T10:31:13.832 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compaction_filter: None 2023-10-09T10:31:13.832 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T10:31:13.833 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T10:31:13.833 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T10:31:13.833 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T10:31:13.833 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5628cba96fa0) 2023-10-09T10:31:13.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: cache_index_and_filter_blocks: 1 2023-10-09T10:31:13.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T10:31:13.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T10:31:13.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: pin_top_level_index_and_filter: 1 2023-10-09T10:31:13.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: index_type: 0 2023-10-09T10:31:13.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: data_block_index_type: 0 2023-10-09T10:31:13.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: index_shortening: 1 2023-10-09T10:31:13.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T10:31:13.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: hash_index_allow_collision: 1 2023-10-09T10:31:13.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: checksum: 1 2023-10-09T10:31:13.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: no_block_cache: 0 2023-10-09T10:31:13.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_cache: 0x5628cbac9090 2023-10-09T10:31:13.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_cache_name: BinnedLRUCache 2023-10-09T10:31:13.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_cache_options: 2023-10-09T10:31:13.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: capacity : 536870912 2023-10-09T10:31:13.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: num_shard_bits : 4 2023-10-09T10:31:13.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: strict_capacity_limit : 0 2023-10-09T10:31:13.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: high_pri_pool_ratio: 0.000 2023-10-09T10:31:13.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_cache_compressed: (nil) 2023-10-09T10:31:13.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: persistent_cache: (nil) 2023-10-09T10:31:13.835 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_size: 4096 2023-10-09T10:31:13.835 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_size_deviation: 10 2023-10-09T10:31:13.835 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_restart_interval: 16 2023-10-09T10:31:13.835 INFO:journalctl@ceph.mon.a.smithi125.stdout: index_block_restart_interval: 1 2023-10-09T10:31:13.835 INFO:journalctl@ceph.mon.a.smithi125.stdout: metadata_block_size: 4096 2023-10-09T10:31:13.835 INFO:journalctl@ceph.mon.a.smithi125.stdout: partition_filters: 0 2023-10-09T10:31:13.835 INFO:journalctl@ceph.mon.a.smithi125.stdout: use_delta_encoding: 1 2023-10-09T10:31:13.835 INFO:journalctl@ceph.mon.a.smithi125.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T10:31:13.835 INFO:journalctl@ceph.mon.a.smithi125.stdout: whole_key_filtering: 1 2023-10-09T10:31:13.835 INFO:journalctl@ceph.mon.a.smithi125.stdout: verify_compression: 0 2023-10-09T10:31:13.835 INFO:journalctl@ceph.mon.a.smithi125.stdout: read_amp_bytes_per_bit: 0 2023-10-09T10:31:13.836 INFO:journalctl@ceph.mon.a.smithi125.stdout: format_version: 4 2023-10-09T10:31:13.836 INFO:journalctl@ceph.mon.a.smithi125.stdout: enable_index_compression: 1 2023-10-09T10:31:13.836 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_align: 0 2023-10-09T10:31:13.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T10:31:13.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T10:31:13.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compression: NoCompression 2023-10-09T10:31:13.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T10:31:13.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T10:31:13.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T10:31:13.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.num_levels: 7 2023-10-09T10:31:13.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T10:31:13.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T10:31:13.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T10:31:13.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T10:31:13.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T10:31:13.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T10:31:13.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T10:31:13.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T10:31:13.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T10:31:13.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T10:31:13.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T10:31:13.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T10:31:13.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T10:31:13.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T10:31:13.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T10:31:13.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T10:31:13.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T10:31:13.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T10:31:13.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T10:31:13.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T10:31:13.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T10:31:13.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T10:31:13.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T10:31:13.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T10:31:13.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T10:31:13.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T10:31:13.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T10:31:13.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T10:31:13.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T10:31:13.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T10:31:13.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T10:31:13.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T10:31:13.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T10:31:13.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T10:31:13.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.arena_block_size: 4194304 2023-10-09T10:31:13.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T10:31:13.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T10:31:13.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T10:31:13.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T10:31:13.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T10:31:13.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T10:31:13.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T10:31:13.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T10:31:13.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T10:31:13.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T10:31:13.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T10:31:13.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T10:31:13.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T10:31:13.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T10:31:13.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.table_properties_collectors: 2023-10-09T10:31:13.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.inplace_update_support: 0 2023-10-09T10:31:13.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T10:31:13.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T10:31:13.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T10:31:13.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T10:31:13.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.bloom_locality: 0 2023-10-09T10:31:13.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.max_successive_merges: 0 2023-10-09T10:31:13.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T10:31:13.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T10:31:13.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T10:31:13.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T10:31:13.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.ttl: 2592000 2023-10-09T10:31:13.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T10:31:13.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.enable_blob_files: false 2023-10-09T10:31:13.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.min_blob_size: 0 2023-10-09T10:31:13.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T10:31:13.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T10:31:13.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T10:31:13.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T10:31:13.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T10:31:13.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-10-09T10:31:13.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-10-09T10:31:13.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696847473425310, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-10-09T10:31:13.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-10-09T10:31:13.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T10:31:13.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696847473425960, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80156, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78424, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72632, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696847473, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "ec023398-428d-4df6-a06d-e76b997c1785", "db_session_id": "V5Y6H74YBUJRNS7KCWZT"}} 2023-10-09T10:31:13.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-10-09T10:31:13.844 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696847473426294, "job": 1, "event": "recovery_finished"} 2023-10-09T10:31:13.844 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T10:31:13.844 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5628cba5d180 2023-10-09T10:31:13.844 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: DB pointer 0x5628cbb32000 2023-10-09T10:31:13.844 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T10:31:13.844 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T10:31:13.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: ** DB Stats ** 2023-10-09T10:31:13.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T10:31:13.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T10:31:13.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T10:31:13.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T10:31:13.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T10:31:13.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T10:31:13.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T10:31:13.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:31:13.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: ** Compaction Stats [default] ** 2023-10-09T10:31:13.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T10:31:13.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T10:31:13.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: L0 2/0 80.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 182.4 0.00 0.00 1 0.000 0 0 2023-10-09T10:31:13.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: Sum 2/0 80.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 182.4 0.00 0.00 1 0.000 0 0 2023-10-09T10:31:13.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 182.4 0.00 0.00 1 0.000 0 0 2023-10-09T10:31:13.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:31:13.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: ** Compaction Stats [default] ** 2023-10-09T10:31:13.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T10:31:13.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T10:31:13.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 182.4 0.00 0.00 1 0.000 0 0 2023-10-09T10:31:13.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T10:31:13.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T10:31:13.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T10:31:13.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T10:31:13.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T10:31:13.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T10:31:13.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: Cumulative compaction: 0.00 GB write, 26.36 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:31:13.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: Interval compaction: 0.00 GB write, 26.36 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:31:13.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T10:31:13.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:31:13.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T10:31:13.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:31:13.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: ** Compaction Stats [default] ** 2023-10-09T10:31:13.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T10:31:13.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T10:31:13.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: L0 2/0 80.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 182.4 0.00 0.00 1 0.000 0 0 2023-10-09T10:31:13.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: Sum 2/0 80.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 182.4 0.00 0.00 1 0.000 0 0 2023-10-09T10:31:13.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T10:31:13.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:31:13.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: ** Compaction Stats [default] ** 2023-10-09T10:31:13.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T10:31:13.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T10:31:13.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 182.4 0.00 0.00 1 0.000 0 0 2023-10-09T10:31:13.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T10:31:13.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T10:31:13.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T10:31:13.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T10:31:13.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T10:31:13.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T10:31:13.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: Cumulative compaction: 0.00 GB write, 26.09 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:31:13.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:31:13.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T10:31:13.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:31:13.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T10:31:13.849 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: starting mon.a rank 0 at public addrs [v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0] at bind addrs [v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:31:13.849 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: mon.a@-1(???) e1 preinit fsid daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:31:13.849 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: mon.a@-1(???).mds e1 new map 2023-10-09T10:31:13.849 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: mon.a@-1(???).mds e1 print_map 2023-10-09T10:31:13.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: e1 2023-10-09T10:31:13.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T10:31:13.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T10:31:13.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: legacy client fscid: -1 2023-10-09T10:31:13.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:31:13.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: No filesystems configured 2023-10-09T10:31:13.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T10:31:13.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T10:31:13.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T10:31:13.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T10:31:13.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-09T10:31:13.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: expand_channel_meta expand map: {default=false} 2023-10-09T10:31:13.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: expand_channel_meta from 'false' to 'false' 2023-10-09T10:31:13.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: expand_channel_meta expanded map: {default=false} 2023-10-09T10:31:13.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: expand_channel_meta expand map: {default=info} 2023-10-09T10:31:13.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: expand_channel_meta from 'info' to 'info' 2023-10-09T10:31:13.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: expand_channel_meta expanded map: {default=info} 2023-10-09T10:31:13.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: expand_channel_meta expand map: {default=daemon} 2023-10-09T10:31:13.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T10:31:13.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T10:31:13.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: expand_channel_meta expand map: {default=debug} 2023-10-09T10:31:13.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T10:31:13.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: expand_channel_meta expanded map: {default=debug} 2023-10-09T10:31:13.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 bash[112552]: 310e442a0e90bde1cfcc23011aa4c7aa835a739aed90ce1dcb02fa13560a4e31 2023-10-09T10:31:13.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 systemd[1]: Started Ceph mon.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:31:13.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T10:31:13.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: monmap e1: 1 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0]} removed_ranks: {} 2023-10-09T10:31:13.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: fsmap 2023-10-09T10:31:13.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: osdmap e1: 0 total, 0 up, 0 in 2023-10-09T10:31:13.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:13 smithi125 ceph-mon[112589]: mgrmap e1: no daemons active 2023-10-09T10:31:14.201 INFO:teuthology.orchestra.run.smithi125.stdout:Wrote config to /etc/ceph/ceph.conf 2023-10-09T10:31:14.201 INFO:teuthology.orchestra.run.smithi125.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-09T10:31:14.201 INFO:teuthology.orchestra.run.smithi125.stdout:Creating mgr... 2023-10-09T10:31:14.202 INFO:teuthology.orchestra.run.smithi125.stdout:Verifying port 9283 ... 2023-10-09T10:31:14.325 INFO:teuthology.orchestra.run.smithi125.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T10:31:14.326 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stderr Failed to reset failed state of unit ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.y.service: Unit ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.y.service not loaded. 2023-10-09T10:31:14.415 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-daf690ae-668e-11ee-8db5-212e2dc638e7.target.wants/ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@.service. 2023-10-09T10:31:14.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:14 smithi125 systemd[1]: Starting Ceph mgr.y for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:31:14.839 INFO:teuthology.orchestra.run.smithi125.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T10:31:14.839 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stdout disabled 2023-10-09T10:31:14.847 INFO:teuthology.orchestra.run.smithi125.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T10:31:14.848 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stdout inactive 2023-10-09T10:31:14.848 INFO:teuthology.orchestra.run.smithi125.stdout:firewalld.service is not enabled 2023-10-09T10:31:14.848 INFO:teuthology.orchestra.run.smithi125.stdout:Not possible to enable service . firewalld.service is not available 2023-10-09T10:31:14.856 INFO:teuthology.orchestra.run.smithi125.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T10:31:14.857 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stdout disabled 2023-10-09T10:31:14.865 INFO:teuthology.orchestra.run.smithi125.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T10:31:14.865 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stdout inactive 2023-10-09T10:31:14.865 INFO:teuthology.orchestra.run.smithi125.stdout:firewalld.service is not enabled 2023-10-09T10:31:14.865 INFO:teuthology.orchestra.run.smithi125.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-10-09T10:31:14.865 INFO:teuthology.orchestra.run.smithi125.stdout:Waiting for mgr to start... 2023-10-09T10:31:14.865 INFO:teuthology.orchestra.run.smithi125.stdout:Waiting for mgr... 2023-10-09T10:31:15.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:14 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/37120849' entity='client.admin' 2023-10-09T10:31:15.075 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:14 smithi125 bash[112823]: e55280bfaf2525bb14a966a4d6edd41bcfb9d1ca0780930106ac2513784c0c0c 2023-10-09T10:31:15.075 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:14 smithi125 systemd[1]: Started Ceph mgr.y for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:31:15.075 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:14.954+0000 7f5cd9f131c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T10:31:15.460 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:15 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:15.125+0000 7f5cd9f131c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T10:31:15.778 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:15 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:15.485+0000 7f5cd9f131c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T10:31:15.825 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 2023-10-09T10:31:15.825 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout { 2023-10-09T10:31:15.825 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "fsid": "daf690ae-668e-11ee-8db5-212e2dc638e7", 2023-10-09T10:31:15.825 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T10:31:15.826 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T10:31:15.826 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T10:31:15.826 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T10:31:15.826 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:15.826 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T10:31:15.826 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T10:31:15.826 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 0 2023-10-09T10:31:15.826 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout ], 2023-10-09T10:31:15.826 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T10:31:15.826 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T10:31:15.827 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout ], 2023-10-09T10:31:15.827 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "quorum_age": 1, 2023-10-09T10:31:15.827 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T10:31:15.827 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:15.827 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T10:31:15.827 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T10:31:15.827 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:15.827 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T10:31:15.827 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:15.827 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T10:31:15.827 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T10:31:15.827 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T10:31:15.828 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T10:31:15.828 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T10:31:15.828 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T10:31:15.828 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:15.828 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T10:31:15.828 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T10:31:15.828 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T10:31:15.828 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T10:31:15.828 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T10:31:15.828 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T10:31:15.828 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T10:31:15.828 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T10:31:15.828 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T10:31:15.829 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:15.829 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T10:31:15.829 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:15.829 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T10:31:15.829 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T10:31:15.829 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:15.829 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T10:31:15.829 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T10:31:15.829 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T10:31:15.829 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T10:31:15.829 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T10:31:15.829 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T10:31:15.830 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T10:31:15.830 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout ], 2023-10-09T10:31:15.830 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T10:31:15.830 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:15.830 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T10:31:15.830 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:15.830 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T10:31:07.768278+0000", 2023-10-09T10:31:15.830 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T10:31:15.830 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:15.830 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T10:31:15.830 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout } 2023-10-09T10:31:15.831 INFO:teuthology.orchestra.run.smithi125.stdout:mgr not available, waiting (1/15)... 2023-10-09T10:31:16.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:15 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2238743247' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T10:31:16.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:16 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:16.018+0000 7f5cd9f131c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T10:31:16.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:16 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:16.184+0000 7f5cd9f131c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T10:31:17.820 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:16 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:16.570+0000 7f5cd9f131c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T10:31:17.821 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:17 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:17.321+0000 7f5cd9f131c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T10:31:17.821 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:17 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:17.405+0000 7f5cd9f131c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T10:31:18.279 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:18 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:18.011+0000 7f5cd9f131c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T10:31:18.280 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:18 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:18.096+0000 7f5cd9f131c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T10:31:18.280 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:18 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:18.185+0000 7f5cd9f131c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T10:31:18.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:18 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1207262172' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T10:31:18.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:18 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:18.279+0000 7f5cd9f131c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T10:31:18.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:18 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:18.509+0000 7f5cd9f131c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T10:31:18.771 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 2023-10-09T10:31:18.771 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout { 2023-10-09T10:31:18.771 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "fsid": "daf690ae-668e-11ee-8db5-212e2dc638e7", 2023-10-09T10:31:18.772 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T10:31:18.772 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T10:31:18.772 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T10:31:18.772 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T10:31:18.772 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:18.772 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T10:31:18.772 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T10:31:18.772 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 0 2023-10-09T10:31:18.772 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout ], 2023-10-09T10:31:18.772 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T10:31:18.772 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T10:31:18.772 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout ], 2023-10-09T10:31:18.773 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "quorum_age": 4, 2023-10-09T10:31:18.773 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T10:31:18.773 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:18.773 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T10:31:18.773 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T10:31:18.773 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:18.773 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T10:31:18.773 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:18.773 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T10:31:18.773 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T10:31:18.773 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T10:31:18.773 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T10:31:18.774 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T10:31:18.774 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T10:31:18.774 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:18.774 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T10:31:18.774 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T10:31:18.774 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T10:31:18.774 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T10:31:18.774 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T10:31:18.774 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T10:31:18.774 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T10:31:18.774 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T10:31:18.774 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T10:31:18.775 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:18.775 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T10:31:18.775 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:18.775 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T10:31:18.775 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T10:31:18.775 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:18.775 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T10:31:18.775 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T10:31:18.775 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T10:31:18.775 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T10:31:18.775 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T10:31:18.775 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T10:31:18.776 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T10:31:18.776 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout ], 2023-10-09T10:31:18.776 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T10:31:18.776 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:18.776 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T10:31:18.776 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:18.776 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T10:31:07.768278+0000", 2023-10-09T10:31:18.776 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T10:31:18.776 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:18.776 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T10:31:18.776 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout } 2023-10-09T10:31:18.776 INFO:teuthology.orchestra.run.smithi125.stdout:mgr not available, waiting (2/15)... 2023-10-09T10:31:18.908 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:18 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:18.603+0000 7f5cd9f131c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T10:31:18.908 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:18 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:18.814+0000 7f5cd9f131c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T10:31:19.167 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:18 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:18.907+0000 7f5cd9f131c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T10:31:19.167 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:19 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:19.084+0000 7f5cd9f131c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T10:31:19.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:19 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:19.166+0000 7f5cd9f131c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T10:31:20.252 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:19 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:19.895+0000 7f5cd9f131c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T10:31:20.253 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:20 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:20.080+0000 7f5cd9f131c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T10:31:20.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:20 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:20.251+0000 7f5cd9f131c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:31:21.212 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:20 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:20.845+0000 7f5cd9f131c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T10:31:21.212 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:21.028+0000 7f5cd9f131c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:31:21.535 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:21 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1981207542' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T10:31:21.535 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:21.211+0000 7f5cd9f131c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T10:31:21.535 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:21.329+0000 7f5cd9f131c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T10:31:21.571 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 2023-10-09T10:31:21.571 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout { 2023-10-09T10:31:21.571 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "fsid": "daf690ae-668e-11ee-8db5-212e2dc638e7", 2023-10-09T10:31:21.572 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T10:31:21.572 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T10:31:21.572 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T10:31:21.572 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T10:31:21.572 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:21.572 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T10:31:21.572 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T10:31:21.572 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 0 2023-10-09T10:31:21.572 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout ], 2023-10-09T10:31:21.572 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T10:31:21.572 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T10:31:21.572 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout ], 2023-10-09T10:31:21.573 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "quorum_age": 7, 2023-10-09T10:31:21.573 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T10:31:21.573 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:21.573 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T10:31:21.573 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T10:31:21.573 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:21.573 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T10:31:21.573 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:21.573 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T10:31:21.573 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T10:31:21.573 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T10:31:21.573 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T10:31:21.574 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T10:31:21.574 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T10:31:21.574 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:21.574 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T10:31:21.574 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T10:31:21.574 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T10:31:21.574 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T10:31:21.574 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T10:31:21.574 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T10:31:21.574 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T10:31:21.574 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T10:31:21.575 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T10:31:21.575 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:21.575 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T10:31:21.575 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:21.575 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T10:31:21.575 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T10:31:21.575 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:21.575 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T10:31:21.575 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T10:31:21.575 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T10:31:21.575 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T10:31:21.575 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T10:31:21.576 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T10:31:21.576 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T10:31:21.576 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout ], 2023-10-09T10:31:21.576 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T10:31:21.576 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:21.576 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T10:31:21.576 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:21.576 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T10:31:07.768278+0000", 2023-10-09T10:31:21.576 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T10:31:21.576 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:21.576 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T10:31:21.576 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout } 2023-10-09T10:31:21.577 INFO:teuthology.orchestra.run.smithi125.stdout:mgr not available, waiting (3/15)... 2023-10-09T10:31:22.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:22 smithi125 ceph-mon[112589]: Activating manager daemon y 2023-10-09T10:31:22.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:22 smithi125 ceph-mon[112589]: mgrmap e2: y(active, starting, since 0.0030051s) 2023-10-09T10:31:22.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:22 smithi125 ceph-mon[112589]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:31:22.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:22 smithi125 ceph-mon[112589]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:31:22.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:22 smithi125 ceph-mon[112589]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:31:22.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:22 smithi125 ceph-mon[112589]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:31:22.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:22 smithi125 ceph-mon[112589]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:31:22.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:22 smithi125 ceph-mon[112589]: Manager daemon y is now available 2023-10-09T10:31:22.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:22 smithi125 ceph-mon[112589]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:31:22.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:22 smithi125 ceph-mon[112589]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T10:31:22.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:22 smithi125 ceph-mon[112589]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' 2023-10-09T10:31:22.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:22 smithi125 ceph-mon[112589]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' 2023-10-09T10:31:22.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:22 smithi125 ceph-mon[112589]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' 2023-10-09T10:31:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:23 smithi125 ceph-mon[112589]: mgrmap e3: y(active, since 1.00846s) 2023-10-09T10:31:25.343 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 2023-10-09T10:31:25.344 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout { 2023-10-09T10:31:25.344 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "fsid": "daf690ae-668e-11ee-8db5-212e2dc638e7", 2023-10-09T10:31:25.344 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T10:31:25.344 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T10:31:25.344 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T10:31:25.344 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T10:31:25.344 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:25.344 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T10:31:25.344 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T10:31:25.344 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 0 2023-10-09T10:31:25.345 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout ], 2023-10-09T10:31:25.345 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T10:31:25.345 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T10:31:25.345 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout ], 2023-10-09T10:31:25.345 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "quorum_age": 11, 2023-10-09T10:31:25.345 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T10:31:25.345 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:25.345 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T10:31:25.345 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T10:31:25.345 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:25.345 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T10:31:25.345 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:25.346 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T10:31:25.346 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T10:31:25.346 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T10:31:25.346 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T10:31:25.346 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T10:31:25.346 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T10:31:25.349 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:25.349 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T10:31:25.349 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T10:31:25.349 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T10:31:25.349 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T10:31:25.350 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T10:31:25.350 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T10:31:25.350 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T10:31:25.350 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T10:31:25.350 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T10:31:25.350 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:25.350 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T10:31:25.350 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:25.350 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T10:31:25.350 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T10:31:25.351 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:25.351 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T10:31:25.351 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T10:31:25.351 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T10:31:25.351 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T10:31:25.351 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T10:31:25.351 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T10:31:25.351 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T10:31:25.351 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout ], 2023-10-09T10:31:25.351 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T10:31:25.351 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:25.351 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T10:31:25.351 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T10:31:25.351 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T10:31:07.768278+0000", 2023-10-09T10:31:25.352 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T10:31:25.352 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout }, 2023-10-09T10:31:25.352 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T10:31:25.352 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout } 2023-10-09T10:31:25.352 INFO:teuthology.orchestra.run.smithi125.stdout:mgr is available 2023-10-09T10:31:25.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:25 smithi125 ceph-mon[112589]: mgrmap e4: y(active, since 3s) 2023-10-09T10:31:25.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:25 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2255373679' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T10:31:26.225 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 2023-10-09T10:31:26.225 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout [global] 2023-10-09T10:31:26.225 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout fsid = daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:31:26.225 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-09T10:31:26.225 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-09T10:31:26.226 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-09T10:31:26.226 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-09T10:31:26.226 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 2023-10-09T10:31:26.226 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-09T10:31:26.226 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-09T10:31:26.226 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 2023-10-09T10:31:26.226 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout [osd] 2023-10-09T10:31:26.226 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-09T10:31:26.226 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-09T10:31:26.226 INFO:teuthology.orchestra.run.smithi125.stdout:Enabling cephadm module... 2023-10-09T10:31:26.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:26 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2878275609' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T10:31:27.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:27 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1753067490' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-09T10:31:27.645 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:27 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ignoring --setuser ceph since I am not root 2023-10-09T10:31:27.645 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:27 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ignoring --setgroup ceph since I am not root 2023-10-09T10:31:27.645 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:27 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:27.507+0000 7fc8a6a911c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T10:31:27.983 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:27 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:27.669+0000 7fc8a6a911c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T10:31:28.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:27 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:27.982+0000 7fc8a6a911c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T10:31:28.642 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:28 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:28.525+0000 7fc8a6a911c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T10:31:28.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:28 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1753067490' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-09T10:31:28.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:28 smithi125 ceph-mon[112589]: mgrmap e5: y(active, since 6s) 2023-10-09T10:31:28.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:28 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1063816205' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T10:31:28.674 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout { 2023-10-09T10:31:28.675 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-10-09T10:31:28.675 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T10:31:28.675 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-10-09T10:31:28.675 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-09T10:31:28.675 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout } 2023-10-09T10:31:28.675 INFO:teuthology.orchestra.run.smithi125.stdout:Waiting for the mgr to restart... 2023-10-09T10:31:28.675 INFO:teuthology.orchestra.run.smithi125.stdout:Waiting for mgr epoch 5... 2023-10-09T10:31:29.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:28 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:28.697+0000 7fc8a6a911c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T10:31:29.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:29 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:29.107+0000 7fc8a6a911c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T10:31:30.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:29 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:29.861+0000 7fc8a6a911c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T10:31:30.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:29 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:29.945+0000 7fc8a6a911c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T10:31:30.803 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:30 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:30.540+0000 7fc8a6a911c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T10:31:30.804 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:30 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:30.623+0000 7fc8a6a911c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T10:31:30.804 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:30 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:30.709+0000 7fc8a6a911c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T10:31:30.804 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:30 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:30.802+0000 7fc8a6a911c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T10:31:31.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:31.040+0000 7fc8a6a911c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T10:31:31.433 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:31.130+0000 7fc8a6a911c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T10:31:31.433 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:31.338+0000 7fc8a6a911c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T10:31:31.695 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:31.431+0000 7fc8a6a911c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T10:31:31.695 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:31.611+0000 7fc8a6a911c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T10:31:32.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:31.693+0000 7fc8a6a911c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T10:31:32.783 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:32 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:32.423+0000 7fc8a6a911c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T10:31:32.783 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:32 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:32.610+0000 7fc8a6a911c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T10:31:33.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:32 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:32.781+0000 7fc8a6a911c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:31:33.745 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:33 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:33.391+0000 7fc8a6a911c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T10:31:33.745 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:33 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:33.567+0000 7fc8a6a911c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:31:34.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:33 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:33.744+0000 7fc8a6a911c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T10:31:34.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:33 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:33.857+0000 7fc8a6a911c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T10:31:34.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: Active manager daemon y restarted 2023-10-09T10:31:34.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: Activating manager daemon y 2023-10-09T10:31:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: osdmap e2: 0 total, 0 up, 0 in 2023-10-09T10:31:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: mgrmap e6: y(active, starting, since 0.00331106s) 2023-10-09T10:31:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:31:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:31:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:31:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:31:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:31:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: Manager daemon y is now available 2023-10-09T10:31:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: Found migration_current of "None". Setting to last migration. 2023-10-09T10:31:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:31:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:31:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:31:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:31:34.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:31:34.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:31:34.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:34 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T10:31:36.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:35 smithi125 ceph-mon[112589]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T10:31:36.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:35 smithi125 ceph-mon[112589]: mgrmap e7: y(active, since 1.00561s) 2023-10-09T10:31:36.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:35 smithi125 ceph-mon[112589]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T10:31:37.140 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout { 2023-10-09T10:31:37.140 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-10-09T10:31:37.140 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-09T10:31:37.140 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout } 2023-10-09T10:31:37.140 INFO:teuthology.orchestra.run.smithi125.stdout:mgr epoch 5 is available 2023-10-09T10:31:37.140 INFO:teuthology.orchestra.run.smithi125.stdout:Setting orchestrator backend to cephadm... 2023-10-09T10:31:37.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:37 smithi125 ceph-mon[112589]: [09/Oct/2023:10:31:35] ENGINE Bus STARTING 2023-10-09T10:31:37.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:37 smithi125 ceph-mon[112589]: [09/Oct/2023:10:31:36] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T10:31:37.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:37 smithi125 ceph-mon[112589]: [09/Oct/2023:10:31:36] ENGINE Bus STARTED 2023-10-09T10:31:37.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:37 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:31:37.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:37 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:31:37.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:37 smithi125 ceph-mon[112589]: mgrmap e8: y(active, since 3s) 2023-10-09T10:31:39.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:38 smithi125 ceph-mon[112589]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:31:39.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:38 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:31:39.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:38 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:31:39.147 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout value unchanged 2023-10-09T10:31:39.147 INFO:teuthology.orchestra.run.smithi125.stdout:Generating ssh key... 2023-10-09T10:31:40.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-mon[112589]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:31:40.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:31:40.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:31:40.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: Generating public/private rsa key pair. 2023-10-09T10:31:40.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: Your identification has been saved in /tmp/tmpmflxt22_/key. 2023-10-09T10:31:40.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: Your public key has been saved in /tmp/tmpmflxt22_/key.pub. 2023-10-09T10:31:40.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: The key fingerprint is: 2023-10-09T10:31:40.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: SHA256:3VqEoAODFhvnMlpNc4tIJkMbODinBrZMtdlLNR5s96A ceph-daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:31:40.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: The key's randomart image is: 2023-10-09T10:31:40.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: +---[RSA 3072]----+ 2023-10-09T10:31:40.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: |*o*+* o+. | 2023-10-09T10:31:40.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: |*B*O+*o=o+ . | 2023-10-09T10:31:40.072 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: |=BBo+o=.o + . | 2023-10-09T10:31:40.072 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: |.* o. .E . + | 2023-10-09T10:31:40.072 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: |o . S . o | 2023-10-09T10:31:40.072 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: | o | 2023-10-09T10:31:40.072 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: | . | 2023-10-09T10:31:40.072 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: | | 2023-10-09T10:31:40.072 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: | | 2023-10-09T10:31:40.072 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: +----[SHA256]-----+ 2023-10-09T10:31:40.982 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:40 smithi125 ceph-mon[112589]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:31:40.982 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:40 smithi125 ceph-mon[112589]: Generating ssh key... 2023-10-09T10:31:41.005 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNA8Hc4He/jqD2r0FCG+c8gdB9dVFreFSKdqHS/B25KoCnwZ77Z887UAtjUrScHzFGvXu7Y2phDXfI840SAfVmohsKpb2LED1J899Eeo78uZhLex7x3guHUGfnPC6r/pItzq3CdqYAe01lBePFRrRnjCk4hqOk5MNuBAaOYvLZiCJW3DiLQ9rq1pYtFzVU8wfG0KH+sprEfy01HmNh2fncS4WoRm9BpVWH3WKyAgu/ogHTJg/U4eFKTQmrK4mXjW/j9AGcQyUEv3RRGytNZp/Lr94QRunkyxv0FiIPfLpSowM/kpCzalI6uMR85egOhY+N33Mfc8G0JGzvVQFWwuOFpfcYkls3O+k2nplc/oxwp32jAQkb3GiSsXKhqm04gyBMbvW8iiQBNy6cgfhES4BQ+B5mG2HK1xE1rTdN0GF//NrKPB6h88sHiY4IAVws4N7r8Mcyj7HkqO0lVrvQjo1JKyisogEBQVefRJ004QPuwhqLoEXWN6CfOMnxlpIq3cU= ceph-daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:31:41.005 INFO:teuthology.orchestra.run.smithi125.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-09T10:31:41.006 INFO:teuthology.orchestra.run.smithi125.stdout:Adding key to root@localhost authorized_keys... 2023-10-09T10:31:41.006 INFO:teuthology.orchestra.run.smithi125.stdout:Adding host smithi125... 2023-10-09T10:31:41.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:41 smithi125 ceph-mon[112589]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:31:43.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:42 smithi125 ceph-mon[112589]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi125", "addr": "172.21.15.125", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:31:43.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:42 smithi125 ceph-mon[112589]: Deploying cephadm binary to smithi125 2023-10-09T10:31:43.608 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout Added host 'smithi125' with addr '172.21.15.125' 2023-10-09T10:31:43.608 INFO:teuthology.orchestra.run.smithi125.stdout:Deploying unmanaged mon service... 2023-10-09T10:31:44.545 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:44 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:31:44.545 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:44 smithi125 ceph-mon[112589]: Added host smithi125 2023-10-09T10:31:44.545 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:44 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:31:44.545 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:44 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:31:44.813 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-10-09T10:31:44.813 INFO:teuthology.orchestra.run.smithi125.stdout:Deploying unmanaged mgr service... 2023-10-09T10:31:45.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:45 smithi125 ceph-mon[112589]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:31:45.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:45 smithi125 ceph-mon[112589]: Saving service mon spec with placement count:5 2023-10-09T10:31:45.938 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-10-09T10:31:46.803 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:46 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:31:46.803 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:46 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:31:46.803 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:46 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:31:47.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:47 smithi125 ceph-mon[112589]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:31:47.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:47 smithi125 ceph-mon[112589]: Saving service mgr spec with placement count:2 2023-10-09T10:31:47.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:47 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3524517480' entity='client.admin' 2023-10-09T10:31:47.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:47 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:31:47.962 INFO:teuthology.orchestra.run.smithi125.stdout:Enabling the dashboard module... 2023-10-09T10:31:48.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:48 smithi125 ceph-mon[112589]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:31:48.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:48 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/4181157997' entity='client.admin' 2023-10-09T10:31:49.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:49 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/624321191' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-09T10:31:49.821 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:49 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ignoring --setuser ceph since I am not root 2023-10-09T10:31:49.821 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:49 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ignoring --setgroup ceph since I am not root 2023-10-09T10:31:49.821 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:49 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:49.698+0000 7f465d9ec1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T10:31:50.179 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:49 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:49.861+0000 7f465d9ec1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T10:31:50.505 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:50.177+0000 7f465d9ec1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T10:31:50.797 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:50.715+0000 7f465d9ec1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T10:31:50.798 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:50 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/624321191' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-09T10:31:50.798 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:50 smithi125 ceph-mon[112589]: mgrmap e9: y(active, since 15s) 2023-10-09T10:31:50.798 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:50 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/4047547662' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T10:31:50.845 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout { 2023-10-09T10:31:50.846 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-10-09T10:31:50.846 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T10:31:50.846 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-10-09T10:31:50.846 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-09T10:31:50.846 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout } 2023-10-09T10:31:50.846 INFO:teuthology.orchestra.run.smithi125.stdout:Waiting for the mgr to restart... 2023-10-09T10:31:50.846 INFO:teuthology.orchestra.run.smithi125.stdout:Waiting for mgr epoch 9... 2023-10-09T10:31:51.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:50.891+0000 7f465d9ec1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T10:31:51.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:51.304+0000 7f465d9ec1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T10:31:52.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:52 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:52.056+0000 7f465d9ec1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T10:31:52.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:52 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:52.141+0000 7f465d9ec1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T10:31:52.992 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:52 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:52.733+0000 7f465d9ec1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T10:31:52.992 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:52 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:52.815+0000 7f465d9ec1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T10:31:52.992 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:52 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:52.901+0000 7f465d9ec1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T10:31:52.992 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:52 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:52.991+0000 7f465d9ec1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T10:31:53.303 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:53 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:53.212+0000 7f465d9ec1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T10:31:53.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:53 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:53.302+0000 7f465d9ec1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T10:31:53.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:53 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:53.512+0000 7f465d9ec1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T10:31:53.868 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:53 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:53.606+0000 7f465d9ec1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T10:31:53.868 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:53 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:53.784+0000 7f465d9ec1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T10:31:53.868 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:53 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:53.867+0000 7f465d9ec1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T10:31:54.960 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:54 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:54.599+0000 7f465d9ec1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T10:31:54.961 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:54 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:54.786+0000 7f465d9ec1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T10:31:54.961 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:54 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:54.959+0000 7f465d9ec1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:31:55.820 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:55 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:55.548+0000 7f465d9ec1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T10:31:55.821 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:55 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:55.724+0000 7f465d9ec1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:31:56.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:55 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:55.902+0000 7f465d9ec1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T10:31:56.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:31:56 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:31:56.015+0000 7f465d9ec1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T10:31:56.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:56 smithi125 ceph-mon[112589]: Active manager daemon y restarted 2023-10-09T10:31:56.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:56 smithi125 ceph-mon[112589]: Activating manager daemon y 2023-10-09T10:31:56.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:56 smithi125 ceph-mon[112589]: osdmap e3: 0 total, 0 up, 0 in 2023-10-09T10:31:56.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:56 smithi125 ceph-mon[112589]: mgrmap e10: y(active, starting, since 0.00548656s) 2023-10-09T10:31:56.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:31:56.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:31:56.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:31:56.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:31:56.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:31:56.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:56 smithi125 ceph-mon[112589]: Manager daemon y is now available 2023-10-09T10:31:56.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:31:56.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:31:57.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:57 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:31:57.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:57 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T10:31:57.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:57 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:31:57.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:57 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:31:57.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:57 smithi125 ceph-mon[112589]: mgrmap e11: y(active, since 1.00521s) 2023-10-09T10:31:57.378 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout { 2023-10-09T10:31:57.378 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-10-09T10:31:57.378 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-09T10:31:57.378 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout } 2023-10-09T10:31:57.378 INFO:teuthology.orchestra.run.smithi125.stdout:mgr epoch 9 is available 2023-10-09T10:31:57.378 INFO:teuthology.orchestra.run.smithi125.stdout:Generating a dashboard self-signed certificate... 2023-10-09T10:31:58.348 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:58 smithi125 ceph-mon[112589]: [09/Oct/2023:10:31:56] ENGINE Bus STARTING 2023-10-09T10:31:58.348 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:58 smithi125 ceph-mon[112589]: [09/Oct/2023:10:31:56] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T10:31:58.348 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:58 smithi125 ceph-mon[112589]: [09/Oct/2023:10:31:56] ENGINE Bus STARTED 2023-10-09T10:31:58.348 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:58 smithi125 ceph-mon[112589]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T10:31:58.348 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:58 smithi125 ceph-mon[112589]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T10:31:58.348 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:58 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:31:59.080 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-10-09T10:31:59.080 INFO:teuthology.orchestra.run.smithi125.stdout:Creating initial admin user... 2023-10-09T10:31:59.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:59 smithi125 ceph-mon[112589]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:31:59.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:59 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:31:59.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:59 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:31:59.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:31:59 smithi125 ceph-mon[112589]: mgrmap e12: y(active, since 2s) 2023-10-09T10:32:00.260 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$Ieq0a6ycZZpX1cB9MSpHluAbPRBl7nSBfzwBknatRcvw/PwiY/xDG", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696847519, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-09T10:32:00.260 INFO:teuthology.orchestra.run.smithi125.stdout:Fetching dashboard port number... 2023-10-09T10:32:01.112 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:00 smithi125 ceph-mon[112589]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:01.112 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:00 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:01.112 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:00 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2012692736' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-09T10:32:01.202 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stdout 8443 2023-10-09T10:32:01.211 INFO:teuthology.orchestra.run.smithi125.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T10:32:01.211 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stdout disabled 2023-10-09T10:32:01.219 INFO:teuthology.orchestra.run.smithi125.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T10:32:01.219 INFO:teuthology.orchestra.run.smithi125.stdout:systemctl: stdout inactive 2023-10-09T10:32:01.219 INFO:teuthology.orchestra.run.smithi125.stdout:firewalld.service is not enabled 2023-10-09T10:32:01.219 INFO:teuthology.orchestra.run.smithi125.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-09T10:32:01.221 INFO:teuthology.orchestra.run.smithi125.stdout:Ceph Dashboard is now available at: 2023-10-09T10:32:01.222 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:32:01.222 INFO:teuthology.orchestra.run.smithi125.stdout: URL: https://smithi125.front.sepia.ceph.com:8443/ 2023-10-09T10:32:01.222 INFO:teuthology.orchestra.run.smithi125.stdout: User: admin 2023-10-09T10:32:01.222 INFO:teuthology.orchestra.run.smithi125.stdout: Password: ghmrxiqd74 2023-10-09T10:32:01.222 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:32:01.222 INFO:teuthology.orchestra.run.smithi125.stdout:Saving cluster configuration to /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config directory 2023-10-09T10:32:01.222 INFO:teuthology.orchestra.run.smithi125.stdout:Enabling autotune for osd_memory_target 2023-10-09T10:32:03.152 INFO:teuthology.orchestra.run.smithi125.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-10-09T10:32:03.152 INFO:teuthology.orchestra.run.smithi125.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-09T10:32:03.152 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:32:03.152 INFO:teuthology.orchestra.run.smithi125.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-09T10:32:03.152 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:32:03.152 INFO:teuthology.orchestra.run.smithi125.stdout:Or, if you are only running a single cluster on this host: 2023-10-09T10:32:03.152 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:32:03.152 INFO:teuthology.orchestra.run.smithi125.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-09T10:32:03.152 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:32:03.153 INFO:teuthology.orchestra.run.smithi125.stdout:Please consider enabling telemetry to help improve Ceph: 2023-10-09T10:32:03.153 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:32:03.153 INFO:teuthology.orchestra.run.smithi125.stdout: ceph telemetry on 2023-10-09T10:32:03.153 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:32:03.153 INFO:teuthology.orchestra.run.smithi125.stdout:For more information see: 2023-10-09T10:32:03.153 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:32:03.153 INFO:teuthology.orchestra.run.smithi125.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-10-09T10:32:03.153 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:32:03.153 INFO:teuthology.orchestra.run.smithi125.stdout:Bootstrap complete. 2023-10-09T10:32:03.184 INFO:tasks.cephadm:Fetching config... 2023-10-09T10:32:03.185 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:32:03.185 DEBUG:teuthology.orchestra.run.smithi125:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-09T10:32:03.203 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-09T10:32:03.203 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:32:03.203 DEBUG:teuthology.orchestra.run.smithi125:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-09T10:32:03.263 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-09T10:32:03.263 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:32:03.263 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd if=/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-09T10:32:03.331 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-09T10:32:03.331 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:32:03.332 DEBUG:teuthology.orchestra.run.smithi125:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-09T10:32:03.358 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:03 smithi125 ceph-mon[112589]: mgrmap e13: y(active, since 6s) 2023-10-09T10:32:03.358 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:03 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3943785909' entity='client.admin' 2023-10-09T10:32:03.376 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-09T10:32:03.376 DEBUG:teuthology.orchestra.run.smithi125:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNA8Hc4He/jqD2r0FCG+c8gdB9dVFreFSKdqHS/B25KoCnwZ77Z887UAtjUrScHzFGvXu7Y2phDXfI840SAfVmohsKpb2LED1J899Eeo78uZhLex7x3guHUGfnPC6r/pItzq3CdqYAe01lBePFRrRnjCk4hqOk5MNuBAaOYvLZiCJW3DiLQ9rq1pYtFzVU8wfG0KH+sprEfy01HmNh2fncS4WoRm9BpVWH3WKyAgu/ogHTJg/U4eFKTQmrK4mXjW/j9AGcQyUEv3RRGytNZp/Lr94QRunkyxv0FiIPfLpSowM/kpCzalI6uMR85egOhY+N33Mfc8G0JGzvVQFWwuOFpfcYkls3O+k2nplc/oxwp32jAQkb3GiSsXKhqm04gyBMbvW8iiQBNy6cgfhES4BQ+B5mG2HK1xE1rTdN0GF//NrKPB6h88sHiY4IAVws4N7r8Mcyj7HkqO0lVrvQjo1JKyisogEBQVefRJ004QPuwhqLoEXWN6CfOMnxlpIq3cU= ceph-daf690ae-668e-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T10:32:03.456 INFO:teuthology.orchestra.run.smithi125.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNA8Hc4He/jqD2r0FCG+c8gdB9dVFreFSKdqHS/B25KoCnwZ77Z887UAtjUrScHzFGvXu7Y2phDXfI840SAfVmohsKpb2LED1J899Eeo78uZhLex7x3guHUGfnPC6r/pItzq3CdqYAe01lBePFRrRnjCk4hqOk5MNuBAaOYvLZiCJW3DiLQ9rq1pYtFzVU8wfG0KH+sprEfy01HmNh2fncS4WoRm9BpVWH3WKyAgu/ogHTJg/U4eFKTQmrK4mXjW/j9AGcQyUEv3RRGytNZp/Lr94QRunkyxv0FiIPfLpSowM/kpCzalI6uMR85egOhY+N33Mfc8G0JGzvVQFWwuOFpfcYkls3O+k2nplc/oxwp32jAQkb3GiSsXKhqm04gyBMbvW8iiQBNy6cgfhES4BQ+B5mG2HK1xE1rTdN0GF//NrKPB6h88sHiY4IAVws4N7r8Mcyj7HkqO0lVrvQjo1JKyisogEBQVefRJ004QPuwhqLoEXWN6CfOMnxlpIq3cU= ceph-daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:32:03.467 DEBUG:teuthology.orchestra.run.smithi155:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNA8Hc4He/jqD2r0FCG+c8gdB9dVFreFSKdqHS/B25KoCnwZ77Z887UAtjUrScHzFGvXu7Y2phDXfI840SAfVmohsKpb2LED1J899Eeo78uZhLex7x3guHUGfnPC6r/pItzq3CdqYAe01lBePFRrRnjCk4hqOk5MNuBAaOYvLZiCJW3DiLQ9rq1pYtFzVU8wfG0KH+sprEfy01HmNh2fncS4WoRm9BpVWH3WKyAgu/ogHTJg/U4eFKTQmrK4mXjW/j9AGcQyUEv3RRGytNZp/Lr94QRunkyxv0FiIPfLpSowM/kpCzalI6uMR85egOhY+N33Mfc8G0JGzvVQFWwuOFpfcYkls3O+k2nplc/oxwp32jAQkb3GiSsXKhqm04gyBMbvW8iiQBNy6cgfhES4BQ+B5mG2HK1xE1rTdN0GF//NrKPB6h88sHiY4IAVws4N7r8Mcyj7HkqO0lVrvQjo1JKyisogEBQVefRJ004QPuwhqLoEXWN6CfOMnxlpIq3cU= ceph-daf690ae-668e-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T10:32:03.514 INFO:teuthology.orchestra.run.smithi155.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNA8Hc4He/jqD2r0FCG+c8gdB9dVFreFSKdqHS/B25KoCnwZ77Z887UAtjUrScHzFGvXu7Y2phDXfI840SAfVmohsKpb2LED1J899Eeo78uZhLex7x3guHUGfnPC6r/pItzq3CdqYAe01lBePFRrRnjCk4hqOk5MNuBAaOYvLZiCJW3DiLQ9rq1pYtFzVU8wfG0KH+sprEfy01HmNh2fncS4WoRm9BpVWH3WKyAgu/ogHTJg/U4eFKTQmrK4mXjW/j9AGcQyUEv3RRGytNZp/Lr94QRunkyxv0FiIPfLpSowM/kpCzalI6uMR85egOhY+N33Mfc8G0JGzvVQFWwuOFpfcYkls3O+k2nplc/oxwp32jAQkb3GiSsXKhqm04gyBMbvW8iiQBNy6cgfhES4BQ+B5mG2HK1xE1rTdN0GF//NrKPB6h88sHiY4IAVws4N7r8Mcyj7HkqO0lVrvQjo1JKyisogEBQVefRJ004QPuwhqLoEXWN6CfOMnxlpIq3cU= ceph-daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:32:03.526 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-09T10:32:03.799 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:32:05.705 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-09T10:32:05.705 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-09T10:32:05.981 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:32:06.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:05 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:06.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:05 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:06.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:05 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi125", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:32:06.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:05 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:06.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:05 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:06.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:05 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:06.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:05 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2470210639' entity='client.admin' 2023-10-09T10:32:06.020 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:05 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:06.020 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:05 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:07.823 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi155 2023-10-09T10:32:07.823 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T10:32:07.824 DEBUG:teuthology.orchestra.run.smithi155:> dd of=/etc/ceph/ceph.conf 2023-10-09T10:32:07.845 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T10:32:07.846 DEBUG:teuthology.orchestra.run.smithi155:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:32:07.907 INFO:tasks.cephadm:Adding host smithi155 to orchestrator... 2023-10-09T10:32:07.908 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi155 2023-10-09T10:32:08.178 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:32:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:08 smithi125 ceph-mon[112589]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:08 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:08 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:08 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:08 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:08 smithi125 ceph-mon[112589]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:32:09.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:09 smithi125 ceph-mon[112589]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:32:09.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:09 smithi125 ceph-mon[112589]: Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:32:09.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:09 smithi125 ceph-mon[112589]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:32:09.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:09 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:09.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:09 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:09.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:09 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:10.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:10 smithi125 ceph-mon[112589]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi155", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:11.158 INFO:teuthology.orchestra.run.smithi125.stdout:Added host 'smithi155' with addr '172.21.15.155' 2023-10-09T10:32:11.520 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-09T10:32:11.778 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:32:11.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:11 smithi125 ceph-mon[112589]: Deploying cephadm binary to smithi155 2023-10-09T10:32:11.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:11 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:11.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:11 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:11.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:11 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:12.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:12 smithi125 ceph-mon[112589]: Added host smithi155 2023-10-09T10:32:12.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:12 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:12.988 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:32:12.988 INFO:teuthology.orchestra.run.smithi125.stdout:[{"addr": "172.21.15.125", "hostname": "smithi125", "labels": [], "status": ""}, {"addr": "172.21.15.155", "hostname": "smithi155", "labels": [], "status": ""}] 2023-10-09T10:32:13.360 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-09T10:32:13.360 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-09T10:32:13.617 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:32:13.915 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:13 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:13.916 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:13 smithi125 ceph-mon[112589]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:32:14.861 INFO:teuthology.orchestra.run.smithi125.stderr:adjusted tunables profile to default 2023-10-09T10:32:15.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:14 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1333312432' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T10:32:15.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:15.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:15.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:15.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:15.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi155", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:32:15.144 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:15.144 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:15.220 INFO:tasks.cephadm:Adding mon.a on smithi125 2023-10-09T10:32:15.220 INFO:tasks.cephadm:Adding mon.c on smithi125 2023-10-09T10:32:15.220 INFO:tasks.cephadm:Adding mon.b on smithi155 2023-10-09T10:32:15.220 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '3;smithi125:172.21.15.125=a;smithi125:[v2:172.21.15.125:3301,v1:172.21.15.125:6790]=c;smithi155:172.21.15.155=b' 2023-10-09T10:32:15.529 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:32:16.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:15 smithi125 ceph-mon[112589]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:32:16.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:15 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1333312432' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T10:32:16.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:15 smithi125 ceph-mon[112589]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T10:32:16.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:15 smithi125 ceph-mon[112589]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:32:16.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:15 smithi125 ceph-mon[112589]: Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:32:16.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:15 smithi125 ceph-mon[112589]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:32:16.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:15 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:16.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:15 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:16.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:15 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:16.679 INFO:teuthology.orchestra.run.smithi155.stdout:Scheduled mon update... 2023-10-09T10:32:17.085 DEBUG:teuthology.orchestra.run.smithi125:mon.c> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.c.service 2023-10-09T10:32:17.088 DEBUG:teuthology.orchestra.run.smithi155:mon.b> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.b.service 2023-10-09T10:32:17.090 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T10:32:17.090 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T10:32:17.119 INFO:journalctl@ceph.mon.b.smithi155.stdout:-- Logs begin at Mon 2023-10-09 10:18:52 UTC. -- 2023-10-09T10:32:17.125 INFO:journalctl@ceph.mon.c.smithi125.stdout:-- Logs begin at Mon 2023-10-09 10:18:49 UTC. -- 2023-10-09T10:32:17.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:17 smithi125 ceph-mon[112589]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:17.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:17.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:17.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:17.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:17.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:17.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:32:17.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:17.584 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:32:18.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:18 smithi125 ceph-mon[112589]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi125:172.21.15.125=a;smithi125:[v2:172.21.15.125:3301,v1:172.21.15.125:6790]=c;smithi155:172.21.15.155=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:18.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:18 smithi125 ceph-mon[112589]: Saving service mon spec with placement smithi125:172.21.15.125=a;smithi125:[v2:172.21.15.125:3301,v1:172.21.15.125:6790]=c;smithi155:172.21.15.155=b;count:3 2023-10-09T10:32:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:18 smithi125 ceph-mon[112589]: Deploying daemon mon.b on smithi155 2023-10-09T10:32:19.178 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:32:19.178 INFO:teuthology.orchestra.run.smithi155.stdout:{"epoch":1,"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","modified":"2023-10-09T10:31:05.901737Z","created":"2023-10-09T10:31:05.901737Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:3300","nonce":0},{"type":"v1","addr":"172.21.15.125:6789","nonce":0}]},"addr":"172.21.15.125:6789/0","public_addr":"172.21.15.125:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-10-09T10:32:19.179 INFO:teuthology.orchestra.run.smithi155.stderr:dumped monmap epoch 1 2023-10-09T10:32:19.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:19 smithi125 ceph-mon[112589]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:19.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:19 smithi125 ceph-mon[112589]: from='client.? 172.21.15.155:0/290160987' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T10:32:20.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 systemd[1]: Started Ceph mon.b for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:32:20.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T10:32:20.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T10:32:20.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T10:32:20.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T10:32:20.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T10:32:20.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.unordered_write: 0 2023-10-09T10:32:20.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T10:32:20.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T10:32:20.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T10:32:20.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T10:32:20.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.row_cache: None 2023-10-09T10:32:20.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.wal_filter: None 2023-10-09T10:32:20.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T10:32:20.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T10:32:20.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.preserve_deletes: 0 2023-10-09T10:32:20.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.two_write_queues: 0 2023-10-09T10:32:20.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T10:32:20.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.atomic_flush: 0 2023-10-09T10:32:20.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T10:32:20.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T10:32:20.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T10:32:20.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.log_readahead_size: 0 2023-10-09T10:32:20.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T10:32:20.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T10:32:20.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T10:32:20.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T10:32:20.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T10:32:20.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T10:32:20.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_background_jobs: 2 2023-10-09T10:32:20.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_background_compactions: -1 2023-10-09T10:32:20.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_subcompactions: 1 2023-10-09T10:32:20.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T10:32:20.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T10:32:20.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T10:32:20.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T10:32:20.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T10:32:20.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T10:32:20.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T10:32:20.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T10:32:20.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_open_files: -1 2023-10-09T10:32:20.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T10:32:20.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T10:32:20.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T10:32:20.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T10:32:20.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_background_flushes: -1 2023-10-09T10:32:20.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Compression algorithms supported: 2023-10-09T10:32:20.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T10:32:20.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: kZSTD supported: 0 2023-10-09T10:32:20.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: kXpressCompression supported: 0 2023-10-09T10:32:20.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T10:32:20.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: kZlibCompression supported: 1 2023-10-09T10:32:20.226 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: kSnappyCompression supported: 1 2023-10-09T10:32:20.226 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: kLZ4Compression supported: 1 2023-10-09T10:32:20.226 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: kBZip2Compression supported: 0 2023-10-09T10:32:20.226 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T10:32:20.226 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2023-10-09T10:32:20.226 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T10:32:20.226 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T10:32:20.226 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.merge_operator: 2023-10-09T10:32:20.226 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compaction_filter: None 2023-10-09T10:32:20.226 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T10:32:20.226 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T10:32:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T10:32:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T10:32:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5631bb05afa0) 2023-10-09T10:32:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout: cache_index_and_filter_blocks: 1 2023-10-09T10:32:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T10:32:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T10:32:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout: pin_top_level_index_and_filter: 1 2023-10-09T10:32:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout: index_type: 0 2023-10-09T10:32:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout: data_block_index_type: 0 2023-10-09T10:32:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout: index_shortening: 1 2023-10-09T10:32:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T10:32:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout: hash_index_allow_collision: 1 2023-10-09T10:32:20.228 INFO:journalctl@ceph.mon.b.smithi155.stdout: checksum: 1 2023-10-09T10:32:20.228 INFO:journalctl@ceph.mon.b.smithi155.stdout: no_block_cache: 0 2023-10-09T10:32:20.228 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_cache: 0x5631bb08d090 2023-10-09T10:32:20.228 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_cache_name: BinnedLRUCache 2023-10-09T10:32:20.228 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_cache_options: 2023-10-09T10:32:20.228 INFO:journalctl@ceph.mon.b.smithi155.stdout: capacity : 536870912 2023-10-09T10:32:20.228 INFO:journalctl@ceph.mon.b.smithi155.stdout: num_shard_bits : 4 2023-10-09T10:32:20.228 INFO:journalctl@ceph.mon.b.smithi155.stdout: strict_capacity_limit : 0 2023-10-09T10:32:20.228 INFO:journalctl@ceph.mon.b.smithi155.stdout: high_pri_pool_ratio: 0.000 2023-10-09T10:32:20.228 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_cache_compressed: (nil) 2023-10-09T10:32:20.228 INFO:journalctl@ceph.mon.b.smithi155.stdout: persistent_cache: (nil) 2023-10-09T10:32:20.228 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_size: 4096 2023-10-09T10:32:20.229 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_size_deviation: 10 2023-10-09T10:32:20.229 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_restart_interval: 16 2023-10-09T10:32:20.229 INFO:journalctl@ceph.mon.b.smithi155.stdout: index_block_restart_interval: 1 2023-10-09T10:32:20.229 INFO:journalctl@ceph.mon.b.smithi155.stdout: metadata_block_size: 4096 2023-10-09T10:32:20.229 INFO:journalctl@ceph.mon.b.smithi155.stdout: partition_filters: 0 2023-10-09T10:32:20.229 INFO:journalctl@ceph.mon.b.smithi155.stdout: use_delta_encoding: 1 2023-10-09T10:32:20.229 INFO:journalctl@ceph.mon.b.smithi155.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T10:32:20.229 INFO:journalctl@ceph.mon.b.smithi155.stdout: whole_key_filtering: 1 2023-10-09T10:32:20.229 INFO:journalctl@ceph.mon.b.smithi155.stdout: verify_compression: 0 2023-10-09T10:32:20.229 INFO:journalctl@ceph.mon.b.smithi155.stdout: read_amp_bytes_per_bit: 0 2023-10-09T10:32:20.229 INFO:journalctl@ceph.mon.b.smithi155.stdout: format_version: 4 2023-10-09T10:32:20.229 INFO:journalctl@ceph.mon.b.smithi155.stdout: enable_index_compression: 1 2023-10-09T10:32:20.229 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_align: 0 2023-10-09T10:32:20.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T10:32:20.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T10:32:20.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compression: NoCompression 2023-10-09T10:32:20.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T10:32:20.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T10:32:20.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T10:32:20.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.num_levels: 7 2023-10-09T10:32:20.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T10:32:20.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T10:32:20.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T10:32:20.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T10:32:20.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T10:32:20.231 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T10:32:20.231 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T10:32:20.231 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T10:32:20.231 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T10:32:20.231 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T10:32:20.231 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T10:32:20.231 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T10:32:20.231 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T10:32:20.231 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T10:32:20.231 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T10:32:20.231 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T10:32:20.232 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T10:32:20.232 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T10:32:20.232 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T10:32:20.232 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T10:32:20.232 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T10:32:20.232 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T10:32:20.232 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T10:32:20.232 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T10:32:20.232 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T10:32:20.232 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T10:32:20.232 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T10:32:20.232 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T10:32:20.233 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T10:32:20.233 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T10:32:20.233 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T10:32:20.233 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T10:32:20.233 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T10:32:20.233 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T10:32:20.233 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.arena_block_size: 4194304 2023-10-09T10:32:20.233 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T10:32:20.233 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T10:32:20.233 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T10:32:20.233 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T10:32:20.233 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T10:32:20.233 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T10:32:20.234 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T10:32:20.234 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T10:32:20.234 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T10:32:20.234 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T10:32:20.234 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T10:32:20.234 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T10:32:20.234 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T10:32:20.234 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T10:32:20.234 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.table_properties_collectors: 2023-10-09T10:32:20.234 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.inplace_update_support: 0 2023-10-09T10:32:20.234 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T10:32:20.234 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T10:32:20.235 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T10:32:20.235 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T10:32:20.235 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.bloom_locality: 0 2023-10-09T10:32:20.235 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.max_successive_merges: 0 2023-10-09T10:32:20.235 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T10:32:20.236 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T10:32:20.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T10:32:20.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T10:32:20.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.ttl: 2592000 2023-10-09T10:32:20.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T10:32:20.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.enable_blob_files: false 2023-10-09T10:32:20.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.min_blob_size: 0 2023-10-09T10:32:20.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T10:32:20.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T10:32:20.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T10:32:20.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T10:32:20.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T10:32:20.238 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-10-09T10:32:20.238 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-10-09T10:32:20.238 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696847539839259, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-10-09T10:32:20.238 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-10-09T10:32:20.238 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T10:32:20.238 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696847539839654, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696847539, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "ddd781e2-939c-45eb-84eb-7d2c97935899", "db_session_id": "1HQ2YVQIH2NL6ZOGTZ28"}} 2023-10-09T10:32:20.238 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-10-09T10:32:20.238 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696847539840095, "job": 1, "event": "recovery_finished"} 2023-10-09T10:32:20.238 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T10:32:20.238 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5631bb021180 2023-10-09T10:32:20.238 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: DB pointer 0x5631bb0f6000 2023-10-09T10:32:20.238 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T10:32:20.239 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T10:32:20.239 INFO:journalctl@ceph.mon.b.smithi155.stdout: ** DB Stats ** 2023-10-09T10:32:20.239 INFO:journalctl@ceph.mon.b.smithi155.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T10:32:20.239 INFO:journalctl@ceph.mon.b.smithi155.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T10:32:20.239 INFO:journalctl@ceph.mon.b.smithi155.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T10:32:20.239 INFO:journalctl@ceph.mon.b.smithi155.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T10:32:20.239 INFO:journalctl@ceph.mon.b.smithi155.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T10:32:20.239 INFO:journalctl@ceph.mon.b.smithi155.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T10:32:20.239 INFO:journalctl@ceph.mon.b.smithi155.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T10:32:20.239 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:32:20.239 INFO:journalctl@ceph.mon.b.smithi155.stdout: ** Compaction Stats [default] ** 2023-10-09T10:32:20.239 INFO:journalctl@ceph.mon.b.smithi155.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T10:32:20.240 INFO:journalctl@ceph.mon.b.smithi155.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T10:32:20.240 INFO:journalctl@ceph.mon.b.smithi155.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2023-10-09T10:32:20.240 INFO:journalctl@ceph.mon.b.smithi155.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2023-10-09T10:32:20.240 INFO:journalctl@ceph.mon.b.smithi155.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2023-10-09T10:32:20.240 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:32:20.240 INFO:journalctl@ceph.mon.b.smithi155.stdout: ** Compaction Stats [default] ** 2023-10-09T10:32:20.240 INFO:journalctl@ceph.mon.b.smithi155.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T10:32:20.240 INFO:journalctl@ceph.mon.b.smithi155.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T10:32:20.240 INFO:journalctl@ceph.mon.b.smithi155.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2023-10-09T10:32:20.240 INFO:journalctl@ceph.mon.b.smithi155.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T10:32:20.240 INFO:journalctl@ceph.mon.b.smithi155.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T10:32:20.240 INFO:journalctl@ceph.mon.b.smithi155.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T10:32:20.240 INFO:journalctl@ceph.mon.b.smithi155.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T10:32:20.241 INFO:journalctl@ceph.mon.b.smithi155.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T10:32:20.241 INFO:journalctl@ceph.mon.b.smithi155.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T10:32:20.241 INFO:journalctl@ceph.mon.b.smithi155.stdout: Cumulative compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:32:20.241 INFO:journalctl@ceph.mon.b.smithi155.stdout: Interval compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:32:20.241 INFO:journalctl@ceph.mon.b.smithi155.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T10:32:20.241 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:32:20.241 INFO:journalctl@ceph.mon.b.smithi155.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T10:32:20.241 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:32:20.241 INFO:journalctl@ceph.mon.b.smithi155.stdout: ** Compaction Stats [default] ** 2023-10-09T10:32:20.241 INFO:journalctl@ceph.mon.b.smithi155.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T10:32:20.241 INFO:journalctl@ceph.mon.b.smithi155.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T10:32:20.241 INFO:journalctl@ceph.mon.b.smithi155.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2023-10-09T10:32:20.242 INFO:journalctl@ceph.mon.b.smithi155.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2023-10-09T10:32:20.242 INFO:journalctl@ceph.mon.b.smithi155.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T10:32:20.242 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:32:20.242 INFO:journalctl@ceph.mon.b.smithi155.stdout: ** Compaction Stats [default] ** 2023-10-09T10:32:20.242 INFO:journalctl@ceph.mon.b.smithi155.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T10:32:20.242 INFO:journalctl@ceph.mon.b.smithi155.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T10:32:20.242 INFO:journalctl@ceph.mon.b.smithi155.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2023-10-09T10:32:20.242 INFO:journalctl@ceph.mon.b.smithi155.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T10:32:20.242 INFO:journalctl@ceph.mon.b.smithi155.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T10:32:20.242 INFO:journalctl@ceph.mon.b.smithi155.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T10:32:20.242 INFO:journalctl@ceph.mon.b.smithi155.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T10:32:20.242 INFO:journalctl@ceph.mon.b.smithi155.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T10:32:20.243 INFO:journalctl@ceph.mon.b.smithi155.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T10:32:20.243 INFO:journalctl@ceph.mon.b.smithi155.stdout: Cumulative compaction: 0.00 GB write, 0.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:32:20.243 INFO:journalctl@ceph.mon.b.smithi155.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:32:20.243 INFO:journalctl@ceph.mon.b.smithi155.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T10:32:20.243 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:32:20.243 INFO:journalctl@ceph.mon.b.smithi155.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T10:32:20.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-10-09T10:32:20.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: using public_addr v2:172.21.15.155:0/0 -> [v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0] 2023-10-09T10:32:20.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: starting mon.b rank -1 at public addrs [v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0] at bind addrs [v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:32:20.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b@-1(???) e0 preinit fsid daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:32:20.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b@-1(synchronizing).mds e1 new map 2023-10-09T10:32:20.244 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b@-1(synchronizing).mds e1 print_map 2023-10-09T10:32:20.244 INFO:journalctl@ceph.mon.b.smithi155.stdout: e1 2023-10-09T10:32:20.244 INFO:journalctl@ceph.mon.b.smithi155.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T10:32:20.244 INFO:journalctl@ceph.mon.b.smithi155.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T10:32:20.244 INFO:journalctl@ceph.mon.b.smithi155.stdout: legacy client fscid: -1 2023-10-09T10:32:20.244 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:32:20.244 INFO:journalctl@ceph.mon.b.smithi155.stdout: No filesystems configured 2023-10-09T10:32:20.244 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T10:32:20.244 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T10:32:20.244 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T10:32:20.244 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T10:32:20.244 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T10:32:20.244 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T10:32:20.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T10:32:20.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T10:32:20.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T10:32:20.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T10:32:20.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mkfs daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:32:20.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T10:32:20.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T10:32:20.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: monmap e1: 1 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0]} removed_ranks: {} 2023-10-09T10:32:20.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: fsmap 2023-10-09T10:32:20.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: osdmap e1: 0 total, 0 up, 0 in 2023-10-09T10:32:20.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mgrmap e1: no daemons active 2023-10-09T10:32:20.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/4144046919' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-09T10:32:20.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1236053850' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T10:32:20.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1236053850' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-09T10:32:20.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1378375891' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:20.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T10:32:20.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: monmap e1: 1 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0]} removed_ranks: {} 2023-10-09T10:32:20.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: fsmap 2023-10-09T10:32:20.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: osdmap e1: 0 total, 0 up, 0 in 2023-10-09T10:32:20.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mgrmap e1: no daemons active 2023-10-09T10:32:20.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/37120849' entity='client.admin' 2023-10-09T10:32:20.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2238743247' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T10:32:20.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1207262172' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T10:32:20.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1981207542' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T10:32:20.247 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Activating manager daemon y 2023-10-09T10:32:20.247 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mgrmap e2: y(active, starting, since 0.0030051s) 2023-10-09T10:32:20.247 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:32:20.247 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:32:20.247 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:32:20.247 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:32:20.247 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:32:20.247 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Manager daemon y is now available 2023-10-09T10:32:20.247 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:32:20.247 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T10:32:20.247 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' 2023-10-09T10:32:20.247 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' 2023-10-09T10:32:20.248 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14100 172.21.15.125:0/2285238697' entity='mgr.y' 2023-10-09T10:32:20.248 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mgrmap e3: y(active, since 1.00846s) 2023-10-09T10:32:20.248 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mgrmap e4: y(active, since 3s) 2023-10-09T10:32:20.248 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2255373679' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T10:32:20.248 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2878275609' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T10:32:20.248 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1753067490' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-09T10:32:20.248 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1753067490' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-09T10:32:20.248 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mgrmap e5: y(active, since 6s) 2023-10-09T10:32:20.248 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1063816205' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T10:32:20.248 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Active manager daemon y restarted 2023-10-09T10:32:20.248 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Activating manager daemon y 2023-10-09T10:32:20.248 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: osdmap e2: 0 total, 0 up, 0 in 2023-10-09T10:32:20.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mgrmap e6: y(active, starting, since 0.00331106s) 2023-10-09T10:32:20.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:32:20.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:32:20.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:32:20.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:32:20.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:32:20.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Manager daemon y is now available 2023-10-09T10:32:20.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Found migration_current of "None". Setting to last migration. 2023-10-09T10:32:20.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:32:20.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:32:20.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:20.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:20.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:20.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:32:20.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T10:32:20.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T10:32:20.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mgrmap e7: y(active, since 1.00561s) 2023-10-09T10:32:20.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T10:32:20.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: [09/Oct/2023:10:31:35] ENGINE Bus STARTING 2023-10-09T10:32:20.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: [09/Oct/2023:10:31:36] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T10:32:20.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: [09/Oct/2023:10:31:36] ENGINE Bus STARTED 2023-10-09T10:32:20.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:32:20.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:20.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mgrmap e8: y(active, since 3s) 2023-10-09T10:32:20.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:20.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:32:20.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:20.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:20.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:32:20.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:32:20.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:20.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Generating ssh key... 2023-10-09T10:32:20.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:20.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi125", "addr": "172.21.15.125", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:20.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Deploying cephadm binary to smithi125 2023-10-09T10:32:20.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:32:20.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Added host smithi125 2023-10-09T10:32:20.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:20.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:32:20.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:20.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Saving service mon spec with placement count:5 2023-10-09T10:32:20.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:32:20.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:32:20.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:32:20.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:20.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Saving service mgr spec with placement count:2 2023-10-09T10:32:20.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3524517480' entity='client.admin' 2023-10-09T10:32:20.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:32:20.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14120 172.21.15.125:0/2212704781' entity='mgr.y' 2023-10-09T10:32:20.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/4181157997' entity='client.admin' 2023-10-09T10:32:20.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/624321191' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-09T10:32:20.253 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/624321191' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-09T10:32:20.253 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mgrmap e9: y(active, since 15s) 2023-10-09T10:32:20.253 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/4047547662' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T10:32:20.253 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Active manager daemon y restarted 2023-10-09T10:32:20.253 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Activating manager daemon y 2023-10-09T10:32:20.253 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: osdmap e3: 0 total, 0 up, 0 in 2023-10-09T10:32:20.253 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mgrmap e10: y(active, starting, since 0.00548656s) 2023-10-09T10:32:20.253 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:32:20.253 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:32:20.253 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:32:20.253 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:32:20.253 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:32:20.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Manager daemon y is now available 2023-10-09T10:32:20.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:20.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:20.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:32:20.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T10:32:20.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mgrmap e11: y(active, since 1.00521s) 2023-10-09T10:32:20.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: [09/Oct/2023:10:31:56] ENGINE Bus STARTING 2023-10-09T10:32:20.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: [09/Oct/2023:10:31:56] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T10:32:20.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: [09/Oct/2023:10:31:56] ENGINE Bus STARTED 2023-10-09T10:32:20.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T10:32:20.255 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T10:32:20.255 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.255 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:20.255 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.255 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.255 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mgrmap e12: y(active, since 2s) 2023-10-09T10:32:20.255 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:20.255 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.255 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2012692736' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-09T10:32:20.255 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mgrmap e13: y(active, since 6s) 2023-10-09T10:32:20.255 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3943785909' entity='client.admin' 2023-10-09T10:32:20.255 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.255 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.256 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi125", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:32:20.256 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.256 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:20.256 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.256 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2470210639' entity='client.admin' 2023-10-09T10:32:20.256 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:20.256 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.256 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:20.256 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.256 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:20.256 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:20.256 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:20.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:32:20.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:32:20.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:32:20.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:32:20.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi155", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:20.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Deploying cephadm binary to smithi155 2023-10-09T10:32:20.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:20.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Added host smithi155 2023-10-09T10:32:20.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:32:20.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1333312432' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T10:32:20.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi155", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:32:20.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:20.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:20.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:32:20.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1333312432' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T10:32:20.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T10:32:20.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:32:20.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:32:20.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:32:20.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:20.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:20.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:20.260 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:20.260 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:20.260 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:32:20.260 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:20.260 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi125:172.21.15.125=a;smithi125:[v2:172.21.15.125:3301,v1:172.21.15.125:6790]=c;smithi155:172.21.15.155=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:20.260 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Saving service mon spec with placement smithi125:172.21.15.125=a;smithi125:[v2:172.21.15.125:3301,v1:172.21.15.125:6790]=c;smithi155:172.21.15.155=b;count:3 2023-10-09T10:32:20.260 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: Deploying daemon mon.b on smithi155 2023-10-09T10:32:20.260 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:20.260 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.155:0/290160987' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T10:32:20.260 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T10:32:20.260 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:19 smithi155 ceph-mon[117223]: mon.b@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-10-09T10:32:20.537 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T10:32:20.537 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T10:32:20.797 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:32:25.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:32:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: mon.a calling monitor election 2023-10-09T10:32:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: mon.b calling monitor election 2023-10-09T10:32:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T10:32:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: monmap e2: 2 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0]} removed_ranks: {} 2023-10-09T10:32:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: fsmap 2023-10-09T10:32:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T10:32:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: mgrmap e13: y(active, since 28s) 2023-10-09T10:32:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: overall HEALTH_OK 2023-10-09T10:32:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:32:25.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:24 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:25.270 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:32:25.270 INFO:teuthology.orchestra.run.smithi155.stdout:{"epoch":2,"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","modified":"2023-10-09T10:32:19.863262Z","created":"2023-10-09T10:31:05.901737Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:3300","nonce":0},{"type":"v1","addr":"172.21.15.125:6789","nonce":0}]},"addr":"172.21.15.125:6789/0","public_addr":"172.21.15.125:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:3300","nonce":0},{"type":"v1","addr":"172.21.15.155:6789","nonce":0}]},"addr":"172.21.15.155:6789/0","public_addr":"172.21.15.155:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-10-09T10:32:25.272 INFO:teuthology.orchestra.run.smithi155.stderr:dumped monmap epoch 2 2023-10-09T10:32:25.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:32:25.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: mon.a calling monitor election 2023-10-09T10:32:25.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: mon.b calling monitor election 2023-10-09T10:32:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T10:32:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: monmap e2: 2 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0]} removed_ranks: {} 2023-10-09T10:32:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: fsmap 2023-10-09T10:32:25.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T10:32:25.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: mgrmap e13: y(active, since 28s) 2023-10-09T10:32:25.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: overall HEALTH_OK 2023-10-09T10:32:25.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:25.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:25.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:25.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:32:25.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:24 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:26.037 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:25 smithi125 ceph-mon[112589]: Deploying daemon mon.c on smithi125 2023-10-09T10:32:26.038 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:25 smithi125 ceph-mon[112589]: from='client.? 172.21.15.155:0/86953692' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T10:32:26.038 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:25 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:26.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:25 smithi155 ceph-mon[117223]: Deploying daemon mon.c on smithi125 2023-10-09T10:32:26.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:25 smithi155 ceph-mon[117223]: from='client.? 172.21.15.155:0/86953692' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T10:32:26.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:25 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:26.589 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T10:32:26.590 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T10:32:26.856 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:32:26.977 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:26 smithi125 systemd[1]: Starting Ceph mon.c for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:32:27.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:26 smithi155 ceph-mon[117223]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:27.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:26 smithi125 ceph-mon[112589]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:27.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T10:32:27.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 7 2023-10-09T10:32:27.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: pidfile_write: ignore empty --pid-file 2023-10-09T10:32:27.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: load: jerasure load: lrc 2023-10-09T10:32:27.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: RocksDB version: 6.15.5 2023-10-09T10:32:27.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T10:32:27.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Compile date Oct 8 2023 2023-10-09T10:32:27.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: DB SUMMARY 2023-10-09T10:32:27.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: DB Session ID: J9HXR5BY6YL6EBS0EQL0 2023-10-09T10:32:27.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: CURRENT file: CURRENT 2023-10-09T10:32:27.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T10:32:27.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-10-09T10:32:27.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-10-09T10:32:27.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2023-10-09T10:32:27.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.error_if_exists: 0 2023-10-09T10:32:27.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.create_if_missing: 0 2023-10-09T10:32:27.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.paranoid_checks: 1 2023-10-09T10:32:27.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T10:32:27.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.env: 0x557dc56d0a80 2023-10-09T10:32:27.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.fs: Posix File System 2023-10-09T10:32:27.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.info_log: 0x557dc5ff1d80 2023-10-09T10:32:27.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T10:32:27.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.statistics: (nil) 2023-10-09T10:32:27.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.use_fsync: 0 2023-10-09T10:32:27.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_log_file_size: 0 2023-10-09T10:32:27.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T10:32:27.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T10:32:27.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T10:32:27.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T10:32:27.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.allow_fallocate: 1 2023-10-09T10:32:27.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T10:32:27.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T10:32:27.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.use_direct_reads: 0 2023-10-09T10:32:27.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T10:32:27.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T10:32:27.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.db_log_dir: 2023-10-09T10:32:27.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-10-09T10:32:27.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T10:32:27.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T10:32:27.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T10:32:27.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T10:32:27.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T10:32:27.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T10:32:27.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T10:32:27.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T10:32:27.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.write_buffer_manager: 0x557dc6ce4930 2023-10-09T10:32:27.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T10:32:27.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T10:32:27.330 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T10:32:27.330 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T10:32:27.330 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T10:32:27.330 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T10:32:27.330 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T10:32:27.330 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T10:32:27.330 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T10:32:27.330 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.unordered_write: 0 2023-10-09T10:32:27.331 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T10:32:27.331 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T10:32:27.331 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T10:32:27.331 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T10:32:27.331 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.row_cache: None 2023-10-09T10:32:27.331 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.wal_filter: None 2023-10-09T10:32:27.331 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T10:32:27.332 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T10:32:27.332 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.preserve_deletes: 0 2023-10-09T10:32:27.332 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.two_write_queues: 0 2023-10-09T10:32:27.332 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T10:32:27.332 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.atomic_flush: 0 2023-10-09T10:32:27.332 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T10:32:27.332 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T10:32:27.333 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T10:32:27.333 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.log_readahead_size: 0 2023-10-09T10:32:27.333 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T10:32:27.333 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T10:32:27.333 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T10:32:27.333 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T10:32:27.333 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T10:32:27.333 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T10:32:27.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_background_jobs: 2 2023-10-09T10:32:27.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_background_compactions: -1 2023-10-09T10:32:27.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_subcompactions: 1 2023-10-09T10:32:27.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T10:32:27.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T10:32:27.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T10:32:27.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T10:32:27.335 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T10:32:27.335 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T10:32:27.335 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T10:32:27.335 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T10:32:27.335 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_open_files: -1 2023-10-09T10:32:27.335 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T10:32:27.335 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T10:32:27.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T10:32:27.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T10:32:27.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_background_flushes: -1 2023-10-09T10:32:27.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Compression algorithms supported: 2023-10-09T10:32:27.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T10:32:27.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: kZSTD supported: 0 2023-10-09T10:32:27.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: kXpressCompression supported: 0 2023-10-09T10:32:27.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T10:32:27.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: kZlibCompression supported: 1 2023-10-09T10:32:27.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: kSnappyCompression supported: 1 2023-10-09T10:32:27.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: kLZ4Compression supported: 1 2023-10-09T10:32:27.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: kBZip2Compression supported: 0 2023-10-09T10:32:27.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T10:32:27.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-10-09T10:32:27.338 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T10:32:27.338 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T10:32:27.338 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.merge_operator: 2023-10-09T10:32:27.338 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compaction_filter: None 2023-10-09T10:32:27.338 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T10:32:27.338 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T10:32:27.339 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T10:32:27.339 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T10:32:27.339 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557dc5fc4fa0) 2023-10-09T10:32:27.339 INFO:journalctl@ceph.mon.c.smithi125.stdout: cache_index_and_filter_blocks: 1 2023-10-09T10:32:27.339 INFO:journalctl@ceph.mon.c.smithi125.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T10:32:27.339 INFO:journalctl@ceph.mon.c.smithi125.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T10:32:27.339 INFO:journalctl@ceph.mon.c.smithi125.stdout: pin_top_level_index_and_filter: 1 2023-10-09T10:32:27.340 INFO:journalctl@ceph.mon.c.smithi125.stdout: index_type: 0 2023-10-09T10:32:27.340 INFO:journalctl@ceph.mon.c.smithi125.stdout: data_block_index_type: 0 2023-10-09T10:32:27.340 INFO:journalctl@ceph.mon.c.smithi125.stdout: index_shortening: 1 2023-10-09T10:32:27.340 INFO:journalctl@ceph.mon.c.smithi125.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T10:32:27.340 INFO:journalctl@ceph.mon.c.smithi125.stdout: hash_index_allow_collision: 1 2023-10-09T10:32:27.340 INFO:journalctl@ceph.mon.c.smithi125.stdout: checksum: 1 2023-10-09T10:32:27.340 INFO:journalctl@ceph.mon.c.smithi125.stdout: no_block_cache: 0 2023-10-09T10:32:27.340 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_cache: 0x557dc5ff7090 2023-10-09T10:32:27.341 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_cache_name: BinnedLRUCache 2023-10-09T10:32:27.341 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_cache_options: 2023-10-09T10:32:27.341 INFO:journalctl@ceph.mon.c.smithi125.stdout: capacity : 536870912 2023-10-09T10:32:27.341 INFO:journalctl@ceph.mon.c.smithi125.stdout: num_shard_bits : 4 2023-10-09T10:32:27.341 INFO:journalctl@ceph.mon.c.smithi125.stdout: strict_capacity_limit : 0 2023-10-09T10:32:27.341 INFO:journalctl@ceph.mon.c.smithi125.stdout: high_pri_pool_ratio: 0.000 2023-10-09T10:32:27.341 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_cache_compressed: (nil) 2023-10-09T10:32:27.342 INFO:journalctl@ceph.mon.c.smithi125.stdout: persistent_cache: (nil) 2023-10-09T10:32:27.342 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_size: 4096 2023-10-09T10:32:27.342 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_size_deviation: 10 2023-10-09T10:32:27.342 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_restart_interval: 16 2023-10-09T10:32:27.342 INFO:journalctl@ceph.mon.c.smithi125.stdout: index_block_restart_interval: 1 2023-10-09T10:32:27.342 INFO:journalctl@ceph.mon.c.smithi125.stdout: metadata_block_size: 4096 2023-10-09T10:32:27.342 INFO:journalctl@ceph.mon.c.smithi125.stdout: partition_filters: 0 2023-10-09T10:32:27.343 INFO:journalctl@ceph.mon.c.smithi125.stdout: use_delta_encoding: 1 2023-10-09T10:32:27.343 INFO:journalctl@ceph.mon.c.smithi125.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T10:32:27.343 INFO:journalctl@ceph.mon.c.smithi125.stdout: whole_key_filtering: 1 2023-10-09T10:32:27.343 INFO:journalctl@ceph.mon.c.smithi125.stdout: verify_compression: 0 2023-10-09T10:32:27.343 INFO:journalctl@ceph.mon.c.smithi125.stdout: read_amp_bytes_per_bit: 0 2023-10-09T10:32:27.343 INFO:journalctl@ceph.mon.c.smithi125.stdout: format_version: 4 2023-10-09T10:32:27.343 INFO:journalctl@ceph.mon.c.smithi125.stdout: enable_index_compression: 1 2023-10-09T10:32:27.343 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_align: 0 2023-10-09T10:32:27.344 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T10:32:27.344 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T10:32:27.344 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compression: NoCompression 2023-10-09T10:32:27.344 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T10:32:27.344 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T10:32:27.344 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T10:32:27.344 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.num_levels: 7 2023-10-09T10:32:27.344 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T10:32:27.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T10:32:27.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T10:32:27.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T10:32:27.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T10:32:27.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T10:32:27.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T10:32:27.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T10:32:27.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T10:32:27.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T10:32:27.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T10:32:27.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T10:32:27.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T10:32:27.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T10:32:27.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T10:32:27.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T10:32:27.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T10:32:27.347 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T10:32:27.347 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T10:32:27.347 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T10:32:27.347 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T10:32:27.347 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T10:32:27.347 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T10:32:27.347 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T10:32:27.347 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T10:32:27.348 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T10:32:27.348 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T10:32:27.348 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T10:32:27.348 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T10:32:27.348 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T10:32:27.348 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T10:32:27.348 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T10:32:27.348 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T10:32:27.349 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T10:32:27.349 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.arena_block_size: 4194304 2023-10-09T10:32:27.349 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T10:32:27.349 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T10:32:27.349 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T10:32:27.349 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T10:32:27.349 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T10:32:27.349 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T10:32:27.350 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T10:32:27.350 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T10:32:27.350 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T10:32:27.350 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T10:32:27.350 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T10:32:27.350 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T10:32:27.350 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T10:32:27.351 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T10:32:27.351 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.table_properties_collectors: 2023-10-09T10:32:27.351 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.inplace_update_support: 0 2023-10-09T10:32:27.351 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T10:32:27.351 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T10:32:27.351 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T10:32:27.351 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T10:32:27.351 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.bloom_locality: 0 2023-10-09T10:32:27.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.max_successive_merges: 0 2023-10-09T10:32:27.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T10:32:27.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T10:32:27.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T10:32:27.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T10:32:27.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.ttl: 2592000 2023-10-09T10:32:27.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T10:32:27.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.enable_blob_files: false 2023-10-09T10:32:27.353 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.min_blob_size: 0 2023-10-09T10:32:27.353 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T10:32:27.353 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T10:32:27.353 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T10:32:27.353 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T10:32:27.353 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T10:32:27.353 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-10-09T10:32:27.353 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-10-09T10:32:27.354 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696847547127718, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-10-09T10:32:27.354 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-10-09T10:32:27.354 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T10:32:27.354 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696847547127981, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696847547, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "5fb9e08d-f58d-4512-bfae-4b21db07b70b", "db_session_id": "J9HXR5BY6YL6EBS0EQL0"}} 2023-10-09T10:32:27.354 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-10-09T10:32:27.354 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696847547128317, "job": 1, "event": "recovery_finished"} 2023-10-09T10:32:27.354 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T10:32:27.355 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x557dc5f8b180 2023-10-09T10:32:27.355 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: DB pointer 0x557dc6060000 2023-10-09T10:32:27.355 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T10:32:27.355 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T10:32:27.355 INFO:journalctl@ceph.mon.c.smithi125.stdout: ** DB Stats ** 2023-10-09T10:32:27.355 INFO:journalctl@ceph.mon.c.smithi125.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T10:32:27.355 INFO:journalctl@ceph.mon.c.smithi125.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T10:32:27.356 INFO:journalctl@ceph.mon.c.smithi125.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T10:32:27.356 INFO:journalctl@ceph.mon.c.smithi125.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T10:32:27.356 INFO:journalctl@ceph.mon.c.smithi125.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T10:32:27.356 INFO:journalctl@ceph.mon.c.smithi125.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T10:32:27.356 INFO:journalctl@ceph.mon.c.smithi125.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T10:32:27.356 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:32:27.356 INFO:journalctl@ceph.mon.c.smithi125.stdout: ** Compaction Stats [default] ** 2023-10-09T10:32:27.357 INFO:journalctl@ceph.mon.c.smithi125.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T10:32:27.357 INFO:journalctl@ceph.mon.c.smithi125.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T10:32:27.357 INFO:journalctl@ceph.mon.c.smithi125.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.5 0.00 0.00 1 0.000 0 0 2023-10-09T10:32:27.357 INFO:journalctl@ceph.mon.c.smithi125.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.5 0.00 0.00 1 0.000 0 0 2023-10-09T10:32:27.357 INFO:journalctl@ceph.mon.c.smithi125.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.5 0.00 0.00 1 0.000 0 0 2023-10-09T10:32:27.357 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:32:27.357 INFO:journalctl@ceph.mon.c.smithi125.stdout: ** Compaction Stats [default] ** 2023-10-09T10:32:27.358 INFO:journalctl@ceph.mon.c.smithi125.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T10:32:27.358 INFO:journalctl@ceph.mon.c.smithi125.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T10:32:27.358 INFO:journalctl@ceph.mon.c.smithi125.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.5 0.00 0.00 1 0.000 0 0 2023-10-09T10:32:27.358 INFO:journalctl@ceph.mon.c.smithi125.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T10:32:27.358 INFO:journalctl@ceph.mon.c.smithi125.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T10:32:27.358 INFO:journalctl@ceph.mon.c.smithi125.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T10:32:27.359 INFO:journalctl@ceph.mon.c.smithi125.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T10:32:27.359 INFO:journalctl@ceph.mon.c.smithi125.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T10:32:27.359 INFO:journalctl@ceph.mon.c.smithi125.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T10:32:27.359 INFO:journalctl@ceph.mon.c.smithi125.stdout: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:32:27.359 INFO:journalctl@ceph.mon.c.smithi125.stdout: Interval compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:32:27.359 INFO:journalctl@ceph.mon.c.smithi125.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T10:32:27.359 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:32:27.359 INFO:journalctl@ceph.mon.c.smithi125.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T10:32:27.360 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:32:27.360 INFO:journalctl@ceph.mon.c.smithi125.stdout: ** Compaction Stats [default] ** 2023-10-09T10:32:27.360 INFO:journalctl@ceph.mon.c.smithi125.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T10:32:27.360 INFO:journalctl@ceph.mon.c.smithi125.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T10:32:27.360 INFO:journalctl@ceph.mon.c.smithi125.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.5 0.00 0.00 1 0.000 0 0 2023-10-09T10:32:27.360 INFO:journalctl@ceph.mon.c.smithi125.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.5 0.00 0.00 1 0.000 0 0 2023-10-09T10:32:27.360 INFO:journalctl@ceph.mon.c.smithi125.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T10:32:27.361 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:32:27.361 INFO:journalctl@ceph.mon.c.smithi125.stdout: ** Compaction Stats [default] ** 2023-10-09T10:32:27.361 INFO:journalctl@ceph.mon.c.smithi125.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T10:32:27.361 INFO:journalctl@ceph.mon.c.smithi125.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T10:32:27.361 INFO:journalctl@ceph.mon.c.smithi125.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.5 0.00 0.00 1 0.000 0 0 2023-10-09T10:32:27.361 INFO:journalctl@ceph.mon.c.smithi125.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T10:32:27.361 INFO:journalctl@ceph.mon.c.smithi125.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T10:32:27.362 INFO:journalctl@ceph.mon.c.smithi125.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T10:32:27.362 INFO:journalctl@ceph.mon.c.smithi125.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T10:32:27.362 INFO:journalctl@ceph.mon.c.smithi125.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T10:32:27.362 INFO:journalctl@ceph.mon.c.smithi125.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T10:32:27.362 INFO:journalctl@ceph.mon.c.smithi125.stdout: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:32:27.362 INFO:journalctl@ceph.mon.c.smithi125.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:32:27.362 INFO:journalctl@ceph.mon.c.smithi125.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T10:32:27.363 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:32:27.363 INFO:journalctl@ceph.mon.c.smithi125.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T10:32:27.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-10-09T10:32:27.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: using public_addrv [v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0] 2023-10-09T10:32:27.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: starting mon.c rank -1 at public addrs [v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0] at bind addrs [v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:32:27.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c@-1(???) e0 preinit fsid daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:32:27.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 bash[126988]: 6524823f3114eb8a2bebf76466cbb68c448eb73ef40998ff8cda7c75a80dc4a8 2023-10-09T10:32:27.364 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 systemd[1]: Started Ceph mon.c for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:32:27.364 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c@-1(synchronizing).mds e1 new map 2023-10-09T10:32:27.364 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c@-1(synchronizing).mds e1 print_map 2023-10-09T10:32:27.364 INFO:journalctl@ceph.mon.c.smithi125.stdout: e1 2023-10-09T10:32:27.364 INFO:journalctl@ceph.mon.c.smithi125.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T10:32:27.364 INFO:journalctl@ceph.mon.c.smithi125.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T10:32:27.364 INFO:journalctl@ceph.mon.c.smithi125.stdout: legacy client fscid: -1 2023-10-09T10:32:27.365 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:32:27.365 INFO:journalctl@ceph.mon.c.smithi125.stdout: No filesystems configured 2023-10-09T10:32:27.365 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T10:32:27.365 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T10:32:27.365 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T10:32:27.365 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T10:32:27.366 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T10:32:27.366 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T10:32:27.366 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T10:32:27.366 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T10:32:27.366 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T10:32:27.366 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T10:32:27.366 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:27.367 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T10:32:27.367 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:27 smithi125 ceph-mon[127026]: mon.c@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-10-09T10:32:32.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:32:32.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: mon.a calling monitor election 2023-10-09T10:32:32.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: mon.b calling monitor election 2023-10-09T10:32:32.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:32.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:32.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: mon.c calling monitor election 2023-10-09T10:32:32.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:32.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T10:32:32.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T10:32:32.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: fsmap 2023-10-09T10:32:32.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T10:32:32.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: mgrmap e13: y(active, since 36s) 2023-10-09T10:32:32.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: overall HEALTH_OK 2023-10-09T10:32:32.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:32.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:32.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:32:32.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: mon.a calling monitor election 2023-10-09T10:32:32.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: mon.b calling monitor election 2023-10-09T10:32:32.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:32.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:32.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: mon.c calling monitor election 2023-10-09T10:32:32.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:32.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T10:32:32.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T10:32:32.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: fsmap 2023-10-09T10:32:32.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T10:32:32.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: mgrmap e13: y(active, since 36s) 2023-10-09T10:32:32.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: overall HEALTH_OK 2023-10-09T10:32:32.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:32.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:32 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:32.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:32:32.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: mon.a calling monitor election 2023-10-09T10:32:32.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: mon.b calling monitor election 2023-10-09T10:32:32.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:32:32.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:32.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: mon.c calling monitor election 2023-10-09T10:32:32.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:32.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:32.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T10:32:32.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T10:32:32.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: fsmap 2023-10-09T10:32:32.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T10:32:32.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: mgrmap e13: y(active, since 36s) 2023-10-09T10:32:32.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: overall HEALTH_OK 2023-10-09T10:32:32.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:32.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:32 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[112589]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:33.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:33.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:33.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:33.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:33.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[127026]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:33.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:33.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:33.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:33.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:33 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:33 smithi155 ceph-mon[117223]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:33 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:33 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:33 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:33.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:33 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:33 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:33 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:32:33.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:33 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:33 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:33.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:33 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:33.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:33 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:34.027 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T10:32:34.027 INFO:teuthology.orchestra.run.smithi155.stdout:{"epoch":3,"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","modified":"2023-10-09T10:32:27.155380Z","created":"2023-10-09T10:31:05.901737Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:3300","nonce":0},{"type":"v1","addr":"172.21.15.125:6789","nonce":0}]},"addr":"172.21.15.125:6789/0","public_addr":"172.21.15.125:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:3300","nonce":0},{"type":"v1","addr":"172.21.15.155:6789","nonce":0}]},"addr":"172.21.15.155:6789/0","public_addr":"172.21.15.155:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:3301","nonce":0},{"type":"v1","addr":"172.21.15.125:6790","nonce":0}]},"addr":"172.21.15.125:6790/0","public_addr":"172.21.15.125:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-10-09T10:32:34.029 INFO:teuthology.orchestra.run.smithi155.stderr:dumped monmap epoch 3 2023-10-09T10:32:34.380 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-09T10:32:34.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-09T10:32:34.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[112589]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:32:34.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[112589]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:32:34.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[112589]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:32:34.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[112589]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:32:34.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:34.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:34.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:34.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:34.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:34.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:32:34.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:32:34.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:34.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[112589]: from='client.? 172.21.15.155:0/1415143057' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T10:32:34.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[127026]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:32:34.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[127026]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:32:34.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[127026]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:32:34.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[127026]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:32:34.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:34.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:34.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:34.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:34.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:34.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:32:34.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:32:34.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:34.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:34 smithi125 ceph-mon[127026]: from='client.? 172.21.15.155:0/1415143057' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T10:32:34.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:34 smithi155 ceph-mon[117223]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:32:34.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:34 smithi155 ceph-mon[117223]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:32:34.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:34 smithi155 ceph-mon[117223]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:32:34.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:34 smithi155 ceph-mon[117223]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:32:34.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:34.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:34.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:34.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:34.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:34.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:32:34.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:32:34.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:34.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:34 smithi155 ceph-mon[117223]: from='client.? 172.21.15.155:0/1415143057' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T10:32:34.853 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:32:35.594 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[112589]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T10:32:35.594 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[112589]: Reconfiguring daemon mon.a on smithi125 2023-10-09T10:32:35.594 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[112589]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:35.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:35.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:35.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:32:35.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:32:35.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:35.595 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[127026]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T10:32:35.595 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[127026]: Reconfiguring daemon mon.a on smithi125 2023-10-09T10:32:35.595 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[127026]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:35.595 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:35.595 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:35.596 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:32:35.596 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:32:35.596 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:35 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:35 smithi155 ceph-mon[117223]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T10:32:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:35 smithi155 ceph-mon[117223]: Reconfiguring daemon mon.a on smithi125 2023-10-09T10:32:35.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:35 smithi155 ceph-mon[117223]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:35.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:35 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:35.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:35 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:35.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:35 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:32:35.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:35 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:32:35.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:35 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:36.112 INFO:teuthology.orchestra.run.smithi125.stdout:# minimal ceph.conf for daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:32:36.113 INFO:teuthology.orchestra.run.smithi125.stdout:[global] 2023-10-09T10:32:36.113 INFO:teuthology.orchestra.run.smithi125.stdout: fsid = daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:32:36.113 INFO:teuthology.orchestra.run.smithi125.stdout: mon_host = [v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0] [v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0] [v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0] 2023-10-09T10:32:36.531 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-09T10:32:36.531 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:32:36.531 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T10:32:36.562 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:32:36.562 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:32:36.629 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T10:32:36.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T10:32:36.659 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T10:32:36.659 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:32:36.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:36 smithi155 ceph-mon[117223]: Reconfiguring mon.c (monmap changed)... 2023-10-09T10:32:36.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:36 smithi155 ceph-mon[117223]: Reconfiguring daemon mon.c on smithi125 2023-10-09T10:32:36.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:36 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:36.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:36 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2180336439' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:36.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:36 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:36.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:36 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:36.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:36 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:32:36.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:36 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:32:36.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:36 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:36.730 INFO:tasks.cephadm:Adding mgr.y on smithi125 2023-10-09T10:32:36.730 INFO:tasks.cephadm:Adding mgr.x on smithi155 2023-10-09T10:32:36.730 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '2;smithi125=y;smithi155=x' 2023-10-09T10:32:36.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[112589]: Reconfiguring mon.c (monmap changed)... 2023-10-09T10:32:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[112589]: Reconfiguring daemon mon.c on smithi125 2023-10-09T10:32:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2180336439' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:32:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:32:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[127026]: Reconfiguring mon.c (monmap changed)... 2023-10-09T10:32:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[127026]: Reconfiguring daemon mon.c on smithi125 2023-10-09T10:32:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2180336439' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:32:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:32:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:36 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:37.278 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:32:37.716 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:37 smithi155 ceph-mon[117223]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:37.716 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:37 smithi155 ceph-mon[117223]: Reconfiguring mon.b (monmap changed)... 2023-10-09T10:32:37.717 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:37 smithi155 ceph-mon[117223]: Reconfiguring daemon mon.b on smithi155 2023-10-09T10:32:37.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:37 smithi125 ceph-mon[112589]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:37 smithi125 ceph-mon[112589]: Reconfiguring mon.b (monmap changed)... 2023-10-09T10:32:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:37 smithi125 ceph-mon[112589]: Reconfiguring daemon mon.b on smithi155 2023-10-09T10:32:37.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:37 smithi125 ceph-mon[127026]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:37.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:37 smithi125 ceph-mon[127026]: Reconfiguring mon.b (monmap changed)... 2023-10-09T10:32:37.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:37 smithi125 ceph-mon[127026]: Reconfiguring daemon mon.b on smithi155 2023-10-09T10:32:38.462 INFO:teuthology.orchestra.run.smithi155.stdout:Scheduled mgr update... 2023-10-09T10:32:38.832 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:38.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:38.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:38.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:38.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:38.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:38.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:38.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi125=y;smithi155=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:38.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: Saving service mgr spec with placement smithi125=y;smithi155=x;count:2 2023-10-09T10:32:38.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:38.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:38.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:38.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:38.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:38.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:32:38.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T10:32:38.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:32:38.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:38.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:38 smithi155 ceph-mon[117223]: Deploying daemon mgr.x on smithi155 2023-10-09T10:32:38.836 DEBUG:teuthology.orchestra.run.smithi155:mgr.x> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-09T10:32:38.838 INFO:tasks.cephadm:Deploying OSDs... 2023-10-09T10:32:38.838 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:32:38.838 DEBUG:teuthology.orchestra.run.smithi125:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T10:32:38.861 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T10:32:38.861 DEBUG:teuthology.orchestra.run.smithi125:> stat /dev/vg_nvme/lv_1 2023-10-09T10:32:38.869 INFO:journalctl@ceph.mgr.x.smithi155.stdout:-- Logs begin at Mon 2023-10-09 10:18:52 UTC. -- 2023-10-09T10:32:38.923 INFO:teuthology.orchestra.run.smithi125.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T10:32:38.923 INFO:teuthology.orchestra.run.smithi125.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T10:32:38.923 INFO:teuthology.orchestra.run.smithi125.stdout:Device: 6h/6d Inode: 390727 Links: 1 2023-10-09T10:32:38.923 INFO:teuthology.orchestra.run.smithi125.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T10:32:38.923 INFO:teuthology.orchestra.run.smithi125.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T10:32:38.923 INFO:teuthology.orchestra.run.smithi125.stdout:Access: 2023-10-09 10:32:26.768838074 +0000 2023-10-09T10:32:38.923 INFO:teuthology.orchestra.run.smithi125.stdout:Modify: 2023-10-09 10:26:00.148563403 +0000 2023-10-09T10:32:38.923 INFO:teuthology.orchestra.run.smithi125.stdout:Change: 2023-10-09 10:26:00.148563403 +0000 2023-10-09T10:32:38.924 INFO:teuthology.orchestra.run.smithi125.stdout: Birth: - 2023-10-09T10:32:38.924 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T10:32:38.993 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:38.993 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:38.993 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:38.993 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:38.993 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:38.993 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:38.993 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:38.993 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi125=y;smithi155=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:38.993 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: Saving service mgr spec with placement smithi125=y;smithi155=x;count:2 2023-10-09T10:32:38.993 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:38.993 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:38.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:38.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:38.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:38.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:32:38.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T10:32:38.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:32:38.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:38.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[112589]: Deploying daemon mgr.x on smithi155 2023-10-09T10:32:38.994 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:38.995 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:38.995 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:38.995 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:38.995 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:38.995 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:38.995 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:38.995 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi125=y;smithi155=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:38.995 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: Saving service mgr spec with placement smithi125=y;smithi155=x;count:2 2023-10-09T10:32:38.995 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:38.995 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:38.995 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:38.996 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:38.996 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:38.996 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:32:38.996 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T10:32:38.996 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:32:38.997 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:38.997 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:38 smithi125 ceph-mon[127026]: Deploying daemon mgr.x on smithi155 2023-10-09T10:32:38.997 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records in 2023-10-09T10:32:38.998 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records out 2023-10-09T10:32:38.998 INFO:teuthology.orchestra.run.smithi125.stderr:512 bytes copied, 0.000279678 s, 1.8 MB/s 2023-10-09T10:32:38.998 DEBUG:teuthology.orchestra.run.smithi125:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T10:32:39.061 DEBUG:teuthology.orchestra.run.smithi125:> stat /dev/vg_nvme/lv_2 2023-10-09T10:32:39.124 INFO:teuthology.orchestra.run.smithi125.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T10:32:39.124 INFO:teuthology.orchestra.run.smithi125.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T10:32:39.124 INFO:teuthology.orchestra.run.smithi125.stdout:Device: 6h/6d Inode: 398411 Links: 1 2023-10-09T10:32:39.125 INFO:teuthology.orchestra.run.smithi125.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T10:32:39.125 INFO:teuthology.orchestra.run.smithi125.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T10:32:39.125 INFO:teuthology.orchestra.run.smithi125.stdout:Access: 2023-10-09 10:32:26.768838074 +0000 2023-10-09T10:32:39.125 INFO:teuthology.orchestra.run.smithi125.stdout:Modify: 2023-10-09 10:26:00.706553761 +0000 2023-10-09T10:32:39.125 INFO:teuthology.orchestra.run.smithi125.stdout:Change: 2023-10-09 10:26:00.706553761 +0000 2023-10-09T10:32:39.125 INFO:teuthology.orchestra.run.smithi125.stdout: Birth: - 2023-10-09T10:32:39.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T10:32:39.197 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records in 2023-10-09T10:32:39.197 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records out 2023-10-09T10:32:39.197 INFO:teuthology.orchestra.run.smithi125.stderr:512 bytes copied, 0.000238373 s, 2.1 MB/s 2023-10-09T10:32:39.199 DEBUG:teuthology.orchestra.run.smithi125:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T10:32:39.262 DEBUG:teuthology.orchestra.run.smithi125:> stat /dev/vg_nvme/lv_3 2023-10-09T10:32:39.324 INFO:teuthology.orchestra.run.smithi125.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T10:32:39.324 INFO:teuthology.orchestra.run.smithi125.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T10:32:39.324 INFO:teuthology.orchestra.run.smithi125.stdout:Device: 6h/6d Inode: 390840 Links: 1 2023-10-09T10:32:39.324 INFO:teuthology.orchestra.run.smithi125.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T10:32:39.324 INFO:teuthology.orchestra.run.smithi125.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T10:32:39.324 INFO:teuthology.orchestra.run.smithi125.stdout:Access: 2023-10-09 10:32:26.768838074 +0000 2023-10-09T10:32:39.325 INFO:teuthology.orchestra.run.smithi125.stdout:Modify: 2023-10-09 10:26:01.221544862 +0000 2023-10-09T10:32:39.325 INFO:teuthology.orchestra.run.smithi125.stdout:Change: 2023-10-09 10:26:01.221544862 +0000 2023-10-09T10:32:39.325 INFO:teuthology.orchestra.run.smithi125.stdout: Birth: - 2023-10-09T10:32:39.325 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T10:32:39.396 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records in 2023-10-09T10:32:39.396 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records out 2023-10-09T10:32:39.396 INFO:teuthology.orchestra.run.smithi125.stderr:512 bytes copied, 0.00024908 s, 2.1 MB/s 2023-10-09T10:32:39.397 DEBUG:teuthology.orchestra.run.smithi125:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T10:32:39.461 DEBUG:teuthology.orchestra.run.smithi125:> stat /dev/vg_nvme/lv_4 2023-10-09T10:32:39.523 INFO:teuthology.orchestra.run.smithi125.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T10:32:39.523 INFO:teuthology.orchestra.run.smithi125.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T10:32:39.523 INFO:teuthology.orchestra.run.smithi125.stdout:Device: 6h/6d Inode: 391066 Links: 1 2023-10-09T10:32:39.523 INFO:teuthology.orchestra.run.smithi125.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T10:32:39.523 INFO:teuthology.orchestra.run.smithi125.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T10:32:39.524 INFO:teuthology.orchestra.run.smithi125.stdout:Access: 2023-10-09 10:32:26.769838057 +0000 2023-10-09T10:32:39.524 INFO:teuthology.orchestra.run.smithi125.stdout:Modify: 2023-10-09 10:26:01.753535669 +0000 2023-10-09T10:32:39.524 INFO:teuthology.orchestra.run.smithi125.stdout:Change: 2023-10-09 10:26:01.753535669 +0000 2023-10-09T10:32:39.524 INFO:teuthology.orchestra.run.smithi125.stdout: Birth: - 2023-10-09T10:32:39.524 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T10:32:39.595 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records in 2023-10-09T10:32:39.595 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records out 2023-10-09T10:32:39.595 INFO:teuthology.orchestra.run.smithi125.stderr:512 bytes copied, 0.000251371 s, 2.0 MB/s 2023-10-09T10:32:39.597 DEBUG:teuthology.orchestra.run.smithi125:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T10:32:39.660 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T10:32:39.660 DEBUG:teuthology.orchestra.run.smithi155:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T10:32:39.681 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T10:32:39.681 DEBUG:teuthology.orchestra.run.smithi155:> stat /dev/vg_nvme/lv_1 2023-10-09T10:32:39.742 INFO:teuthology.orchestra.run.smithi155.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T10:32:39.742 INFO:teuthology.orchestra.run.smithi155.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T10:32:39.742 INFO:teuthology.orchestra.run.smithi155.stdout:Device: 6h/6d Inode: 395522 Links: 1 2023-10-09T10:32:39.742 INFO:teuthology.orchestra.run.smithi155.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T10:32:39.743 INFO:teuthology.orchestra.run.smithi155.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T10:32:39.743 INFO:teuthology.orchestra.run.smithi155.stdout:Access: 2023-10-09 10:32:39.628997409 +0000 2023-10-09T10:32:39.743 INFO:teuthology.orchestra.run.smithi155.stdout:Modify: 2023-10-09 10:25:59.778457896 +0000 2023-10-09T10:32:39.743 INFO:teuthology.orchestra.run.smithi155.stdout:Change: 2023-10-09 10:25:59.778457896 +0000 2023-10-09T10:32:39.743 INFO:teuthology.orchestra.run.smithi155.stdout: Birth: - 2023-10-09T10:32:39.743 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T10:32:39.814 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records in 2023-10-09T10:32:39.814 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records out 2023-10-09T10:32:39.814 INFO:teuthology.orchestra.run.smithi155.stderr:512 bytes copied, 0.00031704 s, 1.6 MB/s 2023-10-09T10:32:39.815 DEBUG:teuthology.orchestra.run.smithi155:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T10:32:39.874 DEBUG:teuthology.orchestra.run.smithi155:> stat /dev/vg_nvme/lv_2 2023-10-09T10:32:39.932 INFO:teuthology.orchestra.run.smithi155.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T10:32:39.932 INFO:teuthology.orchestra.run.smithi155.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T10:32:39.933 INFO:teuthology.orchestra.run.smithi155.stdout:Device: 6h/6d Inode: 399553 Links: 1 2023-10-09T10:32:39.933 INFO:teuthology.orchestra.run.smithi155.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T10:32:39.933 INFO:teuthology.orchestra.run.smithi155.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T10:32:39.933 INFO:teuthology.orchestra.run.smithi155.stdout:Access: 2023-10-09 10:32:39.628997409 +0000 2023-10-09T10:32:39.933 INFO:teuthology.orchestra.run.smithi155.stdout:Modify: 2023-10-09 10:26:00.305447290 +0000 2023-10-09T10:32:39.933 INFO:teuthology.orchestra.run.smithi155.stdout:Change: 2023-10-09 10:26:00.305447290 +0000 2023-10-09T10:32:39.933 INFO:teuthology.orchestra.run.smithi155.stdout: Birth: - 2023-10-09T10:32:39.933 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T10:32:39.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:39 smithi155 systemd[1]: Starting Ceph mgr.x for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:32:39.996 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records in 2023-10-09T10:32:39.996 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records out 2023-10-09T10:32:39.998 INFO:teuthology.orchestra.run.smithi155.stderr:512 bytes copied, 0.000217393 s, 2.4 MB/s 2023-10-09T10:32:39.999 DEBUG:teuthology.orchestra.run.smithi155:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T10:32:40.059 DEBUG:teuthology.orchestra.run.smithi155:> stat /dev/vg_nvme/lv_3 2023-10-09T10:32:40.119 INFO:teuthology.orchestra.run.smithi155.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T10:32:40.119 INFO:teuthology.orchestra.run.smithi155.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T10:32:40.120 INFO:teuthology.orchestra.run.smithi155.stdout:Device: 6h/6d Inode: 397819 Links: 1 2023-10-09T10:32:40.120 INFO:teuthology.orchestra.run.smithi155.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T10:32:40.120 INFO:teuthology.orchestra.run.smithi155.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T10:32:40.120 INFO:teuthology.orchestra.run.smithi155.stdout:Access: 2023-10-09 10:32:39.628997409 +0000 2023-10-09T10:32:40.120 INFO:teuthology.orchestra.run.smithi155.stdout:Modify: 2023-10-09 10:26:00.877435778 +0000 2023-10-09T10:32:40.120 INFO:teuthology.orchestra.run.smithi155.stdout:Change: 2023-10-09 10:26:00.877435778 +0000 2023-10-09T10:32:40.120 INFO:teuthology.orchestra.run.smithi155.stdout: Birth: - 2023-10-09T10:32:40.120 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T10:32:40.187 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records in 2023-10-09T10:32:40.187 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records out 2023-10-09T10:32:40.187 INFO:teuthology.orchestra.run.smithi155.stderr:512 bytes copied, 0.000197389 s, 2.6 MB/s 2023-10-09T10:32:40.188 DEBUG:teuthology.orchestra.run.smithi155:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T10:32:40.246 DEBUG:teuthology.orchestra.run.smithi155:> stat /dev/vg_nvme/lv_4 2023-10-09T10:32:40.309 INFO:teuthology.orchestra.run.smithi155.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T10:32:40.310 INFO:teuthology.orchestra.run.smithi155.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T10:32:40.310 INFO:teuthology.orchestra.run.smithi155.stdout:Device: 6h/6d Inode: 397954 Links: 1 2023-10-09T10:32:40.310 INFO:teuthology.orchestra.run.smithi155.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T10:32:40.310 INFO:teuthology.orchestra.run.smithi155.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T10:32:40.310 INFO:teuthology.orchestra.run.smithi155.stdout:Access: 2023-10-09 10:32:39.628997409 +0000 2023-10-09T10:32:40.310 INFO:teuthology.orchestra.run.smithi155.stdout:Modify: 2023-10-09 10:26:01.402425212 +0000 2023-10-09T10:32:40.310 INFO:teuthology.orchestra.run.smithi155.stdout:Change: 2023-10-09 10:26:01.402425212 +0000 2023-10-09T10:32:40.310 INFO:teuthology.orchestra.run.smithi155.stdout: Birth: - 2023-10-09T10:32:40.310 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T10:32:40.370 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:40 smithi155 ceph-mgr[119399]: -- 172.21.15.155:0/1283809723 <== mon.1 v2:172.21.15.155:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5572c92f69c0 con 0x5572c92cc800 2023-10-09T10:32:40.370 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:40 smithi155 bash[119265]: a1dbb2209d73b4366f2a6f3d204cdd6d4cc8c60d0465e8212377b5fe06e2ce6b 2023-10-09T10:32:40.370 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:40 smithi155 systemd[1]: Started Ceph mgr.x for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:32:40.370 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:40 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:40.187+0000 7fb40352e1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T10:32:40.370 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:40 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:40.363+0000 7fb40352e1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T10:32:40.387 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records in 2023-10-09T10:32:40.387 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records out 2023-10-09T10:32:40.387 INFO:teuthology.orchestra.run.smithi155.stderr:512 bytes copied, 0.00022022 s, 2.3 MB/s 2023-10-09T10:32:40.388 DEBUG:teuthology.orchestra.run.smithi155:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T10:32:40.447 INFO:tasks.cephadm:Deploying osd.0 on smithi125 with /dev/vg_nvme/lv_4... 2023-10-09T10:32:40.448 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T10:32:40.711 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:32:40.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:40 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:40.694+0000 7fb40352e1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T10:32:41.421 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:41.246+0000 7fb40352e1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T10:32:41.421 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:41 smithi155 ceph-mon[117223]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:41.421 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:41 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:41.421 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:41 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:41.421 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:41 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:41.422 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:41 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:41.422 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:41 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:41.422 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:41 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:41.552 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:41 smithi125 ceph-mon[112589]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:41.552 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:41 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:41.552 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:41 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:41.552 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:41 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:41.552 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:41 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:41.552 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:41 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:41.552 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:41 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:41.553 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:41 smithi125 ceph-mon[127026]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:41.553 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:41 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:41.553 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:41 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:41.553 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:41 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:41.553 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:41 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:41.553 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:41 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:41.553 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:41 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:42.116 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T10:32:42.116 INFO:teuthology.orchestra.run.smithi125.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T10:32:42.116 INFO:teuthology.orchestra.run.smithi125.stderr: stderr: 10+0 records in 2023-10-09T10:32:42.116 INFO:teuthology.orchestra.run.smithi125.stderr:10+0 records out 2023-10-09T10:32:42.116 INFO:teuthology.orchestra.run.smithi125.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0116537 s, 900 MB/s 2023-10-09T10:32:42.116 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping successful for: 2023-10-09T10:32:42.160 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:41.419+0000 7fb40352e1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T10:32:42.160 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:41.808+0000 7fb40352e1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T10:32:42.427 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:42.427 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:42.427 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:42.427 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:42.427 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:42.427 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[112589]: Reconfiguring mgr.y (unknown last config time)... 2023-10-09T10:32:42.428 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:32:42.428 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:32:42.428 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:42.428 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[112589]: Reconfiguring daemon mgr.y on smithi125 2023-10-09T10:32:42.428 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:42.428 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:42.428 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:42.428 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:42.429 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:42.429 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[127026]: Reconfiguring mgr.y (unknown last config time)... 2023-10-09T10:32:42.429 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:32:42.429 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:32:42.429 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:42.429 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:42 smithi125 ceph-mon[127026]: Reconfiguring daemon mgr.y on smithi125 2023-10-09T10:32:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:42 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:42 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:42 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:42 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:42 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:42.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:42 smithi155 ceph-mon[117223]: Reconfiguring mgr.y (unknown last config time)... 2023-10-09T10:32:42.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:42 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:32:42.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:42 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:32:42.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:42 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:42.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:42 smithi155 ceph-mon[117223]: Reconfiguring daemon mgr.y on smithi125 2023-10-09T10:32:42.655 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi125:vg_nvme/lv_4 2023-10-09T10:32:42.913 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:32:42.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:42 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:42.574+0000 7fb40352e1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T10:32:42.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:42 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:42.658+0000 7fb40352e1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T10:32:43.512 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:43 smithi155 ceph-mon[117223]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:43.512 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:43 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:43.512 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:43 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:43.512 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:43 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:43.512 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:43 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:43.512 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:43 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:43.513 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:43 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:43.513 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:43.253+0000 7fb40352e1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T10:32:43.513 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:43.334+0000 7fb40352e1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T10:32:43.513 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:43.419+0000 7fb40352e1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T10:32:43.513 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:43.510+0000 7fb40352e1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T10:32:43.685 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:43 smithi125 ceph-mon[112589]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:43.685 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:43 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:43.685 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:43 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:43.686 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:43 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:43.686 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:43 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:43.686 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:43 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:43.686 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:43 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:43.686 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:43 smithi125 ceph-mon[127026]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:43.686 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:43 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:43.686 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:43 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:43.686 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:43 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:43.686 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:43 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:43.687 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:43 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:43.687 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:43 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:43.826 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:43.734+0000 7fb40352e1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T10:32:44.126 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:43.824+0000 7fb40352e1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T10:32:44.126 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:44.031+0000 7fb40352e1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T10:32:44.126 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:44.124+0000 7fb40352e1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T10:32:44.385 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:44.301+0000 7fb40352e1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T10:32:44.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:44.383+0000 7fb40352e1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T10:32:44.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:44 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:32:44.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:44 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:32:44.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:44 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:44.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:44 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:32:44.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:44 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:32:44.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:44 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:44.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:44 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:32:44.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:44 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:32:44.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:44 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:45.455 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:45 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:45.120+0000 7fb40352e1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T10:32:45.455 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:45 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:45.306+0000 7fb40352e1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T10:32:45.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:45 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:45.477+0000 7fb40352e1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:32:45.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:45 smithi155 ceph-mon[117223]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:45.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:45 smithi155 ceph-mon[117223]: from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:45.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:45 smithi125 ceph-mon[112589]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:45.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:45 smithi125 ceph-mon[112589]: from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:45.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:45 smithi125 ceph-mon[127026]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:45.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:45 smithi125 ceph-mon[127026]: from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:32:46.453 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:46 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:46.099+0000 7fb40352e1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T10:32:46.453 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:46 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:46.274+0000 7fb40352e1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:32:46.453 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:46 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:46.451+0000 7fb40352e1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T10:32:46.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:32:46 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:32:46.567+0000 7fb40352e1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T10:32:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:46 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/982746363' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "874cd96b-ddd8-4164-931e-06f2388dd41b"}]: dispatch 2023-10-09T10:32:46.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:46 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/982746363' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "874cd96b-ddd8-4164-931e-06f2388dd41b"}]': finished 2023-10-09T10:32:46.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:46 smithi155 ceph-mon[117223]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T10:32:46.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:46 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:32:46.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:46 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/982746363' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "874cd96b-ddd8-4164-931e-06f2388dd41b"}]: dispatch 2023-10-09T10:32:46.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:46 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/982746363' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "874cd96b-ddd8-4164-931e-06f2388dd41b"}]': finished 2023-10-09T10:32:46.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:46 smithi125 ceph-mon[112589]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T10:32:46.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:46 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:32:46.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:46 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/982746363' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "874cd96b-ddd8-4164-931e-06f2388dd41b"}]: dispatch 2023-10-09T10:32:46.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:46 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/982746363' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "874cd96b-ddd8-4164-931e-06f2388dd41b"}]': finished 2023-10-09T10:32:46.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:46 smithi125 ceph-mon[127026]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T10:32:46.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:46 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:32:47.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:47 smithi155 ceph-mon[117223]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:47.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:47 smithi155 ceph-mon[117223]: Standby manager daemon x started 2023-10-09T10:32:47.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:47 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/2303976841' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:32:47.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:47 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/2303976841' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:32:47.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:47 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/2303976841' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:32:47.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:47 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/2303976841' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:32:47.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:47 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2327887202' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:32:47.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:47 smithi125 ceph-mon[112589]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:47.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:47 smithi125 ceph-mon[112589]: Standby manager daemon x started 2023-10-09T10:32:47.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:47 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/2303976841' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:32:47.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:47 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/2303976841' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:32:47.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:47 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/2303976841' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:32:47.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:47 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/2303976841' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:32:47.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:47 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2327887202' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:32:47.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:47 smithi125 ceph-mon[127026]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:47.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:47 smithi125 ceph-mon[127026]: Standby manager daemon x started 2023-10-09T10:32:47.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:47 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/2303976841' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:32:47.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:47 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/2303976841' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:32:47.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:47 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/2303976841' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:32:47.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:47 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/2303976841' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:32:47.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:47 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2327887202' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:32:48.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:48 smithi155 ceph-mon[117223]: mgrmap e14: y(active, since 51s), standbys: x 2023-10-09T10:32:48.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:48 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T10:32:48.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:48 smithi125 ceph-mon[112589]: mgrmap e14: y(active, since 51s), standbys: x 2023-10-09T10:32:48.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:48 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T10:32:48.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:48 smithi125 ceph-mon[127026]: mgrmap e14: y(active, since 51s), standbys: x 2023-10-09T10:32:48.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:48 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T10:32:49.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:49 smithi155 ceph-mon[117223]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:49.763 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:49 smithi125 ceph-mon[112589]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:49.763 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:49 smithi125 ceph-mon[127026]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:51.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:51 smithi125 ceph-mon[112589]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:51.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:51 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T10:32:51.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:51 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:51.797 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:51 smithi125 ceph-mon[127026]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:51.798 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:51 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T10:32:51.798 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:51 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:51.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:51 smithi155 ceph-mon[117223]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:51.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:51 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T10:32:51.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:51 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:52.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:52 smithi125 ceph-mon[112589]: Deploying daemon osd.0 on smithi125 2023-10-09T10:32:52.787 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:52 smithi125 ceph-mon[127026]: Deploying daemon osd.0 on smithi125 2023-10-09T10:32:52.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:52 smithi155 ceph-mon[117223]: Deploying daemon osd.0 on smithi125 2023-10-09T10:32:53.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:53 smithi125 ceph-mon[112589]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:53.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:53 smithi125 ceph-mon[127026]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:53 smithi155 ceph-mon[117223]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:55.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:55 smithi125 ceph-mon[112589]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:55.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:55 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:55.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:55 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:55.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:55 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:55.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:55 smithi125 ceph-mon[127026]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:55.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:55 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:55.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:55 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:55.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:55 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:55.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:55 smithi155 ceph-mon[117223]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:55.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:55 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:55.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:55 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:55.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:55 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:56.882 INFO:teuthology.orchestra.run.smithi125.stdout:Created osd(s) 0 on host 'smithi125' 2023-10-09T10:32:57.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:57.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:57.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:57.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:57.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:57.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[112589]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:57.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:57.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:57.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:57.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:57.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:57.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:57.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:57.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:57.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[127026]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:57.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:57.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:57.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:57.330 DEBUG:teuthology.orchestra.run.smithi125:osd.0> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T10:32:57.333 INFO:tasks.cephadm:Deploying osd.1 on smithi125 with /dev/vg_nvme/lv_3... 2023-10-09T10:32:57.333 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T10:32:57.364 INFO:journalctl@ceph.osd.0.smithi125.stdout:-- Logs begin at Mon 2023-10-09 10:18:49 UTC. -- 2023-10-09T10:32:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:57 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:57 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:57 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:32:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:57 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:32:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:57 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:57 smithi155 ceph-mon[117223]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:57 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:32:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:57 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:57 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:32:57.927 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:32:58.020 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:32:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-0[131685]: 2023-10-09T10:32:57.876+0000 7fbb1059f540 -1 osd.0 0 log_to_monitors true 2023-10-09T10:32:58.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:58 smithi125 ceph-mon[112589]: from='osd.0 [v2:172.21.15.125:6802/3322074820,v1:172.21.15.125:6803/3322074820]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T10:32:58.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:58 smithi125 ceph-mon[127026]: from='osd.0 [v2:172.21.15.125:6802/3322074820,v1:172.21.15.125:6803/3322074820]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T10:32:58.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:58 smithi155 ceph-mon[117223]: from='osd.0 [v2:172.21.15.125:6802/3322074820,v1:172.21.15.125:6803/3322074820]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T10:32:59.241 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:32:59 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-0[131685]: 2023-10-09T10:32:59.031+0000 7fbb08587700 -1 osd.0 0 waiting for initial osdmap 2023-10-09T10:32:59.242 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:32:59 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-0[131685]: 2023-10-09T10:32:59.034+0000 7fbb0137e700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:32:59.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:59 smithi125 ceph-mon[112589]: from='osd.0 [v2:172.21.15.125:6802/3322074820,v1:172.21.15.125:6803/3322074820]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T10:32:59.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:59 smithi125 ceph-mon[112589]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T10:32:59.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:59 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:32:59.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:59 smithi125 ceph-mon[112589]: from='osd.0 [v2:172.21.15.125:6802/3322074820,v1:172.21.15.125:6803/3322074820]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:32:59.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:32:59 smithi125 ceph-mon[112589]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:59.243 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:59 smithi125 ceph-mon[127026]: from='osd.0 [v2:172.21.15.125:6802/3322074820,v1:172.21.15.125:6803/3322074820]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T10:32:59.243 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:59 smithi125 ceph-mon[127026]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T10:32:59.243 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:59 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:32:59.243 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:59 smithi125 ceph-mon[127026]: from='osd.0 [v2:172.21.15.125:6802/3322074820,v1:172.21.15.125:6803/3322074820]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:32:59.243 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:32:59 smithi125 ceph-mon[127026]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:59.404 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T10:32:59.404 INFO:teuthology.orchestra.run.smithi125.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T10:32:59.404 INFO:teuthology.orchestra.run.smithi125.stderr: stderr: 10+0 records in 2023-10-09T10:32:59.404 INFO:teuthology.orchestra.run.smithi125.stderr:10+0 records out 2023-10-09T10:32:59.404 INFO:teuthology.orchestra.run.smithi125.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0131785 s, 796 MB/s 2023-10-09T10:32:59.404 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping successful for: 2023-10-09T10:32:59.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:59 smithi155 ceph-mon[117223]: from='osd.0 [v2:172.21.15.125:6802/3322074820,v1:172.21.15.125:6803/3322074820]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T10:32:59.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:59 smithi155 ceph-mon[117223]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T10:32:59.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:59 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:32:59.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:59 smithi155 ceph-mon[117223]: from='osd.0 [v2:172.21.15.125:6802/3322074820,v1:172.21.15.125:6803/3322074820]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:32:59.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:32:59 smithi155 ceph-mon[117223]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:32:59.794 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi125:vg_nvme/lv_3 2023-10-09T10:33:00.056 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:33:00.303 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[112589]: from='osd.0 [v2:172.21.15.125:6802/3322074820,v1:172.21.15.125:6803/3322074820]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T10:33:00.303 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[112589]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T10:33:00.303 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:33:00.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:33:00.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[112589]: Detected new or changed devices on smithi125 2023-10-09T10:33:00.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:00.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:00.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:00.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[112589]: Adjusting osd_memory_target on smithi125 to 16089M 2023-10-09T10:33:00.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:00.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:00.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:00.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:00.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[127026]: from='osd.0 [v2:172.21.15.125:6802/3322074820,v1:172.21.15.125:6803/3322074820]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T10:33:00.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[127026]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T10:33:00.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:33:00.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:33:00.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[127026]: Detected new or changed devices on smithi125 2023-10-09T10:33:00.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:00.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:00.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:00.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[127026]: Adjusting osd_memory_target on smithi125 to 16089M 2023-10-09T10:33:00.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:00.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:00.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:00.306 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:00 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:00 smithi155 ceph-mon[117223]: from='osd.0 [v2:172.21.15.125:6802/3322074820,v1:172.21.15.125:6803/3322074820]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T10:33:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:00 smithi155 ceph-mon[117223]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T10:33:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:00 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:33:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:00 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:33:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:00 smithi155 ceph-mon[117223]: Detected new or changed devices on smithi125 2023-10-09T10:33:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:00 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:00 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:00 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:00 smithi155 ceph-mon[117223]: Adjusting osd_memory_target on smithi125 to 16089M 2023-10-09T10:33:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:00 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:00 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:00 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:00 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:01.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:01 smithi125 ceph-mon[127026]: purged_snaps scrub starts 2023-10-09T10:33:01.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:01 smithi125 ceph-mon[127026]: purged_snaps scrub ok 2023-10-09T10:33:01.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:01 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:33:01.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:01 smithi125 ceph-mon[127026]: osd.0 [v2:172.21.15.125:6802/3322074820,v1:172.21.15.125:6803/3322074820] boot 2023-10-09T10:33:01.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:01 smithi125 ceph-mon[127026]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T10:33:01.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:01 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:33:01.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:01 smithi125 ceph-mon[127026]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:33:01.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:01 smithi125 ceph-mon[112589]: purged_snaps scrub starts 2023-10-09T10:33:01.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:01 smithi125 ceph-mon[112589]: purged_snaps scrub ok 2023-10-09T10:33:01.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:01 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:33:01.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:01 smithi125 ceph-mon[112589]: osd.0 [v2:172.21.15.125:6802/3322074820,v1:172.21.15.125:6803/3322074820] boot 2023-10-09T10:33:01.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:01 smithi125 ceph-mon[112589]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T10:33:01.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:01 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:33:01.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:01 smithi125 ceph-mon[112589]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:33:01.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:01 smithi155 ceph-mon[117223]: purged_snaps scrub starts 2023-10-09T10:33:01.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:01 smithi155 ceph-mon[117223]: purged_snaps scrub ok 2023-10-09T10:33:01.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:01 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:33:01.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:01 smithi155 ceph-mon[117223]: osd.0 [v2:172.21.15.125:6802/3322074820,v1:172.21.15.125:6803/3322074820] boot 2023-10-09T10:33:01.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:01 smithi155 ceph-mon[117223]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T10:33:01.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:01 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:33:01.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:01 smithi155 ceph-mon[117223]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T10:33:02.289 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:02 smithi125 ceph-mon[112589]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T10:33:02.289 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:02 smithi125 ceph-mon[112589]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:33:02.289 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:02 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:33:02.289 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:02 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:33:02.289 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:02 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:02.289 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:02 smithi125 ceph-mon[127026]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T10:33:02.290 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:02 smithi125 ceph-mon[127026]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:33:02.290 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:02 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:33:02.290 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:02 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:33:02.290 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:02 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:02 smithi155 ceph-mon[117223]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T10:33:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:02 smithi155 ceph-mon[117223]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:33:02.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:02 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:33:02.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:02 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:33:02.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:02 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:03 smithi155 ceph-mon[117223]: pgmap v32: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:03.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:03 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/988061675' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e517dd5-6d0b-4841-bb81-1eb04377ba5f"}]: dispatch 2023-10-09T10:33:03.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:03 smithi155 ceph-mon[117223]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e517dd5-6d0b-4841-bb81-1eb04377ba5f"}]: dispatch 2023-10-09T10:33:03.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:03 smithi155 ceph-mon[117223]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0e517dd5-6d0b-4841-bb81-1eb04377ba5f"}]': finished 2023-10-09T10:33:03.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:03 smithi155 ceph-mon[117223]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T10:33:03.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:03 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:33:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:03 smithi125 ceph-mon[112589]: pgmap v32: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:03 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/988061675' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e517dd5-6d0b-4841-bb81-1eb04377ba5f"}]: dispatch 2023-10-09T10:33:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:03 smithi125 ceph-mon[112589]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e517dd5-6d0b-4841-bb81-1eb04377ba5f"}]: dispatch 2023-10-09T10:33:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:03 smithi125 ceph-mon[112589]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0e517dd5-6d0b-4841-bb81-1eb04377ba5f"}]': finished 2023-10-09T10:33:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:03 smithi125 ceph-mon[112589]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T10:33:03.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:03 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:33:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:03 smithi125 ceph-mon[127026]: pgmap v32: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:03 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/988061675' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e517dd5-6d0b-4841-bb81-1eb04377ba5f"}]: dispatch 2023-10-09T10:33:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:03 smithi125 ceph-mon[127026]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e517dd5-6d0b-4841-bb81-1eb04377ba5f"}]: dispatch 2023-10-09T10:33:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:03 smithi125 ceph-mon[127026]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0e517dd5-6d0b-4841-bb81-1eb04377ba5f"}]': finished 2023-10-09T10:33:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:03 smithi125 ceph-mon[127026]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T10:33:03.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:03 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:33:04.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:04 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3061329545' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:33:04.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:04 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3061329545' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:33:04.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:04 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3061329545' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:33:05.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:05 smithi155 ceph-mon[117223]: pgmap v34: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:05.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:05 smithi125 ceph-mon[112589]: pgmap v34: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:05.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:05 smithi125 ceph-mon[127026]: pgmap v34: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:07.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:07 smithi155 ceph-mon[117223]: pgmap v35: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:07.777 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:07 smithi125 ceph-mon[112589]: pgmap v35: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:07.777 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:07 smithi125 ceph-mon[127026]: pgmap v35: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:08.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:08 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T10:33:08.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:08 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:08.743 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:08 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T10:33:08.743 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:08 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:08.743 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:08 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T10:33:08.743 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:08 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:09.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:09 smithi155 ceph-mon[117223]: pgmap v36: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:09.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:09 smithi155 ceph-mon[117223]: Deploying daemon osd.1 on smithi125 2023-10-09T10:33:09.730 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:09 smithi125 ceph-mon[127026]: pgmap v36: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:09.730 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:09 smithi125 ceph-mon[127026]: Deploying daemon osd.1 on smithi125 2023-10-09T10:33:09.730 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:09 smithi125 ceph-mon[112589]: pgmap v36: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:09.730 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:09 smithi125 ceph-mon[112589]: Deploying daemon osd.1 on smithi125 2023-10-09T10:33:11.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:11 smithi155 ceph-mon[117223]: pgmap v37: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:11.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:11 smithi125 ceph-mon[112589]: pgmap v37: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:11.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:11 smithi125 ceph-mon[127026]: pgmap v37: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:12.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:12 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:12.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:12 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:12.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:12 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:12.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:12 smithi125 ceph-mon[112589]: pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:12.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:12 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:12.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:12 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:12.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:12 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:12.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:12 smithi125 ceph-mon[127026]: pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:12.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:12 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:12.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:12 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:12.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:12 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:12.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:12 smithi155 ceph-mon[117223]: pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:13.833 INFO:teuthology.orchestra.run.smithi125.stdout:Created osd(s) 1 on host 'smithi125' 2023-10-09T10:33:14.213 DEBUG:teuthology.orchestra.run.smithi125:osd.1> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-09T10:33:14.216 INFO:tasks.cephadm:Deploying osd.2 on smithi125 with /dev/vg_nvme/lv_2... 2023-10-09T10:33:14.216 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T10:33:14.247 INFO:journalctl@ceph.osd.1.smithi125.stdout:-- Logs begin at Mon 2023-10-09 10:18:49 UTC. -- 2023-10-09T10:33:14.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:14.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:14.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:14.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:14.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:14.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:14.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:14.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:14.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:14.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:14.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:14.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:14.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:14.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:14.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:14.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:14.615 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:33:14.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:14 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:14.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:14 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:14 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:14 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:14 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:14 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:14 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:14 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:15.057 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:33:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[138711]: 2023-10-09T10:33:14.847+0000 7f9ca696d540 -1 osd.1 0 log_to_monitors true 2023-10-09T10:33:15.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:15 smithi125 ceph-mon[127026]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:15.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:15 smithi125 ceph-mon[127026]: from='osd.1 [v2:172.21.15.125:6810/3260318814,v1:172.21.15.125:6811/3260318814]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T10:33:15.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:15 smithi125 ceph-mon[112589]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:15.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:15 smithi125 ceph-mon[112589]: from='osd.1 [v2:172.21.15.125:6810/3260318814,v1:172.21.15.125:6811/3260318814]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T10:33:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:15 smithi155 ceph-mon[117223]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:15 smithi155 ceph-mon[117223]: from='osd.1 [v2:172.21.15.125:6810/3260318814,v1:172.21.15.125:6811/3260318814]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T10:33:16.107 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T10:33:16.107 INFO:teuthology.orchestra.run.smithi125.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T10:33:16.107 INFO:teuthology.orchestra.run.smithi125.stderr: stderr: 10+0 records in 2023-10-09T10:33:16.107 INFO:teuthology.orchestra.run.smithi125.stderr:10+0 records out 2023-10-09T10:33:16.107 INFO:teuthology.orchestra.run.smithi125.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0141968 s, 739 MB/s 2023-10-09T10:33:16.107 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping successful for: 2023-10-09T10:33:16.447 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi125:vg_nvme/lv_2 2023-10-09T10:33:16.767 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:33:16.953 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:33:16 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[138711]: 2023-10-09T10:33:16.809+0000 7f9c9e955700 -1 osd.1 0 waiting for initial osdmap 2023-10-09T10:33:17.056 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:33:16 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[138711]: 2023-10-09T10:33:16.813+0000 7f9c95f49700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:33:17.057 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:16 smithi125 ceph-mon[112589]: from='osd.1 [v2:172.21.15.125:6810/3260318814,v1:172.21.15.125:6811/3260318814]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T10:33:17.057 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:16 smithi125 ceph-mon[112589]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T10:33:17.057 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:16 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:33:17.057 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:16 smithi125 ceph-mon[112589]: from='osd.1 [v2:172.21.15.125:6810/3260318814,v1:172.21.15.125:6811/3260318814]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:33:17.057 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:16 smithi125 ceph-mon[112589]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:17.057 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:16 smithi125 ceph-mon[127026]: from='osd.1 [v2:172.21.15.125:6810/3260318814,v1:172.21.15.125:6811/3260318814]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T10:33:17.057 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:16 smithi125 ceph-mon[127026]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T10:33:17.057 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:16 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:33:17.057 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:16 smithi125 ceph-mon[127026]: from='osd.1 [v2:172.21.15.125:6810/3260318814,v1:172.21.15.125:6811/3260318814]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:33:17.057 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:16 smithi125 ceph-mon[127026]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:17.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:16 smithi155 ceph-mon[117223]: from='osd.1 [v2:172.21.15.125:6810/3260318814,v1:172.21.15.125:6811/3260318814]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T10:33:17.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:16 smithi155 ceph-mon[117223]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T10:33:17.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:16 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:33:17.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:16 smithi155 ceph-mon[117223]: from='osd.1 [v2:172.21.15.125:6810/3260318814,v1:172.21.15.125:6811/3260318814]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:33:17.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:16 smithi155 ceph-mon[117223]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:18.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[112589]: from='osd.1 [v2:172.21.15.125:6810/3260318814,v1:172.21.15.125:6811/3260318814]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T10:33:18.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[112589]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T10:33:18.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:33:18.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:33:18.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[112589]: Detected new or changed devices on smithi125 2023-10-09T10:33:18.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:18.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:18.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:18.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:18.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[112589]: Adjusting osd_memory_target on smithi125 to 8044M 2023-10-09T10:33:18.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:18.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:18.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:18.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:18.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[127026]: from='osd.1 [v2:172.21.15.125:6810/3260318814,v1:172.21.15.125:6811/3260318814]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T10:33:18.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[127026]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T10:33:18.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:33:18.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:33:18.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[127026]: Detected new or changed devices on smithi125 2023-10-09T10:33:18.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:18.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:18.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:18.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:18.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[127026]: Adjusting osd_memory_target on smithi125 to 8044M 2023-10-09T10:33:18.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:18.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:18.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:18.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:17 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:18.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:17 smithi155 ceph-mon[117223]: from='osd.1 [v2:172.21.15.125:6810/3260318814,v1:172.21.15.125:6811/3260318814]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T10:33:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:17 smithi155 ceph-mon[117223]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T10:33:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:17 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:33:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:17 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:33:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:17 smithi155 ceph-mon[117223]: Detected new or changed devices on smithi125 2023-10-09T10:33:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:17 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:17 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:17 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:17 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:17 smithi155 ceph-mon[117223]: Adjusting osd_memory_target on smithi125 to 8044M 2023-10-09T10:33:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:17 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:17 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:18.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:17 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:18.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:17 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:19.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[112589]: purged_snaps scrub starts 2023-10-09T10:33:19.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[112589]: purged_snaps scrub ok 2023-10-09T10:33:19.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[112589]: osd.1 [v2:172.21.15.125:6810/3260318814,v1:172.21.15.125:6811/3260318814] boot 2023-10-09T10:33:19.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[112589]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T10:33:19.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:33:19.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[112589]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:33:19.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:33:19.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:33:19.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:19.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[112589]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:19.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[127026]: purged_snaps scrub starts 2023-10-09T10:33:19.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[127026]: purged_snaps scrub ok 2023-10-09T10:33:19.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[127026]: osd.1 [v2:172.21.15.125:6810/3260318814,v1:172.21.15.125:6811/3260318814] boot 2023-10-09T10:33:19.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[127026]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T10:33:19.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:33:19.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[127026]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:33:19.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:33:19.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:33:19.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:19.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:18 smithi125 ceph-mon[127026]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:18 smithi155 ceph-mon[117223]: purged_snaps scrub starts 2023-10-09T10:33:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:18 smithi155 ceph-mon[117223]: purged_snaps scrub ok 2023-10-09T10:33:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:18 smithi155 ceph-mon[117223]: osd.1 [v2:172.21.15.125:6810/3260318814,v1:172.21.15.125:6811/3260318814] boot 2023-10-09T10:33:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:18 smithi155 ceph-mon[117223]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T10:33:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:18 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:33:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:18 smithi155 ceph-mon[117223]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:33:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:18 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:33:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:18 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:33:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:18 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:18 smithi155 ceph-mon[117223]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T10:33:20.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:19 smithi155 ceph-mon[117223]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T10:33:20.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1382961306' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "643c1f3c-ecd9-464a-ad53-bf35a4deee5e"}]: dispatch 2023-10-09T10:33:20.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:19 smithi155 ceph-mon[117223]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "643c1f3c-ecd9-464a-ad53-bf35a4deee5e"}]: dispatch 2023-10-09T10:33:20.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:19 smithi155 ceph-mon[117223]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "643c1f3c-ecd9-464a-ad53-bf35a4deee5e"}]': finished 2023-10-09T10:33:20.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:19 smithi155 ceph-mon[117223]: osdmap e15: 3 total, 2 up, 3 in 2023-10-09T10:33:20.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:20.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:19 smithi125 ceph-mon[112589]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T10:33:20.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:19 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1382961306' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "643c1f3c-ecd9-464a-ad53-bf35a4deee5e"}]: dispatch 2023-10-09T10:33:20.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:19 smithi125 ceph-mon[112589]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "643c1f3c-ecd9-464a-ad53-bf35a4deee5e"}]: dispatch 2023-10-09T10:33:20.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:19 smithi125 ceph-mon[112589]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "643c1f3c-ecd9-464a-ad53-bf35a4deee5e"}]': finished 2023-10-09T10:33:20.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:19 smithi125 ceph-mon[112589]: osdmap e15: 3 total, 2 up, 3 in 2023-10-09T10:33:20.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:19 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:20.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:19 smithi125 ceph-mon[127026]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T10:33:20.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:19 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/1382961306' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "643c1f3c-ecd9-464a-ad53-bf35a4deee5e"}]: dispatch 2023-10-09T10:33:20.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:19 smithi125 ceph-mon[127026]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "643c1f3c-ecd9-464a-ad53-bf35a4deee5e"}]: dispatch 2023-10-09T10:33:20.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:19 smithi125 ceph-mon[127026]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "643c1f3c-ecd9-464a-ad53-bf35a4deee5e"}]': finished 2023-10-09T10:33:20.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:19 smithi125 ceph-mon[127026]: osdmap e15: 3 total, 2 up, 3 in 2023-10-09T10:33:20.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:19 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:20 smithi155 ceph-mon[117223]: pgmap v47: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:20 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/552734422' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:33:21.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:20 smithi125 ceph-mon[112589]: pgmap v47: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:21.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:20 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/552734422' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:33:21.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:20 smithi125 ceph-mon[127026]: pgmap v47: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:21.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:20 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/552734422' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:33:23.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:23 smithi125 ceph-mon[112589]: pgmap v48: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:23 smithi125 ceph-mon[127026]: pgmap v48: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:23 smithi155 ceph-mon[117223]: pgmap v48: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:25.619 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:25 smithi125 ceph-mon[127026]: pgmap v49: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:25.619 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:25 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T10:33:25.619 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:25 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:25.619 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:25 smithi125 ceph-mon[112589]: pgmap v49: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:25.620 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:25 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T10:33:25.620 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:25 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:25.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:25 smithi155 ceph-mon[117223]: pgmap v49: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:25.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:25 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T10:33:25.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:25 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:26.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:26 smithi125 ceph-mon[112589]: Deploying daemon osd.2 on smithi125 2023-10-09T10:33:26.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:26 smithi125 ceph-mon[127026]: Deploying daemon osd.2 on smithi125 2023-10-09T10:33:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:26 smithi155 ceph-mon[117223]: Deploying daemon osd.2 on smithi125 2023-10-09T10:33:27.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:27 smithi155 ceph-mon[117223]: pgmap v50: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:27 smithi125 ceph-mon[112589]: pgmap v50: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:27 smithi125 ceph-mon[127026]: pgmap v50: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:29.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:29 smithi125 ceph-mon[112589]: pgmap v51: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:29.503 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:29 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:29.503 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:29 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:29.503 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:29 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:29.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:29 smithi125 ceph-mon[127026]: pgmap v51: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:29.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:29 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:29.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:29 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:29.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:29 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:29.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:29 smithi155 ceph-mon[117223]: pgmap v51: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:29.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:29 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:29.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:29 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:29.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:29 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:30.791 INFO:teuthology.orchestra.run.smithi125.stdout:Created osd(s) 2 on host 'smithi125' 2023-10-09T10:33:31.182 DEBUG:teuthology.orchestra.run.smithi125:osd.2> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-09T10:33:31.184 INFO:tasks.cephadm:Deploying osd.3 on smithi125 with /dev/vg_nvme/lv_1... 2023-10-09T10:33:31.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T10:33:31.217 INFO:journalctl@ceph.osd.2.smithi125.stdout:-- Logs begin at Mon 2023-10-09 10:18:49 UTC. -- 2023-10-09T10:33:31.241 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:31.241 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:31.241 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:31.241 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:31.241 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:31.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[112589]: pgmap v52: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:31.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:31.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:31.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:31.242 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:31.242 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:31.242 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:31.242 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:31.242 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:31.242 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[127026]: pgmap v52: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:31.243 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:31.243 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:31.243 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:30 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:30 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:30 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:30 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:30 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:30 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:30 smithi155 ceph-mon[117223]: pgmap v52: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:30 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:30 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:30 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:31.710 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:33:31.824 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:33:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-2[146683]: 2023-10-09T10:33:31.596+0000 7fa03591a540 -1 osd.2 0 log_to_monitors true 2023-10-09T10:33:32.126 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:31 smithi125 ceph-mon[112589]: from='osd.2 [v2:172.21.15.125:6818/4010155657,v1:172.21.15.125:6819/4010155657]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T10:33:32.126 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:31 smithi125 ceph-mon[112589]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T10:33:32.126 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:31 smithi125 ceph-mon[127026]: from='osd.2 [v2:172.21.15.125:6818/4010155657,v1:172.21.15.125:6819/4010155657]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T10:33:32.126 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:31 smithi125 ceph-mon[127026]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T10:33:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:31 smithi155 ceph-mon[117223]: from='osd.2 [v2:172.21.15.125:6818/4010155657,v1:172.21.15.125:6819/4010155657]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T10:33:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:31 smithi155 ceph-mon[117223]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T10:33:33.222 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T10:33:33.222 INFO:teuthology.orchestra.run.smithi125.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T10:33:33.222 INFO:teuthology.orchestra.run.smithi125.stderr: stderr: 10+0 records in 2023-10-09T10:33:33.222 INFO:teuthology.orchestra.run.smithi125.stderr:10+0 records out 2023-10-09T10:33:33.223 INFO:teuthology.orchestra.run.smithi125.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0143567 s, 730 MB/s 2023-10-09T10:33:33.223 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping successful for: 2023-10-09T10:33:33.263 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:33:33 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-2[146683]: 2023-10-09T10:33:33.014+0000 7fa02c0ff700 -1 osd.2 0 waiting for initial osdmap 2023-10-09T10:33:33.263 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:33:33 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-2[146683]: 2023-10-09T10:33:33.018+0000 7fa0276fb700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:33:33.263 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:33 smithi125 ceph-mon[112589]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T10:33:33.263 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:33 smithi125 ceph-mon[112589]: osdmap e16: 3 total, 2 up, 3 in 2023-10-09T10:33:33.264 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:33 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:33.264 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:33 smithi125 ceph-mon[112589]: from='osd.2 [v2:172.21.15.125:6818/4010155657,v1:172.21.15.125:6819/4010155657]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:33:33.264 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:33 smithi125 ceph-mon[112589]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:33:33.264 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:33 smithi125 ceph-mon[112589]: pgmap v54: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:33.264 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:33 smithi125 ceph-mon[127026]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T10:33:33.264 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:33 smithi125 ceph-mon[127026]: osdmap e16: 3 total, 2 up, 3 in 2023-10-09T10:33:33.264 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:33 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:33.264 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:33 smithi125 ceph-mon[127026]: from='osd.2 [v2:172.21.15.125:6818/4010155657,v1:172.21.15.125:6819/4010155657]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:33:33.264 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:33 smithi125 ceph-mon[127026]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:33:33.265 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:33 smithi125 ceph-mon[127026]: pgmap v54: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:33 smithi155 ceph-mon[117223]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T10:33:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:33 smithi155 ceph-mon[117223]: osdmap e16: 3 total, 2 up, 3 in 2023-10-09T10:33:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:33 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:33.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:33 smithi155 ceph-mon[117223]: from='osd.2 [v2:172.21.15.125:6818/4010155657,v1:172.21.15.125:6819/4010155657]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:33:33.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:33 smithi155 ceph-mon[117223]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:33:33.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:33 smithi155 ceph-mon[117223]: pgmap v54: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:33.571 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi125:vg_nvme/lv_1 2023-10-09T10:33:33.991 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:33:34.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[112589]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T10:33:34.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[112589]: osdmap e17: 3 total, 2 up, 3 in 2023-10-09T10:33:34.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:34.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:34.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:34.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:34.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:34.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:34.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:34.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:34.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:34.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:34.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:34.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[127026]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T10:33:34.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[127026]: osdmap e17: 3 total, 2 up, 3 in 2023-10-09T10:33:34.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:34.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:34.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:34.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:34.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:34.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:34.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:34.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:34.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:34.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:34.144 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:34 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:34.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:34 smithi155 ceph-mon[117223]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T10:33:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:34 smithi155 ceph-mon[117223]: osdmap e17: 3 total, 2 up, 3 in 2023-10-09T10:33:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:34 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:35.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[112589]: purged_snaps scrub starts 2023-10-09T10:33:35.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[112589]: purged_snaps scrub ok 2023-10-09T10:33:35.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[112589]: Detected new or changed devices on smithi125 2023-10-09T10:33:35.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[112589]: Adjusting osd_memory_target on smithi125 to 5363M 2023-10-09T10:33:35.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:35.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[112589]: osd.2 [v2:172.21.15.125:6818/4010155657,v1:172.21.15.125:6819/4010155657] boot 2023-10-09T10:33:35.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[112589]: osdmap e18: 3 total, 3 up, 3 in 2023-10-09T10:33:35.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:35.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[112589]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:35.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T10:33:35.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[127026]: purged_snaps scrub starts 2023-10-09T10:33:35.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[127026]: purged_snaps scrub ok 2023-10-09T10:33:35.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[127026]: Detected new or changed devices on smithi125 2023-10-09T10:33:35.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[127026]: Adjusting osd_memory_target on smithi125 to 5363M 2023-10-09T10:33:35.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:35.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[127026]: osd.2 [v2:172.21.15.125:6818/4010155657,v1:172.21.15.125:6819/4010155657] boot 2023-10-09T10:33:35.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[127026]: osdmap e18: 3 total, 3 up, 3 in 2023-10-09T10:33:35.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:35.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[127026]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:35.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:35 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T10:33:35.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:35 smithi155 ceph-mon[117223]: purged_snaps scrub starts 2023-10-09T10:33:35.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:35 smithi155 ceph-mon[117223]: purged_snaps scrub ok 2023-10-09T10:33:35.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:35 smithi155 ceph-mon[117223]: Detected new or changed devices on smithi125 2023-10-09T10:33:35.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:35 smithi155 ceph-mon[117223]: Adjusting osd_memory_target on smithi125 to 5363M 2023-10-09T10:33:35.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:35 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:35.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:35 smithi155 ceph-mon[117223]: osd.2 [v2:172.21.15.125:6818/4010155657,v1:172.21.15.125:6819/4010155657] boot 2023-10-09T10:33:35.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:35 smithi155 ceph-mon[117223]: osdmap e18: 3 total, 3 up, 3 in 2023-10-09T10:33:35.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:35 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:33:35.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:35 smithi155 ceph-mon[117223]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T10:33:35.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:35 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T10:33:36.309 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:36 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T10:33:36.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:36 smithi125 ceph-mon[112589]: osdmap e19: 3 total, 3 up, 3 in 2023-10-09T10:33:36.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:36 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:33:36.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:36 smithi125 ceph-mon[112589]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:33:36.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:36 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:33:36.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:36 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:33:36.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:36 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:36.310 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:36 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T10:33:36.310 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:36 smithi125 ceph-mon[127026]: osdmap e19: 3 total, 3 up, 3 in 2023-10-09T10:33:36.310 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:36 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:33:36.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:36 smithi125 ceph-mon[127026]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:33:36.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:36 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:33:36.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:36 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:33:36.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:36 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:36.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:36 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T10:33:36.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:36 smithi155 ceph-mon[117223]: osdmap e19: 3 total, 3 up, 3 in 2023-10-09T10:33:36.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:36 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:33:36.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:36 smithi155 ceph-mon[117223]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:33:36.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:36 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:33:36.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:36 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:33:36.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:36 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:36.570 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:33:36 smithi125 sudo[152664]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T10:33:36.570 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:33:36 smithi125 sudo[152664]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T10:33:36.571 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:33:36 smithi125 sudo[152664]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T10:33:36.571 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:33:36 smithi125 sudo[152664]: pam_unix(sudo:session): session closed for user root 2023-10-09T10:33:36.891 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:33:36 smithi125 sudo[152677]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T10:33:36.891 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:33:36 smithi125 sudo[152677]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T10:33:36.891 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:33:36 smithi125 sudo[152677]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T10:33:36.891 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:33:36 smithi125 sudo[152677]: pam_unix(sudo:session): session closed for user root 2023-10-09T10:33:37.148 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:37 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:33:37.148 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:37 smithi125 ceph-mon[112589]: osdmap e20: 3 total, 3 up, 3 in 2023-10-09T10:33:37.148 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:37 smithi125 ceph-mon[112589]: pgmap v60: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T10:33:37.148 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:37 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:37.148 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:37 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:33:37.148 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:37 smithi125 ceph-mon[127026]: osdmap e20: 3 total, 3 up, 3 in 2023-10-09T10:33:37.149 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:37 smithi125 ceph-mon[127026]: pgmap v60: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T10:33:37.149 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:37 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:37.149 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:33:36 smithi125 sudo[152712]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T10:33:37.149 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:33:36 smithi125 sudo[152712]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T10:33:37.149 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:33:36 smithi125 sudo[152712]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T10:33:37.149 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:33:36 smithi125 sudo[152712]: pam_unix(sudo:session): session closed for user root 2023-10-09T10:33:37.406 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152733]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T10:33:37.406 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152733]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T10:33:37.406 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152733]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T10:33:37.406 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152721]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T10:33:37.406 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152721]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T10:33:37.407 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152721]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T10:33:37.407 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152721]: pam_unix(sudo:session): session closed for user root 2023-10-09T10:33:37.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:37 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:33:37.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:37 smithi155 ceph-mon[117223]: osdmap e20: 3 total, 3 up, 3 in 2023-10-09T10:33:37.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:37 smithi155 ceph-mon[117223]: pgmap v60: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T10:33:37.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:37 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:37.820 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152733]: pam_unix(sudo:session): session closed for user root 2023-10-09T10:33:37.821 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152756]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T10:33:37.821 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152756]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T10:33:37.821 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152756]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T10:33:37.821 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152756]: pam_unix(sudo:session): session closed for user root 2023-10-09T10:33:38.149 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152808]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T10:33:38.149 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152808]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T10:33:38.149 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152808]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T10:33:38.149 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:37 smithi125 sudo[152808]: pam_unix(sudo:session): session closed for user root 2023-10-09T10:33:38.149 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:38 smithi125 sudo[152813]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T10:33:38.150 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:38 smithi125 sudo[152813]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T10:33:38.150 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:38 smithi125 sudo[152813]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T10:33:38.150 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:38 smithi125 sudo[152813]: pam_unix(sudo:session): session closed for user root 2023-10-09T10:33:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 ceph-mon[117223]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T10:33:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 ceph-mon[117223]: mgrmap e15: y(active, since 101s), standbys: x 2023-10-09T10:33:38.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/641578152' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3015cd6e-b270-44c3-8e6e-6bcd979bcd21"}]: dispatch 2023-10-09T10:33:38.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 ceph-mon[117223]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3015cd6e-b270-44c3-8e6e-6bcd979bcd21"}]: dispatch 2023-10-09T10:33:38.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 ceph-mon[117223]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3015cd6e-b270-44c3-8e6e-6bcd979bcd21"}]': finished 2023-10-09T10:33:38.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 ceph-mon[117223]: osdmap e22: 4 total, 3 up, 4 in 2023-10-09T10:33:38.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:38.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 ceph-mon[117223]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T10:33:38.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 sudo[120326]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T10:33:38.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 sudo[120326]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T10:33:38.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 sudo[120326]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T10:33:38.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 sudo[120326]: pam_unix(sudo:session): session closed for user root 2023-10-09T10:33:38.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[112589]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T10:33:38.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[112589]: mgrmap e15: y(active, since 101s), standbys: x 2023-10-09T10:33:38.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/641578152' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3015cd6e-b270-44c3-8e6e-6bcd979bcd21"}]: dispatch 2023-10-09T10:33:38.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[112589]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3015cd6e-b270-44c3-8e6e-6bcd979bcd21"}]: dispatch 2023-10-09T10:33:38.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[112589]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3015cd6e-b270-44c3-8e6e-6bcd979bcd21"}]': finished 2023-10-09T10:33:38.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[112589]: osdmap e22: 4 total, 3 up, 4 in 2023-10-09T10:33:38.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:38.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[112589]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T10:33:38.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[127026]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T10:33:38.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[127026]: mgrmap e15: y(active, since 101s), standbys: x 2023-10-09T10:33:38.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/641578152' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3015cd6e-b270-44c3-8e6e-6bcd979bcd21"}]: dispatch 2023-10-09T10:33:38.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[127026]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3015cd6e-b270-44c3-8e6e-6bcd979bcd21"}]: dispatch 2023-10-09T10:33:38.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[127026]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3015cd6e-b270-44c3-8e6e-6bcd979bcd21"}]': finished 2023-10-09T10:33:38.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[127026]: osdmap e22: 4 total, 3 up, 4 in 2023-10-09T10:33:38.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:38.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:38 smithi125 ceph-mon[127026]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T10:33:38.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 sudo[120329]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T10:33:38.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 sudo[120329]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T10:33:38.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 sudo[120329]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T10:33:38.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:38 smithi155 sudo[120329]: pam_unix(sudo:session): session closed for user root 2023-10-09T10:33:39.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:38 smithi125 sudo[153190]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T10:33:39.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:38 smithi125 sudo[153190]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T10:33:39.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:38 smithi125 sudo[153190]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T10:33:39.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:38 smithi125 sudo[153190]: pam_unix(sudo:session): session closed for user root 2023-10-09T10:33:39.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3812985304' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:33:39.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: pgmap v63: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T10:33:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T10:33:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:33:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:33:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:33:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T10:33:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:33:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:33:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:33:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T10:33:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T10:33:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:33:39.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:33:39.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:33:39.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:39 smithi155 ceph-mon[117223]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T10:33:39.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3812985304' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:33:39.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: pgmap v63: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T10:33:39.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T10:33:39.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:33:39.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:33:39.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:33:39.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T10:33:39.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:33:39.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:33:39.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:33:39.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T10:33:39.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:39.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T10:33:39.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:33:39.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:33:39.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:33:39.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[112589]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T10:33:39.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 sudo[153380]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T10:33:39.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 sudo[153380]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T10:33:39.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 sudo[153380]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T10:33:39.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 sudo[153380]: pam_unix(sudo:session): session closed for user root 2023-10-09T10:33:39.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3812985304' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:33:39.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: pgmap v63: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T10:33:39.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T10:33:39.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:33:39.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:33:39.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:33:39.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T10:33:39.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:33:39.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:33:39.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:33:39.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T10:33:39.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:39.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T10:33:39.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:33:39.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:33:39.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:33:39.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:39 smithi125 ceph-mon[127026]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T10:33:41.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:41 smithi125 ceph-mon[112589]: pgmap v64: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T10:33:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:41 smithi125 ceph-mon[127026]: pgmap v64: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T10:33:41.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:41 smithi155 ceph-mon[117223]: pgmap v64: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T10:33:43.316 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:43 smithi125 ceph-mon[112589]: pgmap v65: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T10:33:43.316 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:43 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T10:33:43.316 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:43 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:43.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:43 smithi125 ceph-mon[112589]: Deploying daemon osd.3 on smithi125 2023-10-09T10:33:43.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:43 smithi125 ceph-mon[127026]: pgmap v65: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T10:33:43.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:43 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T10:33:43.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:43 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:43.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:43 smithi125 ceph-mon[127026]: Deploying daemon osd.3 on smithi125 2023-10-09T10:33:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:43 smithi155 ceph-mon[117223]: pgmap v65: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T10:33:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:43 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T10:33:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:43 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:43.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:43 smithi155 ceph-mon[117223]: Deploying daemon osd.3 on smithi125 2023-10-09T10:33:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:45 smithi155 ceph-mon[117223]: pgmap v66: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 65 KiB/s wr, 5 op/s 2023-10-09T10:33:45.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:45 smithi125 ceph-mon[112589]: pgmap v66: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 65 KiB/s wr, 5 op/s 2023-10-09T10:33:45.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:45 smithi125 ceph-mon[127026]: pgmap v66: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 65 KiB/s wr, 5 op/s 2023-10-09T10:33:46.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:46 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:46.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:46 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:46.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:46 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:46.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:46 smithi125 ceph-mon[112589]: pgmap v67: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 44 KiB/s wr, 3 op/s 2023-10-09T10:33:46.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:46 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:46.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:46 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:46.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:46 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:46.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:46 smithi125 ceph-mon[127026]: pgmap v67: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 44 KiB/s wr, 3 op/s 2023-10-09T10:33:46.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:46 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:46.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:46 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:46.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:46 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:46.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:46 smithi155 ceph-mon[117223]: pgmap v67: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 44 KiB/s wr, 3 op/s 2023-10-09T10:33:47.869 INFO:teuthology.orchestra.run.smithi125.stdout:Created osd(s) 3 on host 'smithi125' 2023-10-09T10:33:48.229 DEBUG:teuthology.orchestra.run.smithi125:osd.3> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-09T10:33:48.231 INFO:tasks.cephadm:Deploying osd.4 on smithi155 with /dev/vg_nvme/lv_4... 2023-10-09T10:33:48.231 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T10:33:48.263 INFO:journalctl@ceph.osd.3.smithi125.stdout:-- Logs begin at Mon 2023-10-09 10:18:49 UTC. -- 2023-10-09T10:33:48.500 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:33:48.578 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:48 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:48.578 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:48 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:48.579 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:48 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:48.579 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:48 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:48.579 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:48 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:48.579 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:48 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:48.579 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:48 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:48.579 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:48 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:48.756 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:48.756 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:48.756 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:48.756 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:48.757 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:48.757 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:48.757 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:48.757 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:48.757 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:48.757 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:48.757 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:48.757 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:48.758 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:48.758 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:48.758 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:48.758 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:33:49.070 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:33:48 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[154655]: 2023-10-09T10:33:48.915+0000 7f57aae66540 -1 osd.3 0 log_to_monitors true 2023-10-09T10:33:49.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:49 smithi155 ceph-mon[117223]: pgmap v68: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.1 KiB/s rd, 33 KiB/s wr, 2 op/s 2023-10-09T10:33:49.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:49 smithi155 ceph-mon[117223]: from='osd.3 [v2:172.21.15.125:6826/2950193979,v1:172.21.15.125:6827/2950193979]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:33:49.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:49 smithi155 ceph-mon[117223]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:33:49.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:49 smithi125 ceph-mon[112589]: pgmap v68: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.1 KiB/s rd, 33 KiB/s wr, 2 op/s 2023-10-09T10:33:49.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:49 smithi125 ceph-mon[112589]: from='osd.3 [v2:172.21.15.125:6826/2950193979,v1:172.21.15.125:6827/2950193979]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:33:49.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:49 smithi125 ceph-mon[112589]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:33:49.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:49 smithi125 ceph-mon[127026]: pgmap v68: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.1 KiB/s rd, 33 KiB/s wr, 2 op/s 2023-10-09T10:33:49.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:49 smithi125 ceph-mon[127026]: from='osd.3 [v2:172.21.15.125:6826/2950193979,v1:172.21.15.125:6827/2950193979]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:33:49.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:49 smithi125 ceph-mon[127026]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:33:49.825 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T10:33:49.825 INFO:teuthology.orchestra.run.smithi155.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T10:33:49.825 INFO:teuthology.orchestra.run.smithi155.stderr: stderr: 10+0 records in 2023-10-09T10:33:49.826 INFO:teuthology.orchestra.run.smithi155.stderr:10+0 records out 2023-10-09T10:33:49.826 INFO:teuthology.orchestra.run.smithi155.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112002 s, 936 MB/s 2023-10-09T10:33:49.826 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping successful for: 2023-10-09T10:33:50.217 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi155:vg_nvme/lv_4 2023-10-09T10:33:50.480 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:33:51.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T10:33:51.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: osdmap e23: 4 total, 3 up, 4 in 2023-10-09T10:33:51.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:51.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: from='osd.3 [v2:172.21.15.125:6826/2950193979,v1:172.21.15.125:6827/2950193979]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:33:51.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:33:51.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: pgmap v70: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 921 B/s rd, 26 KiB/s wr, 2 op/s 2023-10-09T10:33:51.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: Detected new or changed devices on smithi125 2023-10-09T10:33:51.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:51.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:51.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:51.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:51.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:51.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:51.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: Adjusting osd_memory_target on smithi125 to 4022M 2023-10-09T10:33:51.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:51.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:51.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:51.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:50 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:51.321 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[154655]: 2023-10-09T10:33:50.852+0000 7f57a164b700 -1 osd.3 0 waiting for initial osdmap 2023-10-09T10:33:51.321 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[154655]: 2023-10-09T10:33:50.857+0000 7f579bc45700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:33:51.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T10:33:51.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: osdmap e23: 4 total, 3 up, 4 in 2023-10-09T10:33:51.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:51.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: from='osd.3 [v2:172.21.15.125:6826/2950193979,v1:172.21.15.125:6827/2950193979]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:33:51.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:33:51.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: pgmap v70: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 921 B/s rd, 26 KiB/s wr, 2 op/s 2023-10-09T10:33:51.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: Detected new or changed devices on smithi125 2023-10-09T10:33:51.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:51.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:51.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:51.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:51.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:51.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:51.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: Adjusting osd_memory_target on smithi125 to 4022M 2023-10-09T10:33:51.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:51.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:51.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:51.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:51.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T10:33:51.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: osdmap e23: 4 total, 3 up, 4 in 2023-10-09T10:33:51.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:51.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: from='osd.3 [v2:172.21.15.125:6826/2950193979,v1:172.21.15.125:6827/2950193979]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:33:51.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:33:51.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: pgmap v70: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 921 B/s rd, 26 KiB/s wr, 2 op/s 2023-10-09T10:33:51.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: Detected new or changed devices on smithi125 2023-10-09T10:33:51.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:51.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:51.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:51.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:51.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:51.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:33:51.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: Adjusting osd_memory_target on smithi125 to 4022M 2023-10-09T10:33:51.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:51.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:51.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:33:51.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:50 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:33:52.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:51 smithi155 ceph-mon[117223]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T10:33:52.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:51 smithi155 ceph-mon[117223]: osdmap e24: 4 total, 3 up, 4 in 2023-10-09T10:33:52.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:51 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:52.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:51 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:52.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:51 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:33:52.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:51 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:33:52.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:51 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:52.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:51 smithi125 ceph-mon[112589]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T10:33:52.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:51 smithi125 ceph-mon[112589]: osdmap e24: 4 total, 3 up, 4 in 2023-10-09T10:33:52.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:51 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:52.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:51 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:52.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:51 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:33:52.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:51 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:33:52.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:51 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:52.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:51 smithi125 ceph-mon[127026]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T10:33:52.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:51 smithi125 ceph-mon[127026]: osdmap e24: 4 total, 3 up, 4 in 2023-10-09T10:33:52.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:51 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:52.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:51 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:52.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:51 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:33:52.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:51 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:33:52.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:51 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:52.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:52 smithi155 ceph-mon[117223]: purged_snaps scrub starts 2023-10-09T10:33:52.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:52 smithi155 ceph-mon[117223]: purged_snaps scrub ok 2023-10-09T10:33:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:52 smithi155 ceph-mon[117223]: from='client.24196 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:33:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:52 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:52 smithi155 ceph-mon[117223]: osd.3 [v2:172.21.15.125:6826/2950193979,v1:172.21.15.125:6827/2950193979] boot 2023-10-09T10:33:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:52 smithi155 ceph-mon[117223]: osdmap e25: 4 total, 4 up, 4 in 2023-10-09T10:33:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:52 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:52 smithi155 ceph-mon[117223]: pgmap v73: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T10:33:53.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[112589]: purged_snaps scrub starts 2023-10-09T10:33:53.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[112589]: purged_snaps scrub ok 2023-10-09T10:33:53.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[112589]: from='client.24196 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:33:53.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:53.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[112589]: osd.3 [v2:172.21.15.125:6826/2950193979,v1:172.21.15.125:6827/2950193979] boot 2023-10-09T10:33:53.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[112589]: osdmap e25: 4 total, 4 up, 4 in 2023-10-09T10:33:53.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:53.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[112589]: pgmap v73: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T10:33:53.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[127026]: purged_snaps scrub starts 2023-10-09T10:33:53.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[127026]: purged_snaps scrub ok 2023-10-09T10:33:53.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[127026]: from='client.24196 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:33:53.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:53.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[127026]: osd.3 [v2:172.21.15.125:6826/2950193979,v1:172.21.15.125:6827/2950193979] boot 2023-10-09T10:33:53.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[127026]: osdmap e25: 4 total, 4 up, 4 in 2023-10-09T10:33:53.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:33:53.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:52 smithi125 ceph-mon[127026]: pgmap v73: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T10:33:54.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:53 smithi155 ceph-mon[117223]: osdmap e26: 4 total, 4 up, 4 in 2023-10-09T10:33:54.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:53 smithi155 ceph-mon[117223]: from='client.? 172.21.15.155:0/2373979020' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "272c7b95-f89c-4fe1-a299-a861e0ec9043"}]: dispatch 2023-10-09T10:33:54.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:53 smithi155 ceph-mon[117223]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "272c7b95-f89c-4fe1-a299-a861e0ec9043"}]: dispatch 2023-10-09T10:33:54.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:53 smithi155 ceph-mon[117223]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "272c7b95-f89c-4fe1-a299-a861e0ec9043"}]': finished 2023-10-09T10:33:54.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:53 smithi155 ceph-mon[117223]: osdmap e27: 5 total, 4 up, 5 in 2023-10-09T10:33:54.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:53 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:33:54.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:53 smithi125 ceph-mon[112589]: osdmap e26: 4 total, 4 up, 4 in 2023-10-09T10:33:54.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:53 smithi125 ceph-mon[112589]: from='client.? 172.21.15.155:0/2373979020' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "272c7b95-f89c-4fe1-a299-a861e0ec9043"}]: dispatch 2023-10-09T10:33:54.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:53 smithi125 ceph-mon[112589]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "272c7b95-f89c-4fe1-a299-a861e0ec9043"}]: dispatch 2023-10-09T10:33:54.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:53 smithi125 ceph-mon[112589]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "272c7b95-f89c-4fe1-a299-a861e0ec9043"}]': finished 2023-10-09T10:33:54.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:53 smithi125 ceph-mon[112589]: osdmap e27: 5 total, 4 up, 5 in 2023-10-09T10:33:54.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:53 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:33:54.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:53 smithi125 ceph-mon[127026]: osdmap e26: 4 total, 4 up, 4 in 2023-10-09T10:33:54.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:53 smithi125 ceph-mon[127026]: from='client.? 172.21.15.155:0/2373979020' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "272c7b95-f89c-4fe1-a299-a861e0ec9043"}]: dispatch 2023-10-09T10:33:54.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:53 smithi125 ceph-mon[127026]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "272c7b95-f89c-4fe1-a299-a861e0ec9043"}]: dispatch 2023-10-09T10:33:54.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:53 smithi125 ceph-mon[127026]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "272c7b95-f89c-4fe1-a299-a861e0ec9043"}]': finished 2023-10-09T10:33:54.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:53 smithi125 ceph-mon[127026]: osdmap e27: 5 total, 4 up, 5 in 2023-10-09T10:33:54.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:53 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:33:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:54 smithi155 ceph-mon[117223]: from='client.? 172.21.15.155:0/1571954457' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:33:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:54 smithi155 ceph-mon[117223]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:33:55.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:54 smithi125 ceph-mon[112589]: from='client.? 172.21.15.155:0/1571954457' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:33:55.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:54 smithi125 ceph-mon[112589]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:33:55.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:54 smithi125 ceph-mon[127026]: from='client.? 172.21.15.155:0/1571954457' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:33:55.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:54 smithi125 ceph-mon[127026]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:33:57.463 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:57 smithi155 ceph-mon[117223]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:33:57.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:57 smithi125 ceph-mon[112589]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:33:57.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:57 smithi125 ceph-mon[127026]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:33:59.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:59 smithi155 ceph-mon[117223]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:33:59.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:59 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T10:33:59.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:59 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:59.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:33:59 smithi155 ceph-mon[117223]: Deploying daemon osd.4 on smithi155 2023-10-09T10:33:59.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:59 smithi125 ceph-mon[112589]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:33:59.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:59 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T10:33:59.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:59 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:59.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:33:59 smithi125 ceph-mon[112589]: Deploying daemon osd.4 on smithi155 2023-10-09T10:33:59.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:59 smithi125 ceph-mon[127026]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:33:59.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:59 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T10:33:59.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:59 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:33:59.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:33:59 smithi125 ceph-mon[127026]: Deploying daemon osd.4 on smithi155 2023-10-09T10:34:01.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:01 smithi155 ceph-mon[117223]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:34:01.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:01 smithi125 ceph-mon[112589]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:34:01.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:01 smithi125 ceph-mon[127026]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:34:03.113 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:02 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:03.114 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:02 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:03.114 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:02 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:03.114 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:02 smithi155 ceph-mon[117223]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:34:03.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:02 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:03.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:02 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:03.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:02 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:03.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:02 smithi125 ceph-mon[127026]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:34:03.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:02 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:03.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:02 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:03.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:02 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:03.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:02 smithi125 ceph-mon[112589]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:34:03.895 INFO:teuthology.orchestra.run.smithi155.stdout:Created osd(s) 4 on host 'smithi155' 2023-10-09T10:34:04.070 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:04 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:04.071 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:04 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:04.071 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:04 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:04.071 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:04 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:04.071 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:04 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:04.071 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:04 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:04.071 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:04 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:04.071 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:04 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:04.274 DEBUG:teuthology.orchestra.run.smithi155:osd.4> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T10:34:04.277 INFO:tasks.cephadm:Deploying osd.5 on smithi155 with /dev/vg_nvme/lv_3... 2023-10-09T10:34:04.277 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T10:34:04.309 INFO:journalctl@ceph.osd.4.smithi155.stdout:-- Logs begin at Mon 2023-10-09 10:18:52 UTC. -- 2023-10-09T10:34:04.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:04.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:04.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:04.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:04.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:04.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:04.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:04.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:04.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:04.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:04.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:04.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:04.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:04.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:04.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:04.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:04 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:04.875 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:34:05.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:05 smithi125 ceph-mon[112589]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:34:05.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:05 smithi125 ceph-mon[127026]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:34:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:05 smithi155 ceph-mon[117223]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:34:05.470 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 10:34:05 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-4[123099]: 2023-10-09T10:34:05.161+0000 7f0a89a60540 -1 osd.4 0 log_to_monitors true 2023-10-09T10:34:06.283 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T10:34:06.283 INFO:teuthology.orchestra.run.smithi155.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T10:34:06.283 INFO:teuthology.orchestra.run.smithi155.stderr: stderr: 10+0 records in 2023-10-09T10:34:06.283 INFO:teuthology.orchestra.run.smithi155.stderr:10+0 records out 2023-10-09T10:34:06.283 INFO:teuthology.orchestra.run.smithi155.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0134983 s, 777 MB/s 2023-10-09T10:34:06.283 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping successful for: 2023-10-09T10:34:06.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:06 smithi125 ceph-mon[112589]: from='osd.4 [v2:172.21.15.155:6800/1086065767,v1:172.21.15.155:6801/1086065767]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T10:34:06.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:06 smithi125 ceph-mon[112589]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T10:34:06.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:06 smithi125 ceph-mon[127026]: from='osd.4 [v2:172.21.15.155:6800/1086065767,v1:172.21.15.155:6801/1086065767]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T10:34:06.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:06 smithi125 ceph-mon[127026]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T10:34:06.327 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:06 smithi155 ceph-mon[117223]: from='osd.4 [v2:172.21.15.155:6800/1086065767,v1:172.21.15.155:6801/1086065767]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T10:34:06.327 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:06 smithi155 ceph-mon[117223]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T10:34:06.793 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi155:vg_nvme/lv_3 2023-10-09T10:34:07.051 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:34:07.317 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 10:34:07 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-4[123099]: 2023-10-09T10:34:07.055+0000 7f0a80245700 -1 osd.4 0 waiting for initial osdmap 2023-10-09T10:34:07.317 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 10:34:07 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-4[123099]: 2023-10-09T10:34:07.059+0000 7f0a7a83f700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:34:07.317 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:07 smithi155 ceph-mon[117223]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T10:34:07.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:07 smithi155 ceph-mon[117223]: osdmap e28: 5 total, 4 up, 5 in 2023-10-09T10:34:07.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:07 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:34:07.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:07 smithi155 ceph-mon[117223]: from='osd.4 [v2:172.21.15.155:6800/1086065767,v1:172.21.15.155:6801/1086065767]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:07.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:07 smithi155 ceph-mon[117223]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:07.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:07 smithi155 ceph-mon[117223]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:34:07.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:07 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:07.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:07 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:07.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:07 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:07.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:07 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:07.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:07 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:07.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:07 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:07.319 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:07 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:07.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[112589]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T10:34:07.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[112589]: osdmap e28: 5 total, 4 up, 5 in 2023-10-09T10:34:07.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:34:07.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[112589]: from='osd.4 [v2:172.21.15.155:6800/1086065767,v1:172.21.15.155:6801/1086065767]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:07.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[112589]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:07.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[112589]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:34:07.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:07.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:07.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:07.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:07.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:07.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:07.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:07.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[127026]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T10:34:07.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[127026]: osdmap e28: 5 total, 4 up, 5 in 2023-10-09T10:34:07.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:34:07.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[127026]: from='osd.4 [v2:172.21.15.155:6800/1086065767,v1:172.21.15.155:6801/1086065767]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:07.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[127026]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:07.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[127026]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T10:34:07.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:07.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:07.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:07.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:07.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:07.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:07.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:07 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:08.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:08 smithi155 ceph-mon[117223]: Detected new or changed devices on smithi155 2023-10-09T10:34:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:08 smithi155 ceph-mon[117223]: Adjusting osd_memory_target on smithi155 to 17114M 2023-10-09T10:34:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:08 smithi155 ceph-mon[117223]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T10:34:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:08 smithi155 ceph-mon[117223]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T10:34:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:08 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:34:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:08 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:34:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:08 smithi155 ceph-mon[117223]: osd.4 [v2:172.21.15.155:6800/1086065767,v1:172.21.15.155:6801/1086065767] boot 2023-10-09T10:34:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:08 smithi155 ceph-mon[117223]: osdmap e30: 5 total, 5 up, 5 in 2023-10-09T10:34:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:08 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:34:08.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[112589]: Detected new or changed devices on smithi155 2023-10-09T10:34:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[112589]: Adjusting osd_memory_target on smithi155 to 17114M 2023-10-09T10:34:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[112589]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T10:34:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[112589]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T10:34:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:34:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:34:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[112589]: osd.4 [v2:172.21.15.155:6800/1086065767,v1:172.21.15.155:6801/1086065767] boot 2023-10-09T10:34:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[112589]: osdmap e30: 5 total, 5 up, 5 in 2023-10-09T10:34:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:34:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[127026]: Detected new or changed devices on smithi155 2023-10-09T10:34:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[127026]: Adjusting osd_memory_target on smithi155 to 17114M 2023-10-09T10:34:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[127026]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T10:34:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[127026]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T10:34:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:34:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:34:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[127026]: osd.4 [v2:172.21.15.155:6800/1086065767,v1:172.21.15.155:6801/1086065767] boot 2023-10-09T10:34:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[127026]: osdmap e30: 5 total, 5 up, 5 in 2023-10-09T10:34:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:08 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:34:09.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:09 smithi155 ceph-mon[117223]: purged_snaps scrub starts 2023-10-09T10:34:09.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:09 smithi155 ceph-mon[117223]: purged_snaps scrub ok 2023-10-09T10:34:09.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:09 smithi155 ceph-mon[117223]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T10:34:09.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:09 smithi155 ceph-mon[117223]: from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:34:09.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:09 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:34:09.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:09 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:34:09.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:09 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:09.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:09 smithi155 ceph-mon[117223]: osdmap e31: 5 total, 5 up, 5 in 2023-10-09T10:34:09.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[112589]: purged_snaps scrub starts 2023-10-09T10:34:09.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[112589]: purged_snaps scrub ok 2023-10-09T10:34:09.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[112589]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T10:34:09.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[112589]: from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:34:09.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:34:09.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:34:09.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:09.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[112589]: osdmap e31: 5 total, 5 up, 5 in 2023-10-09T10:34:09.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[127026]: purged_snaps scrub starts 2023-10-09T10:34:09.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[127026]: purged_snaps scrub ok 2023-10-09T10:34:09.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[127026]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T10:34:09.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[127026]: from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:34:09.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:34:09.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:34:09.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:09.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:09 smithi125 ceph-mon[127026]: osdmap e31: 5 total, 5 up, 5 in 2023-10-09T10:34:10.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:10 smithi155 ceph-mon[117223]: osdmap e32: 5 total, 5 up, 5 in 2023-10-09T10:34:10.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:10 smithi155 ceph-mon[117223]: from='client.? 172.21.15.155:0/2682900876' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "201d51f2-233c-40fc-a42d-649d225954fb"}]: dispatch 2023-10-09T10:34:10.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:10 smithi155 ceph-mon[117223]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "201d51f2-233c-40fc-a42d-649d225954fb"}]: dispatch 2023-10-09T10:34:10.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:10 smithi155 ceph-mon[117223]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "201d51f2-233c-40fc-a42d-649d225954fb"}]': finished 2023-10-09T10:34:10.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:10 smithi155 ceph-mon[117223]: osdmap e33: 6 total, 5 up, 6 in 2023-10-09T10:34:10.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:10 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:10.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:10 smithi125 ceph-mon[127026]: osdmap e32: 5 total, 5 up, 5 in 2023-10-09T10:34:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:10 smithi125 ceph-mon[127026]: from='client.? 172.21.15.155:0/2682900876' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "201d51f2-233c-40fc-a42d-649d225954fb"}]: dispatch 2023-10-09T10:34:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:10 smithi125 ceph-mon[127026]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "201d51f2-233c-40fc-a42d-649d225954fb"}]: dispatch 2023-10-09T10:34:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:10 smithi125 ceph-mon[127026]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "201d51f2-233c-40fc-a42d-649d225954fb"}]': finished 2023-10-09T10:34:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:10 smithi125 ceph-mon[127026]: osdmap e33: 6 total, 5 up, 6 in 2023-10-09T10:34:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:10 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:10.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:10 smithi125 ceph-mon[112589]: osdmap e32: 5 total, 5 up, 5 in 2023-10-09T10:34:10.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:10 smithi125 ceph-mon[112589]: from='client.? 172.21.15.155:0/2682900876' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "201d51f2-233c-40fc-a42d-649d225954fb"}]: dispatch 2023-10-09T10:34:10.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:10 smithi125 ceph-mon[112589]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "201d51f2-233c-40fc-a42d-649d225954fb"}]: dispatch 2023-10-09T10:34:10.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:10 smithi125 ceph-mon[112589]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "201d51f2-233c-40fc-a42d-649d225954fb"}]': finished 2023-10-09T10:34:10.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:10 smithi125 ceph-mon[112589]: osdmap e33: 6 total, 5 up, 6 in 2023-10-09T10:34:10.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:10 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:11.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:11 smithi125 ceph-mon[112589]: pgmap v90: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T10:34:11.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:11 smithi125 ceph-mon[112589]: from='client.? 172.21.15.155:0/3460328598' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:34:11.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:11 smithi125 ceph-mon[112589]: osdmap e34: 6 total, 5 up, 6 in 2023-10-09T10:34:11.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:11 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:11.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:11 smithi125 ceph-mon[127026]: pgmap v90: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T10:34:11.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:11 smithi125 ceph-mon[127026]: from='client.? 172.21.15.155:0/3460328598' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:34:11.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:11 smithi125 ceph-mon[127026]: osdmap e34: 6 total, 5 up, 6 in 2023-10-09T10:34:11.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:11 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:11 smithi155 ceph-mon[117223]: pgmap v90: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T10:34:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:11 smithi155 ceph-mon[117223]: from='client.? 172.21.15.155:0/3460328598' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:34:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:11 smithi155 ceph-mon[117223]: osdmap e34: 6 total, 5 up, 6 in 2023-10-09T10:34:11.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:11 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:13.789 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:13 smithi155 ceph-mon[117223]: pgmap v92: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T10:34:13.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:13 smithi125 ceph-mon[112589]: pgmap v92: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T10:34:13.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:13 smithi125 ceph-mon[127026]: pgmap v92: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T10:34:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:15 smithi155 ceph-mon[117223]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-10-09T10:34:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:15 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T10:34:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:15 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:15.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:15 smithi125 ceph-mon[112589]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-10-09T10:34:15.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:15 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T10:34:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:15 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:15.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:15 smithi125 ceph-mon[127026]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-10-09T10:34:15.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:15 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T10:34:15.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:15 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:16.746 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:16 smithi155 ceph-mon[117223]: Deploying daemon osd.5 on smithi155 2023-10-09T10:34:16.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:16 smithi125 ceph-mon[112589]: Deploying daemon osd.5 on smithi155 2023-10-09T10:34:16.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:16 smithi125 ceph-mon[127026]: Deploying daemon osd.5 on smithi155 2023-10-09T10:34:17.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:17 smithi155 ceph-mon[117223]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 87 KiB/s, 0 objects/s recovering 2023-10-09T10:34:17.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:17 smithi125 ceph-mon[112589]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 87 KiB/s, 0 objects/s recovering 2023-10-09T10:34:17.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:17 smithi125 ceph-mon[127026]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 87 KiB/s, 0 objects/s recovering 2023-10-09T10:34:19.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:19 smithi125 ceph-mon[112589]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T10:34:19.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:19 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:19.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:19 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:19.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:19 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:19.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:19 smithi125 ceph-mon[127026]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T10:34:19.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:19 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:19.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:19 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:19.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:19 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:19.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:19 smithi155 ceph-mon[117223]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T10:34:19.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:19.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:19.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:19 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:20.587 INFO:teuthology.orchestra.run.smithi155.stdout:Created osd(s) 5 on host 'smithi155' 2023-10-09T10:34:20.946 DEBUG:teuthology.orchestra.run.smithi155:osd.5> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-09T10:34:20.948 INFO:tasks.cephadm:Deploying osd.6 on smithi155 with /dev/vg_nvme/lv_2... 2023-10-09T10:34:20.948 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T10:34:20.977 INFO:journalctl@ceph.osd.5.smithi155.stdout:-- Logs begin at Mon 2023-10-09 10:18:52 UTC. -- 2023-10-09T10:34:21.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:20 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:21.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:20 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:21.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:20 smithi155 ceph-mon[117223]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T10:34:21.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:20 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:21.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:20 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:21.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:20 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:21.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:20 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:21.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:20 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:21.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:20 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:21.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[127026]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T10:34:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:21.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:21.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:21.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[112589]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T10:34:21.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:21.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:21.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:21.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:21.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:21.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:20 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:21.388 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:34:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:21 smithi155 ceph-mon[117223]: from='osd.5 [v2:172.21.15.155:6808/104536199,v1:172.21.15.155:6809/104536199]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T10:34:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:21 smithi155 ceph-mon[117223]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T10:34:22.221 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 10:34:21 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T10:34:21.756+0000 7fbf202a7540 -1 osd.5 0 log_to_monitors true 2023-10-09T10:34:22.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:21 smithi125 ceph-mon[112589]: from='osd.5 [v2:172.21.15.155:6808/104536199,v1:172.21.15.155:6809/104536199]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T10:34:22.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:21 smithi125 ceph-mon[112589]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T10:34:22.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:21 smithi125 ceph-mon[127026]: from='osd.5 [v2:172.21.15.155:6808/104536199,v1:172.21.15.155:6809/104536199]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T10:34:22.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:21 smithi125 ceph-mon[127026]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T10:34:22.949 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T10:34:22.949 INFO:teuthology.orchestra.run.smithi155.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T10:34:22.949 INFO:teuthology.orchestra.run.smithi155.stderr: stderr: 10+0 records in 2023-10-09T10:34:22.950 INFO:teuthology.orchestra.run.smithi155.stderr:10+0 records out 2023-10-09T10:34:22.950 INFO:teuthology.orchestra.run.smithi155.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0125994 s, 832 MB/s 2023-10-09T10:34:22.950 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping successful for: 2023-10-09T10:34:23.146 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 10:34:22 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T10:34:22.864+0000 7fbf16a8c700 -1 osd.5 0 waiting for initial osdmap 2023-10-09T10:34:23.146 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 10:34:22 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T10:34:22.869+0000 7fbf0f883700 -1 osd.5 36 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:34:23.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:22 smithi155 ceph-mon[117223]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T10:34:23.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:22 smithi155 ceph-mon[117223]: osdmap e35: 6 total, 5 up, 6 in 2023-10-09T10:34:23.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:22 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:23.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:22 smithi155 ceph-mon[117223]: from='osd.5 [v2:172.21.15.155:6808/104536199,v1:172.21.15.155:6809/104536199]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:23.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:22 smithi155 ceph-mon[117223]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:23.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:22 smithi155 ceph-mon[117223]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T10:34:23.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:22 smithi125 ceph-mon[112589]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T10:34:23.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:22 smithi125 ceph-mon[112589]: osdmap e35: 6 total, 5 up, 6 in 2023-10-09T10:34:23.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:22 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:23.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:22 smithi125 ceph-mon[112589]: from='osd.5 [v2:172.21.15.155:6808/104536199,v1:172.21.15.155:6809/104536199]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:23.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:22 smithi125 ceph-mon[112589]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:23.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:22 smithi125 ceph-mon[112589]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T10:34:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:22 smithi125 ceph-mon[127026]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T10:34:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:22 smithi125 ceph-mon[127026]: osdmap e35: 6 total, 5 up, 6 in 2023-10-09T10:34:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:22 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:22 smithi125 ceph-mon[127026]: from='osd.5 [v2:172.21.15.155:6808/104536199,v1:172.21.15.155:6809/104536199]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:22 smithi125 ceph-mon[127026]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:22 smithi125 ceph-mon[127026]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T10:34:23.498 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi155:vg_nvme/lv_2 2023-10-09T10:34:23.757 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:34:24.209 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T10:34:24.209 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: osdmap e36: 6 total, 5 up, 6 in 2023-10-09T10:34:24.209 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:24.209 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:24.209 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: Detected new or changed devices on smithi155 2023-10-09T10:34:24.209 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:24.209 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:24.209 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:24.209 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:24.209 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: Adjusting osd_memory_target on smithi155 to 8557M 2023-10-09T10:34:24.210 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:24.210 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:24.210 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:24.210 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:24.210 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: osd.5 [v2:172.21.15.155:6808/104536199,v1:172.21.15.155:6809/104536199] boot 2023-10-09T10:34:24.210 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: osdmap e37: 6 total, 6 up, 6 in 2023-10-09T10:34:24.210 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:23 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:24.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T10:34:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: osdmap e36: 6 total, 5 up, 6 in 2023-10-09T10:34:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: Detected new or changed devices on smithi155 2023-10-09T10:34:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: Adjusting osd_memory_target on smithi155 to 8557M 2023-10-09T10:34:24.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:24.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:24.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:24.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:24.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: osd.5 [v2:172.21.15.155:6808/104536199,v1:172.21.15.155:6809/104536199] boot 2023-10-09T10:34:24.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: osdmap e37: 6 total, 6 up, 6 in 2023-10-09T10:34:24.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:24.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T10:34:24.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: osdmap e36: 6 total, 5 up, 6 in 2023-10-09T10:34:24.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:24.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:24.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: Detected new or changed devices on smithi155 2023-10-09T10:34:24.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:24.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:24.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:24.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:24.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: Adjusting osd_memory_target on smithi155 to 8557M 2023-10-09T10:34:24.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:24.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:24.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:24.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:24.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: osd.5 [v2:172.21.15.155:6808/104536199,v1:172.21.15.155:6809/104536199] boot 2023-10-09T10:34:24.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: osdmap e37: 6 total, 6 up, 6 in 2023-10-09T10:34:24.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:23 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:34:25.633 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:25 smithi155 ceph-mon[117223]: purged_snaps scrub starts 2023-10-09T10:34:25.634 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:25 smithi155 ceph-mon[117223]: purged_snaps scrub ok 2023-10-09T10:34:25.634 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:25 smithi155 ceph-mon[117223]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:25.634 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:25 smithi155 ceph-mon[117223]: osdmap e38: 6 total, 6 up, 6 in 2023-10-09T10:34:25.634 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:25 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:34:25.634 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:25 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:34:25.634 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:25 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:25.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:25 smithi125 ceph-mon[112589]: purged_snaps scrub starts 2023-10-09T10:34:25.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:25 smithi125 ceph-mon[112589]: purged_snaps scrub ok 2023-10-09T10:34:25.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:25 smithi125 ceph-mon[112589]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:25.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:25 smithi125 ceph-mon[112589]: osdmap e38: 6 total, 6 up, 6 in 2023-10-09T10:34:25.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:25 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:34:25.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:25 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:34:25.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:25 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:25.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:25 smithi125 ceph-mon[127026]: purged_snaps scrub starts 2023-10-09T10:34:25.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:25 smithi125 ceph-mon[127026]: purged_snaps scrub ok 2023-10-09T10:34:25.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:25 smithi125 ceph-mon[127026]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:25.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:25 smithi125 ceph-mon[127026]: osdmap e38: 6 total, 6 up, 6 in 2023-10-09T10:34:25.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:25 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:34:25.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:25 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:34:25.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:25 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:26.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:26 smithi125 ceph-mon[112589]: from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:34:26.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:26 smithi125 ceph-mon[112589]: osdmap e39: 6 total, 6 up, 6 in 2023-10-09T10:34:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:26 smithi125 ceph-mon[127026]: from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:34:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:26 smithi125 ceph-mon[127026]: osdmap e39: 6 total, 6 up, 6 in 2023-10-09T10:34:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:26 smithi155 ceph-mon[117223]: from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:34:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:26 smithi155 ceph-mon[117223]: osdmap e39: 6 total, 6 up, 6 in 2023-10-09T10:34:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[112589]: pgmap v104: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[112589]: osdmap e40: 6 total, 6 up, 6 in 2023-10-09T10:34:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[112589]: from='client.? 172.21.15.155:0/756842423' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ec66642-108e-401f-aff3-e7ab258bc083"}]: dispatch 2023-10-09T10:34:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[112589]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ec66642-108e-401f-aff3-e7ab258bc083"}]: dispatch 2023-10-09T10:34:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[112589]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4ec66642-108e-401f-aff3-e7ab258bc083"}]': finished 2023-10-09T10:34:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[112589]: osdmap e41: 7 total, 6 up, 7 in 2023-10-09T10:34:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:34:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[112589]: from='client.? 172.21.15.155:0/1078753644' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:34:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[127026]: pgmap v104: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:27.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[127026]: osdmap e40: 6 total, 6 up, 6 in 2023-10-09T10:34:27.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[127026]: from='client.? 172.21.15.155:0/756842423' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ec66642-108e-401f-aff3-e7ab258bc083"}]: dispatch 2023-10-09T10:34:27.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[127026]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ec66642-108e-401f-aff3-e7ab258bc083"}]: dispatch 2023-10-09T10:34:27.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[127026]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4ec66642-108e-401f-aff3-e7ab258bc083"}]': finished 2023-10-09T10:34:27.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[127026]: osdmap e41: 7 total, 6 up, 7 in 2023-10-09T10:34:27.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:34:27.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:27 smithi125 ceph-mon[127026]: from='client.? 172.21.15.155:0/1078753644' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:34:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:27 smithi155 ceph-mon[117223]: pgmap v104: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:27 smithi155 ceph-mon[117223]: osdmap e40: 6 total, 6 up, 6 in 2023-10-09T10:34:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:27 smithi155 ceph-mon[117223]: from='client.? 172.21.15.155:0/756842423' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ec66642-108e-401f-aff3-e7ab258bc083"}]: dispatch 2023-10-09T10:34:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:27 smithi155 ceph-mon[117223]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ec66642-108e-401f-aff3-e7ab258bc083"}]: dispatch 2023-10-09T10:34:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:27 smithi155 ceph-mon[117223]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4ec66642-108e-401f-aff3-e7ab258bc083"}]': finished 2023-10-09T10:34:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:27 smithi155 ceph-mon[117223]: osdmap e41: 7 total, 6 up, 7 in 2023-10-09T10:34:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:27 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:34:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:27 smithi155 ceph-mon[117223]: from='client.? 172.21.15.155:0/1078753644' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:34:29.766 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:29 smithi155 ceph-mon[117223]: pgmap v107: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:29.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:29 smithi125 ceph-mon[112589]: pgmap v107: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:29 smithi125 ceph-mon[127026]: pgmap v107: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:31.803 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:31 smithi155 ceph-mon[117223]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:31.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:31 smithi125 ceph-mon[112589]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:31.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:31 smithi125 ceph-mon[127026]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:32.657 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:32 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T10:34:32.658 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:32 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:32.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:32 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T10:34:32.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:32 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:32.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:32 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T10:34:32.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:32 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:33 smithi155 ceph-mon[117223]: Deploying daemon osd.6 on smithi155 2023-10-09T10:34:33.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:33 smithi155 ceph-mon[117223]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:33.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:33 smithi125 ceph-mon[112589]: Deploying daemon osd.6 on smithi155 2023-10-09T10:34:33.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:33 smithi125 ceph-mon[112589]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:33.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:33 smithi125 ceph-mon[127026]: Deploying daemon osd.6 on smithi155 2023-10-09T10:34:33.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:33 smithi125 ceph-mon[127026]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:35 smithi155 ceph-mon[117223]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:35 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:35 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:35.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:35 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:35.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:35 smithi125 ceph-mon[112589]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:35.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:35 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:35.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:35 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:35.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:35 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:35.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:35 smithi125 ceph-mon[127026]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:35.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:35 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:35.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:35 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:35.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:35 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:37.271 INFO:teuthology.orchestra.run.smithi155.stdout:Created osd(s) 6 on host 'smithi155' 2023-10-09T10:34:37.566 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:37 smithi155 ceph-mon[117223]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:37.566 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:37 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:37.566 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:37 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:37.566 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:37 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:37.566 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:37 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:37.566 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:37 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:37.567 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:37 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:37.567 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:37 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:37.567 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:37 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:37.642 DEBUG:teuthology.orchestra.run.smithi155:osd.6> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-09T10:34:37.644 INFO:tasks.cephadm:Deploying osd.7 on smithi155 with /dev/vg_nvme/lv_1... 2023-10-09T10:34:37.644 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T10:34:37.677 INFO:journalctl@ceph.osd.6.smithi155.stdout:-- Logs begin at Mon 2023-10-09 10:18:52 UTC. -- 2023-10-09T10:34:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[112589]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:37.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[127026]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:37.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:37.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:37.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:37.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:37.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:37.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:37.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:37.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:37 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:38.358 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:34:38.582 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 10:34:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T10:34:38.568+0000 7fecce0c8540 -1 osd.6 0 log_to_monitors true 2023-10-09T10:34:39.770 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T10:34:39.770 INFO:teuthology.orchestra.run.smithi155.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T10:34:39.770 INFO:teuthology.orchestra.run.smithi155.stderr: stderr: 10+0 records in 2023-10-09T10:34:39.771 INFO:teuthology.orchestra.run.smithi155.stderr:10+0 records out 2023-10-09T10:34:39.771 INFO:teuthology.orchestra.run.smithi155.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111718 s, 939 MB/s 2023-10-09T10:34:39.771 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping successful for: 2023-10-09T10:34:39.803 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:39 smithi155 ceph-mon[117223]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:39.804 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:39 smithi155 ceph-mon[117223]: from='osd.6 [v2:172.21.15.155:6816/1730141492,v1:172.21.15.155:6817/1730141492]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T10:34:39.804 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:39 smithi155 ceph-mon[117223]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T10:34:39.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:39 smithi125 ceph-mon[112589]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:39.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:39 smithi125 ceph-mon[112589]: from='osd.6 [v2:172.21.15.155:6816/1730141492,v1:172.21.15.155:6817/1730141492]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T10:34:39.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:39 smithi125 ceph-mon[112589]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T10:34:39.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:39 smithi125 ceph-mon[127026]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:39.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:39 smithi125 ceph-mon[127026]: from='osd.6 [v2:172.21.15.155:6816/1730141492,v1:172.21.15.155:6817/1730141492]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T10:34:39.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:39 smithi125 ceph-mon[127026]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T10:34:40.186 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi155:vg_nvme/lv_1 2023-10-09T10:34:40.451 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:34:40.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[112589]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T10:34:40.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[112589]: osdmap e42: 7 total, 6 up, 7 in 2023-10-09T10:34:40.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:34:40.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[112589]: from='osd.6 [v2:172.21.15.155:6816/1730141492,v1:172.21.15.155:6817/1730141492]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:40.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[112589]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:40.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:40.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:40.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:40.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:40.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:40.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:40.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:40.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:40.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:40.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[127026]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T10:34:40.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[127026]: osdmap e42: 7 total, 6 up, 7 in 2023-10-09T10:34:40.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:34:40.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[127026]: from='osd.6 [v2:172.21.15.155:6816/1730141492,v1:172.21.15.155:6817/1730141492]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:40.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[127026]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:40.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:40.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:40.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:40.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:40.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:40.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:40.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:40.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:40.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:40 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:40.873 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T10:34:40.536+0000 7fecc48ad700 -1 osd.6 0 waiting for initial osdmap 2023-10-09T10:34:40.873 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T10:34:40.542+0000 7fecbfea9700 -1 osd.6 43 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:34:40.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-mon[117223]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T10:34:40.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-mon[117223]: osdmap e42: 7 total, 6 up, 7 in 2023-10-09T10:34:40.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:34:40.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-mon[117223]: from='osd.6 [v2:172.21.15.155:6816/1730141492,v1:172.21.15.155:6817/1730141492]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:40.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-mon[117223]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:40.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:40.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:40.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:40.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:40.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:40.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:40.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:40.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:40.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:40 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:41.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[112589]: Detected new or changed devices on smithi155 2023-10-09T10:34:41.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[112589]: Adjusting osd_memory_target on smithi155 to 5704M 2023-10-09T10:34:41.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[112589]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:41.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[112589]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T10:34:41.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[112589]: osdmap e43: 7 total, 6 up, 7 in 2023-10-09T10:34:41.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:34:41.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:34:41.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[112589]: osd.6 [v2:172.21.15.155:6816/1730141492,v1:172.21.15.155:6817/1730141492] boot 2023-10-09T10:34:41.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[112589]: osdmap e44: 7 total, 7 up, 7 in 2023-10-09T10:34:41.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:34:41.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[127026]: Detected new or changed devices on smithi155 2023-10-09T10:34:41.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[127026]: Adjusting osd_memory_target on smithi155 to 5704M 2023-10-09T10:34:41.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[127026]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:41.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[127026]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T10:34:41.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[127026]: osdmap e43: 7 total, 6 up, 7 in 2023-10-09T10:34:41.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:34:41.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:34:41.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[127026]: osd.6 [v2:172.21.15.155:6816/1730141492,v1:172.21.15.155:6817/1730141492] boot 2023-10-09T10:34:41.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[127026]: osdmap e44: 7 total, 7 up, 7 in 2023-10-09T10:34:41.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:41 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:34:41.953 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:41 smithi155 ceph-mon[117223]: Detected new or changed devices on smithi155 2023-10-09T10:34:41.953 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:41 smithi155 ceph-mon[117223]: Adjusting osd_memory_target on smithi155 to 5704M 2023-10-09T10:34:41.953 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:41 smithi155 ceph-mon[117223]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T10:34:41.953 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:41 smithi155 ceph-mon[117223]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T10:34:41.953 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:41 smithi155 ceph-mon[117223]: osdmap e43: 7 total, 6 up, 7 in 2023-10-09T10:34:41.953 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:41 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:34:41.953 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:41 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:34:41.953 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:41 smithi155 ceph-mon[117223]: osd.6 [v2:172.21.15.155:6816/1730141492,v1:172.21.15.155:6817/1730141492] boot 2023-10-09T10:34:41.954 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:41 smithi155 ceph-mon[117223]: osdmap e44: 7 total, 7 up, 7 in 2023-10-09T10:34:41.954 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:41 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:34:42.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:42 smithi125 ceph-mon[112589]: purged_snaps scrub starts 2023-10-09T10:34:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:42 smithi125 ceph-mon[112589]: purged_snaps scrub ok 2023-10-09T10:34:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:42 smithi125 ceph-mon[112589]: from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:34:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:42 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:34:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:42 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:34:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:42 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:42 smithi125 ceph-mon[112589]: osdmap e45: 7 total, 7 up, 7 in 2023-10-09T10:34:42.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:42 smithi125 ceph-mon[127026]: purged_snaps scrub starts 2023-10-09T10:34:42.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:42 smithi125 ceph-mon[127026]: purged_snaps scrub ok 2023-10-09T10:34:42.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:42 smithi125 ceph-mon[127026]: from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:34:42.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:42 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:34:42.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:42 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:34:42.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:42 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:42.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:42 smithi125 ceph-mon[127026]: osdmap e45: 7 total, 7 up, 7 in 2023-10-09T10:34:42.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:42 smithi155 ceph-mon[117223]: purged_snaps scrub starts 2023-10-09T10:34:42.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:42 smithi155 ceph-mon[117223]: purged_snaps scrub ok 2023-10-09T10:34:42.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:42 smithi155 ceph-mon[117223]: from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:34:42.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:42 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T10:34:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:42 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T10:34:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:42 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:42 smithi155 ceph-mon[117223]: osdmap e45: 7 total, 7 up, 7 in 2023-10-09T10:34:43.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:43 smithi125 ceph-mon[112589]: pgmap v117: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:34:43.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:43 smithi125 ceph-mon[112589]: from='client.? 172.21.15.155:0/3223741' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8eccbf5-4ad0-43e7-be9a-00c05a362e57"}]: dispatch 2023-10-09T10:34:43.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:43 smithi125 ceph-mon[112589]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8eccbf5-4ad0-43e7-be9a-00c05a362e57"}]: dispatch 2023-10-09T10:34:43.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:43 smithi125 ceph-mon[112589]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f8eccbf5-4ad0-43e7-be9a-00c05a362e57"}]': finished 2023-10-09T10:34:43.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:43 smithi125 ceph-mon[112589]: osdmap e46: 8 total, 7 up, 8 in 2023-10-09T10:34:43.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:43 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:43.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:43 smithi125 ceph-mon[127026]: pgmap v117: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:34:43.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:43 smithi125 ceph-mon[127026]: from='client.? 172.21.15.155:0/3223741' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8eccbf5-4ad0-43e7-be9a-00c05a362e57"}]: dispatch 2023-10-09T10:34:43.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:43 smithi125 ceph-mon[127026]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8eccbf5-4ad0-43e7-be9a-00c05a362e57"}]: dispatch 2023-10-09T10:34:43.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:43 smithi125 ceph-mon[127026]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f8eccbf5-4ad0-43e7-be9a-00c05a362e57"}]': finished 2023-10-09T10:34:43.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:43 smithi125 ceph-mon[127026]: osdmap e46: 8 total, 7 up, 8 in 2023-10-09T10:34:43.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:43 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:43 smithi155 ceph-mon[117223]: pgmap v117: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:34:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:43 smithi155 ceph-mon[117223]: from='client.? 172.21.15.155:0/3223741' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8eccbf5-4ad0-43e7-be9a-00c05a362e57"}]: dispatch 2023-10-09T10:34:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:43 smithi155 ceph-mon[117223]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8eccbf5-4ad0-43e7-be9a-00c05a362e57"}]: dispatch 2023-10-09T10:34:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:43 smithi155 ceph-mon[117223]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f8eccbf5-4ad0-43e7-be9a-00c05a362e57"}]': finished 2023-10-09T10:34:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:43 smithi155 ceph-mon[117223]: osdmap e46: 8 total, 7 up, 8 in 2023-10-09T10:34:43.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:43 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:44.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:44 smithi125 ceph-mon[112589]: from='client.? 172.21.15.155:0/1183231433' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:34:44.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:44 smithi125 ceph-mon[127026]: from='client.? 172.21.15.155:0/1183231433' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:34:44.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:44 smithi155 ceph-mon[117223]: from='client.? 172.21.15.155:0/1183231433' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T10:34:45.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:45 smithi125 ceph-mon[112589]: pgmap v120: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:34:45.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:45 smithi125 ceph-mon[112589]: osdmap e47: 8 total, 7 up, 8 in 2023-10-09T10:34:45.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:45 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:45.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:45 smithi125 ceph-mon[127026]: pgmap v120: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:34:45.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:45 smithi125 ceph-mon[127026]: osdmap e47: 8 total, 7 up, 8 in 2023-10-09T10:34:45.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:45 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:45.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:45 smithi155 ceph-mon[117223]: pgmap v120: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:34:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:45 smithi155 ceph-mon[117223]: osdmap e47: 8 total, 7 up, 8 in 2023-10-09T10:34:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:45 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:46.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:46 smithi125 ceph-mon[112589]: osdmap e48: 8 total, 7 up, 8 in 2023-10-09T10:34:46.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:46 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:46.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:46 smithi125 ceph-mon[127026]: osdmap e48: 8 total, 7 up, 8 in 2023-10-09T10:34:46.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:46 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:46.953 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:46 smithi155 ceph-mon[117223]: osdmap e48: 8 total, 7 up, 8 in 2023-10-09T10:34:46.953 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:46 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:47.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:47 smithi125 ceph-mon[112589]: pgmap v123: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:34:47.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:47 smithi125 ceph-mon[127026]: pgmap v123: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:34:47.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:47 smithi155 ceph-mon[117223]: pgmap v123: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:34:48.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:48 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T10:34:48.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:48 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:49.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:48 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T10:34:49.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:48 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:49.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:48 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T10:34:49.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:48 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:49.846 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:49 smithi155 ceph-mon[117223]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 104 KiB/s, 0 objects/s recovering 2023-10-09T10:34:49.847 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:49 smithi155 ceph-mon[117223]: Deploying daemon osd.7 on smithi155 2023-10-09T10:34:50.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:49 smithi125 ceph-mon[112589]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 104 KiB/s, 0 objects/s recovering 2023-10-09T10:34:50.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:49 smithi125 ceph-mon[112589]: Deploying daemon osd.7 on smithi155 2023-10-09T10:34:50.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:49 smithi125 ceph-mon[127026]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 104 KiB/s, 0 objects/s recovering 2023-10-09T10:34:50.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:49 smithi125 ceph-mon[127026]: Deploying daemon osd.7 on smithi155 2023-10-09T10:34:51.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:51 smithi155 ceph-mon[117223]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2023-10-09T10:34:52.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:51 smithi125 ceph-mon[112589]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2023-10-09T10:34:52.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:51 smithi125 ceph-mon[127026]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2023-10-09T10:34:53.123 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:52 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:53.123 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:52 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:53.123 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:52 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:53.123 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:52 smithi155 ceph-mon[117223]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T10:34:53.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:52 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:53.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:52 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:53.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:52 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:53.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:52 smithi125 ceph-mon[127026]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T10:34:53.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:52 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:53.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:52 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:53.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:52 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:53.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:52 smithi125 ceph-mon[112589]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T10:34:53.985 INFO:teuthology.orchestra.run.smithi155.stdout:Created osd(s) 7 on host 'smithi155' 2023-10-09T10:34:54.370 DEBUG:teuthology.orchestra.run.smithi155:osd.7> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-09T10:34:54.372 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-10-09T10:34:54.372 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T10:34:54.406 INFO:journalctl@ceph.osd.7.smithi155.stdout:-- Logs begin at Mon 2023-10-09 10:18:52 UTC. -- 2023-10-09T10:34:54.647 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:34:54.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:54 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:54.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:54 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:54.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:54 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:54.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:54 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:54.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:54 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:54.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:54 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:54.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:54 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:54.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:54 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:54.799 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:54.799 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:54.799 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:54.799 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:54.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:54.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:54.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:54.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:54.800 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:54.800 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:54.800 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:54.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:54.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:54.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:54.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:54.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:54 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:34:55.457 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 10:34:55 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T10:34:55.096+0000 7f4fe8324540 -1 osd.7 0 log_to_monitors true 2023-10-09T10:34:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:55 smithi155 ceph-mon[117223]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-10-09T10:34:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:55 smithi155 ceph-mon[117223]: from='osd.7 [v2:172.21.15.155:6824/3094552858,v1:172.21.15.155:6825/3094552858]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T10:34:55.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:55 smithi155 ceph-mon[117223]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T10:34:55.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:55 smithi125 ceph-mon[127026]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-10-09T10:34:55.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:55 smithi125 ceph-mon[127026]: from='osd.7 [v2:172.21.15.155:6824/3094552858,v1:172.21.15.155:6825/3094552858]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T10:34:55.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:55 smithi125 ceph-mon[127026]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T10:34:55.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:55 smithi125 ceph-mon[112589]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-10-09T10:34:55.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:55 smithi125 ceph-mon[112589]: from='osd.7 [v2:172.21.15.155:6824/3094552858,v1:172.21.15.155:6825/3094552858]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T10:34:55.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:55 smithi125 ceph-mon[112589]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T10:34:55.897 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:34:56.302 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":48,"num_osds":8,"num_up_osds":7,"osd_up_since":1696847681,"num_in_osds":8,"osd_in_since":1696847683,"num_remapped_pgs":0} 2023-10-09T10:34:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2405254373' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T10:34:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T10:34:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: osdmap e49: 8 total, 7 up, 8 in 2023-10-09T10:34:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: from='osd.7 [v2:172.21.15.155:6824/3094552858,v1:172.21.15.155:6825/3094552858]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:56.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:56.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:56.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:56.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:56.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:56.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:56.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2405254373' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T10:34:56.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T10:34:56.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: osdmap e49: 8 total, 7 up, 8 in 2023-10-09T10:34:56.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:56.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: from='osd.7 [v2:172.21.15.155:6824/3094552858,v1:172.21.15.155:6825/3094552858]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:56.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:56.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:56.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:56.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:56.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:56.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:56.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:56.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:56.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:56.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:56.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:56 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2405254373' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T10:34:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T10:34:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: osdmap e49: 8 total, 7 up, 8 in 2023-10-09T10:34:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: from='osd.7 [v2:172.21.15.155:6824/3094552858,v1:172.21.15.155:6825/3094552858]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T10:34:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:34:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:56.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:34:56.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:34:56.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:34:57.220 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T10:34:56.956+0000 7f4fe030c700 -1 osd.7 0 waiting for initial osdmap 2023-10-09T10:34:57.221 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 10:34:56 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T10:34:56.964+0000 7f4fd9103700 -1 osd.7 50 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:34:57.303 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T10:34:57.576 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:34:57.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:57 smithi125 ceph-mon[112589]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T10:34:57.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:57 smithi125 ceph-mon[112589]: Detected new or changed devices on smithi155 2023-10-09T10:34:57.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:57 smithi125 ceph-mon[112589]: Adjusting osd_memory_target on smithi155 to 4278M 2023-10-09T10:34:57.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:57 smithi125 ceph-mon[112589]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T10:34:57.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:57 smithi125 ceph-mon[112589]: osdmap e50: 8 total, 7 up, 8 in 2023-10-09T10:34:57.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:57 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:57.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:57 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:57.668 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:57 smithi125 ceph-mon[127026]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T10:34:57.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:57 smithi125 ceph-mon[127026]: Detected new or changed devices on smithi155 2023-10-09T10:34:57.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:57 smithi125 ceph-mon[127026]: Adjusting osd_memory_target on smithi155 to 4278M 2023-10-09T10:34:57.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:57 smithi125 ceph-mon[127026]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T10:34:57.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:57 smithi125 ceph-mon[127026]: osdmap e50: 8 total, 7 up, 8 in 2023-10-09T10:34:57.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:57 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:57.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:57 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:57.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:57 smithi155 ceph-mon[117223]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T10:34:57.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:57 smithi155 ceph-mon[117223]: Detected new or changed devices on smithi155 2023-10-09T10:34:57.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:57 smithi155 ceph-mon[117223]: Adjusting osd_memory_target on smithi155 to 4278M 2023-10-09T10:34:57.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:57 smithi155 ceph-mon[117223]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T10:34:57.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:57 smithi155 ceph-mon[117223]: osdmap e50: 8 total, 7 up, 8 in 2023-10-09T10:34:57.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:57 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:57.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:57 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:58.725 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:34:59.065 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[112589]: purged_snaps scrub starts 2023-10-09T10:34:59.065 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[112589]: purged_snaps scrub ok 2023-10-09T10:34:59.065 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[112589]: osd.7 [v2:172.21.15.155:6824/3094552858,v1:172.21.15.155:6825/3094552858] boot 2023-10-09T10:34:59.065 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[112589]: osdmap e51: 8 total, 8 up, 8 in 2023-10-09T10:34:59.065 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:59.065 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[112589]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:34:59.065 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[112589]: osdmap e52: 8 total, 8 up, 8 in 2023-10-09T10:34:59.066 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/510573835' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T10:34:59.066 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[127026]: purged_snaps scrub starts 2023-10-09T10:34:59.066 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[127026]: purged_snaps scrub ok 2023-10-09T10:34:59.066 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[127026]: osd.7 [v2:172.21.15.155:6824/3094552858,v1:172.21.15.155:6825/3094552858] boot 2023-10-09T10:34:59.066 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[127026]: osdmap e51: 8 total, 8 up, 8 in 2023-10-09T10:34:59.066 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:59.066 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[127026]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:34:59.066 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[127026]: osdmap e52: 8 total, 8 up, 8 in 2023-10-09T10:34:59.066 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:34:58 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/510573835' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T10:34:59.107 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":52,"num_osds":8,"num_up_osds":8,"osd_up_since":1696847697,"num_in_osds":8,"osd_in_since":1696847683,"num_remapped_pgs":1} 2023-10-09T10:34:59.107 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T10:34:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:58 smithi155 ceph-mon[117223]: purged_snaps scrub starts 2023-10-09T10:34:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:58 smithi155 ceph-mon[117223]: purged_snaps scrub ok 2023-10-09T10:34:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:58 smithi155 ceph-mon[117223]: osd.7 [v2:172.21.15.155:6824/3094552858,v1:172.21.15.155:6825/3094552858] boot 2023-10-09T10:34:59.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:58 smithi155 ceph-mon[117223]: osdmap e51: 8 total, 8 up, 8 in 2023-10-09T10:34:59.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:58 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:34:59.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:58 smithi155 ceph-mon[117223]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:34:59.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:58 smithi155 ceph-mon[117223]: osdmap e52: 8 total, 8 up, 8 in 2023-10-09T10:34:59.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:34:58 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/510573835' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T10:34:59.367 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:00.557 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:35:00.558 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":54,"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","created":"2023-10-09T10:31:07.767709+0000","modified":"2023-10-09T10:35:00.476280+0000","last_up_change":"2023-10-09T10:34:57.946717+0000","last_in_change":"2023-10-09T10:34:43.351594+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T10:33:34.342993+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"874cd96b-ddd8-4164-931e-06f2388dd41b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6803","nonce":3322074820}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6805","nonce":3322074820}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6809","nonce":3322074820}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6807","nonce":3322074820}]},"public_addr":"172.21.15.125:6803/3322074820","cluster_addr":"172.21.15.125:6805/3322074820","heartbeat_back_addr":"172.21.15.125:6809/3322074820","heartbeat_front_addr":"172.21.15.125:6807/3322074820","state":["exists","up"]},{"osd":1,"uuid":"0e517dd5-6d0b-4841-bb81-1eb04377ba5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6811","nonce":3260318814}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6813","nonce":3260318814}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6817","nonce":3260318814}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6815","nonce":3260318814}]},"public_addr":"172.21.15.125:6811/3260318814","cluster_addr":"172.21.15.125:6813/3260318814","heartbeat_back_addr":"172.21.15.125:6817/3260318814","heartbeat_front_addr":"172.21.15.125:6815/3260318814","state":["exists","up"]},{"osd":2,"uuid":"643c1f3c-ecd9-464a-ad53-bf35a4deee5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6819","nonce":4010155657}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6821","nonce":4010155657}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6825","nonce":4010155657}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6823","nonce":4010155657}]},"public_addr":"172.21.15.125:6819/4010155657","cluster_addr":"172.21.15.125:6821/4010155657","heartbeat_back_addr":"172.21.15.125:6825/4010155657","heartbeat_front_addr":"172.21.15.125:6823/4010155657","state":["exists","up"]},{"osd":3,"uuid":"3015cd6e-b270-44c3-8e6e-6bcd979bcd21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6827","nonce":2950193979}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6829","nonce":2950193979}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6833","nonce":2950193979}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6831","nonce":2950193979}]},"public_addr":"172.21.15.125:6827/2950193979","cluster_addr":"172.21.15.125:6829/2950193979","heartbeat_back_addr":"172.21.15.125:6833/2950193979","heartbeat_front_addr":"172.21.15.125:6831/2950193979","state":["exists","up"]},{"osd":4,"uuid":"272c7b95-f89c-4fe1-a299-a861e0ec9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6801","nonce":1086065767}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6803","nonce":1086065767}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6807","nonce":1086065767}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6805","nonce":1086065767}]},"public_addr":"172.21.15.155:6801/1086065767","cluster_addr":"172.21.15.155:6803/1086065767","heartbeat_back_addr":"172.21.15.155:6807/1086065767","heartbeat_front_addr":"172.21.15.155:6805/1086065767","state":["exists","up"]},{"osd":5,"uuid":"201d51f2-233c-40fc-a42d-649d225954fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6809","nonce":104536199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6811","nonce":104536199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6815","nonce":104536199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6813","nonce":104536199}]},"public_addr":"172.21.15.155:6809/104536199","cluster_addr":"172.21.15.155:6811/104536199","heartbeat_back_addr":"172.21.15.155:6815/104536199","heartbeat_front_addr":"172.21.15.155:6813/104536199","state":["exists","up"]},{"osd":6,"uuid":"4ec66642-108e-401f-aff3-e7ab258bc083","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":47,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6817","nonce":1730141492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6819","nonce":1730141492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6823","nonce":1730141492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6821","nonce":1730141492}]},"public_addr":"172.21.15.155:6817/1730141492","cluster_addr":"172.21.15.155:6819/1730141492","heartbeat_back_addr":"172.21.15.155:6823/1730141492","heartbeat_front_addr":"172.21.15.155:6821/1730141492","state":["exists","up"]},{"osd":7,"uuid":"f8eccbf5-4ad0-43e7-be9a-00c05a362e57","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6825","nonce":3094552858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6827","nonce":3094552858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6831","nonce":3094552858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6829","nonce":3094552858}]},"public_addr":"172.21.15.155:6825/3094552858","cluster_addr":"172.21.15.155:6827/3094552858","heartbeat_back_addr":"172.21.15.155:6831/3094552858","heartbeat_front_addr":"172.21.15.155:6829/3094552858","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:32:58.904066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:15.883014+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:32.639145+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:49.937002+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:06.212675+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:22.755647+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:39.598906+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:56.143619+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:6801/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6800/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6800/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/1623791415":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/1101683240":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/725980307":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/1709045642":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/2611643565":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6801/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/178379618":"2023-10-10T10:31:33.872122+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T10:35:00.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:00 smithi125 ceph-mon[112589]: osdmap e53: 8 total, 8 up, 8 in 2023-10-09T10:35:00.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:00 smithi125 ceph-mon[127026]: osdmap e53: 8 total, 8 up, 8 in 2023-10-09T10:35:00.941 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-10-09T10:33:34.342993+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-10-09T10:35:00.941 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-10-09T10:35:00.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:00 smithi155 ceph-mon[117223]: osdmap e53: 8 total, 8 up, 8 in 2023-10-09T10:35:01.201 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:01.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:01 smithi125 ceph-mon[112589]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:01.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:01 smithi125 ceph-mon[112589]: osdmap e54: 8 total, 8 up, 8 in 2023-10-09T10:35:01.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:01 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2968463201' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:35:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:01 smithi125 ceph-mon[127026]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:01 smithi125 ceph-mon[127026]: osdmap e54: 8 total, 8 up, 8 in 2023-10-09T10:35:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:01 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2968463201' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:35:01.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:01 smithi155 ceph-mon[117223]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:01.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:01 smithi155 ceph-mon[117223]: osdmap e54: 8 total, 8 up, 8 in 2023-10-09T10:35:01.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:01 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2968463201' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:35:02.362 INFO:teuthology.orchestra.run.smithi125.stdout:pg_num: 1 2023-10-09T10:35:02.731 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:02 smithi125 ceph-mon[112589]: osdmap e55: 8 total, 8 up, 8 in 2023-10-09T10:35:02.731 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:02 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2765642235' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T10:35:02.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:02 smithi125 ceph-mon[127026]: osdmap e55: 8 total, 8 up, 8 in 2023-10-09T10:35:02.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:02 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2765642235' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T10:35:02.750 INFO:tasks.cephadm:Adding prometheus.a on smithi155 2023-10-09T10:35:02.750 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch apply prometheus '1;smithi155=a' 2023-10-09T10:35:02.784 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:02 smithi155 ceph-mon[117223]: osdmap e55: 8 total, 8 up, 8 in 2023-10-09T10:35:02.784 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:02 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2765642235' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T10:35:03.025 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:35:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:03 smithi155 ceph-mon[117223]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:03.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:03 smithi125 ceph-mon[112589]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:03 smithi125 ceph-mon[127026]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:04.210 INFO:teuthology.orchestra.run.smithi155.stdout:Scheduled prometheus update... 2023-10-09T10:35:04.590 DEBUG:teuthology.orchestra.run.smithi155:prometheus.a> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@prometheus.a.service 2023-10-09T10:35:04.632 INFO:tasks.cephadm:Adding node-exporter.a on smithi125 2023-10-09T10:35:04.632 INFO:tasks.cephadm:Adding node-exporter.b on smithi155 2023-10-09T10:35:04.632 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi125=a;smithi155=b' 2023-10-09T10:35:04.663 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:-- Logs begin at Mon 2023-10-09 10:18:52 UTC. -- 2023-10-09T10:35:04.921 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:35:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:05 smithi155 ceph-mon[117223]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-10-09T10:35:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:05 smithi155 ceph-mon[117223]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi155=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:35:05.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:05 smithi155 ceph-mon[117223]: Saving service prometheus spec with placement smithi155=a;count:1 2023-10-09T10:35:05.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:05 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:05.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:05 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:35:05.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:05 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:35:05.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:05 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:35:05.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:05 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:05.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:05 smithi155 ceph-mon[117223]: Deploying daemon prometheus.a on smithi155 2023-10-09T10:35:05.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[112589]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-10-09T10:35:05.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[112589]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi155=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:35:05.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[112589]: Saving service prometheus spec with placement smithi155=a;count:1 2023-10-09T10:35:05.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:05.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:35:05.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:35:05.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:35:05.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:05.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[112589]: Deploying daemon prometheus.a on smithi155 2023-10-09T10:35:05.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[127026]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-10-09T10:35:05.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[127026]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi155=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:35:05.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[127026]: Saving service prometheus spec with placement smithi155=a;count:1 2023-10-09T10:35:05.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:05.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:35:05.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:35:05.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:35:05.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:05.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:05 smithi125 ceph-mon[127026]: Deploying daemon prometheus.a on smithi155 2023-10-09T10:35:06.824 INFO:teuthology.orchestra.run.smithi155.stdout:Scheduled node-exporter update... 2023-10-09T10:35:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:07 smithi155 ceph-mon[117223]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2023-10-09T10:35:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:07 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:07.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:07 smithi125 ceph-mon[112589]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2023-10-09T10:35:07.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:07 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:07.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:07 smithi125 ceph-mon[127026]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2023-10-09T10:35:07.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:07 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:08 smithi155 ceph-mon[117223]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi125=a;smithi155=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:35:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:08 smithi155 ceph-mon[117223]: Saving service node-exporter spec with placement smithi125=a;smithi155=b;count:2 2023-10-09T10:35:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:08 smithi125 ceph-mon[112589]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi125=a;smithi155=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:35:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:08 smithi125 ceph-mon[112589]: Saving service node-exporter spec with placement smithi125=a;smithi155=b;count:2 2023-10-09T10:35:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:08 smithi125 ceph-mon[127026]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi125=a;smithi155=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:35:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:08 smithi125 ceph-mon[127026]: Saving service node-exporter spec with placement smithi125=a;smithi155=b;count:2 2023-10-09T10:35:08.648 DEBUG:teuthology.orchestra.run.smithi125:node-exporter.a> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@node-exporter.a.service 2023-10-09T10:35:08.651 DEBUG:teuthology.orchestra.run.smithi155:node-exporter.b> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@node-exporter.b.service 2023-10-09T10:35:08.653 INFO:tasks.cephadm:Adding alertmanager.a on smithi125 2023-10-09T10:35:08.653 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi125=a' 2023-10-09T10:35:08.683 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:-- Logs begin at Mon 2023-10-09 10:18:52 UTC. -- 2023-10-09T10:35:08.688 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:-- Logs begin at Mon 2023-10-09 10:18:49 UTC. -- 2023-10-09T10:35:09.393 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:35:09.785 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:09 smithi155 ceph-mon[117223]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T10:35:09.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:09 smithi125 ceph-mon[112589]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T10:35:09.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:09 smithi125 ceph-mon[127026]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T10:35:11.141 INFO:teuthology.orchestra.run.smithi155.stdout:Scheduled alertmanager update... 2023-10-09T10:35:11.532 DEBUG:teuthology.orchestra.run.smithi125:alertmanager.a> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@alertmanager.a.service 2023-10-09T10:35:11.535 INFO:tasks.cephadm:Adding grafana.a on smithi155 2023-10-09T10:35:11.535 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph orch apply grafana '1;smithi155=a' 2023-10-09T10:35:11.568 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:-- Logs begin at Mon 2023-10-09 10:18:49 UTC. -- 2023-10-09T10:35:11.705 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:11 smithi155 ceph-mon[117223]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-10-09T10:35:11.705 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:11 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:11.816 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:35:11.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:11 smithi125 ceph-mon[112589]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-10-09T10:35:11.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:11 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:11.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:11 smithi125 ceph-mon[127026]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-10-09T10:35:11.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:11 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:12 smithi155 ceph-mon[117223]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi125=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:35:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:12 smithi155 ceph-mon[117223]: Saving service alertmanager spec with placement smithi125=a;count:1 2023-10-09T10:35:12.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:12 smithi125 ceph-mon[112589]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi125=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:35:12.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:12 smithi125 ceph-mon[112589]: Saving service alertmanager spec with placement smithi125=a;count:1 2023-10-09T10:35:12.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:12 smithi125 ceph-mon[127026]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi125=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:35:12.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:12 smithi125 ceph-mon[127026]: Saving service alertmanager spec with placement smithi125=a;count:1 2023-10-09T10:35:13.158 INFO:teuthology.orchestra.run.smithi155.stdout:Scheduled grafana update... 2023-10-09T10:35:13.181 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:12 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[150882]: ts=2023-10-09T10:35:12.812Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T10:35:13.539 DEBUG:teuthology.orchestra.run.smithi155:grafana.a> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@grafana.a.service 2023-10-09T10:35:13.541 INFO:tasks.cephadm:Setting up client nodes... 2023-10-09T10:35:13.541 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T10:35:13.572 INFO:journalctl@ceph.grafana.a.smithi155.stdout:-- Logs begin at Mon 2023-10-09 10:18:52 UTC. -- 2023-10-09T10:35:13.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:13 smithi155 ceph-mon[117223]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-10-09T10:35:13.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:13 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:13.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:13 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:13.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:13 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:13.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:13 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T10:35:13.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:13 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:13.813 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:13.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:13 smithi125 ceph-mon[127026]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-10-09T10:35:13.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:13 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:13.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:13 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:13.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:13 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:13.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:13 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T10:35:13.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:13 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:13.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:13 smithi125 ceph-mon[112589]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-10-09T10:35:13.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:13 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:13.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:13 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:13.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:13 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:13.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:13 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T10:35:13.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:13 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' 2023-10-09T10:35:14.216 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:13 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: ignoring --setuser ceph since I am not root 2023-10-09T10:35:14.216 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:13 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: ignoring --setgroup ceph since I am not root 2023-10-09T10:35:14.216 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:13 smithi155 ceph-mgr[119399]: -- 172.21.15.155:0/979959065 <== mon.2 v2:172.21.15.125:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5627617689c0 con 0x56276173e800 2023-10-09T10:35:14.216 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:14 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:14.053+0000 7f3ad22481c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T10:35:14.240 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:13 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ignoring --setuser ceph since I am not root 2023-10-09T10:35:14.240 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:13 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ignoring --setgroup ceph since I am not root 2023-10-09T10:35:14.240 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:13 smithi125 ceph-mgr[112861]: -- 172.21.15.125:0/1818552692 <== mon.0 v2:172.21.15.125:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559f402d29c0 con 0x559f402a8800 2023-10-09T10:35:14.240 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:14.059+0000 7f3ed48731c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T10:35:14.240 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:14.228+0000 7f3ed48731c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T10:35:14.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:14 smithi125 ceph-mon[112589]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi155=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:35:14.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:14 smithi125 ceph-mon[112589]: Saving service grafana spec with placement smithi155=a;count:1 2023-10-09T10:35:14.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:14 smithi125 ceph-mon[112589]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T10:35:14.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:14 smithi125 ceph-mon[112589]: mgrmap e16: y(active, since 3m), standbys: x 2023-10-09T10:35:14.821 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:14.538+0000 7f3ed48731c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T10:35:14.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:14 smithi125 ceph-mon[127026]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi155=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:35:14.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:14 smithi125 ceph-mon[127026]: Saving service grafana spec with placement smithi155=a;count:1 2023-10-09T10:35:14.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:14 smithi125 ceph-mon[127026]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T10:35:14.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:14 smithi125 ceph-mon[127026]: mgrmap e16: y(active, since 3m), standbys: x 2023-10-09T10:35:14.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:14 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:14.214+0000 7f3ad22481c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T10:35:14.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:14 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:14.510+0000 7f3ad22481c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T10:35:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:14 smithi155 ceph-mon[117223]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi155=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:35:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:14 smithi155 ceph-mon[117223]: Saving service grafana spec with placement smithi155=a;count:1 2023-10-09T10:35:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:14 smithi155 ceph-mon[117223]: from='mgr.14152 172.21.15.125:0/2538763433' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T10:35:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:14 smithi155 ceph-mon[117223]: mgrmap e16: y(active, since 3m), standbys: x 2023-10-09T10:35:15.108 INFO:teuthology.orchestra.run.smithi125.stdout:[client.0] 2023-10-09T10:35:15.108 INFO:teuthology.orchestra.run.smithi125.stdout: key = AQBj1yNlTL88BhAAamUsG3FOxe9qt/rd3gMUdg== 2023-10-09T10:35:15.462 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:15 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:15.075+0000 7f3ed48731c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T10:35:15.462 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:15 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:15.252+0000 7f3ed48731c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T10:35:15.470 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:15 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:15.021+0000 7f3ad22481c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T10:35:15.470 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:15 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:15.187+0000 7f3ad22481c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T10:35:15.517 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:35:15.517 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-09T10:35:15.517 DEBUG:teuthology.orchestra.run.smithi125:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-09T10:35:15.559 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T10:35:15.820 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:15 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:15.664+0000 7f3ed48731c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T10:35:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:15 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3710000054' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T10:35:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:15 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3710000054' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T10:35:15.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:15 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3710000054' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T10:35:15.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:15 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3710000054' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T10:35:15.832 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:35:15.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:15 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:15.585+0000 7f3ad22481c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T10:35:15.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:15 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3710000054' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T10:35:15.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:15 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3710000054' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T10:35:16.676 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:16 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:16.400+0000 7f3ad22481c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T10:35:16.676 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:16 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:16.488+0000 7f3ad22481c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T10:35:16.820 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:16 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:16.420+0000 7f3ed48731c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T10:35:16.820 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:16 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:16.504+0000 7f3ed48731c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T10:35:17.148 INFO:teuthology.orchestra.run.smithi155.stdout:[client.1] 2023-10-09T10:35:17.148 INFO:teuthology.orchestra.run.smithi155.stdout: key = AQBl1yNlx2CeCBAAmaXcO9vsMjsK29KHDJGVfg== 2023-10-09T10:35:17.366 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:17 smithi125 ceph-mon[112589]: from='client.? 172.21.15.155:0/1182109714' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T10:35:17.367 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:17 smithi125 ceph-mon[112589]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T10:35:17.367 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:17 smithi125 ceph-mon[112589]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T10:35:17.367 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:17 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:17.102+0000 7f3ed48731c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T10:35:17.367 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:17 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:17.186+0000 7f3ed48731c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T10:35:17.367 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:17 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:17.274+0000 7f3ed48731c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T10:35:17.367 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:17 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:17.365+0000 7f3ed48731c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T10:35:17.368 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:17 smithi125 ceph-mon[127026]: from='client.? 172.21.15.155:0/1182109714' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T10:35:17.368 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:17 smithi125 ceph-mon[127026]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T10:35:17.368 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:17 smithi125 ceph-mon[127026]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T10:35:17.411 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:17 smithi155 ceph-mon[117223]: from='client.? 172.21.15.155:0/1182109714' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T10:35:17.411 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:17 smithi155 ceph-mon[117223]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T10:35:17.412 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:17 smithi155 ceph-mon[117223]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T10:35:17.412 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:17 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:17.133+0000 7f3ad22481c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T10:35:17.412 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:17 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:17.223+0000 7f3ad22481c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T10:35:17.412 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:17 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:17.314+0000 7f3ad22481c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T10:35:17.561 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T10:35:17.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-10-09T10:35:17.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-10-09T10:35:17.606 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-10-09T10:35:17.606 DEBUG:teuthology.orchestra.run.smithi125:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-10-09T10:35:17.683 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:17 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:17.587+0000 7f3ed48731c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T10:35:17.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:17 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:17.410+0000 7f3ad22481c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T10:35:17.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:17 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:17.648+0000 7f3ad22481c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T10:35:17.933 INFO:teuthology.misc.health.smithi125.stdout: 2023-10-09T10:35:17.933 INFO:teuthology.misc.health.smithi125.stdout:{"epoch":55,"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","created":"2023-10-09T10:31:07.767709+0000","modified":"2023-10-09T10:35:01.483200+0000","last_up_change":"2023-10-09T10:34:57.946717+0000","last_in_change":"2023-10-09T10:34:43.351594+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T10:33:34.342993+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"874cd96b-ddd8-4164-931e-06f2388dd41b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6803","nonce":3322074820}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6805","nonce":3322074820}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6809","nonce":3322074820}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6807","nonce":3322074820}]},"public_addr":"172.21.15.125:6803/3322074820","cluster_addr":"172.21.15.125:6805/3322074820","heartbeat_back_addr":"172.21.15.125:6809/3322074820","heartbeat_front_addr":"172.21.15.125:6807/3322074820","state":["exists","up"]},{"osd":1,"uuid":"0e517dd5-6d0b-4841-bb81-1eb04377ba5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6811","nonce":3260318814}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6813","nonce":3260318814}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6817","nonce":3260318814}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6815","nonce":3260318814}]},"public_addr":"172.21.15.125:6811/3260318814","cluster_addr":"172.21.15.125:6813/3260318814","heartbeat_back_addr":"172.21.15.125:6817/3260318814","heartbeat_front_addr":"172.21.15.125:6815/3260318814","state":["exists","up"]},{"osd":2,"uuid":"643c1f3c-ecd9-464a-ad53-bf35a4deee5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6819","nonce":4010155657}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6821","nonce":4010155657}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6825","nonce":4010155657}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6823","nonce":4010155657}]},"public_addr":"172.21.15.125:6819/4010155657","cluster_addr":"172.21.15.125:6821/4010155657","heartbeat_back_addr":"172.21.15.125:6825/4010155657","heartbeat_front_addr":"172.21.15.125:6823/4010155657","state":["exists","up"]},{"osd":3,"uuid":"3015cd6e-b270-44c3-8e6e-6bcd979bcd21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6827","nonce":2950193979}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6829","nonce":2950193979}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6833","nonce":2950193979}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6831","nonce":2950193979}]},"public_addr":"172.21.15.125:6827/2950193979","cluster_addr":"172.21.15.125:6829/2950193979","heartbeat_back_addr":"172.21.15.125:6833/2950193979","heartbeat_front_addr":"172.21.15.125:6831/2950193979","state":["exists","up"]},{"osd":4,"uuid":"272c7b95-f89c-4fe1-a299-a861e0ec9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6801","nonce":1086065767}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6803","nonce":1086065767}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6807","nonce":1086065767}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6805","nonce":1086065767}]},"public_addr":"172.21.15.155:6801/1086065767","cluster_addr":"172.21.15.155:6803/1086065767","heartbeat_back_addr":"172.21.15.155:6807/1086065767","heartbeat_front_addr":"172.21.15.155:6805/1086065767","state":["exists","up"]},{"osd":5,"uuid":"201d51f2-233c-40fc-a42d-649d225954fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6809","nonce":104536199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6811","nonce":104536199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6815","nonce":104536199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6813","nonce":104536199}]},"public_addr":"172.21.15.155:6809/104536199","cluster_addr":"172.21.15.155:6811/104536199","heartbeat_back_addr":"172.21.15.155:6815/104536199","heartbeat_front_addr":"172.21.15.155:6813/104536199","state":["exists","up"]},{"osd":6,"uuid":"4ec66642-108e-401f-aff3-e7ab258bc083","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":47,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6817","nonce":1730141492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6819","nonce":1730141492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6823","nonce":1730141492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6821","nonce":1730141492}]},"public_addr":"172.21.15.155:6817/1730141492","cluster_addr":"172.21.15.155:6819/1730141492","heartbeat_back_addr":"172.21.15.155:6823/1730141492","heartbeat_front_addr":"172.21.15.155:6821/1730141492","state":["exists","up"]},{"osd":7,"uuid":"f8eccbf5-4ad0-43e7-be9a-00c05a362e57","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6825","nonce":3094552858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6827","nonce":3094552858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6831","nonce":3094552858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6829","nonce":3094552858}]},"public_addr":"172.21.15.155:6825/3094552858","cluster_addr":"172.21.15.155:6827/3094552858","heartbeat_back_addr":"172.21.15.155:6831/3094552858","heartbeat_front_addr":"172.21.15.155:6829/3094552858","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:32:58.904066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:15.883014+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:32.639145+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:49.937002+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:06.212675+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:22.755647+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:39.598906+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:56.143619+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:6801/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6800/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6800/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/1623791415":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/1101683240":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/725980307":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/1709045642":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/2611643565":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6801/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/178379618":"2023-10-10T10:31:33.872122+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T10:35:17.947 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-10-09T10:35:17.947 INFO:tasks.cephadm:Creating RBD pool 2023-10-09T10:35:17.947 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-10-09T10:35:17.972 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:17 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:17.682+0000 7f3ed48731c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T10:35:17.972 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:17 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:17.903+0000 7f3ed48731c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T10:35:18.046 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:17 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:17.740+0000 7f3ad22481c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T10:35:18.046 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:17 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:17.951+0000 7f3ad22481c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T10:35:18.213 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:18.300 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:18 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3446621921' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:35:18.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:18 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3446621921' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:35:18.301 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:18 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:18.007+0000 7f3ed48731c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T10:35:18.301 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:18 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:18.207+0000 7f3ed48731c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T10:35:18.313 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:18 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3446621921' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:35:18.314 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:18 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:18.045+0000 7f3ad22481c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T10:35:18.314 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:18 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:18.226+0000 7f3ad22481c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T10:35:18.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:18 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:18.298+0000 7f3ed48731c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T10:35:18.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:18 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:18.310+0000 7f3ad22481c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T10:35:19.415 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:19 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:19.049+0000 7f3ad22481c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T10:35:19.415 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:19 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:19.238+0000 7f3ad22481c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T10:35:19.469 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:19 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:19.080+0000 7f3ed48731c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T10:35:19.470 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:19 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:19.283+0000 7f3ed48731c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T10:35:19.470 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:19 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:19.468+0000 7f3ed48731c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:35:19.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:19 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:19.414+0000 7f3ad22481c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:35:19.867 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'rbd' created 2023-10-09T10:35:20.125 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:19 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3874921892' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T10:35:20.125 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:20.088+0000 7f3ed48731c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T10:35:20.125 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:19 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3874921892' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T10:35:20.195 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3874921892' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T10:35:20.195 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:20.013+0000 7f3ad22481c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T10:35:20.313 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-10-09T10:35:20.380 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:20.274+0000 7f3ed48731c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:35:20.470 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:20.194+0000 7f3ad22481c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:35:20.470 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:20.374+0000 7f3ad22481c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T10:35:20.630 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:20.685 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:20.467+0000 7f3ed48731c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T10:35:20.685 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:20.592+0000 7f3ed48731c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T10:35:20.685 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:20] ENGINE Bus STARTING 2023-10-09T10:35:20.799 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:20.490+0000 7f3ad22481c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T10:35:20.799 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: [09/Oct/2023:10:35:20] ENGINE Bus STARTING 2023-10-09T10:35:20.799 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: CherryPy Checker: 2023-10-09T10:35:20.799 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: The Application mounted at '' has an empty config. 2023-10-09T10:35:20.799 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: 2023-10-09T10:35:20.799 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: [09/Oct/2023:10:35:20] ENGINE Serving on http://:::9283 2023-10-09T10:35:20.800 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: [09/Oct/2023:10:35:20] ENGINE Bus STARTED 2023-10-09T10:35:21.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: CherryPy Checker: 2023-10-09T10:35:21.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: The Application mounted at '' has an empty config. 2023-10-09T10:35:21.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:35:21.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:20] ENGINE Serving on http://:::9283 2023-10-09T10:35:21.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:20] ENGINE Bus STARTED 2023-10-09T10:35:21.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3874921892' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T10:35:21.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: osdmap e56: 8 total, 8 up, 8 in 2023-10-09T10:35:21.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: Standby manager daemon x restarted 2023-10-09T10:35:21.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: Standby manager daemon x started 2023-10-09T10:35:21.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/3250509796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:35:21.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/3250509796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:35:21.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/3250509796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:35:21.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/3250509796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:35:21.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: Active manager daemon y restarted 2023-10-09T10:35:21.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: Activating manager daemon y 2023-10-09T10:35:21.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: osdmap e57: 8 total, 8 up, 8 in 2023-10-09T10:35:21.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: mgrmap e17: y(active, starting, since 0.00772326s), standbys: x 2023-10-09T10:35:21.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:35:21.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:35:21.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:35:21.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:35:21.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T10:35:21.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:35:21.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:35:21.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:35:21.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:35:21.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:35:21.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:35:21.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:35:21.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:35:21.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:35:21.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:35:21.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:35:21.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: Manager daemon y is now available 2023-10-09T10:35:21.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3874921892' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T10:35:21.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: osdmap e56: 8 total, 8 up, 8 in 2023-10-09T10:35:21.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: Standby manager daemon x restarted 2023-10-09T10:35:21.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: Standby manager daemon x started 2023-10-09T10:35:21.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/3250509796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:35:21.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/3250509796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:35:21.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/3250509796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:35:21.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/3250509796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:35:21.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: Active manager daemon y restarted 2023-10-09T10:35:21.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: Activating manager daemon y 2023-10-09T10:35:21.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: osdmap e57: 8 total, 8 up, 8 in 2023-10-09T10:35:21.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: mgrmap e17: y(active, starting, since 0.00772326s), standbys: x 2023-10-09T10:35:21.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:35:21.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:35:21.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:35:21.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:35:21.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T10:35:21.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:35:21.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:35:21.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:35:21.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:35:21.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:35:21.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:35:21.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:35:21.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:35:21.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:35:21.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:35:21.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:35:21.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: Manager daemon y is now available 2023-10-09T10:35:21.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:35:21.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:35:21.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:21.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:35:21.078 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:35:21.078 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T10:35:21.078 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:35:21.078 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:35:21.078 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:21.078 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:35:21.078 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:35:21.078 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T10:35:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3874921892' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T10:35:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: osdmap e56: 8 total, 8 up, 8 in 2023-10-09T10:35:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: Standby manager daemon x restarted 2023-10-09T10:35:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: Standby manager daemon x started 2023-10-09T10:35:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/3250509796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:35:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/3250509796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:35:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/3250509796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:35:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/3250509796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:35:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: Active manager daemon y restarted 2023-10-09T10:35:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: Activating manager daemon y 2023-10-09T10:35:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: osdmap e57: 8 total, 8 up, 8 in 2023-10-09T10:35:21.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: mgrmap e17: y(active, starting, since 0.00772326s), standbys: x 2023-10-09T10:35:21.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:35:21.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:35:21.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:35:21.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:35:21.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T10:35:21.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:35:21.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:35:21.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:35:21.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:35:21.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:35:21.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:35:21.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:35:21.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:35:21.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:35:21.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:35:21.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:35:21.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: Manager daemon y is now available 2023-10-09T10:35:21.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:35:21.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:35:21.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:21.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:35:21.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:35:21.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T10:35:22.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:21 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:35:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T10:35:22.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[112589]: [09/Oct/2023:10:35:21] ENGINE Bus STARTING 2023-10-09T10:35:22.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[112589]: [09/Oct/2023:10:35:21] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T10:35:22.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[112589]: [09/Oct/2023:10:35:21] ENGINE Bus STARTED 2023-10-09T10:35:22.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:22.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[112589]: osdmap e58: 8 total, 8 up, 8 in 2023-10-09T10:35:22.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[112589]: mgrmap e18: y(active, since 1.01323s), standbys: x 2023-10-09T10:35:22.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[112589]: pgmap v4: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:22.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/398555181' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:35:22.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[112589]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:35:22.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:22.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:22.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:22.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:22.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[127026]: [09/Oct/2023:10:35:21] ENGINE Bus STARTING 2023-10-09T10:35:22.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[127026]: [09/Oct/2023:10:35:21] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T10:35:22.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[127026]: [09/Oct/2023:10:35:21] ENGINE Bus STARTED 2023-10-09T10:35:22.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:22.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[127026]: osdmap e58: 8 total, 8 up, 8 in 2023-10-09T10:35:22.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[127026]: mgrmap e18: y(active, since 1.01323s), standbys: x 2023-10-09T10:35:22.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[127026]: pgmap v4: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:22.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/398555181' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:35:22.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[127026]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:35:22.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:22.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:22.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:22.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:22 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:22.633 INFO:teuthology.orchestra.run.smithi125.stderr:enabled application 'rbd' on pool 'rbd' 2023-10-09T10:35:22.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:22 smithi155 ceph-mon[117223]: [09/Oct/2023:10:35:21] ENGINE Bus STARTING 2023-10-09T10:35:22.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:22 smithi155 ceph-mon[117223]: [09/Oct/2023:10:35:21] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T10:35:22.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:22 smithi155 ceph-mon[117223]: [09/Oct/2023:10:35:21] ENGINE Bus STARTED 2023-10-09T10:35:22.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:22 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:22.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:22 smithi155 ceph-mon[117223]: osdmap e58: 8 total, 8 up, 8 in 2023-10-09T10:35:22.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:22 smithi155 ceph-mon[117223]: mgrmap e18: y(active, since 1.01323s), standbys: x 2023-10-09T10:35:22.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:22 smithi155 ceph-mon[117223]: pgmap v4: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:22.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:22 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/398555181' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:35:22.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:22 smithi155 ceph-mon[117223]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:35:22.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:22 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:22.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:22 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:22.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:22 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:22.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:22 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.033 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T10:35:23.033 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T10:35:23.033 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T10:35:23.353 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:23.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[127026]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T10:35:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[127026]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[127026]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:35:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[127026]: osdmap e59: 8 total, 8 up, 8 in 2023-10-09T10:35:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:35:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:35:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:35:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:35:23.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:35:23.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:35:23.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[112589]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T10:35:23.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[112589]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:23.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[112589]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:35:23.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[112589]: osdmap e59: 8 total, 8 up, 8 in 2023-10-09T10:35:23.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:35:23.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:35:23.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:35:23.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:35:23.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:35:23.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:23 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:35:23.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:23 smithi155 ceph-mon[117223]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T10:35:23.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:23 smithi155 ceph-mon[117223]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:23.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:23 smithi155 ceph-mon[117223]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:35:23.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:23 smithi155 ceph-mon[117223]: osdmap e59: 8 total, 8 up, 8 in 2023-10-09T10:35:23.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:23 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:23 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:23 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:23 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:23 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:35:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:23 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:35:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:23 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:35:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:23 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:35:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:23 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:23 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:35:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:23 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:35:24.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[127026]: Adjusting osd_memory_target on smithi155 to 4022M 2023-10-09T10:35:24.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[127026]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:35:24.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[127026]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:35:24.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[127026]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:35:24.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[127026]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:35:24.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[127026]: mgrmap e19: y(active, since 2s), standbys: x 2023-10-09T10:35:24.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[127026]: Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:35:24.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[127026]: Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:35:24.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[127026]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:35:24.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[127026]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:35:24.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:24.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:24.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:24.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:24.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:24.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[112589]: Adjusting osd_memory_target on smithi155 to 4022M 2023-10-09T10:35:24.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[112589]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:35:24.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[112589]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:35:24.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[112589]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:35:24.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[112589]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:35:24.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[112589]: mgrmap e19: y(active, since 2s), standbys: x 2023-10-09T10:35:24.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[112589]: Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:35:24.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[112589]: Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:35:24.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[112589]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:35:24.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[112589]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:35:24.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:24.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:24.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:24.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:24.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:24 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:24.720 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:35:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:24 smithi155 ceph-mon[117223]: Adjusting osd_memory_target on smithi155 to 4022M 2023-10-09T10:35:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:24 smithi155 ceph-mon[117223]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:35:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:24 smithi155 ceph-mon[117223]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:35:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:24 smithi155 ceph-mon[117223]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:35:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:24 smithi155 ceph-mon[117223]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:35:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:24 smithi155 ceph-mon[117223]: mgrmap e19: y(active, since 2s), standbys: x 2023-10-09T10:35:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:24 smithi155 ceph-mon[117223]: Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:35:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:24 smithi155 ceph-mon[117223]: Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:35:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:24 smithi155 ceph-mon[117223]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:35:24.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:24 smithi155 ceph-mon[117223]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:35:24.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:24 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:24.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:24 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:24.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:24 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:24.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:24 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:24.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:24 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:25.136 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":19,"active_gid":24362,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6800","nonce":1128053553},{"type":"v1","addr":"172.21.15.125:6801","nonce":1128053553}]},"active_addr":"172.21.15.125:6801/1128053553","active_change":"2023-10-09T10:35:20.608067+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24346,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":57,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":86510625}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":1483568239}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":2757467822}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":3709143923}]}]} 2023-10-09T10:35:25.140 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T10:35:25.140 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T10:35:25.140 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T10:35:25.425 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:25.584 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:25 smithi125 ceph-mon[112589]: Deploying daemon alertmanager.a on smithi125 2023-10-09T10:35:25.584 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:25 smithi125 ceph-mon[112589]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:25.585 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:25 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2166052607' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T10:35:25.585 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:25 smithi125 ceph-mon[127026]: Deploying daemon alertmanager.a on smithi125 2023-10-09T10:35:25.585 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:25 smithi125 ceph-mon[127026]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:25.585 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:25 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2166052607' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T10:35:25.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:25 smithi155 ceph-mon[117223]: Deploying daemon alertmanager.a on smithi125 2023-10-09T10:35:25.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:25 smithi155 ceph-mon[117223]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:25.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:25 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2166052607' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T10:35:26.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:26 smithi125 ceph-mon[112589]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T10:35:26.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:26 smithi125 ceph-mon[112589]: Cluster is now healthy 2023-10-09T10:35:26.677 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:26 smithi125 ceph-mon[127026]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T10:35:26.677 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:26 smithi125 ceph-mon[127026]: Cluster is now healthy 2023-10-09T10:35:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:26 smithi155 ceph-mon[117223]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T10:35:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:26 smithi155 ceph-mon[117223]: Cluster is now healthy 2023-10-09T10:35:27.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:27 smithi125 ceph-mon[112589]: pgmap v8: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:27.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:27 smithi125 ceph-mon[127026]: pgmap v8: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:27.713 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:35:27.713 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":59,"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","created":"2023-10-09T10:31:07.767709+0000","modified":"2023-10-09T10:35:22.616608+0000","last_up_change":"2023-10-09T10:34:57.946717+0000","last_in_change":"2023-10-09T10:34:43.351594+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T10:33:34.342993+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T10:35:19.807378+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"874cd96b-ddd8-4164-931e-06f2388dd41b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6803","nonce":3322074820}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6805","nonce":3322074820}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6809","nonce":3322074820}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6807","nonce":3322074820}]},"public_addr":"172.21.15.125:6803/3322074820","cluster_addr":"172.21.15.125:6805/3322074820","heartbeat_back_addr":"172.21.15.125:6809/3322074820","heartbeat_front_addr":"172.21.15.125:6807/3322074820","state":["exists","up"]},{"osd":1,"uuid":"0e517dd5-6d0b-4841-bb81-1eb04377ba5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6811","nonce":3260318814}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6813","nonce":3260318814}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6817","nonce":3260318814}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6815","nonce":3260318814}]},"public_addr":"172.21.15.125:6811/3260318814","cluster_addr":"172.21.15.125:6813/3260318814","heartbeat_back_addr":"172.21.15.125:6817/3260318814","heartbeat_front_addr":"172.21.15.125:6815/3260318814","state":["exists","up"]},{"osd":2,"uuid":"643c1f3c-ecd9-464a-ad53-bf35a4deee5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6819","nonce":4010155657}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6821","nonce":4010155657}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6825","nonce":4010155657}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6823","nonce":4010155657}]},"public_addr":"172.21.15.125:6819/4010155657","cluster_addr":"172.21.15.125:6821/4010155657","heartbeat_back_addr":"172.21.15.125:6825/4010155657","heartbeat_front_addr":"172.21.15.125:6823/4010155657","state":["exists","up"]},{"osd":3,"uuid":"3015cd6e-b270-44c3-8e6e-6bcd979bcd21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6827","nonce":2950193979}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6829","nonce":2950193979}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6833","nonce":2950193979}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6831","nonce":2950193979}]},"public_addr":"172.21.15.125:6827/2950193979","cluster_addr":"172.21.15.125:6829/2950193979","heartbeat_back_addr":"172.21.15.125:6833/2950193979","heartbeat_front_addr":"172.21.15.125:6831/2950193979","state":["exists","up"]},{"osd":4,"uuid":"272c7b95-f89c-4fe1-a299-a861e0ec9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6801","nonce":1086065767}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6803","nonce":1086065767}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6807","nonce":1086065767}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6805","nonce":1086065767}]},"public_addr":"172.21.15.155:6801/1086065767","cluster_addr":"172.21.15.155:6803/1086065767","heartbeat_back_addr":"172.21.15.155:6807/1086065767","heartbeat_front_addr":"172.21.15.155:6805/1086065767","state":["exists","up"]},{"osd":5,"uuid":"201d51f2-233c-40fc-a42d-649d225954fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6809","nonce":104536199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6811","nonce":104536199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6815","nonce":104536199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6813","nonce":104536199}]},"public_addr":"172.21.15.155:6809/104536199","cluster_addr":"172.21.15.155:6811/104536199","heartbeat_back_addr":"172.21.15.155:6815/104536199","heartbeat_front_addr":"172.21.15.155:6813/104536199","state":["exists","up"]},{"osd":6,"uuid":"4ec66642-108e-401f-aff3-e7ab258bc083","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6817","nonce":1730141492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6819","nonce":1730141492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6823","nonce":1730141492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6821","nonce":1730141492}]},"public_addr":"172.21.15.155:6817/1730141492","cluster_addr":"172.21.15.155:6819/1730141492","heartbeat_back_addr":"172.21.15.155:6823/1730141492","heartbeat_front_addr":"172.21.15.155:6821/1730141492","state":["exists","up"]},{"osd":7,"uuid":"f8eccbf5-4ad0-43e7-be9a-00c05a362e57","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6825","nonce":3094552858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6827","nonce":3094552858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6831","nonce":3094552858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6829","nonce":3094552858}]},"public_addr":"172.21.15.155:6825/3094552858","cluster_addr":"172.21.15.155:6827/3094552858","heartbeat_back_addr":"172.21.15.155:6831/3094552858","heartbeat_front_addr":"172.21.15.155:6829/3094552858","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:32:58.904066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:15.883014+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:32.639145+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:49.937002+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:06.212675+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:22.755647+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:39.598906+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:56.143619+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:0/1316191185":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6801/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6800/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/2429796720":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6801/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6800/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6800/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/1623791415":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/461781620":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/1101683240":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/2789680395":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/725980307":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/1709045642":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/2611643565":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6801/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/178379618":"2023-10-10T10:31:33.872122+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T10:35:27.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:27 smithi155 ceph-mon[117223]: pgmap v8: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:28.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:28 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1261715824' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:35:28.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:28 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/1261715824' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:35:28.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:28 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1261715824' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:35:28.916 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T10:35:28.916 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T10:35:29.191 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:29.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:29 smithi125 ceph-mon[112589]: pgmap v9: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:29.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:29 smithi125 ceph-mon[127026]: pgmap v9: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:29 smithi155 ceph-mon[117223]: pgmap v9: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:30.207 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:30 smithi125 systemd[1]: Started Ceph alertmanager.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:35:30.207 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:30 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[163900]: ts=2023-10-09T10:35:30.065Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T10:35:30.207 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:30 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[163900]: ts=2023-10-09T10:35:30.065Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T10:35:30.207 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:30 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[163900]: ts=2023-10-09T10:35:30.068Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T10:35:30.207 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:30 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[163900]: ts=2023-10-09T10:35:30.068Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T10:35:30.573 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:30 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:30] ENGINE Bus STOPPING 2023-10-09T10:35:30.573 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:30 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T10:35:30.573 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:30 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:30] ENGINE Bus STOPPED 2023-10-09T10:35:30.573 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:30 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:30] ENGINE Bus STARTING 2023-10-09T10:35:30.573 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:30 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:30] ENGINE Serving on http://:::9283 2023-10-09T10:35:30.573 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:30 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:30] ENGINE Bus STARTED 2023-10-09T10:35:30.753 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:35:30.754 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":59,"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","created":"2023-10-09T10:31:07.767709+0000","modified":"2023-10-09T10:35:22.616608+0000","last_up_change":"2023-10-09T10:34:57.946717+0000","last_in_change":"2023-10-09T10:34:43.351594+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T10:33:34.342993+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T10:35:19.807378+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"874cd96b-ddd8-4164-931e-06f2388dd41b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6803","nonce":3322074820}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6805","nonce":3322074820}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6809","nonce":3322074820}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6807","nonce":3322074820}]},"public_addr":"172.21.15.125:6803/3322074820","cluster_addr":"172.21.15.125:6805/3322074820","heartbeat_back_addr":"172.21.15.125:6809/3322074820","heartbeat_front_addr":"172.21.15.125:6807/3322074820","state":["exists","up"]},{"osd":1,"uuid":"0e517dd5-6d0b-4841-bb81-1eb04377ba5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6811","nonce":3260318814}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6813","nonce":3260318814}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6817","nonce":3260318814}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6815","nonce":3260318814}]},"public_addr":"172.21.15.125:6811/3260318814","cluster_addr":"172.21.15.125:6813/3260318814","heartbeat_back_addr":"172.21.15.125:6817/3260318814","heartbeat_front_addr":"172.21.15.125:6815/3260318814","state":["exists","up"]},{"osd":2,"uuid":"643c1f3c-ecd9-464a-ad53-bf35a4deee5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6819","nonce":4010155657}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6821","nonce":4010155657}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6825","nonce":4010155657}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6823","nonce":4010155657}]},"public_addr":"172.21.15.125:6819/4010155657","cluster_addr":"172.21.15.125:6821/4010155657","heartbeat_back_addr":"172.21.15.125:6825/4010155657","heartbeat_front_addr":"172.21.15.125:6823/4010155657","state":["exists","up"]},{"osd":3,"uuid":"3015cd6e-b270-44c3-8e6e-6bcd979bcd21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6827","nonce":2950193979}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6829","nonce":2950193979}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6833","nonce":2950193979}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6831","nonce":2950193979}]},"public_addr":"172.21.15.125:6827/2950193979","cluster_addr":"172.21.15.125:6829/2950193979","heartbeat_back_addr":"172.21.15.125:6833/2950193979","heartbeat_front_addr":"172.21.15.125:6831/2950193979","state":["exists","up"]},{"osd":4,"uuid":"272c7b95-f89c-4fe1-a299-a861e0ec9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6801","nonce":1086065767}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6803","nonce":1086065767}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6807","nonce":1086065767}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6805","nonce":1086065767}]},"public_addr":"172.21.15.155:6801/1086065767","cluster_addr":"172.21.15.155:6803/1086065767","heartbeat_back_addr":"172.21.15.155:6807/1086065767","heartbeat_front_addr":"172.21.15.155:6805/1086065767","state":["exists","up"]},{"osd":5,"uuid":"201d51f2-233c-40fc-a42d-649d225954fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6809","nonce":104536199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6811","nonce":104536199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6815","nonce":104536199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6813","nonce":104536199}]},"public_addr":"172.21.15.155:6809/104536199","cluster_addr":"172.21.15.155:6811/104536199","heartbeat_back_addr":"172.21.15.155:6815/104536199","heartbeat_front_addr":"172.21.15.155:6813/104536199","state":["exists","up"]},{"osd":6,"uuid":"4ec66642-108e-401f-aff3-e7ab258bc083","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6817","nonce":1730141492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6819","nonce":1730141492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6823","nonce":1730141492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6821","nonce":1730141492}]},"public_addr":"172.21.15.155:6817/1730141492","cluster_addr":"172.21.15.155:6819/1730141492","heartbeat_back_addr":"172.21.15.155:6823/1730141492","heartbeat_front_addr":"172.21.15.155:6821/1730141492","state":["exists","up"]},{"osd":7,"uuid":"f8eccbf5-4ad0-43e7-be9a-00c05a362e57","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6825","nonce":3094552858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6827","nonce":3094552858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6831","nonce":3094552858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6829","nonce":3094552858}]},"public_addr":"172.21.15.155:6825/3094552858","cluster_addr":"172.21.15.155:6827/3094552858","heartbeat_back_addr":"172.21.15.155:6831/3094552858","heartbeat_front_addr":"172.21.15.155:6829/3094552858","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:32:58.904066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:15.883014+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:32.639145+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:49.937002+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:06.212675+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:22.755647+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:39.598906+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:56.143619+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:0/1316191185":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6801/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6800/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/2429796720":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6801/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6800/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6800/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/1623791415":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/461781620":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/1101683240":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/2789680395":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/725980307":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/1709045642":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/2611643565":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6801/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/178379618":"2023-10-10T10:31:33.872122+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T10:35:31.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T10:35:31.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T10:35:31.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T10:35:31.170 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T10:35:31.170 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T10:35:31.171 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T10:35:31.171 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T10:35:31.172 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T10:35:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:31 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:31 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:31 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:31 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:31 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:31 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:31 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T10:35:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:31 smithi155 ceph-mon[117223]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T10:35:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:31 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:31 smithi155 ceph-mon[117223]: Deploying daemon grafana.a on smithi155 2023-10-09T10:35:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:31 smithi155 ceph-mon[117223]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:31 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:31 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2947740242' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:35:31.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T10:35:31.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[112589]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T10:35:31.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[112589]: Deploying daemon grafana.a on smithi155 2023-10-09T10:35:31.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[112589]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:31.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2947740242' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:35:31.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T10:35:31.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[127026]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T10:35:31.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[127026]: Deploying daemon grafana.a on smithi155 2023-10-09T10:35:31.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[127026]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:31.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:31.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:31 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2947740242' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:35:31.674 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:31.680 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:31.711 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:31.725 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:32.194 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:32 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[163900]: ts=2023-10-09T10:35:32.041Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000376001s 2023-10-09T10:35:32.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:31 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:35:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T10:35:32.840 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:33.208 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:33.412 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:33.473 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:33.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:33 smithi125 ceph-mon[112589]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:33.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:33 smithi125 ceph-mon[127026]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:33 smithi155 ceph-mon[117223]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:34.760 INFO:teuthology.orchestra.run.smithi125.stdout:219043332105 2023-10-09T10:35:34.760 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T10:35:35.967 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:35 smithi125 ceph-mon[112589]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:35.968 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:35 smithi125 ceph-mon[127026]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:35 smithi155 ceph-mon[117223]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:35.984 INFO:teuthology.orchestra.run.smithi125.stdout:34359738399 2023-10-09T10:35:35.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T10:35:36.152 INFO:teuthology.orchestra.run.smithi125.stdout:128849018899 2023-10-09T10:35:36.152 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T10:35:36.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:36 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:35:36.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:36 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:35:36.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:36 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:35:37.598 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:37.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:37 smithi125 ceph-mon[112589]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:37.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:37 smithi125 ceph-mon[127026]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:37.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:37 smithi155 ceph-mon[117223]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:38.122 INFO:teuthology.orchestra.run.smithi125.stdout:55834574879 2023-10-09T10:35:38.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T10:35:38.319 INFO:teuthology.orchestra.run.smithi125.stdout:188978561037 2023-10-09T10:35:38.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T10:35:38.652 INFO:teuthology.orchestra.run.smithi125.stdout:107374182424 2023-10-09T10:35:38.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T10:35:38.750 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:38.870 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:38.937 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:39.071 INFO:teuthology.orchestra.run.smithi125.stdout:77309411353 2023-10-09T10:35:39.071 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T10:35:39.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:39 smithi125 ceph-mon[112589]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:39.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:39 smithi125 ceph-mon[127026]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:39.615 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:39.882 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:39 smithi155 ceph-mon[117223]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:40.028 INFO:teuthology.orchestra.run.smithi125.stdout:34359738400 2023-10-09T10:35:40.141 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:40 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[163900]: ts=2023-10-09T10:35:40.044Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003139887s 2023-10-09T10:35:40.160 INFO:teuthology.orchestra.run.smithi125.stdout:158913789969 2023-10-09T10:35:40.161 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T10:35:40.642 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738399 got 34359738400 for osd.0 2023-10-09T10:35:40.643 DEBUG:teuthology.parallel:result is None 2023-10-09T10:35:40.757 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:40 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3390840651' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T10:35:40.758 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:40 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3390840651' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T10:35:40.771 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:40.968 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:40 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3390840651' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T10:35:41.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:41 smithi125 ceph-mon[112589]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:41.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:41 smithi125 ceph-mon[127026]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:41.753 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:41 smithi155 ceph-mon[117223]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:41.933 INFO:teuthology.orchestra.run.smithi125.stdout:219043332106 2023-10-09T10:35:41.971 INFO:teuthology.orchestra.run.smithi125.stdout:188978561037 2023-10-09T10:35:41.975 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:41.975 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:42.113 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:35:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T10:35:42.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:42 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1656883982' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T10:35:42.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:42 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3138472146' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T10:35:42.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:42 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/1656883982' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T10:35:42.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:42 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3138472146' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T10:35:42.694 INFO:tasks.cephadm.ceph_manager.ceph:need seq 219043332105 got 219043332106 for osd.7 2023-10-09T10:35:42.695 DEBUG:teuthology.parallel:result is None 2023-10-09T10:35:42.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:42 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1656883982' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T10:35:42.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:42 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3138472146' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T10:35:42.721 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:42 smithi155 systemd[1]: Starting Ceph grafana.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:35:43.204 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:42 smithi155 bash[153297]: 591b62826625dc622989154d830cbbdc57bb02e1aea4407431da43f207ef7fdb 2023-10-09T10:35:43.204 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:42 smithi155 systemd[1]: Started Ceph grafana.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:35:43.204 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=settings t=2023-10-09T10:35:43.093659065Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-10-09T10:35:43.204 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=settings t=2023-10-09T10:35:43.093880851Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-10-09T10:35:43.204 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=settings t=2023-10-09T10:35:43.093897881Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-10-09T10:35:43.204 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=settings t=2023-10-09T10:35:43.093905367Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-10-09T10:35:43.205 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=settings t=2023-10-09T10:35:43.09391157Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T10:35:43.205 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=settings t=2023-10-09T10:35:43.093917462Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T10:35:43.205 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=settings t=2023-10-09T10:35:43.093923759Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T10:35:43.205 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=settings t=2023-10-09T10:35:43.093929696Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T10:35:43.205 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=settings t=2023-10-09T10:35:43.093935689Z level=info msg="Path Home" path=/usr/share/grafana 2023-10-09T10:35:43.205 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=settings t=2023-10-09T10:35:43.093941315Z level=info msg="Path Data" path=/var/lib/grafana 2023-10-09T10:35:43.205 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=settings t=2023-10-09T10:35:43.093946578Z level=info msg="Path Logs" path=/var/log/grafana 2023-10-09T10:35:43.205 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=settings t=2023-10-09T10:35:43.09395208Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-10-09T10:35:43.205 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=settings t=2023-10-09T10:35:43.093957579Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-10-09T10:35:43.205 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=settings t=2023-10-09T10:35:43.093963448Z level=info msg="App mode production" 2023-10-09T10:35:43.205 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=sqlstore t=2023-10-09T10:35:43.094027052Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-10-09T10:35:43.205 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=sqlstore t=2023-10-09T10:35:43.094049096Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T10:35:43.206 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.09491042Z level=info msg="Starting DB migrations" 2023-10-09T10:35:43.206 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.095008036Z level=info msg="Executing migration" id="create migration_log table" 2023-10-09T10:35:43.206 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.095584425Z level=info msg="Executing migration" id="create user table" 2023-10-09T10:35:43.206 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.096057879Z level=info msg="Executing migration" id="add unique index user.login" 2023-10-09T10:35:43.206 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.098981101Z level=info msg="Executing migration" id="add unique index user.email" 2023-10-09T10:35:43.206 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.101941657Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-10-09T10:35:43.206 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.102429044Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-10-09T10:35:43.206 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.105315599Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-10-09T10:35:43.206 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.108274074Z level=info msg="Executing migration" id="create user table v2" 2023-10-09T10:35:43.206 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.111214447Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-10-09T10:35:43.206 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.113713618Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-10-09T10:35:43.207 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.116815846Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-10-09T10:35:43.207 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.119688121Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-10-09T10:35:43.207 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.120144994Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-10-09T10:35:43.207 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.122732679Z level=info msg="Executing migration" id="Update user table charset" 2023-10-09T10:35:43.207 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.123039653Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-10-09T10:35:43.207 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.125422374Z level=info msg="Executing migration" id="Add missing user data" 2023-10-09T10:35:43.207 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.125817048Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-10-09T10:35:43.207 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.128234183Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-10-09T10:35:43.207 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.128677415Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-10-09T10:35:43.207 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.129132691Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-10-09T10:35:43.208 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.131798813Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-10-09T10:35:43.208 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.134505277Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-10-09T10:35:43.208 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.137281838Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-10-09T10:35:43.208 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.139884745Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-10-09T10:35:43.208 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.142587038Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-10-09T10:35:43.208 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.14514283Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-10-09T10:35:43.208 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.147636442Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-10-09T10:35:43.209 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.150540157Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-10-09T10:35:43.209 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.153038586Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-10-09T10:35:43.209 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.155587357Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-10-09T10:35:43.209 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.156373885Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-10-09T10:35:43.209 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.157232018Z level=info msg="Executing migration" id="create temp_user v2" 2023-10-09T10:35:43.209 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.158219635Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-10-09T10:35:43.209 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.159061074Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-10-09T10:35:43.209 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.15971012Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-10-09T10:35:43.209 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.16066185Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-10-09T10:35:43.209 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.161375619Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-10-09T10:35:43.209 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.162161838Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-10-09T10:35:43.209 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.163152739Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-10-09T10:35:43.210 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.163773352Z level=info msg="Executing migration" id="create star table" 2023-10-09T10:35:43.210 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.164164366Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-10-09T10:35:43.210 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.165140597Z level=info msg="Executing migration" id="create org table v1" 2023-10-09T10:35:43.210 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.165814541Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-10-09T10:35:43.210 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.166651397Z level=info msg="Executing migration" id="create org_user table v1" 2023-10-09T10:35:43.210 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.167617827Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-10-09T10:35:43.210 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.168242134Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-10-09T10:35:43.210 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.169108881Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-10-09T10:35:43.210 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.169511732Z level=info msg="Executing migration" id="Update org table charset" 2023-10-09T10:35:43.210 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.170461569Z level=info msg="Executing migration" id="Update org_user table charset" 2023-10-09T10:35:43.210 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.171267565Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-10-09T10:35:43.210 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.172139776Z level=info msg="Executing migration" id="create dashboard table" 2023-10-09T10:35:43.211 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.172544755Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-10-09T10:35:43.211 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.172954557Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-10-09T10:35:43.211 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.173815448Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-10-09T10:35:43.211 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.174754383Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-10-09T10:35:43.211 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.175501299Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-10-09T10:35:43.211 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.175908801Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-10-09T10:35:43.211 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.176808724Z level=info msg="Executing migration" id="create dashboard v2" 2023-10-09T10:35:43.211 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.177638015Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-10-09T10:35:43.211 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.178627699Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-10-09T10:35:43.211 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.179196234Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-10-09T10:35:43.211 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.180047558Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-10-09T10:35:43.211 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.180514396Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-10-09T10:35:43.212 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.18085285Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-10-09T10:35:43.212 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.18168291Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-10-09T10:35:43.212 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.182214759Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-10-09T10:35:43.212 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.182722305Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-10-09T10:35:43.212 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.185733547Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-10-09T10:35:43.212 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.188697633Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-10-09T10:35:43.212 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.191525595Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-10-09T10:35:43.212 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.19199708Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-10-09T10:35:43.212 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.194811101Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-10-09T10:35:43.212 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.195137138Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-10-09T10:35:43.212 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.19760893Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-10-09T10:35:43.213 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.200532715Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-10-09T10:35:43.213 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.201040449Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-10-09T10:35:43.213 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.203886619Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-10-09T10:35:43.213 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.204285976Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-10-09T10:35:43.279 INFO:tasks.cephadm.ceph_manager.ceph:need seq 188978561037 got 188978561037 for osd.6 2023-10-09T10:35:43.279 DEBUG:teuthology.parallel:result is None 2023-10-09T10:35:43.454 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.206552392Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-10-09T10:35:43.455 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.209269778Z level=info msg="Executing migration" id="Update dashboard title length" 2023-10-09T10:35:43.455 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.212101669Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-10-09T10:35:43.455 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.215136414Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-10-09T10:35:43.455 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.217851518Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-10-09T10:35:43.455 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.22061039Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-10-09T10:35:43.455 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.221059248Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-10-09T10:35:43.455 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.223753711Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-10-09T10:35:43.455 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.226579806Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-10-09T10:35:43.455 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.229051501Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-10-09T10:35:43.455 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.229450261Z level=info msg="Executing migration" id="Add check_sum column" 2023-10-09T10:35:43.455 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.232215311Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-10-09T10:35:43.456 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.235001389Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-10-09T10:35:43.456 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.237637165Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-10-09T10:35:43.456 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.237962488Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-10-09T10:35:43.456 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.240754194Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-10-09T10:35:43.456 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.243389126Z level=info msg="Executing migration" id="create data_source table" 2023-10-09T10:35:43.456 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.246083575Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-10-09T10:35:43.456 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.246556042Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-10-09T10:35:43.456 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.249324771Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-10-09T10:35:43.456 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.251932813Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-10-09T10:35:43.456 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.25479851Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-10-09T10:35:43.456 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.255996173Z level=info msg="Executing migration" id="create data_source table v2" 2023-10-09T10:35:43.456 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.256846938Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-10-09T10:35:43.457 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.257652412Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-10-09T10:35:43.457 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.258487177Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-10-09T10:35:43.457 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.258924425Z level=info msg="Executing migration" id="Add column with_credentials" 2023-10-09T10:35:43.457 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.259648276Z level=info msg="Executing migration" id="Add secure json data column" 2023-10-09T10:35:43.457 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.260199254Z level=info msg="Executing migration" id="Update data_source table charset" 2023-10-09T10:35:43.457 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.260496848Z level=info msg="Executing migration" id="Update initial version to 1" 2023-10-09T10:35:43.457 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.260839063Z level=info msg="Executing migration" id="Add read_only data column" 2023-10-09T10:35:43.457 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.26138268Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-10-09T10:35:43.457 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.261730916Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-10-09T10:35:43.457 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.262052611Z level=info msg="Executing migration" id="Add uid column" 2023-10-09T10:35:43.457 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.262904185Z level=info msg="Executing migration" id="Update uid value" 2023-10-09T10:35:43.457 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.263743762Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-10-09T10:35:43.458 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.264699757Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-10-09T10:35:43.458 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.265227569Z level=info msg="Executing migration" id="create api_key table" 2023-10-09T10:35:43.459 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.266003343Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-10-09T10:35:43.459 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.266875889Z level=info msg="Executing migration" id="add index api_key.key" 2023-10-09T10:35:43.459 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.267665784Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-10-09T10:35:43.459 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.268150709Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-10-09T10:35:43.459 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.268986788Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-10-09T10:35:43.459 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.269441126Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-10-09T10:35:43.460 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.269906161Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-10-09T10:35:43.460 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.2709991Z level=info msg="Executing migration" id="create api_key table v2" 2023-10-09T10:35:43.460 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.271869402Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-10-09T10:35:43.460 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.272636303Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-10-09T10:35:43.460 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.275439991Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-10-09T10:35:43.460 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.278041753Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-10-09T10:35:43.460 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.280842918Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-10-09T10:35:43.460 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.281256811Z level=info msg="Executing migration" id="Update api_key table charset" 2023-10-09T10:35:43.460 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.28399988Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-10-09T10:35:43.460 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.284666432Z level=info msg="Executing migration" id="Add service account foreign key" 2023-10-09T10:35:43.460 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.286617645Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-10-09T10:35:43.460 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.289427711Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-10-09T10:35:43.461 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.292019091Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-10-09T10:35:43.461 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.294750839Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-10-09T10:35:43.461 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.297681969Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-10-09T10:35:43.461 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.300367878Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-10-09T10:35:43.461 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.302867674Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-10-09T10:35:43.461 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.305574336Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-10-09T10:35:43.461 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.308345065Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-10-09T10:35:43.461 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.311070307Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-10-09T10:35:43.461 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.313788407Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-10-09T10:35:43.461 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.316628725Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-10-09T10:35:43.461 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.31922479Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-10-09T10:35:43.461 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.321938921Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-10-09T10:35:43.462 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.32477241Z level=info msg="Executing migration" id="create quota table v1" 2023-10-09T10:35:43.462 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.327319525Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-10-09T10:35:43.462 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.330094741Z level=info msg="Executing migration" id="Update quota table charset" 2023-10-09T10:35:43.462 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.332615918Z level=info msg="Executing migration" id="create plugin_setting table" 2023-10-09T10:35:43.462 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.3330544Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-10-09T10:35:43.462 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.335651806Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-10-09T10:35:43.462 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.338302046Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-10-09T10:35:43.462 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.340599487Z level=info msg="Executing migration" id="create session table" 2023-10-09T10:35:43.462 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.341109222Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-10-09T10:35:43.462 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.341414531Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-10-09T10:35:43.462 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.343585903Z level=info msg="Executing migration" id="create playlist table v2" 2023-10-09T10:35:43.463 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.344020207Z level=info msg="Executing migration" id="create playlist item table v2" 2023-10-09T10:35:43.463 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.346828241Z level=info msg="Executing migration" id="Update playlist table charset" 2023-10-09T10:35:43.463 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.349355162Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-10-09T10:35:43.463 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.349729662Z level=info msg="Executing migration" id="drop preferences table v2" 2023-10-09T10:35:43.463 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.35005557Z level=info msg="Executing migration" id="drop preferences table v3" 2023-10-09T10:35:43.463 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.350341168Z level=info msg="Executing migration" id="create preferences table v3" 2023-10-09T10:35:43.463 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.350765466Z level=info msg="Executing migration" id="Update preferences table charset" 2023-10-09T10:35:43.463 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.353505437Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-10-09T10:35:43.463 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.354124326Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-10-09T10:35:43.463 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.354437947Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-10-09T10:35:43.463 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.356848728Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-10-09T10:35:43.464 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.359409043Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-10-09T10:35:43.464 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.362022137Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-10-09T10:35:43.464 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.362496711Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-10-09T10:35:43.464 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.36518766Z level=info msg="Executing migration" id="create alert table v1" 2023-10-09T10:35:43.464 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.367699942Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-10-09T10:35:43.464 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.368335344Z level=info msg="Executing migration" id="add index alert state" 2023-10-09T10:35:43.464 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.369124634Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-10-09T10:35:43.464 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.369564017Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-10-09T10:35:43.464 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.369947759Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-10-09T10:35:43.464 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.370375703Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-10-09T10:35:43.464 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.370789346Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-10-09T10:35:43.464 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.372366666Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-10-09T10:35:43.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.37277118Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-10-09T10:35:43.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.373205977Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-10-09T10:35:43.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.373519347Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-10-09T10:35:43.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.373887558Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-10-09T10:35:43.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.374254669Z level=info msg="Executing migration" id="Add column is_default" 2023-10-09T10:35:43.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.374955105Z level=info msg="Executing migration" id="Add column frequency" 2023-10-09T10:35:43.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.375644002Z level=info msg="Executing migration" id="Add column send_reminder" 2023-10-09T10:35:43.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.376279461Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-10-09T10:35:43.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.376929461Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-10-09T10:35:43.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.377381807Z level=info msg="Executing migration" id="Update alert table charset" 2023-10-09T10:35:43.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.377708815Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-10-09T10:35:43.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.378028613Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-10-09T10:35:43.466 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.378440478Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-10-09T10:35:43.466 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.378910413Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-10-09T10:35:43.466 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.379288205Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-10-09T10:35:43.466 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.379712784Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-10-09T10:35:43.466 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.380151729Z level=info msg="Executing migration" id="Add for to alert table" 2023-10-09T10:35:43.466 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.380834757Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-10-09T10:35:43.466 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.381518181Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-10-09T10:35:43.466 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.3818626Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-10-09T10:35:43.466 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.382307537Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-10-09T10:35:43.466 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.382745567Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-10-09T10:35:43.466 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.383398421Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-10-09T10:35:43.466 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.383733349Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-10-09T10:35:43.467 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.38415519Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-10-09T10:35:43.467 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.384603762Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-10-09T10:35:43.467 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.384945778Z level=info msg="Executing migration" id="create annotation table v5" 2023-10-09T10:35:43.467 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.385581202Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-10-09T10:35:43.467 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.386304198Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-10-09T10:35:43.467 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.386771137Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-10-09T10:35:43.467 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.387216504Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-10-09T10:35:43.467 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.387712339Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-10-09T10:35:43.467 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.388527944Z level=info msg="Executing migration" id="Update annotation table charset" 2023-10-09T10:35:43.467 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.389295214Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-10-09T10:35:43.467 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.390023181Z level=info msg="Executing migration" id="Drop category_id index" 2023-10-09T10:35:43.467 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.39084121Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-10-09T10:35:43.468 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.39159206Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-10-09T10:35:43.468 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.392409308Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-10-09T10:35:43.468 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.393210918Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-10-09T10:35:43.468 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.39400538Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-10-09T10:35:43.468 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.395702585Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-10-09T10:35:43.468 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.396539171Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-10-09T10:35:43.468 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.397337709Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-10-09T10:35:43.468 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.398220804Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-10-09T10:35:43.468 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.398949438Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-10-09T10:35:43.468 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.399775472Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-10-09T10:35:43.468 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.400564426Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-10-09T10:35:43.468 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.401355623Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-10-09T10:35:43.468 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.402359208Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-10-09T10:35:43.469 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.403150194Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-10-09T10:35:43.469 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.403878129Z level=info msg="Executing migration" id="Add epoch_end column" 2023-10-09T10:35:43.469 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.404649499Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-10-09T10:35:43.469 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.405122618Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-10-09T10:35:43.469 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.405858294Z level=info msg="Executing migration" id="Move region to single row" 2023-10-09T10:35:43.469 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.406278126Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-10-09T10:35:43.469 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.406726657Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-10-09T10:35:43.469 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.407551564Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-10-09T10:35:43.469 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.408610528Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-10-09T10:35:43.469 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.409239558Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-10-09T10:35:43.469 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.410152692Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-10-09T10:35:43.469 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.410982033Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-10-09T10:35:43.470 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.411873696Z level=info msg="Executing migration" id="create test_data table" 2023-10-09T10:35:43.470 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.412563729Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-10-09T10:35:43.470 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.413010754Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-10-09T10:35:43.470 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.41378001Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-10-09T10:35:43.470 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.414724798Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-10-09T10:35:43.470 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.415269441Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-10-09T10:35:43.470 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.416306364Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-10-09T10:35:43.470 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.416923075Z level=info msg="Executing migration" id="create team table" 2023-10-09T10:35:43.470 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.417719914Z level=info msg="Executing migration" id="add index team.org_id" 2023-10-09T10:35:43.470 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.418675826Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-10-09T10:35:43.470 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.419394894Z level=info msg="Executing migration" id="create team member table" 2023-10-09T10:35:43.470 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.420198801Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-10-09T10:35:43.471 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.420982428Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-10-09T10:35:43.471 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.421935983Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-10-09T10:35:43.471 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.422631754Z level=info msg="Executing migration" id="Add column email to team table" 2023-10-09T10:35:43.471 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.423499078Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-10-09T10:35:43.471 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.424254812Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-10-09T10:35:43.471 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.425123332Z level=info msg="Executing migration" id="create dashboard acl table" 2023-10-09T10:35:43.471 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.426006252Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-10-09T10:35:43.471 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.427014776Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-10-09T10:35:43.471 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.427726209Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-10-09T10:35:43.471 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.428579049Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-10-09T10:35:43.471 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.429600541Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-10-09T10:35:43.471 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.430285164Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-10-09T10:35:43.472 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.430784377Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-10-09T10:35:43.472 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.431620818Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-10-09T10:35:43.472 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.432536621Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-10-09T10:35:43.472 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.433477618Z level=info msg="Executing migration" id="create tag table" 2023-10-09T10:35:43.472 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.433939156Z level=info msg="Executing migration" id="add index tag.key_value" 2023-10-09T10:35:43.472 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.434948167Z level=info msg="Executing migration" id="create login attempt table" 2023-10-09T10:35:43.472 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.435650054Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-10-09T10:35:43.472 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.436536544Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-10-09T10:35:43.472 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.436998748Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-10-09T10:35:43.472 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.4389847Z level=info msg="Executing migration" id="create login_attempt v2" 2023-10-09T10:35:43.472 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.441722561Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-10-09T10:35:43.472 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.444587013Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-10-09T10:35:43.473 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.444985135Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-10-09T10:35:43.473 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.447858782Z level=info msg="Executing migration" id="create user auth table" 2023-10-09T10:35:43.473 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.448325958Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-10-09T10:35:43.473 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.450671368Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-10-09T10:35:43.473 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.453674102Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-10-09T10:35:43.473 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.454527481Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-10-09T10:35:43.473 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.457264847Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-10-09T10:35:43.557 INFO:teuthology.orchestra.run.smithi125.stdout:107374182425 2023-10-09T10:35:43.600 INFO:teuthology.orchestra.run.smithi125.stdout:55834574880 2023-10-09T10:35:43.704 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-mon[117223]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:43.704 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:43.704 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:43.704 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:43.705 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:43.705 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.460022564Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-10-09T10:35:43.706 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.462719387Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-10-09T10:35:43.706 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.463180053Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-10-09T10:35:43.706 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.466107955Z level=info msg="Executing migration" id="create server_lock table" 2023-10-09T10:35:43.706 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.46886742Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-10-09T10:35:43.706 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.471616704Z level=info msg="Executing migration" id="create user auth token table" 2023-10-09T10:35:43.706 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.472108237Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-10-09T10:35:43.706 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.474833213Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-10-09T10:35:43.706 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.477575913Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-10-09T10:35:43.706 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.480237374Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-10-09T10:35:43.706 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.481069754Z level=info msg="Executing migration" id="create cache_data table" 2023-10-09T10:35:43.706 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.483801742Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-10-09T10:35:43.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.486610674Z level=info msg="Executing migration" id="create short_url table v1" 2023-10-09T10:35:43.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.491742729Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-10-09T10:35:43.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.492229696Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-10-09T10:35:43.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.492549537Z level=info msg="Executing migration" id="delete alert_definition table" 2023-10-09T10:35:43.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.492871571Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-10-09T10:35:43.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.498149574Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-10-09T10:35:43.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.498792614Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-10-09T10:35:43.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.504082986Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-10-09T10:35:43.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.504444887Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-10-09T10:35:43.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.507114838Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-10-09T10:35:43.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.509837712Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-10-09T10:35:43.708 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.512618559Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-10-09T10:35:43.708 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.513108537Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-10-09T10:35:43.708 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.515832188Z level=info msg="Executing migration" id="drop alert_definition table" 2023-10-09T10:35:43.708 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.516663339Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-10-09T10:35:43.708 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.517528986Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-10-09T10:35:43.708 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.517985929Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-10-09T10:35:43.708 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.518884877Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-10-09T10:35:43.708 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.51944288Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-10-09T10:35:43.708 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.519757563Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-10-09T10:35:43.708 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.520573496Z level=info msg="Executing migration" id="create alert_instance table" 2023-10-09T10:35:43.708 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.521000102Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-10-09T10:35:43.709 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.521401409Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-10-09T10:35:43.709 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.521857099Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-10-09T10:35:43.709 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.522683789Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-10-09T10:35:43.709 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.523674075Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-10-09T10:35:43.709 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.524343792Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-10-09T10:35:43.709 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.527795979Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-10-09T10:35:43.709 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.531239905Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-10-09T10:35:43.709 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.532239557Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-10-09T10:35:43.709 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.532870718Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-10-09T10:35:43.709 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.533729696Z level=info msg="Executing migration" id="create alert_rule table" 2023-10-09T10:35:43.709 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.534552177Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-10-09T10:35:43.709 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.53560012Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-10-09T10:35:43.710 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.536255529Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-10-09T10:35:43.710 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.536738064Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-10-09T10:35:43.710 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.537704179Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-10-09T10:35:43.710 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.538758517Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-10-09T10:35:43.710 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.539629631Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-10-09T10:35:43.710 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.540476974Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-10-09T10:35:43.710 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.541341052Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-10-09T10:35:43.710 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.542171362Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-10-09T10:35:43.710 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.543051118Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-10-09T10:35:43.710 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.543929137Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-10-09T10:35:43.710 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.544800314Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-10-09T10:35:43.711 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.545711433Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-10-09T10:35:43.711 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.54657508Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-10-09T10:35:43.711 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.546905581Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-10-09T10:35:43.711 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.54777371Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-10-09T10:35:43.711 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.550591374Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-10-09T10:35:43.711 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.55352389Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-10-09T10:35:43.711 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.554014188Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-10-09T10:35:43.711 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.556979727Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-10-09T10:35:43.711 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.559775483Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-10-09T10:35:43.711 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.562774362Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-10-09T10:35:43.711 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.563756904Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-10-09T10:35:43.711 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.566575973Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-10-09T10:35:43.712 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.569508873Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-10-09T10:35:43.712 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.572137368Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-10-09T10:35:43.712 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.573047669Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-10-09T10:35:43.712 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.575791361Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-10-09T10:35:43.712 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.578609227Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-10-09T10:35:43.712 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.581363841Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-10-09T10:35:43.712 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.582259637Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-10-09T10:35:43.712 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.58520971Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-10-09T10:35:43.712 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.588112871Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-10-09T10:35:43.712 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.590823712Z level=info msg="Executing migration" id="create provenance_type table" 2023-10-09T10:35:43.712 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.592021342Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-10-09T10:35:43.712 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.594365143Z level=info msg="Executing migration" id="create alert_image table" 2023-10-09T10:35:43.713 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.597158958Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-10-09T10:35:43.713 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.599699213Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-10-09T10:35:43.713 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.600044526Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-10-09T10:35:43.713 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.602674465Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-10-09T10:35:43.713 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.603109129Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-10-09T10:35:43.713 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.603193422Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-10-09T10:35:43.713 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.605785206Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-10-09T10:35:43.713 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.606192437Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-10-09T10:35:43.713 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.606671657Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-10-09T10:35:43.713 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.606810866Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-10-09T10:35:43.713 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.606938306Z level=info msg="alerts found to migrate" alerts=0 2023-10-09T10:35:43.713 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.609327949Z level=info msg="Executing migration" id="create library_element table v1" 2023-10-09T10:35:43.714 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.612128011Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-10-09T10:35:43.714 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.614854865Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-10-09T10:35:43.714 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.61530396Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-10-09T10:35:43.714 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.61815597Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-10-09T10:35:43.714 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.620896789Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-10-09T10:35:43.714 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.623392035Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-10-09T10:35:43.714 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.623919369Z level=info msg="Executing migration" id="create data_keys table" 2023-10-09T10:35:43.714 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.626668179Z level=info msg="Executing migration" id="create secrets table" 2023-10-09T10:35:43.714 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.629422003Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-10-09T10:35:43.714 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.633797552Z level=info msg="Executing migration" id="add name column into data_keys" 2023-10-09T10:35:43.714 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.634740473Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-10-09T10:35:43.714 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.637432086Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-10-09T10:35:43.715 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.641792093Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-10-09T10:35:43.715 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.646142182Z level=info msg="Executing migration" id="create kv_store table v1" 2023-10-09T10:35:43.715 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.646570833Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-10-09T10:35:43.715 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.649454449Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-10-09T10:35:43.715 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.651944343Z level=info msg="Executing migration" id="create permission table" 2023-10-09T10:35:43.715 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.654652078Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-10-09T10:35:43.715 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.6551289Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-10-09T10:35:43.715 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.657961213Z level=info msg="Executing migration" id="create role table" 2023-10-09T10:35:43.715 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.66040137Z level=info msg="Executing migration" id="add column display_name" 2023-10-09T10:35:43.715 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.663074586Z level=info msg="Executing migration" id="add column group_name" 2023-10-09T10:35:43.715 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.664095022Z level=info msg="Executing migration" id="add index role.org_id" 2023-10-09T10:35:43.716 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.664884325Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-10-09T10:35:43.716 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.665799329Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-10-09T10:35:43.716 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.666461004Z level=info msg="Executing migration" id="create team role table" 2023-10-09T10:35:43.716 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.667194477Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-10-09T10:35:43.716 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.668149661Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-10-09T10:35:43.716 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.668788167Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-10-09T10:35:43.716 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.66957249Z level=info msg="Executing migration" id="create user role table" 2023-10-09T10:35:43.716 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.669980969Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-10-09T10:35:43.716 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.670879512Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-10-09T10:35:43.716 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.671579615Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-10-09T10:35:43.716 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.672528234Z level=info msg="Executing migration" id="create builtin role table" 2023-10-09T10:35:43.716 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.673128054Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-10-09T10:35:43.717 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.673923999Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-10-09T10:35:43.717 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.674693228Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-10-09T10:35:43.717 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.675707894Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-10-09T10:35:43.717 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.676234834Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-10-09T10:35:43.718 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.676990152Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-10-09T10:35:43.718 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.677431702Z level=info msg="Executing migration" id="add unique index role.uid" 2023-10-09T10:35:43.718 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.677902904Z level=info msg="Executing migration" id="create seed assignment table" 2023-10-09T10:35:43.718 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.678296304Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-10-09T10:35:43.718 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.678752109Z level=info msg="Executing migration" id="add column hidden to role table" 2023-10-09T10:35:43.718 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.679751749Z level=info msg="Executing migration" id="create query_history table v1" 2023-10-09T10:35:43.718 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.680167912Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-10-09T10:35:43.718 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.68062774Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-10-09T10:35:43.718 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.680930113Z level=info msg="Executing migration" id="teams permissions migration" 2023-10-09T10:35:43.718 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.681311016Z level=info msg="Executing migration" id="dashboard permissions" 2023-10-09T10:35:43.718 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.681791317Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-10-09T10:35:43.718 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.682170394Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-10-09T10:35:43.719 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.682459407Z level=info msg="Executing migration" id="alerting notification permissions" 2023-10-09T10:35:43.719 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.682840763Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-10-09T10:35:43.719 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.683263462Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-10-09T10:35:43.719 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.683719324Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-10-09T10:35:43.719 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.68472081Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-10-09T10:35:43.719 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.685034858Z level=info msg="Executing migration" id="create correlation table v1" 2023-10-09T10:35:43.719 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.685459459Z level=info msg="Executing migration" id="add index correlations.uid" 2023-10-09T10:35:43.719 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.685932049Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-10-09T10:35:43.719 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.686365791Z level=info msg="Executing migration" id="add correlation config column" 2023-10-09T10:35:43.719 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.687368321Z level=info msg="Executing migration" id="create entity_events table" 2023-10-09T10:35:43.719 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.687821363Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-10-09T10:35:43.719 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.688590111Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-10-09T10:35:43.719 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.688685449Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-10-09T10:35:43.720 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.688984025Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T10:35:43.720 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.689070421Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T10:35:43.720 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.689322654Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-10-09T10:35:43.720 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.689767427Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-10-09T10:35:43.720 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.69020109Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-10-09T10:35:43.720 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.690649667Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T10:35:43.720 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.691106699Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-10-09T10:35:43.720 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.691503712Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-10-09T10:35:43.720 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.691937395Z level=info msg="Executing migration" id="Drop public config table" 2023-10-09T10:35:43.720 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.692330936Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-10-09T10:35:43.720 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.692779868Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-10-09T10:35:43.720 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.6932184Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-10-09T10:35:43.720 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.693700553Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-10-09T10:35:43.721 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.694147966Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-10-09T10:35:43.721 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.697180722Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-10-09T10:35:43.721 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.698228942Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-10-09T10:35:43.721 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.699367601Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-10-09T10:35:43.721 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.699712957Z level=info msg="Executing migration" id="add share column" 2023-10-09T10:35:43.721 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.700767298Z level=info msg="Executing migration" id="create default alerting folders" 2023-10-09T10:35:43.721 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.70121217Z level=info msg="Executing migration" id="create file table" 2023-10-09T10:35:43.721 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.701677292Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-10-09T10:35:43.721 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.702110851Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-10-09T10:35:43.721 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.702596291Z level=info msg="Executing migration" id="create file_meta table" 2023-10-09T10:35:43.721 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.703419536Z level=info msg="Executing migration" id="file table idx: path key" 2023-10-09T10:35:43.721 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.704338208Z level=info msg="Executing migration" id="set path collation in file table" 2023-10-09T10:35:43.722 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.70477937Z level=info msg="Executing migration" id="managed permissions migration" 2023-10-09T10:35:43.722 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.705583178Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-10-09T10:35:43.722 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.705930354Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-10-09T10:35:43.722 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.706398458Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-10-09T10:35:43.771 INFO:teuthology.orchestra.run.smithi125.stdout:128849018901 2023-10-09T10:35:43.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:43 smithi125 ceph-mon[112589]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:43.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:43 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:43.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:43 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:43.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:43 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:43.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:43 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:43.800 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:43 smithi125 ceph-mon[127026]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:43.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:43 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:43.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:43 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:43.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:43 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:43.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:43 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:43.970 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.707451859Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-10-09T10:35:43.970 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.707801304Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-10-09T10:35:43.970 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.708750341Z level=info msg="Executing migration" id="update group index for alert rules" 2023-10-09T10:35:43.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.709492932Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-10-09T10:35:43.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.710255282Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-10-09T10:35:43.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.711099139Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-10-09T10:35:43.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.712186379Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-10-09T10:35:43.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.713237627Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-10-09T10:35:43.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.714085892Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-10-09T10:35:43.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.72420488Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-10-09T10:35:43.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.726992982Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-10-09T10:35:43.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.729679273Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-10-09T10:35:43.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.733121457Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-10-09T10:35:43.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.735852522Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-10-09T10:35:43.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.738447564Z level=info msg="Executing migration" id="create folder table" 2023-10-09T10:35:43.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.741071024Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-10-09T10:35:43.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.743900927Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-10-09T10:35:43.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.746591412Z level=info msg="Executing migration" id="Update folder title length" 2023-10-09T10:35:43.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.749455447Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-10-09T10:35:43.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=migrator t=2023-10-09T10:35:43.752196485Z level=info msg="migrations completed" performed=485 skipped=0 duration=657.211492ms 2023-10-09T10:35:43.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=sqlstore t=2023-10-09T10:35:43.752657412Z level=info msg="Created default organization" 2023-10-09T10:35:43.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=plugin.loader t=2023-10-09T10:35:43.811242402Z level=info msg="Plugin registered" pluginID=input 2023-10-09T10:35:43.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=plugin.loader t=2023-10-09T10:35:43.83992767Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-10-09T10:35:43.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=plugin.loader t=2023-10-09T10:35:43.839958271Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-10-09T10:35:43.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=secrets t=2023-10-09T10:35:43.839992189Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-10-09T10:35:43.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=query_data t=2023-10-09T10:35:43.841714725Z level=info msg="Query Service initialization" 2023-10-09T10:35:43.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=live.push_http t=2023-10-09T10:35:43.844044201Z level=info msg="Live Push Gateway initialization" 2023-10-09T10:35:44.555 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574879 got 55834574880 for osd.1 2023-10-09T10:35:44.555 DEBUG:teuthology.parallel:result is None 2023-10-09T10:35:44.688 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:44 smithi125 ceph-mon[112589]: Deploying daemon node-exporter.a on smithi125 2023-10-09T10:35:44.688 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:44 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3311427584' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T10:35:44.688 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:44 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1014011223' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T10:35:44.688 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:44 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3147415659' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T10:35:44.688 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:44 smithi125 ceph-mon[127026]: Deploying daemon node-exporter.a on smithi125 2023-10-09T10:35:44.689 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:44 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3311427584' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T10:35:44.689 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:44 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/1014011223' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T10:35:44.689 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:44 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3147415659' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T10:35:44.711 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182424 got 107374182425 for osd.3 2023-10-09T10:35:44.712 DEBUG:teuthology.parallel:result is None 2023-10-09T10:35:44.899 INFO:teuthology.orchestra.run.smithi125.stdout:158913789970 2023-10-09T10:35:44.921 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018899 got 128849018901 for osd.4 2023-10-09T10:35:44.921 DEBUG:teuthology.parallel:result is None 2023-10-09T10:35:44.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-mon[117223]: Deploying daemon node-exporter.a on smithi125 2023-10-09T10:35:44.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3311427584' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T10:35:44.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1014011223' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T10:35:44.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3147415659' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T10:35:44.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=infra.usagestats.collector t=2023-10-09T10:35:44.576354118Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-10-09T10:35:44.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=provisioning.datasources t=2023-10-09T10:35:44.576693844Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-10-09T10:35:44.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=provisioning.datasources t=2023-10-09T10:35:44.609338587Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T10:35:44.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=provisioning.datasources t=2023-10-09T10:35:44.641682166Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-10-09T10:35:44.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=provisioning.alerting t=2023-10-09T10:35:44.66315985Z level=info msg="starting to provision alerting" 2023-10-09T10:35:44.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=provisioning.alerting t=2023-10-09T10:35:44.663178647Z level=info msg="finished to provision alerting" 2023-10-09T10:35:44.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=ngalert.state.manager t=2023-10-09T10:35:44.663331485Z level=info msg="Warming state cache for startup" 2023-10-09T10:35:44.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=grafanaStorageLogger t=2023-10-09T10:35:44.663352596Z level=info msg="storage starting" 2023-10-09T10:35:44.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=http.server t=2023-10-09T10:35:44.66513288Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T10:35:44.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=sqlstore.transactions t=2023-10-09T10:35:44.674519478Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T10:35:44.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=ngalert.state.manager t=2023-10-09T10:35:44.67454616Z level=info msg="State cache has been initialized" states=0 duration=11.213551ms 2023-10-09T10:35:44.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=ngalert.multiorg.alertmanager t=2023-10-09T10:35:44.674583594Z level=info msg="starting MultiOrg Alertmanager" 2023-10-09T10:35:44.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=ticker t=2023-10-09T10:35:44.674616978Z level=info msg=starting first_tick=2023-10-09T10:35:50Z 2023-10-09T10:35:44.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=sqlstore.transactions t=2023-10-09T10:35:44.719460429Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T10:35:44.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:35:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[153311]: logger=sqlstore.transactions t=2023-10-09T10:35:44.72998309Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2023-10-09T10:35:45.439 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913789969 got 158913789970 for osd.5 2023-10-09T10:35:45.439 DEBUG:teuthology.parallel:result is None 2023-10-09T10:35:45.555 INFO:teuthology.orchestra.run.smithi125.stdout:77309411354 2023-10-09T10:35:45.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:45 smithi125 ceph-mon[112589]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:45.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:45 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3872077028' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T10:35:45.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:45 smithi125 ceph-mon[127026]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:45.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:45 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3872077028' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T10:35:45.821 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:45 smithi125 bash[167645]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-10-09T10:35:45.942 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411353 got 77309411354 for osd.2 2023-10-09T10:35:45.942 DEBUG:teuthology.parallel:result is None 2023-10-09T10:35:45.942 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T10:35:45.942 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:35:45.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:45 smithi155 ceph-mon[117223]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:45.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:45 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3872077028' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T10:35:46.320 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:45 smithi125 bash[167645]: Getting image source signatures 2023-10-09T10:35:46.320 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:45 smithi125 bash[167645]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-10-09T10:35:46.320 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:45 smithi125 bash[167645]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-10-09T10:35:46.321 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:45 smithi125 bash[167645]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-10-09T10:35:46.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:46 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3849233538' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T10:35:46.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:46 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:46.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:46 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3849233538' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T10:35:46.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:46 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:46.964 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:46.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:46 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3849233538' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T10:35:46.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:46 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:47.307 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:46 smithi125 bash[167645]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-10-09T10:35:47.307 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 bash[167645]: Writing manifest to image destination 2023-10-09T10:35:47.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-mon[112589]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:47.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-mon[127026]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:47.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:47 smithi155 ceph-mon[117223]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:48.071 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.907Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T10:35:48.071 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.907Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T10:35:48.071 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.908Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T10:35:48.071 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.908Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T10:35:48.071 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.908Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T10:35:48.071 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.908Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T10:35:48.071 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T10:35:48.071 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T10:35:48.071 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T10:35:48.071 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T10:35:48.072 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T10:35:48.072 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T10:35:48.072 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T10:35:48.072 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T10:35:48.072 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T10:35:48.072 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T10:35:48.072 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T10:35:48.072 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T10:35:48.072 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T10:35:48.072 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T10:35:48.072 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T10:35:48.072 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T10:35:48.072 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T10:35:48.073 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T10:35:48.073 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T10:35:48.073 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T10:35:48.073 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T10:35:48.073 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T10:35:48.073 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T10:35:48.073 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T10:35:48.073 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T10:35:48.073 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T10:35:48.073 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T10:35:48.073 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T10:35:48.073 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T10:35:48.074 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T10:35:48.074 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T10:35:48.074 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T10:35:48.074 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T10:35:48.074 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T10:35:48.074 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T10:35:48.074 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T10:35:48.074 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T10:35:48.074 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T10:35:48.074 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T10:35:48.074 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T10:35:48.074 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T10:35:48.074 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T10:35:48.075 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T10:35:48.075 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T10:35:48.075 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T10:35:48.075 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T10:35:48.075 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[167834]: ts=2023-10-09T10:35:47.909Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T10:35:48.075 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 bash[167645]: 28a7f76287d680eb5dc16124fc70e8deb63fdd86aaf6aa489b41471a7ae436cc 2023-10-09T10:35:48.075 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:35:47 smithi125 systemd[1]: Started Ceph node-exporter.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:35:48.342 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:35:48.344 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:35:48.721 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:48 smithi155 systemd[1]: Starting Ceph node-exporter.b for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:35:48.750 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2023-10-09T10:35:46.628488+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2384604,"kb_used_data":6300,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747576612,"statfs":{"total":767960285184,"available":765518450688,"internally_reserved":0,"allocated":6451200,"data_stored":3719822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002207"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.774840+0000","last_change":"2023-10-09T10:35:20.619783+0000","last_active":"2023-10-09T10:35:21.774840+0000","last_peered":"2023-10-09T10:35:21.774840+0000","last_clean":"2023-10-09T10:35:21.774840+0000","last_became_active":"2023-10-09T10:35:20.619441+0000","last_became_peered":"2023-10-09T10:35:20.619441+0000","last_unstale":"2023-10-09T10:35:21.774840+0000","last_undegraded":"2023-10-09T10:35:21.774840+0000","last_fullsized":"2023-10-09T10:35:21.774840+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:00.454996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.613352+0000","last_change":"2023-10-09T10:35:20.621958+0000","last_active":"2023-10-09T10:35:21.613352+0000","last_peered":"2023-10-09T10:35:21.613352+0000","last_clean":"2023-10-09T10:35:21.613352+0000","last_became_active":"2023-10-09T10:35:20.621656+0000","last_became_peered":"2023-10-09T10:35:20.621656+0000","last_unstale":"2023-10-09T10:35:21.613352+0000","last_undegraded":"2023-10-09T10:35:21.613352+0000","last_fullsized":"2023-10-09T10:35:21.613352+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:32.121613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.774648+0000","last_change":"2023-10-09T10:35:20.619239+0000","last_active":"2023-10-09T10:35:21.774648+0000","last_peered":"2023-10-09T10:35:21.774648+0000","last_clean":"2023-10-09T10:35:21.774648+0000","last_became_active":"2023-10-09T10:35:20.618892+0000","last_became_peered":"2023-10-09T10:35:20.618892+0000","last_unstale":"2023-10-09T10:35:21.774648+0000","last_undegraded":"2023-10-09T10:35:21.774648+0000","last_fullsized":"2023-10-09T10:35:21.774648+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:55.019433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.774611+0000","last_change":"2023-10-09T10:35:20.619992+0000","last_active":"2023-10-09T10:35:21.774611+0000","last_peered":"2023-10-09T10:35:21.774611+0000","last_clean":"2023-10-09T10:35:21.774611+0000","last_became_active":"2023-10-09T10:35:20.619719+0000","last_became_peered":"2023-10-09T10:35:20.619719+0000","last_unstale":"2023-10-09T10:35:21.774611+0000","last_undegraded":"2023-10-09T10:35:21.774611+0000","last_fullsized":"2023-10-09T10:35:21.774611+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:31.964380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"57'128","reported_seq":270,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.774613+0000","last_change":"2023-10-09T10:35:01.491534+0000","last_active":"2023-10-09T10:35:21.774613+0000","last_peered":"2023-10-09T10:35:21.774613+0000","last_clean":"2023-10-09T10:35:21.774613+0000","last_became_active":"2023-10-09T10:35:01.491048+0000","last_became_peered":"2023-10-09T10:35:01.491048+0000","last_unstale":"2023-10-09T10:35:21.774613+0000","last_undegraded":"2023-10-09T10:35:21.774613+0000","last_fullsized":"2023-10-09T10:35:21.774613+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:00:35.672924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.613436+0000","last_change":"2023-10-09T10:35:20.622020+0000","last_active":"2023-10-09T10:35:21.613436+0000","last_peered":"2023-10-09T10:35:21.613436+0000","last_clean":"2023-10-09T10:35:21.613436+0000","last_became_active":"2023-10-09T10:35:20.621703+0000","last_became_peered":"2023-10-09T10:35:20.621703+0000","last_unstale":"2023-10-09T10:35:21.613436+0000","last_undegraded":"2023-10-09T10:35:21.613436+0000","last_fullsized":"2023-10-09T10:35:21.613436+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:59:34.492019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.774456+0000","last_change":"2023-10-09T10:35:20.625994+0000","last_active":"2023-10-09T10:35:21.774456+0000","last_peered":"2023-10-09T10:35:21.774456+0000","last_clean":"2023-10-09T10:35:21.774456+0000","last_became_active":"2023-10-09T10:35:20.625638+0000","last_became_peered":"2023-10-09T10:35:20.625638+0000","last_unstale":"2023-10-09T10:35:21.774456+0000","last_undegraded":"2023-10-09T10:35:21.774456+0000","last_fullsized":"2023-10-09T10:35:21.774456+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:26.622626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.613436+0000","last_change":"2023-10-09T10:35:20.621825+0000","last_active":"2023-10-09T10:35:21.613436+0000","last_peered":"2023-10-09T10:35:21.613436+0000","last_clean":"2023-10-09T10:35:21.613436+0000","last_became_active":"2023-10-09T10:35:20.621526+0000","last_became_peered":"2023-10-09T10:35:20.621526+0000","last_unstale":"2023-10-09T10:35:21.613436+0000","last_undegraded":"2023-10-09T10:35:21.613436+0000","last_fullsized":"2023-10-09T10:35:21.613436+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:59.814738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.613270+0000","last_change":"2023-10-09T10:35:20.621757+0000","last_active":"2023-10-09T10:35:21.613270+0000","last_peered":"2023-10-09T10:35:21.613270+0000","last_clean":"2023-10-09T10:35:21.613270+0000","last_became_active":"2023-10-09T10:35:20.621505+0000","last_became_peered":"2023-10-09T10:35:20.621505+0000","last_unstale":"2023-10-09T10:35:21.613270+0000","last_undegraded":"2023-10-09T10:35:21.613270+0000","last_fullsized":"2023-10-09T10:35:21.613270+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:35.203999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332108,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298436,"kb_used_data":1148,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446716,"statfs":{"total":95995035648,"available":95689437184,"internally_reserved":0,"allocated":1175552,"data_stored":833789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41599999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51500000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45900000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.499}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48299999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54900000000000004}]}]},{"osd":6,"up_from":44,"seq":188978561039,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298436,"kb_used_data":1148,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446716,"statfs":{"total":95995035648,"available":95689437184,"internally_reserved":0,"allocated":1175552,"data_stored":833789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.46100000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.41599999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.435}]},{"osd":3,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.49099999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.40200000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.377},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.505}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48299999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574881,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297856,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447296,"statfs":{"total":95995035648,"available":95690031104,"internally_reserved":0,"allocated":581632,"data_stored":243479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:18 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.371,"15min":0.36499999999999999},"min":{"1min":0.25800000000000001,"5min":0.247,"15min":0.247},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.38100000000000001,"15min":0.375},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.26700000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:35:35 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.42899999999999999,"15min":0.435},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.54400000000000004,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.435,"15min":0.438},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.38600000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:34:54 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.40100000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:35:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.42299999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.314},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.41199999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:35:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.44400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47999999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738402,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298440,"kb_used_data":1152,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446712,"statfs":{"total":95995035648,"available":95689433088,"internally_reserved":0,"allocated":1179648,"data_stored":834112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.36699999999999999,"15min":0.36299999999999999},"min":{"1min":0.26400000000000001,"5min":0.222,"15min":0.222},"max":{"1min":0.496,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.254},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.40100000000000002,"15min":0.39000000000000001},"min":{"1min":0.34100000000000003,"5min":0.221,"15min":0.221},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.28399999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.29999999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.42199999999999999,"15min":0.42499999999999999},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.23999999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.312},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.318}]},{"osd":4,"last update":"Mon Oct 9 10:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.64100000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.622}]},{"osd":6,"last update":"Mon Oct 9 10:35:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.58999999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65400000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411355,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297860,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447292,"statfs":{"total":95995035648,"available":95690027008,"internally_reserved":0,"allocated":585728,"data_stored":243744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.36099999999999999,"15min":0.35899999999999999},"min":{"1min":0.255,"5min":0.22,"15min":0.22},"max":{"1min":0.57799999999999996,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.72999999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.38900000000000001,"15min":0.38700000000000001},"min":{"1min":0.255,"5min":0.246,"15min":0.246},"max":{"1min":0.55800000000000005,"5min":0.63,"15min":0.63},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.41099999999999998,"15min":0.40899999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.52000000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.63400000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.76400000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:35:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.85099999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.747}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81299999999999994}]}]},{"osd":3,"up_from":25,"seq":107374182426,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297860,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447292,"statfs":{"total":95995035648,"available":95690027008,"internally_reserved":0,"allocated":585728,"data_stored":243744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.313}]},{"osd":1,"last update":"Mon Oct 9 10:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.246},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.33100000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.29199999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:35:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.72199999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:35:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":2.9580000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":3.9399999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.8380000000000001,"5min":1.8380000000000001,"15min":1.8380000000000001},"last":1.8380000000000001},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":1.022}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.8980000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018902,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297860,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447292,"statfs":{"total":95995035648,"available":95690027008,"internally_reserved":0,"allocated":585728,"data_stored":243744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.40799999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.498},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.51000000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.48899999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.45000000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:35:23 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.39600000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.54900000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56499999999999995}]}]},{"osd":5,"up_from":37,"seq":158913789971,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297856,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447296,"statfs":{"total":95995035648,"available":95690031104,"internally_reserved":0,"allocated":581632,"data_stored":243421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.376}]},{"osd":1,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.47999999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.58499999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.53500000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.505,"5min":0.505,"15min":0.505},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.62}]},{"osd":6,"last update":"Mon Oct 9 10:35:43 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.55500000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68200000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:35:48.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:35:49.016 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:49.216 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:48 smithi155 bash[153581]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-10-09T10:35:49.216 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:49 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:49.216 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:49 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:49.217 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:49 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:49.217 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:49 smithi155 ceph-mon[117223]: Deploying daemon node-exporter.b on smithi155 2023-10-09T10:35:49.217 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:49 smithi155 ceph-mon[117223]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:35:49.217 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:49 smithi155 ceph-mon[117223]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:49.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:49 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:49.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:49 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:49.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:49 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:49.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:49 smithi125 ceph-mon[112589]: Deploying daemon node-exporter.b on smithi155 2023-10-09T10:35:49.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:49 smithi125 ceph-mon[112589]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:35:49.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:49 smithi125 ceph-mon[112589]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:49.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:49 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:49.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:49 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:49.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:49 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:49.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:49 smithi125 ceph-mon[127026]: Deploying daemon node-exporter.b on smithi155 2023-10-09T10:35:49.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:49 smithi125 ceph-mon[127026]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:35:49.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:49 smithi125 ceph-mon[127026]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:49.470 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:49 smithi155 bash[153581]: Getting image source signatures 2023-10-09T10:35:49.470 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:49 smithi155 bash[153581]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-10-09T10:35:49.470 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:49 smithi155 bash[153581]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-10-09T10:35:49.470 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:49 smithi155 bash[153581]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-10-09T10:35:50.444 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:35:50.445 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:35:50.470 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 bash[153581]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-10-09T10:35:50.470 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 bash[153581]: Writing manifest to image destination 2023-10-09T10:35:50.836 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2023-10-09T10:35:48.628928+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2384604,"kb_used_data":6300,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747576612,"statfs":{"total":767960285184,"available":765518450688,"internally_reserved":0,"allocated":6451200,"data_stored":3719822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.774840+0000","last_change":"2023-10-09T10:35:20.619783+0000","last_active":"2023-10-09T10:35:21.774840+0000","last_peered":"2023-10-09T10:35:21.774840+0000","last_clean":"2023-10-09T10:35:21.774840+0000","last_became_active":"2023-10-09T10:35:20.619441+0000","last_became_peered":"2023-10-09T10:35:20.619441+0000","last_unstale":"2023-10-09T10:35:21.774840+0000","last_undegraded":"2023-10-09T10:35:21.774840+0000","last_fullsized":"2023-10-09T10:35:21.774840+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:00.454996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.613352+0000","last_change":"2023-10-09T10:35:20.621958+0000","last_active":"2023-10-09T10:35:21.613352+0000","last_peered":"2023-10-09T10:35:21.613352+0000","last_clean":"2023-10-09T10:35:21.613352+0000","last_became_active":"2023-10-09T10:35:20.621656+0000","last_became_peered":"2023-10-09T10:35:20.621656+0000","last_unstale":"2023-10-09T10:35:21.613352+0000","last_undegraded":"2023-10-09T10:35:21.613352+0000","last_fullsized":"2023-10-09T10:35:21.613352+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:32.121613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.774648+0000","last_change":"2023-10-09T10:35:20.619239+0000","last_active":"2023-10-09T10:35:21.774648+0000","last_peered":"2023-10-09T10:35:21.774648+0000","last_clean":"2023-10-09T10:35:21.774648+0000","last_became_active":"2023-10-09T10:35:20.618892+0000","last_became_peered":"2023-10-09T10:35:20.618892+0000","last_unstale":"2023-10-09T10:35:21.774648+0000","last_undegraded":"2023-10-09T10:35:21.774648+0000","last_fullsized":"2023-10-09T10:35:21.774648+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:55.019433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.774611+0000","last_change":"2023-10-09T10:35:20.619992+0000","last_active":"2023-10-09T10:35:21.774611+0000","last_peered":"2023-10-09T10:35:21.774611+0000","last_clean":"2023-10-09T10:35:21.774611+0000","last_became_active":"2023-10-09T10:35:20.619719+0000","last_became_peered":"2023-10-09T10:35:20.619719+0000","last_unstale":"2023-10-09T10:35:21.774611+0000","last_undegraded":"2023-10-09T10:35:21.774611+0000","last_fullsized":"2023-10-09T10:35:21.774611+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:31.964380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"57'128","reported_seq":270,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.774613+0000","last_change":"2023-10-09T10:35:01.491534+0000","last_active":"2023-10-09T10:35:21.774613+0000","last_peered":"2023-10-09T10:35:21.774613+0000","last_clean":"2023-10-09T10:35:21.774613+0000","last_became_active":"2023-10-09T10:35:01.491048+0000","last_became_peered":"2023-10-09T10:35:01.491048+0000","last_unstale":"2023-10-09T10:35:21.774613+0000","last_undegraded":"2023-10-09T10:35:21.774613+0000","last_fullsized":"2023-10-09T10:35:21.774613+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:00:35.672924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.613436+0000","last_change":"2023-10-09T10:35:20.622020+0000","last_active":"2023-10-09T10:35:21.613436+0000","last_peered":"2023-10-09T10:35:21.613436+0000","last_clean":"2023-10-09T10:35:21.613436+0000","last_became_active":"2023-10-09T10:35:20.621703+0000","last_became_peered":"2023-10-09T10:35:20.621703+0000","last_unstale":"2023-10-09T10:35:21.613436+0000","last_undegraded":"2023-10-09T10:35:21.613436+0000","last_fullsized":"2023-10-09T10:35:21.613436+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:59:34.492019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.774456+0000","last_change":"2023-10-09T10:35:20.625994+0000","last_active":"2023-10-09T10:35:21.774456+0000","last_peered":"2023-10-09T10:35:21.774456+0000","last_clean":"2023-10-09T10:35:21.774456+0000","last_became_active":"2023-10-09T10:35:20.625638+0000","last_became_peered":"2023-10-09T10:35:20.625638+0000","last_unstale":"2023-10-09T10:35:21.774456+0000","last_undegraded":"2023-10-09T10:35:21.774456+0000","last_fullsized":"2023-10-09T10:35:21.774456+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:26.622626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.613436+0000","last_change":"2023-10-09T10:35:20.621825+0000","last_active":"2023-10-09T10:35:21.613436+0000","last_peered":"2023-10-09T10:35:21.613436+0000","last_clean":"2023-10-09T10:35:21.613436+0000","last_became_active":"2023-10-09T10:35:20.621526+0000","last_became_peered":"2023-10-09T10:35:20.621526+0000","last_unstale":"2023-10-09T10:35:21.613436+0000","last_undegraded":"2023-10-09T10:35:21.613436+0000","last_fullsized":"2023-10-09T10:35:21.613436+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:59.814738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-10-09T10:35:21.613270+0000","last_change":"2023-10-09T10:35:20.621757+0000","last_active":"2023-10-09T10:35:21.613270+0000","last_peered":"2023-10-09T10:35:21.613270+0000","last_clean":"2023-10-09T10:35:21.613270+0000","last_became_active":"2023-10-09T10:35:20.621505+0000","last_became_peered":"2023-10-09T10:35:20.621505+0000","last_unstale":"2023-10-09T10:35:21.613270+0000","last_undegraded":"2023-10-09T10:35:21.613270+0000","last_fullsized":"2023-10-09T10:35:21.613270+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:35.203999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332108,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298436,"kb_used_data":1148,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446716,"statfs":{"total":95995035648,"available":95689437184,"internally_reserved":0,"allocated":1175552,"data_stored":833789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41599999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51500000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45900000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.499}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48299999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54900000000000004}]}]},{"osd":6,"up_from":44,"seq":188978561039,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298436,"kb_used_data":1148,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446716,"statfs":{"total":95995035648,"available":95689437184,"internally_reserved":0,"allocated":1175552,"data_stored":833789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.46100000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.41599999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.435}]},{"osd":3,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.49099999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.40200000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.377},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.505}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48299999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574881,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297856,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447296,"statfs":{"total":95995035648,"available":95690031104,"internally_reserved":0,"allocated":581632,"data_stored":243479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:18 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.371,"15min":0.36499999999999999},"min":{"1min":0.25800000000000001,"5min":0.247,"15min":0.247},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.38100000000000001,"15min":0.375},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.26700000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:35:35 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.42899999999999999,"15min":0.435},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.54400000000000004,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.435,"15min":0.438},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.38600000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:34:54 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.40100000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:35:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.42299999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.314},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.41199999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:35:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.44400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47999999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738402,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298440,"kb_used_data":1152,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446712,"statfs":{"total":95995035648,"available":95689433088,"internally_reserved":0,"allocated":1179648,"data_stored":834112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.36699999999999999,"15min":0.36299999999999999},"min":{"1min":0.26400000000000001,"5min":0.222,"15min":0.222},"max":{"1min":0.496,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.254},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.40100000000000002,"15min":0.39000000000000001},"min":{"1min":0.34100000000000003,"5min":0.221,"15min":0.221},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.28399999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.29999999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.42199999999999999,"15min":0.42499999999999999},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.23999999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.312},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.318}]},{"osd":4,"last update":"Mon Oct 9 10:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.64100000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.622}]},{"osd":6,"last update":"Mon Oct 9 10:35:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.58999999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65400000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411355,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297860,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447292,"statfs":{"total":95995035648,"available":95690027008,"internally_reserved":0,"allocated":585728,"data_stored":243744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.36099999999999999,"15min":0.35899999999999999},"min":{"1min":0.255,"5min":0.22,"15min":0.22},"max":{"1min":0.57799999999999996,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.72999999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.38900000000000001,"15min":0.38700000000000001},"min":{"1min":0.255,"5min":0.246,"15min":0.246},"max":{"1min":0.55800000000000005,"5min":0.63,"15min":0.63},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.41099999999999998,"15min":0.40899999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.52000000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.63400000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.76400000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:35:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.85099999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.747}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81299999999999994}]}]},{"osd":3,"up_from":25,"seq":107374182426,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297860,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447292,"statfs":{"total":95995035648,"available":95690027008,"internally_reserved":0,"allocated":585728,"data_stored":243744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.313}]},{"osd":1,"last update":"Mon Oct 9 10:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.246},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.33100000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.29199999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:35:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.72199999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:35:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":2.9580000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":3.9399999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.8380000000000001,"5min":1.8380000000000001,"15min":1.8380000000000001},"last":1.8380000000000001},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":1.022}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.8980000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018902,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297860,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447292,"statfs":{"total":95995035648,"available":95690027008,"internally_reserved":0,"allocated":585728,"data_stored":243744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.40799999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.498},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.51000000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.48899999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.45000000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:35:23 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.39600000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.54900000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56499999999999995}]}]},{"osd":5,"up_from":37,"seq":158913789971,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297856,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447296,"statfs":{"total":95995035648,"available":95690031104,"internally_reserved":0,"allocated":581632,"data_stored":243421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.376}]},{"osd":1,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.47999999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.58499999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.53500000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.505,"5min":0.505,"15min":0.505},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.62}]},{"osd":6,"last update":"Mon Oct 9 10:35:43 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.55500000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68200000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:35:50.838 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T10:35:50.838 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T10:35:50.838 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T10:35:50.838 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T10:35:51.114 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:51.220 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.748Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T10:35:51.220 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.748Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T10:35:51.221 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.749Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T10:35:51.221 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.749Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T10:35:51.221 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.749Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T10:35:51.221 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.749Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T10:35:51.221 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T10:35:51.221 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T10:35:51.221 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T10:35:51.221 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T10:35:51.221 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T10:35:51.221 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T10:35:51.221 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T10:35:51.222 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T10:35:51.222 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T10:35:51.222 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T10:35:51.222 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T10:35:51.222 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T10:35:51.222 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T10:35:51.222 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T10:35:51.222 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T10:35:51.222 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T10:35:51.222 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T10:35:51.222 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T10:35:51.223 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T10:35:51.223 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T10:35:51.223 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T10:35:51.223 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T10:35:51.223 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T10:35:51.223 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T10:35:51.223 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T10:35:51.223 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T10:35:51.223 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T10:35:51.223 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T10:35:51.223 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T10:35:51.224 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T10:35:51.224 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T10:35:51.224 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T10:35:51.224 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T10:35:51.224 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T10:35:51.224 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T10:35:51.224 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T10:35:51.224 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T10:35:51.224 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T10:35:51.224 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T10:35:51.224 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T10:35:51.224 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T10:35:51.225 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T10:35:51.225 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T10:35:51.225 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T10:35:51.225 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T10:35:51.225 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T10:35:51.225 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[153632]: ts=2023-10-09T10:35:50.750Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T10:35:51.225 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 bash[153581]: 2e7ced8915fc35c0e7857c2261fecf5ade3c0fa6ec9192fac6d6aabe39cc7842 2023-10-09T10:35:51.225 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:35:50 smithi155 systemd[1]: Started Ceph node-exporter.b for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:35:51.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:51 smithi155 ceph-mon[117223]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:35:51.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:51 smithi155 ceph-mon[117223]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:51.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:51 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:35:51.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:51 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:51.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:51 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:51.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:51 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:51.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:51 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:51.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:51 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:51.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:51 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:35:52.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[112589]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:35:52.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[112589]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:52.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:35:52.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:52.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:52.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:52.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:52.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:52.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:35:52.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[127026]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:35:52.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[127026]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:52.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:35:52.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:52.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:52.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:52.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:52.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:52.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:51 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:35:52.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:35:51 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:35:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T10:35:52.714 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:35:52.714 INFO:teuthology.orchestra.run.smithi125.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T10:35:53.215 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T10:35:53.215 INFO:tasks.cephadm:Setup complete, yielding 2023-10-09T10:35:53.216 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-09T10:35:53.225 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi125.front.sepia.ceph.com 2023-10-09T10:35:53.226 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-10-09T10:35:53.714 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:53.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:53 smithi125 ceph-mon[112589]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:53.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:53 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3066483715' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T10:35:53.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:53 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:53.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:53 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:53.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:53 smithi125 ceph-mon[127026]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:53.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:53 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3066483715' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T10:35:53.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:53 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:53.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:53 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:53 smithi155 ceph-mon[117223]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:53 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3066483715' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T10:35:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:53 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:53 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:55.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:54 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:55.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:54 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:55.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:54 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:35:55.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:54 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:35:55.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:54 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:55.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:54 smithi125 ceph-mon[112589]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T10:35:55.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:54 smithi125 ceph-mon[112589]: Reconfiguring daemon alertmanager.a on smithi125 2023-10-09T10:35:55.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:54 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:55.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:54 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:55.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:54 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:35:55.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:54 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:35:55.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:54 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:55.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:54 smithi125 ceph-mon[127026]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T10:35:55.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:54 smithi125 ceph-mon[127026]: Reconfiguring daemon alertmanager.a on smithi125 2023-10-09T10:35:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:54 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:54 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:54 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:35:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:54 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:35:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:54 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:55.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:54 smithi155 ceph-mon[117223]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T10:35:55.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:54 smithi155 ceph-mon[117223]: Reconfiguring daemon alertmanager.a on smithi125 2023-10-09T10:35:56.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:56 smithi125 ceph-mon[112589]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:56.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:56 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T10:35:56.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:56 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:56.105 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:56 smithi125 ceph-mon[127026]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:56.105 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:56 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T10:35:56.105 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:56 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:56.106 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:55 smithi125 systemd[1]: Stopping Ceph alertmanager.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:35:56.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:56 smithi155 ceph-mon[117223]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:56.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:56 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T10:35:56.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:56 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:56.570 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:56 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[163900]: ts=2023-10-09T10:35:56.103Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T10:35:56.571 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:56 smithi125 bash[169383]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T10:35:56.903 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:56 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[112566]: 2023-10-09T10:35:56.814+0000 7fe908588700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T10:35:56.903 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:56 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T10:35:56.904 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:56 smithi125 systemd[1]: Stopped Ceph alertmanager.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:35:56.904 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:56 smithi125 systemd[1]: Starting Ceph alertmanager.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:35:57.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[112589]: from='client.24551 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:35:57.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T10:35:57.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[112589]: osdmap e60: 8 total, 8 up, 8 in 2023-10-09T10:35:57.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T10:35:57.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[112589]: pgmap v24: 41 pgs: 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:57.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T10:35:57.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[112589]: osdmap e61: 8 total, 8 up, 8 in 2023-10-09T10:35:57.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T10:35:57.204 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[112589]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T10:35:57.204 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[112589]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T10:35:57.204 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[112589]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T10:35:57.204 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T10:35:57.204 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[112589]: fsmap foo:0 2023-10-09T10:35:57.204 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[112589]: Saving service mds.foo spec with placement count:2 2023-10-09T10:35:57.204 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:57.204 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[127026]: from='client.24551 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:35:57.204 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T10:35:57.204 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[127026]: osdmap e60: 8 total, 8 up, 8 in 2023-10-09T10:35:57.205 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T10:35:57.205 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[127026]: pgmap v24: 41 pgs: 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:57.205 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T10:35:57.205 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[127026]: osdmap e61: 8 total, 8 up, 8 in 2023-10-09T10:35:57.205 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T10:35:57.205 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[127026]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T10:35:57.205 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[127026]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T10:35:57.205 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[127026]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T10:35:57.205 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T10:35:57.205 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[127026]: fsmap foo:0 2023-10-09T10:35:57.205 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[127026]: Saving service mds.foo spec with placement count:2 2023-10-09T10:35:57.205 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:57.249 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-10-09T10:35:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:57 smithi155 ceph-mon[117223]: from='client.24551 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:35:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:57 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T10:35:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:57 smithi155 ceph-mon[117223]: osdmap e60: 8 total, 8 up, 8 in 2023-10-09T10:35:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:57 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T10:35:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:57 smithi155 ceph-mon[117223]: pgmap v24: 41 pgs: 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:57 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T10:35:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:57 smithi155 ceph-mon[117223]: osdmap e61: 8 total, 8 up, 8 in 2023-10-09T10:35:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:57 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T10:35:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:57 smithi155 ceph-mon[117223]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T10:35:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:57 smithi155 ceph-mon[117223]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T10:35:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:57 smithi155 ceph-mon[117223]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T10:35:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:57 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T10:35:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:57 smithi155 ceph-mon[117223]: fsmap foo:0 2023-10-09T10:35:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:57 smithi155 ceph-mon[117223]: Saving service mds.foo spec with placement count:2 2023-10-09T10:35:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:57 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:57.519 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:57.571 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[169522]: ts=2023-10-09T10:35:57.202Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-10-09T10:35:57.571 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[169522]: ts=2023-10-09T10:35:57.202Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-10-09T10:35:57.571 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[169522]: ts=2023-10-09T10:35:57.203Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.125 port=9094 2023-10-09T10:35:57.571 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[169522]: ts=2023-10-09T10:35:57.204Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T10:35:57.571 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:57 smithi125 bash[169500]: 93d5ea37b5d245b6c3567f5b4e322f39c07f8e73a32b88886e6495e3992175f8 2023-10-09T10:35:57.571 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:57 smithi125 systemd[1]: Started Ceph alertmanager.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:35:57.571 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[169522]: ts=2023-10-09T10:35:57.230Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T10:35:57.571 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[169522]: ts=2023-10-09T10:35:57.231Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T10:35:57.571 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[169522]: ts=2023-10-09T10:35:57.234Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T10:35:57.571 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[169522]: ts=2023-10-09T10:35:57.234Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T10:35:58.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:58 smithi125 ceph-mon[127026]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T10:35:58.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:58 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:58.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:58 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:58.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:58 smithi125 ceph-mon[127026]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T10:35:58.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:58 smithi125 ceph-mon[127026]: Reconfiguring daemon prometheus.a on smithi155 2023-10-09T10:35:58.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:58 smithi125 ceph-mon[127026]: osdmap e63: 8 total, 8 up, 8 in 2023-10-09T10:35:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:58 smithi125 ceph-mon[112589]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T10:35:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:58 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:58 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:58 smithi125 ceph-mon[112589]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T10:35:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:58 smithi125 ceph-mon[112589]: Reconfiguring daemon prometheus.a on smithi155 2023-10-09T10:35:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:58 smithi125 ceph-mon[112589]: osdmap e63: 8 total, 8 up, 8 in 2023-10-09T10:35:58.347 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:58 smithi155 ceph-mon[117223]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T10:35:58.347 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:58 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:58.347 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:58 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:35:58.347 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:58 smithi155 ceph-mon[117223]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T10:35:58.347 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:58 smithi155 ceph-mon[117223]: Reconfiguring daemon prometheus.a on smithi155 2023-10-09T10:35:58.347 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:58 smithi155 ceph-mon[117223]: osdmap e63: 8 total, 8 up, 8 in 2023-10-09T10:35:58.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:58 smithi155 systemd[1]: Stopping Ceph prometheus.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:35:59.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-10-09T10:35:59.411 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[150882]: ts=2023-10-09T10:35:59.073Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T10:35:59.411 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[150882]: ts=2023-10-09T10:35:59.073Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T10:35:59.411 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[150882]: ts=2023-10-09T10:35:59.073Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T10:35:59.411 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[150882]: ts=2023-10-09T10:35:59.073Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T10:35:59.411 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[150882]: ts=2023-10-09T10:35:59.073Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T10:35:59.411 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[150882]: ts=2023-10-09T10:35:59.073Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T10:35:59.412 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[150882]: ts=2023-10-09T10:35:59.073Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T10:35:59.412 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[150882]: ts=2023-10-09T10:35:59.073Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T10:35:59.412 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[150882]: ts=2023-10-09T10:35:59.074Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T10:35:59.412 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[150882]: ts=2023-10-09T10:35:59.074Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T10:35:59.412 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[150882]: ts=2023-10-09T10:35:59.074Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T10:35:59.412 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[150882]: ts=2023-10-09T10:35:59.074Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T10:35:59.412 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 bash[154535]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T10:35:59.452 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:35:59.492 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:35:59 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[169522]: ts=2023-10-09T10:35:59.204Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000006163s 2023-10-09T10:35:59.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:59 smithi125 ceph-mon[127026]: pgmap v28: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:59.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:35:59 smithi125 ceph-mon[127026]: osdmap e64: 8 total, 8 up, 8 in 2023-10-09T10:35:59.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-mon[117223]: pgmap v28: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:59.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-mon[117223]: osdmap e64: 8 total, 8 up, 8 in 2023-10-09T10:35:59.721 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T10:35:59.721 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 systemd[1]: Stopped Ceph prometheus.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:35:59.721 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 systemd[1]: Starting Ceph prometheus.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:35:59.776 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:59 smithi125 ceph-mon[112589]: pgmap v28: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:35:59.776 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:35:59 smithi125 ceph-mon[112589]: osdmap e64: 8 total, 8 up, 8 in 2023-10-09T10:36:00.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:59 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:59] ENGINE Bus STOPPING 2023-10-09T10:36:00.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:59 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T10:36:00.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:59 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:59] ENGINE Bus STOPPED 2023-10-09T10:36:00.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:59 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:59] ENGINE Bus STARTING 2023-10-09T10:36:00.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:59 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:59] ENGINE Serving on http://:::9283 2023-10-09T10:36:00.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:59 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:59] ENGINE Bus STARTED 2023-10-09T10:36:00.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:59 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:59] ENGINE Bus STOPPING 2023-10-09T10:36:00.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:59 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T10:36:00.072 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:59 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:59] ENGINE Bus STOPPED 2023-10-09T10:36:00.072 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:35:59 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:35:59] ENGINE Bus STARTING 2023-10-09T10:36:00.220 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 bash[154653]: 55ccec467a9863307948038219698c0db962e7284fd24acf0388ce6472f649af 2023-10-09T10:36:00.220 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.759Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T10:36:00.220 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.759Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T10:36:00.221 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.759Z caller=main.go:570 level=info host_details="(Linux 4.18.0-514.el8.x86_64 #1 SMP Tue Sep 19 13:19:58 UTC 2023 x86_64 smithi155 (none))" 2023-10-09T10:36:00.221 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.759Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T10:36:00.221 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.759Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T10:36:00.221 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.761Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T10:36:00.221 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 systemd[1]: Started Ceph prometheus.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:36:00.221 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.762Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T10:36:00.221 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.764Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T10:36:00.221 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.764Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T10:36:00.221 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.765Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T10:36:00.221 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.765Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.127µs 2023-10-09T10:36:00.221 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.765Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T10:36:00.221 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.765Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-10-09T10:36:00.221 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.765Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-10-09T10:36:00.222 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.765Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.808µs wal_replay_duration=459.317µs wbl_replay_duration=123ns total_replay_duration=520.627µs 2023-10-09T10:36:00.222 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.766Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T10:36:00.222 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.766Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T10:36:00.222 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.766Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T10:36:00.222 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.782Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.057992ms db_storage=775ns remote_storage=1.545µs web_handler=410ns query_engine=606ns scrape=573.336µs scrape_sd=29.787µs notify=49.432µs notify_sd=7.375µs rules=15.179082ms tracing=6.186µs 2023-10-09T10:36:00.222 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.782Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T10:36:00.222 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:35:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:35:59.782Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T10:36:00.493 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:36:00] ENGINE Serving on http://:::9283 2023-10-09T10:36:00.493 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:36:00] ENGINE Bus STARTED 2023-10-09T10:36:00.494 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:36:00] ENGINE Bus STOPPING 2023-10-09T10:36:00.494 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:36:00] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T10:36:00.494 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:36:00] ENGINE Bus STOPPED 2023-10-09T10:36:00.494 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:36:00] ENGINE Bus STARTING 2023-10-09T10:36:00.494 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:36:00] ENGINE Serving on http://:::9283 2023-10-09T10:36:00.494 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:36:00] ENGINE Bus STARTED 2023-10-09T10:36:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T10:36:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T10:36:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T10:36:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi125.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T10:36:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi125.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T10:36:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:36:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:36:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi155.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T10:36:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi155.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T10:36:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T10:36:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T10:36:00.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi155.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T10:36:00.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi155.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T10:36:00.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:00.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:00 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:36:00.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T10:36:00.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:00.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:00.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T10:36:00.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T10:36:00.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi125.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T10:36:00.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi125.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T10:36:00.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:00.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:36:00.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:36:00.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi155.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T10:36:00.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi155.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T10:36:00.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:00.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T10:36:00.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T10:36:00.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi155.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T10:36:00.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi155.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T10:36:00.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:00.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:36:00.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T10:36:00.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:00.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:00.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T10:36:00.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T10:36:00.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi125.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T10:36:00.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi125.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T10:36:00.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:00.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:36:00.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:36:00.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi155.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T10:36:00.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi155.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T10:36:00.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:00.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T10:36:00.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T10:36:00.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi155.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T10:36:00.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi155.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T10:36:00.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:00.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:00 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:36:01.663 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:01 smithi125 ceph-mon[112589]: pgmap v30: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:01.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:01 smithi125 ceph-mon[127026]: pgmap v30: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:01.689 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T10:36:01.698 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T10:36:01.698 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T10:36:01.698 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T10:36:01.717 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:01 smithi155 ceph-mon[117223]: pgmap v30: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:02.086 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:03.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:02 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:03.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:02 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:03.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:02 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:03.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:02 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:03.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:02 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:03.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:02 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:03.468 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:36:03.898 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":19,"active_gid":24362,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6800","nonce":1128053553},{"type":"v1","addr":"172.21.15.125:6801","nonce":1128053553}]},"active_addr":"172.21.15.125:6801/1128053553","active_change":"2023-10-09T10:35:20.608067+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24346,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":57,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":86510625}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":1483568239}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":2757467822}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":3709143923}]}]} 2023-10-09T10:36:03.904 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T10:36:03.904 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T10:36:03.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T10:36:04.018 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[112589]: pgmap v31: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:04.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:04.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:04.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:04.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:36:04.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:04.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.satgiw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T10:36:04.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.satgiw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T10:36:04.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:04.020 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2073095022' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T10:36:04.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[127026]: pgmap v31: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:04.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:04.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:04.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:04.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:36:04.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:04.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.satgiw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T10:36:04.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.satgiw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T10:36:04.021 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:04.021 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:03 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2073095022' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T10:36:04.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:03 smithi155 ceph-mon[117223]: pgmap v31: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:04.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:03 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:04.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:03 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:04.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:03 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:04.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:03 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:36:04.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:03 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:04.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:03 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.satgiw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T10:36:04.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:03 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.satgiw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T10:36:04.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:03 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:04.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:03 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2073095022' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T10:36:04.225 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:05.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:04 smithi155 ceph-mon[117223]: Deploying daemon mds.foo.smithi125.satgiw on smithi125 2023-10-09T10:36:05.279 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:04 smithi125 ceph-mon[112589]: Deploying daemon mds.foo.smithi125.satgiw on smithi125 2023-10-09T10:36:05.279 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:04 smithi125 ceph-mon[127026]: Deploying daemon mds.foo.smithi125.satgiw on smithi125 2023-10-09T10:36:05.706 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:36:05.707 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":64,"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","created":"2023-10-09T10:31:07.767709+0000","modified":"2023-10-09T10:35:59.058548+0000","last_up_change":"2023-10-09T10:34:57.946717+0000","last_in_change":"2023-10-09T10:34:43.351594+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T10:33:34.342993+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T10:35:19.807378+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T10:35:55.201745+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T10:35:56.049529+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"874cd96b-ddd8-4164-931e-06f2388dd41b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6803","nonce":3322074820}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6805","nonce":3322074820}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6809","nonce":3322074820}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6807","nonce":3322074820}]},"public_addr":"172.21.15.125:6803/3322074820","cluster_addr":"172.21.15.125:6805/3322074820","heartbeat_back_addr":"172.21.15.125:6809/3322074820","heartbeat_front_addr":"172.21.15.125:6807/3322074820","state":["exists","up"]},{"osd":1,"uuid":"0e517dd5-6d0b-4841-bb81-1eb04377ba5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6811","nonce":3260318814}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6813","nonce":3260318814}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6817","nonce":3260318814}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6815","nonce":3260318814}]},"public_addr":"172.21.15.125:6811/3260318814","cluster_addr":"172.21.15.125:6813/3260318814","heartbeat_back_addr":"172.21.15.125:6817/3260318814","heartbeat_front_addr":"172.21.15.125:6815/3260318814","state":["exists","up"]},{"osd":2,"uuid":"643c1f3c-ecd9-464a-ad53-bf35a4deee5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6819","nonce":4010155657}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6821","nonce":4010155657}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6825","nonce":4010155657}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6823","nonce":4010155657}]},"public_addr":"172.21.15.125:6819/4010155657","cluster_addr":"172.21.15.125:6821/4010155657","heartbeat_back_addr":"172.21.15.125:6825/4010155657","heartbeat_front_addr":"172.21.15.125:6823/4010155657","state":["exists","up"]},{"osd":3,"uuid":"3015cd6e-b270-44c3-8e6e-6bcd979bcd21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6827","nonce":2950193979}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6829","nonce":2950193979}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6833","nonce":2950193979}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6831","nonce":2950193979}]},"public_addr":"172.21.15.125:6827/2950193979","cluster_addr":"172.21.15.125:6829/2950193979","heartbeat_back_addr":"172.21.15.125:6833/2950193979","heartbeat_front_addr":"172.21.15.125:6831/2950193979","state":["exists","up"]},{"osd":4,"uuid":"272c7b95-f89c-4fe1-a299-a861e0ec9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6801","nonce":1086065767}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6803","nonce":1086065767}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6807","nonce":1086065767}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6805","nonce":1086065767}]},"public_addr":"172.21.15.155:6801/1086065767","cluster_addr":"172.21.15.155:6803/1086065767","heartbeat_back_addr":"172.21.15.155:6807/1086065767","heartbeat_front_addr":"172.21.15.155:6805/1086065767","state":["exists","up"]},{"osd":5,"uuid":"201d51f2-233c-40fc-a42d-649d225954fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6809","nonce":104536199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6811","nonce":104536199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6815","nonce":104536199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6813","nonce":104536199}]},"public_addr":"172.21.15.155:6809/104536199","cluster_addr":"172.21.15.155:6811/104536199","heartbeat_back_addr":"172.21.15.155:6815/104536199","heartbeat_front_addr":"172.21.15.155:6813/104536199","state":["exists","up"]},{"osd":6,"uuid":"4ec66642-108e-401f-aff3-e7ab258bc083","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6817","nonce":1730141492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6819","nonce":1730141492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6823","nonce":1730141492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6821","nonce":1730141492}]},"public_addr":"172.21.15.155:6817/1730141492","cluster_addr":"172.21.15.155:6819/1730141492","heartbeat_back_addr":"172.21.15.155:6823/1730141492","heartbeat_front_addr":"172.21.15.155:6821/1730141492","state":["exists","up"]},{"osd":7,"uuid":"f8eccbf5-4ad0-43e7-be9a-00c05a362e57","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6825","nonce":3094552858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6827","nonce":3094552858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6831","nonce":3094552858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6829","nonce":3094552858}]},"public_addr":"172.21.15.155:6825/3094552858","cluster_addr":"172.21.15.155:6827/3094552858","heartbeat_back_addr":"172.21.15.155:6831/3094552858","heartbeat_front_addr":"172.21.15.155:6829/3094552858","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:32:58.904066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:15.883014+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:32.639145+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:49.937002+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:06.212675+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:22.755647+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:39.598906+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:56.143619+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:0/1316191185":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6801/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6800/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/2429796720":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6801/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6800/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6800/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/1623791415":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/461781620":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/1101683240":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/2789680395":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/725980307":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/1709045642":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/2611643565":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6801/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/178379618":"2023-10-10T10:31:33.872122+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T10:36:06.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[112589]: pgmap v32: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:06.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:06.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:06.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:06.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.ydxmyq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T10:36:06.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.ydxmyq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T10:36:06.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:06.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[112589]: Deploying daemon mds.foo.smithi155.ydxmyq on smithi155 2023-10-09T10:36:06.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:06.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:36:06.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2368177802' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:36:06.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[127026]: pgmap v32: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:06.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:06.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:06.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:06.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.ydxmyq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T10:36:06.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.ydxmyq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T10:36:06.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:06.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[127026]: Deploying daemon mds.foo.smithi155.ydxmyq on smithi155 2023-10-09T10:36:06.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:06.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:36:06.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:05 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2368177802' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:36:06.117 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T10:36:06.118 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T10:36:06.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:05 smithi155 ceph-mon[117223]: pgmap v32: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:06.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:05 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:06.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:05 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:06.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:05 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:06.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:05 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.ydxmyq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T10:36:06.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:05 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.ydxmyq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T10:36:06.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:05 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:06.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:05 smithi155 ceph-mon[117223]: Deploying daemon mds.foo.smithi155.ydxmyq on smithi155 2023-10-09T10:36:06.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:05 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:06.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:05 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:36:06.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:05 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2368177802' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:36:06.385 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:07.208 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[112589]: mds.? [v2:172.21.15.125:6834/1945587275,v1:172.21.15.125:6835/1945587275] up:boot 2023-10-09T10:36:07.208 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[112589]: daemon mds.foo.smithi125.satgiw assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T10:36:07.208 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[112589]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T10:36:07.208 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[112589]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T10:36:07.208 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[112589]: Cluster is now healthy 2023-10-09T10:36:07.209 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[112589]: fsmap foo:0 1 up:standby 2023-10-09T10:36:07.209 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.satgiw"}]: dispatch 2023-10-09T10:36:07.209 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[112589]: fsmap foo:1 {0=foo.smithi125.satgiw=up:creating} 2023-10-09T10:36:07.209 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[112589]: daemon mds.foo.smithi125.satgiw is now active in filesystem foo as rank 0 2023-10-09T10:36:07.209 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[112589]: pgmap v33: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2023-10-09T10:36:07.209 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[127026]: mds.? [v2:172.21.15.125:6834/1945587275,v1:172.21.15.125:6835/1945587275] up:boot 2023-10-09T10:36:07.209 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[127026]: daemon mds.foo.smithi125.satgiw assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T10:36:07.209 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[127026]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T10:36:07.209 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[127026]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T10:36:07.209 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[127026]: Cluster is now healthy 2023-10-09T10:36:07.209 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[127026]: fsmap foo:0 1 up:standby 2023-10-09T10:36:07.210 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.satgiw"}]: dispatch 2023-10-09T10:36:07.210 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[127026]: fsmap foo:1 {0=foo.smithi125.satgiw=up:creating} 2023-10-09T10:36:07.210 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[127026]: daemon mds.foo.smithi125.satgiw is now active in filesystem foo as rank 0 2023-10-09T10:36:07.210 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:06 smithi125 ceph-mon[127026]: pgmap v33: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2023-10-09T10:36:07.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:06 smithi155 ceph-mon[117223]: mds.? [v2:172.21.15.125:6834/1945587275,v1:172.21.15.125:6835/1945587275] up:boot 2023-10-09T10:36:07.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:06 smithi155 ceph-mon[117223]: daemon mds.foo.smithi125.satgiw assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T10:36:07.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:06 smithi155 ceph-mon[117223]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T10:36:07.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:06 smithi155 ceph-mon[117223]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T10:36:07.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:06 smithi155 ceph-mon[117223]: Cluster is now healthy 2023-10-09T10:36:07.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:06 smithi155 ceph-mon[117223]: fsmap foo:0 1 up:standby 2023-10-09T10:36:07.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:06 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.satgiw"}]: dispatch 2023-10-09T10:36:07.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:06 smithi155 ceph-mon[117223]: fsmap foo:1 {0=foo.smithi125.satgiw=up:creating} 2023-10-09T10:36:07.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:06 smithi155 ceph-mon[117223]: daemon mds.foo.smithi125.satgiw is now active in filesystem foo as rank 0 2023-10-09T10:36:07.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:06 smithi155 ceph-mon[117223]: pgmap v33: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2023-10-09T10:36:07.570 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[169522]: ts=2023-10-09T10:36:07.207Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002464037s 2023-10-09T10:36:07.612 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:36:07.612 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":64,"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","created":"2023-10-09T10:31:07.767709+0000","modified":"2023-10-09T10:35:59.058548+0000","last_up_change":"2023-10-09T10:34:57.946717+0000","last_in_change":"2023-10-09T10:34:43.351594+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T10:33:34.342993+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T10:35:19.807378+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T10:35:55.201745+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T10:35:56.049529+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"874cd96b-ddd8-4164-931e-06f2388dd41b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6803","nonce":3322074820}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6805","nonce":3322074820}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6809","nonce":3322074820}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":3322074820},{"type":"v1","addr":"172.21.15.125:6807","nonce":3322074820}]},"public_addr":"172.21.15.125:6803/3322074820","cluster_addr":"172.21.15.125:6805/3322074820","heartbeat_back_addr":"172.21.15.125:6809/3322074820","heartbeat_front_addr":"172.21.15.125:6807/3322074820","state":["exists","up"]},{"osd":1,"uuid":"0e517dd5-6d0b-4841-bb81-1eb04377ba5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6811","nonce":3260318814}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6813","nonce":3260318814}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6817","nonce":3260318814}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":3260318814},{"type":"v1","addr":"172.21.15.125:6815","nonce":3260318814}]},"public_addr":"172.21.15.125:6811/3260318814","cluster_addr":"172.21.15.125:6813/3260318814","heartbeat_back_addr":"172.21.15.125:6817/3260318814","heartbeat_front_addr":"172.21.15.125:6815/3260318814","state":["exists","up"]},{"osd":2,"uuid":"643c1f3c-ecd9-464a-ad53-bf35a4deee5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6819","nonce":4010155657}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6821","nonce":4010155657}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6825","nonce":4010155657}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":4010155657},{"type":"v1","addr":"172.21.15.125:6823","nonce":4010155657}]},"public_addr":"172.21.15.125:6819/4010155657","cluster_addr":"172.21.15.125:6821/4010155657","heartbeat_back_addr":"172.21.15.125:6825/4010155657","heartbeat_front_addr":"172.21.15.125:6823/4010155657","state":["exists","up"]},{"osd":3,"uuid":"3015cd6e-b270-44c3-8e6e-6bcd979bcd21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6827","nonce":2950193979}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6829","nonce":2950193979}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6833","nonce":2950193979}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":2950193979},{"type":"v1","addr":"172.21.15.125:6831","nonce":2950193979}]},"public_addr":"172.21.15.125:6827/2950193979","cluster_addr":"172.21.15.125:6829/2950193979","heartbeat_back_addr":"172.21.15.125:6833/2950193979","heartbeat_front_addr":"172.21.15.125:6831/2950193979","state":["exists","up"]},{"osd":4,"uuid":"272c7b95-f89c-4fe1-a299-a861e0ec9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6801","nonce":1086065767}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6803","nonce":1086065767}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6807","nonce":1086065767}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6805","nonce":1086065767}]},"public_addr":"172.21.15.155:6801/1086065767","cluster_addr":"172.21.15.155:6803/1086065767","heartbeat_back_addr":"172.21.15.155:6807/1086065767","heartbeat_front_addr":"172.21.15.155:6805/1086065767","state":["exists","up"]},{"osd":5,"uuid":"201d51f2-233c-40fc-a42d-649d225954fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6809","nonce":104536199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6811","nonce":104536199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6815","nonce":104536199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6813","nonce":104536199}]},"public_addr":"172.21.15.155:6809/104536199","cluster_addr":"172.21.15.155:6811/104536199","heartbeat_back_addr":"172.21.15.155:6815/104536199","heartbeat_front_addr":"172.21.15.155:6813/104536199","state":["exists","up"]},{"osd":6,"uuid":"4ec66642-108e-401f-aff3-e7ab258bc083","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6817","nonce":1730141492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6819","nonce":1730141492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6823","nonce":1730141492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6821","nonce":1730141492}]},"public_addr":"172.21.15.155:6817/1730141492","cluster_addr":"172.21.15.155:6819/1730141492","heartbeat_back_addr":"172.21.15.155:6823/1730141492","heartbeat_front_addr":"172.21.15.155:6821/1730141492","state":["exists","up"]},{"osd":7,"uuid":"f8eccbf5-4ad0-43e7-be9a-00c05a362e57","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6825","nonce":3094552858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6827","nonce":3094552858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6831","nonce":3094552858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6829","nonce":3094552858}]},"public_addr":"172.21.15.155:6825/3094552858","cluster_addr":"172.21.15.155:6827/3094552858","heartbeat_back_addr":"172.21.15.155:6831/3094552858","heartbeat_front_addr":"172.21.15.155:6829/3094552858","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:32:58.904066+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:15.883014+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:32.639145+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:49.937002+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:06.212675+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:22.755647+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:39.598906+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:56.143619+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:0/1316191185":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6801/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6800/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/2429796720":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6801/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6800/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6800/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/1623791415":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/461781620":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/1101683240":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/2789680395":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/725980307":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/1709045642":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/2611643565":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6801/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/178379618":"2023-10-10T10:31:33.872122+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T10:36:07.966 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:07.966 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[127026]: mds.? [v2:172.21.15.125:6834/1945587275,v1:172.21.15.125:6835/1945587275] up:active 2023-10-09T10:36:07.966 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[127026]: mds.? [v2:172.21.15.155:6832/914042814,v1:172.21.15.155:6833/914042814] up:boot 2023-10-09T10:36:07.966 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[127026]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:36:07.966 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.ydxmyq"}]: dispatch 2023-10-09T10:36:07.966 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:07.966 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[127026]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:36:08.114 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T10:36:08.114 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T10:36:08.115 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T10:36:08.115 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T10:36:08.116 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T10:36:08.116 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T10:36:08.116 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T10:36:08.117 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T10:36:08.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:07 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:07 smithi155 ceph-mon[117223]: mds.? [v2:172.21.15.125:6834/1945587275,v1:172.21.15.125:6835/1945587275] up:active 2023-10-09T10:36:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:07 smithi155 ceph-mon[117223]: mds.? [v2:172.21.15.155:6832/914042814,v1:172.21.15.155:6833/914042814] up:boot 2023-10-09T10:36:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:07 smithi155 ceph-mon[117223]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:36:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:07 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.ydxmyq"}]: dispatch 2023-10-09T10:36:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:07 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:07 smithi155 ceph-mon[117223]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:36:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:07 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:07 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:07 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:07 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:36:08.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:07 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/4122148014' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:36:08.333 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:08.333 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[112589]: mds.? [v2:172.21.15.125:6834/1945587275,v1:172.21.15.125:6835/1945587275] up:active 2023-10-09T10:36:08.333 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[112589]: mds.? [v2:172.21.15.155:6832/914042814,v1:172.21.15.155:6833/914042814] up:boot 2023-10-09T10:36:08.333 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[112589]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:36:08.333 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.ydxmyq"}]: dispatch 2023-10-09T10:36:08.333 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:08.333 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[112589]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:36:08.333 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:08.333 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:08.333 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:08.333 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:36:08.334 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/4122148014' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:36:08.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:08.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:08.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:08.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:36:08.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:07 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/4122148014' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:36:08.645 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:09.027 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:08 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:36:08] "GET /metrics HTTP/1.1" 200 208142 "" "Prometheus/2.43.0" 2023-10-09T10:36:09.204 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:09.693 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:09 smithi125 ceph-mon[112589]: pgmap v34: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2023-10-09T10:36:09.693 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:09 smithi125 ceph-mon[127026]: pgmap v34: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2023-10-09T10:36:09.918 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:09.918 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:09.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:09 smithi155 ceph-mon[117223]: pgmap v34: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2023-10-09T10:36:10.264 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:10.264 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:10.634 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:11.128 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:11.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:11 smithi125 ceph-mon[112589]: mds.? [v2:172.21.15.125:6834/1945587275,v1:172.21.15.125:6835/1945587275] up:active 2023-10-09T10:36:11.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:11 smithi125 ceph-mon[112589]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:36:11.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:11 smithi125 ceph-mon[112589]: pgmap v35: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T10:36:11.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:11 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:11.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:11 smithi125 ceph-mon[127026]: mds.? [v2:172.21.15.125:6834/1945587275,v1:172.21.15.125:6835/1945587275] up:active 2023-10-09T10:36:11.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:11 smithi125 ceph-mon[127026]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:36:11.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:11 smithi125 ceph-mon[127026]: pgmap v35: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T10:36:11.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:11 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:11.824 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:11 smithi155 ceph-mon[117223]: mds.? [v2:172.21.15.125:6834/1945587275,v1:172.21.15.125:6835/1945587275] up:active 2023-10-09T10:36:11.824 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:11 smithi155 ceph-mon[117223]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:36:11.824 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:11 smithi155 ceph-mon[117223]: pgmap v35: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T10:36:11.824 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:11 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:12.079 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:36:11 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:36:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T10:36:12.804 INFO:teuthology.orchestra.run.smithi125.stdout:55834574886 2023-10-09T10:36:12.804 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T10:36:12.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:12 smithi155 ceph-mon[117223]: mds.? [v2:172.21.15.155:6832/914042814,v1:172.21.15.155:6833/914042814] up:standby 2023-10-09T10:36:12.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:12 smithi155 ceph-mon[117223]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:36:12.985 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:12 smithi125 ceph-mon[112589]: mds.? [v2:172.21.15.155:6832/914042814,v1:172.21.15.155:6833/914042814] up:standby 2023-10-09T10:36:12.985 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:12 smithi125 ceph-mon[112589]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:36:12.985 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:12 smithi125 ceph-mon[127026]: mds.? [v2:172.21.15.155:6832/914042814,v1:172.21.15.155:6833/914042814] up:standby 2023-10-09T10:36:12.985 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:12 smithi125 ceph-mon[127026]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:36:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:13 smithi155 ceph-mon[117223]: pgmap v36: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T10:36:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:13 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:13 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:13.980 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:13 smithi125 ceph-mon[112589]: pgmap v36: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T10:36:13.980 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:13 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:13.980 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:13 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:13.980 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:13 smithi125 ceph-mon[127026]: pgmap v36: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T10:36:13.980 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:13 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:13.980 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:13 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:14.095 INFO:teuthology.orchestra.run.smithi125.stdout:158913789977 2023-10-09T10:36:14.095 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T10:36:15.486 INFO:teuthology.orchestra.run.smithi125.stdout:107374182432 2023-10-09T10:36:15.486 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T10:36:15.778 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:15 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:15.778 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:15 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:15.778 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:15 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:15.778 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:15 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:36:15.778 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:15 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:15.778 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:15 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:36:15.779 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:15 smithi155 ceph-mon[117223]: pgmap v37: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T10:36:15.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:15 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:15.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:15 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:15.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:15 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:15.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:15 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:36:15.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:15 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:15.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:15 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:36:15.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:15 smithi125 ceph-mon[127026]: pgmap v37: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T10:36:15.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:15 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:15.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:15 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:15.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:15 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:15.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:15 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:36:15.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:15 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:15.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:15 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:36:15.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:15 smithi125 ceph-mon[112589]: pgmap v37: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T10:36:15.972 INFO:teuthology.orchestra.run.smithi125.stdout:219043332114 2023-10-09T10:36:15.973 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T10:36:16.138 INFO:teuthology.orchestra.run.smithi125.stdout:34359738408 2023-10-09T10:36:16.138 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T10:36:16.157 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:16.342 INFO:teuthology.orchestra.run.smithi125.stdout:128849018908 2023-10-09T10:36:16.342 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T10:36:16.498 INFO:teuthology.orchestra.run.smithi125.stdout:77309411361 2023-10-09T10:36:16.498 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T10:36:16.885 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:16.952 INFO:teuthology.orchestra.run.smithi125.stdout:188978561045 2023-10-09T10:36:16.952 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T10:36:17.139 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:17.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:17 smithi125 ceph-mon[112589]: pgmap v38: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T10:36:17.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:17 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:17.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:17 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:17.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:17 smithi125 ceph-mon[127026]: pgmap v38: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T10:36:17.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:17 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:17.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:17 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:18.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:17 smithi155 ceph-mon[117223]: pgmap v38: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T10:36:18.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:17 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:18.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:17 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:18.227 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:18.483 INFO:teuthology.orchestra.run.smithi125.stdout:158913789978 2023-10-09T10:36:18.485 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:18.714 INFO:teuthology.orchestra.run.smithi125.stdout:55834574888 2023-10-09T10:36:18.737 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:18.938 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913789977 got 158913789978 for osd.5 2023-10-09T10:36:18.939 DEBUG:teuthology.parallel:result is None 2023-10-09T10:36:18.985 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:18 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1408090344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T10:36:18.985 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:18 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/464794166' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T10:36:18.985 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:18 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/1408090344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T10:36:18.986 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:18 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/464794166' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T10:36:18.986 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:18 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:36:18] "GET /metrics HTTP/1.1" 200 208142 "" "Prometheus/2.43.0" 2023-10-09T10:36:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:18 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1408090344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T10:36:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:18 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/464794166' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T10:36:19.301 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:19.617 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:19.980 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:19 smithi125 ceph-mon[112589]: pgmap v39: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T10:36:19.980 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:19 smithi125 ceph-mon[127026]: pgmap v39: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T10:36:20.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:19 smithi155 ceph-mon[117223]: pgmap v39: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T10:36:20.872 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T10:36:20.872 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-10-09T10:36:20.872 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-09T10:36:20.872 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:20.872 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:36:20.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T10:36:20.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-10-09T10:36:20.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-09T10:36:20.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:20.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:36:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T10:36:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-10-09T10:36:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-09T10:36:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:36:21.971 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574886 got 55834574888 for osd.1 2023-10-09T10:36:21.971 DEBUG:teuthology.parallel:result is None 2023-10-09T10:36:22.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:21 smithi125 ceph-mon[112589]: pgmap v40: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T10:36:22.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:21 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T10:36:22.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:21 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-10-09T10:36:22.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:21 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-09T10:36:22.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:21 smithi125 ceph-mon[112589]: osdmap e65: 8 total, 8 up, 8 in 2023-10-09T10:36:22.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:21 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2148686334' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T10:36:22.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:21 smithi125 ceph-mon[127026]: pgmap v40: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T10:36:22.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:21 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T10:36:22.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:21 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-10-09T10:36:22.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:21 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-09T10:36:22.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:21 smithi125 ceph-mon[127026]: osdmap e65: 8 total, 8 up, 8 in 2023-10-09T10:36:22.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:21 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2148686334' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T10:36:22.221 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:36:21 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:36:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T10:36:22.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:21 smithi155 ceph-mon[117223]: pgmap v40: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T10:36:22.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:21 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T10:36:22.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:21 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-10-09T10:36:22.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:21 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-09T10:36:22.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:21 smithi155 ceph-mon[117223]: osdmap e65: 8 total, 8 up, 8 in 2023-10-09T10:36:22.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:21 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2148686334' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T10:36:22.254 INFO:teuthology.orchestra.run.smithi125.stdout:34359738409 2023-10-09T10:36:22.496 INFO:teuthology.orchestra.run.smithi125.stdout:107374182433 2023-10-09T10:36:22.783 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738408 got 34359738409 for osd.0 2023-10-09T10:36:22.783 DEBUG:teuthology.parallel:result is None 2023-10-09T10:36:22.907 INFO:teuthology.orchestra.run.smithi125.stdout:128849018910 2023-10-09T10:36:23.023 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:22 smithi125 ceph-mon[112589]: osdmap e66: 8 total, 8 up, 8 in 2023-10-09T10:36:23.024 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:22 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/446737367' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T10:36:23.024 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:22 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/618525939' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T10:36:23.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:22 smithi125 ceph-mon[127026]: osdmap e66: 8 total, 8 up, 8 in 2023-10-09T10:36:23.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:22 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/446737367' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T10:36:23.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:22 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/618525939' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T10:36:23.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:22 smithi155 ceph-mon[117223]: osdmap e66: 8 total, 8 up, 8 in 2023-10-09T10:36:23.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:22 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/446737367' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T10:36:23.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:22 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/618525939' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T10:36:23.516 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182432 got 107374182433 for osd.3 2023-10-09T10:36:23.516 DEBUG:teuthology.parallel:result is None 2023-10-09T10:36:23.520 INFO:teuthology.orchestra.run.smithi125.stdout:77309411362 2023-10-09T10:36:23.995 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:23 smithi125 ceph-mon[112589]: pgmap v43: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:23.995 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:23 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3810998940' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T10:36:23.996 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:23 smithi125 ceph-mon[127026]: pgmap v43: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:23.996 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:23 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3810998940' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T10:36:23.996 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018908 got 128849018910 for osd.4 2023-10-09T10:36:23.996 DEBUG:teuthology.parallel:result is None 2023-10-09T10:36:24.162 INFO:teuthology.orchestra.run.smithi125.stdout:219043332116 2023-10-09T10:36:24.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:23 smithi155 ceph-mon[117223]: pgmap v43: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:24.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:23 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3810998940' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T10:36:24.274 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411361 got 77309411362 for osd.2 2023-10-09T10:36:24.274 DEBUG:teuthology.parallel:result is None 2023-10-09T10:36:24.737 INFO:tasks.cephadm.ceph_manager.ceph:need seq 219043332114 got 219043332116 for osd.7 2023-10-09T10:36:24.737 DEBUG:teuthology.parallel:result is None 2023-10-09T10:36:24.782 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:24 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/322178737' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T10:36:24.999 INFO:teuthology.orchestra.run.smithi125.stdout:188978561047 2023-10-09T10:36:25.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:24 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/322178737' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T10:36:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:24 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/322178737' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T10:36:25.379 INFO:tasks.cephadm.ceph_manager.ceph:need seq 188978561045 got 188978561047 for osd.6 2023-10-09T10:36:25.379 DEBUG:teuthology.parallel:result is None 2023-10-09T10:36:25.379 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T10:36:25.379 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:36:25.665 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:26.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:25 smithi125 ceph-mon[112589]: pgmap v44: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:26.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:25 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/4108578890' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T10:36:26.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:25 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:26.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:25 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:26.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:25 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:26.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:25 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:36:26.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:25 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:26.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:25 smithi125 ceph-mon[127026]: pgmap v44: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:26.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:25 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/4108578890' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T10:36:26.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:25 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:26.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:25 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:26.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:25 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:26.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:25 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:36:26.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:25 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:26.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:25 smithi155 ceph-mon[117223]: pgmap v44: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:26.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:25 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/4108578890' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T10:36:26.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:25 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:26.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:25 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:26.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:25 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:26.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:25 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:36:26.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:25 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:26.923 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:36:26.924 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:36:27.284 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":45,"stamp":"2023-10-09T10:36:26.636620+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":218,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385372,"kb_used_data":7068,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747575844,"statfs":{"total":767960285184,"available":765517664256,"internally_reserved":0,"allocated":7237632,"data_stored":4130156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001329"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771446+0000","last_change":"2023-10-09T10:35:58.712102+0000","last_active":"2023-10-09T10:36:20.771446+0000","last_peered":"2023-10-09T10:36:20.771446+0000","last_clean":"2023-10-09T10:36:20.771446+0000","last_became_active":"2023-10-09T10:35:58.711824+0000","last_became_peered":"2023-10-09T10:35:58.711824+0000","last_unstale":"2023-10-09T10:36:20.771446+0000","last_undegraded":"2023-10-09T10:36:20.771446+0000","last_fullsized":"2023-10-09T10:36:20.771446+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:42.811844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765810+0000","last_change":"2023-10-09T10:35:56.814681+0000","last_active":"2023-10-09T10:36:20.765810+0000","last_peered":"2023-10-09T10:36:20.765810+0000","last_clean":"2023-10-09T10:36:20.765810+0000","last_became_active":"2023-10-09T10:35:56.814401+0000","last_became_peered":"2023-10-09T10:35:56.814401+0000","last_unstale":"2023-10-09T10:36:20.765810+0000","last_undegraded":"2023-10-09T10:36:20.765810+0000","last_fullsized":"2023-10-09T10:36:20.765810+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:15:46.517618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770109+0000","last_change":"2023-10-09T10:35:56.836445+0000","last_active":"2023-10-09T10:36:20.770109+0000","last_peered":"2023-10-09T10:36:20.770109+0000","last_clean":"2023-10-09T10:36:20.770109+0000","last_became_active":"2023-10-09T10:35:56.836158+0000","last_became_peered":"2023-10-09T10:35:56.836158+0000","last_unstale":"2023-10-09T10:36:20.770109+0000","last_undegraded":"2023-10-09T10:36:20.770109+0000","last_fullsized":"2023-10-09T10:36:20.770109+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:29.431572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770254+0000","last_change":"2023-10-09T10:35:56.815445+0000","last_active":"2023-10-09T10:36:20.770254+0000","last_peered":"2023-10-09T10:36:20.770254+0000","last_clean":"2023-10-09T10:36:20.770254+0000","last_became_active":"2023-10-09T10:35:56.815152+0000","last_became_peered":"2023-10-09T10:35:56.815152+0000","last_unstale":"2023-10-09T10:36:20.770254+0000","last_undegraded":"2023-10-09T10:36:20.770254+0000","last_fullsized":"2023-10-09T10:36:20.770254+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:04.416880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771581+0000","last_change":"2023-10-09T10:35:58.242519+0000","last_active":"2023-10-09T10:36:20.771581+0000","last_peered":"2023-10-09T10:36:20.771581+0000","last_clean":"2023-10-09T10:36:20.771581+0000","last_became_active":"2023-10-09T10:35:58.242240+0000","last_became_peered":"2023-10-09T10:35:58.242240+0000","last_unstale":"2023-10-09T10:36:20.771581+0000","last_undegraded":"2023-10-09T10:36:20.771581+0000","last_fullsized":"2023-10-09T10:36:20.771581+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:26.836816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.766130+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:36:20.766130+0000","last_peered":"2023-10-09T10:36:20.766130+0000","last_clean":"2023-10-09T10:36:20.766130+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:36:20.766130+0000","last_undegraded":"2023-10-09T10:36:20.766130+0000","last_fullsized":"2023-10-09T10:36:20.766130+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771810+0000","last_change":"2023-10-09T10:35:58.130037+0000","last_active":"2023-10-09T10:36:20.771810+0000","last_peered":"2023-10-09T10:36:20.771810+0000","last_clean":"2023-10-09T10:36:20.771810+0000","last_became_active":"2023-10-09T10:35:58.129751+0000","last_became_peered":"2023-10-09T10:35:58.129751+0000","last_unstale":"2023-10-09T10:36:20.771810+0000","last_undegraded":"2023-10-09T10:36:20.771810+0000","last_fullsized":"2023-10-09T10:36:20.771810+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:53.201853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.766006+0000","last_change":"2023-10-09T10:35:56.821768+0000","last_active":"2023-10-09T10:36:20.766006+0000","last_peered":"2023-10-09T10:36:20.766006+0000","last_clean":"2023-10-09T10:36:20.766006+0000","last_became_active":"2023-10-09T10:35:56.821452+0000","last_became_peered":"2023-10-09T10:35:56.821452+0000","last_unstale":"2023-10-09T10:36:20.766006+0000","last_undegraded":"2023-10-09T10:36:20.766006+0000","last_fullsized":"2023-10-09T10:36:20.766006+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:32.659779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769165+0000","last_change":"2023-10-09T10:35:56.841741+0000","last_active":"2023-10-09T10:36:20.769165+0000","last_peered":"2023-10-09T10:36:20.769165+0000","last_clean":"2023-10-09T10:36:20.769165+0000","last_became_active":"2023-10-09T10:35:56.841473+0000","last_became_peered":"2023-10-09T10:35:56.841473+0000","last_unstale":"2023-10-09T10:36:20.769165+0000","last_undegraded":"2023-10-09T10:36:20.769165+0000","last_fullsized":"2023-10-09T10:36:20.769165+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:17:07.385347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765589+0000","last_change":"2023-10-09T10:35:56.817791+0000","last_active":"2023-10-09T10:36:20.765589+0000","last_peered":"2023-10-09T10:36:20.765589+0000","last_clean":"2023-10-09T10:36:20.765589+0000","last_became_active":"2023-10-09T10:35:56.817533+0000","last_became_peered":"2023-10-09T10:35:56.817533+0000","last_unstale":"2023-10-09T10:36:20.765589+0000","last_undegraded":"2023-10-09T10:36:20.765589+0000","last_fullsized":"2023-10-09T10:36:20.765589+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:20.056489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.773113+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:36:20.773113+0000","last_peered":"2023-10-09T10:36:20.773113+0000","last_clean":"2023-10-09T10:36:20.773113+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:36:20.773113+0000","last_undegraded":"2023-10-09T10:36:20.773113+0000","last_fullsized":"2023-10-09T10:36:20.773113+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765873+0000","last_change":"2023-10-09T10:35:56.817387+0000","last_active":"2023-10-09T10:36:20.765873+0000","last_peered":"2023-10-09T10:36:20.765873+0000","last_clean":"2023-10-09T10:36:20.765873+0000","last_became_active":"2023-10-09T10:35:56.817101+0000","last_became_peered":"2023-10-09T10:35:56.817101+0000","last_unstale":"2023-10-09T10:36:20.765873+0000","last_undegraded":"2023-10-09T10:36:20.765873+0000","last_fullsized":"2023-10-09T10:36:20.765873+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:50.703008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.971795+0000","last_change":"2023-10-09T10:35:20.619783+0000","last_active":"2023-10-09T10:36:20.971795+0000","last_peered":"2023-10-09T10:36:20.971795+0000","last_clean":"2023-10-09T10:36:20.971795+0000","last_became_active":"2023-10-09T10:35:20.619441+0000","last_became_peered":"2023-10-09T10:35:20.619441+0000","last_unstale":"2023-10-09T10:36:20.971795+0000","last_undegraded":"2023-10-09T10:36:20.971795+0000","last_fullsized":"2023-10-09T10:36:20.971795+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:00.454996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769880+0000","last_change":"2023-10-09T10:35:56.840463+0000","last_active":"2023-10-09T10:36:20.769880+0000","last_peered":"2023-10-09T10:36:20.769880+0000","last_clean":"2023-10-09T10:36:20.769880+0000","last_became_active":"2023-10-09T10:35:56.839880+0000","last_became_peered":"2023-10-09T10:35:56.839880+0000","last_unstale":"2023-10-09T10:36:20.769880+0000","last_undegraded":"2023-10-09T10:36:20.769880+0000","last_fullsized":"2023-10-09T10:36:20.769880+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:38.048996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769557+0000","last_change":"2023-10-09T10:35:20.621958+0000","last_active":"2023-10-09T10:36:20.769557+0000","last_peered":"2023-10-09T10:36:20.769557+0000","last_clean":"2023-10-09T10:36:20.769557+0000","last_became_active":"2023-10-09T10:35:20.621656+0000","last_became_peered":"2023-10-09T10:35:20.621656+0000","last_unstale":"2023-10-09T10:36:20.769557+0000","last_undegraded":"2023-10-09T10:36:20.769557+0000","last_fullsized":"2023-10-09T10:36:20.769557+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:32.121613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770352+0000","last_change":"2023-10-09T10:35:56.820393+0000","last_active":"2023-10-09T10:36:20.770352+0000","last_peered":"2023-10-09T10:36:20.770352+0000","last_clean":"2023-10-09T10:36:20.770352+0000","last_became_active":"2023-10-09T10:35:56.820157+0000","last_became_peered":"2023-10-09T10:35:56.820157+0000","last_unstale":"2023-10-09T10:36:20.770352+0000","last_undegraded":"2023-10-09T10:36:20.770352+0000","last_fullsized":"2023-10-09T10:36:20.770352+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:35.968592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765424+0000","last_change":"2023-10-09T10:35:57.228099+0000","last_active":"2023-10-09T10:36:20.765424+0000","last_peered":"2023-10-09T10:36:20.765424+0000","last_clean":"2023-10-09T10:36:20.765424+0000","last_became_active":"2023-10-09T10:35:57.227754+0000","last_became_peered":"2023-10-09T10:35:57.227754+0000","last_unstale":"2023-10-09T10:36:20.765424+0000","last_undegraded":"2023-10-09T10:36:20.765424+0000","last_fullsized":"2023-10-09T10:36:20.765424+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:23.783778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768886+0000","last_change":"2023-10-09T10:35:56.816296+0000","last_active":"2023-10-09T10:36:20.768886+0000","last_peered":"2023-10-09T10:36:20.768886+0000","last_clean":"2023-10-09T10:36:20.768886+0000","last_became_active":"2023-10-09T10:35:56.816012+0000","last_became_peered":"2023-10-09T10:35:56.816012+0000","last_unstale":"2023-10-09T10:36:20.768886+0000","last_undegraded":"2023-10-09T10:36:20.768886+0000","last_fullsized":"2023-10-09T10:36:20.768886+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:16.247390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.772893+0000","last_change":"2023-10-09T10:35:20.619239+0000","last_active":"2023-10-09T10:36:20.772893+0000","last_peered":"2023-10-09T10:36:20.772893+0000","last_clean":"2023-10-09T10:36:20.772893+0000","last_became_active":"2023-10-09T10:35:20.618892+0000","last_became_peered":"2023-10-09T10:35:20.618892+0000","last_unstale":"2023-10-09T10:36:20.772893+0000","last_undegraded":"2023-10-09T10:36:20.772893+0000","last_fullsized":"2023-10-09T10:36:20.772893+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:55.019433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-10-09T10:36:21.886869+0000","last_change":"2023-10-09T10:36:21.886869+0000","last_active":"2023-10-09T10:36:21.886869+0000","last_peered":"2023-10-09T10:36:21.886869+0000","last_clean":"2023-10-09T10:36:21.886869+0000","last_became_active":"2023-10-09T10:36:21.886559+0000","last_became_peered":"2023-10-09T10:36:21.886559+0000","last_unstale":"2023-10-09T10:36:21.886869+0000","last_undegraded":"2023-10-09T10:36:21.886869+0000","last_fullsized":"2023-10-09T10:36:21.886869+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:12.576014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.764457+0000","last_change":"2023-10-09T10:35:56.820290+0000","last_active":"2023-10-09T10:36:20.764457+0000","last_peered":"2023-10-09T10:36:20.764457+0000","last_clean":"2023-10-09T10:36:20.764457+0000","last_became_active":"2023-10-09T10:35:56.819986+0000","last_became_peered":"2023-10-09T10:35:56.819986+0000","last_unstale":"2023-10-09T10:36:20.764457+0000","last_undegraded":"2023-10-09T10:36:20.764457+0000","last_fullsized":"2023-10-09T10:36:20.764457+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:31.721604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.772389+0000","last_change":"2023-10-09T10:35:20.619992+0000","last_active":"2023-10-09T10:36:20.772389+0000","last_peered":"2023-10-09T10:36:20.772389+0000","last_clean":"2023-10-09T10:36:20.772389+0000","last_became_active":"2023-10-09T10:35:20.619719+0000","last_became_peered":"2023-10-09T10:35:20.619719+0000","last_unstale":"2023-10-09T10:36:20.772389+0000","last_undegraded":"2023-10-09T10:36:20.772389+0000","last_fullsized":"2023-10-09T10:36:20.772389+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:31.964380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.971770+0000","last_change":"2023-10-09T10:35:57.165395+0000","last_active":"2023-10-09T10:36:20.971770+0000","last_peered":"2023-10-09T10:36:20.971770+0000","last_clean":"2023-10-09T10:36:20.971770+0000","last_became_active":"2023-10-09T10:35:57.165077+0000","last_became_peered":"2023-10-09T10:35:57.165077+0000","last_unstale":"2023-10-09T10:36:20.971770+0000","last_undegraded":"2023-10-09T10:36:20.971770+0000","last_fullsized":"2023-10-09T10:36:20.971770+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:53.159616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770024+0000","last_change":"2023-10-09T10:35:56.818871+0000","last_active":"2023-10-09T10:36:20.770024+0000","last_peered":"2023-10-09T10:36:20.770024+0000","last_clean":"2023-10-09T10:36:20.770024+0000","last_became_active":"2023-10-09T10:35:56.818634+0000","last_became_peered":"2023-10-09T10:35:56.818634+0000","last_unstale":"2023-10-09T10:36:20.770024+0000","last_undegraded":"2023-10-09T10:36:20.770024+0000","last_fullsized":"2023-10-09T10:36:20.770024+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:14.248324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"57'128","reported_seq":277,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.772317+0000","last_change":"2023-10-09T10:35:01.491534+0000","last_active":"2023-10-09T10:36:20.772317+0000","last_peered":"2023-10-09T10:36:20.772317+0000","last_clean":"2023-10-09T10:36:20.772317+0000","last_became_active":"2023-10-09T10:35:01.491048+0000","last_became_peered":"2023-10-09T10:35:01.491048+0000","last_unstale":"2023-10-09T10:36:20.772317+0000","last_undegraded":"2023-10-09T10:36:20.772317+0000","last_fullsized":"2023-10-09T10:36:20.772317+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:00:35.672924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765817+0000","last_change":"2023-10-09T10:35:20.622020+0000","last_active":"2023-10-09T10:36:20.765817+0000","last_peered":"2023-10-09T10:36:20.765817+0000","last_clean":"2023-10-09T10:36:20.765817+0000","last_became_active":"2023-10-09T10:35:20.621703+0000","last_became_peered":"2023-10-09T10:35:20.621703+0000","last_unstale":"2023-10-09T10:36:20.765817+0000","last_undegraded":"2023-10-09T10:36:20.765817+0000","last_fullsized":"2023-10-09T10:36:20.765817+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:59:34.492019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769021+0000","last_change":"2023-10-09T10:35:57.228907+0000","last_active":"2023-10-09T10:36:20.769021+0000","last_peered":"2023-10-09T10:36:20.769021+0000","last_clean":"2023-10-09T10:36:20.769021+0000","last_became_active":"2023-10-09T10:35:57.228408+0000","last_became_peered":"2023-10-09T10:35:57.228408+0000","last_unstale":"2023-10-09T10:36:20.769021+0000","last_undegraded":"2023-10-09T10:36:20.769021+0000","last_fullsized":"2023-10-09T10:36:20.769021+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:03.936095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768713+0000","last_change":"2023-10-09T10:35:56.814727+0000","last_active":"2023-10-09T10:36:20.768713+0000","last_peered":"2023-10-09T10:36:20.768713+0000","last_clean":"2023-10-09T10:36:20.768713+0000","last_became_active":"2023-10-09T10:35:56.813636+0000","last_became_peered":"2023-10-09T10:35:56.813636+0000","last_unstale":"2023-10-09T10:36:20.768713+0000","last_undegraded":"2023-10-09T10:36:20.768713+0000","last_fullsized":"2023-10-09T10:36:20.768713+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:49.853322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769493+0000","last_change":"2023-10-09T10:35:20.625994+0000","last_active":"2023-10-09T10:36:20.769493+0000","last_peered":"2023-10-09T10:36:20.769493+0000","last_clean":"2023-10-09T10:36:20.769493+0000","last_became_active":"2023-10-09T10:35:20.625638+0000","last_became_peered":"2023-10-09T10:35:20.625638+0000","last_unstale":"2023-10-09T10:36:20.769493+0000","last_undegraded":"2023-10-09T10:36:20.769493+0000","last_fullsized":"2023-10-09T10:36:20.769493+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:26.622626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768881+0000","last_change":"2023-10-09T10:35:56.840023+0000","last_active":"2023-10-09T10:36:20.768881+0000","last_peered":"2023-10-09T10:36:20.768881+0000","last_clean":"2023-10-09T10:36:20.768881+0000","last_became_active":"2023-10-09T10:35:56.839737+0000","last_became_peered":"2023-10-09T10:35:56.839737+0000","last_unstale":"2023-10-09T10:36:20.768881+0000","last_undegraded":"2023-10-09T10:36:20.768881+0000","last_fullsized":"2023-10-09T10:36:20.768881+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:20.699635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771705+0000","last_change":"2023-10-09T10:35:56.820004+0000","last_active":"2023-10-09T10:36:20.771705+0000","last_peered":"2023-10-09T10:36:20.771705+0000","last_clean":"2023-10-09T10:36:20.771705+0000","last_became_active":"2023-10-09T10:35:56.819736+0000","last_became_peered":"2023-10-09T10:35:56.819736+0000","last_unstale":"2023-10-09T10:36:20.771705+0000","last_undegraded":"2023-10-09T10:36:20.771705+0000","last_fullsized":"2023-10-09T10:36:20.771705+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:31:07.010228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765850+0000","last_change":"2023-10-09T10:35:20.621825+0000","last_active":"2023-10-09T10:36:20.765850+0000","last_peered":"2023-10-09T10:36:20.765850+0000","last_clean":"2023-10-09T10:36:20.765850+0000","last_became_active":"2023-10-09T10:35:20.621526+0000","last_became_peered":"2023-10-09T10:35:20.621526+0000","last_unstale":"2023-10-09T10:36:20.765850+0000","last_undegraded":"2023-10-09T10:36:20.765850+0000","last_fullsized":"2023-10-09T10:36:20.765850+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:59.814738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768308+0000","last_change":"2023-10-09T10:35:57.228124+0000","last_active":"2023-10-09T10:36:20.768308+0000","last_peered":"2023-10-09T10:36:20.768308+0000","last_clean":"2023-10-09T10:36:20.768308+0000","last_became_active":"2023-10-09T10:35:57.227833+0000","last_became_peered":"2023-10-09T10:35:57.227833+0000","last_unstale":"2023-10-09T10:36:20.768308+0000","last_undegraded":"2023-10-09T10:36:20.768308+0000","last_fullsized":"2023-10-09T10:36:20.768308+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:24.892825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.766101+0000","last_change":"2023-10-09T10:35:56.821198+0000","last_active":"2023-10-09T10:36:20.766101+0000","last_peered":"2023-10-09T10:36:20.766101+0000","last_clean":"2023-10-09T10:36:20.766101+0000","last_became_active":"2023-10-09T10:35:56.820864+0000","last_became_peered":"2023-10-09T10:35:56.820864+0000","last_unstale":"2023-10-09T10:36:20.766101+0000","last_undegraded":"2023-10-09T10:36:20.766101+0000","last_fullsized":"2023-10-09T10:36:20.766101+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:42.990821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768809+0000","last_change":"2023-10-09T10:35:20.621757+0000","last_active":"2023-10-09T10:36:20.768809+0000","last_peered":"2023-10-09T10:36:20.768809+0000","last_clean":"2023-10-09T10:36:20.768809+0000","last_became_active":"2023-10-09T10:35:20.621505+0000","last_became_peered":"2023-10-09T10:35:20.621505+0000","last_unstale":"2023-10-09T10:36:20.768809+0000","last_undegraded":"2023-10-09T10:36:20.768809+0000","last_fullsized":"2023-10-09T10:36:20.768809+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:35.203999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768978+0000","last_change":"2023-10-09T10:35:57.165557+0000","last_active":"2023-10-09T10:36:20.768978+0000","last_peered":"2023-10-09T10:36:20.768978+0000","last_clean":"2023-10-09T10:36:20.768978+0000","last_became_active":"2023-10-09T10:35:57.165215+0000","last_became_peered":"2023-10-09T10:35:57.165215+0000","last_unstale":"2023-10-09T10:36:20.768978+0000","last_undegraded":"2023-10-09T10:36:20.768978+0000","last_fullsized":"2023-10-09T10:36:20.768978+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:45.875941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769903+0000","last_change":"2023-10-09T10:35:56.822872+0000","last_active":"2023-10-09T10:36:20.769903+0000","last_peered":"2023-10-09T10:36:20.769903+0000","last_clean":"2023-10-09T10:36:20.769903+0000","last_became_active":"2023-10-09T10:35:56.822591+0000","last_became_peered":"2023-10-09T10:35:56.822591+0000","last_unstale":"2023-10-09T10:36:20.769903+0000","last_undegraded":"2023-10-09T10:36:20.769903+0000","last_fullsized":"2023-10-09T10:36:20.769903+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:27:57.817744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771825+0000","last_change":"2023-10-09T10:35:58.711128+0000","last_active":"2023-10-09T10:36:20.771825+0000","last_peered":"2023-10-09T10:36:20.771825+0000","last_clean":"2023-10-09T10:36:20.771825+0000","last_became_active":"2023-10-09T10:35:58.710718+0000","last_became_peered":"2023-10-09T10:35:58.710718+0000","last_unstale":"2023-10-09T10:36:20.771825+0000","last_undegraded":"2023-10-09T10:36:20.771825+0000","last_fullsized":"2023-10-09T10:36:20.771825+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:05.436822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.773093+0000","last_change":"2023-10-09T10:35:56.823476+0000","last_active":"2023-10-09T10:36:20.773093+0000","last_peered":"2023-10-09T10:36:20.773093+0000","last_clean":"2023-10-09T10:36:20.773093+0000","last_became_active":"2023-10-09T10:35:56.823138+0000","last_became_peered":"2023-10-09T10:35:56.823138+0000","last_unstale":"2023-10-09T10:36:20.773093+0000","last_undegraded":"2023-10-09T10:36:20.773093+0000","last_fullsized":"2023-10-09T10:36:20.773093+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:18.228816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771880+0000","last_change":"2023-10-09T10:35:58.321123+0000","last_active":"2023-10-09T10:36:20.771880+0000","last_peered":"2023-10-09T10:36:20.771880+0000","last_clean":"2023-10-09T10:36:20.771880+0000","last_became_active":"2023-10-09T10:35:58.320867+0000","last_became_peered":"2023-10-09T10:35:58.320867+0000","last_unstale":"2023-10-09T10:36:20.771880+0000","last_undegraded":"2023-10-09T10:36:20.771880+0000","last_fullsized":"2023-10-09T10:36:20.771880+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:33.609435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.970937+0000","last_change":"2023-10-09T10:35:56.816746+0000","last_active":"2023-10-09T10:36:20.970937+0000","last_peered":"2023-10-09T10:36:20.970937+0000","last_clean":"2023-10-09T10:36:20.970937+0000","last_became_active":"2023-10-09T10:35:56.816395+0000","last_became_peered":"2023-10-09T10:35:56.816395+0000","last_unstale":"2023-10-09T10:36:20.970937+0000","last_undegraded":"2023-10-09T10:36:20.970937+0000","last_fullsized":"2023-10-09T10:36:20.970937+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:24.841288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771768+0000","last_change":"2023-10-09T10:35:58.711019+0000","last_active":"2023-10-09T10:36:20.771768+0000","last_peered":"2023-10-09T10:36:20.771768+0000","last_clean":"2023-10-09T10:36:20.771768+0000","last_became_active":"2023-10-09T10:35:58.710741+0000","last_became_peered":"2023-10-09T10:35:58.710741+0000","last_unstale":"2023-10-09T10:36:20.771768+0000","last_undegraded":"2023-10-09T10:36:20.771768+0000","last_fullsized":"2023-10-09T10:36:20.771768+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:25:45.120858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770116+0000","last_change":"2023-10-09T10:35:56.826248+0000","last_active":"2023-10-09T10:36:20.770116+0000","last_peered":"2023-10-09T10:36:20.770116+0000","last_clean":"2023-10-09T10:36:20.770116+0000","last_became_active":"2023-10-09T10:35:56.825932+0000","last_became_peered":"2023-10-09T10:35:56.825932+0000","last_unstale":"2023-10-09T10:36:20.770116+0000","last_undegraded":"2023-10-09T10:36:20.770116+0000","last_fullsized":"2023-10-09T10:36:20.770116+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:24:40.951251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765334+0000","last_change":"2023-10-09T10:35:57.165538+0000","last_active":"2023-10-09T10:36:20.765334+0000","last_peered":"2023-10-09T10:36:20.765334+0000","last_clean":"2023-10-09T10:36:20.765334+0000","last_became_active":"2023-10-09T10:35:57.165187+0000","last_became_peered":"2023-10-09T10:35:57.165187+0000","last_unstale":"2023-10-09T10:36:20.765334+0000","last_undegraded":"2023-10-09T10:36:20.765334+0000","last_fullsized":"2023-10-09T10:36:20.765334+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:53.035298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765892+0000","last_change":"2023-10-09T10:35:56.820965+0000","last_active":"2023-10-09T10:36:20.765892+0000","last_peered":"2023-10-09T10:36:20.765892+0000","last_clean":"2023-10-09T10:36:20.765892+0000","last_became_active":"2023-10-09T10:35:56.820615+0000","last_became_peered":"2023-10-09T10:35:56.820615+0000","last_unstale":"2023-10-09T10:36:20.765892+0000","last_undegraded":"2023-10-09T10:36:20.765892+0000","last_fullsized":"2023-10-09T10:36:20.765892+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:05.292205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.971122+0000","last_change":"2023-10-09T10:35:56.838233+0000","last_active":"2023-10-09T10:36:20.971122+0000","last_peered":"2023-10-09T10:36:20.971122+0000","last_clean":"2023-10-09T10:36:20.971122+0000","last_became_active":"2023-10-09T10:35:56.837911+0000","last_became_peered":"2023-10-09T10:35:56.837911+0000","last_unstale":"2023-10-09T10:36:20.971122+0000","last_undegraded":"2023-10-09T10:36:20.971122+0000","last_fullsized":"2023-10-09T10:36:20.971122+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:58.465137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.772920+0000","last_change":"2023-10-09T10:35:56.822899+0000","last_active":"2023-10-09T10:36:20.772920+0000","last_peered":"2023-10-09T10:36:20.772920+0000","last_clean":"2023-10-09T10:36:20.772920+0000","last_became_active":"2023-10-09T10:35:56.822564+0000","last_became_peered":"2023-10-09T10:35:56.822564+0000","last_unstale":"2023-10-09T10:36:20.772920+0000","last_undegraded":"2023-10-09T10:36:20.772920+0000","last_fullsized":"2023-10-09T10:36:20.772920+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:25:03.398416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-10-09T10:36:21.887280+0000","last_change":"2023-10-09T10:36:21.887280+0000","last_active":"2023-10-09T10:36:21.887280+0000","last_peered":"2023-10-09T10:36:21.887280+0000","last_clean":"2023-10-09T10:36:21.887280+0000","last_became_active":"2023-10-09T10:36:21.886938+0000","last_became_peered":"2023-10-09T10:36:21.886938+0000","last_unstale":"2023-10-09T10:36:21.887280+0000","last_undegraded":"2023-10-09T10:36:21.887280+0000","last_fullsized":"2023-10-09T10:36:21.887280+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:27:07.146829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.773425+0000","last_change":"2023-10-09T10:35:56.823663+0000","last_active":"2023-10-09T10:36:20.773425+0000","last_peered":"2023-10-09T10:36:20.773425+0000","last_clean":"2023-10-09T10:36:20.773425+0000","last_became_active":"2023-10-09T10:35:56.823345+0000","last_became_peered":"2023-10-09T10:35:56.823345+0000","last_unstale":"2023-10-09T10:36:20.773425+0000","last_undegraded":"2023-10-09T10:36:20.773425+0000","last_fullsized":"2023-10-09T10:36:20.773425+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:36:39.917373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765757+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:36:20.765757+0000","last_peered":"2023-10-09T10:36:20.765757+0000","last_clean":"2023-10-09T10:36:20.765757+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:36:20.765757+0000","last_undegraded":"2023-10-09T10:36:20.765757+0000","last_fullsized":"2023-10-09T10:36:20.765757+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.773516+0000","last_change":"2023-10-09T10:35:56.827477+0000","last_active":"2023-10-09T10:36:20.773516+0000","last_peered":"2023-10-09T10:36:20.773516+0000","last_clean":"2023-10-09T10:36:20.773516+0000","last_became_active":"2023-10-09T10:35:56.827078+0000","last_became_peered":"2023-10-09T10:35:56.827078+0000","last_unstale":"2023-10-09T10:36:20.773516+0000","last_undegraded":"2023-10-09T10:36:20.773516+0000","last_fullsized":"2023-10-09T10:36:20.773516+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:29:01.299890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770217+0000","last_change":"2023-10-09T10:35:56.839916+0000","last_active":"2023-10-09T10:36:20.770217+0000","last_peered":"2023-10-09T10:36:20.770217+0000","last_clean":"2023-10-09T10:36:20.770217+0000","last_became_active":"2023-10-09T10:35:56.839605+0000","last_became_peered":"2023-10-09T10:35:56.839605+0000","last_unstale":"2023-10-09T10:36:20.770217+0000","last_undegraded":"2023-10-09T10:36:20.770217+0000","last_fullsized":"2023-10-09T10:36:20.770217+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:24.801950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":22,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.970877+0000","last_change":"2023-10-09T10:35:56.814073+0000","last_active":"2023-10-09T10:36:20.970877+0000","last_peered":"2023-10-09T10:36:20.970877+0000","last_clean":"2023-10-09T10:36:20.970877+0000","last_became_active":"2023-10-09T10:35:56.813780+0000","last_became_peered":"2023-10-09T10:35:56.813780+0000","last_unstale":"2023-10-09T10:36:20.970877+0000","last_undegraded":"2023-10-09T10:36:20.970877+0000","last_fullsized":"2023-10-09T10:36:20.970877+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:04:22.997463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765709+0000","last_change":"2023-10-09T10:35:56.837722+0000","last_active":"2023-10-09T10:36:20.765709+0000","last_peered":"2023-10-09T10:36:20.765709+0000","last_clean":"2023-10-09T10:36:20.765709+0000","last_became_active":"2023-10-09T10:35:56.837445+0000","last_became_peered":"2023-10-09T10:35:56.837445+0000","last_unstale":"2023-10-09T10:36:20.765709+0000","last_undegraded":"2023-10-09T10:36:20.765709+0000","last_fullsized":"2023-10-09T10:36:20.765709+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:02:33.282785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.772389+0000","last_change":"2023-10-09T10:35:56.824257+0000","last_active":"2023-10-09T10:36:20.772389+0000","last_peered":"2023-10-09T10:36:20.772389+0000","last_clean":"2023-10-09T10:36:20.772389+0000","last_became_active":"2023-10-09T10:35:56.823152+0000","last_became_peered":"2023-10-09T10:35:56.823152+0000","last_unstale":"2023-10-09T10:36:20.772389+0000","last_undegraded":"2023-10-09T10:36:20.772389+0000","last_fullsized":"2023-10-09T10:36:20.772389+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:52.392494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.766024+0000","last_change":"2023-10-09T10:35:58.173629+0000","last_active":"2023-10-09T10:36:20.766024+0000","last_peered":"2023-10-09T10:36:20.766024+0000","last_clean":"2023-10-09T10:36:20.766024+0000","last_became_active":"2023-10-09T10:35:58.173331+0000","last_became_peered":"2023-10-09T10:35:58.173331+0000","last_unstale":"2023-10-09T10:36:20.766024+0000","last_undegraded":"2023-10-09T10:36:20.766024+0000","last_fullsized":"2023-10-09T10:36:20.766024+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:13.102708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765306+0000","last_change":"2023-10-09T10:35:56.812650+0000","last_active":"2023-10-09T10:36:20.765306+0000","last_peered":"2023-10-09T10:36:20.765306+0000","last_clean":"2023-10-09T10:36:20.765306+0000","last_became_active":"2023-10-09T10:35:56.812327+0000","last_became_peered":"2023-10-09T10:35:56.812327+0000","last_unstale":"2023-10-09T10:36:20.765306+0000","last_undegraded":"2023-10-09T10:36:20.765306+0000","last_fullsized":"2023-10-09T10:36:20.765306+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:33.929027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-10-09T10:36:21.887125+0000","last_change":"2023-10-09T10:36:21.887125+0000","last_active":"2023-10-09T10:36:21.887125+0000","last_peered":"2023-10-09T10:36:21.887125+0000","last_clean":"2023-10-09T10:36:21.887125+0000","last_became_active":"2023-10-09T10:36:21.886811+0000","last_became_peered":"2023-10-09T10:36:21.886811+0000","last_unstale":"2023-10-09T10:36:21.887125+0000","last_undegraded":"2023-10-09T10:36:21.887125+0000","last_fullsized":"2023-10-09T10:36:21.887125+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:08.364391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.772659+0000","last_change":"2023-10-09T10:35:56.823702+0000","last_active":"2023-10-09T10:36:20.772659+0000","last_peered":"2023-10-09T10:36:20.772659+0000","last_clean":"2023-10-09T10:36:20.772659+0000","last_became_active":"2023-10-09T10:35:56.823401+0000","last_became_peered":"2023-10-09T10:35:56.823401+0000","last_unstale":"2023-10-09T10:36:20.772659+0000","last_undegraded":"2023-10-09T10:36:20.772659+0000","last_fullsized":"2023-10-09T10:36:20.772659+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:13:29.588613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771601+0000","last_change":"2023-10-09T10:35:58.711035+0000","last_active":"2023-10-09T10:36:20.771601+0000","last_peered":"2023-10-09T10:36:20.771601+0000","last_clean":"2023-10-09T10:36:20.771601+0000","last_became_active":"2023-10-09T10:35:58.710703+0000","last_became_peered":"2023-10-09T10:35:58.710703+0000","last_unstale":"2023-10-09T10:36:20.771601+0000","last_undegraded":"2023-10-09T10:36:20.771601+0000","last_fullsized":"2023-10-09T10:36:20.771601+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:46:10.223117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"64'6","reported_seq":20,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771885+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:36:20.771885+0000","last_peered":"2023-10-09T10:36:20.771885+0000","last_clean":"2023-10-09T10:36:20.771885+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:36:20.771885+0000","last_undegraded":"2023-10-09T10:36:20.771885+0000","last_fullsized":"2023-10-09T10:36:20.771885+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768231+0000","last_change":"2023-10-09T10:35:56.837977+0000","last_active":"2023-10-09T10:36:20.768231+0000","last_peered":"2023-10-09T10:36:20.768231+0000","last_clean":"2023-10-09T10:36:20.768231+0000","last_became_active":"2023-10-09T10:35:56.837717+0000","last_became_peered":"2023-10-09T10:35:56.837717+0000","last_unstale":"2023-10-09T10:36:20.768231+0000","last_undegraded":"2023-10-09T10:36:20.768231+0000","last_fullsized":"2023-10-09T10:36:20.768231+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:57.707641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":17,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.772794+0000","last_change":"2023-10-09T10:35:56.824846+0000","last_active":"2023-10-09T10:36:20.772794+0000","last_peered":"2023-10-09T10:36:20.772794+0000","last_clean":"2023-10-09T10:36:20.772794+0000","last_became_active":"2023-10-09T10:35:56.824507+0000","last_became_peered":"2023-10-09T10:35:56.824507+0000","last_unstale":"2023-10-09T10:36:20.772794+0000","last_undegraded":"2023-10-09T10:36:20.772794+0000","last_fullsized":"2023-10-09T10:36:20.772794+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:53:20.462356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770196+0000","last_change":"2023-10-09T10:35:56.839600+0000","last_active":"2023-10-09T10:36:20.770196+0000","last_peered":"2023-10-09T10:36:20.770196+0000","last_clean":"2023-10-09T10:36:20.770196+0000","last_became_active":"2023-10-09T10:35:56.839326+0000","last_became_peered":"2023-10-09T10:35:56.839326+0000","last_unstale":"2023-10-09T10:36:20.770196+0000","last_undegraded":"2023-10-09T10:36:20.770196+0000","last_fullsized":"2023-10-09T10:36:20.770196+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:26.287940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.766081+0000","last_change":"2023-10-09T10:35:56.820752+0000","last_active":"2023-10-09T10:36:20.766081+0000","last_peered":"2023-10-09T10:36:20.766081+0000","last_clean":"2023-10-09T10:36:20.766081+0000","last_became_active":"2023-10-09T10:35:56.819748+0000","last_became_peered":"2023-10-09T10:35:56.819748+0000","last_unstale":"2023-10-09T10:36:20.766081+0000","last_undegraded":"2023-10-09T10:36:20.766081+0000","last_fullsized":"2023-10-09T10:36:20.766081+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:22:00.092032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769970+0000","last_change":"2023-10-09T10:35:56.840607+0000","last_active":"2023-10-09T10:36:20.769970+0000","last_peered":"2023-10-09T10:36:20.769970+0000","last_clean":"2023-10-09T10:36:20.769970+0000","last_became_active":"2023-10-09T10:35:56.840273+0000","last_became_peered":"2023-10-09T10:35:56.840273+0000","last_unstale":"2023-10-09T10:36:20.769970+0000","last_undegraded":"2023-10-09T10:36:20.769970+0000","last_fullsized":"2023-10-09T10:36:20.769970+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:30.114050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765762+0000","last_change":"2023-10-09T10:35:56.820150+0000","last_active":"2023-10-09T10:36:20.765762+0000","last_peered":"2023-10-09T10:36:20.765762+0000","last_clean":"2023-10-09T10:36:20.765762+0000","last_became_active":"2023-10-09T10:35:56.819873+0000","last_became_peered":"2023-10-09T10:35:56.819873+0000","last_unstale":"2023-10-09T10:36:20.765762+0000","last_undegraded":"2023-10-09T10:36:20.765762+0000","last_fullsized":"2023-10-09T10:36:20.765762+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:41.465182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.971774+0000","last_change":"2023-10-09T10:35:57.228230+0000","last_active":"2023-10-09T10:36:20.971774+0000","last_peered":"2023-10-09T10:36:20.971774+0000","last_clean":"2023-10-09T10:36:20.971774+0000","last_became_active":"2023-10-09T10:35:57.227863+0000","last_became_peered":"2023-10-09T10:35:57.227863+0000","last_unstale":"2023-10-09T10:36:20.971774+0000","last_undegraded":"2023-10-09T10:36:20.971774+0000","last_fullsized":"2023-10-09T10:36:20.971774+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:27.674913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769994+0000","last_change":"2023-10-09T10:35:56.818870+0000","last_active":"2023-10-09T10:36:20.769994+0000","last_peered":"2023-10-09T10:36:20.769994+0000","last_clean":"2023-10-09T10:36:20.769994+0000","last_became_active":"2023-10-09T10:35:56.818589+0000","last_became_peered":"2023-10-09T10:35:56.818589+0000","last_unstale":"2023-10-09T10:36:20.769994+0000","last_undegraded":"2023-10-09T10:36:20.769994+0000","last_fullsized":"2023-10-09T10:36:20.769994+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:27:54.174201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765787+0000","last_change":"2023-10-09T10:35:56.837777+0000","last_active":"2023-10-09T10:36:20.765787+0000","last_peered":"2023-10-09T10:36:20.765787+0000","last_clean":"2023-10-09T10:36:20.765787+0000","last_became_active":"2023-10-09T10:35:56.837479+0000","last_became_peered":"2023-10-09T10:35:56.837479+0000","last_unstale":"2023-10-09T10:36:20.765787+0000","last_undegraded":"2023-10-09T10:36:20.765787+0000","last_fullsized":"2023-10-09T10:36:20.765787+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:00.136014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768652+0000","last_change":"2023-10-09T10:35:56.824512+0000","last_active":"2023-10-09T10:36:20.768652+0000","last_peered":"2023-10-09T10:36:20.768652+0000","last_clean":"2023-10-09T10:36:20.768652+0000","last_became_active":"2023-10-09T10:35:56.823293+0000","last_became_peered":"2023-10-09T10:35:56.823293+0000","last_unstale":"2023-10-09T10:36:20.768652+0000","last_undegraded":"2023-10-09T10:36:20.768652+0000","last_fullsized":"2023-10-09T10:36:20.768652+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:34.986666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770727+0000","last_change":"2023-10-09T10:35:57.165630+0000","last_active":"2023-10-09T10:36:20.770727+0000","last_peered":"2023-10-09T10:36:20.770727+0000","last_clean":"2023-10-09T10:36:20.770727+0000","last_became_active":"2023-10-09T10:35:57.165339+0000","last_became_peered":"2023-10-09T10:35:57.165339+0000","last_unstale":"2023-10-09T10:36:20.770727+0000","last_undegraded":"2023-10-09T10:36:20.770727+0000","last_fullsized":"2023-10-09T10:36:20.770727+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:08.227472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771569+0000","last_change":"2023-10-09T10:35:56.823266+0000","last_active":"2023-10-09T10:36:20.771569+0000","last_peered":"2023-10-09T10:36:20.771569+0000","last_clean":"2023-10-09T10:36:20.771569+0000","last_became_active":"2023-10-09T10:35:56.822981+0000","last_became_peered":"2023-10-09T10:35:56.822981+0000","last_unstale":"2023-10-09T10:36:20.771569+0000","last_undegraded":"2023-10-09T10:36:20.771569+0000","last_fullsized":"2023-10-09T10:36:20.771569+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:40.228328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332117,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1260,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1290240,"data_stored":885968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.58599999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.61199999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.56599999999999995}]},{"osd":3,"last update":"Mon Oct 9 10:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.6719999999999999,"5min":1.6719999999999999,"15min":1.6719999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.625}]},{"osd":4,"last update":"Mon Oct 9 10:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.44800000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.432},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.0649999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.47799999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.51400000000000001}]}]},{"osd":6,"up_from":44,"seq":188978561048,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298536,"kb_used_data":1248,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446616,"statfs":{"total":95995035648,"available":95689334784,"internally_reserved":0,"allocated":1277952,"data_stored":885822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.68500000000000005}]},{"osd":1,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.72999999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.64200000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.747},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.80400000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.46800000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.378}]},{"osd":7,"last update":"Mon Oct 9 10:36:02 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.442},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":0.42799999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574890,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297940,"kb_used_data":652,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447212,"statfs":{"total":95995035648,"available":95689945088,"internally_reserved":0,"allocated":667648,"data_stored":293906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:36:20 2023","interfaces":[{"interface":"back","average":{"1min":0.308,"5min":0.35999999999999999,"15min":0.36099999999999999},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":0.46800000000000003,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.254},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.36599999999999999,"15min":0.37},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.38500000000000001,"5min":0.622,"15min":0.622},"last":0.26700000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:35:35 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.42899999999999999,"15min":0.435},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.54400000000000004,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.435,"15min":0.438},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.30199999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.437,"15min":0.44500000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.48199999999999998,"15min":0.49399999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.56699999999999995,"5min":0.625,"15min":0.625},"last":0.28899999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:36:10 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.57399999999999995,"15min":0.58799999999999997},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.63,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.57299999999999995,"15min":0.58299999999999996},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.82199999999999995,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.37}]},{"osd":5,"last update":"Mon Oct 9 10:36:25 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.57899999999999996,"15min":0.58999999999999997},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.73199999999999998,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.57899999999999996,"15min":0.58899999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.64500000000000002,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.54400000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:35:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.63},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.57399999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.40600000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738411,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298532,"kb_used_data":1244,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446620,"statfs":{"total":95995035648,"available":95689338880,"internally_reserved":0,"allocated":1273856,"data_stored":885115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:36:24 2023","interfaces":[{"interface":"back","average":{"1min":0.32600000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.22800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":0.46800000000000003,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.23100000000000001},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.39700000000000002,"15min":0.38900000000000001},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.59299999999999997,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.59299999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.29999999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.37},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.42199999999999999,"15min":0.42499999999999999},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.29699999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:35:53 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.49399999999999999,"15min":0.499},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.181,"5min":1.181,"15min":1.181},"last":0.312},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.48299999999999998,"15min":0.49199999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.55200000000000005,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.46100000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.53000000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:36:24 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.57199999999999995,"15min":0.57399999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.61599999999999999,"15min":0.622},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.628}]},{"osd":6,"last update":"Mon Oct 9 10:35:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.48299999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.54300000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411364,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297952,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447200,"statfs":{"total":95995035648,"available":95689932800,"internally_reserved":0,"allocated":679936,"data_stored":294759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.36099999999999999,"15min":0.35899999999999999},"min":{"1min":0.255,"5min":0.22,"15min":0.22},"max":{"1min":0.57799999999999996,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.25800000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.38900000000000001,"15min":0.38700000000000001},"min":{"1min":0.255,"5min":0.246,"15min":0.246},"max":{"1min":0.55800000000000005,"5min":0.63,"15min":0.63},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.41099999999999998,"15min":0.40899999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.33800000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.46600000000000003,"15min":0.47199999999999998},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.63,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.436,"15min":0.44},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.34899999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:36:10 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.57299999999999995,"15min":0.57599999999999996},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.60499999999999998,"15min":0.61099999999999999},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.78400000000000003,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.64800000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:35:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.628}]},{"osd":6,"last update":"Mon Oct 9 10:35:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":2.0339999999999998,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.59199999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.67500000000000004}]}]},{"osd":3,"up_from":25,"seq":107374182435,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297948,"kb_used_data":660,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447204,"statfs":{"total":95995035648,"available":95689936896,"internally_reserved":0,"allocated":675840,"data_stored":294261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:54 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.39700000000000002,"15min":0.39900000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.45100000000000001,"15min":0.46300000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.54700000000000004,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.35399999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:35:54 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.47199999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.57399999999999995,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.307},{"interface":"front","average":{"1min":0.435,"5min":0.44,"15min":0.441},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.28999999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:35:54 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.48599999999999999,"15min":0.495},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.47099999999999997,"15min":0.47699999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.33700000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:36:14 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.55800000000000005,"15min":0.56799999999999995},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.57199999999999995,"15min":0.57799999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.55000000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:36:23 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.59099999999999997,"15min":0.57199999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":2.9580000000000002,"5min":2.9580000000000002,"15min":2.9580000000000002},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.60299999999999998,"15min":0.58099999999999996},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":3.9399999999999999,"5min":3.9399999999999999,"15min":3.9399999999999999},"last":0.51600000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.8380000000000001,"5min":1.8380000000000001,"15min":1.8380000000000001},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.56999999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":2.8980000000000001,"5min":2.8980000000000001,"15min":2.8980000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.748,"5min":0.748,"15min":0.748},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":3.0169999999999999,"5min":3.0169999999999999,"15min":3.0169999999999999},"last":0.48899999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018911,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":696320,"data_stored":295853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.53300000000000003,"15min":0.54100000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.34799999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.498}]},{"osd":1,"last update":"Mon Oct 9 10:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.52700000000000002,"15min":0.53100000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.623},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.53400000000000003,"15min":0.54100000000000004},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.53900000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.56599999999999995,"15min":0.55800000000000005},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.3530000000000002,"5min":2.3530000000000002,"15min":2.3530000000000002},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.56899999999999995,"15min":0.57699999999999996},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79200000000000004,"5min":0.875,"15min":0.875},"last":0.63500000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56100000000000005,"15min":0.56000000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.5800000000000001,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":0.66500000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:36:26 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.40999999999999998,"15min":0.40300000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.434},{"interface":"front","average":{"1min":0.438,"5min":0.41099999999999998,"15min":0.40699999999999997},"min":{"1min":0.29899999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.44800000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.505},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.52200000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.56399999999999995}]}]},{"osd":5,"up_from":37,"seq":158913789980,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297948,"kb_used_data":660,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447204,"statfs":{"total":95995035648,"available":95689936896,"internally_reserved":0,"allocated":675840,"data_stored":294472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.40500000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.57299999999999995}]},{"osd":2,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.5},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.59899999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.65500000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.505,"5min":0.505,"15min":0.505},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.753}]},{"osd":6,"last update":"Mon Oct 9 10:35:43 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.72799999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.69899999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:36:27.290 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:36:27.554 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:28.021 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:27 smithi125 ceph-mon[112589]: pgmap v45: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:28.021 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:27 smithi125 ceph-mon[112589]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:36:28.022 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:27 smithi125 ceph-mon[127026]: pgmap v45: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:28.022 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:27 smithi125 ceph-mon[127026]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:36:28.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:27 smithi155 ceph-mon[117223]: pgmap v45: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:28.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:27 smithi155 ceph-mon[117223]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:36:28.803 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:36:28.805 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:36:29.005 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:28 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:36:28] "GET /metrics HTTP/1.1" 200 238835 "" "Prometheus/2.43.0" 2023-10-09T10:36:29.184 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":46,"stamp":"2023-10-09T10:36:28.637058+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":218,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385372,"kb_used_data":7068,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747575844,"statfs":{"total":767960285184,"available":765517664256,"internally_reserved":0,"allocated":7237632,"data_stored":4130156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001437"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771446+0000","last_change":"2023-10-09T10:35:58.712102+0000","last_active":"2023-10-09T10:36:20.771446+0000","last_peered":"2023-10-09T10:36:20.771446+0000","last_clean":"2023-10-09T10:36:20.771446+0000","last_became_active":"2023-10-09T10:35:58.711824+0000","last_became_peered":"2023-10-09T10:35:58.711824+0000","last_unstale":"2023-10-09T10:36:20.771446+0000","last_undegraded":"2023-10-09T10:36:20.771446+0000","last_fullsized":"2023-10-09T10:36:20.771446+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:42.811844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765810+0000","last_change":"2023-10-09T10:35:56.814681+0000","last_active":"2023-10-09T10:36:20.765810+0000","last_peered":"2023-10-09T10:36:20.765810+0000","last_clean":"2023-10-09T10:36:20.765810+0000","last_became_active":"2023-10-09T10:35:56.814401+0000","last_became_peered":"2023-10-09T10:35:56.814401+0000","last_unstale":"2023-10-09T10:36:20.765810+0000","last_undegraded":"2023-10-09T10:36:20.765810+0000","last_fullsized":"2023-10-09T10:36:20.765810+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:15:46.517618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770109+0000","last_change":"2023-10-09T10:35:56.836445+0000","last_active":"2023-10-09T10:36:20.770109+0000","last_peered":"2023-10-09T10:36:20.770109+0000","last_clean":"2023-10-09T10:36:20.770109+0000","last_became_active":"2023-10-09T10:35:56.836158+0000","last_became_peered":"2023-10-09T10:35:56.836158+0000","last_unstale":"2023-10-09T10:36:20.770109+0000","last_undegraded":"2023-10-09T10:36:20.770109+0000","last_fullsized":"2023-10-09T10:36:20.770109+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:29.431572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770254+0000","last_change":"2023-10-09T10:35:56.815445+0000","last_active":"2023-10-09T10:36:20.770254+0000","last_peered":"2023-10-09T10:36:20.770254+0000","last_clean":"2023-10-09T10:36:20.770254+0000","last_became_active":"2023-10-09T10:35:56.815152+0000","last_became_peered":"2023-10-09T10:35:56.815152+0000","last_unstale":"2023-10-09T10:36:20.770254+0000","last_undegraded":"2023-10-09T10:36:20.770254+0000","last_fullsized":"2023-10-09T10:36:20.770254+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:04.416880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771581+0000","last_change":"2023-10-09T10:35:58.242519+0000","last_active":"2023-10-09T10:36:20.771581+0000","last_peered":"2023-10-09T10:36:20.771581+0000","last_clean":"2023-10-09T10:36:20.771581+0000","last_became_active":"2023-10-09T10:35:58.242240+0000","last_became_peered":"2023-10-09T10:35:58.242240+0000","last_unstale":"2023-10-09T10:36:20.771581+0000","last_undegraded":"2023-10-09T10:36:20.771581+0000","last_fullsized":"2023-10-09T10:36:20.771581+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:26.836816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.766130+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:36:20.766130+0000","last_peered":"2023-10-09T10:36:20.766130+0000","last_clean":"2023-10-09T10:36:20.766130+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:36:20.766130+0000","last_undegraded":"2023-10-09T10:36:20.766130+0000","last_fullsized":"2023-10-09T10:36:20.766130+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771810+0000","last_change":"2023-10-09T10:35:58.130037+0000","last_active":"2023-10-09T10:36:20.771810+0000","last_peered":"2023-10-09T10:36:20.771810+0000","last_clean":"2023-10-09T10:36:20.771810+0000","last_became_active":"2023-10-09T10:35:58.129751+0000","last_became_peered":"2023-10-09T10:35:58.129751+0000","last_unstale":"2023-10-09T10:36:20.771810+0000","last_undegraded":"2023-10-09T10:36:20.771810+0000","last_fullsized":"2023-10-09T10:36:20.771810+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:53.201853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.766006+0000","last_change":"2023-10-09T10:35:56.821768+0000","last_active":"2023-10-09T10:36:20.766006+0000","last_peered":"2023-10-09T10:36:20.766006+0000","last_clean":"2023-10-09T10:36:20.766006+0000","last_became_active":"2023-10-09T10:35:56.821452+0000","last_became_peered":"2023-10-09T10:35:56.821452+0000","last_unstale":"2023-10-09T10:36:20.766006+0000","last_undegraded":"2023-10-09T10:36:20.766006+0000","last_fullsized":"2023-10-09T10:36:20.766006+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:32.659779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769165+0000","last_change":"2023-10-09T10:35:56.841741+0000","last_active":"2023-10-09T10:36:20.769165+0000","last_peered":"2023-10-09T10:36:20.769165+0000","last_clean":"2023-10-09T10:36:20.769165+0000","last_became_active":"2023-10-09T10:35:56.841473+0000","last_became_peered":"2023-10-09T10:35:56.841473+0000","last_unstale":"2023-10-09T10:36:20.769165+0000","last_undegraded":"2023-10-09T10:36:20.769165+0000","last_fullsized":"2023-10-09T10:36:20.769165+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:17:07.385347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765589+0000","last_change":"2023-10-09T10:35:56.817791+0000","last_active":"2023-10-09T10:36:20.765589+0000","last_peered":"2023-10-09T10:36:20.765589+0000","last_clean":"2023-10-09T10:36:20.765589+0000","last_became_active":"2023-10-09T10:35:56.817533+0000","last_became_peered":"2023-10-09T10:35:56.817533+0000","last_unstale":"2023-10-09T10:36:20.765589+0000","last_undegraded":"2023-10-09T10:36:20.765589+0000","last_fullsized":"2023-10-09T10:36:20.765589+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:20.056489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.773113+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:36:20.773113+0000","last_peered":"2023-10-09T10:36:20.773113+0000","last_clean":"2023-10-09T10:36:20.773113+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:36:20.773113+0000","last_undegraded":"2023-10-09T10:36:20.773113+0000","last_fullsized":"2023-10-09T10:36:20.773113+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765873+0000","last_change":"2023-10-09T10:35:56.817387+0000","last_active":"2023-10-09T10:36:20.765873+0000","last_peered":"2023-10-09T10:36:20.765873+0000","last_clean":"2023-10-09T10:36:20.765873+0000","last_became_active":"2023-10-09T10:35:56.817101+0000","last_became_peered":"2023-10-09T10:35:56.817101+0000","last_unstale":"2023-10-09T10:36:20.765873+0000","last_undegraded":"2023-10-09T10:36:20.765873+0000","last_fullsized":"2023-10-09T10:36:20.765873+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:50.703008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.971795+0000","last_change":"2023-10-09T10:35:20.619783+0000","last_active":"2023-10-09T10:36:20.971795+0000","last_peered":"2023-10-09T10:36:20.971795+0000","last_clean":"2023-10-09T10:36:20.971795+0000","last_became_active":"2023-10-09T10:35:20.619441+0000","last_became_peered":"2023-10-09T10:35:20.619441+0000","last_unstale":"2023-10-09T10:36:20.971795+0000","last_undegraded":"2023-10-09T10:36:20.971795+0000","last_fullsized":"2023-10-09T10:36:20.971795+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:00.454996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769880+0000","last_change":"2023-10-09T10:35:56.840463+0000","last_active":"2023-10-09T10:36:20.769880+0000","last_peered":"2023-10-09T10:36:20.769880+0000","last_clean":"2023-10-09T10:36:20.769880+0000","last_became_active":"2023-10-09T10:35:56.839880+0000","last_became_peered":"2023-10-09T10:35:56.839880+0000","last_unstale":"2023-10-09T10:36:20.769880+0000","last_undegraded":"2023-10-09T10:36:20.769880+0000","last_fullsized":"2023-10-09T10:36:20.769880+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:38.048996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769557+0000","last_change":"2023-10-09T10:35:20.621958+0000","last_active":"2023-10-09T10:36:20.769557+0000","last_peered":"2023-10-09T10:36:20.769557+0000","last_clean":"2023-10-09T10:36:20.769557+0000","last_became_active":"2023-10-09T10:35:20.621656+0000","last_became_peered":"2023-10-09T10:35:20.621656+0000","last_unstale":"2023-10-09T10:36:20.769557+0000","last_undegraded":"2023-10-09T10:36:20.769557+0000","last_fullsized":"2023-10-09T10:36:20.769557+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:09:32.121613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770352+0000","last_change":"2023-10-09T10:35:56.820393+0000","last_active":"2023-10-09T10:36:20.770352+0000","last_peered":"2023-10-09T10:36:20.770352+0000","last_clean":"2023-10-09T10:36:20.770352+0000","last_became_active":"2023-10-09T10:35:56.820157+0000","last_became_peered":"2023-10-09T10:35:56.820157+0000","last_unstale":"2023-10-09T10:36:20.770352+0000","last_undegraded":"2023-10-09T10:36:20.770352+0000","last_fullsized":"2023-10-09T10:36:20.770352+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:35.968592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765424+0000","last_change":"2023-10-09T10:35:57.228099+0000","last_active":"2023-10-09T10:36:20.765424+0000","last_peered":"2023-10-09T10:36:20.765424+0000","last_clean":"2023-10-09T10:36:20.765424+0000","last_became_active":"2023-10-09T10:35:57.227754+0000","last_became_peered":"2023-10-09T10:35:57.227754+0000","last_unstale":"2023-10-09T10:36:20.765424+0000","last_undegraded":"2023-10-09T10:36:20.765424+0000","last_fullsized":"2023-10-09T10:36:20.765424+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:23.783778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768886+0000","last_change":"2023-10-09T10:35:56.816296+0000","last_active":"2023-10-09T10:36:20.768886+0000","last_peered":"2023-10-09T10:36:20.768886+0000","last_clean":"2023-10-09T10:36:20.768886+0000","last_became_active":"2023-10-09T10:35:56.816012+0000","last_became_peered":"2023-10-09T10:35:56.816012+0000","last_unstale":"2023-10-09T10:36:20.768886+0000","last_undegraded":"2023-10-09T10:36:20.768886+0000","last_fullsized":"2023-10-09T10:36:20.768886+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:16.247390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.772893+0000","last_change":"2023-10-09T10:35:20.619239+0000","last_active":"2023-10-09T10:36:20.772893+0000","last_peered":"2023-10-09T10:36:20.772893+0000","last_clean":"2023-10-09T10:36:20.772893+0000","last_became_active":"2023-10-09T10:35:20.618892+0000","last_became_peered":"2023-10-09T10:35:20.618892+0000","last_unstale":"2023-10-09T10:36:20.772893+0000","last_undegraded":"2023-10-09T10:36:20.772893+0000","last_fullsized":"2023-10-09T10:36:20.772893+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:55.019433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-10-09T10:36:21.886869+0000","last_change":"2023-10-09T10:36:21.886869+0000","last_active":"2023-10-09T10:36:21.886869+0000","last_peered":"2023-10-09T10:36:21.886869+0000","last_clean":"2023-10-09T10:36:21.886869+0000","last_became_active":"2023-10-09T10:36:21.886559+0000","last_became_peered":"2023-10-09T10:36:21.886559+0000","last_unstale":"2023-10-09T10:36:21.886869+0000","last_undegraded":"2023-10-09T10:36:21.886869+0000","last_fullsized":"2023-10-09T10:36:21.886869+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:12.576014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.764457+0000","last_change":"2023-10-09T10:35:56.820290+0000","last_active":"2023-10-09T10:36:20.764457+0000","last_peered":"2023-10-09T10:36:20.764457+0000","last_clean":"2023-10-09T10:36:20.764457+0000","last_became_active":"2023-10-09T10:35:56.819986+0000","last_became_peered":"2023-10-09T10:35:56.819986+0000","last_unstale":"2023-10-09T10:36:20.764457+0000","last_undegraded":"2023-10-09T10:36:20.764457+0000","last_fullsized":"2023-10-09T10:36:20.764457+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:31.721604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.772389+0000","last_change":"2023-10-09T10:35:20.619992+0000","last_active":"2023-10-09T10:36:20.772389+0000","last_peered":"2023-10-09T10:36:20.772389+0000","last_clean":"2023-10-09T10:36:20.772389+0000","last_became_active":"2023-10-09T10:35:20.619719+0000","last_became_peered":"2023-10-09T10:35:20.619719+0000","last_unstale":"2023-10-09T10:36:20.772389+0000","last_undegraded":"2023-10-09T10:36:20.772389+0000","last_fullsized":"2023-10-09T10:36:20.772389+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:31.964380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.971770+0000","last_change":"2023-10-09T10:35:57.165395+0000","last_active":"2023-10-09T10:36:20.971770+0000","last_peered":"2023-10-09T10:36:20.971770+0000","last_clean":"2023-10-09T10:36:20.971770+0000","last_became_active":"2023-10-09T10:35:57.165077+0000","last_became_peered":"2023-10-09T10:35:57.165077+0000","last_unstale":"2023-10-09T10:36:20.971770+0000","last_undegraded":"2023-10-09T10:36:20.971770+0000","last_fullsized":"2023-10-09T10:36:20.971770+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:53.159616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770024+0000","last_change":"2023-10-09T10:35:56.818871+0000","last_active":"2023-10-09T10:36:20.770024+0000","last_peered":"2023-10-09T10:36:20.770024+0000","last_clean":"2023-10-09T10:36:20.770024+0000","last_became_active":"2023-10-09T10:35:56.818634+0000","last_became_peered":"2023-10-09T10:35:56.818634+0000","last_unstale":"2023-10-09T10:36:20.770024+0000","last_undegraded":"2023-10-09T10:36:20.770024+0000","last_fullsized":"2023-10-09T10:36:20.770024+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:14.248324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"57'128","reported_seq":277,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.772317+0000","last_change":"2023-10-09T10:35:01.491534+0000","last_active":"2023-10-09T10:36:20.772317+0000","last_peered":"2023-10-09T10:36:20.772317+0000","last_clean":"2023-10-09T10:36:20.772317+0000","last_became_active":"2023-10-09T10:35:01.491048+0000","last_became_peered":"2023-10-09T10:35:01.491048+0000","last_unstale":"2023-10-09T10:36:20.772317+0000","last_undegraded":"2023-10-09T10:36:20.772317+0000","last_fullsized":"2023-10-09T10:36:20.772317+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:00:35.672924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765817+0000","last_change":"2023-10-09T10:35:20.622020+0000","last_active":"2023-10-09T10:36:20.765817+0000","last_peered":"2023-10-09T10:36:20.765817+0000","last_clean":"2023-10-09T10:36:20.765817+0000","last_became_active":"2023-10-09T10:35:20.621703+0000","last_became_peered":"2023-10-09T10:35:20.621703+0000","last_unstale":"2023-10-09T10:36:20.765817+0000","last_undegraded":"2023-10-09T10:36:20.765817+0000","last_fullsized":"2023-10-09T10:36:20.765817+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:59:34.492019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769021+0000","last_change":"2023-10-09T10:35:57.228907+0000","last_active":"2023-10-09T10:36:20.769021+0000","last_peered":"2023-10-09T10:36:20.769021+0000","last_clean":"2023-10-09T10:36:20.769021+0000","last_became_active":"2023-10-09T10:35:57.228408+0000","last_became_peered":"2023-10-09T10:35:57.228408+0000","last_unstale":"2023-10-09T10:36:20.769021+0000","last_undegraded":"2023-10-09T10:36:20.769021+0000","last_fullsized":"2023-10-09T10:36:20.769021+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:03.936095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768713+0000","last_change":"2023-10-09T10:35:56.814727+0000","last_active":"2023-10-09T10:36:20.768713+0000","last_peered":"2023-10-09T10:36:20.768713+0000","last_clean":"2023-10-09T10:36:20.768713+0000","last_became_active":"2023-10-09T10:35:56.813636+0000","last_became_peered":"2023-10-09T10:35:56.813636+0000","last_unstale":"2023-10-09T10:36:20.768713+0000","last_undegraded":"2023-10-09T10:36:20.768713+0000","last_fullsized":"2023-10-09T10:36:20.768713+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:49.853322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769493+0000","last_change":"2023-10-09T10:35:20.625994+0000","last_active":"2023-10-09T10:36:20.769493+0000","last_peered":"2023-10-09T10:36:20.769493+0000","last_clean":"2023-10-09T10:36:20.769493+0000","last_became_active":"2023-10-09T10:35:20.625638+0000","last_became_peered":"2023-10-09T10:35:20.625638+0000","last_unstale":"2023-10-09T10:36:20.769493+0000","last_undegraded":"2023-10-09T10:36:20.769493+0000","last_fullsized":"2023-10-09T10:36:20.769493+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:26.622626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768881+0000","last_change":"2023-10-09T10:35:56.840023+0000","last_active":"2023-10-09T10:36:20.768881+0000","last_peered":"2023-10-09T10:36:20.768881+0000","last_clean":"2023-10-09T10:36:20.768881+0000","last_became_active":"2023-10-09T10:35:56.839737+0000","last_became_peered":"2023-10-09T10:35:56.839737+0000","last_unstale":"2023-10-09T10:36:20.768881+0000","last_undegraded":"2023-10-09T10:36:20.768881+0000","last_fullsized":"2023-10-09T10:36:20.768881+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:20.699635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771705+0000","last_change":"2023-10-09T10:35:56.820004+0000","last_active":"2023-10-09T10:36:20.771705+0000","last_peered":"2023-10-09T10:36:20.771705+0000","last_clean":"2023-10-09T10:36:20.771705+0000","last_became_active":"2023-10-09T10:35:56.819736+0000","last_became_peered":"2023-10-09T10:35:56.819736+0000","last_unstale":"2023-10-09T10:36:20.771705+0000","last_undegraded":"2023-10-09T10:36:20.771705+0000","last_fullsized":"2023-10-09T10:36:20.771705+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:31:07.010228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765850+0000","last_change":"2023-10-09T10:35:20.621825+0000","last_active":"2023-10-09T10:36:20.765850+0000","last_peered":"2023-10-09T10:36:20.765850+0000","last_clean":"2023-10-09T10:36:20.765850+0000","last_became_active":"2023-10-09T10:35:20.621526+0000","last_became_peered":"2023-10-09T10:35:20.621526+0000","last_unstale":"2023-10-09T10:36:20.765850+0000","last_undegraded":"2023-10-09T10:36:20.765850+0000","last_fullsized":"2023-10-09T10:36:20.765850+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:59.814738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768308+0000","last_change":"2023-10-09T10:35:57.228124+0000","last_active":"2023-10-09T10:36:20.768308+0000","last_peered":"2023-10-09T10:36:20.768308+0000","last_clean":"2023-10-09T10:36:20.768308+0000","last_became_active":"2023-10-09T10:35:57.227833+0000","last_became_peered":"2023-10-09T10:35:57.227833+0000","last_unstale":"2023-10-09T10:36:20.768308+0000","last_undegraded":"2023-10-09T10:36:20.768308+0000","last_fullsized":"2023-10-09T10:36:20.768308+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:24.892825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.766101+0000","last_change":"2023-10-09T10:35:56.821198+0000","last_active":"2023-10-09T10:36:20.766101+0000","last_peered":"2023-10-09T10:36:20.766101+0000","last_clean":"2023-10-09T10:36:20.766101+0000","last_became_active":"2023-10-09T10:35:56.820864+0000","last_became_peered":"2023-10-09T10:35:56.820864+0000","last_unstale":"2023-10-09T10:36:20.766101+0000","last_undegraded":"2023-10-09T10:36:20.766101+0000","last_fullsized":"2023-10-09T10:36:20.766101+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:42.990821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768809+0000","last_change":"2023-10-09T10:35:20.621757+0000","last_active":"2023-10-09T10:36:20.768809+0000","last_peered":"2023-10-09T10:36:20.768809+0000","last_clean":"2023-10-09T10:36:20.768809+0000","last_became_active":"2023-10-09T10:35:20.621505+0000","last_became_peered":"2023-10-09T10:35:20.621505+0000","last_unstale":"2023-10-09T10:36:20.768809+0000","last_undegraded":"2023-10-09T10:36:20.768809+0000","last_fullsized":"2023-10-09T10:36:20.768809+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:35.203999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768978+0000","last_change":"2023-10-09T10:35:57.165557+0000","last_active":"2023-10-09T10:36:20.768978+0000","last_peered":"2023-10-09T10:36:20.768978+0000","last_clean":"2023-10-09T10:36:20.768978+0000","last_became_active":"2023-10-09T10:35:57.165215+0000","last_became_peered":"2023-10-09T10:35:57.165215+0000","last_unstale":"2023-10-09T10:36:20.768978+0000","last_undegraded":"2023-10-09T10:36:20.768978+0000","last_fullsized":"2023-10-09T10:36:20.768978+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:45.875941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769903+0000","last_change":"2023-10-09T10:35:56.822872+0000","last_active":"2023-10-09T10:36:20.769903+0000","last_peered":"2023-10-09T10:36:20.769903+0000","last_clean":"2023-10-09T10:36:20.769903+0000","last_became_active":"2023-10-09T10:35:56.822591+0000","last_became_peered":"2023-10-09T10:35:56.822591+0000","last_unstale":"2023-10-09T10:36:20.769903+0000","last_undegraded":"2023-10-09T10:36:20.769903+0000","last_fullsized":"2023-10-09T10:36:20.769903+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:27:57.817744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771825+0000","last_change":"2023-10-09T10:35:58.711128+0000","last_active":"2023-10-09T10:36:20.771825+0000","last_peered":"2023-10-09T10:36:20.771825+0000","last_clean":"2023-10-09T10:36:20.771825+0000","last_became_active":"2023-10-09T10:35:58.710718+0000","last_became_peered":"2023-10-09T10:35:58.710718+0000","last_unstale":"2023-10-09T10:36:20.771825+0000","last_undegraded":"2023-10-09T10:36:20.771825+0000","last_fullsized":"2023-10-09T10:36:20.771825+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:05.436822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.773093+0000","last_change":"2023-10-09T10:35:56.823476+0000","last_active":"2023-10-09T10:36:20.773093+0000","last_peered":"2023-10-09T10:36:20.773093+0000","last_clean":"2023-10-09T10:36:20.773093+0000","last_became_active":"2023-10-09T10:35:56.823138+0000","last_became_peered":"2023-10-09T10:35:56.823138+0000","last_unstale":"2023-10-09T10:36:20.773093+0000","last_undegraded":"2023-10-09T10:36:20.773093+0000","last_fullsized":"2023-10-09T10:36:20.773093+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:18.228816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771880+0000","last_change":"2023-10-09T10:35:58.321123+0000","last_active":"2023-10-09T10:36:20.771880+0000","last_peered":"2023-10-09T10:36:20.771880+0000","last_clean":"2023-10-09T10:36:20.771880+0000","last_became_active":"2023-10-09T10:35:58.320867+0000","last_became_peered":"2023-10-09T10:35:58.320867+0000","last_unstale":"2023-10-09T10:36:20.771880+0000","last_undegraded":"2023-10-09T10:36:20.771880+0000","last_fullsized":"2023-10-09T10:36:20.771880+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:33.609435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.970937+0000","last_change":"2023-10-09T10:35:56.816746+0000","last_active":"2023-10-09T10:36:20.970937+0000","last_peered":"2023-10-09T10:36:20.970937+0000","last_clean":"2023-10-09T10:36:20.970937+0000","last_became_active":"2023-10-09T10:35:56.816395+0000","last_became_peered":"2023-10-09T10:35:56.816395+0000","last_unstale":"2023-10-09T10:36:20.970937+0000","last_undegraded":"2023-10-09T10:36:20.970937+0000","last_fullsized":"2023-10-09T10:36:20.970937+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:24.841288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771768+0000","last_change":"2023-10-09T10:35:58.711019+0000","last_active":"2023-10-09T10:36:20.771768+0000","last_peered":"2023-10-09T10:36:20.771768+0000","last_clean":"2023-10-09T10:36:20.771768+0000","last_became_active":"2023-10-09T10:35:58.710741+0000","last_became_peered":"2023-10-09T10:35:58.710741+0000","last_unstale":"2023-10-09T10:36:20.771768+0000","last_undegraded":"2023-10-09T10:36:20.771768+0000","last_fullsized":"2023-10-09T10:36:20.771768+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:25:45.120858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770116+0000","last_change":"2023-10-09T10:35:56.826248+0000","last_active":"2023-10-09T10:36:20.770116+0000","last_peered":"2023-10-09T10:36:20.770116+0000","last_clean":"2023-10-09T10:36:20.770116+0000","last_became_active":"2023-10-09T10:35:56.825932+0000","last_became_peered":"2023-10-09T10:35:56.825932+0000","last_unstale":"2023-10-09T10:36:20.770116+0000","last_undegraded":"2023-10-09T10:36:20.770116+0000","last_fullsized":"2023-10-09T10:36:20.770116+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:24:40.951251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765334+0000","last_change":"2023-10-09T10:35:57.165538+0000","last_active":"2023-10-09T10:36:20.765334+0000","last_peered":"2023-10-09T10:36:20.765334+0000","last_clean":"2023-10-09T10:36:20.765334+0000","last_became_active":"2023-10-09T10:35:57.165187+0000","last_became_peered":"2023-10-09T10:35:57.165187+0000","last_unstale":"2023-10-09T10:36:20.765334+0000","last_undegraded":"2023-10-09T10:36:20.765334+0000","last_fullsized":"2023-10-09T10:36:20.765334+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:53.035298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765892+0000","last_change":"2023-10-09T10:35:56.820965+0000","last_active":"2023-10-09T10:36:20.765892+0000","last_peered":"2023-10-09T10:36:20.765892+0000","last_clean":"2023-10-09T10:36:20.765892+0000","last_became_active":"2023-10-09T10:35:56.820615+0000","last_became_peered":"2023-10-09T10:35:56.820615+0000","last_unstale":"2023-10-09T10:36:20.765892+0000","last_undegraded":"2023-10-09T10:36:20.765892+0000","last_fullsized":"2023-10-09T10:36:20.765892+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:05.292205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.971122+0000","last_change":"2023-10-09T10:35:56.838233+0000","last_active":"2023-10-09T10:36:20.971122+0000","last_peered":"2023-10-09T10:36:20.971122+0000","last_clean":"2023-10-09T10:36:20.971122+0000","last_became_active":"2023-10-09T10:35:56.837911+0000","last_became_peered":"2023-10-09T10:35:56.837911+0000","last_unstale":"2023-10-09T10:36:20.971122+0000","last_undegraded":"2023-10-09T10:36:20.971122+0000","last_fullsized":"2023-10-09T10:36:20.971122+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:58.465137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.772920+0000","last_change":"2023-10-09T10:35:56.822899+0000","last_active":"2023-10-09T10:36:20.772920+0000","last_peered":"2023-10-09T10:36:20.772920+0000","last_clean":"2023-10-09T10:36:20.772920+0000","last_became_active":"2023-10-09T10:35:56.822564+0000","last_became_peered":"2023-10-09T10:35:56.822564+0000","last_unstale":"2023-10-09T10:36:20.772920+0000","last_undegraded":"2023-10-09T10:36:20.772920+0000","last_fullsized":"2023-10-09T10:36:20.772920+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:25:03.398416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-10-09T10:36:21.887280+0000","last_change":"2023-10-09T10:36:21.887280+0000","last_active":"2023-10-09T10:36:21.887280+0000","last_peered":"2023-10-09T10:36:21.887280+0000","last_clean":"2023-10-09T10:36:21.887280+0000","last_became_active":"2023-10-09T10:36:21.886938+0000","last_became_peered":"2023-10-09T10:36:21.886938+0000","last_unstale":"2023-10-09T10:36:21.887280+0000","last_undegraded":"2023-10-09T10:36:21.887280+0000","last_fullsized":"2023-10-09T10:36:21.887280+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:27:07.146829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.773425+0000","last_change":"2023-10-09T10:35:56.823663+0000","last_active":"2023-10-09T10:36:20.773425+0000","last_peered":"2023-10-09T10:36:20.773425+0000","last_clean":"2023-10-09T10:36:20.773425+0000","last_became_active":"2023-10-09T10:35:56.823345+0000","last_became_peered":"2023-10-09T10:35:56.823345+0000","last_unstale":"2023-10-09T10:36:20.773425+0000","last_undegraded":"2023-10-09T10:36:20.773425+0000","last_fullsized":"2023-10-09T10:36:20.773425+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:36:39.917373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765757+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:36:20.765757+0000","last_peered":"2023-10-09T10:36:20.765757+0000","last_clean":"2023-10-09T10:36:20.765757+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:36:20.765757+0000","last_undegraded":"2023-10-09T10:36:20.765757+0000","last_fullsized":"2023-10-09T10:36:20.765757+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.773516+0000","last_change":"2023-10-09T10:35:56.827477+0000","last_active":"2023-10-09T10:36:20.773516+0000","last_peered":"2023-10-09T10:36:20.773516+0000","last_clean":"2023-10-09T10:36:20.773516+0000","last_became_active":"2023-10-09T10:35:56.827078+0000","last_became_peered":"2023-10-09T10:35:56.827078+0000","last_unstale":"2023-10-09T10:36:20.773516+0000","last_undegraded":"2023-10-09T10:36:20.773516+0000","last_fullsized":"2023-10-09T10:36:20.773516+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:29:01.299890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770217+0000","last_change":"2023-10-09T10:35:56.839916+0000","last_active":"2023-10-09T10:36:20.770217+0000","last_peered":"2023-10-09T10:36:20.770217+0000","last_clean":"2023-10-09T10:36:20.770217+0000","last_became_active":"2023-10-09T10:35:56.839605+0000","last_became_peered":"2023-10-09T10:35:56.839605+0000","last_unstale":"2023-10-09T10:36:20.770217+0000","last_undegraded":"2023-10-09T10:36:20.770217+0000","last_fullsized":"2023-10-09T10:36:20.770217+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:24.801950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":22,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.970877+0000","last_change":"2023-10-09T10:35:56.814073+0000","last_active":"2023-10-09T10:36:20.970877+0000","last_peered":"2023-10-09T10:36:20.970877+0000","last_clean":"2023-10-09T10:36:20.970877+0000","last_became_active":"2023-10-09T10:35:56.813780+0000","last_became_peered":"2023-10-09T10:35:56.813780+0000","last_unstale":"2023-10-09T10:36:20.970877+0000","last_undegraded":"2023-10-09T10:36:20.970877+0000","last_fullsized":"2023-10-09T10:36:20.970877+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:04:22.997463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765709+0000","last_change":"2023-10-09T10:35:56.837722+0000","last_active":"2023-10-09T10:36:20.765709+0000","last_peered":"2023-10-09T10:36:20.765709+0000","last_clean":"2023-10-09T10:36:20.765709+0000","last_became_active":"2023-10-09T10:35:56.837445+0000","last_became_peered":"2023-10-09T10:35:56.837445+0000","last_unstale":"2023-10-09T10:36:20.765709+0000","last_undegraded":"2023-10-09T10:36:20.765709+0000","last_fullsized":"2023-10-09T10:36:20.765709+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:02:33.282785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.772389+0000","last_change":"2023-10-09T10:35:56.824257+0000","last_active":"2023-10-09T10:36:20.772389+0000","last_peered":"2023-10-09T10:36:20.772389+0000","last_clean":"2023-10-09T10:36:20.772389+0000","last_became_active":"2023-10-09T10:35:56.823152+0000","last_became_peered":"2023-10-09T10:35:56.823152+0000","last_unstale":"2023-10-09T10:36:20.772389+0000","last_undegraded":"2023-10-09T10:36:20.772389+0000","last_fullsized":"2023-10-09T10:36:20.772389+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:52.392494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.766024+0000","last_change":"2023-10-09T10:35:58.173629+0000","last_active":"2023-10-09T10:36:20.766024+0000","last_peered":"2023-10-09T10:36:20.766024+0000","last_clean":"2023-10-09T10:36:20.766024+0000","last_became_active":"2023-10-09T10:35:58.173331+0000","last_became_peered":"2023-10-09T10:35:58.173331+0000","last_unstale":"2023-10-09T10:36:20.766024+0000","last_undegraded":"2023-10-09T10:36:20.766024+0000","last_fullsized":"2023-10-09T10:36:20.766024+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:13.102708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765306+0000","last_change":"2023-10-09T10:35:56.812650+0000","last_active":"2023-10-09T10:36:20.765306+0000","last_peered":"2023-10-09T10:36:20.765306+0000","last_clean":"2023-10-09T10:36:20.765306+0000","last_became_active":"2023-10-09T10:35:56.812327+0000","last_became_peered":"2023-10-09T10:35:56.812327+0000","last_unstale":"2023-10-09T10:36:20.765306+0000","last_undegraded":"2023-10-09T10:36:20.765306+0000","last_fullsized":"2023-10-09T10:36:20.765306+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:33.929027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-10-09T10:36:21.887125+0000","last_change":"2023-10-09T10:36:21.887125+0000","last_active":"2023-10-09T10:36:21.887125+0000","last_peered":"2023-10-09T10:36:21.887125+0000","last_clean":"2023-10-09T10:36:21.887125+0000","last_became_active":"2023-10-09T10:36:21.886811+0000","last_became_peered":"2023-10-09T10:36:21.886811+0000","last_unstale":"2023-10-09T10:36:21.887125+0000","last_undegraded":"2023-10-09T10:36:21.887125+0000","last_fullsized":"2023-10-09T10:36:21.887125+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:08.364391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.772659+0000","last_change":"2023-10-09T10:35:56.823702+0000","last_active":"2023-10-09T10:36:20.772659+0000","last_peered":"2023-10-09T10:36:20.772659+0000","last_clean":"2023-10-09T10:36:20.772659+0000","last_became_active":"2023-10-09T10:35:56.823401+0000","last_became_peered":"2023-10-09T10:35:56.823401+0000","last_unstale":"2023-10-09T10:36:20.772659+0000","last_undegraded":"2023-10-09T10:36:20.772659+0000","last_fullsized":"2023-10-09T10:36:20.772659+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:13:29.588613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771601+0000","last_change":"2023-10-09T10:35:58.711035+0000","last_active":"2023-10-09T10:36:20.771601+0000","last_peered":"2023-10-09T10:36:20.771601+0000","last_clean":"2023-10-09T10:36:20.771601+0000","last_became_active":"2023-10-09T10:35:58.710703+0000","last_became_peered":"2023-10-09T10:35:58.710703+0000","last_unstale":"2023-10-09T10:36:20.771601+0000","last_undegraded":"2023-10-09T10:36:20.771601+0000","last_fullsized":"2023-10-09T10:36:20.771601+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:46:10.223117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"64'6","reported_seq":20,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771885+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:36:20.771885+0000","last_peered":"2023-10-09T10:36:20.771885+0000","last_clean":"2023-10-09T10:36:20.771885+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:36:20.771885+0000","last_undegraded":"2023-10-09T10:36:20.771885+0000","last_fullsized":"2023-10-09T10:36:20.771885+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768231+0000","last_change":"2023-10-09T10:35:56.837977+0000","last_active":"2023-10-09T10:36:20.768231+0000","last_peered":"2023-10-09T10:36:20.768231+0000","last_clean":"2023-10-09T10:36:20.768231+0000","last_became_active":"2023-10-09T10:35:56.837717+0000","last_became_peered":"2023-10-09T10:35:56.837717+0000","last_unstale":"2023-10-09T10:36:20.768231+0000","last_undegraded":"2023-10-09T10:36:20.768231+0000","last_fullsized":"2023-10-09T10:36:20.768231+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:57.707641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":17,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.772794+0000","last_change":"2023-10-09T10:35:56.824846+0000","last_active":"2023-10-09T10:36:20.772794+0000","last_peered":"2023-10-09T10:36:20.772794+0000","last_clean":"2023-10-09T10:36:20.772794+0000","last_became_active":"2023-10-09T10:35:56.824507+0000","last_became_peered":"2023-10-09T10:35:56.824507+0000","last_unstale":"2023-10-09T10:36:20.772794+0000","last_undegraded":"2023-10-09T10:36:20.772794+0000","last_fullsized":"2023-10-09T10:36:20.772794+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:53:20.462356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770196+0000","last_change":"2023-10-09T10:35:56.839600+0000","last_active":"2023-10-09T10:36:20.770196+0000","last_peered":"2023-10-09T10:36:20.770196+0000","last_clean":"2023-10-09T10:36:20.770196+0000","last_became_active":"2023-10-09T10:35:56.839326+0000","last_became_peered":"2023-10-09T10:35:56.839326+0000","last_unstale":"2023-10-09T10:36:20.770196+0000","last_undegraded":"2023-10-09T10:36:20.770196+0000","last_fullsized":"2023-10-09T10:36:20.770196+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:26.287940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.766081+0000","last_change":"2023-10-09T10:35:56.820752+0000","last_active":"2023-10-09T10:36:20.766081+0000","last_peered":"2023-10-09T10:36:20.766081+0000","last_clean":"2023-10-09T10:36:20.766081+0000","last_became_active":"2023-10-09T10:35:56.819748+0000","last_became_peered":"2023-10-09T10:35:56.819748+0000","last_unstale":"2023-10-09T10:36:20.766081+0000","last_undegraded":"2023-10-09T10:36:20.766081+0000","last_fullsized":"2023-10-09T10:36:20.766081+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:22:00.092032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769970+0000","last_change":"2023-10-09T10:35:56.840607+0000","last_active":"2023-10-09T10:36:20.769970+0000","last_peered":"2023-10-09T10:36:20.769970+0000","last_clean":"2023-10-09T10:36:20.769970+0000","last_became_active":"2023-10-09T10:35:56.840273+0000","last_became_peered":"2023-10-09T10:35:56.840273+0000","last_unstale":"2023-10-09T10:36:20.769970+0000","last_undegraded":"2023-10-09T10:36:20.769970+0000","last_fullsized":"2023-10-09T10:36:20.769970+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:30.114050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765762+0000","last_change":"2023-10-09T10:35:56.820150+0000","last_active":"2023-10-09T10:36:20.765762+0000","last_peered":"2023-10-09T10:36:20.765762+0000","last_clean":"2023-10-09T10:36:20.765762+0000","last_became_active":"2023-10-09T10:35:56.819873+0000","last_became_peered":"2023-10-09T10:35:56.819873+0000","last_unstale":"2023-10-09T10:36:20.765762+0000","last_undegraded":"2023-10-09T10:36:20.765762+0000","last_fullsized":"2023-10-09T10:36:20.765762+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:41.465182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.971774+0000","last_change":"2023-10-09T10:35:57.228230+0000","last_active":"2023-10-09T10:36:20.971774+0000","last_peered":"2023-10-09T10:36:20.971774+0000","last_clean":"2023-10-09T10:36:20.971774+0000","last_became_active":"2023-10-09T10:35:57.227863+0000","last_became_peered":"2023-10-09T10:35:57.227863+0000","last_unstale":"2023-10-09T10:36:20.971774+0000","last_undegraded":"2023-10-09T10:36:20.971774+0000","last_fullsized":"2023-10-09T10:36:20.971774+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:27.674913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.769994+0000","last_change":"2023-10-09T10:35:56.818870+0000","last_active":"2023-10-09T10:36:20.769994+0000","last_peered":"2023-10-09T10:36:20.769994+0000","last_clean":"2023-10-09T10:36:20.769994+0000","last_became_active":"2023-10-09T10:35:56.818589+0000","last_became_peered":"2023-10-09T10:35:56.818589+0000","last_unstale":"2023-10-09T10:36:20.769994+0000","last_undegraded":"2023-10-09T10:36:20.769994+0000","last_fullsized":"2023-10-09T10:36:20.769994+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:27:54.174201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.765787+0000","last_change":"2023-10-09T10:35:56.837777+0000","last_active":"2023-10-09T10:36:20.765787+0000","last_peered":"2023-10-09T10:36:20.765787+0000","last_clean":"2023-10-09T10:36:20.765787+0000","last_became_active":"2023-10-09T10:35:56.837479+0000","last_became_peered":"2023-10-09T10:35:56.837479+0000","last_unstale":"2023-10-09T10:36:20.765787+0000","last_undegraded":"2023-10-09T10:36:20.765787+0000","last_fullsized":"2023-10-09T10:36:20.765787+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:00.136014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.768652+0000","last_change":"2023-10-09T10:35:56.824512+0000","last_active":"2023-10-09T10:36:20.768652+0000","last_peered":"2023-10-09T10:36:20.768652+0000","last_clean":"2023-10-09T10:36:20.768652+0000","last_became_active":"2023-10-09T10:35:56.823293+0000","last_became_peered":"2023-10-09T10:35:56.823293+0000","last_unstale":"2023-10-09T10:36:20.768652+0000","last_undegraded":"2023-10-09T10:36:20.768652+0000","last_fullsized":"2023-10-09T10:36:20.768652+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:34.986666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.770727+0000","last_change":"2023-10-09T10:35:57.165630+0000","last_active":"2023-10-09T10:36:20.770727+0000","last_peered":"2023-10-09T10:36:20.770727+0000","last_clean":"2023-10-09T10:36:20.770727+0000","last_became_active":"2023-10-09T10:35:57.165339+0000","last_became_peered":"2023-10-09T10:35:57.165339+0000","last_unstale":"2023-10-09T10:36:20.770727+0000","last_undegraded":"2023-10-09T10:36:20.770727+0000","last_fullsized":"2023-10-09T10:36:20.770727+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:08.227472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-10-09T10:36:20.771569+0000","last_change":"2023-10-09T10:35:56.823266+0000","last_active":"2023-10-09T10:36:20.771569+0000","last_peered":"2023-10-09T10:36:20.771569+0000","last_clean":"2023-10-09T10:36:20.771569+0000","last_became_active":"2023-10-09T10:35:56.822981+0000","last_became_peered":"2023-10-09T10:35:56.822981+0000","last_unstale":"2023-10-09T10:36:20.771569+0000","last_undegraded":"2023-10-09T10:36:20.771569+0000","last_fullsized":"2023-10-09T10:36:20.771569+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:40.228328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332117,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1260,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1290240,"data_stored":885968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.58599999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.61199999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.56599999999999995}]},{"osd":3,"last update":"Mon Oct 9 10:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.6719999999999999,"5min":1.6719999999999999,"15min":1.6719999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.625}]},{"osd":4,"last update":"Mon Oct 9 10:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.44800000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.432},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.0649999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.47799999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.51400000000000001}]}]},{"osd":6,"up_from":44,"seq":188978561048,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298536,"kb_used_data":1248,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446616,"statfs":{"total":95995035648,"available":95689334784,"internally_reserved":0,"allocated":1277952,"data_stored":885822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.68500000000000005}]},{"osd":1,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.72999999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.64200000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.747},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.80400000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.46800000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.378}]},{"osd":7,"last update":"Mon Oct 9 10:36:02 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.442},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":0.42799999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574890,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297940,"kb_used_data":652,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447212,"statfs":{"total":95995035648,"available":95689945088,"internally_reserved":0,"allocated":667648,"data_stored":293906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:36:20 2023","interfaces":[{"interface":"back","average":{"1min":0.308,"5min":0.35999999999999999,"15min":0.36099999999999999},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":0.46800000000000003,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.254},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.36599999999999999,"15min":0.37},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.38500000000000001,"5min":0.622,"15min":0.622},"last":0.26700000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:35:35 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.42899999999999999,"15min":0.435},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.54400000000000004,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.435,"15min":0.438},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.30199999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.437,"15min":0.44500000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.48199999999999998,"15min":0.49399999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.56699999999999995,"5min":0.625,"15min":0.625},"last":0.28899999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:36:10 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.57399999999999995,"15min":0.58799999999999997},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.63,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.57299999999999995,"15min":0.58299999999999996},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.82199999999999995,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.37}]},{"osd":5,"last update":"Mon Oct 9 10:36:25 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.57899999999999996,"15min":0.58999999999999997},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.73199999999999998,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.57899999999999996,"15min":0.58899999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.64500000000000002,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.54400000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:35:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.63},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.57399999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.40600000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738411,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298532,"kb_used_data":1244,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446620,"statfs":{"total":95995035648,"available":95689338880,"internally_reserved":0,"allocated":1273856,"data_stored":885115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:36:24 2023","interfaces":[{"interface":"back","average":{"1min":0.32600000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.22800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":0.46800000000000003,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.23100000000000001},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.39700000000000002,"15min":0.38900000000000001},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.59299999999999997,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.59299999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.29999999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.37},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.42199999999999999,"15min":0.42499999999999999},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.29699999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:35:53 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.49399999999999999,"15min":0.499},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.181,"5min":1.181,"15min":1.181},"last":0.312},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.48299999999999998,"15min":0.49199999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.55200000000000005,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.46100000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.53000000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:36:24 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.57199999999999995,"15min":0.57399999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.61599999999999999,"15min":0.622},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.628}]},{"osd":6,"last update":"Mon Oct 9 10:35:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.48299999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.54300000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411364,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297952,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447200,"statfs":{"total":95995035648,"available":95689932800,"internally_reserved":0,"allocated":679936,"data_stored":294759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.36099999999999999,"15min":0.35899999999999999},"min":{"1min":0.255,"5min":0.22,"15min":0.22},"max":{"1min":0.57799999999999996,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.25800000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:35:36 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.38900000000000001,"15min":0.38700000000000001},"min":{"1min":0.255,"5min":0.246,"15min":0.246},"max":{"1min":0.55800000000000005,"5min":0.63,"15min":0.63},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.41099999999999998,"15min":0.40899999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.33800000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.46600000000000003,"15min":0.47199999999999998},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.63,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.436,"15min":0.44},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.34899999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:36:10 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.57299999999999995,"15min":0.57599999999999996},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.60499999999999998,"15min":0.61099999999999999},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.78400000000000003,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.64800000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:35:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.628}]},{"osd":6,"last update":"Mon Oct 9 10:35:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":2.0339999999999998,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.59199999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.67500000000000004}]}]},{"osd":3,"up_from":25,"seq":107374182435,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297948,"kb_used_data":660,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447204,"statfs":{"total":95995035648,"available":95689936896,"internally_reserved":0,"allocated":675840,"data_stored":294261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:54 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.39700000000000002,"15min":0.39900000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.45100000000000001,"15min":0.46300000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.54700000000000004,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.35399999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:35:54 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.47199999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.57399999999999995,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.307},{"interface":"front","average":{"1min":0.435,"5min":0.44,"15min":0.441},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.28999999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:35:54 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.48599999999999999,"15min":0.495},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.47099999999999997,"15min":0.47699999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.33700000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:36:14 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.55800000000000005,"15min":0.56799999999999995},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.57199999999999995,"15min":0.57799999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.55000000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:36:23 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.59099999999999997,"15min":0.57199999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":2.9580000000000002,"5min":2.9580000000000002,"15min":2.9580000000000002},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.60299999999999998,"15min":0.58099999999999996},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":3.9399999999999999,"5min":3.9399999999999999,"15min":3.9399999999999999},"last":0.51600000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.8380000000000001,"5min":1.8380000000000001,"15min":1.8380000000000001},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.56999999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":2.8980000000000001,"5min":2.8980000000000001,"15min":2.8980000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.748,"5min":0.748,"15min":0.748},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":3.0169999999999999,"5min":3.0169999999999999,"15min":3.0169999999999999},"last":0.48899999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018911,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":696320,"data_stored":295853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.53300000000000003,"15min":0.54100000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.34799999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.498}]},{"osd":1,"last update":"Mon Oct 9 10:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.52700000000000002,"15min":0.53100000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.623},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.53400000000000003,"15min":0.54100000000000004},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.53900000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.56599999999999995,"15min":0.55800000000000005},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.3530000000000002,"5min":2.3530000000000002,"15min":2.3530000000000002},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.56899999999999995,"15min":0.57699999999999996},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79200000000000004,"5min":0.875,"15min":0.875},"last":0.63500000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56100000000000005,"15min":0.56000000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.5800000000000001,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":0.66500000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:36:26 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.40999999999999998,"15min":0.40300000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.434},{"interface":"front","average":{"1min":0.438,"5min":0.41099999999999998,"15min":0.40699999999999997},"min":{"1min":0.29899999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.44800000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.505},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.52200000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.56399999999999995}]}]},{"osd":5,"up_from":37,"seq":158913789980,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297948,"kb_used_data":660,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447204,"statfs":{"total":95995035648,"available":95689936896,"internally_reserved":0,"allocated":675840,"data_stored":294472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.40500000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.57299999999999995}]},{"osd":2,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.5},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.59899999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.65500000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.505,"5min":0.505,"15min":0.505},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.753}]},{"osd":6,"last update":"Mon Oct 9 10:35:43 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.72799999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.69899999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:36:29.189 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T10:36:29.190 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T10:36:29.190 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T10:36:29.190 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T10:36:29.456 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:29.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:29 smithi125 ceph-mon[112589]: pgmap v46: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:29.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:29 smithi125 ceph-mon[112589]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:36:29.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:29 smithi125 ceph-mon[127026]: pgmap v46: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:29.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:29 smithi125 ceph-mon[127026]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:36:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:29 smithi155 ceph-mon[117223]: pgmap v46: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:29 smithi155 ceph-mon[117223]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:36:30.731 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:36:30.732 INFO:teuthology.orchestra.run.smithi125.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T10:36:31.117 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T10:36:31.117 INFO:teuthology.run_tasks:Running task print... 2023-10-09T10:36:31.127 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-10-09T10:36:31.127 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T10:36:31.153 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T10:36:31.153 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T10:36:31.153 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T10:36:31.157 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T10:36:31.158 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T10:36:31.158 DEBUG:teuthology.orchestra.run.smithi125:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T10:36:31.158 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T10:36:31.158 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-10-09T10:36:31.159 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi125.front.sepia.ceph.com 2023-10-09T10:36:31.159 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-10-09T10:36:31.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T10:36:31.177 INFO:teuthology.orchestra.run.smithi125.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T10:36:31.178 DEBUG:teuthology.orchestra.run.smithi125:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T10:36:31.235 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T10:36:31.235 DEBUG:teuthology.orchestra.run.smithi125:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T10:36:31.292 INFO:tasks.workunit:timeout=3h 2023-10-09T10:36:31.292 INFO:tasks.workunit:cleanup=True 2023-10-09T10:36:31.293 DEBUG:teuthology.orchestra.run.smithi125:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T10:36:31.351 INFO:tasks.workunit.client.0.smithi125.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T10:36:31.427 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:31.896 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:31 smithi125 ceph-mon[112589]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:31.896 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:31 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1702932244' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T10:36:31.897 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:31 smithi125 ceph-mon[127026]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:31.897 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:31 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/1702932244' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T10:36:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:31 smithi155 ceph-mon[117223]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:31 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1702932244' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T10:36:31.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:36:31 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:36:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T10:36:32.941 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:32 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:36:32] ENGINE Bus STOPPING 2023-10-09T10:36:32.941 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:32 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:36:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T10:36:32.941 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:32 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:36:32] ENGINE Bus STOPPED 2023-10-09T10:36:32.941 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:32 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:36:32] ENGINE Bus STARTING 2023-10-09T10:36:32.941 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:32 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:36:32] ENGINE Serving on http://:::9283 2023-10-09T10:36:32.941 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:32 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:36:32] ENGINE Bus STARTED 2023-10-09T10:36:32.984 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set global log_to_journald false --force' 2023-10-09T10:36:33.305 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:33.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:33 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/4074965418' entity='client.admin' 2023-10-09T10:36:33.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:33 smithi125 ceph-mon[112589]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:33.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:33 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:36:33.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:33 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:33.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:33 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:36:33.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:33 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:33.754 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:33 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/4074965418' entity='client.admin' 2023-10-09T10:36:33.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:33 smithi125 ceph-mon[127026]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:33.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:33 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:36:33.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:33 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:33.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:33 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:36:33.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:33 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:33 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/4074965418' entity='client.admin' 2023-10-09T10:36:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:33 smithi155 ceph-mon[117223]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:33 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:36:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:33 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:33.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:33 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:36:33.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:33 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:34.908 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-10-09T10:36:35.175 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:35 smithi155 ceph-mon[117223]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:35.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:35 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:36:36.002 DEBUG:teuthology.orchestra.run.smithi125:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T10:36:36.061 INFO:tasks.workunit.client.0.smithi125.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T10:36:36.063 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T10:36:36.063 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T10:36:36.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:35 smithi125 ceph-mon[112589]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:36.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:35 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:36:36.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:35 smithi125 ceph-mon[127026]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:36.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:35 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:36:36.537 INFO:teuthology.orchestra.run.smithi125.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T10:36:37.070 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T10:36:37.115 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T10:36:37.176 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T10:36:37.214 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T10:36:37.216 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T10:36:37.216 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T10:36:37.253 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T10:36:37.257 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T10:36:37.257 DEBUG:teuthology.orchestra.run.smithi125:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T10:36:37.274 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T10:36:37.275 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T10:36:37.276 DEBUG:teuthology.orchestra.run.smithi125:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T10:36:37.347 INFO:tasks.workunit.client.0.smithi125.stdout:Running main() from gmock_main.cc 2023-10-09T10:36:37.348 INFO:tasks.workunit.client.0.smithi125.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T10:36:37.348 INFO:tasks.workunit.client.0.smithi125.stdout:[----------] Global test environment set-up. 2023-10-09T10:36:37.348 INFO:tasks.workunit.client.0.smithi125.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T10:36:37.349 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:37.811 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[112589]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:36:37.811 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[112589]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T10:36:37.812 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:37.812 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:36:37.812 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:37.812 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:36:37.812 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:37.812 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[112589]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:37.812 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[112589]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T10:36:37.812 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[127026]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:36:37.812 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[127026]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T10:36:37.813 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:37.813 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:36:37.813 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:37.813 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:36:37.813 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:37.813 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[127026]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:37.813 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:37 smithi125 ceph-mon[127026]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T10:36:37.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:37 smithi155 ceph-mon[117223]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:36:37.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:37 smithi155 ceph-mon[117223]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T10:36:37.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:37 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:37.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:37 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:36:37.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:37 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:36:37.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:37 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:36:37.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:37 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:36:37.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:37 smithi155 ceph-mon[117223]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:37.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:37 smithi155 ceph-mon[117223]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T10:36:38.594 INFO:teuthology.orchestra.run.smithi125.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T10:36:38.594 INFO:teuthology.orchestra.run.smithi125.stdout:alertmanager.a smithi125 *:9093,9094 running (41s) 13s ago 68s 17.4M - 0.25.0 c8568f914cd2 93d5ea37b5d2 2023-10-09T10:36:38.594 INFO:teuthology.orchestra.run.smithi125.stdout:grafana.a smithi155 *:3000 running (55s) 21s ago 55s 85.5M - 9.4.7 2c41d148cca3 591b62826625 2023-10-09T10:36:38.595 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi125.satgiw smithi125 running (33s) 13s ago 33s 14.1M - 17.2.6-1303-gb2903cb0 5517396c50d4 7b5d28288eab 2023-10-09T10:36:38.595 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi155.ydxmyq smithi155 running (31s) 21s ago 31s 13.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 c2c99605d324 2023-10-09T10:36:38.595 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.x smithi155 *:8443 running (3m) 21s ago 3m 438M - 17.2.6-1303-gb2903cb0 5517396c50d4 a1dbb2209d73 2023-10-09T10:36:38.595 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.y smithi125 *:9283 running (5m) 13s ago 5m 491M - 17.2.6-1303-gb2903cb0 5517396c50d4 e55280bfaf25 2023-10-09T10:36:38.595 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a smithi125 running (5m) 13s ago 5m 81.8M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 310e442a0e90 2023-10-09T10:36:38.595 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b smithi155 running (4m) 21s ago 4m 67.5M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 14cad29646e0 2023-10-09T10:36:38.595 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c smithi125 running (4m) 13s ago 4m 65.3M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 6524823f3114 2023-10-09T10:36:38.595 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.a smithi125 *:9100 running (51s) 13s ago 50s 16.6M - 1.5.0 0da6a335fe13 28a7f76287d6 2023-10-09T10:36:38.595 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.b smithi155 *:9100 running (48s) 21s ago 47s 8292k - 1.5.0 0da6a335fe13 2e7ced8915fc 2023-10-09T10:36:38.595 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 smithi125 running (3m) 13s ago 3m 54.9M 4022M 17.2.6-1303-gb2903cb0 5517396c50d4 346135826df7 2023-10-09T10:36:38.595 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1 smithi125 running (3m) 13s ago 3m 60.5M 4022M 17.2.6-1303-gb2903cb0 5517396c50d4 11af9c8d1b19 2023-10-09T10:36:38.596 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2 smithi125 running (3m) 13s ago 3m 51.9M 4022M 17.2.6-1303-gb2903cb0 5517396c50d4 dc337188d937 2023-10-09T10:36:38.596 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3 smithi125 running (2m) 13s ago 2m 49.6M 4022M 17.2.6-1303-gb2903cb0 5517396c50d4 c92a4cc944fe 2023-10-09T10:36:38.596 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 smithi155 running (2m) 21s ago 2m 52.9M 4022M 17.2.6-1303-gb2903cb0 5517396c50d4 e3a61e2037a2 2023-10-09T10:36:38.596 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5 smithi155 running (2m) 21s ago 2m 49.8M 4022M 17.2.6-1303-gb2903cb0 5517396c50d4 12d4484e25b1 2023-10-09T10:36:38.596 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6 smithi155 running (2m) 21s ago 2m 51.2M 4022M 17.2.6-1303-gb2903cb0 5517396c50d4 c7b7ee1a3182 2023-10-09T10:36:38.596 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7 smithi155 running (106s) 21s ago 106s 50.0M 4022M 17.2.6-1303-gb2903cb0 5517396c50d4 8e225788b8f4 2023-10-09T10:36:38.596 INFO:teuthology.orchestra.run.smithi125.stdout:prometheus.a smithi155 *:9095 running (39s) 21s ago 85s 36.8M - 2.43.0 a07b618ecd1d 55ccec467a98 2023-10-09T10:36:38.684 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T10:36:38.699 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_all_features (16 ms) 2023-10-09T10:36:38.699 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T10:36:38.893 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.copyup (194 ms) 2023-10-09T10:36:38.893 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T10:36:38.910 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.sparse_copyup (17 ms) 2023-10-09T10:36:38.911 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T10:36:38.915 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:38 smithi125 ceph-mon[112589]: osdmap e67: 8 total, 8 up, 8 in 2023-10-09T10:36:38.916 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:38 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/623980962' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-177846-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:36:38.916 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:38 smithi125 ceph-mon[127026]: osdmap e67: 8 total, 8 up, 8 in 2023-10-09T10:36:38.916 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:38 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/623980962' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-177846-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:36:38.917 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:38 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:36:38] "GET /metrics HTTP/1.1" 200 238865 "" "Prometheus/2.43.0" 2023-10-09T10:36:38.922 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_and_set_id (11 ms) 2023-10-09T10:36:38.922 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T10:36:38.934 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.add_remove_child (12 ms) 2023-10-09T10:36:38.934 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T10:36:38.956 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for minority of mons to upgrade' 2023-10-09T10:36:38.961 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.directory_methods (26 ms) 2023-10-09T10:36:38.961 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.create 2023-10-09T10:36:38.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:38 smithi155 ceph-mon[117223]: osdmap e67: 8 total, 8 up, 8 in 2023-10-09T10:36:38.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:38 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/623980962' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-177846-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:36:38.973 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.create (13 ms) 2023-10-09T10:36:38.973 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T10:36:38.974 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2023-10-09T10:36:38.975 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T10:36:38.976 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-10-09T10:36:38.977 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T10:36:38.978 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-10-09T10:36:38.979 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T10:36:38.980 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-10-09T10:36:38.980 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T10:36:38.981 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-10-09T10:36:38.981 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T10:36:38.985 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2023-10-09T10:36:38.985 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T10:36:38.989 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2023-10-09T10:36:38.989 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T10:36:38.994 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.set_size (6 ms) 2023-10-09T10:36:38.994 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T10:36:39.015 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.protection_status (21 ms) 2023-10-09T10:36:39.015 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T10:36:39.023 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2023-10-09T10:36:39.023 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T10:36:39.068 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.parents_v1 (46 ms) 2023-10-09T10:36:39.068 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T10:36:39.098 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.parents_v2 (29 ms) 2023-10-09T10:36:39.098 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T10:36:39.118 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.snapshots (20 ms) 2023-10-09T10:36:39.118 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T10:36:39.123 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2023-10-09T10:36:39.123 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T10:36:39.132 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2023-10-09T10:36:39.132 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T10:36:39.133 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_save (1 ms) 2023-10-09T10:36:39.133 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T10:36:39.140 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_resize (8 ms) 2023-10-09T10:36:39.140 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T10:36:39.150 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_update (10 ms) 2023-10-09T10:36:39.151 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T10:36:39.151 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-10-09T10:36:39.151 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T10:36:39.158 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_snap_add (7 ms) 2023-10-09T10:36:39.158 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T10:36:39.166 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_snap_remove (9 ms) 2023-10-09T10:36:39.166 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T10:36:39.171 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.flags (4 ms) 2023-10-09T10:36:39.171 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T10:36:39.195 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.metadata (24 ms) 2023-10-09T10:36:39.195 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T10:36:39.201 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2023-10-09T10:36:39.201 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T10:36:39.237 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror (36 ms) 2023-10-09T10:36:39.238 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T10:36:39.259 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror_image (21 ms) 2023-10-09T10:36:39.259 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T10:36:39.296 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:39.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:39 smithi125 ceph-mon[127026]: from='client.24737 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:36:39.681 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:39 smithi125 ceph-mon[112589]: from='client.24737 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:36:39.681 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:39 smithi125 ceph-mon[112589]: pgmap v52: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:39.681 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:39 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/623980962' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-177846-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T10:36:39.682 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:39 smithi125 ceph-mon[112589]: osdmap e68: 8 total, 8 up, 8 in 2023-10-09T10:36:39.682 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:39 smithi125 ceph-mon[112589]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T10:36:39.936 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:39 smithi125 ceph-mon[127026]: pgmap v52: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:39.936 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:39 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/623980962' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-177846-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T10:36:39.936 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:39 smithi125 ceph-mon[127026]: osdmap e68: 8 total, 8 up, 8 in 2023-10-09T10:36:39.936 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:39 smithi125 ceph-mon[127026]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T10:36:39.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:39 smithi155 ceph-mon[117223]: from='client.24737 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:36:39.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:39 smithi155 ceph-mon[117223]: pgmap v52: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:39.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:39 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/623980962' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-177846-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T10:36:39.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:39 smithi155 ceph-mon[117223]: osdmap e68: 8 total, 8 up, 8 in 2023-10-09T10:36:39.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:39 smithi155 ceph-mon[117223]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T10:36:40.269 INFO:teuthology.orchestra.run.smithi125.stdout:wait for minority of mons to upgrade 2023-10-09T10:36:40.573 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-10-09T10:36:40.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:40 smithi125 ceph-mon[112589]: osdmap e69: 8 total, 8 up, 8 in 2023-10-09T10:36:40.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:40 smithi125 ceph-mon[127026]: osdmap e69: 8 total, 8 up, 8 in 2023-10-09T10:36:40.952 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:36:40.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:40 smithi155 ceph-mon[117223]: osdmap e69: 8 total, 8 up, 8 in 2023-10-09T10:36:41.625 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror_image_status (2366 ms) 2023-10-09T10:36:41.625 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T10:36:41.704 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror_image_map (79 ms) 2023-10-09T10:36:41.704 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T10:36:41.712 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2023-10-09T10:36:41.712 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T10:36:41.723 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror_snapshot (11 ms) 2023-10-09T10:36:41.723 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T10:36:41.725 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-10-09T10:36:41.725 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T10:36:41.727 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-10-09T10:36:41.727 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T10:36:41.729 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-10-09T10:36:41.730 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T10:36:41.735 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_dir_rename (5 ms) 2023-10-09T10:36:41.735 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T10:36:41.738 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_dir_remove (4 ms) 2023-10-09T10:36:41.738 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T10:36:41.742 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (3 ms) 2023-10-09T10:36:41.742 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T10:36:41.744 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2023-10-09T10:36:41.744 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T10:36:41.747 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_image_remove (4 ms) 2023-10-09T10:36:41.747 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T10:36:41.750 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_image_list (2 ms) 2023-10-09T10:36:41.750 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T10:36:41.755 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_image_clean (5 ms) 2023-10-09T10:36:41.755 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T10:36:41.757 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2023-10-09T10:36:41.757 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T10:36:41.760 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2023-10-09T10:36:41.760 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T10:36:41.762 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-10-09T10:36:41.763 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T10:36:41.764 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-10-09T10:36:41.765 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T10:36:41.766 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-10-09T10:36:41.766 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T10:36:41.769 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-10-09T10:36:41.770 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T10:36:41.772 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (2 ms) 2023-10-09T10:36:41.772 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T10:36:41.774 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2023-10-09T10:36:41.774 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T10:36:41.778 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_list (4 ms) 2023-10-09T10:36:41.778 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T10:36:41.794 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (16 ms) 2023-10-09T10:36:41.794 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T10:36:41.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:41 smithi125 ceph-mon[112589]: pgmap v55: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:41.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:41 smithi125 ceph-mon[112589]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T10:36:41.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:41 smithi125 ceph-mon[112589]: Cluster is now healthy 2023-10-09T10:36:41.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:41 smithi125 ceph-mon[127026]: pgmap v55: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:41.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:41 smithi125 ceph-mon[127026]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T10:36:41.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:41 smithi125 ceph-mon[127026]: Cluster is now healthy 2023-10-09T10:36:41.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:36:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:36:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T10:36:41.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:41 smithi155 ceph-mon[117223]: pgmap v55: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:41.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:41 smithi155 ceph-mon[117223]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T10:36:41.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:41 smithi155 ceph-mon[117223]: Cluster is now healthy 2023-10-09T10:36:41.973 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (179 ms) 2023-10-09T10:36:41.973 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T10:36:41.976 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2023-10-09T10:36:41.976 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T10:36:41.978 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (3 ms) 2023-10-09T10:36:41.978 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T10:36:41.986 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2023-10-09T10:36:41.986 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T10:36:41.993 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2023-10-09T10:36:41.993 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T10:36:42.014 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.clone_parent (21 ms) 2023-10-09T10:36:42.014 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T10:36:42.022 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2023-10-09T10:36:42.022 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T10:36:42.035 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.clone_child (14 ms) 2023-10-09T10:36:42.035 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T10:36:42.043 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2023-10-09T10:36:42.043 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T10:36:42.051 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.migration (9 ms) 2023-10-09T10:36:42.052 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T10:36:42.062 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.migration_v1 (10 ms) 2023-10-09T10:36:42.062 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T10:36:42.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:42 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3511530943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:43.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:42 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3511530943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:43.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:42 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3511530943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:43.708 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1646 ms) 2023-10-09T10:36:43.708 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T10:36:43.747 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.sparsify (39 ms) 2023-10-09T10:36:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:43 smithi155 ceph-mon[117223]: pgmap v56: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.2 MiB/s wr, 2.09k op/s 2023-10-09T10:36:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:43 smithi155 ceph-mon[117223]: osdmap e70: 8 total, 8 up, 8 in 2023-10-09T10:36:44.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:43 smithi125 ceph-mon[112589]: pgmap v56: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.2 MiB/s wr, 2.09k op/s 2023-10-09T10:36:44.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:43 smithi125 ceph-mon[112589]: osdmap e70: 8 total, 8 up, 8 in 2023-10-09T10:36:44.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:43 smithi125 ceph-mon[127026]: pgmap v56: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.2 MiB/s wr, 2.09k op/s 2023-10-09T10:36:44.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:43 smithi125 ceph-mon[127026]: osdmap e70: 8 total, 8 up, 8 in 2023-10-09T10:36:44.726 INFO:tasks.workunit.client.0.smithi125.stdout:[----------] 70 tests from TestClsRbd (5063 ms total) 2023-10-09T10:36:44.726 INFO:tasks.workunit.client.0.smithi125.stdout: 2023-10-09T10:36:44.726 INFO:tasks.workunit.client.0.smithi125.stdout:[----------] Global test environment tear-down 2023-10-09T10:36:44.727 INFO:tasks.workunit.client.0.smithi125.stdout:[==========] 70 tests from 1 test suite ran. (7378 ms total) 2023-10-09T10:36:44.727 INFO:tasks.workunit.client.0.smithi125.stdout:[ PASSED ] 70 tests. 2023-10-09T10:36:44.728 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T10:36:44.728 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T10:36:44.760 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T10:36:44.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T10:36:44.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:44 smithi155 ceph-mon[117223]: osdmap e71: 8 total, 8 up, 8 in 2023-10-09T10:36:45.008 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:44 smithi125 ceph-mon[112589]: osdmap e71: 8 total, 8 up, 8 in 2023-10-09T10:36:45.009 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:44 smithi125 ceph-mon[127026]: osdmap e71: 8 total, 8 up, 8 in 2023-10-09T10:36:45.010 DEBUG:teuthology.parallel:result is None 2023-10-09T10:36:45.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T10:36:45.038 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T10:36:45.038 DEBUG:teuthology.orchestra.run.smithi125:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T10:36:45.105 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T10:36:45.106 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T10:36:45.106 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T10:36:45.106 DEBUG:teuthology.parallel:result is None 2023-10-09T10:36:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:45 smithi155 ceph-mon[117223]: pgmap v59: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.2 MiB/s wr, 2.10k op/s 2023-10-09T10:36:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:45 smithi155 ceph-mon[117223]: osdmap e72: 8 total, 8 up, 8 in 2023-10-09T10:36:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:45 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/667574070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:46.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:45 smithi125 ceph-mon[112589]: pgmap v59: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.2 MiB/s wr, 2.10k op/s 2023-10-09T10:36:46.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:45 smithi125 ceph-mon[112589]: osdmap e72: 8 total, 8 up, 8 in 2023-10-09T10:36:46.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:45 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/667574070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:46.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:45 smithi125 ceph-mon[127026]: pgmap v59: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.2 MiB/s wr, 2.10k op/s 2023-10-09T10:36:46.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:45 smithi125 ceph-mon[127026]: osdmap e72: 8 total, 8 up, 8 in 2023-10-09T10:36:46.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:45 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/667574070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:47.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:47 smithi155 ceph-mon[117223]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:47.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:47 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/739999816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:48.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:47 smithi125 ceph-mon[112589]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:48.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:47 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/739999816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:48.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:47 smithi125 ceph-mon[127026]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:48.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:47 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/739999816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:49.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:48 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:36:48] "GET /metrics HTTP/1.1" 200 238865 "" "Prometheus/2.43.0" 2023-10-09T10:36:49.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:49 smithi125 ceph-mon[112589]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:49.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:49 smithi125 ceph-mon[127026]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:49.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:49 smithi155 ceph-mon[117223]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:50.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:50 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2463075735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:50.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:50 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2463075735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:50.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:50 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2463075735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:51.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:51 smithi125 ceph-mon[112589]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:51.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:51 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:36:51.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:51 smithi125 ceph-mon[127026]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:51.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:51 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:36:51.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:51 smithi155 ceph-mon[117223]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:51.875 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:51 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:36:52.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:36:51 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:36:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T10:36:52.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:52 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/4180262002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:52.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:52 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/4180262002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:52.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:52 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/4180262002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:53.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:53 smithi125 ceph-mon[112589]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:53.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:53 smithi125 ceph-mon[127026]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:53 smithi155 ceph-mon[117223]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:54.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:54 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2686183746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:54.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:54 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2686183746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:54.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:54 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2686183746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:55.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:55 smithi125 ceph-mon[112589]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:55.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:55 smithi125 ceph-mon[127026]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:55.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:55 smithi155 ceph-mon[117223]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:57 smithi155 ceph-mon[117223]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:57 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3008762143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:58.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:57 smithi125 ceph-mon[112589]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:58.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:57 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3008762143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:58.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:57 smithi125 ceph-mon[127026]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:58.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:57 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3008762143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:59.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:36:58 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:36:58] "GET /metrics HTTP/1.1" 200 239810 "" "Prometheus/2.43.0" 2023-10-09T10:36:59.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:59 smithi125 ceph-mon[112589]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:59.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:36:59 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/945932126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:59.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:59 smithi125 ceph-mon[127026]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:59.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:36:59 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/945932126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:36:59.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:59 smithi155 ceph-mon[117223]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:36:59.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:36:59 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/945932126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:01.815 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:01 smithi125 ceph-mon[112589]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:01.815 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:01 smithi125 ceph-mon[127026]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:01.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:01 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:37:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T10:37:01.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:01 smithi155 ceph-mon[117223]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:02 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/306096855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:02 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:02 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:37:02.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:02 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:03.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:02 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/306096855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:03.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:02 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:03.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:02 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:37:03.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:02 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:03.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:02 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/306096855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:03.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:02 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:03.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:02 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:37:03.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:02 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:03.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:03 smithi155 ceph-mon[117223]: Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T10:37:03.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:03 smithi155 ceph-mon[117223]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T10:37:03.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:03 smithi155 ceph-mon[117223]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T10:37:03.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:03 smithi155 ceph-mon[117223]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi155 2023-10-09T10:37:03.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:03 smithi155 ceph-mon[117223]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:04.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:03 smithi125 ceph-mon[127026]: Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T10:37:04.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:03 smithi125 ceph-mon[127026]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T10:37:04.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:03 smithi125 ceph-mon[127026]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T10:37:04.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:03 smithi125 ceph-mon[127026]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi155 2023-10-09T10:37:04.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:03 smithi125 ceph-mon[127026]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:04.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:03 smithi125 ceph-mon[112589]: Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T10:37:04.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:03 smithi125 ceph-mon[112589]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T10:37:04.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:03 smithi125 ceph-mon[112589]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T10:37:04.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:03 smithi125 ceph-mon[112589]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi155 2023-10-09T10:37:04.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:03 smithi125 ceph-mon[112589]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:04 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/4052913685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:05.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:04 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/4052913685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:05.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:04 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/4052913685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:05.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:05 smithi155 ceph-mon[117223]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:05.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:05 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:37:06.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:05 smithi125 ceph-mon[112589]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:06.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:05 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:37:06.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:05 smithi125 ceph-mon[127026]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:06.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:05 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:37:06.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:06 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/717848013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:07.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:06 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/717848013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:07.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:06 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/717848013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:07.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:07 smithi155 ceph-mon[117223]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:08.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:07 smithi125 ceph-mon[112589]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:08.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:07 smithi125 ceph-mon[127026]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:09.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:08 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:37:08] "GET /metrics HTTP/1.1" 200 239749 "" "Prometheus/2.43.0" 2023-10-09T10:37:09.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:09 smithi125 ceph-mon[112589]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:09.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:09 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/257690077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:09.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:09 smithi125 ceph-mon[127026]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:09.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:09 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/257690077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:09.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:09 smithi155 ceph-mon[117223]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:09.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:09 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/257690077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:11 smithi155 ceph-mon[117223]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:11 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3461959964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:11.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:11 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:37:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T10:37:12.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:11 smithi125 ceph-mon[112589]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:12.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:11 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3461959964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:12.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:11 smithi125 ceph-mon[127026]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:12.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:11 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3461959964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:13 smithi155 ceph-mon[117223]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:14.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:13 smithi125 ceph-mon[112589]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:14.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:13 smithi125 ceph-mon[127026]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:14 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1051249274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:15.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:14 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1051249274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:15.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:14 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/1051249274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:15.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:15 smithi155 ceph-mon[117223]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:16.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:15 smithi125 ceph-mon[112589]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:16.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:15 smithi125 ceph-mon[127026]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:16.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:16 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2692161324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:17.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:16 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2692161324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:17.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:16 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2692161324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:17.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:17 smithi155 ceph-mon[117223]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:18.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:17 smithi125 ceph-mon[112589]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:18.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:17 smithi125 ceph-mon[127026]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:19.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:18 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:37:18] "GET /metrics HTTP/1.1" 200 239749 "" "Prometheus/2.43.0" 2023-10-09T10:37:19.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:19 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/622987276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:19.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:19 smithi125 ceph-mon[112589]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:19.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:19 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/622987276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:19.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:19 smithi125 ceph-mon[127026]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:19.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:19 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/622987276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:19.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:19 smithi155 ceph-mon[117223]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:21.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:20 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:37:21.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:20 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:37:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:20 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:37:22.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:21 smithi125 ceph-mon[112589]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:22.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:21 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2088002485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:22.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:21 smithi125 ceph-mon[127026]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:22.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:21 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2088002485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:22.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:21 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[119313]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:37:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T10:37:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:21 smithi155 ceph-mon[117223]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:21 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2088002485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:24.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:23 smithi125 ceph-mon[112589]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:24.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:23 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2082655818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:24.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:23 smithi125 ceph-mon[127026]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:24.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:23 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2082655818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:24.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:23 smithi155 ceph-mon[117223]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:24.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:23 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2082655818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:26.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:25 smithi125 ceph-mon[112589]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:26.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:25 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3672753097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:26.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:25 smithi125 ceph-mon[127026]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:26.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:25 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3672753097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:26.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:25 smithi155 ceph-mon[117223]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:26.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:25 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3672753097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:27.922 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:27 smithi155 ceph-mon[117223]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:27.922 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:27 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:27.922 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:27 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:37:27.923 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:27 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:37:27.923 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:27 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:37:28.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:27 smithi125 ceph-mon[127026]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:28.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:27 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:28.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:27 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:37:28.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:27 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:37:28.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:27 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:37:28.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:27 smithi125 ceph-mon[112589]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:28.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:27 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:28.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:27 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:37:28.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:27 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:37:28.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:27 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:37:28.470 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:28 smithi155 systemd[1]: Stopping Ceph mgr.x for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:37:28.884 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:28 smithi155 bash[157524]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T10:37:28.884 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:28 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T10:37:28.884 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:28 smithi155 ceph-mon[117223]: Upgrade: Updating mgr.x 2023-10-09T10:37:28.884 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:28 smithi155 ceph-mon[117223]: Deploying daemon mgr.x on smithi155 2023-10-09T10:37:28.884 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:28 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1161510068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:29.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:28 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:37:28] "GET /metrics HTTP/1.1" 200 239695 "" "Prometheus/2.43.0" 2023-10-09T10:37:29.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:28 smithi125 ceph-mon[112589]: Upgrade: Updating mgr.x 2023-10-09T10:37:29.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:28 smithi125 ceph-mon[112589]: Deploying daemon mgr.x on smithi155 2023-10-09T10:37:29.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:28 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1161510068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:29.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:28 smithi125 ceph-mon[127026]: Upgrade: Updating mgr.x 2023-10-09T10:37:29.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:28 smithi125 ceph-mon[127026]: Deploying daemon mgr.x on smithi155 2023-10-09T10:37:29.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:28 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/1161510068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:29.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:28 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T10:37:29.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:28 smithi155 systemd[1]: Stopped Ceph mgr.x for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:37:29.221 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:29 smithi155 systemd[1]: Starting Ceph mgr.x for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:37:29.702 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:29 smithi155 ceph-mgr[157714]: -- 172.21.15.155:0/3199440330 <== mon.1 v2:172.21.15.155:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55bf8ba1f1e0 con 0x55bf8b9f4800 2023-10-09T10:37:29.703 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:29 smithi155 bash[157676]: 0a66a44a424a10d083fa904726a449ed6151d9f907bb828caae2c5700ef2694a 2023-10-09T10:37:29.703 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:29 smithi155 systemd[1]: Started Ceph mgr.x for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:37:29.703 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:29 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:29.524+0000 7f24f4295200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T10:37:29.703 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:29 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:29.701+0000 7f24f4295200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T10:37:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:29 smithi155 ceph-mon[117223]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:29 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:29 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:29 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:37:30.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:29 smithi125 ceph-mon[112589]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:30.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:29 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:30.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:29 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:30.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:29 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:37:30.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:29 smithi125 ceph-mon[127026]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:30.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:29 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:30.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:29 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:30.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:29 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:37:30.470 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:30 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:30.077+0000 7f24f4295200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T10:37:30.928 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:30 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/4156793970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:30.928 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:30 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:30.645+0000 7f24f4295200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T10:37:31.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:30 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/4156793970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:31.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:30 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/4156793970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:31.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:30 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:30.928+0000 7f24f4295200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T10:37:31.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:31 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:31.393+0000 7f24f4295200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T10:37:32.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:31 smithi125 ceph-mon[112589]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:32.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:31 smithi125 ceph-mon[127026]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:32.194 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:31 smithi155 ceph-mon[117223]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:32.470 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:32 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:32.194+0000 7f24f4295200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T10:37:32.470 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:32 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:32.288+0000 7f24f4295200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T10:37:33.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:33 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:33.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:33 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:33.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:33 smithi155 ceph-mon[117223]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:33.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:33 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:33.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:33 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:33.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:33 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2766315733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:33.221 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:32 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:32.961+0000 7f24f4295200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T10:37:33.221 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:33 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:33.055+0000 7f24f4295200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T10:37:33.221 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:33 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:33.171+0000 7f24f4295200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T10:37:33.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:33 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:33.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:33 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:33.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:33 smithi125 ceph-mon[112589]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:33.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:33 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:33.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:33 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:33.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:33 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2766315733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:33.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:33 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:33.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:33 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:33.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:33 smithi125 ceph-mon[127026]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:33.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:33 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:33.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:33 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:33.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:33 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2766315733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:33.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:33 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:33.402+0000 7f24f4295200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T10:37:33.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:33 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:33.502+0000 7f24f4295200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T10:37:33.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:33 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:33.602+0000 7f24f4295200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T10:37:34.113 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:33 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:33.828+0000 7f24f4295200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T10:37:34.113 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:33 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:33.929+0000 7f24f4295200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T10:37:34.470 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:34.129+0000 7f24f4295200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T10:37:34.471 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:34.223+0000 7f24f4295200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T10:37:35.308 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:35 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:35.028+0000 7f24f4295200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T10:37:35.308 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:35 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:35.236+0000 7f24f4295200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T10:37:35.707 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:35 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:35.424+0000 7f24f4295200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:37:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:35 smithi155 ceph-mon[117223]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:35 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/554441523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:35 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:35 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:35 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:37:36.034 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:35 smithi125 ceph-mon[112589]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:36.034 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:35 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/554441523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:36.034 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:35 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:36.034 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:35 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:36.034 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:35 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:37:36.034 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:35 smithi125 ceph-mon[127026]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:36.034 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:35 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/554441523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:36.034 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:35 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:36.035 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:35 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:36.035 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:35 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:37:36.461 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:36.086+0000 7f24f4295200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T10:37:36.461 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:36.274+0000 7f24f4295200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:37:36.713 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:36.459+0000 7f24f4295200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T10:37:36.713 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:36.578+0000 7f24f4295200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T10:37:36.713 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: [09/Oct/2023:10:37:36] ENGINE Bus STARTING 2023-10-09T10:37:36.713 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: CherryPy Checker: 2023-10-09T10:37:36.713 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: The Application mounted at '' has an empty config. 2023-10-09T10:37:36.713 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:36.713 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: [09/Oct/2023:10:37:36] ENGINE Serving on http://:::9283 2023-10-09T10:37:36.714 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: [09/Oct/2023:10:37:36] ENGINE Bus STARTED 2023-10-09T10:37:36.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:36 smithi125 ceph-mon[127026]: Standby manager daemon x restarted 2023-10-09T10:37:36.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:36 smithi125 ceph-mon[127026]: Standby manager daemon x started 2023-10-09T10:37:36.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:36 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:37:36.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:36 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:37:36.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:36 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:37:36.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:36 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:37:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:36 smithi125 ceph-mon[112589]: Standby manager daemon x restarted 2023-10-09T10:37:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:36 smithi125 ceph-mon[112589]: Standby manager daemon x started 2023-10-09T10:37:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:36 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:37:36.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:36 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:37:36.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:36 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:37:36.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:36 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:37:36.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-mon[117223]: Standby manager daemon x restarted 2023-10-09T10:37:36.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-mon[117223]: Standby manager daemon x started 2023-10-09T10:37:36.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:37:36.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:37:36.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:37:36.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:36 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:37:37.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:37.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: mgrmap e20: y(active, since 2m), standbys: x 2023-10-09T10:37:37.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:37.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:37.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:37:37.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:37:37.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:37.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:37:37.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T10:37:37.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: osdmap e73: 8 total, 8 up, 8 in 2023-10-09T10:37:37.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T10:37:37.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: mgrmap e21: x(active, starting, since 0.00552386s) 2023-10-09T10:37:37.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:37:37.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:37:37.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:37:37.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.ydxmyq"}]: dispatch 2023-10-09T10:37:37.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.satgiw"}]: dispatch 2023-10-09T10:37:37.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T10:37:37.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:37:37.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:37:37.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:37:37.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:37:37.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:37:37.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:37:37.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:37:37.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:37:37.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2530923618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:37.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:37:37.974 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: [09/Oct/2023:10:37:37] ENGINE Bus STOPPING 2023-10-09T10:37:37.974 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: [09/Oct/2023:10:37:37] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T10:37:37.974 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: [09/Oct/2023:10:37:37] ENGINE Bus STOPPED 2023-10-09T10:37:37.974 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: [09/Oct/2023:10:37:37] ENGINE Bus STARTING 2023-10-09T10:37:37.974 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:37.808+0000 7f244e70c700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:37:37.974 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:37.808+0000 7f244e70c700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:37:37.974 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:37.808+0000 7f244e70c700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:37:37.974 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:37.808+0000 7f244e70c700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:37:37.974 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:37.808+0000 7f244e70c700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:37:37.974 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:37.809+0000 7f2451f13700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:37:37.975 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:37.809+0000 7f2451f13700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:37:37.975 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:37.809+0000 7f2451f13700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:37:37.975 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:37.809+0000 7f2451f13700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:37:37.975 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:37:37.809+0000 7f2451f13700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:37:37.975 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: [09/Oct/2023:10:37:37] ENGINE Serving on http://:::9283 2023-10-09T10:37:37.975 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: [09/Oct/2023:10:37:37] ENGINE Bus STARTED 2023-10-09T10:37:37.975 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:37:37.975 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:37:37.975 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:37 smithi155 ceph-mon[117223]: Manager daemon x is now available 2023-10-09T10:37:37.990 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:37.990 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: mgrmap e20: y(active, since 2m), standbys: x 2023-10-09T10:37:37.990 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:37.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:37.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:37:37.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:37:37.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:37.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:37:37.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T10:37:37.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: osdmap e73: 8 total, 8 up, 8 in 2023-10-09T10:37:37.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T10:37:37.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: mgrmap e21: x(active, starting, since 0.00552386s) 2023-10-09T10:37:37.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:37:37.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:37:37.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:37:37.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.ydxmyq"}]: dispatch 2023-10-09T10:37:37.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.satgiw"}]: dispatch 2023-10-09T10:37:37.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T10:37:37.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:37:37.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:37:37.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:37:37.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:37:37.993 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:37:37.993 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:37:37.993 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:37:37.993 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:37:37.993 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2530923618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:37.993 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:37:37.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:37.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: mgrmap e20: y(active, since 2m), standbys: x 2023-10-09T10:37:37.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:37.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:37.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:37:37.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:37:37.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' 2023-10-09T10:37:37.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:37:37.995 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T10:37:37.995 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: osdmap e73: 8 total, 8 up, 8 in 2023-10-09T10:37:37.995 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24362 172.21.15.125:0/453659801' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T10:37:37.995 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: mgrmap e21: x(active, starting, since 0.00552386s) 2023-10-09T10:37:37.995 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:37:37.995 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:37:37.995 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:37:37.995 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.ydxmyq"}]: dispatch 2023-10-09T10:37:37.996 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.satgiw"}]: dispatch 2023-10-09T10:37:37.996 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T10:37:37.996 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:37:37.996 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:37:37.996 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:37:37.996 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:37:37.996 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:37:37.996 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:37:37.997 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:37:37.997 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:37:37.997 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2530923618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:37.997 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:37:37.997 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:37:37.997 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:37:37.997 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[112589]: Manager daemon x is now available 2023-10-09T10:37:37.998 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:37:37.998 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:37:37.998 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mon[127026]: Manager daemon x is now available 2023-10-09T10:37:37.998 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:37.651+0000 7f3ec9d7e700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T10:37:37.998 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ignoring --setuser ceph since I am not root 2023-10-09T10:37:37.999 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ignoring --setgroup ceph since I am not root 2023-10-09T10:37:37.999 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-mgr[112861]: -- 172.21.15.125:0/913491766 <== mon.0 v2:172.21.15.125:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ffc2bba9c0 con 0x55ffc2b91000 2023-10-09T10:37:37.999 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:37.823+0000 7faff0ed01c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T10:37:37.999 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:37 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:37.989+0000 7faff0ed01c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T10:37:38.720 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:38.321+0000 7faff0ed01c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T10:37:39.013 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:38.842+0000 7faff0ed01c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T10:37:39.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:39.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:37:39.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:37:39.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:37:39.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:37:39.014 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:37:39.014 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T10:37:39.014 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T10:37:39.014 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[112589]: Deploying cephadm binary to smithi125 2023-10-09T10:37:39.014 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[112589]: Deploying cephadm binary to smithi155 2023-10-09T10:37:39.014 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[112589]: mgrmap e22: x(active, since 1.00738s) 2023-10-09T10:37:39.014 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:39.014 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:37:39.014 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:37:39.015 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:37:39.015 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:37:39.015 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:37:39.015 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T10:37:39.015 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T10:37:39.015 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[127026]: Deploying cephadm binary to smithi125 2023-10-09T10:37:39.015 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[127026]: Deploying cephadm binary to smithi155 2023-10-09T10:37:39.015 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:38 smithi125 ceph-mon[127026]: mgrmap e22: x(active, since 1.00738s) 2023-10-09T10:37:39.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:38 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:39.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:38 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:37:39.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:38 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:37:39.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:38 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:37:39.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:38 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:37:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:38 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:37:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:38 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T10:37:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:38 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T10:37:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:38 smithi155 ceph-mon[117223]: Deploying cephadm binary to smithi125 2023-10-09T10:37:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:38 smithi155 ceph-mon[117223]: Deploying cephadm binary to smithi155 2023-10-09T10:37:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:38 smithi155 ceph-mon[117223]: mgrmap e22: x(active, since 1.00738s) 2023-10-09T10:37:39.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:39.011+0000 7faff0ed01c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T10:37:39.820 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:39.404+0000 7faff0ed01c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T10:37:40.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:39 smithi155 ceph-mon[117223]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:40.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:39 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:40.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:39 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:40.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:39 smithi155 ceph-mon[117223]: [09/Oct/2023:10:37:39] ENGINE Bus STARTING 2023-10-09T10:37:40.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:39 smithi155 ceph-mon[117223]: [09/Oct/2023:10:37:39] ENGINE Serving on https://172.21.15.155:7150 2023-10-09T10:37:40.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:39 smithi125 ceph-mon[112589]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:40.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:39 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:40.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:39 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:40.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:39 smithi125 ceph-mon[112589]: [09/Oct/2023:10:37:39] ENGINE Bus STARTING 2023-10-09T10:37:40.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:39 smithi125 ceph-mon[112589]: [09/Oct/2023:10:37:39] ENGINE Serving on https://172.21.15.155:7150 2023-10-09T10:37:40.222 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:39 smithi125 ceph-mon[127026]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:40.222 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:39 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:40.222 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:39 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:40.222 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:39 smithi125 ceph-mon[127026]: [09/Oct/2023:10:37:39] ENGINE Bus STARTING 2023-10-09T10:37:40.223 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:39 smithi125 ceph-mon[127026]: [09/Oct/2023:10:37:39] ENGINE Serving on https://172.21.15.155:7150 2023-10-09T10:37:40.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:40 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:40.220+0000 7faff0ed01c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T10:37:40.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:40 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:40.312+0000 7faff0ed01c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T10:37:41.166 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:40 smithi125 ceph-mon[112589]: [09/Oct/2023:10:37:39] ENGINE Serving on http://172.21.15.155:8765 2023-10-09T10:37:41.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:40 smithi125 ceph-mon[112589]: [09/Oct/2023:10:37:39] ENGINE Bus STARTED 2023-10-09T10:37:41.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:40 smithi125 ceph-mon[112589]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:41.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:40 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1477399522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:41.167 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:40 smithi125 ceph-mon[127026]: [09/Oct/2023:10:37:39] ENGINE Serving on http://172.21.15.155:8765 2023-10-09T10:37:41.167 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:40 smithi125 ceph-mon[127026]: [09/Oct/2023:10:37:39] ENGINE Bus STARTED 2023-10-09T10:37:41.167 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:40 smithi125 ceph-mon[127026]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:41.168 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:40 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/1477399522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:41.168 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:40 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:40.974+0000 7faff0ed01c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T10:37:41.168 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:41.073+0000 7faff0ed01c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T10:37:41.168 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:41.165+0000 7faff0ed01c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T10:37:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:40 smithi155 ceph-mon[117223]: [09/Oct/2023:10:37:39] ENGINE Serving on http://172.21.15.155:8765 2023-10-09T10:37:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:40 smithi155 ceph-mon[117223]: [09/Oct/2023:10:37:39] ENGINE Bus STARTED 2023-10-09T10:37:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:40 smithi155 ceph-mon[117223]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:41.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:40 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1477399522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:41.505 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:41.266+0000 7faff0ed01c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T10:37:41.820 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:41.504+0000 7faff0ed01c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T10:37:41.820 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:41.603+0000 7faff0ed01c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T10:37:42.142 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:41.833+0000 7faff0ed01c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T10:37:42.142 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:41.937+0000 7faff0ed01c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T10:37:42.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:37:41] "GET /metrics HTTP/1.1" 200 31032 "" "Prometheus/2.43.0" 2023-10-09T10:37:42.510 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:42.141+0000 7faff0ed01c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T10:37:42.510 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:42.236+0000 7faff0ed01c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T10:37:42.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: Adjusting osd_memory_target on smithi125 to 2486M 2023-10-09T10:37:42.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3132369544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:42.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:42.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: Adjusting osd_memory_target on smithi125 to 2486M 2023-10-09T10:37:42.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3132369544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:42.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:42.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: Adjusting osd_memory_target on smithi125 to 2486M 2023-10-09T10:37:42.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:42.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:42 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3132369544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:43.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:42 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:42.984+0000 7faff0ed01c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T10:37:43.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:43 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:43.180+0000 7faff0ed01c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T10:37:43.616 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:43 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:43.359+0000 7faff0ed01c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:37:43.968 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:43 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:43.968 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:43 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:43.968 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:43 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:43.968 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:43 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:43 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:43 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:44.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:43 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:43.967+0000 7faff0ed01c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T10:37:44.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:44.150+0000 7faff0ed01c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:37:44.625 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:44.336+0000 7faff0ed01c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T10:37:44.625 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:44.454+0000 7faff0ed01c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T10:37:44.625 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:37:44] ENGINE Bus STARTING 2023-10-09T10:37:44.626 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: CherryPy Checker: 2023-10-09T10:37:44.626 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: The Application mounted at '' has an empty config. 2023-10-09T10:37:44.626 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: 2023-10-09T10:37:44.626 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:37:44] ENGINE Serving on http://:::9283 2023-10-09T10:37:44.626 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: [09/Oct/2023:10:37:44] ENGINE Bus STARTED 2023-10-09T10:37:44.940 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:44 smithi155 ceph-mon[117223]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:44.940 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:44 smithi155 ceph-mon[117223]: Standby manager daemon y started 2023-10-09T10:37:44.940 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:44 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.125:0/2991258522' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T10:37:44.940 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:44 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.125:0/2991258522' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:37:44.940 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:44 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.125:0/2991258522' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T10:37:44.940 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:44 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.125:0/2991258522' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:37:45.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-mon[112589]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:45.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-mon[112589]: Standby manager daemon y started 2023-10-09T10:37:45.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.125:0/2991258522' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T10:37:45.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.125:0/2991258522' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:37:45.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.125:0/2991258522' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T10:37:45.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.125:0/2991258522' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:37:45.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-mon[127026]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:45.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-mon[127026]: Standby manager daemon y started 2023-10-09T10:37:45.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.125:0/2991258522' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T10:37:45.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.125:0/2991258522' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:37:45.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.125:0/2991258522' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T10:37:45.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:44 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.125:0/2991258522' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:37:45.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: mgrmap e23: x(active, since 6s), standbys: y 2023-10-09T10:37:45.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:37:45.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/4021943166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:45.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: Adjusting osd_memory_target on smithi155 to 2486M 2023-10-09T10:37:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:45.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:37:45.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:37:45.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:37:45.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:37:45.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:37:45.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:45 smithi155 ceph-mon[117223]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:37:46.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: mgrmap e23: x(active, since 6s), standbys: y 2023-10-09T10:37:46.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:37:46.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/4021943166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:46.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:46.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:46.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: Adjusting osd_memory_target on smithi155 to 2486M 2023-10-09T10:37:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:37:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:37:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:37:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:37:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:37:46.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[112589]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:37:46.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: mgrmap e23: x(active, since 6s), standbys: y 2023-10-09T10:37:46.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:37:46.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/4021943166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:46.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:46.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:46.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T10:37:46.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: Adjusting osd_memory_target on smithi155 to 2486M 2023-10-09T10:37:46.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:46.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:37:46.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:37:46.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:37:46.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:37:46.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:37:46.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:45 smithi125 ceph-mon[127026]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:37:47.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:37:47.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:37:47.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:37:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:37:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T10:37:47.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:47 smithi155 ceph-mon[117223]: Reconfiguring daemon alertmanager.a on smithi125 2023-10-09T10:37:47.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:37:47.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:37:47.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:47.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:37:47.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:37:47.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T10:37:47.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[112589]: Reconfiguring daemon alertmanager.a on smithi125 2023-10-09T10:37:47.487 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:37:47.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:37:47.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:47.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:37:47.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:37:47.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.489 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:47.489 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T10:37:47.489 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:47 smithi125 ceph-mon[127026]: Reconfiguring daemon alertmanager.a on smithi125 2023-10-09T10:37:48.137 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:48 smithi125 systemd[1]: Stopping Ceph alertmanager.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:37:48.450 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:48 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1515311325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:48.450 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:48 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/1515311325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:48.450 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:48 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[169522]: ts=2023-10-09T10:37:48.264Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T10:37:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:48 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1515311325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:48.748 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:48 smithi125 bash[182161]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T10:37:49.070 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:48 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T10:37:49.070 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:48 smithi125 systemd[1]: Stopped Ceph alertmanager.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:37:49.070 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:48 smithi125 systemd[1]: Starting Ceph alertmanager.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:37:49.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:37:48 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[112838]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:37:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T10:37:49.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:49 smithi155 ceph-mon[117223]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:49.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:49 smithi125 ceph-mon[112589]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:49.547 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:49 smithi125 ceph-mon[127026]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:49.547 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:49 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[182292]: ts=2023-10-09T10:37:49.183Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-10-09T10:37:49.547 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:49 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[182292]: ts=2023-10-09T10:37:49.183Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-10-09T10:37:49.547 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:49 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[182292]: ts=2023-10-09T10:37:49.183Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.125 port=9094 2023-10-09T10:37:49.547 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:49 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[182292]: ts=2023-10-09T10:37:49.184Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T10:37:49.547 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:49 smithi125 bash[182277]: 3c13e6c17b7e301d66cbf51e98186873decd0f6580eb24c9574801b9fde7fb99 2023-10-09T10:37:49.547 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:49 smithi125 systemd[1]: Started Ceph alertmanager.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:37:49.548 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:49 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[182292]: ts=2023-10-09T10:37:49.208Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T10:37:49.548 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:49 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[182292]: ts=2023-10-09T10:37:49.208Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T10:37:49.548 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:49 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[182292]: ts=2023-10-09T10:37:49.211Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T10:37:49.548 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:49 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[182292]: ts=2023-10-09T10:37:49.211Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T10:37:49.820 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:49 smithi125 systemd[1]: Stopping Ceph node-exporter.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:37:50.158 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:49 smithi125 bash[182478]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T10:37:50.158 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T10:37:50.507 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T10:37:50.507 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 systemd[1]: Stopped Ceph node-exporter.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:37:50.507 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 systemd[1]: Starting Ceph node-exporter.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:37:50.507 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:50.507 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:50.508 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-mon[112589]: Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T10:37:50.508 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-mon[112589]: Reconfiguring daemon node-exporter.a on smithi125 2023-10-09T10:37:50.508 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2355223045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:50.508 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:50.508 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:50.508 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-mon[127026]: Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T10:37:50.508 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-mon[127026]: Reconfiguring daemon node-exporter.a on smithi125 2023-10-09T10:37:50.508 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2355223045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:50 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:50 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:50 smithi155 ceph-mon[117223]: Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T10:37:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:50 smithi155 ceph-mon[117223]: Reconfiguring daemon node-exporter.a on smithi125 2023-10-09T10:37:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:50 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2355223045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:50.821 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.506Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T10:37:50.821 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.506Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T10:37:50.821 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.507Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T10:37:50.821 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.507Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T10:37:50.821 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.507Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T10:37:50.821 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.507Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T10:37:50.821 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T10:37:50.822 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T10:37:50.822 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T10:37:50.822 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T10:37:50.822 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T10:37:50.822 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T10:37:50.822 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T10:37:50.822 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T10:37:50.822 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T10:37:50.822 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T10:37:50.822 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T10:37:50.823 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T10:37:50.823 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T10:37:50.823 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T10:37:50.823 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T10:37:50.823 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T10:37:50.823 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T10:37:50.823 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T10:37:50.823 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T10:37:50.823 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T10:37:50.823 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T10:37:50.823 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T10:37:50.823 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T10:37:50.824 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T10:37:50.824 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T10:37:50.824 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T10:37:50.824 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T10:37:50.824 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T10:37:50.824 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T10:37:50.824 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T10:37:50.824 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T10:37:50.824 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T10:37:50.824 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T10:37:50.824 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T10:37:50.825 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T10:37:50.825 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T10:37:50.825 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T10:37:50.825 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T10:37:50.825 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T10:37:50.825 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T10:37:50.825 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T10:37:50.825 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T10:37:50.825 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T10:37:50.825 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T10:37:50.825 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T10:37:50.826 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T10:37:50.826 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a[182613]: ts=2023-10-09T10:37:50.508Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T10:37:50.826 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 bash[182598]: bf5a342348bd193fb50cddbb0e11f526ad78f7acd431fc84ce40663efaf08f6e 2023-10-09T10:37:50.826 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 10:37:50 smithi125 systemd[1]: Started Ceph node-exporter.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:37:51.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:51 smithi155 ceph-mon[117223]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:51.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:51 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:51.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:51 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:51.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:51 smithi125 ceph-mon[112589]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:51.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:51 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:51.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:51 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:51.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:51 smithi125 ceph-mon[127026]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:51.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:51 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:51.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:51 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:51.571 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[182292]: ts=2023-10-09T10:37:51.184Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000197822s 2023-10-09T10:37:52.128 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:51 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:37:51] "GET /metrics HTTP/1.1" 200 31032 "" "Prometheus/2.43.0" 2023-10-09T10:37:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:52 smithi155 ceph-mon[117223]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T10:37:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:52 smithi155 ceph-mon[117223]: Reconfiguring daemon prometheus.a on smithi155 2023-10-09T10:37:52.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:52 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/4257777224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:52.471 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 systemd[1]: Stopping Ceph prometheus.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:37:52.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:52 smithi125 ceph-mon[112589]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T10:37:52.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:52 smithi125 ceph-mon[112589]: Reconfiguring daemon prometheus.a on smithi155 2023-10-09T10:37:52.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:52 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/4257777224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:52.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:52 smithi125 ceph-mon[127026]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T10:37:52.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:52 smithi125 ceph-mon[127026]: Reconfiguring daemon prometheus.a on smithi155 2023-10-09T10:37:52.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:52 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/4257777224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:52.871 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:37:52.525Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T10:37:52.871 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:37:52.525Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T10:37:52.871 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:37:52.525Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T10:37:52.872 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:37:52.525Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T10:37:52.872 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:37:52.525Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T10:37:52.872 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:37:52.525Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T10:37:52.872 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:37:52.525Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T10:37:52.872 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:37:52.525Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T10:37:52.872 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:37:52.525Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T10:37:52.872 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:37:52.526Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T10:37:52.872 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:37:52.526Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T10:37:52.872 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[154667]: ts=2023-10-09T10:37:52.526Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T10:37:52.872 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 bash[168592]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T10:37:53.220 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T10:37:53.220 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 systemd[1]: Stopped Ceph prometheus.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:37:53.220 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:52 smithi155 systemd[1]: Starting Ceph prometheus.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:37:53.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:53 smithi125 ceph-mon[112589]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:53.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:53 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:37:53.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:53 smithi125 ceph-mon[127026]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:53.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:53 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:37:53.600 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-mon[117223]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:53.600 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:37:53.600 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.221Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T10:37:53.600 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.222Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T10:37:53.601 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.222Z caller=main.go:570 level=info host_details="(Linux 4.18.0-514.el8.x86_64 #1 SMP Tue Sep 19 13:19:58 UTC 2023 x86_64 smithi155 (none))" 2023-10-09T10:37:53.601 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.222Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T10:37:53.601 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.222Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T10:37:53.601 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.223Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T10:37:53.601 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.223Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T10:37:53.601 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.224Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T10:37:53.601 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.224Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T10:37:53.601 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 bash[168709]: 6e65f8d4e3b46c82bd12caa6c7a4c00d581212e5d1cc9b50ddb042afa5c2243f 2023-10-09T10:37:53.601 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.227Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T10:37:53.601 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.227Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.486µs 2023-10-09T10:37:53.601 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.227Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T10:37:53.602 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.228Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-10-09T10:37:53.602 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 systemd[1]: Started Ceph prometheus.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:37:53.602 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.242Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-10-09T10:37:53.602 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.242Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-10-09T10:37:53.602 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.242Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=31.668µs wal_replay_duration=14.449466ms wbl_replay_duration=95ns total_replay_duration=14.509765ms 2023-10-09T10:37:53.602 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.245Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T10:37:53.602 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.245Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T10:37:53.602 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.245Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T10:37:53.602 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.263Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.38142ms db_storage=1.416µs remote_storage=1.246µs web_handler=433ns query_engine=731ns scrape=610.656µs scrape_sd=69.124µs notify=15.313µs notify_sd=11.949µs rules=17.435231ms tracing=6.449µs 2023-10-09T10:37:53.602 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.263Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T10:37:53.602 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:37:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:37:53.263Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T10:37:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:54 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:54 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:54 smithi155 ceph-mon[117223]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T10:37:54.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:54 smithi155 ceph-mon[117223]: Reconfiguring daemon grafana.a on smithi155 2023-10-09T10:37:54.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:54 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:54.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:54 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:54.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:54 smithi125 ceph-mon[112589]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T10:37:54.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:54 smithi125 ceph-mon[112589]: Reconfiguring daemon grafana.a on smithi155 2023-10-09T10:37:54.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:54 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:54.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:54 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:37:54.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:54 smithi125 ceph-mon[127026]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T10:37:54.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:54 smithi125 ceph-mon[127026]: Reconfiguring daemon grafana.a on smithi155 2023-10-09T10:37:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:55 smithi155 ceph-mon[117223]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:55 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/40045826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:55.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:55 smithi125 ceph-mon[112589]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:55.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:55 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/40045826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:55.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:55 smithi125 ceph-mon[127026]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:55.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:55 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/40045826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:55.970 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:55 smithi155 systemd[1]: Stopping Ceph grafana.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:37:56.720 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:56 smithi155 bash[169014]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T10:37:57.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:57 smithi125 ceph-mon[127026]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:57.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:57 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2833170220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:57.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:57 smithi125 ceph-mon[112589]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:57.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:57 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2833170220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:57.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:57 smithi155 ceph-mon[117223]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:57.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:57 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2833170220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:37:58.220 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:57 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T10:37:58.220 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:57 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-10-09T10:37:58.220 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:57 smithi155 systemd[1]: Stopped Ceph grafana.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:37:58.221 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:57 smithi155 systemd[1]: Starting Ceph grafana.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:37:59.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:37:59 smithi125 ceph-mon[112589]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:59.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:37:59 smithi125 ceph-mon[127026]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:59.572 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:37:59 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[182292]: ts=2023-10-09T10:37:59.188Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003413966s 2023-10-09T10:37:59.713 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-mon[117223]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:37:59.714 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:37:59] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2023-10-09T10:37:59.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=settings t=2023-10-09T10:37:59.713514265Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-10-09T10:37:59.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=settings t=2023-10-09T10:37:59.713699765Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-10-09T10:37:59.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=settings t=2023-10-09T10:37:59.713717388Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-10-09T10:37:59.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=settings t=2023-10-09T10:37:59.713724236Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-10-09T10:37:59.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=settings t=2023-10-09T10:37:59.713730376Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T10:37:59.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=settings t=2023-10-09T10:37:59.713736435Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T10:37:59.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=settings t=2023-10-09T10:37:59.713742374Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T10:37:59.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=settings t=2023-10-09T10:37:59.713748206Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T10:37:59.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=settings t=2023-10-09T10:37:59.713754108Z level=info msg="Path Home" path=/usr/share/grafana 2023-10-09T10:37:59.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=settings t=2023-10-09T10:37:59.713760563Z level=info msg="Path Data" path=/var/lib/grafana 2023-10-09T10:37:59.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=settings t=2023-10-09T10:37:59.713770556Z level=info msg="Path Logs" path=/var/log/grafana 2023-10-09T10:37:59.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=settings t=2023-10-09T10:37:59.713779684Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-10-09T10:37:59.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=settings t=2023-10-09T10:37:59.713789268Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-10-09T10:37:59.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=settings t=2023-10-09T10:37:59.713799188Z level=info msg="App mode production" 2023-10-09T10:37:59.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=sqlstore t=2023-10-09T10:37:59.713855733Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-10-09T10:37:59.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=sqlstore t=2023-10-09T10:37:59.713874011Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T10:37:59.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=migrator t=2023-10-09T10:37:59.727867469Z level=info msg="Starting DB migrations" 2023-10-09T10:37:59.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=migrator t=2023-10-09T10:37:59.732865997Z level=info msg="migrations completed" performed=0 skipped=481 duration=502.846µs 2023-10-09T10:37:59.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=sqlstore t=2023-10-09T10:37:59.73323149Z level=info msg="Created default organization" 2023-10-09T10:37:59.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=plugin.loader t=2023-10-09T10:37:59.790846078Z level=info msg="Plugin registered" pluginID=input 2023-10-09T10:37:59.972 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 bash[169131]: 0ec8893d264dff004049fb49b7659d15ed0850bf0497c70006591f42f110890c 2023-10-09T10:37:59.973 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 systemd[1]: Started Ceph grafana.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:37:59.973 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=plugin.loader t=2023-10-09T10:37:59.819636974Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-10-09T10:37:59.973 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=plugin.loader t=2023-10-09T10:37:59.819688608Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-10-09T10:37:59.973 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=secrets t=2023-10-09T10:37:59.81976256Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-10-09T10:37:59.973 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=query_data t=2023-10-09T10:37:59.823130226Z level=info msg="Query Service initialization" 2023-10-09T10:37:59.973 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:37:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=live.push_http t=2023-10-09T10:37:59.82637843Z level=info msg="Live Push Gateway initialization" 2023-10-09T10:38:00.470 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:00 smithi155 systemd[1]: Stopping Ceph node-exporter.b for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:38:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:00 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1418967815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:00 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:00 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:00.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:00 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1418967815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:00.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:00 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:00.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:00 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:00.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:00 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/1418967815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:00.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:00 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:00.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:00 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:00.970 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:38:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=infra.usagestats.collector t=2023-10-09T10:38:00.613379196Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-10-09T10:38:00.970 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:38:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=provisioning.datasources t=2023-10-09T10:38:00.701053194Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2023-10-09T10:38:00.970 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:38:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=provisioning.datasources t=2023-10-09T10:38:00.701096817Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2023-10-09T10:38:00.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:38:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=provisioning.datasources t=2023-10-09T10:38:00.712326347Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T10:38:00.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:38:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=provisioning.alerting t=2023-10-09T10:38:00.792340917Z level=info msg="starting to provision alerting" 2023-10-09T10:38:00.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:38:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=provisioning.alerting t=2023-10-09T10:38:00.792363288Z level=info msg="finished to provision alerting" 2023-10-09T10:38:00.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:38:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=ngalert.state.manager t=2023-10-09T10:38:00.792540002Z level=info msg="Warming state cache for startup" 2023-10-09T10:38:00.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:38:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=grafanaStorageLogger t=2023-10-09T10:38:00.792551462Z level=info msg="storage starting" 2023-10-09T10:38:00.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:38:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=http.server t=2023-10-09T10:38:00.794470426Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T10:38:00.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:38:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=ngalert.state.manager t=2023-10-09T10:38:00.802714518Z level=info msg="State cache has been initialized" states=0 duration=10.173297ms 2023-10-09T10:38:00.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:38:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=ticker t=2023-10-09T10:38:00.80277762Z level=info msg=starting first_tick=2023-10-09T10:38:10Z 2023-10-09T10:38:00.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:38:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=ngalert.multiorg.alertmanager t=2023-10-09T10:38:00.802773208Z level=info msg="starting MultiOrg Alertmanager" 2023-10-09T10:38:01.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:01 smithi125 ceph-mon[112589]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:01.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:01 smithi125 ceph-mon[112589]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T10:38:01.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:01 smithi125 ceph-mon[112589]: Reconfiguring daemon node-exporter.b on smithi155 2023-10-09T10:38:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:01 smithi125 ceph-mon[127026]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:01 smithi125 ceph-mon[127026]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T10:38:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:01 smithi125 ceph-mon[127026]: Reconfiguring daemon node-exporter.b on smithi155 2023-10-09T10:38:01.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:01 smithi155 ceph-mon[117223]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:01.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:01 smithi155 ceph-mon[117223]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T10:38:01.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:01 smithi155 ceph-mon[117223]: Reconfiguring daemon node-exporter.b on smithi155 2023-10-09T10:38:01.721 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:01 smithi155 bash[169311]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T10:38:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:02 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/635485722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:02.470 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:02 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T10:38:02.471 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:02 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T10:38:02.471 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:02 smithi155 systemd[1]: Stopped Ceph node-exporter.b for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:38:02.471 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:02 smithi155 systemd[1]: Starting Ceph node-exporter.b for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:38:02.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:02 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/635485722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:02.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:02 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/635485722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:03 smithi125 ceph-mon[112589]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:03.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:03 smithi125 ceph-mon[127026]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:03.621 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-mon[117223]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:03.971 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.621Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T10:38:03.971 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.621Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T10:38:03.971 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.621Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T10:38:03.971 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.621Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T10:38:03.971 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T10:38:03.971 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T10:38:03.971 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T10:38:03.971 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T10:38:03.972 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T10:38:03.972 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T10:38:03.972 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T10:38:03.972 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T10:38:03.972 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T10:38:03.972 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T10:38:03.972 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T10:38:03.972 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T10:38:03.972 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T10:38:03.972 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T10:38:03.972 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T10:38:03.973 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T10:38:03.973 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T10:38:03.973 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T10:38:03.973 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T10:38:03.973 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T10:38:03.973 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T10:38:03.973 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T10:38:03.973 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T10:38:03.973 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T10:38:03.973 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T10:38:03.973 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T10:38:03.973 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T10:38:03.973 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T10:38:03.974 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T10:38:03.974 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T10:38:03.974 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T10:38:03.974 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T10:38:03.974 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T10:38:03.974 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T10:38:03.974 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T10:38:03.974 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T10:38:03.974 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T10:38:03.974 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T10:38:03.974 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T10:38:03.974 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T10:38:03.975 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T10:38:03.975 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T10:38:03.975 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T10:38:03.975 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T10:38:03.975 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T10:38:03.975 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T10:38:03.975 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.622Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T10:38:03.975 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.623Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T10:38:03.975 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b[169445]: ts=2023-10-09T10:38:03.623Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T10:38:03.975 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 bash[169431]: 1320bf6a6ac55260b78e122b4b7ee055366923bb18fbed0b37789bdcb6db2971 2023-10-09T10:38:03.975 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 10:38:03 smithi155 systemd[1]: Started Ceph node-exporter.b for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:38:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:04 smithi155 ceph-mon[117223]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:04 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:04 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:04 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T10:38:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:04 smithi155 ceph-mon[117223]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T10:38:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:04 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:38:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:04 smithi155 ceph-mon[117223]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:38:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:04 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T10:38:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:04 smithi155 ceph-mon[117223]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T10:38:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:04 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:04 smithi155 ceph-mon[117223]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T10:38:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:04 smithi155 ceph-mon[117223]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T10:38:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:04 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2415408737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:05.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[112589]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:05.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:05.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:05.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T10:38:05.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[112589]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T10:38:05.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:38:05.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[112589]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:38:05.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T10:38:05.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[112589]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T10:38:05.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:05.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[112589]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T10:38:05.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[112589]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T10:38:05.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2415408737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:05.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[127026]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:05.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:05.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:05.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T10:38:05.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[127026]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T10:38:05.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:38:05.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[127026]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:38:05.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T10:38:05.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[127026]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T10:38:05.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:05.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[127026]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T10:38:05.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[127026]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T10:38:05.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:04 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2415408737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:05.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:05 smithi125 ceph-mon[112589]: Upgrade: Updating mgr.y 2023-10-09T10:38:05.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:05 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:05.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:05 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:38:05.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:05 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:38:05.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:05 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:38:05.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:05 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:05.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:05 smithi125 ceph-mon[112589]: Deploying daemon mgr.y on smithi125 2023-10-09T10:38:05.779 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:05 smithi125 ceph-mon[127026]: Upgrade: Updating mgr.y 2023-10-09T10:38:05.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:05 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:05.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:05 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:38:05.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:05 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:38:05.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:05 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:38:05.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:05 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:05.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:05 smithi125 ceph-mon[127026]: Deploying daemon mgr.y on smithi125 2023-10-09T10:38:06.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:05 smithi125 systemd[1]: Stopping Ceph mgr.y for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:38:06.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:05 smithi155 ceph-mon[117223]: Upgrade: Updating mgr.y 2023-10-09T10:38:06.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:05 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:06.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:05 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:38:06.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:05 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:38:06.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:05 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:38:06.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:05 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:06.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:05 smithi155 ceph-mon[117223]: Deploying daemon mgr.y on smithi125 2023-10-09T10:38:06.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:06 smithi125 bash[183164]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T10:38:06.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:06 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-10-09T10:38:07.042 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:06 smithi125 ceph-mon[112589]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:07.042 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:06 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3254594496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:07.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:06 smithi125 ceph-mon[127026]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:07.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:06 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3254594496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:07.043 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:06 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-10-09T10:38:07.043 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:06 smithi125 systemd[1]: Stopped Ceph mgr.y for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:38:07.043 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:06 smithi125 systemd[1]: Starting Ceph mgr.y for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:38:07.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:06 smithi155 ceph-mon[117223]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:07.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:06 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3254594496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:07.464 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:07 smithi125 ceph-mgr[183374]: -- 172.21.15.125:0/3320652048 <== mon.2 v2:172.21.15.125:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556e8dfb51e0 con 0x556e8df8ac00 2023-10-09T10:38:07.464 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:07 smithi125 bash[183337]: 107edeccfff9d34ea9b04d810ee78d09ec21e0cc7fb880a08191ada78b5375d8 2023-10-09T10:38:07.465 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:07 smithi125 systemd[1]: Started Ceph mgr.y for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:38:07.465 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:07 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:07.289+0000 7fad8a20c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T10:38:07.465 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:07 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:07.463+0000 7fad8a20c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T10:38:08.268 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:07 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:07.830+0000 7fad8a20c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T10:38:08.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:08 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:08.364+0000 7fad8a20c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T10:38:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:08 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:08 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:08 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:08 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:38:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:08 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:08.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:08 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:08.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:08 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:08.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:08 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:38:08.660 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:08 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:08.660 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:08 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:08.660 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:08 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:08.660 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:08 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:38:08.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:08 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:38:08.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:38:08.980 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:08 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:08.597+0000 7fad8a20c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T10:38:09.276 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:09 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:09.064+0000 7fad8a20c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T10:38:09.276 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:09 smithi125 ceph-mon[112589]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:09.276 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:09 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1574662695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:09.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:09 smithi125 ceph-mon[127026]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:09.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:09 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/1574662695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:09.591 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:09 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:38:09] "GET /metrics HTTP/1.1" 200 33872 "" "Prometheus/2.43.0" 2023-10-09T10:38:09.591 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:09 smithi155 ceph-mon[117223]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:09.592 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:09 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1574662695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:09.940 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:09 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:09.842+0000 7fad8a20c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T10:38:10.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:09 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:09.939+0000 7fad8a20c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T10:38:10.842 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:10 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:10.576+0000 7fad8a20c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T10:38:10.843 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:10 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:10.668+0000 7fad8a20c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T10:38:10.843 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:10 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:10.764+0000 7fad8a20c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T10:38:11.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:10 smithi125 ceph-mon[112589]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:11.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:10 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:11.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:10 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:11.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:10 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:11.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:10 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:11.134 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:11.003+0000 7fad8a20c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T10:38:11.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:10 smithi125 ceph-mon[127026]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:11.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:10 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:11.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:10 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:11.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:10 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:11.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:10 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:11.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:10 smithi155 ceph-mon[117223]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:11.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:10 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:11.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:10 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:11.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:10 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:11.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:10 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:11.486 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:11.132+0000 7fad8a20c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T10:38:11.486 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:11.237+0000 7fad8a20c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T10:38:11.487 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:11.485+0000 7fad8a20c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T10:38:11.793 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:11.593+0000 7fad8a20c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T10:38:11.794 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:11.792+0000 7fad8a20c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T10:38:12.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:11.894+0000 7fad8a20c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T10:38:12.367 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:12 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:12.367 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:12 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:12.367 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:12 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2521225182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:12.367 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:12 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:12.367 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:12 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:12.368 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:12 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2521225182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:12.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:12 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:12.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:12 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:12.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:12 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2521225182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:12.974 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:12 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:12.719+0000 7fad8a20c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T10:38:12.974 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:12 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:12.972+0000 7fad8a20c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T10:38:13.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:13 smithi125 ceph-mon[112589]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:13.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:13 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:13.163+0000 7fad8a20c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:38:13.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:13 smithi125 ceph-mon[127026]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:13.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:13 smithi155 ceph-mon[117223]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:14.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:13 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:13.883+0000 7fad8a20c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T10:38:14.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:13 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:13.940+0000 7f24e7942700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T10:38:14.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:13 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: ignoring --setuser ceph since I am not root 2023-10-09T10:38:14.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:13 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: ignoring --setgroup ceph since I am not root 2023-10-09T10:38:14.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:13 smithi155 ceph-mgr[157714]: -- 172.21.15.155:0/1933622565 <== mon.1 v2:172.21.15.155:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555859a011e0 con 0x5558599d7000 2023-10-09T10:38:14.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:14.110+0000 7f6b31080200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T10:38:14.393 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:14.079+0000 7fad8a20c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:38:14.393 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:14.271+0000 7fad8a20c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T10:38:14.609 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3118117849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:14.609 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:14.609 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:14.609 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:14.609 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:38:14.609 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:14.609 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:14.609 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T10:38:14.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T10:38:14.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: osdmap e74: 8 total, 8 up, 8 in 2023-10-09T10:38:14.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.24760 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T10:38:14.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: mgrmap e24: y(active, starting, since 0.00553895s) 2023-10-09T10:38:14.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: Active manager daemon y restarted 2023-10-09T10:38:14.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: Activating manager daemon y 2023-10-09T10:38:14.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T10:38:14.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: mgrmap e25: y(active, starting, since 0.00489352s) 2023-10-09T10:38:14.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T10:38:14.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T10:38:14.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:38:14.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:38:14.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:38:14.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.ydxmyq"}]: dispatch 2023-10-09T10:38:14.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.satgiw"}]: dispatch 2023-10-09T10:38:14.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:38:14.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:38:14.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:38:14.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:38:14.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:38:14.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:38:14.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:38:14.612 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:14.276+0000 7f6b31080200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T10:38:14.612 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:14.608+0000 7f6b31080200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T10:38:14.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:38:14.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:38:14.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:38:14.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:38:14.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:38:14.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:38:14.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T10:38:14.613 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:14 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:38:14.670 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:14.392+0000 7fad8a20c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T10:38:14.670 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:14] ENGINE Bus STARTING 2023-10-09T10:38:14.670 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: CherryPy Checker: 2023-10-09T10:38:14.671 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: The Application mounted at '' has an empty config. 2023-10-09T10:38:14.671 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:14.671 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:14] ENGINE Serving on http://:::9283 2023-10-09T10:38:14.671 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:14] ENGINE Bus STARTED 2023-10-09T10:38:14.671 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:14] ENGINE Bus STOPPING 2023-10-09T10:38:14.671 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T10:38:14.671 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:14] ENGINE Bus STOPPED 2023-10-09T10:38:14.671 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:14] ENGINE Bus STARTING 2023-10-09T10:38:14.671 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:14.668+0000 7face34c1700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:14.671 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:14.668+0000 7face34c1700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:14.672 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3118117849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:14.672 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:14.672 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:14.672 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:14.672 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:38:14.672 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:14.672 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:14.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T10:38:14.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T10:38:14.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: osdmap e74: 8 total, 8 up, 8 in 2023-10-09T10:38:14.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.24760 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T10:38:14.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: mgrmap e24: y(active, starting, since 0.00553895s) 2023-10-09T10:38:14.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: Active manager daemon y restarted 2023-10-09T10:38:14.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: Activating manager daemon y 2023-10-09T10:38:14.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T10:38:14.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: mgrmap e25: y(active, starting, since 0.00489352s) 2023-10-09T10:38:14.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T10:38:14.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T10:38:14.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:38:14.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:38:14.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:38:14.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.ydxmyq"}]: dispatch 2023-10-09T10:38:14.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.satgiw"}]: dispatch 2023-10-09T10:38:14.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:38:14.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:38:14.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:38:14.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:38:14.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:38:14.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:38:14.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:38:14.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:38:14.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:38:14.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:38:14.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:38:14.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:38:14.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:38:14.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T10:38:14.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:38:14.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3118117849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:14.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:14.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:14.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:14.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:38:14.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' 2023-10-09T10:38:14.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:14.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.24760 172.21.15.155:0/223759589' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T10:38:14.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T10:38:14.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: osdmap e74: 8 total, 8 up, 8 in 2023-10-09T10:38:14.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.24760 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T10:38:14.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: mgrmap e24: y(active, starting, since 0.00553895s) 2023-10-09T10:38:14.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: Active manager daemon y restarted 2023-10-09T10:38:14.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: Activating manager daemon y 2023-10-09T10:38:14.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T10:38:14.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: mgrmap e25: y(active, starting, since 0.00489352s) 2023-10-09T10:38:14.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T10:38:14.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T10:38:14.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:38:14.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:38:14.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:38:14.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.ydxmyq"}]: dispatch 2023-10-09T10:38:14.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.satgiw"}]: dispatch 2023-10-09T10:38:14.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:38:14.678 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:38:14.678 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:38:14.678 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:38:14.678 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:38:14.678 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:38:14.678 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:38:14.678 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:38:14.678 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:38:14.678 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:38:14.678 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:38:14.678 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:38:14.678 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:38:14.679 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T10:38:14.679 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:38:15.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:14.668+0000 7face34c1700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:15.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:14.668+0000 7face34c1700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:15.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:14.668+0000 7face34c1700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:15.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:14.675+0000 7facebcd2700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:15.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:14.675+0000 7facebcd2700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:15.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:14.675+0000 7facebcd2700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:15.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:14.675+0000 7facebcd2700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:15.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:14.675+0000 7facebcd2700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:15.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:14] ENGINE Serving on http://:::9283 2023-10-09T10:38:15.071 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:14] ENGINE Bus STARTED 2023-10-09T10:38:15.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:15 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:15.142+0000 7f6b31080200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T10:38:15.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:15 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:15.392+0000 7f6b31080200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T10:38:15.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:15 smithi155 ceph-mon[117223]: Manager daemon y is now available 2023-10-09T10:38:15.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:15 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:15.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:15 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:15.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:15 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:38:15.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:15 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:38:15.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:15 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T10:38:15.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:15 smithi155 ceph-mon[117223]: [09/Oct/2023:10:38:14] ENGINE Bus STARTING 2023-10-09T10:38:15.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:15 smithi155 ceph-mon[117223]: [09/Oct/2023:10:38:15] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T10:38:15.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[112589]: Manager daemon y is now available 2023-10-09T10:38:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:38:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:38:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T10:38:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[112589]: [09/Oct/2023:10:38:14] ENGINE Bus STARTING 2023-10-09T10:38:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[112589]: [09/Oct/2023:10:38:15] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T10:38:15.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[127026]: Manager daemon y is now available 2023-10-09T10:38:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:38:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:38:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T10:38:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[127026]: [09/Oct/2023:10:38:14] ENGINE Bus STARTING 2023-10-09T10:38:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:15 smithi125 ceph-mon[127026]: [09/Oct/2023:10:38:15] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T10:38:16.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:15 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:15.861+0000 7f6b31080200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T10:38:16.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:16 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:16.644+0000 7f6b31080200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T10:38:16.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:16 smithi155 ceph-mon[117223]: [09/Oct/2023:10:38:15] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T10:38:16.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:16 smithi155 ceph-mon[117223]: [09/Oct/2023:10:38:15] ENGINE Bus STARTED 2023-10-09T10:38:16.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:16 smithi155 ceph-mon[117223]: mgrmap e26: y(active, since 1.12501s) 2023-10-09T10:38:16.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:16 smithi155 ceph-mon[117223]: pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:16.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:16 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1142576955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:16.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:16 smithi125 ceph-mon[112589]: [09/Oct/2023:10:38:15] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T10:38:16.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:16 smithi125 ceph-mon[112589]: [09/Oct/2023:10:38:15] ENGINE Bus STARTED 2023-10-09T10:38:16.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:16 smithi125 ceph-mon[112589]: mgrmap e26: y(active, since 1.12501s) 2023-10-09T10:38:16.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:16 smithi125 ceph-mon[112589]: pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:16.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:16 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1142576955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:16.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:16 smithi125 ceph-mon[127026]: [09/Oct/2023:10:38:15] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T10:38:16.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:16 smithi125 ceph-mon[127026]: [09/Oct/2023:10:38:15] ENGINE Bus STARTED 2023-10-09T10:38:16.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:16 smithi125 ceph-mon[127026]: mgrmap e26: y(active, since 1.12501s) 2023-10-09T10:38:16.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:16 smithi125 ceph-mon[127026]: pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:16.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:16 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/1142576955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:17.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:16 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:16.736+0000 7f6b31080200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T10:38:17.487 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:17 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:17.392+0000 7f6b31080200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T10:38:17.488 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:17 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:17.487+0000 7f6b31080200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T10:38:17.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:17 smithi125 ceph-mon[112589]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:17.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:17 smithi125 ceph-mon[112589]: mgrmap e27: y(active, since 2s) 2023-10-09T10:38:17.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:17 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:17.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:17 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:17.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:17 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:17.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:17 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:17.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:17 smithi125 ceph-mon[127026]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:17.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:17 smithi125 ceph-mon[127026]: mgrmap e27: y(active, since 2s) 2023-10-09T10:38:17.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:17 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:17.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:17 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:17.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:17 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:17.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:17 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:17.828 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:17 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:17.584+0000 7f6b31080200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T10:38:17.829 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:17 smithi155 ceph-mon[117223]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:17.829 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:17 smithi155 ceph-mon[117223]: mgrmap e27: y(active, since 2s) 2023-10-09T10:38:17.829 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:17 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:17.829 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:17 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:17.829 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:17 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:17.829 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:17 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:18.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:17 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:17.828+0000 7f6b31080200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T10:38:18.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:17 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:17.935+0000 7f6b31080200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T10:38:18.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:18 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:18.036+0000 7f6b31080200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T10:38:18.548 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:18 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:18.262+0000 7f6b31080200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T10:38:18.548 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:18 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:18.361+0000 7f6b31080200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T10:38:18.944 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:18 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:18.944 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:18 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:18.944 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:18 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:18.944 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:18 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:18.944 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:18 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/2182238771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:18.944 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:18 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:18.945 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:18 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:18.945 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:18 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:18.945 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:18 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:18.945 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:18 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/2182238771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:18.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:18 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:18.547+0000 7f6b31080200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T10:38:18.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:18 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:18.635+0000 7f6b31080200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T10:38:18.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:18 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:18.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:18 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:18.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:18 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:18.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:18 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:18.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:18 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/2182238771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:18.971 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:18 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:38:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:38:19.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:19 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:19.396+0000 7f6b31080200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T10:38:19.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:19 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:19.594+0000 7f6b31080200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T10:38:20.101 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:19 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:19.777+0000 7f6b31080200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:38:20.177 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:20 smithi125 ceph-mon[112589]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:20.178 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:20 smithi125 ceph-mon[127026]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:20.446 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:20 smithi155 ceph-mon[117223]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:20.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:20 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:20.445+0000 7f6b31080200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T10:38:20.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:20 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:20.646+0000 7f6b31080200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:38:21.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:20 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:20.857+0000 7f6b31080200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T10:38:21.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:20 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:20.984+0000 7f6b31080200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T10:38:21.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: [09/Oct/2023:10:38:20] ENGINE Bus STARTING 2023-10-09T10:38:21.221 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: CherryPy Checker: 2023-10-09T10:38:21.221 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: The Application mounted at '' has an empty config. 2023-10-09T10:38:21.221 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: 2023-10-09T10:38:21.221 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: [09/Oct/2023:10:38:21] ENGINE Serving on http://:::9283 2023-10-09T10:38:21.221 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[157691]: [09/Oct/2023:10:38:21] ENGINE Bus STARTED 2023-10-09T10:38:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:38:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:38:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:38:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:38:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:38:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:38:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:38:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: Standby manager daemon x started 2023-10-09T10:38:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/1924857773' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:38:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/1924857773' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:38:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/1924857773' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:38:21.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/1924857773' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:38:21.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:21 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/282021479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:21.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:21.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:21.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:21.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:38:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:38:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:38:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:38:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:38:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:38:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:38:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: Standby manager daemon x started 2023-10-09T10:38:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/1924857773' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:38:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/1924857773' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:38:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/1924857773' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:38:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/1924857773' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:38:21.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/282021479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:21.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:21.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:21.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:21.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:38:21.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:38:21.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:38:21.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:21.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:38:21.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:38:21.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:38:21.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:38:21.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: Standby manager daemon x started 2023-10-09T10:38:21.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/1924857773' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:38:21.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/1924857773' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:38:21.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/1924857773' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:38:21.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/1924857773' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:38:21.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:21 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/282021479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:22.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:22 smithi155 ceph-mon[117223]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:38:22.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:22 smithi155 ceph-mon[117223]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:38:22.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:22 smithi155 ceph-mon[117223]: mgrmap e28: y(active, since 6s), standbys: x 2023-10-09T10:38:22.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:22 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T10:38:22.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:22 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:22.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:22 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:22.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:22 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:22.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:22 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:22.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:22 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:22.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:22 smithi155 ceph-mon[117223]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T10:38:22.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:22 smithi155 ceph-mon[117223]: Reconfiguring daemon prometheus.a on smithi155 2023-10-09T10:38:22.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[112589]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:38:22.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[112589]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:38:22.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[112589]: mgrmap e28: y(active, since 6s), standbys: x 2023-10-09T10:38:22.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T10:38:22.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:22.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:22.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:22.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:22.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:22.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[112589]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T10:38:22.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[112589]: Reconfiguring daemon prometheus.a on smithi155 2023-10-09T10:38:22.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[127026]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:38:22.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[127026]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:38:22.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[127026]: mgrmap e28: y(active, since 6s), standbys: x 2023-10-09T10:38:22.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T10:38:22.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:22.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:22.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:22.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:22.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:22.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[127026]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T10:38:22.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:22 smithi125 ceph-mon[127026]: Reconfiguring daemon prometheus.a on smithi155 2023-10-09T10:38:23.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:23 smithi155 ceph-mon[117223]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:23.471 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 systemd[1]: Stopping Ceph prometheus.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:38:23.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:23 smithi125 ceph-mon[112589]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:23.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:23 smithi125 ceph-mon[127026]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:23.935 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:38:23.587Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T10:38:23.935 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:38:23.588Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T10:38:23.935 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:38:23.588Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T10:38:23.935 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:38:23.588Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T10:38:23.935 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:38:23.588Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T10:38:23.935 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:38:23.588Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T10:38:23.935 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:38:23.588Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T10:38:23.935 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:38:23.588Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T10:38:23.935 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:38:23.588Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T10:38:23.936 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:38:23.588Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T10:38:23.936 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:38:23.588Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T10:38:23.936 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[168724]: ts=2023-10-09T10:38:23.588Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T10:38:23.936 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 bash[172292]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T10:38:24.210 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T10:38:24.210 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 systemd[1]: Stopped Ceph prometheus.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:38:24.210 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:23 smithi155 systemd[1]: Starting Ceph prometheus.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:38:24.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3259607743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:24.721 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.362Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T10:38:24.721 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.362Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T10:38:24.721 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.362Z caller=main.go:570 level=info host_details="(Linux 4.18.0-514.el8.x86_64 #1 SMP Tue Sep 19 13:19:58 UTC 2023 x86_64 smithi155 (none))" 2023-10-09T10:38:24.721 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.362Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T10:38:24.721 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.362Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T10:38:24.721 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.363Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T10:38:24.721 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.364Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T10:38:24.722 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.365Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T10:38:24.722 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.365Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T10:38:24.722 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.367Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T10:38:24.722 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.367Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.289µs 2023-10-09T10:38:24.722 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.367Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T10:38:24.722 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.367Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-10-09T10:38:24.722 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 bash[172412]: 6412c61399645da7666d796c4ab2f7ce38dea6788e30190906a964c2f03dc1a0 2023-10-09T10:38:24.722 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 systemd[1]: Started Ceph prometheus.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:38:24.722 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.381Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-10-09T10:38:24.722 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.383Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-10-09T10:38:24.722 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.384Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-10-09T10:38:24.723 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.384Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=35.638µs wal_replay_duration=16.490734ms wbl_replay_duration=122ns total_replay_duration=16.555349ms 2023-10-09T10:38:24.723 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.387Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T10:38:24.723 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.388Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T10:38:24.723 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.388Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T10:38:24.723 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.404Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.868272ms db_storage=878ns remote_storage=1.22µs web_handler=441ns query_engine=688ns scrape=502.536µs scrape_sd=77.329µs notify=14.761µs notify_sd=22.105µs rules=16.030139ms tracing=6.734µs 2023-10-09T10:38:24.723 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.404Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T10:38:24.723 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:24.404Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T10:38:24.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:24 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3259607743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:24.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:24 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3259607743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:25.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:25 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:25.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:25 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:25.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:25 smithi155 ceph-mon[117223]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:25.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:25 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:38:25.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:25 smithi155 ceph-mon[117223]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:38:25.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:25 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:25.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:25 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:25.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:25 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:25.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:25 smithi125 ceph-mon[112589]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:25.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:25 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:38:25.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:25 smithi125 ceph-mon[112589]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:38:25.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:25 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:25.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:25 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:25.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:25 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:25.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:25 smithi125 ceph-mon[127026]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:25.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:25 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:38:25.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:25 smithi125 ceph-mon[127026]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T10:38:25.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:25 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:26.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:26 smithi155 systemd[1]: Stopping Ceph mgr.x for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:38:26.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:26 smithi155 ceph-mon[117223]: Upgrade: Updating mgr.x 2023-10-09T10:38:26.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:26 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:38:26.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:26 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:38:26.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:26 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:26.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:26 smithi155 ceph-mon[117223]: Deploying daemon mgr.x on smithi155 2023-10-09T10:38:26.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:26 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/600614372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:26.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:26 smithi125 ceph-mon[112589]: Upgrade: Updating mgr.x 2023-10-09T10:38:26.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:26 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:38:26.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:26 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:38:26.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:26 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:26.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:26 smithi125 ceph-mon[112589]: Deploying daemon mgr.x on smithi155 2023-10-09T10:38:26.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:26 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/600614372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:26 smithi125 ceph-mon[127026]: Upgrade: Updating mgr.x 2023-10-09T10:38:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:26 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:38:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:26 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:38:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:26 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:26 smithi125 ceph-mon[127026]: Deploying daemon mgr.x on smithi155 2023-10-09T10:38:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:26 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/600614372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:27.086 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:26 smithi155 bash[172840]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T10:38:27.086 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:26 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T10:38:27.420 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:27 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T10:38:27.420 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:27 smithi155 systemd[1]: Stopped Ceph mgr.x for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:38:27.420 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:27 smithi155 systemd[1]: Starting Ceph mgr.x for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:38:27.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:27 smithi155 ceph-mon[117223]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:27.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:27 smithi155 ceph-mgr[173027]: -- 172.21.15.155:0/3344772102 <== mon.1 v2:172.21.15.155:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c18b6e31e0 con 0x55c18b6b9000 2023-10-09T10:38:27.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:27 smithi155 bash[172989]: 67bb8d1af523bf987f1fbdbcab8146e243820e64f55a13571971eb498e21ab33 2023-10-09T10:38:27.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:27 smithi155 systemd[1]: Started Ceph mgr.x for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:38:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:27 smithi125 ceph-mon[112589]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:27 smithi125 ceph-mon[127026]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:28.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:27 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:27.725+0000 7f06e2f0c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T10:38:28.221 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:27 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:27.906+0000 7f06e2f0c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T10:38:28.660 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:28 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:28.270+0000 7f06e2f0c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T10:38:28.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:28 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:28.827+0000 7f06e2f0c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T10:38:28.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:28 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:28.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:28 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:28.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:28 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:28.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:28 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/492557124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:29.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:28 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:29.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:28 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:29.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:28 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:29.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:28 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/492557124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:29.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:28 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:29.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:28 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:29.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:28 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:29.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:28 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/492557124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:29.422 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:29 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:29.109+0000 7f06e2f0c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T10:38:29.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:29 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:29.564+0000 7f06e2f0c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T10:38:29.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:29 smithi155 ceph-mon[117223]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:29.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:29 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:38:30.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:29 smithi125 ceph-mon[112589]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:30.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:29 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:38:30.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:29 smithi125 ceph-mon[127026]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:30.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:29 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:38:30.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:30 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:30.377+0000 7f06e2f0c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T10:38:30.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:30 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:30.471+0000 7f06e2f0c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T10:38:31.470 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:31 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:31.124+0000 7f06e2f0c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T10:38:31.470 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:31 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:31.217+0000 7f06e2f0c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T10:38:31.470 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:31 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:31.314+0000 7f06e2f0c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T10:38:31.471 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:38:31] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2023-10-09T10:38:31.774 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:31 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:31.566+0000 7f06e2f0c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T10:38:31.775 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:31 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:31.664+0000 7f06e2f0c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T10:38:31.775 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:31 smithi155 ceph-mon[117223]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:31.775 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:31 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:31.775 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:31 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:31.775 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:31 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/547184216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:31.775 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:31 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:31.775 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:31 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:31.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:31 smithi125 ceph-mon[112589]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:31.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:31 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:31.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:31 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:31.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:31 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/547184216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:31.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:31 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:31.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:31 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:31.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:31 smithi125 ceph-mon[127026]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:31.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:31 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:31.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:31 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:31.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:31 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/547184216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:31.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:31 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:31.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:31 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:32.113 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:31 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:31.773+0000 7f06e2f0c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T10:38:32.113 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:32 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:31.999+0000 7f06e2f0c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T10:38:32.409 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:32 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:32.111+0000 7f06e2f0c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T10:38:32.409 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:32 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:32.314+0000 7f06e2f0c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T10:38:32.409 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:32 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:32.407+0000 7f06e2f0c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T10:38:33.470 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:33 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:33.257+0000 7f06e2f0c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T10:38:33.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:33 smithi125 ceph-mon[112589]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:33.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:33 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/1873353049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:33.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:33 smithi125 ceph-mon[127026]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:33.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:33 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/1873353049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:33 smithi155 ceph-mon[117223]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:33 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/1873353049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:33.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:33 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:33.470+0000 7f06e2f0c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T10:38:33.971 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:33 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:33.662+0000 7f06e2f0c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:38:34.590 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:34.336+0000 7f06e2f0c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T10:38:34.590 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:34.527+0000 7f06e2f0c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:38:34.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:34.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:34.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:34.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:38:34.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:34.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:34.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:38:34.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: Upgrade: Setting container_image for all mgr 2023-10-09T10:38:34.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:34.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T10:38:34.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T10:38:34.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T10:38:34.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T10:38:34.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T10:38:34.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T10:38:34.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[112589]: Upgrade: It appears safe to stop mon.a 2023-10-09T10:38:34.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:34.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:34.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:34.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:38:34.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:34.802 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:34.802 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:38:34.802 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: Upgrade: Setting container_image for all mgr 2023-10-09T10:38:34.802 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:34.802 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T10:38:34.802 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T10:38:34.802 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T10:38:34.802 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T10:38:34.802 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T10:38:34.802 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T10:38:34.802 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:34 smithi125 ceph-mon[127026]: Upgrade: It appears safe to stop mon.a 2023-10-09T10:38:34.854 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:34.716+0000 7f06e2f0c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T10:38:34.854 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:34.837+0000 7f06e2f0c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T10:38:34.855 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:34.855 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:34.855 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:34.855 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:38:34.855 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:34.855 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:34.855 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:38:34.855 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: Upgrade: Setting container_image for all mgr 2023-10-09T10:38:34.855 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:34.855 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T10:38:34.855 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T10:38:34.856 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T10:38:34.856 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T10:38:34.856 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T10:38:34.856 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T10:38:34.856 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-mon[117223]: Upgrade: It appears safe to stop mon.a 2023-10-09T10:38:35.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: [09/Oct/2023:10:38:34] ENGINE Bus STARTING 2023-10-09T10:38:35.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: CherryPy Checker: 2023-10-09T10:38:35.221 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: The Application mounted at '' has an empty config. 2023-10-09T10:38:35.221 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:35.221 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: [09/Oct/2023:10:38:34] ENGINE Serving on http://:::9283 2023-10-09T10:38:35.221 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: [09/Oct/2023:10:38:34] ENGINE Bus STARTED 2023-10-09T10:38:36.038 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[112589]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:36.038 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[112589]: Standby manager daemon x restarted 2023-10-09T10:38:36.039 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[112589]: Standby manager daemon x started 2023-10-09T10:38:36.039 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/3245773985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:38:36.039 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/3245773985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:38:36.039 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/3245773985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:38:36.039 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[112589]: from='mgr.? 172.21.15.155:0/3245773985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:38:36.039 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[112589]: Upgrade: Updating mon.a 2023-10-09T10:38:36.039 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:36.039 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:38:36.039 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:38:36.039 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[112589]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:36.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[112589]: Deploying daemon mon.a on smithi125 2023-10-09T10:38:36.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[112589]: from='client.? 172.21.15.125:0/3840804134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:36.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[127026]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:36.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[127026]: Standby manager daemon x restarted 2023-10-09T10:38:36.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[127026]: Standby manager daemon x started 2023-10-09T10:38:36.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/3245773985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:38:36.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/3245773985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:38:36.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/3245773985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:38:36.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/3245773985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:38:36.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[127026]: Upgrade: Updating mon.a 2023-10-09T10:38:36.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:36.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:38:36.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:38:36.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:36.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[127026]: Deploying daemon mon.a on smithi125 2023-10-09T10:38:36.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:35 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3840804134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:36.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:35 smithi155 ceph-mon[117223]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:36.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:35 smithi155 ceph-mon[117223]: Standby manager daemon x restarted 2023-10-09T10:38:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:35 smithi155 ceph-mon[117223]: Standby manager daemon x started 2023-10-09T10:38:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:35 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/3245773985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:38:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:35 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/3245773985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:38:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:35 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/3245773985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:38:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:35 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/3245773985' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:38:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:35 smithi155 ceph-mon[117223]: Upgrade: Updating mon.a 2023-10-09T10:38:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:35 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' 2023-10-09T10:38:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:35 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:38:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:35 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:38:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:35 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:36.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:35 smithi155 ceph-mon[117223]: Deploying daemon mon.a on smithi125 2023-10-09T10:38:36.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:35 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3840804134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:36.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:36 smithi125 systemd[1]: Stopping Ceph mon.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:38:36.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:36 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[112566]: 2023-10-09T10:38:36.336+0000 7fe90dd93700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:38:36.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:36 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[112566]: 2023-10-09T10:38:36.336+0000 7fe90dd93700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-10-09T10:38:36.640 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:36 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:36] ENGINE Bus STOPPING 2023-10-09T10:38:36.640 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:36 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T10:38:36.640 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:36 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:36] ENGINE Bus STOPPED 2023-10-09T10:38:36.640 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:36 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:36] ENGINE Bus STARTING 2023-10-09T10:38:36.640 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:36 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:36] ENGINE Serving on http://:::9283 2023-10-09T10:38:36.640 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:36 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:36] ENGINE Bus STARTED 2023-10-09T10:38:36.904 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:36 smithi125 bash[195303]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T10:38:37.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:36 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T10:38:37.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:36 smithi125 systemd[1]: Stopped Ceph mon.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:38:37.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 systemd[1]: Starting Ceph mon.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:38:37.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T10:38:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T10:38:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: pidfile_write: ignore empty --pid-file 2023-10-09T10:38:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: load: jerasure load: lrc 2023-10-09T10:38:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: RocksDB version: 7.9.2 2023-10-09T10:38:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Git sha 0 2023-10-09T10:38:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T10:38:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: DB SUMMARY 2023-10-09T10:38:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: DB Session ID: YRC4YZFF6FVVLIBCJLC2 2023-10-09T10:38:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: CURRENT file: CURRENT 2023-10-09T10:38:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T10:38:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: MANIFEST file: MANIFEST-000015 size: 793 Bytes 2023-10-09T10:38:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000027.sst 2023-10-09T10:38:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000025.log size: 1029022 ; 2023-10-09T10:38:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.error_if_exists: 0 2023-10-09T10:38:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.create_if_missing: 0 2023-10-09T10:38:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.paranoid_checks: 1 2023-10-09T10:38:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T10:38:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T10:38:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T10:38:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.env: 0x55b89e204720 2023-10-09T10:38:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T10:38:37.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.info_log: 0x55b89fce7420 2023-10-09T10:38:37.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T10:38:37.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.statistics: (nil) 2023-10-09T10:38:37.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.use_fsync: 0 2023-10-09T10:38:37.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_log_file_size: 0 2023-10-09T10:38:37.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T10:38:37.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T10:38:37.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T10:38:37.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T10:38:37.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.allow_fallocate: 1 2023-10-09T10:38:37.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T10:38:37.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T10:38:37.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.use_direct_reads: 0 2023-10-09T10:38:37.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T10:38:37.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T10:38:37.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.db_log_dir: 2023-10-09T10:38:37.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.wal_dir: 2023-10-09T10:38:37.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T10:38:37.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T10:38:37.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T10:38:37.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T10:38:37.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T10:38:37.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T10:38:37.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T10:38:37.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T10:38:37.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.write_buffer_manager: 0x55b89ef78c80 2023-10-09T10:38:37.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T10:38:37.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T10:38:37.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T10:38:37.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T10:38:37.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T10:38:37.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T10:38:37.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T10:38:37.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T10:38:37.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.unordered_write: 0 2023-10-09T10:38:37.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T10:38:37.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T10:38:37.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T10:38:37.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T10:38:37.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.row_cache: None 2023-10-09T10:38:37.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.wal_filter: None 2023-10-09T10:38:37.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T10:38:37.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T10:38:37.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.two_write_queues: 0 2023-10-09T10:38:37.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T10:38:37.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.wal_compression: 0 2023-10-09T10:38:37.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.atomic_flush: 0 2023-10-09T10:38:37.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T10:38:37.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T10:38:37.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T10:38:37.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.log_readahead_size: 0 2023-10-09T10:38:37.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T10:38:37.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T10:38:37.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T10:38:37.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T10:38:37.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T10:38:37.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T10:38:37.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T10:38:37.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_background_jobs: 2 2023-10-09T10:38:37.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_background_compactions: -1 2023-10-09T10:38:37.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_subcompactions: 1 2023-10-09T10:38:37.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T10:38:37.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T10:38:37.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T10:38:37.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T10:38:37.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T10:38:37.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T10:38:37.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T10:38:37.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T10:38:37.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_open_files: -1 2023-10-09T10:38:37.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T10:38:37.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T10:38:37.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T10:38:37.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T10:38:37.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_background_flushes: -1 2023-10-09T10:38:37.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Compression algorithms supported: 2023-10-09T10:38:37.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T10:38:37.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: kZSTD supported: 0 2023-10-09T10:38:37.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: kXpressCompression supported: 0 2023-10-09T10:38:37.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T10:38:37.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: kZlibCompression supported: 1 2023-10-09T10:38:37.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: kSnappyCompression supported: 1 2023-10-09T10:38:37.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: kLZ4Compression supported: 1 2023-10-09T10:38:37.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: kBZip2Compression supported: 0 2023-10-09T10:38:37.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T10:38:37.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T10:38:37.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2023-10-09T10:38:37.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T10:38:37.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T10:38:37.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.merge_operator: 2023-10-09T10:38:37.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compaction_filter: None 2023-10-09T10:38:37.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T10:38:37.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T10:38:37.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T10:38:37.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T10:38:37.831 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b89fce7520) 2023-10-09T10:38:37.832 INFO:journalctl@ceph.mon.a.smithi125.stdout: cache_index_and_filter_blocks: 1 2023-10-09T10:38:37.832 INFO:journalctl@ceph.mon.a.smithi125.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T10:38:37.832 INFO:journalctl@ceph.mon.a.smithi125.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T10:38:37.832 INFO:journalctl@ceph.mon.a.smithi125.stdout: pin_top_level_index_and_filter: 1 2023-10-09T10:38:37.832 INFO:journalctl@ceph.mon.a.smithi125.stdout: index_type: 0 2023-10-09T10:38:37.832 INFO:journalctl@ceph.mon.a.smithi125.stdout: data_block_index_type: 0 2023-10-09T10:38:37.832 INFO:journalctl@ceph.mon.a.smithi125.stdout: index_shortening: 1 2023-10-09T10:38:37.832 INFO:journalctl@ceph.mon.a.smithi125.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T10:38:37.832 INFO:journalctl@ceph.mon.a.smithi125.stdout: checksum: 4 2023-10-09T10:38:37.832 INFO:journalctl@ceph.mon.a.smithi125.stdout: no_block_cache: 0 2023-10-09T10:38:37.832 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_cache: 0x55b89eff9350 2023-10-09T10:38:37.832 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_cache_name: BinnedLRUCache 2023-10-09T10:38:37.832 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_cache_options: 2023-10-09T10:38:37.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: capacity : 536870912 2023-10-09T10:38:37.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: num_shard_bits : 4 2023-10-09T10:38:37.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: strict_capacity_limit : 0 2023-10-09T10:38:37.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: high_pri_pool_ratio: 0.000 2023-10-09T10:38:37.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_cache_compressed: (nil) 2023-10-09T10:38:37.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: persistent_cache: (nil) 2023-10-09T10:38:37.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_size: 4096 2023-10-09T10:38:37.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_size_deviation: 10 2023-10-09T10:38:37.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_restart_interval: 16 2023-10-09T10:38:37.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: index_block_restart_interval: 1 2023-10-09T10:38:37.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: metadata_block_size: 4096 2023-10-09T10:38:37.833 INFO:journalctl@ceph.mon.a.smithi125.stdout: partition_filters: 0 2023-10-09T10:38:37.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: use_delta_encoding: 1 2023-10-09T10:38:37.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: filter_policy: bloomfilter 2023-10-09T10:38:37.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: whole_key_filtering: 1 2023-10-09T10:38:37.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: verify_compression: 0 2023-10-09T10:38:37.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: read_amp_bytes_per_bit: 0 2023-10-09T10:38:37.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: format_version: 5 2023-10-09T10:38:37.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: enable_index_compression: 1 2023-10-09T10:38:37.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: block_align: 0 2023-10-09T10:38:37.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: max_auto_readahead_size: 262144 2023-10-09T10:38:37.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: prepopulate_block_cache: 0 2023-10-09T10:38:37.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: initial_auto_readahead_size: 8192 2023-10-09T10:38:37.834 INFO:journalctl@ceph.mon.a.smithi125.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T10:38:37.835 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T10:38:37.835 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T10:38:37.835 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compression: NoCompression 2023-10-09T10:38:37.835 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T10:38:37.835 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T10:38:37.835 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T10:38:37.835 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.num_levels: 7 2023-10-09T10:38:37.835 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T10:38:37.835 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T10:38:37.835 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T10:38:37.835 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T10:38:37.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T10:38:37.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T10:38:37.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T10:38:37.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T10:38:37.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T10:38:37.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T10:38:37.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T10:38:37.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T10:38:37.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T10:38:37.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T10:38:37.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T10:38:37.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T10:38:37.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T10:38:37.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T10:38:37.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T10:38:37.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T10:38:37.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T10:38:37.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T10:38:37.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T10:38:37.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T10:38:37.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T10:38:37.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T10:38:37.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T10:38:37.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T10:38:37.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T10:38:37.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T10:38:37.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T10:38:37.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T10:38:37.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T10:38:37.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T10:38:37.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T10:38:37.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T10:38:37.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T10:38:37.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T10:38:37.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T10:38:37.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T10:38:37.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T10:38:37.838 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T10:38:37.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T10:38:37.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T10:38:37.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T10:38:37.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T10:38:37.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T10:38:37.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T10:38:37.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T10:38:37.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T10:38:37.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T10:38:37.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T10:38:37.839 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T10:38:37.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.table_properties_collectors: 2023-10-09T10:38:37.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.inplace_update_support: 0 2023-10-09T10:38:37.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T10:38:37.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T10:38:37.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T10:38:37.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T10:38:37.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.bloom_locality: 0 2023-10-09T10:38:37.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.max_successive_merges: 0 2023-10-09T10:38:37.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T10:38:37.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T10:38:37.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T10:38:37.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T10:38:37.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.ttl: 2592000 2023-10-09T10:38:37.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T10:38:37.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T10:38:37.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T10:38:37.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.enable_blob_files: false 2023-10-09T10:38:37.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.min_blob_size: 0 2023-10-09T10:38:37.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T10:38:37.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T10:38:37.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T10:38:37.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T10:38:37.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T10:38:37.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T10:38:37.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T10:38:37.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T10:38:37.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-10-09T10:38:37.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T10:38:37.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 29, last_sequence is 9962, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T10:38:37.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-10-09T10:38:37.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ec023398-428d-4df6-a06d-e76b997c1785 2023-10-09T10:38:37.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696847917432961, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-10-09T10:38:37.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-10-09T10:38:37.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696847917436523, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 932049, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 9956, "largest_seqno": 10445, "table_properties": {"data_size": 928863, "index_size": 1532, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 645, "raw_key_size": 5683, "raw_average_key_size": 24, "raw_value_size": 923676, "raw_average_value_size": 3947, "num_data_blocks": 69, "num_entries": 234, "num_filter_entries": 234, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696847917, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ec023398-428d-4df6-a06d-e76b997c1785", "db_session_id": "YRC4YZFF6FVVLIBCJLC2", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-10-09T10:38:37.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696847917436611, "job": 1, "event": "recovery_finished"} 2023-10-09T10:38:37.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-10-09T10:38:37.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T10:38:37.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T10:38:37.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b89f096000 2023-10-09T10:38:37.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: DB pointer 0x55b89f082000 2023-10-09T10:38:37.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T10:38:37.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T10:38:37.843 INFO:journalctl@ceph.mon.a.smithi125.stdout: ** DB Stats ** 2023-10-09T10:38:37.843 INFO:journalctl@ceph.mon.a.smithi125.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T10:38:37.843 INFO:journalctl@ceph.mon.a.smithi125.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T10:38:37.843 INFO:journalctl@ceph.mon.a.smithi125.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T10:38:37.843 INFO:journalctl@ceph.mon.a.smithi125.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T10:38:37.843 INFO:journalctl@ceph.mon.a.smithi125.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T10:38:37.843 INFO:journalctl@ceph.mon.a.smithi125.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T10:38:37.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T10:38:37.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:38:37.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: ** Compaction Stats [default] ** 2023-10-09T10:38:37.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T10:38:37.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T10:38:37.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: L0 1/0 910.20 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 432.1 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T10:38:37.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: L6 1/0 6.88 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T10:38:37.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: Sum 2/0 7.77 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 432.1 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T10:38:37.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 432.1 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T10:38:37.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:38:37.844 INFO:journalctl@ceph.mon.a.smithi125.stdout: ** Compaction Stats [default] ** 2023-10-09T10:38:37.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T10:38:37.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T10:38:37.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 432.1 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T10:38:37.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:38:37.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T10:38:37.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:38:37.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T10:38:37.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: Flush(GB): cumulative 0.001, interval 0.001 2023-10-09T10:38:37.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T10:38:37.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T10:38:37.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T10:38:37.845 INFO:journalctl@ceph.mon.a.smithi125.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T10:38:37.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: Cumulative compaction: 0.00 GB write, 142.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:38:37.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: Interval compaction: 0.00 GB write, 142.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:38:37.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T10:38:37.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: Block cache BinnedLRUCache@0x55b89eff9350#7 capacity: 512.00 MB usage: 2.30 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2023-10-09T10:38:37.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.72 KB,0.000137091%) IndexBlock(1,1.58 KB,0.000301003%) Misc(1,0.00 KB,0%) 2023-10-09T10:38:37.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:38:37.846 INFO:journalctl@ceph.mon.a.smithi125.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T10:38:37.846 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: starting mon.a rank 0 at public addrs [v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0] at bind addrs [v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:38:37.846 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 bash[195469]: 335012cc14fab6e1684300b91f2687786388580115b96db3842d7809f400e370 2023-10-09T10:38:37.846 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: mon.a@-1(???) e3 preinit fsid daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:38:37.846 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: mon.a@-1(???).mds e8 new map 2023-10-09T10:38:37.847 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: mon.a@-1(???).mds e8 print_map 2023-10-09T10:38:37.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: e8 2023-10-09T10:38:37.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T10:38:37.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T10:38:37.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: legacy client fscid: 1 2023-10-09T10:38:37.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:38:37.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: Filesystem 'foo' (1) 2023-10-09T10:38:37.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: fs_name foo 2023-10-09T10:38:37.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: epoch 7 2023-10-09T10:38:37.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T10:38:37.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: created 2023-10-09T10:35:56.814729+0000 2023-10-09T10:38:37.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: modified 2023-10-09T10:36:10.497834+0000 2023-10-09T10:38:37.847 INFO:journalctl@ceph.mon.a.smithi125.stdout: tableserver 0 2023-10-09T10:38:37.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: root 0 2023-10-09T10:38:37.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: session_timeout 60 2023-10-09T10:38:37.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: session_autoclose 300 2023-10-09T10:38:37.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: max_file_size 1099511627776 2023-10-09T10:38:37.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: required_client_features {} 2023-10-09T10:38:37.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: last_failure 0 2023-10-09T10:38:37.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: last_failure_osd_epoch 0 2023-10-09T10:38:37.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T10:38:37.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: max_mds 1 2023-10-09T10:38:37.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: in 0 2023-10-09T10:38:37.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: up {0=24575} 2023-10-09T10:38:37.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: failed 2023-10-09T10:38:37.848 INFO:journalctl@ceph.mon.a.smithi125.stdout: damaged 2023-10-09T10:38:37.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: stopped 2023-10-09T10:38:37.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: data_pools [4] 2023-10-09T10:38:37.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: metadata_pool 3 2023-10-09T10:38:37.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: inline_data disabled 2023-10-09T10:38:37.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: balancer 2023-10-09T10:38:37.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: bal_rank_mask -1 2023-10-09T10:38:37.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: standby_count_wanted 1 2023-10-09T10:38:37.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: [mds.foo.smithi125.satgiw{0:24575} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.125:6834/1945587275,v1:172.21.15.125:6835/1945587275] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T10:38:37.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:38:37.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:38:37.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: Standby daemons: 2023-10-09T10:38:37.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: 2023-10-09T10:38:37.849 INFO:journalctl@ceph.mon.a.smithi125.stdout: [mds.foo.smithi155.ydxmyq{-1:24544} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.155:6832/914042814,v1:172.21.15.155:6833/914042814] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T10:38:37.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: mon.a@-1(???).osd e75 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T10:38:37.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: mon.a@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T10:38:37.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: mon.a@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T10:38:37.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: mon.a@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T10:38:37.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: mon.a@-1(???).paxosservice(auth 1..17) refresh upgraded, format 0 -> 3 2023-10-09T10:38:37.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 systemd[1]: Started Ceph mon.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:38:37.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: expand_channel_meta expand map: {default=false} 2023-10-09T10:38:37.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: expand_channel_meta from 'false' to 'false' 2023-10-09T10:38:37.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: expand_channel_meta expanded map: {default=false} 2023-10-09T10:38:37.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: expand_channel_meta expand map: {default=info} 2023-10-09T10:38:37.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: expand_channel_meta from 'info' to 'info' 2023-10-09T10:38:37.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: expand_channel_meta expanded map: {default=info} 2023-10-09T10:38:37.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: expand_channel_meta expand map: {default=daemon} 2023-10-09T10:38:37.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T10:38:37.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T10:38:37.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: expand_channel_meta expand map: {default=debug} 2023-10-09T10:38:37.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T10:38:37.851 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:37 smithi125 ceph-mon[195507]: expand_channel_meta expanded map: {default=debug} 2023-10-09T10:38:38.406 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ignoring --setuser ceph since I am not root 2023-10-09T10:38:38.407 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ignoring --setgroup ceph since I am not root 2023-10-09T10:38:38.407 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mgr[183374]: -- 172.21.15.125:0/3949020111 <== mon.0 v2:172.21.15.125:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5591031491e0 con 0x55910311f000 2023-10-09T10:38:38.407 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:38.240+0000 7fa575400200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T10:38:38.407 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:38.405+0000 7fa575400200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T10:38:38.407 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:38:38.407 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:38:38.408 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[127026]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:38:38.408 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[127026]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:38.408 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[127026]: mon.a calling monitor election 2023-10-09T10:38:38.408 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/446569053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:38.408 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[127026]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T10:38:38.408 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[127026]: monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T10:38:38.408 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[127026]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:38:38.408 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[127026]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T10:38:38.408 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[127026]: mgrmap e29: y(active, since 23s), standbys: x 2023-10-09T10:38:38.408 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[127026]: mgrmap e30: y(active, since 23s), standbys: x 2023-10-09T10:38:38.409 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[127026]: overall HEALTH_OK 2023-10-09T10:38:38.409 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[127026]: from='mgr.15087 ' entity='' 2023-10-09T10:38:38.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[195507]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:38:38.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[195507]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:38:38.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[195507]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:38:38.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[195507]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:38.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[195507]: mon.a calling monitor election 2023-10-09T10:38:38.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/446569053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:38.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[195507]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T10:38:38.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[195507]: monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T10:38:38.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[195507]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:38:38.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[195507]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T10:38:38.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[195507]: mgrmap e29: y(active, since 23s), standbys: x 2023-10-09T10:38:38.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[195507]: mgrmap e30: y(active, since 23s), standbys: x 2023-10-09T10:38:38.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[195507]: overall HEALTH_OK 2023-10-09T10:38:38.411 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-mon[195507]: from='mgr.15087 ' entity='' 2023-10-09T10:38:38.416 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:38:38.416 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:38:38.416 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-mon[117223]: from='mgr.15087 172.21.15.125:0/3064312261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:38:38.416 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-mon[117223]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:38.416 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-mon[117223]: mon.a calling monitor election 2023-10-09T10:38:38.416 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/446569053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:38.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-mon[117223]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T10:38:38.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-mon[117223]: monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T10:38:38.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-mon[117223]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:38:38.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-mon[117223]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T10:38:38.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-mon[117223]: mgrmap e29: y(active, since 23s), standbys: x 2023-10-09T10:38:38.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-mon[117223]: mgrmap e30: y(active, since 23s), standbys: x 2023-10-09T10:38:38.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-mon[117223]: overall HEALTH_OK 2023-10-09T10:38:38.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-mon[117223]: from='mgr.15087 ' entity='' 2023-10-09T10:38:38.417 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: ignoring --setuser ceph since I am not root 2023-10-09T10:38:38.418 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: ignoring --setgroup ceph since I am not root 2023-10-09T10:38:38.418 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-mgr[173027]: -- 172.21.15.155:0/3426496968 <== mon.1 v2:172.21.15.155:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a017a411e0 con 0x55a017a16c00 2023-10-09T10:38:38.418 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:38.247+0000 7f8c992c2200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T10:38:38.418 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:38.414+0000 7f8c992c2200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T10:38:38.720 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:38:39.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:38 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:38.731+0000 7fa575400200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T10:38:39.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:38.745+0000 7f8c992c2200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T10:38:39.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:39.219+0000 7fa575400200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T10:38:39.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:39.443+0000 7fa575400200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T10:38:39.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:39 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:39.234+0000 7f8c992c2200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T10:38:39.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:39 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:39.459+0000 7f8c992c2200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T10:38:40.220 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:39 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:39.882+0000 7f8c992c2200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T10:38:40.239 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 1 2023-10-09T10:38:40.250 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:39.863+0000 7fa575400200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T10:38:40.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:40 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3946217900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:40.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:40 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3946217900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:40.592 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:40 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3946217900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:38:40.639 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T10:38:40.915 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:38:40.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:40 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:40.591+0000 7f8c992c2200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T10:38:40.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:40 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:40.680+0000 7f8c992c2200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T10:38:41.058 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:40 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:40.605+0000 7fa575400200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T10:38:41.058 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:40 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:40.705+0000 7fa575400200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T10:38:41.677 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:41.347+0000 7fa575400200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T10:38:41.678 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:41.439+0000 7fa575400200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T10:38:41.678 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:41.531+0000 7fa575400200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T10:38:41.688 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:41.286+0000 7f8c992c2200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T10:38:41.689 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:41.372+0000 7f8c992c2200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T10:38:41.689 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:41.462+0000 7f8c992c2200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T10:38:41.969 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:41.767+0000 7fa575400200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T10:38:41.969 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:41.868+0000 7fa575400200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T10:38:41.969 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:41.967+0000 7fa575400200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T10:38:41.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:41.687+0000 7f8c992c2200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T10:38:41.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:41.782+0000 7f8c992c2200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T10:38:41.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:41.876+0000 7f8c992c2200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T10:38:42.299 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:42 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:42.198+0000 7fa575400200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T10:38:42.384 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:42 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:42.096+0000 7f8c992c2200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T10:38:42.384 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:42 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:42.194+0000 7f8c992c2200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T10:38:42.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:42 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:42.298+0000 7fa575400200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T10:38:42.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:42 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:42.487+0000 7fa575400200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T10:38:42.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:42 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:42.383+0000 7f8c992c2200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T10:38:42.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:42 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:42.475+0000 7f8c992c2200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T10:38:43.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:42 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:42.576+0000 7fa575400200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T10:38:43.619 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:43.235+0000 7f8c992c2200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T10:38:43.619 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:43.435+0000 7f8c992c2200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T10:38:43.724 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:38:43 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[182292]: ts=2023-10-09T10:38:43.507Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi125.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.125:8443: connect: connection refused" 2023-10-09T10:38:43.724 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:38:43 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[182292]: ts=2023-10-09T10:38:43.508Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi155.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.155:8443: connect: connection refused" 2023-10-09T10:38:43.724 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:43 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:43.335+0000 7fa575400200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T10:38:43.724 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:43 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:43.536+0000 7fa575400200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T10:38:43.970 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:43.618+0000 7f8c992c2200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:38:44.663 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:44.284+0000 7f8c992c2200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T10:38:44.663 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:44.473+0000 7f8c992c2200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:38:44.768 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:43 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:43.722+0000 7fa575400200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:38:44.768 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:44.388+0000 7fa575400200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T10:38:44.769 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:44.578+0000 7fa575400200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T10:38:44.913 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:44.661+0000 7f8c992c2200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T10:38:44.913 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:44.780+0000 7f8c992c2200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T10:38:44.913 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: [09/Oct/2023:10:38:44] ENGINE Bus STARTING 2023-10-09T10:38:44.913 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: CherryPy Checker: 2023-10-09T10:38:44.914 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: The Application mounted at '' has an empty config. 2023-10-09T10:38:44.914 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: 2023-10-09T10:38:44.914 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: [09/Oct/2023:10:38:44] ENGINE Serving on http://:::9283 2023-10-09T10:38:44.914 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x[173004]: [09/Oct/2023:10:38:44] ENGINE Bus STARTED 2023-10-09T10:38:44.914 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-mon[117223]: Standby manager daemon x restarted 2023-10-09T10:38:44.914 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-mon[117223]: Standby manager daemon x started 2023-10-09T10:38:44.914 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/94988117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:38:44.914 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/94988117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:38:44.915 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/94988117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:38:44.915 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-mon[117223]: from='mgr.? 172.21.15.155:0/94988117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:38:45.054 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-mon[195507]: Standby manager daemon x restarted 2023-10-09T10:38:45.054 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-mon[195507]: Standby manager daemon x started 2023-10-09T10:38:45.054 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-mon[195507]: from='mgr.? 172.21.15.155:0/94988117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:38:45.054 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-mon[195507]: from='mgr.? 172.21.15.155:0/94988117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:38:45.054 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-mon[195507]: from='mgr.? 172.21.15.155:0/94988117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:38:45.055 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-mon[195507]: from='mgr.? 172.21.15.155:0/94988117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:38:45.055 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-mon[127026]: Standby manager daemon x restarted 2023-10-09T10:38:45.055 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-mon[127026]: Standby manager daemon x started 2023-10-09T10:38:45.055 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/94988117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T10:38:45.055 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/94988117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T10:38:45.056 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/94988117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T10:38:45.056 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-mon[127026]: from='mgr.? 172.21.15.155:0/94988117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T10:38:45.056 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-c[127003]: 2023-10-09T10:38:44.911+0000 7fbf9e50e700 -1 mon.c@2(peon).osd e76 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T10:38:45.056 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:44.767+0000 7fa575400200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T10:38:45.056 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:44.893+0000 7fa575400200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T10:38:45.056 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:44 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:44] ENGINE Bus STARTING 2023-10-09T10:38:45.056 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:45.052+0000 7fa4d377f700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:45.056 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:45.052+0000 7fa4d377f700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:45.056 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:45.052+0000 7fa4d377f700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:45.057 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:45.052+0000 7fa4d377f700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:45.057 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:45.052+0000 7fa4d377f700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:45.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-b[117200]: 2023-10-09T10:38:44.912+0000 7f2290d2f700 -1 mon.b@1(peon).osd e76 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T10:38:45.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:45.053+0000 7fa4d8789700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:45.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:45.053+0000 7fa4d8789700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:45.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:45.053+0000 7fa4d8789700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:45.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:45.053+0000 7fa4d8789700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:45.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:45.053+0000 7fa4d8789700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T10:38:45.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: CherryPy Checker: 2023-10-09T10:38:45.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: The Application mounted at '' has an empty config. 2023-10-09T10:38:45.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:45.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:45] ENGINE Serving on http://:::9283 2023-10-09T10:38:45.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:10:38:45] ENGINE Bus STARTED 2023-10-09T10:38:45.821 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[182292]: ts=2023-10-09T10:38:45.464Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=4 2023-10-09T10:38:45.934 INFO:teuthology.orchestra.run.smithi125.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T10:38:45.934 INFO:teuthology.orchestra.run.smithi125.stdout:alertmanager.a smithi125 *:9093,9094 running (57s) - 3m 17.4M - 0.25.0 c8568f914cd2 3c13e6c17b7e 2023-10-09T10:38:45.934 INFO:teuthology.orchestra.run.smithi125.stdout:grafana.a smithi155 *:3000 running (47s) - 3m 81.1M - 9.4.7 2c41d148cca3 0ec8893d264d 2023-10-09T10:38:45.934 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi125.satgiw smithi125 running (2m) - 2m 15.5M - 17.2.6-1303-gb2903cb0 5517396c50d4 7b5d28288eab 2023-10-09T10:38:45.934 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi155.ydxmyq smithi155 running (2m) - 2m 14.2M - 17.2.6-1303-gb2903cb0 5517396c50d4 c2c99605d324 2023-10-09T10:38:45.934 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.x smithi155 *:8443,9283,8765 running (18s) - 6m 47.9M - 18.2.0-598-gb4fa663f 3060897fdf97 67bb8d1af523 2023-10-09T10:38:45.934 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.y smithi125 *:8443,9283,8765 running (39s) - 7m 478M - 18.2.0-598-gb4fa663f 3060897fdf97 107edeccfff9 2023-10-09T10:38:45.935 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a smithi125 running (7m) - 7m 92.5M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 310e442a0e90 2023-10-09T10:38:45.935 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b smithi155 running (6m) - 6m 64.6M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 14cad29646e0 2023-10-09T10:38:45.935 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c smithi125 running (6m) - 6m 62.9M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 6524823f3114 2023-10-09T10:38:45.935 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.a smithi125 *:9100 running (55s) - 2m 8984k - 1.5.0 0da6a335fe13 bf5a342348bd 2023-10-09T10:38:45.935 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.b smithi155 *:9100 running (43s) - 2m 8082k - 1.5.0 0da6a335fe13 1320bf6a6ac5 2023-10-09T10:38:45.935 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 smithi125 running (5m) - 5m 69.8M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 346135826df7 2023-10-09T10:38:45.935 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1 smithi125 running (5m) - 5m 75.2M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 11af9c8d1b19 2023-10-09T10:38:45.935 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2 smithi125 running (5m) - 5m 146M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 dc337188d937 2023-10-09T10:38:45.935 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3 smithi125 running (5m) - 5m 63.1M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 c92a4cc944fe 2023-10-09T10:38:45.935 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 smithi155 running (4m) - 4m 74.2M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 e3a61e2037a2 2023-10-09T10:38:45.935 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5 smithi155 running (4m) - 4m 92.5M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 12d4484e25b1 2023-10-09T10:38:45.935 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6 smithi155 running (4m) - 4m 62.6M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 c7b7ee1a3182 2023-10-09T10:38:45.936 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7 smithi155 running (3m) - 3m 97.3M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 8e225788b8f4 2023-10-09T10:38:45.936 INFO:teuthology.orchestra.run.smithi125.stdout:prometheus.a smithi155 *:9095 running (21s) - 3m 35.6M - 2.43.0 a07b618ecd1d 6412c6139964 2023-10-09T10:38:46.106 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:38:45.926+0000 7fa537ed4700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-10-09T10:38:46.107 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: mgrmap e31: y(active, since 30s), standbys: x 2023-10-09T10:38:46.107 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: Active manager daemon y restarted 2023-10-09T10:38:46.107 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: Activating manager daemon y 2023-10-09T10:38:46.107 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T10:38:46.107 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: mgrmap e32: y(active, starting, since 0.00779183s), standbys: x 2023-10-09T10:38:46.107 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: failed to encode map e76 with expected crc 2023-10-09T10:38:46.107 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:38:46.107 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:38:46.107 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:38:46.107 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.ydxmyq"}]: dispatch 2023-10-09T10:38:46.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: failed to encode map e76 with expected crc 2023-10-09T10:38:46.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.satgiw"}]: dispatch 2023-10-09T10:38:46.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:38:46.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T10:38:46.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:38:46.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:38:46.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:38:46.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:38:46.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:38:46.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:38:46.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:38:46.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:38:46.109 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:38:46.109 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:38:46.109 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:38:46.109 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: Manager daemon y is now available 2023-10-09T10:38:46.109 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:46.109 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:46.109 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:38:46.109 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: mgrmap e31: y(active, since 30s), standbys: x 2023-10-09T10:38:46.109 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: Active manager daemon y restarted 2023-10-09T10:38:46.109 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: Activating manager daemon y 2023-10-09T10:38:46.110 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T10:38:46.110 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: mgrmap e32: y(active, starting, since 0.00779183s), standbys: x 2023-10-09T10:38:46.110 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: failed to encode map e76 with expected crc 2023-10-09T10:38:46.110 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:38:46.110 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:38:46.110 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:38:46.110 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.ydxmyq"}]: dispatch 2023-10-09T10:38:46.110 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: failed to encode map e76 with expected crc 2023-10-09T10:38:46.110 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.satgiw"}]: dispatch 2023-10-09T10:38:46.110 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:38:46.110 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T10:38:46.111 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:38:46.111 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:38:46.111 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:38:46.111 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:38:46.111 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:38:46.111 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:38:46.111 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:38:46.111 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:38:46.111 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:38:46.111 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:38:46.111 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:38:46.111 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: Manager daemon y is now available 2023-10-09T10:38:46.112 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:46.112 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:46.112 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:38:46.112 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:38:46.112 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: failed to encode map e76 with expected crc 2023-10-09T10:38:46.112 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: failed to encode map e76 with expected crc 2023-10-09T10:38:46.112 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T10:38:46.112 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:38:46.113 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: failed to encode map e76 with expected crc 2023-10-09T10:38:46.113 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: failed to encode map e76 with expected crc 2023-10-09T10:38:46.113 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T10:38:46.196 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: mgrmap e31: y(active, since 30s), standbys: x 2023-10-09T10:38:46.196 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: Active manager daemon y restarted 2023-10-09T10:38:46.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: Activating manager daemon y 2023-10-09T10:38:46.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T10:38:46.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: mgrmap e32: y(active, starting, since 0.00779183s), standbys: x 2023-10-09T10:38:46.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: failed to encode map e76 with expected crc 2023-10-09T10:38:46.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:38:46.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:38:46.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:38:46.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.ydxmyq"}]: dispatch 2023-10-09T10:38:46.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: failed to encode map e76 with expected crc 2023-10-09T10:38:46.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.satgiw"}]: dispatch 2023-10-09T10:38:46.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T10:38:46.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T10:38:46.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:38:46.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:38:46.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:38:46.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:38:46.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T10:38:46.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T10:38:46.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T10:38:46.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T10:38:46.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T10:38:46.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T10:38:46.199 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T10:38:46.199 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: Manager daemon y is now available 2023-10-09T10:38:46.199 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:46.199 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:46.199 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:38:46.199 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T10:38:46.199 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: failed to encode map e76 with expected crc 2023-10-09T10:38:46.199 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: failed to encode map e76 with expected crc 2023-10-09T10:38:46.199 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T10:38:46.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T10:38:46.820 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[182292]: ts=2023-10-09T10:38:46.633Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=4 2023-10-09T10:38:46.843 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:38:47.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:46 smithi155 ceph-mon[117223]: failed to encode map e76 with expected crc 2023-10-09T10:38:47.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:46 smithi155 ceph-mon[117223]: [09/Oct/2023:10:38:45] ENGINE Bus STARTING 2023-10-09T10:38:47.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:46 smithi155 ceph-mon[117223]: [09/Oct/2023:10:38:45] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T10:38:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:46 smithi155 ceph-mon[117223]: [09/Oct/2023:10:38:45] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T10:38:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:46 smithi155 ceph-mon[117223]: [09/Oct/2023:10:38:45] ENGINE Bus STARTED 2023-10-09T10:38:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:46 smithi155 ceph-mon[117223]: mgrmap e33: y(active, since 1.01333s), standbys: x 2023-10-09T10:38:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:46 smithi155 ceph-mon[117223]: from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:38:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:46 smithi155 ceph-mon[117223]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:47.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[195507]: failed to encode map e76 with expected crc 2023-10-09T10:38:47.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[195507]: [09/Oct/2023:10:38:45] ENGINE Bus STARTING 2023-10-09T10:38:47.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[195507]: [09/Oct/2023:10:38:45] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T10:38:47.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[195507]: [09/Oct/2023:10:38:45] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T10:38:47.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[195507]: [09/Oct/2023:10:38:45] ENGINE Bus STARTED 2023-10-09T10:38:47.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[195507]: mgrmap e33: y(active, since 1.01333s), standbys: x 2023-10-09T10:38:47.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[195507]: from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:38:47.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[195507]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:47.239 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[127026]: failed to encode map e76 with expected crc 2023-10-09T10:38:47.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[127026]: [09/Oct/2023:10:38:45] ENGINE Bus STARTING 2023-10-09T10:38:47.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[127026]: [09/Oct/2023:10:38:45] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T10:38:47.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[127026]: [09/Oct/2023:10:38:45] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T10:38:47.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[127026]: [09/Oct/2023:10:38:45] ENGINE Bus STARTED 2023-10-09T10:38:47.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[127026]: mgrmap e33: y(active, since 1.01333s), standbys: x 2023-10-09T10:38:47.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[127026]: from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:38:47.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:46 smithi125 ceph-mon[127026]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:47.926 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:47 smithi125 ceph-mon[127026]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:47.926 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:47 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:47.926 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:47 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:47.926 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:47 smithi125 ceph-mon[127026]: mgrmap e34: y(active, since 2s), standbys: x 2023-10-09T10:38:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:47 smithi155 ceph-mon[117223]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:47 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:47 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:47 smithi155 ceph-mon[117223]: mgrmap e34: y(active, since 2s), standbys: x 2023-10-09T10:38:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:47 smithi125 ceph-mon[195507]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:47 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:47 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:47 smithi125 ceph-mon[195507]: mgrmap e34: y(active, since 2s), standbys: x 2023-10-09T10:38:48.753 INFO:teuthology.orchestra.run.smithi125.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:38:49.139 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T10:38:49.405 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:38:49.720 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.720 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.720 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.720 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.720 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[195507]: from='client.25169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:38:49.720 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[195507]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:38:49.720 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.720 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[195507]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:49.721 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.721 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.721 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.721 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.721 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.721 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.721 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[127026]: from='client.25169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:38:49.721 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[127026]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:38:49.721 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.721 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[127026]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:49.722 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.722 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:49 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:49 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:49 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:49 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:49 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:49 smithi155 ceph-mon[117223]: from='client.25169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:38:49.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:49 smithi155 ceph-mon[117223]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:38:49.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:49 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:49 smithi155 ceph-mon[117223]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:49.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:49 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:49.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:49 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:51.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:38:51] "GET /metrics HTTP/1.1" 200 31237 "" "Prometheus/2.43.0" 2023-10-09T10:38:51.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:51 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:51.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:51 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:51.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:51 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:51.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:51 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:38:51.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:51 smithi155 ceph-mon[117223]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:38:51.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:51 smithi155 ceph-mon[117223]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:38:51.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:51 smithi155 ceph-mon[117223]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:38:51.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:51 smithi155 ceph-mon[117223]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:38:51.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:51 smithi155 ceph-mon[117223]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:52.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:52.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:52.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:52.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:38:52.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[195507]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:38:52.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[195507]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:38:52.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[195507]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:38:52.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[195507]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:38:52.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[195507]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:52.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:52.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:52.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:52.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:38:52.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[127026]: Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T10:38:52.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[127026]: Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T10:38:52.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[127026]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:38:52.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[127026]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T10:38:52.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:51 smithi125 ceph-mon[127026]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:52.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:52 smithi155 ceph-mon[117223]: Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:38:52.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:52 smithi155 ceph-mon[117223]: Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:38:52.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:52 smithi155 ceph-mon[117223]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:38:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:52 smithi155 ceph-mon[117223]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:38:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:52 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:52 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:52 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:52 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:52 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:52 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:52 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:52 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:38:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:52 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:53.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[195507]: Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:38:53.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[195507]: Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:38:53.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[195507]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:38:53.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[195507]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:38:53.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:53.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:53.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:53.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:53.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:53.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:53.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:53.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:38:53.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:53.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[127026]: Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:38:53.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[127026]: Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T10:38:53.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[127026]: Updating smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:38:53.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[127026]: Updating smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T10:38:53.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:53.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:53.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:53.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:53.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:53.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:38:53.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:38:53.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:38:53.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:52 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:38:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:53 smithi155 ceph-mon[117223]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:54.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:53 smithi125 ceph-mon[195507]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:54.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:53 smithi125 ceph-mon[127026]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:56.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:55 smithi125 ceph-mon[195507]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:56.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:55 smithi125 ceph-mon[127026]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:56.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:55 smithi155 ceph-mon[117223]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:58.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:57 smithi125 ceph-mon[195507]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:58.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:57 smithi125 ceph-mon[127026]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:58.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:57 smithi155 ceph-mon[117223]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:38:58.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:38:58 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:38:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:39:00.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:38:59 smithi125 ceph-mon[195507]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:00.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:38:59 smithi125 ceph-mon[127026]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:38:59 smithi155 ceph-mon[117223]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:01.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:39:01.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:00 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:39:01.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:39:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:39:01] "GET /metrics HTTP/1.1" 200 33872 "" "Prometheus/2.43.0" 2023-10-09T10:39:01.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:00 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:39:02.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:01 smithi125 ceph-mon[195507]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:02.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:01 smithi125 ceph-mon[127026]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:01 smithi155 ceph-mon[117223]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:03 smithi155 ceph-mon[117223]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:03.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:03 smithi125 ceph-mon[195507]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:03 smithi125 ceph-mon[127026]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:06.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:05 smithi125 ceph-mon[195507]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:06.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:05 smithi125 ceph-mon[127026]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:05 smithi155 ceph-mon[117223]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:08.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:07 smithi125 ceph-mon[195507]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:08.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:07 smithi125 ceph-mon[127026]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:07 smithi155 ceph-mon[117223]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:08.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:39:08 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:39:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:39:10.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:09 smithi125 ceph-mon[195507]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:10.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:09 smithi125 ceph-mon[127026]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:10.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:09 smithi155 ceph-mon[117223]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:11.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:39:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:39:11] "GET /metrics HTTP/1.1" 200 33872 "" "Prometheus/2.43.0" 2023-10-09T10:39:12.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:11 smithi125 ceph-mon[195507]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:12.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:11 smithi125 ceph-mon[127026]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:12.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:11 smithi155 ceph-mon[117223]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:13.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:13 smithi155 ceph-mon[117223]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:13.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:13 smithi125 ceph-mon[195507]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:13.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:13 smithi125 ceph-mon[127026]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:16.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:15 smithi125 ceph-mon[195507]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:16.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:39:16.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:15 smithi125 ceph-mon[127026]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:16.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:15 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:39:16.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:15 smithi155 ceph-mon[117223]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:16.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:15 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:39:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:17 smithi125 ceph-mon[195507]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:18.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:17 smithi125 ceph-mon[127026]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:17 smithi155 ceph-mon[117223]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:18.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:39:18 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:39:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:39:20.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:19 smithi125 ceph-mon[195507]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:20.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:19 smithi125 ceph-mon[127026]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:19 smithi155 ceph-mon[117223]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:21.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:39:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:39:21] "GET /metrics HTTP/1.1" 200 33865 "" "Prometheus/2.43.0" 2023-10-09T10:39:22.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:21 smithi125 ceph-mon[195507]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:22.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:21 smithi125 ceph-mon[127026]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:22.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:21 smithi155 ceph-mon[117223]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:23.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:23 smithi155 ceph-mon[117223]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:23.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:23 smithi125 ceph-mon[127026]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:23 smithi125 ceph-mon[195507]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:26.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:25 smithi125 ceph-mon[195507]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:26.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:25 smithi125 ceph-mon[127026]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:25 smithi155 ceph-mon[117223]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:28.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:27 smithi125 ceph-mon[195507]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:28.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:27 smithi125 ceph-mon[127026]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:28.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:27 smithi155 ceph-mon[117223]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:28.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:39:28 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:39:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:39:30.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:29 smithi125 ceph-mon[195507]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:30.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:39:30.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:29 smithi125 ceph-mon[127026]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:30.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:29 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:39:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:29 smithi155 ceph-mon[117223]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:29 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:39:31.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:39:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:39:31] "GET /metrics HTTP/1.1" 200 33867 "" "Prometheus/2.43.0" 2023-10-09T10:39:32.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:31 smithi125 ceph-mon[195507]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:32.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:31 smithi125 ceph-mon[127026]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:31 smithi155 ceph-mon[117223]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:33 smithi155 ceph-mon[117223]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:33.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:33 smithi125 ceph-mon[195507]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:33.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:33 smithi125 ceph-mon[127026]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:36.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:35 smithi125 ceph-mon[195507]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:36.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:35 smithi125 ceph-mon[127026]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:36.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:35 smithi155 ceph-mon[117223]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:37 smithi125 ceph-mon[195507]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:37 smithi125 ceph-mon[127026]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:37 smithi155 ceph-mon[117223]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:38.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:39:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:39:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:39:40.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:39 smithi125 ceph-mon[195507]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:40.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:39 smithi125 ceph-mon[127026]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:39 smithi155 ceph-mon[117223]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:41.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:39:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:39:41] "GET /metrics HTTP/1.1" 200 33867 "" "Prometheus/2.43.0" 2023-10-09T10:39:42.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:41 smithi125 ceph-mon[195507]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:42.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:41 smithi125 ceph-mon[127026]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:41 smithi155 ceph-mon[117223]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:43.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:43 smithi125 ceph-mon[195507]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:43.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:43 smithi125 ceph-mon[127026]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:43 smithi155 ceph-mon[117223]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:45 smithi125 ceph-mon[195507]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:39:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:45 smithi125 ceph-mon[127026]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:45 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:39:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:45 smithi155 ceph-mon[117223]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:45 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:39:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:47 smithi125 ceph-mon[195507]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:47 smithi125 ceph-mon[127026]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:47 smithi155 ceph-mon[117223]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:48.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:39:48 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:39:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:39:50.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:49 smithi125 ceph-mon[195507]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:50.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:49 smithi125 ceph-mon[127026]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:49 smithi155 ceph-mon[117223]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:50.754 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T10:39:51.024 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:39:51.536 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:39:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:39:51] "GET /metrics HTTP/1.1" 200 33867 "" "Prometheus/2.43.0" 2023-10-09T10:39:52.318 INFO:teuthology.orchestra.run.smithi125.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:39:52.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:51 smithi125 ceph-mon[195507]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:52.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:51 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:39:52.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:51 smithi125 ceph-mon[127026]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:52.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:51 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:39:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:51 smithi155 ceph-mon[117223]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:51 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:39:53.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for majority of mons to upgrade' 2023-10-09T10:39:53.531 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:39:53.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:53 smithi125 ceph-mon[127026]: from='client.25175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:39:53.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:53 smithi125 ceph-mon[127026]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:39:53.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:53 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:53.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:53 smithi125 ceph-mon[127026]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:53.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:53 smithi125 ceph-mon[195507]: from='client.25175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:39:53.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:53 smithi125 ceph-mon[195507]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:39:53.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:53 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:53.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:53 smithi125 ceph-mon[195507]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:53 smithi155 ceph-mon[117223]: from='client.25175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:39:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:53 smithi155 ceph-mon[117223]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:39:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:53 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:53 smithi155 ceph-mon[117223]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:54.826 INFO:teuthology.orchestra.run.smithi125.stdout:wait for majority of mons to upgrade 2023-10-09T10:39:55.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-10-09T10:39:55.383 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:55 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:55.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:55 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:55.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:55 smithi125 ceph-mon[195507]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:55.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:55 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:55.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:55 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:55.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:55 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:55.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:55 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:55.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:55 smithi125 ceph-mon[127026]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:55.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:55 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:55.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:55 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:55.634 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:39:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:55 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:55 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:55 smithi155 ceph-mon[117223]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:55.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:55 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:55.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:55 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:56.599 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:39:56.599 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:39:56.599 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:56.600 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:39:56.600 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:39:56.600 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T10:39:56.600 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T10:39:56.600 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[195507]: Upgrade: It appears safe to stop mon.c 2023-10-09T10:39:56.600 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:39:56.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:39:56.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:56.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:39:56.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:39:56.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T10:39:56.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T10:39:56.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:56 smithi125 ceph-mon[127026]: Upgrade: It appears safe to stop mon.c 2023-10-09T10:39:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:56 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:39:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:56 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:39:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:56 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:56 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:39:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:56 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:39:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:56 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T10:39:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:56 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T10:39:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:56 smithi155 ceph-mon[117223]: Upgrade: It appears safe to stop mon.c 2023-10-09T10:39:57.619 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:57 smithi125 ceph-mon[195507]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:57.620 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:57 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3188119817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:39:57.620 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:57 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:57.620 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:57 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:39:57.620 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:57 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:39:57.620 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:57 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:39:57.620 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:57 smithi125 ceph-mon[127026]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:57.620 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:57 smithi125 ceph-mon[127026]: from='client.? 172.21.15.125:0/3188119817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:39:57.620 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:57 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:57.620 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:57 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:39:57.621 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:57 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:39:57.621 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:57 smithi125 ceph-mon[127026]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:39:57.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:57 smithi155 ceph-mon[117223]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:39:57.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:57 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3188119817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:39:57.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:57 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:39:57.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:57 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:39:57.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:57 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:39:57.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:57 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:39:58.465 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:58 smithi125 systemd[1]: Stopping Ceph mon.c for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:39:58.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:58 smithi155 ceph-mon[117223]: Upgrade: Updating mon.c 2023-10-09T10:39:58.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:39:58 smithi155 ceph-mon[117223]: Deploying daemon mon.c on smithi125 2023-10-09T10:39:58.721 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:39:58 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:39:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:39:58.761 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:58 smithi125 ceph-mon[195507]: Upgrade: Updating mon.c 2023-10-09T10:39:58.761 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:39:58 smithi125 ceph-mon[195507]: Deploying daemon mon.c on smithi125 2023-10-09T10:39:58.761 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:58 smithi125 ceph-mon[127026]: Upgrade: Updating mon.c 2023-10-09T10:39:58.761 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:58 smithi125 ceph-mon[127026]: Deploying daemon mon.c on smithi125 2023-10-09T10:39:58.761 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:58 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-c[127003]: 2023-10-09T10:39:58.501+0000 7fbfa3d19700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:39:58.762 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:58 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-c[127003]: 2023-10-09T10:39:58.501+0000 7fbfa3d19700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-10-09T10:39:59.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:58 smithi125 bash[199984]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T10:39:59.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-09T10:39:59.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 systemd[1]: Stopped Ceph mon.c for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:39:59.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 systemd[1]: Starting Ceph mon.c for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:40:00.023 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T10:40:00.023 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T10:40:00.023 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: pidfile_write: ignore empty --pid-file 2023-10-09T10:40:00.023 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 bash[200156]: c47abfeb6d97635ca9629aadc6b07e351f17c8c3e543926b9465fd8e8a405fee 2023-10-09T10:40:00.023 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: load: jerasure load: lrc 2023-10-09T10:40:00.023 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: RocksDB version: 7.9.2 2023-10-09T10:40:00.023 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Git sha 0 2023-10-09T10:40:00.023 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T10:40:00.023 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: DB SUMMARY 2023-10-09T10:40:00.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: DB Session ID: 2RXD59ZZ0967IZLWA0C0 2023-10-09T10:40:00.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: CURRENT file: CURRENT 2023-10-09T10:40:00.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T10:40:00.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: MANIFEST file: MANIFEST-000009 size: 718 Bytes 2023-10-09T10:40:00.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000021.sst 2023-10-09T10:40:00.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000019.log size: 4803286 ; 2023-10-09T10:40:00.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.error_if_exists: 0 2023-10-09T10:40:00.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.create_if_missing: 0 2023-10-09T10:40:00.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.paranoid_checks: 1 2023-10-09T10:40:00.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T10:40:00.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T10:40:00.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T10:40:00.025 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.env: 0x55764873c720 2023-10-09T10:40:00.025 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T10:40:00.025 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.info_log: 0x55764b831420 2023-10-09T10:40:00.025 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T10:40:00.025 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.statistics: (nil) 2023-10-09T10:40:00.025 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.use_fsync: 0 2023-10-09T10:40:00.025 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_log_file_size: 0 2023-10-09T10:40:00.025 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T10:40:00.025 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T10:40:00.025 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T10:40:00.025 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T10:40:00.025 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.allow_fallocate: 1 2023-10-09T10:40:00.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T10:40:00.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T10:40:00.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.use_direct_reads: 0 2023-10-09T10:40:00.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T10:40:00.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T10:40:00.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.db_log_dir: 2023-10-09T10:40:00.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.wal_dir: 2023-10-09T10:40:00.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T10:40:00.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T10:40:00.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T10:40:00.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T10:40:00.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T10:40:00.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T10:40:00.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T10:40:00.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T10:40:00.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.write_buffer_manager: 0x55764aac2c80 2023-10-09T10:40:00.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T10:40:00.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T10:40:00.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T10:40:00.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T10:40:00.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T10:40:00.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T10:40:00.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T10:40:00.028 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T10:40:00.028 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.unordered_write: 0 2023-10-09T10:40:00.028 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T10:40:00.028 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T10:40:00.028 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T10:40:00.028 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T10:40:00.028 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.row_cache: None 2023-10-09T10:40:00.028 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.wal_filter: None 2023-10-09T10:40:00.028 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T10:40:00.028 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T10:40:00.028 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.two_write_queues: 0 2023-10-09T10:40:00.028 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T10:40:00.029 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.wal_compression: 0 2023-10-09T10:40:00.029 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.atomic_flush: 0 2023-10-09T10:40:00.029 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T10:40:00.029 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T10:40:00.029 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T10:40:00.029 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.log_readahead_size: 0 2023-10-09T10:40:00.029 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T10:40:00.029 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T10:40:00.029 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T10:40:00.029 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T10:40:00.029 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T10:40:00.030 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T10:40:00.031 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T10:40:00.031 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_background_jobs: 2 2023-10-09T10:40:00.031 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_background_compactions: -1 2023-10-09T10:40:00.031 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_subcompactions: 1 2023-10-09T10:40:00.031 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T10:40:00.031 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T10:40:00.031 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T10:40:00.031 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T10:40:00.031 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T10:40:00.031 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T10:40:00.031 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T10:40:00.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T10:40:00.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_open_files: -1 2023-10-09T10:40:00.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T10:40:00.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T10:40:00.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T10:40:00.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T10:40:00.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_background_flushes: -1 2023-10-09T10:40:00.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Compression algorithms supported: 2023-10-09T10:40:00.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T10:40:00.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: kZSTD supported: 0 2023-10-09T10:40:00.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: kXpressCompression supported: 0 2023-10-09T10:40:00.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T10:40:00.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: kZlibCompression supported: 1 2023-10-09T10:40:00.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: kSnappyCompression supported: 1 2023-10-09T10:40:00.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: kLZ4Compression supported: 1 2023-10-09T10:40:00.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: kBZip2Compression supported: 0 2023-10-09T10:40:00.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T10:40:00.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T10:40:00.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2023-10-09T10:40:00.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T10:40:00.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T10:40:00.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.merge_operator: 2023-10-09T10:40:00.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compaction_filter: None 2023-10-09T10:40:00.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T10:40:00.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T10:40:00.034 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T10:40:00.034 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T10:40:00.034 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55764b831520) 2023-10-09T10:40:00.034 INFO:journalctl@ceph.mon.c.smithi125.stdout: cache_index_and_filter_blocks: 1 2023-10-09T10:40:00.034 INFO:journalctl@ceph.mon.c.smithi125.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T10:40:00.034 INFO:journalctl@ceph.mon.c.smithi125.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T10:40:00.034 INFO:journalctl@ceph.mon.c.smithi125.stdout: pin_top_level_index_and_filter: 1 2023-10-09T10:40:00.034 INFO:journalctl@ceph.mon.c.smithi125.stdout: index_type: 0 2023-10-09T10:40:00.034 INFO:journalctl@ceph.mon.c.smithi125.stdout: data_block_index_type: 0 2023-10-09T10:40:00.034 INFO:journalctl@ceph.mon.c.smithi125.stdout: index_shortening: 1 2023-10-09T10:40:00.034 INFO:journalctl@ceph.mon.c.smithi125.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T10:40:00.034 INFO:journalctl@ceph.mon.c.smithi125.stdout: checksum: 4 2023-10-09T10:40:00.035 INFO:journalctl@ceph.mon.c.smithi125.stdout: no_block_cache: 0 2023-10-09T10:40:00.035 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_cache: 0x55764ab43350 2023-10-09T10:40:00.035 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_cache_name: BinnedLRUCache 2023-10-09T10:40:00.035 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_cache_options: 2023-10-09T10:40:00.035 INFO:journalctl@ceph.mon.c.smithi125.stdout: capacity : 536870912 2023-10-09T10:40:00.035 INFO:journalctl@ceph.mon.c.smithi125.stdout: num_shard_bits : 4 2023-10-09T10:40:00.035 INFO:journalctl@ceph.mon.c.smithi125.stdout: strict_capacity_limit : 0 2023-10-09T10:40:00.035 INFO:journalctl@ceph.mon.c.smithi125.stdout: high_pri_pool_ratio: 0.000 2023-10-09T10:40:00.035 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_cache_compressed: (nil) 2023-10-09T10:40:00.035 INFO:journalctl@ceph.mon.c.smithi125.stdout: persistent_cache: (nil) 2023-10-09T10:40:00.035 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_size: 4096 2023-10-09T10:40:00.035 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_size_deviation: 10 2023-10-09T10:40:00.035 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_restart_interval: 16 2023-10-09T10:40:00.036 INFO:journalctl@ceph.mon.c.smithi125.stdout: index_block_restart_interval: 1 2023-10-09T10:40:00.036 INFO:journalctl@ceph.mon.c.smithi125.stdout: metadata_block_size: 4096 2023-10-09T10:40:00.036 INFO:journalctl@ceph.mon.c.smithi125.stdout: partition_filters: 0 2023-10-09T10:40:00.036 INFO:journalctl@ceph.mon.c.smithi125.stdout: use_delta_encoding: 1 2023-10-09T10:40:00.036 INFO:journalctl@ceph.mon.c.smithi125.stdout: filter_policy: bloomfilter 2023-10-09T10:40:00.036 INFO:journalctl@ceph.mon.c.smithi125.stdout: whole_key_filtering: 1 2023-10-09T10:40:00.036 INFO:journalctl@ceph.mon.c.smithi125.stdout: verify_compression: 0 2023-10-09T10:40:00.036 INFO:journalctl@ceph.mon.c.smithi125.stdout: read_amp_bytes_per_bit: 0 2023-10-09T10:40:00.036 INFO:journalctl@ceph.mon.c.smithi125.stdout: format_version: 5 2023-10-09T10:40:00.036 INFO:journalctl@ceph.mon.c.smithi125.stdout: enable_index_compression: 1 2023-10-09T10:40:00.036 INFO:journalctl@ceph.mon.c.smithi125.stdout: block_align: 0 2023-10-09T10:40:00.036 INFO:journalctl@ceph.mon.c.smithi125.stdout: max_auto_readahead_size: 262144 2023-10-09T10:40:00.036 INFO:journalctl@ceph.mon.c.smithi125.stdout: prepopulate_block_cache: 0 2023-10-09T10:40:00.037 INFO:journalctl@ceph.mon.c.smithi125.stdout: initial_auto_readahead_size: 8192 2023-10-09T10:40:00.037 INFO:journalctl@ceph.mon.c.smithi125.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T10:40:00.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T10:40:00.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T10:40:00.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compression: NoCompression 2023-10-09T10:40:00.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T10:40:00.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T10:40:00.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T10:40:00.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.num_levels: 7 2023-10-09T10:40:00.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T10:40:00.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T10:40:00.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T10:40:00.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T10:40:00.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T10:40:00.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T10:40:00.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T10:40:00.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T10:40:00.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T10:40:00.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T10:40:00.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T10:40:00.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T10:40:00.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T10:40:00.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T10:40:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T10:40:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T10:40:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T10:40:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T10:40:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T10:40:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T10:40:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T10:40:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T10:40:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T10:40:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T10:40:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T10:40:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T10:40:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T10:40:00.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T10:40:00.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T10:40:00.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T10:40:00.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T10:40:00.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T10:40:00.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T10:40:00.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T10:40:00.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T10:40:00.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T10:40:00.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T10:40:00.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T10:40:00.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T10:40:00.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T10:40:00.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T10:40:00.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T10:40:00.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T10:40:00.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T10:40:00.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T10:40:00.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T10:40:00.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T10:40:00.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T10:40:00.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T10:40:00.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T10:40:00.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T10:40:00.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T10:40:00.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T10:40:00.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.table_properties_collectors: 2023-10-09T10:40:00.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.inplace_update_support: 0 2023-10-09T10:40:00.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T10:40:00.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T10:40:00.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T10:40:00.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T10:40:00.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.bloom_locality: 0 2023-10-09T10:40:00.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.max_successive_merges: 0 2023-10-09T10:40:00.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T10:40:00.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T10:40:00.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T10:40:00.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T10:40:00.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.ttl: 2592000 2023-10-09T10:40:00.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T10:40:00.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T10:40:00.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T10:40:00.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.enable_blob_files: false 2023-10-09T10:40:00.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.min_blob_size: 0 2023-10-09T10:40:00.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T10:40:00.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T10:40:00.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T10:40:00.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T10:40:00.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 systemd[1]: Started Ceph mon.c for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:40:00.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T10:40:00.044 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T10:40:00.044 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T10:40:00.044 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T10:40:00.044 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 21.sst 2023-10-09T10:40:00.044 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T10:40:00.044 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 23, last_sequence is 10342, log_number is 19,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T10:40:00.044 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 19 2023-10-09T10:40:00.044 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 5fb9e08d-f58d-4512-bfae-4b21db07b70b 2023-10-09T10:40:00.044 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696847999594831, "job": 1, "event": "recovery_started", "wal_files": [19]} 2023-10-09T10:40:00.044 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #19 mode 2 2023-10-09T10:40:00.044 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696847999605896, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 24, "file_size": 2977618, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 10339, "largest_seqno": 12102, "table_properties": {"data_size": 2970347, "index_size": 4588, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1669, "raw_key_size": 14943, "raw_average_key_size": 22, "raw_value_size": 2956123, "raw_average_value_size": 4499, "num_data_blocks": 210, "num_entries": 657, "num_filter_entries": 657, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696847999, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "5fb9e08d-f58d-4512-bfae-4b21db07b70b", "db_session_id": "2RXD59ZZ0967IZLWA0C0", "orig_file_number": 24, "seqno_to_time_mapping": "N/A"}} 2023-10-09T10:40:00.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696847999605985, "job": 1, "event": "recovery_finished"} 2023-10-09T10:40:00.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: [db/version_set.cc:5047] Creating manifest 26 2023-10-09T10:40:00.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T10:40:00.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000019.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T10:40:00.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55764abe0000 2023-10-09T10:40:00.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: DB pointer 0x55764abcc000 2023-10-09T10:40:00.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T10:40:00.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T10:40:00.045 INFO:journalctl@ceph.mon.c.smithi125.stdout: ** DB Stats ** 2023-10-09T10:40:00.045 INFO:journalctl@ceph.mon.c.smithi125.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T10:40:00.045 INFO:journalctl@ceph.mon.c.smithi125.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T10:40:00.045 INFO:journalctl@ceph.mon.c.smithi125.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T10:40:00.045 INFO:journalctl@ceph.mon.c.smithi125.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T10:40:00.046 INFO:journalctl@ceph.mon.c.smithi125.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T10:40:00.046 INFO:journalctl@ceph.mon.c.smithi125.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T10:40:00.046 INFO:journalctl@ceph.mon.c.smithi125.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T10:40:00.046 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:40:00.046 INFO:journalctl@ceph.mon.c.smithi125.stdout: ** Compaction Stats [default] ** 2023-10-09T10:40:00.046 INFO:journalctl@ceph.mon.c.smithi125.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T10:40:00.046 INFO:journalctl@ceph.mon.c.smithi125.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T10:40:00.046 INFO:journalctl@ceph.mon.c.smithi125.stdout: L0 1/0 2.84 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 653.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T10:40:00.046 INFO:journalctl@ceph.mon.c.smithi125.stdout: L6 1/0 6.88 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T10:40:00.046 INFO:journalctl@ceph.mon.c.smithi125.stdout: Sum 2/0 9.72 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 653.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T10:40:00.046 INFO:journalctl@ceph.mon.c.smithi125.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 653.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T10:40:00.047 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:40:00.047 INFO:journalctl@ceph.mon.c.smithi125.stdout: ** Compaction Stats [default] ** 2023-10-09T10:40:00.047 INFO:journalctl@ceph.mon.c.smithi125.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T10:40:00.047 INFO:journalctl@ceph.mon.c.smithi125.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T10:40:00.047 INFO:journalctl@ceph.mon.c.smithi125.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 653.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T10:40:00.047 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:40:00.047 INFO:journalctl@ceph.mon.c.smithi125.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T10:40:00.047 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:40:00.047 INFO:journalctl@ceph.mon.c.smithi125.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T10:40:00.047 INFO:journalctl@ceph.mon.c.smithi125.stdout: Flush(GB): cumulative 0.003, interval 0.003 2023-10-09T10:40:00.047 INFO:journalctl@ceph.mon.c.smithi125.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T10:40:00.047 INFO:journalctl@ceph.mon.c.smithi125.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T10:40:00.048 INFO:journalctl@ceph.mon.c.smithi125.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T10:40:00.048 INFO:journalctl@ceph.mon.c.smithi125.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T10:40:00.048 INFO:journalctl@ceph.mon.c.smithi125.stdout: Cumulative compaction: 0.00 GB write, 207.85 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:40:00.048 INFO:journalctl@ceph.mon.c.smithi125.stdout: Interval compaction: 0.00 GB write, 207.85 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:40:00.048 INFO:journalctl@ceph.mon.c.smithi125.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T10:40:00.048 INFO:journalctl@ceph.mon.c.smithi125.stdout: Block cache BinnedLRUCache@0x55764ab43350#7 capacity: 512.00 MB usage: 6.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2023-10-09T10:40:00.048 INFO:journalctl@ceph.mon.c.smithi125.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2023-10-09T10:40:00.048 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:40:00.048 INFO:journalctl@ceph.mon.c.smithi125.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T10:40:00.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: starting mon.c rank 2 at public addrs [v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0] at bind addrs [v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:40:00.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: mon.c@-1(???) e3 preinit fsid daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:40:00.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: mon.c@-1(???).mds e8 new map 2023-10-09T10:40:00.049 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: mon.c@-1(???).mds e8 print_map 2023-10-09T10:40:00.049 INFO:journalctl@ceph.mon.c.smithi125.stdout: e8 2023-10-09T10:40:00.049 INFO:journalctl@ceph.mon.c.smithi125.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T10:40:00.049 INFO:journalctl@ceph.mon.c.smithi125.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T10:40:00.049 INFO:journalctl@ceph.mon.c.smithi125.stdout: legacy client fscid: 1 2023-10-09T10:40:00.049 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:40:00.049 INFO:journalctl@ceph.mon.c.smithi125.stdout: Filesystem 'foo' (1) 2023-10-09T10:40:00.049 INFO:journalctl@ceph.mon.c.smithi125.stdout: fs_name foo 2023-10-09T10:40:00.049 INFO:journalctl@ceph.mon.c.smithi125.stdout: epoch 7 2023-10-09T10:40:00.049 INFO:journalctl@ceph.mon.c.smithi125.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T10:40:00.049 INFO:journalctl@ceph.mon.c.smithi125.stdout: created 2023-10-09T10:35:56.814729+0000 2023-10-09T10:40:00.049 INFO:journalctl@ceph.mon.c.smithi125.stdout: modified 2023-10-09T10:36:10.497834+0000 2023-10-09T10:40:00.050 INFO:journalctl@ceph.mon.c.smithi125.stdout: tableserver 0 2023-10-09T10:40:00.050 INFO:journalctl@ceph.mon.c.smithi125.stdout: root 0 2023-10-09T10:40:00.050 INFO:journalctl@ceph.mon.c.smithi125.stdout: session_timeout 60 2023-10-09T10:40:00.050 INFO:journalctl@ceph.mon.c.smithi125.stdout: session_autoclose 300 2023-10-09T10:40:00.050 INFO:journalctl@ceph.mon.c.smithi125.stdout: max_file_size 1099511627776 2023-10-09T10:40:00.050 INFO:journalctl@ceph.mon.c.smithi125.stdout: required_client_features {} 2023-10-09T10:40:00.050 INFO:journalctl@ceph.mon.c.smithi125.stdout: last_failure 0 2023-10-09T10:40:00.050 INFO:journalctl@ceph.mon.c.smithi125.stdout: last_failure_osd_epoch 0 2023-10-09T10:40:00.050 INFO:journalctl@ceph.mon.c.smithi125.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T10:40:00.050 INFO:journalctl@ceph.mon.c.smithi125.stdout: max_mds 1 2023-10-09T10:40:00.050 INFO:journalctl@ceph.mon.c.smithi125.stdout: in 0 2023-10-09T10:40:00.050 INFO:journalctl@ceph.mon.c.smithi125.stdout: up {0=24575} 2023-10-09T10:40:00.051 INFO:journalctl@ceph.mon.c.smithi125.stdout: failed 2023-10-09T10:40:00.051 INFO:journalctl@ceph.mon.c.smithi125.stdout: damaged 2023-10-09T10:40:00.051 INFO:journalctl@ceph.mon.c.smithi125.stdout: stopped 2023-10-09T10:40:00.051 INFO:journalctl@ceph.mon.c.smithi125.stdout: data_pools [4] 2023-10-09T10:40:00.051 INFO:journalctl@ceph.mon.c.smithi125.stdout: metadata_pool 3 2023-10-09T10:40:00.051 INFO:journalctl@ceph.mon.c.smithi125.stdout: inline_data disabled 2023-10-09T10:40:00.051 INFO:journalctl@ceph.mon.c.smithi125.stdout: balancer 2023-10-09T10:40:00.051 INFO:journalctl@ceph.mon.c.smithi125.stdout: bal_rank_mask -1 2023-10-09T10:40:00.051 INFO:journalctl@ceph.mon.c.smithi125.stdout: standby_count_wanted 1 2023-10-09T10:40:00.051 INFO:journalctl@ceph.mon.c.smithi125.stdout: [mds.foo.smithi125.satgiw{0:24575} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.125:6834/1945587275,v1:172.21.15.125:6835/1945587275] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T10:40:00.051 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:40:00.051 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:40:00.051 INFO:journalctl@ceph.mon.c.smithi125.stdout: Standby daemons: 2023-10-09T10:40:00.052 INFO:journalctl@ceph.mon.c.smithi125.stdout: 2023-10-09T10:40:00.052 INFO:journalctl@ceph.mon.c.smithi125.stdout: [mds.foo.smithi155.ydxmyq{-1:24544} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.155:6832/914042814,v1:172.21.15.155:6833/914042814] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T10:40:00.052 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: mon.c@-1(???).osd e76 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T10:40:00.052 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: mon.c@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T10:40:00.052 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: mon.c@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T10:40:00.052 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: mon.c@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T10:40:00.052 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:39:59 smithi125 ceph-mon[200197]: mon.c@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2023-10-09T10:40:00.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:00 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/3498618260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:40:00.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:00 smithi155 ceph-mon[117223]: mon.c calling monitor election 2023-10-09T10:40:00.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:00 smithi155 ceph-mon[117223]: mon.a calling monitor election 2023-10-09T10:40:00.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:00 smithi155 ceph-mon[117223]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T10:40:00.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:00 smithi155 ceph-mon[117223]: monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T10:40:00.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:00 smithi155 ceph-mon[117223]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:40:00.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:00 smithi155 ceph-mon[117223]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T10:40:00.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:00 smithi155 ceph-mon[117223]: mgrmap e34: y(active, since 74s), standbys: x 2023-10-09T10:40:00.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:00 smithi155 ceph-mon[117223]: overall HEALTH_OK 2023-10-09T10:40:00.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:00 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:00.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:00 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:00.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:00 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:40:00.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:00 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:40:00.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:00 smithi155 ceph-mon[117223]: overall HEALTH_OK 2023-10-09T10:40:01.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3498618260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:40:01.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[195507]: mon.c calling monitor election 2023-10-09T10:40:01.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[195507]: mon.a calling monitor election 2023-10-09T10:40:01.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[195507]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T10:40:01.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[195507]: monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T10:40:01.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[195507]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:40:01.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[195507]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T10:40:01.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[195507]: mgrmap e34: y(active, since 74s), standbys: x 2023-10-09T10:40:01.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[195507]: overall HEALTH_OK 2023-10-09T10:40:01.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:01.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:01.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:40:01.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:40:01.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[195507]: overall HEALTH_OK 2023-10-09T10:40:01.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3498618260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:40:01.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[200197]: mon.c calling monitor election 2023-10-09T10:40:01.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[200197]: mon.a calling monitor election 2023-10-09T10:40:01.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[200197]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T10:40:01.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[200197]: monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T10:40:01.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[200197]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:40:01.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[200197]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T10:40:01.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[200197]: mgrmap e34: y(active, since 74s), standbys: x 2023-10-09T10:40:01.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[200197]: overall HEALTH_OK 2023-10-09T10:40:01.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:01.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:01.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:40:01.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:40:01.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:00 smithi125 ceph-mon[200197]: overall HEALTH_OK 2023-10-09T10:40:01.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:40:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:40:01] "GET /metrics HTTP/1.1" 200 33793 "" "Prometheus/2.43.0" 2023-10-09T10:40:01.969 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T10:40:01.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:01 smithi155 ceph-mon[117223]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:01.983 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:01 smithi125 ceph-mon[195507]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:01.984 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:01 smithi125 ceph-mon[200197]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:02.349 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T10:40:02.671 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:40:03.210 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:02 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:03.210 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:02 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:03.210 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:02 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/475144691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:40:03.210 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:02 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:03.210 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:02 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:03.210 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:02 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:03.211 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:02 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:03.211 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:02 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/475144691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:40:03.211 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:02 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:03.211 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:02 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:03.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:02 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:03.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:02 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:03.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:02 smithi155 ceph-mon[117223]: from='client.? 172.21.15.125:0/475144691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:40:03.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:02 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:03.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:02 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:04.104 INFO:teuthology.orchestra.run.smithi125.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T10:40:04.105 INFO:teuthology.orchestra.run.smithi125.stdout:alertmanager.a smithi125 *:9093,9094 running (2m) - 4m 27.3M - 0.25.0 c8568f914cd2 3c13e6c17b7e 2023-10-09T10:40:04.105 INFO:teuthology.orchestra.run.smithi125.stdout:grafana.a smithi155 *:3000 running (2m) - 4m 85.5M - 9.4.7 2c41d148cca3 0ec8893d264d 2023-10-09T10:40:04.105 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi125.satgiw smithi125 running (3m) - 3m 16.9M - 17.2.6-1303-gb2903cb0 5517396c50d4 7b5d28288eab 2023-10-09T10:40:04.105 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi155.ydxmyq smithi155 running (3m) - 3m 14.6M - 17.2.6-1303-gb2903cb0 5517396c50d4 c2c99605d324 2023-10-09T10:40:04.105 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.x smithi155 *:8443,9283,8765 running (96s) - 7m 447M - 18.2.0-598-gb4fa663f 3060897fdf97 67bb8d1af523 2023-10-09T10:40:04.105 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.y smithi125 *:8443,9283,8765 running (117s) - 8m 501M - 18.2.0-598-gb4fa663f 3060897fdf97 107edeccfff9 2023-10-09T10:40:04.105 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a smithi125 running (86s) - 8m 48.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 335012cc14fa 2023-10-09T10:40:04.105 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b smithi155 running (7m) - 7m 77.0M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 14cad29646e0 2023-10-09T10:40:04.105 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c smithi125 running (4s) - 7m 26.2M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 c47abfeb6d97 2023-10-09T10:40:04.105 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.a smithi125 *:9100 running (2m) - 4m 17.2M - 1.5.0 0da6a335fe13 bf5a342348bd 2023-10-09T10:40:04.105 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.b smithi155 *:9100 running (2m) - 4m 8422k - 1.5.0 0da6a335fe13 1320bf6a6ac5 2023-10-09T10:40:04.105 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 smithi125 running (7m) - 7m 74.8M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 346135826df7 2023-10-09T10:40:04.106 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1 smithi125 running (6m) - 6m 80.4M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 11af9c8d1b19 2023-10-09T10:40:04.106 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2 smithi125 running (6m) - 6m 150M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 dc337188d937 2023-10-09T10:40:04.106 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3 smithi125 running (6m) - 6m 68.6M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 c92a4cc944fe 2023-10-09T10:40:04.106 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 smithi155 running (6m) - 6m 78.8M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 e3a61e2037a2 2023-10-09T10:40:04.106 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5 smithi155 running (5m) - 5m 95.8M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 12d4484e25b1 2023-10-09T10:40:04.106 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6 smithi155 running (5m) - 5m 66.5M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 c7b7ee1a3182 2023-10-09T10:40:04.106 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7 smithi155 running (5m) - 5m 102M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 8e225788b8f4 2023-10-09T10:40:04.106 INFO:teuthology.orchestra.run.smithi125.stdout:prometheus.a smithi155 *:9095 running (100s) - 4m 48.2M - 2.43.0 a07b618ecd1d 6412c6139964 2023-10-09T10:40:04.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:03 smithi155 ceph-mon[117223]: pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:04.300 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:03 smithi125 ceph-mon[195507]: pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:04.301 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:03 smithi125 ceph-mon[200197]: pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:04.618 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T10:40:04.911 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:40:04.933 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:40:04 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:40:04.614+0000 7fa537ed4700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-10-09T10:40:05.197 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:04 smithi125 ceph-mon[195507]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:40:05.197 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:04 smithi125 ceph-mon[200197]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:40:05.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:04 smithi155 ceph-mon[117223]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:40:06.064 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:05 smithi125 ceph-mon[195507]: pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:06.065 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:05 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:06.065 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:05 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:06.065 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:05 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:06.065 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:05 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:40:06.065 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:05 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:06.065 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:05 smithi125 ceph-mon[200197]: pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:06.065 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:05 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:06.065 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:05 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:06.065 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:05 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:06.066 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:05 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:40:06.066 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:05 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:06.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:05 smithi155 ceph-mon[117223]: pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:06.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:05 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:06.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:05 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:06.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:05 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:06.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:05 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:40:06.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:05 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:06.452 INFO:teuthology.orchestra.run.smithi125.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:40:06.821 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T10:40:07.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:40:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:40:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T10:40:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T10:40:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:06 smithi125 ceph-mon[195507]: Upgrade: It appears safe to stop mon.b 2023-10-09T10:40:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:07.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:40:07.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:40:07.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T10:40:07.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T10:40:07.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:06 smithi125 ceph-mon[200197]: Upgrade: It appears safe to stop mon.b 2023-10-09T10:40:07.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:07.089 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:40:07.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:06 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:40:07.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:06 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:40:07.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:06 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T10:40:07.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:06 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T10:40:07.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:06 smithi155 ceph-mon[117223]: Upgrade: It appears safe to stop mon.b 2023-10-09T10:40:07.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:06 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:08.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:07 smithi155 ceph-mon[117223]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:40:08.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:07 smithi155 ceph-mon[117223]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:40:08.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:07 smithi155 ceph-mon[117223]: pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:08.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:07 smithi155 ceph-mon[117223]: Upgrade: Updating mon.b 2023-10-09T10:40:08.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:07 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:08.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:07 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:40:08.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:07 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:40:08.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:07 smithi155 ceph-mon[117223]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:08.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:07 smithi155 ceph-mon[117223]: Deploying daemon mon.b on smithi155 2023-10-09T10:40:08.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:07 smithi155 systemd[1]: Stopping Ceph mon.b for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:40:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[195507]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:40:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[195507]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:40:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[195507]: pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[195507]: Upgrade: Updating mon.b 2023-10-09T10:40:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:40:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:40:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:08.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[195507]: Deploying daemon mon.b on smithi155 2023-10-09T10:40:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[200197]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:40:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[200197]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:40:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[200197]: pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[200197]: Upgrade: Updating mon.b 2023-10-09T10:40:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:40:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:40:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:07 smithi125 ceph-mon[200197]: Deploying daemon mon.b on smithi155 2023-10-09T10:40:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-b[117200]: 2023-10-09T10:40:08.150+0000 7f229653a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:40:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-b[117200]: 2023-10-09T10:40:08.150+0000 7f229653a700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-10-09T10:40:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 bash[184667]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T10:40:08.919 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.b.service: Succeeded. 2023-10-09T10:40:08.920 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 systemd[1]: Stopped Ceph mon.b for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:40:08.920 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 systemd[1]: Starting Ceph mon.b for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:40:08.920 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:40:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:40:09.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T10:40:09.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T10:40:09.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: pidfile_write: ignore empty --pid-file 2023-10-09T10:40:09.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: load: jerasure load: lrc 2023-10-09T10:40:09.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 bash[184816]: 9b477d1850dcdf0bf0a711c26b56ab6e3952f8fd42c321d5727032c33f9060cd 2023-10-09T10:40:09.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: RocksDB version: 7.9.2 2023-10-09T10:40:09.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Git sha 0 2023-10-09T10:40:09.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T10:40:09.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: DB SUMMARY 2023-10-09T10:40:09.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: DB Session ID: B5A1Q90ZNB8YN5O2XFFC 2023-10-09T10:40:09.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: CURRENT file: CURRENT 2023-10-09T10:40:09.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T10:40:09.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: MANIFEST file: MANIFEST-000009 size: 718 Bytes 2023-10-09T10:40:09.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000021.sst 2023-10-09T10:40:09.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000019.log size: 5261234 ; 2023-10-09T10:40:09.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.error_if_exists: 0 2023-10-09T10:40:09.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.create_if_missing: 0 2023-10-09T10:40:09.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.paranoid_checks: 1 2023-10-09T10:40:09.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T10:40:09.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T10:40:09.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T10:40:09.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.env: 0x55d48fc2a720 2023-10-09T10:40:09.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T10:40:09.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.info_log: 0x55d4920c9420 2023-10-09T10:40:09.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T10:40:09.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.statistics: (nil) 2023-10-09T10:40:09.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.use_fsync: 0 2023-10-09T10:40:09.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_log_file_size: 0 2023-10-09T10:40:09.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T10:40:09.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T10:40:09.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T10:40:09.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T10:40:09.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.allow_fallocate: 1 2023-10-09T10:40:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T10:40:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T10:40:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.use_direct_reads: 0 2023-10-09T10:40:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T10:40:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T10:40:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.db_log_dir: 2023-10-09T10:40:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.wal_dir: 2023-10-09T10:40:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T10:40:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T10:40:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T10:40:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T10:40:09.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T10:40:09.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T10:40:09.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T10:40:09.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T10:40:09.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.write_buffer_manager: 0x55d49135ac80 2023-10-09T10:40:09.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T10:40:09.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T10:40:09.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T10:40:09.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T10:40:09.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T10:40:09.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T10:40:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T10:40:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T10:40:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.unordered_write: 0 2023-10-09T10:40:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T10:40:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T10:40:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T10:40:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T10:40:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.row_cache: None 2023-10-09T10:40:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.wal_filter: None 2023-10-09T10:40:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T10:40:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T10:40:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.two_write_queues: 0 2023-10-09T10:40:09.477 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T10:40:09.477 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.wal_compression: 0 2023-10-09T10:40:09.477 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.atomic_flush: 0 2023-10-09T10:40:09.477 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T10:40:09.477 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T10:40:09.477 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T10:40:09.477 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.log_readahead_size: 0 2023-10-09T10:40:09.477 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T10:40:09.477 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T10:40:09.477 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T10:40:09.477 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T10:40:09.478 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T10:40:09.479 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T10:40:09.479 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T10:40:09.479 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_background_jobs: 2 2023-10-09T10:40:09.479 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_background_compactions: -1 2023-10-09T10:40:09.479 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_subcompactions: 1 2023-10-09T10:40:09.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T10:40:09.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T10:40:09.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T10:40:09.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T10:40:09.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T10:40:09.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T10:40:09.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T10:40:09.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T10:40:09.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_open_files: -1 2023-10-09T10:40:09.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T10:40:09.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T10:40:09.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T10:40:09.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T10:40:09.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_background_flushes: -1 2023-10-09T10:40:09.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Compression algorithms supported: 2023-10-09T10:40:09.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T10:40:09.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: kZSTD supported: 0 2023-10-09T10:40:09.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: kXpressCompression supported: 0 2023-10-09T10:40:09.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T10:40:09.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: kZlibCompression supported: 1 2023-10-09T10:40:09.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: kSnappyCompression supported: 1 2023-10-09T10:40:09.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: kLZ4Compression supported: 1 2023-10-09T10:40:09.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: kBZip2Compression supported: 0 2023-10-09T10:40:09.482 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T10:40:09.482 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T10:40:09.482 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2023-10-09T10:40:09.482 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T10:40:09.482 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T10:40:09.482 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.merge_operator: 2023-10-09T10:40:09.482 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compaction_filter: None 2023-10-09T10:40:09.482 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T10:40:09.482 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T10:40:09.482 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T10:40:09.482 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T10:40:09.482 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d4920c9520) 2023-10-09T10:40:09.483 INFO:journalctl@ceph.mon.b.smithi155.stdout: cache_index_and_filter_blocks: 1 2023-10-09T10:40:09.483 INFO:journalctl@ceph.mon.b.smithi155.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T10:40:09.483 INFO:journalctl@ceph.mon.b.smithi155.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T10:40:09.483 INFO:journalctl@ceph.mon.b.smithi155.stdout: pin_top_level_index_and_filter: 1 2023-10-09T10:40:09.483 INFO:journalctl@ceph.mon.b.smithi155.stdout: index_type: 0 2023-10-09T10:40:09.483 INFO:journalctl@ceph.mon.b.smithi155.stdout: data_block_index_type: 0 2023-10-09T10:40:09.483 INFO:journalctl@ceph.mon.b.smithi155.stdout: index_shortening: 1 2023-10-09T10:40:09.483 INFO:journalctl@ceph.mon.b.smithi155.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T10:40:09.483 INFO:journalctl@ceph.mon.b.smithi155.stdout: checksum: 4 2023-10-09T10:40:09.483 INFO:journalctl@ceph.mon.b.smithi155.stdout: no_block_cache: 0 2023-10-09T10:40:09.483 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_cache: 0x55d4913db350 2023-10-09T10:40:09.483 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_cache_name: BinnedLRUCache 2023-10-09T10:40:09.483 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_cache_options: 2023-10-09T10:40:09.484 INFO:journalctl@ceph.mon.b.smithi155.stdout: capacity : 536870912 2023-10-09T10:40:09.484 INFO:journalctl@ceph.mon.b.smithi155.stdout: num_shard_bits : 4 2023-10-09T10:40:09.484 INFO:journalctl@ceph.mon.b.smithi155.stdout: strict_capacity_limit : 0 2023-10-09T10:40:09.484 INFO:journalctl@ceph.mon.b.smithi155.stdout: high_pri_pool_ratio: 0.000 2023-10-09T10:40:09.484 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_cache_compressed: (nil) 2023-10-09T10:40:09.484 INFO:journalctl@ceph.mon.b.smithi155.stdout: persistent_cache: (nil) 2023-10-09T10:40:09.484 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_size: 4096 2023-10-09T10:40:09.484 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_size_deviation: 10 2023-10-09T10:40:09.484 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_restart_interval: 16 2023-10-09T10:40:09.484 INFO:journalctl@ceph.mon.b.smithi155.stdout: index_block_restart_interval: 1 2023-10-09T10:40:09.484 INFO:journalctl@ceph.mon.b.smithi155.stdout: metadata_block_size: 4096 2023-10-09T10:40:09.484 INFO:journalctl@ceph.mon.b.smithi155.stdout: partition_filters: 0 2023-10-09T10:40:09.484 INFO:journalctl@ceph.mon.b.smithi155.stdout: use_delta_encoding: 1 2023-10-09T10:40:09.485 INFO:journalctl@ceph.mon.b.smithi155.stdout: filter_policy: bloomfilter 2023-10-09T10:40:09.485 INFO:journalctl@ceph.mon.b.smithi155.stdout: whole_key_filtering: 1 2023-10-09T10:40:09.485 INFO:journalctl@ceph.mon.b.smithi155.stdout: verify_compression: 0 2023-10-09T10:40:09.485 INFO:journalctl@ceph.mon.b.smithi155.stdout: read_amp_bytes_per_bit: 0 2023-10-09T10:40:09.485 INFO:journalctl@ceph.mon.b.smithi155.stdout: format_version: 5 2023-10-09T10:40:09.485 INFO:journalctl@ceph.mon.b.smithi155.stdout: enable_index_compression: 1 2023-10-09T10:40:09.485 INFO:journalctl@ceph.mon.b.smithi155.stdout: block_align: 0 2023-10-09T10:40:09.485 INFO:journalctl@ceph.mon.b.smithi155.stdout: max_auto_readahead_size: 262144 2023-10-09T10:40:09.485 INFO:journalctl@ceph.mon.b.smithi155.stdout: prepopulate_block_cache: 0 2023-10-09T10:40:09.485 INFO:journalctl@ceph.mon.b.smithi155.stdout: initial_auto_readahead_size: 8192 2023-10-09T10:40:09.485 INFO:journalctl@ceph.mon.b.smithi155.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T10:40:09.485 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T10:40:09.485 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T10:40:09.486 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compression: NoCompression 2023-10-09T10:40:09.486 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T10:40:09.486 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T10:40:09.486 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T10:40:09.486 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.num_levels: 7 2023-10-09T10:40:09.486 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T10:40:09.486 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T10:40:09.486 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T10:40:09.486 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T10:40:09.486 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T10:40:09.486 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T10:40:09.486 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T10:40:09.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T10:40:09.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T10:40:09.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T10:40:09.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T10:40:09.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T10:40:09.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T10:40:09.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T10:40:09.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T10:40:09.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T10:40:09.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T10:40:09.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T10:40:09.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T10:40:09.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T10:40:09.488 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T10:40:09.488 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T10:40:09.488 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T10:40:09.488 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T10:40:09.488 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T10:40:09.488 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T10:40:09.488 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T10:40:09.488 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T10:40:09.488 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T10:40:09.488 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T10:40:09.488 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T10:40:09.488 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T10:40:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T10:40:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T10:40:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T10:40:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T10:40:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T10:40:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T10:40:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T10:40:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T10:40:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T10:40:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T10:40:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T10:40:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T10:40:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T10:40:09.490 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T10:40:09.490 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T10:40:09.490 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T10:40:09.490 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T10:40:09.490 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T10:40:09.490 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T10:40:09.490 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T10:40:09.490 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T10:40:09.490 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.table_properties_collectors: 2023-10-09T10:40:09.490 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.inplace_update_support: 0 2023-10-09T10:40:09.490 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T10:40:09.490 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T10:40:09.490 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T10:40:09.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T10:40:09.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.bloom_locality: 0 2023-10-09T10:40:09.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.max_successive_merges: 0 2023-10-09T10:40:09.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T10:40:09.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T10:40:09.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T10:40:09.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T10:40:09.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.ttl: 2592000 2023-10-09T10:40:09.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T10:40:09.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T10:40:09.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T10:40:09.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.enable_blob_files: false 2023-10-09T10:40:09.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.min_blob_size: 0 2023-10-09T10:40:09.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T10:40:09.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T10:40:09.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T10:40:09.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T10:40:09.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T10:40:09.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T10:40:09.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T10:40:09.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T10:40:09.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 21.sst 2023-10-09T10:40:09.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T10:40:09.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 23, last_sequence is 10406, log_number is 19,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T10:40:09.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 19 2023-10-09T10:40:09.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ddd781e2-939c-45eb-84eb-7d2c97935899 2023-10-09T10:40:09.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696848008981682, "job": 1, "event": "recovery_started", "wal_files": [19]} 2023-10-09T10:40:09.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #19 mode 2 2023-10-09T10:40:09.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 systemd[1]: Started Ceph mon.b for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:40:09.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696848008993553, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 24, "file_size": 3223090, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 10403, "largest_seqno": 12454, "table_properties": {"data_size": 3214914, "index_size": 5236, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1925, "raw_key_size": 17189, "raw_average_key_size": 22, "raw_value_size": 3198637, "raw_average_value_size": 4231, "num_data_blocks": 240, "num_entries": 756, "num_filter_entries": 756, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696848008, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ddd781e2-939c-45eb-84eb-7d2c97935899", "db_session_id": "B5A1Q90ZNB8YN5O2XFFC", "orig_file_number": 24, "seqno_to_time_mapping": "N/A"}} 2023-10-09T10:40:09.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696848008993646, "job": 1, "event": "recovery_finished"} 2023-10-09T10:40:09.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: [db/version_set.cc:5047] Creating manifest 26 2023-10-09T10:40:09.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T10:40:09.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000019.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T10:40:09.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55d491478000 2023-10-09T10:40:09.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: DB pointer 0x55d491464000 2023-10-09T10:40:09.494 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T10:40:09.494 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T10:40:09.494 INFO:journalctl@ceph.mon.b.smithi155.stdout: ** DB Stats ** 2023-10-09T10:40:09.494 INFO:journalctl@ceph.mon.b.smithi155.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T10:40:09.494 INFO:journalctl@ceph.mon.b.smithi155.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T10:40:09.494 INFO:journalctl@ceph.mon.b.smithi155.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T10:40:09.494 INFO:journalctl@ceph.mon.b.smithi155.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T10:40:09.494 INFO:journalctl@ceph.mon.b.smithi155.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T10:40:09.494 INFO:journalctl@ceph.mon.b.smithi155.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T10:40:09.494 INFO:journalctl@ceph.mon.b.smithi155.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T10:40:09.494 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:40:09.494 INFO:journalctl@ceph.mon.b.smithi155.stdout: ** Compaction Stats [default] ** 2023-10-09T10:40:09.495 INFO:journalctl@ceph.mon.b.smithi155.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T10:40:09.495 INFO:journalctl@ceph.mon.b.smithi155.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T10:40:09.495 INFO:journalctl@ceph.mon.b.smithi155.stdout: L0 1/0 3.07 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 680.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T10:40:09.495 INFO:journalctl@ceph.mon.b.smithi155.stdout: L6 1/0 6.88 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T10:40:09.495 INFO:journalctl@ceph.mon.b.smithi155.stdout: Sum 2/0 9.96 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 680.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T10:40:09.495 INFO:journalctl@ceph.mon.b.smithi155.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 680.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T10:40:09.495 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:40:09.495 INFO:journalctl@ceph.mon.b.smithi155.stdout: ** Compaction Stats [default] ** 2023-10-09T10:40:09.495 INFO:journalctl@ceph.mon.b.smithi155.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T10:40:09.495 INFO:journalctl@ceph.mon.b.smithi155.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T10:40:09.495 INFO:journalctl@ceph.mon.b.smithi155.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 680.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T10:40:09.495 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:40:09.496 INFO:journalctl@ceph.mon.b.smithi155.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T10:40:09.496 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:40:09.496 INFO:journalctl@ceph.mon.b.smithi155.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T10:40:09.496 INFO:journalctl@ceph.mon.b.smithi155.stdout: Flush(GB): cumulative 0.003, interval 0.003 2023-10-09T10:40:09.496 INFO:journalctl@ceph.mon.b.smithi155.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T10:40:09.496 INFO:journalctl@ceph.mon.b.smithi155.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T10:40:09.496 INFO:journalctl@ceph.mon.b.smithi155.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T10:40:09.496 INFO:journalctl@ceph.mon.b.smithi155.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T10:40:09.496 INFO:journalctl@ceph.mon.b.smithi155.stdout: Cumulative compaction: 0.00 GB write, 207.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:40:09.496 INFO:journalctl@ceph.mon.b.smithi155.stdout: Interval compaction: 0.00 GB write, 207.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T10:40:09.496 INFO:journalctl@ceph.mon.b.smithi155.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T10:40:09.496 INFO:journalctl@ceph.mon.b.smithi155.stdout: Block cache BinnedLRUCache@0x55d4913db350#7 capacity: 512.00 MB usage: 8.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2023-10-09T10:40:09.497 INFO:journalctl@ceph.mon.b.smithi155.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2023-10-09T10:40:09.497 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:40:09.497 INFO:journalctl@ceph.mon.b.smithi155.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T10:40:09.497 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: starting mon.b rank 1 at public addrs [v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0] at bind addrs [v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:40:09.497 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: mon.b@-1(???) e3 preinit fsid daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:40:09.497 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: mon.b@-1(???).mds e8 new map 2023-10-09T10:40:09.497 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: mon.b@-1(???).mds e8 print_map 2023-10-09T10:40:09.497 INFO:journalctl@ceph.mon.b.smithi155.stdout: e8 2023-10-09T10:40:09.497 INFO:journalctl@ceph.mon.b.smithi155.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T10:40:09.497 INFO:journalctl@ceph.mon.b.smithi155.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T10:40:09.497 INFO:journalctl@ceph.mon.b.smithi155.stdout: legacy client fscid: 1 2023-10-09T10:40:09.497 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:40:09.498 INFO:journalctl@ceph.mon.b.smithi155.stdout: Filesystem 'foo' (1) 2023-10-09T10:40:09.498 INFO:journalctl@ceph.mon.b.smithi155.stdout: fs_name foo 2023-10-09T10:40:09.498 INFO:journalctl@ceph.mon.b.smithi155.stdout: epoch 7 2023-10-09T10:40:09.498 INFO:journalctl@ceph.mon.b.smithi155.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T10:40:09.498 INFO:journalctl@ceph.mon.b.smithi155.stdout: created 2023-10-09T10:35:56.814729+0000 2023-10-09T10:40:09.498 INFO:journalctl@ceph.mon.b.smithi155.stdout: modified 2023-10-09T10:36:10.497834+0000 2023-10-09T10:40:09.498 INFO:journalctl@ceph.mon.b.smithi155.stdout: tableserver 0 2023-10-09T10:40:09.498 INFO:journalctl@ceph.mon.b.smithi155.stdout: root 0 2023-10-09T10:40:09.498 INFO:journalctl@ceph.mon.b.smithi155.stdout: session_timeout 60 2023-10-09T10:40:09.498 INFO:journalctl@ceph.mon.b.smithi155.stdout: session_autoclose 300 2023-10-09T10:40:09.498 INFO:journalctl@ceph.mon.b.smithi155.stdout: max_file_size 1099511627776 2023-10-09T10:40:09.498 INFO:journalctl@ceph.mon.b.smithi155.stdout: required_client_features {} 2023-10-09T10:40:09.499 INFO:journalctl@ceph.mon.b.smithi155.stdout: last_failure 0 2023-10-09T10:40:09.499 INFO:journalctl@ceph.mon.b.smithi155.stdout: last_failure_osd_epoch 0 2023-10-09T10:40:09.499 INFO:journalctl@ceph.mon.b.smithi155.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T10:40:09.499 INFO:journalctl@ceph.mon.b.smithi155.stdout: max_mds 1 2023-10-09T10:40:09.499 INFO:journalctl@ceph.mon.b.smithi155.stdout: in 0 2023-10-09T10:40:09.499 INFO:journalctl@ceph.mon.b.smithi155.stdout: up {0=24575} 2023-10-09T10:40:09.499 INFO:journalctl@ceph.mon.b.smithi155.stdout: failed 2023-10-09T10:40:09.499 INFO:journalctl@ceph.mon.b.smithi155.stdout: damaged 2023-10-09T10:40:09.499 INFO:journalctl@ceph.mon.b.smithi155.stdout: stopped 2023-10-09T10:40:09.499 INFO:journalctl@ceph.mon.b.smithi155.stdout: data_pools [4] 2023-10-09T10:40:09.499 INFO:journalctl@ceph.mon.b.smithi155.stdout: metadata_pool 3 2023-10-09T10:40:09.499 INFO:journalctl@ceph.mon.b.smithi155.stdout: inline_data disabled 2023-10-09T10:40:09.500 INFO:journalctl@ceph.mon.b.smithi155.stdout: balancer 2023-10-09T10:40:09.500 INFO:journalctl@ceph.mon.b.smithi155.stdout: bal_rank_mask -1 2023-10-09T10:40:09.500 INFO:journalctl@ceph.mon.b.smithi155.stdout: standby_count_wanted 1 2023-10-09T10:40:09.500 INFO:journalctl@ceph.mon.b.smithi155.stdout: [mds.foo.smithi125.satgiw{0:24575} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.125:6834/1945587275,v1:172.21.15.125:6835/1945587275] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T10:40:09.500 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:40:09.500 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:40:09.500 INFO:journalctl@ceph.mon.b.smithi155.stdout: Standby daemons: 2023-10-09T10:40:09.500 INFO:journalctl@ceph.mon.b.smithi155.stdout: 2023-10-09T10:40:09.500 INFO:journalctl@ceph.mon.b.smithi155.stdout: [mds.foo.smithi155.ydxmyq{-1:24544} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.155:6832/914042814,v1:172.21.15.155:6833/914042814] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T10:40:09.500 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: mon.b@-1(???).osd e76 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T10:40:09.500 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: mon.b@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T10:40:09.500 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: mon.b@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T10:40:09.500 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: mon.b@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T10:40:09.501 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:08 smithi155 ceph-mon[184854]: mon.b@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-10-09T10:40:09.501 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:09 smithi155 ceph-mon[184854]: expand_channel_meta expand map: {default=false} 2023-10-09T10:40:09.501 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:09 smithi155 ceph-mon[184854]: expand_channel_meta from 'false' to 'false' 2023-10-09T10:40:09.501 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:09 smithi155 ceph-mon[184854]: expand_channel_meta expanded map: {default=false} 2023-10-09T10:40:09.501 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:09 smithi155 ceph-mon[184854]: expand_channel_meta expand map: {default=info} 2023-10-09T10:40:09.501 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:09 smithi155 ceph-mon[184854]: expand_channel_meta from 'info' to 'info' 2023-10-09T10:40:09.501 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:09 smithi155 ceph-mon[184854]: expand_channel_meta expanded map: {default=info} 2023-10-09T10:40:09.501 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:09 smithi155 ceph-mon[184854]: expand_channel_meta expand map: {default=daemon} 2023-10-09T10:40:09.501 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:09 smithi155 ceph-mon[184854]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T10:40:09.501 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:09 smithi155 ceph-mon[184854]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T10:40:09.501 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:09 smithi155 ceph-mon[184854]: expand_channel_meta expand map: {default=debug} 2023-10-09T10:40:09.501 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:09 smithi155 ceph-mon[184854]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T10:40:09.501 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:09 smithi155 ceph-mon[184854]: expand_channel_meta expanded map: {default=debug} 2023-10-09T10:40:11.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:40:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:40:11] "GET /metrics HTTP/1.1" 200 33793 "" "Prometheus/2.43.0" 2023-10-09T10:40:14.306 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:40:14.003+0000 7fa537ed4700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T10:40:14.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: overall HEALTH_OK 2023-10-09T10:40:14.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:40:14.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: mon.a calling monitor election 2023-10-09T10:40:14.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: mon.c calling monitor election 2023-10-09T10:40:14.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: mon.b calling monitor election 2023-10-09T10:40:14.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:40:14.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:40:14.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:14.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:14.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2023-10-09T10:40:14.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: overall HEALTH_OK 2023-10-09T10:40:14.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: mon.a calling monitor election 2023-10-09T10:40:14.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T10:40:14.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: monmap e4: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T10:40:14.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:40:14.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T10:40:14.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: mgrmap e34: y(active, since 89s), standbys: x 2023-10-09T10:40:14.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: overall HEALTH_OK 2023-10-09T10:40:14.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:14.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:14.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:40:14.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: overall HEALTH_OK 2023-10-09T10:40:14.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:40:14.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: mon.a calling monitor election 2023-10-09T10:40:14.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: mon.c calling monitor election 2023-10-09T10:40:14.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: mon.b calling monitor election 2023-10-09T10:40:14.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:40:14.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:40:14.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:14.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:14.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2023-10-09T10:40:14.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: overall HEALTH_OK 2023-10-09T10:40:14.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: mon.a calling monitor election 2023-10-09T10:40:14.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T10:40:14.577 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: monmap e4: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T10:40:14.577 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:40:14.577 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T10:40:14.577 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: mgrmap e34: y(active, since 89s), standbys: x 2023-10-09T10:40:14.578 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: overall HEALTH_OK 2023-10-09T10:40:14.578 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:14.578 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:14.578 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:40:14.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: overall HEALTH_OK 2023-10-09T10:40:14.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T10:40:14.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: mon.a calling monitor election 2023-10-09T10:40:14.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: mon.c calling monitor election 2023-10-09T10:40:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: mon.b calling monitor election 2023-10-09T10:40:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T10:40:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T10:40:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2023-10-09T10:40:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: overall HEALTH_OK 2023-10-09T10:40:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: mon.a calling monitor election 2023-10-09T10:40:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T10:40:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: monmap e4: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T10:40:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: fsmap foo:1 {0=foo.smithi125.satgiw=up:active} 1 up:standby 2023-10-09T10:40:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T10:40:14.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: mgrmap e34: y(active, since 89s), standbys: x 2023-10-09T10:40:14.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: overall HEALTH_OK 2023-10-09T10:40:14.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:14.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:14.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:40:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:15 smithi125 ceph-mon[195507]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:40:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:15 smithi125 ceph-mon[200197]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:40:15.716 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:15 smithi155 ceph-mon[184854]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:15.716 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:40:16.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:16 smithi125 ceph-mon[195507]: mgrmap e35: y(active, since 90s), standbys: x 2023-10-09T10:40:16.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:16 smithi125 ceph-mon[200197]: mgrmap e35: y(active, since 90s), standbys: x 2023-10-09T10:40:16.633 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:16 smithi155 ceph-mon[184854]: mgrmap e35: y(active, since 90s), standbys: x 2023-10-09T10:40:18.205 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:17 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:18.205 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:17 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:18.206 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:17 smithi155 ceph-mon[184854]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:18.206 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:17 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:18.206 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:17 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:18.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:17 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:18.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:17 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:18.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:17 smithi125 ceph-mon[200197]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:18.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:17 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:18.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:17 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:17 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:17 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:17 smithi125 ceph-mon[195507]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:17 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:17 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:18.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:40:18 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:40:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:40:20.287 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[195507]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:20.288 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:20.288 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:20.288 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:20.288 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:40:20.288 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:20.288 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:40:20.288 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:40:20.288 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:20.288 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[200197]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:20.288 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:20.288 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:20.289 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:20.289 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:40:20.289 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:20.289 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:40:20.289 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:40:20.289 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:20 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:20 smithi155 ceph-mon[184854]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:20 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:20 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:20 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:20 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:40:20.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:20 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:20.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:20 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:40:20.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:20 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:40:20.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:20 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:21.270 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:40:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:40:21] "GET /metrics HTTP/1.1" 200 33856 "" "Prometheus/2.43.0" 2023-10-09T10:40:21.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:21 smithi125 ceph-mon[195507]: Reconfiguring mon.a (monmap changed)... 2023-10-09T10:40:21.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:21 smithi125 ceph-mon[195507]: Reconfiguring daemon mon.a on smithi125 2023-10-09T10:40:21.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:21 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:21.272 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:21 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:21.272 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:21 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:40:21.272 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:21 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:40:21.272 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:21 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:21.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:21 smithi125 ceph-mon[200197]: Reconfiguring mon.a (monmap changed)... 2023-10-09T10:40:21.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:21 smithi125 ceph-mon[200197]: Reconfiguring daemon mon.a on smithi125 2023-10-09T10:40:21.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:21 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:21.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:21 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:21.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:21 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:40:21.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:21 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:40:21.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:21 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:21.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:21 smithi155 ceph-mon[184854]: Reconfiguring mon.a (monmap changed)... 2023-10-09T10:40:21.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:21 smithi155 ceph-mon[184854]: Reconfiguring daemon mon.a on smithi125 2023-10-09T10:40:21.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:21 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:21.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:21 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:21.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:21 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:40:21.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:21 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:40:21.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:21 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:22.267 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[195507]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T10:40:22.267 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[195507]: Reconfiguring daemon mgr.y on smithi125 2023-10-09T10:40:22.267 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[195507]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:22.268 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:22.268 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:22.268 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:40:22.268 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:40:22.268 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:22.268 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[200197]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T10:40:22.268 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[200197]: Reconfiguring daemon mgr.y on smithi125 2023-10-09T10:40:22.268 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[200197]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:22.268 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:22.269 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:22.269 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:40:22.269 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:40:22.269 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:22 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:22.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:22 smithi155 ceph-mon[184854]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T10:40:22.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:22 smithi155 ceph-mon[184854]: Reconfiguring daemon mgr.y on smithi125 2023-10-09T10:40:22.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:22 smithi155 ceph-mon[184854]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:22.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:22 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:22.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:22 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:22.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:22 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:40:22.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:22 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:40:22.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:22 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:23.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:23 smithi125 ceph-mon[195507]: Reconfiguring mon.c (monmap changed)... 2023-10-09T10:40:23.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:23 smithi125 ceph-mon[195507]: Reconfiguring daemon mon.c on smithi125 2023-10-09T10:40:23.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:23.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:23.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T10:40:23.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:23 smithi125 ceph-mon[200197]: Reconfiguring mon.c (monmap changed)... 2023-10-09T10:40:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:23 smithi125 ceph-mon[200197]: Reconfiguring daemon mon.c on smithi125 2023-10-09T10:40:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T10:40:23.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:23 smithi155 ceph-mon[184854]: Reconfiguring mon.c (monmap changed)... 2023-10-09T10:40:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:23 smithi155 ceph-mon[184854]: Reconfiguring daemon mon.c on smithi125 2023-10-09T10:40:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:23.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:23.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T10:40:23.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:24.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:24 smithi125 ceph-mon[195507]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T10:40:24.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:24 smithi125 ceph-mon[195507]: Reconfiguring daemon osd.0 on smithi125 2023-10-09T10:40:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:24 smithi125 ceph-mon[195507]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T10:40:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:24.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:24 smithi125 ceph-mon[200197]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T10:40:24.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:24 smithi125 ceph-mon[200197]: Reconfiguring daemon osd.0 on smithi125 2023-10-09T10:40:24.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:24 smithi125 ceph-mon[200197]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:24.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:24.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:24.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T10:40:24.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:24 smithi155 ceph-mon[184854]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T10:40:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:24 smithi155 ceph-mon[184854]: Reconfiguring daemon osd.0 on smithi125 2023-10-09T10:40:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:24 smithi155 ceph-mon[184854]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:24.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:24.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:24.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T10:40:24.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:25.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:25 smithi125 ceph-mon[195507]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T10:40:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:25 smithi125 ceph-mon[195507]: Reconfiguring daemon osd.1 on smithi125 2023-10-09T10:40:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:25 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:25 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:25 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T10:40:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:25 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:25.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:25 smithi125 ceph-mon[200197]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T10:40:25.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:25 smithi125 ceph-mon[200197]: Reconfiguring daemon osd.1 on smithi125 2023-10-09T10:40:25.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:25 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:25.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:25 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:25.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:25 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T10:40:25.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:25 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:25 smithi155 ceph-mon[184854]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T10:40:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:25 smithi155 ceph-mon[184854]: Reconfiguring daemon osd.1 on smithi125 2023-10-09T10:40:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:25 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:25 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:25.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:25 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T10:40:25.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:25 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:26.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:26 smithi125 ceph-mon[195507]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T10:40:26.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:26 smithi125 ceph-mon[195507]: Reconfiguring daemon osd.2 on smithi125 2023-10-09T10:40:26.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:26 smithi125 ceph-mon[195507]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:26.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:26 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:26.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:26 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:26.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:26 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T10:40:26.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:26 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:26.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:26 smithi125 ceph-mon[200197]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T10:40:26.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:26 smithi125 ceph-mon[200197]: Reconfiguring daemon osd.2 on smithi125 2023-10-09T10:40:26.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:26 smithi125 ceph-mon[200197]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:26.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:26 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:26.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:26 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:26.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:26 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T10:40:26.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:26 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:26 smithi155 ceph-mon[184854]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T10:40:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:26 smithi155 ceph-mon[184854]: Reconfiguring daemon osd.2 on smithi125 2023-10-09T10:40:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:26 smithi155 ceph-mon[184854]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:26 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:26.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:26 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:26.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:26 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T10:40:26.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:26 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:27.296 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:27 smithi125 ceph-mon[195507]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T10:40:27.296 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:27 smithi125 ceph-mon[195507]: Reconfiguring daemon osd.3 on smithi125 2023-10-09T10:40:27.296 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:27 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:27.296 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:27 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:27.296 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:27 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.satgiw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T10:40:27.296 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:27 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:27.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:27 smithi125 ceph-mon[200197]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T10:40:27.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:27 smithi125 ceph-mon[200197]: Reconfiguring daemon osd.3 on smithi125 2023-10-09T10:40:27.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:27 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:27.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:27 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:27.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:27 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.satgiw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T10:40:27.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:27 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:27.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:27 smithi155 ceph-mon[184854]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T10:40:27.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:27 smithi155 ceph-mon[184854]: Reconfiguring daemon osd.3 on smithi125 2023-10-09T10:40:27.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:27 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:27.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:27 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:27.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:27 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.satgiw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T10:40:27.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:27 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:28.283 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:28 smithi155 ceph-mon[184854]: Reconfiguring mds.foo.smithi125.satgiw (monmap changed)... 2023-10-09T10:40:28.283 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:28 smithi155 ceph-mon[184854]: Reconfiguring daemon mds.foo.smithi125.satgiw on smithi125 2023-10-09T10:40:28.283 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:28 smithi155 ceph-mon[184854]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:28.283 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:28 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:28.285 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:28 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:28.285 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:28 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:40:28.285 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:28 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:40:28.285 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:28 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:28.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[195507]: Reconfiguring mds.foo.smithi125.satgiw (monmap changed)... 2023-10-09T10:40:28.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[195507]: Reconfiguring daemon mds.foo.smithi125.satgiw on smithi125 2023-10-09T10:40:28.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[195507]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:28.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:28.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:28.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:40:28.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:40:28.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:28.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[200197]: Reconfiguring mds.foo.smithi125.satgiw (monmap changed)... 2023-10-09T10:40:28.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[200197]: Reconfiguring daemon mds.foo.smithi125.satgiw on smithi125 2023-10-09T10:40:28.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[200197]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T10:40:28.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:28.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:28.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T10:40:28.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T10:40:28.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:28 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:28.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:40:28 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:40:28.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:40:29.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:29 smithi125 ceph-mon[200197]: Reconfiguring mon.b (monmap changed)... 2023-10-09T10:40:29.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:29 smithi125 ceph-mon[200197]: Reconfiguring daemon mon.b on smithi155 2023-10-09T10:40:29.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:29.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:29.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:40:29.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:40:29.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:29.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:29 smithi125 ceph-mon[195507]: Reconfiguring mon.b (monmap changed)... 2023-10-09T10:40:29.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:29 smithi125 ceph-mon[195507]: Reconfiguring daemon mon.b on smithi155 2023-10-09T10:40:29.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:29.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:29.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:40:29.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:40:29.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:29.344 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:29 smithi155 ceph-mon[184854]: Reconfiguring mon.b (monmap changed)... 2023-10-09T10:40:29.344 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:29 smithi155 ceph-mon[184854]: Reconfiguring daemon mon.b on smithi155 2023-10-09T10:40:29.344 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:29.344 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:29.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T10:40:29.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T10:40:29.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:30.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:30 smithi155 ceph-mon[184854]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T10:40:30.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:30 smithi155 ceph-mon[184854]: Reconfiguring daemon mgr.x on smithi155 2023-10-09T10:40:30.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:30 smithi155 ceph-mon[184854]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:30.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:30.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:30.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T10:40:30.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:30.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:40:30.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[195507]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T10:40:30.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[195507]: Reconfiguring daemon mgr.x on smithi155 2023-10-09T10:40:30.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[195507]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:30.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:30.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:30.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T10:40:30.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:30.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:40:30.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[200197]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T10:40:30.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[200197]: Reconfiguring daemon mgr.x on smithi155 2023-10-09T10:40:30.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[200197]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:30.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:30.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:30.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T10:40:30.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:30.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:40:31.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:31 smithi155 ceph-mon[184854]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T10:40:31.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:31 smithi155 ceph-mon[184854]: Reconfiguring daemon osd.4 on smithi155 2023-10-09T10:40:31.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:31 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:31.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:31 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:31.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:31 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T10:40:31.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:31 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:31.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:40:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:40:31] "GET /metrics HTTP/1.1" 200 33858 "" "Prometheus/2.43.0" 2023-10-09T10:40:31.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:31 smithi125 ceph-mon[195507]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T10:40:31.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:31 smithi125 ceph-mon[195507]: Reconfiguring daemon osd.4 on smithi155 2023-10-09T10:40:31.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:31 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:31.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:31 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:31.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:31 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T10:40:31.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:31 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:31.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:31 smithi125 ceph-mon[200197]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T10:40:31.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:31 smithi125 ceph-mon[200197]: Reconfiguring daemon osd.4 on smithi155 2023-10-09T10:40:31.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:31 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:31.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:31 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:31.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:31 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T10:40:31.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:31 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:32.194 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:32 smithi155 ceph-mon[184854]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T10:40:32.194 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:32 smithi155 ceph-mon[184854]: Reconfiguring daemon osd.5 on smithi155 2023-10-09T10:40:32.194 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:32 smithi155 ceph-mon[184854]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:32.194 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:32 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:32.194 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:32 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:32.194 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:32 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T10:40:32.195 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:32 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:32.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:32 smithi125 ceph-mon[195507]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T10:40:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:32 smithi125 ceph-mon[195507]: Reconfiguring daemon osd.5 on smithi155 2023-10-09T10:40:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:32 smithi125 ceph-mon[195507]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:32 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:32 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:32 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T10:40:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:32 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:32.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:32 smithi125 ceph-mon[200197]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T10:40:32.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:32 smithi125 ceph-mon[200197]: Reconfiguring daemon osd.5 on smithi155 2023-10-09T10:40:32.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:32 smithi125 ceph-mon[200197]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:32.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:32 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:32.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:32 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:32.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:32 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T10:40:32.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:32 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:33.127 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:33 smithi155 ceph-mon[184854]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T10:40:33.127 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:33 smithi155 ceph-mon[184854]: Reconfiguring daemon osd.6 on smithi155 2023-10-09T10:40:33.127 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:33 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:33.127 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:33 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:33.128 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:33 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T10:40:33.128 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:33 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:33.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:33 smithi125 ceph-mon[195507]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T10:40:33.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:33 smithi125 ceph-mon[195507]: Reconfiguring daemon osd.6 on smithi155 2023-10-09T10:40:33.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:33 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:33.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:33 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:33.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:33 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T10:40:33.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:33 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:33.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:33 smithi125 ceph-mon[200197]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T10:40:33.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:33 smithi125 ceph-mon[200197]: Reconfiguring daemon osd.6 on smithi155 2023-10-09T10:40:33.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:33 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:33.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:33 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:33.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:33 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T10:40:33.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:33 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:34 smithi125 ceph-mon[195507]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T10:40:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:34 smithi125 ceph-mon[195507]: Reconfiguring daemon osd.7 on smithi155 2023-10-09T10:40:34.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:34 smithi125 ceph-mon[195507]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:34.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:34 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:34.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:34 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:34.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:34 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.ydxmyq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T10:40:34.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:34 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:34.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:34 smithi125 ceph-mon[200197]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T10:40:34.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:34 smithi125 ceph-mon[200197]: Reconfiguring daemon osd.7 on smithi155 2023-10-09T10:40:34.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:34 smithi125 ceph-mon[200197]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:34.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:34 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:34.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:34 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:34.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:34 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.ydxmyq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T10:40:34.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:34 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:34.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:34 smithi155 ceph-mon[184854]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T10:40:34.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:34 smithi155 ceph-mon[184854]: Reconfiguring daemon osd.7 on smithi155 2023-10-09T10:40:34.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:34 smithi155 ceph-mon[184854]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:34 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:34 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:34 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.ydxmyq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T10:40:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:34 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:35.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[195507]: Reconfiguring mds.foo.smithi155.ydxmyq (monmap changed)... 2023-10-09T10:40:35.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[195507]: Reconfiguring daemon mds.foo.smithi155.ydxmyq on smithi155 2023-10-09T10:40:35.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:35.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:35.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:40:35.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:35.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:40:35.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:35.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[200197]: Reconfiguring mds.foo.smithi155.ydxmyq (monmap changed)... 2023-10-09T10:40:35.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[200197]: Reconfiguring daemon mds.foo.smithi155.ydxmyq on smithi155 2023-10-09T10:40:35.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:35.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:35.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:40:35.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:35.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:40:35.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:35.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:35 smithi155 ceph-mon[184854]: Reconfiguring mds.foo.smithi155.ydxmyq (monmap changed)... 2023-10-09T10:40:35.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:35 smithi155 ceph-mon[184854]: Reconfiguring daemon mds.foo.smithi155.ydxmyq on smithi155 2023-10-09T10:40:35.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:35.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:35.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:40:35.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:40:35.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:40:35.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:40:36.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:36 smithi125 ceph-mon[195507]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:36.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:36 smithi125 ceph-mon[200197]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:36.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:36 smithi155 ceph-mon[184854]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:38 smithi125 ceph-mon[195507]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:38 smithi125 ceph-mon[200197]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:38 smithi155 ceph-mon[184854]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:38.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:40:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:40:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:40:40.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:40 smithi125 ceph-mon[195507]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:40.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:40 smithi125 ceph-mon[200197]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:40 smithi155 ceph-mon[184854]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:41.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:40:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:40:41] "GET /metrics HTTP/1.1" 200 33858 "" "Prometheus/2.43.0" 2023-10-09T10:40:42.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:42 smithi125 ceph-mon[195507]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:42.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:42 smithi125 ceph-mon[200197]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:42 smithi155 ceph-mon[184854]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:43.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:43 smithi125 ceph-mon[195507]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:43.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:43 smithi125 ceph-mon[200197]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:43 smithi155 ceph-mon[184854]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:45.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:40:45.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:40:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:40:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:46 smithi125 ceph-mon[195507]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:46.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:46 smithi125 ceph-mon[200197]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:46 smithi155 ceph-mon[184854]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:48 smithi125 ceph-mon[195507]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:48 smithi125 ceph-mon[200197]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:48 smithi155 ceph-mon[184854]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:48.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:40:48 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:40:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:40:50.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:50 smithi125 ceph-mon[195507]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:50.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:50 smithi125 ceph-mon[200197]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:50 smithi155 ceph-mon[184854]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:51.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:40:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:40:51] "GET /metrics HTTP/1.1" 200 33857 "" "Prometheus/2.43.0" 2023-10-09T10:40:52.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:52 smithi125 ceph-mon[195507]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:52.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:52 smithi125 ceph-mon[200197]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:52 smithi155 ceph-mon[184854]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:53.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:53 smithi125 ceph-mon[195507]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:53.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:53 smithi125 ceph-mon[200197]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:53 smithi155 ceph-mon[184854]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:56.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:56 smithi125 ceph-mon[195507]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:56.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:56 smithi125 ceph-mon[200197]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:56.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:56 smithi155 ceph-mon[184854]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:58.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:40:58 smithi125 ceph-mon[195507]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:58.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:40:58 smithi125 ceph-mon[200197]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:58.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:40:58 smithi155 ceph-mon[184854]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:40:58.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:40:58 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:40:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:41:00.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:00 smithi125 ceph-mon[195507]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:00.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:41:00.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:00 smithi125 ceph-mon[200197]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:00.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:41:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:00 smithi155 ceph-mon[184854]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:41:01.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:41:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:41:01] "GET /metrics HTTP/1.1" 200 33797 "" "Prometheus/2.43.0" 2023-10-09T10:41:02.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:02 smithi125 ceph-mon[195507]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:02.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:02 smithi125 ceph-mon[200197]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:02 smithi155 ceph-mon[184854]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:03.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:03 smithi125 ceph-mon[195507]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:03 smithi125 ceph-mon[200197]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:03.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:03 smithi155 ceph-mon[184854]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:06.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:06 smithi125 ceph-mon[195507]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:06.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:06 smithi125 ceph-mon[200197]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:06 smithi155 ceph-mon[184854]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:08.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:08 smithi125 ceph-mon[195507]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:08.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:08 smithi125 ceph-mon[200197]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:08.358 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T10:41:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:08 smithi155 ceph-mon[184854]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:08.627 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:41:08.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:41:08 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:41:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:41:09.903 INFO:teuthology.orchestra.run.smithi125.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:41:10.293 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:10 smithi125 ceph-mon[195507]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:10.293 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:10 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:10.293 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:10 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:41:10.293 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:10 smithi125 ceph-mon[200197]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:10.294 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:10 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:10.294 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:10 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:41:10.316 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for all mons to upgrade' 2023-10-09T10:41:10.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:10 smithi155 ceph-mon[184854]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:10.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:10 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:10.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:10 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:41:10.873 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:41:11.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:11 smithi125 ceph-mon[195507]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:41:11.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:11 smithi125 ceph-mon[195507]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:41:11.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:41:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:41:11] "GET /metrics HTTP/1.1" 200 33797 "" "Prometheus/2.43.0" 2023-10-09T10:41:11.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:11 smithi125 ceph-mon[200197]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:41:11.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:11 smithi125 ceph-mon[200197]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:41:11.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:11 smithi155 ceph-mon[184854]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:41:11.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:11 smithi155 ceph-mon[184854]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:41:12.254 INFO:teuthology.orchestra.run.smithi125.stdout:wait for all mons to upgrade 2023-10-09T10:41:12.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:12 smithi125 ceph-mon[195507]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:12.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:12 smithi125 ceph-mon[200197]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:12.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:12 smithi155 ceph-mon[184854]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:12.793 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-10-09T10:41:13.220 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:41:13.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:13 smithi125 ceph-mon[195507]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:13.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:13 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:13.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:13 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:13.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:13 smithi125 ceph-mon[200197]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:13.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:13 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:13.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:13 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:13 smithi155 ceph-mon[184854]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:13 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:13 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:14.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:41:14.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:41:14.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:14.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:41:14.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:41:14.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:41:14.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: Upgrade: Setting container_image for all mon 2023-10-09T10:41:14.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:14.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T10:41:14.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T10:41:14.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T10:41:14.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T10:41:14.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T10:41:14.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T10:41:14.759 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T10:41:14.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:41:14.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: Upgrade: Setting container_image for all crash 2023-10-09T10:41:14.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:14.772 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T10:41:14.772 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T10:41:14.772 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[200197]: Upgrade: osd.0 is safe to restart 2023-10-09T10:41:14.772 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:41:14.772 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:41:14.772 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:14.772 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:41:14.772 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:41:14.772 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:41:14.773 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: Upgrade: Setting container_image for all mon 2023-10-09T10:41:14.773 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:14.773 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T10:41:14.773 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T10:41:14.773 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T10:41:14.773 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T10:41:14.773 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T10:41:14.773 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T10:41:14.773 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:41:14.773 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: Upgrade: Setting container_image for all crash 2023-10-09T10:41:14.773 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:14.773 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T10:41:14.774 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T10:41:14.774 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:14 smithi125 ceph-mon[195507]: Upgrade: osd.0 is safe to restart 2023-10-09T10:41:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:41:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:41:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:41:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:41:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:41:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: Upgrade: Setting container_image for all mon 2023-10-09T10:41:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T10:41:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T10:41:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T10:41:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T10:41:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T10:41:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T10:41:14.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:41:14.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: Upgrade: Setting container_image for all crash 2023-10-09T10:41:14.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:14.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T10:41:14.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T10:41:14.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:14 smithi155 ceph-mon[184854]: Upgrade: osd.0 is safe to restart 2023-10-09T10:41:15.256 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T10:41:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2861296891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:41:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[195507]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[195507]: Upgrade: Updating osd.0 2023-10-09T10:41:15.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:15.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T10:41:15.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:41:15.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[195507]: Deploying daemon osd.0 on smithi125 2023-10-09T10:41:15.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:41:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2861296891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:41:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[200197]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[200197]: Upgrade: Updating osd.0 2023-10-09T10:41:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T10:41:15.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:41:15.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[200197]: Deploying daemon osd.0 on smithi125 2023-10-09T10:41:15.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:41:15.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:15 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2861296891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T10:41:15.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:15 smithi155 ceph-mon[184854]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:15.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:15 smithi155 ceph-mon[184854]: Upgrade: Updating osd.0 2023-10-09T10:41:15.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:15.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T10:41:15.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:41:15.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:15 smithi155 ceph-mon[184854]: Deploying daemon osd.0 on smithi125 2023-10-09T10:41:15.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:41:15.983 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:41:17.570 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:17 smithi125 systemd[1]: Stopping Ceph osd.0 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:41:17.655 INFO:teuthology.orchestra.run.smithi125.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T10:41:17.655 INFO:teuthology.orchestra.run.smithi125.stdout:alertmanager.a smithi125 *:9093,9094 running (3m) - 5m 27.3M - 0.25.0 c8568f914cd2 3c13e6c17b7e 2023-10-09T10:41:17.655 INFO:teuthology.orchestra.run.smithi125.stdout:grafana.a smithi155 *:3000 running (3m) - 5m 87.6M - 9.4.7 2c41d148cca3 0ec8893d264d 2023-10-09T10:41:17.655 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi125.satgiw smithi125 running (5m) - 5m 17.2M - 17.2.6-1303-gb2903cb0 5517396c50d4 7b5d28288eab 2023-10-09T10:41:17.656 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi155.ydxmyq smithi155 running (5m) - 5m 15.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 c2c99605d324 2023-10-09T10:41:17.656 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.x smithi155 *:8443,9283,8765 running (2m) - 8m 448M - 18.2.0-598-gb4fa663f 3060897fdf97 67bb8d1af523 2023-10-09T10:41:17.656 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.y smithi125 *:8443,9283,8765 running (3m) - 10m 506M - 18.2.0-598-gb4fa663f 3060897fdf97 107edeccfff9 2023-10-09T10:41:17.656 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a smithi125 running (2m) - 10m 64.8M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 335012cc14fa 2023-10-09T10:41:17.656 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b smithi155 running (68s) - 8m 28.3M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 9b477d1850dc 2023-10-09T10:41:17.656 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c smithi125 running (78s) - 8m 50.4M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 c47abfeb6d97 2023-10-09T10:41:17.656 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.a smithi125 *:9100 running (3m) - 5m 17.2M - 1.5.0 0da6a335fe13 bf5a342348bd 2023-10-09T10:41:17.656 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.b smithi155 *:9100 running (3m) - 5m 9051k - 1.5.0 0da6a335fe13 1320bf6a6ac5 2023-10-09T10:41:17.656 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 smithi125 running (8m) - 8m 77.7M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 346135826df7 2023-10-09T10:41:17.656 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1 smithi125 running (8m) - 8m 83.5M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 11af9c8d1b19 2023-10-09T10:41:17.656 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2 smithi125 running (7m) - 7m 152M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 dc337188d937 2023-10-09T10:41:17.656 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3 smithi125 running (7m) - 7m 71.7M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 c92a4cc944fe 2023-10-09T10:41:17.657 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 smithi155 running (7m) - 7m 79.8M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 e3a61e2037a2 2023-10-09T10:41:17.657 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5 smithi155 running (6m) - 6m 96.8M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 12d4484e25b1 2023-10-09T10:41:17.657 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6 smithi155 running (6m) - 6m 67.3M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 c7b7ee1a3182 2023-10-09T10:41:17.657 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7 smithi155 running (6m) - 6m 103M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 8e225788b8f4 2023-10-09T10:41:17.657 INFO:teuthology.orchestra.run.smithi125.stdout:prometheus.a smithi155 *:9095 running (2m) - 6m 52.7M - 2.43.0 a07b618ecd1d 6412c6139964 2023-10-09T10:41:18.012 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:17 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-0[131685]: 2023-10-09T10:41:17.608+0000 7fbb07d86700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:41:18.013 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:17 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-0[131685]: 2023-10-09T10:41:17.608+0000 7fbb07d86700 -1 osd.0 76 *** Got signal Terminated *** 2023-10-09T10:41:18.013 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:17 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-0[131685]: 2023-10-09T10:41:17.608+0000 7fbb07d86700 -1 osd.0 76 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:41:18.055 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T10:41:18.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:18 smithi125 ceph-mon[195507]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:18.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:18 smithi125 ceph-mon[195507]: osd.0 marked itself down and dead 2023-10-09T10:41:18.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:18 smithi125 ceph-mon[200197]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:18.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:18 smithi125 ceph-mon[200197]: osd.0 marked itself down and dead 2023-10-09T10:41:18.321 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:41:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:18 smithi155 ceph-mon[184854]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:18 smithi155 ceph-mon[184854]: osd.0 marked itself down and dead 2023-10-09T10:41:18.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:41:18 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:41:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:41:19.320 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:19 smithi125 bash[208463]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T10:41:19.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:19 smithi125 ceph-mon[195507]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:41:19.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:19 smithi125 ceph-mon[195507]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:41:19.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:19 smithi125 ceph-mon[195507]: osdmap e77: 8 total, 7 up, 8 in 2023-10-09T10:41:19.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:19 smithi125 ceph-mon[200197]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:41:19.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:19 smithi125 ceph-mon[200197]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:41:19.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:19 smithi125 ceph-mon[200197]: osdmap e77: 8 total, 7 up, 8 in 2023-10-09T10:41:19.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:19 smithi155 ceph-mon[184854]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:41:19.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:19 smithi155 ceph-mon[184854]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:41:19.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:19 smithi155 ceph-mon[184854]: osdmap e77: 8 total, 7 up, 8 in 2023-10-09T10:41:20.158 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[195507]: failed to encode map e77 with expected crc 2023-10-09T10:41:20.158 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[195507]: failed to encode map e77 with expected crc 2023-10-09T10:41:20.158 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[195507]: pgmap v81: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:20.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[195507]: osdmap e78: 8 total, 7 up, 8 in 2023-10-09T10:41:20.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[195507]: failed to encode map e78 with expected crc 2023-10-09T10:41:20.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[195507]: failed to encode map e78 with expected crc 2023-10-09T10:41:20.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[195507]: failed to encode map e78 with expected crc 2023-10-09T10:41:20.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[195507]: failed to encode map e78 with expected crc 2023-10-09T10:41:20.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[195507]: failed to encode map e78 with expected crc 2023-10-09T10:41:20.159 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[200197]: failed to encode map e77 with expected crc 2023-10-09T10:41:20.159 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[200197]: failed to encode map e77 with expected crc 2023-10-09T10:41:20.159 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[200197]: pgmap v81: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:20.159 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[200197]: osdmap e78: 8 total, 7 up, 8 in 2023-10-09T10:41:20.160 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[200197]: failed to encode map e78 with expected crc 2023-10-09T10:41:20.160 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[200197]: failed to encode map e78 with expected crc 2023-10-09T10:41:20.160 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[200197]: failed to encode map e78 with expected crc 2023-10-09T10:41:20.160 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[200197]: failed to encode map e78 with expected crc 2023-10-09T10:41:20.160 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:20 smithi125 ceph-mon[200197]: failed to encode map e78 with expected crc 2023-10-09T10:41:20.300 INFO:teuthology.orchestra.run.smithi125.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:41:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:20 smithi155 ceph-mon[184854]: failed to encode map e77 with expected crc 2023-10-09T10:41:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:20 smithi155 ceph-mon[184854]: failed to encode map e77 with expected crc 2023-10-09T10:41:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:20 smithi155 ceph-mon[184854]: pgmap v81: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:20 smithi155 ceph-mon[184854]: osdmap e78: 8 total, 7 up, 8 in 2023-10-09T10:41:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:20 smithi155 ceph-mon[184854]: failed to encode map e78 with expected crc 2023-10-09T10:41:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:20 smithi155 ceph-mon[184854]: failed to encode map e78 with expected crc 2023-10-09T10:41:20.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:20 smithi155 ceph-mon[184854]: failed to encode map e78 with expected crc 2023-10-09T10:41:20.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:20 smithi155 ceph-mon[184854]: failed to encode map e78 with expected crc 2023-10-09T10:41:20.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:20 smithi155 ceph-mon[184854]: failed to encode map e78 with expected crc 2023-10-09T10:41:20.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T10:41:21.064 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:41:21.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:21 smithi125 ceph-mon[200197]: failed to encode map e78 with expected crc 2023-10-09T10:41:21.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:21 smithi125 ceph-mon[200197]: failed to encode map e78 with expected crc 2023-10-09T10:41:21.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:21 smithi125 ceph-mon[200197]: failed to encode map e78 with expected crc 2023-10-09T10:41:21.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:21 smithi125 ceph-mon[200197]: failed to encode map e78 with expected crc 2023-10-09T10:41:21.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:21 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:21.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:21 smithi125 ceph-mon[195507]: failed to encode map e78 with expected crc 2023-10-09T10:41:21.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:21 smithi125 ceph-mon[195507]: failed to encode map e78 with expected crc 2023-10-09T10:41:21.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:21 smithi125 ceph-mon[195507]: failed to encode map e78 with expected crc 2023-10-09T10:41:21.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:21 smithi125 ceph-mon[195507]: failed to encode map e78 with expected crc 2023-10-09T10:41:21.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:21 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:21.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:21 smithi155 ceph-mon[184854]: failed to encode map e78 with expected crc 2023-10-09T10:41:21.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:21 smithi155 ceph-mon[184854]: failed to encode map e78 with expected crc 2023-10-09T10:41:21.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:21 smithi155 ceph-mon[184854]: failed to encode map e78 with expected crc 2023-10-09T10:41:21.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:21 smithi155 ceph-mon[184854]: failed to encode map e78 with expected crc 2023-10-09T10:41:21.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:21 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:21.538 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:41:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:41:21] "GET /metrics HTTP/1.1" 200 33869 "" "Prometheus/2.43.0" 2023-10-09T10:41:21.538 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:21 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.0.service: Succeeded. 2023-10-09T10:41:21.538 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:21 smithi125 systemd[1]: Stopped Ceph osd.0 for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:41:21.538 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:21 smithi125 systemd[1]: Starting Ceph osd.0 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:41:22.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:22 smithi125 ceph-mon[195507]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:41:22.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:22 smithi125 ceph-mon[195507]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:41:22.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:22 smithi125 ceph-mon[195507]: pgmap v83: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T10:41:22.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:22 smithi125 ceph-mon[195507]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:41:22.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:22 smithi125 ceph-mon[200197]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:41:22.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:22 smithi125 ceph-mon[200197]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:41:22.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:22 smithi125 ceph-mon[200197]: pgmap v83: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T10:41:22.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:22 smithi125 ceph-mon[200197]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:41:22.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:22 smithi155 ceph-mon[184854]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:41:22.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:22 smithi155 ceph-mon[184854]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:41:22.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:22 smithi155 ceph-mon[184854]: pgmap v83: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T10:41:22.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:22 smithi155 ceph-mon[184854]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:41:22.820 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:22 smithi125 bash[209121]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T10:41:22.821 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:22 smithi125 bash[209121]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T10:41:22.821 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:22 smithi125 bash[209121]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-09T10:41:22.821 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:22 smithi125 bash[209121]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-09T10:41:22.821 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:22 smithi125 bash[209121]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-09T10:41:22.821 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:22 smithi125 bash[209121]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T10:41:22.821 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:22 smithi125 bash[209121]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T10:41:22.821 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:22 smithi125 bash[209121]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-09T10:41:23.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:23 smithi125 ceph-mon[195507]: pgmap v84: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T10:41:23.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:23.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:23.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:41:23.571 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:23 smithi125 ceph-osd[209441]: -- 172.21.15.125:0/3542871789 <== mon.2 v2:172.21.15.125:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c6b7aea4e0 con 0x55c6b7abb400 2023-10-09T10:41:23.571 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:23 smithi125 bash[209404]: 0ebace3b73a3a0546c6fa534744a3e9f2f20af8a7cd0b1f3c470a690968480a6 2023-10-09T10:41:23.571 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:23 smithi125 systemd[1]: Started Ceph osd.0 for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:41:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:23 smithi125 ceph-mon[200197]: pgmap v84: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T10:41:23.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:23.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:23.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:41:23.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:23 smithi155 ceph-mon[184854]: pgmap v84: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T10:41:23.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:23.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:23.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:41:24.070 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:23 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-0[209418]: 2023-10-09T10:41:23.810+0000 7f4ef98e4640 -1 Falling back to public interface 2023-10-09T10:41:26.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:26 smithi125 ceph-mon[200197]: pgmap v85: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T10:41:26.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:26 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:26.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:26 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:26.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:26 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:26.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:26 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:26.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:26 smithi125 ceph-mon[195507]: pgmap v85: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T10:41:26.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:26 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:26.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:26 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:26.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:26 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:26.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:26 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:26 smithi155 ceph-mon[184854]: pgmap v85: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T10:41:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:26 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:26 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:26 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:26.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:26 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:26.820 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:26 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-0[209418]: 2023-10-09T10:41:26.549+0000 7f4ef98e4640 -1 osd.0 76 log_to_monitors true 2023-10-09T10:41:27.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:27 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:27.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:27 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:27.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:27 smithi125 ceph-mon[195507]: from='osd.0 [v2:172.21.15.125:6802/2302743228,v1:172.21.15.125:6803/2302743228]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T10:41:27.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:27 smithi125 ceph-mon[195507]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T10:41:27.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:27 smithi125 ceph-mon[195507]: pgmap v86: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T10:41:27.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:27 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:27.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:27 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:27.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:27 smithi125 ceph-mon[200197]: from='osd.0 [v2:172.21.15.125:6802/2302743228,v1:172.21.15.125:6803/2302743228]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T10:41:27.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:27 smithi125 ceph-mon[200197]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T10:41:27.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:27 smithi125 ceph-mon[200197]: pgmap v86: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T10:41:27.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:27 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:27.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:27 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:27.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:27 smithi155 ceph-mon[184854]: from='osd.0 [v2:172.21.15.125:6802/2302743228,v1:172.21.15.125:6803/2302743228]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T10:41:27.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:27 smithi155 ceph-mon[184854]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T10:41:27.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:27 smithi155 ceph-mon[184854]: pgmap v86: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T10:41:27.820 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 10:41:27 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-0[209418]: 2023-10-09T10:41:27.490+0000 7f4ee8d4c700 -1 osd.0 76 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:41:28.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:28 smithi125 ceph-mon[195507]: failed to encode map e79 with expected crc 2023-10-09T10:41:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:28 smithi125 ceph-mon[195507]: failed to encode map e79 with expected crc 2023-10-09T10:41:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:28 smithi125 ceph-mon[195507]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T10:41:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:28 smithi125 ceph-mon[195507]: osdmap e79: 8 total, 7 up, 8 in 2023-10-09T10:41:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:28 smithi125 ceph-mon[195507]: from='osd.0 [v2:172.21.15.125:6802/2302743228,v1:172.21.15.125:6803/2302743228]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:41:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:28 smithi125 ceph-mon[195507]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:41:28.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:28 smithi125 ceph-mon[200197]: failed to encode map e79 with expected crc 2023-10-09T10:41:28.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:28 smithi125 ceph-mon[200197]: failed to encode map e79 with expected crc 2023-10-09T10:41:28.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:28 smithi125 ceph-mon[200197]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T10:41:28.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:28 smithi125 ceph-mon[200197]: osdmap e79: 8 total, 7 up, 8 in 2023-10-09T10:41:28.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:28 smithi125 ceph-mon[200197]: from='osd.0 [v2:172.21.15.125:6802/2302743228,v1:172.21.15.125:6803/2302743228]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:41:28.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:28 smithi125 ceph-mon[200197]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:41:28.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:28 smithi155 ceph-mon[184854]: failed to encode map e79 with expected crc 2023-10-09T10:41:28.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:28 smithi155 ceph-mon[184854]: failed to encode map e79 with expected crc 2023-10-09T10:41:28.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:28 smithi155 ceph-mon[184854]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T10:41:28.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:28 smithi155 ceph-mon[184854]: osdmap e79: 8 total, 7 up, 8 in 2023-10-09T10:41:28.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:28 smithi155 ceph-mon[184854]: from='osd.0 [v2:172.21.15.125:6802/2302743228,v1:172.21.15.125:6803/2302743228]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:41:28.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:28 smithi155 ceph-mon[184854]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:41:28.971 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:41:28 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:41:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:41:29.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: failed to encode map e79 with expected crc 2023-10-09T10:41:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:41:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: failed to encode map e80 with expected crc 2023-10-09T10:41:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: failed to encode map e80 with expected crc 2023-10-09T10:41:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: osd.0 [v2:172.21.15.125:6802/2302743228,v1:172.21.15.125:6803/2302743228] boot 2023-10-09T10:41:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T10:41:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:41:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:41:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:41:29.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:29.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: pgmap v89: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T10:41:29.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:41:29.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:41:29.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:41:29.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:29.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: failed to encode map e79 with expected crc 2023-10-09T10:41:29.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:41:29.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: failed to encode map e80 with expected crc 2023-10-09T10:41:29.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: failed to encode map e80 with expected crc 2023-10-09T10:41:29.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: osd.0 [v2:172.21.15.125:6802/2302743228,v1:172.21.15.125:6803/2302743228] boot 2023-10-09T10:41:29.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T10:41:29.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:41:29.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:29.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:29.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:41:29.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:41:29.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:29.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: pgmap v89: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T10:41:29.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:41:29.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:41:29.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:41:29.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: failed to encode map e79 with expected crc 2023-10-09T10:41:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:41:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: failed to encode map e80 with expected crc 2023-10-09T10:41:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: failed to encode map e80 with expected crc 2023-10-09T10:41:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: osd.0 [v2:172.21.15.125:6802/2302743228,v1:172.21.15.125:6803/2302743228] boot 2023-10-09T10:41:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T10:41:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T10:41:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:41:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:41:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: pgmap v89: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T10:41:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:41:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:41:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:41:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:30.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[195507]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[195507]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[195507]: osdmap e81: 8 total, 8 up, 8 in 2023-10-09T10:41:30.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[195507]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[195507]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[195507]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[195507]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[195507]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[195507]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[195507]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[195507]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:30.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:41:30.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[200197]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[200197]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[200197]: osdmap e81: 8 total, 8 up, 8 in 2023-10-09T10:41:30.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[200197]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[200197]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[200197]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[200197]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[200197]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[200197]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[200197]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[200197]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:30.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:41:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:30 smithi155 ceph-mon[184854]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:30 smithi155 ceph-mon[184854]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:30 smithi155 ceph-mon[184854]: osdmap e81: 8 total, 8 up, 8 in 2023-10-09T10:41:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:30 smithi155 ceph-mon[184854]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:30 smithi155 ceph-mon[184854]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:30 smithi155 ceph-mon[184854]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:30 smithi155 ceph-mon[184854]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:30 smithi155 ceph-mon[184854]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:30 smithi155 ceph-mon[184854]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:30 smithi155 ceph-mon[184854]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:30 smithi155 ceph-mon[184854]: failed to encode map e81 with expected crc 2023-10-09T10:41:30.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:30.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:41:31.500 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:41:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:41:31] "GET /metrics HTTP/1.1" 200 33784 "" "Prometheus/2.43.0" 2023-10-09T10:41:31.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:31 smithi125 ceph-mon[195507]: pgmap v91: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T10:41:31.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:31 smithi125 ceph-mon[200197]: pgmap v91: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T10:41:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:31 smithi155 ceph-mon[184854]: pgmap v91: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T10:41:32.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:32 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:41:32.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:32 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:41:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:32 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:41:33.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:33 smithi125 ceph-mon[195507]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:33.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:33 smithi125 ceph-mon[200197]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:33 smithi155 ceph-mon[184854]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:34.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:34 smithi125 ceph-mon[200197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-10-09T10:41:34.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:34 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T10:41:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:34 smithi125 ceph-mon[195507]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-10-09T10:41:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:34 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T10:41:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:34 smithi155 ceph-mon[184854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-10-09T10:41:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:34 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T10:41:35.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:35 smithi125 ceph-mon[195507]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:35.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:35 smithi125 ceph-mon[200197]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:35 smithi155 ceph-mon[184854]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:38 smithi125 ceph-mon[195507]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:38 smithi125 ceph-mon[200197]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:38 smithi155 ceph-mon[184854]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:38.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:41:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:41:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:41:40.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:40 smithi125 ceph-mon[195507]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:40.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:40 smithi125 ceph-mon[200197]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:40 smithi155 ceph-mon[184854]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:41.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:41 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:41.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:41 smithi125 ceph-mon[200197]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:41.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:41 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:41.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:41 smithi125 ceph-mon[195507]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:41.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:41:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:41:41] "GET /metrics HTTP/1.1" 200 33784 "" "Prometheus/2.43.0" 2023-10-09T10:41:41.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:41 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:41.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:41 smithi155 ceph-mon[184854]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:43.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:43 smithi125 ceph-mon[195507]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:43.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:43 smithi125 ceph-mon[200197]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:43 smithi155 ceph-mon[184854]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:45 smithi125 ceph-mon[195507]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:41:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:45 smithi125 ceph-mon[200197]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:41:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:45 smithi155 ceph-mon[184854]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:41:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:41:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:47 smithi125 ceph-mon[195507]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:48.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:47 smithi125 ceph-mon[200197]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:47 smithi155 ceph-mon[184854]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:48.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:41:48 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:41:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:41:50.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:49 smithi125 ceph-mon[195507]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:50.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:50 smithi125 ceph-mon[200197]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:50 smithi155 ceph-mon[184854]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:51.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:41:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:41:51] "GET /metrics HTTP/1.1" 200 33932 "" "Prometheus/2.43.0" 2023-10-09T10:41:52.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:52 smithi125 ceph-mon[195507]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:52.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:52 smithi125 ceph-mon[200197]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:52 smithi155 ceph-mon[184854]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:53.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:53 smithi125 ceph-mon[195507]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:53.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:53 smithi125 ceph-mon[200197]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:53 smithi155 ceph-mon[184854]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:56.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:56 smithi125 ceph-mon[195507]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:56.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:56 smithi125 ceph-mon[200197]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:56.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:56 smithi155 ceph-mon[184854]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:58.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:41:58 smithi125 ceph-mon[195507]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:58.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:41:58 smithi125 ceph-mon[200197]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:58.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:41:58 smithi155 ceph-mon[184854]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:41:58.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:41:58 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:41:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:42:00.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:00 smithi125 ceph-mon[195507]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:00.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:42:00.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:00 smithi125 ceph-mon[200197]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:00.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:42:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:00 smithi155 ceph-mon[184854]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:42:01.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:42:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:42:01] "GET /metrics HTTP/1.1" 200 33928 "" "Prometheus/2.43.0" 2023-10-09T10:42:02.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:02 smithi125 ceph-mon[195507]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:02.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:02 smithi125 ceph-mon[200197]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:02 smithi155 ceph-mon[184854]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:03.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:03 smithi125 ceph-mon[195507]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:03.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:03 smithi125 ceph-mon[200197]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:03.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:03 smithi155 ceph-mon[184854]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:06.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:06 smithi125 ceph-mon[195507]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:06.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:06 smithi125 ceph-mon[200197]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:06 smithi155 ceph-mon[184854]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:08.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:08 smithi125 ceph-mon[195507]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:08.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:08 smithi125 ceph-mon[200197]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:08 smithi155 ceph-mon[184854]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:08.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:42:08 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:42:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:42:10.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:10 smithi125 ceph-mon[195507]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:10.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:10 smithi125 ceph-mon[200197]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:10.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:10 smithi155 ceph-mon[184854]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:11.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:42:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:42:11] "GET /metrics HTTP/1.1" 200 33928 "" "Prometheus/2.43.0" 2023-10-09T10:42:12.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:12 smithi125 ceph-mon[195507]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:12.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:12 smithi125 ceph-mon[200197]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:12.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:12 smithi155 ceph-mon[184854]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:13.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:13 smithi125 ceph-mon[195507]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:13.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:13 smithi125 ceph-mon[200197]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:13 smithi155 ceph-mon[184854]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:15.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:42:15.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:42:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:42:16.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:16 smithi125 ceph-mon[195507]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:16.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:16 smithi125 ceph-mon[200197]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:16.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:16 smithi155 ceph-mon[184854]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:18.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:18 smithi125 ceph-mon[195507]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:18.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:18 smithi125 ceph-mon[200197]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:18 smithi155 ceph-mon[184854]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:18.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:42:18 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:42:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:42:20.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:20 smithi125 ceph-mon[195507]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:20.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:20 smithi125 ceph-mon[200197]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:20 smithi155 ceph-mon[184854]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:21.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:42:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:42:21] "GET /metrics HTTP/1.1" 200 33930 "" "Prometheus/2.43.0" 2023-10-09T10:42:22.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:22 smithi125 ceph-mon[195507]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:22.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:22 smithi125 ceph-mon[200197]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:22.447 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T10:42:22.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:22 smithi155 ceph-mon[184854]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:22.727 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:42:23.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:23 smithi125 ceph-mon[195507]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:23.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:23 smithi125 ceph-mon[200197]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:23.937 INFO:teuthology.orchestra.run.smithi125.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:42:23.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:23 smithi155 ceph-mon[184854]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:24.348 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for half of osds to upgrade' 2023-10-09T10:42:24.635 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:42:25.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:42:25.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:42:25.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:42:25.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:42:25.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:42:25.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:42:25.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:25.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:25.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:25.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T10:42:25.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:42:25.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:42:25.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:42:25.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:42:25.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:42:25.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:42:25.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:25.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:25.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:25.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T10:42:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:42:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:42:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:42:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:42:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:42:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:42:25.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:25.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:25.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:25.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T10:42:25.987 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:25 smithi125 ceph-mon[200197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T10:42:25.987 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:25 smithi125 ceph-mon[200197]: Upgrade: osd.1 is safe to restart 2023-10-09T10:42:25.987 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:25 smithi125 ceph-mon[200197]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:25.987 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:25 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.987 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:25 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T10:42:25.987 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:25 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:42:25.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:25 smithi125 ceph-mon[195507]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T10:42:25.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:25 smithi125 ceph-mon[195507]: Upgrade: osd.1 is safe to restart 2023-10-09T10:42:25.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:25 smithi125 ceph-mon[195507]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:25.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:25 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:25.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:25 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T10:42:25.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:25 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:42:26.077 INFO:teuthology.orchestra.run.smithi125.stdout:wait for half of osds to upgrade 2023-10-09T10:42:26.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:25 smithi155 ceph-mon[184854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T10:42:26.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:25 smithi155 ceph-mon[184854]: Upgrade: osd.1 is safe to restart 2023-10-09T10:42:26.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:25 smithi155 ceph-mon[184854]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:26.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:25 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:26.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:25 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T10:42:26.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:25 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:42:26.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-10-09T10:42:27.088 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:42:27.206 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:26 smithi125 ceph-mon[195507]: Upgrade: Updating osd.1 2023-10-09T10:42:27.206 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:26 smithi125 ceph-mon[195507]: Deploying daemon osd.1 on smithi125 2023-10-09T10:42:27.207 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:26 smithi125 ceph-mon[200197]: Upgrade: Updating osd.1 2023-10-09T10:42:27.207 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:26 smithi125 ceph-mon[200197]: Deploying daemon osd.1 on smithi125 2023-10-09T10:42:27.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:26 smithi155 ceph-mon[184854]: Upgrade: Updating osd.1 2023-10-09T10:42:27.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:26 smithi155 ceph-mon[184854]: Deploying daemon osd.1 on smithi125 2023-10-09T10:42:28.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:27 smithi125 ceph-mon[195507]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:28.046 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:27 smithi125 ceph-mon[200197]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:28.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:27 smithi155 ceph-mon[184854]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:28.948 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:28 smithi125 systemd[1]: Stopping Ceph osd.1 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:42:28.948 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:28 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[138711]: 2023-10-09T10:42:28.882+0000 7f9c9e154700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:42:28.949 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:28 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[138711]: 2023-10-09T10:42:28.882+0000 7f9c9e154700 -1 osd.1 81 *** Got signal Terminated *** 2023-10-09T10:42:28.949 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:28 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[138711]: 2023-10-09T10:42:28.882+0000 7f9c9e154700 -1 osd.1 81 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:42:28.949 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:28 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2968751344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:28.950 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:28 smithi125 ceph-mon[195507]: osd.1 marked itself down and dead 2023-10-09T10:42:28.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:28 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2968751344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:28.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:28 smithi125 ceph-mon[200197]: osd.1 marked itself down and dead 2023-10-09T10:42:28.953 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:42:28 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:42:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:42:28.954 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:28 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2968751344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:28.954 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:28 smithi155 ceph-mon[184854]: osd.1 marked itself down and dead 2023-10-09T10:42:29.820 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:29 smithi125 bash[215440]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T10:42:30.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:29 smithi155 ceph-mon[184854]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:42:30.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:29 smithi155 ceph-mon[184854]: failed to encode map e82 with expected crc 2023-10-09T10:42:30.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:29 smithi155 ceph-mon[184854]: failed to encode map e82 with expected crc 2023-10-09T10:42:30.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:29 smithi155 ceph-mon[184854]: osdmap e82: 8 total, 7 up, 8 in 2023-10-09T10:42:30.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:29 smithi155 ceph-mon[184854]: pgmap v121: 73 pgs: 10 stale+active+clean, 63 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:30.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:29 smithi155 ceph-mon[184854]: osdmap e83: 8 total, 7 up, 8 in 2023-10-09T10:42:30.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:29 smithi125 ceph-mon[195507]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:42:30.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:29 smithi125 ceph-mon[195507]: failed to encode map e82 with expected crc 2023-10-09T10:42:30.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:29 smithi125 ceph-mon[195507]: failed to encode map e82 with expected crc 2023-10-09T10:42:30.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:29 smithi125 ceph-mon[195507]: osdmap e82: 8 total, 7 up, 8 in 2023-10-09T10:42:30.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:29 smithi125 ceph-mon[195507]: pgmap v121: 73 pgs: 10 stale+active+clean, 63 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:30.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:29 smithi125 ceph-mon[195507]: osdmap e83: 8 total, 7 up, 8 in 2023-10-09T10:42:30.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:29 smithi125 ceph-mon[200197]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:42:30.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:29 smithi125 ceph-mon[200197]: failed to encode map e82 with expected crc 2023-10-09T10:42:30.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:29 smithi125 ceph-mon[200197]: failed to encode map e82 with expected crc 2023-10-09T10:42:30.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:29 smithi125 ceph-mon[200197]: osdmap e82: 8 total, 7 up, 8 in 2023-10-09T10:42:30.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:29 smithi125 ceph-mon[200197]: pgmap v121: 73 pgs: 10 stale+active+clean, 63 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:30.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:29 smithi125 ceph-mon[200197]: osdmap e83: 8 total, 7 up, 8 in 2023-10-09T10:42:30.958 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:30 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.1.service: Succeeded. 2023-10-09T10:42:30.959 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:30 smithi125 systemd[1]: Stopped Ceph osd.1 for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:42:30.959 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:30 smithi125 systemd[1]: Starting Ceph osd.1 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:42:30.960 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[195507]: failed to encode map e83 with expected crc 2023-10-09T10:42:30.960 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[195507]: failed to encode map e83 with expected crc 2023-10-09T10:42:30.960 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[195507]: failed to encode map e83 with expected crc 2023-10-09T10:42:30.960 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[195507]: failed to encode map e83 with expected crc 2023-10-09T10:42:30.960 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[195507]: failed to encode map e83 with expected crc 2023-10-09T10:42:30.960 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[195507]: failed to encode map e83 with expected crc 2023-10-09T10:42:30.960 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[195507]: failed to encode map e83 with expected crc 2023-10-09T10:42:30.961 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[195507]: failed to encode map e83 with expected crc 2023-10-09T10:42:30.961 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:30.961 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:42:31.214 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[200197]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[200197]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[200197]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[200197]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[200197]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[200197]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[200197]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[200197]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:31.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:42:31.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:30 smithi155 ceph-mon[184854]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:30 smithi155 ceph-mon[184854]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:30 smithi155 ceph-mon[184854]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:30 smithi155 ceph-mon[184854]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:30 smithi155 ceph-mon[184854]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:30 smithi155 ceph-mon[184854]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:30 smithi155 ceph-mon[184854]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:30 smithi155 ceph-mon[184854]: failed to encode map e83 with expected crc 2023-10-09T10:42:31.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:31.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:42:31.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:42:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:42:31] "GET /metrics HTTP/1.1" 200 33991 "" "Prometheus/2.43.0" 2023-10-09T10:42:31.963 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:31 smithi125 bash[215730]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T10:42:31.963 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:31 smithi125 bash[215730]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T10:42:31.963 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:31 smithi125 bash[215730]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-09T10:42:31.963 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:31 smithi125 bash[215730]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-09T10:42:31.963 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:31 smithi125 bash[215730]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-09T10:42:31.963 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:31 smithi125 bash[215730]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T10:42:31.964 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:31 smithi125 bash[215730]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T10:42:31.964 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:31 smithi125 bash[215730]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-09T10:42:32.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:31 smithi155 ceph-mon[184854]: pgmap v123: 73 pgs: 2 active+undersized+degraded, 1 stale+active+clean, 27 active+undersized, 43 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T10:42:32.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/747328401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:32.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:31 smithi125 ceph-mon[195507]: pgmap v123: 73 pgs: 2 active+undersized+degraded, 1 stale+active+clean, 27 active+undersized, 43 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T10:42:32.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/747328401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:32.285 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:31 smithi125 ceph-mon[200197]: pgmap v123: 73 pgs: 2 active+undersized+degraded, 1 stale+active+clean, 27 active+undersized, 43 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T10:42:32.286 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/747328401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:32.570 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:32 smithi125 ceph-osd[216037]: -- 172.21.15.125:0/2467934613 <== mon.0 v2:172.21.15.125:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564c4992c4e0 con 0x564c498fcc00 2023-10-09T10:42:32.570 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:32 smithi125 bash[215999]: 743f3b6c2116cb779abc0e16d353626ca9fa7c956fee3af00a525e825a9ed942 2023-10-09T10:42:32.570 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:32 smithi125 systemd[1]: Started Ceph osd.1 for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:42:33.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:32 smithi155 ceph-mon[184854]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:42:33.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:32 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:33.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:32 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:33.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:32 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:42:33.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:32 smithi125 ceph-mon[195507]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:42:33.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:32 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:33.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:32 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:33.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:32 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:42:33.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:32 smithi125 ceph-mon[200197]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:42:33.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:32 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:33.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:32 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:33.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:32 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:42:33.322 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:32 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[216014]: 2023-10-09T10:42:32.888+0000 7fe07192d640 -1 Falling back to public interface 2023-10-09T10:42:34.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:33 smithi155 ceph-mon[184854]: pgmap v124: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T10:42:34.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:33 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2360069609' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:33 smithi125 ceph-mon[195507]: pgmap v124: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T10:42:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:33 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2360069609' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:34.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:33 smithi125 ceph-mon[200197]: pgmap v124: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T10:42:34.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:33 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2360069609' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:35.892 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[216014]: 2023-10-09T10:42:35.630+0000 7fe07192d640 -1 osd.1 81 log_to_monitors true 2023-10-09T10:42:35.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:35.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:35.893 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:35.893 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:35.893 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[195507]: pgmap v125: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T10:42:35.893 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:35.893 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:35.893 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[195507]: from='osd.1 [v2:172.21.15.125:6810/1934730282,v1:172.21.15.125:6811/1934730282]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T10:42:35.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:35.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:35.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:35.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:35.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[200197]: pgmap v125: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T10:42:35.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:35.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:35.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:35 smithi125 ceph-mon[200197]: from='osd.1 [v2:172.21.15.125:6810/1934730282,v1:172.21.15.125:6811/1934730282]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T10:42:36.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:36.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:36.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:35 smithi155 ceph-mon[184854]: pgmap v125: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T10:42:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:35 smithi155 ceph-mon[184854]: from='osd.1 [v2:172.21.15.125:6810/1934730282,v1:172.21.15.125:6811/1934730282]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T10:42:36.610 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:42:36 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[216014]: 2023-10-09T10:42:36.415+0000 7fe062d99700 -1 osd.1 81 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:42:37.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:36 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2738934044' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:37.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:36 smithi125 ceph-mon[195507]: from='osd.1 [v2:172.21.15.125:6810/1934730282,v1:172.21.15.125:6811/1934730282]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T10:42:37.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:36 smithi125 ceph-mon[195507]: osdmap e84: 8 total, 7 up, 8 in 2023-10-09T10:42:37.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:36 smithi125 ceph-mon[195507]: from='osd.1 [v2:172.21.15.125:6810/1934730282,v1:172.21.15.125:6811/1934730282]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:42:37.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:36 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2738934044' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:37.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:36 smithi125 ceph-mon[200197]: from='osd.1 [v2:172.21.15.125:6810/1934730282,v1:172.21.15.125:6811/1934730282]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T10:42:37.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:36 smithi125 ceph-mon[200197]: osdmap e84: 8 total, 7 up, 8 in 2023-10-09T10:42:37.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:36 smithi125 ceph-mon[200197]: from='osd.1 [v2:172.21.15.125:6810/1934730282,v1:172.21.15.125:6811/1934730282]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:42:37.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:36 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2738934044' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:37.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:36 smithi155 ceph-mon[184854]: from='osd.1 [v2:172.21.15.125:6810/1934730282,v1:172.21.15.125:6811/1934730282]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T10:42:37.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:36 smithi155 ceph-mon[184854]: osdmap e84: 8 total, 7 up, 8 in 2023-10-09T10:42:37.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:36 smithi155 ceph-mon[184854]: from='osd.1 [v2:172.21.15.125:6810/1934730282,v1:172.21.15.125:6811/1934730282]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:42:37.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[200197]: failed to encode map e84 with expected crc 2023-10-09T10:42:37.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[200197]: failed to encode map e84 with expected crc 2023-10-09T10:42:37.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[200197]: pgmap v127: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T10:42:37.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[200197]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:42:37.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[200197]: osd.1 [v2:172.21.15.125:6810/1934730282,v1:172.21.15.125:6811/1934730282] boot 2023-10-09T10:42:37.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[200197]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T10:42:37.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:42:37.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:42:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[195507]: failed to encode map e84 with expected crc 2023-10-09T10:42:37.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[195507]: failed to encode map e84 with expected crc 2023-10-09T10:42:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[195507]: pgmap v127: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T10:42:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[195507]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:42:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[195507]: osd.1 [v2:172.21.15.125:6810/1934730282,v1:172.21.15.125:6811/1934730282] boot 2023-10-09T10:42:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[195507]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T10:42:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:42:37.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:37 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:42:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:37 smithi155 ceph-mon[184854]: failed to encode map e84 with expected crc 2023-10-09T10:42:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:37 smithi155 ceph-mon[184854]: failed to encode map e84 with expected crc 2023-10-09T10:42:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:37 smithi155 ceph-mon[184854]: pgmap v127: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T10:42:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:37 smithi155 ceph-mon[184854]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:42:38.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:37 smithi155 ceph-mon[184854]: osd.1 [v2:172.21.15.125:6810/1934730282,v1:172.21.15.125:6811/1934730282] boot 2023-10-09T10:42:38.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:37 smithi155 ceph-mon[184854]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T10:42:38.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:37 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T10:42:38.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:37 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:42:38.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:42:38.119+0000 7fa537ed4700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T10:42:38.968 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:42:38.660Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2023-10-09T10:42:38.968 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:42:38.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:42:39.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: failed to encode map e85 with expected crc 2023-10-09T10:42:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:42:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:42:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:42:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T10:42:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T10:42:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T10:42:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2477577322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: failed to encode map e86 with expected crc 2023-10-09T10:42:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: osdmap e86: 8 total, 8 up, 8 in 2023-10-09T10:42:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: failed to encode map e86 with expected crc 2023-10-09T10:42:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: failed to encode map e86 with expected crc 2023-10-09T10:42:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: failed to encode map e86 with expected crc 2023-10-09T10:42:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:38 smithi155 ceph-mon[184854]: failed to encode map e86 with expected crc 2023-10-09T10:42:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: failed to encode map e85 with expected crc 2023-10-09T10:42:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:42:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:42:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:42:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T10:42:39.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T10:42:39.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T10:42:39.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2477577322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:39.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: failed to encode map e86 with expected crc 2023-10-09T10:42:39.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: osdmap e86: 8 total, 8 up, 8 in 2023-10-09T10:42:39.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: failed to encode map e86 with expected crc 2023-10-09T10:42:39.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: failed to encode map e86 with expected crc 2023-10-09T10:42:39.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: failed to encode map e86 with expected crc 2023-10-09T10:42:39.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[195507]: failed to encode map e86 with expected crc 2023-10-09T10:42:39.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: failed to encode map e85 with expected crc 2023-10-09T10:42:39.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:39.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:39.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:42:39.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:42:39.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:39.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:42:39.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:39.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:39.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:42:39.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T10:42:39.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T10:42:39.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T10:42:39.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2477577322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:39.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: failed to encode map e86 with expected crc 2023-10-09T10:42:39.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: osdmap e86: 8 total, 8 up, 8 in 2023-10-09T10:42:39.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: failed to encode map e86 with expected crc 2023-10-09T10:42:39.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: failed to encode map e86 with expected crc 2023-10-09T10:42:39.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: failed to encode map e86 with expected crc 2023-10-09T10:42:39.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:38 smithi125 ceph-mon[200197]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[195507]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[195507]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[195507]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[195507]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[195507]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[195507]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[195507]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[195507]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[195507]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[195507]: pgmap v130: 73 pgs: 2 peering, 2 active+undersized+degraded, 28 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T10:42:40.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[200197]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[200197]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[200197]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[200197]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[200197]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[200197]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[200197]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[200197]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[200197]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:39 smithi125 ceph-mon[200197]: pgmap v130: 73 pgs: 2 peering, 2 active+undersized+degraded, 28 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T10:42:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:39 smithi155 ceph-mon[184854]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:39 smithi155 ceph-mon[184854]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:39 smithi155 ceph-mon[184854]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:39 smithi155 ceph-mon[184854]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:39 smithi155 ceph-mon[184854]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:39 smithi155 ceph-mon[184854]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:39 smithi155 ceph-mon[184854]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:39 smithi155 ceph-mon[184854]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:39 smithi155 ceph-mon[184854]: failed to encode map e86 with expected crc 2023-10-09T10:42:40.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:39 smithi155 ceph-mon[184854]: pgmap v130: 73 pgs: 2 peering, 2 active+undersized+degraded, 28 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T10:42:41.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:40 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2350886546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:40 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2350886546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:41.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:42:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:42:41] "GET /metrics HTTP/1.1" 200 33991 "" "Prometheus/2.43.0" 2023-10-09T10:42:41.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:40 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2350886546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:42.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:41 smithi125 ceph-mon[195507]: pgmap v131: 73 pgs: 2 peering, 10 active+undersized, 61 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:42.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:41 smithi125 ceph-mon[200197]: pgmap v131: 73 pgs: 2 peering, 10 active+undersized, 61 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:41 smithi155 ceph-mon[184854]: pgmap v131: 73 pgs: 2 peering, 10 active+undersized, 61 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:43.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:42 smithi125 ceph-mon[200197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T10:42:43.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:42 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T10:42:43.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:42 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2296302976' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:42 smithi125 ceph-mon[195507]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T10:42:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:42 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T10:42:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:42 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2296302976' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:42 smithi155 ceph-mon[184854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T10:42:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:42 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T10:42:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:42 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2296302976' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:44.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:43 smithi125 ceph-mon[195507]: pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:44.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:43 smithi125 ceph-mon[200197]: pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:44.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:43 smithi155 ceph-mon[184854]: pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:45 smithi125 ceph-mon[195507]: pgmap v133: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:42:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:45 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/370785856' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:45 smithi125 ceph-mon[200197]: pgmap v133: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:42:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:45 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/370785856' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:45 smithi155 ceph-mon[184854]: pgmap v133: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:46.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:42:46.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:45 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/370785856' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:47 smithi125 ceph-mon[195507]: pgmap v134: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:48.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/749140374' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:47 smithi125 ceph-mon[200197]: pgmap v134: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/749140374' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:48 smithi155 ceph-mon[184854]: pgmap v134: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:48 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/749140374' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:48.970 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:42:48 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:42:48.659Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2023-10-09T10:42:48.971 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 10:42:48 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T10:42:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T10:42:50.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:50 smithi125 ceph-mon[195507]: pgmap v135: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:50.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:50 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/301921122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:50.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:50 smithi125 ceph-mon[200197]: pgmap v135: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:50.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:50 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/301921122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:50 smithi155 ceph-mon[184854]: pgmap v135: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:50 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/301921122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:51.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:42:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:42:51] "GET /metrics HTTP/1.1" 200 33986 "" "Prometheus/2.43.0" 2023-10-09T10:42:51.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:51 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:51.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:51 smithi125 ceph-mon[195507]: pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:51.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:51 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:51.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:51 smithi125 ceph-mon[200197]: pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:51.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:51 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:51.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:51 smithi155 ceph-mon[184854]: pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:52.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:52 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2673282683' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:52.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:52 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2673282683' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:52.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:52 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2673282683' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:53.781 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:53 smithi125 ceph-mon[195507]: pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:53.781 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:53 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T10:42:53.781 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:53 smithi125 ceph-mon[195507]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T10:42:53.781 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:53 smithi125 ceph-mon[195507]: Upgrade: osd.2 is safe to restart 2023-10-09T10:42:53.781 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:53 smithi125 ceph-mon[200197]: pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:53.781 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:53 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T10:42:53.781 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:53 smithi125 ceph-mon[200197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T10:42:53.781 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:53 smithi125 ceph-mon[200197]: Upgrade: osd.2 is safe to restart 2023-10-09T10:42:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:53 smithi155 ceph-mon[184854]: pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:53 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T10:42:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:53 smithi155 ceph-mon[184854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T10:42:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:53 smithi155 ceph-mon[184854]: Upgrade: osd.2 is safe to restart 2023-10-09T10:42:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:55 smithi155 ceph-mon[184854]: Upgrade: Updating osd.2 2023-10-09T10:42:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:55 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:55.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:55 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T10:42:55.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:55 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:42:55.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:55 smithi155 ceph-mon[184854]: Deploying daemon osd.2 on smithi125 2023-10-09T10:42:55.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:55 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/471789164' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:55.568 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:55 smithi125 ceph-mon[195507]: Upgrade: Updating osd.2 2023-10-09T10:42:55.568 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:55 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:55.568 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:55 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T10:42:55.568 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:55 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:42:55.568 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:55 smithi125 ceph-mon[195507]: Deploying daemon osd.2 on smithi125 2023-10-09T10:42:55.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:55 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/471789164' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:55.569 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:55 smithi125 ceph-mon[200197]: Upgrade: Updating osd.2 2023-10-09T10:42:55.569 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:55 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:42:55.569 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:55 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T10:42:55.569 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:55 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:42:55.569 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:55 smithi125 ceph-mon[200197]: Deploying daemon osd.2 on smithi125 2023-10-09T10:42:55.569 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:55 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/471789164' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:56.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:56 smithi155 ceph-mon[184854]: pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:56.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:56 smithi125 ceph-mon[195507]: pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:56 smithi125 ceph-mon[200197]: pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:56.571 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:56 smithi125 systemd[1]: Stopping Ceph osd.2 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:42:56.571 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:56 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-2[146683]: 2023-10-09T10:42:56.378+0000 7fa02d101700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:42:56.571 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:56 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-2[146683]: 2023-10-09T10:42:56.378+0000 7fa02d101700 -1 osd.2 86 *** Got signal Terminated *** 2023-10-09T10:42:56.571 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:56 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-2[146683]: 2023-10-09T10:42:56.378+0000 7fa02d101700 -1 osd.2 86 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:42:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:57 smithi155 ceph-mon[184854]: osd.2 marked itself down and dead 2023-10-09T10:42:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:57 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1889309799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:57.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:57 smithi125 ceph-mon[195507]: osd.2 marked itself down and dead 2023-10-09T10:42:57.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:57 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1889309799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:57.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:57 smithi125 ceph-mon[200197]: osd.2 marked itself down and dead 2023-10-09T10:42:57.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:57 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1889309799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:58.070 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:57 smithi125 bash[222216]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T10:42:58.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:58 smithi155 ceph-mon[184854]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:58.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:58 smithi155 ceph-mon[184854]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:42:58.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:58 smithi155 ceph-mon[184854]: failed to encode map e87 with expected crc 2023-10-09T10:42:58.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:58 smithi155 ceph-mon[184854]: osdmap e87: 8 total, 7 up, 8 in 2023-10-09T10:42:58.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:58 smithi155 ceph-mon[184854]: failed to encode map e87 with expected crc 2023-10-09T10:42:58.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:58 smithi155 ceph-mon[184854]: failed to encode map e87 with expected crc 2023-10-09T10:42:58.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:58 smithi155 ceph-mon[184854]: failed to encode map e87 with expected crc 2023-10-09T10:42:58.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:58 smithi155 ceph-mon[184854]: failed to encode map e87 with expected crc 2023-10-09T10:42:58.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[195507]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:58.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[195507]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:42:58.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[195507]: failed to encode map e87 with expected crc 2023-10-09T10:42:58.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[195507]: osdmap e87: 8 total, 7 up, 8 in 2023-10-09T10:42:58.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[195507]: failed to encode map e87 with expected crc 2023-10-09T10:42:58.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[195507]: failed to encode map e87 with expected crc 2023-10-09T10:42:58.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[195507]: failed to encode map e87 with expected crc 2023-10-09T10:42:58.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[195507]: failed to encode map e87 with expected crc 2023-10-09T10:42:58.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[200197]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:42:58.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[200197]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:42:58.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[200197]: failed to encode map e87 with expected crc 2023-10-09T10:42:58.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[200197]: osdmap e87: 8 total, 7 up, 8 in 2023-10-09T10:42:58.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[200197]: failed to encode map e87 with expected crc 2023-10-09T10:42:58.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[200197]: failed to encode map e87 with expected crc 2023-10-09T10:42:58.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[200197]: failed to encode map e87 with expected crc 2023-10-09T10:42:58.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:58 smithi125 ceph-mon[200197]: failed to encode map e87 with expected crc 2023-10-09T10:42:59.322 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:59 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.2.service: Succeeded. 2023-10-09T10:42:59.322 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:59 smithi125 systemd[1]: Stopped Ceph osd.2 for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:42:59.322 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:59 smithi125 systemd[1]: Starting Ceph osd.2 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:42:59.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:59 smithi125 ceph-mon[195507]: failed to encode map e88 with expected crc 2023-10-09T10:42:59.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:59 smithi125 ceph-mon[195507]: failed to encode map e88 with expected crc 2023-10-09T10:42:59.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:59 smithi125 ceph-mon[195507]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T10:42:59.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:59 smithi125 ceph-mon[195507]: failed to encode map e88 with expected crc 2023-10-09T10:42:59.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:59 smithi125 ceph-mon[195507]: failed to encode map e88 with expected crc 2023-10-09T10:42:59.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:59 smithi125 ceph-mon[195507]: failed to encode map e88 with expected crc 2023-10-09T10:42:59.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:42:59 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2963640909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:59.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:59 smithi125 ceph-mon[200197]: failed to encode map e88 with expected crc 2023-10-09T10:42:59.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:59 smithi125 ceph-mon[200197]: failed to encode map e88 with expected crc 2023-10-09T10:42:59.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:59 smithi125 ceph-mon[200197]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T10:42:59.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:59 smithi125 ceph-mon[200197]: failed to encode map e88 with expected crc 2023-10-09T10:42:59.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:59 smithi125 ceph-mon[200197]: failed to encode map e88 with expected crc 2023-10-09T10:42:59.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:59 smithi125 ceph-mon[200197]: failed to encode map e88 with expected crc 2023-10-09T10:42:59.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:42:59 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2963640909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:42:59.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:59 smithi155 ceph-mon[184854]: failed to encode map e88 with expected crc 2023-10-09T10:42:59.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:59 smithi155 ceph-mon[184854]: failed to encode map e88 with expected crc 2023-10-09T10:42:59.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:59 smithi155 ceph-mon[184854]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T10:42:59.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:59 smithi155 ceph-mon[184854]: failed to encode map e88 with expected crc 2023-10-09T10:42:59.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:59 smithi155 ceph-mon[184854]: failed to encode map e88 with expected crc 2023-10-09T10:42:59.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:59 smithi155 ceph-mon[184854]: failed to encode map e88 with expected crc 2023-10-09T10:42:59.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:42:59 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2963640909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:00.172 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:59 smithi125 bash[222532]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T10:43:00.172 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:59 smithi125 bash[222532]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T10:43:00.172 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:59 smithi125 bash[222532]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-09T10:43:00.172 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:59 smithi125 bash[222532]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-09T10:43:00.172 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:59 smithi125 bash[222532]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-09T10:43:00.173 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:59 smithi125 bash[222532]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T10:43:00.173 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:59 smithi125 bash[222532]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T10:43:00.173 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:42:59 smithi125 bash[222532]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-09T10:43:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:00 smithi155 ceph-mon[184854]: failed to encode map e88 with expected crc 2023-10-09T10:43:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:00 smithi155 ceph-mon[184854]: pgmap v142: 73 pgs: 2 peering, 2 stale+active+clean, 69 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:43:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:43:00.508 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:00 smithi125 ceph-mon[195507]: failed to encode map e88 with expected crc 2023-10-09T10:43:00.509 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:00 smithi125 ceph-mon[195507]: pgmap v142: 73 pgs: 2 peering, 2 stale+active+clean, 69 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:43:00.509 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:00.509 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:43:00.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:00 smithi125 ceph-mon[200197]: failed to encode map e88 with expected crc 2023-10-09T10:43:00.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:00 smithi125 ceph-mon[200197]: pgmap v142: 73 pgs: 2 peering, 2 stale+active+clean, 69 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T10:43:00.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:00.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:43:00.821 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:43:00 smithi125 ceph-osd[222829]: -- 172.21.15.125:0/1751947819 <== mon.0 v2:172.21.15.125:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b844eb64e0 con 0x55b844e86c00 2023-10-09T10:43:00.822 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:43:00 smithi125 bash[222791]: eeba73e62c12a56e4863997b51bc7c29144fa6495b50ead6e0549f00e84d0fe1 2023-10-09T10:43:00.822 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:43:00 smithi125 systemd[1]: Started Ceph osd.2 for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:43:01.571 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:43:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-2[222806]: 2023-10-09T10:43:01.115+0000 7f780e3e2640 -1 Falling back to public interface 2023-10-09T10:43:01.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:43:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:43:01] "GET /metrics HTTP/1.1" 200 33986 "" "Prometheus/2.43.0" 2023-10-09T10:43:01.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:01.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:01.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:01.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:01 smithi155 ceph-mon[184854]: pgmap v143: 73 pgs: 1 active+undersized+degraded, 2 peering, 1 stale+active+clean, 11 active+undersized, 58 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T10:43:01.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1270007782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:02.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:02.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:02.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:02.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:01 smithi125 ceph-mon[200197]: pgmap v143: 73 pgs: 1 active+undersized+degraded, 2 peering, 1 stale+active+clean, 11 active+undersized, 58 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T10:43:02.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1270007782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:02.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:02.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:02.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:02.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:01 smithi125 ceph-mon[195507]: pgmap v143: 73 pgs: 1 active+undersized+degraded, 2 peering, 1 stale+active+clean, 11 active+undersized, 58 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T10:43:02.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1270007782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:02 smithi155 ceph-mon[184854]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:43:03.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:02 smithi125 ceph-mon[195507]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:43:03.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:02 smithi125 ceph-mon[200197]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:43:04.212 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:43:03 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-2[222806]: 2023-10-09T10:43:03.882+0000 7f780e3e2640 -1 osd.2 86 log_to_monitors true 2023-10-09T10:43:04.213 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:04 smithi125 ceph-mon[195507]: pgmap v144: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T10:43:04.213 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:04 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:04.213 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:04 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:04.213 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:04 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:04.213 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:04 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:04.213 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:04 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3301199260' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:04.213 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:04 smithi125 ceph-mon[195507]: from='osd.2 [v2:172.21.15.125:6818/3577506432,v1:172.21.15.125:6819/3577506432]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T10:43:04.214 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:04 smithi125 ceph-mon[200197]: pgmap v144: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T10:43:04.214 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:04 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:04.214 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:04 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:04.214 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:04 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:04.214 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:04 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:04.214 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:04 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3301199260' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:04.214 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:04 smithi125 ceph-mon[200197]: from='osd.2 [v2:172.21.15.125:6818/3577506432,v1:172.21.15.125:6819/3577506432]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T10:43:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:04 smithi155 ceph-mon[184854]: pgmap v144: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T10:43:04.663 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:04 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:04.663 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:04 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:04.663 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:04 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:04.663 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:04 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:04.663 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:04 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3301199260' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:04.663 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:04 smithi155 ceph-mon[184854]: from='osd.2 [v2:172.21.15.125:6818/3577506432,v1:172.21.15.125:6819/3577506432]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T10:43:05.070 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 10:43:04 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-2[222806]: 2023-10-09T10:43:04.670+0000 7f77fe04b700 -1 osd.2 86 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:43:05.929 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:05 smithi125 ceph-mon[195507]: failed to encode map e89 with expected crc 2023-10-09T10:43:05.929 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:05 smithi125 ceph-mon[195507]: from='osd.2 [v2:172.21.15.125:6818/3577506432,v1:172.21.15.125:6819/3577506432]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T10:43:05.929 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:05 smithi125 ceph-mon[195507]: osdmap e89: 8 total, 7 up, 8 in 2023-10-09T10:43:05.929 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:05 smithi125 ceph-mon[195507]: from='osd.2 [v2:172.21.15.125:6818/3577506432,v1:172.21.15.125:6819/3577506432]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:43:05.929 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:05 smithi125 ceph-mon[195507]: pgmap v146: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T10:43:05.930 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:05 smithi125 ceph-mon[200197]: failed to encode map e89 with expected crc 2023-10-09T10:43:05.930 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:05 smithi125 ceph-mon[200197]: from='osd.2 [v2:172.21.15.125:6818/3577506432,v1:172.21.15.125:6819/3577506432]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T10:43:05.930 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:05 smithi125 ceph-mon[200197]: osdmap e89: 8 total, 7 up, 8 in 2023-10-09T10:43:05.930 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:05 smithi125 ceph-mon[200197]: from='osd.2 [v2:172.21.15.125:6818/3577506432,v1:172.21.15.125:6819/3577506432]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:43:05.930 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:05 smithi125 ceph-mon[200197]: pgmap v146: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T10:43:05.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:05 smithi155 ceph-mon[184854]: failed to encode map e89 with expected crc 2023-10-09T10:43:05.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:05 smithi155 ceph-mon[184854]: from='osd.2 [v2:172.21.15.125:6818/3577506432,v1:172.21.15.125:6819/3577506432]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T10:43:05.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:05 smithi155 ceph-mon[184854]: osdmap e89: 8 total, 7 up, 8 in 2023-10-09T10:43:05.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:05 smithi155 ceph-mon[184854]: from='osd.2 [v2:172.21.15.125:6818/3577506432,v1:172.21.15.125:6819/3577506432]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:43:05.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:05 smithi155 ceph-mon[184854]: pgmap v146: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T10:43:06.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:43:06.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: failed to encode map e90 with expected crc 2023-10-09T10:43:06.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: failed to encode map e90 with expected crc 2023-10-09T10:43:06.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: osd.2 [v2:172.21.15.125:6818/3577506432,v1:172.21.15.125:6819/3577506432] boot 2023-10-09T10:43:06.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: osdmap e90: 8 total, 8 up, 8 in 2023-10-09T10:43:06.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:43:06.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2656528256' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:06.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:06.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:06.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:43:06.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:43:06.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:06.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:06.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:06.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:06.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:06.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T10:43:06.676 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:43:06.329+0000 7fa537ed4700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-09T10:43:06.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:43:06.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: failed to encode map e90 with expected crc 2023-10-09T10:43:06.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: failed to encode map e90 with expected crc 2023-10-09T10:43:06.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: osd.2 [v2:172.21.15.125:6818/3577506432,v1:172.21.15.125:6819/3577506432] boot 2023-10-09T10:43:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: osdmap e90: 8 total, 8 up, 8 in 2023-10-09T10:43:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:43:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2656528256' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:43:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:43:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:06.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:06 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T10:43:07.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:43:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: failed to encode map e90 with expected crc 2023-10-09T10:43:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: failed to encode map e90 with expected crc 2023-10-09T10:43:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: osd.2 [v2:172.21.15.125:6818/3577506432,v1:172.21.15.125:6819/3577506432] boot 2023-10-09T10:43:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: osdmap e90: 8 total, 8 up, 8 in 2023-10-09T10:43:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T10:43:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2656528256' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:43:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:43:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:07.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:07.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:07.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:07.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:07.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T10:43:07.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T10:43:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-09T10:43:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: failed to encode map e91 with expected crc 2023-10-09T10:43:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: osdmap e91: 8 total, 8 up, 8 in 2023-10-09T10:43:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: failed to encode map e91 with expected crc 2023-10-09T10:43:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: failed to encode map e91 with expected crc 2023-10-09T10:43:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: failed to encode map e91 with expected crc 2023-10-09T10:43:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: failed to encode map e91 with expected crc 2023-10-09T10:43:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: failed to encode map e91 with expected crc 2023-10-09T10:43:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: failed to encode map e91 with expected crc 2023-10-09T10:43:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: failed to encode map e91 with expected crc 2023-10-09T10:43:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: failed to encode map e91 with expected crc 2023-10-09T10:43:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: failed to encode map e91 with expected crc 2023-10-09T10:43:07.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: failed to encode map e91 with expected crc 2023-10-09T10:43:07.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: pgmap v149: 73 pgs: 15 peering, 1 active+undersized+degraded, 3 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T10:43:07.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:07 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:43:08.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T10:43:08.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-09T10:43:08.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: osdmap e91: 8 total, 8 up, 8 in 2023-10-09T10:43:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: pgmap v149: 73 pgs: 15 peering, 1 active+undersized+degraded, 3 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T10:43:08.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:43:08.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T10:43:08.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-09T10:43:08.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: osdmap e91: 8 total, 8 up, 8 in 2023-10-09T10:43:08.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: failed to encode map e91 with expected crc 2023-10-09T10:43:08.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: pgmap v149: 73 pgs: 15 peering, 1 active+undersized+degraded, 3 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T10:43:08.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:07 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:43:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:08 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1092642688' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:09.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:08 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1092642688' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:09.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:08 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1092642688' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:09.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:09 smithi155 ceph-mon[184854]: pgmap v150: 73 pgs: 15 peering, 2 active+undersized, 56 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:10.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:09 smithi125 ceph-mon[195507]: pgmap v150: 73 pgs: 15 peering, 2 active+undersized, 56 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:10.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:09 smithi125 ceph-mon[200197]: pgmap v150: 73 pgs: 15 peering, 2 active+undersized, 56 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:10.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:10 smithi155 ceph-mon[184854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T10:43:10.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:10 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T10:43:10.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:10 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:10.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:10 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1127392306' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:11.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:10 smithi125 ceph-mon[195507]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T10:43:11.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:10 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T10:43:11.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:10 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:11.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:10 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1127392306' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:11.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:10 smithi125 ceph-mon[200197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T10:43:11.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:10 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T10:43:11.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:10 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:11.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:10 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1127392306' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:11.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:43:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:43:11] "GET /metrics HTTP/1.1" 200 33986 "" "Prometheus/2.43.0" 2023-10-09T10:43:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:11 smithi155 ceph-mon[184854]: pgmap v151: 73 pgs: 4 peering, 69 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:12.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:11 smithi125 ceph-mon[195507]: pgmap v151: 73 pgs: 4 peering, 69 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:12.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:11 smithi125 ceph-mon[200197]: pgmap v151: 73 pgs: 4 peering, 69 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:13.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:13 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1497909549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:13.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:13 smithi125 ceph-mon[195507]: pgmap v152: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:13.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:13 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1497909549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:13.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:13 smithi125 ceph-mon[200197]: pgmap v152: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:13 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1497909549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:13 smithi155 ceph-mon[184854]: pgmap v152: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:16.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:15 smithi125 ceph-mon[200197]: pgmap v153: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:16.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:16.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:43:16.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:15 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/253459403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:16.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:15 smithi125 ceph-mon[195507]: pgmap v153: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:16.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:16.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:43:16.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:15 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/253459403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:16.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:15 smithi155 ceph-mon[184854]: pgmap v153: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:16.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:16.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:43:16.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:15 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/253459403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:18.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:17 smithi125 ceph-mon[200197]: pgmap v154: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:18.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2279599405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:17 smithi125 ceph-mon[195507]: pgmap v154: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2279599405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:18 smithi155 ceph-mon[184854]: pgmap v154: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2279599405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:20.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:20 smithi125 ceph-mon[200197]: pgmap v155: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:20.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:20 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3839279094' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:20.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:20 smithi125 ceph-mon[195507]: pgmap v155: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:20.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:20 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3839279094' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:20 smithi155 ceph-mon[184854]: pgmap v155: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:20 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3839279094' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:21.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:21 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:21.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:21 smithi125 ceph-mon[195507]: pgmap v156: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:21.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:43:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:43:21] "GET /metrics HTTP/1.1" 200 33985 "" "Prometheus/2.43.0" 2023-10-09T10:43:21.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:21 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:21.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:21 smithi125 ceph-mon[200197]: pgmap v156: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:21 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:21 smithi155 ceph-mon[184854]: pgmap v156: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:22.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:22 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T10:43:22.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:22 smithi125 ceph-mon[195507]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T10:43:22.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:22 smithi125 ceph-mon[195507]: Upgrade: osd.3 is safe to restart 2023-10-09T10:43:22.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:22 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4136383124' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:22.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:22 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T10:43:22.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:22 smithi125 ceph-mon[200197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T10:43:22.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:22 smithi125 ceph-mon[200197]: Upgrade: osd.3 is safe to restart 2023-10-09T10:43:22.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:22 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4136383124' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:22.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:22 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T10:43:22.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:22 smithi155 ceph-mon[184854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T10:43:22.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:22 smithi155 ceph-mon[184854]: Upgrade: osd.3 is safe to restart 2023-10-09T10:43:22.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:22 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4136383124' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:23.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:23 smithi155 ceph-mon[184854]: Upgrade: Updating osd.3 2023-10-09T10:43:23.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:23.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T10:43:23.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:43:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:23 smithi155 ceph-mon[184854]: Deploying daemon osd.3 on smithi125 2023-10-09T10:43:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:23 smithi155 ceph-mon[184854]: pgmap v157: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:23.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:23 smithi125 ceph-mon[200197]: Upgrade: Updating osd.3 2023-10-09T10:43:23.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:23.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T10:43:23.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:43:23.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:23 smithi125 ceph-mon[200197]: Deploying daemon osd.3 on smithi125 2023-10-09T10:43:23.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:23 smithi125 ceph-mon[200197]: pgmap v157: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:23 smithi125 ceph-mon[195507]: Upgrade: Updating osd.3 2023-10-09T10:43:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T10:43:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:43:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:23 smithi125 ceph-mon[195507]: Deploying daemon osd.3 on smithi125 2023-10-09T10:43:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:23 smithi125 ceph-mon[195507]: pgmap v157: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:24.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:24 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1349555415' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:24.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:24 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1349555415' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:24.821 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:24 smithi125 systemd[1]: Stopping Ceph osd.3 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:43:24.821 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:24 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[154655]: 2023-10-09T10:43:24.714+0000 7f57a264d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:43:24.821 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:24 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[154655]: 2023-10-09T10:43:24.714+0000 7f57a264d700 -1 osd.3 91 *** Got signal Terminated *** 2023-10-09T10:43:24.821 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:24 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[154655]: 2023-10-09T10:43:24.714+0000 7f57a264d700 -1 osd.3 91 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:43:24.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:24 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1349555415' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:25.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:25 smithi155 ceph-mon[184854]: osd.3 marked itself down and dead 2023-10-09T10:43:25.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:25 smithi155 ceph-mon[184854]: pgmap v158: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:26.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:25 smithi125 ceph-mon[195507]: osd.3 marked itself down and dead 2023-10-09T10:43:26.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:25 smithi125 ceph-mon[195507]: pgmap v158: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:26.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:25 smithi125 ceph-mon[200197]: osd.3 marked itself down and dead 2023-10-09T10:43:26.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:25 smithi125 ceph-mon[200197]: pgmap v158: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:26.570 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:26 smithi125 bash[228913]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T10:43:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:26 smithi155 ceph-mon[184854]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:43:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:26 smithi155 ceph-mon[184854]: failed to encode map e92 with expected crc 2023-10-09T10:43:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:26 smithi155 ceph-mon[184854]: failed to encode map e92 with expected crc 2023-10-09T10:43:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:26 smithi155 ceph-mon[184854]: osdmap e92: 8 total, 7 up, 8 in 2023-10-09T10:43:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:26 smithi155 ceph-mon[184854]: failed to encode map e92 with expected crc 2023-10-09T10:43:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:26 smithi155 ceph-mon[184854]: failed to encode map e92 with expected crc 2023-10-09T10:43:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:26 smithi155 ceph-mon[184854]: failed to encode map e92 with expected crc 2023-10-09T10:43:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:26 smithi155 ceph-mon[184854]: failed to encode map e92 with expected crc 2023-10-09T10:43:27.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[195507]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:43:27.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[195507]: failed to encode map e92 with expected crc 2023-10-09T10:43:27.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[195507]: failed to encode map e92 with expected crc 2023-10-09T10:43:27.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[195507]: osdmap e92: 8 total, 7 up, 8 in 2023-10-09T10:43:27.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[195507]: failed to encode map e92 with expected crc 2023-10-09T10:43:27.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[195507]: failed to encode map e92 with expected crc 2023-10-09T10:43:27.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[195507]: failed to encode map e92 with expected crc 2023-10-09T10:43:27.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[195507]: failed to encode map e92 with expected crc 2023-10-09T10:43:27.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[200197]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:43:27.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[200197]: failed to encode map e92 with expected crc 2023-10-09T10:43:27.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[200197]: failed to encode map e92 with expected crc 2023-10-09T10:43:27.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[200197]: osdmap e92: 8 total, 7 up, 8 in 2023-10-09T10:43:27.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[200197]: failed to encode map e92 with expected crc 2023-10-09T10:43:27.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[200197]: failed to encode map e92 with expected crc 2023-10-09T10:43:27.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[200197]: failed to encode map e92 with expected crc 2023-10-09T10:43:27.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:26 smithi125 ceph-mon[200197]: failed to encode map e92 with expected crc 2023-10-09T10:43:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:27 smithi155 ceph-mon[184854]: failed to encode map e93 with expected crc 2023-10-09T10:43:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:27 smithi155 ceph-mon[184854]: failed to encode map e93 with expected crc 2023-10-09T10:43:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:27 smithi155 ceph-mon[184854]: osdmap e93: 8 total, 7 up, 8 in 2023-10-09T10:43:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:27 smithi155 ceph-mon[184854]: failed to encode map e93 with expected crc 2023-10-09T10:43:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:27 smithi155 ceph-mon[184854]: failed to encode map e93 with expected crc 2023-10-09T10:43:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:27 smithi155 ceph-mon[184854]: failed to encode map e93 with expected crc 2023-10-09T10:43:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:27 smithi155 ceph-mon[184854]: failed to encode map e93 with expected crc 2023-10-09T10:43:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:27 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/950034599' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:27 smithi155 ceph-mon[184854]: pgmap v161: 73 pgs: 20 peering, 4 stale+active+clean, 49 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:28.006 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:27 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.3.service: Succeeded. 2023-10-09T10:43:28.006 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:27 smithi125 systemd[1]: Stopped Ceph osd.3 for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:43:28.006 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:27 smithi125 systemd[1]: Starting Ceph osd.3 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:43:28.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[195507]: failed to encode map e93 with expected crc 2023-10-09T10:43:28.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[195507]: failed to encode map e93 with expected crc 2023-10-09T10:43:28.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[195507]: osdmap e93: 8 total, 7 up, 8 in 2023-10-09T10:43:28.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[195507]: failed to encode map e93 with expected crc 2023-10-09T10:43:28.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[195507]: failed to encode map e93 with expected crc 2023-10-09T10:43:28.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[195507]: failed to encode map e93 with expected crc 2023-10-09T10:43:28.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[195507]: failed to encode map e93 with expected crc 2023-10-09T10:43:28.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/950034599' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:28.008 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[195507]: pgmap v161: 73 pgs: 20 peering, 4 stale+active+clean, 49 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:28.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[200197]: failed to encode map e93 with expected crc 2023-10-09T10:43:28.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[200197]: failed to encode map e93 with expected crc 2023-10-09T10:43:28.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[200197]: osdmap e93: 8 total, 7 up, 8 in 2023-10-09T10:43:28.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[200197]: failed to encode map e93 with expected crc 2023-10-09T10:43:28.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[200197]: failed to encode map e93 with expected crc 2023-10-09T10:43:28.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[200197]: failed to encode map e93 with expected crc 2023-10-09T10:43:28.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[200197]: failed to encode map e93 with expected crc 2023-10-09T10:43:28.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/950034599' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:28.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:27 smithi125 ceph-mon[200197]: pgmap v161: 73 pgs: 20 peering, 4 stale+active+clean, 49 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:28.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:28 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T10:43:28.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:28 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T10:43:28.821 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:28 smithi125 bash[229204]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T10:43:28.821 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:28 smithi125 bash[229204]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T10:43:28.821 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:28 smithi125 bash[229204]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-09T10:43:28.821 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:28 smithi125 bash[229204]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-09T10:43:28.821 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:28 smithi125 bash[229204]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-09T10:43:28.821 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:28 smithi125 bash[229204]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T10:43:28.821 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:28 smithi125 bash[229204]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T10:43:28.821 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:28 smithi125 bash[229204]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-09T10:43:28.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:28 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T10:43:29.570 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:29 smithi125 ceph-osd[229506]: -- 172.21.15.125:0/2560233912 <== mon.0 v2:172.21.15.125:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558af1f144e0 con 0x558af1ee5000 2023-10-09T10:43:29.570 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:29 smithi125 bash[229449]: f85926783d9a308a5741043bc4925aa88c98d99332ac4041d8668b0b99931b0f 2023-10-09T10:43:29.570 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:29 smithi125 systemd[1]: Started Ceph osd.3 for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:43:29.920 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:29 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[229483]: 2023-10-09T10:43:29.709+0000 7f8ed8ad5640 -1 Falling back to public interface 2023-10-09T10:43:29.921 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:29 smithi125 ceph-mon[195507]: pgmap v162: 73 pgs: 1 active+undersized+degraded, 20 peering, 2 stale+active+clean, 6 active+undersized, 44 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T10:43:29.921 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:29 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3476028640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:29.921 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:29.921 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:29.921 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:29.921 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:29 smithi125 ceph-mon[200197]: pgmap v162: 73 pgs: 1 active+undersized+degraded, 20 peering, 2 stale+active+clean, 6 active+undersized, 44 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T10:43:29.921 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:29 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3476028640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:29.922 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:29.922 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:29.922 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:29 smithi155 ceph-mon[184854]: pgmap v162: 73 pgs: 1 active+undersized+degraded, 20 peering, 2 stale+active+clean, 6 active+undersized, 44 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T10:43:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:29 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3476028640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:30.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:30 smithi125 ceph-mon[195507]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:43:30.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:30.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:43:30.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:30 smithi125 ceph-mon[200197]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:43:30.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:30.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:43:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:30 smithi155 ceph-mon[184854]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:43:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:43:31.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:43:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:43:31] "GET /metrics HTTP/1.1" 200 34052 "" "Prometheus/2.43.0" 2023-10-09T10:43:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:31 smithi155 ceph-mon[184854]: pgmap v163: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T10:43:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/966253951' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:32.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:31 smithi125 ceph-mon[195507]: pgmap v163: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T10:43:32.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/966253951' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:32.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:31 smithi125 ceph-mon[200197]: pgmap v163: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T10:43:32.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/966253951' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:32.820 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:32 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[229483]: 2023-10-09T10:43:32.477+0000 7f8ed8ad5640 -1 osd.3 91 log_to_monitors true 2023-10-09T10:43:32.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:32 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-10-09T10:43:32.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:32 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:32.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:32 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:32.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:32 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:32.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:32 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:32.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:32 smithi125 ceph-mon[195507]: from='osd.3 [v2:172.21.15.125:6826/1513104923,v1:172.21.15.125:6827/1513104923]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:43:32.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:32 smithi125 ceph-mon[195507]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:43:32.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:32 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-10-09T10:43:32.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:32 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:32.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:32 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:32.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:32 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:32.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:32 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:32.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:32 smithi125 ceph-mon[200197]: from='osd.3 [v2:172.21.15.125:6826/1513104923,v1:172.21.15.125:6827/1513104923]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:43:32.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:32 smithi125 ceph-mon[200197]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:43:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:32 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-10-09T10:43:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:32 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:32 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:32 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:32 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:32 smithi155 ceph-mon[184854]: from='osd.3 [v2:172.21.15.125:6826/1513104923,v1:172.21.15.125:6827/1513104923]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:43:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:32 smithi155 ceph-mon[184854]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T10:43:33.820 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 10:43:33 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[229483]: 2023-10-09T10:43:33.501+0000 7f8ec5f39700 -1 osd.3 91 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T10:43:33.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:33 smithi125 ceph-mon[195507]: pgmap v164: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T10:43:33.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:33 smithi125 ceph-mon[195507]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T10:43:33.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:33 smithi125 ceph-mon[195507]: osdmap e94: 8 total, 7 up, 8 in 2023-10-09T10:43:33.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:33 smithi125 ceph-mon[195507]: from='osd.3 [v2:172.21.15.125:6826/1513104923,v1:172.21.15.125:6827/1513104923]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:43:33.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:33 smithi125 ceph-mon[195507]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:43:33.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:33 smithi125 ceph-mon[200197]: pgmap v164: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T10:43:33.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:33 smithi125 ceph-mon[200197]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T10:43:33.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:33 smithi125 ceph-mon[200197]: osdmap e94: 8 total, 7 up, 8 in 2023-10-09T10:43:33.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:33 smithi125 ceph-mon[200197]: from='osd.3 [v2:172.21.15.125:6826/1513104923,v1:172.21.15.125:6827/1513104923]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:43:33.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:33 smithi125 ceph-mon[200197]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:43:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:33 smithi155 ceph-mon[184854]: pgmap v164: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T10:43:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:33 smithi155 ceph-mon[184854]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T10:43:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:33 smithi155 ceph-mon[184854]: osdmap e94: 8 total, 7 up, 8 in 2023-10-09T10:43:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:33 smithi155 ceph-mon[184854]: from='osd.3 [v2:172.21.15.125:6826/1513104923,v1:172.21.15.125:6827/1513104923]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:43:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:33 smithi155 ceph-mon[184854]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T10:43:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:34 smithi155 ceph-mon[184854]: failed to encode map e94 with expected crc 2023-10-09T10:43:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:34 smithi155 ceph-mon[184854]: failed to encode map e94 with expected crc 2023-10-09T10:43:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:34 smithi155 ceph-mon[184854]: failed to encode map e94 with expected crc 2023-10-09T10:43:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:34 smithi155 ceph-mon[184854]: failed to encode map e94 with expected crc 2023-10-09T10:43:34.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:34 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1165016599' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:34.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:34 smithi155 ceph-mon[184854]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:43:34.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:34 smithi155 ceph-mon[184854]: osd.3 [v2:172.21.15.125:6826/1513104923,v1:172.21.15.125:6827/1513104923] boot 2023-10-09T10:43:34.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:34 smithi155 ceph-mon[184854]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T10:43:34.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:34 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:43:34.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[195507]: failed to encode map e94 with expected crc 2023-10-09T10:43:34.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[195507]: failed to encode map e94 with expected crc 2023-10-09T10:43:34.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[195507]: failed to encode map e94 with expected crc 2023-10-09T10:43:34.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[195507]: failed to encode map e94 with expected crc 2023-10-09T10:43:34.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1165016599' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:34.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[195507]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:43:34.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[195507]: osd.3 [v2:172.21.15.125:6826/1513104923,v1:172.21.15.125:6827/1513104923] boot 2023-10-09T10:43:34.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[195507]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T10:43:34.978 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:43:34.978 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[200197]: failed to encode map e94 with expected crc 2023-10-09T10:43:34.978 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[200197]: failed to encode map e94 with expected crc 2023-10-09T10:43:34.978 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[200197]: failed to encode map e94 with expected crc 2023-10-09T10:43:34.978 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[200197]: failed to encode map e94 with expected crc 2023-10-09T10:43:34.978 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1165016599' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:34.978 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[200197]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T10:43:34.978 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[200197]: osd.3 [v2:172.21.15.125:6826/1513104923,v1:172.21.15.125:6827/1513104923] boot 2023-10-09T10:43:34.978 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[200197]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T10:43:34.978 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:34 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T10:43:35.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T10:43:35.197+0000 7fa537ed4700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T10:43:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:35 smithi155 ceph-mon[184854]: failed to encode map e95 with expected crc 2023-10-09T10:43:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:35 smithi155 ceph-mon[184854]: pgmap v167: 73 pgs: 2 peering, 3 active+undersized+degraded, 26 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T10:43:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:43:35.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:43:35.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:35.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:35.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:35.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:35.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:35.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T10:43:35.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:35 smithi155 ceph-mon[184854]: osdmap e96: 8 total, 8 up, 8 in 2023-10-09T10:43:36.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[200197]: failed to encode map e95 with expected crc 2023-10-09T10:43:36.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[200197]: pgmap v167: 73 pgs: 2 peering, 3 active+undersized+degraded, 26 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T10:43:36.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:36.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:36.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:43:36.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:43:36.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:36.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:36.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:36.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:36.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:36.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T10:43:36.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[200197]: osdmap e96: 8 total, 8 up, 8 in 2023-10-09T10:43:36.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[195507]: failed to encode map e95 with expected crc 2023-10-09T10:43:36.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[195507]: pgmap v167: 73 pgs: 2 peering, 3 active+undersized+degraded, 26 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T10:43:36.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:36.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:36.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:43:36.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:43:36.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:36.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:36.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:36.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:36.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:36.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T10:43:36.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:35 smithi125 ceph-mon[195507]: osdmap e96: 8 total, 8 up, 8 in 2023-10-09T10:43:36.192 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T10:43:36.553 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T10:43:36.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[195507]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T10:43:36.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[195507]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T10:43:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[195507]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[195507]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[195507]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[195507]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[195507]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[195507]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[195507]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2389236081' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:36.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[200197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T10:43:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[200197]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T10:43:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[200197]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[200197]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[200197]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[200197]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[200197]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[200197]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[200197]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:36 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2389236081' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:36.822 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:36.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:36 smithi155 ceph-mon[184854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T10:43:36.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:36 smithi155 ceph-mon[184854]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T10:43:36.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:36 smithi155 ceph-mon[184854]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:36 smithi155 ceph-mon[184854]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:36 smithi155 ceph-mon[184854]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:36 smithi155 ceph-mon[184854]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:36 smithi155 ceph-mon[184854]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:36 smithi155 ceph-mon[184854]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:36 smithi155 ceph-mon[184854]: failed to encode map e96 with expected crc 2023-10-09T10:43:36.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:36 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2389236081' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T10:43:37.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:37 smithi155 ceph-mon[184854]: pgmap v169: 73 pgs: 2 peering, 2 active+undersized+degraded, 16 active+undersized, 53 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T10:43:37.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:37 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:43:38.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:37 smithi125 ceph-mon[195507]: pgmap v169: 73 pgs: 2 peering, 2 active+undersized+degraded, 16 active+undersized, 53 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T10:43:38.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:37 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:43:38.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:37 smithi125 ceph-mon[200197]: pgmap v169: 73 pgs: 2 peering, 2 active+undersized+degraded, 16 active+undersized, 53 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T10:43:38.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:37 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:43:38.114 INFO:teuthology.orchestra.run.smithi125.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:43:38.528 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T10:43:38.794 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:39.264 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:39 smithi125 ceph-mon[195507]: from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:43:39.264 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:39 smithi125 ceph-mon[195507]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:43:39.265 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:39 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:39.265 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:39 smithi125 ceph-mon[200197]: from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:43:39.265 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:39 smithi125 ceph-mon[200197]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:43:39.265 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:39 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:39.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:39 smithi155 ceph-mon[184854]: from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:43:39.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:39 smithi155 ceph-mon[184854]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T10:43:39.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:39 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:40.063 INFO:teuthology.orchestra.run.smithi125.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T10:43:40.063 INFO:teuthology.orchestra.run.smithi125.stdout:alertmanager.a smithi125 *:9093,9094 running (5m) - 8m 27.4M - 0.25.0 c8568f914cd2 3c13e6c17b7e 2023-10-09T10:43:40.063 INFO:teuthology.orchestra.run.smithi125.stdout:grafana.a smithi155 *:3000 running (5m) - 7m 89.0M - 9.4.7 2c41d148cca3 0ec8893d264d 2023-10-09T10:43:40.063 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi125.satgiw smithi125 running (7m) - 7m 17.9M - 17.2.6-1303-gb2903cb0 5517396c50d4 7b5d28288eab 2023-10-09T10:43:40.063 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi155.ydxmyq smithi155 running (7m) - 7m 15.4M - 17.2.6-1303-gb2903cb0 5517396c50d4 c2c99605d324 2023-10-09T10:43:40.063 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.x smithi155 *:8443,9283,8765 running (5m) - 10m 449M - 18.2.0-598-gb4fa663f 3060897fdf97 67bb8d1af523 2023-10-09T10:43:40.063 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.y smithi125 *:8443,9283,8765 running (5m) - 12m 512M - 18.2.0-598-gb4fa663f 3060897fdf97 107edeccfff9 2023-10-09T10:43:40.063 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a smithi125 running (5m) - 12m 80.9M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 335012cc14fa 2023-10-09T10:43:40.063 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b smithi155 running (3m) - 11m 51.1M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 9b477d1850dc 2023-10-09T10:43:40.063 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c smithi125 running (3m) - 11m 62.2M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 c47abfeb6d97 2023-10-09T10:43:40.064 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.a smithi125 *:9100 running (5m) - 7m 17.8M - 1.5.0 0da6a335fe13 bf5a342348bd 2023-10-09T10:43:40.064 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.b smithi155 *:9100 running (5m) - 7m 16.1M - 1.5.0 0da6a335fe13 1320bf6a6ac5 2023-10-09T10:43:40.064 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 smithi125 running (2m) - 10m 57.9M 2486M 18.2.0-598-gb4fa663f 3060897fdf97 0ebace3b73a3 2023-10-09T10:43:40.064 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1 smithi125 running (67s) - 10m 53.2M 2486M 18.2.0-598-gb4fa663f 3060897fdf97 743f3b6c2116 2023-10-09T10:43:40.064 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2 smithi125 running (39s) - 10m 48.7M 2486M 18.2.0-598-gb4fa663f 3060897fdf97 eeba73e62c12 2023-10-09T10:43:40.064 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3 smithi125 running (11s) - 9m 11.5M 2486M 18.2.0-598-gb4fa663f 3060897fdf97 f85926783d9a 2023-10-09T10:43:40.064 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 smithi155 running (9m) - 9m 90.9M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 e3a61e2037a2 2023-10-09T10:43:40.064 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5 smithi155 running (9m) - 9m 104M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 12d4484e25b1 2023-10-09T10:43:40.064 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6 smithi155 running (9m) - 9m 74.9M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 c7b7ee1a3182 2023-10-09T10:43:40.064 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7 smithi155 running (8m) - 8m 113M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 8e225788b8f4 2023-10-09T10:43:40.064 INFO:teuthology.orchestra.run.smithi125.stdout:prometheus.a smithi155 *:9095 running (5m) - 8m 61.7M - 2.43.0 a07b618ecd1d 6412c6139964 2023-10-09T10:43:40.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:40 smithi125 ceph-mon[195507]: pgmap v170: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:40.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:40 smithi125 ceph-mon[195507]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T10:43:40.435 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:40 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T10:43:40.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:40 smithi125 ceph-mon[200197]: pgmap v170: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:40.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:40 smithi125 ceph-mon[200197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T10:43:40.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:40 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T10:43:40.452 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T10:43:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:40 smithi155 ceph-mon[184854]: pgmap v170: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:40 smithi155 ceph-mon[184854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T10:43:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:40 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T10:43:40.714 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:41.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:41 smithi155 ceph-mon[184854]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:43:41.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:41 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:41.531 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:41 smithi125 ceph-mon[195507]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:43:41.531 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:41 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:41.531 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:43:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:43:41] "GET /metrics HTTP/1.1" 200 34052 "" "Prometheus/2.43.0" 2023-10-09T10:43:41.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:41 smithi125 ceph-mon[200197]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:43:41.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:41 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:41.921 INFO:teuthology.orchestra.run.smithi125.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T10:43:41.921 INFO:teuthology.orchestra.run.smithi125.stdout:alertmanager.a smithi125 *:9093,9094 running (5m) - 8m 27.4M - 0.25.0 c8568f914cd2 3c13e6c17b7e 2023-10-09T10:43:41.921 INFO:teuthology.orchestra.run.smithi125.stdout:grafana.a smithi155 *:3000 running (5m) - 7m 89.0M - 9.4.7 2c41d148cca3 0ec8893d264d 2023-10-09T10:43:41.921 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi125.satgiw smithi125 running (7m) - 7m 17.9M - 17.2.6-1303-gb2903cb0 5517396c50d4 7b5d28288eab 2023-10-09T10:43:41.921 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi155.ydxmyq smithi155 running (7m) - 7m 15.4M - 17.2.6-1303-gb2903cb0 5517396c50d4 c2c99605d324 2023-10-09T10:43:41.921 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.x smithi155 *:8443,9283,8765 running (5m) - 11m 449M - 18.2.0-598-gb4fa663f 3060897fdf97 67bb8d1af523 2023-10-09T10:43:41.921 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.y smithi125 *:8443,9283,8765 running (5m) - 12m 512M - 18.2.0-598-gb4fa663f 3060897fdf97 107edeccfff9 2023-10-09T10:43:41.922 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a smithi125 running (5m) - 12m 80.9M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 335012cc14fa 2023-10-09T10:43:41.922 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b smithi155 running (3m) - 11m 51.1M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 9b477d1850dc 2023-10-09T10:43:41.922 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c smithi125 running (3m) - 11m 62.2M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 c47abfeb6d97 2023-10-09T10:43:41.922 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.a smithi125 *:9100 running (5m) - 7m 17.8M - 1.5.0 0da6a335fe13 bf5a342348bd 2023-10-09T10:43:41.922 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.b smithi155 *:9100 running (5m) - 7m 16.1M - 1.5.0 0da6a335fe13 1320bf6a6ac5 2023-10-09T10:43:41.922 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 smithi125 running (2m) - 10m 57.9M 2486M 18.2.0-598-gb4fa663f 3060897fdf97 0ebace3b73a3 2023-10-09T10:43:41.922 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1 smithi125 running (69s) - 10m 53.2M 2486M 18.2.0-598-gb4fa663f 3060897fdf97 743f3b6c2116 2023-10-09T10:43:41.922 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2 smithi125 running (41s) - 10m 48.7M 2486M 18.2.0-598-gb4fa663f 3060897fdf97 eeba73e62c12 2023-10-09T10:43:41.922 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3 smithi125 running (12s) - 9m 11.5M 2486M 18.2.0-598-gb4fa663f 3060897fdf97 f85926783d9a 2023-10-09T10:43:41.922 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 smithi155 running (9m) - 9m 90.9M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 e3a61e2037a2 2023-10-09T10:43:41.922 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5 smithi155 running (9m) - 9m 104M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 12d4484e25b1 2023-10-09T10:43:41.922 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6 smithi155 running (9m) - 9m 74.9M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 c7b7ee1a3182 2023-10-09T10:43:41.923 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7 smithi155 running (8m) - 8m 113M 2486M 17.2.6-1303-gb2903cb0 5517396c50d4 8e225788b8f4 2023-10-09T10:43:41.923 INFO:teuthology.orchestra.run.smithi125.stdout:prometheus.a smithi155 *:9095 running (5m) - 8m 61.7M - 2.43.0 a07b618ecd1d 6412c6139964 2023-10-09T10:43:42.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:42 smithi125 ceph-mon[195507]: pgmap v171: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:42.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:42 smithi125 ceph-mon[200197]: pgmap v171: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:42.336 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph versions' 2023-10-09T10:43:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:42 smithi155 ceph-mon[184854]: pgmap v171: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:42.604 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:43.297 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:43 smithi125 ceph-mon[195507]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:43:43.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:43 smithi125 ceph-mon[200197]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:43:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:43 smithi155 ceph-mon[184854]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T10:43:44.131 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:43:44.131 INFO:teuthology.orchestra.run.smithi125.stdout: "mon": { 2023-10-09T10:43:44.131 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T10:43:44.132 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:43:44.132 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr": { 2023-10-09T10:43:44.132 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T10:43:44.132 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:43:44.132 INFO:teuthology.orchestra.run.smithi125.stdout: "osd": { 2023-10-09T10:43:44.132 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 4, 2023-10-09T10:43:44.132 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T10:43:44.132 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:43:44.132 INFO:teuthology.orchestra.run.smithi125.stdout: "mds": { 2023-10-09T10:43:44.132 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 2 2023-10-09T10:43:44.132 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:43:44.132 INFO:teuthology.orchestra.run.smithi125.stdout: "overall": { 2023-10-09T10:43:44.133 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 6, 2023-10-09T10:43:44.133 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 9 2023-10-09T10:43:44.133 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:43:44.133 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:43:44.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:44 smithi125 ceph-mon[195507]: pgmap v172: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:44.416 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:44 smithi125 ceph-mon[200197]: pgmap v172: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:44.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:44 smithi155 ceph-mon[184854]: pgmap v172: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:44.517 DEBUG:teuthology.parallel:result is None 2023-10-09T10:43:44.518 INFO:teuthology.run_tasks:Running task print... 2023-10-09T10:43:44.528 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-10-09T10:43:44.528 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T10:43:44.560 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T10:43:44.560 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T10:43:44.560 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T10:43:44.825 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:45.296 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:45 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/716753961' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:45.296 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:45.297 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:43:45.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:45 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/716753961' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:45.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:45.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:43:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:45 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/716753961' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T10:43:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:43:46.141 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:43:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:46 smithi155 ceph-mon[184854]: pgmap v173: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:46.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:46 smithi125 ceph-mon[195507]: pgmap v173: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:46.564 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:46 smithi125 ceph-mon[200197]: pgmap v173: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:46.582 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":35,"active_gid":34107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6800","nonce":3278714741},{"type":"v1","addr":"172.21.15.125:6801","nonce":3278714741}]},"active_addr":"172.21.15.125:6801/3278714741","active_change":"2023-10-09T10:38:44.908378+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25000,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":76,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":2971004502}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":3450765050}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":2200521575}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":3208295983}]}]} 2023-10-09T10:43:46.588 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T10:43:46.588 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T10:43:46.588 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T10:43:46.856 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:47.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/351227377' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T10:43:47.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/351227377' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T10:43:47.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:47 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/351227377' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T10:43:48.198 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:43:48.198 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":96,"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","created":"2023-10-09T10:31:07.767709+0000","modified":"2023-10-09T10:43:35.473754+0000","last_up_change":"2023-10-09T10:43:34.471676+0000","last_in_change":"2023-10-09T10:34:43.351594+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T10:33:34.342993+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T10:35:19.807378+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T10:35:55.201745+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T10:35:56.049529+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"874cd96b-ddd8-4164-931e-06f2388dd41b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":76,"up_from":80,"up_thru":95,"down_at":77,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6803","nonce":2302743228}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6805","nonce":2302743228}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6809","nonce":2302743228}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6807","nonce":2302743228}]},"public_addr":"172.21.15.125:6803/2302743228","cluster_addr":"172.21.15.125:6805/2302743228","heartbeat_back_addr":"172.21.15.125:6809/2302743228","heartbeat_front_addr":"172.21.15.125:6807/2302743228","state":["exists","up"]},{"osd":1,"uuid":"0e517dd5-6d0b-4841-bb81-1eb04377ba5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":81,"up_from":85,"up_thru":95,"down_at":82,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":1934730282},{"type":"v1","addr":"172.21.15.125:6811","nonce":1934730282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":1934730282},{"type":"v1","addr":"172.21.15.125:6813","nonce":1934730282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":1934730282},{"type":"v1","addr":"172.21.15.125:6817","nonce":1934730282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":1934730282},{"type":"v1","addr":"172.21.15.125:6815","nonce":1934730282}]},"public_addr":"172.21.15.125:6811/1934730282","cluster_addr":"172.21.15.125:6813/1934730282","heartbeat_back_addr":"172.21.15.125:6817/1934730282","heartbeat_front_addr":"172.21.15.125:6815/1934730282","state":["exists","up"]},{"osd":2,"uuid":"643c1f3c-ecd9-464a-ad53-bf35a4deee5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":86,"up_from":90,"up_thru":95,"down_at":87,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6819","nonce":3577506432}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6821","nonce":3577506432}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6825","nonce":3577506432}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6823","nonce":3577506432}]},"public_addr":"172.21.15.125:6819/3577506432","cluster_addr":"172.21.15.125:6821/3577506432","heartbeat_back_addr":"172.21.15.125:6825/3577506432","heartbeat_front_addr":"172.21.15.125:6823/3577506432","state":["exists","up"]},{"osd":3,"uuid":"3015cd6e-b270-44c3-8e6e-6bcd979bcd21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":91,"up_from":95,"up_thru":95,"down_at":92,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":1513104923},{"type":"v1","addr":"172.21.15.125:6827","nonce":1513104923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":1513104923},{"type":"v1","addr":"172.21.15.125:6829","nonce":1513104923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":1513104923},{"type":"v1","addr":"172.21.15.125:6833","nonce":1513104923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":1513104923},{"type":"v1","addr":"172.21.15.125:6831","nonce":1513104923}]},"public_addr":"172.21.15.125:6827/1513104923","cluster_addr":"172.21.15.125:6829/1513104923","heartbeat_back_addr":"172.21.15.125:6833/1513104923","heartbeat_front_addr":"172.21.15.125:6831/1513104923","state":["exists","up"]},{"osd":4,"uuid":"272c7b95-f89c-4fe1-a299-a861e0ec9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6801","nonce":1086065767}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6803","nonce":1086065767}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6807","nonce":1086065767}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6805","nonce":1086065767}]},"public_addr":"172.21.15.155:6801/1086065767","cluster_addr":"172.21.15.155:6803/1086065767","heartbeat_back_addr":"172.21.15.155:6807/1086065767","heartbeat_front_addr":"172.21.15.155:6805/1086065767","state":["exists","up"]},{"osd":5,"uuid":"201d51f2-233c-40fc-a42d-649d225954fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6809","nonce":104536199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6811","nonce":104536199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6815","nonce":104536199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6813","nonce":104536199}]},"public_addr":"172.21.15.155:6809/104536199","cluster_addr":"172.21.15.155:6811/104536199","heartbeat_back_addr":"172.21.15.155:6815/104536199","heartbeat_front_addr":"172.21.15.155:6813/104536199","state":["exists","up"]},{"osd":6,"uuid":"4ec66642-108e-401f-aff3-e7ab258bc083","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6817","nonce":1730141492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6819","nonce":1730141492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6823","nonce":1730141492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6821","nonce":1730141492}]},"public_addr":"172.21.15.155:6817/1730141492","cluster_addr":"172.21.15.155:6819/1730141492","heartbeat_back_addr":"172.21.15.155:6823/1730141492","heartbeat_front_addr":"172.21.15.155:6821/1730141492","state":["exists","up"]},{"osd":7,"uuid":"f8eccbf5-4ad0-43e7-be9a-00c05a362e57","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6825","nonce":3094552858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6827","nonce":3094552858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6831","nonce":3094552858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6829","nonce":3094552858}]},"public_addr":"172.21.15.155:6825/3094552858","cluster_addr":"172.21.15.155:6827/3094552858","heartbeat_back_addr":"172.21.15.155:6831/3094552858","heartbeat_front_addr":"172.21.15.155:6829/3094552858","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:32:58.904066+0000","dead_epoch":76},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:15.883014+0000","dead_epoch":81},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:32.639145+0000","dead_epoch":86},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:49.937002+0000","dead_epoch":91},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:06.212675+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:22.755647+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:39.598906+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:56.143619+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.9","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:6801/1533297380":"2023-10-10T10:38:44.908167+0000","172.21.15.125:6800/1533297380":"2023-10-10T10:38:44.908167+0000","172.21.15.125:0/2429796720":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/2611643565":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/178379618":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/1316191185":"2023-10-10T10:35:20.607863+0000","172.21.15.155:0/1923144979":"2023-10-10T10:38:13.935349+0000","172.21.15.125:6800/1128053553":"2023-10-10T10:37:37.647752+0000","172.21.15.125:0/1483568239":"2023-10-10T10:37:37.647752+0000","172.21.15.125:6801/1128053553":"2023-10-10T10:37:37.647752+0000","172.21.15.125:6800/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.155:6835/2184630814":"2023-10-10T10:38:13.935349+0000","172.21.15.155:0/1920197956":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/2789680395":"2023-10-10T10:35:20.607863+0000","172.21.15.155:0/1738428098":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/1507781272":"2023-10-10T10:38:44.908167+0000","172.21.15.155:6834/2184630814":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/725980307":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6800/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/86510625":"2023-10-10T10:37:37.647752+0000","172.21.15.125:0/1623791415":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6801/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/461781620":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6801/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.155:0/2043466629":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/3709143923":"2023-10-10T10:37:37.647752+0000","172.21.15.125:0/1709045642":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/2757467822":"2023-10-10T10:37:37.647752+0000","172.21.15.125:6800/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/3693269321":"2023-10-10T10:38:44.908167+0000","172.21.15.155:0/48952571":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/3028032775":"2023-10-10T10:38:44.908167+0000","172.21.15.125:0/1101683240":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/653697639":"2023-10-10T10:38:44.908167+0000","172.21.15.125:6801/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/3810210930":"2023-10-10T10:38:44.908167+0000","172.21.15.155:0/565975522":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/1633295829":"2023-10-10T10:38:44.908167+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T10:43:48.405 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:48 smithi125 ceph-mon[195507]: pgmap v174: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:48.405 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:48 smithi125 ceph-mon[200197]: pgmap v174: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:48 smithi155 ceph-mon[184854]: pgmap v174: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:48.592 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T10:43:48.592 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T10:43:48.856 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:49.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:49 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1803979496' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:43:49.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:49 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1803979496' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:43:49.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:49 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1803979496' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:43:50.201 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:43:50.201 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":96,"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","created":"2023-10-09T10:31:07.767709+0000","modified":"2023-10-09T10:43:35.473754+0000","last_up_change":"2023-10-09T10:43:34.471676+0000","last_in_change":"2023-10-09T10:34:43.351594+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T10:33:34.342993+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T10:35:19.807378+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T10:35:55.201745+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T10:35:56.049529+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"874cd96b-ddd8-4164-931e-06f2388dd41b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":76,"up_from":80,"up_thru":95,"down_at":77,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6803","nonce":2302743228}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6805","nonce":2302743228}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6809","nonce":2302743228}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6807","nonce":2302743228}]},"public_addr":"172.21.15.125:6803/2302743228","cluster_addr":"172.21.15.125:6805/2302743228","heartbeat_back_addr":"172.21.15.125:6809/2302743228","heartbeat_front_addr":"172.21.15.125:6807/2302743228","state":["exists","up"]},{"osd":1,"uuid":"0e517dd5-6d0b-4841-bb81-1eb04377ba5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":81,"up_from":85,"up_thru":95,"down_at":82,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":1934730282},{"type":"v1","addr":"172.21.15.125:6811","nonce":1934730282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":1934730282},{"type":"v1","addr":"172.21.15.125:6813","nonce":1934730282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":1934730282},{"type":"v1","addr":"172.21.15.125:6817","nonce":1934730282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":1934730282},{"type":"v1","addr":"172.21.15.125:6815","nonce":1934730282}]},"public_addr":"172.21.15.125:6811/1934730282","cluster_addr":"172.21.15.125:6813/1934730282","heartbeat_back_addr":"172.21.15.125:6817/1934730282","heartbeat_front_addr":"172.21.15.125:6815/1934730282","state":["exists","up"]},{"osd":2,"uuid":"643c1f3c-ecd9-464a-ad53-bf35a4deee5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":86,"up_from":90,"up_thru":95,"down_at":87,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6819","nonce":3577506432}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6821","nonce":3577506432}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6825","nonce":3577506432}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6823","nonce":3577506432}]},"public_addr":"172.21.15.125:6819/3577506432","cluster_addr":"172.21.15.125:6821/3577506432","heartbeat_back_addr":"172.21.15.125:6825/3577506432","heartbeat_front_addr":"172.21.15.125:6823/3577506432","state":["exists","up"]},{"osd":3,"uuid":"3015cd6e-b270-44c3-8e6e-6bcd979bcd21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":91,"up_from":95,"up_thru":95,"down_at":92,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":1513104923},{"type":"v1","addr":"172.21.15.125:6827","nonce":1513104923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":1513104923},{"type":"v1","addr":"172.21.15.125:6829","nonce":1513104923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":1513104923},{"type":"v1","addr":"172.21.15.125:6833","nonce":1513104923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":1513104923},{"type":"v1","addr":"172.21.15.125:6831","nonce":1513104923}]},"public_addr":"172.21.15.125:6827/1513104923","cluster_addr":"172.21.15.125:6829/1513104923","heartbeat_back_addr":"172.21.15.125:6833/1513104923","heartbeat_front_addr":"172.21.15.125:6831/1513104923","state":["exists","up"]},{"osd":4,"uuid":"272c7b95-f89c-4fe1-a299-a861e0ec9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6801","nonce":1086065767}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6803","nonce":1086065767}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6807","nonce":1086065767}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6805","nonce":1086065767}]},"public_addr":"172.21.15.155:6801/1086065767","cluster_addr":"172.21.15.155:6803/1086065767","heartbeat_back_addr":"172.21.15.155:6807/1086065767","heartbeat_front_addr":"172.21.15.155:6805/1086065767","state":["exists","up"]},{"osd":5,"uuid":"201d51f2-233c-40fc-a42d-649d225954fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6809","nonce":104536199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6811","nonce":104536199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6815","nonce":104536199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6813","nonce":104536199}]},"public_addr":"172.21.15.155:6809/104536199","cluster_addr":"172.21.15.155:6811/104536199","heartbeat_back_addr":"172.21.15.155:6815/104536199","heartbeat_front_addr":"172.21.15.155:6813/104536199","state":["exists","up"]},{"osd":6,"uuid":"4ec66642-108e-401f-aff3-e7ab258bc083","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":92,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6817","nonce":1730141492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6819","nonce":1730141492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6823","nonce":1730141492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6821","nonce":1730141492}]},"public_addr":"172.21.15.155:6817/1730141492","cluster_addr":"172.21.15.155:6819/1730141492","heartbeat_back_addr":"172.21.15.155:6823/1730141492","heartbeat_front_addr":"172.21.15.155:6821/1730141492","state":["exists","up"]},{"osd":7,"uuid":"f8eccbf5-4ad0-43e7-be9a-00c05a362e57","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":95,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6825","nonce":3094552858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6827","nonce":3094552858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6831","nonce":3094552858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6829","nonce":3094552858}]},"public_addr":"172.21.15.155:6825/3094552858","cluster_addr":"172.21.15.155:6827/3094552858","heartbeat_back_addr":"172.21.15.155:6831/3094552858","heartbeat_front_addr":"172.21.15.155:6829/3094552858","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:32:58.904066+0000","dead_epoch":76},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:15.883014+0000","dead_epoch":81},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:32.639145+0000","dead_epoch":86},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:49.937002+0000","dead_epoch":91},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:06.212675+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:22.755647+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:39.598906+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:56.143619+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.9","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.155:0/48952571":"2023-10-10T10:38:13.935349+0000","172.21.15.125:6801/1533297380":"2023-10-10T10:38:44.908167+0000","172.21.15.125:0/1316191185":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/3693269321":"2023-10-10T10:38:44.908167+0000","172.21.15.125:6800/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6800/1128053553":"2023-10-10T10:37:37.647752+0000","172.21.15.125:6800/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.155:6835/2184630814":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/653697639":"2023-10-10T10:38:44.908167+0000","172.21.15.125:0/1101683240":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/1623791415":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6801/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.155:0/1920197956":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/2789680395":"2023-10-10T10:35:20.607863+0000","172.21.15.155:0/1738428098":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/1507781272":"2023-10-10T10:38:44.908167+0000","172.21.15.155:0/1923144979":"2023-10-10T10:38:13.935349+0000","172.21.15.125:6801/1128053553":"2023-10-10T10:37:37.647752+0000","172.21.15.125:0/1483568239":"2023-10-10T10:37:37.647752+0000","172.21.15.155:6834/2184630814":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/3028032775":"2023-10-10T10:38:44.908167+0000","172.21.15.125:0/2429796720":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6800/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/86510625":"2023-10-10T10:37:37.647752+0000","172.21.15.125:0/725980307":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/178379618":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/2611643565":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/461781620":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6801/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.155:0/2043466629":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/3709143923":"2023-10-10T10:37:37.647752+0000","172.21.15.125:0/1709045642":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/2757467822":"2023-10-10T10:37:37.647752+0000","172.21.15.125:0/3810210930":"2023-10-10T10:38:44.908167+0000","172.21.15.125:6801/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/1633295829":"2023-10-10T10:38:44.908167+0000","172.21.15.155:0/565975522":"2023-10-10T10:38:13.935349+0000","172.21.15.125:6800/1533297380":"2023-10-10T10:38:44.908167+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T10:43:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:50 smithi155 ceph-mon[184854]: pgmap v175: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:50.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:50 smithi125 ceph-mon[195507]: pgmap v175: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:50.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:50 smithi125 ceph-mon[200197]: pgmap v175: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:50.575 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T10:43:50.575 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T10:43:50.575 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T10:43:50.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T10:43:50.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T10:43:50.577 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T10:43:50.577 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T10:43:50.578 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T10:43:51.126 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:51.126 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:51.279 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:51.436 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1053738720' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:43:51.437 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:51 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:51.437 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:51 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:51.437 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1053738720' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:43:51.437 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:51 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:51.437 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:51 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:51.438 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:43:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:43:51] "GET /metrics HTTP/1.1" 200 34050 "" "Prometheus/2.43.0" 2023-10-09T10:43:51.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1053738720' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T10:43:51.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:51 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:51.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:51 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:51.834 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:51.886 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:51.986 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:52.100 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:52.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:52 smithi125 ceph-mon[195507]: pgmap v176: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:52.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:52 smithi125 ceph-mon[200197]: pgmap v176: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:52.415 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:52 smithi155 ceph-mon[184854]: pgmap v176: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:52.417 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:53.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:53.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:53.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:43:53.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:43:53.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:53.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[195507]: pgmap v177: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:53.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:53.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:43:53.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:43:53.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:53.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:53.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:53.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:43:53.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:43:53.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:53.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[200197]: pgmap v177: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:53.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:53.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:43:53.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:43:53.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:53 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:53 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:53 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:53 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:43:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:53 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:43:53.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:53 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:53.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:53 smithi155 ceph-mon[184854]: pgmap v177: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:53.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:53 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:43:53.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:53 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:43:53.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:53 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:43:53.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:53 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:43:54.767 INFO:teuthology.orchestra.run.smithi125.stdout:188978561138 2023-10-09T10:43:54.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T10:43:56.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:56 smithi125 ceph-mon[195507]: pgmap v178: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:56.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:56 smithi125 ceph-mon[200197]: pgmap v178: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:56.344 INFO:teuthology.orchestra.run.smithi125.stdout:128849019001 2023-10-09T10:43:56.345 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T10:43:56.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:56 smithi155 ceph-mon[184854]: pgmap v178: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:57.147 INFO:teuthology.orchestra.run.smithi125.stdout:158913790070 2023-10-09T10:43:57.148 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T10:43:57.651 INFO:teuthology.orchestra.run.smithi125.stdout:386547056652 2023-10-09T10:43:57.651 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T10:43:58.178 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:43:58 smithi125 ceph-mon[195507]: pgmap v179: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:58.178 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:43:58 smithi125 ceph-mon[200197]: pgmap v179: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:58.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:43:58 smithi155 ceph-mon[184854]: pgmap v179: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:43:58.619 INFO:teuthology.orchestra.run.smithi125.stdout:408021893126 2023-10-09T10:43:58.620 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T10:43:58.621 INFO:teuthology.orchestra.run.smithi125.stdout:365072220178 2023-10-09T10:43:58.621 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T10:43:58.749 INFO:teuthology.orchestra.run.smithi125.stdout:343597383711 2023-10-09T10:43:58.750 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T10:43:58.759 INFO:teuthology.orchestra.run.smithi125.stdout:219043332208 2023-10-09T10:43:58.760 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T10:43:58.962 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:58.962 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:58.965 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:58.965 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:59.162 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:43:59.704 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:00.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:00 smithi125 ceph-mon[195507]: pgmap v180: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:00.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:44:00.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:00 smithi125 ceph-mon[200197]: pgmap v180: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:00.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:44:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:00 smithi155 ceph-mon[184854]: pgmap v180: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:44:01.459 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:44:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:44:01] "GET /metrics HTTP/1.1" 200 34049 "" "Prometheus/2.43.0" 2023-10-09T10:44:01.468 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:02.073 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:02.166 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:02 smithi125 ceph-mon[195507]: pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:02.167 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:02 smithi125 ceph-mon[200197]: pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:02.279 INFO:teuthology.orchestra.run.smithi125.stdout:128849019003 2023-10-09T10:44:02.328 INFO:teuthology.orchestra.run.smithi125.stdout:219043332209 2023-10-09T10:44:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:02 smithi155 ceph-mon[184854]: pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:02.557 INFO:teuthology.orchestra.run.smithi125.stdout:158913790072 2023-10-09T10:44:02.665 INFO:teuthology.orchestra.run.smithi125.stdout:408021893127 2023-10-09T10:44:03.281 INFO:tasks.cephadm.ceph_manager.ceph:need seq 219043332208 got 219043332209 for osd.7 2023-10-09T10:44:03.282 DEBUG:teuthology.parallel:result is None 2023-10-09T10:44:03.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1979104677' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T10:44:03.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2222126416' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T10:44:03.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1565586939' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T10:44:03.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3946684361' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T10:44:03.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1979104677' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T10:44:03.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2222126416' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T10:44:03.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1565586939' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T10:44:03.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3946684361' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T10:44:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1979104677' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T10:44:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2222126416' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T10:44:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1565586939' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T10:44:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3946684361' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T10:44:03.603 INFO:teuthology.orchestra.run.smithi125.stdout:386547056653 2023-10-09T10:44:03.883 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019001 got 128849019003 for osd.4 2023-10-09T10:44:03.883 DEBUG:teuthology.parallel:result is None 2023-10-09T10:44:04.071 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913790070 got 158913790072 for osd.5 2023-10-09T10:44:04.072 DEBUG:teuthology.parallel:result is None 2023-10-09T10:44:04.200 INFO:teuthology.orchestra.run.smithi125.stdout:188978561140 2023-10-09T10:44:04.227 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:04 smithi125 ceph-mon[195507]: pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:04.227 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:04 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/182086665' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T10:44:04.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:04 smithi125 ceph-mon[200197]: pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:04.228 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:04 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/182086665' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T10:44:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:04 smithi155 ceph-mon[184854]: pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:04 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/182086665' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T10:44:04.799 INFO:tasks.cephadm.ceph_manager.ceph:need seq 386547056652 got 386547056653 for osd.2 2023-10-09T10:44:04.800 DEBUG:teuthology.parallel:result is None 2023-10-09T10:44:04.941 INFO:tasks.cephadm.ceph_manager.ceph:need seq 188978561138 got 188978561140 for osd.6 2023-10-09T10:44:04.942 DEBUG:teuthology.parallel:result is None 2023-10-09T10:44:05.130 INFO:tasks.cephadm.ceph_manager.ceph:need seq 408021893126 got 408021893127 for osd.3 2023-10-09T10:44:05.131 DEBUG:teuthology.parallel:result is None 2023-10-09T10:44:05.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:05 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3799350743' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T10:44:05.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:05 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3799350743' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T10:44:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:05 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3799350743' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T10:44:05.498 INFO:teuthology.orchestra.run.smithi125.stdout:365072220179 2023-10-09T10:44:05.695 INFO:teuthology.orchestra.run.smithi125.stdout:343597383713 2023-10-09T10:44:06.105 INFO:tasks.cephadm.ceph_manager.ceph:need seq 365072220178 got 365072220179 for osd.1 2023-10-09T10:44:06.106 DEBUG:teuthology.parallel:result is None 2023-10-09T10:44:06.146 INFO:tasks.cephadm.ceph_manager.ceph:need seq 343597383711 got 343597383713 for osd.0 2023-10-09T10:44:06.203 DEBUG:teuthology.parallel:result is None 2023-10-09T10:44:06.203 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T10:44:06.204 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:44:06.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:06 smithi125 ceph-mon[195507]: pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:06.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:06 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1166561611' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T10:44:06.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:06 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1947343648' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T10:44:06.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:06 smithi125 ceph-mon[200197]: pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:06.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:06 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1166561611' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T10:44:06.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:06 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1947343648' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T10:44:06.468 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:06 smithi155 ceph-mon[184854]: pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:06 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1166561611' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T10:44:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:06 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1947343648' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T10:44:07.665 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:44:07.666 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:44:08.090 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":184,"stamp":"2023-10-09T10:44:07.001182+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1333248,"kb_used_data":9472,"kb_used_omap":126,"kb_used_meta":1323585,"kb_avail":748627968,"statfs":{"total":767960285184,"available":766595039232,"internally_reserved":0,"allocated":9699328,"data_stored":6025331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1355351675},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004515"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485359+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:43:34.485359+0000","last_peered":"2023-10-09T10:43:34.485359+0000","last_clean":"2023-10-09T10:43:34.485359+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:43:34.485359+0000","last_undegraded":"2023-10-09T10:43:34.485359+0000","last_fullsized":"2023-10-09T10:43:34.485359+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":57,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.494552+0000","last_change":"2023-10-09T10:43:06.674549+0000","last_active":"2023-10-09T10:43:34.494552+0000","last_peered":"2023-10-09T10:43:34.494552+0000","last_clean":"2023-10-09T10:43:34.494552+0000","last_became_active":"2023-10-09T10:43:06.673510+0000","last_became_peered":"2023-10-09T10:43:06.673510+0000","last_unstale":"2023-10-09T10:43:34.494552+0000","last_undegraded":"2023-10-09T10:43:34.494552+0000","last_fullsized":"2023-10-09T10:43:34.494552+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:32.996795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":84,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487344+0000","last_change":"2023-10-09T10:43:35.487344+0000","last_active":"2023-10-09T10:43:35.487344+0000","last_peered":"2023-10-09T10:43:35.487344+0000","last_clean":"2023-10-09T10:43:35.487344+0000","last_became_active":"2023-10-09T10:43:35.486672+0000","last_became_peered":"2023-10-09T10:43:35.486672+0000","last_unstale":"2023-10-09T10:43:35.487344+0000","last_undegraded":"2023-10-09T10:43:35.487344+0000","last_fullsized":"2023-10-09T10:43:35.487344+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:44.665687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":69,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.485790+0000","last_change":"2023-10-09T10:43:35.485790+0000","last_active":"2023-10-09T10:43:35.485790+0000","last_peered":"2023-10-09T10:43:35.485790+0000","last_clean":"2023-10-09T10:43:35.485790+0000","last_became_active":"2023-10-09T10:43:35.485436+0000","last_became_peered":"2023-10-09T10:43:35.485436+0000","last_unstale":"2023-10-09T10:43:35.485790+0000","last_undegraded":"2023-10-09T10:43:35.485790+0000","last_fullsized":"2023-10-09T10:43:35.485790+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:45:20.897767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":77,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.492236+0000","last_change":"2023-10-09T10:43:35.492236+0000","last_active":"2023-10-09T10:43:35.492236+0000","last_peered":"2023-10-09T10:43:35.492236+0000","last_clean":"2023-10-09T10:43:35.492236+0000","last_became_active":"2023-10-09T10:43:35.491913+0000","last_became_peered":"2023-10-09T10:43:35.491913+0000","last_unstale":"2023-10-09T10:43:35.492236+0000","last_undegraded":"2023-10-09T10:43:35.492236+0000","last_fullsized":"2023-10-09T10:43:35.492236+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:34:13.448179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":46,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.480679+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:43:34.480679+0000","last_peered":"2023-10-09T10:43:34.480679+0000","last_clean":"2023-10-09T10:43:34.480679+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:43:34.480679+0000","last_undegraded":"2023-10-09T10:43:34.480679+0000","last_fullsized":"2023-10-09T10:43:34.480679+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":77,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.492737+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:43:35.492737+0000","last_peered":"2023-10-09T10:43:35.492737+0000","last_clean":"2023-10-09T10:43:35.492737+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:43:35.492737+0000","last_undegraded":"2023-10-09T10:43:35.492737+0000","last_fullsized":"2023-10-09T10:43:35.492737+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479260+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:43:34.479260+0000","last_peered":"2023-10-09T10:43:34.479260+0000","last_clean":"2023-10-09T10:43:34.479260+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:43:34.479260+0000","last_undegraded":"2023-10-09T10:43:34.479260+0000","last_fullsized":"2023-10-09T10:43:34.479260+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":82,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.489655+0000","last_change":"2023-10-09T10:43:35.489655+0000","last_active":"2023-10-09T10:43:35.489655+0000","last_peered":"2023-10-09T10:43:35.489655+0000","last_clean":"2023-10-09T10:43:35.489655+0000","last_became_active":"2023-10-09T10:43:35.489149+0000","last_became_peered":"2023-10-09T10:43:35.489149+0000","last_unstale":"2023-10-09T10:43:35.489655+0000","last_undegraded":"2023-10-09T10:43:35.489655+0000","last_fullsized":"2023-10-09T10:43:35.489655+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:30.598962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":42,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483874+0000","last_change":"2023-10-09T10:41:29.514075+0000","last_active":"2023-10-09T10:43:35.483874+0000","last_peered":"2023-10-09T10:43:35.483874+0000","last_clean":"2023-10-09T10:43:35.483874+0000","last_became_active":"2023-10-09T10:41:29.513347+0000","last_became_peered":"2023-10-09T10:41:29.513347+0000","last_unstale":"2023-10-09T10:43:35.483874+0000","last_undegraded":"2023-10-09T10:43:35.483874+0000","last_fullsized":"2023-10-09T10:43:35.483874+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:17.504888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485850+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:43:34.485850+0000","last_peered":"2023-10-09T10:43:34.485850+0000","last_clean":"2023-10-09T10:43:34.485850+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:43:34.485850+0000","last_undegraded":"2023-10-09T10:43:34.485850+0000","last_fullsized":"2023-10-09T10:43:34.485850+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":57,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483729+0000","last_change":"2023-10-09T10:42:38.407058+0000","last_active":"2023-10-09T10:43:35.483729+0000","last_peered":"2023-10-09T10:43:35.483729+0000","last_clean":"2023-10-09T10:43:35.483729+0000","last_became_active":"2023-10-09T10:42:38.406768+0000","last_became_peered":"2023-10-09T10:42:38.406768+0000","last_unstale":"2023-10-09T10:43:35.483729+0000","last_undegraded":"2023-10-09T10:43:35.483729+0000","last_fullsized":"2023-10-09T10:43:35.483729+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:41.198843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482345+0000","last_change":"2023-10-09T10:43:06.674387+0000","last_active":"2023-10-09T10:43:34.482345+0000","last_peered":"2023-10-09T10:43:34.482345+0000","last_clean":"2023-10-09T10:43:34.482345+0000","last_became_active":"2023-10-09T10:43:06.674032+0000","last_became_peered":"2023-10-09T10:43:06.674032+0000","last_unstale":"2023-10-09T10:43:34.482345+0000","last_undegraded":"2023-10-09T10:43:34.482345+0000","last_fullsized":"2023-10-09T10:43:34.482345+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:58.222034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":58,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486342+0000","last_change":"2023-10-09T10:43:35.486342+0000","last_active":"2023-10-09T10:43:35.486342+0000","last_peered":"2023-10-09T10:43:35.486342+0000","last_clean":"2023-10-09T10:43:35.486342+0000","last_became_active":"2023-10-09T10:43:35.486026+0000","last_became_peered":"2023-10-09T10:43:35.486026+0000","last_unstale":"2023-10-09T10:43:35.486342+0000","last_undegraded":"2023-10-09T10:43:35.486342+0000","last_fullsized":"2023-10-09T10:43:35.486342+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:16.616429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489374+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:43:34.489374+0000","last_peered":"2023-10-09T10:43:34.489374+0000","last_clean":"2023-10-09T10:43:34.489374+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:43:34.489374+0000","last_undegraded":"2023-10-09T10:43:34.489374+0000","last_fullsized":"2023-10-09T10:43:34.489374+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":61,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487842+0000","last_change":"2023-10-09T10:43:35.487842+0000","last_active":"2023-10-09T10:43:35.487842+0000","last_peered":"2023-10-09T10:43:35.487842+0000","last_clean":"2023-10-09T10:43:35.487842+0000","last_became_active":"2023-10-09T10:43:35.487192+0000","last_became_peered":"2023-10-09T10:43:35.487192+0000","last_unstale":"2023-10-09T10:43:35.487842+0000","last_undegraded":"2023-10-09T10:43:35.487842+0000","last_fullsized":"2023-10-09T10:43:35.487842+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:35.477280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.492904+0000","last_change":"2023-10-09T10:41:29.513802+0000","last_active":"2023-10-09T10:43:34.492904+0000","last_peered":"2023-10-09T10:43:34.492904+0000","last_clean":"2023-10-09T10:43:34.492904+0000","last_became_active":"2023-10-09T10:41:29.512895+0000","last_became_peered":"2023-10-09T10:41:29.512895+0000","last_unstale":"2023-10-09T10:43:34.492904+0000","last_undegraded":"2023-10-09T10:43:34.492904+0000","last_fullsized":"2023-10-09T10:43:34.492904+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:06.177976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489173+0000","last_change":"2023-10-09T10:43:06.671356+0000","last_active":"2023-10-09T10:43:34.489173+0000","last_peered":"2023-10-09T10:43:34.489173+0000","last_clean":"2023-10-09T10:43:34.489173+0000","last_became_active":"2023-10-09T10:43:06.671040+0000","last_became_peered":"2023-10-09T10:43:06.671040+0000","last_unstale":"2023-10-09T10:43:34.489173+0000","last_undegraded":"2023-10-09T10:43:34.489173+0000","last_fullsized":"2023-10-09T10:43:34.489173+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:47.601448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486509+0000","last_change":"2023-10-09T10:41:29.512225+0000","last_active":"2023-10-09T10:43:34.486509+0000","last_peered":"2023-10-09T10:43:34.486509+0000","last_clean":"2023-10-09T10:43:34.486509+0000","last_became_active":"2023-10-09T10:41:29.511907+0000","last_became_peered":"2023-10-09T10:41:29.511907+0000","last_unstale":"2023-10-09T10:43:34.486509+0000","last_undegraded":"2023-10-09T10:43:34.486509+0000","last_fullsized":"2023-10-09T10:43:34.486509+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:25.351050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483811+0000","last_change":"2023-10-09T10:43:06.673430+0000","last_active":"2023-10-09T10:43:35.483811+0000","last_peered":"2023-10-09T10:43:35.483811+0000","last_clean":"2023-10-09T10:43:35.483811+0000","last_became_active":"2023-10-09T10:43:06.673110+0000","last_became_peered":"2023-10-09T10:43:06.673110+0000","last_unstale":"2023-10-09T10:43:35.483811+0000","last_undegraded":"2023-10-09T10:43:35.483811+0000","last_fullsized":"2023-10-09T10:43:35.483811+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:35.429715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491431+0000","last_change":"2023-10-09T10:43:35.491431+0000","last_active":"2023-10-09T10:43:35.491431+0000","last_peered":"2023-10-09T10:43:35.491431+0000","last_clean":"2023-10-09T10:43:35.491431+0000","last_became_active":"2023-10-09T10:43:35.491153+0000","last_became_peered":"2023-10-09T10:43:35.491153+0000","last_unstale":"2023-10-09T10:43:35.491431+0000","last_undegraded":"2023-10-09T10:43:35.491431+0000","last_fullsized":"2023-10-09T10:43:35.491431+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:58.495730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":82,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485088+0000","last_change":"2023-10-09T10:42:38.417090+0000","last_active":"2023-10-09T10:43:34.485088+0000","last_peered":"2023-10-09T10:43:34.485088+0000","last_clean":"2023-10-09T10:43:34.485088+0000","last_became_active":"2023-10-09T10:42:38.416758+0000","last_became_peered":"2023-10-09T10:42:38.416758+0000","last_unstale":"2023-10-09T10:43:34.485088+0000","last_undegraded":"2023-10-09T10:43:34.485088+0000","last_fullsized":"2023-10-09T10:43:34.485088+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:52.373940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482379+0000","last_change":"2023-10-09T10:35:57.165395+0000","last_active":"2023-10-09T10:43:34.482379+0000","last_peered":"2023-10-09T10:43:34.482379+0000","last_clean":"2023-10-09T10:43:34.482379+0000","last_became_active":"2023-10-09T10:35:57.165077+0000","last_became_peered":"2023-10-09T10:35:57.165077+0000","last_unstale":"2023-10-09T10:43:34.482379+0000","last_undegraded":"2023-10-09T10:43:34.482379+0000","last_fullsized":"2023-10-09T10:43:34.482379+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:53.159616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.485969+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:43:35.485969+0000","last_peered":"2023-10-09T10:43:35.485969+0000","last_clean":"2023-10-09T10:43:35.485969+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:43:35.485969+0000","last_undegraded":"2023-10-09T10:43:35.485969+0000","last_fullsized":"2023-10-09T10:43:35.485969+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":441,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.484997+0000","last_change":"2023-10-09T10:41:29.515426+0000","last_active":"2023-10-09T10:43:34.484997+0000","last_peered":"2023-10-09T10:43:34.484997+0000","last_clean":"2023-10-09T10:43:34.484997+0000","last_became_active":"2023-10-09T10:41:29.515012+0000","last_became_peered":"2023-10-09T10:41:29.515012+0000","last_unstale":"2023-10-09T10:43:34.484997+0000","last_undegraded":"2023-10-09T10:43:34.484997+0000","last_fullsized":"2023-10-09T10:43:34.484997+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:23.973968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479081+0000","last_change":"2023-10-09T10:43:06.673931+0000","last_active":"2023-10-09T10:43:34.479081+0000","last_peered":"2023-10-09T10:43:34.479081+0000","last_clean":"2023-10-09T10:43:34.479081+0000","last_became_active":"2023-10-09T10:43:06.673605+0000","last_became_peered":"2023-10-09T10:43:06.673605+0000","last_unstale":"2023-10-09T10:43:34.479081+0000","last_undegraded":"2023-10-09T10:43:34.479081+0000","last_fullsized":"2023-10-09T10:43:34.479081+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:19.088672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":67,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489950+0000","last_change":"2023-10-09T10:42:38.406633+0000","last_active":"2023-10-09T10:43:34.489950+0000","last_peered":"2023-10-09T10:43:34.489950+0000","last_clean":"2023-10-09T10:43:34.489950+0000","last_became_active":"2023-10-09T10:42:38.406317+0000","last_became_peered":"2023-10-09T10:42:38.406317+0000","last_unstale":"2023-10-09T10:43:34.489950+0000","last_undegraded":"2023-10-09T10:43:34.489950+0000","last_fullsized":"2023-10-09T10:43:34.489950+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:02.600240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487885+0000","last_change":"2023-10-09T10:43:06.672862+0000","last_active":"2023-10-09T10:43:34.487885+0000","last_peered":"2023-10-09T10:43:34.487885+0000","last_clean":"2023-10-09T10:43:34.487885+0000","last_became_active":"2023-10-09T10:43:06.672572+0000","last_became_peered":"2023-10-09T10:43:06.672572+0000","last_unstale":"2023-10-09T10:43:34.487885+0000","last_undegraded":"2023-10-09T10:43:34.487885+0000","last_fullsized":"2023-10-09T10:43:34.487885+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:23.743910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":83,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.490168+0000","last_change":"2023-10-09T10:43:35.490168+0000","last_active":"2023-10-09T10:43:35.490168+0000","last_peered":"2023-10-09T10:43:35.490168+0000","last_clean":"2023-10-09T10:43:35.490168+0000","last_became_active":"2023-10-09T10:43:35.489834+0000","last_became_peered":"2023-10-09T10:43:35.489834+0000","last_unstale":"2023-10-09T10:43:35.490168+0000","last_undegraded":"2023-10-09T10:43:35.490168+0000","last_fullsized":"2023-10-09T10:43:35.490168+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:59:26.252052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.489332+0000","last_change":"2023-10-09T10:43:35.489332+0000","last_active":"2023-10-09T10:43:35.489332+0000","last_peered":"2023-10-09T10:43:35.489332+0000","last_clean":"2023-10-09T10:43:35.489332+0000","last_became_active":"2023-10-09T10:43:35.489044+0000","last_became_peered":"2023-10-09T10:43:35.489044+0000","last_unstale":"2023-10-09T10:43:35.489332+0000","last_undegraded":"2023-10-09T10:43:35.489332+0000","last_fullsized":"2023-10-09T10:43:35.489332+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:42.168915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487678+0000","last_change":"2023-10-09T10:41:29.511298+0000","last_active":"2023-10-09T10:43:34.487678+0000","last_peered":"2023-10-09T10:43:34.487678+0000","last_clean":"2023-10-09T10:43:34.487678+0000","last_became_active":"2023-10-09T10:41:29.510978+0000","last_became_peered":"2023-10-09T10:41:29.510978+0000","last_unstale":"2023-10-09T10:43:34.487678+0000","last_undegraded":"2023-10-09T10:43:34.487678+0000","last_fullsized":"2023-10-09T10:43:34.487678+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:20:11.933459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479143+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:43:34.479143+0000","last_peered":"2023-10-09T10:43:34.479143+0000","last_clean":"2023-10-09T10:43:34.479143+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:43:34.479143+0000","last_undegraded":"2023-10-09T10:43:34.479143+0000","last_fullsized":"2023-10-09T10:43:34.479143+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486160+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:43:34.486160+0000","last_peered":"2023-10-09T10:43:34.486160+0000","last_clean":"2023-10-09T10:43:34.486160+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:43:34.486160+0000","last_undegraded":"2023-10-09T10:43:34.486160+0000","last_fullsized":"2023-10-09T10:43:34.486160+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":78,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486524+0000","last_change":"2023-10-09T10:43:35.486524+0000","last_active":"2023-10-09T10:43:35.486524+0000","last_peered":"2023-10-09T10:43:35.486524+0000","last_clean":"2023-10-09T10:43:35.486524+0000","last_became_active":"2023-10-09T10:43:35.486152+0000","last_became_peered":"2023-10-09T10:43:35.486152+0000","last_unstale":"2023-10-09T10:43:35.486524+0000","last_undegraded":"2023-10-09T10:43:35.486524+0000","last_fullsized":"2023-10-09T10:43:35.486524+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:47.200522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":72,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489284+0000","last_change":"2023-10-09T10:42:38.416111+0000","last_active":"2023-10-09T10:43:34.489284+0000","last_peered":"2023-10-09T10:43:34.489284+0000","last_clean":"2023-10-09T10:43:34.489284+0000","last_became_active":"2023-10-09T10:42:38.415861+0000","last_became_peered":"2023-10-09T10:42:38.415861+0000","last_unstale":"2023-10-09T10:43:34.489284+0000","last_undegraded":"2023-10-09T10:43:34.489284+0000","last_fullsized":"2023-10-09T10:43:34.489284+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:10:40.572274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.489126+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:43:35.489126+0000","last_peered":"2023-10-09T10:43:35.489126+0000","last_clean":"2023-10-09T10:43:35.489126+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:43:35.489126+0000","last_undegraded":"2023-10-09T10:43:35.489126+0000","last_fullsized":"2023-10-09T10:43:35.489126+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":64,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486030+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:43:35.486030+0000","last_peered":"2023-10-09T10:43:35.486030+0000","last_clean":"2023-10-09T10:43:35.486030+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:43:35.486030+0000","last_undegraded":"2023-10-09T10:43:35.486030+0000","last_fullsized":"2023-10-09T10:43:35.486030+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486884+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:43:34.486884+0000","last_peered":"2023-10-09T10:43:34.486884+0000","last_clean":"2023-10-09T10:43:34.486884+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:43:34.486884+0000","last_undegraded":"2023-10-09T10:43:34.486884+0000","last_fullsized":"2023-10-09T10:43:34.486884+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485938+0000","last_change":"2023-10-09T10:43:06.676577+0000","last_active":"2023-10-09T10:43:34.485938+0000","last_peered":"2023-10-09T10:43:34.485938+0000","last_clean":"2023-10-09T10:43:34.485938+0000","last_became_active":"2023-10-09T10:43:06.676165+0000","last_became_peered":"2023-10-09T10:43:06.676165+0000","last_unstale":"2023-10-09T10:43:34.485938+0000","last_undegraded":"2023-10-09T10:43:34.485938+0000","last_fullsized":"2023-10-09T10:43:34.485938+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:52.614985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":77,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487389+0000","last_change":"2023-10-09T10:43:06.676367+0000","last_active":"2023-10-09T10:43:34.487389+0000","last_peered":"2023-10-09T10:43:34.487389+0000","last_clean":"2023-10-09T10:43:34.487389+0000","last_became_active":"2023-10-09T10:43:06.676017+0000","last_became_peered":"2023-10-09T10:43:06.676017+0000","last_unstale":"2023-10-09T10:43:34.487389+0000","last_undegraded":"2023-10-09T10:43:34.487389+0000","last_fullsized":"2023-10-09T10:43:34.487389+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:54.387617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482664+0000","last_change":"2023-10-09T10:42:38.412169+0000","last_active":"2023-10-09T10:43:34.482664+0000","last_peered":"2023-10-09T10:43:34.482664+0000","last_clean":"2023-10-09T10:43:34.482664+0000","last_became_active":"2023-10-09T10:42:38.411829+0000","last_became_peered":"2023-10-09T10:42:38.411829+0000","last_unstale":"2023-10-09T10:43:34.482664+0000","last_undegraded":"2023-10-09T10:43:34.482664+0000","last_fullsized":"2023-10-09T10:43:34.482664+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:36:35.699293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.493287+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:43:35.493287+0000","last_peered":"2023-10-09T10:43:35.493287+0000","last_clean":"2023-10-09T10:43:35.493287+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:43:35.493287+0000","last_undegraded":"2023-10-09T10:43:35.493287+0000","last_fullsized":"2023-10-09T10:43:35.493287+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":61,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491435+0000","last_change":"2023-10-09T10:43:35.491435+0000","last_active":"2023-10-09T10:43:35.491435+0000","last_peered":"2023-10-09T10:43:35.491435+0000","last_clean":"2023-10-09T10:43:35.491435+0000","last_became_active":"2023-10-09T10:43:35.491096+0000","last_became_peered":"2023-10-09T10:43:35.491096+0000","last_unstale":"2023-10-09T10:43:35.491435+0000","last_undegraded":"2023-10-09T10:43:35.491435+0000","last_fullsized":"2023-10-09T10:43:35.491435+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:09.581525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483788+0000","last_change":"2023-10-09T10:42:38.407703+0000","last_active":"2023-10-09T10:43:35.483788+0000","last_peered":"2023-10-09T10:43:35.483788+0000","last_clean":"2023-10-09T10:43:35.483788+0000","last_became_active":"2023-10-09T10:42:38.407431+0000","last_became_peered":"2023-10-09T10:42:38.407431+0000","last_unstale":"2023-10-09T10:43:35.483788+0000","last_undegraded":"2023-10-09T10:43:35.483788+0000","last_fullsized":"2023-10-09T10:43:35.483788+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:23.450634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487326+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:43:35.487326+0000","last_peered":"2023-10-09T10:43:35.487326+0000","last_clean":"2023-10-09T10:43:35.487326+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:43:35.487326+0000","last_undegraded":"2023-10-09T10:43:35.487326+0000","last_fullsized":"2023-10-09T10:43:35.487326+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482266+0000","last_change":"2023-10-09T10:42:38.411928+0000","last_active":"2023-10-09T10:43:34.482266+0000","last_peered":"2023-10-09T10:43:34.482266+0000","last_clean":"2023-10-09T10:43:34.482266+0000","last_became_active":"2023-10-09T10:42:38.411560+0000","last_became_peered":"2023-10-09T10:42:38.411560+0000","last_unstale":"2023-10-09T10:43:34.482266+0000","last_undegraded":"2023-10-09T10:43:34.482266+0000","last_fullsized":"2023-10-09T10:43:34.482266+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:07.711753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":45,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487828+0000","last_change":"2023-10-09T10:35:56.822899+0000","last_active":"2023-10-09T10:43:34.487828+0000","last_peered":"2023-10-09T10:43:34.487828+0000","last_clean":"2023-10-09T10:43:34.487828+0000","last_became_active":"2023-10-09T10:35:56.822564+0000","last_became_peered":"2023-10-09T10:35:56.822564+0000","last_unstale":"2023-10-09T10:43:34.487828+0000","last_undegraded":"2023-10-09T10:43:34.487828+0000","last_fullsized":"2023-10-09T10:43:34.487828+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:25:03.398416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.493278+0000","last_change":"2023-10-09T10:43:35.493278+0000","last_active":"2023-10-09T10:43:35.493278+0000","last_peered":"2023-10-09T10:43:35.493278+0000","last_clean":"2023-10-09T10:43:35.493278+0000","last_became_active":"2023-10-09T10:43:35.492915+0000","last_became_peered":"2023-10-09T10:43:35.492915+0000","last_unstale":"2023-10-09T10:43:35.493278+0000","last_undegraded":"2023-10-09T10:43:35.493278+0000","last_fullsized":"2023-10-09T10:43:35.493278+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:42.227364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487004+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:43:34.487004+0000","last_peered":"2023-10-09T10:43:34.487004+0000","last_clean":"2023-10-09T10:43:34.487004+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:43:34.487004+0000","last_undegraded":"2023-10-09T10:43:34.487004+0000","last_fullsized":"2023-10-09T10:43:34.487004+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479053+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:43:34.479053+0000","last_peered":"2023-10-09T10:43:34.479053+0000","last_clean":"2023-10-09T10:43:34.479053+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:43:34.479053+0000","last_undegraded":"2023-10-09T10:43:34.479053+0000","last_fullsized":"2023-10-09T10:43:34.479053+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487772+0000","last_change":"2023-10-09T10:41:29.516129+0000","last_active":"2023-10-09T10:43:34.487772+0000","last_peered":"2023-10-09T10:43:34.487772+0000","last_clean":"2023-10-09T10:43:34.487772+0000","last_became_active":"2023-10-09T10:41:29.515741+0000","last_became_peered":"2023-10-09T10:41:29.515741+0000","last_unstale":"2023-10-09T10:43:34.487772+0000","last_undegraded":"2023-10-09T10:43:34.487772+0000","last_fullsized":"2023-10-09T10:43:34.487772+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:29:14.059627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":63,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487647+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:43:35.487647+0000","last_peered":"2023-10-09T10:43:35.487647+0000","last_clean":"2023-10-09T10:43:35.487647+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:43:35.487647+0000","last_undegraded":"2023-10-09T10:43:35.487647+0000","last_fullsized":"2023-10-09T10:43:35.487647+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":69,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.481806+0000","last_change":"2023-10-09T10:41:29.512929+0000","last_active":"2023-10-09T10:43:34.481806+0000","last_peered":"2023-10-09T10:43:34.481806+0000","last_clean":"2023-10-09T10:43:34.481806+0000","last_became_active":"2023-10-09T10:41:29.512591+0000","last_became_peered":"2023-10-09T10:41:29.512591+0000","last_unstale":"2023-10-09T10:43:34.481806+0000","last_undegraded":"2023-10-09T10:43:34.481806+0000","last_fullsized":"2023-10-09T10:43:34.481806+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:35.703322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.490293+0000","last_change":"2023-10-09T10:43:35.487796+0000","last_active":"2023-10-09T10:43:35.490293+0000","last_peered":"2023-10-09T10:43:35.490293+0000","last_clean":"2023-10-09T10:43:35.490293+0000","last_became_active":"2023-10-09T10:43:35.487518+0000","last_became_peered":"2023-10-09T10:43:35.487518+0000","last_unstale":"2023-10-09T10:43:35.490293+0000","last_undegraded":"2023-10-09T10:43:35.490293+0000","last_fullsized":"2023-10-09T10:43:35.490293+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:47.089685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":44,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.484279+0000","last_change":"2023-10-09T10:35:56.824257+0000","last_active":"2023-10-09T10:43:34.484279+0000","last_peered":"2023-10-09T10:43:34.484279+0000","last_clean":"2023-10-09T10:43:34.484279+0000","last_became_active":"2023-10-09T10:35:56.823152+0000","last_became_peered":"2023-10-09T10:35:56.823152+0000","last_unstale":"2023-10-09T10:43:34.484279+0000","last_undegraded":"2023-10-09T10:43:34.484279+0000","last_fullsized":"2023-10-09T10:43:34.484279+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:52.392494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486785+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:43:35.486785+0000","last_peered":"2023-10-09T10:43:35.486785+0000","last_clean":"2023-10-09T10:43:35.486785+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:43:35.486785+0000","last_undegraded":"2023-10-09T10:43:35.486785+0000","last_fullsized":"2023-10-09T10:43:35.486785+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487384+0000","last_change":"2023-10-09T10:43:35.486914+0000","last_active":"2023-10-09T10:43:35.487384+0000","last_peered":"2023-10-09T10:43:35.487384+0000","last_clean":"2023-10-09T10:43:35.487384+0000","last_became_active":"2023-10-09T10:43:35.486623+0000","last_became_peered":"2023-10-09T10:43:35.486623+0000","last_unstale":"2023-10-09T10:43:35.487384+0000","last_undegraded":"2023-10-09T10:43:35.487384+0000","last_fullsized":"2023-10-09T10:43:35.487384+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:13.339801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":77,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487942+0000","last_change":"2023-10-09T10:43:35.487942+0000","last_active":"2023-10-09T10:43:35.487942+0000","last_peered":"2023-10-09T10:43:35.487942+0000","last_clean":"2023-10-09T10:43:35.487942+0000","last_became_active":"2023-10-09T10:43:35.487651+0000","last_became_peered":"2023-10-09T10:43:35.487651+0000","last_unstale":"2023-10-09T10:43:35.487942+0000","last_undegraded":"2023-10-09T10:43:35.487942+0000","last_fullsized":"2023-10-09T10:43:35.487942+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:51.882595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485380+0000","last_change":"2023-10-09T10:43:06.677115+0000","last_active":"2023-10-09T10:43:34.485380+0000","last_peered":"2023-10-09T10:43:34.485380+0000","last_clean":"2023-10-09T10:43:34.485380+0000","last_became_active":"2023-10-09T10:43:06.676739+0000","last_became_peered":"2023-10-09T10:43:06.676739+0000","last_unstale":"2023-10-09T10:43:34.485380+0000","last_undegraded":"2023-10-09T10:43:34.485380+0000","last_fullsized":"2023-10-09T10:43:34.485380+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:02.520770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487176+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:43:34.487176+0000","last_peered":"2023-10-09T10:43:34.487176+0000","last_clean":"2023-10-09T10:43:34.487176+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:43:34.487176+0000","last_undegraded":"2023-10-09T10:43:34.487176+0000","last_fullsized":"2023-10-09T10:43:34.487176+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":66,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486392+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:43:34.486392+0000","last_peered":"2023-10-09T10:43:34.486392+0000","last_clean":"2023-10-09T10:43:34.486392+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:43:34.486392+0000","last_undegraded":"2023-10-09T10:43:34.486392+0000","last_fullsized":"2023-10-09T10:43:34.486392+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486237+0000","last_change":"2023-10-09T10:43:06.672114+0000","last_active":"2023-10-09T10:43:34.486237+0000","last_peered":"2023-10-09T10:43:34.486237+0000","last_clean":"2023-10-09T10:43:34.486237+0000","last_became_active":"2023-10-09T10:43:06.671827+0000","last_became_peered":"2023-10-09T10:43:06.671827+0000","last_unstale":"2023-10-09T10:43:34.486237+0000","last_undegraded":"2023-10-09T10:43:34.486237+0000","last_fullsized":"2023-10-09T10:43:34.486237+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:41.701409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":64,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491382+0000","last_change":"2023-10-09T10:43:35.491382+0000","last_active":"2023-10-09T10:43:35.491382+0000","last_peered":"2023-10-09T10:43:35.491382+0000","last_clean":"2023-10-09T10:43:35.491382+0000","last_became_active":"2023-10-09T10:43:35.491062+0000","last_became_peered":"2023-10-09T10:43:35.491062+0000","last_unstale":"2023-10-09T10:43:35.491382+0000","last_undegraded":"2023-10-09T10:43:35.491382+0000","last_fullsized":"2023-10-09T10:43:35.491382+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:10.396267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487546+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:43:35.487546+0000","last_peered":"2023-10-09T10:43:35.487546+0000","last_clean":"2023-10-09T10:43:35.487546+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:43:35.487546+0000","last_undegraded":"2023-10-09T10:43:35.487546+0000","last_fullsized":"2023-10-09T10:43:35.487546+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479334+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:43:34.479334+0000","last_peered":"2023-10-09T10:43:34.479334+0000","last_clean":"2023-10-09T10:43:34.479334+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:43:34.479334+0000","last_undegraded":"2023-10-09T10:43:34.479334+0000","last_fullsized":"2023-10-09T10:43:34.479334+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":63,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.484250+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:43:35.484250+0000","last_peered":"2023-10-09T10:43:35.484250+0000","last_clean":"2023-10-09T10:43:35.484250+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:43:35.484250+0000","last_undegraded":"2023-10-09T10:43:35.484250+0000","last_fullsized":"2023-10-09T10:43:35.484250+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.488427+0000","last_change":"2023-10-09T10:43:35.487114+0000","last_active":"2023-10-09T10:43:35.488427+0000","last_peered":"2023-10-09T10:43:35.488427+0000","last_clean":"2023-10-09T10:43:35.488427+0000","last_became_active":"2023-10-09T10:43:35.486830+0000","last_became_peered":"2023-10-09T10:43:35.486830+0000","last_unstale":"2023-10-09T10:43:35.488427+0000","last_undegraded":"2023-10-09T10:43:35.488427+0000","last_fullsized":"2023-10-09T10:43:35.488427+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:08.836969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482547+0000","last_change":"2023-10-09T10:42:38.413301+0000","last_active":"2023-10-09T10:43:34.482547+0000","last_peered":"2023-10-09T10:43:34.482547+0000","last_clean":"2023-10-09T10:43:34.482547+0000","last_became_active":"2023-10-09T10:42:38.413006+0000","last_became_peered":"2023-10-09T10:42:38.413006+0000","last_unstale":"2023-10-09T10:43:34.482547+0000","last_undegraded":"2023-10-09T10:43:34.482547+0000","last_fullsized":"2023-10-09T10:43:34.482547+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:33.462419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":81,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.485235+0000","last_change":"2023-10-09T10:43:35.485235+0000","last_active":"2023-10-09T10:43:35.485235+0000","last_peered":"2023-10-09T10:43:35.485235+0000","last_clean":"2023-10-09T10:43:35.485235+0000","last_became_active":"2023-10-09T10:43:35.484917+0000","last_became_peered":"2023-10-09T10:43:35.484917+0000","last_unstale":"2023-10-09T10:43:35.485235+0000","last_undegraded":"2023-10-09T10:43:35.485235+0000","last_fullsized":"2023-10-09T10:43:35.485235+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:55.417326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487224+0000","last_change":"2023-10-09T10:43:35.487224+0000","last_active":"2023-10-09T10:43:35.487224+0000","last_peered":"2023-10-09T10:43:35.487224+0000","last_clean":"2023-10-09T10:43:35.487224+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:43:35.487224+0000","last_undegraded":"2023-10-09T10:43:35.487224+0000","last_fullsized":"2023-10-09T10:43:35.487224+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:44:52.066778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491388+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:43:35.491388+0000","last_peered":"2023-10-09T10:43:35.491388+0000","last_clean":"2023-10-09T10:43:35.491388+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:43:35.491388+0000","last_undegraded":"2023-10-09T10:43:35.491388+0000","last_fullsized":"2023-10-09T10:43:35.491388+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":63,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.490890+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:43:35.490890+0000","last_peered":"2023-10-09T10:43:35.490890+0000","last_clean":"2023-10-09T10:43:35.490890+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:43:35.490890+0000","last_undegraded":"2023-10-09T10:43:35.490890+0000","last_fullsized":"2023-10-09T10:43:35.490890+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":79,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486285+0000","last_change":"2023-10-09T10:43:06.676414+0000","last_active":"2023-10-09T10:43:34.486285+0000","last_peered":"2023-10-09T10:43:34.486285+0000","last_clean":"2023-10-09T10:43:34.486285+0000","last_became_active":"2023-10-09T10:43:06.676050+0000","last_became_peered":"2023-10-09T10:43:06.676050+0000","last_unstale":"2023-10-09T10:43:34.486285+0000","last_undegraded":"2023-10-09T10:43:34.486285+0000","last_fullsized":"2023-10-09T10:43:34.486285+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:28.011477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332210,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311116,"kb_used_data":1540,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93434036,"statfs":{"total":95995035648,"available":95676452864,"internally_reserved":0,"allocated":1576960,"data_stored":1120859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:31 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.56399999999999995,"15min":0.57399999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.67200000000000004,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.61799999999999999,"15min":0.63400000000000001},"min":{"1min":0.36499999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.69799999999999995,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.625}]},{"osd":1,"last update":"Mon Oct 9 10:43:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.46600000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:42:10 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.55600000000000005,"15min":0.53100000000000003},"min":{"1min":0.39500000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.80200000000000005,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.57599999999999996,"15min":0.54900000000000004},"min":{"1min":0.41399999999999998,"5min":0.36599999999999999,"15min":0.27300000000000002},"max":{"1min":0.82499999999999996,"5min":1.2609999999999999,"15min":1.2609999999999999},"last":0.52400000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.58399999999999996,"15min":0.56699999999999995},"min":{"1min":0.41399999999999998,"5min":0.35099999999999998,"15min":0.33800000000000002},"max":{"1min":0.74099999999999999,"5min":2.569,"15min":2.569},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59099999999999997,"15min":0.57399999999999995},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.76900000000000002,"5min":0.80100000000000005,"15min":1.232},"last":0.54900000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48599999999999999,"15min":0.47299999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.73999999999999999,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.502,"5min":0.497,"15min":0.47399999999999998},"min":{"1min":0.36899999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.67100000000000004,"5min":0.80600000000000005,"15min":0.83699999999999997},"last":0.49399999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.52100000000000002,"15min":0.504},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":1.1950000000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.55300000000000005,"15min":0.54000000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.25700000000000001},"max":{"1min":0.86099999999999999,"5min":0.876,"15min":1.0649999999999999},"last":0.69799999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58199999999999996,"15min":0.55700000000000005},"min":{"1min":0.40500000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.81699999999999995,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.57399999999999995,"15min":0.54300000000000004},"min":{"1min":0.39000000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.79700000000000004,"5min":0.89200000000000002,"15min":0.93500000000000005},"last":0.59599999999999997}]}]},{"osd":6,"up_from":44,"seq":188978561141,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298816,"kb_used_data":1528,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446336,"statfs":{"total":95995035648,"available":95689048064,"internally_reserved":0,"allocated":1564672,"data_stored":1120713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:34 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.59699999999999998,"15min":0.60599999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.73999999999999999,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.63,"5min":0.61199999999999999,"15min":0.60899999999999999},"min":{"1min":0.46600000000000003,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.64300000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.57799999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.52500000000000002,"15min":0.501},"min":{"1min":0.39500000000000002,"5min":0.35899999999999999,"15min":0.26600000000000001},"max":{"1min":0.66700000000000004,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.54400000000000004,"15min":0.52900000000000003},"min":{"1min":0.36399999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.77400000000000002,"5min":0.86299999999999999,"15min":1.0109999999999999},"last":0.52800000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56100000000000005,"15min":0.52200000000000002},"min":{"1min":0.375,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72499999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54700000000000004,"15min":0.53200000000000003},"min":{"1min":0.39800000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80400000000000005},"last":0.47499999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.45900000000000002,"15min":0.44400000000000001},"min":{"1min":0.38900000000000001,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":0.59699999999999998,"5min":0.67400000000000004,"15min":0.70999999999999996},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.505,"15min":0.49199999999999999},"min":{"1min":0.35299999999999998,"5min":0.29899999999999999,"15min":0.29699999999999999},"max":{"1min":0.74399999999999999,"5min":0.77000000000000002,"15min":0.78500000000000003},"last":0.45100000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.51500000000000001,"15min":0.48399999999999999},"min":{"1min":0.40100000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.67600000000000005,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.55000000000000004,"15min":0.53000000000000003},"min":{"1min":0.372,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.84399999999999997},"last":0.59199999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:43:18 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.53100000000000003,"15min":0.52600000000000002},"min":{"1min":0.33900000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.72599999999999998,"5min":0.78600000000000003,"15min":0.79100000000000004},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.438,"5min":0.34100000000000003,"15min":0.34000000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":1.018},"last":0.621}]}]},{"osd":1,"up_from":85,"seq":365072220180,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28448,"kb_used_data":984,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716704,"statfs":{"total":95995035648,"available":95965904896,"internally_reserved":0,"allocated":1007616,"data_stored":536014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.56000000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58699999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57799999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.98899999999999999},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.83199999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.78500000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.88200000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":1.0369999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.90000000000000002}]}]},{"osd":0,"up_from":80,"seq":343597383713,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29108,"kb_used_data":1580,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93716044,"statfs":{"total":95995035648,"available":95965229056,"internally_reserved":0,"allocated":1617920,"data_stored":1126957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:43:38 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.436},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.42199999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:42:30 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.48699999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:42:30 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.501}]},{"osd":4,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.61699999999999999,"15min":0.61899999999999999},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61699999999999999,"15min":0.61799999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.83499999999999996,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.70499999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.62,"15min":0.624},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.749,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.625,"5min":0.64400000000000002,"15min":0.64700000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.60999999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.628,"15min":0.63},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.63700000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.64700000000000002,"15min":0.65100000000000002},"min":{"1min":0.44600000000000001,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.66900000000000004,"15min":0.67500000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.68500000000000005}]}]},{"osd":2,"up_from":90,"seq":386547056654,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28572,"kb_used_data":980,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716580,"statfs":{"total":95995035648,"available":95965777920,"internally_reserved":0,"allocated":1003520,"data_stored":533482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35499999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.371}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81100000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76800000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56200000000000006}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51200000000000001}]}]},{"osd":3,"up_from":95,"seq":408021893128,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28424,"kb_used_data":960,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716728,"statfs":{"total":95995035648,"available":95965929472,"internally_reserved":0,"allocated":983040,"data_stored":530397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46100000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.372}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44900000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48499999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61399999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58799999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60499999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019004,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302344,"kb_used_data":960,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93442808,"statfs":{"total":95995035648,"available":95685435392,"internally_reserved":0,"allocated":983040,"data_stored":530744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.55500000000000005,"15min":0.55700000000000005},"min":{"1min":0.35199999999999998,"5min":0.311,"15min":0.311},"max":{"1min":0.74299999999999999,"5min":0.755,"15min":0.755},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.57899999999999996,"15min":0.58299999999999996},"min":{"1min":0.41599999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.747,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.49299999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.53800000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:42:21 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.54100000000000004,"15min":0.55200000000000005},"min":{"1min":0.34799999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.78300000000000003,"5min":0.82999999999999996,"15min":2.3530000000000002},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.56599999999999995,"15min":0.57199999999999995},"min":{"1min":0.35899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.80800000000000005,"5min":0.86499999999999999,"15min":0.875},"last":0.65100000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:43:21 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.58499999999999996,"15min":0.56200000000000006},"min":{"1min":0.45700000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.84799999999999998,"5min":0.85099999999999998,"15min":0.86399999999999999},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.57299999999999995,"15min":0.56799999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.83699999999999997,"5min":0.86599999999999999,"15min":2.0910000000000002},"last":0.71699999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.47399999999999998,"15min":0.438},"min":{"1min":0.34300000000000003,"5min":0.253,"15min":0.253},"max":{"1min":0.65000000000000002,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.48199999999999998,"15min":0.442},"min":{"1min":0.34599999999999997,"5min":0.28499999999999998,"15min":0.26400000000000001},"max":{"1min":0.73299999999999998,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.68999999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.50800000000000001,"15min":0.46899999999999997},"min":{"1min":0.32000000000000001,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":0.752,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59499999999999997,"15min":0.56000000000000005},"min":{"1min":0.46500000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.82299999999999995,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.67200000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:43:11 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.57399999999999995,"15min":0.56899999999999995},"min":{"1min":0.376,"5min":0.34999999999999998,"15min":0.308},"max":{"1min":0.872,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.626,"5min":0.56899999999999995,"15min":0.53800000000000003},"min":{"1min":0.32500000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.80700000000000005,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.82799999999999996}]}]},{"osd":5,"up_from":37,"seq":158913790073,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306420,"kb_used_data":940,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93438732,"statfs":{"total":95995035648,"available":95681261568,"internally_reserved":0,"allocated":962560,"data_stored":526165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.66200000000000003,"15min":0.66700000000000004},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.61399999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.63600000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:44:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.497},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.48799999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:42:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.54500000000000004,"15min":0.52600000000000002},"min":{"1min":0.39600000000000002,"5min":0.36899999999999999,"15min":0.35699999999999998},"max":{"1min":0.76100000000000001,"5min":0.79000000000000004,"15min":2.976},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.59199999999999997,"15min":0.59099999999999997},"min":{"1min":0.44700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.78400000000000003,"5min":0.92400000000000004,"15min":4.7949999999999999},"last":0.45900000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47599999999999998,"15min":0.43099999999999999},"min":{"1min":0.35399999999999998,"5min":0.30499999999999999,"15min":0.25},"max":{"1min":0.69099999999999995,"5min":0.75600000000000001,"15min":2.4649999999999999},"last":0.372},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.5,"15min":0.45100000000000001},"min":{"1min":0.371,"5min":0.30099999999999999,"15min":0.28799999999999998},"max":{"1min":0.60199999999999998,"5min":0.85299999999999998,"15min":1.98},"last":0.51600000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.53200000000000003,"15min":0.45800000000000002},"min":{"1min":0.39300000000000002,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":0.67500000000000004,"5min":0.85599999999999998,"15min":2.048},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56599999999999995,"15min":0.49099999999999999},"min":{"1min":0.47099999999999997,"5min":0.33500000000000002,"15min":0.30299999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.42399999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:43:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.57199999999999995,"15min":0.53500000000000003},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.32400000000000001},"max":{"1min":0.80700000000000005,"5min":0.879,"15min":2.1110000000000002},"last":0.44},{"interface":"front","average":{"1min":0.627,"5min":0.59699999999999998,"15min":0.53600000000000003},"min":{"1min":0.48799999999999999,"5min":0.35699999999999998,"15min":0.31900000000000001},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":1.845},"last":0.54400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:44:08.095 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:44:08.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:08 smithi125 ceph-mon[195507]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:08.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:08 smithi125 ceph-mon[200197]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:08.359 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:08 smithi155 ceph-mon[184854]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:09.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:09 smithi155 ceph-mon[184854]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:44:09.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:09 smithi125 ceph-mon[195507]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:44:09.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:09 smithi125 ceph-mon[200197]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:44:09.610 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:44:09.611 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:44:09.972 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":185,"stamp":"2023-10-09T10:44:09.001604+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1333248,"kb_used_data":9472,"kb_used_omap":126,"kb_used_meta":1323585,"kb_avail":748627968,"statfs":{"total":767960285184,"available":766595039232,"internally_reserved":0,"allocated":9699328,"data_stored":6025331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1355351675},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004559"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485359+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:43:34.485359+0000","last_peered":"2023-10-09T10:43:34.485359+0000","last_clean":"2023-10-09T10:43:34.485359+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:43:34.485359+0000","last_undegraded":"2023-10-09T10:43:34.485359+0000","last_fullsized":"2023-10-09T10:43:34.485359+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":57,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.494552+0000","last_change":"2023-10-09T10:43:06.674549+0000","last_active":"2023-10-09T10:43:34.494552+0000","last_peered":"2023-10-09T10:43:34.494552+0000","last_clean":"2023-10-09T10:43:34.494552+0000","last_became_active":"2023-10-09T10:43:06.673510+0000","last_became_peered":"2023-10-09T10:43:06.673510+0000","last_unstale":"2023-10-09T10:43:34.494552+0000","last_undegraded":"2023-10-09T10:43:34.494552+0000","last_fullsized":"2023-10-09T10:43:34.494552+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:32.996795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":84,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487344+0000","last_change":"2023-10-09T10:43:35.487344+0000","last_active":"2023-10-09T10:43:35.487344+0000","last_peered":"2023-10-09T10:43:35.487344+0000","last_clean":"2023-10-09T10:43:35.487344+0000","last_became_active":"2023-10-09T10:43:35.486672+0000","last_became_peered":"2023-10-09T10:43:35.486672+0000","last_unstale":"2023-10-09T10:43:35.487344+0000","last_undegraded":"2023-10-09T10:43:35.487344+0000","last_fullsized":"2023-10-09T10:43:35.487344+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:44.665687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":69,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.485790+0000","last_change":"2023-10-09T10:43:35.485790+0000","last_active":"2023-10-09T10:43:35.485790+0000","last_peered":"2023-10-09T10:43:35.485790+0000","last_clean":"2023-10-09T10:43:35.485790+0000","last_became_active":"2023-10-09T10:43:35.485436+0000","last_became_peered":"2023-10-09T10:43:35.485436+0000","last_unstale":"2023-10-09T10:43:35.485790+0000","last_undegraded":"2023-10-09T10:43:35.485790+0000","last_fullsized":"2023-10-09T10:43:35.485790+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:45:20.897767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":77,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.492236+0000","last_change":"2023-10-09T10:43:35.492236+0000","last_active":"2023-10-09T10:43:35.492236+0000","last_peered":"2023-10-09T10:43:35.492236+0000","last_clean":"2023-10-09T10:43:35.492236+0000","last_became_active":"2023-10-09T10:43:35.491913+0000","last_became_peered":"2023-10-09T10:43:35.491913+0000","last_unstale":"2023-10-09T10:43:35.492236+0000","last_undegraded":"2023-10-09T10:43:35.492236+0000","last_fullsized":"2023-10-09T10:43:35.492236+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:34:13.448179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":46,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.480679+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:43:34.480679+0000","last_peered":"2023-10-09T10:43:34.480679+0000","last_clean":"2023-10-09T10:43:34.480679+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:43:34.480679+0000","last_undegraded":"2023-10-09T10:43:34.480679+0000","last_fullsized":"2023-10-09T10:43:34.480679+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":77,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.492737+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:43:35.492737+0000","last_peered":"2023-10-09T10:43:35.492737+0000","last_clean":"2023-10-09T10:43:35.492737+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:43:35.492737+0000","last_undegraded":"2023-10-09T10:43:35.492737+0000","last_fullsized":"2023-10-09T10:43:35.492737+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479260+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:43:34.479260+0000","last_peered":"2023-10-09T10:43:34.479260+0000","last_clean":"2023-10-09T10:43:34.479260+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:43:34.479260+0000","last_undegraded":"2023-10-09T10:43:34.479260+0000","last_fullsized":"2023-10-09T10:43:34.479260+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":82,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.489655+0000","last_change":"2023-10-09T10:43:35.489655+0000","last_active":"2023-10-09T10:43:35.489655+0000","last_peered":"2023-10-09T10:43:35.489655+0000","last_clean":"2023-10-09T10:43:35.489655+0000","last_became_active":"2023-10-09T10:43:35.489149+0000","last_became_peered":"2023-10-09T10:43:35.489149+0000","last_unstale":"2023-10-09T10:43:35.489655+0000","last_undegraded":"2023-10-09T10:43:35.489655+0000","last_fullsized":"2023-10-09T10:43:35.489655+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:30.598962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":42,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483874+0000","last_change":"2023-10-09T10:41:29.514075+0000","last_active":"2023-10-09T10:43:35.483874+0000","last_peered":"2023-10-09T10:43:35.483874+0000","last_clean":"2023-10-09T10:43:35.483874+0000","last_became_active":"2023-10-09T10:41:29.513347+0000","last_became_peered":"2023-10-09T10:41:29.513347+0000","last_unstale":"2023-10-09T10:43:35.483874+0000","last_undegraded":"2023-10-09T10:43:35.483874+0000","last_fullsized":"2023-10-09T10:43:35.483874+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:17.504888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485850+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:43:34.485850+0000","last_peered":"2023-10-09T10:43:34.485850+0000","last_clean":"2023-10-09T10:43:34.485850+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:43:34.485850+0000","last_undegraded":"2023-10-09T10:43:34.485850+0000","last_fullsized":"2023-10-09T10:43:34.485850+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":57,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483729+0000","last_change":"2023-10-09T10:42:38.407058+0000","last_active":"2023-10-09T10:43:35.483729+0000","last_peered":"2023-10-09T10:43:35.483729+0000","last_clean":"2023-10-09T10:43:35.483729+0000","last_became_active":"2023-10-09T10:42:38.406768+0000","last_became_peered":"2023-10-09T10:42:38.406768+0000","last_unstale":"2023-10-09T10:43:35.483729+0000","last_undegraded":"2023-10-09T10:43:35.483729+0000","last_fullsized":"2023-10-09T10:43:35.483729+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:41.198843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482345+0000","last_change":"2023-10-09T10:43:06.674387+0000","last_active":"2023-10-09T10:43:34.482345+0000","last_peered":"2023-10-09T10:43:34.482345+0000","last_clean":"2023-10-09T10:43:34.482345+0000","last_became_active":"2023-10-09T10:43:06.674032+0000","last_became_peered":"2023-10-09T10:43:06.674032+0000","last_unstale":"2023-10-09T10:43:34.482345+0000","last_undegraded":"2023-10-09T10:43:34.482345+0000","last_fullsized":"2023-10-09T10:43:34.482345+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:58.222034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":58,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486342+0000","last_change":"2023-10-09T10:43:35.486342+0000","last_active":"2023-10-09T10:43:35.486342+0000","last_peered":"2023-10-09T10:43:35.486342+0000","last_clean":"2023-10-09T10:43:35.486342+0000","last_became_active":"2023-10-09T10:43:35.486026+0000","last_became_peered":"2023-10-09T10:43:35.486026+0000","last_unstale":"2023-10-09T10:43:35.486342+0000","last_undegraded":"2023-10-09T10:43:35.486342+0000","last_fullsized":"2023-10-09T10:43:35.486342+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:16.616429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489374+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:43:34.489374+0000","last_peered":"2023-10-09T10:43:34.489374+0000","last_clean":"2023-10-09T10:43:34.489374+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:43:34.489374+0000","last_undegraded":"2023-10-09T10:43:34.489374+0000","last_fullsized":"2023-10-09T10:43:34.489374+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":61,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487842+0000","last_change":"2023-10-09T10:43:35.487842+0000","last_active":"2023-10-09T10:43:35.487842+0000","last_peered":"2023-10-09T10:43:35.487842+0000","last_clean":"2023-10-09T10:43:35.487842+0000","last_became_active":"2023-10-09T10:43:35.487192+0000","last_became_peered":"2023-10-09T10:43:35.487192+0000","last_unstale":"2023-10-09T10:43:35.487842+0000","last_undegraded":"2023-10-09T10:43:35.487842+0000","last_fullsized":"2023-10-09T10:43:35.487842+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:35.477280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.492904+0000","last_change":"2023-10-09T10:41:29.513802+0000","last_active":"2023-10-09T10:43:34.492904+0000","last_peered":"2023-10-09T10:43:34.492904+0000","last_clean":"2023-10-09T10:43:34.492904+0000","last_became_active":"2023-10-09T10:41:29.512895+0000","last_became_peered":"2023-10-09T10:41:29.512895+0000","last_unstale":"2023-10-09T10:43:34.492904+0000","last_undegraded":"2023-10-09T10:43:34.492904+0000","last_fullsized":"2023-10-09T10:43:34.492904+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:06.177976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489173+0000","last_change":"2023-10-09T10:43:06.671356+0000","last_active":"2023-10-09T10:43:34.489173+0000","last_peered":"2023-10-09T10:43:34.489173+0000","last_clean":"2023-10-09T10:43:34.489173+0000","last_became_active":"2023-10-09T10:43:06.671040+0000","last_became_peered":"2023-10-09T10:43:06.671040+0000","last_unstale":"2023-10-09T10:43:34.489173+0000","last_undegraded":"2023-10-09T10:43:34.489173+0000","last_fullsized":"2023-10-09T10:43:34.489173+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:47.601448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486509+0000","last_change":"2023-10-09T10:41:29.512225+0000","last_active":"2023-10-09T10:43:34.486509+0000","last_peered":"2023-10-09T10:43:34.486509+0000","last_clean":"2023-10-09T10:43:34.486509+0000","last_became_active":"2023-10-09T10:41:29.511907+0000","last_became_peered":"2023-10-09T10:41:29.511907+0000","last_unstale":"2023-10-09T10:43:34.486509+0000","last_undegraded":"2023-10-09T10:43:34.486509+0000","last_fullsized":"2023-10-09T10:43:34.486509+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:25.351050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483811+0000","last_change":"2023-10-09T10:43:06.673430+0000","last_active":"2023-10-09T10:43:35.483811+0000","last_peered":"2023-10-09T10:43:35.483811+0000","last_clean":"2023-10-09T10:43:35.483811+0000","last_became_active":"2023-10-09T10:43:06.673110+0000","last_became_peered":"2023-10-09T10:43:06.673110+0000","last_unstale":"2023-10-09T10:43:35.483811+0000","last_undegraded":"2023-10-09T10:43:35.483811+0000","last_fullsized":"2023-10-09T10:43:35.483811+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:35.429715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491431+0000","last_change":"2023-10-09T10:43:35.491431+0000","last_active":"2023-10-09T10:43:35.491431+0000","last_peered":"2023-10-09T10:43:35.491431+0000","last_clean":"2023-10-09T10:43:35.491431+0000","last_became_active":"2023-10-09T10:43:35.491153+0000","last_became_peered":"2023-10-09T10:43:35.491153+0000","last_unstale":"2023-10-09T10:43:35.491431+0000","last_undegraded":"2023-10-09T10:43:35.491431+0000","last_fullsized":"2023-10-09T10:43:35.491431+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:58.495730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":82,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485088+0000","last_change":"2023-10-09T10:42:38.417090+0000","last_active":"2023-10-09T10:43:34.485088+0000","last_peered":"2023-10-09T10:43:34.485088+0000","last_clean":"2023-10-09T10:43:34.485088+0000","last_became_active":"2023-10-09T10:42:38.416758+0000","last_became_peered":"2023-10-09T10:42:38.416758+0000","last_unstale":"2023-10-09T10:43:34.485088+0000","last_undegraded":"2023-10-09T10:43:34.485088+0000","last_fullsized":"2023-10-09T10:43:34.485088+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:52.373940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482379+0000","last_change":"2023-10-09T10:35:57.165395+0000","last_active":"2023-10-09T10:43:34.482379+0000","last_peered":"2023-10-09T10:43:34.482379+0000","last_clean":"2023-10-09T10:43:34.482379+0000","last_became_active":"2023-10-09T10:35:57.165077+0000","last_became_peered":"2023-10-09T10:35:57.165077+0000","last_unstale":"2023-10-09T10:43:34.482379+0000","last_undegraded":"2023-10-09T10:43:34.482379+0000","last_fullsized":"2023-10-09T10:43:34.482379+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:53.159616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.485969+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:43:35.485969+0000","last_peered":"2023-10-09T10:43:35.485969+0000","last_clean":"2023-10-09T10:43:35.485969+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:43:35.485969+0000","last_undegraded":"2023-10-09T10:43:35.485969+0000","last_fullsized":"2023-10-09T10:43:35.485969+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":441,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.484997+0000","last_change":"2023-10-09T10:41:29.515426+0000","last_active":"2023-10-09T10:43:34.484997+0000","last_peered":"2023-10-09T10:43:34.484997+0000","last_clean":"2023-10-09T10:43:34.484997+0000","last_became_active":"2023-10-09T10:41:29.515012+0000","last_became_peered":"2023-10-09T10:41:29.515012+0000","last_unstale":"2023-10-09T10:43:34.484997+0000","last_undegraded":"2023-10-09T10:43:34.484997+0000","last_fullsized":"2023-10-09T10:43:34.484997+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:23.973968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479081+0000","last_change":"2023-10-09T10:43:06.673931+0000","last_active":"2023-10-09T10:43:34.479081+0000","last_peered":"2023-10-09T10:43:34.479081+0000","last_clean":"2023-10-09T10:43:34.479081+0000","last_became_active":"2023-10-09T10:43:06.673605+0000","last_became_peered":"2023-10-09T10:43:06.673605+0000","last_unstale":"2023-10-09T10:43:34.479081+0000","last_undegraded":"2023-10-09T10:43:34.479081+0000","last_fullsized":"2023-10-09T10:43:34.479081+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:19.088672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":67,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489950+0000","last_change":"2023-10-09T10:42:38.406633+0000","last_active":"2023-10-09T10:43:34.489950+0000","last_peered":"2023-10-09T10:43:34.489950+0000","last_clean":"2023-10-09T10:43:34.489950+0000","last_became_active":"2023-10-09T10:42:38.406317+0000","last_became_peered":"2023-10-09T10:42:38.406317+0000","last_unstale":"2023-10-09T10:43:34.489950+0000","last_undegraded":"2023-10-09T10:43:34.489950+0000","last_fullsized":"2023-10-09T10:43:34.489950+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:02.600240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487885+0000","last_change":"2023-10-09T10:43:06.672862+0000","last_active":"2023-10-09T10:43:34.487885+0000","last_peered":"2023-10-09T10:43:34.487885+0000","last_clean":"2023-10-09T10:43:34.487885+0000","last_became_active":"2023-10-09T10:43:06.672572+0000","last_became_peered":"2023-10-09T10:43:06.672572+0000","last_unstale":"2023-10-09T10:43:34.487885+0000","last_undegraded":"2023-10-09T10:43:34.487885+0000","last_fullsized":"2023-10-09T10:43:34.487885+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:23.743910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":83,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.490168+0000","last_change":"2023-10-09T10:43:35.490168+0000","last_active":"2023-10-09T10:43:35.490168+0000","last_peered":"2023-10-09T10:43:35.490168+0000","last_clean":"2023-10-09T10:43:35.490168+0000","last_became_active":"2023-10-09T10:43:35.489834+0000","last_became_peered":"2023-10-09T10:43:35.489834+0000","last_unstale":"2023-10-09T10:43:35.490168+0000","last_undegraded":"2023-10-09T10:43:35.490168+0000","last_fullsized":"2023-10-09T10:43:35.490168+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:59:26.252052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.489332+0000","last_change":"2023-10-09T10:43:35.489332+0000","last_active":"2023-10-09T10:43:35.489332+0000","last_peered":"2023-10-09T10:43:35.489332+0000","last_clean":"2023-10-09T10:43:35.489332+0000","last_became_active":"2023-10-09T10:43:35.489044+0000","last_became_peered":"2023-10-09T10:43:35.489044+0000","last_unstale":"2023-10-09T10:43:35.489332+0000","last_undegraded":"2023-10-09T10:43:35.489332+0000","last_fullsized":"2023-10-09T10:43:35.489332+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:42.168915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487678+0000","last_change":"2023-10-09T10:41:29.511298+0000","last_active":"2023-10-09T10:43:34.487678+0000","last_peered":"2023-10-09T10:43:34.487678+0000","last_clean":"2023-10-09T10:43:34.487678+0000","last_became_active":"2023-10-09T10:41:29.510978+0000","last_became_peered":"2023-10-09T10:41:29.510978+0000","last_unstale":"2023-10-09T10:43:34.487678+0000","last_undegraded":"2023-10-09T10:43:34.487678+0000","last_fullsized":"2023-10-09T10:43:34.487678+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:20:11.933459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479143+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:43:34.479143+0000","last_peered":"2023-10-09T10:43:34.479143+0000","last_clean":"2023-10-09T10:43:34.479143+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:43:34.479143+0000","last_undegraded":"2023-10-09T10:43:34.479143+0000","last_fullsized":"2023-10-09T10:43:34.479143+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486160+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:43:34.486160+0000","last_peered":"2023-10-09T10:43:34.486160+0000","last_clean":"2023-10-09T10:43:34.486160+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:43:34.486160+0000","last_undegraded":"2023-10-09T10:43:34.486160+0000","last_fullsized":"2023-10-09T10:43:34.486160+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":78,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486524+0000","last_change":"2023-10-09T10:43:35.486524+0000","last_active":"2023-10-09T10:43:35.486524+0000","last_peered":"2023-10-09T10:43:35.486524+0000","last_clean":"2023-10-09T10:43:35.486524+0000","last_became_active":"2023-10-09T10:43:35.486152+0000","last_became_peered":"2023-10-09T10:43:35.486152+0000","last_unstale":"2023-10-09T10:43:35.486524+0000","last_undegraded":"2023-10-09T10:43:35.486524+0000","last_fullsized":"2023-10-09T10:43:35.486524+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:47.200522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":72,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489284+0000","last_change":"2023-10-09T10:42:38.416111+0000","last_active":"2023-10-09T10:43:34.489284+0000","last_peered":"2023-10-09T10:43:34.489284+0000","last_clean":"2023-10-09T10:43:34.489284+0000","last_became_active":"2023-10-09T10:42:38.415861+0000","last_became_peered":"2023-10-09T10:42:38.415861+0000","last_unstale":"2023-10-09T10:43:34.489284+0000","last_undegraded":"2023-10-09T10:43:34.489284+0000","last_fullsized":"2023-10-09T10:43:34.489284+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:10:40.572274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.489126+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:43:35.489126+0000","last_peered":"2023-10-09T10:43:35.489126+0000","last_clean":"2023-10-09T10:43:35.489126+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:43:35.489126+0000","last_undegraded":"2023-10-09T10:43:35.489126+0000","last_fullsized":"2023-10-09T10:43:35.489126+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":64,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486030+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:43:35.486030+0000","last_peered":"2023-10-09T10:43:35.486030+0000","last_clean":"2023-10-09T10:43:35.486030+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:43:35.486030+0000","last_undegraded":"2023-10-09T10:43:35.486030+0000","last_fullsized":"2023-10-09T10:43:35.486030+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486884+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:43:34.486884+0000","last_peered":"2023-10-09T10:43:34.486884+0000","last_clean":"2023-10-09T10:43:34.486884+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:43:34.486884+0000","last_undegraded":"2023-10-09T10:43:34.486884+0000","last_fullsized":"2023-10-09T10:43:34.486884+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485938+0000","last_change":"2023-10-09T10:43:06.676577+0000","last_active":"2023-10-09T10:43:34.485938+0000","last_peered":"2023-10-09T10:43:34.485938+0000","last_clean":"2023-10-09T10:43:34.485938+0000","last_became_active":"2023-10-09T10:43:06.676165+0000","last_became_peered":"2023-10-09T10:43:06.676165+0000","last_unstale":"2023-10-09T10:43:34.485938+0000","last_undegraded":"2023-10-09T10:43:34.485938+0000","last_fullsized":"2023-10-09T10:43:34.485938+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:52.614985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":77,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487389+0000","last_change":"2023-10-09T10:43:06.676367+0000","last_active":"2023-10-09T10:43:34.487389+0000","last_peered":"2023-10-09T10:43:34.487389+0000","last_clean":"2023-10-09T10:43:34.487389+0000","last_became_active":"2023-10-09T10:43:06.676017+0000","last_became_peered":"2023-10-09T10:43:06.676017+0000","last_unstale":"2023-10-09T10:43:34.487389+0000","last_undegraded":"2023-10-09T10:43:34.487389+0000","last_fullsized":"2023-10-09T10:43:34.487389+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:54.387617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482664+0000","last_change":"2023-10-09T10:42:38.412169+0000","last_active":"2023-10-09T10:43:34.482664+0000","last_peered":"2023-10-09T10:43:34.482664+0000","last_clean":"2023-10-09T10:43:34.482664+0000","last_became_active":"2023-10-09T10:42:38.411829+0000","last_became_peered":"2023-10-09T10:42:38.411829+0000","last_unstale":"2023-10-09T10:43:34.482664+0000","last_undegraded":"2023-10-09T10:43:34.482664+0000","last_fullsized":"2023-10-09T10:43:34.482664+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:36:35.699293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.493287+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:43:35.493287+0000","last_peered":"2023-10-09T10:43:35.493287+0000","last_clean":"2023-10-09T10:43:35.493287+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:43:35.493287+0000","last_undegraded":"2023-10-09T10:43:35.493287+0000","last_fullsized":"2023-10-09T10:43:35.493287+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":61,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491435+0000","last_change":"2023-10-09T10:43:35.491435+0000","last_active":"2023-10-09T10:43:35.491435+0000","last_peered":"2023-10-09T10:43:35.491435+0000","last_clean":"2023-10-09T10:43:35.491435+0000","last_became_active":"2023-10-09T10:43:35.491096+0000","last_became_peered":"2023-10-09T10:43:35.491096+0000","last_unstale":"2023-10-09T10:43:35.491435+0000","last_undegraded":"2023-10-09T10:43:35.491435+0000","last_fullsized":"2023-10-09T10:43:35.491435+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:09.581525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483788+0000","last_change":"2023-10-09T10:42:38.407703+0000","last_active":"2023-10-09T10:43:35.483788+0000","last_peered":"2023-10-09T10:43:35.483788+0000","last_clean":"2023-10-09T10:43:35.483788+0000","last_became_active":"2023-10-09T10:42:38.407431+0000","last_became_peered":"2023-10-09T10:42:38.407431+0000","last_unstale":"2023-10-09T10:43:35.483788+0000","last_undegraded":"2023-10-09T10:43:35.483788+0000","last_fullsized":"2023-10-09T10:43:35.483788+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:23.450634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487326+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:43:35.487326+0000","last_peered":"2023-10-09T10:43:35.487326+0000","last_clean":"2023-10-09T10:43:35.487326+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:43:35.487326+0000","last_undegraded":"2023-10-09T10:43:35.487326+0000","last_fullsized":"2023-10-09T10:43:35.487326+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482266+0000","last_change":"2023-10-09T10:42:38.411928+0000","last_active":"2023-10-09T10:43:34.482266+0000","last_peered":"2023-10-09T10:43:34.482266+0000","last_clean":"2023-10-09T10:43:34.482266+0000","last_became_active":"2023-10-09T10:42:38.411560+0000","last_became_peered":"2023-10-09T10:42:38.411560+0000","last_unstale":"2023-10-09T10:43:34.482266+0000","last_undegraded":"2023-10-09T10:43:34.482266+0000","last_fullsized":"2023-10-09T10:43:34.482266+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:07.711753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":45,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487828+0000","last_change":"2023-10-09T10:35:56.822899+0000","last_active":"2023-10-09T10:43:34.487828+0000","last_peered":"2023-10-09T10:43:34.487828+0000","last_clean":"2023-10-09T10:43:34.487828+0000","last_became_active":"2023-10-09T10:35:56.822564+0000","last_became_peered":"2023-10-09T10:35:56.822564+0000","last_unstale":"2023-10-09T10:43:34.487828+0000","last_undegraded":"2023-10-09T10:43:34.487828+0000","last_fullsized":"2023-10-09T10:43:34.487828+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:25:03.398416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.493278+0000","last_change":"2023-10-09T10:43:35.493278+0000","last_active":"2023-10-09T10:43:35.493278+0000","last_peered":"2023-10-09T10:43:35.493278+0000","last_clean":"2023-10-09T10:43:35.493278+0000","last_became_active":"2023-10-09T10:43:35.492915+0000","last_became_peered":"2023-10-09T10:43:35.492915+0000","last_unstale":"2023-10-09T10:43:35.493278+0000","last_undegraded":"2023-10-09T10:43:35.493278+0000","last_fullsized":"2023-10-09T10:43:35.493278+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:42.227364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487004+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:43:34.487004+0000","last_peered":"2023-10-09T10:43:34.487004+0000","last_clean":"2023-10-09T10:43:34.487004+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:43:34.487004+0000","last_undegraded":"2023-10-09T10:43:34.487004+0000","last_fullsized":"2023-10-09T10:43:34.487004+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479053+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:43:34.479053+0000","last_peered":"2023-10-09T10:43:34.479053+0000","last_clean":"2023-10-09T10:43:34.479053+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:43:34.479053+0000","last_undegraded":"2023-10-09T10:43:34.479053+0000","last_fullsized":"2023-10-09T10:43:34.479053+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487772+0000","last_change":"2023-10-09T10:41:29.516129+0000","last_active":"2023-10-09T10:43:34.487772+0000","last_peered":"2023-10-09T10:43:34.487772+0000","last_clean":"2023-10-09T10:43:34.487772+0000","last_became_active":"2023-10-09T10:41:29.515741+0000","last_became_peered":"2023-10-09T10:41:29.515741+0000","last_unstale":"2023-10-09T10:43:34.487772+0000","last_undegraded":"2023-10-09T10:43:34.487772+0000","last_fullsized":"2023-10-09T10:43:34.487772+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:29:14.059627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":63,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487647+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:43:35.487647+0000","last_peered":"2023-10-09T10:43:35.487647+0000","last_clean":"2023-10-09T10:43:35.487647+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:43:35.487647+0000","last_undegraded":"2023-10-09T10:43:35.487647+0000","last_fullsized":"2023-10-09T10:43:35.487647+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":69,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.481806+0000","last_change":"2023-10-09T10:41:29.512929+0000","last_active":"2023-10-09T10:43:34.481806+0000","last_peered":"2023-10-09T10:43:34.481806+0000","last_clean":"2023-10-09T10:43:34.481806+0000","last_became_active":"2023-10-09T10:41:29.512591+0000","last_became_peered":"2023-10-09T10:41:29.512591+0000","last_unstale":"2023-10-09T10:43:34.481806+0000","last_undegraded":"2023-10-09T10:43:34.481806+0000","last_fullsized":"2023-10-09T10:43:34.481806+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:35.703322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.490293+0000","last_change":"2023-10-09T10:43:35.487796+0000","last_active":"2023-10-09T10:43:35.490293+0000","last_peered":"2023-10-09T10:43:35.490293+0000","last_clean":"2023-10-09T10:43:35.490293+0000","last_became_active":"2023-10-09T10:43:35.487518+0000","last_became_peered":"2023-10-09T10:43:35.487518+0000","last_unstale":"2023-10-09T10:43:35.490293+0000","last_undegraded":"2023-10-09T10:43:35.490293+0000","last_fullsized":"2023-10-09T10:43:35.490293+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:47.089685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":44,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.484279+0000","last_change":"2023-10-09T10:35:56.824257+0000","last_active":"2023-10-09T10:43:34.484279+0000","last_peered":"2023-10-09T10:43:34.484279+0000","last_clean":"2023-10-09T10:43:34.484279+0000","last_became_active":"2023-10-09T10:35:56.823152+0000","last_became_peered":"2023-10-09T10:35:56.823152+0000","last_unstale":"2023-10-09T10:43:34.484279+0000","last_undegraded":"2023-10-09T10:43:34.484279+0000","last_fullsized":"2023-10-09T10:43:34.484279+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:52.392494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486785+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:43:35.486785+0000","last_peered":"2023-10-09T10:43:35.486785+0000","last_clean":"2023-10-09T10:43:35.486785+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:43:35.486785+0000","last_undegraded":"2023-10-09T10:43:35.486785+0000","last_fullsized":"2023-10-09T10:43:35.486785+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487384+0000","last_change":"2023-10-09T10:43:35.486914+0000","last_active":"2023-10-09T10:43:35.487384+0000","last_peered":"2023-10-09T10:43:35.487384+0000","last_clean":"2023-10-09T10:43:35.487384+0000","last_became_active":"2023-10-09T10:43:35.486623+0000","last_became_peered":"2023-10-09T10:43:35.486623+0000","last_unstale":"2023-10-09T10:43:35.487384+0000","last_undegraded":"2023-10-09T10:43:35.487384+0000","last_fullsized":"2023-10-09T10:43:35.487384+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:13.339801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":77,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487942+0000","last_change":"2023-10-09T10:43:35.487942+0000","last_active":"2023-10-09T10:43:35.487942+0000","last_peered":"2023-10-09T10:43:35.487942+0000","last_clean":"2023-10-09T10:43:35.487942+0000","last_became_active":"2023-10-09T10:43:35.487651+0000","last_became_peered":"2023-10-09T10:43:35.487651+0000","last_unstale":"2023-10-09T10:43:35.487942+0000","last_undegraded":"2023-10-09T10:43:35.487942+0000","last_fullsized":"2023-10-09T10:43:35.487942+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:51.882595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485380+0000","last_change":"2023-10-09T10:43:06.677115+0000","last_active":"2023-10-09T10:43:34.485380+0000","last_peered":"2023-10-09T10:43:34.485380+0000","last_clean":"2023-10-09T10:43:34.485380+0000","last_became_active":"2023-10-09T10:43:06.676739+0000","last_became_peered":"2023-10-09T10:43:06.676739+0000","last_unstale":"2023-10-09T10:43:34.485380+0000","last_undegraded":"2023-10-09T10:43:34.485380+0000","last_fullsized":"2023-10-09T10:43:34.485380+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:02.520770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487176+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:43:34.487176+0000","last_peered":"2023-10-09T10:43:34.487176+0000","last_clean":"2023-10-09T10:43:34.487176+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:43:34.487176+0000","last_undegraded":"2023-10-09T10:43:34.487176+0000","last_fullsized":"2023-10-09T10:43:34.487176+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":66,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486392+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:43:34.486392+0000","last_peered":"2023-10-09T10:43:34.486392+0000","last_clean":"2023-10-09T10:43:34.486392+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:43:34.486392+0000","last_undegraded":"2023-10-09T10:43:34.486392+0000","last_fullsized":"2023-10-09T10:43:34.486392+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486237+0000","last_change":"2023-10-09T10:43:06.672114+0000","last_active":"2023-10-09T10:43:34.486237+0000","last_peered":"2023-10-09T10:43:34.486237+0000","last_clean":"2023-10-09T10:43:34.486237+0000","last_became_active":"2023-10-09T10:43:06.671827+0000","last_became_peered":"2023-10-09T10:43:06.671827+0000","last_unstale":"2023-10-09T10:43:34.486237+0000","last_undegraded":"2023-10-09T10:43:34.486237+0000","last_fullsized":"2023-10-09T10:43:34.486237+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:41.701409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":64,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491382+0000","last_change":"2023-10-09T10:43:35.491382+0000","last_active":"2023-10-09T10:43:35.491382+0000","last_peered":"2023-10-09T10:43:35.491382+0000","last_clean":"2023-10-09T10:43:35.491382+0000","last_became_active":"2023-10-09T10:43:35.491062+0000","last_became_peered":"2023-10-09T10:43:35.491062+0000","last_unstale":"2023-10-09T10:43:35.491382+0000","last_undegraded":"2023-10-09T10:43:35.491382+0000","last_fullsized":"2023-10-09T10:43:35.491382+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:10.396267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487546+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:43:35.487546+0000","last_peered":"2023-10-09T10:43:35.487546+0000","last_clean":"2023-10-09T10:43:35.487546+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:43:35.487546+0000","last_undegraded":"2023-10-09T10:43:35.487546+0000","last_fullsized":"2023-10-09T10:43:35.487546+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479334+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:43:34.479334+0000","last_peered":"2023-10-09T10:43:34.479334+0000","last_clean":"2023-10-09T10:43:34.479334+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:43:34.479334+0000","last_undegraded":"2023-10-09T10:43:34.479334+0000","last_fullsized":"2023-10-09T10:43:34.479334+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":63,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.484250+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:43:35.484250+0000","last_peered":"2023-10-09T10:43:35.484250+0000","last_clean":"2023-10-09T10:43:35.484250+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:43:35.484250+0000","last_undegraded":"2023-10-09T10:43:35.484250+0000","last_fullsized":"2023-10-09T10:43:35.484250+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.488427+0000","last_change":"2023-10-09T10:43:35.487114+0000","last_active":"2023-10-09T10:43:35.488427+0000","last_peered":"2023-10-09T10:43:35.488427+0000","last_clean":"2023-10-09T10:43:35.488427+0000","last_became_active":"2023-10-09T10:43:35.486830+0000","last_became_peered":"2023-10-09T10:43:35.486830+0000","last_unstale":"2023-10-09T10:43:35.488427+0000","last_undegraded":"2023-10-09T10:43:35.488427+0000","last_fullsized":"2023-10-09T10:43:35.488427+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:08.836969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482547+0000","last_change":"2023-10-09T10:42:38.413301+0000","last_active":"2023-10-09T10:43:34.482547+0000","last_peered":"2023-10-09T10:43:34.482547+0000","last_clean":"2023-10-09T10:43:34.482547+0000","last_became_active":"2023-10-09T10:42:38.413006+0000","last_became_peered":"2023-10-09T10:42:38.413006+0000","last_unstale":"2023-10-09T10:43:34.482547+0000","last_undegraded":"2023-10-09T10:43:34.482547+0000","last_fullsized":"2023-10-09T10:43:34.482547+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:33.462419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":81,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.485235+0000","last_change":"2023-10-09T10:43:35.485235+0000","last_active":"2023-10-09T10:43:35.485235+0000","last_peered":"2023-10-09T10:43:35.485235+0000","last_clean":"2023-10-09T10:43:35.485235+0000","last_became_active":"2023-10-09T10:43:35.484917+0000","last_became_peered":"2023-10-09T10:43:35.484917+0000","last_unstale":"2023-10-09T10:43:35.485235+0000","last_undegraded":"2023-10-09T10:43:35.485235+0000","last_fullsized":"2023-10-09T10:43:35.485235+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:55.417326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487224+0000","last_change":"2023-10-09T10:43:35.487224+0000","last_active":"2023-10-09T10:43:35.487224+0000","last_peered":"2023-10-09T10:43:35.487224+0000","last_clean":"2023-10-09T10:43:35.487224+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:43:35.487224+0000","last_undegraded":"2023-10-09T10:43:35.487224+0000","last_fullsized":"2023-10-09T10:43:35.487224+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:44:52.066778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491388+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:43:35.491388+0000","last_peered":"2023-10-09T10:43:35.491388+0000","last_clean":"2023-10-09T10:43:35.491388+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:43:35.491388+0000","last_undegraded":"2023-10-09T10:43:35.491388+0000","last_fullsized":"2023-10-09T10:43:35.491388+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":63,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.490890+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:43:35.490890+0000","last_peered":"2023-10-09T10:43:35.490890+0000","last_clean":"2023-10-09T10:43:35.490890+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:43:35.490890+0000","last_undegraded":"2023-10-09T10:43:35.490890+0000","last_fullsized":"2023-10-09T10:43:35.490890+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":79,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486285+0000","last_change":"2023-10-09T10:43:06.676414+0000","last_active":"2023-10-09T10:43:34.486285+0000","last_peered":"2023-10-09T10:43:34.486285+0000","last_clean":"2023-10-09T10:43:34.486285+0000","last_became_active":"2023-10-09T10:43:06.676050+0000","last_became_peered":"2023-10-09T10:43:06.676050+0000","last_unstale":"2023-10-09T10:43:34.486285+0000","last_undegraded":"2023-10-09T10:43:34.486285+0000","last_fullsized":"2023-10-09T10:43:34.486285+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:28.011477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332210,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311116,"kb_used_data":1540,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93434036,"statfs":{"total":95995035648,"available":95676452864,"internally_reserved":0,"allocated":1576960,"data_stored":1120859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:31 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.56399999999999995,"15min":0.57399999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.67200000000000004,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.61799999999999999,"15min":0.63400000000000001},"min":{"1min":0.36499999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.69799999999999995,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.625}]},{"osd":1,"last update":"Mon Oct 9 10:43:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.46600000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:42:10 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.55600000000000005,"15min":0.53100000000000003},"min":{"1min":0.39500000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.80200000000000005,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.57599999999999996,"15min":0.54900000000000004},"min":{"1min":0.41399999999999998,"5min":0.36599999999999999,"15min":0.27300000000000002},"max":{"1min":0.82499999999999996,"5min":1.2609999999999999,"15min":1.2609999999999999},"last":0.52400000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.58399999999999996,"15min":0.56699999999999995},"min":{"1min":0.41399999999999998,"5min":0.35099999999999998,"15min":0.33800000000000002},"max":{"1min":0.74099999999999999,"5min":2.569,"15min":2.569},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59099999999999997,"15min":0.57399999999999995},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.76900000000000002,"5min":0.80100000000000005,"15min":1.232},"last":0.54900000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48599999999999999,"15min":0.47299999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.73999999999999999,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.502,"5min":0.497,"15min":0.47399999999999998},"min":{"1min":0.36899999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.67100000000000004,"5min":0.80600000000000005,"15min":0.83699999999999997},"last":0.49399999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.52100000000000002,"15min":0.504},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":1.1950000000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.55300000000000005,"15min":0.54000000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.25700000000000001},"max":{"1min":0.86099999999999999,"5min":0.876,"15min":1.0649999999999999},"last":0.69799999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58199999999999996,"15min":0.55700000000000005},"min":{"1min":0.40500000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.81699999999999995,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.57399999999999995,"15min":0.54300000000000004},"min":{"1min":0.39000000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.79700000000000004,"5min":0.89200000000000002,"15min":0.93500000000000005},"last":0.59599999999999997}]}]},{"osd":6,"up_from":44,"seq":188978561141,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298816,"kb_used_data":1528,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446336,"statfs":{"total":95995035648,"available":95689048064,"internally_reserved":0,"allocated":1564672,"data_stored":1120713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:34 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.59699999999999998,"15min":0.60599999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.73999999999999999,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.63,"5min":0.61199999999999999,"15min":0.60899999999999999},"min":{"1min":0.46600000000000003,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.64300000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.57799999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.52500000000000002,"15min":0.501},"min":{"1min":0.39500000000000002,"5min":0.35899999999999999,"15min":0.26600000000000001},"max":{"1min":0.66700000000000004,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.54400000000000004,"15min":0.52900000000000003},"min":{"1min":0.36399999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.77400000000000002,"5min":0.86299999999999999,"15min":1.0109999999999999},"last":0.52800000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56100000000000005,"15min":0.52200000000000002},"min":{"1min":0.375,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72499999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54700000000000004,"15min":0.53200000000000003},"min":{"1min":0.39800000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80400000000000005},"last":0.47499999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.45900000000000002,"15min":0.44400000000000001},"min":{"1min":0.38900000000000001,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":0.59699999999999998,"5min":0.67400000000000004,"15min":0.70999999999999996},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.505,"15min":0.49199999999999999},"min":{"1min":0.35299999999999998,"5min":0.29899999999999999,"15min":0.29699999999999999},"max":{"1min":0.74399999999999999,"5min":0.77000000000000002,"15min":0.78500000000000003},"last":0.45100000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.51500000000000001,"15min":0.48399999999999999},"min":{"1min":0.40100000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.67600000000000005,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.55000000000000004,"15min":0.53000000000000003},"min":{"1min":0.372,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.84399999999999997},"last":0.59199999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:43:18 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.53100000000000003,"15min":0.52600000000000002},"min":{"1min":0.33900000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.72599999999999998,"5min":0.78600000000000003,"15min":0.79100000000000004},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.438,"5min":0.34100000000000003,"15min":0.34000000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":1.018},"last":0.621}]}]},{"osd":1,"up_from":85,"seq":365072220180,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28448,"kb_used_data":984,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716704,"statfs":{"total":95995035648,"available":95965904896,"internally_reserved":0,"allocated":1007616,"data_stored":536014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.56000000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58699999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57799999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.98899999999999999},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.83199999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.78500000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.88200000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":1.0369999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.90000000000000002}]}]},{"osd":0,"up_from":80,"seq":343597383714,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29108,"kb_used_data":1580,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93716044,"statfs":{"total":95995035648,"available":95965229056,"internally_reserved":0,"allocated":1617920,"data_stored":1126957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:43:38 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.55500000000000005}]},{"osd":2,"last update":"Mon Oct 9 10:42:30 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.47499999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:42:30 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.56699999999999995}]},{"osd":4,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.61699999999999999,"15min":0.61899999999999999},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61699999999999999,"15min":0.61799999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.83499999999999996,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.80300000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.62,"15min":0.624},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.749,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.625,"5min":0.64400000000000002,"15min":0.64700000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.67400000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.628,"15min":0.63},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.73699999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.64700000000000002,"15min":0.65100000000000002},"min":{"1min":0.44600000000000001,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.66900000000000004,"15min":0.67500000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.71899999999999997}]}]},{"osd":2,"up_from":90,"seq":386547056654,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28572,"kb_used_data":980,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716580,"statfs":{"total":95995035648,"available":95965777920,"internally_reserved":0,"allocated":1003520,"data_stored":533482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35499999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.371}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81100000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76800000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56200000000000006}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51200000000000001}]}]},{"osd":3,"up_from":95,"seq":408021893129,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28424,"kb_used_data":960,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716728,"statfs":{"total":95995035648,"available":95965929472,"internally_reserved":0,"allocated":983040,"data_stored":530397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30399999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.318}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40000000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58899999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67000000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65000000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019004,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302344,"kb_used_data":960,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93442808,"statfs":{"total":95995035648,"available":95685435392,"internally_reserved":0,"allocated":983040,"data_stored":530744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.55500000000000005,"15min":0.55700000000000005},"min":{"1min":0.35199999999999998,"5min":0.311,"15min":0.311},"max":{"1min":0.74299999999999999,"5min":0.755,"15min":0.755},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.57899999999999996,"15min":0.58299999999999996},"min":{"1min":0.41599999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.747,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.49299999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.53800000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:42:21 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.54100000000000004,"15min":0.55200000000000005},"min":{"1min":0.34799999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.78300000000000003,"5min":0.82999999999999996,"15min":2.3530000000000002},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.56599999999999995,"15min":0.57199999999999995},"min":{"1min":0.35899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.80800000000000005,"5min":0.86499999999999999,"15min":0.875},"last":0.65100000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:43:21 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.58499999999999996,"15min":0.56200000000000006},"min":{"1min":0.45700000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.84799999999999998,"5min":0.85099999999999998,"15min":0.86399999999999999},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.57299999999999995,"15min":0.56799999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.83699999999999997,"5min":0.86599999999999999,"15min":2.0910000000000002},"last":0.71699999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.47399999999999998,"15min":0.438},"min":{"1min":0.34300000000000003,"5min":0.253,"15min":0.253},"max":{"1min":0.65000000000000002,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.48199999999999998,"15min":0.442},"min":{"1min":0.34599999999999997,"5min":0.28499999999999998,"15min":0.26400000000000001},"max":{"1min":0.73299999999999998,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.68999999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.50800000000000001,"15min":0.46899999999999997},"min":{"1min":0.32000000000000001,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":0.752,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59499999999999997,"15min":0.56000000000000005},"min":{"1min":0.46500000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.82299999999999995,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.67200000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:43:11 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.57399999999999995,"15min":0.56899999999999995},"min":{"1min":0.376,"5min":0.34999999999999998,"15min":0.308},"max":{"1min":0.872,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.626,"5min":0.56899999999999995,"15min":0.53800000000000003},"min":{"1min":0.32500000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.80700000000000005,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.82799999999999996}]}]},{"osd":5,"up_from":37,"seq":158913790073,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306420,"kb_used_data":940,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93438732,"statfs":{"total":95995035648,"available":95681261568,"internally_reserved":0,"allocated":962560,"data_stored":526165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.66200000000000003,"15min":0.66700000000000004},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.61399999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.63600000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:44:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.497},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.48799999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:42:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.54500000000000004,"15min":0.52600000000000002},"min":{"1min":0.39600000000000002,"5min":0.36899999999999999,"15min":0.35699999999999998},"max":{"1min":0.76100000000000001,"5min":0.79000000000000004,"15min":2.976},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.59199999999999997,"15min":0.59099999999999997},"min":{"1min":0.44700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.78400000000000003,"5min":0.92400000000000004,"15min":4.7949999999999999},"last":0.45900000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47599999999999998,"15min":0.43099999999999999},"min":{"1min":0.35399999999999998,"5min":0.30499999999999999,"15min":0.25},"max":{"1min":0.69099999999999995,"5min":0.75600000000000001,"15min":2.4649999999999999},"last":0.372},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.5,"15min":0.45100000000000001},"min":{"1min":0.371,"5min":0.30099999999999999,"15min":0.28799999999999998},"max":{"1min":0.60199999999999998,"5min":0.85299999999999998,"15min":1.98},"last":0.51600000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.53200000000000003,"15min":0.45800000000000002},"min":{"1min":0.39300000000000002,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":0.67500000000000004,"5min":0.85599999999999998,"15min":2.048},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56599999999999995,"15min":0.49099999999999999},"min":{"1min":0.47099999999999997,"5min":0.33500000000000002,"15min":0.30299999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.42399999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:43:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.57199999999999995,"15min":0.53500000000000003},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.32400000000000001},"max":{"1min":0.80700000000000005,"5min":0.879,"15min":2.1110000000000002},"last":0.44},{"interface":"front","average":{"1min":0.627,"5min":0.59699999999999998,"15min":0.53600000000000003},"min":{"1min":0.48799999999999999,"5min":0.35699999999999998,"15min":0.31900000000000001},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":1.845},"last":0.54400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:44:09.977 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T10:44:09.977 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T10:44:09.977 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T10:44:09.977 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T10:44:10.243 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:10.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:10 smithi125 ceph-mon[195507]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:10.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:10 smithi125 ceph-mon[200197]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:10.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:10 smithi155 ceph-mon[184854]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:11.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:11 smithi125 ceph-mon[195507]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:44:11.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:11 smithi125 ceph-mon[200197]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:44:11.322 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:44:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:44:11] "GET /metrics HTTP/1.1" 200 34049 "" "Prometheus/2.43.0" 2023-10-09T10:44:11.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:11 smithi155 ceph-mon[184854]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:44:11.512 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:44:11.512 INFO:teuthology.orchestra.run.smithi125.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T10:44:11.888 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T10:44:11.888 INFO:teuthology.run_tasks:Running task print... 2023-10-09T10:44:11.905 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-10-09T10:44:11.905 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T10:44:11.989 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T10:44:11.990 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T10:44:11.990 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-10-09T10:44:11.991 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T10:44:11.992 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'a340fb5463aa3715e93e7472d5f60dfdcf184ab3'}} 2023-10-09T10:44:11.992 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T10:44:11.992 INFO:tasks.thrashosds:Beginning thrashosds... 2023-10-09T10:44:11.992 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T10:44:11.992 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:44:12.257 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:12.374 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:12 smithi125 ceph-mon[195507]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:12.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:12 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3116561838' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T10:44:12.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:12 smithi125 ceph-mon[200197]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:12.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:12 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3116561838' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T10:44:12.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:12 smithi155 ceph-mon[184854]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:12.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:12 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3116561838' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T10:44:13.481 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:44:13.482 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:44:13.658 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:13 smithi125 ceph-mon[195507]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:13.659 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:13 smithi125 ceph-mon[200197]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:13.862 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":187,"stamp":"2023-10-09T10:44:13.002432+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1333248,"kb_used_data":9472,"kb_used_omap":126,"kb_used_meta":1323585,"kb_avail":748627968,"statfs":{"total":767960285184,"available":766595039232,"internally_reserved":0,"allocated":9699328,"data_stored":6025331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1355351675},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002375"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485359+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:43:34.485359+0000","last_peered":"2023-10-09T10:43:34.485359+0000","last_clean":"2023-10-09T10:43:34.485359+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:43:34.485359+0000","last_undegraded":"2023-10-09T10:43:34.485359+0000","last_fullsized":"2023-10-09T10:43:34.485359+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":57,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.494552+0000","last_change":"2023-10-09T10:43:06.674549+0000","last_active":"2023-10-09T10:43:34.494552+0000","last_peered":"2023-10-09T10:43:34.494552+0000","last_clean":"2023-10-09T10:43:34.494552+0000","last_became_active":"2023-10-09T10:43:06.673510+0000","last_became_peered":"2023-10-09T10:43:06.673510+0000","last_unstale":"2023-10-09T10:43:34.494552+0000","last_undegraded":"2023-10-09T10:43:34.494552+0000","last_fullsized":"2023-10-09T10:43:34.494552+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:32.996795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":84,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487344+0000","last_change":"2023-10-09T10:43:35.487344+0000","last_active":"2023-10-09T10:43:35.487344+0000","last_peered":"2023-10-09T10:43:35.487344+0000","last_clean":"2023-10-09T10:43:35.487344+0000","last_became_active":"2023-10-09T10:43:35.486672+0000","last_became_peered":"2023-10-09T10:43:35.486672+0000","last_unstale":"2023-10-09T10:43:35.487344+0000","last_undegraded":"2023-10-09T10:43:35.487344+0000","last_fullsized":"2023-10-09T10:43:35.487344+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:44.665687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":69,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.485790+0000","last_change":"2023-10-09T10:43:35.485790+0000","last_active":"2023-10-09T10:43:35.485790+0000","last_peered":"2023-10-09T10:43:35.485790+0000","last_clean":"2023-10-09T10:43:35.485790+0000","last_became_active":"2023-10-09T10:43:35.485436+0000","last_became_peered":"2023-10-09T10:43:35.485436+0000","last_unstale":"2023-10-09T10:43:35.485790+0000","last_undegraded":"2023-10-09T10:43:35.485790+0000","last_fullsized":"2023-10-09T10:43:35.485790+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:45:20.897767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":77,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.492236+0000","last_change":"2023-10-09T10:43:35.492236+0000","last_active":"2023-10-09T10:43:35.492236+0000","last_peered":"2023-10-09T10:43:35.492236+0000","last_clean":"2023-10-09T10:43:35.492236+0000","last_became_active":"2023-10-09T10:43:35.491913+0000","last_became_peered":"2023-10-09T10:43:35.491913+0000","last_unstale":"2023-10-09T10:43:35.492236+0000","last_undegraded":"2023-10-09T10:43:35.492236+0000","last_fullsized":"2023-10-09T10:43:35.492236+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:34:13.448179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":46,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.480679+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:43:34.480679+0000","last_peered":"2023-10-09T10:43:34.480679+0000","last_clean":"2023-10-09T10:43:34.480679+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:43:34.480679+0000","last_undegraded":"2023-10-09T10:43:34.480679+0000","last_fullsized":"2023-10-09T10:43:34.480679+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":77,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.492737+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:43:35.492737+0000","last_peered":"2023-10-09T10:43:35.492737+0000","last_clean":"2023-10-09T10:43:35.492737+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:43:35.492737+0000","last_undegraded":"2023-10-09T10:43:35.492737+0000","last_fullsized":"2023-10-09T10:43:35.492737+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479260+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:43:34.479260+0000","last_peered":"2023-10-09T10:43:34.479260+0000","last_clean":"2023-10-09T10:43:34.479260+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:43:34.479260+0000","last_undegraded":"2023-10-09T10:43:34.479260+0000","last_fullsized":"2023-10-09T10:43:34.479260+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":82,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.489655+0000","last_change":"2023-10-09T10:43:35.489655+0000","last_active":"2023-10-09T10:43:35.489655+0000","last_peered":"2023-10-09T10:43:35.489655+0000","last_clean":"2023-10-09T10:43:35.489655+0000","last_became_active":"2023-10-09T10:43:35.489149+0000","last_became_peered":"2023-10-09T10:43:35.489149+0000","last_unstale":"2023-10-09T10:43:35.489655+0000","last_undegraded":"2023-10-09T10:43:35.489655+0000","last_fullsized":"2023-10-09T10:43:35.489655+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:30.598962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":42,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483874+0000","last_change":"2023-10-09T10:41:29.514075+0000","last_active":"2023-10-09T10:43:35.483874+0000","last_peered":"2023-10-09T10:43:35.483874+0000","last_clean":"2023-10-09T10:43:35.483874+0000","last_became_active":"2023-10-09T10:41:29.513347+0000","last_became_peered":"2023-10-09T10:41:29.513347+0000","last_unstale":"2023-10-09T10:43:35.483874+0000","last_undegraded":"2023-10-09T10:43:35.483874+0000","last_fullsized":"2023-10-09T10:43:35.483874+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:17.504888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485850+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:43:34.485850+0000","last_peered":"2023-10-09T10:43:34.485850+0000","last_clean":"2023-10-09T10:43:34.485850+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:43:34.485850+0000","last_undegraded":"2023-10-09T10:43:34.485850+0000","last_fullsized":"2023-10-09T10:43:34.485850+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":57,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483729+0000","last_change":"2023-10-09T10:42:38.407058+0000","last_active":"2023-10-09T10:43:35.483729+0000","last_peered":"2023-10-09T10:43:35.483729+0000","last_clean":"2023-10-09T10:43:35.483729+0000","last_became_active":"2023-10-09T10:42:38.406768+0000","last_became_peered":"2023-10-09T10:42:38.406768+0000","last_unstale":"2023-10-09T10:43:35.483729+0000","last_undegraded":"2023-10-09T10:43:35.483729+0000","last_fullsized":"2023-10-09T10:43:35.483729+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:41.198843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482345+0000","last_change":"2023-10-09T10:43:06.674387+0000","last_active":"2023-10-09T10:43:34.482345+0000","last_peered":"2023-10-09T10:43:34.482345+0000","last_clean":"2023-10-09T10:43:34.482345+0000","last_became_active":"2023-10-09T10:43:06.674032+0000","last_became_peered":"2023-10-09T10:43:06.674032+0000","last_unstale":"2023-10-09T10:43:34.482345+0000","last_undegraded":"2023-10-09T10:43:34.482345+0000","last_fullsized":"2023-10-09T10:43:34.482345+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:58.222034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":58,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486342+0000","last_change":"2023-10-09T10:43:35.486342+0000","last_active":"2023-10-09T10:43:35.486342+0000","last_peered":"2023-10-09T10:43:35.486342+0000","last_clean":"2023-10-09T10:43:35.486342+0000","last_became_active":"2023-10-09T10:43:35.486026+0000","last_became_peered":"2023-10-09T10:43:35.486026+0000","last_unstale":"2023-10-09T10:43:35.486342+0000","last_undegraded":"2023-10-09T10:43:35.486342+0000","last_fullsized":"2023-10-09T10:43:35.486342+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:16.616429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489374+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:43:34.489374+0000","last_peered":"2023-10-09T10:43:34.489374+0000","last_clean":"2023-10-09T10:43:34.489374+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:43:34.489374+0000","last_undegraded":"2023-10-09T10:43:34.489374+0000","last_fullsized":"2023-10-09T10:43:34.489374+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":61,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487842+0000","last_change":"2023-10-09T10:43:35.487842+0000","last_active":"2023-10-09T10:43:35.487842+0000","last_peered":"2023-10-09T10:43:35.487842+0000","last_clean":"2023-10-09T10:43:35.487842+0000","last_became_active":"2023-10-09T10:43:35.487192+0000","last_became_peered":"2023-10-09T10:43:35.487192+0000","last_unstale":"2023-10-09T10:43:35.487842+0000","last_undegraded":"2023-10-09T10:43:35.487842+0000","last_fullsized":"2023-10-09T10:43:35.487842+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:35.477280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.492904+0000","last_change":"2023-10-09T10:41:29.513802+0000","last_active":"2023-10-09T10:43:34.492904+0000","last_peered":"2023-10-09T10:43:34.492904+0000","last_clean":"2023-10-09T10:43:34.492904+0000","last_became_active":"2023-10-09T10:41:29.512895+0000","last_became_peered":"2023-10-09T10:41:29.512895+0000","last_unstale":"2023-10-09T10:43:34.492904+0000","last_undegraded":"2023-10-09T10:43:34.492904+0000","last_fullsized":"2023-10-09T10:43:34.492904+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:06.177976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489173+0000","last_change":"2023-10-09T10:43:06.671356+0000","last_active":"2023-10-09T10:43:34.489173+0000","last_peered":"2023-10-09T10:43:34.489173+0000","last_clean":"2023-10-09T10:43:34.489173+0000","last_became_active":"2023-10-09T10:43:06.671040+0000","last_became_peered":"2023-10-09T10:43:06.671040+0000","last_unstale":"2023-10-09T10:43:34.489173+0000","last_undegraded":"2023-10-09T10:43:34.489173+0000","last_fullsized":"2023-10-09T10:43:34.489173+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:47.601448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486509+0000","last_change":"2023-10-09T10:41:29.512225+0000","last_active":"2023-10-09T10:43:34.486509+0000","last_peered":"2023-10-09T10:43:34.486509+0000","last_clean":"2023-10-09T10:43:34.486509+0000","last_became_active":"2023-10-09T10:41:29.511907+0000","last_became_peered":"2023-10-09T10:41:29.511907+0000","last_unstale":"2023-10-09T10:43:34.486509+0000","last_undegraded":"2023-10-09T10:43:34.486509+0000","last_fullsized":"2023-10-09T10:43:34.486509+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:25.351050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483811+0000","last_change":"2023-10-09T10:43:06.673430+0000","last_active":"2023-10-09T10:43:35.483811+0000","last_peered":"2023-10-09T10:43:35.483811+0000","last_clean":"2023-10-09T10:43:35.483811+0000","last_became_active":"2023-10-09T10:43:06.673110+0000","last_became_peered":"2023-10-09T10:43:06.673110+0000","last_unstale":"2023-10-09T10:43:35.483811+0000","last_undegraded":"2023-10-09T10:43:35.483811+0000","last_fullsized":"2023-10-09T10:43:35.483811+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:35.429715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491431+0000","last_change":"2023-10-09T10:43:35.491431+0000","last_active":"2023-10-09T10:43:35.491431+0000","last_peered":"2023-10-09T10:43:35.491431+0000","last_clean":"2023-10-09T10:43:35.491431+0000","last_became_active":"2023-10-09T10:43:35.491153+0000","last_became_peered":"2023-10-09T10:43:35.491153+0000","last_unstale":"2023-10-09T10:43:35.491431+0000","last_undegraded":"2023-10-09T10:43:35.491431+0000","last_fullsized":"2023-10-09T10:43:35.491431+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:58.495730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":82,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485088+0000","last_change":"2023-10-09T10:42:38.417090+0000","last_active":"2023-10-09T10:43:34.485088+0000","last_peered":"2023-10-09T10:43:34.485088+0000","last_clean":"2023-10-09T10:43:34.485088+0000","last_became_active":"2023-10-09T10:42:38.416758+0000","last_became_peered":"2023-10-09T10:42:38.416758+0000","last_unstale":"2023-10-09T10:43:34.485088+0000","last_undegraded":"2023-10-09T10:43:34.485088+0000","last_fullsized":"2023-10-09T10:43:34.485088+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:52.373940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482379+0000","last_change":"2023-10-09T10:35:57.165395+0000","last_active":"2023-10-09T10:43:34.482379+0000","last_peered":"2023-10-09T10:43:34.482379+0000","last_clean":"2023-10-09T10:43:34.482379+0000","last_became_active":"2023-10-09T10:35:57.165077+0000","last_became_peered":"2023-10-09T10:35:57.165077+0000","last_unstale":"2023-10-09T10:43:34.482379+0000","last_undegraded":"2023-10-09T10:43:34.482379+0000","last_fullsized":"2023-10-09T10:43:34.482379+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:53.159616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.485969+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:43:35.485969+0000","last_peered":"2023-10-09T10:43:35.485969+0000","last_clean":"2023-10-09T10:43:35.485969+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:43:35.485969+0000","last_undegraded":"2023-10-09T10:43:35.485969+0000","last_fullsized":"2023-10-09T10:43:35.485969+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":441,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.484997+0000","last_change":"2023-10-09T10:41:29.515426+0000","last_active":"2023-10-09T10:43:34.484997+0000","last_peered":"2023-10-09T10:43:34.484997+0000","last_clean":"2023-10-09T10:43:34.484997+0000","last_became_active":"2023-10-09T10:41:29.515012+0000","last_became_peered":"2023-10-09T10:41:29.515012+0000","last_unstale":"2023-10-09T10:43:34.484997+0000","last_undegraded":"2023-10-09T10:43:34.484997+0000","last_fullsized":"2023-10-09T10:43:34.484997+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:23.973968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479081+0000","last_change":"2023-10-09T10:43:06.673931+0000","last_active":"2023-10-09T10:43:34.479081+0000","last_peered":"2023-10-09T10:43:34.479081+0000","last_clean":"2023-10-09T10:43:34.479081+0000","last_became_active":"2023-10-09T10:43:06.673605+0000","last_became_peered":"2023-10-09T10:43:06.673605+0000","last_unstale":"2023-10-09T10:43:34.479081+0000","last_undegraded":"2023-10-09T10:43:34.479081+0000","last_fullsized":"2023-10-09T10:43:34.479081+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:19.088672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":67,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489950+0000","last_change":"2023-10-09T10:42:38.406633+0000","last_active":"2023-10-09T10:43:34.489950+0000","last_peered":"2023-10-09T10:43:34.489950+0000","last_clean":"2023-10-09T10:43:34.489950+0000","last_became_active":"2023-10-09T10:42:38.406317+0000","last_became_peered":"2023-10-09T10:42:38.406317+0000","last_unstale":"2023-10-09T10:43:34.489950+0000","last_undegraded":"2023-10-09T10:43:34.489950+0000","last_fullsized":"2023-10-09T10:43:34.489950+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:02.600240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487885+0000","last_change":"2023-10-09T10:43:06.672862+0000","last_active":"2023-10-09T10:43:34.487885+0000","last_peered":"2023-10-09T10:43:34.487885+0000","last_clean":"2023-10-09T10:43:34.487885+0000","last_became_active":"2023-10-09T10:43:06.672572+0000","last_became_peered":"2023-10-09T10:43:06.672572+0000","last_unstale":"2023-10-09T10:43:34.487885+0000","last_undegraded":"2023-10-09T10:43:34.487885+0000","last_fullsized":"2023-10-09T10:43:34.487885+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:23.743910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":83,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.490168+0000","last_change":"2023-10-09T10:43:35.490168+0000","last_active":"2023-10-09T10:43:35.490168+0000","last_peered":"2023-10-09T10:43:35.490168+0000","last_clean":"2023-10-09T10:43:35.490168+0000","last_became_active":"2023-10-09T10:43:35.489834+0000","last_became_peered":"2023-10-09T10:43:35.489834+0000","last_unstale":"2023-10-09T10:43:35.490168+0000","last_undegraded":"2023-10-09T10:43:35.490168+0000","last_fullsized":"2023-10-09T10:43:35.490168+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:59:26.252052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.489332+0000","last_change":"2023-10-09T10:43:35.489332+0000","last_active":"2023-10-09T10:43:35.489332+0000","last_peered":"2023-10-09T10:43:35.489332+0000","last_clean":"2023-10-09T10:43:35.489332+0000","last_became_active":"2023-10-09T10:43:35.489044+0000","last_became_peered":"2023-10-09T10:43:35.489044+0000","last_unstale":"2023-10-09T10:43:35.489332+0000","last_undegraded":"2023-10-09T10:43:35.489332+0000","last_fullsized":"2023-10-09T10:43:35.489332+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:42.168915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487678+0000","last_change":"2023-10-09T10:41:29.511298+0000","last_active":"2023-10-09T10:43:34.487678+0000","last_peered":"2023-10-09T10:43:34.487678+0000","last_clean":"2023-10-09T10:43:34.487678+0000","last_became_active":"2023-10-09T10:41:29.510978+0000","last_became_peered":"2023-10-09T10:41:29.510978+0000","last_unstale":"2023-10-09T10:43:34.487678+0000","last_undegraded":"2023-10-09T10:43:34.487678+0000","last_fullsized":"2023-10-09T10:43:34.487678+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:20:11.933459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479143+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:43:34.479143+0000","last_peered":"2023-10-09T10:43:34.479143+0000","last_clean":"2023-10-09T10:43:34.479143+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:43:34.479143+0000","last_undegraded":"2023-10-09T10:43:34.479143+0000","last_fullsized":"2023-10-09T10:43:34.479143+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486160+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:43:34.486160+0000","last_peered":"2023-10-09T10:43:34.486160+0000","last_clean":"2023-10-09T10:43:34.486160+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:43:34.486160+0000","last_undegraded":"2023-10-09T10:43:34.486160+0000","last_fullsized":"2023-10-09T10:43:34.486160+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":78,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486524+0000","last_change":"2023-10-09T10:43:35.486524+0000","last_active":"2023-10-09T10:43:35.486524+0000","last_peered":"2023-10-09T10:43:35.486524+0000","last_clean":"2023-10-09T10:43:35.486524+0000","last_became_active":"2023-10-09T10:43:35.486152+0000","last_became_peered":"2023-10-09T10:43:35.486152+0000","last_unstale":"2023-10-09T10:43:35.486524+0000","last_undegraded":"2023-10-09T10:43:35.486524+0000","last_fullsized":"2023-10-09T10:43:35.486524+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:47.200522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":72,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489284+0000","last_change":"2023-10-09T10:42:38.416111+0000","last_active":"2023-10-09T10:43:34.489284+0000","last_peered":"2023-10-09T10:43:34.489284+0000","last_clean":"2023-10-09T10:43:34.489284+0000","last_became_active":"2023-10-09T10:42:38.415861+0000","last_became_peered":"2023-10-09T10:42:38.415861+0000","last_unstale":"2023-10-09T10:43:34.489284+0000","last_undegraded":"2023-10-09T10:43:34.489284+0000","last_fullsized":"2023-10-09T10:43:34.489284+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:10:40.572274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.489126+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:43:35.489126+0000","last_peered":"2023-10-09T10:43:35.489126+0000","last_clean":"2023-10-09T10:43:35.489126+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:43:35.489126+0000","last_undegraded":"2023-10-09T10:43:35.489126+0000","last_fullsized":"2023-10-09T10:43:35.489126+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":64,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486030+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:43:35.486030+0000","last_peered":"2023-10-09T10:43:35.486030+0000","last_clean":"2023-10-09T10:43:35.486030+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:43:35.486030+0000","last_undegraded":"2023-10-09T10:43:35.486030+0000","last_fullsized":"2023-10-09T10:43:35.486030+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486884+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:43:34.486884+0000","last_peered":"2023-10-09T10:43:34.486884+0000","last_clean":"2023-10-09T10:43:34.486884+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:43:34.486884+0000","last_undegraded":"2023-10-09T10:43:34.486884+0000","last_fullsized":"2023-10-09T10:43:34.486884+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485938+0000","last_change":"2023-10-09T10:43:06.676577+0000","last_active":"2023-10-09T10:43:34.485938+0000","last_peered":"2023-10-09T10:43:34.485938+0000","last_clean":"2023-10-09T10:43:34.485938+0000","last_became_active":"2023-10-09T10:43:06.676165+0000","last_became_peered":"2023-10-09T10:43:06.676165+0000","last_unstale":"2023-10-09T10:43:34.485938+0000","last_undegraded":"2023-10-09T10:43:34.485938+0000","last_fullsized":"2023-10-09T10:43:34.485938+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:52.614985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":77,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487389+0000","last_change":"2023-10-09T10:43:06.676367+0000","last_active":"2023-10-09T10:43:34.487389+0000","last_peered":"2023-10-09T10:43:34.487389+0000","last_clean":"2023-10-09T10:43:34.487389+0000","last_became_active":"2023-10-09T10:43:06.676017+0000","last_became_peered":"2023-10-09T10:43:06.676017+0000","last_unstale":"2023-10-09T10:43:34.487389+0000","last_undegraded":"2023-10-09T10:43:34.487389+0000","last_fullsized":"2023-10-09T10:43:34.487389+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:54.387617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482664+0000","last_change":"2023-10-09T10:42:38.412169+0000","last_active":"2023-10-09T10:43:34.482664+0000","last_peered":"2023-10-09T10:43:34.482664+0000","last_clean":"2023-10-09T10:43:34.482664+0000","last_became_active":"2023-10-09T10:42:38.411829+0000","last_became_peered":"2023-10-09T10:42:38.411829+0000","last_unstale":"2023-10-09T10:43:34.482664+0000","last_undegraded":"2023-10-09T10:43:34.482664+0000","last_fullsized":"2023-10-09T10:43:34.482664+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:36:35.699293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.493287+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:43:35.493287+0000","last_peered":"2023-10-09T10:43:35.493287+0000","last_clean":"2023-10-09T10:43:35.493287+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:43:35.493287+0000","last_undegraded":"2023-10-09T10:43:35.493287+0000","last_fullsized":"2023-10-09T10:43:35.493287+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":61,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491435+0000","last_change":"2023-10-09T10:43:35.491435+0000","last_active":"2023-10-09T10:43:35.491435+0000","last_peered":"2023-10-09T10:43:35.491435+0000","last_clean":"2023-10-09T10:43:35.491435+0000","last_became_active":"2023-10-09T10:43:35.491096+0000","last_became_peered":"2023-10-09T10:43:35.491096+0000","last_unstale":"2023-10-09T10:43:35.491435+0000","last_undegraded":"2023-10-09T10:43:35.491435+0000","last_fullsized":"2023-10-09T10:43:35.491435+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:09.581525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483788+0000","last_change":"2023-10-09T10:42:38.407703+0000","last_active":"2023-10-09T10:43:35.483788+0000","last_peered":"2023-10-09T10:43:35.483788+0000","last_clean":"2023-10-09T10:43:35.483788+0000","last_became_active":"2023-10-09T10:42:38.407431+0000","last_became_peered":"2023-10-09T10:42:38.407431+0000","last_unstale":"2023-10-09T10:43:35.483788+0000","last_undegraded":"2023-10-09T10:43:35.483788+0000","last_fullsized":"2023-10-09T10:43:35.483788+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:23.450634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487326+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:43:35.487326+0000","last_peered":"2023-10-09T10:43:35.487326+0000","last_clean":"2023-10-09T10:43:35.487326+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:43:35.487326+0000","last_undegraded":"2023-10-09T10:43:35.487326+0000","last_fullsized":"2023-10-09T10:43:35.487326+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482266+0000","last_change":"2023-10-09T10:42:38.411928+0000","last_active":"2023-10-09T10:43:34.482266+0000","last_peered":"2023-10-09T10:43:34.482266+0000","last_clean":"2023-10-09T10:43:34.482266+0000","last_became_active":"2023-10-09T10:42:38.411560+0000","last_became_peered":"2023-10-09T10:42:38.411560+0000","last_unstale":"2023-10-09T10:43:34.482266+0000","last_undegraded":"2023-10-09T10:43:34.482266+0000","last_fullsized":"2023-10-09T10:43:34.482266+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:07.711753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":45,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487828+0000","last_change":"2023-10-09T10:35:56.822899+0000","last_active":"2023-10-09T10:43:34.487828+0000","last_peered":"2023-10-09T10:43:34.487828+0000","last_clean":"2023-10-09T10:43:34.487828+0000","last_became_active":"2023-10-09T10:35:56.822564+0000","last_became_peered":"2023-10-09T10:35:56.822564+0000","last_unstale":"2023-10-09T10:43:34.487828+0000","last_undegraded":"2023-10-09T10:43:34.487828+0000","last_fullsized":"2023-10-09T10:43:34.487828+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:25:03.398416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.493278+0000","last_change":"2023-10-09T10:43:35.493278+0000","last_active":"2023-10-09T10:43:35.493278+0000","last_peered":"2023-10-09T10:43:35.493278+0000","last_clean":"2023-10-09T10:43:35.493278+0000","last_became_active":"2023-10-09T10:43:35.492915+0000","last_became_peered":"2023-10-09T10:43:35.492915+0000","last_unstale":"2023-10-09T10:43:35.493278+0000","last_undegraded":"2023-10-09T10:43:35.493278+0000","last_fullsized":"2023-10-09T10:43:35.493278+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:42.227364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487004+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:43:34.487004+0000","last_peered":"2023-10-09T10:43:34.487004+0000","last_clean":"2023-10-09T10:43:34.487004+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:43:34.487004+0000","last_undegraded":"2023-10-09T10:43:34.487004+0000","last_fullsized":"2023-10-09T10:43:34.487004+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479053+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:43:34.479053+0000","last_peered":"2023-10-09T10:43:34.479053+0000","last_clean":"2023-10-09T10:43:34.479053+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:43:34.479053+0000","last_undegraded":"2023-10-09T10:43:34.479053+0000","last_fullsized":"2023-10-09T10:43:34.479053+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487772+0000","last_change":"2023-10-09T10:41:29.516129+0000","last_active":"2023-10-09T10:43:34.487772+0000","last_peered":"2023-10-09T10:43:34.487772+0000","last_clean":"2023-10-09T10:43:34.487772+0000","last_became_active":"2023-10-09T10:41:29.515741+0000","last_became_peered":"2023-10-09T10:41:29.515741+0000","last_unstale":"2023-10-09T10:43:34.487772+0000","last_undegraded":"2023-10-09T10:43:34.487772+0000","last_fullsized":"2023-10-09T10:43:34.487772+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:29:14.059627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":63,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487647+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:43:35.487647+0000","last_peered":"2023-10-09T10:43:35.487647+0000","last_clean":"2023-10-09T10:43:35.487647+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:43:35.487647+0000","last_undegraded":"2023-10-09T10:43:35.487647+0000","last_fullsized":"2023-10-09T10:43:35.487647+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":69,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.481806+0000","last_change":"2023-10-09T10:41:29.512929+0000","last_active":"2023-10-09T10:43:34.481806+0000","last_peered":"2023-10-09T10:43:34.481806+0000","last_clean":"2023-10-09T10:43:34.481806+0000","last_became_active":"2023-10-09T10:41:29.512591+0000","last_became_peered":"2023-10-09T10:41:29.512591+0000","last_unstale":"2023-10-09T10:43:34.481806+0000","last_undegraded":"2023-10-09T10:43:34.481806+0000","last_fullsized":"2023-10-09T10:43:34.481806+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:35.703322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.490293+0000","last_change":"2023-10-09T10:43:35.487796+0000","last_active":"2023-10-09T10:43:35.490293+0000","last_peered":"2023-10-09T10:43:35.490293+0000","last_clean":"2023-10-09T10:43:35.490293+0000","last_became_active":"2023-10-09T10:43:35.487518+0000","last_became_peered":"2023-10-09T10:43:35.487518+0000","last_unstale":"2023-10-09T10:43:35.490293+0000","last_undegraded":"2023-10-09T10:43:35.490293+0000","last_fullsized":"2023-10-09T10:43:35.490293+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:47.089685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":44,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.484279+0000","last_change":"2023-10-09T10:35:56.824257+0000","last_active":"2023-10-09T10:43:34.484279+0000","last_peered":"2023-10-09T10:43:34.484279+0000","last_clean":"2023-10-09T10:43:34.484279+0000","last_became_active":"2023-10-09T10:35:56.823152+0000","last_became_peered":"2023-10-09T10:35:56.823152+0000","last_unstale":"2023-10-09T10:43:34.484279+0000","last_undegraded":"2023-10-09T10:43:34.484279+0000","last_fullsized":"2023-10-09T10:43:34.484279+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:52.392494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486785+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:43:35.486785+0000","last_peered":"2023-10-09T10:43:35.486785+0000","last_clean":"2023-10-09T10:43:35.486785+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:43:35.486785+0000","last_undegraded":"2023-10-09T10:43:35.486785+0000","last_fullsized":"2023-10-09T10:43:35.486785+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487384+0000","last_change":"2023-10-09T10:43:35.486914+0000","last_active":"2023-10-09T10:43:35.487384+0000","last_peered":"2023-10-09T10:43:35.487384+0000","last_clean":"2023-10-09T10:43:35.487384+0000","last_became_active":"2023-10-09T10:43:35.486623+0000","last_became_peered":"2023-10-09T10:43:35.486623+0000","last_unstale":"2023-10-09T10:43:35.487384+0000","last_undegraded":"2023-10-09T10:43:35.487384+0000","last_fullsized":"2023-10-09T10:43:35.487384+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:13.339801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":77,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487942+0000","last_change":"2023-10-09T10:43:35.487942+0000","last_active":"2023-10-09T10:43:35.487942+0000","last_peered":"2023-10-09T10:43:35.487942+0000","last_clean":"2023-10-09T10:43:35.487942+0000","last_became_active":"2023-10-09T10:43:35.487651+0000","last_became_peered":"2023-10-09T10:43:35.487651+0000","last_unstale":"2023-10-09T10:43:35.487942+0000","last_undegraded":"2023-10-09T10:43:35.487942+0000","last_fullsized":"2023-10-09T10:43:35.487942+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:51.882595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485380+0000","last_change":"2023-10-09T10:43:06.677115+0000","last_active":"2023-10-09T10:43:34.485380+0000","last_peered":"2023-10-09T10:43:34.485380+0000","last_clean":"2023-10-09T10:43:34.485380+0000","last_became_active":"2023-10-09T10:43:06.676739+0000","last_became_peered":"2023-10-09T10:43:06.676739+0000","last_unstale":"2023-10-09T10:43:34.485380+0000","last_undegraded":"2023-10-09T10:43:34.485380+0000","last_fullsized":"2023-10-09T10:43:34.485380+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:02.520770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487176+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:43:34.487176+0000","last_peered":"2023-10-09T10:43:34.487176+0000","last_clean":"2023-10-09T10:43:34.487176+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:43:34.487176+0000","last_undegraded":"2023-10-09T10:43:34.487176+0000","last_fullsized":"2023-10-09T10:43:34.487176+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":66,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486392+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:43:34.486392+0000","last_peered":"2023-10-09T10:43:34.486392+0000","last_clean":"2023-10-09T10:43:34.486392+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:43:34.486392+0000","last_undegraded":"2023-10-09T10:43:34.486392+0000","last_fullsized":"2023-10-09T10:43:34.486392+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486237+0000","last_change":"2023-10-09T10:43:06.672114+0000","last_active":"2023-10-09T10:43:34.486237+0000","last_peered":"2023-10-09T10:43:34.486237+0000","last_clean":"2023-10-09T10:43:34.486237+0000","last_became_active":"2023-10-09T10:43:06.671827+0000","last_became_peered":"2023-10-09T10:43:06.671827+0000","last_unstale":"2023-10-09T10:43:34.486237+0000","last_undegraded":"2023-10-09T10:43:34.486237+0000","last_fullsized":"2023-10-09T10:43:34.486237+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:41.701409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":64,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491382+0000","last_change":"2023-10-09T10:43:35.491382+0000","last_active":"2023-10-09T10:43:35.491382+0000","last_peered":"2023-10-09T10:43:35.491382+0000","last_clean":"2023-10-09T10:43:35.491382+0000","last_became_active":"2023-10-09T10:43:35.491062+0000","last_became_peered":"2023-10-09T10:43:35.491062+0000","last_unstale":"2023-10-09T10:43:35.491382+0000","last_undegraded":"2023-10-09T10:43:35.491382+0000","last_fullsized":"2023-10-09T10:43:35.491382+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:10.396267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487546+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:43:35.487546+0000","last_peered":"2023-10-09T10:43:35.487546+0000","last_clean":"2023-10-09T10:43:35.487546+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:43:35.487546+0000","last_undegraded":"2023-10-09T10:43:35.487546+0000","last_fullsized":"2023-10-09T10:43:35.487546+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479334+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:43:34.479334+0000","last_peered":"2023-10-09T10:43:34.479334+0000","last_clean":"2023-10-09T10:43:34.479334+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:43:34.479334+0000","last_undegraded":"2023-10-09T10:43:34.479334+0000","last_fullsized":"2023-10-09T10:43:34.479334+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":63,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.484250+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:43:35.484250+0000","last_peered":"2023-10-09T10:43:35.484250+0000","last_clean":"2023-10-09T10:43:35.484250+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:43:35.484250+0000","last_undegraded":"2023-10-09T10:43:35.484250+0000","last_fullsized":"2023-10-09T10:43:35.484250+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.488427+0000","last_change":"2023-10-09T10:43:35.487114+0000","last_active":"2023-10-09T10:43:35.488427+0000","last_peered":"2023-10-09T10:43:35.488427+0000","last_clean":"2023-10-09T10:43:35.488427+0000","last_became_active":"2023-10-09T10:43:35.486830+0000","last_became_peered":"2023-10-09T10:43:35.486830+0000","last_unstale":"2023-10-09T10:43:35.488427+0000","last_undegraded":"2023-10-09T10:43:35.488427+0000","last_fullsized":"2023-10-09T10:43:35.488427+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:08.836969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482547+0000","last_change":"2023-10-09T10:42:38.413301+0000","last_active":"2023-10-09T10:43:34.482547+0000","last_peered":"2023-10-09T10:43:34.482547+0000","last_clean":"2023-10-09T10:43:34.482547+0000","last_became_active":"2023-10-09T10:42:38.413006+0000","last_became_peered":"2023-10-09T10:42:38.413006+0000","last_unstale":"2023-10-09T10:43:34.482547+0000","last_undegraded":"2023-10-09T10:43:34.482547+0000","last_fullsized":"2023-10-09T10:43:34.482547+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:33.462419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":81,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.485235+0000","last_change":"2023-10-09T10:43:35.485235+0000","last_active":"2023-10-09T10:43:35.485235+0000","last_peered":"2023-10-09T10:43:35.485235+0000","last_clean":"2023-10-09T10:43:35.485235+0000","last_became_active":"2023-10-09T10:43:35.484917+0000","last_became_peered":"2023-10-09T10:43:35.484917+0000","last_unstale":"2023-10-09T10:43:35.485235+0000","last_undegraded":"2023-10-09T10:43:35.485235+0000","last_fullsized":"2023-10-09T10:43:35.485235+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:55.417326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487224+0000","last_change":"2023-10-09T10:43:35.487224+0000","last_active":"2023-10-09T10:43:35.487224+0000","last_peered":"2023-10-09T10:43:35.487224+0000","last_clean":"2023-10-09T10:43:35.487224+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:43:35.487224+0000","last_undegraded":"2023-10-09T10:43:35.487224+0000","last_fullsized":"2023-10-09T10:43:35.487224+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:44:52.066778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491388+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:43:35.491388+0000","last_peered":"2023-10-09T10:43:35.491388+0000","last_clean":"2023-10-09T10:43:35.491388+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:43:35.491388+0000","last_undegraded":"2023-10-09T10:43:35.491388+0000","last_fullsized":"2023-10-09T10:43:35.491388+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":63,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.490890+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:43:35.490890+0000","last_peered":"2023-10-09T10:43:35.490890+0000","last_clean":"2023-10-09T10:43:35.490890+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:43:35.490890+0000","last_undegraded":"2023-10-09T10:43:35.490890+0000","last_fullsized":"2023-10-09T10:43:35.490890+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":79,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486285+0000","last_change":"2023-10-09T10:43:06.676414+0000","last_active":"2023-10-09T10:43:34.486285+0000","last_peered":"2023-10-09T10:43:34.486285+0000","last_clean":"2023-10-09T10:43:34.486285+0000","last_became_active":"2023-10-09T10:43:06.676050+0000","last_became_peered":"2023-10-09T10:43:06.676050+0000","last_unstale":"2023-10-09T10:43:34.486285+0000","last_undegraded":"2023-10-09T10:43:34.486285+0000","last_fullsized":"2023-10-09T10:43:34.486285+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:28.011477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332211,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311116,"kb_used_data":1540,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93434036,"statfs":{"total":95995035648,"available":95676452864,"internally_reserved":0,"allocated":1576960,"data_stored":1120859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:31 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.56399999999999995,"15min":0.57399999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.67200000000000004,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.61799999999999999,"15min":0.63400000000000001},"min":{"1min":0.36499999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.69799999999999995,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.65700000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:43:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.433},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.45500000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.54300000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.58399999999999996,"15min":0.56699999999999995},"min":{"1min":0.41399999999999998,"5min":0.35099999999999998,"15min":0.33800000000000002},"max":{"1min":0.74099999999999999,"5min":2.569,"15min":2.569},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59099999999999997,"15min":0.57399999999999995},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.76900000000000002,"5min":0.80100000000000005,"15min":1.232},"last":0.64100000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48599999999999999,"15min":0.47299999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.73999999999999999,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.502,"5min":0.497,"15min":0.47399999999999998},"min":{"1min":0.36899999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.67100000000000004,"5min":0.80600000000000005,"15min":0.83699999999999997},"last":0.68200000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.52100000000000002,"15min":0.504},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":1.1950000000000001},"last":0.628},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.55300000000000005,"15min":0.54000000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.25700000000000001},"max":{"1min":0.86099999999999999,"5min":0.876,"15min":1.0649999999999999},"last":0.71599999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58199999999999996,"15min":0.55700000000000005},"min":{"1min":0.40500000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.81699999999999995,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.57399999999999995,"15min":0.54300000000000004},"min":{"1min":0.39000000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.79700000000000004,"5min":0.89200000000000002,"15min":0.93500000000000005},"last":0.70499999999999996}]}]},{"osd":6,"up_from":44,"seq":188978561142,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298816,"kb_used_data":1528,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446336,"statfs":{"total":95995035648,"available":95689048064,"internally_reserved":0,"allocated":1564672,"data_stored":1120713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:34 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.59699999999999998,"15min":0.60599999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.73999999999999999,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.63,"5min":0.61199999999999999,"15min":0.60899999999999999},"min":{"1min":0.46600000000000003,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.66800000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.747},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.71199999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.68700000000000006}]},{"osd":3,"last update":"Mon Oct 9 10:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56100000000000005,"15min":0.52200000000000002},"min":{"1min":0.375,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72499999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54700000000000004,"15min":0.53200000000000003},"min":{"1min":0.39800000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80400000000000005},"last":0.63700000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.45900000000000002,"15min":0.44400000000000001},"min":{"1min":0.38900000000000001,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":0.59699999999999998,"5min":0.67400000000000004,"15min":0.70999999999999996},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.505,"15min":0.49199999999999999},"min":{"1min":0.35299999999999998,"5min":0.29899999999999999,"15min":0.29699999999999999},"max":{"1min":0.74399999999999999,"5min":0.77000000000000002,"15min":0.78500000000000003},"last":0.51200000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.51500000000000001,"15min":0.48399999999999999},"min":{"1min":0.40100000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.67600000000000005,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.55000000000000004,"15min":0.53000000000000003},"min":{"1min":0.372,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.84399999999999997},"last":0.56100000000000005}]},{"osd":7,"last update":"Mon Oct 9 10:43:18 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.53100000000000003,"15min":0.52600000000000002},"min":{"1min":0.33900000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.72599999999999998,"5min":0.78600000000000003,"15min":0.79100000000000004},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.438,"5min":0.34100000000000003,"15min":0.34000000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":1.018},"last":0.61699999999999999}]}]},{"osd":1,"up_from":85,"seq":365072220181,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28448,"kb_used_data":984,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716704,"statfs":{"total":95995035648,"available":95965904896,"internally_reserved":0,"allocated":1007616,"data_stored":536014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.42699999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:44:08 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.47999999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45000000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.82299999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.86399999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.71099999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.75900000000000001}]}]},{"osd":0,"up_from":80,"seq":343597383715,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29108,"kb_used_data":1580,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93716044,"statfs":{"total":95995035648,"available":95965229056,"internally_reserved":0,"allocated":1617920,"data_stored":1126957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:43:38 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.29299999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"last":0.249}]},{"osd":3,"last update":"Mon Oct 9 10:42:30 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.38100000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.61699999999999999,"15min":0.61899999999999999},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61699999999999999,"15min":0.61799999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.83499999999999996,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.496}]},{"osd":5,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.62,"15min":0.624},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.749,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.625,"5min":0.64400000000000002,"15min":0.64700000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.59399999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.628,"15min":0.63},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.61599999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.64700000000000002,"15min":0.65100000000000002},"min":{"1min":0.44600000000000001,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.66900000000000004,"15min":0.67500000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.48599999999999999}]}]},{"osd":2,"up_from":90,"seq":386547056655,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28572,"kb_used_data":980,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716580,"statfs":{"total":95995035648,"available":95965777920,"internally_reserved":0,"allocated":1003520,"data_stored":533482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.312}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.371}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34699999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.623}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54200000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71999999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68899999999999995}]}]},{"osd":3,"up_from":95,"seq":408021893129,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28424,"kb_used_data":960,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716728,"statfs":{"total":95995035648,"available":95965929472,"internally_reserved":0,"allocated":983040,"data_stored":530397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30399999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.318}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40000000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58899999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67000000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65000000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019005,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302344,"kb_used_data":960,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93442808,"statfs":{"total":95995035648,"available":95685435392,"internally_reserved":0,"allocated":983040,"data_stored":530744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.55500000000000005,"15min":0.55700000000000005},"min":{"1min":0.35199999999999998,"5min":0.311,"15min":0.311},"max":{"1min":0.74299999999999999,"5min":0.755,"15min":0.755},"last":0.621},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.57899999999999996,"15min":0.58299999999999996},"min":{"1min":0.41599999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.747,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.60099999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.65800000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:44:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.496}]},{"osd":3,"last update":"Mon Oct 9 10:43:21 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.58499999999999996,"15min":0.56200000000000006},"min":{"1min":0.45700000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.84799999999999998,"5min":0.85099999999999998,"15min":0.86399999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.57299999999999995,"15min":0.56799999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.83699999999999997,"5min":0.86599999999999999,"15min":2.0910000000000002},"last":0.73599999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.47399999999999998,"15min":0.438},"min":{"1min":0.34300000000000003,"5min":0.253,"15min":0.253},"max":{"1min":0.65000000000000002,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.48199999999999998,"15min":0.442},"min":{"1min":0.34599999999999997,"5min":0.28499999999999998,"15min":0.26400000000000001},"max":{"1min":0.73299999999999998,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.70199999999999996}]},{"osd":6,"last update":"Mon Oct 9 10:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.50800000000000001,"15min":0.46899999999999997},"min":{"1min":0.32000000000000001,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":0.752,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59499999999999997,"15min":0.56000000000000005},"min":{"1min":0.46500000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.82299999999999995,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.71599999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:43:11 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.57399999999999995,"15min":0.56899999999999995},"min":{"1min":0.376,"5min":0.34999999999999998,"15min":0.308},"max":{"1min":0.872,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.626,"5min":0.56899999999999995,"15min":0.53800000000000003},"min":{"1min":0.32500000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.80700000000000005,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.76400000000000001}]}]},{"osd":5,"up_from":37,"seq":158913790074,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306420,"kb_used_data":940,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93438732,"statfs":{"total":95995035648,"available":95681261568,"internally_reserved":0,"allocated":962560,"data_stored":526165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.66200000000000003,"15min":0.66700000000000004},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.81699999999999995}]},{"osd":1,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.77300000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:44:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.74199999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:42:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.54500000000000004,"15min":0.52600000000000002},"min":{"1min":0.39600000000000002,"5min":0.36899999999999999,"15min":0.35699999999999998},"max":{"1min":0.76100000000000001,"5min":0.79000000000000004,"15min":2.976},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.59199999999999997,"15min":0.59099999999999997},"min":{"1min":0.44700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.78400000000000003,"5min":0.92400000000000004,"15min":4.7949999999999999},"last":0.71199999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47599999999999998,"15min":0.43099999999999999},"min":{"1min":0.35399999999999998,"5min":0.30499999999999999,"15min":0.25},"max":{"1min":0.69099999999999995,"5min":0.75600000000000001,"15min":2.4649999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.5,"15min":0.45100000000000001},"min":{"1min":0.371,"5min":0.30099999999999999,"15min":0.28799999999999998},"max":{"1min":0.60199999999999998,"5min":0.85299999999999998,"15min":1.98},"last":0.51000000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.53200000000000003,"15min":0.45800000000000002},"min":{"1min":0.39300000000000002,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":0.67500000000000004,"5min":0.85599999999999998,"15min":2.048},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56599999999999995,"15min":0.49099999999999999},"min":{"1min":0.47099999999999997,"5min":0.33500000000000002,"15min":0.30299999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.56200000000000006}]},{"osd":7,"last update":"Mon Oct 9 10:43:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.57199999999999995,"15min":0.53500000000000003},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.32400000000000001},"max":{"1min":0.80700000000000005,"5min":0.879,"15min":2.1110000000000002},"last":0.627},{"interface":"front","average":{"1min":0.627,"5min":0.59699999999999998,"15min":0.53600000000000003},"min":{"1min":0.48799999999999999,"5min":0.35699999999999998,"15min":0.31900000000000001},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":1.845},"last":0.69399999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:44:13.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:44:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:13 smithi155 ceph-mon[184854]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:14.130 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:14.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:14 smithi125 ceph-mon[195507]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:44:14.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:14 smithi125 ceph-mon[200197]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:44:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:14 smithi155 ceph-mon[184854]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:44:15.419 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:44:15.421 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:44:15.753 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:44:15.753 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:15 smithi125 ceph-mon[195507]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:15.753 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:44:15.753 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:15 smithi125 ceph-mon[200197]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:15.804 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":188,"stamp":"2023-10-09T10:44:15.003537+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1333248,"kb_used_data":9472,"kb_used_omap":126,"kb_used_meta":1323585,"kb_avail":748627968,"statfs":{"total":767960285184,"available":766595039232,"internally_reserved":0,"allocated":9699328,"data_stored":6025331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1355351675},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003094"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485359+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:43:34.485359+0000","last_peered":"2023-10-09T10:43:34.485359+0000","last_clean":"2023-10-09T10:43:34.485359+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:43:34.485359+0000","last_undegraded":"2023-10-09T10:43:34.485359+0000","last_fullsized":"2023-10-09T10:43:34.485359+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":57,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.494552+0000","last_change":"2023-10-09T10:43:06.674549+0000","last_active":"2023-10-09T10:43:34.494552+0000","last_peered":"2023-10-09T10:43:34.494552+0000","last_clean":"2023-10-09T10:43:34.494552+0000","last_became_active":"2023-10-09T10:43:06.673510+0000","last_became_peered":"2023-10-09T10:43:06.673510+0000","last_unstale":"2023-10-09T10:43:34.494552+0000","last_undegraded":"2023-10-09T10:43:34.494552+0000","last_fullsized":"2023-10-09T10:43:34.494552+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:32.996795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":84,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487344+0000","last_change":"2023-10-09T10:43:35.487344+0000","last_active":"2023-10-09T10:43:35.487344+0000","last_peered":"2023-10-09T10:43:35.487344+0000","last_clean":"2023-10-09T10:43:35.487344+0000","last_became_active":"2023-10-09T10:43:35.486672+0000","last_became_peered":"2023-10-09T10:43:35.486672+0000","last_unstale":"2023-10-09T10:43:35.487344+0000","last_undegraded":"2023-10-09T10:43:35.487344+0000","last_fullsized":"2023-10-09T10:43:35.487344+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:44.665687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":69,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.485790+0000","last_change":"2023-10-09T10:43:35.485790+0000","last_active":"2023-10-09T10:43:35.485790+0000","last_peered":"2023-10-09T10:43:35.485790+0000","last_clean":"2023-10-09T10:43:35.485790+0000","last_became_active":"2023-10-09T10:43:35.485436+0000","last_became_peered":"2023-10-09T10:43:35.485436+0000","last_unstale":"2023-10-09T10:43:35.485790+0000","last_undegraded":"2023-10-09T10:43:35.485790+0000","last_fullsized":"2023-10-09T10:43:35.485790+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:45:20.897767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":77,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.492236+0000","last_change":"2023-10-09T10:43:35.492236+0000","last_active":"2023-10-09T10:43:35.492236+0000","last_peered":"2023-10-09T10:43:35.492236+0000","last_clean":"2023-10-09T10:43:35.492236+0000","last_became_active":"2023-10-09T10:43:35.491913+0000","last_became_peered":"2023-10-09T10:43:35.491913+0000","last_unstale":"2023-10-09T10:43:35.492236+0000","last_undegraded":"2023-10-09T10:43:35.492236+0000","last_fullsized":"2023-10-09T10:43:35.492236+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:34:13.448179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":46,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.480679+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:43:34.480679+0000","last_peered":"2023-10-09T10:43:34.480679+0000","last_clean":"2023-10-09T10:43:34.480679+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:43:34.480679+0000","last_undegraded":"2023-10-09T10:43:34.480679+0000","last_fullsized":"2023-10-09T10:43:34.480679+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":77,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.492737+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:43:35.492737+0000","last_peered":"2023-10-09T10:43:35.492737+0000","last_clean":"2023-10-09T10:43:35.492737+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:43:35.492737+0000","last_undegraded":"2023-10-09T10:43:35.492737+0000","last_fullsized":"2023-10-09T10:43:35.492737+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479260+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:43:34.479260+0000","last_peered":"2023-10-09T10:43:34.479260+0000","last_clean":"2023-10-09T10:43:34.479260+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:43:34.479260+0000","last_undegraded":"2023-10-09T10:43:34.479260+0000","last_fullsized":"2023-10-09T10:43:34.479260+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":82,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.489655+0000","last_change":"2023-10-09T10:43:35.489655+0000","last_active":"2023-10-09T10:43:35.489655+0000","last_peered":"2023-10-09T10:43:35.489655+0000","last_clean":"2023-10-09T10:43:35.489655+0000","last_became_active":"2023-10-09T10:43:35.489149+0000","last_became_peered":"2023-10-09T10:43:35.489149+0000","last_unstale":"2023-10-09T10:43:35.489655+0000","last_undegraded":"2023-10-09T10:43:35.489655+0000","last_fullsized":"2023-10-09T10:43:35.489655+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:30.598962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":42,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483874+0000","last_change":"2023-10-09T10:41:29.514075+0000","last_active":"2023-10-09T10:43:35.483874+0000","last_peered":"2023-10-09T10:43:35.483874+0000","last_clean":"2023-10-09T10:43:35.483874+0000","last_became_active":"2023-10-09T10:41:29.513347+0000","last_became_peered":"2023-10-09T10:41:29.513347+0000","last_unstale":"2023-10-09T10:43:35.483874+0000","last_undegraded":"2023-10-09T10:43:35.483874+0000","last_fullsized":"2023-10-09T10:43:35.483874+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:17.504888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485850+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:43:34.485850+0000","last_peered":"2023-10-09T10:43:34.485850+0000","last_clean":"2023-10-09T10:43:34.485850+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:43:34.485850+0000","last_undegraded":"2023-10-09T10:43:34.485850+0000","last_fullsized":"2023-10-09T10:43:34.485850+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":57,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483729+0000","last_change":"2023-10-09T10:42:38.407058+0000","last_active":"2023-10-09T10:43:35.483729+0000","last_peered":"2023-10-09T10:43:35.483729+0000","last_clean":"2023-10-09T10:43:35.483729+0000","last_became_active":"2023-10-09T10:42:38.406768+0000","last_became_peered":"2023-10-09T10:42:38.406768+0000","last_unstale":"2023-10-09T10:43:35.483729+0000","last_undegraded":"2023-10-09T10:43:35.483729+0000","last_fullsized":"2023-10-09T10:43:35.483729+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:41.198843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482345+0000","last_change":"2023-10-09T10:43:06.674387+0000","last_active":"2023-10-09T10:43:34.482345+0000","last_peered":"2023-10-09T10:43:34.482345+0000","last_clean":"2023-10-09T10:43:34.482345+0000","last_became_active":"2023-10-09T10:43:06.674032+0000","last_became_peered":"2023-10-09T10:43:06.674032+0000","last_unstale":"2023-10-09T10:43:34.482345+0000","last_undegraded":"2023-10-09T10:43:34.482345+0000","last_fullsized":"2023-10-09T10:43:34.482345+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:58.222034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":58,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486342+0000","last_change":"2023-10-09T10:43:35.486342+0000","last_active":"2023-10-09T10:43:35.486342+0000","last_peered":"2023-10-09T10:43:35.486342+0000","last_clean":"2023-10-09T10:43:35.486342+0000","last_became_active":"2023-10-09T10:43:35.486026+0000","last_became_peered":"2023-10-09T10:43:35.486026+0000","last_unstale":"2023-10-09T10:43:35.486342+0000","last_undegraded":"2023-10-09T10:43:35.486342+0000","last_fullsized":"2023-10-09T10:43:35.486342+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:16.616429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489374+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:43:34.489374+0000","last_peered":"2023-10-09T10:43:34.489374+0000","last_clean":"2023-10-09T10:43:34.489374+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:43:34.489374+0000","last_undegraded":"2023-10-09T10:43:34.489374+0000","last_fullsized":"2023-10-09T10:43:34.489374+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":61,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487842+0000","last_change":"2023-10-09T10:43:35.487842+0000","last_active":"2023-10-09T10:43:35.487842+0000","last_peered":"2023-10-09T10:43:35.487842+0000","last_clean":"2023-10-09T10:43:35.487842+0000","last_became_active":"2023-10-09T10:43:35.487192+0000","last_became_peered":"2023-10-09T10:43:35.487192+0000","last_unstale":"2023-10-09T10:43:35.487842+0000","last_undegraded":"2023-10-09T10:43:35.487842+0000","last_fullsized":"2023-10-09T10:43:35.487842+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:35.477280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.492904+0000","last_change":"2023-10-09T10:41:29.513802+0000","last_active":"2023-10-09T10:43:34.492904+0000","last_peered":"2023-10-09T10:43:34.492904+0000","last_clean":"2023-10-09T10:43:34.492904+0000","last_became_active":"2023-10-09T10:41:29.512895+0000","last_became_peered":"2023-10-09T10:41:29.512895+0000","last_unstale":"2023-10-09T10:43:34.492904+0000","last_undegraded":"2023-10-09T10:43:34.492904+0000","last_fullsized":"2023-10-09T10:43:34.492904+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:06.177976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489173+0000","last_change":"2023-10-09T10:43:06.671356+0000","last_active":"2023-10-09T10:43:34.489173+0000","last_peered":"2023-10-09T10:43:34.489173+0000","last_clean":"2023-10-09T10:43:34.489173+0000","last_became_active":"2023-10-09T10:43:06.671040+0000","last_became_peered":"2023-10-09T10:43:06.671040+0000","last_unstale":"2023-10-09T10:43:34.489173+0000","last_undegraded":"2023-10-09T10:43:34.489173+0000","last_fullsized":"2023-10-09T10:43:34.489173+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:47.601448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486509+0000","last_change":"2023-10-09T10:41:29.512225+0000","last_active":"2023-10-09T10:43:34.486509+0000","last_peered":"2023-10-09T10:43:34.486509+0000","last_clean":"2023-10-09T10:43:34.486509+0000","last_became_active":"2023-10-09T10:41:29.511907+0000","last_became_peered":"2023-10-09T10:41:29.511907+0000","last_unstale":"2023-10-09T10:43:34.486509+0000","last_undegraded":"2023-10-09T10:43:34.486509+0000","last_fullsized":"2023-10-09T10:43:34.486509+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:25.351050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483811+0000","last_change":"2023-10-09T10:43:06.673430+0000","last_active":"2023-10-09T10:43:35.483811+0000","last_peered":"2023-10-09T10:43:35.483811+0000","last_clean":"2023-10-09T10:43:35.483811+0000","last_became_active":"2023-10-09T10:43:06.673110+0000","last_became_peered":"2023-10-09T10:43:06.673110+0000","last_unstale":"2023-10-09T10:43:35.483811+0000","last_undegraded":"2023-10-09T10:43:35.483811+0000","last_fullsized":"2023-10-09T10:43:35.483811+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:35.429715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491431+0000","last_change":"2023-10-09T10:43:35.491431+0000","last_active":"2023-10-09T10:43:35.491431+0000","last_peered":"2023-10-09T10:43:35.491431+0000","last_clean":"2023-10-09T10:43:35.491431+0000","last_became_active":"2023-10-09T10:43:35.491153+0000","last_became_peered":"2023-10-09T10:43:35.491153+0000","last_unstale":"2023-10-09T10:43:35.491431+0000","last_undegraded":"2023-10-09T10:43:35.491431+0000","last_fullsized":"2023-10-09T10:43:35.491431+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:58.495730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":82,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485088+0000","last_change":"2023-10-09T10:42:38.417090+0000","last_active":"2023-10-09T10:43:34.485088+0000","last_peered":"2023-10-09T10:43:34.485088+0000","last_clean":"2023-10-09T10:43:34.485088+0000","last_became_active":"2023-10-09T10:42:38.416758+0000","last_became_peered":"2023-10-09T10:42:38.416758+0000","last_unstale":"2023-10-09T10:43:34.485088+0000","last_undegraded":"2023-10-09T10:43:34.485088+0000","last_fullsized":"2023-10-09T10:43:34.485088+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:52.373940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482379+0000","last_change":"2023-10-09T10:35:57.165395+0000","last_active":"2023-10-09T10:43:34.482379+0000","last_peered":"2023-10-09T10:43:34.482379+0000","last_clean":"2023-10-09T10:43:34.482379+0000","last_became_active":"2023-10-09T10:35:57.165077+0000","last_became_peered":"2023-10-09T10:35:57.165077+0000","last_unstale":"2023-10-09T10:43:34.482379+0000","last_undegraded":"2023-10-09T10:43:34.482379+0000","last_fullsized":"2023-10-09T10:43:34.482379+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:53.159616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.485969+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:43:35.485969+0000","last_peered":"2023-10-09T10:43:35.485969+0000","last_clean":"2023-10-09T10:43:35.485969+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:43:35.485969+0000","last_undegraded":"2023-10-09T10:43:35.485969+0000","last_fullsized":"2023-10-09T10:43:35.485969+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":441,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.484997+0000","last_change":"2023-10-09T10:41:29.515426+0000","last_active":"2023-10-09T10:43:34.484997+0000","last_peered":"2023-10-09T10:43:34.484997+0000","last_clean":"2023-10-09T10:43:34.484997+0000","last_became_active":"2023-10-09T10:41:29.515012+0000","last_became_peered":"2023-10-09T10:41:29.515012+0000","last_unstale":"2023-10-09T10:43:34.484997+0000","last_undegraded":"2023-10-09T10:43:34.484997+0000","last_fullsized":"2023-10-09T10:43:34.484997+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:23.973968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479081+0000","last_change":"2023-10-09T10:43:06.673931+0000","last_active":"2023-10-09T10:43:34.479081+0000","last_peered":"2023-10-09T10:43:34.479081+0000","last_clean":"2023-10-09T10:43:34.479081+0000","last_became_active":"2023-10-09T10:43:06.673605+0000","last_became_peered":"2023-10-09T10:43:06.673605+0000","last_unstale":"2023-10-09T10:43:34.479081+0000","last_undegraded":"2023-10-09T10:43:34.479081+0000","last_fullsized":"2023-10-09T10:43:34.479081+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:19.088672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":67,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489950+0000","last_change":"2023-10-09T10:42:38.406633+0000","last_active":"2023-10-09T10:43:34.489950+0000","last_peered":"2023-10-09T10:43:34.489950+0000","last_clean":"2023-10-09T10:43:34.489950+0000","last_became_active":"2023-10-09T10:42:38.406317+0000","last_became_peered":"2023-10-09T10:42:38.406317+0000","last_unstale":"2023-10-09T10:43:34.489950+0000","last_undegraded":"2023-10-09T10:43:34.489950+0000","last_fullsized":"2023-10-09T10:43:34.489950+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:02.600240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487885+0000","last_change":"2023-10-09T10:43:06.672862+0000","last_active":"2023-10-09T10:43:34.487885+0000","last_peered":"2023-10-09T10:43:34.487885+0000","last_clean":"2023-10-09T10:43:34.487885+0000","last_became_active":"2023-10-09T10:43:06.672572+0000","last_became_peered":"2023-10-09T10:43:06.672572+0000","last_unstale":"2023-10-09T10:43:34.487885+0000","last_undegraded":"2023-10-09T10:43:34.487885+0000","last_fullsized":"2023-10-09T10:43:34.487885+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:23.743910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":83,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.490168+0000","last_change":"2023-10-09T10:43:35.490168+0000","last_active":"2023-10-09T10:43:35.490168+0000","last_peered":"2023-10-09T10:43:35.490168+0000","last_clean":"2023-10-09T10:43:35.490168+0000","last_became_active":"2023-10-09T10:43:35.489834+0000","last_became_peered":"2023-10-09T10:43:35.489834+0000","last_unstale":"2023-10-09T10:43:35.490168+0000","last_undegraded":"2023-10-09T10:43:35.490168+0000","last_fullsized":"2023-10-09T10:43:35.490168+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:59:26.252052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.489332+0000","last_change":"2023-10-09T10:43:35.489332+0000","last_active":"2023-10-09T10:43:35.489332+0000","last_peered":"2023-10-09T10:43:35.489332+0000","last_clean":"2023-10-09T10:43:35.489332+0000","last_became_active":"2023-10-09T10:43:35.489044+0000","last_became_peered":"2023-10-09T10:43:35.489044+0000","last_unstale":"2023-10-09T10:43:35.489332+0000","last_undegraded":"2023-10-09T10:43:35.489332+0000","last_fullsized":"2023-10-09T10:43:35.489332+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:42.168915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487678+0000","last_change":"2023-10-09T10:41:29.511298+0000","last_active":"2023-10-09T10:43:34.487678+0000","last_peered":"2023-10-09T10:43:34.487678+0000","last_clean":"2023-10-09T10:43:34.487678+0000","last_became_active":"2023-10-09T10:41:29.510978+0000","last_became_peered":"2023-10-09T10:41:29.510978+0000","last_unstale":"2023-10-09T10:43:34.487678+0000","last_undegraded":"2023-10-09T10:43:34.487678+0000","last_fullsized":"2023-10-09T10:43:34.487678+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:20:11.933459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":65,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479143+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:43:34.479143+0000","last_peered":"2023-10-09T10:43:34.479143+0000","last_clean":"2023-10-09T10:43:34.479143+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:43:34.479143+0000","last_undegraded":"2023-10-09T10:43:34.479143+0000","last_fullsized":"2023-10-09T10:43:34.479143+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486160+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:43:34.486160+0000","last_peered":"2023-10-09T10:43:34.486160+0000","last_clean":"2023-10-09T10:43:34.486160+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:43:34.486160+0000","last_undegraded":"2023-10-09T10:43:34.486160+0000","last_fullsized":"2023-10-09T10:43:34.486160+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":78,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486524+0000","last_change":"2023-10-09T10:43:35.486524+0000","last_active":"2023-10-09T10:43:35.486524+0000","last_peered":"2023-10-09T10:43:35.486524+0000","last_clean":"2023-10-09T10:43:35.486524+0000","last_became_active":"2023-10-09T10:43:35.486152+0000","last_became_peered":"2023-10-09T10:43:35.486152+0000","last_unstale":"2023-10-09T10:43:35.486524+0000","last_undegraded":"2023-10-09T10:43:35.486524+0000","last_fullsized":"2023-10-09T10:43:35.486524+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:47.200522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":72,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.489284+0000","last_change":"2023-10-09T10:42:38.416111+0000","last_active":"2023-10-09T10:43:34.489284+0000","last_peered":"2023-10-09T10:43:34.489284+0000","last_clean":"2023-10-09T10:43:34.489284+0000","last_became_active":"2023-10-09T10:42:38.415861+0000","last_became_peered":"2023-10-09T10:42:38.415861+0000","last_unstale":"2023-10-09T10:43:34.489284+0000","last_undegraded":"2023-10-09T10:43:34.489284+0000","last_fullsized":"2023-10-09T10:43:34.489284+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:10:40.572274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.489126+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:43:35.489126+0000","last_peered":"2023-10-09T10:43:35.489126+0000","last_clean":"2023-10-09T10:43:35.489126+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:43:35.489126+0000","last_undegraded":"2023-10-09T10:43:35.489126+0000","last_fullsized":"2023-10-09T10:43:35.489126+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":64,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486030+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:43:35.486030+0000","last_peered":"2023-10-09T10:43:35.486030+0000","last_clean":"2023-10-09T10:43:35.486030+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:43:35.486030+0000","last_undegraded":"2023-10-09T10:43:35.486030+0000","last_fullsized":"2023-10-09T10:43:35.486030+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486884+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:43:34.486884+0000","last_peered":"2023-10-09T10:43:34.486884+0000","last_clean":"2023-10-09T10:43:34.486884+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:43:34.486884+0000","last_undegraded":"2023-10-09T10:43:34.486884+0000","last_fullsized":"2023-10-09T10:43:34.486884+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485938+0000","last_change":"2023-10-09T10:43:06.676577+0000","last_active":"2023-10-09T10:43:34.485938+0000","last_peered":"2023-10-09T10:43:34.485938+0000","last_clean":"2023-10-09T10:43:34.485938+0000","last_became_active":"2023-10-09T10:43:06.676165+0000","last_became_peered":"2023-10-09T10:43:06.676165+0000","last_unstale":"2023-10-09T10:43:34.485938+0000","last_undegraded":"2023-10-09T10:43:34.485938+0000","last_fullsized":"2023-10-09T10:43:34.485938+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:52.614985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":77,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487389+0000","last_change":"2023-10-09T10:43:06.676367+0000","last_active":"2023-10-09T10:43:34.487389+0000","last_peered":"2023-10-09T10:43:34.487389+0000","last_clean":"2023-10-09T10:43:34.487389+0000","last_became_active":"2023-10-09T10:43:06.676017+0000","last_became_peered":"2023-10-09T10:43:06.676017+0000","last_unstale":"2023-10-09T10:43:34.487389+0000","last_undegraded":"2023-10-09T10:43:34.487389+0000","last_fullsized":"2023-10-09T10:43:34.487389+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:54.387617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482664+0000","last_change":"2023-10-09T10:42:38.412169+0000","last_active":"2023-10-09T10:43:34.482664+0000","last_peered":"2023-10-09T10:43:34.482664+0000","last_clean":"2023-10-09T10:43:34.482664+0000","last_became_active":"2023-10-09T10:42:38.411829+0000","last_became_peered":"2023-10-09T10:42:38.411829+0000","last_unstale":"2023-10-09T10:43:34.482664+0000","last_undegraded":"2023-10-09T10:43:34.482664+0000","last_fullsized":"2023-10-09T10:43:34.482664+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:36:35.699293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.493287+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:43:35.493287+0000","last_peered":"2023-10-09T10:43:35.493287+0000","last_clean":"2023-10-09T10:43:35.493287+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:43:35.493287+0000","last_undegraded":"2023-10-09T10:43:35.493287+0000","last_fullsized":"2023-10-09T10:43:35.493287+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":61,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491435+0000","last_change":"2023-10-09T10:43:35.491435+0000","last_active":"2023-10-09T10:43:35.491435+0000","last_peered":"2023-10-09T10:43:35.491435+0000","last_clean":"2023-10-09T10:43:35.491435+0000","last_became_active":"2023-10-09T10:43:35.491096+0000","last_became_peered":"2023-10-09T10:43:35.491096+0000","last_unstale":"2023-10-09T10:43:35.491435+0000","last_undegraded":"2023-10-09T10:43:35.491435+0000","last_fullsized":"2023-10-09T10:43:35.491435+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:09.581525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.483788+0000","last_change":"2023-10-09T10:42:38.407703+0000","last_active":"2023-10-09T10:43:35.483788+0000","last_peered":"2023-10-09T10:43:35.483788+0000","last_clean":"2023-10-09T10:43:35.483788+0000","last_became_active":"2023-10-09T10:42:38.407431+0000","last_became_peered":"2023-10-09T10:42:38.407431+0000","last_unstale":"2023-10-09T10:43:35.483788+0000","last_undegraded":"2023-10-09T10:43:35.483788+0000","last_fullsized":"2023-10-09T10:43:35.483788+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:23.450634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487326+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:43:35.487326+0000","last_peered":"2023-10-09T10:43:35.487326+0000","last_clean":"2023-10-09T10:43:35.487326+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:43:35.487326+0000","last_undegraded":"2023-10-09T10:43:35.487326+0000","last_fullsized":"2023-10-09T10:43:35.487326+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482266+0000","last_change":"2023-10-09T10:42:38.411928+0000","last_active":"2023-10-09T10:43:34.482266+0000","last_peered":"2023-10-09T10:43:34.482266+0000","last_clean":"2023-10-09T10:43:34.482266+0000","last_became_active":"2023-10-09T10:42:38.411560+0000","last_became_peered":"2023-10-09T10:42:38.411560+0000","last_unstale":"2023-10-09T10:43:34.482266+0000","last_undegraded":"2023-10-09T10:43:34.482266+0000","last_fullsized":"2023-10-09T10:43:34.482266+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:07.711753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":45,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487828+0000","last_change":"2023-10-09T10:35:56.822899+0000","last_active":"2023-10-09T10:43:34.487828+0000","last_peered":"2023-10-09T10:43:34.487828+0000","last_clean":"2023-10-09T10:43:34.487828+0000","last_became_active":"2023-10-09T10:35:56.822564+0000","last_became_peered":"2023-10-09T10:35:56.822564+0000","last_unstale":"2023-10-09T10:43:34.487828+0000","last_undegraded":"2023-10-09T10:43:34.487828+0000","last_fullsized":"2023-10-09T10:43:34.487828+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:25:03.398416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.493278+0000","last_change":"2023-10-09T10:43:35.493278+0000","last_active":"2023-10-09T10:43:35.493278+0000","last_peered":"2023-10-09T10:43:35.493278+0000","last_clean":"2023-10-09T10:43:35.493278+0000","last_became_active":"2023-10-09T10:43:35.492915+0000","last_became_peered":"2023-10-09T10:43:35.492915+0000","last_unstale":"2023-10-09T10:43:35.493278+0000","last_undegraded":"2023-10-09T10:43:35.493278+0000","last_fullsized":"2023-10-09T10:43:35.493278+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:42.227364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487004+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:43:34.487004+0000","last_peered":"2023-10-09T10:43:34.487004+0000","last_clean":"2023-10-09T10:43:34.487004+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:43:34.487004+0000","last_undegraded":"2023-10-09T10:43:34.487004+0000","last_fullsized":"2023-10-09T10:43:34.487004+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":43,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479053+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:43:34.479053+0000","last_peered":"2023-10-09T10:43:34.479053+0000","last_clean":"2023-10-09T10:43:34.479053+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:43:34.479053+0000","last_undegraded":"2023-10-09T10:43:34.479053+0000","last_fullsized":"2023-10-09T10:43:34.479053+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":62,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487772+0000","last_change":"2023-10-09T10:41:29.516129+0000","last_active":"2023-10-09T10:43:34.487772+0000","last_peered":"2023-10-09T10:43:34.487772+0000","last_clean":"2023-10-09T10:43:34.487772+0000","last_became_active":"2023-10-09T10:41:29.515741+0000","last_became_peered":"2023-10-09T10:41:29.515741+0000","last_unstale":"2023-10-09T10:43:34.487772+0000","last_undegraded":"2023-10-09T10:43:34.487772+0000","last_fullsized":"2023-10-09T10:43:34.487772+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:29:14.059627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":63,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487647+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:43:35.487647+0000","last_peered":"2023-10-09T10:43:35.487647+0000","last_clean":"2023-10-09T10:43:35.487647+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:43:35.487647+0000","last_undegraded":"2023-10-09T10:43:35.487647+0000","last_fullsized":"2023-10-09T10:43:35.487647+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":69,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.481806+0000","last_change":"2023-10-09T10:41:29.512929+0000","last_active":"2023-10-09T10:43:34.481806+0000","last_peered":"2023-10-09T10:43:34.481806+0000","last_clean":"2023-10-09T10:43:34.481806+0000","last_became_active":"2023-10-09T10:41:29.512591+0000","last_became_peered":"2023-10-09T10:41:29.512591+0000","last_unstale":"2023-10-09T10:43:34.481806+0000","last_undegraded":"2023-10-09T10:43:34.481806+0000","last_fullsized":"2023-10-09T10:43:34.481806+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:35.703322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.490293+0000","last_change":"2023-10-09T10:43:35.487796+0000","last_active":"2023-10-09T10:43:35.490293+0000","last_peered":"2023-10-09T10:43:35.490293+0000","last_clean":"2023-10-09T10:43:35.490293+0000","last_became_active":"2023-10-09T10:43:35.487518+0000","last_became_peered":"2023-10-09T10:43:35.487518+0000","last_unstale":"2023-10-09T10:43:35.490293+0000","last_undegraded":"2023-10-09T10:43:35.490293+0000","last_fullsized":"2023-10-09T10:43:35.490293+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:47.089685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":44,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.484279+0000","last_change":"2023-10-09T10:35:56.824257+0000","last_active":"2023-10-09T10:43:34.484279+0000","last_peered":"2023-10-09T10:43:34.484279+0000","last_clean":"2023-10-09T10:43:34.484279+0000","last_became_active":"2023-10-09T10:35:56.823152+0000","last_became_peered":"2023-10-09T10:35:56.823152+0000","last_unstale":"2023-10-09T10:43:34.484279+0000","last_undegraded":"2023-10-09T10:43:34.484279+0000","last_fullsized":"2023-10-09T10:43:34.484279+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:52.392494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.486785+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:43:35.486785+0000","last_peered":"2023-10-09T10:43:35.486785+0000","last_clean":"2023-10-09T10:43:35.486785+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:43:35.486785+0000","last_undegraded":"2023-10-09T10:43:35.486785+0000","last_fullsized":"2023-10-09T10:43:35.486785+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487384+0000","last_change":"2023-10-09T10:43:35.486914+0000","last_active":"2023-10-09T10:43:35.487384+0000","last_peered":"2023-10-09T10:43:35.487384+0000","last_clean":"2023-10-09T10:43:35.487384+0000","last_became_active":"2023-10-09T10:43:35.486623+0000","last_became_peered":"2023-10-09T10:43:35.486623+0000","last_unstale":"2023-10-09T10:43:35.487384+0000","last_undegraded":"2023-10-09T10:43:35.487384+0000","last_fullsized":"2023-10-09T10:43:35.487384+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:13.339801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":77,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487942+0000","last_change":"2023-10-09T10:43:35.487942+0000","last_active":"2023-10-09T10:43:35.487942+0000","last_peered":"2023-10-09T10:43:35.487942+0000","last_clean":"2023-10-09T10:43:35.487942+0000","last_became_active":"2023-10-09T10:43:35.487651+0000","last_became_peered":"2023-10-09T10:43:35.487651+0000","last_unstale":"2023-10-09T10:43:35.487942+0000","last_undegraded":"2023-10-09T10:43:35.487942+0000","last_fullsized":"2023-10-09T10:43:35.487942+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:51.882595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":63,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.485380+0000","last_change":"2023-10-09T10:43:06.677115+0000","last_active":"2023-10-09T10:43:34.485380+0000","last_peered":"2023-10-09T10:43:34.485380+0000","last_clean":"2023-10-09T10:43:34.485380+0000","last_became_active":"2023-10-09T10:43:06.676739+0000","last_became_peered":"2023-10-09T10:43:06.676739+0000","last_unstale":"2023-10-09T10:43:34.485380+0000","last_undegraded":"2023-10-09T10:43:34.485380+0000","last_fullsized":"2023-10-09T10:43:34.485380+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:02.520770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.487176+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:43:34.487176+0000","last_peered":"2023-10-09T10:43:34.487176+0000","last_clean":"2023-10-09T10:43:34.487176+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:43:34.487176+0000","last_undegraded":"2023-10-09T10:43:34.487176+0000","last_fullsized":"2023-10-09T10:43:34.487176+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":66,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486392+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:43:34.486392+0000","last_peered":"2023-10-09T10:43:34.486392+0000","last_clean":"2023-10-09T10:43:34.486392+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:43:34.486392+0000","last_undegraded":"2023-10-09T10:43:34.486392+0000","last_fullsized":"2023-10-09T10:43:34.486392+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486237+0000","last_change":"2023-10-09T10:43:06.672114+0000","last_active":"2023-10-09T10:43:34.486237+0000","last_peered":"2023-10-09T10:43:34.486237+0000","last_clean":"2023-10-09T10:43:34.486237+0000","last_became_active":"2023-10-09T10:43:06.671827+0000","last_became_peered":"2023-10-09T10:43:06.671827+0000","last_unstale":"2023-10-09T10:43:34.486237+0000","last_undegraded":"2023-10-09T10:43:34.486237+0000","last_fullsized":"2023-10-09T10:43:34.486237+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:41.701409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":64,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491382+0000","last_change":"2023-10-09T10:43:35.491382+0000","last_active":"2023-10-09T10:43:35.491382+0000","last_peered":"2023-10-09T10:43:35.491382+0000","last_clean":"2023-10-09T10:43:35.491382+0000","last_became_active":"2023-10-09T10:43:35.491062+0000","last_became_peered":"2023-10-09T10:43:35.491062+0000","last_unstale":"2023-10-09T10:43:35.491382+0000","last_undegraded":"2023-10-09T10:43:35.491382+0000","last_fullsized":"2023-10-09T10:43:35.491382+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:10.396267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487546+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:43:35.487546+0000","last_peered":"2023-10-09T10:43:35.487546+0000","last_clean":"2023-10-09T10:43:35.487546+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:43:35.487546+0000","last_undegraded":"2023-10-09T10:43:35.487546+0000","last_fullsized":"2023-10-09T10:43:35.487546+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.479334+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:43:34.479334+0000","last_peered":"2023-10-09T10:43:34.479334+0000","last_clean":"2023-10-09T10:43:34.479334+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:43:34.479334+0000","last_undegraded":"2023-10-09T10:43:34.479334+0000","last_fullsized":"2023-10-09T10:43:34.479334+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":63,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.484250+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:43:35.484250+0000","last_peered":"2023-10-09T10:43:35.484250+0000","last_clean":"2023-10-09T10:43:35.484250+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:43:35.484250+0000","last_undegraded":"2023-10-09T10:43:35.484250+0000","last_fullsized":"2023-10-09T10:43:35.484250+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.488427+0000","last_change":"2023-10-09T10:43:35.487114+0000","last_active":"2023-10-09T10:43:35.488427+0000","last_peered":"2023-10-09T10:43:35.488427+0000","last_clean":"2023-10-09T10:43:35.488427+0000","last_became_active":"2023-10-09T10:43:35.486830+0000","last_became_peered":"2023-10-09T10:43:35.486830+0000","last_unstale":"2023-10-09T10:43:35.488427+0000","last_undegraded":"2023-10-09T10:43:35.488427+0000","last_fullsized":"2023-10-09T10:43:35.488427+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:08.836969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.482547+0000","last_change":"2023-10-09T10:42:38.413301+0000","last_active":"2023-10-09T10:43:34.482547+0000","last_peered":"2023-10-09T10:43:34.482547+0000","last_clean":"2023-10-09T10:43:34.482547+0000","last_became_active":"2023-10-09T10:42:38.413006+0000","last_became_peered":"2023-10-09T10:42:38.413006+0000","last_unstale":"2023-10-09T10:43:34.482547+0000","last_undegraded":"2023-10-09T10:43:34.482547+0000","last_fullsized":"2023-10-09T10:43:34.482547+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:33.462419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":81,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.485235+0000","last_change":"2023-10-09T10:43:35.485235+0000","last_active":"2023-10-09T10:43:35.485235+0000","last_peered":"2023-10-09T10:43:35.485235+0000","last_clean":"2023-10-09T10:43:35.485235+0000","last_became_active":"2023-10-09T10:43:35.484917+0000","last_became_peered":"2023-10-09T10:43:35.484917+0000","last_unstale":"2023-10-09T10:43:35.485235+0000","last_undegraded":"2023-10-09T10:43:35.485235+0000","last_fullsized":"2023-10-09T10:43:35.485235+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:55.417326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.487224+0000","last_change":"2023-10-09T10:43:35.487224+0000","last_active":"2023-10-09T10:43:35.487224+0000","last_peered":"2023-10-09T10:43:35.487224+0000","last_clean":"2023-10-09T10:43:35.487224+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:43:35.487224+0000","last_undegraded":"2023-10-09T10:43:35.487224+0000","last_fullsized":"2023-10-09T10:43:35.487224+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:44:52.066778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.491388+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:43:35.491388+0000","last_peered":"2023-10-09T10:43:35.491388+0000","last_clean":"2023-10-09T10:43:35.491388+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:43:35.491388+0000","last_undegraded":"2023-10-09T10:43:35.491388+0000","last_fullsized":"2023-10-09T10:43:35.491388+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":63,"reported_epoch":96,"state":"active+clean","last_fresh":"2023-10-09T10:43:35.490890+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:43:35.490890+0000","last_peered":"2023-10-09T10:43:35.490890+0000","last_clean":"2023-10-09T10:43:35.490890+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:43:35.490890+0000","last_undegraded":"2023-10-09T10:43:35.490890+0000","last_fullsized":"2023-10-09T10:43:35.490890+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":79,"reported_epoch":95,"state":"active+clean","last_fresh":"2023-10-09T10:43:34.486285+0000","last_change":"2023-10-09T10:43:06.676414+0000","last_active":"2023-10-09T10:43:34.486285+0000","last_peered":"2023-10-09T10:43:34.486285+0000","last_clean":"2023-10-09T10:43:34.486285+0000","last_became_active":"2023-10-09T10:43:06.676050+0000","last_became_peered":"2023-10-09T10:43:06.676050+0000","last_unstale":"2023-10-09T10:43:34.486285+0000","last_undegraded":"2023-10-09T10:43:34.486285+0000","last_fullsized":"2023-10-09T10:43:34.486285+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:28.011477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332211,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311116,"kb_used_data":1540,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93434036,"statfs":{"total":95995035648,"available":95676452864,"internally_reserved":0,"allocated":1576960,"data_stored":1120859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:31 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.56399999999999995,"15min":0.57399999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.67200000000000004,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.61799999999999999,"15min":0.63400000000000001},"min":{"1min":0.36499999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.69799999999999995,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.65700000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:43:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.433},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.45500000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.54300000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.58399999999999996,"15min":0.56699999999999995},"min":{"1min":0.41399999999999998,"5min":0.35099999999999998,"15min":0.33800000000000002},"max":{"1min":0.74099999999999999,"5min":2.569,"15min":2.569},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59099999999999997,"15min":0.57399999999999995},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.76900000000000002,"5min":0.80100000000000005,"15min":1.232},"last":0.64100000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48599999999999999,"15min":0.47299999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.73999999999999999,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.502,"5min":0.497,"15min":0.47399999999999998},"min":{"1min":0.36899999999999999,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.67100000000000004,"5min":0.80600000000000005,"15min":0.83699999999999997},"last":0.68200000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.52100000000000002,"15min":0.504},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":1.1950000000000001},"last":0.628},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.55300000000000005,"15min":0.54000000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.25700000000000001},"max":{"1min":0.86099999999999999,"5min":0.876,"15min":1.0649999999999999},"last":0.71599999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58199999999999996,"15min":0.55700000000000005},"min":{"1min":0.40500000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.81699999999999995,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.57399999999999995,"15min":0.54300000000000004},"min":{"1min":0.39000000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.79700000000000004,"5min":0.89200000000000002,"15min":0.93500000000000005},"last":0.70499999999999996}]}]},{"osd":6,"up_from":44,"seq":188978561142,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298816,"kb_used_data":1528,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446336,"statfs":{"total":95995035648,"available":95689048064,"internally_reserved":0,"allocated":1564672,"data_stored":1120713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:34 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.59699999999999998,"15min":0.60599999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.73999999999999999,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.63,"5min":0.61199999999999999,"15min":0.60899999999999999},"min":{"1min":0.46600000000000003,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.66800000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.747},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.71199999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.68700000000000006}]},{"osd":3,"last update":"Mon Oct 9 10:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56100000000000005,"15min":0.52200000000000002},"min":{"1min":0.375,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72499999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54700000000000004,"15min":0.53200000000000003},"min":{"1min":0.39800000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80400000000000005},"last":0.63700000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.45900000000000002,"15min":0.44400000000000001},"min":{"1min":0.38900000000000001,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":0.59699999999999998,"5min":0.67400000000000004,"15min":0.70999999999999996},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.505,"15min":0.49199999999999999},"min":{"1min":0.35299999999999998,"5min":0.29899999999999999,"15min":0.29699999999999999},"max":{"1min":0.74399999999999999,"5min":0.77000000000000002,"15min":0.78500000000000003},"last":0.51200000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.51500000000000001,"15min":0.48399999999999999},"min":{"1min":0.40100000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.67600000000000005,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.55000000000000004,"15min":0.53000000000000003},"min":{"1min":0.372,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.84399999999999997},"last":0.56100000000000005}]},{"osd":7,"last update":"Mon Oct 9 10:43:18 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.53100000000000003,"15min":0.52600000000000002},"min":{"1min":0.33900000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.72599999999999998,"5min":0.78600000000000003,"15min":0.79100000000000004},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.438,"5min":0.34100000000000003,"15min":0.34000000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":1.018},"last":0.61699999999999999}]}]},{"osd":1,"up_from":85,"seq":365072220181,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28448,"kb_used_data":984,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716704,"statfs":{"total":95995035648,"available":95965904896,"internally_reserved":0,"allocated":1007616,"data_stored":536014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.42699999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:44:08 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.47999999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45000000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.82299999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.86399999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.71099999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.75900000000000001}]}]},{"osd":0,"up_from":80,"seq":343597383715,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29108,"kb_used_data":1580,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93716044,"statfs":{"total":95995035648,"available":95965229056,"internally_reserved":0,"allocated":1617920,"data_stored":1126957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:43:38 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.29299999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"last":0.249}]},{"osd":3,"last update":"Mon Oct 9 10:42:30 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.38100000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.61699999999999999,"15min":0.61899999999999999},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61699999999999999,"15min":0.61799999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.83499999999999996,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.496}]},{"osd":5,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.62,"15min":0.624},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.749,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.625,"5min":0.64400000000000002,"15min":0.64700000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.59399999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.628,"15min":0.63},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.61599999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.64700000000000002,"15min":0.65100000000000002},"min":{"1min":0.44600000000000001,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.66900000000000004,"15min":0.67500000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.48599999999999999}]}]},{"osd":2,"up_from":90,"seq":386547056656,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28572,"kb_used_data":980,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716580,"statfs":{"total":95995035648,"available":95965777920,"internally_reserved":0,"allocated":1003520,"data_stored":533482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"last":0.503}]},{"osd":1,"last update":"Mon Oct 9 10:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.435}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56200000000000006}]},{"osd":4,"last update":"Mon Oct 9 10:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.78200000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.72299999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.76200000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.68100000000000005}]}]},{"osd":3,"up_from":95,"seq":408021893130,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28424,"kb_used_data":960,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716728,"statfs":{"total":95995035648,"available":95965929472,"internally_reserved":0,"allocated":983040,"data_stored":530397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29699999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53000000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39800000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.379}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55900000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019005,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302344,"kb_used_data":960,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93442808,"statfs":{"total":95995035648,"available":95685435392,"internally_reserved":0,"allocated":983040,"data_stored":530744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.55500000000000005,"15min":0.55700000000000005},"min":{"1min":0.35199999999999998,"5min":0.311,"15min":0.311},"max":{"1min":0.74299999999999999,"5min":0.755,"15min":0.755},"last":0.621},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.57899999999999996,"15min":0.58299999999999996},"min":{"1min":0.41599999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.747,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.60099999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.65800000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:44:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.496}]},{"osd":3,"last update":"Mon Oct 9 10:43:21 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.58499999999999996,"15min":0.56200000000000006},"min":{"1min":0.45700000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.84799999999999998,"5min":0.85099999999999998,"15min":0.86399999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.57299999999999995,"15min":0.56799999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.83699999999999997,"5min":0.86599999999999999,"15min":2.0910000000000002},"last":0.73599999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.47399999999999998,"15min":0.438},"min":{"1min":0.34300000000000003,"5min":0.253,"15min":0.253},"max":{"1min":0.65000000000000002,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.48199999999999998,"15min":0.442},"min":{"1min":0.34599999999999997,"5min":0.28499999999999998,"15min":0.26400000000000001},"max":{"1min":0.73299999999999998,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.70199999999999996}]},{"osd":6,"last update":"Mon Oct 9 10:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.50800000000000001,"15min":0.46899999999999997},"min":{"1min":0.32000000000000001,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":0.752,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59499999999999997,"15min":0.56000000000000005},"min":{"1min":0.46500000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.82299999999999995,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.71599999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:43:11 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.57399999999999995,"15min":0.56899999999999995},"min":{"1min":0.376,"5min":0.34999999999999998,"15min":0.308},"max":{"1min":0.872,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.626,"5min":0.56899999999999995,"15min":0.53800000000000003},"min":{"1min":0.32500000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.80700000000000005,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.76400000000000001}]}]},{"osd":5,"up_from":37,"seq":158913790074,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306420,"kb_used_data":940,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93438732,"statfs":{"total":95995035648,"available":95681261568,"internally_reserved":0,"allocated":962560,"data_stored":526165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.66200000000000003,"15min":0.66700000000000004},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.81699999999999995}]},{"osd":1,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.77300000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:44:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.74199999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:42:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.54500000000000004,"15min":0.52600000000000002},"min":{"1min":0.39600000000000002,"5min":0.36899999999999999,"15min":0.35699999999999998},"max":{"1min":0.76100000000000001,"5min":0.79000000000000004,"15min":2.976},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.59199999999999997,"15min":0.59099999999999997},"min":{"1min":0.44700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.78400000000000003,"5min":0.92400000000000004,"15min":4.7949999999999999},"last":0.71199999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47599999999999998,"15min":0.43099999999999999},"min":{"1min":0.35399999999999998,"5min":0.30499999999999999,"15min":0.25},"max":{"1min":0.69099999999999995,"5min":0.75600000000000001,"15min":2.4649999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.5,"15min":0.45100000000000001},"min":{"1min":0.371,"5min":0.30099999999999999,"15min":0.28799999999999998},"max":{"1min":0.60199999999999998,"5min":0.85299999999999998,"15min":1.98},"last":0.51000000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.53200000000000003,"15min":0.45800000000000002},"min":{"1min":0.39300000000000002,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":0.67500000000000004,"5min":0.85599999999999998,"15min":2.048},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56599999999999995,"15min":0.49099999999999999},"min":{"1min":0.47099999999999997,"5min":0.33500000000000002,"15min":0.30299999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.56200000000000006}]},{"osd":7,"last update":"Mon Oct 9 10:43:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.57199999999999995,"15min":0.53500000000000003},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.32400000000000001},"max":{"1min":0.80700000000000005,"5min":0.879,"15min":2.1110000000000002},"last":0.627},{"interface":"front","average":{"1min":0.627,"5min":0.59699999999999998,"15min":0.53600000000000003},"min":{"1min":0.48799999999999999,"5min":0.35699999999999998,"15min":0.31900000000000001},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":1.845},"last":0.69399999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:44:15.809 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T10:44:15.809 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd dump 2023-10-09T10:44:15.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:44:15.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:15 smithi155 ceph-mon[184854]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:16.073 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:16.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:16 smithi125 ceph-mon[195507]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:44:16.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:16 smithi125 ceph-mon[200197]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:44:16.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:16 smithi155 ceph-mon[184854]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:44:17.316 INFO:teuthology.orchestra.run.smithi125.stdout:epoch 96 2023-10-09T10:44:17.316 INFO:teuthology.orchestra.run.smithi125.stdout:fsid daf690ae-668e-11ee-8db5-212e2dc638e7 2023-10-09T10:44:17.317 INFO:teuthology.orchestra.run.smithi125.stdout:created 2023-10-09T10:31:07.767709+0000 2023-10-09T10:44:17.317 INFO:teuthology.orchestra.run.smithi125.stdout:modified 2023-10-09T10:43:35.473754+0000 2023-10-09T10:44:17.317 INFO:teuthology.orchestra.run.smithi125.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-10-09T10:44:17.317 INFO:teuthology.orchestra.run.smithi125.stdout:crush_version 22 2023-10-09T10:44:17.317 INFO:teuthology.orchestra.run.smithi125.stdout:full_ratio 0.95 2023-10-09T10:44:17.317 INFO:teuthology.orchestra.run.smithi125.stdout:backfillfull_ratio 0.9 2023-10-09T10:44:17.317 INFO:teuthology.orchestra.run.smithi125.stdout:nearfull_ratio 0.85 2023-10-09T10:44:17.317 INFO:teuthology.orchestra.run.smithi125.stdout:require_min_compat_client luminous 2023-10-09T10:44:17.317 INFO:teuthology.orchestra.run.smithi125.stdout:min_compat_client luminous 2023-10-09T10:44:17.317 INFO:teuthology.orchestra.run.smithi125.stdout:require_osd_release quincy 2023-10-09T10:44:17.318 INFO:teuthology.orchestra.run.smithi125.stdout:stretch_mode_enabled false 2023-10-09T10:44:17.318 INFO:teuthology.orchestra.run.smithi125.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-10-09T10:44:17.318 INFO:teuthology.orchestra.run.smithi125.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 59 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-10-09T10:44:17.318 INFO:teuthology.orchestra.run.smithi125.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 63 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-10-09T10:44:17.318 INFO:teuthology.orchestra.run.smithi125.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 64 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-10-09T10:44:17.318 INFO:teuthology.orchestra.run.smithi125.stdout:max_osd 8 2023-10-09T10:44:17.318 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 up in weight 1 up_from 80 up_thru 95 down_at 77 last_clean_interval [8,76) [v2:172.21.15.125:6802/2302743228,v1:172.21.15.125:6803/2302743228] [v2:172.21.15.125:6804/2302743228,v1:172.21.15.125:6805/2302743228] exists,up 874cd96b-ddd8-4164-931e-06f2388dd41b 2023-10-09T10:44:17.318 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1 up in weight 1 up_from 85 up_thru 95 down_at 82 last_clean_interval [13,81) [v2:172.21.15.125:6810/1934730282,v1:172.21.15.125:6811/1934730282] [v2:172.21.15.125:6812/1934730282,v1:172.21.15.125:6813/1934730282] exists,up 0e517dd5-6d0b-4841-bb81-1eb04377ba5f 2023-10-09T10:44:17.318 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2 up in weight 1 up_from 90 up_thru 95 down_at 87 last_clean_interval [18,86) [v2:172.21.15.125:6818/3577506432,v1:172.21.15.125:6819/3577506432] [v2:172.21.15.125:6820/3577506432,v1:172.21.15.125:6821/3577506432] exists,up 643c1f3c-ecd9-464a-ad53-bf35a4deee5e 2023-10-09T10:44:17.318 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3 up in weight 1 up_from 95 up_thru 95 down_at 92 last_clean_interval [25,91) [v2:172.21.15.125:6826/1513104923,v1:172.21.15.125:6827/1513104923] [v2:172.21.15.125:6828/1513104923,v1:172.21.15.125:6829/1513104923] exists,up 3015cd6e-b270-44c3-8e6e-6bcd979bcd21 2023-10-09T10:44:17.318 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 up in weight 1 up_from 30 up_thru 95 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6800/1086065767,v1:172.21.15.155:6801/1086065767] [v2:172.21.15.155:6802/1086065767,v1:172.21.15.155:6803/1086065767] exists,up 272c7b95-f89c-4fe1-a299-a861e0ec9043 2023-10-09T10:44:17.319 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5 up in weight 1 up_from 37 up_thru 95 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6808/104536199,v1:172.21.15.155:6809/104536199] [v2:172.21.15.155:6810/104536199,v1:172.21.15.155:6811/104536199] exists,up 201d51f2-233c-40fc-a42d-649d225954fb 2023-10-09T10:44:17.319 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6 up in weight 1 up_from 44 up_thru 92 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6816/1730141492,v1:172.21.15.155:6817/1730141492] [v2:172.21.15.155:6818/1730141492,v1:172.21.15.155:6819/1730141492] exists,up 4ec66642-108e-401f-aff3-e7ab258bc083 2023-10-09T10:44:17.319 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7 up in weight 1 up_from 51 up_thru 95 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6824/3094552858,v1:172.21.15.155:6825/3094552858] [v2:172.21.15.155:6826/3094552858,v1:172.21.15.155:6827/3094552858] exists,up f8eccbf5-4ad0-43e7-be9a-00c05a362e57 2023-10-09T10:44:17.319 INFO:teuthology.orchestra.run.smithi125.stdout:pg_upmap_items 4.6 [1,5] 2023-10-09T10:44:17.319 INFO:teuthology.orchestra.run.smithi125.stdout:pg_upmap_items 4.9 [1,2] 2023-10-09T10:44:17.319 INFO:teuthology.orchestra.run.smithi125.stdout:pg_upmap_items 4.14 [1,2] 2023-10-09T10:44:17.319 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.155:0/48952571 expires 2023-10-10T10:38:13.935349+0000 2023-10-09T10:44:17.319 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6801/1533297380 expires 2023-10-10T10:38:44.908167+0000 2023-10-09T10:44:17.319 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/1316191185 expires 2023-10-10T10:35:20.607863+0000 2023-10-09T10:44:17.319 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/3693269321 expires 2023-10-10T10:38:44.908167+0000 2023-10-09T10:44:17.319 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6800/459376588 expires 2023-10-10T10:35:20.607863+0000 2023-10-09T10:44:17.320 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6800/1128053553 expires 2023-10-10T10:37:37.647752+0000 2023-10-09T10:44:17.320 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6800/1633421367 expires 2023-10-10T10:31:33.872122+0000 2023-10-09T10:44:17.320 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.155:6835/2184630814 expires 2023-10-10T10:38:13.935349+0000 2023-10-09T10:44:17.320 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/653697639 expires 2023-10-10T10:38:44.908167+0000 2023-10-09T10:44:17.320 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/1101683240 expires 2023-10-10T10:31:33.872122+0000 2023-10-09T10:44:17.320 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/1623791415 expires 2023-10-10T10:31:56.030149+0000 2023-10-09T10:44:17.320 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6801/1633421367 expires 2023-10-10T10:31:33.872122+0000 2023-10-09T10:44:17.320 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.155:0/1920197956 expires 2023-10-10T10:38:13.935349+0000 2023-10-09T10:44:17.320 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/2789680395 expires 2023-10-10T10:35:20.607863+0000 2023-10-09T10:44:17.320 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.155:0/1738428098 expires 2023-10-10T10:38:13.935349+0000 2023-10-09T10:44:17.320 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/1507781272 expires 2023-10-10T10:38:44.908167+0000 2023-10-09T10:44:17.320 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.155:0/1923144979 expires 2023-10-10T10:38:13.935349+0000 2023-10-09T10:44:17.321 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6801/1128053553 expires 2023-10-10T10:37:37.647752+0000 2023-10-09T10:44:17.321 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/1483568239 expires 2023-10-10T10:37:37.647752+0000 2023-10-09T10:44:17.321 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.155:6834/2184630814 expires 2023-10-10T10:38:13.935349+0000 2023-10-09T10:44:17.321 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/3028032775 expires 2023-10-10T10:38:44.908167+0000 2023-10-09T10:44:17.321 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/2429796720 expires 2023-10-10T10:35:20.607863+0000 2023-10-09T10:44:17.321 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6800/3950444489 expires 2023-10-10T10:31:56.030149+0000 2023-10-09T10:44:17.321 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/86510625 expires 2023-10-10T10:37:37.647752+0000 2023-10-09T10:44:17.321 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/725980307 expires 2023-10-10T10:31:56.030149+0000 2023-10-09T10:44:17.321 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/178379618 expires 2023-10-10T10:31:33.872122+0000 2023-10-09T10:44:17.321 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/2611643565 expires 2023-10-10T10:31:56.030149+0000 2023-10-09T10:44:17.321 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/461781620 expires 2023-10-10T10:35:20.607863+0000 2023-10-09T10:44:17.321 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6801/3950444489 expires 2023-10-10T10:31:56.030149+0000 2023-10-09T10:44:17.322 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.155:0/2043466629 expires 2023-10-10T10:38:13.935349+0000 2023-10-09T10:44:17.322 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/3709143923 expires 2023-10-10T10:37:37.647752+0000 2023-10-09T10:44:17.322 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/1709045642 expires 2023-10-10T10:31:33.872122+0000 2023-10-09T10:44:17.322 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/2757467822 expires 2023-10-10T10:37:37.647752+0000 2023-10-09T10:44:17.322 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/3810210930 expires 2023-10-10T10:38:44.908167+0000 2023-10-09T10:44:17.322 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6801/459376588 expires 2023-10-10T10:35:20.607863+0000 2023-10-09T10:44:17.322 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/1633295829 expires 2023-10-10T10:38:44.908167+0000 2023-10-09T10:44:17.322 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.155:0/565975522 expires 2023-10-10T10:38:13.935349+0000 2023-10-09T10:44:17.322 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6800/1533297380 expires 2023-10-10T10:38:44.908167+0000 2023-10-09T10:44:17.678 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:17 smithi125 ceph-mon[195507]: pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:17.678 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3753387505' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T10:44:17.678 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:17 smithi125 ceph-mon[200197]: pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:17.678 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3753387505' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T10:44:17.733 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 80 up_thru 95 down_at 77 last_clean_interval [8,76) [v2:172.21.15.125:6802/2302743228,v1:172.21.15.125:6803/2302743228] [v2:172.21.15.125:6804/2302743228,v1:172.21.15.125:6805/2302743228] exists,up 874cd96b-ddd8-4164-931e-06f2388dd41b', 'osd.1 up in weight 1 up_from 85 up_thru 95 down_at 82 last_clean_interval [13,81) [v2:172.21.15.125:6810/1934730282,v1:172.21.15.125:6811/1934730282] [v2:172.21.15.125:6812/1934730282,v1:172.21.15.125:6813/1934730282] exists,up 0e517dd5-6d0b-4841-bb81-1eb04377ba5f', 'osd.2 up in weight 1 up_from 90 up_thru 95 down_at 87 last_clean_interval [18,86) [v2:172.21.15.125:6818/3577506432,v1:172.21.15.125:6819/3577506432] [v2:172.21.15.125:6820/3577506432,v1:172.21.15.125:6821/3577506432] exists,up 643c1f3c-ecd9-464a-ad53-bf35a4deee5e', 'osd.3 up in weight 1 up_from 95 up_thru 95 down_at 92 last_clean_interval [25,91) [v2:172.21.15.125:6826/1513104923,v1:172.21.15.125:6827/1513104923] [v2:172.21.15.125:6828/1513104923,v1:172.21.15.125:6829/1513104923] exists,up 3015cd6e-b270-44c3-8e6e-6bcd979bcd21', 'osd.4 up in weight 1 up_from 30 up_thru 95 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6800/1086065767,v1:172.21.15.155:6801/1086065767] [v2:172.21.15.155:6802/1086065767,v1:172.21.15.155:6803/1086065767] exists,up 272c7b95-f89c-4fe1-a299-a861e0ec9043', 'osd.5 up in weight 1 up_from 37 up_thru 95 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6808/104536199,v1:172.21.15.155:6809/104536199] [v2:172.21.15.155:6810/104536199,v1:172.21.15.155:6811/104536199] exists,up 201d51f2-233c-40fc-a42d-649d225954fb', 'osd.6 up in weight 1 up_from 44 up_thru 92 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6816/1730141492,v1:172.21.15.155:6817/1730141492] [v2:172.21.15.155:6818/1730141492,v1:172.21.15.155:6819/1730141492] exists,up 4ec66642-108e-401f-aff3-e7ab258bc083', 'osd.7 up in weight 1 up_from 51 up_thru 95 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6824/3094552858,v1:172.21.15.155:6825/3094552858] [v2:172.21.15.155:6826/3094552858,v1:172.21.15.155:6827/3094552858] exists,up f8eccbf5-4ad0-43e7-be9a-00c05a362e57'] 2023-10-09T10:44:17.734 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon mon.a config show 2023-10-09T10:44:17.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:17 smithi155 ceph-mon[184854]: pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:17.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3753387505' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T10:44:17.996 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:19.107 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:44:19.107 INFO:teuthology.orchestra.run.smithi125.stdout: "name": "mon.a", 2023-10-09T10:44:19.107 INFO:teuthology.orchestra.run.smithi125.stdout: "cluster": "ceph", 2023-10-09T10:44:19.107 INFO:teuthology.orchestra.run.smithi125.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-10-09T10:44:19.107 INFO:teuthology.orchestra.run.smithi125.stdout: "admin_socket_mode": "", 2023-10-09T10:44:19.107 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-10-09T10:44:19.107 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_client_required": "cephx, none", 2023-10-09T10:44:19.108 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_cluster_required": "cephx", 2023-10-09T10:44:19.108 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_debug": "true", 2023-10-09T10:44:19.108 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-10-09T10:44:19.108 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-10-09T10:44:19.108 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_service_required": "cephx", 2023-10-09T10:44:19.108 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_service_ticket_ttl": "240.000000", 2023-10-09T10:44:19.108 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_supported": "", 2023-10-09T10:44:19.108 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_aio": "true", 2023-10-09T10:44:19.108 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_aio_max_queue_depth": "1024", 2023-10-09T10:44:19.109 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_aio_poll_ms": "250", 2023-10-09T10:44:19.109 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_aio_reap_max": "16", 2023-10-09T10:44:19.109 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_async_discard": "false", 2023-10-09T10:44:19.109 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_block_size": "4096", 2023-10-09T10:44:19.109 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_debug_aio": "false", 2023-10-09T10:44:19.109 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-10-09T10:44:19.109 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-10-09T10:44:19.109 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_debug_inflight_ios": "false", 2023-10-09T10:44:19.109 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_enable_discard": "false", 2023-10-09T10:44:19.109 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_flock_retry": "3", 2023-10-09T10:44:19.110 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_flock_retry_interval": "0.100000", 2023-10-09T10:44:19.110 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_inject_crash": "0", 2023-10-09T10:44:19.110 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_inject_crash_flush_delay": "2", 2023-10-09T10:44:19.110 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_ioring": "false", 2023-10-09T10:44:19.110 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_ioring_hipri": "false", 2023-10-09T10:44:19.110 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_ioring_sqthread_poll": "false", 2023-10-09T10:44:19.110 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-10-09T10:44:19.110 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_read_buffer_alignment": "4096", 2023-10-09T10:44:19.110 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-10-09T10:44:19.111 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_type": "", 2023-10-09T10:44:19.111 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_alloc_size": "1048576", 2023-10-09T10:44:19.111 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_allocator": "hybrid", 2023-10-09T10:44:19.111 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_buffered_io": "true", 2023-10-09T10:44:19.111 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_check_for_zeros": "false", 2023-10-09T10:44:19.111 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_check_volume_selector_often": "false", 2023-10-09T10:44:19.111 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-10-09T10:44:19.111 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_compact_log_sync": "false", 2023-10-09T10:44:19.111 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-10-09T10:44:19.112 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-10-09T10:44:19.112 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_log_compact_min_size": "16777216", 2023-10-09T10:44:19.112 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_log_replay_check_allocations": "true", 2023-10-09T10:44:19.112 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_max_log_runway": "4194304", 2023-10-09T10:44:19.112 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_max_prefetch": "1048576", 2023-10-09T10:44:19.112 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_min_flush_size": "524288", 2023-10-09T10:44:19.112 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_min_log_runway": "1048576", 2023-10-09T10:44:19.112 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_replay_recovery": "false", 2023-10-09T10:44:19.112 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-10-09T10:44:19.112 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_shared_alloc_size": "65536", 2023-10-09T10:44:19.113 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_sync_write": "false", 2023-10-09T10:44:19.113 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-10-09T10:44:19.113 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-10-09T10:44:19.113 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-10-09T10:44:19.113 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_allocation_from_file": "true", 2023-10-09T10:44:19.113 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_allocator": "hybrid", 2023-10-09T10:44:19.113 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-10-09T10:44:19.113 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-10-09T10:44:19.113 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-10-09T10:44:19.113 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-10-09T10:44:19.114 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-10-09T10:44:19.114 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-10-09T10:44:19.114 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_blobid_prealloc": "10240", 2023-10-09T10:44:19.114 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_create": "true", 2023-10-09T10:44:19.114 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_db_create": "false", 2023-10-09T10:44:19.114 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_db_path": "", 2023-10-09T10:44:19.114 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_db_size": "0", 2023-10-09T10:44:19.114 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_path": "", 2023-10-09T10:44:19.114 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_preallocate_file": "false", 2023-10-09T10:44:19.115 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_size": "107374182400", 2023-10-09T10:44:19.115 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_wal_create": "false", 2023-10-09T10:44:19.115 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_wal_path": "", 2023-10-09T10:44:19.115 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_wal_size": "100663296", 2023-10-09T10:44:19.115 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_bluefs": "true", 2023-10-09T10:44:19.115 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-10-09T10:44:19.115 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_bluefs_env_mirror": "false", 2023-10-09T10:44:19.115 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-10-09T10:44:19.115 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-10-09T10:44:19.115 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T10:44:19.115 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T10:44:19.116 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-10-09T10:44:19.116 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T10:44:19.116 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_autotune": "true", 2023-10-09T10:44:19.116 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-10-09T10:44:19.116 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-10-09T10:44:19.116 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-10-09T10:44:19.116 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-10-09T10:44:19.116 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_size": "0", 2023-10-09T10:44:19.116 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-10-09T10:44:19.117 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-10-09T10:44:19.117 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-10-09T10:44:19.117 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-10-09T10:44:19.117 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_type": "2q", 2023-10-09T10:44:19.117 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-10-09T10:44:19.117 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_clone_cow": "true", 2023-10-09T10:44:19.117 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_algorithm": "snappy", 2023-10-09T10:44:19.117 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_max_blob_size": "0", 2023-10-09T10:44:19.117 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-10-09T10:44:19.117 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-10-09T10:44:19.118 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_min_blob_size": "0", 2023-10-09T10:44:19.118 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-10-09T10:44:19.118 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-10-09T10:44:19.118 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_mode": "none", 2023-10-09T10:44:19.118 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-10-09T10:44:19.118 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_csum_type": "crc32c", 2023-10-09T10:44:19.118 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_enforce_settings": "default", 2023-10-09T10:44:19.118 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_freelist": "false", 2023-10-09T10:44:19.118 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_fsck_abort": "false", 2023-10-09T10:44:19.118 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-10-09T10:44:19.119 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_inject_bug21040": "false", 2023-10-09T10:44:19.119 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-10-09T10:44:19.119 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_inject_read_err": "false", 2023-10-09T10:44:19.119 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_legacy_omap": "false", 2023-10-09T10:44:19.119 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-10-09T10:44:19.119 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-10-09T10:44:19.119 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-10-09T10:44:19.119 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-10-09T10:44:19.119 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_prefill": "0.000000", 2023-10-09T10:44:19.119 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-10-09T10:44:19.119 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-10-09T10:44:19.120 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-10-09T10:44:19.120 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_small_allocations": "0", 2023-10-09T10:44:19.120 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-10-09T10:44:19.120 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_default_buffered_read": "true", 2023-10-09T10:44:19.120 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_default_buffered_write": "false", 2023-10-09T10:44:19.120 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_deferred_batch_ops": "0", 2023-10-09T10:44:19.120 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-10-09T10:44:19.120 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-10-09T10:44:19.120 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-10-09T10:44:19.120 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-10-09T10:44:19.121 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-10-09T10:44:19.121 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-10-09T10:44:19.121 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-10-09T10:44:19.121 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fail_eio": "false", 2023-10-09T10:44:19.121 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-10-09T10:44:19.121 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-10-09T10:44:19.121 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-10-09T10:44:19.121 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-10-09T10:44:19.121 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_on_mkfs": "true", 2023-10-09T10:44:19.121 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-10-09T10:44:19.122 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_on_mount": "false", 2023-10-09T10:44:19.122 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-10-09T10:44:19.122 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_on_umount": "false", 2023-10-09T10:44:19.122 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-10-09T10:44:19.122 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-10-09T10:44:19.122 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-10-09T10:44:19.122 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-10-09T10:44:19.122 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-10-09T10:44:19.122 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-10-09T10:44:19.122 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-10-09T10:44:19.123 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-10-09T10:44:19.123 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_ignore_data_csum": "false", 2023-10-09T10:44:19.123 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-10-09T10:44:19.123 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_kvbackend": "rocksdb", 2023-10-09T10:44:19.123 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-10-09T10:44:19.123 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-10-09T10:44:19.123 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_log_op_age": "5.000000", 2023-10-09T10:44:19.123 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_max_alloc_size": "0", 2023-10-09T10:44:19.123 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_max_blob_size": "0", 2023-10-09T10:44:19.123 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-10-09T10:44:19.124 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-10-09T10:44:19.124 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_max_defer_interval": "3.000000", 2023-10-09T10:44:19.124 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_max_deferred_txc": "32", 2023-10-09T10:44:19.124 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_min_alloc_size": "0", 2023-10-09T10:44:19.124 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-10-09T10:44:19.124 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-10-09T10:44:19.124 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_nid_prealloc": "1024", 2023-10-09T10:44:19.124 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_prefer_deferred_size": "0", 2023-10-09T10:44:19.124 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-10-09T10:44:19.124 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-10-09T10:44:19.125 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_qfsck_on_mount": "true", 2023-10-09T10:44:19.125 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_retry_disk_reads": "3", 2023-10-09T10:44:19.125 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_rocksdb_cf": "true", 2023-10-09T10:44:19.125 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-10-09T10:44:19.125 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-10-09T10:44:19.125 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_rocksdb_options_annex": "", 2023-10-09T10:44:19.125 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_spdk_coremask": "0x1", 2023-10-09T10:44:19.125 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_spdk_io_sleep": "5", 2023-10-09T10:44:19.125 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_spdk_max_io_completion": "0", 2023-10-09T10:44:19.125 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_spdk_mem": "512", 2023-10-09T10:44:19.126 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_sync_submit_transaction": "false", 2023-10-09T10:44:19.126 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_throttle_bytes": "67108864", 2023-10-09T10:44:19.126 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_throttle_cost_per_io": "0", 2023-10-09T10:44:19.126 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-10-09T10:44:19.126 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-10-09T10:44:19.126 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-10-09T10:44:19.126 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-10-09T10:44:19.126 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_tracing": "false", 2023-10-09T10:44:19.126 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-10-09T10:44:19.127 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-10-09T10:44:19.127 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_volume_selection_reserved": "0", 2023-10-09T10:44:19.127 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-10-09T10:44:19.127 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-10-09T10:44:19.127 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-10-09T10:44:19.127 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-10-09T10:44:19.127 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-10-09T10:44:19.127 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-10-09T10:44:19.127 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_zero_block_detection": "false", 2023-10-09T10:44:19.127 INFO:teuthology.orchestra.run.smithi125.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-10-09T10:44:19.128 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_action_update_interval": "2", 2023-10-09T10:44:19.128 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-10-09T10:44:19.128 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-10-09T10:44:19.128 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-10-09T10:44:19.128 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-10-09T10:44:19.128 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_mount_timeout": "10", 2023-10-09T10:44:19.128 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-10-09T10:44:19.128 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-10-09T10:44:19.128 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-10-09T10:44:19.128 INFO:teuthology.orchestra.run.smithi125.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-10-09T10:44:19.129 INFO:teuthology.orchestra.run.smithi125.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-10-09T10:44:19.129 INFO:teuthology.orchestra.run.smithi125.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-10-09T10:44:19.129 INFO:teuthology.orchestra.run.smithi125.stdout: "cephx_cluster_require_signatures": "false", 2023-10-09T10:44:19.129 INFO:teuthology.orchestra.run.smithi125.stdout: "cephx_cluster_require_version": "2", 2023-10-09T10:44:19.129 INFO:teuthology.orchestra.run.smithi125.stdout: "cephx_require_signatures": "false", 2023-10-09T10:44:19.129 INFO:teuthology.orchestra.run.smithi125.stdout: "cephx_require_version": "2", 2023-10-09T10:44:19.129 INFO:teuthology.orchestra.run.smithi125.stdout: "cephx_service_require_signatures": "false", 2023-10-09T10:44:19.129 INFO:teuthology.orchestra.run.smithi125.stdout: "cephx_service_require_version": "2", 2023-10-09T10:44:19.129 INFO:teuthology.orchestra.run.smithi125.stdout: "cephx_sign_messages": "true", 2023-10-09T10:44:19.129 INFO:teuthology.orchestra.run.smithi125.stdout: "chdir": "", 2023-10-09T10:44:19.130 INFO:teuthology.orchestra.run.smithi125.stdout: "client_acl_type": "", 2023-10-09T10:44:19.130 INFO:teuthology.orchestra.run.smithi125.stdout: "client_asio_thread_count": "2", 2023-10-09T10:44:19.130 INFO:teuthology.orchestra.run.smithi125.stdout: "client_cache_mid": "0.750000", 2023-10-09T10:44:19.130 INFO:teuthology.orchestra.run.smithi125.stdout: "client_cache_size": "16384", 2023-10-09T10:44:19.130 INFO:teuthology.orchestra.run.smithi125.stdout: "client_caps_release_delay": "5", 2023-10-09T10:44:19.130 INFO:teuthology.orchestra.run.smithi125.stdout: "client_check_pool_perm": "true", 2023-10-09T10:44:19.130 INFO:teuthology.orchestra.run.smithi125.stdout: "client_collect_and_send_global_metrics": "false", 2023-10-09T10:44:19.130 INFO:teuthology.orchestra.run.smithi125.stdout: "client_debug_force_sync_read": "false", 2023-10-09T10:44:19.130 INFO:teuthology.orchestra.run.smithi125.stdout: "client_debug_getattr_caps": "false", 2023-10-09T10:44:19.131 INFO:teuthology.orchestra.run.smithi125.stdout: "client_debug_inject_tick_delay": "0", 2023-10-09T10:44:19.131 INFO:teuthology.orchestra.run.smithi125.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-10-09T10:44:19.131 INFO:teuthology.orchestra.run.smithi125.stdout: "client_die_on_failed_remount": "false", 2023-10-09T10:44:19.131 INFO:teuthology.orchestra.run.smithi125.stdout: "client_dirsize_rbytes": "true", 2023-10-09T10:44:19.131 INFO:teuthology.orchestra.run.smithi125.stdout: "client_force_lazyio": "false", 2023-10-09T10:44:19.131 INFO:teuthology.orchestra.run.smithi125.stdout: "client_fs": "", 2023-10-09T10:44:19.131 INFO:teuthology.orchestra.run.smithi125.stdout: "client_inject_fixed_oldest_tid": "false", 2023-10-09T10:44:19.131 INFO:teuthology.orchestra.run.smithi125.stdout: "client_inject_release_failure": "false", 2023-10-09T10:44:19.131 INFO:teuthology.orchestra.run.smithi125.stdout: "client_max_inline_size": "4096", 2023-10-09T10:44:19.131 INFO:teuthology.orchestra.run.smithi125.stdout: "client_max_retries_on_remount_failure": "5", 2023-10-09T10:44:19.131 INFO:teuthology.orchestra.run.smithi125.stdout: "client_mds_namespace": "", 2023-10-09T10:44:19.132 INFO:teuthology.orchestra.run.smithi125.stdout: "client_metadata": "", 2023-10-09T10:44:19.132 INFO:teuthology.orchestra.run.smithi125.stdout: "client_mount_gid": "-1", 2023-10-09T10:44:19.132 INFO:teuthology.orchestra.run.smithi125.stdout: "client_mount_timeout": "300", 2023-10-09T10:44:19.132 INFO:teuthology.orchestra.run.smithi125.stdout: "client_mount_uid": "-1", 2023-10-09T10:44:19.132 INFO:teuthology.orchestra.run.smithi125.stdout: "client_mountpoint": "/", 2023-10-09T10:44:19.132 INFO:teuthology.orchestra.run.smithi125.stdout: "client_notify_timeout": "10", 2023-10-09T10:44:19.132 INFO:teuthology.orchestra.run.smithi125.stdout: "client_oc": "true", 2023-10-09T10:44:19.132 INFO:teuthology.orchestra.run.smithi125.stdout: "client_oc_max_dirty": "104857600", 2023-10-09T10:44:19.132 INFO:teuthology.orchestra.run.smithi125.stdout: "client_oc_max_dirty_age": "5.000000", 2023-10-09T10:44:19.132 INFO:teuthology.orchestra.run.smithi125.stdout: "client_oc_max_objects": "1000", 2023-10-09T10:44:19.133 INFO:teuthology.orchestra.run.smithi125.stdout: "client_oc_size": "209715200", 2023-10-09T10:44:19.133 INFO:teuthology.orchestra.run.smithi125.stdout: "client_oc_target_dirty": "8388608", 2023-10-09T10:44:19.133 INFO:teuthology.orchestra.run.smithi125.stdout: "client_permissions": "true", 2023-10-09T10:44:19.133 INFO:teuthology.orchestra.run.smithi125.stdout: "client_quota": "true", 2023-10-09T10:44:19.133 INFO:teuthology.orchestra.run.smithi125.stdout: "client_quota_df": "true", 2023-10-09T10:44:19.133 INFO:teuthology.orchestra.run.smithi125.stdout: "client_readahead_max_bytes": "0", 2023-10-09T10:44:19.133 INFO:teuthology.orchestra.run.smithi125.stdout: "client_readahead_max_periods": "4", 2023-10-09T10:44:19.133 INFO:teuthology.orchestra.run.smithi125.stdout: "client_readahead_min": "131072", 2023-10-09T10:44:19.133 INFO:teuthology.orchestra.run.smithi125.stdout: "client_reconnect_stale": "false", 2023-10-09T10:44:19.133 INFO:teuthology.orchestra.run.smithi125.stdout: "client_shutdown_timeout": "30", 2023-10-09T10:44:19.134 INFO:teuthology.orchestra.run.smithi125.stdout: "client_snapdir": ".snap", 2023-10-09T10:44:19.134 INFO:teuthology.orchestra.run.smithi125.stdout: "client_tick_interval": "1", 2023-10-09T10:44:19.134 INFO:teuthology.orchestra.run.smithi125.stdout: "client_trace": "", 2023-10-09T10:44:19.134 INFO:teuthology.orchestra.run.smithi125.stdout: "client_try_dentry_invalidate": "false", 2023-10-09T10:44:19.134 INFO:teuthology.orchestra.run.smithi125.stdout: "client_use_faked_inos": "false", 2023-10-09T10:44:19.134 INFO:teuthology.orchestra.run.smithi125.stdout: "client_use_random_mds": "false", 2023-10-09T10:44:19.134 INFO:teuthology.orchestra.run.smithi125.stdout: "clog_to_graylog": "false", 2023-10-09T10:44:19.134 INFO:teuthology.orchestra.run.smithi125.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-10-09T10:44:19.134 INFO:teuthology.orchestra.run.smithi125.stdout: "clog_to_graylog_port": "12201", 2023-10-09T10:44:19.134 INFO:teuthology.orchestra.run.smithi125.stdout: "clog_to_monitors": "default=true", 2023-10-09T10:44:19.135 INFO:teuthology.orchestra.run.smithi125.stdout: "clog_to_syslog": "false", 2023-10-09T10:44:19.135 INFO:teuthology.orchestra.run.smithi125.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-10-09T10:44:19.135 INFO:teuthology.orchestra.run.smithi125.stdout: "clog_to_syslog_level": "info", 2023-10-09T10:44:19.135 INFO:teuthology.orchestra.run.smithi125.stdout: "cluster_addr": "-", 2023-10-09T10:44:19.135 INFO:teuthology.orchestra.run.smithi125.stdout: "cluster_network": "", 2023-10-09T10:44:19.135 INFO:teuthology.orchestra.run.smithi125.stdout: "cluster_network_interface": "", 2023-10-09T10:44:19.135 INFO:teuthology.orchestra.run.smithi125.stdout: "compressor_zlib_isal": "false", 2023-10-09T10:44:19.135 INFO:teuthology.orchestra.run.smithi125.stdout: "compressor_zlib_level": "5", 2023-10-09T10:44:19.135 INFO:teuthology.orchestra.run.smithi125.stdout: "compressor_zlib_winsize": "-15", 2023-10-09T10:44:19.135 INFO:teuthology.orchestra.run.smithi125.stdout: "compressor_zstd_level": "1", 2023-10-09T10:44:19.136 INFO:teuthology.orchestra.run.smithi125.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c", 2023-10-09T10:44:19.136 INFO:teuthology.orchestra.run.smithi125.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-10-09T10:44:19.136 INFO:teuthology.orchestra.run.smithi125.stdout: "crimson_alien_op_num_threads": "6", 2023-10-09T10:44:19.136 INFO:teuthology.orchestra.run.smithi125.stdout: "crimson_alien_thread_cpu_cores": "", 2023-10-09T10:44:19.136 INFO:teuthology.orchestra.run.smithi125.stdout: "crimson_osd_obc_lru_size": "10", 2023-10-09T10:44:19.136 INFO:teuthology.orchestra.run.smithi125.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-10-09T10:44:19.136 INFO:teuthology.orchestra.run.smithi125.stdout: "crimson_seastar_smp": "1", 2023-10-09T10:44:19.137 INFO:teuthology.orchestra.run.smithi125.stdout: "crush_location": "", 2023-10-09T10:44:19.137 INFO:teuthology.orchestra.run.smithi125.stdout: "crush_location_hook": "", 2023-10-09T10:44:19.137 INFO:teuthology.orchestra.run.smithi125.stdout: "crush_location_hook_timeout": "10", 2023-10-09T10:44:19.137 INFO:teuthology.orchestra.run.smithi125.stdout: "daemonize": "false", 2023-10-09T10:44:19.137 INFO:teuthology.orchestra.run.smithi125.stdout: "daos_pool": "tank", 2023-10-09T10:44:19.137 INFO:teuthology.orchestra.run.smithi125.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-10-09T10:44:19.137 INFO:teuthology.orchestra.run.smithi125.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-10-09T10:44:19.138 INFO:teuthology.orchestra.run.smithi125.stdout: "dbstore_db_name_prefix": "dbstore", 2023-10-09T10:44:19.138 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_alienstore": "0/5", 2023-10-09T10:44:19.138 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_allow_any_pool_priority": "false", 2023-10-09T10:44:19.138 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_asok": "1/5", 2023-10-09T10:44:19.138 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_asok_assert_abort": "false", 2023-10-09T10:44:19.138 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_asserts_on_shutdown": "true", 2023-10-09T10:44:19.138 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_auth": "1/5", 2023-10-09T10:44:19.138 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_bdev": "1/3", 2023-10-09T10:44:19.139 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_bluefs": "1/5", 2023-10-09T10:44:19.139 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_bluestore": "1/5", 2023-10-09T10:44:19.139 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_buffer": "0/1", 2023-10-09T10:44:19.139 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_ceph_exporter": "1/5", 2023-10-09T10:44:19.139 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_cephfs_mirror": "0/5", 2023-10-09T10:44:19.139 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_cephsqlite": "0/5", 2023-10-09T10:44:19.139 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_client": "0/5", 2023-10-09T10:44:19.139 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_compressor": "1/5", 2023-10-09T10:44:19.139 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_context": "0/1", 2023-10-09T10:44:19.139 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_crush": "1/1", 2023-10-09T10:44:19.140 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_crypto": "1/5", 2023-10-09T10:44:19.140 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_cyanstore": "0/5", 2023-10-09T10:44:19.140 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_deliberately_leak_memory": "false", 2023-10-09T10:44:19.140 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_disable_randomized_ping": "false", 2023-10-09T10:44:19.140 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_dpdk": "1/5", 2023-10-09T10:44:19.140 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_eventtrace": "1/5", 2023-10-09T10:44:19.140 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_filer": "0/1", 2023-10-09T10:44:19.140 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_filestore": "1/3", 2023-10-09T10:44:19.140 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_finisher": "1/1", 2023-10-09T10:44:19.141 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_fuse": "1/5", 2023-10-09T10:44:19.141 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_heartbeat_testing_span": "0", 2023-10-09T10:44:19.141 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_heartbeatmap": "1/5", 2023-10-09T10:44:19.141 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_immutable_obj_cache": "0/5", 2023-10-09T10:44:19.141 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_javaclient": "1/5", 2023-10-09T10:44:19.141 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_journal": "1/3", 2023-10-09T10:44:19.141 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_journaler": "0/5", 2023-10-09T10:44:19.141 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_kstore": "1/5", 2023-10-09T10:44:19.141 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_leveldb": "4/5", 2023-10-09T10:44:19.141 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_lockdep": "0/1", 2023-10-09T10:44:19.141 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mclock": "1/5", 2023-10-09T10:44:19.142 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mds": "1/5", 2023-10-09T10:44:19.142 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mds_balancer": "1/5", 2023-10-09T10:44:19.142 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mds_locker": "1/5", 2023-10-09T10:44:19.142 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mds_log": "1/5", 2023-10-09T10:44:19.142 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mds_log_expire": "1/5", 2023-10-09T10:44:19.142 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mds_migrator": "1/5", 2023-10-09T10:44:19.142 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_memstore": "1/5", 2023-10-09T10:44:19.142 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mgr": "2/5", 2023-10-09T10:44:19.142 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mgrc": "1/5", 2023-10-09T10:44:19.142 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mon": "20/20", 2023-10-09T10:44:19.143 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_monc": "0/10", 2023-10-09T10:44:19.143 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_ms": "1/1", 2023-10-09T10:44:19.143 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_none": "0/5", 2023-10-09T10:44:19.143 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_objclass": "0/5", 2023-10-09T10:44:19.143 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_objectcacher": "0/5", 2023-10-09T10:44:19.143 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_objecter": "0/1", 2023-10-09T10:44:19.143 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_optracker": "0/5", 2023-10-09T10:44:19.143 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_osd": "1/5", 2023-10-09T10:44:19.143 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_paxos": "20/20", 2023-10-09T10:44:19.143 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_perfcounter": "1/5", 2023-10-09T10:44:19.144 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_prioritycache": "1/5", 2023-10-09T10:44:19.144 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rados": "0/5", 2023-10-09T10:44:19.144 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rbd": "0/5", 2023-10-09T10:44:19.144 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rbd_mirror": "0/5", 2023-10-09T10:44:19.144 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rbd_pwl": "0/5", 2023-10-09T10:44:19.144 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rbd_replay": "0/5", 2023-10-09T10:44:19.144 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_refs": "0/0", 2023-10-09T10:44:19.144 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_reserver": "1/1", 2023-10-09T10:44:19.144 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rgw": "1/5", 2023-10-09T10:44:19.144 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rgw_access": "1/5", 2023-10-09T10:44:19.145 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rgw_datacache": "1/5", 2023-10-09T10:44:19.145 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rgw_dbstore": "1/5", 2023-10-09T10:44:19.145 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rgw_flight": "1/5", 2023-10-09T10:44:19.145 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rgw_sync": "1/5", 2023-10-09T10:44:19.145 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rocksdb": "4/5", 2023-10-09T10:44:19.145 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore": "0/5", 2023-10-09T10:44:19.145 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_backref": "0/5", 2023-10-09T10:44:19.145 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_cache": "0/5", 2023-10-09T10:44:19.145 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_cleaner": "0/5", 2023-10-09T10:44:19.145 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_device": "0/5", 2023-10-09T10:44:19.145 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_epm": "0/5", 2023-10-09T10:44:19.146 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-10-09T10:44:19.146 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_journal": "0/5", 2023-10-09T10:44:19.146 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_lba": "0/5", 2023-10-09T10:44:19.146 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_odata": "0/5", 2023-10-09T10:44:19.146 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_omap": "0/5", 2023-10-09T10:44:19.146 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_onode": "0/5", 2023-10-09T10:44:19.146 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_t": "0/5", 2023-10-09T10:44:19.146 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_tm": "0/5", 2023-10-09T10:44:19.146 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_striper": "0/1", 2023-10-09T10:44:19.146 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_test": "0/5", 2023-10-09T10:44:19.146 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_throttle": "1/1", 2023-10-09T10:44:19.147 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_timer": "0/1", 2023-10-09T10:44:19.147 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_tp": "0/5", 2023-10-09T10:44:19.147 INFO:teuthology.orchestra.run.smithi125.stdout: "device_failure_prediction_mode": "none", 2023-10-09T10:44:19.147 INFO:teuthology.orchestra.run.smithi125.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-10-09T10:44:19.147 INFO:teuthology.orchestra.run.smithi125.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-10-09T10:44:19.147 INFO:teuthology.orchestra.run.smithi125.stdout: "err_to_graylog": "false", 2023-10-09T10:44:19.147 INFO:teuthology.orchestra.run.smithi125.stdout: "err_to_journald": "false", 2023-10-09T10:44:19.147 INFO:teuthology.orchestra.run.smithi125.stdout: "err_to_stderr": "true", 2023-10-09T10:44:19.147 INFO:teuthology.orchestra.run.smithi125.stdout: "err_to_syslog": "false", 2023-10-09T10:44:19.147 INFO:teuthology.orchestra.run.smithi125.stdout: "event_tracing": "false", 2023-10-09T10:44:19.148 INFO:teuthology.orchestra.run.smithi125.stdout: "exporter_addr": "0.0.0.0", 2023-10-09T10:44:19.148 INFO:teuthology.orchestra.run.smithi125.stdout: "exporter_http_port": "9926", 2023-10-09T10:44:19.148 INFO:teuthology.orchestra.run.smithi125.stdout: "exporter_prio_limit": "5", 2023-10-09T10:44:19.148 INFO:teuthology.orchestra.run.smithi125.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-10-09T10:44:19.148 INFO:teuthology.orchestra.run.smithi125.stdout: "exporter_sort_metrics": "true", 2023-10-09T10:44:19.148 INFO:teuthology.orchestra.run.smithi125.stdout: "exporter_stats_period": "5", 2023-10-09T10:44:19.148 INFO:teuthology.orchestra.run.smithi125.stdout: "fake_statfs_for_testing": "0", 2023-10-09T10:44:19.148 INFO:teuthology.orchestra.run.smithi125.stdout: "fatal_signal_handlers": "true", 2023-10-09T10:44:19.148 INFO:teuthology.orchestra.run.smithi125.stdout: "filer_max_purge_ops": "10", 2023-10-09T10:44:19.148 INFO:teuthology.orchestra.run.smithi125.stdout: "filer_max_truncate_ops": "128", 2023-10-09T10:44:19.148 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_apply_finisher_threads": "1", 2023-10-09T10:44:19.149 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_blackhole": "false", 2023-10-09T10:44:19.149 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_btrfs_clone_range": "true", 2023-10-09T10:44:19.149 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_btrfs_snap": "true", 2023-10-09T10:44:19.149 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_caller_concurrency": "10", 2023-10-09T10:44:19.149 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_collect_device_partition_information": "true", 2023-10-09T10:44:19.149 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_commit_timeout": "600.000000", 2023-10-09T10:44:19.149 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_debug_inject_read_err": "false", 2023-10-09T10:44:19.149 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_debug_omap_check": "false", 2023-10-09T10:44:19.149 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_debug_random_read_err": "0.000000", 2023-10-09T10:44:19.149 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_debug_verify_split": "false", 2023-10-09T10:44:19.150 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_dump_file": "", 2023-10-09T10:44:19.150 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-10-09T10:44:19.150 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-10-09T10:44:19.150 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_fadvise": "true", 2023-10-09T10:44:19.150 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_fail_eio": "true", 2023-10-09T10:44:19.150 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_fd_cache_shards": "16", 2023-10-09T10:44:19.150 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_fd_cache_size": "128", 2023-10-09T10:44:19.150 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_fiemap": "false", 2023-10-09T10:44:19.150 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_fiemap_threshold": "4096", 2023-10-09T10:44:19.150 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-10-09T10:44:19.151 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_index_retry_probability": "0.000000", 2023-10-09T10:44:19.151 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_inject_stall": "0", 2023-10-09T10:44:19.151 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_journal_parallel": "false", 2023-10-09T10:44:19.151 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_journal_trailing": "false", 2023-10-09T10:44:19.151 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_journal_writeahead": "false", 2023-10-09T10:44:19.151 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_kill_at": "0", 2023-10-09T10:44:19.151 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-10-09T10:44:19.151 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattr_size": "0", 2023-10-09T10:44:19.151 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-10-09T10:44:19.151 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-10-09T10:44:19.151 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-10-09T10:44:19.152 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattrs": "0", 2023-10-09T10:44:19.152 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-10-09T10:44:19.152 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattrs_other": "2", 2023-10-09T10:44:19.152 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-10-09T10:44:19.152 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_sync_interval": "5.000000", 2023-10-09T10:44:19.152 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_xattr_value_size": "0", 2023-10-09T10:44:19.152 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-10-09T10:44:19.152 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-10-09T10:44:19.152 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-10-09T10:44:19.152 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_merge_threshold": "-10", 2023-10-09T10:44:19.153 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_min_sync_interval": "0.010000", 2023-10-09T10:44:19.153 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_odsync_write": "false", 2023-10-09T10:44:19.153 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_omap_backend": "rocksdb", 2023-10-09T10:44:19.153 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_omap_backend_path": "", 2023-10-09T10:44:19.153 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_omap_header_cache_size": "1024", 2023-10-09T10:44:19.153 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_ondisk_finisher_threads": "1", 2023-10-09T10:44:19.153 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-10-09T10:44:19.153 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_op_thread_timeout": "60", 2023-10-09T10:44:19.153 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_op_threads": "2", 2023-10-09T10:44:19.153 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_punch_hole": "false", 2023-10-09T10:44:19.154 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-10-09T10:44:19.154 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-10-09T10:44:19.154 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-10-09T10:44:19.154 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-10-09T10:44:19.154 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-10-09T10:44:19.154 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_max_bytes": "104857600", 2023-10-09T10:44:19.154 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-10-09T10:44:19.154 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-10-09T10:44:19.154 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-10-09T10:44:19.155 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_max_ops": "50", 2023-10-09T10:44:19.155 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-10-09T10:44:19.155 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_seek_data_hole": "false", 2023-10-09T10:44:19.155 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_sloppy_crc": "false", 2023-10-09T10:44:19.155 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-10-09T10:44:19.155 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_splice": "false", 2023-10-09T10:44:19.155 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_split_multiple": "2", 2023-10-09T10:44:19.155 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_split_rand_factor": "20", 2023-10-09T10:44:19.155 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_update_to": "1000", 2023-10-09T10:44:19.155 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-10-09T10:44:19.156 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-10-09T10:44:19.156 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-10-09T10:44:19.156 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-10-09T10:44:19.156 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-10-09T10:44:19.156 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-10-09T10:44:19.156 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_enable": "true", 2023-10-09T10:44:19.156 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-10-09T10:44:19.156 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-10-09T10:44:19.156 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-10-09T10:44:19.156 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-10-09T10:44:19.157 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-10-09T10:44:19.157 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-10-09T10:44:19.157 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_xfs_extsize": "false", 2023-10-09T10:44:19.157 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_zfs_snap": "false", 2023-10-09T10:44:19.157 INFO:teuthology.orchestra.run.smithi125.stdout: "fio_dir": "/tmp/fio", 2023-10-09T10:44:19.157 INFO:teuthology.orchestra.run.smithi125.stdout: "fsid": "daf690ae-668e-11ee-8db5-212e2dc638e7", 2023-10-09T10:44:19.157 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_allow_other": "true", 2023-10-09T10:44:19.157 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_atomic_o_trunc": "true", 2023-10-09T10:44:19.157 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_big_writes": "true", 2023-10-09T10:44:19.157 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_debug": "false", 2023-10-09T10:44:19.158 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_default_permissions": "false", 2023-10-09T10:44:19.158 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_disable_pagecache": "false", 2023-10-09T10:44:19.158 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_max_write": "0", 2023-10-09T10:44:19.158 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_multithreaded": "true", 2023-10-09T10:44:19.158 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_require_active_mds": "true", 2023-10-09T10:44:19.158 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_set_user_groups": "true", 2023-10-09T10:44:19.158 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_splice_move": "true", 2023-10-09T10:44:19.158 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_splice_read": "true", 2023-10-09T10:44:19.159 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_splice_write": "true", 2023-10-09T10:44:19.159 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_syncfs_on_mksnap": "true", 2023-10-09T10:44:19.159 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_use_invalidate_cb": "true", 2023-10-09T10:44:19.159 INFO:teuthology.orchestra.run.smithi125.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-10-09T10:44:19.159 INFO:teuthology.orchestra.run.smithi125.stdout: "gss_target_name": "ceph", 2023-10-09T10:44:19.159 INFO:teuthology.orchestra.run.smithi125.stdout: "heartbeat_file": "", 2023-10-09T10:44:19.159 INFO:teuthology.orchestra.run.smithi125.stdout: "heartbeat_inject_failure": "0", 2023-10-09T10:44:19.160 INFO:teuthology.orchestra.run.smithi125.stdout: "heartbeat_interval": "5", 2023-10-09T10:44:19.160 INFO:teuthology.orchestra.run.smithi125.stdout: "host": "smithi125", 2023-10-09T10:44:19.160 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-10-09T10:44:19.160 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-10-09T10:44:19.160 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_max_size": "1073741824", 2023-10-09T10:44:19.160 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-10-09T10:44:19.160 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-10-09T10:44:19.160 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-10-09T10:44:19.161 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-10-09T10:44:19.161 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-10-09T10:44:19.161 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-10-09T10:44:19.161 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-10-09T10:44:19.161 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-10-09T10:44:19.161 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-10-09T10:44:19.161 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_watermark": "0.900000", 2023-10-09T10:44:19.161 INFO:teuthology.orchestra.run.smithi125.stdout: "inject_early_sigterm": "false", 2023-10-09T10:44:19.162 INFO:teuthology.orchestra.run.smithi125.stdout: "jaeger_agent_port": "6799", 2023-10-09T10:44:19.162 INFO:teuthology.orchestra.run.smithi125.stdout: "jaeger_tracing_enable": "false", 2023-10-09T10:44:19.162 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_aio": "true", 2023-10-09T10:44:19.162 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_align_min_size": "65536", 2023-10-09T10:44:19.162 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_block_align": "true", 2023-10-09T10:44:19.162 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_block_size": "4096", 2023-10-09T10:44:19.162 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_dio": "true", 2023-10-09T10:44:19.162 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_discard": "false", 2023-10-09T10:44:19.163 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_force_aio": "false", 2023-10-09T10:44:19.163 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_ignore_corruption": "false", 2023-10-09T10:44:19.163 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_max_write_bytes": "10485760", 2023-10-09T10:44:19.163 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_max_write_entries": "100", 2023-10-09T10:44:19.163 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_replay_from": "0", 2023-10-09T10:44:19.163 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_throttle_high_multiple": "0.000000", 2023-10-09T10:44:19.163 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-10-09T10:44:19.164 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-10-09T10:44:19.164 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_throttle_max_multiple": "0.000000", 2023-10-09T10:44:19.164 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_write_header_frequency": "0", 2023-10-09T10:44:19.164 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_zero_on_create": "false", 2023-10-09T10:44:19.164 INFO:teuthology.orchestra.run.smithi125.stdout: "journaler_prefetch_periods": "10", 2023-10-09T10:44:19.164 INFO:teuthology.orchestra.run.smithi125.stdout: "journaler_prezero_periods": "5", 2023-10-09T10:44:19.164 INFO:teuthology.orchestra.run.smithi125.stdout: "journaler_write_head_interval": "15", 2023-10-09T10:44:19.164 INFO:teuthology.orchestra.run.smithi125.stdout: "key": "", 2023-10-09T10:44:19.164 INFO:teuthology.orchestra.run.smithi125.stdout: "keyfile": "", 2023-10-09T10:44:19.165 INFO:teuthology.orchestra.run.smithi125.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-10-09T10:44:19.165 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_backend": "rocksdb", 2023-10-09T10:44:19.165 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_default_stripe_size": "65536", 2023-10-09T10:44:19.165 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_fsck_on_mount": "false", 2023-10-09T10:44:19.165 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_fsck_on_mount_deep": "true", 2023-10-09T10:44:19.165 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_max_bytes": "67108864", 2023-10-09T10:44:19.165 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_max_ops": "512", 2023-10-09T10:44:19.165 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_nid_prealloc": "1024", 2023-10-09T10:44:19.166 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_onode_map_size": "1024", 2023-10-09T10:44:19.166 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-10-09T10:44:19.166 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_sync_submit_transaction": "false", 2023-10-09T10:44:19.166 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_sync_transaction": "false", 2023-10-09T10:44:19.166 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_block_size": "65536", 2023-10-09T10:44:19.166 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_bloom_size": "0", 2023-10-09T10:44:19.166 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_cache_size": "536870912", 2023-10-09T10:44:19.166 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_compact_on_mount": "false", 2023-10-09T10:44:19.166 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_compression": "false", 2023-10-09T10:44:19.167 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_log": "", 2023-10-09T10:44:19.167 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_log_to_ceph_log": "true", 2023-10-09T10:44:19.167 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_max_open_files": "0", 2023-10-09T10:44:19.167 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_paranoid": "false", 2023-10-09T10:44:19.167 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_write_buffer_size": "33554432", 2023-10-09T10:44:19.167 INFO:teuthology.orchestra.run.smithi125.stdout: "librados_thread_count": "2", 2023-10-09T10:44:19.167 INFO:teuthology.orchestra.run.smithi125.stdout: "lockdep": "false", 2023-10-09T10:44:19.167 INFO:teuthology.orchestra.run.smithi125.stdout: "lockdep_force_backtrace": "false", 2023-10-09T10:44:19.168 INFO:teuthology.orchestra.run.smithi125.stdout: "log_coarse_timestamps": "true", 2023-10-09T10:44:19.168 INFO:teuthology.orchestra.run.smithi125.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-10-09T10:44:19.168 INFO:teuthology.orchestra.run.smithi125.stdout: "log_flush_on_exit": "false", 2023-10-09T10:44:19.168 INFO:teuthology.orchestra.run.smithi125.stdout: "log_graylog_host": "127.0.0.1", 2023-10-09T10:44:19.168 INFO:teuthology.orchestra.run.smithi125.stdout: "log_graylog_port": "12201", 2023-10-09T10:44:19.168 INFO:teuthology.orchestra.run.smithi125.stdout: "log_max_new": "1000", 2023-10-09T10:44:19.168 INFO:teuthology.orchestra.run.smithi125.stdout: "log_max_recent": "10000", 2023-10-09T10:44:19.168 INFO:teuthology.orchestra.run.smithi125.stdout: "log_stderr_prefix": "", 2023-10-09T10:44:19.169 INFO:teuthology.orchestra.run.smithi125.stdout: "log_stop_at_utilization": "0.970000", 2023-10-09T10:44:19.169 INFO:teuthology.orchestra.run.smithi125.stdout: "log_to_file": "true", 2023-10-09T10:44:19.169 INFO:teuthology.orchestra.run.smithi125.stdout: "log_to_graylog": "false", 2023-10-09T10:44:19.169 INFO:teuthology.orchestra.run.smithi125.stdout: "log_to_journald": "false", 2023-10-09T10:44:19.169 INFO:teuthology.orchestra.run.smithi125.stdout: "log_to_stderr": "false", 2023-10-09T10:44:19.169 INFO:teuthology.orchestra.run.smithi125.stdout: "log_to_syslog": "false", 2023-10-09T10:44:19.169 INFO:teuthology.orchestra.run.smithi125.stdout: "max_rotating_auth_attempts": "10", 2023-10-09T10:44:19.169 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-10-09T10:44:19.170 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_action_on_write_error": "1", 2023-10-09T10:44:19.170 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_alternate_name_max": "8192", 2023-10-09T10:44:19.170 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_asio_thread_count": "2", 2023-10-09T10:44:19.170 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_export_pin": "true", 2023-10-09T10:44:19.170 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_fragment_dirs": "true", 2023-10-09T10:44:19.170 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-10-09T10:44:19.170 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_fragment_interval": "5", 2023-10-09T10:44:19.171 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_fragment_size_max": "100000", 2023-10-09T10:44:19.171 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_idle_threshold": "0.000000", 2023-10-09T10:44:19.171 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_interval": "10", 2023-10-09T10:44:19.171 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_max": "-1", 2023-10-09T10:44:19.171 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_max_until": "-1", 2023-10-09T10:44:19.171 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_merge_size": "50", 2023-10-09T10:44:19.171 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_midchunk": "0.300000", 2023-10-09T10:44:19.171 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_min_rebalance": "0.100000", 2023-10-09T10:44:19.172 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_min_start": "0.200000", 2023-10-09T10:44:19.172 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_minchunk": "0.001000", 2023-10-09T10:44:19.172 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_mode": "0", 2023-10-09T10:44:19.172 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_need_max": "1.200000", 2023-10-09T10:44:19.172 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_need_min": "0.800000", 2023-10-09T10:44:19.172 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-10-09T10:44:19.172 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_sample_interval": "3.000000", 2023-10-09T10:44:19.172 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_split_bits": "3", 2023-10-09T10:44:19.172 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_split_rd": "25000.000000", 2023-10-09T10:44:19.172 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_split_size": "10000", 2023-10-09T10:44:19.173 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_split_wr": "10000.000000", 2023-10-09T10:44:19.173 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_target_decay": "10.000000", 2023-10-09T10:44:19.173 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-10-09T10:44:19.173 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_beacon_grace": "15.000000", 2023-10-09T10:44:19.173 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_beacon_interval": "4.000000", 2023-10-09T10:44:19.173 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_beacon_mon_down_grace": "60", 2023-10-09T10:44:19.173 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cache_memory_limit": "4294967296", 2023-10-09T10:44:19.173 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cache_mid": "0.700000", 2023-10-09T10:44:19.173 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cache_release_free_interval": "10", 2023-10-09T10:44:19.173 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cache_reservation": "0.050000", 2023-10-09T10:44:19.174 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-10-09T10:44:19.174 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cache_trim_interval": "1", 2023-10-09T10:44:19.174 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cache_trim_threshold": "262144", 2023-10-09T10:44:19.174 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-10-09T10:44:19.174 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-10-09T10:44:19.174 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_client_delegate_inos_pct": "50", 2023-10-09T10:44:19.174 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_client_prealloc_inos": "1000", 2023-10-09T10:44:19.174 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-10-09T10:44:19.174 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_connect_bootstrapping": "false", 2023-10-09T10:44:19.174 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_damage_table_max_entries": "10000", 2023-10-09T10:44:19.175 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-10-09T10:44:19.175 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_debug_auth_pins": "false", 2023-10-09T10:44:19.175 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_debug_frag": "false", 2023-10-09T10:44:19.175 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_debug_scatterstat": "false", 2023-10-09T10:44:19.175 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_debug_subtrees": "false", 2023-10-09T10:44:19.175 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_decay_halflife": "5.000000", 2023-10-09T10:44:19.175 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_default_dir_hash": "2", 2023-10-09T10:44:19.175 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_defer_session_stale": "true", 2023-10-09T10:44:19.175 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_deny_all_reconnect": "false", 2023-10-09T10:44:19.175 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dir_keys_per_op": "16384", 2023-10-09T10:44:19.175 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dir_max_commit_size": "10", 2023-10-09T10:44:19.176 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dir_max_entries": "0", 2023-10-09T10:44:19.176 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dir_prefetch": "true", 2023-10-09T10:44:19.176 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dirstat_min_interval": "1.000000", 2023-10-09T10:44:19.176 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dump_cache_after_rejoin": "false", 2023-10-09T10:44:19.176 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dump_cache_on_map": "false", 2023-10-09T10:44:19.176 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dump_cache_threshold_file": "0", 2023-10-09T10:44:19.176 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-10-09T10:44:19.176 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_early_reply": "true", 2023-10-09T10:44:19.176 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_enable_op_tracker": "true", 2023-10-09T10:44:19.176 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_enforce_unique_name": "true", 2023-10-09T10:44:19.176 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_export_ephemeral_distributed": "true", 2023-10-09T10:44:19.177 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-10-09T10:44:19.177 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_export_ephemeral_random": "true", 2023-10-09T10:44:19.177 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-10-09T10:44:19.177 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-10-09T10:44:19.177 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_forward_all_requests_to_auth": "false", 2023-10-09T10:44:19.177 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-10-09T10:44:19.177 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-10-09T10:44:19.177 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-10-09T10:44:19.177 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-10-09T10:44:19.177 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_health_cache_threshold": "1.500000", 2023-10-09T10:44:19.178 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_health_summarize_threshold": "10", 2023-10-09T10:44:19.178 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_heartbeat_grace": "15.000000", 2023-10-09T10:44:19.178 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_heartbeat_reset_grace": "1000", 2023-10-09T10:44:19.178 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_inject_health_dummy": "false", 2023-10-09T10:44:19.178 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-10-09T10:44:19.178 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_inject_migrator_session_race": "false", 2023-10-09T10:44:19.178 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-10-09T10:44:19.178 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-10-09T10:44:19.178 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-10-09T10:44:19.178 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_join_fs": "", 2023-10-09T10:44:19.178 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_journal_format": "1", 2023-10-09T10:44:19.179 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_create_at": "0", 2023-10-09T10:44:19.179 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_export_at": "0", 2023-10-09T10:44:19.179 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_import_at": "0", 2023-10-09T10:44:19.179 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_journal_at": "0", 2023-10-09T10:44:19.179 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_journal_expire_at": "0", 2023-10-09T10:44:19.179 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_journal_replay_at": "0", 2023-10-09T10:44:19.179 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_link_at": "0", 2023-10-09T10:44:19.179 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_mdstable_at": "0", 2023-10-09T10:44:19.179 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_openc_at": "0", 2023-10-09T10:44:19.179 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_rename_at": "0", 2023-10-09T10:44:19.180 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-10-09T10:44:19.180 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_log_events_per_segment": "1024", 2023-10-09T10:44:19.180 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_log_max_events": "-1", 2023-10-09T10:44:19.180 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_log_max_segments": "128", 2023-10-09T10:44:19.180 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_log_pause": "false", 2023-10-09T10:44:19.180 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_log_segment_size": "0", 2023-10-09T10:44:19.180 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_log_skip_corrupt_events": "false", 2023-10-09T10:44:19.180 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_log_warn_factor": "2.000000", 2023-10-09T10:44:19.180 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_caps_per_client": "1048576", 2023-10-09T10:44:19.180 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_completed_flushes": "100000", 2023-10-09T10:44:19.181 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_completed_requests": "100000", 2023-10-09T10:44:19.181 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_export_size": "20971520", 2023-10-09T10:44:19.181 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_file_recover": "32", 2023-10-09T10:44:19.181 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_purge_files": "64", 2023-10-09T10:44:19.181 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_purge_ops": "8192", 2023-10-09T10:44:19.181 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-10-09T10:44:19.181 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-10-09T10:44:19.181 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_snaps_per_dir": "100", 2023-10-09T10:44:19.181 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_xattr_pairs_size": "65536", 2023-10-09T10:44:19.181 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_metrics_update_interval": "2", 2023-10-09T10:44:19.182 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_min_caps_per_client": "100", 2023-10-09T10:44:19.182 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_min_caps_working_set": "10000", 2023-10-09T10:44:19.182 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-10-09T10:44:19.182 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_numa_node": "-1", 2023-10-09T10:44:19.182 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-10-09T10:44:19.182 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_op_complaint_time": "30.000000", 2023-10-09T10:44:19.182 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_op_history_duration": "600", 2023-10-09T10:44:19.182 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_op_history_size": "20", 2023-10-09T10:44:19.182 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_op_log_threshold": "5", 2023-10-09T10:44:19.182 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_ping_grace": "15", 2023-10-09T10:44:19.183 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_ping_interval": "5", 2023-10-09T10:44:19.183 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-10-09T10:44:19.183 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-10-09T10:44:19.183 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_recall_max_caps": "30000", 2023-10-09T10:44:19.183 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-10-09T10:44:19.183 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_recall_max_decay_threshold": "131072", 2023-10-09T10:44:19.183 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-10-09T10:44:19.183 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_recall_warning_threshold": "262144", 2023-10-09T10:44:19.183 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_reconnect_timeout": "45.000000", 2023-10-09T10:44:19.183 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_replay_interval": "1.000000", 2023-10-09T10:44:19.184 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-10-09T10:44:19.184 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-10-09T10:44:19.184 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_root_ino_gid": "0", 2023-10-09T10:44:19.184 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_root_ino_uid": "0", 2023-10-09T10:44:19.184 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-10-09T10:44:19.184 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_session_blocklist_on_evict": "true", 2023-10-09T10:44:19.184 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_session_blocklist_on_timeout": "true", 2023-10-09T10:44:19.184 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-10-09T10:44:19.184 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-10-09T10:44:19.185 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-10-09T10:44:19.185 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-10-09T10:44:19.185 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-10-09T10:44:19.185 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-10-09T10:44:19.185 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_shutdown_check": "0", 2023-10-09T10:44:19.185 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_skip_ino": "0", 2023-10-09T10:44:19.185 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_sleep_rank_change": "0.000000", 2023-10-09T10:44:19.185 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_snap_max_uid": "4294967294", 2023-10-09T10:44:19.185 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_snap_min_uid": "0", 2023-10-09T10:44:19.186 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_snap_rstat": "false", 2023-10-09T10:44:19.186 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_standby_replay_damaged": "false", 2023-10-09T10:44:19.186 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_symlink_recovery": "true", 2023-10-09T10:44:19.186 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_task_status_update_interval": "2.000000", 2023-10-09T10:44:19.186 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_thrash_exports": "0", 2023-10-09T10:44:19.186 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_thrash_fragments": "0", 2023-10-09T10:44:19.186 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_tick_interval": "5.000000", 2023-10-09T10:44:19.186 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_valgrind_exit": "false", 2023-10-09T10:44:19.186 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_verify_backtrace": "1", 2023-10-09T10:44:19.186 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_verify_scatter": "false", 2023-10-09T10:44:19.187 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_wipe_ino_prealloc": "false", 2023-10-09T10:44:19.187 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_wipe_sessions": "false", 2023-10-09T10:44:19.187 INFO:teuthology.orchestra.run.smithi125.stdout: "mempool_debug": "false", 2023-10-09T10:44:19.187 INFO:teuthology.orchestra.run.smithi125.stdout: "memstore_debug_omit_block_device_write": "false", 2023-10-09T10:44:19.187 INFO:teuthology.orchestra.run.smithi125.stdout: "memstore_device_bytes": "1073741824", 2023-10-09T10:44:19.187 INFO:teuthology.orchestra.run.smithi125.stdout: "memstore_page_set": "false", 2023-10-09T10:44:19.187 INFO:teuthology.orchestra.run.smithi125.stdout: "memstore_page_size": "65536", 2023-10-09T10:44:19.187 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_client_bytes": "134217728", 2023-10-09T10:44:19.187 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_client_messages": "512", 2023-10-09T10:44:19.188 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-10-09T10:44:19.188 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_connect_retry_interval": "1.000000", 2023-10-09T10:44:19.188 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-10-09T10:44:19.188 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-10-09T10:44:19.188 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_disabled_modules": "", 2023-10-09T10:44:19.188 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-10-09T10:44:19.188 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_max_pg_num_change": "128", 2023-10-09T10:44:19.188 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_mds_bytes": "134217728", 2023-10-09T10:44:19.188 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_mds_messages": "128", 2023-10-09T10:44:19.189 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-10-09T10:44:19.189 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_mon_bytes": "134217728", 2023-10-09T10:44:19.189 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_mon_messages": "128", 2023-10-09T10:44:19.189 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_osd_bytes": "536870912", 2023-10-09T10:44:19.189 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_osd_messages": "8192", 2023-10-09T10:44:19.189 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_pool": "true", 2023-10-09T10:44:19.189 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_service_beacon_grace": "60.000000", 2023-10-09T10:44:19.189 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_standby_modules": "true", 2023-10-09T10:44:19.189 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_stats_period": "5", 2023-10-09T10:44:19.189 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_stats_threshold": "5", 2023-10-09T10:44:19.190 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_tick_period": "2", 2023-10-09T10:44:19.190 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-10-09T10:44:19.190 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_accept_timeout_factor": "2.000000", 2023-10-09T10:44:19.190 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_allow_pool_delete": "true", 2023-10-09T10:44:19.190 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_allow_pool_size_one": "false", 2023-10-09T10:44:19.190 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_auth_validate_all_caps": "true", 2023-10-09T10:44:19.190 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-10-09T10:44:19.190 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-10-09T10:44:19.191 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_bytes": "104857600", 2023-10-09T10:44:19.191 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_directed_command_retry": "2", 2023-10-09T10:44:19.191 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_hunt_interval": "3.000000", 2023-10-09T10:44:19.191 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-10-09T10:44:19.191 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-10-09T10:44:19.191 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-10-09T10:44:19.191 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_hunt_parallel": "3", 2023-10-09T10:44:19.191 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_log_interval": "1.000000", 2023-10-09T10:44:19.191 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-10-09T10:44:19.192 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_ping_interval": "10.000000", 2023-10-09T10:44:19.192 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_ping_timeout": "30.000000", 2023-10-09T10:44:19.192 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_clock_drift_allowed": "1.000000", 2023-10-09T10:44:19.192 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-10-09T10:44:19.192 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-10-09T10:44:19.192 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_file_level": "debug", 2023-10-09T10:44:19.192 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_file": "false", 2023-10-09T10:44:19.192 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_graylog": "false", 2023-10-09T10:44:19.193 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-10-09T10:44:19.193 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-10-09T10:44:19.193 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_journald": "true", 2023-10-09T10:44:19.193 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_stderr": "false", 2023-10-09T10:44:19.193 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-10-09T10:44:19.193 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-10-09T10:44:19.193 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-10-09T10:44:19.193 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_compact_on_bootstrap": "false", 2023-10-09T10:44:19.193 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_compact_on_start": "false", 2023-10-09T10:44:19.193 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_compact_on_trim": "true", 2023-10-09T10:44:19.194 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_con_tracker_persist_interval": "10", 2023-10-09T10:44:19.194 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_con_tracker_score_halflife": "43200", 2023-10-09T10:44:19.194 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_config_key_max_entry_size": "65536", 2023-10-09T10:44:19.194 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cpu_threads": "4", 2023-10-09T10:44:19.194 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_crush_min_required_version": "hammer", 2023-10-09T10:44:19.194 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_daemon_bytes": "419430400", 2023-10-09T10:44:19.194 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-10-09T10:44:19.195 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_data_avail_crit": "5", 2023-10-09T10:44:19.195 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_data_avail_warn": "5", 2023-10-09T10:44:19.195 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_data_size_warn": "16106127360", 2023-10-09T10:44:19.195 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_block_osdmap_trim": "false", 2023-10-09T10:44:19.195 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-10-09T10:44:19.195 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_dump_json": "false", 2023-10-09T10:44:19.195 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-10-09T10:44:19.195 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_dump_transactions": "false", 2023-10-09T10:44:19.195 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_extra_checks": "false", 2023-10-09T10:44:19.196 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-10-09T10:44:19.196 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-10-09T10:44:19.196 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_no_require_quincy": "false", 2023-10-09T10:44:19.196 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_no_require_reef": "false", 2023-10-09T10:44:19.196 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-10-09T10:44:19.196 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_delta_reset_interval": "10.000000", 2023-10-09T10:44:19.196 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_dns_srv_name": "ceph-mon", 2023-10-09T10:44:19.196 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_down_mkfs_grace": "60", 2023-10-09T10:44:19.197 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_election_default_strategy": "1", 2023-10-09T10:44:19.197 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_election_timeout": "5.000000", 2023-10-09T10:44:19.197 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-10-09T10:44:19.197 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_elector_ping_divisor": "2", 2023-10-09T10:44:19.197 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_elector_ping_timeout": "2.000000", 2023-10-09T10:44:19.197 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_enable_op_tracker": "true", 2023-10-09T10:44:19.197 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_fake_pool_delete": "false", 2023-10-09T10:44:19.197 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_force_quorum_join": "false", 2023-10-09T10:44:19.197 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_globalid_prealloc": "10000", 2023-10-09T10:44:19.198 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_health_detail_to_clog": "true", 2023-10-09T10:44:19.198 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_health_log_update_period": "5", 2023-10-09T10:44:19.198 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_health_max_detail": "50", 2023-10-09T10:44:19.198 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_health_to_clog": "true", 2023-10-09T10:44:19.198 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_health_to_clog_interval": "600", 2023-10-09T10:44:19.198 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-10-09T10:44:19.198 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_host": "[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0] [v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0] [v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]", 2023-10-09T10:44:19.198 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_host_override": "", 2023-10-09T10:44:19.199 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_initial_members": "", 2023-10-09T10:44:19.199 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-10-09T10:44:19.199 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-10-09T10:44:19.199 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-10-09T10:44:19.199 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-10-09T10:44:19.199 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_keyvaluedb": "rocksdb", 2023-10-09T10:44:19.199 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_lease": "5.000000", 2023-10-09T10:44:19.199 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-10-09T10:44:19.199 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-10-09T10:44:19.200 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_log_full_interval": "50", 2023-10-09T10:44:19.200 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_log_max": "10000", 2023-10-09T10:44:19.200 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_log_max_summary": "50", 2023-10-09T10:44:19.200 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_log_entries_per_event": "4096", 2023-10-09T10:44:19.200 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_log_epochs": "500", 2023-10-09T10:44:19.200 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_mdsmap_epochs": "500", 2023-10-09T10:44:19.200 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_mgrmap_epochs": "500", 2023-10-09T10:44:19.200 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_osd": "10000", 2023-10-09T10:44:19.201 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_pg_per_osd": "10000", 2023-10-09T10:44:19.201 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_pool_pg_num": "65536", 2023-10-09T10:44:19.201 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-10-09T10:44:19.201 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-10-09T10:44:19.201 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mds_force_trim_to": "0", 2023-10-09T10:44:19.201 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mds_skip_sanity": "true", 2023-10-09T10:44:19.201 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_memory_autotune": "true", 2023-10-09T10:44:19.201 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_memory_target": "2147483648", 2023-10-09T10:44:19.202 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mgr_beacon_grace": "30", 2023-10-09T10:44:19.202 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-10-09T10:44:19.202 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mgr_digest_period": "5", 2023-10-09T10:44:19.202 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mgr_inactive_grace": "60", 2023-10-09T10:44:19.202 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mgr_mkfs_grace": "240", 2023-10-09T10:44:19.202 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-10-09T10:44:19.202 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_min_osdmap_epochs": "500", 2023-10-09T10:44:19.203 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_op_complaint_time": "30", 2023-10-09T10:44:19.203 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_op_history_duration": "600", 2023-10-09T10:44:19.203 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_op_history_size": "20", 2023-10-09T10:44:19.203 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_op_history_slow_op_size": "20", 2023-10-09T10:44:19.203 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_op_history_slow_op_threshold": "10", 2023-10-09T10:44:19.203 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_op_log_threshold": "5", 2023-10-09T10:44:19.203 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-10-09T10:44:19.203 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-10-09T10:44:19.203 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-10-09T10:44:19.203 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_auto_mark_in": "false", 2023-10-09T10:44:19.204 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_auto_mark_new_in": "true", 2023-10-09T10:44:19.204 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-10-09T10:44:19.204 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-10-09T10:44:19.204 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_cache_size": "500", 2023-10-09T10:44:19.204 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_cache_size_min": "134217728", 2023-10-09T10:44:19.204 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_crush_smoke_test": "true", 2023-10-09T10:44:19.204 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_destroyed_out_interval": "600", 2023-10-09T10:44:19.204 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_down_out_interval": "600", 2023-10-09T10:44:19.204 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-10-09T10:44:19.204 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-10-09T10:44:19.205 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_force_trim_to": "0", 2023-10-09T10:44:19.205 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_full_ratio": "0.950000", 2023-10-09T10:44:19.205 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-10-09T10:44:19.205 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_laggy_halflife": "3600", 2023-10-09T10:44:19.205 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_laggy_max_interval": "300", 2023-10-09T10:44:19.205 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_laggy_weight": "0.300000", 2023-10-09T10:44:19.205 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-10-09T10:44:19.205 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_max_creating_pgs": "1024", 2023-10-09T10:44:19.205 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_max_initial_pgs": "1024", 2023-10-09T10:44:19.205 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_min_down_reporters": "2", 2023-10-09T10:44:19.206 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-10-09T10:44:19.206 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-10-09T10:44:19.206 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-10-09T10:44:19.206 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_prime_pg_temp": "true", 2023-10-09T10:44:19.206 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-10-09T10:44:19.206 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-10-09T10:44:19.207 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_report_timeout": "900", 2023-10-09T10:44:19.207 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-10-09T10:44:19.207 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-10-09T10:44:19.207 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_warn_num_repaired": "10", 2023-10-09T10:44:19.207 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_warn_op_age": "32.000000", 2023-10-09T10:44:19.207 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-10-09T10:44:19.207 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osdmap_full_prune_interval": "10", 2023-10-09T10:44:19.208 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osdmap_full_prune_min": "10000", 2023-10-09T10:44:19.208 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-10-09T10:44:19.208 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-10-09T10:44:19.208 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pg_stuck_threshold": "60", 2023-10-09T10:44:19.208 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-10-09T10:44:19.208 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pg_warn_min_objects": "10000", 2023-10-09T10:44:19.208 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pg_warn_min_per_osd": "0", 2023-10-09T10:44:19.208 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-10-09T10:44:19.208 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pool_quota_crit_threshold": "0", 2023-10-09T10:44:19.208 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pool_quota_warn_threshold": "0", 2023-10-09T10:44:19.209 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_probe_timeout": "2.000000", 2023-10-09T10:44:19.209 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_reweight_max_change": "0.050000", 2023-10-09T10:44:19.209 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_reweight_max_osds": "4", 2023-10-09T10:44:19.209 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-10-09T10:44:19.209 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-10-09T10:44:19.209 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-10-09T10:44:19.209 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-10-09T10:44:19.209 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-10-09T10:44:19.209 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_scrub_interval": "86400", 2023-10-09T10:44:19.210 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_scrub_max_keys": "100", 2023-10-09T10:44:19.210 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_scrub_timeout": "300", 2023-10-09T10:44:19.210 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_session_timeout": "300", 2023-10-09T10:44:19.210 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_smart_report_timeout": "5", 2023-10-09T10:44:19.210 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_stat_smooth_intervals": "6", 2023-10-09T10:44:19.210 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-10-09T10:44:19.210 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_stretch_pool_min_size": "2", 2023-10-09T10:44:19.210 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_stretch_pool_size": "4", 2023-10-09T10:44:19.210 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-10-09T10:44:19.210 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_subscribe_interval": "86400.000000", 2023-10-09T10:44:19.211 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_sync_debug": "false", 2023-10-09T10:44:19.211 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_sync_max_payload_keys": "2000", 2023-10-09T10:44:19.211 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_sync_max_payload_size": "1048576", 2023-10-09T10:44:19.211 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_sync_provider_kill_at": "0", 2023-10-09T10:44:19.211 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_sync_requester_kill_at": "0", 2023-10-09T10:44:19.211 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_sync_timeout": "60.000000", 2023-10-09T10:44:19.211 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_target_pg_per_osd": "100", 2023-10-09T10:44:19.211 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_tick_interval": "5", 2023-10-09T10:44:19.211 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_timecheck_interval": "300.000000", 2023-10-09T10:44:19.211 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-10-09T10:44:19.212 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_use_min_delay_socket": "false", 2023-10-09T10:44:19.212 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_older_version_delay": "604800", 2023-10-09T10:44:19.212 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-10-09T10:44:19.212 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-10-09T10:44:19.212 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-10-09T10:44:19.212 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_filestore_osds": "true", 2023-10-09T10:44:19.212 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-10-09T10:44:19.212 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-10-09T10:44:19.212 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-10-09T10:44:19.213 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_misplaced": "false", 2023-10-09T10:44:19.213 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-10-09T10:44:19.213 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_older_version": "true", 2023-10-09T10:44:19.213 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-10-09T10:44:19.213 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_pool_no_app": "true", 2023-10-09T10:44:19.213 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-10-09T10:44:19.213 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-10-09T10:44:19.213 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-10-09T10:44:19.213 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-10-09T10:44:19.214 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_too_few_osds": "false", 2023-10-09T10:44:19.214 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-10-09T10:44:19.214 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-10-09T10:44:19.214 INFO:teuthology.orchestra.run.smithi125.stdout: "monmap": "", 2023-10-09T10:44:19.214 INFO:teuthology.orchestra.run.smithi125.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T10:44:19.214 INFO:teuthology.orchestra.run.smithi125.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-10-09T10:44:19.214 INFO:teuthology.orchestra.run.smithi125.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-10-09T10:44:19.214 INFO:teuthology.orchestra.run.smithi125.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T10:44:19.214 INFO:teuthology.orchestra.run.smithi125.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-10-09T10:44:19.215 INFO:teuthology.orchestra.run.smithi125.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-10-09T10:44:19.215 INFO:teuthology.orchestra.run.smithi125.stdout: "motr_tracing_enabled": "false", 2023-10-09T10:44:19.215 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_op_threads": "3", 2023-10-09T10:44:19.215 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_buffer_size": "131072", 2023-10-09T10:44:19.215 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_cm": "false", 2023-10-09T10:44:19.215 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_device_name": "", 2023-10-09T10:44:19.215 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_dscp": "96", 2023-10-09T10:44:19.215 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-10-09T10:44:19.215 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_gid_idx": "0", 2023-10-09T10:44:19.215 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_local_gid": "", 2023-10-09T10:44:19.216 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_polling_us": "1000", 2023-10-09T10:44:19.216 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_port_num": "1", 2023-10-09T10:44:19.216 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-10-09T10:44:19.216 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-10-09T10:44:19.216 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_roce_ver": "1", 2023-10-09T10:44:19.216 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_send_buffers": "1024", 2023-10-09T10:44:19.216 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_sl": "3", 2023-10-09T10:44:19.216 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_support_srq": "true", 2023-10-09T10:44:19.216 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_type": "ib", 2023-10-09T10:44:19.216 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_reap_threshold": "5", 2023-10-09T10:44:19.217 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_before_connect": "false", 2023-10-09T10:44:19.217 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_ipv4": "true", 2023-10-09T10:44:19.217 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_ipv6": "false", 2023-10-09T10:44:19.217 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_msgr1": "true", 2023-10-09T10:44:19.217 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_msgr2": "true", 2023-10-09T10:44:19.217 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_port_max": "7568", 2023-10-09T10:44:19.217 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_port_min": "6800", 2023-10-09T10:44:19.217 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_prefer_ipv4": "false", 2023-10-09T10:44:19.217 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_retry_count": "3", 2023-10-09T10:44:19.218 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_retry_delay": "5", 2023-10-09T10:44:19.218 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_blackhole_client": "false", 2023-10-09T10:44:19.218 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_blackhole_mds": "false", 2023-10-09T10:44:19.218 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_blackhole_mgr": "false", 2023-10-09T10:44:19.218 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_blackhole_mon": "false", 2023-10-09T10:44:19.218 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_blackhole_osd": "false", 2023-10-09T10:44:19.218 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_client_mode": "crc secure", 2023-10-09T10:44:19.218 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_cluster_mode": "crc secure", 2023-10-09T10:44:19.219 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_cluster_type": "", 2023-10-09T10:44:19.219 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_compress_secure": "false", 2023-10-09T10:44:19.219 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_connection_idle_timeout": "900", 2023-10-09T10:44:19.219 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_connection_ready_timeout": "10", 2023-10-09T10:44:19.219 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_crc_data": "true", 2023-10-09T10:44:19.219 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_crc_header": "true", 2023-10-09T10:44:19.219 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_die_on_bad_msg": "false", 2023-10-09T10:44:19.219 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_die_on_bug": "true", 2023-10-09T10:44:19.219 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_die_on_old_message": "true", 2023-10-09T10:44:19.220 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_die_on_skipped_message": "false", 2023-10-09T10:44:19.220 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_die_on_unhandled_msg": "false", 2023-10-09T10:44:19.220 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-10-09T10:44:19.220 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_coremask": "0xF", 2023-10-09T10:44:19.220 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-10-09T10:44:19.220 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_devs_allowlist": "", 2023-10-09T10:44:19.220 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_enable_tso": "true", 2023-10-09T10:44:19.220 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-10-09T10:44:19.220 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-10-09T10:44:19.221 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_hugepages": "", 2023-10-09T10:44:19.221 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_hw_flow_control": "true", 2023-10-09T10:44:19.221 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-10-09T10:44:19.221 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_lro": "true", 2023-10-09T10:44:19.221 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_memory_channel": "4", 2023-10-09T10:44:19.221 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-10-09T10:44:19.221 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_pmd": "", 2023-10-09T10:44:19.221 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_port_id": "0", 2023-10-09T10:44:19.221 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-10-09T10:44:19.221 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dump_corrupt_message_level": "1", 2023-10-09T10:44:19.222 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dump_on_send": "false", 2023-10-09T10:44:19.222 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_initial_backoff": "0.200000", 2023-10-09T10:44:19.222 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_inject_delay_max": "1.000000", 2023-10-09T10:44:19.222 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_inject_delay_probability": "0.000000", 2023-10-09T10:44:19.222 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_inject_delay_type": "", 2023-10-09T10:44:19.222 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_inject_internal_delays": "0.000000", 2023-10-09T10:44:19.222 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_inject_network_congestion": "0", 2023-10-09T10:44:19.222 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_inject_socket_failures": "0", 2023-10-09T10:44:19.222 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_learn_addr_from_peer": "true", 2023-10-09T10:44:19.222 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_max_accept_failures": "4", 2023-10-09T10:44:19.223 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_max_backoff": "15.000000", 2023-10-09T10:44:19.223 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_mon_client_mode": "secure crc", 2023-10-09T10:44:19.223 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_mon_cluster_mode": "secure crc", 2023-10-09T10:44:19.223 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_mon_service_mode": "secure crc", 2023-10-09T10:44:19.223 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_osd_compress_min_size": "1024", 2023-10-09T10:44:19.223 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_osd_compress_mode": "none", 2023-10-09T10:44:19.223 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_osd_compression_algorithm": "snappy", 2023-10-09T10:44:19.223 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-10-09T10:44:19.223 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_pq_min_cost": "65536", 2023-10-09T10:44:19.223 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_public_type": "", 2023-10-09T10:44:19.224 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_service_mode": "crc secure", 2023-10-09T10:44:19.224 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_tcp_listen_backlog": "512", 2023-10-09T10:44:19.224 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_tcp_nodelay": "true", 2023-10-09T10:44:19.224 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-10-09T10:44:19.224 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_tcp_rcvbuf": "0", 2023-10-09T10:44:19.224 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_type": "async+posix", 2023-10-09T10:44:19.224 INFO:teuthology.orchestra.run.smithi125.stdout: "no_config_file": "false", 2023-10-09T10:44:19.224 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_completion_locks_per_session": "32", 2023-10-09T10:44:19.224 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_debug_inject_relock_delay": "false", 2023-10-09T10:44:19.224 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_inflight_op_bytes": "104857600", 2023-10-09T10:44:19.224 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_inflight_ops": "1024", 2023-10-09T10:44:19.225 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_inject_no_watch_ping": "false", 2023-10-09T10:44:19.225 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-10-09T10:44:19.225 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_tick_interval": "5.000000", 2023-10-09T10:44:19.225 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_timeout": "10.000000", 2023-10-09T10:44:19.225 INFO:teuthology.orchestra.run.smithi125.stdout: "objectstore_blackhole": "false", 2023-10-09T10:44:19.225 INFO:teuthology.orchestra.run.smithi125.stdout: "openssl_engine_opts": "", 2023-10-09T10:44:19.225 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_agent_delay_time": "5.000000", 2023-10-09T10:44:19.225 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_agent_hist_halflife": "1000", 2023-10-09T10:44:19.225 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_agent_max_low_ops": "2", 2023-10-09T10:44:19.225 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_agent_max_ops": "4", 2023-10-09T10:44:19.226 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-10-09T10:44:19.226 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_agent_quantize_effort": "0.100000", 2023-10-09T10:44:19.226 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_agent_slop": "0.020000", 2023-10-09T10:44:19.226 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-10-09T10:44:19.226 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_allow_recovery_below_min_size": "true", 2023-10-09T10:44:19.226 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_asio_thread_count": "2", 2023-10-09T10:44:19.226 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_async_recovery_min_cost": "100", 2023-10-09T10:44:19.226 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_auto_mark_unfound_lost": "false", 2023-10-09T10:44:19.226 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_backfill_retry_interval": "30.000000", 2023-10-09T10:44:19.226 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_backfill_scan_max": "512", 2023-10-09T10:44:19.226 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_backfill_scan_min": "64", 2023-10-09T10:44:19.227 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_backoff_on_degraded": "false", 2023-10-09T10:44:19.227 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_backoff_on_peering": "false", 2023-10-09T10:44:19.227 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_backoff_on_unfound": "true", 2023-10-09T10:44:19.227 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_beacon_report_interval": "300", 2023-10-09T10:44:19.227 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_bench_duration": "30", 2023-10-09T10:44:19.227 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-10-09T10:44:19.227 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_bench_max_block_size": "67108864", 2023-10-09T10:44:19.228 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_bench_small_size_max_iops": "100", 2023-10-09T10:44:19.228 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_blkin_trace_all": "false", 2023-10-09T10:44:19.228 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_blocked_scrub_grace_period": "120", 2023-10-09T10:44:19.228 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-10-09T10:44:19.228 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-10-09T10:44:19.228 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-10-09T10:44:19.228 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_check_for_log_corruption": "false", 2023-10-09T10:44:19.228 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-10-09T10:44:19.228 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T10:44:19.228 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-10-09T10:44:19.229 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T10:44:19.229 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_class_update_on_start": "true", 2023-10-09T10:44:19.229 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_client_message_cap": "256", 2023-10-09T10:44:19.229 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_client_message_size_cap": "524288000", 2023-10-09T10:44:19.229 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_client_op_priority": "63", 2023-10-09T10:44:19.229 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_client_watch_timeout": "30", 2023-10-09T10:44:19.229 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_command_max_records": "256", 2023-10-09T10:44:19.229 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_compact_on_start": "false", 2023-10-09T10:44:19.230 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-10-09T10:44:19.230 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_crush_chooseleaf_type": "1", 2023-10-09T10:44:19.230 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_crush_initial_weight": "-1.000000", 2023-10-09T10:44:19.230 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_crush_update_on_start": "true", 2023-10-09T10:44:19.230 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_crush_update_weight_set": "true", 2023-10-09T10:44:19.230 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-10-09T10:44:19.230 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-10-09T10:44:19.230 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-10-09T10:44:19.230 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_drop_ping_duration": "0", 2023-10-09T10:44:19.231 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-10-09T10:44:19.231 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_feed_pullee": "-1", 2023-10-09T10:44:19.231 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-10-09T10:44:19.231 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-10-09T10:44:19.231 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-10-09T10:44:19.231 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_misdirected_ops": "false", 2023-10-09T10:44:19.231 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_no_acting_change": "false", 2023-10-09T10:44:19.231 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_no_purge_strays": "false", 2023-10-09T10:44:19.232 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_op_order": "false", 2023-10-09T10:44:19.232 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_pg_log_writeout": "false", 2023-10-09T10:44:19.232 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_pretend_recovery_active": "false", 2023-10-09T10:44:19.232 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-10-09T10:44:19.232 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-10-09T10:44:19.232 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_shutdown": "false", 2023-10-09T10:44:19.232 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-10-09T10:44:19.232 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-10-09T10:44:19.233 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_verify_cached_snaps": "false", 2023-10-09T10:44:19.233 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_verify_missing_on_start": "false", 2023-10-09T10:44:19.233 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_verify_snaps": "false", 2023-10-09T10:44:19.233 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-10-09T10:44:19.233 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-10-09T10:44:19.233 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_deep_scrub_keys": "1024", 2023-10-09T10:44:19.233 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-10-09T10:44:19.233 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-10-09T10:44:19.234 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-10-09T10:44:19.234 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_deep_scrub_stride": "524288", 2023-10-09T10:44:19.234 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-10-09T10:44:19.234 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_default_data_pool_replay_window": "45", 2023-10-09T10:44:19.234 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_default_notify_timeout": "30", 2023-10-09T10:44:19.234 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_delete_sleep": "0.000000", 2023-10-09T10:44:19.234 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-10-09T10:44:19.234 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-10-09T10:44:19.235 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-10-09T10:44:19.235 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_discard_disconnected_ops": "true", 2023-10-09T10:44:19.235 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_enable_op_tracker": "true", 2023-10-09T10:44:19.235 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-10-09T10:44:19.235 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_extblkdev_plugins": "vdo", 2023-10-09T10:44:19.235 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-10-09T10:44:19.235 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-10-09T10:44:19.235 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_fast_info": "true", 2023-10-09T10:44:19.235 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_fast_shutdown": "true", 2023-10-09T10:44:19.236 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-10-09T10:44:19.236 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_fast_shutdown_timeout": "15", 2023-10-09T10:44:19.236 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-10-09T10:44:19.236 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-10-09T10:44:19.236 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-10-09T10:44:19.236 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_function_tracing": "false", 2023-10-09T10:44:19.236 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_heartbeat_grace": "20", 2023-10-09T10:44:19.236 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_heartbeat_interval": "6", 2023-10-09T10:44:19.237 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-10-09T10:44:19.237 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_heartbeat_min_peers": "10", 2023-10-09T10:44:19.237 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_heartbeat_min_size": "2000", 2023-10-09T10:44:19.237 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_heartbeat_stale": "600", 2023-10-09T10:44:19.237 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-10-09T10:44:19.237 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_hit_set_max_size": "100000", 2023-10-09T10:44:19.237 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_hit_set_min_size": "1000", 2023-10-09T10:44:19.237 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-10-09T10:44:19.238 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-10-09T10:44:19.238 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-10-09T10:44:19.238 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-10-09T10:44:19.238 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-10-09T10:44:19.238 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_journal_flush_on_shutdown": "true", 2023-10-09T10:44:19.238 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_journal_size": "5120", 2023-10-09T10:44:19.238 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_kill_backfill_at": "0", 2023-10-09T10:44:19.239 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_loop_before_reset_tphandle": "64", 2023-10-09T10:44:19.239 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_map_cache_size": "50", 2023-10-09T10:44:19.239 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_map_dedup": "true", 2023-10-09T10:44:19.239 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_map_message_max": "40", 2023-10-09T10:44:19.239 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_map_message_max_bytes": "10485760", 2023-10-09T10:44:19.239 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_map_share_max_epochs": "40", 2023-10-09T10:44:19.239 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_attr_name_len": "100", 2023-10-09T10:44:19.239 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_attr_size": "0", 2023-10-09T10:44:19.240 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_backfills": "1", 2023-10-09T10:44:19.240 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_markdown_count": "5", 2023-10-09T10:44:19.240 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_markdown_period": "600", 2023-10-09T10:44:19.240 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_object_name_len": "2048", 2023-10-09T10:44:19.240 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_object_namespace_len": "256", 2023-10-09T10:44:19.240 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_object_size": "134217728", 2023-10-09T10:44:19.240 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-10-09T10:44:19.240 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_omap_entries_per_request": "1024", 2023-10-09T10:44:19.240 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_pg_blocked_by": "16", 2023-10-09T10:44:19.241 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_pg_log_entries": "10000", 2023-10-09T10:44:19.241 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-10-09T10:44:19.241 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_pgls": "1024", 2023-10-09T10:44:19.241 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_push_cost": "8388608", 2023-10-09T10:44:19.241 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_push_objects": "10", 2023-10-09T10:44:19.241 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_scrubs": "1", 2023-10-09T10:44:19.241 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-10-09T10:44:19.241 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_trimming_pgs": "2", 2023-10-09T10:44:19.241 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_write_op_reply_len": "64", 2023-10-09T10:44:19.241 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_write_size": "90", 2023-10-09T10:44:19.242 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-10-09T10:44:19.242 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-10-09T10:44:19.242 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-10-09T10:44:19.242 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-10-09T10:44:19.242 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-10-09T10:44:19.242 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-10-09T10:44:19.242 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-10-09T10:44:19.242 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_override_recovery_settings": "false", 2023-10-09T10:44:19.242 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_profile": "balanced", 2023-10-09T10:44:19.243 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-10-09T10:44:19.243 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-10-09T10:44:19.243 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-10-09T10:44:19.243 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-10-09T10:44:19.243 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-10-09T10:44:19.243 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-10-09T10:44:19.243 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-10-09T10:44:19.243 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-10-09T10:44:19.243 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-10-09T10:44:19.244 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-10-09T10:44:19.244 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_skip_benchmark": "false", 2023-10-09T10:44:19.244 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_memory_base": "805306368", 2023-10-09T10:44:19.244 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_memory_cache_min": "134217728", 2023-10-09T10:44:19.244 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-10-09T10:44:19.244 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-10-09T10:44:19.244 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_memory_target": "4294967296", 2023-10-09T10:44:19.244 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_memory_target_autotune": "false", 2023-10-09T10:44:19.244 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-10-09T10:44:19.244 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_min_pg_log_entries": "250", 2023-10-09T10:44:19.245 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_min_recovery_priority": "0", 2023-10-09T10:44:19.245 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mon_heartbeat_interval": "30", 2023-10-09T10:44:19.245 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-10-09T10:44:19.245 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mon_report_interval": "5", 2023-10-09T10:44:19.245 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mon_report_max_in_flight": "2", 2023-10-09T10:44:19.245 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-10-09T10:44:19.245 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_num_cache_shards": "32", 2023-10-09T10:44:19.245 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_num_op_tracker_shard": "32", 2023-10-09T10:44:19.245 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_numa_auto_affinity": "true", 2023-10-09T10:44:19.246 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_numa_node": "-1", 2023-10-09T10:44:19.246 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_numa_prefer_iface": "true", 2023-10-09T10:44:19.246 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-10-09T10:44:19.246 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_objecter_finishers": "1", 2023-10-09T10:44:19.246 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_objectstore": "bluestore", 2023-10-09T10:44:19.246 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_objectstore_fuse": "false", 2023-10-09T10:44:19.246 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_objectstore_tracing": "false", 2023-10-09T10:44:19.246 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_complaint_time": "30.000000", 2023-10-09T10:44:19.246 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_history_duration": "600", 2023-10-09T10:44:19.246 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_history_size": "20", 2023-10-09T10:44:19.247 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_history_slow_op_size": "20", 2023-10-09T10:44:19.247 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-10-09T10:44:19.247 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_log_threshold": "5", 2023-10-09T10:44:19.247 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_num_shards": "0", 2023-10-09T10:44:19.247 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_num_shards_hdd": "5", 2023-10-09T10:44:19.247 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_num_shards_ssd": "8", 2023-10-09T10:44:19.247 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_num_threads_per_shard": "0", 2023-10-09T10:44:19.247 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-10-09T10:44:19.247 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-10-09T10:44:19.248 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-10-09T10:44:19.248 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_pq_min_cost": "65536", 2023-10-09T10:44:19.248 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_queue": "mclock_scheduler", 2023-10-09T10:44:19.248 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_queue_cut_off": "high", 2023-10-09T10:44:19.248 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_thread_suicide_timeout": "150", 2023-10-09T10:44:19.248 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_thread_timeout": "15", 2023-10-09T10:44:19.248 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_open_classes_on_start": "true", 2023-10-09T10:44:19.248 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_os_flags": "0", 2023-10-09T10:44:19.248 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_peering_op_priority": "255", 2023-10-09T10:44:19.248 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_delete_cost": "1048576", 2023-10-09T10:44:19.249 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_delete_priority": "5", 2023-10-09T10:44:19.249 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-10-09T10:44:19.249 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-10-09T10:44:19.249 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_log_dups_tracked": "3000", 2023-10-09T10:44:19.249 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_log_trim_max": "10000", 2023-10-09T10:44:19.249 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_log_trim_min": "100", 2023-10-09T10:44:19.249 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-10-09T10:44:19.249 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_object_context_cache_count": "64", 2023-10-09T10:44:19.249 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_stat_report_interval_max": "500", 2023-10-09T10:44:19.249 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-10-09T10:44:19.250 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-10-09T10:44:19.250 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-10-09T10:44:19.250 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-10-09T10:44:19.250 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-10-09T10:44:19.250 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-10-09T10:44:19.250 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_crimson": "false", 2023-10-09T10:44:19.250 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_crush_rule": "-1", 2023-10-09T10:44:19.250 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_ec_fast_read": "false", 2023-10-09T10:44:19.250 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-10-09T10:44:19.251 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_flag_bulk": "false", 2023-10-09T10:44:19.251 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-10-09T10:44:19.251 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_flag_nodelete": "false", 2023-10-09T10:44:19.251 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-10-09T10:44:19.251 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-10-09T10:44:19.251 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_flags": "0", 2023-10-09T10:44:19.251 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-10-09T10:44:19.251 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_min_size": "0", 2023-10-09T10:44:19.251 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-10-09T10:44:19.251 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_pg_num": "32", 2023-10-09T10:44:19.252 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_pgp_num": "0", 2023-10-09T10:44:19.252 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-10-09T10:44:19.252 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_size": "3", 2023-10-09T10:44:19.252 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_type": "replicated", 2023-10-09T10:44:19.252 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-10-09T10:44:19.252 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_use_gmt_hitset": "true", 2023-10-09T10:44:19.252 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_push_per_object_cost": "1000", 2023-10-09T10:44:19.252 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_read_ec_check_for_errors": "false", 2023-10-09T10:44:19.252 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recover_clone_overlap": "true", 2023-10-09T10:44:19.252 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recover_clone_overlap_limit": "10", 2023-10-09T10:44:19.253 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_cost": "20971520", 2023-10-09T10:44:19.253 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_delay_start": "0.000000", 2023-10-09T10:44:19.253 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_max_active": "0", 2023-10-09T10:44:19.253 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_max_active_hdd": "3", 2023-10-09T10:44:19.253 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_max_active_ssd": "10", 2023-10-09T10:44:19.253 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_max_chunk": "8388608", 2023-10-09T10:44:19.253 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-10-09T10:44:19.253 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_max_single_start": "1", 2023-10-09T10:44:19.253 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_op_priority": "3", 2023-10-09T10:44:19.254 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_op_warn_multiple": "16", 2023-10-09T10:44:19.254 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_priority": "5", 2023-10-09T10:44:19.254 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_retry_interval": "30.000000", 2023-10-09T10:44:19.254 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_sleep": "0.000000", 2023-10-09T10:44:19.254 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-10-09T10:44:19.254 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-10-09T10:44:19.254 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-10-09T10:44:19.254 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_repair_during_recovery": "false", 2023-10-09T10:44:19.254 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_requested_scrub_priority": "120", 2023-10-09T10:44:19.254 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-10-09T10:44:19.255 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_rollback_to_cluster_snap": "", 2023-10-09T10:44:19.255 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_auto_repair": "false", 2023-10-09T10:44:19.255 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-10-09T10:44:19.255 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-10-09T10:44:19.255 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_begin_hour": "0", 2023-10-09T10:44:19.255 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_begin_week_day": "0", 2023-10-09T10:44:19.255 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_chunk_max": "25", 2023-10-09T10:44:19.255 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_chunk_min": "5", 2023-10-09T10:44:19.255 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_cost": "52428800", 2023-10-09T10:44:19.255 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_during_recovery": "false", 2023-10-09T10:44:19.256 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_end_hour": "0", 2023-10-09T10:44:19.256 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_end_week_day": "0", 2023-10-09T10:44:19.256 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_event_cost": "4096", 2023-10-09T10:44:19.256 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-10-09T10:44:19.256 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-10-09T10:44:19.256 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_invalid_stats": "true", 2023-10-09T10:44:19.256 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_load_threshold": "0.500000", 2023-10-09T10:44:19.256 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_max_interval": "604800.000000", 2023-10-09T10:44:19.256 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_max_preemptions": "5", 2023-10-09T10:44:19.256 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_min_interval": "86400.000000", 2023-10-09T10:44:19.257 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_priority": "5", 2023-10-09T10:44:19.257 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_reservation_timeout": "5000", 2023-10-09T10:44:19.257 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_sleep": "0.000000", 2023-10-09T10:44:19.257 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-10-09T10:44:19.257 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-10-09T10:44:19.257 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-10-09T10:44:19.257 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_shutdown_pgref_assert": "false", 2023-10-09T10:44:19.257 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_skip_data_digest": "false", 2023-10-09T10:44:19.257 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_smart_report_timeout": "5", 2023-10-09T10:44:19.257 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_snap_trim_cost": "1048576", 2023-10-09T10:44:19.257 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_snap_trim_priority": "5", 2023-10-09T10:44:19.258 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_snap_trim_sleep": "0.000000", 2023-10-09T10:44:19.258 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-10-09T10:44:19.258 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-10-09T10:44:19.258 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-10-09T10:44:19.258 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-10-09T10:44:19.258 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_stats_update_period_scrubbing": "15", 2023-10-09T10:44:19.258 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-10-09T10:44:19.258 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_target_transaction_size": "30", 2023-10-09T10:44:19.258 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-10-09T10:44:19.259 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-10-09T10:44:19.261 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-10-09T10:44:19.261 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-10-09T10:44:19.261 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-10-09T10:44:19.261 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-10-09T10:44:19.261 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-10-09T10:44:19.261 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_mode": "writeback", 2023-10-09T10:44:19.261 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-10-09T10:44:19.262 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-10-09T10:44:19.262 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tracing": "false", 2023-10-09T10:44:19.262 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_use_stale_snap": "false", 2023-10-09T10:44:19.262 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-10-09T10:44:19.262 INFO:teuthology.orchestra.run.smithi125.stdout: "osdc_blkin_trace_all": "false", 2023-10-09T10:44:19.262 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_kill_at": "0", 2023-10-09T10:44:19.262 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_max_join_drift": "10", 2023-10-09T10:44:19.262 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_min": "500", 2023-10-09T10:44:19.262 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_min_wait": "0.050000", 2023-10-09T10:44:19.263 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_propose_interval": "1.000000", 2023-10-09T10:44:19.263 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_service_trim_max": "500", 2023-10-09T10:44:19.263 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_service_trim_max_multiplier": "20", 2023-10-09T10:44:19.263 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_service_trim_min": "250", 2023-10-09T10:44:19.263 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_stash_full_interval": "25", 2023-10-09T10:44:19.263 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_trim_max": "500", 2023-10-09T10:44:19.263 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_trim_min": "250", 2023-10-09T10:44:19.263 INFO:teuthology.orchestra.run.smithi125.stdout: "perf": "true", 2023-10-09T10:44:19.264 INFO:teuthology.orchestra.run.smithi125.stdout: "pid_file": "", 2023-10-09T10:44:19.264 INFO:teuthology.orchestra.run.smithi125.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-10-09T10:44:19.264 INFO:teuthology.orchestra.run.smithi125.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-10-09T10:44:19.264 INFO:teuthology.orchestra.run.smithi125.stdout: "public_addr": "-", 2023-10-09T10:44:19.264 INFO:teuthology.orchestra.run.smithi125.stdout: "public_addrv": "", 2023-10-09T10:44:19.264 INFO:teuthology.orchestra.run.smithi125.stdout: "public_bind_addr": "-", 2023-10-09T10:44:19.264 INFO:teuthology.orchestra.run.smithi125.stdout: "public_network": "172.21.0.0/20", 2023-10-09T10:44:19.264 INFO:teuthology.orchestra.run.smithi125.stdout: "public_network_interface": "", 2023-10-09T10:44:19.264 INFO:teuthology.orchestra.run.smithi125.stdout: "qat_compressor_enabled": "false", 2023-10-09T10:44:19.265 INFO:teuthology.orchestra.run.smithi125.stdout: "qat_compressor_session_max_number": "256", 2023-10-09T10:44:19.265 INFO:teuthology.orchestra.run.smithi125.stdout: "rados_mon_op_timeout": "0", 2023-10-09T10:44:19.265 INFO:teuthology.orchestra.run.smithi125.stdout: "rados_osd_op_timeout": "0", 2023-10-09T10:44:19.265 INFO:teuthology.orchestra.run.smithi125.stdout: "rados_tracing": "false", 2023-10-09T10:44:19.265 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_atime_update_interval": "60", 2023-10-09T10:44:19.265 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-10-09T10:44:19.265 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_balance_parent_reads": "false", 2023-10-09T10:44:19.266 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_balance_snap_reads": "false", 2023-10-09T10:44:19.266 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_blkin_trace_all": "false", 2023-10-09T10:44:19.266 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_blocklist_expire_seconds": "0", 2023-10-09T10:44:19.266 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_blocklist_on_break_lock": "true", 2023-10-09T10:44:19.266 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache": "true", 2023-10-09T10:44:19.266 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_block_writes_upfront": "false", 2023-10-09T10:44:19.266 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_max_dirty": "25165824", 2023-10-09T10:44:19.266 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-10-09T10:44:19.266 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_max_dirty_object": "0", 2023-10-09T10:44:19.267 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_policy": "writearound", 2023-10-09T10:44:19.267 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_size": "33554432", 2023-10-09T10:44:19.267 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_target_dirty": "16777216", 2023-10-09T10:44:19.267 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-10-09T10:44:19.267 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_clone_copy_on_read": "false", 2023-10-09T10:44:19.267 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_compression_hint": "none", 2023-10-09T10:44:19.267 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_concurrent_management_ops": "10", 2023-10-09T10:44:19.267 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-10-09T10:44:19.268 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_clone_format": "auto", 2023-10-09T10:44:19.268 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_data_pool": "", 2023-10-09T10:44:19.268 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_features": "61", 2023-10-09T10:44:19.268 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_format": "2", 2023-10-09T10:44:19.268 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_map_options": "", 2023-10-09T10:44:19.268 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_order": "22", 2023-10-09T10:44:19.268 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_pool": "rbd", 2023-10-09T10:44:19.268 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-10-09T10:44:19.269 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_stripe_count": "0", 2023-10-09T10:44:19.269 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_stripe_unit": "0", 2023-10-09T10:44:19.269 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_disable_zero_copy_writes": "true", 2023-10-09T10:44:19.269 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_discard_granularity_bytes": "65536", 2023-10-09T10:44:19.269 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-10-09T10:44:19.269 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_enable_alloc_hint": "true", 2023-10-09T10:44:19.269 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-10-09T10:44:19.269 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_io_scheduler": "simple", 2023-10-09T10:44:19.269 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-10-09T10:44:19.270 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_commit_age": "5.000000", 2023-10-09T10:44:19.270 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-10-09T10:44:19.270 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-10-09T10:44:19.270 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-10-09T10:44:19.270 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-10-09T10:44:19.270 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_object_flush_interval": "0", 2023-10-09T10:44:19.270 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-10-09T10:44:19.270 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-10-09T10:44:19.271 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_order": "24", 2023-10-09T10:44:19.271 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_pool": "", 2023-10-09T10:44:19.271 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_splay_width": "4", 2023-10-09T10:44:19.271 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_localize_parent_reads": "false", 2023-10-09T10:44:19.271 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_localize_snap_reads": "false", 2023-10-09T10:44:19.271 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-10-09T10:44:19.271 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-10-09T10:44:19.272 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-10-09T10:44:19.272 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-10-09T10:44:19.272 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-10-09T10:44:19.272 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-10-09T10:44:19.272 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_image_policy_type": "simple", 2023-10-09T10:44:19.272 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-10-09T10:44:19.272 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-10-09T10:44:19.272 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-10-09T10:44:19.273 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-10-09T10:44:19.273 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-10-09T10:44:19.273 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-10-09T10:44:19.273 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-10-09T10:44:19.273 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_memory_autotune": "true", 2023-10-09T10:44:19.273 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_memory_base": "805306368", 2023-10-09T10:44:19.273 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-10-09T10:44:19.274 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-10-09T10:44:19.274 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-10-09T10:44:19.274 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-10-09T10:44:19.274 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_memory_target": "4294967296", 2023-10-09T10:44:19.274 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-10-09T10:44:19.274 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-10-09T10:44:19.274 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-10-09T10:44:19.274 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirroring_delete_delay": "0", 2023-10-09T10:44:19.274 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-10-09T10:44:19.275 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirroring_replay_delay": "0", 2023-10-09T10:44:19.275 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-10-09T10:44:19.275 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-10-09T10:44:19.275 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_move_to_trash_on_remove": "false", 2023-10-09T10:44:19.275 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-10-09T10:44:19.275 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mtime_update_interval": "60", 2023-10-09T10:44:19.275 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_non_blocking_aio": "true", 2023-10-09T10:44:19.275 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_op_thread_timeout": "60", 2023-10-09T10:44:19.275 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_op_threads": "1", 2023-10-09T10:44:19.276 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_parent_cache_enabled": "false", 2023-10-09T10:44:19.276 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_persistent_cache_mode": "disabled", 2023-10-09T10:44:19.276 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_persistent_cache_path": "/tmp", 2023-10-09T10:44:19.276 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_persistent_cache_size": "1073741824", 2023-10-09T10:44:19.276 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_plugins": "", 2023-10-09T10:44:19.276 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_bps_burst": "0", 2023-10-09T10:44:19.276 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-10-09T10:44:19.276 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_bps_limit": "0", 2023-10-09T10:44:19.276 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_exclude_ops": "0", 2023-10-09T10:44:19.277 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_iops_burst": "0", 2023-10-09T10:44:19.277 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-10-09T10:44:19.277 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_iops_limit": "0", 2023-10-09T10:44:19.277 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_read_bps_burst": "0", 2023-10-09T10:44:19.277 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-10-09T10:44:19.277 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_read_bps_limit": "0", 2023-10-09T10:44:19.277 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_read_iops_burst": "0", 2023-10-09T10:44:19.277 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-10-09T10:44:19.277 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_read_iops_limit": "0", 2023-10-09T10:44:19.277 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_schedule_tick_min": "50", 2023-10-09T10:44:19.277 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_write_bps_burst": "0", 2023-10-09T10:44:19.278 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-10-09T10:44:19.278 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_write_bps_limit": "0", 2023-10-09T10:44:19.278 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_write_iops_burst": "0", 2023-10-09T10:44:19.278 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-10-09T10:44:19.278 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_write_iops_limit": "0", 2023-10-09T10:44:19.278 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_quiesce_notification_attempts": "10", 2023-10-09T10:44:19.278 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_read_from_replica_policy": "default", 2023-10-09T10:44:19.278 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-10-09T10:44:19.279 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_readahead_max_bytes": "524288", 2023-10-09T10:44:19.279 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_readahead_trigger_requests": "10", 2023-10-09T10:44:19.279 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_request_timed_out_seconds": "30", 2023-10-09T10:44:19.279 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_skip_partial_discard": "true", 2023-10-09T10:44:19.279 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-10-09T10:44:19.279 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_tracing": "false", 2023-10-09T10:44:19.279 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_validate_names": "true", 2023-10-09T10:44:19.279 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_validate_pool": "true", 2023-10-09T10:44:19.279 INFO:teuthology.orchestra.run.smithi125.stdout: "restapi_base_url": "", 2023-10-09T10:44:19.279 INFO:teuthology.orchestra.run.smithi125.stdout: "restapi_log_level": "", 2023-10-09T10:44:19.280 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_acl_grants_max_num": "100", 2023-10-09T10:44:19.280 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_admin_entry": "admin", 2023-10-09T10:44:19.280 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-10-09T10:44:19.280 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_backend_store": "rados", 2023-10-09T10:44:19.280 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_barbican_url": "", 2023-10-09T10:44:19.280 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_beast_enable_async": "true", 2023-10-09T10:44:19.280 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-10-09T10:44:19.280 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-10-09T10:44:19.280 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_bucket_index_max_aio": "128", 2023-10-09T10:44:19.280 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-10-09T10:44:19.281 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-10-09T10:44:19.281 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_bucket_quota_ttl": "600", 2023-10-09T10:44:19.281 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-10-09T10:44:19.281 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_cache_enabled": "true", 2023-10-09T10:44:19.281 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_cache_expiry_interval": "900", 2023-10-09T10:44:19.281 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_cache_lru_size": "10000", 2023-10-09T10:44:19.281 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_config_store": "rados", 2023-10-09T10:44:19.281 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_content_length_compat": "false", 2023-10-09T10:44:19.281 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_copy_obj_progress": "true", 2023-10-09T10:44:19.281 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-10-09T10:44:19.281 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_cors_rules_max_num": "100", 2023-10-09T10:44:19.282 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_cross_domain_policy": "", 2023-10-09T10:44:19.282 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_default_encryption_key": "", 2023-10-09T10:44:19.282 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_addr": "", 2023-10-09T10:44:19.282 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_ca_path": "", 2023-10-09T10:44:19.282 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_client_cert": "", 2023-10-09T10:44:19.282 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_client_key": "", 2023-10-09T10:44:19.282 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-10-09T10:44:19.282 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_password": "", 2023-10-09T10:44:19.282 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-10-09T10:44:19.283 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_username": "", 2023-10-09T10:44:19.283 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_require_ssl": "true", 2023-10-09T10:44:19.283 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-10-09T10:44:19.283 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-10-09T10:44:19.283 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-10-09T10:44:19.283 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-10-09T10:44:19.283 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-10-09T10:44:19.283 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-10-09T10:44:19.283 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-10-09T10:44:19.283 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-10-09T10:44:19.284 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-10-09T10:44:19.284 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-10-09T10:44:19.284 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-10-09T10:44:19.284 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-10-09T10:44:19.284 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-10-09T10:44:19.284 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-10-09T10:44:19.284 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_suppress_logs": "true", 2023-10-09T10:44:19.284 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_addr": "", 2023-10-09T10:44:19.284 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_auth": "token", 2023-10-09T10:44:19.285 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_namespace": "", 2023-10-09T10:44:19.285 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_prefix": "", 2023-10-09T10:44:19.285 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-10-09T10:44:19.285 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-10-09T10:44:19.285 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-10-09T10:44:19.285 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-10-09T10:44:19.285 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_token_file": "", 2023-10-09T10:44:19.285 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-10-09T10:44:19.285 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_curl_buffersize": "524288", 2023-10-09T10:44:19.286 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_curl_low_speed_limit": "1024", 2023-10-09T10:44:19.286 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_curl_low_speed_time": "300", 2023-10-09T10:44:19.286 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_curl_tcp_keepalive": "0", 2023-10-09T10:44:19.286 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-10-09T10:44:19.286 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-10-09T10:44:19.286 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-10-09T10:44:19.286 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-10-09T10:44:19.286 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-10-09T10:44:19.286 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_l1_fadvise": "4", 2023-10-09T10:44:19.287 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-10-09T10:44:19.287 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-10-09T10:44:19.287 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-10-09T10:44:19.287 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-10-09T10:44:19.287 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data_log_changes_size": "1000", 2023-10-09T10:44:19.287 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data_log_num_shards": "128", 2023-10-09T10:44:19.287 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-10-09T10:44:19.287 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data_log_window": "30", 2023-10-09T10:44:19.287 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data_notify_interval_msec": "0", 2023-10-09T10:44:19.287 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data_sync_poll_interval": "20", 2023-10-09T10:44:19.288 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data_sync_spawn_window": "20", 2023-10-09T10:44:19.288 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-10-09T10:44:19.288 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-10-09T10:44:19.288 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_default_data_log_backing": "fifo", 2023-10-09T10:44:19.288 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-10-09T10:44:19.288 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_default_region_info_oid": "default.region", 2023-10-09T10:44:19.288 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-10-09T10:44:19.288 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-10-09T10:44:19.288 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_defer_to_bucket_acls": "", 2023-10-09T10:44:19.288 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-10-09T10:44:19.288 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-10-09T10:44:19.289 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-10-09T10:44:19.289 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-10-09T10:44:19.289 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-10-09T10:44:19.289 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-10-09T10:44:19.289 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-10-09T10:44:19.289 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-10-09T10:44:19.289 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_data_res": "500.000000", 2023-10-09T10:44:19.289 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-10-09T10:44:19.289 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-10-09T10:44:19.289 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-10-09T10:44:19.290 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-10-09T10:44:19.290 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dns_name": "", 2023-10-09T10:44:19.290 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dns_s3website_name": "", 2023-10-09T10:44:19.290 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dynamic_resharding": "true", 2023-10-09T10:44:19.290 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-10-09T10:44:19.290 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enable_gc_threads": "true", 2023-10-09T10:44:19.290 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enable_lc_threads": "true", 2023-10-09T10:44:19.290 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enable_ops_log": "false", 2023-10-09T10:44:19.290 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enable_quota_threads": "true", 2023-10-09T10:44:19.290 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enable_static_website": "false", 2023-10-09T10:44:19.291 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enable_usage_log": "false", 2023-10-09T10:44:19.291 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enforce_swift_acls": "true", 2023-10-09T10:44:19.291 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_exit_timeout_secs": "120", 2023-10-09T10:44:19.291 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_expose_bucket": "false", 2023-10-09T10:44:19.291 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_extended_http_attrs": "", 2023-10-09T10:44:19.291 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_filter": "none", 2023-10-09T10:44:19.291 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-10-09T10:44:19.291 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_frontends": "beast port=7480", 2023-10-09T10:44:19.291 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_max_concurrent_io": "10", 2023-10-09T10:44:19.291 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_max_deferred": "50", 2023-10-09T10:44:19.292 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-10-09T10:44:19.292 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_max_objs": "32", 2023-10-09T10:44:19.292 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_max_queue_size": "134213632", 2023-10-09T10:44:19.292 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_max_trim_chunk": "16", 2023-10-09T10:44:19.292 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_obj_min_wait": "7200", 2023-10-09T10:44:19.292 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_processor_max_time": "3600", 2023-10-09T10:44:19.292 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_processor_period": "3600", 2023-10-09T10:44:19.292 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-10-09T10:44:19.292 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_get_obj_window_size": "16777216", 2023-10-09T10:44:19.292 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_healthcheck_disabling_path": "", 2023-10-09T10:44:19.293 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ignore_get_invalid_range": "false", 2023-10-09T10:44:19.293 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_init_timeout": "300", 2023-10-09T10:44:19.293 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-10-09T10:44:19.293 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-10-09T10:44:19.293 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-10-09T10:44:19.293 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-10-09T10:44:19.293 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_domain": "", 2023-10-09T10:44:19.293 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_password": "", 2023-10-09T10:44:19.293 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_password_path": "", 2023-10-09T10:44:19.293 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_project": "", 2023-10-09T10:44:19.294 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_tenant": "", 2023-10-09T10:44:19.294 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_token": "", 2023-10-09T10:44:19.294 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_token_path": "", 2023-10-09T10:44:19.294 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_user": "", 2023-10-09T10:44:19.294 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_api_version": "2", 2023-10-09T10:44:19.294 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_barbican_domain": "", 2023-10-09T10:44:19.294 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_barbican_password": "", 2023-10-09T10:44:19.294 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_barbican_project": "", 2023-10-09T10:44:19.294 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_barbican_tenant": "", 2023-10-09T10:44:19.294 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_barbican_user": "", 2023-10-09T10:44:19.295 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-10-09T10:44:19.295 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_implicit_tenants": "false", 2023-10-09T10:44:19.295 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-10-09T10:44:19.295 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_service_token_enabled": "false", 2023-10-09T10:44:19.295 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_token_cache_size": "10000", 2023-10-09T10:44:19.295 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_url": "", 2023-10-09T10:44:19.295 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_verify_ssl": "true", 2023-10-09T10:44:19.295 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lc_debug_interval": "-1", 2023-10-09T10:44:19.295 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lc_lock_max_time": "90", 2023-10-09T10:44:19.295 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lc_max_objs": "32", 2023-10-09T10:44:19.296 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lc_max_rules": "1000", 2023-10-09T10:44:19.296 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lc_max_worker": "3", 2023-10-09T10:44:19.296 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lc_max_wp_worker": "3", 2023-10-09T10:44:19.296 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lc_thread_delay": "0", 2023-10-09T10:44:19.296 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-10-09T10:44:19.296 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ldap_dnattr": "uid", 2023-10-09T10:44:19.296 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-10-09T10:44:19.297 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ldap_searchfilter": "", 2023-10-09T10:44:19.297 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-10-09T10:44:19.297 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ldap_uri": "ldaps://", 2023-10-09T10:44:19.297 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-10-09T10:44:19.297 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-10-09T10:44:19.297 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-10-09T10:44:19.297 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_log_http_headers": "", 2023-10-09T10:44:19.297 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_log_nonexistent_bucket": "false", 2023-10-09T10:44:19.297 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-10-09T10:44:19.298 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_log_object_name_utc": "false", 2023-10-09T10:44:19.298 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-10-09T10:44:19.298 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_attr_name_len": "0", 2023-10-09T10:44:19.298 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_attr_size": "0", 2023-10-09T10:44:19.298 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_attrs_num_in_req": "0", 2023-10-09T10:44:19.298 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_chunk_size": "4194304", 2023-10-09T10:44:19.298 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_concurrent_requests": "1024", 2023-10-09T10:44:19.299 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-10-09T10:44:19.299 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_dynamic_shards": "1999", 2023-10-09T10:44:19.299 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_listing_results": "1000", 2023-10-09T10:44:19.299 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_notify_retries": "10", 2023-10-09T10:44:19.299 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_objs_per_shard": "100000", 2023-10-09T10:44:19.299 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_put_param_size": "1048576", 2023-10-09T10:44:19.299 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_put_size": "5368709120", 2023-10-09T10:44:19.299 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_slo_entries": "1000", 2023-10-09T10:44:19.300 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_md_log_max_shards": "64", 2023-10-09T10:44:19.300 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_md_notify_interval_msec": "200", 2023-10-09T10:44:19.300 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_meta_sync_poll_interval": "20", 2023-10-09T10:44:19.300 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_meta_sync_spawn_window": "20", 2023-10-09T10:44:19.300 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-10-09T10:44:19.300 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_mp_lock_max_time": "600", 2023-10-09T10:44:19.300 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-10-09T10:44:19.300 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_multipart_min_part_size": "5242880", 2023-10-09T10:44:19.300 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-10-09T10:44:19.301 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-10-09T10:44:19.301 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_fhcache_size": "2017", 2023-10-09T10:44:19.301 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-10-09T10:44:19.301 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-10-09T10:44:19.301 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_lru_lanes": "5", 2023-10-09T10:44:19.301 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_max_gc": "300", 2023-10-09T10:44:19.301 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-10-09T10:44:19.301 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_run_gc_threads": "false", 2023-10-09T10:44:19.302 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_run_lc_threads": "false", 2023-10-09T10:44:19.302 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_run_quota_threads": "false", 2023-10-09T10:44:19.302 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_run_sync_thread": "false", 2023-10-09T10:44:19.302 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-10-09T10:44:19.302 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-10-09T10:44:19.302 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_num_async_rados_threads": "32", 2023-10-09T10:44:19.302 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_num_control_oids": "8", 2023-10-09T10:44:19.302 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_numa_node": "-1", 2023-10-09T10:44:19.303 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_obj_stripe_size": "4194304", 2023-10-09T10:44:19.303 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-10-09T10:44:19.303 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_objexp_chunk_size": "100", 2023-10-09T10:44:19.303 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_objexp_gc_interval": "600", 2023-10-09T10:44:19.303 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_objexp_hints_num_shards": "127", 2023-10-09T10:44:19.303 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-10-09T10:44:19.303 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-10-09T10:44:19.303 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_op_thread_timeout": "600", 2023-10-09T10:44:19.304 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_op_tracing": "false", 2023-10-09T10:44:19.304 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_opa_token": "", 2023-10-09T10:44:19.304 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_opa_url": "", 2023-10-09T10:44:19.304 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_opa_verify_ssl": "true", 2023-10-09T10:44:19.304 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-10-09T10:44:19.304 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-10-09T10:44:19.304 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ops_log_rados": "false", 2023-10-09T10:44:19.305 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ops_log_socket_path": "", 2023-10-09T10:44:19.305 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-10-09T10:44:19.305 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-10-09T10:44:19.305 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-10-09T10:44:19.305 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_period_push_interval": "2.000000", 2023-10-09T10:44:19.305 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_period_push_interval_max": "30.000000", 2023-10-09T10:44:19.305 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_period_root_pool": ".rgw.root", 2023-10-09T10:44:19.306 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-10-09T10:44:19.306 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_print_continue": "true", 2023-10-09T10:44:19.306 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_print_prohibited_content_length": "false", 2023-10-09T10:44:19.306 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-10-09T10:44:19.306 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-10-09T10:44:19.306 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-10-09T10:44:19.306 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-10-09T10:44:19.306 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_rados_tracing": "false", 2023-10-09T10:44:19.307 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_realm": "", 2023-10-09T10:44:19.307 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_realm_id": "", 2023-10-09T10:44:19.307 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-10-09T10:44:19.307 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_region": "", 2023-10-09T10:44:19.307 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_region_root_pool": ".rgw.root", 2023-10-09T10:44:19.307 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_relaxed_region_enforcement": "false", 2023-10-09T10:44:19.307 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-10-09T10:44:19.308 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-10-09T10:44:19.308 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_request_uri": "", 2023-10-09T10:44:19.308 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_reshard_batch_size": "64", 2023-10-09T10:44:19.308 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-10-09T10:44:19.308 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_reshard_max_aio": "128", 2023-10-09T10:44:19.308 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_reshard_num_logs": "16", 2023-10-09T10:44:19.309 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_reshard_thread_interval": "600", 2023-10-09T10:44:19.309 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_resolve_cname": "false", 2023-10-09T10:44:19.309 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_rest_getusage_op_compat": "false", 2023-10-09T10:44:19.309 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_run_sync_thread": "true", 2023-10-09T10:44:19.309 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-10-09T10:44:19.309 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_s3_auth_use_keystone": "false", 2023-10-09T10:44:19.309 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_s3_auth_use_ldap": "false", 2023-10-09T10:44:19.309 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_s3_auth_use_rados": "true", 2023-10-09T10:44:19.309 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_s3_auth_use_sts": "false", 2023-10-09T10:44:19.310 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-10-09T10:44:19.310 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_s3_success_create_obj_status": "0", 2023-10-09T10:44:19.310 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-10-09T10:44:19.310 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_scheduler_type": "throttler", 2023-10-09T10:44:19.310 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_script_uri": "", 2023-10-09T10:44:19.310 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_service_provider_name": "", 2023-10-09T10:44:19.310 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-10-09T10:44:19.310 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sts_client_id": "", 2023-10-09T10:44:19.310 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sts_client_secret": "", 2023-10-09T10:44:19.310 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sts_entry": "sts", 2023-10-09T10:44:19.310 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sts_key": "sts", 2023-10-09T10:44:19.311 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sts_max_session_duration": "43200", 2023-10-09T10:44:19.311 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sts_min_session_duration": "900", 2023-10-09T10:44:19.311 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sts_token_introspection_url": "", 2023-10-09T10:44:19.311 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_account_in_url": "false", 2023-10-09T10:44:19.311 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_auth_entry": "auth", 2023-10-09T10:44:19.311 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_auth_url": "", 2023-10-09T10:44:19.311 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_custom_header": "", 2023-10-09T10:44:19.311 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_enforce_content_length": "false", 2023-10-09T10:44:19.311 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_need_stats": "true", 2023-10-09T10:44:19.311 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_tenant_name": "", 2023-10-09T10:44:19.312 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_token_expiration": "86400", 2023-10-09T10:44:19.312 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_url": "", 2023-10-09T10:44:19.312 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_url_prefix": "swift", 2023-10-09T10:44:19.312 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_versioning_enabled": "false", 2023-10-09T10:44:19.312 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-10-09T10:44:19.312 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-10-09T10:44:19.312 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_lease_period": "120", 2023-10-09T10:44:19.312 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-10-09T10:44:19.312 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_log_trim_interval": "1200", 2023-10-09T10:44:19.313 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-10-09T10:44:19.313 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-10-09T10:44:19.313 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-10-09T10:44:19.313 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_obj_etag_verify": "false", 2023-10-09T10:44:19.313 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_trace_history_size": "4096", 2023-10-09T10:44:19.313 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-10-09T10:44:19.313 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-10-09T10:44:19.313 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_thread_pool_size": "512", 2023-10-09T10:44:19.313 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_torrent_comment": "", 2023-10-09T10:44:19.314 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_torrent_createby": "", 2023-10-09T10:44:19.314 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_torrent_encoding": "", 2023-10-09T10:44:19.314 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_torrent_flag": "false", 2023-10-09T10:44:19.314 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_torrent_origin": "", 2023-10-09T10:44:19.314 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_torrent_sha_unit": "524288", 2023-10-09T10:44:19.314 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_torrent_tracker": "", 2023-10-09T10:44:19.314 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_trust_forwarded_https": "false", 2023-10-09T10:44:19.314 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-10-09T10:44:19.314 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_usage_log_tick_interval": "30", 2023-10-09T10:44:19.314 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_usage_max_shards": "32", 2023-10-09T10:44:19.315 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_usage_max_user_shards": "1", 2023-10-09T10:44:19.315 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_use_opa_authz": "false", 2023-10-09T10:44:19.315 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-10-09T10:44:19.315 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_default_quota_max_size": "-1", 2023-10-09T10:44:19.315 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_max_buckets": "1000", 2023-10-09T10:44:19.315 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_policies_max_num": "100", 2023-10-09T10:44:19.315 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-10-09T10:44:19.315 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-10-09T10:44:19.315 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_quota_sync_interval": "86400", 2023-10-09T10:44:19.315 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-10-09T10:44:19.316 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_unique_email": "true", 2023-10-09T10:44:19.316 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_verify_ssl": "true", 2023-10-09T10:44:19.316 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_website_routing_rules_max_num": "50", 2023-10-09T10:44:19.316 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_zone": "", 2023-10-09T10:44:19.316 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_zone_id": "", 2023-10-09T10:44:19.316 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-10-09T10:44:19.316 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_zonegroup": "", 2023-10-09T10:44:19.316 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_zonegroup_id": "", 2023-10-09T10:44:19.316 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-10-09T10:44:19.317 INFO:teuthology.orchestra.run.smithi125.stdout: "rgwlc_auto_session_clear": "true", 2023-10-09T10:44:19.317 INFO:teuthology.orchestra.run.smithi125.stdout: "rgwlc_skip_bucket_step": "false", 2023-10-09T10:44:19.317 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_block_size": "4096", 2023-10-09T10:44:19.317 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-10-09T10:44:19.317 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-10-09T10:44:19.317 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-10-09T10:44:19.317 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-10-09T10:44:19.317 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cache_shard_bits": "4", 2023-10-09T10:44:19.317 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cache_size": "536870912", 2023-10-09T10:44:19.317 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cache_type": "binned_lru", 2023-10-09T10:44:19.318 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-10-09T10:44:19.318 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-10-09T10:44:19.318 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-10-09T10:44:19.318 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_collect_compaction_stats": "false", 2023-10-09T10:44:19.318 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_collect_extended_stats": "false", 2023-10-09T10:44:19.318 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_collect_memory_stats": "false", 2023-10-09T10:44:19.318 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-10-09T10:44:19.318 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_index_type": "binary_search", 2023-10-09T10:44:19.318 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_log_to_ceph_log": "true", 2023-10-09T10:44:19.318 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_metadata_block_size": "4096", 2023-10-09T10:44:19.319 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_partition_filters": "false", 2023-10-09T10:44:19.319 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_perf": "false", 2023-10-09T10:44:19.319 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-10-09T10:44:19.319 INFO:teuthology.orchestra.run.smithi125.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-10-09T10:44:19.319 INFO:teuthology.orchestra.run.smithi125.stdout: "rotating_keys_renewal_timeout": "10", 2023-10-09T10:44:19.319 INFO:teuthology.orchestra.run.smithi125.stdout: "run_dir": "/var/run/ceph", 2023-10-09T10:44:19.319 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_block_create": "true", 2023-10-09T10:44:19.319 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_cache_lru_size": "67108864", 2023-10-09T10:44:19.319 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_cbjournal_size": "5368709120", 2023-10-09T10:44:19.320 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_default_max_object_size": "16777216", 2023-10-09T10:44:19.320 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-10-09T10:44:19.320 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_device_size": "53687091200", 2023-10-09T10:44:19.320 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_journal_batch_capacity": "16", 2023-10-09T10:44:19.320 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-10-09T10:44:19.320 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-10-09T10:44:19.320 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_journal_iodepth_limit": "5", 2023-10-09T10:44:19.320 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_main_device_type": "SSD", 2023-10-09T10:44:19.320 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_max_concurrent_transactions": "8", 2023-10-09T10:44:19.320 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-10-09T10:44:19.321 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-10-09T10:44:19.321 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-10-09T10:44:19.321 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-10-09T10:44:19.321 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_segment_size": "67108864", 2023-10-09T10:44:19.321 INFO:teuthology.orchestra.run.smithi125.stdout: "set_keepcaps": "false", 2023-10-09T10:44:19.321 INFO:teuthology.orchestra.run.smithi125.stdout: "setgroup": "ceph", 2023-10-09T10:44:19.321 INFO:teuthology.orchestra.run.smithi125.stdout: "setuser": "ceph", 2023-10-09T10:44:19.321 INFO:teuthology.orchestra.run.smithi125.stdout: "setuser_match_path": "", 2023-10-09T10:44:19.321 INFO:teuthology.orchestra.run.smithi125.stdout: "target_max_misplaced_ratio": "0.050000", 2023-10-09T10:44:19.322 INFO:teuthology.orchestra.run.smithi125.stdout: "thp": "false", 2023-10-09T10:44:19.322 INFO:teuthology.orchestra.run.smithi125.stdout: "threadpool_default_timeout": "60", 2023-10-09T10:44:19.322 INFO:teuthology.orchestra.run.smithi125.stdout: "threadpool_empty_queue_max_wait": "2", 2023-10-09T10:44:19.322 INFO:teuthology.orchestra.run.smithi125.stdout: "throttler_perf_counter": "true" 2023-10-09T10:44:19.322 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:44:19.497 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-10-09T10:44:19.761 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:20.157 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:20 smithi125 ceph-mon[195507]: pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:20.157 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:20 smithi125 ceph-mon[200197]: pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:20 smithi155 ceph-mon[184854]: pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:20.719 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a: {} 2023-10-09T10:44:20.731 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b: {} 2023-10-09T10:44:20.740 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c: {} 2023-10-09T10:44:20.742 INFO:teuthology.orchestra.run.smithi125.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T10:44:20.742 INFO:teuthology.orchestra.run.smithi125.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T10:44:20.742 INFO:teuthology.orchestra.run.smithi125.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T10:44:21.145 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-10-09T10:44:21.145 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T10:44:21.146 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T10:44:21.147 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T10:44:21.147 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-10-09T10:44:21.148 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-10-09T10:44:21.148 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-10-09T10:44:21.148 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:44:21.148 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:44:21.149 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T10:44:21.149 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:44:21.149 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:44:21.149 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T10:44:21.149 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-10-09T10:44:21.149 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:21.150 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-10-09T10:44:21.150 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:44:21.150 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-10-09T10:44:21.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:44:21.151 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-10-09T10:44:21.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:44:21.183 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:21 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3597346603' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T10:44:21.184 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:21 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3597346603' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T10:44:21.189 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:21.218 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:21.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:21.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:21.323 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:21.357 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:21.379 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:21.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:21.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:21.429 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:44:21.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:21 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3597346603' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T10:44:21.483 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:21.505 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:21.508 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:44:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:44:21] "GET /metrics HTTP/1.1" 200 34051 "" "Prometheus/2.43.0" 2023-10-09T10:44:21.511 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:21.535 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:21.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:21.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:21.638 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:21.657 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:21.670 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:21.692 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:21.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:21.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:21.796 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:21.826 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:21.999 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:22.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:22.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:22.102 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:22.128 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:22 smithi155 ceph-mon[184854]: pgmap v191: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:22.130 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:22.199 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:22.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:22.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:22.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:22.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:22 smithi125 ceph-mon[195507]: pgmap v191: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:22.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:22 smithi125 ceph-mon[200197]: pgmap v191: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:22.333 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:22.353 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:22.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:22.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:22.370 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:44:22.370 INFO:teuthology.orchestra.run.smithi155.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T10:44:22.370 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:44:22.457 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:22.486 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:22.493 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:22.504 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:22.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:22.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:22.608 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:22.636 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:22.656 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:22.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:22.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:22.760 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:22.793 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:22.838 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:44:22.839 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:44:22.839 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 0 2023-10-09T10:44:22.839 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:44:22.917 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:22.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:22.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:23.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:44:23.049 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:23.068 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:44:23.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:23.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:44:23.171 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:23.199 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:23.217 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:23.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:23.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:23.283 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:23.321 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:23.352 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:23.467 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:44:23.657 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:44:23.675 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:44:23.694 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:44:23.713 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:44:23.733 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:44:23.753 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:44:23.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:23 smithi125 ceph-mon[195507]: pgmap v192: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:23.758 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:23 smithi125 ceph-mon[200197]: pgmap v192: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:23.776 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:44:23.777 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:23.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:23.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:23.796 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:44:23.798 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:44:23.798 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:44:23.798 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:44:23.798 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:44:23.798 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:44:23.798 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:44:23.798 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:44:23.799 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:44:23.881 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:23.910 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:23 smithi155 ceph-mon[184854]: pgmap v192: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:23.912 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:23.932 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:23.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:23.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:24.036 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:44:24.066 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:24.085 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:44:24.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:24.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:44:24.189 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:24.217 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:24.235 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:24.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:24.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:24.339 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:24.366 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:24.385 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:24.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:24.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:24.488 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:24.560 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:24.581 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:24.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:24.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:24.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:24.726 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:24 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4246885366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:24.726 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:24 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4246885366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:24.729 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:24.783 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:24.797 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:24.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:24.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:24.903 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:24.933 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:24.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:24 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4246885366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:25.052 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:25.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:25.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:25.156 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:44:25.188 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:25.209 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:44:25.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:25.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:44:25.313 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:25.341 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:25.510 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:25.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:25.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:25.557 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:44:25.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1877099660' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:44:25.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:25 smithi125 ceph-mon[195507]: pgmap v193: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:25.596 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1877099660' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:44:25.596 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:25 smithi125 ceph-mon[200197]: pgmap v193: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:25.614 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:25.647 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:25.668 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:25.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:25.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:25.772 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:25.802 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:25.824 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:25.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:25.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:25.891 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:44:25.891 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:44:25.927 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:25.954 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:25.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1877099660' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:44:25.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:25 smithi155 ceph-mon[184854]: pgmap v193: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:26.125 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:26.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:26.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:26.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:26.255 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:26.323 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:26.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:26.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:26.376 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:44:26.427 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:26.460 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:26.481 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:26.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:26.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:26.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:44:26.565 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_0' created 2023-10-09T10:44:26.584 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:26.614 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:26.635 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:26.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:26.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:26 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:44:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:26 smithi155 ceph-mon[184854]: failed to encode map e97 with expected crc 2023-10-09T10:44:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:26 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1877099660' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:44:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:26 smithi155 ceph-mon[184854]: osdmap e97: 8 total, 8 up, 8 in 2023-10-09T10:44:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:26 smithi155 ceph-mon[184854]: failed to encode map e97 with expected crc 2023-10-09T10:44:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:26 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/498049491' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T10:44:26.739 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:26.765 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:26.765 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:26 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:44:26.765 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:26 smithi125 ceph-mon[195507]: failed to encode map e97 with expected crc 2023-10-09T10:44:26.765 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:26 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1877099660' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:44:26.766 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:26 smithi125 ceph-mon[195507]: osdmap e97: 8 total, 8 up, 8 in 2023-10-09T10:44:26.766 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:26 smithi125 ceph-mon[195507]: failed to encode map e97 with expected crc 2023-10-09T10:44:26.766 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:26 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/498049491' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T10:44:26.766 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:26 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:44:26.766 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:26 smithi125 ceph-mon[200197]: failed to encode map e97 with expected crc 2023-10-09T10:44:26.766 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:26 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1877099660' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:44:26.766 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:26 smithi125 ceph-mon[200197]: osdmap e97: 8 total, 8 up, 8 in 2023-10-09T10:44:26.766 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:26 smithi125 ceph-mon[200197]: failed to encode map e97 with expected crc 2023-10-09T10:44:26.767 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:26 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/498049491' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T10:44:26.768 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:26.993 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:26.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:26.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:26.997 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-10-09T10:44:27.097 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:27.129 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:27.151 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:27.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:27.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:27.254 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:27.281 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:27.302 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:27.550 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:27.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:27.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:27.626 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:27.654 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:27.682 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:27.697 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:44:27.697 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:44:27.698 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:44:27.698 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.490484 2023-10-09T10:44:27.698 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 0.49048385262288674 2023-10-09T10:44:27.700 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:27.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:27.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:27.807 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/498049491' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T10:44:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[195507]: osdmap e98: 8 total, 8 up, 8 in 2023-10-09T10:44:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[195507]: failed to encode map e97 with expected crc 2023-10-09T10:44:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[195507]: failed to encode map e97 with expected crc 2023-10-09T10:44:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[195507]: failed to encode map e98 with expected crc 2023-10-09T10:44:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[195507]: failed to encode map e98 with expected crc 2023-10-09T10:44:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[195507]: failed to encode map e98 with expected crc 2023-10-09T10:44:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[195507]: failed to encode map e98 with expected crc 2023-10-09T10:44:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[195507]: pgmap v196: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/498049491' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T10:44:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[200197]: osdmap e98: 8 total, 8 up, 8 in 2023-10-09T10:44:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[200197]: failed to encode map e97 with expected crc 2023-10-09T10:44:27.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[200197]: failed to encode map e97 with expected crc 2023-10-09T10:44:27.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[200197]: failed to encode map e98 with expected crc 2023-10-09T10:44:27.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[200197]: failed to encode map e98 with expected crc 2023-10-09T10:44:27.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[200197]: failed to encode map e98 with expected crc 2023-10-09T10:44:27.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[200197]: failed to encode map e98 with expected crc 2023-10-09T10:44:27.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:27 smithi125 ceph-mon[200197]: pgmap v196: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:27.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:27 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/498049491' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T10:44:27.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:27 smithi155 ceph-mon[184854]: osdmap e98: 8 total, 8 up, 8 in 2023-10-09T10:44:27.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:27 smithi155 ceph-mon[184854]: failed to encode map e97 with expected crc 2023-10-09T10:44:27.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:27 smithi155 ceph-mon[184854]: failed to encode map e97 with expected crc 2023-10-09T10:44:27.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:27 smithi155 ceph-mon[184854]: failed to encode map e98 with expected crc 2023-10-09T10:44:27.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:27 smithi155 ceph-mon[184854]: failed to encode map e98 with expected crc 2023-10-09T10:44:27.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:27 smithi155 ceph-mon[184854]: failed to encode map e98 with expected crc 2023-10-09T10:44:27.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:27 smithi155 ceph-mon[184854]: failed to encode map e98 with expected crc 2023-10-09T10:44:27.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:27 smithi155 ceph-mon[184854]: pgmap v196: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:27.840 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:27.864 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:27.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:27.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:27.968 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:27.998 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:28.018 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:28.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:28.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:28.062 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:44:28.122 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:28.153 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:28.174 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:28.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:28.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:28.277 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:28.306 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:28.324 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:28.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:28.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:28.380 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:44:28.380 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:44:28.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:28.455 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:28.474 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:28.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:28.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:28.578 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:28.608 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:28.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:28 smithi155 ceph-mon[184854]: failed to encode map e99 with expected crc 2023-10-09T10:44:28.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:28 smithi155 ceph-mon[184854]: failed to encode map e99 with expected crc 2023-10-09T10:44:28.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:28 smithi155 ceph-mon[184854]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T10:44:28.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:28 smithi155 ceph-mon[184854]: failed to encode map e99 with expected crc 2023-10-09T10:44:28.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:28 smithi155 ceph-mon[184854]: failed to encode map e99 with expected crc 2023-10-09T10:44:28.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:28 smithi155 ceph-mon[184854]: failed to encode map e99 with expected crc 2023-10-09T10:44:28.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:28 smithi125 ceph-mon[200197]: failed to encode map e99 with expected crc 2023-10-09T10:44:28.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:28 smithi125 ceph-mon[200197]: failed to encode map e99 with expected crc 2023-10-09T10:44:28.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:28 smithi125 ceph-mon[200197]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T10:44:28.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:28 smithi125 ceph-mon[200197]: failed to encode map e99 with expected crc 2023-10-09T10:44:28.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:28 smithi125 ceph-mon[200197]: failed to encode map e99 with expected crc 2023-10-09T10:44:28.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:28 smithi125 ceph-mon[200197]: failed to encode map e99 with expected crc 2023-10-09T10:44:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:28 smithi125 ceph-mon[195507]: failed to encode map e99 with expected crc 2023-10-09T10:44:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:28 smithi125 ceph-mon[195507]: failed to encode map e99 with expected crc 2023-10-09T10:44:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:28 smithi125 ceph-mon[195507]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T10:44:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:28 smithi125 ceph-mon[195507]: failed to encode map e99 with expected crc 2023-10-09T10:44:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:28 smithi125 ceph-mon[195507]: failed to encode map e99 with expected crc 2023-10-09T10:44:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:28 smithi125 ceph-mon[195507]: failed to encode map e99 with expected crc 2023-10-09T10:44:28.877 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:28.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:28.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:28.981 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:29.010 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:29.029 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:29.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:29.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:29.132 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:29.162 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:29.222 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:44:29.240 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:44:29.258 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:44:29.277 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:44:29.300 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:44:29.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:44:29.322 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:44:29.342 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:29.345 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:44:29.366 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:44:29.367 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:44:29.367 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:44:29.368 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:44:29.368 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:44:29.368 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:44:29.368 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:44:29.368 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:44:29.368 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:44:29.784 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:29.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:29 smithi125 ceph-mon[195507]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T10:44:29.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:29 smithi125 ceph-mon[195507]: failed to encode map e100 with expected crc 2023-10-09T10:44:29.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:29 smithi125 ceph-mon[195507]: pgmap v199: 89 pgs: 2 creating+peering, 11 unknown, 76 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:29.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:29 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3330158091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:29.786 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:29 smithi125 ceph-mon[200197]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T10:44:29.786 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:29 smithi125 ceph-mon[200197]: failed to encode map e100 with expected crc 2023-10-09T10:44:29.786 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:29 smithi125 ceph-mon[200197]: pgmap v199: 89 pgs: 2 creating+peering, 11 unknown, 76 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:29.787 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:29 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3330158091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:29.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:29.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:29.796 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:29.888 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:29.926 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:29 smithi155 ceph-mon[184854]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T10:44:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:29 smithi155 ceph-mon[184854]: failed to encode map e100 with expected crc 2023-10-09T10:44:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:29 smithi155 ceph-mon[184854]: pgmap v199: 89 pgs: 2 creating+peering, 11 unknown, 76 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:29 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3330158091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:30.589 INFO:teuthology.orchestra.run.smithi125.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-10-09T10:44:30.597 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:30.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:30.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:30.602 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:30.616 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:30 smithi125 ceph-mon[195507]: failed to encode map e100 with expected crc 2023-10-09T10:44:30.616 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:30 smithi125 ceph-mon[195507]: failed to encode map e100 with expected crc 2023-10-09T10:44:30.616 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:30 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3427640842' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:44:30.616 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:30 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:44:30.616 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:44:30.616 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:44:30.616 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:30 smithi125 ceph-mon[200197]: failed to encode map e100 with expected crc 2023-10-09T10:44:30.617 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:30 smithi125 ceph-mon[200197]: failed to encode map e100 with expected crc 2023-10-09T10:44:30.617 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:30 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3427640842' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:44:30.617 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:30 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:44:30.617 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:44:30.617 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:44:30.702 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:30.736 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:30.757 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:30.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:30.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:30.860 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:30.891 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:30 smithi155 ceph-mon[184854]: failed to encode map e100 with expected crc 2023-10-09T10:44:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:30 smithi155 ceph-mon[184854]: failed to encode map e100 with expected crc 2023-10-09T10:44:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:30 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3427640842' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:44:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:30 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:44:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:44:30.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:44:31.212 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:31.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:31.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:31.317 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:31.346 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:31.414 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:31.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:31.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:31.483 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:44:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:44:31] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2023-10-09T10:44:31.518 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:31.551 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:31.572 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:31.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:31.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:31.676 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:31.705 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:31.724 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:31.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:31.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:31.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:31.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:31 smithi125 ceph-mon[195507]: failed to encode map e101 with expected crc 2023-10-09T10:44:31.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:31 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:44:31.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:31 smithi125 ceph-mon[195507]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T10:44:31.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:31 smithi125 ceph-mon[195507]: pgmap v201: 89 pgs: 2 creating+peering, 2 unknown, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:31.843 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:31 smithi125 ceph-mon[200197]: failed to encode map e101 with expected crc 2023-10-09T10:44:31.843 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:31 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:44:31.843 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:31 smithi125 ceph-mon[200197]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T10:44:31.843 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:31 smithi125 ceph-mon[200197]: pgmap v201: 89 pgs: 2 creating+peering, 2 unknown, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:31.861 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:31.881 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:31.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:31.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:31 smithi155 ceph-mon[184854]: failed to encode map e101 with expected crc 2023-10-09T10:44:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:31 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:44:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:31 smithi155 ceph-mon[184854]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T10:44:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:31 smithi155 ceph-mon[184854]: pgmap v201: 89 pgs: 2 creating+peering, 2 unknown, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T10:44:31.986 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:32.014 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:32.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:32.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:32.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:32.089 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:44:32.089 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:44:32.089 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 0 2023-10-09T10:44:32.089 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:44:32.138 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:32.167 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:32.236 DEBUG:teuthology.orchestra.run.smithi125:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-10-09T10:44:32.239 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T10:44:32.239 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T10:44:32.286 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:hints = 1 2023-10-09T10:44:32.286 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T10:44:32.286 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object prefix: benchmark_data_smithi125_244951 2023-10-09T10:44:32.286 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:44:32.287 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 0 0 0 0 0 0 - 0 2023-10-09T10:44:32.373 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:44:32.487 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:32.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:32.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:32.512 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:44:32.597 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:32.612 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:44:32.612 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.7 primary-affinity to 0.490484 (8321442) 2023-10-09T10:44:32.659 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:32.683 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:32.685 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:32 smithi125 ceph-mon[200197]: failed to encode map e101 with expected crc 2023-10-09T10:44:32.685 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:32 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2194925419' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.49048385262288674}]: dispatch 2023-10-09T10:44:32.686 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:32 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2236972651' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:32.686 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:32 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:32.686 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:32 smithi125 ceph-mon[195507]: failed to encode map e101 with expected crc 2023-10-09T10:44:32.686 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:32 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2194925419' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.49048385262288674}]: dispatch 2023-10-09T10:44:32.686 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:32 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2236972651' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:32.686 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:32 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:32.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:32.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:32.788 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:32.816 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:32.834 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:32.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:32.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:32.937 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:32.965 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:32 smithi155 ceph-mon[184854]: failed to encode map e101 with expected crc 2023-10-09T10:44:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:32 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2194925419' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.49048385262288674}]: dispatch 2023-10-09T10:44:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:32 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2236972651' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:32 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:33.134 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:33.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:33.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:33.241 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:33.270 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:33.286 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 1 16 1184 1168 72.9927 73 0.00115787 0.00492173 2023-10-09T10:44:33.289 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:33.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:33.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:33.335 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:33.393 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:33.421 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:33.440 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:33.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:33.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:33.543 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:33.571 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:33.590 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:33.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:33.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:33.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:33.736 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:33.745 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:33.827 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:33.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:33.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:33.933 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:33 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:44:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:33 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2194925419' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.49048385262288674}]': finished 2023-10-09T10:44:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:33 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:44:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:33 smithi155 ceph-mon[184854]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T10:44:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:33 smithi155 ceph-mon[184854]: failed to encode map e102 with expected crc 2023-10-09T10:44:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:33 smithi155 ceph-mon[184854]: failed to encode map e102 with expected crc 2023-10-09T10:44:33.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:33 smithi155 ceph-mon[184854]: failed to encode map e102 with expected crc 2023-10-09T10:44:33.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:33 smithi155 ceph-mon[184854]: failed to encode map e102 with expected crc 2023-10-09T10:44:33.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:33 smithi155 ceph-mon[184854]: failed to encode map e102 with expected crc 2023-10-09T10:44:33.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:33 smithi155 ceph-mon[184854]: failed to encode map e102 with expected crc 2023-10-09T10:44:33.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:33 smithi155 ceph-mon[184854]: failed to encode map e102 with expected crc 2023-10-09T10:44:33.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:33 smithi155 ceph-mon[184854]: pgmap v203: 89 pgs: 89 active+clean; 6.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s wr, 18 op/s 2023-10-09T10:44:33.976 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:34.002 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:34.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:34.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:34.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:44:34.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2194925419' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.49048385262288674}]': finished 2023-10-09T10:44:34.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:44:34.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[195507]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T10:44:34.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[195507]: failed to encode map e102 with expected crc 2023-10-09T10:44:34.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[195507]: failed to encode map e102 with expected crc 2023-10-09T10:44:34.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[195507]: failed to encode map e102 with expected crc 2023-10-09T10:44:34.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[195507]: failed to encode map e102 with expected crc 2023-10-09T10:44:34.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[195507]: failed to encode map e102 with expected crc 2023-10-09T10:44:34.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[195507]: failed to encode map e102 with expected crc 2023-10-09T10:44:34.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[195507]: failed to encode map e102 with expected crc 2023-10-09T10:44:34.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[195507]: pgmap v203: 89 pgs: 89 active+clean; 6.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s wr, 18 op/s 2023-10-09T10:44:34.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:44:34.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2194925419' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.49048385262288674}]': finished 2023-10-09T10:44:34.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:44:34.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[200197]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T10:44:34.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[200197]: failed to encode map e102 with expected crc 2023-10-09T10:44:34.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[200197]: failed to encode map e102 with expected crc 2023-10-09T10:44:34.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[200197]: failed to encode map e102 with expected crc 2023-10-09T10:44:34.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[200197]: failed to encode map e102 with expected crc 2023-10-09T10:44:34.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[200197]: failed to encode map e102 with expected crc 2023-10-09T10:44:34.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[200197]: failed to encode map e102 with expected crc 2023-10-09T10:44:34.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[200197]: failed to encode map e102 with expected crc 2023-10-09T10:44:34.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:33 smithi125 ceph-mon[200197]: pgmap v203: 89 pgs: 89 active+clean; 6.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s wr, 18 op/s 2023-10-09T10:44:34.107 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:34.158 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:34.287 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 2 16 3351 3335 104.175 135.438 0.00288121 0.00958257 2023-10-09T10:44:34.493 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:34.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:34.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:34.598 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:34.658 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:34.799 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:44:34.834 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:44:34.874 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:44:34.880 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:34 smithi125 ceph-mon[195507]: failed to encode map e103 with expected crc 2023-10-09T10:44:34.880 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:34 smithi125 ceph-mon[195507]: failed to encode map e103 with expected crc 2023-10-09T10:44:34.880 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:34 smithi125 ceph-mon[195507]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T10:44:34.880 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:34 smithi125 ceph-mon[195507]: failed to encode map e103 with expected crc 2023-10-09T10:44:34.880 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:34 smithi125 ceph-mon[195507]: failed to encode map e103 with expected crc 2023-10-09T10:44:34.880 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:34 smithi125 ceph-mon[195507]: failed to encode map e103 with expected crc 2023-10-09T10:44:34.881 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:34 smithi125 ceph-mon[200197]: failed to encode map e103 with expected crc 2023-10-09T10:44:34.881 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:34 smithi125 ceph-mon[200197]: failed to encode map e103 with expected crc 2023-10-09T10:44:34.881 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:34 smithi125 ceph-mon[200197]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T10:44:34.881 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:34 smithi125 ceph-mon[200197]: failed to encode map e103 with expected crc 2023-10-09T10:44:34.881 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:34 smithi125 ceph-mon[200197]: failed to encode map e103 with expected crc 2023-10-09T10:44:34.881 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:34 smithi125 ceph-mon[200197]: failed to encode map e103 with expected crc 2023-10-09T10:44:34.888 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:34.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:34.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:34.925 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:44:34.965 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:44:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:34 smithi155 ceph-mon[184854]: failed to encode map e103 with expected crc 2023-10-09T10:44:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:34 smithi155 ceph-mon[184854]: failed to encode map e103 with expected crc 2023-10-09T10:44:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:34 smithi155 ceph-mon[184854]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T10:44:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:34 smithi155 ceph-mon[184854]: failed to encode map e103 with expected crc 2023-10-09T10:44:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:34 smithi155 ceph-mon[184854]: failed to encode map e103 with expected crc 2023-10-09T10:44:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:34 smithi155 ceph-mon[184854]: failed to encode map e103 with expected crc 2023-10-09T10:44:34.992 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:44:35.000 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:35.023 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:44:35.039 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:35.060 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:44:35.063 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:35.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:35.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:35.074 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:44:35.074 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:44:35.074 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:44:35.074 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:44:35.074 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:44:35.075 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:44:35.075 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:44:35.075 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:44:35.117 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:44:35.168 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:35.213 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:35.287 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 3 16 6015 5999 124.927 166.5 0.0078063 0.00799357 2023-10-09T10:44:35.340 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:35.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:35.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:35.450 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:44:35.477 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:44:35.477 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:44:35.487 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:35.515 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:44:35.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:35.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:44:35.618 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:35.667 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:35.768 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:35 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2894513622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:35.768 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:35 smithi125 ceph-mon[195507]: pgmap v205: 89 pgs: 1 peering, 88 active+clean; 55 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s wr, 143 op/s 2023-10-09T10:44:35.768 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:35 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2894513622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:35.768 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:35 smithi125 ceph-mon[200197]: pgmap v205: 89 pgs: 1 peering, 88 active+clean; 55 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s wr, 143 op/s 2023-10-09T10:44:35.950 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:35.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:35.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:35 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2894513622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:35 smithi155 ceph-mon[184854]: pgmap v205: 89 pgs: 1 peering, 88 active+clean; 55 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s wr, 143 op/s 2023-10-09T10:44:36.024 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:36.055 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:44:36.093 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:36.107 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:44:36.126 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:44:36.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:36.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:44:36.230 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:36.263 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:36.287 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 4 16 8624 8608 134.454 163.062 0.00590572 0.00742877 2023-10-09T10:44:36.289 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:36.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:36.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:36.394 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:36.459 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:36.731 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:36 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:44:36.732 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:36 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:44:36.797 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:36.840 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:36.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:36.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:36.948 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:36.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:36 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:44:36.995 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:37.288 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 5 16 11180 11164 139.492 159.75 0.00407447 0.00716247 2023-10-09T10:44:37.375 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:37.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:37.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:37.479 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:37.546 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:37.596 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:37.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:37.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:37.650 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:44:37.655 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:37 smithi125 ceph-mon[195507]: pgmap v206: 89 pgs: 1 peering, 88 active+clean; 336 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s 2023-10-09T10:44:37.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:37 smithi125 ceph-mon[200197]: pgmap v206: 89 pgs: 1 peering, 88 active+clean; 336 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s 2023-10-09T10:44:37.702 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:37.753 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:37.835 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:37.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:37.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:37.942 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:37.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:37 smithi155 ceph-mon[184854]: pgmap v206: 89 pgs: 1 peering, 88 active+clean; 336 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s 2023-10-09T10:44:37.982 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:38.006 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:38.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:38.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:38.109 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:38.110 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:44:38.110 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:44:38.116 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:38.173 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:38.276 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:44:38.277 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:44:38.277 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2023-10-09T10:44:38.277 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:44:38.288 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 6 16 13266 13250 137.969 130.375 0.00567158 0.00724258 2023-10-09T10:44:38.351 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:38.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:38.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:38.456 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:38.492 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:38.519 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:38.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:38.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:38.625 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:38.668 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:38.676 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:44:38.786 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:44:38.848 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:38.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:38.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:38.962 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:38.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:38 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/964601958' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:44:38.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:38 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:44:39.001 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:39.010 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:38 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/964601958' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:44:39.010 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:38 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:44:39.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:38 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/964601958' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:44:39.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:38 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:44:39.024 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:39.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:39.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:39.129 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:39.162 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:39.194 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:39.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:39.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:39.288 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 7 16 15494 15478 138.151 139.25 0.00190035 0.0072279 2023-10-09T10:44:39.300 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:39.356 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:39.384 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:39.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:39.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:39.489 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:44:39.528 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:39.559 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:44:39.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:39.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:44:39.663 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:39.682 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:44:39.704 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:39.720 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:44:39.735 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:39.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:39.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:39.749 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:39.770 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:44:39.800 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:39.808 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:44:39.839 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:40.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:39 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:44:40.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:39 smithi155 ceph-mon[184854]: failed to encode map e104 with expected crc 2023-10-09T10:44:40.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:39 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:44:40.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:39 smithi155 ceph-mon[184854]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T10:44:40.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:39 smithi155 ceph-mon[184854]: failed to encode map e104 with expected crc 2023-10-09T10:44:40.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:39 smithi155 ceph-mon[184854]: pgmap v208: 89 pgs: 89 active+clean; 572 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T10:44:40.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:39 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2563297315' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:40.247 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:44:40.247 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:44:40.247 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:44:40.247 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:39 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:44:40.247 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:39 smithi125 ceph-mon[200197]: failed to encode map e104 with expected crc 2023-10-09T10:44:40.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:39 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:44:40.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:39 smithi125 ceph-mon[200197]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T10:44:40.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:39 smithi125 ceph-mon[200197]: failed to encode map e104 with expected crc 2023-10-09T10:44:40.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:39 smithi125 ceph-mon[200197]: pgmap v208: 89 pgs: 89 active+clean; 572 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T10:44:40.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:39 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2563297315' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:40.248 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:39 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:44:40.248 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:39 smithi125 ceph-mon[195507]: failed to encode map e104 with expected crc 2023-10-09T10:44:40.248 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:39 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:44:40.248 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:39 smithi125 ceph-mon[195507]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T10:44:40.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:39 smithi125 ceph-mon[195507]: failed to encode map e104 with expected crc 2023-10-09T10:44:40.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:39 smithi125 ceph-mon[195507]: pgmap v208: 89 pgs: 89 active+clean; 572 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T10:44:40.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:39 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2563297315' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:40.249 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:44:40.249 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:44:40.249 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:44:40.249 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:44:40.250 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:44:40.250 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:44:40.250 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:44:40.250 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:44:40.250 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:44:40.289 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 8 16 17638 17622 137.63 134 0.0099554 0.00725918 2023-10-09T10:44:40.306 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:40.738 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:40.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:40.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:40.862 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:40.878 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:40 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:44:40.878 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:40 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:44:40.879 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:40 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:44:40.880 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:40 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:44:40.898 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:40.925 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:40.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:40.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:40.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:40 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:44:40.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:40 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:44:41.030 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:41.069 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:41.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:41.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:41.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:41.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:41.218 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:44:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:44:41] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2023-10-09T10:44:41.262 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:41.288 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 9 16 19867 19851 137.816 139.312 0.0280591 0.0072434 2023-10-09T10:44:41.298 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:44:41.389 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:41.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:41.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:41.495 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:41.578 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:41.633 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:44:41.633 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:44:41.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.633 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:20396 6.8 6:109bd17d:::benchmark_data_smithi125_244951_object20395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:44:41.634 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.611714+0000", 2023-10-09T10:44:41.634 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.013565499999999999, 2023-10-09T10:44:41.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013607391, 2023-10-09T10:44:41.634 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.634 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:44:41.634 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:41.634 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:41.634 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:41.634 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 20396 2023-10-09T10:44:41.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.635 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.611714+0000", 2023-10-09T10:44:41.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.611714+0000", 2023-10-09T10:44:41.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.611712+0000", 2023-10-09T10:44:41.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:44:41.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.611733+0000", 2023-10-09T10:44:41.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0866e-05 2023-10-09T10:44:41.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.611749+0000", 2023-10-09T10:44:41.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5451000000000002e-05 2023-10-09T10:44:41.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.611757+0000", 2023-10-09T10:44:41.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2009999999999999e-06 2023-10-09T10:44:41.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:41.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.613456+0000", 2023-10-09T10:44:41.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00169921 2023-10-09T10:44:41.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.638 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:44:41.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.613580+0000", 2023-10-09T10:44:41.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000123981 2023-10-09T10:44:41.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.638 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:44:41.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.613661+0000", 2023-10-09T10:44:41.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1112000000000005e-05 2023-10-09T10:44:41.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.638 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:44:41.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.618501+0000", 2023-10-09T10:44:41.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0048401909999999998 2023-10-09T10:44:41.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:44:41.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.618603+0000", 2023-10-09T10:44:41.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010199100000000001 2023-10-09T10:44:41.639 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.639 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.639 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.639 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:20403 6.f 6:f0e47c5c:::benchmark_data_smithi125_244951_object20402:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:44:41.640 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.618267+0000", 2023-10-09T10:44:41.640 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0070121580000000001, 2023-10-09T10:44:41.640 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0071652089999999996, 2023-10-09T10:44:41.640 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.640 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:44:41.640 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:41.640 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:41.640 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:41.640 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 20403 2023-10-09T10:44:41.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.640 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.640 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.618267+0000", 2023-10-09T10:44:41.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.641 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.618267+0000", 2023-10-09T10:44:41.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.641 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.618265+0000", 2023-10-09T10:44:41.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T10:44:41.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.642 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.642 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.618293+0000", 2023-10-09T10:44:41.642 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8014999999999999e-05 2023-10-09T10:44:41.642 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.642 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.642 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.642 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.618310+0000", 2023-10-09T10:44:41.642 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.677e-05 2023-10-09T10:44:41.642 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.642 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.642 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.618318+0000", 2023-10-09T10:44:41.643 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.456e-06 2023-10-09T10:44:41.643 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.643 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.643 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:41.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.620219+0000", 2023-10-09T10:44:41.643 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001901163 2023-10-09T10:44:41.643 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.643 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.643 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:44:41.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.620376+0000", 2023-10-09T10:44:41.643 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015683 2023-10-09T10:44:41.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:44:41.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.620532+0000", 2023-10-09T10:44:41.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000156031 2023-10-09T10:44:41.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:44:41.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623427+0000", 2023-10-09T10:44:41.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002895042 2023-10-09T10:44:41.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:44:41.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623435+0000", 2023-10-09T10:44:41.645 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2179999999999999e-06 2023-10-09T10:44:41.645 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.645 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.645 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.645 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.645 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.645 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:20405 6.8 6:1721b5a4:::benchmark_data_smithi125_244951_object20404:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:44:41.645 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.620486+0000", 2023-10-09T10:44:41.645 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0047931839999999998, 2023-10-09T10:44:41.645 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0051419539999999998, 2023-10-09T10:44:41.645 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.645 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:44:41.646 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:41.646 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:41.646 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:41.646 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 20405 2023-10-09T10:44:41.646 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.646 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.646 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.646 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.646 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.620486+0000", 2023-10-09T10:44:41.646 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.646 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.646 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.647 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.647 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.620486+0000", 2023-10-09T10:44:41.647 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.647 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.647 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.647 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.647 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.620485+0000", 2023-10-09T10:44:41.647 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:44:41.647 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.647 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.647 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.647 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.620528+0000", 2023-10-09T10:44:41.648 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3377999999999997e-05 2023-10-09T10:44:41.648 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.648 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.648 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.648 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.620538+0000", 2023-10-09T10:44:41.648 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6719999999999999e-06 2023-10-09T10:44:41.648 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.648 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.648 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.648 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.620544+0000", 2023-10-09T10:44:41.648 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9070000000000004e-06 2023-10-09T10:44:41.648 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:41.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623470+0000", 2023-10-09T10:44:41.649 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029264970000000001 2023-10-09T10:44:41.649 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:44:41.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623596+0000", 2023-10-09T10:44:41.649 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012571899999999999 2023-10-09T10:44:41.649 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:44:41.650 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623742+0000", 2023-10-09T10:44:41.650 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014586300000000001 2023-10-09T10:44:41.650 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.650 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.650 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.650 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.650 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.650 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44570.0:20411 6.2 e104/98 6:4651cdb0:::benchmark_data_smithi125_244951_object20410:head v 104'1267, mlcod=104'1265)", 2023-10-09T10:44:41.650 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.621056+0000", 2023-10-09T10:44:41.650 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0042230230000000002, 2023-10-09T10:44:41.650 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046550419999999999, 2023-10-09T10:44:41.650 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.651 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:44:41.651 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.651 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.651 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.651 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.621056+0000", 2023-10-09T10:44:41.651 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.651 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.651 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.651 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.651 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.621056+0000", 2023-10-09T10:44:41.651 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.651 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.651 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.652 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.652 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.621056+0000", 2023-10-09T10:44:41.652 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:44:41.652 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.652 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.652 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.652 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.622153+0000", 2023-10-09T10:44:41.652 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001097722 2023-10-09T10:44:41.652 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.652 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.652 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.652 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.622170+0000", 2023-10-09T10:44:41.653 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6540000000000001e-05 2023-10-09T10:44:41.653 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.653 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.653 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.653 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.622177+0000", 2023-10-09T10:44:41.653 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5460000000000003e-06 2023-10-09T10:44:41.653 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.653 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.653 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:41.653 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.622582+0000", 2023-10-09T10:44:41.653 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00040445299999999998 2023-10-09T10:44:41.653 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.654 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.654 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:44:41.654 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.622603+0000", 2023-10-09T10:44:41.654 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1055000000000001e-05 2023-10-09T10:44:41.654 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.654 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.654 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.654 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.654 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.654 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44570.0:20413 6.3 e104/98 6:c175ae6e:::benchmark_data_smithi125_244951_object20412:head v 104'1268, mlcod=104'1265)", 2023-10-09T10:44:41.654 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.623085+0000", 2023-10-09T10:44:41.654 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0021947569999999999, 2023-10-09T10:44:41.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026921789999999998, 2023-10-09T10:44:41.655 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.655 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:44:41.655 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.655 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.655 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623085+0000", 2023-10-09T10:44:41.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.655 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.655 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.655 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623085+0000", 2023-10-09T10:44:41.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.656 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.656 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.656 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.656 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623084+0000", 2023-10-09T10:44:41.656 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:41.656 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.656 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.656 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.656 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623120+0000", 2023-10-09T10:44:41.656 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6267999999999999e-05 2023-10-09T10:44:41.656 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.656 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.657 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623132+0000", 2023-10-09T10:44:41.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2072000000000001e-05 2023-10-09T10:44:41.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.657 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623140+0000", 2023-10-09T10:44:41.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6969999999999993e-06 2023-10-09T10:44:41.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.657 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:41.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.625464+0000", 2023-10-09T10:44:41.658 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023236400000000001 2023-10-09T10:44:41.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.658 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:44:41.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.625478+0000", 2023-10-09T10:44:41.658 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4207e-05 2023-10-09T10:44:41.658 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.658 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.658 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.659 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:20406 6.8 6:17c401ce:::benchmark_data_smithi125_244951_object20405:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:44:41.659 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.623158+0000", 2023-10-09T10:44:41.659 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0021214910000000001, 2023-10-09T10:44:41.659 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026895629999999998, 2023-10-09T10:44:41.659 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.659 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:44:41.659 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:41.659 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:41.659 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:41.659 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 20406 2023-10-09T10:44:41.659 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.659 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.659 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.660 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.660 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623158+0000", 2023-10-09T10:44:41.660 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.660 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.660 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.660 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.660 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623158+0000", 2023-10-09T10:44:41.660 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.660 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.660 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.660 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.660 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623156+0000", 2023-10-09T10:44:41.661 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:44:41.661 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.661 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.661 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.661 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623200+0000", 2023-10-09T10:44:41.661 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3980999999999998e-05 2023-10-09T10:44:41.661 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.661 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.661 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.661 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623217+0000", 2023-10-09T10:44:41.661 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6299000000000001e-05 2023-10-09T10:44:41.662 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.662 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.662 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.662 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623223+0000", 2023-10-09T10:44:41.662 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5509999999999996e-06 2023-10-09T10:44:41.662 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.662 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.662 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:41.662 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.624545+0000", 2023-10-09T10:44:41.662 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013217470000000001 2023-10-09T10:44:41.662 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.662 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.663 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:44:41.663 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.624672+0000", 2023-10-09T10:44:41.663 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012714000000000001 2023-10-09T10:44:41.663 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.663 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.663 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:44:41.663 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.624804+0000", 2023-10-09T10:44:41.663 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000132175 2023-10-09T10:44:41.663 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.663 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.663 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.663 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.664 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.664 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:20409 6.8 6:1407d6f6:::benchmark_data_smithi125_244951_object20408:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:44:41.664 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.623229+0000", 2023-10-09T10:44:41.664 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0020501249999999999, 2023-10-09T10:44:41.664 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027041669999999999, 2023-10-09T10:44:41.664 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.664 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:44:41.664 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:41.664 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:41.665 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:41.665 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 20409 2023-10-09T10:44:41.665 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.665 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.665 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.665 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.665 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623229+0000", 2023-10-09T10:44:41.665 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.665 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.665 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.665 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.665 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623229+0000", 2023-10-09T10:44:41.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.666 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.666 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.666 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.666 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623229+0000", 2023-10-09T10:44:41.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:41.666 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.666 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.666 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.666 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623254+0000", 2023-10-09T10:44:41.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5318999999999999e-05 2023-10-09T10:44:41.666 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.667 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.667 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.667 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623261+0000", 2023-10-09T10:44:41.667 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2960000000000001e-06 2023-10-09T10:44:41.667 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.667 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.667 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.667 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623266+0000", 2023-10-09T10:44:41.667 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1340000000000003e-06 2023-10-09T10:44:41.667 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.667 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.668 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:41.668 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.625318+0000", 2023-10-09T10:44:41.668 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020517500000000002 2023-10-09T10:44:41.668 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.668 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.668 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:44:41.668 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.625406+0000", 2023-10-09T10:44:41.668 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7607000000000003e-05 2023-10-09T10:44:41.668 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.668 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.668 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:44:41.668 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.625509+0000", 2023-10-09T10:44:41.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010370999999999999 2023-10-09T10:44:41.669 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.669 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.669 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.669 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:20412 6.8 6:1abbbdde:::benchmark_data_smithi125_244951_object20411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:44:41.669 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.623271+0000", 2023-10-09T10:44:41.669 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0020084299999999999, 2023-10-09T10:44:41.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027531090000000001, 2023-10-09T10:44:41.669 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.669 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:44:41.670 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:41.670 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:41.670 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:41.670 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 20412 2023-10-09T10:44:41.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.670 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.670 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.670 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623271+0000", 2023-10-09T10:44:41.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623271+0000", 2023-10-09T10:44:41.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.671 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623270+0000", 2023-10-09T10:44:41.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:41.671 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623296+0000", 2023-10-09T10:44:41.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5641999999999999e-05 2023-10-09T10:44:41.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623303+0000", 2023-10-09T10:44:41.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2919999999999998e-06 2023-10-09T10:44:41.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623309+0000", 2023-10-09T10:44:41.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5679999999999999e-06 2023-10-09T10:44:41.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.673 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.673 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:41.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.625974+0000", 2023-10-09T10:44:41.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026650580000000001 2023-10-09T10:44:41.673 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.673 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.673 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.673 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.673 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.673 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44570.0:20414 6.9 e104/98 6:9b3d6238:::benchmark_data_smithi125_244951_object20413:head v 104'1238, mlcod=104'1236)", 2023-10-09T10:44:41.673 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.623330+0000", 2023-10-09T10:44:41.674 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0019495339999999999, 2023-10-09T10:44:41.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028226890000000002, 2023-10-09T10:44:41.674 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.674 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:44:41.674 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.674 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.674 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623330+0000", 2023-10-09T10:44:41.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.674 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623330+0000", 2023-10-09T10:44:41.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.675 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.675 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623329+0000", 2023-10-09T10:44:41.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:41.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.675 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.675 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623347+0000", 2023-10-09T10:44:41.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7269e-05 2023-10-09T10:44:41.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.676 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.676 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623354+0000", 2023-10-09T10:44:41.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1579999999999999e-06 2023-10-09T10:44:41.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.676 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.676 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623359+0000", 2023-10-09T10:44:41.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9099999999999996e-06 2023-10-09T10:44:41.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.677 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:41.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623393+0000", 2023-10-09T10:44:41.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4541e-05 2023-10-09T10:44:41.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.677 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:44:41.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623416+0000", 2023-10-09T10:44:41.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2248000000000001e-05 2023-10-09T10:44:41.677 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.677 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.677 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.678 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.678 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44570.0:20415 6.e e104/98 6:7fcc73e5:::benchmark_data_smithi125_244951_object20414:head v 104'1235, mlcod=104'1233)", 2023-10-09T10:44:41.678 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.623818+0000", 2023-10-09T10:44:41.678 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.001460958, 2023-10-09T10:44:41.678 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043828000000000001, 2023-10-09T10:44:41.678 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.678 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:44:41.678 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.678 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.678 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623818+0000", 2023-10-09T10:44:41.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623818+0000", 2023-10-09T10:44:41.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623818+0000", 2023-10-09T10:44:41.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:41.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623850+0000", 2023-10-09T10:44:41.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2431000000000002e-05 2023-10-09T10:44:41.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623860+0000", 2023-10-09T10:44:41.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9510000000000001e-06 2023-10-09T10:44:41.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623867+0000", 2023-10-09T10:44:41.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.438e-06 2023-10-09T10:44:41.681 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.681 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:41.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623910+0000", 2023-10-09T10:44:41.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2981000000000001e-05 2023-10-09T10:44:41.681 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.681 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:44:41.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.623931+0000", 2023-10-09T10:44:41.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1469999999999999e-05 2023-10-09T10:44:41.681 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.682 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.682 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.682 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.44570.0:20396 6.8 e104/102)", 2023-10-09T10:44:41.682 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.626263+0000", 2023-10-09T10:44:41.682 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4294967295.9990168, 2023-10-09T10:44:41.682 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020221879999999999, 2023-10-09T10:44:41.682 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.682 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:44:41.682 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626263+0000", 2023-10-09T10:44:41.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626263+0000", 2023-10-09T10:44:41.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626262+0000", 2023-10-09T10:44:41.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:41.684 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.684 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.684 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.684 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626267+0000", 2023-10-09T10:44:41.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0520000000000001e-06 2023-10-09T10:44:41.684 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.684 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.684 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.684 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626278+0000", 2023-10-09T10:44:41.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0652999999999999e-05 2023-10-09T10:44:41.684 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.685 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.685 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626289+0000", 2023-10-09T10:44:41.685 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1715e-05 2023-10-09T10:44:41.685 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.685 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.685 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.685 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.685 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.44570.0:20405 6.8 e104/102)", 2023-10-09T10:44:41.685 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.626869+0000", 2023-10-09T10:44:41.685 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4294967295.9984102, 2023-10-09T10:44:41.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001470205, 2023-10-09T10:44:41.686 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.686 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:44:41.686 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626869+0000", 2023-10-09T10:44:41.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626869+0000", 2023-10-09T10:44:41.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626869+0000", 2023-10-09T10:44:41.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:41.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626873+0000", 2023-10-09T10:44:41.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1500000000000001e-06 2023-10-09T10:44:41.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.688 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626881+0000", 2023-10-09T10:44:41.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3410000000000003e-06 2023-10-09T10:44:41.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.688 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626887+0000", 2023-10-09T10:44:41.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9989999999999997e-06 2023-10-09T10:44:41.688 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.688 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.688 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.689 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.689 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.44570.0:20406 6.8 e104/102)", 2023-10-09T10:44:41.689 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.626895+0000", 2023-10-09T10:44:41.689 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4294967295.998384, 2023-10-09T10:44:41.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014994170000000001, 2023-10-09T10:44:41.689 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.689 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:44:41.689 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.689 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.689 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626895+0000", 2023-10-09T10:44:41.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626895+0000", 2023-10-09T10:44:41.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626895+0000", 2023-10-09T10:44:41.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:41.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626898+0000", 2023-10-09T10:44:41.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5349999999999999e-06 2023-10-09T10:44:41.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626904+0000", 2023-10-09T10:44:41.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5599999999999999e-06 2023-10-09T10:44:41.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626909+0000", 2023-10-09T10:44:41.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3730000000000003e-06 2023-10-09T10:44:41.692 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.692 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.692 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.692 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.44570.0:20409 6.8 e104/102)", 2023-10-09T10:44:41.692 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.626940+0000", 2023-10-09T10:44:41.692 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4294967295.9983397, 2023-10-09T10:44:41.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001518096, 2023-10-09T10:44:41.692 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.692 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:44:41.692 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626940+0000", 2023-10-09T10:44:41.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626940+0000", 2023-10-09T10:44:41.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626939+0000", 2023-10-09T10:44:41.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:41.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626943+0000", 2023-10-09T10:44:41.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2880000000000001e-06 2023-10-09T10:44:41.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626953+0000", 2023-10-09T10:44:41.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0219e-05 2023-10-09T10:44:41.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.626958+0000", 2023-10-09T10:44:41.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5130000000000004e-06 2023-10-09T10:44:41.695 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.695 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.695 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.695 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.695 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.44570.0:20403 6.f e104/98)", 2023-10-09T10:44:41.695 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.627127+0000", 2023-10-09T10:44:41.695 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4294967295.9981527, 2023-10-09T10:44:41.696 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013922470000000001, 2023-10-09T10:44:41.696 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.696 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:44:41.696 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.696 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.696 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.696 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.627127+0000", 2023-10-09T10:44:41.696 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.696 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.696 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.696 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.696 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.627127+0000", 2023-10-09T10:44:41.696 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.697 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.697 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.697 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.697 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.627126+0000", 2023-10-09T10:44:41.697 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:41.697 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.697 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.697 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.697 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.627130+0000", 2023-10-09T10:44:41.697 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9060000000000004e-06 2023-10-09T10:44:41.697 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.697 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.698 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.698 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.627141+0000", 2023-10-09T10:44:41.698 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1148999999999999e-05 2023-10-09T10:44:41.698 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.698 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.698 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.698 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.627147+0000", 2023-10-09T10:44:41.698 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.964e-06 2023-10-09T10:44:41.698 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.698 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.698 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.699 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.699 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.699 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44570.0:20418 6.2 e104/98)", 2023-10-09T10:44:41.699 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:41.628072+0000", 2023-10-09T10:44:41.699 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4294967295.9972076, 2023-10-09T10:44:41.699 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00052621199999999997, 2023-10-09T10:44:41.699 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:41.699 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:44:41.699 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:41.699 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.699 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:41.699 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.628072+0000", 2023-10-09T10:44:41.699 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:41.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.628072+0000", 2023-10-09T10:44:41.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:41.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:41.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.628071+0000", 2023-10-09T10:44:41.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:44:41.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.701 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:41.701 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.628100+0000", 2023-10-09T10:44:41.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.955e-05 2023-10-09T10:44:41.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.701 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:41.701 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.628109+0000", 2023-10-09T10:44:41.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6840000000000002e-06 2023-10-09T10:44:41.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:41.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:41.701 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:41.702 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:41.628114+0000", 2023-10-09T10:44:41.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0159999999999999e-06 2023-10-09T10:44:41.702 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.702 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:41.702 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.702 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:41.702 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:44:41.702 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 16 2023-10-09T10:44:41.702 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:44:41.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:41.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:41.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:41.804 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:41.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:41 smithi125 ceph-mon[200197]: pgmap v209: 89 pgs: 89 active+clean; 978 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T10:44:41.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:41 smithi125 ceph-mon[195507]: pgmap v209: 89 pgs: 89 active+clean; 978 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T10:44:41.849 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:41.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:41 smithi155 ceph-mon[184854]: pgmap v209: 89 pgs: 89 active+clean; 978 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T10:44:42.040 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:42.041 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:42.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:42.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:42.063 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:44:42.134 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:44:42.147 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:42.148 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:44:42.179 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:42.202 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:42.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:42.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:42.290 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 10 16 21573 21557 134.688 106.625 0.0114055 0.00741748 2023-10-09T10:44:42.307 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:42.344 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:42.374 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:42.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:42.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:42.478 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:42.521 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:42.548 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:42.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:42.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:42.652 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:44:42.693 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:42.703 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:44:42.719 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:44:42.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:42.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:44:42.757 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":209,"stamp":"2023-10-09T10:44:41.011284+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1025578912,"num_objects":15664,"num_object_clones":0,"num_object_copies":46992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15664,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":15940,"num_write_kb":1003405,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":15856,"ondisk_log_size":15856,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4061900,"kb_used_data":2709452,"kb_used_omap":126,"kb_used_meta":1352257,"kb_avail":745899316,"statfs":{"total":767960285184,"available":763800899584,"internally_reserved":0,"allocated":2774478848,"data_stored":2770706141,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1384711803},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1018363904,"num_objects":15539,"num_object_clones":0,"num_object_copies":46617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15539,"num_whiteouts":0,"num_read":-14,"num_read_kb":-12,"num_write":15539,"num_write_kb":994496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003446"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":87,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.616355+0000","last_change":"2023-10-09T10:43:06.676414+0000","last_active":"2023-10-09T10:44:33.616355+0000","last_peered":"2023-10-09T10:44:33.616355+0000","last_clean":"2023-10-09T10:44:33.616355+0000","last_became_active":"2023-10-09T10:43:06.676050+0000","last_became_peered":"2023-10-09T10:43:06.676050+0000","last_unstale":"2023-10-09T10:44:33.616355+0000","last_undegraded":"2023-10-09T10:44:33.616355+0000","last_fullsized":"2023-10-09T10:44:33.616355+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:28.011477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":70,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.635900+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:44:32.635900+0000","last_peered":"2023-10-09T10:44:32.635900+0000","last_clean":"2023-10-09T10:44:32.635900+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:44:32.635900+0000","last_undegraded":"2023-10-09T10:44:32.635900+0000","last_fullsized":"2023-10-09T10:44:32.635900+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":53,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.618601+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:44:32.618601+0000","last_peered":"2023-10-09T10:44:32.618601+0000","last_clean":"2023-10-09T10:44:32.618601+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:44:32.618601+0000","last_undegraded":"2023-10-09T10:44:32.618601+0000","last_fullsized":"2023-10-09T10:44:32.618601+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":59,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.607405+0000","last_change":"2023-10-09T10:43:35.487224+0000","last_active":"2023-10-09T10:44:32.607405+0000","last_peered":"2023-10-09T10:44:32.607405+0000","last_clean":"2023-10-09T10:44:32.607405+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:44:32.607405+0000","last_undegraded":"2023-10-09T10:44:32.607405+0000","last_fullsized":"2023-10-09T10:44:32.607405+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:44:52.066778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":88,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.634281+0000","last_change":"2023-10-09T10:43:35.485235+0000","last_active":"2023-10-09T10:44:32.634281+0000","last_peered":"2023-10-09T10:44:32.634281+0000","last_clean":"2023-10-09T10:44:32.634281+0000","last_became_active":"2023-10-09T10:43:35.484917+0000","last_became_peered":"2023-10-09T10:43:35.484917+0000","last_unstale":"2023-10-09T10:44:32.634281+0000","last_undegraded":"2023-10-09T10:44:32.634281+0000","last_fullsized":"2023-10-09T10:44:32.634281+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:55.417326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":67,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.617375+0000","last_change":"2023-10-09T10:42:38.413301+0000","last_active":"2023-10-09T10:44:33.617375+0000","last_peered":"2023-10-09T10:44:33.617375+0000","last_clean":"2023-10-09T10:44:33.617375+0000","last_became_active":"2023-10-09T10:42:38.413006+0000","last_became_peered":"2023-10-09T10:42:38.413006+0000","last_unstale":"2023-10-09T10:44:33.617375+0000","last_undegraded":"2023-10-09T10:44:33.617375+0000","last_fullsized":"2023-10-09T10:44:33.617375+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:33.462419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":59,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.608009+0000","last_change":"2023-10-09T10:43:35.487114+0000","last_active":"2023-10-09T10:44:32.608009+0000","last_peered":"2023-10-09T10:44:32.608009+0000","last_clean":"2023-10-09T10:44:32.608009+0000","last_became_active":"2023-10-09T10:43:35.486830+0000","last_became_peered":"2023-10-09T10:43:35.486830+0000","last_unstale":"2023-10-09T10:44:32.608009+0000","last_undegraded":"2023-10-09T10:44:32.608009+0000","last_fullsized":"2023-10-09T10:44:32.608009+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:08.836969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":70,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.634190+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:44:32.634190+0000","last_peered":"2023-10-09T10:44:32.634190+0000","last_clean":"2023-10-09T10:44:32.634190+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:44:32.634190+0000","last_undegraded":"2023-10-09T10:44:32.634190+0000","last_fullsized":"2023-10-09T10:44:32.634190+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.617728+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:44:33.617728+0000","last_peered":"2023-10-09T10:44:33.617728+0000","last_clean":"2023-10-09T10:44:33.617728+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:44:33.617728+0000","last_undegraded":"2023-10-09T10:44:33.617728+0000","last_fullsized":"2023-10-09T10:44:33.617728+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":29,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.633920+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:44:32.633920+0000","last_peered":"2023-10-09T10:44:32.633920+0000","last_clean":"2023-10-09T10:44:32.633920+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:44:32.633920+0000","last_undegraded":"2023-10-09T10:44:32.633920+0000","last_fullsized":"2023-10-09T10:44:32.633920+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":23,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.621433+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:44:33.621433+0000","last_peered":"2023-10-09T10:44:33.621433+0000","last_clean":"2023-10-09T10:44:33.621433+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:44:33.621433+0000","last_undegraded":"2023-10-09T10:44:33.621433+0000","last_fullsized":"2023-10-09T10:44:33.621433+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":53,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.617928+0000","last_change":"2023-10-09T10:43:06.672114+0000","last_active":"2023-10-09T10:44:32.617928+0000","last_peered":"2023-10-09T10:44:32.617928+0000","last_clean":"2023-10-09T10:44:32.617928+0000","last_became_active":"2023-10-09T10:43:06.671827+0000","last_became_peered":"2023-10-09T10:43:06.671827+0000","last_unstale":"2023-10-09T10:44:32.617928+0000","last_undegraded":"2023-10-09T10:44:32.617928+0000","last_fullsized":"2023-10-09T10:44:32.617928+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:41.701409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":74,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.616409+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:44:33.616409+0000","last_peered":"2023-10-09T10:44:33.616409+0000","last_clean":"2023-10-09T10:44:33.616409+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:44:33.616409+0000","last_undegraded":"2023-10-09T10:44:33.616409+0000","last_fullsized":"2023-10-09T10:44:33.616409+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.616445+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:44:33.616445+0000","last_peered":"2023-10-09T10:44:33.616445+0000","last_clean":"2023-10-09T10:44:33.616445+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:44:33.616445+0000","last_undegraded":"2023-10-09T10:44:33.616445+0000","last_fullsized":"2023-10-09T10:44:33.616445+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.615262+0000","last_change":"2023-10-09T10:43:06.677115+0000","last_active":"2023-10-09T10:44:33.615262+0000","last_peered":"2023-10-09T10:44:33.615262+0000","last_clean":"2023-10-09T10:44:33.615262+0000","last_became_active":"2023-10-09T10:43:06.676739+0000","last_became_peered":"2023-10-09T10:43:06.676739+0000","last_unstale":"2023-10-09T10:44:33.615262+0000","last_undegraded":"2023-10-09T10:44:33.615262+0000","last_fullsized":"2023-10-09T10:44:33.615262+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:02.520770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":84,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.634858+0000","last_change":"2023-10-09T10:43:35.487942+0000","last_active":"2023-10-09T10:44:32.634858+0000","last_peered":"2023-10-09T10:44:32.634858+0000","last_clean":"2023-10-09T10:44:32.634858+0000","last_became_active":"2023-10-09T10:43:35.487651+0000","last_became_peered":"2023-10-09T10:43:35.487651+0000","last_unstale":"2023-10-09T10:44:32.634858+0000","last_undegraded":"2023-10-09T10:44:32.634858+0000","last_fullsized":"2023-10-09T10:44:32.634858+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:51.882595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":59,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.607334+0000","last_change":"2023-10-09T10:43:35.486914+0000","last_active":"2023-10-09T10:44:32.607334+0000","last_peered":"2023-10-09T10:44:32.607334+0000","last_clean":"2023-10-09T10:44:32.607334+0000","last_became_active":"2023-10-09T10:43:35.486623+0000","last_became_peered":"2023-10-09T10:43:35.486623+0000","last_unstale":"2023-10-09T10:44:32.607334+0000","last_undegraded":"2023-10-09T10:44:32.607334+0000","last_fullsized":"2023-10-09T10:44:32.607334+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:13.339801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.618508+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:44:33.618508+0000","last_peered":"2023-10-09T10:44:33.618508+0000","last_clean":"2023-10-09T10:44:33.618508+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:44:33.618508+0000","last_undegraded":"2023-10-09T10:44:33.618508+0000","last_fullsized":"2023-10-09T10:44:33.618508+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.621350+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:44:33.621350+0000","last_peered":"2023-10-09T10:44:33.621350+0000","last_clean":"2023-10-09T10:44:33.621350+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:44:33.621350+0000","last_undegraded":"2023-10-09T10:44:33.621350+0000","last_fullsized":"2023-10-09T10:44:33.621350+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":59,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.607323+0000","last_change":"2023-10-09T10:43:35.487796+0000","last_active":"2023-10-09T10:44:32.607323+0000","last_peered":"2023-10-09T10:44:32.607323+0000","last_clean":"2023-10-09T10:44:32.607323+0000","last_became_active":"2023-10-09T10:43:35.487518+0000","last_became_peered":"2023-10-09T10:43:35.487518+0000","last_unstale":"2023-10-09T10:44:32.607323+0000","last_undegraded":"2023-10-09T10:44:32.607323+0000","last_fullsized":"2023-10-09T10:44:32.607323+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:47.089685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":76,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.616424+0000","last_change":"2023-10-09T10:41:29.512929+0000","last_active":"2023-10-09T10:44:33.616424+0000","last_peered":"2023-10-09T10:44:33.616424+0000","last_clean":"2023-10-09T10:44:33.616424+0000","last_became_active":"2023-10-09T10:41:29.512591+0000","last_became_peered":"2023-10-09T10:41:29.512591+0000","last_unstale":"2023-10-09T10:44:33.616424+0000","last_undegraded":"2023-10-09T10:44:33.616424+0000","last_fullsized":"2023-10-09T10:44:33.616424+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:35.703322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":70,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.635646+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:44:32.635646+0000","last_peered":"2023-10-09T10:44:32.635646+0000","last_clean":"2023-10-09T10:44:32.635646+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:44:32.635646+0000","last_undegraded":"2023-10-09T10:44:32.635646+0000","last_fullsized":"2023-10-09T10:44:32.635646+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"104'1268","reported_seq":1282,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T10:44:40.903171+0000","last_change":"2023-10-09T10:44:27.566986+0000","last_active":"2023-10-09T10:44:40.903171+0000","last_peered":"2023-10-09T10:44:40.903171+0000","last_clean":"2023-10-09T10:44:40.903171+0000","last_became_active":"2023-10-09T10:44:27.566682+0000","last_became_peered":"2023-10-09T10:44:27.566682+0000","last_unstale":"2023-10-09T10:44:40.903171+0000","last_undegraded":"2023-10-09T10:44:40.903171+0000","last_fullsized":"2023-10-09T10:44:40.903171+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":1268,"log_dups_size":0,"ondisk_log_size":1268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:36.504111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83099648,"num_objects":1268,"num_object_clones":0,"num_object_copies":3804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1268,"num_write_kb":81152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.614953+0000","last_change":"2023-10-09T10:41:29.516129+0000","last_active":"2023-10-09T10:44:33.614953+0000","last_peered":"2023-10-09T10:44:33.614953+0000","last_clean":"2023-10-09T10:44:33.614953+0000","last_became_active":"2023-10-09T10:41:29.515741+0000","last_became_peered":"2023-10-09T10:41:29.515741+0000","last_unstale":"2023-10-09T10:44:33.614953+0000","last_undegraded":"2023-10-09T10:44:33.614953+0000","last_fullsized":"2023-10-09T10:44:33.614953+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:29:14.059627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.617878+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:44:33.617878+0000","last_peered":"2023-10-09T10:44:33.617878+0000","last_clean":"2023-10-09T10:44:33.617878+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:44:33.617878+0000","last_undegraded":"2023-10-09T10:44:33.617878+0000","last_fullsized":"2023-10-09T10:44:33.617878+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"103'886","reported_seq":900,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:38.461510+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:44:38.461510+0000","last_peered":"2023-10-09T10:44:38.461510+0000","last_clean":"2023-10-09T10:44:38.461510+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:44:38.461510+0000","last_undegraded":"2023-10-09T10:44:38.461510+0000","last_fullsized":"2023-10-09T10:44:38.461510+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":886,"log_dups_size":0,"ondisk_log_size":886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58064896,"num_objects":886,"num_object_clones":0,"num_object_copies":2658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":886,"num_write_kb":56704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.614941+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:44:33.614941+0000","last_peered":"2023-10-09T10:44:33.614941+0000","last_clean":"2023-10-09T10:44:33.614941+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:44:33.614941+0000","last_undegraded":"2023-10-09T10:44:33.614941+0000","last_fullsized":"2023-10-09T10:44:33.614941+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":95,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.616818+0000","last_change":"2023-10-09T10:43:35.493278+0000","last_active":"2023-10-09T10:44:33.616818+0000","last_peered":"2023-10-09T10:44:33.616818+0000","last_clean":"2023-10-09T10:44:33.616818+0000","last_became_active":"2023-10-09T10:43:35.492915+0000","last_became_peered":"2023-10-09T10:43:35.492915+0000","last_unstale":"2023-10-09T10:44:33.616818+0000","last_undegraded":"2023-10-09T10:44:33.616818+0000","last_fullsized":"2023-10-09T10:44:33.616818+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:42.227364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"104'1005","reported_seq":1023,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T10:44:39.658195+0000","last_change":"2023-10-09T10:44:33.618884+0000","last_active":"2023-10-09T10:44:39.658195+0000","last_peered":"2023-10-09T10:44:39.658195+0000","last_clean":"2023-10-09T10:44:39.658195+0000","last_became_active":"2023-10-09T10:44:33.618237+0000","last_became_peered":"2023-10-09T10:44:33.618237+0000","last_unstale":"2023-10-09T10:44:39.658195+0000","last_undegraded":"2023-10-09T10:44:39.658195+0000","last_fullsized":"2023-10-09T10:44:39.658195+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":1005,"log_dups_size":0,"ondisk_log_size":1005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:36:19.641515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65863680,"num_objects":1005,"num_object_clones":0,"num_object_copies":3015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1005,"num_write_kb":64320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":25,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.625588+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:44:33.625588+0000","last_peered":"2023-10-09T10:44:33.625588+0000","last_clean":"2023-10-09T10:44:33.625588+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:44:33.625588+0000","last_undegraded":"2023-10-09T10:44:33.625588+0000","last_fullsized":"2023-10-09T10:44:33.625588+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":67,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.618120+0000","last_change":"2023-10-09T10:42:38.411928+0000","last_active":"2023-10-09T10:44:33.618120+0000","last_peered":"2023-10-09T10:44:33.618120+0000","last_clean":"2023-10-09T10:44:33.618120+0000","last_became_active":"2023-10-09T10:42:38.411560+0000","last_became_peered":"2023-10-09T10:42:38.411560+0000","last_unstale":"2023-10-09T10:44:33.618120+0000","last_undegraded":"2023-10-09T10:44:33.618120+0000","last_fullsized":"2023-10-09T10:44:33.618120+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:07.711753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"103'718","reported_seq":732,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:37.510509+0000","last_change":"2023-10-09T10:44:27.565408+0000","last_active":"2023-10-09T10:44:37.510509+0000","last_peered":"2023-10-09T10:44:37.510509+0000","last_clean":"2023-10-09T10:44:37.510509+0000","last_became_active":"2023-10-09T10:44:27.565091+0000","last_became_peered":"2023-10-09T10:44:27.565091+0000","last_unstale":"2023-10-09T10:44:37.510509+0000","last_undegraded":"2023-10-09T10:44:37.510509+0000","last_fullsized":"2023-10-09T10:44:37.510509+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":718,"log_dups_size":0,"ondisk_log_size":718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:03.848914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47054848,"num_objects":718,"num_object_clones":0,"num_object_copies":2154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":718,"num_write_kb":45952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.618150+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:44:33.618150+0000","last_peered":"2023-10-09T10:44:33.618150+0000","last_clean":"2023-10-09T10:44:33.618150+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:44:33.618150+0000","last_undegraded":"2023-10-09T10:44:33.618150+0000","last_fullsized":"2023-10-09T10:44:33.618150+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":59,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.607544+0000","last_change":"2023-10-09T10:42:38.407703+0000","last_active":"2023-10-09T10:44:32.607544+0000","last_peered":"2023-10-09T10:44:32.607544+0000","last_clean":"2023-10-09T10:44:32.607544+0000","last_became_active":"2023-10-09T10:42:38.407431+0000","last_became_peered":"2023-10-09T10:42:38.407431+0000","last_unstale":"2023-10-09T10:44:32.607544+0000","last_undegraded":"2023-10-09T10:44:32.607544+0000","last_fullsized":"2023-10-09T10:44:32.607544+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:23.450634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"104'1172","reported_seq":1186,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T10:44:40.933134+0000","last_change":"2023-10-09T10:44:27.574527+0000","last_active":"2023-10-09T10:44:40.933134+0000","last_peered":"2023-10-09T10:44:40.933134+0000","last_clean":"2023-10-09T10:44:40.933134+0000","last_became_active":"2023-10-09T10:44:27.574205+0000","last_became_peered":"2023-10-09T10:44:27.574205+0000","last_unstale":"2023-10-09T10:44:40.933134+0000","last_undegraded":"2023-10-09T10:44:40.933134+0000","last_fullsized":"2023-10-09T10:44:40.933134+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":1172,"log_dups_size":0,"ondisk_log_size":1172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:54.868034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76808192,"num_objects":1172,"num_object_clones":0,"num_object_copies":3516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1172,"num_write_kb":75008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":68,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.634750+0000","last_change":"2023-10-09T10:43:35.491435+0000","last_active":"2023-10-09T10:44:32.634750+0000","last_peered":"2023-10-09T10:44:32.634750+0000","last_clean":"2023-10-09T10:44:32.634750+0000","last_became_active":"2023-10-09T10:43:35.491096+0000","last_became_peered":"2023-10-09T10:43:35.491096+0000","last_unstale":"2023-10-09T10:44:32.634750+0000","last_undegraded":"2023-10-09T10:44:32.634750+0000","last_fullsized":"2023-10-09T10:44:32.634750+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:09.581525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.616318+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:44:33.616318+0000","last_peered":"2023-10-09T10:44:33.616318+0000","last_clean":"2023-10-09T10:44:33.616318+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:44:33.616318+0000","last_undegraded":"2023-10-09T10:44:33.616318+0000","last_fullsized":"2023-10-09T10:44:33.616318+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"104'996","reported_seq":1011,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T10:44:39.657950+0000","last_change":"2023-10-09T10:44:27.574938+0000","last_active":"2023-10-09T10:44:39.657950+0000","last_peered":"2023-10-09T10:44:39.657950+0000","last_clean":"2023-10-09T10:44:39.657950+0000","last_became_active":"2023-10-09T10:44:27.574662+0000","last_became_peered":"2023-10-09T10:44:27.574662+0000","last_unstale":"2023-10-09T10:44:39.657950+0000","last_undegraded":"2023-10-09T10:44:39.657950+0000","last_fullsized":"2023-10-09T10:44:39.657950+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":996,"log_dups_size":0,"ondisk_log_size":996,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:35:19.975925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65273856,"num_objects":996,"num_object_clones":0,"num_object_copies":2988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":996,"num_write_kb":63744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.618250+0000","last_change":"2023-10-09T10:42:38.412169+0000","last_active":"2023-10-09T10:44:33.618250+0000","last_peered":"2023-10-09T10:44:33.618250+0000","last_clean":"2023-10-09T10:44:33.618250+0000","last_became_active":"2023-10-09T10:42:38.411829+0000","last_became_peered":"2023-10-09T10:42:38.411829+0000","last_unstale":"2023-10-09T10:44:33.618250+0000","last_undegraded":"2023-10-09T10:44:33.618250+0000","last_fullsized":"2023-10-09T10:44:33.618250+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:36:35.699293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":85,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.616891+0000","last_change":"2023-10-09T10:43:06.676367+0000","last_active":"2023-10-09T10:44:33.616891+0000","last_peered":"2023-10-09T10:44:33.616891+0000","last_clean":"2023-10-09T10:44:33.616891+0000","last_became_active":"2023-10-09T10:43:06.676017+0000","last_became_peered":"2023-10-09T10:43:06.676017+0000","last_unstale":"2023-10-09T10:44:33.616891+0000","last_undegraded":"2023-10-09T10:44:33.616891+0000","last_fullsized":"2023-10-09T10:44:33.616891+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:54.387617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"103'867","reported_seq":881,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:38.483943+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:44:38.483943+0000","last_peered":"2023-10-09T10:44:38.483943+0000","last_clean":"2023-10-09T10:44:38.483943+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:44:38.483943+0000","last_undegraded":"2023-10-09T10:44:38.483943+0000","last_fullsized":"2023-10-09T10:44:38.483943+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":867,"log_dups_size":0,"ondisk_log_size":867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56819712,"num_objects":867,"num_object_clones":0,"num_object_copies":2601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":867,"num_write_kb":55488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.616607+0000","last_change":"2023-10-09T10:43:06.676577+0000","last_active":"2023-10-09T10:44:33.616607+0000","last_peered":"2023-10-09T10:44:33.616607+0000","last_clean":"2023-10-09T10:44:33.616607+0000","last_became_active":"2023-10-09T10:43:06.676165+0000","last_became_peered":"2023-10-09T10:43:06.676165+0000","last_unstale":"2023-10-09T10:44:33.616607+0000","last_undegraded":"2023-10-09T10:44:33.616607+0000","last_fullsized":"2023-10-09T10:44:33.616607+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:52.614985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.616747+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:44:33.616747+0000","last_peered":"2023-10-09T10:44:33.616747+0000","last_clean":"2023-10-09T10:44:33.616747+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:44:33.616747+0000","last_undegraded":"2023-10-09T10:44:33.616747+0000","last_fullsized":"2023-10-09T10:44:33.616747+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"104'1220","reported_seq":1234,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T10:44:40.939971+0000","last_change":"2023-10-09T10:44:27.569554+0000","last_active":"2023-10-09T10:44:40.939971+0000","last_peered":"2023-10-09T10:44:40.939971+0000","last_clean":"2023-10-09T10:44:40.939971+0000","last_became_active":"2023-10-09T10:44:27.569215+0000","last_became_peered":"2023-10-09T10:44:27.569215+0000","last_unstale":"2023-10-09T10:44:40.939971+0000","last_undegraded":"2023-10-09T10:44:40.939971+0000","last_fullsized":"2023-10-09T10:44:40.939971+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":1220,"log_dups_size":0,"ondisk_log_size":1220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:41.558109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79953920,"num_objects":1220,"num_object_clones":0,"num_object_copies":3660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1220,"num_write_kb":78080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":71,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.634434+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:44:32.634434+0000","last_peered":"2023-10-09T10:44:32.634434+0000","last_clean":"2023-10-09T10:44:32.634434+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:44:32.634434+0000","last_undegraded":"2023-10-09T10:44:32.634434+0000","last_fullsized":"2023-10-09T10:44:32.634434+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":53,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.618136+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:44:32.618136+0000","last_peered":"2023-10-09T10:44:32.618136+0000","last_clean":"2023-10-09T10:44:32.618136+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:44:32.618136+0000","last_undegraded":"2023-10-09T10:44:32.618136+0000","last_fullsized":"2023-10-09T10:44:32.618136+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":78,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.618483+0000","last_change":"2023-10-09T10:42:38.416111+0000","last_active":"2023-10-09T10:44:32.618483+0000","last_peered":"2023-10-09T10:44:32.618483+0000","last_clean":"2023-10-09T10:44:32.618483+0000","last_became_active":"2023-10-09T10:42:38.415861+0000","last_became_peered":"2023-10-09T10:42:38.415861+0000","last_unstale":"2023-10-09T10:44:32.618483+0000","last_undegraded":"2023-10-09T10:44:32.618483+0000","last_fullsized":"2023-10-09T10:44:32.618483+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:10:40.572274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.618588+0000","last_change":"2023-10-09T10:43:35.486524+0000","last_active":"2023-10-09T10:44:33.618588+0000","last_peered":"2023-10-09T10:44:33.618588+0000","last_clean":"2023-10-09T10:44:33.618588+0000","last_became_active":"2023-10-09T10:43:35.486152+0000","last_became_peered":"2023-10-09T10:43:35.486152+0000","last_unstale":"2023-10-09T10:44:33.618588+0000","last_undegraded":"2023-10-09T10:44:33.618588+0000","last_fullsized":"2023-10-09T10:44:33.618588+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:47.200522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":38,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.618069+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:44:32.618069+0000","last_peered":"2023-10-09T10:44:32.618069+0000","last_clean":"2023-10-09T10:44:32.618069+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:44:32.618069+0000","last_undegraded":"2023-10-09T10:44:32.618069+0000","last_fullsized":"2023-10-09T10:44:32.618069+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"103'758","reported_seq":772,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:37.516954+0000","last_change":"2023-10-09T10:44:27.567512+0000","last_active":"2023-10-09T10:44:37.516954+0000","last_peered":"2023-10-09T10:44:37.516954+0000","last_clean":"2023-10-09T10:44:37.516954+0000","last_became_active":"2023-10-09T10:44:27.567216+0000","last_became_peered":"2023-10-09T10:44:27.567216+0000","last_unstale":"2023-10-09T10:44:37.516954+0000","last_undegraded":"2023-10-09T10:44:37.516954+0000","last_fullsized":"2023-10-09T10:44:37.516954+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":758,"log_dups_size":0,"ondisk_log_size":758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:08.445397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49676288,"num_objects":758,"num_object_clones":0,"num_object_copies":2274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":758,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":758,"num_write_kb":48512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"103'852","reported_seq":866,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:38.489046+0000","last_change":"2023-10-09T10:44:27.574737+0000","last_active":"2023-10-09T10:44:38.489046+0000","last_peered":"2023-10-09T10:44:38.489046+0000","last_clean":"2023-10-09T10:44:38.489046+0000","last_became_active":"2023-10-09T10:44:27.574481+0000","last_became_peered":"2023-10-09T10:44:27.574481+0000","last_unstale":"2023-10-09T10:44:38.489046+0000","last_undegraded":"2023-10-09T10:44:38.489046+0000","last_fullsized":"2023-10-09T10:44:38.489046+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":852,"log_dups_size":0,"ondisk_log_size":852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:48:55.395091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55836672,"num_objects":852,"num_object_clones":0,"num_object_copies":2556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":852,"num_write_kb":54528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.618719+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:44:33.618719+0000","last_peered":"2023-10-09T10:44:33.618719+0000","last_clean":"2023-10-09T10:44:33.618719+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:44:33.618719+0000","last_undegraded":"2023-10-09T10:44:33.618719+0000","last_fullsized":"2023-10-09T10:44:33.618719+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.616615+0000","last_change":"2023-10-09T10:41:29.511298+0000","last_active":"2023-10-09T10:44:33.616615+0000","last_peered":"2023-10-09T10:44:33.616615+0000","last_clean":"2023-10-09T10:44:33.616615+0000","last_became_active":"2023-10-09T10:41:29.510978+0000","last_became_peered":"2023-10-09T10:41:29.510978+0000","last_unstale":"2023-10-09T10:44:33.616615+0000","last_undegraded":"2023-10-09T10:44:33.616615+0000","last_fullsized":"2023-10-09T10:44:33.616615+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:20:11.933459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":68,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.618415+0000","last_change":"2023-10-09T10:43:35.489332+0000","last_active":"2023-10-09T10:44:32.618415+0000","last_peered":"2023-10-09T10:44:32.618415+0000","last_clean":"2023-10-09T10:44:32.618415+0000","last_became_active":"2023-10-09T10:43:35.489044+0000","last_became_peered":"2023-10-09T10:43:35.489044+0000","last_unstale":"2023-10-09T10:44:32.618415+0000","last_undegraded":"2023-10-09T10:44:32.618415+0000","last_fullsized":"2023-10-09T10:44:32.618415+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:42.168915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"104'1233","reported_seq":1255,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T10:44:40.947812+0000","last_change":"2023-10-09T10:44:33.624061+0000","last_active":"2023-10-09T10:44:40.947812+0000","last_peered":"2023-10-09T10:44:40.947812+0000","last_clean":"2023-10-09T10:44:40.947812+0000","last_became_active":"2023-10-09T10:44:33.619704+0000","last_became_peered":"2023-10-09T10:44:33.619704+0000","last_unstale":"2023-10-09T10:44:40.947812+0000","last_undegraded":"2023-10-09T10:44:40.947812+0000","last_fullsized":"2023-10-09T10:44:40.947812+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":1233,"log_dups_size":0,"ondisk_log_size":1233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:40:51.405653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80805888,"num_objects":1233,"num_object_clones":0,"num_object_copies":3699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1233,"num_write_kb":78912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":90,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.614145+0000","last_change":"2023-10-09T10:43:35.490168+0000","last_active":"2023-10-09T10:44:33.614145+0000","last_peered":"2023-10-09T10:44:33.614145+0000","last_clean":"2023-10-09T10:44:33.614145+0000","last_became_active":"2023-10-09T10:43:35.489834+0000","last_became_peered":"2023-10-09T10:43:35.489834+0000","last_unstale":"2023-10-09T10:44:33.614145+0000","last_undegraded":"2023-10-09T10:44:33.614145+0000","last_fullsized":"2023-10-09T10:44:33.614145+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:59:26.252052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.614873+0000","last_change":"2023-10-09T10:43:06.672862+0000","last_active":"2023-10-09T10:44:33.614873+0000","last_peered":"2023-10-09T10:44:33.614873+0000","last_clean":"2023-10-09T10:44:33.614873+0000","last_became_active":"2023-10-09T10:43:06.672572+0000","last_became_peered":"2023-10-09T10:43:06.672572+0000","last_unstale":"2023-10-09T10:44:33.614873+0000","last_undegraded":"2023-10-09T10:44:33.614873+0000","last_fullsized":"2023-10-09T10:44:33.614873+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:23.743910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":73,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.618060+0000","last_change":"2023-10-09T10:42:38.406633+0000","last_active":"2023-10-09T10:44:32.618060+0000","last_peered":"2023-10-09T10:44:32.618060+0000","last_clean":"2023-10-09T10:44:32.618060+0000","last_became_active":"2023-10-09T10:42:38.406317+0000","last_became_peered":"2023-10-09T10:42:38.406317+0000","last_unstale":"2023-10-09T10:44:32.618060+0000","last_undegraded":"2023-10-09T10:44:32.618060+0000","last_fullsized":"2023-10-09T10:44:32.618060+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:02.600240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"104'1154","reported_seq":1168,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T10:44:40.945423+0000","last_change":"2023-10-09T10:44:27.574669+0000","last_active":"2023-10-09T10:44:40.945423+0000","last_peered":"2023-10-09T10:44:40.945423+0000","last_clean":"2023-10-09T10:44:40.945423+0000","last_became_active":"2023-10-09T10:44:27.574254+0000","last_became_peered":"2023-10-09T10:44:27.574254+0000","last_unstale":"2023-10-09T10:44:40.945423+0000","last_undegraded":"2023-10-09T10:44:40.945423+0000","last_fullsized":"2023-10-09T10:44:40.945423+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":1154,"log_dups_size":0,"ondisk_log_size":1154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:41.104607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75628544,"num_objects":1154,"num_object_clones":0,"num_object_copies":3462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1154,"num_write_kb":73856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":73,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.617588+0000","last_change":"2023-10-09T10:43:06.673931+0000","last_active":"2023-10-09T10:44:33.617588+0000","last_peered":"2023-10-09T10:44:33.617588+0000","last_clean":"2023-10-09T10:44:33.617588+0000","last_became_active":"2023-10-09T10:43:06.673605+0000","last_became_peered":"2023-10-09T10:43:06.673605+0000","last_unstale":"2023-10-09T10:44:33.617588+0000","last_undegraded":"2023-10-09T10:44:33.617588+0000","last_fullsized":"2023-10-09T10:44:33.617588+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:19.088672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":414,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.620080+0000","last_change":"2023-10-09T10:44:33.619837+0000","last_active":"2023-10-09T10:44:33.620080+0000","last_peered":"2023-10-09T10:44:33.620080+0000","last_clean":"2023-10-09T10:44:33.620080+0000","last_became_active":"2023-10-09T10:44:33.619483+0000","last_became_peered":"2023-10-09T10:44:33.619483+0000","last_unstale":"2023-10-09T10:44:33.620080+0000","last_undegraded":"2023-10-09T10:44:33.620080+0000","last_fullsized":"2023-10-09T10:44:33.620080+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:21.313472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":29,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.634110+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:44:32.634110+0000","last_peered":"2023-10-09T10:44:32.634110+0000","last_clean":"2023-10-09T10:44:32.634110+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:44:32.634110+0000","last_undegraded":"2023-10-09T10:44:32.634110+0000","last_fullsized":"2023-10-09T10:44:32.634110+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":50,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.617576+0000","last_change":"2023-10-09T10:35:57.165395+0000","last_active":"2023-10-09T10:44:33.617576+0000","last_peered":"2023-10-09T10:44:33.617576+0000","last_clean":"2023-10-09T10:44:33.617576+0000","last_became_active":"2023-10-09T10:35:57.165077+0000","last_became_peered":"2023-10-09T10:35:57.165077+0000","last_unstale":"2023-10-09T10:44:33.617576+0000","last_undegraded":"2023-10-09T10:44:33.617576+0000","last_fullsized":"2023-10-09T10:44:33.617576+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:53.159616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":75,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.617437+0000","last_change":"2023-10-09T10:44:33.617149+0000","last_active":"2023-10-09T10:44:33.617437+0000","last_peered":"2023-10-09T10:44:33.617437+0000","last_clean":"2023-10-09T10:44:33.617437+0000","last_became_active":"2023-10-09T10:44:33.616822+0000","last_became_peered":"2023-10-09T10:44:33.616822+0000","last_unstale":"2023-10-09T10:44:33.617437+0000","last_undegraded":"2023-10-09T10:44:33.617437+0000","last_fullsized":"2023-10-09T10:44:33.617437+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:06:17.307378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":59,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.606633+0000","last_change":"2023-10-09T10:43:35.491431+0000","last_active":"2023-10-09T10:44:32.606633+0000","last_peered":"2023-10-09T10:44:32.606633+0000","last_clean":"2023-10-09T10:44:32.606633+0000","last_became_active":"2023-10-09T10:43:35.491153+0000","last_became_peered":"2023-10-09T10:43:35.491153+0000","last_unstale":"2023-10-09T10:44:32.606633+0000","last_undegraded":"2023-10-09T10:44:32.606633+0000","last_fullsized":"2023-10-09T10:44:32.606633+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:58.495730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":73,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.608223+0000","last_change":"2023-10-09T10:43:06.673430+0000","last_active":"2023-10-09T10:44:32.608223+0000","last_peered":"2023-10-09T10:44:32.608223+0000","last_clean":"2023-10-09T10:44:32.608223+0000","last_became_active":"2023-10-09T10:43:06.673110+0000","last_became_peered":"2023-10-09T10:43:06.673110+0000","last_unstale":"2023-10-09T10:44:32.608223+0000","last_undegraded":"2023-10-09T10:44:32.608223+0000","last_fullsized":"2023-10-09T10:44:32.608223+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:35.429715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"103'575","reported_seq":589,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:36.399027+0000","last_change":"2023-10-09T10:44:27.569117+0000","last_active":"2023-10-09T10:44:36.399027+0000","last_peered":"2023-10-09T10:44:36.399027+0000","last_clean":"2023-10-09T10:44:36.399027+0000","last_became_active":"2023-10-09T10:44:27.568850+0000","last_became_peered":"2023-10-09T10:44:27.568850+0000","last_unstale":"2023-10-09T10:44:36.399027+0000","last_undegraded":"2023-10-09T10:44:36.399027+0000","last_fullsized":"2023-10-09T10:44:36.399027+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":575,"log_dups_size":0,"ondisk_log_size":575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:07.282805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37683200,"num_objects":575,"num_object_clones":0,"num_object_copies":1725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":575,"num_write_kb":36800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.615311+0000","last_change":"2023-10-09T10:41:29.512225+0000","last_active":"2023-10-09T10:44:33.615311+0000","last_peered":"2023-10-09T10:44:33.615311+0000","last_clean":"2023-10-09T10:44:33.615311+0000","last_became_active":"2023-10-09T10:41:29.511907+0000","last_became_peered":"2023-10-09T10:41:29.511907+0000","last_unstale":"2023-10-09T10:44:33.615311+0000","last_undegraded":"2023-10-09T10:44:33.615311+0000","last_fullsized":"2023-10-09T10:44:33.615311+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:25.351050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":53,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.618345+0000","last_change":"2023-10-09T10:43:06.671356+0000","last_active":"2023-10-09T10:44:32.618345+0000","last_peered":"2023-10-09T10:44:32.618345+0000","last_clean":"2023-10-09T10:44:32.618345+0000","last_became_active":"2023-10-09T10:43:06.671040+0000","last_became_peered":"2023-10-09T10:43:06.671040+0000","last_unstale":"2023-10-09T10:44:32.618345+0000","last_undegraded":"2023-10-09T10:44:32.618345+0000","last_fullsized":"2023-10-09T10:44:32.618345+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:47.601448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":44,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.607408+0000","last_change":"2023-10-09T10:41:29.513802+0000","last_active":"2023-10-09T10:44:32.607408+0000","last_peered":"2023-10-09T10:44:32.607408+0000","last_clean":"2023-10-09T10:44:32.607408+0000","last_became_active":"2023-10-09T10:41:29.512895+0000","last_became_peered":"2023-10-09T10:41:29.512895+0000","last_unstale":"2023-10-09T10:44:32.607408+0000","last_undegraded":"2023-10-09T10:44:32.607408+0000","last_fullsized":"2023-10-09T10:44:32.607408+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:06.177976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"103'549","reported_seq":563,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:36.378595+0000","last_change":"2023-10-09T10:44:27.569094+0000","last_active":"2023-10-09T10:44:36.378595+0000","last_peered":"2023-10-09T10:44:36.378595+0000","last_clean":"2023-10-09T10:44:36.378595+0000","last_became_active":"2023-10-09T10:44:27.568809+0000","last_became_peered":"2023-10-09T10:44:27.568809+0000","last_unstale":"2023-10-09T10:44:36.378595+0000","last_undegraded":"2023-10-09T10:44:36.378595+0000","last_fullsized":"2023-10-09T10:44:36.378595+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":549,"log_dups_size":0,"ondisk_log_size":549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:01:30.022053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35979264,"num_objects":549,"num_object_clones":0,"num_object_copies":1647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":549,"num_write_kb":35136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":68,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.635743+0000","last_change":"2023-10-09T10:43:35.487842+0000","last_active":"2023-10-09T10:44:32.635743+0000","last_peered":"2023-10-09T10:44:32.635743+0000","last_clean":"2023-10-09T10:44:32.635743+0000","last_became_active":"2023-10-09T10:43:35.487192+0000","last_became_peered":"2023-10-09T10:43:35.487192+0000","last_unstale":"2023-10-09T10:44:32.635743+0000","last_undegraded":"2023-10-09T10:44:32.635743+0000","last_fullsized":"2023-10-09T10:44:32.635743+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:35.477280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":38,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.618772+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:44:32.618772+0000","last_peered":"2023-10-09T10:44:32.618772+0000","last_clean":"2023-10-09T10:44:32.618772+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:44:32.618772+0000","last_undegraded":"2023-10-09T10:44:32.618772+0000","last_fullsized":"2023-10-09T10:44:32.618772+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.634355+0000","last_change":"2023-10-09T10:43:35.486342+0000","last_active":"2023-10-09T10:44:32.634355+0000","last_peered":"2023-10-09T10:44:32.634355+0000","last_clean":"2023-10-09T10:44:32.634355+0000","last_became_active":"2023-10-09T10:43:35.486026+0000","last_became_peered":"2023-10-09T10:43:35.486026+0000","last_unstale":"2023-10-09T10:44:32.634355+0000","last_undegraded":"2023-10-09T10:44:32.634355+0000","last_fullsized":"2023-10-09T10:44:32.634355+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:16.616429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"104'1182","reported_seq":1196,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T10:44:40.956509+0000","last_change":"2023-10-09T10:44:27.574746+0000","last_active":"2023-10-09T10:44:40.956509+0000","last_peered":"2023-10-09T10:44:40.956509+0000","last_clean":"2023-10-09T10:44:40.956509+0000","last_became_active":"2023-10-09T10:44:27.574292+0000","last_became_peered":"2023-10-09T10:44:27.574292+0000","last_unstale":"2023-10-09T10:44:40.956509+0000","last_undegraded":"2023-10-09T10:44:40.956509+0000","last_fullsized":"2023-10-09T10:44:40.956509+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":1182,"log_dups_size":0,"ondisk_log_size":1182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:17.093260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77463552,"num_objects":1182,"num_object_clones":0,"num_object_copies":3546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1182,"num_write_kb":75648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.617292+0000","last_change":"2023-10-09T10:43:06.674387+0000","last_active":"2023-10-09T10:44:33.617292+0000","last_peered":"2023-10-09T10:44:33.617292+0000","last_clean":"2023-10-09T10:44:33.617292+0000","last_became_active":"2023-10-09T10:43:06.674032+0000","last_became_peered":"2023-10-09T10:43:06.674032+0000","last_unstale":"2023-10-09T10:44:33.617292+0000","last_undegraded":"2023-10-09T10:44:33.617292+0000","last_fullsized":"2023-10-09T10:44:33.617292+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:58.222034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":63,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.607492+0000","last_change":"2023-10-09T10:42:38.407058+0000","last_active":"2023-10-09T10:44:32.607492+0000","last_peered":"2023-10-09T10:44:32.607492+0000","last_clean":"2023-10-09T10:44:32.607492+0000","last_became_active":"2023-10-09T10:42:38.406768+0000","last_became_peered":"2023-10-09T10:42:38.406768+0000","last_unstale":"2023-10-09T10:44:32.607492+0000","last_undegraded":"2023-10-09T10:44:32.607492+0000","last_fullsized":"2023-10-09T10:44:32.607492+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:41.198843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.616528+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:44:33.616528+0000","last_peered":"2023-10-09T10:44:33.616528+0000","last_clean":"2023-10-09T10:44:33.616528+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:44:33.616528+0000","last_undegraded":"2023-10-09T10:44:33.616528+0000","last_fullsized":"2023-10-09T10:44:33.616528+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"104'1205","reported_seq":1219,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T10:44:40.956082+0000","last_change":"2023-10-09T10:44:27.568846+0000","last_active":"2023-10-09T10:44:40.956082+0000","last_peered":"2023-10-09T10:44:40.956082+0000","last_clean":"2023-10-09T10:44:40.956082+0000","last_became_active":"2023-10-09T10:44:27.568538+0000","last_became_peered":"2023-10-09T10:44:27.568538+0000","last_unstale":"2023-10-09T10:44:40.956082+0000","last_undegraded":"2023-10-09T10:44:40.956082+0000","last_fullsized":"2023-10-09T10:44:40.956082+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":1205,"log_dups_size":0,"ondisk_log_size":1205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:53.581603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78970880,"num_objects":1205,"num_object_clones":0,"num_object_copies":3615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1205,"num_write_kb":77120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":48,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.607629+0000","last_change":"2023-10-09T10:41:29.514075+0000","last_active":"2023-10-09T10:44:32.607629+0000","last_peered":"2023-10-09T10:44:32.607629+0000","last_clean":"2023-10-09T10:44:32.607629+0000","last_became_active":"2023-10-09T10:41:29.513347+0000","last_became_peered":"2023-10-09T10:41:29.513347+0000","last_unstale":"2023-10-09T10:44:32.607629+0000","last_undegraded":"2023-10-09T10:44:32.607629+0000","last_fullsized":"2023-10-09T10:44:32.607629+0000","mapping_epoch":80,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":81,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:17.504888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":89,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.614192+0000","last_change":"2023-10-09T10:43:35.489655+0000","last_active":"2023-10-09T10:44:33.614192+0000","last_peered":"2023-10-09T10:44:33.614192+0000","last_clean":"2023-10-09T10:44:33.614192+0000","last_became_active":"2023-10-09T10:43:35.489149+0000","last_became_peered":"2023-10-09T10:43:35.489149+0000","last_unstale":"2023-10-09T10:44:33.614192+0000","last_undegraded":"2023-10-09T10:44:33.614192+0000","last_fullsized":"2023-10-09T10:44:33.614192+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:30.598962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.618290+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:44:33.618290+0000","last_peered":"2023-10-09T10:44:33.618290+0000","last_clean":"2023-10-09T10:44:33.618290+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:44:33.618290+0000","last_undegraded":"2023-10-09T10:44:33.618290+0000","last_fullsized":"2023-10-09T10:44:33.618290+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.616754+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:44:33.616754+0000","last_peered":"2023-10-09T10:44:33.616754+0000","last_clean":"2023-10-09T10:44:33.616754+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:44:33.616754+0000","last_undegraded":"2023-10-09T10:44:33.616754+0000","last_fullsized":"2023-10-09T10:44:33.616754+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.618727+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:44:33.618727+0000","last_peered":"2023-10-09T10:44:33.618727+0000","last_clean":"2023-10-09T10:44:33.618727+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:44:33.618727+0000","last_undegraded":"2023-10-09T10:44:33.618727+0000","last_fullsized":"2023-10-09T10:44:33.618727+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.616431+0000","last_change":"2023-10-09T10:43:35.492236+0000","last_active":"2023-10-09T10:44:33.616431+0000","last_peered":"2023-10-09T10:44:33.616431+0000","last_clean":"2023-10-09T10:44:33.616431+0000","last_became_active":"2023-10-09T10:43:35.491913+0000","last_became_peered":"2023-10-09T10:43:35.491913+0000","last_unstale":"2023-10-09T10:44:33.616431+0000","last_undegraded":"2023-10-09T10:44:33.616431+0000","last_fullsized":"2023-10-09T10:44:33.616431+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:34:13.448179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":76,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.635295+0000","last_change":"2023-10-09T10:43:35.485790+0000","last_active":"2023-10-09T10:44:32.635295+0000","last_peered":"2023-10-09T10:44:32.635295+0000","last_clean":"2023-10-09T10:44:32.635295+0000","last_became_active":"2023-10-09T10:43:35.485436+0000","last_became_peered":"2023-10-09T10:43:35.485436+0000","last_unstale":"2023-10-09T10:44:32.635295+0000","last_undegraded":"2023-10-09T10:44:32.635295+0000","last_fullsized":"2023-10-09T10:44:32.635295+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:45:20.897767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":91,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.633997+0000","last_change":"2023-10-09T10:43:35.487344+0000","last_active":"2023-10-09T10:44:32.633997+0000","last_peered":"2023-10-09T10:44:32.633997+0000","last_clean":"2023-10-09T10:44:32.633997+0000","last_became_active":"2023-10-09T10:43:35.486672+0000","last_became_peered":"2023-10-09T10:43:35.486672+0000","last_unstale":"2023-10-09T10:44:32.633997+0000","last_undegraded":"2023-10-09T10:44:32.633997+0000","last_fullsized":"2023-10-09T10:44:32.633997+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:44.665687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":63,"reported_epoch":102,"state":"active+clean","last_fresh":"2023-10-09T10:44:32.607934+0000","last_change":"2023-10-09T10:43:06.674549+0000","last_active":"2023-10-09T10:44:32.607934+0000","last_peered":"2023-10-09T10:44:32.607934+0000","last_clean":"2023-10-09T10:44:32.607934+0000","last_became_active":"2023-10-09T10:43:06.673510+0000","last_became_peered":"2023-10-09T10:43:06.673510+0000","last_unstale":"2023-10-09T10:44:32.607934+0000","last_undegraded":"2023-10-09T10:44:32.607934+0000","last_fullsized":"2023-10-09T10:44:32.607934+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:32.996795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T10:44:33.616112+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:44:33.616112+0000","last_peered":"2023-10-09T10:44:33.616112+0000","last_clean":"2023-10-09T10:44:33.616112+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:44:33.616112+0000","last_undegraded":"2023-10-09T10:44:33.616112+0000","last_fullsized":"2023-10-09T10:44:33.616112+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1024983040,"num_objects":15640,"num_object_clones":0,"num_object_copies":46920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15640,"num_write_kb":1000960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3132162048,"data_stored":3132162048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":15640,"ondisk_log_size":15640,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332217,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":726988,"kb_used_data":425604,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93018164,"statfs":{"total":95995035648,"available":95250599936,"internally_reserved":0,"allocated":435818496,"data_stored":435361355,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:44:32 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.54400000000000004,"15min":0.56699999999999995},"min":{"1min":0.34599999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.20999999999999999},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.59999999999999998,"15min":0.628},"min":{"1min":0.39300000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.91400000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:43:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":2.077},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.32400000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":1.006},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.92600000000000005}]},{"osd":3,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.88500000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":2.3809999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.48999999999999999,"15min":0.47499999999999998},"min":{"1min":0.308,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.69299999999999995,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.373},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.50700000000000001,"15min":0.47899999999999998},"min":{"1min":0.34300000000000003,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":0.68200000000000005,"5min":0.73899999999999999,"15min":0.83699999999999997},"last":0.26200000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.52700000000000002,"15min":0.50800000000000001},"min":{"1min":0.41799999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.65700000000000003,"5min":0.83699999999999997,"15min":1.1950000000000001},"last":0.247},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.55900000000000005,"15min":0.54200000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.25700000000000001},"max":{"1min":0.73999999999999999,"5min":0.876,"15min":1.0649999999999999},"last":0.40000000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.59299999999999997,"15min":0.56100000000000005},"min":{"1min":0.39100000000000001,"5min":0.36699999999999999,"15min":0.34300000000000003},"max":{"1min":0.73799999999999999,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57899999999999996,"15min":0.54600000000000004},"min":{"1min":0.38100000000000001,"5min":0.35999999999999999,"15min":0.30599999999999999},"max":{"1min":0.72599999999999998,"5min":0.81299999999999994,"15min":0.93500000000000005},"last":0.29399999999999998}]}]},{"osd":6,"up_from":44,"seq":188978561148,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":663360,"kb_used_data":357880,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93081792,"statfs":{"total":95995035648,"available":95315755008,"internally_reserved":0,"allocated":366469120,"data_stored":366024121,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.57699999999999996,"15min":0.59899999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.72599999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.59899999999999998,"15min":0.60499999999999998},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.76300000000000001,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.49099999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.749}]},{"osd":2,"last update":"Mon Oct 9 10:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.28399999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.59499999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.45900000000000002,"15min":0.44400000000000001},"min":{"1min":0.38900000000000001,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":0.59699999999999998,"5min":0.67400000000000004,"15min":0.70999999999999996},"last":0.378},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.505,"15min":0.49199999999999999},"min":{"1min":0.35299999999999998,"5min":0.29899999999999999,"15min":0.29699999999999999},"max":{"1min":0.74399999999999999,"5min":0.77000000000000002,"15min":0.78500000000000003},"last":0.72499999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.51500000000000001,"15min":0.48399999999999999},"min":{"1min":0.40100000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.67600000000000005,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.55000000000000004,"15min":0.53000000000000003},"min":{"1min":0.372,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.84399999999999997},"last":0.76400000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:44:20 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.55400000000000005,"15min":0.53000000000000003},"min":{"1min":0.438,"5min":0.33900000000000002,"15min":0.32900000000000001},"max":{"1min":0.70099999999999996,"5min":0.78600000000000003,"15min":0.79100000000000004},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.629,"5min":0.59699999999999998,"15min":0.57899999999999996},"min":{"1min":0.41199999999999998,"5min":0.34100000000000003,"15min":0.34000000000000002},"max":{"1min":0.72799999999999998,"5min":0.84299999999999997,"15min":1.018},"last":0.79600000000000004}]}]},{"osd":1,"up_from":85,"seq":365072220186,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":174772,"kb_used_data":139116,"kb_used_omap":9,"kb_used_meta":35638,"kb_avail":93570380,"statfs":{"total":95995035648,"available":95816069120,"internally_reserved":0,"allocated":142454784,"data_stored":141956988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":36494313},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.377}]},{"osd":2,"last update":"Mon Oct 9 10:44:08 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.42499999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.3879999999999999,"5min":2.3879999999999999,"15min":2.3879999999999999},"last":2.3879999999999999},{"interface":"front","average":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.26400000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.38900000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.318},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.498}]},{"osd":6,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.56399999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.746}]}]},{"osd":0,"up_from":80,"seq":343597383720,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":192960,"kb_used_data":157240,"kb_used_omap":28,"kb_used_meta":35683,"kb_avail":93552192,"statfs":{"total":95995035648,"available":95797444608,"internally_reserved":0,"allocated":161013760,"data_stored":160503466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":36540240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:43:38 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.38},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":3.7429999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":3.2440000000000002},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"last":0.55900000000000005}]},{"osd":3,"last update":"Mon Oct 9 10:44:36 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.1680000000000001,"5min":5.1680000000000001,"15min":5.1680000000000001},"last":5.1680000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.2839999999999998,"5min":3.2839999999999998,"15min":3.2839999999999998},"last":3.2839999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:44:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.59999999999999998,"15min":0.61399999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.747,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.60799999999999998,"15min":0.61499999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.80300000000000005,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":3.806}]},{"osd":5,"last update":"Mon Oct 9 10:44:32 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.60799999999999998,"15min":0.62},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.79000000000000004,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":3.7810000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.63100000000000001,"15min":0.64300000000000002},"min":{"1min":0.39200000000000002,"5min":0.38,"15min":0.38},"max":{"1min":0.76400000000000001,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.52200000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:44:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.61699999999999999,"15min":0.626},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.75800000000000001,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":3.831},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.63700000000000001,"15min":0.64600000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.77300000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.48999999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:44:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.63200000000000001,"15min":0.64600000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.79300000000000004,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.65200000000000002,"15min":0.66900000000000004},"min":{"1min":0.45800000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.76800000000000002,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.63}]}]},{"osd":2,"up_from":90,"seq":386547056661,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":494068,"kb_used_data":462380,"kb_used_omap":81,"kb_used_meta":31598,"kb_avail":93251084,"statfs":{"total":95995035648,"available":95489110016,"internally_reserved":0,"allocated":473477120,"data_stored":472984251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":32356570},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":1.181},{"interface":"front","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"last":1.2490000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":1.0840000000000001},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.375}]},{"osd":3,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":1.333},{"interface":"front","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.5649999999999999,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":1.113}]},{"osd":4,"last update":"Mon Oct 9 10:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.48799999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":1.1279999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.42299999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":1.143}]},{"osd":7,"last update":"Mon Oct 9 10:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":1.157}]}]},{"osd":3,"up_from":95,"seq":408021893135,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":498908,"kb_used_data":471444,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93246244,"statfs":{"total":95995035648,"available":95484153856,"internally_reserved":0,"allocated":482758656,"data_stored":482279819,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.219},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.992,"5min":1.992,"15min":1.992},"last":1.992}]},{"osd":1,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.59299999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.69499999999999995}]},{"osd":4,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.61299999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.47999999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.44800000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.38700000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019011,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":656200,"kb_used_data":350720,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93088952,"statfs":{"total":95995035648,"available":95323086848,"internally_reserved":0,"allocated":359137280,"data_stored":358683944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.54200000000000004,"15min":0.55200000000000005},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.311},"max":{"1min":0.621,"5min":0.755,"15min":0.755},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.57099999999999995,"15min":0.57999999999999996},"min":{"1min":0.40799999999999997,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.69999999999999996,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.66500000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:44:40 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":0.60299999999999998,"15min":0.56899999999999995},"min":{"1min":0.35499999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.867,"5min":4.867,"15min":4.867},"last":1.4770000000000001},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.65200000000000002,"15min":0.63200000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.8120000000000003,"5min":4.8120000000000003,"15min":4.8120000000000003},"last":0.504}]},{"osd":2,"last update":"Mon Oct 9 10:44:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.57799999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:44:34 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.335,"5min":4.335,"15min":4.335},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.63500000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:44:40 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47899999999999998,"15min":0.443},"min":{"1min":0.25800000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.73599999999999999,"5min":0.79400000000000004,"15min":0.80900000000000005},"last":0.25800000000000001},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.503,"15min":0.45300000000000001},"min":{"1min":0.34599999999999997,"5min":0.28499999999999998,"15min":0.26400000000000001},"max":{"1min":1.0609999999999999,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.54800000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.50800000000000001,"15min":0.46899999999999997},"min":{"1min":0.32000000000000001,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":0.752,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59499999999999997,"15min":0.56000000000000005},"min":{"1min":0.46500000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.82299999999999995,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.42399999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:44:12 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.58699999999999997,"15min":0.56999999999999995},"min":{"1min":0.36699999999999999,"5min":0.35299999999999998,"15min":0.308},"max":{"1min":0.82899999999999996,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.379},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.59299999999999997,"15min":0.54600000000000004},"min":{"1min":0.40400000000000003,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":0.88800000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.95799999999999996}]}]},{"osd":5,"up_from":37,"seq":158913790080,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":654644,"kb_used_data":345068,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93090508,"statfs":{"total":95995035648,"available":95324680192,"internally_reserved":0,"allocated":353349632,"data_stored":352912197,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:44:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.59299999999999997,"15min":0.59599999999999997},"min":{"1min":0.40999999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.65700000000000003,"15min":0.66500000000000004},"min":{"1min":0.443,"5min":0.439,"15min":0.439},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.41599999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:44:40 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.59699999999999998,"15min":0.60399999999999998},"min":{"1min":0.38500000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.69999999999999996,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.64100000000000001,"15min":0.60799999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.5110000000000001,"5min":5.5110000000000001,"15min":5.5110000000000001},"last":5.5110000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:44:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":3.3820000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":1.105}]},{"osd":3,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":3.7240000000000002,"5min":3.7240000000000002,"15min":3.7240000000000002},"last":1.536},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.415,"5min":1.415,"15min":1.415},"last":3.5819999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:44:40 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.48299999999999998,"15min":0.437},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.25},"max":{"1min":0.73199999999999998,"5min":0.75600000000000001,"15min":2.4649999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.52500000000000002,"15min":0.46300000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28799999999999998},"max":{"1min":1.923,"5min":1.923,"15min":1.98},"last":1.923}]},{"osd":6,"last update":"Mon Oct 9 10:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.53200000000000003,"15min":0.45800000000000002},"min":{"1min":0.39300000000000002,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":0.67500000000000004,"5min":0.85599999999999998,"15min":2.048},"last":0.436},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56599999999999995,"15min":0.49099999999999999},"min":{"1min":0.47099999999999997,"5min":0.33500000000000002,"15min":0.30299999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.629}]},{"osd":7,"last update":"Mon Oct 9 10:44:18 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.58599999999999997,"15min":0.54000000000000004},"min":{"1min":0.44,"5min":0.32900000000000001,"15min":0.32400000000000001},"max":{"1min":0.72299999999999998,"5min":0.879,"15min":2.1110000000000002},"last":1.6439999999999999},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.61299999999999999,"15min":0.54600000000000004},"min":{"1min":0.51500000000000001,"5min":0.35699999999999998,"15min":0.31900000000000001},"max":{"1min":0.76900000000000002,"5min":0.86899999999999999,"15min":1.845},"last":1.1379999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":196149248,"data_stored":196149248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":186777600,"data_stored":186777600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":529596416,"data_stored":529596416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":561119232,"data_stored":561119232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":388038656,"data_stored":388038656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":398327808,"data_stored":398327808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":399507456,"data_stored":399507456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":472645632,"data_stored":472645632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:44:42.764 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:44:42.764 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pg_num 26 2023-10-09T10:44:42.823 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:42.898 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:42.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:42 smithi155 ceph-mon[184854]: from='client.54472 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:44:42.979 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:42 smithi125 ceph-mon[195507]: from='client.54472 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:44:42.979 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:42 smithi125 ceph-mon[200197]: from='client.54472 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:44:42.987 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:43.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:43.001 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:43.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:43.102 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:43.142 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:43.170 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:43.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:43.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:43.275 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:43.290 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 11 16 23313 23297 132.322 108.75 0.0101945 0.00755033 2023-10-09T10:44:43.321 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:43.348 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:43.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:43.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:43.452 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:43.518 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:43.540 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:43.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:43.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:43.649 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:43.688 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:43.715 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:43.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:43.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:43.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:43 smithi125 ceph-mon[195507]: pgmap v210: 89 pgs: 89 active+clean; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-09T10:44:43.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:43 smithi125 ceph-mon[200197]: pgmap v210: 89 pgs: 89 active+clean; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-09T10:44:43.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:43.875 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:43.914 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:43.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:43.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:43 smithi155 ceph-mon[184854]: pgmap v210: 89 pgs: 89 active+clean; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-09T10:44:43.970 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:44.028 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:44.063 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:44.092 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:44.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:44.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:44.197 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:44.241 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:44.268 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:44.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:44.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:44.290 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 12 16 25193 25177 131.086 117.5 0.00474445 0.00762271 2023-10-09T10:44:44.374 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:44.424 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:44.768 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:44:44.773 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:44.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:44.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:44.794 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:44.880 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:44:44.880 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:44:44.880 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:44:44.881 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:44:44.881 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:44:44.882 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:44.925 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:44.954 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:44.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:44.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:45.057 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:45.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:44 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4057780599' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:45.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:44 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:45.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:44 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4057780599' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:45.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:44 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:45.096 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:44 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4057780599' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:45.096 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:44 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:45.098 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:45.130 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:45.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:45.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:45.234 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:45.276 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:45.290 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 13 16 27164 27148 130.478 123.188 0.00715779 0.00765893 2023-10-09T10:44:45.301 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:45.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:45.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:45.406 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:45.446 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:45.475 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:45.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:45.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:45.582 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:45.617 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:45.643 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:45.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:45.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:45.648 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:44:45.748 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:44:45.787 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:45.816 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:44:45.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:45.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:44:45.916 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:44:45.917 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T10:44:45.917 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[195507]: failed to encode map e105 with expected crc 2023-10-09T10:44:45.917 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:44:45.917 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[195507]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T10:44:45.917 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[195507]: failed to encode map e105 with expected crc 2023-10-09T10:44:45.917 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[195507]: failed to encode map e105 with expected crc 2023-10-09T10:44:45.917 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:44:45.917 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:44:45.918 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[195507]: pgmap v212: 89 pgs: 89 active+clean; 1.3 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T10:44:45.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:44:45.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T10:44:45.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[200197]: failed to encode map e105 with expected crc 2023-10-09T10:44:45.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:44:45.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[200197]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T10:44:45.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[200197]: failed to encode map e105 with expected crc 2023-10-09T10:44:45.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[200197]: failed to encode map e105 with expected crc 2023-10-09T10:44:45.919 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:44:45.919 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:44:45.919 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:45 smithi125 ceph-mon[200197]: pgmap v212: 89 pgs: 89 active+clean; 1.3 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T10:44:45.920 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:45.957 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:45.980 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:45.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:45.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:46.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:46.125 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:46.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:46.208 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:46.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:46.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:45 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:44:46.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:45 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T10:44:46.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:45 smithi155 ceph-mon[184854]: failed to encode map e105 with expected crc 2023-10-09T10:44:46.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:45 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:44:46.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:45 smithi155 ceph-mon[184854]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T10:44:46.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:45 smithi155 ceph-mon[184854]: failed to encode map e105 with expected crc 2023-10-09T10:44:46.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:45 smithi155 ceph-mon[184854]: failed to encode map e105 with expected crc 2023-10-09T10:44:46.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:44:46.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:44:46.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:45 smithi155 ceph-mon[184854]: pgmap v212: 89 pgs: 89 active+clean; 1.3 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T10:44:46.290 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 14 16 29024 29008 129.459 116.25 0.00906322 0.00771847 2023-10-09T10:44:46.309 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:46.344 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:46.369 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:46.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:46.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:46.472 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:46.508 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:46.534 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:46.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:46.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:46.638 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:46.675 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:46.707 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:46.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:46.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:46.796 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:46.813 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:46.844 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:46.870 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:46.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:46.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:46.974 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:47.028 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:47.055 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 6 pg_num to 26 2023-10-09T10:44:47.290 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 15 16 30630 30614 127.519 100.375 0.0088619 0.00783569 2023-10-09T10:44:47.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4214960661' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:44:47.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:47 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:44:47.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4214960661' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:44:47.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:47 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:44:47.451 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:44:47.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:47 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4214960661' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:44:47.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:47 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:44:47.472 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:47.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:47.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:47.500 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:44:47.531 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:44:47.579 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:47.581 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:44:47.613 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:44:47.622 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:44:47.647 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:47.649 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:44:47.684 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:44:47.725 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:44:47.740 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:44:47.740 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:44:47.741 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:44:47.741 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:44:47.741 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:44:47.741 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:44:47.741 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:44:47.741 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:44:47.780 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:47.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:47.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:47.887 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:47.918 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:47.945 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:47.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:47.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:48.049 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:48.107 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:48.185 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:48.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:48.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:48.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:48 smithi155 ceph-mon[184854]: pgmap v213: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 130 MiB/s wr, 2.09k op/s 2023-10-09T10:44:48.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:48 smithi155 ceph-mon[184854]: failed to encode map e106 with expected crc 2023-10-09T10:44:48.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:48 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:44:48.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:48 smithi155 ceph-mon[184854]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T10:44:48.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:48 smithi155 ceph-mon[184854]: failed to encode map e106 with expected crc 2023-10-09T10:44:48.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:48 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/394291021' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:48.291 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:48.292 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 16 16 32239 32223 125.831 100.562 0.00444639 0.00794205 2023-10-09T10:44:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:48 smithi125 ceph-mon[195507]: pgmap v213: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 130 MiB/s wr, 2.09k op/s 2023-10-09T10:44:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:48 smithi125 ceph-mon[195507]: failed to encode map e106 with expected crc 2023-10-09T10:44:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:48 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:44:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:48 smithi125 ceph-mon[195507]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T10:44:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:48 smithi125 ceph-mon[195507]: failed to encode map e106 with expected crc 2023-10-09T10:44:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:48 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/394291021' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:48 smithi125 ceph-mon[200197]: pgmap v213: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 130 MiB/s wr, 2.09k op/s 2023-10-09T10:44:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:48 smithi125 ceph-mon[200197]: failed to encode map e106 with expected crc 2023-10-09T10:44:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:48 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:44:48.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:48 smithi125 ceph-mon[200197]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T10:44:48.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:48 smithi125 ceph-mon[200197]: failed to encode map e106 with expected crc 2023-10-09T10:44:48.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:48 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/394291021' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:48.326 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:48.355 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:48.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:48.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:48.459 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:48.464 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:48.533 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:48.612 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:44:48.612 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:44:48.618 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:48.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:48.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:48.725 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:48.781 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:48.961 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:48.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:48.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:49.065 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:49.137 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:49.170 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:49.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:49.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:49.192 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:44:49.219 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:49 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:44:49.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:49 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:44:49.273 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:49.291 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 17 16 33502 33486 123.073 78.9375 0.00223643 0.00801138 2023-10-09T10:44:49.298 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:49 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:44:49.300 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:49.318 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:49.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:49.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:49.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:44:49.449 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:49.467 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:44:49.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:49.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:44:49.570 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:49.603 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:49.672 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:49.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:49.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:49.743 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:49.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:49.802 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:50.067 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:44:50.217 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:44:50.291 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 18 16 33987 33971 117.92 30.3125 0.0102696 0.00847529 2023-10-09T10:44:50.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[195507]: pgmap v215: 89 pgs: 89 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-10-09T10:44:50.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[195507]: failed to encode map e107 with expected crc 2023-10-09T10:44:50.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[195507]: failed to encode map e107 with expected crc 2023-10-09T10:44:50.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:44:50.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[195507]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T10:44:50.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[195507]: failed to encode map e107 with expected crc 2023-10-09T10:44:50.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[195507]: failed to encode map e107 with expected crc 2023-10-09T10:44:50.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3976381088' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:44:50.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:44:50.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[200197]: pgmap v215: 89 pgs: 89 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-10-09T10:44:50.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[200197]: failed to encode map e107 with expected crc 2023-10-09T10:44:50.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[200197]: failed to encode map e107 with expected crc 2023-10-09T10:44:50.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:44:50.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[200197]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T10:44:50.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[200197]: failed to encode map e107 with expected crc 2023-10-09T10:44:50.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[200197]: failed to encode map e107 with expected crc 2023-10-09T10:44:50.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3976381088' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:44:50.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:50 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:44:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:50 smithi155 ceph-mon[184854]: pgmap v215: 89 pgs: 89 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-10-09T10:44:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:50 smithi155 ceph-mon[184854]: failed to encode map e107 with expected crc 2023-10-09T10:44:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:50 smithi155 ceph-mon[184854]: failed to encode map e107 with expected crc 2023-10-09T10:44:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:50 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:44:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:50 smithi155 ceph-mon[184854]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T10:44:50.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:50 smithi155 ceph-mon[184854]: failed to encode map e107 with expected crc 2023-10-09T10:44:50.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:50 smithi155 ceph-mon[184854]: failed to encode map e107 with expected crc 2023-10-09T10:44:50.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:50 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3976381088' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:44:50.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:50 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:44:50.631 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:50.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:50.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:50.739 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:50.791 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:50.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:50.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:50.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:51.024 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:51.049 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:44:51.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[195507]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[195507]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:44:51.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[195507]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T10:44:51.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[195507]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[195507]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[195507]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[195507]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[195507]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[195507]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.051 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[195507]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:44:51.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[200197]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[200197]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:44:51.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[200197]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T10:44:51.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[200197]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[200197]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[200197]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.052 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[200197]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.052 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[200197]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.052 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[200197]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.052 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-mon[200197]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.078 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:51.207 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:51.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:51.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:51.293 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 19 16 35990 35974 118.292 125.188 0.00614119 0.00844826 2023-10-09T10:44:51.308 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:51.312 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:51.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:44:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:44:51] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.43.0" 2023-10-09T10:44:51.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:51 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:44:51.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:51 smithi155 ceph-mon[184854]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:51 smithi155 ceph-mon[184854]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:51 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:44:51.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:51 smithi155 ceph-mon[184854]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T10:44:51.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:51 smithi155 ceph-mon[184854]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:51 smithi155 ceph-mon[184854]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:51 smithi155 ceph-mon[184854]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:51 smithi155 ceph-mon[184854]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:51 smithi155 ceph-mon[184854]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:51 smithi155 ceph-mon[184854]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:51 smithi155 ceph-mon[184854]: failed to encode map e108 with expected crc 2023-10-09T10:44:51.348 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:51.377 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:51.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:51.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:51.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:51.538 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:51.575 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:51.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:51.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:51.679 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:51.719 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:51.744 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:51.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:51.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:51.849 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:51.900 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:51.969 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:44:51.969 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:44:52.088 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:52.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:52.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:52.163 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:52 smithi125 ceph-mon[195507]: pgmap v218: 99 pgs: 1 peering, 6 unknown, 92 active+clean; 2.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s; 11 KiB/s, 0 objects/s recovering 2023-10-09T10:44:52.165 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:52 smithi125 ceph-mon[200197]: pgmap v218: 99 pgs: 1 peering, 6 unknown, 92 active+clean; 2.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s; 11 KiB/s, 0 objects/s recovering 2023-10-09T10:44:52.193 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:52.235 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:52.261 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:52.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:52.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:52.293 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:44:52.293805+0000 min lat: 0.00102371 max lat: 1.04963 avg lat: 0.00844934 2023-10-09T10:44:52.293 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:44:52.293 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 20 16 37880 37864 118.283 118.125 0.00484419 0.00844934 2023-10-09T10:44:52.365 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:52.428 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:44:52.434 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:52 smithi155 ceph-mon[184854]: pgmap v218: 99 pgs: 1 peering, 6 unknown, 92 active+clean; 2.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s; 11 KiB/s, 0 objects/s recovering 2023-10-09T10:44:52.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:52.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:52.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:52.581 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:44:52.653 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:52.696 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:44:52.741 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:44:52.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:52.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:44:52.748 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:44:52.784 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:44:52.824 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:44:52.846 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:52.861 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:44:52.890 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:52.896 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:44:52.905 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:52.925 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:44:52.966 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:44:52.976 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:44:52.976 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:44:52.976 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:44:52.976 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:44:52.977 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:44:52.977 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:44:52.977 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:44:52.977 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:44:52.995 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:53.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:53.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:53.008 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:44:53.008 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:44:53.008 INFO:tasks.thrashosds.thrasher:Reweighting osd 4 to 0.951240334519261 2023-10-09T10:44:53.008 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 0.951240334519261 2023-10-09T10:44:53.102 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:44:53.136 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:53.164 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:44:53.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:53.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:44:53.264 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:44:53.269 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:53.293 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 21 16 39529 39513 117.558 103.062 0.00680935 0.00850155 2023-10-09T10:44:53.404 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:53.564 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:53 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3107794089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:53.564 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:53 smithi125 ceph-mon[200197]: pgmap v219: 99 pgs: 1 peering, 2 unknown, 96 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 11 KiB/s, 0 objects/s recovering 2023-10-09T10:44:53.564 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:53 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:44:53.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:53 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3107794089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:53.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:53 smithi125 ceph-mon[195507]: pgmap v219: 99 pgs: 1 peering, 2 unknown, 96 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 11 KiB/s, 0 objects/s recovering 2023-10-09T10:44:53.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:53 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:44:53.599 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:53.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:53.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:53.704 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:53.764 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:53.841 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:53.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:53.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:53.848 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:53 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3107794089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:44:53.848 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:53 smithi155 ceph-mon[184854]: pgmap v219: 99 pgs: 1 peering, 2 unknown, 96 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 11 KiB/s, 0 objects/s recovering 2023-10-09T10:44:53.848 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:53 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:44:53.917 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:53.949 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:53.972 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:53.986 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:54.070 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:54.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:54.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:54.175 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:54.222 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:54.293 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 22 16 41031 41015 116.481 93.875 0.00771046 0.0085798 2023-10-09T10:44:54.350 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:54.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:54.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:54.454 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:44:54.505 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:54.632 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:44:54.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:54.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:44:54.737 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:54.812 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:54.887 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:44:54.887 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:44:54.888 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:44:54.888 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:44:54.888 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:44:55.038 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:55.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:55.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:55.142 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:55.189 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:55.293 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 23 16 42695 42679 115.938 104 0.00561607 0.00860336 2023-10-09T10:44:55.309 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:55 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T10:44:55.309 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:55 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T10:44:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:55 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T10:44:55.573 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:55.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:55.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:55.642 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:44:55.676 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:55.713 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:55.782 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:55.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:55.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:55.886 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:44:55.914 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:55.935 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:44:55.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:55.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:44:56.039 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:56.051 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:44:56.065 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:56.134 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:56.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:56.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:56.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[195507]: pgmap v220: 99 pgs: 99 active+clean; 2.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 24 KiB/s, 0 objects/s recovering 2023-10-09T10:44:56.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[195507]: failed to encode map e109 with expected crc 2023-10-09T10:44:56.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T10:44:56.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[195507]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T10:44:56.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[195507]: failed to encode map e109 with expected crc 2023-10-09T10:44:56.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[195507]: failed to encode map e109 with expected crc 2023-10-09T10:44:56.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[195507]: failed to encode map e109 with expected crc 2023-10-09T10:44:56.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[195507]: failed to encode map e109 with expected crc 2023-10-09T10:44:56.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[200197]: pgmap v220: 99 pgs: 99 active+clean; 2.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 24 KiB/s, 0 objects/s recovering 2023-10-09T10:44:56.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[200197]: failed to encode map e109 with expected crc 2023-10-09T10:44:56.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T10:44:56.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[200197]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T10:44:56.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[200197]: failed to encode map e109 with expected crc 2023-10-09T10:44:56.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[200197]: failed to encode map e109 with expected crc 2023-10-09T10:44:56.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[200197]: failed to encode map e109 with expected crc 2023-10-09T10:44:56.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:56 smithi125 ceph-mon[200197]: failed to encode map e109 with expected crc 2023-10-09T10:44:56.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:56 smithi155 ceph-mon[184854]: pgmap v220: 99 pgs: 99 active+clean; 2.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 24 KiB/s, 0 objects/s recovering 2023-10-09T10:44:56.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:56 smithi155 ceph-mon[184854]: failed to encode map e109 with expected crc 2023-10-09T10:44:56.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:56 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T10:44:56.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:56 smithi155 ceph-mon[184854]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T10:44:56.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:56 smithi155 ceph-mon[184854]: failed to encode map e109 with expected crc 2023-10-09T10:44:56.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:56 smithi155 ceph-mon[184854]: failed to encode map e109 with expected crc 2023-10-09T10:44:56.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:56 smithi155 ceph-mon[184854]: failed to encode map e109 with expected crc 2023-10-09T10:44:56.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:56 smithi155 ceph-mon[184854]: failed to encode map e109 with expected crc 2023-10-09T10:44:56.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:56.269 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:56.291 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:56.293 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 24 16 42906 42890 111.658 13.1875 0.00163896 0.0085789 2023-10-09T10:44:56.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:56.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:56.395 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:56.423 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:56.442 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:56.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:56.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:56.545 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:44:56.572 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:56.589 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:56.591 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:44:56.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:56.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:44:56.640 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:44:56.695 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:56.725 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:56.797 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:56.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:56.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:56.900 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:56.928 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:56.946 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:56.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:56.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:57.050 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:57.077 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:57.095 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:57.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:57.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:57.117 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:44:57.119 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.4 to 0.95124 (f384) 2023-10-09T10:44:57.200 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:57 smithi155 ceph-mon[184854]: failed to encode map e110 with expected crc 2023-10-09T10:44:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:57 smithi155 ceph-mon[184854]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T10:44:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:57 smithi155 ceph-mon[184854]: failed to encode map e110 with expected crc 2023-10-09T10:44:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:57 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:44:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:57 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:44:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:57 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/623798556' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:57.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:57 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:57.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:57 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1209393932' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.951240334519261}]: dispatch 2023-10-09T10:44:57.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:57 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T10:44:57.248 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:57.294 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 25 16 43048 43032 107.548 8.875 0.00520024 0.00928486 2023-10-09T10:44:57.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:57.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:57.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:57.399 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:57.437 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[195507]: failed to encode map e110 with expected crc 2023-10-09T10:44:57.437 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[195507]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T10:44:57.437 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[195507]: failed to encode map e110 with expected crc 2023-10-09T10:44:57.438 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:44:57.438 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:44:57.438 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/623798556' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:57.438 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:57.438 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1209393932' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.951240334519261}]: dispatch 2023-10-09T10:44:57.438 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T10:44:57.438 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[200197]: failed to encode map e110 with expected crc 2023-10-09T10:44:57.438 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[200197]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T10:44:57.438 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[200197]: failed to encode map e110 with expected crc 2023-10-09T10:44:57.438 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:44:57.439 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:44:57.439 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/623798556' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:57.439 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:44:57.439 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1209393932' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.951240334519261}]: dispatch 2023-10-09T10:44:57.439 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:57 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T10:44:57.446 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:57.924 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:57.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:57.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:58.029 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:44:58.055 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:58.075 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:44:58.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:58.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:44:58.178 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:58.208 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:58.231 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:58.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:58.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:58.293 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 26 16 43385 43369 104.222 21.0625 0.00150213 0.00924412 2023-10-09T10:44:58.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: failed to encode map e110 with expected crc 2023-10-09T10:44:58.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: failed to encode map e110 with expected crc 2023-10-09T10:44:58.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: pgmap v223: 99 pgs: 99 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s; 28 KiB/s, 0 objects/s recovering 2023-10-09T10:44:58.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:44:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: failed to encode map e111 with expected crc 2023-10-09T10:44:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:44:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1209393932' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.951240334519261}]': finished 2023-10-09T10:44:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T10:44:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T10:44:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: failed to encode map e111 with expected crc 2023-10-09T10:44:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: failed to encode map e111 with expected crc 2023-10-09T10:44:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: failed to encode map e111 with expected crc 2023-10-09T10:44:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T10:44:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: failed to encode map e112 with expected crc 2023-10-09T10:44:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: failed to encode map e112 with expected crc 2023-10-09T10:44:58.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: failed to encode map e112 with expected crc 2023-10-09T10:44:58.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[195507]: failed to encode map e112 with expected crc 2023-10-09T10:44:58.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: failed to encode map e110 with expected crc 2023-10-09T10:44:58.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: failed to encode map e110 with expected crc 2023-10-09T10:44:58.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: pgmap v223: 99 pgs: 99 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s; 28 KiB/s, 0 objects/s recovering 2023-10-09T10:44:58.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:44:58.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: failed to encode map e111 with expected crc 2023-10-09T10:44:58.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:44:58.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1209393932' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.951240334519261}]': finished 2023-10-09T10:44:58.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T10:44:58.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T10:44:58.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: failed to encode map e111 with expected crc 2023-10-09T10:44:58.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: failed to encode map e111 with expected crc 2023-10-09T10:44:58.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: failed to encode map e111 with expected crc 2023-10-09T10:44:58.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T10:44:58.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: failed to encode map e112 with expected crc 2023-10-09T10:44:58.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: failed to encode map e112 with expected crc 2023-10-09T10:44:58.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: failed to encode map e112 with expected crc 2023-10-09T10:44:58.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:58 smithi125 ceph-mon[200197]: failed to encode map e112 with expected crc 2023-10-09T10:44:58.335 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:44:58.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: failed to encode map e110 with expected crc 2023-10-09T10:44:58.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: failed to encode map e110 with expected crc 2023-10-09T10:44:58.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: pgmap v223: 99 pgs: 99 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s; 28 KiB/s, 0 objects/s recovering 2023-10-09T10:44:58.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:44:58.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: failed to encode map e111 with expected crc 2023-10-09T10:44:58.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:44:58.361 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1209393932' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.951240334519261}]': finished 2023-10-09T10:44:58.361 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T10:44:58.361 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T10:44:58.361 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: failed to encode map e111 with expected crc 2023-10-09T10:44:58.361 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: failed to encode map e111 with expected crc 2023-10-09T10:44:58.361 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: failed to encode map e111 with expected crc 2023-10-09T10:44:58.361 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T10:44:58.361 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: failed to encode map e112 with expected crc 2023-10-09T10:44:58.361 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: failed to encode map e112 with expected crc 2023-10-09T10:44:58.361 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: failed to encode map e112 with expected crc 2023-10-09T10:44:58.361 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:58 smithi155 ceph-mon[184854]: failed to encode map e112 with expected crc 2023-10-09T10:44:58.362 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:58.381 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:44:58.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:58.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:44:58.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:58.513 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:58.736 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:58.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:58.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:58.840 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:44:58.891 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:58.978 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:44:58.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:58.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:44:59.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:44:59.175 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:59.200 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:44:59.259 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:44:59.261 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:44:59.262 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:44:59.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:44:59.262 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:44:59.262 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.262 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:41068 6.b 6:d88dc4c2:::benchmark_data_smithi125_244951_object41067:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T10:44:59.262 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:54.314901+0000", 2023-10-09T10:44:59.262 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4.9441602639999997, 2023-10-09T10:44:59.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.053471721, 2023-10-09T10:44:59.262 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.262 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:44:59.263 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:59.263 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:59.263 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:59.263 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 41068 2023-10-09T10:44:59.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.263 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.263 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.263 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.314901+0000", 2023-10-09T10:44:59.263 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.314901+0000", 2023-10-09T10:44:59.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.314900+0000", 2023-10-09T10:44:59.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:44:59.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.314922+0000", 2023-10-09T10:44:59.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1713e-05 2023-10-09T10:44:59.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.314930+0000", 2023-10-09T10:44:59.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7749999999999993e-06 2023-10-09T10:44:59.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.314936+0000", 2023-10-09T10:44:59.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0569999999999998e-06 2023-10-09T10:44:59.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.317053+0000", 2023-10-09T10:44:59.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021176260000000001 2023-10-09T10:44:59.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:44:59.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.317167+0000", 2023-10-09T10:44:59.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011346200000000001 2023-10-09T10:44:59.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,7", 2023-10-09T10:44:59.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.317295+0000", 2023-10-09T10:44:59.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012772900000000001 2023-10-09T10:44:59.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:44:59.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.318872+0000", 2023-10-09T10:44:59.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015771369999999999 2023-10-09T10:44:59.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:44:59.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.367633+0000", 2023-10-09T10:44:59.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.048761521000000002 2023-10-09T10:44:59.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:44:59.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.368297+0000", 2023-10-09T10:44:59.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00066341600000000001 2023-10-09T10:44:59.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:44:59.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.368338+0000", 2023-10-09T10:44:59.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1276000000000003e-05 2023-10-09T10:44:59.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.368373+0000", 2023-10-09T10:44:59.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4779e-05 2023-10-09T10:44:59.269 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.269 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.269 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.270 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44570.0:41070 6.4 e108/107 6:22362fc4:::benchmark_data_smithi125_244951_object41069:head v 108'2346, mlcod=108'2344)", 2023-10-09T10:44:59.270 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:54.318712+0000", 2023-10-09T10:44:59.270 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4.9403492719999997, 2023-10-09T10:44:59.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.048806879999999997, 2023-10-09T10:44:59.270 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.270 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:44:59.270 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.318712+0000", 2023-10-09T10:44:59.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.318712+0000", 2023-10-09T10:44:59.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.318711+0000", 2023-10-09T10:44:59.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:59.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.272 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.272 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.318731+0000", 2023-10-09T10:44:59.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9324999999999999e-05 2023-10-09T10:44:59.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.272 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.272 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.318740+0000", 2023-10-09T10:44:59.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9290000000000004e-06 2023-10-09T10:44:59.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.318745+0000", 2023-10-09T10:44:59.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1359999999999996e-06 2023-10-09T10:44:59.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.318842+0000", 2023-10-09T10:44:59.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7157000000000005e-05 2023-10-09T10:44:59.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:44:59.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.318863+0000", 2023-10-09T10:44:59.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1114000000000001e-05 2023-10-09T10:44:59.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:44:59.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.322296+0000", 2023-10-09T10:44:59.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003433274 2023-10-09T10:44:59.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.367519+0000", 2023-10-09T10:44:59.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.045222627000000001 2023-10-09T10:44:59.275 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.275 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.275 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.275 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44570.0:41071 6.5 e108/107 6:a5e834b5:::benchmark_data_smithi125_244951_object41070:head v 108'2335, mlcod=108'2333)", 2023-10-09T10:44:59.275 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:54.320392+0000", 2023-10-09T10:44:59.275 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4.9386690870000001, 2023-10-09T10:44:59.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.048910365999999997, 2023-10-09T10:44:59.276 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.276 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:44:59.276 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.320392+0000", 2023-10-09T10:44:59.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.320392+0000", 2023-10-09T10:44:59.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.320392+0000", 2023-10-09T10:44:59.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:59.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.320424+0000", 2023-10-09T10:44:59.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.218e-05 2023-10-09T10:44:59.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.320431+0000", 2023-10-09T10:44:59.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3819999999999998e-06 2023-10-09T10:44:59.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.320437+0000", 2023-10-09T10:44:59.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.502e-06 2023-10-09T10:44:59.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.320467+0000", 2023-10-09T10:44:59.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0565999999999998e-05 2023-10-09T10:44:59.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:44:59.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.320480+0000", 2023-10-09T10:44:59.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2738000000000001e-05 2023-10-09T10:44:59.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:44:59.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.369261+0000", 2023-10-09T10:44:59.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.048780721999999999 2023-10-09T10:44:59.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.369302+0000", 2023-10-09T10:44:59.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1751000000000001e-05 2023-10-09T10:44:59.280 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.280 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.280 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.281 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44570.0:41075 6.15 e108/107 6:aa34514a:::benchmark_data_smithi125_244951_object41074:head v 108'2349, mlcod=108'2346)", 2023-10-09T10:44:59.281 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:54.320445+0000", 2023-10-09T10:44:59.281 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4.9386159630000002, 2023-10-09T10:44:59.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.048880966999999997, 2023-10-09T10:44:59.281 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.281 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:44:59.281 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.320445+0000", 2023-10-09T10:44:59.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.320445+0000", 2023-10-09T10:44:59.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.320445+0000", 2023-10-09T10:44:59.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:59.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.320473+0000", 2023-10-09T10:44:59.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7841999999999999e-05 2023-10-09T10:44:59.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.320479+0000", 2023-10-09T10:44:59.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1809999999999998e-06 2023-10-09T10:44:59.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.320483+0000", 2023-10-09T10:44:59.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4970000000000003e-06 2023-10-09T10:44:59.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.321078+0000", 2023-10-09T10:44:59.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00059408400000000004 2023-10-09T10:44:59.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:44:59.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.321096+0000", 2023-10-09T10:44:59.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8417999999999999e-05 2023-10-09T10:44:59.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:44:59.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.369306+0000", 2023-10-09T10:44:59.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.048210255 2023-10-09T10:44:59.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.369326+0000", 2023-10-09T10:44:59.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0047e-05 2023-10-09T10:44:59.285 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.285 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.285 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.286 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44570.0:41073 6.2 e108/107 6:44027eb5:::benchmark_data_smithi125_244951_object41072:head v 108'2345, mlcod=108'2343)", 2023-10-09T10:44:59.286 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:54.321515+0000", 2023-10-09T10:44:59.286 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4.9375466660000003, 2023-10-09T10:44:59.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.050156882999999999, 2023-10-09T10:44:59.286 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.286 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:44:59.286 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.321515+0000", 2023-10-09T10:44:59.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.321515+0000", 2023-10-09T10:44:59.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.321514+0000", 2023-10-09T10:44:59.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:59.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.288 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.321542+0000", 2023-10-09T10:44:59.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7469000000000001e-05 2023-10-09T10:44:59.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.288 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.321548+0000", 2023-10-09T10:44:59.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7839999999999998e-06 2023-10-09T10:44:59.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.321553+0000", 2023-10-09T10:44:59.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4530000000000004e-06 2023-10-09T10:44:59.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.321599+0000", 2023-10-09T10:44:59.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6004000000000002e-05 2023-10-09T10:44:59.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:44:59.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.367960+0000", 2023-10-09T10:44:59.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.046361375000000003 2023-10-09T10:44:59.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:44:59.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.371636+0000", 2023-10-09T10:44:59.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003676257 2023-10-09T10:44:59.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.371671+0000", 2023-10-09T10:44:59.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5015000000000003e-05 2023-10-09T10:44:59.291 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.291 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.291 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.291 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44570.0:41080 6.2 e108/107 6:45f640b2:::benchmark_data_smithi125_244951_object41079:head v 108'2346, mlcod=108'2343)", 2023-10-09T10:44:59.291 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:54.322551+0000", 2023-10-09T10:44:59.291 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4.9365099969999999, 2023-10-09T10:44:59.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.049139861999999999, 2023-10-09T10:44:59.291 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.291 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:44:59.292 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.322551+0000", 2023-10-09T10:44:59.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.322551+0000", 2023-10-09T10:44:59.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.322551+0000", 2023-10-09T10:44:59.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:59.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.322612+0000", 2023-10-09T10:44:59.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1093000000000006e-05 2023-10-09T10:44:59.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.367917+0000", 2023-10-09T10:44:59.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.045305007000000001 2023-10-09T10:44:59.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.367923+0000", 2023-10-09T10:44:59.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5830000000000003e-06 2023-10-09T10:44:59.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.368778+0000", 2023-10-09T10:44:59.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000854589 2023-10-09T10:44:59.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:44:59.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.368794+0000", 2023-10-09T10:44:59.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6608000000000001e-05 2023-10-09T10:44:59.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:44:59.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.371674+0000", 2023-10-09T10:44:59.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00287966 2023-10-09T10:44:59.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:54.371691+0000", 2023-10-09T10:44:59.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7008999999999999e-05 2023-10-09T10:44:59.296 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.296 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.296 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.296 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42934 6.11 6.374e9c51 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T10:44:59.296 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.210857+0000", 2023-10-09T10:44:59.296 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 2.0482041209999999, 2023-10-09T10:44:59.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33813347999999999, 2023-10-09T10:44:59.297 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.297 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:44:59.297 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:59.297 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:59.297 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:59.297 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42934 2023-10-09T10:44:59.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.297 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.210857+0000", 2023-10-09T10:44:59.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.210857+0000", 2023-10-09T10:44:59.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.210856+0000", 2023-10-09T10:44:59.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:44:59.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.210876+0000", 2023-10-09T10:44:59.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0159000000000001e-05 2023-10-09T10:44:59.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.210890+0000", 2023-10-09T10:44:59.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4301000000000001e-05 2023-10-09T10:44:59.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.210897+0000", 2023-10-09T10:44:59.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9750000000000001e-06 2023-10-09T10:44:59.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.210932+0000", 2023-10-09T10:44:59.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5438999999999997e-05 2023-10-09T10:44:59.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:44:59.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.210939+0000", 2023-10-09T10:44:59.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3269999999999997e-06 2023-10-09T10:44:59.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.548980+0000", 2023-10-09T10:44:59.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33804157000000001 2023-10-09T10:44:59.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.548991+0000", 2023-10-09T10:44:59.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.024e-05 2023-10-09T10:44:59.302 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.302 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.302 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.302 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42936 6.11 6.858b9731 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T10:44:59.302 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.212290+0000", 2023-10-09T10:44:59.302 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 2.0467714560000001, 2023-10-09T10:44:59.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33673148600000002, 2023-10-09T10:44:59.302 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.303 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:44:59.303 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:59.303 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:59.303 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:59.303 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42936 2023-10-09T10:44:59.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.303 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.212290+0000", 2023-10-09T10:44:59.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.212290+0000", 2023-10-09T10:44:59.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.212288+0000", 2023-10-09T10:44:59.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:44:59.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.212309+0000", 2023-10-09T10:44:59.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0933000000000001e-05 2023-10-09T10:44:59.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.212320+0000", 2023-10-09T10:44:59.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0739e-05 2023-10-09T10:44:59.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.212327+0000", 2023-10-09T10:44:59.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7780000000000002e-06 2023-10-09T10:44:59.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.212958+0000", 2023-10-09T10:44:59.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00063159299999999995 2023-10-09T10:44:59.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:44:59.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.212967+0000", 2023-10-09T10:44:59.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0780000000000002e-06 2023-10-09T10:44:59.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549014+0000", 2023-10-09T10:44:59.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33604640699999999 2023-10-09T10:44:59.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549021+0000", 2023-10-09T10:44:59.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3880000000000002e-06 2023-10-09T10:44:59.307 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.307 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.307 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.308 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42950 6.11 6.382ddad1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T10:44:59.308 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.225355+0000", 2023-10-09T10:44:59.308 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 2.0337059630000001, 2023-10-09T10:44:59.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32369242999999998, 2023-10-09T10:44:59.308 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.308 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:44:59.308 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:59.308 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:59.308 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:59.308 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42950 2023-10-09T10:44:59.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.309 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.225355+0000", 2023-10-09T10:44:59.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.225355+0000", 2023-10-09T10:44:59.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.225354+0000", 2023-10-09T10:44:59.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:44:59.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.225372+0000", 2023-10-09T10:44:59.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8440999999999998e-05 2023-10-09T10:44:59.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.225384+0000", 2023-10-09T10:44:59.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2165e-05 2023-10-09T10:44:59.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.225393+0000", 2023-10-09T10:44:59.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4349999999999997e-06 2023-10-09T10:44:59.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.225553+0000", 2023-10-09T10:44:59.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016031899999999999 2023-10-09T10:44:59.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:44:59.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.225561+0000", 2023-10-09T10:44:59.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1130000000000001e-06 2023-10-09T10:44:59.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549040+0000", 2023-10-09T10:44:59.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32347902899999997 2023-10-09T10:44:59.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549048+0000", 2023-10-09T10:44:59.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2880000000000004e-06 2023-10-09T10:44:59.313 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.313 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.313 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.313 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42962 6.11 6.2d31cf51 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T10:44:59.313 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.233229+0000", 2023-10-09T10:44:59.313 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 2.0258326950000001, 2023-10-09T10:44:59.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.31584466, 2023-10-09T10:44:59.314 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.314 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:44:59.314 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:59.314 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:59.314 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:59.314 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42962 2023-10-09T10:44:59.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.314 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.233229+0000", 2023-10-09T10:44:59.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.233229+0000", 2023-10-09T10:44:59.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.233227+0000", 2023-10-09T10:44:59.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:44:59.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.233252+0000", 2023-10-09T10:44:59.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4501000000000001e-05 2023-10-09T10:44:59.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.233263+0000", 2023-10-09T10:44:59.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.17e-05 2023-10-09T10:44:59.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.233271+0000", 2023-10-09T10:44:59.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7109999999999995e-06 2023-10-09T10:44:59.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.234273+0000", 2023-10-09T10:44:59.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001001668 2023-10-09T10:44:59.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:44:59.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.234283+0000", 2023-10-09T10:44:59.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0193999999999999e-05 2023-10-09T10:44:59.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549066+0000", 2023-10-09T10:44:59.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.31478316499999998 2023-10-09T10:44:59.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549073+0000", 2023-10-09T10:44:59.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1539999999999996e-06 2023-10-09T10:44:59.318 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.318 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.318 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.319 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43001 6.11 6.5b8150f1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T10:44:59.319 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.260702+0000", 2023-10-09T10:44:59.319 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 1.9983589770000001, 2023-10-09T10:44:59.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.28840815400000003, 2023-10-09T10:44:59.319 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.319 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:44:59.319 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:59.319 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:59.319 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:59.319 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43001 2023-10-09T10:44:59.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.319 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260702+0000", 2023-10-09T10:44:59.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260702+0000", 2023-10-09T10:44:59.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260702+0000", 2023-10-09T10:44:59.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:59.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260719+0000", 2023-10-09T10:44:59.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7326999999999999e-05 2023-10-09T10:44:59.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260726+0000", 2023-10-09T10:44:59.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4139999999999997e-06 2023-10-09T10:44:59.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260732+0000", 2023-10-09T10:44:59.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2889999999999997e-06 2023-10-09T10:44:59.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260824+0000", 2023-10-09T10:44:59.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2175000000000004e-05 2023-10-09T10:44:59.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:44:59.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260830+0000", 2023-10-09T10:44:59.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5320000000000003e-06 2023-10-09T10:44:59.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549103+0000", 2023-10-09T10:44:59.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.28827289499999997 2023-10-09T10:44:59.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549110+0000", 2023-10-09T10:44:59.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.153e-06 2023-10-09T10:44:59.324 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.324 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.324 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.324 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43002 6.11 6.4589ebd1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T10:44:59.324 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.260740+0000", 2023-10-09T10:44:59.324 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 1.998320724, 2023-10-09T10:44:59.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.28839398900000002, 2023-10-09T10:44:59.324 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.324 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:44:59.325 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:59.325 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:59.325 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:59.325 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43002 2023-10-09T10:44:59.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.325 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260740+0000", 2023-10-09T10:44:59.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260740+0000", 2023-10-09T10:44:59.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260740+0000", 2023-10-09T10:44:59.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:59.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260757+0000", 2023-10-09T10:44:59.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7274000000000001e-05 2023-10-09T10:44:59.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260763+0000", 2023-10-09T10:44:59.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4389999999999999e-06 2023-10-09T10:44:59.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260767+0000", 2023-10-09T10:44:59.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7730000000000001e-06 2023-10-09T10:44:59.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260843+0000", 2023-10-09T10:44:59.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6286999999999997e-05 2023-10-09T10:44:59.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:44:59.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.260846+0000", 2023-10-09T10:44:59.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7530000000000002e-06 2023-10-09T10:44:59.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549128+0000", 2023-10-09T10:44:59.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.28828205299999998 2023-10-09T10:44:59.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549134+0000", 2023-10-09T10:44:59.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7839999999999998e-06 2023-10-09T10:44:59.329 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.329 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.329 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.330 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43015 6.11 6.40195eb1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T10:44:59.330 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.271377+0000", 2023-10-09T10:44:59.330 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 1.987683793, 2023-10-09T10:44:59.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.27778185100000002, 2023-10-09T10:44:59.330 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.330 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:44:59.330 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:59.330 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:59.330 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:59.330 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43015 2023-10-09T10:44:59.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.331 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.271377+0000", 2023-10-09T10:44:59.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.271377+0000", 2023-10-09T10:44:59.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.271376+0000", 2023-10-09T10:44:59.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:44:59.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.271399+0000", 2023-10-09T10:44:59.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2870999999999998e-05 2023-10-09T10:44:59.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.271414+0000", 2023-10-09T10:44:59.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.47e-05 2023-10-09T10:44:59.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.271421+0000", 2023-10-09T10:44:59.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1640000000000004e-06 2023-10-09T10:44:59.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.271779+0000", 2023-10-09T10:44:59.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035777399999999999 2023-10-09T10:44:59.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:44:59.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.271787+0000", 2023-10-09T10:44:59.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0350000000000001e-06 2023-10-09T10:44:59.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549152+0000", 2023-10-09T10:44:59.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.27736544200000002 2023-10-09T10:44:59.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.335 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549159+0000", 2023-10-09T10:44:59.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0600000000000002e-06 2023-10-09T10:44:59.335 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.335 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.335 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.335 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43040 6.11 6.26b7f491 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T10:44:59.335 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.286546+0000", 2023-10-09T10:44:59.335 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 1.9725151489999999, 2023-10-09T10:44:59.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.262638339, 2023-10-09T10:44:59.336 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.336 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:44:59.336 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:59.336 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:59.336 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:59.336 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43040 2023-10-09T10:44:59.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.336 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.286546+0000", 2023-10-09T10:44:59.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.286546+0000", 2023-10-09T10:44:59.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.286545+0000", 2023-10-09T10:44:59.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:44:59.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.286566+0000", 2023-10-09T10:44:59.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0937999999999998e-05 2023-10-09T10:44:59.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.286574+0000", 2023-10-09T10:44:59.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5460000000000003e-06 2023-10-09T10:44:59.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.286580+0000", 2023-10-09T10:44:59.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6910000000000002e-06 2023-10-09T10:44:59.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.287550+0000", 2023-10-09T10:44:59.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00097011700000000005 2023-10-09T10:44:59.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:44:59.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.287556+0000", 2023-10-09T10:44:59.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0129999999999999e-06 2023-10-09T10:44:59.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549177+0000", 2023-10-09T10:44:59.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.26162144900000001 2023-10-09T10:44:59.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549184+0000", 2023-10-09T10:44:59.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2740000000000002e-06 2023-10-09T10:44:59.340 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.340 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.341 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.341 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43116 6.11 6.af376611 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T10:44:59.341 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.337467+0000", 2023-10-09T10:44:59.341 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 1.921593927, 2023-10-09T10:44:59.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.21176484600000001, 2023-10-09T10:44:59.341 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.341 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:44:59.341 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:59.341 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:59.342 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:59.342 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43116 2023-10-09T10:44:59.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.342 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.337467+0000", 2023-10-09T10:44:59.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.337467+0000", 2023-10-09T10:44:59.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.337466+0000", 2023-10-09T10:44:59.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:44:59.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.337487+0000", 2023-10-09T10:44:59.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1005999999999998e-05 2023-10-09T10:44:59.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.344 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.344 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.337495+0000", 2023-10-09T10:44:59.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3860000000000007e-06 2023-10-09T10:44:59.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.344 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.344 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.337502+0000", 2023-10-09T10:44:59.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8050000000000001e-06 2023-10-09T10:44:59.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.344 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.344 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.338367+0000", 2023-10-09T10:44:59.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00086484300000000001 2023-10-09T10:44:59.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:44:59.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.338375+0000", 2023-10-09T10:44:59.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1580000000000005e-06 2023-10-09T10:44:59.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549202+0000", 2023-10-09T10:44:59.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.210827349 2023-10-09T10:44:59.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549232+0000", 2023-10-09T10:44:59.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9672999999999999e-05 2023-10-09T10:44:59.346 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.346 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.346 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.346 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43152 6.11 6.88f15af1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T10:44:59.346 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.357009+0000", 2023-10-09T10:44:59.347 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 1.902052356, 2023-10-09T10:44:59.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19224941100000001, 2023-10-09T10:44:59.347 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.347 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:44:59.347 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:59.347 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:59.347 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:59.347 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43152 2023-10-09T10:44:59.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.347 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.357009+0000", 2023-10-09T10:44:59.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.357009+0000", 2023-10-09T10:44:59.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.357007+0000", 2023-10-09T10:44:59.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:44:59.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.357031+0000", 2023-10-09T10:44:59.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3410000000000001e-05 2023-10-09T10:44:59.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.357044+0000", 2023-10-09T10:44:59.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2886e-05 2023-10-09T10:44:59.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.357052+0000", 2023-10-09T10:44:59.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1009999999999992e-06 2023-10-09T10:44:59.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.357097+0000", 2023-10-09T10:44:59.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4718000000000001e-05 2023-10-09T10:44:59.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:44:59.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.357102+0000", 2023-10-09T10:44:59.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8989999999999998e-06 2023-10-09T10:44:59.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549251+0000", 2023-10-09T10:44:59.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.192148718 2023-10-09T10:44:59.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549258+0000", 2023-10-09T10:44:59.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1369999999999996e-06 2023-10-09T10:44:59.352 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.352 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.352 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.352 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43224 6.11 6.e803d0d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T10:44:59.352 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.408219+0000", 2023-10-09T10:44:59.352 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 1.850842281, 2023-10-09T10:44:59.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.14106487600000001, 2023-10-09T10:44:59.352 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.352 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:44:59.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:59.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:59.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:59.353 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43224 2023-10-09T10:44:59.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.353 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.353 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.353 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.408219+0000", 2023-10-09T10:44:59.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.408219+0000", 2023-10-09T10:44:59.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.408217+0000", 2023-10-09T10:44:59.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:44:59.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.408268+0000", 2023-10-09T10:44:59.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0651e-05 2023-10-09T10:44:59.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.408285+0000", 2023-10-09T10:44:59.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.791e-05 2023-10-09T10:44:59.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.408294+0000", 2023-10-09T10:44:59.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5699999999999993e-06 2023-10-09T10:44:59.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.408354+0000", 2023-10-09T10:44:59.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9905000000000003e-05 2023-10-09T10:44:59.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:44:59.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.408363+0000", 2023-10-09T10:44:59.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8799999999999997e-06 2023-10-09T10:44:59.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549277+0000", 2023-10-09T10:44:59.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.140914132 2023-10-09T10:44:59.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549284+0000", 2023-10-09T10:44:59.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.827e-06 2023-10-09T10:44:59.357 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.357 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.357 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.357 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43255 6.11 6.c92abb11 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T10:44:59.358 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.436971+0000", 2023-10-09T10:44:59.358 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 1.8220905460000001, 2023-10-09T10:44:59.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.112340493, 2023-10-09T10:44:59.358 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.358 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:44:59.358 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:59.358 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:59.358 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:59.358 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43255 2023-10-09T10:44:59.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.358 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.436971+0000", 2023-10-09T10:44:59.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.436971+0000", 2023-10-09T10:44:59.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.436969+0000", 2023-10-09T10:44:59.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:44:59.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.436994+0000", 2023-10-09T10:44:59.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4916e-05 2023-10-09T10:44:59.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.437005+0000", 2023-10-09T10:44:59.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.066e-05 2023-10-09T10:44:59.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.437012+0000", 2023-10-09T10:44:59.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3520000000000001e-06 2023-10-09T10:44:59.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.437067+0000", 2023-10-09T10:44:59.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4700000000000001e-05 2023-10-09T10:44:59.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:44:59.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.437076+0000", 2023-10-09T10:44:59.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4110000000000002e-06 2023-10-09T10:44:59.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.362 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549304+0000", 2023-10-09T10:44:59.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.11222781900000001 2023-10-09T10:44:59.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.362 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549311+0000", 2023-10-09T10:44:59.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9330000000000004e-06 2023-10-09T10:44:59.362 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.362 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.362 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.363 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43315 6.11 6.e23aedf1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T10:44:59.363 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.490819+0000", 2023-10-09T10:44:59.363 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 1.7682423709999999, 2023-10-09T10:44:59.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.058514997999999999, 2023-10-09T10:44:59.363 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.363 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:44:59.363 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:59.363 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:59.363 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:59.363 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43315 2023-10-09T10:44:59.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.364 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.490819+0000", 2023-10-09T10:44:59.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.490819+0000", 2023-10-09T10:44:59.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.490818+0000", 2023-10-09T10:44:59.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:44:59.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.490840+0000", 2023-10-09T10:44:59.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2586999999999999e-05 2023-10-09T10:44:59.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.490852+0000", 2023-10-09T10:44:59.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2054e-05 2023-10-09T10:44:59.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.490860+0000", 2023-10-09T10:44:59.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9710000000000004e-06 2023-10-09T10:44:59.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.490910+0000", 2023-10-09T10:44:59.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0028999999999998e-05 2023-10-09T10:44:59.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:44:59.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.490918+0000", 2023-10-09T10:44:59.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8040000000000002e-06 2023-10-09T10:44:59.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549327+0000", 2023-10-09T10:44:59.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.058409098 2023-10-09T10:44:59.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549334+0000", 2023-10-09T10:44:59.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6259999999999997e-06 2023-10-09T10:44:59.368 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.368 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.368 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.368 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43318 6.11 6.27e5b0f1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T10:44:59.368 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.494281+0000", 2023-10-09T10:44:59.368 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 1.7647799370000001, 2023-10-09T10:44:59.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.055080362000000001, 2023-10-09T10:44:59.368 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:44:59.368 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:44:59.369 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:44:59.369 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:44:59.369 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:44:59.369 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43318 2023-10-09T10:44:59.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.369 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:44:59.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:44:59.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.494281+0000", 2023-10-09T10:44:59.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:44:59.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.494281+0000", 2023-10-09T10:44:59.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:44:59.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:44:59.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.494280+0000", 2023-10-09T10:44:59.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:44:59.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:44:59.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.494309+0000", 2023-10-09T10:44:59.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9193999999999999e-05 2023-10-09T10:44:59.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:44:59.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.494329+0000", 2023-10-09T10:44:59.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9983000000000002e-05 2023-10-09T10:44:59.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:44:59.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.494338+0000", 2023-10-09T10:44:59.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8050000000000001e-06 2023-10-09T10:44:59.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.494394+0000", 2023-10-09T10:44:59.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5868000000000001e-05 2023-10-09T10:44:59.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:44:59.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.494404+0000", 2023-10-09T10:44:59.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6709999999999994e-06 2023-10-09T10:44:59.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:44:59.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549355+0000", 2023-10-09T10:44:59.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.054950488999999998 2023-10-09T10:44:59.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:44:59.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:44:59.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:44:59.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549362+0000", 2023-10-09T10:44:59.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1289999999999998e-06 2023-10-09T10:44:59.373 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.373 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.373 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.373 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:44:59.373 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:44:59.374 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:44:59.374 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:44:59.374 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:59 smithi125 ceph-mon[195507]: failed to encode map e112 with expected crc 2023-10-09T10:44:59.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:59 smithi125 ceph-mon[195507]: failed to encode map e112 with expected crc 2023-10-09T10:44:59.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:59 smithi125 ceph-mon[195507]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T10:44:59.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:59 smithi125 ceph-mon[195507]: failed to encode map e113 with expected crc 2023-10-09T10:44:59.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:59 smithi125 ceph-mon[195507]: failed to encode map e113 with expected crc 2023-10-09T10:44:59.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:44:59 smithi125 ceph-mon[195507]: failed to encode map e113 with expected crc 2023-10-09T10:44:59.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:59 smithi125 ceph-mon[200197]: failed to encode map e112 with expected crc 2023-10-09T10:44:59.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:59 smithi125 ceph-mon[200197]: failed to encode map e112 with expected crc 2023-10-09T10:44:59.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:59 smithi125 ceph-mon[200197]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T10:44:59.376 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:59 smithi125 ceph-mon[200197]: failed to encode map e113 with expected crc 2023-10-09T10:44:59.376 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:59 smithi125 ceph-mon[200197]: failed to encode map e113 with expected crc 2023-10-09T10:44:59.376 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:44:59 smithi125 ceph-mon[200197]: failed to encode map e113 with expected crc 2023-10-09T10:44:59.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:59.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:44:59.377 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:44:59.377 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 27 16 44030 44014 101.853 40.3125 0.00681185 0.00981264 2023-10-09T10:44:59.377 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:44:59.393 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:44:59.449 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:44:59.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:59 smithi155 ceph-mon[184854]: failed to encode map e112 with expected crc 2023-10-09T10:44:59.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:59 smithi155 ceph-mon[184854]: failed to encode map e112 with expected crc 2023-10-09T10:44:59.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:59 smithi155 ceph-mon[184854]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T10:44:59.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:59 smithi155 ceph-mon[184854]: failed to encode map e113 with expected crc 2023-10-09T10:44:59.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:59 smithi155 ceph-mon[184854]: failed to encode map e113 with expected crc 2023-10-09T10:44:59.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:44:59 smithi155 ceph-mon[184854]: failed to encode map e113 with expected crc 2023-10-09T10:44:59.478 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:44:59.513 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:59.519 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:44:59.538 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:44:59.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:59.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:44:59.565 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:44:59.567 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:44:59.567 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:44:59.567 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:44:59.567 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:44:59.567 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:44:59.567 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:44:59.567 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:44:59.568 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:44:59.642 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:44:59.703 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:44:59.884 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:44:59.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:44:59.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:44:59.961 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:44:59.990 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:00.058 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:00.093 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:00.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:00.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:00.166 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:45:00.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[200197]: failed to encode map e113 with expected crc 2023-10-09T10:45:00.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[200197]: failed to encode map e113 with expected crc 2023-10-09T10:45:00.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[200197]: pgmap v227: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 97 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 2612/128757 objects misplaced (2.029%); 520 KiB/s, 8 objects/s recovering 2023-10-09T10:45:00.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/209764577' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:00.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:00.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:00.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:00.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:45:00.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[195507]: failed to encode map e113 with expected crc 2023-10-09T10:45:00.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[195507]: failed to encode map e113 with expected crc 2023-10-09T10:45:00.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[195507]: pgmap v227: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 97 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 2612/128757 objects misplaced (2.029%); 520 KiB/s, 8 objects/s recovering 2023-10-09T10:45:00.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/209764577' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:00.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:00.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:00.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:00.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:45:00.198 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:00.270 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:00.294 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 28 16 45529 45513 101.561 93.6875 0.0165426 0.00983988 2023-10-09T10:45:00.307 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:00.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:00.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:00.331 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:45:00.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:00.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:00 smithi155 ceph-mon[184854]: failed to encode map e113 with expected crc 2023-10-09T10:45:00.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:00 smithi155 ceph-mon[184854]: failed to encode map e113 with expected crc 2023-10-09T10:45:00.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:00 smithi155 ceph-mon[184854]: pgmap v227: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 97 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 2612/128757 objects misplaced (2.029%); 520 KiB/s, 8 objects/s recovering 2023-10-09T10:45:00.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:00 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/209764577' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:00.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:00.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:00.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:00.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:45:00.484 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:00.524 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:00.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:00.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:00.637 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:00.655 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:00.677 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:00.758 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:00.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:00.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:00.864 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:00.912 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:00.940 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:00.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:00.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:01.045 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:01.101 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:01.180 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:01.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:01.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:01.219 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:45:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:45:01] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2023-10-09T10:45:01.286 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:01.294 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 29 16 47041 47025 101.317 94.5 0.00729654 0.00986544 2023-10-09T10:45:01.329 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:01.352 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:01.352 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:45:01.352 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:45:01.352 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:01.352 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:01 smithi155 ceph-mon[184854]: pgmap v228: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 97 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 2612/134613 objects misplaced (1.940%); 416 KiB/s, 6 objects/s recovering 2023-10-09T10:45:01.357 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:01.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:01.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:01.461 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:01.485 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:45:01.485 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:45:01.508 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:01.537 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:01.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:01.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:01.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:01.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:45:01.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:45:01.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:01.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:01 smithi125 ceph-mon[195507]: pgmap v228: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 97 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 2612/134613 objects misplaced (1.940%); 416 KiB/s, 6 objects/s recovering 2023-10-09T10:45:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:45:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:45:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:01 smithi125 ceph-mon[200197]: pgmap v228: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 97 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 2612/134613 objects misplaced (1.940%); 416 KiB/s, 6 objects/s recovering 2023-10-09T10:45:01.642 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:01.677 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:01.847 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:01.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:01.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:01.881 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:45:01.954 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:02.025 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:02.051 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:02.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:02.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:02.156 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:02.202 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:02.229 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:45:02.294 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 30 16 48578 48562 101.142 96.0625 0.00609614 0.00988217 2023-10-09T10:45:02.316 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:45:02.739 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:45:02.739 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:45:02.739 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:45:02.740 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.021074 2023-10-09T10:45:02.740 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 0.02107422246760171 2023-10-09T10:45:03.291 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:03.294 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 31 16 50182 50166 101.113 100.25 0.0299995 0.00988359 2023-10-09T10:45:03.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:03.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:03.374 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:03.397 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:03.442 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:03.503 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:45:03.503 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:45:03.576 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:03.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:03.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:03.684 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:03.726 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:03.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:03 smithi125 ceph-mon[195507]: pgmap v229: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 2.8 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 46 MiB/s wr, 742 op/s; 2708/138189 objects degraded (1.960%); 2948/138189 objects misplaced (2.133%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T10:45:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:03 smithi125 ceph-mon[200197]: pgmap v229: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 2.8 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 46 MiB/s wr, 742 op/s; 2708/138189 objects degraded (1.960%); 2948/138189 objects misplaced (2.133%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T10:45:03.825 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:03 smithi155 ceph-mon[184854]: pgmap v229: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 2.8 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 46 MiB/s wr, 742 op/s; 2708/138189 objects degraded (1.960%); 2948/138189 objects misplaced (2.133%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T10:45:03.853 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:03.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:03.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:03.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:45:03.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:04.014 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:04.038 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:04.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:04.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:04.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:04.200 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:04.261 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:04.268 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:45:04.281 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:04.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:04.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:04.300 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 32 16 51446 51430 100.407 79 0.00624588 0.00995118 2023-10-09T10:45:04.388 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:04.445 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:04.480 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:04.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:04.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:04.588 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:04.606 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:45:04.644 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:04.671 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:04.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:04.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:04.694 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:04 smithi155 ceph-mon[184854]: Health check failed: Degraded data redundancy: 2708/138189 objects degraded (1.960%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:45:04.694 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:04 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2286342354' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:04.694 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:04 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:04.778 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:04.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:04 smithi125 ceph-mon[195507]: Health check failed: Degraded data redundancy: 2708/138189 objects degraded (1.960%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:45:04.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:04 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2286342354' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:04.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:04 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:04.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:04 smithi125 ceph-mon[200197]: Health check failed: Degraded data redundancy: 2708/138189 objects degraded (1.960%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:45:04.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:04 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2286342354' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:04.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:04 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:04.832 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:04.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:04.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:04.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:04.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:05.025 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:05.299 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 33 16 52879 52863 100.078 89.5625 0.00804902 0.00998783 2023-10-09T10:45:05.345 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:45:05.345 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:45:05.561 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:05.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:05.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:05.667 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:05.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:05 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:05.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:05 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:45:05.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:05 smithi155 ceph-mon[184854]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T10:45:05.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:05 smithi155 ceph-mon[184854]: failed to encode map e114 with expected crc 2023-10-09T10:45:05.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:05 smithi155 ceph-mon[184854]: failed to encode map e114 with expected crc 2023-10-09T10:45:05.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:05 smithi155 ceph-mon[184854]: failed to encode map e114 with expected crc 2023-10-09T10:45:05.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:05 smithi155 ceph-mon[184854]: pgmap v231: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 2708/144753 objects degraded (1.871%); 2948/144753 objects misplaced (2.037%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:45:05.727 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:05.736 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:45:05.789 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:05 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:05.789 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:05 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:45:05.789 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:05 smithi125 ceph-mon[195507]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T10:45:05.789 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:05 smithi125 ceph-mon[195507]: failed to encode map e114 with expected crc 2023-10-09T10:45:05.789 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:05 smithi125 ceph-mon[195507]: failed to encode map e114 with expected crc 2023-10-09T10:45:05.789 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:05 smithi125 ceph-mon[195507]: failed to encode map e114 with expected crc 2023-10-09T10:45:05.790 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:05 smithi125 ceph-mon[195507]: pgmap v231: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 2708/144753 objects degraded (1.871%); 2948/144753 objects misplaced (2.037%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:45:05.790 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:05 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:05.790 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:05 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:45:05.790 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:05 smithi125 ceph-mon[200197]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T10:45:05.790 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:05 smithi125 ceph-mon[200197]: failed to encode map e114 with expected crc 2023-10-09T10:45:05.790 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:05 smithi125 ceph-mon[200197]: failed to encode map e114 with expected crc 2023-10-09T10:45:05.790 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:05 smithi125 ceph-mon[200197]: failed to encode map e114 with expected crc 2023-10-09T10:45:05.790 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:05 smithi125 ceph-mon[200197]: pgmap v231: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 2708/144753 objects degraded (1.871%); 2948/144753 objects misplaced (2.037%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:45:05.815 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:05.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:05.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:05.925 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:05.992 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:06.022 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:06.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:06.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:06.096 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:45:06.105 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:45:06.128 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:06.136 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:45:06.171 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:06.176 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:45:06.215 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:45:06.251 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:06.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:06.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:06.258 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:45:06.299 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 34 16 54225 54209 99.6083 84.125 0.00849897 0.0100344 2023-10-09T10:45:06.303 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:45:06.335 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:45:06.356 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:06.375 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:45:06.396 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:45:06.396 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:45:06.396 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:45:06.396 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:45:06.396 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:45:06.396 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:45:06.397 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:45:06.397 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:45:06.405 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:06.433 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:06.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:06.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:06.537 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:06.578 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:06.622 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:06 smithi155 ceph-mon[184854]: 6.3 scrub starts 2023-10-09T10:45:06.622 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:06 smithi155 ceph-mon[184854]: 6.2 scrub starts 2023-10-09T10:45:06.637 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.6 primary-affinity to 0.0210742 (813812) 2023-10-09T10:45:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:06 smithi155 ceph-mon[184854]: 6.3 scrub ok 2023-10-09T10:45:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:06 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3869302889' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.02107422246760171}]: dispatch 2023-10-09T10:45:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:06 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.02107422246760171}]: dispatch 2023-10-09T10:45:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:06 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3058779803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:06.755 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:06.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:06.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:06.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:06 smithi125 ceph-mon[195507]: 6.3 scrub starts 2023-10-09T10:45:06.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:06 smithi125 ceph-mon[195507]: 6.2 scrub starts 2023-10-09T10:45:06.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:06 smithi125 ceph-mon[195507]: 6.3 scrub ok 2023-10-09T10:45:06.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:06 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3869302889' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.02107422246760171}]: dispatch 2023-10-09T10:45:06.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:06 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.02107422246760171}]: dispatch 2023-10-09T10:45:06.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:06 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3058779803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:06.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:06 smithi125 ceph-mon[200197]: 6.3 scrub starts 2023-10-09T10:45:06.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:06 smithi125 ceph-mon[200197]: 6.2 scrub starts 2023-10-09T10:45:06.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:06 smithi125 ceph-mon[200197]: 6.3 scrub ok 2023-10-09T10:45:06.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:06 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3869302889' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.02107422246760171}]: dispatch 2023-10-09T10:45:06.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:06 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.02107422246760171}]: dispatch 2023-10-09T10:45:06.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:06 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3058779803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:06.860 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:06.895 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:07.015 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:07.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:07.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:07.120 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:07.150 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:07.194 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:45:07.194 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:45:07.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:07.194 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "MOSDPGPush(6.10 115/110 [PushOp(6:0a26db6b:::benchmark_data_smithi125_244951_object33128:head, version: 106'2070, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:0a26db6b:::benchmark_data_smithi125_244951_object33128:head@106'2070, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:45:07.194 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:07.194010+0000", 2023-10-09T10:45:07.195 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.00023994500000000001, 2023-10-09T10:45:07.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00027270899999999999, 2023-10-09T10:45:07.195 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:07.195 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:45:07.195 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:07.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:07.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:07.195 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:07.194010+0000", 2023-10-09T10:45:07.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:07.195 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:07.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:07.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:07.195 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:07.194010+0000", 2023-10-09T10:45:07.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:07.196 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:07.196 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:07.196 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:07.196 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:07.193994+0000", 2023-10-09T10:45:07.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999838 2023-10-09T10:45:07.196 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:07.196 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:07.196 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:07.196 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:07.194063+0000", 2023-10-09T10:45:07.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9286e-05 2023-10-09T10:45:07.196 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:07.196 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:07.197 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:07.197 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:07.194081+0000", 2023-10-09T10:45:07.197 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7920000000000001e-05 2023-10-09T10:45:07.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:07.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:07.197 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:07.197 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:07.194093+0000", 2023-10-09T10:45:07.197 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2245999999999999e-05 2023-10-09T10:45:07.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:07.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:07.197 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:07.197 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:07.194173+0000", 2023-10-09T10:45:07.198 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9707000000000001e-05 2023-10-09T10:45:07.198 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:07.198 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:07.198 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:07.198 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:07.194181+0000", 2023-10-09T10:45:07.198 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0870000000000007e-06 2023-10-09T10:45:07.198 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:07.198 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:07.198 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:07.198 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:07.198 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T10:45:07.198 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 1 2023-10-09T10:45:07.199 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:45:07.215 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:45:07.222 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:07.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:07.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:07.300 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 35 16 55064 55048 98.2609 52.4375 0.00175006 0.0100123 2023-10-09T10:45:07.327 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:07.357 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:07.478 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:07.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:07.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:07.526 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:07.565 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:45:07.582 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:07.654 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:07.687 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:07.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:07.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:07.793 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:07.846 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:07.879 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:07.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:07.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:07.887 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:07 smithi155 ceph-mon[184854]: 6.5 scrub starts 2023-10-09T10:45:07.887 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:07 smithi155 ceph-mon[184854]: 6.12 scrub starts 2023-10-09T10:45:07.887 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:07 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.02107422246760171}]': finished 2023-10-09T10:45:07.887 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:07 smithi155 ceph-mon[184854]: osdmap e115: 8 total, 8 up, 8 in 2023-10-09T10:45:07.887 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:07 smithi155 ceph-mon[184854]: failed to encode map e115 with expected crc 2023-10-09T10:45:07.887 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:07 smithi155 ceph-mon[184854]: pgmap v233: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s; 2708/157446 objects degraded (1.720%); 2948/157446 objects misplaced (1.872%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T10:45:07.887 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:07 smithi155 ceph-mon[184854]: osdmap e116: 8 total, 8 up, 8 in 2023-10-09T10:45:07.966 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:45:07.984 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:08.027 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:08.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:07 smithi125 ceph-mon[195507]: 6.5 scrub starts 2023-10-09T10:45:08.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:07 smithi125 ceph-mon[195507]: 6.12 scrub starts 2023-10-09T10:45:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:07 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.02107422246760171}]': finished 2023-10-09T10:45:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:07 smithi125 ceph-mon[195507]: osdmap e115: 8 total, 8 up, 8 in 2023-10-09T10:45:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:07 smithi125 ceph-mon[195507]: failed to encode map e115 with expected crc 2023-10-09T10:45:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:07 smithi125 ceph-mon[195507]: pgmap v233: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s; 2708/157446 objects degraded (1.720%); 2948/157446 objects misplaced (1.872%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T10:45:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:07 smithi125 ceph-mon[195507]: osdmap e116: 8 total, 8 up, 8 in 2023-10-09T10:45:08.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:07 smithi125 ceph-mon[200197]: 6.5 scrub starts 2023-10-09T10:45:08.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:07 smithi125 ceph-mon[200197]: 6.12 scrub starts 2023-10-09T10:45:08.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:07 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.02107422246760171}]': finished 2023-10-09T10:45:08.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:07 smithi125 ceph-mon[200197]: osdmap e115: 8 total, 8 up, 8 in 2023-10-09T10:45:08.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:07 smithi125 ceph-mon[200197]: failed to encode map e115 with expected crc 2023-10-09T10:45:08.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:07 smithi125 ceph-mon[200197]: pgmap v233: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s; 2708/157446 objects degraded (1.720%); 2948/157446 objects misplaced (1.872%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T10:45:08.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:07 smithi125 ceph-mon[200197]: osdmap e116: 8 total, 8 up, 8 in 2023-10-09T10:45:08.300 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 36 16 56128 56112 97.3787 66.5 0.0116241 0.0102646 2023-10-09T10:45:08.610 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:08.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:08.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:08.714 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:08.771 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:08.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:08 smithi155 ceph-mon[184854]: 6.8 scrub starts 2023-10-09T10:45:08.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:08 smithi155 ceph-mon[184854]: 6.13 scrub starts 2023-10-09T10:45:08.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:08 smithi155 ceph-mon[184854]: 6.13 scrub ok 2023-10-09T10:45:08.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:08 smithi155 ceph-mon[184854]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:08 smithi155 ceph-mon[184854]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:08 smithi155 ceph-mon[184854]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:08 smithi155 ceph-mon[184854]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:08 smithi155 ceph-mon[184854]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:08 smithi155 ceph-mon[184854]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.931 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:45:08.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[195507]: 6.8 scrub starts 2023-10-09T10:45:08.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[195507]: 6.13 scrub starts 2023-10-09T10:45:08.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[195507]: 6.13 scrub ok 2023-10-09T10:45:08.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[195507]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[195507]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[195507]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[195507]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[195507]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[195507]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.972 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[200197]: 6.8 scrub starts 2023-10-09T10:45:08.972 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[200197]: 6.13 scrub starts 2023-10-09T10:45:08.972 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[200197]: 6.13 scrub ok 2023-10-09T10:45:08.972 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[200197]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.972 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[200197]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.972 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[200197]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.972 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[200197]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.972 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[200197]: failed to encode map e116 with expected crc 2023-10-09T10:45:08.972 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:08 smithi125 ceph-mon[200197]: failed to encode map e116 with expected crc 2023-10-09T10:45:09.300 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 37 16 57312 57296 96.7468 74 0.00173492 0.0102178 2023-10-09T10:45:09.449 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:09.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:09.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:09.506 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:45:09.507 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:45:09.507 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:45:09.507 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:45:09.507 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:45:09.554 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:09.581 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:09.620 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:09.658 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:09.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:09.700 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:09 smithi125 ceph-mon[195507]: failed to encode map e117 with expected crc 2023-10-09T10:45:09.700 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:09 smithi125 ceph-mon[195507]: osdmap e117: 8 total, 8 up, 8 in 2023-10-09T10:45:09.700 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:09 smithi125 ceph-mon[195507]: pgmap v236: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 2708/163434 objects degraded (1.657%); 1670/163434 objects misplaced (1.022%); 9.0 MiB/s, 143 objects/s recovering 2023-10-09T10:45:09.700 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:09 smithi125 ceph-mon[200197]: failed to encode map e117 with expected crc 2023-10-09T10:45:09.700 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:09 smithi125 ceph-mon[200197]: osdmap e117: 8 total, 8 up, 8 in 2023-10-09T10:45:09.700 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:09 smithi125 ceph-mon[200197]: pgmap v236: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 2708/163434 objects degraded (1.657%); 1670/163434 objects misplaced (1.022%); 9.0 MiB/s, 143 objects/s recovering 2023-10-09T10:45:09.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:09.802 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:09.844 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:09 smithi155 ceph-mon[184854]: failed to encode map e117 with expected crc 2023-10-09T10:45:09.845 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:09 smithi155 ceph-mon[184854]: osdmap e117: 8 total, 8 up, 8 in 2023-10-09T10:45:09.845 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:09 smithi155 ceph-mon[184854]: pgmap v236: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 2708/163434 objects degraded (1.657%); 1670/163434 objects misplaced (1.022%); 9.0 MiB/s, 143 objects/s recovering 2023-10-09T10:45:09.858 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:09.891 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:45:09.892 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:09.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:09.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:09.999 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:10.050 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:10.187 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:10.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:10.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:10.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:10.300 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 38 16 58247 58231 95.7388 58.4375 0.0094002 0.0104406 2023-10-09T10:45:10.313 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:45:10.340 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:10.420 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:10.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:10.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:10.528 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:10.562 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:10.639 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:10.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:10.743 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:10.823 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:10.880 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:10 smithi155 ceph-mon[184854]: 6.2 scrub starts 2023-10-09T10:45:10.880 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:10 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:45:10.880 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:10 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 2708/163434 objects degraded (1.657%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:45:10.880 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:10 smithi155 ceph-mon[184854]: failed to encode map e118 with expected crc 2023-10-09T10:45:10.880 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:10 smithi155 ceph-mon[184854]: failed to encode map e118 with expected crc 2023-10-09T10:45:10.881 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:10 smithi155 ceph-mon[184854]: osdmap e118: 8 total, 8 up, 8 in 2023-10-09T10:45:10.881 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:10 smithi155 ceph-mon[184854]: failed to encode map e118 with expected crc 2023-10-09T10:45:10.881 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:10 smithi155 ceph-mon[184854]: failed to encode map e118 with expected crc 2023-10-09T10:45:10.881 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:10 smithi155 ceph-mon[184854]: failed to encode map e118 with expected crc 2023-10-09T10:45:10.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[200197]: 6.2 scrub starts 2023-10-09T10:45:10.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:45:10.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 2708/163434 objects degraded (1.657%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:45:10.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[200197]: failed to encode map e118 with expected crc 2023-10-09T10:45:10.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[200197]: failed to encode map e118 with expected crc 2023-10-09T10:45:10.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[200197]: osdmap e118: 8 total, 8 up, 8 in 2023-10-09T10:45:10.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[200197]: failed to encode map e118 with expected crc 2023-10-09T10:45:10.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[200197]: failed to encode map e118 with expected crc 2023-10-09T10:45:10.936 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[200197]: failed to encode map e118 with expected crc 2023-10-09T10:45:10.936 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[195507]: 6.2 scrub starts 2023-10-09T10:45:10.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:45:10.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 2708/163434 objects degraded (1.657%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:45:10.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[195507]: failed to encode map e118 with expected crc 2023-10-09T10:45:10.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[195507]: failed to encode map e118 with expected crc 2023-10-09T10:45:10.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[195507]: osdmap e118: 8 total, 8 up, 8 in 2023-10-09T10:45:10.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[195507]: failed to encode map e118 with expected crc 2023-10-09T10:45:10.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[195507]: failed to encode map e118 with expected crc 2023-10-09T10:45:10.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:10 smithi125 ceph-mon[195507]: failed to encode map e118 with expected crc 2023-10-09T10:45:11.008 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:11.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:11.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:11.111 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:11.158 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:11.185 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:11.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:11.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:11.289 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:11.301 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 39 16 59540 59524 95.3533 80.8125 0.0125173 0.0104825 2023-10-09T10:45:11.341 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:11.370 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:11.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:11.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:11.415 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:45:11.415 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:45:11.415 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:45:11.415 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:45:11.415 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.415 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:1084 6.7 6:e18689b9:::benchmark_data_smithi125_244951_object1083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e101)", 2023-10-09T10:45:11.415 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:32.604614+0000", 2023-10-09T10:45:11.415 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 38.807139167999999, 2023-10-09T10:45:11.415 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026147479000000001, 2023-10-09T10:45:11.415 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.416 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.416 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.416 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.416 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.416 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 1084 2023-10-09T10:45:11.416 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.416 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.416 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.416 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.416 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.604614+0000", 2023-10-09T10:45:11.416 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.416 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.416 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.417 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.417 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.604614+0000", 2023-10-09T10:45:11.417 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.417 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.417 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.417 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.417 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.604614+0000", 2023-10-09T10:45:11.417 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:11.417 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.417 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.417 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.417 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.604657+0000", 2023-10-09T10:45:11.417 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3377000000000002e-05 2023-10-09T10:45:11.418 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.418 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.418 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.418 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.604667+0000", 2023-10-09T10:45:11.418 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7229999999999999e-06 2023-10-09T10:45:11.418 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.418 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.418 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.418 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.604670+0000", 2023-10-09T10:45:11.418 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3830000000000001e-06 2023-10-09T10:45:11.418 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.418 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.419 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.419 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.604716+0000", 2023-10-09T10:45:11.419 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5704999999999997e-05 2023-10-09T10:45:11.419 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.419 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.419 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.419 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.604822+0000", 2023-10-09T10:45:11.419 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106072 2023-10-09T10:45:11.419 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.419 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.419 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:45:11.420 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.604956+0000", 2023-10-09T10:45:11.420 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000133984 2023-10-09T10:45:11.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.420 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.420 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.420 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.606158+0000", 2023-10-09T10:45:11.420 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012021180000000001 2023-10-09T10:45:11.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.420 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.420 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.420 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.607352+0000", 2023-10-09T10:45:11.420 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011933480000000001 2023-10-09T10:45:11.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.421 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.421 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.630669+0000", 2023-10-09T10:45:11.421 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023317579000000001 2023-10-09T10:45:11.421 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.421 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.421 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.630740+0000", 2023-10-09T10:45:11.421 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1055000000000003e-05 2023-10-09T10:45:11.421 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.421 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.421 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.630762+0000", 2023-10-09T10:45:11.422 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1416e-05 2023-10-09T10:45:11.422 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.422 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.422 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.422 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.422 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.422 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:21359 6.d 6:b0b6555d:::benchmark_data_smithi125_244951_object21358:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:45:11.422 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:42.149926+0000", 2023-10-09T10:45:11.422 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 29.261827843999999, 2023-10-09T10:45:11.422 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.049235387999999998, 2023-10-09T10:45:11.422 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.422 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.423 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.423 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.423 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.423 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 21359 2023-10-09T10:45:11.423 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.423 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.423 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.423 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.423 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.149926+0000", 2023-10-09T10:45:11.423 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.423 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.424 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.424 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.424 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.149926+0000", 2023-10-09T10:45:11.424 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.424 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.424 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.424 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.424 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.149924+0000", 2023-10-09T10:45:11.424 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:45:11.424 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.424 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.424 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.425 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.150028+0000", 2023-10-09T10:45:11.425 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104262 2023-10-09T10:45:11.425 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.425 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.425 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.425 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.150051+0000", 2023-10-09T10:45:11.425 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3465e-05 2023-10-09T10:45:11.425 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.425 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.425 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.425 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.150058+0000", 2023-10-09T10:45:11.426 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.139e-06 2023-10-09T10:45:11.426 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.426 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.426 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.426 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.150177+0000", 2023-10-09T10:45:11.426 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000119485 2023-10-09T10:45:11.426 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.426 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.426 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.426 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.150288+0000", 2023-10-09T10:45:11.426 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000111146 2023-10-09T10:45:11.426 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.427 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.427 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:45:11.427 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.150419+0000", 2023-10-09T10:45:11.427 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013029900000000001 2023-10-09T10:45:11.427 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.427 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.427 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.427 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.151599+0000", 2023-10-09T10:45:11.427 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011805780000000001 2023-10-09T10:45:11.427 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.427 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.428 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.428 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.159012+0000", 2023-10-09T10:45:11.428 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0074126849999999996 2023-10-09T10:45:11.428 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.428 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.428 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.428 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.199073+0000", 2023-10-09T10:45:11.428 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040060996000000001 2023-10-09T10:45:11.428 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.428 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.428 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.428 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.199122+0000", 2023-10-09T10:45:11.428 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9104000000000003e-05 2023-10-09T10:45:11.429 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.429 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.429 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.429 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.199161+0000", 2023-10-09T10:45:11.429 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9035000000000001e-05 2023-10-09T10:45:11.429 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.429 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.429 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.429 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.429 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.429 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:21366 6.a 6:5b41f12e:::benchmark_data_smithi125_244951_object21365:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:45:11.429 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:42.156283+0000", 2023-10-09T10:45:11.430 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 29.255469989000002, 2023-10-09T10:45:11.430 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.042486267000000001, 2023-10-09T10:45:11.430 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.430 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.430 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.430 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.430 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.430 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 21366 2023-10-09T10:45:11.430 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.430 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.430 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.431 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.431 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.156283+0000", 2023-10-09T10:45:11.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.431 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.431 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.431 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.431 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.156283+0000", 2023-10-09T10:45:11.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.431 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.431 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.431 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.431 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.156282+0000", 2023-10-09T10:45:11.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:11.432 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.432 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.432 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.432 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.156324+0000", 2023-10-09T10:45:11.432 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1687999999999998e-05 2023-10-09T10:45:11.432 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.432 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.432 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.432 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.156346+0000", 2023-10-09T10:45:11.432 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2327000000000002e-05 2023-10-09T10:45:11.432 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.432 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.432 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.433 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.156352+0000", 2023-10-09T10:45:11.433 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.003e-06 2023-10-09T10:45:11.433 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.433 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.433 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.433 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.156401+0000", 2023-10-09T10:45:11.433 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8665000000000002e-05 2023-10-09T10:45:11.433 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.433 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.433 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.433 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.156548+0000", 2023-10-09T10:45:11.433 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000147811 2023-10-09T10:45:11.434 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.434 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.434 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:45:11.434 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.156690+0000", 2023-10-09T10:45:11.434 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014171499999999999 2023-10-09T10:45:11.434 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.434 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.434 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.434 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.158839+0000", 2023-10-09T10:45:11.434 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021490910000000001 2023-10-09T10:45:11.434 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.434 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.435 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.159626+0000", 2023-10-09T10:45:11.435 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00078711500000000002 2023-10-09T10:45:11.435 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.435 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.198677+0000", 2023-10-09T10:45:11.435 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039050856000000002 2023-10-09T10:45:11.435 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.435 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.198726+0000", 2023-10-09T10:45:11.435 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8739000000000002e-05 2023-10-09T10:45:11.436 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.436 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.436 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.436 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.198770+0000", 2023-10-09T10:45:11.436 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3682e-05 2023-10-09T10:45:11.436 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.436 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.436 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.436 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.436 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.436 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:21378 6.7 6:ed991cc4:::benchmark_data_smithi125_244951_object21377:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:45:11.436 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:42.160556+0000", 2023-10-09T10:45:11.436 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 29.251197033, 2023-10-09T10:45:11.437 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039208826000000002, 2023-10-09T10:45:11.437 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.437 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.437 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.437 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.437 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.437 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 21378 2023-10-09T10:45:11.437 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.437 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.437 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.437 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.437 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.160556+0000", 2023-10-09T10:45:11.437 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.438 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.438 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.438 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.438 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.160556+0000", 2023-10-09T10:45:11.438 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.438 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.438 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.438 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.438 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.160555+0000", 2023-10-09T10:45:11.438 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:45:11.438 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.438 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.439 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.439 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.160673+0000", 2023-10-09T10:45:11.439 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000117884 2023-10-09T10:45:11.439 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.439 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.439 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.439 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.160696+0000", 2023-10-09T10:45:11.439 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3039000000000001e-05 2023-10-09T10:45:11.439 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.439 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.439 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.440 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.160703+0000", 2023-10-09T10:45:11.440 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6270000000000002e-06 2023-10-09T10:45:11.440 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.440 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.440 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.440 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.160748+0000", 2023-10-09T10:45:11.440 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5312000000000003e-05 2023-10-09T10:45:11.440 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.440 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.440 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.440 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.160864+0000", 2023-10-09T10:45:11.440 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115952 2023-10-09T10:45:11.440 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.441 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.441 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:45:11.441 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.160984+0000", 2023-10-09T10:45:11.441 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000119614 2023-10-09T10:45:11.441 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.441 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.441 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.441 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.199652+0000", 2023-10-09T10:45:11.441 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038668853000000003 2023-10-09T10:45:11.441 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.441 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.441 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.441 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.199658+0000", 2023-10-09T10:45:11.442 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0220000000000001e-06 2023-10-09T10:45:11.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.199689+0000", 2023-10-09T10:45:11.442 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0267999999999999e-05 2023-10-09T10:45:11.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.199732+0000", 2023-10-09T10:45:11.442 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3424000000000003e-05 2023-10-09T10:45:11.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.443 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.443 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.443 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.199765+0000", 2023-10-09T10:45:11.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3015000000000002e-05 2023-10-09T10:45:11.443 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.443 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.443 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.443 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.443 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.443 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:21380 6.7 6:e5c3dcb4:::benchmark_data_smithi125_244951_object21379:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:45:11.443 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:42.161491+0000", 2023-10-09T10:45:11.443 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 29.25026214, 2023-10-09T10:45:11.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.043331895000000002, 2023-10-09T10:45:11.444 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.444 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.444 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.444 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.444 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.444 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 21380 2023-10-09T10:45:11.444 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.444 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.444 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.444 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.444 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.161491+0000", 2023-10-09T10:45:11.444 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.444 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.445 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.161491+0000", 2023-10-09T10:45:11.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.445 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.445 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.161490+0000", 2023-10-09T10:45:11.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:45:11.445 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.445 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.161561+0000", 2023-10-09T10:45:11.446 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0154000000000006e-05 2023-10-09T10:45:11.446 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.446 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.446 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.446 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.161572+0000", 2023-10-09T10:45:11.446 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1080000000000001e-05 2023-10-09T10:45:11.446 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.446 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.446 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.446 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.161578+0000", 2023-10-09T10:45:11.446 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1099999999999999e-06 2023-10-09T10:45:11.447 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.447 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.447 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.447 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.161723+0000", 2023-10-09T10:45:11.447 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000144931 2023-10-09T10:45:11.447 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.447 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.447 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.447 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.161837+0000", 2023-10-09T10:45:11.447 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000113829 2023-10-09T10:45:11.447 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.447 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.447 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:45:11.448 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.198723+0000", 2023-10-09T10:45:11.448 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.036886534999999998 2023-10-09T10:45:11.448 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.448 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.448 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.448 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.202839+0000", 2023-10-09T10:45:11.448 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041155920000000004 2023-10-09T10:45:11.448 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.448 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.448 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.448 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.203067+0000", 2023-10-09T10:45:11.448 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000228027 2023-10-09T10:45:11.448 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.449 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.449 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.204737+0000", 2023-10-09T10:45:11.449 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0016705960000000001 2023-10-09T10:45:11.449 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.449 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.449 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.204785+0000", 2023-10-09T10:45:11.449 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7302999999999998e-05 2023-10-09T10:45:11.449 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.449 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.450 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.204823+0000", 2023-10-09T10:45:11.450 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8563999999999999e-05 2023-10-09T10:45:11.450 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.450 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.450 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.450 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.450 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.450 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop_reply(client.44570.0:21378 6.7 e104/98 ondisk, result = 0)", 2023-10-09T10:45:11.450 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:42.162303+0000", 2023-10-09T10:45:11.450 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 29.249450013000001, 2023-10-09T10:45:11.450 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.037355846999999998, 2023-10-09T10:45:11.451 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.451 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:45:11.451 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.451 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.451 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.162303+0000", 2023-10-09T10:45:11.451 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.451 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.451 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.451 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.162303+0000", 2023-10-09T10:45:11.451 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.162303+0000", 2023-10-09T10:45:11.452 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:11.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.162307+0000", 2023-10-09T10:45:11.452 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2429999999999999e-06 2023-10-09T10:45:11.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.198607+0000", 2023-10-09T10:45:11.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.036299653000000001 2023-10-09T10:45:11.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.453 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.453 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.198625+0000", 2023-10-09T10:45:11.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8610999999999999e-05 2023-10-09T10:45:11.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.453 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.453 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.199641+0000", 2023-10-09T10:45:11.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001015981 2023-10-09T10:45:11.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.454 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.454 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.454 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.199647+0000", 2023-10-09T10:45:11.454 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9329999999999998e-06 2023-10-09T10:45:11.454 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.454 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.454 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.454 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.199659+0000", 2023-10-09T10:45:11.454 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.199e-05 2023-10-09T10:45:11.454 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.454 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.454 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.455 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.455 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.455 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:21381 6.c e104/98 6:3e452211:::benchmark_data_smithi125_244951_object21380:head v 104'1356, mlcod=104'1354)", 2023-10-09T10:45:11.455 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:42.162457+0000", 2023-10-09T10:45:11.455 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 29.24929685, 2023-10-09T10:45:11.455 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.041519371999999999, 2023-10-09T10:45:11.455 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.455 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.455 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.455 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.455 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.455 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.162457+0000", 2023-10-09T10:45:11.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.456 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.456 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.456 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.162457+0000", 2023-10-09T10:45:11.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.456 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.456 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.456 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.162456+0000", 2023-10-09T10:45:11.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:11.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.456 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.457 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.162576+0000", 2023-10-09T10:45:11.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000119963 2023-10-09T10:45:11.457 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.457 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.457 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.198612+0000", 2023-10-09T10:45:11.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.036035997 2023-10-09T10:45:11.457 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.457 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.457 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.198628+0000", 2023-10-09T10:45:11.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6410999999999999e-05 2023-10-09T10:45:11.458 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.458 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.458 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.199378+0000", 2023-10-09T10:45:11.458 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00074967499999999999 2023-10-09T10:45:11.458 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.458 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.458 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.199397+0000", 2023-10-09T10:45:11.458 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9414000000000001e-05 2023-10-09T10:45:11.458 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.458 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.203944+0000", 2023-10-09T10:45:11.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045462990000000002 2023-10-09T10:45:11.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.459 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.203976+0000", 2023-10-09T10:45:11.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2196999999999997e-05 2023-10-09T10:45:11.459 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.459 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.459 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.459 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop_reply(client.44570.0:21366 6.a e104/98 ondisk, result = 0)", 2023-10-09T10:45:11.460 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:42.163515+0000", 2023-10-09T10:45:11.460 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 29.248238381, 2023-10-09T10:45:11.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.035261764000000001, 2023-10-09T10:45:11.460 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.460 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:45:11.460 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.163515+0000", 2023-10-09T10:45:11.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.163515+0000", 2023-10-09T10:45:11.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.163514+0000", 2023-10-09T10:45:11.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:11.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.163519+0000", 2023-10-09T10:45:11.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5850000000000001e-06 2023-10-09T10:45:11.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.198610+0000", 2023-10-09T10:45:11.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.035091325999999999 2023-10-09T10:45:11.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.198622+0000", 2023-10-09T10:45:11.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1977999999999999e-05 2023-10-09T10:45:11.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.463 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.463 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.198662+0000", 2023-10-09T10:45:11.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9827999999999999e-05 2023-10-09T10:45:11.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.463 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.463 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.198671+0000", 2023-10-09T10:45:11.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8710000000000003e-06 2023-10-09T10:45:11.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.463 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.463 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.198777+0000", 2023-10-09T10:45:11.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000105816 2023-10-09T10:45:11.464 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.464 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.464 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.464 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:22470 6.d 6:b41d58d6:::benchmark_data_smithi125_244951_object22469:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:45:11.464 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:42.711148+0000", 2023-10-09T10:45:11.464 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 28.700605487000001, 2023-10-09T10:45:11.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022648462000000001, 2023-10-09T10:45:11.464 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.464 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.464 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.465 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.465 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.465 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 22470 2023-10-09T10:45:11.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.465 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.465 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.711148+0000", 2023-10-09T10:45:11.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.466 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.711148+0000", 2023-10-09T10:45:11.466 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.466 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.466 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.466 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.711147+0000", 2023-10-09T10:45:11.466 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:11.466 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.466 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.466 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.711177+0000", 2023-10-09T10:45:11.466 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9456000000000001e-05 2023-10-09T10:45:11.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.467 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.711184+0000", 2023-10-09T10:45:11.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8789999999999997e-06 2023-10-09T10:45:11.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.467 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.711188+0000", 2023-10-09T10:45:11.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3030000000000001e-06 2023-10-09T10:45:11.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.712023+0000", 2023-10-09T10:45:11.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00083509199999999995 2023-10-09T10:45:11.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.712130+0000", 2023-10-09T10:45:11.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106949 2023-10-09T10:45:11.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:45:11.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.712258+0000", 2023-10-09T10:45:11.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000127916 2023-10-09T10:45:11.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.469 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.469 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.713694+0000", 2023-10-09T10:45:11.469 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001435925 2023-10-09T10:45:11.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.469 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.469 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.730649+0000", 2023-10-09T10:45:11.469 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016955366999999999 2023-10-09T10:45:11.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.469 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.470 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.733732+0000", 2023-10-09T10:45:11.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030829299999999998 2023-10-09T10:45:11.470 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.470 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.470 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.470 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.733767+0000", 2023-10-09T10:45:11.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4841000000000001e-05 2023-10-09T10:45:11.470 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.470 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.470 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.470 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.733796+0000", 2023-10-09T10:45:11.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9380999999999999e-05 2023-10-09T10:45:11.470 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.471 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.471 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.471 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:22593 6.7 6:ea2dd48c:::benchmark_data_smithi125_244951_object22592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:45:11.471 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:42.795459+0000", 2023-10-09T10:45:11.471 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 28.616294397000001, 2023-10-09T10:45:11.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.032953246999999998, 2023-10-09T10:45:11.471 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.471 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.471 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.471 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.472 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.472 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 22593 2023-10-09T10:45:11.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.472 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.795459+0000", 2023-10-09T10:45:11.472 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.795459+0000", 2023-10-09T10:45:11.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.795458+0000", 2023-10-09T10:45:11.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:45:11.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.795627+0000", 2023-10-09T10:45:11.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016872299999999999 2023-10-09T10:45:11.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.795638+0000", 2023-10-09T10:45:11.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0917000000000001e-05 2023-10-09T10:45:11.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.795643+0000", 2023-10-09T10:45:11.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6940000000000003e-06 2023-10-09T10:45:11.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.795681+0000", 2023-10-09T10:45:11.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8661999999999997e-05 2023-10-09T10:45:11.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.475 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.475 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.795766+0000", 2023-10-09T10:45:11.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4893000000000002e-05 2023-10-09T10:45:11.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.475 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:45:11.475 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.795858+0000", 2023-10-09T10:45:11.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1566999999999999e-05 2023-10-09T10:45:11.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.476 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.476 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.796984+0000", 2023-10-09T10:45:11.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011260280000000001 2023-10-09T10:45:11.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.476 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.476 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.797200+0000", 2023-10-09T10:45:11.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000215918 2023-10-09T10:45:11.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.476 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.476 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.828343+0000", 2023-10-09T10:45:11.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.031143301000000002 2023-10-09T10:45:11.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.828377+0000", 2023-10-09T10:45:11.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3856000000000002e-05 2023-10-09T10:45:11.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.828412+0000", 2023-10-09T10:45:11.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5475000000000002e-05 2023-10-09T10:45:11.477 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.478 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.478 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.478 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:22884 6.7 6:e0681e98:::benchmark_data_smithi125_244951_object22883:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:45:11.478 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:43.034958+0000", 2023-10-09T10:45:11.478 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 28.376794951000001, 2023-10-09T10:45:11.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027421518999999998, 2023-10-09T10:45:11.478 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.478 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.478 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.478 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.478 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.479 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 22884 2023-10-09T10:45:11.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.479 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.034958+0000", 2023-10-09T10:45:11.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.034958+0000", 2023-10-09T10:45:11.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.034958+0000", 2023-10-09T10:45:11.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:45:11.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.035006+0000", 2023-10-09T10:45:11.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8049e-05 2023-10-09T10:45:11.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.035016+0000", 2023-10-09T10:45:11.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0672e-05 2023-10-09T10:45:11.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.035021+0000", 2023-10-09T10:45:11.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6190000000000002e-06 2023-10-09T10:45:11.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.035056+0000", 2023-10-09T10:45:11.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4615e-05 2023-10-09T10:45:11.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.035146+0000", 2023-10-09T10:45:11.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0736999999999995e-05 2023-10-09T10:45:11.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:45:11.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.035250+0000", 2023-10-09T10:45:11.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010386399999999999 2023-10-09T10:45:11.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.483 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.036164+0000", 2023-10-09T10:45:11.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00091379199999999997 2023-10-09T10:45:11.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.483 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.483 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.036422+0000", 2023-10-09T10:45:11.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025777299999999999 2023-10-09T10:45:11.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.483 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.483 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.062323+0000", 2023-10-09T10:45:11.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025901786999999999 2023-10-09T10:45:11.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.062355+0000", 2023-10-09T10:45:11.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1417000000000001e-05 2023-10-09T10:45:11.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.062380+0000", 2023-10-09T10:45:11.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5052e-05 2023-10-09T10:45:11.484 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.484 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.484 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.485 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:22891 6.7 6:e6aea7b8:::benchmark_data_smithi125_244951_object22890:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:45:11.485 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:43.037815+0000", 2023-10-09T10:45:11.485 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 28.373938047999999, 2023-10-09T10:45:11.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024701219, 2023-10-09T10:45:11.485 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.485 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.485 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.485 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.485 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.485 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 22891 2023-10-09T10:45:11.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.485 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.037815+0000", 2023-10-09T10:45:11.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.037815+0000", 2023-10-09T10:45:11.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.037798+0000", 2023-10-09T10:45:11.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999828 2023-10-09T10:45:11.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.037845+0000", 2023-10-09T10:45:11.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6851000000000003e-05 2023-10-09T10:45:11.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.037859+0000", 2023-10-09T10:45:11.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.393e-05 2023-10-09T10:45:11.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.037866+0000", 2023-10-09T10:45:11.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8650000000000003e-06 2023-10-09T10:45:11.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.037905+0000", 2023-10-09T10:45:11.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9722999999999999e-05 2023-10-09T10:45:11.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.038025+0000", 2023-10-09T10:45:11.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011943300000000001 2023-10-09T10:45:11.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:45:11.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.038169+0000", 2023-10-09T10:45:11.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000144019 2023-10-09T10:45:11.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.039332+0000", 2023-10-09T10:45:11.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001163049 2023-10-09T10:45:11.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.490 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.490 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.040222+0000", 2023-10-09T10:45:11.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00089030999999999995 2023-10-09T10:45:11.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.490 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.490 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.490 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.062419+0000", 2023-10-09T10:45:11.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022197070999999999 2023-10-09T10:45:11.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.490 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.490 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.490 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.062476+0000", 2023-10-09T10:45:11.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7169000000000001e-05 2023-10-09T10:45:11.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.062517+0000", 2023-10-09T10:45:11.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0067000000000001e-05 2023-10-09T10:45:11.491 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.491 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.491 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.491 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:24429 6.a 6:5dfa8424:::benchmark_data_smithi125_244951_object24428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:45:11.491 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:43.882167+0000", 2023-10-09T10:45:11.492 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 27.529585942000001, 2023-10-09T10:45:11.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024532286, 2023-10-09T10:45:11.492 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.492 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.492 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.492 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.492 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.492 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 24429 2023-10-09T10:45:11.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.492 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.882167+0000", 2023-10-09T10:45:11.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.882167+0000", 2023-10-09T10:45:11.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.882166+0000", 2023-10-09T10:45:11.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:45:11.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.882246+0000", 2023-10-09T10:45:11.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9282000000000004e-05 2023-10-09T10:45:11.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.882258+0000", 2023-10-09T10:45:11.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1946e-05 2023-10-09T10:45:11.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.882263+0000", 2023-10-09T10:45:11.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3129999999999998e-06 2023-10-09T10:45:11.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.882305+0000", 2023-10-09T10:45:11.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2307e-05 2023-10-09T10:45:11.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.882424+0000", 2023-10-09T10:45:11.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118502 2023-10-09T10:45:11.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:45:11.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.882577+0000", 2023-10-09T10:45:11.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015378700000000001 2023-10-09T10:45:11.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.883500+0000", 2023-10-09T10:45:11.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00092297199999999999 2023-10-09T10:45:11.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.883732+0000", 2023-10-09T10:45:11.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00023152000000000001 2023-10-09T10:45:11.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.906605+0000", 2023-10-09T10:45:11.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.02287341 2023-10-09T10:45:11.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.906658+0000", 2023-10-09T10:45:11.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2197999999999997e-05 2023-10-09T10:45:11.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.498 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.498 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.906700+0000", 2023-10-09T10:45:11.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2169000000000002e-05 2023-10-09T10:45:11.498 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.498 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.498 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.498 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:24434 6.a 6:578de90e:::benchmark_data_smithi125_244951_object24433:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:45:11.498 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:43.889632+0000", 2023-10-09T10:45:11.499 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 27.522121468999998, 2023-10-09T10:45:11.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022145304000000001, 2023-10-09T10:45:11.499 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.499 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.499 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.499 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.499 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.499 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 24434 2023-10-09T10:45:11.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.499 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.889632+0000", 2023-10-09T10:45:11.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.889632+0000", 2023-10-09T10:45:11.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.889631+0000", 2023-10-09T10:45:11.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:45:11.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.889680+0000", 2023-10-09T10:45:11.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8909999999999998e-05 2023-10-09T10:45:11.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.889690+0000", 2023-10-09T10:45:11.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0428000000000001e-05 2023-10-09T10:45:11.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.889695+0000", 2023-10-09T10:45:11.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5129999999999998e-06 2023-10-09T10:45:11.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.889739+0000", 2023-10-09T10:45:11.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4345999999999999e-05 2023-10-09T10:45:11.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.889859+0000", 2023-10-09T10:45:11.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000119262 2023-10-09T10:45:11.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:45:11.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.889994+0000", 2023-10-09T10:45:11.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013496299999999999 2023-10-09T10:45:11.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.891846+0000", 2023-10-09T10:45:11.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018526059999999999 2023-10-09T10:45:11.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.892004+0000", 2023-10-09T10:45:11.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000157725 2023-10-09T10:45:11.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.911690+0000", 2023-10-09T10:45:11.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019686565 2023-10-09T10:45:11.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.911741+0000", 2023-10-09T10:45:11.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0309999999999998e-05 2023-10-09T10:45:11.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.505 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.505 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.911777+0000", 2023-10-09T10:45:11.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6464000000000002e-05 2023-10-09T10:45:11.505 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.505 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.505 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.505 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:24436 6.a 6:5d09b924:::benchmark_data_smithi125_244951_object24435:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:45:11.505 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:43.889703+0000", 2023-10-09T10:45:11.506 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 27.522049943999999, 2023-10-09T10:45:11.506 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022349379999999999, 2023-10-09T10:45:11.506 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.506 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.506 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.506 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.506 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.506 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 24436 2023-10-09T10:45:11.506 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.506 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.506 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.506 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.507 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.889703+0000", 2023-10-09T10:45:11.507 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.507 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.507 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.889703+0000", 2023-10-09T10:45:11.507 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.507 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.507 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.889703+0000", 2023-10-09T10:45:11.507 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:11.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.508 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.508 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.508 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.889795+0000", 2023-10-09T10:45:11.508 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2138000000000005e-05 2023-10-09T10:45:11.508 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.508 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.508 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.508 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.889811+0000", 2023-10-09T10:45:11.508 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5821e-05 2023-10-09T10:45:11.508 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.508 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.508 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.509 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.889817+0000", 2023-10-09T10:45:11.509 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5809999999999996e-06 2023-10-09T10:45:11.509 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.509 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.890841+0000", 2023-10-09T10:45:11.509 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001023999 2023-10-09T10:45:11.509 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.509 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.890951+0000", 2023-10-09T10:45:11.509 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000109937 2023-10-09T10:45:11.509 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.510 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.510 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:45:11.510 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.891098+0000", 2023-10-09T10:45:11.510 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014735099999999999 2023-10-09T10:45:11.510 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.510 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.510 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.510 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.892009+0000", 2023-10-09T10:45:11.510 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00091079300000000002 2023-10-09T10:45:11.510 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.510 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.510 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.511 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.892141+0000", 2023-10-09T10:45:11.511 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013189399999999999 2023-10-09T10:45:11.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.511 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.511 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.511 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.911974+0000", 2023-10-09T10:45:11.511 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019833157000000001 2023-10-09T10:45:11.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.511 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.511 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.511 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.912018+0000", 2023-10-09T10:45:11.512 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4032000000000001e-05 2023-10-09T10:45:11.512 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:43.912053+0000", 2023-10-09T10:45:11.512 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5021999999999998e-05 2023-10-09T10:45:11.512 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.512 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.512 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.512 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.512 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:39918 6.7 6:e02d5c49:::benchmark_data_smithi125_244951_object39917:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T10:45:11.512 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:53.589524+0000", 2023-10-09T10:45:11.513 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 17.822228924000001, 2023-10-09T10:45:11.513 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024124699999999999, 2023-10-09T10:45:11.513 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.513 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.513 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.513 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.513 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.513 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39918 2023-10-09T10:45:11.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.513 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.513 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.513 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.589524+0000", 2023-10-09T10:45:11.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.514 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.514 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.589524+0000", 2023-10-09T10:45:11.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.514 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.514 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.589522+0000", 2023-10-09T10:45:11.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:45:11.514 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.515 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.515 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.589609+0000", 2023-10-09T10:45:11.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6823000000000005e-05 2023-10-09T10:45:11.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.515 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.515 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.589627+0000", 2023-10-09T10:45:11.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7859e-05 2023-10-09T10:45:11.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.515 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.515 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.589635+0000", 2023-10-09T10:45:11.516 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2029999999999992e-06 2023-10-09T10:45:11.516 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.516 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.516 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.516 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.589835+0000", 2023-10-09T10:45:11.516 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000200192 2023-10-09T10:45:11.516 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.516 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.516 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.516 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.589974+0000", 2023-10-09T10:45:11.516 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013888600000000001 2023-10-09T10:45:11.516 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.517 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:45:11.517 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.590120+0000", 2023-10-09T10:45:11.517 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000145537 2023-10-09T10:45:11.517 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.517 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.517 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.592053+0000", 2023-10-09T10:45:11.517 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019333519999999999 2023-10-09T10:45:11.517 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.518 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.518 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.593935+0000", 2023-10-09T10:45:11.518 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001881726 2023-10-09T10:45:11.518 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.518 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.518 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.518 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.613574+0000", 2023-10-09T10:45:11.518 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019639090000000001 2023-10-09T10:45:11.518 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.518 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.518 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.518 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.613617+0000", 2023-10-09T10:45:11.518 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2565000000000003e-05 2023-10-09T10:45:11.519 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.519 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.519 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.519 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.613649+0000", 2023-10-09T10:45:11.519 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2530999999999997e-05 2023-10-09T10:45:11.519 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.519 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.519 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.519 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.519 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.519 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:39921 6.17 6:eecd3c2f:::benchmark_data_smithi125_244951_object39920:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T10:45:11.519 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:53.589675+0000", 2023-10-09T10:45:11.520 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 17.822077966999998, 2023-10-09T10:45:11.520 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024118903000000001, 2023-10-09T10:45:11.520 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.520 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.520 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.520 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.520 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.520 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39921 2023-10-09T10:45:11.520 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.520 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.520 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.520 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.521 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.589675+0000", 2023-10-09T10:45:11.521 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.521 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.521 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.589675+0000", 2023-10-09T10:45:11.521 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.521 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.521 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.589675+0000", 2023-10-09T10:45:11.521 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:11.522 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.522 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.522 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.522 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.589738+0000", 2023-10-09T10:45:11.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2872000000000004e-05 2023-10-09T10:45:11.522 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.522 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.522 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.522 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.589755+0000", 2023-10-09T10:45:11.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7139999999999999e-05 2023-10-09T10:45:11.522 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.523 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.589760+0000", 2023-10-09T10:45:11.523 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4729999999999999e-06 2023-10-09T10:45:11.523 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.523 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.590978+0000", 2023-10-09T10:45:11.523 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001218027 2023-10-09T10:45:11.523 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.525 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.525 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.591086+0000", 2023-10-09T10:45:11.525 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000108 2023-10-09T10:45:11.525 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.525 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.525 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:45:11.525 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.591211+0000", 2023-10-09T10:45:11.525 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012455999999999999 2023-10-09T10:45:11.525 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.525 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.526 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.526 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.592396+0000", 2023-10-09T10:45:11.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011852189999999999 2023-10-09T10:45:11.526 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.526 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.526 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.526 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.595037+0000", 2023-10-09T10:45:11.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026406279999999999 2023-10-09T10:45:11.526 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.526 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.526 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.527 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.613742+0000", 2023-10-09T10:45:11.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018705741000000001 2023-10-09T10:45:11.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.527 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.527 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.527 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.613770+0000", 2023-10-09T10:45:11.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7960999999999999e-05 2023-10-09T10:45:11.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.527 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.527 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.527 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.613794+0000", 2023-10-09T10:45:11.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3983000000000001e-05 2023-10-09T10:45:11.528 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.528 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.528 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.528 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.528 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:43910 6.d 6:b519bad3:::benchmark_data_smithi125_244951_object43909:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2023-10-09T10:45:11.528 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:59.192667+0000", 2023-10-09T10:45:11.528 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 12.219086169000001, 2023-10-09T10:45:11.528 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022535241000000001, 2023-10-09T10:45:11.528 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.528 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.528 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.529 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.529 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.529 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 43910 2023-10-09T10:45:11.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.529 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.529 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.192667+0000", 2023-10-09T10:45:11.529 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.529 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.192667+0000", 2023-10-09T10:45:11.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.530 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.530 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.192663+0000", 2023-10-09T10:45:11.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999957 2023-10-09T10:45:11.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.530 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.530 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.192738+0000", 2023-10-09T10:45:11.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5406000000000003e-05 2023-10-09T10:45:11.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.531 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.531 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.531 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.192748+0000", 2023-10-09T10:45:11.531 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6190000000000006e-06 2023-10-09T10:45:11.531 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.531 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.531 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.531 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.192752+0000", 2023-10-09T10:45:11.531 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2699999999999998e-06 2023-10-09T10:45:11.531 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.531 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.531 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.532 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.192809+0000", 2023-10-09T10:45:11.532 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6975999999999998e-05 2023-10-09T10:45:11.532 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.532 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.532 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.532 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.192925+0000", 2023-10-09T10:45:11.532 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011633099999999999 2023-10-09T10:45:11.532 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.532 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.532 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:45:11.532 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.193067+0000", 2023-10-09T10:45:11.532 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000141588 2023-10-09T10:45:11.532 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.533 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.533 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.533 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.194413+0000", 2023-10-09T10:45:11.533 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001345962 2023-10-09T10:45:11.533 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.533 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.533 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.533 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.198732+0000", 2023-10-09T10:45:11.533 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043193379999999998 2023-10-09T10:45:11.533 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.533 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.534 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.534 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.215120+0000", 2023-10-09T10:45:11.534 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016387616000000001 2023-10-09T10:45:11.534 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.534 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.534 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.534 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.215167+0000", 2023-10-09T10:45:11.534 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6823999999999998e-05 2023-10-09T10:45:11.534 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.534 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.534 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.534 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.215202+0000", 2023-10-09T10:45:11.535 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5799000000000001e-05 2023-10-09T10:45:11.535 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.535 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.535 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.535 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.535 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.535 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:44091 6.d 6:be3b006a:::benchmark_data_smithi125_244951_object44090:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2023-10-09T10:45:11.535 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:59.350670+0000", 2023-10-09T10:45:11.535 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 12.061082984, 2023-10-09T10:45:11.535 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024362460999999998, 2023-10-09T10:45:11.535 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.535 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.535 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.536 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.536 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.536 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 44091 2023-10-09T10:45:11.536 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.536 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.536 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.536 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.536 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.350670+0000", 2023-10-09T10:45:11.536 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.536 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.536 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.536 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.536 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.350670+0000", 2023-10-09T10:45:11.537 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.537 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.537 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.350668+0000", 2023-10-09T10:45:11.537 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T10:45:11.537 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.537 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.350725+0000", 2023-10-09T10:45:11.537 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7513000000000003e-05 2023-10-09T10:45:11.537 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.538 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.350741+0000", 2023-10-09T10:45:11.538 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5461e-05 2023-10-09T10:45:11.538 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.538 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.538 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.350746+0000", 2023-10-09T10:45:11.538 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4859999999999997e-06 2023-10-09T10:45:11.538 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.538 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.538 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.539 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.350791+0000", 2023-10-09T10:45:11.539 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4737000000000002e-05 2023-10-09T10:45:11.539 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.539 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.539 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.539 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.350887+0000", 2023-10-09T10:45:11.539 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5820000000000001e-05 2023-10-09T10:45:11.539 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.539 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.539 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:45:11.539 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.350992+0000", 2023-10-09T10:45:11.539 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010492099999999999 2023-10-09T10:45:11.539 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.540 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.540 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.352597+0000", 2023-10-09T10:45:11.540 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00160545 2023-10-09T10:45:11.540 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.540 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.540 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.355109+0000", 2023-10-09T10:45:11.540 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0025117960000000002 2023-10-09T10:45:11.540 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.540 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.374935+0000", 2023-10-09T10:45:11.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019825412000000001 2023-10-09T10:45:11.541 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.541 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.541 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.374988+0000", 2023-10-09T10:45:11.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3936e-05 2023-10-09T10:45:11.541 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.541 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.541 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:59.375033+0000", 2023-10-09T10:45:11.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4369000000000002e-05 2023-10-09T10:45:11.541 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.542 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.542 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.542 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.542 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.542 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:59584 6.a 6:5e25878d:::benchmark_data_smithi125_244951_object59583:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-10-09T10:45:11.542 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:11.334699+0000", 2023-10-09T10:45:11.542 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.077054028999999996, 2023-10-09T10:45:11.542 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024863695000000002, 2023-10-09T10:45:11.542 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:11.542 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:11.542 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:11.542 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:11.543 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:11.543 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 59584 2023-10-09T10:45:11.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.543 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:11.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:11.543 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:11.334699+0000", 2023-10-09T10:45:11.543 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:11.543 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:11.334699+0000", 2023-10-09T10:45:11.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:11.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:11.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:11.334698+0000", 2023-10-09T10:45:11.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:45:11.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:11.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:11.334796+0000", 2023-10-09T10:45:11.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7393999999999996e-05 2023-10-09T10:45:11.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:11.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:11.334809+0000", 2023-10-09T10:45:11.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2886e-05 2023-10-09T10:45:11.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:11.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:11.334815+0000", 2023-10-09T10:45:11.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4320000000000004e-06 2023-10-09T10:45:11.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:11.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:11.338257+0000", 2023-10-09T10:45:11.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034415689999999998 2023-10-09T10:45:11.546 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.546 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.546 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:11.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:11.338378+0000", 2023-10-09T10:45:11.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012108900000000001 2023-10-09T10:45:11.546 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.546 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.546 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:45:11.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:11.338508+0000", 2023-10-09T10:45:11.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012992099999999999 2023-10-09T10:45:11.546 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.547 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:11.547 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:11.341421+0000", 2023-10-09T10:45:11.547 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029138290000000002 2023-10-09T10:45:11.547 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.547 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.547 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:11.345277+0000", 2023-10-09T10:45:11.547 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003855958 2023-10-09T10:45:11.547 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.547 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:11.547 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:11.359490+0000", 2023-10-09T10:45:11.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014212862 2023-10-09T10:45:11.548 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.548 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.548 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:11.548 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:11.359536+0000", 2023-10-09T10:45:11.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6249999999999999e-05 2023-10-09T10:45:11.548 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:11.548 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:11.548 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:11.548 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:11.359563+0000", 2023-10-09T10:45:11.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6576000000000001e-05 2023-10-09T10:45:11.548 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.548 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.549 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.549 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:11.549 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:11.549 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:45:11.549 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:11.550 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:45:11.555 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:45:11.574 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:45:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:45:11] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2023-10-09T10:45:11.588 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:45:11.592 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:11.621 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:45:11.653 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:45:11.694 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:45:11.719 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:11.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:11.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:11.724 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:45:11.730 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:45:11.784 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:45:11.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:45:11.801 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:45:11.801 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:45:11.801 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:45:11.801 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:45:11.801 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:45:11.802 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:45:11.802 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:45:11.802 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:45:11.824 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:11.881 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:11.912 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:11.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:11.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:11 smithi155 ceph-mon[184854]: failed to encode map e118 with expected crc 2023-10-09T10:45:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:11 smithi155 ceph-mon[184854]: pgmap v238: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 2708/171369 objects degraded (1.580%); 1670/171369 objects misplaced (0.975%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T10:45:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:11 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/237579685' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:11 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1434420480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:12.019 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:12.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:11 smithi125 ceph-mon[200197]: failed to encode map e118 with expected crc 2023-10-09T10:45:12.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:11 smithi125 ceph-mon[200197]: pgmap v238: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 2708/171369 objects degraded (1.580%); 1670/171369 objects misplaced (0.975%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T10:45:12.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:11 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/237579685' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:12.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:11 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1434420480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:12.042 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:11 smithi125 ceph-mon[195507]: failed to encode map e118 with expected crc 2023-10-09T10:45:12.042 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:11 smithi125 ceph-mon[195507]: pgmap v238: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 2708/171369 objects degraded (1.580%); 1670/171369 objects misplaced (0.975%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T10:45:12.042 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:11 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/237579685' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:12.042 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:11 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1434420480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:12.075 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:12.176 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:45:12.284 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:45:12.284 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:45:12.284 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:45:12.284 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd out 0 2023-10-09T10:45:12.302 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:45:12.302724+0000 min lat: 0.00102371 max lat: 2.05736 avg lat: 0.0105126 2023-10-09T10:45:12.302 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:45:12.302 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 40 16 60892 60876 95.0806 84.5 0.0212391 0.0105126 2023-10-09T10:45:12.408 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:12.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:12.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:12.511 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:12.550 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:12.712 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:12 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:45:12.712 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:12 smithi125 ceph-mon[195507]: failed to encode map e119 with expected crc 2023-10-09T10:45:12.712 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:12 smithi125 ceph-mon[195507]: failed to encode map e119 with expected crc 2023-10-09T10:45:12.712 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:12 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/237579685' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:45:12.713 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:12 smithi125 ceph-mon[195507]: osdmap e119: 8 total, 8 up, 8 in 2023-10-09T10:45:12.737 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:12.807 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:12 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:45:12.807 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:12 smithi155 ceph-mon[184854]: failed to encode map e119 with expected crc 2023-10-09T10:45:12.807 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:12 smithi155 ceph-mon[184854]: failed to encode map e119 with expected crc 2023-10-09T10:45:12.807 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:12 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/237579685' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:45:12.807 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:12 smithi155 ceph-mon[184854]: osdmap e119: 8 total, 8 up, 8 in 2023-10-09T10:45:12.829 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:12.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:12.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:12.932 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:12.982 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:12 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:45:12.982 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:12 smithi125 ceph-mon[200197]: failed to encode map e119 with expected crc 2023-10-09T10:45:12.983 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:12 smithi125 ceph-mon[200197]: failed to encode map e119 with expected crc 2023-10-09T10:45:12.983 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:12 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/237579685' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:45:12.983 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:12 smithi125 ceph-mon[200197]: osdmap e119: 8 total, 8 up, 8 in 2023-10-09T10:45:12.988 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:13.024 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:13.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:13.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:13.129 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:13.182 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:13.211 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:13.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:13.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:13.293 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:45:13.293 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:45:13.303 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 41 16 62227 62211 94.7946 83.4375 0.0105797 0.0105447 2023-10-09T10:45:13.317 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:13.368 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:13.595 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:13.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:13.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:13.683 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:45:13.698 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:13.774 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:13.865 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:13.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:13.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:13 smithi155 ceph-mon[184854]: 6.18 scrub starts 2023-10-09T10:45:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:13 smithi155 ceph-mon[184854]: pgmap v240: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 42 MiB/s wr, 678 op/s; 3459/175734 objects misplaced (1.968%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T10:45:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:13 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T10:45:13.972 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:14.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:13 smithi125 ceph-mon[195507]: 6.18 scrub starts 2023-10-09T10:45:14.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:13 smithi125 ceph-mon[195507]: pgmap v240: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 42 MiB/s wr, 678 op/s; 3459/175734 objects misplaced (1.968%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T10:45:14.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:13 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T10:45:14.013 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:13 smithi125 ceph-mon[200197]: 6.18 scrub starts 2023-10-09T10:45:14.013 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:13 smithi125 ceph-mon[200197]: pgmap v240: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 42 MiB/s wr, 678 op/s; 3459/175734 objects misplaced (1.968%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T10:45:14.013 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:13 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T10:45:14.016 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:14.027 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:45:14.043 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:14.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:14.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:14.148 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:14.179 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:14.204 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:14.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:14.231 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:45:14.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T10:45:14.207+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e120 definitely_dead 0 2023-10-09T10:45:14.303 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 42 16 63227 63211 94.0258 62.5 0.00212743 0.0105372 2023-10-09T10:45:14.307 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:14.342 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:14.368 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:14.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:14.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:14.422 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:45:14.472 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:14.507 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:14.573 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:14.581 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:14.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:14.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:14.688 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:14.726 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:14.749 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:14.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:14.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:14.764 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.0. 2023-10-09T10:45:14.853 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:14.870 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:45:14.870 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[195507]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2708/171369 objects degraded (1.580%), 1 pg degraded) 2023-10-09T10:45:14.870 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T10:45:14.870 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[195507]: failed to encode map e120 with expected crc 2023-10-09T10:45:14.870 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T10:45:14.870 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[195507]: osdmap e120: 8 total, 8 up, 8 in 2023-10-09T10:45:14.870 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[195507]: failed to encode map e120 with expected crc 2023-10-09T10:45:14.870 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3792962386' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T10:45:14.870 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[195507]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T10:45:14.871 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:45:14.871 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[200197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2708/171369 objects degraded (1.580%), 1 pg degraded) 2023-10-09T10:45:14.871 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T10:45:14.871 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[200197]: failed to encode map e120 with expected crc 2023-10-09T10:45:14.871 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T10:45:14.871 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[200197]: osdmap e120: 8 total, 8 up, 8 in 2023-10-09T10:45:14.871 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[200197]: failed to encode map e120 with expected crc 2023-10-09T10:45:14.871 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3792962386' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T10:45:14.871 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:14 smithi125 ceph-mon[200197]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T10:45:14.880 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:14.900 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:14.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:14.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:14.994 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:45:14.994 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:45:15.003 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:15.013 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:14 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:45:15.013 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:14 smithi155 ceph-mon[184854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2708/171369 objects degraded (1.580%), 1 pg degraded) 2023-10-09T10:45:15.013 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:14 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T10:45:15.013 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:14 smithi155 ceph-mon[184854]: failed to encode map e120 with expected crc 2023-10-09T10:45:15.013 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T10:45:15.013 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:14 smithi155 ceph-mon[184854]: osdmap e120: 8 total, 8 up, 8 in 2023-10-09T10:45:15.013 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:14 smithi155 ceph-mon[184854]: failed to encode map e120 with expected crc 2023-10-09T10:45:15.013 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:14 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3792962386' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T10:45:15.013 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:14 smithi155 ceph-mon[184854]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T10:45:15.033 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:15.052 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:15.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:15.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:15.155 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:15.182 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:15.218 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:15.251 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:15.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:15.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:15.303 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 43 16 63274 63258 91.9081 2.9375 0.00251834 0.0105312 2023-10-09T10:45:15.354 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:15.384 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:15.391 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:45:15.403 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:15.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:15.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:15.507 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:15.535 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:15.663 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:45:15.704 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:15.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:15.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:15.807 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:15.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[195507]: 6.2 scrub starts 2023-10-09T10:45:15.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3792962386' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T10:45:15.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[195507]: osdmap e121: 8 total, 8 up, 7 in 2023-10-09T10:45:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[195507]: failed to encode map e121 with expected crc 2023-10-09T10:45:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[195507]: failed to encode map e121 with expected crc 2023-10-09T10:45:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[195507]: failed to encode map e121 with expected crc 2023-10-09T10:45:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[195507]: failed to encode map e121 with expected crc 2023-10-09T10:45:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[195507]: failed to encode map e121 with expected crc 2023-10-09T10:45:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:45:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[195507]: pgmap v243: 99 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 97 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 56 MiB/s wr, 895 op/s; 3459/182901 objects misplaced (1.891%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:45:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[195507]: 6.4 scrub starts 2023-10-09T10:45:15.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[195507]: 6.4 scrub ok 2023-10-09T10:45:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[200197]: 6.2 scrub starts 2023-10-09T10:45:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3792962386' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T10:45:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[200197]: osdmap e121: 8 total, 8 up, 7 in 2023-10-09T10:45:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[200197]: failed to encode map e121 with expected crc 2023-10-09T10:45:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[200197]: failed to encode map e121 with expected crc 2023-10-09T10:45:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[200197]: failed to encode map e121 with expected crc 2023-10-09T10:45:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[200197]: failed to encode map e121 with expected crc 2023-10-09T10:45:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[200197]: failed to encode map e121 with expected crc 2023-10-09T10:45:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:45:15.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[200197]: pgmap v243: 99 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 97 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 56 MiB/s wr, 895 op/s; 3459/182901 objects misplaced (1.891%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:45:15.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[200197]: 6.4 scrub starts 2023-10-09T10:45:15.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:15 smithi125 ceph-mon[200197]: 6.4 scrub ok 2023-10-09T10:45:15.843 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:16.103 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:15 smithi155 ceph-mon[184854]: 6.2 scrub starts 2023-10-09T10:45:16.103 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:15 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3792962386' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T10:45:16.104 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:15 smithi155 ceph-mon[184854]: osdmap e121: 8 total, 8 up, 7 in 2023-10-09T10:45:16.104 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:15 smithi155 ceph-mon[184854]: failed to encode map e121 with expected crc 2023-10-09T10:45:16.104 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:15 smithi155 ceph-mon[184854]: failed to encode map e121 with expected crc 2023-10-09T10:45:16.104 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:15 smithi155 ceph-mon[184854]: failed to encode map e121 with expected crc 2023-10-09T10:45:16.104 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:15 smithi155 ceph-mon[184854]: failed to encode map e121 with expected crc 2023-10-09T10:45:16.104 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:15 smithi155 ceph-mon[184854]: failed to encode map e121 with expected crc 2023-10-09T10:45:16.104 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:16.104 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:45:16.104 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:15 smithi155 ceph-mon[184854]: pgmap v243: 99 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 97 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 56 MiB/s wr, 895 op/s; 3459/182901 objects misplaced (1.891%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:45:16.104 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:15 smithi155 ceph-mon[184854]: 6.4 scrub starts 2023-10-09T10:45:16.105 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:15 smithi155 ceph-mon[184854]: 6.4 scrub ok 2023-10-09T10:45:16.303 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 44 16 63323 63307 89.8895 3.0625 0.00207382 0.0110051 2023-10-09T10:45:16.571 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:45:16.572 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:45:16.653 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:16.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:16.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:16.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:16.806 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:16.889 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:16.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:16.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:16.966 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:45:16.993 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:17.010 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:16 smithi155 ceph-mon[184854]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.010 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:16 smithi155 ceph-mon[184854]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.010 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:16 smithi155 ceph-mon[184854]: osdmap e122: 8 total, 8 up, 7 in 2023-10-09T10:45:17.010 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:16 smithi155 ceph-mon[184854]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.010 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:16 smithi155 ceph-mon[184854]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.010 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:16 smithi155 ceph-mon[184854]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.010 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:16 smithi155 ceph-mon[184854]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.010 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:16 smithi155 ceph-mon[184854]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.010 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:16 smithi155 ceph-mon[184854]: osdmap e123: 8 total, 8 up, 7 in 2023-10-09T10:45:17.063 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:17.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[195507]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[195507]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[195507]: osdmap e122: 8 total, 8 up, 7 in 2023-10-09T10:45:17.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[195507]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[195507]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[195507]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[195507]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[195507]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[195507]: osdmap e123: 8 total, 8 up, 7 in 2023-10-09T10:45:17.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[200197]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[200197]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[200197]: osdmap e122: 8 total, 8 up, 7 in 2023-10-09T10:45:17.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[200197]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[200197]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[200197]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[200197]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[200197]: failed to encode map e122 with expected crc 2023-10-09T10:45:17.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:16 smithi125 ceph-mon[200197]: osdmap e123: 8 total, 8 up, 7 in 2023-10-09T10:45:17.100 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:17.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:17.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:17.208 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:17.262 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:17.295 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:17.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:17.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:17.303 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 45 16 63721 63705 88.4451 24.875 0.0115301 0.0113009 2023-10-09T10:45:17.373 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:45:17.400 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:17.460 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:17.496 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:17.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:17.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:17.601 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:17.644 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:17.674 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:17.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:17.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:17.701 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:45:17.749 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:45:17.777 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:17.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[195507]: failed to encode map e123 with expected crc 2023-10-09T10:45:17.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[195507]: failed to encode map e123 with expected crc 2023-10-09T10:45:17.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[195507]: failed to encode map e123 with expected crc 2023-10-09T10:45:17.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[195507]: failed to encode map e123 with expected crc 2023-10-09T10:45:17.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[195507]: pgmap v246: 99 pgs: 4 remapped+peering, 1 unknown, 94 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 4.0 MiB/s, 65 objects/s recovering 2023-10-09T10:45:17.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1777048253' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:45:17.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:45:17.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2506575611' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:17.789 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:45:17.812 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:45:17.830 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:45:17.848 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:17.866 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:45:17.889 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:17.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:17.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:17.909 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:45:17.962 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:45:17.996 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:18.002 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:45:18.023 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:45:18.024 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:45:18.024 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:45:18.024 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:45:18.024 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:45:18.024 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:45:18.024 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:45:18.024 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:45:18.047 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:18.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[200197]: failed to encode map e123 with expected crc 2023-10-09T10:45:18.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[200197]: failed to encode map e123 with expected crc 2023-10-09T10:45:18.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[200197]: failed to encode map e123 with expected crc 2023-10-09T10:45:18.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[200197]: failed to encode map e123 with expected crc 2023-10-09T10:45:18.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[200197]: pgmap v246: 99 pgs: 4 remapped+peering, 1 unknown, 94 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 4.0 MiB/s, 65 objects/s recovering 2023-10-09T10:45:18.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1777048253' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:45:18.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:45:18.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2506575611' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:18.083 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:18.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:18.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:18.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:17 smithi155 ceph-mon[184854]: failed to encode map e123 with expected crc 2023-10-09T10:45:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:17 smithi155 ceph-mon[184854]: failed to encode map e123 with expected crc 2023-10-09T10:45:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:17 smithi155 ceph-mon[184854]: failed to encode map e123 with expected crc 2023-10-09T10:45:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:17 smithi155 ceph-mon[184854]: failed to encode map e123 with expected crc 2023-10-09T10:45:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:17 smithi155 ceph-mon[184854]: pgmap v246: 99 pgs: 4 remapped+peering, 1 unknown, 94 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 4.0 MiB/s, 65 objects/s recovering 2023-10-09T10:45:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1777048253' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:45:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:17 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:45:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2506575611' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:18.244 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:18.304 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 46 16 64779 64763 87.9586 66.125 0.0087971 0.0113636 2023-10-09T10:45:18.374 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:18.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:18.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:18.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:18.516 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:45:18.516 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:45:18.516 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T10:45:18.516 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:45:18.537 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:18.623 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:18.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:18.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:18.726 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:18.797 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:18.823 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:18 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:18.823 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:18 smithi155 ceph-mon[184854]: failed to encode map e124 with expected crc 2023-10-09T10:45:18.823 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:18 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:45:18.823 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:18 smithi155 ceph-mon[184854]: osdmap e124: 8 total, 8 up, 7 in 2023-10-09T10:45:18.831 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:18.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:18.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:18.897 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:45:18.937 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:18.983 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:18 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:18.984 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:18 smithi125 ceph-mon[200197]: failed to encode map e124 with expected crc 2023-10-09T10:45:18.984 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:18 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:45:18.984 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:18 smithi125 ceph-mon[200197]: osdmap e124: 8 total, 8 up, 7 in 2023-10-09T10:45:18.985 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:18 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:18.985 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:18 smithi125 ceph-mon[195507]: failed to encode map e124 with expected crc 2023-10-09T10:45:18.986 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:18 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:45:18.986 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:18 smithi125 ceph-mon[195507]: osdmap e124: 8 total, 8 up, 7 in 2023-10-09T10:45:18.986 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:19.017 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:19.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:19.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:19.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:19.179 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:19.213 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:19.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:19.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:19.298 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:45:19.304 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 47 16 65978 65962 87.6815 74.9375 0.0147484 0.0114002 2023-10-09T10:45:19.320 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:19.355 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:19.398 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:19.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:19.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:19.504 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:19.543 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:19.675 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:19.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:19.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:19.784 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:19.835 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:19.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:19 smithi155 ceph-mon[184854]: pgmap v248: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 remapped+peering, 1 unknown, 92 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 11 MiB/s wr, 169 op/s; 1803/190917 objects misplaced (0.944%); 5.9 MiB/s, 97 objects/s recovering 2023-10-09T10:45:19.971 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:19.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:19.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:20.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:19 smithi125 ceph-mon[200197]: pgmap v248: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 remapped+peering, 1 unknown, 92 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 11 MiB/s wr, 169 op/s; 1803/190917 objects misplaced (0.944%); 5.9 MiB/s, 97 objects/s recovering 2023-10-09T10:45:20.076 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:20.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:19 smithi125 ceph-mon[195507]: pgmap v248: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 remapped+peering, 1 unknown, 92 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 11 MiB/s wr, 169 op/s; 1803/190917 objects misplaced (0.944%); 5.9 MiB/s, 97 objects/s recovering 2023-10-09T10:45:20.126 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:20.259 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:20.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:20.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:20.304 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 48 16 67258 67242 87.5204 80 0.0120802 0.0114211 2023-10-09T10:45:20.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:45:20.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:20.420 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:20.472 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:20.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:20.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:20.578 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:20.594 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:45:20.594 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:45:20.594 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T10:45:20.595 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:45:20.639 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:20.657 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:45:20.674 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:45:20.674 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:45:20.674 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:45:20.674 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:45:20.674 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:45:20.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:20.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:20.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:20.793 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:20.832 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:20.835 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:20.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:20.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:20.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:20.966 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:20.999 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:21.022 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:21.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:21.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:21.056 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:45:21.129 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:21.191 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:21.229 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:21.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:21.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:21.305 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 49 16 68187 68171 86.9193 58.0625 0.0239157 0.0114988 2023-10-09T10:45:21.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:21 smithi125 ceph-mon[195507]: Health check failed: Degraded data redundancy: 14063/198564 objects degraded (7.082%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:21.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:45:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:45:21] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.43.0" 2023-10-09T10:45:21.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:21 smithi125 ceph-mon[200197]: Health check failed: Degraded data redundancy: 14063/198564 objects degraded (7.082%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:21.334 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:21.391 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:21.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:21 smithi155 ceph-mon[184854]: Health check failed: Degraded data redundancy: 14063/198564 objects degraded (7.082%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:21.474 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:45:21.648 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:21.681 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:21.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:21.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:21.703 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:21.794 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:21.843 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:22.283 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:22.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:22 smithi125 ceph-mon[195507]: pgmap v249: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 remapped+peering, 90 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 34 MiB/s wr, 545 op/s; 14063/198564 objects degraded (7.082%); 3379/198564 objects misplaced (1.702%); 8.7 MiB/s, 140 objects/s recovering 2023-10-09T10:45:22.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:22 smithi125 ceph-mon[200197]: pgmap v249: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 remapped+peering, 90 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 34 MiB/s wr, 545 op/s; 14063/198564 objects degraded (7.082%); 3379/198564 objects misplaced (1.702%); 8.7 MiB/s, 140 objects/s recovering 2023-10-09T10:45:22.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:22.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:22.305 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 50 16 69267 69251 86.5309 67.5 0.0211305 0.0115518 2023-10-09T10:45:22.380 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:22 smithi155 ceph-mon[184854]: pgmap v249: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 remapped+peering, 90 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 34 MiB/s wr, 545 op/s; 14063/198564 objects degraded (7.082%); 3379/198564 objects misplaced (1.702%); 8.7 MiB/s, 140 objects/s recovering 2023-10-09T10:45:22.388 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:22.442 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:22.525 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:22.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:22.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:22.630 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:22.672 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:45:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:45:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:45:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:45:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:1070 6.8 6:1676aaf7:::benchmark_data_smithi125_244951_object1069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e101)", 2023-10-09T10:45:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:32.597309+0000", 2023-10-09T10:45:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 50.072900742000002, 2023-10-09T10:45:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026379626, 2023-10-09T10:45:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:45:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 1070 2023-10-09T10:45:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.597309+0000", 2023-10-09T10:45:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.597309+0000", 2023-10-09T10:45:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.597309+0000", 2023-10-09T10:45:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.597376+0000", 2023-10-09T10:45:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7115999999999999e-05 2023-10-09T10:45:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.597385+0000", 2023-10-09T10:45:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7920000000000004e-06 2023-10-09T10:45:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.597390+0000", 2023-10-09T10:45:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5059999999999997e-06 2023-10-09T10:45:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.597418+0000", 2023-10-09T10:45:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7682e-05 2023-10-09T10:45:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.597503+0000", 2023-10-09T10:45:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5877000000000005e-05 2023-10-09T10:45:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:45:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.597584+0000", 2023-10-09T10:45:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0593000000000006e-05 2023-10-09T10:45:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.597986+0000", 2023-10-09T10:45:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00040200000000000001 2023-10-09T10:45:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.611178+0000", 2023-10-09T10:45:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013192229 2023-10-09T10:45:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:32.623689+0000", 2023-10-09T10:45:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012510522 2023-10-09T10:45:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:17770 6.3 6:c33290b9:::benchmark_data_smithi125_244951_object17769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:45:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:40.368491+0000", 2023-10-09T10:45:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 42.301718594, 2023-10-09T10:45:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018773253, 2023-10-09T10:45:22.680 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.680 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.680 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.680 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.680 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.680 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 17770 2023-10-09T10:45:22.680 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.680 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.680 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.680 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.680 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.368491+0000", 2023-10-09T10:45:22.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.680 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.681 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.368491+0000", 2023-10-09T10:45:22.681 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.681 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.368490+0000", 2023-10-09T10:45:22.681 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:45:22.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.681 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.368527+0000", 2023-10-09T10:45:22.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7057000000000001e-05 2023-10-09T10:45:22.682 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.682 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.682 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.368552+0000", 2023-10-09T10:45:22.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4888999999999998e-05 2023-10-09T10:45:22.682 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.682 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.682 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.368559+0000", 2023-10-09T10:45:22.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1899999999999998e-06 2023-10-09T10:45:22.682 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.683 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.683 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.368626+0000", 2023-10-09T10:45:22.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7192000000000003e-05 2023-10-09T10:45:22.683 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.683 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.683 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.368775+0000", 2023-10-09T10:45:22.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000149166 2023-10-09T10:45:22.683 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.683 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.683 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.368935+0000", 2023-10-09T10:45:22.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016027599999999999 2023-10-09T10:45:22.684 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.684 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.684 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.684 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.373304+0000", 2023-10-09T10:45:22.684 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043682829999999997 2023-10-09T10:45:22.684 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.684 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.684 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.684 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.373538+0000", 2023-10-09T10:45:22.684 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000234718 2023-10-09T10:45:22.684 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.684 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.684 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.685 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.387171+0000", 2023-10-09T10:45:22.685 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013633081 2023-10-09T10:45:22.685 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.685 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.685 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.686 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.387224+0000", 2023-10-09T10:45:22.686 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2561e-05 2023-10-09T10:45:22.686 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.686 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.686 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.686 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.387265+0000", 2023-10-09T10:45:22.687 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.057e-05 2023-10-09T10:45:22.687 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.687 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.687 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.687 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.687 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.687 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:17776 6.3 6:cbc897b2:::benchmark_data_smithi125_244951_object17775:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:45:22.687 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:40.368588+0000", 2023-10-09T10:45:22.687 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 42.301622037999998, 2023-10-09T10:45:22.687 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018812431000000001, 2023-10-09T10:45:22.687 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.687 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.687 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.688 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.688 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.688 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 17776 2023-10-09T10:45:22.688 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.688 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.688 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.688 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.688 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.368588+0000", 2023-10-09T10:45:22.688 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.688 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.688 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.688 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.688 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.368588+0000", 2023-10-09T10:45:22.689 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.689 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.689 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.689 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.689 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.368587+0000", 2023-10-09T10:45:22.689 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:22.689 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.689 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.689 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.689 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.368676+0000", 2023-10-09T10:45:22.689 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8837000000000003e-05 2023-10-09T10:45:22.689 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.689 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.690 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.690 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.368688+0000", 2023-10-09T10:45:22.690 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1888999999999999e-05 2023-10-09T10:45:22.690 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.690 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.690 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.690 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.368693+0000", 2023-10-09T10:45:22.690 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8280000000000002e-06 2023-10-09T10:45:22.690 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.690 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.690 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.690 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.369757+0000", 2023-10-09T10:45:22.690 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010644809999999999 2023-10-09T10:45:22.691 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.691 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.691 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.691 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.369877+0000", 2023-10-09T10:45:22.691 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011957500000000001 2023-10-09T10:45:22.691 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.691 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.691 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.691 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.370021+0000", 2023-10-09T10:45:22.691 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014385 2023-10-09T10:45:22.691 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.691 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.691 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.691 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.373438+0000", 2023-10-09T10:45:22.692 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034176710000000002 2023-10-09T10:45:22.692 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.692 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.692 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.692 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.373755+0000", 2023-10-09T10:45:22.692 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00031640800000000001 2023-10-09T10:45:22.692 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.692 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.692 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.692 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.387316+0000", 2023-10-09T10:45:22.692 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013561230000000001 2023-10-09T10:45:22.692 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.692 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.693 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.693 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.387362+0000", 2023-10-09T10:45:22.693 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5745000000000003e-05 2023-10-09T10:45:22.693 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.693 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.693 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.693 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:40.387400+0000", 2023-10-09T10:45:22.693 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8513000000000002e-05 2023-10-09T10:45:22.693 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.693 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.693 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.693 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.693 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.694 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:39146 6.3 6:c49d57a4:::benchmark_data_smithi125_244951_object39145:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T10:45:22.694 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:53.069653+0000", 2023-10-09T10:45:22.694 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 29.600556822000001, 2023-10-09T10:45:22.694 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020034074999999998, 2023-10-09T10:45:22.694 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.694 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.694 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.694 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.694 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.694 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39146 2023-10-09T10:45:22.694 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.694 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.695 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.695 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.695 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.069653+0000", 2023-10-09T10:45:22.695 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.695 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.695 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.695 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.695 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.069653+0000", 2023-10-09T10:45:22.695 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.695 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.695 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.695 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.695 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.069653+0000", 2023-10-09T10:45:22.696 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:22.696 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.696 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.696 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.696 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.069685+0000", 2023-10-09T10:45:22.696 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2783000000000001e-05 2023-10-09T10:45:22.696 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.696 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.696 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.696 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.069693+0000", 2023-10-09T10:45:22.696 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1160000000000001e-06 2023-10-09T10:45:22.696 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.696 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.697 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.697 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.069696+0000", 2023-10-09T10:45:22.697 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9550000000000002e-06 2023-10-09T10:45:22.697 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.697 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.697 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.697 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.071432+0000", 2023-10-09T10:45:22.697 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0017352509999999999 2023-10-09T10:45:22.697 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.697 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.697 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.698 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.071543+0000", 2023-10-09T10:45:22.698 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000111 2023-10-09T10:45:22.698 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.698 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.698 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.698 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.071680+0000", 2023-10-09T10:45:22.698 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013747400000000001 2023-10-09T10:45:22.698 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.698 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.698 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.698 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.072535+0000", 2023-10-09T10:45:22.699 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00085487599999999996 2023-10-09T10:45:22.699 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.699 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.699 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.699 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.072925+0000", 2023-10-09T10:45:22.699 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000390262 2023-10-09T10:45:22.699 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.699 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.699 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.699 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.089627+0000", 2023-10-09T10:45:22.699 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016701965999999999 2023-10-09T10:45:22.699 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.699 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.089662+0000", 2023-10-09T10:45:22.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5139999999999999e-05 2023-10-09T10:45:22.700 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.700 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.089687+0000", 2023-10-09T10:45:22.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4768000000000001e-05 2023-10-09T10:45:22.700 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.700 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.700 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.700 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.700 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.701 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:39324 6.3 6:c3da0f33:::benchmark_data_smithi125_244951_object39323:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T10:45:22.701 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:53.172895+0000", 2023-10-09T10:45:22.701 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 29.497315129, 2023-10-09T10:45:22.701 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020531750000000001, 2023-10-09T10:45:22.701 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.701 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.701 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.701 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.701 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.701 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39324 2023-10-09T10:45:22.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.701 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.701 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.172895+0000", 2023-10-09T10:45:22.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.702 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.172895+0000", 2023-10-09T10:45:22.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.702 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.172894+0000", 2023-10-09T10:45:22.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:22.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.172949+0000", 2023-10-09T10:45:22.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5034000000000002e-05 2023-10-09T10:45:22.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.172958+0000", 2023-10-09T10:45:22.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0089999999999996e-06 2023-10-09T10:45:22.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.704 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.172963+0000", 2023-10-09T10:45:22.704 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4349999999999996e-06 2023-10-09T10:45:22.704 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.704 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.704 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.704 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.173001+0000", 2023-10-09T10:45:22.704 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8087000000000003e-05 2023-10-09T10:45:22.704 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.704 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.704 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.704 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.173121+0000", 2023-10-09T10:45:22.704 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000119465 2023-10-09T10:45:22.704 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.705 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.705 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.173253+0000", 2023-10-09T10:45:22.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000131963 2023-10-09T10:45:22.705 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.705 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.705 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.174131+0000", 2023-10-09T10:45:22.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00087799799999999995 2023-10-09T10:45:22.705 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.705 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.705 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.174434+0000", 2023-10-09T10:45:22.706 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030381800000000002 2023-10-09T10:45:22.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.706 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.706 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.706 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.193356+0000", 2023-10-09T10:45:22.706 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018921306999999998 2023-10-09T10:45:22.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.706 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.706 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.706 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.193401+0000", 2023-10-09T10:45:22.706 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5077000000000002e-05 2023-10-09T10:45:22.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.706 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.707 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.707 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.193427+0000", 2023-10-09T10:45:22.707 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5763e-05 2023-10-09T10:45:22.707 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.707 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.707 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.707 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.707 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.707 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:39811 6.13 6:c8b8345e:::benchmark_data_smithi125_244951_object39810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T10:45:22.707 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:53.505709+0000", 2023-10-09T10:45:22.707 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 29.164500925999999, 2023-10-09T10:45:22.707 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018949191000000001, 2023-10-09T10:45:22.708 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.708 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.708 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.708 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.708 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.708 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39811 2023-10-09T10:45:22.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.708 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.708 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.708 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.505709+0000", 2023-10-09T10:45:22.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.709 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.505709+0000", 2023-10-09T10:45:22.709 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.709 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.709 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.505708+0000", 2023-10-09T10:45:22.709 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:22.709 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.709 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.505756+0000", 2023-10-09T10:45:22.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8340000000000001e-05 2023-10-09T10:45:22.710 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.710 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.710 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.710 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.505768+0000", 2023-10-09T10:45:22.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1909999999999999e-05 2023-10-09T10:45:22.710 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.710 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.710 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.710 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.505774+0000", 2023-10-09T10:45:22.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4670000000000001e-06 2023-10-09T10:45:22.710 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.710 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.711 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.711 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.506102+0000", 2023-10-09T10:45:22.711 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000327752 2023-10-09T10:45:22.711 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.711 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.711 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.711 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.506229+0000", 2023-10-09T10:45:22.711 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012688900000000001 2023-10-09T10:45:22.711 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.711 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.711 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.711 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.506367+0000", 2023-10-09T10:45:22.711 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013809599999999999 2023-10-09T10:45:22.712 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.712 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.712 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.507873+0000", 2023-10-09T10:45:22.712 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001505525 2023-10-09T10:45:22.712 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.712 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.712 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.507993+0000", 2023-10-09T10:45:22.712 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000120531 2023-10-09T10:45:22.712 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.712 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.712 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.524560+0000", 2023-10-09T10:45:22.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016566688999999999 2023-10-09T10:45:22.713 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.713 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.713 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.713 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.524614+0000", 2023-10-09T10:45:22.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3961000000000001e-05 2023-10-09T10:45:22.713 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.713 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.713 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.713 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:53.524658+0000", 2023-10-09T10:45:22.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4400999999999997e-05 2023-10-09T10:45:22.713 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.713 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.714 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.714 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.714 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.714 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:41867 6.3 6:c507f2bf:::benchmark_data_smithi125_244951_object41866:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T10:45:22.714 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:54.819363+0000", 2023-10-09T10:45:22.714 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 27.850846722, 2023-10-09T10:45:22.714 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026608659, 2023-10-09T10:45:22.714 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.714 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.715 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.715 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.715 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.715 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 41867 2023-10-09T10:45:22.715 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.715 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.819363+0000", 2023-10-09T10:45:22.715 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.715 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.819363+0000", 2023-10-09T10:45:22.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.716 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.716 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.819362+0000", 2023-10-09T10:45:22.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:22.716 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.716 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.819413+0000", 2023-10-09T10:45:22.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1578e-05 2023-10-09T10:45:22.717 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.717 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.717 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.717 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.819431+0000", 2023-10-09T10:45:22.717 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8088999999999999e-05 2023-10-09T10:45:22.717 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.717 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.717 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.717 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.819440+0000", 2023-10-09T10:45:22.717 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3280000000000006e-06 2023-10-09T10:45:22.717 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.717 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.718 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.718 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.819500+0000", 2023-10-09T10:45:22.718 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9902999999999999e-05 2023-10-09T10:45:22.718 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.718 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.718 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.718 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.819601+0000", 2023-10-09T10:45:22.718 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010180500000000001 2023-10-09T10:45:22.718 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.718 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.718 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.819700+0000", 2023-10-09T10:45:22.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8671000000000004e-05 2023-10-09T10:45:22.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.719 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.719 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.821531+0000", 2023-10-09T10:45:22.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018308490000000001 2023-10-09T10:45:22.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.719 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.719 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.821693+0000", 2023-10-09T10:45:22.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000162411 2023-10-09T10:45:22.720 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.720 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.845861+0000", 2023-10-09T10:45:22.720 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024167296000000001 2023-10-09T10:45:22.720 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.720 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.845923+0000", 2023-10-09T10:45:22.720 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2618000000000003e-05 2023-10-09T10:45:22.720 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.721 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.721 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.845972+0000", 2023-10-09T10:45:22.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8701e-05 2023-10-09T10:45:22.721 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.721 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.721 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.721 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.721 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.721 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:41878 6.13 6:c94f914a:::benchmark_data_smithi125_244951_object41877:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T10:45:22.721 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:54.821585+0000", 2023-10-09T10:45:22.721 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 27.848624524999998, 2023-10-09T10:45:22.722 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024521202999999998, 2023-10-09T10:45:22.722 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.722 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.722 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.722 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.722 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.722 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 41878 2023-10-09T10:45:22.722 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.722 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.722 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.722 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.821585+0000", 2023-10-09T10:45:22.723 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.723 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.723 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.723 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.821585+0000", 2023-10-09T10:45:22.723 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.723 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.723 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.723 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.821584+0000", 2023-10-09T10:45:22.723 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:22.724 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.724 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.724 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.724 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.821682+0000", 2023-10-09T10:45:22.724 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8160999999999997e-05 2023-10-09T10:45:22.724 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.724 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.724 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.724 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.821692+0000", 2023-10-09T10:45:22.724 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6269999999999995e-06 2023-10-09T10:45:22.724 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.724 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.725 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.725 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.821697+0000", 2023-10-09T10:45:22.725 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2909999999999998e-06 2023-10-09T10:45:22.725 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.725 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.725 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.725 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.821733+0000", 2023-10-09T10:45:22.725 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5460000000000003e-05 2023-10-09T10:45:22.725 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.725 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.725 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.726 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.821828+0000", 2023-10-09T10:45:22.726 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5205999999999996e-05 2023-10-09T10:45:22.726 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.726 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.726 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.726 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.821955+0000", 2023-10-09T10:45:22.726 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012671399999999999 2023-10-09T10:45:22.726 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.726 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.726 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.726 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.823129+0000", 2023-10-09T10:45:22.726 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011747210000000001 2023-10-09T10:45:22.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.727 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.727 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.824624+0000", 2023-10-09T10:45:22.727 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014947739999999999 2023-10-09T10:45:22.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.727 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.727 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.846016+0000", 2023-10-09T10:45:22.727 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021391712 2023-10-09T10:45:22.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.846067+0000", 2023-10-09T10:45:22.728 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1609e-05 2023-10-09T10:45:22.728 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.728 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.846107+0000", 2023-10-09T10:45:22.728 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9229e-05 2023-10-09T10:45:22.728 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.728 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.728 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.729 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.729 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.729 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:42929 6.3 6:c4b1f97c:::benchmark_data_smithi125_244951_object42928:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T10:45:22.729 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:57.205519+0000", 2023-10-09T10:45:22.729 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 25.46469128, 2023-10-09T10:45:22.729 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017257405, 2023-10-09T10:45:22.729 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.729 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.729 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.729 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.729 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.729 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 42929 2023-10-09T10:45:22.730 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.730 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.730 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.730 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:57.205519+0000", 2023-10-09T10:45:22.730 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.730 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.730 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.730 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:57.205519+0000", 2023-10-09T10:45:22.730 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.731 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.731 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.731 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:57.205518+0000", 2023-10-09T10:45:22.731 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:45:22.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.731 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.731 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.731 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:57.205587+0000", 2023-10-09T10:45:22.731 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9173999999999999e-05 2023-10-09T10:45:22.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.731 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:57.205601+0000", 2023-10-09T10:45:22.732 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.436e-05 2023-10-09T10:45:22.732 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:57.205608+0000", 2023-10-09T10:45:22.732 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1380000000000001e-06 2023-10-09T10:45:22.732 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:57.205662+0000", 2023-10-09T10:45:22.733 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3675999999999999e-05 2023-10-09T10:45:22.733 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.733 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.733 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.733 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:57.205803+0000", 2023-10-09T10:45:22.733 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014098800000000001 2023-10-09T10:45:22.733 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.733 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.733 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.733 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:57.205966+0000", 2023-10-09T10:45:22.733 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016256899999999999 2023-10-09T10:45:22.733 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.734 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.734 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.734 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:57.207094+0000", 2023-10-09T10:45:22.734 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001128701 2023-10-09T10:45:22.734 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.734 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.734 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.734 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:57.208160+0000", 2023-10-09T10:45:22.734 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001065355 2023-10-09T10:45:22.734 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.735 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.735 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:57.222688+0000", 2023-10-09T10:45:22.735 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014528746 2023-10-09T10:45:22.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.735 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.735 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:57.222737+0000", 2023-10-09T10:45:22.735 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8751999999999997e-05 2023-10-09T10:45:22.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.735 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.735 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:57.222776+0000", 2023-10-09T10:45:22.736 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8970999999999997e-05 2023-10-09T10:45:22.736 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.736 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.736 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.736 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.736 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:64670 6.13 6:cb749ced:::benchmark_data_smithi125_244951_object64669:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:45:22.736 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:18.201365+0000", 2023-10-09T10:45:22.736 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 4.4688445989999996, 2023-10-09T10:45:22.736 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017687443000000001, 2023-10-09T10:45:22.736 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.736 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.736 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.737 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.737 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.737 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 64670 2023-10-09T10:45:22.737 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.737 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.737 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.737 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.737 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.201365+0000", 2023-10-09T10:45:22.737 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.737 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.737 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.737 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.737 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.201365+0000", 2023-10-09T10:45:22.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.738 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.738 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.738 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.738 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.201365+0000", 2023-10-09T10:45:22.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:22.738 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.738 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.738 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.738 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.201428+0000", 2023-10-09T10:45:22.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2867999999999995e-05 2023-10-09T10:45:22.739 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.739 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.739 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.739 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.201615+0000", 2023-10-09T10:45:22.739 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018730900000000001 2023-10-09T10:45:22.739 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.739 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.739 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.739 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.201622+0000", 2023-10-09T10:45:22.739 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0859999999999996e-06 2023-10-09T10:45:22.739 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.739 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.739 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.740 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.202968+0000", 2023-10-09T10:45:22.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0013460729999999999 2023-10-09T10:45:22.740 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.740 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.740 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.740 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.203101+0000", 2023-10-09T10:45:22.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000133242 2023-10-09T10:45:22.740 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.740 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.740 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.740 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.203255+0000", 2023-10-09T10:45:22.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000153888 2023-10-09T10:45:22.741 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.741 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.741 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.741 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.206708+0000", 2023-10-09T10:45:22.741 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034523470000000001 2023-10-09T10:45:22.741 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.741 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.741 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.741 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.207587+0000", 2023-10-09T10:45:22.741 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00087925700000000004 2023-10-09T10:45:22.741 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.741 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.742 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.742 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.218946+0000", 2023-10-09T10:45:22.742 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011358971000000001 2023-10-09T10:45:22.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.742 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.742 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.742 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.219010+0000", 2023-10-09T10:45:22.742 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3812999999999994e-05 2023-10-09T10:45:22.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.742 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.742 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.742 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.219053+0000", 2023-10-09T10:45:22.742 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3121000000000003e-05 2023-10-09T10:45:22.743 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.743 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.743 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.743 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.743 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.743 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:64690 6.3 6:c2939563:::benchmark_data_smithi125_244951_object64689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:45:22.743 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:18.214844+0000", 2023-10-09T10:45:22.743 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 4.4553663009999998, 2023-10-09T10:45:22.743 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018931539000000001, 2023-10-09T10:45:22.743 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.743 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.743 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.743 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.744 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.744 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 64690 2023-10-09T10:45:22.744 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.744 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.744 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.744 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.744 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.214844+0000", 2023-10-09T10:45:22.744 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.744 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.744 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.744 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.744 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.214844+0000", 2023-10-09T10:45:22.744 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.745 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.214842+0000", 2023-10-09T10:45:22.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:22.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.745 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.215011+0000", 2023-10-09T10:45:22.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016916300000000001 2023-10-09T10:45:22.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.215027+0000", 2023-10-09T10:45:22.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5293000000000001e-05 2023-10-09T10:45:22.746 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.746 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.215035+0000", 2023-10-09T10:45:22.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.755e-06 2023-10-09T10:45:22.746 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.746 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.215079+0000", 2023-10-09T10:45:22.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3667000000000001e-05 2023-10-09T10:45:22.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.747 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.747 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.215214+0000", 2023-10-09T10:45:22.747 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000134917 2023-10-09T10:45:22.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.747 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.747 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.215368+0000", 2023-10-09T10:45:22.747 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015401999999999999 2023-10-09T10:45:22.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.747 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.217829+0000", 2023-10-09T10:45:22.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024611960000000001 2023-10-09T10:45:22.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.748 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.748 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.222308+0000", 2023-10-09T10:45:22.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0044789230000000001 2023-10-09T10:45:22.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.748 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.748 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.233713+0000", 2023-10-09T10:45:22.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011404878 2023-10-09T10:45:22.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.233748+0000", 2023-10-09T10:45:22.749 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4919999999999998e-05 2023-10-09T10:45:22.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.233775+0000", 2023-10-09T10:45:22.749 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7161999999999998e-05 2023-10-09T10:45:22.749 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.749 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.749 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.750 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:65184 6.13 6:cbdfc7fa:::benchmark_data_smithi125_244951_object65183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:45:22.750 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:18.646640+0000", 2023-10-09T10:45:22.750 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 4.023569932, 2023-10-09T10:45:22.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018973692, 2023-10-09T10:45:22.750 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.750 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.750 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.750 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.751 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.751 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 65184 2023-10-09T10:45:22.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.751 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.646640+0000", 2023-10-09T10:45:22.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.646640+0000", 2023-10-09T10:45:22.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.646628+0000", 2023-10-09T10:45:22.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999881 2023-10-09T10:45:22.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.646679+0000", 2023-10-09T10:45:22.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0896000000000002e-05 2023-10-09T10:45:22.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.646691+0000", 2023-10-09T10:45:22.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1364e-05 2023-10-09T10:45:22.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.646695+0000", 2023-10-09T10:45:22.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.189e-06 2023-10-09T10:45:22.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.648133+0000", 2023-10-09T10:45:22.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014385019999999999 2023-10-09T10:45:22.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.754 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.648304+0000", 2023-10-09T10:45:22.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017115300000000001 2023-10-09T10:45:22.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.754 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.648490+0000", 2023-10-09T10:45:22.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018528800000000001 2023-10-09T10:45:22.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.755 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.651226+0000", 2023-10-09T10:45:22.755 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002735946 2023-10-09T10:45:22.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.755 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.755 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.651240+0000", 2023-10-09T10:45:22.755 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4634000000000001e-05 2023-10-09T10:45:22.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.755 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.755 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.665515+0000", 2023-10-09T10:45:22.755 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014274617999999999 2023-10-09T10:45:22.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.665569+0000", 2023-10-09T10:45:22.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3797e-05 2023-10-09T10:45:22.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.665614+0000", 2023-10-09T10:45:22.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5040999999999998e-05 2023-10-09T10:45:22.756 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.756 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.756 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.757 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:65185 6.13 6:c9e51282:::benchmark_data_smithi125_244951_object65184:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:45:22.757 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:18.646701+0000", 2023-10-09T10:45:22.757 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 4.0235093239999999, 2023-10-09T10:45:22.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020231605999999999, 2023-10-09T10:45:22.757 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.757 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.757 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.757 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.757 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.757 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 65185 2023-10-09T10:45:22.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.757 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.646701+0000", 2023-10-09T10:45:22.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.646701+0000", 2023-10-09T10:45:22.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.646700+0000", 2023-10-09T10:45:22.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:22.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.646729+0000", 2023-10-09T10:45:22.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9040999999999999e-05 2023-10-09T10:45:22.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.761 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.646734+0000", 2023-10-09T10:45:22.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5970000000000002e-06 2023-10-09T10:45:22.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.761 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.761 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.761 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.646737+0000", 2023-10-09T10:45:22.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9620000000000002e-06 2023-10-09T10:45:22.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.761 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.649775+0000", 2023-10-09T10:45:22.762 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003038069 2023-10-09T10:45:22.762 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.649897+0000", 2023-10-09T10:45:22.762 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000122731 2023-10-09T10:45:22.762 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.650046+0000", 2023-10-09T10:45:22.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014884800000000001 2023-10-09T10:45:22.763 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.763 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.763 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.763 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.651245+0000", 2023-10-09T10:45:22.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011984509999999999 2023-10-09T10:45:22.763 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.763 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.763 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.763 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.651430+0000", 2023-10-09T10:45:22.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018483200000000001 2023-10-09T10:45:22.763 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.763 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.764 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.764 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.666800+0000", 2023-10-09T10:45:22.764 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015369971 2023-10-09T10:45:22.764 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.764 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.764 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.764 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.666890+0000", 2023-10-09T10:45:22.764 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0628e-05 2023-10-09T10:45:22.764 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.764 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.764 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.764 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:18.666932+0000", 2023-10-09T10:45:22.764 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2024000000000003e-05 2023-10-09T10:45:22.765 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.765 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.765 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.765 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.765 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.765 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:67569 6.13 6:cbc4efb2:::benchmark_data_smithi125_244951_object67568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:45:22.765 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:20.601074+0000", 2023-10-09T10:45:22.765 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 2.069136232, 2023-10-09T10:45:22.765 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021996068000000001, 2023-10-09T10:45:22.765 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.765 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.765 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.765 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.766 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.766 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 67569 2023-10-09T10:45:22.766 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.766 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.766 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.766 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.766 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.601074+0000", 2023-10-09T10:45:22.766 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.766 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.766 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.766 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.766 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.601074+0000", 2023-10-09T10:45:22.767 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.767 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.767 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.767 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.767 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.601072+0000", 2023-10-09T10:45:22.767 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:22.767 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.767 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.767 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.767 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.601101+0000", 2023-10-09T10:45:22.767 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8793999999999999e-05 2023-10-09T10:45:22.767 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.768 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.601118+0000", 2023-10-09T10:45:22.768 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7365999999999999e-05 2023-10-09T10:45:22.768 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.768 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.601125+0000", 2023-10-09T10:45:22.768 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5379999999999999e-06 2023-10-09T10:45:22.768 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.768 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.769 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.601191+0000", 2023-10-09T10:45:22.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5976999999999995e-05 2023-10-09T10:45:22.769 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.769 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.769 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.769 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.601328+0000", 2023-10-09T10:45:22.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013738499999999999 2023-10-09T10:45:22.769 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.769 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.769 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.769 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.601477+0000", 2023-10-09T10:45:22.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014926300000000001 2023-10-09T10:45:22.770 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.770 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.770 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.770 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.603957+0000", 2023-10-09T10:45:22.770 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024792780000000001 2023-10-09T10:45:22.770 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.770 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.770 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.770 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.604472+0000", 2023-10-09T10:45:22.770 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00051493299999999997 2023-10-09T10:45:22.770 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.770 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.622972+0000", 2023-10-09T10:45:22.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018500526 2023-10-09T10:45:22.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.623029+0000", 2023-10-09T10:45:22.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6959999999999997e-05 2023-10-09T10:45:22.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.623070+0000", 2023-10-09T10:45:22.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0682000000000001e-05 2023-10-09T10:45:22.772 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.772 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.772 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.772 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.772 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.772 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:67640 6.13 6:cdad8e56:::benchmark_data_smithi125_244951_object67639:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:45:22.772 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:20.688149+0000", 2023-10-09T10:45:22.772 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 1.9820607689999998, 2023-10-09T10:45:22.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020359735, 2023-10-09T10:45:22.772 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.772 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.772 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.772 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.773 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.773 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 67640 2023-10-09T10:45:22.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.773 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.688149+0000", 2023-10-09T10:45:22.773 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.688149+0000", 2023-10-09T10:45:22.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.774 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.774 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.774 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.774 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.688147+0000", 2023-10-09T10:45:22.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:45:22.774 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.774 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.774 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.774 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.688205+0000", 2023-10-09T10:45:22.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7853000000000003e-05 2023-10-09T10:45:22.774 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.775 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.688220+0000", 2023-10-09T10:45:22.775 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.521e-05 2023-10-09T10:45:22.775 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.775 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.688229+0000", 2023-10-09T10:45:22.775 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3960000000000006e-06 2023-10-09T10:45:22.775 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.776 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.688268+0000", 2023-10-09T10:45:22.776 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9682999999999999e-05 2023-10-09T10:45:22.776 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.776 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.776 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.776 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.688373+0000", 2023-10-09T10:45:22.776 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104256 2023-10-09T10:45:22.776 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.776 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.776 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.776 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.688497+0000", 2023-10-09T10:45:22.776 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000124363 2023-10-09T10:45:22.777 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.777 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.777 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.777 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.689562+0000", 2023-10-09T10:45:22.777 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010647180000000001 2023-10-09T10:45:22.777 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.777 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.777 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.777 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.689702+0000", 2023-10-09T10:45:22.777 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014018799999999999 2023-10-09T10:45:22.777 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.777 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.778 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.778 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.708389+0000", 2023-10-09T10:45:22.778 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018686566000000002 2023-10-09T10:45:22.778 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.778 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.778 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.778 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.708448+0000", 2023-10-09T10:45:22.778 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9599000000000003e-05 2023-10-09T10:45:22.778 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.778 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.778 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.778 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.708509+0000", 2023-10-09T10:45:22.779 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0744e-05 2023-10-09T10:45:22.779 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.779 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.779 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.779 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.779 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.779 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:67654 6.13 6:cebb48a6:::benchmark_data_smithi125_244951_object67653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:45:22.779 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:20.711273+0000", 2023-10-09T10:45:22.779 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 1.958936926, 2023-10-09T10:45:22.779 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019351084000000001, 2023-10-09T10:45:22.779 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.779 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.780 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.780 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.780 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.780 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 67654 2023-10-09T10:45:22.780 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.780 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.780 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.780 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.780 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.711273+0000", 2023-10-09T10:45:22.780 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.780 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.780 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.780 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.781 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.711273+0000", 2023-10-09T10:45:22.781 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.781 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.781 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.781 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.781 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.711272+0000", 2023-10-09T10:45:22.781 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:22.781 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.781 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.781 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.781 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.711310+0000", 2023-10-09T10:45:22.781 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7876999999999997e-05 2023-10-09T10:45:22.782 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.782 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.782 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.782 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.711322+0000", 2023-10-09T10:45:22.782 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.189e-05 2023-10-09T10:45:22.782 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.782 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.782 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.782 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.711327+0000", 2023-10-09T10:45:22.782 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5979999999999996e-06 2023-10-09T10:45:22.782 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.782 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.783 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.783 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.711364+0000", 2023-10-09T10:45:22.783 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6678999999999999e-05 2023-10-09T10:45:22.783 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.783 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.783 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.783 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.711499+0000", 2023-10-09T10:45:22.783 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013469799999999999 2023-10-09T10:45:22.783 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.783 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.783 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.783 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.711601+0000", 2023-10-09T10:45:22.784 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000102638 2023-10-09T10:45:22.784 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.784 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.784 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.784 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.712659+0000", 2023-10-09T10:45:22.784 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001058067 2023-10-09T10:45:22.784 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.784 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.784 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.784 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.712857+0000", 2023-10-09T10:45:22.784 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00019796700000000001 2023-10-09T10:45:22.784 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.785 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.785 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.785 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.730507+0000", 2023-10-09T10:45:22.785 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017649339 2023-10-09T10:45:22.785 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.785 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.785 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.785 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.730577+0000", 2023-10-09T10:45:22.785 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0198999999999997e-05 2023-10-09T10:45:22.785 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.785 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.785 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.785 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.730624+0000", 2023-10-09T10:45:22.786 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7379000000000002e-05 2023-10-09T10:45:22.786 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.786 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.786 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.786 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.786 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.786 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:67660 6.13 6:cad900bd:::benchmark_data_smithi125_244951_object67659:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:45:22.786 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:20.713528+0000", 2023-10-09T10:45:22.786 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 1.956682292, 2023-10-09T10:45:22.786 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017184347999999999, 2023-10-09T10:45:22.786 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.786 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.787 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.787 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.787 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.787 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 67660 2023-10-09T10:45:22.787 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.787 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.787 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.787 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.787 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.713528+0000", 2023-10-09T10:45:22.787 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.787 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.787 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.787 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.788 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.713528+0000", 2023-10-09T10:45:22.788 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.788 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.788 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.788 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.788 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.713526+0000", 2023-10-09T10:45:22.788 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:22.788 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.788 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.788 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.788 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.713592+0000", 2023-10-09T10:45:22.788 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5926999999999994e-05 2023-10-09T10:45:22.789 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.789 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.789 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.789 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.713605+0000", 2023-10-09T10:45:22.789 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2418e-05 2023-10-09T10:45:22.789 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.789 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.789 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.789 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.713611+0000", 2023-10-09T10:45:22.789 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2999999999999998e-06 2023-10-09T10:45:22.789 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.789 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.789 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.790 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.713654+0000", 2023-10-09T10:45:22.790 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2710999999999998e-05 2023-10-09T10:45:22.790 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.790 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.790 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.790 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.713779+0000", 2023-10-09T10:45:22.790 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012574999999999999 2023-10-09T10:45:22.790 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.790 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.790 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.790 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.713925+0000", 2023-10-09T10:45:22.790 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014590499999999999 2023-10-09T10:45:22.790 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.715035+0000", 2023-10-09T10:45:22.791 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011099580000000001 2023-10-09T10:45:22.791 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.715496+0000", 2023-10-09T10:45:22.791 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00046112900000000001 2023-10-09T10:45:22.791 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.730661+0000", 2023-10-09T10:45:22.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015164545 2023-10-09T10:45:22.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.792 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.730693+0000", 2023-10-09T10:45:22.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2048999999999998e-05 2023-10-09T10:45:22.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.792 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.730712+0000", 2023-10-09T10:45:22.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8972999999999999e-05 2023-10-09T10:45:22.792 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.792 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.793 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.793 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.793 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.793 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:67678 6.13 6:cfab9cf4:::benchmark_data_smithi125_244951_object67677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:45:22.793 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:20.745438+0000", 2023-10-09T10:45:22.793 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 1.9247714899999999, 2023-10-09T10:45:22.793 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018511783, 2023-10-09T10:45:22.793 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.793 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.793 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.793 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.793 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.793 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 67678 2023-10-09T10:45:22.793 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.794 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.794 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.794 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.745438+0000", 2023-10-09T10:45:22.794 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.794 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.794 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.745438+0000", 2023-10-09T10:45:22.794 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.745437+0000", 2023-10-09T10:45:22.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:22.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.795 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.745494+0000", 2023-10-09T10:45:22.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7216999999999998e-05 2023-10-09T10:45:22.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.795 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.745511+0000", 2023-10-09T10:45:22.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6515999999999999e-05 2023-10-09T10:45:22.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.796 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.796 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.745517+0000", 2023-10-09T10:45:22.796 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7390000000000002e-06 2023-10-09T10:45:22.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.796 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.796 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.745573+0000", 2023-10-09T10:45:22.796 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5439999999999998e-05 2023-10-09T10:45:22.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.797 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.797 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.745722+0000", 2023-10-09T10:45:22.797 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014969699999999999 2023-10-09T10:45:22.797 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.797 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.797 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.797 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.745880+0000", 2023-10-09T10:45:22.797 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015719 2023-10-09T10:45:22.797 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.797 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.797 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.797 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.747320+0000", 2023-10-09T10:45:22.797 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014405010000000001 2023-10-09T10:45:22.798 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.798 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.798 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.798 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.748982+0000", 2023-10-09T10:45:22.798 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00166142 2023-10-09T10:45:22.798 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.798 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.798 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.798 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.763857+0000", 2023-10-09T10:45:22.798 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014875138 2023-10-09T10:45:22.798 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.798 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.799 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.799 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.763908+0000", 2023-10-09T10:45:22.799 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1400000000000003e-05 2023-10-09T10:45:22.799 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.799 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.799 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.799 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.763950+0000", 2023-10-09T10:45:22.799 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2104000000000002e-05 2023-10-09T10:45:22.799 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.799 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.799 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.799 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.799 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.800 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:68556 6.13 6:cddf35e3:::benchmark_data_smithi125_244951_object68555:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:45:22.800 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:21.627212+0000", 2023-10-09T10:45:22.800 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 1.0429980190000001, 2023-10-09T10:45:22.800 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018974901999999998, 2023-10-09T10:45:22.800 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.800 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.800 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.800 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.800 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.800 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 68556 2023-10-09T10:45:22.800 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.801 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.801 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.801 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.801 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:21.627212+0000", 2023-10-09T10:45:22.801 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.801 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.801 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.801 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.801 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:21.627212+0000", 2023-10-09T10:45:22.801 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.801 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.801 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.801 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.802 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:21.627210+0000", 2023-10-09T10:45:22.802 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:22.802 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.802 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.802 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.802 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:21.627260+0000", 2023-10-09T10:45:22.802 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9227999999999997e-05 2023-10-09T10:45:22.802 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.802 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.802 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.802 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:21.627275+0000", 2023-10-09T10:45:22.803 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5033e-05 2023-10-09T10:45:22.803 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.803 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.803 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.803 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:21.627282+0000", 2023-10-09T10:45:22.803 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.306e-06 2023-10-09T10:45:22.803 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.803 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.803 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.803 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:21.627331+0000", 2023-10-09T10:45:22.803 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9310000000000001e-05 2023-10-09T10:45:22.803 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.803 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.804 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.804 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:21.627492+0000", 2023-10-09T10:45:22.804 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016066 2023-10-09T10:45:22.804 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.804 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.804 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.804 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:21.627646+0000", 2023-10-09T10:45:22.804 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000153726 2023-10-09T10:45:22.804 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.804 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.804 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.804 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:21.629747+0000", 2023-10-09T10:45:22.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002101847 2023-10-09T10:45:22.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.805 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.805 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:21.629976+0000", 2023-10-09T10:45:22.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022848899999999999 2023-10-09T10:45:22.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.805 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.805 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:21.646098+0000", 2023-10-09T10:45:22.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016122206 2023-10-09T10:45:22.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.806 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.806 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:21.646150+0000", 2023-10-09T10:45:22.806 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2027999999999997e-05 2023-10-09T10:45:22.806 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.806 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.806 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.806 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:21.646187+0000", 2023-10-09T10:45:22.806 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6711000000000001e-05 2023-10-09T10:45:22.806 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.806 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.806 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.806 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.807 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.807 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:69276 6.13 6:cf69c14e:::benchmark_data_smithi125_244951_object69275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:45:22.807 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:22.313789+0000", 2023-10-09T10:45:22.807 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.35642111399999998, 2023-10-09T10:45:22.807 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026275216000000001, 2023-10-09T10:45:22.807 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:45:22.807 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:22.807 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:45:22.807 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:45:22.807 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:22.807 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 69276 2023-10-09T10:45:22.807 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.807 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:45:22.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:45:22.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.313789+0000", 2023-10-09T10:45:22.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:45:22.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.313789+0000", 2023-10-09T10:45:22.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:45:22.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:45:22.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.313787+0000", 2023-10-09T10:45:22.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:22.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:45:22.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.313828+0000", 2023-10-09T10:45:22.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0061999999999997e-05 2023-10-09T10:45:22.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:45:22.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.313840+0000", 2023-10-09T10:45:22.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2680000000000001e-05 2023-10-09T10:45:22.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:45:22.810 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.313848+0000", 2023-10-09T10:45:22.810 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3699999999999997e-06 2023-10-09T10:45:22.810 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.810 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.810 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:45:22.810 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.313891+0000", 2023-10-09T10:45:22.810 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3078000000000003e-05 2023-10-09T10:45:22.810 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.810 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.810 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:45:22.810 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.313994+0000", 2023-10-09T10:45:22.811 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010359 2023-10-09T10:45:22.811 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.811 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.811 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:45:22.811 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.314109+0000", 2023-10-09T10:45:22.811 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011481600000000001 2023-10-09T10:45:22.811 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.811 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.811 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:45:22.811 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.316380+0000", 2023-10-09T10:45:22.811 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022705540000000001 2023-10-09T10:45:22.811 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.811 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.812 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.812 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.318651+0000", 2023-10-09T10:45:22.812 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022717290000000001 2023-10-09T10:45:22.812 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.812 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.812 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:22.812 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.339946+0000", 2023-10-09T10:45:22.812 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021294495 2023-10-09T10:45:22.812 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.812 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.812 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:45:22.812 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.340011+0000", 2023-10-09T10:45:22.812 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5387000000000003e-05 2023-10-09T10:45:22.812 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:45:22.813 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:45:22.813 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:45:22.813 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.340064+0000", 2023-10-09T10:45:22.813 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2815999999999997e-05 2023-10-09T10:45:22.813 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.813 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.813 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.813 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:45:22.813 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:45:22.813 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:45:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:22.814 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:22.815 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:22.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:22.915 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:22.956 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:22.984 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:22.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:22.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:23.061 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:45:23.090 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:23.130 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:23.155 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:23.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:23.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:23.260 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:23.308 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 51 16 70269 70253 86.0563 62.625 0.0299719 0.0116137 2023-10-09T10:45:23.393 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:23.423 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:23.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:23.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:23.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:23.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:23 smithi125 ceph-mon[195507]: pgmap v250: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 90 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 33 MiB/s wr, 532 op/s; 15586/200379 objects degraded (7.778%); 4276/200379 objects misplaced (2.134%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T10:45:23.645 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:23 smithi125 ceph-mon[200197]: pgmap v250: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 90 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 33 MiB/s wr, 532 op/s; 15586/200379 objects degraded (7.778%); 4276/200379 objects misplaced (2.134%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T10:45:23.653 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:23.673 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:23.719 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:23.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:23.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:23.778 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:45:23.824 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:45:23.831 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:23.852 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:45:23.875 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:45:23.886 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:45:23.902 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:23.951 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:45:23.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:23 smithi155 ceph-mon[184854]: pgmap v250: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 90 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 33 MiB/s wr, 532 op/s; 15586/200379 objects degraded (7.778%); 4276/200379 objects misplaced (2.134%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T10:45:24.001 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:45:24.037 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:45:24.080 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:45:24.117 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:45:24.137 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:24.141 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:45:24.141 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:45:24.141 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:45:24.141 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:45:24.141 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:45:24.141 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:45:24.141 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:45:24.141 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:45:24.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:24.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:24.246 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:24.296 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:24.311 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 52 16 71054 71038 85.3428 49.0625 0.0110562 0.0117129 2023-10-09T10:45:24.326 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:24.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:24.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:24.412 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":250,"stamp":"2023-10-09T10:45:23.033686+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4376336288,"num_objects":66793,"num_object_clones":0,"num_object_copies":200379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15586,"num_objects_misplaced":4276,"num_objects_unfound":0,"num_objects_dirty":66793,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":61821,"num_write_kb":3939757,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2393,"num_bytes_recovered":157550752,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87799,"ondisk_log_size":87799,"up":297,"acting":289,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13469056,"kb_used_data":12013704,"kb_used_omap":98,"kb_used_meta":1455197,"kb_avail":642747008,"statfs":{"total":671965249536,"available":658172936192,"internally_reserved":0,"allocated":12302032896,"data_stored":12298519367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":100565,"internal_metadata":1490122539},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,5,4,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":381779968,"num_objects":5826,"num_object_clones":0,"num_object_copies":17478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15586,"num_objects_misplaced":817,"num_objects_unfound":0,"num_objects_dirty":5826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3880,"num_write_kb":248288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":845,"num_bytes_recovered":54723136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.290687"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":107,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.768344+0000","last_change":"2023-10-09T10:43:06.676414+0000","last_active":"2023-10-09T10:45:16.768344+0000","last_peered":"2023-10-09T10:45:16.768344+0000","last_clean":"2023-10-09T10:45:16.768344+0000","last_became_active":"2023-10-09T10:43:06.676050+0000","last_became_peered":"2023-10-09T10:43:06.676050+0000","last_unstale":"2023-10-09T10:45:16.768344+0000","last_undegraded":"2023-10-09T10:45:16.768344+0000","last_fullsized":"2023-10-09T10:45:16.768344+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:28.011477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":91,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751418+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:45:16.751418+0000","last_peered":"2023-10-09T10:45:16.751418+0000","last_clean":"2023-10-09T10:45:16.751418+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:45:16.751418+0000","last_undegraded":"2023-10-09T10:45:16.751418+0000","last_fullsized":"2023-10-09T10:45:16.751418+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":75,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.747980+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:45:16.747980+0000","last_peered":"2023-10-09T10:45:16.747980+0000","last_clean":"2023-10-09T10:45:16.747980+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:45:16.747980+0000","last_undegraded":"2023-10-09T10:45:16.747980+0000","last_fullsized":"2023-10-09T10:45:16.747980+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":58,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:17.799627+0000","last_change":"2023-10-09T10:45:15.767462+0000","last_active":"2023-10-09T10:45:17.799627+0000","last_peered":"2023-10-09T10:45:17.799627+0000","last_clean":"2023-10-09T10:45:17.799627+0000","last_became_active":"2023-10-09T10:45:15.767116+0000","last_became_peered":"2023-10-09T10:45:15.767116+0000","last_unstale":"2023-10-09T10:45:17.799627+0000","last_undegraded":"2023-10-09T10:45:17.799627+0000","last_fullsized":"2023-10-09T10:45:17.799627+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:42.610234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":118,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.762984+0000","last_change":"2023-10-09T10:45:15.768276+0000","last_active":"2023-10-09T10:45:16.762984+0000","last_peered":"2023-10-09T10:45:16.762984+0000","last_clean":"2023-10-09T10:45:16.762984+0000","last_became_active":"2023-10-09T10:45:15.767836+0000","last_became_peered":"2023-10-09T10:45:15.767836+0000","last_unstale":"2023-10-09T10:45:16.762984+0000","last_undegraded":"2023-10-09T10:45:16.762984+0000","last_fullsized":"2023-10-09T10:45:16.762984+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:40.987628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":62,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751341+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:45:16.751341+0000","last_peered":"2023-10-09T10:45:16.751341+0000","last_clean":"2023-10-09T10:45:16.751341+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:45:16.751341+0000","last_undegraded":"2023-10-09T10:45:16.751341+0000","last_fullsized":"2023-10-09T10:45:16.751341+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"124'3182","reported_seq":4094,"reported_epoch":124,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:45:19.678440+0000","last_change":"2023-10-09T10:45:15.825805+0000","last_active":"2023-10-09T10:45:19.678440+0000","last_peered":"2023-10-09T10:45:19.678440+0000","last_clean":"2023-10-09T10:45:13.701802+0000","last_became_active":"2023-10-09T10:45:15.811026+0000","last_became_peered":"2023-10-09T10:45:15.811026+0000","last_unstale":"2023-10-09T10:45:19.678440+0000","last_undegraded":"2023-10-09T10:45:19.678440+0000","last_fullsized":"2023-10-09T10:45:15.751309+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3182,"log_dups_size":0,"ondisk_log_size":3182,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140607488,"num_objects":2146,"num_object_clones":0,"num_object_copies":6438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1576,"num_objects_unfound":0,"num_objects_dirty":2146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2146,"num_write_kb":137312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":429,"num_bytes_recovered":28114944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,4],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3,4","objects":1576},{"shards":"2,4","objects":570}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":58,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751410+0000","last_change":"2023-10-09T10:45:15.763750+0000","last_active":"2023-10-09T10:45:16.751410+0000","last_peered":"2023-10-09T10:45:16.751410+0000","last_clean":"2023-10-09T10:45:16.751410+0000","last_became_active":"2023-10-09T10:45:15.762902+0000","last_became_peered":"2023-10-09T10:45:15.762902+0000","last_unstale":"2023-10-09T10:45:16.751410+0000","last_undegraded":"2023-10-09T10:45:16.751410+0000","last_fullsized":"2023-10-09T10:45:16.751410+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:08.151291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":91,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.763057+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:45:16.763057+0000","last_peered":"2023-10-09T10:45:16.763057+0000","last_clean":"2023-10-09T10:45:16.763057+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:45:16.763057+0000","last_undegraded":"2023-10-09T10:45:16.763057+0000","last_fullsized":"2023-10-09T10:45:16.763057+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"124'3129","reported_seq":3249,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:20.917525+0000","last_change":"2023-10-09T10:45:07.544591+0000","last_active":"2023-10-09T10:45:20.917525+0000","last_peered":"2023-10-09T10:45:20.917525+0000","last_clean":"2023-10-09T10:45:20.917525+0000","last_became_active":"2023-10-09T10:44:50.071995+0000","last_became_peered":"2023-10-09T10:44:50.071995+0000","last_unstale":"2023-10-09T10:45:20.917525+0000","last_undegraded":"2023-10-09T10:45:20.917525+0000","last_fullsized":"2023-10-09T10:45:20.917525+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":3129,"log_dups_size":0,"ondisk_log_size":3129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:45.046717+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136773632,"num_objects":2087,"num_object_clones":0,"num_object_copies":6261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":414,"num_write_kb":26496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":89,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.750757+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:45:16.750757+0000","last_peered":"2023-10-09T10:45:16.750757+0000","last_clean":"2023-10-09T10:45:16.750757+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:45:16.750757+0000","last_undegraded":"2023-10-09T10:45:16.750757+0000","last_fullsized":"2023-10-09T10:45:16.750757+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":50,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751502+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:45:16.751502+0000","last_peered":"2023-10-09T10:45:16.751502+0000","last_clean":"2023-10-09T10:45:16.751502+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:45:16.751502+0000","last_undegraded":"2023-10-09T10:45:16.751502+0000","last_fullsized":"2023-10-09T10:45:16.751502+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"124'3165","reported_seq":7251,"reported_epoch":124,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:45:20.860868+0000","last_change":"2023-10-09T10:45:16.836984+0000","last_active":"2023-10-09T10:45:20.860868+0000","last_peered":"2023-10-09T10:45:20.860868+0000","last_clean":"2023-10-09T10:45:14.179874+0000","last_became_active":"2023-10-09T10:45:16.829821+0000","last_became_peered":"2023-10-09T10:45:16.829821+0000","last_unstale":"2023-10-09T10:45:20.860868+0000","last_undegraded":"2023-10-09T10:45:16.765335+0000","last_fullsized":"2023-10-09T10:45:16.750477+0000","mapping_epoch":122,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3165,"log_dups_size":0,"ondisk_log_size":3165,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138838016,"num_objects":2119,"num_object_clones":0,"num_object_copies":6357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1983,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2119,"num_write_kb":135584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1338,"num_bytes_recovered":87687168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1],"avail_no_missing":["5","0","1","2","3"],"object_location_counts":[{"shards":"0,1,5","objects":1983},{"shards":"1,5","objects":136}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":44,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751338+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:45:16.751338+0000","last_peered":"2023-10-09T10:45:16.751338+0000","last_clean":"2023-10-09T10:45:16.751338+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:45:16.751338+0000","last_undegraded":"2023-10-09T10:45:16.751338+0000","last_fullsized":"2023-10-09T10:45:16.751338+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":75,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.747291+0000","last_change":"2023-10-09T10:43:06.672114+0000","last_active":"2023-10-09T10:45:16.747291+0000","last_peered":"2023-10-09T10:45:16.747291+0000","last_clean":"2023-10-09T10:45:16.747291+0000","last_became_active":"2023-10-09T10:43:06.671827+0000","last_became_peered":"2023-10-09T10:43:06.671827+0000","last_unstale":"2023-10-09T10:45:16.747291+0000","last_undegraded":"2023-10-09T10:45:16.747291+0000","last_fullsized":"2023-10-09T10:45:16.747291+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:41.701409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"124'3136","reported_seq":5025,"reported_epoch":124,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:45:21.382076+0000","last_change":"2023-10-09T10:45:16.870647+0000","last_active":"2023-10-09T10:45:21.382076+0000","last_peered":"2023-10-09T10:45:21.382076+0000","last_clean":"2023-10-09T10:44:56.105580+0000","last_became_active":"2023-10-09T10:45:16.866250+0000","last_became_peered":"2023-10-09T10:45:16.866250+0000","last_unstale":"2023-10-09T10:45:21.382076+0000","last_undegraded":"2023-10-09T10:45:16.812712+0000","last_fullsized":"2023-10-09T10:45:16.747501+0000","mapping_epoch":122,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3136,"log_dups_size":0,"ondisk_log_size":3136,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139493376,"num_objects":2129,"num_object_clones":0,"num_object_copies":6387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1523,"num_objects_misplaced":897,"num_objects_unfound":0,"num_objects_dirty":2129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2129,"num_write_kb":136224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":457,"num_bytes_recovered":29949952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[1,2],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"0,1,3,5","objects":627},{"shards":"1,2","objects":605},{"shards":"1,2,5,6","objects":1},{"shards":"1,2,6","objects":896}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":94,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.767928+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:45:16.767928+0000","last_peered":"2023-10-09T10:45:16.767928+0000","last_clean":"2023-10-09T10:45:16.767928+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:45:16.767928+0000","last_undegraded":"2023-10-09T10:45:16.767928+0000","last_fullsized":"2023-10-09T10:45:16.767928+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":88,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.768077+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:45:16.768077+0000","last_peered":"2023-10-09T10:45:16.768077+0000","last_clean":"2023-10-09T10:45:16.768077+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:45:16.768077+0000","last_undegraded":"2023-10-09T10:45:16.768077+0000","last_fullsized":"2023-10-09T10:45:16.768077+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"124'3055","reported_seq":3102,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:18.496436+0000","last_change":"2023-10-09T10:44:50.093642+0000","last_active":"2023-10-09T10:45:18.496436+0000","last_peered":"2023-10-09T10:45:18.496436+0000","last_clean":"2023-10-09T10:45:18.496436+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:45:18.496436+0000","last_undegraded":"2023-10-09T10:45:18.496436+0000","last_fullsized":"2023-10-09T10:45:18.496436+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3055,"log_dups_size":0,"ondisk_log_size":3055,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131858432,"num_objects":2012,"num_object_clones":0,"num_object_copies":6036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2012,"num_write_kb":128768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":90,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.747452+0000","last_change":"2023-10-09T10:43:06.677115+0000","last_active":"2023-10-09T10:45:16.747452+0000","last_peered":"2023-10-09T10:45:16.747452+0000","last_clean":"2023-10-09T10:45:16.747452+0000","last_became_active":"2023-10-09T10:43:06.676739+0000","last_became_peered":"2023-10-09T10:43:06.676739+0000","last_unstale":"2023-10-09T10:45:16.747452+0000","last_undegraded":"2023-10-09T10:45:16.747452+0000","last_fullsized":"2023-10-09T10:45:16.747452+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:02.520770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":105,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.752375+0000","last_change":"2023-10-09T10:43:35.487942+0000","last_active":"2023-10-09T10:45:16.752375+0000","last_peered":"2023-10-09T10:45:16.752375+0000","last_clean":"2023-10-09T10:45:16.752375+0000","last_became_active":"2023-10-09T10:43:35.487651+0000","last_became_peered":"2023-10-09T10:43:35.487651+0000","last_unstale":"2023-10-09T10:45:16.752375+0000","last_undegraded":"2023-10-09T10:45:16.752375+0000","last_fullsized":"2023-10-09T10:45:16.752375+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:51.882595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"124'3129","reported_seq":3172,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:20.940999+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:45:20.940999+0000","last_peered":"2023-10-09T10:45:20.940999+0000","last_clean":"2023-10-09T10:45:20.940999+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:45:20.940999+0000","last_undegraded":"2023-10-09T10:45:20.940999+0000","last_fullsized":"2023-10-09T10:45:20.940999+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3129,"log_dups_size":0,"ondisk_log_size":3129,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139001856,"num_objects":2121,"num_object_clones":0,"num_object_copies":6363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2121,"num_write_kb":135744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":57,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:17.798509+0000","last_change":"2023-10-09T10:45:15.769708+0000","last_active":"2023-10-09T10:45:17.798509+0000","last_peered":"2023-10-09T10:45:17.798509+0000","last_clean":"2023-10-09T10:45:17.798509+0000","last_became_active":"2023-10-09T10:45:15.769370+0000","last_became_peered":"2023-10-09T10:45:15.769370+0000","last_unstale":"2023-10-09T10:45:17.798509+0000","last_undegraded":"2023-10-09T10:45:17.798509+0000","last_fullsized":"2023-10-09T10:45:17.798509+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:33:21.951569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":88,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.784458+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:45:16.784458+0000","last_peered":"2023-10-09T10:45:16.784458+0000","last_clean":"2023-10-09T10:45:16.784458+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:45:16.784458+0000","last_undegraded":"2023-10-09T10:45:16.784458+0000","last_fullsized":"2023-10-09T10:45:16.784458+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"124'3166","reported_seq":3210,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:21.386142+0000","last_change":"2023-10-09T10:44:50.084697+0000","last_active":"2023-10-09T10:45:21.386142+0000","last_peered":"2023-10-09T10:45:21.386142+0000","last_clean":"2023-10-09T10:45:21.386142+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:45:21.386142+0000","last_undegraded":"2023-10-09T10:45:21.386142+0000","last_fullsized":"2023-10-09T10:45:21.386142+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3166,"log_dups_size":0,"ondisk_log_size":3166,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138379264,"num_objects":2112,"num_object_clones":0,"num_object_copies":6336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2112,"num_write_kb":135136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":34,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.768167+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:45:16.768167+0000","last_peered":"2023-10-09T10:45:16.768167+0000","last_clean":"2023-10-09T10:45:16.768167+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:45:16.768167+0000","last_undegraded":"2023-10-09T10:45:16.768167+0000","last_fullsized":"2023-10-09T10:45:16.768167+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":57,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:17.799693+0000","last_change":"2023-10-09T10:45:15.766405+0000","last_active":"2023-10-09T10:45:17.799693+0000","last_peered":"2023-10-09T10:45:17.799693+0000","last_clean":"2023-10-09T10:45:17.799693+0000","last_became_active":"2023-10-09T10:45:15.766088+0000","last_became_peered":"2023-10-09T10:45:15.766088+0000","last_unstale":"2023-10-09T10:45:17.799693+0000","last_undegraded":"2023-10-09T10:45:17.799693+0000","last_fullsized":"2023-10-09T10:45:17.799693+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:07.504491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.15","version":"124'3114","reported_seq":3171,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:18.509269+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:45:18.509269+0000","last_peered":"2023-10-09T10:45:18.509269+0000","last_clean":"2023-10-09T10:45:18.509269+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:45:18.509269+0000","last_undegraded":"2023-10-09T10:45:18.509269+0000","last_fullsized":"2023-10-09T10:45:18.509269+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3114,"log_dups_size":0,"ondisk_log_size":3114,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134545408,"num_objects":2053,"num_object_clones":0,"num_object_copies":6159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2053,"num_write_kb":131392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":36,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.766668+0000","last_change":"2023-10-09T10:45:15.782117+0000","last_active":"2023-10-09T10:45:16.766668+0000","last_peered":"2023-10-09T10:45:16.766668+0000","last_clean":"2023-10-09T10:45:16.766668+0000","last_became_active":"2023-10-09T10:45:15.765683+0000","last_became_peered":"2023-10-09T10:45:15.765683+0000","last_unstale":"2023-10-09T10:45:16.766668+0000","last_undegraded":"2023-10-09T10:45:16.766668+0000","last_fullsized":"2023-10-09T10:45:16.766668+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:39.454280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":91,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751676+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:45:16.751676+0000","last_peered":"2023-10-09T10:45:16.751676+0000","last_clean":"2023-10-09T10:45:16.751676+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:45:16.751676+0000","last_undegraded":"2023-10-09T10:45:16.751676+0000","last_fullsized":"2023-10-09T10:45:16.751676+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"124'4448","reported_seq":4492,"reported_epoch":124,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:45:20.919220+0000","last_change":"2023-10-09T10:45:16.985715+0000","last_active":"2023-10-09T10:45:20.919220+0000","last_peered":"2023-10-09T10:45:20.919220+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:45:16.949545+0000","last_became_peered":"2023-10-09T10:45:16.949545+0000","last_unstale":"2023-10-09T10:45:20.919220+0000","last_undegraded":"2023-10-09T10:45:16.782589+0000","last_fullsized":"2023-10-09T10:45:16.750475+0000","mapping_epoch":122,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4448,"log_dups_size":0,"ondisk_log_size":4448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:44.131701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291504128,"num_objects":4448,"num_object_clones":0,"num_object_copies":13344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4166,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4448,"num_write_kb":284672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"0,5,6","objects":4166},{"shards":"5,6","objects":282}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":103,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.747812+0000","last_change":"2023-10-09T10:45:15.787875+0000","last_active":"2023-10-09T10:45:16.747812+0000","last_peered":"2023-10-09T10:45:16.747812+0000","last_clean":"2023-10-09T10:45:16.747812+0000","last_became_active":"2023-10-09T10:45:15.766410+0000","last_became_peered":"2023-10-09T10:45:15.766410+0000","last_unstale":"2023-10-09T10:45:16.747812+0000","last_undegraded":"2023-10-09T10:45:16.747812+0000","last_fullsized":"2023-10-09T10:45:16.747812+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:49.209544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":71,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.766740+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:45:16.766740+0000","last_peered":"2023-10-09T10:45:16.766740+0000","last_clean":"2023-10-09T10:45:16.766740+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:45:16.766740+0000","last_undegraded":"2023-10-09T10:45:16.766740+0000","last_fullsized":"2023-10-09T10:45:16.766740+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"124'4014","reported_seq":4049,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:18.528559+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:45:18.528559+0000","last_peered":"2023-10-09T10:45:18.528559+0000","last_clean":"2023-10-09T10:45:18.528559+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:45:18.528559+0000","last_undegraded":"2023-10-09T10:45:18.528559+0000","last_fullsized":"2023-10-09T10:45:18.528559+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4014,"log_dups_size":0,"ondisk_log_size":4014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263061504,"num_objects":4014,"num_object_clones":0,"num_object_copies":12042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4014,"num_write_kb":256896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":90,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.747948+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:45:16.747948+0000","last_peered":"2023-10-09T10:45:16.747948+0000","last_clean":"2023-10-09T10:45:16.747948+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:45:16.747948+0000","last_undegraded":"2023-10-09T10:45:16.747948+0000","last_fullsized":"2023-10-09T10:45:16.747948+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":115,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.767649+0000","last_change":"2023-10-09T10:43:35.493278+0000","last_active":"2023-10-09T10:45:16.767649+0000","last_peered":"2023-10-09T10:45:16.767649+0000","last_clean":"2023-10-09T10:45:16.767649+0000","last_became_active":"2023-10-09T10:43:35.492915+0000","last_became_peered":"2023-10-09T10:43:35.492915+0000","last_unstale":"2023-10-09T10:45:16.767649+0000","last_undegraded":"2023-10-09T10:45:16.767649+0000","last_fullsized":"2023-10-09T10:45:16.767649+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:42.227364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"124'3176","reported_seq":3225,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:19.638814+0000","last_change":"2023-10-09T10:45:06.950897+0000","last_active":"2023-10-09T10:45:19.638814+0000","last_peered":"2023-10-09T10:45:19.638814+0000","last_clean":"2023-10-09T10:45:19.638814+0000","last_became_active":"2023-10-09T10:44:50.063797+0000","last_became_peered":"2023-10-09T10:44:50.063797+0000","last_unstale":"2023-10-09T10:45:19.638814+0000","last_undegraded":"2023-10-09T10:45:19.638814+0000","last_fullsized":"2023-10-09T10:45:19.638814+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3176,"log_dups_size":0,"ondisk_log_size":3176,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138379264,"num_objects":2111,"num_object_clones":0,"num_object_copies":6333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2111,"num_write_kb":135136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":45,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.784339+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:45:16.784339+0000","last_peered":"2023-10-09T10:45:16.784339+0000","last_clean":"2023-10-09T10:45:16.784339+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:45:16.784339+0000","last_undegraded":"2023-10-09T10:45:16.784339+0000","last_fullsized":"2023-10-09T10:45:16.784339+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":61,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.747613+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:45:16.747613+0000","last_peered":"2023-10-09T10:45:16.747613+0000","last_clean":"2023-10-09T10:45:16.747613+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:45:16.747613+0000","last_undegraded":"2023-10-09T10:45:16.747613+0000","last_fullsized":"2023-10-09T10:45:16.747613+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"124'3139","reported_seq":3189,"reported_epoch":124,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:45:19.661548+0000","last_change":"2023-10-09T10:45:16.848181+0000","last_active":"2023-10-09T10:45:19.661548+0000","last_peered":"2023-10-09T10:45:19.661548+0000","last_clean":"2023-10-09T10:45:14.284950+0000","last_became_active":"2023-10-09T10:45:16.845243+0000","last_became_peered":"2023-10-09T10:45:16.845243+0000","last_unstale":"2023-10-09T10:45:19.661548+0000","last_undegraded":"2023-10-09T10:45:16.764961+0000","last_fullsized":"2023-10-09T10:45:16.748120+0000","mapping_epoch":122,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3139,"log_dups_size":0,"ondisk_log_size":3139,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137428992,"num_objects":2097,"num_object_clones":0,"num_object_copies":6291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2003,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2097,"num_write_kb":134208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2,7","objects":2003},{"shards":"2,7","objects":94}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":89,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751853+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:45:16.751853+0000","last_peered":"2023-10-09T10:45:16.751853+0000","last_clean":"2023-10-09T10:45:16.751853+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:45:16.751853+0000","last_undegraded":"2023-10-09T10:45:16.751853+0000","last_fullsized":"2023-10-09T10:45:16.751853+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":47,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.750810+0000","last_change":"2023-10-09T10:45:15.767409+0000","last_active":"2023-10-09T10:45:16.750810+0000","last_peered":"2023-10-09T10:45:16.750810+0000","last_clean":"2023-10-09T10:45:16.750810+0000","last_became_active":"2023-10-09T10:45:15.766857+0000","last_became_peered":"2023-10-09T10:45:15.766857+0000","last_unstale":"2023-10-09T10:45:16.750810+0000","last_undegraded":"2023-10-09T10:45:16.750810+0000","last_fullsized":"2023-10-09T10:45:16.750810+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:06.417379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.e","version":"124'4152","reported_seq":4185,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:20.941043+0000","last_change":"2023-10-09T10:44:27.574527+0000","last_active":"2023-10-09T10:45:20.941043+0000","last_peered":"2023-10-09T10:45:20.941043+0000","last_clean":"2023-10-09T10:45:20.941043+0000","last_became_active":"2023-10-09T10:44:27.574205+0000","last_became_peered":"2023-10-09T10:44:27.574205+0000","last_unstale":"2023-10-09T10:45:20.941043+0000","last_undegraded":"2023-10-09T10:45:20.941043+0000","last_fullsized":"2023-10-09T10:45:20.941043+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4152,"log_dups_size":0,"ondisk_log_size":4152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:54.868034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272105472,"num_objects":4152,"num_object_clones":0,"num_object_copies":12456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4152,"num_write_kb":265728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":102,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.752102+0000","last_change":"2023-10-09T10:45:15.789677+0000","last_active":"2023-10-09T10:45:16.752102+0000","last_peered":"2023-10-09T10:45:16.752102+0000","last_clean":"2023-10-09T10:45:16.752102+0000","last_became_active":"2023-10-09T10:45:15.775547+0000","last_became_peered":"2023-10-09T10:45:15.775547+0000","last_unstale":"2023-10-09T10:45:16.752102+0000","last_undegraded":"2023-10-09T10:45:16.752102+0000","last_fullsized":"2023-10-09T10:45:16.752102+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:13.878981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":88,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.767849+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:45:16.767849+0000","last_peered":"2023-10-09T10:45:16.767849+0000","last_clean":"2023-10-09T10:45:16.767849+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:45:16.767849+0000","last_undegraded":"2023-10-09T10:45:16.767849+0000","last_fullsized":"2023-10-09T10:45:16.767849+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"124'4065","reported_seq":4100,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:19.635365+0000","last_change":"2023-10-09T10:44:27.574938+0000","last_active":"2023-10-09T10:45:19.635365+0000","last_peered":"2023-10-09T10:45:19.635365+0000","last_clean":"2023-10-09T10:45:19.635365+0000","last_became_active":"2023-10-09T10:44:27.574662+0000","last_became_peered":"2023-10-09T10:44:27.574662+0000","last_unstale":"2023-10-09T10:45:19.635365+0000","last_undegraded":"2023-10-09T10:45:19.635365+0000","last_fullsized":"2023-10-09T10:45:19.635365+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4065,"log_dups_size":0,"ondisk_log_size":4065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:35:19.975925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266403840,"num_objects":4065,"num_object_clones":0,"num_object_copies":12195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4065,"num_write_kb":260160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":63,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.768341+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:45:16.768341+0000","last_peered":"2023-10-09T10:45:16.768341+0000","last_clean":"2023-10-09T10:45:16.768341+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:45:16.768341+0000","last_undegraded":"2023-10-09T10:45:16.768341+0000","last_fullsized":"2023-10-09T10:45:16.768341+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":105,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.767910+0000","last_change":"2023-10-09T10:43:06.676367+0000","last_active":"2023-10-09T10:45:16.767910+0000","last_peered":"2023-10-09T10:45:16.767910+0000","last_clean":"2023-10-09T10:45:16.767910+0000","last_became_active":"2023-10-09T10:43:06.676017+0000","last_became_peered":"2023-10-09T10:43:06.676017+0000","last_unstale":"2023-10-09T10:45:16.767910+0000","last_undegraded":"2023-10-09T10:45:16.767910+0000","last_fullsized":"2023-10-09T10:45:16.767910+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:54.387617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"124'4129","reported_seq":4164,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:18.511813+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:45:18.511813+0000","last_peered":"2023-10-09T10:45:18.511813+0000","last_clean":"2023-10-09T10:45:18.511813+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:45:18.511813+0000","last_undegraded":"2023-10-09T10:45:18.511813+0000","last_fullsized":"2023-10-09T10:45:18.511813+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4129,"log_dups_size":0,"ondisk_log_size":4129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270598144,"num_objects":4129,"num_object_clones":0,"num_object_copies":12387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4129,"num_write_kb":264256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":89,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.767636+0000","last_change":"2023-10-09T10:43:06.676577+0000","last_active":"2023-10-09T10:45:16.767636+0000","last_peered":"2023-10-09T10:45:16.767636+0000","last_clean":"2023-10-09T10:45:16.767636+0000","last_became_active":"2023-10-09T10:43:06.676165+0000","last_became_peered":"2023-10-09T10:43:06.676165+0000","last_unstale":"2023-10-09T10:45:16.767636+0000","last_undegraded":"2023-10-09T10:45:16.767636+0000","last_fullsized":"2023-10-09T10:45:16.767636+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:52.614985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":88,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.767931+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:45:16.767931+0000","last_peered":"2023-10-09T10:45:16.767931+0000","last_clean":"2023-10-09T10:45:16.767931+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:45:16.767931+0000","last_undegraded":"2023-10-09T10:45:16.767931+0000","last_fullsized":"2023-10-09T10:45:16.767931+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"124'4271","reported_seq":4313,"reported_epoch":124,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:45:20.918477+0000","last_change":"2023-10-09T10:45:16.933804+0000","last_active":"2023-10-09T10:45:20.918477+0000","last_peered":"2023-10-09T10:45:20.918477+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:45:16.923149+0000","last_became_peered":"2023-10-09T10:45:16.923149+0000","last_unstale":"2023-10-09T10:45:20.918477+0000","last_undegraded":"2023-10-09T10:45:16.782245+0000","last_fullsized":"2023-10-09T10:45:16.750546+0000","mapping_epoch":122,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4271,"log_dups_size":0,"ondisk_log_size":4271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:57.297705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279904256,"num_objects":4271,"num_object_clones":0,"num_object_copies":12813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3979,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4271,"num_write_kb":273344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1],"avail_no_missing":["5","0","1"],"object_location_counts":[{"shards":"0,1,5","objects":3979},{"shards":"1,5","objects":292}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":92,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.762837+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:45:16.762837+0000","last_peered":"2023-10-09T10:45:16.762837+0000","last_clean":"2023-10-09T10:45:16.762837+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:45:16.762837+0000","last_undegraded":"2023-10-09T10:45:16.762837+0000","last_fullsized":"2023-10-09T10:45:16.762837+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":75,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.750301+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:45:16.750301+0000","last_peered":"2023-10-09T10:45:16.750301+0000","last_clean":"2023-10-09T10:45:16.750301+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:45:16.750301+0000","last_undegraded":"2023-10-09T10:45:16.750301+0000","last_fullsized":"2023-10-09T10:45:16.750301+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":109,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751064+0000","last_change":"2023-10-09T10:45:15.764916+0000","last_active":"2023-10-09T10:45:16.751064+0000","last_peered":"2023-10-09T10:45:16.751064+0000","last_clean":"2023-10-09T10:45:16.751064+0000","last_became_active":"2023-10-09T10:45:15.764607+0000","last_became_peered":"2023-10-09T10:45:15.764607+0000","last_unstale":"2023-10-09T10:45:16.751064+0000","last_undegraded":"2023-10-09T10:45:16.751064+0000","last_fullsized":"2023-10-09T10:45:16.751064+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:21.140378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":106,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.784641+0000","last_change":"2023-10-09T10:43:35.486524+0000","last_active":"2023-10-09T10:45:16.784641+0000","last_peered":"2023-10-09T10:45:16.784641+0000","last_clean":"2023-10-09T10:45:16.784641+0000","last_became_active":"2023-10-09T10:43:35.486152+0000","last_became_peered":"2023-10-09T10:43:35.486152+0000","last_unstale":"2023-10-09T10:45:16.784641+0000","last_undegraded":"2023-10-09T10:45:16.784641+0000","last_fullsized":"2023-10-09T10:45:16.784641+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:47.200522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":60,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.747378+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:45:16.747378+0000","last_peered":"2023-10-09T10:45:16.747378+0000","last_clean":"2023-10-09T10:45:16.747378+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:45:16.747378+0000","last_undegraded":"2023-10-09T10:45:16.747378+0000","last_fullsized":"2023-10-09T10:45:16.747378+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"124'3071","reported_seq":3423,"reported_epoch":124,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:45:18.521813+0000","last_change":"2023-10-09T10:45:16.856420+0000","last_active":"2023-10-09T10:45:18.521813+0000","last_peered":"2023-10-09T10:45:18.521813+0000","last_clean":"2023-10-09T10:45:14.144268+0000","last_became_active":"2023-10-09T10:45:16.839118+0000","last_became_peered":"2023-10-09T10:45:16.839118+0000","last_unstale":"2023-10-09T10:45:18.521813+0000","last_undegraded":"2023-10-09T10:45:18.521813+0000","last_fullsized":"2023-10-09T10:45:16.751479+0000","mapping_epoch":122,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3071,"log_dups_size":0,"ondisk_log_size":3071,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132677632,"num_objects":2024,"num_object_clones":0,"num_object_copies":6072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1801,"num_objects_unfound":0,"num_objects_dirty":2024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2024,"num_write_kb":129568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":9830400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":1801},{"shards":"2,3","objects":223}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.6","version":"124'3087","reported_seq":3134,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:18.432095+0000","last_change":"2023-10-09T10:44:50.088013+0000","last_active":"2023-10-09T10:45:18.432095+0000","last_peered":"2023-10-09T10:45:18.432095+0000","last_clean":"2023-10-09T10:45:18.432095+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:45:18.432095+0000","last_undegraded":"2023-10-09T10:45:18.432095+0000","last_fullsized":"2023-10-09T10:45:18.432095+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3087,"log_dups_size":0,"ondisk_log_size":3087,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133955584,"num_objects":2044,"num_object_clones":0,"num_object_copies":6132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2044,"num_write_kb":130816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":93,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.784567+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:45:16.784567+0000","last_peered":"2023-10-09T10:45:16.784567+0000","last_clean":"2023-10-09T10:45:16.784567+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:45:16.784567+0000","last_undegraded":"2023-10-09T10:45:16.784567+0000","last_fullsized":"2023-10-09T10:45:16.784567+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.768175+0000","last_change":"2023-10-09T10:45:15.770080+0000","last_active":"2023-10-09T10:45:16.768175+0000","last_peered":"2023-10-09T10:45:16.768175+0000","last_clean":"2023-10-09T10:45:16.768175+0000","last_became_active":"2023-10-09T10:45:15.769640+0000","last_became_peered":"2023-10-09T10:45:15.769640+0000","last_unstale":"2023-10-09T10:45:16.768175+0000","last_undegraded":"2023-10-09T10:45:16.768175+0000","last_fullsized":"2023-10-09T10:45:16.768175+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:17.992596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":90,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.752724+0000","last_change":"2023-10-09T10:43:35.489332+0000","last_active":"2023-10-09T10:45:16.752724+0000","last_peered":"2023-10-09T10:45:16.752724+0000","last_clean":"2023-10-09T10:45:16.752724+0000","last_became_active":"2023-10-09T10:43:35.489044+0000","last_became_peered":"2023-10-09T10:43:35.489044+0000","last_unstale":"2023-10-09T10:45:16.752724+0000","last_undegraded":"2023-10-09T10:45:16.752724+0000","last_fullsized":"2023-10-09T10:45:16.752724+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:42.168915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"124'3110","reported_seq":3170,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:18.509145+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:45:18.509145+0000","last_peered":"2023-10-09T10:45:18.509145+0000","last_clean":"2023-10-09T10:45:18.509145+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:45:18.509145+0000","last_undegraded":"2023-10-09T10:45:18.509145+0000","last_fullsized":"2023-10-09T10:45:18.509145+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3110,"log_dups_size":0,"ondisk_log_size":3110,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134283264,"num_objects":2049,"num_object_clones":0,"num_object_copies":6147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2049,"num_write_kb":131136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":120,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.747877+0000","last_change":"2023-10-09T10:45:15.768379+0000","last_active":"2023-10-09T10:45:16.747877+0000","last_peered":"2023-10-09T10:45:16.747877+0000","last_clean":"2023-10-09T10:45:16.747877+0000","last_became_active":"2023-10-09T10:45:15.767987+0000","last_became_peered":"2023-10-09T10:45:15.767987+0000","last_unstale":"2023-10-09T10:45:16.747877+0000","last_undegraded":"2023-10-09T10:45:16.747877+0000","last_fullsized":"2023-10-09T10:45:16.747877+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:55.314341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":75,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.747706+0000","last_change":"2023-10-09T10:43:06.672862+0000","last_active":"2023-10-09T10:45:16.747706+0000","last_peered":"2023-10-09T10:45:16.747706+0000","last_clean":"2023-10-09T10:45:16.747706+0000","last_became_active":"2023-10-09T10:43:06.672572+0000","last_became_peered":"2023-10-09T10:43:06.672572+0000","last_unstale":"2023-10-09T10:45:16.747706+0000","last_undegraded":"2023-10-09T10:45:16.747706+0000","last_fullsized":"2023-10-09T10:45:16.747706+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:23.743910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":104,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.750225+0000","last_change":"2023-10-09T10:45:15.773125+0000","last_active":"2023-10-09T10:45:16.750225+0000","last_peered":"2023-10-09T10:45:16.750225+0000","last_clean":"2023-10-09T10:45:16.750225+0000","last_became_active":"2023-10-09T10:45:15.772652+0000","last_became_peered":"2023-10-09T10:45:15.772652+0000","last_unstale":"2023-10-09T10:45:16.750225+0000","last_undegraded":"2023-10-09T10:45:16.750225+0000","last_fullsized":"2023-10-09T10:45:16.750225+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:23:38.484481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"124'3113","reported_seq":3161,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:20.940884+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:45:20.940884+0000","last_peered":"2023-10-09T10:45:20.940884+0000","last_clean":"2023-10-09T10:45:20.940884+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:45:20.940884+0000","last_undegraded":"2023-10-09T10:45:20.940884+0000","last_fullsized":"2023-10-09T10:45:20.940884+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3113,"log_dups_size":0,"ondisk_log_size":3113,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137953280,"num_objects":2105,"num_object_clones":0,"num_object_copies":6315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2105,"num_write_kb":134720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":93,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751028+0000","last_change":"2023-10-09T10:43:06.673931+0000","last_active":"2023-10-09T10:45:16.751028+0000","last_peered":"2023-10-09T10:45:16.751028+0000","last_clean":"2023-10-09T10:45:16.751028+0000","last_became_active":"2023-10-09T10:43:06.673605+0000","last_became_peered":"2023-10-09T10:43:06.673605+0000","last_unstale":"2023-10-09T10:45:16.751028+0000","last_undegraded":"2023-10-09T10:45:16.751028+0000","last_fullsized":"2023-10-09T10:45:16.751028+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:19.088672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":441,"reported_epoch":123,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:45:16.758679+0000","last_change":"2023-10-09T10:45:16.758679+0000","last_active":"2023-10-09T10:45:16.758679+0000","last_peered":"2023-10-09T10:45:16.758679+0000","last_clean":"2023-10-09T10:45:13.733443+0000","last_became_active":"2023-10-09T10:45:16.758278+0000","last_became_peered":"2023-10-09T10:45:16.758278+0000","last_unstale":"2023-10-09T10:45:16.758679+0000","last_undegraded":"2023-10-09T10:45:16.758679+0000","last_fullsized":"2023-10-09T10:45:16.758679+0000","mapping_epoch":122,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:56.077188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[0,7,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":50,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751541+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:45:16.751541+0000","last_peered":"2023-10-09T10:45:16.751541+0000","last_clean":"2023-10-09T10:45:16.751541+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:45:16.751541+0000","last_undegraded":"2023-10-09T10:45:16.751541+0000","last_fullsized":"2023-10-09T10:45:16.751541+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":21,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.767723+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:45:16.767723+0000","last_peered":"2023-10-09T10:45:16.767723+0000","last_clean":"2023-10-09T10:45:16.767723+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:45:16.767723+0000","last_undegraded":"2023-10-09T10:45:16.767723+0000","last_fullsized":"2023-10-09T10:45:16.767723+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":105,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.747843+0000","last_change":"2023-10-09T10:45:15.763912+0000","last_active":"2023-10-09T10:45:16.747843+0000","last_peered":"2023-10-09T10:45:16.747843+0000","last_clean":"2023-10-09T10:45:16.747843+0000","last_became_active":"2023-10-09T10:45:15.763634+0000","last_became_peered":"2023-10-09T10:45:15.763634+0000","last_unstale":"2023-10-09T10:45:16.747843+0000","last_undegraded":"2023-10-09T10:45:16.747843+0000","last_fullsized":"2023-10-09T10:45:16.747843+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:42:50.962688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":82,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:17.799126+0000","last_change":"2023-10-09T10:45:15.762810+0000","last_active":"2023-10-09T10:45:17.799126+0000","last_peered":"2023-10-09T10:45:17.799126+0000","last_clean":"2023-10-09T10:45:17.799126+0000","last_became_active":"2023-10-09T10:45:15.762469+0000","last_became_peered":"2023-10-09T10:45:15.762469+0000","last_unstale":"2023-10-09T10:45:17.799126+0000","last_undegraded":"2023-10-09T10:45:17.799126+0000","last_fullsized":"2023-10-09T10:45:17.799126+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:09.451806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.750845+0000","last_change":"2023-10-09T10:45:15.768399+0000","last_active":"2023-10-09T10:45:16.750845+0000","last_peered":"2023-10-09T10:45:16.750845+0000","last_clean":"2023-10-09T10:45:16.750845+0000","last_became_active":"2023-10-09T10:45:15.768067+0000","last_became_peered":"2023-10-09T10:45:15.768067+0000","last_unstale":"2023-10-09T10:45:16.750845+0000","last_undegraded":"2023-10-09T10:45:16.750845+0000","last_fullsized":"2023-10-09T10:45:16.750845+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:43.467271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.4","version":"124'3192","reported_seq":3286,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:21.345933+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:45:21.345933+0000","last_peered":"2023-10-09T10:45:21.345933+0000","last_clean":"2023-10-09T10:45:21.345933+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:45:21.345933+0000","last_undegraded":"2023-10-09T10:45:21.345933+0000","last_fullsized":"2023-10-09T10:45:21.345933+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":3192,"log_dups_size":0,"ondisk_log_size":3192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138149888,"num_objects":2108,"num_object_clones":0,"num_object_copies":6324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":162,"num_write_kb":10368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":102,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.748083+0000","last_change":"2023-10-09T10:45:15.766600+0000","last_active":"2023-10-09T10:45:16.748083+0000","last_peered":"2023-10-09T10:45:16.748083+0000","last_clean":"2023-10-09T10:45:16.748083+0000","last_became_active":"2023-10-09T10:45:15.766192+0000","last_became_peered":"2023-10-09T10:45:15.766192+0000","last_unstale":"2023-10-09T10:45:16.748083+0000","last_undegraded":"2023-10-09T10:45:16.748083+0000","last_fullsized":"2023-10-09T10:45:16.748083+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:01:13.866396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":75,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.752772+0000","last_change":"2023-10-09T10:43:06.671356+0000","last_active":"2023-10-09T10:45:16.752772+0000","last_peered":"2023-10-09T10:45:16.752772+0000","last_clean":"2023-10-09T10:45:16.752772+0000","last_became_active":"2023-10-09T10:43:06.671040+0000","last_became_peered":"2023-10-09T10:43:06.671040+0000","last_unstale":"2023-10-09T10:45:16.752772+0000","last_undegraded":"2023-10-09T10:45:16.752772+0000","last_fullsized":"2023-10-09T10:45:16.752772+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:47.601448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":28,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751174+0000","last_change":"2023-10-09T10:45:15.769154+0000","last_active":"2023-10-09T10:45:16.751174+0000","last_peered":"2023-10-09T10:45:16.751174+0000","last_clean":"2023-10-09T10:45:16.751174+0000","last_became_active":"2023-10-09T10:45:15.768827+0000","last_became_peered":"2023-10-09T10:45:15.768827+0000","last_unstale":"2023-10-09T10:45:16.751174+0000","last_undegraded":"2023-10-09T10:45:16.751174+0000","last_fullsized":"2023-10-09T10:45:16.751174+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:40.970054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1","version":"124'3089","reported_seq":3133,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:21.337569+0000","last_change":"2023-10-09T10:44:50.065999+0000","last_active":"2023-10-09T10:45:21.337569+0000","last_peered":"2023-10-09T10:45:21.337569+0000","last_clean":"2023-10-09T10:45:21.337569+0000","last_became_active":"2023-10-09T10:44:50.064940+0000","last_became_peered":"2023-10-09T10:44:50.064940+0000","last_unstale":"2023-10-09T10:45:21.337569+0000","last_undegraded":"2023-10-09T10:45:21.337569+0000","last_fullsized":"2023-10-09T10:45:21.337569+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3089,"log_dups_size":0,"ondisk_log_size":3089,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136413184,"num_objects":2081,"num_object_clones":0,"num_object_copies":6243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2081,"num_write_kb":133216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":102,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751720+0000","last_change":"2023-10-09T10:45:15.778104+0000","last_active":"2023-10-09T10:45:16.751720+0000","last_peered":"2023-10-09T10:45:16.751720+0000","last_clean":"2023-10-09T10:45:16.751720+0000","last_became_active":"2023-10-09T10:45:15.766880+0000","last_became_peered":"2023-10-09T10:45:15.766880+0000","last_unstale":"2023-10-09T10:45:16.751720+0000","last_undegraded":"2023-10-09T10:45:16.751720+0000","last_fullsized":"2023-10-09T10:45:16.751720+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:59:39.885831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":60,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.752446+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:45:16.752446+0000","last_peered":"2023-10-09T10:45:16.752446+0000","last_clean":"2023-10-09T10:45:16.752446+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:45:16.752446+0000","last_undegraded":"2023-10-09T10:45:16.752446+0000","last_fullsized":"2023-10-09T10:45:16.752446+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":95,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.762911+0000","last_change":"2023-10-09T10:45:15.768090+0000","last_active":"2023-10-09T10:45:16.762911+0000","last_peered":"2023-10-09T10:45:16.762911+0000","last_clean":"2023-10-09T10:45:16.762911+0000","last_became_active":"2023-10-09T10:45:15.767438+0000","last_became_peered":"2023-10-09T10:45:15.767438+0000","last_unstale":"2023-10-09T10:45:16.762911+0000","last_undegraded":"2023-10-09T10:45:16.762911+0000","last_fullsized":"2023-10-09T10:45:16.762911+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:54.095543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"124'3174","reported_seq":3228,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:20.918283+0000","last_change":"2023-10-09T10:45:14.330179+0000","last_active":"2023-10-09T10:45:20.918283+0000","last_peered":"2023-10-09T10:45:20.918283+0000","last_clean":"2023-10-09T10:45:20.918283+0000","last_became_active":"2023-10-09T10:44:50.060284+0000","last_became_peered":"2023-10-09T10:44:50.060284+0000","last_unstale":"2023-10-09T10:45:20.918283+0000","last_undegraded":"2023-10-09T10:45:20.918283+0000","last_fullsized":"2023-10-09T10:45:20.918283+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3174,"log_dups_size":0,"ondisk_log_size":3174,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140083200,"num_objects":2137,"num_object_clones":0,"num_object_copies":6411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2137,"num_write_kb":136800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":71,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.748904+0000","last_change":"2023-10-09T10:45:07.620938+0000","last_active":"2023-10-09T10:45:16.748904+0000","last_peered":"2023-10-09T10:45:16.748904+0000","last_clean":"2023-10-09T10:45:16.748904+0000","last_became_active":"2023-10-09T10:45:07.620627+0000","last_became_peered":"2023-10-09T10:45:07.620627+0000","last_unstale":"2023-10-09T10:45:16.748904+0000","last_undegraded":"2023-10-09T10:45:16.748904+0000","last_fullsized":"2023-10-09T10:45:16.748904+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:13.415622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":39,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.752562+0000","last_change":"2023-10-09T10:45:15.783236+0000","last_active":"2023-10-09T10:45:16.752562+0000","last_peered":"2023-10-09T10:45:16.752562+0000","last_clean":"2023-10-09T10:45:16.752562+0000","last_became_active":"2023-10-09T10:45:15.770551+0000","last_became_peered":"2023-10-09T10:45:15.770551+0000","last_unstale":"2023-10-09T10:45:16.752562+0000","last_undegraded":"2023-10-09T10:45:16.752562+0000","last_fullsized":"2023-10-09T10:45:16.752562+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:15:37.696584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":71,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.767552+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:45:16.767552+0000","last_peered":"2023-10-09T10:45:16.767552+0000","last_clean":"2023-10-09T10:45:16.767552+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:45:16.767552+0000","last_undegraded":"2023-10-09T10:45:16.767552+0000","last_fullsized":"2023-10-09T10:45:16.767552+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"124'3102","reported_seq":3223,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:20.905241+0000","last_change":"2023-10-09T10:45:05.770329+0000","last_active":"2023-10-09T10:45:20.905241+0000","last_peered":"2023-10-09T10:45:20.905241+0000","last_clean":"2023-10-09T10:45:20.905241+0000","last_became_active":"2023-10-09T10:44:50.061195+0000","last_became_peered":"2023-10-09T10:44:50.061195+0000","last_unstale":"2023-10-09T10:45:20.905241+0000","last_undegraded":"2023-10-09T10:45:20.905241+0000","last_fullsized":"2023-10-09T10:45:20.905241+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":3102,"log_dups_size":0,"ondisk_log_size":3102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:02.077286+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136708096,"num_objects":2086,"num_object_clones":0,"num_object_copies":6258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":457,"num_write_kb":29248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":25,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.768190+0000","last_change":"2023-10-09T10:45:15.770124+0000","last_active":"2023-10-09T10:45:16.768190+0000","last_peered":"2023-10-09T10:45:16.768190+0000","last_clean":"2023-10-09T10:45:16.768190+0000","last_became_active":"2023-10-09T10:45:15.769787+0000","last_became_peered":"2023-10-09T10:45:15.769787+0000","last_unstale":"2023-10-09T10:45:16.768190+0000","last_undegraded":"2023-10-09T10:45:16.768190+0000","last_fullsized":"2023-10-09T10:45:16.768190+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:09.577196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":109,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.748618+0000","last_change":"2023-10-09T10:43:35.489655+0000","last_active":"2023-10-09T10:45:16.748618+0000","last_peered":"2023-10-09T10:45:16.748618+0000","last_clean":"2023-10-09T10:45:16.748618+0000","last_became_active":"2023-10-09T10:43:35.489149+0000","last_became_peered":"2023-10-09T10:43:35.489149+0000","last_unstale":"2023-10-09T10:45:16.748618+0000","last_undegraded":"2023-10-09T10:45:16.748618+0000","last_fullsized":"2023-10-09T10:45:16.748618+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:30.598962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"124'3072","reported_seq":3123,"reported_epoch":124,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:45:19.661521+0000","last_change":"2023-10-09T10:45:16.848179+0000","last_active":"2023-10-09T10:45:19.661521+0000","last_peered":"2023-10-09T10:45:19.661521+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:45:16.845016+0000","last_became_peered":"2023-10-09T10:45:16.845016+0000","last_unstale":"2023-10-09T10:45:19.661521+0000","last_undegraded":"2023-10-09T10:45:16.765670+0000","last_fullsized":"2023-10-09T10:45:16.747355+0000","mapping_epoch":122,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3072,"log_dups_size":0,"ondisk_log_size":3072,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133038080,"num_objects":2030,"num_object_clones":0,"num_object_copies":6090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1932,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2030,"num_write_kb":129920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2,7","objects":1932},{"shards":"2,7","objects":98}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":91,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751987+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:45:16.751987+0000","last_peered":"2023-10-09T10:45:16.751987+0000","last_clean":"2023-10-09T10:45:16.751987+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:45:16.751987+0000","last_undegraded":"2023-10-09T10:45:16.751987+0000","last_fullsized":"2023-10-09T10:45:16.751987+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":105,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.768267+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:45:16.768267+0000","last_peered":"2023-10-09T10:45:16.768267+0000","last_clean":"2023-10-09T10:45:16.768267+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:45:16.768267+0000","last_undegraded":"2023-10-09T10:45:16.768267+0000","last_fullsized":"2023-10-09T10:45:16.768267+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"124'3103","reported_seq":3152,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-10-09T10:45:19.628313+0000","last_change":"2023-10-09T10:45:12.912569+0000","last_active":"2023-10-09T10:45:19.628313+0000","last_peered":"2023-10-09T10:45:19.628313+0000","last_clean":"2023-10-09T10:45:19.628313+0000","last_became_active":"2023-10-09T10:44:50.076612+0000","last_became_peered":"2023-10-09T10:44:50.076612+0000","last_unstale":"2023-10-09T10:45:19.628313+0000","last_undegraded":"2023-10-09T10:45:19.628313+0000","last_fullsized":"2023-10-09T10:45:19.628313+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3103,"log_dups_size":0,"ondisk_log_size":3103,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133595136,"num_objects":2039,"num_object_clones":0,"num_object_copies":6117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2039,"num_write_kb":130464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":74,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.784669+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:45:16.784669+0000","last_peered":"2023-10-09T10:45:16.784669+0000","last_clean":"2023-10-09T10:45:16.784669+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:45:16.784669+0000","last_undegraded":"2023-10-09T10:45:16.784669+0000","last_fullsized":"2023-10-09T10:45:16.784669+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":101,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751641+0000","last_change":"2023-10-09T10:45:15.769793+0000","last_active":"2023-10-09T10:45:16.751641+0000","last_peered":"2023-10-09T10:45:16.751641+0000","last_clean":"2023-10-09T10:45:16.751641+0000","last_became_active":"2023-10-09T10:45:15.769546+0000","last_became_peered":"2023-10-09T10:45:15.769546+0000","last_unstale":"2023-10-09T10:45:16.751641+0000","last_undegraded":"2023-10-09T10:45:16.751641+0000","last_fullsized":"2023-10-09T10:45:16.751641+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:32.371324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":97,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751361+0000","last_change":"2023-10-09T10:43:35.485790+0000","last_active":"2023-10-09T10:45:16.751361+0000","last_peered":"2023-10-09T10:45:16.751361+0000","last_clean":"2023-10-09T10:45:16.751361+0000","last_became_active":"2023-10-09T10:43:35.485436+0000","last_became_peered":"2023-10-09T10:43:35.485436+0000","last_unstale":"2023-10-09T10:45:16.751361+0000","last_undegraded":"2023-10-09T10:45:16.751361+0000","last_fullsized":"2023-10-09T10:45:16.751361+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:45:20.897767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":121,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751426+0000","last_change":"2023-10-09T10:45:15.761078+0000","last_active":"2023-10-09T10:45:16.751426+0000","last_peered":"2023-10-09T10:45:16.751426+0000","last_clean":"2023-10-09T10:45:16.751426+0000","last_became_active":"2023-10-09T10:45:15.760721+0000","last_became_peered":"2023-10-09T10:45:15.760721+0000","last_unstale":"2023-10-09T10:45:16.751426+0000","last_undegraded":"2023-10-09T10:45:16.751426+0000","last_fullsized":"2023-10-09T10:45:16.751426+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:33.535124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":42,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.751260+0000","last_change":"2023-10-09T10:45:15.774365+0000","last_active":"2023-10-09T10:45:16.751260+0000","last_peered":"2023-10-09T10:45:16.751260+0000","last_clean":"2023-10-09T10:45:16.751260+0000","last_became_active":"2023-10-09T10:45:15.763429+0000","last_became_peered":"2023-10-09T10:45:15.763429+0000","last_unstale":"2023-10-09T10:45:16.751260+0000","last_undegraded":"2023-10-09T10:45:16.751260+0000","last_fullsized":"2023-10-09T10:45:16.751260+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:29.201640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":88,"reported_epoch":123,"state":"active+clean","last_fresh":"2023-10-09T10:45:16.767656+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:45:16.767656+0000","last_peered":"2023-10-09T10:45:16.767656+0000","last_clean":"2023-10-09T10:45:16.767656+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:45:16.767656+0000","last_undegraded":"2023-10-09T10:45:16.767656+0000","last_fullsized":"2023-10-09T10:45:16.767656+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":4375740416,"num_objects":66769,"num_object_clones":0,"num_object_copies":200307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15586,"num_objects_misplaced":4274,"num_objects_unfound":0,"num_objects_dirty":66769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61521,"num_write_kb":3937312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2377,"num_bytes_recovered":155779072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13527678976,"data_stored":13527678976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87583,"ondisk_log_size":87583,"up":78,"acting":70,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":576,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332225,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1962680,"kb_used_data":1657200,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91782472,"statfs":{"total":95995035648,"available":93985251328,"internally_reserved":0,"allocated":1696972800,"data_stored":1696503456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:44:32 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.54400000000000004,"15min":0.56699999999999995},"min":{"1min":0.34599999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":1.897},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.59999999999999998,"15min":0.628},"min":{"1min":0.39300000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":1.9890000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:44:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.55000000000000004,"15min":0.54400000000000004},"min":{"1min":0.39200000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":2.077,"5min":2.077,"15min":2.077},"last":3.121},{"interface":"front","average":{"1min":0.84599999999999997,"5min":0.59399999999999997,"15min":0.55200000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.4939999999999998,"5min":6.4939999999999998,"15min":6.4939999999999998},"last":0.34100000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:45:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":0.69699999999999995,"15min":0.57299999999999995},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.2690000000000001,"5min":6.2690000000000001,"15min":6.2690000000000001},"last":1.413},{"interface":"front","average":{"1min":0.84999999999999998,"5min":0.58999999999999997,"15min":0.54700000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.157,"5min":2.157,"15min":2.157},"last":1.462}]},{"osd":3,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":1.2330000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:45:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.51600000000000001,"15min":0.48499999999999999},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.1600000000000001,"5min":3.1600000000000001,"15min":3.1600000000000001},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.54200000000000004,"15min":0.49399999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.1699999999999999,"5min":3.1699999999999999,"15min":3.1699999999999999},"last":1.179}]},{"osd":5,"last update":"Mon Oct 9 10:45:16 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.55000000000000004,"15min":0.51700000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":2.1989999999999998,"5min":2.1989999999999998,"15min":2.1989999999999998},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.58699999999999997,"15min":0.55200000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":2.7549999999999999,"5min":2.7549999999999999,"15min":2.7549999999999999},"last":0.36199999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:45:16 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.628,"15min":0.57399999999999995},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.1880000000000002,"5min":3.1880000000000002,"15min":3.1880000000000002},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.58399999999999996,"15min":0.55200000000000005},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":1.2110000000000001}]}]},{"osd":6,"up_from":44,"seq":188978561156,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1626160,"kb_used_data":1316584,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92118992,"statfs":{"total":95995035648,"available":94329847808,"internally_reserved":0,"allocated":1348182016,"data_stored":1347721272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.57699999999999996,"15min":0.59899999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.72599999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.59899999999999998,"15min":0.60499999999999998},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.76300000000000001,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.39700000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:44:41 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.58599999999999997,"15min":0.58899999999999997},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":1.343},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":1.355}]},{"osd":2,"last update":"Mon Oct 9 10:45:07 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.54900000000000004,"15min":0.53900000000000003},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":2.0960000000000001,"5min":2.0960000000000001,"15min":2.0960000000000001},"last":0.90400000000000003},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.64900000000000002,"15min":0.58999999999999997},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.7349999999999999,"5min":3.7349999999999999,"15min":3.7349999999999999},"last":0.38200000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":2.7440000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:45:00 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.46899999999999997,"15min":0.44900000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.55,"5min":1.55,"15min":1.55},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.52200000000000002,"15min":0.496},"min":{"1min":0.29999999999999999,"5min":0.29899999999999999,"15min":0.29699999999999999},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.439}]},{"osd":5,"last update":"Mon Oct 9 10:45:00 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.53000000000000003,"15min":0.49099999999999999},"min":{"1min":0.34999999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.762,"5min":1.762,"15min":1.762},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.56999999999999995,"15min":0.53500000000000003},"min":{"1min":0.33900000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.84399999999999997},"last":0.83099999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:44:20 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.55400000000000005,"15min":0.53000000000000003},"min":{"1min":0.438,"5min":0.33900000000000002,"15min":0.32900000000000001},"max":{"1min":0.70099999999999996,"5min":0.78600000000000003,"15min":0.79100000000000004},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.629,"5min":0.59699999999999998,"15min":0.57899999999999996},"min":{"1min":0.41199999999999998,"5min":0.34100000000000003,"15min":0.34000000000000002},"max":{"1min":0.72799999999999998,"5min":0.84299999999999997,"15min":1.018},"last":0.503}]}]},{"osd":1,"up_from":85,"seq":365072220195,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1564516,"kb_used_data":1492764,"kb_used_omap":9,"kb_used_meta":71734,"kb_avail":92180636,"statfs":{"total":95995035648,"available":94392971264,"internally_reserved":0,"allocated":1528590336,"data_stored":1528025891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":73456617},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:44:42 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.38500000000000001,"15min":0.371},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.7949999999999999,"5min":2.7949999999999999,"15min":2.7949999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.42899999999999999,"15min":0.40899999999999997},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.6789999999999998,"5min":3.6789999999999998,"15min":3.6789999999999998},"last":1.919}]},{"osd":2,"last update":"Mon Oct 9 10:45:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":0.55300000000000005,"15min":0.42899999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":8.6859999999999999,"5min":8.6859999999999999,"15min":8.6859999999999999},"last":2.3159999999999998},{"interface":"front","average":{"1min":1.7470000000000001,"5min":0.65700000000000003,"15min":0.47499999999999998},"min":{"1min":0.26200000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":14.137,"5min":14.137,"15min":14.137},"last":0.52600000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.3879999999999999,"5min":2.3879999999999999,"15min":2.3879999999999999},"last":1.51},{"interface":"front","average":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":3.8620000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:44:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.58199999999999996,"15min":0.58899999999999997},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.61399999999999999,"15min":0.61799999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.9330000000000001,"5min":1.9330000000000001,"15min":1.9330000000000001},"last":0.85499999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:44:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.61899999999999999,"15min":0.625},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":1.9430000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.627,"15min":0.64000000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.42599999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:44:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.621,"15min":0.628},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.129,"5min":1.129,"15min":1.129},"last":3.7559999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.63,"15min":0.64100000000000001},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.96299999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:44:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.63700000000000001,"15min":0.64500000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.64700000000000002,"15min":0.64600000000000002},"min":{"1min":0.42599999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.8939999999999999,"5min":1.8939999999999999,"15min":1.8939999999999999},"last":0.54400000000000004}]}]},{"osd":0,"up_from":0,"seq":343597383729,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":90,"seq":386547056669,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2078812,"kb_used_data":2011028,"kb_used_omap":81,"kb_used_meta":67694,"kb_avail":91666340,"statfs":{"total":95995035648,"available":93866332160,"internally_reserved":0,"allocated":2059292672,"data_stored":2058733865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":69318874},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.39500000000000002,"15min":0.34499999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":3.8370000000000002,"5min":3.8370000000000002,"15min":3.8370000000000002},"last":0.21099999999999999},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.46100000000000002,"15min":0.40300000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.0049999999999999,"5min":4.0049999999999999,"15min":4.0049999999999999},"last":4.3979999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":0.54800000000000004,"15min":0.44400000000000001},"min":{"1min":0.27900000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.1909999999999998,"5min":5.1909999999999998,"15min":5.1909999999999998},"last":3.1120000000000001},{"interface":"front","average":{"1min":0.877,"5min":0.46200000000000002,"15min":0.39300000000000002},"min":{"1min":0.23799999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.2039999999999997,"5min":5.2039999999999997,"15min":5.2039999999999997},"last":3.1349999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.5649999999999999,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":7.0419999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":0.68200000000000005,"15min":0.621},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":8.1029999999999998,"5min":8.1029999999999998,"15min":8.1029999999999998},"last":3.6429999999999998},{"interface":"front","average":{"1min":1.216,"5min":0.71799999999999997,"15min":0.63500000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":8.1259999999999994,"5min":8.1259999999999994,"15min":8.1259999999999994},"last":3.5209999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":0.754,"15min":0.64800000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.3659999999999997,"5min":8.3659999999999997,"15min":8.3659999999999997},"last":2.7469999999999999},{"interface":"front","average":{"1min":1.286,"5min":0.73399999999999999,"15min":0.64200000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":8.3979999999999997,"5min":8.3979999999999997,"15min":8.3979999999999997},"last":2.3420000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.62,"15min":0.58999999999999997},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":3.3730000000000002,"5min":3.3730000000000002,"15min":3.3730000000000002},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.1100000000000001,"5min":0.68999999999999995,"15min":0.62},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":5.2199999999999998,"5min":5.2199999999999998,"15min":5.2199999999999998},"last":3.6949999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.629,"15min":0.59799999999999998},"min":{"1min":0.433,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":3.3500000000000001,"5min":3.3500000000000001,"15min":3.3500000000000001},"last":2.3029999999999999},{"interface":"front","average":{"1min":1.1419999999999999,"5min":0.69499999999999995,"15min":0.621},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":5.2690000000000001,"5min":5.2690000000000001,"15min":5.2690000000000001},"last":2.7799999999999998}]}]},{"osd":3,"up_from":95,"seq":408021893143,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2664772,"kb_used_data":2579068,"kb_used_omap":7,"kb_used_meta":85688,"kb_avail":91080380,"statfs":{"total":95995035648,"available":93266309120,"internally_reserved":0,"allocated":2640965632,"data_stored":2640420620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":87745128},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":1.2170000000000001},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.992,"5min":1.992,"15min":1.992},"last":2.7360000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.96799999999999997},{"interface":"front","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":1.2969999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":5.3739999999999997},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":5.5179999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":1.9259999999999999},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":2.6989999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":2.613},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":2.226}]},{"osd":6,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":2.5819999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":1.022}]},{"osd":7,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":1.2669999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019019,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1653428,"kb_used_data":1339756,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":92091724,"statfs":{"total":95995035648,"available":94301925376,"internally_reserved":0,"allocated":1371910144,"data_stored":1371444605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.54200000000000004,"15min":0.55200000000000005},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.311},"max":{"1min":0.621,"5min":0.755,"15min":0.755},"last":2.1819999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.57099999999999995,"15min":0.57999999999999996},"min":{"1min":0.40799999999999997,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.69999999999999996,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":6.4240000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:44:40 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":0.60299999999999998,"15min":0.56899999999999995},"min":{"1min":0.35499999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.867,"5min":4.867,"15min":4.867},"last":2.2280000000000002},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.65200000000000002,"15min":0.63200000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.8120000000000003,"5min":4.8120000000000003,"15min":4.8120000000000003},"last":1.0600000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:45:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":0.71799999999999997,"15min":0.62},"min":{"1min":0.46100000000000002,"5min":0.373,"15min":0.373},"max":{"1min":4.6340000000000003,"5min":4.6340000000000003,"15min":4.6340000000000003},"last":1.78},{"interface":"front","average":{"1min":1.1319999999999999,"5min":0.67600000000000005,"15min":0.59999999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.2939999999999996,"5min":6.2939999999999996,"15min":6.2939999999999996},"last":1.8140000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:44:34 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.335,"5min":4.335,"15min":4.335},"last":4.173},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":2.2040000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:44:40 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47899999999999998,"15min":0.443},"min":{"1min":0.25800000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.73599999999999999,"5min":0.79400000000000004,"15min":0.80900000000000005},"last":5.5410000000000004},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.503,"15min":0.45300000000000001},"min":{"1min":0.34599999999999997,"5min":0.28499999999999998,"15min":0.26400000000000001},"max":{"1min":1.0609999999999999,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":1.8779999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:45:04 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.51000000000000001,"15min":0.47299999999999998},"min":{"1min":0.313,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":0.75800000000000001,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":2.1419999999999999},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.60399999999999998,"15min":0.56699999999999995},"min":{"1min":0.34300000000000003,"5min":0.314,"15min":0.314},"max":{"1min":2.1320000000000001,"5min":2.1320000000000001,"15min":2.1320000000000001},"last":1.101}]},{"osd":7,"last update":"Mon Oct 9 10:45:14 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.628,"15min":0.58399999999999996},"min":{"1min":0.36799999999999999,"5min":0.35299999999999998,"15min":0.308},"max":{"1min":2.7589999999999999,"5min":2.7589999999999999,"15min":2.7589999999999999},"last":3.375},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.622,"15min":0.55800000000000005},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.456,"5min":2.456,"15min":2.456},"last":1.8480000000000001}]}]},{"osd":5,"up_from":37,"seq":158913790088,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1918688,"kb_used_data":1617304,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91826464,"statfs":{"total":95995035648,"available":94030299136,"internally_reserved":0,"allocated":1656119296,"data_stored":1655669658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:44:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.59299999999999997,"15min":0.59599999999999997},"min":{"1min":0.40999999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":3.694},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.65700000000000003,"15min":0.66500000000000004},"min":{"1min":0.443,"5min":0.439,"15min":0.439},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":3.722}]},{"osd":1,"last update":"Mon Oct 9 10:44:40 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.59699999999999998,"15min":0.60399999999999998},"min":{"1min":0.38500000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.69999999999999996,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":1.9650000000000001},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.64100000000000001,"15min":0.60799999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.5110000000000001,"5min":5.5110000000000001,"15min":5.5110000000000001},"last":1.893}]},{"osd":2,"last update":"Mon Oct 9 10:45:06 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.65400000000000003,"15min":0.59899999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.8490000000000002,"5min":3.8490000000000002,"15min":3.8490000000000002},"last":3.1920000000000002},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.59199999999999997,"15min":0.57999999999999996},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.823,"5min":1.823,"15min":1.823},"last":1.9279999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":3.7240000000000002,"5min":3.7240000000000002,"15min":3.7240000000000002},"last":2.1259999999999999},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.415,"5min":1.415,"15min":1.415},"last":0.56999999999999995}]},{"osd":4,"last update":"Mon Oct 9 10:44:40 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.48299999999999998,"15min":0.437},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.25},"max":{"1min":0.73199999999999998,"5min":0.75600000000000001,"15min":2.4649999999999999},"last":1.2769999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.52500000000000002,"15min":0.46300000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28799999999999998},"max":{"1min":1.923,"5min":1.923,"15min":1.98},"last":0.53900000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:45:03 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.52900000000000003,"15min":0.46700000000000003},"min":{"1min":0.317,"5min":0.317,"15min":0.28100000000000003},"max":{"1min":1.274,"5min":1.274,"15min":2.048},"last":2.9529999999999998},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.58599999999999997,"15min":0.505},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.30299999999999999},"max":{"1min":1.746,"5min":1.746,"15min":1.746},"last":1.3140000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:45:18 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.622,"15min":0.55500000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32400000000000001},"max":{"1min":4.0510000000000002,"5min":4.0510000000000002,"15min":4.0510000000000002},"last":4.0510000000000002},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.66700000000000004,"15min":0.56999999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":2.6869999999999998,"5min":2.6869999999999998,"15min":2.6869999999999998},"last":2.6869999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1120534528,"data_stored":1120534528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1529741312,"data_stored":1529741312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2085945344,"data_stored":2085945344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2680422400,"data_stored":2680422400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1381171200,"data_stored":1381171200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1660551168,"data_stored":1660551168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1361313792,"data_stored":1361313792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1707999232,"data_stored":1707999232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:45:24.419 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:45:24.420 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 11 2023-10-09T10:45:24.430 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:24.470 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:24.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:24.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:24.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:24.603 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:24.639 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:45:24.644 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:24.666 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:24.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:24.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:24.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:24.798 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:24 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3471020407' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:24.799 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:24 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2858676490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:24.799 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:24 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:24.799 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:24 smithi125 ceph-mon[195507]: from='client.54550 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:45:24.799 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:24 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3471020407' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:24.799 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:24 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2858676490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:24.799 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:24 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:24.799 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:24 smithi125 ceph-mon[200197]: from='client.54550 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:45:24.814 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:24.851 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:24.856 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:24 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3471020407' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:24.856 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:24 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2858676490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:24.856 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:24 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:24.856 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:24 smithi155 ceph-mon[184854]: from='client.54550 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:45:24.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:24.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:24.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:25.017 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:25.043 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:25.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:25.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:25.147 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:25.187 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:25.218 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:25.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:25.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:25.310 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 53 16 72083 72067 84.946 64.3125 0.0149619 0.0117669 2023-10-09T10:45:25.324 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:25.360 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:25.393 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:25.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:25.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:25.429 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:25.498 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:25.530 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:45:25.531 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:45:25.540 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:25.571 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:25.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:25.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:25.675 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:25.733 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:25.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:25 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:25.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:25 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:45:25.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:25 smithi125 ceph-mon[195507]: osdmap e125: 8 total, 8 up, 7 in 2023-10-09T10:45:25.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:25 smithi125 ceph-mon[195507]: failed to encode map e125 with expected crc 2023-10-09T10:45:25.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:25 smithi125 ceph-mon[195507]: pgmap v252: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 90 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 51 MiB/s wr, 808 op/s; 15586/208134 objects degraded (7.488%); 3473/208134 objects misplaced (1.669%); 11 MiB/s, 172 objects/s recovering 2023-10-09T10:45:25.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:25 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:25.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:25 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:45:25.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:25 smithi125 ceph-mon[200197]: osdmap e125: 8 total, 8 up, 7 in 2023-10-09T10:45:25.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:25 smithi125 ceph-mon[200197]: failed to encode map e125 with expected crc 2023-10-09T10:45:25.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:25 smithi125 ceph-mon[200197]: pgmap v252: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 90 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 51 MiB/s wr, 808 op/s; 15586/208134 objects degraded (7.488%); 3473/208134 objects misplaced (1.669%); 11 MiB/s, 172 objects/s recovering 2023-10-09T10:45:25.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:25 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:25.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:25 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:45:25.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:25 smithi155 ceph-mon[184854]: osdmap e125: 8 total, 8 up, 7 in 2023-10-09T10:45:25.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:25 smithi155 ceph-mon[184854]: failed to encode map e125 with expected crc 2023-10-09T10:45:25.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:25 smithi155 ceph-mon[184854]: pgmap v252: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 90 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 51 MiB/s wr, 808 op/s; 15586/208134 objects degraded (7.488%); 3473/208134 objects misplaced (1.669%); 11 MiB/s, 172 objects/s recovering 2023-10-09T10:45:25.976 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:25.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:25.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:26.084 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:26.100 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:45:26.149 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:26.310 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 54 16 73148 73132 84.6056 66.5625 0.0131298 0.0118144 2023-10-09T10:45:26.340 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:26.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:26.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:26.449 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:26.500 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:26.576 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:26.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:26.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:26.680 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:26.730 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:26.760 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:26.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:26.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:26.823 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:26.864 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:26.917 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:26.957 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:26.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:26.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:27.061 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:27.067 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:45:27.094 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:27.120 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:27.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:27.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:27.163 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:27 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 15586/215577 objects degraded (7.230%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:27.163 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:27 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 15586/215577 objects degraded (7.230%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:27.226 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:27.290 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:45:27.291 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:27.311 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 55 16 74034 74018 84.0726 55.375 0.0328076 0.0118887 2023-10-09T10:45:27.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:27 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 15586/215577 objects degraded (7.230%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:27.552 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:27.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:27.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:27.681 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:27.693 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:27.718 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:27.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:27.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:27.824 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:27.894 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:27.902 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:28.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:28 smithi125 ceph-mon[195507]: pgmap v253: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 90 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 60 MiB/s wr, 966 op/s; 15586/215577 objects degraded (7.230%); 3473/215577 objects misplaced (1.611%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T10:45:28.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:28 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/901985442' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T10:45:28.176 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 1 pg_num to 11 2023-10-09T10:45:28.311 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 56 16 75051 75035 83.7064 63.5625 0.0163282 0.011942 2023-10-09T10:45:28.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:28 smithi125 ceph-mon[200197]: pgmap v253: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 90 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 60 MiB/s wr, 966 op/s; 15586/215577 objects degraded (7.230%); 3473/215577 objects misplaced (1.611%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T10:45:28.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:28 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/901985442' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T10:45:28.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:28 smithi155 ceph-mon[184854]: pgmap v253: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 90 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 60 MiB/s wr, 966 op/s; 15586/215577 objects degraded (7.230%); 3473/215577 objects misplaced (1.611%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T10:45:28.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:28 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/901985442' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T10:45:28.856 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:28.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:28.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:28.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:29.010 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:45:29.010 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:45:29.051 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:29.311 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 57 16 76094 76078 83.3816 65.1875 0.0114958 0.0119887 2023-10-09T10:45:29.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:29 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/901985442' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-10-09T10:45:29.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:29 smithi125 ceph-mon[200197]: osdmap e126: 8 total, 8 up, 7 in 2023-10-09T10:45:29.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:29 smithi125 ceph-mon[200197]: failed to encode map e126 with expected crc 2023-10-09T10:45:29.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:29 smithi125 ceph-mon[200197]: failed to encode map e126 with expected crc 2023-10-09T10:45:29.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:29 smithi125 ceph-mon[200197]: failed to encode map e126 with expected crc 2023-10-09T10:45:29.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T10:45:29.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:29 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/901985442' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-10-09T10:45:29.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:29 smithi125 ceph-mon[195507]: osdmap e126: 8 total, 8 up, 7 in 2023-10-09T10:45:29.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:29 smithi125 ceph-mon[195507]: failed to encode map e126 with expected crc 2023-10-09T10:45:29.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:29 smithi125 ceph-mon[195507]: failed to encode map e126 with expected crc 2023-10-09T10:45:29.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:29 smithi125 ceph-mon[195507]: failed to encode map e126 with expected crc 2023-10-09T10:45:29.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T10:45:29.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:29 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/901985442' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-10-09T10:45:29.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:29 smithi155 ceph-mon[184854]: osdmap e126: 8 total, 8 up, 7 in 2023-10-09T10:45:29.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:29 smithi155 ceph-mon[184854]: failed to encode map e126 with expected crc 2023-10-09T10:45:29.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:29 smithi155 ceph-mon[184854]: failed to encode map e126 with expected crc 2023-10-09T10:45:29.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:29 smithi155 ceph-mon[184854]: failed to encode map e126 with expected crc 2023-10-09T10:45:29.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T10:45:29.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:29.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:29.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:29.716 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:29.772 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:29.804 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:29.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:29.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:29.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:29.976 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:30.059 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:30.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:30.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:30.090 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:45:30.164 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:30.203 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:30.237 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:30.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:30.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:30.313 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 58 16 77164 77148 83.0969 66.875 0.0111952 0.012029 2023-10-09T10:45:30.343 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:30.433 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:30.443 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:30 smithi125 ceph-mon[195507]: pgmap v255: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 90 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 55 MiB/s wr, 881 op/s; 15586/219729 objects degraded (7.093%); 2982/219729 objects misplaced (1.357%); 10 MiB/s, 161 objects/s recovering 2023-10-09T10:45:30.443 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:30 smithi125 ceph-mon[195507]: failed to encode map e127 with expected crc 2023-10-09T10:45:30.443 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-10-09T10:45:30.443 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:30 smithi125 ceph-mon[195507]: osdmap e127: 8 total, 8 up, 7 in 2023-10-09T10:45:30.444 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:30.444 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:45:30.444 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:30 smithi125 ceph-mon[200197]: pgmap v255: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 90 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 55 MiB/s wr, 881 op/s; 15586/219729 objects degraded (7.093%); 2982/219729 objects misplaced (1.357%); 10 MiB/s, 161 objects/s recovering 2023-10-09T10:45:30.444 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:30 smithi125 ceph-mon[200197]: failed to encode map e127 with expected crc 2023-10-09T10:45:30.444 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-10-09T10:45:30.444 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:30 smithi125 ceph-mon[200197]: osdmap e127: 8 total, 8 up, 7 in 2023-10-09T10:45:30.444 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:30.444 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:45:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:30 smithi155 ceph-mon[184854]: pgmap v255: 99 pgs: 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 90 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 55 MiB/s wr, 881 op/s; 15586/219729 objects degraded (7.093%); 2982/219729 objects misplaced (1.357%); 10 MiB/s, 161 objects/s recovering 2023-10-09T10:45:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:30 smithi155 ceph-mon[184854]: failed to encode map e127 with expected crc 2023-10-09T10:45:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-10-09T10:45:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:30 smithi155 ceph-mon[184854]: osdmap e127: 8 total, 8 up, 7 in 2023-10-09T10:45:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:45:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:45:30.509 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:30.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:30.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:30.618 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:30.704 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:30.708 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:30.749 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:30.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:30.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:30.809 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:45:30.855 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:30.864 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:45:30.891 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:30.912 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:45:30.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:30.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:30.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:30.969 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:45:31.024 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:31.035 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:45:31.072 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:45:31.073 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:31.104 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:31.108 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:45:31.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:31.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:31.141 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:45:31.150 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:45:31.150 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:45:31.151 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:45:31.151 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:45:31.151 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:45:31.151 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:45:31.151 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:45:31.151 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:45:31.211 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:31.263 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:45:31.282 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:31.311 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 59 16 77983 77967 82.5562 51.1875 0.0193181 0.0121083 2023-10-09T10:45:31.322 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:45:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:45:31] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.43.0" 2023-10-09T10:45:31.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:31 smithi125 ceph-mon[200197]: failed to encode map e128 with expected crc 2023-10-09T10:45:31.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:31 smithi125 ceph-mon[200197]: failed to encode map e128 with expected crc 2023-10-09T10:45:31.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:31 smithi125 ceph-mon[200197]: osdmap e128: 8 total, 8 up, 7 in 2023-10-09T10:45:31.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:31 smithi125 ceph-mon[200197]: failed to encode map e128 with expected crc 2023-10-09T10:45:31.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2728629283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:31.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1676206750' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:31.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:31 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:31.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:31 smithi125 ceph-mon[195507]: failed to encode map e128 with expected crc 2023-10-09T10:45:31.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:31 smithi125 ceph-mon[195507]: failed to encode map e128 with expected crc 2023-10-09T10:45:31.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:31 smithi125 ceph-mon[195507]: osdmap e128: 8 total, 8 up, 7 in 2023-10-09T10:45:31.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:31 smithi125 ceph-mon[195507]: failed to encode map e128 with expected crc 2023-10-09T10:45:31.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2728629283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:31.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1676206750' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:31.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:31 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:31 smithi155 ceph-mon[184854]: failed to encode map e128 with expected crc 2023-10-09T10:45:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:31 smithi155 ceph-mon[184854]: failed to encode map e128 with expected crc 2023-10-09T10:45:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:31 smithi155 ceph-mon[184854]: osdmap e128: 8 total, 8 up, 7 in 2023-10-09T10:45:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:31 smithi155 ceph-mon[184854]: failed to encode map e128 with expected crc 2023-10-09T10:45:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2728629283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1676206750' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:31 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:31.515 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:31.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:31.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:31.618 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:31.682 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:31.711 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:31.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:31.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:31.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:31.874 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:31.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:31.957 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:31.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:32.058 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:32.105 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:32.137 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:32.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:32.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:32.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:32 smithi125 ceph-mon[195507]: pgmap v258: 109 pgs: 10 unknown, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 90 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 13603/228648 objects degraded (5.949%); 4314/228648 objects misplaced (1.887%); 12 MiB/s, 190 objects/s recovering 2023-10-09T10:45:32.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:32 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:32.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:32 smithi125 ceph-mon[195507]: failed to encode map e129 with expected crc 2023-10-09T10:45:32.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:32 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:45:32.223 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:32 smithi125 ceph-mon[195507]: osdmap e129: 8 total, 8 up, 7 in 2023-10-09T10:45:32.223 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:32 smithi125 ceph-mon[195507]: failed to encode map e129 with expected crc 2023-10-09T10:45:32.223 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:32 smithi125 ceph-mon[200197]: pgmap v258: 109 pgs: 10 unknown, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 90 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 13603/228648 objects degraded (5.949%); 4314/228648 objects misplaced (1.887%); 12 MiB/s, 190 objects/s recovering 2023-10-09T10:45:32.223 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:32 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:32.223 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:32 smithi125 ceph-mon[200197]: failed to encode map e129 with expected crc 2023-10-09T10:45:32.223 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:32 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:45:32.223 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:32 smithi125 ceph-mon[200197]: osdmap e129: 8 total, 8 up, 7 in 2023-10-09T10:45:32.223 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:32 smithi125 ceph-mon[200197]: failed to encode map e129 with expected crc 2023-10-09T10:45:32.227 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:45:32.227 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:45:32.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:45:32.227 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:45:32.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.227 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42384 6.10 6:0c7bd3cf:::benchmark_data_smithi125_244951_object42383:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.228 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:45:32.228 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 37.101999393, 2023-10-09T10:45:32.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0577403670000001, 2023-10-09T10:45:32.228 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.228 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.228 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.228 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.228 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.228 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42384 2023-10-09T10:45:32.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.228 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.228 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:45:32.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:45:32.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117185+0000", 2023-10-09T10:45:32.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:32.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117225+0000", 2023-10-09T10:45:32.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0364000000000002e-05 2023-10-09T10:45:32.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117240+0000", 2023-10-09T10:45:32.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4425e-05 2023-10-09T10:45:32.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117248+0000", 2023-10-09T10:45:32.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4999999999999999e-06 2023-10-09T10:45:32.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117301+0000", 2023-10-09T10:45:32.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2685999999999997e-05 2023-10-09T10:45:32.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117312+0000", 2023-10-09T10:45:32.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1273e-05 2023-10-09T10:45:32.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103469+0000", 2023-10-09T10:45:32.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98615701300000003 2023-10-09T10:45:32.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103483+0000", 2023-10-09T10:45:32.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3207000000000001e-05 2023-10-09T10:45:32.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.148764+0000", 2023-10-09T10:45:32.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0452812789999999 2023-10-09T10:45:32.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.148882+0000", 2023-10-09T10:45:32.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118095 2023-10-09T10:45:32.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.149057+0000", 2023-10-09T10:45:32.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017516400000000001 2023-10-09T10:45:32.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174105+0000", 2023-10-09T10:45:32.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.025048237000000001 2023-10-09T10:45:32.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174158+0000", 2023-10-09T10:45:32.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2330999999999998e-05 2023-10-09T10:45:32.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174325+0000", 2023-10-09T10:45:32.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016690100000000001 2023-10-09T10:45:32.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174367+0000", 2023-10-09T10:45:32.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2759000000000002e-05 2023-10-09T10:45:32.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174927+0000", 2023-10-09T10:45:32.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00055942800000000005 2023-10-09T10:45:32.235 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.236 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.236 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.236 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42398 6.10 6:0867fefc:::benchmark_data_smithi125_244951_object42397:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.236 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:45:32.236 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 37.091220624000002, 2023-10-09T10:45:32.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0510695769999998, 2023-10-09T10:45:32.236 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.236 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.236 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.236 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.236 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.237 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42398 2023-10-09T10:45:32.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.237 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:45:32.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:45:32.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.238 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127964+0000", 2023-10-09T10:45:32.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:32.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.238 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127989+0000", 2023-10-09T10:45:32.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4811e-05 2023-10-09T10:45:32.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.238 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128005+0000", 2023-10-09T10:45:32.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6436999999999999e-05 2023-10-09T10:45:32.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128014+0000", 2023-10-09T10:45:32.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5990000000000002e-06 2023-10-09T10:45:32.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128072+0000", 2023-10-09T10:45:32.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7980999999999999e-05 2023-10-09T10:45:32.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128083+0000", 2023-10-09T10:45:32.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1051e-05 2023-10-09T10:45:32.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103714+0000", 2023-10-09T10:45:32.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97563124099999998 2023-10-09T10:45:32.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103725+0000", 2023-10-09T10:45:32.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1121000000000001e-05 2023-10-09T10:45:32.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174971+0000", 2023-10-09T10:45:32.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0712457310000001 2023-10-09T10:45:32.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.175149+0000", 2023-10-09T10:45:32.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000178584 2023-10-09T10:45:32.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.175358+0000", 2023-10-09T10:45:32.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000208848 2023-10-09T10:45:32.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178406+0000", 2023-10-09T10:45:32.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030478049999999998 2023-10-09T10:45:32.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178418+0000", 2023-10-09T10:45:32.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1499e-05 2023-10-09T10:45:32.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178466+0000", 2023-10-09T10:45:32.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8715999999999999e-05 2023-10-09T10:45:32.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178522+0000", 2023-10-09T10:45:32.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5952999999999998e-05 2023-10-09T10:45:32.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.244 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179035+0000", 2023-10-09T10:45:32.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051254299999999996 2023-10-09T10:45:32.244 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.244 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.244 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.244 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42518 6.10 6:0fbcc8bc:::benchmark_data_smithi125_244951_object42517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.244 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:45:32.245 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 37.024823298999998, 2023-10-09T10:45:32.245 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9852423510000001, 2023-10-09T10:45:32.245 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.245 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.245 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.245 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.245 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.245 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42518 2023-10-09T10:45:32.245 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.245 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.245 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.245 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.245 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:45:32.245 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.246 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.246 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.246 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:45:32.246 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.246 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.246 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.246 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194361+0000", 2023-10-09T10:45:32.246 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:32.246 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.247 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.247 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194385+0000", 2023-10-09T10:45:32.247 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4017000000000001e-05 2023-10-09T10:45:32.247 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.247 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.247 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.247 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194401+0000", 2023-10-09T10:45:32.247 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5841e-05 2023-10-09T10:45:32.247 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.247 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.247 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.247 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194408+0000", 2023-10-09T10:45:32.247 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0450000000000003e-06 2023-10-09T10:45:32.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194460+0000", 2023-10-09T10:45:32.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2015000000000003e-05 2023-10-09T10:45:32.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194474+0000", 2023-10-09T10:45:32.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4486e-05 2023-10-09T10:45:32.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103750+0000", 2023-10-09T10:45:32.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.90927595699999997 2023-10-09T10:45:32.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103758+0000", 2023-10-09T10:45:32.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9000000000000006e-06 2023-10-09T10:45:32.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176100+0000", 2023-10-09T10:45:32.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.07234187 2023-10-09T10:45:32.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176198+0000", 2023-10-09T10:45:32.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7408e-05 2023-10-09T10:45:32.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176317+0000", 2023-10-09T10:45:32.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011998900000000001 2023-10-09T10:45:32.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178425+0000", 2023-10-09T10:45:32.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021072 2023-10-09T10:45:32.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179085+0000", 2023-10-09T10:45:32.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00066011800000000001 2023-10-09T10:45:32.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179122+0000", 2023-10-09T10:45:32.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6831e-05 2023-10-09T10:45:32.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179155+0000", 2023-10-09T10:45:32.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3080000000000002e-05 2023-10-09T10:45:32.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179605+0000", 2023-10-09T10:45:32.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000450121 2023-10-09T10:45:32.252 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.252 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.252 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.253 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42546 6.10 6:0889293a:::benchmark_data_smithi125_244951_object42545:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.253 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:45:32.253 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 37.010393866000001, 2023-10-09T10:45:32.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.971488017, 2023-10-09T10:45:32.253 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.253 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.253 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.253 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.253 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.253 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42546 2023-10-09T10:45:32.253 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.253 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.253 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.253 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.254 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:45:32.254 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.254 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.254 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.254 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:45:32.254 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.254 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.254 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.254 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208790+0000", 2023-10-09T10:45:32.254 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:45:32.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.255 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.255 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208815+0000", 2023-10-09T10:45:32.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5103e-05 2023-10-09T10:45:32.255 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.255 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.255 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208831+0000", 2023-10-09T10:45:32.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5235000000000001e-05 2023-10-09T10:45:32.255 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.255 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208838+0000", 2023-10-09T10:45:32.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6510000000000001e-06 2023-10-09T10:45:32.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208895+0000", 2023-10-09T10:45:32.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6817999999999997e-05 2023-10-09T10:45:32.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208906+0000", 2023-10-09T10:45:32.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0550000000000001e-05 2023-10-09T10:45:32.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103780+0000", 2023-10-09T10:45:32.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.89487466699999996 2023-10-09T10:45:32.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103788+0000", 2023-10-09T10:45:32.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0970000000000006e-06 2023-10-09T10:45:32.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176855+0000", 2023-10-09T10:45:32.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0730664299999999 2023-10-09T10:45:32.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176993+0000", 2023-10-09T10:45:32.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013810999999999999 2023-10-09T10:45:32.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.177166+0000", 2023-10-09T10:45:32.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017289900000000001 2023-10-09T10:45:32.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179093+0000", 2023-10-09T10:45:32.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019276969999999999 2023-10-09T10:45:32.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179674+0000", 2023-10-09T10:45:32.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00058078000000000003 2023-10-09T10:45:32.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179730+0000", 2023-10-09T10:45:32.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5418999999999999e-05 2023-10-09T10:45:32.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179788+0000", 2023-10-09T10:45:32.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8510000000000001e-05 2023-10-09T10:45:32.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180280+0000", 2023-10-09T10:45:32.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00049179400000000004 2023-10-09T10:45:32.261 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.261 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.261 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.261 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42570 6.10 6:0f6ea5a0:::benchmark_data_smithi125_244951_object42569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.261 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:45:32.261 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 36.996309373000003, 2023-10-09T10:45:32.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.969653487, 2023-10-09T10:45:32.261 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.262 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.262 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.262 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.262 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.262 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42570 2023-10-09T10:45:32.262 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.262 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.262 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.262 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.262 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:45:32.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.262 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.263 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.263 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:45:32.263 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.263 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.263 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222875+0000", 2023-10-09T10:45:32.263 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:32.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.263 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222897+0000", 2023-10-09T10:45:32.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1678000000000001e-05 2023-10-09T10:45:32.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222906+0000", 2023-10-09T10:45:32.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4190000000000008e-06 2023-10-09T10:45:32.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222913+0000", 2023-10-09T10:45:32.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2659999999999998e-06 2023-10-09T10:45:32.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222958+0000", 2023-10-09T10:45:32.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5979000000000001e-05 2023-10-09T10:45:32.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222965+0000", 2023-10-09T10:45:32.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0099999999999998e-06 2023-10-09T10:45:32.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103823+0000", 2023-10-09T10:45:32.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.880857891 2023-10-09T10:45:32.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103833+0000", 2023-10-09T10:45:32.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8910000000000007e-06 2023-10-09T10:45:32.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180446+0000", 2023-10-09T10:45:32.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0766125440000001 2023-10-09T10:45:32.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180594+0000", 2023-10-09T10:45:32.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014846600000000001 2023-10-09T10:45:32.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180775+0000", 2023-10-09T10:45:32.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018071999999999999 2023-10-09T10:45:32.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183638+0000", 2023-10-09T10:45:32.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028626440000000001 2023-10-09T10:45:32.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183677+0000", 2023-10-09T10:45:32.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9545000000000002e-05 2023-10-09T10:45:32.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191583+0000", 2023-10-09T10:45:32.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0079059569999999999 2023-10-09T10:45:32.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191643+0000", 2023-10-09T10:45:32.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9590999999999999e-05 2023-10-09T10:45:32.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192530+0000", 2023-10-09T10:45:32.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00088716399999999996 2023-10-09T10:45:32.269 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.269 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.269 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.270 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42586 6.10 6:090d6b8d:::benchmark_data_smithi125_244951_object42585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.270 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:45:32.270 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 36.989714831999997, 2023-10-09T10:45:32.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.963875493, 2023-10-09T10:45:32.270 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.270 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.270 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.270 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.270 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.270 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42586 2023-10-09T10:45:32.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.270 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:45:32.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:45:32.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229470+0000", 2023-10-09T10:45:32.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:45:32.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.272 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.272 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229492+0000", 2023-10-09T10:45:32.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1716999999999999e-05 2023-10-09T10:45:32.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.272 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.272 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229501+0000", 2023-10-09T10:45:32.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0929999999999992e-06 2023-10-09T10:45:32.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229507+0000", 2023-10-09T10:45:32.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6760000000000001e-06 2023-10-09T10:45:32.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229565+0000", 2023-10-09T10:45:32.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7450000000000001e-05 2023-10-09T10:45:32.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229572+0000", 2023-10-09T10:45:32.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0820000000000001e-06 2023-10-09T10:45:32.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103865+0000", 2023-10-09T10:45:32.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.87429282200000003 2023-10-09T10:45:32.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103873+0000", 2023-10-09T10:45:32.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2090000000000005e-06 2023-10-09T10:45:32.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181656+0000", 2023-10-09T10:45:32.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.077783462 2023-10-09T10:45:32.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181791+0000", 2023-10-09T10:45:32.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000134351 2023-10-09T10:45:32.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181948+0000", 2023-10-09T10:45:32.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015678600000000001 2023-10-09T10:45:32.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183685+0000", 2023-10-09T10:45:32.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00173705 2023-10-09T10:45:32.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183719+0000", 2023-10-09T10:45:32.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4301999999999998e-05 2023-10-09T10:45:32.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192640+0000", 2023-10-09T10:45:32.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0089210899999999996 2023-10-09T10:45:32.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192696+0000", 2023-10-09T10:45:32.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5733000000000003e-05 2023-10-09T10:45:32.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193346+0000", 2023-10-09T10:45:32.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00065071200000000001 2023-10-09T10:45:32.278 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.278 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.278 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.278 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42606 6.10 6:0826a955:::benchmark_data_smithi125_244951_object42605:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.278 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:45:32.278 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 36.980955237000003, 2023-10-09T10:45:32.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.955680458, 2023-10-09T10:45:32.278 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.279 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.279 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.279 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.279 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.279 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42606 2023-10-09T10:45:32.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.279 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:45:32.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:45:32.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238230+0000", 2023-10-09T10:45:32.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:45:32.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238257+0000", 2023-10-09T10:45:32.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7551000000000001e-05 2023-10-09T10:45:32.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238267+0000", 2023-10-09T10:45:32.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5820000000000008e-06 2023-10-09T10:45:32.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238273+0000", 2023-10-09T10:45:32.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5490000000000003e-06 2023-10-09T10:45:32.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238322+0000", 2023-10-09T10:45:32.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8467999999999997e-05 2023-10-09T10:45:32.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238332+0000", 2023-10-09T10:45:32.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9000000000000001e-06 2023-10-09T10:45:32.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103898+0000", 2023-10-09T10:45:32.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.86556675699999996 2023-10-09T10:45:32.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103906+0000", 2023-10-09T10:45:32.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7710000000000006e-06 2023-10-09T10:45:32.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.182699+0000", 2023-10-09T10:45:32.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0787928490000001 2023-10-09T10:45:32.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.182847+0000", 2023-10-09T10:45:32.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014758600000000001 2023-10-09T10:45:32.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183016+0000", 2023-10-09T10:45:32.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000169326 2023-10-09T10:45:32.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186922+0000", 2023-10-09T10:45:32.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039060269999999998 2023-10-09T10:45:32.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186980+0000", 2023-10-09T10:45:32.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8313999999999998e-05 2023-10-09T10:45:32.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193472+0000", 2023-10-09T10:45:32.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0064917020000000002 2023-10-09T10:45:32.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193522+0000", 2023-10-09T10:45:32.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9672999999999997e-05 2023-10-09T10:45:32.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193911+0000", 2023-10-09T10:45:32.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00038944099999999997 2023-10-09T10:45:32.286 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.286 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.286 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.286 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42608 6.10 6:0c1c992a:::benchmark_data_smithi125_244951_object42607:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.286 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:45:32.286 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 36.977863536999998, 2023-10-09T10:45:32.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9589951289999998, 2023-10-09T10:45:32.286 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.287 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.287 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.287 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.287 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.287 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42608 2023-10-09T10:45:32.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.287 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:45:32.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:45:32.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.288 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241321+0000", 2023-10-09T10:45:32.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:45:32.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.288 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241343+0000", 2023-10-09T10:45:32.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1987000000000001e-05 2023-10-09T10:45:32.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241355+0000", 2023-10-09T10:45:32.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1395e-05 2023-10-09T10:45:32.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241362+0000", 2023-10-09T10:45:32.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7759999999999997e-06 2023-10-09T10:45:32.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241406+0000", 2023-10-09T10:45:32.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3189999999999998e-05 2023-10-09T10:45:32.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241413+0000", 2023-10-09T10:45:32.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5800000000000003e-06 2023-10-09T10:45:32.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103930+0000", 2023-10-09T10:45:32.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.86251731399999998 2023-10-09T10:45:32.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103938+0000", 2023-10-09T10:45:32.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7349999999999996e-06 2023-10-09T10:45:32.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183741+0000", 2023-10-09T10:45:32.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0798032179999999 2023-10-09T10:45:32.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183865+0000", 2023-10-09T10:45:32.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012327200000000001 2023-10-09T10:45:32.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.184011+0000", 2023-10-09T10:45:32.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014617499999999999 2023-10-09T10:45:32.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186989+0000", 2023-10-09T10:45:32.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029777340000000001 2023-10-09T10:45:32.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187029+0000", 2023-10-09T10:45:32.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0513000000000003e-05 2023-10-09T10:45:32.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199414+0000", 2023-10-09T10:45:32.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012385022000000001 2023-10-09T10:45:32.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199472+0000", 2023-10-09T10:45:32.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7998999999999998e-05 2023-10-09T10:45:32.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200317+0000", 2023-10-09T10:45:32.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000845245 2023-10-09T10:45:32.294 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.294 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.294 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.294 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42717 6.10 6:0e29612e:::benchmark_data_smithi125_244951_object42716:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.294 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:45:32.294 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 36.911735079000003, 2023-10-09T10:45:32.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8935023019999999, 2023-10-09T10:45:32.295 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.295 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.295 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.295 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.295 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.295 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42717 2023-10-09T10:45:32.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.295 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:45:32.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:45:32.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307449+0000", 2023-10-09T10:45:32.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:32.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307473+0000", 2023-10-09T10:45:32.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3349e-05 2023-10-09T10:45:32.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307490+0000", 2023-10-09T10:45:32.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7133e-05 2023-10-09T10:45:32.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307496+0000", 2023-10-09T10:45:32.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7349999999999999e-06 2023-10-09T10:45:32.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307561+0000", 2023-10-09T10:45:32.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4226999999999993e-05 2023-10-09T10:45:32.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307575+0000", 2023-10-09T10:45:32.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4504e-05 2023-10-09T10:45:32.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103963+0000", 2023-10-09T10:45:32.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.79638768500000001 2023-10-09T10:45:32.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103971+0000", 2023-10-09T10:45:32.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9629999999999998e-06 2023-10-09T10:45:32.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.185776+0000", 2023-10-09T10:45:32.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0818051479999999 2023-10-09T10:45:32.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.185949+0000", 2023-10-09T10:45:32.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000172659 2023-10-09T10:45:32.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186122+0000", 2023-10-09T10:45:32.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017296799999999999 2023-10-09T10:45:32.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189418+0000", 2023-10-09T10:45:32.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032959679999999998 2023-10-09T10:45:32.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189457+0000", 2023-10-09T10:45:32.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9669000000000003e-05 2023-10-09T10:45:32.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200372+0000", 2023-10-09T10:45:32.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010914633999999999 2023-10-09T10:45:32.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200421+0000", 2023-10-09T10:45:32.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9047e-05 2023-10-09T10:45:32.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200953+0000", 2023-10-09T10:45:32.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00053212500000000005 2023-10-09T10:45:32.302 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.302 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.302 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.302 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42722 6.10 6:0ee7a776:::benchmark_data_smithi125_244951_object42721:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.302 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:45:32.303 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 36.911676870999997, 2023-10-09T10:45:32.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8972440939999999, 2023-10-09T10:45:32.303 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.303 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.303 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.303 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.303 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.303 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42722 2023-10-09T10:45:32.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.303 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:45:32.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:45:32.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307508+0000", 2023-10-09T10:45:32.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:32.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307529+0000", 2023-10-09T10:45:32.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0866999999999999e-05 2023-10-09T10:45:32.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307539+0000", 2023-10-09T10:45:32.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2690000000000005e-06 2023-10-09T10:45:32.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307544+0000", 2023-10-09T10:45:32.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7930000000000003e-06 2023-10-09T10:45:32.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307599+0000", 2023-10-09T10:45:32.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4876000000000001e-05 2023-10-09T10:45:32.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307607+0000", 2023-10-09T10:45:32.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8760000000000005e-06 2023-10-09T10:45:32.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103995+0000", 2023-10-09T10:45:32.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.79638814099999999 2023-10-09T10:45:32.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104003+0000", 2023-10-09T10:45:32.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7819999999999994e-06 2023-10-09T10:45:32.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187058+0000", 2023-10-09T10:45:32.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.083054712 2023-10-09T10:45:32.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187234+0000", 2023-10-09T10:45:32.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017615799999999999 2023-10-09T10:45:32.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187416+0000", 2023-10-09T10:45:32.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018243 2023-10-09T10:45:32.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189466+0000", 2023-10-09T10:45:32.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020493210000000002 2023-10-09T10:45:32.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189504+0000", 2023-10-09T10:45:32.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8287000000000001e-05 2023-10-09T10:45:32.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.201001+0000", 2023-10-09T10:45:32.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011496632999999999 2023-10-09T10:45:32.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.201044+0000", 2023-10-09T10:45:32.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3078000000000003e-05 2023-10-09T10:45:32.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204753+0000", 2023-10-09T10:45:32.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037093209999999998 2023-10-09T10:45:32.310 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.310 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.310 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.311 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42757 6.10 6:08e1bcf5:::benchmark_data_smithi125_244951_object42756:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.311 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.333229+0000", 2023-10-09T10:45:32.311 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 36.885956712000002, 2023-10-09T10:45:32.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.872111463, 2023-10-09T10:45:32.311 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.311 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.311 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.311 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.311 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.311 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42757 2023-10-09T10:45:32.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.311 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333229+0000", 2023-10-09T10:45:32.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333229+0000", 2023-10-09T10:45:32.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333228+0000", 2023-10-09T10:45:32.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:45:32.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333280+0000", 2023-10-09T10:45:32.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1999999999999997e-05 2023-10-09T10:45:32.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333291+0000", 2023-10-09T10:45:32.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0738e-05 2023-10-09T10:45:32.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333298+0000", 2023-10-09T10:45:32.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7730000000000002e-06 2023-10-09T10:45:32.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333360+0000", 2023-10-09T10:45:32.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2230000000000006e-05 2023-10-09T10:45:32.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333371+0000", 2023-10-09T10:45:32.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0669999999999999e-05 2023-10-09T10:45:32.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104027+0000", 2023-10-09T10:45:32.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.77065640400000002 2023-10-09T10:45:32.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104035+0000", 2023-10-09T10:45:32.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7400000000000004e-06 2023-10-09T10:45:32.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.188337+0000", 2023-10-09T10:45:32.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.084302281 2023-10-09T10:45:32.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.188495+0000", 2023-10-09T10:45:32.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015774699999999999 2023-10-09T10:45:32.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.188674+0000", 2023-10-09T10:45:32.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000178729 2023-10-09T10:45:32.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191542+0000", 2023-10-09T10:45:32.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028680799999999998 2023-10-09T10:45:32.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192592+0000", 2023-10-09T10:45:32.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010502140000000001 2023-10-09T10:45:32.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204808+0000", 2023-10-09T10:45:32.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012216595 2023-10-09T10:45:32.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204867+0000", 2023-10-09T10:45:32.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8251999999999997e-05 2023-10-09T10:45:32.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205341+0000", 2023-10-09T10:45:32.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047385999999999999 2023-10-09T10:45:32.318 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.319 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.319 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.319 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42857 6.10 6:0e113cab:::benchmark_data_smithi125_244951_object42856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.319 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.409599+0000", 2023-10-09T10:45:32.319 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 36.809587143000002, 2023-10-09T10:45:32.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.796322089, 2023-10-09T10:45:32.319 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.319 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.319 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.319 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.319 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.319 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42857 2023-10-09T10:45:32.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.320 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409599+0000", 2023-10-09T10:45:32.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409599+0000", 2023-10-09T10:45:32.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409597+0000", 2023-10-09T10:45:32.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:32.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409619+0000", 2023-10-09T10:45:32.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1202000000000001e-05 2023-10-09T10:45:32.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409631+0000", 2023-10-09T10:45:32.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2546e-05 2023-10-09T10:45:32.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409638+0000", 2023-10-09T10:45:32.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9269999999999999e-06 2023-10-09T10:45:32.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409694+0000", 2023-10-09T10:45:32.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6025e-05 2023-10-09T10:45:32.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409706+0000", 2023-10-09T10:45:32.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.164e-05 2023-10-09T10:45:32.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104059+0000", 2023-10-09T10:45:32.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.69435323100000002 2023-10-09T10:45:32.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104067+0000", 2023-10-09T10:45:32.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9060000000000002e-06 2023-10-09T10:45:32.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.324 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189529+0000", 2023-10-09T10:45:32.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.085462189 2023-10-09T10:45:32.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.324 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189675+0000", 2023-10-09T10:45:32.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014598599999999999 2023-10-09T10:45:32.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189846+0000", 2023-10-09T10:45:32.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017139799999999999 2023-10-09T10:45:32.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192547+0000", 2023-10-09T10:45:32.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027008760000000001 2023-10-09T10:45:32.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193395+0000", 2023-10-09T10:45:32.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00084811200000000004 2023-10-09T10:45:32.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205386+0000", 2023-10-09T10:45:32.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011990763999999999 2023-10-09T10:45:32.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205435+0000", 2023-10-09T10:45:32.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8467999999999997e-05 2023-10-09T10:45:32.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205921+0000", 2023-10-09T10:45:32.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00048609599999999999 2023-10-09T10:45:32.327 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.327 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.327 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.327 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42860 6.10 6:08b59805:::benchmark_data_smithi125_244951_object42859:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.327 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.411356+0000", 2023-10-09T10:45:32.327 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 36.807829380000001, 2023-10-09T10:45:32.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7952376650000001, 2023-10-09T10:45:32.327 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.328 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.328 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.328 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.328 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.328 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42860 2023-10-09T10:45:32.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.328 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411356+0000", 2023-10-09T10:45:32.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411356+0000", 2023-10-09T10:45:32.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411355+0000", 2023-10-09T10:45:32.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:32.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411383+0000", 2023-10-09T10:45:32.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7348999999999999e-05 2023-10-09T10:45:32.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.330 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411398+0000", 2023-10-09T10:45:32.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5424e-05 2023-10-09T10:45:32.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.330 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411407+0000", 2023-10-09T10:45:32.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2429999999999994e-06 2023-10-09T10:45:32.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411465+0000", 2023-10-09T10:45:32.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7892000000000001e-05 2023-10-09T10:45:32.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411475+0000", 2023-10-09T10:45:32.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0066e-05 2023-10-09T10:45:32.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104091+0000", 2023-10-09T10:45:32.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.69261584300000001 2023-10-09T10:45:32.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104099+0000", 2023-10-09T10:45:32.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8760000000000005e-06 2023-10-09T10:45:32.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.190559+0000", 2023-10-09T10:45:32.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.086459659 2023-10-09T10:45:32.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.190698+0000", 2023-10-09T10:45:32.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013982400000000001 2023-10-09T10:45:32.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.190864+0000", 2023-10-09T10:45:32.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016535200000000001 2023-10-09T10:45:32.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192601+0000", 2023-10-09T10:45:32.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0017371999999999999 2023-10-09T10:45:32.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193434+0000", 2023-10-09T10:45:32.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00083267900000000001 2023-10-09T10:45:32.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206008+0000", 2023-10-09T10:45:32.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012574058000000001 2023-10-09T10:45:32.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206060+0000", 2023-10-09T10:45:32.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2710999999999997e-05 2023-10-09T10:45:32.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.335 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206594+0000", 2023-10-09T10:45:32.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00053372399999999996 2023-10-09T10:45:32.335 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.335 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.335 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.335 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42870 6.10 6:0e8b4417:::benchmark_data_smithi125_244951_object42869:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.336 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.417961+0000", 2023-10-09T10:45:32.336 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 36.801224689999998, 2023-10-09T10:45:32.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8001529010000001, 2023-10-09T10:45:32.336 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.336 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.336 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.336 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.336 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.336 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42870 2023-10-09T10:45:32.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.336 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417961+0000", 2023-10-09T10:45:32.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417961+0000", 2023-10-09T10:45:32.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417960+0000", 2023-10-09T10:45:32.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:32.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417982+0000", 2023-10-09T10:45:32.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2585000000000001e-05 2023-10-09T10:45:32.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417999+0000", 2023-10-09T10:45:32.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6711999999999999e-05 2023-10-09T10:45:32.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.418005+0000", 2023-10-09T10:45:32.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3180000000000003e-06 2023-10-09T10:45:32.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.418044+0000", 2023-10-09T10:45:32.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9171999999999997e-05 2023-10-09T10:45:32.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.418052+0000", 2023-10-09T10:45:32.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9880000000000004e-06 2023-10-09T10:45:32.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104123+0000", 2023-10-09T10:45:32.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.68607037299999996 2023-10-09T10:45:32.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104131+0000", 2023-10-09T10:45:32.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0730000000000005e-06 2023-10-09T10:45:32.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193936+0000", 2023-10-09T10:45:32.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0898054349999999 2023-10-09T10:45:32.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.194029+0000", 2023-10-09T10:45:32.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2313999999999997e-05 2023-10-09T10:45:32.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.194131+0000", 2023-10-09T10:45:32.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000102384 2023-10-09T10:45:32.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199366+0000", 2023-10-09T10:45:32.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0052347640000000003 2023-10-09T10:45:32.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205969+0000", 2023-10-09T10:45:32.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066036840000000003 2023-10-09T10:45:32.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.217437+0000", 2023-10-09T10:45:32.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011467252000000001 2023-10-09T10:45:32.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.217474+0000", 2023-10-09T10:45:32.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7177e-05 2023-10-09T10:45:32.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218114+0000", 2023-10-09T10:45:32.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000640231 2023-10-09T10:45:32.343 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.343 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.344 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.344 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42902 6.10 6:0d48f256:::benchmark_data_smithi125_244951_object42901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.344 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.448090+0000", 2023-10-09T10:45:32.344 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 36.771096231999998, 2023-10-09T10:45:32.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7704487879999999, 2023-10-09T10:45:32.344 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.344 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.344 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.344 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.344 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.344 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42902 2023-10-09T10:45:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448090+0000", 2023-10-09T10:45:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448090+0000", 2023-10-09T10:45:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448088+0000", 2023-10-09T10:45:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:45:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448108+0000", 2023-10-09T10:45:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9888999999999999e-05 2023-10-09T10:45:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448124+0000", 2023-10-09T10:45:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5529999999999999e-05 2023-10-09T10:45:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448131+0000", 2023-10-09T10:45:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9500000000000004e-06 2023-10-09T10:45:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448189+0000", 2023-10-09T10:45:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8010000000000002e-05 2023-10-09T10:45:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448201+0000", 2023-10-09T10:45:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2368000000000001e-05 2023-10-09T10:45:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104155+0000", 2023-10-09T10:45:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.65595361500000005 2023-10-09T10:45:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104163+0000", 2023-10-09T10:45:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9759999999999995e-06 2023-10-09T10:45:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.194936+0000", 2023-10-09T10:45:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.090773378 2023-10-09T10:45:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.195074+0000", 2023-10-09T10:45:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000138298 2023-10-09T10:45:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.196676+0000", 2023-10-09T10:45:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001601327 2023-10-09T10:45:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199373+0000", 2023-10-09T10:45:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026971320000000001 2023-10-09T10:45:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206643+0000", 2023-10-09T10:45:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0072700519999999999 2023-10-09T10:45:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218147+0000", 2023-10-09T10:45:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011503904000000001 2023-10-09T10:45:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218182+0000", 2023-10-09T10:45:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4752999999999997e-05 2023-10-09T10:45:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218538+0000", 2023-10-09T10:45:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035677400000000002 2023-10-09T10:45:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42906 6.10 6:0945c66e:::benchmark_data_smithi125_244951_object42905:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:45:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.453858+0000", 2023-10-09T10:45:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 36.765328119000003, 2023-10-09T10:45:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7650414240000001, 2023-10-09T10:45:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42906 2023-10-09T10:45:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453858+0000", 2023-10-09T10:45:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453858+0000", 2023-10-09T10:45:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453857+0000", 2023-10-09T10:45:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:45:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453877+0000", 2023-10-09T10:45:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0857999999999999e-05 2023-10-09T10:45:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453891+0000", 2023-10-09T10:45:32.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3249e-05 2023-10-09T10:45:32.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453897+0000", 2023-10-09T10:45:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1129999999999997e-06 2023-10-09T10:45:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453938+0000", 2023-10-09T10:45:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0799e-05 2023-10-09T10:45:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453945+0000", 2023-10-09T10:45:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2849999999999997e-06 2023-10-09T10:45:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104187+0000", 2023-10-09T10:45:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.65024259699999998 2023-10-09T10:45:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104195+0000", 2023-10-09T10:45:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9710000000000004e-06 2023-10-09T10:45:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.197298+0000", 2023-10-09T10:45:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.093102646 2023-10-09T10:45:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.197461+0000", 2023-10-09T10:45:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016253599999999999 2023-10-09T10:45:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.197621+0000", 2023-10-09T10:45:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000160178 2023-10-09T10:45:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200332+0000", 2023-10-09T10:45:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027108029999999999 2023-10-09T10:45:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206690+0000", 2023-10-09T10:45:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0063584990000000001 2023-10-09T10:45:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218563+0000", 2023-10-09T10:45:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011872562 2023-10-09T10:45:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218587+0000", 2023-10-09T10:45:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4586000000000001e-05 2023-10-09T10:45:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218899+0000", 2023-10-09T10:45:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000311883 2023-10-09T10:45:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:56713 6.10 6:086e6a33:::benchmark_data_smithi125_244951_object56712:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-10-09T10:45:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:08.647257+0000", 2023-10-09T10:45:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 23.571928942, 2023-10-09T10:45:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0123530810000001, 2023-10-09T10:45:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:45:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 56713 2023-10-09T10:45:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647257+0000", 2023-10-09T10:45:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647257+0000", 2023-10-09T10:45:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647256+0000", 2023-10-09T10:45:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647273+0000", 2023-10-09T10:45:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6075000000000001e-05 2023-10-09T10:45:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647281+0000", 2023-10-09T10:45:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6589999999999996e-06 2023-10-09T10:45:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647287+0000", 2023-10-09T10:45:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4940000000000003e-06 2023-10-09T10:45:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647464+0000", 2023-10-09T10:45:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017730999999999999 2023-10-09T10:45:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647667+0000", 2023-10-09T10:45:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020269 2023-10-09T10:45:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647835+0000", 2023-10-09T10:45:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000168677 2023-10-09T10:45:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.649455+0000", 2023-10-09T10:45:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016191999999999999 2023-10-09T10:45:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.655385+0000", 2023-10-09T10:45:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0059305549999999997 2023-10-09T10:45:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.658642+0000", 2023-10-09T10:45:32.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032573540000000001 2023-10-09T10:45:32.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.658657+0000", 2023-10-09T10:45:32.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4354e-05 2023-10-09T10:45:32.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.659536+0000", 2023-10-09T10:45:32.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0008789929999999 2023-10-09T10:45:32.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.659610+0000", 2023-10-09T10:45:32.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4134999999999994e-05 2023-10-09T10:45:32.369 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.369 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.369 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.369 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:56783 6.10 6:0d692e98:::benchmark_data_smithi125_244951_object56782:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-10-09T10:45:32.369 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:08.699618+0000", 2023-10-09T10:45:32.369 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 23.519567646999999, 2023-10-09T10:45:32.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96364151399999998, 2023-10-09T10:45:32.369 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.370 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.370 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.370 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.370 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.370 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 56783 2023-10-09T10:45:32.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.370 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.699618+0000", 2023-10-09T10:45:32.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.699618+0000", 2023-10-09T10:45:32.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.699616+0000", 2023-10-09T10:45:32.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:45:32.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.699656+0000", 2023-10-09T10:45:32.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9555999999999999e-05 2023-10-09T10:45:32.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.699669+0000", 2023-10-09T10:45:32.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2816000000000001e-05 2023-10-09T10:45:32.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.699677+0000", 2023-10-09T10:45:32.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9410000000000007e-06 2023-10-09T10:45:32.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.699847+0000", 2023-10-09T10:45:32.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017032399999999999 2023-10-09T10:45:32.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.699858+0000", 2023-10-09T10:45:32.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0682999999999999e-05 2023-10-09T10:45:32.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.659891+0000", 2023-10-09T10:45:32.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96003320700000006 2023-10-09T10:45:32.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.660048+0000", 2023-10-09T10:45:32.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000156725 2023-10-09T10:45:32.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.660190+0000", 2023-10-09T10:45:32.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000142564 2023-10-09T10:45:32.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.663058+0000", 2023-10-09T10:45:32.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028680200000000002 2023-10-09T10:45:32.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.663176+0000", 2023-10-09T10:45:32.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0001177 2023-10-09T10:45:32.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.663181+0000", 2023-10-09T10:45:32.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7079999999999996e-06 2023-10-09T10:45:32.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.663221+0000", 2023-10-09T10:45:32.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0160000000000002e-05 2023-10-09T10:45:32.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.663260+0000", 2023-10-09T10:45:32.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8794999999999997e-05 2023-10-09T10:45:32.376 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.376 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.376 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.377 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:56793 6.10 6:089c55e8:::benchmark_data_smithi125_244951_object56792:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-10-09T10:45:32.377 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:08.705082+0000", 2023-10-09T10:45:32.377 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 23.514104271000001, 2023-10-09T10:45:32.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96299103500000005, 2023-10-09T10:45:32.377 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.377 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.377 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.377 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.377 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.377 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 56793 2023-10-09T10:45:32.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.377 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.705082+0000", 2023-10-09T10:45:32.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.705082+0000", 2023-10-09T10:45:32.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.379 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.705080+0000", 2023-10-09T10:45:32.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:32.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.379 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.379 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.705110+0000", 2023-10-09T10:45:32.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0395e-05 2023-10-09T10:45:32.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.379 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.379 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.705124+0000", 2023-10-09T10:45:32.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3984e-05 2023-10-09T10:45:32.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.705132+0000", 2023-10-09T10:45:32.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6329999999999996e-06 2023-10-09T10:45:32.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.705755+0000", 2023-10-09T10:45:32.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00062351099999999999 2023-10-09T10:45:32.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.705766+0000", 2023-10-09T10:45:32.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0831e-05 2023-10-09T10:45:32.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.660960+0000", 2023-10-09T10:45:32.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.955193391 2023-10-09T10:45:32.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.661090+0000", 2023-10-09T10:45:32.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013029900000000001 2023-10-09T10:45:32.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.661242+0000", 2023-10-09T10:45:32.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015233600000000001 2023-10-09T10:45:32.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.663303+0000", 2023-10-09T10:45:32.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002061149 2023-10-09T10:45:32.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.665427+0000", 2023-10-09T10:45:32.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00212385 2023-10-09T10:45:32.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.667983+0000", 2023-10-09T10:45:32.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002556034 2023-10-09T10:45:32.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.668034+0000", 2023-10-09T10:45:32.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0387999999999999e-05 2023-10-09T10:45:32.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.384 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.668073+0000", 2023-10-09T10:45:32.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8831000000000001e-05 2023-10-09T10:45:32.384 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.384 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.384 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.384 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.384 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.384 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:56801 6.10 6:0d92d488:::benchmark_data_smithi125_244951_object56800:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-10-09T10:45:32.384 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:08.706963+0000", 2023-10-09T10:45:32.384 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 23.512222795, 2023-10-09T10:45:32.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96120153799999997, 2023-10-09T10:45:32.384 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:32.384 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:45:32.385 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:32.385 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:32.385 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:32.385 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 56801 2023-10-09T10:45:32.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.385 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:32.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:32.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.706963+0000", 2023-10-09T10:45:32.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:32.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.706963+0000", 2023-10-09T10:45:32.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:32.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:32.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.706962+0000", 2023-10-09T10:45:32.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:45:32.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:32.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.706982+0000", 2023-10-09T10:45:32.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0256999999999999e-05 2023-10-09T10:45:32.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:32.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.706994+0000", 2023-10-09T10:45:32.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1552000000000001e-05 2023-10-09T10:45:32.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:32.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.707000+0000", 2023-10-09T10:45:32.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2149999999999998e-06 2023-10-09T10:45:32.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.707048+0000", 2023-10-09T10:45:32.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7672e-05 2023-10-09T10:45:32.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:45:32.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.707060+0000", 2023-10-09T10:45:32.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1933000000000001e-05 2023-10-09T10:45:32.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:32.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.661974+0000", 2023-10-09T10:45:32.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95491459099999998 2023-10-09T10:45:32.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:32.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.662093+0000", 2023-10-09T10:45:32.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118523 2023-10-09T10:45:32.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:45:32.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.662236+0000", 2023-10-09T10:45:32.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014298300000000001 2023-10-09T10:45:32.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.663338+0000", 2023-10-09T10:45:32.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011025220000000001 2023-10-09T10:45:32.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:45:32.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.665433+0000", 2023-10-09T10:45:32.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020944890000000002 2023-10-09T10:45:32.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:45:32.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.668122+0000", 2023-10-09T10:45:32.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026890550000000001 2023-10-09T10:45:32.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:45:32.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.668147+0000", 2023-10-09T10:45:32.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5262e-05 2023-10-09T10:45:32.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:32.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:32.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:45:32.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.668165+0000", 2023-10-09T10:45:32.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7541999999999999e-05 2023-10-09T10:45:32.391 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.391 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.391 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.391 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:32.391 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:32.391 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:45:32.392 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:32.393 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:45:32.312468+0000 min lat: 0.00102371 max lat: 2.05736 avg lat: 0.0121481 2023-10-09T10:45:32.393 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:45:32.393 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 60 16 79041 79025 82.2824 66.125 0.0125618 0.0121481 2023-10-09T10:45:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:32 smithi155 ceph-mon[184854]: pgmap v258: 109 pgs: 10 unknown, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 90 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 13603/228648 objects degraded (5.949%); 4314/228648 objects misplaced (1.887%); 12 MiB/s, 190 objects/s recovering 2023-10-09T10:45:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:32 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:32 smithi155 ceph-mon[184854]: failed to encode map e129 with expected crc 2023-10-09T10:45:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:32 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:45:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:32 smithi155 ceph-mon[184854]: osdmap e129: 8 total, 8 up, 7 in 2023-10-09T10:45:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:32 smithi155 ceph-mon[184854]: failed to encode map e129 with expected crc 2023-10-09T10:45:32.495 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:32.629 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:32.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:32.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:32.673 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:45:32.739 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:32.790 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:32.821 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:32.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:32.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:32.928 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:32.977 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:33.006 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:33.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:33.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:33.111 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:33.146 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:33.158 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:33.182 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:33.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:33.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:33.287 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:33.311 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 61 16 80136 80120 82.0554 68.4375 0.00878624 0.0121822 2023-10-09T10:45:33.327 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:33.355 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:33.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:33.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:33.468 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:33 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 13603/228648 objects degraded (5.949%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:33.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:33 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 13603/228648 objects degraded (5.949%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:33.518 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:33 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 13603/228648 objects degraded (5.949%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:33.526 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:33.705 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:33.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:33.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:33.795 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:45:33.795 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:45:33.795 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T10:45:33.796 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:45:33.796 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:45:33.796 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T10:45:33.811 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:33.881 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:45:33.916 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:33.951 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:33.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:33.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:34.065 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:34.119 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:34.170 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:34.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:34.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:34.214 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:45:34.275 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:34.312 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 62 16 81196 81180 81.8004 66.25 0.0148854 0.0122201 2023-10-09T10:45:34.346 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:34.393 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:34.393 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:45:34.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:34.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:34.498 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:34.539 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:34 smithi155 ceph-mon[184854]: pgmap v260: 109 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 99 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 13605/230466 objects degraded (5.903%); 4312/230466 objects misplaced (1.871%); 15 MiB/s, 233 objects/s recovering 2023-10-09T10:45:34.542 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:34.558 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:34 smithi125 ceph-mon[195507]: pgmap v260: 109 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 99 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 13605/230466 objects degraded (5.903%); 4312/230466 objects misplaced (1.871%); 15 MiB/s, 233 objects/s recovering 2023-10-09T10:45:34.559 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:34 smithi125 ceph-mon[200197]: pgmap v260: 109 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 99 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 13605/230466 objects degraded (5.903%); 4312/230466 objects misplaced (1.871%); 15 MiB/s, 233 objects/s recovering 2023-10-09T10:45:34.725 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:34.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:34.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:34.830 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:34.886 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:34.931 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:34.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:34.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:34.952 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:35.035 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:35.090 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:35.175 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:35.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:35.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:35.282 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:35.312 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 63 16 82204 82188 81.502 63 0.00988837 0.0122641 2023-10-09T10:45:35.338 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:35.370 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:35.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:35.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:35.381 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:35 smithi155 ceph-mon[184854]: pgmap v261: 109 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 99 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 13605/237891 objects degraded (5.719%); 3345/237891 objects misplaced (1.406%); 17 MiB/s, 269 objects/s recovering 2023-10-09T10:45:35.399 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:35.433 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:45:35.434 INFO:teuthology.orchestra.run.smithi155.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T10:45:35.434 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:45:35.479 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:35.557 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:35.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:35 smithi125 ceph-mon[195507]: pgmap v261: 109 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 99 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 13605/237891 objects degraded (5.719%); 3345/237891 objects misplaced (1.406%); 17 MiB/s, 269 objects/s recovering 2023-10-09T10:45:35.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:35 smithi125 ceph-mon[200197]: pgmap v261: 109 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 99 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 13605/237891 objects degraded (5.719%); 3345/237891 objects misplaced (1.406%); 17 MiB/s, 269 objects/s recovering 2023-10-09T10:45:35.754 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:35.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:35.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:35.809 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:45:35.809 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:45:35.809 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:35.809 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:82675 6.11 6:8a858241:::benchmark_data_smithi125_244951_object82674:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T10:45:35.810 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:35.807184+0000", 2023-10-09T10:45:35.810 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.002240096, 2023-10-09T10:45:35.810 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022747560000000002, 2023-10-09T10:45:35.810 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:35.810 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:45:35.810 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:35.810 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:35.810 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:35.810 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 82675 2023-10-09T10:45:35.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:35.811 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:35.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:35.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:35.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:35.807184+0000", 2023-10-09T10:45:35.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:35.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:35.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:35.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:35.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:35.807184+0000", 2023-10-09T10:45:35.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:35.812 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:35.812 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:35.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:35.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:35.807182+0000", 2023-10-09T10:45:35.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:45:35.812 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:35.812 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:35.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:35.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:35.807228+0000", 2023-10-09T10:45:35.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5636999999999997e-05 2023-10-09T10:45:35.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:35.813 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:35.813 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:35.813 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:35.807246+0000", 2023-10-09T10:45:35.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7583000000000001e-05 2023-10-09T10:45:35.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:35.813 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:35.813 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:35.813 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:35.807255+0000", 2023-10-09T10:45:35.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5340000000000005e-06 2023-10-09T10:45:35.814 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:35.814 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:35.814 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:35.814 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:35.807308+0000", 2023-10-09T10:45:35.814 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3029000000000003e-05 2023-10-09T10:45:35.814 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:35.814 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:35.814 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:35.814 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:35.807467+0000", 2023-10-09T10:45:35.814 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015864199999999999 2023-10-09T10:45:35.814 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:35.815 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:35.815 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T10:45:35.815 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:35.807604+0000", 2023-10-09T10:45:35.815 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013726999999999999 2023-10-09T10:45:35.815 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:35.815 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:35.815 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:35.815 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:35.815 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:45:35.815 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 1 2023-10-09T10:45:35.816 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:45:35.858 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:35.905 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:35.935 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:35.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:35.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:36.039 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:36.087 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:36.120 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:36.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:36.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:36.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:36.284 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:36.312 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 64 16 83196 83180 81.1973 62 0.0164864 0.0123095 2023-10-09T10:45:36.326 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:36.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:36.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:36.339 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:45:36.431 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:36.507 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:36.610 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:36.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:36.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:36.716 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:36.755 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:36.787 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:36.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:36.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:36.896 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:36.966 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:37.000 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:36 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1663881943' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:37.000 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:36 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:37.043 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:45:37.194 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:37.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:37.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:37.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:36 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1663881943' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:37.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:36 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:37.286 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:37.300 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:37.315 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 65 16 84118 84102 80.8328 57.625 0.0114413 0.0123653 2023-10-09T10:45:37.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:37 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1663881943' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:37.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:37 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:37.378 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:37.430 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:45:37.486 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:45:37.526 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:45:37.529 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:37.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:37.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:37.569 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:45:37.602 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:45:37.643 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:45:37.644 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:37.680 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:45:37.707 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:37.717 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:45:37.724 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:45:37.724 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:45:37.724 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:45:37.724 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:45:37.724 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:45:37.724 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:45:37.724 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:45:37.724 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:45:37.784 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:37.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:37.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:37.890 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:37.942 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:38.000 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:45:38.001 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:45:38.001 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[195507]: osdmap e130: 8 total, 8 up, 7 in 2023-10-09T10:45:38.001 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[195507]: failed to encode map e130 with expected crc 2023-10-09T10:45:38.001 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[195507]: pgmap v263: 109 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 99 active+clean; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 13605/245838 objects degraded (5.534%); 2798/245838 objects misplaced (1.138%); 16 MiB/s, 253 objects/s recovering 2023-10-09T10:45:38.001 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[195507]: failed to encode map e130 with expected crc 2023-10-09T10:45:38.001 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/56737339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:38.001 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 13605/237891 objects degraded (5.719%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:38.001 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[195507]: failed to encode map e131 with expected crc 2023-10-09T10:45:38.001 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:45:38.002 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:45:38.002 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[200197]: osdmap e130: 8 total, 8 up, 7 in 2023-10-09T10:45:38.002 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[200197]: failed to encode map e130 with expected crc 2023-10-09T10:45:38.172 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:38.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:38.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:38.276 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:38.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:38 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:45:38.302 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:38 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:45:38.302 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:38 smithi155 ceph-mon[184854]: osdmap e130: 8 total, 8 up, 7 in 2023-10-09T10:45:38.302 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:38 smithi155 ceph-mon[184854]: failed to encode map e130 with expected crc 2023-10-09T10:45:38.302 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:38 smithi155 ceph-mon[184854]: pgmap v263: 109 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 99 active+clean; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 13605/245838 objects degraded (5.534%); 2798/245838 objects misplaced (1.138%); 16 MiB/s, 253 objects/s recovering 2023-10-09T10:45:38.302 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:38 smithi155 ceph-mon[184854]: failed to encode map e130 with expected crc 2023-10-09T10:45:38.302 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:38 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/56737339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:38.302 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:38 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 13605/237891 objects degraded (5.719%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:38.302 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:38 smithi155 ceph-mon[184854]: failed to encode map e131 with expected crc 2023-10-09T10:45:38.302 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:38 smithi155 ceph-mon[184854]: osdmap e131: 8 total, 8 up, 7 in 2023-10-09T10:45:38.302 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:38 smithi155 ceph-mon[184854]: failed to encode map e131 with expected crc 2023-10-09T10:45:38.302 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:38 smithi155 ceph-mon[184854]: failed to encode map e131 with expected crc 2023-10-09T10:45:38.304 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:38.313 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 66 16 84731 84715 80.1887 38.3125 0.00229293 0.0123773 2023-10-09T10:45:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[195507]: osdmap e131: 8 total, 8 up, 7 in 2023-10-09T10:45:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[195507]: failed to encode map e131 with expected crc 2023-10-09T10:45:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[195507]: failed to encode map e131 with expected crc 2023-10-09T10:45:38.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[200197]: pgmap v263: 109 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 99 active+clean; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 13605/245838 objects degraded (5.534%); 2798/245838 objects misplaced (1.138%); 16 MiB/s, 253 objects/s recovering 2023-10-09T10:45:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[200197]: failed to encode map e130 with expected crc 2023-10-09T10:45:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/56737339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 13605/237891 objects degraded (5.719%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[200197]: failed to encode map e131 with expected crc 2023-10-09T10:45:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[200197]: osdmap e131: 8 total, 8 up, 7 in 2023-10-09T10:45:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[200197]: failed to encode map e131 with expected crc 2023-10-09T10:45:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:38 smithi125 ceph-mon[200197]: failed to encode map e131 with expected crc 2023-10-09T10:45:38.324 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:38.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:38.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:38.428 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:38.456 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:38.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:38.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:38.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:38.582 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:38.645 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:38.733 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:38.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:38.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:38.843 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:38.911 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:38.937 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:38.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:38.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:39.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:39 smithi125 ceph-mon[195507]: failed to encode map e131 with expected crc 2023-10-09T10:45:39.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:39 smithi125 ceph-mon[195507]: failed to encode map e131 with expected crc 2023-10-09T10:45:39.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:39 smithi125 ceph-mon[195507]: failed to encode map e131 with expected crc 2023-10-09T10:45:39.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:39 smithi125 ceph-mon[195507]: failed to encode map e131 with expected crc 2023-10-09T10:45:39.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:39 smithi125 ceph-mon[195507]: failed to encode map e131 with expected crc 2023-10-09T10:45:39.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:39 smithi125 ceph-mon[195507]: osdmap e132: 8 total, 8 up, 7 in 2023-10-09T10:45:39.044 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:39.050 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:45:39.050 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:45:39.097 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:39.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:39 smithi155 ceph-mon[184854]: failed to encode map e131 with expected crc 2023-10-09T10:45:39.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:39 smithi155 ceph-mon[184854]: failed to encode map e131 with expected crc 2023-10-09T10:45:39.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:39 smithi155 ceph-mon[184854]: failed to encode map e131 with expected crc 2023-10-09T10:45:39.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:39 smithi155 ceph-mon[184854]: failed to encode map e131 with expected crc 2023-10-09T10:45:39.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:39 smithi155 ceph-mon[184854]: failed to encode map e131 with expected crc 2023-10-09T10:45:39.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:39 smithi155 ceph-mon[184854]: osdmap e132: 8 total, 8 up, 7 in 2023-10-09T10:45:39.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:39 smithi155 ceph-mon[184854]: failed to encode map e132 with expected crc 2023-10-09T10:45:39.313 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 67 16 85260 85244 79.4855 33.0625 0.00242324 0.0125043 2023-10-09T10:45:39.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:39 smithi125 ceph-mon[195507]: failed to encode map e132 with expected crc 2023-10-09T10:45:39.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:39 smithi125 ceph-mon[200197]: failed to encode map e131 with expected crc 2023-10-09T10:45:39.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:39 smithi125 ceph-mon[200197]: failed to encode map e131 with expected crc 2023-10-09T10:45:39.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:39 smithi125 ceph-mon[200197]: failed to encode map e131 with expected crc 2023-10-09T10:45:39.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:39 smithi125 ceph-mon[200197]: failed to encode map e131 with expected crc 2023-10-09T10:45:39.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:39 smithi125 ceph-mon[200197]: failed to encode map e131 with expected crc 2023-10-09T10:45:39.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:39 smithi125 ceph-mon[200197]: osdmap e132: 8 total, 8 up, 7 in 2023-10-09T10:45:39.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:39 smithi125 ceph-mon[200197]: failed to encode map e132 with expected crc 2023-10-09T10:45:39.527 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:39.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:39.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:39.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:45:39.569 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:45:39.630 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:39.675 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:39.697 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:39.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:39.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:39.802 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:39.877 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:39.919 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:39.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:39.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:40.025 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:40.028 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:40.065 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:40.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:40.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:40.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:40.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:45:40.131 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:40.185 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:40 smithi125 ceph-mon[195507]: failed to encode map e132 with expected crc 2023-10-09T10:45:40.185 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:40 smithi125 ceph-mon[195507]: failed to encode map e132 with expected crc 2023-10-09T10:45:40.186 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:40 smithi125 ceph-mon[195507]: pgmap v266: 109 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 99 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 65 MiB/s wr, 1.05k op/s; 13605/249333 objects degraded (5.457%); 2397/249333 objects misplaced (0.961%); 20 MiB/s, 318 objects/s recovering 2023-10-09T10:45:40.186 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:40 smithi125 ceph-mon[195507]: osdmap e133: 8 total, 8 up, 7 in 2023-10-09T10:45:40.186 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:40 smithi125 ceph-mon[195507]: failed to encode map e133 with expected crc 2023-10-09T10:45:40.186 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:40 smithi125 ceph-mon[195507]: failed to encode map e133 with expected crc 2023-10-09T10:45:40.186 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:40 smithi125 ceph-mon[200197]: failed to encode map e132 with expected crc 2023-10-09T10:45:40.186 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:40 smithi125 ceph-mon[200197]: failed to encode map e132 with expected crc 2023-10-09T10:45:40.186 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:40 smithi125 ceph-mon[200197]: pgmap v266: 109 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 99 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 65 MiB/s wr, 1.05k op/s; 13605/249333 objects degraded (5.457%); 2397/249333 objects misplaced (0.961%); 20 MiB/s, 318 objects/s recovering 2023-10-09T10:45:40.186 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:40 smithi125 ceph-mon[200197]: osdmap e133: 8 total, 8 up, 7 in 2023-10-09T10:45:40.187 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:40 smithi125 ceph-mon[200197]: failed to encode map e133 with expected crc 2023-10-09T10:45:40.187 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:40 smithi125 ceph-mon[200197]: failed to encode map e133 with expected crc 2023-10-09T10:45:40.206 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:40.250 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:40.275 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:40.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:40.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:40.314 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 68 16 85891 85875 78.8967 39.4375 0.00925461 0.0126704 2023-10-09T10:45:40.379 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:40.439 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:40 smithi155 ceph-mon[184854]: failed to encode map e132 with expected crc 2023-10-09T10:45:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:40 smithi155 ceph-mon[184854]: failed to encode map e132 with expected crc 2023-10-09T10:45:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:40 smithi155 ceph-mon[184854]: pgmap v266: 109 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 99 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 65 MiB/s wr, 1.05k op/s; 13605/249333 objects degraded (5.457%); 2397/249333 objects misplaced (0.961%); 20 MiB/s, 318 objects/s recovering 2023-10-09T10:45:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:40 smithi155 ceph-mon[184854]: osdmap e133: 8 total, 8 up, 7 in 2023-10-09T10:45:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:40 smithi155 ceph-mon[184854]: failed to encode map e133 with expected crc 2023-10-09T10:45:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:40 smithi155 ceph-mon[184854]: failed to encode map e133 with expected crc 2023-10-09T10:45:40.856 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:45:40.856 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:45:40.856 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T10:45:40.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:45:41.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:41 smithi125 ceph-mon[195507]: 6.2 scrub starts 2023-10-09T10:45:41.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:41 smithi125 ceph-mon[195507]: failed to encode map e133 with expected crc 2023-10-09T10:45:41.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:41 smithi125 ceph-mon[195507]: failed to encode map e133 with expected crc 2023-10-09T10:45:41.222 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:41 smithi125 ceph-mon[200197]: 6.2 scrub starts 2023-10-09T10:45:41.222 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:41 smithi125 ceph-mon[200197]: failed to encode map e133 with expected crc 2023-10-09T10:45:41.222 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:41 smithi125 ceph-mon[200197]: failed to encode map e133 with expected crc 2023-10-09T10:45:41.223 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:45:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:45:41] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.43.0" 2023-10-09T10:45:41.314 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 69 16 86936 86920 78.6998 65.3125 0.0112703 0.0126927 2023-10-09T10:45:41.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:41 smithi155 ceph-mon[184854]: 6.2 scrub starts 2023-10-09T10:45:41.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:41 smithi155 ceph-mon[184854]: failed to encode map e133 with expected crc 2023-10-09T10:45:41.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:41 smithi155 ceph-mon[184854]: failed to encode map e133 with expected crc 2023-10-09T10:45:41.783 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:41.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:41.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:41.887 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:41.916 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:41.935 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:41.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:41.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:42.039 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:42.067 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:42.088 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:42.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:42.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:42.193 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:42.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:42 smithi155 ceph-mon[184854]: pgmap v268: 109 pgs: 2 active+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 100 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53 MiB/s wr, 854 op/s; 11673/256146 objects degraded (4.557%); 2493/256146 objects misplaced (0.973%); 35 MiB/s, 556 objects/s recovering 2023-10-09T10:45:42.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:42 smithi155 ceph-mon[184854]: osdmap e134: 8 total, 8 up, 7 in 2023-10-09T10:45:42.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:42 smithi155 ceph-mon[184854]: failed to encode map e134 with expected crc 2023-10-09T10:45:42.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:42 smithi155 ceph-mon[184854]: failed to encode map e134 with expected crc 2023-10-09T10:45:42.246 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:42 smithi125 ceph-mon[195507]: pgmap v268: 109 pgs: 2 active+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 100 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53 MiB/s wr, 854 op/s; 11673/256146 objects degraded (4.557%); 2493/256146 objects misplaced (0.973%); 35 MiB/s, 556 objects/s recovering 2023-10-09T10:45:42.247 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:42 smithi125 ceph-mon[195507]: osdmap e134: 8 total, 8 up, 7 in 2023-10-09T10:45:42.247 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:42 smithi125 ceph-mon[195507]: failed to encode map e134 with expected crc 2023-10-09T10:45:42.247 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:42 smithi125 ceph-mon[195507]: failed to encode map e134 with expected crc 2023-10-09T10:45:42.247 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:42 smithi125 ceph-mon[200197]: pgmap v268: 109 pgs: 2 active+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 100 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53 MiB/s wr, 854 op/s; 11673/256146 objects degraded (4.557%); 2493/256146 objects misplaced (0.973%); 35 MiB/s, 556 objects/s recovering 2023-10-09T10:45:42.247 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:42 smithi125 ceph-mon[200197]: osdmap e134: 8 total, 8 up, 7 in 2023-10-09T10:45:42.247 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:42 smithi125 ceph-mon[200197]: failed to encode map e134 with expected crc 2023-10-09T10:45:42.247 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:42 smithi125 ceph-mon[200197]: failed to encode map e134 with expected crc 2023-10-09T10:45:42.253 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:42.316 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 70 16 87291 87275 77.8927 22.1875 0.0170024 0.0128335 2023-10-09T10:45:42.394 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:42.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:42.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:42.500 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:42.573 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:42.666 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:42.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:42.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:42.775 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:42.799 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:42.859 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:43.051 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:43.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:43.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:43.159 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:43.199 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:43.223 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:43.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:43.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:43.314 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 71 16 88270 88254 77.6572 61.1875 0.0203544 0.0128716 2023-10-09T10:45:43.326 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:43.446 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[195507]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.446 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[195507]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.447 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[195507]: osdmap e135: 8 total, 8 up, 7 in 2023-10-09T10:45:43.447 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[195507]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.447 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[195507]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.447 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[195507]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.447 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[195507]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.447 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 11673/256146 objects degraded (4.557%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:43.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[200197]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[200197]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[200197]: osdmap e135: 8 total, 8 up, 7 in 2023-10-09T10:45:43.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[200197]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[200197]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[200197]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[200197]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.452 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:43 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 11673/256146 objects degraded (4.557%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:43.460 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:43.469 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:45:43.469 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:45:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:43 smithi155 ceph-mon[184854]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:43 smithi155 ceph-mon[184854]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:43 smithi155 ceph-mon[184854]: osdmap e135: 8 total, 8 up, 7 in 2023-10-09T10:45:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:43 smithi155 ceph-mon[184854]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:43 smithi155 ceph-mon[184854]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:43 smithi155 ceph-mon[184854]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:43 smithi155 ceph-mon[184854]: failed to encode map e135 with expected crc 2023-10-09T10:45:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:43 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 11673/256146 objects degraded (4.557%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:43.541 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:43.695 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:43.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:43.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:43.806 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:43.844 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:43.874 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:43.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:43.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:43.979 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:44.048 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:44.078 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:44.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:44.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:44.124 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:45:44.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:44.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:44 smithi155 ceph-mon[184854]: pgmap v271: 109 pgs: 2 active+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 100 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39 MiB/s wr, 616 op/s; 11673/257586 objects degraded (4.532%); 2493/257586 objects misplaced (0.968%); 26 MiB/s, 411 objects/s recovering 2023-10-09T10:45:44.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:44 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2058261839' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:45:44.239 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:44.314 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 72 16 89254 89238 77.4327 61.5 0.0157326 0.0129098 2023-10-09T10:45:44.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:44 smithi125 ceph-mon[195507]: pgmap v271: 109 pgs: 2 active+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 100 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39 MiB/s wr, 616 op/s; 11673/257586 objects degraded (4.532%); 2493/257586 objects misplaced (0.968%); 26 MiB/s, 411 objects/s recovering 2023-10-09T10:45:44.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:44 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2058261839' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:45:44.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:44 smithi125 ceph-mon[200197]: pgmap v271: 109 pgs: 2 active+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 100 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39 MiB/s wr, 616 op/s; 11673/257586 objects degraded (4.532%); 2493/257586 objects misplaced (0.968%); 26 MiB/s, 411 objects/s recovering 2023-10-09T10:45:44.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:44 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2058261839' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:45:44.399 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:45:45.033 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:45.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:45.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:45.140 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:45.174 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:45.200 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:45.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:45.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:45.304 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:45.316 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 73 16 90283 90267 77.2514 64.3125 0.0253356 0.0129395 2023-10-09T10:45:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:45 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:45 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2058261839' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:45:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:45 smithi125 ceph-mon[195507]: osdmap e136: 8 total, 8 up, 7 in 2023-10-09T10:45:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:45 smithi125 ceph-mon[195507]: failed to encode map e136 with expected crc 2023-10-09T10:45:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:45 smithi125 ceph-mon[195507]: failed to encode map e136 with expected crc 2023-10-09T10:45:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:45:45.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:45 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:45.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:45 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2058261839' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:45:45.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:45 smithi125 ceph-mon[200197]: osdmap e136: 8 total, 8 up, 7 in 2023-10-09T10:45:45.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:45 smithi125 ceph-mon[200197]: failed to encode map e136 with expected crc 2023-10-09T10:45:45.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:45 smithi125 ceph-mon[200197]: failed to encode map e136 with expected crc 2023-10-09T10:45:45.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:45:45.405 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:45.442 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:45.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:45.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:45 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:45 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2058261839' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:45:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:45 smithi155 ceph-mon[184854]: osdmap e136: 8 total, 8 up, 7 in 2023-10-09T10:45:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:45 smithi155 ceph-mon[184854]: failed to encode map e136 with expected crc 2023-10-09T10:45:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:45 smithi155 ceph-mon[184854]: failed to encode map e136 with expected crc 2023-10-09T10:45:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:45:45.548 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:45.597 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:45.663 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:45.750 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:45:45.763 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:45:45.769 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:45:45.792 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:45.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:45.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:45.810 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:45:45.864 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:45:45.899 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:45.909 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:45:45.941 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:45.956 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:45:45.996 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:45:45.998 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:46.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:46.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:46.025 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:45:46.063 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:45:46.080 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:45:46.080 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:45:46.080 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:45:46.080 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:45:46.081 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:45:46.081 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:45:46.081 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:45:46.081 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:45:46.102 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:46.145 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:46.171 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:46.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:46.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:46.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:46.316 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 74 16 91187 91171 76.971 56.5 0.0168028 0.0129864 2023-10-09T10:45:46.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:46 smithi125 ceph-mon[195507]: pgmap v273: 109 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 101 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 54 MiB/s wr, 861 op/s; 11673/263448 objects degraded (4.431%); 2052/263448 objects misplaced (0.779%); 26 MiB/s, 416 objects/s recovering 2023-10-09T10:45:46.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:46 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1586182308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:46.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:46 smithi125 ceph-mon[200197]: pgmap v273: 109 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 101 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 54 MiB/s wr, 861 op/s; 11673/263448 objects degraded (4.431%); 2052/263448 objects misplaced (0.779%); 26 MiB/s, 416 objects/s recovering 2023-10-09T10:45:46.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:46 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1586182308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:46.342 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:46 smithi155 ceph-mon[184854]: pgmap v273: 109 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 101 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 54 MiB/s wr, 861 op/s; 11673/263448 objects degraded (4.431%); 2052/263448 objects misplaced (0.779%); 26 MiB/s, 416 objects/s recovering 2023-10-09T10:45:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:46 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1586182308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:46.693 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:46.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:46.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:46.726 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":273,"stamp":"2023-10-09T10:45:45.043389+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5754099616,"num_objects":87816,"num_object_clones":0,"num_object_copies":263448,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":11673,"num_objects_misplaced":2052,"num_objects_unfound":0,"num_objects_dirty":87816,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":82844,"num_write_kb":5285229,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8530,"num_bytes_recovered":559745184,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110432,"ondisk_log_size":110432,"up":327,"acting":321,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":332,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17893192,"kb_used_data":16395984,"kb_used_omap":98,"kb_used_meta":1497053,"kb_avail":638322872,"statfs":{"total":671965249536,"available":653642620928,"internally_reserved":0,"allocated":16789487616,"data_stored":16785632396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":100565,"internal_metadata":1532983083},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":308346880,"num_objects":4705,"num_object_clones":0,"num_object_copies":14115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1932,"num_objects_misplaced":-345,"num_objects_unfound":0,"num_objects_dirty":4705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4705,"num_write_kb":301120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2277,"num_bytes_recovered":149225472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.463703"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":116,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592826+0000","last_change":"2023-10-09T10:43:06.676414+0000","last_active":"2023-10-09T10:45:38.592826+0000","last_peered":"2023-10-09T10:45:38.592826+0000","last_clean":"2023-10-09T10:45:38.592826+0000","last_became_active":"2023-10-09T10:43:06.676050+0000","last_became_peered":"2023-10-09T10:43:06.676050+0000","last_unstale":"2023-10-09T10:45:38.592826+0000","last_undegraded":"2023-10-09T10:45:38.592826+0000","last_fullsized":"2023-10-09T10:45:38.592826+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:28.011477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":102,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.099475+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:45:41.099475+0000","last_peered":"2023-10-09T10:45:41.099475+0000","last_clean":"2023-10-09T10:45:41.099475+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:45:41.099475+0000","last_undegraded":"2023-10-09T10:45:41.099475+0000","last_fullsized":"2023-10-09T10:45:41.099475+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":85,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.618339+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:45:39.618339+0000","last_peered":"2023-10-09T10:45:39.618339+0000","last_clean":"2023-10-09T10:45:39.618339+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:45:39.618339+0000","last_undegraded":"2023-10-09T10:45:39.618339+0000","last_fullsized":"2023-10-09T10:45:39.618339+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":69,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.099030+0000","last_change":"2023-10-09T10:45:15.767462+0000","last_active":"2023-10-09T10:45:41.099030+0000","last_peered":"2023-10-09T10:45:41.099030+0000","last_clean":"2023-10-09T10:45:41.099030+0000","last_became_active":"2023-10-09T10:45:15.767116+0000","last_became_peered":"2023-10-09T10:45:15.767116+0000","last_unstale":"2023-10-09T10:45:41.099030+0000","last_undegraded":"2023-10-09T10:45:41.099030+0000","last_fullsized":"2023-10-09T10:45:41.099030+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:42.610234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":129,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.094592+0000","last_change":"2023-10-09T10:45:15.768276+0000","last_active":"2023-10-09T10:45:41.094592+0000","last_peered":"2023-10-09T10:45:41.094592+0000","last_clean":"2023-10-09T10:45:41.094592+0000","last_became_active":"2023-10-09T10:45:15.767836+0000","last_became_peered":"2023-10-09T10:45:15.767836+0000","last_unstale":"2023-10-09T10:45:41.094592+0000","last_undegraded":"2023-10-09T10:45:41.094592+0000","last_fullsized":"2023-10-09T10:45:41.094592+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:40.987628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":71,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.585174+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:45:38.585174+0000","last_peered":"2023-10-09T10:45:38.585174+0000","last_clean":"2023-10-09T10:45:38.585174+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:45:38.585174+0000","last_undegraded":"2023-10-09T10:45:38.585174+0000","last_fullsized":"2023-10-09T10:45:38.585174+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"136'3914","reported_seq":8000,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T10:45:44.649623+0000","last_change":"2023-10-09T10:45:38.597219+0000","last_active":"2023-10-09T10:45:44.649623+0000","last_peered":"2023-10-09T10:45:44.649623+0000","last_clean":"2023-10-09T10:45:44.649623+0000","last_became_active":"2023-10-09T10:45:38.593601+0000","last_became_peered":"2023-10-09T10:45:38.593601+0000","last_unstale":"2023-10-09T10:45:44.649623+0000","last_undegraded":"2023-10-09T10:45:44.649623+0000","last_fullsized":"2023-10-09T10:45:44.649623+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3914,"log_dups_size":0,"ondisk_log_size":3914,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188579840,"num_objects":2878,"num_object_clones":0,"num_object_copies":8634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2878,"num_write_kb":184160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2005,"num_bytes_recovered":131399680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":67,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.585248+0000","last_change":"2023-10-09T10:45:15.763750+0000","last_active":"2023-10-09T10:45:38.585248+0000","last_peered":"2023-10-09T10:45:38.585248+0000","last_clean":"2023-10-09T10:45:38.585248+0000","last_became_active":"2023-10-09T10:45:15.762902+0000","last_became_peered":"2023-10-09T10:45:15.762902+0000","last_unstale":"2023-10-09T10:45:38.585248+0000","last_undegraded":"2023-10-09T10:45:38.585248+0000","last_fullsized":"2023-10-09T10:45:38.585248+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:08.151291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":102,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.094662+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:45:41.094662+0000","last_peered":"2023-10-09T10:45:41.094662+0000","last_clean":"2023-10-09T10:45:41.094662+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:45:41.094662+0000","last_undegraded":"2023-10-09T10:45:41.094662+0000","last_fullsized":"2023-10-09T10:45:41.094662+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"133'3725","reported_seq":3854,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:40.900552+0000","last_change":"2023-10-09T10:45:07.544591+0000","last_active":"2023-10-09T10:45:40.900552+0000","last_peered":"2023-10-09T10:45:40.900552+0000","last_clean":"2023-10-09T10:45:40.900552+0000","last_became_active":"2023-10-09T10:44:50.071995+0000","last_became_peered":"2023-10-09T10:44:50.071995+0000","last_unstale":"2023-10-09T10:45:40.900552+0000","last_undegraded":"2023-10-09T10:45:40.900552+0000","last_fullsized":"2023-10-09T10:45:40.900552+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":3725,"log_dups_size":0,"ondisk_log_size":3725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:45.046717+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175833088,"num_objects":2683,"num_object_clones":0,"num_object_copies":8049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1010,"num_write_kb":64640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.586820+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:45:38.586820+0000","last_peered":"2023-10-09T10:45:38.586820+0000","last_clean":"2023-10-09T10:45:38.586820+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:45:38.586820+0000","last_undegraded":"2023-10-09T10:45:38.586820+0000","last_fullsized":"2023-10-09T10:45:38.586820+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":61,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.096697+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:45:41.096697+0000","last_peered":"2023-10-09T10:45:41.096697+0000","last_clean":"2023-10-09T10:45:41.096697+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:45:41.096697+0000","last_undegraded":"2023-10-09T10:45:41.096697+0000","last_fullsized":"2023-10-09T10:45:41.096697+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"133'3773","reported_seq":11836,"reported_epoch":133,"state":"active+undersized+remapped","last_fresh":"2023-10-09T10:45:40.967589+0000","last_change":"2023-10-09T10:45:40.967589+0000","last_active":"2023-10-09T10:45:40.967589+0000","last_peered":"2023-10-09T10:45:40.967589+0000","last_clean":"2023-10-09T10:45:14.179874+0000","last_became_active":"2023-10-09T10:45:16.829821+0000","last_became_peered":"2023-10-09T10:45:16.829821+0000","last_unstale":"2023-10-09T10:45:40.967589+0000","last_undegraded":"2023-10-09T10:45:40.967589+0000","last_fullsized":"2023-10-09T10:45:16.750477+0000","mapping_epoch":122,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3773,"log_dups_size":0,"ondisk_log_size":3773,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178683904,"num_objects":2727,"num_object_clones":0,"num_object_copies":8181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2727,"num_write_kb":174496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1],"avail_no_missing":["5","0","1","2","3","6"],"object_location_counts":[{"shards":"1,5","objects":2727}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":55,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.099399+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:45:41.099399+0000","last_peered":"2023-10-09T10:45:41.099399+0000","last_clean":"2023-10-09T10:45:41.099399+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:45:41.099399+0000","last_undegraded":"2023-10-09T10:45:41.099399+0000","last_fullsized":"2023-10-09T10:45:41.099399+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":85,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.616173+0000","last_change":"2023-10-09T10:43:06.672114+0000","last_active":"2023-10-09T10:45:39.616173+0000","last_peered":"2023-10-09T10:45:39.616173+0000","last_clean":"2023-10-09T10:45:39.616173+0000","last_became_active":"2023-10-09T10:43:06.671827+0000","last_became_peered":"2023-10-09T10:43:06.671827+0000","last_unstale":"2023-10-09T10:45:39.616173+0000","last_undegraded":"2023-10-09T10:45:39.616173+0000","last_fullsized":"2023-10-09T10:45:39.616173+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:41.701409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"134'3691","reported_seq":5590,"reported_epoch":134,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:45:41.333684+0000","last_change":"2023-10-09T10:45:16.870647+0000","last_active":"2023-10-09T10:45:41.333684+0000","last_peered":"2023-10-09T10:45:41.333684+0000","last_clean":"2023-10-09T10:44:56.105580+0000","last_became_active":"2023-10-09T10:45:16.866250+0000","last_became_peered":"2023-10-09T10:45:16.866250+0000","last_unstale":"2023-10-09T10:45:41.333684+0000","last_undegraded":"2023-10-09T10:45:16.812712+0000","last_fullsized":"2023-10-09T10:45:16.747501+0000","mapping_epoch":122,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3691,"log_dups_size":0,"ondisk_log_size":3691,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175865856,"num_objects":2684,"num_object_clones":0,"num_object_copies":8052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1523,"num_objects_misplaced":897,"num_objects_unfound":0,"num_objects_dirty":2684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2684,"num_write_kb":171744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":457,"num_bytes_recovered":29949952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[1,2],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"0,1,3,5","objects":627},{"shards":"1,2","objects":1160},{"shards":"1,2,5,6","objects":1},{"shards":"1,2,6","objects":896}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":103,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592613+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:45:38.592613+0000","last_peered":"2023-10-09T10:45:38.592613+0000","last_clean":"2023-10-09T10:45:38.592613+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:45:38.592613+0000","last_undegraded":"2023-10-09T10:45:38.592613+0000","last_fullsized":"2023-10-09T10:45:38.592613+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":97,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592810+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:45:38.592810+0000","last_peered":"2023-10-09T10:45:38.592810+0000","last_clean":"2023-10-09T10:45:38.592810+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:45:38.592810+0000","last_undegraded":"2023-10-09T10:45:38.592810+0000","last_fullsized":"2023-10-09T10:45:38.592810+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"135'3766","reported_seq":3824,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T10:45:43.448675+0000","last_change":"2023-10-09T10:44:50.093642+0000","last_active":"2023-10-09T10:45:43.448675+0000","last_peered":"2023-10-09T10:45:43.448675+0000","last_clean":"2023-10-09T10:45:43.448675+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:45:43.448675+0000","last_undegraded":"2023-10-09T10:45:43.448675+0000","last_fullsized":"2023-10-09T10:45:43.448675+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3766,"log_dups_size":0,"ondisk_log_size":3766,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178454528,"num_objects":2723,"num_object_clones":0,"num_object_copies":8169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2723,"num_write_kb":174272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":99,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.590279+0000","last_change":"2023-10-09T10:43:06.677115+0000","last_active":"2023-10-09T10:45:38.590279+0000","last_peered":"2023-10-09T10:45:38.590279+0000","last_clean":"2023-10-09T10:45:38.590279+0000","last_became_active":"2023-10-09T10:43:06.676739+0000","last_became_peered":"2023-10-09T10:43:06.676739+0000","last_unstale":"2023-10-09T10:45:38.590279+0000","last_undegraded":"2023-10-09T10:45:38.590279+0000","last_fullsized":"2023-10-09T10:45:38.590279+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:02.520770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":116,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.100971+0000","last_change":"2023-10-09T10:43:35.487942+0000","last_active":"2023-10-09T10:45:41.100971+0000","last_peered":"2023-10-09T10:45:41.100971+0000","last_clean":"2023-10-09T10:45:41.100971+0000","last_became_active":"2023-10-09T10:43:35.487651+0000","last_became_peered":"2023-10-09T10:43:35.487651+0000","last_unstale":"2023-10-09T10:45:41.100971+0000","last_undegraded":"2023-10-09T10:45:41.100971+0000","last_fullsized":"2023-10-09T10:45:41.100971+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:51.882595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"133'3668","reported_seq":3720,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:40.848632+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:45:40.848632+0000","last_peered":"2023-10-09T10:45:40.848632+0000","last_clean":"2023-10-09T10:45:40.848632+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:45:40.848632+0000","last_undegraded":"2023-10-09T10:45:40.848632+0000","last_fullsized":"2023-10-09T10:45:40.848632+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3668,"log_dups_size":0,"ondisk_log_size":3668,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174325760,"num_objects":2660,"num_object_clones":0,"num_object_copies":7980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2660,"num_write_kb":170240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":68,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.097226+0000","last_change":"2023-10-09T10:45:15.769708+0000","last_active":"2023-10-09T10:45:41.097226+0000","last_peered":"2023-10-09T10:45:41.097226+0000","last_clean":"2023-10-09T10:45:41.097226+0000","last_became_active":"2023-10-09T10:45:15.769370+0000","last_became_peered":"2023-10-09T10:45:15.769370+0000","last_unstale":"2023-10-09T10:45:41.097226+0000","last_undegraded":"2023-10-09T10:45:41.097226+0000","last_fullsized":"2023-10-09T10:45:41.097226+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:33:21.951569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":97,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.585623+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:45:38.585623+0000","last_peered":"2023-10-09T10:45:38.585623+0000","last_clean":"2023-10-09T10:45:38.585623+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:45:38.585623+0000","last_undegraded":"2023-10-09T10:45:38.585623+0000","last_fullsized":"2023-10-09T10:45:38.585623+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"134'3771","reported_seq":3825,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.394299+0000","last_change":"2023-10-09T10:44:50.084697+0000","last_active":"2023-10-09T10:45:41.394299+0000","last_peered":"2023-10-09T10:45:41.394299+0000","last_clean":"2023-10-09T10:45:41.394299+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:45:41.394299+0000","last_undegraded":"2023-10-09T10:45:41.394299+0000","last_fullsized":"2023-10-09T10:45:41.394299+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3771,"log_dups_size":0,"ondisk_log_size":3771,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178028544,"num_objects":2717,"num_object_clones":0,"num_object_copies":8151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2717,"num_write_kb":173856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":43,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592730+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:45:38.592730+0000","last_peered":"2023-10-09T10:45:38.592730+0000","last_clean":"2023-10-09T10:45:38.592730+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:45:38.592730+0000","last_undegraded":"2023-10-09T10:45:38.592730+0000","last_fullsized":"2023-10-09T10:45:38.592730+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":68,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.099143+0000","last_change":"2023-10-09T10:45:15.766405+0000","last_active":"2023-10-09T10:45:41.099143+0000","last_peered":"2023-10-09T10:45:41.099143+0000","last_clean":"2023-10-09T10:45:41.099143+0000","last_became_active":"2023-10-09T10:45:15.766088+0000","last_became_peered":"2023-10-09T10:45:15.766088+0000","last_unstale":"2023-10-09T10:45:41.099143+0000","last_undegraded":"2023-10-09T10:45:41.099143+0000","last_fullsized":"2023-10-09T10:45:41.099143+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:07.504491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.15","version":"135'3856","reported_seq":3924,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T10:45:43.552240+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:45:43.552240+0000","last_peered":"2023-10-09T10:45:43.552240+0000","last_clean":"2023-10-09T10:45:43.552240+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:45:43.552240+0000","last_undegraded":"2023-10-09T10:45:43.552240+0000","last_fullsized":"2023-10-09T10:45:43.552240+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3856,"log_dups_size":0,"ondisk_log_size":3856,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183173120,"num_objects":2795,"num_object_clones":0,"num_object_copies":8385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2795,"num_write_kb":178880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":45,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.585242+0000","last_change":"2023-10-09T10:45:15.782117+0000","last_active":"2023-10-09T10:45:38.585242+0000","last_peered":"2023-10-09T10:45:38.585242+0000","last_clean":"2023-10-09T10:45:38.585242+0000","last_became_active":"2023-10-09T10:45:15.765683+0000","last_became_peered":"2023-10-09T10:45:15.765683+0000","last_unstale":"2023-10-09T10:45:38.585242+0000","last_undegraded":"2023-10-09T10:45:38.585242+0000","last_fullsized":"2023-10-09T10:45:38.585242+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:39.454280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":102,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.099219+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:45:41.099219+0000","last_peered":"2023-10-09T10:45:41.099219+0000","last_clean":"2023-10-09T10:45:41.099219+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:45:41.099219+0000","last_undegraded":"2023-10-09T10:45:41.099219+0000","last_fullsized":"2023-10-09T10:45:41.099219+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"133'5649","reported_seq":5702,"reported_epoch":133,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:45:40.928444+0000","last_change":"2023-10-09T10:45:16.985715+0000","last_active":"2023-10-09T10:45:40.928444+0000","last_peered":"2023-10-09T10:45:40.928444+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:45:16.949545+0000","last_became_peered":"2023-10-09T10:45:16.949545+0000","last_unstale":"2023-10-09T10:45:40.928444+0000","last_undegraded":"2023-10-09T10:45:16.782589+0000","last_fullsized":"2023-10-09T10:45:16.750475+0000","mapping_epoch":122,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5649,"log_dups_size":0,"ondisk_log_size":5649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:44.131701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370212864,"num_objects":5649,"num_object_clones":0,"num_object_copies":16947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4166,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5649,"num_write_kb":361536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"0,5,6","objects":4166},{"shards":"5,6","objects":1483}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":112,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.590786+0000","last_change":"2023-10-09T10:45:15.787875+0000","last_active":"2023-10-09T10:45:38.590786+0000","last_peered":"2023-10-09T10:45:38.590786+0000","last_clean":"2023-10-09T10:45:38.590786+0000","last_became_active":"2023-10-09T10:45:15.766410+0000","last_became_peered":"2023-10-09T10:45:15.766410+0000","last_unstale":"2023-10-09T10:45:38.590786+0000","last_undegraded":"2023-10-09T10:45:38.590786+0000","last_fullsized":"2023-10-09T10:45:38.590786+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:49.209544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":80,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.585261+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:45:38.585261+0000","last_peered":"2023-10-09T10:45:38.585261+0000","last_clean":"2023-10-09T10:45:38.585261+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:45:38.585261+0000","last_undegraded":"2023-10-09T10:45:38.585261+0000","last_fullsized":"2023-10-09T10:45:38.585261+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"135'5475","reported_seq":5521,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T10:45:43.504761+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:45:43.504761+0000","last_peered":"2023-10-09T10:45:43.504761+0000","last_clean":"2023-10-09T10:45:43.504761+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:45:43.504761+0000","last_undegraded":"2023-10-09T10:45:43.504761+0000","last_fullsized":"2023-10-09T10:45:43.504761+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5475,"log_dups_size":0,"ondisk_log_size":5475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358809600,"num_objects":5475,"num_object_clones":0,"num_object_copies":16425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5475,"num_write_kb":350400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":99,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.591522+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:45:38.591522+0000","last_peered":"2023-10-09T10:45:38.591522+0000","last_clean":"2023-10-09T10:45:38.591522+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:45:38.591522+0000","last_undegraded":"2023-10-09T10:45:38.591522+0000","last_fullsized":"2023-10-09T10:45:38.591522+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":124,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592380+0000","last_change":"2023-10-09T10:43:35.493278+0000","last_active":"2023-10-09T10:45:38.592380+0000","last_peered":"2023-10-09T10:45:38.592380+0000","last_clean":"2023-10-09T10:45:38.592380+0000","last_became_active":"2023-10-09T10:43:35.492915+0000","last_became_peered":"2023-10-09T10:43:35.492915+0000","last_unstale":"2023-10-09T10:45:38.592380+0000","last_undegraded":"2023-10-09T10:45:38.592380+0000","last_fullsized":"2023-10-09T10:45:38.592380+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:42.227364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"136'3886","reported_seq":3947,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T10:45:44.692917+0000","last_change":"2023-10-09T10:45:06.950897+0000","last_active":"2023-10-09T10:45:44.692917+0000","last_peered":"2023-10-09T10:45:44.692917+0000","last_clean":"2023-10-09T10:45:44.692917+0000","last_became_active":"2023-10-09T10:44:50.063797+0000","last_became_peered":"2023-10-09T10:44:50.063797+0000","last_unstale":"2023-10-09T10:45:44.692917+0000","last_undegraded":"2023-10-09T10:45:44.692917+0000","last_fullsized":"2023-10-09T10:45:44.692917+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3886,"log_dups_size":0,"ondisk_log_size":3886,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184909824,"num_objects":2821,"num_object_clones":0,"num_object_copies":8463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2821,"num_write_kb":180576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":54,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.585558+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:45:38.585558+0000","last_peered":"2023-10-09T10:45:38.585558+0000","last_clean":"2023-10-09T10:45:38.585558+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:45:38.585558+0000","last_undegraded":"2023-10-09T10:45:38.585558+0000","last_fullsized":"2023-10-09T10:45:38.585558+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":71,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.616502+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:45:39.616502+0000","last_peered":"2023-10-09T10:45:39.616502+0000","last_clean":"2023-10-09T10:45:39.616502+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:45:39.616502+0000","last_undegraded":"2023-10-09T10:45:39.616502+0000","last_fullsized":"2023-10-09T10:45:39.616502+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"136'3834","reported_seq":3897,"reported_epoch":136,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:45:44.682115+0000","last_change":"2023-10-09T10:45:16.848181+0000","last_active":"2023-10-09T10:45:44.682115+0000","last_peered":"2023-10-09T10:45:44.682115+0000","last_clean":"2023-10-09T10:45:14.284950+0000","last_became_active":"2023-10-09T10:45:16.845243+0000","last_became_peered":"2023-10-09T10:45:16.845243+0000","last_unstale":"2023-10-09T10:45:44.682115+0000","last_undegraded":"2023-10-09T10:45:16.764961+0000","last_fullsized":"2023-10-09T10:45:16.748120+0000","mapping_epoch":122,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3834,"log_dups_size":0,"ondisk_log_size":3834,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182976512,"num_objects":2792,"num_object_clones":0,"num_object_copies":8376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2003,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2792,"num_write_kb":178688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2,7","objects":2003},{"shards":"2,7","objects":789}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.584884+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:45:38.584884+0000","last_peered":"2023-10-09T10:45:38.584884+0000","last_clean":"2023-10-09T10:45:38.584884+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:45:38.584884+0000","last_undegraded":"2023-10-09T10:45:38.584884+0000","last_fullsized":"2023-10-09T10:45:38.584884+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":57,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.616642+0000","last_change":"2023-10-09T10:45:15.767409+0000","last_active":"2023-10-09T10:45:39.616642+0000","last_peered":"2023-10-09T10:45:39.616642+0000","last_clean":"2023-10-09T10:45:39.616642+0000","last_became_active":"2023-10-09T10:45:15.766857+0000","last_became_peered":"2023-10-09T10:45:15.766857+0000","last_unstale":"2023-10-09T10:45:39.616642+0000","last_undegraded":"2023-10-09T10:45:39.616642+0000","last_fullsized":"2023-10-09T10:45:39.616642+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:06.417379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":451,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.618058+0000","last_change":"2023-10-09T10:45:31.260184+0000","last_active":"2023-10-09T10:45:39.618058+0000","last_peered":"2023-10-09T10:45:39.618058+0000","last_clean":"2023-10-09T10:45:39.618058+0000","last_became_active":"2023-10-09T10:45:31.259865+0000","last_became_peered":"2023-10-09T10:45:31.259865+0000","last_unstale":"2023-10-09T10:45:39.618058+0000","last_undegraded":"2023-10-09T10:45:39.618058+0000","last_fullsized":"2023-10-09T10:45:39.618058+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.e","version":"133'5356","reported_seq":5399,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:40.947333+0000","last_change":"2023-10-09T10:44:27.574527+0000","last_active":"2023-10-09T10:45:40.947333+0000","last_peered":"2023-10-09T10:45:40.947333+0000","last_clean":"2023-10-09T10:45:40.947333+0000","last_became_active":"2023-10-09T10:44:27.574205+0000","last_became_peered":"2023-10-09T10:44:27.574205+0000","last_unstale":"2023-10-09T10:45:40.947333+0000","last_undegraded":"2023-10-09T10:45:40.947333+0000","last_fullsized":"2023-10-09T10:45:40.947333+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5356,"log_dups_size":0,"ondisk_log_size":5356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:54.868034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351010816,"num_objects":5356,"num_object_clones":0,"num_object_copies":16068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5356,"num_write_kb":342784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":113,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.101927+0000","last_change":"2023-10-09T10:45:15.789677+0000","last_active":"2023-10-09T10:45:41.101927+0000","last_peered":"2023-10-09T10:45:41.101927+0000","last_clean":"2023-10-09T10:45:41.101927+0000","last_became_active":"2023-10-09T10:45:15.775547+0000","last_became_peered":"2023-10-09T10:45:15.775547+0000","last_unstale":"2023-10-09T10:45:41.101927+0000","last_undegraded":"2023-10-09T10:45:41.101927+0000","last_fullsized":"2023-10-09T10:45:41.101927+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:13.878981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":97,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592524+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:45:38.592524+0000","last_peered":"2023-10-09T10:45:38.592524+0000","last_clean":"2023-10-09T10:45:38.592524+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:45:38.592524+0000","last_undegraded":"2023-10-09T10:45:38.592524+0000","last_fullsized":"2023-10-09T10:45:38.592524+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":449,"reported_epoch":133,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:45:39.615718+0000","last_change":"2023-10-09T10:45:31.258387+0000","last_active":"2023-10-09T10:45:39.615718+0000","last_peered":"2023-10-09T10:45:39.615718+0000","last_clean":"2023-10-09T10:45:13.733443+0000","last_became_active":"2023-10-09T10:45:31.258076+0000","last_became_peered":"2023-10-09T10:45:31.258076+0000","last_unstale":"2023-10-09T10:45:39.615718+0000","last_undegraded":"2023-10-09T10:45:31.234301+0000","last_fullsized":"2023-10-09T10:45:39.615718+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":1},{"shards":"1,6,7","objects":-1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.f","version":"136'5446","reported_seq":5493,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T10:45:44.710184+0000","last_change":"2023-10-09T10:44:27.574938+0000","last_active":"2023-10-09T10:45:44.710184+0000","last_peered":"2023-10-09T10:45:44.710184+0000","last_clean":"2023-10-09T10:45:44.710184+0000","last_became_active":"2023-10-09T10:44:27.574662+0000","last_became_peered":"2023-10-09T10:44:27.574662+0000","last_unstale":"2023-10-09T10:45:44.710184+0000","last_undegraded":"2023-10-09T10:45:44.710184+0000","last_fullsized":"2023-10-09T10:45:44.710184+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5446,"log_dups_size":0,"ondisk_log_size":5446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:35:19.975925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356909056,"num_objects":5446,"num_object_clones":0,"num_object_copies":16338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5446,"num_write_kb":348544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":72,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592775+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:45:38.592775+0000","last_peered":"2023-10-09T10:45:38.592775+0000","last_clean":"2023-10-09T10:45:38.592775+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:45:38.592775+0000","last_undegraded":"2023-10-09T10:45:38.592775+0000","last_fullsized":"2023-10-09T10:45:38.592775+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":114,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592641+0000","last_change":"2023-10-09T10:43:06.676367+0000","last_active":"2023-10-09T10:45:38.592641+0000","last_peered":"2023-10-09T10:45:38.592641+0000","last_clean":"2023-10-09T10:45:38.592641+0000","last_became_active":"2023-10-09T10:43:06.676017+0000","last_became_peered":"2023-10-09T10:43:06.676017+0000","last_unstale":"2023-10-09T10:45:38.592641+0000","last_undegraded":"2023-10-09T10:45:38.592641+0000","last_fullsized":"2023-10-09T10:45:38.592641+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:54.387617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"135'5497","reported_seq":5543,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T10:45:43.508452+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:45:43.508452+0000","last_peered":"2023-10-09T10:45:43.508452+0000","last_clean":"2023-10-09T10:45:43.508452+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:45:43.508452+0000","last_undegraded":"2023-10-09T10:45:43.508452+0000","last_fullsized":"2023-10-09T10:45:43.508452+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5497,"log_dups_size":0,"ondisk_log_size":5497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360251392,"num_objects":5497,"num_object_clones":0,"num_object_copies":16491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5497,"num_write_kb":351808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592381+0000","last_change":"2023-10-09T10:43:06.676577+0000","last_active":"2023-10-09T10:45:38.592381+0000","last_peered":"2023-10-09T10:45:38.592381+0000","last_clean":"2023-10-09T10:45:38.592381+0000","last_became_active":"2023-10-09T10:43:06.676165+0000","last_became_peered":"2023-10-09T10:43:06.676165+0000","last_unstale":"2023-10-09T10:45:38.592381+0000","last_undegraded":"2023-10-09T10:45:38.592381+0000","last_fullsized":"2023-10-09T10:45:38.592381+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:52.614985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":97,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592989+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:45:38.592989+0000","last_peered":"2023-10-09T10:45:38.592989+0000","last_clean":"2023-10-09T10:45:38.592989+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:45:38.592989+0000","last_undegraded":"2023-10-09T10:45:38.592989+0000","last_fullsized":"2023-10-09T10:45:38.592989+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":451,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.616005+0000","last_change":"2023-10-09T10:45:31.256030+0000","last_active":"2023-10-09T10:45:39.616005+0000","last_peered":"2023-10-09T10:45:39.616005+0000","last_clean":"2023-10-09T10:45:39.616005+0000","last_became_active":"2023-10-09T10:45:31.253175+0000","last_became_peered":"2023-10-09T10:45:31.253175+0000","last_unstale":"2023-10-09T10:45:39.616005+0000","last_undegraded":"2023-10-09T10:45:39.616005+0000","last_fullsized":"2023-10-09T10:45:39.616005+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.d","version":"133'5529","reported_seq":5582,"reported_epoch":133,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:45:40.956772+0000","last_change":"2023-10-09T10:45:16.933804+0000","last_active":"2023-10-09T10:45:40.956772+0000","last_peered":"2023-10-09T10:45:40.956772+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:45:16.923149+0000","last_became_peered":"2023-10-09T10:45:16.923149+0000","last_unstale":"2023-10-09T10:45:40.956772+0000","last_undegraded":"2023-10-09T10:45:16.782245+0000","last_fullsized":"2023-10-09T10:45:16.750546+0000","mapping_epoch":122,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5529,"log_dups_size":0,"ondisk_log_size":5529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:57.297705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362348544,"num_objects":5529,"num_object_clones":0,"num_object_copies":16587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3979,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5529,"num_write_kb":353856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1],"avail_no_missing":["5","0","1"],"object_location_counts":[{"shards":"0,1,5","objects":3979},{"shards":"1,5","objects":1550}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":104,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T10:45:42.085281+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:45:42.085281+0000","last_peered":"2023-10-09T10:45:42.085281+0000","last_clean":"2023-10-09T10:45:42.085281+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:45:42.085281+0000","last_undegraded":"2023-10-09T10:45:42.085281+0000","last_fullsized":"2023-10-09T10:45:42.085281+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":85,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.616842+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:45:39.616842+0000","last_peered":"2023-10-09T10:45:39.616842+0000","last_clean":"2023-10-09T10:45:39.616842+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:45:39.616842+0000","last_undegraded":"2023-10-09T10:45:39.616842+0000","last_fullsized":"2023-10-09T10:45:39.616842+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":451,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.616923+0000","last_change":"2023-10-09T10:45:31.264377+0000","last_active":"2023-10-09T10:45:39.616923+0000","last_peered":"2023-10-09T10:45:39.616923+0000","last_clean":"2023-10-09T10:45:39.616923+0000","last_became_active":"2023-10-09T10:45:31.262888+0000","last_became_peered":"2023-10-09T10:45:31.262888+0000","last_unstale":"2023-10-09T10:45:39.616923+0000","last_undegraded":"2023-10-09T10:45:39.616923+0000","last_fullsized":"2023-10-09T10:45:39.616923+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":119,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.617362+0000","last_change":"2023-10-09T10:45:15.764916+0000","last_active":"2023-10-09T10:45:39.617362+0000","last_peered":"2023-10-09T10:45:39.617362+0000","last_clean":"2023-10-09T10:45:39.617362+0000","last_became_active":"2023-10-09T10:45:15.764607+0000","last_became_peered":"2023-10-09T10:45:15.764607+0000","last_unstale":"2023-10-09T10:45:39.617362+0000","last_undegraded":"2023-10-09T10:45:39.617362+0000","last_fullsized":"2023-10-09T10:45:39.617362+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:21.140378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":115,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.585687+0000","last_change":"2023-10-09T10:43:35.486524+0000","last_active":"2023-10-09T10:45:38.585687+0000","last_peered":"2023-10-09T10:45:38.585687+0000","last_clean":"2023-10-09T10:45:38.585687+0000","last_became_active":"2023-10-09T10:43:35.486152+0000","last_became_peered":"2023-10-09T10:43:35.486152+0000","last_unstale":"2023-10-09T10:45:38.585687+0000","last_undegraded":"2023-10-09T10:45:38.585687+0000","last_fullsized":"2023-10-09T10:45:38.585687+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:47.200522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.616260+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:45:39.616260+0000","last_peered":"2023-10-09T10:45:39.616260+0000","last_clean":"2023-10-09T10:45:39.616260+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:45:39.616260+0000","last_undegraded":"2023-10-09T10:45:39.616260+0000","last_fullsized":"2023-10-09T10:45:39.616260+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"135'3781","reported_seq":7757,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T10:45:43.474657+0000","last_change":"2023-10-09T10:45:39.594616+0000","last_active":"2023-10-09T10:45:43.474657+0000","last_peered":"2023-10-09T10:45:43.474657+0000","last_clean":"2023-10-09T10:45:43.474657+0000","last_became_active":"2023-10-09T10:45:39.590952+0000","last_became_peered":"2023-10-09T10:45:39.590952+0000","last_unstale":"2023-10-09T10:45:43.474657+0000","last_undegraded":"2023-10-09T10:45:43.474657+0000","last_fullsized":"2023-10-09T10:45:43.474657+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3781,"log_dups_size":0,"ondisk_log_size":3781,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179208192,"num_objects":2734,"num_object_clones":0,"num_object_copies":8202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2734,"num_write_kb":175008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":450,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.618399+0000","last_change":"2023-10-09T10:45:31.262336+0000","last_active":"2023-10-09T10:45:39.618399+0000","last_peered":"2023-10-09T10:45:39.618399+0000","last_clean":"2023-10-09T10:45:39.618399+0000","last_became_active":"2023-10-09T10:45:31.259459+0000","last_became_peered":"2023-10-09T10:45:31.259459+0000","last_unstale":"2023-10-09T10:45:39.618399+0000","last_undegraded":"2023-10-09T10:45:39.618399+0000","last_fullsized":"2023-10-09T10:45:39.618399+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"135'3860","reported_seq":3918,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T10:45:43.512813+0000","last_change":"2023-10-09T10:44:50.088013+0000","last_active":"2023-10-09T10:45:43.512813+0000","last_peered":"2023-10-09T10:45:43.512813+0000","last_clean":"2023-10-09T10:45:43.512813+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:45:43.512813+0000","last_undegraded":"2023-10-09T10:45:43.512813+0000","last_fullsized":"2023-10-09T10:45:43.512813+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3860,"log_dups_size":0,"ondisk_log_size":3860,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184614912,"num_objects":2817,"num_object_clones":0,"num_object_copies":8451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2817,"num_write_kb":180288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":102,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.584796+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:45:38.584796+0000","last_peered":"2023-10-09T10:45:38.584796+0000","last_clean":"2023-10-09T10:45:38.584796+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:45:38.584796+0000","last_undegraded":"2023-10-09T10:45:38.584796+0000","last_fullsized":"2023-10-09T10:45:38.584796+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":108,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592618+0000","last_change":"2023-10-09T10:45:15.770080+0000","last_active":"2023-10-09T10:45:38.592618+0000","last_peered":"2023-10-09T10:45:38.592618+0000","last_clean":"2023-10-09T10:45:38.592618+0000","last_became_active":"2023-10-09T10:45:15.769640+0000","last_became_peered":"2023-10-09T10:45:15.769640+0000","last_unstale":"2023-10-09T10:45:38.592618+0000","last_undegraded":"2023-10-09T10:45:38.592618+0000","last_fullsized":"2023-10-09T10:45:38.592618+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:17.992596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":100,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.617432+0000","last_change":"2023-10-09T10:43:35.489332+0000","last_active":"2023-10-09T10:45:39.617432+0000","last_peered":"2023-10-09T10:45:39.617432+0000","last_clean":"2023-10-09T10:45:39.617432+0000","last_became_active":"2023-10-09T10:43:35.489044+0000","last_became_peered":"2023-10-09T10:43:35.489044+0000","last_unstale":"2023-10-09T10:45:39.617432+0000","last_undegraded":"2023-10-09T10:45:39.617432+0000","last_fullsized":"2023-10-09T10:45:39.617432+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:42.168915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":451,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.616097+0000","last_change":"2023-10-09T10:45:31.253491+0000","last_active":"2023-10-09T10:45:39.616097+0000","last_peered":"2023-10-09T10:45:39.616097+0000","last_clean":"2023-10-09T10:45:39.616097+0000","last_became_active":"2023-10-09T10:45:31.252758+0000","last_became_peered":"2023-10-09T10:45:31.252758+0000","last_unstale":"2023-10-09T10:45:39.616097+0000","last_undegraded":"2023-10-09T10:45:39.616097+0000","last_fullsized":"2023-10-09T10:45:39.616097+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"135'3836","reported_seq":3907,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T10:45:43.525439+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:45:43.525439+0000","last_peered":"2023-10-09T10:45:43.525439+0000","last_clean":"2023-10-09T10:45:43.525439+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:45:43.525439+0000","last_undegraded":"2023-10-09T10:45:43.525439+0000","last_fullsized":"2023-10-09T10:45:43.525439+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3836,"log_dups_size":0,"ondisk_log_size":3836,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181862400,"num_objects":2775,"num_object_clones":0,"num_object_copies":8325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2775,"num_write_kb":177600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":132,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T10:45:42.088526+0000","last_change":"2023-10-09T10:45:15.768379+0000","last_active":"2023-10-09T10:45:42.088526+0000","last_peered":"2023-10-09T10:45:42.088526+0000","last_clean":"2023-10-09T10:45:42.088526+0000","last_became_active":"2023-10-09T10:45:15.767987+0000","last_became_peered":"2023-10-09T10:45:15.767987+0000","last_unstale":"2023-10-09T10:45:42.088526+0000","last_undegraded":"2023-10-09T10:45:42.088526+0000","last_fullsized":"2023-10-09T10:45:42.088526+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:55.314341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":85,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.616005+0000","last_change":"2023-10-09T10:43:06.672862+0000","last_active":"2023-10-09T10:45:39.616005+0000","last_peered":"2023-10-09T10:45:39.616005+0000","last_clean":"2023-10-09T10:45:39.616005+0000","last_became_active":"2023-10-09T10:43:06.672572+0000","last_became_peered":"2023-10-09T10:43:06.672572+0000","last_unstale":"2023-10-09T10:45:39.616005+0000","last_undegraded":"2023-10-09T10:45:39.616005+0000","last_fullsized":"2023-10-09T10:45:39.616005+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:23.743910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":114,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.616775+0000","last_change":"2023-10-09T10:45:15.773125+0000","last_active":"2023-10-09T10:45:39.616775+0000","last_peered":"2023-10-09T10:45:39.616775+0000","last_clean":"2023-10-09T10:45:39.616775+0000","last_became_active":"2023-10-09T10:45:15.772652+0000","last_became_peered":"2023-10-09T10:45:15.772652+0000","last_unstale":"2023-10-09T10:45:39.616775+0000","last_undegraded":"2023-10-09T10:45:39.616775+0000","last_fullsized":"2023-10-09T10:45:39.616775+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:23:38.484481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"133'3709","reported_seq":3765,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:40.928290+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:45:40.928290+0000","last_peered":"2023-10-09T10:45:40.928290+0000","last_clean":"2023-10-09T10:45:40.928290+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:45:40.928290+0000","last_undegraded":"2023-10-09T10:45:40.928290+0000","last_fullsized":"2023-10-09T10:45:40.928290+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3709,"log_dups_size":0,"ondisk_log_size":3709,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177012736,"num_objects":2701,"num_object_clones":0,"num_object_copies":8103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2701,"num_write_kb":172864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":102,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.584623+0000","last_change":"2023-10-09T10:43:06.673931+0000","last_active":"2023-10-09T10:45:38.584623+0000","last_peered":"2023-10-09T10:45:38.584623+0000","last_clean":"2023-10-09T10:45:38.584623+0000","last_became_active":"2023-10-09T10:43:06.673605+0000","last_became_peered":"2023-10-09T10:43:06.673605+0000","last_unstale":"2023-10-09T10:45:38.584623+0000","last_undegraded":"2023-10-09T10:45:38.584623+0000","last_fullsized":"2023-10-09T10:45:38.584623+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:19.088672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":451,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.615811+0000","last_change":"2023-10-09T10:45:31.256839+0000","last_active":"2023-10-09T10:45:39.615811+0000","last_peered":"2023-10-09T10:45:39.615811+0000","last_clean":"2023-10-09T10:45:39.615811+0000","last_became_active":"2023-10-09T10:45:31.256514+0000","last_became_peered":"2023-10-09T10:45:31.256514+0000","last_unstale":"2023-10-09T10:45:39.615811+0000","last_undegraded":"2023-10-09T10:45:39.615811+0000","last_fullsized":"2023-10-09T10:45:39.615811+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":62,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.097053+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:45:41.097053+0000","last_peered":"2023-10-09T10:45:41.097053+0000","last_clean":"2023-10-09T10:45:41.097053+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:45:41.097053+0000","last_undegraded":"2023-10-09T10:45:41.097053+0000","last_fullsized":"2023-10-09T10:45:41.097053+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":30,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592517+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:45:38.592517+0000","last_peered":"2023-10-09T10:45:38.592517+0000","last_clean":"2023-10-09T10:45:38.592517+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:45:38.592517+0000","last_undegraded":"2023-10-09T10:45:38.592517+0000","last_fullsized":"2023-10-09T10:45:38.592517+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":451,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.616304+0000","last_change":"2023-10-09T10:45:31.263832+0000","last_active":"2023-10-09T10:45:39.616304+0000","last_peered":"2023-10-09T10:45:39.616304+0000","last_clean":"2023-10-09T10:45:39.616304+0000","last_became_active":"2023-10-09T10:45:31.263496+0000","last_became_peered":"2023-10-09T10:45:31.263496+0000","last_unstale":"2023-10-09T10:45:39.616304+0000","last_undegraded":"2023-10-09T10:45:39.616304+0000","last_fullsized":"2023-10-09T10:45:39.616304+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":115,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.616053+0000","last_change":"2023-10-09T10:45:15.763912+0000","last_active":"2023-10-09T10:45:39.616053+0000","last_peered":"2023-10-09T10:45:39.616053+0000","last_clean":"2023-10-09T10:45:39.616053+0000","last_became_active":"2023-10-09T10:45:15.763634+0000","last_became_peered":"2023-10-09T10:45:15.763634+0000","last_unstale":"2023-10-09T10:45:39.616053+0000","last_undegraded":"2023-10-09T10:45:39.616053+0000","last_fullsized":"2023-10-09T10:45:39.616053+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:42:50.962688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":93,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.096878+0000","last_change":"2023-10-09T10:45:15.762810+0000","last_active":"2023-10-09T10:45:41.096878+0000","last_peered":"2023-10-09T10:45:41.096878+0000","last_clean":"2023-10-09T10:45:41.096878+0000","last_became_active":"2023-10-09T10:45:15.762469+0000","last_became_peered":"2023-10-09T10:45:15.762469+0000","last_unstale":"2023-10-09T10:45:41.096878+0000","last_undegraded":"2023-10-09T10:45:41.096878+0000","last_fullsized":"2023-10-09T10:45:41.096878+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:09.451806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":76,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.585118+0000","last_change":"2023-10-09T10:45:15.768399+0000","last_active":"2023-10-09T10:45:38.585118+0000","last_peered":"2023-10-09T10:45:38.585118+0000","last_clean":"2023-10-09T10:45:38.585118+0000","last_became_active":"2023-10-09T10:45:15.768067+0000","last_became_peered":"2023-10-09T10:45:15.768067+0000","last_unstale":"2023-10-09T10:45:38.585118+0000","last_undegraded":"2023-10-09T10:45:38.585118+0000","last_fullsized":"2023-10-09T10:45:38.585118+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:43.467271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.4","version":"134'3787","reported_seq":3891,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.389950+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:45:41.389950+0000","last_peered":"2023-10-09T10:45:41.389950+0000","last_clean":"2023-10-09T10:45:41.389950+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:45:41.389950+0000","last_undegraded":"2023-10-09T10:45:41.389950+0000","last_fullsized":"2023-10-09T10:45:41.389950+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":3787,"log_dups_size":0,"ondisk_log_size":3787,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177143808,"num_objects":2703,"num_object_clones":0,"num_object_copies":8109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":757,"num_write_kb":48448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":451,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.616928+0000","last_change":"2023-10-09T10:45:31.261246+0000","last_active":"2023-10-09T10:45:39.616928+0000","last_peered":"2023-10-09T10:45:39.616928+0000","last_clean":"2023-10-09T10:45:39.616928+0000","last_became_active":"2023-10-09T10:45:31.260910+0000","last_became_peered":"2023-10-09T10:45:31.260910+0000","last_unstale":"2023-10-09T10:45:39.616928+0000","last_undegraded":"2023-10-09T10:45:39.616928+0000","last_fullsized":"2023-10-09T10:45:39.616928+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.590778+0000","last_change":"2023-10-09T10:45:15.766600+0000","last_active":"2023-10-09T10:45:38.590778+0000","last_peered":"2023-10-09T10:45:38.590778+0000","last_clean":"2023-10-09T10:45:38.590778+0000","last_became_active":"2023-10-09T10:45:15.766192+0000","last_became_peered":"2023-10-09T10:45:15.766192+0000","last_unstale":"2023-10-09T10:45:38.590778+0000","last_undegraded":"2023-10-09T10:45:38.590778+0000","last_fullsized":"2023-10-09T10:45:38.590778+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:01:13.866396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":85,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.617511+0000","last_change":"2023-10-09T10:43:06.671356+0000","last_active":"2023-10-09T10:45:39.617511+0000","last_peered":"2023-10-09T10:45:39.617511+0000","last_clean":"2023-10-09T10:45:39.617511+0000","last_became_active":"2023-10-09T10:43:06.671040+0000","last_became_peered":"2023-10-09T10:43:06.671040+0000","last_unstale":"2023-10-09T10:45:39.617511+0000","last_undegraded":"2023-10-09T10:45:39.617511+0000","last_fullsized":"2023-10-09T10:45:39.617511+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:47.601448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.584738+0000","last_change":"2023-10-09T10:45:15.769154+0000","last_active":"2023-10-09T10:45:38.584738+0000","last_peered":"2023-10-09T10:45:38.584738+0000","last_clean":"2023-10-09T10:45:38.584738+0000","last_became_active":"2023-10-09T10:45:15.768827+0000","last_became_peered":"2023-10-09T10:45:15.768827+0000","last_unstale":"2023-10-09T10:45:38.584738+0000","last_undegraded":"2023-10-09T10:45:38.584738+0000","last_fullsized":"2023-10-09T10:45:38.584738+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:40.970054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.1","version":"134'3702","reported_seq":3756,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.414603+0000","last_change":"2023-10-09T10:44:50.065999+0000","last_active":"2023-10-09T10:45:41.414603+0000","last_peered":"2023-10-09T10:45:41.414603+0000","last_clean":"2023-10-09T10:45:41.414603+0000","last_became_active":"2023-10-09T10:44:50.064940+0000","last_became_peered":"2023-10-09T10:44:50.064940+0000","last_unstale":"2023-10-09T10:45:41.414603+0000","last_undegraded":"2023-10-09T10:45:41.414603+0000","last_fullsized":"2023-10-09T10:45:41.414603+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3702,"log_dups_size":0,"ondisk_log_size":3702,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176586752,"num_objects":2694,"num_object_clones":0,"num_object_copies":8082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2694,"num_write_kb":172448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":451,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.617942+0000","last_change":"2023-10-09T10:45:31.259135+0000","last_active":"2023-10-09T10:45:39.617942+0000","last_peered":"2023-10-09T10:45:39.617942+0000","last_clean":"2023-10-09T10:45:39.617942+0000","last_became_active":"2023-10-09T10:45:31.258933+0000","last_became_peered":"2023-10-09T10:45:31.258933+0000","last_unstale":"2023-10-09T10:45:39.617942+0000","last_undegraded":"2023-10-09T10:45:39.617942+0000","last_fullsized":"2023-10-09T10:45:39.617942+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":114,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T10:45:42.086723+0000","last_change":"2023-10-09T10:45:15.778104+0000","last_active":"2023-10-09T10:45:42.086723+0000","last_peered":"2023-10-09T10:45:42.086723+0000","last_clean":"2023-10-09T10:45:42.086723+0000","last_became_active":"2023-10-09T10:45:15.766880+0000","last_became_peered":"2023-10-09T10:45:15.766880+0000","last_unstale":"2023-10-09T10:45:42.086723+0000","last_undegraded":"2023-10-09T10:45:42.086723+0000","last_fullsized":"2023-10-09T10:45:42.086723+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:59:39.885831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":70,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.617029+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:45:39.617029+0000","last_peered":"2023-10-09T10:45:39.617029+0000","last_clean":"2023-10-09T10:45:39.617029+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:45:39.617029+0000","last_undegraded":"2023-10-09T10:45:39.617029+0000","last_fullsized":"2023-10-09T10:45:39.617029+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":107,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.094509+0000","last_change":"2023-10-09T10:45:15.768090+0000","last_active":"2023-10-09T10:45:41.094509+0000","last_peered":"2023-10-09T10:45:41.094509+0000","last_clean":"2023-10-09T10:45:41.094509+0000","last_became_active":"2023-10-09T10:45:15.767438+0000","last_became_peered":"2023-10-09T10:45:15.767438+0000","last_unstale":"2023-10-09T10:45:41.094509+0000","last_undegraded":"2023-10-09T10:45:41.094509+0000","last_fullsized":"2023-10-09T10:45:41.094509+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:54.095543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"133'3748","reported_seq":3814,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:40.940433+0000","last_change":"2023-10-09T10:45:39.513186+0000","last_active":"2023-10-09T10:45:40.940433+0000","last_peered":"2023-10-09T10:45:40.940433+0000","last_clean":"2023-10-09T10:45:40.940433+0000","last_became_active":"2023-10-09T10:44:50.060284+0000","last_became_peered":"2023-10-09T10:44:50.060284+0000","last_unstale":"2023-10-09T10:45:40.940433+0000","last_undegraded":"2023-10-09T10:45:40.940433+0000","last_fullsized":"2023-10-09T10:45:40.940433+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3748,"log_dups_size":0,"ondisk_log_size":3748,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177700864,"num_objects":2711,"num_object_clones":0,"num_object_copies":8133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2711,"num_write_kb":173536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":449,"reported_epoch":133,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:45:39.617112+0000","last_change":"2023-10-09T10:45:31.259173+0000","last_active":"2023-10-09T10:45:39.617112+0000","last_peered":"2023-10-09T10:45:39.617112+0000","last_clean":"2023-10-09T10:45:13.733443+0000","last_became_active":"2023-10-09T10:45:31.258999+0000","last_became_peered":"2023-10-09T10:45:31.258999+0000","last_unstale":"2023-10-09T10:45:39.617112+0000","last_undegraded":"2023-10-09T10:45:31.238260+0000","last_fullsized":"2023-10-09T10:45:39.617112+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":1},{"shards":"1,6,7","objects":-1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":83,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T10:45:42.088600+0000","last_change":"2023-10-09T10:45:07.620938+0000","last_active":"2023-10-09T10:45:42.088600+0000","last_peered":"2023-10-09T10:45:42.088600+0000","last_clean":"2023-10-09T10:45:42.088600+0000","last_became_active":"2023-10-09T10:45:07.620627+0000","last_became_peered":"2023-10-09T10:45:07.620627+0000","last_unstale":"2023-10-09T10:45:42.088600+0000","last_undegraded":"2023-10-09T10:45:42.088600+0000","last_fullsized":"2023-10-09T10:45:42.088600+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:13.415622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":49,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:39.617107+0000","last_change":"2023-10-09T10:45:15.783236+0000","last_active":"2023-10-09T10:45:39.617107+0000","last_peered":"2023-10-09T10:45:39.617107+0000","last_clean":"2023-10-09T10:45:39.617107+0000","last_became_active":"2023-10-09T10:45:15.770551+0000","last_became_peered":"2023-10-09T10:45:15.770551+0000","last_unstale":"2023-10-09T10:45:39.617107+0000","last_undegraded":"2023-10-09T10:45:39.617107+0000","last_fullsized":"2023-10-09T10:45:39.617107+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:15:37.696584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":80,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592324+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:45:38.592324+0000","last_peered":"2023-10-09T10:45:38.592324+0000","last_clean":"2023-10-09T10:45:38.592324+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:45:38.592324+0000","last_undegraded":"2023-10-09T10:45:38.592324+0000","last_fullsized":"2023-10-09T10:45:38.592324+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"133'3695","reported_seq":3825,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-09T10:45:40.869918+0000","last_change":"2023-10-09T10:45:05.770329+0000","last_active":"2023-10-09T10:45:40.869918+0000","last_peered":"2023-10-09T10:45:40.869918+0000","last_clean":"2023-10-09T10:45:40.869918+0000","last_became_active":"2023-10-09T10:44:50.061195+0000","last_became_peered":"2023-10-09T10:44:50.061195+0000","last_unstale":"2023-10-09T10:45:40.869918+0000","last_undegraded":"2023-10-09T10:45:40.869918+0000","last_fullsized":"2023-10-09T10:45:40.869918+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":3695,"log_dups_size":0,"ondisk_log_size":3695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:02.077286+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175570944,"num_objects":2679,"num_object_clones":0,"num_object_copies":8037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1050,"num_write_kb":67200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":34,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592709+0000","last_change":"2023-10-09T10:45:15.770124+0000","last_active":"2023-10-09T10:45:38.592709+0000","last_peered":"2023-10-09T10:45:38.592709+0000","last_clean":"2023-10-09T10:45:38.592709+0000","last_became_active":"2023-10-09T10:45:15.769787+0000","last_became_peered":"2023-10-09T10:45:15.769787+0000","last_unstale":"2023-10-09T10:45:38.592709+0000","last_undegraded":"2023-10-09T10:45:38.592709+0000","last_fullsized":"2023-10-09T10:45:38.592709+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:09.577196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":122,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T10:45:42.088992+0000","last_change":"2023-10-09T10:43:35.489655+0000","last_active":"2023-10-09T10:45:42.088992+0000","last_peered":"2023-10-09T10:45:42.088992+0000","last_clean":"2023-10-09T10:45:42.088992+0000","last_became_active":"2023-10-09T10:43:35.489149+0000","last_became_peered":"2023-10-09T10:43:35.489149+0000","last_unstale":"2023-10-09T10:45:42.088992+0000","last_undegraded":"2023-10-09T10:45:42.088992+0000","last_fullsized":"2023-10-09T10:45:42.088992+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:30.598962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"136'3823","reported_seq":5441,"reported_epoch":136,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:45:44.724357+0000","last_change":"2023-10-09T10:45:37.514594+0000","last_active":"2023-10-09T10:45:44.724357+0000","last_peered":"2023-10-09T10:45:44.724357+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:45:16.845016+0000","last_became_peered":"2023-10-09T10:45:16.845016+0000","last_unstale":"2023-10-09T10:45:44.724357+0000","last_undegraded":"2023-10-09T10:45:44.724357+0000","last_fullsized":"2023-10-09T10:45:16.747355+0000","mapping_epoch":122,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3823,"log_dups_size":0,"ondisk_log_size":3823,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182255616,"num_objects":2781,"num_object_clones":0,"num_object_copies":8343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1155,"num_objects_unfound":0,"num_objects_dirty":2781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2781,"num_write_kb":177984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":777,"num_bytes_recovered":50921472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2,7","objects":1155},{"shards":"2,7","objects":1626}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":100,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.585029+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:45:38.585029+0000","last_peered":"2023-10-09T10:45:38.585029+0000","last_clean":"2023-10-09T10:45:38.585029+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:45:38.585029+0000","last_undegraded":"2023-10-09T10:45:38.585029+0000","last_fullsized":"2023-10-09T10:45:38.585029+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":114,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592787+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:45:38.592787+0000","last_peered":"2023-10-09T10:45:38.592787+0000","last_clean":"2023-10-09T10:45:38.592787+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:45:38.592787+0000","last_undegraded":"2023-10-09T10:45:38.592787+0000","last_fullsized":"2023-10-09T10:45:38.592787+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"136'3829","reported_seq":3890,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T10:45:44.673201+0000","last_change":"2023-10-09T10:45:12.912569+0000","last_active":"2023-10-09T10:45:44.673201+0000","last_peered":"2023-10-09T10:45:44.673201+0000","last_clean":"2023-10-09T10:45:44.673201+0000","last_became_active":"2023-10-09T10:44:50.076612+0000","last_became_peered":"2023-10-09T10:44:50.076612+0000","last_unstale":"2023-10-09T10:45:44.673201+0000","last_undegraded":"2023-10-09T10:45:44.673201+0000","last_fullsized":"2023-10-09T10:45:44.673201+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":3829,"log_dups_size":0,"ondisk_log_size":3829,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181174272,"num_objects":2765,"num_object_clones":0,"num_object_copies":8295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2765,"num_write_kb":176928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":83,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.585758+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:45:38.585758+0000","last_peered":"2023-10-09T10:45:38.585758+0000","last_clean":"2023-10-09T10:45:38.585758+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:45:38.585758+0000","last_undegraded":"2023-10-09T10:45:38.585758+0000","last_fullsized":"2023-10-09T10:45:38.585758+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":112,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.097132+0000","last_change":"2023-10-09T10:45:15.769793+0000","last_active":"2023-10-09T10:45:41.097132+0000","last_peered":"2023-10-09T10:45:41.097132+0000","last_clean":"2023-10-09T10:45:41.097132+0000","last_became_active":"2023-10-09T10:45:15.769546+0000","last_became_peered":"2023-10-09T10:45:15.769546+0000","last_unstale":"2023-10-09T10:45:41.097132+0000","last_undegraded":"2023-10-09T10:45:41.097132+0000","last_fullsized":"2023-10-09T10:45:41.097132+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:32.371324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":109,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.098984+0000","last_change":"2023-10-09T10:43:35.485790+0000","last_active":"2023-10-09T10:45:41.098984+0000","last_peered":"2023-10-09T10:45:41.098984+0000","last_clean":"2023-10-09T10:45:41.098984+0000","last_became_active":"2023-10-09T10:43:35.485436+0000","last_became_peered":"2023-10-09T10:43:35.485436+0000","last_unstale":"2023-10-09T10:45:41.098984+0000","last_undegraded":"2023-10-09T10:45:41.098984+0000","last_fullsized":"2023-10-09T10:45:41.098984+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:45:20.897767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":132,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T10:45:41.096619+0000","last_change":"2023-10-09T10:45:15.761078+0000","last_active":"2023-10-09T10:45:41.096619+0000","last_peered":"2023-10-09T10:45:41.096619+0000","last_clean":"2023-10-09T10:45:41.096619+0000","last_became_active":"2023-10-09T10:45:15.760721+0000","last_became_peered":"2023-10-09T10:45:15.760721+0000","last_unstale":"2023-10-09T10:45:41.096619+0000","last_undegraded":"2023-10-09T10:45:41.096619+0000","last_fullsized":"2023-10-09T10:45:41.096619+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:33.535124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":51,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.585150+0000","last_change":"2023-10-09T10:45:15.774365+0000","last_active":"2023-10-09T10:45:38.585150+0000","last_peered":"2023-10-09T10:45:38.585150+0000","last_clean":"2023-10-09T10:45:38.585150+0000","last_became_active":"2023-10-09T10:45:15.763429+0000","last_became_peered":"2023-10-09T10:45:15.763429+0000","last_unstale":"2023-10-09T10:45:38.585150+0000","last_undegraded":"2023-10-09T10:45:38.585150+0000","last_fullsized":"2023-10-09T10:45:38.585150+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:29.201640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":97,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-09T10:45:38.592301+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:45:38.592301+0000","last_peered":"2023-10-09T10:45:38.592301+0000","last_clean":"2023-10-09T10:45:38.592301+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:45:38.592301+0000","last_undegraded":"2023-10-09T10:45:38.592301+0000","last_fullsized":"2023-10-09T10:45:38.592301+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":5753503744,"num_objects":87792,"num_object_clones":0,"num_object_copies":263376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11671,"num_objects_misplaced":2052,"num_objects_unfound":0,"num_objects_dirty":87792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":82544,"num_write_kb":5282784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8514,"num_bytes_recovered":557973504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17976786944,"data_stored":17976786944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108606,"ondisk_log_size":108606,"up":78,"acting":72,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":576,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332229,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2535588,"kb_used_data":2234204,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91209564,"statfs":{"total":95995035648,"available":93398593536,"internally_reserved":0,"allocated":2287824896,"data_stored":2287320180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:45:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2609999999999999,"5min":0.68100000000000005,"15min":0.61299999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.4349999999999996,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.876,"5min":0.64700000000000002,"15min":0.64300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.9260000000000002,"5min":2.9260000000000002,"15min":2.9260000000000002},"last":0.39000000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:44:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.55000000000000004,"15min":0.54400000000000004},"min":{"1min":0.39200000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":2.077,"5min":2.077,"15min":2.077},"last":1.8380000000000001},{"interface":"front","average":{"1min":0.84599999999999997,"5min":0.59399999999999997,"15min":0.55200000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.4939999999999998,"5min":6.4939999999999998,"15min":6.4939999999999998},"last":0.29699999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:45:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":0.69699999999999995,"15min":0.57299999999999995},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.2690000000000001,"5min":6.2690000000000001,"15min":6.2690000000000001},"last":0.96699999999999997},{"interface":"front","average":{"1min":0.84999999999999998,"5min":0.58999999999999997,"15min":0.54700000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.157,"5min":2.157,"15min":2.157},"last":0.379}]},{"osd":3,"last update":"Mon Oct 9 10:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":0.80900000000000005,"15min":0.66000000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":8.1639999999999997,"5min":8.1639999999999997,"15min":8.1639999999999997},"last":0.5},{"interface":"front","average":{"1min":1.9419999999999999,"5min":0.86799999999999999,"15min":0.68899999999999995},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":12.451000000000001,"5min":12.451000000000001,"15min":12.451000000000001},"last":2.1520000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:45:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.51600000000000001,"15min":0.48499999999999999},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.1600000000000001,"5min":3.1600000000000001,"15min":3.1600000000000001},"last":0.433},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.54200000000000004,"15min":0.49399999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.1699999999999999,"5min":3.1699999999999999,"15min":3.1699999999999999},"last":0.89500000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:45:16 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.55000000000000004,"15min":0.51700000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":2.1989999999999998,"5min":2.1989999999999998,"15min":2.1989999999999998},"last":1.659},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.58699999999999997,"15min":0.55200000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":2.7549999999999999,"5min":2.7549999999999999,"15min":2.7549999999999999},"last":0.47299999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:45:16 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.628,"15min":0.57399999999999995},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.1880000000000002,"5min":3.1880000000000002,"15min":3.1880000000000002},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.58399999999999996,"15min":0.55200000000000005},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":0.93500000000000005}]}]},{"osd":6,"up_from":44,"seq":188978561160,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2109788,"kb_used_data":1804308,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91635364,"statfs":{"total":95995035648,"available":93834612736,"internally_reserved":0,"allocated":1847611392,"data_stored":1847115276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:45:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":0.66300000000000003,"15min":0.628},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.4219999999999997,"5min":5.4219999999999997,"15min":5.4219999999999997},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.63800000000000001,"15min":0.61799999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.7250000000000001,"5min":2.7250000000000001,"15min":2.7250000000000001},"last":0.44}]},{"osd":1,"last update":"Mon Oct 9 10:44:41 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.58599999999999997,"15min":0.58899999999999997},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":3.0190000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:45:07 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.54900000000000004,"15min":0.53900000000000003},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":2.0960000000000001,"5min":2.0960000000000001,"15min":2.0960000000000001},"last":2.984},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.64900000000000002,"15min":0.58999999999999997},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.7349999999999999,"5min":3.7349999999999999,"15min":3.7349999999999999},"last":0.41899999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:45:38 2023","interfaces":[{"interface":"back","average":{"1min":1.419,"5min":0.71399999999999997,"15min":0.59599999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.7030000000000003,"5min":4.7030000000000003,"15min":4.7030000000000003},"last":0.622},{"interface":"front","average":{"1min":1.2729999999999999,"5min":0.69199999999999995,"15min":0.59499999999999997},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.867,"5min":2.867,"15min":2.867},"last":3.0459999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:45:00 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.46899999999999997,"15min":0.44900000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.55,"5min":1.55,"15min":1.55},"last":1.3759999999999999},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.52200000000000002,"15min":0.496},"min":{"1min":0.29999999999999999,"5min":0.29899999999999999,"15min":0.29699999999999999},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.48699999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:45:00 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.53000000000000003,"15min":0.49099999999999999},"min":{"1min":0.34999999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.762,"5min":1.762,"15min":1.762},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.56999999999999995,"15min":0.53500000000000003},"min":{"1min":0.33900000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.84399999999999997},"last":2.3599999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:45:21 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.59399999999999997,"15min":0.54500000000000004},"min":{"1min":0.39300000000000002,"5min":0.33900000000000002,"15min":0.32900000000000001},"max":{"1min":2.0870000000000002,"5min":2.0870000000000002,"15min":2.0870000000000002},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.63800000000000001,"15min":0.59199999999999997},"min":{"1min":0.42199999999999999,"5min":0.34100000000000003,"15min":0.34000000000000002},"max":{"1min":1.921,"5min":1.921,"15min":1.921},"last":0.64800000000000002}]}]},{"osd":1,"up_from":85,"seq":365072220199,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1991532,"kb_used_data":1915684,"kb_used_omap":9,"kb_used_meta":75830,"kb_avail":91753620,"statfs":{"total":95995035648,"available":93955706880,"internally_reserved":0,"allocated":1961660416,"data_stored":1961041970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":77650921},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:44:42 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.38500000000000001,"15min":0.371},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.7949999999999999,"5min":2.7949999999999999,"15min":2.7949999999999999},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.42899999999999999,"15min":0.40899999999999997},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.6789999999999998,"5min":3.6789999999999998,"15min":3.6789999999999998},"last":0.248}]},{"osd":2,"last update":"Mon Oct 9 10:45:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":0.55300000000000005,"15min":0.42899999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":8.6859999999999999,"5min":8.6859999999999999,"15min":8.6859999999999999},"last":1.0129999999999999},{"interface":"front","average":{"1min":1.7470000000000001,"5min":0.65700000000000003,"15min":0.47499999999999998},"min":{"1min":0.26200000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":14.137,"5min":14.137,"15min":14.137},"last":0.93200000000000005}]},{"osd":3,"last update":"Mon Oct 9 10:45:37 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":0.92000000000000004,"15min":0.621},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":14.404999999999999,"5min":14.404999999999999,"15min":14.404999999999999},"last":0.69099999999999995},{"interface":"front","average":{"1min":2.4279999999999999,"5min":0.76100000000000001,"15min":0.48299999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":7.8620000000000001,"5min":7.8620000000000001,"15min":7.8620000000000001},"last":0.29399999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:44:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.58199999999999996,"15min":0.58899999999999997},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":1.123},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.61399999999999999,"15min":0.61799999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.9330000000000001,"5min":1.9330000000000001,"15min":1.9330000000000001},"last":1.998}]},{"osd":5,"last update":"Mon Oct 9 10:44:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.61899999999999999,"15min":0.625},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":1.915},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.627,"15min":0.64000000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":2.6379999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:44:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.621,"15min":0.628},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.129,"5min":1.129,"15min":1.129},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.63,"15min":0.64100000000000001},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":4.3659999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:44:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.63700000000000001,"15min":0.64500000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":1.1579999999999999},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.64700000000000002,"15min":0.64600000000000002},"min":{"1min":0.42599999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.8939999999999999,"5min":1.8939999999999999,"15min":1.8939999999999999},"last":1.474}]}]},{"osd":0,"up_from":0,"seq":343597383733,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":90,"seq":386547056674,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3006968,"kb_used_data":2894896,"kb_used_omap":81,"kb_used_meta":111982,"kb_avail":90738184,"statfs":{"total":95995035648,"available":92915900416,"internally_reserved":0,"allocated":2964373504,"data_stored":2963749190,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":114669786},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.39500000000000002,"15min":0.34499999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":3.8370000000000002,"5min":3.8370000000000002,"15min":3.8370000000000002},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.46100000000000002,"15min":0.40300000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.0049999999999999,"5min":4.0049999999999999,"15min":4.0049999999999999},"last":0.23499999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":0.54800000000000004,"15min":0.44400000000000001},"min":{"1min":0.27900000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.1909999999999998,"5min":5.1909999999999998,"15min":5.1909999999999998},"last":5.8360000000000003},{"interface":"front","average":{"1min":0.877,"5min":0.46200000000000002,"15min":0.39300000000000002},"min":{"1min":0.23799999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.2039999999999997,"5min":5.2039999999999997,"15min":5.2039999999999997},"last":0.76800000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:45:37 2023","interfaces":[{"interface":"back","average":{"1min":1.958,"5min":0.72599999999999998,"15min":0.52000000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":12.076000000000001,"5min":12.076000000000001,"15min":12.076000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":2.8199999999999998,"5min":0.91600000000000004,"15min":0.59899999999999998},"min":{"1min":0.29299999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":10.249000000000001,"5min":10.249000000000001,"15min":10.249000000000001},"last":0.20999999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":0.68200000000000005,"15min":0.621},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":8.1029999999999998,"5min":8.1029999999999998,"15min":8.1029999999999998},"last":0.78600000000000003},{"interface":"front","average":{"1min":1.216,"5min":0.71799999999999997,"15min":0.63500000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":8.1259999999999994,"5min":8.1259999999999994,"15min":8.1259999999999994},"last":0.30199999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":0.754,"15min":0.64800000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.3659999999999997,"5min":8.3659999999999997,"15min":8.3659999999999997},"last":0.29299999999999998},{"interface":"front","average":{"1min":1.286,"5min":0.73399999999999999,"15min":0.64200000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":8.3979999999999997,"5min":8.3979999999999997,"15min":8.3979999999999997},"last":0.91200000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.62,"15min":0.58999999999999997},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":3.3730000000000002,"5min":3.3730000000000002,"15min":3.3730000000000002},"last":0.373},{"interface":"front","average":{"1min":1.1100000000000001,"5min":0.68999999999999995,"15min":0.62},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":5.2199999999999998,"5min":5.2199999999999998,"15min":5.2199999999999998},"last":0.311}]},{"osd":7,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.629,"15min":0.59799999999999998},"min":{"1min":0.433,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":3.3500000000000001,"5min":3.3500000000000001,"15min":3.3500000000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.1419999999999999,"5min":0.69499999999999995,"15min":0.621},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":5.2690000000000001,"5min":5.2690000000000001,"15min":5.2690000000000001},"last":5.9779999999999998}]}]},{"osd":3,"up_from":95,"seq":408021893148,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3720248,"kb_used_data":3620592,"kb_used_omap":7,"kb_used_meta":99640,"kb_avail":90024904,"statfs":{"total":95995035648,"available":92185501696,"internally_reserved":0,"allocated":3707486208,"data_stored":3706860501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":102031976},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":0.51200000000000001,"15min":0.38400000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":4.3159999999999998,"5min":4.3159999999999998,"15min":4.3159999999999998},"last":0.38100000000000001},{"interface":"front","average":{"1min":2.1070000000000002,"5min":0.747,"15min":0.52000000000000002},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":7.0510000000000002,"5min":7.0510000000000002,"15min":7.0510000000000002},"last":0.22800000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":3.782,"5min":1.02,"15min":0.56000000000000005},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":14.707000000000001,"5min":14.707000000000001,"15min":14.707000000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.6659999999999999,"5min":0.63200000000000001,"15min":0.46000000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":6.4870000000000001,"5min":6.4870000000000001,"15min":6.4870000000000001},"last":0.32400000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":3.4089999999999998,"5min":0.97499999999999998,"15min":0.56899999999999995},"min":{"1min":0.31,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":13.803000000000001,"5min":13.803000000000001,"15min":13.803000000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":2.7000000000000002,"5min":0.85599999999999998,"15min":0.54900000000000004},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.0579999999999998,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":0.26900000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":0.752,"15min":0.63200000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.798,"5min":3.798,"15min":3.798},"last":0.504},{"interface":"front","average":{"1min":1.6890000000000001,"5min":0.82399999999999995,"15min":0.68000000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":5.4269999999999996,"5min":5.4269999999999996,"15min":5.4269999999999996},"last":0.44900000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":0.81399999999999995,"15min":0.66200000000000003},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.4580000000000002,"5min":5.4580000000000002,"15min":5.4580000000000002},"last":0.41799999999999998},{"interface":"front","average":{"1min":3.1200000000000001,"5min":1.0740000000000001,"15min":0.73299999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":13.845000000000001,"5min":13.845000000000001,"15min":13.845000000000001},"last":0.45700000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0640000000000001,"5min":0.88300000000000001,"15min":0.68600000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.4789999999999992,"5min":8.4789999999999992,"15min":8.4789999999999992},"last":0.496},{"interface":"front","average":{"1min":3.2919999999999998,"5min":1.1379999999999999,"15min":0.77900000000000003},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":13.891999999999999,"5min":13.891999999999999,"15min":13.891999999999999},"last":0.48199999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":3.214,"5min":1.1160000000000001,"15min":0.76600000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":13.869999999999999,"5min":13.869999999999999,"15min":13.869999999999999},"last":0.434},{"interface":"front","average":{"1min":1.611,"5min":0.78100000000000003,"15min":0.64300000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.831,"5min":3.831,"15min":3.831},"last":0.52400000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019023,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2015328,"kb_used_data":1713944,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91729824,"statfs":{"total":95995035648,"available":93931339776,"internally_reserved":0,"allocated":1755078656,"data_stored":1754577745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":0.66100000000000003,"15min":0.59199999999999997},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.8550000000000004,"5min":6.8550000000000004,"15min":6.8550000000000004},"last":2.9420000000000002},{"interface":"front","average":{"1min":1.6919999999999999,"5min":0.79200000000000004,"15min":0.65400000000000003},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.4240000000000004,"5min":6.4240000000000004,"15min":6.4240000000000004},"last":0.36399999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":0.82199999999999995,"15min":0.64200000000000002},"min":{"1min":0.35699999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.907,"5min":6.907,"15min":6.907},"last":2.903},{"interface":"front","average":{"1min":1.3120000000000001,"5min":0.79000000000000004,"15min":0.67800000000000005},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.2010000000000001,"5min":4.8120000000000003,"15min":4.8120000000000003},"last":3.2010000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:45:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":0.71799999999999997,"15min":0.62},"min":{"1min":0.46100000000000002,"5min":0.373,"15min":0.373},"max":{"1min":4.6340000000000003,"5min":4.6340000000000003,"15min":4.6340000000000003},"last":7.5449999999999999},{"interface":"front","average":{"1min":1.1319999999999999,"5min":0.67600000000000005,"15min":0.59999999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.2939999999999996,"5min":6.2939999999999996,"15min":6.2939999999999996},"last":0.26300000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":0.86499999999999999,"15min":0.76300000000000001},"min":{"1min":0.40799999999999997,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.7750000000000004,"5min":4.7750000000000004,"15min":4.7750000000000004},"last":3.2200000000000002},{"interface":"front","average":{"1min":1.052,"5min":0.68000000000000005,"15min":0.61799999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.2320000000000002,"5min":3.2320000000000002,"15min":3.2320000000000002},"last":0.75800000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":0.66700000000000004,"15min":0.51000000000000001},"min":{"1min":0.28999999999999998,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":5.5410000000000004,"5min":5.5410000000000004,"15min":5.5410000000000004},"last":3.4649999999999999},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.60599999999999998,"15min":0.49099999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.5150000000000001,"5min":2.5150000000000001,"15min":2.5150000000000001},"last":0.29099999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:45:04 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.51000000000000001,"15min":0.47299999999999998},"min":{"1min":0.313,"5min":0.27700000000000002,"15min":0.246},"max":{"1min":0.75800000000000001,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":2.9820000000000002},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.60399999999999998,"15min":0.56699999999999995},"min":{"1min":0.34300000000000003,"5min":0.314,"15min":0.314},"max":{"1min":2.1320000000000001,"5min":2.1320000000000001,"15min":2.1320000000000001},"last":3.181}]},{"osd":7,"last update":"Mon Oct 9 10:45:14 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.628,"15min":0.58399999999999996},"min":{"1min":0.36799999999999999,"5min":0.35299999999999998,"15min":0.308},"max":{"1min":2.7589999999999999,"5min":2.7589999999999999,"15min":2.7589999999999999},"last":3.1499999999999999},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.622,"15min":0.55800000000000005},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.456,"5min":2.456,"15min":2.456},"last":0.78700000000000003}]}]},{"osd":5,"up_from":37,"seq":158913790092,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2513740,"kb_used_data":2212356,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91231412,"statfs":{"total":95995035648,"available":93420965888,"internally_reserved":0,"allocated":2265452544,"data_stored":2264967534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:45:32 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.68899999999999995,"15min":0.628},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":4.8380000000000001,"5min":4.8380000000000001,"15min":4.8380000000000001},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.131,"5min":0.749,"15min":0.69599999999999995},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.8650000000000002,"5min":4.8650000000000002,"15min":4.8650000000000002},"last":0.36899999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:44:40 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.59699999999999998,"15min":0.60399999999999998},"min":{"1min":0.38500000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.69999999999999996,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.64100000000000001,"15min":0.60799999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.5110000000000001,"5min":5.5110000000000001,"15min":5.5110000000000001},"last":0.32000000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:45:06 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.65400000000000003,"15min":0.59899999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.8490000000000002,"5min":3.8490000000000002,"15min":3.8490000000000002},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.59199999999999997,"15min":0.57999999999999996},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.823,"5min":1.823,"15min":1.823},"last":0.432}]},{"osd":3,"last update":"Mon Oct 9 10:45:36 2023","interfaces":[{"interface":"back","average":{"1min":1.5649999999999999,"5min":0.89200000000000002,"15min":0.78000000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.0069999999999997,"5min":4.0069999999999997,"15min":4.0069999999999997},"last":0.93799999999999994},{"interface":"front","average":{"1min":1.5249999999999999,"5min":0.79500000000000004,"15min":0.67300000000000004},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.0940000000000003,"5min":6.0940000000000003,"15min":6.0940000000000003},"last":1.5700000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:44:40 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.48299999999999998,"15min":0.437},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.25},"max":{"1min":0.73199999999999998,"5min":0.75600000000000001,"15min":2.4649999999999999},"last":0.252},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.52500000000000002,"15min":0.46300000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28799999999999998},"max":{"1min":1.923,"5min":1.923,"15min":1.98},"last":0.33900000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:45:03 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.52900000000000003,"15min":0.46700000000000003},"min":{"1min":0.317,"5min":0.317,"15min":0.28100000000000003},"max":{"1min":1.274,"5min":1.274,"15min":2.048},"last":0.84799999999999998},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.58599999999999997,"15min":0.505},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.30299999999999999},"max":{"1min":1.746,"5min":1.746,"15min":1.746},"last":0.58999999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:45:18 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.622,"15min":0.55500000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32400000000000001},"max":{"1min":4.0510000000000002,"5min":4.0510000000000002,"15min":4.0510000000000002},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.66700000000000004,"15min":0.56999999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":2.6869999999999998,"5min":2.6869999999999998,"15min":2.6869999999999998},"last":0.40600000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1112801280,"data_stored":1112801280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1960247296,"data_stored":1960247296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2989948928,"data_stored":2989948928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3706126336,"data_stored":3706126336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1763115008,"data_stored":1763115008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2264137728,"data_stored":2264137728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1875247104,"data_stored":1875247104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2305163264,"data_stored":2305163264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:45:46.734 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:45:46.734 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 21 2023-10-09T10:45:46.756 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:45:46.801 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:46.871 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:46.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:46.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:46.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:47.074 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:47.131 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:47.203 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:47.209 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:47 smithi125 ceph-mon[195507]: from='client.34800 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:45:47.210 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:47 smithi125 ceph-mon[200197]: from='client.34800 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:45:47.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:47.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:47.311 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:47.316 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 75 16 92160 92144 76.7556 60.8125 0.0125139 0.0130233 2023-10-09T10:45:47.376 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:47.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:47 smithi155 ceph-mon[184854]: from='client.34800 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:45:47.979 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:47.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:47.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:48.087 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:48.139 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:48.170 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:48.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:48.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:48.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:48.316 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 76 16 93243 93227 76.6362 67.6875 0.0174789 0.0130429 2023-10-09T10:45:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:48 smithi125 ceph-mon[195507]: pgmap v274: 109 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 102 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 761 op/s; 11673/269868 objects degraded (4.325%); 2052/269868 objects misplaced (0.760%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T10:45:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:48 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 11673/269868 objects degraded (4.325%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:48 smithi125 ceph-mon[200197]: pgmap v274: 109 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 102 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 761 op/s; 11673/269868 objects degraded (4.325%); 2052/269868 objects misplaced (0.760%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T10:45:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:48 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 11673/269868 objects degraded (4.325%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:48.349 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:48.386 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:45:48.386 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:45:48.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.387 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:93286 6.19 6:9f30f200:::benchmark_data_smithi125_244951_object93285:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-10-09T10:45:48.387 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:48.368778+0000", 2023-10-09T10:45:48.387 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0061652449999999998, 2023-10-09T10:45:48.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.006207242, 2023-10-09T10:45:48.387 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:48.387 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:45:48.387 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:48.387 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:48.387 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:48.387 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 93286 2023-10-09T10:45:48.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.388 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:48.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:48.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368778+0000", 2023-10-09T10:45:48.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:48.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:48.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368778+0000", 2023-10-09T10:45:48.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:48.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:48.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368776+0000", 2023-10-09T10:45:48.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:45:48.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:48.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368814+0000", 2023-10-09T10:45:48.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8504999999999998e-05 2023-10-09T10:45:48.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:48.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368829+0000", 2023-10-09T10:45:48.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.456e-05 2023-10-09T10:45:48.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:48.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368839+0000", 2023-10-09T10:45:48.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4739999999999994e-06 2023-10-09T10:45:48.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:48.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368891+0000", 2023-10-09T10:45:48.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2441999999999997e-05 2023-10-09T10:45:48.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:48.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.369037+0000", 2023-10-09T10:45:48.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014592699999999999 2023-10-09T10:45:48.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:45:48.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.369203+0000", 2023-10-09T10:45:48.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000166231 2023-10-09T10:45:48.391 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:48.391 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:48.391 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:48.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.392 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:93289 6.8 6:10752442:::benchmark_data_smithi125_244951_object93288:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-10-09T10:45:48.392 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:48.368851+0000", 2023-10-09T10:45:48.392 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.006092236, 2023-10-09T10:45:48.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0061887699999999997, 2023-10-09T10:45:48.392 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:48.392 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:45:48.392 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:48.392 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:48.392 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:48.392 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 93289 2023-10-09T10:45:48.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.393 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:48.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.393 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:48.393 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368851+0000", 2023-10-09T10:45:48.393 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:48.393 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.393 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:48.393 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368851+0000", 2023-10-09T10:45:48.393 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:48.393 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:48.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368850+0000", 2023-10-09T10:45:48.394 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:48.394 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.394 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:48.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368870+0000", 2023-10-09T10:45:48.394 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9916e-05 2023-10-09T10:45:48.394 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.394 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:48.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368876+0000", 2023-10-09T10:45:48.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3790000000000003e-06 2023-10-09T10:45:48.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.395 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:48.395 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368882+0000", 2023-10-09T10:45:48.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7400000000000001e-06 2023-10-09T10:45:48.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.395 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:48.395 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368940+0000", 2023-10-09T10:45:48.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7593000000000002e-05 2023-10-09T10:45:48.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:48.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.369070+0000", 2023-10-09T10:45:48.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013054000000000001 2023-10-09T10:45:48.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:45:48.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.369225+0000", 2023-10-09T10:45:48.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000155017 2023-10-09T10:45:48.396 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:48.396 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:48.396 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:48.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.397 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:93290 6.8 6:1052b16f:::benchmark_data_smithi125_244951_object93289:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-10-09T10:45:48.397 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:48.368897+0000", 2023-10-09T10:45:48.397 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0060453970000000001, 2023-10-09T10:45:48.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0061867720000000001, 2023-10-09T10:45:48.397 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:48.397 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:45:48.397 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:45:48.397 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:45:48.397 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:45:48.397 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 93290 2023-10-09T10:45:48.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.398 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:48.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.398 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:48.398 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368897+0000", 2023-10-09T10:45:48.398 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:48.398 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.398 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:48.398 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368897+0000", 2023-10-09T10:45:48.398 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:48.398 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.399 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:48.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368897+0000", 2023-10-09T10:45:48.399 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:48.399 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.399 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.399 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:48.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368915+0000", 2023-10-09T10:45:48.399 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8406999999999998e-05 2023-10-09T10:45:48.399 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.399 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.399 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:48.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368922+0000", 2023-10-09T10:45:48.399 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1850000000000001e-06 2023-10-09T10:45:48.400 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.400 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.400 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:48.400 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.368926+0000", 2023-10-09T10:45:48.400 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6979999999999997e-06 2023-10-09T10:45:48.400 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.400 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.400 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:48.400 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.373059+0000", 2023-10-09T10:45:48.400 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041324700000000001 2023-10-09T10:45:48.400 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.400 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.401 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:48.401 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.373202+0000", 2023-10-09T10:45:48.401 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014342200000000001 2023-10-09T10:45:48.401 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.401 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.401 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:45:48.401 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.373352+0000", 2023-10-09T10:45:48.401 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000149567 2023-10-09T10:45:48.401 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:48.401 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:48.401 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:48.401 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.401 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.402 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44570.0:93288 6.e e136/98 6:7e7dfa29:::benchmark_data_smithi125_244951_object93287:head v 136'5819, mlcod=136'5817)", 2023-10-09T10:45:48.402 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:48.369433+0000", 2023-10-09T10:45:48.402 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.005509964, 2023-10-09T10:45:48.402 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0056946870000000004, 2023-10-09T10:45:48.402 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:48.402 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:45:48.402 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:48.402 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.402 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:48.402 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.369433+0000", 2023-10-09T10:45:48.402 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:48.402 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.403 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.403 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:48.403 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.369433+0000", 2023-10-09T10:45:48.403 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:48.403 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.403 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.403 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:48.403 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.369432+0000", 2023-10-09T10:45:48.403 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:45:48.403 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.403 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.403 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:48.404 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.369630+0000", 2023-10-09T10:45:48.404 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019767600000000001 2023-10-09T10:45:48.404 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.404 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.404 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:48.404 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.369640+0000", 2023-10-09T10:45:48.404 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9250000000000007e-06 2023-10-09T10:45:48.404 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.404 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.404 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:48.404 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.369645+0000", 2023-10-09T10:45:48.404 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.164e-06 2023-10-09T10:45:48.405 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.405 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.405 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:48.405 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.369686+0000", 2023-10-09T10:45:48.405 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0519000000000003e-05 2023-10-09T10:45:48.405 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.405 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.405 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:48.405 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.369700+0000", 2023-10-09T10:45:48.405 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4809e-05 2023-10-09T10:45:48.405 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:48.405 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:48.405 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:48.406 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.406 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.406 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44570.0:93292 6.0 e136/132 6:068e7832:::benchmark_data_smithi125_244951_object93291:head v 136'3942, mlcod=136'3940)", 2023-10-09T10:45:48.406 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:48.372202+0000", 2023-10-09T10:45:48.406 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0027409689999999998, 2023-10-09T10:45:48.406 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029604010000000001, 2023-10-09T10:45:48.406 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:45:48.406 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:45:48.406 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:45:48.406 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.406 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:45:48.406 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.372202+0000", 2023-10-09T10:45:48.407 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:48.407 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.407 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.407 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:45:48.407 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.372202+0000", 2023-10-09T10:45:48.407 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:45:48.407 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.407 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.407 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:45:48.407 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.372201+0000", 2023-10-09T10:45:48.407 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:45:48.407 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.408 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.408 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:45:48.408 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.372230+0000", 2023-10-09T10:45:48.408 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9320999999999999e-05 2023-10-09T10:45:48.408 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.408 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.408 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:45:48.408 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.372242+0000", 2023-10-09T10:45:48.408 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1216999999999999e-05 2023-10-09T10:45:48.408 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.408 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.408 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:45:48.408 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.372249+0000", 2023-10-09T10:45:48.409 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7400000000000004e-06 2023-10-09T10:45:48.409 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.409 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.409 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:45:48.409 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.372325+0000", 2023-10-09T10:45:48.409 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5964000000000007e-05 2023-10-09T10:45:48.409 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:45:48.409 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:45:48.409 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:45:48.409 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:48.372344+0000", 2023-10-09T10:45:48.409 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8756000000000001e-05 2023-10-09T10:45:48.409 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:48.410 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:45:48.410 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:48.410 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:45:48.410 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:45:48.410 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 5 2023-10-09T10:45:48.410 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:45:48.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:48.411 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:48.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:48.432 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:48 smithi155 ceph-mon[184854]: pgmap v274: 109 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 102 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 761 op/s; 11673/269868 objects degraded (4.325%); 2052/269868 objects misplaced (0.760%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T10:45:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:48 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 11673/269868 objects degraded (4.325%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:48.512 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:48.536 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:48.574 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:48.688 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:45:48.759 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:48.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:48.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:48.792 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:45:48.868 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:48.903 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:48.926 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:48.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:48.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:49.031 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:49.062 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:49.083 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:49.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:49.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:49.190 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:49.239 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:49.316 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 77 16 94129 94113 76.3601 55.375 0.0228482 0.0130913 2023-10-09T10:45:49.317 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:49.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:49.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:49.424 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:49.468 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:49.498 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:49.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:49.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:49.603 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:49.644 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:49.667 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:49.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:49.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:49.740 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:49.771 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:49.794 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:49.843 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:50.191 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:50.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:50.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:50.297 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:50.316 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 78 16 95127 95111 76.1808 62.375 0.015495 0.0131219 2023-10-09T10:45:50.353 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:50.411 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:50 smithi125 ceph-mon[195507]: pgmap v275: 109 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 102 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 762 op/s; 11673/274371 objects degraded (4.254%); 2052/274371 objects misplaced (0.748%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T10:45:50.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:50 smithi125 ceph-mon[200197]: pgmap v275: 109 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 102 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 762 op/s; 11673/274371 objects degraded (4.254%); 2052/274371 objects misplaced (0.748%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T10:45:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:50 smithi155 ceph-mon[184854]: pgmap v275: 109 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 102 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 762 op/s; 11673/274371 objects degraded (4.254%); 2052/274371 objects misplaced (0.748%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T10:45:50.949 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:50.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:50.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:51.057 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:45:51.137 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:51.179 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 1 pg_num to 21 2023-10-09T10:45:51.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1600151321' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T10:45:51.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:51 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T10:45:51.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1600151321' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T10:45:51.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:51 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T10:45:51.260 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:45:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:45:51] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.43.0" 2023-10-09T10:45:51.317 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 79 15 96014 95999 75.9188 55.5 0.0353312 0.0131677 2023-10-09T10:45:51.335 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:45:51.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:51.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:45:51.441 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:51.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1600151321' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T10:45:51.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:51 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T10:45:51.492 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:51.826 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:51.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:51.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:51.939 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:51.971 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:51.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:52.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:52.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:52.050 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:45:52.050 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:45:52.050 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:45:52.051 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:45:52.051 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:45:52.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:52.182 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:52.209 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:52.209 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:45:52.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:52.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:52 smithi155 ceph-mon[184854]: pgmap v276: 109 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 102 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 58 MiB/s wr, 935 op/s; 11673/282747 objects degraded (4.128%); 1757/282747 objects misplaced (0.621%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T10:45:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:52 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-10-09T10:45:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:52 smithi155 ceph-mon[184854]: osdmap e137: 8 total, 8 up, 7 in 2023-10-09T10:45:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:52 smithi155 ceph-mon[184854]: failed to encode map e137 with expected crc 2023-10-09T10:45:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:52 smithi155 ceph-mon[184854]: failed to encode map e137 with expected crc 2023-10-09T10:45:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:52 smithi155 ceph-mon[184854]: failed to encode map e137 with expected crc 2023-10-09T10:45:52.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:52 smithi155 ceph-mon[184854]: failed to encode map e137 with expected crc 2023-10-09T10:45:52.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:52 smithi155 ceph-mon[184854]: failed to encode map e137 with expected crc 2023-10-09T10:45:52.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:52 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3485867613' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:52.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:52 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:52.314 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:52.321 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:45:52.320234+0000 min lat: 0.00102371 max lat: 2.05736 avg lat: 0.0132047 2023-10-09T10:45:52.321 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:45:52.321 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 80 16 96955 96939 75.7019 58.75 0.0234666 0.0132047 2023-10-09T10:45:52.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[195507]: pgmap v276: 109 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 102 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 58 MiB/s wr, 935 op/s; 11673/282747 objects degraded (4.128%); 1757/282747 objects misplaced (0.621%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T10:45:52.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-10-09T10:45:52.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[195507]: osdmap e137: 8 total, 8 up, 7 in 2023-10-09T10:45:52.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[195507]: failed to encode map e137 with expected crc 2023-10-09T10:45:52.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[195507]: failed to encode map e137 with expected crc 2023-10-09T10:45:52.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[195507]: failed to encode map e137 with expected crc 2023-10-09T10:45:52.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[195507]: failed to encode map e137 with expected crc 2023-10-09T10:45:52.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[195507]: failed to encode map e137 with expected crc 2023-10-09T10:45:52.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3485867613' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:52.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:52.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[200197]: pgmap v276: 109 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 102 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 58 MiB/s wr, 935 op/s; 11673/282747 objects degraded (4.128%); 1757/282747 objects misplaced (0.621%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T10:45:52.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-10-09T10:45:52.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[200197]: osdmap e137: 8 total, 8 up, 7 in 2023-10-09T10:45:52.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[200197]: failed to encode map e137 with expected crc 2023-10-09T10:45:52.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[200197]: failed to encode map e137 with expected crc 2023-10-09T10:45:52.402 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[200197]: failed to encode map e137 with expected crc 2023-10-09T10:45:52.402 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[200197]: failed to encode map e137 with expected crc 2023-10-09T10:45:52.402 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[200197]: failed to encode map e137 with expected crc 2023-10-09T10:45:52.402 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3485867613' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:52.402 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:52 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:45:52.408 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:52.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:52.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:52.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:52.534 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:45:52.534 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:45:52.534 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:45:52.552 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:45:52.582 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:45:52.603 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:52.616 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:45:52.669 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:45:52.674 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:52.708 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:45:52.733 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:45:52.733 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:45:52.734 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:45:52.734 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:45:52.734 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:45:52.734 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:45:52.734 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:45:52.734 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:45:53.121 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:53.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:53.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:53.138 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:45:53.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:53.307 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:53.307 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[195507]: failed to encode map e138 with expected crc 2023-10-09T10:45:53.307 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:45:53.307 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[195507]: osdmap e138: 8 total, 8 up, 7 in 2023-10-09T10:45:53.307 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[195507]: failed to encode map e138 with expected crc 2023-10-09T10:45:53.307 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2244672543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:53.307 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 11673/282747 objects degraded (4.128%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:53.307 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-10-09T10:45:53.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:53.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[200197]: failed to encode map e138 with expected crc 2023-10-09T10:45:53.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:45:53.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[200197]: osdmap e138: 8 total, 8 up, 7 in 2023-10-09T10:45:53.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[200197]: failed to encode map e138 with expected crc 2023-10-09T10:45:53.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2244672543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:53.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 11673/282747 objects degraded (4.128%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:53.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:53 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-10-09T10:45:53.312 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:53.319 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 81 16 97945 97929 75.5311 61.875 0.0110363 0.0132347 2023-10-09T10:45:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:53 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:45:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:53 smithi155 ceph-mon[184854]: failed to encode map e138 with expected crc 2023-10-09T10:45:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:53 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:45:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:53 smithi155 ceph-mon[184854]: osdmap e138: 8 total, 8 up, 7 in 2023-10-09T10:45:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:53 smithi155 ceph-mon[184854]: failed to encode map e138 with expected crc 2023-10-09T10:45:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:53 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2244672543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:45:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:53 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 11673/282747 objects degraded (4.128%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:53.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:53 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-10-09T10:45:53.651 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:53.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:53.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:53.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:53.822 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:54.321 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 82 16 99055 99039 75.4552 69.375 0.0197453 0.0132483 2023-10-09T10:45:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:54 smithi155 ceph-mon[184854]: pgmap v279: 109 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 102 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 55 MiB/s wr, 882 op/s; 11673/284649 objects degraded (4.101%); 1757/284649 objects misplaced (0.617%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T10:45:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:54 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-10-09T10:45:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:54 smithi155 ceph-mon[184854]: osdmap e139: 8 total, 8 up, 7 in 2023-10-09T10:45:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:54 smithi155 ceph-mon[184854]: failed to encode map e139 with expected crc 2023-10-09T10:45:54.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:54 smithi125 ceph-mon[195507]: pgmap v279: 109 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 102 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 55 MiB/s wr, 882 op/s; 11673/284649 objects degraded (4.101%); 1757/284649 objects misplaced (0.617%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T10:45:54.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:54 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-10-09T10:45:54.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:54 smithi125 ceph-mon[195507]: osdmap e139: 8 total, 8 up, 7 in 2023-10-09T10:45:54.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:54 smithi125 ceph-mon[195507]: failed to encode map e139 with expected crc 2023-10-09T10:45:54.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:54 smithi125 ceph-mon[200197]: pgmap v279: 109 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 102 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 55 MiB/s wr, 882 op/s; 11673/284649 objects degraded (4.101%); 1757/284649 objects misplaced (0.617%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T10:45:54.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:54 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-10-09T10:45:54.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:54 smithi125 ceph-mon[200197]: osdmap e139: 8 total, 8 up, 7 in 2023-10-09T10:45:54.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:54 smithi125 ceph-mon[200197]: failed to encode map e139 with expected crc 2023-10-09T10:45:55.078 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:45:55.194 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:55.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:55.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:55.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:55.323 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 83 16 100103 100087 75.3332 65.5 0.0329845 0.0132688 2023-10-09T10:45:55.323 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:55.334 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:55.365 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:55.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:55.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:55.468 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:55 smithi155 ceph-mon[184854]: failed to encode map e139 with expected crc 2023-10-09T10:45:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:55 smithi155 ceph-mon[184854]: failed to encode map e140 with expected crc 2023-10-09T10:45:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:55 smithi155 ceph-mon[184854]: osdmap e140: 8 total, 8 up, 7 in 2023-10-09T10:45:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:55 smithi155 ceph-mon[184854]: failed to encode map e140 with expected crc 2023-10-09T10:45:55.513 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:55.543 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:55.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:55.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:55.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:55 smithi125 ceph-mon[195507]: failed to encode map e139 with expected crc 2023-10-09T10:45:55.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:55 smithi125 ceph-mon[195507]: failed to encode map e140 with expected crc 2023-10-09T10:45:55.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:55 smithi125 ceph-mon[195507]: osdmap e140: 8 total, 8 up, 7 in 2023-10-09T10:45:55.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:55 smithi125 ceph-mon[195507]: failed to encode map e140 with expected crc 2023-10-09T10:45:55.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:55 smithi125 ceph-mon[200197]: failed to encode map e139 with expected crc 2023-10-09T10:45:55.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:55 smithi125 ceph-mon[200197]: failed to encode map e140 with expected crc 2023-10-09T10:45:55.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:55 smithi125 ceph-mon[200197]: osdmap e140: 8 total, 8 up, 7 in 2023-10-09T10:45:55.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:55 smithi125 ceph-mon[200197]: failed to encode map e140 with expected crc 2023-10-09T10:45:55.647 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:55.704 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:55.994 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:56.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:56.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:56.105 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:56.167 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:56.267 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:56.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:56.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:56.270 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:56 smithi125 ceph-mon[195507]: failed to encode map e140 with expected crc 2023-10-09T10:45:56.270 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:56 smithi125 ceph-mon[195507]: failed to encode map e140 with expected crc 2023-10-09T10:45:56.270 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:56 smithi125 ceph-mon[195507]: pgmap v282: 119 pgs: 10 unknown, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 102 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 764 op/s; 11673/291930 objects degraded (3.999%); 1428/291930 objects misplaced (0.489%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T10:45:56.271 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:56 smithi125 ceph-mon[200197]: failed to encode map e140 with expected crc 2023-10-09T10:45:56.271 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:56 smithi125 ceph-mon[200197]: failed to encode map e140 with expected crc 2023-10-09T10:45:56.271 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:56 smithi125 ceph-mon[200197]: pgmap v282: 119 pgs: 10 unknown, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 102 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 764 op/s; 11673/291930 objects degraded (3.999%); 1428/291930 objects misplaced (0.489%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T10:45:56.271 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:56.323 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 84 16 101116 101100 75.1901 63.3125 0.0140009 0.0132949 2023-10-09T10:45:56.371 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:56.415 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:56.440 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:56.445 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:56 smithi155 ceph-mon[184854]: failed to encode map e140 with expected crc 2023-10-09T10:45:56.445 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:56 smithi155 ceph-mon[184854]: failed to encode map e140 with expected crc 2023-10-09T10:45:56.445 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:56 smithi155 ceph-mon[184854]: pgmap v282: 119 pgs: 10 unknown, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 102 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 764 op/s; 11673/291930 objects degraded (3.999%); 1428/291930 objects misplaced (0.489%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T10:45:56.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:56.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:56.547 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:56.579 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:56.605 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:56.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:56.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:56.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:56.785 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:56.958 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:56.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:56.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:57.066 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:57.098 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:57.125 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:57.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:57.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:57.199 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:45:57.200 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:45:57.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:45:57.229 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:57.272 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:57.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:57.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:57.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:57.323 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 85 16 102103 102087 75.0312 61.6875 0.0207502 0.0133234 2023-10-09T10:45:57.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:57.447 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:57.472 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:57.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:57.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:57.526 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:45:57.527 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:45:57.527 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-10-09T10:45:57.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd in 0 2023-10-09T10:45:57.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:57.625 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:57.804 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:45:57.863 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:57.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:57.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:57.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:58.031 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:58.077 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:58.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:58.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:58.183 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:45:58.201 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:58.205 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:58.257 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:58.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:45:58.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:58.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:45:58.323 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 86 16 102875 102859 74.7199 48.25 0.0184468 0.0133782 2023-10-09T10:45:58.403 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:45:58.449 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:58 smithi125 ceph-mon[195507]: pgmap v283: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 112 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 59 MiB/s wr, 939 op/s; 11673/299391 objects degraded (3.899%); 1428/299391 objects misplaced (0.477%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T10:45:58.450 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:58 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 11673/299391 objects degraded (3.899%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:58.450 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:58 smithi125 ceph-mon[200197]: pgmap v283: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 112 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 59 MiB/s wr, 939 op/s; 11673/299391 objects degraded (3.899%); 1428/299391 objects misplaced (0.477%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T10:45:58.450 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:58 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 11673/299391 objects degraded (3.899%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:58.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:58 smithi155 ceph-mon[184854]: pgmap v283: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 112 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 59 MiB/s wr, 939 op/s; 11673/299391 objects degraded (3.899%); 1428/299391 objects misplaced (0.477%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T10:45:58.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:58 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 11673/299391 objects degraded (3.899%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:45:58.482 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:58.508 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:45:58.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:58.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:45:58.551 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:45:58.615 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:58.654 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:58.680 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:58.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:58.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:58.783 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:58.817 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:58.847 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:58.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:58.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:58.953 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:45:58.993 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:59.022 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:45:59.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:59.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:45:59.127 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:59.153 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:59.188 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:59.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:59.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:45:59.211 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:59 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2613982150' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:59.212 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:45:59 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:59.212 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:59 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2613982150' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:59.212 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:45:59 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:59.226 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:45:59.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:45:59.323 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 87 16 103739 103723 74.4818 54 0.0132694 0.0134204 2023-10-09T10:45:59.323 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:59 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2613982150' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:59.324 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:45:59 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:45:59.327 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:59.358 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:45:59.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:59.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:45:59.463 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:45:59.499 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:59.532 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:45:59.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:59.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:45:59.637 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:45:59.686 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:59.720 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:45:59.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:59.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:45:59.825 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:45:59.873 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:45:59.907 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:45:59.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:45:59.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:00.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:00.077 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:00.211 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:00.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:00.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:00.323 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 88 16 104733 104717 74.3413 62.125 0.0180985 0.0134466 2023-10-09T10:46:00.334 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:00.388 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:00.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[195507]: pgmap v284: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 112 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 907 op/s; 11673/303378 objects degraded (3.848%); 1428/303378 objects misplaced (0.471%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T10:46:00.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:00.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:46:00.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[195507]: osdmap e141: 8 total, 8 up, 7 in 2023-10-09T10:46:00.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[195507]: failed to encode map e141 with expected crc 2023-10-09T10:46:00.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[195507]: failed to encode map e141 with expected crc 2023-10-09T10:46:00.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[195507]: failed to encode map e141 with expected crc 2023-10-09T10:46:00.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[195507]: failed to encode map e141 with expected crc 2023-10-09T10:46:00.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:46:00.396 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[200197]: pgmap v284: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 112 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 907 op/s; 11673/303378 objects degraded (3.848%); 1428/303378 objects misplaced (0.471%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T10:46:00.397 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:00.397 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:46:00.397 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[200197]: osdmap e141: 8 total, 8 up, 7 in 2023-10-09T10:46:00.397 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[200197]: failed to encode map e141 with expected crc 2023-10-09T10:46:00.397 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[200197]: failed to encode map e141 with expected crc 2023-10-09T10:46:00.397 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[200197]: failed to encode map e141 with expected crc 2023-10-09T10:46:00.397 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[200197]: failed to encode map e141 with expected crc 2023-10-09T10:46:00.397 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:46:00.415 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:00.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:00.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:00 smithi155 ceph-mon[184854]: pgmap v284: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 112 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 907 op/s; 11673/303378 objects degraded (3.848%); 1428/303378 objects misplaced (0.471%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T10:46:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:00 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:00 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:46:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:00 smithi155 ceph-mon[184854]: osdmap e141: 8 total, 8 up, 7 in 2023-10-09T10:46:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:00 smithi155 ceph-mon[184854]: failed to encode map e141 with expected crc 2023-10-09T10:46:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:00 smithi155 ceph-mon[184854]: failed to encode map e141 with expected crc 2023-10-09T10:46:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:00 smithi155 ceph-mon[184854]: failed to encode map e141 with expected crc 2023-10-09T10:46:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:00 smithi155 ceph-mon[184854]: failed to encode map e141 with expected crc 2023-10-09T10:46:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:46:00.519 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:00.597 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:00.774 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:00.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:00.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:00.880 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:00.976 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:01.095 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:46:01.095 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:46:01.233 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:46:01.233 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:46:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:46:01] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.43.0" 2023-10-09T10:46:01.281 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:01.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:01.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:01.323 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 89 16 105565 105549 74.0902 52 0.015934 0.0134917 2023-10-09T10:46:01.324 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:46:01.374 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:46:01.386 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:01.406 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:46:01.421 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:01.448 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:46:01.485 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:46:01.531 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:46:01.569 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:46:01.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T10:46:01.397+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e141 definitely_dead 0 2023-10-09T10:46:01.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:46:01.613 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:46:01.619 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:46:01.620 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:46:01.620 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:46:01.620 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:46:01.620 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:46:01.620 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:46:01.620 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:46:01.620 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:46:01.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:46:01.908 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:01.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:01.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:01.960 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:46:02.014 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:02.071 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:02.277 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.0. 2023-10-09T10:46:02.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:02 smithi125 ceph-mon[195507]: pgmap v286: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 112 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 11673/311805 objects degraded (3.744%); 1064/311805 objects misplaced (0.341%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T10:46:02.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:02 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1842017067' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:02.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:02 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2488972267' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T10:46:02.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:02 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T10:46:02.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:02 smithi125 ceph-mon[200197]: pgmap v286: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 112 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 11673/311805 objects degraded (3.744%); 1064/311805 objects misplaced (0.341%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T10:46:02.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:02 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1842017067' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:02.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:02 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2488972267' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T10:46:02.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:02 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T10:46:02.323 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 90 9 106440 106431 73.8795 55.125 0.0207807 0.013524 2023-10-09T10:46:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:02 smithi155 ceph-mon[184854]: pgmap v286: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 112 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 11673/311805 objects degraded (3.744%); 1064/311805 objects misplaced (0.341%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T10:46:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:02 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1842017067' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:02.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:02 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2488972267' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T10:46:02.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:02 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T10:46:03.324 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 91 4 106440 106436 73.0713 0.3125 0.471457 0.0135456 2023-10-09T10:46:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:03 smithi155 ceph-mon[184854]: 6.2 scrub starts 2023-10-09T10:46:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:03 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T10:46:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:03 smithi155 ceph-mon[184854]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T10:46:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:03 smithi155 ceph-mon[184854]: failed to encode map e142 with expected crc 2023-10-09T10:46:03.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:03 smithi155 ceph-mon[184854]: failed to encode map e142 with expected crc 2023-10-09T10:46:03.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:03 smithi155 ceph-mon[184854]: 6.6 scrub starts 2023-10-09T10:46:03.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:03 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 11673/311805 objects degraded (3.744%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:03.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:03 smithi155 ceph-mon[184854]: failed to encode map e143 with expected crc 2023-10-09T10:46:03.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:03 smithi155 ceph-mon[184854]: failed to encode map e143 with expected crc 2023-10-09T10:46:03.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:03 smithi155 ceph-mon[184854]: failed to encode map e143 with expected crc 2023-10-09T10:46:03.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:03 smithi155 ceph-mon[184854]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T10:46:03.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:03 smithi155 ceph-mon[184854]: failed to encode map e143 with expected crc 2023-10-09T10:46:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[195507]: 6.2 scrub starts 2023-10-09T10:46:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T10:46:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[195507]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T10:46:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[195507]: failed to encode map e142 with expected crc 2023-10-09T10:46:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[195507]: failed to encode map e142 with expected crc 2023-10-09T10:46:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[195507]: 6.6 scrub starts 2023-10-09T10:46:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 11673/311805 objects degraded (3.744%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[195507]: failed to encode map e143 with expected crc 2023-10-09T10:46:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[195507]: failed to encode map e143 with expected crc 2023-10-09T10:46:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[195507]: failed to encode map e143 with expected crc 2023-10-09T10:46:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[195507]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T10:46:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[195507]: failed to encode map e143 with expected crc 2023-10-09T10:46:03.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[200197]: 6.2 scrub starts 2023-10-09T10:46:03.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T10:46:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[200197]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T10:46:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[200197]: failed to encode map e142 with expected crc 2023-10-09T10:46:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[200197]: failed to encode map e142 with expected crc 2023-10-09T10:46:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[200197]: 6.6 scrub starts 2023-10-09T10:46:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 11673/311805 objects degraded (3.744%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[200197]: failed to encode map e143 with expected crc 2023-10-09T10:46:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[200197]: failed to encode map e143 with expected crc 2023-10-09T10:46:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[200197]: failed to encode map e143 with expected crc 2023-10-09T10:46:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[200197]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T10:46:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:03 smithi125 ceph-mon[200197]: failed to encode map e143 with expected crc 2023-10-09T10:46:04.324 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 92 2 106440 106438 72.2787 0.125 1.41747 0.0135721 2023-10-09T10:46:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: failed to encode map e143 with expected crc 2023-10-09T10:46:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: failed to encode map e143 with expected crc 2023-10-09T10:46:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: failed to encode map e143 with expected crc 2023-10-09T10:46:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: 6.6 scrub ok 2023-10-09T10:46:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: pgmap v289: 119 pgs: 17 peering, 19 unknown, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 7.3 MiB/s wr, 87 op/s; 11671/301581 objects degraded (3.870%); 897/301581 objects misplaced (0.297%) 2023-10-09T10:46:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: 6.16 scrub starts 2023-10-09T10:46:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T10:46:04.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: 6.10 starting backfill to osd.0 from (0'0,120'3010] MIN to 141'4440 2023-10-09T10:46:04.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: 6.0 starting backfill to osd.0 from (0'0,120'3017] MIN to 141'4369 2023-10-09T10:46:04.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 6 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2023-10-09T10:46:04.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:04 smithi155 ceph-mon[184854]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: failed to encode map e143 with expected crc 2023-10-09T10:46:04.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: failed to encode map e143 with expected crc 2023-10-09T10:46:04.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: failed to encode map e143 with expected crc 2023-10-09T10:46:04.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: 6.6 scrub ok 2023-10-09T10:46:04.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: pgmap v289: 119 pgs: 17 peering, 19 unknown, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 7.3 MiB/s wr, 87 op/s; 11671/301581 objects degraded (3.870%); 897/301581 objects misplaced (0.297%) 2023-10-09T10:46:04.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: 6.16 scrub starts 2023-10-09T10:46:04.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T10:46:04.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: 6.10 starting backfill to osd.0 from (0'0,120'3010] MIN to 141'4440 2023-10-09T10:46:04.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: 6.0 starting backfill to osd.0 from (0'0,120'3017] MIN to 141'4369 2023-10-09T10:46:04.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 6 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2023-10-09T10:46:04.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[195507]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: failed to encode map e143 with expected crc 2023-10-09T10:46:04.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: failed to encode map e143 with expected crc 2023-10-09T10:46:04.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: failed to encode map e143 with expected crc 2023-10-09T10:46:04.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: 6.6 scrub ok 2023-10-09T10:46:04.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: pgmap v289: 119 pgs: 17 peering, 19 unknown, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 7.3 MiB/s wr, 87 op/s; 11671/301581 objects degraded (3.870%); 897/301581 objects misplaced (0.297%) 2023-10-09T10:46:04.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: 6.16 scrub starts 2023-10-09T10:46:04.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T10:46:04.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: 6.10 starting backfill to osd.0 from (0'0,120'3010] MIN to 141'4440 2023-10-09T10:46:04.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: 6.0 starting backfill to osd.0 from (0'0,120'3017] MIN to 141'4369 2023-10-09T10:46:04.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 6 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2023-10-09T10:46:04.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:04 smithi125 ceph-mon[200197]: failed to encode map e144 with expected crc 2023-10-09T10:46:04.602 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:04.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:04.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:04.651 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:46:04.709 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:04.738 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:04.756 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:04.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:04.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:04.860 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:04.887 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:04.906 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:04.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:04.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:05.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:05.037 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:05.055 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:05.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:05.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:05.159 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:05.191 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:05.324 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 93 2 106440 106438 71.5017 0 - 0.0135721 2023-10-09T10:46:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:05 smithi155 ceph-mon[184854]: 6.2 scrub starts 2023-10-09T10:46:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:05 smithi155 ceph-mon[184854]: failed to encode map e144 with expected crc 2023-10-09T10:46:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:05 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:05 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:05 smithi155 ceph-mon[184854]: osdmap e145: 8 total, 8 up, 8 in 2023-10-09T10:46:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:05 smithi155 ceph-mon[184854]: failed to encode map e145 with expected crc 2023-10-09T10:46:05.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:05 smithi155 ceph-mon[184854]: failed to encode map e145 with expected crc 2023-10-09T10:46:05.485 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:05 smithi125 ceph-mon[200197]: 6.2 scrub starts 2023-10-09T10:46:05.485 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:05 smithi125 ceph-mon[200197]: failed to encode map e144 with expected crc 2023-10-09T10:46:05.485 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:05 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:05.485 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:05 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:05.485 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:05 smithi125 ceph-mon[200197]: osdmap e145: 8 total, 8 up, 8 in 2023-10-09T10:46:05.485 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:05 smithi125 ceph-mon[200197]: failed to encode map e145 with expected crc 2023-10-09T10:46:05.485 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:05 smithi125 ceph-mon[200197]: failed to encode map e145 with expected crc 2023-10-09T10:46:05.485 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:05 smithi125 ceph-mon[195507]: 6.2 scrub starts 2023-10-09T10:46:05.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:05 smithi125 ceph-mon[195507]: failed to encode map e144 with expected crc 2023-10-09T10:46:05.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:05 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:05.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:05 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:05.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:05 smithi125 ceph-mon[195507]: osdmap e145: 8 total, 8 up, 8 in 2023-10-09T10:46:05.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:05 smithi125 ceph-mon[195507]: failed to encode map e145 with expected crc 2023-10-09T10:46:05.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:05 smithi125 ceph-mon[195507]: failed to encode map e145 with expected crc 2023-10-09T10:46:05.516 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:05.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:05.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:05.620 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:05.649 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:06.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:06 smithi125 ceph-mon[200197]: failed to encode map e145 with expected crc 2023-10-09T10:46:06.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:06 smithi125 ceph-mon[200197]: failed to encode map e145 with expected crc 2023-10-09T10:46:06.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:06 smithi125 ceph-mon[200197]: pgmap v292: 119 pgs: 1 active+recovering+undersized+remapped, 78 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 19 unknown, 15 peering, 1 remapped+peering, 1 active+clean+scrubbing; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 9551/313704 objects degraded (3.045%); 4090/313704 objects misplaced (1.304%) 2023-10-09T10:46:06.263 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:06 smithi125 ceph-mon[200197]: osdmap e146: 8 total, 8 up, 8 in 2023-10-09T10:46:06.263 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:06 smithi125 ceph-mon[200197]: failed to encode map e146 with expected crc 2023-10-09T10:46:06.263 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:06 smithi125 ceph-mon[200197]: failed to encode map e146 with expected crc 2023-10-09T10:46:06.324 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 94 2 106440 106438 70.7413 0 - 0.0135721 2023-10-09T10:46:06.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:06 smithi125 ceph-mon[195507]: failed to encode map e145 with expected crc 2023-10-09T10:46:06.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:06 smithi125 ceph-mon[195507]: failed to encode map e145 with expected crc 2023-10-09T10:46:06.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:06 smithi125 ceph-mon[195507]: pgmap v292: 119 pgs: 1 active+recovering+undersized+remapped, 78 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 19 unknown, 15 peering, 1 remapped+peering, 1 active+clean+scrubbing; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 9551/313704 objects degraded (3.045%); 4090/313704 objects misplaced (1.304%) 2023-10-09T10:46:06.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:06 smithi125 ceph-mon[195507]: osdmap e146: 8 total, 8 up, 8 in 2023-10-09T10:46:06.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:06 smithi125 ceph-mon[195507]: failed to encode map e146 with expected crc 2023-10-09T10:46:06.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:06 smithi125 ceph-mon[195507]: failed to encode map e146 with expected crc 2023-10-09T10:46:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:06 smithi155 ceph-mon[184854]: failed to encode map e145 with expected crc 2023-10-09T10:46:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:06 smithi155 ceph-mon[184854]: failed to encode map e145 with expected crc 2023-10-09T10:46:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:06 smithi155 ceph-mon[184854]: pgmap v292: 119 pgs: 1 active+recovering+undersized+remapped, 78 active+clean, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 19 unknown, 15 peering, 1 remapped+peering, 1 active+clean+scrubbing; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 9551/313704 objects degraded (3.045%); 4090/313704 objects misplaced (1.304%) 2023-10-09T10:46:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:06 smithi155 ceph-mon[184854]: osdmap e146: 8 total, 8 up, 8 in 2023-10-09T10:46:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:06 smithi155 ceph-mon[184854]: failed to encode map e146 with expected crc 2023-10-09T10:46:06.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:06 smithi155 ceph-mon[184854]: failed to encode map e146 with expected crc 2023-10-09T10:46:07.164 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:07.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:07.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:07.190 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:07.233 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-10-09T10:46:07.268 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:07.302 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:07.324 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 95 2 106440 106438 69.9967 0 - 0.0135721 2023-10-09T10:46:07.508 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:46:07.932 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:07.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:07.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:08.040 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:08.069 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:08.087 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:08.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:08.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:08.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:08.222 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:08.324 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total time run: 95.1452 2023-10-09T10:46:08.324 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total writes made: 106440 2023-10-09T10:46:08.325 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Write size: 65536 2023-10-09T10:46:08.325 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object size: 65536 2023-10-09T10:46:08.325 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Bandwidth (MB/sec): 69.9194 2023-10-09T10:46:08.325 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Bandwidth: 37.1228 2023-10-09T10:46:08.325 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max bandwidth (MB/sec): 166.5 2023-10-09T10:46:08.325 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min bandwidth (MB/sec): 0 2023-10-09T10:46:08.325 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average IOPS: 1118 2023-10-09T10:46:08.325 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev IOPS: 593.964 2023-10-09T10:46:08.325 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max IOPS: 2664 2023-10-09T10:46:08.325 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min IOPS: 0 2023-10-09T10:46:08.325 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average Latency(s): 0.0136693 2023-10-09T10:46:08.325 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Latency(s): 0.0541373 2023-10-09T10:46:08.326 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max latency(s): 5.19684 2023-10-09T10:46:08.326 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min latency(s): 0.00102371 2023-10-09T10:46:08.335 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T10:46:08.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:08 smithi155 ceph-mon[184854]: pgmap v294: 119 pgs: 1 active+recovering+undersized+remapped, 78 active+clean, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 19 unknown, 14 peering, 1 remapped+peering, 1 active+clean+scrubbing; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 35 MiB/s wr, 565 op/s; 6772/319314 objects degraded (2.121%); 7484/319314 objects misplaced (2.344%); 89 MiB/s, 1.43k objects/s recovering 2023-10-09T10:46:08.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:08 smithi155 ceph-mon[184854]: failed to encode map e147 with expected crc 2023-10-09T10:46:08.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:08 smithi155 ceph-mon[184854]: failed to encode map e147 with expected crc 2023-10-09T10:46:08.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:08 smithi155 ceph-mon[184854]: osdmap e147: 8 total, 8 up, 8 in 2023-10-09T10:46:08.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:08 smithi155 ceph-mon[184854]: failed to encode map e147 with expected crc 2023-10-09T10:46:08.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:08 smithi155 ceph-mon[184854]: 6.1 scrub starts 2023-10-09T10:46:08.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:08 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 6772/319314 objects degraded (2.121%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:08.495 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:08.501 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:08.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:08.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:08.504 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:08 smithi125 ceph-mon[195507]: pgmap v294: 119 pgs: 1 active+recovering+undersized+remapped, 78 active+clean, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 19 unknown, 14 peering, 1 remapped+peering, 1 active+clean+scrubbing; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 35 MiB/s wr, 565 op/s; 6772/319314 objects degraded (2.121%); 7484/319314 objects misplaced (2.344%); 89 MiB/s, 1.43k objects/s recovering 2023-10-09T10:46:08.504 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:08 smithi125 ceph-mon[195507]: failed to encode map e147 with expected crc 2023-10-09T10:46:08.504 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:08 smithi125 ceph-mon[195507]: failed to encode map e147 with expected crc 2023-10-09T10:46:08.504 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:08 smithi125 ceph-mon[195507]: osdmap e147: 8 total, 8 up, 8 in 2023-10-09T10:46:08.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:08 smithi125 ceph-mon[195507]: failed to encode map e147 with expected crc 2023-10-09T10:46:08.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:08 smithi125 ceph-mon[195507]: 6.1 scrub starts 2023-10-09T10:46:08.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:08 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 6772/319314 objects degraded (2.121%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:08.505 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:08 smithi125 ceph-mon[200197]: pgmap v294: 119 pgs: 1 active+recovering+undersized+remapped, 78 active+clean, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 19 unknown, 14 peering, 1 remapped+peering, 1 active+clean+scrubbing; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 35 MiB/s wr, 565 op/s; 6772/319314 objects degraded (2.121%); 7484/319314 objects misplaced (2.344%); 89 MiB/s, 1.43k objects/s recovering 2023-10-09T10:46:08.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:08 smithi125 ceph-mon[200197]: failed to encode map e147 with expected crc 2023-10-09T10:46:08.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:08 smithi125 ceph-mon[200197]: failed to encode map e147 with expected crc 2023-10-09T10:46:08.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:08 smithi125 ceph-mon[200197]: osdmap e147: 8 total, 8 up, 8 in 2023-10-09T10:46:08.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:08 smithi125 ceph-mon[200197]: failed to encode map e147 with expected crc 2023-10-09T10:46:08.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:08 smithi125 ceph-mon[200197]: 6.1 scrub starts 2023-10-09T10:46:08.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:08 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 6772/319314 objects degraded (2.121%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:08.605 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:08.689 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:08.781 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:08.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:08.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:08.885 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:08.921 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:08.951 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:08.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:08.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:09.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:09.088 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:09.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:09.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:09.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:09.220 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:09.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[195507]: 6.2 scrub starts 2023-10-09T10:46:09.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[195507]: failed to encode map e147 with expected crc 2023-10-09T10:46:09.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[195507]: failed to encode map e147 with expected crc 2023-10-09T10:46:09.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[195507]: failed to encode map e148 with expected crc 2023-10-09T10:46:09.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[195507]: osdmap e148: 8 total, 8 up, 8 in 2023-10-09T10:46:09.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[195507]: failed to encode map e148 with expected crc 2023-10-09T10:46:09.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[195507]: failed to encode map e148 with expected crc 2023-10-09T10:46:09.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[195507]: pgmap v297: 119 pgs: 1 peering, 112 active+clean, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 180 KiB/s rd, 71 MiB/s wr, 1.29k op/s; 10782/319125 objects degraded (3.379%); 6716/319125 objects misplaced (2.105%); 83 MiB/s, 1.32k objects/s recovering 2023-10-09T10:46:09.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[200197]: 6.2 scrub starts 2023-10-09T10:46:09.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[200197]: failed to encode map e147 with expected crc 2023-10-09T10:46:09.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[200197]: failed to encode map e147 with expected crc 2023-10-09T10:46:09.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[200197]: failed to encode map e148 with expected crc 2023-10-09T10:46:09.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[200197]: osdmap e148: 8 total, 8 up, 8 in 2023-10-09T10:46:09.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[200197]: failed to encode map e148 with expected crc 2023-10-09T10:46:09.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[200197]: failed to encode map e148 with expected crc 2023-10-09T10:46:09.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:09 smithi125 ceph-mon[200197]: pgmap v297: 119 pgs: 1 peering, 112 active+clean, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 180 KiB/s rd, 71 MiB/s wr, 1.29k op/s; 10782/319125 objects degraded (3.379%); 6716/319125 objects misplaced (2.105%); 83 MiB/s, 1.32k objects/s recovering 2023-10-09T10:46:09.334 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:09.414 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:09.417 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:09.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:09.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:09.528 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:09.621 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:09.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:09 smithi155 ceph-mon[184854]: 6.2 scrub starts 2023-10-09T10:46:09.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:09 smithi155 ceph-mon[184854]: failed to encode map e147 with expected crc 2023-10-09T10:46:09.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:09 smithi155 ceph-mon[184854]: failed to encode map e147 with expected crc 2023-10-09T10:46:09.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:09 smithi155 ceph-mon[184854]: failed to encode map e148 with expected crc 2023-10-09T10:46:09.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:09 smithi155 ceph-mon[184854]: osdmap e148: 8 total, 8 up, 8 in 2023-10-09T10:46:09.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:09 smithi155 ceph-mon[184854]: failed to encode map e148 with expected crc 2023-10-09T10:46:09.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:09 smithi155 ceph-mon[184854]: failed to encode map e148 with expected crc 2023-10-09T10:46:09.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:09 smithi155 ceph-mon[184854]: pgmap v297: 119 pgs: 1 peering, 112 active+clean, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 180 KiB/s rd, 71 MiB/s wr, 1.29k op/s; 10782/319125 objects degraded (3.379%); 6716/319125 objects misplaced (2.105%); 83 MiB/s, 1.32k objects/s recovering 2023-10-09T10:46:09.857 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:09.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:09.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:09.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:10.021 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:10.104 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:10.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:10.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:10.209 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:10.243 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:10.269 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:10.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:10.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:10.314 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:46:10.373 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:10.456 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:10 smithi125 ceph-mon[195507]: failed to encode map e148 with expected crc 2023-10-09T10:46:10.456 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:10 smithi125 ceph-mon[195507]: 6.2 scrub starts 2023-10-09T10:46:10.456 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:10 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 11 pgs peering) 2023-10-09T10:46:10.456 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:10 smithi125 ceph-mon[195507]: 6.2 scrub starts 2023-10-09T10:46:10.456 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:10 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3610099852' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:10.456 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:10 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:10.457 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:10 smithi125 ceph-mon[200197]: failed to encode map e148 with expected crc 2023-10-09T10:46:10.457 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:10 smithi125 ceph-mon[200197]: 6.2 scrub starts 2023-10-09T10:46:10.457 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:10 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 11 pgs peering) 2023-10-09T10:46:10.457 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:10 smithi125 ceph-mon[200197]: 6.2 scrub starts 2023-10-09T10:46:10.457 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:10 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3610099852' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:10.457 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:10 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:10.460 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:10.539 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:10.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:10.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:10.647 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:10.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:10 smithi155 ceph-mon[184854]: failed to encode map e148 with expected crc 2023-10-09T10:46:10.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:10 smithi155 ceph-mon[184854]: 6.2 scrub starts 2023-10-09T10:46:10.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:10 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 11 pgs peering) 2023-10-09T10:46:10.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:10 smithi155 ceph-mon[184854]: 6.2 scrub starts 2023-10-09T10:46:10.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:10 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3610099852' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:10.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:10 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:10.684 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:10.716 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:10.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:10.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:10.826 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:10.877 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:11.065 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:11.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:11.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:11.174 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:11.248 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:11.257 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:46:11.257 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:46:11.257 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:46:11.257 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:46:11.257 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:46:11.497 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:11.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:11.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:11.503 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:11 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:46:11.503 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:11 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:46:11.503 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:11 smithi125 ceph-mon[195507]: osdmap e149: 8 total, 8 up, 8 in 2023-10-09T10:46:11.503 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:11 smithi125 ceph-mon[195507]: failed to encode map e149 with expected crc 2023-10-09T10:46:11.504 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:11 smithi125 ceph-mon[195507]: pgmap v299: 119 pgs: 1 peering, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 147 KiB/s rd, 22 MiB/s wr, 659 op/s; 8135/316053 objects degraded (2.574%); 8912/316053 objects misplaced (2.820%); 22 MiB/s, 344 objects/s recovering 2023-10-09T10:46:11.506 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:46:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:46:11] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.43.0" 2023-10-09T10:46:11.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:11 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:46:11.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:11 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:46:11.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:11 smithi125 ceph-mon[200197]: osdmap e149: 8 total, 8 up, 8 in 2023-10-09T10:46:11.507 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:11 smithi125 ceph-mon[200197]: failed to encode map e149 with expected crc 2023-10-09T10:46:11.507 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:11 smithi125 ceph-mon[200197]: pgmap v299: 119 pgs: 1 peering, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 147 KiB/s rd, 22 MiB/s wr, 659 op/s; 8135/316053 objects degraded (2.574%); 8912/316053 objects misplaced (2.820%); 22 MiB/s, 344 objects/s recovering 2023-10-09T10:46:11.604 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:11.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:11 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:46:11.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:11 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:46:11.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:11 smithi155 ceph-mon[184854]: osdmap e149: 8 total, 8 up, 8 in 2023-10-09T10:46:11.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:11 smithi155 ceph-mon[184854]: failed to encode map e149 with expected crc 2023-10-09T10:46:11.636 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:11 smithi155 ceph-mon[184854]: pgmap v299: 119 pgs: 1 peering, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 147 KiB/s rd, 22 MiB/s wr, 659 op/s; 8135/316053 objects degraded (2.574%); 8912/316053 objects misplaced (2.820%); 22 MiB/s, 344 objects/s recovering 2023-10-09T10:46:11.638 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:11.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:11.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:11.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:11.780 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:11.811 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:11.834 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:11.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:11.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:11.928 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:46:11.928 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:46:11.928 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:46:11.939 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:11.957 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:46:11.969 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:11.994 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:46:11.995 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:11.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:12.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:12.035 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:46:12.076 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:46:12.100 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:12.108 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:46:12.130 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:46:12.130 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:46:12.130 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:46:12.130 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:46:12.130 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:46:12.130 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:46:12.130 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:46:12.131 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:46:12.173 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:12.234 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:46:12.234 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:46:12.234 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T10:46:12.234 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:46:12.262 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:46:12.550 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:12 smithi125 ceph-mon[195507]: 6.14 scrub starts 2023-10-09T10:46:12.550 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:12 smithi125 ceph-mon[195507]: 6.2 deep-scrub starts 2023-10-09T10:46:12.550 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:12 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3566186782' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:12.551 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:12 smithi125 ceph-mon[200197]: 6.14 scrub starts 2023-10-09T10:46:12.551 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:12 smithi125 ceph-mon[200197]: 6.2 deep-scrub starts 2023-10-09T10:46:12.551 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:12 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3566186782' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:12 smithi155 ceph-mon[184854]: 6.14 scrub starts 2023-10-09T10:46:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:12 smithi155 ceph-mon[184854]: 6.2 deep-scrub starts 2023-10-09T10:46:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:12 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3566186782' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:12.879 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:12.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:12.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:12.989 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:13.083 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:13.118 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:13.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:13.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:13.151 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:13.155 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:13.221 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:13.289 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:13.331 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:13.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:13.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:13.437 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:13.468 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:13.493 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:13.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:13.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:13.539 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:46:13.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:13 smithi125 ceph-mon[195507]: 6.1 deep-scrub starts 2023-10-09T10:46:13.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:13 smithi125 ceph-mon[195507]: 6.2 scrub starts 2023-10-09T10:46:13.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:13 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 8135/316053 objects degraded (2.574%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:13.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:13 smithi125 ceph-mon[195507]: pgmap v300: 119 pgs: 1 active+clean+scrubbing, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 134 KiB/s rd, 354 KiB/s wr, 363 op/s; 8135/314688 objects degraded (2.585%); 8828/314688 objects misplaced (2.805%); 13 MiB/s, 199 objects/s recovering 2023-10-09T10:46:13.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:13 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:13.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:13 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:13.569 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:13 smithi125 ceph-mon[200197]: 6.1 deep-scrub starts 2023-10-09T10:46:13.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:13 smithi125 ceph-mon[200197]: 6.2 scrub starts 2023-10-09T10:46:13.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:13 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 8135/316053 objects degraded (2.574%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:13.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:13 smithi125 ceph-mon[200197]: pgmap v300: 119 pgs: 1 active+clean+scrubbing, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 134 KiB/s rd, 354 KiB/s wr, 363 op/s; 8135/314688 objects degraded (2.585%); 8828/314688 objects misplaced (2.805%); 13 MiB/s, 199 objects/s recovering 2023-10-09T10:46:13.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:13 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:13.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:13 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:13.598 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:13.667 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:13.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:13 smithi155 ceph-mon[184854]: 6.1 deep-scrub starts 2023-10-09T10:46:13.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:13 smithi155 ceph-mon[184854]: 6.2 scrub starts 2023-10-09T10:46:13.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:13 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 8135/316053 objects degraded (2.574%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:13.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:13 smithi155 ceph-mon[184854]: pgmap v300: 119 pgs: 1 active+clean+scrubbing, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 134 KiB/s rd, 354 KiB/s wr, 363 op/s; 8135/314688 objects degraded (2.585%); 8828/314688 objects misplaced (2.805%); 13 MiB/s, 199 objects/s recovering 2023-10-09T10:46:13.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:13 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:13.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:13 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:13.851 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:13.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:13.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:13.958 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:13.998 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:14.026 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:14.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:14.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:14.133 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:14.211 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:14.310 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:14.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:14.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:14.424 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:14.461 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:14.484 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:14.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:14.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:14.502 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:14.588 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:14.620 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:14.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:14.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:14.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:14.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:14 smithi155 ceph-mon[184854]: 6.14 scrub starts 2023-10-09T10:46:14.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:14 smithi155 ceph-mon[184854]: 6.2 scrub starts 2023-10-09T10:46:14.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:46:14.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:46:14.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:14.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:14.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:46:14.795 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:14 smithi125 ceph-mon[195507]: 6.14 scrub starts 2023-10-09T10:46:14.795 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:14 smithi125 ceph-mon[195507]: 6.2 scrub starts 2023-10-09T10:46:14.796 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:46:14.796 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:46:14.796 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:14.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:14 smithi125 ceph-mon[200197]: 6.14 scrub starts 2023-10-09T10:46:14.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:14 smithi125 ceph-mon[200197]: 6.2 scrub starts 2023-10-09T10:46:14.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:46:14.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:46:14.797 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:14.831 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:15.150 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:15.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:15.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:15.254 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:15.273 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:46:15.273 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:46:15.291 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:15.317 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:15.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:15.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:15.421 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:15.475 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:15.595 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:46:15.608 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:46:16.175 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:16.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:16.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:16.211 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":301,"stamp":"2023-10-09T10:46:15.065340+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6742710200,"num_objects":102902,"num_object_clones":0,"num_object_copies":308706,"num_objects_missing_on_primary":897,"num_objects_missing":897,"num_objects_degraded":8135,"num_objects_misplaced":8700,"num_objects_unfound":0,"num_objects_dirty":102902,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":101630,"num_write_kb":6260974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11424,"num_bytes_recovered":749144246,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132201,"ondisk_log_size":132201,"up":357,"acting":353,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":369,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22337568,"kb_used_data":20704096,"kb_used_omap":126,"kb_used_meta":1632833,"kb_avail":727623648,"statfs":{"total":767960285184,"available":745086615552,"internally_reserved":0,"allocated":21200994304,"data_stored":21195875708,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1672021627},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-231413072,"num_objects":-3536,"num_object_clones":0,"num_object_copies":-10608,"num_objects_missing_on_primary":897,"num_objects_missing":890,"num_objects_degraded":1363,"num_objects_misplaced":1216,"num_objects_unfound":0,"num_objects_dirty":-3536,"num_whiteouts":0,"num_read":428,"num_read_kb":802,"num_write":3749,"num_write_kb":2124,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1196,"num_bytes_recovered":79151603,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.796997"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":132,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.340741+0000","last_change":"2023-10-09T10:43:06.676414+0000","last_active":"2023-10-09T10:46:08.340741+0000","last_peered":"2023-10-09T10:46:08.340741+0000","last_clean":"2023-10-09T10:46:08.340741+0000","last_became_active":"2023-10-09T10:43:06.676050+0000","last_became_peered":"2023-10-09T10:43:06.676050+0000","last_unstale":"2023-10-09T10:46:08.340741+0000","last_undegraded":"2023-10-09T10:46:08.340741+0000","last_fullsized":"2023-10-09T10:46:08.340741+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:28.011477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":117,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.277085+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:46:08.277085+0000","last_peered":"2023-10-09T10:46:08.277085+0000","last_clean":"2023-10-09T10:46:08.277085+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:46:08.277085+0000","last_undegraded":"2023-10-09T10:46:08.277085+0000","last_fullsized":"2023-10-09T10:46:08.277085+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":100,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279523+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:46:08.279523+0000","last_peered":"2023-10-09T10:46:08.279523+0000","last_clean":"2023-10-09T10:46:08.279523+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:46:08.279523+0000","last_undegraded":"2023-10-09T10:46:08.279523+0000","last_fullsized":"2023-10-09T10:46:08.279523+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":17,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278563+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:46:08.278563+0000","last_peered":"2023-10-09T10:46:08.278563+0000","last_clean":"2023-10-09T10:46:08.278563+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:46:08.278563+0000","last_undegraded":"2023-10-09T10:46:08.278563+0000","last_fullsized":"2023-10-09T10:46:08.278563+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":153,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.275855+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:46:08.275855+0000","last_peered":"2023-10-09T10:46:08.275855+0000","last_clean":"2023-10-09T10:46:08.275855+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:46:08.275855+0000","last_undegraded":"2023-10-09T10:46:08.275855+0000","last_fullsized":"2023-10-09T10:46:08.275855+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":87,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.286054+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:46:08.286054+0000","last_peered":"2023-10-09T10:46:08.286054+0000","last_clean":"2023-10-09T10:46:08.286054+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:46:08.286054+0000","last_undegraded":"2023-10-09T10:46:08.286054+0000","last_fullsized":"2023-10-09T10:46:08.286054+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"149'4591","reported_seq":8690,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:14.589165+0000","last_change":"2023-10-09T10:45:38.597219+0000","last_active":"2023-10-09T10:46:14.589165+0000","last_peered":"2023-10-09T10:46:14.589165+0000","last_clean":"2023-10-09T10:46:14.589165+0000","last_became_active":"2023-10-09T10:45:38.593601+0000","last_became_peered":"2023-10-09T10:45:38.593601+0000","last_unstale":"2023-10-09T10:46:14.589165+0000","last_undegraded":"2023-10-09T10:46:14.589165+0000","last_fullsized":"2023-10-09T10:46:14.589165+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4591,"log_dups_size":0,"ondisk_log_size":4591,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216432640,"num_objects":3303,"num_object_clones":0,"num_object_copies":9909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3555,"num_write_kb":219424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2005,"num_bytes_recovered":131399680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":17,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279145+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:46:08.279145+0000","last_peered":"2023-10-09T10:46:08.279145+0000","last_clean":"2023-10-09T10:46:08.279145+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:46:08.279145+0000","last_undegraded":"2023-10-09T10:46:08.279145+0000","last_fullsized":"2023-10-09T10:46:08.279145+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":117,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.275924+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:46:08.275924+0000","last_peered":"2023-10-09T10:46:08.275924+0000","last_clean":"2023-10-09T10:46:08.275924+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:46:08.275924+0000","last_undegraded":"2023-10-09T10:46:08.275924+0000","last_fullsized":"2023-10-09T10:46:08.275924+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":482,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278312+0000","last_change":"2023-10-09T10:46:05.620365+0000","last_active":"2023-10-09T10:46:08.278312+0000","last_peered":"2023-10-09T10:46:08.278312+0000","last_clean":"2023-10-09T10:46:08.278312+0000","last_became_active":"2023-10-09T10:46:05.620024+0000","last_became_peered":"2023-10-09T10:46:05.620024+0000","last_unstale":"2023-10-09T10:46:08.278312+0000","last_undegraded":"2023-10-09T10:46:08.278312+0000","last_fullsized":"2023-10-09T10:46:08.278312+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.13","version":"149'4428","reported_seq":4573,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:10.930336+0000","last_change":"2023-10-09T10:45:07.544591+0000","last_active":"2023-10-09T10:46:10.930336+0000","last_peered":"2023-10-09T10:46:10.930336+0000","last_clean":"2023-10-09T10:46:10.930336+0000","last_became_active":"2023-10-09T10:44:50.071995+0000","last_became_peered":"2023-10-09T10:44:50.071995+0000","last_unstale":"2023-10-09T10:46:10.930336+0000","last_undegraded":"2023-10-09T10:46:10.930336+0000","last_fullsized":"2023-10-09T10:46:10.930336+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":4428,"log_dups_size":0,"ondisk_log_size":4428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:45.046717+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212598784,"num_objects":3244,"num_object_clones":0,"num_object_copies":9732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1713,"num_write_kb":105088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":114,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.285962+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:46:08.285962+0000","last_peered":"2023-10-09T10:46:08.285962+0000","last_clean":"2023-10-09T10:46:08.285962+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:46:08.285962+0000","last_undegraded":"2023-10-09T10:46:08.285962+0000","last_fullsized":"2023-10-09T10:46:08.285962+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":74,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.275973+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:46:08.275973+0000","last_peered":"2023-10-09T10:46:08.275973+0000","last_clean":"2023-10-09T10:46:08.275973+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:46:08.275973+0000","last_undegraded":"2023-10-09T10:46:08.275973+0000","last_fullsized":"2023-10-09T10:46:08.275973+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"149'4524","reported_seq":12620,"reported_epoch":149,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:46:11.248095+0000","last_change":"2023-10-09T10:46:03.683196+0000","last_active":"2023-10-09T10:46:11.248095+0000","last_peered":"2023-10-09T10:46:11.248095+0000","last_clean":"2023-10-09T10:46:02.177309+0000","last_became_active":"2023-10-09T10:46:03.681058+0000","last_became_peered":"2023-10-09T10:46:03.681058+0000","last_unstale":"2023-10-09T10:46:11.248095+0000","last_undegraded":"2023-10-09T10:46:11.248095+0000","last_fullsized":"2023-10-09T10:46:11.248095+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4524,"log_dups_size":0,"ondisk_log_size":4524,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216891392,"num_objects":3310,"num_object_clones":0,"num_object_copies":9930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3310,"num_objects_unfound":0,"num_objects_dirty":3310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3478,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[1,5,6],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":3310}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":69,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.277014+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:46:08.277014+0000","last_peered":"2023-10-09T10:46:08.277014+0000","last_clean":"2023-10-09T10:46:08.277014+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:46:08.277014+0000","last_undegraded":"2023-10-09T10:46:08.277014+0000","last_fullsized":"2023-10-09T10:46:08.277014+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":100,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279899+0000","last_change":"2023-10-09T10:43:06.672114+0000","last_active":"2023-10-09T10:46:08.279899+0000","last_peered":"2023-10-09T10:46:08.279899+0000","last_clean":"2023-10-09T10:46:08.279899+0000","last_became_active":"2023-10-09T10:43:06.671827+0000","last_became_peered":"2023-10-09T10:43:06.671827+0000","last_unstale":"2023-10-09T10:46:08.279899+0000","last_undegraded":"2023-10-09T10:46:08.279899+0000","last_fullsized":"2023-10-09T10:46:08.279899+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:41.701409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"149'4435","reported_seq":6389,"reported_epoch":149,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:46:13.716436+0000","last_change":"2023-10-09T10:46:07.423351+0000","last_active":"2023-10-09T10:46:13.716436+0000","last_peered":"2023-10-09T10:46:13.716436+0000","last_clean":"2023-10-09T10:44:56.105580+0000","last_became_active":"2023-10-09T10:46:07.402111+0000","last_became_peered":"2023-10-09T10:46:07.402111+0000","last_unstale":"2023-10-09T10:46:13.716436+0000","last_undegraded":"2023-10-09T10:46:05.816165+0000","last_fullsized":"2023-10-09T10:46:05.812663+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4435,"log_dups_size":0,"ondisk_log_size":4435,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206143488,"num_objects":3146,"num_object_clones":0,"num_object_copies":9438,"num_objects_missing_on_primary":897,"num_objects_missing":897,"num_objects_degraded":4010,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3428,"num_write_kb":210336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":474,"num_bytes_recovered":31064064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,2,3,5","objects":1319},{"shards":"1,2,5,6","objects":1},{"shards":"1,2,6","objects":896},{"shards":"1,3,5","objects":1},{"shards":"3,5","objects":929}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":119,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.339963+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:46:08.339963+0000","last_peered":"2023-10-09T10:46:08.339963+0000","last_clean":"2023-10-09T10:46:08.339963+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:46:08.339963+0000","last_undegraded":"2023-10-09T10:46:08.339963+0000","last_fullsized":"2023-10-09T10:46:08.339963+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":113,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.340101+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:46:08.340101+0000","last_peered":"2023-10-09T10:46:08.340101+0000","last_clean":"2023-10-09T10:46:08.340101+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:46:08.340101+0000","last_undegraded":"2023-10-09T10:46:08.340101+0000","last_fullsized":"2023-10-09T10:46:08.340101+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":482,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278794+0000","last_change":"2023-10-09T10:46:05.618276+0000","last_active":"2023-10-09T10:46:08.278794+0000","last_peered":"2023-10-09T10:46:08.278794+0000","last_clean":"2023-10-09T10:46:08.278794+0000","last_became_active":"2023-10-09T10:46:05.616509+0000","last_became_peered":"2023-10-09T10:46:05.616509+0000","last_unstale":"2023-10-09T10:46:08.278794+0000","last_undegraded":"2023-10-09T10:46:08.278794+0000","last_fullsized":"2023-10-09T10:46:08.278794+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.16","version":"149'4479","reported_seq":4552,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:13.747932+0000","last_change":"2023-10-09T10:46:03.592682+0000","last_active":"2023-10-09T10:46:13.747932+0000","last_peered":"2023-10-09T10:46:13.747932+0000","last_clean":"2023-10-09T10:46:13.747932+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:46:13.747932+0000","last_undegraded":"2023-10-09T10:46:13.747932+0000","last_fullsized":"2023-10-09T10:46:13.747932+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4479,"log_dups_size":0,"ondisk_log_size":4479,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206438400,"num_objects":3150,"num_object_clones":0,"num_object_copies":9450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3436,"num_write_kb":210752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":115,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.281417+0000","last_change":"2023-10-09T10:43:06.677115+0000","last_active":"2023-10-09T10:46:08.281417+0000","last_peered":"2023-10-09T10:46:08.281417+0000","last_clean":"2023-10-09T10:46:08.281417+0000","last_became_active":"2023-10-09T10:43:06.676739+0000","last_became_peered":"2023-10-09T10:43:06.676739+0000","last_unstale":"2023-10-09T10:46:08.281417+0000","last_undegraded":"2023-10-09T10:46:08.281417+0000","last_fullsized":"2023-10-09T10:46:08.281417+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:02.520770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":131,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.276636+0000","last_change":"2023-10-09T10:43:35.487942+0000","last_active":"2023-10-09T10:46:08.276636+0000","last_peered":"2023-10-09T10:46:08.276636+0000","last_clean":"2023-10-09T10:46:08.276636+0000","last_became_active":"2023-10-09T10:43:35.487651+0000","last_became_peered":"2023-10-09T10:43:35.487651+0000","last_unstale":"2023-10-09T10:46:08.276636+0000","last_undegraded":"2023-10-09T10:46:08.276636+0000","last_fullsized":"2023-10-09T10:46:08.276636+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:51.882595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":483,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278834+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:46:08.278834+0000","last_peered":"2023-10-09T10:46:08.278834+0000","last_clean":"2023-10-09T10:46:08.278834+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:46:08.278834+0000","last_undegraded":"2023-10-09T10:46:08.278834+0000","last_fullsized":"2023-10-09T10:46:08.278834+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.17","version":"149'4330","reported_seq":4398,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:10.937036+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:46:10.937036+0000","last_peered":"2023-10-09T10:46:10.937036+0000","last_clean":"2023-10-09T10:46:10.937036+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:46:10.937036+0000","last_undegraded":"2023-10-09T10:46:10.937036+0000","last_fullsized":"2023-10-09T10:46:10.937036+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4330,"log_dups_size":0,"ondisk_log_size":4330,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208666624,"num_objects":3184,"num_object_clones":0,"num_object_copies":9552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3322,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":17,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278981+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:46:08.278981+0000","last_peered":"2023-10-09T10:46:08.278981+0000","last_clean":"2023-10-09T10:46:08.278981+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:46:08.278981+0000","last_undegraded":"2023-10-09T10:46:08.278981+0000","last_fullsized":"2023-10-09T10:46:08.278981+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":113,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.286001+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:46:08.286001+0000","last_peered":"2023-10-09T10:46:08.286001+0000","last_clean":"2023-10-09T10:46:08.286001+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:46:08.286001+0000","last_undegraded":"2023-10-09T10:46:08.286001+0000","last_fullsized":"2023-10-09T10:46:08.286001+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":483,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278227+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:46:08.278227+0000","last_peered":"2023-10-09T10:46:08.278227+0000","last_clean":"2023-10-09T10:46:08.278227+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:46:08.278227+0000","last_undegraded":"2023-10-09T10:46:08.278227+0000","last_fullsized":"2023-10-09T10:46:08.278227+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"149'4470","reported_seq":4540,"reported_epoch":149,"state":"active+clean+scrubbing","last_fresh":"2023-10-09T10:46:11.377932+0000","last_change":"2023-10-09T10:46:11.388035+0000","last_active":"2023-10-09T10:46:11.377932+0000","last_peered":"2023-10-09T10:46:11.377932+0000","last_clean":"2023-10-09T10:46:11.377932+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:46:11.377932+0000","last_undegraded":"2023-10-09T10:46:11.377932+0000","last_fullsized":"2023-10-09T10:46:11.377932+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4470,"log_dups_size":0,"ondisk_log_size":4470,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213483520,"num_objects":3258,"num_object_clones":0,"num_object_copies":9774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3416,"num_write_kb":213536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":59,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.339932+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:46:08.339932+0000","last_peered":"2023-10-09T10:46:08.339932+0000","last_clean":"2023-10-09T10:46:08.339932+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:46:08.339932+0000","last_undegraded":"2023-10-09T10:46:08.339932+0000","last_fullsized":"2023-10-09T10:46:08.339932+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":17,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279298+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:46:08.279298+0000","last_peered":"2023-10-09T10:46:08.279298+0000","last_clean":"2023-10-09T10:46:08.279298+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:46:08.279298+0000","last_undegraded":"2023-10-09T10:46:08.279298+0000","last_fullsized":"2023-10-09T10:46:08.279298+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":483,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279248+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:46:08.279248+0000","last_peered":"2023-10-09T10:46:08.279248+0000","last_clean":"2023-10-09T10:46:08.279248+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:46:08.279248+0000","last_undegraded":"2023-10-09T10:46:08.279248+0000","last_fullsized":"2023-10-09T10:46:08.279248+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"149'4526","reported_seq":4608,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:13.622405+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:46:13.622405+0000","last_peered":"2023-10-09T10:46:13.622405+0000","last_clean":"2023-10-09T10:46:13.622405+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:46:13.622405+0000","last_undegraded":"2023-10-09T10:46:13.622405+0000","last_fullsized":"2023-10-09T10:46:13.622405+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4526,"log_dups_size":0,"ondisk_log_size":4526,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209911808,"num_objects":3203,"num_object_clones":0,"num_object_copies":9609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3465,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":27,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278083+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:46:08.278083+0000","last_peered":"2023-10-09T10:46:08.278083+0000","last_clean":"2023-10-09T10:46:08.278083+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:46:08.278083+0000","last_undegraded":"2023-10-09T10:46:08.278083+0000","last_fullsized":"2023-10-09T10:46:08.278083+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":117,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.277617+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:46:08.277617+0000","last_peered":"2023-10-09T10:46:08.277617+0000","last_clean":"2023-10-09T10:46:08.277617+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:46:08.277617+0000","last_undegraded":"2023-10-09T10:46:08.277617+0000","last_fullsized":"2023-10-09T10:46:08.277617+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":483,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.281406+0000","last_change":"2023-10-09T10:46:05.620574+0000","last_active":"2023-10-09T10:46:08.281406+0000","last_peered":"2023-10-09T10:46:08.281406+0000","last_clean":"2023-10-09T10:46:08.281406+0000","last_became_active":"2023-10-09T10:46:05.620212+0000","last_became_peered":"2023-10-09T10:46:05.620212+0000","last_unstale":"2023-10-09T10:46:08.281406+0000","last_undegraded":"2023-10-09T10:46:08.281406+0000","last_fullsized":"2023-10-09T10:46:08.281406+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"149'6955","reported_seq":7865,"reported_epoch":149,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:46:10.980850+0000","last_change":"2023-10-09T10:46:06.806749+0000","last_active":"2023-10-09T10:46:10.980850+0000","last_peered":"2023-10-09T10:46:10.980850+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:46:02.739355+0000","last_became_peered":"2023-10-09T10:46:02.739355+0000","last_unstale":"2023-10-09T10:46:10.980850+0000","last_undegraded":"2023-10-09T10:46:02.608805+0000","last_fullsized":"2023-10-09T10:46:02.583589+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6955,"log_dups_size":0,"ondisk_log_size":6955,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:32.899634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437583872,"num_objects":6677,"num_object_clones":0,"num_object_copies":20031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":2231,"num_objects_unfound":0,"num_objects_dirty":6677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6955,"num_write_kb":436224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":416,"num_bytes_recovered":27262976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":2231},{"shards":"5,6","objects":4445}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":142,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.281384+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:46:08.281384+0000","last_peered":"2023-10-09T10:46:08.281384+0000","last_clean":"2023-10-09T10:46:08.281384+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:46:08.281384+0000","last_undegraded":"2023-10-09T10:46:08.281384+0000","last_fullsized":"2023-10-09T10:46:08.281384+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":96,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.285726+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:46:08.285726+0000","last_peered":"2023-10-09T10:46:08.285726+0000","last_clean":"2023-10-09T10:46:08.285726+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:46:08.285726+0000","last_undegraded":"2023-10-09T10:46:08.285726+0000","last_fullsized":"2023-10-09T10:46:08.285726+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":463,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278454+0000","last_change":"2023-10-09T10:46:02.631378+0000","last_active":"2023-10-09T10:46:08.278454+0000","last_peered":"2023-10-09T10:46:08.278454+0000","last_clean":"2023-10-09T10:46:08.278454+0000","last_became_active":"2023-10-09T10:46:02.629142+0000","last_became_peered":"2023-10-09T10:46:02.629142+0000","last_unstale":"2023-10-09T10:46:08.278454+0000","last_undegraded":"2023-10-09T10:46:08.278454+0000","last_fullsized":"2023-10-09T10:46:08.278454+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.b","version":"149'6883","reported_seq":6943,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:13.739140+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:46:13.739140+0000","last_peered":"2023-10-09T10:46:13.739140+0000","last_clean":"2023-10-09T10:46:13.739140+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:46:13.739140+0000","last_undegraded":"2023-10-09T10:46:13.739140+0000","last_fullsized":"2023-10-09T10:46:13.739140+0000","mapping_epoch":98,"log_start":"104'1000","ondisk_log_start":"104'1000","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5883,"log_dups_size":0,"ondisk_log_size":5883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412680192,"num_objects":6297,"num_object_clones":0,"num_object_copies":18891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6883,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":115,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.281440+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:46:08.281440+0000","last_peered":"2023-10-09T10:46:08.281440+0000","last_clean":"2023-10-09T10:46:08.281440+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:46:08.281440+0000","last_undegraded":"2023-10-09T10:46:08.281440+0000","last_fullsized":"2023-10-09T10:46:08.281440+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":140,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.339859+0000","last_change":"2023-10-09T10:43:35.493278+0000","last_active":"2023-10-09T10:46:08.339859+0000","last_peered":"2023-10-09T10:46:08.339859+0000","last_clean":"2023-10-09T10:46:08.339859+0000","last_became_active":"2023-10-09T10:43:35.492915+0000","last_became_peered":"2023-10-09T10:43:35.492915+0000","last_unstale":"2023-10-09T10:46:08.339859+0000","last_undegraded":"2023-10-09T10:46:08.339859+0000","last_fullsized":"2023-10-09T10:46:08.339859+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:42.227364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":483,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278877+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:46:08.278877+0000","last_peered":"2023-10-09T10:46:08.278877+0000","last_clean":"2023-10-09T10:46:08.278877+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:46:08.278877+0000","last_undegraded":"2023-10-09T10:46:08.278877+0000","last_fullsized":"2023-10-09T10:46:08.278877+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"149'4566","reported_seq":4640,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:14.711361+0000","last_change":"2023-10-09T10:45:06.950897+0000","last_active":"2023-10-09T10:46:14.711361+0000","last_peered":"2023-10-09T10:46:14.711361+0000","last_clean":"2023-10-09T10:46:14.711361+0000","last_became_active":"2023-10-09T10:44:50.063797+0000","last_became_peered":"2023-10-09T10:44:50.063797+0000","last_unstale":"2023-10-09T10:46:14.711361+0000","last_undegraded":"2023-10-09T10:46:14.711361+0000","last_fullsized":"2023-10-09T10:46:14.711361+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4566,"log_dups_size":0,"ondisk_log_size":4566,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208764928,"num_objects":3185,"num_object_clones":0,"num_object_copies":9555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3501,"num_write_kb":213984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":70,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.285924+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:46:08.285924+0000","last_peered":"2023-10-09T10:46:08.285924+0000","last_clean":"2023-10-09T10:46:08.285924+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:46:08.285924+0000","last_undegraded":"2023-10-09T10:46:08.285924+0000","last_fullsized":"2023-10-09T10:46:08.285924+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":86,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.280168+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:46:08.280168+0000","last_peered":"2023-10-09T10:46:08.280168+0000","last_clean":"2023-10-09T10:46:08.280168+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:46:08.280168+0000","last_undegraded":"2023-10-09T10:46:08.280168+0000","last_fullsized":"2023-10-09T10:46:08.280168+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":483,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278388+0000","last_change":"2023-10-09T10:46:05.620030+0000","last_active":"2023-10-09T10:46:08.278388+0000","last_peered":"2023-10-09T10:46:08.278388+0000","last_clean":"2023-10-09T10:46:08.278388+0000","last_became_active":"2023-10-09T10:46:05.619694+0000","last_became_peered":"2023-10-09T10:46:05.619694+0000","last_unstale":"2023-10-09T10:46:08.278388+0000","last_undegraded":"2023-10-09T10:46:08.278388+0000","last_fullsized":"2023-10-09T10:46:08.278388+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.9","version":"149'4470","reported_seq":4552,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:12.487219+0000","last_change":"2023-10-09T10:46:08.287244+0000","last_active":"2023-10-09T10:46:12.487219+0000","last_peered":"2023-10-09T10:46:12.487219+0000","last_clean":"2023-10-09T10:46:12.487219+0000","last_became_active":"2023-10-09T10:46:08.284512+0000","last_became_peered":"2023-10-09T10:46:08.284512+0000","last_unstale":"2023-10-09T10:46:12.487219+0000","last_undegraded":"2023-10-09T10:46:12.487219+0000","last_fullsized":"2023-10-09T10:46:12.487219+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4470,"log_dups_size":0,"ondisk_log_size":4470,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208797696,"num_objects":3186,"num_object_clones":0,"num_object_copies":9558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3428,"num_write_kb":211648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1303,"num_bytes_recovered":85393408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":114,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.285909+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:46:08.285909+0000","last_peered":"2023-10-09T10:46:08.285909+0000","last_clean":"2023-10-09T10:46:08.285909+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:46:08.285909+0000","last_undegraded":"2023-10-09T10:46:08.285909+0000","last_fullsized":"2023-10-09T10:46:08.285909+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":17,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279348+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:46:08.279348+0000","last_peered":"2023-10-09T10:46:08.279348+0000","last_clean":"2023-10-09T10:46:08.279348+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:46:08.279348+0000","last_undegraded":"2023-10-09T10:46:08.279348+0000","last_fullsized":"2023-10-09T10:46:08.279348+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":474,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278564+0000","last_change":"2023-10-09T10:46:05.618134+0000","last_active":"2023-10-09T10:46:08.278564+0000","last_peered":"2023-10-09T10:46:08.278564+0000","last_clean":"2023-10-09T10:46:08.278564+0000","last_became_active":"2023-10-09T10:46:05.616057+0000","last_became_peered":"2023-10-09T10:46:05.616057+0000","last_unstale":"2023-10-09T10:46:08.278564+0000","last_undegraded":"2023-10-09T10:46:08.278564+0000","last_fullsized":"2023-10-09T10:46:08.278564+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"149'6756","reported_seq":6815,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:10.959445+0000","last_change":"2023-10-09T10:44:27.574527+0000","last_active":"2023-10-09T10:46:10.959445+0000","last_peered":"2023-10-09T10:46:10.959445+0000","last_clean":"2023-10-09T10:46:10.959445+0000","last_became_active":"2023-10-09T10:44:27.574205+0000","last_became_peered":"2023-10-09T10:44:27.574205+0000","last_unstale":"2023-10-09T10:46:10.959445+0000","last_undegraded":"2023-10-09T10:46:10.959445+0000","last_fullsized":"2023-10-09T10:46:10.959445+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6756,"log_dups_size":0,"ondisk_log_size":6756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:54.868034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424542208,"num_objects":6478,"num_object_clones":0,"num_object_copies":19434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6756,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":141,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.276024+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:46:08.276024+0000","last_peered":"2023-10-09T10:46:08.276024+0000","last_clean":"2023-10-09T10:46:08.276024+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:46:08.276024+0000","last_undegraded":"2023-10-09T10:46:08.276024+0000","last_fullsized":"2023-10-09T10:46:08.276024+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":113,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.339883+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:46:08.339883+0000","last_peered":"2023-10-09T10:46:08.339883+0000","last_clean":"2023-10-09T10:46:08.339883+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:46:08.339883+0000","last_undegraded":"2023-10-09T10:46:08.339883+0000","last_fullsized":"2023-10-09T10:46:08.339883+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":463,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278546+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:46:08.278546+0000","last_peered":"2023-10-09T10:46:08.278546+0000","last_clean":"2023-10-09T10:46:08.278546+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:46:08.278546+0000","last_undegraded":"2023-10-09T10:46:08.278546+0000","last_fullsized":"2023-10-09T10:46:08.278546+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.f","version":"149'6829","reported_seq":6889,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:14.718942+0000","last_change":"2023-10-09T10:44:27.574938+0000","last_active":"2023-10-09T10:46:14.718942+0000","last_peered":"2023-10-09T10:46:14.718942+0000","last_clean":"2023-10-09T10:46:14.718942+0000","last_became_active":"2023-10-09T10:44:27.574662+0000","last_became_peered":"2023-10-09T10:44:27.574662+0000","last_unstale":"2023-10-09T10:46:14.718942+0000","last_undegraded":"2023-10-09T10:46:14.718942+0000","last_fullsized":"2023-10-09T10:46:14.718942+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6829,"log_dups_size":0,"ondisk_log_size":6829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:35:19.975925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405864448,"num_objects":6193,"num_object_clones":0,"num_object_copies":18579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6829,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":88,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.340801+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:46:08.340801+0000","last_peered":"2023-10-09T10:46:08.340801+0000","last_clean":"2023-10-09T10:46:08.340801+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:46:08.340801+0000","last_undegraded":"2023-10-09T10:46:08.340801+0000","last_fullsized":"2023-10-09T10:46:08.340801+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":130,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.340106+0000","last_change":"2023-10-09T10:43:06.676367+0000","last_active":"2023-10-09T10:46:08.340106+0000","last_peered":"2023-10-09T10:46:08.340106+0000","last_clean":"2023-10-09T10:46:08.340106+0000","last_became_active":"2023-10-09T10:43:06.676017+0000","last_became_peered":"2023-10-09T10:43:06.676017+0000","last_unstale":"2023-10-09T10:46:08.340106+0000","last_undegraded":"2023-10-09T10:46:08.340106+0000","last_fullsized":"2023-10-09T10:46:08.340106+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:54.387617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":483,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278400+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:46:08.278400+0000","last_peered":"2023-10-09T10:46:08.278400+0000","last_clean":"2023-10-09T10:46:08.278400+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:46:08.278400+0000","last_undegraded":"2023-10-09T10:46:08.278400+0000","last_fullsized":"2023-10-09T10:46:08.278400+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"149'6932","reported_seq":6992,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:13.767513+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:46:13.767513+0000","last_peered":"2023-10-09T10:46:13.767513+0000","last_clean":"2023-10-09T10:46:13.767513+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:46:13.767513+0000","last_undegraded":"2023-10-09T10:46:13.767513+0000","last_fullsized":"2023-10-09T10:46:13.767513+0000","mapping_epoch":98,"log_start":"104'1000","ondisk_log_start":"104'1000","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5932,"log_dups_size":0,"ondisk_log_size":5932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416284672,"num_objects":6352,"num_object_clones":0,"num_object_copies":19056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6932,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":114,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.339791+0000","last_change":"2023-10-09T10:43:06.676577+0000","last_active":"2023-10-09T10:46:08.339791+0000","last_peered":"2023-10-09T10:46:08.339791+0000","last_clean":"2023-10-09T10:46:08.339791+0000","last_became_active":"2023-10-09T10:43:06.676165+0000","last_became_peered":"2023-10-09T10:43:06.676165+0000","last_unstale":"2023-10-09T10:46:08.339791+0000","last_undegraded":"2023-10-09T10:46:08.339791+0000","last_fullsized":"2023-10-09T10:46:08.339791+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:52.614985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":113,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.340548+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:46:08.340548+0000","last_peered":"2023-10-09T10:46:08.340548+0000","last_clean":"2023-10-09T10:46:08.340548+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:46:08.340548+0000","last_undegraded":"2023-10-09T10:46:08.340548+0000","last_fullsized":"2023-10-09T10:46:08.340548+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":474,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279148+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:46:08.279148+0000","last_peered":"2023-10-09T10:46:08.279148+0000","last_clean":"2023-10-09T10:46:08.279148+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:46:08.279148+0000","last_undegraded":"2023-10-09T10:46:08.279148+0000","last_fullsized":"2023-10-09T10:46:08.279148+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"149'6848","reported_seq":6927,"reported_epoch":149,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:46:10.955557+0000","last_change":"2023-10-09T10:46:02.754507+0000","last_active":"2023-10-09T10:46:10.955557+0000","last_peered":"2023-10-09T10:46:10.955557+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:46:02.747510+0000","last_became_peered":"2023-10-09T10:46:02.747510+0000","last_unstale":"2023-10-09T10:46:10.955557+0000","last_undegraded":"2023-10-09T10:46:02.605987+0000","last_fullsized":"2023-10-09T10:46:02.585248+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6848,"log_dups_size":0,"ondisk_log_size":6848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:59.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429064216,"num_objects":6548,"num_object_clones":0,"num_object_copies":19644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6548,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6848,"num_write_kb":428609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,2,5","objects":2718},{"shards":"1,5","objects":3830}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":118,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.275751+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:46:08.275751+0000","last_peered":"2023-10-09T10:46:08.275751+0000","last_clean":"2023-10-09T10:46:08.275751+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:46:08.275751+0000","last_undegraded":"2023-10-09T10:46:08.275751+0000","last_fullsized":"2023-10-09T10:46:08.275751+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":100,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279635+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:46:08.279635+0000","last_peered":"2023-10-09T10:46:08.279635+0000","last_clean":"2023-10-09T10:46:08.279635+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:46:08.279635+0000","last_undegraded":"2023-10-09T10:46:08.279635+0000","last_fullsized":"2023-10-09T10:46:08.279635+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":483,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279028+0000","last_change":"2023-10-09T10:46:05.617983+0000","last_active":"2023-10-09T10:46:08.279028+0000","last_peered":"2023-10-09T10:46:08.279028+0000","last_clean":"2023-10-09T10:46:08.279028+0000","last_became_active":"2023-10-09T10:46:05.616021+0000","last_became_peered":"2023-10-09T10:46:05.616021+0000","last_unstale":"2023-10-09T10:46:08.279028+0000","last_undegraded":"2023-10-09T10:46:08.279028+0000","last_fullsized":"2023-10-09T10:46:08.279028+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":143,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279074+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:46:08.279074+0000","last_peered":"2023-10-09T10:46:08.279074+0000","last_clean":"2023-10-09T10:46:08.279074+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:46:08.279074+0000","last_undegraded":"2023-10-09T10:46:08.279074+0000","last_fullsized":"2023-10-09T10:46:08.279074+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":131,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.286075+0000","last_change":"2023-10-09T10:43:35.486524+0000","last_active":"2023-10-09T10:46:08.286075+0000","last_peered":"2023-10-09T10:46:08.286075+0000","last_clean":"2023-10-09T10:46:08.286075+0000","last_became_active":"2023-10-09T10:43:35.486152+0000","last_became_peered":"2023-10-09T10:43:35.486152+0000","last_unstale":"2023-10-09T10:46:08.286075+0000","last_undegraded":"2023-10-09T10:46:08.286075+0000","last_fullsized":"2023-10-09T10:46:08.286075+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:47.200522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":85,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279966+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:46:08.279966+0000","last_peered":"2023-10-09T10:46:08.279966+0000","last_clean":"2023-10-09T10:46:08.279966+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:46:08.279966+0000","last_undegraded":"2023-10-09T10:46:08.279966+0000","last_fullsized":"2023-10-09T10:46:08.279966+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"149'4532","reported_seq":8528,"reported_epoch":149,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:46:14.717113+0000","last_change":"2023-10-09T10:46:03.681935+0000","last_active":"2023-10-09T10:46:14.717113+0000","last_peered":"2023-10-09T10:46:14.717113+0000","last_clean":"2023-10-09T10:46:02.181912+0000","last_became_active":"2023-10-09T10:46:03.679114+0000","last_became_peered":"2023-10-09T10:46:03.679114+0000","last_unstale":"2023-10-09T10:46:14.717113+0000","last_undegraded":"2023-10-09T10:46:14.717113+0000","last_fullsized":"2023-10-09T10:46:14.717113+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4532,"log_dups_size":0,"ondisk_log_size":4532,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207060992,"num_objects":3159,"num_object_clones":0,"num_object_copies":9477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3159,"num_objects_unfound":0,"num_objects_dirty":3159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3485,"num_write_kb":212640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":3159}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":482,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278056+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:46:08.278056+0000","last_peered":"2023-10-09T10:46:08.278056+0000","last_clean":"2023-10-09T10:46:08.278056+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:46:08.278056+0000","last_undegraded":"2023-10-09T10:46:08.278056+0000","last_fullsized":"2023-10-09T10:46:08.278056+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"149'4537","reported_seq":4687,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:13.718680+0000","last_change":"2023-10-09T10:46:02.760963+0000","last_active":"2023-10-09T10:46:13.718680+0000","last_peered":"2023-10-09T10:46:13.718680+0000","last_clean":"2023-10-09T10:46:13.718680+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:46:13.718680+0000","last_undegraded":"2023-10-09T10:46:13.718680+0000","last_fullsized":"2023-10-09T10:46:13.718680+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"141'4416","last_scrub_stamp":"2023-10-09T10:46:02.760810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:46:02.760810+0000","objects_scrubbed":3378,"log_size":4537,"log_dups_size":0,"ondisk_log_size":4537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:35.863002+0000","scrub_duration":0.47769323899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213450752,"num_objects":3257,"num_object_clones":0,"num_object_copies":9771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":121,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":118,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.285761+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:46:08.285761+0000","last_peered":"2023-10-09T10:46:08.285761+0000","last_clean":"2023-10-09T10:46:08.285761+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:46:08.285761+0000","last_undegraded":"2023-10-09T10:46:08.285761+0000","last_fullsized":"2023-10-09T10:46:08.285761+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":134,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.340223+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:46:08.340223+0000","last_peered":"2023-10-09T10:46:08.340223+0000","last_clean":"2023-10-09T10:46:08.340223+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:46:08.340223+0000","last_undegraded":"2023-10-09T10:46:08.340223+0000","last_fullsized":"2023-10-09T10:46:08.340223+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":115,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279139+0000","last_change":"2023-10-09T10:43:35.489332+0000","last_active":"2023-10-09T10:46:08.279139+0000","last_peered":"2023-10-09T10:46:08.279139+0000","last_clean":"2023-10-09T10:46:08.279139+0000","last_became_active":"2023-10-09T10:43:35.489044+0000","last_became_peered":"2023-10-09T10:43:35.489044+0000","last_unstale":"2023-10-09T10:46:08.279139+0000","last_undegraded":"2023-10-09T10:46:08.279139+0000","last_fullsized":"2023-10-09T10:46:08.279139+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:42.168915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":483,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279064+0000","last_change":"2023-10-09T10:46:05.620057+0000","last_active":"2023-10-09T10:46:08.279064+0000","last_peered":"2023-10-09T10:46:08.279064+0000","last_clean":"2023-10-09T10:46:08.279064+0000","last_became_active":"2023-10-09T10:46:05.619703+0000","last_became_peered":"2023-10-09T10:46:05.619703+0000","last_unstale":"2023-10-09T10:46:08.279064+0000","last_undegraded":"2023-10-09T10:46:08.279064+0000","last_fullsized":"2023-10-09T10:46:08.279064+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"149'4538","reported_seq":4623,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:13.776041+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:46:13.776041+0000","last_peered":"2023-10-09T10:46:13.776041+0000","last_clean":"2023-10-09T10:46:13.776041+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:46:13.776041+0000","last_undegraded":"2023-10-09T10:46:13.776041+0000","last_fullsized":"2023-10-09T10:46:13.776041+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4538,"log_dups_size":0,"ondisk_log_size":4538,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209911808,"num_objects":3203,"num_object_clones":0,"num_object_copies":9609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3477,"num_write_kb":213760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":154,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.283493+0000","last_change":"2023-10-09T10:46:02.643455+0000","last_active":"2023-10-09T10:46:08.283493+0000","last_peered":"2023-10-09T10:46:08.283493+0000","last_clean":"2023-10-09T10:46:08.283493+0000","last_became_active":"2023-10-09T10:46:02.643110+0000","last_became_peered":"2023-10-09T10:46:02.643110+0000","last_unstale":"2023-10-09T10:46:08.283493+0000","last_undegraded":"2023-10-09T10:46:08.283493+0000","last_fullsized":"2023-10-09T10:46:08.283493+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:44:15.909240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":100,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.280745+0000","last_change":"2023-10-09T10:43:06.672862+0000","last_active":"2023-10-09T10:46:08.280745+0000","last_peered":"2023-10-09T10:46:08.280745+0000","last_clean":"2023-10-09T10:46:08.280745+0000","last_became_active":"2023-10-09T10:43:06.672572+0000","last_became_peered":"2023-10-09T10:43:06.672572+0000","last_unstale":"2023-10-09T10:46:08.280745+0000","last_undegraded":"2023-10-09T10:46:08.280745+0000","last_fullsized":"2023-10-09T10:46:08.280745+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:23.743910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":138,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279707+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:46:08.279707+0000","last_peered":"2023-10-09T10:46:08.279707+0000","last_clean":"2023-10-09T10:46:08.279707+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:46:08.279707+0000","last_undegraded":"2023-10-09T10:46:08.279707+0000","last_fullsized":"2023-10-09T10:46:08.279707+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"149'4405","reported_seq":4478,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:10.851226+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:46:10.851226+0000","last_peered":"2023-10-09T10:46:10.851226+0000","last_clean":"2023-10-09T10:46:10.851226+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:46:10.851226+0000","last_undegraded":"2023-10-09T10:46:10.851226+0000","last_fullsized":"2023-10-09T10:46:10.851226+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4405,"log_dups_size":0,"ondisk_log_size":4405,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213188608,"num_objects":3253,"num_object_clones":0,"num_object_copies":9759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3397,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":118,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.285675+0000","last_change":"2023-10-09T10:43:06.673931+0000","last_active":"2023-10-09T10:46:08.285675+0000","last_peered":"2023-10-09T10:46:08.285675+0000","last_clean":"2023-10-09T10:46:08.285675+0000","last_became_active":"2023-10-09T10:43:06.673605+0000","last_became_peered":"2023-10-09T10:43:06.673605+0000","last_unstale":"2023-10-09T10:46:08.285675+0000","last_undegraded":"2023-10-09T10:46:08.285675+0000","last_fullsized":"2023-10-09T10:46:08.285675+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:19.088672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":483,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.277981+0000","last_change":"2023-10-09T10:46:05.617146+0000","last_active":"2023-10-09T10:46:08.277981+0000","last_peered":"2023-10-09T10:46:08.277981+0000","last_clean":"2023-10-09T10:46:08.277981+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:46:08.277981+0000","last_undegraded":"2023-10-09T10:46:08.277981+0000","last_fullsized":"2023-10-09T10:46:08.277981+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":76,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.276355+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:46:08.276355+0000","last_peered":"2023-10-09T10:46:08.276355+0000","last_clean":"2023-10-09T10:46:08.276355+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:46:08.276355+0000","last_undegraded":"2023-10-09T10:46:08.276355+0000","last_fullsized":"2023-10-09T10:46:08.276355+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":46,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.339816+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:46:08.339816+0000","last_peered":"2023-10-09T10:46:08.339816+0000","last_clean":"2023-10-09T10:46:08.339816+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:46:08.339816+0000","last_undegraded":"2023-10-09T10:46:08.339816+0000","last_fullsized":"2023-10-09T10:46:08.339816+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":483,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278309+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:46:08.278309+0000","last_peered":"2023-10-09T10:46:08.278309+0000","last_clean":"2023-10-09T10:46:08.278309+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:46:08.278309+0000","last_undegraded":"2023-10-09T10:46:08.278309+0000","last_fullsized":"2023-10-09T10:46:08.278309+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":139,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.280883+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:46:08.280883+0000","last_peered":"2023-10-09T10:46:08.280883+0000","last_clean":"2023-10-09T10:46:08.280883+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:46:08.280883+0000","last_undegraded":"2023-10-09T10:46:08.280883+0000","last_fullsized":"2023-10-09T10:46:08.280883+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":17,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.277973+0000","last_change":"2023-10-09T10:46:02.652277+0000","last_active":"2023-10-09T10:46:08.277973+0000","last_peered":"2023-10-09T10:46:08.277973+0000","last_clean":"2023-10-09T10:46:08.277973+0000","last_became_active":"2023-10-09T10:46:02.642109+0000","last_became_peered":"2023-10-09T10:46:02.642109+0000","last_unstale":"2023-10-09T10:46:08.277973+0000","last_undegraded":"2023-10-09T10:46:08.277973+0000","last_fullsized":"2023-10-09T10:46:08.277973+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:04.271063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":17,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279234+0000","last_change":"2023-10-09T10:46:02.615790+0000","last_active":"2023-10-09T10:46:08.279234+0000","last_peered":"2023-10-09T10:46:08.279234+0000","last_clean":"2023-10-09T10:46:08.279234+0000","last_became_active":"2023-10-09T10:46:02.615452+0000","last_became_peered":"2023-10-09T10:46:02.615452+0000","last_unstale":"2023-10-09T10:46:08.279234+0000","last_undegraded":"2023-10-09T10:46:08.279234+0000","last_fullsized":"2023-10-09T10:46:08.279234+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:49:21.175606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"149'4522","reported_seq":4641,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:11.418834+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:46:11.418834+0000","last_peered":"2023-10-09T10:46:11.418834+0000","last_clean":"2023-10-09T10:46:11.418834+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:46:11.418834+0000","last_undegraded":"2023-10-09T10:46:11.418834+0000","last_fullsized":"2023-10-09T10:46:11.418834+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":4522,"log_dups_size":0,"ondisk_log_size":4522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213385216,"num_objects":3256,"num_object_clones":0,"num_object_copies":9768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1492,"num_write_kb":89664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":483,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278479+0000","last_change":"2023-10-09T10:46:05.620764+0000","last_active":"2023-10-09T10:46:08.278479+0000","last_peered":"2023-10-09T10:46:08.278479+0000","last_clean":"2023-10-09T10:46:08.278479+0000","last_became_active":"2023-10-09T10:46:05.619870+0000","last_became_peered":"2023-10-09T10:46:05.619870+0000","last_unstale":"2023-10-09T10:46:08.278479+0000","last_undegraded":"2023-10-09T10:46:08.278479+0000","last_fullsized":"2023-10-09T10:46:08.278479+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":137,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.281429+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:46:08.281429+0000","last_peered":"2023-10-09T10:46:08.281429+0000","last_clean":"2023-10-09T10:46:08.281429+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:46:08.281429+0000","last_undegraded":"2023-10-09T10:46:08.281429+0000","last_fullsized":"2023-10-09T10:46:08.281429+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":100,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279221+0000","last_change":"2023-10-09T10:43:06.671356+0000","last_active":"2023-10-09T10:46:08.279221+0000","last_peered":"2023-10-09T10:46:08.279221+0000","last_clean":"2023-10-09T10:46:08.279221+0000","last_became_active":"2023-10-09T10:43:06.671040+0000","last_became_peered":"2023-10-09T10:43:06.671040+0000","last_unstale":"2023-10-09T10:46:08.279221+0000","last_undegraded":"2023-10-09T10:46:08.279221+0000","last_fullsized":"2023-10-09T10:46:08.279221+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:47.601448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":17,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278476+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:46:08.278476+0000","last_peered":"2023-10-09T10:46:08.278476+0000","last_clean":"2023-10-09T10:46:08.278476+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:46:08.278476+0000","last_undegraded":"2023-10-09T10:46:08.278476+0000","last_fullsized":"2023-10-09T10:46:08.278476+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"149'4389","reported_seq":4459,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:11.371979+0000","last_change":"2023-10-09T10:46:08.279429+0000","last_active":"2023-10-09T10:46:11.371979+0000","last_peered":"2023-10-09T10:46:11.371979+0000","last_clean":"2023-10-09T10:46:11.371979+0000","last_became_active":"2023-10-09T10:44:50.064940+0000","last_became_peered":"2023-10-09T10:44:50.064940+0000","last_unstale":"2023-10-09T10:46:11.371979+0000","last_undegraded":"2023-10-09T10:46:11.371979+0000","last_fullsized":"2023-10-09T10:46:11.371979+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4389,"log_dups_size":0,"ondisk_log_size":4389,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211124224,"num_objects":3221,"num_object_clones":0,"num_object_copies":9663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3381,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":483,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.281243+0000","last_change":"2023-10-09T10:46:05.618443+0000","last_active":"2023-10-09T10:46:08.281243+0000","last_peered":"2023-10-09T10:46:08.281243+0000","last_clean":"2023-10-09T10:46:08.281243+0000","last_became_active":"2023-10-09T10:46:05.616588+0000","last_became_peered":"2023-10-09T10:46:05.616588+0000","last_unstale":"2023-10-09T10:46:08.281243+0000","last_undegraded":"2023-10-09T10:46:08.281243+0000","last_fullsized":"2023-10-09T10:46:08.281243+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":141,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.277542+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:46:08.277542+0000","last_peered":"2023-10-09T10:46:08.277542+0000","last_clean":"2023-10-09T10:46:08.277542+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:46:08.277542+0000","last_undegraded":"2023-10-09T10:46:08.277542+0000","last_fullsized":"2023-10-09T10:46:08.277542+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":85,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.279725+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:46:08.279725+0000","last_peered":"2023-10-09T10:46:08.279725+0000","last_clean":"2023-10-09T10:46:08.279725+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:46:08.279725+0000","last_undegraded":"2023-10-09T10:46:08.279725+0000","last_fullsized":"2023-10-09T10:46:08.279725+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":130,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.276106+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:46:08.276106+0000","last_peered":"2023-10-09T10:46:08.276106+0000","last_clean":"2023-10-09T10:46:08.276106+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:46:08.276106+0000","last_undegraded":"2023-10-09T10:46:08.276106+0000","last_fullsized":"2023-10-09T10:46:08.276106+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"149'4438","reported_seq":4535,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:10.926066+0000","last_change":"2023-10-09T10:46:09.481639+0000","last_active":"2023-10-09T10:46:10.926066+0000","last_peered":"2023-10-09T10:46:10.926066+0000","last_clean":"2023-10-09T10:46:10.926066+0000","last_became_active":"2023-10-09T10:44:50.060284+0000","last_became_peered":"2023-10-09T10:44:50.060284+0000","last_unstale":"2023-10-09T10:46:10.926066+0000","last_undegraded":"2023-10-09T10:46:10.926066+0000","last_fullsized":"2023-10-09T10:46:10.926066+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4438,"log_dups_size":0,"ondisk_log_size":4438,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213876736,"num_objects":3263,"num_object_clones":0,"num_object_copies":9789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3401,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":482,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278221+0000","last_change":"2023-10-09T10:46:05.617589+0000","last_active":"2023-10-09T10:46:08.278221+0000","last_peered":"2023-10-09T10:46:08.278221+0000","last_clean":"2023-10-09T10:46:08.278221+0000","last_became_active":"2023-10-09T10:46:05.616400+0000","last_became_peered":"2023-10-09T10:46:05.616400+0000","last_unstale":"2023-10-09T10:46:08.278221+0000","last_undegraded":"2023-10-09T10:46:08.278221+0000","last_fullsized":"2023-10-09T10:46:08.278221+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":96,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.282397+0000","last_change":"2023-10-09T10:45:07.620938+0000","last_active":"2023-10-09T10:46:08.282397+0000","last_peered":"2023-10-09T10:46:08.282397+0000","last_clean":"2023-10-09T10:46:08.282397+0000","last_became_active":"2023-10-09T10:45:07.620627+0000","last_became_peered":"2023-10-09T10:45:07.620627+0000","last_unstale":"2023-10-09T10:46:08.282397+0000","last_undegraded":"2023-10-09T10:46:08.282397+0000","last_fullsized":"2023-10-09T10:46:08.282397+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:13.415622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":21,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.281521+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:46:08.281521+0000","last_peered":"2023-10-09T10:46:08.281521+0000","last_clean":"2023-10-09T10:46:08.281521+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:46:08.281521+0000","last_undegraded":"2023-10-09T10:46:08.281521+0000","last_fullsized":"2023-10-09T10:46:08.281521+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":96,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.339707+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:46:08.339707+0000","last_peered":"2023-10-09T10:46:08.339707+0000","last_clean":"2023-10-09T10:46:08.339707+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:46:08.339707+0000","last_undegraded":"2023-10-09T10:46:08.339707+0000","last_fullsized":"2023-10-09T10:46:08.339707+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"149'4390","reported_seq":4536,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:10.804759+0000","last_change":"2023-10-09T10:45:05.770329+0000","last_active":"2023-10-09T10:46:10.804759+0000","last_peered":"2023-10-09T10:46:10.804759+0000","last_clean":"2023-10-09T10:46:10.804759+0000","last_became_active":"2023-10-09T10:44:50.061195+0000","last_became_peered":"2023-10-09T10:44:50.061195+0000","last_unstale":"2023-10-09T10:46:10.804759+0000","last_undegraded":"2023-10-09T10:46:10.804759+0000","last_fullsized":"2023-10-09T10:46:10.804759+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":4390,"log_dups_size":0,"ondisk_log_size":4390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:02.077286+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212074496,"num_objects":3236,"num_object_clones":0,"num_object_copies":9708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1745,"num_write_kb":107264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":17,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278562+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:46:08.278562+0000","last_peered":"2023-10-09T10:46:08.278562+0000","last_clean":"2023-10-09T10:46:08.278562+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:46:08.278562+0000","last_undegraded":"2023-10-09T10:46:08.278562+0000","last_fullsized":"2023-10-09T10:46:08.278562+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":135,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.282102+0000","last_change":"2023-10-09T10:43:35.489655+0000","last_active":"2023-10-09T10:46:08.282102+0000","last_peered":"2023-10-09T10:46:08.282102+0000","last_clean":"2023-10-09T10:46:08.282102+0000","last_became_active":"2023-10-09T10:43:35.489149+0000","last_became_peered":"2023-10-09T10:43:35.489149+0000","last_unstale":"2023-10-09T10:46:08.282102+0000","last_undegraded":"2023-10-09T10:46:08.282102+0000","last_fullsized":"2023-10-09T10:46:08.282102+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:30.598962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"149'4524","reported_seq":8470,"reported_epoch":149,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:46:14.621687+0000","last_change":"2023-10-09T10:46:03.657336+0000","last_active":"2023-10-09T10:46:14.621687+0000","last_peered":"2023-10-09T10:46:14.621687+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:46:03.652844+0000","last_became_peered":"2023-10-09T10:46:03.652844+0000","last_unstale":"2023-10-09T10:46:14.621687+0000","last_undegraded":"2023-10-09T10:46:03.599894+0000","last_fullsized":"2023-10-09T10:46:03.585379+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4524,"log_dups_size":0,"ondisk_log_size":4524,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209190912,"num_objects":3192,"num_object_clones":0,"num_object_copies":9576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1406,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3482,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1932,"num_bytes_recovered":126615552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[2,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1406},{"shards":"2,7","objects":1786}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":116,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.285989+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:46:08.285989+0000","last_peered":"2023-10-09T10:46:08.285989+0000","last_clean":"2023-10-09T10:46:08.285989+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:46:08.285989+0000","last_undegraded":"2023-10-09T10:46:08.285989+0000","last_fullsized":"2023-10-09T10:46:08.285989+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":130,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.340339+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:46:08.340339+0000","last_peered":"2023-10-09T10:46:08.340339+0000","last_clean":"2023-10-09T10:46:08.340339+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:46:08.340339+0000","last_undegraded":"2023-10-09T10:46:08.340339+0000","last_fullsized":"2023-10-09T10:46:08.340339+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"149'4468","reported_seq":4542,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T10:46:14.577873+0000","last_change":"2023-10-09T10:45:12.912569+0000","last_active":"2023-10-09T10:46:14.577873+0000","last_peered":"2023-10-09T10:46:14.577873+0000","last_clean":"2023-10-09T10:46:14.577873+0000","last_became_active":"2023-10-09T10:44:50.076612+0000","last_became_peered":"2023-10-09T10:44:50.076612+0000","last_unstale":"2023-10-09T10:46:14.577873+0000","last_undegraded":"2023-10-09T10:46:14.577873+0000","last_fullsized":"2023-10-09T10:46:14.577873+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":4468,"log_dups_size":0,"ondisk_log_size":4468,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204701696,"num_objects":3124,"num_object_clones":0,"num_object_copies":9372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3404,"num_write_kb":208864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":99,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.286155+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:46:08.286155+0000","last_peered":"2023-10-09T10:46:08.286155+0000","last_clean":"2023-10-09T10:46:08.286155+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:46:08.286155+0000","last_undegraded":"2023-10-09T10:46:08.286155+0000","last_fullsized":"2023-10-09T10:46:08.286155+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":135,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.276417+0000","last_change":"2023-10-09T10:46:02.633774+0000","last_active":"2023-10-09T10:46:08.276417+0000","last_peered":"2023-10-09T10:46:08.276417+0000","last_clean":"2023-10-09T10:46:08.276417+0000","last_became_active":"2023-10-09T10:46:02.633507+0000","last_became_peered":"2023-10-09T10:46:02.633507+0000","last_unstale":"2023-10-09T10:46:08.276417+0000","last_undegraded":"2023-10-09T10:46:08.276417+0000","last_fullsized":"2023-10-09T10:46:08.276417+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:21:11.356777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":123,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.277610+0000","last_change":"2023-10-09T10:43:35.485790+0000","last_active":"2023-10-09T10:46:08.277610+0000","last_peered":"2023-10-09T10:46:08.277610+0000","last_clean":"2023-10-09T10:46:08.277610+0000","last_became_active":"2023-10-09T10:43:35.485436+0000","last_became_peered":"2023-10-09T10:43:35.485436+0000","last_unstale":"2023-10-09T10:46:08.277610+0000","last_undegraded":"2023-10-09T10:46:08.277610+0000","last_fullsized":"2023-10-09T10:46:08.277610+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:45:20.897767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":154,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.275902+0000","last_change":"2023-10-09T10:46:02.694921+0000","last_active":"2023-10-09T10:46:08.275902+0000","last_peered":"2023-10-09T10:46:08.275902+0000","last_clean":"2023-10-09T10:46:08.275902+0000","last_became_active":"2023-10-09T10:46:02.694564+0000","last_became_peered":"2023-10-09T10:46:02.694564+0000","last_unstale":"2023-10-09T10:46:08.275902+0000","last_undegraded":"2023-10-09T10:46:08.275902+0000","last_fullsized":"2023-10-09T10:46:08.275902+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:24.937192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":23,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.278954+0000","last_change":"2023-10-09T10:46:02.636151+0000","last_active":"2023-10-09T10:46:08.278954+0000","last_peered":"2023-10-09T10:46:08.278954+0000","last_clean":"2023-10-09T10:46:08.278954+0000","last_became_active":"2023-10-09T10:46:02.615571+0000","last_became_peered":"2023-10-09T10:46:02.615571+0000","last_unstale":"2023-10-09T10:46:08.278954+0000","last_undegraded":"2023-10-09T10:46:08.278954+0000","last_fullsized":"2023-10-09T10:46:08.278954+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:38.225155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":113,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-10-09T10:46:08.339703+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:46:08.339703+0000","last_peered":"2023-10-09T10:46:08.339703+0000","last_clean":"2023-10-09T10:46:08.339703+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:46:08.339703+0000","last_undegraded":"2023-10-09T10:46:08.339703+0000","last_fullsized":"2023-10-09T10:46:08.339703+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":6742114328,"num_objects":102878,"num_object_clones":0,"num_object_copies":308634,"num_objects_missing_on_primary":897,"num_objects_missing":897,"num_objects_degraded":8135,"num_objects_misplaced":8700,"num_objects_unfound":0,"num_objects_dirty":102878,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":101330,"num_write_kb":6258529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11404,"num_bytes_recovered":747372544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21110140928,"data_stored":21110128712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128765,"ondisk_log_size":128765,"up":78,"acting":74,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":598,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332235,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3025124,"kb_used_data":2719516,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":90720028,"statfs":{"total":95995035648,"available":92897308672,"internally_reserved":0,"allocated":2784784384,"data_stored":2784217068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:45:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2609999999999999,"5min":0.68100000000000005,"15min":0.61299999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.4349999999999996,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.876,"5min":0.64700000000000002,"15min":0.64300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.9260000000000002,"5min":2.9260000000000002,"15min":2.9260000000000002},"last":0.309}]},{"osd":1,"last update":"Mon Oct 9 10:45:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":0.72099999999999997,"15min":0.60099999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":3.9860000000000002,"5min":3.9860000000000002,"15min":3.9860000000000002},"last":1.3839999999999999},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.66600000000000004,"15min":0.57599999999999996},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":3.9860000000000002,"5min":6.4939999999999998,"15min":6.4939999999999998},"last":0.94899999999999995}]},{"osd":2,"last update":"Mon Oct 9 10:45:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":0.69699999999999995,"15min":0.57299999999999995},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.2690000000000001,"5min":6.2690000000000001,"15min":6.2690000000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.84999999999999998,"5min":0.58999999999999997,"15min":0.54700000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.157,"5min":2.157,"15min":2.157},"last":0.873}]},{"osd":3,"last update":"Mon Oct 9 10:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":0.80900000000000005,"15min":0.66000000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":8.1639999999999997,"5min":8.1639999999999997,"15min":8.1639999999999997},"last":5.3840000000000003},{"interface":"front","average":{"1min":1.9419999999999999,"5min":0.86799999999999999,"15min":0.68899999999999995},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":12.451000000000001,"5min":12.451000000000001,"15min":12.451000000000001},"last":1.9419999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:45:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.51600000000000001,"15min":0.48499999999999999},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.1600000000000001,"5min":3.1600000000000001,"15min":3.1600000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.54200000000000004,"15min":0.49399999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.1699999999999999,"5min":3.1699999999999999,"15min":3.1699999999999999},"last":0.37}]},{"osd":5,"last update":"Mon Oct 9 10:45:16 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.55000000000000004,"15min":0.51700000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":2.1989999999999998,"5min":2.1989999999999998,"15min":2.1989999999999998},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.58699999999999997,"15min":0.55200000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.25700000000000001},"max":{"1min":2.7549999999999999,"5min":2.7549999999999999,"15min":2.7549999999999999},"last":0.54400000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:45:16 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.628,"15min":0.57399999999999995},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.1880000000000002,"5min":3.1880000000000002,"15min":3.1880000000000002},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.58399999999999996,"15min":0.55200000000000005},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":0.46700000000000003}]}]},{"osd":6,"up_from":44,"seq":188978561166,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2498444,"kb_used_data":2201156,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91246708,"statfs":{"total":95995035648,"available":93436628992,"internally_reserved":0,"allocated":2253983744,"data_stored":2253432546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:45:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":0.66300000000000003,"15min":0.628},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.4219999999999997,"5min":5.4219999999999997,"15min":5.4219999999999997},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.63800000000000001,"15min":0.61799999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.7250000000000001,"5min":2.7250000000000001,"15min":2.7250000000000001},"last":0.45400000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:45:42 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":0.82199999999999995,"15min":0.66800000000000004},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":16.747,"5min":16.747,"15min":16.747},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.9690000000000001,"5min":0.83399999999999996,"15min":0.64500000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":16.734000000000002,"5min":16.734000000000002,"15min":16.734000000000002},"last":0.441}]},{"osd":2,"last update":"Mon Oct 9 10:45:07 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.54900000000000004,"15min":0.53900000000000003},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":2.0960000000000001,"5min":2.0960000000000001,"15min":2.0960000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.64900000000000002,"15min":0.58999999999999997},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.7349999999999999,"5min":3.7349999999999999,"15min":3.7349999999999999},"last":0.42399999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:45:38 2023","interfaces":[{"interface":"back","average":{"1min":1.419,"5min":0.71399999999999997,"15min":0.59599999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.7030000000000003,"5min":4.7030000000000003,"15min":4.7030000000000003},"last":0.498},{"interface":"front","average":{"1min":1.2729999999999999,"5min":0.69199999999999995,"15min":0.59499999999999997},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.867,"5min":2.867,"15min":2.867},"last":0.36699999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.54400000000000004,"15min":0.47299999999999998},"min":{"1min":0.26100000000000001,"5min":0.255,"15min":0.255},"max":{"1min":2.9649999999999999,"5min":2.9649999999999999,"15min":2.9649999999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.80800000000000005,"5min":0.58499999999999996,"15min":0.51700000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.9039999999999999,"5min":2.9039999999999999,"15min":2.9039999999999999},"last":0.51500000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.626,"15min":0.52300000000000002},"min":{"1min":0.317,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.4849999999999999,"5min":3.4849999999999999,"15min":3.4849999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.075,"5min":0.68400000000000005,"15min":0.57099999999999995},"min":{"1min":0.316,"5min":0.316,"15min":0.27600000000000002},"max":{"1min":2.992,"5min":2.992,"15min":2.992},"last":0.47699999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:45:21 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.59399999999999997,"15min":0.54500000000000004},"min":{"1min":0.39300000000000002,"5min":0.33900000000000002,"15min":0.32900000000000001},"max":{"1min":2.0870000000000002,"5min":2.0870000000000002,"15min":2.0870000000000002},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.63800000000000001,"15min":0.59199999999999997},"min":{"1min":0.42199999999999999,"5min":0.34100000000000003,"15min":0.34000000000000002},"max":{"1min":1.921,"5min":1.921,"15min":1.921},"last":0.54900000000000004}]}]},{"osd":1,"up_from":85,"seq":365072220205,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2374028,"kb_used_data":2292356,"kb_used_omap":9,"kb_used_meta":81590,"kb_avail":91371124,"statfs":{"total":95995035648,"available":93564030976,"internally_reserved":0,"allocated":2347372544,"data_stored":2346656854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":83549161},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:45:44 2023","interfaces":[{"interface":"back","average":{"1min":2.7490000000000001,"5min":0.86199999999999999,"15min":0.53000000000000003},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":14.1,"5min":14.1,"15min":14.1},"last":0.47499999999999998},{"interface":"front","average":{"1min":2.431,"5min":0.83499999999999996,"15min":0.54400000000000004},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":14.465,"5min":14.465,"15min":14.465},"last":0.68400000000000005}]},{"osd":2,"last update":"Mon Oct 9 10:45:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":0.55300000000000005,"15min":0.42899999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":8.6859999999999999,"5min":8.6859999999999999,"15min":8.6859999999999999},"last":4.2850000000000001},{"interface":"front","average":{"1min":1.7470000000000001,"5min":0.65700000000000003,"15min":0.47499999999999998},"min":{"1min":0.26200000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":14.137,"5min":14.137,"15min":14.137},"last":2.8799999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:45:37 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":0.92000000000000004,"15min":0.621},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":14.404999999999999,"5min":14.404999999999999,"15min":14.404999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.4279999999999999,"5min":0.76100000000000001,"15min":0.48299999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":7.8620000000000001,"5min":7.8620000000000001,"15min":7.8620000000000001},"last":0.71599999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:45:44 2023","interfaces":[{"interface":"back","average":{"1min":2.1600000000000001,"5min":0.89500000000000002,"15min":0.69299999999999995},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.0129999999999999,"5min":8.0129999999999999,"15min":8.0129999999999999},"last":4.1879999999999997},{"interface":"front","average":{"1min":1.8080000000000001,"5min":0.85199999999999998,"15min":0.69699999999999995},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":8.2899999999999991,"5min":8.2899999999999991,"15min":8.2899999999999991},"last":0.53300000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:45:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":0.76900000000000002,"15min":0.67500000000000004},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.3660000000000001,"5min":3.3660000000000001,"15min":3.3660000000000001},"last":0.73699999999999999},{"interface":"front","average":{"1min":2.254,"5min":0.94899999999999995,"15min":0.747},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":7.923,"5min":7.923,"15min":7.923},"last":4.2290000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:45:44 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":0.78100000000000003,"15min":0.68100000000000005},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.7559999999999998,"5min":3.7559999999999998,"15min":3.7559999999999998},"last":0.75600000000000001},{"interface":"front","average":{"1min":2.4020000000000001,"5min":0.98099999999999998,"15min":0.75800000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.9569999999999999,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":4.2679999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:45:44 2023","interfaces":[{"interface":"back","average":{"1min":2.222,"5min":0.95099999999999996,"15min":0.75},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":7.9859999999999998,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":4.2489999999999997},{"interface":"front","average":{"1min":1.728,"5min":0.86399999999999999,"15min":0.71799999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":8.2469999999999999,"5min":8.2469999999999999,"15min":8.2469999999999999},"last":0.50600000000000001}]}]},{"osd":0,"up_from":80,"seq":343597383739,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1151196,"kb_used_data":1087508,"kb_used_omap":28,"kb_used_meta":63587,"kb_avail":92593956,"statfs":{"total":95995035648,"available":94816210944,"internally_reserved":0,"allocated":1113608192,"data_stored":1112875118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":65113936},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:45:42 2023","interfaces":[{"interface":"back","average":{"1min":3.0150000000000001,"5min":0.91100000000000003,"15min":0.54900000000000004},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":25.475000000000001,"5min":25.475000000000001,"15min":25.475000000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":2.3769999999999998,"5min":0.86299999999999999,"15min":0.53100000000000003},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":20.832999999999998,"5min":20.832999999999998,"15min":20.832999999999998},"last":0.30099999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:45:13 2023","interfaces":[{"interface":"back","average":{"1min":1.758,"5min":0.63,"15min":0.442},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":8.9420000000000002,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":0.313},{"interface":"front","average":{"1min":1.444,"5min":0.56299999999999994,"15min":0.41599999999999998},"min":{"1min":0.22500000000000001,"5min":0.223,"15min":0.223},"max":{"1min":7.4980000000000002,"5min":7.4980000000000002,"15min":7.4980000000000002},"last":0.33700000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:45:38 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":1.0669999999999999,"15min":0.78900000000000003},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":14.926,"5min":14.926,"15min":14.926},"last":0.23100000000000001},{"interface":"front","average":{"1min":2.6389999999999998,"5min":0.96099999999999997,"15min":0.68100000000000005},"min":{"1min":0.40699999999999997,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":11.113,"5min":11.113,"15min":11.113},"last":0.24399999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:45:33 2023","interfaces":[{"interface":"back","average":{"1min":1.8879999999999999,"5min":0.85399999999999998,"15min":0.69799999999999995},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.71,"5min":5.71,"15min":5.71},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.956,"5min":0.876,"15min":0.70399999999999996},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.9269999999999996,"5min":9.9269999999999996,"15min":9.9269999999999996},"last":0.44400000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:45:33 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":0.874,"15min":0.70799999999999996},"min":{"1min":0.36599999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":9.9900000000000002,"5min":9.9900000000000002,"15min":9.9900000000000002},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.7549999999999999,"5min":0.85299999999999998,"15min":0.71699999999999997},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.9190000000000005,"5min":8.9190000000000005,"15min":8.9190000000000005},"last":0.501}]},{"osd":6,"last update":"Mon Oct 9 10:45:33 2023","interfaces":[{"interface":"back","average":{"1min":1.958,"5min":0.88300000000000001,"15min":0.71499999999999997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":9.9659999999999993,"5min":9.9659999999999993,"15min":9.9659999999999993},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.9019999999999999,"5min":0.88700000000000001,"15min":0.72899999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":8.8450000000000006,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":0.52900000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:45:33 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":0.86699999999999999,"15min":0.72399999999999998},"min":{"1min":0.42099999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":8.875,"5min":8.875,"15min":8.875},"last":0.54800000000000004},{"interface":"front","average":{"1min":2.1030000000000002,"5min":0.93700000000000006,"15min":0.76400000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":7.3899999999999997,"5min":7.3899999999999997,"15min":7.3899999999999997},"last":0.52200000000000002}]}]},{"osd":2,"up_from":90,"seq":386547056680,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3563460,"kb_used_data":3433212,"kb_used_omap":81,"kb_used_meta":130030,"kb_avail":90181692,"statfs":{"total":95995035648,"available":92346052608,"internally_reserved":0,"allocated":3515609088,"data_stored":3514899063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":133150938},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.39500000000000002,"15min":0.34499999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":3.8370000000000002,"5min":3.8370000000000002,"15min":3.8370000000000002},"last":2.101},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.46100000000000002,"15min":0.40300000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.0049999999999999,"5min":4.0049999999999999,"15min":4.0049999999999999},"last":2.4670000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":0.54800000000000004,"15min":0.44400000000000001},"min":{"1min":0.27900000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.1909999999999998,"5min":5.1909999999999998,"15min":5.1909999999999998},"last":2.5030000000000001},{"interface":"front","average":{"1min":0.877,"5min":0.46200000000000002,"15min":0.39300000000000002},"min":{"1min":0.23799999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.2039999999999997,"5min":5.2039999999999997,"15min":5.2039999999999997},"last":2.0150000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:45:37 2023","interfaces":[{"interface":"back","average":{"1min":1.958,"5min":0.72599999999999998,"15min":0.52000000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":12.076000000000001,"5min":12.076000000000001,"15min":12.076000000000001},"last":2.3799999999999999},{"interface":"front","average":{"1min":2.8199999999999998,"5min":0.91600000000000004,"15min":0.59899999999999998},"min":{"1min":0.29299999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":10.249000000000001,"5min":10.249000000000001,"15min":10.249000000000001},"last":8.9960000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":0.68200000000000005,"15min":0.621},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":8.1029999999999998,"5min":8.1029999999999998,"15min":8.1029999999999998},"last":2.0840000000000001},{"interface":"front","average":{"1min":1.216,"5min":0.71799999999999997,"15min":0.63500000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":8.1259999999999994,"5min":8.1259999999999994,"15min":8.1259999999999994},"last":2.21}]},{"osd":5,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":0.754,"15min":0.64800000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.3659999999999997,"5min":8.3659999999999997,"15min":8.3659999999999997},"last":2.5329999999999999},{"interface":"front","average":{"1min":1.286,"5min":0.73399999999999999,"15min":0.64200000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":8.3979999999999997,"5min":8.3979999999999997,"15min":8.3979999999999997},"last":2.0489999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.62,"15min":0.58999999999999997},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":3.3730000000000002,"5min":3.3730000000000002,"15min":3.3730000000000002},"last":2.1309999999999998},{"interface":"front","average":{"1min":1.1100000000000001,"5min":0.68999999999999995,"15min":0.62},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":5.2199999999999998,"5min":5.2199999999999998,"15min":5.2199999999999998},"last":2.5609999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.629,"15min":0.59799999999999998},"min":{"1min":0.433,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":3.3500000000000001,"5min":3.3500000000000001,"15min":3.3500000000000001},"last":2.1579999999999999},{"interface":"front","average":{"1min":1.1419999999999999,"5min":0.69499999999999995,"15min":0.621},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":5.2690000000000001,"5min":5.2690000000000001,"15min":5.2690000000000001},"last":2.548}]}]},{"osd":3,"up_from":95,"seq":408021893154,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4355860,"kb_used_data":4211916,"kb_used_omap":7,"kb_used_meta":143928,"kb_avail":89389292,"statfs":{"total":95995035648,"available":91534635008,"internally_reserved":0,"allocated":4313001984,"data_stored":4312275884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":147382888},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":0.51200000000000001,"15min":0.38400000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":4.3159999999999998,"5min":4.3159999999999998,"15min":4.3159999999999998},"last":7.0659999999999998},{"interface":"front","average":{"1min":2.1070000000000002,"5min":0.747,"15min":0.52000000000000002},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":7.0510000000000002,"5min":7.0510000000000002,"15min":7.0510000000000002},"last":0.188}]},{"osd":1,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":3.782,"5min":1.02,"15min":0.56000000000000005},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":14.707000000000001,"5min":14.707000000000001,"15min":14.707000000000001},"last":13.967000000000001},{"interface":"front","average":{"1min":1.6659999999999999,"5min":0.63200000000000001,"15min":0.46000000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":6.4870000000000001,"5min":6.4870000000000001,"15min":6.4870000000000001},"last":7.0330000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":3.4089999999999998,"5min":0.97499999999999998,"15min":0.56899999999999995},"min":{"1min":0.31,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":13.803000000000001,"5min":13.803000000000001,"15min":13.803000000000001},"last":13.853},{"interface":"front","average":{"1min":2.7000000000000002,"5min":0.85599999999999998,"15min":0.54900000000000004},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.0579999999999998,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":0.60499999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":0.752,"15min":0.63200000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.798,"5min":3.798,"15min":3.798},"last":2.8969999999999998},{"interface":"front","average":{"1min":1.6890000000000001,"5min":0.82399999999999995,"15min":0.68000000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":5.4269999999999996,"5min":5.4269999999999996,"15min":5.4269999999999996},"last":0.64000000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":0.81399999999999995,"15min":0.66200000000000003},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.4580000000000002,"5min":5.4580000000000002,"15min":5.4580000000000002},"last":0.66400000000000003},{"interface":"front","average":{"1min":3.1200000000000001,"5min":1.0740000000000001,"15min":0.73299999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":13.845000000000001,"5min":13.845000000000001,"15min":13.845000000000001},"last":11.396000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0640000000000001,"5min":0.88300000000000001,"15min":0.68600000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.4789999999999992,"5min":8.4789999999999992,"15min":8.4789999999999992},"last":0.626},{"interface":"front","average":{"1min":3.2919999999999998,"5min":1.1379999999999999,"15min":0.77900000000000003},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":13.891999999999999,"5min":13.891999999999999,"15min":13.891999999999999},"last":11.470000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":3.214,"5min":1.1160000000000001,"15min":0.76600000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":13.869999999999999,"5min":13.869999999999999,"15min":13.869999999999999},"last":11.441000000000001},{"interface":"front","average":{"1min":1.611,"5min":0.78100000000000003,"15min":0.64300000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.831,"5min":3.831,"15min":3.831},"last":2.9420000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019029,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2397408,"kb_used_data":2087768,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91347744,"statfs":{"total":95995035648,"available":93540089856,"internally_reserved":0,"allocated":2137874432,"data_stored":2137310921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":0.66100000000000003,"15min":0.59199999999999997},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.8550000000000004,"5min":6.8550000000000004,"15min":6.8550000000000004},"last":0.23799999999999999},{"interface":"front","average":{"1min":1.6919999999999999,"5min":0.79200000000000004,"15min":0.65400000000000003},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.4240000000000004,"5min":6.4240000000000004,"15min":6.4240000000000004},"last":0.27300000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":0.82199999999999995,"15min":0.64200000000000002},"min":{"1min":0.35699999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.907,"5min":6.907,"15min":6.907},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.3120000000000001,"5min":0.79000000000000004,"15min":0.67800000000000005},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.2010000000000001,"5min":4.8120000000000003,"15min":4.8120000000000003},"last":0.436}]},{"osd":2,"last update":"Mon Oct 9 10:45:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":0.71799999999999997,"15min":0.62},"min":{"1min":0.46100000000000002,"5min":0.373,"15min":0.373},"max":{"1min":4.6340000000000003,"5min":4.6340000000000003,"15min":4.6340000000000003},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.1319999999999999,"5min":0.67600000000000005,"15min":0.59999999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.2939999999999996,"5min":6.2939999999999996,"15min":6.2939999999999996},"last":0.45600000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":0.86499999999999999,"15min":0.76300000000000001},"min":{"1min":0.40799999999999997,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.7750000000000004,"5min":4.7750000000000004,"15min":4.7750000000000004},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.052,"5min":0.68000000000000005,"15min":0.61799999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.2320000000000002,"5min":3.2320000000000002,"15min":3.2320000000000002},"last":0.42399999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:45:40 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":0.66700000000000004,"15min":0.51000000000000001},"min":{"1min":0.28999999999999998,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":5.5410000000000004,"5min":5.5410000000000004,"15min":5.5410000000000004},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.60599999999999998,"15min":0.49099999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.5150000000000001,"5min":2.5150000000000001,"15min":2.5150000000000001},"last":0.32600000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:46:07 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":0.64900000000000002,"15min":0.52100000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.246},"max":{"1min":5.1509999999999998,"5min":5.1509999999999998,"15min":5.1509999999999998},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.68799999999999994,"15min":0.59599999999999997},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.314},"max":{"1min":3.181,"5min":3.181,"15min":3.181},"last":0.52600000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:45:14 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.628,"15min":0.58399999999999996},"min":{"1min":0.36799999999999999,"5min":0.35299999999999998,"15min":0.308},"max":{"1min":2.7589999999999999,"5min":2.7589999999999999,"15min":2.7589999999999999},"last":0.505},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.622,"15min":0.55800000000000005},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.456,"5min":2.456,"15min":2.456},"last":0.49199999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790098,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2972048,"kb_used_data":2670664,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":90773104,"statfs":{"total":95995035648,"available":92951658496,"internally_reserved":0,"allocated":2734759936,"data_stored":2734208254,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:45:32 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":0.68899999999999995,"15min":0.628},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":4.8380000000000001,"5min":4.8380000000000001,"15min":4.8380000000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.131,"5min":0.749,"15min":0.69599999999999995},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.8650000000000002,"5min":4.8650000000000002,"15min":4.8650000000000002},"last":0.52200000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:45:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":0.68700000000000006,"15min":0.63400000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.577,"5min":3.577,"15min":3.577},"last":0.438},{"interface":"front","average":{"1min":1.514,"5min":0.82499999999999996,"15min":0.66900000000000004},"min":{"1min":0.26700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.165,"5min":7.165,"15min":7.165},"last":0.38600000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:45:06 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.65400000000000003,"15min":0.59899999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.8490000000000002,"5min":3.8490000000000002,"15min":3.8490000000000002},"last":0.371},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.59199999999999997,"15min":0.57999999999999996},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.823,"5min":1.823,"15min":1.823},"last":0.54300000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:45:36 2023","interfaces":[{"interface":"back","average":{"1min":1.5649999999999999,"5min":0.89200000000000002,"15min":0.78000000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.0069999999999997,"5min":4.0069999999999997,"15min":4.0069999999999997},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.5249999999999999,"5min":0.79500000000000004,"15min":0.67300000000000004},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.0940000000000003,"5min":6.0940000000000003,"15min":6.0940000000000003},"last":0.40100000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:45:43 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.51800000000000002,"15min":0.45300000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.25},"max":{"1min":1.7929999999999999,"5min":1.7929999999999999,"15min":2.4649999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.57899999999999996,"15min":0.48599999999999999},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.28799999999999998},"max":{"1min":2.1419999999999999,"5min":2.1419999999999999,"15min":2.1419999999999999},"last":0.32700000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:46:05 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.59399999999999997,"15min":0.497},"min":{"1min":0.311,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":2.9529999999999998,"5min":2.9529999999999998,"15min":2.9529999999999998},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.58899999999999997,"15min":0.51200000000000001},"min":{"1min":0.34300000000000003,"5min":0.33300000000000002,"15min":0.30299999999999999},"max":{"1min":1.3140000000000001,"5min":1.746,"15min":1.746},"last":0.34499999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:45:18 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.622,"15min":0.55500000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32400000000000001},"max":{"1min":4.0510000000000002,"5min":4.0510000000000002,"15min":4.0510000000000002},"last":0.505},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.66700000000000004,"15min":0.56999999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":2.6869999999999998,"5min":2.6869999999999998,"15min":2.6869999999999998},"last":0.41599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1104809984,"data_stored":1104805912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2344357888,"data_stored":2344353816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3499229184,"data_stored":3499229184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4295491584,"data_stored":4295491584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2126446592,"data_stored":2126446592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2729054208,"data_stored":2729050136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2236153856,"data_stored":2236153856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2774597632,"data_stored":2774597632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:46:16.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 21 2023-10-09T10:46:16.282 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:16.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:16.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:46:16.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:16 smithi125 ceph-mon[195507]: pgmap v301: 119 pgs: 1 active+clean+scrubbing, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 103 KiB/s rd, 272 KiB/s wr, 535 op/s; 8135/308706 objects degraded (2.635%); 8700/308706 objects misplaced (2.818%); 9.7 MiB/s, 153 objects/s recovering 2023-10-09T10:46:16.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:16.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:46:16.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:16 smithi125 ceph-mon[200197]: pgmap v301: 119 pgs: 1 active+clean+scrubbing, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 103 KiB/s rd, 272 KiB/s wr, 535 op/s; 8135/308706 objects degraded (2.635%); 8700/308706 objects misplaced (2.818%); 9.7 MiB/s, 153 objects/s recovering 2023-10-09T10:46:16.317 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:16.317 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:46:16.317 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:16 smithi155 ceph-mon[184854]: pgmap v301: 119 pgs: 1 active+clean+scrubbing, 112 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 103 KiB/s rd, 272 KiB/s wr, 535 op/s; 8135/308706 objects degraded (2.635%); 8700/308706 objects misplaced (2.818%); 9.7 MiB/s, 153 objects/s recovering 2023-10-09T10:46:16.320 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:16.343 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:16.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:16.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:16.448 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:16.482 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:16.510 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:16.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:16.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:16.595 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:46:16.613 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:16.614 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:16.671 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:16.695 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:16.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:16.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:16.799 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:16.871 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:16.902 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:16.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:16.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:16.950 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:46:17.009 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:17.071 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:46:17.085 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:17.101 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:17 smithi125 ceph-mon[195507]: 6.2 scrub starts 2023-10-09T10:46:17.101 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:17 smithi125 ceph-mon[195507]: from='client.34836 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:46:17.101 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3263584107' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:46:17.101 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:17 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:46:17.101 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:17 smithi125 ceph-mon[200197]: 6.2 scrub starts 2023-10-09T10:46:17.101 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:17 smithi125 ceph-mon[200197]: from='client.34836 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:46:17.101 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3263584107' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:46:17.101 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:17 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:46:17.149 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:17.228 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:17.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:17.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:17.345 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:17.383 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:17 smithi155 ceph-mon[184854]: 6.2 scrub starts 2023-10-09T10:46:17.383 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:17 smithi155 ceph-mon[184854]: from='client.34836 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:46:17.383 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3263584107' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:46:17.383 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:17 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:46:17.387 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:17.577 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:17.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:17.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:17.681 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:17.723 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:17.806 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:17.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:17.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:17.910 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:17.966 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:18.005 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:18.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:18.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:18.059 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:46:18.059 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:46:18.111 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:18.187 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:46:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[195507]: osdmap e150: 8 total, 8 up, 8 in 2023-10-09T10:46:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[195507]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[195507]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[195507]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[195507]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[195507]: pgmap v303: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 127 B/s wr, 653 op/s; 8135/303423 objects degraded (2.681%); 8307/303423 objects misplaced (2.738%); 16 MiB/s, 248 objects/s recovering 2023-10-09T10:46:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[195507]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[195507]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 8135/308706 objects degraded (2.635%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:18.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:18.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:46:18.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[200197]: osdmap e150: 8 total, 8 up, 8 in 2023-10-09T10:46:18.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[200197]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[200197]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[200197]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[200197]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[200197]: pgmap v303: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 127 B/s wr, 653 op/s; 8135/303423 objects degraded (2.681%); 8307/303423 objects misplaced (2.738%); 16 MiB/s, 248 objects/s recovering 2023-10-09T10:46:18.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[200197]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[200197]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:18 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 8135/308706 objects degraded (2.635%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:18.362 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:18 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:18.362 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:18 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:46:18.362 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:18 smithi155 ceph-mon[184854]: osdmap e150: 8 total, 8 up, 8 in 2023-10-09T10:46:18.362 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:18 smithi155 ceph-mon[184854]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.363 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:18 smithi155 ceph-mon[184854]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.363 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:18 smithi155 ceph-mon[184854]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.363 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:18 smithi155 ceph-mon[184854]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.363 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:18 smithi155 ceph-mon[184854]: pgmap v303: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 127 B/s wr, 653 op/s; 8135/303423 objects degraded (2.681%); 8307/303423 objects misplaced (2.738%); 16 MiB/s, 248 objects/s recovering 2023-10-09T10:46:18.363 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:18 smithi155 ceph-mon[184854]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.363 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:18 smithi155 ceph-mon[184854]: failed to encode map e150 with expected crc 2023-10-09T10:46:18.363 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:18 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 8135/308706 objects degraded (2.635%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:18.451 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:46:18.865 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:46:19.193 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:19.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:19.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:19.297 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:19.342 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:19.467 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:19.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:19.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:19.573 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:19.612 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:46:19.616 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:19.870 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:19.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:19.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:19.892 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:46:19.892 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:46:19.901 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:46:19.941 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:46:19.974 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:19.999 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:46:20.052 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:46:20.088 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:20.095 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:46:20.128 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:20.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:20.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:20.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:20 smithi125 ceph-mon[200197]: pgmap v304: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 116 B/s rd, 116 B/s wr, 715 op/s; 8135/299574 objects degraded (2.716%); 8307/299574 objects misplaced (2.773%); 14 MiB/s, 227 objects/s recovering 2023-10-09T10:46:20.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:20 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1630435817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:20.141 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:46:20.172 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:46:20.187 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:20.206 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:46:20.227 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:46:20.227 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:46:20.227 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:46:20.227 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:46:20.227 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:46:20.227 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:46:20.227 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:46:20.227 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:46:20.237 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:20.270 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:46:20.302 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:20.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:20 smithi125 ceph-mon[195507]: pgmap v304: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 116 B/s rd, 116 B/s wr, 715 op/s; 8135/299574 objects degraded (2.716%); 8307/299574 objects misplaced (2.773%); 14 MiB/s, 227 objects/s recovering 2023-10-09T10:46:20.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:20 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1630435817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:20 smithi155 ceph-mon[184854]: pgmap v304: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 116 B/s rd, 116 B/s wr, 715 op/s; 8135/299574 objects degraded (2.716%); 8307/299574 objects misplaced (2.773%); 14 MiB/s, 227 objects/s recovering 2023-10-09T10:46:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:20 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1630435817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:20.664 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:46:20.702 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:20.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:20.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:20.807 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:20.873 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:21.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:21.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:21.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:21.126 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:21.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:21 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2092435982' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T10:46:21.182 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:21.198 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 1 pgp_num to 21 2023-10-09T10:46:21.258 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:21.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:21.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:21.263 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:21 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2092435982' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T10:46:21.263 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:21 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2092435982' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T10:46:21.264 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:46:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:46:21] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.43.0" 2023-10-09T10:46:21.362 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:21.414 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:21.452 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:21.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:21.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:21.558 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:21.604 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:21.694 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:21.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:21.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:21.799 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:21.803 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:46:21.803 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:46:21.803 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:46:21.803 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:46:21.803 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.804 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:22586 6.2 6:442872f5:::benchmark_data_smithi125_244951_object22585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e104)", 2023-10-09T10:46:21.804 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:42.792081+0000", 2023-10-09T10:46:21.804 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 99.008367824000004, 2023-10-09T10:46:21.804 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.036352778000000002, 2023-10-09T10:46:21.804 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.804 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.804 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.804 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.804 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.804 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 22586 2023-10-09T10:46:21.804 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.804 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.804 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.805 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.805 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.792081+0000", 2023-10-09T10:46:21.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.805 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.805 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.792081+0000", 2023-10-09T10:46:21.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.805 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.805 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.792080+0000", 2023-10-09T10:46:21.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:46:21.806 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.806 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.806 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.806 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.792156+0000", 2023-10-09T10:46:21.806 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5389999999999995e-05 2023-10-09T10:46:21.806 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.806 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.806 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.806 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.792166+0000", 2023-10-09T10:46:21.806 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8300000000000008e-06 2023-10-09T10:46:21.806 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.806 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.807 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.807 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.792170+0000", 2023-10-09T10:46:21.807 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7380000000000002e-06 2023-10-09T10:46:21.807 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.807 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.807 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.807 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.792218+0000", 2023-10-09T10:46:21.807 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7281999999999999e-05 2023-10-09T10:46:21.807 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.807 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.807 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.807 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.792353+0000", 2023-10-09T10:46:21.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013511099999999999 2023-10-09T10:46:21.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:46:21.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.792499+0000", 2023-10-09T10:46:21.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014664300000000001 2023-10-09T10:46:21.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.793314+0000", 2023-10-09T10:46:21.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00081455299999999998 2023-10-09T10:46:21.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.796363+0000", 2023-10-09T10:46:21.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030487159999999999 2023-10-09T10:46:21.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.828349+0000", 2023-10-09T10:46:21.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.031986094999999999 2023-10-09T10:46:21.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.828398+0000", 2023-10-09T10:46:21.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8847999999999997e-05 2023-10-09T10:46:21.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.810 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.810 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.810 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:42.828434+0000", 2023-10-09T10:46:21.810 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6264999999999999e-05 2023-10-09T10:46:21.810 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.810 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.810 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.810 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.810 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.810 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:41073 6.2 6:44027eb5:::benchmark_data_smithi125_244951_object41072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T10:46:21.810 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:54.318944+0000", 2023-10-09T10:46:21.810 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 87.481505190999997, 2023-10-09T10:46:21.811 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.054273323999999998, 2023-10-09T10:46:21.811 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.811 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.811 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.811 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.811 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.811 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 41073 2023-10-09T10:46:21.811 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.811 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.811 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.811 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.811 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.318944+0000", 2023-10-09T10:46:21.811 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.811 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.812 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.812 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.812 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.318944+0000", 2023-10-09T10:46:21.812 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.812 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.812 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.812 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.812 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.318942+0000", 2023-10-09T10:46:21.812 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:46:21.812 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.812 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.812 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.812 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.319018+0000", 2023-10-09T10:46:21.813 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5895000000000004e-05 2023-10-09T10:46:21.813 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.813 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.813 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.813 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.319028+0000", 2023-10-09T10:46:21.813 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6209999999999999e-06 2023-10-09T10:46:21.813 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.813 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.813 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.813 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.319034+0000", 2023-10-09T10:46:21.813 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0100000000000001e-06 2023-10-09T10:46:21.813 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.814 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.814 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.814 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.319082+0000", 2023-10-09T10:46:21.814 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8625000000000003e-05 2023-10-09T10:46:21.814 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.814 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.814 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.814 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.319208+0000", 2023-10-09T10:46:21.814 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000125649 2023-10-09T10:46:21.814 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.814 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.814 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:46:21.815 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.319358+0000", 2023-10-09T10:46:21.815 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000150209 2023-10-09T10:46:21.815 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.815 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.815 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.815 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.320771+0000", 2023-10-09T10:46:21.815 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014123149999999999 2023-10-09T10:46:21.815 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.815 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.815 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.815 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.322458+0000", 2023-10-09T10:46:21.815 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001687256 2023-10-09T10:46:21.815 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.816 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.816 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.816 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.373121+0000", 2023-10-09T10:46:21.816 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.050663073000000003 2023-10-09T10:46:21.816 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.816 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.816 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.816 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.373174+0000", 2023-10-09T10:46:21.816 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2935000000000001e-05 2023-10-09T10:46:21.816 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.816 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.816 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.817 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.373217+0000", 2023-10-09T10:46:21.817 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3210000000000001e-05 2023-10-09T10:46:21.817 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.817 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.817 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.817 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.817 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.817 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:41080 6.2 6:45f640b2:::benchmark_data_smithi125_244951_object41079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T10:46:21.817 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:44:54.320996+0000", 2023-10-09T10:46:21.817 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 87.479452959, 2023-10-09T10:46:21.817 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.052382713999999997, 2023-10-09T10:46:21.817 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.817 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.818 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.818 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.818 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.818 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 41080 2023-10-09T10:46:21.818 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.818 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.818 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.818 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.818 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.320996+0000", 2023-10-09T10:46:21.818 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.818 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.818 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.819 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.819 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.320996+0000", 2023-10-09T10:46:21.819 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.819 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.819 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.819 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.819 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.320995+0000", 2023-10-09T10:46:21.819 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:46:21.819 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.819 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.819 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.819 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.321154+0000", 2023-10-09T10:46:21.820 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000159419 2023-10-09T10:46:21.820 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.820 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.820 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.820 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.321166+0000", 2023-10-09T10:46:21.820 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1958e-05 2023-10-09T10:46:21.820 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.820 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.820 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.820 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.321172+0000", 2023-10-09T10:46:21.820 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1829999999999999e-06 2023-10-09T10:46:21.820 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.820 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.820 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.821 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.321211+0000", 2023-10-09T10:46:21.821 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8649000000000002e-05 2023-10-09T10:46:21.821 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.821 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.821 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.821 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.321330+0000", 2023-10-09T10:46:21.821 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000119375 2023-10-09T10:46:21.821 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.821 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.821 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:46:21.821 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.321487+0000", 2023-10-09T10:46:21.821 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000156496 2023-10-09T10:46:21.822 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.822 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.822 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.822 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.323297+0000", 2023-10-09T10:46:21.822 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001810025 2023-10-09T10:46:21.822 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.822 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.822 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.822 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.324641+0000", 2023-10-09T10:46:21.822 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001344594 2023-10-09T10:46:21.822 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.822 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.822 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.823 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.373258+0000", 2023-10-09T10:46:21.823 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.048616566999999999 2023-10-09T10:46:21.823 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.823 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.823 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.823 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.373338+0000", 2023-10-09T10:46:21.823 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0162999999999998e-05 2023-10-09T10:46:21.823 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.823 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.823 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.823 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:44:54.373379+0000", 2023-10-09T10:46:21.824 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0546000000000001e-05 2023-10-09T10:46:21.824 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.824 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.824 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.824 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.824 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.824 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:67628 6.2 6:422521ec:::benchmark_data_smithi125_244951_object67627:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:46:21.824 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:20.675541+0000", 2023-10-09T10:46:21.824 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 61.124908114999997, 2023-10-09T10:46:21.824 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.033588075000000002, 2023-10-09T10:46:21.824 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.824 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.824 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.825 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.825 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.825 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 67628 2023-10-09T10:46:21.825 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.825 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.825 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.825 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.825 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.675541+0000", 2023-10-09T10:46:21.825 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.825 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.825 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.825 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.825 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.675541+0000", 2023-10-09T10:46:21.826 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.826 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.826 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.826 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.826 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.675539+0000", 2023-10-09T10:46:21.826 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:46:21.826 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.826 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.826 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.826 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.675597+0000", 2023-10-09T10:46:21.826 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7327000000000002e-05 2023-10-09T10:46:21.826 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.826 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.827 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.827 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.675614+0000", 2023-10-09T10:46:21.827 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7146000000000002e-05 2023-10-09T10:46:21.827 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.827 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.827 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.827 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.675621+0000", 2023-10-09T10:46:21.827 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.063e-06 2023-10-09T10:46:21.827 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.827 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.827 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.827 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.675680+0000", 2023-10-09T10:46:21.828 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8526999999999997e-05 2023-10-09T10:46:21.828 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.828 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.828 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.828 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.675810+0000", 2023-10-09T10:46:21.828 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013080000000000001 2023-10-09T10:46:21.828 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.828 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.828 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:46:21.828 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.675954+0000", 2023-10-09T10:46:21.828 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014388399999999999 2023-10-09T10:46:21.828 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.828 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.829 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.829 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.679699+0000", 2023-10-09T10:46:21.829 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0037444090000000002 2023-10-09T10:46:21.829 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.829 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.829 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.829 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.697728+0000", 2023-10-09T10:46:21.829 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018029367000000001 2023-10-09T10:46:21.829 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.829 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.829 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.829 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.709062+0000", 2023-10-09T10:46:21.830 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011333582 2023-10-09T10:46:21.830 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.830 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.830 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.830 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.709099+0000", 2023-10-09T10:46:21.830 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7694999999999997e-05 2023-10-09T10:46:21.830 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.830 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.830 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.830 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:20.709129+0000", 2023-10-09T10:46:21.830 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9638999999999999e-05 2023-10-09T10:46:21.830 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.831 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.831 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.831 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.831 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.831 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:69269 6.e 6:72c72736:::benchmark_data_smithi125_244951_object69268:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:46:21.831 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:22.309155+0000", 2023-10-09T10:46:21.831 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 59.491293939999998, 2023-10-09T10:46:21.831 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.036662966999999998, 2023-10-09T10:46:21.831 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.831 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.831 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.831 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.831 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.832 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 69269 2023-10-09T10:46:21.832 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.832 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.832 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.832 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.832 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.309155+0000", 2023-10-09T10:46:21.832 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.832 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.832 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.832 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.832 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.309155+0000", 2023-10-09T10:46:21.832 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.833 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.833 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.833 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.833 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.309153+0000", 2023-10-09T10:46:21.833 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:46:21.833 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.833 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.833 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.833 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.309197+0000", 2023-10-09T10:46:21.833 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3435e-05 2023-10-09T10:46:21.833 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.833 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.834 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.834 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.309213+0000", 2023-10-09T10:46:21.834 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5654e-05 2023-10-09T10:46:21.834 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.834 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.834 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.834 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.309220+0000", 2023-10-09T10:46:21.834 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.041e-06 2023-10-09T10:46:21.834 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.834 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.834 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.834 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.309278+0000", 2023-10-09T10:46:21.835 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8764000000000002e-05 2023-10-09T10:46:21.835 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.835 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.835 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.835 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.309448+0000", 2023-10-09T10:46:21.835 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000169183 2023-10-09T10:46:21.835 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.835 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.835 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:46:21.835 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.309614+0000", 2023-10-09T10:46:21.835 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016692599999999999 2023-10-09T10:46:21.835 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.835 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.836 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.836 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.313498+0000", 2023-10-09T10:46:21.836 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0038830420000000002 2023-10-09T10:46:21.836 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.836 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.836 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.836 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.313934+0000", 2023-10-09T10:46:21.836 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00043665799999999998 2023-10-09T10:46:21.836 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.836 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.836 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.836 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.345742+0000", 2023-10-09T10:46:21.836 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.031807710000000003 2023-10-09T10:46:21.837 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.837 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.837 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.837 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.345778+0000", 2023-10-09T10:46:21.837 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5871000000000003e-05 2023-10-09T10:46:21.837 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.837 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.837 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.837 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:22.345818+0000", 2023-10-09T10:46:21.837 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0228000000000002e-05 2023-10-09T10:46:21.837 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.837 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.838 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.838 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.838 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.838 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70160 6.2 6:46f40ad6:::benchmark_data_smithi125_244951_object70159:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:46:21.838 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.154245+0000", 2023-10-09T10:46:21.838 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 58.646204296000001, 2023-10-09T10:46:21.838 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.042055607000000002, 2023-10-09T10:46:21.838 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.838 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.838 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.838 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.838 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.839 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70160 2023-10-09T10:46:21.839 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.839 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.839 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.839 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.839 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.154245+0000", 2023-10-09T10:46:21.839 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.839 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.839 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.839 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.839 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.154245+0000", 2023-10-09T10:46:21.839 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.839 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.840 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.840 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.840 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.154241+0000", 2023-10-09T10:46:21.840 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999967 2023-10-09T10:46:21.840 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.840 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.840 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.840 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.154317+0000", 2023-10-09T10:46:21.840 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5700999999999999e-05 2023-10-09T10:46:21.840 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.840 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.840 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.840 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.154337+0000", 2023-10-09T10:46:21.841 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0165000000000001e-05 2023-10-09T10:46:21.841 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.841 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.841 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.841 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.154345+0000", 2023-10-09T10:46:21.841 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7929999999999998e-06 2023-10-09T10:46:21.841 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.841 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.841 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.841 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.154414+0000", 2023-10-09T10:46:21.841 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8435999999999993e-05 2023-10-09T10:46:21.841 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.842 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.842 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.842 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.154585+0000", 2023-10-09T10:46:21.842 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017107400000000001 2023-10-09T10:46:21.842 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.842 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.842 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:46:21.842 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.154745+0000", 2023-10-09T10:46:21.842 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000160543 2023-10-09T10:46:21.842 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.842 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.842 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.842 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.155969+0000", 2023-10-09T10:46:21.842 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012236930000000001 2023-10-09T10:46:21.843 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.843 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.843 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.843 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.162581+0000", 2023-10-09T10:46:21.843 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0066118449999999999 2023-10-09T10:46:21.843 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.843 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.843 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.843 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.196201+0000", 2023-10-09T10:46:21.843 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.033619934999999997 2023-10-09T10:46:21.843 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.843 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.844 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.844 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.196257+0000", 2023-10-09T10:46:21.844 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6255999999999998e-05 2023-10-09T10:46:21.844 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.844 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.844 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.844 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.196300+0000", 2023-10-09T10:46:21.844 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3418999999999999e-05 2023-10-09T10:46:21.844 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.844 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.844 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.844 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.844 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.845 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70162 6.e 6:7912b3a3:::benchmark_data_smithi125_244951_object70161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:46:21.845 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.156954+0000", 2023-10-09T10:46:21.845 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 58.643494480999998, 2023-10-09T10:46:21.845 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.044544122999999998, 2023-10-09T10:46:21.845 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.845 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.845 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.845 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.845 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.845 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70162 2023-10-09T10:46:21.845 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.845 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.845 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.846 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.846 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.156954+0000", 2023-10-09T10:46:21.846 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.846 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.846 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.846 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.846 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.156954+0000", 2023-10-09T10:46:21.846 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.846 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.846 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.846 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.846 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.156953+0000", 2023-10-09T10:46:21.847 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:46:21.847 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.847 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.847 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.847 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.157014+0000", 2023-10-09T10:46:21.847 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1309000000000005e-05 2023-10-09T10:46:21.847 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.847 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.847 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.847 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.157028+0000", 2023-10-09T10:46:21.847 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4008999999999999e-05 2023-10-09T10:46:21.847 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.847 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.848 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.848 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.157035+0000", 2023-10-09T10:46:21.848 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.533e-06 2023-10-09T10:46:21.848 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.848 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.848 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.848 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.157078+0000", 2023-10-09T10:46:21.848 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3047000000000003e-05 2023-10-09T10:46:21.848 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.848 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.848 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.848 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.157178+0000", 2023-10-09T10:46:21.849 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9834000000000007e-05 2023-10-09T10:46:21.849 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.849 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.849 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:46:21.849 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.157285+0000", 2023-10-09T10:46:21.849 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000107389 2023-10-09T10:46:21.849 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.849 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.849 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.849 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.159737+0000", 2023-10-09T10:46:21.849 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024519730000000001 2023-10-09T10:46:21.849 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.850 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.850 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.850 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.159954+0000", 2023-10-09T10:46:21.850 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00021661900000000001 2023-10-09T10:46:21.850 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.850 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.850 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.850 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.201431+0000", 2023-10-09T10:46:21.850 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.041477100000000003 2023-10-09T10:46:21.850 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.850 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.850 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.851 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.201471+0000", 2023-10-09T10:46:21.851 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0460000000000002e-05 2023-10-09T10:46:21.851 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.851 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.851 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.851 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.201499+0000", 2023-10-09T10:46:21.851 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7189999999999999e-05 2023-10-09T10:46:21.851 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.851 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.851 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.851 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.851 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.851 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70172 6.e 6:700a3695:::benchmark_data_smithi125_244951_object70171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:46:21.852 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.166200+0000", 2023-10-09T10:46:21.852 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 58.634249412999999, 2023-10-09T10:46:21.852 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.046290348000000002, 2023-10-09T10:46:21.852 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.852 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.852 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.852 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.852 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.852 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70172 2023-10-09T10:46:21.852 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.852 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.852 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.853 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.853 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.166200+0000", 2023-10-09T10:46:21.853 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.853 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.853 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.853 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.853 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.166200+0000", 2023-10-09T10:46:21.853 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.853 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.853 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.853 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.853 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.166198+0000", 2023-10-09T10:46:21.854 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:46:21.854 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.854 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.854 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.854 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.166248+0000", 2023-10-09T10:46:21.854 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0142000000000002e-05 2023-10-09T10:46:21.854 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.854 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.854 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.854 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.166264+0000", 2023-10-09T10:46:21.854 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5724999999999999e-05 2023-10-09T10:46:21.854 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.854 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.855 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.855 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.166271+0000", 2023-10-09T10:46:21.855 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9890000000000003e-06 2023-10-09T10:46:21.855 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.855 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.855 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.855 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.166326+0000", 2023-10-09T10:46:21.855 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5371999999999998e-05 2023-10-09T10:46:21.855 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.855 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.855 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.855 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.166477+0000", 2023-10-09T10:46:21.855 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000150848 2023-10-09T10:46:21.856 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.856 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.856 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:46:21.856 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.166631+0000", 2023-10-09T10:46:21.856 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015367899999999999 2023-10-09T10:46:21.856 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.856 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.856 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.856 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.168532+0000", 2023-10-09T10:46:21.856 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001900917 2023-10-09T10:46:21.856 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.856 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.857 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.857 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.176221+0000", 2023-10-09T10:46:21.857 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0076894870000000001 2023-10-09T10:46:21.857 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.857 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.857 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.857 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.212408+0000", 2023-10-09T10:46:21.857 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.036186541000000003 2023-10-09T10:46:21.857 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.857 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.857 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.857 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.212444+0000", 2023-10-09T10:46:21.857 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6649000000000001e-05 2023-10-09T10:46:21.858 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.858 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.858 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.858 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.212490+0000", 2023-10-09T10:46:21.858 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5469000000000001e-05 2023-10-09T10:46:21.858 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.858 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.858 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.858 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.858 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.858 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70667 6.2 6:41f96b88:::benchmark_data_smithi125_244951_object70666:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:46:21.858 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.914837+0000", 2023-10-09T10:46:21.859 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 57.885611877999999, 2023-10-09T10:46:21.859 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038692030000000002, 2023-10-09T10:46:21.859 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.859 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.859 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.859 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.859 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.859 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70667 2023-10-09T10:46:21.859 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.859 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.859 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.859 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.859 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.914837+0000", 2023-10-09T10:46:21.860 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.860 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.860 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.860 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.860 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.914837+0000", 2023-10-09T10:46:21.860 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.860 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.860 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.860 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.860 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.914836+0000", 2023-10-09T10:46:21.860 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:46:21.860 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.861 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.861 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.861 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.914893+0000", 2023-10-09T10:46:21.861 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7256000000000002e-05 2023-10-09T10:46:21.861 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.861 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.861 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.861 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.914904+0000", 2023-10-09T10:46:21.861 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1450999999999999e-05 2023-10-09T10:46:21.861 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.861 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.861 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.861 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.914911+0000", 2023-10-09T10:46:21.862 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5420000000000002e-06 2023-10-09T10:46:21.862 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.862 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.862 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.862 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.914959+0000", 2023-10-09T10:46:21.862 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8092e-05 2023-10-09T10:46:21.862 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.862 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.862 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.862 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.915086+0000", 2023-10-09T10:46:21.862 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012746099999999999 2023-10-09T10:46:21.862 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.863 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.863 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:46:21.863 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.915248+0000", 2023-10-09T10:46:21.863 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016123999999999999 2023-10-09T10:46:21.863 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.863 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.863 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.863 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.916735+0000", 2023-10-09T10:46:21.863 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014875940000000001 2023-10-09T10:46:21.863 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.863 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.863 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.864 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.919588+0000", 2023-10-09T10:46:21.864 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028527520000000001 2023-10-09T10:46:21.864 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.864 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.864 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.864 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.953430+0000", 2023-10-09T10:46:21.864 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.033842301999999998 2023-10-09T10:46:21.864 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.864 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.864 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.864 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.953487+0000", 2023-10-09T10:46:21.864 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6963999999999999e-05 2023-10-09T10:46:21.864 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.865 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.865 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.865 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.953529+0000", 2023-10-09T10:46:21.865 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1834e-05 2023-10-09T10:46:21.865 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.865 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.865 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.865 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.865 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.865 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70678 6.2 6:46681df3:::benchmark_data_smithi125_244951_object70677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:46:21.865 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.928282+0000", 2023-10-09T10:46:21.865 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 57.872167179999998, 2023-10-09T10:46:21.865 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.035938168, 2023-10-09T10:46:21.866 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.866 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.866 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.866 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.866 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.866 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70678 2023-10-09T10:46:21.866 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.866 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.866 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.866 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.866 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.928282+0000", 2023-10-09T10:46:21.866 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.867 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.867 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.867 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.867 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.928282+0000", 2023-10-09T10:46:21.867 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.867 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.867 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.867 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.867 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.928280+0000", 2023-10-09T10:46:21.867 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:46:21.867 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.867 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.868 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.868 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.928334+0000", 2023-10-09T10:46:21.868 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4311999999999998e-05 2023-10-09T10:46:21.868 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.868 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.868 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.868 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.928349+0000", 2023-10-09T10:46:21.868 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.451e-05 2023-10-09T10:46:21.868 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.868 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.868 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.868 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.928356+0000", 2023-10-09T10:46:21.868 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8199999999999999e-06 2023-10-09T10:46:21.869 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.869 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.869 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.869 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.928400+0000", 2023-10-09T10:46:21.869 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4585000000000001e-05 2023-10-09T10:46:21.869 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.869 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.869 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.869 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.928505+0000", 2023-10-09T10:46:21.869 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104786 2023-10-09T10:46:21.869 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.869 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.869 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:46:21.870 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.928600+0000", 2023-10-09T10:46:21.870 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4648000000000006e-05 2023-10-09T10:46:21.870 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.870 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.870 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.870 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.929706+0000", 2023-10-09T10:46:21.870 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011063430000000001 2023-10-09T10:46:21.870 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.870 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.870 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.870 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.938024+0000", 2023-10-09T10:46:21.870 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0083179080000000006 2023-10-09T10:46:21.870 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.871 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.871 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.871 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.964138+0000", 2023-10-09T10:46:21.871 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026113684000000002 2023-10-09T10:46:21.871 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.871 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.871 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.871 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.964181+0000", 2023-10-09T10:46:21.871 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3854999999999999e-05 2023-10-09T10:46:21.871 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.871 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.871 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.872 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.964220+0000", 2023-10-09T10:46:21.872 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8485999999999997e-05 2023-10-09T10:46:21.872 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.872 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.872 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.872 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.872 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.872 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:77920 6.e 6:7756cfbe:::benchmark_data_smithi125_244951_object77919:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T10:46:21.872 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:31.218724+0000", 2023-10-09T10:46:21.872 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 50.581724688000001, 2023-10-09T10:46:21.872 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.044091884999999997, 2023-10-09T10:46:21.872 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.872 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.873 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.873 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.873 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.873 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 77920 2023-10-09T10:46:21.873 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.873 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.873 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.873 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.873 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.218724+0000", 2023-10-09T10:46:21.873 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.873 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.873 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.873 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.874 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.218724+0000", 2023-10-09T10:46:21.874 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.874 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.874 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.874 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.874 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.218723+0000", 2023-10-09T10:46:21.874 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:46:21.874 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.874 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.874 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.874 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.218825+0000", 2023-10-09T10:46:21.874 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000102568 2023-10-09T10:46:21.875 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.875 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.875 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.875 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.218845+0000", 2023-10-09T10:46:21.875 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9564000000000001e-05 2023-10-09T10:46:21.875 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.875 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.875 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.875 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.218852+0000", 2023-10-09T10:46:21.875 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5480000000000004e-06 2023-10-09T10:46:21.875 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.875 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.875 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.876 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.218901+0000", 2023-10-09T10:46:21.876 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8398e-05 2023-10-09T10:46:21.876 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.876 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.876 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.876 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.219021+0000", 2023-10-09T10:46:21.876 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011995800000000001 2023-10-09T10:46:21.876 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.876 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.876 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:46:21.876 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.219159+0000", 2023-10-09T10:46:21.876 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013842600000000001 2023-10-09T10:46:21.876 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.877 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.877 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.877 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.221756+0000", 2023-10-09T10:46:21.877 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0025973350000000001 2023-10-09T10:46:21.877 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.877 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.877 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.877 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.233416+0000", 2023-10-09T10:46:21.877 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011659902999999999 2023-10-09T10:46:21.877 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.877 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.877 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.877 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.262710+0000", 2023-10-09T10:46:21.878 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029294192 2023-10-09T10:46:21.878 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.878 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.878 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.878 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.262774+0000", 2023-10-09T10:46:21.878 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4063e-05 2023-10-09T10:46:21.878 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.878 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.878 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.878 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.262816+0000", 2023-10-09T10:46:21.878 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1656000000000002e-05 2023-10-09T10:46:21.878 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.879 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.879 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.879 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.879 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.879 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:77928 6.e 6:764033fb:::benchmark_data_smithi125_244951_object77927:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T10:46:21.879 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:31.225238+0000", 2023-10-09T10:46:21.879 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 50.575210462000001, 2023-10-09T10:46:21.879 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.037711020999999997, 2023-10-09T10:46:21.879 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.879 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.879 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.879 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.879 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.880 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 77928 2023-10-09T10:46:21.880 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.880 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.880 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.880 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.880 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.225238+0000", 2023-10-09T10:46:21.880 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.880 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.880 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.880 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.880 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.225238+0000", 2023-10-09T10:46:21.880 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.881 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.881 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.881 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.881 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.225237+0000", 2023-10-09T10:46:21.881 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:46:21.881 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.881 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.881 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.881 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.225290+0000", 2023-10-09T10:46:21.881 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3350000000000003e-05 2023-10-09T10:46:21.881 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.881 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.881 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.882 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.225302+0000", 2023-10-09T10:46:21.882 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1837e-05 2023-10-09T10:46:21.882 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.882 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.882 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.882 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.225307+0000", 2023-10-09T10:46:21.882 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1479999999999997e-06 2023-10-09T10:46:21.882 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.882 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.882 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.882 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.225352+0000", 2023-10-09T10:46:21.882 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.464e-05 2023-10-09T10:46:21.882 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.225494+0000", 2023-10-09T10:46:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014222099999999999 2023-10-09T10:46:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:46:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.225649+0000", 2023-10-09T10:46:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000155642 2023-10-09T10:46:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.883 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.231662+0000", 2023-10-09T10:46:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.006012791 2023-10-09T10:46:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.233443+0000", 2023-10-09T10:46:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0017808399999999999 2023-10-09T10:46:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.262863+0000", 2023-10-09T10:46:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029419636999999998 2023-10-09T10:46:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.884 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.885 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.885 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.262913+0000", 2023-10-09T10:46:21.885 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0318999999999997e-05 2023-10-09T10:46:21.885 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.885 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.885 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.885 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.262950+0000", 2023-10-09T10:46:21.885 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6470000000000001e-05 2023-10-09T10:46:21.885 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.885 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.885 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.885 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.885 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.885 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:77934 6.2 6:4337b86b:::benchmark_data_smithi125_244951_object77933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T10:46:21.886 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:31.238712+0000", 2023-10-09T10:46:21.886 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 50.561737393000001, 2023-10-09T10:46:21.886 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.032939229, 2023-10-09T10:46:21.886 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.886 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.886 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.886 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.886 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.886 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 77934 2023-10-09T10:46:21.886 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.886 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.886 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.886 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.887 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.238712+0000", 2023-10-09T10:46:21.887 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.887 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.887 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.887 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.887 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.238712+0000", 2023-10-09T10:46:21.887 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.887 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.887 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.887 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.887 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.238706+0000", 2023-10-09T10:46:21.887 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999948 2023-10-09T10:46:21.888 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.888 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.888 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.888 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.238766+0000", 2023-10-09T10:46:21.888 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9907000000000001e-05 2023-10-09T10:46:21.888 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.888 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.888 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.888 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.238784+0000", 2023-10-09T10:46:21.888 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7574000000000001e-05 2023-10-09T10:46:21.888 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.888 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.888 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.889 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.238792+0000", 2023-10-09T10:46:21.889 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1839999999999999e-06 2023-10-09T10:46:21.889 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.889 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.889 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.889 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.238843+0000", 2023-10-09T10:46:21.889 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1137000000000002e-05 2023-10-09T10:46:21.889 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.889 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.889 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op must wait for map", 2023-10-09T10:46:21.889 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.238850+0000", 2023-10-09T10:46:21.889 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0899999999999999e-06 2023-10-09T10:46:21.889 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.890 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.890 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.890 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.242576+0000", 2023-10-09T10:46:21.890 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0037259760000000002 2023-10-09T10:46:21.890 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.890 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.890 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.890 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.242721+0000", 2023-10-09T10:46:21.890 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014536700000000001 2023-10-09T10:46:21.890 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.890 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.890 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:46:21.890 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.242888+0000", 2023-10-09T10:46:21.891 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000166618 2023-10-09T10:46:21.891 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.891 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.891 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.891 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.244564+0000", 2023-10-09T10:46:21.891 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001675738 2023-10-09T10:46:21.891 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.891 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.891 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.891 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.246597+0000", 2023-10-09T10:46:21.891 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020328400000000002 2023-10-09T10:46:21.891 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.891 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.891 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.892 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.271544+0000", 2023-10-09T10:46:21.892 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024947463 2023-10-09T10:46:21.892 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.892 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.892 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.892 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.271601+0000", 2023-10-09T10:46:21.892 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6733e-05 2023-10-09T10:46:21.892 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.892 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.892 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.892 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:31.271651+0000", 2023-10-09T10:46:21.892 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0006000000000002e-05 2023-10-09T10:46:21.892 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.893 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.893 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.893 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.893 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.893 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:95396 6.e 6:7010460e:::benchmark_data_smithi125_244951_object95395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-10-09T10:46:21.893 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:50.610096+0000", 2023-10-09T10:46:21.893 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 31.190352722, 2023-10-09T10:46:21.893 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.034449430000000003, 2023-10-09T10:46:21.893 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.893 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.893 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.893 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.893 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.894 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 95396 2023-10-09T10:46:21.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.894 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.894 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.894 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.894 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:50.610096+0000", 2023-10-09T10:46:21.894 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.894 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.894 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.894 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:50.610096+0000", 2023-10-09T10:46:21.894 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.894 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.895 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.895 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:50.610095+0000", 2023-10-09T10:46:21.895 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:46:21.895 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.895 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.895 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.895 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:50.610291+0000", 2023-10-09T10:46:21.895 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00019603399999999999 2023-10-09T10:46:21.895 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.895 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.895 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.895 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:50.610308+0000", 2023-10-09T10:46:21.895 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6824e-05 2023-10-09T10:46:21.896 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.896 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.896 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.896 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:50.610315+0000", 2023-10-09T10:46:21.896 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0670000000000003e-06 2023-10-09T10:46:21.896 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.896 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.896 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.896 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:50.610357+0000", 2023-10-09T10:46:21.896 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2469000000000003e-05 2023-10-09T10:46:21.896 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.896 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.896 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.896 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:50.610479+0000", 2023-10-09T10:46:21.897 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000121531 2023-10-09T10:46:21.897 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.897 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.897 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:46:21.897 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:50.610603+0000", 2023-10-09T10:46:21.897 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012378500000000001 2023-10-09T10:46:21.897 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.897 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.897 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.897 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:50.616024+0000", 2023-10-09T10:46:21.897 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0054213179999999996 2023-10-09T10:46:21.897 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.897 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.897 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:50.621261+0000", 2023-10-09T10:46:21.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0052373480000000002 2023-10-09T10:46:21.898 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.898 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:50.644440+0000", 2023-10-09T10:46:21.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023178473000000002 2023-10-09T10:46:21.898 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.898 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:50.644504+0000", 2023-10-09T10:46:21.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3824000000000004e-05 2023-10-09T10:46:21.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.899 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.899 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.899 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:50.644546+0000", 2023-10-09T10:46:21.899 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2135999999999998e-05 2023-10-09T10:46:21.899 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.899 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.899 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.899 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.899 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106044 6.e 6:7e277d56:::benchmark_data_smithi125_244951_object106043:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:46:21.899 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:01.825735+0000", 2023-10-09T10:46:21.899 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 19.974713640000001, 2023-10-09T10:46:21.900 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.042768898, 2023-10-09T10:46:21.900 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.900 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.900 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.900 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.900 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.900 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106044 2023-10-09T10:46:21.900 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.900 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.900 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.900 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.900 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.825735+0000", 2023-10-09T10:46:21.900 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.900 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.901 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.901 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.901 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.825735+0000", 2023-10-09T10:46:21.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.901 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.901 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.901 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.825734+0000", 2023-10-09T10:46:21.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:46:21.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.901 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.901 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.901 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.825775+0000", 2023-10-09T10:46:21.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1013999999999997e-05 2023-10-09T10:46:21.902 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.902 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.825789+0000", 2023-10-09T10:46:21.902 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4531e-05 2023-10-09T10:46:21.902 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.902 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.825796+0000", 2023-10-09T10:46:21.902 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6800000000000004e-06 2023-10-09T10:46:21.902 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.903 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.825841+0000", 2023-10-09T10:46:21.903 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4808999999999997e-05 2023-10-09T10:46:21.903 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.903 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.903 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.903 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.826000+0000", 2023-10-09T10:46:21.903 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000159474 2023-10-09T10:46:21.903 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.903 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.903 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:46:21.903 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.826146+0000", 2023-10-09T10:46:21.903 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014548 2023-10-09T10:46:21.903 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.903 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.904 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.904 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.829324+0000", 2023-10-09T10:46:21.904 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0031787120000000002 2023-10-09T10:46:21.904 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.904 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.904 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.904 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.832495+0000", 2023-10-09T10:46:21.904 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0031701049999999999 2023-10-09T10:46:21.904 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.904 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.904 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.904 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.868438+0000", 2023-10-09T10:46:21.904 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.035942927 2023-10-09T10:46:21.905 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.905 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.905 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.868478+0000", 2023-10-09T10:46:21.905 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0414000000000003e-05 2023-10-09T10:46:21.905 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.905 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.905 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.868504+0000", 2023-10-09T10:46:21.905 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6282e-05 2023-10-09T10:46:21.905 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.905 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.905 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.905 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.906 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.906 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106419 6.e 6:753f6003:::benchmark_data_smithi125_244951_object106418:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:46:21.906 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.234314+0000", 2023-10-09T10:46:21.906 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 19.566135268, 2023-10-09T10:46:21.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.042116606000000001, 2023-10-09T10:46:21.906 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.906 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.906 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.906 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.906 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.906 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106419 2023-10-09T10:46:21.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.907 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.907 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.907 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234314+0000", 2023-10-09T10:46:21.907 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.907 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.907 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.907 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234314+0000", 2023-10-09T10:46:21.907 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.907 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.907 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.908 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234313+0000", 2023-10-09T10:46:21.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:46:21.908 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.908 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.908 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.908 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234350+0000", 2023-10-09T10:46:21.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6497e-05 2023-10-09T10:46:21.908 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.908 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.908 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.908 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234357+0000", 2023-10-09T10:46:21.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3250000000000002e-06 2023-10-09T10:46:21.908 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.909 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.909 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234362+0000", 2023-10-09T10:46:21.909 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0209999999999999e-06 2023-10-09T10:46:21.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.909 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.909 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234408+0000", 2023-10-09T10:46:21.909 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5970999999999997e-05 2023-10-09T10:46:21.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.909 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.909 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234531+0000", 2023-10-09T10:46:21.909 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012280599999999999 2023-10-09T10:46:21.910 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.910 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:46:21.910 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234649+0000", 2023-10-09T10:46:21.910 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011771 2023-10-09T10:46:21.910 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.910 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.910 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.237903+0000", 2023-10-09T10:46:21.910 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003254659 2023-10-09T10:46:21.910 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.910 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.911 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.246942+0000", 2023-10-09T10:46:21.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0090389019999999997 2023-10-09T10:46:21.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.911 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.911 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.911 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.276345+0000", 2023-10-09T10:46:21.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029402414000000002 2023-10-09T10:46:21.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.911 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.911 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.911 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.276394+0000", 2023-10-09T10:46:21.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9537999999999999e-05 2023-10-09T10:46:21.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.276430+0000", 2023-10-09T10:46:21.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6202999999999999e-05 2023-10-09T10:46:21.912 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.912 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.912 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.912 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106431 6.2 6:45f89566:::benchmark_data_smithi125_244951_object106430:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:46:21.912 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.249006+0000", 2023-10-09T10:46:21.912 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 19.551443294999999, 2023-10-09T10:46:21.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.042410528000000003, 2023-10-09T10:46:21.912 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.913 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.913 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.913 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.913 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.913 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106431 2023-10-09T10:46:21.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.913 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.913 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249006+0000", 2023-10-09T10:46:21.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.914 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.914 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.914 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249006+0000", 2023-10-09T10:46:21.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.914 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.914 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.914 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249004+0000", 2023-10-09T10:46:21.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:46:21.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.914 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.914 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.914 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249076+0000", 2023-10-09T10:46:21.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1767000000000006e-05 2023-10-09T10:46:21.915 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.915 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249091+0000", 2023-10-09T10:46:21.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5156e-05 2023-10-09T10:46:21.915 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.915 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249098+0000", 2023-10-09T10:46:21.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4350000000000002e-06 2023-10-09T10:46:21.915 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249143+0000", 2023-10-09T10:46:21.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4950999999999997e-05 2023-10-09T10:46:21.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.916 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249271+0000", 2023-10-09T10:46:21.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012820900000000001 2023-10-09T10:46:21.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.916 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T10:46:21.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249374+0000", 2023-10-09T10:46:21.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000102947 2023-10-09T10:46:21.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.917 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.917 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.251043+0000", 2023-10-09T10:46:21.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0016687659999999999 2023-10-09T10:46:21.917 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.917 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.917 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.261568+0000", 2023-10-09T10:46:21.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010525889 2023-10-09T10:46:21.917 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.917 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.917 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.291343+0000", 2023-10-09T10:46:21.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029774345000000001 2023-10-09T10:46:21.918 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.918 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.918 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.918 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.291387+0000", 2023-10-09T10:46:21.918 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3803000000000001e-05 2023-10-09T10:46:21.918 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.918 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.918 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.918 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.291416+0000", 2023-10-09T10:46:21.918 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9624e-05 2023-10-09T10:46:21.918 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.918 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.918 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.919 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106673 6.e 6:7abcc940:::benchmark_data_smithi125_244951_object230:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-10-09T10:46:21.919 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:08.567554+0000", 2023-10-09T10:46:21.919 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 13.232895186, 2023-10-09T10:46:21.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.032343465000000002, 2023-10-09T10:46:21.919 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.919 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.919 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.919 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.919 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.919 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106673 2023-10-09T10:46:21.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.920 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.567554+0000", 2023-10-09T10:46:21.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.567554+0000", 2023-10-09T10:46:21.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.567553+0000", 2023-10-09T10:46:21.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:46:21.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.567556+0000", 2023-10-09T10:46:21.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3159999999999999e-06 2023-10-09T10:46:21.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.567568+0000", 2023-10-09T10:46:21.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.237e-05 2023-10-09T10:46:21.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.567572+0000", 2023-10-09T10:46:21.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9700000000000001e-06 2023-10-09T10:46:21.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.572878+0000", 2023-10-09T10:46:21.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0053054160000000003 2023-10-09T10:46:21.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.573008+0000", 2023-10-09T10:46:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013029 2023-10-09T10:46:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:46:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.573133+0000", 2023-10-09T10:46:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012481499999999999 2023-10-09T10:46:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.575688+0000", 2023-10-09T10:46:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0025554900000000001 2023-10-09T10:46:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.598866+0000", 2023-10-09T10:46:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023177745 2023-10-09T10:46:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.599818+0000", 2023-10-09T10:46:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00095225699999999997 2023-10-09T10:46:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.599862+0000", 2023-10-09T10:46:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4066000000000001e-05 2023-10-09T10:46:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.599897+0000", 2023-10-09T10:46:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5070999999999997e-05 2023-10-09T10:46:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:107523 6.e 6:7e796fd2:::benchmark_data_smithi125_244951_object1080:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-10-09T10:46:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:09.627332+0000", 2023-10-09T10:46:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 12.173117059999999, 2023-10-09T10:46:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.035484849999999998, 2023-10-09T10:46:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 107523 2023-10-09T10:46:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.627332+0000", 2023-10-09T10:46:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.627332+0000", 2023-10-09T10:46:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.627330+0000", 2023-10-09T10:46:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:46:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.627338+0000", 2023-10-09T10:46:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1419999999999993e-06 2023-10-09T10:46:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.627362+0000", 2023-10-09T10:46:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3527999999999999e-05 2023-10-09T10:46:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.627370+0000", 2023-10-09T10:46:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5920000000000001e-06 2023-10-09T10:46:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.627424+0000", 2023-10-09T10:46:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3421000000000002e-05 2023-10-09T10:46:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.627560+0000", 2023-10-09T10:46:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000135974 2023-10-09T10:46:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:46:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.627674+0000", 2023-10-09T10:46:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000114379 2023-10-09T10:46:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.633665+0000", 2023-10-09T10:46:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.005990698 2023-10-09T10:46:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.647830+0000", 2023-10-09T10:46:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014165000000000001 2023-10-09T10:46:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.662727+0000", 2023-10-09T10:46:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014897305 2023-10-09T10:46:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.662777+0000", 2023-10-09T10:46:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9812999999999999e-05 2023-10-09T10:46:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.662817+0000", 2023-10-09T10:46:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9657999999999999e-05 2023-10-09T10:46:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:107529 6.e 6:7f6820c7:::benchmark_data_smithi125_244951_object1086:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-10-09T10:46:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:09.627420+0000", 2023-10-09T10:46:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 12.173028468, 2023-10-09T10:46:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.035486878999999999, 2023-10-09T10:46:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:46:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 107529 2023-10-09T10:46:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.627420+0000", 2023-10-09T10:46:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.627420+0000", 2023-10-09T10:46:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.627420+0000", 2023-10-09T10:46:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:46:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.627423+0000", 2023-10-09T10:46:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.242e-06 2023-10-09T10:46:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.627434+0000", 2023-10-09T10:46:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1114000000000001e-05 2023-10-09T10:46:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.627439+0000", 2023-10-09T10:46:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8670000000000002e-06 2023-10-09T10:46:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.630675+0000", 2023-10-09T10:46:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003236311 2023-10-09T10:46:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.630805+0000", 2023-10-09T10:46:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000129958 2023-10-09T10:46:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:46:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.630917+0000", 2023-10-09T10:46:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000111178 2023-10-09T10:46:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.633786+0000", 2023-10-09T10:46:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028690679999999998 2023-10-09T10:46:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.651844+0000", 2023-10-09T10:46:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018058146000000001 2023-10-09T10:46:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.662853+0000", 2023-10-09T10:46:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011009207 2023-10-09T10:46:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:46:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.662884+0000", 2023-10-09T10:46:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.074e-05 2023-10-09T10:46:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:46:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:09.662907+0000", 2023-10-09T10:46:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.357e-05 2023-10-09T10:46:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:21.939 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:46:21.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:21.941 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:21.941 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:21.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:22.042 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:22.111 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:22.136 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:22.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:22.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:22.166 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:22 smithi125 ceph-mon[195507]: pgmap v305: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 8135/292557 objects degraded (2.781%); 7904/292557 objects misplaced (2.702%); 12 MiB/s, 185 objects/s recovering 2023-10-09T10:46:22.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:22 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2092435982' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-10-09T10:46:22.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:22 smithi125 ceph-mon[195507]: osdmap e151: 8 total, 8 up, 8 in 2023-10-09T10:46:22.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:22 smithi125 ceph-mon[195507]: failed to encode map e151 with expected crc 2023-10-09T10:46:22.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:22 smithi125 ceph-mon[195507]: failed to encode map e151 with expected crc 2023-10-09T10:46:22.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:22 smithi125 ceph-mon[195507]: failed to encode map e151 with expected crc 2023-10-09T10:46:22.167 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:22 smithi125 ceph-mon[200197]: pgmap v305: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 8135/292557 objects degraded (2.781%); 7904/292557 objects misplaced (2.702%); 12 MiB/s, 185 objects/s recovering 2023-10-09T10:46:22.167 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:22 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2092435982' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-10-09T10:46:22.167 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:22 smithi125 ceph-mon[200197]: osdmap e151: 8 total, 8 up, 8 in 2023-10-09T10:46:22.167 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:22 smithi125 ceph-mon[200197]: failed to encode map e151 with expected crc 2023-10-09T10:46:22.168 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:22 smithi125 ceph-mon[200197]: failed to encode map e151 with expected crc 2023-10-09T10:46:22.168 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:22 smithi125 ceph-mon[200197]: failed to encode map e151 with expected crc 2023-10-09T10:46:22.177 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:46:22.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:22.307 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:22.348 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:22.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:22.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:22.454 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:22.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:22 smithi155 ceph-mon[184854]: pgmap v305: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 8135/292557 objects degraded (2.781%); 7904/292557 objects misplaced (2.702%); 12 MiB/s, 185 objects/s recovering 2023-10-09T10:46:22.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:22 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2092435982' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-10-09T10:46:22.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:22 smithi155 ceph-mon[184854]: osdmap e151: 8 total, 8 up, 8 in 2023-10-09T10:46:22.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:22 smithi155 ceph-mon[184854]: failed to encode map e151 with expected crc 2023-10-09T10:46:22.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:22 smithi155 ceph-mon[184854]: failed to encode map e151 with expected crc 2023-10-09T10:46:22.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:22 smithi155 ceph-mon[184854]: failed to encode map e151 with expected crc 2023-10-09T10:46:22.520 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:22.550 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:22.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:22.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:22.575 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:46:22.654 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:22.704 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:22.736 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:22.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:22.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:22.841 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:22.877 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:23.034 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:46:23.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:23 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3865326381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:23.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:23 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 8135/292557 objects degraded (2.781%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:23.226 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:46:23.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:23.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:23.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:23.413 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:23.454 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:23.496 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:23 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3865326381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:23.496 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:23 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 8135/292557 objects degraded (2.781%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:23.496 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:23 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3865326381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:23.497 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:23 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 8135/292557 objects degraded (2.781%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:23.531 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:23.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:23.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:23.636 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:23.677 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:23.697 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:23.751 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:23.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:23.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:23.860 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:23.921 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:23.948 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:23.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:23.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:23.983 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:46:23.983 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:46:23.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.983 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:119236 6.a 6:5cab942c:::benchmark_data_smithi125_244951_object12793:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-10-09T10:46:23.983 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:23.966776+0000", 2023-10-09T10:46:23.984 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.015937422, 2023-10-09T10:46:23.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015969100999999999, 2023-10-09T10:46:23.984 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:23.984 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:46:23.984 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:23.984 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:23.984 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:23.984 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 119236 2023-10-09T10:46:23.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.984 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:23.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:23.985 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.966776+0000", 2023-10-09T10:46:23.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:23.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.985 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:23.985 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.966776+0000", 2023-10-09T10:46:23.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:23.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.985 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:23.985 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.966775+0000", 2023-10-09T10:46:23.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:46:23.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:23.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.966781+0000", 2023-10-09T10:46:23.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.494e-06 2023-10-09T10:46:23.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:23.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.966792+0000", 2023-10-09T10:46:23.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0288999999999999e-05 2023-10-09T10:46:23.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:23.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.966802+0000", 2023-10-09T10:46:23.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0154e-05 2023-10-09T10:46:23.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:23.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.971136+0000", 2023-10-09T10:46:23.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043345600000000003 2023-10-09T10:46:23.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:23.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.971284+0000", 2023-10-09T10:46:23.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014743800000000001 2023-10-09T10:46:23.988 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.988 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:46:23.988 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.971438+0000", 2023-10-09T10:46:23.988 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015417600000000001 2023-10-09T10:46:23.988 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.988 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:23.988 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.980232+0000", 2023-10-09T10:46:23.988 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0087942890000000003 2023-10-09T10:46:23.988 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.988 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:23.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.980298+0000", 2023-10-09T10:46:23.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5689000000000001e-05 2023-10-09T10:46:23.989 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:23.989 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:23.989 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:23.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.989 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:119242 6.d 6:bc53aa11:::benchmark_data_smithi125_244951_object12799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-10-09T10:46:23.989 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:23.978334+0000", 2023-10-09T10:46:23.989 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0043797840000000003, 2023-10-09T10:46:23.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0044748690000000002, 2023-10-09T10:46:23.989 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:23.989 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:46:23.989 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:23.990 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:23.990 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:23.990 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 119242 2023-10-09T10:46:23.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.990 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:23.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:23.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978334+0000", 2023-10-09T10:46:23.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:23.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:23.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978334+0000", 2023-10-09T10:46:23.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:23.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:23.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978333+0000", 2023-10-09T10:46:23.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:46:23.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:23.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978339+0000", 2023-10-09T10:46:23.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2639999999999997e-06 2023-10-09T10:46:23.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:23.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978351+0000", 2023-10-09T10:46:23.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2312e-05 2023-10-09T10:46:23.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:23.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978357+0000", 2023-10-09T10:46:23.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2530000000000001e-06 2023-10-09T10:46:23.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:23.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978438+0000", 2023-10-09T10:46:23.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0496999999999999e-05 2023-10-09T10:46:23.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:23.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978580+0000", 2023-10-09T10:46:23.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000142053 2023-10-09T10:46:23.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:46:23.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978716+0000", 2023-10-09T10:46:23.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013628199999999999 2023-10-09T10:46:23.995 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.995 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.995 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:23.995 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.982051+0000", 2023-10-09T10:46:23.995 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0033350839999999999 2023-10-09T10:46:23.995 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.995 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.995 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:23.995 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.982759+0000", 2023-10-09T10:46:23.995 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00070807499999999996 2023-10-09T10:46:23.995 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:23.995 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:23.995 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:23.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.996 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:119245 6.a 6:56587171:::benchmark_data_smithi125_244951_object12802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-10-09T10:46:23.996 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:23.978380+0000", 2023-10-09T10:46:23.996 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0043337269999999999, 2023-10-09T10:46:23.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0044786339999999996, 2023-10-09T10:46:23.996 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:23.996 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:46:23.996 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:23.996 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:23.996 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:23.996 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 119245 2023-10-09T10:46:23.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.997 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:23.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:23.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978380+0000", 2023-10-09T10:46:23.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:23.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:23.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978380+0000", 2023-10-09T10:46:23.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:23.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:23.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978379+0000", 2023-10-09T10:46:23.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:46:23.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:23.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978382+0000", 2023-10-09T10:46:23.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9129999999999999e-06 2023-10-09T10:46:23.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:23.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978388+0000", 2023-10-09T10:46:23.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9270000000000002e-06 2023-10-09T10:46:23.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:23.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978391+0000", 2023-10-09T10:46:23.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8710000000000001e-06 2023-10-09T10:46:23.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:23.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.980342+0000", 2023-10-09T10:46:23.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001951249 2023-10-09T10:46:23.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:23.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:23.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:23.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.980510+0000", 2023-10-09T10:46:23.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016796899999999999 2023-10-09T10:46:24.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.000 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:46:24.000 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.980666+0000", 2023-10-09T10:46:24.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015576000000000001 2023-10-09T10:46:24.000 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:24.000 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:24.000 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:24.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.000 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:119246 6.a 6.890dad1a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-10-09T10:46:24.000 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:23.978395+0000", 2023-10-09T10:46:24.000 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0043183969999999999, 2023-10-09T10:46:24.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045049729999999998, 2023-10-09T10:46:24.001 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:24.001 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "queued for pg", 2023-10-09T10:46:24.001 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:24.001 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:24.001 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:24.001 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 119246 2023-10-09T10:46:24.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.001 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:24.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:24.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978395+0000", 2023-10-09T10:46:24.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:24.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:24.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978395+0000", 2023-10-09T10:46:24.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:24.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:24.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978395+0000", 2023-10-09T10:46:24.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967296 2023-10-09T10:46:24.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:24.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978397+0000", 2023-10-09T10:46:24.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.84e-06 2023-10-09T10:46:24.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:24.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978400+0000", 2023-10-09T10:46:24.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6550000000000002e-06 2023-10-09T10:46:24.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:24.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.978404+0000", 2023-10-09T10:46:24.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.411e-06 2023-10-09T10:46:24.003 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:24.003 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:24.004 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:24.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.004 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:119181 6.11 e152/143 6:8a265801:::benchmark_data_smithi125_244951_object12738:head v 152'4701, mlcod=104'883)", 2023-10-09T10:46:24.004 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:23.982448+0000", 2023-10-09T10:46:24.004 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.00026579000000000001, 2023-10-09T10:46:24.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00049250200000000002, 2023-10-09T10:46:24.004 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:24.004 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:46:24.004 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:24.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:24.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.982448+0000", 2023-10-09T10:46:24.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:24.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:24.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.982448+0000", 2023-10-09T10:46:24.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:24.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:24.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.982447+0000", 2023-10-09T10:46:24.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:46:24.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:24.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.982452+0000", 2023-10-09T10:46:24.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.378e-06 2023-10-09T10:46:24.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:24.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.982462+0000", 2023-10-09T10:46:24.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5999999999999996e-06 2023-10-09T10:46:24.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:24.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.982472+0000", 2023-10-09T10:46:24.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0121e-05 2023-10-09T10:46:24.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.007 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:24.007 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.982509+0000", 2023-10-09T10:46:24.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6529999999999998e-05 2023-10-09T10:46:24.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:24.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:24.007 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:24.007 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:23.982532+0000", 2023-10-09T10:46:24.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3366e-05 2023-10-09T10:46:24.007 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:24.008 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:24.008 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:24.008 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:24.008 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T10:46:24.008 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 5 2023-10-09T10:46:24.008 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:46:24.053 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:24.100 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:24.133 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:24.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:24.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:24.181 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:24 smithi125 ceph-mon[195507]: pgmap v307: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 8135/290505 objects degraded (2.800%); 7734/290505 objects misplaced (2.662%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:46:24.181 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:24 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:24.181 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:24 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3865326381' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:46:24.182 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:24 smithi125 ceph-mon[195507]: osdmap e152: 8 total, 8 up, 8 in 2023-10-09T10:46:24.182 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:24 smithi125 ceph-mon[195507]: failed to encode map e152 with expected crc 2023-10-09T10:46:24.182 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:24 smithi125 ceph-mon[195507]: failed to encode map e152 with expected crc 2023-10-09T10:46:24.182 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:24 smithi125 ceph-mon[195507]: failed to encode map e152 with expected crc 2023-10-09T10:46:24.182 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:24 smithi125 ceph-mon[200197]: pgmap v307: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 8135/290505 objects degraded (2.800%); 7734/290505 objects misplaced (2.662%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:46:24.182 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:24 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:24.182 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:24 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3865326381' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:46:24.182 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:24 smithi125 ceph-mon[200197]: osdmap e152: 8 total, 8 up, 8 in 2023-10-09T10:46:24.182 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:24 smithi125 ceph-mon[200197]: failed to encode map e152 with expected crc 2023-10-09T10:46:24.183 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:24 smithi125 ceph-mon[200197]: failed to encode map e152 with expected crc 2023-10-09T10:46:24.183 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:24 smithi125 ceph-mon[200197]: failed to encode map e152 with expected crc 2023-10-09T10:46:24.237 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:24.272 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:24 smithi155 ceph-mon[184854]: pgmap v307: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 8135/290505 objects degraded (2.800%); 7734/290505 objects misplaced (2.662%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:46:24.272 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:24 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:24.272 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:24 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3865326381' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:46:24.272 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:24 smithi155 ceph-mon[184854]: osdmap e152: 8 total, 8 up, 8 in 2023-10-09T10:46:24.272 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:24 smithi155 ceph-mon[184854]: failed to encode map e152 with expected crc 2023-10-09T10:46:24.272 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:24 smithi155 ceph-mon[184854]: failed to encode map e152 with expected crc 2023-10-09T10:46:24.272 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:24 smithi155 ceph-mon[184854]: failed to encode map e152 with expected crc 2023-10-09T10:46:24.290 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:24.324 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:24.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:24.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:24.377 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:46:24.429 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:24.476 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:24.504 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:24.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:24.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:24.615 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:24.667 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:24.693 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:24.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:24.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:24.774 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:46:24.801 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:24.850 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:24.943 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:24.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:24.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:25.046 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:25.088 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:25.117 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:25.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:25.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:25.159 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:46:25.177 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:46:25.222 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:46:25.224 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:25.270 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:46:25.281 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:25.310 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:46:25.318 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:25.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:25.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:25.348 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:46:25.383 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:46:25.425 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:25.440 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:46:25.457 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:46:25.457 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:46:25.458 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:46:25.458 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:46:25.458 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:46:25.458 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:46:25.458 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:46:25.458 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:46:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3587058214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:25.483 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3587058214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:25.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3587058214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:25.498 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:25.533 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:25.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:25.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:25.643 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:25.690 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:46:25.698 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:25.727 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:25.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:25.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:25.833 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:25.876 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:46:25.876 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:46:25.915 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:25.959 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:25.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:25.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:26.064 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:26.134 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:26.262 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:46:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:26 smithi155 ceph-mon[184854]: pgmap v309: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 8135/284256 objects degraded (2.862%); 7615/284256 objects misplaced (2.679%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T10:46:26.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:26 smithi125 ceph-mon[195507]: pgmap v309: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 8135/284256 objects degraded (2.862%); 7615/284256 objects misplaced (2.679%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T10:46:26.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:26 smithi125 ceph-mon[200197]: pgmap v309: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 8135/284256 objects degraded (2.862%); 7615/284256 objects misplaced (2.679%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T10:46:26.602 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:46:26.775 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:26.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:26.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:26.861 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:26.882 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:26.931 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:26.941 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:46:26.941 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:46:26.941 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T10:46:26.941 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-09T10:46:27.625 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:46:27.625 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:46:28.004 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:46:28.208 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:28 smithi155 ceph-mon[184854]: pgmap v310: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 8135/278130 objects degraded (2.925%); 7217/278130 objects misplaced (2.595%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:46:28.208 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:28 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 8135/278130 objects degraded (2.925%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:28.325 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:46:28.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:28 smithi125 ceph-mon[200197]: pgmap v310: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 8135/278130 objects degraded (2.925%); 7217/278130 objects misplaced (2.595%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:46:28.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:28 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 8135/278130 objects degraded (2.925%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:28.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:28 smithi125 ceph-mon[195507]: pgmap v310: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 8135/278130 objects degraded (2.925%); 7217/278130 objects misplaced (2.595%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:46:28.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:28 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 8135/278130 objects degraded (2.925%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:28.691 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:28.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:28.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:28.799 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:28.831 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:29.209 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:29.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:29.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:29.318 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:29.340 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:46:29.340 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:46:29.355 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:29.379 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:29.381 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:29.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:29.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:29.483 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:29.519 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:29.694 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:29.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:29.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:29.714 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:46:29.799 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:29.833 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:29.858 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:29.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:29.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:29.962 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:30.008 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:30.070 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:46:30.097 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:30.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:30.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:30.201 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:30.212 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:30 smithi125 ceph-mon[195507]: pgmap v311: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 8020/274905 objects degraded (2.917%); 7217/274905 objects misplaced (2.625%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T10:46:30.212 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:30.212 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:46:30.250 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:30.285 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:30.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:30.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:30.391 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:30.424 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:30 smithi155 ceph-mon[184854]: pgmap v311: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 8020/274905 objects degraded (2.917%); 7217/274905 objects misplaced (2.625%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T10:46:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:46:30.522 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:30 smithi125 ceph-mon[200197]: pgmap v311: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 8020/274905 objects degraded (2.917%); 7217/274905 objects misplaced (2.625%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T10:46:30.522 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:46:30.522 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:46:30.649 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:30.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:30.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:30.753 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:30.796 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:30.820 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:30.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:30.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:30.924 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:30.969 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:31.004 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:31.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:31.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:31.042 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:46:31.043 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:46:31.109 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:31.146 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:31.162 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:46:31.171 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:31.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:31.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:31.252 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.2. 2023-10-09T10:46:31.252 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:46:31.276 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:31.311 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:31.400 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4169482249' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:46:31.400 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:31 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:46:31.400 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2149170422' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T10:46:31.400 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:31 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T10:46:31.400 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:31 smithi155 ceph-mon[184854]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T10:46:31.402 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:46:31.485 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T10:46:31.112+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e152 definitely_dead 0 2023-10-09T10:46:31.485 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4169482249' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:46:31.485 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:31 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:46:31.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2149170422' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T10:46:31.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:31 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T10:46:31.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:31 smithi125 ceph-mon[195507]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T10:46:31.486 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4169482249' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:46:31.486 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:31 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:46:31.486 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2149170422' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T10:46:31.486 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:31 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T10:46:31.487 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:31 smithi125 ceph-mon[200197]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T10:46:31.487 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:46:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:46:31] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.43.0" 2023-10-09T10:46:31.724 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:46:31.742 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:31.743 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:31.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:31.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:31.849 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:31.879 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:31.899 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:31.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:31.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:32.002 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:32.032 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:32.103 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:32.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:32.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:32.209 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:32.250 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:32.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[195507]: pgmap v312: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 8020/270417 objects degraded (2.966%); 6577/270417 objects misplaced (2.432%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T10:46:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[195507]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[195507]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[195507]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[195507]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[195507]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[195507]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[195507]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[195507]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:46:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T10:46:32.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[195507]: osdmap e153: 8 total, 8 up, 7 in 2023-10-09T10:46:32.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[195507]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[195507]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[200197]: pgmap v312: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 8020/270417 objects degraded (2.966%); 6577/270417 objects misplaced (2.432%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T10:46:32.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:32.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[200197]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[200197]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[200197]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[200197]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[200197]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[200197]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[200197]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[200197]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:46:32.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T10:46:32.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[200197]: osdmap e153: 8 total, 8 up, 7 in 2023-10-09T10:46:32.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[200197]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:32 smithi125 ceph-mon[200197]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:32.324 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:32.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:32.425 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:32.453 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:32 smithi155 ceph-mon[184854]: pgmap v312: 119 pgs: 113 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 8020/270417 objects degraded (2.966%); 6577/270417 objects misplaced (2.432%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T10:46:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:32 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:32 smithi155 ceph-mon[184854]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:32 smithi155 ceph-mon[184854]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:32 smithi155 ceph-mon[184854]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:32 smithi155 ceph-mon[184854]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:32 smithi155 ceph-mon[184854]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:32 smithi155 ceph-mon[184854]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:32 smithi155 ceph-mon[184854]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:32 smithi155 ceph-mon[184854]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:32 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:46:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:32 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T10:46:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:32 smithi155 ceph-mon[184854]: osdmap e153: 8 total, 8 up, 7 in 2023-10-09T10:46:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:32 smithi155 ceph-mon[184854]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:32 smithi155 ceph-mon[184854]: failed to encode map e153 with expected crc 2023-10-09T10:46:32.672 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:32.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:32.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:32.675 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:46:32.675 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:46:32.775 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:32.804 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:32.975 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:32.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:32.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:33.042 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:46:33.054 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:46:33.075 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:46:33.078 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:33.094 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:46:33.113 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:33.114 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:46:33.132 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:33.135 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:46:33.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:33.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:33.156 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:46:33.176 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:46:33.194 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:46:33.196 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:46:33.197 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:46:33.197 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:46:33.197 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:46:33.197 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:46:33.197 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:46:33.197 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:46:33.197 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:46:33.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:33.353 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:33.378 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:33.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:33.413 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:46:33.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: osdmap e154: 8 total, 8 up, 7 in 2023-10-09T10:46:33.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: 6.0 continuing backfill to osd.0 from (0'0,152'4875] MIN to 152'4875 2023-10-09T10:46:33.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: 6.14 scrub starts 2023-10-09T10:46:33.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 8020/270417 objects degraded (2.966%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:33.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:33 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/717061399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:33.493 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:33.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: osdmap e154: 8 total, 8 up, 7 in 2023-10-09T10:46:33.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: 6.0 continuing backfill to osd.0 from (0'0,152'4875] MIN to 152'4875 2023-10-09T10:46:33.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.547 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.547 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.547 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.547 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.547 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.547 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: 6.14 scrub starts 2023-10-09T10:46:33.547 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 8020/270417 objects degraded (2.966%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:33.547 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/717061399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:33.547 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.547 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: osdmap e154: 8 total, 8 up, 7 in 2023-10-09T10:46:33.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: 6.0 continuing backfill to osd.0 from (0'0,152'4875] MIN to 152'4875 2023-10-09T10:46:33.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: failed to encode map e154 with expected crc 2023-10-09T10:46:33.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: 6.14 scrub starts 2023-10-09T10:46:33.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 8020/270417 objects degraded (2.966%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:33.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:33 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/717061399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:33.557 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:33.596 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:33.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:33.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:33.682 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:46:33.704 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:33.761 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:34.011 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:34.033 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:34.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:34.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:34.136 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:34.164 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:34.233 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:34.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:34.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:34.337 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:34.365 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:34.385 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:34.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:34.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:34.431 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:34 smithi155 ceph-mon[184854]: pgmap v315: 119 pgs: 2 remapped+peering, 6 peering, 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 6614/269493 objects degraded (2.454%); 6525/269493 objects misplaced (2.421%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T10:46:34.431 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:34 smithi155 ceph-mon[184854]: failed to encode map e155 with expected crc 2023-10-09T10:46:34.432 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:34 smithi155 ceph-mon[184854]: osdmap e155: 8 total, 8 up, 7 in 2023-10-09T10:46:34.432 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:34 smithi155 ceph-mon[184854]: failed to encode map e155 with expected crc 2023-10-09T10:46:34.432 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:34 smithi155 ceph-mon[184854]: failed to encode map e155 with expected crc 2023-10-09T10:46:34.432 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:34 smithi155 ceph-mon[184854]: failed to encode map e155 with expected crc 2023-10-09T10:46:34.432 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:34 smithi155 ceph-mon[184854]: 6.12 starting backfill to osd.3 from (0'0,119'3046] MIN to 152'4999 2023-10-09T10:46:34.432 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:34 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:46:34.488 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:34.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[195507]: pgmap v315: 119 pgs: 2 remapped+peering, 6 peering, 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 6614/269493 objects degraded (2.454%); 6525/269493 objects misplaced (2.421%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T10:46:34.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[195507]: failed to encode map e155 with expected crc 2023-10-09T10:46:34.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[195507]: osdmap e155: 8 total, 8 up, 7 in 2023-10-09T10:46:34.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[195507]: failed to encode map e155 with expected crc 2023-10-09T10:46:34.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[195507]: failed to encode map e155 with expected crc 2023-10-09T10:46:34.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[195507]: failed to encode map e155 with expected crc 2023-10-09T10:46:34.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[195507]: 6.12 starting backfill to osd.3 from (0'0,119'3046] MIN to 152'4999 2023-10-09T10:46:34.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:46:34.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[200197]: pgmap v315: 119 pgs: 2 remapped+peering, 6 peering, 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 6614/269493 objects degraded (2.454%); 6525/269493 objects misplaced (2.421%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T10:46:34.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[200197]: failed to encode map e155 with expected crc 2023-10-09T10:46:34.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[200197]: osdmap e155: 8 total, 8 up, 7 in 2023-10-09T10:46:34.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[200197]: failed to encode map e155 with expected crc 2023-10-09T10:46:34.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[200197]: failed to encode map e155 with expected crc 2023-10-09T10:46:34.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[200197]: failed to encode map e155 with expected crc 2023-10-09T10:46:34.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[200197]: 6.12 starting backfill to osd.3 from (0'0,119'3046] MIN to 152'4999 2023-10-09T10:46:34.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:34 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:46:34.517 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:34.544 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:34.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:34.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:34.649 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:34.654 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:46:34.654 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:46:34.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:34.654 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:123837 6.3 6:c75b61b9:::benchmark_data_smithi125_244951_object17394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-10-09T10:46:34.654 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:34.643130+0000", 2023-10-09T10:46:34.655 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.011143693, 2023-10-09T10:46:34.655 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011178064999999999, 2023-10-09T10:46:34.655 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:46:34.655 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:46:34.655 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:46:34.655 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:46:34.655 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:46:34.655 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 123837 2023-10-09T10:46:34.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:34.655 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:46:34.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:34.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:46:34.656 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:34.643130+0000", 2023-10-09T10:46:34.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:34.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:34.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:34.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:46:34.656 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:34.643130+0000", 2023-10-09T10:46:34.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:46:34.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:34.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:34.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:46:34.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:34.643129+0000", 2023-10-09T10:46:34.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:46:34.657 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:34.657 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:34.657 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:46:34.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:34.643133+0000", 2023-10-09T10:46:34.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0729999999999998e-06 2023-10-09T10:46:34.657 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:34.657 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:34.657 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:46:34.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:34.643142+0000", 2023-10-09T10:46:34.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7110000000000001e-06 2023-10-09T10:46:34.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:34.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:34.658 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:46:34.658 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:34.643146+0000", 2023-10-09T10:46:34.658 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6829999999999999e-06 2023-10-09T10:46:34.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:34.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:34.658 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:46:34.658 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:34.644485+0000", 2023-10-09T10:46:34.658 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001339167 2023-10-09T10:46:34.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:34.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:34.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:46:34.659 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:34.644616+0000", 2023-10-09T10:46:34.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000130903 2023-10-09T10:46:34.659 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:34.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:34.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:46:34.659 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:34.645002+0000", 2023-10-09T10:46:34.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00038582900000000002 2023-10-09T10:46:34.659 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:34.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:34.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:46:34.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:34.650094+0000", 2023-10-09T10:46:34.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0050919909999999997 2023-10-09T10:46:34.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:46:34.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:46:34.660 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:46:34.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:34.650344+0000", 2023-10-09T10:46:34.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025023599999999998 2023-10-09T10:46:34.660 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:34.660 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:46:34.660 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:34.660 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:46:34.660 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T10:46:34.660 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 1 2023-10-09T10:46:34.661 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:46:34.689 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:34.723 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:34.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:34.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:34.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:34.892 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:35.038 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:35.040 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:46:35.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:35.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:35.149 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:35.203 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:35.235 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:35.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:35.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:35.347 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:35.405 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:35.438 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:46:35.446 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:35.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:35.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:35.555 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:35.603 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:46:35.643 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:35.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:35.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:35.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:35.770 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:35.810 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:35.898 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:35.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:35.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:36.002 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:36.030 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:36.059 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:36.151 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:36.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:36.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:36.262 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:36.320 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:46:36.364 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:36.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:36 smithi155 ceph-mon[184854]: pgmap v317: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 6 peering, 103 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 388 op/s; 15837/267909 objects degraded (5.911%); 10196/267909 objects misplaced (3.806%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T10:46:36.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:36 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/974056363' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:36.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:36 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:36.565 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:36 smithi125 ceph-mon[195507]: pgmap v317: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 6 peering, 103 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 388 op/s; 15837/267909 objects degraded (5.911%); 10196/267909 objects misplaced (3.806%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T10:46:36.566 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:36 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/974056363' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:36.566 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:36 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:36.566 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:36 smithi125 ceph-mon[200197]: pgmap v317: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 6 peering, 103 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 388 op/s; 15837/267909 objects degraded (5.911%); 10196/267909 objects misplaced (3.806%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T10:46:36.566 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:36 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/974056363' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:36.566 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:36 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:36.627 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:46:36.627 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:46:36.627 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:46:36.627 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:46:36.627 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:46:36.909 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:36.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:36.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:36.978 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:46:36.978 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T10:46:37.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:37.053 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:46:37.067 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:37.094 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:37.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:37.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:37.198 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:37.276 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:37 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:46:37.285 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:37.336 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:37.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:37.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:37.423 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:46:37.445 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:37.479 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:37 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:46:37.479 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:37 smithi125 ceph-mon[195507]: failed to encode map e156 with expected crc 2023-10-09T10:46:37.479 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:37 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:46:37.479 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:37 smithi125 ceph-mon[195507]: osdmap e156: 8 total, 8 up, 7 in 2023-10-09T10:46:37.479 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:37 smithi125 ceph-mon[195507]: failed to encode map e156 with expected crc 2023-10-09T10:46:37.484 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:37 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:46:37.484 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:37 smithi125 ceph-mon[200197]: failed to encode map e156 with expected crc 2023-10-09T10:46:37.484 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:37 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:46:37.484 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:37 smithi125 ceph-mon[200197]: osdmap e156: 8 total, 8 up, 7 in 2023-10-09T10:46:37.485 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:37 smithi125 ceph-mon[200197]: failed to encode map e156 with expected crc 2023-10-09T10:46:37.511 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:37.546 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:37 smithi155 ceph-mon[184854]: failed to encode map e156 with expected crc 2023-10-09T10:46:37.546 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:37 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:46:37.546 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:37 smithi155 ceph-mon[184854]: osdmap e156: 8 total, 8 up, 7 in 2023-10-09T10:46:37.547 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:37 smithi155 ceph-mon[184854]: failed to encode map e156 with expected crc 2023-10-09T10:46:37.550 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:37.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:37.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:37.656 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:37.659 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:37.721 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:37.745 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:37.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:37.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:37.751 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:46:37.778 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:46:37.809 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:46:37.836 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:46:37.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:37.865 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:46:37.889 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:37.894 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:46:37.916 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:46:37.941 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:46:37.943 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:46:37.943 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:46:37.944 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:46:37.944 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:46:37.944 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:46:37.944 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:46:37.944 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:46:37.944 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:46:37.976 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:37.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:37.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:38.081 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:38.120 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:38.154 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:38.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:38.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:38.260 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:38.295 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:38.372 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:38.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:38.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:38.459 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:46:38.459 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:46:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:38 smithi155 ceph-mon[184854]: pgmap v319: 119 pgs: 2 active+recovering+undersized+remapped, 2 remapped+peering, 103 active+clean, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 33036/265797 objects degraded (12.429%); 14721/265797 objects misplaced (5.538%); 5.4 MiB/s, 1 keys/s, 86 objects/s recovering 2023-10-09T10:46:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:38 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:46:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:38 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 33036/265797 objects degraded (12.429%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:38 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/904388340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:38.478 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:38.512 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:38 smithi125 ceph-mon[195507]: pgmap v319: 119 pgs: 2 active+recovering+undersized+remapped, 2 remapped+peering, 103 active+clean, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 33036/265797 objects degraded (12.429%); 14721/265797 objects misplaced (5.538%); 5.4 MiB/s, 1 keys/s, 86 objects/s recovering 2023-10-09T10:46:38.512 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:38 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:46:38.512 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:38 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 33036/265797 objects degraded (12.429%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:38.512 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:38 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/904388340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:38.513 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:38 smithi125 ceph-mon[200197]: pgmap v319: 119 pgs: 2 active+recovering+undersized+remapped, 2 remapped+peering, 103 active+clean, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 33036/265797 objects degraded (12.429%); 14721/265797 objects misplaced (5.538%); 5.4 MiB/s, 1 keys/s, 86 objects/s recovering 2023-10-09T10:46:38.513 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:38 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:46:38.513 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:38 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 33036/265797 objects degraded (12.429%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:38.513 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:38 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/904388340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:38.514 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:38.641 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:38.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:38.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:38.746 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:38.779 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:38.857 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:38.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:38.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:38.876 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:46:38.961 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:39.004 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:39.046 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:39.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:39.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:39.155 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:39.185 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:39.275 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:39.923 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:39.926 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:46:39.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:39.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:40.031 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:40.091 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:40.113 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:40.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:40.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:40.223 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:40.261 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:40.324 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T10:46:40.486 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:40.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:40.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:40.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:40 smithi125 ceph-mon[200197]: pgmap v320: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 359 op/s; 36956/262293 objects degraded (14.090%); 14588/262293 objects misplaced (5.562%); 5.0 MiB/s, 1 keys/s, 79 objects/s recovering 2023-10-09T10:46:40.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:40 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1261283760' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:46:40.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:40 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:46:40.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:40 smithi125 ceph-mon[195507]: pgmap v320: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 359 op/s; 36956/262293 objects degraded (14.090%); 14588/262293 objects misplaced (5.562%); 5.0 MiB/s, 1 keys/s, 79 objects/s recovering 2023-10-09T10:46:40.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:40 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1261283760' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:46:40.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:40 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:46:40.597 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:40.631 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:40.658 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:46:40.659 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:40.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:40.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:40.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:40 smithi155 ceph-mon[184854]: pgmap v320: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 359 op/s; 36956/262293 objects degraded (14.090%); 14588/262293 objects misplaced (5.562%); 5.0 MiB/s, 1 keys/s, 79 objects/s recovering 2023-10-09T10:46:40.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:40 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1261283760' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:46:40.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:40 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:46:40.764 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:40.806 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:40.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:40.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:40.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:40.960 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:40.995 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:41.018 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:41.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:41.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:41.074 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:41.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T10:46:41.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:41.211 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:41.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:41 smithi155 ceph-mon[184854]: failed to encode map e157 with expected crc 2023-10-09T10:46:41.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:41 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:46:41.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:41 smithi155 ceph-mon[184854]: osdmap e157: 8 total, 8 up, 7 in 2023-10-09T10:46:41.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:41 smithi155 ceph-mon[184854]: failed to encode map e157 with expected crc 2023-10-09T10:46:41.342 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:41.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:41.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:41.412 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:46:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:46:41] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.43.0" 2023-10-09T10:46:41.413 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:41 smithi125 ceph-mon[195507]: failed to encode map e157 with expected crc 2023-10-09T10:46:41.413 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:41 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:46:41.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:41 smithi125 ceph-mon[195507]: osdmap e157: 8 total, 8 up, 7 in 2023-10-09T10:46:41.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:41 smithi125 ceph-mon[195507]: failed to encode map e157 with expected crc 2023-10-09T10:46:41.414 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:41 smithi125 ceph-mon[200197]: failed to encode map e157 with expected crc 2023-10-09T10:46:41.414 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:41 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:46:41.414 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:41 smithi125 ceph-mon[200197]: osdmap e157: 8 total, 8 up, 7 in 2023-10-09T10:46:41.414 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:41 smithi125 ceph-mon[200197]: failed to encode map e157 with expected crc 2023-10-09T10:46:41.454 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:41.485 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:41.509 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:41.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:41.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:41.613 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:41.697 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:41.819 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:41.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:41.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:41.939 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:41.957 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:41.979 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:41.992 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:42.234 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:42.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:42.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:42.331 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:46:42.345 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:42.391 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:42.419 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:42.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:42.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:42.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:42 smithi125 ceph-mon[195507]: pgmap v322: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 36956/259524 objects degraded (14.240%); 14419/259524 objects misplaced (5.556%); 4.3 MiB/s, 1 keys/s, 69 objects/s recovering 2023-10-09T10:46:42.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:42 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1532167785' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:46:42.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:42 smithi125 ceph-mon[200197]: pgmap v322: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 36956/259524 objects degraded (14.240%); 14419/259524 objects misplaced (5.556%); 4.3 MiB/s, 1 keys/s, 69 objects/s recovering 2023-10-09T10:46:42.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:42 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1532167785' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:46:42.523 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:42.562 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:42.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:42 smithi155 ceph-mon[184854]: pgmap v322: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 36956/259524 objects degraded (14.240%); 14419/259524 objects misplaced (5.556%); 4.3 MiB/s, 1 keys/s, 69 objects/s recovering 2023-10-09T10:46:42.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:42 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1532167785' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:46:42.847 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:42.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:42.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:42.952 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:43.000 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:43.025 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:43.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:43.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:43.129 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:43.167 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:43.190 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:43.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:43.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:43.267 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:46:43.267 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:46:43.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:43.344 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:43.367 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:43.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:43.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:43 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:43 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1532167785' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:46:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:43 smithi155 ceph-mon[184854]: osdmap e158: 8 total, 8 up, 7 in 2023-10-09T10:46:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:43 smithi155 ceph-mon[184854]: failed to encode map e158 with expected crc 2023-10-09T10:46:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:43 smithi155 ceph-mon[184854]: failed to encode map e158 with expected crc 2023-10-09T10:46:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:43 smithi155 ceph-mon[184854]: failed to encode map e158 with expected crc 2023-10-09T10:46:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:43 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 36956/259524 objects degraded (14.240%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:43.474 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:43.513 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:43.536 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:43.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:43.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:43.568 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:43 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:43.568 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:43 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1532167785' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:46:43.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:43 smithi125 ceph-mon[195507]: osdmap e158: 8 total, 8 up, 7 in 2023-10-09T10:46:43.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:43 smithi125 ceph-mon[195507]: failed to encode map e158 with expected crc 2023-10-09T10:46:43.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:43 smithi125 ceph-mon[195507]: failed to encode map e158 with expected crc 2023-10-09T10:46:43.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:43 smithi125 ceph-mon[195507]: failed to encode map e158 with expected crc 2023-10-09T10:46:43.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:43 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 36956/259524 objects degraded (14.240%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:43.569 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:43 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:43.569 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:43 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1532167785' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:46:43.569 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:43 smithi125 ceph-mon[200197]: osdmap e158: 8 total, 8 up, 7 in 2023-10-09T10:46:43.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:43 smithi125 ceph-mon[200197]: failed to encode map e158 with expected crc 2023-10-09T10:46:43.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:43 smithi125 ceph-mon[200197]: failed to encode map e158 with expected crc 2023-10-09T10:46:43.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:43 smithi125 ceph-mon[200197]: failed to encode map e158 with expected crc 2023-10-09T10:46:43.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:43 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 36956/259524 objects degraded (14.240%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:43.639 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:43.670 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:43.695 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:43.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:43.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:43.798 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:43.841 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:43.965 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:43.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:43.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:44.069 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:44.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:46:44.133 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:44.320 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:44.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:44.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:44.399 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:46:44.427 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:44.434 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:46:44.476 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:46:44.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:44 smithi125 ceph-mon[195507]: pgmap v324: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 383 op/s; 36956/258534 objects degraded (14.294%); 13953/258534 objects misplaced (5.397%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T10:46:44.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:44 smithi125 ceph-mon[200197]: pgmap v324: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 383 op/s; 36956/258534 objects degraded (14.294%); 13953/258534 objects misplaced (5.397%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T10:46:44.521 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:44.521 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:46:44.555 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:46:44.555 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:44.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:44.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:44.567 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:44.582 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:46:44.633 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:46:44.668 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:44.681 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:46:44.699 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:46:44.699 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:46:44.699 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:46:44.700 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:46:44.700 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:46:44.700 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:46:44.700 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:46:44.700 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:46:44.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:44 smithi155 ceph-mon[184854]: pgmap v324: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 383 op/s; 36956/258534 objects degraded (14.294%); 13953/258534 objects misplaced (5.397%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T10:46:44.769 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:44.855 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:44.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:44.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:44.960 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:44.991 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:45.019 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:45.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:45.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:45.058 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:46:45.125 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:45.156 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:45.178 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:45.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:45.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:45.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:45.323 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:45.368 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T10:46:45.402 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:45 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/925031117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:45.402 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:45 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3873853039' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:46:45.402 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:45 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:46:45.402 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:46:45.402 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:45 smithi125 ceph-mon[200197]: pgmap v325: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 36956/256026 objects degraded (14.434%); 13709/256026 objects misplaced (5.355%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T10:46:45.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:45 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/925031117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:45.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:45 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3873853039' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:46:45.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:45 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:46:45.403 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:46:45.403 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:45 smithi125 ceph-mon[195507]: pgmap v325: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 36956/256026 objects degraded (14.434%); 13709/256026 objects misplaced (5.355%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T10:46:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:45 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/925031117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:45 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3873853039' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:46:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:45 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:46:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:46:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:45 smithi155 ceph-mon[184854]: pgmap v325: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 36956/256026 objects degraded (14.434%); 13709/256026 objects misplaced (5.355%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T10:46:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:46 smithi155 ceph-mon[184854]: failed to encode map e159 with expected crc 2023-10-09T10:46:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:46 smithi155 ceph-mon[184854]: failed to encode map e159 with expected crc 2023-10-09T10:46:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:46 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:46:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:46 smithi155 ceph-mon[184854]: osdmap e159: 8 total, 8 up, 7 in 2023-10-09T10:46:46.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:46 smithi125 ceph-mon[195507]: failed to encode map e159 with expected crc 2023-10-09T10:46:46.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:46 smithi125 ceph-mon[195507]: failed to encode map e159 with expected crc 2023-10-09T10:46:46.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:46 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:46:46.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:46 smithi125 ceph-mon[195507]: osdmap e159: 8 total, 8 up, 7 in 2023-10-09T10:46:46.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:46 smithi125 ceph-mon[200197]: failed to encode map e159 with expected crc 2023-10-09T10:46:46.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:46 smithi125 ceph-mon[200197]: failed to encode map e159 with expected crc 2023-10-09T10:46:46.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:46 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:46:46.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:46 smithi125 ceph-mon[200197]: osdmap e159: 8 total, 8 up, 7 in 2023-10-09T10:46:47.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:47 smithi155 ceph-mon[184854]: pgmap v327: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 36956/253164 objects degraded (14.598%); 13112/253164 objects misplaced (5.179%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T10:46:47.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:47 smithi125 ceph-mon[195507]: pgmap v327: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 36956/253164 objects degraded (14.598%); 13112/253164 objects misplaced (5.179%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T10:46:47.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:47 smithi125 ceph-mon[200197]: pgmap v327: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 36956/253164 objects degraded (14.598%); 13112/253164 objects misplaced (5.179%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T10:46:48.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:48.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:48.512 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:48.567 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:48.678 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:48 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 36956/253164 objects degraded (14.598%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:48.683 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:48 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 36956/253164 objects degraded (14.598%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:48.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:48 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 36956/253164 objects degraded (14.598%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:49.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:49 smithi155 ceph-mon[184854]: pgmap v328: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 383 op/s; 36956/250326 objects degraded (14.763%); 12915/250326 objects misplaced (5.159%); 8.1 MiB/s, 128 objects/s recovering 2023-10-09T10:46:49.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:49 smithi125 ceph-mon[200197]: pgmap v328: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 383 op/s; 36956/250326 objects degraded (14.763%); 12915/250326 objects misplaced (5.159%); 8.1 MiB/s, 128 objects/s recovering 2023-10-09T10:46:49.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:49 smithi125 ceph-mon[195507]: pgmap v328: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 383 op/s; 36956/250326 objects degraded (14.763%); 12915/250326 objects misplaced (5.159%); 8.1 MiB/s, 128 objects/s recovering 2023-10-09T10:46:50.697 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:50.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:46:50.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:50.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:50.864 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:50.895 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:50.922 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:50.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:50.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:50.983 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T10:46:51.027 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:51.061 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:51.083 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:51.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:51.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:51.187 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:51.224 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:51.251 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:51.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:51.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:51.295 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:46:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:46:51] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.43.0" 2023-10-09T10:46:51.355 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:51.383 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:51.392 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:51.406 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:51.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:51.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:51.444 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:51.510 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:51.550 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:51.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:51.577 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:51.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:51.669 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:51.678 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:46:51.714 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:51.738 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:46:51.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:51.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:46:51.844 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:51.889 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:52.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:52 smithi125 ceph-mon[195507]: pgmap v329: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 424 op/s; 36956/247674 objects degraded (14.921%); 12795/247674 objects misplaced (5.166%); 4.4 MiB/s, 71 objects/s recovering 2023-10-09T10:46:52.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:52 smithi125 ceph-mon[200197]: pgmap v329: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 424 op/s; 36956/247674 objects degraded (14.921%); 12795/247674 objects misplaced (5.166%); 4.4 MiB/s, 71 objects/s recovering 2023-10-09T10:46:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:52 smithi155 ceph-mon[184854]: pgmap v329: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 424 op/s; 36956/247674 objects degraded (14.921%); 12795/247674 objects misplaced (5.166%); 4.4 MiB/s, 71 objects/s recovering 2023-10-09T10:46:52.575 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:52.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:52.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:52.684 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:52.716 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:52.736 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:52.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:52.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:52.842 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:52.878 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:52.906 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:52.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:52.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:53.014 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:53.075 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:53.110 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:53.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:53.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:53.142 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:46:53.143 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:46:53.143 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:46:53.143 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:46:53.143 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:46:53.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:53.258 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:53 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 36956/247674 objects degraded (14.921%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:53.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:53 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 36956/247674 objects degraded (14.921%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:53.262 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:53 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 36956/247674 objects degraded (14.921%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:53.542 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:53.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:53.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:53.647 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:53.680 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:53.704 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:53.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:53.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:53.810 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:53.842 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:53.868 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:53.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:53.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:53.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:46:53.976 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:54.005 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:54.027 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:54.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:54.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:54.131 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:54.160 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:54.180 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:54.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:54.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:54.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:54 smithi125 ceph-mon[195507]: pgmap v330: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 36956/246303 objects degraded (15.004%); 12118/246303 objects misplaced (4.920%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T10:46:54.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:54 smithi125 ceph-mon[200197]: pgmap v330: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 36956/246303 objects degraded (15.004%); 12118/246303 objects misplaced (4.920%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T10:46:54.284 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:54.345 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:54 smithi155 ceph-mon[184854]: pgmap v330: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 36956/246303 objects degraded (15.004%); 12118/246303 objects misplaced (4.920%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T10:46:54.694 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:54.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:54.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:54.801 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:54.804 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:54.840 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:54.879 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:54.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:54.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:54.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:55.030 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:55.170 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:55 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2098151009' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:55.170 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:55 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:55.171 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:55 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2098151009' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:55.171 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:55 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:55.181 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:46:55.365 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:55.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:55.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:55.430 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:46:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:55 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2098151009' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:55 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:46:55.471 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:55.474 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:46:55.507 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:46:55.532 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:55.560 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:46:55.601 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:46:55.640 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:46:55.680 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:46:55.712 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:46:55.719 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:46:55.719 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:46:55.720 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:46:55.720 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:46:55.720 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:46:55.720 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:46:55.720 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:46:55.720 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:46:55.821 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:55.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:55.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:55.927 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:46:55.978 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:56.059 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:46:56.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:56.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:46:56.165 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:56.176 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T10:46:56.200 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:56.220 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:56.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:56.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:56.325 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:56.358 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:56.380 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:56.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:56.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:56.399 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[195507]: pgmap v331: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 429 op/s; 36956/243129 objects degraded (15.200%); 11847/243129 objects misplaced (4.873%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T10:46:56.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:56.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:46:56.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[195507]: osdmap e160: 8 total, 8 up, 7 in 2023-10-09T10:46:56.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[195507]: failed to encode map e160 with expected crc 2023-10-09T10:46:56.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[195507]: failed to encode map e160 with expected crc 2023-10-09T10:46:56.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[195507]: failed to encode map e160 with expected crc 2023-10-09T10:46:56.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[195507]: failed to encode map e160 with expected crc 2023-10-09T10:46:56.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2173054463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:56.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/486625247' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:46:56.400 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[200197]: pgmap v331: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 429 op/s; 36956/243129 objects degraded (15.200%); 11847/243129 objects misplaced (4.873%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T10:46:56.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:56.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:46:56.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[200197]: osdmap e160: 8 total, 8 up, 7 in 2023-10-09T10:46:56.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[200197]: failed to encode map e160 with expected crc 2023-10-09T10:46:56.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[200197]: failed to encode map e160 with expected crc 2023-10-09T10:46:56.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[200197]: failed to encode map e160 with expected crc 2023-10-09T10:46:56.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[200197]: failed to encode map e160 with expected crc 2023-10-09T10:46:56.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2173054463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:56.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/486625247' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:46:56.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:56 smithi155 ceph-mon[184854]: pgmap v331: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 429 op/s; 36956/243129 objects degraded (15.200%); 11847/243129 objects misplaced (4.873%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T10:46:56.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:56 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:46:56.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:56 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:46:56.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:56 smithi155 ceph-mon[184854]: osdmap e160: 8 total, 8 up, 7 in 2023-10-09T10:46:56.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:56 smithi155 ceph-mon[184854]: failed to encode map e160 with expected crc 2023-10-09T10:46:56.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:56 smithi155 ceph-mon[184854]: failed to encode map e160 with expected crc 2023-10-09T10:46:56.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:56 smithi155 ceph-mon[184854]: failed to encode map e160 with expected crc 2023-10-09T10:46:56.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:56 smithi155 ceph-mon[184854]: failed to encode map e160 with expected crc 2023-10-09T10:46:56.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2173054463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:46:56.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/486625247' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:46:56.486 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:56.533 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:56.679 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T10:46:56.767 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:56.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:56.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:56.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:46:56.923 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:57.001 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:46:57.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:57.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:46:57.101 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:46:57.102 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:46:57.105 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:57.130 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:57.137 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:57.157 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:57.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:57.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:57.179 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:57 smithi125 ceph-mon[195507]: failed to encode map e161 with expected crc 2023-10-09T10:46:57.179 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:57 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/486625247' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:46:57.179 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:57 smithi125 ceph-mon[195507]: osdmap e161: 8 total, 8 up, 7 in 2023-10-09T10:46:57.179 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:57 smithi125 ceph-mon[195507]: failed to encode map e161 with expected crc 2023-10-09T10:46:57.179 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:57 smithi125 ceph-mon[200197]: failed to encode map e161 with expected crc 2023-10-09T10:46:57.180 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:57 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/486625247' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:46:57.180 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:57 smithi125 ceph-mon[200197]: osdmap e161: 8 total, 8 up, 7 in 2023-10-09T10:46:57.180 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:57 smithi125 ceph-mon[200197]: failed to encode map e161 with expected crc 2023-10-09T10:46:57.260 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:57.291 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:57.312 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:57.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:57.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:57.416 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:46:57.449 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:57 smithi155 ceph-mon[184854]: failed to encode map e161 with expected crc 2023-10-09T10:46:57.449 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:57 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/486625247' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:46:57.449 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:57 smithi155 ceph-mon[184854]: osdmap e161: 8 total, 8 up, 7 in 2023-10-09T10:46:57.449 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:57 smithi155 ceph-mon[184854]: failed to encode map e161 with expected crc 2023-10-09T10:46:57.451 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:57.472 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:46:57.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:57.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:46:57.576 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:57.605 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:57.614 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:46:57.629 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:57.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:57.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:57.734 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:57.782 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:57.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:46:58.160 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:58.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:58.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:58.214 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:58.270 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:46:58.304 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:58.327 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:46:58.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:58.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:46:58.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[195507]: pgmap v334: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 36956/239526 objects degraded (15.429%); 11028/239526 objects misplaced (4.604%); 10 MiB/s, 167 objects/s recovering 2023-10-09T10:46:58.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[195507]: failed to encode map e162 with expected crc 2023-10-09T10:46:58.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[195507]: failed to encode map e162 with expected crc 2023-10-09T10:46:58.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[195507]: osdmap e162: 8 total, 8 up, 7 in 2023-10-09T10:46:58.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[195507]: failed to encode map e162 with expected crc 2023-10-09T10:46:58.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[195507]: failed to encode map e162 with expected crc 2023-10-09T10:46:58.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[195507]: failed to encode map e162 with expected crc 2023-10-09T10:46:58.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 36956/239526 objects degraded (15.429%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:58.397 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[200197]: pgmap v334: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 36956/239526 objects degraded (15.429%); 11028/239526 objects misplaced (4.604%); 10 MiB/s, 167 objects/s recovering 2023-10-09T10:46:58.397 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[200197]: failed to encode map e162 with expected crc 2023-10-09T10:46:58.397 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[200197]: failed to encode map e162 with expected crc 2023-10-09T10:46:58.398 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[200197]: osdmap e162: 8 total, 8 up, 7 in 2023-10-09T10:46:58.398 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[200197]: failed to encode map e162 with expected crc 2023-10-09T10:46:58.398 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[200197]: failed to encode map e162 with expected crc 2023-10-09T10:46:58.398 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[200197]: failed to encode map e162 with expected crc 2023-10-09T10:46:58.398 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:46:58 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 36956/239526 objects degraded (15.429%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:58.430 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:58.468 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:58 smithi155 ceph-mon[184854]: pgmap v334: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 36956/239526 objects degraded (15.429%); 11028/239526 objects misplaced (4.604%); 10 MiB/s, 167 objects/s recovering 2023-10-09T10:46:58.468 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:58 smithi155 ceph-mon[184854]: failed to encode map e162 with expected crc 2023-10-09T10:46:58.468 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:58 smithi155 ceph-mon[184854]: failed to encode map e162 with expected crc 2023-10-09T10:46:58.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:58 smithi155 ceph-mon[184854]: osdmap e162: 8 total, 8 up, 7 in 2023-10-09T10:46:58.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:58 smithi155 ceph-mon[184854]: failed to encode map e162 with expected crc 2023-10-09T10:46:58.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:58 smithi155 ceph-mon[184854]: failed to encode map e162 with expected crc 2023-10-09T10:46:58.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:58 smithi155 ceph-mon[184854]: failed to encode map e162 with expected crc 2023-10-09T10:46:58.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:46:58 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 36956/239526 objects degraded (15.429%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:46:58.471 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:58.498 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:58.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:58.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:58.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:58.613 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:46:58.640 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:58.654 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:46:58.663 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:58.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:58.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:58.767 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:46:58.803 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:58.826 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:46:58.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:58.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:46:58.933 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:58.973 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:59.151 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:59.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:59.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:59.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:59.324 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:59.408 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:46:59.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:59.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:46:59.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:46:59.557 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:59.640 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:46:59.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:46:59.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:46:59.746 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:46:59.798 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:46:59.875 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:46:59.876 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:47:00.037 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:00.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:00.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:00.149 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:00.186 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:00.221 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T10:47:00.330 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:00.337 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:00.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:00.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:00.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:47:00.441 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:00.462 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:00 smithi125 ceph-mon[195507]: pgmap v336: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 399 op/s; 36956/239109 objects degraded (15.456%); 11028/239109 objects misplaced (4.612%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T10:47:00.462 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:00 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/844521230' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:47:00.462 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:47:00.462 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:00 smithi125 ceph-mon[200197]: pgmap v336: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 399 op/s; 36956/239109 objects degraded (15.456%); 11028/239109 objects misplaced (4.612%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T10:47:00.463 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:00 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/844521230' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:47:00.463 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:47:00.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:00 smithi155 ceph-mon[184854]: pgmap v336: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 399 op/s; 36956/239109 objects degraded (15.456%); 11028/239109 objects misplaced (4.612%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T10:47:00.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:00 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/844521230' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:47:00.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:47:00.472 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:00.498 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:00.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:00.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:00.604 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:00.665 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:00.946 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:00.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:00.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:01.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:01.087 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:01.204 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T10:47:01.247 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:47:01.416 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:01.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:01.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:01.517 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:01.523 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:01.556 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:01 smithi125 ceph-mon[195507]: failed to encode map e163 with expected crc 2023-10-09T10:47:01.557 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/844521230' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:47:01.557 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:01 smithi125 ceph-mon[195507]: osdmap e163: 8 total, 8 up, 7 in 2023-10-09T10:47:01.557 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/757849079' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:47:01.557 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:01 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:47:01.557 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:01 smithi125 ceph-mon[200197]: failed to encode map e163 with expected crc 2023-10-09T10:47:01.557 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/844521230' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:47:01.557 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:01 smithi125 ceph-mon[200197]: osdmap e163: 8 total, 8 up, 7 in 2023-10-09T10:47:01.557 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/757849079' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:47:01.557 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:01 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:47:01.558 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:47:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:47:01] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.43.0" 2023-10-09T10:47:01.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:01 smithi155 ceph-mon[184854]: failed to encode map e163 with expected crc 2023-10-09T10:47:01.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/844521230' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:47:01.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:01 smithi155 ceph-mon[184854]: osdmap e163: 8 total, 8 up, 7 in 2023-10-09T10:47:01.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/757849079' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:47:01.560 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:01 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:47:01.563 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:01.592 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:01.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:01.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:01.697 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:01.731 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:01.750 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:01.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:01.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:01.854 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:01.884 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:01.908 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:01.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:01.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:02.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:02.041 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:47:02.042 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:02.067 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:02.069 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:47:02.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:02.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:02.095 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:47:02.116 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:47:02.151 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:47:02.171 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:02.172 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:47:02.194 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:47:02.200 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:02.217 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:47:02.218 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:47:02.218 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:47:02.218 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:47:02.218 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:47:02.218 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:47:02.218 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:47:02.218 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:47:02.218 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:47:02.220 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:02.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:02.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:02.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:02.368 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:02.412 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:02.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:02.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:02 smithi155 ceph-mon[184854]: pgmap v338: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 34009/233910 objects degraded (14.539%); 13068/233910 objects misplaced (5.587%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T10:47:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:02 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:02 smithi155 ceph-mon[184854]: failed to encode map e164 with expected crc 2023-10-09T10:47:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:02 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:47:02.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:02 smithi155 ceph-mon[184854]: osdmap e164: 8 total, 8 up, 7 in 2023-10-09T10:47:02.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:02 smithi155 ceph-mon[184854]: failed to encode map e164 with expected crc 2023-10-09T10:47:02.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:02 smithi155 ceph-mon[184854]: failed to encode map e164 with expected crc 2023-10-09T10:47:02.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:02 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/293761579' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:02.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[195507]: pgmap v338: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 34009/233910 objects degraded (14.539%); 13068/233910 objects misplaced (5.587%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T10:47:02.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:02.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[195507]: failed to encode map e164 with expected crc 2023-10-09T10:47:02.482 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:47:02.482 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[195507]: osdmap e164: 8 total, 8 up, 7 in 2023-10-09T10:47:02.482 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[195507]: failed to encode map e164 with expected crc 2023-10-09T10:47:02.482 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[195507]: failed to encode map e164 with expected crc 2023-10-09T10:47:02.482 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/293761579' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:02.482 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[200197]: pgmap v338: 119 pgs: 2 active+recovering+undersized+remapped, 103 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 34009/233910 objects degraded (14.539%); 13068/233910 objects misplaced (5.587%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T10:47:02.482 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:02.482 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[200197]: failed to encode map e164 with expected crc 2023-10-09T10:47:02.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:47:02.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[200197]: osdmap e164: 8 total, 8 up, 7 in 2023-10-09T10:47:02.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[200197]: failed to encode map e164 with expected crc 2023-10-09T10:47:02.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[200197]: failed to encode map e164 with expected crc 2023-10-09T10:47:02.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:02 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/293761579' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:02.516 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:02.568 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:02.745 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:02.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:02.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:02.852 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:02.896 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:02.923 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:02.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:02.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:03.028 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:03.064 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:03.086 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:03.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:03.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:03.190 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:03.227 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:03.252 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:03.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:03.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:03.356 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:03.397 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:03.421 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:03.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:03.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:03.529 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:03.565 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:03 smithi125 ceph-mon[195507]: osdmap e165: 8 total, 8 up, 7 in 2023-10-09T10:47:03.565 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:03 smithi125 ceph-mon[195507]: failed to encode map e165 with expected crc 2023-10-09T10:47:03.565 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:03 smithi125 ceph-mon[195507]: failed to encode map e165 with expected crc 2023-10-09T10:47:03.566 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:03 smithi125 ceph-mon[195507]: failed to encode map e165 with expected crc 2023-10-09T10:47:03.566 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:03 smithi125 ceph-mon[195507]: failed to encode map e165 with expected crc 2023-10-09T10:47:03.566 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:03 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 34009/233910 objects degraded (14.539%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:47:03.566 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:03 smithi125 ceph-mon[200197]: osdmap e165: 8 total, 8 up, 7 in 2023-10-09T10:47:03.566 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:03 smithi125 ceph-mon[200197]: failed to encode map e165 with expected crc 2023-10-09T10:47:03.566 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:03 smithi125 ceph-mon[200197]: failed to encode map e165 with expected crc 2023-10-09T10:47:03.567 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:03 smithi125 ceph-mon[200197]: failed to encode map e165 with expected crc 2023-10-09T10:47:03.567 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:03 smithi125 ceph-mon[200197]: failed to encode map e165 with expected crc 2023-10-09T10:47:03.567 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:03 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 34009/233910 objects degraded (14.539%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:47:03.568 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:03 smithi155 ceph-mon[184854]: osdmap e165: 8 total, 8 up, 7 in 2023-10-09T10:47:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:03 smithi155 ceph-mon[184854]: failed to encode map e165 with expected crc 2023-10-09T10:47:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:03 smithi155 ceph-mon[184854]: failed to encode map e165 with expected crc 2023-10-09T10:47:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:03 smithi155 ceph-mon[184854]: failed to encode map e165 with expected crc 2023-10-09T10:47:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:03 smithi155 ceph-mon[184854]: failed to encode map e165 with expected crc 2023-10-09T10:47:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:03 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 34009/233910 objects degraded (14.539%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:47:03.843 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:47:03.871 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:04.104 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:04.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:04.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:04.211 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:04.258 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:04.327 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:47:04.327 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:47:04.327 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:47:04.327 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:47:04.327 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:47:04.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:04 smithi125 ceph-mon[200197]: pgmap v341: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 103 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 381 op/s; 34009/232749 objects degraded (14.612%); 12601/232749 objects misplaced (5.414%); 5.2 MiB/s, 168 objects/s recovering 2023-10-09T10:47:04.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:04 smithi125 ceph-mon[195507]: pgmap v341: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 103 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 381 op/s; 34009/232749 objects degraded (14.612%); 12601/232749 objects misplaced (5.414%); 5.2 MiB/s, 168 objects/s recovering 2023-10-09T10:47:04.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:04 smithi155 ceph-mon[184854]: pgmap v341: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 103 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 381 op/s; 34009/232749 objects degraded (14.612%); 12601/232749 objects misplaced (5.414%); 5.2 MiB/s, 168 objects/s recovering 2023-10-09T10:47:04.796 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:04.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:04.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:04.901 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:04.958 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:05.035 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:05.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:05.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:05.057 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:47:05.073 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:05.142 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:05.207 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:05.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:05.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:05.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:05.336 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:05.387 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:05.416 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:05.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:05.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:05.519 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:05.594 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:05.729 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:05.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:05.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:05.836 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:05.892 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:47:05.901 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:06.008 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:06.047 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:06.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:06.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:06.154 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:06.189 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:06.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:06.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:06.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:06.310 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T10:47:06.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:06.360 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:06.385 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:06.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:06.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:06 smithi155 ceph-mon[184854]: pgmap v342: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 103 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 34009/229500 objects degraded (14.819%); 12297/229500 objects misplaced (5.358%); 5.8 MiB/s, 178 objects/s recovering 2023-10-09T10:47:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:06 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2646822814' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:47:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:06 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:47:06.491 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:06.567 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:06 smithi125 ceph-mon[200197]: pgmap v342: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 103 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 34009/229500 objects degraded (14.819%); 12297/229500 objects misplaced (5.358%); 5.8 MiB/s, 178 objects/s recovering 2023-10-09T10:47:06.567 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:06 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2646822814' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:47:06.567 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:06 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:47:06.567 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:06 smithi125 ceph-mon[195507]: pgmap v342: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 103 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 34009/229500 objects degraded (14.819%); 12297/229500 objects misplaced (5.358%); 5.8 MiB/s, 178 objects/s recovering 2023-10-09T10:47:06.567 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:06 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2646822814' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:47:06.567 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:06 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:47:06.574 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:06.658 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:06.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:06.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:06.763 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:06.824 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:06.948 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:06.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:06.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:06.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T10:47:07.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:07.083 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:07.102 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:07.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:07.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:07.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:07.228 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:07.257 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:07.304 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:47:07.431 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:07 smithi125 ceph-mon[195507]: failed to encode map e166 with expected crc 2023-10-09T10:47:07.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:07 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:47:07.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:07 smithi125 ceph-mon[195507]: osdmap e166: 8 total, 8 up, 7 in 2023-10-09T10:47:07.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:07 smithi125 ceph-mon[195507]: failed to encode map e166 with expected crc 2023-10-09T10:47:07.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:07 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1738359528' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:07.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:07 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:07.432 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:07 smithi125 ceph-mon[200197]: failed to encode map e166 with expected crc 2023-10-09T10:47:07.432 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:07 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:47:07.432 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:07 smithi125 ceph-mon[200197]: osdmap e166: 8 total, 8 up, 7 in 2023-10-09T10:47:07.432 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:07 smithi125 ceph-mon[200197]: failed to encode map e166 with expected crc 2023-10-09T10:47:07.432 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:07 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1738359528' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:07.433 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:07 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:07.450 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:07.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:07.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:07 smithi155 ceph-mon[184854]: failed to encode map e166 with expected crc 2023-10-09T10:47:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:07 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:47:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:07 smithi155 ceph-mon[184854]: osdmap e166: 8 total, 8 up, 7 in 2023-10-09T10:47:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:07 smithi155 ceph-mon[184854]: failed to encode map e166 with expected crc 2023-10-09T10:47:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:07 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1738359528' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:07 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:07.556 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:07.592 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:07.636 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:07.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:07.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:07.740 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:07.791 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:08.024 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:08.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:08.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:08.132 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:08.161 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:08.182 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:08.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:08.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:08.193 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:47:08.193 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:47:08.193 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:47:08.193 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:47:08.193 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.193 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42934 6.11 6:8a3972ec:::benchmark_data_smithi125_244951_object42933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.194 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.549876+0000", 2023-10-09T10:47:08.194 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.64019452799999, 2023-10-09T10:47:08.194 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.229929598, 2023-10-09T10:47:08.194 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.194 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.194 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.194 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.194 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.194 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42934 2023-10-09T10:47:08.194 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.194 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.194 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.195 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.195 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549876+0000", 2023-10-09T10:47:08.195 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.195 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.195 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.195 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549876+0000", 2023-10-09T10:47:08.195 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.195 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.195 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549874+0000", 2023-10-09T10:47:08.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:47:08.196 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.196 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.196 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549907+0000", 2023-10-09T10:47:08.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2579000000000001e-05 2023-10-09T10:47:08.196 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.196 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.196 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549922+0000", 2023-10-09T10:47:08.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5420000000000001e-05 2023-10-09T10:47:08.197 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.197 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.197 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.197 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549934+0000", 2023-10-09T10:47:08.197 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1724e-05 2023-10-09T10:47:08.197 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.197 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.197 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.197 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550289+0000", 2023-10-09T10:47:08.197 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035548900000000001 2023-10-09T10:47:08.197 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.198 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.198 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op must wait for map", 2023-10-09T10:47:08.198 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550297+0000", 2023-10-09T10:47:08.198 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8869999999999992e-06 2023-10-09T10:47:08.198 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.198 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.198 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.198 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551339+0000", 2023-10-09T10:47:08.198 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010423069999999999 2023-10-09T10:47:08.198 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.198 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551349+0000", 2023-10-09T10:47:08.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8060000000000006e-06 2023-10-09T10:47:08.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.199 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.776033+0000", 2023-10-09T10:47:08.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.224684135 2023-10-09T10:47:08.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.199 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.776159+0000", 2023-10-09T10:47:08.200 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012583900000000001 2023-10-09T10:47:08.200 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.200 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.200 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.200 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.776290+0000", 2023-10-09T10:47:08.200 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000130705 2023-10-09T10:47:08.200 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.200 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.200 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.200 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778810+0000", 2023-10-09T10:47:08.200 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025205549999999998 2023-10-09T10:47:08.200 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.201 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.201 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.201 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778863+0000", 2023-10-09T10:47:08.201 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2821000000000001e-05 2023-10-09T10:47:08.201 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.201 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.201 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.201 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778971+0000", 2023-10-09T10:47:08.201 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108244 2023-10-09T10:47:08.201 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.201 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.202 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.202 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779012+0000", 2023-10-09T10:47:08.202 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0612999999999999e-05 2023-10-09T10:47:08.202 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.202 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.203 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.203 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779058+0000", 2023-10-09T10:47:08.203 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5948999999999997e-05 2023-10-09T10:47:08.203 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.203 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.204 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.204 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779805+0000", 2023-10-09T10:47:08.204 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00074730900000000004 2023-10-09T10:47:08.204 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.204 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.204 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.204 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.204 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.204 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42936 6.11 6:8ce9d1a1:::benchmark_data_smithi125_244951_object42935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.204 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.549950+0000", 2023-10-09T10:47:08.205 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.64012051500001, 2023-10-09T10:47:08.205 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.235619359, 2023-10-09T10:47:08.205 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.205 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.205 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.205 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.205 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.205 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42936 2023-10-09T10:47:08.205 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.205 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.205 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.205 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.206 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549950+0000", 2023-10-09T10:47:08.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.206 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.206 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.206 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.206 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549950+0000", 2023-10-09T10:47:08.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.206 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.206 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.206 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.206 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549949+0000", 2023-10-09T10:47:08.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:08.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.207 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549976+0000", 2023-10-09T10:47:08.207 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7223999999999999e-05 2023-10-09T10:47:08.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.207 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549987+0000", 2023-10-09T10:47:08.207 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0375e-05 2023-10-09T10:47:08.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549996+0000", 2023-10-09T10:47:08.208 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4010000000000003e-06 2023-10-09T10:47:08.208 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.208 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.208 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550347+0000", 2023-10-09T10:47:08.208 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035085200000000002 2023-10-09T10:47:08.208 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.208 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.208 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:47:08.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550352+0000", 2023-10-09T10:47:08.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2079999999999999e-06 2023-10-09T10:47:08.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.209 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.209 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.209 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551460+0000", 2023-10-09T10:47:08.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001107762 2023-10-09T10:47:08.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.209 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.209 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.209 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551467+0000", 2023-10-09T10:47:08.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0350000000000004e-06 2023-10-09T10:47:08.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.210 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.777841+0000", 2023-10-09T10:47:08.210 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2263741750000001 2023-10-09T10:47:08.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.210 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778003+0000", 2023-10-09T10:47:08.210 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016172699999999999 2023-10-09T10:47:08.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.211 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.211 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778128+0000", 2023-10-09T10:47:08.211 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012541000000000001 2023-10-09T10:47:08.211 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.211 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.211 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780770+0000", 2023-10-09T10:47:08.211 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026419500000000001 2023-10-09T10:47:08.211 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.212 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.212 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.212 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780795+0000", 2023-10-09T10:47:08.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4363e-05 2023-10-09T10:47:08.212 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.212 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.212 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.212 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784699+0000", 2023-10-09T10:47:08.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039046739999999999 2023-10-09T10:47:08.212 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.212 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.212 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.212 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784714+0000", 2023-10-09T10:47:08.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4956999999999999e-05 2023-10-09T10:47:08.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784774+0000", 2023-10-09T10:47:08.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9994999999999997e-05 2023-10-09T10:47:08.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785569+0000", 2023-10-09T10:47:08.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00079485999999999999 2023-10-09T10:47:08.213 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.213 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.214 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.214 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.214 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42950 6.11 6:8b5bb41c:::benchmark_data_smithi125_244951_object42949:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.214 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:47:08.214 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.64006812, 2023-10-09T10:47:08.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2390020909999999, 2023-10-09T10:47:08.214 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.214 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.214 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.214 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.214 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.215 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42950 2023-10-09T10:47:08.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.215 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.215 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.215 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:47:08.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.215 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.215 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:47:08.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:47:08.216 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:08.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550021+0000", 2023-10-09T10:47:08.216 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8941999999999999e-05 2023-10-09T10:47:08.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550034+0000", 2023-10-09T10:47:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3267e-05 2023-10-09T10:47:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550040+0000", 2023-10-09T10:47:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5799999999999997e-06 2023-10-09T10:47:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.218 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550377+0000", 2023-10-09T10:47:08.218 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00033659800000000002 2023-10-09T10:47:08.218 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:47:08.218 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550382+0000", 2023-10-09T10:47:08.218 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6589999999999998e-06 2023-10-09T10:47:08.218 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.218 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551483+0000", 2023-10-09T10:47:08.218 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011012210000000001 2023-10-09T10:47:08.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.219 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551489+0000", 2023-10-09T10:47:08.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4910000000000002e-06 2023-10-09T10:47:08.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.219 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779851+0000", 2023-10-09T10:47:08.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.22836144 2023-10-09T10:47:08.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.220 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779959+0000", 2023-10-09T10:47:08.220 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108546 2023-10-09T10:47:08.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.220 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780061+0000", 2023-10-09T10:47:08.220 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010196600000000001 2023-10-09T10:47:08.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782707+0000", 2023-10-09T10:47:08.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002646074 2023-10-09T10:47:08.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.221 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785603+0000", 2023-10-09T10:47:08.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028958410000000001 2023-10-09T10:47:08.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.221 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785626+0000", 2023-10-09T10:47:08.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2371e-05 2023-10-09T10:47:08.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.222 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.222 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.787983+0000", 2023-10-09T10:47:08.222 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023577619999999998 2023-10-09T10:47:08.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.222 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.222 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.788053+0000", 2023-10-09T10:47:08.222 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9797999999999998e-05 2023-10-09T10:47:08.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.222 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.223 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789004+0000", 2023-10-09T10:47:08.223 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00095101500000000004 2023-10-09T10:47:08.223 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.223 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.223 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.223 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.223 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.223 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42962 6.11 6:8af38cb4:::benchmark_data_smithi125_244951_object42961:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.223 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550048+0000", 2023-10-09T10:47:08.223 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.64002259099999, 2023-10-09T10:47:08.223 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2398821879999999, 2023-10-09T10:47:08.224 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.224 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.224 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.224 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.224 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.224 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42962 2023-10-09T10:47:08.224 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.224 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.224 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.224 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.224 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550048+0000", 2023-10-09T10:47:08.224 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550048+0000", 2023-10-09T10:47:08.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550047+0000", 2023-10-09T10:47:08.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:08.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550067+0000", 2023-10-09T10:47:08.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0228999999999999e-05 2023-10-09T10:47:08.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550075+0000", 2023-10-09T10:47:08.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0979999999999996e-06 2023-10-09T10:47:08.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550080+0000", 2023-10-09T10:47:08.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7780000000000004e-06 2023-10-09T10:47:08.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550404+0000", 2023-10-09T10:47:08.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000323648 2023-10-09T10:47:08.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:47:08.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550409+0000", 2023-10-09T10:47:08.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6630000000000001e-06 2023-10-09T10:47:08.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.228 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.228 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551523+0000", 2023-10-09T10:47:08.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001113953 2023-10-09T10:47:08.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.228 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.228 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551531+0000", 2023-10-09T10:47:08.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5860000000000005e-06 2023-10-09T10:47:08.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.228 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.228 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780812+0000", 2023-10-09T10:47:08.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2292809120000001 2023-10-09T10:47:08.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780947+0000", 2023-10-09T10:47:08.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000135265 2023-10-09T10:47:08.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.781098+0000", 2023-10-09T10:47:08.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015082499999999999 2023-10-09T10:47:08.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782749+0000", 2023-10-09T10:47:08.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001651057 2023-10-09T10:47:08.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785607+0000", 2023-10-09T10:47:08.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028579569999999999 2023-10-09T10:47:08.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785661+0000", 2023-10-09T10:47:08.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4151999999999999e-05 2023-10-09T10:47:08.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789184+0000", 2023-10-09T10:47:08.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035224309999999999 2023-10-09T10:47:08.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789241+0000", 2023-10-09T10:47:08.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6962000000000001e-05 2023-10-09T10:47:08.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789930+0000", 2023-10-09T10:47:08.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000689148 2023-10-09T10:47:08.232 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.232 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.232 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.232 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43001 6.11 6:8f0a81da:::benchmark_data_smithi125_244951_object43000:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.232 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:47:08.232 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.63998488199999, 2023-10-09T10:47:08.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.240501734, 2023-10-09T10:47:08.232 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.232 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.233 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.233 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.233 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.233 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43001 2023-10-09T10:47:08.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.233 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:47:08.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:47:08.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:47:08.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:08.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550104+0000", 2023-10-09T10:47:08.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.912e-05 2023-10-09T10:47:08.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550110+0000", 2023-10-09T10:47:08.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3779999999999998e-06 2023-10-09T10:47:08.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550116+0000", 2023-10-09T10:47:08.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.609e-06 2023-10-09T10:47:08.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550429+0000", 2023-10-09T10:47:08.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00031264800000000001 2023-10-09T10:47:08.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:47:08.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550433+0000", 2023-10-09T10:47:08.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8420000000000004e-06 2023-10-09T10:47:08.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551574+0000", 2023-10-09T10:47:08.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011402210000000001 2023-10-09T10:47:08.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551584+0000", 2023-10-09T10:47:08.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9690000000000006e-06 2023-10-09T10:47:08.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.781792+0000", 2023-10-09T10:47:08.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.230208454 2023-10-09T10:47:08.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.238 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.781903+0000", 2023-10-09T10:47:08.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110733 2023-10-09T10:47:08.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.238 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782036+0000", 2023-10-09T10:47:08.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000133545 2023-10-09T10:47:08.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785598+0000", 2023-10-09T10:47:08.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035616100000000002 2023-10-09T10:47:08.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785679+0000", 2023-10-09T10:47:08.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0624999999999995e-05 2023-10-09T10:47:08.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785703+0000", 2023-10-09T10:47:08.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4150000000000001e-05 2023-10-09T10:47:08.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790058+0000", 2023-10-09T10:47:08.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043551919999999999 2023-10-09T10:47:08.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790110+0000", 2023-10-09T10:47:08.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2278000000000003e-05 2023-10-09T10:47:08.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790587+0000", 2023-10-09T10:47:08.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047690599999999998 2023-10-09T10:47:08.241 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.241 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.241 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.242 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43002 6.11 6:8bd791a2:::benchmark_data_smithi125_244951_object43001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.242 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550121+0000", 2023-10-09T10:47:08.242 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.63994958800001, 2023-10-09T10:47:08.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.241166963, 2023-10-09T10:47:08.242 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.242 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.242 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.242 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.242 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.243 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43002 2023-10-09T10:47:08.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.243 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550121+0000", 2023-10-09T10:47:08.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550121+0000", 2023-10-09T10:47:08.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.244 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550120+0000", 2023-10-09T10:47:08.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:08.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.244 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550139+0000", 2023-10-09T10:47:08.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8644999999999999e-05 2023-10-09T10:47:08.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.245 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.245 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.245 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550145+0000", 2023-10-09T10:47:08.245 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5180000000000001e-06 2023-10-09T10:47:08.245 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.245 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.245 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.245 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550151+0000", 2023-10-09T10:47:08.245 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0399999999999998e-06 2023-10-09T10:47:08.245 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.245 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.246 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.246 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551616+0000", 2023-10-09T10:47:08.246 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001464449 2023-10-09T10:47:08.246 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.246 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.246 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551623+0000", 2023-10-09T10:47:08.246 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0090000000000001e-06 2023-10-09T10:47:08.246 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.246 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.247 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782780+0000", 2023-10-09T10:47:08.247 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2311575050000001 2023-10-09T10:47:08.247 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.247 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.247 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.247 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782892+0000", 2023-10-09T10:47:08.247 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111826 2023-10-09T10:47:08.247 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.247 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.247 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.247 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782998+0000", 2023-10-09T10:47:08.247 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010603399999999999 2023-10-09T10:47:08.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785644+0000", 2023-10-09T10:47:08.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026456489999999999 2023-10-09T10:47:08.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785699+0000", 2023-10-09T10:47:08.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4778999999999998e-05 2023-10-09T10:47:08.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786762+0000", 2023-10-09T10:47:08.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001063172 2023-10-09T10:47:08.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790708+0000", 2023-10-09T10:47:08.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039466409999999999 2023-10-09T10:47:08.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790775+0000", 2023-10-09T10:47:08.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6155000000000006e-05 2023-10-09T10:47:08.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791288+0000", 2023-10-09T10:47:08.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051300599999999999 2023-10-09T10:47:08.250 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.250 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.250 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.250 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43015 6.11 6:8d7a9802:::benchmark_data_smithi125_244951_object43014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.250 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:47:08.251 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.63991389, 2023-10-09T10:47:08.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.241685028, 2023-10-09T10:47:08.251 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.251 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.251 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.251 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.251 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.251 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43015 2023-10-09T10:47:08.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.251 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:47:08.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:47:08.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:47:08.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:08.253 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.253 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.253 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.253 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550174+0000", 2023-10-09T10:47:08.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8289000000000001e-05 2023-10-09T10:47:08.253 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.253 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.253 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.253 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550181+0000", 2023-10-09T10:47:08.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.024e-06 2023-10-09T10:47:08.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.254 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.254 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.254 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550187+0000", 2023-10-09T10:47:08.254 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6659999999999996e-06 2023-10-09T10:47:08.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.254 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.254 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.254 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551652+0000", 2023-10-09T10:47:08.254 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001465406 2023-10-09T10:47:08.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.254 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.254 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.255 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551659+0000", 2023-10-09T10:47:08.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1130000000000003e-06 2023-10-09T10:47:08.255 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.255 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.255 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.783679+0000", 2023-10-09T10:47:08.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.232019406 2023-10-09T10:47:08.255 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.255 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.255 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.783831+0000", 2023-10-09T10:47:08.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000151769 2023-10-09T10:47:08.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784019+0000", 2023-10-09T10:47:08.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018824400000000001 2023-10-09T10:47:08.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786769+0000", 2023-10-09T10:47:08.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027502529999999998 2023-10-09T10:47:08.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789091+0000", 2023-10-09T10:47:08.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023215200000000001 2023-10-09T10:47:08.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789131+0000", 2023-10-09T10:47:08.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0766999999999998e-05 2023-10-09T10:47:08.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791345+0000", 2023-10-09T10:47:08.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022138349999999999 2023-10-09T10:47:08.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791404+0000", 2023-10-09T10:47:08.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8301000000000003e-05 2023-10-09T10:47:08.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791841+0000", 2023-10-09T10:47:08.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00043785599999999998 2023-10-09T10:47:08.259 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.259 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.259 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.259 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43040 6.11 6:892fed64:::benchmark_data_smithi125_244951_object43039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.259 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550192+0000", 2023-10-09T10:47:08.259 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.63987851100001, 2023-10-09T10:47:08.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2421168730000001, 2023-10-09T10:47:08.259 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.259 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.259 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.260 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.260 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.260 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43040 2023-10-09T10:47:08.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.260 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550192+0000", 2023-10-09T10:47:08.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550192+0000", 2023-10-09T10:47:08.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.261 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550191+0000", 2023-10-09T10:47:08.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:08.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.261 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550265+0000", 2023-10-09T10:47:08.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3668e-05 2023-10-09T10:47:08.262 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.262 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.262 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.262 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550273+0000", 2023-10-09T10:47:08.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2230000000000007e-06 2023-10-09T10:47:08.262 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.262 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.262 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.262 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550280+0000", 2023-10-09T10:47:08.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3729999999999998e-06 2023-10-09T10:47:08.262 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.263 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.263 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551688+0000", 2023-10-09T10:47:08.263 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001408011 2023-10-09T10:47:08.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.263 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.263 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551694+0000", 2023-10-09T10:47:08.263 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8120000000000002e-06 2023-10-09T10:47:08.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785718+0000", 2023-10-09T10:47:08.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2340238640000001 2023-10-09T10:47:08.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785825+0000", 2023-10-09T10:47:08.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000107161 2023-10-09T10:47:08.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785964+0000", 2023-10-09T10:47:08.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013889800000000001 2023-10-09T10:47:08.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789139+0000", 2023-10-09T10:47:08.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0031749249999999999 2023-10-09T10:47:08.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789979+0000", 2023-10-09T10:47:08.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00083957300000000003 2023-10-09T10:47:08.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790018+0000", 2023-10-09T10:47:08.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8816999999999998e-05 2023-10-09T10:47:08.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791894+0000", 2023-10-09T10:47:08.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018764160000000001 2023-10-09T10:47:08.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791933+0000", 2023-10-09T10:47:08.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8889e-05 2023-10-09T10:47:08.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792309+0000", 2023-10-09T10:47:08.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00037573100000000003 2023-10-09T10:47:08.267 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.267 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.267 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.268 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43116 6.11 6:8866ecf5:::benchmark_data_smithi125_244951_object43115:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.268 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550285+0000", 2023-10-09T10:47:08.268 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.63978536400001, 2023-10-09T10:47:08.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2425320870000001, 2023-10-09T10:47:08.268 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.268 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.268 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.268 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.268 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.268 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43116 2023-10-09T10:47:08.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.269 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550285+0000", 2023-10-09T10:47:08.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550285+0000", 2023-10-09T10:47:08.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550284+0000", 2023-10-09T10:47:08.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:08.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550303+0000", 2023-10-09T10:47:08.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8817e-05 2023-10-09T10:47:08.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550310+0000", 2023-10-09T10:47:08.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7889999999999997e-06 2023-10-09T10:47:08.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.272 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550315+0000", 2023-10-09T10:47:08.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4539999999999998e-06 2023-10-09T10:47:08.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.272 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.272 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551718+0000", 2023-10-09T10:47:08.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001402585 2023-10-09T10:47:08.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.272 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.272 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551725+0000", 2023-10-09T10:47:08.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9319999999999999e-06 2023-10-09T10:47:08.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786811+0000", 2023-10-09T10:47:08.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.23508555 2023-10-09T10:47:08.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786947+0000", 2023-10-09T10:47:08.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013601499999999999 2023-10-09T10:47:08.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.787112+0000", 2023-10-09T10:47:08.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016519000000000001 2023-10-09T10:47:08.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789147+0000", 2023-10-09T10:47:08.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020350899999999998 2023-10-09T10:47:08.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790634+0000", 2023-10-09T10:47:08.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014872209999999999 2023-10-09T10:47:08.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790670+0000", 2023-10-09T10:47:08.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6395e-05 2023-10-09T10:47:08.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792355+0000", 2023-10-09T10:47:08.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016850179999999999 2023-10-09T10:47:08.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792391+0000", 2023-10-09T10:47:08.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5484000000000001e-05 2023-10-09T10:47:08.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792817+0000", 2023-10-09T10:47:08.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042605300000000002 2023-10-09T10:47:08.276 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.276 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.276 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.277 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43152 6.11 6:8f5a8f11:::benchmark_data_smithi125_244951_object43151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.277 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:47:08.277 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.63974902199999, 2023-10-09T10:47:08.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.254750676, 2023-10-09T10:47:08.277 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.277 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.277 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.277 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.277 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.277 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43152 2023-10-09T10:47:08.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.278 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:47:08.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:47:08.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:47:08.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:08.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550339+0000", 2023-10-09T10:47:08.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7940000000000001e-05 2023-10-09T10:47:08.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550346+0000", 2023-10-09T10:47:08.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4789999999999999e-06 2023-10-09T10:47:08.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550353+0000", 2023-10-09T10:47:08.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7510000000000002e-06 2023-10-09T10:47:08.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551748+0000", 2023-10-09T10:47:08.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013946119999999999 2023-10-09T10:47:08.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551754+0000", 2023-10-09T10:47:08.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8460000000000002e-06 2023-10-09T10:47:08.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792855+0000", 2023-10-09T10:47:08.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.241101091 2023-10-09T10:47:08.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792993+0000", 2023-10-09T10:47:08.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013794 2023-10-09T10:47:08.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.793143+0000", 2023-10-09T10:47:08.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014960799999999999 2023-10-09T10:47:08.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797271+0000", 2023-10-09T10:47:08.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041283810000000004 2023-10-09T10:47:08.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797282+0000", 2023-10-09T10:47:08.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0543999999999999e-05 2023-10-09T10:47:08.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799296+0000", 2023-10-09T10:47:08.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020138999999999999 2023-10-09T10:47:08.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803962+0000", 2023-10-09T10:47:08.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046658840000000004 2023-10-09T10:47:08.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.804015+0000", 2023-10-09T10:47:08.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3229999999999997e-05 2023-10-09T10:47:08.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805072+0000", 2023-10-09T10:47:08.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010569690000000001 2023-10-09T10:47:08.285 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.285 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.285 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.285 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43224 6.11 6:8b0bc017:::benchmark_data_smithi125_244951_object43223:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.285 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550374+0000", 2023-10-09T10:47:08.285 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.63969663399999, 2023-10-09T10:47:08.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2556577980000001, 2023-10-09T10:47:08.286 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.286 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.286 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.286 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.286 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.286 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43224 2023-10-09T10:47:08.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.286 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550374+0000", 2023-10-09T10:47:08.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550374+0000", 2023-10-09T10:47:08.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550373+0000", 2023-10-09T10:47:08.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:08.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550388+0000", 2023-10-09T10:47:08.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.453e-05 2023-10-09T10:47:08.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.288 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550393+0000", 2023-10-09T10:47:08.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7119999999999999e-06 2023-10-09T10:47:08.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.288 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550397+0000", 2023-10-09T10:47:08.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6789999999999996e-06 2023-10-09T10:47:08.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551777+0000", 2023-10-09T10:47:08.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001380196 2023-10-09T10:47:08.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551784+0000", 2023-10-09T10:47:08.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0369999999999997e-06 2023-10-09T10:47:08.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.793904+0000", 2023-10-09T10:47:08.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.242119346 2023-10-09T10:47:08.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.794042+0000", 2023-10-09T10:47:08.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013829700000000001 2023-10-09T10:47:08.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.794190+0000", 2023-10-09T10:47:08.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014784600000000001 2023-10-09T10:47:08.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797289+0000", 2023-10-09T10:47:08.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003099152 2023-10-09T10:47:08.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799339+0000", 2023-10-09T10:47:08.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020501059999999999 2023-10-09T10:47:08.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803853+0000", 2023-10-09T10:47:08.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045136179999999996 2023-10-09T10:47:08.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805211+0000", 2023-10-09T10:47:08.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001358349 2023-10-09T10:47:08.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805266+0000", 2023-10-09T10:47:08.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5297999999999998e-05 2023-10-09T10:47:08.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806031+0000", 2023-10-09T10:47:08.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00076497100000000003 2023-10-09T10:47:08.293 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.293 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.293 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.293 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43255 6.11 6:88dd5493:::benchmark_data_smithi125_244951_object43254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.294 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:47:08.294 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.639670115, 2023-10-09T10:47:08.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2563999859999999, 2023-10-09T10:47:08.294 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.294 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.294 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.294 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.294 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.294 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43255 2023-10-09T10:47:08.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.294 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:47:08.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:47:08.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:47:08.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:08.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550413+0000", 2023-10-09T10:47:08.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3052e-05 2023-10-09T10:47:08.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550417+0000", 2023-10-09T10:47:08.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5999999999999998e-06 2023-10-09T10:47:08.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550420+0000", 2023-10-09T10:47:08.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6519999999999999e-06 2023-10-09T10:47:08.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551807+0000", 2023-10-09T10:47:08.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001386998 2023-10-09T10:47:08.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551814+0000", 2023-10-09T10:47:08.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3960000000000003e-06 2023-10-09T10:47:08.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.794932+0000", 2023-10-09T10:47:08.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2431179880000001 2023-10-09T10:47:08.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.795097+0000", 2023-10-09T10:47:08.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000165248 2023-10-09T10:47:08.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.795276+0000", 2023-10-09T10:47:08.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017957699999999999 2023-10-09T10:47:08.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799383+0000", 2023-10-09T10:47:08.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004106569 2023-10-09T10:47:08.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801446+0000", 2023-10-09T10:47:08.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020628370000000001 2023-10-09T10:47:08.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803918+0000", 2023-10-09T10:47:08.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024717239999999998 2023-10-09T10:47:08.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806076+0000", 2023-10-09T10:47:08.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002158395 2023-10-09T10:47:08.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806122+0000", 2023-10-09T10:47:08.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5807999999999999e-05 2023-10-09T10:47:08.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806800+0000", 2023-10-09T10:47:08.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00067843299999999999 2023-10-09T10:47:08.301 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.301 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.301 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.302 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43315 6.11 6:8fb75c47:::benchmark_data_smithi125_244951_object43314:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.302 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550436+0000", 2023-10-09T10:47:08.302 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.639634621, 2023-10-09T10:47:08.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2571070289999999, 2023-10-09T10:47:08.302 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.302 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.302 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.302 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.302 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.302 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43315 2023-10-09T10:47:08.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.303 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550436+0000", 2023-10-09T10:47:08.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550436+0000", 2023-10-09T10:47:08.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550435+0000", 2023-10-09T10:47:08.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:08.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550453+0000", 2023-10-09T10:47:08.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7504999999999999e-05 2023-10-09T10:47:08.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550460+0000", 2023-10-09T10:47:08.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2470000000000002e-06 2023-10-09T10:47:08.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550466+0000", 2023-10-09T10:47:08.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8699999999999997e-06 2023-10-09T10:47:08.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551836+0000", 2023-10-09T10:47:08.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001370664 2023-10-09T10:47:08.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551843+0000", 2023-10-09T10:47:08.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5320000000000003e-06 2023-10-09T10:47:08.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.796128+0000", 2023-10-09T10:47:08.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2442850169999999 2023-10-09T10:47:08.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.796286+0000", 2023-10-09T10:47:08.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015828500000000001 2023-10-09T10:47:08.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.796448+0000", 2023-10-09T10:47:08.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016164200000000001 2023-10-09T10:47:08.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801434+0000", 2023-10-09T10:47:08.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049856279999999998 2023-10-09T10:47:08.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801530+0000", 2023-10-09T10:47:08.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.645e-05 2023-10-09T10:47:08.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805133+0000", 2023-10-09T10:47:08.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036025850000000002 2023-10-09T10:47:08.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806935+0000", 2023-10-09T10:47:08.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018020410000000001 2023-10-09T10:47:08.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.807002+0000", 2023-10-09T10:47:08.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6883000000000003e-05 2023-10-09T10:47:08.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.807543+0000", 2023-10-09T10:47:08.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000541115 2023-10-09T10:47:08.309 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.310 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.310 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.310 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43318 6.11 6:8f0da7e4:::benchmark_data_smithi125_244951_object43317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.310 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550471+0000", 2023-10-09T10:47:08.310 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.63959974299999, 2023-10-09T10:47:08.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2582329400000001, 2023-10-09T10:47:08.310 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.310 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.310 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.310 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.311 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.311 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43318 2023-10-09T10:47:08.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.311 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550471+0000", 2023-10-09T10:47:08.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550471+0000", 2023-10-09T10:47:08.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550470+0000", 2023-10-09T10:47:08.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:08.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550518+0000", 2023-10-09T10:47:08.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8108000000000001e-05 2023-10-09T10:47:08.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550526+0000", 2023-10-09T10:47:08.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0199999999999994e-06 2023-10-09T10:47:08.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550532+0000", 2023-10-09T10:47:08.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9109999999999998e-06 2023-10-09T10:47:08.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551864+0000", 2023-10-09T10:47:08.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001332317 2023-10-09T10:47:08.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551870+0000", 2023-10-09T10:47:08.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6489999999999996e-06 2023-10-09T10:47:08.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797329+0000", 2023-10-09T10:47:08.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.245458701 2023-10-09T10:47:08.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797476+0000", 2023-10-09T10:47:08.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014769199999999999 2023-10-09T10:47:08.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797644+0000", 2023-10-09T10:47:08.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000167481 2023-10-09T10:47:08.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801478+0000", 2023-10-09T10:47:08.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038341780000000002 2023-10-09T10:47:08.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801537+0000", 2023-10-09T10:47:08.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8879000000000003e-05 2023-10-09T10:47:08.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806859+0000", 2023-10-09T10:47:08.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0053216640000000003 2023-10-09T10:47:08.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.807979+0000", 2023-10-09T10:47:08.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011203770000000001 2023-10-09T10:47:08.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808056+0000", 2023-10-09T10:47:08.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6568000000000003e-05 2023-10-09T10:47:08.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808703+0000", 2023-10-09T10:47:08.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00064785600000000004 2023-10-09T10:47:08.318 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.318 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.318 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.318 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43334 6.11 6:8b093292:::benchmark_data_smithi125_244951_object43333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.318 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:47:08.318 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.63953207399999, 2023-10-09T10:47:08.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.266648716, 2023-10-09T10:47:08.319 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.319 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.319 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.319 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.319 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.319 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43334 2023-10-09T10:47:08.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.319 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:47:08.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:47:08.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:47:08.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:08.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550579+0000", 2023-10-09T10:47:08.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1733000000000002e-05 2023-10-09T10:47:08.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550587+0000", 2023-10-09T10:47:08.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6260000000000003e-06 2023-10-09T10:47:08.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550592+0000", 2023-10-09T10:47:08.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4160000000000003e-06 2023-10-09T10:47:08.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551890+0000", 2023-10-09T10:47:08.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001297702 2023-10-09T10:47:08.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551896+0000", 2023-10-09T10:47:08.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4559999999999999e-06 2023-10-09T10:47:08.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.798375+0000", 2023-10-09T10:47:08.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2464794800000001 2023-10-09T10:47:08.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.798514+0000", 2023-10-09T10:47:08.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000139159 2023-10-09T10:47:08.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.798661+0000", 2023-10-09T10:47:08.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000146312 2023-10-09T10:47:08.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.324 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801519+0000", 2023-10-09T10:47:08.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028583720000000001 2023-10-09T10:47:08.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.324 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803865+0000", 2023-10-09T10:47:08.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002345752 2023-10-09T10:47:08.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808789+0000", 2023-10-09T10:47:08.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049239790000000002 2023-10-09T10:47:08.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.816450+0000", 2023-10-09T10:47:08.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076614550000000002 2023-10-09T10:47:08.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.816501+0000", 2023-10-09T10:47:08.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1298000000000003e-05 2023-10-09T10:47:08.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817187+0000", 2023-10-09T10:47:08.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00068546500000000001 2023-10-09T10:47:08.326 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.326 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.326 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.327 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43385 6.11 6:8e6a343d:::benchmark_data_smithi125_244951_object43384:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:47:08.327 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.636508+0000", 2023-10-09T10:47:08.327 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.55356216600001, 2023-10-09T10:47:08.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1811192319999999, 2023-10-09T10:47:08.327 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.327 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.327 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.327 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.327 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.327 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43385 2023-10-09T10:47:08.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.328 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636508+0000", 2023-10-09T10:47:08.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636508+0000", 2023-10-09T10:47:08.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636507+0000", 2023-10-09T10:47:08.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:08.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636527+0000", 2023-10-09T10:47:08.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9748000000000001e-05 2023-10-09T10:47:08.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636539+0000", 2023-10-09T10:47:08.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2197e-05 2023-10-09T10:47:08.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.330 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636545+0000", 2023-10-09T10:47:08.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1999999999999999e-06 2023-10-09T10:47:08.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.330 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636582+0000", 2023-10-09T10:47:08.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6791000000000001e-05 2023-10-09T10:47:08.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636590+0000", 2023-10-09T10:47:08.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4309999999999994e-06 2023-10-09T10:47:08.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799411+0000", 2023-10-09T10:47:08.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1628212659999999 2023-10-09T10:47:08.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799550+0000", 2023-10-09T10:47:08.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013822300000000001 2023-10-09T10:47:08.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:47:08.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799660+0000", 2023-10-09T10:47:08.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110701 2023-10-09T10:47:08.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801589+0000", 2023-10-09T10:47:08.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019285470000000001 2023-10-09T10:47:08.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803871+0000", 2023-10-09T10:47:08.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022818180000000001 2023-10-09T10:47:08.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808835+0000", 2023-10-09T10:47:08.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049646200000000003 2023-10-09T10:47:08.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817231+0000", 2023-10-09T10:47:08.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083956959999999994 2023-10-09T10:47:08.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817263+0000", 2023-10-09T10:47:08.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1899000000000001e-05 2023-10-09T10:47:08.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.335 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817627+0000", 2023-10-09T10:47:08.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00036434500000000002 2023-10-09T10:47:08.335 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.335 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.335 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.335 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106397 6.1 6:856288a5:::benchmark_data_smithi125_244951_object106396:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:47:08.336 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.204308+0000", 2023-10-09T10:47:08.336 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 65.985762152000007, 2023-10-09T10:47:08.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.075713931999999998, 2023-10-09T10:47:08.336 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.336 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.336 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.336 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.336 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.336 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106397 2023-10-09T10:47:08.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.336 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204308+0000", 2023-10-09T10:47:08.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204308+0000", 2023-10-09T10:47:08.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204307+0000", 2023-10-09T10:47:08.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:47:08.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204330+0000", 2023-10-09T10:47:08.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2852000000000001e-05 2023-10-09T10:47:08.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204340+0000", 2023-10-09T10:47:08.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3989999999999993e-06 2023-10-09T10:47:08.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204346+0000", 2023-10-09T10:47:08.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5989999999999998e-06 2023-10-09T10:47:08.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.207434+0000", 2023-10-09T10:47:08.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030874600000000002 2023-10-09T10:47:08.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227524+0000", 2023-10-09T10:47:08.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.020090545000000001 2023-10-09T10:47:08.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:47:08.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227667+0000", 2023-10-09T10:47:08.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014324699999999999 2023-10-09T10:47:08.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.233515+0000", 2023-10-09T10:47:08.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0058477219999999996 2023-10-09T10:47:08.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.234252+0000", 2023-10-09T10:47:08.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00073691800000000003 2023-10-09T10:47:08.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279940+0000", 2023-10-09T10:47:08.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.045688232000000002 2023-10-09T10:47:08.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279985+0000", 2023-10-09T10:47:08.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4916000000000002e-05 2023-10-09T10:47:08.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.280022+0000", 2023-10-09T10:47:08.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6871e-05 2023-10-09T10:47:08.342 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.343 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.343 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.343 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106414 6.11 6:8c9e6868:::benchmark_data_smithi125_244951_object106413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:47:08.343 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.227135+0000", 2023-10-09T10:47:08.343 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 65.962935462999994, 2023-10-09T10:47:08.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.37275067699999997, 2023-10-09T10:47:08.344 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.344 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:47:08.344 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.344 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.344 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.344 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106414 2023-10-09T10:47:08.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.344 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227135+0000", 2023-10-09T10:47:08.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227135+0000", 2023-10-09T10:47:08.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227134+0000", 2023-10-09T10:47:08.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:08.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227156+0000", 2023-10-09T10:47:08.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2059999999999999e-05 2023-10-09T10:47:08.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227163+0000", 2023-10-09T10:47:08.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4169999999999997e-06 2023-10-09T10:47:08.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227168+0000", 2023-10-09T10:47:08.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9509999999999997e-06 2023-10-09T10:47:08.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.229108+0000", 2023-10-09T10:47:08.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001940396 2023-10-09T10:47:08.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.229279+0000", 2023-10-09T10:47:08.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017114100000000001 2023-10-09T10:47:08.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T10:47:08.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.229487+0000", 2023-10-09T10:47:08.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000207977 2023-10-09T10:47:08.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.233575+0000", 2023-10-09T10:47:08.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040875140000000004 2023-10-09T10:47:08.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.240998+0000", 2023-10-09T10:47:08.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074230249999999998 2023-10-09T10:47:08.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.245203+0000", 2023-10-09T10:47:08.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042052039999999997 2023-10-09T10:47:08.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274084+0000", 2023-10-09T10:47:08.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.028880626999999999 2023-10-09T10:47:08.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274097+0000", 2023-10-09T10:47:08.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.344e-05 2023-10-09T10:47:08.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.352 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.352 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599869+0000", 2023-10-09T10:47:08.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32577187699999999 2023-10-09T10:47:08.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.352 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.352 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599885+0000", 2023-10-09T10:47:08.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6515e-05 2023-10-09T10:47:08.352 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.352 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.353 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.353 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106433 6.11 6:8d28bd42:::benchmark_data_smithi125_244951_object106432:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:47:08.353 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.251731+0000", 2023-10-09T10:47:08.353 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 65.938338994999995, 2023-10-09T10:47:08.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.34819989800000001, 2023-10-09T10:47:08.353 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.353 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:47:08.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.354 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106433 2023-10-09T10:47:08.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.354 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251731+0000", 2023-10-09T10:47:08.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251731+0000", 2023-10-09T10:47:08.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251729+0000", 2023-10-09T10:47:08.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:47:08.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251758+0000", 2023-10-09T10:47:08.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8909000000000001e-05 2023-10-09T10:47:08.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251780+0000", 2023-10-09T10:47:08.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2011999999999999e-05 2023-10-09T10:47:08.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251791+0000", 2023-10-09T10:47:08.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0147e-05 2023-10-09T10:47:08.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.253110+0000", 2023-10-09T10:47:08.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013193739999999999 2023-10-09T10:47:08.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.253287+0000", 2023-10-09T10:47:08.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017752800000000001 2023-10-09T10:47:08.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T10:47:08.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.253484+0000", 2023-10-09T10:47:08.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000196622 2023-10-09T10:47:08.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.257296+0000", 2023-10-09T10:47:08.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038119550000000001 2023-10-09T10:47:08.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.257345+0000", 2023-10-09T10:47:08.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8745000000000002e-05 2023-10-09T10:47:08.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279630+0000", 2023-10-09T10:47:08.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.02228538 2023-10-09T10:47:08.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279642+0000", 2023-10-09T10:47:08.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2328e-05 2023-10-09T10:47:08.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599919+0000", 2023-10-09T10:47:08.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32027628800000002 2023-10-09T10:47:08.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599931+0000", 2023-10-09T10:47:08.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2439000000000001e-05 2023-10-09T10:47:08.360 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.360 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.360 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.360 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:136053 6.1 6:832e3320:::benchmark_data_smithi125_244951_object29610:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-10-09T10:47:08.360 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:02.144996+0000", 2023-10-09T10:47:08.360 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 6.0450745829999999, 2023-10-09T10:47:08.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.155053459, 2023-10-09T10:47:08.360 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:08.361 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:08.361 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:08.361 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:08.361 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:08.361 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 136053 2023-10-09T10:47:08.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.361 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:08.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:08.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.144996+0000", 2023-10-09T10:47:08.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:08.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.144996+0000", 2023-10-09T10:47:08.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:08.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.362 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:08.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.144994+0000", 2023-10-09T10:47:08.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:08.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.362 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:08.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145002+0000", 2023-10-09T10:47:08.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8429999999999993e-06 2023-10-09T10:47:08.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:08.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145014+0000", 2023-10-09T10:47:08.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2163e-05 2023-10-09T10:47:08.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:08.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145024+0000", 2023-10-09T10:47:08.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7610000000000002e-06 2023-10-09T10:47:08.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145083+0000", 2023-10-09T10:47:08.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8882999999999998e-05 2023-10-09T10:47:08.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:08.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145095+0000", 2023-10-09T10:47:08.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2255999999999999e-05 2023-10-09T10:47:08.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:08.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.287386+0000", 2023-10-09T10:47:08.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.14229140700000001 2023-10-09T10:47:08.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:08.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.287495+0000", 2023-10-09T10:47:08.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108651 2023-10-09T10:47:08.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:47:08.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.287593+0000", 2023-10-09T10:47:08.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8348000000000001e-05 2023-10-09T10:47:08.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.295087+0000", 2023-10-09T10:47:08.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074939550000000001 2023-10-09T10:47:08.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:08.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.295761+0000", 2023-10-09T10:47:08.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00067362100000000001 2023-10-09T10:47:08.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:08.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.299957+0000", 2023-10-09T10:47:08.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041957419999999997 2023-10-09T10:47:08.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:08.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.300009+0000", 2023-10-09T10:47:08.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2193e-05 2023-10-09T10:47:08.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:08.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:08.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:08.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.300049+0000", 2023-10-09T10:47:08.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0156e-05 2023-10-09T10:47:08.368 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.368 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.368 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.368 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:08.368 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:08.368 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:47:08.368 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:08.412 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:08 smithi155 ceph-mon[184854]: pgmap v344: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 464 op/s; 34009/225555 objects degraded (15.078%); 11746/225555 objects misplaced (5.208%); 8.8 MiB/s, 226 objects/s recovering 2023-10-09T10:47:08.412 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:08 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:47:08.412 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:08 smithi155 ceph-mon[184854]: failed to encode map e167 with expected crc 2023-10-09T10:47:08.412 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:08 smithi155 ceph-mon[184854]: failed to encode map e167 with expected crc 2023-10-09T10:47:08.412 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:08 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:47:08.412 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:08 smithi155 ceph-mon[184854]: osdmap e167: 8 total, 8 up, 7 in 2023-10-09T10:47:08.413 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:08 smithi155 ceph-mon[184854]: failed to encode map e167 with expected crc 2023-10-09T10:47:08.413 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:08 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 34009/225555 objects degraded (15.078%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:08.415 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:08.437 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:08.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:08.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:08.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[195507]: pgmap v344: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 464 op/s; 34009/225555 objects degraded (15.078%); 11746/225555 objects misplaced (5.208%); 8.8 MiB/s, 226 objects/s recovering 2023-10-09T10:47:08.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:47:08.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[195507]: failed to encode map e167 with expected crc 2023-10-09T10:47:08.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[195507]: failed to encode map e167 with expected crc 2023-10-09T10:47:08.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:47:08.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[195507]: osdmap e167: 8 total, 8 up, 7 in 2023-10-09T10:47:08.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[195507]: failed to encode map e167 with expected crc 2023-10-09T10:47:08.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 34009/225555 objects degraded (15.078%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:08.498 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[200197]: pgmap v344: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 464 op/s; 34009/225555 objects degraded (15.078%); 11746/225555 objects misplaced (5.208%); 8.8 MiB/s, 226 objects/s recovering 2023-10-09T10:47:08.498 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:47:08.498 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[200197]: failed to encode map e167 with expected crc 2023-10-09T10:47:08.498 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[200197]: failed to encode map e167 with expected crc 2023-10-09T10:47:08.498 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:47:08.498 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[200197]: osdmap e167: 8 total, 8 up, 7 in 2023-10-09T10:47:08.498 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[200197]: failed to encode map e167 with expected crc 2023-10-09T10:47:08.498 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:08 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 34009/225555 objects degraded (15.078%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:08.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:08.584 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:08.614 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:08.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:08.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:08.691 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:47:08.720 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:08.755 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:08.783 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:08.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:08.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:08.892 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:08.952 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:09.072 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:09.095 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:09.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:09.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:09.108 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:47:09.141 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:47:09.177 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:47:09.202 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:09.203 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:47:09.230 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:47:09.232 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:09.249 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:09.254 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:09.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:09.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:09.265 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:47:09.297 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:47:09.329 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:47:09.330 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:47:09.331 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:47:09.331 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:47:09.331 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:47:09.331 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:47:09.331 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:47:09.331 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:47:09.331 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:47:09.354 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:09 smithi125 ceph-mon[195507]: failed to encode map e167 with expected crc 2023-10-09T10:47:09.354 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:09 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1293661814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:09.355 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:09 smithi125 ceph-mon[200197]: failed to encode map e167 with expected crc 2023-10-09T10:47:09.355 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:09 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1293661814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:09.360 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:09.416 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:09.449 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:09.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:09.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:09.555 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:09.614 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:09.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:09 smithi155 ceph-mon[184854]: failed to encode map e167 with expected crc 2023-10-09T10:47:09.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:09 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1293661814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:09.830 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:47:10.158 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:10.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:10.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:10.263 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:10.306 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:10.472 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:10 smithi125 ceph-mon[195507]: pgmap v346: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 34009/224997 objects degraded (15.115%); 11746/224997 objects misplaced (5.221%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:47:10.473 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:10 smithi125 ceph-mon[200197]: pgmap v346: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 34009/224997 objects degraded (15.115%); 11746/224997 objects misplaced (5.221%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:47:10.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:10 smithi155 ceph-mon[184854]: pgmap v346: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 34009/224997 objects degraded (15.115%); 11746/224997 objects misplaced (5.221%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:47:10.906 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:10.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:10.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:11.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:11.076 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:11.483 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:47:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:47:11] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.43.0" 2023-10-09T10:47:11.505 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:11.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:11.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:11.608 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:11.651 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:11.733 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:11.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:11.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:11.843 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:11.895 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:11.919 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:11.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:11.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:12.024 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:12.051 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:12.102 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:12.132 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:12.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:12.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:12.237 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:12.279 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:12.299 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:12.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:12.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:12.403 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:12.446 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:12 smithi125 ceph-mon[195507]: pgmap v347: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 34009/219288 objects degraded (15.509%); 10930/219288 objects misplaced (4.984%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T10:47:12.447 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:12 smithi125 ceph-mon[200197]: pgmap v347: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 34009/219288 objects degraded (15.509%); 10930/219288 objects misplaced (4.984%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T10:47:12.457 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:12.472 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:47:12.472 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:47:12.690 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:12.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:12.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:12 smithi155 ceph-mon[184854]: pgmap v347: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 34009/219288 objects degraded (15.509%); 10930/219288 objects misplaced (4.984%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T10:47:12.794 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:12.827 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:12.853 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:12.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:12.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:12.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:13.007 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:13.087 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:13.093 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:47:13.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:13.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:13.195 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:13.250 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:13.328 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:13.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:13.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:13.366 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T10:47:13.443 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:13.476 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:13.503 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:47:13.503 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:13.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:13.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:13.556 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:13 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2678769506' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:47:13.556 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:13 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 34009/219288 objects degraded (15.509%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:13.556 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:13 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2678769506' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:47:13.556 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:13 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 34009/219288 objects degraded (15.509%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:13.609 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:13.636 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:13 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2678769506' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:47:13.636 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:13 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 34009/219288 objects degraded (15.509%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:13.641 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:13.667 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:13.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:13.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:13.773 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:13.802 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:13.826 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:13.827 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:13.830 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T10:47:13.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:13.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:13.932 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:13.975 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:14.019 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:14.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:14.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:14.124 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:14.157 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:14.186 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:14.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:14.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:14.294 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:14.335 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:14.357 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:14.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:14 smithi155 ceph-mon[184854]: pgmap v348: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 34009/217980 objects degraded (15.602%); 10856/217980 objects misplaced (4.980%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T10:47:14.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:14 smithi155 ceph-mon[184854]: failed to encode map e168 with expected crc 2023-10-09T10:47:14.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:14 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2678769506' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:47:14.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:14 smithi155 ceph-mon[184854]: osdmap e168: 8 total, 8 up, 7 in 2023-10-09T10:47:14.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:14 smithi155 ceph-mon[184854]: failed to encode map e168 with expected crc 2023-10-09T10:47:14.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:14 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2644684495' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:47:14.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:14 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:47:14.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:47:14.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[195507]: pgmap v348: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 34009/217980 objects degraded (15.602%); 10856/217980 objects misplaced (4.980%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T10:47:14.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[195507]: failed to encode map e168 with expected crc 2023-10-09T10:47:14.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2678769506' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:47:14.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[195507]: osdmap e168: 8 total, 8 up, 7 in 2023-10-09T10:47:14.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[195507]: failed to encode map e168 with expected crc 2023-10-09T10:47:14.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2644684495' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:47:14.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:47:14.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:47:14.474 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[200197]: pgmap v348: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 34009/217980 objects degraded (15.602%); 10856/217980 objects misplaced (4.980%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T10:47:14.474 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[200197]: failed to encode map e168 with expected crc 2023-10-09T10:47:14.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2678769506' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:47:14.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[200197]: osdmap e168: 8 total, 8 up, 7 in 2023-10-09T10:47:14.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[200197]: failed to encode map e168 with expected crc 2023-10-09T10:47:14.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2644684495' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:47:14.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:47:14.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:47:14.667 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:47:14.940 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:14.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:14.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:15.045 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:15.080 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:15.205 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:15.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:15.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:15.310 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:15.349 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:15.405 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:15.426 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:15.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:15.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:15.531 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:15.581 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:15.630 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:47:15.631 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:47:15.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:15 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:15.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:15 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:47:15.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:15 smithi125 ceph-mon[195507]: osdmap e169: 8 total, 8 up, 7 in 2023-10-09T10:47:15.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:15 smithi125 ceph-mon[195507]: failed to encode map e169 with expected crc 2023-10-09T10:47:15.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:15 smithi125 ceph-mon[195507]: failed to encode map e169 with expected crc 2023-10-09T10:47:15.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:15 smithi125 ceph-mon[195507]: failed to encode map e169 with expected crc 2023-10-09T10:47:15.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:47:15.661 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:15 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:15.661 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:15 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:47:15.661 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:15 smithi125 ceph-mon[200197]: osdmap e169: 8 total, 8 up, 7 in 2023-10-09T10:47:15.661 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:15 smithi125 ceph-mon[200197]: failed to encode map e169 with expected crc 2023-10-09T10:47:15.662 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:15 smithi125 ceph-mon[200197]: failed to encode map e169 with expected crc 2023-10-09T10:47:15.662 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:15 smithi125 ceph-mon[200197]: failed to encode map e169 with expected crc 2023-10-09T10:47:15.662 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:47:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:15 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:15 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:47:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:15 smithi155 ceph-mon[184854]: osdmap e169: 8 total, 8 up, 7 in 2023-10-09T10:47:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:15 smithi155 ceph-mon[184854]: failed to encode map e169 with expected crc 2023-10-09T10:47:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:15 smithi155 ceph-mon[184854]: failed to encode map e169 with expected crc 2023-10-09T10:47:15.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:15 smithi155 ceph-mon[184854]: failed to encode map e169 with expected crc 2023-10-09T10:47:15.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:47:16.544 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:16.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:16.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:16.598 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:16 smithi125 ceph-mon[195507]: pgmap v351: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 34009/215100 objects degraded (15.811%); 10559/215100 objects misplaced (4.909%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T10:47:16.598 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:16 smithi125 ceph-mon[200197]: pgmap v351: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 34009/215100 objects degraded (15.811%); 10559/215100 objects misplaced (4.909%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T10:47:16.652 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:16.681 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:16 smithi155 ceph-mon[184854]: pgmap v351: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 34009/215100 objects degraded (15.811%); 10559/215100 objects misplaced (4.909%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T10:47:16.719 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:16.843 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:16.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:16.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:16.916 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:47:16.920 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:47:16.949 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:16.957 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:47:16.982 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:16.990 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:47:17.011 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:17.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:17.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:17.032 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:47:17.070 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:47:17.114 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:17.127 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:47:17.162 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:47:17.162 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:17.195 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:47:17.197 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:17.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:17.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:17.210 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:47:17.210 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:47:17.210 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:47:17.210 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:47:17.211 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:47:17.211 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:47:17.211 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:47:17.211 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:47:17.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:17.333 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:17.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3579236532' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:17.411 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:17 smithi125 ceph-mon[195507]: pgmap v352: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 34009/212190 objects degraded (16.028%); 9899/212190 objects misplaced (4.665%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T10:47:17.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3579236532' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:17.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:17 smithi125 ceph-mon[200197]: pgmap v352: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 34009/212190 objects degraded (16.028%); 9899/212190 objects misplaced (4.665%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T10:47:17.454 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:17.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:17.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:17.557 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:17.605 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:17.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3579236532' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:17.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:17 smithi155 ceph-mon[184854]: pgmap v352: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 34009/212190 objects degraded (16.028%); 9899/212190 objects misplaced (4.665%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T10:47:17.639 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:17.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:17.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:17.755 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:17.795 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:17.827 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:17.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:17.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:17.916 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:17.931 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:17.961 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:17.985 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:17.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:17.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:18.089 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:18.148 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:18.227 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:18.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:18.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:18.334 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:18.387 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:18.555 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:47:18.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:18 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 34009/212190 objects degraded (16.028%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:18.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:18 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:47:18.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:18 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:47:18.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:18 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2864412677' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:47:18.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:18 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:47:18.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:18 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 34009/212190 objects degraded (16.028%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:18.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:18 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:47:18.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:18 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:47:18.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:18 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2864412677' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:47:18.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:18 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:47:18.572 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:18.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:18.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:18.681 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:18.718 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:18 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 34009/212190 objects degraded (16.028%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:18.718 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:18 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:47:18.718 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:18 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:47:18.718 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2864412677' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:47:18.718 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:18 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:47:18.721 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:18.747 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:18.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:18.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:18.852 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:18.890 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:18.913 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:18.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:18.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:18.920 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T10:47:18.982 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:19.016 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:19.046 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:19.067 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:19.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:19.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:19.171 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:19.209 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:19.237 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:19.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:19.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:19.342 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:19.377 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:19.504 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:19.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:19.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:19.539 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T10:47:19.539 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:47:19.613 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:19.658 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:19.690 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:19.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:19.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:19.794 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:19.845 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:19.879 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:19.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:19.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:19.927 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:47:19.989 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:20.055 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:47:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:47:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:47:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:47:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1070 6.8 6:1676aaf7:::benchmark_data_smithi125_244951_object1069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:47:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.621927+0000", 2023-10-09T10:47:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 167.4265667, 2023-10-09T10:47:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99674418099999995, 2023-10-09T10:47:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:47:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1070 2023-10-09T10:47:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621927+0000", 2023-10-09T10:47:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621927+0000", 2023-10-09T10:47:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621926+0000", 2023-10-09T10:47:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621950+0000", 2023-10-09T10:47:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3706999999999999e-05 2023-10-09T10:47:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621964+0000", 2023-10-09T10:47:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4589999999999999e-05 2023-10-09T10:47:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621972+0000", 2023-10-09T10:47:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3510000000000002e-06 2023-10-09T10:47:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.622022+0000", 2023-10-09T10:47:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9088000000000002e-05 2023-10-09T10:47:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.622034+0000", 2023-10-09T10:47:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2584e-05 2023-10-09T10:47:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.618557+0000", 2023-10-09T10:47:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99652308199999995 2023-10-09T10:47:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.618672+0000", 2023-10-09T10:47:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011433 2023-10-09T10:47:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1095 6.8 6:164b39e5:::benchmark_data_smithi125_244951_object1094:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:47:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.623092+0000", 2023-10-09T10:47:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 167.425402499, 2023-10-09T10:47:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99981824900000005, 2023-10-09T10:47:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1095 2023-10-09T10:47:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623092+0000", 2023-10-09T10:47:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623092+0000", 2023-10-09T10:47:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623090+0000", 2023-10-09T10:47:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:47:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623113+0000", 2023-10-09T10:47:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2691e-05 2023-10-09T10:47:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623130+0000", 2023-10-09T10:47:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7594000000000001e-05 2023-10-09T10:47:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623139+0000", 2023-10-09T10:47:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5359999999999992e-06 2023-10-09T10:47:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623187+0000", 2023-10-09T10:47:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7874000000000003e-05 2023-10-09T10:47:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623197+0000", 2023-10-09T10:47:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0889e-05 2023-10-09T10:47:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.618984+0000", 2023-10-09T10:47:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99578652199999995 2023-10-09T10:47:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.619093+0000", 2023-10-09T10:47:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108948 2023-10-09T10:47:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:47:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.619248+0000", 2023-10-09T10:47:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000154747 2023-10-09T10:47:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620532+0000", 2023-10-09T10:47:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012847729999999999 2023-10-09T10:47:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621605+0000", 2023-10-09T10:47:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010723569999999999 2023-10-09T10:47:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622824+0000", 2023-10-09T10:47:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012196939999999999 2023-10-09T10:47:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622872+0000", 2023-10-09T10:47:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7573e-05 2023-10-09T10:47:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622910+0000", 2023-10-09T10:47:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7790999999999998e-05 2023-10-09T10:47:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1097 6.8 6:1cc6c81b:::benchmark_data_smithi125_244951_object1096:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:47:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.625080+0000", 2023-10-09T10:47:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 167.42341388299999, 2023-10-09T10:47:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.997987073, 2023-10-09T10:47:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1097 2023-10-09T10:47:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625080+0000", 2023-10-09T10:47:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625080+0000", 2023-10-09T10:47:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625078+0000", 2023-10-09T10:47:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T10:47:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625108+0000", 2023-10-09T10:47:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9626000000000001e-05 2023-10-09T10:47:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625132+0000", 2023-10-09T10:47:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4236999999999999e-05 2023-10-09T10:47:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625145+0000", 2023-10-09T10:47:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2814e-05 2023-10-09T10:47:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625215+0000", 2023-10-09T10:47:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0779999999999997e-05 2023-10-09T10:47:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625228+0000", 2023-10-09T10:47:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2989e-05 2023-10-09T10:47:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.619577+0000", 2023-10-09T10:47:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99434825299999996 2023-10-09T10:47:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.619653+0000", 2023-10-09T10:47:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.682e-05 2023-10-09T10:47:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:47:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.619810+0000", 2023-10-09T10:47:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000156737 2023-10-09T10:47:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621611+0000", 2023-10-09T10:47:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001800472 2023-10-09T10:47:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621664+0000", 2023-10-09T10:47:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2967999999999998e-05 2023-10-09T10:47:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622991+0000", 2023-10-09T10:47:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001327122 2023-10-09T10:47:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623034+0000", 2023-10-09T10:47:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2772000000000003e-05 2023-10-09T10:47:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623067+0000", 2023-10-09T10:47:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3767999999999999e-05 2023-10-09T10:47:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1100 6.8 6:1c8c7e3c:::benchmark_data_smithi125_244951_object1099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:47:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.627382+0000", 2023-10-09T10:47:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 167.42111211700001, 2023-10-09T10:47:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99583719999999998, 2023-10-09T10:47:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1100 2023-10-09T10:47:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627382+0000", 2023-10-09T10:47:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627382+0000", 2023-10-09T10:47:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627380+0000", 2023-10-09T10:47:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:47:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627405+0000", 2023-10-09T10:47:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5058e-05 2023-10-09T10:47:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627419+0000", 2023-10-09T10:47:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3431999999999999e-05 2023-10-09T10:47:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627426+0000", 2023-10-09T10:47:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.34e-06 2023-10-09T10:47:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627534+0000", 2023-10-09T10:47:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010761599999999999 2023-10-09T10:47:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627543+0000", 2023-10-09T10:47:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2660000000000007e-06 2023-10-09T10:47:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620079+0000", 2023-10-09T10:47:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99253606000000005 2023-10-09T10:47:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620154+0000", 2023-10-09T10:47:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5461000000000001e-05 2023-10-09T10:47:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:47:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620263+0000", 2023-10-09T10:47:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010842599999999999 2023-10-09T10:47:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621714+0000", 2023-10-09T10:47:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001451265 2023-10-09T10:47:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621722+0000", 2023-10-09T10:47:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2740000000000007e-06 2023-10-09T10:47:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623143+0000", 2023-10-09T10:47:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014206570000000001 2023-10-09T10:47:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623183+0000", 2023-10-09T10:47:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9660999999999999e-05 2023-10-09T10:47:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623219+0000", 2023-10-09T10:47:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6433000000000002e-05 2023-10-09T10:47:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1136 6.8 6:193c408d:::benchmark_data_smithi125_244951_object1135:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:47:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.650385+0000", 2023-10-09T10:47:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 167.39810949, 2023-10-09T10:47:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97294367199999998, 2023-10-09T10:47:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1136 2023-10-09T10:47:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650385+0000", 2023-10-09T10:47:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650385+0000", 2023-10-09T10:47:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650383+0000", 2023-10-09T10:47:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650411+0000", 2023-10-09T10:47:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7837000000000001e-05 2023-10-09T10:47:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650421+0000", 2023-10-09T10:47:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8779999999999993e-06 2023-10-09T10:47:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650427+0000", 2023-10-09T10:47:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7270000000000001e-06 2023-10-09T10:47:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650504+0000", 2023-10-09T10:47:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6800999999999999e-05 2023-10-09T10:47:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650514+0000", 2023-10-09T10:47:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9490000000000008e-06 2023-10-09T10:47:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620561+0000", 2023-10-09T10:47:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97004676599999995 2023-10-09T10:47:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620656+0000", 2023-10-09T10:47:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5345000000000003e-05 2023-10-09T10:47:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:47:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620791+0000", 2023-10-09T10:47:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013480999999999999 2023-10-09T10:47:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621728+0000", 2023-10-09T10:47:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00093716100000000005 2023-10-09T10:47:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621777+0000", 2023-10-09T10:47:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8569999999999997e-05 2023-10-09T10:47:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623256+0000", 2023-10-09T10:47:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014793569999999999 2023-10-09T10:47:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623297+0000", 2023-10-09T10:47:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1012e-05 2023-10-09T10:47:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623328+0000", 2023-10-09T10:47:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1044999999999998e-05 2023-10-09T10:47:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1140 6.8 6:1bbcb053:::benchmark_data_smithi125_244951_object1139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:47:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.651480+0000", 2023-10-09T10:47:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 167.39701456, 2023-10-09T10:47:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97196035599999997, 2023-10-09T10:47:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1140 2023-10-09T10:47:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651480+0000", 2023-10-09T10:47:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651480+0000", 2023-10-09T10:47:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651479+0000", 2023-10-09T10:47:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651498+0000", 2023-10-09T10:47:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9128999999999999e-05 2023-10-09T10:47:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651507+0000", 2023-10-09T10:47:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0410000000000003e-06 2023-10-09T10:47:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651513+0000", 2023-10-09T10:47:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4990000000000002e-06 2023-10-09T10:47:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651606+0000", 2023-10-09T10:47:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3546000000000002e-05 2023-10-09T10:47:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651618+0000", 2023-10-09T10:47:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1600000000000001e-05 2023-10-09T10:47:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621048+0000", 2023-10-09T10:47:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96943005599999998 2023-10-09T10:47:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621174+0000", 2023-10-09T10:47:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012588899999999999 2023-10-09T10:47:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:47:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621311+0000", 2023-10-09T10:47:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000137415 2023-10-09T10:47:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622787+0000", 2023-10-09T10:47:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014754880000000001 2023-10-09T10:47:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622958+0000", 2023-10-09T10:47:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017176800000000001 2023-10-09T10:47:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623365+0000", 2023-10-09T10:47:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00040624999999999998 2023-10-09T10:47:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623405+0000", 2023-10-09T10:47:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0680999999999999e-05 2023-10-09T10:47:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623440+0000", 2023-10-09T10:47:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4474000000000002e-05 2023-10-09T10:47:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1141 6.8 6:1ec53f00:::benchmark_data_smithi125_244951_object1140:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:47:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.651524+0000", 2023-10-09T10:47:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 167.39697017, 2023-10-09T10:47:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97203539800000005, 2023-10-09T10:47:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1141 2023-10-09T10:47:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651524+0000", 2023-10-09T10:47:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651524+0000", 2023-10-09T10:47:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651523+0000", 2023-10-09T10:47:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651544+0000", 2023-10-09T10:47:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.014e-05 2023-10-09T10:47:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651552+0000", 2023-10-09T10:47:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6179999999999995e-06 2023-10-09T10:47:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651561+0000", 2023-10-09T10:47:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7860000000000002e-06 2023-10-09T10:47:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651639+0000", 2023-10-09T10:47:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8362999999999995e-05 2023-10-09T10:47:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651646+0000", 2023-10-09T10:47:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6610000000000003e-06 2023-10-09T10:47:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621816+0000", 2023-10-09T10:47:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97017014700000004 2023-10-09T10:47:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621928+0000", 2023-10-09T10:47:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111427 2023-10-09T10:47:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:47:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622053+0000", 2023-10-09T10:47:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012548999999999999 2023-10-09T10:47:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622921+0000", 2023-10-09T10:47:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00086811799999999997 2023-10-09T10:47:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623114+0000", 2023-10-09T10:47:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019289700000000001 2023-10-09T10:47:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623487+0000", 2023-10-09T10:47:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00037276399999999998 2023-10-09T10:47:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623528+0000", 2023-10-09T10:47:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1179e-05 2023-10-09T10:47:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623559+0000", 2023-10-09T10:47:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1245999999999999e-05 2023-10-09T10:47:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1151 6.8 6:15b7e58c:::benchmark_data_smithi125_244951_object1150:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:47:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.658348+0000", 2023-10-09T10:47:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 167.390146488, 2023-10-09T10:47:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96534045700000004, 2023-10-09T10:47:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1151 2023-10-09T10:47:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658348+0000", 2023-10-09T10:47:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658348+0000", 2023-10-09T10:47:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658347+0000", 2023-10-09T10:47:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658366+0000", 2023-10-09T10:47:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8757e-05 2023-10-09T10:47:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658378+0000", 2023-10-09T10:47:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.216e-05 2023-10-09T10:47:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658384+0000", 2023-10-09T10:47:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0170000000000002e-06 2023-10-09T10:47:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658460+0000", 2023-10-09T10:47:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5543999999999994e-05 2023-10-09T10:47:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658481+0000", 2023-10-09T10:47:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1916000000000002e-05 2023-10-09T10:47:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622329+0000", 2023-10-09T10:47:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96384757600000004 2023-10-09T10:47:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622438+0000", 2023-10-09T10:47:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108985 2023-10-09T10:47:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:47:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622573+0000", 2023-10-09T10:47:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013480499999999999 2023-10-09T10:47:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623082+0000", 2023-10-09T10:47:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00050925299999999998 2023-10-09T10:47:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623595+0000", 2023-10-09T10:47:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051302300000000004 2023-10-09T10:47:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623625+0000", 2023-10-09T10:47:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0309000000000001e-05 2023-10-09T10:47:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623657+0000", 2023-10-09T10:47:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2027999999999999e-05 2023-10-09T10:47:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623688+0000", 2023-10-09T10:47:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0593000000000003e-05 2023-10-09T10:47:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1184 6.8 6:157bd3fc:::benchmark_data_smithi125_244951_object1183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:47:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.696219+0000", 2023-10-09T10:47:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 167.352275507, 2023-10-09T10:47:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.92991913699999995, 2023-10-09T10:47:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1184 2023-10-09T10:47:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696219+0000", 2023-10-09T10:47:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696219+0000", 2023-10-09T10:47:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696217+0000", 2023-10-09T10:47:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696237+0000", 2023-10-09T10:47:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9983000000000002e-05 2023-10-09T10:47:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696249+0000", 2023-10-09T10:47:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2215000000000001e-05 2023-10-09T10:47:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696257+0000", 2023-10-09T10:47:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.072e-06 2023-10-09T10:47:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696297+0000", 2023-10-09T10:47:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9852999999999999e-05 2023-10-09T10:47:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696305+0000", 2023-10-09T10:47:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7870000000000002e-06 2023-10-09T10:47:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623715+0000", 2023-10-09T10:47:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.92741050999999997 2023-10-09T10:47:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623814+0000", 2023-10-09T10:47:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8057e-05 2023-10-09T10:47:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:47:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623938+0000", 2023-10-09T10:47:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012472899999999999 2023-10-09T10:47:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.625153+0000", 2023-10-09T10:47:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012150970000000001 2023-10-09T10:47:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.625631+0000", 2023-10-09T10:47:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047751299999999999 2023-10-09T10:47:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.626058+0000", 2023-10-09T10:47:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042723499999999998 2023-10-09T10:47:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.626104+0000", 2023-10-09T10:47:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5569999999999999e-05 2023-10-09T10:47:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.626138+0000", 2023-10-09T10:47:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4069999999999997e-05 2023-10-09T10:47:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44570.0:70220 6.d e124/122 6:bb7bde6c:::benchmark_data_smithi125_244951_object70219:head v 124'4435, mlcod=0'0)", 2023-10-09T10:47:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:23.234057+0000", 2023-10-09T10:47:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 116.814437492, 2023-10-09T10:47:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.054302571000000001, 2023-10-09T10:47:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.234057+0000", 2023-10-09T10:47:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.234057+0000", 2023-10-09T10:47:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.234056+0000", 2023-10-09T10:47:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.234088+0000", 2023-10-09T10:47:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1903000000000003e-05 2023-10-09T10:47:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.234094+0000", 2023-10-09T10:47:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8080000000000001e-06 2023-10-09T10:47:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.234098+0000", 2023-10-09T10:47:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2390000000000004e-06 2023-10-09T10:47:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.248393+0000", 2023-10-09T10:47:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014295006000000001 2023-10-09T10:47:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.248418+0000", 2023-10-09T10:47:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4870000000000001e-05 2023-10-09T10:47:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.288320+0000", 2023-10-09T10:47:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.039902083999999997 2023-10-09T10:47:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.288359+0000", 2023-10-09T10:47:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8911000000000001e-05 2023-10-09T10:47:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:70301 6.19 6:98815d50:::benchmark_data_smithi125_244951_object70300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:47:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:23.364524+0000", 2023-10-09T10:47:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 116.683970436, 2023-10-09T10:47:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.054642890999999999, 2023-10-09T10:47:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 70301 2023-10-09T10:47:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364524+0000", 2023-10-09T10:47:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364524+0000", 2023-10-09T10:47:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364522+0000", 2023-10-09T10:47:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:47:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364544+0000", 2023-10-09T10:47:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1990000000000001e-05 2023-10-09T10:47:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364556+0000", 2023-10-09T10:47:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2181000000000001e-05 2023-10-09T10:47:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364564+0000", 2023-10-09T10:47:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0499999999999992e-06 2023-10-09T10:47:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364698+0000", 2023-10-09T10:47:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013398800000000001 2023-10-09T10:47:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364826+0000", 2023-10-09T10:47:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012756099999999999 2023-10-09T10:47:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:47:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364936+0000", 2023-10-09T10:47:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011065000000000001 2023-10-09T10:47:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.366921+0000", 2023-10-09T10:47:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019843679999999998 2023-10-09T10:47:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.366945+0000", 2023-10-09T10:47:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3995999999999999e-05 2023-10-09T10:47:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.418241+0000", 2023-10-09T10:47:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.051296003 2023-10-09T10:47:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.418305+0000", 2023-10-09T10:47:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4815999999999997e-05 2023-10-09T10:47:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.419167+0000", 2023-10-09T10:47:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00086109200000000004 2023-10-09T10:47:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:70319 6.18 6:1f556d03:::benchmark_data_smithi125_244951_object70318:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:47:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:23.385122+0000", 2023-10-09T10:47:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 116.663372533, 2023-10-09T10:47:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.059906651999999998, 2023-10-09T10:47:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 70319 2023-10-09T10:47:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385122+0000", 2023-10-09T10:47:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385122+0000", 2023-10-09T10:47:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385120+0000", 2023-10-09T10:47:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385145+0000", 2023-10-09T10:47:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4414e-05 2023-10-09T10:47:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385155+0000", 2023-10-09T10:47:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0805000000000001e-05 2023-10-09T10:47:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385163+0000", 2023-10-09T10:47:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0330000000000008e-06 2023-10-09T10:47:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385219+0000", 2023-10-09T10:47:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5894999999999999e-05 2023-10-09T10:47:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385382+0000", 2023-10-09T10:47:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016238299999999999 2023-10-09T10:47:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:47:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385553+0000", 2023-10-09T10:47:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017152199999999999 2023-10-09T10:47:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.399377+0000", 2023-10-09T10:47:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013823291 2023-10-09T10:47:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.400837+0000", 2023-10-09T10:47:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014599890000000001 2023-10-09T10:47:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.444954+0000", 2023-10-09T10:47:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.044117351999999999 2023-10-09T10:47:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.445001+0000", 2023-10-09T10:47:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7105999999999999e-05 2023-10-09T10:47:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.445028+0000", 2023-10-09T10:47:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7172e-05 2023-10-09T10:47:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106423 6.9 6:930bb25f:::benchmark_data_smithi125_244951_object106422:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:47:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.239301+0000", 2023-10-09T10:47:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 77.809193559999997, 2023-10-09T10:47:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.46007959199999998, 2023-10-09T10:47:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:47:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106423 2023-10-09T10:47:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239301+0000", 2023-10-09T10:47:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239301+0000", 2023-10-09T10:47:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239299+0000", 2023-10-09T10:47:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:47:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239340+0000", 2023-10-09T10:47:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0917000000000001e-05 2023-10-09T10:47:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239360+0000", 2023-10-09T10:47:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0649000000000002e-05 2023-10-09T10:47:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239371+0000", 2023-10-09T10:47:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0244000000000001e-05 2023-10-09T10:47:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.244269+0000", 2023-10-09T10:47:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0048985590000000002 2023-10-09T10:47:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.244451+0000", 2023-10-09T10:47:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000181526 2023-10-09T10:47:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:47:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.244646+0000", 2023-10-09T10:47:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000195133 2023-10-09T10:47:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.271437+0000", 2023-10-09T10:47:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.026791150999999999 2023-10-09T10:47:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274828+0000", 2023-10-09T10:47:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003391088 2023-10-09T10:47:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274837+0000", 2023-10-09T10:47:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6899999999999998e-06 2023-10-09T10:47:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.699260+0000", 2023-10-09T10:47:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.42442315200000003 2023-10-09T10:47:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.699380+0000", 2023-10-09T10:47:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120189 2023-10-09T10:47:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106425 6.9 6:966db102:::benchmark_data_smithi125_244951_object106424:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:47:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.239425+0000", 2023-10-09T10:47:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 77.809068886999995, 2023-10-09T10:47:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.46673965099999998, 2023-10-09T10:47:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:47:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106425 2023-10-09T10:47:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239425+0000", 2023-10-09T10:47:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239425+0000", 2023-10-09T10:47:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239425+0000", 2023-10-09T10:47:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239473+0000", 2023-10-09T10:47:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8177000000000003e-05 2023-10-09T10:47:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239480+0000", 2023-10-09T10:47:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0559999999999999e-06 2023-10-09T10:47:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239485+0000", 2023-10-09T10:47:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.242e-06 2023-10-09T10:47:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.249603+0000", 2023-10-09T10:47:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010118121000000001 2023-10-09T10:47:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.249760+0000", 2023-10-09T10:47:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000156331 2023-10-09T10:47:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:47:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.249953+0000", 2023-10-09T10:47:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019313500000000001 2023-10-09T10:47:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274855+0000", 2023-10-09T10:47:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.024902173999999999 2023-10-09T10:47:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274860+0000", 2023-10-09T10:47:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8380000000000001e-06 2023-10-09T10:47:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.700402+0000", 2023-10-09T10:47:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.42554259900000002 2023-10-09T10:47:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.701238+0000", 2023-10-09T10:47:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000835825 2023-10-09T10:47:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.706109+0000", 2023-10-09T10:47:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0048705440000000001 2023-10-09T10:47:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.706165+0000", 2023-10-09T10:47:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6017999999999998e-05 2023-10-09T10:47:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106422 6.19 6:9914323d:::benchmark_data_smithi125_244951_object106421:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T10:47:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.601105+0000", 2023-10-09T10:47:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 77.447389176000001, 2023-10-09T10:47:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0560851339999999, 2023-10-09T10:47:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:47:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106422 2023-10-09T10:47:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601105+0000", 2023-10-09T10:47:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601105+0000", 2023-10-09T10:47:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601103+0000", 2023-10-09T10:47:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:47:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601130+0000", 2023-10-09T10:47:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7222000000000001e-05 2023-10-09T10:47:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601147+0000", 2023-10-09T10:47:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6600999999999999e-05 2023-10-09T10:47:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601162+0000", 2023-10-09T10:47:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4929000000000001e-05 2023-10-09T10:47:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.643541+0000", 2023-10-09T10:47:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.042379130000000001 2023-10-09T10:47:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.643555+0000", 2023-10-09T10:47:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3465000000000001e-05 2023-10-09T10:47:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:03.657089+0000", 2023-10-09T10:47:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0135346009999999 2023-10-09T10:47:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:03.657190+0000", 2023-10-09T10:47:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000100945 2023-10-09T10:47:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106426 6.19 6:9f6a94cf:::benchmark_data_smithi125_244951_object106425:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T10:47:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.601170+0000", 2023-10-09T10:47:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 77.447323667000006, 2023-10-09T10:47:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.057006321, 2023-10-09T10:47:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:47:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106426 2023-10-09T10:47:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601170+0000", 2023-10-09T10:47:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601170+0000", 2023-10-09T10:47:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601170+0000", 2023-10-09T10:47:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601191+0000", 2023-10-09T10:47:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0746000000000001e-05 2023-10-09T10:47:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601199+0000", 2023-10-09T10:47:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8019999999999998e-06 2023-10-09T10:47:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601222+0000", 2023-10-09T10:47:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2801000000000001e-05 2023-10-09T10:47:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.643580+0000", 2023-10-09T10:47:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.042357842 2023-10-09T10:47:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.643589+0000", 2023-10-09T10:47:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1749999999999994e-06 2023-10-09T10:47:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:03.658076+0000", 2023-10-09T10:47:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0144863019999999 2023-10-09T10:47:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:03.658177+0000", 2023-10-09T10:47:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000101201 2023-10-09T10:47:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:115958 6.f 6:fa100500:::benchmark_data_smithi125_244951_object9515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-10-09T10:47:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:20.075222+0000", 2023-10-09T10:47:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 59.973272319000003, 2023-10-09T10:47:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.084394099, 2023-10-09T10:47:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115958 2023-10-09T10:47:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075222+0000", 2023-10-09T10:47:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075222+0000", 2023-10-09T10:47:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075220+0000", 2023-10-09T10:47:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:47:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075229+0000", 2023-10-09T10:47:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7949999999999996e-06 2023-10-09T10:47:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075244+0000", 2023-10-09T10:47:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5364000000000001e-05 2023-10-09T10:47:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075254+0000", 2023-10-09T10:47:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4609999999999997e-06 2023-10-09T10:47:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.076549+0000", 2023-10-09T10:47:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012954900000000001 2023-10-09T10:47:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.076690+0000", 2023-10-09T10:47:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014043200000000001 2023-10-09T10:47:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:47:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.076818+0000", 2023-10-09T10:47:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012810400000000001 2023-10-09T10:47:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.085804+0000", 2023-10-09T10:47:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0089861170000000001 2023-10-09T10:47:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.087330+0000", 2023-10-09T10:47:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001526108 2023-10-09T10:47:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159461+0000", 2023-10-09T10:47:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.072131120000000007 2023-10-09T10:47:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159562+0000", 2023-10-09T10:47:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000101475 2023-10-09T10:47:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159616+0000", 2023-10-09T10:47:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3387000000000002e-05 2023-10-09T10:47:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:115960 6.f 6:f825ff14:::benchmark_data_smithi125_244951_object9517:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-10-09T10:47:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:20.081466+0000", 2023-10-09T10:47:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 59.967027856999998, 2023-10-09T10:47:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.078315183999999996, 2023-10-09T10:47:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115960 2023-10-09T10:47:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081466+0000", 2023-10-09T10:47:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081466+0000", 2023-10-09T10:47:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081464+0000", 2023-10-09T10:47:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:47:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081473+0000", 2023-10-09T10:47:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3960000000000006e-06 2023-10-09T10:47:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081490+0000", 2023-10-09T10:47:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7730000000000001e-05 2023-10-09T10:47:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081502+0000", 2023-10-09T10:47:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1612e-05 2023-10-09T10:47:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.082766+0000", 2023-10-09T10:47:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012642350000000001 2023-10-09T10:47:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.082875+0000", 2023-10-09T10:47:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010849999999999999 2023-10-09T10:47:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:47:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.082969+0000", 2023-10-09T10:47:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4368000000000002e-05 2023-10-09T10:47:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.091397+0000", 2023-10-09T10:47:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0084276379999999994 2023-10-09T10:47:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.105493+0000", 2023-10-09T10:47:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014096630000000001 2023-10-09T10:47:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159659+0000", 2023-10-09T10:47:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.05416588 2023-10-09T10:47:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159740+0000", 2023-10-09T10:47:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0735999999999994e-05 2023-10-09T10:47:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159781+0000", 2023-10-09T10:47:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1458999999999997e-05 2023-10-09T10:47:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:115962 6.f 6:f37830ff:::benchmark_data_smithi125_244951_object9519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-10-09T10:47:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:20.095380+0000", 2023-10-09T10:47:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 59.953114118000002, 2023-10-09T10:47:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.064565055999999996, 2023-10-09T10:47:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115962 2023-10-09T10:47:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095380+0000", 2023-10-09T10:47:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095380+0000", 2023-10-09T10:47:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095377+0000", 2023-10-09T10:47:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T10:47:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095391+0000", 2023-10-09T10:47:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3073e-05 2023-10-09T10:47:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095412+0000", 2023-10-09T10:47:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1406000000000001e-05 2023-10-09T10:47:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095423+0000", 2023-10-09T10:47:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0764000000000001e-05 2023-10-09T10:47:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.097612+0000", 2023-10-09T10:47:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021888530000000002 2023-10-09T10:47:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.097704+0000", 2023-10-09T10:47:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2219000000000006e-05 2023-10-09T10:47:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:47:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.097775+0000", 2023-10-09T10:47:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1330999999999999e-05 2023-10-09T10:47:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.101037+0000", 2023-10-09T10:47:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032613899999999999 2023-10-09T10:47:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.122053+0000", 2023-10-09T10:47:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.021016743000000001 2023-10-09T10:47:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159823+0000", 2023-10-09T10:47:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.037769619999999997 2023-10-09T10:47:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159902+0000", 2023-10-09T10:47:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9357999999999995e-05 2023-10-09T10:47:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159945+0000", 2023-10-09T10:47:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2772999999999999e-05 2023-10-09T10:47:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:115966 6.f 6:f2b073c2:::benchmark_data_smithi125_244951_object9523:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-10-09T10:47:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:20.095468+0000", 2023-10-09T10:47:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 59.953026311999999, 2023-10-09T10:47:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.067415613999999999, 2023-10-09T10:47:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115966 2023-10-09T10:47:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095468+0000", 2023-10-09T10:47:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095468+0000", 2023-10-09T10:47:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095468+0000", 2023-10-09T10:47:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095470+0000", 2023-10-09T10:47:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2520000000000002e-06 2023-10-09T10:47:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095475+0000", 2023-10-09T10:47:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8919999999999999e-06 2023-10-09T10:47:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095479+0000", 2023-10-09T10:47:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.476e-06 2023-10-09T10:47:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.101070+0000", 2023-10-09T10:47:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0055904309999999999 2023-10-09T10:47:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.101187+0000", 2023-10-09T10:47:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000117111 2023-10-09T10:47:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:47:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.101301+0000", 2023-10-09T10:47:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000113842 2023-10-09T10:47:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.111354+0000", 2023-10-09T10:47:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010053945 2023-10-09T10:47:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.122063+0000", 2023-10-09T10:47:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010709000999999999 2023-10-09T10:47:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159984+0000", 2023-10-09T10:47:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.037920483999999997 2023-10-09T10:47:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.162830+0000", 2023-10-09T10:47:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002845888 2023-10-09T10:47:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.162883+0000", 2023-10-09T10:47:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3554999999999998e-05 2023-10-09T10:47:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:20.226 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:47:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:19 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:47:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:19 smithi155 ceph-mon[184854]: osdmap e170: 8 total, 8 up, 7 in 2023-10-09T10:47:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:19 smithi155 ceph-mon[184854]: failed to encode map e170 with expected crc 2023-10-09T10:47:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:19 smithi155 ceph-mon[184854]: failed to encode map e170 with expected crc 2023-10-09T10:47:20.227 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:19 smithi155 ceph-mon[184854]: failed to encode map e170 with expected crc 2023-10-09T10:47:20.228 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:19 smithi155 ceph-mon[184854]: failed to encode map e170 with expected crc 2023-10-09T10:47:20.228 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:19 smithi155 ceph-mon[184854]: failed to encode map e170 with expected crc 2023-10-09T10:47:20.228 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:19 smithi155 ceph-mon[184854]: pgmap v354: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 346 op/s; 34009/211749 objects degraded (16.061%); 9899/211749 objects misplaced (4.675%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T10:47:20.228 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:20.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:47:20.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[195507]: osdmap e170: 8 total, 8 up, 7 in 2023-10-09T10:47:20.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[195507]: failed to encode map e170 with expected crc 2023-10-09T10:47:20.229 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[195507]: failed to encode map e170 with expected crc 2023-10-09T10:47:20.229 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[195507]: failed to encode map e170 with expected crc 2023-10-09T10:47:20.229 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[195507]: failed to encode map e170 with expected crc 2023-10-09T10:47:20.229 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[195507]: failed to encode map e170 with expected crc 2023-10-09T10:47:20.229 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[195507]: pgmap v354: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 346 op/s; 34009/211749 objects degraded (16.061%); 9899/211749 objects misplaced (4.675%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T10:47:20.229 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:47:20.229 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[200197]: osdmap e170: 8 total, 8 up, 7 in 2023-10-09T10:47:20.229 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[200197]: failed to encode map e170 with expected crc 2023-10-09T10:47:20.230 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[200197]: failed to encode map e170 with expected crc 2023-10-09T10:47:20.230 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[200197]: failed to encode map e170 with expected crc 2023-10-09T10:47:20.230 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[200197]: failed to encode map e170 with expected crc 2023-10-09T10:47:20.230 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[200197]: failed to encode map e170 with expected crc 2023-10-09T10:47:20.230 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:19 smithi125 ceph-mon[200197]: pgmap v354: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 346 op/s; 34009/211749 objects degraded (16.061%); 9899/211749 objects misplaced (4.675%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T10:47:20.374 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:20.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:20.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:20.478 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:20.526 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:20.818 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:20.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:20.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:20.910 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:47:20.925 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:21.003 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:21.040 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:21.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:21.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:21.148 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:21.193 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:21.526 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:21.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:21.533 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:47:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:47:21] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.43.0" 2023-10-09T10:47:21.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:21.638 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:21.671 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:22.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:22 smithi125 ceph-mon[195507]: pgmap v355: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 34009/206577 objects degraded (16.463%); 8974/206577 objects misplaced (4.344%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T10:47:22.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:22 smithi125 ceph-mon[200197]: pgmap v355: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 34009/206577 objects degraded (16.463%); 8974/206577 objects misplaced (4.344%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T10:47:22.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:22 smithi155 ceph-mon[184854]: pgmap v355: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 34009/206577 objects degraded (16.463%); 8974/206577 objects misplaced (4.344%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T10:47:22.962 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:22.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:22.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:23.072 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:23.114 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:23.179 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:23.181 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:23.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:23.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:23.186 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:23.249 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:23.284 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:23.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:23 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 34009/206577 objects degraded (16.463%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:23.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:47:23.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:47:23.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:23 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 34009/206577 objects degraded (16.463%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:47:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:47:23.331 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:23.359 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:23.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:23.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:23.465 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:23 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 34009/206577 objects degraded (16.463%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:47:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:47:23.521 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:23.543 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:23.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:23.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:23.647 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:23.690 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:23.719 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:23.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:23.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:23.823 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:23.883 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:24.431 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:24 smithi125 ceph-mon[200197]: pgmap v356: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 394 op/s; 34009/205308 objects degraded (16.565%); 8917/205308 objects misplaced (4.343%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T10:47:24.431 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:47:24.432 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:47:24.432 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:47:24.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:24 smithi125 ceph-mon[195507]: pgmap v356: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 394 op/s; 34009/205308 objects degraded (16.565%); 8917/205308 objects misplaced (4.343%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T10:47:24.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:47:24.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:47:24.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:47:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:24 smithi155 ceph-mon[184854]: pgmap v356: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 394 op/s; 34009/205308 objects degraded (16.565%); 8917/205308 objects misplaced (4.343%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T10:47:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:47:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:47:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:47:24.581 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:24.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:24.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:24.686 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:24.718 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:24.752 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:24.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:24.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:24.857 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:24.907 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:25.081 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:25.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:25.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:25.188 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:25.188 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:47:25.221 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:25.254 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:25.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:25.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:25.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1591643955' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:25.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:25 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:25.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1591643955' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:25.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:25 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:25.358 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:25.404 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1591643955' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:25 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:25.882 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:25.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:25.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:25.990 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:26.035 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:47:26.035 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:47:26.035 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.035 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:147277 6.11 6.dfdbecd1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T10:47:26.035 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:25.973945+0000", 2023-10-09T10:47:26.035 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.059445892, 2023-10-09T10:47:26.035 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.059467429000000002, 2023-10-09T10:47:26.036 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.036 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:47:26.036 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:26.036 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:26.036 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:26.036 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 147277 2023-10-09T10:47:26.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.036 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.036 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.036 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.973945+0000", 2023-10-09T10:47:26.037 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.037 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.037 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.037 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.037 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.973945+0000", 2023-10-09T10:47:26.037 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.037 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.037 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.037 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.037 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.973945+0000", 2023-10-09T10:47:26.037 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:26.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.973947+0000", 2023-10-09T10:47:26.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2680000000000001e-06 2023-10-09T10:47:26.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.973953+0000", 2023-10-09T10:47:26.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3850000000000001e-06 2023-10-09T10:47:26.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.039 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.039 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.973957+0000", 2023-10-09T10:47:26.039 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6519999999999997e-06 2023-10-09T10:47:26.039 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.039 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.039 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.039 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPushReply(6.11 171/143 [PushReplyOp(6:8f9b3312:::benchmark_data_smithi125_244951_object40609:head)])", 2023-10-09T10:47:26.040 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:25.976740+0000", 2023-10-09T10:47:26.040 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.056651343, 2023-10-09T10:47:26.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.056713842, 2023-10-09T10:47:26.040 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.040 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:47:26.040 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.040 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.040 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.976740+0000", 2023-10-09T10:47:26.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.040 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.976740+0000", 2023-10-09T10:47:26.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.976739+0000", 2023-10-09T10:47:26.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:47:26.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.042 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.976747+0000", 2023-10-09T10:47:26.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2369999999999992e-06 2023-10-09T10:47:26.042 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.042 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.042 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.042 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.976759+0000", 2023-10-09T10:47:26.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1566000000000001e-05 2023-10-09T10:47:26.042 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.042 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.042 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.042 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.976768+0000", 2023-10-09T10:47:26.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7040000000000006e-06 2023-10-09T10:47:26.043 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.043 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.043 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.043 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.043 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.043 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPushReply(6.11 171/143 [PushReplyOp(6:8b74cc26:::benchmark_data_smithi125_244951_object40635:head)])", 2023-10-09T10:47:26.043 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:25.979463+0000", 2023-10-09T10:47:26.043 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.053927937000000002, 2023-10-09T10:47:26.043 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.054019286, 2023-10-09T10:47:26.043 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.043 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:47:26.043 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.044 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.979463+0000", 2023-10-09T10:47:26.044 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.044 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.979463+0000", 2023-10-09T10:47:26.044 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.979463+0000", 2023-10-09T10:47:26.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:26.045 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.045 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.979467+0000", 2023-10-09T10:47:26.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9190000000000001e-06 2023-10-09T10:47:26.045 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.045 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.979478+0000", 2023-10-09T10:47:26.046 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1331e-05 2023-10-09T10:47:26.046 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.046 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.046 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.046 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.979484+0000", 2023-10-09T10:47:26.046 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2160000000000003e-06 2023-10-09T10:47:26.046 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.046 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.046 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.046 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.046 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.046 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44570.0:147296 6.19 e171/154)", 2023-10-09T10:47:26.046 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:26.024371+0000", 2023-10-09T10:47:26.047 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0090198989999999996, 2023-10-09T10:47:26.047 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0091389669999999996, 2023-10-09T10:47:26.047 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.047 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:47:26.047 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.047 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.047 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.024371+0000", 2023-10-09T10:47:26.047 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.047 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.047 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.024371+0000", 2023-10-09T10:47:26.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.048 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.024371+0000", 2023-10-09T10:47:26.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:47:26.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.048 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.024378+0000", 2023-10-09T10:47:26.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8580000000000002e-06 2023-10-09T10:47:26.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.024389+0000", 2023-10-09T10:47:26.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1249e-05 2023-10-09T10:47:26.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.024396+0000", 2023-10-09T10:47:26.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1409999999999999e-06 2023-10-09T10:47:26.049 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.049 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.049 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.050 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.44570.0:146993 6.11 e171/143)", 2023-10-09T10:47:26.050 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:26.029048+0000", 2023-10-09T10:47:26.050 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0043431199999999998, 2023-10-09T10:47:26.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0044893329999999999, 2023-10-09T10:47:26.050 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.050 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:47:26.050 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.029048+0000", 2023-10-09T10:47:26.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.029048+0000", 2023-10-09T10:47:26.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.029048+0000", 2023-10-09T10:47:26.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:26.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.029052+0000", 2023-10-09T10:47:26.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3939999999999998e-06 2023-10-09T10:47:26.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.029068+0000", 2023-10-09T10:47:26.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6558e-05 2023-10-09T10:47:26.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.029077+0000", 2023-10-09T10:47:26.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3529999999999995e-06 2023-10-09T10:47:26.052 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.053 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.053 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.053 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.053 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPushReply(6.11 171/143 [PushReplyOp(6:8b74cc26:::benchmark_data_smithi125_244951_object40635:head)])", 2023-10-09T10:47:26.053 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:25.983685+0000", 2023-10-09T10:47:26.053 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.049706081999999999, 2023-10-09T10:47:26.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.049881078000000002, 2023-10-09T10:47:26.053 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.053 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:47:26.053 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.054 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.054 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.983685+0000", 2023-10-09T10:47:26.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.054 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.054 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.054 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.054 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.983685+0000", 2023-10-09T10:47:26.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.054 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.054 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.054 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.054 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.983685+0000", 2023-10-09T10:47:26.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:26.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.983690+0000", 2023-10-09T10:47:26.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0889999999999999e-06 2023-10-09T10:47:26.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.983699+0000", 2023-10-09T10:47:26.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4190000000000008e-06 2023-10-09T10:47:26.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.056 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.983705+0000", 2023-10-09T10:47:26.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9900000000000002e-06 2023-10-09T10:47:26.056 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.056 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.056 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.056 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.44570.0:146993 6.11 e171/143)", 2023-10-09T10:47:26.056 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:26.029783+0000", 2023-10-09T10:47:26.056 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0036084770000000001, 2023-10-09T10:47:26.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003809869, 2023-10-09T10:47:26.057 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.057 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:47:26.057 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.029783+0000", 2023-10-09T10:47:26.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.029783+0000", 2023-10-09T10:47:26.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.029782+0000", 2023-10-09T10:47:26.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:26.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.029786+0000", 2023-10-09T10:47:26.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0640000000000004e-06 2023-10-09T10:47:26.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.059 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.059 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.059 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.029797+0000", 2023-10-09T10:47:26.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1028e-05 2023-10-09T10:47:26.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.059 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.059 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.059 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.029805+0000", 2023-10-09T10:47:26.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.481e-06 2023-10-09T10:47:26.059 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.059 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.059 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.060 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:147305 6.b 6:d9b05049:::benchmark_data_smithi125_244951_object40862:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T10:47:26.060 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:26.030710+0000", 2023-10-09T10:47:26.060 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0026808510000000002, 2023-10-09T10:47:26.060 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029128359999999998, 2023-10-09T10:47:26.060 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.060 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:47:26.060 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:26.060 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:26.060 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:26.060 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 147305 2023-10-09T10:47:26.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.060 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.030710+0000", 2023-10-09T10:47:26.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.030710+0000", 2023-10-09T10:47:26.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.030709+0000", 2023-10-09T10:47:26.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:26.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.030717+0000", 2023-10-09T10:47:26.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0760000000000003e-06 2023-10-09T10:47:26.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.030735+0000", 2023-10-09T10:47:26.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7689999999999998e-05 2023-10-09T10:47:26.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.030744+0000", 2023-10-09T10:47:26.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2199999999999998e-06 2023-10-09T10:47:26.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:26.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.030797+0000", 2023-10-09T10:47:26.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3179e-05 2023-10-09T10:47:26.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.064 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:26.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.030927+0000", 2023-10-09T10:47:26.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013012500000000001 2023-10-09T10:47:26.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.064 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,7", 2023-10-09T10:47:26.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.031048+0000", 2023-10-09T10:47:26.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012071200000000001 2023-10-09T10:47:26.064 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.064 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.064 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.065 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:147307 6.11 6.fe3076f1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T10:47:26.065 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:26.032386+0000", 2023-10-09T10:47:26.065 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.001005213, 2023-10-09T10:47:26.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001280448, 2023-10-09T10:47:26.065 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.065 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:47:26.065 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:26.065 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:26.065 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:26.065 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 147307 2023-10-09T10:47:26.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.066 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.032386+0000", 2023-10-09T10:47:26.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.032386+0000", 2023-10-09T10:47:26.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.032385+0000", 2023-10-09T10:47:26.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:47:26.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.067 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.032392+0000", 2023-10-09T10:47:26.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7299999999999999e-06 2023-10-09T10:47:26.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.067 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.032405+0000", 2023-10-09T10:47:26.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2994e-05 2023-10-09T10:47:26.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.068 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.032413+0000", 2023-10-09T10:47:26.068 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2570000000000007e-06 2023-10-09T10:47:26.068 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.068 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.068 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.068 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.068 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44570.0:147279 6.9 e171/154)", 2023-10-09T10:47:26.068 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:25.986284+0000", 2023-10-09T10:47:26.069 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.047107674000000002, 2023-10-09T10:47:26.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.047414650000000003, 2023-10-09T10:47:26.069 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.069 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:47:26.069 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.069 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.986284+0000", 2023-10-09T10:47:26.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.070 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.070 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.986284+0000", 2023-10-09T10:47:26.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.070 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.070 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.070 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.986283+0000", 2023-10-09T10:47:26.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:26.070 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.070 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.070 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.986289+0000", 2023-10-09T10:47:26.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0909999999999998e-06 2023-10-09T10:47:26.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.071 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.986298+0000", 2023-10-09T10:47:26.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4069999999999993e-06 2023-10-09T10:47:26.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.071 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.986303+0000", 2023-10-09T10:47:26.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5160000000000002e-06 2023-10-09T10:47:26.071 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.071 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.071 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.072 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:146993 6.11 6:8bb9f8ce:::benchmark_data_smithi125_244951_object40550:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T10:47:26.072 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:25.464991+0000", 2023-10-09T10:47:26.072 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.56840082700000005, 2023-10-09T10:47:26.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.56873751299999997, 2023-10-09T10:47:26.072 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.072 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:47:26.072 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:26.072 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:26.072 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:26.072 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 146993 2023-10-09T10:47:26.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.073 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.464991+0000", 2023-10-09T10:47:26.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.464991+0000", 2023-10-09T10:47:26.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.464989+0000", 2023-10-09T10:47:26.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:26.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.464996+0000", 2023-10-09T10:47:26.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2810000000000003e-06 2023-10-09T10:47:26.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.465008+0000", 2023-10-09T10:47:26.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1131999999999999e-05 2023-10-09T10:47:26.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.465015+0000", 2023-10-09T10:47:26.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5220000000000001e-06 2023-10-09T10:47:26.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:26.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.517426+0000", 2023-10-09T10:47:26.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.052410845999999997 2023-10-09T10:47:26.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.076 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:47:26.076 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.519834+0000", 2023-10-09T10:47:26.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002408005 2023-10-09T10:47:26.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.076 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:26.076 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.541364+0000", 2023-10-09T10:47:26.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.021530128999999999 2023-10-09T10:47:26.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.076 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:26.076 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.542832+0000", 2023-10-09T10:47:26.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014682689999999999 2023-10-09T10:47:26.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:26.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.018466+0000", 2023-10-09T10:47:26.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.47563410499999997 2023-10-09T10:47:26.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:26.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.020703+0000", 2023-10-09T10:47:26.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022363249999999999 2023-10-09T10:47:26.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:26.078 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.025375+0000", 2023-10-09T10:47:26.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046725450000000002 2023-10-09T10:47:26.078 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.078 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.078 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.078 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:147052 6.11 6:8f9b3312:::benchmark_data_smithi125_244951_object40609:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T10:47:26.078 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:25.582394+0000", 2023-10-09T10:47:26.078 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.450997706, 2023-10-09T10:47:26.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.45139069599999998, 2023-10-09T10:47:26.079 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.079 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "delayed", 2023-10-09T10:47:26.079 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:26.079 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:26.079 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:26.079 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 147052 2023-10-09T10:47:26.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.079 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.582394+0000", 2023-10-09T10:47:26.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.080 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.080 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.582394+0000", 2023-10-09T10:47:26.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.080 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.080 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.582392+0000", 2023-10-09T10:47:26.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:26.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.081 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.081 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.582399+0000", 2023-10-09T10:47:26.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1199999999999999e-06 2023-10-09T10:47:26.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.081 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.081 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.081 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.582411+0000", 2023-10-09T10:47:26.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2870999999999999e-05 2023-10-09T10:47:26.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.081 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.081 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.081 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.582418+0000", 2023-10-09T10:47:26.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0260000000000002e-06 2023-10-09T10:47:26.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:26.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.592569+0000", 2023-10-09T10:47:26.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.01015072 2023-10-09T10:47:26.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:47:26.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.595545+0000", 2023-10-09T10:47:26.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029762629999999998 2023-10-09T10:47:26.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:26.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.631431+0000", 2023-10-09T10:47:26.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.035885591000000001 2023-10-09T10:47:26.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.083 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:26.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.632697+0000", 2023-10-09T10:47:26.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001266334 2023-10-09T10:47:26.083 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.083 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.083 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.084 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:147160 6.11 6:89397033:::benchmark_data_smithi125_244951_object40717:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T10:47:26.084 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:25.767091+0000", 2023-10-09T10:47:26.084 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.26630030599999999, 2023-10-09T10:47:26.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.26674008900000001, 2023-10-09T10:47:26.084 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.084 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "delayed", 2023-10-09T10:47:26.084 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:26.084 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:26.084 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:26.084 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 147160 2023-10-09T10:47:26.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.085 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.767091+0000", 2023-10-09T10:47:26.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.767091+0000", 2023-10-09T10:47:26.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.086 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.767090+0000", 2023-10-09T10:47:26.086 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:26.086 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.086 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.086 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.086 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.767096+0000", 2023-10-09T10:47:26.086 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5699999999999998e-06 2023-10-09T10:47:26.086 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.086 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.086 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.767108+0000", 2023-10-09T10:47:26.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2119999999999999e-05 2023-10-09T10:47:26.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.767117+0000", 2023-10-09T10:47:26.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1599999999999998e-06 2023-10-09T10:47:26.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:26.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.828044+0000", 2023-10-09T10:47:26.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.060927282999999999 2023-10-09T10:47:26.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.088 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:26.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.829516+0000", 2023-10-09T10:47:26.088 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00147215 2023-10-09T10:47:26.088 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.088 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.088 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.088 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:147224 6.11 6:8ea6c692:::benchmark_data_smithi125_244951_object40781:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T10:47:26.088 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:25.869690+0000", 2023-10-09T10:47:26.089 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.16370116200000001, 2023-10-09T10:47:26.089 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.16418086600000001, 2023-10-09T10:47:26.089 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.089 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "delayed", 2023-10-09T10:47:26.089 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:26.089 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:26.089 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:26.089 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 147224 2023-10-09T10:47:26.090 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.090 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.090 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.090 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.090 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.869690+0000", 2023-10-09T10:47:26.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.090 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.090 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.090 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.090 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.869690+0000", 2023-10-09T10:47:26.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.091 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.091 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.869689+0000", 2023-10-09T10:47:26.091 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:47:26.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.091 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.091 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.869695+0000", 2023-10-09T10:47:26.091 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0560000000000001e-06 2023-10-09T10:47:26.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.091 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.092 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.869708+0000", 2023-10-09T10:47:26.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2872e-05 2023-10-09T10:47:26.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.092 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.092 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.869715+0000", 2023-10-09T10:47:26.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2429999999999999e-06 2023-10-09T10:47:26.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.092 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:26.092 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.947013+0000", 2023-10-09T10:47:26.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.077297563 2023-10-09T10:47:26.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:47:26.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.949683+0000", 2023-10-09T10:47:26.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026705800000000001 2023-10-09T10:47:26.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:26.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.986199+0000", 2023-10-09T10:47:26.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.036515678000000003 2023-10-09T10:47:26.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:26.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.987409+0000", 2023-10-09T10:47:26.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012104119999999999 2023-10-09T10:47:26.094 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.094 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.094 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.094 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:147078 6.11 6:8b74cc26:::benchmark_data_smithi125_244951_object40635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T10:47:26.094 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:25.622818+0000", 2023-10-09T10:47:26.094 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.41057291699999998, 2023-10-09T10:47:26.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.41110322500000002, 2023-10-09T10:47:26.094 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.094 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "delayed", 2023-10-09T10:47:26.095 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:26.095 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:26.095 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:26.095 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 147078 2023-10-09T10:47:26.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.095 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.622818+0000", 2023-10-09T10:47:26.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.622818+0000", 2023-10-09T10:47:26.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.622817+0000", 2023-10-09T10:47:26.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:26.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.622824+0000", 2023-10-09T10:47:26.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6649999999999997e-06 2023-10-09T10:47:26.097 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.097 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.097 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.097 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.622835+0000", 2023-10-09T10:47:26.097 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1705000000000001e-05 2023-10-09T10:47:26.097 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.097 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.097 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.097 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.622843+0000", 2023-10-09T10:47:26.097 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2089999999999999e-06 2023-10-09T10:47:26.097 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.097 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:26.098 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.641384+0000", 2023-10-09T10:47:26.098 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018541150999999999 2023-10-09T10:47:26.098 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.098 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:47:26.098 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.643876+0000", 2023-10-09T10:47:26.098 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024920789999999999 2023-10-09T10:47:26.098 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.098 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:26.099 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.662390+0000", 2023-10-09T10:47:26.099 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018514065999999999 2023-10-09T10:47:26.099 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.099 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.099 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:26.099 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.663740+0000", 2023-10-09T10:47:26.099 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013497800000000001 2023-10-09T10:47:26.099 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.099 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.099 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.099 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.099 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.099 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:147226 6.11 6:8cc5678a:::benchmark_data_smithi125_244951_object40783:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T10:47:26.100 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:25.874502+0000", 2023-10-09T10:47:26.100 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.15888982099999999, 2023-10-09T10:47:26.100 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.159466618, 2023-10-09T10:47:26.100 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.100 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "delayed", 2023-10-09T10:47:26.100 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:26.100 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:26.100 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:26.100 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 147226 2023-10-09T10:47:26.100 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.100 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.874502+0000", 2023-10-09T10:47:26.101 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.101 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.101 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.874502+0000", 2023-10-09T10:47:26.101 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.101 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.101 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.874500+0000", 2023-10-09T10:47:26.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:26.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.102 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.102 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.102 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.874507+0000", 2023-10-09T10:47:26.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.148e-06 2023-10-09T10:47:26.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.102 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.102 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.102 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.874520+0000", 2023-10-09T10:47:26.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2626000000000001e-05 2023-10-09T10:47:26.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.874529+0000", 2023-10-09T10:47:26.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9970000000000004e-06 2023-10-09T10:47:26.103 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:26.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.949956+0000", 2023-10-09T10:47:26.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.075426644000000001 2023-10-09T10:47:26.103 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:47:26.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.955891+0000", 2023-10-09T10:47:26.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0059354860000000002 2023-10-09T10:47:26.104 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.104 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.104 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:26.104 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.998195+0000", 2023-10-09T10:47:26.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.042303495000000003 2023-10-09T10:47:26.104 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.104 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.104 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:26.104 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.999375+0000", 2023-10-09T10:47:26.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001179998 2023-10-09T10:47:26.104 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.104 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.105 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.105 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.105 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.105 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPushReply(6.11 171/143 [PushReplyOp(6:8f9b3312:::benchmark_data_smithi125_244951_object40609:head)])", 2023-10-09T10:47:26.105 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:25.973776+0000", 2023-10-09T10:47:26.105 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.059615686000000001, 2023-10-09T10:47:26.105 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.060239932000000003, 2023-10-09T10:47:26.105 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:26.105 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:47:26.105 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:26.105 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.105 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:26.106 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.973776+0000", 2023-10-09T10:47:26.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.106 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.106 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.106 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:26.106 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.973776+0000", 2023-10-09T10:47:26.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:26.106 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.106 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.106 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:26.106 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.973775+0000", 2023-10-09T10:47:26.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:47:26.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:26.107 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.973781+0000", 2023-10-09T10:47:26.107 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1160000000000004e-06 2023-10-09T10:47:26.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:26.107 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.973798+0000", 2023-10-09T10:47:26.107 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6674999999999999e-05 2023-10-09T10:47:26.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:26.108 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.973806+0000", 2023-10-09T10:47:26.108 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7949999999999996e-06 2023-10-09T10:47:26.108 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:26.108 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:26.108 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:26.108 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:26.029978+0000", 2023-10-09T10:47:26.108 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.056171746000000002 2023-10-09T10:47:26.108 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.108 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:26.108 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.108 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:26.108 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:47:26.109 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 17 2023-10-09T10:47:26.109 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:47:26.109 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:26.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:26 smithi125 ceph-mon[195507]: pgmap v357: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 34009/201936 objects degraded (16.841%); 8598/201936 objects misplaced (4.258%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T10:47:26.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:26 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:26.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:26 smithi125 ceph-mon[195507]: failed to encode map e171 with expected crc 2023-10-09T10:47:26.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:26 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:47:26.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:26 smithi125 ceph-mon[195507]: osdmap e171: 8 total, 8 up, 7 in 2023-10-09T10:47:26.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:26 smithi125 ceph-mon[195507]: failed to encode map e171 with expected crc 2023-10-09T10:47:26.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:26 smithi125 ceph-mon[195507]: failed to encode map e171 with expected crc 2023-10-09T10:47:26.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:26 smithi125 ceph-mon[200197]: pgmap v357: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 34009/201936 objects degraded (16.841%); 8598/201936 objects misplaced (4.258%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T10:47:26.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:26 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:26.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:26 smithi125 ceph-mon[200197]: failed to encode map e171 with expected crc 2023-10-09T10:47:26.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:26 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:47:26.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:26 smithi125 ceph-mon[200197]: osdmap e171: 8 total, 8 up, 7 in 2023-10-09T10:47:26.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:26 smithi125 ceph-mon[200197]: failed to encode map e171 with expected crc 2023-10-09T10:47:26.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:26 smithi125 ceph-mon[200197]: failed to encode map e171 with expected crc 2023-10-09T10:47:26.410 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:26.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:26.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:26 smithi155 ceph-mon[184854]: pgmap v357: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 34009/201936 objects degraded (16.841%); 8598/201936 objects misplaced (4.258%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T10:47:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:26 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:26 smithi155 ceph-mon[184854]: failed to encode map e171 with expected crc 2023-10-09T10:47:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:26 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:47:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:26 smithi155 ceph-mon[184854]: osdmap e171: 8 total, 8 up, 7 in 2023-10-09T10:47:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:26 smithi155 ceph-mon[184854]: failed to encode map e171 with expected crc 2023-10-09T10:47:26.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:26 smithi155 ceph-mon[184854]: failed to encode map e171 with expected crc 2023-10-09T10:47:26.515 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:26.560 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:26.639 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:26.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:26.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:26.748 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:26.814 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:26.904 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:47:26.912 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:47:27.118 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:47:27.140 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:47:27.140 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:47:27.140 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:47:27.140 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:47:27.153 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:27.158 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:47:27.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:27.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:27.186 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:47:27.223 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:47:27.252 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:47:27.253 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:47:27.253 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:47:27.253 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:47:27.253 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:47:27.253 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:47:27.254 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:47:27.254 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:47:27.254 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:47:27.263 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:27.283 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:27 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3012321160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:27.284 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:27 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3012321160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:27.305 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:27.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:27 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3012321160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:27.891 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":358,"stamp":"2023-10-09T10:47:25.178851+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4410283960,"num_objects":67312,"num_object_clones":0,"num_object_copies":201936,"num_objects_missing_on_primary":1503,"num_objects_missing":1503,"num_objects_degraded":34009,"num_objects_misplaced":8598,"num_objects_unfound":0,"num_objects_dirty":67312,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":137220,"num_write_kb":6260974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19048,"num_bytes_recovered":1215040258,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":157201,"ondisk_log_size":157201,"up":357,"acting":345,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13385656,"kb_used_data":11777664,"kb_used_omap":44,"kb_used_meta":1607763,"kb_avail":642830408,"statfs":{"total":671965249536,"available":658258337792,"internally_reserved":0,"allocated":12060327936,"data_stored":12055198097,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1646349729},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,1,3,1,2,3,0,3,2,4,3],"upper_bound":2048},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-224002048,"num_objects":-3418,"num_object_clones":0,"num_object_copies":-10254,"num_objects_missing_on_primary":-160,"num_objects_missing":-160,"num_objects_degraded":0,"num_objects_misplaced":-1301,"num_objects_unfound":0,"num_objects_dirty":-3418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3418,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":699,"num_bytes_recovered":45809664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.080577"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":163,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.678472+0000","last_change":"2023-10-09T10:46:32.250979+0000","last_active":"2023-10-09T10:47:14.678472+0000","last_peered":"2023-10-09T10:47:14.678472+0000","last_clean":"2023-10-09T10:47:14.678472+0000","last_became_active":"2023-10-09T10:46:32.250625+0000","last_became_peered":"2023-10-09T10:46:32.250625+0000","last_unstale":"2023-10-09T10:47:14.678472+0000","last_undegraded":"2023-10-09T10:47:14.678472+0000","last_fullsized":"2023-10-09T10:47:14.678472+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:34.858348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":138,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.677787+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:47:14.677787+0000","last_peered":"2023-10-09T10:47:14.677787+0000","last_clean":"2023-10-09T10:47:14.677787+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:47:14.677787+0000","last_undegraded":"2023-10-09T10:47:14.677787+0000","last_fullsized":"2023-10-09T10:47:14.677787+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":121,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.670384+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:47:14.670384+0000","last_peered":"2023-10-09T10:47:14.670384+0000","last_clean":"2023-10-09T10:47:14.670384+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:47:14.670384+0000","last_undegraded":"2023-10-09T10:47:14.670384+0000","last_fullsized":"2023-10-09T10:47:14.670384+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":38,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669511+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:47:14.669511+0000","last_peered":"2023-10-09T10:47:14.669511+0000","last_clean":"2023-10-09T10:47:14.669511+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:47:14.669511+0000","last_undegraded":"2023-10-09T10:47:14.669511+0000","last_fullsized":"2023-10-09T10:47:14.669511+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":174,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.677011+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:47:14.677011+0000","last_peered":"2023-10-09T10:47:14.677011+0000","last_clean":"2023-10-09T10:47:14.677011+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:47:14.677011+0000","last_undegraded":"2023-10-09T10:47:14.677011+0000","last_fullsized":"2023-10-09T10:47:14.677011+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":108,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.662973+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:47:14.662973+0000","last_peered":"2023-10-09T10:47:14.662973+0000","last_clean":"2023-10-09T10:47:14.662973+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:47:14.662973+0000","last_undegraded":"2023-10-09T10:47:14.662973+0000","last_fullsized":"2023-10-09T10:47:14.662973+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"170'5656","reported_seq":9800,"reported_epoch":170,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:20.826540+0000","last_change":"2023-10-09T10:46:33.521602+0000","last_active":"2023-10-09T10:47:20.826540+0000","last_peered":"2023-10-09T10:47:20.826540+0000","last_clean":"2023-10-09T10:46:31.055751+0000","last_became_active":"2023-10-09T10:46:33.364058+0000","last_became_peered":"2023-10-09T10:46:33.364058+0000","last_unstale":"2023-10-09T10:47:20.826540+0000","last_undegraded":"2023-10-09T10:47:20.826540+0000","last_fullsized":"2023-10-09T10:47:20.826540+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5656,"log_dups_size":0,"ondisk_log_size":5656,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146636800,"num_objects":2238,"num_object_clones":0,"num_object_copies":6714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2236,"num_objects_unfound":0,"num_objects_dirty":2238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4620,"num_write_kb":219424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2010,"num_bytes_recovered":131727360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":2238}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":38,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.671165+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:47:14.671165+0000","last_peered":"2023-10-09T10:47:14.671165+0000","last_clean":"2023-10-09T10:47:14.671165+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:47:14.671165+0000","last_undegraded":"2023-10-09T10:47:14.671165+0000","last_fullsized":"2023-10-09T10:47:14.671165+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":138,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.677077+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:47:14.677077+0000","last_peered":"2023-10-09T10:47:14.677077+0000","last_clean":"2023-10-09T10:47:14.677077+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:47:14.677077+0000","last_undegraded":"2023-10-09T10:47:14.677077+0000","last_fullsized":"2023-10-09T10:47:14.677077+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":503,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.671570+0000","last_change":"2023-10-09T10:46:05.620365+0000","last_active":"2023-10-09T10:47:14.671570+0000","last_peered":"2023-10-09T10:47:14.671570+0000","last_clean":"2023-10-09T10:47:14.671570+0000","last_became_active":"2023-10-09T10:46:05.620024+0000","last_became_peered":"2023-10-09T10:46:05.620024+0000","last_unstale":"2023-10-09T10:47:14.671570+0000","last_undegraded":"2023-10-09T10:47:14.671570+0000","last_fullsized":"2023-10-09T10:47:14.671570+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.13","version":"170'5518","reported_seq":5694,"reported_epoch":170,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:20.935946+0000","last_change":"2023-10-09T10:46:33.470698+0000","last_active":"2023-10-09T10:47:20.935946+0000","last_peered":"2023-10-09T10:47:20.935946+0000","last_clean":"2023-10-09T10:46:31.195699+0000","last_became_active":"2023-10-09T10:46:33.455632+0000","last_became_peered":"2023-10-09T10:46:33.455632+0000","last_unstale":"2023-10-09T10:47:20.935946+0000","last_undegraded":"2023-10-09T10:46:33.269323+0000","last_fullsized":"2023-10-09T10:46:33.241226+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":5518,"log_dups_size":0,"ondisk_log_size":5518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:07:12.547333+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141164544,"num_objects":2154,"num_object_clones":0,"num_object_copies":6462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3315,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2803,"num_write_kb":105088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":2154},{"shards":"6,7","objects":-621}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":135,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.664622+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:47:14.664622+0000","last_peered":"2023-10-09T10:47:14.664622+0000","last_clean":"2023-10-09T10:47:14.664622+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:47:14.664622+0000","last_undegraded":"2023-10-09T10:47:14.664622+0000","last_fullsized":"2023-10-09T10:47:14.664622+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":95,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.698802+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:47:14.698802+0000","last_peered":"2023-10-09T10:47:14.698802+0000","last_clean":"2023-10-09T10:47:14.698802+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:47:14.698802+0000","last_undegraded":"2023-10-09T10:47:14.698802+0000","last_fullsized":"2023-10-09T10:47:14.698802+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"170'5656","reported_seq":13773,"reported_epoch":170,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:21.381930+0000","last_change":"2023-10-09T10:46:03.683196+0000","last_active":"2023-10-09T10:47:21.381930+0000","last_peered":"2023-10-09T10:47:21.381930+0000","last_clean":"2023-10-09T10:46:02.177309+0000","last_became_active":"2023-10-09T10:46:03.681058+0000","last_became_peered":"2023-10-09T10:46:03.681058+0000","last_unstale":"2023-10-09T10:47:21.381930+0000","last_undegraded":"2023-10-09T10:47:21.381930+0000","last_fullsized":"2023-10-09T10:47:21.381930+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5656,"log_dups_size":0,"ondisk_log_size":5656,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142704640,"num_objects":2178,"num_object_clones":0,"num_object_copies":6534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2178,"num_objects_unfound":0,"num_objects_dirty":2178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4610,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[1,5,6],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":2178}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":91,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.677721+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:47:14.677721+0000","last_peered":"2023-10-09T10:47:14.677721+0000","last_clean":"2023-10-09T10:47:14.677721+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:47:14.677721+0000","last_undegraded":"2023-10-09T10:47:14.677721+0000","last_fullsized":"2023-10-09T10:47:14.677721+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":131,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:16.509752+0000","last_change":"2023-10-09T10:46:32.249921+0000","last_active":"2023-10-09T10:47:16.509752+0000","last_peered":"2023-10-09T10:47:16.509752+0000","last_clean":"2023-10-09T10:47:16.509752+0000","last_became_active":"2023-10-09T10:46:32.249590+0000","last_became_peered":"2023-10-09T10:46:32.249590+0000","last_unstale":"2023-10-09T10:47:16.509752+0000","last_undegraded":"2023-10-09T10:47:16.509752+0000","last_fullsized":"2023-10-09T10:47:16.509752+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:25.169752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"170'5544","reported_seq":10700,"reported_epoch":170,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:47:24.519426+0000","last_change":"2023-10-09T10:46:32.280686+0000","last_active":"2023-10-09T10:47:24.519426+0000","last_peered":"2023-10-09T10:47:24.519426+0000","last_clean":"2023-10-09T10:44:56.105580+0000","last_became_active":"2023-10-09T10:46:07.402111+0000","last_became_peered":"2023-10-09T10:46:07.402111+0000","last_unstale":"2023-10-09T10:47:24.519426+0000","last_undegraded":"2023-10-09T10:47:24.519426+0000","last_fullsized":"2023-10-09T10:46:05.812663+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5544,"log_dups_size":0,"ondisk_log_size":5544,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133464064,"num_objects":2037,"num_object_clones":0,"num_object_copies":6111,"num_objects_missing_on_primary":97,"num_objects_missing":97,"num_objects_degraded":0,"num_objects_misplaced":1624,"num_objects_unfound":0,"num_objects_dirty":2037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4537,"num_write_kb":210336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1267,"num_bytes_recovered":83034112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,2,3,5","objects":1319},{"shards":"1,2,3,5,6","objects":1},{"shards":"1,2,3,6","objects":6},{"shards":"1,2,6","objects":97},{"shards":"1,3,5","objects":1},{"shards":"3,5","objects":613}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":140,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.678970+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:47:14.678970+0000","last_peered":"2023-10-09T10:47:14.678970+0000","last_clean":"2023-10-09T10:47:14.678970+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:47:14.678970+0000","last_undegraded":"2023-10-09T10:47:14.678970+0000","last_fullsized":"2023-10-09T10:47:14.678970+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":134,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.678737+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:47:14.678737+0000","last_peered":"2023-10-09T10:47:14.678737+0000","last_clean":"2023-10-09T10:47:14.678737+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:47:14.678737+0000","last_undegraded":"2023-10-09T10:47:14.678737+0000","last_fullsized":"2023-10-09T10:47:14.678737+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":503,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.674031+0000","last_change":"2023-10-09T10:46:05.618276+0000","last_active":"2023-10-09T10:47:14.674031+0000","last_peered":"2023-10-09T10:47:14.674031+0000","last_clean":"2023-10-09T10:47:14.674031+0000","last_became_active":"2023-10-09T10:46:05.616509+0000","last_became_peered":"2023-10-09T10:46:05.616509+0000","last_unstale":"2023-10-09T10:47:14.674031+0000","last_undegraded":"2023-10-09T10:47:14.674031+0000","last_fullsized":"2023-10-09T10:47:14.674031+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.16","version":"170'5560","reported_seq":5655,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:24.162324+0000","last_change":"2023-10-09T10:46:03.592682+0000","last_active":"2023-10-09T10:47:24.162324+0000","last_peered":"2023-10-09T10:47:24.162324+0000","last_clean":"2023-10-09T10:47:24.162324+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:47:24.162324+0000","last_undegraded":"2023-10-09T10:47:24.162324+0000","last_fullsized":"2023-10-09T10:47:24.162324+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5560,"log_dups_size":0,"ondisk_log_size":5560,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135593984,"num_objects":2069,"num_object_clones":0,"num_object_copies":6207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4517,"num_write_kb":210752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":152,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.675192+0000","last_change":"2023-10-09T10:46:32.250256+0000","last_active":"2023-10-09T10:47:14.675192+0000","last_peered":"2023-10-09T10:47:14.675192+0000","last_clean":"2023-10-09T10:47:14.675192+0000","last_became_active":"2023-10-09T10:46:32.243312+0000","last_became_peered":"2023-10-09T10:46:32.243312+0000","last_unstale":"2023-10-09T10:47:14.675192+0000","last_undegraded":"2023-10-09T10:47:14.675192+0000","last_fullsized":"2023-10-09T10:47:14.675192+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:08.059777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":161,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.682406+0000","last_change":"2023-10-09T10:46:32.237577+0000","last_active":"2023-10-09T10:47:14.682406+0000","last_peered":"2023-10-09T10:47:14.682406+0000","last_clean":"2023-10-09T10:47:14.682406+0000","last_became_active":"2023-10-09T10:46:32.237316+0000","last_became_peered":"2023-10-09T10:46:32.237316+0000","last_unstale":"2023-10-09T10:47:14.682406+0000","last_undegraded":"2023-10-09T10:47:14.682406+0000","last_fullsized":"2023-10-09T10:47:14.682406+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:39.684602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":504,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.668904+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:47:14.668904+0000","last_peered":"2023-10-09T10:47:14.668904+0000","last_clean":"2023-10-09T10:47:14.668904+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:47:14.668904+0000","last_undegraded":"2023-10-09T10:47:14.668904+0000","last_fullsized":"2023-10-09T10:47:14.668904+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.17","version":"170'5440","reported_seq":5529,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:21.019839+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:47:21.019839+0000","last_peered":"2023-10-09T10:47:21.019839+0000","last_clean":"2023-10-09T10:47:21.019839+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:47:21.019839+0000","last_undegraded":"2023-10-09T10:47:21.019839+0000","last_fullsized":"2023-10-09T10:47:21.019839+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5440,"log_dups_size":0,"ondisk_log_size":5440,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135921664,"num_objects":2074,"num_object_clones":0,"num_object_copies":6222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4432,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":38,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669294+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:47:14.669294+0000","last_peered":"2023-10-09T10:47:14.669294+0000","last_clean":"2023-10-09T10:47:14.669294+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:47:14.669294+0000","last_undegraded":"2023-10-09T10:47:14.669294+0000","last_fullsized":"2023-10-09T10:47:14.669294+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":134,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.663540+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:47:14.663540+0000","last_peered":"2023-10-09T10:47:14.663540+0000","last_clean":"2023-10-09T10:47:14.663540+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:47:14.663540+0000","last_undegraded":"2023-10-09T10:47:14.663540+0000","last_fullsized":"2023-10-09T10:47:14.663540+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":504,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669947+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:47:14.669947+0000","last_peered":"2023-10-09T10:47:14.669947+0000","last_clean":"2023-10-09T10:47:14.669947+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:47:14.669947+0000","last_undegraded":"2023-10-09T10:47:14.669947+0000","last_fullsized":"2023-10-09T10:47:14.669947+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"170'5609","reported_seq":5703,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:21.476581+0000","last_change":"2023-10-09T10:46:33.243540+0000","last_active":"2023-10-09T10:47:21.476581+0000","last_peered":"2023-10-09T10:47:21.476581+0000","last_clean":"2023-10-09T10:47:21.476581+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:47:21.476581+0000","last_undegraded":"2023-10-09T10:47:21.476581+0000","last_fullsized":"2023-10-09T10:47:21.476581+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5609,"log_dups_size":0,"ondisk_log_size":5609,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138838016,"num_objects":2119,"num_object_clones":0,"num_object_copies":6357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4555,"num_write_kb":213536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":80,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.678343+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:47:14.678343+0000","last_peered":"2023-10-09T10:47:14.678343+0000","last_clean":"2023-10-09T10:47:14.678343+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:47:14.678343+0000","last_undegraded":"2023-10-09T10:47:14.678343+0000","last_fullsized":"2023-10-09T10:47:14.678343+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":38,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669744+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:47:14.669744+0000","last_peered":"2023-10-09T10:47:14.669744+0000","last_clean":"2023-10-09T10:47:14.669744+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:47:14.669744+0000","last_undegraded":"2023-10-09T10:47:14.669744+0000","last_fullsized":"2023-10-09T10:47:14.669744+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":504,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.668992+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:47:14.668992+0000","last_peered":"2023-10-09T10:47:14.668992+0000","last_clean":"2023-10-09T10:47:14.668992+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:47:14.668992+0000","last_undegraded":"2023-10-09T10:47:14.668992+0000","last_fullsized":"2023-10-09T10:47:14.668992+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"170'5681","reported_seq":5784,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:24.526368+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:47:24.526368+0000","last_peered":"2023-10-09T10:47:24.526368+0000","last_clean":"2023-10-09T10:47:24.526368+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:47:24.526368+0000","last_undegraded":"2023-10-09T10:47:24.526368+0000","last_fullsized":"2023-10-09T10:47:24.526368+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5681,"log_dups_size":0,"ondisk_log_size":5681,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134217728,"num_objects":2048,"num_object_clones":0,"num_object_copies":6144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4620,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":48,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.668825+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:47:14.668825+0000","last_peered":"2023-10-09T10:47:14.668825+0000","last_clean":"2023-10-09T10:47:14.668825+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:47:14.668825+0000","last_undegraded":"2023-10-09T10:47:14.668825+0000","last_fullsized":"2023-10-09T10:47:14.668825+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":138,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.680006+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:47:14.680006+0000","last_peered":"2023-10-09T10:47:14.680006+0000","last_clean":"2023-10-09T10:47:14.680006+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:47:14.680006+0000","last_undegraded":"2023-10-09T10:47:14.680006+0000","last_fullsized":"2023-10-09T10:47:14.680006+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":504,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669650+0000","last_change":"2023-10-09T10:46:05.620574+0000","last_active":"2023-10-09T10:47:14.669650+0000","last_peered":"2023-10-09T10:47:14.669650+0000","last_clean":"2023-10-09T10:47:14.669650+0000","last_became_active":"2023-10-09T10:46:05.620212+0000","last_became_peered":"2023-10-09T10:46:05.620212+0000","last_unstale":"2023-10-09T10:47:14.669650+0000","last_undegraded":"2023-10-09T10:47:14.669650+0000","last_fullsized":"2023-10-09T10:47:14.669650+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"170'9327","reported_seq":13472,"reported_epoch":170,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:20.959600+0000","last_change":"2023-10-09T10:46:32.247903+0000","last_active":"2023-10-09T10:47:20.959600+0000","last_peered":"2023-10-09T10:47:20.959600+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:46:02.739355+0000","last_became_peered":"2023-10-09T10:46:02.739355+0000","last_unstale":"2023-10-09T10:47:20.959600+0000","last_undegraded":"2023-10-09T10:46:02.608805+0000","last_fullsized":"2023-10-09T10:46:02.583589+0000","mapping_epoch":142,"log_start":"103'500","ondisk_log_start":"103'500","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8827,"log_dups_size":0,"ondisk_log_size":8827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:32.899634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282132480,"num_objects":4305,"num_object_clones":0,"num_object_copies":12915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9327,"num_write_kb":436224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2022,"num_bytes_recovered":132513792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":625},{"shards":"5,6","objects":3679}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":163,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.674208+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:47:14.674208+0000","last_peered":"2023-10-09T10:47:14.674208+0000","last_clean":"2023-10-09T10:47:14.674208+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:47:14.674208+0000","last_undegraded":"2023-10-09T10:47:14.674208+0000","last_fullsized":"2023-10-09T10:47:14.674208+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":117,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.662717+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:47:14.662717+0000","last_peered":"2023-10-09T10:47:14.662717+0000","last_clean":"2023-10-09T10:47:14.662717+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:47:14.662717+0000","last_undegraded":"2023-10-09T10:47:14.662717+0000","last_fullsized":"2023-10-09T10:47:14.662717+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":484,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.671688+0000","last_change":"2023-10-09T10:46:02.631378+0000","last_active":"2023-10-09T10:47:14.671688+0000","last_peered":"2023-10-09T10:47:14.671688+0000","last_clean":"2023-10-09T10:47:14.671688+0000","last_became_active":"2023-10-09T10:46:02.629142+0000","last_became_peered":"2023-10-09T10:46:02.629142+0000","last_unstale":"2023-10-09T10:47:14.671688+0000","last_undegraded":"2023-10-09T10:47:14.671688+0000","last_fullsized":"2023-10-09T10:47:14.671688+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.b","version":"170'9101","reported_seq":9183,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:24.465386+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:47:24.465386+0000","last_peered":"2023-10-09T10:47:24.465386+0000","last_clean":"2023-10-09T10:47:24.465386+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:47:24.465386+0000","last_undegraded":"2023-10-09T10:47:24.465386+0000","last_fullsized":"2023-10-09T10:47:24.465386+0000","mapping_epoch":98,"log_start":"116'3425","ondisk_log_start":"116'3425","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5676,"log_dups_size":0,"ondisk_log_size":5676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267321344,"num_objects":4079,"num_object_clones":0,"num_object_copies":12237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9101,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":136,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.674206+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:47:14.674206+0000","last_peered":"2023-10-09T10:47:14.674206+0000","last_clean":"2023-10-09T10:47:14.674206+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:47:14.674206+0000","last_undegraded":"2023-10-09T10:47:14.674206+0000","last_fullsized":"2023-10-09T10:47:14.674206+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":171,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.678323+0000","last_change":"2023-10-09T10:46:32.251035+0000","last_active":"2023-10-09T10:47:14.678323+0000","last_peered":"2023-10-09T10:47:14.678323+0000","last_clean":"2023-10-09T10:47:14.678323+0000","last_became_active":"2023-10-09T10:46:32.250293+0000","last_became_peered":"2023-10-09T10:46:32.250293+0000","last_unstale":"2023-10-09T10:47:14.678323+0000","last_undegraded":"2023-10-09T10:47:14.678323+0000","last_fullsized":"2023-10-09T10:47:14.678323+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.787766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":504,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.670852+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:47:14.670852+0000","last_peered":"2023-10-09T10:47:14.670852+0000","last_clean":"2023-10-09T10:47:14.670852+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:47:14.670852+0000","last_undegraded":"2023-10-09T10:47:14.670852+0000","last_fullsized":"2023-10-09T10:47:14.670852+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"170'5666","reported_seq":5768,"reported_epoch":170,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:24.486393+0000","last_change":"2023-10-09T10:46:33.540719+0000","last_active":"2023-10-09T10:47:24.486393+0000","last_peered":"2023-10-09T10:47:24.486393+0000","last_clean":"2023-10-09T10:46:31.137696+0000","last_became_active":"2023-10-09T10:46:33.527650+0000","last_became_peered":"2023-10-09T10:46:33.527650+0000","last_unstale":"2023-10-09T10:47:24.486393+0000","last_undegraded":"2023-10-09T10:46:33.285468+0000","last_fullsized":"2023-10-09T10:46:33.235683+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5666,"log_dups_size":0,"ondisk_log_size":5666,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136675328,"num_objects":2085,"num_object_clones":0,"num_object_copies":6255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4601,"num_write_kb":213984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2073},{"shards":"3,7","objects":-724}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":91,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.663416+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:47:14.663416+0000","last_peered":"2023-10-09T10:47:14.663416+0000","last_clean":"2023-10-09T10:47:14.663416+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:47:14.663416+0000","last_undegraded":"2023-10-09T10:47:14.663416+0000","last_fullsized":"2023-10-09T10:47:14.663416+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":108,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.909400+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:47:18.909400+0000","last_peered":"2023-10-09T10:47:18.909400+0000","last_clean":"2023-10-09T10:47:18.909400+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:47:18.909400+0000","last_undegraded":"2023-10-09T10:47:18.909400+0000","last_fullsized":"2023-10-09T10:47:18.909400+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":504,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669352+0000","last_change":"2023-10-09T10:46:05.620030+0000","last_active":"2023-10-09T10:47:14.669352+0000","last_peered":"2023-10-09T10:47:14.669352+0000","last_clean":"2023-10-09T10:47:14.669352+0000","last_became_active":"2023-10-09T10:46:05.619694+0000","last_became_peered":"2023-10-09T10:46:05.619694+0000","last_unstale":"2023-10-09T10:47:14.669352+0000","last_undegraded":"2023-10-09T10:47:14.669352+0000","last_fullsized":"2023-10-09T10:47:14.669352+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.9","version":"170'5591","reported_seq":5704,"reported_epoch":170,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:22.440535+0000","last_change":"2023-10-09T10:46:34.405499+0000","last_active":"2023-10-09T10:47:22.440535+0000","last_peered":"2023-10-09T10:47:22.440535+0000","last_clean":"2023-10-09T10:46:31.190603+0000","last_became_active":"2023-10-09T10:46:34.396448+0000","last_became_peered":"2023-10-09T10:46:34.396448+0000","last_unstale":"2023-10-09T10:47:22.440535+0000","last_undegraded":"2023-10-09T10:46:33.246949+0000","last_fullsized":"2023-10-09T10:46:33.237968+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5591,"log_dups_size":0,"ondisk_log_size":5591,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135331840,"num_objects":2065,"num_object_clones":0,"num_object_copies":6195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2004,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4549,"num_write_kb":211648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1303,"num_bytes_recovered":85393408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":762},{"shards":"0,7","objects":597}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":135,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.662884+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:47:14.662884+0000","last_peered":"2023-10-09T10:47:14.662884+0000","last_clean":"2023-10-09T10:47:14.662884+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:47:14.662884+0000","last_undegraded":"2023-10-09T10:47:14.662884+0000","last_fullsized":"2023-10-09T10:47:14.662884+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":38,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669263+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:47:14.669263+0000","last_peered":"2023-10-09T10:47:14.669263+0000","last_clean":"2023-10-09T10:47:14.669263+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:47:14.669263+0000","last_undegraded":"2023-10-09T10:47:14.669263+0000","last_fullsized":"2023-10-09T10:47:14.669263+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":495,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.673870+0000","last_change":"2023-10-09T10:46:05.618134+0000","last_active":"2023-10-09T10:47:14.673870+0000","last_peered":"2023-10-09T10:47:14.673870+0000","last_clean":"2023-10-09T10:47:14.673870+0000","last_became_active":"2023-10-09T10:46:05.616057+0000","last_became_peered":"2023-10-09T10:46:05.616057+0000","last_unstale":"2023-10-09T10:47:14.673870+0000","last_undegraded":"2023-10-09T10:47:14.673870+0000","last_fullsized":"2023-10-09T10:47:14.673870+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"170'8951","reported_seq":9041,"reported_epoch":170,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:20.952929+0000","last_change":"2023-10-09T10:46:33.576530+0000","last_active":"2023-10-09T10:47:20.952929+0000","last_peered":"2023-10-09T10:47:20.952929+0000","last_clean":"2023-10-09T10:46:31.183481+0000","last_became_active":"2023-10-09T10:46:33.542304+0000","last_became_peered":"2023-10-09T10:46:33.542304+0000","last_unstale":"2023-10-09T10:47:20.952929+0000","last_undegraded":"2023-10-09T10:46:33.300581+0000","last_fullsized":"2023-10-09T10:46:33.246047+0000","mapping_epoch":154,"log_start":"104'1577","ondisk_log_start":"104'1577","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7374,"log_dups_size":0,"ondisk_log_size":7374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:26.739488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280690688,"num_objects":4283,"num_object_clones":0,"num_object_copies":12849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6617,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8951,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":4283},{"shards":"1,4","objects":-1278}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":162,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.677363+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:47:14.677363+0000","last_peered":"2023-10-09T10:47:14.677363+0000","last_clean":"2023-10-09T10:47:14.677363+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:47:14.677363+0000","last_undegraded":"2023-10-09T10:47:14.677363+0000","last_fullsized":"2023-10-09T10:47:14.677363+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":134,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.678518+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:47:14.678518+0000","last_peered":"2023-10-09T10:47:14.678518+0000","last_clean":"2023-10-09T10:47:14.678518+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:47:14.678518+0000","last_undegraded":"2023-10-09T10:47:14.678518+0000","last_fullsized":"2023-10-09T10:47:14.678518+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":484,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.668980+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:47:14.668980+0000","last_peered":"2023-10-09T10:47:14.668980+0000","last_clean":"2023-10-09T10:47:14.668980+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:47:14.668980+0000","last_undegraded":"2023-10-09T10:47:14.668980+0000","last_fullsized":"2023-10-09T10:47:14.668980+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.f","version":"170'8998","reported_seq":9086,"reported_epoch":170,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:24.405970+0000","last_change":"2023-10-09T10:46:33.651756+0000","last_active":"2023-10-09T10:47:24.405970+0000","last_peered":"2023-10-09T10:47:24.405970+0000","last_clean":"2023-10-09T10:46:31.156458+0000","last_became_active":"2023-10-09T10:46:33.621003+0000","last_became_peered":"2023-10-09T10:46:33.621003+0000","last_unstale":"2023-10-09T10:47:24.405970+0000","last_undegraded":"2023-10-09T10:46:33.309813+0000","last_fullsized":"2023-10-09T10:46:33.236508+0000","mapping_epoch":154,"log_start":"114'3310","ondisk_log_start":"114'3310","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5688,"log_dups_size":0,"ondisk_log_size":5688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:13.704346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263716864,"num_objects":4024,"num_object_clones":0,"num_object_copies":12072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8998,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":4024},{"shards":"3,4","objects":-1428}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":109,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.678631+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:47:14.678631+0000","last_peered":"2023-10-09T10:47:14.678631+0000","last_clean":"2023-10-09T10:47:14.678631+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:47:14.678631+0000","last_undegraded":"2023-10-09T10:47:14.678631+0000","last_fullsized":"2023-10-09T10:47:14.678631+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":161,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.678719+0000","last_change":"2023-10-09T10:46:32.245583+0000","last_active":"2023-10-09T10:47:14.678719+0000","last_peered":"2023-10-09T10:47:14.678719+0000","last_clean":"2023-10-09T10:47:14.678719+0000","last_became_active":"2023-10-09T10:46:32.245271+0000","last_became_peered":"2023-10-09T10:46:32.245271+0000","last_unstale":"2023-10-09T10:47:14.678719+0000","last_undegraded":"2023-10-09T10:47:14.678719+0000","last_fullsized":"2023-10-09T10:47:14.678719+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.025051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":504,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669012+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:47:14.669012+0000","last_peered":"2023-10-09T10:47:14.669012+0000","last_clean":"2023-10-09T10:47:14.669012+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:47:14.669012+0000","last_undegraded":"2023-10-09T10:47:14.669012+0000","last_fullsized":"2023-10-09T10:47:14.669012+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"170'9220","reported_seq":9302,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:24.526662+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:47:24.526662+0000","last_peered":"2023-10-09T10:47:24.526662+0000","last_clean":"2023-10-09T10:47:24.526662+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:47:24.526662+0000","last_undegraded":"2023-10-09T10:47:24.526662+0000","last_fullsized":"2023-10-09T10:47:24.526662+0000","mapping_epoch":98,"log_start":"114'3478","ondisk_log_start":"114'3478","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5742,"log_dups_size":0,"ondisk_log_size":5742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266338304,"num_objects":4064,"num_object_clones":0,"num_object_copies":12192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9220,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":145,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.678264+0000","last_change":"2023-10-09T10:46:32.245685+0000","last_active":"2023-10-09T10:47:14.678264+0000","last_peered":"2023-10-09T10:47:14.678264+0000","last_clean":"2023-10-09T10:47:14.678264+0000","last_became_active":"2023-10-09T10:46:32.245314+0000","last_became_peered":"2023-10-09T10:46:32.245314+0000","last_unstale":"2023-10-09T10:47:14.678264+0000","last_undegraded":"2023-10-09T10:47:14.678264+0000","last_fullsized":"2023-10-09T10:47:14.678264+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:51.716860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":134,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.680327+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:47:14.680327+0000","last_peered":"2023-10-09T10:47:14.680327+0000","last_clean":"2023-10-09T10:47:14.680327+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:47:14.680327+0000","last_undegraded":"2023-10-09T10:47:14.680327+0000","last_fullsized":"2023-10-09T10:47:14.680327+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":495,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669087+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:47:14.669087+0000","last_peered":"2023-10-09T10:47:14.669087+0000","last_clean":"2023-10-09T10:47:14.669087+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:47:14.669087+0000","last_undegraded":"2023-10-09T10:47:14.669087+0000","last_fullsized":"2023-10-09T10:47:14.669087+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"170'9138","reported_seq":9238,"reported_epoch":170,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:20.966830+0000","last_change":"2023-10-09T10:46:02.754507+0000","last_active":"2023-10-09T10:47:20.966830+0000","last_peered":"2023-10-09T10:47:20.966830+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:46:02.747510+0000","last_became_peered":"2023-10-09T10:46:02.747510+0000","last_unstale":"2023-10-09T10:47:20.966830+0000","last_undegraded":"2023-10-09T10:46:02.605987+0000","last_fullsized":"2023-10-09T10:46:02.585248+0000","mapping_epoch":142,"log_start":"103'300","ondisk_log_start":"103'300","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8838,"log_dups_size":0,"ondisk_log_size":8838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:59.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278986776,"num_objects":4258,"num_object_clones":0,"num_object_copies":12774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4258,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9138,"num_write_kb":428609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,2,5","objects":2718},{"shards":"1,5","objects":1540}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":139,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.676867+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:47:14.676867+0000","last_peered":"2023-10-09T10:47:14.676867+0000","last_clean":"2023-10-09T10:47:14.676867+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:47:14.676867+0000","last_undegraded":"2023-10-09T10:47:14.676867+0000","last_fullsized":"2023-10-09T10:47:14.676867+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":122,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.673347+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:47:14.673347+0000","last_peered":"2023-10-09T10:47:14.673347+0000","last_clean":"2023-10-09T10:47:14.673347+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:47:14.673347+0000","last_undegraded":"2023-10-09T10:47:14.673347+0000","last_fullsized":"2023-10-09T10:47:14.673347+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":504,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.671027+0000","last_change":"2023-10-09T10:46:05.617983+0000","last_active":"2023-10-09T10:47:14.671027+0000","last_peered":"2023-10-09T10:47:14.671027+0000","last_clean":"2023-10-09T10:47:14.671027+0000","last_became_active":"2023-10-09T10:46:05.616021+0000","last_became_peered":"2023-10-09T10:46:05.616021+0000","last_unstale":"2023-10-09T10:47:14.671027+0000","last_undegraded":"2023-10-09T10:47:14.671027+0000","last_fullsized":"2023-10-09T10:47:14.671027+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":165,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669506+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:47:14.669506+0000","last_peered":"2023-10-09T10:47:14.669506+0000","last_clean":"2023-10-09T10:47:14.669506+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:47:14.669506+0000","last_undegraded":"2023-10-09T10:47:14.669506+0000","last_fullsized":"2023-10-09T10:47:14.669506+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":162,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.663626+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:47:14.663626+0000","last_peered":"2023-10-09T10:47:14.663626+0000","last_clean":"2023-10-09T10:47:14.663626+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:47:14.663626+0000","last_undegraded":"2023-10-09T10:47:14.663626+0000","last_fullsized":"2023-10-09T10:47:14.663626+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":107,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669033+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:47:14.669033+0000","last_peered":"2023-10-09T10:47:14.669033+0000","last_clean":"2023-10-09T10:47:14.669033+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:47:14.669033+0000","last_undegraded":"2023-10-09T10:47:14.669033+0000","last_fullsized":"2023-10-09T10:47:14.669033+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"170'5604","reported_seq":9630,"reported_epoch":170,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:24.745419+0000","last_change":"2023-10-09T10:46:32.248218+0000","last_active":"2023-10-09T10:47:24.745419+0000","last_peered":"2023-10-09T10:47:24.745419+0000","last_clean":"2023-10-09T10:46:02.181912+0000","last_became_active":"2023-10-09T10:46:32.226237+0000","last_became_peered":"2023-10-09T10:46:32.226237+0000","last_unstale":"2023-10-09T10:47:24.745419+0000","last_undegraded":"2023-10-09T10:47:24.745419+0000","last_fullsized":"2023-10-09T10:47:24.745419+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5604,"log_dups_size":0,"ondisk_log_size":5604,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136806400,"num_objects":2087,"num_object_clones":0,"num_object_copies":6261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2087,"num_objects_unfound":0,"num_objects_dirty":2087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4557,"num_write_kb":212640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":2087}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":503,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669956+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:47:14.669956+0000","last_peered":"2023-10-09T10:47:14.669956+0000","last_clean":"2023-10-09T10:47:14.669956+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:47:14.669956+0000","last_undegraded":"2023-10-09T10:47:14.669956+0000","last_fullsized":"2023-10-09T10:47:14.669956+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"170'5680","reported_seq":5852,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:24.273410+0000","last_change":"2023-10-09T10:46:02.760963+0000","last_active":"2023-10-09T10:47:24.273410+0000","last_peered":"2023-10-09T10:47:24.273410+0000","last_clean":"2023-10-09T10:47:24.273410+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:47:24.273410+0000","last_undegraded":"2023-10-09T10:47:24.273410+0000","last_fullsized":"2023-10-09T10:47:24.273410+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"141'4416","last_scrub_stamp":"2023-10-09T10:46:02.760810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:46:02.760810+0000","objects_scrubbed":3378,"log_size":5680,"log_dups_size":0,"ondisk_log_size":5680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:35.863002+0000","scrub_duration":0.47769323899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138543104,"num_objects":2114,"num_object_clones":0,"num_object_copies":6342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1264,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":139,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.663329+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:47:14.663329+0000","last_peered":"2023-10-09T10:47:14.663329+0000","last_clean":"2023-10-09T10:47:14.663329+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:47:14.663329+0000","last_undegraded":"2023-10-09T10:47:14.663329+0000","last_fullsized":"2023-10-09T10:47:14.663329+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":155,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.678879+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:47:14.678879+0000","last_peered":"2023-10-09T10:47:14.678879+0000","last_clean":"2023-10-09T10:47:14.678879+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:47:14.678879+0000","last_undegraded":"2023-10-09T10:47:14.678879+0000","last_fullsized":"2023-10-09T10:47:14.678879+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":146,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669578+0000","last_change":"2023-10-09T10:46:32.245608+0000","last_active":"2023-10-09T10:47:14.669578+0000","last_peered":"2023-10-09T10:47:14.669578+0000","last_clean":"2023-10-09T10:47:14.669578+0000","last_became_active":"2023-10-09T10:46:32.245290+0000","last_became_peered":"2023-10-09T10:46:32.245290+0000","last_unstale":"2023-10-09T10:47:14.669578+0000","last_undegraded":"2023-10-09T10:47:14.669578+0000","last_fullsized":"2023-10-09T10:47:14.669578+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:48.832515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":504,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669185+0000","last_change":"2023-10-09T10:46:05.620057+0000","last_active":"2023-10-09T10:47:14.669185+0000","last_peered":"2023-10-09T10:47:14.669185+0000","last_clean":"2023-10-09T10:47:14.669185+0000","last_became_active":"2023-10-09T10:46:05.619703+0000","last_became_peered":"2023-10-09T10:46:05.619703+0000","last_unstale":"2023-10-09T10:47:14.669185+0000","last_undegraded":"2023-10-09T10:47:14.669185+0000","last_fullsized":"2023-10-09T10:47:14.669185+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"170'5619","reported_seq":5726,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:24.465344+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:47:24.465344+0000","last_peered":"2023-10-09T10:47:24.465344+0000","last_clean":"2023-10-09T10:47:24.465344+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:47:24.465344+0000","last_undegraded":"2023-10-09T10:47:24.465344+0000","last_fullsized":"2023-10-09T10:47:24.465344+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5619,"log_dups_size":0,"ondisk_log_size":5619,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139067392,"num_objects":2122,"num_object_clones":0,"num_object_copies":6366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4558,"num_write_kb":213760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":167,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.698970+0000","last_change":"2023-10-09T10:46:32.241041+0000","last_active":"2023-10-09T10:47:14.698970+0000","last_peered":"2023-10-09T10:47:14.698970+0000","last_clean":"2023-10-09T10:47:14.698970+0000","last_became_active":"2023-10-09T10:46:32.238683+0000","last_became_peered":"2023-10-09T10:46:32.238683+0000","last_unstale":"2023-10-09T10:47:14.698970+0000","last_undegraded":"2023-10-09T10:47:14.698970+0000","last_fullsized":"2023-10-09T10:47:14.698970+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:25.817713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":131,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.672431+0000","last_change":"2023-10-09T10:46:32.250104+0000","last_active":"2023-10-09T10:47:14.672431+0000","last_peered":"2023-10-09T10:47:14.672431+0000","last_clean":"2023-10-09T10:47:14.672431+0000","last_became_active":"2023-10-09T10:46:32.249805+0000","last_became_peered":"2023-10-09T10:46:32.249805+0000","last_unstale":"2023-10-09T10:47:14.672431+0000","last_undegraded":"2023-10-09T10:47:14.672431+0000","last_fullsized":"2023-10-09T10:47:14.672431+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:07.230672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":160,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.673411+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:47:14.673411+0000","last_peered":"2023-10-09T10:47:14.673411+0000","last_clean":"2023-10-09T10:47:14.673411+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:47:14.673411+0000","last_undegraded":"2023-10-09T10:47:14.673411+0000","last_fullsized":"2023-10-09T10:47:14.673411+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"170'5447","reported_seq":5541,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:21.019350+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:47:21.019350+0000","last_peered":"2023-10-09T10:47:21.019350+0000","last_clean":"2023-10-09T10:47:21.019350+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:47:21.019350+0000","last_undegraded":"2023-10-09T10:47:21.019350+0000","last_fullsized":"2023-10-09T10:47:21.019350+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5447,"log_dups_size":0,"ondisk_log_size":5447,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144900096,"num_objects":2211,"num_object_clones":0,"num_object_copies":6633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4439,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":149,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.662732+0000","last_change":"2023-10-09T10:46:32.235346+0000","last_active":"2023-10-09T10:47:14.662732+0000","last_peered":"2023-10-09T10:47:14.662732+0000","last_clean":"2023-10-09T10:47:14.662732+0000","last_became_active":"2023-10-09T10:46:32.234906+0000","last_became_peered":"2023-10-09T10:46:32.234906+0000","last_unstale":"2023-10-09T10:47:14.662732+0000","last_undegraded":"2023-10-09T10:47:14.662732+0000","last_fullsized":"2023-10-09T10:47:14.662732+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.199251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":504,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.668679+0000","last_change":"2023-10-09T10:46:05.617146+0000","last_active":"2023-10-09T10:47:14.668679+0000","last_peered":"2023-10-09T10:47:14.668679+0000","last_clean":"2023-10-09T10:47:14.668679+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:47:14.668679+0000","last_undegraded":"2023-10-09T10:47:14.668679+0000","last_fullsized":"2023-10-09T10:47:14.668679+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":97,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.676851+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:47:14.676851+0000","last_peered":"2023-10-09T10:47:14.676851+0000","last_clean":"2023-10-09T10:47:14.676851+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:47:14.676851+0000","last_undegraded":"2023-10-09T10:47:14.676851+0000","last_fullsized":"2023-10-09T10:47:14.676851+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":67,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.678560+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:47:14.678560+0000","last_peered":"2023-10-09T10:47:14.678560+0000","last_clean":"2023-10-09T10:47:14.678560+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:47:14.678560+0000","last_undegraded":"2023-10-09T10:47:14.678560+0000","last_fullsized":"2023-10-09T10:47:14.678560+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":504,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.668926+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:47:14.668926+0000","last_peered":"2023-10-09T10:47:14.668926+0000","last_clean":"2023-10-09T10:47:14.668926+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:47:14.668926+0000","last_undegraded":"2023-10-09T10:47:14.668926+0000","last_fullsized":"2023-10-09T10:47:14.668926+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":161,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.672555+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:47:14.672555+0000","last_peered":"2023-10-09T10:47:14.672555+0000","last_clean":"2023-10-09T10:47:14.672555+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:47:14.672555+0000","last_undegraded":"2023-10-09T10:47:14.672555+0000","last_fullsized":"2023-10-09T10:47:14.672555+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":47,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669873+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:47:14.669873+0000","last_peered":"2023-10-09T10:47:14.669873+0000","last_clean":"2023-10-09T10:47:14.669873+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:47:14.669873+0000","last_undegraded":"2023-10-09T10:47:14.669873+0000","last_fullsized":"2023-10-09T10:47:14.669873+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":47,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669675+0000","last_change":"2023-10-09T10:46:32.236202+0000","last_active":"2023-10-09T10:47:14.669675+0000","last_peered":"2023-10-09T10:47:14.669675+0000","last_clean":"2023-10-09T10:47:14.669675+0000","last_became_active":"2023-10-09T10:46:32.235901+0000","last_became_peered":"2023-10-09T10:46:32.235901+0000","last_unstale":"2023-10-09T10:47:14.669675+0000","last_undegraded":"2023-10-09T10:47:14.669675+0000","last_fullsized":"2023-10-09T10:47:14.669675+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:48.260403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"170'5605","reported_seq":5746,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:21.344544+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:47:21.344544+0000","last_peered":"2023-10-09T10:47:21.344544+0000","last_clean":"2023-10-09T10:47:21.344544+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:47:21.344544+0000","last_undegraded":"2023-10-09T10:47:21.344544+0000","last_fullsized":"2023-10-09T10:47:21.344544+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":5605,"log_dups_size":0,"ondisk_log_size":5605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142409728,"num_objects":2173,"num_object_clones":0,"num_object_copies":6519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2575,"num_write_kb":89664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":504,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669435+0000","last_change":"2023-10-09T10:46:05.620764+0000","last_active":"2023-10-09T10:47:14.669435+0000","last_peered":"2023-10-09T10:47:14.669435+0000","last_clean":"2023-10-09T10:47:14.669435+0000","last_became_active":"2023-10-09T10:46:05.619870+0000","last_became_peered":"2023-10-09T10:46:05.619870+0000","last_unstale":"2023-10-09T10:47:14.669435+0000","last_undegraded":"2023-10-09T10:47:14.669435+0000","last_fullsized":"2023-10-09T10:47:14.669435+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":158,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.673877+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:47:14.673877+0000","last_peered":"2023-10-09T10:47:14.673877+0000","last_clean":"2023-10-09T10:47:14.673877+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:47:14.673877+0000","last_undegraded":"2023-10-09T10:47:14.673877+0000","last_fullsized":"2023-10-09T10:47:14.673877+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":131,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669664+0000","last_change":"2023-10-09T10:46:32.251450+0000","last_active":"2023-10-09T10:47:14.669664+0000","last_peered":"2023-10-09T10:47:14.669664+0000","last_clean":"2023-10-09T10:47:14.669664+0000","last_became_active":"2023-10-09T10:46:32.250933+0000","last_became_peered":"2023-10-09T10:46:32.250933+0000","last_unstale":"2023-10-09T10:47:14.669664+0000","last_undegraded":"2023-10-09T10:47:14.669664+0000","last_fullsized":"2023-10-09T10:47:14.669664+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:04.116178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669092+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:47:14.669092+0000","last_peered":"2023-10-09T10:47:14.669092+0000","last_clean":"2023-10-09T10:47:14.669092+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:47:14.669092+0000","last_undegraded":"2023-10-09T10:47:14.669092+0000","last_fullsized":"2023-10-09T10:47:14.669092+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"170'5473","reported_seq":12159,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:21.378730+0000","last_change":"2023-10-09T10:47:02.269289+0000","last_active":"2023-10-09T10:47:21.378730+0000","last_peered":"2023-10-09T10:47:21.378730+0000","last_clean":"2023-10-09T10:47:21.378730+0000","last_became_active":"2023-10-09T10:47:02.257854+0000","last_became_peered":"2023-10-09T10:47:02.257854+0000","last_unstale":"2023-10-09T10:47:21.378730+0000","last_undegraded":"2023-10-09T10:47:21.378730+0000","last_fullsized":"2023-10-09T10:47:21.378730+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5473,"log_dups_size":0,"ondisk_log_size":5473,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140083200,"num_objects":2137,"num_object_clones":0,"num_object_copies":6411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4465,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3287,"num_bytes_recovered":181927936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":504,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669494+0000","last_change":"2023-10-09T10:46:05.618443+0000","last_active":"2023-10-09T10:47:14.669494+0000","last_peered":"2023-10-09T10:47:14.669494+0000","last_clean":"2023-10-09T10:47:14.669494+0000","last_became_active":"2023-10-09T10:46:05.616588+0000","last_became_peered":"2023-10-09T10:46:05.616588+0000","last_unstale":"2023-10-09T10:47:14.669494+0000","last_undegraded":"2023-10-09T10:47:14.669494+0000","last_fullsized":"2023-10-09T10:47:14.669494+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":162,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.679933+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:47:14.679933+0000","last_peered":"2023-10-09T10:47:14.679933+0000","last_clean":"2023-10-09T10:47:14.679933+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:47:14.679933+0000","last_undegraded":"2023-10-09T10:47:14.679933+0000","last_fullsized":"2023-10-09T10:47:14.679933+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":107,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.672464+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:47:14.672464+0000","last_peered":"2023-10-09T10:47:14.672464+0000","last_clean":"2023-10-09T10:47:14.672464+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:47:14.672464+0000","last_undegraded":"2023-10-09T10:47:14.672464+0000","last_fullsized":"2023-10-09T10:47:14.672464+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":151,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.676938+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:47:14.676938+0000","last_peered":"2023-10-09T10:47:14.676938+0000","last_clean":"2023-10-09T10:47:14.676938+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:47:14.676938+0000","last_undegraded":"2023-10-09T10:47:14.676938+0000","last_fullsized":"2023-10-09T10:47:14.676938+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"170'5568","reported_seq":5708,"reported_epoch":170,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:20.826537+0000","last_change":"2023-10-09T10:46:33.507626+0000","last_active":"2023-10-09T10:47:20.826537+0000","last_peered":"2023-10-09T10:47:20.826537+0000","last_clean":"2023-10-09T10:46:31.130083+0000","last_became_active":"2023-10-09T10:46:33.493288+0000","last_became_peered":"2023-10-09T10:46:33.493288+0000","last_unstale":"2023-10-09T10:47:20.826537+0000","last_undegraded":"2023-10-09T10:46:33.283624+0000","last_fullsized":"2023-10-09T10:46:33.244043+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5568,"log_dups_size":0,"ondisk_log_size":5568,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139821056,"num_objects":2133,"num_object_clones":0,"num_object_copies":6399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3337,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4531,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":2138},{"shards":"3,4","objects":-682}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":503,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.671437+0000","last_change":"2023-10-09T10:46:05.617589+0000","last_active":"2023-10-09T10:47:14.671437+0000","last_peered":"2023-10-09T10:47:14.671437+0000","last_clean":"2023-10-09T10:47:14.671437+0000","last_became_active":"2023-10-09T10:46:05.616400+0000","last_became_peered":"2023-10-09T10:46:05.616400+0000","last_unstale":"2023-10-09T10:47:14.671437+0000","last_undegraded":"2023-10-09T10:47:14.671437+0000","last_fullsized":"2023-10-09T10:47:14.671437+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":27,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.679649+0000","last_change":"2023-10-09T10:46:32.245106+0000","last_active":"2023-10-09T10:47:14.679649+0000","last_peered":"2023-10-09T10:47:14.679649+0000","last_clean":"2023-10-09T10:47:14.679649+0000","last_became_active":"2023-10-09T10:46:32.244826+0000","last_became_peered":"2023-10-09T10:46:32.244826+0000","last_unstale":"2023-10-09T10:47:14.679649+0000","last_undegraded":"2023-10-09T10:47:14.679649+0000","last_fullsized":"2023-10-09T10:47:14.679649+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:30.322834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":42,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669598+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:47:14.669598+0000","last_peered":"2023-10-09T10:47:14.669598+0000","last_clean":"2023-10-09T10:47:14.669598+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:47:14.669598+0000","last_undegraded":"2023-10-09T10:47:14.669598+0000","last_fullsized":"2023-10-09T10:47:14.669598+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":117,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.678244+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:47:14.678244+0000","last_peered":"2023-10-09T10:47:14.678244+0000","last_clean":"2023-10-09T10:47:14.678244+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:47:14.678244+0000","last_undegraded":"2023-10-09T10:47:14.678244+0000","last_fullsized":"2023-10-09T10:47:14.678244+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"170'5488","reported_seq":9524,"reported_epoch":170,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:20.992227+0000","last_change":"2023-10-09T10:47:00.660718+0000","last_active":"2023-10-09T10:47:20.992227+0000","last_peered":"2023-10-09T10:47:20.992227+0000","last_clean":"2023-10-09T10:46:31.098576+0000","last_became_active":"2023-10-09T10:46:33.454803+0000","last_became_peered":"2023-10-09T10:46:33.454803+0000","last_unstale":"2023-10-09T10:47:20.992227+0000","last_undegraded":"2023-10-09T10:46:33.267772+0000","last_fullsized":"2023-10-09T10:46:33.242755+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":5488,"log_dups_size":0,"ondisk_log_size":5488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:03.811205+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140115968,"num_objects":2138,"num_object_clones":0,"num_object_copies":6414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":358,"num_objects_misplaced":473,"num_objects_unfound":0,"num_objects_dirty":2138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2843,"num_write_kb":107264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1929,"num_bytes_recovered":126418944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":473},{"shards":"6,7","objects":1307}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":38,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669179+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:47:14.669179+0000","last_peered":"2023-10-09T10:47:14.669179+0000","last_clean":"2023-10-09T10:47:14.669179+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:47:14.669179+0000","last_undegraded":"2023-10-09T10:47:14.669179+0000","last_fullsized":"2023-10-09T10:47:14.669179+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":101,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.669742+0000","last_change":"2023-10-09T10:46:32.251242+0000","last_active":"2023-10-09T10:47:14.669742+0000","last_peered":"2023-10-09T10:47:14.669742+0000","last_clean":"2023-10-09T10:47:14.669742+0000","last_became_active":"2023-10-09T10:46:32.250534+0000","last_became_peered":"2023-10-09T10:46:32.250534+0000","last_unstale":"2023-10-09T10:47:14.669742+0000","last_undegraded":"2023-10-09T10:47:14.669742+0000","last_fullsized":"2023-10-09T10:47:14.669742+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:33.726041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"170'5601","reported_seq":9575,"reported_epoch":170,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:22.619799+0000","last_change":"2023-10-09T10:46:34.514479+0000","last_active":"2023-10-09T10:47:22.619799+0000","last_peered":"2023-10-09T10:47:22.619799+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:46:34.409667+0000","last_became_peered":"2023-10-09T10:46:34.409667+0000","last_unstale":"2023-10-09T10:47:22.619799+0000","last_undegraded":"2023-10-09T10:46:34.376635+0000","last_fullsized":"2023-10-09T10:46:33.237998+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5601,"log_dups_size":0,"ondisk_log_size":5601,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138608640,"num_objects":2115,"num_object_clones":0,"num_object_copies":6345,"num_objects_missing_on_primary":1406,"num_objects_missing":1406,"num_objects_degraded":1916,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4559,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1932,"num_bytes_recovered":126615552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,7","objects":709},{"shards":"2,3,7","objects":1406}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":137,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.663024+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:47:14.663024+0000","last_peered":"2023-10-09T10:47:14.663024+0000","last_clean":"2023-10-09T10:47:14.663024+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:47:14.663024+0000","last_undegraded":"2023-10-09T10:47:14.663024+0000","last_fullsized":"2023-10-09T10:47:14.663024+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":151,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.679098+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:47:14.679098+0000","last_peered":"2023-10-09T10:47:14.679098+0000","last_clean":"2023-10-09T10:47:14.679098+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:47:14.679098+0000","last_undegraded":"2023-10-09T10:47:14.679098+0000","last_fullsized":"2023-10-09T10:47:14.679098+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"170'5614","reported_seq":5716,"reported_epoch":170,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:24.391618+0000","last_change":"2023-10-09T10:46:33.546991+0000","last_active":"2023-10-09T10:47:24.391618+0000","last_peered":"2023-10-09T10:47:24.391618+0000","last_clean":"2023-10-09T10:46:31.009484+0000","last_became_active":"2023-10-09T10:46:33.535665+0000","last_became_peered":"2023-10-09T10:46:33.535665+0000","last_unstale":"2023-10-09T10:47:24.391618+0000","last_undegraded":"2023-10-09T10:46:33.277168+0000","last_fullsized":"2023-10-09T10:46:33.235038+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5614,"log_dups_size":0,"ondisk_log_size":5614,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129597440,"num_objects":1978,"num_object_clones":0,"num_object_copies":5934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4550,"num_write_kb":208864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1990},{"shards":"3,7","objects":-780}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":120,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.663697+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:47:14.663697+0000","last_peered":"2023-10-09T10:47:14.663697+0000","last_clean":"2023-10-09T10:47:14.663697+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:47:14.663697+0000","last_undegraded":"2023-10-09T10:47:14.663697+0000","last_fullsized":"2023-10-09T10:47:14.663697+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":18,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.678545+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:47:14.678545+0000","last_peered":"2023-10-09T10:47:14.678545+0000","last_clean":"2023-10-09T10:47:14.678545+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:47:14.678545+0000","last_undegraded":"2023-10-09T10:47:14.678545+0000","last_fullsized":"2023-10-09T10:47:14.678545+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":153,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.677399+0000","last_change":"2023-10-09T10:46:32.244607+0000","last_active":"2023-10-09T10:47:14.677399+0000","last_peered":"2023-10-09T10:47:14.677399+0000","last_clean":"2023-10-09T10:47:14.677399+0000","last_became_active":"2023-10-09T10:46:32.242628+0000","last_became_peered":"2023-10-09T10:46:32.242628+0000","last_unstale":"2023-10-09T10:47:14.677399+0000","last_undegraded":"2023-10-09T10:47:14.677399+0000","last_fullsized":"2023-10-09T10:47:14.677399+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:10.738819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":184,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.698733+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:47:14.698733+0000","last_peered":"2023-10-09T10:47:14.698733+0000","last_clean":"2023-10-09T10:47:14.698733+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:47:14.698733+0000","last_undegraded":"2023-10-09T10:47:14.698733+0000","last_fullsized":"2023-10-09T10:47:14.698733+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":59,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.670943+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:47:14.670943+0000","last_peered":"2023-10-09T10:47:14.670943+0000","last_clean":"2023-10-09T10:47:14.670943+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:47:14.670943+0000","last_undegraded":"2023-10-09T10:47:14.670943+0000","last_fullsized":"2023-10-09T10:47:14.670943+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":134,"reported_epoch":169,"state":"active+clean","last_fresh":"2023-10-09T10:47:14.678136+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:47:14.678136+0000","last_peered":"2023-10-09T10:47:14.678136+0000","last_clean":"2023-10-09T10:47:14.678136+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:47:14.678136+0000","last_undegraded":"2023-10-09T10:47:14.678136+0000","last_fullsized":"2023-10-09T10:47:14.678136+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":4409688088,"num_objects":67288,"num_object_clones":0,"num_object_copies":201864,"num_objects_missing_on_primary":1503,"num_objects_missing":1503,"num_objects_degraded":34009,"num_objects_misplaced":8598,"num_objects_unfound":0,"num_objects_dirty":67288,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":136920,"num_write_kb":6258529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19024,"num_bytes_recovered":1213267968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15011229696,"data_stored":15011217480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":153765,"ondisk_log_size":153765,"up":78,"acting":66,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1186,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332249,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2236512,"kb_used_data":1922840,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":91508640,"statfs":{"total":95995035648,"available":93704847360,"internally_reserved":0,"allocated":1968988160,"data_stored":1968341099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.70099999999999996,"15min":0.61899999999999999},"min":{"1min":0.23100000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.0919999999999996,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.3640000000000001,"5min":0.79100000000000004,"15min":0.69099999999999995},"min":{"1min":0.309,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":10.207000000000001,"5min":10.207000000000001,"15min":10.207000000000001},"last":2.5979999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:46:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":0.99399999999999999,"15min":0.69199999999999995},"min":{"1min":0.42199999999999999,"5min":0.318,"15min":0.318},"max":{"1min":8.7360000000000007,"5min":8.7360000000000007,"15min":8.7360000000000007},"last":3.419},{"interface":"front","average":{"1min":1.2230000000000001,"5min":0.80500000000000005,"15min":0.623},"min":{"1min":0.27800000000000002,"5min":0.245,"15min":0.245},"max":{"1min":3.6080000000000001,"5min":6.4939999999999998,"15min":6.4939999999999998},"last":5.0140000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.1499999999999999,"15min":0.72399999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.4770000000000003,"5min":6.4770000000000003,"15min":6.4770000000000003},"last":3.548},{"interface":"front","average":{"1min":1.272,"5min":0.85699999999999998,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.251,"15min":0.251},"max":{"1min":3.1480000000000001,"5min":3.7120000000000002,"15min":3.7120000000000002},"last":0.59699999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:46:36 2023","interfaces":[{"interface":"back","average":{"1min":1.9079999999999999,"5min":1.073,"15min":0.748},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.8129999999999997,"5min":8.1639999999999997,"15min":8.1639999999999997},"last":3.0619999999999998},{"interface":"front","average":{"1min":1.3720000000000001,"5min":1.022,"15min":0.73999999999999999},"min":{"1min":0.377,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.3609999999999998,"5min":12.451000000000001,"15min":12.451000000000001},"last":1.107}]},{"osd":4,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.73299999999999998,"15min":0.56200000000000006},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.1420000000000003,"5min":7.4509999999999996,"15min":7.4509999999999996},"last":2.9700000000000002},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.65300000000000002,"15min":0.53600000000000003},"min":{"1min":0.29799999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.2749999999999999,"5min":3.1699999999999999,"15min":3.1699999999999999},"last":1.0780000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.56399999999999995,"15min":0.52700000000000002},"min":{"1min":0.34300000000000003,"5min":0.247,"15min":0.247},"max":{"1min":2.2349999999999999,"5min":2.2349999999999999,"15min":2.2349999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.81299999999999994,"15min":0.63},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":5.0860000000000003,"5min":10.33,"15min":10.33},"last":3.032}]},{"osd":6,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.83399999999999996,"15min":0.64800000000000002},"min":{"1min":0.311,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.109,"5min":10.164999999999999,"15min":10.164999999999999},"last":3.0049999999999999},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.66000000000000003,"15min":0.58299999999999996},"min":{"1min":0.33300000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.306,"5min":2.8079999999999998,"15min":2.8079999999999998},"last":1.129}]}]},{"osd":6,"up_from":44,"seq":188978561180,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1767964,"kb_used_data":1458388,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91977188,"statfs":{"total":95995035648,"available":94184640512,"internally_reserved":0,"allocated":1493389312,"data_stored":1492746789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.772,"5min":0.89500000000000002,"15min":0.70499999999999996},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.3220000000000001,"5min":7.3220000000000001,"15min":7.3220000000000001},"last":0.29099999999999998},{"interface":"front","average":{"1min":2.02,"5min":0.92000000000000004,"15min":0.71199999999999997},"min":{"1min":0.26700000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.5309999999999997,"5min":7.5309999999999997,"15min":7.5309999999999997},"last":0.375}]},{"osd":1,"last update":"Mon Oct 9 10:46:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8180000000000001,"5min":1.0680000000000001,"15min":0.75},"min":{"1min":0.29899999999999999,"5min":0.245,"15min":0.245},"max":{"1min":7.327,"5min":16.747,"15min":16.747},"last":0.25700000000000001},{"interface":"front","average":{"1min":1.883,"5min":1.101,"15min":0.73399999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.4260000000000002,"5min":16.734000000000002,"15min":16.734000000000002},"last":1.9550000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":0.95799999999999996,"15min":0.67500000000000004},"min":{"1min":0.28699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5069999999999997,"5min":7.5069999999999997,"15min":7.5069999999999997},"last":1.99},{"interface":"front","average":{"1min":1.675,"5min":1.143,"15min":0.755},"min":{"1min":0.28599999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.3899999999999997,"5min":12.571,"15min":12.571},"last":0.64000000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":0.89600000000000002,"15min":0.65700000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.8799999999999999,"5min":4.7030000000000003,"15min":4.7030000000000003},"last":0.34599999999999997},{"interface":"front","average":{"1min":2.4399999999999999,"5min":1.0700000000000001,"15min":0.72099999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.4770000000000003,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":5.8230000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":0.66600000000000004,"15min":0.51500000000000001},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":4.0739999999999998,"5min":4.0739999999999998,"15min":4.0739999999999998},"last":3.891},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.67300000000000004,"15min":0.54600000000000004},"min":{"1min":0.34899999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6600000000000001,"5min":3.6600000000000001,"15min":3.6600000000000001},"last":0.495}]},{"osd":5,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":0.76000000000000001,"15min":0.56999999999999995},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.8949999999999996,"5min":4.8949999999999996,"15min":4.8949999999999996},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.76900000000000002,"15min":0.59999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":4.3639999999999999,"5min":4.3639999999999999,"15min":4.3639999999999999},"last":0.66600000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:46:22 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.67200000000000004,"15min":0.56899999999999995},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":3.472,"5min":3.472,"15min":3.472},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.119,"5min":0.754,"15min":0.628},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.34000000000000002},"max":{"1min":3.7949999999999999,"5min":3.7949999999999999,"15min":3.7949999999999999},"last":0.46800000000000003}]}]},{"osd":1,"up_from":85,"seq":365072220219,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1796540,"kb_used_data":1692660,"kb_used_omap":9,"kb_used_meta":103734,"kb_avail":91948612,"statfs":{"total":95995035648,"available":94155378688,"internally_reserved":0,"allocated":1733283840,"data_stored":1732440460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":106224617},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.3160000000000001,"15min":0.68100000000000005},"min":{"1min":0.255,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":8.3759999999999994,"5min":14.1,"15min":14.1},"last":4.2800000000000002},{"interface":"front","average":{"1min":2.5499999999999998,"5min":1.266,"15min":0.68799999999999994},"min":{"1min":0.27300000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":13.462999999999999,"5min":14.465,"15min":14.465},"last":5.8090000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:47:13 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.1200000000000001,"15min":0.61799999999999999},"min":{"1min":0.21199999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.4829999999999997,"5min":8.6859999999999999,"15min":8.6859999999999999},"last":0.91000000000000003},{"interface":"front","average":{"1min":1.919,"5min":1.4690000000000001,"15min":0.746},"min":{"1min":0.29099999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.2829999999999995,"5min":14.234,"15min":14.234},"last":0.28199999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.2,"15min":0.71399999999999997},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.3539999999999992,"5min":14.404999999999999,"15min":14.404999999999999},"last":4.2069999999999999},{"interface":"front","average":{"1min":2.077,"5min":1.1080000000000001,"15min":0.59899999999999998},"min":{"1min":0.255,"5min":0.251,"15min":0.251},"max":{"1min":9.6820000000000004,"5min":9.6820000000000004,"15min":9.6820000000000004},"last":0.35599999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.0880000000000001,"15min":0.75800000000000001},"min":{"1min":0.29099999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.1420000000000003,"5min":8.0129999999999999,"15min":8.0129999999999999},"last":0.94299999999999995},{"interface":"front","average":{"1min":1.4850000000000001,"5min":1.0249999999999999,"15min":0.755},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":8.327,"5min":8.327,"15min":8.327},"last":4.1260000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.571,"5min":0.95699999999999996,"15min":0.73699999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.7720000000000002,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.1359999999999999,"15min":0.81000000000000005},"min":{"1min":0.32700000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.1799999999999997,"5min":7.923,"15min":7.923},"last":0.995}]},{"osd":6,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":0.97199999999999998,"15min":0.745},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.7379999999999995,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.617,"5min":1.175,"15min":0.82299999999999995},"min":{"1min":0.39500000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.2069999999999999,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":0.96899999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6699999999999999,"5min":1.155,"15min":0.81799999999999995},"min":{"1min":0.41599999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":5.2309999999999999,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":1.0189999999999999},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.022,"15min":0.77100000000000002},"min":{"1min":0.42499999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":4.6399999999999997,"5min":8.2469999999999999,"15min":8.2469999999999999},"last":4.0810000000000004}]}]},{"osd":0,"up_from":80,"seq":343597383753,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":819796,"kb_used_data":721740,"kb_used_omap":28,"kb_used_meta":98019,"kb_avail":92925356,"statfs":{"total":95995035648,"available":95155564544,"internally_reserved":0,"allocated":739061760,"data_stored":738193358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":100372304},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:46:43 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.1679999999999999,"15min":0.63400000000000001},"min":{"1min":0.217,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.0449999999999999,"5min":25.475000000000001,"15min":25.475000000000001},"last":1.504},{"interface":"front","average":{"1min":2.0099999999999998,"5min":1.1919999999999999,"15min":0.64000000000000001},"min":{"1min":0.23499999999999999,"5min":0.191,"15min":0.191},"max":{"1min":6.2290000000000001,"5min":20.832999999999998,"15min":20.832999999999998},"last":3.4119999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":1.2350000000000001,"15min":0.64400000000000002},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.907,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":0.432},{"interface":"front","average":{"1min":1.754,"5min":1.5349999999999999,"15min":0.73999999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.5619999999999994,"5min":14.391999999999999,"15min":14.391999999999999},"last":3.0779999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9689999999999999,"5min":1.5309999999999999,"15min":0.94399999999999995},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":9.3889999999999993,"5min":14.926,"15min":14.926},"last":3.4409999999999998},{"interface":"front","average":{"1min":2.02,"5min":1.2569999999999999,"15min":0.78000000000000003},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":11.868,"5min":11.868,"15min":11.868},"last":1.018}]},{"osd":4,"last update":"Mon Oct 9 10:46:33 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":0.94899999999999995,"15min":0.72999999999999998},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.3949999999999996,"5min":5.71,"15min":5.71},"last":3.177},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.0509999999999999,"15min":0.76300000000000001},"min":{"1min":0.38900000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.093,"5min":9.9269999999999996,"15min":9.9269999999999996},"last":3.3039999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:46:33 2023","interfaces":[{"interface":"back","average":{"1min":1.8560000000000001,"5min":1.1200000000000001,"15min":0.79000000000000004},"min":{"1min":0.378,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.1299999999999999,"5min":9.9900000000000002,"15min":9.9900000000000002},"last":3.339},{"interface":"front","average":{"1min":1.149,"5min":0.95299999999999996,"15min":0.75},"min":{"1min":0.32300000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.9199999999999999,"5min":8.9190000000000005,"15min":8.9190000000000005},"last":0.35199999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:46:33 2023","interfaces":[{"interface":"back","average":{"1min":1.679,"5min":1.0920000000000001,"15min":0.78400000000000003},"min":{"1min":0.42699999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.1719999999999997,"5min":9.9659999999999993,"15min":9.9659999999999993},"last":3.3820000000000001},{"interface":"front","average":{"1min":1.1379999999999999,"5min":0.98399999999999999,"15min":0.76200000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.8890000000000002,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":0.38400000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:46:33 2023","interfaces":[{"interface":"back","average":{"1min":1.252,"5min":0.98699999999999999,"15min":0.76400000000000001},"min":{"1min":0.39700000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.9649999999999999,"5min":8.875,"15min":8.875},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.1679999999999999,"5min":1.0349999999999999,"15min":0.79700000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.4390000000000001,"5min":7.3899999999999997,"15min":7.3899999999999997},"last":4.2729999999999997}]}]},{"osd":2,"up_from":0,"seq":386547056694,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893168,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3011716,"kb_used_data":2831676,"kb_used_omap":7,"kb_used_meta":180024,"kb_avail":90733436,"statfs":{"total":95995035648,"available":92911038464,"internally_reserved":0,"allocated":2899636224,"data_stored":2898782434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":184345192},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2,3,0,3,2,4,3],"upper_bound":2048},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":2.734,"5min":0.995,"15min":0.54500000000000004},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":11.557,"5min":11.557,"15min":11.557},"last":5.4500000000000002},{"interface":"front","average":{"1min":1.359,"5min":0.93700000000000006,"15min":0.58299999999999996},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":4.1369999999999996,"5min":7.0510000000000002,"15min":7.0510000000000002},"last":0.57399999999999995}]},{"osd":1,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":1.4179999999999999,"15min":0.69199999999999995},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":13.967000000000001,"5min":14.707000000000001,"15min":14.707000000000001},"last":4.4850000000000003},{"interface":"front","average":{"1min":1.911,"5min":0.93899999999999995,"15min":0.56200000000000006},"min":{"1min":0.218,"5min":0.193,"15min":0.193},"max":{"1min":11.629,"5min":11.629,"15min":11.629},"last":5.4020000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1299999999999999,"5min":1.3280000000000001,"15min":0.68700000000000006},"min":{"1min":0.30099999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":13.853,"5min":13.853,"15min":13.853},"last":3.899},{"interface":"front","average":{"1min":1.514,"5min":1.0800000000000001,"15min":0.624},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.3019999999999996,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":5.7300000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":1.0229999999999999,"15min":0.72199999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":11.532,"5min":11.532,"15min":11.532},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.335,"5min":0.96899999999999997,"15min":0.72799999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.6070000000000002,"5min":5.6070000000000002,"15min":5.6070000000000002},"last":0.62}]},{"osd":5,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":4.3609999999999998,"5min":1.5680000000000001,"15min":0.91400000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":51.509999999999998,"5min":51.509999999999998,"15min":51.509999999999998},"last":0.65200000000000002},{"interface":"front","average":{"1min":5.141,"5min":1.99,"15min":1.038},"min":{"1min":0.32300000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":55.338999999999999,"5min":55.338999999999999,"15min":55.338999999999999},"last":3.7250000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":0.99199999999999999,"15min":0.72199999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.8490000000000002,"5min":8.4789999999999992,"15min":8.4789999999999992},"last":0.67700000000000005},{"interface":"front","average":{"1min":2.0150000000000001,"5min":1.421,"15min":0.873},"min":{"1min":0.378,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":11.470000000000001,"5min":13.891999999999999,"15min":13.891999999999999},"last":3.6720000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":4.9960000000000004,"5min":1.9970000000000001,"15min":1.0600000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":51.744,"5min":51.744,"15min":51.744},"last":3.7069999999999999},{"interface":"front","average":{"1min":4.6749999999999998,"5min":1.601,"15min":0.91600000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":53.399000000000001,"5min":53.399000000000001,"15min":53.399000000000001},"last":0.42399999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019043,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1681180,"kb_used_data":1375700,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92063972,"statfs":{"total":95995035648,"available":94273507328,"internally_reserved":0,"allocated":1408716800,"data_stored":1408073544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":0.90600000000000003,"15min":0.67400000000000004},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.387,"5min":10.387,"15min":10.387},"last":0.252},{"interface":"front","average":{"1min":1.5369999999999999,"5min":0.98299999999999998,"15min":0.71699999999999997},"min":{"1min":0.26600000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":10.311999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":1.3919999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:46:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":0.98599999999999999,"15min":0.69699999999999995},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.4189999999999996,"5min":6.907,"15min":6.907},"last":2.2639999999999998},{"interface":"front","average":{"1min":1.627,"5min":0.99099999999999999,"15min":0.745},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.1790000000000003,"5min":6.1790000000000003,"15min":6.1790000000000003},"last":1.4910000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:47:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2170000000000001,"5min":1.1000000000000001,"15min":0.747},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.2149999999999999,"5min":7.5449999999999999,"15min":7.5449999999999999},"last":7.1440000000000001},{"interface":"front","average":{"1min":1.579,"5min":1.1639999999999999,"15min":0.76300000000000001},"min":{"1min":0.36699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.5110000000000001,"5min":6.2939999999999996,"15min":6.2939999999999996},"last":1.4319999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:46:38 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.107,"15min":0.84299999999999997},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.0730000000000004,"5min":5.0730000000000004,"15min":5.0730000000000004},"last":0.93899999999999995},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.004,"15min":0.72599999999999998},"min":{"1min":0.35999999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":8.9809999999999999,"5min":8.9809999999999999,"15min":8.9809999999999999},"last":0.34300000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:46:41 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.73099999999999998,"15min":0.53500000000000003},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":3.9319999999999999,"5min":5.5410000000000004,"15min":5.5410000000000004},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.71099999999999997,"15min":0.52900000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":3.8980000000000001,"5min":3.8980000000000001,"15min":3.8980000000000001},"last":0.28699999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.74099999999999999,"15min":0.55700000000000005},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":7.1580000000000004,"5min":7.1580000000000004,"15min":7.1580000000000004},"last":0.375},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.71599999999999997,"15min":0.60899999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":3.6150000000000002,"5min":3.6150000000000002,"15min":3.6150000000000002},"last":0.39300000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.80800000000000005,"15min":0.64600000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.1050000000000004,"5min":4.1050000000000004,"15min":4.1050000000000004},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.748,"15min":0.60799999999999998},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0919999999999996,"5min":4.0919999999999996,"15min":4.0919999999999996},"last":0.40899999999999997}]}]},{"osd":5,"up_from":37,"seq":158913790112,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2071948,"kb_used_data":1774660,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91673204,"statfs":{"total":95995035648,"available":93873360896,"internally_reserved":0,"allocated":1817251840,"data_stored":1816620413,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.72199999999999998,"15min":0.63900000000000001},"min":{"1min":0.25800000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.722,"5min":4.8380000000000001,"15min":4.8380000000000001},"last":3.9860000000000002},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.78000000000000003,"15min":0.70599999999999996},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.7549999999999999,"5min":4.8650000000000002,"15min":4.8650000000000002},"last":4.0209999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":0.85599999999999998,"15min":0.68999999999999995},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.7789999999999999,"5min":3.7789999999999999,"15min":3.7789999999999999},"last":1.875},{"interface":"front","average":{"1min":1.1200000000000001,"5min":0.93100000000000005,"15min":0.70499999999999996},"min":{"1min":0.26200000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.77,"5min":7.165,"15min":7.165},"last":1.9970000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:47:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.02,"15min":0.72099999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.4569999999999999,"5min":7.9000000000000004,"15min":7.9000000000000004},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.7569999999999999,"5min":0.90400000000000003,"15min":0.68400000000000005},"min":{"1min":0.38,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":10.352,"5min":10.352,"15min":10.352},"last":0.26200000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:46:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5960000000000001,"5min":1.266,"15min":0.90400000000000003},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":9.984,"5min":9.984,"15min":9.984},"last":4.1349999999999998},{"interface":"front","average":{"1min":2.6509999999999998,"5min":1.2030000000000001,"15min":0.80900000000000005},"min":{"1min":0.34999999999999998,"5min":0.317,"15min":0.317},"max":{"1min":11.657,"5min":11.657,"15min":11.657},"last":2.484}]},{"osd":4,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.53200000000000003,"15min":0.46700000000000003},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":3.7919999999999998,"5min":3.7919999999999998,"15min":3.7919999999999998},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.048,"5min":0.68500000000000005,"15min":0.52900000000000003},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.9569999999999999,"5min":4.9569999999999999,"15min":4.9569999999999999},"last":2.4390000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.66300000000000003,"15min":0.53500000000000003},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":2.8900000000000001,"5min":2.9529999999999998,"15min":2.9529999999999998},"last":1.337},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.63300000000000001,"15min":0.53800000000000003},"min":{"1min":0.34100000000000003,"5min":0.33300000000000002,"15min":0.30299999999999999},"max":{"1min":3.8159999999999998,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":1.605}]},{"osd":7,"last update":"Mon Oct 9 10:46:21 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.67200000000000004,"15min":0.57899999999999996},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.782,"5min":4.0510000000000002,"15min":4.0510000000000002},"last":1.3049999999999999},{"interface":"front","average":{"1min":0.625,"5min":0.67200000000000004,"15min":0.57899999999999996},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":1.986,"5min":2.6869999999999998,"15min":2.6869999999999998},"last":1.702}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":731648000,"data_stored":731643928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1725501440,"data_stored":1725497368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3018457088,"data_stored":3018457088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2877751296,"data_stored":2877751296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1400766464,"data_stored":1400766464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1810698240,"data_stored":1810694168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1490223104,"data_stored":1490223104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1956184064,"data_stored":1956184064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:47:27.898 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:47:27.899 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:27.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:27.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:27.963 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:28.002 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:28.048 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:28.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:28.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:28.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:28.192 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:28.232 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:28 smithi125 ceph-mon[195507]: from='client.54751 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:47:28.232 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:28 smithi125 ceph-mon[195507]: pgmap v359: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 34009/197910 objects degraded (17.184%); 7944/197910 objects misplaced (4.014%); 8.9 MiB/s, 175 objects/s recovering 2023-10-09T10:47:28.232 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:28 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 34009/197910 objects degraded (17.184%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:28.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:28 smithi125 ceph-mon[200197]: from='client.54751 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:47:28.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:28 smithi125 ceph-mon[200197]: pgmap v359: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 34009/197910 objects degraded (17.184%); 7944/197910 objects misplaced (4.014%); 8.9 MiB/s, 175 objects/s recovering 2023-10-09T10:47:28.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:28 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 34009/197910 objects degraded (17.184%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:28.236 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:28.414 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:28.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:28.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:28.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:28 smithi155 ceph-mon[184854]: from='client.54751 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:47:28.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:28 smithi155 ceph-mon[184854]: pgmap v359: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 34009/197910 objects degraded (17.184%); 7944/197910 objects misplaced (4.014%); 8.9 MiB/s, 175 objects/s recovering 2023-10-09T10:47:28.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:28 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 34009/197910 objects degraded (17.184%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:28.519 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:28.575 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:28.594 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:28.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:28.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:28.666 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:47:28.667 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:28.699 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:28.728 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:28.753 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:28.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:28.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:28.857 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:28.903 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:28.927 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:28.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:28.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:29.032 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:29.082 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:29.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:29.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:29.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:29.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:29.264 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:29.290 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:47:29.290 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:47:29.340 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:29.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:29.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:29.465 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:29.506 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:29.533 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:29.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:29.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:29.642 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:29.684 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:47:29.688 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:29.724 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:29.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:29.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:29.830 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:29.882 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:30.018 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:30.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:30.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:30.102 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:30.124 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:30.161 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:30.217 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:30.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:30.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:30.260 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:30 smithi125 ceph-mon[195507]: pgmap v360: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 34009/197472 objects degraded (17.222%); 7944/197472 objects misplaced (4.023%); 7.3 MiB/s, 143 objects/s recovering 2023-10-09T10:47:30.260 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:47:30.261 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:30 smithi125 ceph-mon[200197]: pgmap v360: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 34009/197472 objects degraded (17.222%); 7944/197472 objects misplaced (4.023%); 7.3 MiB/s, 143 objects/s recovering 2023-10-09T10:47:30.261 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:47:30.321 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:30.386 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:30 smithi155 ceph-mon[184854]: pgmap v360: 119 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 34009/197472 objects degraded (17.222%); 7944/197472 objects misplaced (4.023%); 7.3 MiB/s, 143 objects/s recovering 2023-10-09T10:47:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:47:30.743 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:30.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:30.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:30.760 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:47:30.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:30.902 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:30.932 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:30.940 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:30.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:30.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:30.983 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:47:30.989 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:47:31.050 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:31.085 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:31.305 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:31.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:31.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:31.312 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:31 smithi125 ceph-mon[195507]: osdmap e172: 8 total, 8 up, 7 in 2023-10-09T10:47:31.312 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:31 smithi125 ceph-mon[195507]: failed to encode map e172 with expected crc 2023-10-09T10:47:31.312 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:31 smithi125 ceph-mon[195507]: failed to encode map e172 with expected crc 2023-10-09T10:47:31.312 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:31 smithi125 ceph-mon[195507]: failed to encode map e172 with expected crc 2023-10-09T10:47:31.315 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:47:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:47:31] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.43.0" 2023-10-09T10:47:31.316 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:31 smithi125 ceph-mon[200197]: osdmap e172: 8 total, 8 up, 7 in 2023-10-09T10:47:31.316 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:31 smithi125 ceph-mon[200197]: failed to encode map e172 with expected crc 2023-10-09T10:47:31.316 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:31 smithi125 ceph-mon[200197]: failed to encode map e172 with expected crc 2023-10-09T10:47:31.316 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:31 smithi125 ceph-mon[200197]: failed to encode map e172 with expected crc 2023-10-09T10:47:31.412 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:31.469 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:31 smithi155 ceph-mon[184854]: osdmap e172: 8 total, 8 up, 7 in 2023-10-09T10:47:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:31 smithi155 ceph-mon[184854]: failed to encode map e172 with expected crc 2023-10-09T10:47:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:31 smithi155 ceph-mon[184854]: failed to encode map e172 with expected crc 2023-10-09T10:47:31.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:31 smithi155 ceph-mon[184854]: failed to encode map e172 with expected crc 2023-10-09T10:47:31.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:31.517 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:31.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:31.617 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:31.689 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:31.860 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":361,"stamp":"2023-10-09T10:47:30.213199+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4241987512,"num_objects":64744,"num_object_clones":0,"num_object_copies":194232,"num_objects_missing_on_primary":1415,"num_objects_missing":1415,"num_objects_degraded":34009,"num_objects_misplaced":7612,"num_objects_unfound":0,"num_objects_dirty":64744,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":139788,"num_write_kb":6260974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19874,"num_bytes_recovered":1251609346,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":158969,"ondisk_log_size":158969,"up":357,"acting":345,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12950872,"kb_used_data":11371680,"kb_used_omap":44,"kb_used_meta":1579091,"kb_avail":643265192,"statfs":{"total":671965249536,"available":658703556608,"internally_reserved":0,"allocated":11644600320,"data_stored":11639432629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1616989601},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,4,0,1,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-269680640,"num_objects":-4115,"num_object_clones":0,"num_object_copies":-12345,"num_objects_missing_on_primary":-169,"num_objects_missing":-169,"num_objects_degraded":0,"num_objects_misplaced":-1362,"num_objects_unfound":0,"num_objects_dirty":-4115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4115,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":909,"num_bytes_recovered":42008576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.111638"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":164,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931272+0000","last_change":"2023-10-09T10:46:32.250979+0000","last_active":"2023-10-09T10:47:18.931272+0000","last_peered":"2023-10-09T10:47:18.931272+0000","last_clean":"2023-10-09T10:47:18.931272+0000","last_became_active":"2023-10-09T10:46:32.250625+0000","last_became_peered":"2023-10-09T10:46:32.250625+0000","last_unstale":"2023-10-09T10:47:18.931272+0000","last_undegraded":"2023-10-09T10:47:18.931272+0000","last_fullsized":"2023-10-09T10:47:18.931272+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:34.858348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":139,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.930701+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:47:18.930701+0000","last_peered":"2023-10-09T10:47:18.930701+0000","last_clean":"2023-10-09T10:47:18.930701+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:47:18.930701+0000","last_undegraded":"2023-10-09T10:47:18.930701+0000","last_fullsized":"2023-10-09T10:47:18.930701+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":123,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T10:47:25.197951+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:47:25.197951+0000","last_peered":"2023-10-09T10:47:25.197951+0000","last_clean":"2023-10-09T10:47:25.197951+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:47:25.197951+0000","last_undegraded":"2023-10-09T10:47:25.197951+0000","last_fullsized":"2023-10-09T10:47:25.197951+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":39,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.911588+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:47:18.911588+0000","last_peered":"2023-10-09T10:47:18.911588+0000","last_clean":"2023-10-09T10:47:18.911588+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:47:18.911588+0000","last_undegraded":"2023-10-09T10:47:18.911588+0000","last_fullsized":"2023-10-09T10:47:18.911588+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":175,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.926678+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:47:18.926678+0000","last_peered":"2023-10-09T10:47:18.926678+0000","last_clean":"2023-10-09T10:47:18.926678+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:47:18.926678+0000","last_undegraded":"2023-10-09T10:47:18.926678+0000","last_fullsized":"2023-10-09T10:47:18.926678+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":109,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.917839+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:47:18.917839+0000","last_peered":"2023-10-09T10:47:18.917839+0000","last_clean":"2023-10-09T10:47:18.917839+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:47:18.917839+0000","last_undegraded":"2023-10-09T10:47:18.917839+0000","last_fullsized":"2023-10-09T10:47:18.917839+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"171'5748","reported_seq":9893,"reported_epoch":171,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:25.858616+0000","last_change":"2023-10-09T10:46:33.521602+0000","last_active":"2023-10-09T10:47:25.858616+0000","last_peered":"2023-10-09T10:47:25.858616+0000","last_clean":"2023-10-09T10:46:31.055751+0000","last_became_active":"2023-10-09T10:46:33.364058+0000","last_became_peered":"2023-10-09T10:46:33.364058+0000","last_unstale":"2023-10-09T10:47:25.858616+0000","last_undegraded":"2023-10-09T10:47:25.858616+0000","last_fullsized":"2023-10-09T10:47:25.858616+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5748,"log_dups_size":0,"ondisk_log_size":5748,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140607488,"num_objects":2146,"num_object_clones":0,"num_object_copies":6438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2144,"num_objects_unfound":0,"num_objects_dirty":2146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4712,"num_write_kb":219424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2010,"num_bytes_recovered":131727360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":2146}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":39,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.911893+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:47:18.911893+0000","last_peered":"2023-10-09T10:47:18.911893+0000","last_clean":"2023-10-09T10:47:18.911893+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:47:18.911893+0000","last_undegraded":"2023-10-09T10:47:18.911893+0000","last_fullsized":"2023-10-09T10:47:18.911893+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":139,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.926738+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:47:18.926738+0000","last_peered":"2023-10-09T10:47:18.926738+0000","last_clean":"2023-10-09T10:47:18.926738+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:47:18.926738+0000","last_undegraded":"2023-10-09T10:47:18.926738+0000","last_fullsized":"2023-10-09T10:47:18.926738+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":504,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.911500+0000","last_change":"2023-10-09T10:46:05.620365+0000","last_active":"2023-10-09T10:47:18.911500+0000","last_peered":"2023-10-09T10:47:18.911500+0000","last_clean":"2023-10-09T10:47:18.911500+0000","last_became_active":"2023-10-09T10:46:05.620024+0000","last_became_peered":"2023-10-09T10:46:05.620024+0000","last_unstale":"2023-10-09T10:47:18.911500+0000","last_undegraded":"2023-10-09T10:47:18.911500+0000","last_fullsized":"2023-10-09T10:47:18.911500+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.13","version":"171'5613","reported_seq":5790,"reported_epoch":171,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:25.796785+0000","last_change":"2023-10-09T10:46:33.470698+0000","last_active":"2023-10-09T10:47:25.796785+0000","last_peered":"2023-10-09T10:47:25.796785+0000","last_clean":"2023-10-09T10:46:31.195699+0000","last_became_active":"2023-10-09T10:46:33.455632+0000","last_became_peered":"2023-10-09T10:46:33.455632+0000","last_unstale":"2023-10-09T10:47:25.796785+0000","last_undegraded":"2023-10-09T10:46:33.269323+0000","last_fullsized":"2023-10-09T10:46:33.241226+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":5613,"log_dups_size":0,"ondisk_log_size":5613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:07:12.547333+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134938624,"num_objects":2059,"num_object_clones":0,"num_object_copies":6177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3315,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2898,"num_write_kb":105088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":2059},{"shards":"6,7","objects":-716}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":136,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.917614+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:47:18.917614+0000","last_peered":"2023-10-09T10:47:18.917614+0000","last_clean":"2023-10-09T10:47:18.917614+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:47:18.917614+0000","last_undegraded":"2023-10-09T10:47:18.917614+0000","last_fullsized":"2023-10-09T10:47:18.917614+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":96,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.959045+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:47:18.959045+0000","last_peered":"2023-10-09T10:47:18.959045+0000","last_clean":"2023-10-09T10:47:18.959045+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:47:18.959045+0000","last_undegraded":"2023-10-09T10:47:18.959045+0000","last_fullsized":"2023-10-09T10:47:18.959045+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"171'5745","reported_seq":13863,"reported_epoch":171,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:26.406925+0000","last_change":"2023-10-09T10:46:03.683196+0000","last_active":"2023-10-09T10:47:26.406925+0000","last_peered":"2023-10-09T10:47:26.406925+0000","last_clean":"2023-10-09T10:46:02.177309+0000","last_became_active":"2023-10-09T10:46:03.681058+0000","last_became_peered":"2023-10-09T10:46:03.681058+0000","last_unstale":"2023-10-09T10:47:26.406925+0000","last_undegraded":"2023-10-09T10:47:26.406925+0000","last_fullsized":"2023-10-09T10:47:26.406925+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136871936,"num_objects":2089,"num_object_clones":0,"num_object_copies":6267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2089,"num_objects_unfound":0,"num_objects_dirty":2089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4699,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[1,5,6],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":2089}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":92,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.930630+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:47:18.930630+0000","last_peered":"2023-10-09T10:47:18.930630+0000","last_clean":"2023-10-09T10:47:18.930630+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:47:18.930630+0000","last_undegraded":"2023-10-09T10:47:18.930630+0000","last_fullsized":"2023-10-09T10:47:18.930630+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":132,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.909119+0000","last_change":"2023-10-09T10:46:32.249921+0000","last_active":"2023-10-09T10:47:18.909119+0000","last_peered":"2023-10-09T10:47:18.909119+0000","last_clean":"2023-10-09T10:47:18.909119+0000","last_became_active":"2023-10-09T10:46:32.249590+0000","last_became_peered":"2023-10-09T10:46:32.249590+0000","last_unstale":"2023-10-09T10:47:18.909119+0000","last_undegraded":"2023-10-09T10:47:18.909119+0000","last_fullsized":"2023-10-09T10:47:18.909119+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:25.169752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"171'5629","reported_seq":11129,"reported_epoch":171,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:47:29.540724+0000","last_change":"2023-10-09T10:46:32.280686+0000","last_active":"2023-10-09T10:47:29.540724+0000","last_peered":"2023-10-09T10:47:29.540724+0000","last_clean":"2023-10-09T10:44:56.105580+0000","last_became_active":"2023-10-09T10:46:07.402111+0000","last_became_peered":"2023-10-09T10:46:07.402111+0000","last_unstale":"2023-10-09T10:47:29.540724+0000","last_undegraded":"2023-10-09T10:47:29.540724+0000","last_fullsized":"2023-10-09T10:46:05.812663+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5629,"log_dups_size":0,"ondisk_log_size":5629,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127893504,"num_objects":1952,"num_object_clones":0,"num_object_copies":5856,"num_objects_missing_on_primary":9,"num_objects_missing":9,"num_objects_degraded":0,"num_objects_misplaced":1366,"num_objects_unfound":0,"num_objects_dirty":1952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4622,"num_write_kb":210336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1352,"num_bytes_recovered":88604672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,2,3,5","objects":1319},{"shards":"1,2,3,5,6","objects":1},{"shards":"1,2,3,6","objects":9},{"shards":"1,2,6","objects":9},{"shards":"1,3,5","objects":1},{"shards":"3,5","objects":613}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":141,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931507+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:47:18.931507+0000","last_peered":"2023-10-09T10:47:18.931507+0000","last_clean":"2023-10-09T10:47:18.931507+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:47:18.931507+0000","last_undegraded":"2023-10-09T10:47:18.931507+0000","last_fullsized":"2023-10-09T10:47:18.931507+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":135,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931451+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:47:18.931451+0000","last_peered":"2023-10-09T10:47:18.931451+0000","last_clean":"2023-10-09T10:47:18.931451+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:47:18.931451+0000","last_undegraded":"2023-10-09T10:47:18.931451+0000","last_fullsized":"2023-10-09T10:47:18.931451+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":504,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.912920+0000","last_change":"2023-10-09T10:46:05.618276+0000","last_active":"2023-10-09T10:47:18.912920+0000","last_peered":"2023-10-09T10:47:18.912920+0000","last_clean":"2023-10-09T10:47:18.912920+0000","last_became_active":"2023-10-09T10:46:05.616509+0000","last_became_peered":"2023-10-09T10:46:05.616509+0000","last_unstale":"2023-10-09T10:47:18.912920+0000","last_undegraded":"2023-10-09T10:47:18.912920+0000","last_fullsized":"2023-10-09T10:47:18.912920+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.16","version":"171'5642","reported_seq":5738,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T10:47:29.501087+0000","last_change":"2023-10-09T10:46:03.592682+0000","last_active":"2023-10-09T10:47:29.501087+0000","last_peered":"2023-10-09T10:47:29.501087+0000","last_clean":"2023-10-09T10:47:29.501087+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:47:29.501087+0000","last_undegraded":"2023-10-09T10:47:29.501087+0000","last_fullsized":"2023-10-09T10:47:29.501087+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5642,"log_dups_size":0,"ondisk_log_size":5642,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130220032,"num_objects":1987,"num_object_clones":0,"num_object_copies":5961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4599,"num_write_kb":210752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":153,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.920004+0000","last_change":"2023-10-09T10:46:32.250256+0000","last_active":"2023-10-09T10:47:18.920004+0000","last_peered":"2023-10-09T10:47:18.920004+0000","last_clean":"2023-10-09T10:47:18.920004+0000","last_became_active":"2023-10-09T10:46:32.243312+0000","last_became_peered":"2023-10-09T10:46:32.243312+0000","last_unstale":"2023-10-09T10:47:18.920004+0000","last_undegraded":"2023-10-09T10:47:18.920004+0000","last_fullsized":"2023-10-09T10:47:18.920004+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:08.059777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":162,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.932686+0000","last_change":"2023-10-09T10:46:32.237577+0000","last_active":"2023-10-09T10:47:18.932686+0000","last_peered":"2023-10-09T10:47:18.932686+0000","last_clean":"2023-10-09T10:47:18.932686+0000","last_became_active":"2023-10-09T10:46:32.237316+0000","last_became_peered":"2023-10-09T10:46:32.237316+0000","last_unstale":"2023-10-09T10:47:18.932686+0000","last_undegraded":"2023-10-09T10:47:18.932686+0000","last_fullsized":"2023-10-09T10:47:18.932686+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:39.684602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":505,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910651+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:47:18.910651+0000","last_peered":"2023-10-09T10:47:18.910651+0000","last_clean":"2023-10-09T10:47:18.910651+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:47:18.910651+0000","last_undegraded":"2023-10-09T10:47:18.910651+0000","last_fullsized":"2023-10-09T10:47:18.910651+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.17","version":"171'5510","reported_seq":5600,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T10:47:25.910579+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:47:25.910579+0000","last_peered":"2023-10-09T10:47:25.910579+0000","last_clean":"2023-10-09T10:47:25.910579+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:47:25.910579+0000","last_undegraded":"2023-10-09T10:47:25.910579+0000","last_fullsized":"2023-10-09T10:47:25.910579+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5510,"log_dups_size":0,"ondisk_log_size":5510,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131334144,"num_objects":2004,"num_object_clones":0,"num_object_copies":6012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4502,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":39,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910873+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:47:18.910873+0000","last_peered":"2023-10-09T10:47:18.910873+0000","last_clean":"2023-10-09T10:47:18.910873+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:47:18.910873+0000","last_undegraded":"2023-10-09T10:47:18.910873+0000","last_fullsized":"2023-10-09T10:47:18.910873+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":135,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.918034+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:47:18.918034+0000","last_peered":"2023-10-09T10:47:18.918034+0000","last_clean":"2023-10-09T10:47:18.918034+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:47:18.918034+0000","last_undegraded":"2023-10-09T10:47:18.918034+0000","last_fullsized":"2023-10-09T10:47:18.918034+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":505,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.911853+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:47:18.911853+0000","last_peered":"2023-10-09T10:47:18.911853+0000","last_clean":"2023-10-09T10:47:18.911853+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:47:18.911853+0000","last_undegraded":"2023-10-09T10:47:18.911853+0000","last_fullsized":"2023-10-09T10:47:18.911853+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"171'5699","reported_seq":5794,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T10:47:26.370655+0000","last_change":"2023-10-09T10:46:33.243540+0000","last_active":"2023-10-09T10:47:26.370655+0000","last_peered":"2023-10-09T10:47:26.370655+0000","last_clean":"2023-10-09T10:47:26.370655+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:47:26.370655+0000","last_undegraded":"2023-10-09T10:47:26.370655+0000","last_fullsized":"2023-10-09T10:47:26.370655+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5699,"log_dups_size":0,"ondisk_log_size":5699,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132939776,"num_objects":2029,"num_object_clones":0,"num_object_copies":6087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4645,"num_write_kb":213536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":81,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931227+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:47:18.931227+0000","last_peered":"2023-10-09T10:47:18.931227+0000","last_clean":"2023-10-09T10:47:18.931227+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:47:18.931227+0000","last_undegraded":"2023-10-09T10:47:18.931227+0000","last_fullsized":"2023-10-09T10:47:18.931227+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":39,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.911792+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:47:18.911792+0000","last_peered":"2023-10-09T10:47:18.911792+0000","last_clean":"2023-10-09T10:47:18.911792+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:47:18.911792+0000","last_undegraded":"2023-10-09T10:47:18.911792+0000","last_fullsized":"2023-10-09T10:47:18.911792+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":505,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910653+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:47:18.910653+0000","last_peered":"2023-10-09T10:47:18.910653+0000","last_clean":"2023-10-09T10:47:18.910653+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:47:18.910653+0000","last_undegraded":"2023-10-09T10:47:18.910653+0000","last_fullsized":"2023-10-09T10:47:18.910653+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"171'5747","reported_seq":5852,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T10:47:29.543763+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:47:29.543763+0000","last_peered":"2023-10-09T10:47:29.543763+0000","last_clean":"2023-10-09T10:47:29.543763+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:47:29.543763+0000","last_undegraded":"2023-10-09T10:47:29.543763+0000","last_fullsized":"2023-10-09T10:47:29.543763+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5747,"log_dups_size":0,"ondisk_log_size":5747,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129892352,"num_objects":1982,"num_object_clones":0,"num_object_copies":5946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4686,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":49,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910568+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:47:18.910568+0000","last_peered":"2023-10-09T10:47:18.910568+0000","last_clean":"2023-10-09T10:47:18.910568+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:47:18.910568+0000","last_undegraded":"2023-10-09T10:47:18.910568+0000","last_fullsized":"2023-10-09T10:47:18.910568+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":139,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931936+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:47:18.931936+0000","last_peered":"2023-10-09T10:47:18.931936+0000","last_clean":"2023-10-09T10:47:18.931936+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:47:18.931936+0000","last_undegraded":"2023-10-09T10:47:18.931936+0000","last_fullsized":"2023-10-09T10:47:18.931936+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":505,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.911172+0000","last_change":"2023-10-09T10:46:05.620574+0000","last_active":"2023-10-09T10:47:18.911172+0000","last_peered":"2023-10-09T10:47:18.911172+0000","last_clean":"2023-10-09T10:47:18.911172+0000","last_became_active":"2023-10-09T10:46:05.620212+0000","last_became_peered":"2023-10-09T10:46:05.620212+0000","last_unstale":"2023-10-09T10:47:18.911172+0000","last_undegraded":"2023-10-09T10:47:18.911172+0000","last_fullsized":"2023-10-09T10:47:18.911172+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"171'9498","reported_seq":13644,"reported_epoch":171,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:26.015938+0000","last_change":"2023-10-09T10:46:32.247903+0000","last_active":"2023-10-09T10:47:26.015938+0000","last_peered":"2023-10-09T10:47:26.015938+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:46:02.739355+0000","last_became_peered":"2023-10-09T10:46:02.739355+0000","last_unstale":"2023-10-09T10:47:26.015938+0000","last_undegraded":"2023-10-09T10:46:02.608805+0000","last_fullsized":"2023-10-09T10:46:02.583589+0000","mapping_epoch":142,"log_start":"103'600","ondisk_log_start":"103'600","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8898,"log_dups_size":0,"ondisk_log_size":8898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:32.899634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270925824,"num_objects":4134,"num_object_clones":0,"num_object_copies":12402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9498,"num_write_kb":436224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2022,"num_bytes_recovered":132513792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":625},{"shards":"5,6","objects":3508}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":164,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.920341+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:47:18.920341+0000","last_peered":"2023-10-09T10:47:18.920341+0000","last_clean":"2023-10-09T10:47:18.920341+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:47:18.920341+0000","last_undegraded":"2023-10-09T10:47:18.920341+0000","last_fullsized":"2023-10-09T10:47:18.920341+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":118,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.917402+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:47:18.917402+0000","last_peered":"2023-10-09T10:47:18.917402+0000","last_clean":"2023-10-09T10:47:18.917402+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:47:18.917402+0000","last_undegraded":"2023-10-09T10:47:18.917402+0000","last_fullsized":"2023-10-09T10:47:18.917402+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":485,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.911635+0000","last_change":"2023-10-09T10:46:02.631378+0000","last_active":"2023-10-09T10:47:18.911635+0000","last_peered":"2023-10-09T10:47:18.911635+0000","last_clean":"2023-10-09T10:47:18.911635+0000","last_became_active":"2023-10-09T10:46:02.629142+0000","last_became_peered":"2023-10-09T10:46:02.629142+0000","last_unstale":"2023-10-09T10:47:18.911635+0000","last_undegraded":"2023-10-09T10:47:18.911635+0000","last_fullsized":"2023-10-09T10:47:18.911635+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.b","version":"171'9256","reported_seq":9339,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T10:47:29.441070+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:47:29.441070+0000","last_peered":"2023-10-09T10:47:29.441070+0000","last_clean":"2023-10-09T10:47:29.441070+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:47:29.441070+0000","last_undegraded":"2023-10-09T10:47:29.441070+0000","last_fullsized":"2023-10-09T10:47:29.441070+0000","mapping_epoch":98,"log_start":"117'3525","ondisk_log_start":"117'3525","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5731,"log_dups_size":0,"ondisk_log_size":5731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257163264,"num_objects":3924,"num_object_clones":0,"num_object_copies":11772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9256,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":137,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.923451+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:47:18.923451+0000","last_peered":"2023-10-09T10:47:18.923451+0000","last_clean":"2023-10-09T10:47:18.923451+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:47:18.923451+0000","last_undegraded":"2023-10-09T10:47:18.923451+0000","last_fullsized":"2023-10-09T10:47:18.923451+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":172,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931190+0000","last_change":"2023-10-09T10:46:32.251035+0000","last_active":"2023-10-09T10:47:18.931190+0000","last_peered":"2023-10-09T10:47:18.931190+0000","last_clean":"2023-10-09T10:47:18.931190+0000","last_became_active":"2023-10-09T10:46:32.250293+0000","last_became_peered":"2023-10-09T10:46:32.250293+0000","last_unstale":"2023-10-09T10:47:18.931190+0000","last_undegraded":"2023-10-09T10:47:18.931190+0000","last_fullsized":"2023-10-09T10:47:18.931190+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.787766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":505,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910619+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:47:18.910619+0000","last_peered":"2023-10-09T10:47:18.910619+0000","last_clean":"2023-10-09T10:47:18.910619+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:47:18.910619+0000","last_undegraded":"2023-10-09T10:47:18.910619+0000","last_fullsized":"2023-10-09T10:47:18.910619+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"171'5733","reported_seq":5836,"reported_epoch":171,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:29.399872+0000","last_change":"2023-10-09T10:46:33.540719+0000","last_active":"2023-10-09T10:47:29.399872+0000","last_peered":"2023-10-09T10:47:29.399872+0000","last_clean":"2023-10-09T10:46:31.137696+0000","last_became_active":"2023-10-09T10:46:33.527650+0000","last_became_peered":"2023-10-09T10:46:33.527650+0000","last_unstale":"2023-10-09T10:47:29.399872+0000","last_undegraded":"2023-10-09T10:46:33.285468+0000","last_fullsized":"2023-10-09T10:46:33.235683+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5733,"log_dups_size":0,"ondisk_log_size":5733,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132284416,"num_objects":2018,"num_object_clones":0,"num_object_copies":6054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4668,"num_write_kb":213984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2006},{"shards":"3,7","objects":-791}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":92,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.917963+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:47:18.917963+0000","last_peered":"2023-10-09T10:47:18.917963+0000","last_clean":"2023-10-09T10:47:18.917963+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:47:18.917963+0000","last_undegraded":"2023-10-09T10:47:18.917963+0000","last_fullsized":"2023-10-09T10:47:18.917963+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":109,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:21.512810+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:47:21.512810+0000","last_peered":"2023-10-09T10:47:21.512810+0000","last_clean":"2023-10-09T10:47:21.512810+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:47:21.512810+0000","last_undegraded":"2023-10-09T10:47:21.512810+0000","last_fullsized":"2023-10-09T10:47:21.512810+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":505,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.911433+0000","last_change":"2023-10-09T10:46:05.620030+0000","last_active":"2023-10-09T10:47:18.911433+0000","last_peered":"2023-10-09T10:47:18.911433+0000","last_clean":"2023-10-09T10:47:18.911433+0000","last_became_active":"2023-10-09T10:46:05.619694+0000","last_became_peered":"2023-10-09T10:46:05.619694+0000","last_unstale":"2023-10-09T10:47:18.911433+0000","last_undegraded":"2023-10-09T10:47:18.911433+0000","last_fullsized":"2023-10-09T10:47:18.911433+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.9","version":"171'5669","reported_seq":5783,"reported_epoch":171,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:27.411327+0000","last_change":"2023-10-09T10:46:34.405499+0000","last_active":"2023-10-09T10:47:27.411327+0000","last_peered":"2023-10-09T10:47:27.411327+0000","last_clean":"2023-10-09T10:46:31.190603+0000","last_became_active":"2023-10-09T10:46:34.396448+0000","last_became_peered":"2023-10-09T10:46:34.396448+0000","last_unstale":"2023-10-09T10:47:27.411327+0000","last_undegraded":"2023-10-09T10:46:33.246949+0000","last_fullsized":"2023-10-09T10:46:33.237968+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5669,"log_dups_size":0,"ondisk_log_size":5669,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130220032,"num_objects":1987,"num_object_clones":0,"num_object_copies":5961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2004,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4627,"num_write_kb":211648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1303,"num_bytes_recovered":85393408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":684},{"shards":"0,7","objects":519}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":136,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.917736+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:47:18.917736+0000","last_peered":"2023-10-09T10:47:18.917736+0000","last_clean":"2023-10-09T10:47:18.917736+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:47:18.917736+0000","last_undegraded":"2023-10-09T10:47:18.917736+0000","last_fullsized":"2023-10-09T10:47:18.917736+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":39,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.912250+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:47:18.912250+0000","last_peered":"2023-10-09T10:47:18.912250+0000","last_clean":"2023-10-09T10:47:18.912250+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:47:18.912250+0000","last_undegraded":"2023-10-09T10:47:18.912250+0000","last_fullsized":"2023-10-09T10:47:18.912250+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":496,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.912754+0000","last_change":"2023-10-09T10:46:05.618134+0000","last_active":"2023-10-09T10:47:18.912754+0000","last_peered":"2023-10-09T10:47:18.912754+0000","last_clean":"2023-10-09T10:47:18.912754+0000","last_became_active":"2023-10-09T10:46:05.616057+0000","last_became_peered":"2023-10-09T10:46:05.616057+0000","last_unstale":"2023-10-09T10:47:18.912754+0000","last_undegraded":"2023-10-09T10:47:18.912754+0000","last_fullsized":"2023-10-09T10:47:18.912754+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"171'9116","reported_seq":9206,"reported_epoch":171,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:26.006400+0000","last_change":"2023-10-09T10:46:33.576530+0000","last_active":"2023-10-09T10:47:26.006400+0000","last_peered":"2023-10-09T10:47:26.006400+0000","last_clean":"2023-10-09T10:46:31.183481+0000","last_became_active":"2023-10-09T10:46:33.542304+0000","last_became_peered":"2023-10-09T10:46:33.542304+0000","last_unstale":"2023-10-09T10:47:26.006400+0000","last_undegraded":"2023-10-09T10:46:33.300581+0000","last_fullsized":"2023-10-09T10:46:33.246047+0000","mapping_epoch":154,"log_start":"105'1777","ondisk_log_start":"105'1777","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7339,"log_dups_size":0,"ondisk_log_size":7339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:26.739488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269877248,"num_objects":4118,"num_object_clones":0,"num_object_copies":12354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6617,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9116,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":4118},{"shards":"1,4","objects":-1443}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":163,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.930514+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:47:18.930514+0000","last_peered":"2023-10-09T10:47:18.930514+0000","last_clean":"2023-10-09T10:47:18.930514+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:47:18.930514+0000","last_undegraded":"2023-10-09T10:47:18.930514+0000","last_fullsized":"2023-10-09T10:47:18.930514+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":135,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931391+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:47:18.931391+0000","last_peered":"2023-10-09T10:47:18.931391+0000","last_clean":"2023-10-09T10:47:18.931391+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:47:18.931391+0000","last_undegraded":"2023-10-09T10:47:18.931391+0000","last_fullsized":"2023-10-09T10:47:18.931391+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":485,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910738+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:47:18.910738+0000","last_peered":"2023-10-09T10:47:18.910738+0000","last_clean":"2023-10-09T10:47:18.910738+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:47:18.910738+0000","last_undegraded":"2023-10-09T10:47:18.910738+0000","last_fullsized":"2023-10-09T10:47:18.910738+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.f","version":"171'9154","reported_seq":9243,"reported_epoch":171,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:29.503353+0000","last_change":"2023-10-09T10:46:33.651756+0000","last_active":"2023-10-09T10:47:29.503353+0000","last_peered":"2023-10-09T10:47:29.503353+0000","last_clean":"2023-10-09T10:46:31.156458+0000","last_became_active":"2023-10-09T10:46:33.621003+0000","last_became_peered":"2023-10-09T10:46:33.621003+0000","last_unstale":"2023-10-09T10:47:29.503353+0000","last_undegraded":"2023-10-09T10:46:33.309813+0000","last_fullsized":"2023-10-09T10:46:33.236508+0000","mapping_epoch":154,"log_start":"116'3410","ondisk_log_start":"116'3410","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5744,"log_dups_size":0,"ondisk_log_size":5744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:13.704346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253493248,"num_objects":3868,"num_object_clones":0,"num_object_copies":11604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9154,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":3868},{"shards":"3,4","objects":-1584}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":110,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931510+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:47:18.931510+0000","last_peered":"2023-10-09T10:47:18.931510+0000","last_clean":"2023-10-09T10:47:18.931510+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:47:18.931510+0000","last_undegraded":"2023-10-09T10:47:18.931510+0000","last_fullsized":"2023-10-09T10:47:18.931510+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":162,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931556+0000","last_change":"2023-10-09T10:46:32.245583+0000","last_active":"2023-10-09T10:47:18.931556+0000","last_peered":"2023-10-09T10:47:18.931556+0000","last_clean":"2023-10-09T10:47:18.931556+0000","last_became_active":"2023-10-09T10:46:32.245271+0000","last_became_peered":"2023-10-09T10:46:32.245271+0000","last_unstale":"2023-10-09T10:47:18.931556+0000","last_undegraded":"2023-10-09T10:47:18.931556+0000","last_fullsized":"2023-10-09T10:47:18.931556+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.025051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":505,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.912015+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:47:18.912015+0000","last_peered":"2023-10-09T10:47:18.912015+0000","last_clean":"2023-10-09T10:47:18.912015+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:47:18.912015+0000","last_undegraded":"2023-10-09T10:47:18.912015+0000","last_fullsized":"2023-10-09T10:47:18.912015+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"171'9392","reported_seq":9475,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T10:47:29.482788+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:47:29.482788+0000","last_peered":"2023-10-09T10:47:29.482788+0000","last_clean":"2023-10-09T10:47:29.482788+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:47:29.482788+0000","last_undegraded":"2023-10-09T10:47:29.482788+0000","last_fullsized":"2023-10-09T10:47:29.482788+0000","mapping_epoch":98,"log_start":"118'3678","ondisk_log_start":"118'3678","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5714,"log_dups_size":0,"ondisk_log_size":5714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255066112,"num_objects":3892,"num_object_clones":0,"num_object_copies":11676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9392,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":146,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931104+0000","last_change":"2023-10-09T10:46:32.245685+0000","last_active":"2023-10-09T10:47:18.931104+0000","last_peered":"2023-10-09T10:47:18.931104+0000","last_clean":"2023-10-09T10:47:18.931104+0000","last_became_active":"2023-10-09T10:46:32.245314+0000","last_became_peered":"2023-10-09T10:46:32.245314+0000","last_unstale":"2023-10-09T10:47:18.931104+0000","last_undegraded":"2023-10-09T10:47:18.931104+0000","last_fullsized":"2023-10-09T10:47:18.931104+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:51.716860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":135,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931603+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:47:18.931603+0000","last_peered":"2023-10-09T10:47:18.931603+0000","last_clean":"2023-10-09T10:47:18.931603+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:47:18.931603+0000","last_undegraded":"2023-10-09T10:47:18.931603+0000","last_fullsized":"2023-10-09T10:47:18.931603+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":496,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910729+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:47:18.910729+0000","last_peered":"2023-10-09T10:47:18.910729+0000","last_clean":"2023-10-09T10:47:18.910729+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:47:18.910729+0000","last_undegraded":"2023-10-09T10:47:18.910729+0000","last_fullsized":"2023-10-09T10:47:18.910729+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"171'9278","reported_seq":9379,"reported_epoch":171,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:25.973446+0000","last_change":"2023-10-09T10:46:02.754507+0000","last_active":"2023-10-09T10:47:25.973446+0000","last_peered":"2023-10-09T10:47:25.973446+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:46:02.747510+0000","last_became_peered":"2023-10-09T10:46:02.747510+0000","last_unstale":"2023-10-09T10:47:25.973446+0000","last_undegraded":"2023-10-09T10:46:02.605987+0000","last_fullsized":"2023-10-09T10:46:02.585248+0000","mapping_epoch":142,"log_start":"103'400","ondisk_log_start":"103'400","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8878,"log_dups_size":0,"ondisk_log_size":8878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:59.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269811736,"num_objects":4118,"num_object_clones":0,"num_object_copies":12354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4118,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9278,"num_write_kb":428609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,2,5","objects":2718},{"shards":"1,5","objects":1400}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":140,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.926548+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:47:18.926548+0000","last_peered":"2023-10-09T10:47:18.926548+0000","last_clean":"2023-10-09T10:47:18.926548+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:47:18.926548+0000","last_undegraded":"2023-10-09T10:47:18.926548+0000","last_fullsized":"2023-10-09T10:47:18.926548+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":123,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910217+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:47:18.910217+0000","last_peered":"2023-10-09T10:47:18.910217+0000","last_clean":"2023-10-09T10:47:18.910217+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:47:18.910217+0000","last_undegraded":"2023-10-09T10:47:18.910217+0000","last_fullsized":"2023-10-09T10:47:18.910217+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":505,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910787+0000","last_change":"2023-10-09T10:46:05.617983+0000","last_active":"2023-10-09T10:47:18.910787+0000","last_peered":"2023-10-09T10:47:18.910787+0000","last_clean":"2023-10-09T10:47:18.910787+0000","last_became_active":"2023-10-09T10:46:05.616021+0000","last_became_peered":"2023-10-09T10:46:05.616021+0000","last_unstale":"2023-10-09T10:47:18.910787+0000","last_undegraded":"2023-10-09T10:47:18.910787+0000","last_fullsized":"2023-10-09T10:47:18.910787+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":166,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.909689+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:47:18.909689+0000","last_peered":"2023-10-09T10:47:18.909689+0000","last_clean":"2023-10-09T10:47:18.909689+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:47:18.909689+0000","last_undegraded":"2023-10-09T10:47:18.909689+0000","last_fullsized":"2023-10-09T10:47:18.909689+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":163,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.918104+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:47:18.918104+0000","last_peered":"2023-10-09T10:47:18.918104+0000","last_clean":"2023-10-09T10:47:18.918104+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:47:18.918104+0000","last_undegraded":"2023-10-09T10:47:18.918104+0000","last_fullsized":"2023-10-09T10:47:18.918104+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":108,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.909194+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:47:18.909194+0000","last_peered":"2023-10-09T10:47:18.909194+0000","last_clean":"2023-10-09T10:47:18.909194+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:47:18.909194+0000","last_undegraded":"2023-10-09T10:47:18.909194+0000","last_fullsized":"2023-10-09T10:47:18.909194+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"171'5678","reported_seq":9705,"reported_epoch":171,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:29.665162+0000","last_change":"2023-10-09T10:46:32.248218+0000","last_active":"2023-10-09T10:47:29.665162+0000","last_peered":"2023-10-09T10:47:29.665162+0000","last_clean":"2023-10-09T10:46:02.181912+0000","last_became_active":"2023-10-09T10:46:32.226237+0000","last_became_peered":"2023-10-09T10:46:32.226237+0000","last_unstale":"2023-10-09T10:47:29.665162+0000","last_undegraded":"2023-10-09T10:47:29.665162+0000","last_fullsized":"2023-10-09T10:47:29.665162+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5678,"log_dups_size":0,"ondisk_log_size":5678,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131956736,"num_objects":2013,"num_object_clones":0,"num_object_copies":6039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2013,"num_objects_unfound":0,"num_objects_dirty":2013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4631,"num_write_kb":212640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":2013}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":504,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910458+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:47:18.910458+0000","last_peered":"2023-10-09T10:47:18.910458+0000","last_clean":"2023-10-09T10:47:18.910458+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:47:18.910458+0000","last_undegraded":"2023-10-09T10:47:18.910458+0000","last_fullsized":"2023-10-09T10:47:18.910458+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"171'5744","reported_seq":5917,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T10:47:29.355361+0000","last_change":"2023-10-09T10:46:02.760963+0000","last_active":"2023-10-09T10:47:29.355361+0000","last_peered":"2023-10-09T10:47:29.355361+0000","last_clean":"2023-10-09T10:47:29.355361+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:47:29.355361+0000","last_undegraded":"2023-10-09T10:47:29.355361+0000","last_fullsized":"2023-10-09T10:47:29.355361+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"141'4416","last_scrub_stamp":"2023-10-09T10:46:02.760810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:46:02.760810+0000","objects_scrubbed":3378,"log_size":5744,"log_dups_size":0,"ondisk_log_size":5744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:35.863002+0000","scrub_duration":0.47769323899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134348800,"num_objects":2050,"num_object_clones":0,"num_object_copies":6150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1328,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":140,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.917738+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:47:18.917738+0000","last_peered":"2023-10-09T10:47:18.917738+0000","last_clean":"2023-10-09T10:47:18.917738+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:47:18.917738+0000","last_undegraded":"2023-10-09T10:47:18.917738+0000","last_fullsized":"2023-10-09T10:47:18.917738+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":156,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931625+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:47:18.931625+0000","last_peered":"2023-10-09T10:47:18.931625+0000","last_clean":"2023-10-09T10:47:18.931625+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:47:18.931625+0000","last_undegraded":"2023-10-09T10:47:18.931625+0000","last_fullsized":"2023-10-09T10:47:18.931625+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":147,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.909762+0000","last_change":"2023-10-09T10:46:32.245608+0000","last_active":"2023-10-09T10:47:18.909762+0000","last_peered":"2023-10-09T10:47:18.909762+0000","last_clean":"2023-10-09T10:47:18.909762+0000","last_became_active":"2023-10-09T10:46:32.245290+0000","last_became_peered":"2023-10-09T10:46:32.245290+0000","last_unstale":"2023-10-09T10:47:18.909762+0000","last_undegraded":"2023-10-09T10:47:18.909762+0000","last_fullsized":"2023-10-09T10:47:18.909762+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:48.832515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":505,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910801+0000","last_change":"2023-10-09T10:46:05.620057+0000","last_active":"2023-10-09T10:47:18.910801+0000","last_peered":"2023-10-09T10:47:18.910801+0000","last_clean":"2023-10-09T10:47:18.910801+0000","last_became_active":"2023-10-09T10:46:05.619703+0000","last_became_peered":"2023-10-09T10:46:05.619703+0000","last_unstale":"2023-10-09T10:47:18.910801+0000","last_undegraded":"2023-10-09T10:47:18.910801+0000","last_fullsized":"2023-10-09T10:47:18.910801+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"171'5698","reported_seq":5806,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T10:47:29.436649+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:47:29.436649+0000","last_peered":"2023-10-09T10:47:29.436649+0000","last_clean":"2023-10-09T10:47:29.436649+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:47:29.436649+0000","last_undegraded":"2023-10-09T10:47:29.436649+0000","last_fullsized":"2023-10-09T10:47:29.436649+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5698,"log_dups_size":0,"ondisk_log_size":5698,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133890048,"num_objects":2043,"num_object_clones":0,"num_object_copies":6129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4637,"num_write_kb":213760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":168,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.959195+0000","last_change":"2023-10-09T10:46:32.241041+0000","last_active":"2023-10-09T10:47:18.959195+0000","last_peered":"2023-10-09T10:47:18.959195+0000","last_clean":"2023-10-09T10:47:18.959195+0000","last_became_active":"2023-10-09T10:46:32.238683+0000","last_became_peered":"2023-10-09T10:46:32.238683+0000","last_unstale":"2023-10-09T10:47:18.959195+0000","last_undegraded":"2023-10-09T10:47:18.959195+0000","last_fullsized":"2023-10-09T10:47:18.959195+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:25.817713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":132,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.908618+0000","last_change":"2023-10-09T10:46:32.250104+0000","last_active":"2023-10-09T10:47:18.908618+0000","last_peered":"2023-10-09T10:47:18.908618+0000","last_clean":"2023-10-09T10:47:18.908618+0000","last_became_active":"2023-10-09T10:46:32.249805+0000","last_became_peered":"2023-10-09T10:46:32.249805+0000","last_unstale":"2023-10-09T10:47:18.908618+0000","last_undegraded":"2023-10-09T10:47:18.908618+0000","last_fullsized":"2023-10-09T10:47:18.908618+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:07.230672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":161,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910288+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:47:18.910288+0000","last_peered":"2023-10-09T10:47:18.910288+0000","last_clean":"2023-10-09T10:47:18.910288+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:47:18.910288+0000","last_undegraded":"2023-10-09T10:47:18.910288+0000","last_fullsized":"2023-10-09T10:47:18.910288+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"171'5519","reported_seq":5614,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T10:47:25.867068+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:47:25.867068+0000","last_peered":"2023-10-09T10:47:25.867068+0000","last_clean":"2023-10-09T10:47:25.867068+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:47:25.867068+0000","last_undegraded":"2023-10-09T10:47:25.867068+0000","last_fullsized":"2023-10-09T10:47:25.867068+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5519,"log_dups_size":0,"ondisk_log_size":5519,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140181504,"num_objects":2139,"num_object_clones":0,"num_object_copies":6417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4511,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":150,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.917410+0000","last_change":"2023-10-09T10:46:32.235346+0000","last_active":"2023-10-09T10:47:18.917410+0000","last_peered":"2023-10-09T10:47:18.917410+0000","last_clean":"2023-10-09T10:47:18.917410+0000","last_became_active":"2023-10-09T10:46:32.234906+0000","last_became_peered":"2023-10-09T10:46:32.234906+0000","last_unstale":"2023-10-09T10:47:18.917410+0000","last_undegraded":"2023-10-09T10:47:18.917410+0000","last_fullsized":"2023-10-09T10:47:18.917410+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.199251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":505,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910417+0000","last_change":"2023-10-09T10:46:05.617146+0000","last_active":"2023-10-09T10:47:18.910417+0000","last_peered":"2023-10-09T10:47:18.910417+0000","last_clean":"2023-10-09T10:47:18.910417+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:47:18.910417+0000","last_undegraded":"2023-10-09T10:47:18.910417+0000","last_fullsized":"2023-10-09T10:47:18.910417+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":98,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.930139+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:47:18.930139+0000","last_peered":"2023-10-09T10:47:18.930139+0000","last_clean":"2023-10-09T10:47:18.930139+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:47:18.930139+0000","last_undegraded":"2023-10-09T10:47:18.930139+0000","last_fullsized":"2023-10-09T10:47:18.930139+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":68,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931307+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:47:18.931307+0000","last_peered":"2023-10-09T10:47:18.931307+0000","last_clean":"2023-10-09T10:47:18.931307+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:47:18.931307+0000","last_undegraded":"2023-10-09T10:47:18.931307+0000","last_fullsized":"2023-10-09T10:47:18.931307+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":505,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.911933+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:47:18.911933+0000","last_peered":"2023-10-09T10:47:18.911933+0000","last_clean":"2023-10-09T10:47:18.911933+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:47:18.911933+0000","last_undegraded":"2023-10-09T10:47:18.911933+0000","last_fullsized":"2023-10-09T10:47:18.911933+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":162,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.908681+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:47:18.908681+0000","last_peered":"2023-10-09T10:47:18.908681+0000","last_clean":"2023-10-09T10:47:18.908681+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:47:18.908681+0000","last_undegraded":"2023-10-09T10:47:18.908681+0000","last_fullsized":"2023-10-09T10:47:18.908681+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":48,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910369+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:47:18.910369+0000","last_peered":"2023-10-09T10:47:18.910369+0000","last_clean":"2023-10-09T10:47:18.910369+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:47:18.910369+0000","last_undegraded":"2023-10-09T10:47:18.910369+0000","last_fullsized":"2023-10-09T10:47:18.910369+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":48,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.911730+0000","last_change":"2023-10-09T10:46:32.236202+0000","last_active":"2023-10-09T10:47:18.911730+0000","last_peered":"2023-10-09T10:47:18.911730+0000","last_clean":"2023-10-09T10:47:18.911730+0000","last_became_active":"2023-10-09T10:46:32.235901+0000","last_became_peered":"2023-10-09T10:46:32.235901+0000","last_unstale":"2023-10-09T10:47:18.911730+0000","last_undegraded":"2023-10-09T10:47:18.911730+0000","last_fullsized":"2023-10-09T10:47:18.911730+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:48.260403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"171'5694","reported_seq":5836,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T10:47:26.421255+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:47:26.421255+0000","last_peered":"2023-10-09T10:47:26.421255+0000","last_clean":"2023-10-09T10:47:26.421255+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:47:26.421255+0000","last_undegraded":"2023-10-09T10:47:26.421255+0000","last_fullsized":"2023-10-09T10:47:26.421255+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":5694,"log_dups_size":0,"ondisk_log_size":5694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136577024,"num_objects":2084,"num_object_clones":0,"num_object_copies":6252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2664,"num_write_kb":89664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":505,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.911521+0000","last_change":"2023-10-09T10:46:05.620764+0000","last_active":"2023-10-09T10:47:18.911521+0000","last_peered":"2023-10-09T10:47:18.911521+0000","last_clean":"2023-10-09T10:47:18.911521+0000","last_became_active":"2023-10-09T10:46:05.619870+0000","last_became_peered":"2023-10-09T10:46:05.619870+0000","last_unstale":"2023-10-09T10:47:18.911521+0000","last_undegraded":"2023-10-09T10:47:18.911521+0000","last_fullsized":"2023-10-09T10:47:18.911521+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":159,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.921401+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:47:18.921401+0000","last_peered":"2023-10-09T10:47:18.921401+0000","last_clean":"2023-10-09T10:47:18.921401+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:47:18.921401+0000","last_undegraded":"2023-10-09T10:47:18.921401+0000","last_fullsized":"2023-10-09T10:47:18.921401+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":132,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.909843+0000","last_change":"2023-10-09T10:46:32.251450+0000","last_active":"2023-10-09T10:47:18.909843+0000","last_peered":"2023-10-09T10:47:18.909843+0000","last_clean":"2023-10-09T10:47:18.909843+0000","last_became_active":"2023-10-09T10:46:32.250933+0000","last_became_peered":"2023-10-09T10:46:32.250933+0000","last_unstale":"2023-10-09T10:47:18.909843+0000","last_undegraded":"2023-10-09T10:47:18.909843+0000","last_fullsized":"2023-10-09T10:47:18.909843+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:04.116178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":39,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.912087+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:47:18.912087+0000","last_peered":"2023-10-09T10:47:18.912087+0000","last_clean":"2023-10-09T10:47:18.912087+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:47:18.912087+0000","last_undegraded":"2023-10-09T10:47:18.912087+0000","last_fullsized":"2023-10-09T10:47:18.912087+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"171'5552","reported_seq":12239,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T10:47:26.410265+0000","last_change":"2023-10-09T10:47:02.269289+0000","last_active":"2023-10-09T10:47:26.410265+0000","last_peered":"2023-10-09T10:47:26.410265+0000","last_clean":"2023-10-09T10:47:26.410265+0000","last_became_active":"2023-10-09T10:47:02.257854+0000","last_became_peered":"2023-10-09T10:47:02.257854+0000","last_unstale":"2023-10-09T10:47:26.410265+0000","last_undegraded":"2023-10-09T10:47:26.410265+0000","last_fullsized":"2023-10-09T10:47:26.410265+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5552,"log_dups_size":0,"ondisk_log_size":5552,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134905856,"num_objects":2058,"num_object_clones":0,"num_object_copies":6174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4544,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3287,"num_bytes_recovered":181927936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":505,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.911024+0000","last_change":"2023-10-09T10:46:05.618443+0000","last_active":"2023-10-09T10:47:18.911024+0000","last_peered":"2023-10-09T10:47:18.911024+0000","last_clean":"2023-10-09T10:47:18.911024+0000","last_became_active":"2023-10-09T10:46:05.616588+0000","last_became_peered":"2023-10-09T10:46:05.616588+0000","last_unstale":"2023-10-09T10:47:18.911024+0000","last_undegraded":"2023-10-09T10:47:18.911024+0000","last_fullsized":"2023-10-09T10:47:18.911024+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":163,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931859+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:47:18.931859+0000","last_peered":"2023-10-09T10:47:18.931859+0000","last_clean":"2023-10-09T10:47:18.931859+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:47:18.931859+0000","last_undegraded":"2023-10-09T10:47:18.931859+0000","last_fullsized":"2023-10-09T10:47:18.931859+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":108,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910735+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:47:18.910735+0000","last_peered":"2023-10-09T10:47:18.910735+0000","last_clean":"2023-10-09T10:47:18.910735+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:47:18.910735+0000","last_undegraded":"2023-10-09T10:47:18.910735+0000","last_fullsized":"2023-10-09T10:47:18.910735+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":152,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.926615+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:47:18.926615+0000","last_peered":"2023-10-09T10:47:18.926615+0000","last_clean":"2023-10-09T10:47:18.926615+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:47:18.926615+0000","last_undegraded":"2023-10-09T10:47:18.926615+0000","last_fullsized":"2023-10-09T10:47:18.926615+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"171'5670","reported_seq":5811,"reported_epoch":171,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:25.910160+0000","last_change":"2023-10-09T10:46:33.507626+0000","last_active":"2023-10-09T10:47:25.910160+0000","last_peered":"2023-10-09T10:47:25.910160+0000","last_clean":"2023-10-09T10:46:31.130083+0000","last_became_active":"2023-10-09T10:46:33.493288+0000","last_became_peered":"2023-10-09T10:46:33.493288+0000","last_unstale":"2023-10-09T10:47:25.910160+0000","last_undegraded":"2023-10-09T10:46:33.283624+0000","last_fullsized":"2023-10-09T10:46:33.244043+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5670,"log_dups_size":0,"ondisk_log_size":5670,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133136384,"num_objects":2031,"num_object_clones":0,"num_object_copies":6093,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3337,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2031,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4633,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":2036},{"shards":"3,4","objects":-784}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":504,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.911355+0000","last_change":"2023-10-09T10:46:05.617589+0000","last_active":"2023-10-09T10:47:18.911355+0000","last_peered":"2023-10-09T10:47:18.911355+0000","last_clean":"2023-10-09T10:47:18.911355+0000","last_became_active":"2023-10-09T10:46:05.616400+0000","last_became_peered":"2023-10-09T10:46:05.616400+0000","last_unstale":"2023-10-09T10:47:18.911355+0000","last_undegraded":"2023-10-09T10:47:18.911355+0000","last_fullsized":"2023-10-09T10:47:18.911355+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931609+0000","last_change":"2023-10-09T10:46:32.245106+0000","last_active":"2023-10-09T10:47:18.931609+0000","last_peered":"2023-10-09T10:47:18.931609+0000","last_clean":"2023-10-09T10:47:18.931609+0000","last_became_active":"2023-10-09T10:46:32.244826+0000","last_became_peered":"2023-10-09T10:46:32.244826+0000","last_unstale":"2023-10-09T10:47:18.931609+0000","last_undegraded":"2023-10-09T10:47:18.931609+0000","last_fullsized":"2023-10-09T10:47:18.931609+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:30.322834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":43,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.911664+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:47:18.911664+0000","last_peered":"2023-10-09T10:47:18.911664+0000","last_clean":"2023-10-09T10:47:18.911664+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:47:18.911664+0000","last_undegraded":"2023-10-09T10:47:18.911664+0000","last_fullsized":"2023-10-09T10:47:18.911664+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":118,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931098+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:47:18.931098+0000","last_peered":"2023-10-09T10:47:18.931098+0000","last_clean":"2023-10-09T10:47:18.931098+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:47:18.931098+0000","last_undegraded":"2023-10-09T10:47:18.931098+0000","last_fullsized":"2023-10-09T10:47:18.931098+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"171'5576","reported_seq":11095,"reported_epoch":171,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:25.989400+0000","last_change":"2023-10-09T10:47:00.660718+0000","last_active":"2023-10-09T10:47:25.989400+0000","last_peered":"2023-10-09T10:47:25.989400+0000","last_clean":"2023-10-09T10:46:31.098576+0000","last_became_active":"2023-10-09T10:46:33.454803+0000","last_became_peered":"2023-10-09T10:46:33.454803+0000","last_unstale":"2023-10-09T10:47:25.989400+0000","last_undegraded":"2023-10-09T10:46:33.267772+0000","last_fullsized":"2023-10-09T10:46:33.242755+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":5576,"log_dups_size":0,"ondisk_log_size":5576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:40:03.811205+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134348800,"num_objects":2050,"num_object_clones":0,"num_object_copies":6150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":358,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2931,"num_write_kb":107264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2670,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"6,7","objects":1692}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":39,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.912168+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:47:18.912168+0000","last_peered":"2023-10-09T10:47:18.912168+0000","last_clean":"2023-10-09T10:47:18.912168+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:47:18.912168+0000","last_undegraded":"2023-10-09T10:47:18.912168+0000","last_fullsized":"2023-10-09T10:47:18.912168+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":102,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.909917+0000","last_change":"2023-10-09T10:46:32.251242+0000","last_active":"2023-10-09T10:47:18.909917+0000","last_peered":"2023-10-09T10:47:18.909917+0000","last_clean":"2023-10-09T10:47:18.909917+0000","last_became_active":"2023-10-09T10:46:32.250534+0000","last_became_peered":"2023-10-09T10:46:32.250534+0000","last_unstale":"2023-10-09T10:47:18.909917+0000","last_undegraded":"2023-10-09T10:47:18.909917+0000","last_fullsized":"2023-10-09T10:47:18.909917+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:33.726041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"171'5669","reported_seq":9644,"reported_epoch":171,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:27.575978+0000","last_change":"2023-10-09T10:46:34.514479+0000","last_active":"2023-10-09T10:47:27.575978+0000","last_peered":"2023-10-09T10:47:27.575978+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:46:34.409667+0000","last_became_peered":"2023-10-09T10:46:34.409667+0000","last_unstale":"2023-10-09T10:47:27.575978+0000","last_undegraded":"2023-10-09T10:46:34.376635+0000","last_fullsized":"2023-10-09T10:46:33.237998+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5669,"log_dups_size":0,"ondisk_log_size":5669,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134152192,"num_objects":2047,"num_object_clones":0,"num_object_copies":6141,"num_objects_missing_on_primary":1406,"num_objects_missing":1406,"num_objects_degraded":1916,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4627,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1932,"num_bytes_recovered":126615552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,7","objects":641},{"shards":"2,3,7","objects":1406}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":138,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.917873+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:47:18.917873+0000","last_peered":"2023-10-09T10:47:18.917873+0000","last_clean":"2023-10-09T10:47:18.917873+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:47:18.917873+0000","last_undegraded":"2023-10-09T10:47:18.917873+0000","last_fullsized":"2023-10-09T10:47:18.917873+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":152,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931769+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:47:18.931769+0000","last_peered":"2023-10-09T10:47:18.931769+0000","last_clean":"2023-10-09T10:47:18.931769+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:47:18.931769+0000","last_undegraded":"2023-10-09T10:47:18.931769+0000","last_fullsized":"2023-10-09T10:47:18.931769+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"171'5694","reported_seq":5797,"reported_epoch":171,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:29.436756+0000","last_change":"2023-10-09T10:46:33.546991+0000","last_active":"2023-10-09T10:47:29.436756+0000","last_peered":"2023-10-09T10:47:29.436756+0000","last_clean":"2023-10-09T10:46:31.009484+0000","last_became_active":"2023-10-09T10:46:33.535665+0000","last_became_peered":"2023-10-09T10:46:33.535665+0000","last_unstale":"2023-10-09T10:47:29.436756+0000","last_undegraded":"2023-10-09T10:46:33.277168+0000","last_fullsized":"2023-10-09T10:46:33.235038+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5694,"log_dups_size":0,"ondisk_log_size":5694,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124354560,"num_objects":1898,"num_object_clones":0,"num_object_copies":5694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4630,"num_write_kb":208864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1910},{"shards":"3,7","objects":-860}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":121,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.918157+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:47:18.918157+0000","last_peered":"2023-10-09T10:47:18.918157+0000","last_clean":"2023-10-09T10:47:18.918157+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:47:18.918157+0000","last_undegraded":"2023-10-09T10:47:18.918157+0000","last_fullsized":"2023-10-09T10:47:18.918157+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":19,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931347+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:47:18.931347+0000","last_peered":"2023-10-09T10:47:18.931347+0000","last_clean":"2023-10-09T10:47:18.931347+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:47:18.931347+0000","last_undegraded":"2023-10-09T10:47:18.931347+0000","last_fullsized":"2023-10-09T10:47:18.931347+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":154,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.930554+0000","last_change":"2023-10-09T10:46:32.244607+0000","last_active":"2023-10-09T10:47:18.930554+0000","last_peered":"2023-10-09T10:47:18.930554+0000","last_clean":"2023-10-09T10:47:18.930554+0000","last_became_active":"2023-10-09T10:46:32.242628+0000","last_became_peered":"2023-10-09T10:46:32.242628+0000","last_unstale":"2023-10-09T10:47:18.930554+0000","last_undegraded":"2023-10-09T10:47:18.930554+0000","last_fullsized":"2023-10-09T10:47:18.930554+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:10.738819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":185,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.958961+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:47:18.958961+0000","last_peered":"2023-10-09T10:47:18.958961+0000","last_clean":"2023-10-09T10:47:18.958961+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:47:18.958961+0000","last_undegraded":"2023-10-09T10:47:18.958961+0000","last_fullsized":"2023-10-09T10:47:18.958961+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":60,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.910703+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:47:18.910703+0000","last_peered":"2023-10-09T10:47:18.910703+0000","last_clean":"2023-10-09T10:47:18.910703+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:47:18.910703+0000","last_undegraded":"2023-10-09T10:47:18.910703+0000","last_fullsized":"2023-10-09T10:47:18.910703+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":135,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T10:47:18.931002+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:47:18.931002+0000","last_peered":"2023-10-09T10:47:18.931002+0000","last_clean":"2023-10-09T10:47:18.931002+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:47:18.931002+0000","last_undegraded":"2023-10-09T10:47:18.931002+0000","last_fullsized":"2023-10-09T10:47:18.931002+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":4241391640,"num_objects":64720,"num_object_clones":0,"num_object_copies":194160,"num_objects_missing_on_primary":1415,"num_objects_missing":1415,"num_objects_degraded":34009,"num_objects_misplaced":7612,"num_objects_unfound":0,"num_objects_dirty":64720,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":139488,"num_write_kb":6258529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19850,"num_bytes_recovered":1249837056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14589898752,"data_stored":14589886536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":155533,"ondisk_log_size":155533,"up":78,"acting":66,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1186,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332250,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2147244,"kb_used_data":1845860,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91597908,"statfs":{"total":95995035648,"available":93796257792,"internally_reserved":0,"allocated":1890160640,"data_stored":1889509810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.70099999999999996,"15min":0.61899999999999999},"min":{"1min":0.23100000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.0919999999999996,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.3640000000000001,"5min":0.79100000000000004,"15min":0.69099999999999995},"min":{"1min":0.309,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":10.207000000000001,"5min":10.207000000000001,"15min":10.207000000000001},"last":2.5979999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:46:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":0.99399999999999999,"15min":0.69199999999999995},"min":{"1min":0.42199999999999999,"5min":0.318,"15min":0.318},"max":{"1min":8.7360000000000007,"5min":8.7360000000000007,"15min":8.7360000000000007},"last":3.419},{"interface":"front","average":{"1min":1.2230000000000001,"5min":0.80500000000000005,"15min":0.623},"min":{"1min":0.27800000000000002,"5min":0.245,"15min":0.245},"max":{"1min":3.6080000000000001,"5min":6.4939999999999998,"15min":6.4939999999999998},"last":5.0140000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.1499999999999999,"15min":0.72399999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.4770000000000003,"5min":6.4770000000000003,"15min":6.4770000000000003},"last":3.548},{"interface":"front","average":{"1min":1.272,"5min":0.85699999999999998,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.251,"15min":0.251},"max":{"1min":3.1480000000000001,"5min":3.7120000000000002,"15min":3.7120000000000002},"last":0.59699999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:46:36 2023","interfaces":[{"interface":"back","average":{"1min":1.9079999999999999,"5min":1.073,"15min":0.748},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.8129999999999997,"5min":8.1639999999999997,"15min":8.1639999999999997},"last":3.0619999999999998},{"interface":"front","average":{"1min":1.3720000000000001,"5min":1.022,"15min":0.73999999999999999},"min":{"1min":0.377,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.3609999999999998,"5min":12.451000000000001,"15min":12.451000000000001},"last":1.107}]},{"osd":4,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.73299999999999998,"15min":0.56200000000000006},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.1420000000000003,"5min":7.4509999999999996,"15min":7.4509999999999996},"last":2.9700000000000002},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.65300000000000002,"15min":0.53600000000000003},"min":{"1min":0.29799999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.2749999999999999,"5min":3.1699999999999999,"15min":3.1699999999999999},"last":1.0780000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.56399999999999995,"15min":0.52700000000000002},"min":{"1min":0.34300000000000003,"5min":0.247,"15min":0.247},"max":{"1min":2.2349999999999999,"5min":2.2349999999999999,"15min":2.2349999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.81299999999999994,"15min":0.63},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":5.0860000000000003,"5min":10.33,"15min":10.33},"last":3.032}]},{"osd":6,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.83399999999999996,"15min":0.64800000000000002},"min":{"1min":0.311,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.109,"5min":10.164999999999999,"15min":10.164999999999999},"last":3.0049999999999999},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.66000000000000003,"15min":0.58299999999999996},"min":{"1min":0.33300000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.306,"5min":2.8079999999999998,"15min":2.8079999999999998},"last":1.129}]}]},{"osd":6,"up_from":44,"seq":188978561181,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1702888,"kb_used_data":1405600,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92042264,"statfs":{"total":95995035648,"available":94251278336,"internally_reserved":0,"allocated":1439334400,"data_stored":1438688108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.772,"5min":0.89500000000000002,"15min":0.70499999999999996},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.3220000000000001,"5min":7.3220000000000001,"15min":7.3220000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":2.02,"5min":0.92000000000000004,"15min":0.71199999999999997},"min":{"1min":0.26700000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.5309999999999997,"5min":7.5309999999999997,"15min":7.5309999999999997},"last":0.58599999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:46:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8180000000000001,"5min":1.0680000000000001,"15min":0.75},"min":{"1min":0.29899999999999999,"5min":0.245,"15min":0.245},"max":{"1min":7.327,"5min":16.747,"15min":16.747},"last":0.499},{"interface":"front","average":{"1min":1.883,"5min":1.101,"15min":0.73399999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.4260000000000002,"5min":16.734000000000002,"15min":16.734000000000002},"last":0.34100000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":0.95799999999999996,"15min":0.67500000000000004},"min":{"1min":0.28699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5069999999999997,"5min":7.5069999999999997,"15min":7.5069999999999997},"last":0.29999999999999999},{"interface":"front","average":{"1min":1.675,"5min":1.143,"15min":0.755},"min":{"1min":0.28599999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.3899999999999997,"5min":12.571,"15min":12.571},"last":1.3480000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":0.89600000000000002,"15min":0.65700000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.8799999999999999,"5min":4.7030000000000003,"15min":4.7030000000000003},"last":1.456},{"interface":"front","average":{"1min":2.4399999999999999,"5min":1.0700000000000001,"15min":0.72099999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.4770000000000003,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":0.41499999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":0.66600000000000004,"15min":0.51500000000000001},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":4.0739999999999998,"5min":4.0739999999999998,"15min":4.0739999999999998},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.67300000000000004,"15min":0.54600000000000004},"min":{"1min":0.34899999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6600000000000001,"5min":3.6600000000000001,"15min":3.6600000000000001},"last":0.56100000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":0.76000000000000001,"15min":0.56999999999999995},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.8949999999999996,"5min":4.8949999999999996,"15min":4.8949999999999996},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.76900000000000002,"15min":0.59999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":4.3639999999999999,"5min":4.3639999999999999,"15min":4.3639999999999999},"last":0.63400000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:47:25 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69899999999999995,"15min":0.57999999999999996},"min":{"1min":0.35799999999999998,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":2.5049999999999999,"5min":3.472,"15min":3.472},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.84299999999999997,"15min":0.65700000000000003},"min":{"1min":0.373,"5min":0.36899999999999999,"15min":0.34000000000000002},"max":{"1min":4.0800000000000001,"5min":4.0800000000000001,"15min":4.0800000000000001},"last":0.60899999999999999}]}]},{"osd":1,"up_from":85,"seq":365072220220,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1781516,"kb_used_data":1669572,"kb_used_omap":9,"kb_used_meta":111926,"kb_avail":91963636,"statfs":{"total":95995035648,"available":94170763264,"internally_reserved":0,"allocated":1709641728,"data_stored":1708790756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":114613225},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.3160000000000001,"15min":0.68100000000000005},"min":{"1min":0.255,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":8.3759999999999994,"5min":14.1,"15min":14.1},"last":0.311},{"interface":"front","average":{"1min":2.5499999999999998,"5min":1.266,"15min":0.68799999999999994},"min":{"1min":0.27300000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":13.462999999999999,"5min":14.465,"15min":14.465},"last":0.26400000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:47:13 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.1200000000000001,"15min":0.61799999999999999},"min":{"1min":0.21199999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.4829999999999997,"5min":8.6859999999999999,"15min":8.6859999999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.919,"5min":1.4690000000000001,"15min":0.746},"min":{"1min":0.29099999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.2829999999999995,"5min":14.234,"15min":14.234},"last":0.34999999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.2,"15min":0.71399999999999997},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.3539999999999992,"5min":14.404999999999999,"15min":14.404999999999999},"last":0.65200000000000002},{"interface":"front","average":{"1min":2.077,"5min":1.1080000000000001,"15min":0.59899999999999998},"min":{"1min":0.255,"5min":0.251,"15min":0.251},"max":{"1min":9.6820000000000004,"5min":9.6820000000000004,"15min":9.6820000000000004},"last":0.38700000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.0880000000000001,"15min":0.75800000000000001},"min":{"1min":0.29099999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.1420000000000003,"5min":8.0129999999999999,"15min":8.0129999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.4850000000000001,"5min":1.0249999999999999,"15min":0.755},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":8.327,"5min":8.327,"15min":8.327},"last":0.53200000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.571,"5min":0.95699999999999996,"15min":0.73699999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.7720000000000002,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":0.504},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.1359999999999999,"15min":0.81000000000000005},"min":{"1min":0.32700000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.1799999999999997,"5min":7.923,"15min":7.923},"last":0.45500000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":0.97199999999999998,"15min":0.745},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.7379999999999995,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":2.4119999999999999},{"interface":"front","average":{"1min":1.617,"5min":1.175,"15min":0.82299999999999995},"min":{"1min":0.39500000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.2069999999999999,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":0.42299999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6699999999999999,"5min":1.155,"15min":0.81799999999999995},"min":{"1min":0.41599999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":5.2309999999999999,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.022,"15min":0.77100000000000002},"min":{"1min":0.42499999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":4.6399999999999997,"5min":8.2469999999999999,"15min":8.2469999999999999},"last":0.59799999999999998}]}]},{"osd":0,"up_from":80,"seq":343597383754,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":773028,"kb_used_data":691356,"kb_used_omap":28,"kb_used_meta":81635,"kb_avail":92972124,"statfs":{"total":95995035648,"available":95203454976,"internally_reserved":0,"allocated":707948544,"data_stored":707072550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":83595088},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:46:43 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.1679999999999999,"15min":0.63400000000000001},"min":{"1min":0.217,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.0449999999999999,"5min":25.475000000000001,"15min":25.475000000000001},"last":1.9630000000000001},{"interface":"front","average":{"1min":2.0099999999999998,"5min":1.1919999999999999,"15min":0.64000000000000001},"min":{"1min":0.23499999999999999,"5min":0.191,"15min":0.191},"max":{"1min":6.2290000000000001,"5min":20.832999999999998,"15min":20.832999999999998},"last":10.593999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":1.2350000000000001,"15min":0.64400000000000002},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.907,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":0.25900000000000001},{"interface":"front","average":{"1min":1.754,"5min":1.5349999999999999,"15min":0.73999999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.5619999999999994,"5min":14.391999999999999,"15min":14.391999999999999},"last":0.27300000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9689999999999999,"5min":1.5309999999999999,"15min":0.94399999999999995},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":9.3889999999999993,"5min":14.926,"15min":14.926},"last":10.644},{"interface":"front","average":{"1min":2.02,"5min":1.2569999999999999,"15min":0.78000000000000003},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":11.868,"5min":11.868,"15min":11.868},"last":0.41199999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:46:33 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":0.94899999999999995,"15min":0.72999999999999998},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.3949999999999996,"5min":5.71,"15min":5.71},"last":1.8560000000000001},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.0509999999999999,"15min":0.76300000000000001},"min":{"1min":0.38900000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.093,"5min":9.9269999999999996,"15min":9.9269999999999996},"last":10.676}]},{"osd":5,"last update":"Mon Oct 9 10:46:33 2023","interfaces":[{"interface":"back","average":{"1min":1.8560000000000001,"5min":1.1200000000000001,"15min":0.79000000000000004},"min":{"1min":0.378,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.1299999999999999,"5min":9.9900000000000002,"15min":9.9900000000000002},"last":10.734},{"interface":"front","average":{"1min":1.149,"5min":0.95299999999999996,"15min":0.75},"min":{"1min":0.32300000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.9199999999999999,"5min":8.9190000000000005,"15min":8.9190000000000005},"last":0.49299999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:46:33 2023","interfaces":[{"interface":"back","average":{"1min":1.679,"5min":1.0920000000000001,"15min":0.78400000000000003},"min":{"1min":0.42699999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.1719999999999997,"5min":9.9659999999999993,"15min":9.9659999999999993},"last":10.706},{"interface":"front","average":{"1min":1.1379999999999999,"5min":0.98399999999999999,"15min":0.76200000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.8890000000000002,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":1.403}]},{"osd":7,"last update":"Mon Oct 9 10:46:33 2023","interfaces":[{"interface":"back","average":{"1min":1.252,"5min":0.98699999999999999,"15min":0.76400000000000001},"min":{"1min":0.39700000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.9649999999999999,"5min":8.875,"15min":8.875},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.1679999999999999,"5min":1.0349999999999999,"15min":0.79700000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.4390000000000001,"5min":7.3899999999999997,"15min":7.3899999999999997},"last":0.32200000000000001}]}]},{"osd":2,"up_from":0,"seq":386547056695,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893169,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2892372,"kb_used_data":2720524,"kb_used_omap":7,"kb_used_meta":171832,"kb_avail":90852780,"statfs":{"total":95995035648,"available":93033246720,"internally_reserved":0,"allocated":2785816576,"data_stored":2784955194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":175956584},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":2.734,"5min":0.995,"15min":0.54500000000000004},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":11.557,"5min":11.557,"15min":11.557},"last":1.474},{"interface":"front","average":{"1min":1.359,"5min":0.93700000000000006,"15min":0.58299999999999996},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":4.1369999999999996,"5min":7.0510000000000002,"15min":7.0510000000000002},"last":2.6219999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":1.4179999999999999,"15min":0.69199999999999995},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":13.967000000000001,"5min":14.707000000000001,"15min":14.707000000000001},"last":0.27700000000000002},{"interface":"front","average":{"1min":1.911,"5min":0.93899999999999995,"15min":0.56200000000000006},"min":{"1min":0.218,"5min":0.193,"15min":0.193},"max":{"1min":11.629,"5min":11.629,"15min":11.629},"last":0.19400000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1299999999999999,"5min":1.3280000000000001,"15min":0.68700000000000006},"min":{"1min":0.30099999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":13.853,"5min":13.853,"15min":13.853},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.514,"5min":1.0800000000000001,"15min":0.624},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.3019999999999996,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":4.6580000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":1.0229999999999999,"15min":0.72199999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":11.532,"5min":11.532,"15min":11.532},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.335,"5min":0.96899999999999997,"15min":0.72799999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.6070000000000002,"5min":5.6070000000000002,"15min":5.6070000000000002},"last":2.6579999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":4.3609999999999998,"5min":1.5680000000000001,"15min":0.91400000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":51.509999999999998,"5min":51.509999999999998,"15min":51.509999999999998},"last":2.7090000000000001},{"interface":"front","average":{"1min":5.141,"5min":1.99,"15min":1.038},"min":{"1min":0.32300000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":55.338999999999999,"5min":55.338999999999999,"15min":55.338999999999999},"last":0.36799999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":0.99199999999999999,"15min":0.72199999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.8490000000000002,"5min":8.4789999999999992,"15min":8.4789999999999992},"last":2.6859999999999999},{"interface":"front","average":{"1min":2.0150000000000001,"5min":1.421,"15min":0.873},"min":{"1min":0.378,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":11.470000000000001,"5min":13.891999999999999,"15min":13.891999999999999},"last":0.39600000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":4.9960000000000004,"5min":1.9970000000000001,"15min":1.0600000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":51.744,"5min":51.744,"15min":51.744},"last":0.42399999999999999},{"interface":"front","average":{"1min":4.6749999999999998,"5min":1.601,"15min":0.91600000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":53.399000000000001,"5min":53.399000000000001,"15min":53.399000000000001},"last":5.476}]}]},{"osd":4,"up_from":30,"seq":128849019044,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1634600,"kb_used_data":1325024,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92110552,"statfs":{"total":95995035648,"available":94321205248,"internally_reserved":0,"allocated":1356824576,"data_stored":1356177551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":0.90600000000000003,"15min":0.67400000000000004},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.387,"5min":10.387,"15min":10.387},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.5369999999999999,"5min":0.98299999999999998,"15min":0.71699999999999997},"min":{"1min":0.26600000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":10.311999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.249}]},{"osd":1,"last update":"Mon Oct 9 10:46:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":0.98599999999999999,"15min":0.69699999999999995},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.4189999999999996,"5min":6.907,"15min":6.907},"last":0.81399999999999995},{"interface":"front","average":{"1min":1.627,"5min":0.99099999999999999,"15min":0.745},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.1790000000000003,"5min":6.1790000000000003,"15min":6.1790000000000003},"last":0.80300000000000005}]},{"osd":2,"last update":"Mon Oct 9 10:47:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2170000000000001,"5min":1.1000000000000001,"15min":0.747},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.2149999999999999,"5min":7.5449999999999999,"15min":7.5449999999999999},"last":0.75700000000000001},{"interface":"front","average":{"1min":1.579,"5min":1.1639999999999999,"15min":0.76300000000000001},"min":{"1min":0.36699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.5110000000000001,"5min":6.2939999999999996,"15min":6.2939999999999996},"last":0.36799999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:46:38 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.107,"15min":0.84299999999999997},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.0730000000000004,"5min":5.0730000000000004,"15min":5.0730000000000004},"last":0.38100000000000001},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.004,"15min":0.72599999999999998},"min":{"1min":0.35999999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":8.9809999999999999,"5min":8.9809999999999999,"15min":8.9809999999999999},"last":0.40699999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:46:41 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.73099999999999998,"15min":0.53500000000000003},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":3.9319999999999999,"5min":5.5410000000000004,"15min":5.5410000000000004},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.71099999999999997,"15min":0.52900000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":3.8980000000000001,"5min":3.8980000000000001,"15min":3.8980000000000001},"last":0.71799999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.74099999999999999,"15min":0.55700000000000005},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":7.1580000000000004,"5min":7.1580000000000004,"15min":7.1580000000000004},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.71599999999999997,"15min":0.60899999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":3.6150000000000002,"5min":3.6150000000000002,"15min":3.6150000000000002},"last":0.51100000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.80800000000000005,"15min":0.64600000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.1050000000000004,"5min":4.1050000000000004,"15min":4.1050000000000004},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.748,"15min":0.60799999999999998},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0919999999999996,"5min":4.0919999999999996,"15min":4.0919999999999996},"last":0.55900000000000005}]}]},{"osd":5,"up_from":37,"seq":158913790113,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2019224,"kb_used_data":1713744,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91725928,"statfs":{"total":95995035648,"available":93927350272,"internally_reserved":0,"allocated":1754873856,"data_stored":1754238660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.72199999999999998,"15min":0.63900000000000001},"min":{"1min":0.25800000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.722,"5min":4.8380000000000001,"15min":4.8380000000000001},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.78000000000000003,"15min":0.70599999999999996},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.7549999999999999,"5min":4.8650000000000002,"15min":4.8650000000000002},"last":0.41999999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":0.85599999999999998,"15min":0.68999999999999995},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.7789999999999999,"5min":3.7789999999999999,"15min":3.7789999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.1200000000000001,"5min":0.93100000000000005,"15min":0.70499999999999996},"min":{"1min":0.26200000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.77,"5min":7.165,"15min":7.165},"last":3.879}]},{"osd":2,"last update":"Mon Oct 9 10:47:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.02,"15min":0.72099999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.4569999999999999,"5min":7.9000000000000004,"15min":7.9000000000000004},"last":0.84999999999999998},{"interface":"front","average":{"1min":1.7569999999999999,"5min":0.90400000000000003,"15min":0.68400000000000005},"min":{"1min":0.38,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":10.352,"5min":10.352,"15min":10.352},"last":0.30299999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:46:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5960000000000001,"5min":1.266,"15min":0.90400000000000003},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":9.984,"5min":9.984,"15min":9.984},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.6509999999999998,"5min":1.2030000000000001,"15min":0.80900000000000005},"min":{"1min":0.34999999999999998,"5min":0.317,"15min":0.317},"max":{"1min":11.657,"5min":11.657,"15min":11.657},"last":0.36399999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.53200000000000003,"15min":0.46700000000000003},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":3.7919999999999998,"5min":3.7919999999999998,"15min":3.7919999999999998},"last":0.26500000000000001},{"interface":"front","average":{"1min":1.048,"5min":0.68500000000000005,"15min":0.52900000000000003},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.9569999999999999,"5min":4.9569999999999999,"15min":4.9569999999999999},"last":0.40899999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.66300000000000003,"15min":0.53500000000000003},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":2.8900000000000001,"5min":2.9529999999999998,"15min":2.9529999999999998},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.63300000000000001,"15min":0.53800000000000003},"min":{"1min":0.34100000000000003,"5min":0.33300000000000002,"15min":0.30299999999999999},"max":{"1min":3.8159999999999998,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":0.375}]},{"osd":7,"last update":"Mon Oct 9 10:47:22 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.67100000000000004,"15min":0.58799999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.4729999999999999,"5min":4.0510000000000002,"15min":4.0510000000000002},"last":0.435},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.70899999999999996,"15min":0.60099999999999998},"min":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":4.1289999999999996,"5min":4.1289999999999996,"15min":4.1289999999999996},"last":0.46000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":701566976,"data_stored":701562904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1695354880,"data_stored":1695350808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2998403072,"data_stored":2998403072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2782461952,"data_stored":2782461952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1347092480,"data_stored":1347092480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1748439040,"data_stored":1748434968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1436090368,"data_stored":1436090368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1880489984,"data_stored":1880489984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:47:31.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:47:31.927 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:31.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:31.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:32.035 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:32.088 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:32.095 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:32.124 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:32.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:32.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:32.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:32.252 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:47:32.287 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:32 smithi155 ceph-mon[184854]: from='client.44849 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:47:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:32 smithi155 ceph-mon[184854]: pgmap v362: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 31152/191442 objects degraded (16.272%); 9397/191442 objects misplaced (4.909%); 5.8 MiB/s, 206 objects/s recovering 2023-10-09T10:47:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:32 smithi155 ceph-mon[184854]: failed to encode map e173 with expected crc 2023-10-09T10:47:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:32 smithi155 ceph-mon[184854]: failed to encode map e173 with expected crc 2023-10-09T10:47:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:32 smithi155 ceph-mon[184854]: failed to encode map e173 with expected crc 2023-10-09T10:47:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:32 smithi155 ceph-mon[184854]: osdmap e173: 8 total, 8 up, 7 in 2023-10-09T10:47:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:32 smithi155 ceph-mon[184854]: failed to encode map e173 with expected crc 2023-10-09T10:47:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:32 smithi155 ceph-mon[184854]: failed to encode map e173 with expected crc 2023-10-09T10:47:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:32 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1248281503' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:47:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:32 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:47:32.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[195507]: from='client.44849 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:47:32.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[195507]: pgmap v362: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 31152/191442 objects degraded (16.272%); 9397/191442 objects misplaced (4.909%); 5.8 MiB/s, 206 objects/s recovering 2023-10-09T10:47:32.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[195507]: failed to encode map e173 with expected crc 2023-10-09T10:47:32.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[195507]: failed to encode map e173 with expected crc 2023-10-09T10:47:32.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[195507]: failed to encode map e173 with expected crc 2023-10-09T10:47:32.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[195507]: osdmap e173: 8 total, 8 up, 7 in 2023-10-09T10:47:32.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[195507]: failed to encode map e173 with expected crc 2023-10-09T10:47:32.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[195507]: failed to encode map e173 with expected crc 2023-10-09T10:47:32.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1248281503' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:47:32.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:47:32.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[200197]: from='client.44849 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:47:32.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[200197]: pgmap v362: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 31152/191442 objects degraded (16.272%); 9397/191442 objects misplaced (4.909%); 5.8 MiB/s, 206 objects/s recovering 2023-10-09T10:47:32.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[200197]: failed to encode map e173 with expected crc 2023-10-09T10:47:32.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[200197]: failed to encode map e173 with expected crc 2023-10-09T10:47:32.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[200197]: failed to encode map e173 with expected crc 2023-10-09T10:47:32.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[200197]: osdmap e173: 8 total, 8 up, 7 in 2023-10-09T10:47:32.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[200197]: failed to encode map e173 with expected crc 2023-10-09T10:47:32.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[200197]: failed to encode map e173 with expected crc 2023-10-09T10:47:32.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1248281503' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:47:32.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:32 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:47:32.624 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:32.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:32.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:32.732 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:32.775 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:33.022 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:33.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:33.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:33.127 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:33.186 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:33.214 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:33.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:33.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:33.319 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:33.372 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:33.409 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:33.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:33.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:33.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:33 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:33.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:33 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:47:33.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:33 smithi155 ceph-mon[184854]: osdmap e174: 8 total, 8 up, 7 in 2023-10-09T10:47:33.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:33 smithi155 ceph-mon[184854]: failed to encode map e174 with expected crc 2023-10-09T10:47:33.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:33 smithi155 ceph-mon[184854]: failed to encode map e174 with expected crc 2023-10-09T10:47:33.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:33 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 31152/191442 objects degraded (16.272%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:33.486 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:33.523 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:33.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:33 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:33.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:33 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:47:33.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:33 smithi125 ceph-mon[195507]: osdmap e174: 8 total, 8 up, 7 in 2023-10-09T10:47:33.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:33 smithi125 ceph-mon[195507]: failed to encode map e174 with expected crc 2023-10-09T10:47:33.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:33 smithi125 ceph-mon[195507]: failed to encode map e174 with expected crc 2023-10-09T10:47:33.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:33 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 31152/191442 objects degraded (16.272%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:33.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:33 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:33.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:33 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:47:33.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:33 smithi125 ceph-mon[200197]: osdmap e174: 8 total, 8 up, 7 in 2023-10-09T10:47:33.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:33 smithi125 ceph-mon[200197]: failed to encode map e174 with expected crc 2023-10-09T10:47:33.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:33 smithi125 ceph-mon[200197]: failed to encode map e174 with expected crc 2023-10-09T10:47:33.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:33 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 31152/191442 objects degraded (16.272%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:33.575 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:47:33.596 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:33.621 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:47:33.630 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:33.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:33.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:33.667 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:47:33.704 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:47:33.739 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:33.744 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:47:33.776 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:33.786 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:47:33.813 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:33.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:33.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:33.866 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:47:33.916 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:47:33.919 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:33.920 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:47:33.920 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:47:33.920 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:47:33.920 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:47:33.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:47:33.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:47:33.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:47:33.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:47:33.957 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:47:33.958 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:47:33.958 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:47:33.958 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:47:33.958 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.958 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106414 6.11 6:8c9e6868:::benchmark_data_smithi125_244951_object106413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T10:47:33.958 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.602371+0000", 2023-10-09T10:47:33.958 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 91.350904768999996, 2023-10-09T10:47:33.958 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8167812699999999, 2023-10-09T10:47:33.958 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:33.958 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:47:33.958 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:33.959 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:33.959 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:33.959 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106414 2023-10-09T10:47:33.959 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.959 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:33.959 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.959 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:33.959 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602371+0000", 2023-10-09T10:47:33.959 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:33.959 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.959 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.959 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:33.960 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602371+0000", 2023-10-09T10:47:33.960 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:33.960 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.960 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.960 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:33.960 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602370+0000", 2023-10-09T10:47:33.960 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:33.960 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.960 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.960 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:33.960 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602394+0000", 2023-10-09T10:47:33.960 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4910999999999999e-05 2023-10-09T10:47:33.960 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.961 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.961 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:33.961 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602407+0000", 2023-10-09T10:47:33.961 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2272999999999999e-05 2023-10-09T10:47:33.961 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.961 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.961 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:33.961 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602417+0000", 2023-10-09T10:47:33.961 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9170000000000001e-06 2023-10-09T10:47:33.961 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.961 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.961 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:33.962 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602505+0000", 2023-10-09T10:47:33.962 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8245999999999995e-05 2023-10-09T10:47:33.962 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.962 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.962 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:33.962 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602517+0000", 2023-10-09T10:47:33.962 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1804000000000001e-05 2023-10-09T10:47:33.962 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.962 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.962 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:33.962 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:07.416738+0000", 2023-10-09T10:47:33.962 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8142212950000003 2023-10-09T10:47:33.963 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.963 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.963 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:33.963 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:07.419152+0000", 2023-10-09T10:47:33.963 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002414167 2023-10-09T10:47:33.963 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:33.963 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:33.963 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:33.963 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.963 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.963 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106433 6.11 6:8d28bd42:::benchmark_data_smithi125_244951_object106432:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T10:47:33.963 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.602427+0000", 2023-10-09T10:47:33.963 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 91.350848701999993, 2023-10-09T10:47:33.964 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.829405951, 2023-10-09T10:47:33.964 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:33.964 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:47:33.964 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:33.964 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:33.964 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:33.964 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106433 2023-10-09T10:47:33.964 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.964 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:33.964 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.964 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:33.964 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602427+0000", 2023-10-09T10:47:33.965 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:33.965 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.965 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.965 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:33.965 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602427+0000", 2023-10-09T10:47:33.965 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:33.965 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.965 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.965 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:33.965 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602426+0000", 2023-10-09T10:47:33.965 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:33.965 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.965 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.966 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:33.966 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602447+0000", 2023-10-09T10:47:33.966 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0704e-05 2023-10-09T10:47:33.966 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.966 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.966 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:33.966 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602454+0000", 2023-10-09T10:47:33.966 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.131e-06 2023-10-09T10:47:33.966 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.966 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.966 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:33.966 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602460+0000", 2023-10-09T10:47:33.967 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.964e-06 2023-10-09T10:47:33.967 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.967 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.967 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:33.967 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602542+0000", 2023-10-09T10:47:33.967 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1854000000000006e-05 2023-10-09T10:47:33.967 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.967 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.967 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:47:33.967 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.602551+0000", 2023-10-09T10:47:33.967 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2739999999999996e-06 2023-10-09T10:47:33.967 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.968 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.968 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:33.968 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:07.429395+0000", 2023-10-09T10:47:33.968 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8268439179999998 2023-10-09T10:47:33.968 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.968 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.968 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:33.968 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:07.431833+0000", 2023-10-09T10:47:33.968 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024376409999999999 2023-10-09T10:47:33.968 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:33.968 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:33.968 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:33.968 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.969 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.969 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:128431 6.11 6:8cae1542:::benchmark_data_smithi125_244951_object21988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T10:47:33.969 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:44.912769+0000", 2023-10-09T10:47:33.969 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 49.040506422999997, 2023-10-09T10:47:33.969 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.307313492, 2023-10-09T10:47:33.969 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:33.969 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:33.969 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:33.969 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:33.969 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:33.969 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 128431 2023-10-09T10:47:33.969 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.969 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:33.970 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.970 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:33.970 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:44.912769+0000", 2023-10-09T10:47:33.970 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:33.970 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.970 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.970 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:33.970 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:44.912769+0000", 2023-10-09T10:47:33.970 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:33.970 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.970 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.970 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:33.971 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:44.912769+0000", 2023-10-09T10:47:33.971 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:33.971 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.971 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.971 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:33.971 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:44.912772+0000", 2023-10-09T10:47:33.971 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7629999999999998e-06 2023-10-09T10:47:33.971 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.971 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.971 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:33.971 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:44.912781+0000", 2023-10-09T10:47:33.971 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0100000000000001e-06 2023-10-09T10:47:33.971 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.972 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.972 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:33.972 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:44.912787+0000", 2023-10-09T10:47:33.972 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9660000000000001e-06 2023-10-09T10:47:33.972 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.972 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.972 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:33.972 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:44.914017+0000", 2023-10-09T10:47:33.972 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001229696 2023-10-09T10:47:33.972 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.972 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.972 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:47:33.972 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:44.915691+0000", 2023-10-09T10:47:33.973 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001673957 2023-10-09T10:47:33.973 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.973 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.973 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:33.973 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:44.946306+0000", 2023-10-09T10:47:33.973 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.030614709 2023-10-09T10:47:33.973 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.973 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.973 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:33.973 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:44.948467+0000", 2023-10-09T10:47:33.973 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021609929999999999 2023-10-09T10:47:33.973 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.974 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.974 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:33.974 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.185916+0000", 2023-10-09T10:47:33.974 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2374496589999999 2023-10-09T10:47:33.974 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.974 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.974 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:33.974 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.188782+0000", 2023-10-09T10:47:33.974 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028658429999999999 2023-10-09T10:47:33.974 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.974 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.974 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:33.975 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.193275+0000", 2023-10-09T10:47:33.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0044926209999999996 2023-10-09T10:47:33.975 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.975 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:33.975 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.208805+0000", 2023-10-09T10:47:33.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.01552985 2023-10-09T10:47:33.975 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.975 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:33.975 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.209833+0000", 2023-10-09T10:47:33.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001027888 2023-10-09T10:47:33.975 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.976 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:33.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.210913+0000", 2023-10-09T10:47:33.976 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010808040000000001 2023-10-09T10:47:33.976 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.976 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:33.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.212301+0000", 2023-10-09T10:47:33.976 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001387723 2023-10-09T10:47:33.976 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.976 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:33.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.220083+0000", 2023-10-09T10:47:33.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0077816209999999999 2023-10-09T10:47:33.977 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:33.977 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:33.977 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:33.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.977 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:128764 6.11 6:890703b0:::benchmark_data_smithi125_244951_object22321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T10:47:33.977 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:45.655926+0000", 2023-10-09T10:47:33.977 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 48.297349660000002, 2023-10-09T10:47:33.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5326520179999998, 2023-10-09T10:47:33.977 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:33.977 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:33.978 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:33.978 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:33.978 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:33.978 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 128764 2023-10-09T10:47:33.978 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.978 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:33.978 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.978 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:33.978 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:45.655926+0000", 2023-10-09T10:47:33.978 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:33.978 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.979 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.979 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:33.979 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:45.655926+0000", 2023-10-09T10:47:33.979 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:33.979 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.979 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.979 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:33.979 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:45.655924+0000", 2023-10-09T10:47:33.979 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:47:33.979 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.979 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.979 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:33.980 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:45.655934+0000", 2023-10-09T10:47:33.980 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6849999999999996e-06 2023-10-09T10:47:33.980 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.980 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.980 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:33.980 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:45.655947+0000", 2023-10-09T10:47:33.980 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3587000000000001e-05 2023-10-09T10:47:33.980 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.980 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.980 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:33.980 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:45.655957+0000", 2023-10-09T10:47:33.980 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7429999999999997e-06 2023-10-09T10:47:33.980 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.981 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.981 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:33.981 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:45.768870+0000", 2023-10-09T10:47:33.981 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.112913396 2023-10-09T10:47:33.981 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.981 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.981 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:33.981 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:45.770567+0000", 2023-10-09T10:47:33.981 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016967830000000001 2023-10-09T10:47:33.981 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.981 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.981 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:33.981 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.154803+0000", 2023-10-09T10:47:33.981 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.384235498 2023-10-09T10:47:33.982 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.982 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.982 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:33.982 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.157437+0000", 2023-10-09T10:47:33.982 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026340130000000002 2023-10-09T10:47:33.982 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.982 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.982 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:33.982 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.161693+0000", 2023-10-09T10:47:33.982 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042558700000000001 2023-10-09T10:47:33.982 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.982 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.983 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:33.983 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.177405+0000", 2023-10-09T10:47:33.983 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015712806999999999 2023-10-09T10:47:33.983 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.983 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.983 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:33.983 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.178384+0000", 2023-10-09T10:47:33.983 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000978542 2023-10-09T10:47:33.983 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.983 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.983 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:33.983 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.178750+0000", 2023-10-09T10:47:33.984 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00036581 2023-10-09T10:47:33.984 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.984 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.984 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:33.984 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.180454+0000", 2023-10-09T10:47:33.984 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0017046450000000001 2023-10-09T10:47:33.984 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.984 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.984 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:33.984 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.188578+0000", 2023-10-09T10:47:33.984 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0081236429999999998 2023-10-09T10:47:33.984 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:33.985 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:33.985 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:33.985 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.985 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.985 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:129049 6.11 6:8c5b5882:::benchmark_data_smithi125_244951_object22606:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T10:47:33.985 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:46.415395+0000", 2023-10-09T10:47:33.985 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 47.537880172999998, 2023-10-09T10:47:33.985 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6541893600000002, 2023-10-09T10:47:33.985 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:33.985 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:33.985 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:33.985 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:33.986 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:33.986 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 129049 2023-10-09T10:47:33.986 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.986 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:33.986 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.986 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:33.986 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.415395+0000", 2023-10-09T10:47:33.986 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:33.986 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.986 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.986 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:33.986 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.415395+0000", 2023-10-09T10:47:33.986 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:33.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.987 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.987 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:33.987 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.415394+0000", 2023-10-09T10:47:33.987 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:33.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.987 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.987 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:33.987 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.415401+0000", 2023-10-09T10:47:33.987 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0049999999999998e-06 2023-10-09T10:47:33.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.987 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.987 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:33.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.415412+0000", 2023-10-09T10:47:33.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0869999999999999e-05 2023-10-09T10:47:33.988 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.988 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:33.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.415419+0000", 2023-10-09T10:47:33.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6390000000000003e-06 2023-10-09T10:47:33.988 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.988 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:33.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.435520+0000", 2023-10-09T10:47:33.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.02010168 2023-10-09T10:47:33.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.989 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:47:33.989 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.437082+0000", 2023-10-09T10:47:33.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001561989 2023-10-09T10:47:33.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.989 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:33.989 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.451196+0000", 2023-10-09T10:47:33.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014113491000000001 2023-10-09T10:47:33.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.990 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:33.990 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:46.453080+0000", 2023-10-09T10:47:33.990 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018843530000000001 2023-10-09T10:47:33.990 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.990 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.990 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:33.990 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.031854+0000", 2023-10-09T10:47:33.990 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5787737850000001 2023-10-09T10:47:33.990 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.990 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.990 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:33.990 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.034219+0000", 2023-10-09T10:47:33.990 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023646800000000001 2023-10-09T10:47:33.991 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.991 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.991 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:33.991 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.038470+0000", 2023-10-09T10:47:33.991 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042516050000000003 2023-10-09T10:47:33.991 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.991 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.991 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:33.991 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.048123+0000", 2023-10-09T10:47:33.991 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0096528199999999995 2023-10-09T10:47:33.993 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.993 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.993 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:33.993 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.049169+0000", 2023-10-09T10:47:33.993 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010462690000000001 2023-10-09T10:47:33.993 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.993 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.993 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:33.993 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.061330+0000", 2023-10-09T10:47:33.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012161137000000001 2023-10-09T10:47:33.994 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.994 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.994 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:33.994 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.062587+0000", 2023-10-09T10:47:33.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012563229999999999 2023-10-09T10:47:33.994 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.994 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.994 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:33.994 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.069585+0000", 2023-10-09T10:47:33.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0069981339999999996 2023-10-09T10:47:33.994 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:33.995 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:33.995 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:33.995 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.995 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:129526 6.11 6:89e83fb7:::benchmark_data_smithi125_244951_object23083:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T10:47:33.995 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:47.846316+0000", 2023-10-09T10:47:33.995 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 46.106959263, 2023-10-09T10:47:33.995 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1346659370000001, 2023-10-09T10:47:33.995 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:33.995 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:33.995 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:33.995 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:33.995 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:33.996 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 129526 2023-10-09T10:47:33.996 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.996 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:33.996 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.996 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:33.996 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.846316+0000", 2023-10-09T10:47:33.996 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:33.996 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.996 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.996 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:33.996 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.846316+0000", 2023-10-09T10:47:33.996 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:33.997 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.997 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:33.997 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.846315+0000", 2023-10-09T10:47:33.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:33.997 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.997 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:33.997 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.846322+0000", 2023-10-09T10:47:33.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8499999999999996e-06 2023-10-09T10:47:33.997 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.997 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:33.998 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.846335+0000", 2023-10-09T10:47:33.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3698e-05 2023-10-09T10:47:33.998 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.998 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.998 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:33.998 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.846343+0000", 2023-10-09T10:47:33.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.994e-06 2023-10-09T10:47:33.998 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.998 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.998 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:33.998 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.855810+0000", 2023-10-09T10:47:33.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0094664350000000005 2023-10-09T10:47:33.999 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:33.999 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:47.857856+0000", 2023-10-09T10:47:33.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020458020000000002 2023-10-09T10:47:33.999 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:33.999 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.941492+0000", 2023-10-09T10:47:33.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0836365619999999 2023-10-09T10:47:33.999 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:33.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:33.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:34.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.944325+0000", 2023-10-09T10:47:34.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028328239999999998 2023-10-09T10:47:34.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.000 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:34.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.949308+0000", 2023-10-09T10:47:34.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049832419999999997 2023-10-09T10:47:34.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.000 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.958095+0000", 2023-10-09T10:47:34.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0087865400000000007 2023-10-09T10:47:34.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.958901+0000", 2023-10-09T10:47:34.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000806477 2023-10-09T10:47:34.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:34.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.972234+0000", 2023-10-09T10:47:34.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013332759 2023-10-09T10:47:34.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:34.002 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.973806+0000", 2023-10-09T10:47:34.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001571598 2023-10-09T10:47:34.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.002 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:34.002 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.980982+0000", 2023-10-09T10:47:34.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0071766349999999998 2023-10-09T10:47:34.002 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.002 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.002 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.003 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:129998 6.11 6:8d110458:::benchmark_data_smithi125_244951_object23555:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T10:47:34.003 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:48.717822+0000", 2023-10-09T10:47:34.003 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 45.235453333000002, 2023-10-09T10:47:34.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.382684391, 2023-10-09T10:47:34.003 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:34.003 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:34.003 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:34.003 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:34.003 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:34.003 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 129998 2023-10-09T10:47:34.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.003 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:34.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:34.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.717822+0000", 2023-10-09T10:47:34.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:34.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.717822+0000", 2023-10-09T10:47:34.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:34.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.717821+0000", 2023-10-09T10:47:34.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:47:34.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:34.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.717827+0000", 2023-10-09T10:47:34.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7089999999999999e-06 2023-10-09T10:47:34.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:34.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.717837+0000", 2023-10-09T10:47:34.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0176e-05 2023-10-09T10:47:34.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.006 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.006 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:34.006 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.717844+0000", 2023-10-09T10:47:34.006 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8159999999999996e-06 2023-10-09T10:47:34.006 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.006 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.006 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.006 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.837403+0000", 2023-10-09T10:47:34.006 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.11955855 2023-10-09T10:47:34.006 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.006 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.006 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:47:34.007 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.838863+0000", 2023-10-09T10:47:34.007 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014599000000000001 2023-10-09T10:47:34.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.007 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.007 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.922516+0000", 2023-10-09T10:47:34.007 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.083653925000000004 2023-10-09T10:47:34.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.007 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:34.007 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:48.924579+0000", 2023-10-09T10:47:34.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020621839999999999 2023-10-09T10:47:34.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.064505+0000", 2023-10-09T10:47:34.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1399258889999999 2023-10-09T10:47:34.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:34.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.066991+0000", 2023-10-09T10:47:34.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024869340000000001 2023-10-09T10:47:34.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:34.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.071691+0000", 2023-10-09T10:47:34.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0047000280000000002 2023-10-09T10:47:34.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.090792+0000", 2023-10-09T10:47:34.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.019100500999999999 2023-10-09T10:47:34.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:34.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.091122+0000", 2023-10-09T10:47:34.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000330248 2023-10-09T10:47:34.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.010 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.010 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.010 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.092184+0000", 2023-10-09T10:47:34.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010615869999999999 2023-10-09T10:47:34.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.010 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.010 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:34.010 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.093694+0000", 2023-10-09T10:47:34.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001509846 2023-10-09T10:47:34.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:34.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.100507+0000", 2023-10-09T10:47:34.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0068130769999999998 2023-10-09T10:47:34.011 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.011 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.011 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.011 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:130268 6.11 6:8a293a7f:::benchmark_data_smithi125_244951_object23825:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T10:47:34.011 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:49.256642+0000", 2023-10-09T10:47:34.011 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 44.696633802000001, 2023-10-09T10:47:34.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.219981108, 2023-10-09T10:47:34.012 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:34.012 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:34.012 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:34.012 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:34.012 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:34.012 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 130268 2023-10-09T10:47:34.012 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.012 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:34.012 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.012 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:34.012 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:49.256642+0000", 2023-10-09T10:47:34.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.012 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.013 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:34.013 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:49.256642+0000", 2023-10-09T10:47:34.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.013 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:34.013 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:49.256640+0000", 2023-10-09T10:47:34.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:34.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.013 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:34.013 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:49.256647+0000", 2023-10-09T10:47:34.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5810000000000002e-06 2023-10-09T10:47:34.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:34.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:49.256660+0000", 2023-10-09T10:47:34.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3369e-05 2023-10-09T10:47:34.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:34.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:49.256669+0000", 2023-10-09T10:47:34.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5480000000000002e-06 2023-10-09T10:47:34.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:49.322167+0000", 2023-10-09T10:47:34.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.065497823999999996 2023-10-09T10:47:34.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:34.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:49.323850+0000", 2023-10-09T10:47:34.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001683056 2023-10-09T10:47:34.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.397649+0000", 2023-10-09T10:47:34.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0737986369999999 2023-10-09T10:47:34.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:34.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.400567+0000", 2023-10-09T10:47:34.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029189139999999999 2023-10-09T10:47:34.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:34.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.404758+0000", 2023-10-09T10:47:34.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041909520000000004 2023-10-09T10:47:34.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.017 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:34.017 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.438409+0000", 2023-10-09T10:47:34.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.033650600000000003 2023-10-09T10:47:34.017 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.017 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.017 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.467084+0000", 2023-10-09T10:47:34.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.028674583 2023-10-09T10:47:34.017 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.017 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.017 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.467875+0000", 2023-10-09T10:47:34.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00079136099999999997 2023-10-09T10:47:34.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.018 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:34.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.469438+0000", 2023-10-09T10:47:34.018 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015634970000000001 2023-10-09T10:47:34.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.018 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:34.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.476623+0000", 2023-10-09T10:47:34.018 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0071845609999999999 2023-10-09T10:47:34.018 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.018 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.019 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.019 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:130663 6.11 6:896d123d:::benchmark_data_smithi125_244951_object24220:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T10:47:34.019 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:50.079717+0000", 2023-10-09T10:47:34.019 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 43.873559081000003, 2023-10-09T10:47:34.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4123892310000001, 2023-10-09T10:47:34.019 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:34.019 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:34.019 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:34.019 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:34.019 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:34.019 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 130663 2023-10-09T10:47:34.020 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.020 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:34.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.020 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:34.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.079717+0000", 2023-10-09T10:47:34.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.020 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.020 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:34.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.079717+0000", 2023-10-09T10:47:34.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.020 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.021 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:34.021 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.079716+0000", 2023-10-09T10:47:34.021 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:47:34.021 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.021 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.021 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:34.021 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.079722+0000", 2023-10-09T10:47:34.021 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4969999999999998e-06 2023-10-09T10:47:34.021 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.021 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.021 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:34.021 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.079735+0000", 2023-10-09T10:47:34.021 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2931e-05 2023-10-09T10:47:34.022 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:34.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.079743+0000", 2023-10-09T10:47:34.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5650000000000004e-06 2023-10-09T10:47:34.022 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.101767+0000", 2023-10-09T10:47:34.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.022024467999999998 2023-10-09T10:47:34.022 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:34.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.103463+0000", 2023-10-09T10:47:34.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001696066 2023-10-09T10:47:34.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:51.437532+0000", 2023-10-09T10:47:34.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.334069314 2023-10-09T10:47:34.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:34.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:51.441279+0000", 2023-10-09T10:47:34.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037465480000000002 2023-10-09T10:47:34.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:34.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:51.445823+0000", 2023-10-09T10:47:34.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045444270000000002 2023-10-09T10:47:34.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:51.464483+0000", 2023-10-09T10:47:34.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018659347 2023-10-09T10:47:34.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:51.470553+0000", 2023-10-09T10:47:34.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0060704319999999997 2023-10-09T10:47:34.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:34.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:51.483592+0000", 2023-10-09T10:47:34.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013039123999999999 2023-10-09T10:47:34.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:34.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:51.485186+0000", 2023-10-09T10:47:34.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001593283 2023-10-09T10:47:34.026 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.026 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:34.026 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:51.492106+0000", 2023-10-09T10:47:34.026 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00692028 2023-10-09T10:47:34.026 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.026 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.026 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.026 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.026 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:131017 6.11 6:8c0d6c19:::benchmark_data_smithi125_244951_object24574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T10:47:34.026 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:50.824237+0000", 2023-10-09T10:47:34.026 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 43.129038209000001, 2023-10-09T10:47:34.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2568209770000001, 2023-10-09T10:47:34.027 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:34.027 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:34.027 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:34.027 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:34.027 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:34.027 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 131017 2023-10-09T10:47:34.027 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.027 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:34.027 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.027 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:34.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.824237+0000", 2023-10-09T10:47:34.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.028 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:34.028 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.824237+0000", 2023-10-09T10:47:34.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.028 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:34.028 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.824236+0000", 2023-10-09T10:47:34.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:47:34.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.029 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:34.029 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.824242+0000", 2023-10-09T10:47:34.029 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6490000000000002e-06 2023-10-09T10:47:34.029 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.029 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.029 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:34.029 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.824255+0000", 2023-10-09T10:47:34.029 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2252000000000001e-05 2023-10-09T10:47:34.029 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.029 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.029 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:34.029 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.824261+0000", 2023-10-09T10:47:34.029 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4609999999999996e-06 2023-10-09T10:47:34.030 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.964901+0000", 2023-10-09T10:47:34.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.140639758 2023-10-09T10:47:34.030 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:34.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:50.966557+0000", 2023-10-09T10:47:34.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016562199999999999 2023-10-09T10:47:34.030 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.031 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.031 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:52.018120+0000", 2023-10-09T10:47:34.031 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0515630499999999 2023-10-09T10:47:34.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.031 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.031 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:34.031 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:52.021165+0000", 2023-10-09T10:47:34.031 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030445279999999999 2023-10-09T10:47:34.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.031 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.031 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:34.031 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:52.025774+0000", 2023-10-09T10:47:34.031 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046090000000000002 2023-10-09T10:47:34.032 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.032 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.032 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:34.032 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:52.069558+0000", 2023-10-09T10:47:34.032 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.043784557000000002 2023-10-09T10:47:34.032 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.032 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.032 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.032 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:52.070517+0000", 2023-10-09T10:47:34.032 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00095887200000000004 2023-10-09T10:47:34.032 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.032 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:52.071543+0000", 2023-10-09T10:47:34.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010259469999999999 2023-10-09T10:47:34.033 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:34.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:52.073132+0000", 2023-10-09T10:47:34.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015895620000000001 2023-10-09T10:47:34.033 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:34.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:52.081058+0000", 2023-10-09T10:47:34.034 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0079259670000000008 2023-10-09T10:47:34.034 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.034 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.034 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.034 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.034 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.034 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:132226 6.11 6:89c53517:::benchmark_data_smithi125_244951_object25783:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T10:47:34.034 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:53.094543+0000", 2023-10-09T10:47:34.034 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 40.858732347999997, 2023-10-09T10:47:34.034 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4342427340000001, 2023-10-09T10:47:34.034 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:34.034 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:34.035 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:34.035 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:34.035 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:34.035 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 132226 2023-10-09T10:47:34.035 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.035 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:34.035 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.035 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:34.035 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.094543+0000", 2023-10-09T10:47:34.035 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.035 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.035 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.035 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:34.036 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.094543+0000", 2023-10-09T10:47:34.036 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.036 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:34.036 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.094542+0000", 2023-10-09T10:47:34.036 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:34.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.036 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:34.036 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.094549+0000", 2023-10-09T10:47:34.036 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6599999999999998e-06 2023-10-09T10:47:34.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.037 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.037 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:34.037 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.094560+0000", 2023-10-09T10:47:34.037 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1287999999999999e-05 2023-10-09T10:47:34.037 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.037 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.037 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:34.037 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.094568+0000", 2023-10-09T10:47:34.037 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9419999999999995e-06 2023-10-09T10:47:34.037 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.037 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.037 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.221247+0000", 2023-10-09T10:47:34.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12667890000000001 2023-10-09T10:47:34.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:34.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.223190+0000", 2023-10-09T10:47:34.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019430630000000001 2023-10-09T10:47:34.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:54.491807+0000", 2023-10-09T10:47:34.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2686171479999999 2023-10-09T10:47:34.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.039 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:34.039 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:54.494166+0000", 2023-10-09T10:47:34.039 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023585260000000001 2023-10-09T10:47:34.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.039 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:34.039 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:54.498384+0000", 2023-10-09T10:47:34.039 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042179979999999997 2023-10-09T10:47:34.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.040 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:54.518184+0000", 2023-10-09T10:47:34.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.019800477 2023-10-09T10:47:34.040 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.040 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.040 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:34.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:54.518445+0000", 2023-10-09T10:47:34.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00026047499999999998 2023-10-09T10:47:34.040 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.040 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.040 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:54.519767+0000", 2023-10-09T10:47:34.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013220009999999999 2023-10-09T10:47:34.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:34.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:54.521247+0000", 2023-10-09T10:47:34.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001480727 2023-10-09T10:47:34.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:34.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:54.528786+0000", 2023-10-09T10:47:34.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0075387379999999997 2023-10-09T10:47:34.041 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.041 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.041 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.042 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.042 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.042 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:132525 6.11 6:8d4b977b:::benchmark_data_smithi125_244951_object26082:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T10:47:34.042 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:53.751707+0000", 2023-10-09T10:47:34.042 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 40.201568365, 2023-10-09T10:47:34.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.285650382, 2023-10-09T10:47:34.042 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:34.042 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:34.042 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:34.042 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:34.042 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:34.042 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 132525 2023-10-09T10:47:34.043 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.043 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:34.043 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.043 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:34.043 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.751707+0000", 2023-10-09T10:47:34.043 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.043 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.043 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.043 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:34.043 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.751707+0000", 2023-10-09T10:47:34.043 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.043 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.043 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:34.044 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.751706+0000", 2023-10-09T10:47:34.044 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:34.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:34.044 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.751713+0000", 2023-10-09T10:47:34.044 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8809999999999998e-06 2023-10-09T10:47:34.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:34.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.751727+0000", 2023-10-09T10:47:34.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.394e-05 2023-10-09T10:47:34.045 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.045 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:34.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.751734+0000", 2023-10-09T10:47:34.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.058e-06 2023-10-09T10:47:34.045 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.045 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.786286+0000", 2023-10-09T10:47:34.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.034552353000000001 2023-10-09T10:47:34.046 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.046 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.046 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:47:34.046 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.787701+0000", 2023-10-09T10:47:34.046 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014152290000000001 2023-10-09T10:47:34.046 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.046 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.046 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.046 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.807402+0000", 2023-10-09T10:47:34.046 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.019700856999999999 2023-10-09T10:47:34.046 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.047 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:34.047 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:53.808994+0000", 2023-10-09T10:47:34.047 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015919339999999999 2023-10-09T10:47:34.047 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.047 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.047 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:54.943032+0000", 2023-10-09T10:47:34.047 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1340380219999999 2023-10-09T10:47:34.047 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.047 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:34.047 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:54.945936+0000", 2023-10-09T10:47:34.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029039539999999998 2023-10-09T10:47:34.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.048 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:34.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:54.950688+0000", 2023-10-09T10:47:34.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0047518969999999997 2023-10-09T10:47:34.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.048 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:34.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:54.981273+0000", 2023-10-09T10:47:34.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.030585004999999998 2023-10-09T10:47:34.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:55.022806+0000", 2023-10-09T10:47:34.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.041533187999999999 2023-10-09T10:47:34.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:55.028233+0000", 2023-10-09T10:47:34.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0054270259999999997 2023-10-09T10:47:34.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:34.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:55.030054+0000", 2023-10-09T10:47:34.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001820248 2023-10-09T10:47:34.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:34.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:55.037358+0000", 2023-10-09T10:47:34.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0073040969999999998 2023-10-09T10:47:34.050 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.050 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.050 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.050 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:136503 6.11 6:8bb765c4:::benchmark_data_smithi125_244951_object30060:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:47:34.051 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:02.970011+0000", 2023-10-09T10:47:34.051 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 30.983264948999999, 2023-10-09T10:47:34.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1524496529999999, 2023-10-09T10:47:34.051 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:34.051 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:34.051 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:34.051 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:34.051 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:34.051 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 136503 2023-10-09T10:47:34.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.051 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:34.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:34.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.970011+0000", 2023-10-09T10:47:34.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:34.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.970011+0000", 2023-10-09T10:47:34.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:34.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.970010+0000", 2023-10-09T10:47:34.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:47:34.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.053 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:34.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.970014+0000", 2023-10-09T10:47:34.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0640000000000004e-06 2023-10-09T10:47:34.053 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.053 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:34.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.970024+0000", 2023-10-09T10:47:34.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0414999999999999e-05 2023-10-09T10:47:34.053 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.053 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:34.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.970031+0000", 2023-10-09T10:47:34.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2349999999999996e-06 2023-10-09T10:47:34.054 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.054 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.054 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.054 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:03.000042+0000", 2023-10-09T10:47:34.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.030011283999999999 2023-10-09T10:47:34.054 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.054 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.054 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:47:34.054 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:03.001327+0000", 2023-10-09T10:47:34.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012854489999999999 2023-10-09T10:47:34.054 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:03.014103+0000", 2023-10-09T10:47:34.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012775737000000001 2023-10-09T10:47:34.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:34.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:03.015941+0000", 2023-10-09T10:47:34.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018377300000000001 2023-10-09T10:47:34.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:04.007597+0000", 2023-10-09T10:47:34.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.991655693 2023-10-09T10:47:34.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.056 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:34.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:04.009957+0000", 2023-10-09T10:47:34.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023599649999999999 2023-10-09T10:47:34.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.056 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:34.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:04.014448+0000", 2023-10-09T10:47:34.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0044911059999999999 2023-10-09T10:47:34.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:04.026077+0000", 2023-10-09T10:47:34.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011629096 2023-10-09T10:47:34.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:04.026965+0000", 2023-10-09T10:47:34.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00088870799999999999 2023-10-09T10:47:34.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:34.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:04.111808+0000", 2023-10-09T10:47:34.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.084842694999999996 2023-10-09T10:47:34.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:34.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:04.113004+0000", 2023-10-09T10:47:34.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001195357 2023-10-09T10:47:34.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:34.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:04.122460+0000", 2023-10-09T10:47:34.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0094568410000000006 2023-10-09T10:47:34.059 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.059 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.059 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.059 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.059 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:138630 6.11 6:89ba69be:::benchmark_data_smithi125_244951_object32187:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2023-10-09T10:47:34.059 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:06.718410+0000", 2023-10-09T10:47:34.059 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 27.234865889999998, 2023-10-09T10:47:34.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3670684689999999, 2023-10-09T10:47:34.059 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:34.059 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:34.059 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:34.060 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:34.060 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:34.060 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 138630 2023-10-09T10:47:34.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.060 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:34.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:34.060 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:06.718410+0000", 2023-10-09T10:47:34.060 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:34.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:06.718410+0000", 2023-10-09T10:47:34.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:34.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:06.718409+0000", 2023-10-09T10:47:34.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:34.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:34.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:06.718412+0000", 2023-10-09T10:47:34.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.616e-06 2023-10-09T10:47:34.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:34.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:06.718418+0000", 2023-10-09T10:47:34.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5820000000000001e-06 2023-10-09T10:47:34.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:34.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:06.718426+0000", 2023-10-09T10:47:34.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.49e-06 2023-10-09T10:47:34.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:06.756795+0000", 2023-10-09T10:47:34.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.038369159 2023-10-09T10:47:34.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:47:34.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:06.758190+0000", 2023-10-09T10:47:34.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013948280000000001 2023-10-09T10:47:34.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:06.859139+0000", 2023-10-09T10:47:34.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.100948914 2023-10-09T10:47:34.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.064 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:34.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:06.860503+0000", 2023-10-09T10:47:34.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013642820000000001 2023-10-09T10:47:34.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.064 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.061998+0000", 2023-10-09T10:47:34.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.201495218 2023-10-09T10:47:34.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.064 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:34.065 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.064100+0000", 2023-10-09T10:47:34.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021016659999999999 2023-10-09T10:47:34.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:34.065 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.067648+0000", 2023-10-09T10:47:34.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003547999 2023-10-09T10:47:34.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.065 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.077394+0000", 2023-10-09T10:47:34.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0097461000000000006 2023-10-09T10:47:34.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:34.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.077686+0000", 2023-10-09T10:47:34.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00029188800000000002 2023-10-09T10:47:34.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.078453+0000", 2023-10-09T10:47:34.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00076705100000000004 2023-10-09T10:47:34.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:34.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.079687+0000", 2023-10-09T10:47:34.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012339339999999999 2023-10-09T10:47:34.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.067 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:34.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.085478+0000", 2023-10-09T10:47:34.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0057912010000000002 2023-10-09T10:47:34.067 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.067 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.067 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.068 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:138778 6.11 6:8bc6ce58:::benchmark_data_smithi125_244951_object32335:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2023-10-09T10:47:34.068 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:07.193532+0000", 2023-10-09T10:47:34.068 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 26.759743969999999, 2023-10-09T10:47:34.068 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.446739373, 2023-10-09T10:47:34.068 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:34.068 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:34.068 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:34.068 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:34.068 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:34.068 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 138778 2023-10-09T10:47:34.068 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.068 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:34.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:34.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:07.193532+0000", 2023-10-09T10:47:34.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.069 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:34.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:07.193532+0000", 2023-10-09T10:47:34.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.069 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:34.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:07.193530+0000", 2023-10-09T10:47:34.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:47:34.070 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.070 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:34.070 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:07.193539+0000", 2023-10-09T10:47:34.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0259999999999996e-06 2023-10-09T10:47:34.070 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.070 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:34.070 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:07.193553+0000", 2023-10-09T10:47:34.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3896999999999999e-05 2023-10-09T10:47:34.070 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.071 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:34.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:07.193561+0000", 2023-10-09T10:47:34.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6340000000000007e-06 2023-10-09T10:47:34.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.071 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:07.205743+0000", 2023-10-09T10:47:34.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012181908999999999 2023-10-09T10:47:34.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.071 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:34.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:07.207407+0000", 2023-10-09T10:47:34.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016637259999999999 2023-10-09T10:47:34.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.597661+0000", 2023-10-09T10:47:34.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3902541509999999 2023-10-09T10:47:34.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:34.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.599779+0000", 2023-10-09T10:47:34.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002117725 2023-10-09T10:47:34.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:34.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.603699+0000", 2023-10-09T10:47:34.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039196489999999999 2023-10-09T10:47:34.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:34.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.628825+0000", 2023-10-09T10:47:34.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.025126454999999999 2023-10-09T10:47:34.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.631555+0000", 2023-10-09T10:47:34.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027296769999999998 2023-10-09T10:47:34.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.632300+0000", 2023-10-09T10:47:34.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00074536800000000005 2023-10-09T10:47:34.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:34.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.633447+0000", 2023-10-09T10:47:34.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011466200000000001 2023-10-09T10:47:34.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:34.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:08.640271+0000", 2023-10-09T10:47:34.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0068242989999999998 2023-10-09T10:47:34.075 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.075 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.075 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.075 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:143306 6.11 6:8e553f64:::benchmark_data_smithi125_244951_object36863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-10-09T10:47:34.076 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:17.681127+0000", 2023-10-09T10:47:34.076 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 16.272149022000001, 2023-10-09T10:47:34.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1391514759999999, 2023-10-09T10:47:34.076 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:34.076 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:34.076 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:34.076 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:34.076 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:34.076 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 143306 2023-10-09T10:47:34.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.076 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:34.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:34.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:17.681127+0000", 2023-10-09T10:47:34.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:34.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:17.681127+0000", 2023-10-09T10:47:34.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:34.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:17.681126+0000", 2023-10-09T10:47:34.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:47:34.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.078 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:34.078 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:17.681132+0000", 2023-10-09T10:47:34.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.818e-06 2023-10-09T10:47:34.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.078 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:34.078 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:17.681143+0000", 2023-10-09T10:47:34.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1575e-05 2023-10-09T10:47:34.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:34.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:17.681151+0000", 2023-10-09T10:47:34.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5440000000000001e-06 2023-10-09T10:47:34.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:17.774373+0000", 2023-10-09T10:47:34.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.093222439000000004 2023-10-09T10:47:34.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:34.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:17.775672+0000", 2023-10-09T10:47:34.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012993550000000001 2023-10-09T10:47:34.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.080 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.080 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.753420+0000", 2023-10-09T10:47:34.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97774716399999995 2023-10-09T10:47:34.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.080 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:34.080 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.755314+0000", 2023-10-09T10:47:34.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018947230000000001 2023-10-09T10:47:34.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.080 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:34.081 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.758979+0000", 2023-10-09T10:47:34.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036650530000000001 2023-10-09T10:47:34.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.081 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.081 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:34.081 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.777863+0000", 2023-10-09T10:47:34.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018883395000000001 2023-10-09T10:47:34.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.081 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.081 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.081 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.810263+0000", 2023-10-09T10:47:34.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.032400391000000001 2023-10-09T10:47:34.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.812186+0000", 2023-10-09T10:47:34.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001922836 2023-10-09T10:47:34.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:34.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.813566+0000", 2023-10-09T10:47:34.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013802370000000001 2023-10-09T10:47:34.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:34.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.820278+0000", 2023-10-09T10:47:34.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0067118660000000004 2023-10-09T10:47:34.083 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.083 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.083 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.083 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:143494 6.11 6:8968c3a5:::benchmark_data_smithi125_244951_object37051:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-10-09T10:47:34.083 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:18.257328+0000", 2023-10-09T10:47:34.083 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 15.695947395999999, 2023-10-09T10:47:34.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.116860338, 2023-10-09T10:47:34.083 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:34.084 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:34.084 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:34.084 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:34.084 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:34.084 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 143494 2023-10-09T10:47:34.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.084 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:34.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:34.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.257328+0000", 2023-10-09T10:47:34.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:34.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.257328+0000", 2023-10-09T10:47:34.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:34.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.257327+0000", 2023-10-09T10:47:34.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:34.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:34.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.257334+0000", 2023-10-09T10:47:34.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1319999999999996e-06 2023-10-09T10:47:34.086 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.086 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.086 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:34.086 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.257349+0000", 2023-10-09T10:47:34.086 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5534999999999999e-05 2023-10-09T10:47:34.086 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.086 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.086 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:34.086 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.257357+0000", 2023-10-09T10:47:34.086 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6480000000000003e-06 2023-10-09T10:47:34.086 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.086 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.367782+0000", 2023-10-09T10:47:34.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.110425385 2023-10-09T10:47:34.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:34.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.369382+0000", 2023-10-09T10:47:34.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001599989 2023-10-09T10:47:34.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.347685+0000", 2023-10-09T10:47:34.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.978302687 2023-10-09T10:47:34.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.088 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:34.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.349692+0000", 2023-10-09T10:47:34.088 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020070370000000001 2023-10-09T10:47:34.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.088 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:34.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.353572+0000", 2023-10-09T10:47:34.088 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038799749999999999 2023-10-09T10:47:34.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.088 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.089 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.362922+0000", 2023-10-09T10:47:34.089 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0093499169999999993 2023-10-09T10:47:34.089 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.089 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.089 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.089 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.363748+0000", 2023-10-09T10:47:34.089 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00082581699999999996 2023-10-09T10:47:34.089 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.089 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.089 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:34.089 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.367234+0000", 2023-10-09T10:47:34.089 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0034856829999999998 2023-10-09T10:47:34.089 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.090 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.090 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:34.090 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.368526+0000", 2023-10-09T10:47:34.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012921460000000001 2023-10-09T10:47:34.090 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.090 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.090 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:34.090 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.374189+0000", 2023-10-09T10:47:34.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0056629080000000004 2023-10-09T10:47:34.090 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.090 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.090 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.090 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.091 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:143745 6.11 6:88eac2c0:::benchmark_data_smithi125_244951_object37302:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-10-09T10:47:34.091 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:18.905155+0000", 2023-10-09T10:47:34.091 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 15.048121075999999, 2023-10-09T10:47:34.091 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1255730289999999, 2023-10-09T10:47:34.091 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:34.091 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:34.091 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:34.091 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:34.091 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:34.091 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 143745 2023-10-09T10:47:34.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.092 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:34.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.092 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:34.092 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.905155+0000", 2023-10-09T10:47:34.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.092 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:34.092 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.905155+0000", 2023-10-09T10:47:34.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.092 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:34.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.905153+0000", 2023-10-09T10:47:34.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:34.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:34.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.905159+0000", 2023-10-09T10:47:34.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7769999999999999e-06 2023-10-09T10:47:34.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:34.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.905171+0000", 2023-10-09T10:47:34.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2359e-05 2023-10-09T10:47:34.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:34.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.905179+0000", 2023-10-09T10:47:34.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6320000000000008e-06 2023-10-09T10:47:34.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.985892+0000", 2023-10-09T10:47:34.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.080712882 2023-10-09T10:47:34.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:34.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:18.988295+0000", 2023-10-09T10:47:34.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024026809999999998 2023-10-09T10:47:34.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.997628+0000", 2023-10-09T10:47:34.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.009333322 2023-10-09T10:47:34.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:34.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.999890+0000", 2023-10-09T10:47:34.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022621469999999999 2023-10-09T10:47:34.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:34.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:20.003419+0000", 2023-10-09T10:47:34.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035284399999999999 2023-10-09T10:47:34.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:34.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:20.018667+0000", 2023-10-09T10:47:34.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015248309999999999 2023-10-09T10:47:34.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.097 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.097 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.097 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:20.022171+0000", 2023-10-09T10:47:34.097 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003503857 2023-10-09T10:47:34.097 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.097 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.097 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.097 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:20.023050+0000", 2023-10-09T10:47:34.097 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000878903 2023-10-09T10:47:34.097 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.097 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.097 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:34.098 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:20.024339+0000", 2023-10-09T10:47:34.098 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012893900000000001 2023-10-09T10:47:34.098 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.098 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:34.098 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:20.030728+0000", 2023-10-09T10:47:34.098 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.006388599 2023-10-09T10:47:34.098 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.098 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.098 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.098 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.098 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.099 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:143932 6.11 6:89f4da8e:::benchmark_data_smithi125_244951_object37489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e170)", 2023-10-09T10:47:34.099 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:19.448342+0000", 2023-10-09T10:47:34.099 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 14.504933877999999, 2023-10-09T10:47:34.099 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.127445539, 2023-10-09T10:47:34.099 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:34.099 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:34.099 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:34.099 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:34.099 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:34.099 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 143932 2023-10-09T10:47:34.099 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.099 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:34.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.100 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:34.100 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.448342+0000", 2023-10-09T10:47:34.100 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.100 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.100 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:34.100 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.448342+0000", 2023-10-09T10:47:34.100 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.100 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.100 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:34.100 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.448341+0000", 2023-10-09T10:47:34.101 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:34.101 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.101 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:34.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.448344+0000", 2023-10-09T10:47:34.101 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7700000000000002e-06 2023-10-09T10:47:34.101 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.101 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:34.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.448352+0000", 2023-10-09T10:47:34.101 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5100000000000001e-06 2023-10-09T10:47:34.101 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.101 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:34.102 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.448357+0000", 2023-10-09T10:47:34.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4770000000000002e-06 2023-10-09T10:47:34.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.102 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.102 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.102 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.563486+0000", 2023-10-09T10:47:34.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.11512900700000001 2023-10-09T10:47:34.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.102 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.102 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:34.102 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:19.564742+0000", 2023-10-09T10:47:34.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001255993 2023-10-09T10:47:34.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:20.525239+0000", 2023-10-09T10:47:34.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96049710799999999 2023-10-09T10:47:34.103 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:34.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:20.527528+0000", 2023-10-09T10:47:34.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022887889999999998 2023-10-09T10:47:34.103 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:34.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:20.531597+0000", 2023-10-09T10:47:34.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040685349999999999 2023-10-09T10:47:34.104 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.104 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.104 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.104 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:20.549479+0000", 2023-10-09T10:47:34.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.017882431000000001 2023-10-09T10:47:34.104 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.104 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.104 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:34.104 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:20.559991+0000", 2023-10-09T10:47:34.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010512044 2023-10-09T10:47:34.104 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.104 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.104 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.105 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:20.569553+0000", 2023-10-09T10:47:34.105 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0095621690000000006 2023-10-09T10:47:34.105 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.105 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.105 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:34.105 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:20.570617+0000", 2023-10-09T10:47:34.105 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010638889999999999 2023-10-09T10:47:34.105 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.105 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.105 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:34.105 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:20.575787+0000", 2023-10-09T10:47:34.105 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0051702789999999998 2023-10-09T10:47:34.105 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.106 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.106 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.106 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.106 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.106 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:146307 6.11 6:89f89e1c:::benchmark_data_smithi125_244951_object39864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e170)", 2023-10-09T10:47:34.106 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:23.938110+0000", 2023-10-09T10:47:34.106 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 10.015165870000001, 2023-10-09T10:47:34.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0644314260000001, 2023-10-09T10:47:34.106 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:47:34.106 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:34.106 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:47:34.106 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:47:34.107 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:34.107 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 146307 2023-10-09T10:47:34.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.107 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:47:34.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:47:34.107 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:23.938110+0000", 2023-10-09T10:47:34.107 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:47:34.107 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:23.938110+0000", 2023-10-09T10:47:34.107 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:47:34.108 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.108 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.108 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:47:34.108 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:23.938109+0000", 2023-10-09T10:47:34.108 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:47:34.108 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.108 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.108 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:47:34.108 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:23.938115+0000", 2023-10-09T10:47:34.108 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8429999999999998e-06 2023-10-09T10:47:34.108 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.108 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.108 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:47:34.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:23.938125+0000", 2023-10-09T10:47:34.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.061e-05 2023-10-09T10:47:34.109 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.109 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.109 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:47:34.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:23.938133+0000", 2023-10-09T10:47:34.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5279999999999998e-06 2023-10-09T10:47:34.109 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.109 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.109 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:23.987150+0000", 2023-10-09T10:47:34.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.049017505000000003 2023-10-09T10:47:34.109 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.110 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:47:34.110 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:23.988494+0000", 2023-10-09T10:47:34.110 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013432660000000001 2023-10-09T10:47:34.110 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.110 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:47:34.110 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:24.979417+0000", 2023-10-09T10:47:34.110 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99092309199999995 2023-10-09T10:47:34.110 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.110 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:47:34.111 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:24.981640+0000", 2023-10-09T10:47:34.111 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022229630000000001 2023-10-09T10:47:34.111 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.111 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.111 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:47:34.111 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:24.985270+0000", 2023-10-09T10:47:34.111 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036302679999999999 2023-10-09T10:47:34.111 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.111 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.111 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.111 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:24.989890+0000", 2023-10-09T10:47:34.111 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046198080000000004 2023-10-09T10:47:34.111 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.112 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.112 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:34.112 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:24.990506+0000", 2023-10-09T10:47:34.112 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00061581899999999998 2023-10-09T10:47:34.112 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.112 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.112 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:47:34.112 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:24.996059+0000", 2023-10-09T10:47:34.112 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0055530390000000001 2023-10-09T10:47:34.112 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.112 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.112 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:47:34.112 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:24.997187+0000", 2023-10-09T10:47:34.112 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001127926 2023-10-09T10:47:34.113 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:47:34.113 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:47:34.113 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:47:34.113 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:25.002541+0000", 2023-10-09T10:47:34.113 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0053546929999999998 2023-10-09T10:47:34.113 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.113 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.113 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.113 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:47:34.113 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:47:34.113 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:47:34.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:34.115 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:34.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:34.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:34.216 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:34.290 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:34 smithi155 ceph-mon[184854]: pgmap v365: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 439 op/s; 31152/189996 objects degraded (16.396%); 9323/189996 objects misplaced (4.907%); 2.0 MiB/s, 137 objects/s recovering 2023-10-09T10:47:34.290 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:34 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1835045593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:34.291 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:34.327 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:34.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:34.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:34.434 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:34.469 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:34 smithi125 ceph-mon[195507]: pgmap v365: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 439 op/s; 31152/189996 objects degraded (16.396%); 9323/189996 objects misplaced (4.907%); 2.0 MiB/s, 137 objects/s recovering 2023-10-09T10:47:34.469 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:34 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1835045593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:34.470 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:34 smithi125 ceph-mon[200197]: pgmap v365: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 439 op/s; 31152/189996 objects degraded (16.396%); 9323/189996 objects misplaced (4.907%); 2.0 MiB/s, 137 objects/s recovering 2023-10-09T10:47:34.470 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:34 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1835045593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:34.482 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:34.567 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:34.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:34.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:34.673 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:34.747 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:34.794 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:34.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:34.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:34.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:47:34.902 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:47:34.906 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:35.003 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:35.090 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:35.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:35.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:35.195 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:35.277 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:35.300 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:35.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:35.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:35.333 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:47:35.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:35.455 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:35.887 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:35.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:35.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:35.994 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:36.065 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:36.121 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:36.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:36.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:36.207 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:36.227 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:36.261 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:36 smithi125 ceph-mon[195507]: pgmap v366: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 31152/185640 objects degraded (16.781%); 8918/185640 objects misplaced (4.804%); 5.1 MiB/s, 186 objects/s recovering 2023-10-09T10:47:36.261 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:36 smithi125 ceph-mon[200197]: pgmap v366: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 31152/185640 objects degraded (16.781%); 8918/185640 objects misplaced (4.804%); 5.1 MiB/s, 186 objects/s recovering 2023-10-09T10:47:36.306 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:36.412 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:36.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:36.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:36.442 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:47:36.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:36 smithi155 ceph-mon[184854]: pgmap v366: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 104 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 31152/185640 objects degraded (16.781%); 8918/185640 objects misplaced (4.804%); 5.1 MiB/s, 186 objects/s recovering 2023-10-09T10:47:36.525 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:36.550 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:47:36.550 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:47:36.550 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.550 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "MOSDPGPushReply(6.2 174/154 [PushReplyOp(6:410a5df0:::benchmark_data_smithi125_244951_object55660:head)])", 2023-10-09T10:47:36.550 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:36.547195+0000", 2023-10-09T10:47:36.551 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.00305273, 2023-10-09T10:47:36.551 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030746229999999999, 2023-10-09T10:47:36.551 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:36.551 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T10:47:36.551 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:36.551 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.551 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:36.551 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.547195+0000", 2023-10-09T10:47:36.551 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:36.551 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:36.551 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:36.552 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.547195+0000", 2023-10-09T10:47:36.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:36.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:36.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:36.552 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.547194+0000", 2023-10-09T10:47:36.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:47:36.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:36.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:36.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.547213+0000", 2023-10-09T10:47:36.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9253999999999999e-05 2023-10-09T10:47:36.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:36.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:36.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.547233+0000", 2023-10-09T10:47:36.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0296e-05 2023-10-09T10:47:36.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:36.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:36.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.547242+0000", 2023-10-09T10:47:36.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1530000000000003e-06 2023-10-09T10:47:36.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:36.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:36.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.548528+0000", 2023-10-09T10:47:36.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001285298 2023-10-09T10:47:36.554 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:36.554 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:36.554 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:36.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:36.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.555 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:153704 6.16 e174/107 6:6d122e73:::benchmark_data_smithi125_244951_object47261:head v 174'5794, mlcod=174'5792)", 2023-10-09T10:47:36.555 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:36.548915+0000", 2023-10-09T10:47:36.555 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0013328719999999999, 2023-10-09T10:47:36.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0013925669999999999, 2023-10-09T10:47:36.555 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:36.555 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:47:36.555 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:36.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:36.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.548915+0000", 2023-10-09T10:47:36.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:36.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:36.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:36.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.548915+0000", 2023-10-09T10:47:36.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:36.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:36.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:36.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.548914+0000", 2023-10-09T10:47:36.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:47:36.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:36.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.557 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:36.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.548926+0000", 2023-10-09T10:47:36.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.217e-05 2023-10-09T10:47:36.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:36.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.557 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:36.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.548941+0000", 2023-10-09T10:47:36.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5187e-05 2023-10-09T10:47:36.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:36.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:36.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.548947+0000", 2023-10-09T10:47:36.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9499999999999998e-06 2023-10-09T10:47:36.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:36.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:36.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.549007+0000", 2023-10-09T10:47:36.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9747000000000002e-05 2023-10-09T10:47:36.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:36.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:36.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:36.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.549025+0000", 2023-10-09T10:47:36.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7652e-05 2023-10-09T10:47:36.559 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:36.559 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:36.559 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:36.559 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:36.559 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T10:47:36.559 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 2 2023-10-09T10:47:36.559 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:47:36.601 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:36.741 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:36.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:36.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:36.804 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:47:36.804 INFO:teuthology.orchestra.run.smithi125.stdout:{"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 31152/185640 objects degraded (16.781%), 10 pgs degraded, 11 pgs undersized","count":21},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":442,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":174,"num_osds":8,"num_up_osds":8,"osd_up_since":1696848214,"num_in_osds":7,"osd_in_since":1696848391,"num_remapped_pgs":14},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":104},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":9},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"peering","count":1},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":61880,"data_bytes":4054292408,"bytes_used":12847566848,"bytes_avail":659117682688,"bytes_total":671965249536,"inactive_pgs_ratio":0.0084033617749810219,"degraded_objects":31152,"degraded_total":185640,"degraded_ratio":0.16780866192630894,"misplaced_objects":8918,"misplaced_total":185640,"misplaced_ratio":0.04803921568627452,"recovering_objects_per_sec":186,"recovering_bytes_per_sec":5295815,"recovering_keys_per_sec":0,"num_objects_recovered":1120,"num_bytes_recovered":31784960,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":657},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi125.satgiw","status":"up:active","gid":24575}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"}},"servicemap":{"epoch":20,"modified":"2023-10-09T10:44:35.008237+0000","services":{}},"progress_events":{"4db0080d-306f-4da6-a38c-4c702d14cb34":{"message":"Global Recovery Event (2m)\n [========================....] (remaining: 21s)","progress":0.87394958734512329,"add_to_ceph_s":true},"585851ac-6e70-4462-9176-ac50987fab77":{"message":"Upgrade to 18.2.0-598-gb4fa663f (3m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T10:47:36.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:36.914 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:36.989 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:47:37.101 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:37.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:37.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:37.206 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:37.305 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:37.339 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:37.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:37.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:37.444 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:37.456 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:37.458 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:47:37.458 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:47:37.465 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:47:37.527 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:37.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:37 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1476148369' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:47:37.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:37 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1476148369' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:47:37.621 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:37.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:37.628 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:37 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1476148369' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:47:37.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:37.730 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:37.808 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:37.840 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:37.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:37.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:37.945 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:38.001 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:38.097 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:38.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:38.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:38.204 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:38.252 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:38.365 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:38 smithi125 ceph-mon[195507]: pgmap v367: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 31152/180345 objects degraded (17.274%); 8371/180345 objects misplaced (4.642%); 6.2 MiB/s, 191 objects/s recovering 2023-10-09T10:47:38.365 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:38 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 31152/180345 objects degraded (17.274%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:38.365 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:38 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3594624738' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:38.365 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:38 smithi125 ceph-mon[200197]: pgmap v367: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 31152/180345 objects degraded (17.274%); 8371/180345 objects misplaced (4.642%); 6.2 MiB/s, 191 objects/s recovering 2023-10-09T10:47:38.366 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:38 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 31152/180345 objects degraded (17.274%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:38.366 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:38 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3594624738' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:38.396 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:38.398 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:38.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:38.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:38.507 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:38.582 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:38 smithi155 ceph-mon[184854]: pgmap v367: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 31152/180345 objects degraded (17.274%); 8371/180345 objects misplaced (4.642%); 6.2 MiB/s, 191 objects/s recovering 2023-10-09T10:47:38.583 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:38 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 31152/180345 objects degraded (17.274%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:38.583 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:38 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3594624738' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:38.591 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:38.643 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:38.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:38.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:38.655 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:47:38.655 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:47:38.656 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:47:38.656 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:47:38.656 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:47:38.692 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:47:38.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:38.826 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:38.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:38.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:38.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:39.045 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:47:39.073 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:39.123 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:39.140 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:47:39.178 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:47:39.201 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:47:39.229 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:47:39.270 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:47:39.321 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:47:39.370 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:47:39.411 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:47:39.433 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:47:39.433 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:47:39.433 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:47:39.433 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:47:39.434 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:47:39.434 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:47:39.434 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:47:39.434 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:47:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:39 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:47:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:39 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3594624738' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:47:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:39 smithi155 ceph-mon[184854]: osdmap e175: 8 total, 8 up, 7 in 2023-10-09T10:47:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:39 smithi155 ceph-mon[184854]: failed to encode map e175 with expected crc 2023-10-09T10:47:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:39 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4140079128' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:39.542 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:39 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:47:39.542 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:39 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3594624738' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:47:39.543 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:39 smithi125 ceph-mon[195507]: osdmap e175: 8 total, 8 up, 7 in 2023-10-09T10:47:39.543 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:39 smithi125 ceph-mon[195507]: failed to encode map e175 with expected crc 2023-10-09T10:47:39.543 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:39 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4140079128' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:39.543 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:39 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:47:39.543 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:39 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3594624738' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:47:39.543 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:39 smithi125 ceph-mon[200197]: osdmap e175: 8 total, 8 up, 7 in 2023-10-09T10:47:39.543 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:39 smithi125 ceph-mon[200197]: failed to encode map e175 with expected crc 2023-10-09T10:47:39.543 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:39 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4140079128' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:39.544 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:47:39.922 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:39.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:39.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:40.027 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:40.093 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:40.120 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:40.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:40.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:40.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:40.296 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:40.350 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:40.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:40.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:40.455 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:40.480 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:47:40.483 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:47:40.515 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:40.565 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:40.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:40.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:40.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:40 smithi125 ceph-mon[195507]: failed to encode map e175 with expected crc 2023-10-09T10:47:40.575 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:40 smithi125 ceph-mon[195507]: pgmap v369: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 31152/179706 objects degraded (17.335%); 8371/179706 objects misplaced (4.658%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T10:47:40.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:40 smithi125 ceph-mon[200197]: failed to encode map e175 with expected crc 2023-10-09T10:47:40.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:40 smithi125 ceph-mon[200197]: pgmap v369: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 31152/179706 objects degraded (17.335%); 8371/179706 objects misplaced (4.658%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T10:47:40.576 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:40 smithi155 ceph-mon[184854]: failed to encode map e175 with expected crc 2023-10-09T10:47:40.576 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:40 smithi155 ceph-mon[184854]: pgmap v369: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 31152/179706 objects degraded (17.335%); 8371/179706 objects misplaced (4.658%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T10:47:40.674 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:40.758 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:40.812 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:40.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:40.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:40.852 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:47:40.853 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:47:40.918 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":369,"stamp":"2023-10-09T10:47:39.115105+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3924662200,"num_objects":59902,"num_object_clones":0,"num_object_copies":179706,"num_objects_missing_on_primary":1406,"num_objects_missing":1406,"num_objects_degraded":31152,"num_objects_misplaced":8371,"num_objects_unfound":0,"num_objects_dirty":59902,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":144630,"num_write_kb":6260974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21196,"num_bytes_recovered":1296632578,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161889,"ondisk_log_size":161889,"up":357,"acting":346,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12130928,"kb_used_data":10524984,"kb_used_omap":44,"kb_used_meta":1605331,"kb_avail":644085136,"statfs":{"total":671965249536,"available":659543179264,"internally_reserved":0,"allocated":10777583616,"data_stored":10772309488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1643859361},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,1,2,8,6,4,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-256376832,"num_objects":-3912,"num_object_clones":0,"num_object_copies":-11736,"num_objects_missing_on_primary":-9,"num_objects_missing":-9,"num_objects_degraded":0,"num_objects_misplaced":-1026,"num_objects_unfound":0,"num_objects_dirty":-3912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3912,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":580,"num_bytes_recovered":38010880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.899478"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":167,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.228091+0000","last_change":"2023-10-09T10:46:32.250979+0000","last_active":"2023-10-09T10:47:31.228091+0000","last_peered":"2023-10-09T10:47:31.228091+0000","last_clean":"2023-10-09T10:47:31.228091+0000","last_became_active":"2023-10-09T10:46:32.250625+0000","last_became_peered":"2023-10-09T10:46:32.250625+0000","last_unstale":"2023-10-09T10:47:31.228091+0000","last_undegraded":"2023-10-09T10:47:31.228091+0000","last_fullsized":"2023-10-09T10:47:31.228091+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:34.858348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":142,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.259808+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:47:31.259808+0000","last_peered":"2023-10-09T10:47:31.259808+0000","last_clean":"2023-10-09T10:47:31.259808+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:47:31.259808+0000","last_undegraded":"2023-10-09T10:47:31.259808+0000","last_fullsized":"2023-10-09T10:47:31.259808+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":126,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.221027+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:47:31.221027+0000","last_peered":"2023-10-09T10:47:31.221027+0000","last_clean":"2023-10-09T10:47:31.221027+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:47:31.221027+0000","last_undegraded":"2023-10-09T10:47:31.221027+0000","last_fullsized":"2023-10-09T10:47:31.221027+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":42,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263078+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:47:31.263078+0000","last_peered":"2023-10-09T10:47:31.263078+0000","last_clean":"2023-10-09T10:47:31.263078+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:47:31.263078+0000","last_undegraded":"2023-10-09T10:47:31.263078+0000","last_fullsized":"2023-10-09T10:47:31.263078+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":178,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.258288+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:47:31.258288+0000","last_peered":"2023-10-09T10:47:31.258288+0000","last_clean":"2023-10-09T10:47:31.258288+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:47:31.258288+0000","last_undegraded":"2023-10-09T10:47:31.258288+0000","last_fullsized":"2023-10-09T10:47:31.258288+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":112,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262625+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:47:31.262625+0000","last_peered":"2023-10-09T10:47:31.262625+0000","last_clean":"2023-10-09T10:47:31.262625+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:47:31.262625+0000","last_undegraded":"2023-10-09T10:47:31.262625+0000","last_fullsized":"2023-10-09T10:47:31.262625+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"174'5964","reported_seq":10112,"reported_epoch":174,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:35.998758+0000","last_change":"2023-10-09T10:46:33.521602+0000","last_active":"2023-10-09T10:47:35.998758+0000","last_peered":"2023-10-09T10:47:35.998758+0000","last_clean":"2023-10-09T10:46:31.055751+0000","last_became_active":"2023-10-09T10:46:33.364058+0000","last_became_peered":"2023-10-09T10:46:33.364058+0000","last_unstale":"2023-10-09T10:47:35.998758+0000","last_undegraded":"2023-10-09T10:47:35.998758+0000","last_fullsized":"2023-10-09T10:47:35.998758+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5964,"log_dups_size":0,"ondisk_log_size":5964,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126451712,"num_objects":1930,"num_object_clones":0,"num_object_copies":5790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1928,"num_objects_unfound":0,"num_objects_dirty":1930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4928,"num_write_kb":219424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2010,"num_bytes_recovered":131727360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":1930}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":42,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263157+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:47:31.263157+0000","last_peered":"2023-10-09T10:47:31.263157+0000","last_clean":"2023-10-09T10:47:31.263157+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:47:31.263157+0000","last_undegraded":"2023-10-09T10:47:31.263157+0000","last_fullsized":"2023-10-09T10:47:31.263157+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":142,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.258359+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:47:31.258359+0000","last_peered":"2023-10-09T10:47:31.258359+0000","last_clean":"2023-10-09T10:47:31.258359+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:47:31.258359+0000","last_undegraded":"2023-10-09T10:47:31.258359+0000","last_fullsized":"2023-10-09T10:47:31.258359+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":507,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.265091+0000","last_change":"2023-10-09T10:46:05.620365+0000","last_active":"2023-10-09T10:47:31.265091+0000","last_peered":"2023-10-09T10:47:31.265091+0000","last_clean":"2023-10-09T10:47:31.265091+0000","last_became_active":"2023-10-09T10:46:05.620024+0000","last_became_peered":"2023-10-09T10:46:05.620024+0000","last_unstale":"2023-10-09T10:47:31.265091+0000","last_undegraded":"2023-10-09T10:47:31.265091+0000","last_fullsized":"2023-10-09T10:47:31.265091+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.13","version":"174'5787","reported_seq":5967,"reported_epoch":174,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:35.963659+0000","last_change":"2023-10-09T10:46:33.470698+0000","last_active":"2023-10-09T10:47:35.963659+0000","last_peered":"2023-10-09T10:47:35.963659+0000","last_clean":"2023-10-09T10:46:31.195699+0000","last_became_active":"2023-10-09T10:46:33.455632+0000","last_became_peered":"2023-10-09T10:46:33.455632+0000","last_unstale":"2023-10-09T10:47:35.963659+0000","last_undegraded":"2023-10-09T10:46:33.269323+0000","last_fullsized":"2023-10-09T10:46:33.241226+0000","mapping_epoch":154,"log_start":"103'107","ondisk_log_start":"103'107","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":5680,"log_dups_size":0,"ondisk_log_size":5680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:07:12.547333+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123535360,"num_objects":1885,"num_object_clones":0,"num_object_copies":5655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3315,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3072,"num_write_kb":105088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":1885},{"shards":"6,7","objects":-890}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":139,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262908+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:47:31.262908+0000","last_peered":"2023-10-09T10:47:31.262908+0000","last_clean":"2023-10-09T10:47:31.262908+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:47:31.262908+0000","last_undegraded":"2023-10-09T10:47:31.262908+0000","last_fullsized":"2023-10-09T10:47:31.262908+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":99,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.260199+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:47:31.260199+0000","last_peered":"2023-10-09T10:47:31.260199+0000","last_clean":"2023-10-09T10:47:31.260199+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:47:31.260199+0000","last_undegraded":"2023-10-09T10:47:31.260199+0000","last_fullsized":"2023-10-09T10:47:31.260199+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"174'5948","reported_seq":14069,"reported_epoch":174,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:36.453859+0000","last_change":"2023-10-09T10:46:03.683196+0000","last_active":"2023-10-09T10:47:36.453859+0000","last_peered":"2023-10-09T10:47:36.453859+0000","last_clean":"2023-10-09T10:46:02.177309+0000","last_became_active":"2023-10-09T10:46:03.681058+0000","last_became_peered":"2023-10-09T10:46:03.681058+0000","last_unstale":"2023-10-09T10:47:36.453859+0000","last_undegraded":"2023-10-09T10:47:36.453859+0000","last_fullsized":"2023-10-09T10:47:36.453859+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5948,"log_dups_size":0,"ondisk_log_size":5948,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123568128,"num_objects":1886,"num_object_clones":0,"num_object_copies":5658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1886,"num_objects_unfound":0,"num_objects_dirty":1886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4902,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[1,5,6],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1886}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":95,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.259736+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:47:31.259736+0000","last_peered":"2023-10-09T10:47:31.259736+0000","last_clean":"2023-10-09T10:47:31.259736+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:47:31.259736+0000","last_undegraded":"2023-10-09T10:47:31.259736+0000","last_fullsized":"2023-10-09T10:47:31.259736+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":135,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.220718+0000","last_change":"2023-10-09T10:46:32.249921+0000","last_active":"2023-10-09T10:47:31.220718+0000","last_peered":"2023-10-09T10:47:31.220718+0000","last_clean":"2023-10-09T10:47:31.220718+0000","last_became_active":"2023-10-09T10:46:32.249590+0000","last_became_peered":"2023-10-09T10:46:32.249590+0000","last_unstale":"2023-10-09T10:47:31.220718+0000","last_undegraded":"2023-10-09T10:47:31.220718+0000","last_fullsized":"2023-10-09T10:47:31.220718+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:25.169752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"174'5749","reported_seq":11864,"reported_epoch":174,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:47:34.559577+0000","last_change":"2023-10-09T10:46:32.280686+0000","last_active":"2023-10-09T10:47:34.559577+0000","last_peered":"2023-10-09T10:47:34.559577+0000","last_clean":"2023-10-09T10:44:56.105580+0000","last_became_active":"2023-10-09T10:46:07.402111+0000","last_became_peered":"2023-10-09T10:46:07.402111+0000","last_unstale":"2023-10-09T10:47:34.559577+0000","last_undegraded":"2023-10-09T10:47:34.559577+0000","last_fullsized":"2023-10-09T10:46:05.812663+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5749,"log_dups_size":0,"ondisk_log_size":5749,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120029184,"num_objects":1832,"num_object_clones":0,"num_object_copies":5496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1046,"num_objects_unfound":0,"num_objects_dirty":1832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4742,"num_write_kb":210336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1645,"num_bytes_recovered":107806720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5],"avail_no_missing":["3","1","5","6"],"object_location_counts":[{"shards":"1,2,3,5","objects":1045},{"shards":"1,3,5","objects":1},{"shards":"3,5","objects":786}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":144,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.228044+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:47:31.228044+0000","last_peered":"2023-10-09T10:47:31.228044+0000","last_clean":"2023-10-09T10:47:31.228044+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:47:31.228044+0000","last_undegraded":"2023-10-09T10:47:31.228044+0000","last_fullsized":"2023-10-09T10:47:31.228044+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":138,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.227074+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:47:31.227074+0000","last_peered":"2023-10-09T10:47:31.227074+0000","last_clean":"2023-10-09T10:47:31.227074+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:47:31.227074+0000","last_undegraded":"2023-10-09T10:47:31.227074+0000","last_fullsized":"2023-10-09T10:47:31.227074+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":507,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.264413+0000","last_change":"2023-10-09T10:46:05.618276+0000","last_active":"2023-10-09T10:47:31.264413+0000","last_peered":"2023-10-09T10:47:31.264413+0000","last_clean":"2023-10-09T10:47:31.264413+0000","last_became_active":"2023-10-09T10:46:05.616509+0000","last_became_peered":"2023-10-09T10:46:05.616509+0000","last_unstale":"2023-10-09T10:47:31.264413+0000","last_undegraded":"2023-10-09T10:47:31.264413+0000","last_fullsized":"2023-10-09T10:47:31.264413+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.16","version":"174'5751","reported_seq":5850,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:34.382860+0000","last_change":"2023-10-09T10:46:03.592682+0000","last_active":"2023-10-09T10:47:34.382860+0000","last_peered":"2023-10-09T10:47:34.382860+0000","last_clean":"2023-10-09T10:47:34.382860+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:47:34.382860+0000","last_undegraded":"2023-10-09T10:47:34.382860+0000","last_fullsized":"2023-10-09T10:47:34.382860+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5751,"log_dups_size":0,"ondisk_log_size":5751,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123076608,"num_objects":1878,"num_object_clones":0,"num_object_copies":5634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4708,"num_write_kb":210752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":156,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.227597+0000","last_change":"2023-10-09T10:46:32.250256+0000","last_active":"2023-10-09T10:47:31.227597+0000","last_peered":"2023-10-09T10:47:31.227597+0000","last_clean":"2023-10-09T10:47:31.227597+0000","last_became_active":"2023-10-09T10:46:32.243312+0000","last_became_peered":"2023-10-09T10:46:32.243312+0000","last_unstale":"2023-10-09T10:47:31.227597+0000","last_undegraded":"2023-10-09T10:47:31.227597+0000","last_fullsized":"2023-10-09T10:47:31.227597+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:08.059777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":165,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.258944+0000","last_change":"2023-10-09T10:46:32.237577+0000","last_active":"2023-10-09T10:47:31.258944+0000","last_peered":"2023-10-09T10:47:31.258944+0000","last_clean":"2023-10-09T10:47:31.258944+0000","last_became_active":"2023-10-09T10:46:32.237316+0000","last_became_peered":"2023-10-09T10:46:32.237316+0000","last_unstale":"2023-10-09T10:47:31.258944+0000","last_undegraded":"2023-10-09T10:47:31.258944+0000","last_fullsized":"2023-10-09T10:47:31.258944+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:39.684602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":508,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262710+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:47:31.262710+0000","last_peered":"2023-10-09T10:47:31.262710+0000","last_clean":"2023-10-09T10:47:31.262710+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:47:31.262710+0000","last_undegraded":"2023-10-09T10:47:31.262710+0000","last_fullsized":"2023-10-09T10:47:31.262710+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.17","version":"174'5699","reported_seq":5792,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:35.995586+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:47:35.995586+0000","last_peered":"2023-10-09T10:47:35.995586+0000","last_clean":"2023-10-09T10:47:35.995586+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:47:35.995586+0000","last_undegraded":"2023-10-09T10:47:35.995586+0000","last_fullsized":"2023-10-09T10:47:35.995586+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5699,"log_dups_size":0,"ondisk_log_size":5699,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118947840,"num_objects":1815,"num_object_clones":0,"num_object_copies":5445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4691,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":42,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262827+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:47:31.262827+0000","last_peered":"2023-10-09T10:47:31.262827+0000","last_clean":"2023-10-09T10:47:31.262827+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:47:31.262827+0000","last_undegraded":"2023-10-09T10:47:31.262827+0000","last_fullsized":"2023-10-09T10:47:31.262827+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":138,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262959+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:47:31.262959+0000","last_peered":"2023-10-09T10:47:31.262959+0000","last_clean":"2023-10-09T10:47:31.262959+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:47:31.262959+0000","last_undegraded":"2023-10-09T10:47:31.262959+0000","last_fullsized":"2023-10-09T10:47:31.262959+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":508,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263290+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:47:31.263290+0000","last_peered":"2023-10-09T10:47:31.263290+0000","last_clean":"2023-10-09T10:47:31.263290+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:47:31.263290+0000","last_undegraded":"2023-10-09T10:47:31.263290+0000","last_fullsized":"2023-10-09T10:47:31.263290+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"174'5867","reported_seq":5965,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:36.448682+0000","last_change":"2023-10-09T10:46:33.243540+0000","last_active":"2023-10-09T10:47:36.448682+0000","last_peered":"2023-10-09T10:47:36.448682+0000","last_clean":"2023-10-09T10:47:36.448682+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:47:36.448682+0000","last_undegraded":"2023-10-09T10:47:36.448682+0000","last_fullsized":"2023-10-09T10:47:36.448682+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5867,"log_dups_size":0,"ondisk_log_size":5867,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121929728,"num_objects":1861,"num_object_clones":0,"num_object_copies":5583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4813,"num_write_kb":213536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":84,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.226913+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:47:31.226913+0000","last_peered":"2023-10-09T10:47:31.226913+0000","last_clean":"2023-10-09T10:47:31.226913+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:47:31.226913+0000","last_undegraded":"2023-10-09T10:47:31.226913+0000","last_fullsized":"2023-10-09T10:47:31.226913+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":42,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263312+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:47:31.263312+0000","last_peered":"2023-10-09T10:47:31.263312+0000","last_clean":"2023-10-09T10:47:31.263312+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:47:31.263312+0000","last_undegraded":"2023-10-09T10:47:31.263312+0000","last_fullsized":"2023-10-09T10:47:31.263312+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":508,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262615+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:47:31.262615+0000","last_peered":"2023-10-09T10:47:31.262615+0000","last_clean":"2023-10-09T10:47:31.262615+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:47:31.262615+0000","last_undegraded":"2023-10-09T10:47:31.262615+0000","last_fullsized":"2023-10-09T10:47:31.262615+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"174'5854","reported_seq":5962,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:34.554455+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:47:34.554455+0000","last_peered":"2023-10-09T10:47:34.554455+0000","last_clean":"2023-10-09T10:47:34.554455+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:47:34.554455+0000","last_undegraded":"2023-10-09T10:47:34.554455+0000","last_fullsized":"2023-10-09T10:47:34.554455+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5854,"log_dups_size":0,"ondisk_log_size":5854,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122880000,"num_objects":1875,"num_object_clones":0,"num_object_copies":5625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4793,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":52,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262633+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:47:31.262633+0000","last_peered":"2023-10-09T10:47:31.262633+0000","last_clean":"2023-10-09T10:47:31.262633+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:47:31.262633+0000","last_undegraded":"2023-10-09T10:47:31.262633+0000","last_fullsized":"2023-10-09T10:47:31.262633+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":142,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.260830+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:47:31.260830+0000","last_peered":"2023-10-09T10:47:31.260830+0000","last_clean":"2023-10-09T10:47:31.260830+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:47:31.260830+0000","last_undegraded":"2023-10-09T10:47:31.260830+0000","last_fullsized":"2023-10-09T10:47:31.260830+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":508,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263546+0000","last_change":"2023-10-09T10:46:05.620574+0000","last_active":"2023-10-09T10:47:31.263546+0000","last_peered":"2023-10-09T10:47:31.263546+0000","last_clean":"2023-10-09T10:47:31.263546+0000","last_became_active":"2023-10-09T10:46:05.620212+0000","last_became_peered":"2023-10-09T10:46:05.620212+0000","last_unstale":"2023-10-09T10:47:31.263546+0000","last_undegraded":"2023-10-09T10:47:31.263546+0000","last_fullsized":"2023-10-09T10:47:31.263546+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"174'9902","reported_seq":14051,"reported_epoch":174,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:36.046690+0000","last_change":"2023-10-09T10:46:32.247903+0000","last_active":"2023-10-09T10:47:36.046690+0000","last_peered":"2023-10-09T10:47:36.046690+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:46:02.739355+0000","last_became_peered":"2023-10-09T10:46:02.739355+0000","last_unstale":"2023-10-09T10:47:36.046690+0000","last_undegraded":"2023-10-09T10:46:02.608805+0000","last_fullsized":"2023-10-09T10:46:02.583589+0000","mapping_epoch":142,"log_start":"104'1000","ondisk_log_start":"104'1000","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8902,"log_dups_size":0,"ondisk_log_size":8902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:32.899634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244449280,"num_objects":3730,"num_object_clones":0,"num_object_copies":11190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9902,"num_write_kb":436224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2022,"num_bytes_recovered":132513792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":625},{"shards":"5,6","objects":3104}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":167,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.225681+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:47:31.225681+0000","last_peered":"2023-10-09T10:47:31.225681+0000","last_clean":"2023-10-09T10:47:31.225681+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:47:31.225681+0000","last_undegraded":"2023-10-09T10:47:31.225681+0000","last_fullsized":"2023-10-09T10:47:31.225681+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":121,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262312+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:47:31.262312+0000","last_peered":"2023-10-09T10:47:31.262312+0000","last_clean":"2023-10-09T10:47:31.262312+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:47:31.262312+0000","last_undegraded":"2023-10-09T10:47:31.262312+0000","last_fullsized":"2023-10-09T10:47:31.262312+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":488,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.265160+0000","last_change":"2023-10-09T10:46:02.631378+0000","last_active":"2023-10-09T10:47:31.265160+0000","last_peered":"2023-10-09T10:47:31.265160+0000","last_clean":"2023-10-09T10:47:31.265160+0000","last_became_active":"2023-10-09T10:46:02.629142+0000","last_became_peered":"2023-10-09T10:46:02.629142+0000","last_unstale":"2023-10-09T10:47:31.265160+0000","last_undegraded":"2023-10-09T10:47:31.265160+0000","last_fullsized":"2023-10-09T10:47:31.265160+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.b","version":"174'9437","reported_seq":9523,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:34.535924+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:47:34.535924+0000","last_peered":"2023-10-09T10:47:34.535924+0000","last_clean":"2023-10-09T10:47:34.535924+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:47:34.535924+0000","last_undegraded":"2023-10-09T10:47:34.535924+0000","last_fullsized":"2023-10-09T10:47:34.535924+0000","mapping_epoch":98,"log_start":"119'3725","ondisk_log_start":"119'3725","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5712,"log_dups_size":0,"ondisk_log_size":5712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245301248,"num_objects":3743,"num_object_clones":0,"num_object_copies":11229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9437,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":140,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.225975+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:47:31.225975+0000","last_peered":"2023-10-09T10:47:31.225975+0000","last_clean":"2023-10-09T10:47:31.225975+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:47:31.225975+0000","last_undegraded":"2023-10-09T10:47:31.225975+0000","last_fullsized":"2023-10-09T10:47:31.225975+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":175,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.226834+0000","last_change":"2023-10-09T10:46:32.251035+0000","last_active":"2023-10-09T10:47:31.226834+0000","last_peered":"2023-10-09T10:47:31.226834+0000","last_clean":"2023-10-09T10:47:31.226834+0000","last_became_active":"2023-10-09T10:46:32.250293+0000","last_became_peered":"2023-10-09T10:46:32.250293+0000","last_unstale":"2023-10-09T10:47:31.226834+0000","last_undegraded":"2023-10-09T10:47:31.226834+0000","last_fullsized":"2023-10-09T10:47:31.226834+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.787766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":508,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262884+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:47:31.262884+0000","last_peered":"2023-10-09T10:47:31.262884+0000","last_clean":"2023-10-09T10:47:31.262884+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:47:31.262884+0000","last_undegraded":"2023-10-09T10:47:31.262884+0000","last_fullsized":"2023-10-09T10:47:31.262884+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"174'5862","reported_seq":5968,"reported_epoch":174,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:34.484424+0000","last_change":"2023-10-09T10:46:33.540719+0000","last_active":"2023-10-09T10:47:34.484424+0000","last_peered":"2023-10-09T10:47:34.484424+0000","last_clean":"2023-10-09T10:46:31.137696+0000","last_became_active":"2023-10-09T10:46:33.527650+0000","last_became_peered":"2023-10-09T10:46:33.527650+0000","last_unstale":"2023-10-09T10:47:34.484424+0000","last_undegraded":"2023-10-09T10:46:33.285468+0000","last_fullsized":"2023-10-09T10:46:33.235683+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5862,"log_dups_size":0,"ondisk_log_size":5862,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123830272,"num_objects":1889,"num_object_clones":0,"num_object_copies":5667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4797,"num_write_kb":213984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1877},{"shards":"3,7","objects":-920}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":95,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262878+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:47:31.262878+0000","last_peered":"2023-10-09T10:47:31.262878+0000","last_clean":"2023-10-09T10:47:31.262878+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:47:31.262878+0000","last_undegraded":"2023-10-09T10:47:31.262878+0000","last_fullsized":"2023-10-09T10:47:31.262878+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":112,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.220909+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:47:31.220909+0000","last_peered":"2023-10-09T10:47:31.220909+0000","last_clean":"2023-10-09T10:47:31.220909+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:47:31.220909+0000","last_undegraded":"2023-10-09T10:47:31.220909+0000","last_fullsized":"2023-10-09T10:47:31.220909+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":508,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263248+0000","last_change":"2023-10-09T10:46:05.620030+0000","last_active":"2023-10-09T10:47:31.263248+0000","last_peered":"2023-10-09T10:47:31.263248+0000","last_clean":"2023-10-09T10:47:31.263248+0000","last_became_active":"2023-10-09T10:46:05.619694+0000","last_became_peered":"2023-10-09T10:46:05.619694+0000","last_unstale":"2023-10-09T10:47:31.263248+0000","last_undegraded":"2023-10-09T10:47:31.263248+0000","last_fullsized":"2023-10-09T10:47:31.263248+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.9","version":"174'5882","reported_seq":5999,"reported_epoch":174,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:37.553150+0000","last_change":"2023-10-09T10:46:34.405499+0000","last_active":"2023-10-09T10:47:37.553150+0000","last_peered":"2023-10-09T10:47:37.553150+0000","last_clean":"2023-10-09T10:46:31.190603+0000","last_became_active":"2023-10-09T10:46:34.396448+0000","last_became_peered":"2023-10-09T10:46:34.396448+0000","last_unstale":"2023-10-09T10:47:37.553150+0000","last_undegraded":"2023-10-09T10:46:33.246949+0000","last_fullsized":"2023-10-09T10:46:33.237968+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5882,"log_dups_size":0,"ondisk_log_size":5882,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116260864,"num_objects":1774,"num_object_clones":0,"num_object_copies":5322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2004,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4840,"num_write_kb":211648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1303,"num_bytes_recovered":85393408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":471},{"shards":"0,7","objects":306}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":139,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262725+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:47:31.262725+0000","last_peered":"2023-10-09T10:47:31.262725+0000","last_clean":"2023-10-09T10:47:31.262725+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:47:31.262725+0000","last_undegraded":"2023-10-09T10:47:31.262725+0000","last_fullsized":"2023-10-09T10:47:31.262725+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":42,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263539+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:47:31.263539+0000","last_peered":"2023-10-09T10:47:31.263539+0000","last_clean":"2023-10-09T10:47:31.263539+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:47:31.263539+0000","last_undegraded":"2023-10-09T10:47:31.263539+0000","last_fullsized":"2023-10-09T10:47:31.263539+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":499,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.259955+0000","last_change":"2023-10-09T10:46:05.618134+0000","last_active":"2023-10-09T10:47:31.259955+0000","last_peered":"2023-10-09T10:47:31.259955+0000","last_clean":"2023-10-09T10:47:31.259955+0000","last_became_active":"2023-10-09T10:46:05.616057+0000","last_became_peered":"2023-10-09T10:46:05.616057+0000","last_unstale":"2023-10-09T10:47:31.259955+0000","last_undegraded":"2023-10-09T10:47:31.259955+0000","last_fullsized":"2023-10-09T10:47:31.259955+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"174'9487","reported_seq":9581,"reported_epoch":174,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:35.999112+0000","last_change":"2023-10-09T10:46:33.576530+0000","last_active":"2023-10-09T10:47:35.999112+0000","last_peered":"2023-10-09T10:47:35.999112+0000","last_clean":"2023-10-09T10:46:31.183481+0000","last_became_active":"2023-10-09T10:46:33.542304+0000","last_became_peered":"2023-10-09T10:46:33.542304+0000","last_unstale":"2023-10-09T10:47:35.999112+0000","last_undegraded":"2023-10-09T10:46:33.300581+0000","last_fullsized":"2023-10-09T10:46:33.246047+0000","mapping_epoch":154,"log_start":"106'2077","ondisk_log_start":"106'2077","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7410,"log_dups_size":0,"ondisk_log_size":7410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:26.739488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245563392,"num_objects":3747,"num_object_clones":0,"num_object_copies":11241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6617,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9487,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":3747},{"shards":"1,4","objects":-1814}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":166,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.258757+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:47:31.258757+0000","last_peered":"2023-10-09T10:47:31.258757+0000","last_clean":"2023-10-09T10:47:31.258757+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:47:31.258757+0000","last_undegraded":"2023-10-09T10:47:31.258757+0000","last_fullsized":"2023-10-09T10:47:31.258757+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":138,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.226539+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:47:31.226539+0000","last_peered":"2023-10-09T10:47:31.226539+0000","last_clean":"2023-10-09T10:47:31.226539+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:47:31.226539+0000","last_undegraded":"2023-10-09T10:47:31.226539+0000","last_fullsized":"2023-10-09T10:47:31.226539+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":488,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262785+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:47:31.262785+0000","last_peered":"2023-10-09T10:47:31.262785+0000","last_clean":"2023-10-09T10:47:31.262785+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:47:31.262785+0000","last_undegraded":"2023-10-09T10:47:31.262785+0000","last_fullsized":"2023-10-09T10:47:31.262785+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.f","version":"174'9355","reported_seq":9446,"reported_epoch":174,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:34.547917+0000","last_change":"2023-10-09T10:46:33.651756+0000","last_active":"2023-10-09T10:47:34.547917+0000","last_peered":"2023-10-09T10:47:34.547917+0000","last_clean":"2023-10-09T10:46:31.156458+0000","last_became_active":"2023-10-09T10:46:33.621003+0000","last_became_peered":"2023-10-09T10:46:33.621003+0000","last_unstale":"2023-10-09T10:47:34.547917+0000","last_undegraded":"2023-10-09T10:46:33.309813+0000","last_fullsized":"2023-10-09T10:46:33.236508+0000","mapping_epoch":154,"log_start":"118'3610","ondisk_log_start":"118'3610","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:13.704346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240320512,"num_objects":3667,"num_object_clones":0,"num_object_copies":11001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9355,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":3667},{"shards":"3,4","objects":-1785}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":113,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.228257+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:47:31.228257+0000","last_peered":"2023-10-09T10:47:31.228257+0000","last_clean":"2023-10-09T10:47:31.228257+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:47:31.228257+0000","last_undegraded":"2023-10-09T10:47:31.228257+0000","last_fullsized":"2023-10-09T10:47:31.228257+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":165,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.227832+0000","last_change":"2023-10-09T10:46:32.245583+0000","last_active":"2023-10-09T10:47:31.227832+0000","last_peered":"2023-10-09T10:47:31.227832+0000","last_clean":"2023-10-09T10:47:31.227832+0000","last_became_active":"2023-10-09T10:46:32.245271+0000","last_became_peered":"2023-10-09T10:46:32.245271+0000","last_unstale":"2023-10-09T10:47:31.227832+0000","last_undegraded":"2023-10-09T10:47:31.227832+0000","last_fullsized":"2023-10-09T10:47:31.227832+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.025051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":508,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263394+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:47:31.263394+0000","last_peered":"2023-10-09T10:47:31.263394+0000","last_clean":"2023-10-09T10:47:31.263394+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:47:31.263394+0000","last_undegraded":"2023-10-09T10:47:31.263394+0000","last_fullsized":"2023-10-09T10:47:31.263394+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"174'9617","reported_seq":9703,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:34.508553+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:47:34.508553+0000","last_peered":"2023-10-09T10:47:34.508553+0000","last_clean":"2023-10-09T10:47:34.508553+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:47:34.508553+0000","last_undegraded":"2023-10-09T10:47:34.508553+0000","last_fullsized":"2023-10-09T10:47:34.508553+0000","mapping_epoch":98,"log_start":"119'3878","ondisk_log_start":"119'3878","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5739,"log_dups_size":0,"ondisk_log_size":5739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240320512,"num_objects":3667,"num_object_clones":0,"num_object_copies":11001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9617,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":149,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.226759+0000","last_change":"2023-10-09T10:46:32.245685+0000","last_active":"2023-10-09T10:47:31.226759+0000","last_peered":"2023-10-09T10:47:31.226759+0000","last_clean":"2023-10-09T10:47:31.226759+0000","last_became_active":"2023-10-09T10:46:32.245314+0000","last_became_peered":"2023-10-09T10:46:32.245314+0000","last_unstale":"2023-10-09T10:47:31.226759+0000","last_undegraded":"2023-10-09T10:47:31.226759+0000","last_fullsized":"2023-10-09T10:47:31.226759+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:51.716860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":138,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.228660+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:47:31.228660+0000","last_peered":"2023-10-09T10:47:31.228660+0000","last_clean":"2023-10-09T10:47:31.228660+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:47:31.228660+0000","last_undegraded":"2023-10-09T10:47:31.228660+0000","last_fullsized":"2023-10-09T10:47:31.228660+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":499,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262711+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:47:31.262711+0000","last_peered":"2023-10-09T10:47:31.262711+0000","last_clean":"2023-10-09T10:47:31.262711+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:47:31.262711+0000","last_undegraded":"2023-10-09T10:47:31.262711+0000","last_fullsized":"2023-10-09T10:47:31.262711+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"174'9642","reported_seq":9746,"reported_epoch":174,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:36.029698+0000","last_change":"2023-10-09T10:46:02.754507+0000","last_active":"2023-10-09T10:47:36.029698+0000","last_peered":"2023-10-09T10:47:36.029698+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:46:02.747510+0000","last_became_peered":"2023-10-09T10:46:02.747510+0000","last_unstale":"2023-10-09T10:47:36.029698+0000","last_undegraded":"2023-10-09T10:46:02.605987+0000","last_fullsized":"2023-10-09T10:46:02.585248+0000","mapping_epoch":142,"log_start":"103'800","ondisk_log_start":"103'800","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8842,"log_dups_size":0,"ondisk_log_size":8842,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:59.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245956632,"num_objects":3754,"num_object_clones":0,"num_object_copies":11262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3754,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9642,"num_write_kb":428609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,2,5","objects":2718},{"shards":"1,5","objects":1036}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":143,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.258122+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:47:31.258122+0000","last_peered":"2023-10-09T10:47:31.258122+0000","last_clean":"2023-10-09T10:47:31.258122+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:47:31.258122+0000","last_undegraded":"2023-10-09T10:47:31.258122+0000","last_fullsized":"2023-10-09T10:47:31.258122+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":126,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.222952+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:47:31.222952+0000","last_peered":"2023-10-09T10:47:31.222952+0000","last_clean":"2023-10-09T10:47:31.222952+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:47:31.222952+0000","last_undegraded":"2023-10-09T10:47:31.222952+0000","last_fullsized":"2023-10-09T10:47:31.222952+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":508,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263024+0000","last_change":"2023-10-09T10:46:05.617983+0000","last_active":"2023-10-09T10:47:31.263024+0000","last_peered":"2023-10-09T10:47:31.263024+0000","last_clean":"2023-10-09T10:47:31.263024+0000","last_became_active":"2023-10-09T10:46:05.616021+0000","last_became_peered":"2023-10-09T10:46:05.616021+0000","last_unstale":"2023-10-09T10:47:31.263024+0000","last_undegraded":"2023-10-09T10:47:31.263024+0000","last_fullsized":"2023-10-09T10:47:31.263024+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":169,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.221297+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:47:31.221297+0000","last_peered":"2023-10-09T10:47:31.221297+0000","last_clean":"2023-10-09T10:47:31.221297+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:47:31.221297+0000","last_undegraded":"2023-10-09T10:47:31.221297+0000","last_fullsized":"2023-10-09T10:47:31.221297+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":166,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263034+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:47:31.263034+0000","last_peered":"2023-10-09T10:47:31.263034+0000","last_clean":"2023-10-09T10:47:31.263034+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:47:31.263034+0000","last_undegraded":"2023-10-09T10:47:31.263034+0000","last_fullsized":"2023-10-09T10:47:31.263034+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":111,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.220814+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:47:31.220814+0000","last_peered":"2023-10-09T10:47:31.220814+0000","last_clean":"2023-10-09T10:47:31.220814+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:47:31.220814+0000","last_undegraded":"2023-10-09T10:47:31.220814+0000","last_fullsized":"2023-10-09T10:47:31.220814+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"174'5763","reported_seq":9793,"reported_epoch":174,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:34.737950+0000","last_change":"2023-10-09T10:46:32.248218+0000","last_active":"2023-10-09T10:47:34.737950+0000","last_peered":"2023-10-09T10:47:34.737950+0000","last_clean":"2023-10-09T10:46:02.181912+0000","last_became_active":"2023-10-09T10:46:32.226237+0000","last_became_peered":"2023-10-09T10:46:32.226237+0000","last_unstale":"2023-10-09T10:47:34.737950+0000","last_undegraded":"2023-10-09T10:47:34.737950+0000","last_fullsized":"2023-10-09T10:47:34.737950+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5763,"log_dups_size":0,"ondisk_log_size":5763,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126386176,"num_objects":1928,"num_object_clones":0,"num_object_copies":5784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1928,"num_objects_unfound":0,"num_objects_dirty":1928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4716,"num_write_kb":212640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1928}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":507,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.258369+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:47:31.258369+0000","last_peered":"2023-10-09T10:47:31.258369+0000","last_clean":"2023-10-09T10:47:31.258369+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:47:31.258369+0000","last_undegraded":"2023-10-09T10:47:31.258369+0000","last_fullsized":"2023-10-09T10:47:31.258369+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"174'5850","reported_seq":6026,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:34.515553+0000","last_change":"2023-10-09T10:46:02.760963+0000","last_active":"2023-10-09T10:47:34.515553+0000","last_peered":"2023-10-09T10:47:34.515553+0000","last_clean":"2023-10-09T10:47:34.515553+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:47:34.515553+0000","last_undegraded":"2023-10-09T10:47:34.515553+0000","last_fullsized":"2023-10-09T10:47:34.515553+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"141'4416","last_scrub_stamp":"2023-10-09T10:46:02.760810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:46:02.760810+0000","objects_scrubbed":3378,"log_size":5850,"log_dups_size":0,"ondisk_log_size":5850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:35.863002+0000","scrub_duration":0.47769323899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127401984,"num_objects":1944,"num_object_clones":0,"num_object_copies":5832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1434,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":143,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262606+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:47:31.262606+0000","last_peered":"2023-10-09T10:47:31.262606+0000","last_clean":"2023-10-09T10:47:31.262606+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:47:31.262606+0000","last_undegraded":"2023-10-09T10:47:31.262606+0000","last_fullsized":"2023-10-09T10:47:31.262606+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":159,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.227216+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:47:31.227216+0000","last_peered":"2023-10-09T10:47:31.227216+0000","last_clean":"2023-10-09T10:47:31.227216+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:47:31.227216+0000","last_undegraded":"2023-10-09T10:47:31.227216+0000","last_fullsized":"2023-10-09T10:47:31.227216+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":150,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.221348+0000","last_change":"2023-10-09T10:46:32.245608+0000","last_active":"2023-10-09T10:47:31.221348+0000","last_peered":"2023-10-09T10:47:31.221348+0000","last_clean":"2023-10-09T10:47:31.221348+0000","last_became_active":"2023-10-09T10:46:32.245290+0000","last_became_peered":"2023-10-09T10:46:32.245290+0000","last_unstale":"2023-10-09T10:47:31.221348+0000","last_undegraded":"2023-10-09T10:47:31.221348+0000","last_fullsized":"2023-10-09T10:47:31.221348+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:48.832515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":508,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262776+0000","last_change":"2023-10-09T10:46:05.620057+0000","last_active":"2023-10-09T10:47:31.262776+0000","last_peered":"2023-10-09T10:47:31.262776+0000","last_clean":"2023-10-09T10:47:31.262776+0000","last_became_active":"2023-10-09T10:46:05.619703+0000","last_became_peered":"2023-10-09T10:46:05.619703+0000","last_unstale":"2023-10-09T10:47:31.262776+0000","last_undegraded":"2023-10-09T10:47:31.262776+0000","last_fullsized":"2023-10-09T10:47:31.262776+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"174'5798","reported_seq":5909,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:34.473352+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:47:34.473352+0000","last_peered":"2023-10-09T10:47:34.473352+0000","last_clean":"2023-10-09T10:47:34.473352+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:47:34.473352+0000","last_undegraded":"2023-10-09T10:47:34.473352+0000","last_fullsized":"2023-10-09T10:47:34.473352+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5798,"log_dups_size":0,"ondisk_log_size":5798,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127336448,"num_objects":1943,"num_object_clones":0,"num_object_copies":5829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4737,"num_write_kb":213760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":171,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.260327+0000","last_change":"2023-10-09T10:46:32.241041+0000","last_active":"2023-10-09T10:47:31.260327+0000","last_peered":"2023-10-09T10:47:31.260327+0000","last_clean":"2023-10-09T10:47:31.260327+0000","last_became_active":"2023-10-09T10:46:32.238683+0000","last_became_peered":"2023-10-09T10:46:32.238683+0000","last_unstale":"2023-10-09T10:47:31.260327+0000","last_undegraded":"2023-10-09T10:47:31.260327+0000","last_fullsized":"2023-10-09T10:47:31.260327+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:25.817713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":135,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.220984+0000","last_change":"2023-10-09T10:46:32.250104+0000","last_active":"2023-10-09T10:47:31.220984+0000","last_peered":"2023-10-09T10:47:31.220984+0000","last_clean":"2023-10-09T10:47:31.220984+0000","last_became_active":"2023-10-09T10:46:32.249805+0000","last_became_peered":"2023-10-09T10:46:32.249805+0000","last_unstale":"2023-10-09T10:47:31.220984+0000","last_undegraded":"2023-10-09T10:47:31.220984+0000","last_fullsized":"2023-10-09T10:47:31.220984+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:07.230672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":164,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.223026+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:47:31.223026+0000","last_peered":"2023-10-09T10:47:31.223026+0000","last_clean":"2023-10-09T10:47:31.223026+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:47:31.223026+0000","last_undegraded":"2023-10-09T10:47:31.223026+0000","last_fullsized":"2023-10-09T10:47:31.223026+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"174'5697","reported_seq":5795,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:36.014801+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:47:36.014801+0000","last_peered":"2023-10-09T10:47:36.014801+0000","last_clean":"2023-10-09T10:47:36.014801+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:47:36.014801+0000","last_undegraded":"2023-10-09T10:47:36.014801+0000","last_fullsized":"2023-10-09T10:47:36.014801+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5697,"log_dups_size":0,"ondisk_log_size":5697,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128516096,"num_objects":1961,"num_object_clones":0,"num_object_copies":5883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4689,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":153,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262368+0000","last_change":"2023-10-09T10:46:32.235346+0000","last_active":"2023-10-09T10:47:31.262368+0000","last_peered":"2023-10-09T10:47:31.262368+0000","last_clean":"2023-10-09T10:47:31.262368+0000","last_became_active":"2023-10-09T10:46:32.234906+0000","last_became_peered":"2023-10-09T10:46:32.234906+0000","last_unstale":"2023-10-09T10:47:31.262368+0000","last_undegraded":"2023-10-09T10:47:31.262368+0000","last_fullsized":"2023-10-09T10:47:31.262368+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.199251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":508,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262499+0000","last_change":"2023-10-09T10:46:05.617146+0000","last_active":"2023-10-09T10:47:31.262499+0000","last_peered":"2023-10-09T10:47:31.262499+0000","last_clean":"2023-10-09T10:47:31.262499+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:47:31.262499+0000","last_undegraded":"2023-10-09T10:47:31.262499+0000","last_fullsized":"2023-10-09T10:47:31.262499+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":101,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.258401+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:47:31.258401+0000","last_peered":"2023-10-09T10:47:31.258401+0000","last_clean":"2023-10-09T10:47:31.258401+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:47:31.258401+0000","last_undegraded":"2023-10-09T10:47:31.258401+0000","last_fullsized":"2023-10-09T10:47:31.258401+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":71,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.227640+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:47:31.227640+0000","last_peered":"2023-10-09T10:47:31.227640+0000","last_clean":"2023-10-09T10:47:31.227640+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:47:31.227640+0000","last_undegraded":"2023-10-09T10:47:31.227640+0000","last_fullsized":"2023-10-09T10:47:31.227640+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":508,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263337+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:47:31.263337+0000","last_peered":"2023-10-09T10:47:31.263337+0000","last_clean":"2023-10-09T10:47:31.263337+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:47:31.263337+0000","last_undegraded":"2023-10-09T10:47:31.263337+0000","last_fullsized":"2023-10-09T10:47:31.263337+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":165,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.221107+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:47:31.221107+0000","last_peered":"2023-10-09T10:47:31.221107+0000","last_clean":"2023-10-09T10:47:31.221107+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:47:31.221107+0000","last_undegraded":"2023-10-09T10:47:31.221107+0000","last_fullsized":"2023-10-09T10:47:31.221107+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":51,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.258284+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:47:31.258284+0000","last_peered":"2023-10-09T10:47:31.258284+0000","last_clean":"2023-10-09T10:47:31.258284+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:47:31.258284+0000","last_undegraded":"2023-10-09T10:47:31.258284+0000","last_fullsized":"2023-10-09T10:47:31.258284+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":51,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263244+0000","last_change":"2023-10-09T10:46:32.236202+0000","last_active":"2023-10-09T10:47:31.263244+0000","last_peered":"2023-10-09T10:47:31.263244+0000","last_clean":"2023-10-09T10:47:31.263244+0000","last_became_active":"2023-10-09T10:46:32.235901+0000","last_became_peered":"2023-10-09T10:46:32.235901+0000","last_unstale":"2023-10-09T10:47:31.263244+0000","last_undegraded":"2023-10-09T10:47:31.263244+0000","last_fullsized":"2023-10-09T10:47:31.263244+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:48.260403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"174'5879","reported_seq":6024,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:36.455129+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:47:36.455129+0000","last_peered":"2023-10-09T10:47:36.455129+0000","last_clean":"2023-10-09T10:47:36.455129+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:47:36.455129+0000","last_undegraded":"2023-10-09T10:47:36.455129+0000","last_fullsized":"2023-10-09T10:47:36.455129+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":5879,"log_dups_size":0,"ondisk_log_size":5879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124452864,"num_objects":1899,"num_object_clones":0,"num_object_copies":5697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2849,"num_write_kb":89664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":508,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263003+0000","last_change":"2023-10-09T10:46:05.620764+0000","last_active":"2023-10-09T10:47:31.263003+0000","last_peered":"2023-10-09T10:47:31.263003+0000","last_clean":"2023-10-09T10:47:31.263003+0000","last_became_active":"2023-10-09T10:46:05.619870+0000","last_became_peered":"2023-10-09T10:46:05.619870+0000","last_unstale":"2023-10-09T10:47:31.263003+0000","last_undegraded":"2023-10-09T10:47:31.263003+0000","last_fullsized":"2023-10-09T10:47:31.263003+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":162,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.225256+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:47:31.225256+0000","last_peered":"2023-10-09T10:47:31.225256+0000","last_clean":"2023-10-09T10:47:31.225256+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:47:31.225256+0000","last_undegraded":"2023-10-09T10:47:31.225256+0000","last_fullsized":"2023-10-09T10:47:31.225256+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":135,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.221394+0000","last_change":"2023-10-09T10:46:32.251450+0000","last_active":"2023-10-09T10:47:31.221394+0000","last_peered":"2023-10-09T10:47:31.221394+0000","last_clean":"2023-10-09T10:47:31.221394+0000","last_became_active":"2023-10-09T10:46:32.250933+0000","last_became_peered":"2023-10-09T10:46:32.250933+0000","last_unstale":"2023-10-09T10:47:31.221394+0000","last_undegraded":"2023-10-09T10:47:31.221394+0000","last_fullsized":"2023-10-09T10:47:31.221394+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:04.116178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":42,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263449+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:47:31.263449+0000","last_peered":"2023-10-09T10:47:31.263449+0000","last_clean":"2023-10-09T10:47:31.263449+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:47:31.263449+0000","last_undegraded":"2023-10-09T10:47:31.263449+0000","last_fullsized":"2023-10-09T10:47:31.263449+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"174'5721","reported_seq":12411,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:36.457444+0000","last_change":"2023-10-09T10:47:02.269289+0000","last_active":"2023-10-09T10:47:36.457444+0000","last_peered":"2023-10-09T10:47:36.457444+0000","last_clean":"2023-10-09T10:47:36.457444+0000","last_became_active":"2023-10-09T10:47:02.257854+0000","last_became_peered":"2023-10-09T10:47:02.257854+0000","last_unstale":"2023-10-09T10:47:36.457444+0000","last_undegraded":"2023-10-09T10:47:36.457444+0000","last_fullsized":"2023-10-09T10:47:36.457444+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5721,"log_dups_size":0,"ondisk_log_size":5721,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123830272,"num_objects":1889,"num_object_clones":0,"num_object_copies":5667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4713,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3287,"num_bytes_recovered":181927936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":508,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263392+0000","last_change":"2023-10-09T10:46:05.618443+0000","last_active":"2023-10-09T10:47:31.263392+0000","last_peered":"2023-10-09T10:47:31.263392+0000","last_clean":"2023-10-09T10:47:31.263392+0000","last_became_active":"2023-10-09T10:46:05.616588+0000","last_became_peered":"2023-10-09T10:46:05.616588+0000","last_unstale":"2023-10-09T10:47:31.263392+0000","last_undegraded":"2023-10-09T10:47:31.263392+0000","last_fullsized":"2023-10-09T10:47:31.263392+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":166,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.260757+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:47:31.260757+0000","last_peered":"2023-10-09T10:47:31.260757+0000","last_clean":"2023-10-09T10:47:31.260757+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:47:31.260757+0000","last_undegraded":"2023-10-09T10:47:31.260757+0000","last_fullsized":"2023-10-09T10:47:31.260757+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":111,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.221653+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:47:31.221653+0000","last_peered":"2023-10-09T10:47:31.221653+0000","last_clean":"2023-10-09T10:47:31.221653+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:47:31.221653+0000","last_undegraded":"2023-10-09T10:47:31.221653+0000","last_fullsized":"2023-10-09T10:47:31.221653+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":156,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.281386+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:47:32.281386+0000","last_peered":"2023-10-09T10:47:32.281386+0000","last_clean":"2023-10-09T10:47:32.281386+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:47:32.281386+0000","last_undegraded":"2023-10-09T10:47:32.281386+0000","last_fullsized":"2023-10-09T10:47:32.281386+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"174'5860","reported_seq":6793,"reported_epoch":174,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:36.016249+0000","last_change":"2023-10-09T10:47:29.622439+0000","last_active":"2023-10-09T10:47:36.016249+0000","last_peered":"2023-10-09T10:47:36.016249+0000","last_clean":"2023-10-09T10:46:31.130083+0000","last_became_active":"2023-10-09T10:46:33.493288+0000","last_became_peered":"2023-10-09T10:46:33.493288+0000","last_unstale":"2023-10-09T10:47:36.016249+0000","last_undegraded":"2023-10-09T10:46:33.283624+0000","last_fullsized":"2023-10-09T10:46:33.244043+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5860,"log_dups_size":0,"ondisk_log_size":5860,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120684544,"num_objects":1841,"num_object_clones":0,"num_object_copies":5523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":838,"num_objects_misplaced":1583,"num_objects_unfound":0,"num_objects_dirty":1841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4823,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":394,"num_bytes_recovered":25821184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":1583},{"shards":"3,4","objects":-580}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":507,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.265010+0000","last_change":"2023-10-09T10:46:05.617589+0000","last_active":"2023-10-09T10:47:31.265010+0000","last_peered":"2023-10-09T10:47:31.265010+0000","last_clean":"2023-10-09T10:47:31.265010+0000","last_became_active":"2023-10-09T10:46:05.616400+0000","last_became_peered":"2023-10-09T10:46:05.616400+0000","last_unstale":"2023-10-09T10:47:31.265010+0000","last_undegraded":"2023-10-09T10:47:31.265010+0000","last_fullsized":"2023-10-09T10:47:31.265010+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":31,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.260495+0000","last_change":"2023-10-09T10:46:32.245106+0000","last_active":"2023-10-09T10:47:31.260495+0000","last_peered":"2023-10-09T10:47:31.260495+0000","last_clean":"2023-10-09T10:47:31.260495+0000","last_became_active":"2023-10-09T10:46:32.244826+0000","last_became_peered":"2023-10-09T10:46:32.244826+0000","last_unstale":"2023-10-09T10:47:31.260495+0000","last_undegraded":"2023-10-09T10:47:31.260495+0000","last_fullsized":"2023-10-09T10:47:31.260495+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:30.322834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":46,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263156+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:47:31.263156+0000","last_peered":"2023-10-09T10:47:31.263156+0000","last_clean":"2023-10-09T10:47:31.263156+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:47:31.263156+0000","last_undegraded":"2023-10-09T10:47:31.263156+0000","last_fullsized":"2023-10-09T10:47:31.263156+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":121,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.226683+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:47:31.226683+0000","last_peered":"2023-10-09T10:47:31.226683+0000","last_clean":"2023-10-09T10:47:31.226683+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:47:31.226683+0000","last_undegraded":"2023-10-09T10:47:31.226683+0000","last_fullsized":"2023-10-09T10:47:31.226683+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"174'5759","reported_seq":12562,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:35.825051+0000","last_change":"2023-10-09T10:47:31.271196+0000","last_active":"2023-10-09T10:47:35.825051+0000","last_peered":"2023-10-09T10:47:35.825051+0000","last_clean":"2023-10-09T10:47:35.825051+0000","last_became_active":"2023-10-09T10:47:31.229504+0000","last_became_peered":"2023-10-09T10:47:31.229504+0000","last_unstale":"2023-10-09T10:47:35.825051+0000","last_undegraded":"2023-10-09T10:47:35.825051+0000","last_fullsized":"2023-10-09T10:47:35.825051+0000","mapping_epoch":172,"log_start":"103'115","ondisk_log_start":"103'115","created":98,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":5644,"log_dups_size":0,"ondisk_log_size":5644,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:26.533476+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122355712,"num_objects":1867,"num_object_clones":0,"num_object_copies":5601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3114,"num_write_kb":107264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3305,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":42,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263497+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:47:31.263497+0000","last_peered":"2023-10-09T10:47:31.263497+0000","last_clean":"2023-10-09T10:47:31.263497+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:47:31.263497+0000","last_undegraded":"2023-10-09T10:47:31.263497+0000","last_fullsized":"2023-10-09T10:47:31.263497+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":105,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.221440+0000","last_change":"2023-10-09T10:46:32.251242+0000","last_active":"2023-10-09T10:47:31.221440+0000","last_peered":"2023-10-09T10:47:31.221440+0000","last_clean":"2023-10-09T10:47:31.221440+0000","last_became_active":"2023-10-09T10:46:32.250534+0000","last_became_peered":"2023-10-09T10:46:32.250534+0000","last_unstale":"2023-10-09T10:47:31.221440+0000","last_undegraded":"2023-10-09T10:47:31.221440+0000","last_fullsized":"2023-10-09T10:47:31.221440+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:33.726041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"174'5852","reported_seq":9830,"reported_epoch":174,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:37.600012+0000","last_change":"2023-10-09T10:46:34.514479+0000","last_active":"2023-10-09T10:47:37.600012+0000","last_peered":"2023-10-09T10:47:37.600012+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:46:34.409667+0000","last_became_peered":"2023-10-09T10:46:34.409667+0000","last_unstale":"2023-10-09T10:47:37.600012+0000","last_undegraded":"2023-10-09T10:46:34.376635+0000","last_fullsized":"2023-10-09T10:46:33.237998+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5852,"log_dups_size":0,"ondisk_log_size":5852,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122159104,"num_objects":1864,"num_object_clones":0,"num_object_copies":5592,"num_objects_missing_on_primary":1406,"num_objects_missing":1406,"num_objects_degraded":1916,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4810,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1932,"num_bytes_recovered":126615552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,7","objects":458},{"shards":"2,3,7","objects":1406}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":141,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262812+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:47:31.262812+0000","last_peered":"2023-10-09T10:47:31.262812+0000","last_clean":"2023-10-09T10:47:31.262812+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:47:31.262812+0000","last_undegraded":"2023-10-09T10:47:31.262812+0000","last_fullsized":"2023-10-09T10:47:31.262812+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":155,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.227355+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:47:31.227355+0000","last_peered":"2023-10-09T10:47:31.227355+0000","last_clean":"2023-10-09T10:47:31.227355+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:47:31.227355+0000","last_undegraded":"2023-10-09T10:47:31.227355+0000","last_fullsized":"2023-10-09T10:47:31.227355+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"174'5783","reported_seq":5889,"reported_epoch":174,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:34.414455+0000","last_change":"2023-10-09T10:46:33.546991+0000","last_active":"2023-10-09T10:47:34.414455+0000","last_peered":"2023-10-09T10:47:34.414455+0000","last_clean":"2023-10-09T10:46:31.009484+0000","last_became_active":"2023-10-09T10:46:33.535665+0000","last_became_peered":"2023-10-09T10:46:33.535665+0000","last_unstale":"2023-10-09T10:47:34.414455+0000","last_undegraded":"2023-10-09T10:46:33.277168+0000","last_fullsized":"2023-10-09T10:46:33.235038+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5783,"log_dups_size":0,"ondisk_log_size":5783,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118521856,"num_objects":1809,"num_object_clones":0,"num_object_copies":5427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4719,"num_write_kb":208864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1821},{"shards":"3,7","objects":-949}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":124,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.263112+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:47:31.263112+0000","last_peered":"2023-10-09T10:47:31.263112+0000","last_clean":"2023-10-09T10:47:31.263112+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:47:31.263112+0000","last_undegraded":"2023-10-09T10:47:31.263112+0000","last_fullsized":"2023-10-09T10:47:31.263112+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":22,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.228175+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:47:31.228175+0000","last_peered":"2023-10-09T10:47:31.228175+0000","last_clean":"2023-10-09T10:47:31.228175+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:47:31.228175+0000","last_undegraded":"2023-10-09T10:47:31.228175+0000","last_fullsized":"2023-10-09T10:47:31.228175+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":158,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.277987+0000","last_change":"2023-10-09T10:46:32.244607+0000","last_active":"2023-10-09T10:47:32.277987+0000","last_peered":"2023-10-09T10:47:32.277987+0000","last_clean":"2023-10-09T10:47:32.277987+0000","last_became_active":"2023-10-09T10:46:32.242628+0000","last_became_peered":"2023-10-09T10:46:32.242628+0000","last_unstale":"2023-10-09T10:47:32.277987+0000","last_undegraded":"2023-10-09T10:47:32.277987+0000","last_fullsized":"2023-10-09T10:47:32.277987+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:10.738819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":188,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.260134+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:47:31.260134+0000","last_peered":"2023-10-09T10:47:31.260134+0000","last_clean":"2023-10-09T10:47:31.260134+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:47:31.260134+0000","last_undegraded":"2023-10-09T10:47:31.260134+0000","last_fullsized":"2023-10-09T10:47:31.260134+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":63,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.262957+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:47:31.262957+0000","last_peered":"2023-10-09T10:47:31.262957+0000","last_clean":"2023-10-09T10:47:31.262957+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:47:31.262957+0000","last_undegraded":"2023-10-09T10:47:31.262957+0000","last_fullsized":"2023-10-09T10:47:31.262957+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":138,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T10:47:31.226470+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:47:31.226470+0000","last_peered":"2023-10-09T10:47:31.226470+0000","last_clean":"2023-10-09T10:47:31.226470+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:47:31.226470+0000","last_undegraded":"2023-10-09T10:47:31.226470+0000","last_fullsized":"2023-10-09T10:47:31.226470+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":3924066328,"num_objects":59878,"num_object_clones":0,"num_object_copies":179634,"num_objects_missing_on_primary":1406,"num_objects_missing":1406,"num_objects_degraded":31152,"num_objects_misplaced":8371,"num_objects_unfound":0,"num_objects_dirty":59878,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":144330,"num_write_kb":6258529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21172,"num_bytes_recovered":1294860288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13672001536,"data_stored":13671989320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":158453,"ondisk_log_size":158453,"up":78,"acting":67,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1186,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332252,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1990672,"kb_used_data":1681096,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91754480,"statfs":{"total":95995035648,"available":93956587520,"internally_reserved":0,"allocated":1721442304,"data_stored":1720780080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.70099999999999996,"15min":0.61899999999999999},"min":{"1min":0.23100000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.0919999999999996,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":0.308},{"interface":"front","average":{"1min":1.3640000000000001,"5min":0.79100000000000004,"15min":0.69099999999999995},"min":{"1min":0.309,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":10.207000000000001,"5min":10.207000000000001,"15min":10.207000000000001},"last":0.379}]},{"osd":1,"last update":"Mon Oct 9 10:46:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":0.99399999999999999,"15min":0.69199999999999995},"min":{"1min":0.42199999999999999,"5min":0.318,"15min":0.318},"max":{"1min":8.7360000000000007,"5min":8.7360000000000007,"15min":8.7360000000000007},"last":0.44},{"interface":"front","average":{"1min":1.2230000000000001,"5min":0.80500000000000005,"15min":0.623},"min":{"1min":0.27800000000000002,"5min":0.245,"15min":0.245},"max":{"1min":3.6080000000000001,"5min":6.4939999999999998,"15min":6.4939999999999998},"last":0.83499999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.1499999999999999,"15min":0.72399999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.4770000000000003,"5min":6.4770000000000003,"15min":6.4770000000000003},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.272,"5min":0.85699999999999998,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.251,"15min":0.251},"max":{"1min":3.1480000000000001,"5min":3.7120000000000002,"15min":3.7120000000000002},"last":0.35799999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:46:36 2023","interfaces":[{"interface":"back","average":{"1min":1.9079999999999999,"5min":1.073,"15min":0.748},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.8129999999999997,"5min":8.1639999999999997,"15min":8.1639999999999997},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.3720000000000001,"5min":1.022,"15min":0.73999999999999999},"min":{"1min":0.377,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.3609999999999998,"5min":12.451000000000001,"15min":12.451000000000001},"last":0.42699999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.73299999999999998,"15min":0.56200000000000006},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.1420000000000003,"5min":7.4509999999999996,"15min":7.4509999999999996},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.65300000000000002,"15min":0.53600000000000003},"min":{"1min":0.29799999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.2749999999999999,"5min":3.1699999999999999,"15min":3.1699999999999999},"last":0.45800000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.56399999999999995,"15min":0.52700000000000002},"min":{"1min":0.34300000000000003,"5min":0.247,"15min":0.247},"max":{"1min":2.2349999999999999,"5min":2.2349999999999999,"15min":2.2349999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.81299999999999994,"15min":0.63},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":5.0860000000000003,"5min":10.33,"15min":10.33},"last":0.52100000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.83399999999999996,"15min":0.64800000000000002},"min":{"1min":0.311,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.109,"5min":10.164999999999999,"15min":10.164999999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.66000000000000003,"15min":0.58299999999999996},"min":{"1min":0.33300000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.306,"5min":2.8079999999999998,"15min":2.8079999999999998},"last":0.53300000000000003}]}]},{"osd":6,"up_from":44,"seq":188978561183,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1614732,"kb_used_data":1309124,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92130420,"statfs":{"total":95995035648,"available":94341550080,"internally_reserved":0,"allocated":1340542976,"data_stored":1339885290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.772,"5min":0.89500000000000002,"15min":0.70499999999999996},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.3220000000000001,"5min":7.3220000000000001,"15min":7.3220000000000001},"last":1.044},{"interface":"front","average":{"1min":2.02,"5min":0.92000000000000004,"15min":0.71199999999999997},"min":{"1min":0.26700000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.5309999999999997,"5min":7.5309999999999997,"15min":7.5309999999999997},"last":1.9850000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:46:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8180000000000001,"5min":1.0680000000000001,"15min":0.75},"min":{"1min":0.29899999999999999,"5min":0.245,"15min":0.245},"max":{"1min":7.327,"5min":16.747,"15min":16.747},"last":1.238},{"interface":"front","average":{"1min":1.883,"5min":1.101,"15min":0.73399999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.4260000000000002,"5min":16.734000000000002,"15min":16.734000000000002},"last":1.204}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":0.95799999999999996,"15min":0.67500000000000004},"min":{"1min":0.28699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5069999999999997,"5min":7.5069999999999997,"15min":7.5069999999999997},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.675,"5min":1.143,"15min":0.755},"min":{"1min":0.28599999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.3899999999999997,"5min":12.571,"15min":12.571},"last":5.5739999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":0.89600000000000002,"15min":0.65700000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.8799999999999999,"5min":4.7030000000000003,"15min":4.7030000000000003},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.4399999999999999,"5min":1.0700000000000001,"15min":0.72099999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.4770000000000003,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":6.6020000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":0.66600000000000004,"15min":0.51500000000000001},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":4.0739999999999998,"5min":4.0739999999999998,"15min":4.0739999999999998},"last":7.4340000000000002},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.67300000000000004,"15min":0.54600000000000004},"min":{"1min":0.34899999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6600000000000001,"5min":3.6600000000000001,"15min":3.6600000000000001},"last":0.66500000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":0.76000000000000001,"15min":0.56999999999999995},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.8949999999999996,"5min":4.8949999999999996,"15min":4.8949999999999996},"last":6.1470000000000002},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.76900000000000002,"15min":0.59999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":4.3639999999999999,"5min":4.3639999999999999,"15min":4.3639999999999999},"last":0.96699999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:47:25 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69899999999999995,"15min":0.57999999999999996},"min":{"1min":0.35799999999999998,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":2.5049999999999999,"5min":3.472,"15min":3.472},"last":5.6289999999999996},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.84299999999999997,"15min":0.65700000000000003},"min":{"1min":0.373,"5min":0.36899999999999999,"15min":0.34000000000000002},"max":{"1min":4.0800000000000001,"5min":4.0800000000000001,"15min":4.0800000000000001},"last":0.46899999999999997}]}]},{"osd":1,"up_from":85,"seq":365072220222,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1667960,"kb_used_data":1542064,"kb_used_omap":9,"kb_used_meta":125878,"kb_avail":92077192,"statfs":{"total":95995035648,"available":94287044608,"internally_reserved":0,"allocated":1579073536,"data_stored":1578203532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":128900073},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.3160000000000001,"15min":0.68100000000000005},"min":{"1min":0.255,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":8.3759999999999994,"5min":14.1,"15min":14.1},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.5499999999999998,"5min":1.266,"15min":0.68799999999999994},"min":{"1min":0.27300000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":13.462999999999999,"5min":14.465,"15min":14.465},"last":4.6299999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:47:13 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.1200000000000001,"15min":0.61799999999999999},"min":{"1min":0.21199999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.4829999999999997,"5min":8.6859999999999999,"15min":8.6859999999999999},"last":4.8220000000000001},{"interface":"front","average":{"1min":1.919,"5min":1.4690000000000001,"15min":0.746},"min":{"1min":0.29099999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.2829999999999995,"5min":14.234,"15min":14.234},"last":0.35099999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.2,"15min":0.71399999999999997},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.3539999999999992,"5min":14.404999999999999,"15min":14.404999999999999},"last":4.7400000000000002},{"interface":"front","average":{"1min":2.077,"5min":1.1080000000000001,"15min":0.59899999999999998},"min":{"1min":0.255,"5min":0.251,"15min":0.251},"max":{"1min":9.6820000000000004,"5min":9.6820000000000004,"15min":9.6820000000000004},"last":0.65500000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.0880000000000001,"15min":0.75800000000000001},"min":{"1min":0.29099999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.1420000000000003,"5min":8.0129999999999999,"15min":8.0129999999999999},"last":4.7709999999999999},{"interface":"front","average":{"1min":1.4850000000000001,"5min":1.0249999999999999,"15min":0.755},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":8.327,"5min":8.327,"15min":8.327},"last":0.38100000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.571,"5min":0.95699999999999996,"15min":0.73699999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.7720000000000002,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":0.871},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.1359999999999999,"15min":0.81000000000000005},"min":{"1min":0.32700000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.1799999999999997,"5min":7.923,"15min":7.923},"last":4.7990000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":0.97199999999999998,"15min":0.745},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.7379999999999995,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":0.746},{"interface":"front","average":{"1min":1.617,"5min":1.175,"15min":0.82299999999999995},"min":{"1min":0.39500000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.2069999999999999,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":4.8440000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6699999999999999,"5min":1.155,"15min":0.81799999999999995},"min":{"1min":0.41599999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":5.2309999999999999,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":4.8689999999999998},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.022,"15min":0.77100000000000002},"min":{"1min":0.42499999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":4.6399999999999997,"5min":8.2469999999999999,"15min":8.2469999999999999},"last":4.7140000000000004}]}]},{"osd":0,"up_from":80,"seq":343597383756,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":724692,"kb_used_data":634828,"kb_used_omap":28,"kb_used_meta":89827,"kb_avail":93020460,"statfs":{"total":95995035648,"available":95252951040,"internally_reserved":0,"allocated":650063872,"data_stored":649165052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":91983696},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,1,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:46:43 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.1679999999999999,"15min":0.63400000000000001},"min":{"1min":0.217,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.0449999999999999,"5min":25.475000000000001,"15min":25.475000000000001},"last":2.7869999999999999},{"interface":"front","average":{"1min":2.0099999999999998,"5min":1.1919999999999999,"15min":0.64000000000000001},"min":{"1min":0.23499999999999999,"5min":0.191,"15min":0.191},"max":{"1min":6.2290000000000001,"5min":20.832999999999998,"15min":20.832999999999998},"last":1.123}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":1.2350000000000001,"15min":0.64400000000000002},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.907,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":2.8250000000000002},{"interface":"front","average":{"1min":1.754,"5min":1.5349999999999999,"15min":0.73999999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.5619999999999994,"5min":14.391999999999999,"15min":14.391999999999999},"last":4.3780000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9689999999999999,"5min":1.5309999999999999,"15min":0.94399999999999995},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":9.3889999999999993,"5min":14.926,"15min":14.926},"last":2.6589999999999998},{"interface":"front","average":{"1min":2.02,"5min":1.2569999999999999,"15min":0.78000000000000003},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":11.868,"5min":11.868,"15min":11.868},"last":2.863}]},{"osd":4,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.2250000000000001,"15min":0.82199999999999995},"min":{"1min":0.38400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.8150000000000004,"5min":6.8150000000000004,"15min":6.8150000000000004},"last":4.4429999999999996},{"interface":"front","average":{"1min":2.391,"5min":1.405,"15min":0.88100000000000001},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.676,"5min":10.676,"15min":10.676},"last":2.0579999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":1.4359999999999999,"15min":0.89600000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":10.734,"5min":10.734,"15min":10.734},"last":2.1309999999999998},{"interface":"front","average":{"1min":1.716,"5min":1.1659999999999999,"15min":0.82099999999999995},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.54,"5min":8.9190000000000005,"15min":8.9190000000000005},"last":2.8889999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.391,"15min":0.88400000000000001},"min":{"1min":0.36599999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.706,"5min":10.706,"15min":10.706},"last":2.1070000000000002},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.0780000000000001,"15min":0.79300000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.0499999999999998,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":2.9140000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.474,"5min":1.151,"15min":0.81899999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.5380000000000003,"5min":8.875,"15min":8.875},"last":4.5380000000000003},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.425,"15min":0.92700000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":14.394,"5min":14.394,"15min":14.394},"last":4.4180000000000001}]}]},{"osd":2,"up_from":0,"seq":386547056696,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893170,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2770624,"kb_used_data":2590584,"kb_used_omap":7,"kb_used_meta":180024,"kb_avail":90974528,"statfs":{"total":95995035648,"available":93157916672,"internally_reserved":0,"allocated":2652758016,"data_stored":2651877631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":184345192},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":2.734,"5min":0.995,"15min":0.54500000000000004},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":11.557,"5min":11.557,"15min":11.557},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.359,"5min":0.93700000000000006,"15min":0.58299999999999996},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":4.1369999999999996,"5min":7.0510000000000002,"15min":7.0510000000000002},"last":2.8610000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":1.4179999999999999,"15min":0.69199999999999995},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":13.967000000000001,"5min":14.707000000000001,"15min":14.707000000000001},"last":0.31},{"interface":"front","average":{"1min":1.911,"5min":0.93899999999999995,"15min":0.56200000000000006},"min":{"1min":0.218,"5min":0.193,"15min":0.193},"max":{"1min":11.629,"5min":11.629,"15min":11.629},"last":0.59699999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1299999999999999,"5min":1.3280000000000001,"15min":0.68700000000000006},"min":{"1min":0.30099999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":13.853,"5min":13.853,"15min":13.853},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.514,"5min":1.0800000000000001,"15min":0.624},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.3019999999999996,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":2.8199999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":1.0229999999999999,"15min":0.72199999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":11.532,"5min":11.532,"15min":11.532},"last":0.92600000000000005},{"interface":"front","average":{"1min":1.335,"5min":0.96899999999999997,"15min":0.72799999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.6070000000000002,"5min":5.6070000000000002,"15min":5.6070000000000002},"last":2.891}]},{"osd":5,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":4.3609999999999998,"5min":1.5680000000000001,"15min":0.91400000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":51.509999999999998,"5min":51.509999999999998,"15min":51.509999999999998},"last":0.36099999999999999},{"interface":"front","average":{"1min":5.141,"5min":1.99,"15min":1.038},"min":{"1min":0.32300000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":55.338999999999999,"5min":55.338999999999999,"15min":55.338999999999999},"last":0.47099999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":0.99199999999999999,"15min":0.72199999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.8490000000000002,"5min":8.4789999999999992,"15min":8.4789999999999992},"last":0.40699999999999997},{"interface":"front","average":{"1min":2.0150000000000001,"5min":1.421,"15min":0.873},"min":{"1min":0.378,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":11.470000000000001,"5min":13.891999999999999,"15min":13.891999999999999},"last":0.499}]},{"osd":7,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":4.9960000000000004,"5min":1.9970000000000001,"15min":1.0600000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":51.744,"5min":51.744,"15min":51.744},"last":2.8410000000000002},{"interface":"front","average":{"1min":4.6749999999999998,"5min":1.601,"15min":0.91600000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":53.399000000000001,"5min":53.399000000000001,"15min":53.399000000000001},"last":0.53400000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019046,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1502988,"kb_used_data":1205572,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92242164,"statfs":{"total":95995035648,"available":94455975936,"internally_reserved":0,"allocated":1234505728,"data_stored":1233847309,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.0129999999999999,"15min":0.70899999999999996},"min":{"1min":0.24099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.5419999999999998,"5min":10.387,"15min":10.387},"last":4.5419999999999998},{"interface":"front","average":{"1min":1.581,"5min":1.1819999999999999,"15min":0.78400000000000003},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.5999999999999996,"5min":10.311999999999999,"15min":10.311999999999999},"last":4.0810000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:46:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":0.98599999999999999,"15min":0.69699999999999995},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.4189999999999996,"5min":6.907,"15min":6.907},"last":4.5119999999999996},{"interface":"front","average":{"1min":1.627,"5min":0.99099999999999999,"15min":0.745},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.1790000000000003,"5min":6.1790000000000003,"15min":6.1790000000000003},"last":3.7149999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:47:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2170000000000001,"5min":1.1000000000000001,"15min":0.747},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.2149999999999999,"5min":7.5449999999999999,"15min":7.5449999999999999},"last":4.1040000000000001},{"interface":"front","average":{"1min":1.579,"5min":1.1639999999999999,"15min":0.76300000000000001},"min":{"1min":0.36699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.5110000000000001,"5min":6.2939999999999996,"15min":6.2939999999999996},"last":7.4139999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:46:38 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.107,"15min":0.84299999999999997},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.0730000000000004,"5min":5.0730000000000004,"15min":5.0730000000000004},"last":3.6789999999999998},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.004,"15min":0.72599999999999998},"min":{"1min":0.35999999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":8.9809999999999999,"5min":8.9809999999999999,"15min":8.9809999999999999},"last":7.625}]},{"osd":5,"last update":"Mon Oct 9 10:46:41 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.73099999999999998,"15min":0.53500000000000003},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":3.9319999999999999,"5min":5.5410000000000004,"15min":5.5410000000000004},"last":3.7429999999999999},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.71099999999999997,"15min":0.52900000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":3.8980000000000001,"5min":3.8980000000000001,"15min":3.8980000000000001},"last":4.7060000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.74099999999999999,"15min":0.55700000000000005},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":7.1580000000000004,"5min":7.1580000000000004,"15min":7.1580000000000004},"last":4.4569999999999999},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.71599999999999997,"15min":0.60899999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":3.6150000000000002,"5min":3.6150000000000002,"15min":3.6150000000000002},"last":3.7730000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.80800000000000005,"15min":0.64600000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.1050000000000004,"5min":4.1050000000000004,"15min":4.1050000000000004},"last":5.1479999999999997},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.748,"15min":0.60799999999999998},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0919999999999996,"5min":4.0919999999999996,"15min":4.0919999999999996},"last":5.0800000000000001}]}]},{"osd":5,"up_from":37,"seq":158913790115,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1859260,"kb_used_data":1561716,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91885892,"statfs":{"total":95995035648,"available":94091153408,"internally_reserved":0,"allocated":1599197184,"data_stored":1598550594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.72199999999999998,"15min":0.63900000000000001},"min":{"1min":0.25800000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.722,"5min":4.8380000000000001,"15min":4.8380000000000001},"last":1.3740000000000001},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.78000000000000003,"15min":0.70599999999999996},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.7549999999999999,"5min":4.8650000000000002,"15min":4.8650000000000002},"last":1.4039999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":0.85599999999999998,"15min":0.68999999999999995},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.7789999999999999,"5min":3.7789999999999999,"15min":3.7789999999999999},"last":2.3519999999999999},{"interface":"front","average":{"1min":1.1200000000000001,"5min":0.93100000000000005,"15min":0.70499999999999996},"min":{"1min":0.26200000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.77,"5min":7.165,"15min":7.165},"last":0.27600000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:47:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.02,"15min":0.72099999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.4569999999999999,"5min":7.9000000000000004,"15min":7.9000000000000004},"last":2.3809999999999998},{"interface":"front","average":{"1min":1.7569999999999999,"5min":0.90400000000000003,"15min":0.68400000000000005},"min":{"1min":0.38,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":10.352,"5min":10.352,"15min":10.352},"last":1.4850000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:46:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5960000000000001,"5min":1.266,"15min":0.90400000000000003},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":9.984,"5min":9.984,"15min":9.984},"last":7.2839999999999998},{"interface":"front","average":{"1min":2.6509999999999998,"5min":1.2030000000000001,"15min":0.80900000000000005},"min":{"1min":0.34999999999999998,"5min":0.317,"15min":0.317},"max":{"1min":11.657,"5min":11.657,"15min":11.657},"last":7.4059999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.53200000000000003,"15min":0.46700000000000003},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":3.7919999999999998,"5min":3.7919999999999998,"15min":3.7919999999999998},"last":3.4780000000000002},{"interface":"front","average":{"1min":1.048,"5min":0.68500000000000005,"15min":0.52900000000000003},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.9569999999999999,"5min":4.9569999999999999,"15min":4.9569999999999999},"last":7.3220000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.66300000000000003,"15min":0.53500000000000003},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":2.8900000000000001,"5min":2.9529999999999998,"15min":2.9529999999999998},"last":2.3149999999999999},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.63300000000000001,"15min":0.53800000000000003},"min":{"1min":0.34100000000000003,"5min":0.33300000000000002,"15min":0.30299999999999999},"max":{"1min":3.8159999999999998,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":0.60299999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:47:22 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.67100000000000004,"15min":0.58799999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.4729999999999999,"5min":4.0510000000000002,"15min":4.0510000000000002},"last":2.4209999999999998},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.70899999999999996,"15min":0.60099999999999998},"min":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":4.1289999999999996,"5min":4.1289999999999996,"15min":4.1289999999999996},"last":0.54700000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":644222976,"data_stored":644218904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1561726976,"data_stored":1561722904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2967142400,"data_stored":2967142400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2650341376,"data_stored":2650341376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1221328896,"data_stored":1221328896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1590038528,"data_stored":1590034456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1327366144,"data_stored":1327366144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1709834240,"data_stored":1709834240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:47:40.929 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:40.982 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:41.250 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:41.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:41.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:41.287 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:47:41.362 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:41.478 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:41.542 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:47:41.557 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:41.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:41.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:41.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:47:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:47:41] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.43.0" 2023-10-09T10:47:41.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:41.734 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:41.778 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:41.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:41.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:41.839 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:47:41.889 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:41.958 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:41.965 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:47:41.980 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:42.010 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:42.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:42.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:42.113 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:42.173 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:42.308 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:42.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:42.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:42.311 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:42 smithi125 ceph-mon[195507]: from='client.34977 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:47:42.311 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:42 smithi125 ceph-mon[195507]: pgmap v370: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 31152/171129 objects degraded (18.204%); 7550/171129 objects misplaced (4.412%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T10:47:42.412 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:42.471 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:42.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:42 smithi125 ceph-mon[200197]: from='client.34977 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:47:42.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:42 smithi125 ceph-mon[200197]: pgmap v370: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 31152/171129 objects degraded (18.204%); 7550/171129 objects misplaced (4.412%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T10:47:42.699 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:42.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:42.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:42.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:42 smithi155 ceph-mon[184854]: from='client.34977 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:47:42.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:42 smithi155 ceph-mon[184854]: pgmap v370: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 31152/171129 objects degraded (18.204%); 7550/171129 objects misplaced (4.412%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T10:47:42.726 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:42.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:42.862 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:43.526 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:43.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:43.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:43.544 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:43 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 31152/171129 objects degraded (18.204%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:43.545 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:43 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 31152/171129 objects degraded (18.204%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:43.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:43 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 31152/171129 objects degraded (18.204%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:43.633 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:43.737 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:43.875 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:43.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:43.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:43.929 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:47:43.980 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:47:43.980 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:47:43.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:44.032 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:44.111 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:44.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:44.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:44.216 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:44.277 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:44.290 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:47:44.313 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:47:44.332 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:44.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:44.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:44.365 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:47:44.380 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:47:44.386 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:44.405 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:47:44.433 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:47:44.444 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:44.464 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:47:44.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:44 smithi155 ceph-mon[184854]: pgmap v371: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 31152/169137 objects degraded (18.418%); 7449/169137 objects misplaced (4.404%); 7.4 MiB/s, 117 objects/s recovering 2023-10-09T10:47:44.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:44 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1341485617' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:47:44.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:44 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2914937780' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:44.494 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:47:44.528 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:47:44.535 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:47:44.535 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:47:44.536 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:47:44.536 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:47:44.536 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:47:44.536 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:47:44.536 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:47:44.536 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:47:44.554 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:44.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:44 smithi125 ceph-mon[195507]: pgmap v371: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 31152/169137 objects degraded (18.418%); 7449/169137 objects misplaced (4.404%); 7.4 MiB/s, 117 objects/s recovering 2023-10-09T10:47:44.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:44 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1341485617' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:47:44.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:44 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2914937780' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:44.580 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:44 smithi125 ceph-mon[200197]: pgmap v371: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 31152/169137 objects degraded (18.418%); 7449/169137 objects misplaced (4.404%); 7.4 MiB/s, 117 objects/s recovering 2023-10-09T10:47:44.580 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:44 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1341485617' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:47:44.581 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:44 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2914937780' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:44.603 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:44.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:44.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:44.702 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:47:44.717 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:44.780 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:44.821 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:44.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:44.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:44.924 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:47:44.928 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:45.026 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:45.111 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:45.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:45.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:45.215 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:45.295 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:45.336 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:45.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:45.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:45.371 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:45 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:45.371 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:45 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1341485617' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:47:45.371 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:45 smithi155 ceph-mon[184854]: osdmap e176: 8 total, 8 up, 7 in 2023-10-09T10:47:45.371 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:45 smithi155 ceph-mon[184854]: failed to encode map e176 with expected crc 2023-10-09T10:47:45.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:45 smithi155 ceph-mon[184854]: failed to encode map e176 with expected crc 2023-10-09T10:47:45.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:47:45.442 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:45.485 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:45.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:45 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:45.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:45 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1341485617' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:47:45.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:45 smithi125 ceph-mon[195507]: osdmap e176: 8 total, 8 up, 7 in 2023-10-09T10:47:45.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:45 smithi125 ceph-mon[195507]: failed to encode map e176 with expected crc 2023-10-09T10:47:45.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:45 smithi125 ceph-mon[195507]: failed to encode map e176 with expected crc 2023-10-09T10:47:45.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:47:45.644 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:45 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:45.644 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:45 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1341485617' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:47:45.644 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:45 smithi125 ceph-mon[200197]: osdmap e176: 8 total, 8 up, 7 in 2023-10-09T10:47:45.644 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:45 smithi125 ceph-mon[200197]: failed to encode map e176 with expected crc 2023-10-09T10:47:45.644 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:45 smithi125 ceph-mon[200197]: failed to encode map e176 with expected crc 2023-10-09T10:47:45.644 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:47:45.700 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:45.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:45.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:45.804 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:45.865 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:45.898 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:45.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:45.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:46.004 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:46.060 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:46.101 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:46.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:46.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:46.209 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:46.220 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:47:46.221 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:47:46.221 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:47:46.221 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:47:46.221 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:47:46.307 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:46.370 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:47:46.373 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:46.379 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:47:46.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:46.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:46.484 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:46.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:46 smithi155 ceph-mon[184854]: pgmap v373: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 31152/164535 objects degraded (18.933%); 6879/164535 objects misplaced (4.181%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T10:47:46.537 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:46.566 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:46.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:46.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:46.599 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:47:46.670 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:46.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:46 smithi125 ceph-mon[195507]: pgmap v373: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 31152/164535 objects degraded (18.933%); 6879/164535 objects misplaced (4.181%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T10:47:46.703 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:46 smithi125 ceph-mon[200197]: pgmap v373: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 31152/164535 objects degraded (18.933%); 6879/164535 objects misplaced (4.181%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T10:47:46.705 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:46.738 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:46.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:46.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:46.809 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":373,"stamp":"2023-10-09T10:47:45.119082+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3593246648,"num_objects":54845,"num_object_clones":0,"num_object_copies":164535,"num_objects_missing_on_primary":1406,"num_objects_missing":1406,"num_objects_degraded":31152,"num_objects_misplaced":6879,"num_objects_unfound":0,"num_objects_dirty":54845,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":149687,"num_write_kb":6260974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22257,"num_bytes_recovered":1366166274,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":164328,"ondisk_log_size":164328,"up":357,"acting":346,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11461188,"kb_used_data":9825036,"kb_used_omap":44,"kb_used_meta":1635667,"kb_avail":644754876,"statfs":{"total":671965249536,"available":660228993024,"internally_reserved":0,"allocated":10060836864,"data_stored":10055536150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1674923425},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,2,0,3,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-345374720,"num_objects":-5270,"num_object_clones":0,"num_object_copies":-15810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1492,"num_objects_unfound":0,"num_objects_dirty":-5270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5270,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1061,"num_bytes_recovered":69533696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.004722"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":168,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.255227+0000","last_change":"2023-10-09T10:46:32.250979+0000","last_active":"2023-10-09T10:47:32.255227+0000","last_peered":"2023-10-09T10:47:32.255227+0000","last_clean":"2023-10-09T10:47:32.255227+0000","last_became_active":"2023-10-09T10:46:32.250625+0000","last_became_peered":"2023-10-09T10:46:32.250625+0000","last_unstale":"2023-10-09T10:47:32.255227+0000","last_undegraded":"2023-10-09T10:47:32.255227+0000","last_fullsized":"2023-10-09T10:47:32.255227+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:34.858348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":143,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.277359+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:47:32.277359+0000","last_peered":"2023-10-09T10:47:32.277359+0000","last_clean":"2023-10-09T10:47:32.277359+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:47:32.277359+0000","last_undegraded":"2023-10-09T10:47:32.277359+0000","last_fullsized":"2023-10-09T10:47:32.277359+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":127,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.264350+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:47:32.264350+0000","last_peered":"2023-10-09T10:47:32.264350+0000","last_clean":"2023-10-09T10:47:32.264350+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:47:32.264350+0000","last_undegraded":"2023-10-09T10:47:32.264350+0000","last_fullsized":"2023-10-09T10:47:32.264350+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":43,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.265917+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:47:32.265917+0000","last_peered":"2023-10-09T10:47:32.265917+0000","last_clean":"2023-10-09T10:47:32.265917+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:47:32.265917+0000","last_undegraded":"2023-10-09T10:47:32.265917+0000","last_fullsized":"2023-10-09T10:47:32.265917+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":179,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.281477+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:47:32.281477+0000","last_peered":"2023-10-09T10:47:32.281477+0000","last_clean":"2023-10-09T10:47:32.281477+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:47:32.281477+0000","last_undegraded":"2023-10-09T10:47:32.281477+0000","last_fullsized":"2023-10-09T10:47:32.281477+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":113,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.255808+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:47:32.255808+0000","last_peered":"2023-10-09T10:47:32.255808+0000","last_clean":"2023-10-09T10:47:32.255808+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:47:32.255808+0000","last_undegraded":"2023-10-09T10:47:32.255808+0000","last_fullsized":"2023-10-09T10:47:32.255808+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"175'6064","reported_seq":10213,"reported_epoch":175,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:40.973394+0000","last_change":"2023-10-09T10:46:33.521602+0000","last_active":"2023-10-09T10:47:40.973394+0000","last_peered":"2023-10-09T10:47:40.973394+0000","last_clean":"2023-10-09T10:46:31.055751+0000","last_became_active":"2023-10-09T10:46:33.364058+0000","last_became_peered":"2023-10-09T10:46:33.364058+0000","last_unstale":"2023-10-09T10:47:40.973394+0000","last_undegraded":"2023-10-09T10:47:40.973394+0000","last_fullsized":"2023-10-09T10:47:40.973394+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6064,"log_dups_size":0,"ondisk_log_size":6064,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119898112,"num_objects":1830,"num_object_clones":0,"num_object_copies":5490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1828,"num_objects_unfound":0,"num_objects_dirty":1830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5028,"num_write_kb":219424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2010,"num_bytes_recovered":131727360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":1830}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":43,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.266632+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:47:32.266632+0000","last_peered":"2023-10-09T10:47:32.266632+0000","last_clean":"2023-10-09T10:47:32.266632+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:47:32.266632+0000","last_undegraded":"2023-10-09T10:47:32.266632+0000","last_fullsized":"2023-10-09T10:47:32.266632+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":143,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.281541+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:47:32.281541+0000","last_peered":"2023-10-09T10:47:32.281541+0000","last_clean":"2023-10-09T10:47:32.281541+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:47:32.281541+0000","last_undegraded":"2023-10-09T10:47:32.281541+0000","last_fullsized":"2023-10-09T10:47:32.281541+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":508,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.266439+0000","last_change":"2023-10-09T10:46:05.620365+0000","last_active":"2023-10-09T10:47:32.266439+0000","last_peered":"2023-10-09T10:47:32.266439+0000","last_clean":"2023-10-09T10:47:32.266439+0000","last_became_active":"2023-10-09T10:46:05.620024+0000","last_became_peered":"2023-10-09T10:46:05.620024+0000","last_unstale":"2023-10-09T10:47:32.266439+0000","last_undegraded":"2023-10-09T10:47:32.266439+0000","last_fullsized":"2023-10-09T10:47:32.266439+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.13","version":"175'5890","reported_seq":6071,"reported_epoch":175,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:40.963109+0000","last_change":"2023-10-09T10:46:33.470698+0000","last_active":"2023-10-09T10:47:40.963109+0000","last_peered":"2023-10-09T10:47:40.963109+0000","last_clean":"2023-10-09T10:46:31.195699+0000","last_became_active":"2023-10-09T10:46:33.455632+0000","last_became_peered":"2023-10-09T10:46:33.455632+0000","last_unstale":"2023-10-09T10:47:40.963109+0000","last_undegraded":"2023-10-09T10:46:33.269323+0000","last_fullsized":"2023-10-09T10:46:33.241226+0000","mapping_epoch":154,"log_start":"103'303","ondisk_log_start":"103'303","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":5587,"log_dups_size":0,"ondisk_log_size":5587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:07:12.547333+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116785152,"num_objects":1782,"num_object_clones":0,"num_object_copies":5346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3315,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3175,"num_write_kb":105088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":1782},{"shards":"6,7","objects":-993}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":140,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.256568+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:47:32.256568+0000","last_peered":"2023-10-09T10:47:32.256568+0000","last_clean":"2023-10-09T10:47:32.256568+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:47:32.256568+0000","last_undegraded":"2023-10-09T10:47:32.256568+0000","last_fullsized":"2023-10-09T10:47:32.256568+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":100,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.281167+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:47:32.281167+0000","last_peered":"2023-10-09T10:47:32.281167+0000","last_clean":"2023-10-09T10:47:32.281167+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:47:32.281167+0000","last_undegraded":"2023-10-09T10:47:32.281167+0000","last_fullsized":"2023-10-09T10:47:32.281167+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"175'6049","reported_seq":14171,"reported_epoch":175,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:41.416397+0000","last_change":"2023-10-09T10:46:03.683196+0000","last_active":"2023-10-09T10:47:41.416397+0000","last_peered":"2023-10-09T10:47:41.416397+0000","last_clean":"2023-10-09T10:46:02.177309+0000","last_became_active":"2023-10-09T10:46:03.681058+0000","last_became_peered":"2023-10-09T10:46:03.681058+0000","last_unstale":"2023-10-09T10:47:41.416397+0000","last_undegraded":"2023-10-09T10:47:41.416397+0000","last_fullsized":"2023-10-09T10:47:41.416397+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6049,"log_dups_size":0,"ondisk_log_size":6049,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116948992,"num_objects":1785,"num_object_clones":0,"num_object_copies":5355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1785,"num_objects_unfound":0,"num_objects_dirty":1785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5003,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[1,5,6],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1785}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":96,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.277288+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:47:32.277288+0000","last_peered":"2023-10-09T10:47:32.277288+0000","last_clean":"2023-10-09T10:47:32.277288+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:47:32.277288+0000","last_undegraded":"2023-10-09T10:47:32.277288+0000","last_fullsized":"2023-10-09T10:47:32.277288+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":136,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.259518+0000","last_change":"2023-10-09T10:46:32.249921+0000","last_active":"2023-10-09T10:47:32.259518+0000","last_peered":"2023-10-09T10:47:32.259518+0000","last_clean":"2023-10-09T10:47:32.259518+0000","last_became_active":"2023-10-09T10:46:32.249590+0000","last_became_peered":"2023-10-09T10:46:32.249590+0000","last_unstale":"2023-10-09T10:47:32.259518+0000","last_undegraded":"2023-10-09T10:47:32.259518+0000","last_fullsized":"2023-10-09T10:47:32.259518+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:25.169752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"175'5970","reported_seq":13696,"reported_epoch":175,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:47:44.571888+0000","last_change":"2023-10-09T10:46:32.280686+0000","last_active":"2023-10-09T10:47:44.571888+0000","last_peered":"2023-10-09T10:47:44.571888+0000","last_clean":"2023-10-09T10:44:56.105580+0000","last_became_active":"2023-10-09T10:46:07.402111+0000","last_became_peered":"2023-10-09T10:46:07.402111+0000","last_unstale":"2023-10-09T10:47:44.571888+0000","last_undegraded":"2023-10-09T10:47:44.571888+0000","last_fullsized":"2023-10-09T10:46:05.812663+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5970,"log_dups_size":0,"ondisk_log_size":5970,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105545728,"num_objects":1611,"num_object_clones":0,"num_object_copies":4833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":241,"num_objects_unfound":0,"num_objects_dirty":1611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4963,"num_write_kb":210336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2450,"num_bytes_recovered":160563200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5],"avail_no_missing":["3","1","5","6"],"object_location_counts":[{"shards":"1,2,3,5","objects":240},{"shards":"1,3,5","objects":1},{"shards":"3,5","objects":1370}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":145,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.250919+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:47:32.250919+0000","last_peered":"2023-10-09T10:47:32.250919+0000","last_clean":"2023-10-09T10:47:32.250919+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:47:32.250919+0000","last_undegraded":"2023-10-09T10:47:32.250919+0000","last_fullsized":"2023-10-09T10:47:32.250919+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":139,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.250648+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:47:32.250648+0000","last_peered":"2023-10-09T10:47:32.250648+0000","last_clean":"2023-10-09T10:47:32.250648+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:47:32.250648+0000","last_undegraded":"2023-10-09T10:47:32.250648+0000","last_fullsized":"2023-10-09T10:47:32.250648+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":508,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.269072+0000","last_change":"2023-10-09T10:46:05.618276+0000","last_active":"2023-10-09T10:47:32.269072+0000","last_peered":"2023-10-09T10:47:32.269072+0000","last_clean":"2023-10-09T10:47:32.269072+0000","last_became_active":"2023-10-09T10:46:05.616509+0000","last_became_peered":"2023-10-09T10:46:05.616509+0000","last_unstale":"2023-10-09T10:47:32.269072+0000","last_undegraded":"2023-10-09T10:47:32.269072+0000","last_fullsized":"2023-10-09T10:47:32.269072+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.16","version":"175'5984","reported_seq":6084,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-10-09T10:47:44.521071+0000","last_change":"2023-10-09T10:46:03.592682+0000","last_active":"2023-10-09T10:47:44.521071+0000","last_peered":"2023-10-09T10:47:44.521071+0000","last_clean":"2023-10-09T10:47:44.521071+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:47:44.521071+0000","last_undegraded":"2023-10-09T10:47:44.521071+0000","last_fullsized":"2023-10-09T10:47:44.521071+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5984,"log_dups_size":0,"ondisk_log_size":5984,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107806720,"num_objects":1645,"num_object_clones":0,"num_object_copies":4935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4941,"num_write_kb":210752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":157,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.275591+0000","last_change":"2023-10-09T10:46:32.250256+0000","last_active":"2023-10-09T10:47:32.275591+0000","last_peered":"2023-10-09T10:47:32.275591+0000","last_clean":"2023-10-09T10:47:32.275591+0000","last_became_active":"2023-10-09T10:46:32.243312+0000","last_became_peered":"2023-10-09T10:46:32.243312+0000","last_unstale":"2023-10-09T10:47:32.275591+0000","last_undegraded":"2023-10-09T10:47:32.275591+0000","last_fullsized":"2023-10-09T10:47:32.275591+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:08.059777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":166,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.277103+0000","last_change":"2023-10-09T10:46:32.237577+0000","last_active":"2023-10-09T10:47:32.277103+0000","last_peered":"2023-10-09T10:47:32.277103+0000","last_clean":"2023-10-09T10:47:32.277103+0000","last_became_active":"2023-10-09T10:46:32.237316+0000","last_became_peered":"2023-10-09T10:46:32.237316+0000","last_unstale":"2023-10-09T10:47:32.277103+0000","last_undegraded":"2023-10-09T10:47:32.277103+0000","last_fullsized":"2023-10-09T10:47:32.277103+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:39.684602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":509,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.268195+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:47:32.268195+0000","last_peered":"2023-10-09T10:47:32.268195+0000","last_clean":"2023-10-09T10:47:32.268195+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:47:32.268195+0000","last_undegraded":"2023-10-09T10:47:32.268195+0000","last_fullsized":"2023-10-09T10:47:32.268195+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.17","version":"175'5806","reported_seq":5900,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-10-09T10:47:41.049592+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:47:41.049592+0000","last_peered":"2023-10-09T10:47:41.049592+0000","last_clean":"2023-10-09T10:47:41.049592+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:47:41.049592+0000","last_undegraded":"2023-10-09T10:47:41.049592+0000","last_fullsized":"2023-10-09T10:47:41.049592+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5806,"log_dups_size":0,"ondisk_log_size":5806,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111935488,"num_objects":1708,"num_object_clones":0,"num_object_copies":5124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4798,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":43,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.267038+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:47:32.267038+0000","last_peered":"2023-10-09T10:47:32.267038+0000","last_clean":"2023-10-09T10:47:32.267038+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:47:32.267038+0000","last_undegraded":"2023-10-09T10:47:32.267038+0000","last_fullsized":"2023-10-09T10:47:32.267038+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":139,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.260354+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:47:32.260354+0000","last_peered":"2023-10-09T10:47:32.260354+0000","last_clean":"2023-10-09T10:47:32.260354+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:47:32.260354+0000","last_undegraded":"2023-10-09T10:47:32.260354+0000","last_fullsized":"2023-10-09T10:47:32.260354+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":509,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.265662+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:47:32.265662+0000","last_peered":"2023-10-09T10:47:32.265662+0000","last_clean":"2023-10-09T10:47:32.265662+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:47:32.265662+0000","last_undegraded":"2023-10-09T10:47:32.265662+0000","last_fullsized":"2023-10-09T10:47:32.265662+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"175'5963","reported_seq":6062,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-10-09T10:47:41.470984+0000","last_change":"2023-10-09T10:46:33.243540+0000","last_active":"2023-10-09T10:47:41.470984+0000","last_peered":"2023-10-09T10:47:41.470984+0000","last_clean":"2023-10-09T10:47:41.470984+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:47:41.470984+0000","last_undegraded":"2023-10-09T10:47:41.470984+0000","last_fullsized":"2023-10-09T10:47:41.470984+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5963,"log_dups_size":0,"ondisk_log_size":5963,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115638272,"num_objects":1765,"num_object_clones":0,"num_object_copies":5295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4909,"num_write_kb":213536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":85,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.250729+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:47:32.250729+0000","last_peered":"2023-10-09T10:47:32.250729+0000","last_clean":"2023-10-09T10:47:32.250729+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:47:32.250729+0000","last_undegraded":"2023-10-09T10:47:32.250729+0000","last_fullsized":"2023-10-09T10:47:32.250729+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":43,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.266126+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:47:32.266126+0000","last_peered":"2023-10-09T10:47:32.266126+0000","last_clean":"2023-10-09T10:47:32.266126+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:47:32.266126+0000","last_undegraded":"2023-10-09T10:47:32.266126+0000","last_fullsized":"2023-10-09T10:47:32.266126+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":509,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.266816+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:47:32.266816+0000","last_peered":"2023-10-09T10:47:32.266816+0000","last_clean":"2023-10-09T10:47:32.266816+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:47:32.266816+0000","last_undegraded":"2023-10-09T10:47:32.266816+0000","last_fullsized":"2023-10-09T10:47:32.266816+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"175'6082","reported_seq":6191,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-10-09T10:47:44.518771+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:47:44.518771+0000","last_peered":"2023-10-09T10:47:44.518771+0000","last_clean":"2023-10-09T10:47:44.518771+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:47:44.518771+0000","last_undegraded":"2023-10-09T10:47:44.518771+0000","last_fullsized":"2023-10-09T10:47:44.518771+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6082,"log_dups_size":0,"ondisk_log_size":6082,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107937792,"num_objects":1647,"num_object_clones":0,"num_object_copies":4941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5021,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":53,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.268114+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:47:32.268114+0000","last_peered":"2023-10-09T10:47:32.268114+0000","last_clean":"2023-10-09T10:47:32.268114+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:47:32.268114+0000","last_undegraded":"2023-10-09T10:47:32.268114+0000","last_fullsized":"2023-10-09T10:47:32.268114+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":143,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.279596+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:47:32.279596+0000","last_peered":"2023-10-09T10:47:32.279596+0000","last_clean":"2023-10-09T10:47:32.279596+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:47:32.279596+0000","last_undegraded":"2023-10-09T10:47:32.279596+0000","last_fullsized":"2023-10-09T10:47:32.279596+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":509,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.267401+0000","last_change":"2023-10-09T10:46:05.620574+0000","last_active":"2023-10-09T10:47:32.267401+0000","last_peered":"2023-10-09T10:47:32.267401+0000","last_clean":"2023-10-09T10:47:32.267401+0000","last_became_active":"2023-10-09T10:46:05.620212+0000","last_became_peered":"2023-10-09T10:46:05.620212+0000","last_unstale":"2023-10-09T10:47:32.267401+0000","last_undegraded":"2023-10-09T10:47:32.267401+0000","last_fullsized":"2023-10-09T10:47:32.267401+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"175'10139","reported_seq":14289,"reported_epoch":175,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:41.036158+0000","last_change":"2023-10-09T10:46:32.247903+0000","last_active":"2023-10-09T10:47:41.036158+0000","last_peered":"2023-10-09T10:47:41.036158+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:46:02.739355+0000","last_became_peered":"2023-10-09T10:46:02.739355+0000","last_unstale":"2023-10-09T10:47:41.036158+0000","last_undegraded":"2023-10-09T10:46:02.608805+0000","last_fullsized":"2023-10-09T10:46:02.583589+0000","mapping_epoch":142,"log_start":"104'1300","ondisk_log_start":"104'1300","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8839,"log_dups_size":0,"ondisk_log_size":8839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:32.899634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228917248,"num_objects":3493,"num_object_clones":0,"num_object_copies":10479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10139,"num_write_kb":436224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2022,"num_bytes_recovered":132513792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":625},{"shards":"5,6","objects":2867}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":168,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.274892+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:47:32.274892+0000","last_peered":"2023-10-09T10:47:32.274892+0000","last_clean":"2023-10-09T10:47:32.274892+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:47:32.274892+0000","last_undegraded":"2023-10-09T10:47:32.274892+0000","last_fullsized":"2023-10-09T10:47:32.274892+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":122,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.255217+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:47:32.255217+0000","last_peered":"2023-10-09T10:47:32.255217+0000","last_clean":"2023-10-09T10:47:32.255217+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:47:32.255217+0000","last_undegraded":"2023-10-09T10:47:32.255217+0000","last_fullsized":"2023-10-09T10:47:32.255217+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":489,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.266581+0000","last_change":"2023-10-09T10:46:02.631378+0000","last_active":"2023-10-09T10:47:32.266581+0000","last_peered":"2023-10-09T10:47:32.266581+0000","last_clean":"2023-10-09T10:47:32.266581+0000","last_became_active":"2023-10-09T10:46:02.629142+0000","last_became_peered":"2023-10-09T10:46:02.629142+0000","last_unstale":"2023-10-09T10:47:32.266581+0000","last_undegraded":"2023-10-09T10:47:32.266581+0000","last_fullsized":"2023-10-09T10:47:32.266581+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.b","version":"175'9880","reported_seq":9966,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-10-09T10:47:44.522340+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:47:44.522340+0000","last_peered":"2023-10-09T10:47:44.522340+0000","last_clean":"2023-10-09T10:47:44.522340+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:47:44.522340+0000","last_undegraded":"2023-10-09T10:47:44.522340+0000","last_fullsized":"2023-10-09T10:47:44.522340+0000","mapping_epoch":98,"log_start":"124'4125","ondisk_log_start":"124'4125","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5755,"log_dups_size":0,"ondisk_log_size":5755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216268800,"num_objects":3300,"num_object_clones":0,"num_object_copies":9900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9880,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":141,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.275406+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:47:32.275406+0000","last_peered":"2023-10-09T10:47:32.275406+0000","last_clean":"2023-10-09T10:47:32.275406+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:47:32.275406+0000","last_undegraded":"2023-10-09T10:47:32.275406+0000","last_fullsized":"2023-10-09T10:47:32.275406+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":176,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.250656+0000","last_change":"2023-10-09T10:46:32.251035+0000","last_active":"2023-10-09T10:47:32.250656+0000","last_peered":"2023-10-09T10:47:32.250656+0000","last_clean":"2023-10-09T10:47:32.250656+0000","last_became_active":"2023-10-09T10:46:32.250293+0000","last_became_peered":"2023-10-09T10:46:32.250293+0000","last_unstale":"2023-10-09T10:47:32.250656+0000","last_undegraded":"2023-10-09T10:47:32.250656+0000","last_fullsized":"2023-10-09T10:47:32.250656+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.787766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":509,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.266337+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:47:32.266337+0000","last_peered":"2023-10-09T10:47:32.266337+0000","last_clean":"2023-10-09T10:47:32.266337+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:47:32.266337+0000","last_undegraded":"2023-10-09T10:47:32.266337+0000","last_fullsized":"2023-10-09T10:47:32.266337+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"175'6073","reported_seq":6180,"reported_epoch":175,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:44.400946+0000","last_change":"2023-10-09T10:46:33.540719+0000","last_active":"2023-10-09T10:47:44.400946+0000","last_peered":"2023-10-09T10:47:44.400946+0000","last_clean":"2023-10-09T10:46:31.137696+0000","last_became_active":"2023-10-09T10:46:33.527650+0000","last_became_peered":"2023-10-09T10:46:33.527650+0000","last_unstale":"2023-10-09T10:47:44.400946+0000","last_undegraded":"2023-10-09T10:46:33.285468+0000","last_fullsized":"2023-10-09T10:46:33.235683+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6073,"log_dups_size":0,"ondisk_log_size":6073,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110002176,"num_objects":1678,"num_object_clones":0,"num_object_copies":5034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5008,"num_write_kb":213984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1666},{"shards":"3,7","objects":-1131}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":96,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.260279+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:47:32.260279+0000","last_peered":"2023-10-09T10:47:32.260279+0000","last_clean":"2023-10-09T10:47:32.260279+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:47:32.260279+0000","last_undegraded":"2023-10-09T10:47:32.260279+0000","last_fullsized":"2023-10-09T10:47:32.260279+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":113,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.259754+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:47:32.259754+0000","last_peered":"2023-10-09T10:47:32.259754+0000","last_clean":"2023-10-09T10:47:32.259754+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:47:32.259754+0000","last_undegraded":"2023-10-09T10:47:32.259754+0000","last_fullsized":"2023-10-09T10:47:32.259754+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":509,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.265773+0000","last_change":"2023-10-09T10:46:05.620030+0000","last_active":"2023-10-09T10:47:32.265773+0000","last_peered":"2023-10-09T10:47:32.265773+0000","last_clean":"2023-10-09T10:47:32.265773+0000","last_became_active":"2023-10-09T10:46:05.619694+0000","last_became_peered":"2023-10-09T10:46:05.619694+0000","last_unstale":"2023-10-09T10:47:32.265773+0000","last_undegraded":"2023-10-09T10:47:32.265773+0000","last_fullsized":"2023-10-09T10:47:32.265773+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.9","version":"175'5993","reported_seq":6111,"reported_epoch":175,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:42.566595+0000","last_change":"2023-10-09T10:46:34.405499+0000","last_active":"2023-10-09T10:47:42.566595+0000","last_peered":"2023-10-09T10:47:42.566595+0000","last_clean":"2023-10-09T10:46:31.190603+0000","last_became_active":"2023-10-09T10:46:34.396448+0000","last_became_peered":"2023-10-09T10:46:34.396448+0000","last_unstale":"2023-10-09T10:47:42.566595+0000","last_undegraded":"2023-10-09T10:46:33.246949+0000","last_fullsized":"2023-10-09T10:46:33.237968+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5993,"log_dups_size":0,"ondisk_log_size":5993,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108986368,"num_objects":1663,"num_object_clones":0,"num_object_copies":4989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2004,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4951,"num_write_kb":211648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1303,"num_bytes_recovered":85393408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":360},{"shards":"0,7","objects":195}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":140,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.256285+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:47:32.256285+0000","last_peered":"2023-10-09T10:47:32.256285+0000","last_clean":"2023-10-09T10:47:32.256285+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:47:32.256285+0000","last_undegraded":"2023-10-09T10:47:32.256285+0000","last_fullsized":"2023-10-09T10:47:32.256285+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":43,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.266066+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:47:32.266066+0000","last_peered":"2023-10-09T10:47:32.266066+0000","last_clean":"2023-10-09T10:47:32.266066+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:47:32.266066+0000","last_undegraded":"2023-10-09T10:47:32.266066+0000","last_fullsized":"2023-10-09T10:47:32.266066+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":500,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.268896+0000","last_change":"2023-10-09T10:46:05.618134+0000","last_active":"2023-10-09T10:47:32.268896+0000","last_peered":"2023-10-09T10:47:32.268896+0000","last_clean":"2023-10-09T10:47:32.268896+0000","last_became_active":"2023-10-09T10:46:05.616057+0000","last_became_peered":"2023-10-09T10:46:05.616057+0000","last_unstale":"2023-10-09T10:47:32.268896+0000","last_undegraded":"2023-10-09T10:47:32.268896+0000","last_fullsized":"2023-10-09T10:47:32.268896+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"175'9698","reported_seq":9791,"reported_epoch":175,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:41.011824+0000","last_change":"2023-10-09T10:46:33.576530+0000","last_active":"2023-10-09T10:47:41.011824+0000","last_peered":"2023-10-09T10:47:41.011824+0000","last_clean":"2023-10-09T10:46:31.183481+0000","last_became_active":"2023-10-09T10:46:33.542304+0000","last_became_peered":"2023-10-09T10:46:33.542304+0000","last_unstale":"2023-10-09T10:47:41.011824+0000","last_undegraded":"2023-10-09T10:46:33.300581+0000","last_fullsized":"2023-10-09T10:46:33.246047+0000","mapping_epoch":154,"log_start":"108'2377","ondisk_log_start":"108'2377","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7321,"log_dups_size":0,"ondisk_log_size":7321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:26.739488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231735296,"num_objects":3536,"num_object_clones":0,"num_object_copies":10608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6617,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9698,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":3536},{"shards":"1,4","objects":-2025}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":167,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.278770+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:47:32.278770+0000","last_peered":"2023-10-09T10:47:32.278770+0000","last_clean":"2023-10-09T10:47:32.278770+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:47:32.278770+0000","last_undegraded":"2023-10-09T10:47:32.278770+0000","last_fullsized":"2023-10-09T10:47:32.278770+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":139,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.250859+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:47:32.250859+0000","last_peered":"2023-10-09T10:47:32.250859+0000","last_clean":"2023-10-09T10:47:32.250859+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:47:32.250859+0000","last_undegraded":"2023-10-09T10:47:32.250859+0000","last_fullsized":"2023-10-09T10:47:32.250859+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":489,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.268282+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:47:32.268282+0000","last_peered":"2023-10-09T10:47:32.268282+0000","last_clean":"2023-10-09T10:47:32.268282+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:47:32.268282+0000","last_undegraded":"2023-10-09T10:47:32.268282+0000","last_fullsized":"2023-10-09T10:47:32.268282+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.f","version":"175'9794","reported_seq":9887,"reported_epoch":175,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:44.497523+0000","last_change":"2023-10-09T10:46:33.651756+0000","last_active":"2023-10-09T10:47:44.497523+0000","last_peered":"2023-10-09T10:47:44.497523+0000","last_clean":"2023-10-09T10:46:31.156458+0000","last_became_active":"2023-10-09T10:46:33.621003+0000","last_became_peered":"2023-10-09T10:46:33.621003+0000","last_unstale":"2023-10-09T10:47:44.497523+0000","last_undegraded":"2023-10-09T10:46:33.309813+0000","last_fullsized":"2023-10-09T10:46:33.236508+0000","mapping_epoch":154,"log_start":"124'4110","ondisk_log_start":"124'4110","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5684,"log_dups_size":0,"ondisk_log_size":5684,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:13.704346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211550208,"num_objects":3228,"num_object_clones":0,"num_object_copies":9684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9794,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":3228},{"shards":"3,4","objects":-2224}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":114,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.255379+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:47:32.255379+0000","last_peered":"2023-10-09T10:47:32.255379+0000","last_clean":"2023-10-09T10:47:32.255379+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:47:32.255379+0000","last_undegraded":"2023-10-09T10:47:32.255379+0000","last_fullsized":"2023-10-09T10:47:32.255379+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":166,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.251177+0000","last_change":"2023-10-09T10:46:32.245583+0000","last_active":"2023-10-09T10:47:32.251177+0000","last_peered":"2023-10-09T10:47:32.251177+0000","last_clean":"2023-10-09T10:47:32.251177+0000","last_became_active":"2023-10-09T10:46:32.245271+0000","last_became_peered":"2023-10-09T10:46:32.245271+0000","last_unstale":"2023-10-09T10:47:32.251177+0000","last_undegraded":"2023-10-09T10:47:32.251177+0000","last_fullsized":"2023-10-09T10:47:32.251177+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.025051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":509,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.265826+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:47:32.265826+0000","last_peered":"2023-10-09T10:47:32.265826+0000","last_clean":"2023-10-09T10:47:32.265826+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:47:32.265826+0000","last_undegraded":"2023-10-09T10:47:32.265826+0000","last_fullsized":"2023-10-09T10:47:32.265826+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"175'10040","reported_seq":10127,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-10-09T10:47:44.491104+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:47:44.491104+0000","last_peered":"2023-10-09T10:47:44.491104+0000","last_clean":"2023-10-09T10:47:44.491104+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:47:44.491104+0000","last_undegraded":"2023-10-09T10:47:44.491104+0000","last_fullsized":"2023-10-09T10:47:44.491104+0000","mapping_epoch":98,"log_start":"124'4378","ondisk_log_start":"124'4378","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5662,"log_dups_size":0,"ondisk_log_size":5662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212598784,"num_objects":3244,"num_object_clones":0,"num_object_copies":9732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10040,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":150,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.250588+0000","last_change":"2023-10-09T10:46:32.245685+0000","last_active":"2023-10-09T10:47:32.250588+0000","last_peered":"2023-10-09T10:47:32.250588+0000","last_clean":"2023-10-09T10:47:32.250588+0000","last_became_active":"2023-10-09T10:46:32.245314+0000","last_became_peered":"2023-10-09T10:46:32.245314+0000","last_unstale":"2023-10-09T10:47:32.250588+0000","last_undegraded":"2023-10-09T10:47:32.250588+0000","last_fullsized":"2023-10-09T10:47:32.250588+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:51.716860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":139,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.253139+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:47:32.253139+0000","last_peered":"2023-10-09T10:47:32.253139+0000","last_clean":"2023-10-09T10:47:32.253139+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:47:32.253139+0000","last_undegraded":"2023-10-09T10:47:32.253139+0000","last_fullsized":"2023-10-09T10:47:32.253139+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":500,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.266891+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:47:32.266891+0000","last_peered":"2023-10-09T10:47:32.266891+0000","last_clean":"2023-10-09T10:47:32.266891+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:47:32.266891+0000","last_undegraded":"2023-10-09T10:47:32.266891+0000","last_fullsized":"2023-10-09T10:47:32.266891+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"175'9888","reported_seq":9993,"reported_epoch":175,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:41.020949+0000","last_change":"2023-10-09T10:46:02.754507+0000","last_active":"2023-10-09T10:47:41.020949+0000","last_peered":"2023-10-09T10:47:41.020949+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:46:02.747510+0000","last_became_peered":"2023-10-09T10:46:02.747510+0000","last_unstale":"2023-10-09T10:47:41.020949+0000","last_undegraded":"2023-10-09T10:46:02.605987+0000","last_fullsized":"2023-10-09T10:46:02.585248+0000","mapping_epoch":142,"log_start":"104'1000","ondisk_log_start":"104'1000","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8888,"log_dups_size":0,"ondisk_log_size":8888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:59.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229834776,"num_objects":3508,"num_object_clones":0,"num_object_copies":10524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3508,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9888,"num_write_kb":428609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,2,5","objects":2718},{"shards":"1,5","objects":790}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":144,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.281301+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:47:32.281301+0000","last_peered":"2023-10-09T10:47:32.281301+0000","last_clean":"2023-10-09T10:47:32.281301+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:47:32.281301+0000","last_undegraded":"2023-10-09T10:47:32.281301+0000","last_fullsized":"2023-10-09T10:47:32.281301+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":127,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.260519+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:47:32.260519+0000","last_peered":"2023-10-09T10:47:32.260519+0000","last_clean":"2023-10-09T10:47:32.260519+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:47:32.260519+0000","last_undegraded":"2023-10-09T10:47:32.260519+0000","last_fullsized":"2023-10-09T10:47:32.260519+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":509,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.266495+0000","last_change":"2023-10-09T10:46:05.617983+0000","last_active":"2023-10-09T10:47:32.266495+0000","last_peered":"2023-10-09T10:47:32.266495+0000","last_clean":"2023-10-09T10:47:32.266495+0000","last_became_active":"2023-10-09T10:46:05.616021+0000","last_became_peered":"2023-10-09T10:46:05.616021+0000","last_unstale":"2023-10-09T10:47:32.266495+0000","last_undegraded":"2023-10-09T10:47:32.266495+0000","last_fullsized":"2023-10-09T10:47:32.266495+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":170,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.260716+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:47:32.260716+0000","last_peered":"2023-10-09T10:47:32.260716+0000","last_clean":"2023-10-09T10:47:32.260716+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:47:32.260716+0000","last_undegraded":"2023-10-09T10:47:32.260716+0000","last_fullsized":"2023-10-09T10:47:32.260716+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":167,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.260423+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:47:32.260423+0000","last_peered":"2023-10-09T10:47:32.260423+0000","last_clean":"2023-10-09T10:47:32.260423+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:47:32.260423+0000","last_undegraded":"2023-10-09T10:47:32.260423+0000","last_fullsized":"2023-10-09T10:47:32.260423+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":112,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.259587+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:47:32.259587+0000","last_peered":"2023-10-09T10:47:32.259587+0000","last_clean":"2023-10-09T10:47:32.259587+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:47:32.259587+0000","last_undegraded":"2023-10-09T10:47:32.259587+0000","last_fullsized":"2023-10-09T10:47:32.259587+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"175'5993","reported_seq":10025,"reported_epoch":175,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:44.650939+0000","last_change":"2023-10-09T10:46:32.248218+0000","last_active":"2023-10-09T10:47:44.650939+0000","last_peered":"2023-10-09T10:47:44.650939+0000","last_clean":"2023-10-09T10:46:02.181912+0000","last_became_active":"2023-10-09T10:46:32.226237+0000","last_became_peered":"2023-10-09T10:46:32.226237+0000","last_unstale":"2023-10-09T10:47:44.650939+0000","last_undegraded":"2023-10-09T10:47:44.650939+0000","last_fullsized":"2023-10-09T10:47:44.650939+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5993,"log_dups_size":0,"ondisk_log_size":5993,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111312896,"num_objects":1698,"num_object_clones":0,"num_object_copies":5094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1698,"num_objects_unfound":0,"num_objects_dirty":1698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4946,"num_write_kb":212640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1698}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":508,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.266495+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:47:32.266495+0000","last_peered":"2023-10-09T10:47:32.266495+0000","last_clean":"2023-10-09T10:47:32.266495+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:47:32.266495+0000","last_undegraded":"2023-10-09T10:47:32.266495+0000","last_fullsized":"2023-10-09T10:47:32.266495+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"175'6061","reported_seq":6238,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-10-09T10:47:44.520970+0000","last_change":"2023-10-09T10:46:02.760963+0000","last_active":"2023-10-09T10:47:44.520970+0000","last_peered":"2023-10-09T10:47:44.520970+0000","last_clean":"2023-10-09T10:47:44.520970+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:47:44.520970+0000","last_undegraded":"2023-10-09T10:47:44.520970+0000","last_fullsized":"2023-10-09T10:47:44.520970+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"141'4416","last_scrub_stamp":"2023-10-09T10:46:02.760810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:46:02.760810+0000","objects_scrubbed":3378,"log_size":6061,"log_dups_size":0,"ondisk_log_size":6061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:35.863002+0000","scrub_duration":0.47769323899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113573888,"num_objects":1733,"num_object_clones":0,"num_object_copies":5199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1645,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":144,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.257970+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:47:32.257970+0000","last_peered":"2023-10-09T10:47:32.257970+0000","last_clean":"2023-10-09T10:47:32.257970+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:47:32.257970+0000","last_undegraded":"2023-10-09T10:47:32.257970+0000","last_fullsized":"2023-10-09T10:47:32.257970+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":160,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.251328+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:47:32.251328+0000","last_peered":"2023-10-09T10:47:32.251328+0000","last_clean":"2023-10-09T10:47:32.251328+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:47:32.251328+0000","last_undegraded":"2023-10-09T10:47:32.251328+0000","last_fullsized":"2023-10-09T10:47:32.251328+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":151,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.260785+0000","last_change":"2023-10-09T10:46:32.245608+0000","last_active":"2023-10-09T10:47:32.260785+0000","last_peered":"2023-10-09T10:47:32.260785+0000","last_clean":"2023-10-09T10:47:32.260785+0000","last_became_active":"2023-10-09T10:46:32.245290+0000","last_became_peered":"2023-10-09T10:46:32.245290+0000","last_unstale":"2023-10-09T10:47:32.260785+0000","last_undegraded":"2023-10-09T10:47:32.260785+0000","last_fullsized":"2023-10-09T10:47:32.260785+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:48.832515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":509,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.266963+0000","last_change":"2023-10-09T10:46:05.620057+0000","last_active":"2023-10-09T10:47:32.266963+0000","last_peered":"2023-10-09T10:47:32.266963+0000","last_clean":"2023-10-09T10:47:32.266963+0000","last_became_active":"2023-10-09T10:46:05.619703+0000","last_became_peered":"2023-10-09T10:46:05.619703+0000","last_unstale":"2023-10-09T10:47:32.266963+0000","last_undegraded":"2023-10-09T10:47:32.266963+0000","last_fullsized":"2023-10-09T10:47:32.266963+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"175'6041","reported_seq":6153,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-10-09T10:47:44.388767+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:47:44.388767+0000","last_peered":"2023-10-09T10:47:44.388767+0000","last_clean":"2023-10-09T10:47:44.388767+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:47:44.388767+0000","last_undegraded":"2023-10-09T10:47:44.388767+0000","last_fullsized":"2023-10-09T10:47:44.388767+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6041,"log_dups_size":0,"ondisk_log_size":6041,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111411200,"num_objects":1700,"num_object_clones":0,"num_object_copies":5100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4980,"num_write_kb":213760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":172,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.281334+0000","last_change":"2023-10-09T10:46:32.241041+0000","last_active":"2023-10-09T10:47:32.281334+0000","last_peered":"2023-10-09T10:47:32.281334+0000","last_clean":"2023-10-09T10:47:32.281334+0000","last_became_active":"2023-10-09T10:46:32.238683+0000","last_became_peered":"2023-10-09T10:46:32.238683+0000","last_unstale":"2023-10-09T10:47:32.281334+0000","last_undegraded":"2023-10-09T10:47:32.281334+0000","last_fullsized":"2023-10-09T10:47:32.281334+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:25.817713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":136,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.259740+0000","last_change":"2023-10-09T10:46:32.250104+0000","last_active":"2023-10-09T10:47:32.259740+0000","last_peered":"2023-10-09T10:47:32.259740+0000","last_clean":"2023-10-09T10:47:32.259740+0000","last_became_active":"2023-10-09T10:46:32.249805+0000","last_became_peered":"2023-10-09T10:46:32.249805+0000","last_unstale":"2023-10-09T10:47:32.259740+0000","last_undegraded":"2023-10-09T10:47:32.259740+0000","last_fullsized":"2023-10-09T10:47:32.259740+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:07.230672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":165,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.259957+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:47:32.259957+0000","last_peered":"2023-10-09T10:47:32.259957+0000","last_clean":"2023-10-09T10:47:32.259957+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:47:32.259957+0000","last_undegraded":"2023-10-09T10:47:32.259957+0000","last_fullsized":"2023-10-09T10:47:32.259957+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"175'5827","reported_seq":5926,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-10-09T10:47:41.049706+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:47:41.049706+0000","last_peered":"2023-10-09T10:47:41.049706+0000","last_clean":"2023-10-09T10:47:41.049706+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:47:41.049706+0000","last_undegraded":"2023-10-09T10:47:41.049706+0000","last_fullsized":"2023-10-09T10:47:41.049706+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5827,"log_dups_size":0,"ondisk_log_size":5827,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119996416,"num_objects":1831,"num_object_clones":0,"num_object_copies":5493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4819,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":154,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.255551+0000","last_change":"2023-10-09T10:46:32.235346+0000","last_active":"2023-10-09T10:47:32.255551+0000","last_peered":"2023-10-09T10:47:32.255551+0000","last_clean":"2023-10-09T10:47:32.255551+0000","last_became_active":"2023-10-09T10:46:32.234906+0000","last_became_peered":"2023-10-09T10:46:32.234906+0000","last_unstale":"2023-10-09T10:47:32.255551+0000","last_undegraded":"2023-10-09T10:47:32.255551+0000","last_fullsized":"2023-10-09T10:47:32.255551+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.199251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":509,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.267984+0000","last_change":"2023-10-09T10:46:05.617146+0000","last_active":"2023-10-09T10:47:32.267984+0000","last_peered":"2023-10-09T10:47:32.267984+0000","last_clean":"2023-10-09T10:47:32.267984+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:47:32.267984+0000","last_undegraded":"2023-10-09T10:47:32.267984+0000","last_fullsized":"2023-10-09T10:47:32.267984+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":103,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-10-09T10:47:38.685674+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:47:38.685674+0000","last_peered":"2023-10-09T10:47:38.685674+0000","last_clean":"2023-10-09T10:47:38.685674+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:47:38.685674+0000","last_undegraded":"2023-10-09T10:47:38.685674+0000","last_fullsized":"2023-10-09T10:47:38.685674+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":72,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.251007+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:47:32.251007+0000","last_peered":"2023-10-09T10:47:32.251007+0000","last_clean":"2023-10-09T10:47:32.251007+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:47:32.251007+0000","last_undegraded":"2023-10-09T10:47:32.251007+0000","last_fullsized":"2023-10-09T10:47:32.251007+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":509,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.265739+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:47:32.265739+0000","last_peered":"2023-10-09T10:47:32.265739+0000","last_clean":"2023-10-09T10:47:32.265739+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:47:32.265739+0000","last_undegraded":"2023-10-09T10:47:32.265739+0000","last_fullsized":"2023-10-09T10:47:32.265739+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":166,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.259853+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:47:32.259853+0000","last_peered":"2023-10-09T10:47:32.259853+0000","last_clean":"2023-10-09T10:47:32.259853+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:47:32.259853+0000","last_undegraded":"2023-10-09T10:47:32.259853+0000","last_fullsized":"2023-10-09T10:47:32.259853+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.266417+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:47:32.266417+0000","last_peered":"2023-10-09T10:47:32.266417+0000","last_clean":"2023-10-09T10:47:32.266417+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:47:32.266417+0000","last_undegraded":"2023-10-09T10:47:32.266417+0000","last_fullsized":"2023-10-09T10:47:32.266417+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":52,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.266062+0000","last_change":"2023-10-09T10:46:32.236202+0000","last_active":"2023-10-09T10:47:32.266062+0000","last_peered":"2023-10-09T10:47:32.266062+0000","last_clean":"2023-10-09T10:47:32.266062+0000","last_became_active":"2023-10-09T10:46:32.235901+0000","last_became_peered":"2023-10-09T10:46:32.235901+0000","last_unstale":"2023-10-09T10:47:32.266062+0000","last_undegraded":"2023-10-09T10:47:32.266062+0000","last_fullsized":"2023-10-09T10:47:32.266062+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:48.260403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"175'5999","reported_seq":6145,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-10-09T10:47:41.488907+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:47:41.488907+0000","last_peered":"2023-10-09T10:47:41.488907+0000","last_clean":"2023-10-09T10:47:41.488907+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:47:41.488907+0000","last_undegraded":"2023-10-09T10:47:41.488907+0000","last_fullsized":"2023-10-09T10:47:41.488907+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":5999,"log_dups_size":0,"ondisk_log_size":5999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116588544,"num_objects":1779,"num_object_clones":0,"num_object_copies":5337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2969,"num_write_kb":89664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":509,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.265850+0000","last_change":"2023-10-09T10:46:05.620764+0000","last_active":"2023-10-09T10:47:32.265850+0000","last_peered":"2023-10-09T10:47:32.265850+0000","last_clean":"2023-10-09T10:47:32.265850+0000","last_became_active":"2023-10-09T10:46:05.619870+0000","last_became_peered":"2023-10-09T10:46:05.619870+0000","last_unstale":"2023-10-09T10:47:32.265850+0000","last_undegraded":"2023-10-09T10:47:32.265850+0000","last_fullsized":"2023-10-09T10:47:32.265850+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":163,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.274402+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:47:32.274402+0000","last_peered":"2023-10-09T10:47:32.274402+0000","last_clean":"2023-10-09T10:47:32.274402+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:47:32.274402+0000","last_undegraded":"2023-10-09T10:47:32.274402+0000","last_fullsized":"2023-10-09T10:47:32.274402+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":136,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.260867+0000","last_change":"2023-10-09T10:46:32.251450+0000","last_active":"2023-10-09T10:47:32.260867+0000","last_peered":"2023-10-09T10:47:32.260867+0000","last_clean":"2023-10-09T10:47:32.260867+0000","last_became_active":"2023-10-09T10:46:32.250933+0000","last_became_peered":"2023-10-09T10:46:32.250933+0000","last_unstale":"2023-10-09T10:47:32.260867+0000","last_undegraded":"2023-10-09T10:47:32.260867+0000","last_fullsized":"2023-10-09T10:47:32.260867+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:04.116178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":43,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.265919+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:47:32.265919+0000","last_peered":"2023-10-09T10:47:32.265919+0000","last_clean":"2023-10-09T10:47:32.265919+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:47:32.265919+0000","last_undegraded":"2023-10-09T10:47:32.265919+0000","last_fullsized":"2023-10-09T10:47:32.265919+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"175'5836","reported_seq":12527,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-10-09T10:47:41.410023+0000","last_change":"2023-10-09T10:47:02.269289+0000","last_active":"2023-10-09T10:47:41.410023+0000","last_peered":"2023-10-09T10:47:41.410023+0000","last_clean":"2023-10-09T10:47:41.410023+0000","last_became_active":"2023-10-09T10:47:02.257854+0000","last_became_peered":"2023-10-09T10:47:02.257854+0000","last_unstale":"2023-10-09T10:47:41.410023+0000","last_undegraded":"2023-10-09T10:47:41.410023+0000","last_fullsized":"2023-10-09T10:47:41.410023+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5836,"log_dups_size":0,"ondisk_log_size":5836,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116293632,"num_objects":1774,"num_object_clones":0,"num_object_copies":5322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4828,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3287,"num_bytes_recovered":181927936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":509,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.265597+0000","last_change":"2023-10-09T10:46:05.618443+0000","last_active":"2023-10-09T10:47:32.265597+0000","last_peered":"2023-10-09T10:47:32.265597+0000","last_clean":"2023-10-09T10:47:32.265597+0000","last_became_active":"2023-10-09T10:46:05.616588+0000","last_became_peered":"2023-10-09T10:46:05.616588+0000","last_unstale":"2023-10-09T10:47:32.265597+0000","last_undegraded":"2023-10-09T10:47:32.265597+0000","last_fullsized":"2023-10-09T10:47:32.265597+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":167,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.279530+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:47:32.279530+0000","last_peered":"2023-10-09T10:47:32.279530+0000","last_clean":"2023-10-09T10:47:32.279530+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:47:32.279530+0000","last_undegraded":"2023-10-09T10:47:32.279530+0000","last_fullsized":"2023-10-09T10:47:32.279530+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":112,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.260214+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:47:32.260214+0000","last_peered":"2023-10-09T10:47:32.260214+0000","last_clean":"2023-10-09T10:47:32.260214+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:47:32.260214+0000","last_undegraded":"2023-10-09T10:47:32.260214+0000","last_fullsized":"2023-10-09T10:47:32.260214+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":157,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:34.562169+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:47:34.562169+0000","last_peered":"2023-10-09T10:47:34.562169+0000","last_clean":"2023-10-09T10:47:34.562169+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:47:34.562169+0000","last_undegraded":"2023-10-09T10:47:34.562169+0000","last_fullsized":"2023-10-09T10:47:34.562169+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"175'5964","reported_seq":7410,"reported_epoch":175,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:41.014049+0000","last_change":"2023-10-09T10:47:29.622439+0000","last_active":"2023-10-09T10:47:41.014049+0000","last_peered":"2023-10-09T10:47:41.014049+0000","last_clean":"2023-10-09T10:46:31.130083+0000","last_became_active":"2023-10-09T10:46:33.493288+0000","last_became_peered":"2023-10-09T10:46:33.493288+0000","last_unstale":"2023-10-09T10:47:41.014049+0000","last_undegraded":"2023-10-09T10:46:33.283624+0000","last_fullsized":"2023-10-09T10:46:33.244043+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5964,"log_dups_size":0,"ondisk_log_size":5964,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113868800,"num_objects":1737,"num_object_clones":0,"num_object_copies":5211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":838,"num_objects_misplaced":1327,"num_objects_unfound":0,"num_objects_dirty":1737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4927,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":650,"num_bytes_recovered":42598400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":1327},{"shards":"3,4","objects":-428}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":508,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.266612+0000","last_change":"2023-10-09T10:46:05.617589+0000","last_active":"2023-10-09T10:47:32.266612+0000","last_peered":"2023-10-09T10:47:32.266612+0000","last_clean":"2023-10-09T10:47:32.266612+0000","last_became_active":"2023-10-09T10:46:05.616400+0000","last_became_peered":"2023-10-09T10:46:05.616400+0000","last_unstale":"2023-10-09T10:47:32.266612+0000","last_undegraded":"2023-10-09T10:47:32.266612+0000","last_fullsized":"2023-10-09T10:47:32.266612+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":32,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.279298+0000","last_change":"2023-10-09T10:46:32.245106+0000","last_active":"2023-10-09T10:47:32.279298+0000","last_peered":"2023-10-09T10:47:32.279298+0000","last_clean":"2023-10-09T10:47:32.279298+0000","last_became_active":"2023-10-09T10:46:32.244826+0000","last_became_peered":"2023-10-09T10:46:32.244826+0000","last_unstale":"2023-10-09T10:47:32.279298+0000","last_undegraded":"2023-10-09T10:47:32.279298+0000","last_fullsized":"2023-10-09T10:47:32.279298+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:30.322834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":47,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.265995+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:47:32.265995+0000","last_peered":"2023-10-09T10:47:32.265995+0000","last_clean":"2023-10-09T10:47:32.265995+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:47:32.265995+0000","last_undegraded":"2023-10-09T10:47:32.265995+0000","last_fullsized":"2023-10-09T10:47:32.265995+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":122,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.250426+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:47:32.250426+0000","last_peered":"2023-10-09T10:47:32.250426+0000","last_clean":"2023-10-09T10:47:32.250426+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:47:32.250426+0000","last_undegraded":"2023-10-09T10:47:32.250426+0000","last_fullsized":"2023-10-09T10:47:32.250426+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"175'5867","reported_seq":12671,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-10-09T10:47:40.941736+0000","last_change":"2023-10-09T10:47:31.271196+0000","last_active":"2023-10-09T10:47:40.941736+0000","last_peered":"2023-10-09T10:47:40.941736+0000","last_clean":"2023-10-09T10:47:40.941736+0000","last_became_active":"2023-10-09T10:47:31.229504+0000","last_became_peered":"2023-10-09T10:47:31.229504+0000","last_unstale":"2023-10-09T10:47:40.941736+0000","last_undegraded":"2023-10-09T10:47:40.941736+0000","last_fullsized":"2023-10-09T10:47:40.941736+0000","mapping_epoch":172,"log_start":"103'337","ondisk_log_start":"103'337","created":98,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":5530,"log_dups_size":0,"ondisk_log_size":5530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:26.533476+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115277824,"num_objects":1759,"num_object_clones":0,"num_object_copies":5277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3222,"num_write_kb":107264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3305,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":43,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.265997+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:47:32.265997+0000","last_peered":"2023-10-09T10:47:32.265997+0000","last_clean":"2023-10-09T10:47:32.265997+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:47:32.265997+0000","last_undegraded":"2023-10-09T10:47:32.265997+0000","last_fullsized":"2023-10-09T10:47:32.265997+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":106,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.260939+0000","last_change":"2023-10-09T10:46:32.251242+0000","last_active":"2023-10-09T10:47:32.260939+0000","last_peered":"2023-10-09T10:47:32.260939+0000","last_clean":"2023-10-09T10:47:32.260939+0000","last_became_active":"2023-10-09T10:46:32.250534+0000","last_became_peered":"2023-10-09T10:46:32.250534+0000","last_unstale":"2023-10-09T10:47:32.260939+0000","last_undegraded":"2023-10-09T10:47:32.260939+0000","last_fullsized":"2023-10-09T10:47:32.260939+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:33.726041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"175'5973","reported_seq":9952,"reported_epoch":175,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:42.541273+0000","last_change":"2023-10-09T10:46:34.514479+0000","last_active":"2023-10-09T10:47:42.541273+0000","last_peered":"2023-10-09T10:47:42.541273+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:46:34.409667+0000","last_became_peered":"2023-10-09T10:46:34.409667+0000","last_unstale":"2023-10-09T10:47:42.541273+0000","last_undegraded":"2023-10-09T10:46:34.376635+0000","last_fullsized":"2023-10-09T10:46:33.237998+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5973,"log_dups_size":0,"ondisk_log_size":5973,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114229248,"num_objects":1743,"num_object_clones":0,"num_object_copies":5229,"num_objects_missing_on_primary":1406,"num_objects_missing":1406,"num_objects_degraded":1916,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4931,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1932,"num_bytes_recovered":126615552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,7","objects":337},{"shards":"2,3,7","objects":1406}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":142,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.256411+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:47:32.256411+0000","last_peered":"2023-10-09T10:47:32.256411+0000","last_clean":"2023-10-09T10:47:32.256411+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:47:32.256411+0000","last_undegraded":"2023-10-09T10:47:32.256411+0000","last_fullsized":"2023-10-09T10:47:32.256411+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":156,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.251397+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:47:32.251397+0000","last_peered":"2023-10-09T10:47:32.251397+0000","last_clean":"2023-10-09T10:47:32.251397+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:47:32.251397+0000","last_undegraded":"2023-10-09T10:47:32.251397+0000","last_fullsized":"2023-10-09T10:47:32.251397+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"175'5948","reported_seq":6055,"reported_epoch":175,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:44.560303+0000","last_change":"2023-10-09T10:46:33.546991+0000","last_active":"2023-10-09T10:47:44.560303+0000","last_peered":"2023-10-09T10:47:44.560303+0000","last_clean":"2023-10-09T10:46:31.009484+0000","last_became_active":"2023-10-09T10:46:33.535665+0000","last_became_peered":"2023-10-09T10:46:33.535665+0000","last_unstale":"2023-10-09T10:47:44.560303+0000","last_undegraded":"2023-10-09T10:46:33.277168+0000","last_fullsized":"2023-10-09T10:46:33.235038+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5948,"log_dups_size":0,"ondisk_log_size":5948,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107708416,"num_objects":1644,"num_object_clones":0,"num_object_copies":4932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4884,"num_write_kb":208864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1656},{"shards":"3,7","objects":-1114}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":125,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.260502+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:47:32.260502+0000","last_peered":"2023-10-09T10:47:32.260502+0000","last_clean":"2023-10-09T10:47:32.260502+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:47:32.260502+0000","last_undegraded":"2023-10-09T10:47:32.260502+0000","last_fullsized":"2023-10-09T10:47:32.260502+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":23,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.255307+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:47:32.255307+0000","last_peered":"2023-10-09T10:47:32.255307+0000","last_clean":"2023-10-09T10:47:32.255307+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:47:32.255307+0000","last_undegraded":"2023-10-09T10:47:32.255307+0000","last_fullsized":"2023-10-09T10:47:32.255307+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":159,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:34.563622+0000","last_change":"2023-10-09T10:46:32.244607+0000","last_active":"2023-10-09T10:47:34.563622+0000","last_peered":"2023-10-09T10:47:34.563622+0000","last_clean":"2023-10-09T10:47:34.563622+0000","last_became_active":"2023-10-09T10:46:32.242628+0000","last_became_peered":"2023-10-09T10:46:32.242628+0000","last_unstale":"2023-10-09T10:47:34.563622+0000","last_undegraded":"2023-10-09T10:47:34.563622+0000","last_fullsized":"2023-10-09T10:47:34.563622+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:10.738819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":189,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.281101+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:47:32.281101+0000","last_peered":"2023-10-09T10:47:32.281101+0000","last_clean":"2023-10-09T10:47:32.281101+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:47:32.281101+0000","last_undegraded":"2023-10-09T10:47:32.281101+0000","last_fullsized":"2023-10-09T10:47:32.281101+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":64,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.266416+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:47:32.266416+0000","last_peered":"2023-10-09T10:47:32.266416+0000","last_clean":"2023-10-09T10:47:32.266416+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:47:32.266416+0000","last_undegraded":"2023-10-09T10:47:32.266416+0000","last_fullsized":"2023-10-09T10:47:32.266416+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":139,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T10:47:32.250426+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:47:32.250426+0000","last_peered":"2023-10-09T10:47:32.250426+0000","last_clean":"2023-10-09T10:47:32.250426+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:47:32.250426+0000","last_undegraded":"2023-10-09T10:47:32.250426+0000","last_fullsized":"2023-10-09T10:47:32.250426+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":3592650776,"num_objects":54821,"num_object_clones":0,"num_object_copies":164463,"num_objects_missing_on_primary":1406,"num_objects_missing":1406,"num_objects_degraded":31152,"num_objects_misplaced":6879,"num_objects_unfound":0,"num_objects_dirty":54821,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":149387,"num_write_kb":6258529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22233,"num_bytes_recovered":1364393984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12880850944,"data_stored":12880838728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":160892,"ondisk_log_size":160892,"up":78,"acting":67,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1186,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332253,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1882012,"kb_used_data":1584596,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91863140,"statfs":{"total":95995035648,"available":94067855360,"internally_reserved":0,"allocated":1622626304,"data_stored":1621960282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":0.79200000000000004,"15min":0.65000000000000002},"min":{"1min":0.26300000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.1479999999999997,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":2.988},{"interface":"front","average":{"1min":1.546,"5min":0.97199999999999998,"15min":0.752},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.9909999999999997,"5min":10.207000000000001,"15min":10.207000000000001},"last":5.0890000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:46:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":0.99399999999999999,"15min":0.69199999999999995},"min":{"1min":0.42199999999999999,"5min":0.318,"15min":0.318},"max":{"1min":8.7360000000000007,"5min":8.7360000000000007,"15min":8.7360000000000007},"last":2.3559999999999999},{"interface":"front","average":{"1min":1.2230000000000001,"5min":0.80500000000000005,"15min":0.623},"min":{"1min":0.27800000000000002,"5min":0.245,"15min":0.245},"max":{"1min":3.6080000000000001,"5min":6.4939999999999998,"15min":6.4939999999999998},"last":3.202}]},{"osd":2,"last update":"Mon Oct 9 10:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.1499999999999999,"15min":0.72399999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.4770000000000003,"5min":6.4770000000000003,"15min":6.4770000000000003},"last":3.181},{"interface":"front","average":{"1min":1.272,"5min":0.85699999999999998,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.251,"15min":0.251},"max":{"1min":3.1480000000000001,"5min":3.7120000000000002,"15min":3.7120000000000002},"last":3.238}]},{"osd":3,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.23,"15min":0.80000000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.0599999999999996,"5min":8.1639999999999997,"15min":8.1639999999999997},"last":2.3300000000000001},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.099,"15min":0.76600000000000001},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.504,"5min":12.451000000000001,"15min":12.451000000000001},"last":2.4489999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.73299999999999998,"15min":0.56200000000000006},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.1420000000000003,"5min":7.4509999999999996,"15min":7.4509999999999996},"last":2.294},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.65300000000000002,"15min":0.53600000000000003},"min":{"1min":0.29799999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.2749999999999999,"5min":3.1699999999999999,"15min":3.1699999999999999},"last":4.0810000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.56399999999999995,"15min":0.52700000000000002},"min":{"1min":0.34300000000000003,"5min":0.247,"15min":0.247},"max":{"1min":2.2349999999999999,"5min":2.2349999999999999,"15min":2.2349999999999999},"last":7.0069999999999997},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.81299999999999994,"15min":0.63},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":5.0860000000000003,"5min":10.33,"15min":10.33},"last":5.1289999999999996}]},{"osd":6,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.83399999999999996,"15min":0.64800000000000002},"min":{"1min":0.311,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.109,"5min":10.164999999999999,"15min":10.164999999999999},"last":5.1520000000000001},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.66000000000000003,"15min":0.58299999999999996},"min":{"1min":0.33300000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.306,"5min":2.8079999999999998,"15min":2.8079999999999998},"last":2.415}]}]},{"osd":6,"up_from":44,"seq":188978561184,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1564568,"kb_used_data":1250896,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":92180584,"statfs":{"total":95995035648,"available":94392918016,"internally_reserved":0,"allocated":1280917504,"data_stored":1280256020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.772,"5min":0.89500000000000002,"15min":0.70499999999999996},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.3220000000000001,"5min":7.3220000000000001,"15min":7.3220000000000001},"last":4.4630000000000001},{"interface":"front","average":{"1min":2.02,"5min":0.92000000000000004,"15min":0.71199999999999997},"min":{"1min":0.26700000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.5309999999999997,"5min":7.5309999999999997,"15min":7.5309999999999997},"last":4.4969999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:46:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8180000000000001,"5min":1.0680000000000001,"15min":0.75},"min":{"1min":0.29899999999999999,"5min":0.245,"15min":0.245},"max":{"1min":7.327,"5min":16.747,"15min":16.747},"last":6.7560000000000002},{"interface":"front","average":{"1min":1.883,"5min":1.101,"15min":0.73399999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.4260000000000002,"5min":16.734000000000002,"15min":16.734000000000002},"last":1.8560000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":0.95799999999999996,"15min":0.67500000000000004},"min":{"1min":0.28699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5069999999999997,"5min":7.5069999999999997,"15min":7.5069999999999997},"last":2.7970000000000002},{"interface":"front","average":{"1min":1.675,"5min":1.143,"15min":0.755},"min":{"1min":0.28599999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.3899999999999997,"5min":12.571,"15min":12.571},"last":1.0960000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":0.89600000000000002,"15min":0.65700000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.8799999999999999,"5min":4.7030000000000003,"15min":4.7030000000000003},"last":6.8070000000000004},{"interface":"front","average":{"1min":2.4399999999999999,"5min":1.0700000000000001,"15min":0.72099999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.4770000000000003,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":0.84999999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":0.66600000000000004,"15min":0.51500000000000001},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":4.0739999999999998,"5min":4.0739999999999998,"15min":4.0739999999999998},"last":1.0620000000000001},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.67300000000000004,"15min":0.54600000000000004},"min":{"1min":0.34899999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6600000000000001,"5min":3.6600000000000001,"15min":3.6600000000000001},"last":5.1260000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":0.76000000000000001,"15min":0.56999999999999995},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.8949999999999996,"5min":4.8949999999999996,"15min":4.8949999999999996},"last":1.26},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.76900000000000002,"15min":0.59999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":4.3639999999999999,"5min":4.3639999999999999,"15min":4.3639999999999999},"last":6.8419999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:47:25 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69899999999999995,"15min":0.57999999999999996},"min":{"1min":0.35799999999999998,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":2.5049999999999999,"5min":3.472,"15min":3.472},"last":8.1470000000000002},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.84299999999999997,"15min":0.65700000000000003},"min":{"1min":0.373,"5min":0.36899999999999999,"15min":0.34000000000000002},"max":{"1min":4.0800000000000001,"5min":4.0800000000000001,"15min":4.0800000000000001},"last":6.7859999999999996}]}]},{"osd":1,"up_from":85,"seq":365072220223,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1595140,"kb_used_data":1465020,"kb_used_omap":9,"kb_used_meta":129974,"kb_avail":92150012,"statfs":{"total":95995035648,"available":94361612288,"internally_reserved":0,"allocated":1500180480,"data_stored":1499306678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":133094377},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.3160000000000001,"15min":0.68100000000000005},"min":{"1min":0.255,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":8.3759999999999994,"5min":14.1,"15min":14.1},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.5499999999999998,"5min":1.266,"15min":0.68799999999999994},"min":{"1min":0.27300000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":13.462999999999999,"5min":14.465,"15min":14.465},"last":4.6299999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:47:13 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.1200000000000001,"15min":0.61799999999999999},"min":{"1min":0.21199999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.4829999999999997,"5min":8.6859999999999999,"15min":8.6859999999999999},"last":4.8220000000000001},{"interface":"front","average":{"1min":1.919,"5min":1.4690000000000001,"15min":0.746},"min":{"1min":0.29099999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.2829999999999995,"5min":14.234,"15min":14.234},"last":0.35099999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:46:39 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.2,"15min":0.71399999999999997},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.3539999999999992,"5min":14.404999999999999,"15min":14.404999999999999},"last":4.7400000000000002},{"interface":"front","average":{"1min":2.077,"5min":1.1080000000000001,"15min":0.59899999999999998},"min":{"1min":0.255,"5min":0.251,"15min":0.251},"max":{"1min":9.6820000000000004,"5min":9.6820000000000004,"15min":9.6820000000000004},"last":0.65500000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.0880000000000001,"15min":0.75800000000000001},"min":{"1min":0.29099999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.1420000000000003,"5min":8.0129999999999999,"15min":8.0129999999999999},"last":4.7709999999999999},{"interface":"front","average":{"1min":1.4850000000000001,"5min":1.0249999999999999,"15min":0.755},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":8.327,"5min":8.327,"15min":8.327},"last":0.38100000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.571,"5min":0.95699999999999996,"15min":0.73699999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.7720000000000002,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":0.871},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.1359999999999999,"15min":0.81000000000000005},"min":{"1min":0.32700000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.1799999999999997,"5min":7.923,"15min":7.923},"last":4.7990000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":0.97199999999999998,"15min":0.745},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.7379999999999995,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":0.746},{"interface":"front","average":{"1min":1.617,"5min":1.175,"15min":0.82299999999999995},"min":{"1min":0.39500000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.2069999999999999,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":4.8440000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6699999999999999,"5min":1.155,"15min":0.81799999999999995},"min":{"1min":0.41599999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":5.2309999999999999,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":4.8689999999999998},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.022,"15min":0.77100000000000002},"min":{"1min":0.42499999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":4.6399999999999997,"5min":8.2469999999999999,"15min":8.2469999999999999},"last":4.7140000000000004}]}]},{"osd":0,"up_from":80,"seq":343597383757,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":708384,"kb_used_data":610328,"kb_used_omap":28,"kb_used_meta":98019,"kb_avail":93036768,"statfs":{"total":95995035648,"available":95269650432,"internally_reserved":0,"allocated":624975872,"data_stored":624073254,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":100372304},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:46:43 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.1679999999999999,"15min":0.63400000000000001},"min":{"1min":0.217,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.0449999999999999,"5min":25.475000000000001,"15min":25.475000000000001},"last":2.2200000000000002},{"interface":"front","average":{"1min":2.0099999999999998,"5min":1.1919999999999999,"15min":0.64000000000000001},"min":{"1min":0.23499999999999999,"5min":0.191,"15min":0.191},"max":{"1min":6.2290000000000001,"5min":20.832999999999998,"15min":20.832999999999998},"last":0.249}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":1.2350000000000001,"15min":0.64400000000000002},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.907,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":2.3639999999999999},{"interface":"front","average":{"1min":1.754,"5min":1.5349999999999999,"15min":0.73999999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.5619999999999994,"5min":14.391999999999999,"15min":14.391999999999999},"last":0.37}]},{"osd":3,"last update":"Mon Oct 9 10:46:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9689999999999999,"5min":1.5309999999999999,"15min":0.94399999999999995},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":9.3889999999999993,"5min":14.926,"15min":14.926},"last":4.5369999999999999},{"interface":"front","average":{"1min":2.02,"5min":1.2569999999999999,"15min":0.78000000000000003},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":11.868,"5min":11.868,"15min":11.868},"last":2.399}]},{"osd":4,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.2250000000000001,"15min":0.82199999999999995},"min":{"1min":0.38400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.8150000000000004,"5min":6.8150000000000004,"15min":6.8150000000000004},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.391,"5min":1.405,"15min":0.88100000000000001},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.676,"5min":10.676,"15min":10.676},"last":0.45100000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":1.4359999999999999,"15min":0.89600000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":10.734,"5min":10.734,"15min":10.734},"last":4.5830000000000002},{"interface":"front","average":{"1min":1.716,"5min":1.1659999999999999,"15min":0.82099999999999995},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.54,"5min":8.9190000000000005,"15min":8.9190000000000005},"last":2.2650000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.391,"15min":0.88400000000000001},"min":{"1min":0.36599999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.706,"5min":10.706,"15min":10.706},"last":4.6150000000000002},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.0780000000000001,"15min":0.79300000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.0499999999999998,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":2.2959999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.474,"5min":1.151,"15min":0.81899999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.5380000000000003,"5min":8.875,"15min":8.875},"last":2.3260000000000001},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.425,"15min":0.92700000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":14.394,"5min":14.394,"15min":14.394},"last":6.2910000000000004}]}]},{"osd":2,"up_from":0,"seq":386547056698,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893172,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2502348,"kb_used_data":2308292,"kb_used_omap":7,"kb_used_meta":193976,"kb_avail":91242804,"statfs":{"total":95995035648,"available":93432631296,"internally_reserved":0,"allocated":2363691008,"data_stored":2362806825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":198632040},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":1.3140000000000001,"15min":0.65200000000000002},"min":{"1min":0.192,"5min":0.183,"15min":0.183},"max":{"1min":7.0289999999999999,"5min":11.557,"15min":11.557},"last":7.0289999999999999},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.1379999999999999,"15min":0.65000000000000002},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":6.681,"5min":7.0510000000000002,"15min":7.0510000000000002},"last":6.681}]},{"osd":1,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":1.6970000000000001,"15min":0.78500000000000003},"min":{"1min":0.27100000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.048,"5min":14.707000000000001,"15min":14.707000000000001},"last":2.9849999999999999},{"interface":"front","average":{"1min":1.6220000000000001,"5min":1.1890000000000001,"15min":0.64500000000000002},"min":{"1min":0.19400000000000001,"5min":0.193,"15min":0.193},"max":{"1min":7.1630000000000003,"5min":11.629,"15min":11.629},"last":7.1630000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.6739999999999999,"15min":0.80200000000000005},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.9180000000000001,"5min":13.853,"15min":13.853},"last":3.0070000000000001},{"interface":"front","average":{"1min":2.5590000000000002,"5min":1.5129999999999999,"15min":0.76800000000000002},"min":{"1min":0.30099999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.71,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":6.71}]},{"osd":4,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.2210000000000001,"15min":0.78800000000000003},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.9909999999999997,"5min":11.532,"15min":11.532},"last":6.9909999999999997},{"interface":"front","average":{"1min":2.2440000000000002,"5min":1.296,"15min":0.83699999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.962999999999999,"5min":11.962999999999999,"15min":11.962999999999999},"last":6.6369999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7589999999999999,"5min":1.8029999999999999,"15min":0.99199999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.907,"5min":51.509999999999998,"15min":51.509999999999998},"last":6.7389999999999999},{"interface":"front","average":{"1min":2.1469999999999998,"5min":2.306,"15min":1.1439999999999999},"min":{"1min":0.36799999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.1139999999999999,"5min":55.338999999999999,"15min":55.338999999999999},"last":6.1139999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":1.3620000000000001,"15min":0.84599999999999997},"min":{"1min":0.378,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":11.92,"5min":11.92,"15min":11.92},"last":11.505000000000001},{"interface":"front","average":{"1min":2.3759999999999999,"5min":1.776,"15min":0.99199999999999999},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":10.528,"5min":13.891999999999999,"15min":13.891999999999999},"last":2.706}]},{"osd":7,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":2.3410000000000002,"15min":1.175},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.6520000000000001,"5min":51.744,"15min":51.744},"last":2.6720000000000002},{"interface":"front","average":{"1min":1.9139999999999999,"5min":1.869,"15min":1.0049999999999999},"min":{"1min":0.33000000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.133,"5min":53.399000000000001,"15min":53.399000000000001},"last":7.133}]}]},{"osd":4,"up_from":30,"seq":128849019047,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1439512,"kb_used_data":1138064,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92305640,"statfs":{"total":95995035648,"available":94520975360,"internally_reserved":0,"allocated":1165377536,"data_stored":1164715319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.0129999999999999,"15min":0.70899999999999996},"min":{"1min":0.24099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.5419999999999998,"5min":10.387,"15min":10.387},"last":3.4020000000000001},{"interface":"front","average":{"1min":1.581,"5min":1.1819999999999999,"15min":0.78400000000000003},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.5999999999999996,"5min":10.311999999999999,"15min":10.311999999999999},"last":3.1800000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:46:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":0.98599999999999999,"15min":0.69699999999999995},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.4189999999999996,"5min":6.907,"15min":6.907},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.627,"5min":0.99099999999999999,"15min":0.745},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.1790000000000003,"5min":6.1790000000000003,"15min":6.1790000000000003},"last":2.069}]},{"osd":2,"last update":"Mon Oct 9 10:47:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2170000000000001,"5min":1.1000000000000001,"15min":0.747},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.2149999999999999,"5min":7.5449999999999999,"15min":7.5449999999999999},"last":2.9750000000000001},{"interface":"front","average":{"1min":1.579,"5min":1.1639999999999999,"15min":0.76300000000000001},"min":{"1min":0.36699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.5110000000000001,"5min":6.2939999999999996,"15min":6.2939999999999996},"last":2.9529999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:47:38 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.333,"15min":0.91900000000000004},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.8170000000000002,"5min":8.8170000000000002,"15min":8.8170000000000002},"last":3.7490000000000001},{"interface":"front","average":{"1min":2.3650000000000002,"5min":1.359,"15min":0.84499999999999997},"min":{"1min":0.34300000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.625,"5min":8.9809999999999999,"15min":8.9809999999999999},"last":3.4289999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:46:41 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.73099999999999998,"15min":0.53500000000000003},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":3.9319999999999999,"5min":5.5410000000000004,"15min":5.5410000000000004},"last":2.6699999999999999},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.71099999999999997,"15min":0.52900000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":3.8980000000000001,"5min":3.8980000000000001,"15min":3.8980000000000001},"last":3.7759999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.74099999999999999,"15min":0.55700000000000005},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":7.1580000000000004,"5min":7.1580000000000004,"15min":7.1580000000000004},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.71599999999999997,"15min":0.60899999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":3.6150000000000002,"5min":3.6150000000000002,"15min":3.6150000000000002},"last":2.637}]},{"osd":7,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.80800000000000005,"15min":0.64600000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.1050000000000004,"5min":4.1050000000000004,"15min":4.1050000000000004},"last":3.9159999999999999},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.748,"15min":0.60799999999999998},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0919999999999996,"5min":4.0919999999999996,"15min":4.0919999999999996},"last":6.0499999999999998}]}]},{"osd":5,"up_from":37,"seq":158913790116,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1769224,"kb_used_data":1467840,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91975928,"statfs":{"total":95995035648,"available":94183350272,"internally_reserved":0,"allocated":1503068160,"data_stored":1502417772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.77600000000000002,"15min":0.65700000000000003},"min":{"1min":0.247,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.9860000000000002,"5min":4.8380000000000001,"15min":4.8380000000000001},"last":1.222},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.84899999999999998,"15min":0.72899999999999998},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.0209999999999999,"5min":4.8650000000000002,"15min":4.8650000000000002},"last":1.244}]},{"osd":1,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":0.85599999999999998,"15min":0.68999999999999995},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.7789999999999999,"5min":3.7789999999999999,"15min":3.7789999999999999},"last":4.9550000000000001},{"interface":"front","average":{"1min":1.1200000000000001,"5min":0.93100000000000005,"15min":0.70499999999999996},"min":{"1min":0.26200000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.77,"5min":7.165,"15min":7.165},"last":3.7429999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:47:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.02,"15min":0.72099999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.4569999999999999,"5min":7.9000000000000004,"15min":7.9000000000000004},"last":5.0140000000000002},{"interface":"front","average":{"1min":1.7569999999999999,"5min":0.90400000000000003,"15min":0.68400000000000005},"min":{"1min":0.38,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":10.352,"5min":10.352,"15min":10.352},"last":2.661}]},{"osd":3,"last update":"Mon Oct 9 10:46:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5960000000000001,"5min":1.266,"15min":0.90400000000000003},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":9.984,"5min":9.984,"15min":9.984},"last":9.1069999999999993},{"interface":"front","average":{"1min":2.6509999999999998,"5min":1.2030000000000001,"15min":0.80900000000000005},"min":{"1min":0.34999999999999998,"5min":0.317,"15min":0.317},"max":{"1min":11.657,"5min":11.657,"15min":11.657},"last":9.1370000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:46:46 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.53200000000000003,"15min":0.46700000000000003},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":3.7919999999999998,"5min":3.7919999999999998,"15min":3.7919999999999998},"last":19.152999999999999},{"interface":"front","average":{"1min":1.048,"5min":0.68500000000000005,"15min":0.52900000000000003},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.9569999999999999,"5min":4.9569999999999999,"15min":4.9569999999999999},"last":6.9909999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.66300000000000003,"15min":0.53500000000000003},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":2.8900000000000001,"5min":2.9529999999999998,"15min":2.9529999999999998},"last":4.8940000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.63300000000000001,"15min":0.53800000000000003},"min":{"1min":0.34100000000000003,"5min":0.33300000000000002,"15min":0.30299999999999999},"max":{"1min":3.8159999999999998,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":0.97799999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:47:22 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.67100000000000004,"15min":0.58799999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.4729999999999999,"5min":4.0510000000000002,"15min":4.0510000000000002},"last":4.9930000000000003},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.70899999999999996,"15min":0.60099999999999998},"min":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":4.1289999999999996,"5min":4.1289999999999996,"15min":4.1289999999999996},"last":1.0049999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":616960000,"data_stored":616955928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1482756096,"data_stored":1482752024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2898395136,"data_stored":2898395136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2361327616,"data_stored":2361327616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1149239296,"data_stored":1149239296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1495732224,"data_stored":1495728152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1265696768,"data_stored":1265696768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1610743808,"data_stored":1610743808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:47:46.816 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:47:46.846 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:46.941 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:46.996 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:47.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:47.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:47.095 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:47:47.106 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:47.160 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:47.209 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:47.245 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:47.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:47.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:47.256 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:47:47.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:47.411 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:47 smithi125 ceph-mon[195507]: from='client.34995 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:47:47.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:47 smithi125 ceph-mon[200197]: from='client.34995 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:47:47.438 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:47.466 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:47.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:47.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:47.572 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:47.619 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:47.665 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:47 smithi155 ceph-mon[184854]: from='client.34995 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:47:47.670 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:47:48.245 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:47:48.245 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:47:48.245 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:47:48.245 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:47:48.245 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.245 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70217 6.d 6:b8b4c26d:::benchmark_data_smithi125_244951_object70216:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:47:48.246 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.226550+0000", 2023-10-09T10:47:48.246 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 145.015580911, 2023-10-09T10:47:48.246 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.064685005000000004, 2023-10-09T10:47:48.246 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.246 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.246 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.246 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.246 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.246 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70217 2023-10-09T10:47:48.246 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.246 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.247 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.247 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.247 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226550+0000", 2023-10-09T10:47:48.247 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.247 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.247 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.247 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.247 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226550+0000", 2023-10-09T10:47:48.247 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.247 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.247 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.248 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.248 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226550+0000", 2023-10-09T10:47:48.248 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:48.248 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.248 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.248 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.248 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226583+0000", 2023-10-09T10:47:48.248 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2808000000000002e-05 2023-10-09T10:47:48.248 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.248 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.248 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.248 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226590+0000", 2023-10-09T10:47:48.249 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8509999999999999e-06 2023-10-09T10:47:48.249 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.249 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.249 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.249 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226595+0000", 2023-10-09T10:47:48.249 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6290000000000001e-06 2023-10-09T10:47:48.249 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.249 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.249 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.249 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226637+0000", 2023-10-09T10:47:48.249 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1734999999999999e-05 2023-10-09T10:47:48.249 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.250 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.250 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.250 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226771+0000", 2023-10-09T10:47:48.250 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013447000000000001 2023-10-09T10:47:48.250 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.250 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.250 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:47:48.250 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226923+0000", 2023-10-09T10:47:48.250 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151977 2023-10-09T10:47:48.250 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.250 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.250 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.251 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.229528+0000", 2023-10-09T10:47:48.251 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026052530000000001 2023-10-09T10:47:48.251 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.251 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.251 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.251 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.241565+0000", 2023-10-09T10:47:48.251 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012036467 2023-10-09T10:47:48.251 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.251 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.251 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.251 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.288660+0000", 2023-10-09T10:47:48.251 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047095342999999998 2023-10-09T10:47:48.252 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.252 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.252 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.252 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.288720+0000", 2023-10-09T10:47:48.252 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0025000000000002e-05 2023-10-09T10:47:48.252 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.252 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.252 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.252 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.291235+0000", 2023-10-09T10:47:48.252 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0025149249999999999 2023-10-09T10:47:48.252 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.252 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.253 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.253 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.253 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.253 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70220 6.d 6:bb7bde6c:::benchmark_data_smithi125_244951_object70219:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:47:48.253 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.231764+0000", 2023-10-09T10:47:48.253 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 145.01036658800001, 2023-10-09T10:47:48.253 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.061391124999999998, 2023-10-09T10:47:48.253 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.253 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.253 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.253 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.254 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.254 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70220 2023-10-09T10:47:48.254 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.254 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.254 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.254 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.254 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231764+0000", 2023-10-09T10:47:48.254 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.254 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.254 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.254 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.254 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231764+0000", 2023-10-09T10:47:48.255 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.255 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.255 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.255 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.255 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231763+0000", 2023-10-09T10:47:48.255 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:47:48.255 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.255 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.255 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.255 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231801+0000", 2023-10-09T10:47:48.255 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7691999999999998e-05 2023-10-09T10:47:48.256 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.256 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.256 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.256 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231812+0000", 2023-10-09T10:47:48.256 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0569e-05 2023-10-09T10:47:48.256 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.256 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.256 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.256 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231817+0000", 2023-10-09T10:47:48.256 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0490000000000003e-06 2023-10-09T10:47:48.256 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.256 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.257 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.257 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231856+0000", 2023-10-09T10:47:48.257 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9002999999999999e-05 2023-10-09T10:47:48.257 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.257 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.257 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.257 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231960+0000", 2023-10-09T10:47:48.257 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010419100000000001 2023-10-09T10:47:48.257 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.257 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.257 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:47:48.257 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.232064+0000", 2023-10-09T10:47:48.258 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010409 2023-10-09T10:47:48.258 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.258 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.258 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.258 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.234485+0000", 2023-10-09T10:47:48.258 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024210680000000002 2023-10-09T10:47:48.258 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.258 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.258 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.258 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.251746+0000", 2023-10-09T10:47:48.258 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017260707 2023-10-09T10:47:48.258 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.259 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.259 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.259 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.291288+0000", 2023-10-09T10:47:48.259 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039542425999999999 2023-10-09T10:47:48.259 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.259 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.259 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.259 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.291380+0000", 2023-10-09T10:47:48.259 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1514999999999994e-05 2023-10-09T10:47:48.259 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.259 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.259 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.259 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.293156+0000", 2023-10-09T10:47:48.260 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001775878 2023-10-09T10:47:48.260 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.260 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.260 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.260 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.260 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.260 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70235 6.d 6:be185add:::benchmark_data_smithi125_244951_object70234:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:47:48.260 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.257199+0000", 2023-10-09T10:47:48.260 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 144.98493251599999, 2023-10-09T10:47:48.260 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.057403978000000001, 2023-10-09T10:47:48.260 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.260 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.261 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.261 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.261 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.261 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70235 2023-10-09T10:47:48.261 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.261 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.261 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.261 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.261 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.257199+0000", 2023-10-09T10:47:48.261 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.261 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.261 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.262 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.262 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.257199+0000", 2023-10-09T10:47:48.262 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.262 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.262 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.262 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.262 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.257198+0000", 2023-10-09T10:47:48.262 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:47:48.262 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.262 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.262 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.263 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.257262+0000", 2023-10-09T10:47:48.263 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3872000000000001e-05 2023-10-09T10:47:48.263 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.263 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.263 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.263 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.257272+0000", 2023-10-09T10:47:48.263 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0726e-05 2023-10-09T10:47:48.263 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.263 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.263 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.263 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.257279+0000", 2023-10-09T10:47:48.264 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9160000000000004e-06 2023-10-09T10:47:48.264 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.264 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.264 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.264 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.259884+0000", 2023-10-09T10:47:48.264 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026049649999999999 2023-10-09T10:47:48.264 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.264 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.264 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.264 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.259970+0000", 2023-10-09T10:47:48.264 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6046999999999998e-05 2023-10-09T10:47:48.264 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.265 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.265 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:47:48.265 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.260071+0000", 2023-10-09T10:47:48.265 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010060099999999999 2023-10-09T10:47:48.265 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.265 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.265 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.265 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.262039+0000", 2023-10-09T10:47:48.265 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019683690000000002 2023-10-09T10:47:48.265 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.265 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.265 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.265 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.268541+0000", 2023-10-09T10:47:48.266 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0065019040000000002 2023-10-09T10:47:48.266 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.266 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.266 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.266 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.312244+0000", 2023-10-09T10:47:48.266 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.04370309 2023-10-09T10:47:48.266 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.266 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.266 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.266 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.312291+0000", 2023-10-09T10:47:48.267 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6499999999999999e-05 2023-10-09T10:47:48.267 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.267 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.267 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.267 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.314603+0000", 2023-10-09T10:47:48.267 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023117799999999998 2023-10-09T10:47:48.267 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.267 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.267 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.267 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.267 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.267 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70535 6.a 6:5d73a126:::benchmark_data_smithi125_244951_object70534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:47:48.268 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.725476+0000", 2023-10-09T10:47:48.268 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 144.51665557000001, 2023-10-09T10:47:48.268 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.053976932999999998, 2023-10-09T10:47:48.268 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.268 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.268 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.268 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.268 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.268 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70535 2023-10-09T10:47:48.268 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.268 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.268 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.725476+0000", 2023-10-09T10:47:48.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.269 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.269 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.725476+0000", 2023-10-09T10:47:48.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.269 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.269 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.725475+0000", 2023-10-09T10:47:48.270 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:48.270 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.270 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.270 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.270 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.725509+0000", 2023-10-09T10:47:48.270 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3624000000000002e-05 2023-10-09T10:47:48.270 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.270 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.270 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.270 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.725516+0000", 2023-10-09T10:47:48.270 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6129999999999998e-06 2023-10-09T10:47:48.270 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.271 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.271 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.271 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.725521+0000", 2023-10-09T10:47:48.271 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8890000000000001e-06 2023-10-09T10:47:48.271 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.271 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.271 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.271 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.727787+0000", 2023-10-09T10:47:48.271 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022656830000000001 2023-10-09T10:47:48.271 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.271 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.272 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.272 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.727937+0000", 2023-10-09T10:47:48.272 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015070099999999999 2023-10-09T10:47:48.272 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.272 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.272 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:47:48.272 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.728111+0000", 2023-10-09T10:47:48.272 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017334699999999999 2023-10-09T10:47:48.272 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.272 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.272 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.272 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.730067+0000", 2023-10-09T10:47:48.272 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019566200000000001 2023-10-09T10:47:48.273 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.273 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.273 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.273 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.730086+0000", 2023-10-09T10:47:48.273 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8334000000000001e-05 2023-10-09T10:47:48.273 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.273 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.273 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.273 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.778216+0000", 2023-10-09T10:47:48.273 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.048130615000000002 2023-10-09T10:47:48.273 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.274 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.274 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.274 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.778254+0000", 2023-10-09T10:47:48.274 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8061e-05 2023-10-09T10:47:48.274 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.274 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.274 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.274 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.779452+0000", 2023-10-09T10:47:48.274 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001198057 2023-10-09T10:47:48.274 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.274 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.274 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.275 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.275 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.275 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70665 6.a 6:5d5b51c9:::benchmark_data_smithi125_244951_object70664:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:47:48.275 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.914403+0000", 2023-10-09T10:47:48.275 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 144.327728114, 2023-10-09T10:47:48.275 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.055020996000000003, 2023-10-09T10:47:48.275 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.275 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.275 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.275 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.275 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.275 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70665 2023-10-09T10:47:48.276 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.276 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.276 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.276 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.276 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.914403+0000", 2023-10-09T10:47:48.276 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.276 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.276 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.276 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.276 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.914403+0000", 2023-10-09T10:47:48.276 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.276 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.276 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.277 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.277 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.914402+0000", 2023-10-09T10:47:48.277 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:48.277 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.277 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.277 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.914456+0000", 2023-10-09T10:47:48.277 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3977999999999997e-05 2023-10-09T10:47:48.277 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.277 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.277 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.914472+0000", 2023-10-09T10:47:48.277 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6716999999999999e-05 2023-10-09T10:47:48.278 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.278 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.278 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.278 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.914479+0000", 2023-10-09T10:47:48.278 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7240000000000003e-06 2023-10-09T10:47:48.278 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.278 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.278 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.278 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.916008+0000", 2023-10-09T10:47:48.278 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015288770000000001 2023-10-09T10:47:48.278 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.278 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.279 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.279 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.916145+0000", 2023-10-09T10:47:48.279 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013761399999999999 2023-10-09T10:47:48.279 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.279 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.279 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:47:48.279 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.916308+0000", 2023-10-09T10:47:48.279 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016281100000000001 2023-10-09T10:47:48.279 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.279 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.279 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.279 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.920537+0000", 2023-10-09T10:47:48.280 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042282409999999998 2023-10-09T10:47:48.280 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.280 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.280 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.280 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.920796+0000", 2023-10-09T10:47:48.280 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025962100000000003 2023-10-09T10:47:48.280 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.280 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.280 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.280 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.967578+0000", 2023-10-09T10:47:48.280 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.046781769000000001 2023-10-09T10:47:48.280 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.280 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.281 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.281 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.967634+0000", 2023-10-09T10:47:48.281 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6530000000000003e-05 2023-10-09T10:47:48.281 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.281 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.281 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.281 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.969424+0000", 2023-10-09T10:47:48.281 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001789524 2023-10-09T10:47:48.281 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.281 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.281 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.281 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.282 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.282 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106038 6.d 6:b7509843:::benchmark_data_smithi125_244951_object106037:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:47:48.282 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:01.817535+0000", 2023-10-09T10:47:48.282 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 106.424595907, 2023-10-09T10:47:48.282 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.056422880000000002, 2023-10-09T10:47:48.282 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.282 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.282 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.282 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.282 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.282 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106038 2023-10-09T10:47:48.282 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.282 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.283 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.817535+0000", 2023-10-09T10:47:48.283 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.283 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.817535+0000", 2023-10-09T10:47:48.283 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.817534+0000", 2023-10-09T10:47:48.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:47:48.284 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.284 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.817588+0000", 2023-10-09T10:47:48.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4048000000000002e-05 2023-10-09T10:47:48.284 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.284 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.817600+0000", 2023-10-09T10:47:48.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1902000000000001e-05 2023-10-09T10:47:48.284 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.285 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.285 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.817612+0000", 2023-10-09T10:47:48.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2004000000000001e-05 2023-10-09T10:47:48.285 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.285 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.285 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.821544+0000", 2023-10-09T10:47:48.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039317329999999998 2023-10-09T10:47:48.285 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.285 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.286 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.821713+0000", 2023-10-09T10:47:48.286 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016872400000000001 2023-10-09T10:47:48.286 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.286 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.286 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:47:48.286 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.821895+0000", 2023-10-09T10:47:48.286 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018211799999999999 2023-10-09T10:47:48.286 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.286 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.286 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.286 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.827739+0000", 2023-10-09T10:47:48.286 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0058439549999999996 2023-10-09T10:47:48.287 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.287 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.287 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.287 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.845999+0000", 2023-10-09T10:47:48.287 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018260456000000001 2023-10-09T10:47:48.287 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.287 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.287 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.287 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.871528+0000", 2023-10-09T10:47:48.287 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.02552828 2023-10-09T10:47:48.287 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.287 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.288 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.288 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.871587+0000", 2023-10-09T10:47:48.288 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9145999999999999e-05 2023-10-09T10:47:48.288 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.288 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.288 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.288 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:01.873958+0000", 2023-10-09T10:47:48.288 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023713559999999998 2023-10-09T10:47:48.288 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.288 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.288 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.289 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.289 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.289 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106432 6.a 6:50fa9909:::benchmark_data_smithi125_244951_object106431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:47:48.289 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.249049+0000", 2023-10-09T10:47:48.289 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 105.993082288, 2023-10-09T10:47:48.289 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.49699488600000002, 2023-10-09T10:47:48.289 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.289 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T10:47:48.289 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.289 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.289 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.289 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106432 2023-10-09T10:47:48.290 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.290 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.290 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.290 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.290 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249049+0000", 2023-10-09T10:47:48.290 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.290 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.290 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.290 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.290 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249049+0000", 2023-10-09T10:47:48.290 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.291 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.291 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.291 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.291 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249047+0000", 2023-10-09T10:47:48.291 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:47:48.291 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.291 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.291 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.291 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249097+0000", 2023-10-09T10:47:48.291 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0584000000000002e-05 2023-10-09T10:47:48.291 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.291 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.292 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.292 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249117+0000", 2023-10-09T10:47:48.292 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9763999999999999e-05 2023-10-09T10:47:48.292 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.292 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.292 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.292 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249125+0000", 2023-10-09T10:47:48.292 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6040000000000004e-06 2023-10-09T10:47:48.292 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.292 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.292 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.292 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249189+0000", 2023-10-09T10:47:48.293 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3779e-05 2023-10-09T10:47:48.293 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.293 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.293 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.293 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249335+0000", 2023-10-09T10:47:48.293 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014603399999999999 2023-10-09T10:47:48.293 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.293 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.293 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:47:48.293 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249521+0000", 2023-10-09T10:47:48.293 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018619099999999999 2023-10-09T10:47:48.293 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.293 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.294 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.294 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.253194+0000", 2023-10-09T10:47:48.294 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003673103 2023-10-09T10:47:48.294 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.294 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.294 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.294 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.253298+0000", 2023-10-09T10:47:48.294 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103828 2023-10-09T10:47:48.294 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.294 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.294 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.294 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.271832+0000", 2023-10-09T10:47:48.295 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018534658999999998 2023-10-09T10:47:48.295 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.295 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.295 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:47:48.295 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.271843+0000", 2023-10-09T10:47:48.295 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0995000000000001e-05 2023-10-09T10:47:48.295 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.295 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.295 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.295 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.745982+0000", 2023-10-09T10:47:48.295 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.47413843 2023-10-09T10:47:48.295 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.296 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.296 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.296 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746044+0000", 2023-10-09T10:47:48.296 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1832e-05 2023-10-09T10:47:48.296 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.296 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.296 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.296 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.296 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.296 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106439 6.a 6:5c3c51c3:::benchmark_data_smithi125_244951_object106438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-10-09T10:47:48.296 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.284123+0000", 2023-10-09T10:47:48.296 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 105.958008083, 2023-10-09T10:47:48.296 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.47113701699999999, 2023-10-09T10:47:48.297 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.297 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.297 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.297 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.297 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.297 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106439 2023-10-09T10:47:48.297 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.297 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.297 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.297 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.297 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284123+0000", 2023-10-09T10:47:48.298 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.298 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.298 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.298 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.298 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284123+0000", 2023-10-09T10:47:48.298 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.298 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.298 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.298 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.298 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284122+0000", 2023-10-09T10:47:48.298 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:48.298 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.299 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.299 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284170+0000", 2023-10-09T10:47:48.299 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7929999999999997e-05 2023-10-09T10:47:48.299 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.299 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.299 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284181+0000", 2023-10-09T10:47:48.299 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1292e-05 2023-10-09T10:47:48.299 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.299 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.299 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284186+0000", 2023-10-09T10:47:48.300 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1089999999999997e-06 2023-10-09T10:47:48.300 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.300 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.300 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.300 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284266+0000", 2023-10-09T10:47:48.300 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0311999999999999e-05 2023-10-09T10:47:48.300 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.300 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.300 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:47:48.300 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284287+0000", 2023-10-09T10:47:48.300 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0704e-05 2023-10-09T10:47:48.300 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.301 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.301 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.301 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746068+0000", 2023-10-09T10:47:48.301 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.46178057300000003 2023-10-09T10:47:48.301 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.301 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.301 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.301 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746295+0000", 2023-10-09T10:47:48.301 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000227335 2023-10-09T10:47:48.301 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.301 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.302 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:47:48.302 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746417+0000", 2023-10-09T10:47:48.302 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000121816 2023-10-09T10:47:48.302 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.302 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.302 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.302 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.752322+0000", 2023-10-09T10:47:48.302 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059055230000000002 2023-10-09T10:47:48.302 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.302 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.302 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.302 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754107+0000", 2023-10-09T10:47:48.303 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0017852149999999999 2023-10-09T10:47:48.303 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.303 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.303 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.303 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754378+0000", 2023-10-09T10:47:48.303 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000270547 2023-10-09T10:47:48.303 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.303 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.303 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.303 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754422+0000", 2023-10-09T10:47:48.303 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4063000000000001e-05 2023-10-09T10:47:48.303 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.303 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.304 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.304 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.755260+0000", 2023-10-09T10:47:48.304 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00083801000000000004 2023-10-09T10:47:48.304 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.304 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.304 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.304 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.304 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.304 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106440 6.a 6:5e7f7caf:::benchmark_data_smithi125_244951_object106439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-10-09T10:47:48.304 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.284197+0000", 2023-10-09T10:47:48.304 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 105.95793455499999, 2023-10-09T10:47:48.304 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.47186083699999998, 2023-10-09T10:47:48.304 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.305 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.305 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.305 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.305 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.305 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106440 2023-10-09T10:47:48.305 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.305 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.305 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.305 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.305 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284197+0000", 2023-10-09T10:47:48.305 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.306 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.306 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.306 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.306 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284197+0000", 2023-10-09T10:47:48.306 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.306 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.306 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.306 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.306 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284196+0000", 2023-10-09T10:47:48.306 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:48.306 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.306 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.307 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.307 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284284+0000", 2023-10-09T10:47:48.307 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7646999999999996e-05 2023-10-09T10:47:48.307 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.307 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.307 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.307 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284292+0000", 2023-10-09T10:47:48.307 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2689999999999999e-06 2023-10-09T10:47:48.307 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.307 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.307 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.308 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284297+0000", 2023-10-09T10:47:48.308 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4419999999999997e-06 2023-10-09T10:47:48.308 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.308 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.308 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.308 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284344+0000", 2023-10-09T10:47:48.308 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6940000000000001e-05 2023-10-09T10:47:48.308 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.308 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.308 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:47:48.308 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284352+0000", 2023-10-09T10:47:48.308 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0749999999999998e-06 2023-10-09T10:47:48.309 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.309 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.309 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.309 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.749219+0000", 2023-10-09T10:47:48.309 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.46486643799999999 2023-10-09T10:47:48.309 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.309 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.309 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.309 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.749378+0000", 2023-10-09T10:47:48.309 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015880000000000001 2023-10-09T10:47:48.309 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.309 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.309 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:47:48.310 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.749563+0000", 2023-10-09T10:47:48.310 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018555399999999999 2023-10-09T10:47:48.310 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.310 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.310 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.310 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754147+0000", 2023-10-09T10:47:48.310 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045842050000000001 2023-10-09T10:47:48.310 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.310 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.310 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.310 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754155+0000", 2023-10-09T10:47:48.310 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2080000000000003e-06 2023-10-09T10:47:48.311 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.311 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.311 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.311 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.755321+0000", 2023-10-09T10:47:48.311 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011663839999999999 2023-10-09T10:47:48.311 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.311 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.311 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.311 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.755360+0000", 2023-10-09T10:47:48.311 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8943000000000003e-05 2023-10-09T10:47:48.311 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.311 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.312 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.312 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.756057+0000", 2023-10-09T10:47:48.312 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00069742899999999995 2023-10-09T10:47:48.312 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.312 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.312 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.312 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.312 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.312 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106891 6.a 6:5a1ea4e1:::benchmark_data_smithi125_244951_object448:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-10-09T10:47:48.312 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:08.815152+0000", 2023-10-09T10:47:48.312 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 99.426979110999994, 2023-10-09T10:47:48.312 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.053066111999999999, 2023-10-09T10:47:48.313 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.313 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.313 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.313 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.313 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.313 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106891 2023-10-09T10:47:48.313 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.313 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.313 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.313 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.313 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.815152+0000", 2023-10-09T10:47:48.313 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.314 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.314 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.314 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.314 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.815152+0000", 2023-10-09T10:47:48.314 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.314 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.314 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.314 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.314 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.815151+0000", 2023-10-09T10:47:48.314 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:48.314 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.314 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.315 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.315 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.815157+0000", 2023-10-09T10:47:48.315 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1569999999999997e-06 2023-10-09T10:47:48.315 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.315 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.315 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.315 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.815166+0000", 2023-10-09T10:47:48.315 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4520000000000003e-06 2023-10-09T10:47:48.315 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.315 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.315 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.315 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.815171+0000", 2023-10-09T10:47:48.315 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7350000000000004e-06 2023-10-09T10:47:48.316 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.316 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.316 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.316 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.815214+0000", 2023-10-09T10:47:48.316 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2809000000000003e-05 2023-10-09T10:47:48.316 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.316 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.316 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.316 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.815336+0000", 2023-10-09T10:47:48.316 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012176 2023-10-09T10:47:48.316 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.316 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.316 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:47:48.316 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.815486+0000", 2023-10-09T10:47:48.317 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015021500000000001 2023-10-09T10:47:48.317 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.317 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.317 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.317 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.821061+0000", 2023-10-09T10:47:48.317 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0055751230000000004 2023-10-09T10:47:48.317 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.317 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.317 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.317 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.821078+0000", 2023-10-09T10:47:48.317 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6949999999999999e-05 2023-10-09T10:47:48.317 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.317 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.318 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.318 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.866886+0000", 2023-10-09T10:47:48.318 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.045808414999999998 2023-10-09T10:47:48.318 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.318 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.318 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.318 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.866937+0000", 2023-10-09T10:47:48.318 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0543e-05 2023-10-09T10:47:48.318 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.318 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.318 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.318 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:08.868218+0000", 2023-10-09T10:47:48.319 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012811840000000001 2023-10-09T10:47:48.319 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.319 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.319 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.319 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.319 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.319 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:119809 6.a 6:578cf30a:::benchmark_data_smithi125_244951_object13366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-10-09T10:47:48.319 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:24.627670+0000", 2023-10-09T10:47:48.319 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 83.614461156999994, 2023-10-09T10:47:48.319 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.056820514000000003, 2023-10-09T10:47:48.319 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.319 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.319 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.320 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.320 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.320 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 119809 2023-10-09T10:47:48.320 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.320 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.320 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.320 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.320 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:24.627670+0000", 2023-10-09T10:47:48.320 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.320 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.320 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.320 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.320 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:24.627670+0000", 2023-10-09T10:47:48.321 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.321 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.321 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.321 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.321 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:24.627669+0000", 2023-10-09T10:47:48.321 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:48.321 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.321 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.321 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.321 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:24.627673+0000", 2023-10-09T10:47:48.321 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2100000000000002e-06 2023-10-09T10:47:48.321 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.321 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.322 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.322 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:24.627680+0000", 2023-10-09T10:47:48.322 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9439999999999999e-06 2023-10-09T10:47:48.322 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.322 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.322 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.322 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:24.627684+0000", 2023-10-09T10:47:48.322 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2180000000000001e-06 2023-10-09T10:47:48.322 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.322 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.322 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.322 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:24.634125+0000", 2023-10-09T10:47:48.323 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064414470000000003 2023-10-09T10:47:48.323 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.323 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.323 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.323 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:24.634292+0000", 2023-10-09T10:47:48.323 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016664300000000001 2023-10-09T10:47:48.323 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.323 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.323 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:47:48.323 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:24.634454+0000", 2023-10-09T10:47:48.323 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016247599999999999 2023-10-09T10:47:48.324 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.324 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.324 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.324 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:24.647583+0000", 2023-10-09T10:47:48.324 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013128702000000001 2023-10-09T10:47:48.324 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.324 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.324 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.324 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:24.652280+0000", 2023-10-09T10:47:48.324 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0046966949999999999 2023-10-09T10:47:48.324 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.324 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.324 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.325 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:24.683270+0000", 2023-10-09T10:47:48.325 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030990542999999999 2023-10-09T10:47:48.325 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.325 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.325 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.325 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:24.683335+0000", 2023-10-09T10:47:48.325 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4430999999999993e-05 2023-10-09T10:47:48.325 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.325 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.325 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.325 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:24.684490+0000", 2023-10-09T10:47:48.325 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011556470000000001 2023-10-09T10:47:48.326 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.326 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.326 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.326 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.326 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.326 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:152170 6.e e174/154 6:7338dbae:::benchmark_data_smithi125_244951_object45727:head v 174'9413, mlcod=104'1056)", 2023-10-09T10:47:48.326 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.371354+0000", 2023-10-09T10:47:48.326 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 13.870776896000001, 2023-10-09T10:47:48.326 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.055038210999999997, 2023-10-09T10:47:48.326 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.326 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.326 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.327 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.327 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.327 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.371354+0000", 2023-10-09T10:47:48.327 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.327 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.327 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.327 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.327 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.371354+0000", 2023-10-09T10:47:48.327 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.327 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.327 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.328 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.328 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.371354+0000", 2023-10-09T10:47:48.328 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:48.328 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.328 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.328 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.328 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.371363+0000", 2023-10-09T10:47:48.328 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.918e-06 2023-10-09T10:47:48.328 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.328 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.328 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.328 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.371372+0000", 2023-10-09T10:47:48.328 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5599999999999999e-06 2023-10-09T10:47:48.328 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.329 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.329 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.329 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.371377+0000", 2023-10-09T10:47:48.329 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9980000000000003e-06 2023-10-09T10:47:48.329 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.329 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.329 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.329 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.371417+0000", 2023-10-09T10:47:48.329 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9891999999999997e-05 2023-10-09T10:47:48.329 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.329 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.330 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.330 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.371445+0000", 2023-10-09T10:47:48.330 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8011999999999999e-05 2023-10-09T10:47:48.330 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.330 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.330 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.330 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.426367+0000", 2023-10-09T10:47:48.330 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.054921650000000002 2023-10-09T10:47:48.330 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.330 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.330 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.330 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.426392+0000", 2023-10-09T10:47:48.331 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5672000000000001e-05 2023-10-09T10:47:48.331 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.331 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.331 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.331 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.331 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.331 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:152172 6.e e174/154 6:7cafcf69:::benchmark_data_smithi125_244951_object45729:head v 174'9414, mlcod=104'1056)", 2023-10-09T10:47:48.331 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.371693+0000", 2023-10-09T10:47:48.331 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 13.870437917, 2023-10-09T10:47:48.331 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.054718740000000002, 2023-10-09T10:47:48.331 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.331 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.332 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.332 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.332 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.332 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.371693+0000", 2023-10-09T10:47:48.332 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.332 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.332 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.332 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.332 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.371693+0000", 2023-10-09T10:47:48.332 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.332 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.332 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.332 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.333 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.371693+0000", 2023-10-09T10:47:48.333 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:48.333 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.333 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.333 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.333 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.371699+0000", 2023-10-09T10:47:48.333 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7780000000000002e-06 2023-10-09T10:47:48.333 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.333 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.333 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.333 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.371713+0000", 2023-10-09T10:47:48.333 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3424000000000001e-05 2023-10-09T10:47:48.334 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.334 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.334 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.334 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.371718+0000", 2023-10-09T10:47:48.334 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5679999999999999e-06 2023-10-09T10:47:48.334 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.334 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.334 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.334 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.380273+0000", 2023-10-09T10:47:48.334 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0085551110000000007 2023-10-09T10:47:48.334 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.334 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.335 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.335 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.380297+0000", 2023-10-09T10:47:48.335 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3081000000000001e-05 2023-10-09T10:47:48.335 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.335 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.335 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.335 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.426397+0000", 2023-10-09T10:47:48.335 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.046100391999999997 2023-10-09T10:47:48.335 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.335 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.335 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.335 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.426412+0000", 2023-10-09T10:47:48.335 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5003e-05 2023-10-09T10:47:48.336 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.336 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.336 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.336 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.336 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.336 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:152994 6.7 6:e184a006:::benchmark_data_smithi125_244951_object46551:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:47:48.336 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:35.519171+0000", 2023-10-09T10:47:48.336 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 12.722960112999999, 2023-10-09T10:47:48.336 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.084211253, 2023-10-09T10:47:48.336 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.336 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.336 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.337 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.337 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.337 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 152994 2023-10-09T10:47:48.337 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.337 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.337 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.337 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.337 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519171+0000", 2023-10-09T10:47:48.337 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.337 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.337 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.338 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.338 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519171+0000", 2023-10-09T10:47:48.338 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.338 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.338 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.338 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.338 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519170+0000", 2023-10-09T10:47:48.338 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:48.338 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.338 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.338 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.338 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519181+0000", 2023-10-09T10:47:48.339 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1884e-05 2023-10-09T10:47:48.339 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.339 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.339 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.339 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519196+0000", 2023-10-09T10:47:48.339 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.448e-05 2023-10-09T10:47:48.339 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.339 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.339 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.339 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519202+0000", 2023-10-09T10:47:48.339 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3670000000000002e-06 2023-10-09T10:47:48.339 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.340 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.340 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.340 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.522571+0000", 2023-10-09T10:47:48.340 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0033686639999999999 2023-10-09T10:47:48.340 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.340 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.340 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.340 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.522712+0000", 2023-10-09T10:47:48.340 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014130800000000001 2023-10-09T10:47:48.340 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.340 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.340 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:47:48.340 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.522828+0000", 2023-10-09T10:47:48.341 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115808 2023-10-09T10:47:48.341 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.341 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.341 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.341 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.527082+0000", 2023-10-09T10:47:48.341 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042537520000000004 2023-10-09T10:47:48.341 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.341 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.341 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.341 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.528250+0000", 2023-10-09T10:47:48.341 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011679310000000001 2023-10-09T10:47:48.341 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.342 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.342 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.342 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.603293+0000", 2023-10-09T10:47:48.342 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.075043377999999994 2023-10-09T10:47:48.342 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.342 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.342 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.342 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.603336+0000", 2023-10-09T10:47:48.342 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2774000000000001e-05 2023-10-09T10:47:48.342 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.342 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.343 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.343 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.603382+0000", 2023-10-09T10:47:48.343 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6279999999999997e-05 2023-10-09T10:47:48.343 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.343 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.343 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.343 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.343 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.343 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:153730 6.d 6:b4bba254:::benchmark_data_smithi125_244951_object47287:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:47:48.343 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:36.580818+0000", 2023-10-09T10:47:48.343 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 11.66131334, 2023-10-09T10:47:48.343 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.060300754999999998, 2023-10-09T10:47:48.344 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.344 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.344 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.344 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.344 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.344 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 153730 2023-10-09T10:47:48.344 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.344 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.344 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.344 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.344 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.580818+0000", 2023-10-09T10:47:48.344 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.345 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.345 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.345 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.345 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.580818+0000", 2023-10-09T10:47:48.345 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.345 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.345 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.345 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.345 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.580817+0000", 2023-10-09T10:47:48.345 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:48.345 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.345 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.346 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.346 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.580821+0000", 2023-10-09T10:47:48.346 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2270000000000002e-06 2023-10-09T10:47:48.346 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.346 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.346 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.346 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.580827+0000", 2023-10-09T10:47:48.346 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7410000000000003e-06 2023-10-09T10:47:48.346 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.346 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.346 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.347 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.580830+0000", 2023-10-09T10:47:48.347 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0469999999999998e-06 2023-10-09T10:47:48.347 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.347 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.347 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.347 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.585573+0000", 2023-10-09T10:47:48.347 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00474311 2023-10-09T10:47:48.347 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.347 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.347 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.347 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.585727+0000", 2023-10-09T10:47:48.347 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015385 2023-10-09T10:47:48.348 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.348 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.348 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:47:48.348 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.585876+0000", 2023-10-09T10:47:48.348 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014922700000000001 2023-10-09T10:47:48.348 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.348 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.348 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.348 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.591228+0000", 2023-10-09T10:47:48.348 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0053518940000000003 2023-10-09T10:47:48.348 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.348 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.348 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.349 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.609925+0000", 2023-10-09T10:47:48.349 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018696833999999999 2023-10-09T10:47:48.349 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.349 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.349 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.349 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.639420+0000", 2023-10-09T10:47:48.349 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029494933000000001 2023-10-09T10:47:48.349 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.349 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.349 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.349 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.639454+0000", 2023-10-09T10:47:48.349 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3986000000000003e-05 2023-10-09T10:47:48.350 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.350 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.350 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.350 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.641118+0000", 2023-10-09T10:47:48.350 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0016643680000000001 2023-10-09T10:47:48.350 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.350 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.350 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.350 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.350 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.350 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:153733 6.d 6:b575b55b:::benchmark_data_smithi125_244951_object47290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:47:48.350 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:36.588580+0000", 2023-10-09T10:47:48.351 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 11.653551237, 2023-10-09T10:47:48.351 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.071524196999999998, 2023-10-09T10:47:48.351 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.351 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.351 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.351 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.351 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.351 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 153733 2023-10-09T10:47:48.351 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.351 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.351 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.351 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.352 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588580+0000", 2023-10-09T10:47:48.352 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.352 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.352 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.352 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.352 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588580+0000", 2023-10-09T10:47:48.352 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.352 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.352 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.352 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.352 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588578+0000", 2023-10-09T10:47:48.352 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:48.352 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.353 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.353 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.353 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588585+0000", 2023-10-09T10:47:48.353 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.139e-06 2023-10-09T10:47:48.353 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.353 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.353 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.353 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588596+0000", 2023-10-09T10:47:48.353 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.163e-05 2023-10-09T10:47:48.353 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.353 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.353 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.354 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588602+0000", 2023-10-09T10:47:48.354 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6869999999999999e-06 2023-10-09T10:47:48.354 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.354 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.354 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.354 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.595518+0000", 2023-10-09T10:47:48.354 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0069160599999999999 2023-10-09T10:47:48.354 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.354 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.354 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.354 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.595663+0000", 2023-10-09T10:47:48.354 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014470299999999999 2023-10-09T10:47:48.355 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.355 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.355 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:47:48.355 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.595807+0000", 2023-10-09T10:47:48.355 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014449800000000001 2023-10-09T10:47:48.355 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.355 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.355 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.355 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.600067+0000", 2023-10-09T10:47:48.355 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042602200000000003 2023-10-09T10:47:48.355 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.355 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.355 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.356 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.641151+0000", 2023-10-09T10:47:48.356 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.041083736000000003 2023-10-09T10:47:48.356 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.356 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.356 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.356 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.657793+0000", 2023-10-09T10:47:48.356 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016641420000000001 2023-10-09T10:47:48.356 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.356 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.356 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.356 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.657848+0000", 2023-10-09T10:47:48.356 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5611e-05 2023-10-09T10:47:48.356 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.357 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.357 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.357 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.660104+0000", 2023-10-09T10:47:48.357 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022558880000000002 2023-10-09T10:47:48.357 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.357 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.357 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.357 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.357 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.357 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:153738 6.d 6:b4e67804:::benchmark_data_smithi125_244951_object47295:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:47:48.357 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:36.603669+0000", 2023-10-09T10:47:48.357 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 11.638461749999999, 2023-10-09T10:47:48.358 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.058424775999999998, 2023-10-09T10:47:48.358 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.358 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.358 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.358 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.358 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.358 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 153738 2023-10-09T10:47:48.358 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.358 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.358 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.358 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.359 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.603669+0000", 2023-10-09T10:47:48.359 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.359 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.359 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.359 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.603669+0000", 2023-10-09T10:47:48.359 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.359 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.359 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.359 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.603668+0000", 2023-10-09T10:47:48.359 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:48.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.360 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.360 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.360 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.603674+0000", 2023-10-09T10:47:48.360 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4169999999999997e-06 2023-10-09T10:47:48.360 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.360 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.360 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.360 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.603687+0000", 2023-10-09T10:47:48.360 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.238e-05 2023-10-09T10:47:48.360 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.360 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.360 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.360 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.603692+0000", 2023-10-09T10:47:48.361 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3260000000000003e-06 2023-10-09T10:47:48.361 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.361 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.361 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.361 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.603734+0000", 2023-10-09T10:47:48.361 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1495999999999997e-05 2023-10-09T10:47:48.361 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.361 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.361 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.361 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.603849+0000", 2023-10-09T10:47:48.361 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011593499999999999 2023-10-09T10:47:48.361 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.362 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.362 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:47:48.362 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.603948+0000", 2023-10-09T10:47:48.362 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8886000000000002e-05 2023-10-09T10:47:48.362 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.362 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.362 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.362 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.609937+0000", 2023-10-09T10:47:48.362 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059888839999999999 2023-10-09T10:47:48.362 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.362 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.362 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.363 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.647846+0000", 2023-10-09T10:47:48.363 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.037908881999999998 2023-10-09T10:47:48.363 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.363 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.363 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.363 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.660146+0000", 2023-10-09T10:47:48.363 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012299826 2023-10-09T10:47:48.363 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.363 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.363 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.363 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.660193+0000", 2023-10-09T10:47:48.363 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7457999999999999e-05 2023-10-09T10:47:48.364 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.364 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.364 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.364 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.662094+0000", 2023-10-09T10:47:48.364 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019007200000000001 2023-10-09T10:47:48.364 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.364 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.364 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.364 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.364 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.364 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:154223 6.d 6:b0b1d51c:::benchmark_data_smithi125_244951_object47780:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:47:48.364 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:37.297363+0000", 2023-10-09T10:47:48.364 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 10.944768357999999, 2023-10-09T10:47:48.365 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.056381402999999997, 2023-10-09T10:47:48.365 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.365 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.365 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.365 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.365 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.365 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 154223 2023-10-09T10:47:48.365 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.365 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.365 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.365 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.365 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:37.297363+0000", 2023-10-09T10:47:48.366 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.366 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.366 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.366 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.366 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:37.297363+0000", 2023-10-09T10:47:48.366 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.366 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.366 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.366 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.366 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:37.297362+0000", 2023-10-09T10:47:48.366 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:48.367 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.367 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.367 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.367 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:37.297365+0000", 2023-10-09T10:47:48.367 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0019999999999998e-06 2023-10-09T10:47:48.367 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.367 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.367 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.367 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:37.297371+0000", 2023-10-09T10:47:48.367 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4589999999999997e-06 2023-10-09T10:47:48.367 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.367 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.368 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.368 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:37.297374+0000", 2023-10-09T10:47:48.368 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3380000000000001e-06 2023-10-09T10:47:48.368 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.368 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.368 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.368 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:37.323243+0000", 2023-10-09T10:47:48.368 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025868347999999999 2023-10-09T10:47:48.368 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.368 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.368 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.368 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:37.323321+0000", 2023-10-09T10:47:48.368 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8527000000000002e-05 2023-10-09T10:47:48.369 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.369 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.369 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:47:48.369 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:37.323397+0000", 2023-10-09T10:47:48.369 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6328999999999994e-05 2023-10-09T10:47:48.369 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.369 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.369 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.369 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:37.330240+0000", 2023-10-09T10:47:48.369 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0068429800000000002 2023-10-09T10:47:48.369 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.369 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.370 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.370 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:37.331020+0000", 2023-10-09T10:47:48.370 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00077940099999999999 2023-10-09T10:47:48.370 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.370 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.370 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.370 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:37.349440+0000", 2023-10-09T10:47:48.370 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018420644 2023-10-09T10:47:48.370 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.370 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.370 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.371 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:37.349493+0000", 2023-10-09T10:47:48.371 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2305999999999997e-05 2023-10-09T10:47:48.371 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.371 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.371 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.371 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:37.353744+0000", 2023-10-09T10:47:48.371 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042514090000000003 2023-10-09T10:47:48.371 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.371 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.371 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.371 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.371 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.372 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:159376 6.a 6:5d3d2f87:::benchmark_data_smithi125_244951_object52933:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-10-09T10:47:48.372 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:44.500622+0000", 2023-10-09T10:47:48.372 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 3.7415091199999999, 2023-10-09T10:47:48.372 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.054690810999999999, 2023-10-09T10:47:48.372 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.372 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.372 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:47:48.372 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:47:48.372 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:47:48.372 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 159376 2023-10-09T10:47:48.372 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.372 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.373 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.373 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.373 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.500622+0000", 2023-10-09T10:47:48.373 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.373 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.373 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.373 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.373 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.500622+0000", 2023-10-09T10:47:48.373 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.373 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.373 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.373 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.374 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.500622+0000", 2023-10-09T10:47:48.374 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:48.374 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.374 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.374 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.374 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.500624+0000", 2023-10-09T10:47:48.374 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4059999999999999e-06 2023-10-09T10:47:48.374 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.374 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.374 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.374 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.500631+0000", 2023-10-09T10:47:48.374 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8340000000000001e-06 2023-10-09T10:47:48.375 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.375 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.375 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.375 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.500635+0000", 2023-10-09T10:47:48.375 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.172e-06 2023-10-09T10:47:48.375 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.375 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.375 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.375 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.500696+0000", 2023-10-09T10:47:48.375 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0526999999999998e-05 2023-10-09T10:47:48.375 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.375 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.375 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.376 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.500814+0000", 2023-10-09T10:47:48.376 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118813 2023-10-09T10:47:48.376 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.376 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.376 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:47:48.376 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.500938+0000", 2023-10-09T10:47:48.376 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012334800000000001 2023-10-09T10:47:48.376 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.376 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.376 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:47:48.376 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.506108+0000", 2023-10-09T10:47:48.376 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051706959999999998 2023-10-09T10:47:48.377 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.377 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.377 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.377 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.510860+0000", 2023-10-09T10:47:48.377 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0047515650000000001 2023-10-09T10:47:48.377 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.377 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.377 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:47:48.377 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.554536+0000", 2023-10-09T10:47:48.377 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.043676056999999997 2023-10-09T10:47:48.377 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.377 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.378 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.378 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.554593+0000", 2023-10-09T10:47:48.378 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7303999999999999e-05 2023-10-09T10:47:48.378 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.378 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.378 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.378 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.555313+0000", 2023-10-09T10:47:48.378 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00071948499999999996 2023-10-09T10:47:48.378 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.378 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.378 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.379 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.379 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.379 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:159662 6.e e176/154 6:771f0157:::benchmark_data_smithi125_244951_object53219:head v 176'9879, mlcod=104'1056)", 2023-10-09T10:47:48.379 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:44.956389+0000", 2023-10-09T10:47:48.379 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 3.2857423830000001, 2023-10-09T10:47:48.379 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.054107889999999999, 2023-10-09T10:47:48.379 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:48.379 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:47:48.379 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:48.379 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.379 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:48.379 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.956389+0000", 2023-10-09T10:47:48.379 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.380 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.380 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.380 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:48.380 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.956389+0000", 2023-10-09T10:47:48.380 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:48.380 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.380 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.380 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:48.380 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.956388+0000", 2023-10-09T10:47:48.380 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967296 2023-10-09T10:47:48.380 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.380 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.381 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:48.381 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.956392+0000", 2023-10-09T10:47:48.381 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6459999999999999e-06 2023-10-09T10:47:48.381 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.381 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.381 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:48.381 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.956397+0000", 2023-10-09T10:47:48.381 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.135e-06 2023-10-09T10:47:48.381 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.381 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.381 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:48.381 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.956401+0000", 2023-10-09T10:47:48.381 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3749999999999999e-06 2023-10-09T10:47:48.382 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.382 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.382 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:48.382 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.972072+0000", 2023-10-09T10:47:48.382 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015671159 2023-10-09T10:47:48.382 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.382 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.382 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:48.382 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.972092+0000", 2023-10-09T10:47:48.382 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0049999999999999e-05 2023-10-09T10:47:48.382 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.382 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.382 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:47:48.383 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:45.010472+0000", 2023-10-09T10:47:48.383 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038380620999999997 2023-10-09T10:47:48.383 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:48.383 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:48.383 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:47:48.383 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:45.010497+0000", 2023-10-09T10:47:48.383 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4139999999999999e-05 2023-10-09T10:47:48.383 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.383 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.383 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.383 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:48.383 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:48.384 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:47:48.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:48.384 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:48.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:48.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:48.532 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:48.552 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:48 smithi155 ceph-mon[184854]: pgmap v374: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 31152/158883 objects degraded (19.607%); 6411/158883 objects misplaced (4.035%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T10:47:48.552 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:48 smithi155 ceph-mon[184854]: failed to encode map e177 with expected crc 2023-10-09T10:47:48.552 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:48 smithi155 ceph-mon[184854]: failed to encode map e177 with expected crc 2023-10-09T10:47:48.552 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:48 smithi155 ceph-mon[184854]: osdmap e177: 8 total, 8 up, 7 in 2023-10-09T10:47:48.552 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:48 smithi155 ceph-mon[184854]: failed to encode map e177 with expected crc 2023-10-09T10:47:48.552 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:48 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 31152/158883 objects degraded (19.607%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:48.634 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:47:48.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:48 smithi125 ceph-mon[195507]: pgmap v374: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 31152/158883 objects degraded (19.607%); 6411/158883 objects misplaced (4.035%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T10:47:48.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:48 smithi125 ceph-mon[195507]: failed to encode map e177 with expected crc 2023-10-09T10:47:48.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:48 smithi125 ceph-mon[195507]: failed to encode map e177 with expected crc 2023-10-09T10:47:48.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:48 smithi125 ceph-mon[195507]: osdmap e177: 8 total, 8 up, 7 in 2023-10-09T10:47:48.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:48 smithi125 ceph-mon[195507]: failed to encode map e177 with expected crc 2023-10-09T10:47:48.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:48 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 31152/158883 objects degraded (19.607%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:48.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:48 smithi125 ceph-mon[200197]: pgmap v374: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 31152/158883 objects degraded (19.607%); 6411/158883 objects misplaced (4.035%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T10:47:48.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:48 smithi125 ceph-mon[200197]: failed to encode map e177 with expected crc 2023-10-09T10:47:48.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:48 smithi125 ceph-mon[200197]: failed to encode map e177 with expected crc 2023-10-09T10:47:48.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:48 smithi125 ceph-mon[200197]: osdmap e177: 8 total, 8 up, 7 in 2023-10-09T10:47:48.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:48 smithi125 ceph-mon[200197]: failed to encode map e177 with expected crc 2023-10-09T10:47:48.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:48 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 31152/158883 objects degraded (19.607%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:49.054 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:49.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:49.060 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:49.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:49.141 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:47:49.162 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:49.198 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:49.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:49 smithi155 ceph-mon[184854]: osdmap e178: 8 total, 8 up, 7 in 2023-10-09T10:47:49.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:49 smithi155 ceph-mon[184854]: failed to encode map e178 with expected crc 2023-10-09T10:47:49.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:49 smithi155 ceph-mon[184854]: failed to encode map e178 with expected crc 2023-10-09T10:47:49.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:49 smithi155 ceph-mon[184854]: failed to encode map e178 with expected crc 2023-10-09T10:47:49.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:49 smithi155 ceph-mon[184854]: pgmap v377: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 31152/158238 objects degraded (19.687%); 6411/158238 objects misplaced (4.051%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T10:47:49.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:49 smithi125 ceph-mon[195507]: osdmap e178: 8 total, 8 up, 7 in 2023-10-09T10:47:49.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:49 smithi125 ceph-mon[195507]: failed to encode map e178 with expected crc 2023-10-09T10:47:49.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:49 smithi125 ceph-mon[195507]: failed to encode map e178 with expected crc 2023-10-09T10:47:49.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:49 smithi125 ceph-mon[195507]: failed to encode map e178 with expected crc 2023-10-09T10:47:49.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:49 smithi125 ceph-mon[195507]: pgmap v377: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 31152/158238 objects degraded (19.687%); 6411/158238 objects misplaced (4.051%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T10:47:49.797 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:49 smithi125 ceph-mon[200197]: osdmap e178: 8 total, 8 up, 7 in 2023-10-09T10:47:49.798 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:49 smithi125 ceph-mon[200197]: failed to encode map e178 with expected crc 2023-10-09T10:47:49.798 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:49 smithi125 ceph-mon[200197]: failed to encode map e178 with expected crc 2023-10-09T10:47:49.798 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:49 smithi125 ceph-mon[200197]: failed to encode map e178 with expected crc 2023-10-09T10:47:49.798 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:49 smithi125 ceph-mon[200197]: pgmap v377: 119 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 31152/158238 objects degraded (19.687%); 6411/158238 objects misplaced (4.051%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T10:47:49.798 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:49.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:49.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:49.837 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:49.903 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:49.982 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:50.031 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:50.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:50.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:50.136 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:50.222 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:50.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:50.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:50.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:50.366 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:50.414 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:50.539 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:50.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:50.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:50.552 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:47:50.553 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:47:50.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.553 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:163389 6.f e178/154 6:f8ee9fad:::benchmark_data_smithi125_244951_object56946:head v 178'10039, mlcod=104'1059)", 2023-10-09T10:47:50.553 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:50.536678+0000", 2023-10-09T10:47:50.553 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.015410027999999999, 2023-10-09T10:47:50.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015442947, 2023-10-09T10:47:50.553 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:50.553 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:47:50.553 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:50.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:50.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.536678+0000", 2023-10-09T10:47:50.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:50.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:50.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.536678+0000", 2023-10-09T10:47:50.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:50.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:50.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.536677+0000", 2023-10-09T10:47:50.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:50.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:50.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.536683+0000", 2023-10-09T10:47:50.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6450000000000002e-06 2023-10-09T10:47:50.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:50.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.536691+0000", 2023-10-09T10:47:50.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2130000000000008e-06 2023-10-09T10:47:50.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:50.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.536696+0000", 2023-10-09T10:47:50.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2579999999999997e-06 2023-10-09T10:47:50.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:50.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.538132+0000", 2023-10-09T10:47:50.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014362839999999999 2023-10-09T10:47:50.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:50.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.538154+0000", 2023-10-09T10:47:50.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1968999999999999e-05 2023-10-09T10:47:50.557 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:50.557 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:50.557 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:50.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.557 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:163390 6.f e178/154 6:fc6c2823:::benchmark_data_smithi125_244951_object56947:head v 178'10040, mlcod=104'1059)", 2023-10-09T10:47:50.557 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:50.542572+0000", 2023-10-09T10:47:50.557 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0095157509999999994, 2023-10-09T10:47:50.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0096057139999999996, 2023-10-09T10:47:50.558 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:50.558 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:47:50.558 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:50.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:50.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.542572+0000", 2023-10-09T10:47:50.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:50.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:50.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.542572+0000", 2023-10-09T10:47:50.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:50.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:50.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.542571+0000", 2023-10-09T10:47:50.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:47:50.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:50.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.542580+0000", 2023-10-09T10:47:50.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7169999999999997e-06 2023-10-09T10:47:50.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:50.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.542592+0000", 2023-10-09T10:47:50.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1902000000000001e-05 2023-10-09T10:47:50.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:50.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.542597+0000", 2023-10-09T10:47:50.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9579999999999998e-06 2023-10-09T10:47:50.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:50.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.546549+0000", 2023-10-09T10:47:50.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039520780000000004 2023-10-09T10:47:50.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:50.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.546575+0000", 2023-10-09T10:47:50.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.569e-05 2023-10-09T10:47:50.561 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:50.561 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:50.561 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:50.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.562 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:163394 6.f e178/154)", 2023-10-09T10:47:50.562 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:50.548744+0000", 2023-10-09T10:47:50.562 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0033442910000000001, 2023-10-09T10:47:50.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034678119999999998, 2023-10-09T10:47:50.562 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:50.562 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "queued for pg", 2023-10-09T10:47:50.562 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:50.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.562 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:50.562 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.548744+0000", 2023-10-09T10:47:50.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:50.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:50.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.548744+0000", 2023-10-09T10:47:50.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:50.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:50.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.548742+0000", 2023-10-09T10:47:50.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:47:50.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:50.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.548755+0000", 2023-10-09T10:47:50.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2155e-05 2023-10-09T10:47:50.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:50.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.548767+0000", 2023-10-09T10:47:50.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2553e-05 2023-10-09T10:47:50.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:50.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.548774+0000", 2023-10-09T10:47:50.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1709999999999996e-06 2023-10-09T10:47:50.565 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:50.565 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:50.565 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:50.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.565 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:163391 6.18 e178/154 6:1d4ae4fd:::benchmark_data_smithi125_244951_object56948:head v 178'6080, mlcod=104'1038)", 2023-10-09T10:47:50.565 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:50.548783+0000", 2023-10-09T10:47:50.565 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0033051510000000001, 2023-10-09T10:47:50.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003456934, 2023-10-09T10:47:50.565 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:50.565 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:47:50.566 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:50.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:50.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.548783+0000", 2023-10-09T10:47:50.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:50.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:50.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.548783+0000", 2023-10-09T10:47:50.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:50.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.567 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:50.567 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.548783+0000", 2023-10-09T10:47:50.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:47:50.567 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.567 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.567 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:50.567 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.548788+0000", 2023-10-09T10:47:50.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1560000000000003e-06 2023-10-09T10:47:50.567 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.567 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.567 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:50.567 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.548795+0000", 2023-10-09T10:47:50.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.002e-06 2023-10-09T10:47:50.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:50.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.548799+0000", 2023-10-09T10:47:50.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.783e-06 2023-10-09T10:47:50.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:50.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.548854+0000", 2023-10-09T10:47:50.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5177999999999999e-05 2023-10-09T10:47:50.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:50.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.548877+0000", 2023-10-09T10:47:50.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2900000000000001e-05 2023-10-09T10:47:50.569 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:50.569 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:50.569 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:50.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.569 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:163397 6.a e178/142 6:5d19d82c:::benchmark_data_smithi125_244951_object56954:head v 178'10565, mlcod=137'6188)", 2023-10-09T10:47:50.569 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:50.549576+0000", 2023-10-09T10:47:50.569 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.002512378, 2023-10-09T10:47:50.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026992510000000002, 2023-10-09T10:47:50.570 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:47:50.570 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:47:50.570 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:47:50.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:47:50.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.549576+0000", 2023-10-09T10:47:50.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:50.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:47:50.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.549576+0000", 2023-10-09T10:47:50.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:47:50.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:47:50.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.549575+0000", 2023-10-09T10:47:50.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:47:50.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:47:50.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.549583+0000", 2023-10-09T10:47:50.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3690000000000007e-06 2023-10-09T10:47:50.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:47:50.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.549595+0000", 2023-10-09T10:47:50.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1933000000000001e-05 2023-10-09T10:47:50.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:47:50.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.549602+0000", 2023-10-09T10:47:50.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.652e-06 2023-10-09T10:47:50.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:47:50.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.549663+0000", 2023-10-09T10:47:50.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0645e-05 2023-10-09T10:47:50.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:47:50.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:47:50.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:47:50.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:50.549686+0000", 2023-10-09T10:47:50.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3343e-05 2023-10-09T10:47:50.573 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:50.573 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:47:50.573 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:50.573 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:47:50.574 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T10:47:50.574 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 5 2023-10-09T10:47:50.574 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:47:50.645 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:50.731 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:50.882 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:50.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:50.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:50.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:47:50.990 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:51.034 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:51.096 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:51.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:51.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:51.136 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:47:51.137 INFO:teuthology.orchestra.run.smithi125.stdout:{"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 31152/158238 objects degraded (19.687%), 10 pgs degraded, 11 pgs undersized","count":21},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":456,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":178,"num_osds":8,"num_up_osds":8,"osd_up_since":1696848214,"num_in_osds":7,"osd_in_since":1696848391,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":105},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":9},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":52746,"data_bytes":3455686584,"bytes_used":11261853696,"bytes_avail":660703395840,"bytes_total":671965249536,"degraded_objects":31152,"degraded_total":158238,"degraded_ratio":0.19686800894854584,"misplaced_objects":6411,"misplaced_total":158238,"misplaced_ratio":0.040514920562696698,"recovering_objects_per_sec":116,"recovering_bytes_per_sec":7654251,"recovering_keys_per_sec":0,"num_objects_recovered":701,"num_bytes_recovered":45940736,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":605},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi125.satgiw","status":"up:active","gid":24575}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"}},"servicemap":{"epoch":20,"modified":"2023-10-09T10:44:35.008237+0000","services":{}},"progress_events":{"4db0080d-306f-4da6-a38c-4c702d14cb34":{"message":"Global Recovery Event (2m)\n [========================....] (remaining: 22s)","progress":0.88235294818878174,"add_to_ceph_s":true},"585851ac-6e70-4462-9176-ac50987fab77":{"message":"Upgrade to 18.2.0-598-gb4fa663f (3m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T10:47:51.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2693902317' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:47:51.210 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:51.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2693902317' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:47:51.250 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2693902317' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:47:51.250 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:47:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:47:51] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.43.0" 2023-10-09T10:47:51.254 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:51.404 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:47:51.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:51.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:51.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:51.665 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:51.710 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:51.712 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:47:51.712 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:47:51.761 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:51.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:51.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:51.875 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:51.945 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:52.080 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:52.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:52.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:52.154 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:52.184 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:52.218 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:47:52.234 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:47:52.249 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:47:52.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:52 smithi155 ceph-mon[184854]: pgmap v378: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 747 op/s; 31152/150426 objects degraded (20.709%); 5659/150426 objects misplaced (3.762%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T10:47:52.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:52 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/105866922' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:52.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:52 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3633802482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:52.273 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:52.278 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:47:52.308 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:47:52.366 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:47:52.413 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:47:52.420 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:52.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:52.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:52.449 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:47:52.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:52 smithi125 ceph-mon[195507]: pgmap v378: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 747 op/s; 31152/150426 objects degraded (20.709%); 5659/150426 objects misplaced (3.762%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T10:47:52.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:52 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/105866922' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:52.494 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:52 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3633802482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:52.494 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:52 smithi125 ceph-mon[200197]: pgmap v378: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 747 op/s; 31152/150426 objects degraded (20.709%); 5659/150426 objects misplaced (3.762%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T10:47:52.494 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:52 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/105866922' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:47:52.494 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:52 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3633802482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:52.494 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:47:52.503 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:47:52.503 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:47:52.503 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:47:52.503 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:47:52.503 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:47:52.503 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:47:52.503 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:47:52.503 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:47:52.528 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:52.632 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:52.661 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:52.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:52.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:52.676 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:47:52.676 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:47:52.766 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:52.827 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:53.010 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:53.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:53.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:53.065 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:47:53.117 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:53.177 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:53.459 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:53.459 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/105866922' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:47:53.459 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[195507]: osdmap e179: 8 total, 8 up, 7 in 2023-10-09T10:47:53.459 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[195507]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.460 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[195507]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.460 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[195507]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.460 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[195507]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.460 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[195507]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.460 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[195507]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.460 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[195507]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.460 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[195507]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.460 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[195507]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.460 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 31152/150426 objects degraded (20.709%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:53.460 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:53.460 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/105866922' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:47:53.461 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[200197]: osdmap e179: 8 total, 8 up, 7 in 2023-10-09T10:47:53.461 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[200197]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.461 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[200197]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.461 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[200197]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.461 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[200197]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.461 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[200197]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.461 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[200197]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.461 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[200197]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.461 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[200197]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.461 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[200197]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.461 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:53 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 31152/150426 objects degraded (20.709%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:53.462 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:53.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:53.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:53.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:53 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:47:53.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:53 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/105866922' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:47:53.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:53 smithi155 ceph-mon[184854]: osdmap e179: 8 total, 8 up, 7 in 2023-10-09T10:47:53.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:53 smithi155 ceph-mon[184854]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:53 smithi155 ceph-mon[184854]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:53 smithi155 ceph-mon[184854]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:53 smithi155 ceph-mon[184854]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:53 smithi155 ceph-mon[184854]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:53 smithi155 ceph-mon[184854]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:53 smithi155 ceph-mon[184854]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:53 smithi155 ceph-mon[184854]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:53 smithi155 ceph-mon[184854]: failed to encode map e179 with expected crc 2023-10-09T10:47:53.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:53 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 31152/150426 objects degraded (20.709%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:53.533 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:47:53.572 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:53.661 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:53.720 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:53.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:53.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:53.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:53.891 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:54.201 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:47:54.204 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:47:54.489 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:54.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:54.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:54.563 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:54 smithi155 ceph-mon[184854]: pgmap v380: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 31152/148602 objects degraded (20.963%); 5570/148602 objects misplaced (3.748%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:47:54.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:54 smithi125 ceph-mon[195507]: pgmap v380: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 31152/148602 objects degraded (20.963%); 5570/148602 objects misplaced (3.748%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:47:54.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:54 smithi125 ceph-mon[200197]: pgmap v380: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 31152/148602 objects degraded (20.963%); 5570/148602 objects misplaced (3.748%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:47:54.596 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:54.655 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":380,"stamp":"2023-10-09T10:47:53.121585+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3245184952,"num_objects":49534,"num_object_clones":0,"num_object_copies":148602,"num_objects_missing_on_primary":1406,"num_objects_missing":1406,"num_objects_degraded":31152,"num_objects_misplaced":5570,"num_objects_unfound":0,"num_objects_dirty":49534,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":154998,"num_write_kb":6260974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23043,"num_bytes_recovered":1417677570,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167020,"ondisk_log_size":167020,"up":357,"acting":347,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10493396,"kb_used_data":8853340,"kb_used_omap":44,"kb_used_meta":1639763,"kb_avail":645722668,"statfs":{"total":671965249536,"available":661220012032,"internally_reserved":0,"allocated":9065820160,"data_stored":9060420220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1679117729},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,2,7,6,10,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-224591872,"num_objects":-3427,"num_object_clones":0,"num_object_copies":-10281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-841,"num_objects_unfound":0,"num_objects_dirty":-3427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3427,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":547,"num_bytes_recovered":35848192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002026"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":171,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386688+0000","last_change":"2023-10-09T10:46:32.250979+0000","last_active":"2023-10-09T10:47:47.386688+0000","last_peered":"2023-10-09T10:47:47.386688+0000","last_clean":"2023-10-09T10:47:47.386688+0000","last_became_active":"2023-10-09T10:46:32.250625+0000","last_became_peered":"2023-10-09T10:46:32.250625+0000","last_unstale":"2023-10-09T10:47:47.386688+0000","last_undegraded":"2023-10-09T10:47:47.386688+0000","last_fullsized":"2023-10-09T10:47:47.386688+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:34.858348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":146,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.387373+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:47:47.387373+0000","last_peered":"2023-10-09T10:47:47.387373+0000","last_clean":"2023-10-09T10:47:47.387373+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:47:47.387373+0000","last_undegraded":"2023-10-09T10:47:47.387373+0000","last_fullsized":"2023-10-09T10:47:47.387373+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":130,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.400632+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:47:47.400632+0000","last_peered":"2023-10-09T10:47:47.400632+0000","last_clean":"2023-10-09T10:47:47.400632+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:47:47.400632+0000","last_undegraded":"2023-10-09T10:47:47.400632+0000","last_fullsized":"2023-10-09T10:47:47.400632+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":47,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385769+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:47:48.385769+0000","last_peered":"2023-10-09T10:47:48.385769+0000","last_clean":"2023-10-09T10:47:48.385769+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:47:48.385769+0000","last_undegraded":"2023-10-09T10:47:48.385769+0000","last_fullsized":"2023-10-09T10:47:48.385769+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":182,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386219+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:47:47.386219+0000","last_peered":"2023-10-09T10:47:47.386219+0000","last_clean":"2023-10-09T10:47:47.386219+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:47:47.386219+0000","last_undegraded":"2023-10-09T10:47:47.386219+0000","last_fullsized":"2023-10-09T10:47:47.386219+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":116,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.389217+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:47:47.389217+0000","last_peered":"2023-10-09T10:47:47.389217+0000","last_clean":"2023-10-09T10:47:47.389217+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:47:47.389217+0000","last_undegraded":"2023-10-09T10:47:47.389217+0000","last_fullsized":"2023-10-09T10:47:47.389217+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"178'6288","reported_seq":10440,"reported_epoch":178,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:51.013018+0000","last_change":"2023-10-09T10:46:33.521602+0000","last_active":"2023-10-09T10:47:51.013018+0000","last_peered":"2023-10-09T10:47:51.013018+0000","last_clean":"2023-10-09T10:46:31.055751+0000","last_became_active":"2023-10-09T10:46:33.364058+0000","last_became_peered":"2023-10-09T10:46:33.364058+0000","last_unstale":"2023-10-09T10:47:51.013018+0000","last_undegraded":"2023-10-09T10:47:51.013018+0000","last_fullsized":"2023-10-09T10:47:51.013018+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6288,"log_dups_size":0,"ondisk_log_size":6288,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105218048,"num_objects":1606,"num_object_clones":0,"num_object_copies":4818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1605,"num_objects_unfound":0,"num_objects_dirty":1606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5252,"num_write_kb":219424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2010,"num_bytes_recovered":131727360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":1606}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":47,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386472+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:47:48.386472+0000","last_peered":"2023-10-09T10:47:48.386472+0000","last_clean":"2023-10-09T10:47:48.386472+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:47:48.386472+0000","last_undegraded":"2023-10-09T10:47:48.386472+0000","last_fullsized":"2023-10-09T10:47:48.386472+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":146,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386281+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:47:47.386281+0000","last_peered":"2023-10-09T10:47:47.386281+0000","last_clean":"2023-10-09T10:47:47.386281+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:47:47.386281+0000","last_undegraded":"2023-10-09T10:47:47.386281+0000","last_fullsized":"2023-10-09T10:47:47.386281+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":512,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385364+0000","last_change":"2023-10-09T10:46:05.620365+0000","last_active":"2023-10-09T10:47:48.385364+0000","last_peered":"2023-10-09T10:47:48.385364+0000","last_clean":"2023-10-09T10:47:48.385364+0000","last_became_active":"2023-10-09T10:46:05.620024+0000","last_became_peered":"2023-10-09T10:46:05.620024+0000","last_unstale":"2023-10-09T10:47:48.385364+0000","last_undegraded":"2023-10-09T10:47:48.385364+0000","last_fullsized":"2023-10-09T10:47:48.385364+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.13","version":"178'6108","reported_seq":6292,"reported_epoch":178,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:50.973319+0000","last_change":"2023-10-09T10:46:33.470698+0000","last_active":"2023-10-09T10:47:50.973319+0000","last_peered":"2023-10-09T10:47:50.973319+0000","last_clean":"2023-10-09T10:46:31.195699+0000","last_became_active":"2023-10-09T10:46:33.455632+0000","last_became_peered":"2023-10-09T10:46:33.455632+0000","last_unstale":"2023-10-09T10:47:50.973319+0000","last_undegraded":"2023-10-09T10:46:33.269323+0000","last_fullsized":"2023-10-09T10:46:33.241226+0000","mapping_epoch":154,"log_start":"103'745","ondisk_log_start":"103'745","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":5363,"log_dups_size":0,"ondisk_log_size":5363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:07:12.547333+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102498304,"num_objects":1564,"num_object_clones":0,"num_object_copies":4692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3315,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3393,"num_write_kb":105088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":1564},{"shards":"6,7","objects":-1211}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":143,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.389975+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:47:47.389975+0000","last_peered":"2023-10-09T10:47:47.389975+0000","last_clean":"2023-10-09T10:47:47.389975+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:47:47.389975+0000","last_undegraded":"2023-10-09T10:47:47.389975+0000","last_fullsized":"2023-10-09T10:47:47.389975+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":103,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.400995+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:47:47.400995+0000","last_peered":"2023-10-09T10:47:47.400995+0000","last_clean":"2023-10-09T10:47:47.400995+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:47:47.400995+0000","last_undegraded":"2023-10-09T10:47:47.400995+0000","last_fullsized":"2023-10-09T10:47:47.400995+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"178'6251","reported_seq":14376,"reported_epoch":178,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:51.496638+0000","last_change":"2023-10-09T10:46:03.683196+0000","last_active":"2023-10-09T10:47:51.496638+0000","last_peered":"2023-10-09T10:47:51.496638+0000","last_clean":"2023-10-09T10:46:02.177309+0000","last_became_active":"2023-10-09T10:46:03.681058+0000","last_became_peered":"2023-10-09T10:46:03.681058+0000","last_unstale":"2023-10-09T10:47:51.496638+0000","last_undegraded":"2023-10-09T10:47:51.496638+0000","last_fullsized":"2023-10-09T10:47:51.496638+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6251,"log_dups_size":0,"ondisk_log_size":6251,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103710720,"num_objects":1583,"num_object_clones":0,"num_object_copies":4749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1583,"num_objects_unfound":0,"num_objects_dirty":1583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5205,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[1,5,6],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1583}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":99,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.387093+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:47:47.387093+0000","last_peered":"2023-10-09T10:47:47.387093+0000","last_clean":"2023-10-09T10:47:47.387093+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:47:47.387093+0000","last_undegraded":"2023-10-09T10:47:47.387093+0000","last_fullsized":"2023-10-09T10:47:47.387093+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":139,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.399973+0000","last_change":"2023-10-09T10:46:32.249921+0000","last_active":"2023-10-09T10:47:47.399973+0000","last_peered":"2023-10-09T10:47:47.399973+0000","last_clean":"2023-10-09T10:47:47.399973+0000","last_became_active":"2023-10-09T10:46:32.249590+0000","last_became_peered":"2023-10-09T10:46:32.249590+0000","last_unstale":"2023-10-09T10:47:47.399973+0000","last_undegraded":"2023-10-09T10:47:47.399973+0000","last_fullsized":"2023-10-09T10:47:47.399973+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:25.169752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"178'6067","reported_seq":14288,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:49.523454+0000","last_change":"2023-10-09T10:47:48.412928+0000","last_active":"2023-10-09T10:47:49.523454+0000","last_peered":"2023-10-09T10:47:49.523454+0000","last_clean":"2023-10-09T10:47:49.523454+0000","last_became_active":"2023-10-09T10:47:48.398813+0000","last_became_peered":"2023-10-09T10:47:48.398813+0000","last_unstale":"2023-10-09T10:47:49.523454+0000","last_undegraded":"2023-10-09T10:47:49.523454+0000","last_fullsized":"2023-10-09T10:47:49.523454+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6067,"log_dups_size":0,"ondisk_log_size":6067,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99188736,"num_objects":1514,"num_object_clones":0,"num_object_copies":4542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5060,"num_write_kb":210336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2691,"num_bytes_recovered":176357376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":148,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386313+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:47:47.386313+0000","last_peered":"2023-10-09T10:47:47.386313+0000","last_clean":"2023-10-09T10:47:47.386313+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:47:47.386313+0000","last_undegraded":"2023-10-09T10:47:47.386313+0000","last_fullsized":"2023-10-09T10:47:47.386313+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":142,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386250+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:47:47.386250+0000","last_peered":"2023-10-09T10:47:47.386250+0000","last_clean":"2023-10-09T10:47:47.386250+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:47:47.386250+0000","last_undegraded":"2023-10-09T10:47:47.386250+0000","last_fullsized":"2023-10-09T10:47:47.386250+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":512,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385954+0000","last_change":"2023-10-09T10:46:05.618276+0000","last_active":"2023-10-09T10:47:48.385954+0000","last_peered":"2023-10-09T10:47:48.385954+0000","last_clean":"2023-10-09T10:47:48.385954+0000","last_became_active":"2023-10-09T10:46:05.616509+0000","last_became_peered":"2023-10-09T10:46:05.616509+0000","last_unstale":"2023-10-09T10:47:48.385954+0000","last_undegraded":"2023-10-09T10:47:48.385954+0000","last_fullsized":"2023-10-09T10:47:48.385954+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.16","version":"178'6071","reported_seq":6173,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:49.469421+0000","last_change":"2023-10-09T10:46:03.592682+0000","last_active":"2023-10-09T10:47:49.469421+0000","last_peered":"2023-10-09T10:47:49.469421+0000","last_clean":"2023-10-09T10:47:49.469421+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:47:49.469421+0000","last_undegraded":"2023-10-09T10:47:49.469421+0000","last_fullsized":"2023-10-09T10:47:49.469421+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6071,"log_dups_size":0,"ondisk_log_size":6071,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102105088,"num_objects":1558,"num_object_clones":0,"num_object_copies":4674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5028,"num_write_kb":210752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":160,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.404363+0000","last_change":"2023-10-09T10:46:32.250256+0000","last_active":"2023-10-09T10:47:47.404363+0000","last_peered":"2023-10-09T10:47:47.404363+0000","last_clean":"2023-10-09T10:47:47.404363+0000","last_became_active":"2023-10-09T10:46:32.243312+0000","last_became_peered":"2023-10-09T10:46:32.243312+0000","last_unstale":"2023-10-09T10:47:47.404363+0000","last_undegraded":"2023-10-09T10:47:47.404363+0000","last_fullsized":"2023-10-09T10:47:47.404363+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:08.059777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":169,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.391381+0000","last_change":"2023-10-09T10:46:32.237577+0000","last_active":"2023-10-09T10:47:47.391381+0000","last_peered":"2023-10-09T10:47:47.391381+0000","last_clean":"2023-10-09T10:47:47.391381+0000","last_became_active":"2023-10-09T10:46:32.237316+0000","last_became_peered":"2023-10-09T10:46:32.237316+0000","last_unstale":"2023-10-09T10:47:47.391381+0000","last_undegraded":"2023-10-09T10:47:47.391381+0000","last_fullsized":"2023-10-09T10:47:47.391381+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:39.684602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384553+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:47:48.384553+0000","last_peered":"2023-10-09T10:47:48.384553+0000","last_clean":"2023-10-09T10:47:48.384553+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:47:48.384553+0000","last_undegraded":"2023-10-09T10:47:48.384553+0000","last_fullsized":"2023-10-09T10:47:48.384553+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.17","version":"178'6045","reported_seq":6142,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:50.875766+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:47:50.875766+0000","last_peered":"2023-10-09T10:47:50.875766+0000","last_clean":"2023-10-09T10:47:50.875766+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:47:50.875766+0000","last_undegraded":"2023-10-09T10:47:50.875766+0000","last_fullsized":"2023-10-09T10:47:50.875766+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6045,"log_dups_size":0,"ondisk_log_size":6045,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96272384,"num_objects":1469,"num_object_clones":0,"num_object_copies":4407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5037,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":47,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386909+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:47:48.386909+0000","last_peered":"2023-10-09T10:47:48.386909+0000","last_clean":"2023-10-09T10:47:48.386909+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:47:48.386909+0000","last_undegraded":"2023-10-09T10:47:48.386909+0000","last_fullsized":"2023-10-09T10:47:48.386909+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":142,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.391620+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:47:47.391620+0000","last_peered":"2023-10-09T10:47:47.391620+0000","last_clean":"2023-10-09T10:47:47.391620+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:47:47.391620+0000","last_undegraded":"2023-10-09T10:47:47.391620+0000","last_fullsized":"2023-10-09T10:47:47.391620+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384741+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:47:48.384741+0000","last_peered":"2023-10-09T10:47:48.384741+0000","last_clean":"2023-10-09T10:47:48.384741+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:47:48.384741+0000","last_undegraded":"2023-10-09T10:47:48.384741+0000","last_fullsized":"2023-10-09T10:47:48.384741+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"178'6201","reported_seq":6303,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:51.480684+0000","last_change":"2023-10-09T10:46:33.243540+0000","last_active":"2023-10-09T10:47:51.480684+0000","last_peered":"2023-10-09T10:47:51.480684+0000","last_clean":"2023-10-09T10:47:51.480684+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:47:51.480684+0000","last_undegraded":"2023-10-09T10:47:51.480684+0000","last_fullsized":"2023-10-09T10:47:51.480684+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6201,"log_dups_size":0,"ondisk_log_size":6201,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100040704,"num_objects":1527,"num_object_clones":0,"num_object_copies":4581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5147,"num_write_kb":213536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":88,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386530+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:47:47.386530+0000","last_peered":"2023-10-09T10:47:47.386530+0000","last_clean":"2023-10-09T10:47:47.386530+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:47:47.386530+0000","last_undegraded":"2023-10-09T10:47:47.386530+0000","last_fullsized":"2023-10-09T10:47:47.386530+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":47,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386007+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:47:48.386007+0000","last_peered":"2023-10-09T10:47:48.386007+0000","last_clean":"2023-10-09T10:47:48.386007+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:47:48.386007+0000","last_undegraded":"2023-10-09T10:47:48.386007+0000","last_fullsized":"2023-10-09T10:47:48.386007+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386682+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:47:48.386682+0000","last_peered":"2023-10-09T10:47:48.386682+0000","last_clean":"2023-10-09T10:47:48.386682+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:47:48.386682+0000","last_undegraded":"2023-10-09T10:47:48.386682+0000","last_fullsized":"2023-10-09T10:47:48.386682+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"178'6179","reported_seq":6291,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:49.519918+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:47:49.519918+0000","last_peered":"2023-10-09T10:47:49.519918+0000","last_clean":"2023-10-09T10:47:49.519918+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:47:49.519918+0000","last_undegraded":"2023-10-09T10:47:49.519918+0000","last_fullsized":"2023-10-09T10:47:49.519918+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6179,"log_dups_size":0,"ondisk_log_size":6179,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101580800,"num_objects":1550,"num_object_clones":0,"num_object_copies":4650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5118,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":57,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384471+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:47:48.384471+0000","last_peered":"2023-10-09T10:47:48.384471+0000","last_clean":"2023-10-09T10:47:48.384471+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:47:48.384471+0000","last_undegraded":"2023-10-09T10:47:48.384471+0000","last_fullsized":"2023-10-09T10:47:48.384471+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":146,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.400179+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:47:47.400179+0000","last_peered":"2023-10-09T10:47:47.400179+0000","last_clean":"2023-10-09T10:47:47.400179+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:47:47.400179+0000","last_undegraded":"2023-10-09T10:47:47.400179+0000","last_fullsized":"2023-10-09T10:47:47.400179+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.387256+0000","last_change":"2023-10-09T10:46:05.620574+0000","last_active":"2023-10-09T10:47:48.387256+0000","last_peered":"2023-10-09T10:47:48.387256+0000","last_clean":"2023-10-09T10:47:48.387256+0000","last_became_active":"2023-10-09T10:46:05.620212+0000","last_became_peered":"2023-10-09T10:46:05.620212+0000","last_unstale":"2023-10-09T10:47:48.387256+0000","last_undegraded":"2023-10-09T10:47:48.387256+0000","last_fullsized":"2023-10-09T10:47:48.387256+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"178'10581","reported_seq":14734,"reported_epoch":178,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:50.998971+0000","last_change":"2023-10-09T10:46:32.247903+0000","last_active":"2023-10-09T10:47:50.998971+0000","last_peered":"2023-10-09T10:47:50.998971+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:46:02.739355+0000","last_became_peered":"2023-10-09T10:46:02.739355+0000","last_unstale":"2023-10-09T10:47:50.998971+0000","last_undegraded":"2023-10-09T10:46:02.608805+0000","last_fullsized":"2023-10-09T10:46:02.583589+0000","mapping_epoch":142,"log_start":"104'1700","ondisk_log_start":"104'1700","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8881,"log_dups_size":0,"ondisk_log_size":8881,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:32.899634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199950336,"num_objects":3051,"num_object_clones":0,"num_object_copies":9153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10581,"num_write_kb":436224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2022,"num_bytes_recovered":132513792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":625},{"shards":"5,6","objects":2425}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":171,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.402215+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:47:47.402215+0000","last_peered":"2023-10-09T10:47:47.402215+0000","last_clean":"2023-10-09T10:47:47.402215+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:47:47.402215+0000","last_undegraded":"2023-10-09T10:47:47.402215+0000","last_fullsized":"2023-10-09T10:47:47.402215+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":125,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.387101+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:47:47.387101+0000","last_peered":"2023-10-09T10:47:47.387101+0000","last_clean":"2023-10-09T10:47:47.387101+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:47:47.387101+0000","last_undegraded":"2023-10-09T10:47:47.387101+0000","last_fullsized":"2023-10-09T10:47:47.387101+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":493,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385497+0000","last_change":"2023-10-09T10:46:02.631378+0000","last_active":"2023-10-09T10:47:48.385497+0000","last_peered":"2023-10-09T10:47:48.385497+0000","last_clean":"2023-10-09T10:47:48.385497+0000","last_became_active":"2023-10-09T10:46:02.629142+0000","last_became_peered":"2023-10-09T10:46:02.629142+0000","last_unstale":"2023-10-09T10:47:48.385497+0000","last_undegraded":"2023-10-09T10:47:48.385497+0000","last_fullsized":"2023-10-09T10:47:48.385497+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.b","version":"178'10054","reported_seq":10143,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:49.552302+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:47:49.552302+0000","last_peered":"2023-10-09T10:47:49.552302+0000","last_clean":"2023-10-09T10:47:49.552302+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:47:49.552302+0000","last_undegraded":"2023-10-09T10:47:49.552302+0000","last_fullsized":"2023-10-09T10:47:49.552302+0000","mapping_epoch":98,"log_start":"124'4325","ondisk_log_start":"124'4325","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5729,"log_dups_size":0,"ondisk_log_size":5729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204865536,"num_objects":3126,"num_object_clones":0,"num_object_copies":9378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10054,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":144,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.402259+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:47:47.402259+0000","last_peered":"2023-10-09T10:47:47.402259+0000","last_clean":"2023-10-09T10:47:47.402259+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:47:47.402259+0000","last_undegraded":"2023-10-09T10:47:47.402259+0000","last_fullsized":"2023-10-09T10:47:47.402259+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":179,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386441+0000","last_change":"2023-10-09T10:46:32.251035+0000","last_active":"2023-10-09T10:47:47.386441+0000","last_peered":"2023-10-09T10:47:47.386441+0000","last_clean":"2023-10-09T10:47:47.386441+0000","last_became_active":"2023-10-09T10:46:32.250293+0000","last_became_peered":"2023-10-09T10:46:32.250293+0000","last_unstale":"2023-10-09T10:47:47.386441+0000","last_undegraded":"2023-10-09T10:47:47.386441+0000","last_fullsized":"2023-10-09T10:47:47.386441+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.787766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386245+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:47:48.386245+0000","last_peered":"2023-10-09T10:47:48.386245+0000","last_clean":"2023-10-09T10:47:48.386245+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:47:48.386245+0000","last_undegraded":"2023-10-09T10:47:48.386245+0000","last_fullsized":"2023-10-09T10:47:48.386245+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"178'6168","reported_seq":6278,"reported_epoch":178,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:49.462632+0000","last_change":"2023-10-09T10:46:33.540719+0000","last_active":"2023-10-09T10:47:49.462632+0000","last_peered":"2023-10-09T10:47:49.462632+0000","last_clean":"2023-10-09T10:46:31.137696+0000","last_became_active":"2023-10-09T10:46:33.527650+0000","last_became_peered":"2023-10-09T10:46:33.527650+0000","last_unstale":"2023-10-09T10:47:49.462632+0000","last_undegraded":"2023-10-09T10:46:33.285468+0000","last_fullsized":"2023-10-09T10:46:33.235683+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6168,"log_dups_size":0,"ondisk_log_size":6168,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103776256,"num_objects":1583,"num_object_clones":0,"num_object_copies":4749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5103,"num_write_kb":213984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1571},{"shards":"3,7","objects":-1226}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":99,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.391550+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:47:47.391550+0000","last_peered":"2023-10-09T10:47:47.391550+0000","last_clean":"2023-10-09T10:47:47.391550+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:47:47.391550+0000","last_undegraded":"2023-10-09T10:47:47.391550+0000","last_fullsized":"2023-10-09T10:47:47.391550+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":116,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.400263+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:47:47.400263+0000","last_peered":"2023-10-09T10:47:47.400263+0000","last_clean":"2023-10-09T10:47:47.400263+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:47:47.400263+0000","last_undegraded":"2023-10-09T10:47:47.400263+0000","last_fullsized":"2023-10-09T10:47:47.400263+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385610+0000","last_change":"2023-10-09T10:46:05.620030+0000","last_active":"2023-10-09T10:47:48.385610+0000","last_peered":"2023-10-09T10:47:48.385610+0000","last_clean":"2023-10-09T10:47:48.385610+0000","last_became_active":"2023-10-09T10:46:05.619694+0000","last_became_peered":"2023-10-09T10:46:05.619694+0000","last_unstale":"2023-10-09T10:47:48.385610+0000","last_undegraded":"2023-10-09T10:47:48.385610+0000","last_fullsized":"2023-10-09T10:47:48.385610+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.9","version":"179'6202","reported_seq":6324,"reported_epoch":179,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:52.429948+0000","last_change":"2023-10-09T10:46:34.405499+0000","last_active":"2023-10-09T10:47:52.429948+0000","last_peered":"2023-10-09T10:47:52.429948+0000","last_clean":"2023-10-09T10:46:31.190603+0000","last_became_active":"2023-10-09T10:46:34.396448+0000","last_became_peered":"2023-10-09T10:46:34.396448+0000","last_unstale":"2023-10-09T10:47:52.429948+0000","last_undegraded":"2023-10-09T10:46:33.246949+0000","last_fullsized":"2023-10-09T10:46:33.237968+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6202,"log_dups_size":0,"ondisk_log_size":6202,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95289344,"num_objects":1454,"num_object_clones":0,"num_object_copies":4362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2004,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5160,"num_write_kb":211648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1303,"num_bytes_recovered":85393408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":151},{"shards":"0,7","objects":-14}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":143,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.394537+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:47:47.394537+0000","last_peered":"2023-10-09T10:47:47.394537+0000","last_clean":"2023-10-09T10:47:47.394537+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:47:47.394537+0000","last_undegraded":"2023-10-09T10:47:47.394537+0000","last_fullsized":"2023-10-09T10:47:47.394537+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":47,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385076+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:47:48.385076+0000","last_peered":"2023-10-09T10:47:48.385076+0000","last_clean":"2023-10-09T10:47:48.385076+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:47:48.385076+0000","last_undegraded":"2023-10-09T10:47:48.385076+0000","last_fullsized":"2023-10-09T10:47:48.385076+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":504,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385702+0000","last_change":"2023-10-09T10:46:05.618134+0000","last_active":"2023-10-09T10:47:48.385702+0000","last_peered":"2023-10-09T10:47:48.385702+0000","last_clean":"2023-10-09T10:47:48.385702+0000","last_became_active":"2023-10-09T10:46:05.616057+0000","last_became_peered":"2023-10-09T10:46:05.616057+0000","last_unstale":"2023-10-09T10:47:48.385702+0000","last_undegraded":"2023-10-09T10:47:48.385702+0000","last_fullsized":"2023-10-09T10:47:48.385702+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"178'10142","reported_seq":10240,"reported_epoch":178,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:51.017643+0000","last_change":"2023-10-09T10:46:33.576530+0000","last_active":"2023-10-09T10:47:51.017643+0000","last_peered":"2023-10-09T10:47:51.017643+0000","last_clean":"2023-10-09T10:46:31.183481+0000","last_became_active":"2023-10-09T10:46:33.542304+0000","last_became_peered":"2023-10-09T10:46:33.542304+0000","last_unstale":"2023-10-09T10:47:51.017643+0000","last_undegraded":"2023-10-09T10:46:33.300581+0000","last_fullsized":"2023-10-09T10:46:33.246047+0000","mapping_epoch":154,"log_start":"113'2777","ondisk_log_start":"113'2777","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7365,"log_dups_size":0,"ondisk_log_size":7365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:26.739488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202637312,"num_objects":3092,"num_object_clones":0,"num_object_copies":9276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6617,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10142,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":3092},{"shards":"1,4","objects":-2469}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":170,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.387534+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:47:47.387534+0000","last_peered":"2023-10-09T10:47:47.387534+0000","last_clean":"2023-10-09T10:47:47.387534+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:47:47.387534+0000","last_undegraded":"2023-10-09T10:47:47.387534+0000","last_fullsized":"2023-10-09T10:47:47.387534+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":142,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386208+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:47:47.386208+0000","last_peered":"2023-10-09T10:47:47.386208+0000","last_clean":"2023-10-09T10:47:47.386208+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:47:47.386208+0000","last_undegraded":"2023-10-09T10:47:47.386208+0000","last_fullsized":"2023-10-09T10:47:47.386208+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":493,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384634+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:47:48.384634+0000","last_peered":"2023-10-09T10:47:48.384634+0000","last_clean":"2023-10-09T10:47:48.384634+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:47:48.384634+0000","last_undegraded":"2023-10-09T10:47:48.384634+0000","last_fullsized":"2023-10-09T10:47:48.384634+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.f","version":"178'9995","reported_seq":10091,"reported_epoch":178,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:49.539585+0000","last_change":"2023-10-09T10:46:33.651756+0000","last_active":"2023-10-09T10:47:49.539585+0000","last_peered":"2023-10-09T10:47:49.539585+0000","last_clean":"2023-10-09T10:46:31.156458+0000","last_became_active":"2023-10-09T10:46:33.621003+0000","last_became_peered":"2023-10-09T10:46:33.621003+0000","last_unstale":"2023-10-09T10:47:49.539585+0000","last_undegraded":"2023-10-09T10:46:33.309813+0000","last_fullsized":"2023-10-09T10:46:33.236508+0000","mapping_epoch":154,"log_start":"124'4310","ondisk_log_start":"124'4310","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5685,"log_dups_size":0,"ondisk_log_size":5685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:13.704346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198377472,"num_objects":3027,"num_object_clones":0,"num_object_copies":9081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9995,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":3027},{"shards":"3,4","objects":-2425}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":117,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386824+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:47:47.386824+0000","last_peered":"2023-10-09T10:47:47.386824+0000","last_clean":"2023-10-09T10:47:47.386824+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:47:47.386824+0000","last_undegraded":"2023-10-09T10:47:47.386824+0000","last_fullsized":"2023-10-09T10:47:47.386824+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":169,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386497+0000","last_change":"2023-10-09T10:46:32.245583+0000","last_active":"2023-10-09T10:47:47.386497+0000","last_peered":"2023-10-09T10:47:47.386497+0000","last_clean":"2023-10-09T10:47:47.386497+0000","last_became_active":"2023-10-09T10:46:32.245271+0000","last_became_peered":"2023-10-09T10:46:32.245271+0000","last_unstale":"2023-10-09T10:47:47.386497+0000","last_undegraded":"2023-10-09T10:47:47.386497+0000","last_fullsized":"2023-10-09T10:47:47.386497+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.025051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384853+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:47:48.384853+0000","last_peered":"2023-10-09T10:47:48.384853+0000","last_clean":"2023-10-09T10:47:48.384853+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:47:48.384853+0000","last_undegraded":"2023-10-09T10:47:48.384853+0000","last_fullsized":"2023-10-09T10:47:48.384853+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"178'10235","reported_seq":10325,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:49.523512+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:47:49.523512+0000","last_peered":"2023-10-09T10:47:49.523512+0000","last_clean":"2023-10-09T10:47:49.523512+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:47:49.523512+0000","last_undegraded":"2023-10-09T10:47:49.523512+0000","last_fullsized":"2023-10-09T10:47:49.523512+0000","mapping_epoch":98,"log_start":"124'4478","ondisk_log_start":"124'4478","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5757,"log_dups_size":0,"ondisk_log_size":5757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199819264,"num_objects":3049,"num_object_clones":0,"num_object_copies":9147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10235,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":153,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386373+0000","last_change":"2023-10-09T10:46:32.245685+0000","last_active":"2023-10-09T10:47:47.386373+0000","last_peered":"2023-10-09T10:47:47.386373+0000","last_clean":"2023-10-09T10:47:47.386373+0000","last_became_active":"2023-10-09T10:46:32.245314+0000","last_became_peered":"2023-10-09T10:46:32.245314+0000","last_unstale":"2023-10-09T10:47:47.386373+0000","last_undegraded":"2023-10-09T10:47:47.386373+0000","last_fullsized":"2023-10-09T10:47:47.386373+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:51.716860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":142,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386838+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:47:47.386838+0000","last_peered":"2023-10-09T10:47:47.386838+0000","last_clean":"2023-10-09T10:47:47.386838+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:47:47.386838+0000","last_undegraded":"2023-10-09T10:47:47.386838+0000","last_fullsized":"2023-10-09T10:47:47.386838+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":504,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386755+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:47:48.386755+0000","last_peered":"2023-10-09T10:47:48.386755+0000","last_clean":"2023-10-09T10:47:48.386755+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:47:48.386755+0000","last_undegraded":"2023-10-09T10:47:48.386755+0000","last_fullsized":"2023-10-09T10:47:48.386755+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"178'10340","reported_seq":10448,"reported_epoch":178,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:51.045269+0000","last_change":"2023-10-09T10:46:02.754507+0000","last_active":"2023-10-09T10:47:51.045269+0000","last_peered":"2023-10-09T10:47:51.045269+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:46:02.747510+0000","last_became_peered":"2023-10-09T10:46:02.747510+0000","last_unstale":"2023-10-09T10:47:51.045269+0000","last_undegraded":"2023-10-09T10:46:02.605987+0000","last_fullsized":"2023-10-09T10:46:02.585248+0000","mapping_epoch":142,"log_start":"104'1500","ondisk_log_start":"104'1500","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8840,"log_dups_size":0,"ondisk_log_size":8840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:59.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200212504,"num_objects":3056,"num_object_clones":0,"num_object_copies":9168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3056,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10340,"num_write_kb":428609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,2,5","objects":2718},{"shards":"1,5","objects":338}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":147,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386070+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:47:47.386070+0000","last_peered":"2023-10-09T10:47:47.386070+0000","last_clean":"2023-10-09T10:47:47.386070+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:47:47.386070+0000","last_undegraded":"2023-10-09T10:47:47.386070+0000","last_fullsized":"2023-10-09T10:47:47.386070+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":130,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.400039+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:47:47.400039+0000","last_peered":"2023-10-09T10:47:47.400039+0000","last_clean":"2023-10-09T10:47:47.400039+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:47:47.400039+0000","last_undegraded":"2023-10-09T10:47:47.400039+0000","last_fullsized":"2023-10-09T10:47:47.400039+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386394+0000","last_change":"2023-10-09T10:46:05.617983+0000","last_active":"2023-10-09T10:47:48.386394+0000","last_peered":"2023-10-09T10:47:48.386394+0000","last_clean":"2023-10-09T10:47:48.386394+0000","last_became_active":"2023-10-09T10:46:05.616021+0000","last_became_peered":"2023-10-09T10:46:05.616021+0000","last_unstale":"2023-10-09T10:47:48.386394+0000","last_undegraded":"2023-10-09T10:47:48.386394+0000","last_fullsized":"2023-10-09T10:47:48.386394+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":173,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.403041+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:47:47.403041+0000","last_peered":"2023-10-09T10:47:47.403041+0000","last_clean":"2023-10-09T10:47:47.403041+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:47:47.403041+0000","last_undegraded":"2023-10-09T10:47:47.403041+0000","last_fullsized":"2023-10-09T10:47:47.403041+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":170,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.391689+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:47:47.391689+0000","last_peered":"2023-10-09T10:47:47.391689+0000","last_clean":"2023-10-09T10:47:47.391689+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:47:47.391689+0000","last_undegraded":"2023-10-09T10:47:47.391689+0000","last_fullsized":"2023-10-09T10:47:47.391689+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":115,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.400057+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:47:47.400057+0000","last_peered":"2023-10-09T10:47:47.400057+0000","last_clean":"2023-10-09T10:47:47.400057+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:47:47.400057+0000","last_undegraded":"2023-10-09T10:47:47.400057+0000","last_fullsized":"2023-10-09T10:47:47.400057+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"178'6091","reported_seq":10125,"reported_epoch":178,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:49.757867+0000","last_change":"2023-10-09T10:46:32.248218+0000","last_active":"2023-10-09T10:47:49.757867+0000","last_peered":"2023-10-09T10:47:49.757867+0000","last_clean":"2023-10-09T10:46:02.181912+0000","last_became_active":"2023-10-09T10:46:32.226237+0000","last_became_peered":"2023-10-09T10:46:32.226237+0000","last_unstale":"2023-10-09T10:47:49.757867+0000","last_undegraded":"2023-10-09T10:47:49.757867+0000","last_fullsized":"2023-10-09T10:47:49.757867+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6091,"log_dups_size":0,"ondisk_log_size":6091,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104890368,"num_objects":1600,"num_object_clones":0,"num_object_copies":4800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1600,"num_objects_unfound":0,"num_objects_dirty":1600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5044,"num_write_kb":212640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1600}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":512,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384554+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:47:48.384554+0000","last_peered":"2023-10-09T10:47:48.384554+0000","last_clean":"2023-10-09T10:47:48.384554+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:47:48.384554+0000","last_undegraded":"2023-10-09T10:47:48.384554+0000","last_fullsized":"2023-10-09T10:47:48.384554+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"178'6173","reported_seq":6353,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:49.542092+0000","last_change":"2023-10-09T10:46:02.760963+0000","last_active":"2023-10-09T10:47:49.542092+0000","last_peered":"2023-10-09T10:47:49.542092+0000","last_clean":"2023-10-09T10:47:49.542092+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:47:49.542092+0000","last_undegraded":"2023-10-09T10:47:49.542092+0000","last_fullsized":"2023-10-09T10:47:49.542092+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"141'4416","last_scrub_stamp":"2023-10-09T10:46:02.760810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:46:02.760810+0000","objects_scrubbed":3378,"log_size":6173,"log_dups_size":0,"ondisk_log_size":6173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:35.863002+0000","scrub_duration":0.47769323899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106233856,"num_objects":1621,"num_object_clones":0,"num_object_copies":4863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1757,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":147,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.388947+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:47:47.388947+0000","last_peered":"2023-10-09T10:47:47.388947+0000","last_clean":"2023-10-09T10:47:47.388947+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:47:47.388947+0000","last_undegraded":"2023-10-09T10:47:47.388947+0000","last_fullsized":"2023-10-09T10:47:47.388947+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":163,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386424+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:47:47.386424+0000","last_peered":"2023-10-09T10:47:47.386424+0000","last_clean":"2023-10-09T10:47:47.386424+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:47:47.386424+0000","last_undegraded":"2023-10-09T10:47:47.386424+0000","last_fullsized":"2023-10-09T10:47:47.386424+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":154,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.403106+0000","last_change":"2023-10-09T10:46:32.245608+0000","last_active":"2023-10-09T10:47:47.403106+0000","last_peered":"2023-10-09T10:47:47.403106+0000","last_clean":"2023-10-09T10:47:47.403106+0000","last_became_active":"2023-10-09T10:46:32.245290+0000","last_became_peered":"2023-10-09T10:46:32.245290+0000","last_unstale":"2023-10-09T10:47:47.403106+0000","last_undegraded":"2023-10-09T10:47:47.403106+0000","last_fullsized":"2023-10-09T10:47:47.403106+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:48.832515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386834+0000","last_change":"2023-10-09T10:46:05.620057+0000","last_active":"2023-10-09T10:47:48.386834+0000","last_peered":"2023-10-09T10:47:48.386834+0000","last_clean":"2023-10-09T10:47:48.386834+0000","last_became_active":"2023-10-09T10:46:05.619703+0000","last_became_peered":"2023-10-09T10:46:05.619703+0000","last_unstale":"2023-10-09T10:47:48.386834+0000","last_undegraded":"2023-10-09T10:47:48.386834+0000","last_fullsized":"2023-10-09T10:47:48.386834+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"178'6148","reported_seq":6263,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:49.527770+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:47:49.527770+0000","last_peered":"2023-10-09T10:47:49.527770+0000","last_clean":"2023-10-09T10:47:49.527770+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:47:49.527770+0000","last_undegraded":"2023-10-09T10:47:49.527770+0000","last_fullsized":"2023-10-09T10:47:49.527770+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6148,"log_dups_size":0,"ondisk_log_size":6148,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104398848,"num_objects":1593,"num_object_clones":0,"num_object_copies":4779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5087,"num_write_kb":213760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":175,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.401112+0000","last_change":"2023-10-09T10:46:32.241041+0000","last_active":"2023-10-09T10:47:47.401112+0000","last_peered":"2023-10-09T10:47:47.401112+0000","last_clean":"2023-10-09T10:47:47.401112+0000","last_became_active":"2023-10-09T10:46:32.238683+0000","last_became_peered":"2023-10-09T10:46:32.238683+0000","last_unstale":"2023-10-09T10:47:47.401112+0000","last_undegraded":"2023-10-09T10:47:47.401112+0000","last_fullsized":"2023-10-09T10:47:47.401112+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:25.817713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":139,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.401636+0000","last_change":"2023-10-09T10:46:32.250104+0000","last_active":"2023-10-09T10:47:47.401636+0000","last_peered":"2023-10-09T10:47:47.401636+0000","last_clean":"2023-10-09T10:47:47.401636+0000","last_became_active":"2023-10-09T10:46:32.249805+0000","last_became_peered":"2023-10-09T10:46:32.249805+0000","last_unstale":"2023-10-09T10:47:47.401636+0000","last_undegraded":"2023-10-09T10:47:47.401636+0000","last_fullsized":"2023-10-09T10:47:47.401636+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:07.230672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":168,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.400140+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:47:47.400140+0000","last_peered":"2023-10-09T10:47:47.400140+0000","last_clean":"2023-10-09T10:47:47.400140+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:47:47.400140+0000","last_undegraded":"2023-10-09T10:47:47.400140+0000","last_fullsized":"2023-10-09T10:47:47.400140+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"178'6036","reported_seq":6138,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:50.702645+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:47:50.702645+0000","last_peered":"2023-10-09T10:47:50.702645+0000","last_clean":"2023-10-09T10:47:50.702645+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:47:50.702645+0000","last_undegraded":"2023-10-09T10:47:50.702645+0000","last_fullsized":"2023-10-09T10:47:50.702645+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6036,"log_dups_size":0,"ondisk_log_size":6036,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106299392,"num_objects":1622,"num_object_clones":0,"num_object_copies":4866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5028,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":157,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.387044+0000","last_change":"2023-10-09T10:46:32.235346+0000","last_active":"2023-10-09T10:47:47.387044+0000","last_peered":"2023-10-09T10:47:47.387044+0000","last_clean":"2023-10-09T10:47:47.387044+0000","last_became_active":"2023-10-09T10:46:32.234906+0000","last_became_peered":"2023-10-09T10:46:32.234906+0000","last_unstale":"2023-10-09T10:47:47.387044+0000","last_undegraded":"2023-10-09T10:47:47.387044+0000","last_fullsized":"2023-10-09T10:47:47.387044+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.199251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384317+0000","last_change":"2023-10-09T10:46:05.617146+0000","last_active":"2023-10-09T10:47:48.384317+0000","last_peered":"2023-10-09T10:47:48.384317+0000","last_clean":"2023-10-09T10:47:48.384317+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:47:48.384317+0000","last_undegraded":"2023-10-09T10:47:48.384317+0000","last_fullsized":"2023-10-09T10:47:48.384317+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":106,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.387169+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:47:47.387169+0000","last_peered":"2023-10-09T10:47:47.387169+0000","last_clean":"2023-10-09T10:47:47.387169+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:47:47.387169+0000","last_undegraded":"2023-10-09T10:47:47.387169+0000","last_fullsized":"2023-10-09T10:47:47.387169+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":75,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386365+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:47:47.386365+0000","last_peered":"2023-10-09T10:47:47.386365+0000","last_clean":"2023-10-09T10:47:47.386365+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:47:47.386365+0000","last_undegraded":"2023-10-09T10:47:47.386365+0000","last_fullsized":"2023-10-09T10:47:47.386365+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384810+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:47:48.384810+0000","last_peered":"2023-10-09T10:47:48.384810+0000","last_clean":"2023-10-09T10:47:48.384810+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:47:48.384810+0000","last_undegraded":"2023-10-09T10:47:48.384810+0000","last_fullsized":"2023-10-09T10:47:48.384810+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":169,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.401747+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:47:47.401747+0000","last_peered":"2023-10-09T10:47:47.401747+0000","last_clean":"2023-10-09T10:47:47.401747+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:47:47.401747+0000","last_undegraded":"2023-10-09T10:47:47.401747+0000","last_fullsized":"2023-10-09T10:47:47.401747+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":56,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384501+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:47:48.384501+0000","last_peered":"2023-10-09T10:47:48.384501+0000","last_clean":"2023-10-09T10:47:48.384501+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:47:48.384501+0000","last_undegraded":"2023-10-09T10:47:48.384501+0000","last_fullsized":"2023-10-09T10:47:48.384501+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":56,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385932+0000","last_change":"2023-10-09T10:46:32.236202+0000","last_active":"2023-10-09T10:47:48.385932+0000","last_peered":"2023-10-09T10:47:48.385932+0000","last_clean":"2023-10-09T10:47:48.385932+0000","last_became_active":"2023-10-09T10:46:32.235901+0000","last_became_peered":"2023-10-09T10:46:32.235901+0000","last_unstale":"2023-10-09T10:47:48.385932+0000","last_undegraded":"2023-10-09T10:47:48.385932+0000","last_fullsized":"2023-10-09T10:47:48.385932+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:48.260403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"178'6221","reported_seq":6370,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:51.329185+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:47:51.329185+0000","last_peered":"2023-10-09T10:47:51.329185+0000","last_clean":"2023-10-09T10:47:51.329185+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:47:51.329185+0000","last_undegraded":"2023-10-09T10:47:51.329185+0000","last_fullsized":"2023-10-09T10:47:51.329185+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":6221,"log_dups_size":0,"ondisk_log_size":6221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102039552,"num_objects":1557,"num_object_clones":0,"num_object_copies":4671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3191,"num_write_kb":89664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385693+0000","last_change":"2023-10-09T10:46:05.620764+0000","last_active":"2023-10-09T10:47:48.385693+0000","last_peered":"2023-10-09T10:47:48.385693+0000","last_clean":"2023-10-09T10:47:48.385693+0000","last_became_active":"2023-10-09T10:46:05.619870+0000","last_became_peered":"2023-10-09T10:46:05.619870+0000","last_unstale":"2023-10-09T10:47:48.385693+0000","last_undegraded":"2023-10-09T10:47:48.385693+0000","last_fullsized":"2023-10-09T10:47:48.385693+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":166,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.402788+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:47:47.402788+0000","last_peered":"2023-10-09T10:47:47.402788+0000","last_clean":"2023-10-09T10:47:47.402788+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:47:47.402788+0000","last_undegraded":"2023-10-09T10:47:47.402788+0000","last_fullsized":"2023-10-09T10:47:47.402788+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":139,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.403185+0000","last_change":"2023-10-09T10:46:32.251450+0000","last_active":"2023-10-09T10:47:47.403185+0000","last_peered":"2023-10-09T10:47:47.403185+0000","last_clean":"2023-10-09T10:47:47.403185+0000","last_became_active":"2023-10-09T10:46:32.250933+0000","last_became_peered":"2023-10-09T10:46:32.250933+0000","last_unstale":"2023-10-09T10:47:47.403185+0000","last_undegraded":"2023-10-09T10:47:47.403185+0000","last_fullsized":"2023-10-09T10:47:47.403185+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:04.116178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":47,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384918+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:47:48.384918+0000","last_peered":"2023-10-09T10:47:48.384918+0000","last_clean":"2023-10-09T10:47:48.384918+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:47:48.384918+0000","last_undegraded":"2023-10-09T10:47:48.384918+0000","last_fullsized":"2023-10-09T10:47:48.384918+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"178'6057","reported_seq":12751,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:51.473320+0000","last_change":"2023-10-09T10:47:02.269289+0000","last_active":"2023-10-09T10:47:51.473320+0000","last_peered":"2023-10-09T10:47:51.473320+0000","last_clean":"2023-10-09T10:47:51.473320+0000","last_became_active":"2023-10-09T10:47:02.257854+0000","last_became_peered":"2023-10-09T10:47:02.257854+0000","last_unstale":"2023-10-09T10:47:51.473320+0000","last_undegraded":"2023-10-09T10:47:51.473320+0000","last_fullsized":"2023-10-09T10:47:51.473320+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6057,"log_dups_size":0,"ondisk_log_size":6057,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101810176,"num_objects":1553,"num_object_clones":0,"num_object_copies":4659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5049,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3287,"num_bytes_recovered":181927936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.387103+0000","last_change":"2023-10-09T10:46:05.618443+0000","last_active":"2023-10-09T10:47:48.387103+0000","last_peered":"2023-10-09T10:47:48.387103+0000","last_clean":"2023-10-09T10:47:48.387103+0000","last_became_active":"2023-10-09T10:46:05.616588+0000","last_became_peered":"2023-10-09T10:46:05.616588+0000","last_unstale":"2023-10-09T10:47:48.387103+0000","last_undegraded":"2023-10-09T10:47:48.387103+0000","last_fullsized":"2023-10-09T10:47:48.387103+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":170,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.400103+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:47:47.400103+0000","last_peered":"2023-10-09T10:47:47.400103+0000","last_clean":"2023-10-09T10:47:47.400103+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:47:47.400103+0000","last_undegraded":"2023-10-09T10:47:47.400103+0000","last_fullsized":"2023-10-09T10:47:47.400103+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":115,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.403384+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:47:47.403384+0000","last_peered":"2023-10-09T10:47:47.403384+0000","last_clean":"2023-10-09T10:47:47.403384+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:47:47.403384+0000","last_undegraded":"2023-10-09T10:47:47.403384+0000","last_fullsized":"2023-10-09T10:47:47.403384+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":160,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386145+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:47:47.386145+0000","last_peered":"2023-10-09T10:47:47.386145+0000","last_clean":"2023-10-09T10:47:47.386145+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:47:47.386145+0000","last_undegraded":"2023-10-09T10:47:47.386145+0000","last_fullsized":"2023-10-09T10:47:47.386145+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"178'6200","reported_seq":8739,"reported_epoch":178,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:51.033937+0000","last_change":"2023-10-09T10:47:29.622439+0000","last_active":"2023-10-09T10:47:51.033937+0000","last_peered":"2023-10-09T10:47:51.033937+0000","last_clean":"2023-10-09T10:46:31.130083+0000","last_became_active":"2023-10-09T10:46:33.493288+0000","last_became_peered":"2023-10-09T10:46:33.493288+0000","last_unstale":"2023-10-09T10:47:51.033937+0000","last_undegraded":"2023-10-09T10:46:33.283624+0000","last_fullsized":"2023-10-09T10:46:33.244043+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6200,"log_dups_size":0,"ondisk_log_size":6200,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98402304,"num_objects":1501,"num_object_clones":0,"num_object_copies":4503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":838,"num_objects_misplaced":782,"num_objects_unfound":0,"num_objects_dirty":1501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5163,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1195,"num_bytes_recovered":78315520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":782},{"shards":"3,4","objects":-119}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":512,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385223+0000","last_change":"2023-10-09T10:46:05.617589+0000","last_active":"2023-10-09T10:47:48.385223+0000","last_peered":"2023-10-09T10:47:48.385223+0000","last_clean":"2023-10-09T10:47:48.385223+0000","last_became_active":"2023-10-09T10:46:05.616400+0000","last_became_peered":"2023-10-09T10:46:05.616400+0000","last_unstale":"2023-10-09T10:47:48.385223+0000","last_undegraded":"2023-10-09T10:47:48.385223+0000","last_fullsized":"2023-10-09T10:47:48.385223+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":35,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.394768+0000","last_change":"2023-10-09T10:46:32.245106+0000","last_active":"2023-10-09T10:47:47.394768+0000","last_peered":"2023-10-09T10:47:47.394768+0000","last_clean":"2023-10-09T10:47:47.394768+0000","last_became_active":"2023-10-09T10:46:32.244826+0000","last_became_peered":"2023-10-09T10:46:32.244826+0000","last_unstale":"2023-10-09T10:47:47.394768+0000","last_undegraded":"2023-10-09T10:47:47.394768+0000","last_fullsized":"2023-10-09T10:47:47.394768+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:30.322834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":51,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385854+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:47:48.385854+0000","last_peered":"2023-10-09T10:47:48.385854+0000","last_clean":"2023-10-09T10:47:48.385854+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:47:48.385854+0000","last_undegraded":"2023-10-09T10:47:48.385854+0000","last_fullsized":"2023-10-09T10:47:48.385854+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":125,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386295+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:47:47.386295+0000","last_peered":"2023-10-09T10:47:47.386295+0000","last_clean":"2023-10-09T10:47:47.386295+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:47:47.386295+0000","last_undegraded":"2023-10-09T10:47:47.386295+0000","last_fullsized":"2023-10-09T10:47:47.386295+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"178'6060","reported_seq":12867,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:50.821934+0000","last_change":"2023-10-09T10:47:31.271196+0000","last_active":"2023-10-09T10:47:50.821934+0000","last_peered":"2023-10-09T10:47:50.821934+0000","last_clean":"2023-10-09T10:47:50.821934+0000","last_became_active":"2023-10-09T10:47:31.229504+0000","last_became_peered":"2023-10-09T10:47:31.229504+0000","last_unstale":"2023-10-09T10:47:50.821934+0000","last_undegraded":"2023-10-09T10:47:50.821934+0000","last_fullsized":"2023-10-09T10:47:50.821934+0000","mapping_epoch":172,"log_start":"103'714","ondisk_log_start":"103'714","created":98,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":5346,"log_dups_size":0,"ondisk_log_size":5346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:26.533476+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102629376,"num_objects":1566,"num_object_clones":0,"num_object_copies":4698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3415,"num_write_kb":107264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3305,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":47,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384996+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:47:48.384996+0000","last_peered":"2023-10-09T10:47:48.384996+0000","last_clean":"2023-10-09T10:47:48.384996+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:47:48.384996+0000","last_undegraded":"2023-10-09T10:47:48.384996+0000","last_fullsized":"2023-10-09T10:47:48.384996+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":109,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.403259+0000","last_change":"2023-10-09T10:46:32.251242+0000","last_active":"2023-10-09T10:47:47.403259+0000","last_peered":"2023-10-09T10:47:47.403259+0000","last_clean":"2023-10-09T10:47:47.403259+0000","last_became_active":"2023-10-09T10:46:32.250534+0000","last_became_peered":"2023-10-09T10:46:32.250534+0000","last_unstale":"2023-10-09T10:47:47.403259+0000","last_undegraded":"2023-10-09T10:47:47.403259+0000","last_fullsized":"2023-10-09T10:47:47.403259+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:33.726041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"179'6172","reported_seq":10155,"reported_epoch":179,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:52.524711+0000","last_change":"2023-10-09T10:46:34.514479+0000","last_active":"2023-10-09T10:47:52.524711+0000","last_peered":"2023-10-09T10:47:52.524711+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:46:34.409667+0000","last_became_peered":"2023-10-09T10:46:34.409667+0000","last_unstale":"2023-10-09T10:47:52.524711+0000","last_undegraded":"2023-10-09T10:46:34.376635+0000","last_fullsized":"2023-10-09T10:46:33.237998+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6172,"log_dups_size":0,"ondisk_log_size":6172,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101187584,"num_objects":1544,"num_object_clones":0,"num_object_copies":4632,"num_objects_missing_on_primary":1406,"num_objects_missing":1406,"num_objects_degraded":1916,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5130,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1932,"num_bytes_recovered":126615552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,7","objects":138},{"shards":"2,3,7","objects":1406}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":145,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.394657+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:47:47.394657+0000","last_peered":"2023-10-09T10:47:47.394657+0000","last_clean":"2023-10-09T10:47:47.394657+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:47:47.394657+0000","last_undegraded":"2023-10-09T10:47:47.394657+0000","last_fullsized":"2023-10-09T10:47:47.394657+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":159,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386585+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:47:47.386585+0000","last_peered":"2023-10-09T10:47:47.386585+0000","last_clean":"2023-10-09T10:47:47.386585+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:47:47.386585+0000","last_undegraded":"2023-10-09T10:47:47.386585+0000","last_fullsized":"2023-10-09T10:47:47.386585+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"178'6048","reported_seq":6158,"reported_epoch":178,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:49.550569+0000","last_change":"2023-10-09T10:46:33.546991+0000","last_active":"2023-10-09T10:47:49.550569+0000","last_peered":"2023-10-09T10:47:49.550569+0000","last_clean":"2023-10-09T10:46:31.009484+0000","last_became_active":"2023-10-09T10:46:33.535665+0000","last_became_peered":"2023-10-09T10:46:33.535665+0000","last_unstale":"2023-10-09T10:47:49.550569+0000","last_undegraded":"2023-10-09T10:46:33.277168+0000","last_fullsized":"2023-10-09T10:46:33.235038+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6048,"log_dups_size":0,"ondisk_log_size":6048,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101154816,"num_objects":1544,"num_object_clones":0,"num_object_copies":4632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4984,"num_write_kb":208864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1556},{"shards":"3,7","objects":-1214}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":128,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.391761+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:47:47.391761+0000","last_peered":"2023-10-09T10:47:47.391761+0000","last_clean":"2023-10-09T10:47:47.391761+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:47:47.391761+0000","last_undegraded":"2023-10-09T10:47:47.391761+0000","last_fullsized":"2023-10-09T10:47:47.391761+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":26,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386756+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:47:47.386756+0000","last_peered":"2023-10-09T10:47:47.386756+0000","last_clean":"2023-10-09T10:47:47.386756+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:47:47.386756+0000","last_undegraded":"2023-10-09T10:47:47.386756+0000","last_fullsized":"2023-10-09T10:47:47.386756+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":162,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.387841+0000","last_change":"2023-10-09T10:46:32.244607+0000","last_active":"2023-10-09T10:47:47.387841+0000","last_peered":"2023-10-09T10:47:47.387841+0000","last_clean":"2023-10-09T10:47:47.387841+0000","last_became_active":"2023-10-09T10:46:32.242628+0000","last_became_peered":"2023-10-09T10:46:32.242628+0000","last_unstale":"2023-10-09T10:47:47.387841+0000","last_undegraded":"2023-10-09T10:47:47.387841+0000","last_fullsized":"2023-10-09T10:47:47.387841+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:10.738819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":192,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.400931+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:47:47.400931+0000","last_peered":"2023-10-09T10:47:47.400931+0000","last_clean":"2023-10-09T10:47:47.400931+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:47:47.400931+0000","last_undegraded":"2023-10-09T10:47:47.400931+0000","last_fullsized":"2023-10-09T10:47:47.400931+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":68,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386325+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:47:48.386325+0000","last_peered":"2023-10-09T10:47:48.386325+0000","last_clean":"2023-10-09T10:47:48.386325+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:47:48.386325+0000","last_undegraded":"2023-10-09T10:47:48.386325+0000","last_fullsized":"2023-10-09T10:47:48.386325+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":142,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-09T10:47:47.386167+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:47:47.386167+0000","last_peered":"2023-10-09T10:47:47.386167+0000","last_clean":"2023-10-09T10:47:47.386167+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:47:47.386167+0000","last_undegraded":"2023-10-09T10:47:47.386167+0000","last_fullsized":"2023-10-09T10:47:47.386167+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":3244589080,"num_objects":49510,"num_object_clones":0,"num_object_copies":148530,"num_objects_missing_on_primary":1406,"num_objects_missing":1406,"num_objects_degraded":31152,"num_objects_misplaced":5570,"num_objects_unfound":0,"num_objects_dirty":49510,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":154698,"num_write_kb":6258529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23019,"num_bytes_recovered":1415905280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11827818496,"data_stored":11827806280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":163584,"ondisk_log_size":163584,"up":78,"acting":68,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1186,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332255,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1703680,"kb_used_data":1394040,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92041472,"statfs":{"total":95995035648,"available":94250467328,"internally_reserved":0,"allocated":1427496960,"data_stored":1426819486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":0.79200000000000004,"15min":0.65000000000000002},"min":{"1min":0.26300000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.1479999999999997,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":1.8500000000000001},{"interface":"front","average":{"1min":1.546,"5min":0.97199999999999998,"15min":0.752},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.9909999999999997,"5min":10.207000000000001,"15min":10.207000000000001},"last":0.33800000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:47:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2210000000000001,"5min":1.3300000000000001,"15min":0.80400000000000005},"min":{"1min":0.43099999999999999,"5min":0.318,"15min":0.318},"max":{"1min":11.826000000000001,"5min":11.826000000000001,"15min":11.826000000000001},"last":0.30199999999999999},{"interface":"front","average":{"1min":2.286,"5min":1.155,"15min":0.73899999999999999},"min":{"1min":0.32100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":11.84,"5min":11.84,"15min":11.84},"last":1.8160000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.1499999999999999,"15min":0.72399999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.4770000000000003,"5min":6.4770000000000003,"15min":6.4770000000000003},"last":0.317},{"interface":"front","average":{"1min":1.272,"5min":0.85699999999999998,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.251,"15min":0.251},"max":{"1min":3.1480000000000001,"5min":3.7120000000000002,"15min":3.7120000000000002},"last":1.8799999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.23,"15min":0.80000000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.0599999999999996,"5min":8.1639999999999997,"15min":8.1639999999999997},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.099,"15min":0.76600000000000001},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.504,"5min":12.451000000000001,"15min":12.451000000000001},"last":0.35299999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.73299999999999998,"15min":0.56200000000000006},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.1420000000000003,"5min":7.4509999999999996,"15min":7.4509999999999996},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.65300000000000002,"15min":0.53600000000000003},"min":{"1min":0.29799999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.2749999999999999,"5min":3.1699999999999999,"15min":3.1699999999999999},"last":0.378}]},{"osd":5,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.56399999999999995,"15min":0.52700000000000002},"min":{"1min":0.34300000000000003,"5min":0.247,"15min":0.247},"max":{"1min":2.2349999999999999,"5min":2.2349999999999999,"15min":2.2349999999999999},"last":12.064},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.81299999999999994,"15min":0.63},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":5.0860000000000003,"5min":10.33,"15min":10.33},"last":9.9670000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.83399999999999996,"15min":0.64800000000000002},"min":{"1min":0.311,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.109,"5min":10.164999999999999,"15min":10.164999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.66000000000000003,"15min":0.58299999999999996},"min":{"1min":0.33300000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.306,"5min":2.8079999999999998,"15min":2.8079999999999998},"last":0.41299999999999998}]}]},{"osd":6,"up_from":44,"seq":188978561186,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1433916,"kb_used_data":1128436,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92311236,"statfs":{"total":95995035648,"available":94526705664,"internally_reserved":0,"allocated":1155518464,"data_stored":1154845528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.93200000000000005,"15min":0.71799999999999997},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.4630000000000001,"5min":7.3220000000000001,"15min":7.3220000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.02,"5min":1.002,"15min":0.73899999999999999},"min":{"1min":0.33900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.4969999999999999,"5min":7.5309999999999997,"15min":7.5309999999999997},"last":0.54800000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.157,"15min":0.77900000000000003},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.245},"max":{"1min":6.7560000000000002,"5min":16.747,"15min":16.747},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.284,"15min":0.79500000000000004},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.3579999999999997,"5min":16.734000000000002,"15min":16.734000000000002},"last":1.647}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":0.95799999999999996,"15min":0.67500000000000004},"min":{"1min":0.28699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5069999999999997,"5min":7.5069999999999997,"15min":7.5069999999999997},"last":1.7270000000000001},{"interface":"front","average":{"1min":1.675,"5min":1.143,"15min":0.755},"min":{"1min":0.28599999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.3899999999999997,"5min":12.571,"15min":12.571},"last":7.6749999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.117,"15min":0.73099999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.8070000000000004,"5min":6.8070000000000004,"15min":6.8070000000000004},"last":0.48599999999999999},{"interface":"front","average":{"1min":2.1219999999999999,"5min":1.385,"15min":0.82599999999999996},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.3940000000000001,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":1.6919999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":0.66600000000000004,"15min":0.51500000000000001},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":4.0739999999999998,"5min":4.0739999999999998,"15min":4.0739999999999998},"last":9.4060000000000006},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.67300000000000004,"15min":0.54600000000000004},"min":{"1min":0.34899999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6600000000000001,"5min":3.6600000000000001,"15min":3.6600000000000001},"last":0.56899999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":0.76000000000000001,"15min":0.56999999999999995},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.8949999999999996,"5min":4.8949999999999996,"15min":4.8949999999999996},"last":8.6969999999999992},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.76900000000000002,"15min":0.59999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":4.3639999999999999,"5min":4.3639999999999999,"15min":4.3639999999999999},"last":0.79600000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:47:25 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69899999999999995,"15min":0.57999999999999996},"min":{"1min":0.35799999999999998,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":2.5049999999999999,"5min":3.472,"15min":3.472},"last":23.358000000000001},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.84299999999999997,"15min":0.65700000000000003},"min":{"1min":0.373,"5min":0.36899999999999999,"15min":0.34000000000000002},"max":{"1min":4.0800000000000001,"5min":4.0800000000000001,"15min":4.0800000000000001},"last":0.66800000000000004}]}]},{"osd":1,"up_from":85,"seq":365072220225,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1430188,"kb_used_data":1304292,"kb_used_omap":9,"kb_used_meta":125878,"kb_avail":92314964,"statfs":{"total":95995035648,"available":94530523136,"internally_reserved":0,"allocated":1335595008,"data_stored":1334705939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":128900073},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.581,"15min":0.76900000000000002},"min":{"1min":0.251,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.2119999999999997,"5min":14.1,"15min":14.1},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.885,"5min":1.5629999999999999,"15min":0.78700000000000003},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.8090000000000002,"5min":14.465,"15min":14.465},"last":5.8079999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:47:13 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.1200000000000001,"15min":0.61799999999999999},"min":{"1min":0.21199999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.4829999999999997,"5min":8.6859999999999999,"15min":8.6859999999999999},"last":2.0680000000000001},{"interface":"front","average":{"1min":1.919,"5min":1.4690000000000001,"15min":0.746},"min":{"1min":0.29099999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.2829999999999995,"5min":14.234,"15min":14.234},"last":0.38100000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":1.5269999999999999,"15min":0.82299999999999995},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":7.234,"5min":14.404999999999999,"15min":14.404999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.429,"15min":0.70599999999999996},"min":{"1min":0.30399999999999999,"5min":0.251,"15min":0.251},"max":{"1min":15.468999999999999,"5min":15.468999999999999,"15min":15.468999999999999},"last":5.6100000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.1679999999999999,"15min":0.78400000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.1449999999999996,"5min":8.0129999999999999,"15min":8.0129999999999999},"last":2.1099999999999999},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.2350000000000001,"15min":0.82499999999999996},"min":{"1min":0.30299999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.1189999999999998,"5min":8.327,"15min":8.327},"last":1.8149999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":1.111,"15min":0.78900000000000003},"min":{"1min":0.28899999999999998,"5min":0.253,"15min":0.253},"max":{"1min":5.4850000000000003,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":9.4809999999999999},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.2929999999999999,"15min":0.86199999999999999},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.1859999999999999,"5min":7.923,"15min":7.923},"last":6.9400000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7909999999999999,"5min":1.2030000000000001,"15min":0.82199999999999995},"min":{"1min":0.35499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.9450000000000003,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":6.6340000000000003},{"interface":"front","average":{"1min":1.246,"5min":1.2949999999999999,"15min":0.86299999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.8440000000000003,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":6.5190000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.22,"15min":0.83999999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.8689999999999998,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":2.141},{"interface":"front","average":{"1min":1.992,"5min":1.2909999999999999,"15min":0.86099999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":7.1589999999999998,"5min":8.2469999999999999,"15min":8.2469999999999999},"last":0.48699999999999999}]}]},{"osd":0,"up_from":80,"seq":343597383759,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":630284,"kb_used_data":540420,"kb_used_omap":28,"kb_used_meta":89827,"kb_avail":93114868,"statfs":{"total":95995035648,"available":95349624832,"internally_reserved":0,"allocated":553390080,"data_stored":552468372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":91983696},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:47:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2509999999999999,"5min":1.5449999999999999,"15min":0.76000000000000001},"min":{"1min":0.22,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":10.564,"5min":25.475000000000001,"15min":25.475000000000001},"last":2.0720000000000001},{"interface":"front","average":{"1min":2.3740000000000001,"5min":1.5940000000000001,"15min":0.77400000000000002},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":10.593999999999999,"5min":20.832999999999998,"15min":20.832999999999998},"last":2.0859999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":1.2350000000000001,"15min":0.64400000000000002},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.907,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":1.7769999999999999},{"interface":"front","average":{"1min":1.754,"5min":1.5349999999999999,"15min":0.73999999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.5619999999999994,"5min":14.391999999999999,"15min":14.391999999999999},"last":3.6600000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:47:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4740000000000002,"5min":1.8959999999999999,"15min":1.0660000000000001},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.215},"max":{"1min":10.644,"5min":14.926,"15min":14.926},"last":0.46200000000000002},{"interface":"front","average":{"1min":2.2269999999999999,"5min":1.5940000000000001,"15min":0.89200000000000002},"min":{"1min":0.34499999999999997,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.228999999999999,"5min":11.868,"15min":11.868},"last":0.224}]},{"osd":4,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.2250000000000001,"15min":0.82199999999999995},"min":{"1min":0.38400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.8150000000000004,"5min":6.8150000000000004,"15min":6.8150000000000004},"last":3.7080000000000002},{"interface":"front","average":{"1min":2.391,"5min":1.405,"15min":0.88100000000000001},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.676,"5min":10.676,"15min":10.676},"last":0.39900000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":1.4359999999999999,"15min":0.89600000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":10.734,"5min":10.734,"15min":10.734},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.716,"5min":1.1659999999999999,"15min":0.82099999999999995},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.54,"5min":8.9190000000000005,"15min":8.9190000000000005},"last":1.802}]},{"osd":6,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.391,"15min":0.88400000000000001},"min":{"1min":0.36599999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.706,"5min":10.706,"15min":10.706},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.0780000000000001,"15min":0.79300000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.0499999999999998,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":4.282}]},{"osd":7,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.474,"5min":1.151,"15min":0.81899999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.5380000000000003,"5min":8.875,"15min":8.875},"last":1.9850000000000001},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.425,"15min":0.92700000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":14.394,"5min":14.394,"15min":14.394},"last":5.5549999999999997}]}]},{"osd":2,"up_from":0,"seq":386547056699,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893173,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2407544,"kb_used_data":2205360,"kb_used_omap":7,"kb_used_meta":202168,"kb_avail":91337608,"statfs":{"total":95995035648,"available":93529710592,"internally_reserved":0,"allocated":2258288640,"data_stored":2257385396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":207020648},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":1.3140000000000001,"15min":0.65200000000000002},"min":{"1min":0.192,"5min":0.183,"15min":0.183},"max":{"1min":7.0289999999999999,"5min":11.557,"15min":11.557},"last":3.8399999999999999},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.1379999999999999,"15min":0.65000000000000002},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":6.681,"5min":7.0510000000000002,"15min":7.0510000000000002},"last":3.7879999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":1.6970000000000001,"15min":0.78500000000000003},"min":{"1min":0.27100000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.048,"5min":14.707000000000001,"15min":14.707000000000001},"last":1.9299999999999999},{"interface":"front","average":{"1min":1.6220000000000001,"5min":1.1890000000000001,"15min":0.64500000000000002},"min":{"1min":0.19400000000000001,"5min":0.193,"15min":0.193},"max":{"1min":7.1630000000000003,"5min":11.629,"15min":11.629},"last":3.8159999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.6739999999999999,"15min":0.80200000000000005},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.9180000000000001,"5min":13.853,"15min":13.853},"last":0.315},{"interface":"front","average":{"1min":2.5590000000000002,"5min":1.5129999999999999,"15min":0.76800000000000002},"min":{"1min":0.30099999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.71,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":0.374}]},{"osd":4,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.2210000000000001,"15min":0.78800000000000003},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.9909999999999997,"5min":11.532,"15min":11.532},"last":3.766},{"interface":"front","average":{"1min":2.2440000000000002,"5min":1.296,"15min":0.83699999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.962999999999999,"5min":11.962999999999999,"15min":11.962999999999999},"last":0.48599999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7589999999999999,"5min":1.8029999999999999,"15min":0.99199999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.907,"5min":51.509999999999998,"15min":51.509999999999998},"last":4.4370000000000003},{"interface":"front","average":{"1min":2.1469999999999998,"5min":2.306,"15min":1.1439999999999999},"min":{"1min":0.36799999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.1139999999999999,"5min":55.338999999999999,"15min":55.338999999999999},"last":8.7449999999999992}]},{"osd":6,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":1.3620000000000001,"15min":0.84599999999999997},"min":{"1min":0.378,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":11.92,"5min":11.92,"15min":11.92},"last":0.65500000000000003},{"interface":"front","average":{"1min":2.3759999999999999,"5min":1.776,"15min":0.99199999999999999},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":10.528,"5min":13.891999999999999,"15min":13.891999999999999},"last":0.67300000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":2.3410000000000002,"15min":1.175},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.6520000000000001,"5min":51.744,"15min":51.744},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.9139999999999999,"5min":1.869,"15min":1.0049999999999999},"min":{"1min":0.33000000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.133,"5min":53.399000000000001,"15min":53.399000000000001},"last":8.4280000000000008}]}]},{"osd":4,"up_from":30,"seq":128849019049,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1306236,"kb_used_data":996660,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92438916,"statfs":{"total":95995035648,"available":94657449984,"internally_reserved":0,"allocated":1020579840,"data_stored":1019906171,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.0129999999999999,"15min":0.70899999999999996},"min":{"1min":0.24099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.5419999999999998,"5min":10.387,"15min":10.387},"last":0.216},{"interface":"front","average":{"1min":1.581,"5min":1.1819999999999999,"15min":0.78400000000000003},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.5999999999999996,"5min":10.311999999999999,"15min":10.311999999999999},"last":4.3700000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.1519999999999999,"15min":0.752},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.9100000000000001,"5min":6.907,"15min":6.907},"last":0.254},{"interface":"front","average":{"1min":1.859,"5min":1.2390000000000001,"15min":0.82699999999999996},"min":{"1min":0.30099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.7669999999999995,"5min":8.7669999999999995,"15min":8.7669999999999995},"last":0.28000000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:47:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2170000000000001,"5min":1.1000000000000001,"15min":0.747},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.2149999999999999,"5min":7.5449999999999999,"15min":7.5449999999999999},"last":4.4109999999999996},{"interface":"front","average":{"1min":1.579,"5min":1.1639999999999999,"15min":0.76300000000000001},"min":{"1min":0.36699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.5110000000000001,"5min":6.2939999999999996,"15min":6.2939999999999996},"last":4.4349999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:47:38 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.333,"15min":0.91900000000000004},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.8170000000000002,"5min":8.8170000000000002,"15min":8.8170000000000002},"last":0.68999999999999995},{"interface":"front","average":{"1min":2.3650000000000002,"5min":1.359,"15min":0.84499999999999997},"min":{"1min":0.34300000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.625,"5min":8.9809999999999999,"15min":8.9809999999999999},"last":4.4589999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":0.89500000000000002,"15min":0.59699999999999998},"min":{"1min":0.25600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":8.6950000000000003,"5min":8.6950000000000003,"15min":8.6950000000000003},"last":15.881},{"interface":"front","average":{"1min":1.6579999999999999,"5min":0.94299999999999995,"15min":0.61299999999999999},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.252},"max":{"1min":4.8879999999999999,"5min":4.8879999999999999,"15min":4.8879999999999999},"last":20.422000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.74099999999999999,"15min":0.55700000000000005},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":7.1580000000000004,"5min":7.1580000000000004,"15min":7.1580000000000004},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.71599999999999997,"15min":0.60899999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":3.6150000000000002,"5min":3.6150000000000002,"15min":3.6150000000000002},"last":0.84999999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.80800000000000005,"15min":0.64600000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.1050000000000004,"5min":4.1050000000000004,"15min":4.1050000000000004},"last":7.3109999999999999},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.748,"15min":0.60799999999999998},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0919999999999996,"5min":4.0919999999999996,"15min":4.0919999999999996},"last":7.423}]}]},{"osd":5,"up_from":37,"seq":158913790118,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1581548,"kb_used_data":1284132,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92163604,"statfs":{"total":95995035648,"available":94375530496,"internally_reserved":0,"allocated":1314951168,"data_stored":1314289328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.77600000000000002,"15min":0.65700000000000003},"min":{"1min":0.247,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.9860000000000002,"5min":4.8380000000000001,"15min":4.8380000000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.84899999999999998,"15min":0.72899999999999998},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.0209999999999999,"5min":4.8650000000000002,"15min":4.8650000000000002},"last":0.60999999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":1.367,"15min":0.86099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":12.999000000000001,"5min":12.999000000000001,"15min":12.999000000000001},"last":3.1829999999999998},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.2370000000000001,"15min":0.80700000000000005},"min":{"1min":0.27600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3730000000000002,"5min":7.3730000000000002,"15min":7.3730000000000002},"last":4.2119999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:47:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.02,"15min":0.72099999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.4569999999999999,"5min":7.9000000000000004,"15min":7.9000000000000004},"last":3.2109999999999999},{"interface":"front","average":{"1min":1.7569999999999999,"5min":0.90400000000000003,"15min":0.68400000000000005},"min":{"1min":0.38,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":10.352,"5min":10.352,"15min":10.352},"last":4.2389999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4969999999999999,"5min":1.621,"15min":1.0229999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":18.016999999999999,"5min":18.016999999999999,"15min":18.016999999999999},"last":1.6759999999999999},{"interface":"front","average":{"1min":2.964,"5min":1.673,"15min":0.96599999999999997},"min":{"1min":0.318,"5min":0.317,"15min":0.317},"max":{"1min":15.555999999999999,"5min":15.555999999999999,"15min":15.555999999999999},"last":1.641}]},{"osd":4,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":0.83799999999999997,"15min":0.57799999999999996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":19.152999999999999,"5min":19.152999999999999,"15min":19.152999999999999},"last":4.1699999999999999},{"interface":"front","average":{"1min":1.986,"5min":0.96999999999999997,"15min":0.63500000000000001},"min":{"1min":0.26800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.3059999999999992,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":1.706}]},{"osd":6,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.66300000000000003,"15min":0.53500000000000003},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":2.8900000000000001,"5min":2.9529999999999998,"15min":2.9529999999999998},"last":5.9900000000000002},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.63300000000000001,"15min":0.53800000000000003},"min":{"1min":0.34100000000000003,"5min":0.33300000000000002,"15min":0.30299999999999999},"max":{"1min":3.8159999999999998,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":4.2839999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:47:22 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.67100000000000004,"15min":0.58799999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.4729999999999999,"5min":4.0510000000000002,"15min":4.0510000000000002},"last":6.0250000000000004},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.70899999999999996,"15min":0.60099999999999998},"min":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":4.1289999999999996,"5min":4.1289999999999996,"15min":4.1289999999999996},"last":4.6630000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":545067008,"data_stored":545062936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1319112704,"data_stored":1319108632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2861694976,"data_stored":2861694976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2229338112,"data_stored":2229338112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1005977600,"data_stored":1005977600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1308889088,"data_stored":1308885016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1140195328,"data_stored":1140195328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1417543680,"data_stored":1417543680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:47:54.681 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:54.745 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:54.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:54.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:54.849 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:54.939 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:54.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:54.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:54.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:55.093 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:47:55.136 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:55.165 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:47:55.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:55.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:47:55.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:47:55.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:55.325 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:47:55.326 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:47:55.369 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:55.460 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:47:55.469 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:55.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:55.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:55.519 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:55 smithi125 ceph-mon[195507]: from='client.44891 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:47:55.519 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:55 smithi125 ceph-mon[200197]: from='client.44891 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:47:55.539 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:55 smithi155 ceph-mon[184854]: from='client.44891 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:47:55.581 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:55.608 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:55.620 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:55.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:55.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:55.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:55.752 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:47:55.772 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:47:55.890 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:55.940 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:47:55.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:55.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:47:56.051 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:56.086 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:56.209 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:47:56.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:56 smithi125 ceph-mon[195507]: pgmap v381: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 31152/143742 objects degraded (21.672%); 5449/143742 objects misplaced (3.791%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:47:56.559 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:56 smithi125 ceph-mon[200197]: pgmap v381: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 31152/143742 objects degraded (21.672%); 5449/143742 objects misplaced (3.791%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:47:56.561 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:56.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:56.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:56.637 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:56 smithi155 ceph-mon[184854]: pgmap v381: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 31152/143742 objects degraded (21.672%); 5449/143742 objects misplaced (3.791%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:47:56.666 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:47:56.715 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:56.741 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:47:56.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:56.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:47:56.846 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:47:56.934 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:57.019 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:47:57.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:57.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:47:57.123 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:57.157 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:57.255 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:57.292 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:57.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:57.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:57.397 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:57.469 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:57.504 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:57.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:57.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:57.611 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:57.618 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:47:57.618 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:47:57.664 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:47:57.698 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:57.782 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:57.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:57.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:57.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:57.964 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:57.975 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:47:58.012 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:47:58.040 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:47:58.042 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:47:58.075 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:47:58.115 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:47:58.147 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:47:58.178 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:47:58.188 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:58.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:58.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:58.205 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:47:58.212 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:47:58.212 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:47:58.213 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:47:58.213 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:47:58.213 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:47:58.213 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:47:58.213 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:47:58.213 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:47:58.295 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:47:58.372 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:47:58.383 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:58.417 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:47:58.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:58.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:47:58.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:58 smithi125 ceph-mon[195507]: pgmap v382: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 758 op/s; 31152/138348 objects degraded (22.517%); 4954/138348 objects misplaced (3.581%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T10:47:58.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:58 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 31152/138348 objects degraded (22.517%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:58.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:47:58 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3167748335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:58.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:58 smithi125 ceph-mon[200197]: pgmap v382: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 758 op/s; 31152/138348 objects degraded (22.517%); 4954/138348 objects misplaced (3.581%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T10:47:58.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:58 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 31152/138348 objects degraded (22.517%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:58.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:47:58 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3167748335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:58.524 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:47:58.529 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:58.606 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:58 smithi155 ceph-mon[184854]: pgmap v382: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 758 op/s; 31152/138348 objects degraded (22.517%); 4954/138348 objects misplaced (3.581%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T10:47:58.606 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:58 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 31152/138348 objects degraded (22.517%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:47:58.606 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:47:58 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3167748335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:47:58.621 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:58.675 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:58.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:58.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:58.780 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:58.832 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:59.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:59.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:59.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:59.219 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:47:59.323 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:59.364 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:47:59.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:59.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:47:59.472 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:47:59.526 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:59.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:59.614 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:47:59.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:47:59.715 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:47:59.790 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:47:59.832 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:47:59.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:47:59.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:47:59.939 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:00.009 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:00.171 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:48:00.196 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:48:00.329 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:48:00.408 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:48:00.408 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:48:00.408 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:48:00.408 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:48:00.408 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:48:00.476 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:00.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:00.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:00.540 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:00 smithi125 ceph-mon[195507]: pgmap v383: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 31152/137703 objects degraded (22.623%); 4954/137703 objects misplaced (3.598%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T10:48:00.540 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:00 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3392810936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:48:00.540 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:48:00.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:00 smithi125 ceph-mon[200197]: pgmap v383: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 31152/137703 objects degraded (22.623%); 4954/137703 objects misplaced (3.598%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T10:48:00.541 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:00 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3392810936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:48:00.541 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:48:00.582 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:00.619 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:00 smithi155 ceph-mon[184854]: pgmap v383: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 31152/137703 objects degraded (22.623%); 4954/137703 objects misplaced (3.598%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T10:48:00.619 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:00 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3392810936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:48:00.619 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:48:00.622 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:00.653 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:00.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:00.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:00.660 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":383,"stamp":"2023-10-09T10:47:59.125872+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3007092664,"num_objects":45901,"num_object_clones":0,"num_object_copies":137703,"num_objects_missing_on_primary":1406,"num_objects_missing":1406,"num_objects_degraded":31152,"num_objects_misplaced":4954,"num_objects_unfound":0,"num_objects_dirty":45901,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":158631,"num_write_kb":6260974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23317,"num_bytes_recovered":1435634434,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":168567,"ondisk_log_size":168567,"up":357,"acting":347,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9854644,"kb_used_data":8226684,"kb_used_omap":44,"kb_used_meta":1627475,"kb_avail":646361420,"statfs":{"total":671965249536,"available":661874094080,"internally_reserved":0,"allocated":8424124416,"data_stored":8418686218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1666534817},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,4,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-448593920,"num_objects":-6845,"num_object_clones":0,"num_object_copies":-20535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1457,"num_objects_unfound":0,"num_objects_dirty":-6845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6845,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":821,"num_bytes_recovered":53805056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005538"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":172,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.433214+0000","last_change":"2023-10-09T10:46:32.250979+0000","last_active":"2023-10-09T10:47:48.433214+0000","last_peered":"2023-10-09T10:47:48.433214+0000","last_clean":"2023-10-09T10:47:48.433214+0000","last_became_active":"2023-10-09T10:46:32.250625+0000","last_became_peered":"2023-10-09T10:46:32.250625+0000","last_unstale":"2023-10-09T10:47:48.433214+0000","last_undegraded":"2023-10-09T10:47:48.433214+0000","last_fullsized":"2023-10-09T10:47:48.433214+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:34.858348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":147,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.391264+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:47:48.391264+0000","last_peered":"2023-10-09T10:47:48.391264+0000","last_clean":"2023-10-09T10:47:48.391264+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:47:48.391264+0000","last_undegraded":"2023-10-09T10:47:48.391264+0000","last_fullsized":"2023-10-09T10:47:48.391264+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":131,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386894+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:47:48.386894+0000","last_peered":"2023-10-09T10:47:48.386894+0000","last_clean":"2023-10-09T10:47:48.386894+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:47:48.386894+0000","last_undegraded":"2023-10-09T10:47:48.386894+0000","last_fullsized":"2023-10-09T10:47:48.386894+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":47,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385769+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:47:48.385769+0000","last_peered":"2023-10-09T10:47:48.385769+0000","last_clean":"2023-10-09T10:47:48.385769+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:47:48.385769+0000","last_undegraded":"2023-10-09T10:47:48.385769+0000","last_fullsized":"2023-10-09T10:47:48.385769+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":183,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.392956+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:47:48.392956+0000","last_peered":"2023-10-09T10:47:48.392956+0000","last_clean":"2023-10-09T10:47:48.392956+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:47:48.392956+0000","last_undegraded":"2023-10-09T10:47:48.392956+0000","last_fullsized":"2023-10-09T10:47:48.392956+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":117,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.397418+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:47:48.397418+0000","last_peered":"2023-10-09T10:47:48.397418+0000","last_clean":"2023-10-09T10:47:48.397418+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:47:48.397418+0000","last_undegraded":"2023-10-09T10:47:48.397418+0000","last_fullsized":"2023-10-09T10:47:48.397418+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"179'6399","reported_seq":10552,"reported_epoch":179,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:56.065370+0000","last_change":"2023-10-09T10:46:33.521602+0000","last_active":"2023-10-09T10:47:56.065370+0000","last_peered":"2023-10-09T10:47:56.065370+0000","last_clean":"2023-10-09T10:46:31.055751+0000","last_became_active":"2023-10-09T10:46:33.364058+0000","last_became_peered":"2023-10-09T10:46:33.364058+0000","last_unstale":"2023-10-09T10:47:56.065370+0000","last_undegraded":"2023-10-09T10:47:56.065370+0000","last_fullsized":"2023-10-09T10:47:56.065370+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6399,"log_dups_size":0,"ondisk_log_size":6399,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97943552,"num_objects":1495,"num_object_clones":0,"num_object_copies":4485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1494,"num_objects_unfound":0,"num_objects_dirty":1495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5363,"num_write_kb":219424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2010,"num_bytes_recovered":131727360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":1495}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":47,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386472+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:47:48.386472+0000","last_peered":"2023-10-09T10:47:48.386472+0000","last_clean":"2023-10-09T10:47:48.386472+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:47:48.386472+0000","last_undegraded":"2023-10-09T10:47:48.386472+0000","last_fullsized":"2023-10-09T10:47:48.386472+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":147,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.393046+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:47:48.393046+0000","last_peered":"2023-10-09T10:47:48.393046+0000","last_clean":"2023-10-09T10:47:48.393046+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:47:48.393046+0000","last_undegraded":"2023-10-09T10:47:48.393046+0000","last_fullsized":"2023-10-09T10:47:48.393046+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":512,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385364+0000","last_change":"2023-10-09T10:46:05.620365+0000","last_active":"2023-10-09T10:47:48.385364+0000","last_peered":"2023-10-09T10:47:48.385364+0000","last_clean":"2023-10-09T10:47:48.385364+0000","last_became_active":"2023-10-09T10:46:05.620024+0000","last_became_peered":"2023-10-09T10:46:05.620024+0000","last_unstale":"2023-10-09T10:47:48.385364+0000","last_undegraded":"2023-10-09T10:47:48.385364+0000","last_fullsized":"2023-10-09T10:47:48.385364+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.13","version":"179'6217","reported_seq":6402,"reported_epoch":179,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:55.910180+0000","last_change":"2023-10-09T10:46:33.470698+0000","last_active":"2023-10-09T10:47:55.910180+0000","last_peered":"2023-10-09T10:47:55.910180+0000","last_clean":"2023-10-09T10:46:31.195699+0000","last_became_active":"2023-10-09T10:46:33.455632+0000","last_became_peered":"2023-10-09T10:46:33.455632+0000","last_unstale":"2023-10-09T10:47:55.910180+0000","last_undegraded":"2023-10-09T10:46:33.269323+0000","last_fullsized":"2023-10-09T10:46:33.241226+0000","mapping_epoch":154,"log_start":"104'971","ondisk_log_start":"104'971","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":5246,"log_dups_size":0,"ondisk_log_size":5246,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:07:12.547333+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95354880,"num_objects":1455,"num_object_clones":0,"num_object_copies":4365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3315,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3502,"num_write_kb":105088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":1455},{"shards":"6,7","objects":-1320}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":144,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.397833+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:47:48.397833+0000","last_peered":"2023-10-09T10:47:48.397833+0000","last_clean":"2023-10-09T10:47:48.397833+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:47:48.397833+0000","last_undegraded":"2023-10-09T10:47:48.397833+0000","last_fullsized":"2023-10-09T10:47:48.397833+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":104,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.390551+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:47:48.390551+0000","last_peered":"2023-10-09T10:47:48.390551+0000","last_clean":"2023-10-09T10:47:48.390551+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:47:48.390551+0000","last_undegraded":"2023-10-09T10:47:48.390551+0000","last_fullsized":"2023-10-09T10:47:48.390551+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"179'6361","reported_seq":14487,"reported_epoch":179,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:56.522233+0000","last_change":"2023-10-09T10:46:03.683196+0000","last_active":"2023-10-09T10:47:56.522233+0000","last_peered":"2023-10-09T10:47:56.522233+0000","last_clean":"2023-10-09T10:46:02.177309+0000","last_became_active":"2023-10-09T10:46:03.681058+0000","last_became_peered":"2023-10-09T10:46:03.681058+0000","last_unstale":"2023-10-09T10:47:56.522233+0000","last_undegraded":"2023-10-09T10:47:56.522233+0000","last_fullsized":"2023-10-09T10:47:56.522233+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6361,"log_dups_size":0,"ondisk_log_size":6361,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96501760,"num_objects":1473,"num_object_clones":0,"num_object_copies":4419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1473,"num_objects_unfound":0,"num_objects_dirty":1473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5315,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[1,5,6],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1473}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":100,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.391515+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:47:48.391515+0000","last_peered":"2023-10-09T10:47:48.391515+0000","last_clean":"2023-10-09T10:47:48.391515+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:47:48.391515+0000","last_undegraded":"2023-10-09T10:47:48.391515+0000","last_fullsized":"2023-10-09T10:47:48.391515+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":140,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385692+0000","last_change":"2023-10-09T10:46:32.249921+0000","last_active":"2023-10-09T10:47:48.385692+0000","last_peered":"2023-10-09T10:47:48.385692+0000","last_clean":"2023-10-09T10:47:48.385692+0000","last_became_active":"2023-10-09T10:46:32.249590+0000","last_became_peered":"2023-10-09T10:46:32.249590+0000","last_unstale":"2023-10-09T10:47:48.385692+0000","last_undegraded":"2023-10-09T10:47:48.385692+0000","last_fullsized":"2023-10-09T10:47:48.385692+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:25.169752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"179'6179","reported_seq":14401,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:54.519898+0000","last_change":"2023-10-09T10:47:48.412928+0000","last_active":"2023-10-09T10:47:54.519898+0000","last_peered":"2023-10-09T10:47:54.519898+0000","last_clean":"2023-10-09T10:47:54.519898+0000","last_became_active":"2023-10-09T10:47:48.398813+0000","last_became_peered":"2023-10-09T10:47:48.398813+0000","last_unstale":"2023-10-09T10:47:54.519898+0000","last_undegraded":"2023-10-09T10:47:54.519898+0000","last_fullsized":"2023-10-09T10:47:54.519898+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6179,"log_dups_size":0,"ondisk_log_size":6179,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91848704,"num_objects":1402,"num_object_clones":0,"num_object_copies":4206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5172,"num_write_kb":210336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2691,"num_bytes_recovered":176357376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":149,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.430534+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:47:48.430534+0000","last_peered":"2023-10-09T10:47:48.430534+0000","last_clean":"2023-10-09T10:47:48.430534+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:47:48.430534+0000","last_undegraded":"2023-10-09T10:47:48.430534+0000","last_fullsized":"2023-10-09T10:47:48.430534+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":143,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.430364+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:47:48.430364+0000","last_peered":"2023-10-09T10:47:48.430364+0000","last_clean":"2023-10-09T10:47:48.430364+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:47:48.430364+0000","last_undegraded":"2023-10-09T10:47:48.430364+0000","last_fullsized":"2023-10-09T10:47:48.430364+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":512,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385954+0000","last_change":"2023-10-09T10:46:05.618276+0000","last_active":"2023-10-09T10:47:48.385954+0000","last_peered":"2023-10-09T10:47:48.385954+0000","last_clean":"2023-10-09T10:47:48.385954+0000","last_became_active":"2023-10-09T10:46:05.616509+0000","last_became_peered":"2023-10-09T10:46:05.616509+0000","last_unstale":"2023-10-09T10:47:48.385954+0000","last_undegraded":"2023-10-09T10:47:48.385954+0000","last_fullsized":"2023-10-09T10:47:48.385954+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.16","version":"179'6177","reported_seq":6281,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:54.461607+0000","last_change":"2023-10-09T10:46:03.592682+0000","last_active":"2023-10-09T10:47:54.461607+0000","last_peered":"2023-10-09T10:47:54.461607+0000","last_clean":"2023-10-09T10:47:54.461607+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:47:54.461607+0000","last_undegraded":"2023-10-09T10:47:54.461607+0000","last_fullsized":"2023-10-09T10:47:54.461607+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6177,"log_dups_size":0,"ondisk_log_size":6177,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95158272,"num_objects":1452,"num_object_clones":0,"num_object_copies":4356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5134,"num_write_kb":210752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":161,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.454473+0000","last_change":"2023-10-09T10:46:32.250256+0000","last_active":"2023-10-09T10:47:48.454473+0000","last_peered":"2023-10-09T10:47:48.454473+0000","last_clean":"2023-10-09T10:47:48.454473+0000","last_became_active":"2023-10-09T10:46:32.243312+0000","last_became_peered":"2023-10-09T10:46:32.243312+0000","last_unstale":"2023-10-09T10:47:48.454473+0000","last_undegraded":"2023-10-09T10:47:48.454473+0000","last_fullsized":"2023-10-09T10:47:48.454473+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:08.059777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":170,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.390965+0000","last_change":"2023-10-09T10:46:32.237577+0000","last_active":"2023-10-09T10:47:48.390965+0000","last_peered":"2023-10-09T10:47:48.390965+0000","last_clean":"2023-10-09T10:47:48.390965+0000","last_became_active":"2023-10-09T10:46:32.237316+0000","last_became_peered":"2023-10-09T10:46:32.237316+0000","last_unstale":"2023-10-09T10:47:48.390965+0000","last_undegraded":"2023-10-09T10:47:48.390965+0000","last_fullsized":"2023-10-09T10:47:48.390965+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:39.684602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384553+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:47:48.384553+0000","last_peered":"2023-10-09T10:47:48.384553+0000","last_clean":"2023-10-09T10:47:48.384553+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:47:48.384553+0000","last_undegraded":"2023-10-09T10:47:48.384553+0000","last_fullsized":"2023-10-09T10:47:48.384553+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.17","version":"179'6161","reported_seq":6259,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:56.065725+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:47:56.065725+0000","last_peered":"2023-10-09T10:47:56.065725+0000","last_clean":"2023-10-09T10:47:56.065725+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:47:56.065725+0000","last_undegraded":"2023-10-09T10:47:56.065725+0000","last_fullsized":"2023-10-09T10:47:56.065725+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6161,"log_dups_size":0,"ondisk_log_size":6161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88670208,"num_objects":1353,"num_object_clones":0,"num_object_copies":4059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5153,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":47,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386909+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:47:48.386909+0000","last_peered":"2023-10-09T10:47:48.386909+0000","last_clean":"2023-10-09T10:47:48.386909+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:47:48.386909+0000","last_undegraded":"2023-10-09T10:47:48.386909+0000","last_fullsized":"2023-10-09T10:47:48.386909+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":143,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.397339+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:47:48.397339+0000","last_peered":"2023-10-09T10:47:48.397339+0000","last_clean":"2023-10-09T10:47:48.397339+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:47:48.397339+0000","last_undegraded":"2023-10-09T10:47:48.397339+0000","last_fullsized":"2023-10-09T10:47:48.397339+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384741+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:47:48.384741+0000","last_peered":"2023-10-09T10:47:48.384741+0000","last_clean":"2023-10-09T10:47:48.384741+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:47:48.384741+0000","last_undegraded":"2023-10-09T10:47:48.384741+0000","last_fullsized":"2023-10-09T10:47:48.384741+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"179'6305","reported_seq":6408,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:56.413394+0000","last_change":"2023-10-09T10:46:33.243540+0000","last_active":"2023-10-09T10:47:56.413394+0000","last_peered":"2023-10-09T10:47:56.413394+0000","last_clean":"2023-10-09T10:47:56.413394+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:47:56.413394+0000","last_undegraded":"2023-10-09T10:47:56.413394+0000","last_fullsized":"2023-10-09T10:47:56.413394+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6305,"log_dups_size":0,"ondisk_log_size":6305,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93224960,"num_objects":1423,"num_object_clones":0,"num_object_copies":4269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5251,"num_write_kb":213536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":89,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.430170+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:47:48.430170+0000","last_peered":"2023-10-09T10:47:48.430170+0000","last_clean":"2023-10-09T10:47:48.430170+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:47:48.430170+0000","last_undegraded":"2023-10-09T10:47:48.430170+0000","last_fullsized":"2023-10-09T10:47:48.430170+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":47,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386007+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:47:48.386007+0000","last_peered":"2023-10-09T10:47:48.386007+0000","last_clean":"2023-10-09T10:47:48.386007+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:47:48.386007+0000","last_undegraded":"2023-10-09T10:47:48.386007+0000","last_fullsized":"2023-10-09T10:47:48.386007+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386682+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:47:48.386682+0000","last_peered":"2023-10-09T10:47:48.386682+0000","last_clean":"2023-10-09T10:47:48.386682+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:47:48.386682+0000","last_undegraded":"2023-10-09T10:47:48.386682+0000","last_fullsized":"2023-10-09T10:47:48.386682+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"179'6291","reported_seq":6404,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:54.512780+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:47:54.512780+0000","last_peered":"2023-10-09T10:47:54.512780+0000","last_clean":"2023-10-09T10:47:54.512780+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:47:54.512780+0000","last_undegraded":"2023-10-09T10:47:54.512780+0000","last_fullsized":"2023-10-09T10:47:54.512780+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6291,"log_dups_size":0,"ondisk_log_size":6291,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94240768,"num_objects":1438,"num_object_clones":0,"num_object_copies":4314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5230,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":57,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384471+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:47:48.384471+0000","last_peered":"2023-10-09T10:47:48.384471+0000","last_clean":"2023-10-09T10:47:48.384471+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:47:48.384471+0000","last_undegraded":"2023-10-09T10:47:48.384471+0000","last_fullsized":"2023-10-09T10:47:48.384471+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":147,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.390575+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:47:48.390575+0000","last_peered":"2023-10-09T10:47:48.390575+0000","last_clean":"2023-10-09T10:47:48.390575+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:47:48.390575+0000","last_undegraded":"2023-10-09T10:47:48.390575+0000","last_fullsized":"2023-10-09T10:47:48.390575+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.387256+0000","last_change":"2023-10-09T10:46:05.620574+0000","last_active":"2023-10-09T10:47:48.387256+0000","last_peered":"2023-10-09T10:47:48.387256+0000","last_clean":"2023-10-09T10:47:48.387256+0000","last_became_active":"2023-10-09T10:46:05.620212+0000","last_became_peered":"2023-10-09T10:46:05.620212+0000","last_unstale":"2023-10-09T10:47:48.387256+0000","last_undegraded":"2023-10-09T10:47:48.387256+0000","last_fullsized":"2023-10-09T10:47:48.387256+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"179'10817","reported_seq":14971,"reported_epoch":179,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:56.025435+0000","last_change":"2023-10-09T10:46:32.247903+0000","last_active":"2023-10-09T10:47:56.025435+0000","last_peered":"2023-10-09T10:47:56.025435+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:46:02.739355+0000","last_became_peered":"2023-10-09T10:46:02.739355+0000","last_unstale":"2023-10-09T10:47:56.025435+0000","last_undegraded":"2023-10-09T10:46:02.608805+0000","last_fullsized":"2023-10-09T10:46:02.583589+0000","mapping_epoch":142,"log_start":"105'1900","ondisk_log_start":"105'1900","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8917,"log_dups_size":0,"ondisk_log_size":8917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:32.899634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184483840,"num_objects":2815,"num_object_clones":0,"num_object_copies":8445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10817,"num_write_kb":436224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2022,"num_bytes_recovered":132513792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":625},{"shards":"5,6","objects":2189}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":172,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.460660+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:47:48.460660+0000","last_peered":"2023-10-09T10:47:48.460660+0000","last_clean":"2023-10-09T10:47:48.460660+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:47:48.460660+0000","last_undegraded":"2023-10-09T10:47:48.460660+0000","last_fullsized":"2023-10-09T10:47:48.460660+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":126,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.397110+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:47:48.397110+0000","last_peered":"2023-10-09T10:47:48.397110+0000","last_clean":"2023-10-09T10:47:48.397110+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:47:48.397110+0000","last_undegraded":"2023-10-09T10:47:48.397110+0000","last_fullsized":"2023-10-09T10:47:48.397110+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":493,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385497+0000","last_change":"2023-10-09T10:46:02.631378+0000","last_active":"2023-10-09T10:47:48.385497+0000","last_peered":"2023-10-09T10:47:48.385497+0000","last_clean":"2023-10-09T10:47:48.385497+0000","last_became_active":"2023-10-09T10:46:02.629142+0000","last_became_peered":"2023-10-09T10:46:02.629142+0000","last_unstale":"2023-10-09T10:47:48.385497+0000","last_undegraded":"2023-10-09T10:47:48.385497+0000","last_fullsized":"2023-10-09T10:47:48.385497+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.b","version":"179'10283","reported_seq":10374,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:54.530506+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:47:54.530506+0000","last_peered":"2023-10-09T10:47:54.530506+0000","last_clean":"2023-10-09T10:47:54.530506+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:47:54.530506+0000","last_undegraded":"2023-10-09T10:47:54.530506+0000","last_fullsized":"2023-10-09T10:47:54.530506+0000","mapping_epoch":98,"log_start":"125'4525","ondisk_log_start":"125'4525","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5758,"log_dups_size":0,"ondisk_log_size":5758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189857792,"num_objects":2897,"num_object_clones":0,"num_object_copies":8691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2897,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10283,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":145,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.451324+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:47:48.451324+0000","last_peered":"2023-10-09T10:47:48.451324+0000","last_clean":"2023-10-09T10:47:48.451324+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:47:48.451324+0000","last_undegraded":"2023-10-09T10:47:48.451324+0000","last_fullsized":"2023-10-09T10:47:48.451324+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":180,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.430126+0000","last_change":"2023-10-09T10:46:32.251035+0000","last_active":"2023-10-09T10:47:48.430126+0000","last_peered":"2023-10-09T10:47:48.430126+0000","last_clean":"2023-10-09T10:47:48.430126+0000","last_became_active":"2023-10-09T10:46:32.250293+0000","last_became_peered":"2023-10-09T10:46:32.250293+0000","last_unstale":"2023-10-09T10:47:48.430126+0000","last_undegraded":"2023-10-09T10:47:48.430126+0000","last_fullsized":"2023-10-09T10:47:48.430126+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.787766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386245+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:47:48.386245+0000","last_peered":"2023-10-09T10:47:48.386245+0000","last_clean":"2023-10-09T10:47:48.386245+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:47:48.386245+0000","last_undegraded":"2023-10-09T10:47:48.386245+0000","last_fullsized":"2023-10-09T10:47:48.386245+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"179'6293","reported_seq":6403,"reported_epoch":179,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:54.567074+0000","last_change":"2023-10-09T10:46:33.540719+0000","last_active":"2023-10-09T10:47:54.567074+0000","last_peered":"2023-10-09T10:47:54.567074+0000","last_clean":"2023-10-09T10:46:31.137696+0000","last_became_active":"2023-10-09T10:46:33.527650+0000","last_became_peered":"2023-10-09T10:46:33.527650+0000","last_unstale":"2023-10-09T10:47:54.567074+0000","last_undegraded":"2023-10-09T10:46:33.285468+0000","last_fullsized":"2023-10-09T10:46:33.235683+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6293,"log_dups_size":0,"ondisk_log_size":6293,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95584256,"num_objects":1458,"num_object_clones":0,"num_object_copies":4374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5228,"num_write_kb":213984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1446},{"shards":"3,7","objects":-1351}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":100,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.397259+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:47:48.397259+0000","last_peered":"2023-10-09T10:47:48.397259+0000","last_clean":"2023-10-09T10:47:48.397259+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:47:48.397259+0000","last_undegraded":"2023-10-09T10:47:48.397259+0000","last_fullsized":"2023-10-09T10:47:48.397259+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":117,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385975+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:47:48.385975+0000","last_peered":"2023-10-09T10:47:48.385975+0000","last_clean":"2023-10-09T10:47:48.385975+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:47:48.385975+0000","last_undegraded":"2023-10-09T10:47:48.385975+0000","last_fullsized":"2023-10-09T10:47:48.385975+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385610+0000","last_change":"2023-10-09T10:46:05.620030+0000","last_active":"2023-10-09T10:47:48.385610+0000","last_peered":"2023-10-09T10:47:48.385610+0000","last_clean":"2023-10-09T10:47:48.385610+0000","last_became_active":"2023-10-09T10:46:05.619694+0000","last_became_peered":"2023-10-09T10:46:05.619694+0000","last_unstale":"2023-10-09T10:47:48.385610+0000","last_undegraded":"2023-10-09T10:47:48.385610+0000","last_fullsized":"2023-10-09T10:47:48.385610+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.9","version":"179'6312","reported_seq":6434,"reported_epoch":179,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:57.508674+0000","last_change":"2023-10-09T10:46:34.405499+0000","last_active":"2023-10-09T10:47:57.508674+0000","last_peered":"2023-10-09T10:47:57.508674+0000","last_clean":"2023-10-09T10:46:31.190603+0000","last_became_active":"2023-10-09T10:46:34.396448+0000","last_became_peered":"2023-10-09T10:46:34.396448+0000","last_unstale":"2023-10-09T10:47:57.508674+0000","last_undegraded":"2023-10-09T10:46:33.246949+0000","last_fullsized":"2023-10-09T10:46:33.237968+0000","mapping_epoch":154,"log_start":"103'167","ondisk_log_start":"103'167","created":98,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6145,"log_dups_size":0,"ondisk_log_size":6145,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88080384,"num_objects":1344,"num_object_clones":0,"num_object_copies":4032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2004,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5270,"num_write_kb":211648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1303,"num_bytes_recovered":85393408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":41},{"shards":"0,7","objects":-124}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":144,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.397404+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:47:48.397404+0000","last_peered":"2023-10-09T10:47:48.397404+0000","last_clean":"2023-10-09T10:47:48.397404+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:47:48.397404+0000","last_undegraded":"2023-10-09T10:47:48.397404+0000","last_fullsized":"2023-10-09T10:47:48.397404+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":47,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385076+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:47:48.385076+0000","last_peered":"2023-10-09T10:47:48.385076+0000","last_clean":"2023-10-09T10:47:48.385076+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:47:48.385076+0000","last_undegraded":"2023-10-09T10:47:48.385076+0000","last_fullsized":"2023-10-09T10:47:48.385076+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":504,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385702+0000","last_change":"2023-10-09T10:46:05.618134+0000","last_active":"2023-10-09T10:47:48.385702+0000","last_peered":"2023-10-09T10:47:48.385702+0000","last_clean":"2023-10-09T10:47:48.385702+0000","last_became_active":"2023-10-09T10:46:05.616057+0000","last_became_peered":"2023-10-09T10:46:05.616057+0000","last_unstale":"2023-10-09T10:47:48.385702+0000","last_undegraded":"2023-10-09T10:47:48.385702+0000","last_fullsized":"2023-10-09T10:47:48.385702+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"179'10367","reported_seq":10466,"reported_epoch":179,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:55.955900+0000","last_change":"2023-10-09T10:46:33.576530+0000","last_active":"2023-10-09T10:47:55.955900+0000","last_peered":"2023-10-09T10:47:55.955900+0000","last_clean":"2023-10-09T10:46:31.183481+0000","last_became_active":"2023-10-09T10:46:33.542304+0000","last_became_peered":"2023-10-09T10:46:33.542304+0000","last_unstale":"2023-10-09T10:47:55.955900+0000","last_undegraded":"2023-10-09T10:46:33.300581+0000","last_fullsized":"2023-10-09T10:46:33.246047+0000","mapping_epoch":154,"log_start":"113'2977","ondisk_log_start":"113'2977","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7390,"log_dups_size":0,"ondisk_log_size":7390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:26.739488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187891712,"num_objects":2867,"num_object_clones":0,"num_object_copies":8601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6617,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10367,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":2867},{"shards":"1,4","objects":-2694}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":171,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.390840+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:47:48.390840+0000","last_peered":"2023-10-09T10:47:48.390840+0000","last_clean":"2023-10-09T10:47:48.390840+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:47:48.390840+0000","last_undegraded":"2023-10-09T10:47:48.390840+0000","last_fullsized":"2023-10-09T10:47:48.390840+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":143,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.430402+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:47:48.430402+0000","last_peered":"2023-10-09T10:47:48.430402+0000","last_clean":"2023-10-09T10:47:48.430402+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:47:48.430402+0000","last_undegraded":"2023-10-09T10:47:48.430402+0000","last_fullsized":"2023-10-09T10:47:48.430402+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":493,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384634+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:47:48.384634+0000","last_peered":"2023-10-09T10:47:48.384634+0000","last_clean":"2023-10-09T10:47:48.384634+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:47:48.384634+0000","last_undegraded":"2023-10-09T10:47:48.384634+0000","last_fullsized":"2023-10-09T10:47:48.384634+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.f","version":"179'10209","reported_seq":10306,"reported_epoch":179,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:54.566862+0000","last_change":"2023-10-09T10:46:33.651756+0000","last_active":"2023-10-09T10:47:54.566862+0000","last_peered":"2023-10-09T10:47:54.566862+0000","last_clean":"2023-10-09T10:46:31.156458+0000","last_became_active":"2023-10-09T10:46:33.621003+0000","last_became_peered":"2023-10-09T10:46:33.621003+0000","last_unstale":"2023-10-09T10:47:54.566862+0000","last_undegraded":"2023-10-09T10:46:33.309813+0000","last_fullsized":"2023-10-09T10:46:33.236508+0000","mapping_epoch":154,"log_start":"125'4510","ondisk_log_start":"125'4510","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5699,"log_dups_size":0,"ondisk_log_size":5699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:13.704346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184352768,"num_objects":2813,"num_object_clones":0,"num_object_copies":8439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10209,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":2813},{"shards":"3,4","objects":-2639}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":118,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.433364+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:47:48.433364+0000","last_peered":"2023-10-09T10:47:48.433364+0000","last_clean":"2023-10-09T10:47:48.433364+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:47:48.433364+0000","last_undegraded":"2023-10-09T10:47:48.433364+0000","last_fullsized":"2023-10-09T10:47:48.433364+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":170,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.430646+0000","last_change":"2023-10-09T10:46:32.245583+0000","last_active":"2023-10-09T10:47:48.430646+0000","last_peered":"2023-10-09T10:47:48.430646+0000","last_clean":"2023-10-09T10:47:48.430646+0000","last_became_active":"2023-10-09T10:46:32.245271+0000","last_became_peered":"2023-10-09T10:46:32.245271+0000","last_unstale":"2023-10-09T10:47:48.430646+0000","last_undegraded":"2023-10-09T10:47:48.430646+0000","last_fullsized":"2023-10-09T10:47:48.430646+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.025051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384853+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:47:48.384853+0000","last_peered":"2023-10-09T10:47:48.384853+0000","last_clean":"2023-10-09T10:47:48.384853+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:47:48.384853+0000","last_undegraded":"2023-10-09T10:47:48.384853+0000","last_fullsized":"2023-10-09T10:47:48.384853+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"179'10458","reported_seq":10549,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:54.496352+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:47:54.496352+0000","last_peered":"2023-10-09T10:47:54.496352+0000","last_clean":"2023-10-09T10:47:54.496352+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:47:54.496352+0000","last_undegraded":"2023-10-09T10:47:54.496352+0000","last_fullsized":"2023-10-09T10:47:54.496352+0000","mapping_epoch":98,"log_start":"126'4778","ondisk_log_start":"126'4778","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5680,"log_dups_size":0,"ondisk_log_size":5680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185204736,"num_objects":2826,"num_object_clones":0,"num_object_copies":8478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10458,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":154,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.430078+0000","last_change":"2023-10-09T10:46:32.245685+0000","last_active":"2023-10-09T10:47:48.430078+0000","last_peered":"2023-10-09T10:47:48.430078+0000","last_clean":"2023-10-09T10:47:48.430078+0000","last_became_active":"2023-10-09T10:46:32.245314+0000","last_became_peered":"2023-10-09T10:46:32.245314+0000","last_unstale":"2023-10-09T10:47:48.430078+0000","last_undegraded":"2023-10-09T10:47:48.430078+0000","last_fullsized":"2023-10-09T10:47:48.430078+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:51.716860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":143,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.430912+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:47:48.430912+0000","last_peered":"2023-10-09T10:47:48.430912+0000","last_clean":"2023-10-09T10:47:48.430912+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:47:48.430912+0000","last_undegraded":"2023-10-09T10:47:48.430912+0000","last_fullsized":"2023-10-09T10:47:48.430912+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":504,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386755+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:47:48.386755+0000","last_peered":"2023-10-09T10:47:48.386755+0000","last_clean":"2023-10-09T10:47:48.386755+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:47:48.386755+0000","last_undegraded":"2023-10-09T10:47:48.386755+0000","last_fullsized":"2023-10-09T10:47:48.386755+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"179'10544","reported_seq":10653,"reported_epoch":179,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:55.966192+0000","last_change":"2023-10-09T10:46:02.754507+0000","last_active":"2023-10-09T10:47:55.966192+0000","last_peered":"2023-10-09T10:47:55.966192+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:46:02.747510+0000","last_became_peered":"2023-10-09T10:46:02.747510+0000","last_unstale":"2023-10-09T10:47:55.966192+0000","last_undegraded":"2023-10-09T10:46:02.605987+0000","last_fullsized":"2023-10-09T10:46:02.585248+0000","mapping_epoch":142,"log_start":"105'1700","ondisk_log_start":"105'1700","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8844,"log_dups_size":0,"ondisk_log_size":8844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:59.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186843160,"num_objects":2852,"num_object_clones":0,"num_object_copies":8556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2852,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10544,"num_write_kb":428609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,2,5","objects":2718},{"shards":"1,5","objects":134}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":149,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.215510+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:47:52.215510+0000","last_peered":"2023-10-09T10:47:52.215510+0000","last_clean":"2023-10-09T10:47:52.215510+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:47:52.215510+0000","last_undegraded":"2023-10-09T10:47:52.215510+0000","last_fullsized":"2023-10-09T10:47:52.215510+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":131,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385939+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:47:48.385939+0000","last_peered":"2023-10-09T10:47:48.385939+0000","last_clean":"2023-10-09T10:47:48.385939+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:47:48.385939+0000","last_undegraded":"2023-10-09T10:47:48.385939+0000","last_fullsized":"2023-10-09T10:47:48.385939+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386394+0000","last_change":"2023-10-09T10:46:05.617983+0000","last_active":"2023-10-09T10:47:48.386394+0000","last_peered":"2023-10-09T10:47:48.386394+0000","last_clean":"2023-10-09T10:47:48.386394+0000","last_became_active":"2023-10-09T10:46:05.616021+0000","last_became_peered":"2023-10-09T10:46:05.616021+0000","last_unstale":"2023-10-09T10:47:48.386394+0000","last_undegraded":"2023-10-09T10:47:48.386394+0000","last_fullsized":"2023-10-09T10:47:48.386394+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":174,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.387734+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:47:48.387734+0000","last_peered":"2023-10-09T10:47:48.387734+0000","last_clean":"2023-10-09T10:47:48.387734+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:47:48.387734+0000","last_undegraded":"2023-10-09T10:47:48.387734+0000","last_fullsized":"2023-10-09T10:47:48.387734+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":171,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.397457+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:47:48.397457+0000","last_peered":"2023-10-09T10:47:48.397457+0000","last_clean":"2023-10-09T10:47:48.397457+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:47:48.397457+0000","last_undegraded":"2023-10-09T10:47:48.397457+0000","last_fullsized":"2023-10-09T10:47:48.397457+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":116,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385781+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:47:48.385781+0000","last_peered":"2023-10-09T10:47:48.385781+0000","last_clean":"2023-10-09T10:47:48.385781+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:47:48.385781+0000","last_undegraded":"2023-10-09T10:47:48.385781+0000","last_fullsized":"2023-10-09T10:47:48.385781+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"179'6212","reported_seq":10247,"reported_epoch":179,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:47:54.621033+0000","last_change":"2023-10-09T10:46:32.248218+0000","last_active":"2023-10-09T10:47:54.621033+0000","last_peered":"2023-10-09T10:47:54.621033+0000","last_clean":"2023-10-09T10:46:02.181912+0000","last_became_active":"2023-10-09T10:46:32.226237+0000","last_became_peered":"2023-10-09T10:46:32.226237+0000","last_unstale":"2023-10-09T10:47:54.621033+0000","last_undegraded":"2023-10-09T10:47:54.621033+0000","last_fullsized":"2023-10-09T10:47:54.621033+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6212,"log_dups_size":0,"ondisk_log_size":6212,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96960512,"num_objects":1479,"num_object_clones":0,"num_object_copies":4437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1479,"num_objects_unfound":0,"num_objects_dirty":1479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5165,"num_write_kb":212640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1479}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":512,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384554+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:47:48.384554+0000","last_peered":"2023-10-09T10:47:48.384554+0000","last_clean":"2023-10-09T10:47:48.384554+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:47:48.384554+0000","last_undegraded":"2023-10-09T10:47:48.384554+0000","last_fullsized":"2023-10-09T10:47:48.384554+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"179'6304","reported_seq":6485,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:54.505020+0000","last_change":"2023-10-09T10:46:02.760963+0000","last_active":"2023-10-09T10:47:54.505020+0000","last_peered":"2023-10-09T10:47:54.505020+0000","last_clean":"2023-10-09T10:47:54.505020+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:47:54.505020+0000","last_undegraded":"2023-10-09T10:47:54.505020+0000","last_fullsized":"2023-10-09T10:47:54.505020+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"141'4416","last_scrub_stamp":"2023-10-09T10:46:02.760810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:46:02.760810+0000","objects_scrubbed":3378,"log_size":6304,"log_dups_size":0,"ondisk_log_size":6304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:35.863002+0000","scrub_duration":0.47769323899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97648640,"num_objects":1490,"num_object_clones":0,"num_object_copies":4470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1888,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":148,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.398045+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:47:48.398045+0000","last_peered":"2023-10-09T10:47:48.398045+0000","last_clean":"2023-10-09T10:47:48.398045+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:47:48.398045+0000","last_undegraded":"2023-10-09T10:47:48.398045+0000","last_fullsized":"2023-10-09T10:47:48.398045+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":164,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.430520+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:47:48.430520+0000","last_peered":"2023-10-09T10:47:48.430520+0000","last_clean":"2023-10-09T10:47:48.430520+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:47:48.430520+0000","last_undegraded":"2023-10-09T10:47:48.430520+0000","last_fullsized":"2023-10-09T10:47:48.430520+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":155,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.387799+0000","last_change":"2023-10-09T10:46:32.245608+0000","last_active":"2023-10-09T10:47:48.387799+0000","last_peered":"2023-10-09T10:47:48.387799+0000","last_clean":"2023-10-09T10:47:48.387799+0000","last_became_active":"2023-10-09T10:46:32.245290+0000","last_became_peered":"2023-10-09T10:46:32.245290+0000","last_unstale":"2023-10-09T10:47:48.387799+0000","last_undegraded":"2023-10-09T10:47:48.387799+0000","last_fullsized":"2023-10-09T10:47:48.387799+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:48.832515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386834+0000","last_change":"2023-10-09T10:46:05.620057+0000","last_active":"2023-10-09T10:47:48.386834+0000","last_peered":"2023-10-09T10:47:48.386834+0000","last_clean":"2023-10-09T10:47:48.386834+0000","last_became_active":"2023-10-09T10:46:05.619703+0000","last_became_peered":"2023-10-09T10:46:05.619703+0000","last_unstale":"2023-10-09T10:47:48.386834+0000","last_undegraded":"2023-10-09T10:47:48.386834+0000","last_fullsized":"2023-10-09T10:47:48.386834+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"179'6268","reported_seq":6384,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:54.539811+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:47:54.539811+0000","last_peered":"2023-10-09T10:47:54.539811+0000","last_clean":"2023-10-09T10:47:54.539811+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:47:54.539811+0000","last_undegraded":"2023-10-09T10:47:54.539811+0000","last_fullsized":"2023-10-09T10:47:54.539811+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6268,"log_dups_size":0,"ondisk_log_size":6268,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96534528,"num_objects":1473,"num_object_clones":0,"num_object_copies":4419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5207,"num_write_kb":213760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":176,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.390719+0000","last_change":"2023-10-09T10:46:32.241041+0000","last_active":"2023-10-09T10:47:48.390719+0000","last_peered":"2023-10-09T10:47:48.390719+0000","last_clean":"2023-10-09T10:47:48.390719+0000","last_became_active":"2023-10-09T10:46:32.238683+0000","last_became_peered":"2023-10-09T10:46:32.238683+0000","last_unstale":"2023-10-09T10:47:48.390719+0000","last_undegraded":"2023-10-09T10:47:48.390719+0000","last_fullsized":"2023-10-09T10:47:48.390719+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:25.817713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":140,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.387037+0000","last_change":"2023-10-09T10:46:32.250104+0000","last_active":"2023-10-09T10:47:48.387037+0000","last_peered":"2023-10-09T10:47:48.387037+0000","last_clean":"2023-10-09T10:47:48.387037+0000","last_became_active":"2023-10-09T10:46:32.249805+0000","last_became_peered":"2023-10-09T10:46:32.249805+0000","last_unstale":"2023-10-09T10:47:48.387037+0000","last_undegraded":"2023-10-09T10:47:48.387037+0000","last_fullsized":"2023-10-09T10:47:48.387037+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:07.230672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":169,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386083+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:47:48.386083+0000","last_peered":"2023-10-09T10:47:48.386083+0000","last_clean":"2023-10-09T10:47:48.386083+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:47:48.386083+0000","last_undegraded":"2023-10-09T10:47:48.386083+0000","last_fullsized":"2023-10-09T10:47:48.386083+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"179'6155","reported_seq":6258,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:56.020638+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:47:56.020638+0000","last_peered":"2023-10-09T10:47:56.020638+0000","last_clean":"2023-10-09T10:47:56.020638+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:47:56.020638+0000","last_undegraded":"2023-10-09T10:47:56.020638+0000","last_fullsized":"2023-10-09T10:47:56.020638+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6155,"log_dups_size":0,"ondisk_log_size":6155,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98500608,"num_objects":1503,"num_object_clones":0,"num_object_copies":4509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5147,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":158,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.397166+0000","last_change":"2023-10-09T10:46:32.235346+0000","last_active":"2023-10-09T10:47:48.397166+0000","last_peered":"2023-10-09T10:47:48.397166+0000","last_clean":"2023-10-09T10:47:48.397166+0000","last_became_active":"2023-10-09T10:46:32.234906+0000","last_became_peered":"2023-10-09T10:46:32.234906+0000","last_unstale":"2023-10-09T10:47:48.397166+0000","last_undegraded":"2023-10-09T10:47:48.397166+0000","last_fullsized":"2023-10-09T10:47:48.397166+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.199251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384317+0000","last_change":"2023-10-09T10:46:05.617146+0000","last_active":"2023-10-09T10:47:48.384317+0000","last_peered":"2023-10-09T10:47:48.384317+0000","last_clean":"2023-10-09T10:47:48.384317+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:47:48.384317+0000","last_undegraded":"2023-10-09T10:47:48.384317+0000","last_fullsized":"2023-10-09T10:47:48.384317+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":107,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.390028+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:47:48.390028+0000","last_peered":"2023-10-09T10:47:48.390028+0000","last_clean":"2023-10-09T10:47:48.390028+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:47:48.390028+0000","last_undegraded":"2023-10-09T10:47:48.390028+0000","last_fullsized":"2023-10-09T10:47:48.390028+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":76,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.430463+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:47:48.430463+0000","last_peered":"2023-10-09T10:47:48.430463+0000","last_clean":"2023-10-09T10:47:48.430463+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:47:48.430463+0000","last_undegraded":"2023-10-09T10:47:48.430463+0000","last_fullsized":"2023-10-09T10:47:48.430463+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384810+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:47:48.384810+0000","last_peered":"2023-10-09T10:47:48.384810+0000","last_clean":"2023-10-09T10:47:48.384810+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:47:48.384810+0000","last_undegraded":"2023-10-09T10:47:48.384810+0000","last_fullsized":"2023-10-09T10:47:48.384810+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":170,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.387196+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:47:48.387196+0000","last_peered":"2023-10-09T10:47:48.387196+0000","last_clean":"2023-10-09T10:47:48.387196+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:47:48.387196+0000","last_undegraded":"2023-10-09T10:47:48.387196+0000","last_fullsized":"2023-10-09T10:47:48.387196+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":56,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384501+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:47:48.384501+0000","last_peered":"2023-10-09T10:47:48.384501+0000","last_clean":"2023-10-09T10:47:48.384501+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:47:48.384501+0000","last_undegraded":"2023-10-09T10:47:48.384501+0000","last_fullsized":"2023-10-09T10:47:48.384501+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":56,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385932+0000","last_change":"2023-10-09T10:46:32.236202+0000","last_active":"2023-10-09T10:47:48.385932+0000","last_peered":"2023-10-09T10:47:48.385932+0000","last_clean":"2023-10-09T10:47:48.385932+0000","last_became_active":"2023-10-09T10:46:32.235901+0000","last_became_peered":"2023-10-09T10:46:32.235901+0000","last_unstale":"2023-10-09T10:47:48.385932+0000","last_undegraded":"2023-10-09T10:47:48.385932+0000","last_fullsized":"2023-10-09T10:47:48.385932+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:48.260403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"179'6321","reported_seq":6471,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:56.472814+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:47:56.472814+0000","last_peered":"2023-10-09T10:47:56.472814+0000","last_clean":"2023-10-09T10:47:56.472814+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:47:56.472814+0000","last_undegraded":"2023-10-09T10:47:56.472814+0000","last_fullsized":"2023-10-09T10:47:56.472814+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":6321,"log_dups_size":0,"ondisk_log_size":6321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95485952,"num_objects":1457,"num_object_clones":0,"num_object_copies":4371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3291,"num_write_kb":89664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385693+0000","last_change":"2023-10-09T10:46:05.620764+0000","last_active":"2023-10-09T10:47:48.385693+0000","last_peered":"2023-10-09T10:47:48.385693+0000","last_clean":"2023-10-09T10:47:48.385693+0000","last_became_active":"2023-10-09T10:46:05.619870+0000","last_became_peered":"2023-10-09T10:46:05.619870+0000","last_unstale":"2023-10-09T10:47:48.385693+0000","last_undegraded":"2023-10-09T10:47:48.385693+0000","last_fullsized":"2023-10-09T10:47:48.385693+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":167,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.462453+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:47:48.462453+0000","last_peered":"2023-10-09T10:47:48.462453+0000","last_clean":"2023-10-09T10:47:48.462453+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:47:48.462453+0000","last_undegraded":"2023-10-09T10:47:48.462453+0000","last_fullsized":"2023-10-09T10:47:48.462453+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":140,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.387868+0000","last_change":"2023-10-09T10:46:32.251450+0000","last_active":"2023-10-09T10:47:48.387868+0000","last_peered":"2023-10-09T10:47:48.387868+0000","last_clean":"2023-10-09T10:47:48.387868+0000","last_became_active":"2023-10-09T10:46:32.250933+0000","last_became_peered":"2023-10-09T10:46:32.250933+0000","last_unstale":"2023-10-09T10:47:48.387868+0000","last_undegraded":"2023-10-09T10:47:48.387868+0000","last_fullsized":"2023-10-09T10:47:48.387868+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:04.116178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":47,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384918+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:47:48.384918+0000","last_peered":"2023-10-09T10:47:48.384918+0000","last_clean":"2023-10-09T10:47:48.384918+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:47:48.384918+0000","last_undegraded":"2023-10-09T10:47:48.384918+0000","last_fullsized":"2023-10-09T10:47:48.384918+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"179'6181","reported_seq":12876,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:56.502999+0000","last_change":"2023-10-09T10:47:02.269289+0000","last_active":"2023-10-09T10:47:56.502999+0000","last_peered":"2023-10-09T10:47:56.502999+0000","last_clean":"2023-10-09T10:47:56.502999+0000","last_became_active":"2023-10-09T10:47:02.257854+0000","last_became_peered":"2023-10-09T10:47:02.257854+0000","last_unstale":"2023-10-09T10:47:56.502999+0000","last_undegraded":"2023-10-09T10:47:56.502999+0000","last_fullsized":"2023-10-09T10:47:56.502999+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6181,"log_dups_size":0,"ondisk_log_size":6181,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93683712,"num_objects":1429,"num_object_clones":0,"num_object_copies":4287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5173,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3287,"num_bytes_recovered":181927936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":513,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.387103+0000","last_change":"2023-10-09T10:46:05.618443+0000","last_active":"2023-10-09T10:47:48.387103+0000","last_peered":"2023-10-09T10:47:48.387103+0000","last_clean":"2023-10-09T10:47:48.387103+0000","last_became_active":"2023-10-09T10:46:05.616588+0000","last_became_peered":"2023-10-09T10:46:05.616588+0000","last_unstale":"2023-10-09T10:47:48.387103+0000","last_undegraded":"2023-10-09T10:47:48.387103+0000","last_fullsized":"2023-10-09T10:47:48.387103+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":172,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.213364+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:47:52.213364+0000","last_peered":"2023-10-09T10:47:52.213364+0000","last_clean":"2023-10-09T10:47:52.213364+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:47:52.213364+0000","last_undegraded":"2023-10-09T10:47:52.213364+0000","last_fullsized":"2023-10-09T10:47:52.213364+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":116,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.387282+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:47:48.387282+0000","last_peered":"2023-10-09T10:47:48.387282+0000","last_clean":"2023-10-09T10:47:48.387282+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:47:48.387282+0000","last_undegraded":"2023-10-09T10:47:48.387282+0000","last_fullsized":"2023-10-09T10:47:48.387282+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":161,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.392863+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:47:48.392863+0000","last_peered":"2023-10-09T10:47:48.392863+0000","last_clean":"2023-10-09T10:47:48.392863+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:47:48.392863+0000","last_undegraded":"2023-10-09T10:47:48.392863+0000","last_fullsized":"2023-10-09T10:47:48.392863+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"179'6317","reported_seq":9404,"reported_epoch":179,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:56.053525+0000","last_change":"2023-10-09T10:47:29.622439+0000","last_active":"2023-10-09T10:47:56.053525+0000","last_peered":"2023-10-09T10:47:56.053525+0000","last_clean":"2023-10-09T10:46:31.130083+0000","last_became_active":"2023-10-09T10:46:33.493288+0000","last_became_peered":"2023-10-09T10:46:33.493288+0000","last_unstale":"2023-10-09T10:47:56.053525+0000","last_undegraded":"2023-10-09T10:46:33.283624+0000","last_fullsized":"2023-10-09T10:46:33.244043+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6317,"log_dups_size":0,"ondisk_log_size":6317,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90734592,"num_objects":1384,"num_object_clones":0,"num_object_copies":4152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":838,"num_objects_misplaced":508,"num_objects_unfound":0,"num_objects_dirty":1384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5280,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1469,"num_bytes_recovered":96272384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":508},{"shards":"3,4","objects":38}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":512,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385223+0000","last_change":"2023-10-09T10:46:05.617589+0000","last_active":"2023-10-09T10:47:48.385223+0000","last_peered":"2023-10-09T10:47:48.385223+0000","last_clean":"2023-10-09T10:47:48.385223+0000","last_became_active":"2023-10-09T10:46:05.616400+0000","last_became_peered":"2023-10-09T10:46:05.616400+0000","last_unstale":"2023-10-09T10:47:48.385223+0000","last_undegraded":"2023-10-09T10:47:48.385223+0000","last_fullsized":"2023-10-09T10:47:48.385223+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.390234+0000","last_change":"2023-10-09T10:46:32.245106+0000","last_active":"2023-10-09T10:47:48.390234+0000","last_peered":"2023-10-09T10:47:48.390234+0000","last_clean":"2023-10-09T10:47:48.390234+0000","last_became_active":"2023-10-09T10:46:32.244826+0000","last_became_peered":"2023-10-09T10:46:32.244826+0000","last_unstale":"2023-10-09T10:47:48.390234+0000","last_undegraded":"2023-10-09T10:47:48.390234+0000","last_fullsized":"2023-10-09T10:47:48.390234+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:30.322834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":51,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.385854+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:47:48.385854+0000","last_peered":"2023-10-09T10:47:48.385854+0000","last_clean":"2023-10-09T10:47:48.385854+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:47:48.385854+0000","last_undegraded":"2023-10-09T10:47:48.385854+0000","last_fullsized":"2023-10-09T10:47:48.385854+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":126,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.430019+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:47:48.430019+0000","last_peered":"2023-10-09T10:47:48.430019+0000","last_clean":"2023-10-09T10:47:48.430019+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:47:48.430019+0000","last_undegraded":"2023-10-09T10:47:48.430019+0000","last_fullsized":"2023-10-09T10:47:48.430019+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"179'6183","reported_seq":12991,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:55.966859+0000","last_change":"2023-10-09T10:47:31.271196+0000","last_active":"2023-10-09T10:47:55.966859+0000","last_peered":"2023-10-09T10:47:55.966859+0000","last_clean":"2023-10-09T10:47:55.966859+0000","last_became_active":"2023-10-09T10:47:31.229504+0000","last_became_peered":"2023-10-09T10:47:31.229504+0000","last_unstale":"2023-10-09T10:47:55.966859+0000","last_undegraded":"2023-10-09T10:47:55.966859+0000","last_fullsized":"2023-10-09T10:47:55.966859+0000","mapping_epoch":172,"log_start":"104'961","ondisk_log_start":"104'961","created":98,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":5222,"log_dups_size":0,"ondisk_log_size":5222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:26.533476+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94568448,"num_objects":1443,"num_object_clones":0,"num_object_copies":4329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3538,"num_write_kb":107264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3305,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":47,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.384996+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:47:48.384996+0000","last_peered":"2023-10-09T10:47:48.384996+0000","last_clean":"2023-10-09T10:47:48.384996+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:47:48.384996+0000","last_undegraded":"2023-10-09T10:47:48.384996+0000","last_fullsized":"2023-10-09T10:47:48.384996+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":110,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.387930+0000","last_change":"2023-10-09T10:46:32.251242+0000","last_active":"2023-10-09T10:47:48.387930+0000","last_peered":"2023-10-09T10:47:48.387930+0000","last_clean":"2023-10-09T10:47:48.387930+0000","last_became_active":"2023-10-09T10:46:32.250534+0000","last_became_peered":"2023-10-09T10:46:32.250534+0000","last_unstale":"2023-10-09T10:47:48.387930+0000","last_undegraded":"2023-10-09T10:47:48.387930+0000","last_fullsized":"2023-10-09T10:47:48.387930+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:33.726041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"179'6277","reported_seq":10260,"reported_epoch":179,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:57.643340+0000","last_change":"2023-10-09T10:46:34.514479+0000","last_active":"2023-10-09T10:47:57.643340+0000","last_peered":"2023-10-09T10:47:57.643340+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:46:34.409667+0000","last_became_peered":"2023-10-09T10:46:34.409667+0000","last_unstale":"2023-10-09T10:47:57.643340+0000","last_undegraded":"2023-10-09T10:46:34.376635+0000","last_fullsized":"2023-10-09T10:46:33.237998+0000","mapping_epoch":154,"log_start":"103'146","ondisk_log_start":"103'146","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6131,"log_dups_size":0,"ondisk_log_size":6131,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94306304,"num_objects":1439,"num_object_clones":0,"num_object_copies":4317,"num_objects_missing_on_primary":1406,"num_objects_missing":1406,"num_objects_degraded":1916,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5235,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1932,"num_bytes_recovered":126615552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,7","objects":33},{"shards":"2,3,7","objects":1406}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":146,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.397691+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:47:48.397691+0000","last_peered":"2023-10-09T10:47:48.397691+0000","last_clean":"2023-10-09T10:47:48.397691+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:47:48.397691+0000","last_undegraded":"2023-10-09T10:47:48.397691+0000","last_fullsized":"2023-10-09T10:47:48.397691+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":160,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.430641+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:47:48.430641+0000","last_peered":"2023-10-09T10:47:48.430641+0000","last_clean":"2023-10-09T10:47:48.430641+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:47:48.430641+0000","last_undegraded":"2023-10-09T10:47:48.430641+0000","last_fullsized":"2023-10-09T10:47:48.430641+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"179'6175","reported_seq":6286,"reported_epoch":179,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:47:54.503756+0000","last_change":"2023-10-09T10:46:33.546991+0000","last_active":"2023-10-09T10:47:54.503756+0000","last_peered":"2023-10-09T10:47:54.503756+0000","last_clean":"2023-10-09T10:46:31.009484+0000","last_became_active":"2023-10-09T10:46:33.535665+0000","last_became_peered":"2023-10-09T10:46:33.535665+0000","last_unstale":"2023-10-09T10:47:54.503756+0000","last_undegraded":"2023-10-09T10:46:33.277168+0000","last_fullsized":"2023-10-09T10:46:33.235038+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6175,"log_dups_size":0,"ondisk_log_size":6175,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92831744,"num_objects":1417,"num_object_clones":0,"num_object_copies":4251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5111,"num_write_kb":208864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1429},{"shards":"3,7","objects":-1341}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":129,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.397723+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:47:48.397723+0000","last_peered":"2023-10-09T10:47:48.397723+0000","last_clean":"2023-10-09T10:47:48.397723+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:47:48.397723+0000","last_undegraded":"2023-10-09T10:47:48.397723+0000","last_fullsized":"2023-10-09T10:47:48.397723+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":27,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.433294+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:47:48.433294+0000","last_peered":"2023-10-09T10:47:48.433294+0000","last_clean":"2023-10-09T10:47:48.433294+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:47:48.433294+0000","last_undegraded":"2023-10-09T10:47:48.433294+0000","last_fullsized":"2023-10-09T10:47:48.433294+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":163,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.394440+0000","last_change":"2023-10-09T10:46:32.244607+0000","last_active":"2023-10-09T10:47:48.394440+0000","last_peered":"2023-10-09T10:47:48.394440+0000","last_clean":"2023-10-09T10:47:48.394440+0000","last_became_active":"2023-10-09T10:46:32.242628+0000","last_became_peered":"2023-10-09T10:46:32.242628+0000","last_unstale":"2023-10-09T10:47:48.394440+0000","last_undegraded":"2023-10-09T10:47:48.394440+0000","last_fullsized":"2023-10-09T10:47:48.394440+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:10.738819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":193,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.390481+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:47:48.390481+0000","last_peered":"2023-10-09T10:47:48.390481+0000","last_clean":"2023-10-09T10:47:48.390481+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:47:48.390481+0000","last_undegraded":"2023-10-09T10:47:48.390481+0000","last_fullsized":"2023-10-09T10:47:48.390481+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":68,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.386325+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:47:48.386325+0000","last_peered":"2023-10-09T10:47:48.386325+0000","last_clean":"2023-10-09T10:47:48.386325+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:47:48.386325+0000","last_undegraded":"2023-10-09T10:47:48.386325+0000","last_fullsized":"2023-10-09T10:47:48.386325+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":143,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-09T10:47:48.429846+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:47:48.429846+0000","last_peered":"2023-10-09T10:47:48.429846+0000","last_clean":"2023-10-09T10:47:48.429846+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:47:48.429846+0000","last_undegraded":"2023-10-09T10:47:48.429846+0000","last_fullsized":"2023-10-09T10:47:48.429846+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":3006496792,"num_objects":45877,"num_object_clones":0,"num_object_copies":137631,"num_objects_missing_on_primary":1406,"num_objects_missing":1406,"num_objects_degraded":31152,"num_objects_misplaced":4954,"num_objects_unfound":0,"num_objects_dirty":45877,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158331,"num_write_kb":6258529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23293,"num_bytes_recovered":1433862144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11143426048,"data_stored":11143413832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":165131,"ondisk_log_size":165131,"up":78,"acting":68,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1186,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332256,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1589068,"kb_used_data":1291716,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92156084,"statfs":{"total":95995035648,"available":94367830016,"internally_reserved":0,"allocated":1322717184,"data_stored":1322035883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":0.79200000000000004,"15min":0.65000000000000002},"min":{"1min":0.26300000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.1479999999999997,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":1.482},{"interface":"front","average":{"1min":1.546,"5min":0.97199999999999998,"15min":0.752},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.9909999999999997,"5min":10.207000000000001,"15min":10.207000000000001},"last":0.48299999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:47:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2210000000000001,"5min":1.3300000000000001,"15min":0.80400000000000005},"min":{"1min":0.43099999999999999,"5min":0.318,"15min":0.318},"max":{"1min":11.826000000000001,"5min":11.826000000000001,"15min":11.826000000000001},"last":0.58799999999999997},{"interface":"front","average":{"1min":2.286,"5min":1.155,"15min":0.73899999999999999},"min":{"1min":0.32100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":11.84,"5min":11.84,"15min":11.84},"last":1.5469999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.1499999999999999,"15min":0.72399999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.4770000000000003,"5min":6.4770000000000003,"15min":6.4770000000000003},"last":0.81799999999999995},{"interface":"front","average":{"1min":1.272,"5min":0.85699999999999998,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.251,"15min":0.251},"max":{"1min":3.1480000000000001,"5min":3.7120000000000002,"15min":3.7120000000000002},"last":1.52}]},{"osd":3,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.23,"15min":0.80000000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.0599999999999996,"5min":8.1639999999999997,"15min":8.1639999999999997},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.099,"15min":0.76600000000000001},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.504,"5min":12.451000000000001,"15min":12.451000000000001},"last":0.76200000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.73299999999999998,"15min":0.56200000000000006},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.1420000000000003,"5min":7.4509999999999996,"15min":7.4509999999999996},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.65300000000000002,"15min":0.53600000000000003},"min":{"1min":0.29799999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.2749999999999999,"5min":3.1699999999999999,"15min":3.1699999999999999},"last":1.5369999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.56399999999999995,"15min":0.52700000000000002},"min":{"1min":0.34300000000000003,"5min":0.247,"15min":0.247},"max":{"1min":2.2349999999999999,"5min":2.2349999999999999,"15min":2.2349999999999999},"last":1.978},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.81299999999999994,"15min":0.63},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":5.0860000000000003,"5min":10.33,"15min":10.33},"last":0.54200000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.83399999999999996,"15min":0.64800000000000002},"min":{"1min":0.311,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.109,"5min":10.164999999999999,"15min":10.164999999999999},"last":1.0389999999999999},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.66000000000000003,"15min":0.58299999999999996},"min":{"1min":0.33300000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.306,"5min":2.8079999999999998,"15min":2.8079999999999998},"last":1.3009999999999999}]}]},{"osd":6,"up_from":44,"seq":188978561187,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1340296,"kb_used_data":1042944,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92404856,"statfs":{"total":95995035648,"available":94622572544,"internally_reserved":0,"allocated":1067974656,"data_stored":1067297893,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.93200000000000005,"15min":0.71799999999999997},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.4630000000000001,"5min":7.3220000000000001,"15min":7.3220000000000001},"last":3.2029999999999998},{"interface":"front","average":{"1min":1.02,"5min":1.002,"15min":0.73899999999999999},"min":{"1min":0.33900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.4969999999999999,"5min":7.5309999999999997,"15min":7.5309999999999997},"last":6.4550000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.157,"15min":0.77900000000000003},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.245},"max":{"1min":6.7560000000000002,"5min":16.747,"15min":16.747},"last":1.272},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.284,"15min":0.79500000000000004},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.3579999999999997,"5min":16.734000000000002,"15min":16.734000000000002},"last":5.8689999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":0.95799999999999996,"15min":0.67500000000000004},"min":{"1min":0.28699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5069999999999997,"5min":7.5069999999999997,"15min":7.5069999999999997},"last":1.948},{"interface":"front","average":{"1min":1.675,"5min":1.143,"15min":0.755},"min":{"1min":0.28599999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.3899999999999997,"5min":12.571,"15min":12.571},"last":0.48699999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.117,"15min":0.73099999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.8070000000000004,"5min":6.8070000000000004,"15min":6.8070000000000004},"last":4.2089999999999996},{"interface":"front","average":{"1min":2.1219999999999999,"5min":1.385,"15min":0.82599999999999996},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.3940000000000001,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":1.9159999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":0.66600000000000004,"15min":0.51500000000000001},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":4.0739999999999998,"5min":4.0739999999999998,"15min":4.0739999999999998},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.67300000000000004,"15min":0.54600000000000004},"min":{"1min":0.34899999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6600000000000001,"5min":3.6600000000000001,"15min":3.6600000000000001},"last":0.75700000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":0.76000000000000001,"15min":0.56999999999999995},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.8949999999999996,"5min":4.8949999999999996,"15min":4.8949999999999996},"last":1.7430000000000001},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.76900000000000002,"15min":0.59999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":4.3639999999999999,"5min":4.3639999999999999,"15min":4.3639999999999999},"last":0.79200000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:47:25 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69899999999999995,"15min":0.57999999999999996},"min":{"1min":0.35799999999999998,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":2.5049999999999999,"5min":3.472,"15min":3.472},"last":2.6230000000000002},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.84299999999999997,"15min":0.65700000000000003},"min":{"1min":0.373,"5min":0.36899999999999999,"15min":0.34000000000000002},"max":{"1min":4.0800000000000001,"5min":4.0800000000000001,"15min":4.0800000000000001},"last":0.81599999999999995}]}]},{"osd":1,"up_from":85,"seq":365072220226,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1347260,"kb_used_data":1217268,"kb_used_omap":9,"kb_used_meta":129974,"kb_avail":92397892,"statfs":{"total":95995035648,"available":94615441408,"internally_reserved":0,"allocated":1246482432,"data_stored":1245585713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":133094377},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.581,"15min":0.76900000000000002},"min":{"1min":0.251,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.2119999999999997,"5min":14.1,"15min":14.1},"last":0.71099999999999997},{"interface":"front","average":{"1min":1.885,"5min":1.5629999999999999,"15min":0.78700000000000003},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.8090000000000002,"5min":14.465,"15min":14.465},"last":0.65900000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:47:13 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.1200000000000001,"15min":0.61799999999999999},"min":{"1min":0.21199999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.4829999999999997,"5min":8.6859999999999999,"15min":8.6859999999999999},"last":5.6070000000000002},{"interface":"front","average":{"1min":1.919,"5min":1.4690000000000001,"15min":0.746},"min":{"1min":0.29099999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.2829999999999995,"5min":14.234,"15min":14.234},"last":0.40999999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":1.5269999999999999,"15min":0.82299999999999995},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":7.234,"5min":14.404999999999999,"15min":14.404999999999999},"last":2.1509999999999998},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.429,"15min":0.70599999999999996},"min":{"1min":0.30399999999999999,"5min":0.251,"15min":0.251},"max":{"1min":15.468999999999999,"5min":15.468999999999999,"15min":15.468999999999999},"last":7.5090000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.1679999999999999,"15min":0.78400000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.1449999999999996,"5min":8.0129999999999999,"15min":8.0129999999999999},"last":3.5030000000000001},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.2350000000000001,"15min":0.82499999999999996},"min":{"1min":0.30299999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.1189999999999998,"5min":8.327,"15min":8.327},"last":0.372}]},{"osd":5,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":1.111,"15min":0.78900000000000003},"min":{"1min":0.28899999999999998,"5min":0.253,"15min":0.253},"max":{"1min":5.4850000000000003,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":0.69099999999999995},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.2929999999999999,"15min":0.86199999999999999},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.1859999999999999,"5min":7.923,"15min":7.923},"last":17.085000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7909999999999999,"5min":1.2030000000000001,"15min":0.82199999999999995},"min":{"1min":0.35499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.9450000000000003,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":5.5170000000000003},{"interface":"front","average":{"1min":1.246,"5min":1.2949999999999999,"15min":0.86299999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.8440000000000003,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":3.4769999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.22,"15min":0.83999999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.8689999999999998,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":3.532},{"interface":"front","average":{"1min":1.992,"5min":1.2909999999999999,"15min":0.86099999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":7.1589999999999998,"5min":8.2469999999999999,"15min":8.2469999999999999},"last":6.9809999999999999}]}]},{"osd":0,"up_from":80,"seq":343597383760,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":584668,"kb_used_data":490708,"kb_used_omap":28,"kb_used_meta":93923,"kb_avail":93160484,"statfs":{"total":95995035648,"available":95396335616,"internally_reserved":0,"allocated":502484992,"data_stored":501555634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":96178000},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:47:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2509999999999999,"5min":1.5449999999999999,"15min":0.76000000000000001},"min":{"1min":0.22,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":10.564,"5min":25.475000000000001,"15min":25.475000000000001},"last":0.21099999999999999},{"interface":"front","average":{"1min":2.3740000000000001,"5min":1.5940000000000001,"15min":0.77400000000000002},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":10.593999999999999,"5min":20.832999999999998,"15min":20.832999999999998},"last":0.311}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":1.2350000000000001,"15min":0.64400000000000002},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.907,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":0.28499999999999998},{"interface":"front","average":{"1min":1.754,"5min":1.5349999999999999,"15min":0.73999999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.5619999999999994,"5min":14.391999999999999,"15min":14.391999999999999},"last":0.27100000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:47:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4740000000000002,"5min":1.8959999999999999,"15min":1.0660000000000001},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.215},"max":{"1min":10.644,"5min":14.926,"15min":14.926},"last":0.34699999999999998},{"interface":"front","average":{"1min":2.2269999999999999,"5min":1.5940000000000001,"15min":0.89200000000000002},"min":{"1min":0.34499999999999997,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.228999999999999,"5min":11.868,"15min":11.868},"last":0.32300000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.2250000000000001,"15min":0.82199999999999995},"min":{"1min":0.38400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.8150000000000004,"5min":6.8150000000000004,"15min":6.8150000000000004},"last":0.65600000000000003},{"interface":"front","average":{"1min":2.391,"5min":1.405,"15min":0.88100000000000001},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.676,"5min":10.676,"15min":10.676},"last":3.2349999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":1.4359999999999999,"15min":0.89600000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":10.734,"5min":10.734,"15min":10.734},"last":1.9259999999999999},{"interface":"front","average":{"1min":1.716,"5min":1.1659999999999999,"15min":0.82099999999999995},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.54,"5min":8.9190000000000005,"15min":8.9190000000000005},"last":0.40500000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.391,"15min":0.88400000000000001},"min":{"1min":0.36599999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.706,"5min":10.706,"15min":10.706},"last":3.8090000000000002},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.0780000000000001,"15min":0.79300000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.0499999999999998,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":2.02}]},{"osd":7,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.474,"5min":1.151,"15min":0.81899999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.5380000000000003,"5min":8.875,"15min":8.875},"last":0.94799999999999995},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.425,"15min":0.92700000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":14.394,"5min":14.394,"15min":14.394},"last":0.621}]}]},{"osd":2,"up_from":0,"seq":386547056700,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893174,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2258056,"kb_used_data":2063808,"kb_used_omap":7,"kb_used_meta":193976,"kb_avail":91487096,"statfs":{"total":95995035648,"available":93682786304,"internally_reserved":0,"allocated":2113339392,"data_stored":2112428498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":198632040},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,4,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":1.3140000000000001,"15min":0.65200000000000002},"min":{"1min":0.192,"5min":0.183,"15min":0.183},"max":{"1min":7.0289999999999999,"5min":11.557,"15min":11.557},"last":3.895},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.1379999999999999,"15min":0.65000000000000002},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":6.681,"5min":7.0510000000000002,"15min":7.0510000000000002},"last":3.093}]},{"osd":1,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":1.6970000000000001,"15min":0.78500000000000003},"min":{"1min":0.27100000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.048,"5min":14.707000000000001,"15min":14.707000000000001},"last":3.1869999999999998},{"interface":"front","average":{"1min":1.6220000000000001,"5min":1.1890000000000001,"15min":0.64500000000000002},"min":{"1min":0.19400000000000001,"5min":0.193,"15min":0.193},"max":{"1min":7.1630000000000003,"5min":11.629,"15min":11.629},"last":3.851}]},{"osd":2,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.6739999999999999,"15min":0.80200000000000005},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.9180000000000001,"5min":13.853,"15min":13.853},"last":0.35999999999999999},{"interface":"front","average":{"1min":2.5590000000000002,"5min":1.5129999999999999,"15min":0.76800000000000002},"min":{"1min":0.30099999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.71,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":0.437}]},{"osd":4,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.2210000000000001,"15min":0.78800000000000003},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.9909999999999997,"5min":11.532,"15min":11.532},"last":14.071999999999999},{"interface":"front","average":{"1min":2.2440000000000002,"5min":1.296,"15min":0.83699999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.962999999999999,"5min":11.962999999999999,"15min":11.962999999999999},"last":14.048}]},{"osd":5,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7589999999999999,"5min":1.8029999999999999,"15min":0.99199999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.907,"5min":51.509999999999998,"15min":51.509999999999998},"last":4.6289999999999996},{"interface":"front","average":{"1min":2.1469999999999998,"5min":2.306,"15min":1.1439999999999999},"min":{"1min":0.36799999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.1139999999999999,"5min":55.338999999999999,"15min":55.338999999999999},"last":3.9950000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":1.3620000000000001,"15min":0.84599999999999997},"min":{"1min":0.378,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":11.92,"5min":11.92,"15min":11.92},"last":0.504},{"interface":"front","average":{"1min":2.3759999999999999,"5min":1.776,"15min":0.99199999999999999},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":10.528,"5min":13.891999999999999,"15min":13.891999999999999},"last":3.1539999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":2.3410000000000002,"15min":1.175},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.6520000000000001,"5min":51.744,"15min":51.744},"last":9.516},{"interface":"front","average":{"1min":1.9139999999999999,"5min":1.869,"15min":1.0049999999999999},"min":{"1min":0.33000000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.133,"5min":53.399000000000001,"15min":53.399000000000001},"last":4.4059999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019050,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1239432,"kb_used_data":925760,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":92505720,"statfs":{"total":95995035648,"available":94725857280,"internally_reserved":0,"allocated":947978240,"data_stored":947300744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.0129999999999999,"15min":0.70899999999999996},"min":{"1min":0.24099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.5419999999999998,"5min":10.387,"15min":10.387},"last":1.3899999999999999},{"interface":"front","average":{"1min":1.581,"5min":1.1819999999999999,"15min":0.78400000000000003},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.5999999999999996,"5min":10.311999999999999,"15min":10.311999999999999},"last":5.375}]},{"osd":1,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.1519999999999999,"15min":0.752},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.9100000000000001,"5min":6.907,"15min":6.907},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.859,"5min":1.2390000000000001,"15min":0.82699999999999996},"min":{"1min":0.30099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.7669999999999995,"5min":8.7669999999999995,"15min":8.7669999999999995},"last":5.4660000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:47:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2170000000000001,"5min":1.1000000000000001,"15min":0.747},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.2149999999999999,"5min":7.5449999999999999,"15min":7.5449999999999999},"last":6.1200000000000001},{"interface":"front","average":{"1min":1.579,"5min":1.1639999999999999,"15min":0.76300000000000001},"min":{"1min":0.36699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.5110000000000001,"5min":6.2939999999999996,"15min":6.2939999999999996},"last":20.545999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:47:38 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.333,"15min":0.91900000000000004},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.8170000000000002,"5min":8.8170000000000002,"15min":8.8170000000000002},"last":5.492},{"interface":"front","average":{"1min":2.3650000000000002,"5min":1.359,"15min":0.84499999999999997},"min":{"1min":0.34300000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.625,"5min":8.9809999999999999,"15min":8.9809999999999999},"last":5.5629999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":0.89500000000000002,"15min":0.59699999999999998},"min":{"1min":0.25600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":8.6950000000000003,"5min":8.6950000000000003,"15min":8.6950000000000003},"last":24.149999999999999},{"interface":"front","average":{"1min":1.6579999999999999,"5min":0.94299999999999995,"15min":0.61299999999999999},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.252},"max":{"1min":4.8879999999999999,"5min":4.8879999999999999,"15min":4.8879999999999999},"last":5.4089999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.74099999999999999,"15min":0.55700000000000005},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":7.1580000000000004,"5min":7.1580000000000004,"15min":7.1580000000000004},"last":12.391999999999999},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.71599999999999997,"15min":0.60899999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":3.6150000000000002,"5min":3.6150000000000002,"15min":3.6150000000000002},"last":12.539}]},{"osd":7,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.80800000000000005,"15min":0.64600000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.1050000000000004,"5min":4.1050000000000004,"15min":4.1050000000000004},"last":5.4409999999999998},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.748,"15min":0.60799999999999998},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0919999999999996,"5min":4.0919999999999996,"15min":4.0919999999999996},"last":5.3390000000000004}]}]},{"osd":5,"up_from":37,"seq":158913790119,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1495864,"kb_used_data":1194480,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92249288,"statfs":{"total":95995035648,"available":94463270912,"internally_reserved":0,"allocated":1223147520,"data_stored":1222481853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.77600000000000002,"15min":0.65700000000000003},"min":{"1min":0.247,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.9860000000000002,"5min":4.8380000000000001,"15min":4.8380000000000001},"last":2.9089999999999998},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.84899999999999998,"15min":0.72899999999999998},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.0209999999999999,"5min":4.8650000000000002,"15min":4.8650000000000002},"last":2.9300000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":1.367,"15min":0.86099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":12.999000000000001,"5min":12.999000000000001,"15min":12.999000000000001},"last":1.2749999999999999},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.2370000000000001,"15min":0.80700000000000005},"min":{"1min":0.27600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3730000000000002,"5min":7.3730000000000002,"15min":7.3730000000000002},"last":2.8319999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:47:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.02,"15min":0.72099999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.4569999999999999,"5min":7.9000000000000004,"15min":7.9000000000000004},"last":1.302},{"interface":"front","average":{"1min":1.7569999999999999,"5min":0.90400000000000003,"15min":0.68400000000000005},"min":{"1min":0.38,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":10.352,"5min":10.352,"15min":10.352},"last":2.8610000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4969999999999999,"5min":1.621,"15min":1.0229999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":18.016999999999999,"5min":18.016999999999999,"15min":18.016999999999999},"last":1.2470000000000001},{"interface":"front","average":{"1min":2.964,"5min":1.673,"15min":0.96599999999999997},"min":{"1min":0.318,"5min":0.317,"15min":0.317},"max":{"1min":15.555999999999999,"5min":15.555999999999999,"15min":15.555999999999999},"last":1.2130000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":0.83799999999999997,"15min":0.57799999999999996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":19.152999999999999,"5min":19.152999999999999,"15min":19.152999999999999},"last":3.722},{"interface":"front","average":{"1min":1.986,"5min":0.96999999999999997,"15min":0.63500000000000001},"min":{"1min":0.26800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.3059999999999992,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":1.169}]},{"osd":6,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.66300000000000003,"15min":0.53500000000000003},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":2.8900000000000001,"5min":2.9529999999999998,"15min":2.9529999999999998},"last":1.3240000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.63300000000000001,"15min":0.53800000000000003},"min":{"1min":0.34100000000000003,"5min":0.33300000000000002,"15min":0.30299999999999999},"max":{"1min":3.8159999999999998,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":5.226}]},{"osd":7,"last update":"Mon Oct 9 10:47:22 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.67100000000000004,"15min":0.58799999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.4729999999999999,"5min":4.0510000000000002,"15min":4.0510000000000002},"last":5.9240000000000004},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.70899999999999996,"15min":0.60099999999999998},"min":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":4.1289999999999996,"5min":4.1289999999999996,"15min":4.1289999999999996},"last":2.8039999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":494800896,"data_stored":494796824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1230114816,"data_stored":1230110744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2817064960,"data_stored":2817064960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2080047104,"data_stored":2080047104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":931725312,"data_stored":931725312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1213534208,"data_stored":1213530136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1064566784,"data_stored":1064566784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1311571968,"data_stored":1311571968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:48:00.668 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:48:00.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:00.807 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:00.884 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:48:00.970 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:48:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=sqlstore.transactions t=2023-10-09T10:48:00.805910721Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T10:48:00.970 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:48:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=cleanup t=2023-10-09T10:48:00.806286632Z level=info msg="Completed cleanup jobs" duration=13.754796ms 2023-10-09T10:48:01.141 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:01.219 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:48:01] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.43.0" 2023-10-09T10:48:01.392 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:01.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:01.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:01.505 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:01.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[195507]: from='client.35025 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:01.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:01.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3392810936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:48:01.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[195507]: osdmap e180: 8 total, 8 up, 7 in 2023-10-09T10:48:01.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[195507]: failed to encode map e180 with expected crc 2023-10-09T10:48:01.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[195507]: failed to encode map e180 with expected crc 2023-10-09T10:48:01.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[195507]: failed to encode map e180 with expected crc 2023-10-09T10:48:01.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[195507]: failed to encode map e180 with expected crc 2023-10-09T10:48:01.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[195507]: failed to encode map e180 with expected crc 2023-10-09T10:48:01.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[200197]: from='client.35025 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:01.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:01.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3392810936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:48:01.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[200197]: osdmap e180: 8 total, 8 up, 7 in 2023-10-09T10:48:01.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[200197]: failed to encode map e180 with expected crc 2023-10-09T10:48:01.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[200197]: failed to encode map e180 with expected crc 2023-10-09T10:48:01.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[200197]: failed to encode map e180 with expected crc 2023-10-09T10:48:01.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[200197]: failed to encode map e180 with expected crc 2023-10-09T10:48:01.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:01 smithi125 ceph-mon[200197]: failed to encode map e180 with expected crc 2023-10-09T10:48:01.562 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:01.592 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:01.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:01.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:01.688 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:01.704 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:01.709 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:01 smithi155 ceph-mon[184854]: from='client.35025 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:01.709 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:01 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:01.709 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3392810936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:48:01.709 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:01 smithi155 ceph-mon[184854]: osdmap e180: 8 total, 8 up, 7 in 2023-10-09T10:48:01.709 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:01 smithi155 ceph-mon[184854]: failed to encode map e180 with expected crc 2023-10-09T10:48:01.709 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:01 smithi155 ceph-mon[184854]: failed to encode map e180 with expected crc 2023-10-09T10:48:01.710 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:01 smithi155 ceph-mon[184854]: failed to encode map e180 with expected crc 2023-10-09T10:48:01.710 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:01 smithi155 ceph-mon[184854]: failed to encode map e180 with expected crc 2023-10-09T10:48:01.710 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:01 smithi155 ceph-mon[184854]: failed to encode map e180 with expected crc 2023-10-09T10:48:01.742 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:02.033 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:02.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:02.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:02.141 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:02.198 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:02.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:02 smithi155 ceph-mon[184854]: pgmap v385: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 31152/129027 objects degraded (24.144%); 4385/129027 objects misplaced (3.399%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:48:02.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:02 smithi125 ceph-mon[200197]: pgmap v385: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 31152/129027 objects degraded (24.144%); 4385/129027 objects misplaced (3.399%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:48:02.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:02 smithi125 ceph-mon[195507]: pgmap v385: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 31152/129027 objects degraded (24.144%); 4385/129027 objects misplaced (3.399%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:48:02.890 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:48:03.180 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:03.181 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:48:03.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:03.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:03.284 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:03.340 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:03.389 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:03.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:03.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:03 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 31152/129027 objects degraded (24.144%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:03.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:03.497 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:03.500 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:03 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 31152/129027 objects degraded (24.144%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:03.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:03 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 31152/129027 objects degraded (24.144%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:03.589 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:03.618 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:48:03.665 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:03.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:03.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:03.715 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:48:03.715 INFO:teuthology.orchestra.run.smithi125.stdout:{"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 31079/127095 objects degraded (24.453%), 10 pgs degraded, 10 pgs undersized","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":469,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":180,"num_osds":8,"num_up_osds":8,"osd_up_since":1696848214,"num_in_osds":7,"osd_in_since":1696848391,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":106},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":9},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":42365,"data_bytes":2775357368,"bytes_used":9468305408,"bytes_avail":662496944128,"bytes_total":671965249536,"degraded_objects":31079,"degraded_total":127095,"degraded_ratio":0.24453361658601833,"misplaced_objects":4273,"misplaced_total":127095,"misplaced_ratio":0.033620520083402182,"recovering_objects_per_sec":68,"recovering_bytes_per_sec":4473621,"recovering_keys_per_sec":0,"num_objects_recovered":683,"num_bytes_recovered":44761088,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":716},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi125.satgiw","status":"up:active","gid":24575}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"}},"servicemap":{"epoch":20,"modified":"2023-10-09T10:44:35.008237+0000","services":{}},"progress_events":{"4db0080d-306f-4da6-a38c-4c702d14cb34":{"message":"Global Recovery Event (3m)\n [========================....] (remaining: 22s)","progress":0.89075630903244019,"add_to_ceph_s":true},"585851ac-6e70-4462-9176-ac50987fab77":{"message":"Upgrade to 18.2.0-598-gb4fa663f (3m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T10:48:03.769 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:03.814 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:03.940 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:03.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:03.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:04.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:04.104 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:04.137 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:48:04.175 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:48:04.213 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:48:04.258 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:48:04.292 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:48:04.343 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:48:04.361 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:04 smithi155 ceph-mon[184854]: pgmap v386: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 31079/127095 objects degraded (24.453%); 4273/127095 objects misplaced (3.362%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:48:04.361 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:04 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/88376124' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:48:04.361 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:04 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/889481174' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:04.384 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:48:04.405 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:04.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:04.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:04.413 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:48:04.419 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:48:04.419 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:48:04.419 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:48:04.419 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:48:04.420 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:48:04.420 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:48:04.420 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:48:04.420 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:48:04.511 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:04.557 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:04 smithi125 ceph-mon[195507]: pgmap v386: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 31079/127095 objects degraded (24.453%); 4273/127095 objects misplaced (3.362%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:48:04.558 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:04 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/88376124' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:48:04.558 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:04 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/889481174' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:04.558 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:04 smithi125 ceph-mon[200197]: pgmap v386: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 31079/127095 objects degraded (24.453%); 4273/127095 objects misplaced (3.362%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:48:04.558 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:04 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/88376124' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:48:04.558 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:04 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/889481174' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:04.560 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:04.642 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:04.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:04.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:04.746 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:04.820 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:04.859 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:04.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:04.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:04.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:04.990 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:48:04.990 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:48:05.021 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:05.113 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:05.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:05.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:05.226 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:05.292 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:05.395 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:05.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:05.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:05.436 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:05 smithi125 ceph-mon[195507]: pgmap v387: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 31079/122745 objects degraded (25.320%); 4160/122745 objects misplaced (3.389%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:48:05.491 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:05.500 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:05.538 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:48:05.538 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:48:05.573 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:05 smithi155 ceph-mon[184854]: pgmap v387: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 31079/122745 objects degraded (25.320%); 4160/122745 objects misplaced (3.389%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:48:05.586 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:05.702 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:05 smithi125 ceph-mon[200197]: pgmap v387: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 31079/122745 objects degraded (25.320%); 4160/122745 objects misplaced (3.389%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:48:05.887 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:05.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:05.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:05.952 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:48:05.992 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:06.050 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:06.089 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:06.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:06.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:06.194 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:06.262 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:06.347 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:06.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:06.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:06.435 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:06.459 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:06.476 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:48:06.534 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:06.572 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:06.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:06.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:06.599 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:06 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/635156496' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:06.599 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:06 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:06.599 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:06 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/635156496' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:06.599 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:06 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:06.685 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:06.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:06 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/635156496' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:06.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:06 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:06.735 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:06.767 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:06.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:06.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:06.875 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:06.943 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:06.981 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:06.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:06.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:07.087 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:07.130 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:07.146 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:48:07.163 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:07.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:07.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:07.270 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:07.348 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:07.496 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:48:07.496 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:48:07.496 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[195507]: osdmap e181: 8 total, 8 up, 7 in 2023-10-09T10:48:07.496 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[195507]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.496 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[195507]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.496 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[195507]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.496 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[195507]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[195507]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[195507]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[195507]: pgmap v389: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 31079/117504 objects degraded (26.449%); 3730/117504 objects misplaced (3.174%); 4.5 MiB/s, 107 objects/s recovering 2023-10-09T10:48:07.552 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:48:07.554 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:07.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:07.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:07.572 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:48:07.658 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:07.700 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:07.728 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:07.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:48:07.772 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:48:07.772 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[200197]: osdmap e181: 8 total, 8 up, 7 in 2023-10-09T10:48:07.772 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[200197]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.772 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[200197]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.772 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[200197]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.772 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[200197]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.772 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[200197]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.772 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[200197]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.772 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:07 smithi125 ceph-mon[200197]: pgmap v389: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 31079/117504 objects degraded (26.449%); 3730/117504 objects misplaced (3.174%); 4.5 MiB/s, 107 objects/s recovering 2023-10-09T10:48:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:07 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:48:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:07 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:48:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:07 smithi155 ceph-mon[184854]: osdmap e181: 8 total, 8 up, 7 in 2023-10-09T10:48:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:07 smithi155 ceph-mon[184854]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:07 smithi155 ceph-mon[184854]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:07 smithi155 ceph-mon[184854]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:07 smithi155 ceph-mon[184854]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:07 smithi155 ceph-mon[184854]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:07 smithi155 ceph-mon[184854]: failed to encode map e181 with expected crc 2023-10-09T10:48:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:07 smithi155 ceph-mon[184854]: pgmap v389: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 31079/117504 objects degraded (26.449%); 3730/117504 objects misplaced (3.174%); 4.5 MiB/s, 107 objects/s recovering 2023-10-09T10:48:08.013 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:08.015 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":389,"stamp":"2023-10-09T10:48:07.138906+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2565838776,"num_objects":39168,"num_object_clones":0,"num_object_copies":117504,"num_objects_missing_on_primary":1333,"num_objects_missing":1333,"num_objects_degraded":31079,"num_objects_misplaced":3730,"num_objects_unfound":0,"num_objects_dirty":39168,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":165364,"num_write_kb":6260974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24181,"num_bytes_recovered":1473710850,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":171382,"ondisk_log_size":171382,"up":357,"acting":347,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8715092,"kb_used_data":7048540,"kb_used_omap":44,"kb_used_meta":1666003,"kb_avail":647500972,"statfs":{"total":671965249536,"available":663040995328,"internally_reserved":0,"allocated":7217704960,"data_stored":7212228504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1705987489},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,3,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-441253888,"num_objects":-6733,"num_object_clones":0,"num_object_copies":-20199,"num_objects_missing_on_primary":-73,"num_objects_missing":-73,"num_objects_degraded":-73,"num_objects_misplaced":-1224,"num_objects_unfound":0,"num_objects_dirty":-6733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6733,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":864,"num_bytes_recovered":38076416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.013034"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":173,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.258316+0000","last_change":"2023-10-09T10:46:32.250979+0000","last_active":"2023-10-09T10:47:52.258316+0000","last_peered":"2023-10-09T10:47:52.258316+0000","last_clean":"2023-10-09T10:47:52.258316+0000","last_became_active":"2023-10-09T10:46:32.250625+0000","last_became_peered":"2023-10-09T10:46:32.250625+0000","last_unstale":"2023-10-09T10:47:52.258316+0000","last_undegraded":"2023-10-09T10:47:52.258316+0000","last_fullsized":"2023-10-09T10:47:52.258316+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:34.858348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":148,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.215825+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:47:52.215825+0000","last_peered":"2023-10-09T10:47:52.215825+0000","last_clean":"2023-10-09T10:47:52.215825+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:47:52.215825+0000","last_undegraded":"2023-10-09T10:47:52.215825+0000","last_fullsized":"2023-10-09T10:47:52.215825+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":133,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.317968+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:48:00.317968+0000","last_peered":"2023-10-09T10:48:00.317968+0000","last_clean":"2023-10-09T10:48:00.317968+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:48:00.317968+0000","last_undegraded":"2023-10-09T10:48:00.317968+0000","last_fullsized":"2023-10-09T10:48:00.317968+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":49,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.322905+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:48:00.322905+0000","last_peered":"2023-10-09T10:48:00.322905+0000","last_clean":"2023-10-09T10:48:00.322905+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:48:00.322905+0000","last_undegraded":"2023-10-09T10:48:00.322905+0000","last_fullsized":"2023-10-09T10:48:00.322905+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":184,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.215658+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:47:52.215658+0000","last_peered":"2023-10-09T10:47:52.215658+0000","last_clean":"2023-10-09T10:47:52.215658+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:47:52.215658+0000","last_undegraded":"2023-10-09T10:47:52.215658+0000","last_fullsized":"2023-10-09T10:47:52.215658+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":118,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.245004+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:47:52.245004+0000","last_peered":"2023-10-09T10:47:52.245004+0000","last_clean":"2023-10-09T10:47:52.245004+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:47:52.245004+0000","last_undegraded":"2023-10-09T10:47:52.245004+0000","last_fullsized":"2023-10-09T10:47:52.245004+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"180'6658","reported_seq":10812,"reported_epoch":180,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:05.894672+0000","last_change":"2023-10-09T10:46:33.521602+0000","last_active":"2023-10-09T10:48:05.894672+0000","last_peered":"2023-10-09T10:48:05.894672+0000","last_clean":"2023-10-09T10:46:31.055751+0000","last_became_active":"2023-10-09T10:46:33.364058+0000","last_became_peered":"2023-10-09T10:46:33.364058+0000","last_unstale":"2023-10-09T10:48:05.894672+0000","last_undegraded":"2023-10-09T10:48:05.894672+0000","last_fullsized":"2023-10-09T10:48:05.894672+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6658,"log_dups_size":0,"ondisk_log_size":6658,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80969728,"num_objects":1236,"num_object_clones":0,"num_object_copies":3708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1235,"num_objects_unfound":0,"num_objects_dirty":1236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5622,"num_write_kb":219424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2010,"num_bytes_recovered":131727360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":1236}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":48,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.246618+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:47:52.246618+0000","last_peered":"2023-10-09T10:47:52.246618+0000","last_clean":"2023-10-09T10:47:52.246618+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:47:52.246618+0000","last_undegraded":"2023-10-09T10:47:52.246618+0000","last_fullsized":"2023-10-09T10:47:52.246618+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":148,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.215723+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:47:52.215723+0000","last_peered":"2023-10-09T10:47:52.215723+0000","last_clean":"2023-10-09T10:47:52.215723+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:47:52.215723+0000","last_undegraded":"2023-10-09T10:47:52.215723+0000","last_fullsized":"2023-10-09T10:47:52.215723+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":513,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.241998+0000","last_change":"2023-10-09T10:46:05.620365+0000","last_active":"2023-10-09T10:47:52.241998+0000","last_peered":"2023-10-09T10:47:52.241998+0000","last_clean":"2023-10-09T10:47:52.241998+0000","last_became_active":"2023-10-09T10:46:05.620024+0000","last_became_peered":"2023-10-09T10:46:05.620024+0000","last_unstale":"2023-10-09T10:47:52.241998+0000","last_undegraded":"2023-10-09T10:47:52.241998+0000","last_fullsized":"2023-10-09T10:47:52.241998+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.13","version":"180'6441","reported_seq":6627,"reported_epoch":180,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:05.873238+0000","last_change":"2023-10-09T10:46:33.470698+0000","last_active":"2023-10-09T10:48:05.873238+0000","last_peered":"2023-10-09T10:48:05.873238+0000","last_clean":"2023-10-09T10:46:31.195699+0000","last_became_active":"2023-10-09T10:46:33.455632+0000","last_became_peered":"2023-10-09T10:46:33.455632+0000","last_unstale":"2023-10-09T10:48:05.873238+0000","last_undegraded":"2023-10-09T10:46:33.269323+0000","last_fullsized":"2023-10-09T10:46:33.241226+0000","mapping_epoch":154,"log_start":"104'1441","ondisk_log_start":"104'1441","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":5000,"log_dups_size":0,"ondisk_log_size":5000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:07:12.547333+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80674816,"num_objects":1231,"num_object_clones":0,"num_object_copies":3693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3315,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3726,"num_write_kb":105088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":1231},{"shards":"6,7","objects":-1544}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":145,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.244907+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:47:52.244907+0000","last_peered":"2023-10-09T10:47:52.244907+0000","last_clean":"2023-10-09T10:47:52.244907+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:47:52.244907+0000","last_undegraded":"2023-10-09T10:47:52.244907+0000","last_fullsized":"2023-10-09T10:47:52.244907+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":105,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.218700+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:47:52.218700+0000","last_peered":"2023-10-09T10:47:52.218700+0000","last_clean":"2023-10-09T10:47:52.218700+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:47:52.218700+0000","last_undegraded":"2023-10-09T10:47:52.218700+0000","last_fullsized":"2023-10-09T10:47:52.218700+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"181'6590","reported_seq":14718,"reported_epoch":181,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:06.466055+0000","last_change":"2023-10-09T10:46:03.683196+0000","last_active":"2023-10-09T10:48:06.466055+0000","last_peered":"2023-10-09T10:48:06.466055+0000","last_clean":"2023-10-09T10:46:02.177309+0000","last_became_active":"2023-10-09T10:46:03.681058+0000","last_became_peered":"2023-10-09T10:46:03.681058+0000","last_unstale":"2023-10-09T10:48:06.466055+0000","last_undegraded":"2023-10-09T10:48:06.466055+0000","last_fullsized":"2023-10-09T10:48:06.466055+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6590,"log_dups_size":0,"ondisk_log_size":6590,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81494016,"num_objects":1244,"num_object_clones":0,"num_object_copies":3732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1244,"num_objects_unfound":0,"num_objects_dirty":1244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5544,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[1,5,6],"avail_no_missing":["1","2","3","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1244}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":101,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.215430+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:47:52.215430+0000","last_peered":"2023-10-09T10:47:52.215430+0000","last_clean":"2023-10-09T10:47:52.215430+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:47:52.215430+0000","last_undegraded":"2023-10-09T10:47:52.215430+0000","last_fullsized":"2023-10-09T10:47:52.215430+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":142,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.317991+0000","last_change":"2023-10-09T10:46:32.249921+0000","last_active":"2023-10-09T10:48:00.317991+0000","last_peered":"2023-10-09T10:48:00.317991+0000","last_clean":"2023-10-09T10:48:00.317991+0000","last_became_active":"2023-10-09T10:46:32.249590+0000","last_became_peered":"2023-10-09T10:46:32.249590+0000","last_unstale":"2023-10-09T10:48:00.317991+0000","last_undegraded":"2023-10-09T10:48:00.317991+0000","last_fullsized":"2023-10-09T10:48:00.317991+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:25.169752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"180'6388","reported_seq":14611,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:04.545049+0000","last_change":"2023-10-09T10:47:48.412928+0000","last_active":"2023-10-09T10:48:04.545049+0000","last_peered":"2023-10-09T10:48:04.545049+0000","last_clean":"2023-10-09T10:48:04.545049+0000","last_became_active":"2023-10-09T10:47:48.398813+0000","last_became_peered":"2023-10-09T10:47:48.398813+0000","last_unstale":"2023-10-09T10:48:04.545049+0000","last_undegraded":"2023-10-09T10:48:04.545049+0000","last_fullsized":"2023-10-09T10:48:04.545049+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6388,"log_dups_size":0,"ondisk_log_size":6388,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78151680,"num_objects":1193,"num_object_clones":0,"num_object_copies":3579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5381,"num_write_kb":210336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2691,"num_bytes_recovered":176357376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":150,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.252870+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:47:52.252870+0000","last_peered":"2023-10-09T10:47:52.252870+0000","last_clean":"2023-10-09T10:47:52.252870+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:47:52.252870+0000","last_undegraded":"2023-10-09T10:47:52.252870+0000","last_fullsized":"2023-10-09T10:47:52.252870+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":144,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.252290+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:47:52.252290+0000","last_peered":"2023-10-09T10:47:52.252290+0000","last_clean":"2023-10-09T10:47:52.252290+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:47:52.252290+0000","last_undegraded":"2023-10-09T10:47:52.252290+0000","last_fullsized":"2023-10-09T10:47:52.252290+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":513,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.253168+0000","last_change":"2023-10-09T10:46:05.618276+0000","last_active":"2023-10-09T10:47:52.253168+0000","last_peered":"2023-10-09T10:47:52.253168+0000","last_clean":"2023-10-09T10:47:52.253168+0000","last_became_active":"2023-10-09T10:46:05.616509+0000","last_became_peered":"2023-10-09T10:46:05.616509+0000","last_unstale":"2023-10-09T10:47:52.253168+0000","last_undegraded":"2023-10-09T10:47:52.253168+0000","last_fullsized":"2023-10-09T10:47:52.253168+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.16","version":"180'6393","reported_seq":6498,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:04.530643+0000","last_change":"2023-10-09T10:46:03.592682+0000","last_active":"2023-10-09T10:48:04.530643+0000","last_peered":"2023-10-09T10:48:04.530643+0000","last_clean":"2023-10-09T10:48:04.530643+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:48:04.530643+0000","last_undegraded":"2023-10-09T10:48:04.530643+0000","last_fullsized":"2023-10-09T10:48:04.530643+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6393,"log_dups_size":0,"ondisk_log_size":6393,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81002496,"num_objects":1236,"num_object_clones":0,"num_object_copies":3708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5350,"num_write_kb":210752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":162,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.248395+0000","last_change":"2023-10-09T10:46:32.250256+0000","last_active":"2023-10-09T10:47:52.248395+0000","last_peered":"2023-10-09T10:47:52.248395+0000","last_clean":"2023-10-09T10:47:52.248395+0000","last_became_active":"2023-10-09T10:46:32.243312+0000","last_became_peered":"2023-10-09T10:46:32.243312+0000","last_unstale":"2023-10-09T10:47:52.248395+0000","last_undegraded":"2023-10-09T10:47:52.248395+0000","last_fullsized":"2023-10-09T10:47:52.248395+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:08.059777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":171,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.214865+0000","last_change":"2023-10-09T10:46:32.237577+0000","last_active":"2023-10-09T10:47:52.214865+0000","last_peered":"2023-10-09T10:47:52.214865+0000","last_clean":"2023-10-09T10:47:52.214865+0000","last_became_active":"2023-10-09T10:46:32.237316+0000","last_became_peered":"2023-10-09T10:46:32.237316+0000","last_unstale":"2023-10-09T10:47:52.214865+0000","last_undegraded":"2023-10-09T10:47:52.214865+0000","last_fullsized":"2023-10-09T10:47:52.214865+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:39.684602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":514,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.246745+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:47:52.246745+0000","last_peered":"2023-10-09T10:47:52.246745+0000","last_clean":"2023-10-09T10:47:52.246745+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:47:52.246745+0000","last_undegraded":"2023-10-09T10:47:52.246745+0000","last_fullsized":"2023-10-09T10:47:52.246745+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.17","version":"180'6417","reported_seq":6516,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.063522+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:48:06.063522+0000","last_peered":"2023-10-09T10:48:06.063522+0000","last_clean":"2023-10-09T10:48:06.063522+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:48:06.063522+0000","last_undegraded":"2023-10-09T10:48:06.063522+0000","last_fullsized":"2023-10-09T10:48:06.063522+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6417,"log_dups_size":0,"ondisk_log_size":6417,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71892992,"num_objects":1097,"num_object_clones":0,"num_object_copies":3291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5409,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":48,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.244444+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:47:52.244444+0000","last_peered":"2023-10-09T10:47:52.244444+0000","last_clean":"2023-10-09T10:47:52.244444+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:47:52.244444+0000","last_undegraded":"2023-10-09T10:47:52.244444+0000","last_fullsized":"2023-10-09T10:47:52.244444+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":144,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.249861+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:47:52.249861+0000","last_peered":"2023-10-09T10:47:52.249861+0000","last_clean":"2023-10-09T10:47:52.249861+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:47:52.249861+0000","last_undegraded":"2023-10-09T10:47:52.249861+0000","last_fullsized":"2023-10-09T10:47:52.249861+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":514,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.241485+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:47:52.241485+0000","last_peered":"2023-10-09T10:47:52.241485+0000","last_clean":"2023-10-09T10:47:52.241485+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:47:52.241485+0000","last_undegraded":"2023-10-09T10:47:52.241485+0000","last_fullsized":"2023-10-09T10:47:52.241485+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"181'6528","reported_seq":6633,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.466878+0000","last_change":"2023-10-09T10:46:33.243540+0000","last_active":"2023-10-09T10:48:06.466878+0000","last_peered":"2023-10-09T10:48:06.466878+0000","last_clean":"2023-10-09T10:48:06.466878+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:48:06.466878+0000","last_undegraded":"2023-10-09T10:48:06.466878+0000","last_fullsized":"2023-10-09T10:48:06.466878+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6528,"log_dups_size":0,"ondisk_log_size":6528,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78610432,"num_objects":1200,"num_object_clones":0,"num_object_copies":3600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5474,"num_write_kb":213536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":90,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.252130+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:47:52.252130+0000","last_peered":"2023-10-09T10:47:52.252130+0000","last_clean":"2023-10-09T10:47:52.252130+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:47:52.252130+0000","last_undegraded":"2023-10-09T10:47:52.252130+0000","last_fullsized":"2023-10-09T10:47:52.252130+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":48,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.242860+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:47:52.242860+0000","last_peered":"2023-10-09T10:47:52.242860+0000","last_clean":"2023-10-09T10:47:52.242860+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:47:52.242860+0000","last_undegraded":"2023-10-09T10:47:52.242860+0000","last_fullsized":"2023-10-09T10:47:52.242860+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":514,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.244184+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:47:52.244184+0000","last_peered":"2023-10-09T10:47:52.244184+0000","last_clean":"2023-10-09T10:47:52.244184+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:47:52.244184+0000","last_undegraded":"2023-10-09T10:47:52.244184+0000","last_fullsized":"2023-10-09T10:47:52.244184+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"180'6540","reported_seq":6654,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:04.393351+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:48:04.393351+0000","last_peered":"2023-10-09T10:48:04.393351+0000","last_clean":"2023-10-09T10:48:04.393351+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:48:04.393351+0000","last_undegraded":"2023-10-09T10:48:04.393351+0000","last_fullsized":"2023-10-09T10:48:04.393351+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6540,"log_dups_size":0,"ondisk_log_size":6540,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77922304,"num_objects":1189,"num_object_clones":0,"num_object_copies":3567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5479,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":58,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.246702+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:47:52.246702+0000","last_peered":"2023-10-09T10:47:52.246702+0000","last_clean":"2023-10-09T10:47:52.246702+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:47:52.246702+0000","last_undegraded":"2023-10-09T10:47:52.246702+0000","last_fullsized":"2023-10-09T10:47:52.246702+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":148,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.213366+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:47:52.213366+0000","last_peered":"2023-10-09T10:47:52.213366+0000","last_clean":"2023-10-09T10:47:52.213366+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:47:52.213366+0000","last_undegraded":"2023-10-09T10:47:52.213366+0000","last_fullsized":"2023-10-09T10:47:52.213366+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":514,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.242431+0000","last_change":"2023-10-09T10:46:05.620574+0000","last_active":"2023-10-09T10:47:52.242431+0000","last_peered":"2023-10-09T10:47:52.242431+0000","last_clean":"2023-10-09T10:47:52.242431+0000","last_became_active":"2023-10-09T10:46:05.620212+0000","last_became_peered":"2023-10-09T10:46:05.620212+0000","last_unstale":"2023-10-09T10:47:52.242431+0000","last_undegraded":"2023-10-09T10:47:52.242431+0000","last_fullsized":"2023-10-09T10:47:52.242431+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"180'11264","reported_seq":15419,"reported_epoch":180,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:06.036675+0000","last_change":"2023-10-09T10:46:32.247903+0000","last_active":"2023-10-09T10:48:06.036675+0000","last_peered":"2023-10-09T10:48:06.036675+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:46:02.739355+0000","last_became_peered":"2023-10-09T10:46:02.739355+0000","last_unstale":"2023-10-09T10:48:06.036675+0000","last_undegraded":"2023-10-09T10:46:02.608805+0000","last_fullsized":"2023-10-09T10:46:02.583589+0000","mapping_epoch":142,"log_start":"108'2400","ondisk_log_start":"108'2400","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8864,"log_dups_size":0,"ondisk_log_size":8864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:32.899634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155189248,"num_objects":2368,"num_object_clones":0,"num_object_copies":7104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11264,"num_write_kb":436224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2022,"num_bytes_recovered":132513792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":625},{"shards":"5,6","objects":1742}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":173,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.246697+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:47:52.246697+0000","last_peered":"2023-10-09T10:47:52.246697+0000","last_clean":"2023-10-09T10:47:52.246697+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:47:52.246697+0000","last_undegraded":"2023-10-09T10:47:52.246697+0000","last_fullsized":"2023-10-09T10:47:52.246697+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":127,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.245308+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:47:52.245308+0000","last_peered":"2023-10-09T10:47:52.245308+0000","last_clean":"2023-10-09T10:47:52.245308+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:47:52.245308+0000","last_undegraded":"2023-10-09T10:47:52.245308+0000","last_fullsized":"2023-10-09T10:47:52.245308+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":494,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.242126+0000","last_change":"2023-10-09T10:46:02.631378+0000","last_active":"2023-10-09T10:47:52.242126+0000","last_peered":"2023-10-09T10:47:52.242126+0000","last_clean":"2023-10-09T10:47:52.242126+0000","last_became_active":"2023-10-09T10:46:02.629142+0000","last_became_peered":"2023-10-09T10:46:02.629142+0000","last_unstale":"2023-10-09T10:47:52.242126+0000","last_undegraded":"2023-10-09T10:47:52.242126+0000","last_fullsized":"2023-10-09T10:47:52.242126+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.b","version":"180'10723","reported_seq":10815,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:04.564182+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:48:04.564182+0000","last_peered":"2023-10-09T10:48:04.564182+0000","last_clean":"2023-10-09T10:48:04.564182+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:48:04.564182+0000","last_undegraded":"2023-10-09T10:48:04.564182+0000","last_fullsized":"2023-10-09T10:48:04.564182+0000","mapping_epoch":98,"log_start":"129'5025","ondisk_log_start":"129'5025","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5698,"log_dups_size":0,"ondisk_log_size":5698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161021952,"num_objects":2457,"num_object_clones":0,"num_object_copies":7371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10723,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":146,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.247132+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:47:52.247132+0000","last_peered":"2023-10-09T10:47:52.247132+0000","last_clean":"2023-10-09T10:47:52.247132+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:47:52.247132+0000","last_undegraded":"2023-10-09T10:47:52.247132+0000","last_fullsized":"2023-10-09T10:47:52.247132+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":181,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.252059+0000","last_change":"2023-10-09T10:46:32.251035+0000","last_active":"2023-10-09T10:47:52.252059+0000","last_peered":"2023-10-09T10:47:52.252059+0000","last_clean":"2023-10-09T10:47:52.252059+0000","last_became_active":"2023-10-09T10:46:32.250293+0000","last_became_peered":"2023-10-09T10:46:32.250293+0000","last_unstale":"2023-10-09T10:47:52.252059+0000","last_undegraded":"2023-10-09T10:47:52.252059+0000","last_fullsized":"2023-10-09T10:47:52.252059+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.787766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":514,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.246403+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:47:52.246403+0000","last_peered":"2023-10-09T10:47:52.246403+0000","last_clean":"2023-10-09T10:47:52.246403+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:47:52.246403+0000","last_undegraded":"2023-10-09T10:47:52.246403+0000","last_fullsized":"2023-10-09T10:47:52.246403+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"180'6515","reported_seq":6627,"reported_epoch":180,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:04.318099+0000","last_change":"2023-10-09T10:46:33.540719+0000","last_active":"2023-10-09T10:48:04.318099+0000","last_peered":"2023-10-09T10:48:04.318099+0000","last_clean":"2023-10-09T10:46:31.137696+0000","last_became_active":"2023-10-09T10:46:33.527650+0000","last_became_peered":"2023-10-09T10:46:33.527650+0000","last_unstale":"2023-10-09T10:48:04.318099+0000","last_undegraded":"2023-10-09T10:46:33.285468+0000","last_fullsized":"2023-10-09T10:46:33.235683+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6515,"log_dups_size":0,"ondisk_log_size":6515,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81035264,"num_objects":1236,"num_object_clones":0,"num_object_copies":3708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5450,"num_write_kb":213984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1224},{"shards":"3,7","objects":-1573}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":101,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.249788+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:47:52.249788+0000","last_peered":"2023-10-09T10:47:52.249788+0000","last_clean":"2023-10-09T10:47:52.249788+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:47:52.249788+0000","last_undegraded":"2023-10-09T10:47:52.249788+0000","last_fullsized":"2023-10-09T10:47:52.249788+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":119,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.318259+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:48:00.318259+0000","last_peered":"2023-10-09T10:48:00.318259+0000","last_clean":"2023-10-09T10:48:00.318259+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:48:00.318259+0000","last_undegraded":"2023-10-09T10:48:00.318259+0000","last_fullsized":"2023-10-09T10:48:00.318259+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":514,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.242518+0000","last_change":"2023-10-09T10:46:05.620030+0000","last_active":"2023-10-09T10:47:52.242518+0000","last_peered":"2023-10-09T10:47:52.242518+0000","last_clean":"2023-10-09T10:47:52.242518+0000","last_became_active":"2023-10-09T10:46:05.619694+0000","last_became_peered":"2023-10-09T10:46:05.619694+0000","last_unstale":"2023-10-09T10:47:52.242518+0000","last_undegraded":"2023-10-09T10:47:52.242518+0000","last_fullsized":"2023-10-09T10:47:52.242518+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.9","version":"180'6413","reported_seq":6536,"reported_epoch":180,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:02.627940+0000","last_change":"2023-10-09T10:46:34.405499+0000","last_active":"2023-10-09T10:48:02.627940+0000","last_peered":"2023-10-09T10:48:02.627940+0000","last_clean":"2023-10-09T10:46:31.190603+0000","last_became_active":"2023-10-09T10:46:34.396448+0000","last_became_peered":"2023-10-09T10:46:34.396448+0000","last_unstale":"2023-10-09T10:48:02.627940+0000","last_undegraded":"2023-10-09T10:46:33.246949+0000","last_fullsized":"2023-10-09T10:46:33.237968+0000","mapping_epoch":154,"log_start":"103'368","ondisk_log_start":"103'368","created":98,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6045,"log_dups_size":0,"ondisk_log_size":6045,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81461248,"num_objects":1243,"num_object_clones":0,"num_object_copies":3729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2004,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5371,"num_write_kb":211648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1303,"num_bytes_recovered":85393408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":1},{"shards":"0,7","objects":-225}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":145,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.251049+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:47:52.251049+0000","last_peered":"2023-10-09T10:47:52.251049+0000","last_clean":"2023-10-09T10:47:52.251049+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:47:52.251049+0000","last_undegraded":"2023-10-09T10:47:52.251049+0000","last_fullsized":"2023-10-09T10:47:52.251049+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":48,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.241704+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:47:52.241704+0000","last_peered":"2023-10-09T10:47:52.241704+0000","last_clean":"2023-10-09T10:47:52.241704+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:47:52.241704+0000","last_undegraded":"2023-10-09T10:47:52.241704+0000","last_fullsized":"2023-10-09T10:47:52.241704+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":505,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.252738+0000","last_change":"2023-10-09T10:46:05.618134+0000","last_active":"2023-10-09T10:47:52.252738+0000","last_peered":"2023-10-09T10:47:52.252738+0000","last_clean":"2023-10-09T10:47:52.252738+0000","last_became_active":"2023-10-09T10:46:05.616057+0000","last_became_peered":"2023-10-09T10:46:05.616057+0000","last_unstale":"2023-10-09T10:47:52.252738+0000","last_undegraded":"2023-10-09T10:47:52.252738+0000","last_fullsized":"2023-10-09T10:47:52.252738+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"180'10765","reported_seq":10865,"reported_epoch":180,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:05.990327+0000","last_change":"2023-10-09T10:46:33.576530+0000","last_active":"2023-10-09T10:48:05.990327+0000","last_peered":"2023-10-09T10:48:05.990327+0000","last_clean":"2023-10-09T10:46:31.183481+0000","last_became_active":"2023-10-09T10:46:33.542304+0000","last_became_peered":"2023-10-09T10:46:33.542304+0000","last_unstale":"2023-10-09T10:48:05.990327+0000","last_undegraded":"2023-10-09T10:46:33.300581+0000","last_fullsized":"2023-10-09T10:46:33.246047+0000","mapping_epoch":154,"log_start":"115'3377","ondisk_log_start":"115'3377","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7388,"log_dups_size":0,"ondisk_log_size":7388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:26.739488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161808384,"num_objects":2469,"num_object_clones":0,"num_object_copies":7407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6617,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10765,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":2469},{"shards":"1,4","objects":-3092}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":173,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:59.592616+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:47:59.592616+0000","last_peered":"2023-10-09T10:47:59.592616+0000","last_clean":"2023-10-09T10:47:59.592616+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:47:59.592616+0000","last_undegraded":"2023-10-09T10:47:59.592616+0000","last_fullsized":"2023-10-09T10:47:59.592616+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":144,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.252794+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:47:52.252794+0000","last_peered":"2023-10-09T10:47:52.252794+0000","last_clean":"2023-10-09T10:47:52.252794+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:47:52.252794+0000","last_undegraded":"2023-10-09T10:47:52.252794+0000","last_fullsized":"2023-10-09T10:47:52.252794+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":494,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.246749+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:47:52.246749+0000","last_peered":"2023-10-09T10:47:52.246749+0000","last_clean":"2023-10-09T10:47:52.246749+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:47:52.246749+0000","last_undegraded":"2023-10-09T10:47:52.246749+0000","last_fullsized":"2023-10-09T10:47:52.246749+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.f","version":"180'10593","reported_seq":10691,"reported_epoch":180,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:04.539525+0000","last_change":"2023-10-09T10:46:33.651756+0000","last_active":"2023-10-09T10:48:04.539525+0000","last_peered":"2023-10-09T10:48:04.539525+0000","last_clean":"2023-10-09T10:46:31.156458+0000","last_became_active":"2023-10-09T10:46:33.621003+0000","last_became_peered":"2023-10-09T10:46:33.621003+0000","last_unstale":"2023-10-09T10:48:04.539525+0000","last_undegraded":"2023-10-09T10:46:33.309813+0000","last_fullsized":"2023-10-09T10:46:33.236508+0000","mapping_epoch":154,"log_start":"129'4910","ondisk_log_start":"129'4910","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5683,"log_dups_size":0,"ondisk_log_size":5683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:13.704346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159186944,"num_objects":2429,"num_object_clones":0,"num_object_copies":7287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10593,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":2429},{"shards":"3,4","objects":-3023}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":119,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.258457+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:47:52.258457+0000","last_peered":"2023-10-09T10:47:52.258457+0000","last_clean":"2023-10-09T10:47:52.258457+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:47:52.258457+0000","last_undegraded":"2023-10-09T10:47:52.258457+0000","last_fullsized":"2023-10-09T10:47:52.258457+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":171,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.253132+0000","last_change":"2023-10-09T10:46:32.245583+0000","last_active":"2023-10-09T10:47:52.253132+0000","last_peered":"2023-10-09T10:47:52.253132+0000","last_clean":"2023-10-09T10:47:52.253132+0000","last_became_active":"2023-10-09T10:46:32.245271+0000","last_became_peered":"2023-10-09T10:46:32.245271+0000","last_unstale":"2023-10-09T10:47:52.253132+0000","last_undegraded":"2023-10-09T10:47:52.253132+0000","last_fullsized":"2023-10-09T10:47:52.253132+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.025051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":514,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.241579+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:47:52.241579+0000","last_peered":"2023-10-09T10:47:52.241579+0000","last_clean":"2023-10-09T10:47:52.241579+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:47:52.241579+0000","last_undegraded":"2023-10-09T10:47:52.241579+0000","last_fullsized":"2023-10-09T10:47:52.241579+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"180'10860","reported_seq":10952,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:04.503831+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:48:04.503831+0000","last_peered":"2023-10-09T10:48:04.503831+0000","last_clean":"2023-10-09T10:48:04.503831+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:48:04.503831+0000","last_undegraded":"2023-10-09T10:48:04.503831+0000","last_fullsized":"2023-10-09T10:48:04.503831+0000","mapping_epoch":98,"log_start":"129'5178","ondisk_log_start":"129'5178","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5682,"log_dups_size":0,"ondisk_log_size":5682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158859264,"num_objects":2424,"num_object_clones":0,"num_object_copies":7272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10860,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":155,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.251994+0000","last_change":"2023-10-09T10:46:32.245685+0000","last_active":"2023-10-09T10:47:52.251994+0000","last_peered":"2023-10-09T10:47:52.251994+0000","last_clean":"2023-10-09T10:47:52.251994+0000","last_became_active":"2023-10-09T10:46:32.245314+0000","last_became_peered":"2023-10-09T10:46:32.245314+0000","last_unstale":"2023-10-09T10:47:52.251994+0000","last_undegraded":"2023-10-09T10:47:52.251994+0000","last_fullsized":"2023-10-09T10:47:52.251994+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:51.716860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":144,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.251811+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:47:52.251811+0000","last_peered":"2023-10-09T10:47:52.251811+0000","last_clean":"2023-10-09T10:47:52.251811+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:47:52.251811+0000","last_undegraded":"2023-10-09T10:47:52.251811+0000","last_fullsized":"2023-10-09T10:47:52.251811+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":505,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.244276+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:47:52.244276+0000","last_peered":"2023-10-09T10:47:52.244276+0000","last_clean":"2023-10-09T10:47:52.244276+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:47:52.244276+0000","last_undegraded":"2023-10-09T10:47:52.244276+0000","last_fullsized":"2023-10-09T10:47:52.244276+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"180'10968","reported_seq":11078,"reported_epoch":180,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:06.093732+0000","last_change":"2023-10-09T10:46:02.754507+0000","last_active":"2023-10-09T10:48:06.093732+0000","last_peered":"2023-10-09T10:48:06.093732+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:46:02.747510+0000","last_became_peered":"2023-10-09T10:46:02.747510+0000","last_unstale":"2023-10-09T10:48:06.093732+0000","last_undegraded":"2023-10-09T10:46:02.605987+0000","last_fullsized":"2023-10-09T10:46:02.585248+0000","mapping_epoch":142,"log_start":"106'2100","ondisk_log_start":"106'2100","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8868,"log_dups_size":0,"ondisk_log_size":8868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:59.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159055896,"num_objects":2428,"num_object_clones":0,"num_object_copies":7284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2428,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10968,"num_write_kb":428609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,2,5","objects":2427},{"shards":"1,5","objects":-290}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":150,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:54.579638+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:47:54.579638+0000","last_peered":"2023-10-09T10:47:54.579638+0000","last_clean":"2023-10-09T10:47:54.579638+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:47:54.579638+0000","last_undegraded":"2023-10-09T10:47:54.579638+0000","last_fullsized":"2023-10-09T10:47:54.579638+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":133,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.325510+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:48:00.325510+0000","last_peered":"2023-10-09T10:48:00.325510+0000","last_clean":"2023-10-09T10:48:00.325510+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:48:00.325510+0000","last_undegraded":"2023-10-09T10:48:00.325510+0000","last_fullsized":"2023-10-09T10:48:00.325510+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":514,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.246496+0000","last_change":"2023-10-09T10:46:05.617983+0000","last_active":"2023-10-09T10:47:52.246496+0000","last_peered":"2023-10-09T10:47:52.246496+0000","last_clean":"2023-10-09T10:47:52.246496+0000","last_became_active":"2023-10-09T10:46:05.616021+0000","last_became_peered":"2023-10-09T10:46:05.616021+0000","last_unstale":"2023-10-09T10:47:52.246496+0000","last_undegraded":"2023-10-09T10:47:52.246496+0000","last_fullsized":"2023-10-09T10:47:52.246496+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":176,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.324395+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:48:00.324395+0000","last_peered":"2023-10-09T10:48:00.324395+0000","last_clean":"2023-10-09T10:48:00.324395+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:48:00.324395+0000","last_undegraded":"2023-10-09T10:48:00.324395+0000","last_fullsized":"2023-10-09T10:48:00.324395+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":172,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.245572+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:47:52.245572+0000","last_peered":"2023-10-09T10:47:52.245572+0000","last_clean":"2023-10-09T10:47:52.245572+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:47:52.245572+0000","last_undegraded":"2023-10-09T10:47:52.245572+0000","last_fullsized":"2023-10-09T10:47:52.245572+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":118,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.318073+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:48:00.318073+0000","last_peered":"2023-10-09T10:48:00.318073+0000","last_clean":"2023-10-09T10:48:00.318073+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:48:00.318073+0000","last_undegraded":"2023-10-09T10:48:00.318073+0000","last_fullsized":"2023-10-09T10:48:00.318073+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"180'6440","reported_seq":10477,"reported_epoch":180,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:04.679625+0000","last_change":"2023-10-09T10:46:32.248218+0000","last_active":"2023-10-09T10:48:04.679625+0000","last_peered":"2023-10-09T10:48:04.679625+0000","last_clean":"2023-10-09T10:46:02.181912+0000","last_became_active":"2023-10-09T10:46:32.226237+0000","last_became_peered":"2023-10-09T10:46:32.226237+0000","last_unstale":"2023-10-09T10:48:04.679625+0000","last_undegraded":"2023-10-09T10:48:04.679625+0000","last_fullsized":"2023-10-09T10:48:04.679625+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6440,"log_dups_size":0,"ondisk_log_size":6440,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82018304,"num_objects":1251,"num_object_clones":0,"num_object_copies":3753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1251,"num_objects_unfound":0,"num_objects_dirty":1251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5393,"num_write_kb":212640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1251}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":513,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.241563+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:47:52.241563+0000","last_peered":"2023-10-09T10:47:52.241563+0000","last_clean":"2023-10-09T10:47:52.241563+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:47:52.241563+0000","last_undegraded":"2023-10-09T10:47:52.241563+0000","last_fullsized":"2023-10-09T10:47:52.241563+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"180'6525","reported_seq":6707,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:04.414202+0000","last_change":"2023-10-09T10:46:02.760963+0000","last_active":"2023-10-09T10:48:04.414202+0000","last_peered":"2023-10-09T10:48:04.414202+0000","last_clean":"2023-10-09T10:48:04.414202+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:48:04.414202+0000","last_undegraded":"2023-10-09T10:48:04.414202+0000","last_fullsized":"2023-10-09T10:48:04.414202+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"141'4416","last_scrub_stamp":"2023-10-09T10:46:02.760810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:46:02.760810+0000","objects_scrubbed":3378,"log_size":6525,"log_dups_size":0,"ondisk_log_size":6525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:35.863002+0000","scrub_duration":0.47769323899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83165184,"num_objects":1269,"num_object_clones":0,"num_object_copies":3807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2109,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":149,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.250524+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:47:52.250524+0000","last_peered":"2023-10-09T10:47:52.250524+0000","last_clean":"2023-10-09T10:47:52.250524+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:47:52.250524+0000","last_undegraded":"2023-10-09T10:47:52.250524+0000","last_fullsized":"2023-10-09T10:47:52.250524+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":165,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.252424+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:47:52.252424+0000","last_peered":"2023-10-09T10:47:52.252424+0000","last_clean":"2023-10-09T10:47:52.252424+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:47:52.252424+0000","last_undegraded":"2023-10-09T10:47:52.252424+0000","last_fullsized":"2023-10-09T10:47:52.252424+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":157,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.324704+0000","last_change":"2023-10-09T10:46:32.245608+0000","last_active":"2023-10-09T10:48:00.324704+0000","last_peered":"2023-10-09T10:48:00.324704+0000","last_clean":"2023-10-09T10:48:00.324704+0000","last_became_active":"2023-10-09T10:46:32.245290+0000","last_became_peered":"2023-10-09T10:46:32.245290+0000","last_unstale":"2023-10-09T10:48:00.324704+0000","last_undegraded":"2023-10-09T10:48:00.324704+0000","last_fullsized":"2023-10-09T10:48:00.324704+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:48.832515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":514,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.244358+0000","last_change":"2023-10-09T10:46:05.620057+0000","last_active":"2023-10-09T10:47:52.244358+0000","last_peered":"2023-10-09T10:47:52.244358+0000","last_clean":"2023-10-09T10:47:52.244358+0000","last_became_active":"2023-10-09T10:46:05.619703+0000","last_became_peered":"2023-10-09T10:46:05.619703+0000","last_unstale":"2023-10-09T10:47:52.244358+0000","last_undegraded":"2023-10-09T10:47:52.244358+0000","last_fullsized":"2023-10-09T10:47:52.244358+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"180'6481","reported_seq":6597,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:04.485475+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:48:04.485475+0000","last_peered":"2023-10-09T10:48:04.485475+0000","last_clean":"2023-10-09T10:48:04.485475+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:48:04.485475+0000","last_undegraded":"2023-10-09T10:48:04.485475+0000","last_fullsized":"2023-10-09T10:48:04.485475+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6481,"log_dups_size":0,"ondisk_log_size":6481,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82575360,"num_objects":1260,"num_object_clones":0,"num_object_copies":3780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5420,"num_write_kb":213760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":177,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.218825+0000","last_change":"2023-10-09T10:46:32.241041+0000","last_active":"2023-10-09T10:47:52.218825+0000","last_peered":"2023-10-09T10:47:52.218825+0000","last_clean":"2023-10-09T10:47:52.218825+0000","last_became_active":"2023-10-09T10:46:32.238683+0000","last_became_peered":"2023-10-09T10:46:32.238683+0000","last_unstale":"2023-10-09T10:47:52.218825+0000","last_undegraded":"2023-10-09T10:47:52.218825+0000","last_fullsized":"2023-10-09T10:47:52.218825+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:25.817713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":142,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.317410+0000","last_change":"2023-10-09T10:46:32.250104+0000","last_active":"2023-10-09T10:48:00.317410+0000","last_peered":"2023-10-09T10:48:00.317410+0000","last_clean":"2023-10-09T10:48:00.317410+0000","last_became_active":"2023-10-09T10:46:32.249805+0000","last_became_peered":"2023-10-09T10:46:32.249805+0000","last_unstale":"2023-10-09T10:48:00.317410+0000","last_undegraded":"2023-10-09T10:48:00.317410+0000","last_fullsized":"2023-10-09T10:48:00.317410+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:07.230672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":171,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.325574+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:48:00.325574+0000","last_peered":"2023-10-09T10:48:00.325574+0000","last_clean":"2023-10-09T10:48:00.325574+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:48:00.325574+0000","last_undegraded":"2023-10-09T10:48:00.325574+0000","last_fullsized":"2023-10-09T10:48:00.325574+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"180'6403","reported_seq":6507,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.063390+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:48:06.063390+0000","last_peered":"2023-10-09T10:48:06.063390+0000","last_clean":"2023-10-09T10:48:06.063390+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:48:06.063390+0000","last_undegraded":"2023-10-09T10:48:06.063390+0000","last_fullsized":"2023-10-09T10:48:06.063390+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6403,"log_dups_size":0,"ondisk_log_size":6403,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82247680,"num_objects":1255,"num_object_clones":0,"num_object_copies":3765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5395,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":159,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.244256+0000","last_change":"2023-10-09T10:46:32.235346+0000","last_active":"2023-10-09T10:47:52.244256+0000","last_peered":"2023-10-09T10:47:52.244256+0000","last_clean":"2023-10-09T10:47:52.244256+0000","last_became_active":"2023-10-09T10:46:32.234906+0000","last_became_peered":"2023-10-09T10:46:32.234906+0000","last_unstale":"2023-10-09T10:47:52.244256+0000","last_undegraded":"2023-10-09T10:47:52.244256+0000","last_fullsized":"2023-10-09T10:47:52.244256+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.199251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":515,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.320226+0000","last_change":"2023-10-09T10:46:05.617146+0000","last_active":"2023-10-09T10:48:00.320226+0000","last_peered":"2023-10-09T10:48:00.320226+0000","last_clean":"2023-10-09T10:48:00.320226+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:48:00.320226+0000","last_undegraded":"2023-10-09T10:48:00.320226+0000","last_fullsized":"2023-10-09T10:48:00.320226+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":108,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.215226+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:47:52.215226+0000","last_peered":"2023-10-09T10:47:52.215226+0000","last_clean":"2023-10-09T10:47:52.215226+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:47:52.215226+0000","last_undegraded":"2023-10-09T10:47:52.215226+0000","last_fullsized":"2023-10-09T10:47:52.215226+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":77,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.252956+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:47:52.252956+0000","last_peered":"2023-10-09T10:47:52.252956+0000","last_clean":"2023-10-09T10:47:52.252956+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:47:52.252956+0000","last_undegraded":"2023-10-09T10:47:52.252956+0000","last_fullsized":"2023-10-09T10:47:52.252956+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":514,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.241534+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:47:52.241534+0000","last_peered":"2023-10-09T10:47:52.241534+0000","last_clean":"2023-10-09T10:47:52.241534+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:47:52.241534+0000","last_undegraded":"2023-10-09T10:47:52.241534+0000","last_fullsized":"2023-10-09T10:47:52.241534+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":172,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.317535+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:48:00.317535+0000","last_peered":"2023-10-09T10:48:00.317535+0000","last_clean":"2023-10-09T10:48:00.317535+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:48:00.317535+0000","last_undegraded":"2023-10-09T10:48:00.317535+0000","last_fullsized":"2023-10-09T10:48:00.317535+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":57,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.241481+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:47:52.241481+0000","last_peered":"2023-10-09T10:47:52.241481+0000","last_clean":"2023-10-09T10:47:52.241481+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:47:52.241481+0000","last_undegraded":"2023-10-09T10:47:52.241481+0000","last_fullsized":"2023-10-09T10:47:52.241481+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":57,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.242796+0000","last_change":"2023-10-09T10:46:32.236202+0000","last_active":"2023-10-09T10:47:52.242796+0000","last_peered":"2023-10-09T10:47:52.242796+0000","last_clean":"2023-10-09T10:47:52.242796+0000","last_became_active":"2023-10-09T10:46:32.235901+0000","last_became_peered":"2023-10-09T10:46:32.235901+0000","last_unstale":"2023-10-09T10:47:52.242796+0000","last_undegraded":"2023-10-09T10:47:52.242796+0000","last_fullsized":"2023-10-09T10:47:52.242796+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:48.260403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"180'6535","reported_seq":6687,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.426797+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:48:06.426797+0000","last_peered":"2023-10-09T10:48:06.426797+0000","last_clean":"2023-10-09T10:48:06.426797+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:48:06.426797+0000","last_undegraded":"2023-10-09T10:48:06.426797+0000","last_fullsized":"2023-10-09T10:48:06.426797+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":6535,"log_dups_size":0,"ondisk_log_size":6535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81461248,"num_objects":1243,"num_object_clones":0,"num_object_copies":3729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3505,"num_write_kb":89664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":514,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.242590+0000","last_change":"2023-10-09T10:46:05.620764+0000","last_active":"2023-10-09T10:47:52.242590+0000","last_peered":"2023-10-09T10:47:52.242590+0000","last_clean":"2023-10-09T10:47:52.242590+0000","last_became_active":"2023-10-09T10:46:05.619870+0000","last_became_peered":"2023-10-09T10:46:05.619870+0000","last_unstale":"2023-10-09T10:47:52.242590+0000","last_undegraded":"2023-10-09T10:47:52.242590+0000","last_fullsized":"2023-10-09T10:47:52.242590+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":168,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.248284+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:47:52.248284+0000","last_peered":"2023-10-09T10:47:52.248284+0000","last_clean":"2023-10-09T10:47:52.248284+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:47:52.248284+0000","last_undegraded":"2023-10-09T10:47:52.248284+0000","last_fullsized":"2023-10-09T10:47:52.248284+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":142,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.324782+0000","last_change":"2023-10-09T10:46:32.251450+0000","last_active":"2023-10-09T10:48:00.324782+0000","last_peered":"2023-10-09T10:48:00.324782+0000","last_clean":"2023-10-09T10:48:00.324782+0000","last_became_active":"2023-10-09T10:46:32.250933+0000","last_became_peered":"2023-10-09T10:46:32.250933+0000","last_unstale":"2023-10-09T10:48:00.324782+0000","last_undegraded":"2023-10-09T10:48:00.324782+0000","last_fullsized":"2023-10-09T10:48:00.324782+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:04.116178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":48,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.241621+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:47:52.241621+0000","last_peered":"2023-10-09T10:47:52.241621+0000","last_clean":"2023-10-09T10:47:52.241621+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:47:52.241621+0000","last_undegraded":"2023-10-09T10:47:52.241621+0000","last_fullsized":"2023-10-09T10:47:52.241621+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"181'6374","reported_seq":13071,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.483532+0000","last_change":"2023-10-09T10:47:02.269289+0000","last_active":"2023-10-09T10:48:06.483532+0000","last_peered":"2023-10-09T10:48:06.483532+0000","last_clean":"2023-10-09T10:48:06.483532+0000","last_became_active":"2023-10-09T10:47:02.257854+0000","last_became_peered":"2023-10-09T10:47:02.257854+0000","last_unstale":"2023-10-09T10:48:06.483532+0000","last_undegraded":"2023-10-09T10:48:06.483532+0000","last_fullsized":"2023-10-09T10:48:06.483532+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6374,"log_dups_size":0,"ondisk_log_size":6374,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81035264,"num_objects":1236,"num_object_clones":0,"num_object_copies":3708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5366,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3287,"num_bytes_recovered":181927936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":514,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.242279+0000","last_change":"2023-10-09T10:46:05.618443+0000","last_active":"2023-10-09T10:47:52.242279+0000","last_peered":"2023-10-09T10:47:52.242279+0000","last_clean":"2023-10-09T10:47:52.242279+0000","last_became_active":"2023-10-09T10:46:05.616588+0000","last_became_peered":"2023-10-09T10:46:05.616588+0000","last_unstale":"2023-10-09T10:47:52.242279+0000","last_undegraded":"2023-10-09T10:47:52.242279+0000","last_fullsized":"2023-10-09T10:47:52.242279+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":173,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:54.582735+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:47:54.582735+0000","last_peered":"2023-10-09T10:47:54.582735+0000","last_clean":"2023-10-09T10:47:54.582735+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:47:54.582735+0000","last_undegraded":"2023-10-09T10:47:54.582735+0000","last_fullsized":"2023-10-09T10:47:54.582735+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":118,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.325792+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:48:00.325792+0000","last_peered":"2023-10-09T10:48:00.325792+0000","last_clean":"2023-10-09T10:48:00.325792+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:48:00.325792+0000","last_undegraded":"2023-10-09T10:48:00.325792+0000","last_fullsized":"2023-10-09T10:48:00.325792+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":162,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.215585+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:47:52.215585+0000","last_peered":"2023-10-09T10:47:52.215585+0000","last_clean":"2023-10-09T10:47:52.215585+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:47:52.215585+0000","last_undegraded":"2023-10-09T10:47:52.215585+0000","last_fullsized":"2023-10-09T10:47:52.215585+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"180'6510","reported_seq":11181,"reported_epoch":180,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:06.076800+0000","last_change":"2023-10-09T10:47:29.622439+0000","last_active":"2023-10-09T10:48:06.076800+0000","last_peered":"2023-10-09T10:48:06.076800+0000","last_clean":"2023-10-09T10:46:31.130083+0000","last_became_active":"2023-10-09T10:46:33.493288+0000","last_became_peered":"2023-10-09T10:46:33.493288+0000","last_unstale":"2023-10-09T10:48:06.076800+0000","last_undegraded":"2023-10-09T10:46:33.283624+0000","last_fullsized":"2023-10-09T10:46:33.244043+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6510,"log_dups_size":0,"ondisk_log_size":6510,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78086144,"num_objects":1191,"num_object_clones":0,"num_object_copies":3573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":838,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5473,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2260,"num_bytes_recovered":129564672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"3,4","objects":353}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":513,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.241860+0000","last_change":"2023-10-09T10:46:05.617589+0000","last_active":"2023-10-09T10:47:52.241860+0000","last_peered":"2023-10-09T10:47:52.241860+0000","last_clean":"2023-10-09T10:47:52.241860+0000","last_became_active":"2023-10-09T10:46:05.616400+0000","last_became_peered":"2023-10-09T10:46:05.616400+0000","last_unstale":"2023-10-09T10:47:52.241860+0000","last_undegraded":"2023-10-09T10:47:52.241860+0000","last_fullsized":"2023-10-09T10:47:52.241860+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":37,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.213108+0000","last_change":"2023-10-09T10:46:32.245106+0000","last_active":"2023-10-09T10:47:52.213108+0000","last_peered":"2023-10-09T10:47:52.213108+0000","last_clean":"2023-10-09T10:47:52.213108+0000","last_became_active":"2023-10-09T10:46:32.244826+0000","last_became_peered":"2023-10-09T10:46:32.244826+0000","last_unstale":"2023-10-09T10:47:52.213108+0000","last_undegraded":"2023-10-09T10:47:52.213108+0000","last_fullsized":"2023-10-09T10:47:52.213108+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:30.322834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":52,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.242730+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:47:52.242730+0000","last_peered":"2023-10-09T10:47:52.242730+0000","last_clean":"2023-10-09T10:47:52.242730+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:47:52.242730+0000","last_undegraded":"2023-10-09T10:47:52.242730+0000","last_fullsized":"2023-10-09T10:47:52.242730+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":127,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.251932+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:47:52.251932+0000","last_peered":"2023-10-09T10:47:52.251932+0000","last_clean":"2023-10-09T10:47:52.251932+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:47:52.251932+0000","last_undegraded":"2023-10-09T10:47:52.251932+0000","last_fullsized":"2023-10-09T10:47:52.251932+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"180'6404","reported_seq":13213,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:05.896429+0000","last_change":"2023-10-09T10:47:31.271196+0000","last_active":"2023-10-09T10:48:05.896429+0000","last_peered":"2023-10-09T10:48:05.896429+0000","last_clean":"2023-10-09T10:48:05.896429+0000","last_became_active":"2023-10-09T10:47:31.229504+0000","last_became_peered":"2023-10-09T10:47:31.229504+0000","last_unstale":"2023-10-09T10:48:05.896429+0000","last_undegraded":"2023-10-09T10:48:05.896429+0000","last_fullsized":"2023-10-09T10:48:05.896429+0000","mapping_epoch":172,"log_start":"104'1388","ondisk_log_start":"104'1388","created":98,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":5016,"log_dups_size":0,"ondisk_log_size":5016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:26.533476+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80084992,"num_objects":1222,"num_object_clones":0,"num_object_copies":3666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3759,"num_write_kb":107264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3305,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":48,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.241664+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:47:52.241664+0000","last_peered":"2023-10-09T10:47:52.241664+0000","last_clean":"2023-10-09T10:47:52.241664+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:47:52.241664+0000","last_undegraded":"2023-10-09T10:47:52.241664+0000","last_fullsized":"2023-10-09T10:47:52.241664+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":112,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.324848+0000","last_change":"2023-10-09T10:46:32.251242+0000","last_active":"2023-10-09T10:48:00.324848+0000","last_peered":"2023-10-09T10:48:00.324848+0000","last_clean":"2023-10-09T10:48:00.324848+0000","last_became_active":"2023-10-09T10:46:32.250534+0000","last_became_peered":"2023-10-09T10:46:32.250534+0000","last_unstale":"2023-10-09T10:48:00.324848+0000","last_undegraded":"2023-10-09T10:48:00.324848+0000","last_fullsized":"2023-10-09T10:48:00.324848+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:33.726041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"180'6384","reported_seq":10514,"reported_epoch":180,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:02.651646+0000","last_change":"2023-10-09T10:46:34.514479+0000","last_active":"2023-10-09T10:48:02.651646+0000","last_peered":"2023-10-09T10:48:02.651646+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:46:34.409667+0000","last_became_peered":"2023-10-09T10:46:34.409667+0000","last_unstale":"2023-10-09T10:48:02.651646+0000","last_undegraded":"2023-10-09T10:46:34.376635+0000","last_fullsized":"2023-10-09T10:46:33.237998+0000","mapping_epoch":154,"log_start":"103'366","ondisk_log_start":"103'366","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6018,"log_dups_size":0,"ondisk_log_size":6018,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87293952,"num_objects":1332,"num_object_clones":0,"num_object_copies":3996,"num_objects_missing_on_primary":1333,"num_objects_missing":1333,"num_objects_degraded":1843,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5342,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2005,"num_bytes_recovered":131399680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,7","objects":-1},{"shards":"2,3,7","objects":1333}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":147,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.251169+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:47:52.251169+0000","last_peered":"2023-10-09T10:47:52.251169+0000","last_clean":"2023-10-09T10:47:52.251169+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:47:52.251169+0000","last_undegraded":"2023-10-09T10:47:52.251169+0000","last_fullsized":"2023-10-09T10:47:52.251169+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":161,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.252601+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:47:52.252601+0000","last_peered":"2023-10-09T10:47:52.252601+0000","last_clean":"2023-10-09T10:47:52.252601+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:47:52.252601+0000","last_undegraded":"2023-10-09T10:47:52.252601+0000","last_fullsized":"2023-10-09T10:47:52.252601+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"180'6387","reported_seq":6499,"reported_epoch":180,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:04.541916+0000","last_change":"2023-10-09T10:46:33.546991+0000","last_active":"2023-10-09T10:48:04.541916+0000","last_peered":"2023-10-09T10:48:04.541916+0000","last_clean":"2023-10-09T10:46:31.009484+0000","last_became_active":"2023-10-09T10:46:33.535665+0000","last_became_peered":"2023-10-09T10:46:33.535665+0000","last_unstale":"2023-10-09T10:48:04.541916+0000","last_undegraded":"2023-10-09T10:46:33.277168+0000","last_fullsized":"2023-10-09T10:46:33.235038+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6387,"log_dups_size":0,"ondisk_log_size":6387,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78938112,"num_objects":1205,"num_object_clones":0,"num_object_copies":3615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5323,"num_write_kb":208864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1217},{"shards":"3,7","objects":-1553}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":130,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.252395+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:47:52.252395+0000","last_peered":"2023-10-09T10:47:52.252395+0000","last_clean":"2023-10-09T10:47:52.252395+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:47:52.252395+0000","last_undegraded":"2023-10-09T10:47:52.252395+0000","last_fullsized":"2023-10-09T10:47:52.252395+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":28,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.258383+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:47:52.258383+0000","last_peered":"2023-10-09T10:47:52.258383+0000","last_clean":"2023-10-09T10:47:52.258383+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:47:52.258383+0000","last_undegraded":"2023-10-09T10:47:52.258383+0000","last_fullsized":"2023-10-09T10:47:52.258383+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":164,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.213439+0000","last_change":"2023-10-09T10:46:32.244607+0000","last_active":"2023-10-09T10:47:52.213439+0000","last_peered":"2023-10-09T10:47:52.213439+0000","last_clean":"2023-10-09T10:47:52.213439+0000","last_became_active":"2023-10-09T10:46:32.242628+0000","last_became_peered":"2023-10-09T10:46:32.242628+0000","last_unstale":"2023-10-09T10:47:52.213439+0000","last_undegraded":"2023-10-09T10:47:52.213439+0000","last_fullsized":"2023-10-09T10:47:52.213439+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:10.738819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":194,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.218636+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:47:52.218636+0000","last_peered":"2023-10-09T10:47:52.218636+0000","last_clean":"2023-10-09T10:47:52.218636+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:47:52.218636+0000","last_undegraded":"2023-10-09T10:47:52.218636+0000","last_fullsized":"2023-10-09T10:47:52.218636+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":69,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.246404+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:47:52.246404+0000","last_peered":"2023-10-09T10:47:52.246404+0000","last_clean":"2023-10-09T10:47:52.246404+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:47:52.246404+0000","last_undegraded":"2023-10-09T10:47:52.246404+0000","last_fullsized":"2023-10-09T10:47:52.246404+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":144,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-09T10:47:52.251710+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:47:52.251710+0000","last_peered":"2023-10-09T10:47:52.251710+0000","last_clean":"2023-10-09T10:47:52.251710+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:47:52.251710+0000","last_undegraded":"2023-10-09T10:47:52.251710+0000","last_fullsized":"2023-10-09T10:47:52.251710+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":2565242904,"num_objects":39144,"num_object_clones":0,"num_object_copies":117432,"num_objects_missing_on_primary":1333,"num_objects_missing":1333,"num_objects_degraded":31079,"num_objects_misplaced":3730,"num_objects_unfound":0,"num_objects_dirty":39144,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":165064,"num_write_kb":6258529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24157,"num_bytes_recovered":1471938560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9878122496,"data_stored":9878110280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167946,"ondisk_log_size":167946,"up":78,"acting":68,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1186,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332258,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1400280,"kb_used_data":1094736,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92344872,"statfs":{"total":95995035648,"available":94561148928,"internally_reserved":0,"allocated":1121009664,"data_stored":1120324536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":0.79200000000000004,"15min":0.65000000000000002},"min":{"1min":0.26300000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.1479999999999997,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":2.512},{"interface":"front","average":{"1min":1.546,"5min":0.97199999999999998,"15min":0.752},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.9909999999999997,"5min":10.207000000000001,"15min":10.207000000000001},"last":0.248}]},{"osd":1,"last update":"Mon Oct 9 10:47:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2210000000000001,"5min":1.3300000000000001,"15min":0.80400000000000005},"min":{"1min":0.43099999999999999,"5min":0.318,"15min":0.318},"max":{"1min":11.826000000000001,"5min":11.826000000000001,"15min":11.826000000000001},"last":2.5760000000000001},{"interface":"front","average":{"1min":2.286,"5min":1.155,"15min":0.73899999999999999},"min":{"1min":0.32100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":11.84,"5min":11.84,"15min":11.84},"last":2.4649999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.1499999999999999,"15min":0.72399999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.4770000000000003,"5min":6.4770000000000003,"15min":6.4770000000000003},"last":2.4300000000000002},{"interface":"front","average":{"1min":1.272,"5min":0.85699999999999998,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.251,"15min":0.251},"max":{"1min":3.1480000000000001,"5min":3.7120000000000002,"15min":3.7120000000000002},"last":2.5590000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.23,"15min":0.80000000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.0599999999999996,"5min":8.1639999999999997,"15min":8.1639999999999997},"last":5.7640000000000002},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.099,"15min":0.76600000000000001},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.504,"5min":12.451000000000001,"15min":12.451000000000001},"last":9.7789999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.73299999999999998,"15min":0.56200000000000006},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.1420000000000003,"5min":7.4509999999999996,"15min":7.4509999999999996},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.65300000000000002,"15min":0.53600000000000003},"min":{"1min":0.29799999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.2749999999999999,"5min":3.1699999999999999,"15min":3.1699999999999999},"last":2.0099999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.56399999999999995,"15min":0.52700000000000002},"min":{"1min":0.34300000000000003,"5min":0.247,"15min":0.247},"max":{"1min":2.2349999999999999,"5min":2.2349999999999999,"15min":2.2349999999999999},"last":2.5870000000000002},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.81299999999999994,"15min":0.63},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":5.0860000000000003,"5min":10.33,"15min":10.33},"last":2.548}]},{"osd":6,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.83399999999999996,"15min":0.64800000000000002},"min":{"1min":0.311,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.109,"5min":10.164999999999999,"15min":10.164999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.66000000000000003,"15min":0.58299999999999996},"min":{"1min":0.33300000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.306,"5min":2.8079999999999998,"15min":2.8079999999999998},"last":2.2109999999999999}]}]},{"osd":6,"up_from":44,"seq":188978561189,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1213268,"kb_used_data":903436,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92531884,"statfs":{"total":95995035648,"available":94752649216,"internally_reserved":0,"allocated":925118464,"data_stored":924437874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.93200000000000005,"15min":0.71799999999999997},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.4630000000000001,"5min":7.3220000000000001,"15min":7.3220000000000001},"last":2.3679999999999999},{"interface":"front","average":{"1min":1.02,"5min":1.002,"15min":0.73899999999999999},"min":{"1min":0.33900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.4969999999999999,"5min":7.5309999999999997,"15min":7.5309999999999997},"last":2.448}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.157,"15min":0.77900000000000003},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.245},"max":{"1min":6.7560000000000002,"5min":16.747,"15min":16.747},"last":2.4129999999999998},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.284,"15min":0.79500000000000004},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.3579999999999997,"5min":16.734000000000002,"15min":16.734000000000002},"last":2.4740000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":0.95799999999999996,"15min":0.67500000000000004},"min":{"1min":0.28699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5069999999999997,"5min":7.5069999999999997,"15min":7.5069999999999997},"last":2.5099999999999998},{"interface":"front","average":{"1min":1.675,"5min":1.143,"15min":0.755},"min":{"1min":0.28599999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.3899999999999997,"5min":12.571,"15min":12.571},"last":0.55800000000000005}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.117,"15min":0.73099999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.8070000000000004,"5min":6.8070000000000004,"15min":6.8070000000000004},"last":5.9710000000000001},{"interface":"front","average":{"1min":2.1219999999999999,"5min":1.385,"15min":0.82599999999999996},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.3940000000000001,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":2.4910000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":0.66600000000000004,"15min":0.51500000000000001},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":4.0739999999999998,"5min":4.0739999999999998,"15min":4.0739999999999998},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.67300000000000004,"15min":0.54600000000000004},"min":{"1min":0.34899999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6600000000000001,"5min":3.6600000000000001,"15min":3.6600000000000001},"last":2.4279999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":0.76000000000000001,"15min":0.56999999999999995},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.8949999999999996,"5min":4.8949999999999996,"15min":4.8949999999999996},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.76900000000000002,"15min":0.59999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":4.3639999999999999,"5min":4.3639999999999999,"15min":4.3639999999999999},"last":2.3900000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:47:25 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69899999999999995,"15min":0.57999999999999996},"min":{"1min":0.35799999999999998,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":2.5049999999999999,"5min":3.472,"15min":3.472},"last":0.90100000000000002},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.84299999999999997,"15min":0.65700000000000003},"min":{"1min":0.373,"5min":0.36899999999999999,"15min":0.34000000000000002},"max":{"1min":4.0800000000000001,"5min":4.0800000000000001,"15min":4.0800000000000001},"last":6.0110000000000001}]}]},{"osd":1,"up_from":85,"seq":365072220228,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1170124,"kb_used_data":1048196,"kb_used_omap":9,"kb_used_meta":121782,"kb_avail":92575028,"statfs":{"total":95995035648,"available":94796828672,"internally_reserved":0,"allocated":1073352704,"data_stored":1072448335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":124705769},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.581,"15min":0.76900000000000002},"min":{"1min":0.251,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.2119999999999997,"5min":14.1,"15min":14.1},"last":2.012},{"interface":"front","average":{"1min":1.885,"5min":1.5629999999999999,"15min":0.78700000000000003},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.8090000000000002,"5min":14.465,"15min":14.465},"last":3.2610000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:47:13 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.1200000000000001,"15min":0.61799999999999999},"min":{"1min":0.21199999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.4829999999999997,"5min":8.6859999999999999,"15min":8.6859999999999999},"last":2.2850000000000001},{"interface":"front","average":{"1min":1.919,"5min":1.4690000000000001,"15min":0.746},"min":{"1min":0.29099999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.2829999999999995,"5min":14.234,"15min":14.234},"last":2.0710000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":1.5269999999999999,"15min":0.82299999999999995},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":7.234,"5min":14.404999999999999,"15min":14.404999999999999},"last":2.0430000000000001},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.429,"15min":0.70599999999999996},"min":{"1min":0.30399999999999999,"5min":0.251,"15min":0.251},"max":{"1min":15.468999999999999,"5min":15.468999999999999,"15min":15.468999999999999},"last":3.29}]},{"osd":4,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.1679999999999999,"15min":0.78400000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.1449999999999996,"5min":8.0129999999999999,"15min":8.0129999999999999},"last":1.2869999999999999},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.2350000000000001,"15min":0.82499999999999996},"min":{"1min":0.30299999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.1189999999999998,"5min":8.327,"15min":8.327},"last":7.524}]},{"osd":5,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":1.111,"15min":0.78900000000000003},"min":{"1min":0.28899999999999998,"5min":0.253,"15min":0.253},"max":{"1min":5.4850000000000003,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":3.984},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.2929999999999999,"15min":0.86199999999999999},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.1859999999999999,"5min":7.923,"15min":7.923},"last":6.9809999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7909999999999999,"5min":1.2030000000000001,"15min":0.82199999999999995},"min":{"1min":0.35499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.9450000000000003,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":6.4240000000000004},{"interface":"front","average":{"1min":1.246,"5min":1.2949999999999999,"15min":0.86299999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.8440000000000003,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":2.6160000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.22,"15min":0.83999999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.8689999999999998,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":1.3149999999999999},{"interface":"front","average":{"1min":1.992,"5min":1.2909999999999999,"15min":0.86099999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":7.1589999999999998,"5min":8.2469999999999999,"15min":8.2469999999999999},"last":7.5599999999999996}]}]},{"osd":0,"up_from":80,"seq":343597383761,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":571692,"kb_used_data":455588,"kb_used_omap":28,"kb_used_meta":116067,"kb_avail":93173460,"statfs":{"total":95995035648,"available":95409623040,"internally_reserved":0,"allocated":466522112,"data_stored":465585104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":118853456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:47:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2509999999999999,"5min":1.5449999999999999,"15min":0.76000000000000001},"min":{"1min":0.22,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":10.564,"5min":25.475000000000001,"15min":25.475000000000001},"last":0.191},{"interface":"front","average":{"1min":2.3740000000000001,"5min":1.5940000000000001,"15min":0.77400000000000002},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":10.593999999999999,"5min":20.832999999999998,"15min":20.832999999999998},"last":1.925}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":1.2350000000000001,"15min":0.64400000000000002},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.907,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":0.38},{"interface":"front","average":{"1min":1.754,"5min":1.5349999999999999,"15min":0.73999999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.5619999999999994,"5min":14.391999999999999,"15min":14.391999999999999},"last":0.624}]},{"osd":3,"last update":"Mon Oct 9 10:47:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4740000000000002,"5min":1.8959999999999999,"15min":1.0660000000000001},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.215},"max":{"1min":10.644,"5min":14.926,"15min":14.926},"last":4.3109999999999999},{"interface":"front","average":{"1min":2.2269999999999999,"5min":1.5940000000000001,"15min":0.89200000000000002},"min":{"1min":0.34499999999999997,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.228999999999999,"5min":11.868,"15min":11.868},"last":1.071}]},{"osd":4,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.2250000000000001,"15min":0.82199999999999995},"min":{"1min":0.38400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.8150000000000004,"5min":6.8150000000000004,"15min":6.8150000000000004},"last":2.4399999999999999},{"interface":"front","average":{"1min":2.391,"5min":1.405,"15min":0.88100000000000001},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.676,"5min":10.676,"15min":10.676},"last":1.966}]},{"osd":5,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":1.4359999999999999,"15min":0.89600000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":10.734,"5min":10.734,"15min":10.734},"last":1.988},{"interface":"front","average":{"1min":1.716,"5min":1.1659999999999999,"15min":0.82099999999999995},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.54,"5min":8.9190000000000005,"15min":8.9190000000000005},"last":12.781000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.391,"15min":0.88400000000000001},"min":{"1min":0.36599999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.706,"5min":10.706,"15min":10.706},"last":5.4720000000000004},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.0780000000000001,"15min":0.79300000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.0499999999999998,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":0.41299999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.474,"5min":1.151,"15min":0.81899999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.5380000000000003,"5min":8.875,"15min":8.875},"last":12.727},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.425,"15min":0.92700000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":14.394,"5min":14.394,"15min":14.394},"last":0.59299999999999997}]}]},{"osd":2,"up_from":0,"seq":386547056702,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893176,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1944024,"kb_used_data":1750032,"kb_used_omap":7,"kb_used_meta":193976,"kb_avail":91801128,"statfs":{"total":95995035648,"available":94004355072,"internally_reserved":0,"allocated":1792032768,"data_stored":1791114224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":198632040},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":1.3140000000000001,"15min":0.65200000000000002},"min":{"1min":0.192,"5min":0.183,"15min":0.183},"max":{"1min":7.0289999999999999,"5min":11.557,"15min":11.557},"last":1.8160000000000001},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.1379999999999999,"15min":0.65000000000000002},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":6.681,"5min":7.0510000000000002,"15min":7.0510000000000002},"last":2.661}]},{"osd":1,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":1.6970000000000001,"15min":0.78500000000000003},"min":{"1min":0.27100000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.048,"5min":14.707000000000001,"15min":14.707000000000001},"last":2.0390000000000001},{"interface":"front","average":{"1min":1.6220000000000001,"5min":1.1890000000000001,"15min":0.64500000000000002},"min":{"1min":0.19400000000000001,"5min":0.193,"15min":0.193},"max":{"1min":7.1630000000000003,"5min":11.629,"15min":11.629},"last":1.7869999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.6739999999999999,"15min":0.80200000000000005},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.9180000000000001,"5min":13.853,"15min":13.853},"last":2.0790000000000002},{"interface":"front","average":{"1min":2.5590000000000002,"5min":1.5129999999999999,"15min":0.76800000000000002},"min":{"1min":0.30099999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.71,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":0.28699999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.2210000000000001,"15min":0.78800000000000003},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.9909999999999997,"5min":11.532,"15min":11.532},"last":0.33900000000000002},{"interface":"front","average":{"1min":2.2440000000000002,"5min":1.296,"15min":0.83699999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.962999999999999,"5min":11.962999999999999,"15min":11.962999999999999},"last":0.38700000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7589999999999999,"5min":1.8029999999999999,"15min":0.99199999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.907,"5min":51.509999999999998,"15min":51.509999999999998},"last":3.2509999999999999},{"interface":"front","average":{"1min":2.1469999999999998,"5min":2.306,"15min":1.1439999999999999},"min":{"1min":0.36799999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.1139999999999999,"5min":55.338999999999999,"15min":55.338999999999999},"last":3.1909999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":1.3620000000000001,"15min":0.84599999999999997},"min":{"1min":0.378,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":11.92,"5min":11.92,"15min":11.92},"last":2.1110000000000002},{"interface":"front","average":{"1min":2.3759999999999999,"5min":1.776,"15min":0.99199999999999999},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":10.528,"5min":13.891999999999999,"15min":13.891999999999999},"last":2.1549999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":2.3410000000000002,"15min":1.175},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.6520000000000001,"5min":51.744,"15min":51.744},"last":2.1760000000000002},{"interface":"front","average":{"1min":1.9139999999999999,"5min":1.869,"15min":1.0049999999999999},"min":{"1min":0.33000000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.133,"5min":53.399000000000001,"15min":53.399000000000001},"last":3.488}]}]},{"osd":4,"up_from":30,"seq":128849019052,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1090964,"kb_used_data":785484,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92654188,"statfs":{"total":95995035648,"available":94877888512,"internally_reserved":0,"allocated":804335616,"data_stored":803654293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.0129999999999999,"15min":0.70899999999999996},"min":{"1min":0.24099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.5419999999999998,"5min":10.387,"15min":10.387},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.581,"5min":1.1819999999999999,"15min":0.78400000000000003},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.5999999999999996,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.53300000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.1519999999999999,"15min":0.752},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.9100000000000001,"5min":6.907,"15min":6.907},"last":0.5},{"interface":"front","average":{"1min":1.859,"5min":1.2390000000000001,"15min":0.82699999999999996},"min":{"1min":0.30099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.7669999999999995,"5min":8.7669999999999995,"15min":8.7669999999999995},"last":4.923}]},{"osd":2,"last update":"Mon Oct 9 10:47:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2170000000000001,"5min":1.1000000000000001,"15min":0.747},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.2149999999999999,"5min":7.5449999999999999,"15min":7.5449999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.579,"5min":1.1639999999999999,"15min":0.76300000000000001},"min":{"1min":0.36699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.5110000000000001,"5min":6.2939999999999996,"15min":6.2939999999999996},"last":0.58699999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:47:38 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.333,"15min":0.91900000000000004},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.8170000000000002,"5min":8.8170000000000002,"15min":8.8170000000000002},"last":5.0430000000000001},{"interface":"front","average":{"1min":2.3650000000000002,"5min":1.359,"15min":0.84499999999999997},"min":{"1min":0.34300000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.625,"5min":8.9809999999999999,"15min":8.9809999999999999},"last":1.0980000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":0.89500000000000002,"15min":0.59699999999999998},"min":{"1min":0.25600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":8.6950000000000003,"5min":8.6950000000000003,"15min":8.6950000000000003},"last":4.9669999999999996},{"interface":"front","average":{"1min":1.6579999999999999,"5min":0.94299999999999995,"15min":0.61299999999999999},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.252},"max":{"1min":4.8879999999999999,"5min":4.8879999999999999,"15min":4.8879999999999999},"last":1.319}]},{"osd":6,"last update":"Mon Oct 9 10:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.74099999999999999,"15min":0.55700000000000005},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":7.1580000000000004,"5min":7.1580000000000004,"15min":7.1580000000000004},"last":2.3010000000000002},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.71599999999999997,"15min":0.60899999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":3.6150000000000002,"5min":3.6150000000000002,"15min":3.6150000000000002},"last":5.0190000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.80800000000000005,"15min":0.64600000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.1050000000000004,"5min":4.1050000000000004,"15min":4.1050000000000004},"last":4.9939999999999998},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.748,"15min":0.60799999999999998},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0919999999999996,"5min":4.0919999999999996,"15min":4.0919999999999996},"last":0.84799999999999998}]}]},{"osd":5,"up_from":37,"seq":158913790121,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1324740,"kb_used_data":1011068,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":92420412,"statfs":{"total":95995035648,"available":94638501888,"internally_reserved":0,"allocated":1035333632,"data_stored":1034664138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.77600000000000002,"15min":0.65700000000000003},"min":{"1min":0.247,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.9860000000000002,"5min":4.8380000000000001,"15min":4.8380000000000001},"last":5.9429999999999996},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.84899999999999998,"15min":0.72899999999999998},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.0209999999999999,"5min":4.8650000000000002,"15min":4.8650000000000002},"last":5.984}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":1.367,"15min":0.86099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":12.999000000000001,"5min":12.999000000000001,"15min":12.999000000000001},"last":8.0909999999999993},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.2370000000000001,"15min":0.80700000000000005},"min":{"1min":0.27600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3730000000000002,"5min":7.3730000000000002,"15min":7.3730000000000002},"last":0.61399999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:47:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.02,"15min":0.72099999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.4569999999999999,"5min":7.9000000000000004,"15min":7.9000000000000004},"last":1.4670000000000001},{"interface":"front","average":{"1min":1.7569999999999999,"5min":0.90400000000000003,"15min":0.68400000000000005},"min":{"1min":0.38,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":10.352,"5min":10.352,"15min":10.352},"last":0.44800000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4969999999999999,"5min":1.621,"15min":1.0229999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":18.016999999999999,"5min":18.016999999999999,"15min":18.016999999999999},"last":0.66800000000000004},{"interface":"front","average":{"1min":2.964,"5min":1.673,"15min":0.96599999999999997},"min":{"1min":0.318,"5min":0.317,"15min":0.317},"max":{"1min":15.555999999999999,"5min":15.555999999999999,"15min":15.555999999999999},"last":0.63900000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":0.83799999999999997,"15min":0.57799999999999996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":19.152999999999999,"5min":19.152999999999999,"15min":19.152999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.986,"5min":0.96999999999999997,"15min":0.63500000000000001},"min":{"1min":0.26800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.3059999999999992,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":0.91600000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:47:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.66300000000000003,"15min":0.53500000000000003},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":2.8900000000000001,"5min":2.9529999999999998,"15min":2.9529999999999998},"last":1.4399999999999999},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.63300000000000001,"15min":0.53800000000000003},"min":{"1min":0.34100000000000003,"5min":0.33300000000000002,"15min":0.30299999999999999},"max":{"1min":3.8159999999999998,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":0.42399999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:47:22 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.67100000000000004,"15min":0.58799999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.4729999999999999,"5min":4.0510000000000002,"15min":4.0510000000000002},"last":1.409},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.70899999999999996,"15min":0.60099999999999998},"min":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":4.1289999999999996,"5min":4.1289999999999996,"15min":4.1289999999999996},"last":0.59099999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":459542528,"data_stored":459538456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1055854592,"data_stored":1055850520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2725838848,"data_stored":2725838848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1788018688,"data_stored":1788018688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":790626304,"data_stored":790626304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1025118208,"data_stored":1025114136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":922550272,"data_stored":922550272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1110573056,"data_stored":1110573056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:48:08.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:08.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:08.155 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:08.208 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:08.399 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:48:08.399 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:48:08.551 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:08.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:08.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:08.656 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:08.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:08 smithi125 ceph-mon[195507]: from='client.54829 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:08.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:08 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 31079/117504 objects degraded (26.449%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:08.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:08 smithi125 ceph-mon[200197]: from='client.54829 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:08.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:08 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 31079/117504 objects degraded (26.449%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:08.721 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:08.916 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:48:08.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:08 smithi155 ceph-mon[184854]: from='client.54829 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:08.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:08 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 31079/117504 objects degraded (26.449%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:09.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:09 smithi125 ceph-mon[195507]: pgmap v390: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 30985/116886 objects degraded (26.509%); 3730/116886 objects misplaced (3.191%); 4.8 MiB/s, 108 objects/s recovering 2023-10-09T10:48:09.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:09 smithi125 ceph-mon[200197]: pgmap v390: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 30985/116886 objects degraded (26.509%); 3730/116886 objects misplaced (3.191%); 4.8 MiB/s, 108 objects/s recovering 2023-10-09T10:48:09.919 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:09.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:09.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:09.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:09 smithi155 ceph-mon[184854]: pgmap v390: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 30985/116886 objects degraded (26.509%); 3730/116886 objects misplaced (3.191%); 4.8 MiB/s, 108 objects/s recovering 2023-10-09T10:48:10.026 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:10.072 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:10.300 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:10.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:10.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:10.406 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:10.464 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:10.496 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:10.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:10.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:10.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:10.654 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:10.687 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:10.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:10.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:10.791 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:10.837 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:10.874 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:10.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:10.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:10.982 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:11.044 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:11.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:48:11.096 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:11.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:11.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:11.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:11.271 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:11.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:48:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:48:11] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.43.0" 2023-10-09T10:48:12.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:12 smithi155 ceph-mon[184854]: pgmap v391: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 693 op/s; 30677/108204 objects degraded (28.351%); 3528/108204 objects misplaced (3.261%); 2.1 MiB/s, 116 objects/s recovering 2023-10-09T10:48:12.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:12 smithi125 ceph-mon[195507]: pgmap v391: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 693 op/s; 30677/108204 objects degraded (28.351%); 3528/108204 objects misplaced (3.261%); 2.1 MiB/s, 116 objects/s recovering 2023-10-09T10:48:12.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:12 smithi125 ceph-mon[200197]: pgmap v391: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 693 op/s; 30677/108204 objects degraded (28.351%); 3528/108204 objects misplaced (3.261%); 2.1 MiB/s, 116 objects/s recovering 2023-10-09T10:48:12.914 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:12.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:12.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:12.932 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:48:13.023 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:13.093 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:13.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:13.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:13.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:13.253 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:13.305 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:13.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:13.340 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:13 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30677/108204 objects degraded (28.351%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:13.340 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:13.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:13.441 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:13.478 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:13.490 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:13.515 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:13.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:13.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:13.536 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:13.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:13 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30677/108204 objects degraded (28.351%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:13.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:13 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30677/108204 objects degraded (28.351%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:13.624 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:13.675 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:14.261 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:14.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:14.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:14.370 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:14.426 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:14.515 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:14 smithi125 ceph-mon[195507]: pgmap v392: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 30559/106239 objects degraded (28.764%); 3425/106239 objects misplaced (3.224%); 2.4 MiB/s, 120 objects/s recovering 2023-10-09T10:48:14.516 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:14 smithi125 ceph-mon[200197]: pgmap v392: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 30559/106239 objects degraded (28.764%); 3425/106239 objects misplaced (3.224%); 2.4 MiB/s, 120 objects/s recovering 2023-10-09T10:48:14.710 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:14.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:14.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:14.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:14 smithi155 ceph-mon[184854]: pgmap v392: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 30559/106239 objects degraded (28.764%); 3425/106239 objects misplaced (3.224%); 2.4 MiB/s, 120 objects/s recovering 2023-10-09T10:48:14.819 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:14.882 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:48:14.896 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:14.930 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:48:14.946 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:14.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:14.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:14.971 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:48:15.007 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:48:15.037 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:15.053 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:48:15.055 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:15.107 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:48:15.115 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:15.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:15.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:15.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:15.164 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:48:15.207 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:48:15.236 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:48:15.236 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:48:15.236 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:48:15.236 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:48:15.236 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:48:15.236 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:48:15.236 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:48:15.236 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:48:15.254 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:15.313 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:15.340 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:15.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:15.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:15.403 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:15 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/583531436' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:15.404 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:15.404 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:48:15.404 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:15 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/583531436' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:15.404 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:15.404 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:48:15.445 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:15.499 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:15 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/583531436' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:15.500 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:15.500 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:48:15.503 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:15.553 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:15.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:15.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:15.663 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:15.734 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:15.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:15.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:15.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:15.898 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:15.976 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:16.464 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:16 smithi125 ceph-mon[195507]: pgmap v393: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 30559/101442 objects degraded (30.125%); 3314/101442 objects misplaced (3.267%); 2.4 MiB/s, 121 objects/s recovering 2023-10-09T10:48:16.465 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:16 smithi125 ceph-mon[200197]: pgmap v393: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 30559/101442 objects degraded (30.125%); 3314/101442 objects misplaced (3.267%); 2.4 MiB/s, 121 objects/s recovering 2023-10-09T10:48:16.471 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:16.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:16.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:16.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:16.619 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:16.706 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:16.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:16.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:16.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:16 smithi155 ceph-mon[184854]: pgmap v393: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 30559/101442 objects degraded (30.125%); 3314/101442 objects misplaced (3.267%); 2.4 MiB/s, 121 objects/s recovering 2023-10-09T10:48:16.811 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:16.862 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:16.959 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:16.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:16.977 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:48:16.977 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:48:16.978 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:48:16.978 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:48:16.978 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:48:17.064 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:17.096 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:17.124 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:17.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:17.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:17.227 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:17.277 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:48:17.277 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:17.407 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:48:17.461 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:17.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:17.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:17.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[195507]: osdmap e182: 8 total, 8 up, 7 in 2023-10-09T10:48:17.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[195507]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[195507]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[195507]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[195507]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[195507]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[195507]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[195507]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[195507]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.471 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/798435630' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:48:17.471 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:48:17.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[200197]: osdmap e182: 8 total, 8 up, 7 in 2023-10-09T10:48:17.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[200197]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[200197]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[200197]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[200197]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[200197]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[200197]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[200197]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.472 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[200197]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.472 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/798435630' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:48:17.472 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:17 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:48:17.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:17 smithi155 ceph-mon[184854]: osdmap e182: 8 total, 8 up, 7 in 2023-10-09T10:48:17.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:17 smithi155 ceph-mon[184854]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:17 smithi155 ceph-mon[184854]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:17 smithi155 ceph-mon[184854]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:17 smithi155 ceph-mon[184854]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:17 smithi155 ceph-mon[184854]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:17 smithi155 ceph-mon[184854]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:17 smithi155 ceph-mon[184854]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:17 smithi155 ceph-mon[184854]: failed to encode map e182 with expected crc 2023-10-09T10:48:17.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/798435630' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:48:17.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:17 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:48:17.517 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:48:17.553 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:48:17.568 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:17.619 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:17.657 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:17.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:17.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:17.766 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:17.820 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:17.832 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:48:17.948 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:17.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:17.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:18.061 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:18.104 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:18.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:18.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:18.243 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:18.252 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:18.279 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:18.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:18.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:18.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:18.413 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:18.454 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[195507]: pgmap v395: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 28481/95982 objects degraded (29.673%); 4068/95982 objects misplaced (4.238%); 1.5 MiB/s, 131 objects/s recovering 2023-10-09T10:48:18.454 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:18.454 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:48:18.454 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[195507]: osdmap e183: 8 total, 8 up, 7 in 2023-10-09T10:48:18.454 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[195507]: failed to encode map e183 with expected crc 2023-10-09T10:48:18.454 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[195507]: failed to encode map e183 with expected crc 2023-10-09T10:48:18.454 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[195507]: failed to encode map e183 with expected crc 2023-10-09T10:48:18.454 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 28481/95982 objects degraded (29.673%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:18.455 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[200197]: pgmap v395: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 28481/95982 objects degraded (29.673%); 4068/95982 objects misplaced (4.238%); 1.5 MiB/s, 131 objects/s recovering 2023-10-09T10:48:18.455 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:18.455 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:48:18.455 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[200197]: osdmap e183: 8 total, 8 up, 7 in 2023-10-09T10:48:18.455 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[200197]: failed to encode map e183 with expected crc 2023-10-09T10:48:18.455 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[200197]: failed to encode map e183 with expected crc 2023-10-09T10:48:18.455 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[200197]: failed to encode map e183 with expected crc 2023-10-09T10:48:18.455 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:18 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 28481/95982 objects degraded (29.673%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:18.471 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":396,"stamp":"2023-10-09T10:48:17.263224+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2095683512,"num_objects":31994,"num_object_clones":0,"num_object_copies":95982,"num_objects_missing_on_primary":1121,"num_objects_missing":1121,"num_objects_degraded":28481,"num_objects_misplaced":4068,"num_objects_unfound":0,"num_objects_dirty":31994,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":172538,"num_write_kb":6260974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25494,"num_bytes_recovered":1489177346,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173856,"ondisk_log_size":173856,"up":357,"acting":347,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7451828,"kb_used_data":5797948,"kb_used_omap":44,"kb_used_meta":1653715,"kb_avail":648764236,"statfs":{"total":671965249536,"available":664334577664,"internally_reserved":0,"allocated":5937098752,"data_stored":5931584038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1693404577},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-456654848,"num_objects":-6968,"num_object_clones":0,"num_object_copies":-20904,"num_objects_missing_on_primary":-118,"num_objects_missing":-118,"num_objects_degraded":-2504,"num_objects_misplaced":338,"num_objects_unfound":0,"num_objects_dirty":-6968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6968,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1219,"num_bytes_recovered":9306112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.123799"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":174,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.329856+0000","last_change":"2023-10-09T10:46:32.250979+0000","last_active":"2023-10-09T10:48:00.329856+0000","last_peered":"2023-10-09T10:48:00.329856+0000","last_clean":"2023-10-09T10:48:00.329856+0000","last_became_active":"2023-10-09T10:46:32.250625+0000","last_became_peered":"2023-10-09T10:46:32.250625+0000","last_unstale":"2023-10-09T10:48:00.329856+0000","last_undegraded":"2023-10-09T10:48:00.329856+0000","last_fullsized":"2023-10-09T10:48:00.329856+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:34.858348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":149,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.333488+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:48:00.333488+0000","last_peered":"2023-10-09T10:48:00.333488+0000","last_clean":"2023-10-09T10:48:00.333488+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:48:00.333488+0000","last_undegraded":"2023-10-09T10:48:00.333488+0000","last_fullsized":"2023-10-09T10:48:00.333488+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":134,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.466330+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:48:06.466330+0000","last_peered":"2023-10-09T10:48:06.466330+0000","last_clean":"2023-10-09T10:48:06.466330+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:48:06.466330+0000","last_undegraded":"2023-10-09T10:48:06.466330+0000","last_fullsized":"2023-10-09T10:48:06.466330+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":50,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:02.705503+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:48:02.705503+0000","last_peered":"2023-10-09T10:48:02.705503+0000","last_clean":"2023-10-09T10:48:02.705503+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:48:02.705503+0000","last_undegraded":"2023-10-09T10:48:02.705503+0000","last_fullsized":"2023-10-09T10:48:02.705503+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":185,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.331634+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:48:00.331634+0000","last_peered":"2023-10-09T10:48:00.331634+0000","last_clean":"2023-10-09T10:48:00.331634+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:48:00.331634+0000","last_undegraded":"2023-10-09T10:48:00.331634+0000","last_fullsized":"2023-10-09T10:48:00.331634+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":119,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.329807+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:48:00.329807+0000","last_peered":"2023-10-09T10:48:00.329807+0000","last_clean":"2023-10-09T10:48:00.329807+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:48:00.329807+0000","last_undegraded":"2023-10-09T10:48:00.329807+0000","last_fullsized":"2023-10-09T10:48:00.329807+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"181'6876","reported_seq":11031,"reported_epoch":181,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:15.909979+0000","last_change":"2023-10-09T10:46:33.521602+0000","last_active":"2023-10-09T10:48:15.909979+0000","last_peered":"2023-10-09T10:48:15.909979+0000","last_clean":"2023-10-09T10:46:31.055751+0000","last_became_active":"2023-10-09T10:46:33.364058+0000","last_became_peered":"2023-10-09T10:46:33.364058+0000","last_unstale":"2023-10-09T10:48:15.909979+0000","last_undegraded":"2023-10-09T10:48:15.909979+0000","last_fullsized":"2023-10-09T10:48:15.909979+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6876,"log_dups_size":0,"ondisk_log_size":6876,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66682880,"num_objects":1018,"num_object_clones":0,"num_object_copies":3054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1018,"num_objects_unfound":0,"num_objects_dirty":1018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5840,"num_write_kb":219424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2010,"num_bytes_recovered":131727360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":1018}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":49,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.325028+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:48:00.325028+0000","last_peered":"2023-10-09T10:48:00.325028+0000","last_clean":"2023-10-09T10:48:00.325028+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:48:00.325028+0000","last_undegraded":"2023-10-09T10:48:00.325028+0000","last_fullsized":"2023-10-09T10:48:00.325028+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":150,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.463748+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:48:06.463748+0000","last_peered":"2023-10-09T10:48:06.463748+0000","last_clean":"2023-10-09T10:48:06.463748+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:48:06.463748+0000","last_undegraded":"2023-10-09T10:48:06.463748+0000","last_fullsized":"2023-10-09T10:48:06.463748+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":514,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.322662+0000","last_change":"2023-10-09T10:46:05.620365+0000","last_active":"2023-10-09T10:48:00.322662+0000","last_peered":"2023-10-09T10:48:00.322662+0000","last_clean":"2023-10-09T10:48:00.322662+0000","last_became_active":"2023-10-09T10:46:05.620024+0000","last_became_peered":"2023-10-09T10:46:05.620024+0000","last_unstale":"2023-10-09T10:48:00.322662+0000","last_undegraded":"2023-10-09T10:48:00.322662+0000","last_fullsized":"2023-10-09T10:48:00.322662+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.13","version":"181'6676","reported_seq":6918,"reported_epoch":181,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:16.004577+0000","last_change":"2023-10-09T10:48:15.511525+0000","last_active":"2023-10-09T10:48:16.004577+0000","last_peered":"2023-10-09T10:48:16.004577+0000","last_clean":"2023-10-09T10:46:31.195699+0000","last_became_active":"2023-10-09T10:46:33.455632+0000","last_became_peered":"2023-10-09T10:46:33.455632+0000","last_unstale":"2023-10-09T10:48:16.004577+0000","last_undegraded":"2023-10-09T10:46:33.269323+0000","last_fullsized":"2023-10-09T10:46:33.241226+0000","mapping_epoch":154,"log_start":"106'1892","ondisk_log_start":"106'1892","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":4784,"log_dups_size":0,"ondisk_log_size":4784,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:07:12.547333+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65273856,"num_objects":996,"num_object_clones":0,"num_object_copies":2988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1767,"num_objects_misplaced":981,"num_objects_unfound":0,"num_objects_dirty":996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3961,"num_write_kb":105088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":1769472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":981},{"shards":"6,7","objects":-1752}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":146,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.329824+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:48:00.329824+0000","last_peered":"2023-10-09T10:48:00.329824+0000","last_clean":"2023-10-09T10:48:00.329824+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:48:00.329824+0000","last_undegraded":"2023-10-09T10:48:00.329824+0000","last_fullsized":"2023-10-09T10:48:00.329824+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":107,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.463494+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:48:06.463494+0000","last_peered":"2023-10-09T10:48:06.463494+0000","last_clean":"2023-10-09T10:48:06.463494+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:48:06.463494+0000","last_undegraded":"2023-10-09T10:48:06.463494+0000","last_fullsized":"2023-10-09T10:48:06.463494+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"182'6805","reported_seq":14934,"reported_epoch":182,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:16.373661+0000","last_change":"2023-10-09T10:46:03.683196+0000","last_active":"2023-10-09T10:48:16.373661+0000","last_peered":"2023-10-09T10:48:16.373661+0000","last_clean":"2023-10-09T10:46:02.177309+0000","last_became_active":"2023-10-09T10:46:03.681058+0000","last_became_peered":"2023-10-09T10:46:03.681058+0000","last_unstale":"2023-10-09T10:48:16.373661+0000","last_undegraded":"2023-10-09T10:48:16.373661+0000","last_fullsized":"2023-10-09T10:48:16.373661+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6805,"log_dups_size":0,"ondisk_log_size":6805,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67403776,"num_objects":1029,"num_object_clones":0,"num_object_copies":3087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1029,"num_objects_unfound":0,"num_objects_dirty":1029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5759,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[1,5,6],"avail_no_missing":["1","2","3","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1029}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":102,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.333413+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:48:00.333413+0000","last_peered":"2023-10-09T10:48:00.333413+0000","last_clean":"2023-10-09T10:48:00.333413+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:48:00.333413+0000","last_undegraded":"2023-10-09T10:48:00.333413+0000","last_fullsized":"2023-10-09T10:48:00.333413+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":143,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.466125+0000","last_change":"2023-10-09T10:46:32.249921+0000","last_active":"2023-10-09T10:48:06.466125+0000","last_peered":"2023-10-09T10:48:06.466125+0000","last_clean":"2023-10-09T10:48:06.466125+0000","last_became_active":"2023-10-09T10:46:32.249590+0000","last_became_peered":"2023-10-09T10:46:32.249590+0000","last_unstale":"2023-10-09T10:48:06.466125+0000","last_undegraded":"2023-10-09T10:48:06.466125+0000","last_fullsized":"2023-10-09T10:48:06.466125+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:25.169752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"181'6609","reported_seq":14833,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:14.512357+0000","last_change":"2023-10-09T10:47:48.412928+0000","last_active":"2023-10-09T10:48:14.512357+0000","last_peered":"2023-10-09T10:48:14.512357+0000","last_clean":"2023-10-09T10:48:14.512357+0000","last_became_active":"2023-10-09T10:47:48.398813+0000","last_became_peered":"2023-10-09T10:47:48.398813+0000","last_unstale":"2023-10-09T10:48:14.512357+0000","last_undegraded":"2023-10-09T10:48:14.512357+0000","last_fullsized":"2023-10-09T10:48:14.512357+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6609,"log_dups_size":0,"ondisk_log_size":6609,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63668224,"num_objects":972,"num_object_clones":0,"num_object_copies":2916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5602,"num_write_kb":210336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2691,"num_bytes_recovered":176357376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":151,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.329596+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:48:00.329596+0000","last_peered":"2023-10-09T10:48:00.329596+0000","last_clean":"2023-10-09T10:48:00.329596+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:48:00.329596+0000","last_undegraded":"2023-10-09T10:48:00.329596+0000","last_fullsized":"2023-10-09T10:48:00.329596+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":145,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.329079+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:48:00.329079+0000","last_peered":"2023-10-09T10:48:00.329079+0000","last_clean":"2023-10-09T10:48:00.329079+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:48:00.329079+0000","last_undegraded":"2023-10-09T10:48:00.329079+0000","last_fullsized":"2023-10-09T10:48:00.329079+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":514,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.338703+0000","last_change":"2023-10-09T10:46:05.618276+0000","last_active":"2023-10-09T10:48:00.338703+0000","last_peered":"2023-10-09T10:48:00.338703+0000","last_clean":"2023-10-09T10:48:00.338703+0000","last_became_active":"2023-10-09T10:46:05.616509+0000","last_became_peered":"2023-10-09T10:46:05.616509+0000","last_unstale":"2023-10-09T10:48:00.338703+0000","last_undegraded":"2023-10-09T10:48:00.338703+0000","last_fullsized":"2023-10-09T10:48:00.338703+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.16","version":"181'6615","reported_seq":6721,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:14.602403+0000","last_change":"2023-10-09T10:46:03.592682+0000","last_active":"2023-10-09T10:48:14.602403+0000","last_peered":"2023-10-09T10:48:14.602403+0000","last_clean":"2023-10-09T10:48:14.602403+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:48:14.602403+0000","last_undegraded":"2023-10-09T10:48:14.602403+0000","last_fullsized":"2023-10-09T10:48:14.602403+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6615,"log_dups_size":0,"ondisk_log_size":6615,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66453504,"num_objects":1014,"num_object_clones":0,"num_object_copies":3042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5572,"num_write_kb":210752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":163,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.345830+0000","last_change":"2023-10-09T10:46:32.250256+0000","last_active":"2023-10-09T10:48:00.345830+0000","last_peered":"2023-10-09T10:48:00.345830+0000","last_clean":"2023-10-09T10:48:00.345830+0000","last_became_active":"2023-10-09T10:46:32.243312+0000","last_became_peered":"2023-10-09T10:46:32.243312+0000","last_unstale":"2023-10-09T10:48:00.345830+0000","last_undegraded":"2023-10-09T10:48:00.345830+0000","last_fullsized":"2023-10-09T10:48:00.345830+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:08.059777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":173,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.462867+0000","last_change":"2023-10-09T10:46:32.237577+0000","last_active":"2023-10-09T10:48:06.462867+0000","last_peered":"2023-10-09T10:48:06.462867+0000","last_clean":"2023-10-09T10:48:06.462867+0000","last_became_active":"2023-10-09T10:46:32.237316+0000","last_became_peered":"2023-10-09T10:46:32.237316+0000","last_unstale":"2023-10-09T10:48:06.462867+0000","last_undegraded":"2023-10-09T10:48:06.462867+0000","last_fullsized":"2023-10-09T10:48:06.462867+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:39.684602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":515,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.320867+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:48:00.320867+0000","last_peered":"2023-10-09T10:48:00.320867+0000","last_clean":"2023-10-09T10:48:00.320867+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:48:00.320867+0000","last_undegraded":"2023-10-09T10:48:00.320867+0000","last_fullsized":"2023-10-09T10:48:00.320867+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.17","version":"181'6632","reported_seq":6732,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:15.869405+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:48:15.869405+0000","last_peered":"2023-10-09T10:48:15.869405+0000","last_clean":"2023-10-09T10:48:15.869405+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:48:15.869405+0000","last_undegraded":"2023-10-09T10:48:15.869405+0000","last_fullsized":"2023-10-09T10:48:15.869405+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6632,"log_dups_size":0,"ondisk_log_size":6632,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57802752,"num_objects":882,"num_object_clones":0,"num_object_copies":2646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":882,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5624,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":49,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.321896+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:48:00.321896+0000","last_peered":"2023-10-09T10:48:00.321896+0000","last_clean":"2023-10-09T10:48:00.321896+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:48:00.321896+0000","last_undegraded":"2023-10-09T10:48:00.321896+0000","last_fullsized":"2023-10-09T10:48:00.321896+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":145,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.333894+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:48:00.333894+0000","last_peered":"2023-10-09T10:48:00.333894+0000","last_clean":"2023-10-09T10:48:00.333894+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:48:00.333894+0000","last_undegraded":"2023-10-09T10:48:00.333894+0000","last_fullsized":"2023-10-09T10:48:00.333894+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":515,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.321945+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:48:00.321945+0000","last_peered":"2023-10-09T10:48:00.321945+0000","last_clean":"2023-10-09T10:48:00.321945+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:48:00.321945+0000","last_undegraded":"2023-10-09T10:48:00.321945+0000","last_fullsized":"2023-10-09T10:48:00.321945+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"182'6734","reported_seq":6840,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.375524+0000","last_change":"2023-10-09T10:46:33.243540+0000","last_active":"2023-10-09T10:48:16.375524+0000","last_peered":"2023-10-09T10:48:16.375524+0000","last_clean":"2023-10-09T10:48:16.375524+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:48:16.375524+0000","last_undegraded":"2023-10-09T10:48:16.375524+0000","last_fullsized":"2023-10-09T10:48:16.375524+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6734,"log_dups_size":0,"ondisk_log_size":6734,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65110016,"num_objects":994,"num_object_clones":0,"num_object_copies":2982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5680,"num_write_kb":213536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":91,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.328719+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:48:00.328719+0000","last_peered":"2023-10-09T10:48:00.328719+0000","last_clean":"2023-10-09T10:48:00.328719+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:48:00.328719+0000","last_undegraded":"2023-10-09T10:48:00.328719+0000","last_fullsized":"2023-10-09T10:48:00.328719+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":49,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.325194+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:48:00.325194+0000","last_peered":"2023-10-09T10:48:00.325194+0000","last_clean":"2023-10-09T10:48:00.325194+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:48:00.325194+0000","last_undegraded":"2023-10-09T10:48:00.325194+0000","last_fullsized":"2023-10-09T10:48:00.325194+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":515,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.321633+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:48:00.321633+0000","last_peered":"2023-10-09T10:48:00.321633+0000","last_clean":"2023-10-09T10:48:00.321633+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:48:00.321633+0000","last_undegraded":"2023-10-09T10:48:00.321633+0000","last_fullsized":"2023-10-09T10:48:00.321633+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"181'6782","reported_seq":6897,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:14.489242+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:48:14.489242+0000","last_peered":"2023-10-09T10:48:14.489242+0000","last_clean":"2023-10-09T10:48:14.489242+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:48:14.489242+0000","last_undegraded":"2023-10-09T10:48:14.489242+0000","last_fullsized":"2023-10-09T10:48:14.489242+0000","mapping_epoch":115,"log_start":"103'179","ondisk_log_start":"103'179","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6603,"log_dups_size":0,"ondisk_log_size":6603,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62062592,"num_objects":947,"num_object_clones":0,"num_object_copies":2841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5721,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":59,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.320353+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:48:00.320353+0000","last_peered":"2023-10-09T10:48:00.320353+0000","last_clean":"2023-10-09T10:48:00.320353+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:48:00.320353+0000","last_undegraded":"2023-10-09T10:48:00.320353+0000","last_fullsized":"2023-10-09T10:48:00.320353+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":149,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.334028+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:48:00.334028+0000","last_peered":"2023-10-09T10:48:00.334028+0000","last_clean":"2023-10-09T10:48:00.334028+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:48:00.334028+0000","last_undegraded":"2023-10-09T10:48:00.334028+0000","last_fullsized":"2023-10-09T10:48:00.334028+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":515,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.326281+0000","last_change":"2023-10-09T10:46:05.620574+0000","last_active":"2023-10-09T10:48:00.326281+0000","last_peered":"2023-10-09T10:48:00.326281+0000","last_clean":"2023-10-09T10:48:00.326281+0000","last_became_active":"2023-10-09T10:46:05.620212+0000","last_became_peered":"2023-10-09T10:46:05.620212+0000","last_unstale":"2023-10-09T10:48:00.326281+0000","last_undegraded":"2023-10-09T10:48:00.326281+0000","last_fullsized":"2023-10-09T10:48:00.326281+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"181'11717","reported_seq":15873,"reported_epoch":181,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:16.085244+0000","last_change":"2023-10-09T10:46:32.247903+0000","last_active":"2023-10-09T10:48:16.085244+0000","last_peered":"2023-10-09T10:48:16.085244+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:46:02.739355+0000","last_became_peered":"2023-10-09T10:46:02.739355+0000","last_unstale":"2023-10-09T10:48:16.085244+0000","last_undegraded":"2023-10-09T10:46:02.608805+0000","last_fullsized":"2023-10-09T10:46:02.583589+0000","mapping_epoch":142,"log_start":"109'2800","ondisk_log_start":"109'2800","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8917,"log_dups_size":0,"ondisk_log_size":8917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:32.899634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125501440,"num_objects":1915,"num_object_clones":0,"num_object_copies":5745,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1915,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11717,"num_write_kb":436224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2022,"num_bytes_recovered":132513792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":625},{"shards":"5,6","objects":1289}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":174,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.344341+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:48:00.344341+0000","last_peered":"2023-10-09T10:48:00.344341+0000","last_clean":"2023-10-09T10:48:00.344341+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:48:00.344341+0000","last_undegraded":"2023-10-09T10:48:00.344341+0000","last_fullsized":"2023-10-09T10:48:00.344341+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":128,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.333775+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:48:00.333775+0000","last_peered":"2023-10-09T10:48:00.333775+0000","last_clean":"2023-10-09T10:48:00.333775+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:48:00.333775+0000","last_undegraded":"2023-10-09T10:48:00.333775+0000","last_fullsized":"2023-10-09T10:48:00.333775+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":495,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.322792+0000","last_change":"2023-10-09T10:46:02.631378+0000","last_active":"2023-10-09T10:48:00.322792+0000","last_peered":"2023-10-09T10:48:00.322792+0000","last_clean":"2023-10-09T10:48:00.322792+0000","last_became_active":"2023-10-09T10:46:02.629142+0000","last_became_peered":"2023-10-09T10:46:02.629142+0000","last_unstale":"2023-10-09T10:48:00.322792+0000","last_undegraded":"2023-10-09T10:48:00.322792+0000","last_fullsized":"2023-10-09T10:48:00.322792+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.b","version":"181'11184","reported_seq":11277,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:14.594364+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:48:14.594364+0000","last_peered":"2023-10-09T10:48:14.594364+0000","last_clean":"2023-10-09T10:48:14.594364+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:48:14.594364+0000","last_undegraded":"2023-10-09T10:48:14.594364+0000","last_fullsized":"2023-10-09T10:48:14.594364+0000","mapping_epoch":98,"log_start":"135'5425","ondisk_log_start":"135'5425","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5759,"log_dups_size":0,"ondisk_log_size":5759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130809856,"num_objects":1996,"num_object_clones":0,"num_object_copies":5988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11184,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":147,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.344511+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:48:00.344511+0000","last_peered":"2023-10-09T10:48:00.344511+0000","last_clean":"2023-10-09T10:48:00.344511+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:48:00.344511+0000","last_undegraded":"2023-10-09T10:48:00.344511+0000","last_fullsized":"2023-10-09T10:48:00.344511+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":182,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.328659+0000","last_change":"2023-10-09T10:46:32.251035+0000","last_active":"2023-10-09T10:48:00.328659+0000","last_peered":"2023-10-09T10:48:00.328659+0000","last_clean":"2023-10-09T10:48:00.328659+0000","last_became_active":"2023-10-09T10:46:32.250293+0000","last_became_peered":"2023-10-09T10:46:32.250293+0000","last_unstale":"2023-10-09T10:48:00.328659+0000","last_undegraded":"2023-10-09T10:48:00.328659+0000","last_fullsized":"2023-10-09T10:48:00.328659+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.787766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":515,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.324775+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:48:00.324775+0000","last_peered":"2023-10-09T10:48:00.324775+0000","last_clean":"2023-10-09T10:48:00.324775+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:48:00.324775+0000","last_undegraded":"2023-10-09T10:48:00.324775+0000","last_fullsized":"2023-10-09T10:48:00.324775+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"181'6748","reported_seq":6861,"reported_epoch":181,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:14.510324+0000","last_change":"2023-10-09T10:46:33.540719+0000","last_active":"2023-10-09T10:48:14.510324+0000","last_peered":"2023-10-09T10:48:14.510324+0000","last_clean":"2023-10-09T10:46:31.137696+0000","last_became_active":"2023-10-09T10:46:33.527650+0000","last_became_peered":"2023-10-09T10:46:33.527650+0000","last_unstale":"2023-10-09T10:48:14.510324+0000","last_undegraded":"2023-10-09T10:46:33.285468+0000","last_fullsized":"2023-10-09T10:46:33.235683+0000","mapping_epoch":154,"log_start":"101'19","ondisk_log_start":"101'19","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6729,"log_dups_size":0,"ondisk_log_size":6729,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65765376,"num_objects":1003,"num_object_clones":0,"num_object_copies":3009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5683,"num_write_kb":213984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":991},{"shards":"3,7","objects":-1806}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":102,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.333834+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:48:00.333834+0000","last_peered":"2023-10-09T10:48:00.333834+0000","last_clean":"2023-10-09T10:48:00.333834+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:48:00.333834+0000","last_undegraded":"2023-10-09T10:48:00.333834+0000","last_fullsized":"2023-10-09T10:48:00.333834+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":120,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.466499+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:48:06.466499+0000","last_peered":"2023-10-09T10:48:06.466499+0000","last_clean":"2023-10-09T10:48:06.466499+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:48:06.466499+0000","last_undegraded":"2023-10-09T10:48:06.466499+0000","last_fullsized":"2023-10-09T10:48:06.466499+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":515,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.321809+0000","last_change":"2023-10-09T10:46:05.620030+0000","last_active":"2023-10-09T10:48:00.321809+0000","last_peered":"2023-10-09T10:48:00.321809+0000","last_clean":"2023-10-09T10:48:00.321809+0000","last_became_active":"2023-10-09T10:46:05.619694+0000","last_became_peered":"2023-10-09T10:46:05.619694+0000","last_unstale":"2023-10-09T10:48:00.321809+0000","last_undegraded":"2023-10-09T10:48:00.321809+0000","last_fullsized":"2023-10-09T10:48:00.321809+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.9","version":"181'6639","reported_seq":6763,"reported_epoch":181,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:12.691594+0000","last_change":"2023-10-09T10:46:34.405499+0000","last_active":"2023-10-09T10:48:12.691594+0000","last_peered":"2023-10-09T10:48:12.691594+0000","last_clean":"2023-10-09T10:46:31.190603+0000","last_became_active":"2023-10-09T10:46:34.396448+0000","last_became_peered":"2023-10-09T10:46:34.396448+0000","last_unstale":"2023-10-09T10:48:12.691594+0000","last_undegraded":"2023-10-09T10:46:33.246949+0000","last_fullsized":"2023-10-09T10:46:33.237968+0000","mapping_epoch":154,"log_start":"103'823","ondisk_log_start":"103'823","created":98,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5816,"log_dups_size":0,"ondisk_log_size":5816,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66650112,"num_objects":1017,"num_object_clones":0,"num_object_copies":3051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2004,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5597,"num_write_kb":211648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1303,"num_bytes_recovered":85393408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":1},{"shards":"0,7","objects":-451}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":146,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.333750+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:48:00.333750+0000","last_peered":"2023-10-09T10:48:00.333750+0000","last_clean":"2023-10-09T10:48:00.333750+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:48:00.333750+0000","last_undegraded":"2023-10-09T10:48:00.333750+0000","last_fullsized":"2023-10-09T10:48:00.333750+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":49,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.322373+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:48:00.322373+0000","last_peered":"2023-10-09T10:48:00.322373+0000","last_clean":"2023-10-09T10:48:00.322373+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:48:00.322373+0000","last_undegraded":"2023-10-09T10:48:00.322373+0000","last_fullsized":"2023-10-09T10:48:00.322373+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":507,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.491188+0000","last_change":"2023-10-09T10:46:05.618134+0000","last_active":"2023-10-09T10:48:06.491188+0000","last_peered":"2023-10-09T10:48:06.491188+0000","last_clean":"2023-10-09T10:48:06.491188+0000","last_became_active":"2023-10-09T10:46:05.616057+0000","last_became_peered":"2023-10-09T10:46:05.616057+0000","last_unstale":"2023-10-09T10:48:06.491188+0000","last_undegraded":"2023-10-09T10:48:06.491188+0000","last_fullsized":"2023-10-09T10:48:06.491188+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"181'11231","reported_seq":11332,"reported_epoch":181,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:16.050779+0000","last_change":"2023-10-09T10:46:33.576530+0000","last_active":"2023-10-09T10:48:16.050779+0000","last_peered":"2023-10-09T10:48:16.050779+0000","last_clean":"2023-10-09T10:46:31.183481+0000","last_became_active":"2023-10-09T10:46:33.542304+0000","last_became_peered":"2023-10-09T10:46:33.542304+0000","last_unstale":"2023-10-09T10:48:16.050779+0000","last_undegraded":"2023-10-09T10:46:33.300581+0000","last_fullsized":"2023-10-09T10:46:33.246047+0000","mapping_epoch":154,"log_start":"120'3877","ondisk_log_start":"120'3877","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7354,"log_dups_size":0,"ondisk_log_size":7354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:26.739488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131268608,"num_objects":2003,"num_object_clones":0,"num_object_copies":6009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6617,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11231,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":2003},{"shards":"1,4","objects":-3558}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":174,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.336033+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:48:00.336033+0000","last_peered":"2023-10-09T10:48:00.336033+0000","last_clean":"2023-10-09T10:48:00.336033+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:48:00.336033+0000","last_undegraded":"2023-10-09T10:48:00.336033+0000","last_fullsized":"2023-10-09T10:48:00.336033+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":145,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.329518+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:48:00.329518+0000","last_peered":"2023-10-09T10:48:00.329518+0000","last_clean":"2023-10-09T10:48:00.329518+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:48:00.329518+0000","last_undegraded":"2023-10-09T10:48:00.329518+0000","last_fullsized":"2023-10-09T10:48:00.329518+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":496,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.485086+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:48:06.485086+0000","last_peered":"2023-10-09T10:48:06.485086+0000","last_clean":"2023-10-09T10:48:06.485086+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:48:06.485086+0000","last_undegraded":"2023-10-09T10:48:06.485086+0000","last_fullsized":"2023-10-09T10:48:06.485086+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.f","version":"181'11013","reported_seq":11112,"reported_epoch":181,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:14.587456+0000","last_change":"2023-10-09T10:46:33.651756+0000","last_active":"2023-10-09T10:48:14.587456+0000","last_peered":"2023-10-09T10:48:14.587456+0000","last_clean":"2023-10-09T10:46:31.156458+0000","last_became_active":"2023-10-09T10:46:33.621003+0000","last_became_peered":"2023-10-09T10:46:33.621003+0000","last_unstale":"2023-10-09T10:48:14.587456+0000","last_undegraded":"2023-10-09T10:46:33.309813+0000","last_fullsized":"2023-10-09T10:46:33.236508+0000","mapping_epoch":154,"log_start":"135'5310","ondisk_log_start":"135'5310","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5703,"log_dups_size":0,"ondisk_log_size":5703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:13.704346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131661824,"num_objects":2009,"num_object_clones":0,"num_object_copies":6027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11013,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":2009},{"shards":"3,4","objects":-3443}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":120,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.330018+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:48:00.330018+0000","last_peered":"2023-10-09T10:48:00.330018+0000","last_clean":"2023-10-09T10:48:00.330018+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:48:00.330018+0000","last_undegraded":"2023-10-09T10:48:00.330018+0000","last_fullsized":"2023-10-09T10:48:00.330018+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":172,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.329879+0000","last_change":"2023-10-09T10:46:32.245583+0000","last_active":"2023-10-09T10:48:00.329879+0000","last_peered":"2023-10-09T10:48:00.329879+0000","last_clean":"2023-10-09T10:48:00.329879+0000","last_became_active":"2023-10-09T10:46:32.245271+0000","last_became_peered":"2023-10-09T10:46:32.245271+0000","last_unstale":"2023-10-09T10:48:00.329879+0000","last_undegraded":"2023-10-09T10:48:00.329879+0000","last_fullsized":"2023-10-09T10:48:00.329879+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.025051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":515,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.322125+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:48:00.322125+0000","last_peered":"2023-10-09T10:48:00.322125+0000","last_clean":"2023-10-09T10:48:00.322125+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:48:00.322125+0000","last_undegraded":"2023-10-09T10:48:00.322125+0000","last_fullsized":"2023-10-09T10:48:00.322125+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"181'11323","reported_seq":11416,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:14.571296+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:48:14.571296+0000","last_peered":"2023-10-09T10:48:14.571296+0000","last_clean":"2023-10-09T10:48:14.571296+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:48:14.571296+0000","last_undegraded":"2023-10-09T10:48:14.571296+0000","last_fullsized":"2023-10-09T10:48:14.571296+0000","mapping_epoch":98,"log_start":"136'5578","ondisk_log_start":"136'5578","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128516096,"num_objects":1961,"num_object_clones":0,"num_object_copies":5883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11323,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":156,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.328603+0000","last_change":"2023-10-09T10:46:32.245685+0000","last_active":"2023-10-09T10:48:00.328603+0000","last_peered":"2023-10-09T10:48:00.328603+0000","last_clean":"2023-10-09T10:48:00.328603+0000","last_became_active":"2023-10-09T10:46:32.245314+0000","last_became_peered":"2023-10-09T10:46:32.245314+0000","last_unstale":"2023-10-09T10:48:00.328603+0000","last_undegraded":"2023-10-09T10:48:00.328603+0000","last_fullsized":"2023-10-09T10:48:00.328603+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:51.716860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":145,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.330417+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:48:00.330417+0000","last_peered":"2023-10-09T10:48:00.330417+0000","last_clean":"2023-10-09T10:48:00.330417+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:48:00.330417+0000","last_undegraded":"2023-10-09T10:48:00.330417+0000","last_fullsized":"2023-10-09T10:48:00.330417+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":507,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.488882+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:48:06.488882+0000","last_peered":"2023-10-09T10:48:06.488882+0000","last_clean":"2023-10-09T10:48:06.488882+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:48:06.488882+0000","last_undegraded":"2023-10-09T10:48:06.488882+0000","last_fullsized":"2023-10-09T10:48:06.488882+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"181'11434","reported_seq":11545,"reported_epoch":181,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:16.083648+0000","last_change":"2023-10-09T10:46:02.754507+0000","last_active":"2023-10-09T10:48:16.083648+0000","last_peered":"2023-10-09T10:48:16.083648+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:46:02.747510+0000","last_became_peered":"2023-10-09T10:46:02.747510+0000","last_unstale":"2023-10-09T10:48:16.083648+0000","last_undegraded":"2023-10-09T10:46:02.605987+0000","last_fullsized":"2023-10-09T10:46:02.585248+0000","mapping_epoch":142,"log_start":"108'2600","ondisk_log_start":"108'2600","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8834,"log_dups_size":0,"ondisk_log_size":8834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:59.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128516120,"num_objects":1962,"num_object_clones":0,"num_object_copies":5886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1962,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11434,"num_write_kb":428609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,2,5","objects":1961},{"shards":"1,5","objects":-756}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":151,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.331541+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:48:00.331541+0000","last_peered":"2023-10-09T10:48:00.331541+0000","last_clean":"2023-10-09T10:48:00.331541+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:48:00.331541+0000","last_undegraded":"2023-10-09T10:48:00.331541+0000","last_fullsized":"2023-10-09T10:48:00.331541+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":134,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.466934+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:48:06.466934+0000","last_peered":"2023-10-09T10:48:06.466934+0000","last_clean":"2023-10-09T10:48:06.466934+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:48:06.466934+0000","last_undegraded":"2023-10-09T10:48:06.466934+0000","last_fullsized":"2023-10-09T10:48:06.466934+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":516,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.487294+0000","last_change":"2023-10-09T10:46:05.617983+0000","last_active":"2023-10-09T10:48:06.487294+0000","last_peered":"2023-10-09T10:48:06.487294+0000","last_clean":"2023-10-09T10:48:06.487294+0000","last_became_active":"2023-10-09T10:46:05.616021+0000","last_became_peered":"2023-10-09T10:46:05.616021+0000","last_unstale":"2023-10-09T10:48:06.487294+0000","last_undegraded":"2023-10-09T10:48:06.487294+0000","last_fullsized":"2023-10-09T10:48:06.487294+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":177,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.466964+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:48:06.466964+0000","last_peered":"2023-10-09T10:48:06.466964+0000","last_clean":"2023-10-09T10:48:06.466964+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:48:06.466964+0000","last_undegraded":"2023-10-09T10:48:06.466964+0000","last_fullsized":"2023-10-09T10:48:06.466964+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":173,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.333977+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:48:00.333977+0000","last_peered":"2023-10-09T10:48:00.333977+0000","last_clean":"2023-10-09T10:48:00.333977+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:48:00.333977+0000","last_undegraded":"2023-10-09T10:48:00.333977+0000","last_fullsized":"2023-10-09T10:48:00.333977+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":119,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.466200+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:48:06.466200+0000","last_peered":"2023-10-09T10:48:06.466200+0000","last_clean":"2023-10-09T10:48:06.466200+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:48:06.466200+0000","last_undegraded":"2023-10-09T10:48:06.466200+0000","last_fullsized":"2023-10-09T10:48:06.466200+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"181'6651","reported_seq":10689,"reported_epoch":181,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:14.642672+0000","last_change":"2023-10-09T10:46:32.248218+0000","last_active":"2023-10-09T10:48:14.642672+0000","last_peered":"2023-10-09T10:48:14.642672+0000","last_clean":"2023-10-09T10:46:02.181912+0000","last_became_active":"2023-10-09T10:46:32.226237+0000","last_became_peered":"2023-10-09T10:46:32.226237+0000","last_unstale":"2023-10-09T10:48:14.642672+0000","last_undegraded":"2023-10-09T10:48:14.642672+0000","last_fullsized":"2023-10-09T10:48:14.642672+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6651,"log_dups_size":0,"ondisk_log_size":6651,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68190208,"num_objects":1040,"num_object_clones":0,"num_object_copies":3120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1040,"num_objects_unfound":0,"num_objects_dirty":1040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5604,"num_write_kb":212640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1040}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":515,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.485614+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:48:06.485614+0000","last_peered":"2023-10-09T10:48:06.485614+0000","last_clean":"2023-10-09T10:48:06.485614+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:48:06.485614+0000","last_undegraded":"2023-10-09T10:48:06.485614+0000","last_fullsized":"2023-10-09T10:48:06.485614+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"181'6757","reported_seq":6940,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:14.576053+0000","last_change":"2023-10-09T10:46:02.760963+0000","last_active":"2023-10-09T10:48:14.576053+0000","last_peered":"2023-10-09T10:48:14.576053+0000","last_clean":"2023-10-09T10:48:14.576053+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:48:14.576053+0000","last_undegraded":"2023-10-09T10:48:14.576053+0000","last_fullsized":"2023-10-09T10:48:14.576053+0000","mapping_epoch":98,"log_start":"103'124","ondisk_log_start":"103'124","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"141'4416","last_scrub_stamp":"2023-10-09T10:46:02.760810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:46:02.760810+0000","objects_scrubbed":3378,"log_size":6633,"log_dups_size":0,"ondisk_log_size":6633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:35.863002+0000","scrub_duration":0.47769323899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67960832,"num_objects":1037,"num_object_clones":0,"num_object_copies":3111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2341,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":150,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.331637+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:48:00.331637+0000","last_peered":"2023-10-09T10:48:00.331637+0000","last_clean":"2023-10-09T10:48:00.331637+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:48:00.331637+0000","last_undegraded":"2023-10-09T10:48:00.331637+0000","last_fullsized":"2023-10-09T10:48:00.331637+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":166,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.329208+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:48:00.329208+0000","last_peered":"2023-10-09T10:48:00.329208+0000","last_clean":"2023-10-09T10:48:00.329208+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:48:00.329208+0000","last_undegraded":"2023-10-09T10:48:00.329208+0000","last_fullsized":"2023-10-09T10:48:00.329208+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":158,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.467627+0000","last_change":"2023-10-09T10:46:32.245608+0000","last_active":"2023-10-09T10:48:06.467627+0000","last_peered":"2023-10-09T10:48:06.467627+0000","last_clean":"2023-10-09T10:48:06.467627+0000","last_became_active":"2023-10-09T10:46:32.245290+0000","last_became_peered":"2023-10-09T10:46:32.245290+0000","last_unstale":"2023-10-09T10:48:06.467627+0000","last_undegraded":"2023-10-09T10:48:06.467627+0000","last_fullsized":"2023-10-09T10:48:06.467627+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:48.832515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":516,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.488679+0000","last_change":"2023-10-09T10:46:05.620057+0000","last_active":"2023-10-09T10:48:06.488679+0000","last_peered":"2023-10-09T10:48:06.488679+0000","last_clean":"2023-10-09T10:48:06.488679+0000","last_became_active":"2023-10-09T10:46:05.619703+0000","last_became_peered":"2023-10-09T10:46:05.619703+0000","last_unstale":"2023-10-09T10:48:06.488679+0000","last_undegraded":"2023-10-09T10:48:06.488679+0000","last_fullsized":"2023-10-09T10:48:06.488679+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"181'6690","reported_seq":6808,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:14.531894+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:48:14.531894+0000","last_peered":"2023-10-09T10:48:14.531894+0000","last_clean":"2023-10-09T10:48:14.531894+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:48:14.531894+0000","last_undegraded":"2023-10-09T10:48:14.531894+0000","last_fullsized":"2023-10-09T10:48:14.531894+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6690,"log_dups_size":0,"ondisk_log_size":6690,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68878336,"num_objects":1051,"num_object_clones":0,"num_object_copies":3153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5629,"num_write_kb":213760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":178,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.334238+0000","last_change":"2023-10-09T10:46:32.241041+0000","last_active":"2023-10-09T10:48:00.334238+0000","last_peered":"2023-10-09T10:48:00.334238+0000","last_clean":"2023-10-09T10:48:00.334238+0000","last_became_active":"2023-10-09T10:46:32.238683+0000","last_became_peered":"2023-10-09T10:46:32.238683+0000","last_unstale":"2023-10-09T10:48:00.334238+0000","last_undegraded":"2023-10-09T10:48:00.334238+0000","last_fullsized":"2023-10-09T10:48:00.334238+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:25.817713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":143,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.466629+0000","last_change":"2023-10-09T10:46:32.250104+0000","last_active":"2023-10-09T10:48:06.466629+0000","last_peered":"2023-10-09T10:48:06.466629+0000","last_clean":"2023-10-09T10:48:06.466629+0000","last_became_active":"2023-10-09T10:46:32.249805+0000","last_became_peered":"2023-10-09T10:46:32.249805+0000","last_unstale":"2023-10-09T10:48:06.466629+0000","last_undegraded":"2023-10-09T10:48:06.466629+0000","last_fullsized":"2023-10-09T10:48:06.466629+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:07.230672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":172,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.467515+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:48:06.467515+0000","last_peered":"2023-10-09T10:48:06.467515+0000","last_clean":"2023-10-09T10:48:06.467515+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:48:06.467515+0000","last_undegraded":"2023-10-09T10:48:06.467515+0000","last_fullsized":"2023-10-09T10:48:06.467515+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"181'6635","reported_seq":6739,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.068260+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:48:16.068260+0000","last_peered":"2023-10-09T10:48:16.068260+0000","last_clean":"2023-10-09T10:48:16.068260+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:48:16.068260+0000","last_undegraded":"2023-10-09T10:48:16.068260+0000","last_fullsized":"2023-10-09T10:48:16.068260+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6635,"log_dups_size":0,"ondisk_log_size":6635,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67043328,"num_objects":1023,"num_object_clones":0,"num_object_copies":3069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5627,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":160,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.330429+0000","last_change":"2023-10-09T10:46:32.235346+0000","last_active":"2023-10-09T10:48:00.330429+0000","last_peered":"2023-10-09T10:48:00.330429+0000","last_clean":"2023-10-09T10:48:00.330429+0000","last_became_active":"2023-10-09T10:46:32.234906+0000","last_became_peered":"2023-10-09T10:46:32.234906+0000","last_unstale":"2023-10-09T10:48:00.330429+0000","last_undegraded":"2023-10-09T10:48:00.330429+0000","last_fullsized":"2023-10-09T10:48:00.330429+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.199251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":516,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:02.704582+0000","last_change":"2023-10-09T10:46:05.617146+0000","last_active":"2023-10-09T10:48:02.704582+0000","last_peered":"2023-10-09T10:48:02.704582+0000","last_clean":"2023-10-09T10:48:02.704582+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:48:02.704582+0000","last_undegraded":"2023-10-09T10:48:02.704582+0000","last_fullsized":"2023-10-09T10:48:02.704582+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":109,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.333537+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:48:00.333537+0000","last_peered":"2023-10-09T10:48:00.333537+0000","last_clean":"2023-10-09T10:48:00.333537+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:48:00.333537+0000","last_undegraded":"2023-10-09T10:48:00.333537+0000","last_fullsized":"2023-10-09T10:48:00.333537+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":78,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.329713+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:48:00.329713+0000","last_peered":"2023-10-09T10:48:00.329713+0000","last_clean":"2023-10-09T10:48:00.329713+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:48:00.329713+0000","last_undegraded":"2023-10-09T10:48:00.329713+0000","last_fullsized":"2023-10-09T10:48:00.329713+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":516,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.485825+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:48:06.485825+0000","last_peered":"2023-10-09T10:48:06.485825+0000","last_clean":"2023-10-09T10:48:06.485825+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:48:06.485825+0000","last_undegraded":"2023-10-09T10:48:06.485825+0000","last_fullsized":"2023-10-09T10:48:06.485825+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":173,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.466776+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:48:06.466776+0000","last_peered":"2023-10-09T10:48:06.466776+0000","last_clean":"2023-10-09T10:48:06.466776+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:48:06.466776+0000","last_undegraded":"2023-10-09T10:48:06.466776+0000","last_fullsized":"2023-10-09T10:48:06.466776+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":59,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.485509+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:48:06.485509+0000","last_peered":"2023-10-09T10:48:06.485509+0000","last_clean":"2023-10-09T10:48:06.485509+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:48:06.485509+0000","last_undegraded":"2023-10-09T10:48:06.485509+0000","last_fullsized":"2023-10-09T10:48:06.485509+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":59,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.486888+0000","last_change":"2023-10-09T10:46:32.236202+0000","last_active":"2023-10-09T10:48:06.486888+0000","last_peered":"2023-10-09T10:48:06.486888+0000","last_clean":"2023-10-09T10:48:06.486888+0000","last_became_active":"2023-10-09T10:46:32.235901+0000","last_became_peered":"2023-10-09T10:46:32.235901+0000","last_unstale":"2023-10-09T10:48:06.486888+0000","last_undegraded":"2023-10-09T10:48:06.486888+0000","last_fullsized":"2023-10-09T10:48:06.486888+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:48.260403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"182'6758","reported_seq":6911,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.375592+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:48:16.375592+0000","last_peered":"2023-10-09T10:48:16.375592+0000","last_clean":"2023-10-09T10:48:16.375592+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:48:16.375592+0000","last_undegraded":"2023-10-09T10:48:16.375592+0000","last_fullsized":"2023-10-09T10:48:16.375592+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":6758,"log_dups_size":0,"ondisk_log_size":6758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66846720,"num_objects":1020,"num_object_clones":0,"num_object_copies":3060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3728,"num_write_kb":89664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":516,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.486684+0000","last_change":"2023-10-09T10:46:05.620764+0000","last_active":"2023-10-09T10:48:06.486684+0000","last_peered":"2023-10-09T10:48:06.486684+0000","last_clean":"2023-10-09T10:48:06.486684+0000","last_became_active":"2023-10-09T10:46:05.619870+0000","last_became_peered":"2023-10-09T10:46:05.619870+0000","last_unstale":"2023-10-09T10:48:06.486684+0000","last_undegraded":"2023-10-09T10:48:06.486684+0000","last_fullsized":"2023-10-09T10:48:06.486684+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":169,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.344991+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:48:00.344991+0000","last_peered":"2023-10-09T10:48:00.344991+0000","last_clean":"2023-10-09T10:48:00.344991+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:48:00.344991+0000","last_undegraded":"2023-10-09T10:48:00.344991+0000","last_fullsized":"2023-10-09T10:48:00.344991+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":143,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.467648+0000","last_change":"2023-10-09T10:46:32.251450+0000","last_active":"2023-10-09T10:48:06.467648+0000","last_peered":"2023-10-09T10:48:06.467648+0000","last_clean":"2023-10-09T10:48:06.467648+0000","last_became_active":"2023-10-09T10:46:32.250933+0000","last_became_peered":"2023-10-09T10:46:32.250933+0000","last_unstale":"2023-10-09T10:48:06.467648+0000","last_undegraded":"2023-10-09T10:48:06.467648+0000","last_fullsized":"2023-10-09T10:48:06.467648+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:04.116178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":50,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.485971+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:48:06.485971+0000","last_peered":"2023-10-09T10:48:06.485971+0000","last_clean":"2023-10-09T10:48:06.485971+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:48:06.485971+0000","last_undegraded":"2023-10-09T10:48:06.485971+0000","last_fullsized":"2023-10-09T10:48:06.485971+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"182'6593","reported_seq":13291,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.479754+0000","last_change":"2023-10-09T10:47:02.269289+0000","last_active":"2023-10-09T10:48:16.479754+0000","last_peered":"2023-10-09T10:48:16.479754+0000","last_clean":"2023-10-09T10:48:16.479754+0000","last_became_active":"2023-10-09T10:47:02.257854+0000","last_became_peered":"2023-10-09T10:47:02.257854+0000","last_unstale":"2023-10-09T10:48:16.479754+0000","last_undegraded":"2023-10-09T10:48:16.479754+0000","last_fullsized":"2023-10-09T10:48:16.479754+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6593,"log_dups_size":0,"ondisk_log_size":6593,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66682880,"num_objects":1017,"num_object_clones":0,"num_object_copies":3051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5585,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3287,"num_bytes_recovered":181927936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":516,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.488439+0000","last_change":"2023-10-09T10:46:05.618443+0000","last_active":"2023-10-09T10:48:06.488439+0000","last_peered":"2023-10-09T10:48:06.488439+0000","last_clean":"2023-10-09T10:48:06.488439+0000","last_became_active":"2023-10-09T10:46:05.616588+0000","last_became_peered":"2023-10-09T10:46:05.616588+0000","last_unstale":"2023-10-09T10:48:06.488439+0000","last_undegraded":"2023-10-09T10:48:06.488439+0000","last_fullsized":"2023-10-09T10:48:06.488439+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":174,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.333954+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:48:00.333954+0000","last_peered":"2023-10-09T10:48:00.333954+0000","last_clean":"2023-10-09T10:48:00.333954+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:48:00.333954+0000","last_undegraded":"2023-10-09T10:48:00.333954+0000","last_fullsized":"2023-10-09T10:48:00.333954+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":119,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.466934+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:48:06.466934+0000","last_peered":"2023-10-09T10:48:06.466934+0000","last_clean":"2023-10-09T10:48:06.466934+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:48:06.466934+0000","last_undegraded":"2023-10-09T10:48:06.466934+0000","last_fullsized":"2023-10-09T10:48:06.466934+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":163,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.331589+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:48:00.331589+0000","last_peered":"2023-10-09T10:48:00.331589+0000","last_clean":"2023-10-09T10:48:00.331589+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:48:00.331589+0000","last_undegraded":"2023-10-09T10:48:00.331589+0000","last_fullsized":"2023-10-09T10:48:00.331589+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"181'6728","reported_seq":13555,"reported_epoch":181,"state":"active+undersized+remapped","last_fresh":"2023-10-09T10:48:16.032297+0000","last_change":"2023-10-09T10:48:15.511163+0000","last_active":"2023-10-09T10:48:16.032297+0000","last_peered":"2023-10-09T10:48:16.032297+0000","last_clean":"2023-10-09T10:46:31.130083+0000","last_became_active":"2023-10-09T10:46:33.493288+0000","last_became_peered":"2023-10-09T10:46:33.493288+0000","last_unstale":"2023-10-09T10:48:16.032297+0000","last_undegraded":"2023-10-09T10:48:16.032297+0000","last_fullsized":"2023-10-09T10:46:33.244043+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6728,"log_dups_size":0,"ondisk_log_size":6728,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63799296,"num_objects":973,"num_object_clones":0,"num_object_copies":2919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5691,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3337,"num_bytes_recovered":129564672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3],"avail_no_missing":["4","2","3","6"],"object_location_counts":[{"shards":"3,4","objects":973}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":515,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.486268+0000","last_change":"2023-10-09T10:46:05.617589+0000","last_active":"2023-10-09T10:48:06.486268+0000","last_peered":"2023-10-09T10:48:06.486268+0000","last_clean":"2023-10-09T10:48:06.486268+0000","last_became_active":"2023-10-09T10:46:05.616400+0000","last_became_peered":"2023-10-09T10:46:05.616400+0000","last_unstale":"2023-10-09T10:48:06.486268+0000","last_undegraded":"2023-10-09T10:48:06.486268+0000","last_fullsized":"2023-10-09T10:48:06.486268+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":38,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.333709+0000","last_change":"2023-10-09T10:46:32.245106+0000","last_active":"2023-10-09T10:48:00.333709+0000","last_peered":"2023-10-09T10:48:00.333709+0000","last_clean":"2023-10-09T10:48:00.333709+0000","last_became_active":"2023-10-09T10:46:32.244826+0000","last_became_peered":"2023-10-09T10:46:32.244826+0000","last_unstale":"2023-10-09T10:48:00.333709+0000","last_undegraded":"2023-10-09T10:48:00.333709+0000","last_fullsized":"2023-10-09T10:48:00.333709+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:30.322834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":54,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.486823+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:48:06.486823+0000","last_peered":"2023-10-09T10:48:06.486823+0000","last_clean":"2023-10-09T10:48:06.486823+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:48:06.486823+0000","last_undegraded":"2023-10-09T10:48:06.486823+0000","last_fullsized":"2023-10-09T10:48:06.486823+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":128,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.328534+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:48:00.328534+0000","last_peered":"2023-10-09T10:48:00.328534+0000","last_clean":"2023-10-09T10:48:00.328534+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:48:00.328534+0000","last_undegraded":"2023-10-09T10:48:00.328534+0000","last_fullsized":"2023-10-09T10:48:00.328534+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"181'6643","reported_seq":13453,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:15.995895+0000","last_change":"2023-10-09T10:47:31.271196+0000","last_active":"2023-10-09T10:48:15.995895+0000","last_peered":"2023-10-09T10:48:15.995895+0000","last_clean":"2023-10-09T10:48:15.995895+0000","last_became_active":"2023-10-09T10:47:31.229504+0000","last_became_peered":"2023-10-09T10:47:31.229504+0000","last_unstale":"2023-10-09T10:48:15.995895+0000","last_undegraded":"2023-10-09T10:48:15.995895+0000","last_fullsized":"2023-10-09T10:48:15.995895+0000","mapping_epoch":172,"log_start":"106'1853","ondisk_log_start":"106'1853","created":98,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":4790,"log_dups_size":0,"ondisk_log_size":4790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:26.533476+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64421888,"num_objects":983,"num_object_clones":0,"num_object_copies":2949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3998,"num_write_kb":107264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3305,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":49,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.322299+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:48:00.322299+0000","last_peered":"2023-10-09T10:48:00.322299+0000","last_clean":"2023-10-09T10:48:00.322299+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:48:00.322299+0000","last_undegraded":"2023-10-09T10:48:00.322299+0000","last_fullsized":"2023-10-09T10:48:00.322299+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":113,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.467719+0000","last_change":"2023-10-09T10:46:32.251242+0000","last_active":"2023-10-09T10:48:06.467719+0000","last_peered":"2023-10-09T10:48:06.467719+0000","last_clean":"2023-10-09T10:48:06.467719+0000","last_became_active":"2023-10-09T10:46:32.250534+0000","last_became_peered":"2023-10-09T10:46:32.250534+0000","last_unstale":"2023-10-09T10:48:06.467719+0000","last_undegraded":"2023-10-09T10:48:06.467719+0000","last_fullsized":"2023-10-09T10:48:06.467719+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:33.726041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"181'6593","reported_seq":11145,"reported_epoch":181,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:12.754477+0000","last_change":"2023-10-09T10:46:34.514479+0000","last_active":"2023-10-09T10:48:12.754477+0000","last_peered":"2023-10-09T10:48:12.754477+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:46:34.409667+0000","last_became_peered":"2023-10-09T10:46:34.409667+0000","last_unstale":"2023-10-09T10:48:12.754477+0000","last_undegraded":"2023-10-09T10:46:34.376635+0000","last_fullsized":"2023-10-09T10:46:33.237998+0000","mapping_epoch":154,"log_start":"103'773","ondisk_log_start":"103'773","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5820,"log_dups_size":0,"ondisk_log_size":5820,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73596928,"num_objects":1123,"num_object_clones":0,"num_object_copies":3369,"num_objects_missing_on_primary":1121,"num_objects_missing":1121,"num_objects_degraded":1631,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5551,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2214,"num_bytes_recovered":145096704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3,7","objects":3},{"shards":"0,7","objects":-1},{"shards":"2,3,7","objects":1121}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":148,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.333875+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:48:00.333875+0000","last_peered":"2023-10-09T10:48:00.333875+0000","last_clean":"2023-10-09T10:48:00.333875+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:48:00.333875+0000","last_undegraded":"2023-10-09T10:48:00.333875+0000","last_fullsized":"2023-10-09T10:48:00.333875+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":162,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.329325+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:48:00.329325+0000","last_peered":"2023-10-09T10:48:00.329325+0000","last_clean":"2023-10-09T10:48:00.329325+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:48:00.329325+0000","last_undegraded":"2023-10-09T10:48:00.329325+0000","last_fullsized":"2023-10-09T10:48:00.329325+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"181'6607","reported_seq":6720,"reported_epoch":181,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:14.465660+0000","last_change":"2023-10-09T10:46:33.546991+0000","last_active":"2023-10-09T10:48:14.465660+0000","last_peered":"2023-10-09T10:48:14.465660+0000","last_clean":"2023-10-09T10:46:31.009484+0000","last_became_active":"2023-10-09T10:46:33.535665+0000","last_became_peered":"2023-10-09T10:46:33.535665+0000","last_unstale":"2023-10-09T10:48:14.465660+0000","last_undegraded":"2023-10-09T10:46:33.277168+0000","last_fullsized":"2023-10-09T10:46:33.235038+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6607,"log_dups_size":0,"ondisk_log_size":6607,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64520192,"num_objects":985,"num_object_clones":0,"num_object_copies":2955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5543,"num_write_kb":208864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":997},{"shards":"3,7","objects":-1773}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":131,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.333971+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:48:00.333971+0000","last_peered":"2023-10-09T10:48:00.333971+0000","last_clean":"2023-10-09T10:48:00.333971+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:48:00.333971+0000","last_undegraded":"2023-10-09T10:48:00.333971+0000","last_fullsized":"2023-10-09T10:48:00.333971+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":29,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.329936+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:48:00.329936+0000","last_peered":"2023-10-09T10:48:00.329936+0000","last_clean":"2023-10-09T10:48:00.329936+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:48:00.329936+0000","last_undegraded":"2023-10-09T10:48:00.329936+0000","last_fullsized":"2023-10-09T10:48:00.329936+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":165,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.335920+0000","last_change":"2023-10-09T10:46:32.244607+0000","last_active":"2023-10-09T10:48:00.335920+0000","last_peered":"2023-10-09T10:48:00.335920+0000","last_clean":"2023-10-09T10:48:00.335920+0000","last_became_active":"2023-10-09T10:46:32.242628+0000","last_became_peered":"2023-10-09T10:46:32.242628+0000","last_unstale":"2023-10-09T10:48:00.335920+0000","last_undegraded":"2023-10-09T10:48:00.335920+0000","last_fullsized":"2023-10-09T10:48:00.335920+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:10.738819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":195,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.334027+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:48:00.334027+0000","last_peered":"2023-10-09T10:48:00.334027+0000","last_clean":"2023-10-09T10:48:00.334027+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:48:00.334027+0000","last_undegraded":"2023-10-09T10:48:00.334027+0000","last_fullsized":"2023-10-09T10:48:00.334027+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":71,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-10-09T10:48:06.487221+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:48:06.487221+0000","last_peered":"2023-10-09T10:48:06.487221+0000","last_clean":"2023-10-09T10:48:06.487221+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:48:06.487221+0000","last_undegraded":"2023-10-09T10:48:06.487221+0000","last_fullsized":"2023-10-09T10:48:06.487221+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":145,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-10-09T10:48:00.328402+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:48:00.328402+0000","last_peered":"2023-10-09T10:48:00.328402+0000","last_clean":"2023-10-09T10:48:00.328402+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:48:00.328402+0000","last_undegraded":"2023-10-09T10:48:00.328402+0000","last_fullsized":"2023-10-09T10:48:00.328402+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":2095087640,"num_objects":31970,"num_object_clones":0,"num_object_copies":95910,"num_objects_missing_on_primary":1121,"num_objects_missing":1121,"num_objects_degraded":28481,"num_objects_misplaced":4068,"num_objects_unfound":0,"num_objects_dirty":31970,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":172238,"num_write_kb":6258529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25470,"num_bytes_recovered":1487405056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8522248192,"data_stored":8522235976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170420,"ondisk_log_size":170420,"up":78,"acting":68,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1186,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332260,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1195172,"kb_used_data":893788,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92549980,"statfs":{"total":95995035648,"available":94771179520,"internally_reserved":0,"allocated":915238912,"data_stored":914549957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":0.79200000000000004,"15min":0.65000000000000002},"min":{"1min":0.26300000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.1479999999999997,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":2.9300000000000002},{"interface":"front","average":{"1min":1.546,"5min":0.97199999999999998,"15min":0.752},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.9909999999999997,"5min":10.207000000000001,"15min":10.207000000000001},"last":2.734}]},{"osd":1,"last update":"Mon Oct 9 10:47:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2210000000000001,"5min":1.3300000000000001,"15min":0.80400000000000005},"min":{"1min":0.43099999999999999,"5min":0.318,"15min":0.318},"max":{"1min":11.826000000000001,"5min":11.826000000000001,"15min":11.826000000000001},"last":4.8209999999999997},{"interface":"front","average":{"1min":2.286,"5min":1.155,"15min":0.73899999999999999},"min":{"1min":0.32100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":11.84,"5min":11.84,"15min":11.84},"last":4.6059999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:47:14 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.1499999999999999,"15min":0.72399999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.4770000000000003,"5min":6.4770000000000003,"15min":6.4770000000000003},"last":2.6640000000000001},{"interface":"front","average":{"1min":1.272,"5min":0.85699999999999998,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.251,"15min":0.251},"max":{"1min":3.1480000000000001,"5min":3.7120000000000002,"15min":3.7120000000000002},"last":2.9910000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.23,"15min":0.80000000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.0599999999999996,"5min":8.1639999999999997,"15min":8.1639999999999997},"last":4.9249999999999998},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.099,"15min":0.76600000000000001},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.504,"5min":12.451000000000001,"15min":12.451000000000001},"last":2.6800000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.73299999999999998,"15min":0.56200000000000006},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.1420000000000003,"5min":7.4509999999999996,"15min":7.4509999999999996},"last":3.3980000000000001},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.65300000000000002,"15min":0.53600000000000003},"min":{"1min":0.29799999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.2749999999999999,"5min":3.1699999999999999,"15min":3.1699999999999999},"last":2.6019999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.56399999999999995,"15min":0.52700000000000002},"min":{"1min":0.34300000000000003,"5min":0.247,"15min":0.247},"max":{"1min":2.2349999999999999,"5min":2.2349999999999999,"15min":2.2349999999999999},"last":4.1509999999999998},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.81299999999999994,"15min":0.63},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":5.0860000000000003,"5min":10.33,"15min":10.33},"last":2.7160000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:47:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.83399999999999996,"15min":0.64800000000000002},"min":{"1min":0.311,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.109,"5min":10.164999999999999,"15min":10.164999999999999},"last":6.1539999999999999},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.66000000000000003,"15min":0.58299999999999996},"min":{"1min":0.33300000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.306,"5min":2.8079999999999998,"15min":2.8079999999999998},"last":2.637}]}]},{"osd":6,"up_from":44,"seq":188978561191,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1023456,"kb_used_data":726168,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92721696,"statfs":{"total":95995035648,"available":94947016704,"internally_reserved":0,"allocated":743596032,"data_stored":742911615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.93200000000000005,"15min":0.71799999999999997},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.4630000000000001,"5min":7.3220000000000001,"15min":7.3220000000000001},"last":0.309},{"interface":"front","average":{"1min":1.02,"5min":1.002,"15min":0.73899999999999999},"min":{"1min":0.33900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.4969999999999999,"5min":7.5309999999999997,"15min":7.5309999999999997},"last":0.22800000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.157,"15min":0.77900000000000003},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.245},"max":{"1min":6.7560000000000002,"5min":16.747,"15min":16.747},"last":0.28999999999999998},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.284,"15min":0.79500000000000004},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.3579999999999997,"5min":16.734000000000002,"15min":16.734000000000002},"last":0.33700000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":0.95799999999999996,"15min":0.67500000000000004},"min":{"1min":0.28699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5069999999999997,"5min":7.5069999999999997,"15min":7.5069999999999997},"last":0.29699999999999999},{"interface":"front","average":{"1min":1.675,"5min":1.143,"15min":0.755},"min":{"1min":0.28599999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.3899999999999997,"5min":12.571,"15min":12.571},"last":4.2750000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.117,"15min":0.73099999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.8070000000000004,"5min":6.8070000000000004,"15min":6.8070000000000004},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.1219999999999999,"5min":1.385,"15min":0.82599999999999996},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.3940000000000001,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":0.45000000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.0629999999999999,"15min":0.65000000000000002},"min":{"1min":0.29899999999999999,"5min":0.255,"15min":0.255},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":3.827},{"interface":"front","average":{"1min":1.6579999999999999,"5min":0.89600000000000002,"15min":0.623},"min":{"1min":0.34100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.1260000000000003,"5min":5.1260000000000003,"15min":5.1260000000000003},"last":0.32800000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":1.1719999999999999,"15min":0.71099999999999997},"min":{"1min":0.35899999999999999,"5min":0.29499999999999998,"15min":0.27800000000000002},"max":{"1min":10.497,"5min":10.497,"15min":10.497},"last":3.911},{"interface":"front","average":{"1min":1.589,"5min":0.96599999999999997,"15min":0.67100000000000004},"min":{"1min":0.40699999999999997,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":6.8419999999999996,"5min":6.8419999999999996,"15min":6.8419999999999996},"last":0.55100000000000005}]},{"osd":7,"last update":"Mon Oct 9 10:47:25 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69899999999999995,"15min":0.57999999999999996},"min":{"1min":0.35799999999999998,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":2.5049999999999999,"5min":3.472,"15min":3.472},"last":3.8780000000000001},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.84299999999999997,"15min":0.65700000000000003},"min":{"1min":0.373,"5min":0.36899999999999999,"15min":0.34000000000000002},"max":{"1min":4.0800000000000001,"5min":4.0800000000000001,"15min":4.0800000000000001},"last":0.40000000000000002}]}]},{"osd":1,"up_from":85,"seq":365072220230,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1002780,"kb_used_data":868692,"kb_used_omap":9,"kb_used_meta":134070,"kb_avail":92742372,"statfs":{"total":95995035648,"available":94968188928,"internally_reserved":0,"allocated":889540608,"data_stored":888628589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":137288681},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.581,"15min":0.76900000000000002},"min":{"1min":0.251,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.2119999999999997,"5min":14.1,"15min":14.1},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.885,"5min":1.5629999999999999,"15min":0.78700000000000003},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.8090000000000002,"5min":14.465,"15min":14.465},"last":2.915}]},{"osd":2,"last update":"Mon Oct 9 10:48:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":1.492,"15min":0.74199999999999999},"min":{"1min":0.221,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":15.356,"5min":15.356,"15min":15.356},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.488,"5min":1.6890000000000001,"15min":0.81899999999999995},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.4130000000000003,"5min":14.234,"15min":14.234},"last":0.47699999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":1.5269999999999999,"15min":0.82299999999999995},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":7.234,"5min":14.404999999999999,"15min":14.404999999999999},"last":0.751},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.429,"15min":0.70599999999999996},"min":{"1min":0.30399999999999999,"5min":0.251,"15min":0.251},"max":{"1min":15.468999999999999,"5min":15.468999999999999,"15min":15.468999999999999},"last":4.7060000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.1679999999999999,"15min":0.78400000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.1449999999999996,"5min":8.0129999999999999,"15min":8.0129999999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.2350000000000001,"15min":0.82499999999999996},"min":{"1min":0.30299999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.1189999999999998,"5min":8.327,"15min":8.327},"last":0.51000000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":1.111,"15min":0.78900000000000003},"min":{"1min":0.28899999999999998,"5min":0.253,"15min":0.253},"max":{"1min":5.4850000000000003,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":2.9590000000000001},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.2929999999999999,"15min":0.86199999999999999},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.1859999999999999,"5min":7.923,"15min":7.923},"last":0.379}]},{"osd":6,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7909999999999999,"5min":1.2030000000000001,"15min":0.82199999999999995},"min":{"1min":0.35499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.9450000000000003,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":2.9849999999999999},{"interface":"front","average":{"1min":1.246,"5min":1.2949999999999999,"15min":0.86299999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.8440000000000003,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":0.41499999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.22,"15min":0.83999999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.8689999999999998,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":4.149},{"interface":"front","average":{"1min":1.992,"5min":1.2909999999999999,"15min":0.86099999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":7.1589999999999998,"5min":8.2469999999999999,"15min":8.2469999999999999},"last":0.57199999999999995}]}]},{"osd":0,"up_from":80,"seq":343597383763,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":506044,"kb_used_data":398132,"kb_used_omap":28,"kb_used_meta":107875,"kb_avail":93239108,"statfs":{"total":95995035648,"available":95476846592,"internally_reserved":0,"allocated":407687168,"data_stored":406742510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":110464848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:47:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2509999999999999,"5min":1.5449999999999999,"15min":0.76000000000000001},"min":{"1min":0.22,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":10.564,"5min":25.475000000000001,"15min":25.475000000000001},"last":0.28999999999999998},{"interface":"front","average":{"1min":2.3740000000000001,"5min":1.5940000000000001,"15min":0.77400000000000002},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":10.593999999999999,"5min":20.832999999999998,"15min":20.832999999999998},"last":0.34799999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":1.2350000000000001,"15min":0.64400000000000002},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.907,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.754,"5min":1.5349999999999999,"15min":0.73999999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.5619999999999994,"5min":14.391999999999999,"15min":14.391999999999999},"last":4.7990000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:47:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4740000000000002,"5min":1.8959999999999999,"15min":1.0660000000000001},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.215},"max":{"1min":10.644,"5min":14.926,"15min":14.926},"last":0.311},{"interface":"front","average":{"1min":2.2269999999999999,"5min":1.5940000000000001,"15min":0.89200000000000002},"min":{"1min":0.34499999999999997,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.228999999999999,"5min":11.868,"15min":11.868},"last":0.56399999999999995}]},{"osd":4,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.2250000000000001,"15min":0.82199999999999995},"min":{"1min":0.38400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.8150000000000004,"5min":6.8150000000000004,"15min":6.8150000000000004},"last":0.434},{"interface":"front","average":{"1min":2.391,"5min":1.405,"15min":0.88100000000000001},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.676,"5min":10.676,"15min":10.676},"last":4.0119999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":1.4359999999999999,"15min":0.89600000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":10.734,"5min":10.734,"15min":10.734},"last":1.085},{"interface":"front","average":{"1min":1.716,"5min":1.1659999999999999,"15min":0.82099999999999995},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.54,"5min":8.9190000000000005,"15min":8.9190000000000005},"last":3.8740000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.391,"15min":0.88400000000000001},"min":{"1min":0.36599999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.706,"5min":10.706,"15min":10.706},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.0780000000000001,"15min":0.79300000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.0499999999999998,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":0.68799999999999994}]},{"osd":7,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.474,"5min":1.151,"15min":0.81899999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.5380000000000003,"5min":8.875,"15min":8.875},"last":1.071},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.425,"15min":0.92700000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":14.394,"5min":14.394,"15min":14.394},"last":0.67100000000000004}]}]},{"osd":2,"up_from":0,"seq":386547056704,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893178,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1649512,"kb_used_data":1455520,"kb_used_omap":7,"kb_used_meta":193976,"kb_avail":92095640,"statfs":{"total":95995035648,"available":94305935360,"internally_reserved":0,"allocated":1490452480,"data_stored":1489526286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":198632040},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":1.3140000000000001,"15min":0.65200000000000002},"min":{"1min":0.192,"5min":0.183,"15min":0.183},"max":{"1min":7.0289999999999999,"5min":11.557,"15min":11.557},"last":0.29499999999999998},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.1379999999999999,"15min":0.65000000000000002},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":6.681,"5min":7.0510000000000002,"15min":7.0510000000000002},"last":1.353}]},{"osd":1,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":1.6970000000000001,"15min":0.78500000000000003},"min":{"1min":0.27100000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.048,"5min":14.707000000000001,"15min":14.707000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.6220000000000001,"5min":1.1890000000000001,"15min":0.64500000000000002},"min":{"1min":0.19400000000000001,"5min":0.193,"15min":0.193},"max":{"1min":7.1630000000000003,"5min":11.629,"15min":11.629},"last":0.35899999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.6739999999999999,"15min":0.80200000000000005},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.9180000000000001,"5min":13.853,"15min":13.853},"last":0.27200000000000002},{"interface":"front","average":{"1min":2.5590000000000002,"5min":1.5129999999999999,"15min":0.76800000000000002},"min":{"1min":0.30099999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.71,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":1.1830000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.2210000000000001,"15min":0.78800000000000003},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.9909999999999997,"5min":11.532,"15min":11.532},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.2440000000000002,"5min":1.296,"15min":0.83699999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.962999999999999,"5min":11.962999999999999,"15min":11.962999999999999},"last":7.0839999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7589999999999999,"5min":1.8029999999999999,"15min":0.99199999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.907,"5min":51.509999999999998,"15min":51.509999999999998},"last":2.6349999999999998},{"interface":"front","average":{"1min":2.1469999999999998,"5min":2.306,"15min":1.1439999999999999},"min":{"1min":0.36799999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.1139999999999999,"5min":55.338999999999999,"15min":55.338999999999999},"last":0.379}]},{"osd":6,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":1.3620000000000001,"15min":0.84599999999999997},"min":{"1min":0.378,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":11.92,"5min":11.92,"15min":11.92},"last":5.2110000000000003},{"interface":"front","average":{"1min":2.3759999999999999,"5min":1.776,"15min":0.99199999999999999},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":10.528,"5min":13.891999999999999,"15min":13.891999999999999},"last":0.442}]},{"osd":7,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":2.3410000000000002,"15min":1.175},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.6520000000000001,"5min":51.744,"15min":51.744},"last":0.69499999999999995},{"interface":"front","average":{"1min":1.9139999999999999,"5min":1.869,"15min":1.0049999999999999},"min":{"1min":0.33000000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.133,"5min":53.399000000000001,"15min":53.399000000000001},"last":2.3109999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019054,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":946464,"kb_used_data":632792,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":92798688,"statfs":{"total":95995035648,"available":95025856512,"internally_reserved":0,"allocated":647979008,"data_stored":647293858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.0129999999999999,"15min":0.70899999999999996},"min":{"1min":0.24099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.5419999999999998,"5min":10.387,"15min":10.387},"last":1.5760000000000001},{"interface":"front","average":{"1min":1.581,"5min":1.1819999999999999,"15min":0.78400000000000003},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.5999999999999996,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.746}]},{"osd":1,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.1519999999999999,"15min":0.752},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.9100000000000001,"5min":6.907,"15min":6.907},"last":7.359},{"interface":"front","average":{"1min":1.859,"5min":1.2390000000000001,"15min":0.82699999999999996},"min":{"1min":0.30099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.7669999999999995,"5min":8.7669999999999995,"15min":8.7669999999999995},"last":2.7650000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:48:13 2023","interfaces":[{"interface":"back","average":{"1min":3.1520000000000001,"5min":1.617,"15min":0.91900000000000004},"min":{"1min":0.52100000000000002,"5min":0.253,"15min":0.253},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":0.77400000000000002},{"interface":"front","average":{"1min":4.016,"5min":1.855,"15min":0.99299999999999999},"min":{"1min":0.36799999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":20.545999999999999,"5min":20.545999999999999,"15min":20.545999999999999},"last":1.5329999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:47:38 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.333,"15min":0.91900000000000004},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.8170000000000002,"5min":8.8170000000000002,"15min":8.8170000000000002},"last":1.3580000000000001},{"interface":"front","average":{"1min":2.3650000000000002,"5min":1.359,"15min":0.84499999999999997},"min":{"1min":0.34300000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.625,"5min":8.9809999999999999,"15min":8.9809999999999999},"last":2.778}]},{"osd":5,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":0.89500000000000002,"15min":0.59699999999999998},"min":{"1min":0.25600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":8.6950000000000003,"5min":8.6950000000000003,"15min":8.6950000000000003},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.6579999999999999,"5min":0.94299999999999995,"15min":0.61299999999999999},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.252},"max":{"1min":4.8879999999999999,"5min":4.8879999999999999,"15min":4.8879999999999999},"last":2.036}]},{"osd":6,"last update":"Mon Oct 9 10:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":1.1319999999999999,"15min":0.69399999999999995},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":12.391999999999999,"5min":12.391999999999999,"15min":12.391999999999999},"last":2.0750000000000002},{"interface":"front","average":{"1min":3.1200000000000001,"5min":1.2130000000000001,"15min":0.78100000000000003},"min":{"1min":0.374,"5min":0.311,"15min":0.311},"max":{"1min":12.539,"5min":12.539,"15min":12.539},"last":2.1230000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.80800000000000005,"15min":0.64600000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.1050000000000004,"5min":4.1050000000000004,"15min":4.1050000000000004},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.748,"15min":0.60799999999999998},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0919999999999996,"5min":4.0919999999999996,"15min":4.0919999999999996},"last":0.84299999999999997}]}]},{"osd":5,"up_from":37,"seq":158913790123,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1128400,"kb_used_data":822856,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92616752,"statfs":{"total":95995035648,"available":94839554048,"internally_reserved":0,"allocated":842604544,"data_stored":841931223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.77600000000000002,"15min":0.65700000000000003},"min":{"1min":0.247,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.9860000000000002,"5min":4.8380000000000001,"15min":4.8380000000000001},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.84899999999999998,"15min":0.72899999999999998},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.0209999999999999,"5min":4.8650000000000002,"15min":4.8650000000000002},"last":0.35999999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":1.367,"15min":0.86099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":12.999000000000001,"5min":12.999000000000001,"15min":12.999000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.2370000000000001,"15min":0.80700000000000005},"min":{"1min":0.27600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3730000000000002,"5min":7.3730000000000002,"15min":7.3730000000000002},"last":0.33700000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:48:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6669999999999998,"5min":1.4390000000000001,"15min":0.86099999999999999},"min":{"1min":0.34799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.3280000000000003,"5min":7.9000000000000004,"15min":7.9000000000000004},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.2350000000000001,"15min":0.79400000000000004},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.3059999999999992,"5min":10.352,"15min":10.352},"last":2.0299999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4969999999999999,"5min":1.621,"15min":1.0229999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":18.016999999999999,"5min":18.016999999999999,"15min":18.016999999999999},"last":5.5999999999999996},{"interface":"front","average":{"1min":2.964,"5min":1.673,"15min":0.96599999999999997},"min":{"1min":0.318,"5min":0.317,"15min":0.317},"max":{"1min":15.555999999999999,"5min":15.555999999999999,"15min":15.555999999999999},"last":5.6349999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":0.83799999999999997,"15min":0.57799999999999996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":19.152999999999999,"5min":19.152999999999999,"15min":19.152999999999999},"last":2.0779999999999998},{"interface":"front","average":{"1min":1.986,"5min":0.96999999999999997,"15min":0.63500000000000001},"min":{"1min":0.26800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.3059999999999992,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":7.0359999999999996}]},{"osd":6,"last update":"Mon Oct 9 10:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":1.127,"15min":0.69899999999999995},"min":{"1min":0.33100000000000002,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":7.7060000000000004,"5min":7.7060000000000004,"15min":7.7060000000000004},"last":6.9100000000000001},{"interface":"front","average":{"1min":1.958,"5min":0.90400000000000003,"15min":0.64000000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.30299999999999999},"max":{"1min":8.0909999999999993,"5min":8.0909999999999993,"15min":8.0909999999999993},"last":2.1030000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:47:22 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.67100000000000004,"15min":0.58799999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.4729999999999999,"5min":4.0510000000000002,"15min":4.0510000000000002},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.70899999999999996,"15min":0.60099999999999998},"min":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":4.1289999999999996,"5min":4.1289999999999996,"15min":4.1289999999999996},"last":2.7120000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":401149952,"data_stored":401145880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":877727744,"data_stored":877723672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2638675968,"data_stored":2638675968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1482817536,"data_stored":1482817536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":644349952,"data_stored":644349952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":834408448,"data_stored":834404376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":740556800,"data_stored":740556800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":902561792,"data_stored":902561792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:48:18.479 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:48:18.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:18 smithi155 ceph-mon[184854]: pgmap v395: 119 pgs: 106 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 28481/95982 objects degraded (29.673%); 4068/95982 objects misplaced (4.238%); 1.5 MiB/s, 131 objects/s recovering 2023-10-09T10:48:18.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:18 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:18.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:18 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:48:18.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:18 smithi155 ceph-mon[184854]: osdmap e183: 8 total, 8 up, 7 in 2023-10-09T10:48:18.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:18 smithi155 ceph-mon[184854]: failed to encode map e183 with expected crc 2023-10-09T10:48:18.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:18 smithi155 ceph-mon[184854]: failed to encode map e183 with expected crc 2023-10-09T10:48:18.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:18 smithi155 ceph-mon[184854]: failed to encode map e183 with expected crc 2023-10-09T10:48:18.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:18 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 28481/95982 objects degraded (29.673%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:18.481 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:18.517 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:18.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:18.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:18.627 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:18.664 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:18.677 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:18.709 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:18.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:18.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:18.814 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:18.857 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:18.913 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:18.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:18.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:19.015 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:19.053 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:19.338 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:19.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:19.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:19.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:19.462 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:19 smithi125 ceph-mon[200197]: from='client.44927 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:19.462 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:19 smithi125 ceph-mon[195507]: from='client.44927 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:19.500 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:19.583 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:19.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:19.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:19.688 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:19.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:19 smithi155 ceph-mon[184854]: from='client.44927 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:19.747 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:19.989 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:19.994 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:19.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:19.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:20.004 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:48:20.079 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:48:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:48:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:48:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:48:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42384 6.10 6:0c7bd3cf:::benchmark_data_smithi125_244951_object42383:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:48:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.95214017199999, 2023-10-09T10:48:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0577403670000001, 2023-10-09T10:48:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42384 2023-10-09T10:48:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:48:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:48:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117185+0000", 2023-10-09T10:48:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117225+0000", 2023-10-09T10:48:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0364000000000002e-05 2023-10-09T10:48:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117240+0000", 2023-10-09T10:48:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4425e-05 2023-10-09T10:48:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117248+0000", 2023-10-09T10:48:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4999999999999999e-06 2023-10-09T10:48:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117301+0000", 2023-10-09T10:48:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2685999999999997e-05 2023-10-09T10:48:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117312+0000", 2023-10-09T10:48:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1273e-05 2023-10-09T10:48:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103469+0000", 2023-10-09T10:48:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98615701300000003 2023-10-09T10:48:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103483+0000", 2023-10-09T10:48:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3207000000000001e-05 2023-10-09T10:48:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.148764+0000", 2023-10-09T10:48:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0452812789999999 2023-10-09T10:48:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.148882+0000", 2023-10-09T10:48:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118095 2023-10-09T10:48:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.149057+0000", 2023-10-09T10:48:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017516400000000001 2023-10-09T10:48:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174105+0000", 2023-10-09T10:48:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.025048237000000001 2023-10-09T10:48:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174158+0000", 2023-10-09T10:48:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2330999999999998e-05 2023-10-09T10:48:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174325+0000", 2023-10-09T10:48:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016690100000000001 2023-10-09T10:48:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174367+0000", 2023-10-09T10:48:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2759000000000002e-05 2023-10-09T10:48:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174927+0000", 2023-10-09T10:48:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00055942800000000005 2023-10-09T10:48:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42398 6.10 6:0867fefc:::benchmark_data_smithi125_244951_object42397:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:48:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.941361403, 2023-10-09T10:48:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0510695769999998, 2023-10-09T10:48:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42398 2023-10-09T10:48:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:48:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:48:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127964+0000", 2023-10-09T10:48:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127989+0000", 2023-10-09T10:48:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4811e-05 2023-10-09T10:48:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128005+0000", 2023-10-09T10:48:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6436999999999999e-05 2023-10-09T10:48:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128014+0000", 2023-10-09T10:48:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5990000000000002e-06 2023-10-09T10:48:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128072+0000", 2023-10-09T10:48:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7980999999999999e-05 2023-10-09T10:48:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128083+0000", 2023-10-09T10:48:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1051e-05 2023-10-09T10:48:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103714+0000", 2023-10-09T10:48:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97563124099999998 2023-10-09T10:48:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103725+0000", 2023-10-09T10:48:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1121000000000001e-05 2023-10-09T10:48:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174971+0000", 2023-10-09T10:48:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0712457310000001 2023-10-09T10:48:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.175149+0000", 2023-10-09T10:48:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000178584 2023-10-09T10:48:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.175358+0000", 2023-10-09T10:48:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000208848 2023-10-09T10:48:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178406+0000", 2023-10-09T10:48:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030478049999999998 2023-10-09T10:48:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178418+0000", 2023-10-09T10:48:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1499e-05 2023-10-09T10:48:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178466+0000", 2023-10-09T10:48:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8715999999999999e-05 2023-10-09T10:48:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178522+0000", 2023-10-09T10:48:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5952999999999998e-05 2023-10-09T10:48:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179035+0000", 2023-10-09T10:48:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051254299999999996 2023-10-09T10:48:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42518 6.10 6:0fbcc8bc:::benchmark_data_smithi125_244951_object42517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:48:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.874964078, 2023-10-09T10:48:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9852423510000001, 2023-10-09T10:48:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42518 2023-10-09T10:48:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:48:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:48:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194361+0000", 2023-10-09T10:48:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194385+0000", 2023-10-09T10:48:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4017000000000001e-05 2023-10-09T10:48:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194401+0000", 2023-10-09T10:48:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5841e-05 2023-10-09T10:48:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194408+0000", 2023-10-09T10:48:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0450000000000003e-06 2023-10-09T10:48:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194460+0000", 2023-10-09T10:48:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2015000000000003e-05 2023-10-09T10:48:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194474+0000", 2023-10-09T10:48:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4486e-05 2023-10-09T10:48:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103750+0000", 2023-10-09T10:48:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.90927595699999997 2023-10-09T10:48:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103758+0000", 2023-10-09T10:48:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9000000000000006e-06 2023-10-09T10:48:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176100+0000", 2023-10-09T10:48:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.07234187 2023-10-09T10:48:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176198+0000", 2023-10-09T10:48:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7408e-05 2023-10-09T10:48:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176317+0000", 2023-10-09T10:48:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011998900000000001 2023-10-09T10:48:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178425+0000", 2023-10-09T10:48:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021072 2023-10-09T10:48:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179085+0000", 2023-10-09T10:48:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00066011800000000001 2023-10-09T10:48:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179122+0000", 2023-10-09T10:48:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6831e-05 2023-10-09T10:48:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179155+0000", 2023-10-09T10:48:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3080000000000002e-05 2023-10-09T10:48:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179605+0000", 2023-10-09T10:48:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000450121 2023-10-09T10:48:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42546 6.10 6:0889293a:::benchmark_data_smithi125_244951_object42545:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:48:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.860534645, 2023-10-09T10:48:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.971488017, 2023-10-09T10:48:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42546 2023-10-09T10:48:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:48:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:48:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208790+0000", 2023-10-09T10:48:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:48:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208815+0000", 2023-10-09T10:48:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5103e-05 2023-10-09T10:48:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208831+0000", 2023-10-09T10:48:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5235000000000001e-05 2023-10-09T10:48:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208838+0000", 2023-10-09T10:48:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6510000000000001e-06 2023-10-09T10:48:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208895+0000", 2023-10-09T10:48:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6817999999999997e-05 2023-10-09T10:48:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208906+0000", 2023-10-09T10:48:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0550000000000001e-05 2023-10-09T10:48:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103780+0000", 2023-10-09T10:48:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.89487466699999996 2023-10-09T10:48:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103788+0000", 2023-10-09T10:48:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0970000000000006e-06 2023-10-09T10:48:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176855+0000", 2023-10-09T10:48:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0730664299999999 2023-10-09T10:48:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176993+0000", 2023-10-09T10:48:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013810999999999999 2023-10-09T10:48:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.177166+0000", 2023-10-09T10:48:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017289900000000001 2023-10-09T10:48:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179093+0000", 2023-10-09T10:48:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019276969999999999 2023-10-09T10:48:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179674+0000", 2023-10-09T10:48:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00058078000000000003 2023-10-09T10:48:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179730+0000", 2023-10-09T10:48:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5418999999999999e-05 2023-10-09T10:48:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179788+0000", 2023-10-09T10:48:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8510000000000001e-05 2023-10-09T10:48:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180280+0000", 2023-10-09T10:48:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00049179400000000004 2023-10-09T10:48:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42570 6.10 6:0f6ea5a0:::benchmark_data_smithi125_244951_object42569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:48:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.84645015199999, 2023-10-09T10:48:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.969653487, 2023-10-09T10:48:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42570 2023-10-09T10:48:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:48:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:48:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222875+0000", 2023-10-09T10:48:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222897+0000", 2023-10-09T10:48:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1678000000000001e-05 2023-10-09T10:48:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222906+0000", 2023-10-09T10:48:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4190000000000008e-06 2023-10-09T10:48:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222913+0000", 2023-10-09T10:48:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2659999999999998e-06 2023-10-09T10:48:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222958+0000", 2023-10-09T10:48:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5979000000000001e-05 2023-10-09T10:48:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222965+0000", 2023-10-09T10:48:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0099999999999998e-06 2023-10-09T10:48:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103823+0000", 2023-10-09T10:48:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.880857891 2023-10-09T10:48:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103833+0000", 2023-10-09T10:48:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8910000000000007e-06 2023-10-09T10:48:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180446+0000", 2023-10-09T10:48:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0766125440000001 2023-10-09T10:48:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180594+0000", 2023-10-09T10:48:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014846600000000001 2023-10-09T10:48:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180775+0000", 2023-10-09T10:48:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018071999999999999 2023-10-09T10:48:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183638+0000", 2023-10-09T10:48:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028626440000000001 2023-10-09T10:48:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183677+0000", 2023-10-09T10:48:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9545000000000002e-05 2023-10-09T10:48:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191583+0000", 2023-10-09T10:48:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0079059569999999999 2023-10-09T10:48:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191643+0000", 2023-10-09T10:48:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9590999999999999e-05 2023-10-09T10:48:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192530+0000", 2023-10-09T10:48:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00088716399999999996 2023-10-09T10:48:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42586 6.10 6:090d6b8d:::benchmark_data_smithi125_244951_object42585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:48:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.83985561099999, 2023-10-09T10:48:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.963875493, 2023-10-09T10:48:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42586 2023-10-09T10:48:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:48:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:48:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229470+0000", 2023-10-09T10:48:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:48:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229492+0000", 2023-10-09T10:48:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1716999999999999e-05 2023-10-09T10:48:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229501+0000", 2023-10-09T10:48:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0929999999999992e-06 2023-10-09T10:48:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229507+0000", 2023-10-09T10:48:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6760000000000001e-06 2023-10-09T10:48:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229565+0000", 2023-10-09T10:48:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7450000000000001e-05 2023-10-09T10:48:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229572+0000", 2023-10-09T10:48:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0820000000000001e-06 2023-10-09T10:48:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103865+0000", 2023-10-09T10:48:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.87429282200000003 2023-10-09T10:48:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103873+0000", 2023-10-09T10:48:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2090000000000005e-06 2023-10-09T10:48:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181656+0000", 2023-10-09T10:48:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.077783462 2023-10-09T10:48:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181791+0000", 2023-10-09T10:48:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000134351 2023-10-09T10:48:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181948+0000", 2023-10-09T10:48:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015678600000000001 2023-10-09T10:48:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183685+0000", 2023-10-09T10:48:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00173705 2023-10-09T10:48:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183719+0000", 2023-10-09T10:48:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4301999999999998e-05 2023-10-09T10:48:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192640+0000", 2023-10-09T10:48:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0089210899999999996 2023-10-09T10:48:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192696+0000", 2023-10-09T10:48:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5733000000000003e-05 2023-10-09T10:48:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193346+0000", 2023-10-09T10:48:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00065071200000000001 2023-10-09T10:48:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42606 6.10 6:0826a955:::benchmark_data_smithi125_244951_object42605:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:48:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.831096016, 2023-10-09T10:48:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.955680458, 2023-10-09T10:48:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42606 2023-10-09T10:48:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:48:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:48:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238230+0000", 2023-10-09T10:48:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:48:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238257+0000", 2023-10-09T10:48:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7551000000000001e-05 2023-10-09T10:48:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238267+0000", 2023-10-09T10:48:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5820000000000008e-06 2023-10-09T10:48:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238273+0000", 2023-10-09T10:48:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5490000000000003e-06 2023-10-09T10:48:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238322+0000", 2023-10-09T10:48:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8467999999999997e-05 2023-10-09T10:48:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238332+0000", 2023-10-09T10:48:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9000000000000001e-06 2023-10-09T10:48:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103898+0000", 2023-10-09T10:48:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.86556675699999996 2023-10-09T10:48:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103906+0000", 2023-10-09T10:48:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7710000000000006e-06 2023-10-09T10:48:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.182699+0000", 2023-10-09T10:48:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0787928490000001 2023-10-09T10:48:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.182847+0000", 2023-10-09T10:48:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014758600000000001 2023-10-09T10:48:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183016+0000", 2023-10-09T10:48:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000169326 2023-10-09T10:48:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186922+0000", 2023-10-09T10:48:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039060269999999998 2023-10-09T10:48:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186980+0000", 2023-10-09T10:48:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8313999999999998e-05 2023-10-09T10:48:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193472+0000", 2023-10-09T10:48:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0064917020000000002 2023-10-09T10:48:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193522+0000", 2023-10-09T10:48:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9672999999999997e-05 2023-10-09T10:48:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193911+0000", 2023-10-09T10:48:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00038944099999999997 2023-10-09T10:48:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42608 6.10 6:0c1c992a:::benchmark_data_smithi125_244951_object42607:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:48:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.828004316, 2023-10-09T10:48:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9589951289999998, 2023-10-09T10:48:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42608 2023-10-09T10:48:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:48:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:48:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241321+0000", 2023-10-09T10:48:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:48:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241343+0000", 2023-10-09T10:48:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1987000000000001e-05 2023-10-09T10:48:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241355+0000", 2023-10-09T10:48:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1395e-05 2023-10-09T10:48:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241362+0000", 2023-10-09T10:48:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7759999999999997e-06 2023-10-09T10:48:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241406+0000", 2023-10-09T10:48:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3189999999999998e-05 2023-10-09T10:48:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241413+0000", 2023-10-09T10:48:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5800000000000003e-06 2023-10-09T10:48:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103930+0000", 2023-10-09T10:48:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.86251731399999998 2023-10-09T10:48:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103938+0000", 2023-10-09T10:48:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7349999999999996e-06 2023-10-09T10:48:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183741+0000", 2023-10-09T10:48:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0798032179999999 2023-10-09T10:48:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183865+0000", 2023-10-09T10:48:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012327200000000001 2023-10-09T10:48:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.184011+0000", 2023-10-09T10:48:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014617499999999999 2023-10-09T10:48:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186989+0000", 2023-10-09T10:48:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029777340000000001 2023-10-09T10:48:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187029+0000", 2023-10-09T10:48:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0513000000000003e-05 2023-10-09T10:48:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199414+0000", 2023-10-09T10:48:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012385022000000001 2023-10-09T10:48:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199472+0000", 2023-10-09T10:48:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7998999999999998e-05 2023-10-09T10:48:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200317+0000", 2023-10-09T10:48:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000845245 2023-10-09T10:48:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42717 6.10 6:0e29612e:::benchmark_data_smithi125_244951_object42716:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:48:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.761875858, 2023-10-09T10:48:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8935023019999999, 2023-10-09T10:48:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42717 2023-10-09T10:48:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:48:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:48:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307449+0000", 2023-10-09T10:48:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307473+0000", 2023-10-09T10:48:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3349e-05 2023-10-09T10:48:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307490+0000", 2023-10-09T10:48:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7133e-05 2023-10-09T10:48:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307496+0000", 2023-10-09T10:48:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7349999999999999e-06 2023-10-09T10:48:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307561+0000", 2023-10-09T10:48:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4226999999999993e-05 2023-10-09T10:48:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307575+0000", 2023-10-09T10:48:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4504e-05 2023-10-09T10:48:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103963+0000", 2023-10-09T10:48:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.79638768500000001 2023-10-09T10:48:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103971+0000", 2023-10-09T10:48:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9629999999999998e-06 2023-10-09T10:48:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.185776+0000", 2023-10-09T10:48:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0818051479999999 2023-10-09T10:48:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.185949+0000", 2023-10-09T10:48:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000172659 2023-10-09T10:48:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186122+0000", 2023-10-09T10:48:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017296799999999999 2023-10-09T10:48:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189418+0000", 2023-10-09T10:48:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032959679999999998 2023-10-09T10:48:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189457+0000", 2023-10-09T10:48:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9669000000000003e-05 2023-10-09T10:48:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200372+0000", 2023-10-09T10:48:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010914633999999999 2023-10-09T10:48:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200421+0000", 2023-10-09T10:48:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9047e-05 2023-10-09T10:48:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200953+0000", 2023-10-09T10:48:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00053212500000000005 2023-10-09T10:48:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42722 6.10 6:0ee7a776:::benchmark_data_smithi125_244951_object42721:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:48:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.76181765000001, 2023-10-09T10:48:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8972440939999999, 2023-10-09T10:48:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42722 2023-10-09T10:48:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:48:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:48:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307508+0000", 2023-10-09T10:48:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307529+0000", 2023-10-09T10:48:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0866999999999999e-05 2023-10-09T10:48:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307539+0000", 2023-10-09T10:48:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2690000000000005e-06 2023-10-09T10:48:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.161 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307544+0000", 2023-10-09T10:48:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7930000000000003e-06 2023-10-09T10:48:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307599+0000", 2023-10-09T10:48:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4876000000000001e-05 2023-10-09T10:48:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.162 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307607+0000", 2023-10-09T10:48:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8760000000000005e-06 2023-10-09T10:48:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103995+0000", 2023-10-09T10:48:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.79638814099999999 2023-10-09T10:48:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104003+0000", 2023-10-09T10:48:20.163 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7819999999999994e-06 2023-10-09T10:48:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187058+0000", 2023-10-09T10:48:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.083054712 2023-10-09T10:48:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187234+0000", 2023-10-09T10:48:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017615799999999999 2023-10-09T10:48:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.164 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187416+0000", 2023-10-09T10:48:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018243 2023-10-09T10:48:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189466+0000", 2023-10-09T10:48:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020493210000000002 2023-10-09T10:48:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.165 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189504+0000", 2023-10-09T10:48:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8287000000000001e-05 2023-10-09T10:48:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.201001+0000", 2023-10-09T10:48:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011496632999999999 2023-10-09T10:48:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.201044+0000", 2023-10-09T10:48:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3078000000000003e-05 2023-10-09T10:48:20.166 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204753+0000", 2023-10-09T10:48:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037093209999999998 2023-10-09T10:48:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42757 6.10 6:08e1bcf5:::benchmark_data_smithi125_244951_object42756:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.333229+0000", 2023-10-09T10:48:20.167 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.73609749100001, 2023-10-09T10:48:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.872111463, 2023-10-09T10:48:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42757 2023-10-09T10:48:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.168 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333229+0000", 2023-10-09T10:48:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333229+0000", 2023-10-09T10:48:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333228+0000", 2023-10-09T10:48:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:48:20.169 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333280+0000", 2023-10-09T10:48:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1999999999999997e-05 2023-10-09T10:48:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333291+0000", 2023-10-09T10:48:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0738e-05 2023-10-09T10:48:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.170 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333298+0000", 2023-10-09T10:48:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7730000000000002e-06 2023-10-09T10:48:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333360+0000", 2023-10-09T10:48:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2230000000000006e-05 2023-10-09T10:48:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333371+0000", 2023-10-09T10:48:20.171 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0669999999999999e-05 2023-10-09T10:48:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104027+0000", 2023-10-09T10:48:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.77065640400000002 2023-10-09T10:48:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104035+0000", 2023-10-09T10:48:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7400000000000004e-06 2023-10-09T10:48:20.172 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.188337+0000", 2023-10-09T10:48:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.084302281 2023-10-09T10:48:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.188495+0000", 2023-10-09T10:48:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015774699999999999 2023-10-09T10:48:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.173 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.188674+0000", 2023-10-09T10:48:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000178729 2023-10-09T10:48:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191542+0000", 2023-10-09T10:48:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028680799999999998 2023-10-09T10:48:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192592+0000", 2023-10-09T10:48:20.174 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010502140000000001 2023-10-09T10:48:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204808+0000", 2023-10-09T10:48:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012216595 2023-10-09T10:48:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204867+0000", 2023-10-09T10:48:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8251999999999997e-05 2023-10-09T10:48:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205341+0000", 2023-10-09T10:48:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047385999999999999 2023-10-09T10:48:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42857 6.10 6:0e113cab:::benchmark_data_smithi125_244951_object42856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.409599+0000", 2023-10-09T10:48:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.659727922, 2023-10-09T10:48:20.176 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.796322089, 2023-10-09T10:48:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42857 2023-10-09T10:48:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409599+0000", 2023-10-09T10:48:20.177 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409599+0000", 2023-10-09T10:48:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409597+0000", 2023-10-09T10:48:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.178 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409619+0000", 2023-10-09T10:48:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1202000000000001e-05 2023-10-09T10:48:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409631+0000", 2023-10-09T10:48:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2546e-05 2023-10-09T10:48:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.179 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409638+0000", 2023-10-09T10:48:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9269999999999999e-06 2023-10-09T10:48:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409694+0000", 2023-10-09T10:48:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6025e-05 2023-10-09T10:48:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409706+0000", 2023-10-09T10:48:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.164e-05 2023-10-09T10:48:20.180 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104059+0000", 2023-10-09T10:48:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.69435323100000002 2023-10-09T10:48:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104067+0000", 2023-10-09T10:48:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9060000000000002e-06 2023-10-09T10:48:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.181 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189529+0000", 2023-10-09T10:48:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.085462189 2023-10-09T10:48:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189675+0000", 2023-10-09T10:48:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014598599999999999 2023-10-09T10:48:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189846+0000", 2023-10-09T10:48:20.182 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017139799999999999 2023-10-09T10:48:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192547+0000", 2023-10-09T10:48:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027008760000000001 2023-10-09T10:48:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193395+0000", 2023-10-09T10:48:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00084811200000000004 2023-10-09T10:48:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.183 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205386+0000", 2023-10-09T10:48:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011990763999999999 2023-10-09T10:48:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205435+0000", 2023-10-09T10:48:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8467999999999997e-05 2023-10-09T10:48:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205921+0000", 2023-10-09T10:48:20.184 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00048609599999999999 2023-10-09T10:48:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42860 6.10 6:08b59805:::benchmark_data_smithi125_244951_object42859:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.411356+0000", 2023-10-09T10:48:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.657970159, 2023-10-09T10:48:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7952376650000001, 2023-10-09T10:48:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.185 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42860 2023-10-09T10:48:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411356+0000", 2023-10-09T10:48:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.186 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411356+0000", 2023-10-09T10:48:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411355+0000", 2023-10-09T10:48:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411383+0000", 2023-10-09T10:48:20.187 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7348999999999999e-05 2023-10-09T10:48:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411398+0000", 2023-10-09T10:48:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5424e-05 2023-10-09T10:48:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411407+0000", 2023-10-09T10:48:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2429999999999994e-06 2023-10-09T10:48:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.188 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411465+0000", 2023-10-09T10:48:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7892000000000001e-05 2023-10-09T10:48:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411475+0000", 2023-10-09T10:48:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0066e-05 2023-10-09T10:48:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.189 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104091+0000", 2023-10-09T10:48:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.69261584300000001 2023-10-09T10:48:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104099+0000", 2023-10-09T10:48:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8760000000000005e-06 2023-10-09T10:48:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.190559+0000", 2023-10-09T10:48:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.086459659 2023-10-09T10:48:20.190 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.190698+0000", 2023-10-09T10:48:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013982400000000001 2023-10-09T10:48:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.190864+0000", 2023-10-09T10:48:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016535200000000001 2023-10-09T10:48:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.191 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192601+0000", 2023-10-09T10:48:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0017371999999999999 2023-10-09T10:48:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193434+0000", 2023-10-09T10:48:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00083267900000000001 2023-10-09T10:48:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206008+0000", 2023-10-09T10:48:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012574058000000001 2023-10-09T10:48:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206060+0000", 2023-10-09T10:48:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2710999999999997e-05 2023-10-09T10:48:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206594+0000", 2023-10-09T10:48:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00053372399999999996 2023-10-09T10:48:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42870 6.10 6:0e8b4417:::benchmark_data_smithi125_244951_object42869:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.417961+0000", 2023-10-09T10:48:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.65136546900001, 2023-10-09T10:48:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8001529010000001, 2023-10-09T10:48:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42870 2023-10-09T10:48:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417961+0000", 2023-10-09T10:48:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417961+0000", 2023-10-09T10:48:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417960+0000", 2023-10-09T10:48:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417982+0000", 2023-10-09T10:48:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2585000000000001e-05 2023-10-09T10:48:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417999+0000", 2023-10-09T10:48:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6711999999999999e-05 2023-10-09T10:48:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.418005+0000", 2023-10-09T10:48:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3180000000000003e-06 2023-10-09T10:48:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.197 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.418044+0000", 2023-10-09T10:48:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9171999999999997e-05 2023-10-09T10:48:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.418052+0000", 2023-10-09T10:48:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9880000000000004e-06 2023-10-09T10:48:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104123+0000", 2023-10-09T10:48:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.68607037299999996 2023-10-09T10:48:20.198 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104131+0000", 2023-10-09T10:48:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0730000000000005e-06 2023-10-09T10:48:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193936+0000", 2023-10-09T10:48:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0898054349999999 2023-10-09T10:48:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.194029+0000", 2023-10-09T10:48:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2313999999999997e-05 2023-10-09T10:48:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.194131+0000", 2023-10-09T10:48:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000102384 2023-10-09T10:48:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.200 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199366+0000", 2023-10-09T10:48:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0052347640000000003 2023-10-09T10:48:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205969+0000", 2023-10-09T10:48:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066036840000000003 2023-10-09T10:48:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.217437+0000", 2023-10-09T10:48:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011467252000000001 2023-10-09T10:48:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.201 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.217474+0000", 2023-10-09T10:48:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7177e-05 2023-10-09T10:48:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218114+0000", 2023-10-09T10:48:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000640231 2023-10-09T10:48:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.202 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42902 6.10 6:0d48f256:::benchmark_data_smithi125_244951_object42901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.448090+0000", 2023-10-09T10:48:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.62123701100001, 2023-10-09T10:48:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7704487879999999, 2023-10-09T10:48:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42902 2023-10-09T10:48:20.203 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448090+0000", 2023-10-09T10:48:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448090+0000", 2023-10-09T10:48:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.204 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448088+0000", 2023-10-09T10:48:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:48:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448108+0000", 2023-10-09T10:48:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9888999999999999e-05 2023-10-09T10:48:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.205 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448124+0000", 2023-10-09T10:48:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5529999999999999e-05 2023-10-09T10:48:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448131+0000", 2023-10-09T10:48:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9500000000000004e-06 2023-10-09T10:48:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448189+0000", 2023-10-09T10:48:20.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8010000000000002e-05 2023-10-09T10:48:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448201+0000", 2023-10-09T10:48:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2368000000000001e-05 2023-10-09T10:48:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104155+0000", 2023-10-09T10:48:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.65595361500000005 2023-10-09T10:48:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104163+0000", 2023-10-09T10:48:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9759999999999995e-06 2023-10-09T10:48:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.194936+0000", 2023-10-09T10:48:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.090773378 2023-10-09T10:48:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.195074+0000", 2023-10-09T10:48:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000138298 2023-10-09T10:48:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.196676+0000", 2023-10-09T10:48:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001601327 2023-10-09T10:48:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199373+0000", 2023-10-09T10:48:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026971320000000001 2023-10-09T10:48:20.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206643+0000", 2023-10-09T10:48:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0072700519999999999 2023-10-09T10:48:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218147+0000", 2023-10-09T10:48:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011503904000000001 2023-10-09T10:48:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218182+0000", 2023-10-09T10:48:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4752999999999997e-05 2023-10-09T10:48:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218538+0000", 2023-10-09T10:48:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035677400000000002 2023-10-09T10:48:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42906 6.10 6:0945c66e:::benchmark_data_smithi125_244951_object42905:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:48:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.453858+0000", 2023-10-09T10:48:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.61546889799999, 2023-10-09T10:48:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7650414240000001, 2023-10-09T10:48:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42906 2023-10-09T10:48:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.212 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453858+0000", 2023-10-09T10:48:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453858+0000", 2023-10-09T10:48:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453857+0000", 2023-10-09T10:48:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:48:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453877+0000", 2023-10-09T10:48:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0857999999999999e-05 2023-10-09T10:48:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453891+0000", 2023-10-09T10:48:20.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3249e-05 2023-10-09T10:48:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453897+0000", 2023-10-09T10:48:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1129999999999997e-06 2023-10-09T10:48:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453938+0000", 2023-10-09T10:48:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0799e-05 2023-10-09T10:48:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453945+0000", 2023-10-09T10:48:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2849999999999997e-06 2023-10-09T10:48:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104187+0000", 2023-10-09T10:48:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.65024259699999998 2023-10-09T10:48:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104195+0000", 2023-10-09T10:48:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9710000000000004e-06 2023-10-09T10:48:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.197298+0000", 2023-10-09T10:48:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.093102646 2023-10-09T10:48:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.197461+0000", 2023-10-09T10:48:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016253599999999999 2023-10-09T10:48:20.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.197621+0000", 2023-10-09T10:48:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000160178 2023-10-09T10:48:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200332+0000", 2023-10-09T10:48:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027108029999999999 2023-10-09T10:48:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206690+0000", 2023-10-09T10:48:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0063584990000000001 2023-10-09T10:48:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218563+0000", 2023-10-09T10:48:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011872562 2023-10-09T10:48:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218587+0000", 2023-10-09T10:48:20.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4586000000000001e-05 2023-10-09T10:48:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218899+0000", 2023-10-09T10:48:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000311883 2023-10-09T10:48:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:56713 6.10 6:086e6a33:::benchmark_data_smithi125_244951_object56712:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-10-09T10:48:20.220 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:08.647257+0000", 2023-10-09T10:48:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 191.42206972100001, 2023-10-09T10:48:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0123530810000001, 2023-10-09T10:48:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:48:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 56713 2023-10-09T10:48:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.221 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647257+0000", 2023-10-09T10:48:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647257+0000", 2023-10-09T10:48:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.222 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647256+0000", 2023-10-09T10:48:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647273+0000", 2023-10-09T10:48:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6075000000000001e-05 2023-10-09T10:48:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647281+0000", 2023-10-09T10:48:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6589999999999996e-06 2023-10-09T10:48:20.223 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647287+0000", 2023-10-09T10:48:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4940000000000003e-06 2023-10-09T10:48:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647464+0000", 2023-10-09T10:48:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017730999999999999 2023-10-09T10:48:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.224 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647667+0000", 2023-10-09T10:48:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020269 2023-10-09T10:48:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:48:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647835+0000", 2023-10-09T10:48:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000168677 2023-10-09T10:48:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.649455+0000", 2023-10-09T10:48:20.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016191999999999999 2023-10-09T10:48:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.655385+0000", 2023-10-09T10:48:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0059305549999999997 2023-10-09T10:48:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.658642+0000", 2023-10-09T10:48:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032573540000000001 2023-10-09T10:48:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:20.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.658657+0000", 2023-10-09T10:48:20.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4354e-05 2023-10-09T10:48:20.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.659536+0000", 2023-10-09T10:48:20.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0008789929999999 2023-10-09T10:48:20.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.659610+0000", 2023-10-09T10:48:20.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4134999999999994e-05 2023-10-09T10:48:20.228 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.228 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.228 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.228 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPush(6.11 155/143 [PushOp(6:8b7ae795:::benchmark_data_smithi125_244951_object18078:head, version: 104'1108, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:8b7ae795:::benchmark_data_smithi125_244951_object18078:head@104'1108, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:48:20.228 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:33.349257+0000", 2023-10-09T10:48:20.228 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 106.72006908100001, 2023-10-09T10:48:20.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1771969579999999, 2023-10-09T10:48:20.228 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.228 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:48:20.228 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349257+0000", 2023-10-09T10:48:20.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349257+0000", 2023-10-09T10:48:20.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349256+0000", 2023-10-09T10:48:20.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:20.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349302+0000", 2023-10-09T10:48:20.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5671999999999999e-05 2023-10-09T10:48:20.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349333+0000", 2023-10-09T10:48:20.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1256e-05 2023-10-09T10:48:20.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349351+0000", 2023-10-09T10:48:20.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8303000000000001e-05 2023-10-09T10:48:20.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.525188+0000", 2023-10-09T10:48:20.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.175836713 2023-10-09T10:48:20.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.525199+0000", 2023-10-09T10:48:20.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0458e-05 2023-10-09T10:48:20.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.526454+0000", 2023-10-09T10:48:20.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001255927 2023-10-09T10:48:20.232 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.232 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.232 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.232 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPush(6.11 155/143 [PushOp(6:8ff63630:::benchmark_data_smithi125_244951_object18084:head, version: 104'1109, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:8ff63630:::benchmark_data_smithi125_244951_object18084:head@104'1109, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:48:20.233 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:33.403563+0000", 2023-10-09T10:48:20.233 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 106.665763829, 2023-10-09T10:48:20.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1637633279999999, 2023-10-09T10:48:20.233 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.233 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:48:20.233 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403563+0000", 2023-10-09T10:48:20.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403563+0000", 2023-10-09T10:48:20.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403561+0000", 2023-10-09T10:48:20.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:20.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403594+0000", 2023-10-09T10:48:20.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2743999999999997e-05 2023-10-09T10:48:20.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403612+0000", 2023-10-09T10:48:20.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8389e-05 2023-10-09T10:48:20.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403626+0000", 2023-10-09T10:48:20.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3458999999999999e-05 2023-10-09T10:48:20.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.565733+0000", 2023-10-09T10:48:20.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.162107429 2023-10-09T10:48:20.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.565745+0000", 2023-10-09T10:48:20.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1355000000000001e-05 2023-10-09T10:48:20.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.567326+0000", 2023-10-09T10:48:20.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015813979999999999 2023-10-09T10:48:20.237 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.237 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.237 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.237 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:123629 6.9 6:932bc8aa:::benchmark_data_smithi125_244951_object17186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-10-09T10:48:20.237 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:33.543425+0000", 2023-10-09T10:48:20.238 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 106.525901236, 2023-10-09T10:48:20.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.002534984, 2023-10-09T10:48:20.238 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:20.238 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:20.238 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:20.238 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:20.238 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:20.238 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 123629 2023-10-09T10:48:20.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.238 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:20.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.238 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:20.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543425+0000", 2023-10-09T10:48:20.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:20.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543425+0000", 2023-10-09T10:48:20.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:20.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:20.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543424+0000", 2023-10-09T10:48:20.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:48:20.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:20.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543432+0000", 2023-10-09T10:48:20.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1389999999999995e-06 2023-10-09T10:48:20.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:20.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543448+0000", 2023-10-09T10:48:20.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5979000000000001e-05 2023-10-09T10:48:20.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:20.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543458+0000", 2023-10-09T10:48:20.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.859e-06 2023-10-09T10:48:20.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:20.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.520483+0000", 2023-10-09T10:48:20.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97702594300000001 2023-10-09T10:48:20.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:20.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.520610+0000", 2023-10-09T10:48:20.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012663799999999999 2023-10-09T10:48:20.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:48:20.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.520719+0000", 2023-10-09T10:48:20.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010933999999999999 2023-10-09T10:48:20.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.525132+0000", 2023-10-09T10:48:20.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0044124419999999999 2023-10-09T10:48:20.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:20.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.534848+0000", 2023-10-09T10:48:20.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0097164620000000004 2023-10-09T10:48:20.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:20.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.544664+0000", 2023-10-09T10:48:20.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0098156279999999999 2023-10-09T10:48:20.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.244 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:20.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.544725+0000", 2023-10-09T10:48:20.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1042000000000003e-05 2023-10-09T10:48:20.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:20.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:20.244 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:20.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.545960+0000", 2023-10-09T10:48:20.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001235311 2023-10-09T10:48:20.244 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.244 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.244 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.244 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:20.245 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:20.245 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:48:20.245 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:20.246 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:48:20.246 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:48:20.246 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:48:20.270 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:48:20.311 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:20.326 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:48:20.364 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:20.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:20.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:20.381 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:48:20.416 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:48:20.463 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:48:20.474 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:20.479 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:48:20.479 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:48:20.479 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:48:20.479 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:48:20.479 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:48:20.479 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:48:20.480 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:48:20.480 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:48:20.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:20 smithi125 ceph-mon[195507]: pgmap v397: 119 pgs: 106 active+clean, 2 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 27851/95370 objects degraded (29.203%); 4068/95370 objects misplaced (4.265%); 1.9 MiB/s, 109 objects/s recovering 2023-10-09T10:48:20.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:20 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/568553656' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:20.550 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:20 smithi125 ceph-mon[200197]: pgmap v397: 119 pgs: 106 active+clean, 2 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 27851/95370 objects degraded (29.203%); 4068/95370 objects misplaced (4.265%); 1.9 MiB/s, 109 objects/s recovering 2023-10-09T10:48:20.550 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:20 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/568553656' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:20.565 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:20.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:20 smithi155 ceph-mon[184854]: pgmap v397: 119 pgs: 106 active+clean, 2 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 27851/95370 objects degraded (29.203%); 4068/95370 objects misplaced (4.265%); 1.9 MiB/s, 109 objects/s recovering 2023-10-09T10:48:20.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:20 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/568553656' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:20.765 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:20.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:20.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:20.872 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:20.920 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:20.953 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:20.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:20.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:21.062 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:21.100 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:48:21.106 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:21.137 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:21.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:21.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:21.242 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:21.296 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:21.334 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:21.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:21.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:21.394 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:21.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:21.470 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:21.503 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:21.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:21.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:21.572 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:48:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:48:21] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T10:48:21.610 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:21.651 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:21.689 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:21.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:21.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:21.796 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:21.882 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:21.925 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:21.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:21.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:22.039 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:22.128 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:22.226 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:22.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:22.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:22.330 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:22.401 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:22.442 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:22.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:22.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:22.467 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:22 smithi125 ceph-mon[195507]: pgmap v398: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 764 op/s; 27851/87876 objects degraded (31.694%); 3559/87876 objects misplaced (4.050%); 3.3 MiB/s, 131 objects/s recovering 2023-10-09T10:48:22.467 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:22 smithi125 ceph-mon[200197]: pgmap v398: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 764 op/s; 27851/87876 objects degraded (31.694%); 3559/87876 objects misplaced (4.050%); 3.3 MiB/s, 131 objects/s recovering 2023-10-09T10:48:22.477 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:22.546 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:22.584 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:22 smithi155 ceph-mon[184854]: pgmap v398: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 764 op/s; 27851/87876 objects degraded (31.694%); 3559/87876 objects misplaced (4.050%); 3.3 MiB/s, 131 objects/s recovering 2023-10-09T10:48:22.586 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:22.621 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:22.636 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:48:22.636 INFO:teuthology.orchestra.run.smithi125.stdout:{"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 27851/87876 objects degraded (31.694%), 9 pgs degraded, 9 pgs undersized","count":18},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":488,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":183,"num_osds":8,"num_up_osds":8,"osd_up_since":1696848214,"num_in_osds":7,"osd_in_since":1696848391,"num_remapped_pgs":12},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":107},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":2}],"num_pgs":119,"num_pools":5,"num_objects":29292,"data_bytes":1918605240,"bytes_used":7179145216,"bytes_avail":664786104320,"bytes_total":671965249536,"degraded_objects":27851,"degraded_total":87876,"degraded_ratio":0.31693522691064679,"misplaced_objects":3559,"misplaced_total":87876,"misplaced_ratio":0.040500250352769808,"recovering_objects_per_sec":131,"recovering_bytes_per_sec":3423000,"recovering_keys_per_sec":0,"num_objects_recovered":1056,"num_bytes_recovered":27394048,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":764},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi125.satgiw","status":"up:active","gid":24575}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"}},"servicemap":{"epoch":20,"modified":"2023-10-09T10:44:35.008237+0000","services":{}},"progress_events":{"4db0080d-306f-4da6-a38c-4c702d14cb34":{"message":"Global Recovery Event (3m)\n [========================....] (remaining: 24s)","progress":0.89075630903244019,"add_to_ceph_s":true},"585851ac-6e70-4462-9176-ac50987fab77":{"message":"Upgrade to 18.2.0-598-gb4fa663f (3m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T10:48:22.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:22.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:22.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:22.806 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:22.843 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:22.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:22.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:22.948 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:23.006 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:23.067 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:48:23.184 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:23.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:23.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:23.302 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:23.365 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:23.545 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:23 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4072570729' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:48:23.545 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:23 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 27851/87876 objects degraded (31.694%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:23.545 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:23 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4072570729' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:48:23.545 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:23 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 27851/87876 objects degraded (31.694%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:23.571 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:48:23.571 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:48:23.709 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:23.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:23.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:23.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:23 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4072570729' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:48:23.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:23 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 27851/87876 objects degraded (31.694%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:23.817 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:23.891 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:23.976 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:23.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:23.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:24.080 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:24.127 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:24.149 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:24.160 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:24.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:24.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:24.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:24.295 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:48:24.295 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:48:24.309 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:24 smithi125 ceph-mon[195507]: pgmap v399: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 27736/86058 objects degraded (32.229%); 3463/86058 objects misplaced (4.024%); 4.2 MiB/s, 163 objects/s recovering 2023-10-09T10:48:24.309 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:24 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/793554836' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:24.309 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:24 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:24.309 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:24 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:48:24.309 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:24 smithi125 ceph-mon[200197]: pgmap v399: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 27736/86058 objects degraded (32.229%); 3463/86058 objects misplaced (4.024%); 4.2 MiB/s, 163 objects/s recovering 2023-10-09T10:48:24.309 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:24 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/793554836' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:24.351 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:24 smithi155 ceph-mon[184854]: pgmap v399: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 27736/86058 objects degraded (32.229%); 3463/86058 objects misplaced (4.024%); 4.2 MiB/s, 163 objects/s recovering 2023-10-09T10:48:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:24 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/793554836' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:24 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:24.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:24 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:48:24.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:24 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:24.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:24 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:48:24.626 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:24.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:24.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:24.673 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:24.725 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:48:24.734 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:24.787 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:25.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:25 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:25.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:25 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:48:25.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:25 smithi125 ceph-mon[195507]: osdmap e184: 8 total, 8 up, 7 in 2023-10-09T10:48:25.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:25 smithi125 ceph-mon[195507]: failed to encode map e184 with expected crc 2023-10-09T10:48:25.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:25 smithi125 ceph-mon[195507]: failed to encode map e184 with expected crc 2023-10-09T10:48:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:25 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:25 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:48:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:25 smithi125 ceph-mon[200197]: osdmap e184: 8 total, 8 up, 7 in 2023-10-09T10:48:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:25 smithi125 ceph-mon[200197]: failed to encode map e184 with expected crc 2023-10-09T10:48:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:25 smithi125 ceph-mon[200197]: failed to encode map e184 with expected crc 2023-10-09T10:48:25.613 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:25 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:25.614 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:25 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:48:25.614 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:25 smithi155 ceph-mon[184854]: osdmap e184: 8 total, 8 up, 7 in 2023-10-09T10:48:25.614 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:25 smithi155 ceph-mon[184854]: failed to encode map e184 with expected crc 2023-10-09T10:48:25.614 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:25 smithi155 ceph-mon[184854]: failed to encode map e184 with expected crc 2023-10-09T10:48:25.680 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:25.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:25.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:25.791 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:25.829 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:25.912 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:48:26.162 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:26.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:26.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:26.272 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:26.351 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:26.408 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:26.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:26.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:26.520 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:26.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:26 smithi125 ceph-mon[195507]: pgmap v401: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 27736/82083 objects degraded (33.790%); 3375/82083 objects misplaced (4.112%); 4.0 MiB/s, 94 objects/s recovering 2023-10-09T10:48:26.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:26 smithi125 ceph-mon[200197]: pgmap v401: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 27736/82083 objects degraded (33.790%); 3375/82083 objects misplaced (4.112%); 4.0 MiB/s, 94 objects/s recovering 2023-10-09T10:48:26.609 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:26.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:26 smithi155 ceph-mon[184854]: pgmap v401: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 27736/82083 objects degraded (33.790%); 3375/82083 objects misplaced (4.112%); 4.0 MiB/s, 94 objects/s recovering 2023-10-09T10:48:26.642 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:26.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:26.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:26.748 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:26.800 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:26.826 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:26.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:26.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:26.931 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:27.003 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:27.136 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:27.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:27.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:27.248 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:27.249 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:27.291 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:27.369 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:27.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:27.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:27.404 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:48:27.445 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:48:27.479 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:27.490 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:48:27.562 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:48:27.571 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:27.609 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:48:27.648 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:48:27.651 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:27.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:27.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:27.693 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:48:27.754 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:48:27.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:27.781 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:48:27.781 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:48:27.781 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:48:27.781 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:48:27.781 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:48:27.781 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:48:27.781 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:48:27.781 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:48:27.832 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:27.919 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:27.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:27.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:28.025 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:28.111 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:48:28.112 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:28.134 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:48:28.337 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:28.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:28.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:28.441 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:28.498 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:28.520 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:48:28.537 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:28.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:28.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:28.646 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:28.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:28 smithi125 ceph-mon[195507]: pgmap v402: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 27736/77142 objects degraded (35.954%); 2869/77142 objects misplaced (3.719%); 5.0 MiB/s, 105 objects/s recovering 2023-10-09T10:48:28.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:28 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4281513809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:28.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:28 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 27736/77142 objects degraded (35.954%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:28.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:28 smithi125 ceph-mon[200197]: pgmap v402: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 27736/77142 objects degraded (35.954%); 2869/77142 objects misplaced (3.719%); 5.0 MiB/s, 105 objects/s recovering 2023-10-09T10:48:28.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:28 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4281513809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:28.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:28 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 27736/77142 objects degraded (35.954%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:28.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:28 smithi155 ceph-mon[184854]: pgmap v402: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 27736/77142 objects degraded (35.954%); 2869/77142 objects misplaced (3.719%); 5.0 MiB/s, 105 objects/s recovering 2023-10-09T10:48:28.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:28 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4281513809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:28.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:28 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 27736/77142 objects degraded (35.954%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:28.730 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:28.955 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":402,"stamp":"2023-10-09T10:48:27.151816+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1684117432,"num_objects":25714,"num_object_clones":0,"num_object_copies":77142,"num_objects_missing_on_primary":913,"num_objects_missing":913,"num_objects_degraded":27736,"num_objects_misplaced":2869,"num_objects_unfound":0,"num_objects_dirty":25714,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":178818,"num_write_kb":6260974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26538,"num_bytes_recovered":1541409538,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":174230,"ondisk_log_size":174230,"up":357,"acting":348,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6426928,"kb_used_data":4731384,"kb_used_omap":44,"kb_used_meta":1695059,"kb_avail":649789136,"statfs":{"total":671965249536,"available":665384075264,"internally_reserved":0,"allocated":4844937216,"data_stored":4839326411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1735740833},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-411566080,"num_objects":-6280,"num_object_clones":0,"num_object_copies":-18840,"num_objects_missing_on_primary":-208,"num_objects_missing":-208,"num_objects_degraded":-745,"num_objects_misplaced":-1199,"num_objects_unfound":0,"num_objects_dirty":-6280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6280,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1044,"num_bytes_recovered":52232192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.888592"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":177,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275431+0000","last_change":"2023-10-09T10:46:32.250979+0000","last_active":"2023-10-09T10:48:17.275431+0000","last_peered":"2023-10-09T10:48:17.275431+0000","last_clean":"2023-10-09T10:48:17.275431+0000","last_became_active":"2023-10-09T10:46:32.250625+0000","last_became_peered":"2023-10-09T10:46:32.250625+0000","last_unstale":"2023-10-09T10:48:17.275431+0000","last_undegraded":"2023-10-09T10:48:17.275431+0000","last_fullsized":"2023-10-09T10:48:17.275431+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:34.858348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":151,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.256392+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:48:16.256392+0000","last_peered":"2023-10-09T10:48:16.256392+0000","last_clean":"2023-10-09T10:48:16.256392+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:48:16.256392+0000","last_undegraded":"2023-10-09T10:48:16.256392+0000","last_fullsized":"2023-10-09T10:48:16.256392+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":136,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267298+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:48:17.267298+0000","last_peered":"2023-10-09T10:48:17.267298+0000","last_clean":"2023-10-09T10:48:17.267298+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:48:17.267298+0000","last_undegraded":"2023-10-09T10:48:17.267298+0000","last_fullsized":"2023-10-09T10:48:17.267298+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.266165+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:48:16.266165+0000","last_peered":"2023-10-09T10:48:16.266165+0000","last_clean":"2023-10-09T10:48:16.266165+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:48:16.266165+0000","last_undegraded":"2023-10-09T10:48:16.266165+0000","last_fullsized":"2023-10-09T10:48:16.266165+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":188,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.276998+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:48:17.276998+0000","last_peered":"2023-10-09T10:48:17.276998+0000","last_clean":"2023-10-09T10:48:17.276998+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:48:17.276998+0000","last_undegraded":"2023-10-09T10:48:17.276998+0000","last_fullsized":"2023-10-09T10:48:17.276998+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":122,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286593+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:48:17.286593+0000","last_peered":"2023-10-09T10:48:17.286593+0000","last_clean":"2023-10-09T10:48:17.286593+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:48:17.286593+0000","last_undegraded":"2023-10-09T10:48:17.286593+0000","last_fullsized":"2023-10-09T10:48:17.286593+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"184'7079","reported_seq":11237,"reported_epoch":184,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:25.942668+0000","last_change":"2023-10-09T10:46:33.521602+0000","last_active":"2023-10-09T10:48:25.942668+0000","last_peered":"2023-10-09T10:48:25.942668+0000","last_clean":"2023-10-09T10:46:31.055751+0000","last_became_active":"2023-10-09T10:46:33.364058+0000","last_became_peered":"2023-10-09T10:46:33.364058+0000","last_unstale":"2023-10-09T10:48:25.942668+0000","last_undegraded":"2023-10-09T10:48:25.942668+0000","last_fullsized":"2023-10-09T10:48:25.942668+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7079,"log_dups_size":0,"ondisk_log_size":7079,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53379072,"num_objects":815,"num_object_clones":0,"num_object_copies":2445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":815,"num_objects_unfound":0,"num_objects_dirty":815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6043,"num_write_kb":219424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2010,"num_bytes_recovered":131727360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":815}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":51,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.265947+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:48:16.265947+0000","last_peered":"2023-10-09T10:48:16.265947+0000","last_clean":"2023-10-09T10:48:16.265947+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:48:16.265947+0000","last_undegraded":"2023-10-09T10:48:16.265947+0000","last_fullsized":"2023-10-09T10:48:16.265947+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":152,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.255238+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:48:16.255238+0000","last_peered":"2023-10-09T10:48:16.255238+0000","last_clean":"2023-10-09T10:48:16.255238+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:48:16.255238+0000","last_undegraded":"2023-10-09T10:48:16.255238+0000","last_fullsized":"2023-10-09T10:48:16.255238+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":517,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267373+0000","last_change":"2023-10-09T10:46:05.620365+0000","last_active":"2023-10-09T10:48:17.267373+0000","last_peered":"2023-10-09T10:48:17.267373+0000","last_clean":"2023-10-09T10:48:17.267373+0000","last_became_active":"2023-10-09T10:46:05.620024+0000","last_became_peered":"2023-10-09T10:46:05.620024+0000","last_unstale":"2023-10-09T10:48:17.267373+0000","last_undegraded":"2023-10-09T10:48:17.267373+0000","last_fullsized":"2023-10-09T10:48:17.267373+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.13","version":"184'6863","reported_seq":8278,"reported_epoch":184,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:26.032357+0000","last_change":"2023-10-09T10:48:15.511525+0000","last_active":"2023-10-09T10:48:26.032357+0000","last_peered":"2023-10-09T10:48:26.032357+0000","last_clean":"2023-10-09T10:46:31.195699+0000","last_became_active":"2023-10-09T10:46:33.455632+0000","last_became_peered":"2023-10-09T10:46:33.455632+0000","last_unstale":"2023-10-09T10:48:26.032357+0000","last_undegraded":"2023-10-09T10:46:33.269323+0000","last_fullsized":"2023-10-09T10:46:33.241226+0000","mapping_epoch":154,"log_start":"108'2175","ondisk_log_start":"108'2175","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":4688,"log_dups_size":0,"ondisk_log_size":4688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:07:12.547333+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53018624,"num_objects":809,"num_object_clones":0,"num_object_copies":2427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1767,"num_objects_misplaced":396,"num_objects_unfound":0,"num_objects_dirty":809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4148,"num_write_kb":105088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":612,"num_bytes_recovered":40108032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":396},{"shards":"6,7","objects":-1354}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":149,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286522+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:48:17.286522+0000","last_peered":"2023-10-09T10:48:17.286522+0000","last_clean":"2023-10-09T10:48:17.286522+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:48:17.286522+0000","last_undegraded":"2023-10-09T10:48:17.286522+0000","last_fullsized":"2023-10-09T10:48:17.286522+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":109,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.251904+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:48:16.251904+0000","last_peered":"2023-10-09T10:48:16.251904+0000","last_clean":"2023-10-09T10:48:16.251904+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:48:16.251904+0000","last_undegraded":"2023-10-09T10:48:16.251904+0000","last_fullsized":"2023-10-09T10:48:16.251904+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"184'7022","reported_seq":15153,"reported_epoch":184,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:26.493447+0000","last_change":"2023-10-09T10:46:03.683196+0000","last_active":"2023-10-09T10:48:26.493447+0000","last_peered":"2023-10-09T10:48:26.493447+0000","last_clean":"2023-10-09T10:46:02.177309+0000","last_became_active":"2023-10-09T10:46:03.681058+0000","last_became_peered":"2023-10-09T10:46:03.681058+0000","last_unstale":"2023-10-09T10:48:26.493447+0000","last_undegraded":"2023-10-09T10:48:26.493447+0000","last_fullsized":"2023-10-09T10:48:26.493447+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7022,"log_dups_size":0,"ondisk_log_size":7022,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53182464,"num_objects":812,"num_object_clones":0,"num_object_copies":2436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":812,"num_objects_unfound":0,"num_objects_dirty":812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5976,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[1,5,6],"avail_no_missing":["1","2","3","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":812}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":104,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.256314+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:48:16.256314+0000","last_peered":"2023-10-09T10:48:16.256314+0000","last_clean":"2023-10-09T10:48:16.256314+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:48:16.256314+0000","last_undegraded":"2023-10-09T10:48:16.256314+0000","last_fullsized":"2023-10-09T10:48:16.256314+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":145,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.266962+0000","last_change":"2023-10-09T10:46:32.249921+0000","last_active":"2023-10-09T10:48:17.266962+0000","last_peered":"2023-10-09T10:48:17.266962+0000","last_clean":"2023-10-09T10:48:17.266962+0000","last_became_active":"2023-10-09T10:46:32.249590+0000","last_became_peered":"2023-10-09T10:46:32.249590+0000","last_unstale":"2023-10-09T10:48:17.266962+0000","last_undegraded":"2023-10-09T10:48:17.266962+0000","last_fullsized":"2023-10-09T10:48:17.266962+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:25.169752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"183'6788","reported_seq":15015,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.617719+0000","last_change":"2023-10-09T10:47:48.412928+0000","last_active":"2023-10-09T10:48:24.617719+0000","last_peered":"2023-10-09T10:48:24.617719+0000","last_clean":"2023-10-09T10:48:24.617719+0000","last_became_active":"2023-10-09T10:47:48.398813+0000","last_became_peered":"2023-10-09T10:47:48.398813+0000","last_unstale":"2023-10-09T10:48:24.617719+0000","last_undegraded":"2023-10-09T10:48:24.617719+0000","last_fullsized":"2023-10-09T10:48:24.617719+0000","mapping_epoch":177,"log_start":"103'261","ondisk_log_start":"103'261","created":107,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6527,"log_dups_size":0,"ondisk_log_size":6527,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51937280,"num_objects":793,"num_object_clones":0,"num_object_copies":2379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5781,"num_write_kb":210336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2691,"num_bytes_recovered":176357376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":154,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.276186+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:48:17.276186+0000","last_peered":"2023-10-09T10:48:17.276186+0000","last_clean":"2023-10-09T10:48:17.276186+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:48:17.276186+0000","last_undegraded":"2023-10-09T10:48:17.276186+0000","last_fullsized":"2023-10-09T10:48:17.276186+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":148,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275916+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:48:17.275916+0000","last_peered":"2023-10-09T10:48:17.275916+0000","last_clean":"2023-10-09T10:48:17.275916+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:48:17.275916+0000","last_undegraded":"2023-10-09T10:48:17.275916+0000","last_fullsized":"2023-10-09T10:48:17.275916+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":517,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.268784+0000","last_change":"2023-10-09T10:46:05.618276+0000","last_active":"2023-10-09T10:48:17.268784+0000","last_peered":"2023-10-09T10:48:17.268784+0000","last_clean":"2023-10-09T10:48:17.268784+0000","last_became_active":"2023-10-09T10:46:05.616509+0000","last_became_peered":"2023-10-09T10:46:05.616509+0000","last_unstale":"2023-10-09T10:48:17.268784+0000","last_undegraded":"2023-10-09T10:48:17.268784+0000","last_fullsized":"2023-10-09T10:48:17.268784+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.16","version":"183'6799","reported_seq":6907,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.606256+0000","last_change":"2023-10-09T10:46:03.592682+0000","last_active":"2023-10-09T10:48:24.606256+0000","last_peered":"2023-10-09T10:48:24.606256+0000","last_clean":"2023-10-09T10:48:24.606256+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:48:24.606256+0000","last_undegraded":"2023-10-09T10:48:24.606256+0000","last_fullsized":"2023-10-09T10:48:24.606256+0000","mapping_epoch":107,"log_start":"103'166","ondisk_log_start":"103'166","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6633,"log_dups_size":0,"ondisk_log_size":6633,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54394880,"num_objects":830,"num_object_clones":0,"num_object_copies":2490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5756,"num_write_kb":210752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":166,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.279501+0000","last_change":"2023-10-09T10:46:32.250256+0000","last_active":"2023-10-09T10:48:17.279501+0000","last_peered":"2023-10-09T10:48:17.279501+0000","last_clean":"2023-10-09T10:48:17.279501+0000","last_became_active":"2023-10-09T10:46:32.243312+0000","last_became_peered":"2023-10-09T10:46:32.243312+0000","last_unstale":"2023-10-09T10:48:17.279501+0000","last_undegraded":"2023-10-09T10:48:17.279501+0000","last_fullsized":"2023-10-09T10:48:17.279501+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:08.059777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":175,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.251148+0000","last_change":"2023-10-09T10:46:32.237577+0000","last_active":"2023-10-09T10:48:16.251148+0000","last_peered":"2023-10-09T10:48:16.251148+0000","last_clean":"2023-10-09T10:48:16.251148+0000","last_became_active":"2023-10-09T10:46:32.237316+0000","last_became_peered":"2023-10-09T10:46:32.237316+0000","last_unstale":"2023-10-09T10:48:16.251148+0000","last_undegraded":"2023-10-09T10:48:16.251148+0000","last_fullsized":"2023-10-09T10:48:16.251148+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:39.684602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":518,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.266544+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:48:17.266544+0000","last_peered":"2023-10-09T10:48:17.266544+0000","last_clean":"2023-10-09T10:48:17.266544+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:48:17.266544+0000","last_undegraded":"2023-10-09T10:48:17.266544+0000","last_fullsized":"2023-10-09T10:48:17.266544+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.17","version":"184'6802","reported_seq":6905,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:26.046542+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:48:26.046542+0000","last_peered":"2023-10-09T10:48:26.046542+0000","last_clean":"2023-10-09T10:48:26.046542+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:48:26.046542+0000","last_undegraded":"2023-10-09T10:48:26.046542+0000","last_fullsized":"2023-10-09T10:48:26.046542+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6802,"log_dups_size":0,"ondisk_log_size":6802,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46661632,"num_objects":712,"num_object_clones":0,"num_object_copies":2136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5794,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.268743+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:48:17.268743+0000","last_peered":"2023-10-09T10:48:17.268743+0000","last_clean":"2023-10-09T10:48:17.268743+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:48:17.268743+0000","last_undegraded":"2023-10-09T10:48:17.268743+0000","last_fullsized":"2023-10-09T10:48:17.268743+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":148,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286752+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:48:17.286752+0000","last_peered":"2023-10-09T10:48:17.286752+0000","last_clean":"2023-10-09T10:48:17.286752+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:48:17.286752+0000","last_undegraded":"2023-10-09T10:48:17.286752+0000","last_fullsized":"2023-10-09T10:48:17.286752+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":518,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.266619+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:48:17.266619+0000","last_peered":"2023-10-09T10:48:17.266619+0000","last_clean":"2023-10-09T10:48:17.266619+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:48:17.266619+0000","last_undegraded":"2023-10-09T10:48:17.266619+0000","last_fullsized":"2023-10-09T10:48:17.266619+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"184'6957","reported_seq":7065,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:26.517649+0000","last_change":"2023-10-09T10:46:33.243540+0000","last_active":"2023-10-09T10:48:26.517649+0000","last_peered":"2023-10-09T10:48:26.517649+0000","last_clean":"2023-10-09T10:48:26.517649+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:48:26.517649+0000","last_undegraded":"2023-10-09T10:48:26.517649+0000","last_fullsized":"2023-10-09T10:48:26.517649+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6957,"log_dups_size":0,"ondisk_log_size":6957,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50495488,"num_objects":771,"num_object_clones":0,"num_object_copies":2313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5903,"num_write_kb":213536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":94,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275638+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:48:17.275638+0000","last_peered":"2023-10-09T10:48:17.275638+0000","last_clean":"2023-10-09T10:48:17.275638+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:48:17.275638+0000","last_undegraded":"2023-10-09T10:48:17.275638+0000","last_fullsized":"2023-10-09T10:48:17.275638+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":51,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.266284+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:48:16.266284+0000","last_peered":"2023-10-09T10:48:16.266284+0000","last_clean":"2023-10-09T10:48:16.266284+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:48:16.266284+0000","last_undegraded":"2023-10-09T10:48:16.266284+0000","last_fullsized":"2023-10-09T10:48:16.266284+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":518,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.268498+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:48:17.268498+0000","last_peered":"2023-10-09T10:48:17.268498+0000","last_clean":"2023-10-09T10:48:17.268498+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:48:17.268498+0000","last_undegraded":"2023-10-09T10:48:17.268498+0000","last_fullsized":"2023-10-09T10:48:17.268498+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"183'6957","reported_seq":7074,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.541500+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:48:24.541500+0000","last_peered":"2023-10-09T10:48:24.541500+0000","last_clean":"2023-10-09T10:48:24.541500+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:48:24.541500+0000","last_undegraded":"2023-10-09T10:48:24.541500+0000","last_fullsized":"2023-10-09T10:48:24.541500+0000","mapping_epoch":115,"log_start":"103'505","ondisk_log_start":"103'505","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6452,"log_dups_size":0,"ondisk_log_size":6452,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50593792,"num_objects":772,"num_object_clones":0,"num_object_copies":2316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5896,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":62,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.266454+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:48:17.266454+0000","last_peered":"2023-10-09T10:48:17.266454+0000","last_clean":"2023-10-09T10:48:17.266454+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:48:17.266454+0000","last_undegraded":"2023-10-09T10:48:17.266454+0000","last_fullsized":"2023-10-09T10:48:17.266454+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":152,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.278609+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:48:17.278609+0000","last_peered":"2023-10-09T10:48:17.278609+0000","last_clean":"2023-10-09T10:48:17.278609+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:48:17.278609+0000","last_undegraded":"2023-10-09T10:48:17.278609+0000","last_fullsized":"2023-10-09T10:48:17.278609+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":518,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267681+0000","last_change":"2023-10-09T10:46:05.620574+0000","last_active":"2023-10-09T10:48:17.267681+0000","last_peered":"2023-10-09T10:48:17.267681+0000","last_clean":"2023-10-09T10:48:17.267681+0000","last_became_active":"2023-10-09T10:46:05.620212+0000","last_became_peered":"2023-10-09T10:46:05.620212+0000","last_unstale":"2023-10-09T10:48:17.267681+0000","last_undegraded":"2023-10-09T10:48:17.267681+0000","last_fullsized":"2023-10-09T10:48:17.267681+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"184'12091","reported_seq":16250,"reported_epoch":184,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:26.015386+0000","last_change":"2023-10-09T10:46:32.247903+0000","last_active":"2023-10-09T10:48:26.015386+0000","last_peered":"2023-10-09T10:48:26.015386+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:46:02.739355+0000","last_became_peered":"2023-10-09T10:46:02.739355+0000","last_unstale":"2023-10-09T10:48:26.015386+0000","last_undegraded":"2023-10-09T10:46:02.608805+0000","last_fullsized":"2023-10-09T10:46:02.583589+0000","mapping_epoch":142,"log_start":"113'3200","ondisk_log_start":"113'3200","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8891,"log_dups_size":0,"ondisk_log_size":8891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:32.899634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100990976,"num_objects":1541,"num_object_clones":0,"num_object_copies":4623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12091,"num_write_kb":436224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2022,"num_bytes_recovered":132513792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":625},{"shards":"5,6","objects":915}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":177,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275403+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:48:17.275403+0000","last_peered":"2023-10-09T10:48:17.275403+0000","last_clean":"2023-10-09T10:48:17.275403+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:48:17.275403+0000","last_undegraded":"2023-10-09T10:48:17.275403+0000","last_fullsized":"2023-10-09T10:48:17.275403+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":131,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286332+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:48:17.286332+0000","last_peered":"2023-10-09T10:48:17.286332+0000","last_clean":"2023-10-09T10:48:17.286332+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:48:17.286332+0000","last_undegraded":"2023-10-09T10:48:17.286332+0000","last_fullsized":"2023-10-09T10:48:17.286332+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":498,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267469+0000","last_change":"2023-10-09T10:46:02.631378+0000","last_active":"2023-10-09T10:48:17.267469+0000","last_peered":"2023-10-09T10:48:17.267469+0000","last_clean":"2023-10-09T10:48:17.267469+0000","last_became_active":"2023-10-09T10:46:02.629142+0000","last_became_peered":"2023-10-09T10:46:02.629142+0000","last_unstale":"2023-10-09T10:48:17.267469+0000","last_undegraded":"2023-10-09T10:48:17.267469+0000","last_fullsized":"2023-10-09T10:48:17.267469+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.b","version":"183'11588","reported_seq":11683,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.507022+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:48:24.507022+0000","last_peered":"2023-10-09T10:48:24.507022+0000","last_clean":"2023-10-09T10:48:24.507022+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:48:24.507022+0000","last_undegraded":"2023-10-09T10:48:24.507022+0000","last_fullsized":"2023-10-09T10:48:24.507022+0000","mapping_epoch":98,"log_start":"136'5925","ondisk_log_start":"136'5925","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5663,"log_dups_size":0,"ondisk_log_size":5663,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104333312,"num_objects":1592,"num_object_clones":0,"num_object_copies":4776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11588,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":150,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275053+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:48:17.275053+0000","last_peered":"2023-10-09T10:48:17.275053+0000","last_clean":"2023-10-09T10:48:17.275053+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:48:17.275053+0000","last_undegraded":"2023-10-09T10:48:17.275053+0000","last_fullsized":"2023-10-09T10:48:17.275053+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":185,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275568+0000","last_change":"2023-10-09T10:46:32.251035+0000","last_active":"2023-10-09T10:48:17.275568+0000","last_peered":"2023-10-09T10:48:17.275568+0000","last_clean":"2023-10-09T10:48:17.275568+0000","last_became_active":"2023-10-09T10:46:32.250293+0000","last_became_peered":"2023-10-09T10:46:32.250293+0000","last_unstale":"2023-10-09T10:48:17.275568+0000","last_undegraded":"2023-10-09T10:48:17.275568+0000","last_fullsized":"2023-10-09T10:48:17.275568+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.787766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":518,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.268063+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:48:17.268063+0000","last_peered":"2023-10-09T10:48:17.268063+0000","last_clean":"2023-10-09T10:48:17.268063+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:48:17.268063+0000","last_undegraded":"2023-10-09T10:48:17.268063+0000","last_fullsized":"2023-10-09T10:48:17.268063+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"183'6942","reported_seq":7057,"reported_epoch":183,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:24.397476+0000","last_change":"2023-10-09T10:46:33.540719+0000","last_active":"2023-10-09T10:48:24.397476+0000","last_peered":"2023-10-09T10:48:24.397476+0000","last_clean":"2023-10-09T10:46:31.137696+0000","last_became_active":"2023-10-09T10:46:33.527650+0000","last_became_peered":"2023-10-09T10:46:33.527650+0000","last_unstale":"2023-10-09T10:48:24.397476+0000","last_undegraded":"2023-10-09T10:46:33.285468+0000","last_fullsized":"2023-10-09T10:46:33.235683+0000","mapping_epoch":154,"log_start":"103'401","ondisk_log_start":"103'401","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6541,"log_dups_size":0,"ondisk_log_size":6541,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53051392,"num_objects":809,"num_object_clones":0,"num_object_copies":2427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5877,"num_write_kb":213984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":797},{"shards":"3,7","objects":-2000}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":105,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286674+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:48:17.286674+0000","last_peered":"2023-10-09T10:48:17.286674+0000","last_clean":"2023-10-09T10:48:17.286674+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:48:17.286674+0000","last_undegraded":"2023-10-09T10:48:17.286674+0000","last_fullsized":"2023-10-09T10:48:17.286674+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":122,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267223+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:48:17.267223+0000","last_peered":"2023-10-09T10:48:17.267223+0000","last_clean":"2023-10-09T10:48:17.267223+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:48:17.267223+0000","last_undegraded":"2023-10-09T10:48:17.267223+0000","last_fullsized":"2023-10-09T10:48:17.267223+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":518,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267701+0000","last_change":"2023-10-09T10:46:05.620030+0000","last_active":"2023-10-09T10:48:17.267701+0000","last_peered":"2023-10-09T10:48:17.267701+0000","last_clean":"2023-10-09T10:48:17.267701+0000","last_became_active":"2023-10-09T10:46:05.619694+0000","last_became_peered":"2023-10-09T10:46:05.619694+0000","last_unstale":"2023-10-09T10:48:17.267701+0000","last_undegraded":"2023-10-09T10:48:17.267701+0000","last_fullsized":"2023-10-09T10:48:17.267701+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.9","version":"183'6832","reported_seq":7455,"reported_epoch":183,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:22.768301+0000","last_change":"2023-10-09T10:48:15.517098+0000","last_active":"2023-10-09T10:48:22.768301+0000","last_peered":"2023-10-09T10:48:22.768301+0000","last_clean":"2023-10-09T10:46:31.190603+0000","last_became_active":"2023-10-09T10:46:34.396448+0000","last_became_peered":"2023-10-09T10:46:34.396448+0000","last_unstale":"2023-10-09T10:48:22.768301+0000","last_undegraded":"2023-10-09T10:46:33.246949+0000","last_fullsized":"2023-10-09T10:46:33.237968+0000","mapping_epoch":154,"log_start":"104'1195","ondisk_log_start":"104'1195","created":98,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5637,"log_dups_size":0,"ondisk_log_size":5637,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54001664,"num_objects":824,"num_object_clones":0,"num_object_copies":2472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1467,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5790,"num_write_kb":211648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1551,"num_bytes_recovered":85458944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,7","objects":-643}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":149,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.287413+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:48:17.287413+0000","last_peered":"2023-10-09T10:48:17.287413+0000","last_clean":"2023-10-09T10:48:17.287413+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:48:17.287413+0000","last_undegraded":"2023-10-09T10:48:17.287413+0000","last_fullsized":"2023-10-09T10:48:17.287413+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267041+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:48:17.267041+0000","last_peered":"2023-10-09T10:48:17.267041+0000","last_clean":"2023-10-09T10:48:17.267041+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:48:17.267041+0000","last_undegraded":"2023-10-09T10:48:17.267041+0000","last_fullsized":"2023-10-09T10:48:17.267041+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":509,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.285672+0000","last_change":"2023-10-09T10:46:05.618134+0000","last_active":"2023-10-09T10:48:16.285672+0000","last_peered":"2023-10-09T10:48:16.285672+0000","last_clean":"2023-10-09T10:48:16.285672+0000","last_became_active":"2023-10-09T10:46:05.616057+0000","last_became_peered":"2023-10-09T10:46:05.616057+0000","last_unstale":"2023-10-09T10:48:16.285672+0000","last_undegraded":"2023-10-09T10:48:16.285672+0000","last_fullsized":"2023-10-09T10:48:16.285672+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"184'11622","reported_seq":11726,"reported_epoch":184,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:26.043367+0000","last_change":"2023-10-09T10:46:33.576530+0000","last_active":"2023-10-09T10:48:26.043367+0000","last_peered":"2023-10-09T10:48:26.043367+0000","last_clean":"2023-10-09T10:46:31.183481+0000","last_became_active":"2023-10-09T10:46:33.542304+0000","last_became_peered":"2023-10-09T10:46:33.542304+0000","last_unstale":"2023-10-09T10:48:26.043367+0000","last_undegraded":"2023-10-09T10:46:33.300581+0000","last_fullsized":"2023-10-09T10:46:33.246047+0000","mapping_epoch":154,"log_start":"124'4277","ondisk_log_start":"124'4277","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7345,"log_dups_size":0,"ondisk_log_size":7345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:26.739488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105644032,"num_objects":1612,"num_object_clones":0,"num_object_copies":4836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6617,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11622,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1612},{"shards":"1,4","objects":-3949}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":176,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.252668+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:48:16.252668+0000","last_peered":"2023-10-09T10:48:16.252668+0000","last_clean":"2023-10-09T10:48:16.252668+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:48:16.252668+0000","last_undegraded":"2023-10-09T10:48:16.252668+0000","last_fullsized":"2023-10-09T10:48:16.252668+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":148,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.276097+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:48:17.276097+0000","last_peered":"2023-10-09T10:48:17.276097+0000","last_clean":"2023-10-09T10:48:17.276097+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:48:17.276097+0000","last_undegraded":"2023-10-09T10:48:17.276097+0000","last_fullsized":"2023-10-09T10:48:17.276097+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":498,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.264802+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:48:16.264802+0000","last_peered":"2023-10-09T10:48:16.264802+0000","last_clean":"2023-10-09T10:48:16.264802+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:48:16.264802+0000","last_undegraded":"2023-10-09T10:48:16.264802+0000","last_fullsized":"2023-10-09T10:48:16.264802+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.f","version":"183'11405","reported_seq":11506,"reported_epoch":183,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:24.579269+0000","last_change":"2023-10-09T10:46:33.651756+0000","last_active":"2023-10-09T10:48:24.579269+0000","last_peered":"2023-10-09T10:48:24.579269+0000","last_clean":"2023-10-09T10:46:31.156458+0000","last_became_active":"2023-10-09T10:46:33.621003+0000","last_became_peered":"2023-10-09T10:46:33.621003+0000","last_unstale":"2023-10-09T10:48:24.579269+0000","last_undegraded":"2023-10-09T10:46:33.309813+0000","last_fullsized":"2023-10-09T10:46:33.236508+0000","mapping_epoch":154,"log_start":"136'5710","ondisk_log_start":"136'5710","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5695,"log_dups_size":0,"ondisk_log_size":5695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:13.704346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105971712,"num_objects":1617,"num_object_clones":0,"num_object_copies":4851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11405,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":1617},{"shards":"3,4","objects":-3835}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":123,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275599+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:48:17.275599+0000","last_peered":"2023-10-09T10:48:17.275599+0000","last_clean":"2023-10-09T10:48:17.275599+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:48:17.275599+0000","last_undegraded":"2023-10-09T10:48:17.275599+0000","last_fullsized":"2023-10-09T10:48:17.275599+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":175,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.276719+0000","last_change":"2023-10-09T10:46:32.245583+0000","last_active":"2023-10-09T10:48:17.276719+0000","last_peered":"2023-10-09T10:48:17.276719+0000","last_clean":"2023-10-09T10:48:17.276719+0000","last_became_active":"2023-10-09T10:46:32.245271+0000","last_became_peered":"2023-10-09T10:46:32.245271+0000","last_unstale":"2023-10-09T10:48:17.276719+0000","last_undegraded":"2023-10-09T10:48:17.276719+0000","last_fullsized":"2023-10-09T10:48:17.276719+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.025051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":518,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.266791+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:48:17.266791+0000","last_peered":"2023-10-09T10:48:17.266791+0000","last_clean":"2023-10-09T10:48:17.266791+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:48:17.266791+0000","last_undegraded":"2023-10-09T10:48:17.266791+0000","last_fullsized":"2023-10-09T10:48:17.266791+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"183'11681","reported_seq":11776,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.606052+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:48:24.606052+0000","last_peered":"2023-10-09T10:48:24.606052+0000","last_clean":"2023-10-09T10:48:24.606052+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:48:24.606052+0000","last_undegraded":"2023-10-09T10:48:24.606052+0000","last_fullsized":"2023-10-09T10:48:24.606052+0000","mapping_epoch":98,"log_start":"136'5978","ondisk_log_start":"136'5978","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5703,"log_dups_size":0,"ondisk_log_size":5703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105054208,"num_objects":1603,"num_object_clones":0,"num_object_copies":4809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11681,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":159,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275500+0000","last_change":"2023-10-09T10:46:32.245685+0000","last_active":"2023-10-09T10:48:17.275500+0000","last_peered":"2023-10-09T10:48:17.275500+0000","last_clean":"2023-10-09T10:48:17.275500+0000","last_became_active":"2023-10-09T10:46:32.245314+0000","last_became_peered":"2023-10-09T10:46:32.245314+0000","last_unstale":"2023-10-09T10:48:17.275500+0000","last_undegraded":"2023-10-09T10:48:17.275500+0000","last_fullsized":"2023-10-09T10:48:17.275500+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:51.716860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":148,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275891+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:48:17.275891+0000","last_peered":"2023-10-09T10:48:17.275891+0000","last_clean":"2023-10-09T10:48:17.275891+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:48:17.275891+0000","last_undegraded":"2023-10-09T10:48:17.275891+0000","last_fullsized":"2023-10-09T10:48:17.275891+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":509,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.266378+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:48:16.266378+0000","last_peered":"2023-10-09T10:48:16.266378+0000","last_clean":"2023-10-09T10:48:16.266378+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:48:16.266378+0000","last_undegraded":"2023-10-09T10:48:16.266378+0000","last_fullsized":"2023-10-09T10:48:16.266378+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"184'11860","reported_seq":11974,"reported_epoch":184,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:26.072461+0000","last_change":"2023-10-09T10:46:02.754507+0000","last_active":"2023-10-09T10:48:26.072461+0000","last_peered":"2023-10-09T10:48:26.072461+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:46:02.747510+0000","last_became_peered":"2023-10-09T10:46:02.747510+0000","last_unstale":"2023-10-09T10:48:26.072461+0000","last_undegraded":"2023-10-09T10:46:02.605987+0000","last_fullsized":"2023-10-09T10:46:02.585248+0000","mapping_epoch":142,"log_start":"113'3000","ondisk_log_start":"113'3000","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8860,"log_dups_size":0,"ondisk_log_size":8860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:59.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100597784,"num_objects":1536,"num_object_clones":0,"num_object_copies":4608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1536,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11860,"num_write_kb":428609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,2,5","objects":1535},{"shards":"1,5","objects":-1182}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":153,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.254995+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:48:16.254995+0000","last_peered":"2023-10-09T10:48:16.254995+0000","last_clean":"2023-10-09T10:48:16.254995+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:48:16.254995+0000","last_undegraded":"2023-10-09T10:48:16.254995+0000","last_fullsized":"2023-10-09T10:48:16.254995+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":136,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267791+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:48:17.267791+0000","last_peered":"2023-10-09T10:48:17.267791+0000","last_clean":"2023-10-09T10:48:17.267791+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:48:17.267791+0000","last_undegraded":"2023-10-09T10:48:17.267791+0000","last_fullsized":"2023-10-09T10:48:17.267791+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":518,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.265874+0000","last_change":"2023-10-09T10:46:05.617983+0000","last_active":"2023-10-09T10:48:16.265874+0000","last_peered":"2023-10-09T10:48:16.265874+0000","last_clean":"2023-10-09T10:48:16.265874+0000","last_became_active":"2023-10-09T10:46:05.616021+0000","last_became_peered":"2023-10-09T10:46:05.616021+0000","last_unstale":"2023-10-09T10:48:16.265874+0000","last_undegraded":"2023-10-09T10:48:16.265874+0000","last_fullsized":"2023-10-09T10:48:16.265874+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":179,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267293+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:48:17.267293+0000","last_peered":"2023-10-09T10:48:17.267293+0000","last_clean":"2023-10-09T10:48:17.267293+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:48:17.267293+0000","last_undegraded":"2023-10-09T10:48:17.267293+0000","last_fullsized":"2023-10-09T10:48:17.267293+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":176,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286825+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:48:17.286825+0000","last_peered":"2023-10-09T10:48:17.286825+0000","last_clean":"2023-10-09T10:48:17.286825+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:48:17.286825+0000","last_undegraded":"2023-10-09T10:48:17.286825+0000","last_fullsized":"2023-10-09T10:48:17.286825+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":121,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267037+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:48:17.267037+0000","last_peered":"2023-10-09T10:48:17.267037+0000","last_clean":"2023-10-09T10:48:17.267037+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:48:17.267037+0000","last_undegraded":"2023-10-09T10:48:17.267037+0000","last_fullsized":"2023-10-09T10:48:17.267037+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"183'6845","reported_seq":10886,"reported_epoch":183,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:24.708429+0000","last_change":"2023-10-09T10:46:32.248218+0000","last_active":"2023-10-09T10:48:24.708429+0000","last_peered":"2023-10-09T10:48:24.708429+0000","last_clean":"2023-10-09T10:46:02.181912+0000","last_became_active":"2023-10-09T10:46:32.226237+0000","last_became_peered":"2023-10-09T10:46:32.226237+0000","last_unstale":"2023-10-09T10:48:24.708429+0000","last_undegraded":"2023-10-09T10:48:24.708429+0000","last_fullsized":"2023-10-09T10:48:24.708429+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6845,"log_dups_size":0,"ondisk_log_size":6845,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55476224,"num_objects":846,"num_object_clones":0,"num_object_copies":2538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":846,"num_objects_unfound":0,"num_objects_dirty":846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5798,"num_write_kb":212640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":846}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":517,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.280666+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:48:16.280666+0000","last_peered":"2023-10-09T10:48:16.280666+0000","last_clean":"2023-10-09T10:48:16.280666+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:48:16.280666+0000","last_undegraded":"2023-10-09T10:48:16.280666+0000","last_fullsized":"2023-10-09T10:48:16.280666+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"183'6943","reported_seq":7128,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.376136+0000","last_change":"2023-10-09T10:46:02.760963+0000","last_active":"2023-10-09T10:48:24.376136+0000","last_peered":"2023-10-09T10:48:24.376136+0000","last_clean":"2023-10-09T10:48:24.376136+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:48:24.376136+0000","last_undegraded":"2023-10-09T10:48:24.376136+0000","last_fullsized":"2023-10-09T10:48:24.376136+0000","mapping_epoch":98,"log_start":"103'518","ondisk_log_start":"103'518","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"141'4416","last_scrub_stamp":"2023-10-09T10:46:02.760810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:46:02.760810+0000","objects_scrubbed":3378,"log_size":6425,"log_dups_size":0,"ondisk_log_size":6425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:35.863002+0000","scrub_duration":0.47769323899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55771136,"num_objects":851,"num_object_clones":0,"num_object_copies":2553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2527,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":153,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286437+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:48:17.286437+0000","last_peered":"2023-10-09T10:48:17.286437+0000","last_clean":"2023-10-09T10:48:17.286437+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:48:17.286437+0000","last_undegraded":"2023-10-09T10:48:17.286437+0000","last_fullsized":"2023-10-09T10:48:17.286437+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":169,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.276047+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:48:17.276047+0000","last_peered":"2023-10-09T10:48:17.276047+0000","last_clean":"2023-10-09T10:48:17.276047+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:48:17.276047+0000","last_undegraded":"2023-10-09T10:48:17.276047+0000","last_fullsized":"2023-10-09T10:48:17.276047+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":160,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267390+0000","last_change":"2023-10-09T10:46:32.245608+0000","last_active":"2023-10-09T10:48:17.267390+0000","last_peered":"2023-10-09T10:48:17.267390+0000","last_clean":"2023-10-09T10:48:17.267390+0000","last_became_active":"2023-10-09T10:46:32.245290+0000","last_became_peered":"2023-10-09T10:46:32.245290+0000","last_unstale":"2023-10-09T10:48:17.267390+0000","last_undegraded":"2023-10-09T10:48:17.267390+0000","last_fullsized":"2023-10-09T10:48:17.267390+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:48.832515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":518,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.266421+0000","last_change":"2023-10-09T10:46:05.620057+0000","last_active":"2023-10-09T10:48:16.266421+0000","last_peered":"2023-10-09T10:48:16.266421+0000","last_clean":"2023-10-09T10:48:16.266421+0000","last_became_active":"2023-10-09T10:46:05.619703+0000","last_became_peered":"2023-10-09T10:46:05.619703+0000","last_unstale":"2023-10-09T10:48:16.266421+0000","last_undegraded":"2023-10-09T10:48:16.266421+0000","last_fullsized":"2023-10-09T10:48:16.266421+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"183'6889","reported_seq":7009,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.567031+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:48:24.567031+0000","last_peered":"2023-10-09T10:48:24.567031+0000","last_clean":"2023-10-09T10:48:24.567031+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:48:24.567031+0000","last_undegraded":"2023-10-09T10:48:24.567031+0000","last_fullsized":"2023-10-09T10:48:24.567031+0000","mapping_epoch":115,"log_start":"103'277","ondisk_log_start":"103'277","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6612,"log_dups_size":0,"ondisk_log_size":6612,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55836672,"num_objects":852,"num_object_clones":0,"num_object_copies":2556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5828,"num_write_kb":213760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":180,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.252038+0000","last_change":"2023-10-09T10:46:32.241041+0000","last_active":"2023-10-09T10:48:16.252038+0000","last_peered":"2023-10-09T10:48:16.252038+0000","last_clean":"2023-10-09T10:48:16.252038+0000","last_became_active":"2023-10-09T10:46:32.238683+0000","last_became_peered":"2023-10-09T10:46:32.238683+0000","last_unstale":"2023-10-09T10:48:16.252038+0000","last_undegraded":"2023-10-09T10:48:16.252038+0000","last_fullsized":"2023-10-09T10:48:16.252038+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:25.817713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":145,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.266963+0000","last_change":"2023-10-09T10:46:32.250104+0000","last_active":"2023-10-09T10:48:17.266963+0000","last_peered":"2023-10-09T10:48:17.266963+0000","last_clean":"2023-10-09T10:48:17.266963+0000","last_became_active":"2023-10-09T10:46:32.249805+0000","last_became_peered":"2023-10-09T10:46:32.249805+0000","last_unstale":"2023-10-09T10:48:17.266963+0000","last_undegraded":"2023-10-09T10:48:17.266963+0000","last_fullsized":"2023-10-09T10:48:17.266963+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:07.230672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":174,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267381+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:48:17.267381+0000","last_peered":"2023-10-09T10:48:17.267381+0000","last_clean":"2023-10-09T10:48:17.267381+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:48:17.267381+0000","last_undegraded":"2023-10-09T10:48:17.267381+0000","last_fullsized":"2023-10-09T10:48:17.267381+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"184'6834","reported_seq":6942,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:26.020299+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:48:26.020299+0000","last_peered":"2023-10-09T10:48:26.020299+0000","last_clean":"2023-10-09T10:48:26.020299+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:48:26.020299+0000","last_undegraded":"2023-10-09T10:48:26.020299+0000","last_fullsized":"2023-10-09T10:48:26.020299+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6834,"log_dups_size":0,"ondisk_log_size":6834,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54001664,"num_objects":824,"num_object_clones":0,"num_object_copies":2472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5826,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":163,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286437+0000","last_change":"2023-10-09T10:46:32.235346+0000","last_active":"2023-10-09T10:48:17.286437+0000","last_peered":"2023-10-09T10:48:17.286437+0000","last_clean":"2023-10-09T10:48:17.286437+0000","last_became_active":"2023-10-09T10:46:32.234906+0000","last_became_peered":"2023-10-09T10:46:32.234906+0000","last_unstale":"2023-10-09T10:48:17.286437+0000","last_undegraded":"2023-10-09T10:48:17.286437+0000","last_fullsized":"2023-10-09T10:48:17.286437+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.199251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":518,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.264535+0000","last_change":"2023-10-09T10:46:05.617146+0000","last_active":"2023-10-09T10:48:16.264535+0000","last_peered":"2023-10-09T10:48:16.264535+0000","last_clean":"2023-10-09T10:48:16.264535+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:48:16.264535+0000","last_undegraded":"2023-10-09T10:48:16.264535+0000","last_fullsized":"2023-10-09T10:48:16.264535+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":111,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.251935+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:48:16.251935+0000","last_peered":"2023-10-09T10:48:16.251935+0000","last_clean":"2023-10-09T10:48:16.251935+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:48:16.251935+0000","last_undegraded":"2023-10-09T10:48:16.251935+0000","last_fullsized":"2023-10-09T10:48:16.251935+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":81,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.276548+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:48:17.276548+0000","last_peered":"2023-10-09T10:48:17.276548+0000","last_clean":"2023-10-09T10:48:17.276548+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:48:17.276548+0000","last_undegraded":"2023-10-09T10:48:17.276548+0000","last_fullsized":"2023-10-09T10:48:17.276548+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":518,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.265078+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:48:16.265078+0000","last_peered":"2023-10-09T10:48:16.265078+0000","last_clean":"2023-10-09T10:48:16.265078+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:48:16.265078+0000","last_undegraded":"2023-10-09T10:48:16.265078+0000","last_fullsized":"2023-10-09T10:48:16.265078+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":175,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267086+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:48:17.267086+0000","last_peered":"2023-10-09T10:48:17.267086+0000","last_clean":"2023-10-09T10:48:17.267086+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:48:17.267086+0000","last_undegraded":"2023-10-09T10:48:17.267086+0000","last_fullsized":"2023-10-09T10:48:17.267086+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":61,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.280575+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:48:16.280575+0000","last_peered":"2023-10-09T10:48:16.280575+0000","last_clean":"2023-10-09T10:48:16.280575+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:48:16.280575+0000","last_undegraded":"2023-10-09T10:48:16.280575+0000","last_fullsized":"2023-10-09T10:48:16.280575+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":61,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.266283+0000","last_change":"2023-10-09T10:46:32.236202+0000","last_active":"2023-10-09T10:48:16.266283+0000","last_peered":"2023-10-09T10:48:16.266283+0000","last_clean":"2023-10-09T10:48:16.266283+0000","last_became_active":"2023-10-09T10:46:32.235901+0000","last_became_peered":"2023-10-09T10:46:32.235901+0000","last_unstale":"2023-10-09T10:48:16.266283+0000","last_undegraded":"2023-10-09T10:48:16.266283+0000","last_fullsized":"2023-10-09T10:48:16.266283+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:48.260403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"184'6970","reported_seq":7125,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:26.290514+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:48:26.290514+0000","last_peered":"2023-10-09T10:48:26.290514+0000","last_clean":"2023-10-09T10:48:26.290514+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:48:26.290514+0000","last_undegraded":"2023-10-09T10:48:26.290514+0000","last_fullsized":"2023-10-09T10:48:26.290514+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":6970,"log_dups_size":0,"ondisk_log_size":6970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52953088,"num_objects":808,"num_object_clones":0,"num_object_copies":2424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3940,"num_write_kb":89664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":518,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.266067+0000","last_change":"2023-10-09T10:46:05.620764+0000","last_active":"2023-10-09T10:48:16.266067+0000","last_peered":"2023-10-09T10:48:16.266067+0000","last_clean":"2023-10-09T10:48:16.266067+0000","last_became_active":"2023-10-09T10:46:05.619870+0000","last_became_peered":"2023-10-09T10:46:05.619870+0000","last_unstale":"2023-10-09T10:48:16.266067+0000","last_undegraded":"2023-10-09T10:48:16.266067+0000","last_fullsized":"2023-10-09T10:48:16.266067+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":172,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.274200+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:48:17.274200+0000","last_peered":"2023-10-09T10:48:17.274200+0000","last_clean":"2023-10-09T10:48:17.274200+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:48:17.274200+0000","last_undegraded":"2023-10-09T10:48:17.274200+0000","last_fullsized":"2023-10-09T10:48:17.274200+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":145,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267482+0000","last_change":"2023-10-09T10:46:32.251450+0000","last_active":"2023-10-09T10:48:17.267482+0000","last_peered":"2023-10-09T10:48:17.267482+0000","last_clean":"2023-10-09T10:48:17.267482+0000","last_became_active":"2023-10-09T10:46:32.250933+0000","last_became_peered":"2023-10-09T10:46:32.250933+0000","last_unstale":"2023-10-09T10:48:17.267482+0000","last_undegraded":"2023-10-09T10:48:17.267482+0000","last_fullsized":"2023-10-09T10:48:17.267482+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:04.116178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":52,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.265159+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:48:16.265159+0000","last_peered":"2023-10-09T10:48:16.265159+0000","last_clean":"2023-10-09T10:48:16.265159+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:48:16.265159+0000","last_undegraded":"2023-10-09T10:48:16.265159+0000","last_fullsized":"2023-10-09T10:48:16.265159+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"184'6795","reported_seq":13495,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:26.463151+0000","last_change":"2023-10-09T10:47:02.269289+0000","last_active":"2023-10-09T10:48:26.463151+0000","last_peered":"2023-10-09T10:48:26.463151+0000","last_clean":"2023-10-09T10:48:26.463151+0000","last_became_active":"2023-10-09T10:47:02.257854+0000","last_became_peered":"2023-10-09T10:47:02.257854+0000","last_unstale":"2023-10-09T10:48:26.463151+0000","last_undegraded":"2023-10-09T10:48:26.463151+0000","last_fullsized":"2023-10-09T10:48:26.463151+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6795,"log_dups_size":0,"ondisk_log_size":6795,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53444608,"num_objects":815,"num_object_clones":0,"num_object_copies":2445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5787,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3287,"num_bytes_recovered":181927936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":518,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.266260+0000","last_change":"2023-10-09T10:46:05.618443+0000","last_active":"2023-10-09T10:48:16.266260+0000","last_peered":"2023-10-09T10:48:16.266260+0000","last_clean":"2023-10-09T10:48:16.266260+0000","last_became_active":"2023-10-09T10:46:05.616588+0000","last_became_peered":"2023-10-09T10:46:05.616588+0000","last_unstale":"2023-10-09T10:48:16.266260+0000","last_undegraded":"2023-10-09T10:48:16.266260+0000","last_fullsized":"2023-10-09T10:48:16.266260+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":176,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.252898+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:48:16.252898+0000","last_peered":"2023-10-09T10:48:16.252898+0000","last_clean":"2023-10-09T10:48:16.252898+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:48:16.252898+0000","last_undegraded":"2023-10-09T10:48:16.252898+0000","last_fullsized":"2023-10-09T10:48:16.252898+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":121,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267491+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:48:17.267491+0000","last_peered":"2023-10-09T10:48:17.267491+0000","last_clean":"2023-10-09T10:48:17.267491+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:48:17.267491+0000","last_undegraded":"2023-10-09T10:48:17.267491+0000","last_fullsized":"2023-10-09T10:48:17.267491+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":165,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.255076+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:48:16.255076+0000","last_peered":"2023-10-09T10:48:16.255076+0000","last_clean":"2023-10-09T10:48:16.255076+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:48:16.255076+0000","last_undegraded":"2023-10-09T10:48:16.255076+0000","last_fullsized":"2023-10-09T10:48:16.255076+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"184'6929","reported_seq":13769,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:25.947684+0000","last_change":"2023-10-09T10:48:17.301310+0000","last_active":"2023-10-09T10:48:25.947684+0000","last_peered":"2023-10-09T10:48:25.947684+0000","last_clean":"2023-10-09T10:48:25.947684+0000","last_became_active":"2023-10-09T10:48:17.283042+0000","last_became_peered":"2023-10-09T10:48:17.283042+0000","last_unstale":"2023-10-09T10:48:25.947684+0000","last_undegraded":"2023-10-09T10:48:25.947684+0000","last_fullsized":"2023-10-09T10:48:25.947684+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6929,"log_dups_size":0,"ondisk_log_size":6929,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50626560,"num_objects":772,"num_object_clones":0,"num_object_copies":2316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5892,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3337,"num_bytes_recovered":129564672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":517,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.265347+0000","last_change":"2023-10-09T10:46:05.617589+0000","last_active":"2023-10-09T10:48:16.265347+0000","last_peered":"2023-10-09T10:48:16.265347+0000","last_clean":"2023-10-09T10:48:16.265347+0000","last_became_active":"2023-10-09T10:46:05.616400+0000","last_became_peered":"2023-10-09T10:46:05.616400+0000","last_unstale":"2023-10-09T10:48:16.265347+0000","last_undegraded":"2023-10-09T10:48:16.265347+0000","last_fullsized":"2023-10-09T10:48:16.265347+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":40,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.252640+0000","last_change":"2023-10-09T10:46:32.245106+0000","last_active":"2023-10-09T10:48:16.252640+0000","last_peered":"2023-10-09T10:48:16.252640+0000","last_clean":"2023-10-09T10:48:16.252640+0000","last_became_active":"2023-10-09T10:46:32.244826+0000","last_became_peered":"2023-10-09T10:46:32.244826+0000","last_unstale":"2023-10-09T10:48:16.252640+0000","last_undegraded":"2023-10-09T10:48:16.252640+0000","last_fullsized":"2023-10-09T10:48:16.252640+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:30.322834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":56,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.266176+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:48:16.266176+0000","last_peered":"2023-10-09T10:48:16.266176+0000","last_clean":"2023-10-09T10:48:16.266176+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:48:16.266176+0000","last_undegraded":"2023-10-09T10:48:16.266176+0000","last_fullsized":"2023-10-09T10:48:16.266176+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":131,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275421+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:48:17.275421+0000","last_peered":"2023-10-09T10:48:17.275421+0000","last_clean":"2023-10-09T10:48:17.275421+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:48:17.275421+0000","last_undegraded":"2023-10-09T10:48:17.275421+0000","last_fullsized":"2023-10-09T10:48:17.275421+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"184'6833","reported_seq":13646,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:26.016142+0000","last_change":"2023-10-09T10:47:31.271196+0000","last_active":"2023-10-09T10:48:26.016142+0000","last_peered":"2023-10-09T10:48:26.016142+0000","last_clean":"2023-10-09T10:48:26.016142+0000","last_became_active":"2023-10-09T10:47:31.229504+0000","last_became_peered":"2023-10-09T10:47:31.229504+0000","last_unstale":"2023-10-09T10:48:26.016142+0000","last_undegraded":"2023-10-09T10:48:26.016142+0000","last_fullsized":"2023-10-09T10:48:26.016142+0000","mapping_epoch":172,"log_start":"108'2115","ondisk_log_start":"108'2115","created":98,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":4718,"log_dups_size":0,"ondisk_log_size":4718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:26.533476+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51970048,"num_objects":793,"num_object_clones":0,"num_object_copies":2379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4188,"num_write_kb":107264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3305,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":51,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.265201+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:48:16.265201+0000","last_peered":"2023-10-09T10:48:16.265201+0000","last_clean":"2023-10-09T10:48:16.265201+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:48:16.265201+0000","last_undegraded":"2023-10-09T10:48:16.265201+0000","last_fullsized":"2023-10-09T10:48:16.265201+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":115,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.269650+0000","last_change":"2023-10-09T10:46:32.251242+0000","last_active":"2023-10-09T10:48:17.269650+0000","last_peered":"2023-10-09T10:48:17.269650+0000","last_clean":"2023-10-09T10:48:17.269650+0000","last_became_active":"2023-10-09T10:46:32.250534+0000","last_became_peered":"2023-10-09T10:46:32.250534+0000","last_unstale":"2023-10-09T10:48:17.269650+0000","last_undegraded":"2023-10-09T10:48:17.269650+0000","last_fullsized":"2023-10-09T10:48:17.269650+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:33.726041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"183'6804","reported_seq":11777,"reported_epoch":183,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:22.745731+0000","last_change":"2023-10-09T10:46:34.514479+0000","last_active":"2023-10-09T10:48:22.745731+0000","last_peered":"2023-10-09T10:48:22.745731+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:46:34.409667+0000","last_became_peered":"2023-10-09T10:46:34.409667+0000","last_unstale":"2023-10-09T10:48:22.745731+0000","last_undegraded":"2023-10-09T10:46:34.376635+0000","last_fullsized":"2023-10-09T10:46:33.237998+0000","mapping_epoch":154,"log_start":"104'1224","ondisk_log_start":"104'1224","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5580,"log_dups_size":0,"ondisk_log_size":5580,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59768832,"num_objects":912,"num_object_clones":0,"num_object_copies":2736,"num_objects_missing_on_primary":913,"num_objects_missing":913,"num_objects_degraded":1423,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5762,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2425,"num_bytes_recovered":158924800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,7","objects":-1},{"shards":"2,3,7","objects":913}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":151,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.287529+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:48:17.287529+0000","last_peered":"2023-10-09T10:48:17.287529+0000","last_clean":"2023-10-09T10:48:17.287529+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:48:17.287529+0000","last_undegraded":"2023-10-09T10:48:17.287529+0000","last_fullsized":"2023-10-09T10:48:17.287529+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":165,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.276170+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:48:17.276170+0000","last_peered":"2023-10-09T10:48:17.276170+0000","last_clean":"2023-10-09T10:48:17.276170+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:48:17.276170+0000","last_undegraded":"2023-10-09T10:48:17.276170+0000","last_fullsized":"2023-10-09T10:48:17.276170+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"183'6823","reported_seq":6938,"reported_epoch":183,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:24.488474+0000","last_change":"2023-10-09T10:46:33.546991+0000","last_active":"2023-10-09T10:48:24.488474+0000","last_peered":"2023-10-09T10:48:24.488474+0000","last_clean":"2023-10-09T10:46:31.009484+0000","last_became_active":"2023-10-09T10:46:33.535665+0000","last_became_peered":"2023-10-09T10:46:33.535665+0000","last_unstale":"2023-10-09T10:48:24.488474+0000","last_undegraded":"2023-10-09T10:46:33.277168+0000","last_fullsized":"2023-10-09T10:46:33.235038+0000","mapping_epoch":154,"log_start":"103'232","ondisk_log_start":"103'232","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6591,"log_dups_size":0,"ondisk_log_size":6591,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50364416,"num_objects":769,"num_object_clones":0,"num_object_copies":2307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5759,"num_write_kb":208864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":781},{"shards":"3,7","objects":-1989}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":134,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286903+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:48:17.286903+0000","last_peered":"2023-10-09T10:48:17.286903+0000","last_clean":"2023-10-09T10:48:17.286903+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:48:17.286903+0000","last_undegraded":"2023-10-09T10:48:17.286903+0000","last_fullsized":"2023-10-09T10:48:17.286903+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":32,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275519+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:48:17.275519+0000","last_peered":"2023-10-09T10:48:17.275519+0000","last_clean":"2023-10-09T10:48:17.275519+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:48:17.275519+0000","last_undegraded":"2023-10-09T10:48:17.275519+0000","last_fullsized":"2023-10-09T10:48:17.275519+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":167,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.255968+0000","last_change":"2023-10-09T10:46:32.244607+0000","last_active":"2023-10-09T10:48:16.255968+0000","last_peered":"2023-10-09T10:48:16.255968+0000","last_clean":"2023-10-09T10:48:16.255968+0000","last_became_active":"2023-10-09T10:46:32.242628+0000","last_became_peered":"2023-10-09T10:46:32.242628+0000","last_unstale":"2023-10-09T10:48:16.255968+0000","last_undegraded":"2023-10-09T10:48:16.255968+0000","last_fullsized":"2023-10-09T10:48:16.255968+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:10.738819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":198,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.278037+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:48:17.278037+0000","last_peered":"2023-10-09T10:48:17.278037+0000","last_clean":"2023-10-09T10:48:17.278037+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:48:17.278037+0000","last_undegraded":"2023-10-09T10:48:17.278037+0000","last_fullsized":"2023-10-09T10:48:17.278037+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":73,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-10-09T10:48:16.265826+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:48:16.265826+0000","last_peered":"2023-10-09T10:48:16.265826+0000","last_clean":"2023-10-09T10:48:16.265826+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:48:16.265826+0000","last_undegraded":"2023-10-09T10:48:16.265826+0000","last_fullsized":"2023-10-09T10:48:16.265826+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":148,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275934+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:48:17.275934+0000","last_peered":"2023-10-09T10:48:17.275934+0000","last_clean":"2023-10-09T10:48:17.275934+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:48:17.275934+0000","last_undegraded":"2023-10-09T10:48:17.275934+0000","last_fullsized":"2023-10-09T10:48:17.275934+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":1683521560,"num_objects":25690,"num_object_clones":0,"num_object_copies":77070,"num_objects_missing_on_primary":913,"num_objects_missing":913,"num_objects_degraded":27736,"num_objects_misplaced":2869,"num_objects_unfound":0,"num_objects_dirty":25690,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":178518,"num_write_kb":6258529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26514,"num_bytes_recovered":1539637248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7334211584,"data_stored":7334199368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170794,"ondisk_log_size":170794,"up":78,"acting":69,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1186,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332262,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1030024,"kb_used_data":720448,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92715128,"statfs":{"total":95995035648,"available":94940291072,"internally_reserved":0,"allocated":737738752,"data_stored":737038229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":0.79200000000000004,"15min":0.65000000000000002},"min":{"1min":0.26300000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.1479999999999997,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":0.23499999999999999},{"interface":"front","average":{"1min":1.546,"5min":0.97199999999999998,"15min":0.752},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.9909999999999997,"5min":10.207000000000001,"15min":10.207000000000001},"last":9.8030000000000008}]},{"osd":1,"last update":"Mon Oct 9 10:47:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2210000000000001,"5min":1.3300000000000001,"15min":0.80400000000000005},"min":{"1min":0.43099999999999999,"5min":0.318,"15min":0.318},"max":{"1min":11.826000000000001,"5min":11.826000000000001,"15min":11.826000000000001},"last":9.8469999999999995},{"interface":"front","average":{"1min":2.286,"5min":1.155,"15min":0.73899999999999999},"min":{"1min":0.32100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":11.84,"5min":11.84,"15min":11.84},"last":0.26100000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:48:18 2023","interfaces":[{"interface":"back","average":{"1min":2.165,"5min":1.4810000000000001,"15min":0.83499999999999996},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.1419999999999995,"5min":9.1419999999999995,"15min":9.1419999999999995},"last":4.6769999999999996},{"interface":"front","average":{"1min":2.1970000000000001,"5min":1.1910000000000001,"15min":0.747},"min":{"1min":0.34999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":7.8120000000000003,"5min":7.8120000000000003,"15min":7.8120000000000003},"last":0.28699999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.23,"15min":0.80000000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.0599999999999996,"5min":8.1639999999999997,"15min":8.1639999999999997},"last":9.8759999999999994},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.099,"15min":0.76600000000000001},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.504,"5min":12.451000000000001,"15min":12.451000000000001},"last":4.6520000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":1.1579999999999999,"15min":0.70499999999999996},"min":{"1min":0.36399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":9.4900000000000002,"5min":9.4900000000000002,"15min":9.4900000000000002},"last":10.009},{"interface":"front","average":{"1min":2.327,"5min":1.018,"15min":0.66100000000000003},"min":{"1min":0.35399999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.0999999999999996,"5min":9.0999999999999996,"15min":9.0999999999999996},"last":4.5890000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.0249999999999999,"5min":1.0609999999999999,"15min":0.69599999999999995},"min":{"1min":0.314,"5min":0.247,"15min":0.247},"max":{"1min":12.064,"5min":12.064,"15min":12.064},"last":4.6260000000000003},{"interface":"front","average":{"1min":3.8839999999999999,"5min":1.4770000000000001,"15min":0.85299999999999998},"min":{"1min":0.435,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":11.741,"5min":11.741,"15min":11.741},"last":9.9100000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2999999999999998,"5min":1.3720000000000001,"15min":0.83199999999999996},"min":{"1min":0.33700000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.029,"5min":13.029,"15min":13.029},"last":9.9339999999999993},{"interface":"front","average":{"1min":2.6869999999999998,"5min":1.079,"15min":0.72699999999999998},"min":{"1min":0.41299999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":19.187999999999999,"5min":19.187999999999999,"15min":19.187999999999999},"last":9.2850000000000001}]}]},{"osd":6,"up_from":44,"seq":188978561193,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":878724,"kb_used_data":573244,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92866428,"statfs":{"total":95995035648,"available":95095222272,"internally_reserved":0,"allocated":587001856,"data_stored":586305871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.93200000000000005,"15min":0.71799999999999997},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.4630000000000001,"5min":7.3220000000000001,"15min":7.3220000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":1.02,"5min":1.002,"15min":0.73899999999999999},"min":{"1min":0.33900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.4969999999999999,"5min":7.5309999999999997,"15min":7.5309999999999997},"last":0.29999999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.157,"15min":0.77900000000000003},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.245},"max":{"1min":6.7560000000000002,"5min":16.747,"15min":16.747},"last":3.3879999999999999},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.284,"15min":0.79500000000000004},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.3579999999999997,"5min":16.734000000000002,"15min":16.734000000000002},"last":3.3690000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:48:20 2023","interfaces":[{"interface":"back","average":{"1min":2.798,"5min":1.411,"15min":0.82599999999999996},"min":{"1min":0.29699999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.208,"5min":13.208,"15min":13.208},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.9420000000000002,"5min":1.619,"15min":0.91300000000000003},"min":{"1min":0.309,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.6749999999999998,"5min":12.571,"15min":12.571},"last":0.51500000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.117,"15min":0.73099999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.8070000000000004,"5min":6.8070000000000004,"15min":6.8070000000000004},"last":0.90100000000000002},{"interface":"front","average":{"1min":2.1219999999999999,"5min":1.385,"15min":0.82599999999999996},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.3940000000000001,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":0.39600000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.0629999999999999,"15min":0.65000000000000002},"min":{"1min":0.29899999999999999,"5min":0.255,"15min":0.255},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":0.438},{"interface":"front","average":{"1min":1.6579999999999999,"5min":0.89600000000000002,"15min":0.623},"min":{"1min":0.34100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.1260000000000003,"5min":5.1260000000000003,"15min":5.1260000000000003},"last":0.39000000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":1.1719999999999999,"15min":0.71099999999999997},"min":{"1min":0.35899999999999999,"5min":0.29499999999999998,"15min":0.27800000000000002},"max":{"1min":10.497,"5min":10.497,"15min":10.497},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.589,"5min":0.96599999999999997,"15min":0.67100000000000004},"min":{"1min":0.40699999999999997,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":6.8419999999999996,"5min":6.8419999999999996,"15min":6.8419999999999996},"last":0.32900000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:47:25 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69899999999999995,"15min":0.57999999999999996},"min":{"1min":0.35799999999999998,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":2.5049999999999999,"5min":3.472,"15min":3.472},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.84299999999999997,"15min":0.65700000000000003},"min":{"1min":0.373,"5min":0.36899999999999999,"15min":0.34000000000000002},"max":{"1min":4.0800000000000001,"5min":4.0800000000000001,"15min":4.0800000000000001},"last":0.35099999999999998}]}]},{"osd":1,"up_from":85,"seq":365072220232,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":856200,"kb_used_data":730304,"kb_used_omap":9,"kb_used_meta":125878,"kb_avail":92888952,"statfs":{"total":95995035648,"available":95118286848,"internally_reserved":0,"allocated":747831296,"data_stored":746900100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":128900073},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.581,"15min":0.76900000000000002},"min":{"1min":0.251,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.2119999999999997,"5min":14.1,"15min":14.1},"last":0.26800000000000002},{"interface":"front","average":{"1min":1.885,"5min":1.5629999999999999,"15min":0.78700000000000003},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.8090000000000002,"5min":14.465,"15min":14.465},"last":7.0190000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:48:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":1.492,"15min":0.74199999999999999},"min":{"1min":0.221,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":15.356,"5min":15.356,"15min":15.356},"last":1.3420000000000001},{"interface":"front","average":{"1min":1.488,"5min":1.6890000000000001,"15min":0.81899999999999995},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.4130000000000003,"5min":14.234,"15min":14.234},"last":0.42199999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":1.5269999999999999,"15min":0.82299999999999995},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":7.234,"5min":14.404999999999999,"15min":14.404999999999999},"last":0.218},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.429,"15min":0.70599999999999996},"min":{"1min":0.30399999999999999,"5min":0.251,"15min":0.251},"max":{"1min":15.468999999999999,"5min":15.468999999999999,"15min":15.468999999999999},"last":7.109}]},{"osd":4,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.1679999999999999,"15min":0.78400000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.1449999999999996,"5min":8.0129999999999999,"15min":8.0129999999999999},"last":3.3769999999999998},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.2350000000000001,"15min":0.82499999999999996},"min":{"1min":0.30299999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.1189999999999998,"5min":8.327,"15min":8.327},"last":0.309}]},{"osd":5,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":1.111,"15min":0.78900000000000003},"min":{"1min":0.28899999999999998,"5min":0.253,"15min":0.253},"max":{"1min":5.4850000000000003,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.2929999999999999,"15min":0.86199999999999999},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.1859999999999999,"5min":7.923,"15min":7.923},"last":1.3240000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7909999999999999,"5min":1.2030000000000001,"15min":0.82199999999999995},"min":{"1min":0.35499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.9450000000000003,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":1.0880000000000001},{"interface":"front","average":{"1min":1.246,"5min":1.2949999999999999,"15min":0.86299999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.8440000000000003,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":1.222}]},{"osd":7,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.22,"15min":0.83999999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.8689999999999998,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":1.248},{"interface":"front","average":{"1min":1.992,"5min":1.2909999999999999,"15min":0.86099999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":7.1589999999999998,"5min":8.2469999999999999,"15min":8.2469999999999999},"last":0.32500000000000001}]}]},{"osd":0,"up_from":80,"seq":343597383765,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":460572,"kb_used_data":348564,"kb_used_omap":28,"kb_used_meta":111971,"kb_avail":93284580,"statfs":{"total":95995035648,"available":95523409920,"internally_reserved":0,"allocated":356929536,"data_stored":355969557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":114659152},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:47:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2509999999999999,"5min":1.5449999999999999,"15min":0.76000000000000001},"min":{"1min":0.22,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":10.564,"5min":25.475000000000001,"15min":25.475000000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":2.3740000000000001,"5min":1.5940000000000001,"15min":0.77400000000000002},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":10.593999999999999,"5min":20.832999999999998,"15min":20.832999999999998},"last":0.60399999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":1.4750000000000001,"15min":0.72399999999999998},"min":{"1min":0.249,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.0890000000000004,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":7.0890000000000004},{"interface":"front","average":{"1min":2.1120000000000001,"5min":1.889,"15min":0.85799999999999998},"min":{"1min":0.27100000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.0329999999999995,"5min":14.391999999999999,"15min":14.391999999999999},"last":9.0329999999999995}]},{"osd":3,"last update":"Mon Oct 9 10:47:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4740000000000002,"5min":1.8959999999999999,"15min":1.0660000000000001},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.215},"max":{"1min":10.644,"5min":14.926,"15min":14.926},"last":0.44800000000000001},{"interface":"front","average":{"1min":2.2269999999999999,"5min":1.5940000000000001,"15min":0.89200000000000002},"min":{"1min":0.34499999999999997,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.228999999999999,"5min":11.868,"15min":11.868},"last":0.77800000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.2250000000000001,"15min":0.82199999999999995},"min":{"1min":0.38400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.8150000000000004,"5min":6.8150000000000004,"15min":6.8150000000000004},"last":0.64500000000000002},{"interface":"front","average":{"1min":2.391,"5min":1.405,"15min":0.88100000000000001},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.676,"5min":10.676,"15min":10.676},"last":0.57199999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":1.4359999999999999,"15min":0.89600000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":10.734,"5min":10.734,"15min":10.734},"last":3.379},{"interface":"front","average":{"1min":1.716,"5min":1.1659999999999999,"15min":0.82099999999999995},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.54,"5min":8.9190000000000005,"15min":8.9190000000000005},"last":1.7569999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.391,"15min":0.88400000000000001},"min":{"1min":0.36599999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.706,"5min":10.706,"15min":10.706},"last":0.84199999999999997},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.0780000000000001,"15min":0.79300000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.0499999999999998,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":0.439}]},{"osd":7,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.474,"5min":1.151,"15min":0.81899999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.5380000000000003,"5min":8.875,"15min":8.875},"last":7.0419999999999998},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.425,"15min":0.92700000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":14.394,"5min":14.394,"15min":14.394},"last":0.67300000000000004}]}]},{"osd":2,"up_from":0,"seq":386547056706,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893180,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1428680,"kb_used_data":1189248,"kb_used_omap":7,"kb_used_meta":239416,"kb_avail":92316472,"statfs":{"total":95995035648,"available":94532067328,"internally_reserved":0,"allocated":1217789952,"data_stored":1216848437,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":245162600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":1.3140000000000001,"15min":0.65200000000000002},"min":{"1min":0.192,"5min":0.183,"15min":0.183},"max":{"1min":7.0289999999999999,"5min":11.557,"15min":11.557},"last":0.65800000000000003},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.1379999999999999,"15min":0.65000000000000002},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":6.681,"5min":7.0510000000000002,"15min":7.0510000000000002},"last":3.7389999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":1.6970000000000001,"15min":0.78500000000000003},"min":{"1min":0.27100000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.048,"5min":14.707000000000001,"15min":14.707000000000001},"last":2.8719999999999999},{"interface":"front","average":{"1min":1.6220000000000001,"5min":1.1890000000000001,"15min":0.64500000000000002},"min":{"1min":0.19400000000000001,"5min":0.193,"15min":0.193},"max":{"1min":7.1630000000000003,"5min":11.629,"15min":11.629},"last":5.0090000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.6739999999999999,"15min":0.80200000000000005},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.9180000000000001,"5min":13.853,"15min":13.853},"last":0.23699999999999999},{"interface":"front","average":{"1min":2.5590000000000002,"5min":1.5129999999999999,"15min":0.76800000000000002},"min":{"1min":0.30099999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.71,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":3.6419999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.2210000000000001,"15min":0.78800000000000003},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.9909999999999997,"5min":11.532,"15min":11.532},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.2440000000000002,"5min":1.296,"15min":0.83699999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.962999999999999,"5min":11.962999999999999,"15min":11.962999999999999},"last":3.71}]},{"osd":5,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7589999999999999,"5min":1.8029999999999999,"15min":0.99199999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.907,"5min":51.509999999999998,"15min":51.509999999999998},"last":3.6850000000000001},{"interface":"front","average":{"1min":2.1469999999999998,"5min":2.306,"15min":1.1439999999999999},"min":{"1min":0.36799999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.1139999999999999,"5min":55.338999999999999,"15min":55.338999999999999},"last":0.41599999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":1.3620000000000001,"15min":0.84599999999999997},"min":{"1min":0.378,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":11.92,"5min":11.92,"15min":11.92},"last":3.7639999999999998},{"interface":"front","average":{"1min":2.3759999999999999,"5min":1.776,"15min":0.99199999999999999},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":10.528,"5min":13.891999999999999,"15min":13.891999999999999},"last":3.5070000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":2.3410000000000002,"15min":1.175},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.6520000000000001,"5min":51.744,"15min":51.744},"last":0.435},{"interface":"front","average":{"1min":1.9139999999999999,"5min":1.869,"15min":1.0049999999999999},"min":{"1min":0.33000000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.133,"5min":53.399000000000001,"15min":53.399000000000001},"last":0.56399999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019056,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":810308,"kb_used_data":508924,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92934844,"statfs":{"total":95995035648,"available":95165280256,"internally_reserved":0,"allocated":521138176,"data_stored":520441458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.0129999999999999,"15min":0.70899999999999996},"min":{"1min":0.24099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.5419999999999998,"5min":10.387,"15min":10.387},"last":2.387},{"interface":"front","average":{"1min":1.581,"5min":1.1819999999999999,"15min":0.78400000000000003},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.5999999999999996,"5min":10.311999999999999,"15min":10.311999999999999},"last":4.2080000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.1519999999999999,"15min":0.752},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.9100000000000001,"5min":6.907,"15min":6.907},"last":2.472},{"interface":"front","average":{"1min":1.859,"5min":1.2390000000000001,"15min":0.82699999999999996},"min":{"1min":0.30099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.7669999999999995,"5min":8.7669999999999995,"15min":8.7669999999999995},"last":2.254}]},{"osd":2,"last update":"Mon Oct 9 10:48:13 2023","interfaces":[{"interface":"back","average":{"1min":3.1520000000000001,"5min":1.617,"15min":0.91900000000000004},"min":{"1min":0.52100000000000002,"5min":0.253,"15min":0.253},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":4.1340000000000003},{"interface":"front","average":{"1min":4.016,"5min":1.855,"15min":0.99299999999999999},"min":{"1min":0.36799999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":20.545999999999999,"5min":20.545999999999999,"15min":20.545999999999999},"last":4.1710000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:47:38 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.333,"15min":0.91900000000000004},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.8170000000000002,"5min":8.8170000000000002,"15min":8.8170000000000002},"last":5.8890000000000002},{"interface":"front","average":{"1min":2.3650000000000002,"5min":1.359,"15min":0.84499999999999997},"min":{"1min":0.34300000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.625,"5min":8.9809999999999999,"15min":8.9809999999999999},"last":4.2389999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":0.89500000000000002,"15min":0.59699999999999998},"min":{"1min":0.25600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":8.6950000000000003,"5min":8.6950000000000003,"15min":8.6950000000000003},"last":2.5489999999999999},{"interface":"front","average":{"1min":1.6579999999999999,"5min":0.94299999999999995,"15min":0.61299999999999999},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.252},"max":{"1min":4.8879999999999999,"5min":4.8879999999999999,"15min":4.8879999999999999},"last":8.4499999999999993}]},{"osd":6,"last update":"Mon Oct 9 10:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":1.1319999999999999,"15min":0.69399999999999995},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":12.391999999999999,"5min":12.391999999999999,"15min":12.391999999999999},"last":2.516},{"interface":"front","average":{"1min":3.1200000000000001,"5min":1.2130000000000001,"15min":0.78100000000000003},"min":{"1min":0.374,"5min":0.311,"15min":0.311},"max":{"1min":12.539,"5min":12.539,"15min":12.539},"last":2.347}]},{"osd":7,"last update":"Mon Oct 9 10:48:21 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":1.2929999999999999,"15min":0.80900000000000005},"min":{"1min":0.33300000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.7919999999999998,"5min":8.7919999999999998,"15min":8.7919999999999998},"last":2.3119999999999998},{"interface":"front","average":{"1min":3.2389999999999999,"5min":1.27,"15min":0.79000000000000004},"min":{"1min":0.55900000000000005,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.423,"5min":7.423,"15min":7.423},"last":4.2629999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790125,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":962420,"kb_used_data":660652,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92782732,"statfs":{"total":95995035648,"available":95009517568,"internally_reserved":0,"allocated":676507648,"data_stored":675822759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.77600000000000002,"15min":0.65700000000000003},"min":{"1min":0.247,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.9860000000000002,"5min":4.8380000000000001,"15min":4.8380000000000001},"last":0.504},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.84899999999999998,"15min":0.72899999999999998},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.0209999999999999,"5min":4.8650000000000002,"15min":4.8650000000000002},"last":0.53000000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":1.367,"15min":0.86099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":12.999000000000001,"5min":12.999000000000001,"15min":12.999000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.2370000000000001,"15min":0.80700000000000005},"min":{"1min":0.27600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3730000000000002,"5min":7.3730000000000002,"15min":7.3730000000000002},"last":0.47399999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:48:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6669999999999998,"5min":1.4390000000000001,"15min":0.86099999999999999},"min":{"1min":0.34799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.3280000000000003,"5min":7.9000000000000004,"15min":7.9000000000000004},"last":0.755},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.2350000000000001,"15min":0.79400000000000004},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.3059999999999992,"5min":10.352,"15min":10.352},"last":0.39000000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4969999999999999,"5min":1.621,"15min":1.0229999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":18.016999999999999,"5min":18.016999999999999,"15min":18.016999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.964,"5min":1.673,"15min":0.96599999999999997},"min":{"1min":0.318,"5min":0.317,"15min":0.317},"max":{"1min":15.555999999999999,"5min":15.555999999999999,"15min":15.555999999999999},"last":0.81499999999999995}]},{"osd":4,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":0.83799999999999997,"15min":0.57799999999999996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":19.152999999999999,"5min":19.152999999999999,"15min":19.152999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.986,"5min":0.96999999999999997,"15min":0.63500000000000001},"min":{"1min":0.26800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.3059999999999992,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":0.36699999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":1.127,"15min":0.69899999999999995},"min":{"1min":0.33100000000000002,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":7.7060000000000004,"5min":7.7060000000000004,"15min":7.7060000000000004},"last":6.2220000000000004},{"interface":"front","average":{"1min":1.958,"5min":0.90400000000000003,"15min":0.64000000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.30299999999999999},"max":{"1min":8.0909999999999993,"5min":8.0909999999999993,"15min":8.0909999999999993},"last":0.432}]},{"osd":7,"last update":"Mon Oct 9 10:47:22 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.67100000000000004,"15min":0.58799999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.4729999999999999,"5min":4.0510000000000002,"15min":4.0510000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.70899999999999996,"15min":0.60099999999999998},"min":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":4.1289999999999996,"5min":4.1289999999999996,"15min":4.1289999999999996},"last":0.56100000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":350949376,"data_stored":350945304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":736956416,"data_stored":736952344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2538078208,"data_stored":2538078208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1215299584,"data_stored":1215299584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":517341184,"data_stored":517341184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":668143616,"data_stored":668139544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":582942720,"data_stored":582942720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":724500480,"data_stored":724500480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:48:29.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:29.131 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:29.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:29.232 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:29.267 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:29.281 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:29.290 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:29.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:29.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:29.393 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:29.450 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:29.619 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:29 smithi125 ceph-mon[195507]: from='client.35064 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:29.619 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:29.619 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:29 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:29.619 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:29 smithi125 ceph-mon[200197]: from='client.35064 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:29.619 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:29.619 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:29 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:29.657 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:29.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:29.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:29.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:29 smithi155 ceph-mon[184854]: from='client.35064 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:29.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:29.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:29 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:29.769 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:29.816 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:29.842 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:29.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:29.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:29.867 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:48:29.867 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:48:29.867 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:48:29.867 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:48:29.867 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:48:29.946 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:29.996 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:30.031 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:30.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:30.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:30.136 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:30.196 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:30.242 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:30.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:30.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:30.352 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:30.411 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:30.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:30 smithi155 ceph-mon[184854]: pgmap v403: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 27619/76530 objects degraded (36.089%); 2869/76530 objects misplaced (3.749%); 5.1 MiB/s, 115 objects/s recovering 2023-10-09T10:48:30.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:30.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:48:30.640 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:30.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:30 smithi125 ceph-mon[195507]: pgmap v403: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 27619/76530 objects degraded (36.089%); 2869/76530 objects misplaced (3.749%); 5.1 MiB/s, 115 objects/s recovering 2023-10-09T10:48:30.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:30.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:48:30.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:30 smithi125 ceph-mon[200197]: pgmap v403: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 27619/76530 objects degraded (36.089%); 2869/76530 objects misplaced (3.749%); 5.1 MiB/s, 115 objects/s recovering 2023-10-09T10:48:30.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:30.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:48:30.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:30.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:30.682 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:48:30.744 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:30.829 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:48:30.845 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:31.487 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:31.490 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:48:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:48:31] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.43.0" 2023-10-09T10:48:31.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:31.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:31.592 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:31.674 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:31.809 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:31.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:31.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:31.915 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:31.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:48:31.993 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:32.009 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:32.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:32.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:32.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:32.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:32.253 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:32.294 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:32.400 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:48:32.594 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:32.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:32.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:32.674 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:32 smithi125 ceph-mon[195507]: pgmap v404: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 27619/68745 objects degraded (40.176%); 2390/68745 objects misplaced (3.477%); 5.0 MiB/s, 114 objects/s recovering 2023-10-09T10:48:32.674 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:32 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1664975236' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:48:32.674 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:32 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:48:32.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:32 smithi125 ceph-mon[200197]: pgmap v404: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 27619/68745 objects degraded (40.176%); 2390/68745 objects misplaced (3.477%); 5.0 MiB/s, 114 objects/s recovering 2023-10-09T10:48:32.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:32 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1664975236' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:48:32.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:32 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:48:32.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:32.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:32 smithi155 ceph-mon[184854]: pgmap v404: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 27619/68745 objects degraded (40.176%); 2390/68745 objects misplaced (3.477%); 5.0 MiB/s, 114 objects/s recovering 2023-10-09T10:48:32.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:32 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1664975236' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:48:32.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:32 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:48:32.748 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:32.791 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:32.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:32.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:32.900 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:32.959 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:32.993 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:32.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:33.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:33.101 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:33.143 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:33.179 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:33.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:33.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:33.284 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:33.391 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:33.430 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:33.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:33.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:33.535 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:33.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:33 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:33.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:33 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:48:33.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:33 smithi125 ceph-mon[195507]: osdmap e185: 8 total, 8 up, 7 in 2023-10-09T10:48:33.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:33 smithi125 ceph-mon[195507]: failed to encode map e185 with expected crc 2023-10-09T10:48:33.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:33 smithi125 ceph-mon[195507]: failed to encode map e185 with expected crc 2023-10-09T10:48:33.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:33 smithi125 ceph-mon[195507]: failed to encode map e185 with expected crc 2023-10-09T10:48:33.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:33 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 27619/68745 objects degraded (40.176%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:33.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:33 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:33.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:33 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:48:33.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:33 smithi125 ceph-mon[200197]: osdmap e185: 8 total, 8 up, 7 in 2023-10-09T10:48:33.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:33 smithi125 ceph-mon[200197]: failed to encode map e185 with expected crc 2023-10-09T10:48:33.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:33 smithi125 ceph-mon[200197]: failed to encode map e185 with expected crc 2023-10-09T10:48:33.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:33 smithi125 ceph-mon[200197]: failed to encode map e185 with expected crc 2023-10-09T10:48:33.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:33 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 27619/68745 objects degraded (40.176%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:33.618 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:33 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:33 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:48:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:33 smithi155 ceph-mon[184854]: osdmap e185: 8 total, 8 up, 7 in 2023-10-09T10:48:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:33 smithi155 ceph-mon[184854]: failed to encode map e185 with expected crc 2023-10-09T10:48:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:33 smithi155 ceph-mon[184854]: failed to encode map e185 with expected crc 2023-10-09T10:48:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:33 smithi155 ceph-mon[184854]: failed to encode map e185 with expected crc 2023-10-09T10:48:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:33 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 27619/68745 objects degraded (40.176%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:33.738 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:48:33.738 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:48:33.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:48:33.738 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:48:33.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.739 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42934 6.11 6:8a3972ec:::benchmark_data_smithi125_244951_object42933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.739 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.549876+0000", 2023-10-09T10:48:33.739 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.18302333299999, 2023-10-09T10:48:33.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.229929598, 2023-10-09T10:48:33.739 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.739 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.739 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.739 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.739 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.739 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42934 2023-10-09T10:48:33.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.739 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549876+0000", 2023-10-09T10:48:33.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549876+0000", 2023-10-09T10:48:33.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549874+0000", 2023-10-09T10:48:33.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:48:33.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549907+0000", 2023-10-09T10:48:33.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2579000000000001e-05 2023-10-09T10:48:33.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549922+0000", 2023-10-09T10:48:33.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5420000000000001e-05 2023-10-09T10:48:33.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549934+0000", 2023-10-09T10:48:33.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1724e-05 2023-10-09T10:48:33.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550289+0000", 2023-10-09T10:48:33.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035548900000000001 2023-10-09T10:48:33.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op must wait for map", 2023-10-09T10:48:33.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550297+0000", 2023-10-09T10:48:33.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8869999999999992e-06 2023-10-09T10:48:33.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551339+0000", 2023-10-09T10:48:33.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010423069999999999 2023-10-09T10:48:33.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551349+0000", 2023-10-09T10:48:33.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8060000000000006e-06 2023-10-09T10:48:33.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.776033+0000", 2023-10-09T10:48:33.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.224684135 2023-10-09T10:48:33.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.776159+0000", 2023-10-09T10:48:33.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012583900000000001 2023-10-09T10:48:33.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.776290+0000", 2023-10-09T10:48:33.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000130705 2023-10-09T10:48:33.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778810+0000", 2023-10-09T10:48:33.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025205549999999998 2023-10-09T10:48:33.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778863+0000", 2023-10-09T10:48:33.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2821000000000001e-05 2023-10-09T10:48:33.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778971+0000", 2023-10-09T10:48:33.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108244 2023-10-09T10:48:33.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779012+0000", 2023-10-09T10:48:33.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0612999999999999e-05 2023-10-09T10:48:33.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779058+0000", 2023-10-09T10:48:33.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5948999999999997e-05 2023-10-09T10:48:33.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779805+0000", 2023-10-09T10:48:33.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00074730900000000004 2023-10-09T10:48:33.747 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.747 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.747 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.747 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42936 6.11 6:8ce9d1a1:::benchmark_data_smithi125_244951_object42935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.747 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.549950+0000", 2023-10-09T10:48:33.747 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.18294932000001, 2023-10-09T10:48:33.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.235619359, 2023-10-09T10:48:33.747 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.748 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.748 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.748 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.748 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.748 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42936 2023-10-09T10:48:33.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.748 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549950+0000", 2023-10-09T10:48:33.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549950+0000", 2023-10-09T10:48:33.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549949+0000", 2023-10-09T10:48:33.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:33.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549976+0000", 2023-10-09T10:48:33.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7223999999999999e-05 2023-10-09T10:48:33.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549987+0000", 2023-10-09T10:48:33.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0375e-05 2023-10-09T10:48:33.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549996+0000", 2023-10-09T10:48:33.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4010000000000003e-06 2023-10-09T10:48:33.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550347+0000", 2023-10-09T10:48:33.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035085200000000002 2023-10-09T10:48:33.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:48:33.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550352+0000", 2023-10-09T10:48:33.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2079999999999999e-06 2023-10-09T10:48:33.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551460+0000", 2023-10-09T10:48:33.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001107762 2023-10-09T10:48:33.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551467+0000", 2023-10-09T10:48:33.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0350000000000004e-06 2023-10-09T10:48:33.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.777841+0000", 2023-10-09T10:48:33.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2263741750000001 2023-10-09T10:48:33.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778003+0000", 2023-10-09T10:48:33.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016172699999999999 2023-10-09T10:48:33.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778128+0000", 2023-10-09T10:48:33.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012541000000000001 2023-10-09T10:48:33.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780770+0000", 2023-10-09T10:48:33.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026419500000000001 2023-10-09T10:48:33.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780795+0000", 2023-10-09T10:48:33.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4363e-05 2023-10-09T10:48:33.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784699+0000", 2023-10-09T10:48:33.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039046739999999999 2023-10-09T10:48:33.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784714+0000", 2023-10-09T10:48:33.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4956999999999999e-05 2023-10-09T10:48:33.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784774+0000", 2023-10-09T10:48:33.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9994999999999997e-05 2023-10-09T10:48:33.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785569+0000", 2023-10-09T10:48:33.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00079485999999999999 2023-10-09T10:48:33.756 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.756 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.756 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.756 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42950 6.11 6:8b5bb41c:::benchmark_data_smithi125_244951_object42949:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.756 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:48:33.756 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.18289692499999, 2023-10-09T10:48:33.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2390020909999999, 2023-10-09T10:48:33.756 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.756 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.757 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.757 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.757 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.757 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42950 2023-10-09T10:48:33.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.757 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:48:33.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:48:33.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:48:33.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:33.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550021+0000", 2023-10-09T10:48:33.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8941999999999999e-05 2023-10-09T10:48:33.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550034+0000", 2023-10-09T10:48:33.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3267e-05 2023-10-09T10:48:33.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550040+0000", 2023-10-09T10:48:33.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5799999999999997e-06 2023-10-09T10:48:33.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550377+0000", 2023-10-09T10:48:33.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00033659800000000002 2023-10-09T10:48:33.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:48:33.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550382+0000", 2023-10-09T10:48:33.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6589999999999998e-06 2023-10-09T10:48:33.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551483+0000", 2023-10-09T10:48:33.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011012210000000001 2023-10-09T10:48:33.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551489+0000", 2023-10-09T10:48:33.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4910000000000002e-06 2023-10-09T10:48:33.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779851+0000", 2023-10-09T10:48:33.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.22836144 2023-10-09T10:48:33.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779959+0000", 2023-10-09T10:48:33.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108546 2023-10-09T10:48:33.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780061+0000", 2023-10-09T10:48:33.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010196600000000001 2023-10-09T10:48:33.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782707+0000", 2023-10-09T10:48:33.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002646074 2023-10-09T10:48:33.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785603+0000", 2023-10-09T10:48:33.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028958410000000001 2023-10-09T10:48:33.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785626+0000", 2023-10-09T10:48:33.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2371e-05 2023-10-09T10:48:33.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.787983+0000", 2023-10-09T10:48:33.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023577619999999998 2023-10-09T10:48:33.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.788053+0000", 2023-10-09T10:48:33.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9797999999999998e-05 2023-10-09T10:48:33.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789004+0000", 2023-10-09T10:48:33.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00095101500000000004 2023-10-09T10:48:33.764 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.764 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.765 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.765 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42962 6.11 6:8af38cb4:::benchmark_data_smithi125_244951_object42961:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.765 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550048+0000", 2023-10-09T10:48:33.765 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.18285139599999, 2023-10-09T10:48:33.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2398821879999999, 2023-10-09T10:48:33.765 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.765 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.765 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.765 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.765 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.765 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42962 2023-10-09T10:48:33.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.766 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550048+0000", 2023-10-09T10:48:33.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550048+0000", 2023-10-09T10:48:33.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550047+0000", 2023-10-09T10:48:33.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:33.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550067+0000", 2023-10-09T10:48:33.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0228999999999999e-05 2023-10-09T10:48:33.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550075+0000", 2023-10-09T10:48:33.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0979999999999996e-06 2023-10-09T10:48:33.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550080+0000", 2023-10-09T10:48:33.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7780000000000004e-06 2023-10-09T10:48:33.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550404+0000", 2023-10-09T10:48:33.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000323648 2023-10-09T10:48:33.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:48:33.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550409+0000", 2023-10-09T10:48:33.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6630000000000001e-06 2023-10-09T10:48:33.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551523+0000", 2023-10-09T10:48:33.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001113953 2023-10-09T10:48:33.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551531+0000", 2023-10-09T10:48:33.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5860000000000005e-06 2023-10-09T10:48:33.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780812+0000", 2023-10-09T10:48:33.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2292809120000001 2023-10-09T10:48:33.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780947+0000", 2023-10-09T10:48:33.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000135265 2023-10-09T10:48:33.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.781098+0000", 2023-10-09T10:48:33.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015082499999999999 2023-10-09T10:48:33.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782749+0000", 2023-10-09T10:48:33.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001651057 2023-10-09T10:48:33.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785607+0000", 2023-10-09T10:48:33.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028579569999999999 2023-10-09T10:48:33.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785661+0000", 2023-10-09T10:48:33.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4151999999999999e-05 2023-10-09T10:48:33.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789184+0000", 2023-10-09T10:48:33.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035224309999999999 2023-10-09T10:48:33.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789241+0000", 2023-10-09T10:48:33.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6962000000000001e-05 2023-10-09T10:48:33.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789930+0000", 2023-10-09T10:48:33.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000689148 2023-10-09T10:48:33.773 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.773 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.773 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.773 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43001 6.11 6:8f0a81da:::benchmark_data_smithi125_244951_object43000:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.773 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:48:33.774 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.18281368699999, 2023-10-09T10:48:33.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.240501734, 2023-10-09T10:48:33.774 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.774 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.774 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.774 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.774 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.774 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43001 2023-10-09T10:48:33.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.774 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:48:33.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:48:33.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:48:33.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:33.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550104+0000", 2023-10-09T10:48:33.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.912e-05 2023-10-09T10:48:33.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550110+0000", 2023-10-09T10:48:33.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3779999999999998e-06 2023-10-09T10:48:33.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550116+0000", 2023-10-09T10:48:33.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.609e-06 2023-10-09T10:48:33.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550429+0000", 2023-10-09T10:48:33.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00031264800000000001 2023-10-09T10:48:33.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:48:33.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550433+0000", 2023-10-09T10:48:33.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8420000000000004e-06 2023-10-09T10:48:33.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551574+0000", 2023-10-09T10:48:33.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011402210000000001 2023-10-09T10:48:33.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551584+0000", 2023-10-09T10:48:33.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9690000000000006e-06 2023-10-09T10:48:33.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.781792+0000", 2023-10-09T10:48:33.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.230208454 2023-10-09T10:48:33.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.781903+0000", 2023-10-09T10:48:33.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110733 2023-10-09T10:48:33.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782036+0000", 2023-10-09T10:48:33.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000133545 2023-10-09T10:48:33.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785598+0000", 2023-10-09T10:48:33.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035616100000000002 2023-10-09T10:48:33.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785679+0000", 2023-10-09T10:48:33.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0624999999999995e-05 2023-10-09T10:48:33.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785703+0000", 2023-10-09T10:48:33.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4150000000000001e-05 2023-10-09T10:48:33.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790058+0000", 2023-10-09T10:48:33.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043551919999999999 2023-10-09T10:48:33.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790110+0000", 2023-10-09T10:48:33.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2278000000000003e-05 2023-10-09T10:48:33.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.782 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790587+0000", 2023-10-09T10:48:33.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047690599999999998 2023-10-09T10:48:33.782 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.782 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.782 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.782 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43002 6.11 6:8bd791a2:::benchmark_data_smithi125_244951_object43001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.782 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550121+0000", 2023-10-09T10:48:33.782 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.18277839300001, 2023-10-09T10:48:33.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.241166963, 2023-10-09T10:48:33.783 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.783 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.783 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.783 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.783 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.783 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43002 2023-10-09T10:48:33.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.783 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550121+0000", 2023-10-09T10:48:33.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550121+0000", 2023-10-09T10:48:33.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550120+0000", 2023-10-09T10:48:33.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:33.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550139+0000", 2023-10-09T10:48:33.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8644999999999999e-05 2023-10-09T10:48:33.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550145+0000", 2023-10-09T10:48:33.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5180000000000001e-06 2023-10-09T10:48:33.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550151+0000", 2023-10-09T10:48:33.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0399999999999998e-06 2023-10-09T10:48:33.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551616+0000", 2023-10-09T10:48:33.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001464449 2023-10-09T10:48:33.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551623+0000", 2023-10-09T10:48:33.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0090000000000001e-06 2023-10-09T10:48:33.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782780+0000", 2023-10-09T10:48:33.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2311575050000001 2023-10-09T10:48:33.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782892+0000", 2023-10-09T10:48:33.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111826 2023-10-09T10:48:33.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782998+0000", 2023-10-09T10:48:33.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010603399999999999 2023-10-09T10:48:33.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785644+0000", 2023-10-09T10:48:33.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026456489999999999 2023-10-09T10:48:33.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785699+0000", 2023-10-09T10:48:33.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4778999999999998e-05 2023-10-09T10:48:33.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786762+0000", 2023-10-09T10:48:33.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001063172 2023-10-09T10:48:33.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790708+0000", 2023-10-09T10:48:33.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039466409999999999 2023-10-09T10:48:33.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790775+0000", 2023-10-09T10:48:33.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6155000000000006e-05 2023-10-09T10:48:33.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791288+0000", 2023-10-09T10:48:33.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051300599999999999 2023-10-09T10:48:33.790 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.790 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.790 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.790 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43015 6.11 6:8d7a9802:::benchmark_data_smithi125_244951_object43014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.790 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:48:33.790 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.182742695, 2023-10-09T10:48:33.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.241685028, 2023-10-09T10:48:33.791 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.791 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.791 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.791 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.791 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.791 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43015 2023-10-09T10:48:33.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.791 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:48:33.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.792 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.792 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:48:33.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.792 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.792 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:48:33.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:33.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.792 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550174+0000", 2023-10-09T10:48:33.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8289000000000001e-05 2023-10-09T10:48:33.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550181+0000", 2023-10-09T10:48:33.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.024e-06 2023-10-09T10:48:33.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550187+0000", 2023-10-09T10:48:33.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6659999999999996e-06 2023-10-09T10:48:33.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551652+0000", 2023-10-09T10:48:33.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001465406 2023-10-09T10:48:33.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551659+0000", 2023-10-09T10:48:33.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1130000000000003e-06 2023-10-09T10:48:33.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.783679+0000", 2023-10-09T10:48:33.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.232019406 2023-10-09T10:48:33.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.783831+0000", 2023-10-09T10:48:33.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000151769 2023-10-09T10:48:33.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784019+0000", 2023-10-09T10:48:33.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018824400000000001 2023-10-09T10:48:33.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786769+0000", 2023-10-09T10:48:33.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027502529999999998 2023-10-09T10:48:33.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789091+0000", 2023-10-09T10:48:33.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023215200000000001 2023-10-09T10:48:33.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789131+0000", 2023-10-09T10:48:33.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0766999999999998e-05 2023-10-09T10:48:33.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791345+0000", 2023-10-09T10:48:33.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022138349999999999 2023-10-09T10:48:33.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791404+0000", 2023-10-09T10:48:33.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8301000000000003e-05 2023-10-09T10:48:33.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.798 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.798 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791841+0000", 2023-10-09T10:48:33.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00043785599999999998 2023-10-09T10:48:33.798 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.798 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.798 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.798 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.798 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43040 6.11 6:892fed64:::benchmark_data_smithi125_244951_object43039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.798 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550192+0000", 2023-10-09T10:48:33.798 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.18270731600001, 2023-10-09T10:48:33.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2421168730000001, 2023-10-09T10:48:33.799 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.799 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.799 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.799 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.799 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.799 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43040 2023-10-09T10:48:33.799 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.799 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550192+0000", 2023-10-09T10:48:33.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550192+0000", 2023-10-09T10:48:33.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550191+0000", 2023-10-09T10:48:33.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:33.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550265+0000", 2023-10-09T10:48:33.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3668e-05 2023-10-09T10:48:33.801 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.801 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550273+0000", 2023-10-09T10:48:33.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2230000000000007e-06 2023-10-09T10:48:33.801 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.801 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550280+0000", 2023-10-09T10:48:33.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3729999999999998e-06 2023-10-09T10:48:33.802 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.802 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.802 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.802 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551688+0000", 2023-10-09T10:48:33.802 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001408011 2023-10-09T10:48:33.802 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.802 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.802 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.802 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551694+0000", 2023-10-09T10:48:33.802 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8120000000000002e-06 2023-10-09T10:48:33.802 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.803 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.803 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.803 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785718+0000", 2023-10-09T10:48:33.803 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2340238640000001 2023-10-09T10:48:33.803 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.803 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.803 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.803 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785825+0000", 2023-10-09T10:48:33.803 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000107161 2023-10-09T10:48:33.803 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.803 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.803 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.803 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785964+0000", 2023-10-09T10:48:33.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013889800000000001 2023-10-09T10:48:33.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.804 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789139+0000", 2023-10-09T10:48:33.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0031749249999999999 2023-10-09T10:48:33.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.804 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789979+0000", 2023-10-09T10:48:33.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00083957300000000003 2023-10-09T10:48:33.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790018+0000", 2023-10-09T10:48:33.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8816999999999998e-05 2023-10-09T10:48:33.805 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.805 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791894+0000", 2023-10-09T10:48:33.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018764160000000001 2023-10-09T10:48:33.805 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.805 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791933+0000", 2023-10-09T10:48:33.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8889e-05 2023-10-09T10:48:33.805 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.806 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.806 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.806 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792309+0000", 2023-10-09T10:48:33.806 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00037573100000000003 2023-10-09T10:48:33.806 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.806 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.806 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.806 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.806 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.806 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43116 6.11 6:8866ecf5:::benchmark_data_smithi125_244951_object43115:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.806 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550285+0000", 2023-10-09T10:48:33.807 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.182614169, 2023-10-09T10:48:33.807 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2425320870000001, 2023-10-09T10:48:33.807 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.807 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.807 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.807 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.807 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.807 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43116 2023-10-09T10:48:33.807 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.807 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.807 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.807 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.808 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550285+0000", 2023-10-09T10:48:33.808 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.808 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.808 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.808 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.808 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550285+0000", 2023-10-09T10:48:33.808 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.808 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.808 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.808 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.808 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550284+0000", 2023-10-09T10:48:33.808 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:33.808 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.809 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.809 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.809 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550303+0000", 2023-10-09T10:48:33.809 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8817e-05 2023-10-09T10:48:33.809 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.809 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.809 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.809 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550310+0000", 2023-10-09T10:48:33.809 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7889999999999997e-06 2023-10-09T10:48:33.809 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.809 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.809 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.809 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550315+0000", 2023-10-09T10:48:33.809 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4539999999999998e-06 2023-10-09T10:48:33.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.810 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.810 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.810 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551718+0000", 2023-10-09T10:48:33.810 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001402585 2023-10-09T10:48:33.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.810 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.810 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.810 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551725+0000", 2023-10-09T10:48:33.810 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9319999999999999e-06 2023-10-09T10:48:33.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.810 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.810 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786811+0000", 2023-10-09T10:48:33.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.23508555 2023-10-09T10:48:33.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786947+0000", 2023-10-09T10:48:33.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013601499999999999 2023-10-09T10:48:33.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.787112+0000", 2023-10-09T10:48:33.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016519000000000001 2023-10-09T10:48:33.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.812 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789147+0000", 2023-10-09T10:48:33.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020350899999999998 2023-10-09T10:48:33.812 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.812 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790634+0000", 2023-10-09T10:48:33.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014872209999999999 2023-10-09T10:48:33.812 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.812 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.813 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790670+0000", 2023-10-09T10:48:33.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6395e-05 2023-10-09T10:48:33.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.813 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.813 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.813 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792355+0000", 2023-10-09T10:48:33.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016850179999999999 2023-10-09T10:48:33.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.813 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.813 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.813 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792391+0000", 2023-10-09T10:48:33.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5484000000000001e-05 2023-10-09T10:48:33.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.814 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.814 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.814 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792817+0000", 2023-10-09T10:48:33.814 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042605300000000002 2023-10-09T10:48:33.814 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.814 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.814 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.814 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.814 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.814 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43152 6.11 6:8f5a8f11:::benchmark_data_smithi125_244951_object43151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.814 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:48:33.814 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.18257782699999, 2023-10-09T10:48:33.815 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.254750676, 2023-10-09T10:48:33.815 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.815 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.815 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.815 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.815 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.815 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43152 2023-10-09T10:48:33.815 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.815 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.815 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.815 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.815 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:48:33.816 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.816 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.816 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.816 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.816 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:48:33.816 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.816 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.816 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.816 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.816 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:48:33.816 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:33.816 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.817 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.817 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.817 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550339+0000", 2023-10-09T10:48:33.817 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7940000000000001e-05 2023-10-09T10:48:33.817 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.817 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.817 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.817 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550346+0000", 2023-10-09T10:48:33.817 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4789999999999999e-06 2023-10-09T10:48:33.817 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.817 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.817 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.817 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550353+0000", 2023-10-09T10:48:33.818 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7510000000000002e-06 2023-10-09T10:48:33.818 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.818 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.818 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.818 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551748+0000", 2023-10-09T10:48:33.818 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013946119999999999 2023-10-09T10:48:33.818 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.818 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.818 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.818 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551754+0000", 2023-10-09T10:48:33.818 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8460000000000002e-06 2023-10-09T10:48:33.818 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.819 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.819 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.819 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792855+0000", 2023-10-09T10:48:33.819 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.241101091 2023-10-09T10:48:33.819 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.819 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.819 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.819 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792993+0000", 2023-10-09T10:48:33.819 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013794 2023-10-09T10:48:33.819 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.819 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.819 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.820 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.793143+0000", 2023-10-09T10:48:33.820 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014960799999999999 2023-10-09T10:48:33.820 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.820 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.820 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.820 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797271+0000", 2023-10-09T10:48:33.820 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041283810000000004 2023-10-09T10:48:33.820 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.820 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.820 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.820 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797282+0000", 2023-10-09T10:48:33.820 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0543999999999999e-05 2023-10-09T10:48:33.821 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.821 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.821 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.821 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799296+0000", 2023-10-09T10:48:33.821 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020138999999999999 2023-10-09T10:48:33.821 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.821 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.821 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.821 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803962+0000", 2023-10-09T10:48:33.821 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046658840000000004 2023-10-09T10:48:33.821 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.821 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.821 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.822 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.804015+0000", 2023-10-09T10:48:33.822 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3229999999999997e-05 2023-10-09T10:48:33.822 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.822 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.822 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.822 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805072+0000", 2023-10-09T10:48:33.822 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010569690000000001 2023-10-09T10:48:33.822 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.822 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.822 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.822 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.822 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.823 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43224 6.11 6:8b0bc017:::benchmark_data_smithi125_244951_object43223:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.823 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550374+0000", 2023-10-09T10:48:33.823 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.18252543899999, 2023-10-09T10:48:33.823 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2556577980000001, 2023-10-09T10:48:33.823 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.823 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.823 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.823 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.823 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.823 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43224 2023-10-09T10:48:33.823 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.823 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.823 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.824 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.824 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550374+0000", 2023-10-09T10:48:33.824 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.824 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.824 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.824 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.824 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550374+0000", 2023-10-09T10:48:33.824 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.824 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.824 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.824 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.824 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550373+0000", 2023-10-09T10:48:33.825 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:33.825 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.825 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.825 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.825 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550388+0000", 2023-10-09T10:48:33.825 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.453e-05 2023-10-09T10:48:33.825 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.825 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.825 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.825 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550393+0000", 2023-10-09T10:48:33.825 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7119999999999999e-06 2023-10-09T10:48:33.825 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.825 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.826 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.826 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550397+0000", 2023-10-09T10:48:33.826 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6789999999999996e-06 2023-10-09T10:48:33.826 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.826 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.826 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.826 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551777+0000", 2023-10-09T10:48:33.826 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001380196 2023-10-09T10:48:33.826 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.826 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.826 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.826 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551784+0000", 2023-10-09T10:48:33.827 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0369999999999997e-06 2023-10-09T10:48:33.827 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.827 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.827 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.827 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.793904+0000", 2023-10-09T10:48:33.827 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.242119346 2023-10-09T10:48:33.827 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.827 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.827 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.827 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.794042+0000", 2023-10-09T10:48:33.827 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013829700000000001 2023-10-09T10:48:33.827 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.828 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.828 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.828 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.794190+0000", 2023-10-09T10:48:33.828 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014784600000000001 2023-10-09T10:48:33.828 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.828 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.828 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.828 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797289+0000", 2023-10-09T10:48:33.828 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003099152 2023-10-09T10:48:33.828 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.828 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.828 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.829 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799339+0000", 2023-10-09T10:48:33.829 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020501059999999999 2023-10-09T10:48:33.829 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.829 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.829 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.829 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803853+0000", 2023-10-09T10:48:33.829 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045136179999999996 2023-10-09T10:48:33.829 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.829 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.829 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.829 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805211+0000", 2023-10-09T10:48:33.829 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001358349 2023-10-09T10:48:33.829 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.830 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.830 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.830 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805266+0000", 2023-10-09T10:48:33.830 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5297999999999998e-05 2023-10-09T10:48:33.830 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.830 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.830 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.830 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806031+0000", 2023-10-09T10:48:33.830 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00076497100000000003 2023-10-09T10:48:33.830 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.830 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.830 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.831 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.831 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.831 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43255 6.11 6:88dd5493:::benchmark_data_smithi125_244951_object43254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.831 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:48:33.831 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.18249892, 2023-10-09T10:48:33.831 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2563999859999999, 2023-10-09T10:48:33.831 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.831 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.831 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.831 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.831 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.831 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43255 2023-10-09T10:48:33.832 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.832 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.832 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.832 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.832 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:48:33.832 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.832 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.832 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.832 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.832 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:48:33.832 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.832 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.833 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.833 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.833 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:48:33.833 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:33.833 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.833 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.833 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.833 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550413+0000", 2023-10-09T10:48:33.833 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3052e-05 2023-10-09T10:48:33.833 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.833 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.833 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.833 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550417+0000", 2023-10-09T10:48:33.834 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5999999999999998e-06 2023-10-09T10:48:33.834 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.834 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.834 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.834 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550420+0000", 2023-10-09T10:48:33.834 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6519999999999999e-06 2023-10-09T10:48:33.834 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.834 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.834 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.834 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551807+0000", 2023-10-09T10:48:33.834 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001386998 2023-10-09T10:48:33.834 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.834 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.835 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.835 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551814+0000", 2023-10-09T10:48:33.835 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3960000000000003e-06 2023-10-09T10:48:33.835 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.835 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.835 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.835 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.794932+0000", 2023-10-09T10:48:33.835 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2431179880000001 2023-10-09T10:48:33.835 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.835 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.835 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.835 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.795097+0000", 2023-10-09T10:48:33.836 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000165248 2023-10-09T10:48:33.836 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.836 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.836 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.795276+0000", 2023-10-09T10:48:33.836 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017957699999999999 2023-10-09T10:48:33.836 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.836 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.836 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799383+0000", 2023-10-09T10:48:33.836 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004106569 2023-10-09T10:48:33.836 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.837 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.837 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801446+0000", 2023-10-09T10:48:33.837 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020628370000000001 2023-10-09T10:48:33.837 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.837 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.837 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.837 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803918+0000", 2023-10-09T10:48:33.837 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024717239999999998 2023-10-09T10:48:33.837 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.837 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.837 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.837 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806076+0000", 2023-10-09T10:48:33.838 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002158395 2023-10-09T10:48:33.838 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.838 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.838 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806122+0000", 2023-10-09T10:48:33.838 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5807999999999999e-05 2023-10-09T10:48:33.838 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.838 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.838 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806800+0000", 2023-10-09T10:48:33.838 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00067843299999999999 2023-10-09T10:48:33.838 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.839 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.839 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.839 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.839 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.839 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43315 6.11 6:8fb75c47:::benchmark_data_smithi125_244951_object43314:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.839 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550436+0000", 2023-10-09T10:48:33.839 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.182463426, 2023-10-09T10:48:33.839 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2571070289999999, 2023-10-09T10:48:33.839 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.839 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.839 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.840 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.840 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.840 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43315 2023-10-09T10:48:33.840 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.840 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.840 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.840 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.840 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550436+0000", 2023-10-09T10:48:33.840 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.840 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.840 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.840 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.840 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550436+0000", 2023-10-09T10:48:33.841 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.841 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.841 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.841 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.841 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550435+0000", 2023-10-09T10:48:33.841 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:33.841 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.841 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.841 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.841 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550453+0000", 2023-10-09T10:48:33.841 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7504999999999999e-05 2023-10-09T10:48:33.841 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.842 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.842 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550460+0000", 2023-10-09T10:48:33.842 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2470000000000002e-06 2023-10-09T10:48:33.842 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.842 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.842 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550466+0000", 2023-10-09T10:48:33.842 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8699999999999997e-06 2023-10-09T10:48:33.842 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.843 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.843 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551836+0000", 2023-10-09T10:48:33.843 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001370664 2023-10-09T10:48:33.843 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.843 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.843 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.843 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551843+0000", 2023-10-09T10:48:33.843 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5320000000000003e-06 2023-10-09T10:48:33.843 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.843 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.843 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.843 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.796128+0000", 2023-10-09T10:48:33.844 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2442850169999999 2023-10-09T10:48:33.844 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.844 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.844 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.844 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.796286+0000", 2023-10-09T10:48:33.844 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015828500000000001 2023-10-09T10:48:33.844 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.844 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.844 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.844 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.796448+0000", 2023-10-09T10:48:33.844 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016164200000000001 2023-10-09T10:48:33.844 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.845 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.845 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.845 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801434+0000", 2023-10-09T10:48:33.845 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049856279999999998 2023-10-09T10:48:33.845 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.845 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.845 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.845 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801530+0000", 2023-10-09T10:48:33.845 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.645e-05 2023-10-09T10:48:33.845 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.845 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.845 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.846 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805133+0000", 2023-10-09T10:48:33.846 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036025850000000002 2023-10-09T10:48:33.846 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.846 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.846 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.846 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806935+0000", 2023-10-09T10:48:33.846 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018020410000000001 2023-10-09T10:48:33.846 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.846 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.846 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.846 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.807002+0000", 2023-10-09T10:48:33.846 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6883000000000003e-05 2023-10-09T10:48:33.847 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.847 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.847 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.847 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.807543+0000", 2023-10-09T10:48:33.847 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000541115 2023-10-09T10:48:33.847 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.847 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.847 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.847 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.847 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.847 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43318 6.11 6:8f0da7e4:::benchmark_data_smithi125_244951_object43317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.847 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550471+0000", 2023-10-09T10:48:33.848 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.18242854799999, 2023-10-09T10:48:33.848 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2582329400000001, 2023-10-09T10:48:33.848 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.848 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.848 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.848 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.848 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.848 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43318 2023-10-09T10:48:33.848 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.848 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.848 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.848 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.849 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550471+0000", 2023-10-09T10:48:33.849 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.849 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.849 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.849 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.849 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550471+0000", 2023-10-09T10:48:33.849 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.849 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.849 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.849 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.849 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550470+0000", 2023-10-09T10:48:33.849 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:33.850 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.850 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.850 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.850 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550518+0000", 2023-10-09T10:48:33.850 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8108000000000001e-05 2023-10-09T10:48:33.850 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.850 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.850 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.850 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550526+0000", 2023-10-09T10:48:33.850 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0199999999999994e-06 2023-10-09T10:48:33.850 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.850 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.851 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.851 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550532+0000", 2023-10-09T10:48:33.851 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9109999999999998e-06 2023-10-09T10:48:33.851 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.851 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.851 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.851 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551864+0000", 2023-10-09T10:48:33.851 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001332317 2023-10-09T10:48:33.851 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.851 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.851 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.851 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551870+0000", 2023-10-09T10:48:33.852 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6489999999999996e-06 2023-10-09T10:48:33.852 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.852 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.852 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.852 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797329+0000", 2023-10-09T10:48:33.852 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.245458701 2023-10-09T10:48:33.852 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.852 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.852 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.852 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797476+0000", 2023-10-09T10:48:33.852 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014769199999999999 2023-10-09T10:48:33.852 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.852 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.853 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.853 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797644+0000", 2023-10-09T10:48:33.853 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000167481 2023-10-09T10:48:33.853 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.853 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.853 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.853 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801478+0000", 2023-10-09T10:48:33.853 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038341780000000002 2023-10-09T10:48:33.853 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.853 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.853 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.853 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801537+0000", 2023-10-09T10:48:33.854 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8879000000000003e-05 2023-10-09T10:48:33.854 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.854 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.854 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.854 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806859+0000", 2023-10-09T10:48:33.854 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0053216640000000003 2023-10-09T10:48:33.854 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.854 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.854 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.854 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.807979+0000", 2023-10-09T10:48:33.854 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011203770000000001 2023-10-09T10:48:33.854 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.855 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.855 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.855 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808056+0000", 2023-10-09T10:48:33.855 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6568000000000003e-05 2023-10-09T10:48:33.855 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.855 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.855 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.855 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808703+0000", 2023-10-09T10:48:33.855 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00064785600000000004 2023-10-09T10:48:33.855 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.855 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.855 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.856 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.856 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.856 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43334 6.11 6:8b093292:::benchmark_data_smithi125_244951_object43333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.856 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:48:33.856 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.18236087899999, 2023-10-09T10:48:33.856 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.266648716, 2023-10-09T10:48:33.856 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.856 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.856 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.856 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.856 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.856 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43334 2023-10-09T10:48:33.857 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.857 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.857 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.857 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.857 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:48:33.857 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.857 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.857 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.857 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.857 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:48:33.857 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.857 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.858 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.858 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.858 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:48:33.858 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:33.859 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.859 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.859 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.859 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550579+0000", 2023-10-09T10:48:33.859 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1733000000000002e-05 2023-10-09T10:48:33.859 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.859 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.859 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.859 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550587+0000", 2023-10-09T10:48:33.859 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6260000000000003e-06 2023-10-09T10:48:33.859 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.860 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.860 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.860 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550592+0000", 2023-10-09T10:48:33.860 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4160000000000003e-06 2023-10-09T10:48:33.860 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.860 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.860 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.860 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551890+0000", 2023-10-09T10:48:33.860 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001297702 2023-10-09T10:48:33.860 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.860 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.860 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.861 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551896+0000", 2023-10-09T10:48:33.861 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4559999999999999e-06 2023-10-09T10:48:33.861 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.861 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.861 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.861 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.798375+0000", 2023-10-09T10:48:33.861 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2464794800000001 2023-10-09T10:48:33.861 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.861 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.861 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.861 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.798514+0000", 2023-10-09T10:48:33.861 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000139159 2023-10-09T10:48:33.862 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.862 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.862 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.862 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.798661+0000", 2023-10-09T10:48:33.862 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000146312 2023-10-09T10:48:33.862 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.862 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.862 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.862 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801519+0000", 2023-10-09T10:48:33.862 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028583720000000001 2023-10-09T10:48:33.862 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.862 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.863 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.863 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803865+0000", 2023-10-09T10:48:33.863 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002345752 2023-10-09T10:48:33.863 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.863 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.863 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.863 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808789+0000", 2023-10-09T10:48:33.863 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049239790000000002 2023-10-09T10:48:33.863 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.863 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.863 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.863 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.816450+0000", 2023-10-09T10:48:33.864 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076614550000000002 2023-10-09T10:48:33.864 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.864 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.864 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.864 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.816501+0000", 2023-10-09T10:48:33.864 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1298000000000003e-05 2023-10-09T10:48:33.864 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.864 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.864 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.864 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817187+0000", 2023-10-09T10:48:33.864 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00068546500000000001 2023-10-09T10:48:33.865 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.865 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.865 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.865 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.865 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.865 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43385 6.11 6:8e6a343d:::benchmark_data_smithi125_244951_object43384:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:48:33.865 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.636508+0000", 2023-10-09T10:48:33.865 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 216.09639097100001, 2023-10-09T10:48:33.865 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1811192319999999, 2023-10-09T10:48:33.865 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.865 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.865 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.865 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.866 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.866 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43385 2023-10-09T10:48:33.866 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.866 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.866 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.866 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.866 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636508+0000", 2023-10-09T10:48:33.866 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.866 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.866 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.866 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.867 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636508+0000", 2023-10-09T10:48:33.867 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.867 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.867 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.867 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.867 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636507+0000", 2023-10-09T10:48:33.867 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:33.867 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.867 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.867 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.867 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636527+0000", 2023-10-09T10:48:33.868 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9748000000000001e-05 2023-10-09T10:48:33.868 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.868 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.868 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.868 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636539+0000", 2023-10-09T10:48:33.868 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2197e-05 2023-10-09T10:48:33.868 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.868 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.868 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.868 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636545+0000", 2023-10-09T10:48:33.868 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1999999999999999e-06 2023-10-09T10:48:33.868 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.868 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.869 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.869 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636582+0000", 2023-10-09T10:48:33.869 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6791000000000001e-05 2023-10-09T10:48:33.869 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.869 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.869 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.869 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636590+0000", 2023-10-09T10:48:33.869 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4309999999999994e-06 2023-10-09T10:48:33.869 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.869 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.869 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.869 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799411+0000", 2023-10-09T10:48:33.870 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1628212659999999 2023-10-09T10:48:33.870 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.870 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.870 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.870 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799550+0000", 2023-10-09T10:48:33.870 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013822300000000001 2023-10-09T10:48:33.870 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.870 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.870 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:33.870 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799660+0000", 2023-10-09T10:48:33.870 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110701 2023-10-09T10:48:33.870 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.871 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.871 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.871 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801589+0000", 2023-10-09T10:48:33.871 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019285470000000001 2023-10-09T10:48:33.871 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.871 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.871 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.871 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803871+0000", 2023-10-09T10:48:33.871 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022818180000000001 2023-10-09T10:48:33.871 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.871 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.871 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.871 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808835+0000", 2023-10-09T10:48:33.872 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049646200000000003 2023-10-09T10:48:33.872 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.872 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.872 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.872 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817231+0000", 2023-10-09T10:48:33.872 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083956959999999994 2023-10-09T10:48:33.872 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.872 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.872 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.872 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817263+0000", 2023-10-09T10:48:33.872 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1899000000000001e-05 2023-10-09T10:48:33.872 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.873 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.873 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.873 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817627+0000", 2023-10-09T10:48:33.873 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00036434500000000002 2023-10-09T10:48:33.873 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.873 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.873 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.873 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.873 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.873 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106397 6.1 6:856288a5:::benchmark_data_smithi125_244951_object106396:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:48:33.873 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.204308+0000", 2023-10-09T10:48:33.873 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 151.52859095700001, 2023-10-09T10:48:33.873 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.075713931999999998, 2023-10-09T10:48:33.874 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.874 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.874 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.874 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.874 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.874 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106397 2023-10-09T10:48:33.874 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.874 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.874 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.874 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.874 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204308+0000", 2023-10-09T10:48:33.874 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.875 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.875 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.875 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.875 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204308+0000", 2023-10-09T10:48:33.875 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.875 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.875 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.875 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.875 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204307+0000", 2023-10-09T10:48:33.875 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:48:33.875 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.875 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.876 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.876 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204330+0000", 2023-10-09T10:48:33.876 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2852000000000001e-05 2023-10-09T10:48:33.876 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.876 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.876 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.876 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204340+0000", 2023-10-09T10:48:33.876 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3989999999999993e-06 2023-10-09T10:48:33.876 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.876 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.876 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.876 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204346+0000", 2023-10-09T10:48:33.877 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5989999999999998e-06 2023-10-09T10:48:33.877 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.877 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.877 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.877 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.207434+0000", 2023-10-09T10:48:33.877 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030874600000000002 2023-10-09T10:48:33.877 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.877 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.877 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.877 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227524+0000", 2023-10-09T10:48:33.877 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.020090545000000001 2023-10-09T10:48:33.877 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.878 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.878 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:48:33.878 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227667+0000", 2023-10-09T10:48:33.878 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014324699999999999 2023-10-09T10:48:33.878 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.878 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.878 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.878 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.233515+0000", 2023-10-09T10:48:33.878 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0058477219999999996 2023-10-09T10:48:33.878 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.878 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.878 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.878 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.234252+0000", 2023-10-09T10:48:33.879 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00073691800000000003 2023-10-09T10:48:33.879 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.879 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.879 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.879 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279940+0000", 2023-10-09T10:48:33.879 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.045688232000000002 2023-10-09T10:48:33.879 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.879 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.879 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.879 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279985+0000", 2023-10-09T10:48:33.879 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4916000000000002e-05 2023-10-09T10:48:33.879 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.880 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.880 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.880 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.280022+0000", 2023-10-09T10:48:33.880 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6871e-05 2023-10-09T10:48:33.880 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.880 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.880 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.880 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.880 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.880 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106414 6.11 6:8c9e6868:::benchmark_data_smithi125_244951_object106413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:48:33.880 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.227135+0000", 2023-10-09T10:48:33.880 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 151.50576426800001, 2023-10-09T10:48:33.880 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.37275067699999997, 2023-10-09T10:48:33.881 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.881 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:48:33.881 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.881 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.881 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.881 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106414 2023-10-09T10:48:33.881 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.881 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.881 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.881 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.881 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227135+0000", 2023-10-09T10:48:33.881 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.882 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.882 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.882 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.882 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227135+0000", 2023-10-09T10:48:33.882 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.882 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.882 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.882 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.882 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227134+0000", 2023-10-09T10:48:33.882 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:33.882 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.882 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.883 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.883 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227156+0000", 2023-10-09T10:48:33.883 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2059999999999999e-05 2023-10-09T10:48:33.883 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.883 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.883 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.883 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227163+0000", 2023-10-09T10:48:33.883 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4169999999999997e-06 2023-10-09T10:48:33.883 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.883 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.883 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.883 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227168+0000", 2023-10-09T10:48:33.884 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9509999999999997e-06 2023-10-09T10:48:33.884 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.884 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.884 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.884 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.229108+0000", 2023-10-09T10:48:33.884 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001940396 2023-10-09T10:48:33.884 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.884 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.884 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.884 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.229279+0000", 2023-10-09T10:48:33.884 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017114100000000001 2023-10-09T10:48:33.884 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.884 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.884 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T10:48:33.885 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.229487+0000", 2023-10-09T10:48:33.885 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000207977 2023-10-09T10:48:33.885 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.885 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.885 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.885 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.233575+0000", 2023-10-09T10:48:33.885 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040875140000000004 2023-10-09T10:48:33.885 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.885 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.885 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.885 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.240998+0000", 2023-10-09T10:48:33.885 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074230249999999998 2023-10-09T10:48:33.885 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.886 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.886 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.886 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.245203+0000", 2023-10-09T10:48:33.886 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042052039999999997 2023-10-09T10:48:33.886 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.886 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.886 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.886 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274084+0000", 2023-10-09T10:48:33.886 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.028880626999999999 2023-10-09T10:48:33.886 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.886 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.886 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.887 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274097+0000", 2023-10-09T10:48:33.887 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.344e-05 2023-10-09T10:48:33.887 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.887 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.887 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.887 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599869+0000", 2023-10-09T10:48:33.887 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32577187699999999 2023-10-09T10:48:33.887 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.887 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.887 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.887 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599885+0000", 2023-10-09T10:48:33.887 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6515e-05 2023-10-09T10:48:33.888 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.888 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.888 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.888 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.888 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.888 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106433 6.11 6:8d28bd42:::benchmark_data_smithi125_244951_object106432:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:48:33.888 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.251731+0000", 2023-10-09T10:48:33.888 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 151.48116780000001, 2023-10-09T10:48:33.888 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.34819989800000001, 2023-10-09T10:48:33.888 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.888 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:48:33.888 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.889 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.889 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.889 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106433 2023-10-09T10:48:33.889 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.889 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.889 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.889 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.889 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251731+0000", 2023-10-09T10:48:33.889 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.889 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.889 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.889 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.889 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251731+0000", 2023-10-09T10:48:33.890 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.890 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.890 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.890 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.890 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251729+0000", 2023-10-09T10:48:33.890 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:48:33.890 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.890 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.890 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.890 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251758+0000", 2023-10-09T10:48:33.890 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8909000000000001e-05 2023-10-09T10:48:33.890 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.890 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.891 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.891 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251780+0000", 2023-10-09T10:48:33.891 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2011999999999999e-05 2023-10-09T10:48:33.891 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.891 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.891 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.891 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251791+0000", 2023-10-09T10:48:33.891 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0147e-05 2023-10-09T10:48:33.891 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.891 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.891 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.891 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.253110+0000", 2023-10-09T10:48:33.892 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013193739999999999 2023-10-09T10:48:33.892 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.892 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.892 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.892 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.253287+0000", 2023-10-09T10:48:33.892 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017752800000000001 2023-10-09T10:48:33.892 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.892 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.892 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T10:48:33.892 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.253484+0000", 2023-10-09T10:48:33.892 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000196622 2023-10-09T10:48:33.892 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.893 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.893 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.893 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.257296+0000", 2023-10-09T10:48:33.893 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038119550000000001 2023-10-09T10:48:33.893 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.893 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.893 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.893 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.257345+0000", 2023-10-09T10:48:33.893 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8745000000000002e-05 2023-10-09T10:48:33.893 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.893 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.893 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.893 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279630+0000", 2023-10-09T10:48:33.894 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.02228538 2023-10-09T10:48:33.894 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.894 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.894 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.894 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279642+0000", 2023-10-09T10:48:33.894 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2328e-05 2023-10-09T10:48:33.894 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.894 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.894 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.894 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599919+0000", 2023-10-09T10:48:33.894 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32027628800000002 2023-10-09T10:48:33.894 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.894 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.895 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.895 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599931+0000", 2023-10-09T10:48:33.895 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2439000000000001e-05 2023-10-09T10:48:33.895 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.895 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.895 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.895 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.895 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.895 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:136053 6.1 6:832e3320:::benchmark_data_smithi125_244951_object29610:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-10-09T10:48:33.895 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:02.144996+0000", 2023-10-09T10:48:33.895 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 91.587903388000001, 2023-10-09T10:48:33.896 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.155053459, 2023-10-09T10:48:33.896 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:33.896 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:33.896 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:33.896 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:33.896 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:33.896 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 136053 2023-10-09T10:48:33.896 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.896 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:33.896 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.896 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:33.896 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.144996+0000", 2023-10-09T10:48:33.897 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.897 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.897 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.897 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:33.897 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.144996+0000", 2023-10-09T10:48:33.897 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:33.897 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.897 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.897 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:33.897 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.144994+0000", 2023-10-09T10:48:33.897 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:33.897 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.897 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.898 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:33.898 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145002+0000", 2023-10-09T10:48:33.898 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8429999999999993e-06 2023-10-09T10:48:33.898 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.898 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.898 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:33.898 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145014+0000", 2023-10-09T10:48:33.898 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2163e-05 2023-10-09T10:48:33.898 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.898 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.898 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:33.898 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145024+0000", 2023-10-09T10:48:33.899 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7610000000000002e-06 2023-10-09T10:48:33.899 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.899 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.899 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.899 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145083+0000", 2023-10-09T10:48:33.899 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8882999999999998e-05 2023-10-09T10:48:33.899 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.899 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.899 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:33.899 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145095+0000", 2023-10-09T10:48:33.899 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2255999999999999e-05 2023-10-09T10:48:33.899 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.900 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.900 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:33.900 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.287386+0000", 2023-10-09T10:48:33.900 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.14229140700000001 2023-10-09T10:48:33.900 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.900 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.900 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:33.900 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.287495+0000", 2023-10-09T10:48:33.900 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108651 2023-10-09T10:48:33.900 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.900 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.900 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:48:33.900 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.287593+0000", 2023-10-09T10:48:33.901 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8348000000000001e-05 2023-10-09T10:48:33.901 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.901 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.901 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.901 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.295087+0000", 2023-10-09T10:48:33.901 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074939550000000001 2023-10-09T10:48:33.901 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.901 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.901 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:33.901 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.295761+0000", 2023-10-09T10:48:33.901 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00067362100000000001 2023-10-09T10:48:33.901 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.902 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.902 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:33.902 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.299957+0000", 2023-10-09T10:48:33.902 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041957419999999997 2023-10-09T10:48:33.902 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.902 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.902 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:33.902 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.300009+0000", 2023-10-09T10:48:33.902 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2193e-05 2023-10-09T10:48:33.902 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:33.902 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:33.902 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:33.902 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.300049+0000", 2023-10-09T10:48:33.903 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0156e-05 2023-10-09T10:48:33.903 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.903 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.903 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.903 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:33.903 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:33.903 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:48:33.903 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:34.141 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:34.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:34.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:34.251 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:34.264 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:48:34.330 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:34.370 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:34.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:34.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:34.477 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:34.568 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:34 smithi125 ceph-mon[195507]: pgmap v406: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 27363/66957 objects degraded (40.867%); 2286/66957 objects misplaced (3.414%); 5.8 MiB/s, 146 objects/s recovering 2023-10-09T10:48:34.568 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:34 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:34.568 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:34 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:34.569 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:34 smithi125 ceph-mon[200197]: pgmap v406: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 27363/66957 objects degraded (40.867%); 2286/66957 objects misplaced (3.414%); 5.8 MiB/s, 146 objects/s recovering 2023-10-09T10:48:34.569 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:34 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:34.569 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:34 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:34.572 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:34.616 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:48:34.684 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:48:34.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:34 smithi155 ceph-mon[184854]: pgmap v406: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 27363/66957 objects degraded (40.867%); 2286/66957 objects misplaced (3.414%); 5.8 MiB/s, 146 objects/s recovering 2023-10-09T10:48:34.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:34 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:34.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:34 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:34.749 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:48:34.757 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:34.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:34.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:34.795 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:48:34.830 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:48:34.865 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:34.871 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:48:34.907 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:48:34.946 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:34.954 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:48:34.972 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:48:34.972 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:48:34.973 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:48:34.973 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:48:34.973 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:48:34.973 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:48:34.973 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:48:34.973 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:48:35.063 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:48:35.280 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:35.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:35.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:35.392 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:35.396 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:35.478 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:35 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2936425376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:35.479 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:48:35.479 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:48:35.479 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:35.479 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:35 smithi125 ceph-mon[195507]: pgmap v407: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 27363/62868 objects degraded (43.525%); 2190/62868 objects misplaced (3.483%); 4.9 MiB/s, 123 objects/s recovering 2023-10-09T10:48:35.479 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:35 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2936425376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:35.479 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:48:35.479 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:48:35.479 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:35.479 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:35 smithi125 ceph-mon[200197]: pgmap v407: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 27363/62868 objects degraded (43.525%); 2190/62868 objects misplaced (3.483%); 4.9 MiB/s, 123 objects/s recovering 2023-10-09T10:48:35.480 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:35.508 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:35 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2936425376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:48:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:48:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:35 smithi155 ceph-mon[184854]: pgmap v407: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 27363/62868 objects degraded (43.525%); 2190/62868 objects misplaced (3.483%); 4.9 MiB/s, 123 objects/s recovering 2023-10-09T10:48:35.871 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:35.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:35.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:35.975 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:36.034 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:36.068 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:36.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:36.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:36.176 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:36.226 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:36.280 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:36.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:36.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:36.384 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:36.435 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:36.465 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:36.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:36.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:36.570 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:36.637 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:36.725 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:36.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:36.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:36.839 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:36.883 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:36.909 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:36.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:36.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:37.014 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:37.019 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:48:37.029 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:48:37.076 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:37.410 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:48:37.467 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:37.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:37.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:37.572 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:37.587 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:48:37.587 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:48:37.687 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:38.070 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:48:38.178 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:38.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:38.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:38.213 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":407,"stamp":"2023-10-09T10:48:35.163525+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1372297144,"num_objects":20956,"num_object_clones":0,"num_object_copies":62868,"num_objects_missing_on_primary":708,"num_objects_missing":708,"num_objects_degraded":27363,"num_objects_misplaced":2190,"num_objects_unfound":0,"num_objects_dirty":20956,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":183576,"num_write_kb":6260974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27483,"num_bytes_recovered":1573391106,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173605,"ondisk_log_size":173605,"up":357,"acting":348,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5719128,"kb_used_data":4027296,"kb_used_omap":44,"kb_used_meta":1690963,"kb_avail":650496936,"statfs":{"total":671965249536,"available":666108862464,"internally_reserved":0,"allocated":4123951104,"data_stored":4118324908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1731546529},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,5,6,8,8],"upper_bound":64},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-419758080,"num_objects":-6405,"num_object_clones":0,"num_object_copies":-19215,"num_objects_missing_on_primary":-205,"num_objects_missing":-205,"num_objects_degraded":-373,"num_objects_misplaced":-1185,"num_objects_unfound":0,"num_objects_dirty":-6405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6405,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1236,"num_bytes_recovered":51052544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.012230"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":177,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275431+0000","last_change":"2023-10-09T10:46:32.250979+0000","last_active":"2023-10-09T10:48:17.275431+0000","last_peered":"2023-10-09T10:48:17.275431+0000","last_clean":"2023-10-09T10:48:17.275431+0000","last_became_active":"2023-10-09T10:46:32.250625+0000","last_became_peered":"2023-10-09T10:46:32.250625+0000","last_unstale":"2023-10-09T10:48:17.275431+0000","last_undegraded":"2023-10-09T10:48:17.275431+0000","last_fullsized":"2023-10-09T10:48:17.275431+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:34.858348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":154,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.390686+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:48:32.390686+0000","last_peered":"2023-10-09T10:48:32.390686+0000","last_clean":"2023-10-09T10:48:32.390686+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:48:32.390686+0000","last_undegraded":"2023-10-09T10:48:32.390686+0000","last_fullsized":"2023-10-09T10:48:32.390686+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":136,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267298+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:48:17.267298+0000","last_peered":"2023-10-09T10:48:17.267298+0000","last_clean":"2023-10-09T10:48:17.267298+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:48:17.267298+0000","last_undegraded":"2023-10-09T10:48:17.267298+0000","last_fullsized":"2023-10-09T10:48:17.267298+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":54,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.719528+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:48:24.719528+0000","last_peered":"2023-10-09T10:48:24.719528+0000","last_clean":"2023-10-09T10:48:24.719528+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:48:24.719528+0000","last_undegraded":"2023-10-09T10:48:24.719528+0000","last_fullsized":"2023-10-09T10:48:24.719528+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":190,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.732992+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:48:24.732992+0000","last_peered":"2023-10-09T10:48:24.732992+0000","last_clean":"2023-10-09T10:48:24.732992+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:48:24.732992+0000","last_undegraded":"2023-10-09T10:48:24.732992+0000","last_fullsized":"2023-10-09T10:48:24.732992+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":122,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286593+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:48:17.286593+0000","last_peered":"2023-10-09T10:48:17.286593+0000","last_clean":"2023-10-09T10:48:17.286593+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:48:17.286593+0000","last_undegraded":"2023-10-09T10:48:17.286593+0000","last_fullsized":"2023-10-09T10:48:17.286593+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"184'7179","reported_seq":11337,"reported_epoch":184,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:31.020473+0000","last_change":"2023-10-09T10:46:33.521602+0000","last_active":"2023-10-09T10:48:31.020473+0000","last_peered":"2023-10-09T10:48:31.020473+0000","last_clean":"2023-10-09T10:46:31.055751+0000","last_became_active":"2023-10-09T10:46:33.364058+0000","last_became_peered":"2023-10-09T10:46:33.364058+0000","last_unstale":"2023-10-09T10:48:31.020473+0000","last_undegraded":"2023-10-09T10:48:31.020473+0000","last_fullsized":"2023-10-09T10:48:31.020473+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7179,"log_dups_size":0,"ondisk_log_size":7179,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46825472,"num_objects":715,"num_object_clones":0,"num_object_copies":2145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":715,"num_objects_unfound":0,"num_objects_dirty":715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6143,"num_write_kb":219424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2010,"num_bytes_recovered":131727360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":715}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":54,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:27.795477+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:48:27.795477+0000","last_peered":"2023-10-09T10:48:27.795477+0000","last_clean":"2023-10-09T10:48:27.795477+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:48:27.795477+0000","last_undegraded":"2023-10-09T10:48:27.795477+0000","last_fullsized":"2023-10-09T10:48:27.795477+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":154,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.733103+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:48:24.733103+0000","last_peered":"2023-10-09T10:48:24.733103+0000","last_clean":"2023-10-09T10:48:24.733103+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:48:24.733103+0000","last_undegraded":"2023-10-09T10:48:24.733103+0000","last_fullsized":"2023-10-09T10:48:24.733103+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":519,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.719098+0000","last_change":"2023-10-09T10:46:05.620365+0000","last_active":"2023-10-09T10:48:24.719098+0000","last_peered":"2023-10-09T10:48:24.719098+0000","last_clean":"2023-10-09T10:48:24.719098+0000","last_became_active":"2023-10-09T10:46:05.620024+0000","last_became_peered":"2023-10-09T10:46:05.620024+0000","last_unstale":"2023-10-09T10:48:24.719098+0000","last_undegraded":"2023-10-09T10:48:24.719098+0000","last_fullsized":"2023-10-09T10:48:24.719098+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.13","version":"184'6976","reported_seq":8957,"reported_epoch":184,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:31.027091+0000","last_change":"2023-10-09T10:48:15.511525+0000","last_active":"2023-10-09T10:48:31.027091+0000","last_peered":"2023-10-09T10:48:31.027091+0000","last_clean":"2023-10-09T10:46:31.195699+0000","last_became_active":"2023-10-09T10:46:33.455632+0000","last_became_peered":"2023-10-09T10:46:33.455632+0000","last_unstale":"2023-10-09T10:48:31.027091+0000","last_undegraded":"2023-10-09T10:46:33.269323+0000","last_fullsized":"2023-10-09T10:46:33.241226+0000","mapping_epoch":154,"log_start":"108'2275","ondisk_log_start":"108'2275","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":4701,"log_dups_size":0,"ondisk_log_size":4701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:07:12.547333+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45613056,"num_objects":696,"num_object_clones":0,"num_object_copies":2088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1767,"num_objects_misplaced":113,"num_objects_unfound":0,"num_objects_dirty":696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4261,"num_write_kb":105088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":895,"num_bytes_recovered":58654720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":113},{"shards":"6,7","objects":-1184}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":149,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286522+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:48:17.286522+0000","last_peered":"2023-10-09T10:48:17.286522+0000","last_clean":"2023-10-09T10:48:17.286522+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:48:17.286522+0000","last_undegraded":"2023-10-09T10:48:17.286522+0000","last_fullsized":"2023-10-09T10:48:17.286522+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":111,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.726216+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:48:24.726216+0000","last_peered":"2023-10-09T10:48:24.726216+0000","last_clean":"2023-10-09T10:48:24.726216+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:48:24.726216+0000","last_undegraded":"2023-10-09T10:48:24.726216+0000","last_fullsized":"2023-10-09T10:48:24.726216+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"184'7126","reported_seq":15258,"reported_epoch":184,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:31.474183+0000","last_change":"2023-10-09T10:46:03.683196+0000","last_active":"2023-10-09T10:48:31.474183+0000","last_peered":"2023-10-09T10:48:31.474183+0000","last_clean":"2023-10-09T10:46:02.177309+0000","last_became_active":"2023-10-09T10:46:03.681058+0000","last_became_peered":"2023-10-09T10:46:03.681058+0000","last_unstale":"2023-10-09T10:48:31.474183+0000","last_undegraded":"2023-10-09T10:48:31.474183+0000","last_fullsized":"2023-10-09T10:48:31.474183+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7126,"log_dups_size":0,"ondisk_log_size":7126,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46366720,"num_objects":708,"num_object_clones":0,"num_object_copies":2124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":708,"num_objects_unfound":0,"num_objects_dirty":708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6080,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[1,5,6],"avail_no_missing":["1","2","3","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":708}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":106,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.726625+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:48:24.726625+0000","last_peered":"2023-10-09T10:48:24.726625+0000","last_clean":"2023-10-09T10:48:24.726625+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:48:24.726625+0000","last_undegraded":"2023-10-09T10:48:24.726625+0000","last_fullsized":"2023-10-09T10:48:24.726625+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":145,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.266962+0000","last_change":"2023-10-09T10:46:32.249921+0000","last_active":"2023-10-09T10:48:17.266962+0000","last_peered":"2023-10-09T10:48:17.266962+0000","last_clean":"2023-10-09T10:48:17.266962+0000","last_became_active":"2023-10-09T10:46:32.249590+0000","last_became_peered":"2023-10-09T10:46:32.249590+0000","last_unstale":"2023-10-09T10:48:17.266962+0000","last_undegraded":"2023-10-09T10:48:17.266962+0000","last_fullsized":"2023-10-09T10:48:17.266962+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:25.169752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"185'6982","reported_seq":15211,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:34.420231+0000","last_change":"2023-10-09T10:47:48.412928+0000","last_active":"2023-10-09T10:48:34.420231+0000","last_peered":"2023-10-09T10:48:34.420231+0000","last_clean":"2023-10-09T10:48:34.420231+0000","last_became_active":"2023-10-09T10:47:48.398813+0000","last_became_peered":"2023-10-09T10:47:48.398813+0000","last_unstale":"2023-10-09T10:48:34.420231+0000","last_undegraded":"2023-10-09T10:48:34.420231+0000","last_fullsized":"2023-10-09T10:48:34.420231+0000","mapping_epoch":177,"log_start":"103'628","ondisk_log_start":"103'628","created":107,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6354,"log_dups_size":0,"ondisk_log_size":6354,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39223296,"num_objects":599,"num_object_clones":0,"num_object_copies":1797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5975,"num_write_kb":210336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2691,"num_bytes_recovered":176357376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":154,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.276186+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:48:17.276186+0000","last_peered":"2023-10-09T10:48:17.276186+0000","last_clean":"2023-10-09T10:48:17.276186+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:48:17.276186+0000","last_undegraded":"2023-10-09T10:48:17.276186+0000","last_fullsized":"2023-10-09T10:48:17.276186+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":148,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275916+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:48:17.275916+0000","last_peered":"2023-10-09T10:48:17.275916+0000","last_clean":"2023-10-09T10:48:17.275916+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:48:17.275916+0000","last_undegraded":"2023-10-09T10:48:17.275916+0000","last_fullsized":"2023-10-09T10:48:17.275916+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":519,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.736847+0000","last_change":"2023-10-09T10:46:05.618276+0000","last_active":"2023-10-09T10:48:24.736847+0000","last_peered":"2023-10-09T10:48:24.736847+0000","last_clean":"2023-10-09T10:48:24.736847+0000","last_became_active":"2023-10-09T10:46:05.616509+0000","last_became_peered":"2023-10-09T10:46:05.616509+0000","last_unstale":"2023-10-09T10:48:24.736847+0000","last_undegraded":"2023-10-09T10:48:24.736847+0000","last_fullsized":"2023-10-09T10:48:24.736847+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.16","version":"185'7006","reported_seq":7116,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:34.313069+0000","last_change":"2023-10-09T10:46:03.592682+0000","last_active":"2023-10-09T10:48:34.313069+0000","last_peered":"2023-10-09T10:48:34.313069+0000","last_clean":"2023-10-09T10:48:34.313069+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:48:34.313069+0000","last_undegraded":"2023-10-09T10:48:34.313069+0000","last_fullsized":"2023-10-09T10:48:34.313069+0000","mapping_epoch":107,"log_start":"103'565","ondisk_log_start":"103'565","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6441,"log_dups_size":0,"ondisk_log_size":6441,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40828928,"num_objects":623,"num_object_clones":0,"num_object_copies":1869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5963,"num_write_kb":210752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":166,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.279501+0000","last_change":"2023-10-09T10:46:32.250256+0000","last_active":"2023-10-09T10:48:17.279501+0000","last_peered":"2023-10-09T10:48:17.279501+0000","last_clean":"2023-10-09T10:48:17.279501+0000","last_became_active":"2023-10-09T10:46:32.243312+0000","last_became_peered":"2023-10-09T10:46:32.243312+0000","last_unstale":"2023-10-09T10:48:17.279501+0000","last_undegraded":"2023-10-09T10:48:17.279501+0000","last_fullsized":"2023-10-09T10:48:17.279501+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:08.059777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":177,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.731008+0000","last_change":"2023-10-09T10:46:32.237577+0000","last_active":"2023-10-09T10:48:24.731008+0000","last_peered":"2023-10-09T10:48:24.731008+0000","last_clean":"2023-10-09T10:48:24.731008+0000","last_became_active":"2023-10-09T10:46:32.237316+0000","last_became_peered":"2023-10-09T10:46:32.237316+0000","last_unstale":"2023-10-09T10:48:24.731008+0000","last_undegraded":"2023-10-09T10:48:24.731008+0000","last_fullsized":"2023-10-09T10:48:24.731008+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:39.684602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":520,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.719338+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:48:24.719338+0000","last_peered":"2023-10-09T10:48:24.719338+0000","last_clean":"2023-10-09T10:48:24.719338+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:48:24.719338+0000","last_undegraded":"2023-10-09T10:48:24.719338+0000","last_fullsized":"2023-10-09T10:48:24.719338+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.17","version":"184'6898","reported_seq":7001,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:31.098713+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:48:31.098713+0000","last_peered":"2023-10-09T10:48:31.098713+0000","last_clean":"2023-10-09T10:48:31.098713+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:48:31.098713+0000","last_undegraded":"2023-10-09T10:48:31.098713+0000","last_fullsized":"2023-10-09T10:48:31.098713+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6898,"log_dups_size":0,"ondisk_log_size":6898,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40370176,"num_objects":616,"num_object_clones":0,"num_object_copies":1848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5890,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":54,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.720165+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:48:24.720165+0000","last_peered":"2023-10-09T10:48:24.720165+0000","last_clean":"2023-10-09T10:48:24.720165+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:48:24.720165+0000","last_undegraded":"2023-10-09T10:48:24.720165+0000","last_fullsized":"2023-10-09T10:48:24.720165+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":148,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286752+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:48:17.286752+0000","last_peered":"2023-10-09T10:48:17.286752+0000","last_clean":"2023-10-09T10:48:17.286752+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:48:17.286752+0000","last_undegraded":"2023-10-09T10:48:17.286752+0000","last_fullsized":"2023-10-09T10:48:17.286752+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":520,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.718568+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:48:24.718568+0000","last_peered":"2023-10-09T10:48:24.718568+0000","last_clean":"2023-10-09T10:48:24.718568+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:48:24.718568+0000","last_undegraded":"2023-10-09T10:48:24.718568+0000","last_fullsized":"2023-10-09T10:48:24.718568+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"184'7060","reported_seq":7168,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:31.399313+0000","last_change":"2023-10-09T10:46:33.243540+0000","last_active":"2023-10-09T10:48:31.399313+0000","last_peered":"2023-10-09T10:48:31.399313+0000","last_clean":"2023-10-09T10:48:31.399313+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:48:31.399313+0000","last_undegraded":"2023-10-09T10:48:31.399313+0000","last_fullsized":"2023-10-09T10:48:31.399313+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7060,"log_dups_size":0,"ondisk_log_size":7060,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43745280,"num_objects":668,"num_object_clones":0,"num_object_copies":2004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6006,"num_write_kb":213536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":94,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275638+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:48:17.275638+0000","last_peered":"2023-10-09T10:48:17.275638+0000","last_clean":"2023-10-09T10:48:17.275638+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:48:17.275638+0000","last_undegraded":"2023-10-09T10:48:17.275638+0000","last_fullsized":"2023-10-09T10:48:17.275638+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":54,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:27.795439+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:48:27.795439+0000","last_peered":"2023-10-09T10:48:27.795439+0000","last_clean":"2023-10-09T10:48:27.795439+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:48:27.795439+0000","last_undegraded":"2023-10-09T10:48:27.795439+0000","last_fullsized":"2023-10-09T10:48:27.795439+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":520,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.719916+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:48:24.719916+0000","last_peered":"2023-10-09T10:48:24.719916+0000","last_clean":"2023-10-09T10:48:24.719916+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:48:24.719916+0000","last_undegraded":"2023-10-09T10:48:24.719916+0000","last_fullsized":"2023-10-09T10:48:24.719916+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"185'7156","reported_seq":7275,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:34.550673+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:48:34.550673+0000","last_peered":"2023-10-09T10:48:34.550673+0000","last_clean":"2023-10-09T10:48:34.550673+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:48:34.550673+0000","last_undegraded":"2023-10-09T10:48:34.550673+0000","last_fullsized":"2023-10-09T10:48:34.550673+0000","mapping_epoch":115,"log_start":"103'878","ondisk_log_start":"103'878","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6278,"log_dups_size":0,"ondisk_log_size":6278,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37552128,"num_objects":573,"num_object_clones":0,"num_object_copies":1719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6095,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":64,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.718858+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:48:24.718858+0000","last_peered":"2023-10-09T10:48:24.718858+0000","last_clean":"2023-10-09T10:48:24.718858+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:48:24.718858+0000","last_undegraded":"2023-10-09T10:48:24.718858+0000","last_fullsized":"2023-10-09T10:48:24.718858+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":154,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.735859+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:48:24.735859+0000","last_peered":"2023-10-09T10:48:24.735859+0000","last_clean":"2023-10-09T10:48:24.735859+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:48:24.735859+0000","last_undegraded":"2023-10-09T10:48:24.735859+0000","last_fullsized":"2023-10-09T10:48:24.735859+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":520,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.726566+0000","last_change":"2023-10-09T10:46:05.620574+0000","last_active":"2023-10-09T10:48:24.726566+0000","last_peered":"2023-10-09T10:48:24.726566+0000","last_clean":"2023-10-09T10:48:24.726566+0000","last_became_active":"2023-10-09T10:46:05.620212+0000","last_became_peered":"2023-10-09T10:46:05.620212+0000","last_unstale":"2023-10-09T10:48:24.726566+0000","last_undegraded":"2023-10-09T10:48:24.726566+0000","last_fullsized":"2023-10-09T10:48:24.726566+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"184'12297","reported_seq":16456,"reported_epoch":184,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:31.127718+0000","last_change":"2023-10-09T10:46:32.247903+0000","last_active":"2023-10-09T10:48:31.127718+0000","last_peered":"2023-10-09T10:48:31.127718+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:46:02.739355+0000","last_became_peered":"2023-10-09T10:46:02.739355+0000","last_unstale":"2023-10-09T10:48:31.127718+0000","last_undegraded":"2023-10-09T10:46:02.608805+0000","last_fullsized":"2023-10-09T10:46:02.583589+0000","mapping_epoch":142,"log_start":"113'3400","ondisk_log_start":"113'3400","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8897,"log_dups_size":0,"ondisk_log_size":8897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:32.899634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87490560,"num_objects":1335,"num_object_clones":0,"num_object_copies":4005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12297,"num_write_kb":436224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2022,"num_bytes_recovered":132513792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":625},{"shards":"5,6","objects":709}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":177,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275403+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:48:17.275403+0000","last_peered":"2023-10-09T10:48:17.275403+0000","last_clean":"2023-10-09T10:48:17.275403+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:48:17.275403+0000","last_undegraded":"2023-10-09T10:48:17.275403+0000","last_fullsized":"2023-10-09T10:48:17.275403+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":131,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286332+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:48:17.286332+0000","last_peered":"2023-10-09T10:48:17.286332+0000","last_clean":"2023-10-09T10:48:17.286332+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:48:17.286332+0000","last_undegraded":"2023-10-09T10:48:17.286332+0000","last_fullsized":"2023-10-09T10:48:17.286332+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":500,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.719174+0000","last_change":"2023-10-09T10:46:02.631378+0000","last_active":"2023-10-09T10:48:24.719174+0000","last_peered":"2023-10-09T10:48:24.719174+0000","last_clean":"2023-10-09T10:48:24.719174+0000","last_became_active":"2023-10-09T10:46:02.629142+0000","last_became_peered":"2023-10-09T10:46:02.629142+0000","last_unstale":"2023-10-09T10:48:24.719174+0000","last_undegraded":"2023-10-09T10:48:24.719174+0000","last_fullsized":"2023-10-09T10:48:24.719174+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.b","version":"185'11976","reported_seq":12072,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:34.627020+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:48:34.627020+0000","last_peered":"2023-10-09T10:48:34.627020+0000","last_clean":"2023-10-09T10:48:34.627020+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:48:34.627020+0000","last_undegraded":"2023-10-09T10:48:34.627020+0000","last_fullsized":"2023-10-09T10:48:34.627020+0000","mapping_epoch":98,"log_start":"140'6225","ondisk_log_start":"140'6225","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5751,"log_dups_size":0,"ondisk_log_size":5751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78905344,"num_objects":1204,"num_object_clones":0,"num_object_copies":3612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11976,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":150,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275053+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:48:17.275053+0000","last_peered":"2023-10-09T10:48:17.275053+0000","last_clean":"2023-10-09T10:48:17.275053+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:48:17.275053+0000","last_undegraded":"2023-10-09T10:48:17.275053+0000","last_fullsized":"2023-10-09T10:48:17.275053+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":185,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275568+0000","last_change":"2023-10-09T10:46:32.251035+0000","last_active":"2023-10-09T10:48:17.275568+0000","last_peered":"2023-10-09T10:48:17.275568+0000","last_clean":"2023-10-09T10:48:17.275568+0000","last_became_active":"2023-10-09T10:46:32.250293+0000","last_became_peered":"2023-10-09T10:46:32.250293+0000","last_unstale":"2023-10-09T10:48:17.275568+0000","last_undegraded":"2023-10-09T10:48:17.275568+0000","last_fullsized":"2023-10-09T10:48:17.275568+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.787766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":520,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.719897+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:48:24.719897+0000","last_peered":"2023-10-09T10:48:24.719897+0000","last_clean":"2023-10-09T10:48:24.719897+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:48:24.719897+0000","last_undegraded":"2023-10-09T10:48:24.719897+0000","last_fullsized":"2023-10-09T10:48:24.719897+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"185'7120","reported_seq":7237,"reported_epoch":185,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:34.349986+0000","last_change":"2023-10-09T10:46:33.540719+0000","last_active":"2023-10-09T10:48:34.349986+0000","last_peered":"2023-10-09T10:48:34.349986+0000","last_clean":"2023-10-09T10:46:31.137696+0000","last_became_active":"2023-10-09T10:46:33.527650+0000","last_became_peered":"2023-10-09T10:46:33.527650+0000","last_unstale":"2023-10-09T10:48:34.349986+0000","last_undegraded":"2023-10-09T10:46:33.285468+0000","last_fullsized":"2023-10-09T10:46:33.235683+0000","mapping_epoch":154,"log_start":"103'760","ondisk_log_start":"103'760","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6360,"log_dups_size":0,"ondisk_log_size":6360,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41385984,"num_objects":631,"num_object_clones":0,"num_object_copies":1893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6055,"num_write_kb":213984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":619},{"shards":"3,7","objects":-2178}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":105,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286674+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:48:17.286674+0000","last_peered":"2023-10-09T10:48:17.286674+0000","last_clean":"2023-10-09T10:48:17.286674+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:48:17.286674+0000","last_undegraded":"2023-10-09T10:48:17.286674+0000","last_fullsized":"2023-10-09T10:48:17.286674+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":122,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267223+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:48:17.267223+0000","last_peered":"2023-10-09T10:48:17.267223+0000","last_clean":"2023-10-09T10:48:17.267223+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:48:17.267223+0000","last_undegraded":"2023-10-09T10:48:17.267223+0000","last_fullsized":"2023-10-09T10:48:17.267223+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":520,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.719439+0000","last_change":"2023-10-09T10:46:05.620030+0000","last_active":"2023-10-09T10:48:24.719439+0000","last_peered":"2023-10-09T10:48:24.719439+0000","last_clean":"2023-10-09T10:48:24.719439+0000","last_became_active":"2023-10-09T10:46:05.619694+0000","last_became_peered":"2023-10-09T10:46:05.619694+0000","last_unstale":"2023-10-09T10:48:24.719439+0000","last_undegraded":"2023-10-09T10:48:24.719439+0000","last_fullsized":"2023-10-09T10:48:24.719439+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.9","version":"185'7019","reported_seq":8558,"reported_epoch":185,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:32.792429+0000","last_change":"2023-10-09T10:48:15.517098+0000","last_active":"2023-10-09T10:48:32.792429+0000","last_peered":"2023-10-09T10:48:32.792429+0000","last_clean":"2023-10-09T10:46:31.190603+0000","last_became_active":"2023-10-09T10:46:34.396448+0000","last_became_peered":"2023-10-09T10:46:34.396448+0000","last_unstale":"2023-10-09T10:48:32.792429+0000","last_undegraded":"2023-10-09T10:46:33.246949+0000","last_fullsized":"2023-10-09T10:46:33.237968+0000","mapping_epoch":154,"log_start":"105'1607","ondisk_log_start":"105'1607","created":98,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5412,"log_dups_size":0,"ondisk_log_size":5412,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41746432,"num_objects":637,"num_object_clones":0,"num_object_copies":1911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1299,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5977,"num_write_kb":211648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2008,"num_bytes_recovered":85458944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,7","objects":-662}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":149,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.287413+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:48:17.287413+0000","last_peered":"2023-10-09T10:48:17.287413+0000","last_clean":"2023-10-09T10:48:17.287413+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:48:17.287413+0000","last_undegraded":"2023-10-09T10:48:17.287413+0000","last_fullsized":"2023-10-09T10:48:17.287413+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":54,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.718844+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:48:24.718844+0000","last_peered":"2023-10-09T10:48:24.718844+0000","last_clean":"2023-10-09T10:48:24.718844+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:48:24.718844+0000","last_undegraded":"2023-10-09T10:48:24.718844+0000","last_fullsized":"2023-10-09T10:48:24.718844+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":511,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.736688+0000","last_change":"2023-10-09T10:46:05.618134+0000","last_active":"2023-10-09T10:48:24.736688+0000","last_peered":"2023-10-09T10:48:24.736688+0000","last_clean":"2023-10-09T10:48:24.736688+0000","last_became_active":"2023-10-09T10:46:05.616057+0000","last_became_peered":"2023-10-09T10:46:05.616057+0000","last_unstale":"2023-10-09T10:48:24.736688+0000","last_undegraded":"2023-10-09T10:48:24.736688+0000","last_fullsized":"2023-10-09T10:48:24.736688+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"184'11829","reported_seq":11931,"reported_epoch":184,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:31.064648+0000","last_change":"2023-10-09T10:46:33.576530+0000","last_active":"2023-10-09T10:48:31.064648+0000","last_peered":"2023-10-09T10:48:31.064648+0000","last_clean":"2023-10-09T10:46:31.183481+0000","last_became_active":"2023-10-09T10:46:33.542304+0000","last_became_peered":"2023-10-09T10:46:33.542304+0000","last_unstale":"2023-10-09T10:48:31.064648+0000","last_undegraded":"2023-10-09T10:46:33.300581+0000","last_fullsized":"2023-10-09T10:46:33.246047+0000","mapping_epoch":154,"log_start":"125'4477","ondisk_log_start":"125'4477","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7352,"log_dups_size":0,"ondisk_log_size":7352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:26.739488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92078080,"num_objects":1405,"num_object_clones":0,"num_object_copies":4215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6617,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11829,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1405},{"shards":"1,4","objects":-4156}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":178,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.725970+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:48:24.725970+0000","last_peered":"2023-10-09T10:48:24.725970+0000","last_clean":"2023-10-09T10:48:24.725970+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:48:24.725970+0000","last_undegraded":"2023-10-09T10:48:24.725970+0000","last_fullsized":"2023-10-09T10:48:24.725970+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":148,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.276097+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:48:17.276097+0000","last_peered":"2023-10-09T10:48:17.276097+0000","last_clean":"2023-10-09T10:48:17.276097+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:48:17.276097+0000","last_undegraded":"2023-10-09T10:48:17.276097+0000","last_fullsized":"2023-10-09T10:48:17.276097+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":500,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.719449+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:48:24.719449+0000","last_peered":"2023-10-09T10:48:24.719449+0000","last_clean":"2023-10-09T10:48:24.719449+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:48:24.719449+0000","last_undegraded":"2023-10-09T10:48:24.719449+0000","last_fullsized":"2023-10-09T10:48:24.719449+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.f","version":"185'11781","reported_seq":11884,"reported_epoch":185,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:34.550985+0000","last_change":"2023-10-09T10:46:33.651756+0000","last_active":"2023-10-09T10:48:34.550985+0000","last_peered":"2023-10-09T10:48:34.550985+0000","last_clean":"2023-10-09T10:46:31.156458+0000","last_became_active":"2023-10-09T10:46:33.621003+0000","last_became_peered":"2023-10-09T10:46:33.621003+0000","last_unstale":"2023-10-09T10:48:34.550985+0000","last_undegraded":"2023-10-09T10:46:33.309813+0000","last_fullsized":"2023-10-09T10:46:33.236508+0000","mapping_epoch":154,"log_start":"140'6110","ondisk_log_start":"140'6110","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5671,"log_dups_size":0,"ondisk_log_size":5671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:13.704346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81330176,"num_objects":1241,"num_object_clones":0,"num_object_copies":3723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11781,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":1241},{"shards":"3,4","objects":-4211}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":123,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275599+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:48:17.275599+0000","last_peered":"2023-10-09T10:48:17.275599+0000","last_clean":"2023-10-09T10:48:17.275599+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:48:17.275599+0000","last_undegraded":"2023-10-09T10:48:17.275599+0000","last_fullsized":"2023-10-09T10:48:17.275599+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":175,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.276719+0000","last_change":"2023-10-09T10:46:32.245583+0000","last_active":"2023-10-09T10:48:17.276719+0000","last_peered":"2023-10-09T10:48:17.276719+0000","last_clean":"2023-10-09T10:48:17.276719+0000","last_became_active":"2023-10-09T10:46:32.245271+0000","last_became_peered":"2023-10-09T10:46:32.245271+0000","last_unstale":"2023-10-09T10:48:17.276719+0000","last_undegraded":"2023-10-09T10:48:17.276719+0000","last_fullsized":"2023-10-09T10:48:17.276719+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.025051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":520,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.718639+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:48:24.718639+0000","last_peered":"2023-10-09T10:48:24.718639+0000","last_clean":"2023-10-09T10:48:24.718639+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:48:24.718639+0000","last_undegraded":"2023-10-09T10:48:24.718639+0000","last_fullsized":"2023-10-09T10:48:24.718639+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"185'12032","reported_seq":12129,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:34.579074+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:48:34.579074+0000","last_peered":"2023-10-09T10:48:34.579074+0000","last_clean":"2023-10-09T10:48:34.579074+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:48:34.579074+0000","last_undegraded":"2023-10-09T10:48:34.579074+0000","last_fullsized":"2023-10-09T10:48:34.579074+0000","mapping_epoch":98,"log_start":"140'6278","ondisk_log_start":"140'6278","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5754,"log_dups_size":0,"ondisk_log_size":5754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82051072,"num_objects":1252,"num_object_clones":0,"num_object_copies":3756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12032,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":159,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275500+0000","last_change":"2023-10-09T10:46:32.245685+0000","last_active":"2023-10-09T10:48:17.275500+0000","last_peered":"2023-10-09T10:48:17.275500+0000","last_clean":"2023-10-09T10:48:17.275500+0000","last_became_active":"2023-10-09T10:46:32.245314+0000","last_became_peered":"2023-10-09T10:46:32.245314+0000","last_unstale":"2023-10-09T10:48:17.275500+0000","last_undegraded":"2023-10-09T10:48:17.275500+0000","last_fullsized":"2023-10-09T10:48:17.275500+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:51.716860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":148,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275891+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:48:17.275891+0000","last_peered":"2023-10-09T10:48:17.275891+0000","last_clean":"2023-10-09T10:48:17.275891+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:48:17.275891+0000","last_undegraded":"2023-10-09T10:48:17.275891+0000","last_fullsized":"2023-10-09T10:48:17.275891+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":511,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.720003+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:48:24.720003+0000","last_peered":"2023-10-09T10:48:24.720003+0000","last_clean":"2023-10-09T10:48:24.720003+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:48:24.720003+0000","last_undegraded":"2023-10-09T10:48:24.720003+0000","last_fullsized":"2023-10-09T10:48:24.720003+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"184'12080","reported_seq":12194,"reported_epoch":184,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:31.139905+0000","last_change":"2023-10-09T10:46:02.754507+0000","last_active":"2023-10-09T10:48:31.139905+0000","last_peered":"2023-10-09T10:48:31.139905+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:46:02.747510+0000","last_became_peered":"2023-10-09T10:46:02.747510+0000","last_unstale":"2023-10-09T10:48:31.139905+0000","last_undegraded":"2023-10-09T10:46:02.605987+0000","last_fullsized":"2023-10-09T10:46:02.585248+0000","mapping_epoch":142,"log_start":"113'3200","ondisk_log_start":"113'3200","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8880,"log_dups_size":0,"ondisk_log_size":8880,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:59.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86179864,"num_objects":1316,"num_object_clones":0,"num_object_copies":3948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1316,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12080,"num_write_kb":428609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,2,5","objects":1315},{"shards":"1,5","objects":-1402}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":155,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.732800+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:48:24.732800+0000","last_peered":"2023-10-09T10:48:24.732800+0000","last_clean":"2023-10-09T10:48:24.732800+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:48:24.732800+0000","last_undegraded":"2023-10-09T10:48:24.732800+0000","last_fullsized":"2023-10-09T10:48:24.732800+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":136,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267791+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:48:17.267791+0000","last_peered":"2023-10-09T10:48:17.267791+0000","last_clean":"2023-10-09T10:48:17.267791+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:48:17.267791+0000","last_undegraded":"2023-10-09T10:48:17.267791+0000","last_fullsized":"2023-10-09T10:48:17.267791+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":520,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.719990+0000","last_change":"2023-10-09T10:46:05.617983+0000","last_active":"2023-10-09T10:48:24.719990+0000","last_peered":"2023-10-09T10:48:24.719990+0000","last_clean":"2023-10-09T10:48:24.719990+0000","last_became_active":"2023-10-09T10:46:05.616021+0000","last_became_peered":"2023-10-09T10:46:05.616021+0000","last_unstale":"2023-10-09T10:48:24.719990+0000","last_undegraded":"2023-10-09T10:48:24.719990+0000","last_fullsized":"2023-10-09T10:48:24.719990+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":179,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267293+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:48:17.267293+0000","last_peered":"2023-10-09T10:48:17.267293+0000","last_clean":"2023-10-09T10:48:17.267293+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:48:17.267293+0000","last_undegraded":"2023-10-09T10:48:17.267293+0000","last_fullsized":"2023-10-09T10:48:17.267293+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":176,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286825+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:48:17.286825+0000","last_peered":"2023-10-09T10:48:17.286825+0000","last_clean":"2023-10-09T10:48:17.286825+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:48:17.286825+0000","last_undegraded":"2023-10-09T10:48:17.286825+0000","last_fullsized":"2023-10-09T10:48:17.286825+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":121,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267037+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:48:17.267037+0000","last_peered":"2023-10-09T10:48:17.267037+0000","last_clean":"2023-10-09T10:48:17.267037+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:48:17.267037+0000","last_undegraded":"2023-10-09T10:48:17.267037+0000","last_fullsized":"2023-10-09T10:48:17.267037+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"185'7037","reported_seq":11079,"reported_epoch":185,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:34.764192+0000","last_change":"2023-10-09T10:46:32.248218+0000","last_active":"2023-10-09T10:48:34.764192+0000","last_peered":"2023-10-09T10:48:34.764192+0000","last_clean":"2023-10-09T10:46:02.181912+0000","last_became_active":"2023-10-09T10:46:32.226237+0000","last_became_peered":"2023-10-09T10:46:32.226237+0000","last_unstale":"2023-10-09T10:48:34.764192+0000","last_undegraded":"2023-10-09T10:48:34.764192+0000","last_fullsized":"2023-10-09T10:48:34.764192+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7037,"log_dups_size":0,"ondisk_log_size":7037,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42893312,"num_objects":654,"num_object_clones":0,"num_object_copies":1962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":654,"num_objects_unfound":0,"num_objects_dirty":654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5990,"num_write_kb":212640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":654}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":519,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.722714+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:48:24.722714+0000","last_peered":"2023-10-09T10:48:24.722714+0000","last_clean":"2023-10-09T10:48:24.722714+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:48:24.722714+0000","last_undegraded":"2023-10-09T10:48:24.722714+0000","last_fullsized":"2023-10-09T10:48:24.722714+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"185'7179","reported_seq":7366,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:34.433663+0000","last_change":"2023-10-09T10:46:02.760963+0000","last_active":"2023-10-09T10:48:34.433663+0000","last_peered":"2023-10-09T10:48:34.433663+0000","last_clean":"2023-10-09T10:48:34.433663+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:48:34.433663+0000","last_undegraded":"2023-10-09T10:48:34.433663+0000","last_fullsized":"2023-10-09T10:48:34.433663+0000","mapping_epoch":98,"log_start":"104'993","ondisk_log_start":"104'993","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"141'4416","last_scrub_stamp":"2023-10-09T10:46:02.760810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:46:02.760810+0000","objects_scrubbed":3378,"log_size":6186,"log_dups_size":0,"ondisk_log_size":6186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:35.863002+0000","scrub_duration":0.47769323899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40304640,"num_objects":615,"num_object_clones":0,"num_object_copies":1845,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":615,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2763,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":153,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286437+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:48:17.286437+0000","last_peered":"2023-10-09T10:48:17.286437+0000","last_clean":"2023-10-09T10:48:17.286437+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:48:17.286437+0000","last_undegraded":"2023-10-09T10:48:17.286437+0000","last_fullsized":"2023-10-09T10:48:17.286437+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":169,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.276047+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:48:17.276047+0000","last_peered":"2023-10-09T10:48:17.276047+0000","last_clean":"2023-10-09T10:48:17.276047+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:48:17.276047+0000","last_undegraded":"2023-10-09T10:48:17.276047+0000","last_fullsized":"2023-10-09T10:48:17.276047+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":160,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267390+0000","last_change":"2023-10-09T10:46:32.245608+0000","last_active":"2023-10-09T10:48:17.267390+0000","last_peered":"2023-10-09T10:48:17.267390+0000","last_clean":"2023-10-09T10:48:17.267390+0000","last_became_active":"2023-10-09T10:46:32.245290+0000","last_became_peered":"2023-10-09T10:46:32.245290+0000","last_unstale":"2023-10-09T10:48:17.267390+0000","last_undegraded":"2023-10-09T10:48:17.267390+0000","last_fullsized":"2023-10-09T10:48:17.267390+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:48.832515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":520,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.720085+0000","last_change":"2023-10-09T10:46:05.620057+0000","last_active":"2023-10-09T10:48:24.720085+0000","last_peered":"2023-10-09T10:48:24.720085+0000","last_clean":"2023-10-09T10:48:24.720085+0000","last_became_active":"2023-10-09T10:46:05.619703+0000","last_became_peered":"2023-10-09T10:46:05.619703+0000","last_unstale":"2023-10-09T10:48:24.720085+0000","last_undegraded":"2023-10-09T10:48:24.720085+0000","last_fullsized":"2023-10-09T10:48:24.720085+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"185'7094","reported_seq":7216,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:34.509480+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:48:34.509480+0000","last_peered":"2023-10-09T10:48:34.509480+0000","last_clean":"2023-10-09T10:48:34.509480+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:48:34.509480+0000","last_undegraded":"2023-10-09T10:48:34.509480+0000","last_fullsized":"2023-10-09T10:48:34.509480+0000","mapping_epoch":115,"log_start":"103'712","ondisk_log_start":"103'712","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6382,"log_dups_size":0,"ondisk_log_size":6382,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42401792,"num_objects":647,"num_object_clones":0,"num_object_copies":1941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6033,"num_write_kb":213760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":182,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.726298+0000","last_change":"2023-10-09T10:46:32.241041+0000","last_active":"2023-10-09T10:48:24.726298+0000","last_peered":"2023-10-09T10:48:24.726298+0000","last_clean":"2023-10-09T10:48:24.726298+0000","last_became_active":"2023-10-09T10:46:32.238683+0000","last_became_peered":"2023-10-09T10:46:32.238683+0000","last_unstale":"2023-10-09T10:48:24.726298+0000","last_undegraded":"2023-10-09T10:48:24.726298+0000","last_fullsized":"2023-10-09T10:48:24.726298+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:25.817713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":145,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.266963+0000","last_change":"2023-10-09T10:46:32.250104+0000","last_active":"2023-10-09T10:48:17.266963+0000","last_peered":"2023-10-09T10:48:17.266963+0000","last_clean":"2023-10-09T10:48:17.266963+0000","last_became_active":"2023-10-09T10:46:32.249805+0000","last_became_peered":"2023-10-09T10:46:32.249805+0000","last_unstale":"2023-10-09T10:48:17.266963+0000","last_undegraded":"2023-10-09T10:48:17.266963+0000","last_fullsized":"2023-10-09T10:48:17.266963+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:07.230672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":174,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267381+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:48:17.267381+0000","last_peered":"2023-10-09T10:48:17.267381+0000","last_clean":"2023-10-09T10:48:17.267381+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:48:17.267381+0000","last_undegraded":"2023-10-09T10:48:17.267381+0000","last_fullsized":"2023-10-09T10:48:17.267381+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"184'6934","reported_seq":7042,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:31.119085+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:48:31.119085+0000","last_peered":"2023-10-09T10:48:31.119085+0000","last_clean":"2023-10-09T10:48:31.119085+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:48:31.119085+0000","last_undegraded":"2023-10-09T10:48:31.119085+0000","last_fullsized":"2023-10-09T10:48:31.119085+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6934,"log_dups_size":0,"ondisk_log_size":6934,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47448064,"num_objects":724,"num_object_clones":0,"num_object_copies":2172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5926,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":163,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286437+0000","last_change":"2023-10-09T10:46:32.235346+0000","last_active":"2023-10-09T10:48:17.286437+0000","last_peered":"2023-10-09T10:48:17.286437+0000","last_clean":"2023-10-09T10:48:17.286437+0000","last_became_active":"2023-10-09T10:46:32.234906+0000","last_became_peered":"2023-10-09T10:46:32.234906+0000","last_unstale":"2023-10-09T10:48:17.286437+0000","last_undegraded":"2023-10-09T10:48:17.286437+0000","last_fullsized":"2023-10-09T10:48:17.286437+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.199251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":520,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.718682+0000","last_change":"2023-10-09T10:46:05.617146+0000","last_active":"2023-10-09T10:48:24.718682+0000","last_peered":"2023-10-09T10:48:24.718682+0000","last_clean":"2023-10-09T10:48:24.718682+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:48:24.718682+0000","last_undegraded":"2023-10-09T10:48:24.718682+0000","last_fullsized":"2023-10-09T10:48:24.718682+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":113,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.725825+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:48:24.725825+0000","last_peered":"2023-10-09T10:48:24.725825+0000","last_clean":"2023-10-09T10:48:24.725825+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:48:24.725825+0000","last_undegraded":"2023-10-09T10:48:24.725825+0000","last_fullsized":"2023-10-09T10:48:24.725825+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":81,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.276548+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:48:17.276548+0000","last_peered":"2023-10-09T10:48:17.276548+0000","last_clean":"2023-10-09T10:48:17.276548+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:48:17.276548+0000","last_undegraded":"2023-10-09T10:48:17.276548+0000","last_fullsized":"2023-10-09T10:48:17.276548+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":520,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.718568+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:48:24.718568+0000","last_peered":"2023-10-09T10:48:24.718568+0000","last_clean":"2023-10-09T10:48:24.718568+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:48:24.718568+0000","last_undegraded":"2023-10-09T10:48:24.718568+0000","last_fullsized":"2023-10-09T10:48:24.718568+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":175,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267086+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:48:17.267086+0000","last_peered":"2023-10-09T10:48:17.267086+0000","last_clean":"2023-10-09T10:48:17.267086+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:48:17.267086+0000","last_undegraded":"2023-10-09T10:48:17.267086+0000","last_fullsized":"2023-10-09T10:48:17.267086+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":63,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.722632+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:48:24.722632+0000","last_peered":"2023-10-09T10:48:24.722632+0000","last_clean":"2023-10-09T10:48:24.722632+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:48:24.722632+0000","last_undegraded":"2023-10-09T10:48:24.722632+0000","last_fullsized":"2023-10-09T10:48:24.722632+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":63,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.719673+0000","last_change":"2023-10-09T10:46:32.236202+0000","last_active":"2023-10-09T10:48:24.719673+0000","last_peered":"2023-10-09T10:48:24.719673+0000","last_clean":"2023-10-09T10:48:24.719673+0000","last_became_active":"2023-10-09T10:46:32.235901+0000","last_became_peered":"2023-10-09T10:46:32.235901+0000","last_unstale":"2023-10-09T10:48:24.719673+0000","last_undegraded":"2023-10-09T10:48:24.719673+0000","last_fullsized":"2023-10-09T10:48:24.719673+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:48.260403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"184'7066","reported_seq":7221,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:31.477992+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:48:31.477992+0000","last_peered":"2023-10-09T10:48:31.477992+0000","last_clean":"2023-10-09T10:48:31.477992+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:48:31.477992+0000","last_undegraded":"2023-10-09T10:48:31.477992+0000","last_fullsized":"2023-10-09T10:48:31.477992+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":7066,"log_dups_size":0,"ondisk_log_size":7066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46661632,"num_objects":712,"num_object_clones":0,"num_object_copies":2136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4036,"num_write_kb":89664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":520,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.719487+0000","last_change":"2023-10-09T10:46:05.620764+0000","last_active":"2023-10-09T10:48:24.719487+0000","last_peered":"2023-10-09T10:48:24.719487+0000","last_clean":"2023-10-09T10:48:24.719487+0000","last_became_active":"2023-10-09T10:46:05.619870+0000","last_became_peered":"2023-10-09T10:46:05.619870+0000","last_unstale":"2023-10-09T10:48:24.719487+0000","last_undegraded":"2023-10-09T10:48:24.719487+0000","last_fullsized":"2023-10-09T10:48:24.719487+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":172,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.274200+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:48:17.274200+0000","last_peered":"2023-10-09T10:48:17.274200+0000","last_clean":"2023-10-09T10:48:17.274200+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:48:17.274200+0000","last_undegraded":"2023-10-09T10:48:17.274200+0000","last_fullsized":"2023-10-09T10:48:17.274200+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":145,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267482+0000","last_change":"2023-10-09T10:46:32.251450+0000","last_active":"2023-10-09T10:48:17.267482+0000","last_peered":"2023-10-09T10:48:17.267482+0000","last_clean":"2023-10-09T10:48:17.267482+0000","last_became_active":"2023-10-09T10:46:32.250933+0000","last_became_peered":"2023-10-09T10:46:32.250933+0000","last_unstale":"2023-10-09T10:48:17.267482+0000","last_undegraded":"2023-10-09T10:48:17.267482+0000","last_fullsized":"2023-10-09T10:48:17.267482+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:04.116178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":54,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.718727+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:48:24.718727+0000","last_peered":"2023-10-09T10:48:24.718727+0000","last_clean":"2023-10-09T10:48:24.718727+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:48:24.718727+0000","last_undegraded":"2023-10-09T10:48:24.718727+0000","last_fullsized":"2023-10-09T10:48:24.718727+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"184'6900","reported_seq":13600,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:31.277565+0000","last_change":"2023-10-09T10:47:02.269289+0000","last_active":"2023-10-09T10:48:31.277565+0000","last_peered":"2023-10-09T10:48:31.277565+0000","last_clean":"2023-10-09T10:48:31.277565+0000","last_became_active":"2023-10-09T10:47:02.257854+0000","last_became_peered":"2023-10-09T10:47:02.257854+0000","last_unstale":"2023-10-09T10:48:31.277565+0000","last_undegraded":"2023-10-09T10:48:31.277565+0000","last_fullsized":"2023-10-09T10:48:31.277565+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6900,"log_dups_size":0,"ondisk_log_size":6900,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46563328,"num_objects":710,"num_object_clones":0,"num_object_copies":2130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5892,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3287,"num_bytes_recovered":181927936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":520,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.723688+0000","last_change":"2023-10-09T10:46:05.618443+0000","last_active":"2023-10-09T10:48:24.723688+0000","last_peered":"2023-10-09T10:48:24.723688+0000","last_clean":"2023-10-09T10:48:24.723688+0000","last_became_active":"2023-10-09T10:46:05.616588+0000","last_became_peered":"2023-10-09T10:46:05.616588+0000","last_unstale":"2023-10-09T10:48:24.723688+0000","last_undegraded":"2023-10-09T10:48:24.723688+0000","last_fullsized":"2023-10-09T10:48:24.723688+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":178,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.735758+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:48:24.735758+0000","last_peered":"2023-10-09T10:48:24.735758+0000","last_clean":"2023-10-09T10:48:24.735758+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:48:24.735758+0000","last_undegraded":"2023-10-09T10:48:24.735758+0000","last_fullsized":"2023-10-09T10:48:24.735758+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":121,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.267491+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:48:17.267491+0000","last_peered":"2023-10-09T10:48:17.267491+0000","last_clean":"2023-10-09T10:48:17.267491+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:48:17.267491+0000","last_undegraded":"2023-10-09T10:48:17.267491+0000","last_fullsized":"2023-10-09T10:48:17.267491+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":167,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.732879+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:48:24.732879+0000","last_peered":"2023-10-09T10:48:24.732879+0000","last_clean":"2023-10-09T10:48:24.732879+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:48:24.732879+0000","last_undegraded":"2023-10-09T10:48:24.732879+0000","last_fullsized":"2023-10-09T10:48:24.732879+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"184'7022","reported_seq":13862,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:31.046673+0000","last_change":"2023-10-09T10:48:17.301310+0000","last_active":"2023-10-09T10:48:31.046673+0000","last_peered":"2023-10-09T10:48:31.046673+0000","last_clean":"2023-10-09T10:48:31.046673+0000","last_became_active":"2023-10-09T10:48:17.283042+0000","last_became_peered":"2023-10-09T10:48:17.283042+0000","last_unstale":"2023-10-09T10:48:31.046673+0000","last_undegraded":"2023-10-09T10:48:31.046673+0000","last_fullsized":"2023-10-09T10:48:31.046673+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7022,"log_dups_size":0,"ondisk_log_size":7022,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44531712,"num_objects":679,"num_object_clones":0,"num_object_copies":2037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5985,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3337,"num_bytes_recovered":129564672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":519,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.718846+0000","last_change":"2023-10-09T10:46:05.617589+0000","last_active":"2023-10-09T10:48:24.718846+0000","last_peered":"2023-10-09T10:48:24.718846+0000","last_clean":"2023-10-09T10:48:24.718846+0000","last_became_active":"2023-10-09T10:46:05.616400+0000","last_became_peered":"2023-10-09T10:46:05.616400+0000","last_unstale":"2023-10-09T10:48:24.718846+0000","last_undegraded":"2023-10-09T10:48:24.718846+0000","last_fullsized":"2023-10-09T10:48:24.718846+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":42,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.735402+0000","last_change":"2023-10-09T10:46:32.245106+0000","last_active":"2023-10-09T10:48:24.735402+0000","last_peered":"2023-10-09T10:48:24.735402+0000","last_clean":"2023-10-09T10:48:24.735402+0000","last_became_active":"2023-10-09T10:46:32.244826+0000","last_became_peered":"2023-10-09T10:46:32.244826+0000","last_unstale":"2023-10-09T10:48:24.735402+0000","last_undegraded":"2023-10-09T10:48:24.735402+0000","last_fullsized":"2023-10-09T10:48:24.735402+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:30.322834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":58,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.719614+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:48:24.719614+0000","last_peered":"2023-10-09T10:48:24.719614+0000","last_clean":"2023-10-09T10:48:24.719614+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:48:24.719614+0000","last_undegraded":"2023-10-09T10:48:24.719614+0000","last_fullsized":"2023-10-09T10:48:24.719614+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":131,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275421+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:48:17.275421+0000","last_peered":"2023-10-09T10:48:17.275421+0000","last_clean":"2023-10-09T10:48:17.275421+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:48:17.275421+0000","last_undegraded":"2023-10-09T10:48:17.275421+0000","last_fullsized":"2023-10-09T10:48:17.275421+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"184'6930","reported_seq":13743,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:30.936088+0000","last_change":"2023-10-09T10:47:31.271196+0000","last_active":"2023-10-09T10:48:30.936088+0000","last_peered":"2023-10-09T10:48:30.936088+0000","last_clean":"2023-10-09T10:48:30.936088+0000","last_became_active":"2023-10-09T10:47:31.229504+0000","last_became_peered":"2023-10-09T10:47:31.229504+0000","last_unstale":"2023-10-09T10:48:30.936088+0000","last_undegraded":"2023-10-09T10:48:30.936088+0000","last_fullsized":"2023-10-09T10:48:30.936088+0000","mapping_epoch":172,"log_start":"108'2215","ondisk_log_start":"108'2215","created":98,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":4715,"log_dups_size":0,"ondisk_log_size":4715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:26.533476+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45613056,"num_objects":696,"num_object_clones":0,"num_object_copies":2088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4285,"num_write_kb":107264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3305,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":54,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.399326+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:48:32.399326+0000","last_peered":"2023-10-09T10:48:32.399326+0000","last_clean":"2023-10-09T10:48:32.399326+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:48:32.399326+0000","last_undegraded":"2023-10-09T10:48:32.399326+0000","last_fullsized":"2023-10-09T10:48:32.399326+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":115,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.269650+0000","last_change":"2023-10-09T10:46:32.251242+0000","last_active":"2023-10-09T10:48:17.269650+0000","last_peered":"2023-10-09T10:48:17.269650+0000","last_clean":"2023-10-09T10:48:17.269650+0000","last_became_active":"2023-10-09T10:46:32.250534+0000","last_became_peered":"2023-10-09T10:46:32.250534+0000","last_unstale":"2023-10-09T10:48:17.269650+0000","last_undegraded":"2023-10-09T10:48:17.269650+0000","last_fullsized":"2023-10-09T10:48:17.269650+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:33.726041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"185'7009","reported_seq":12394,"reported_epoch":185,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:32.690194+0000","last_change":"2023-10-09T10:46:34.514479+0000","last_active":"2023-10-09T10:48:32.690194+0000","last_peered":"2023-10-09T10:48:32.690194+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:46:34.409667+0000","last_became_peered":"2023-10-09T10:46:34.409667+0000","last_unstale":"2023-10-09T10:48:32.690194+0000","last_undegraded":"2023-10-09T10:46:34.376635+0000","last_fullsized":"2023-10-09T10:46:33.237998+0000","mapping_epoch":154,"log_start":"104'1591","ondisk_log_start":"104'1591","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5418,"log_dups_size":0,"ondisk_log_size":5418,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46333952,"num_objects":707,"num_object_clones":0,"num_object_copies":2121,"num_objects_missing_on_primary":708,"num_objects_missing":708,"num_objects_degraded":1218,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5967,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2630,"num_bytes_recovered":172359680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,7","objects":-1},{"shards":"2,3,7","objects":708}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":151,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.287529+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:48:17.287529+0000","last_peered":"2023-10-09T10:48:17.287529+0000","last_clean":"2023-10-09T10:48:17.287529+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:48:17.287529+0000","last_undegraded":"2023-10-09T10:48:17.287529+0000","last_fullsized":"2023-10-09T10:48:17.287529+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":165,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.276170+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:48:17.276170+0000","last_peered":"2023-10-09T10:48:17.276170+0000","last_clean":"2023-10-09T10:48:17.276170+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:48:17.276170+0000","last_undegraded":"2023-10-09T10:48:17.276170+0000","last_fullsized":"2023-10-09T10:48:17.276170+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"185'7023","reported_seq":7140,"reported_epoch":185,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:34.591759+0000","last_change":"2023-10-09T10:46:33.546991+0000","last_active":"2023-10-09T10:48:34.591759+0000","last_peered":"2023-10-09T10:48:34.591759+0000","last_clean":"2023-10-09T10:46:31.009484+0000","last_became_active":"2023-10-09T10:46:33.535665+0000","last_became_peered":"2023-10-09T10:46:33.535665+0000","last_unstale":"2023-10-09T10:48:34.591759+0000","last_undegraded":"2023-10-09T10:46:33.277168+0000","last_fullsized":"2023-10-09T10:46:33.235038+0000","mapping_epoch":154,"log_start":"103'628","ondisk_log_start":"103'628","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6395,"log_dups_size":0,"ondisk_log_size":6395,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37257216,"num_objects":569,"num_object_clones":0,"num_object_copies":1707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5959,"num_write_kb":208864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":581},{"shards":"3,7","objects":-2189}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":134,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.286903+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:48:17.286903+0000","last_peered":"2023-10-09T10:48:17.286903+0000","last_clean":"2023-10-09T10:48:17.286903+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:48:17.286903+0000","last_undegraded":"2023-10-09T10:48:17.286903+0000","last_fullsized":"2023-10-09T10:48:17.286903+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":32,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275519+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:48:17.275519+0000","last_peered":"2023-10-09T10:48:17.275519+0000","last_clean":"2023-10-09T10:48:17.275519+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:48:17.275519+0000","last_undegraded":"2023-10-09T10:48:17.275519+0000","last_fullsized":"2023-10-09T10:48:17.275519+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":169,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.726387+0000","last_change":"2023-10-09T10:46:32.244607+0000","last_active":"2023-10-09T10:48:24.726387+0000","last_peered":"2023-10-09T10:48:24.726387+0000","last_clean":"2023-10-09T10:48:24.726387+0000","last_became_active":"2023-10-09T10:46:32.242628+0000","last_became_peered":"2023-10-09T10:46:32.242628+0000","last_unstale":"2023-10-09T10:48:24.726387+0000","last_undegraded":"2023-10-09T10:48:24.726387+0000","last_fullsized":"2023-10-09T10:48:24.726387+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:10.738819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":200,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.726167+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:48:24.726167+0000","last_peered":"2023-10-09T10:48:24.726167+0000","last_clean":"2023-10-09T10:48:24.726167+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:48:24.726167+0000","last_undegraded":"2023-10-09T10:48:24.726167+0000","last_fullsized":"2023-10-09T10:48:24.726167+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":75,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-10-09T10:48:24.719946+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:48:24.719946+0000","last_peered":"2023-10-09T10:48:24.719946+0000","last_clean":"2023-10-09T10:48:24.719946+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:48:24.719946+0000","last_undegraded":"2023-10-09T10:48:24.719946+0000","last_fullsized":"2023-10-09T10:48:24.719946+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":148,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-10-09T10:48:17.275934+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:48:17.275934+0000","last_peered":"2023-10-09T10:48:17.275934+0000","last_clean":"2023-10-09T10:48:17.275934+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:48:17.275934+0000","last_undegraded":"2023-10-09T10:48:17.275934+0000","last_fullsized":"2023-10-09T10:48:17.275934+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":1371701272,"num_objects":20932,"num_object_clones":0,"num_object_copies":62796,"num_objects_missing_on_primary":708,"num_objects_missing":708,"num_objects_degraded":27363,"num_objects_misplaced":2190,"num_objects_unfound":0,"num_objects_dirty":20932,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":183276,"num_write_kb":6258529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27459,"num_bytes_recovered":1571618816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6513176576,"data_stored":6513164360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170169,"ondisk_log_size":170169,"up":78,"acting":69,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1186,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332263,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":925128,"kb_used_data":627456,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92820024,"statfs":{"total":95995035648,"available":95047704576,"internally_reserved":0,"allocated":642514944,"data_stored":641814421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":0.79200000000000004,"15min":0.65000000000000002},"min":{"1min":0.26300000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.1479999999999997,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.546,"5min":0.97199999999999998,"15min":0.752},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.9909999999999997,"5min":10.207000000000001,"15min":10.207000000000001},"last":0.97799999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:47:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2210000000000001,"5min":1.3300000000000001,"15min":0.80400000000000005},"min":{"1min":0.43099999999999999,"5min":0.318,"15min":0.318},"max":{"1min":11.826000000000001,"5min":11.826000000000001,"15min":11.826000000000001},"last":1.764},{"interface":"front","average":{"1min":2.286,"5min":1.155,"15min":0.73899999999999999},"min":{"1min":0.32100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":11.84,"5min":11.84,"15min":11.84},"last":0.48599999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:48:18 2023","interfaces":[{"interface":"back","average":{"1min":2.165,"5min":1.4810000000000001,"15min":0.83499999999999996},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.1419999999999995,"5min":9.1419999999999995,"15min":9.1419999999999995},"last":0.54700000000000004},{"interface":"front","average":{"1min":2.1970000000000001,"5min":1.1910000000000001,"15min":0.747},"min":{"1min":0.34999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":7.8120000000000003,"5min":7.8120000000000003,"15min":7.8120000000000003},"last":2.6680000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.23,"15min":0.80000000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.0599999999999996,"5min":8.1639999999999997,"15min":8.1639999999999997},"last":1.01},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.099,"15min":0.76600000000000001},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.504,"5min":12.451000000000001,"15min":12.451000000000001},"last":0.69699999999999995}]},{"osd":4,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":1.1579999999999999,"15min":0.70499999999999996},"min":{"1min":0.36399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":9.4900000000000002,"5min":9.4900000000000002,"15min":9.4900000000000002},"last":0.61799999999999999},{"interface":"front","average":{"1min":2.327,"5min":1.018,"15min":0.66100000000000003},"min":{"1min":0.35399999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.0999999999999996,"5min":9.0999999999999996,"15min":9.0999999999999996},"last":0.59499999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.0249999999999999,"5min":1.0609999999999999,"15min":0.69599999999999995},"min":{"1min":0.314,"5min":0.247,"15min":0.247},"max":{"1min":12.064,"5min":12.064,"15min":12.064},"last":0.52300000000000002},{"interface":"front","average":{"1min":3.8839999999999999,"5min":1.4770000000000001,"15min":0.85299999999999998},"min":{"1min":0.435,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":11.741,"5min":11.741,"15min":11.741},"last":0.65400000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2999999999999998,"5min":1.3720000000000001,"15min":0.83199999999999996},"min":{"1min":0.33700000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.029,"5min":13.029,"15min":13.029},"last":3.0640000000000001},{"interface":"front","average":{"1min":2.6869999999999998,"5min":1.079,"15min":0.72699999999999998},"min":{"1min":0.41299999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":19.187999999999999,"5min":19.187999999999999,"15min":19.187999999999999},"last":0.57699999999999996}]}]},{"osd":6,"up_from":44,"seq":188978561194,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":803012,"kb_used_data":493308,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92942140,"statfs":{"total":95995035648,"available":95172751360,"internally_reserved":0,"allocated":505147392,"data_stored":504451407,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.93200000000000005,"15min":0.71799999999999997},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.4630000000000001,"5min":7.3220000000000001,"15min":7.3220000000000001},"last":3.194},{"interface":"front","average":{"1min":1.02,"5min":1.002,"15min":0.73899999999999999},"min":{"1min":0.33900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.4969999999999999,"5min":7.5309999999999997,"15min":7.5309999999999997},"last":3.2570000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.157,"15min":0.77900000000000003},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.245},"max":{"1min":6.7560000000000002,"5min":16.747,"15min":16.747},"last":3.3170000000000002},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.284,"15min":0.79500000000000004},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.3579999999999997,"5min":16.734000000000002,"15min":16.734000000000002},"last":1.2210000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:48:20 2023","interfaces":[{"interface":"back","average":{"1min":2.798,"5min":1.411,"15min":0.82599999999999996},"min":{"1min":0.29699999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.208,"5min":13.208,"15min":13.208},"last":0.29899999999999999},{"interface":"front","average":{"1min":2.9420000000000002,"5min":1.619,"15min":0.91300000000000003},"min":{"1min":0.309,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.6749999999999998,"5min":12.571,"15min":12.571},"last":2.226}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.117,"15min":0.73099999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.8070000000000004,"5min":6.8070000000000004,"15min":6.8070000000000004},"last":3.3439999999999999},{"interface":"front","average":{"1min":2.1219999999999999,"5min":1.385,"15min":0.82599999999999996},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.3940000000000001,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":0.33000000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.0629999999999999,"15min":0.65000000000000002},"min":{"1min":0.29899999999999999,"5min":0.255,"15min":0.255},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":0.309},{"interface":"front","average":{"1min":1.6579999999999999,"5min":0.89600000000000002,"15min":0.623},"min":{"1min":0.34100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.1260000000000003,"5min":5.1260000000000003,"15min":5.1260000000000003},"last":12.231999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":1.1719999999999999,"15min":0.71099999999999997},"min":{"1min":0.35899999999999999,"5min":0.29499999999999998,"15min":0.27800000000000002},"max":{"1min":10.497,"5min":10.497,"15min":10.497},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.589,"5min":0.96599999999999997,"15min":0.67100000000000004},"min":{"1min":0.40699999999999997,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":6.8419999999999996,"5min":6.8419999999999996,"15min":6.8419999999999996},"last":3.2269999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:48:26 2023","interfaces":[{"interface":"back","average":{"1min":3.915,"5min":1.369,"15min":0.80600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":23.358000000000001,"5min":23.358000000000001,"15min":23.358000000000001},"last":12.007},{"interface":"front","average":{"1min":2.6840000000000002,"5min":1.25,"15min":0.79700000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34000000000000002},"max":{"1min":12.404,"5min":12.404,"15min":12.404},"last":3.2850000000000001}]}]},{"osd":1,"up_from":85,"seq":365072220233,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":789768,"kb_used_data":659776,"kb_used_omap":9,"kb_used_meta":129974,"kb_avail":92955384,"statfs":{"total":95995035648,"available":95186313216,"internally_reserved":0,"allocated":675610624,"data_stored":674679428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":133094377},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.581,"15min":0.76900000000000002},"min":{"1min":0.251,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.2119999999999997,"5min":14.1,"15min":14.1},"last":3.8999999999999999},{"interface":"front","average":{"1min":1.885,"5min":1.5629999999999999,"15min":0.78700000000000003},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.8090000000000002,"5min":14.465,"15min":14.465},"last":7.6639999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:48:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":1.492,"15min":0.74199999999999999},"min":{"1min":0.221,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":15.356,"5min":15.356,"15min":15.356},"last":0.223},{"interface":"front","average":{"1min":1.488,"5min":1.6890000000000001,"15min":0.81899999999999995},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.4130000000000003,"5min":14.234,"15min":14.234},"last":3.6829999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":1.5269999999999999,"15min":0.82299999999999995},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":7.234,"5min":14.404999999999999,"15min":14.404999999999999},"last":3.859},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.429,"15min":0.70599999999999996},"min":{"1min":0.30399999999999999,"5min":0.251,"15min":0.251},"max":{"1min":15.468999999999999,"5min":15.468999999999999,"15min":15.468999999999999},"last":7.7009999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.1679999999999999,"15min":0.78400000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.1449999999999996,"5min":8.0129999999999999,"15min":8.0129999999999999},"last":0.28299999999999997},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.2350000000000001,"15min":0.82499999999999996},"min":{"1min":0.30299999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.1189999999999998,"5min":8.327,"15min":8.327},"last":3.044}]},{"osd":5,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":1.111,"15min":0.78900000000000003},"min":{"1min":0.28899999999999998,"5min":0.253,"15min":0.253},"max":{"1min":5.4850000000000003,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":3.6080000000000001},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.2929999999999999,"15min":0.86199999999999999},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.1859999999999999,"5min":7.923,"15min":7.923},"last":0.318}]},{"osd":6,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7909999999999999,"5min":1.2030000000000001,"15min":0.82199999999999995},"min":{"1min":0.35499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.9450000000000003,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":3.6509999999999998},{"interface":"front","average":{"1min":1.246,"5min":1.2949999999999999,"15min":0.86299999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.8440000000000003,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":0.34499999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.22,"15min":0.83999999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.8689999999999998,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.992,"5min":1.2909999999999999,"15min":0.86099999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":7.1589999999999998,"5min":8.2469999999999999,"15min":8.2469999999999999},"last":3.0960000000000001}]}]},{"osd":0,"up_from":80,"seq":343597383767,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":400680,"kb_used_data":296864,"kb_used_omap":28,"kb_used_meta":103779,"kb_avail":93344472,"statfs":{"total":95995035648,"available":95584739328,"internally_reserved":0,"allocated":303988736,"data_stored":303024901,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":106270544},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,0,0,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:47:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2509999999999999,"5min":1.5449999999999999,"15min":0.76000000000000001},"min":{"1min":0.22,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":10.564,"5min":25.475000000000001,"15min":25.475000000000001},"last":11.869},{"interface":"front","average":{"1min":2.3740000000000001,"5min":1.5940000000000001,"15min":0.77400000000000002},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":10.593999999999999,"5min":20.832999999999998,"15min":20.832999999999998},"last":12.276}]},{"osd":2,"last update":"Mon Oct 9 10:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":1.4750000000000001,"15min":0.72399999999999998},"min":{"1min":0.249,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.0890000000000004,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":11.837},{"interface":"front","average":{"1min":2.1120000000000001,"5min":1.889,"15min":0.85799999999999998},"min":{"1min":0.27100000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.0329999999999995,"5min":14.391999999999999,"15min":14.391999999999999},"last":3.5750000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:47:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4740000000000002,"5min":1.8959999999999999,"15min":1.0660000000000001},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.215},"max":{"1min":10.644,"5min":14.926,"15min":14.926},"last":12.307},{"interface":"front","average":{"1min":2.2269999999999999,"5min":1.5940000000000001,"15min":0.89200000000000002},"min":{"1min":0.34499999999999997,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.228999999999999,"5min":11.868,"15min":11.868},"last":11.885999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.2250000000000001,"15min":0.82199999999999995},"min":{"1min":0.38400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.8150000000000004,"5min":6.8150000000000004,"15min":6.8150000000000004},"last":3.5019999999999998},{"interface":"front","average":{"1min":2.391,"5min":1.405,"15min":0.88100000000000001},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.676,"5min":10.676,"15min":10.676},"last":12.242000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":1.4359999999999999,"15min":0.89600000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":10.734,"5min":10.734,"15min":10.734},"last":12.157},{"interface":"front","average":{"1min":1.716,"5min":1.1659999999999999,"15min":0.82099999999999995},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.54,"5min":8.9190000000000005,"15min":8.9190000000000005},"last":0.47499999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.391,"15min":0.88400000000000001},"min":{"1min":0.36599999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.706,"5min":10.706,"15min":10.706},"last":12.207000000000001},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.0780000000000001,"15min":0.79300000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.0499999999999998,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":0.55100000000000005}]},{"osd":7,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.474,"5min":1.151,"15min":0.81899999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.5380000000000003,"5min":8.875,"15min":8.875},"last":0.51200000000000001},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.425,"15min":0.92700000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":14.394,"5min":14.394,"15min":14.394},"last":3.5350000000000001}]}]},{"osd":2,"up_from":0,"seq":386547056708,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893182,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1165668,"kb_used_data":926044,"kb_used_omap":7,"kb_used_meta":239416,"kb_avail":92579484,"statfs":{"total":95995035648,"available":94801391616,"internally_reserved":0,"allocated":948269056,"data_stored":947316006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":245162600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,2,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":1.3140000000000001,"15min":0.65200000000000002},"min":{"1min":0.192,"5min":0.183,"15min":0.183},"max":{"1min":7.0289999999999999,"5min":11.557,"15min":11.557},"last":1.202},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.1379999999999999,"15min":0.65000000000000002},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":6.681,"5min":7.0510000000000002,"15min":7.0510000000000002},"last":2.9260000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":1.6970000000000001,"15min":0.78500000000000003},"min":{"1min":0.27100000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.048,"5min":14.707000000000001,"15min":14.707000000000001},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.6220000000000001,"5min":1.1890000000000001,"15min":0.64500000000000002},"min":{"1min":0.19400000000000001,"5min":0.193,"15min":0.193},"max":{"1min":7.1630000000000003,"5min":11.629,"15min":11.629},"last":1.155}]},{"osd":2,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.6739999999999999,"15min":0.80200000000000005},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.9180000000000001,"5min":13.853,"15min":13.853},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.5590000000000002,"5min":1.5129999999999999,"15min":0.76800000000000002},"min":{"1min":0.30099999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.71,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":1.897}]},{"osd":4,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.2210000000000001,"15min":0.78800000000000003},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.9909999999999997,"5min":11.532,"15min":11.532},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.2440000000000002,"5min":1.296,"15min":0.83699999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.962999999999999,"5min":11.962999999999999,"15min":11.962999999999999},"last":2.1930000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7589999999999999,"5min":1.8029999999999999,"15min":0.99199999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.907,"5min":51.509999999999998,"15min":51.509999999999998},"last":7.3280000000000003},{"interface":"front","average":{"1min":2.1469999999999998,"5min":2.306,"15min":1.1439999999999999},"min":{"1min":0.36799999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.1139999999999999,"5min":55.338999999999999,"15min":55.338999999999999},"last":0.44900000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":1.3620000000000001,"15min":0.84599999999999997},"min":{"1min":0.378,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":11.92,"5min":11.92,"15min":11.92},"last":2.073},{"interface":"front","average":{"1min":2.3759999999999999,"5min":1.776,"15min":0.99199999999999999},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":10.528,"5min":13.891999999999999,"15min":13.891999999999999},"last":0.50800000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":2.3410000000000002,"15min":1.175},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.6520000000000001,"5min":51.744,"15min":51.744},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.9139999999999999,"5min":1.869,"15min":1.0049999999999999},"min":{"1min":0.33000000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.133,"5min":53.399000000000001,"15min":53.399000000000001},"last":1.1819999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019057,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":750468,"kb_used_data":444988,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92994684,"statfs":{"total":95995035648,"available":95226556416,"internally_reserved":0,"allocated":455667712,"data_stored":454970994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.0129999999999999,"15min":0.70899999999999996},"min":{"1min":0.24099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.5419999999999998,"5min":10.387,"15min":10.387},"last":0.308},{"interface":"front","average":{"1min":1.581,"5min":1.1819999999999999,"15min":0.78400000000000003},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.5999999999999996,"5min":10.311999999999999,"15min":10.311999999999999},"last":1.2450000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.1519999999999999,"15min":0.752},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.9100000000000001,"5min":6.907,"15min":6.907},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.859,"5min":1.2390000000000001,"15min":0.82699999999999996},"min":{"1min":0.30099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.7669999999999995,"5min":8.7669999999999995,"15min":8.7669999999999995},"last":0.874}]},{"osd":2,"last update":"Mon Oct 9 10:48:13 2023","interfaces":[{"interface":"back","average":{"1min":3.1520000000000001,"5min":1.617,"15min":0.91900000000000004},"min":{"1min":0.52100000000000002,"5min":0.253,"15min":0.253},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":1.452},{"interface":"front","average":{"1min":4.016,"5min":1.855,"15min":0.99299999999999999},"min":{"1min":0.36799999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":20.545999999999999,"5min":20.545999999999999,"15min":20.545999999999999},"last":1.1000000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:47:38 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.333,"15min":0.91900000000000004},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.8170000000000002,"5min":8.8170000000000002,"15min":8.8170000000000002},"last":0.84999999999999998},{"interface":"front","average":{"1min":2.3650000000000002,"5min":1.359,"15min":0.84499999999999997},"min":{"1min":0.34300000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.625,"5min":8.9809999999999999,"15min":8.9809999999999999},"last":1.3089999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:47:43 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":0.89500000000000002,"15min":0.59699999999999998},"min":{"1min":0.25600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":8.6950000000000003,"5min":8.6950000000000003,"15min":8.6950000000000003},"last":0.82099999999999995},{"interface":"front","average":{"1min":1.6579999999999999,"5min":0.94299999999999995,"15min":0.61299999999999999},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.252},"max":{"1min":4.8879999999999999,"5min":4.8879999999999999,"15min":4.8879999999999999},"last":0.374}]},{"osd":6,"last update":"Mon Oct 9 10:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":1.1319999999999999,"15min":0.69399999999999995},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":12.391999999999999,"5min":12.391999999999999,"15min":12.391999999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":3.1200000000000001,"5min":1.2130000000000001,"15min":0.78100000000000003},"min":{"1min":0.374,"5min":0.311,"15min":0.311},"max":{"1min":12.539,"5min":12.539,"15min":12.539},"last":0.746}]},{"osd":7,"last update":"Mon Oct 9 10:48:21 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":1.2929999999999999,"15min":0.80900000000000005},"min":{"1min":0.33300000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.7919999999999998,"5min":8.7919999999999998,"15min":8.7919999999999998},"last":0.79100000000000004},{"interface":"front","average":{"1min":3.2389999999999999,"5min":1.27,"15min":0.79000000000000004},"min":{"1min":0.55900000000000005,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.423,"5min":7.423,"15min":7.423},"last":0.55800000000000005}]}]},{"osd":5,"up_from":37,"seq":158913790126,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":884404,"kb_used_data":578860,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92860748,"statfs":{"total":95995035648,"available":95089405952,"internally_reserved":0,"allocated":592752640,"data_stored":592067751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:36 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.77600000000000002,"15min":0.65700000000000003},"min":{"1min":0.247,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.9860000000000002,"5min":4.8380000000000001,"15min":4.8380000000000001},"last":11.648999999999999},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.84899999999999998,"15min":0.72899999999999998},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.0209999999999999,"5min":4.8650000000000002,"15min":4.8650000000000002},"last":11.68}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":1.367,"15min":0.86099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":12.999000000000001,"5min":12.999000000000001,"15min":12.999000000000001},"last":5.8689999999999998},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.2370000000000001,"15min":0.80700000000000005},"min":{"1min":0.27600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3730000000000002,"5min":7.3730000000000002,"15min":7.3730000000000002},"last":5.4169999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:48:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6669999999999998,"5min":1.4390000000000001,"15min":0.86099999999999999},"min":{"1min":0.34799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.3280000000000003,"5min":7.9000000000000004,"15min":7.9000000000000004},"last":11.393000000000001},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.2350000000000001,"15min":0.79400000000000004},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.3059999999999992,"5min":10.352,"15min":10.352},"last":2.6669999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4969999999999999,"5min":1.621,"15min":1.0229999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":18.016999999999999,"5min":18.016999999999999,"15min":18.016999999999999},"last":1.335},{"interface":"front","average":{"1min":2.964,"5min":1.673,"15min":0.96599999999999997},"min":{"1min":0.318,"5min":0.317,"15min":0.317},"max":{"1min":15.555999999999999,"5min":15.555999999999999,"15min":15.555999999999999},"last":1.2769999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":0.83799999999999997,"15min":0.57799999999999996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":19.152999999999999,"5min":19.152999999999999,"15min":19.152999999999999},"last":2.6309999999999998},{"interface":"front","average":{"1min":1.986,"5min":0.96999999999999997,"15min":0.63500000000000001},"min":{"1min":0.26800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.3059999999999992,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":1.3089999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":1.127,"15min":0.69899999999999995},"min":{"1min":0.33100000000000002,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":7.7060000000000004,"5min":7.7060000000000004,"15min":7.7060000000000004},"last":5.8259999999999996},{"interface":"front","average":{"1min":1.958,"5min":0.90400000000000003,"15min":0.64000000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.30299999999999999},"max":{"1min":8.0909999999999993,"5min":8.0909999999999993,"15min":8.0909999999999993},"last":2.77}]},{"osd":7,"last update":"Mon Oct 9 10:48:26 2023","interfaces":[{"interface":"back","average":{"1min":2.79,"5min":1.1160000000000001,"15min":0.74099999999999999},"min":{"1min":0.41899999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.3940000000000001,"5min":7.3940000000000001,"15min":7.3940000000000001},"last":5.915},{"interface":"front","average":{"1min":2.2669999999999999,"5min":1.0369999999999999,"15min":0.71999999999999997},"min":{"1min":0.38500000000000001,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":9.0579999999999998,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":14.154}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":297996288,"data_stored":297992216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":665194496,"data_stored":665190424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2434793472,"data_stored":2434793472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":945750016,"data_stored":945750016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":452526080,"data_stored":452526080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":583274496,"data_stored":583270424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":502136832,"data_stored":502136832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":631504896,"data_stored":631504896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:48:38.221 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:48:38.282 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:38.322 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:38.343 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:38.346 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:38.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:38.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:38.448 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:38.536 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:38 smithi125 ceph-mon[195507]: from='client.44951 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:38.537 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:38 smithi125 ceph-mon[195507]: pgmap v408: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 27363/57906 objects degraded (47.254%); 1867/57906 objects misplaced (3.224%); 3.8 MiB/s, 125 objects/s recovering 2023-10-09T10:48:38.537 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:38 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 27363/57906 objects degraded (47.254%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:38.537 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:38 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3984007755' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:38.537 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:38 smithi125 ceph-mon[200197]: from='client.44951 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:38.537 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:38 smithi125 ceph-mon[200197]: pgmap v408: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 27363/57906 objects degraded (47.254%); 1867/57906 objects misplaced (3.224%); 3.8 MiB/s, 125 objects/s recovering 2023-10-09T10:48:38.537 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:38 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 27363/57906 objects degraded (47.254%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:38.538 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:38 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3984007755' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:38.539 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:38.596 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:38.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:38.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:38.702 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:38.711 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:48:38.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:38 smithi155 ceph-mon[184854]: from='client.44951 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:38.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:38 smithi155 ceph-mon[184854]: pgmap v408: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 27363/57906 objects degraded (47.254%); 1867/57906 objects misplaced (3.224%); 3.8 MiB/s, 125 objects/s recovering 2023-10-09T10:48:38.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:38 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 27363/57906 objects degraded (47.254%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:38.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:38 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3984007755' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:38.772 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:38.809 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:38.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:38.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:38.913 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:38.951 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:38.983 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:38.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:38.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:39.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:39.137 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:39.170 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:39.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:39.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:39.256 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:39.279 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:39.310 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:39.338 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:39.383 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:39.387 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:39 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:48:39.387 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:39 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3984007755' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:48:39.387 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:39 smithi155 ceph-mon[184854]: osdmap e186: 8 total, 8 up, 7 in 2023-10-09T10:48:39.387 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:39 smithi155 ceph-mon[184854]: failed to encode map e186 with expected crc 2023-10-09T10:48:39.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:39.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:39.483 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:39 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:48:39.483 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:39 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3984007755' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:48:39.483 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:39 smithi125 ceph-mon[195507]: osdmap e186: 8 total, 8 up, 7 in 2023-10-09T10:48:39.483 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:39 smithi125 ceph-mon[195507]: failed to encode map e186 with expected crc 2023-10-09T10:48:39.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:39 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:48:39.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:39 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3984007755' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:48:39.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:39 smithi125 ceph-mon[200197]: osdmap e186: 8 total, 8 up, 7 in 2023-10-09T10:48:39.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:39 smithi125 ceph-mon[200197]: failed to encode map e186 with expected crc 2023-10-09T10:48:39.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:39.545 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:39.976 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:39.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:39.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:40.086 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:40.120 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:40.156 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:40.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:40.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:40.256 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:40 smithi125 ceph-mon[195507]: failed to encode map e186 with expected crc 2023-10-09T10:48:40.256 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:40 smithi125 ceph-mon[195507]: pgmap v410: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 27050/57270 objects degraded (47.232%); 1867/57270 objects misplaced (3.260%); 2.5 MiB/s, 119 objects/s recovering 2023-10-09T10:48:40.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:40.321 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:40.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:40 smithi155 ceph-mon[184854]: failed to encode map e186 with expected crc 2023-10-09T10:48:40.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:40 smithi155 ceph-mon[184854]: pgmap v410: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 27050/57270 objects degraded (47.232%); 1867/57270 objects misplaced (3.260%); 2.5 MiB/s, 119 objects/s recovering 2023-10-09T10:48:40.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:40 smithi125 ceph-mon[200197]: failed to encode map e186 with expected crc 2023-10-09T10:48:40.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:40 smithi125 ceph-mon[200197]: pgmap v410: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 27050/57270 objects degraded (47.232%); 1867/57270 objects misplaced (3.260%); 2.5 MiB/s, 119 objects/s recovering 2023-10-09T10:48:40.851 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:48:40.870 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:40.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:40.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:40.901 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:48:40.954 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:48:40.983 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:40.996 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:48:41.024 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:41.034 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:48:41.054 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:41.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:41.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:41.074 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:48:41.102 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:48:41.138 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:48:41.165 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:41.176 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:48:41.177 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:48:41.177 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:48:41.177 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:48:41.177 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:48:41.177 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:48:41.177 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:48:41.177 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:48:41.281 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:41.286 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:48:41.335 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:48:41.336 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:48:41.336 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:48:41.336 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:48:41.336 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:48:41.381 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:41.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:41.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:41.493 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:41.526 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:41.560 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:41.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:41.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:41.564 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:41 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2259554892' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:41.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:41 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2259554892' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:41.576 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:48:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:48:41] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.43.0" 2023-10-09T10:48:41.577 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:41 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2259554892' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:41.664 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:41.738 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:41.770 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:41.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:41.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:41.801 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:48:41.877 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:41.916 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:41.949 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:41.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:41.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:41.973 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:48:41.973 INFO:teuthology.orchestra.run.smithi125.stdout:{"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 27050/51063 objects degraded (52.974%), 9 pgs degraded, 9 pgs undersized","count":18},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":507,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":186,"num_osds":8,"num_up_osds":8,"osd_up_since":1696848214,"num_in_osds":7,"osd_in_since":1696848391,"num_remapped_pgs":12},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":107},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":2}],"num_pgs":119,"num_pools":5,"num_objects":17021,"data_bytes":1114412984,"bytes_used":5107027968,"bytes_avail":666858221568,"bytes_total":671965249536,"degraded_objects":27050,"degraded_total":51063,"degraded_ratio":0.52973777490550888,"misplaced_objects":1654,"misplaced_total":51063,"misplaced_ratio":0.032391359692928341,"recovering_objects_per_sec":148,"recovering_bytes_per_sec":2364499,"recovering_keys_per_sec":0,"num_objects_recovered":1303,"num_bytes_recovered":20774912,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":624},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi125.satgiw","status":"up:active","gid":24575}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"}},"servicemap":{"epoch":20,"modified":"2023-10-09T10:44:35.008237+0000","services":{}},"progress_events":{"4db0080d-306f-4da6-a38c-4c702d14cb34":{"message":"Global Recovery Event (3m)\n [=========================...] (remaining: 24s)","progress":0.89915966987609863,"add_to_ceph_s":true},"585851ac-6e70-4462-9176-ac50987fab77":{"message":"Upgrade to 18.2.0-598-gb4fa663f (3m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T10:48:42.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:42.148 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:42.234 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:42.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:42.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:42.342 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:42.391 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:42.418 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:42.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:42 smithi125 ceph-mon[195507]: pgmap v411: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 27050/51063 objects degraded (52.974%); 1654/51063 objects misplaced (3.239%); 2.3 MiB/s, 148 objects/s recovering 2023-10-09T10:48:42.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:42 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2023582052' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:48:42.423 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:42 smithi125 ceph-mon[200197]: pgmap v411: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 27050/51063 objects degraded (52.974%); 1654/51063 objects misplaced (3.239%); 2.3 MiB/s, 148 objects/s recovering 2023-10-09T10:48:42.423 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:42 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2023582052' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:48:42.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:42.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:42.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:42.547 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:42.618 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:42 smithi155 ceph-mon[184854]: pgmap v411: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 27050/51063 objects degraded (52.974%); 1654/51063 objects misplaced (3.239%); 2.3 MiB/s, 148 objects/s recovering 2023-10-09T10:48:42.618 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:42 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2023582052' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:48:42.618 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:42.653 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:48:42.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:48:42.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:42.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:42.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:42.774 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:42.842 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:42.883 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:42.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:42.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:42.992 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:43.063 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:43.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:43.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:43.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:43.221 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:43.260 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:43.323 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:43.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:43.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:43.426 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:43.486 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:43.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:43 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 27050/51063 objects degraded (52.974%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:43.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:43 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 27050/51063 objects degraded (52.974%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:43.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:43 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 27050/51063 objects degraded (52.974%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:43.906 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:43.926 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:43.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:43.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:44.006 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:48:44.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:44.150 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:44.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:44 smithi125 ceph-mon[200197]: pgmap v412: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 982 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 26691/47208 objects degraded (56.539%); 1528/47208 objects misplaced (3.237%); 2.2 MiB/s, 132 objects/s recovering 2023-10-09T10:48:44.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:44 smithi125 ceph-mon[195507]: pgmap v412: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 982 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 26691/47208 objects degraded (56.539%); 1528/47208 objects misplaced (3.237%); 2.2 MiB/s, 132 objects/s recovering 2023-10-09T10:48:44.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:44 smithi155 ceph-mon[184854]: pgmap v412: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 982 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 26691/47208 objects degraded (56.539%); 1528/47208 objects misplaced (3.237%); 2.2 MiB/s, 132 objects/s recovering 2023-10-09T10:48:45.138 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:45.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:45.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:45.251 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:45.342 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:45.600 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:45.639 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:45.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:45.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:45.671 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:45.746 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:45.804 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:45.833 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:45.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:45.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:45.939 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:46.004 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:46.035 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:46.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:46.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:46.076 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:48:46.142 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:46.186 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:46.187 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:48:46.187 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:46 smithi155 ceph-mon[184854]: pgmap v413: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 880 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 685 op/s; 26691/42297 objects degraded (63.104%); 1407/42297 objects misplaced (3.326%); 2.2 MiB/s, 132 objects/s recovering 2023-10-09T10:48:46.187 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:46 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2485933444' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:48:46.187 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:46 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:48:46.189 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:46.226 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:46.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:46.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:46.276 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:46.276 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:48:46.276 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:46 smithi125 ceph-mon[195507]: pgmap v413: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 880 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 685 op/s; 26691/42297 objects degraded (63.104%); 1407/42297 objects misplaced (3.326%); 2.2 MiB/s, 132 objects/s recovering 2023-10-09T10:48:46.276 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:46 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2485933444' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:48:46.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:46 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:48:46.277 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:48:46.277 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:48:46.277 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:46 smithi125 ceph-mon[200197]: pgmap v413: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 880 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 685 op/s; 26691/42297 objects degraded (63.104%); 1407/42297 objects misplaced (3.326%); 2.2 MiB/s, 132 objects/s recovering 2023-10-09T10:48:46.277 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:46 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2485933444' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:48:46.277 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:46 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:48:46.331 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:46.394 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:47.192 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:47 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:47.192 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:47 smithi125 ceph-mon[195507]: failed to encode map e187 with expected crc 2023-10-09T10:48:47.192 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:47 smithi125 ceph-mon[195507]: failed to encode map e187 with expected crc 2023-10-09T10:48:47.192 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:47 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:48:47.192 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:47 smithi125 ceph-mon[195507]: osdmap e187: 8 total, 8 up, 7 in 2023-10-09T10:48:47.192 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:47 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:47.192 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:47 smithi125 ceph-mon[200197]: failed to encode map e187 with expected crc 2023-10-09T10:48:47.192 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:47 smithi125 ceph-mon[200197]: failed to encode map e187 with expected crc 2023-10-09T10:48:47.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:47 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:48:47.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:47 smithi125 ceph-mon[200197]: osdmap e187: 8 total, 8 up, 7 in 2023-10-09T10:48:47.198 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:47.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:47.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:47.301 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:47.368 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:47.448 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:47.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:47.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:47 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:47.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:47 smithi155 ceph-mon[184854]: failed to encode map e187 with expected crc 2023-10-09T10:48:47.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:47 smithi155 ceph-mon[184854]: failed to encode map e187 with expected crc 2023-10-09T10:48:47.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:47 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:48:47.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:47 smithi155 ceph-mon[184854]: osdmap e187: 8 total, 8 up, 7 in 2023-10-09T10:48:47.554 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:47.597 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:47.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:47.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:47.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:47.670 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:48:47.670 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:48:47.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:48:47.670 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:48:47.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.671 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1070 6.8 6:1676aaf7:::benchmark_data_smithi125_244951_object1069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:48:47.671 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.621927+0000", 2023-10-09T10:48:47.671 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 255.04439772699999, 2023-10-09T10:48:47.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99674418099999995, 2023-10-09T10:48:47.671 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.671 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:48:47.671 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.671 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.671 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.672 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1070 2023-10-09T10:48:47.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.672 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621927+0000", 2023-10-09T10:48:47.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621927+0000", 2023-10-09T10:48:47.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.673 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.673 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.673 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621926+0000", 2023-10-09T10:48:47.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:47.673 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.673 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.673 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.674 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621950+0000", 2023-10-09T10:48:47.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3706999999999999e-05 2023-10-09T10:48:47.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.674 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.674 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621964+0000", 2023-10-09T10:48:47.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4589999999999999e-05 2023-10-09T10:48:47.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.674 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621972+0000", 2023-10-09T10:48:47.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3510000000000002e-06 2023-10-09T10:48:47.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.675 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.675 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.622022+0000", 2023-10-09T10:48:47.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9088000000000002e-05 2023-10-09T10:48:47.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.675 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.675 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:47.676 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.622034+0000", 2023-10-09T10:48:47.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2584e-05 2023-10-09T10:48:47.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.676 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.676 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.618557+0000", 2023-10-09T10:48:47.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99652308199999995 2023-10-09T10:48:47.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.618672+0000", 2023-10-09T10:48:47.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011433 2023-10-09T10:48:47.677 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.677 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.677 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.677 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.677 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1095 6.8 6:164b39e5:::benchmark_data_smithi125_244951_object1094:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:48:47.677 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.623092+0000", 2023-10-09T10:48:47.678 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 255.04323352599999, 2023-10-09T10:48:47.678 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99981824900000005, 2023-10-09T10:48:47.678 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.678 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:47.678 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.678 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.678 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.678 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1095 2023-10-09T10:48:47.678 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.679 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623092+0000", 2023-10-09T10:48:47.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623092+0000", 2023-10-09T10:48:47.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623090+0000", 2023-10-09T10:48:47.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:48:47.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623113+0000", 2023-10-09T10:48:47.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2691e-05 2023-10-09T10:48:47.681 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.681 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623130+0000", 2023-10-09T10:48:47.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7594000000000001e-05 2023-10-09T10:48:47.681 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.681 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623139+0000", 2023-10-09T10:48:47.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5359999999999992e-06 2023-10-09T10:48:47.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.682 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.682 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623187+0000", 2023-10-09T10:48:47.682 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7874000000000003e-05 2023-10-09T10:48:47.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.682 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:47.682 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623197+0000", 2023-10-09T10:48:47.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0889e-05 2023-10-09T10:48:47.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.618984+0000", 2023-10-09T10:48:47.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99578652199999995 2023-10-09T10:48:47.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.685 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.619093+0000", 2023-10-09T10:48:47.685 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108948 2023-10-09T10:48:47.685 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.685 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:48:47.685 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.619248+0000", 2023-10-09T10:48:47.685 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000154747 2023-10-09T10:48:47.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620532+0000", 2023-10-09T10:48:47.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012847729999999999 2023-10-09T10:48:47.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:47.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621605+0000", 2023-10-09T10:48:47.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010723569999999999 2023-10-09T10:48:47.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622824+0000", 2023-10-09T10:48:47.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012196939999999999 2023-10-09T10:48:47.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:47.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622872+0000", 2023-10-09T10:48:47.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7573e-05 2023-10-09T10:48:47.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.688 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622910+0000", 2023-10-09T10:48:47.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7790999999999998e-05 2023-10-09T10:48:47.688 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.688 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.688 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.689 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1097 6.8 6:1cc6c81b:::benchmark_data_smithi125_244951_object1096:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:48:47.689 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.625080+0000", 2023-10-09T10:48:47.689 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 255.04124490999999, 2023-10-09T10:48:47.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.997987073, 2023-10-09T10:48:47.689 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.689 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:47.689 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.689 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.690 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.690 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1097 2023-10-09T10:48:47.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.690 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625080+0000", 2023-10-09T10:48:47.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625080+0000", 2023-10-09T10:48:47.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625078+0000", 2023-10-09T10:48:47.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T10:48:47.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625108+0000", 2023-10-09T10:48:47.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9626000000000001e-05 2023-10-09T10:48:47.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625132+0000", 2023-10-09T10:48:47.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4236999999999999e-05 2023-10-09T10:48:47.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625145+0000", 2023-10-09T10:48:47.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2814e-05 2023-10-09T10:48:47.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625215+0000", 2023-10-09T10:48:47.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0779999999999997e-05 2023-10-09T10:48:47.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:47.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625228+0000", 2023-10-09T10:48:47.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2989e-05 2023-10-09T10:48:47.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.619577+0000", 2023-10-09T10:48:47.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99434825299999996 2023-10-09T10:48:47.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.619653+0000", 2023-10-09T10:48:47.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.682e-05 2023-10-09T10:48:47.695 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:48:47.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.619810+0000", 2023-10-09T10:48:47.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000156737 2023-10-09T10:48:47.696 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.696 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.696 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:47.696 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621611+0000", 2023-10-09T10:48:47.696 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001800472 2023-10-09T10:48:47.696 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.696 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.696 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.696 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621664+0000", 2023-10-09T10:48:47.697 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2967999999999998e-05 2023-10-09T10:48:47.697 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.697 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.697 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.697 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622991+0000", 2023-10-09T10:48:47.697 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001327122 2023-10-09T10:48:47.697 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.697 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.697 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:47.697 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623034+0000", 2023-10-09T10:48:47.698 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2772000000000003e-05 2023-10-09T10:48:47.698 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.698 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.698 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.698 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623067+0000", 2023-10-09T10:48:47.698 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3767999999999999e-05 2023-10-09T10:48:47.698 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.698 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.698 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.698 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.699 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.699 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1100 6.8 6:1c8c7e3c:::benchmark_data_smithi125_244951_object1099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:48:47.699 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.627382+0000", 2023-10-09T10:48:47.699 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 255.038943144, 2023-10-09T10:48:47.699 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99583719999999998, 2023-10-09T10:48:47.699 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.699 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:47.699 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.699 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.700 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.700 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1100 2023-10-09T10:48:47.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.700 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627382+0000", 2023-10-09T10:48:47.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.701 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.701 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627382+0000", 2023-10-09T10:48:47.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.701 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.701 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627380+0000", 2023-10-09T10:48:47.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:48:47.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.702 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627405+0000", 2023-10-09T10:48:47.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5058e-05 2023-10-09T10:48:47.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.702 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627419+0000", 2023-10-09T10:48:47.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3431999999999999e-05 2023-10-09T10:48:47.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627426+0000", 2023-10-09T10:48:47.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.34e-06 2023-10-09T10:48:47.703 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627534+0000", 2023-10-09T10:48:47.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010761599999999999 2023-10-09T10:48:47.703 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:47.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627543+0000", 2023-10-09T10:48:47.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2660000000000007e-06 2023-10-09T10:48:47.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620079+0000", 2023-10-09T10:48:47.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99253606000000005 2023-10-09T10:48:47.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620154+0000", 2023-10-09T10:48:47.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5461000000000001e-05 2023-10-09T10:48:47.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:48:47.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620263+0000", 2023-10-09T10:48:47.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010842599999999999 2023-10-09T10:48:47.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.706 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621714+0000", 2023-10-09T10:48:47.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001451265 2023-10-09T10:48:47.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:47.706 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621722+0000", 2023-10-09T10:48:47.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2740000000000007e-06 2023-10-09T10:48:47.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.707 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623143+0000", 2023-10-09T10:48:47.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014206570000000001 2023-10-09T10:48:47.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.707 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.707 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:47.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623183+0000", 2023-10-09T10:48:47.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9660999999999999e-05 2023-10-09T10:48:47.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.707 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.707 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623219+0000", 2023-10-09T10:48:47.708 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6433000000000002e-05 2023-10-09T10:48:47.708 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.708 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.708 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.708 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.708 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1136 6.8 6:193c408d:::benchmark_data_smithi125_244951_object1135:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:48:47.708 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.650385+0000", 2023-10-09T10:48:47.708 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 255.01594051699999, 2023-10-09T10:48:47.708 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97294367199999998, 2023-10-09T10:48:47.709 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.709 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:47.709 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.709 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.709 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.709 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1136 2023-10-09T10:48:47.709 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.709 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.709 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.709 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.709 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650385+0000", 2023-10-09T10:48:47.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.710 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.710 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650385+0000", 2023-10-09T10:48:47.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.710 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.710 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650383+0000", 2023-10-09T10:48:47.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:47.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650411+0000", 2023-10-09T10:48:47.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7837000000000001e-05 2023-10-09T10:48:47.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650421+0000", 2023-10-09T10:48:47.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8779999999999993e-06 2023-10-09T10:48:47.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650427+0000", 2023-10-09T10:48:47.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7270000000000001e-06 2023-10-09T10:48:47.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650504+0000", 2023-10-09T10:48:47.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6800999999999999e-05 2023-10-09T10:48:47.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:47.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650514+0000", 2023-10-09T10:48:47.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9490000000000008e-06 2023-10-09T10:48:47.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620561+0000", 2023-10-09T10:48:47.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97004676599999995 2023-10-09T10:48:47.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620656+0000", 2023-10-09T10:48:47.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5345000000000003e-05 2023-10-09T10:48:47.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:48:47.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620791+0000", 2023-10-09T10:48:47.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013480999999999999 2023-10-09T10:48:47.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:47.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621728+0000", 2023-10-09T10:48:47.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00093716100000000005 2023-10-09T10:48:47.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621777+0000", 2023-10-09T10:48:47.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8569999999999997e-05 2023-10-09T10:48:47.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623256+0000", 2023-10-09T10:48:47.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014793569999999999 2023-10-09T10:48:47.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:47.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623297+0000", 2023-10-09T10:48:47.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1012e-05 2023-10-09T10:48:47.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623328+0000", 2023-10-09T10:48:47.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1044999999999998e-05 2023-10-09T10:48:47.717 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.717 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.717 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.717 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1140 6.8 6:1bbcb053:::benchmark_data_smithi125_244951_object1139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:48:47.717 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.651480+0000", 2023-10-09T10:48:47.717 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 255.014845587, 2023-10-09T10:48:47.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97196035599999997, 2023-10-09T10:48:47.718 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.718 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:47.718 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.718 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.718 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.718 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1140 2023-10-09T10:48:47.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.718 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651480+0000", 2023-10-09T10:48:47.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651480+0000", 2023-10-09T10:48:47.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651479+0000", 2023-10-09T10:48:47.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:47.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651498+0000", 2023-10-09T10:48:47.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9128999999999999e-05 2023-10-09T10:48:47.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651507+0000", 2023-10-09T10:48:47.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0410000000000003e-06 2023-10-09T10:48:47.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651513+0000", 2023-10-09T10:48:47.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4990000000000002e-06 2023-10-09T10:48:47.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651606+0000", 2023-10-09T10:48:47.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3546000000000002e-05 2023-10-09T10:48:47.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:47.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651618+0000", 2023-10-09T10:48:47.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1600000000000001e-05 2023-10-09T10:48:47.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621048+0000", 2023-10-09T10:48:47.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96943005599999998 2023-10-09T10:48:47.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621174+0000", 2023-10-09T10:48:47.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012588899999999999 2023-10-09T10:48:47.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:48:47.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621311+0000", 2023-10-09T10:48:47.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000137415 2023-10-09T10:48:47.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:47.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622787+0000", 2023-10-09T10:48:47.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014754880000000001 2023-10-09T10:48:47.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622958+0000", 2023-10-09T10:48:47.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017176800000000001 2023-10-09T10:48:47.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623365+0000", 2023-10-09T10:48:47.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00040624999999999998 2023-10-09T10:48:47.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:47.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623405+0000", 2023-10-09T10:48:47.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0680999999999999e-05 2023-10-09T10:48:47.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623440+0000", 2023-10-09T10:48:47.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4474000000000002e-05 2023-10-09T10:48:47.726 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.726 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.726 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.727 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.727 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.727 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1141 6.8 6:1ec53f00:::benchmark_data_smithi125_244951_object1140:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:48:47.727 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.651524+0000", 2023-10-09T10:48:47.727 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 255.014801197, 2023-10-09T10:48:47.727 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97203539800000005, 2023-10-09T10:48:47.727 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.727 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:47.727 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.727 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.727 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.728 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1141 2023-10-09T10:48:47.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.728 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651524+0000", 2023-10-09T10:48:47.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651524+0000", 2023-10-09T10:48:47.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651523+0000", 2023-10-09T10:48:47.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:47.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651544+0000", 2023-10-09T10:48:47.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.014e-05 2023-10-09T10:48:47.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651552+0000", 2023-10-09T10:48:47.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6179999999999995e-06 2023-10-09T10:48:47.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651561+0000", 2023-10-09T10:48:47.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7860000000000002e-06 2023-10-09T10:48:47.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651639+0000", 2023-10-09T10:48:47.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8362999999999995e-05 2023-10-09T10:48:47.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:47.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651646+0000", 2023-10-09T10:48:47.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6610000000000003e-06 2023-10-09T10:48:47.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621816+0000", 2023-10-09T10:48:47.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97017014700000004 2023-10-09T10:48:47.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621928+0000", 2023-10-09T10:48:47.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111427 2023-10-09T10:48:47.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:48:47.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622053+0000", 2023-10-09T10:48:47.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012548999999999999 2023-10-09T10:48:47.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:47.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622921+0000", 2023-10-09T10:48:47.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00086811799999999997 2023-10-09T10:48:47.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623114+0000", 2023-10-09T10:48:47.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019289700000000001 2023-10-09T10:48:47.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623487+0000", 2023-10-09T10:48:47.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00037276399999999998 2023-10-09T10:48:47.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:47.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623528+0000", 2023-10-09T10:48:47.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1179e-05 2023-10-09T10:48:47.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623559+0000", 2023-10-09T10:48:47.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1245999999999999e-05 2023-10-09T10:48:47.736 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.736 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.736 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.737 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1151 6.8 6:15b7e58c:::benchmark_data_smithi125_244951_object1150:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:48:47.737 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.658348+0000", 2023-10-09T10:48:47.737 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 255.00797751499999, 2023-10-09T10:48:47.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96534045700000004, 2023-10-09T10:48:47.737 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.737 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:47.737 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.737 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.737 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.737 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1151 2023-10-09T10:48:47.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.738 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658348+0000", 2023-10-09T10:48:47.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658348+0000", 2023-10-09T10:48:47.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658347+0000", 2023-10-09T10:48:47.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:47.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658366+0000", 2023-10-09T10:48:47.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8757e-05 2023-10-09T10:48:47.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658378+0000", 2023-10-09T10:48:47.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.216e-05 2023-10-09T10:48:47.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658384+0000", 2023-10-09T10:48:47.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0170000000000002e-06 2023-10-09T10:48:47.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658460+0000", 2023-10-09T10:48:47.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5543999999999994e-05 2023-10-09T10:48:47.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:47.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658481+0000", 2023-10-09T10:48:47.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1916000000000002e-05 2023-10-09T10:48:47.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622329+0000", 2023-10-09T10:48:47.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96384757600000004 2023-10-09T10:48:47.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622438+0000", 2023-10-09T10:48:47.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108985 2023-10-09T10:48:47.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:48:47.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622573+0000", 2023-10-09T10:48:47.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013480499999999999 2023-10-09T10:48:47.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:47.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623082+0000", 2023-10-09T10:48:47.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00050925299999999998 2023-10-09T10:48:47.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623595+0000", 2023-10-09T10:48:47.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051302300000000004 2023-10-09T10:48:47.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623625+0000", 2023-10-09T10:48:47.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0309000000000001e-05 2023-10-09T10:48:47.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:47.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623657+0000", 2023-10-09T10:48:47.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2027999999999999e-05 2023-10-09T10:48:47.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623688+0000", 2023-10-09T10:48:47.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0593000000000003e-05 2023-10-09T10:48:47.746 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.746 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.746 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.746 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1184 6.8 6:157bd3fc:::benchmark_data_smithi125_244951_object1183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:48:47.746 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.696219+0000", 2023-10-09T10:48:47.746 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 254.970106534, 2023-10-09T10:48:47.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.92991913699999995, 2023-10-09T10:48:47.747 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.747 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:47.747 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.747 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.747 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.747 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1184 2023-10-09T10:48:47.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.747 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696219+0000", 2023-10-09T10:48:47.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696219+0000", 2023-10-09T10:48:47.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696217+0000", 2023-10-09T10:48:47.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:47.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696237+0000", 2023-10-09T10:48:47.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9983000000000002e-05 2023-10-09T10:48:47.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696249+0000", 2023-10-09T10:48:47.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2215000000000001e-05 2023-10-09T10:48:47.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696257+0000", 2023-10-09T10:48:47.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.072e-06 2023-10-09T10:48:47.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696297+0000", 2023-10-09T10:48:47.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9852999999999999e-05 2023-10-09T10:48:47.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:47.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696305+0000", 2023-10-09T10:48:47.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7870000000000002e-06 2023-10-09T10:48:47.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623715+0000", 2023-10-09T10:48:47.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.92741050999999997 2023-10-09T10:48:47.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623814+0000", 2023-10-09T10:48:47.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8057e-05 2023-10-09T10:48:47.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:48:47.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623938+0000", 2023-10-09T10:48:47.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012472899999999999 2023-10-09T10:48:47.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:47.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.625153+0000", 2023-10-09T10:48:47.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012150970000000001 2023-10-09T10:48:47.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.625631+0000", 2023-10-09T10:48:47.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047751299999999999 2023-10-09T10:48:47.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.626058+0000", 2023-10-09T10:48:47.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042723499999999998 2023-10-09T10:48:47.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:47.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.626104+0000", 2023-10-09T10:48:47.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5569999999999999e-05 2023-10-09T10:48:47.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.626138+0000", 2023-10-09T10:48:47.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4069999999999997e-05 2023-10-09T10:48:47.756 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.756 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.756 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.756 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:70301 6.19 6:98815d50:::benchmark_data_smithi125_244951_object70300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:48:47.756 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:23.364524+0000", 2023-10-09T10:48:47.756 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.301801463, 2023-10-09T10:48:47.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.054642890999999999, 2023-10-09T10:48:47.757 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.757 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:47.757 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.757 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.757 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.757 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 70301 2023-10-09T10:48:47.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.757 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364524+0000", 2023-10-09T10:48:47.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364524+0000", 2023-10-09T10:48:47.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364522+0000", 2023-10-09T10:48:47.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:48:47.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364544+0000", 2023-10-09T10:48:47.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1990000000000001e-05 2023-10-09T10:48:47.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364556+0000", 2023-10-09T10:48:47.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2181000000000001e-05 2023-10-09T10:48:47.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364564+0000", 2023-10-09T10:48:47.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0499999999999992e-06 2023-10-09T10:48:47.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364698+0000", 2023-10-09T10:48:47.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013398800000000001 2023-10-09T10:48:47.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364826+0000", 2023-10-09T10:48:47.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012756099999999999 2023-10-09T10:48:47.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:48:47.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364936+0000", 2023-10-09T10:48:47.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011065000000000001 2023-10-09T10:48:47.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.366921+0000", 2023-10-09T10:48:47.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019843679999999998 2023-10-09T10:48:47.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:47.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.366945+0000", 2023-10-09T10:48:47.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3995999999999999e-05 2023-10-09T10:48:47.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.418241+0000", 2023-10-09T10:48:47.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.051296003 2023-10-09T10:48:47.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:47.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.418305+0000", 2023-10-09T10:48:47.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4815999999999997e-05 2023-10-09T10:48:47.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.419167+0000", 2023-10-09T10:48:47.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00086109200000000004 2023-10-09T10:48:47.764 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.764 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.764 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.764 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:70319 6.18 6:1f556d03:::benchmark_data_smithi125_244951_object70318:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:48:47.764 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:23.385122+0000", 2023-10-09T10:48:47.765 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 204.28120355999999, 2023-10-09T10:48:47.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.059906651999999998, 2023-10-09T10:48:47.765 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.765 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:47.765 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.765 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.765 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.765 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 70319 2023-10-09T10:48:47.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.766 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385122+0000", 2023-10-09T10:48:47.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385122+0000", 2023-10-09T10:48:47.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385120+0000", 2023-10-09T10:48:47.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:48:47.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385145+0000", 2023-10-09T10:48:47.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4414e-05 2023-10-09T10:48:47.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385155+0000", 2023-10-09T10:48:47.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0805000000000001e-05 2023-10-09T10:48:47.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385163+0000", 2023-10-09T10:48:47.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0330000000000008e-06 2023-10-09T10:48:47.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385219+0000", 2023-10-09T10:48:47.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5894999999999999e-05 2023-10-09T10:48:47.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385382+0000", 2023-10-09T10:48:47.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016238299999999999 2023-10-09T10:48:47.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:48:47.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385553+0000", 2023-10-09T10:48:47.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017152199999999999 2023-10-09T10:48:47.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.399377+0000", 2023-10-09T10:48:47.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013823291 2023-10-09T10:48:47.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:47.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.400837+0000", 2023-10-09T10:48:47.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014599890000000001 2023-10-09T10:48:47.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.444954+0000", 2023-10-09T10:48:47.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.044117351999999999 2023-10-09T10:48:47.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:47.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.445001+0000", 2023-10-09T10:48:47.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7105999999999999e-05 2023-10-09T10:48:47.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.445028+0000", 2023-10-09T10:48:47.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7172e-05 2023-10-09T10:48:47.773 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.773 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.773 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.774 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106423 6.9 6:930bb25f:::benchmark_data_smithi125_244951_object106422:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:48:47.774 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.239301+0000", 2023-10-09T10:48:47.774 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 165.42702458700001, 2023-10-09T10:48:47.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.46007959199999998, 2023-10-09T10:48:47.774 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.774 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:48:47.774 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.774 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.774 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.775 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106423 2023-10-09T10:48:47.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.775 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239301+0000", 2023-10-09T10:48:47.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239301+0000", 2023-10-09T10:48:47.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239299+0000", 2023-10-09T10:48:47.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:48:47.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239340+0000", 2023-10-09T10:48:47.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0917000000000001e-05 2023-10-09T10:48:47.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239360+0000", 2023-10-09T10:48:47.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0649000000000002e-05 2023-10-09T10:48:47.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239371+0000", 2023-10-09T10:48:47.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0244000000000001e-05 2023-10-09T10:48:47.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.244269+0000", 2023-10-09T10:48:47.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0048985590000000002 2023-10-09T10:48:47.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.244451+0000", 2023-10-09T10:48:47.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000181526 2023-10-09T10:48:47.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:48:47.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.244646+0000", 2023-10-09T10:48:47.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000195133 2023-10-09T10:48:47.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.271437+0000", 2023-10-09T10:48:47.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.026791150999999999 2023-10-09T10:48:47.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274828+0000", 2023-10-09T10:48:47.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003391088 2023-10-09T10:48:47.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:47.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274837+0000", 2023-10-09T10:48:47.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6899999999999998e-06 2023-10-09T10:48:47.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.699260+0000", 2023-10-09T10:48:47.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.42442315200000003 2023-10-09T10:48:47.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.782 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.699380+0000", 2023-10-09T10:48:47.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120189 2023-10-09T10:48:47.782 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.782 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.782 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.782 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106425 6.9 6:966db102:::benchmark_data_smithi125_244951_object106424:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:48:47.783 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.239425+0000", 2023-10-09T10:48:47.783 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 165.42689991399999, 2023-10-09T10:48:47.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.46673965099999998, 2023-10-09T10:48:47.783 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.783 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:48:47.783 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.783 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.783 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.783 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106425 2023-10-09T10:48:47.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.784 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239425+0000", 2023-10-09T10:48:47.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239425+0000", 2023-10-09T10:48:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239425+0000", 2023-10-09T10:48:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239473+0000", 2023-10-09T10:48:47.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8177000000000003e-05 2023-10-09T10:48:47.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239480+0000", 2023-10-09T10:48:47.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0559999999999999e-06 2023-10-09T10:48:47.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239485+0000", 2023-10-09T10:48:47.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.242e-06 2023-10-09T10:48:47.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.249603+0000", 2023-10-09T10:48:47.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010118121000000001 2023-10-09T10:48:47.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.249760+0000", 2023-10-09T10:48:47.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000156331 2023-10-09T10:48:47.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:48:47.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.249953+0000", 2023-10-09T10:48:47.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019313500000000001 2023-10-09T10:48:47.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274855+0000", 2023-10-09T10:48:47.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.024902173999999999 2023-10-09T10:48:47.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:47.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274860+0000", 2023-10-09T10:48:47.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8380000000000001e-06 2023-10-09T10:48:47.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.700402+0000", 2023-10-09T10:48:47.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.42554259900000002 2023-10-09T10:48:47.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:48:47.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.701238+0000", 2023-10-09T10:48:47.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000835825 2023-10-09T10:48:47.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.706109+0000", 2023-10-09T10:48:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0048705440000000001 2023-10-09T10:48:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.706165+0000", 2023-10-09T10:48:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6017999999999998e-05 2023-10-09T10:48:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.792 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106422 6.19 6:9914323d:::benchmark_data_smithi125_244951_object106421:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T10:48:47.792 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.601105+0000", 2023-10-09T10:48:47.792 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 165.065220203, 2023-10-09T10:48:47.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0560851339999999, 2023-10-09T10:48:47.792 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.792 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:48:47.792 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.792 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.792 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.793 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106422 2023-10-09T10:48:47.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.793 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601105+0000", 2023-10-09T10:48:47.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601105+0000", 2023-10-09T10:48:47.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601103+0000", 2023-10-09T10:48:47.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:48:47.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601130+0000", 2023-10-09T10:48:47.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7222000000000001e-05 2023-10-09T10:48:47.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601147+0000", 2023-10-09T10:48:47.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6600999999999999e-05 2023-10-09T10:48:47.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601162+0000", 2023-10-09T10:48:47.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4929000000000001e-05 2023-10-09T10:48:47.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.643541+0000", 2023-10-09T10:48:47.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.042379130000000001 2023-10-09T10:48:47.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:47.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.643555+0000", 2023-10-09T10:48:47.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3465000000000001e-05 2023-10-09T10:48:47.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:03.657089+0000", 2023-10-09T10:48:47.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0135346009999999 2023-10-09T10:48:47.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:03.657190+0000", 2023-10-09T10:48:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000100945 2023-10-09T10:48:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106426 6.19 6:9f6a94cf:::benchmark_data_smithi125_244951_object106425:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T10:48:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.601170+0000", 2023-10-09T10:48:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 165.065154694, 2023-10-09T10:48:47.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.057006321, 2023-10-09T10:48:47.799 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.799 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:48:47.799 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.799 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.799 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.799 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106426 2023-10-09T10:48:47.799 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.799 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601170+0000", 2023-10-09T10:48:47.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601170+0000", 2023-10-09T10:48:47.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601170+0000", 2023-10-09T10:48:47.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:47.801 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.801 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601191+0000", 2023-10-09T10:48:47.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0746000000000001e-05 2023-10-09T10:48:47.801 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.801 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.802 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.802 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601199+0000", 2023-10-09T10:48:47.802 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8019999999999998e-06 2023-10-09T10:48:47.802 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.802 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.802 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.802 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601222+0000", 2023-10-09T10:48:47.802 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2801000000000001e-05 2023-10-09T10:48:47.802 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.803 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.803 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.803 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.643580+0000", 2023-10-09T10:48:47.803 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.042357842 2023-10-09T10:48:47.803 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.803 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.803 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:48:47.803 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.643589+0000", 2023-10-09T10:48:47.803 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1749999999999994e-06 2023-10-09T10:48:47.803 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.804 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:03.658076+0000", 2023-10-09T10:48:47.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0144863019999999 2023-10-09T10:48:47.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.804 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:03.658177+0000", 2023-10-09T10:48:47.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000101201 2023-10-09T10:48:47.805 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.805 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.805 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.805 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.805 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.805 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:115958 6.f 6:fa100500:::benchmark_data_smithi125_244951_object9515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-10-09T10:48:47.805 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:20.075222+0000", 2023-10-09T10:48:47.805 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 147.59110334600001, 2023-10-09T10:48:47.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.084394099, 2023-10-09T10:48:47.805 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.806 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:47.806 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.806 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.806 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.806 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115958 2023-10-09T10:48:47.806 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.806 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.806 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.806 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.807 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075222+0000", 2023-10-09T10:48:47.807 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.807 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.807 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.807 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.807 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075222+0000", 2023-10-09T10:48:47.807 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.807 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.807 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.807 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.808 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075220+0000", 2023-10-09T10:48:47.808 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:48:47.808 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.808 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.808 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.808 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075229+0000", 2023-10-09T10:48:47.808 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7949999999999996e-06 2023-10-09T10:48:47.808 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.808 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.808 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.809 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075244+0000", 2023-10-09T10:48:47.809 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5364000000000001e-05 2023-10-09T10:48:47.809 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.809 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.809 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.809 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075254+0000", 2023-10-09T10:48:47.809 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4609999999999997e-06 2023-10-09T10:48:47.809 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.809 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.809 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.810 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.076549+0000", 2023-10-09T10:48:47.810 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012954900000000001 2023-10-09T10:48:47.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.810 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.810 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.810 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.076690+0000", 2023-10-09T10:48:47.810 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014043200000000001 2023-10-09T10:48:47.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.810 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.810 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:48:47.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.076818+0000", 2023-10-09T10:48:47.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012810400000000001 2023-10-09T10:48:47.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:47.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.085804+0000", 2023-10-09T10:48:47.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0089861170000000001 2023-10-09T10:48:47.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.087330+0000", 2023-10-09T10:48:47.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001526108 2023-10-09T10:48:47.812 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.812 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159461+0000", 2023-10-09T10:48:47.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.072131120000000007 2023-10-09T10:48:47.812 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.812 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:47.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159562+0000", 2023-10-09T10:48:47.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000101475 2023-10-09T10:48:47.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.813 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.813 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.813 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159616+0000", 2023-10-09T10:48:47.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3387000000000002e-05 2023-10-09T10:48:47.813 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.813 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.813 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.814 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.814 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:115960 6.f 6:f825ff14:::benchmark_data_smithi125_244951_object9517:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-10-09T10:48:47.814 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:20.081466+0000", 2023-10-09T10:48:47.814 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 147.584858884, 2023-10-09T10:48:47.814 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.078315183999999996, 2023-10-09T10:48:47.814 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.814 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:47.814 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.814 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.814 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.814 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115960 2023-10-09T10:48:47.815 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.815 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.815 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.815 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.815 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081466+0000", 2023-10-09T10:48:47.815 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.815 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.815 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.815 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.815 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081466+0000", 2023-10-09T10:48:47.816 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.816 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.816 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.816 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.816 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081464+0000", 2023-10-09T10:48:47.816 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:48:47.816 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.816 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.816 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.816 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081473+0000", 2023-10-09T10:48:47.817 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3960000000000006e-06 2023-10-09T10:48:47.817 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.817 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.817 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.817 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081490+0000", 2023-10-09T10:48:47.817 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7730000000000001e-05 2023-10-09T10:48:47.817 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.817 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.817 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.817 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081502+0000", 2023-10-09T10:48:47.818 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1612e-05 2023-10-09T10:48:47.818 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.818 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.818 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.818 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.082766+0000", 2023-10-09T10:48:47.818 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012642350000000001 2023-10-09T10:48:47.818 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.818 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.818 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.818 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.082875+0000", 2023-10-09T10:48:47.819 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010849999999999999 2023-10-09T10:48:47.819 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.819 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.819 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:48:47.819 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.082969+0000", 2023-10-09T10:48:47.819 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4368000000000002e-05 2023-10-09T10:48:47.819 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.819 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.819 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.819 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.091397+0000", 2023-10-09T10:48:47.820 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0084276379999999994 2023-10-09T10:48:47.820 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.820 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.820 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:47.820 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.105493+0000", 2023-10-09T10:48:47.820 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014096630000000001 2023-10-09T10:48:47.820 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.820 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.820 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.820 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159659+0000", 2023-10-09T10:48:47.821 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.05416588 2023-10-09T10:48:47.821 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.821 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.821 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:47.821 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159740+0000", 2023-10-09T10:48:47.821 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0735999999999994e-05 2023-10-09T10:48:47.821 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.821 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.821 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.822 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159781+0000", 2023-10-09T10:48:47.822 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1458999999999997e-05 2023-10-09T10:48:47.822 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.822 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.822 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.822 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.822 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.822 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:115962 6.f 6:f37830ff:::benchmark_data_smithi125_244951_object9519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-10-09T10:48:47.822 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:20.095380+0000", 2023-10-09T10:48:47.823 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 147.570945145, 2023-10-09T10:48:47.823 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.064565055999999996, 2023-10-09T10:48:47.823 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.823 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:47.823 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.823 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.823 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.823 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115962 2023-10-09T10:48:47.824 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.824 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.824 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.824 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.824 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095380+0000", 2023-10-09T10:48:47.824 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.824 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.824 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.824 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.825 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095380+0000", 2023-10-09T10:48:47.825 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.825 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.825 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.825 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.825 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095377+0000", 2023-10-09T10:48:47.825 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T10:48:47.825 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.825 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.825 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.826 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095391+0000", 2023-10-09T10:48:47.826 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3073e-05 2023-10-09T10:48:47.826 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.826 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.826 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.826 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095412+0000", 2023-10-09T10:48:47.826 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1406000000000001e-05 2023-10-09T10:48:47.826 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.826 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.827 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.827 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095423+0000", 2023-10-09T10:48:47.827 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0764000000000001e-05 2023-10-09T10:48:47.827 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.827 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.827 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.827 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.097612+0000", 2023-10-09T10:48:47.827 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021888530000000002 2023-10-09T10:48:47.827 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.827 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.827 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.828 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.097704+0000", 2023-10-09T10:48:47.828 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2219000000000006e-05 2023-10-09T10:48:47.828 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.828 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.828 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:48:47.828 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.097775+0000", 2023-10-09T10:48:47.828 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1330999999999999e-05 2023-10-09T10:48:47.828 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.828 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.828 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.829 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.101037+0000", 2023-10-09T10:48:47.829 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032613899999999999 2023-10-09T10:48:47.829 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.829 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.829 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:47.829 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.122053+0000", 2023-10-09T10:48:47.829 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.021016743000000001 2023-10-09T10:48:47.829 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.829 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.829 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.829 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159823+0000", 2023-10-09T10:48:47.830 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.037769619999999997 2023-10-09T10:48:47.830 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.830 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.830 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:47.830 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159902+0000", 2023-10-09T10:48:47.830 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9357999999999995e-05 2023-10-09T10:48:47.830 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.830 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.830 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.831 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159945+0000", 2023-10-09T10:48:47.831 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2772999999999999e-05 2023-10-09T10:48:47.831 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.831 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.831 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.831 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.831 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.831 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:115966 6.f 6:f2b073c2:::benchmark_data_smithi125_244951_object9523:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-10-09T10:48:47.831 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:20.095468+0000", 2023-10-09T10:48:47.831 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 147.57085733900001, 2023-10-09T10:48:47.831 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.067415613999999999, 2023-10-09T10:48:47.832 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.832 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:47.832 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.832 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.832 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.832 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115966 2023-10-09T10:48:47.832 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.832 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.832 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.833 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.833 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095468+0000", 2023-10-09T10:48:47.833 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.833 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.833 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.833 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.833 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095468+0000", 2023-10-09T10:48:47.833 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.833 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.834 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.834 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.834 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095468+0000", 2023-10-09T10:48:47.834 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:48:47.834 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.834 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.834 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.834 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095470+0000", 2023-10-09T10:48:47.834 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2520000000000002e-06 2023-10-09T10:48:47.835 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.835 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.835 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.835 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095475+0000", 2023-10-09T10:48:47.835 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8919999999999999e-06 2023-10-09T10:48:47.835 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.835 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.835 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.835 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095479+0000", 2023-10-09T10:48:47.835 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.476e-06 2023-10-09T10:48:47.836 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.836 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.836 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.101070+0000", 2023-10-09T10:48:47.836 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0055904309999999999 2023-10-09T10:48:47.836 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.836 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.836 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.101187+0000", 2023-10-09T10:48:47.836 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000117111 2023-10-09T10:48:47.836 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.837 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.837 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:48:47.837 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.101301+0000", 2023-10-09T10:48:47.837 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000113842 2023-10-09T10:48:47.837 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.837 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.837 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.837 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.111354+0000", 2023-10-09T10:48:47.837 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010053945 2023-10-09T10:48:47.837 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.838 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.838 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:47.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.122063+0000", 2023-10-09T10:48:47.838 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010709000999999999 2023-10-09T10:48:47.838 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.838 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.838 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159984+0000", 2023-10-09T10:48:47.838 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.037920483999999997 2023-10-09T10:48:47.839 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.839 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.839 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:47.839 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.162830+0000", 2023-10-09T10:48:47.839 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002845888 2023-10-09T10:48:47.839 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.839 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.839 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.839 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.162883+0000", 2023-10-09T10:48:47.839 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3554999999999998e-05 2023-10-09T10:48:47.839 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.840 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.840 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.840 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.840 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.840 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:153008 6.0 6:05373277:::benchmark_data_smithi125_244951_object46565:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:48:47.840 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:35.545330+0000", 2023-10-09T10:48:47.840 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.120994675999995, 2023-10-09T10:48:47.840 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.057702479000000001, 2023-10-09T10:48:47.840 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:48:47.840 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:48:47.841 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:48:47.841 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:48:47.841 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:48:47.841 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 153008 2023-10-09T10:48:47.841 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.841 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:48:47.841 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.841 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:48:47.841 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545330+0000", 2023-10-09T10:48:47.841 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.842 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.842 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:48:47.842 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545330+0000", 2023-10-09T10:48:47.842 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:48:47.842 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.842 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:48:47.842 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545329+0000", 2023-10-09T10:48:47.842 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:48:47.843 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.843 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.843 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:48:47.843 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545338+0000", 2023-10-09T10:48:47.843 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3919999999999992e-06 2023-10-09T10:48:47.843 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.843 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.843 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:48:47.843 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545359+0000", 2023-10-09T10:48:47.843 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0851000000000001e-05 2023-10-09T10:48:47.844 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.844 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.844 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:48:47.844 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545368+0000", 2023-10-09T10:48:47.844 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7679999999999997e-06 2023-10-09T10:48:47.844 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.844 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.844 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:48:47.844 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545452+0000", 2023-10-09T10:48:47.845 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3881000000000005e-05 2023-10-09T10:48:47.845 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.845 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.845 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:48:47.845 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545599+0000", 2023-10-09T10:48:47.845 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014761900000000001 2023-10-09T10:48:47.845 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.845 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.845 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:48:47.845 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545725+0000", 2023-10-09T10:48:47.845 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000126148 2023-10-09T10:48:47.846 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.846 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.846 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.846 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.547777+0000", 2023-10-09T10:48:47.846 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020514560000000001 2023-10-09T10:48:47.846 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.846 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.846 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:48:47.846 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.547844+0000", 2023-10-09T10:48:47.846 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7282999999999999e-05 2023-10-09T10:48:47.847 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.847 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.847 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.847 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.553741+0000", 2023-10-09T10:48:47.847 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0058972629999999998 2023-10-09T10:48:47.847 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.847 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.847 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:48:47.847 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.602869+0000", 2023-10-09T10:48:47.847 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.049127951000000003 2023-10-09T10:48:47.848 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.848 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.848 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:48:47.848 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.602922+0000", 2023-10-09T10:48:47.848 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2329e-05 2023-10-09T10:48:47.848 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:48:47.848 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:48:47.848 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:48:47.848 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.603033+0000", 2023-10-09T10:48:47.848 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011130599999999999 2023-10-09T10:48:47.849 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.849 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.849 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.849 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:48:47.849 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:48:47.849 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:48:47.849 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:47.922 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:47.968 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:47.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:47.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:48.074 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:48.164 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:48.214 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:48 smithi155 ceph-mon[184854]: pgmap v415: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 773 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 791 op/s; 26351/37161 objects degraded (70.910%); 1187/37161 objects misplaced (3.194%); 945 KiB/s, 123 objects/s recovering 2023-10-09T10:48:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:48 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 26691/42297 objects degraded (63.104%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:48.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:48.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:48.266 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:48:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:48 smithi125 ceph-mon[195507]: pgmap v415: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 773 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 791 op/s; 26351/37161 objects degraded (70.910%); 1187/37161 objects misplaced (3.194%); 945 KiB/s, 123 objects/s recovering 2023-10-09T10:48:48.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:48 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 26691/42297 objects degraded (63.104%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:48 smithi125 ceph-mon[200197]: pgmap v415: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 773 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 791 op/s; 26351/37161 objects degraded (70.910%); 1187/37161 objects misplaced (3.194%); 945 KiB/s, 123 objects/s recovering 2023-10-09T10:48:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:48 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 26691/42297 objects degraded (63.104%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:48.321 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:48.358 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:48:48.361 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:48.367 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:48:48.398 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:48.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:48.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:48.411 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:48:48.419 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:48:48.429 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:48:48.469 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:48:48.505 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:48.517 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:48:48.565 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:48:48.565 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:48:48.572 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:48.613 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:48:48.671 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:48:48.677 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:48:48.678 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:48:48.678 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:48:48.678 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:48:48.678 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:48:48.678 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:48:48.678 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:48:48.678 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:48:48.966 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":415,"stamp":"2023-10-09T10:48:47.172000+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":810719160,"num_objects":12387,"num_object_clones":0,"num_object_copies":37161,"num_objects_missing_on_primary":467,"num_objects_missing":467,"num_objects_degraded":26351,"num_objects_misplaced":1187,"num_objects_unfound":0,"num_objects_dirty":12387,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":192145,"num_write_kb":6260974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29148,"num_bytes_recovered":1596590850,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":174318,"ondisk_log_size":174318,"up":357,"acting":348,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4143168,"kb_used_data":2441992,"kb_used_omap":44,"kb_used_meta":1700819,"kb_avail":652072896,"statfs":{"total":671965249536,"available":667722645504,"internally_reserved":0,"allocated":2500599808,"data_stored":2494912015,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1741639073},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,8,12,5,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-439287808,"num_objects":-6703,"num_object_clones":0,"num_object_copies":-20109,"num_objects_missing_on_primary":-125,"num_objects_missing":-125,"num_objects_degraded":-699,"num_objects_misplaced":-680,"num_objects_unfound":0,"num_objects_dirty":-6703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6703,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1045,"num_bytes_recovered":8192000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.468250"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":180,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.737448+0000","last_change":"2023-10-09T10:46:32.250979+0000","last_active":"2023-10-09T10:48:38.737448+0000","last_peered":"2023-10-09T10:48:38.737448+0000","last_clean":"2023-10-09T10:48:38.737448+0000","last_became_active":"2023-10-09T10:46:32.250625+0000","last_became_peered":"2023-10-09T10:46:32.250625+0000","last_unstale":"2023-10-09T10:48:38.737448+0000","last_undegraded":"2023-10-09T10:48:38.737448+0000","last_fullsized":"2023-10-09T10:48:38.737448+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:34.858348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":155,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:34.644807+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:48:34.644807+0000","last_peered":"2023-10-09T10:48:34.644807+0000","last_clean":"2023-10-09T10:48:34.644807+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:48:34.644807+0000","last_undegraded":"2023-10-09T10:48:34.644807+0000","last_fullsized":"2023-10-09T10:48:34.644807+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":139,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.736565+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:48:38.736565+0000","last_peered":"2023-10-09T10:48:38.736565+0000","last_clean":"2023-10-09T10:48:38.736565+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:48:38.736565+0000","last_undegraded":"2023-10-09T10:48:38.736565+0000","last_fullsized":"2023-10-09T10:48:38.736565+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":55,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.419523+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:48:32.419523+0000","last_peered":"2023-10-09T10:48:32.419523+0000","last_clean":"2023-10-09T10:48:32.419523+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:48:32.419523+0000","last_undegraded":"2023-10-09T10:48:32.419523+0000","last_fullsized":"2023-10-09T10:48:32.419523+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":191,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.395920+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:48:32.395920+0000","last_peered":"2023-10-09T10:48:32.395920+0000","last_clean":"2023-10-09T10:48:32.395920+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:48:32.395920+0000","last_undegraded":"2023-10-09T10:48:32.395920+0000","last_fullsized":"2023-10-09T10:48:32.395920+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":125,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723319+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:48:38.723319+0000","last_peered":"2023-10-09T10:48:38.723319+0000","last_clean":"2023-10-09T10:48:38.723319+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:48:38.723319+0000","last_undegraded":"2023-10-09T10:48:38.723319+0000","last_fullsized":"2023-10-09T10:48:38.723319+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"187'7485","reported_seq":11646,"reported_epoch":187,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:46.047438+0000","last_change":"2023-10-09T10:46:33.521602+0000","last_active":"2023-10-09T10:48:46.047438+0000","last_peered":"2023-10-09T10:48:46.047438+0000","last_clean":"2023-10-09T10:46:31.055751+0000","last_became_active":"2023-10-09T10:46:33.364058+0000","last_became_peered":"2023-10-09T10:46:33.364058+0000","last_unstale":"2023-10-09T10:48:46.047438+0000","last_undegraded":"2023-10-09T10:48:46.047438+0000","last_fullsized":"2023-10-09T10:48:46.047438+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7485,"log_dups_size":0,"ondisk_log_size":7485,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26771456,"num_objects":409,"num_object_clones":0,"num_object_copies":1227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":409,"num_objects_unfound":0,"num_objects_dirty":409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6449,"num_write_kb":219424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2010,"num_bytes_recovered":131727360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":409}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":55,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.401730+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:48:32.401730+0000","last_peered":"2023-10-09T10:48:32.401730+0000","last_clean":"2023-10-09T10:48:32.401730+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:48:32.401730+0000","last_undegraded":"2023-10-09T10:48:32.401730+0000","last_fullsized":"2023-10-09T10:48:32.401730+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":155,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.395983+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:48:32.395983+0000","last_peered":"2023-10-09T10:48:32.395983+0000","last_clean":"2023-10-09T10:48:32.395983+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:48:32.395983+0000","last_undegraded":"2023-10-09T10:48:32.395983+0000","last_fullsized":"2023-10-09T10:48:32.395983+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":520,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.401200+0000","last_change":"2023-10-09T10:46:05.620365+0000","last_active":"2023-10-09T10:48:32.401200+0000","last_peered":"2023-10-09T10:48:32.401200+0000","last_clean":"2023-10-09T10:48:32.401200+0000","last_became_active":"2023-10-09T10:46:05.620024+0000","last_became_peered":"2023-10-09T10:46:05.620024+0000","last_unstale":"2023-10-09T10:48:32.401200+0000","last_undegraded":"2023-10-09T10:48:32.401200+0000","last_fullsized":"2023-10-09T10:48:32.401200+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.13","version":"186'7287","reported_seq":11256,"reported_epoch":186,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:46.030042+0000","last_change":"2023-10-09T10:48:15.511525+0000","last_active":"2023-10-09T10:48:46.030042+0000","last_peered":"2023-10-09T10:48:46.030042+0000","last_clean":"2023-10-09T10:46:31.195699+0000","last_became_active":"2023-10-09T10:46:33.455632+0000","last_became_peered":"2023-10-09T10:46:33.455632+0000","last_unstale":"2023-10-09T10:48:46.030042+0000","last_undegraded":"2023-10-09T10:46:33.269323+0000","last_fullsized":"2023-10-09T10:46:33.241226+0000","mapping_epoch":154,"log_start":"113'2575","ondisk_log_start":"113'2575","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":4712,"log_dups_size":0,"ondisk_log_size":4712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:07:12.547333+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25231360,"num_objects":385,"num_object_clones":0,"num_object_copies":1155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1427,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4572,"num_write_kb":105088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1888,"num_bytes_recovered":66060288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"6,7","objects":-1042}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":152,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.732002+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:48:38.732002+0000","last_peered":"2023-10-09T10:48:38.732002+0000","last_clean":"2023-10-09T10:48:38.732002+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:48:38.732002+0000","last_undegraded":"2023-10-09T10:48:38.732002+0000","last_fullsized":"2023-10-09T10:48:38.732002+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":112,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.406466+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:48:32.406466+0000","last_peered":"2023-10-09T10:48:32.406466+0000","last_clean":"2023-10-09T10:48:32.406466+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:48:32.406466+0000","last_undegraded":"2023-10-09T10:48:32.406466+0000","last_fullsized":"2023-10-09T10:48:32.406466+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"187'7485","reported_seq":15620,"reported_epoch":187,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:46.411782+0000","last_change":"2023-10-09T10:46:03.683196+0000","last_active":"2023-10-09T10:48:46.411782+0000","last_peered":"2023-10-09T10:48:46.411782+0000","last_clean":"2023-10-09T10:46:02.177309+0000","last_became_active":"2023-10-09T10:46:03.681058+0000","last_became_peered":"2023-10-09T10:46:03.681058+0000","last_unstale":"2023-10-09T10:48:46.411782+0000","last_undegraded":"2023-10-09T10:48:46.411782+0000","last_fullsized":"2023-10-09T10:48:46.411782+0000","mapping_epoch":143,"log_start":"103'142","ondisk_log_start":"103'142","created":107,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7343,"log_dups_size":0,"ondisk_log_size":7343,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22839296,"num_objects":349,"num_object_clones":0,"num_object_copies":1047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":349,"num_objects_unfound":0,"num_objects_dirty":349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6439,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[1,5,6],"avail_no_missing":["1","2","3","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":349}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":107,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.390655+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:48:32.390655+0000","last_peered":"2023-10-09T10:48:32.390655+0000","last_clean":"2023-10-09T10:48:32.390655+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:48:32.390655+0000","last_undegraded":"2023-10-09T10:48:32.390655+0000","last_fullsized":"2023-10-09T10:48:32.390655+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":148,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.739447+0000","last_change":"2023-10-09T10:46:32.249921+0000","last_active":"2023-10-09T10:48:38.739447+0000","last_peered":"2023-10-09T10:48:38.739447+0000","last_clean":"2023-10-09T10:48:38.739447+0000","last_became_active":"2023-10-09T10:46:32.249590+0000","last_became_peered":"2023-10-09T10:46:32.249590+0000","last_unstale":"2023-10-09T10:48:38.739447+0000","last_undegraded":"2023-10-09T10:48:38.739447+0000","last_fullsized":"2023-10-09T10:48:38.739447+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:25.169752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"186'7201","reported_seq":15431,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:44.603166+0000","last_change":"2023-10-09T10:47:48.412928+0000","last_active":"2023-10-09T10:48:44.603166+0000","last_peered":"2023-10-09T10:48:44.603166+0000","last_clean":"2023-10-09T10:48:44.603166+0000","last_became_active":"2023-10-09T10:47:48.398813+0000","last_became_peered":"2023-10-09T10:47:48.398813+0000","last_unstale":"2023-10-09T10:48:44.603166+0000","last_undegraded":"2023-10-09T10:48:44.603166+0000","last_fullsized":"2023-10-09T10:48:44.603166+0000","mapping_epoch":177,"log_start":"104'1070","ondisk_log_start":"104'1070","created":107,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6131,"log_dups_size":0,"ondisk_log_size":6131,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24870912,"num_objects":380,"num_object_clones":0,"num_object_copies":1140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6194,"num_write_kb":210336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2691,"num_bytes_recovered":176357376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":157,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.734505+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:48:38.734505+0000","last_peered":"2023-10-09T10:48:38.734505+0000","last_clean":"2023-10-09T10:48:38.734505+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:48:38.734505+0000","last_undegraded":"2023-10-09T10:48:38.734505+0000","last_fullsized":"2023-10-09T10:48:38.734505+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.733906+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:48:38.733906+0000","last_peered":"2023-10-09T10:48:38.733906+0000","last_clean":"2023-10-09T10:48:38.733906+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:48:38.733906+0000","last_undegraded":"2023-10-09T10:48:38.733906+0000","last_fullsized":"2023-10-09T10:48:38.733906+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":520,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.411066+0000","last_change":"2023-10-09T10:46:05.618276+0000","last_active":"2023-10-09T10:48:32.411066+0000","last_peered":"2023-10-09T10:48:32.411066+0000","last_clean":"2023-10-09T10:48:32.411066+0000","last_became_active":"2023-10-09T10:46:05.616509+0000","last_became_peered":"2023-10-09T10:46:05.616509+0000","last_unstale":"2023-10-09T10:48:32.411066+0000","last_undegraded":"2023-10-09T10:48:32.411066+0000","last_fullsized":"2023-10-09T10:48:32.411066+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.16","version":"186'7222","reported_seq":7333,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:44.572145+0000","last_change":"2023-10-09T10:46:03.592682+0000","last_active":"2023-10-09T10:48:44.572145+0000","last_peered":"2023-10-09T10:48:44.572145+0000","last_clean":"2023-10-09T10:48:44.572145+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:48:44.572145+0000","last_undegraded":"2023-10-09T10:48:44.572145+0000","last_fullsized":"2023-10-09T10:48:44.572145+0000","mapping_epoch":107,"log_start":"104'991","ondisk_log_start":"104'991","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6231,"log_dups_size":0,"ondisk_log_size":6231,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26673152,"num_objects":407,"num_object_clones":0,"num_object_copies":1221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6179,"num_write_kb":210752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":168,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.398532+0000","last_change":"2023-10-09T10:46:32.250256+0000","last_active":"2023-10-09T10:48:32.398532+0000","last_peered":"2023-10-09T10:48:32.398532+0000","last_clean":"2023-10-09T10:48:32.398532+0000","last_became_active":"2023-10-09T10:46:32.243312+0000","last_became_peered":"2023-10-09T10:46:32.243312+0000","last_unstale":"2023-10-09T10:48:32.398532+0000","last_undegraded":"2023-10-09T10:48:32.398532+0000","last_fullsized":"2023-10-09T10:48:32.398532+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:08.059777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":178,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.405503+0000","last_change":"2023-10-09T10:46:32.237577+0000","last_active":"2023-10-09T10:48:32.405503+0000","last_peered":"2023-10-09T10:48:32.405503+0000","last_clean":"2023-10-09T10:48:32.405503+0000","last_became_active":"2023-10-09T10:46:32.237316+0000","last_became_peered":"2023-10-09T10:46:32.237316+0000","last_unstale":"2023-10-09T10:48:32.405503+0000","last_undegraded":"2023-10-09T10:48:32.405503+0000","last_fullsized":"2023-10-09T10:48:32.405503+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:39.684602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":521,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.413671+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:48:32.413671+0000","last_peered":"2023-10-09T10:48:32.413671+0000","last_clean":"2023-10-09T10:48:32.413671+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:48:32.413671+0000","last_undegraded":"2023-10-09T10:48:32.413671+0000","last_fullsized":"2023-10-09T10:48:32.413671+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.17","version":"187'7172","reported_seq":7278,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.126174+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:48:46.126174+0000","last_peered":"2023-10-09T10:48:46.126174+0000","last_clean":"2023-10-09T10:48:46.126174+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:48:46.126174+0000","last_undegraded":"2023-10-09T10:48:46.126174+0000","last_fullsized":"2023-10-09T10:48:46.126174+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7172,"log_dups_size":0,"ondisk_log_size":7172,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22413312,"num_objects":342,"num_object_clones":0,"num_object_copies":1026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6164,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":55,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.413521+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:48:32.413521+0000","last_peered":"2023-10-09T10:48:32.413521+0000","last_clean":"2023-10-09T10:48:32.413521+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:48:32.413521+0000","last_undegraded":"2023-10-09T10:48:32.413521+0000","last_fullsized":"2023-10-09T10:48:32.413521+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723345+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:48:38.723345+0000","last_peered":"2023-10-09T10:48:38.723345+0000","last_clean":"2023-10-09T10:48:38.723345+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:48:38.723345+0000","last_undegraded":"2023-10-09T10:48:38.723345+0000","last_fullsized":"2023-10-09T10:48:38.723345+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":521,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.399008+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:48:32.399008+0000","last_peered":"2023-10-09T10:48:32.399008+0000","last_clean":"2023-10-09T10:48:32.399008+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:48:32.399008+0000","last_undegraded":"2023-10-09T10:48:32.399008+0000","last_fullsized":"2023-10-09T10:48:32.399008+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"187'7399","reported_seq":7510,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.498689+0000","last_change":"2023-10-09T10:46:33.243540+0000","last_active":"2023-10-09T10:48:46.498689+0000","last_peered":"2023-10-09T10:48:46.498689+0000","last_clean":"2023-10-09T10:48:46.498689+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:48:46.498689+0000","last_undegraded":"2023-10-09T10:48:46.498689+0000","last_fullsized":"2023-10-09T10:48:46.498689+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7399,"log_dups_size":0,"ondisk_log_size":7399,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21528576,"num_objects":329,"num_object_clones":0,"num_object_copies":987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6345,"num_write_kb":213536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":97,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.733887+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:48:38.733887+0000","last_peered":"2023-10-09T10:48:38.733887+0000","last_clean":"2023-10-09T10:48:38.733887+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:48:38.733887+0000","last_undegraded":"2023-10-09T10:48:38.733887+0000","last_fullsized":"2023-10-09T10:48:38.733887+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":55,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.419760+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:48:32.419760+0000","last_peered":"2023-10-09T10:48:32.419760+0000","last_clean":"2023-10-09T10:48:32.419760+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:48:32.419760+0000","last_undegraded":"2023-10-09T10:48:32.419760+0000","last_fullsized":"2023-10-09T10:48:32.419760+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":521,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.413285+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:48:32.413285+0000","last_peered":"2023-10-09T10:48:32.413285+0000","last_clean":"2023-10-09T10:48:32.413285+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:48:32.413285+0000","last_undegraded":"2023-10-09T10:48:32.413285+0000","last_fullsized":"2023-10-09T10:48:32.413285+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"186'7387","reported_seq":7507,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:44.553738+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:48:44.553738+0000","last_peered":"2023-10-09T10:48:44.553738+0000","last_clean":"2023-10-09T10:48:44.553738+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:48:44.553738+0000","last_undegraded":"2023-10-09T10:48:44.553738+0000","last_fullsized":"2023-10-09T10:48:44.553738+0000","mapping_epoch":115,"log_start":"104'1314","ondisk_log_start":"104'1314","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6073,"log_dups_size":0,"ondisk_log_size":6073,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22413312,"num_objects":342,"num_object_clones":0,"num_object_copies":1026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6326,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":65,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.413598+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:48:32.413598+0000","last_peered":"2023-10-09T10:48:32.413598+0000","last_clean":"2023-10-09T10:48:32.413598+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:48:32.413598+0000","last_undegraded":"2023-10-09T10:48:32.413598+0000","last_fullsized":"2023-10-09T10:48:32.413598+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":155,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.388189+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:48:32.388189+0000","last_peered":"2023-10-09T10:48:32.388189+0000","last_clean":"2023-10-09T10:48:32.388189+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:48:32.388189+0000","last_undegraded":"2023-10-09T10:48:32.388189+0000","last_fullsized":"2023-10-09T10:48:32.388189+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":521,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.410852+0000","last_change":"2023-10-09T10:46:05.620574+0000","last_active":"2023-10-09T10:48:32.410852+0000","last_peered":"2023-10-09T10:48:32.410852+0000","last_clean":"2023-10-09T10:48:32.410852+0000","last_became_active":"2023-10-09T10:46:05.620212+0000","last_became_peered":"2023-10-09T10:46:05.620212+0000","last_unstale":"2023-10-09T10:48:32.410852+0000","last_undegraded":"2023-10-09T10:48:32.410852+0000","last_fullsized":"2023-10-09T10:48:32.410852+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"187'12928","reported_seq":17090,"reported_epoch":187,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:46.147332+0000","last_change":"2023-10-09T10:46:32.247903+0000","last_active":"2023-10-09T10:48:46.147332+0000","last_peered":"2023-10-09T10:48:46.147332+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:46:02.739355+0000","last_became_peered":"2023-10-09T10:46:02.739355+0000","last_unstale":"2023-10-09T10:48:46.147332+0000","last_undegraded":"2023-10-09T10:46:02.608805+0000","last_fullsized":"2023-10-09T10:46:02.583589+0000","mapping_epoch":142,"log_start":"119'4100","ondisk_log_start":"119'4100","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8828,"log_dups_size":0,"ondisk_log_size":8828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:32.899634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46137344,"num_objects":704,"num_object_clones":0,"num_object_copies":2112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12928,"num_write_kb":436224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2022,"num_bytes_recovered":132513792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":625},{"shards":"5,6","objects":78}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":179,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.398166+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:48:32.398166+0000","last_peered":"2023-10-09T10:48:32.398166+0000","last_clean":"2023-10-09T10:48:32.398166+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:48:32.398166+0000","last_undegraded":"2023-10-09T10:48:32.398166+0000","last_fullsized":"2023-10-09T10:48:32.398166+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":134,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723111+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:48:38.723111+0000","last_peered":"2023-10-09T10:48:38.723111+0000","last_clean":"2023-10-09T10:48:38.723111+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:48:38.723111+0000","last_undegraded":"2023-10-09T10:48:38.723111+0000","last_fullsized":"2023-10-09T10:48:38.723111+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":501,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.401338+0000","last_change":"2023-10-09T10:46:02.631378+0000","last_active":"2023-10-09T10:48:32.401338+0000","last_peered":"2023-10-09T10:48:32.401338+0000","last_clean":"2023-10-09T10:48:32.401338+0000","last_became_active":"2023-10-09T10:46:02.629142+0000","last_became_peered":"2023-10-09T10:46:02.629142+0000","last_unstale":"2023-10-09T10:48:32.401338+0000","last_undegraded":"2023-10-09T10:48:32.401338+0000","last_fullsized":"2023-10-09T10:48:32.401338+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.b","version":"186'12385","reported_seq":12483,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:44.646361+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:48:44.646361+0000","last_peered":"2023-10-09T10:48:44.646361+0000","last_clean":"2023-10-09T10:48:44.646361+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:48:44.646361+0000","last_undegraded":"2023-10-09T10:48:44.646361+0000","last_fullsized":"2023-10-09T10:48:44.646361+0000","mapping_epoch":98,"log_start":"148'6625","ondisk_log_start":"148'6625","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5760,"log_dups_size":0,"ondisk_log_size":5760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52101120,"num_objects":795,"num_object_clones":0,"num_object_copies":2385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12385,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":152,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.398105+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:48:32.398105+0000","last_peered":"2023-10-09T10:48:32.398105+0000","last_clean":"2023-10-09T10:48:32.398105+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:48:32.398105+0000","last_undegraded":"2023-10-09T10:48:32.398105+0000","last_fullsized":"2023-10-09T10:48:32.398105+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":188,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.733815+0000","last_change":"2023-10-09T10:46:32.251035+0000","last_active":"2023-10-09T10:48:38.733815+0000","last_peered":"2023-10-09T10:48:38.733815+0000","last_clean":"2023-10-09T10:48:38.733815+0000","last_became_active":"2023-10-09T10:46:32.250293+0000","last_became_peered":"2023-10-09T10:46:32.250293+0000","last_unstale":"2023-10-09T10:48:38.733815+0000","last_undegraded":"2023-10-09T10:48:38.733815+0000","last_fullsized":"2023-10-09T10:48:38.733815+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.787766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":521,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.401517+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:48:32.401517+0000","last_peered":"2023-10-09T10:48:32.401517+0000","last_clean":"2023-10-09T10:48:32.401517+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:48:32.401517+0000","last_undegraded":"2023-10-09T10:48:32.401517+0000","last_fullsized":"2023-10-09T10:48:32.401517+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"186'7337","reported_seq":7456,"reported_epoch":186,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:44.644505+0000","last_change":"2023-10-09T10:46:33.540719+0000","last_active":"2023-10-09T10:48:44.644505+0000","last_peered":"2023-10-09T10:48:44.644505+0000","last_clean":"2023-10-09T10:46:31.137696+0000","last_became_active":"2023-10-09T10:46:33.527650+0000","last_became_peered":"2023-10-09T10:46:33.527650+0000","last_unstale":"2023-10-09T10:48:44.644505+0000","last_undegraded":"2023-10-09T10:46:33.285468+0000","last_fullsized":"2023-10-09T10:46:33.235683+0000","mapping_epoch":154,"log_start":"104'1197","ondisk_log_start":"104'1197","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6140,"log_dups_size":0,"ondisk_log_size":6140,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27164672,"num_objects":414,"num_object_clones":0,"num_object_copies":1242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6272,"num_write_kb":213984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":402},{"shards":"3,7","objects":-2395}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":108,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723223+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:48:38.723223+0000","last_peered":"2023-10-09T10:48:38.723223+0000","last_clean":"2023-10-09T10:48:38.723223+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:48:38.723223+0000","last_undegraded":"2023-10-09T10:48:38.723223+0000","last_fullsized":"2023-10-09T10:48:38.723223+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":125,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.739705+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:48:38.739705+0000","last_peered":"2023-10-09T10:48:38.739705+0000","last_clean":"2023-10-09T10:48:38.739705+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:48:38.739705+0000","last_undegraded":"2023-10-09T10:48:38.739705+0000","last_fullsized":"2023-10-09T10:48:38.739705+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":521,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.419360+0000","last_change":"2023-10-09T10:46:05.620030+0000","last_active":"2023-10-09T10:48:32.419360+0000","last_peered":"2023-10-09T10:48:32.419360+0000","last_clean":"2023-10-09T10:48:32.419360+0000","last_became_active":"2023-10-09T10:46:05.619694+0000","last_became_peered":"2023-10-09T10:46:05.619694+0000","last_unstale":"2023-10-09T10:48:32.419360+0000","last_undegraded":"2023-10-09T10:48:32.419360+0000","last_fullsized":"2023-10-09T10:48:32.419360+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.9","version":"186'7207","reported_seq":9609,"reported_epoch":186,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:42.805787+0000","last_change":"2023-10-09T10:48:15.517098+0000","last_active":"2023-10-09T10:48:42.805787+0000","last_peered":"2023-10-09T10:48:42.805787+0000","last_clean":"2023-10-09T10:46:31.190603+0000","last_became_active":"2023-10-09T10:46:34.396448+0000","last_became_peered":"2023-10-09T10:46:34.396448+0000","last_unstale":"2023-10-09T10:48:42.805787+0000","last_undegraded":"2023-10-09T10:46:33.246949+0000","last_fullsized":"2023-10-09T10:46:33.237968+0000","mapping_epoch":154,"log_start":"106'1941","ondisk_log_start":"106'1941","created":98,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5266,"log_dups_size":0,"ondisk_log_size":5266,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29425664,"num_objects":449,"num_object_clones":0,"num_object_copies":1347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":868,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6165,"num_write_kb":211648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2439,"num_bytes_recovered":85458944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,7","objects":-419}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":152,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723838+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:48:38.723838+0000","last_peered":"2023-10-09T10:48:38.723838+0000","last_clean":"2023-10-09T10:48:38.723838+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:48:38.723838+0000","last_undegraded":"2023-10-09T10:48:38.723838+0000","last_fullsized":"2023-10-09T10:48:38.723838+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":55,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.399400+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:48:32.399400+0000","last_peered":"2023-10-09T10:48:32.399400+0000","last_clean":"2023-10-09T10:48:32.399400+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:48:32.399400+0000","last_undegraded":"2023-10-09T10:48:32.399400+0000","last_fullsized":"2023-10-09T10:48:32.399400+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":512,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.402951+0000","last_change":"2023-10-09T10:46:05.618134+0000","last_active":"2023-10-09T10:48:32.402951+0000","last_peered":"2023-10-09T10:48:32.402951+0000","last_clean":"2023-10-09T10:48:32.402951+0000","last_became_active":"2023-10-09T10:46:05.616057+0000","last_became_peered":"2023-10-09T10:46:05.616057+0000","last_unstale":"2023-10-09T10:48:32.402951+0000","last_undegraded":"2023-10-09T10:48:32.402951+0000","last_fullsized":"2023-10-09T10:48:32.402951+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"187'12507","reported_seq":12614,"reported_epoch":187,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:46.050197+0000","last_change":"2023-10-09T10:46:33.576530+0000","last_active":"2023-10-09T10:48:46.050197+0000","last_peered":"2023-10-09T10:48:46.050197+0000","last_clean":"2023-10-09T10:46:31.183481+0000","last_became_active":"2023-10-09T10:46:33.542304+0000","last_became_peered":"2023-10-09T10:46:33.542304+0000","last_unstale":"2023-10-09T10:48:46.050197+0000","last_undegraded":"2023-10-09T10:46:33.300581+0000","last_fullsized":"2023-10-09T10:46:33.246047+0000","mapping_epoch":154,"log_start":"130'5177","ondisk_log_start":"130'5177","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7330,"log_dups_size":0,"ondisk_log_size":7330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:26.739488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47644672,"num_objects":727,"num_object_clones":0,"num_object_copies":2181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6617,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12507,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":727},{"shards":"1,4","objects":-4834}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":179,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.388644+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:48:32.388644+0000","last_peered":"2023-10-09T10:48:32.388644+0000","last_clean":"2023-10-09T10:48:32.388644+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:48:32.388644+0000","last_undegraded":"2023-10-09T10:48:32.388644+0000","last_fullsized":"2023-10-09T10:48:32.388644+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.734398+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:48:38.734398+0000","last_peered":"2023-10-09T10:48:38.734398+0000","last_clean":"2023-10-09T10:48:38.734398+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:48:38.734398+0000","last_undegraded":"2023-10-09T10:48:38.734398+0000","last_fullsized":"2023-10-09T10:48:38.734398+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":501,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.413743+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:48:32.413743+0000","last_peered":"2023-10-09T10:48:32.413743+0000","last_clean":"2023-10-09T10:48:32.413743+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:48:32.413743+0000","last_undegraded":"2023-10-09T10:48:32.413743+0000","last_fullsized":"2023-10-09T10:48:32.413743+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.f","version":"186'12190","reported_seq":12293,"reported_epoch":186,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:44.649550+0000","last_change":"2023-10-09T10:46:33.651756+0000","last_active":"2023-10-09T10:48:44.649550+0000","last_peered":"2023-10-09T10:48:44.649550+0000","last_clean":"2023-10-09T10:46:31.156458+0000","last_became_active":"2023-10-09T10:46:33.621003+0000","last_became_peered":"2023-10-09T10:46:33.621003+0000","last_unstale":"2023-10-09T10:48:44.649550+0000","last_undegraded":"2023-10-09T10:46:33.309813+0000","last_fullsized":"2023-10-09T10:46:33.236508+0000","mapping_epoch":154,"log_start":"142'6510","ondisk_log_start":"142'6510","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5680,"log_dups_size":0,"ondisk_log_size":5680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:13.704346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54525952,"num_objects":832,"num_object_clones":0,"num_object_copies":2496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12190,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":832},{"shards":"3,4","objects":-4620}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":126,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.737597+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:48:38.737597+0000","last_peered":"2023-10-09T10:48:38.737597+0000","last_clean":"2023-10-09T10:48:38.737597+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:48:38.737597+0000","last_undegraded":"2023-10-09T10:48:38.737597+0000","last_fullsized":"2023-10-09T10:48:38.737597+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":178,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.734524+0000","last_change":"2023-10-09T10:46:32.245583+0000","last_active":"2023-10-09T10:48:38.734524+0000","last_peered":"2023-10-09T10:48:38.734524+0000","last_clean":"2023-10-09T10:48:38.734524+0000","last_became_active":"2023-10-09T10:46:32.245271+0000","last_became_peered":"2023-10-09T10:46:32.245271+0000","last_unstale":"2023-10-09T10:48:38.734524+0000","last_undegraded":"2023-10-09T10:48:38.734524+0000","last_fullsized":"2023-10-09T10:48:38.734524+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.025051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":521,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.399165+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:48:32.399165+0000","last_peered":"2023-10-09T10:48:32.399165+0000","last_clean":"2023-10-09T10:48:32.399165+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:48:32.399165+0000","last_undegraded":"2023-10-09T10:48:32.399165+0000","last_fullsized":"2023-10-09T10:48:32.399165+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"186'12480","reported_seq":12577,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:44.643889+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:48:44.643889+0000","last_peered":"2023-10-09T10:48:44.643889+0000","last_clean":"2023-10-09T10:48:44.643889+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:48:44.643889+0000","last_undegraded":"2023-10-09T10:48:44.643889+0000","last_fullsized":"2023-10-09T10:48:44.643889+0000","mapping_epoch":98,"log_start":"149'6778","ondisk_log_start":"149'6778","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5702,"log_dups_size":0,"ondisk_log_size":5702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52690944,"num_objects":804,"num_object_clones":0,"num_object_copies":2412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12480,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":162,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.733486+0000","last_change":"2023-10-09T10:46:32.245685+0000","last_active":"2023-10-09T10:48:38.733486+0000","last_peered":"2023-10-09T10:48:38.733486+0000","last_clean":"2023-10-09T10:48:38.733486+0000","last_became_active":"2023-10-09T10:46:32.245314+0000","last_became_peered":"2023-10-09T10:46:32.245314+0000","last_unstale":"2023-10-09T10:48:38.733486+0000","last_undegraded":"2023-10-09T10:48:38.733486+0000","last_fullsized":"2023-10-09T10:48:38.733486+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:51.716860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.735061+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:48:38.735061+0000","last_peered":"2023-10-09T10:48:38.735061+0000","last_clean":"2023-10-09T10:48:38.735061+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:48:38.735061+0000","last_undegraded":"2023-10-09T10:48:38.735061+0000","last_fullsized":"2023-10-09T10:48:38.735061+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":512,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.413372+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:48:32.413372+0000","last_peered":"2023-10-09T10:48:32.413372+0000","last_clean":"2023-10-09T10:48:32.413372+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:48:32.413372+0000","last_undegraded":"2023-10-09T10:48:32.413372+0000","last_fullsized":"2023-10-09T10:48:32.413372+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"187'12670","reported_seq":12787,"reported_epoch":187,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:46.146327+0000","last_change":"2023-10-09T10:46:02.754507+0000","last_active":"2023-10-09T10:48:46.146327+0000","last_peered":"2023-10-09T10:48:46.146327+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:46:02.747510+0000","last_became_peered":"2023-10-09T10:46:02.747510+0000","last_unstale":"2023-10-09T10:48:46.146327+0000","last_undegraded":"2023-10-09T10:46:02.605987+0000","last_fullsized":"2023-10-09T10:46:02.585248+0000","mapping_epoch":142,"log_start":"119'3800","ondisk_log_start":"119'3800","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8870,"log_dups_size":0,"ondisk_log_size":8870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:59.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47513624,"num_objects":726,"num_object_clones":0,"num_object_copies":2178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":726,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12670,"num_write_kb":428609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,2,5","objects":725},{"shards":"1,5","objects":-1992}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":156,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.395788+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:48:32.395788+0000","last_peered":"2023-10-09T10:48:32.395788+0000","last_clean":"2023-10-09T10:48:32.395788+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:48:32.395788+0000","last_undegraded":"2023-10-09T10:48:32.395788+0000","last_fullsized":"2023-10-09T10:48:32.395788+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":139,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.744170+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:48:38.744170+0000","last_peered":"2023-10-09T10:48:38.744170+0000","last_clean":"2023-10-09T10:48:38.744170+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:48:38.744170+0000","last_undegraded":"2023-10-09T10:48:38.744170+0000","last_fullsized":"2023-10-09T10:48:38.744170+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":521,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.401648+0000","last_change":"2023-10-09T10:46:05.617983+0000","last_active":"2023-10-09T10:48:32.401648+0000","last_peered":"2023-10-09T10:48:32.401648+0000","last_clean":"2023-10-09T10:48:32.401648+0000","last_became_active":"2023-10-09T10:46:05.616021+0000","last_became_peered":"2023-10-09T10:46:05.616021+0000","last_unstale":"2023-10-09T10:48:32.401648+0000","last_undegraded":"2023-10-09T10:48:32.401648+0000","last_fullsized":"2023-10-09T10:48:32.401648+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":182,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.745099+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:48:38.745099+0000","last_peered":"2023-10-09T10:48:38.745099+0000","last_clean":"2023-10-09T10:48:38.745099+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:48:38.745099+0000","last_undegraded":"2023-10-09T10:48:38.745099+0000","last_fullsized":"2023-10-09T10:48:38.745099+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":179,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723418+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:48:38.723418+0000","last_peered":"2023-10-09T10:48:38.723418+0000","last_clean":"2023-10-09T10:48:38.723418+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:48:38.723418+0000","last_undegraded":"2023-10-09T10:48:38.723418+0000","last_fullsized":"2023-10-09T10:48:38.723418+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":124,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.739525+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:48:38.739525+0000","last_peered":"2023-10-09T10:48:38.739525+0000","last_clean":"2023-10-09T10:48:38.739525+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:48:38.739525+0000","last_undegraded":"2023-10-09T10:48:38.739525+0000","last_fullsized":"2023-10-09T10:48:38.739525+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"186'7262","reported_seq":11305,"reported_epoch":186,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:44.734289+0000","last_change":"2023-10-09T10:46:32.248218+0000","last_active":"2023-10-09T10:48:44.734289+0000","last_peered":"2023-10-09T10:48:44.734289+0000","last_clean":"2023-10-09T10:46:02.181912+0000","last_became_active":"2023-10-09T10:46:32.226237+0000","last_became_peered":"2023-10-09T10:46:32.226237+0000","last_unstale":"2023-10-09T10:48:44.734289+0000","last_undegraded":"2023-10-09T10:48:44.734289+0000","last_fullsized":"2023-10-09T10:48:44.734289+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7262,"log_dups_size":0,"ondisk_log_size":7262,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28147712,"num_objects":429,"num_object_clones":0,"num_object_copies":1287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":429,"num_objects_unfound":0,"num_objects_dirty":429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6215,"num_write_kb":212640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":429}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":520,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.397696+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:48:32.397696+0000","last_peered":"2023-10-09T10:48:32.397696+0000","last_clean":"2023-10-09T10:48:32.397696+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:48:32.397696+0000","last_undegraded":"2023-10-09T10:48:32.397696+0000","last_fullsized":"2023-10-09T10:48:32.397696+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"186'7422","reported_seq":7610,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:44.608061+0000","last_change":"2023-10-09T10:46:02.760963+0000","last_active":"2023-10-09T10:48:44.608061+0000","last_peered":"2023-10-09T10:48:44.608061+0000","last_clean":"2023-10-09T10:48:44.608061+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:48:44.608061+0000","last_undegraded":"2023-10-09T10:48:44.608061+0000","last_fullsized":"2023-10-09T10:48:44.608061+0000","mapping_epoch":98,"log_start":"104'1458","ondisk_log_start":"104'1458","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"141'4416","last_scrub_stamp":"2023-10-09T10:46:02.760810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:46:02.760810+0000","objects_scrubbed":3378,"log_size":5964,"log_dups_size":0,"ondisk_log_size":5964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:35.863002+0000","scrub_duration":0.47769323899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24379392,"num_objects":372,"num_object_clones":0,"num_object_copies":1116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3006,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":156,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723866+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:48:38.723866+0000","last_peered":"2023-10-09T10:48:38.723866+0000","last_clean":"2023-10-09T10:48:38.723866+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:48:38.723866+0000","last_undegraded":"2023-10-09T10:48:38.723866+0000","last_fullsized":"2023-10-09T10:48:38.723866+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":172,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.734678+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:48:38.734678+0000","last_peered":"2023-10-09T10:48:38.734678+0000","last_clean":"2023-10-09T10:48:38.734678+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:48:38.734678+0000","last_undegraded":"2023-10-09T10:48:38.734678+0000","last_fullsized":"2023-10-09T10:48:38.734678+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":163,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.745164+0000","last_change":"2023-10-09T10:46:32.245608+0000","last_active":"2023-10-09T10:48:38.745164+0000","last_peered":"2023-10-09T10:48:38.745164+0000","last_clean":"2023-10-09T10:48:38.745164+0000","last_became_active":"2023-10-09T10:46:32.245290+0000","last_became_peered":"2023-10-09T10:46:32.245290+0000","last_unstale":"2023-10-09T10:48:38.745164+0000","last_undegraded":"2023-10-09T10:48:38.745164+0000","last_fullsized":"2023-10-09T10:48:38.745164+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:48.832515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":521,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.413448+0000","last_change":"2023-10-09T10:46:05.620057+0000","last_active":"2023-10-09T10:48:32.413448+0000","last_peered":"2023-10-09T10:48:32.413448+0000","last_clean":"2023-10-09T10:48:32.413448+0000","last_became_active":"2023-10-09T10:46:05.619703+0000","last_became_peered":"2023-10-09T10:46:05.619703+0000","last_unstale":"2023-10-09T10:48:32.413448+0000","last_undegraded":"2023-10-09T10:48:32.413448+0000","last_fullsized":"2023-10-09T10:48:32.413448+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"186'7325","reported_seq":7448,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:44.623203+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:48:44.623203+0000","last_peered":"2023-10-09T10:48:44.623203+0000","last_clean":"2023-10-09T10:48:44.623203+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:48:44.623203+0000","last_undegraded":"2023-10-09T10:48:44.623203+0000","last_fullsized":"2023-10-09T10:48:44.623203+0000","mapping_epoch":115,"log_start":"104'1207","ondisk_log_start":"104'1207","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6118,"log_dups_size":0,"ondisk_log_size":6118,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27262976,"num_objects":416,"num_object_clones":0,"num_object_copies":1248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6264,"num_write_kb":213760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":184,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.726147+0000","last_change":"2023-10-09T10:46:32.241041+0000","last_active":"2023-10-09T10:48:38.726147+0000","last_peered":"2023-10-09T10:48:38.726147+0000","last_clean":"2023-10-09T10:48:38.726147+0000","last_became_active":"2023-10-09T10:46:32.238683+0000","last_became_peered":"2023-10-09T10:46:32.238683+0000","last_unstale":"2023-10-09T10:48:38.726147+0000","last_undegraded":"2023-10-09T10:48:38.726147+0000","last_fullsized":"2023-10-09T10:48:38.726147+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:25.817713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":149,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.728288+0000","last_change":"2023-10-09T10:46:32.250104+0000","last_active":"2023-10-09T10:48:38.728288+0000","last_peered":"2023-10-09T10:48:38.728288+0000","last_clean":"2023-10-09T10:48:38.728288+0000","last_became_active":"2023-10-09T10:46:32.249805+0000","last_became_peered":"2023-10-09T10:46:32.249805+0000","last_unstale":"2023-10-09T10:48:38.728288+0000","last_undegraded":"2023-10-09T10:48:38.728288+0000","last_fullsized":"2023-10-09T10:48:38.728288+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:07.230672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":177,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.744261+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:48:38.744261+0000","last_peered":"2023-10-09T10:48:38.744261+0000","last_clean":"2023-10-09T10:48:38.744261+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:48:38.744261+0000","last_undegraded":"2023-10-09T10:48:38.744261+0000","last_fullsized":"2023-10-09T10:48:38.744261+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"187'7260","reported_seq":7371,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.117874+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:48:46.117874+0000","last_peered":"2023-10-09T10:48:46.117874+0000","last_clean":"2023-10-09T10:48:46.117874+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:48:46.117874+0000","last_undegraded":"2023-10-09T10:48:46.117874+0000","last_fullsized":"2023-10-09T10:48:46.117874+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7260,"log_dups_size":0,"ondisk_log_size":7260,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26083328,"num_objects":398,"num_object_clones":0,"num_object_copies":1194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6252,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":166,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723071+0000","last_change":"2023-10-09T10:46:32.235346+0000","last_active":"2023-10-09T10:48:38.723071+0000","last_peered":"2023-10-09T10:48:38.723071+0000","last_clean":"2023-10-09T10:48:38.723071+0000","last_became_active":"2023-10-09T10:46:32.234906+0000","last_became_peered":"2023-10-09T10:46:32.234906+0000","last_unstale":"2023-10-09T10:48:38.723071+0000","last_undegraded":"2023-10-09T10:48:38.723071+0000","last_fullsized":"2023-10-09T10:48:38.723071+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.199251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":521,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.413471+0000","last_change":"2023-10-09T10:46:05.617146+0000","last_active":"2023-10-09T10:48:32.413471+0000","last_peered":"2023-10-09T10:48:32.413471+0000","last_clean":"2023-10-09T10:48:32.413471+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:48:32.413471+0000","last_undegraded":"2023-10-09T10:48:32.413471+0000","last_fullsized":"2023-10-09T10:48:32.413471+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":114,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.388986+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:48:32.388986+0000","last_peered":"2023-10-09T10:48:32.388986+0000","last_clean":"2023-10-09T10:48:32.388986+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:48:32.388986+0000","last_undegraded":"2023-10-09T10:48:32.388986+0000","last_fullsized":"2023-10-09T10:48:32.388986+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":84,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.734325+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:48:38.734325+0000","last_peered":"2023-10-09T10:48:38.734325+0000","last_clean":"2023-10-09T10:48:38.734325+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:48:38.734325+0000","last_undegraded":"2023-10-09T10:48:38.734325+0000","last_fullsized":"2023-10-09T10:48:38.734325+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":521,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.399087+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:48:32.399087+0000","last_peered":"2023-10-09T10:48:32.399087+0000","last_clean":"2023-10-09T10:48:32.399087+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:48:32.399087+0000","last_undegraded":"2023-10-09T10:48:32.399087+0000","last_fullsized":"2023-10-09T10:48:32.399087+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":179,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.728411+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:48:38.728411+0000","last_peered":"2023-10-09T10:48:38.728411+0000","last_clean":"2023-10-09T10:48:38.728411+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:48:38.728411+0000","last_undegraded":"2023-10-09T10:48:38.728411+0000","last_fullsized":"2023-10-09T10:48:38.728411+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":64,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.397320+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:48:32.397320+0000","last_peered":"2023-10-09T10:48:32.397320+0000","last_clean":"2023-10-09T10:48:32.397320+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:48:32.397320+0000","last_undegraded":"2023-10-09T10:48:32.397320+0000","last_fullsized":"2023-10-09T10:48:32.397320+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":64,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.419687+0000","last_change":"2023-10-09T10:46:32.236202+0000","last_active":"2023-10-09T10:48:32.419687+0000","last_peered":"2023-10-09T10:48:32.419687+0000","last_clean":"2023-10-09T10:48:32.419687+0000","last_became_active":"2023-10-09T10:46:32.235901+0000","last_became_peered":"2023-10-09T10:46:32.235901+0000","last_unstale":"2023-10-09T10:48:32.419687+0000","last_undegraded":"2023-10-09T10:48:32.419687+0000","last_fullsized":"2023-10-09T10:48:32.419687+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:48.260403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"187'7389","reported_seq":7548,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.449598+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:48:46.449598+0000","last_peered":"2023-10-09T10:48:46.449598+0000","last_clean":"2023-10-09T10:48:46.449598+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:48:46.449598+0000","last_undegraded":"2023-10-09T10:48:46.449598+0000","last_fullsized":"2023-10-09T10:48:46.449598+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":7389,"log_dups_size":0,"ondisk_log_size":7389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25493504,"num_objects":389,"num_object_clones":0,"num_object_copies":1167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4359,"num_write_kb":89664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":521,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.419447+0000","last_change":"2023-10-09T10:46:05.620764+0000","last_active":"2023-10-09T10:48:32.419447+0000","last_peered":"2023-10-09T10:48:32.419447+0000","last_clean":"2023-10-09T10:48:32.419447+0000","last_became_active":"2023-10-09T10:46:05.619870+0000","last_became_peered":"2023-10-09T10:46:05.619870+0000","last_unstale":"2023-10-09T10:48:32.419447+0000","last_undegraded":"2023-10-09T10:48:32.419447+0000","last_fullsized":"2023-10-09T10:48:32.419447+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":174,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.398159+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:48:32.398159+0000","last_peered":"2023-10-09T10:48:32.398159+0000","last_clean":"2023-10-09T10:48:32.398159+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:48:32.398159+0000","last_undegraded":"2023-10-09T10:48:32.398159+0000","last_fullsized":"2023-10-09T10:48:32.398159+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":148,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.745245+0000","last_change":"2023-10-09T10:46:32.251450+0000","last_active":"2023-10-09T10:48:38.745245+0000","last_peered":"2023-10-09T10:48:38.745245+0000","last_clean":"2023-10-09T10:48:38.745245+0000","last_became_active":"2023-10-09T10:46:32.250933+0000","last_became_peered":"2023-10-09T10:46:32.250933+0000","last_unstale":"2023-10-09T10:48:38.745245+0000","last_undegraded":"2023-10-09T10:48:38.745245+0000","last_fullsized":"2023-10-09T10:48:38.745245+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:04.116178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":55,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.399244+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:48:32.399244+0000","last_peered":"2023-10-09T10:48:32.399244+0000","last_clean":"2023-10-09T10:48:32.399244+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:48:32.399244+0000","last_undegraded":"2023-10-09T10:48:32.399244+0000","last_fullsized":"2023-10-09T10:48:32.399244+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"187'7237","reported_seq":13941,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.496155+0000","last_change":"2023-10-09T10:47:02.269289+0000","last_active":"2023-10-09T10:48:46.496155+0000","last_peered":"2023-10-09T10:48:46.496155+0000","last_clean":"2023-10-09T10:48:46.496155+0000","last_became_active":"2023-10-09T10:47:02.257854+0000","last_became_peered":"2023-10-09T10:47:02.257854+0000","last_unstale":"2023-10-09T10:48:46.496155+0000","last_undegraded":"2023-10-09T10:48:46.496155+0000","last_fullsized":"2023-10-09T10:48:46.496155+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7237,"log_dups_size":0,"ondisk_log_size":7237,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24477696,"num_objects":373,"num_object_clones":0,"num_object_copies":1119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6229,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3287,"num_bytes_recovered":181927936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":521,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.410684+0000","last_change":"2023-10-09T10:46:05.618443+0000","last_active":"2023-10-09T10:48:32.410684+0000","last_peered":"2023-10-09T10:48:32.410684+0000","last_clean":"2023-10-09T10:48:32.410684+0000","last_became_active":"2023-10-09T10:46:05.616588+0000","last_became_peered":"2023-10-09T10:46:05.616588+0000","last_unstale":"2023-10-09T10:48:32.410684+0000","last_undegraded":"2023-10-09T10:48:32.410684+0000","last_fullsized":"2023-10-09T10:48:32.410684+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":179,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.388126+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:48:32.388126+0000","last_peered":"2023-10-09T10:48:32.388126+0000","last_clean":"2023-10-09T10:48:32.388126+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:48:32.388126+0000","last_undegraded":"2023-10-09T10:48:32.388126+0000","last_fullsized":"2023-10-09T10:48:32.388126+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":124,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.743551+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:48:38.743551+0000","last_peered":"2023-10-09T10:48:38.743551+0000","last_clean":"2023-10-09T10:48:38.743551+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:48:38.743551+0000","last_undegraded":"2023-10-09T10:48:38.743551+0000","last_fullsized":"2023-10-09T10:48:38.743551+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":168,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.395854+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:48:32.395854+0000","last_peered":"2023-10-09T10:48:32.395854+0000","last_clean":"2023-10-09T10:48:32.395854+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:48:32.395854+0000","last_undegraded":"2023-10-09T10:48:32.395854+0000","last_fullsized":"2023-10-09T10:48:32.395854+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"187'7342","reported_seq":14185,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.047667+0000","last_change":"2023-10-09T10:48:17.301310+0000","last_active":"2023-10-09T10:48:46.047667+0000","last_peered":"2023-10-09T10:48:46.047667+0000","last_clean":"2023-10-09T10:48:46.047667+0000","last_became_active":"2023-10-09T10:48:17.283042+0000","last_became_peered":"2023-10-09T10:48:17.283042+0000","last_unstale":"2023-10-09T10:48:46.047667+0000","last_undegraded":"2023-10-09T10:48:46.047667+0000","last_fullsized":"2023-10-09T10:48:46.047667+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7342,"log_dups_size":0,"ondisk_log_size":7342,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23560192,"num_objects":359,"num_object_clones":0,"num_object_copies":1077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6305,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3337,"num_bytes_recovered":129564672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":520,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.401065+0000","last_change":"2023-10-09T10:46:05.617589+0000","last_active":"2023-10-09T10:48:32.401065+0000","last_peered":"2023-10-09T10:48:32.401065+0000","last_clean":"2023-10-09T10:48:32.401065+0000","last_became_active":"2023-10-09T10:46:05.616400+0000","last_became_peered":"2023-10-09T10:46:05.616400+0000","last_unstale":"2023-10-09T10:48:32.401065+0000","last_undegraded":"2023-10-09T10:48:32.401065+0000","last_fullsized":"2023-10-09T10:48:32.401065+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":44,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.726005+0000","last_change":"2023-10-09T10:46:32.245106+0000","last_active":"2023-10-09T10:48:38.726005+0000","last_peered":"2023-10-09T10:48:38.726005+0000","last_clean":"2023-10-09T10:48:38.726005+0000","last_became_active":"2023-10-09T10:46:32.244826+0000","last_became_peered":"2023-10-09T10:46:32.244826+0000","last_unstale":"2023-10-09T10:48:38.726005+0000","last_undegraded":"2023-10-09T10:48:38.726005+0000","last_fullsized":"2023-10-09T10:48:38.726005+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:30.322834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":59,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.419612+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:48:32.419612+0000","last_peered":"2023-10-09T10:48:32.419612+0000","last_clean":"2023-10-09T10:48:32.419612+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:48:32.419612+0000","last_undegraded":"2023-10-09T10:48:32.419612+0000","last_fullsized":"2023-10-09T10:48:32.419612+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":134,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.733317+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:48:38.733317+0000","last_peered":"2023-10-09T10:48:38.733317+0000","last_clean":"2023-10-09T10:48:38.733317+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:48:38.733317+0000","last_undegraded":"2023-10-09T10:48:38.733317+0000","last_fullsized":"2023-10-09T10:48:38.733317+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"186'7242","reported_seq":14057,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:45.874231+0000","last_change":"2023-10-09T10:47:31.271196+0000","last_active":"2023-10-09T10:48:45.874231+0000","last_peered":"2023-10-09T10:48:45.874231+0000","last_clean":"2023-10-09T10:48:45.874231+0000","last_became_active":"2023-10-09T10:47:31.229504+0000","last_became_peered":"2023-10-09T10:47:31.229504+0000","last_unstale":"2023-10-09T10:48:45.874231+0000","last_undegraded":"2023-10-09T10:48:45.874231+0000","last_fullsized":"2023-10-09T10:48:45.874231+0000","mapping_epoch":172,"log_start":"113'2515","ondisk_log_start":"113'2515","created":98,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":4727,"log_dups_size":0,"ondisk_log_size":4727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:26.533476+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25165824,"num_objects":384,"num_object_clones":0,"num_object_copies":1152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4597,"num_write_kb":107264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3305,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":55,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.798790+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:48:32.798790+0000","last_peered":"2023-10-09T10:48:32.798790+0000","last_clean":"2023-10-09T10:48:32.798790+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:48:32.798790+0000","last_undegraded":"2023-10-09T10:48:32.798790+0000","last_fullsized":"2023-10-09T10:48:32.798790+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":118,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.745312+0000","last_change":"2023-10-09T10:46:32.251242+0000","last_active":"2023-10-09T10:48:38.745312+0000","last_peered":"2023-10-09T10:48:38.745312+0000","last_clean":"2023-10-09T10:48:38.745312+0000","last_became_active":"2023-10-09T10:46:32.250534+0000","last_became_peered":"2023-10-09T10:46:32.250534+0000","last_unstale":"2023-10-09T10:48:38.745312+0000","last_undegraded":"2023-10-09T10:48:38.745312+0000","last_fullsized":"2023-10-09T10:48:38.745312+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:33.726041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"186'7250","reported_seq":13118,"reported_epoch":186,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:42.765377+0000","last_change":"2023-10-09T10:46:34.514479+0000","last_active":"2023-10-09T10:48:42.765377+0000","last_peered":"2023-10-09T10:48:42.765377+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:46:34.409667+0000","last_became_peered":"2023-10-09T10:46:34.409667+0000","last_unstale":"2023-10-09T10:48:42.765377+0000","last_undegraded":"2023-10-09T10:46:34.376635+0000","last_fullsized":"2023-10-09T10:46:33.237998+0000","mapping_epoch":154,"log_start":"106'1997","ondisk_log_start":"106'1997","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5253,"log_dups_size":0,"ondisk_log_size":5253,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30539776,"num_objects":466,"num_object_clones":0,"num_object_copies":1398,"num_objects_missing_on_primary":467,"num_objects_missing":467,"num_objects_degraded":977,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6208,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2871,"num_bytes_recovered":188153856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,7","objects":-1},{"shards":"2,3,7","objects":467}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":154,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723973+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:48:38.723973+0000","last_peered":"2023-10-09T10:48:38.723973+0000","last_clean":"2023-10-09T10:48:38.723973+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:48:38.723973+0000","last_undegraded":"2023-10-09T10:48:38.723973+0000","last_fullsized":"2023-10-09T10:48:38.723973+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":168,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.734803+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:48:38.734803+0000","last_peered":"2023-10-09T10:48:38.734803+0000","last_clean":"2023-10-09T10:48:38.734803+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:48:38.734803+0000","last_undegraded":"2023-10-09T10:48:38.734803+0000","last_fullsized":"2023-10-09T10:48:38.734803+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"186'7209","reported_seq":7327,"reported_epoch":186,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:44.628257+0000","last_change":"2023-10-09T10:46:33.546991+0000","last_active":"2023-10-09T10:48:44.628257+0000","last_peered":"2023-10-09T10:48:44.628257+0000","last_clean":"2023-10-09T10:46:31.009484+0000","last_became_active":"2023-10-09T10:46:33.535665+0000","last_became_peered":"2023-10-09T10:46:33.535665+0000","last_unstale":"2023-10-09T10:48:44.628257+0000","last_undegraded":"2023-10-09T10:46:33.277168+0000","last_fullsized":"2023-10-09T10:46:33.235038+0000","mapping_epoch":154,"log_start":"104'1001","ondisk_log_start":"104'1001","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6208,"log_dups_size":0,"ondisk_log_size":6208,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25067520,"num_objects":383,"num_object_clones":0,"num_object_copies":1149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6145,"num_write_kb":208864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":395},{"shards":"3,7","objects":-2375}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":137,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723528+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:48:38.723528+0000","last_peered":"2023-10-09T10:48:38.723528+0000","last_clean":"2023-10-09T10:48:38.723528+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:48:38.723528+0000","last_undegraded":"2023-10-09T10:48:38.723528+0000","last_fullsized":"2023-10-09T10:48:38.723528+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":35,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.737526+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:48:38.737526+0000","last_peered":"2023-10-09T10:48:38.737526+0000","last_clean":"2023-10-09T10:48:38.737526+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:48:38.737526+0000","last_undegraded":"2023-10-09T10:48:38.737526+0000","last_fullsized":"2023-10-09T10:48:38.737526+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":170,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.389477+0000","last_change":"2023-10-09T10:46:32.244607+0000","last_active":"2023-10-09T10:48:32.389477+0000","last_peered":"2023-10-09T10:48:32.389477+0000","last_clean":"2023-10-09T10:48:32.389477+0000","last_became_active":"2023-10-09T10:46:32.242628+0000","last_became_peered":"2023-10-09T10:46:32.242628+0000","last_unstale":"2023-10-09T10:48:32.389477+0000","last_undegraded":"2023-10-09T10:48:32.389477+0000","last_fullsized":"2023-10-09T10:48:32.389477+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:10.738819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":201,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.406401+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:48:32.406401+0000","last_peered":"2023-10-09T10:48:32.406401+0000","last_clean":"2023-10-09T10:48:32.406401+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:48:32.406401+0000","last_undegraded":"2023-10-09T10:48:32.406401+0000","last_fullsized":"2023-10-09T10:48:32.406401+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":76,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T10:48:32.401599+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:48:32.401599+0000","last_peered":"2023-10-09T10:48:32.401599+0000","last_clean":"2023-10-09T10:48:32.401599+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:48:32.401599+0000","last_undegraded":"2023-10-09T10:48:32.401599+0000","last_fullsized":"2023-10-09T10:48:32.401599+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.733257+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:48:38.733257+0000","last_peered":"2023-10-09T10:48:38.733257+0000","last_clean":"2023-10-09T10:48:38.733257+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:48:38.733257+0000","last_undegraded":"2023-10-09T10:48:38.733257+0000","last_fullsized":"2023-10-09T10:48:38.733257+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":810123288,"num_objects":12363,"num_object_clones":0,"num_object_copies":37089,"num_objects_missing_on_primary":467,"num_objects_missing":467,"num_objects_degraded":26351,"num_objects_misplaced":1187,"num_objects_unfound":0,"num_objects_dirty":12363,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":191845,"num_write_kb":6258529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29124,"num_bytes_recovered":1594818560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4792659968,"data_stored":4792647752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170882,"ondisk_log_size":170882,"up":78,"acting":69,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1186,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332266,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":641376,"kb_used_data":344024,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93103776,"statfs":{"total":95995035648,"available":95338266624,"internally_reserved":0,"allocated":352280576,"data_stored":351572341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":1.155,"15min":0.76600000000000001},"min":{"1min":0.23499999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.476,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":7.2389999999999999},{"interface":"front","average":{"1min":3.1789999999999998,"5min":1.5029999999999999,"15min":0.92100000000000004},"min":{"1min":0.248,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.332000000000001,"5min":11.332000000000001,"15min":11.332000000000001},"last":0.26200000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:47:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2210000000000001,"5min":1.3300000000000001,"15min":0.80400000000000005},"min":{"1min":0.43099999999999999,"5min":0.318,"15min":0.318},"max":{"1min":11.826000000000001,"5min":11.826000000000001,"15min":11.826000000000001},"last":3.028},{"interface":"front","average":{"1min":2.286,"5min":1.155,"15min":0.73899999999999999},"min":{"1min":0.32100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":11.84,"5min":11.84,"15min":11.84},"last":7.2750000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:48:18 2023","interfaces":[{"interface":"back","average":{"1min":2.165,"5min":1.4810000000000001,"15min":0.83499999999999996},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.1419999999999995,"5min":9.1419999999999995,"15min":9.1419999999999995},"last":2.7010000000000001},{"interface":"front","average":{"1min":2.1970000000000001,"5min":1.1910000000000001,"15min":0.747},"min":{"1min":0.34999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":7.8120000000000003,"5min":7.8120000000000003,"15min":7.8120000000000003},"last":7.2949999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":3.903,"5min":1.893,"15min":1.0209999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":11.523,"5min":11.523,"15min":11.523},"last":2.988},{"interface":"front","average":{"1min":4.2789999999999999,"5min":1.835,"15min":1.0109999999999999},"min":{"1min":0.35299999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":19.234999999999999,"5min":19.234999999999999,"15min":19.234999999999999},"last":2.6629999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":1.1579999999999999,"15min":0.70499999999999996},"min":{"1min":0.36399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":9.4900000000000002,"5min":9.4900000000000002,"15min":9.4900000000000002},"last":0.29599999999999999},{"interface":"front","average":{"1min":2.327,"5min":1.018,"15min":0.66100000000000003},"min":{"1min":0.35399999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.0999999999999996,"5min":9.0999999999999996,"15min":9.0999999999999996},"last":7.125}]},{"osd":5,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.0249999999999999,"5min":1.0609999999999999,"15min":0.69599999999999995},"min":{"1min":0.314,"5min":0.247,"15min":0.247},"max":{"1min":12.064,"5min":12.064,"15min":12.064},"last":7.3120000000000003},{"interface":"front","average":{"1min":3.8839999999999999,"5min":1.4770000000000001,"15min":0.85299999999999998},"min":{"1min":0.435,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":11.741,"5min":11.741,"15min":11.741},"last":2.9489999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2999999999999998,"5min":1.3720000000000001,"15min":0.83199999999999996},"min":{"1min":0.33700000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.029,"5min":13.029,"15min":13.029},"last":3.2919999999999998},{"interface":"front","average":{"1min":2.6869999999999998,"5min":1.079,"15min":0.72699999999999998},"min":{"1min":0.41299999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":19.187999999999999,"5min":19.187999999999999,"15min":19.187999999999999},"last":7.0759999999999996}]}]},{"osd":6,"up_from":44,"seq":188978561197,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":572060,"kb_used_data":262484,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93173092,"statfs":{"total":95995035648,"available":95409246208,"internally_reserved":0,"allocated":268783616,"data_stored":268079919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":3.0499999999999998,"5min":1.4330000000000001,"15min":0.88},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.882,"5min":10.882,"15min":10.882},"last":4.2080000000000002},{"interface":"front","average":{"1min":2.4540000000000002,"5min":1.367,"15min":0.86199999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.8710000000000004,"5min":7.8710000000000004,"15min":7.8710000000000004},"last":1.026}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.157,"15min":0.77900000000000003},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.245},"max":{"1min":6.7560000000000002,"5min":16.747,"15min":16.747},"last":2.5739999999999998},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.284,"15min":0.79500000000000004},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.3579999999999997,"5min":16.734000000000002,"15min":16.734000000000002},"last":2.5600000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:48:20 2023","interfaces":[{"interface":"back","average":{"1min":2.798,"5min":1.411,"15min":0.82599999999999996},"min":{"1min":0.29699999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.208,"5min":13.208,"15min":13.208},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.9420000000000002,"5min":1.619,"15min":0.91300000000000003},"min":{"1min":0.309,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.6749999999999998,"5min":12.571,"15min":12.571},"last":1.2250000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":1.496,"15min":0.85699999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":7.7649999999999997,"5min":7.7649999999999997,"15min":7.7649999999999997},"last":0.434},{"interface":"front","average":{"1min":2.254,"5min":1.7270000000000001,"15min":0.93999999999999995},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":13.179,"5min":13.179,"15min":13.179},"last":0.51800000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.0629999999999999,"15min":0.65000000000000002},"min":{"1min":0.29899999999999999,"5min":0.255,"15min":0.255},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":1.2869999999999999},{"interface":"front","average":{"1min":1.6579999999999999,"5min":0.89600000000000002,"15min":0.623},"min":{"1min":0.34100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.1260000000000003,"5min":5.1260000000000003,"15min":5.1260000000000003},"last":0.36899999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":1.1719999999999999,"15min":0.71099999999999997},"min":{"1min":0.35899999999999999,"5min":0.29499999999999998,"15min":0.27800000000000002},"max":{"1min":10.497,"5min":10.497,"15min":10.497},"last":1.3320000000000001},{"interface":"front","average":{"1min":1.589,"5min":0.96599999999999997,"15min":0.67100000000000004},"min":{"1min":0.40699999999999997,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":6.8419999999999996,"5min":6.8419999999999996,"15min":6.8419999999999996},"last":0.71199999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:48:26 2023","interfaces":[{"interface":"back","average":{"1min":3.915,"5min":1.369,"15min":0.80600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":23.358000000000001,"5min":23.358000000000001,"15min":23.358000000000001},"last":1.2609999999999999},{"interface":"front","average":{"1min":2.6840000000000002,"5min":1.25,"15min":0.79700000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34000000000000002},"max":{"1min":12.404,"5min":12.404,"15min":12.404},"last":0.46500000000000002}]}]},{"osd":1,"up_from":85,"seq":365072220236,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":528036,"kb_used_data":392284,"kb_used_omap":9,"kb_used_meta":135734,"kb_avail":93217116,"statfs":{"total":95995035648,"available":95454326784,"internally_reserved":0,"allocated":401698816,"data_stored":400756085,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":138992617},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.581,"15min":0.76900000000000002},"min":{"1min":0.251,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.2119999999999997,"5min":14.1,"15min":14.1},"last":1.129},{"interface":"front","average":{"1min":1.885,"5min":1.5629999999999999,"15min":0.78700000000000003},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.8090000000000002,"5min":14.465,"15min":14.465},"last":0.48899999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:48:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":1.492,"15min":0.74199999999999999},"min":{"1min":0.221,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":15.356,"5min":15.356,"15min":15.356},"last":1.1970000000000001},{"interface":"front","average":{"1min":1.488,"5min":1.6890000000000001,"15min":0.81899999999999995},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.4130000000000003,"5min":14.234,"15min":14.234},"last":1.1020000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.6829999999999998,"5min":2.169,"15min":1.0369999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":18.274999999999999,"5min":18.274999999999999,"15min":18.274999999999999},"last":1.2150000000000001},{"interface":"front","average":{"1min":2.774,"5min":1.915,"15min":0.86799999999999999},"min":{"1min":0.313,"5min":0.251,"15min":0.251},"max":{"1min":8.5999999999999996,"5min":15.468999999999999,"15min":15.468999999999999},"last":1.7869999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.1679999999999999,"15min":0.78400000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.1449999999999996,"5min":8.0129999999999999,"15min":8.0129999999999999},"last":1.149},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.2350000000000001,"15min":0.82499999999999996},"min":{"1min":0.30299999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.1189999999999998,"5min":8.327,"15min":8.327},"last":1.056}]},{"osd":5,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":1.111,"15min":0.78900000000000003},"min":{"1min":0.28899999999999998,"5min":0.253,"15min":0.253},"max":{"1min":5.4850000000000003,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.2929999999999999,"15min":0.86199999999999999},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.1859999999999999,"5min":7.923,"15min":7.923},"last":1.351}]},{"osd":6,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7909999999999999,"5min":1.2030000000000001,"15min":0.82199999999999995},"min":{"1min":0.35499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.9450000000000003,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":5.1520000000000001},{"interface":"front","average":{"1min":1.246,"5min":1.2949999999999999,"15min":0.86299999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.8440000000000003,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":1.3149999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.22,"15min":0.83999999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.8689999999999998,"5min":7.9859999999999998,"15min":7.9859999999999998},"last":3.7450000000000001},{"interface":"front","average":{"1min":1.992,"5min":1.2909999999999999,"15min":0.86099999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":7.1589999999999998,"5min":8.2469999999999999,"15min":8.2469999999999999},"last":1.266}]}]},{"osd":0,"up_from":80,"seq":343597383769,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":357060,"kb_used_data":244924,"kb_used_omap":28,"kb_used_meta":111971,"kb_avail":93388092,"statfs":{"total":95995035648,"available":95629406208,"internally_reserved":0,"allocated":250802176,"data_stored":249826806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":114659152},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4,1,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:47:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2509999999999999,"5min":1.5449999999999999,"15min":0.76000000000000001},"min":{"1min":0.22,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":10.564,"5min":25.475000000000001,"15min":25.475000000000001},"last":8.3569999999999993},{"interface":"front","average":{"1min":2.3740000000000001,"5min":1.5940000000000001,"15min":0.77400000000000002},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":10.593999999999999,"5min":20.832999999999998,"15min":20.832999999999998},"last":8.4079999999999995}]},{"osd":2,"last update":"Mon Oct 9 10:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":1.4750000000000001,"15min":0.72399999999999998},"min":{"1min":0.249,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.0890000000000004,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":11.542999999999999},{"interface":"front","average":{"1min":2.1120000000000001,"5min":1.889,"15min":0.85799999999999998},"min":{"1min":0.27100000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.0329999999999995,"5min":14.391999999999999,"15min":14.391999999999999},"last":4.2279999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:47:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4740000000000002,"5min":1.8959999999999999,"15min":1.0660000000000001},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.215},"max":{"1min":10.644,"5min":14.926,"15min":14.926},"last":2.6920000000000002},{"interface":"front","average":{"1min":2.2269999999999999,"5min":1.5940000000000001,"15min":0.89200000000000002},"min":{"1min":0.34499999999999997,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.228999999999999,"5min":11.868,"15min":11.868},"last":5.3860000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":2.8370000000000002,"5min":1.671,"15min":0.96999999999999997},"min":{"1min":0.434,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.0920000000000005,"5min":8.0920000000000005,"15min":8.0920000000000005},"last":2.6280000000000001},{"interface":"front","average":{"1min":3.2679999999999998,"5min":1.9370000000000001,"15min":1.0569999999999999},"min":{"1min":0.36099999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.318,"5min":12.318,"15min":12.318},"last":2.734}]},{"osd":5,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6179999999999999,"5min":1.8400000000000001,"15min":1.0289999999999999},"min":{"1min":0.35899999999999999,"5min":0.318,"15min":0.318},"max":{"1min":12.157,"5min":12.157,"15min":12.157},"last":2.7709999999999999},{"interface":"front","average":{"1min":3.181,"5min":1.677,"15min":0.98999999999999999},"min":{"1min":0.40500000000000003,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":12.781000000000001,"5min":12.781000000000001,"15min":12.781000000000001},"last":5.3659999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":3.4649999999999999,"5min":1.96,"15min":1.073},"min":{"1min":0.33600000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":12.207000000000001,"5min":12.207000000000001,"15min":12.207000000000001},"last":2.714},{"interface":"front","average":{"1min":2.1269999999999998,"5min":1.3740000000000001,"15min":0.89100000000000001},"min":{"1min":0.41299999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":9.8889999999999993,"5min":9.8889999999999993,"15min":9.8889999999999993},"last":5.343}]},{"osd":7,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":3.6280000000000001,"5min":1.7509999999999999,"15min":1.0169999999999999},"min":{"1min":0.48799999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":12.727,"5min":12.727,"15min":12.727},"last":5.3049999999999997},{"interface":"front","average":{"1min":2.843,"5min":1.8660000000000001,"15min":1.071},"min":{"1min":0.47599999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.6150000000000002,"5min":14.394,"15min":14.394},"last":2.6589999999999998}]}]},{"osd":2,"up_from":0,"seq":386547056710,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893184,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":876340,"kb_used_data":636844,"kb_used_omap":7,"kb_used_meta":239416,"kb_avail":92868812,"statfs":{"total":95995035648,"available":95097663488,"internally_reserved":0,"allocated":652128256,"data_stored":651167527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":245162600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3159999999999998,"5min":1.913,"15min":0.85099999999999998},"min":{"1min":0.248,"5min":0.183,"15min":0.183},"max":{"1min":15.255000000000001,"5min":15.255000000000001,"15min":15.255000000000001},"last":3.9780000000000002},{"interface":"front","average":{"1min":2.9830000000000001,"5min":1.6539999999999999,"15min":0.82199999999999995},"min":{"1min":0.21099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":8.6470000000000002,"5min":8.6470000000000002,"15min":8.6470000000000002},"last":0.48699999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.6400000000000001,"5min":2.359,"15min":1.006},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":10.821,"5min":14.707000000000001,"15min":14.707000000000001},"last":7.2439999999999998},{"interface":"front","average":{"1min":3.4780000000000002,"5min":1.8100000000000001,"15min":0.85199999999999998},"min":{"1min":0.28999999999999998,"5min":0.193,"15min":0.193},"max":{"1min":15.443,"5min":15.443,"15min":15.443},"last":0.28999999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":2.0630000000000002,"15min":0.93200000000000005},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.375,"5min":13.853,"15min":13.853},"last":7.2140000000000004},{"interface":"front","average":{"1min":3.988,"5min":2.2309999999999999,"15min":1.0069999999999999},"min":{"1min":0.28699999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.557,"5min":10.557,"15min":10.557},"last":10.116}]},{"osd":4,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":4.0739999999999998,"5min":1.9219999999999999,"15min":1.022},"min":{"1min":0.33900000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":20.388000000000002,"5min":20.388000000000002,"15min":20.388000000000002},"last":0.42999999999999999},{"interface":"front","average":{"1min":4.4299999999999997,"5min":2.0609999999999999,"15min":1.0920000000000001},"min":{"1min":0.38700000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":19.085999999999999,"5min":19.085999999999999,"15min":19.085999999999999},"last":0.56899999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.7599999999999998,"5min":2.4369999999999998,"15min":1.2030000000000001},"min":{"1min":0.35399999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.3550000000000004,"5min":51.509999999999998,"15min":51.509999999999998},"last":0.59899999999999998},{"interface":"front","average":{"1min":2.6240000000000001,"5min":2.7189999999999999,"15min":1.2809999999999999},"min":{"1min":0.316,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.7449999999999992,"5min":55.338999999999999,"15min":55.338999999999999},"last":2.403}]},{"osd":6,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":1.841,"15min":1.0049999999999999},"min":{"1min":0.43099999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.361000000000001,"5min":12.361000000000001,"15min":12.361000000000001},"last":0.62},{"interface":"front","average":{"1min":2.8140000000000001,"5min":2.2189999999999999,"15min":1.139},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.5839999999999996,"5min":13.891999999999999,"15min":13.891999999999999},"last":5.2990000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.2269999999999999,"5min":2.8679999999999999,"15min":1.3500000000000001},"min":{"1min":0.34000000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.734,"5min":51.744,"15min":51.744},"last":5.1130000000000004},{"interface":"front","average":{"1min":3.927,"5min":2.54,"15min":1.2290000000000001},"min":{"1min":0.46100000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":15.212999999999999,"5min":53.399000000000001,"15min":53.399000000000001},"last":0.46100000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019060,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":544220,"kb_used_data":242836,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93200932,"statfs":{"total":95995035648,"available":95437754368,"internally_reserved":0,"allocated":248664064,"data_stored":247959634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.9850000000000001,"5min":1.302,"15min":0.80500000000000005},"min":{"1min":0.216,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.7160000000000002,"5min":10.387,"15min":10.387},"last":1.53},{"interface":"front","average":{"1min":2.9780000000000002,"5min":1.6659999999999999,"15min":0.94299999999999995},"min":{"1min":0.48499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.0129999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":3.3450000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":2.351,"5min":1.512,"15min":0.872},"min":{"1min":0.254,"5min":0.249,"15min":0.249},"max":{"1min":7.5350000000000001,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":2.4550000000000001},{"interface":"front","average":{"1min":2.931,"5min":1.7010000000000001,"15min":0.98099999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.2919999999999998,"5min":8.7669999999999995,"15min":8.7669999999999995},"last":1.762}]},{"osd":2,"last update":"Mon Oct 9 10:48:13 2023","interfaces":[{"interface":"back","average":{"1min":3.1520000000000001,"5min":1.617,"15min":0.91900000000000004},"min":{"1min":0.52100000000000002,"5min":0.253,"15min":0.253},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":2.3929999999999998},{"interface":"front","average":{"1min":4.016,"5min":1.855,"15min":0.99299999999999999},"min":{"1min":0.36799999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":20.545999999999999,"5min":20.545999999999999,"15min":20.545999999999999},"last":2.4159999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":3.2549999999999999,"5min":1.8420000000000001,"15min":1.0880000000000001},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.0269999999999992,"5min":9.0269999999999992,"15min":9.0269999999999992},"last":1.718},{"interface":"front","average":{"1min":3.9809999999999999,"5min":2.0379999999999998,"15min":1.071},"min":{"1min":0.79400000000000004,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.6029999999999998,"5min":8.9809999999999999,"15min":8.9809999999999999},"last":2.3559999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":5.0960000000000001,"5min":1.8140000000000001,"15min":0.91000000000000003},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":24.149999999999999,"5min":24.149999999999999,"15min":24.149999999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":4.173,"5min":1.6699999999999999,"15min":0.86399999999999999},"min":{"1min":0.373,"5min":0.252,"15min":0.252},"max":{"1min":20.422000000000001,"5min":20.422000000000001,"15min":20.422000000000001},"last":4.9619999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":1.1319999999999999,"15min":0.69399999999999995},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":12.391999999999999,"5min":12.391999999999999,"15min":12.391999999999999},"last":6.3719999999999999},{"interface":"front","average":{"1min":3.1200000000000001,"5min":1.2130000000000001,"15min":0.78100000000000003},"min":{"1min":0.374,"5min":0.311,"15min":0.311},"max":{"1min":12.539,"5min":12.539,"15min":12.539},"last":10.329000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:48:21 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":1.2929999999999999,"15min":0.80900000000000005},"min":{"1min":0.33300000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.7919999999999998,"5min":8.7919999999999998,"15min":8.7919999999999998},"last":4.3659999999999997},{"interface":"front","average":{"1min":3.2389999999999999,"5min":1.27,"15min":0.79000000000000004},"min":{"1min":0.55900000000000005,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.423,"5min":7.423,"15min":7.423},"last":2.6890000000000001}]}]},{"osd":5,"up_from":37,"seq":158913790129,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":624076,"kb_used_data":318596,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93121076,"statfs":{"total":95995035648,"available":95355981824,"internally_reserved":0,"allocated":326242304,"data_stored":325549703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:41 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":1.2210000000000001,"15min":0.80500000000000005},"min":{"1min":0.28899999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.648999999999999,"5min":11.648999999999999,"15min":11.648999999999999},"last":1.401},{"interface":"front","average":{"1min":2.8570000000000002,"5min":1.294,"15min":0.875},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.68,"5min":11.68,"15min":11.68},"last":1.4159999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":1.367,"15min":0.86099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":12.999000000000001,"5min":12.999000000000001,"15min":12.999000000000001},"last":2.2189999999999999},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.2370000000000001,"15min":0.80700000000000005},"min":{"1min":0.27600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3730000000000002,"5min":7.3730000000000002,"15min":7.3730000000000002},"last":2.8540000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:48:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6669999999999998,"5min":1.4390000000000001,"15min":0.86099999999999999},"min":{"1min":0.34799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.3280000000000003,"5min":7.9000000000000004,"15min":7.9000000000000004},"last":0.66400000000000003},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.2350000000000001,"15min":0.79400000000000004},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.3059999999999992,"5min":10.352,"15min":10.352},"last":1.3740000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:47:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4969999999999999,"5min":1.621,"15min":1.0229999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":18.016999999999999,"5min":18.016999999999999,"15min":18.016999999999999},"last":1.7829999999999999},{"interface":"front","average":{"1min":2.964,"5min":1.673,"15min":0.96599999999999997},"min":{"1min":0.318,"5min":0.317,"15min":0.317},"max":{"1min":15.555999999999999,"5min":15.555999999999999,"15min":15.555999999999999},"last":2.8730000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":0.83799999999999997,"15min":0.57799999999999996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":19.152999999999999,"5min":19.152999999999999,"15min":19.152999999999999},"last":1.4510000000000001},{"interface":"front","average":{"1min":1.986,"5min":0.96999999999999997,"15min":0.63500000000000001},"min":{"1min":0.26800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.3059999999999992,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":1.0609999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":1.127,"15min":0.69899999999999995},"min":{"1min":0.33100000000000002,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":7.7060000000000004,"5min":7.7060000000000004,"15min":7.7060000000000004},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.958,"5min":0.90400000000000003,"15min":0.64000000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.30299999999999999},"max":{"1min":8.0909999999999993,"5min":8.0909999999999993,"15min":8.0909999999999993},"last":1.4690000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:48:26 2023","interfaces":[{"interface":"back","average":{"1min":2.79,"5min":1.1160000000000001,"15min":0.74099999999999999},"min":{"1min":0.41899999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.3940000000000001,"5min":7.3940000000000001,"15min":7.3940000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":2.2669999999999999,"5min":1.0369999999999999,"15min":0.71999999999999997},"min":{"1min":0.38500000000000001,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":9.0579999999999998,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":1.4350000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":246288384,"data_stored":246284312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":387125248,"data_stored":387121176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2347433984,"data_stored":2347433984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":649134080,"data_stored":649134080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":243400704,"data_stored":243400704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":313987072,"data_stored":313983000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":264437760,"data_stored":264437760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":340852736,"data_stored":340852736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:48:49.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:49.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:49.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:49.124 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:49.190 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:49.207 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:49 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3550214442' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:49.208 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:49 smithi125 ceph-mon[200197]: from='client.54877 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:49.210 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:49 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3550214442' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:49.210 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:49 smithi125 ceph-mon[195507]: from='client.54877 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:49.216 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:49.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:49.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:49.301 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:49.324 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:49.368 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:49 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3550214442' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:49.368 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:49 smithi155 ceph-mon[184854]: from='client.54877 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:49.370 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:49.413 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:49.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:49.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:49.517 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:49.576 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:49.722 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:50.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:50 smithi155 ceph-mon[184854]: pgmap v416: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 761 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 25864/36555 objects degraded (70.754%); 1187/36555 objects misplaced (3.247%); 1.4 MiB/s, 153 objects/s recovering 2023-10-09T10:48:50.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:50 smithi125 ceph-mon[195507]: pgmap v416: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 761 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 25864/36555 objects degraded (70.754%); 1187/36555 objects misplaced (3.247%); 1.4 MiB/s, 153 objects/s recovering 2023-10-09T10:48:50.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:50 smithi125 ceph-mon[200197]: pgmap v416: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 761 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 25864/36555 objects degraded (70.754%); 1187/36555 objects misplaced (3.247%); 1.4 MiB/s, 153 objects/s recovering 2023-10-09T10:48:51.280 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:48:51.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:48:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:48:51] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2023-10-09T10:48:51.816 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:51.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:51.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:51.975 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:48:51.997 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:52.054 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:52.095 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:52.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:52.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:52.199 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:52.242 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:52.278 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:52.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:52.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:52.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:52 smithi125 ceph-mon[200197]: pgmap v417: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 619 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 25489/29778 objects degraded (85.597%); 937/29778 objects misplaced (3.147%); 1.4 MiB/s, 156 objects/s recovering 2023-10-09T10:48:52.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:52 smithi125 ceph-mon[195507]: pgmap v417: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 619 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 25489/29778 objects degraded (85.597%); 937/29778 objects misplaced (3.147%); 1.4 MiB/s, 156 objects/s recovering 2023-10-09T10:48:52.383 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:52.435 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:52.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:52 smithi155 ceph-mon[184854]: pgmap v417: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 619 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 25489/29778 objects degraded (85.597%); 937/29778 objects misplaced (3.147%); 1.4 MiB/s, 156 objects/s recovering 2023-10-09T10:48:52.502 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:52.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:52.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:52.658 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:52.715 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:52.750 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:52.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:52.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:52.858 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:52.900 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:52.945 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:52.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:52.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:53.058 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:53.119 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:53.149 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:53.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:53.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:53.256 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:53.258 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:53 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 25489/29778 objects degraded (85.597%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:53.276 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:53.334 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:53.339 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:53.386 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:53.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:53.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:53.428 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:48:53.428 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:48:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:53 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 25489/29778 objects degraded (85.597%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:53.528 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:53.529 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:53 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 25489/29778 objects degraded (85.597%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:53.572 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:53.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:53.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:53.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:53.807 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:53.857 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:53.898 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:53.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:53.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:54.005 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:54.051 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:54.091 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:54.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:54.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:54.198 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:54.283 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:54.308 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:48:54.342 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:54.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:54.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:54.446 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:54 smithi155 ceph-mon[184854]: pgmap v418: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 532 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 24927/25578 objects degraded (97.455%); 820/25578 objects misplaced (3.206%); 1.5 MiB/s, 176 objects/s recovering 2023-10-09T10:48:54.500 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:54.529 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:54 smithi125 ceph-mon[195507]: pgmap v418: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 532 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 24927/25578 objects degraded (97.455%); 820/25578 objects misplaced (3.206%); 1.5 MiB/s, 176 objects/s recovering 2023-10-09T10:48:54.529 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:54 smithi125 ceph-mon[200197]: pgmap v418: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 532 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 24927/25578 objects degraded (97.455%); 820/25578 objects misplaced (3.206%); 1.5 MiB/s, 176 objects/s recovering 2023-10-09T10:48:54.530 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:54.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:54.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:54.636 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:48:54.694 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:54.725 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:48:54.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:54.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:48:54.834 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:54.877 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:54.909 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:54.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:54.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:55.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:55.062 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:55.093 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:55.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:55.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:55.200 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:48:55.228 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:55.247 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:48:55.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:55.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:48:55.312 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:48:55.351 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:55.365 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:48:55.413 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:55.418 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:48:55.450 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:55 smithi125 ceph-mon[195507]: osdmap e188: 8 total, 8 up, 7 in 2023-10-09T10:48:55.450 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:55 smithi125 ceph-mon[195507]: failed to encode map e188 with expected crc 2023-10-09T10:48:55.451 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:55 smithi125 ceph-mon[195507]: failed to encode map e188 with expected crc 2023-10-09T10:48:55.451 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:55 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3439543058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:55.452 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:55 smithi125 ceph-mon[200197]: osdmap e188: 8 total, 8 up, 7 in 2023-10-09T10:48:55.452 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:55 smithi125 ceph-mon[200197]: failed to encode map e188 with expected crc 2023-10-09T10:48:55.452 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:55 smithi125 ceph-mon[200197]: failed to encode map e188 with expected crc 2023-10-09T10:48:55.452 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:55 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3439543058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:55.453 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:55.485 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:48:55.546 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:48:55.562 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:55.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:55.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:55.609 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:48:55.652 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:48:55.685 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:55.719 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:48:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:55 smithi155 ceph-mon[184854]: osdmap e188: 8 total, 8 up, 7 in 2023-10-09T10:48:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:55 smithi155 ceph-mon[184854]: failed to encode map e188 with expected crc 2023-10-09T10:48:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:55 smithi155 ceph-mon[184854]: failed to encode map e188 with expected crc 2023-10-09T10:48:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:55 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3439543058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:48:55.742 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:55.766 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:48:55.792 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:48:55.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:55.795 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:55.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:55.802 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:48:55.802 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:48:55.803 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:48:55.803 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:48:55.803 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:48:55.803 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:48:55.803 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:48:55.803 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:48:55.813 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:48:55.896 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:55.953 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:55.990 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:56.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:56.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:56.104 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:56.175 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:56.223 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:56.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:56.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:56.332 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:48:56.407 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:56.558 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[195507]: pgmap v420: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 433 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 24927/20835 objects degraded (119.640%); 713/20835 objects misplaced (3.422%); 1.7 MiB/s, 193 objects/s recovering 2023-10-09T10:48:56.558 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:56.558 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[195507]: failed to encode map e189 with expected crc 2023-10-09T10:48:56.558 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[195507]: failed to encode map e189 with expected crc 2023-10-09T10:48:56.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3439543058' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:48:56.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[195507]: osdmap e189: 8 total, 8 up, 7 in 2023-10-09T10:48:56.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[195507]: failed to encode map e189 with expected crc 2023-10-09T10:48:56.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/637991142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:56.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[195507]: failed to encode map e189 with expected crc 2023-10-09T10:48:56.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[200197]: pgmap v420: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 433 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 24927/20835 objects degraded (119.640%); 713/20835 objects misplaced (3.422%); 1.7 MiB/s, 193 objects/s recovering 2023-10-09T10:48:56.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:56.562 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[200197]: failed to encode map e189 with expected crc 2023-10-09T10:48:56.562 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[200197]: failed to encode map e189 with expected crc 2023-10-09T10:48:56.562 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3439543058' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:48:56.562 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[200197]: osdmap e189: 8 total, 8 up, 7 in 2023-10-09T10:48:56.562 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[200197]: failed to encode map e189 with expected crc 2023-10-09T10:48:56.562 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/637991142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:56.562 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:56 smithi125 ceph-mon[200197]: failed to encode map e189 with expected crc 2023-10-09T10:48:56.590 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:48:56.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:56.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:48:56.704 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:56 smithi155 ceph-mon[184854]: pgmap v420: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 433 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 24927/20835 objects degraded (119.640%); 713/20835 objects misplaced (3.422%); 1.7 MiB/s, 193 objects/s recovering 2023-10-09T10:48:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:56 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:48:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:56 smithi155 ceph-mon[184854]: failed to encode map e189 with expected crc 2023-10-09T10:48:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:56 smithi155 ceph-mon[184854]: failed to encode map e189 with expected crc 2023-10-09T10:48:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3439543058' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:48:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:56 smithi155 ceph-mon[184854]: osdmap e189: 8 total, 8 up, 7 in 2023-10-09T10:48:56.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:56 smithi155 ceph-mon[184854]: failed to encode map e189 with expected crc 2023-10-09T10:48:56.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/637991142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:48:56.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:56 smithi155 ceph-mon[184854]: failed to encode map e189 with expected crc 2023-10-09T10:48:56.764 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:56.896 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":421,"stamp":"2023-10-09T10:48:55.298737+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":454072248,"num_objects":6945,"num_object_clones":0,"num_object_copies":20835,"num_objects_missing_on_primary":219,"num_objects_missing":219,"num_objects_degraded":24927,"num_objects_misplaced":713,"num_objects_unfound":0,"num_objects_dirty":6945,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":197587,"num_write_kb":6260974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30572,"num_bytes_recovered":1612843778,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173525,"ondisk_log_size":173525,"up":357,"acting":348,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3397844,"kb_used_data":1674460,"kb_used_omap":44,"kb_used_meta":1722963,"kb_avail":652818220,"statfs":{"total":671965249536,"available":668485857280,"internally_reserved":0,"allocated":1714647040,"data_stored":1708932255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1764314529},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-356646912,"num_objects":-5442,"num_object_clones":0,"num_object_copies":-16326,"num_objects_missing_on_primary":-248,"num_objects_missing":-248,"num_objects_degraded":-1424,"num_objects_misplaced":-474,"num_objects_unfound":0,"num_objects_dirty":-5442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5442,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1424,"num_bytes_recovered":16252928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.126737"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":180,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.737448+0000","last_change":"2023-10-09T10:46:32.250979+0000","last_active":"2023-10-09T10:48:38.737448+0000","last_peered":"2023-10-09T10:48:38.737448+0000","last_clean":"2023-10-09T10:48:38.737448+0000","last_became_active":"2023-10-09T10:46:32.250625+0000","last_became_peered":"2023-10-09T10:46:32.250625+0000","last_unstale":"2023-10-09T10:48:38.737448+0000","last_undegraded":"2023-10-09T10:48:38.737448+0000","last_fullsized":"2023-10-09T10:48:38.737448+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:34.858348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":157,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.057367+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:48:46.057367+0000","last_peered":"2023-10-09T10:48:46.057367+0000","last_clean":"2023-10-09T10:48:46.057367+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:48:46.057367+0000","last_undegraded":"2023-10-09T10:48:46.057367+0000","last_fullsized":"2023-10-09T10:48:46.057367+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":139,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.736565+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:48:38.736565+0000","last_peered":"2023-10-09T10:48:38.736565+0000","last_clean":"2023-10-09T10:48:38.736565+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:48:38.736565+0000","last_undegraded":"2023-10-09T10:48:38.736565+0000","last_fullsized":"2023-10-09T10:48:38.736565+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":56,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.713228+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:48:38.713228+0000","last_peered":"2023-10-09T10:48:38.713228+0000","last_clean":"2023-10-09T10:48:38.713228+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:48:38.713228+0000","last_undegraded":"2023-10-09T10:48:38.713228+0000","last_fullsized":"2023-10-09T10:48:38.713228+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":193,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.056902+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:48:46.056902+0000","last_peered":"2023-10-09T10:48:46.056902+0000","last_clean":"2023-10-09T10:48:46.056902+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:48:46.056902+0000","last_undegraded":"2023-10-09T10:48:46.056902+0000","last_fullsized":"2023-10-09T10:48:46.056902+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":125,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723319+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:48:38.723319+0000","last_peered":"2023-10-09T10:48:38.723319+0000","last_clean":"2023-10-09T10:48:38.723319+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:48:38.723319+0000","last_undegraded":"2023-10-09T10:48:38.723319+0000","last_fullsized":"2023-10-09T10:48:38.723319+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"187'7621","reported_seq":11782,"reported_epoch":187,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:51.055560+0000","last_change":"2023-10-09T10:46:33.521602+0000","last_active":"2023-10-09T10:48:51.055560+0000","last_peered":"2023-10-09T10:48:51.055560+0000","last_clean":"2023-10-09T10:46:31.055751+0000","last_became_active":"2023-10-09T10:46:33.364058+0000","last_became_peered":"2023-10-09T10:46:33.364058+0000","last_unstale":"2023-10-09T10:48:51.055560+0000","last_undegraded":"2023-10-09T10:48:51.055560+0000","last_fullsized":"2023-10-09T10:48:51.055560+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7621,"log_dups_size":0,"ondisk_log_size":7621,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17858560,"num_objects":273,"num_object_clones":0,"num_object_copies":819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":273,"num_objects_unfound":0,"num_objects_dirty":273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6585,"num_write_kb":219424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2010,"num_bytes_recovered":131727360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":273}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":56,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.713940+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:48:38.713940+0000","last_peered":"2023-10-09T10:48:38.713940+0000","last_clean":"2023-10-09T10:48:38.713940+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:48:38.713940+0000","last_undegraded":"2023-10-09T10:48:38.713940+0000","last_fullsized":"2023-10-09T10:48:38.713940+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":157,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.056969+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:48:46.056969+0000","last_peered":"2023-10-09T10:48:46.056969+0000","last_clean":"2023-10-09T10:48:46.056969+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:48:46.056969+0000","last_undegraded":"2023-10-09T10:48:46.056969+0000","last_fullsized":"2023-10-09T10:48:46.056969+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":521,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.712688+0000","last_change":"2023-10-09T10:46:05.620365+0000","last_active":"2023-10-09T10:48:38.712688+0000","last_peered":"2023-10-09T10:48:38.712688+0000","last_clean":"2023-10-09T10:48:38.712688+0000","last_became_active":"2023-10-09T10:46:05.620024+0000","last_became_peered":"2023-10-09T10:46:05.620024+0000","last_unstale":"2023-10-09T10:48:38.712688+0000","last_undegraded":"2023-10-09T10:48:38.712688+0000","last_fullsized":"2023-10-09T10:48:38.712688+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.13","version":"187'7398","reported_seq":12118,"reported_epoch":187,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:51.056055+0000","last_change":"2023-10-09T10:48:15.511525+0000","last_active":"2023-10-09T10:48:51.056055+0000","last_peered":"2023-10-09T10:48:51.056055+0000","last_clean":"2023-10-09T10:46:31.195699+0000","last_became_active":"2023-10-09T10:46:33.455632+0000","last_became_peered":"2023-10-09T10:46:33.455632+0000","last_unstale":"2023-10-09T10:48:51.056055+0000","last_undegraded":"2023-10-09T10:46:33.269323+0000","last_fullsized":"2023-10-09T10:46:33.241226+0000","mapping_epoch":154,"log_start":"114'2675","ondisk_log_start":"114'2675","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":4723,"log_dups_size":0,"ondisk_log_size":4723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:07:12.547333+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17956864,"num_objects":274,"num_object_clones":0,"num_object_copies":822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1052,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4683,"num_write_kb":105088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2263,"num_bytes_recovered":66060288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"6,7","objects":-778}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":152,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.732002+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:48:38.732002+0000","last_peered":"2023-10-09T10:48:38.732002+0000","last_clean":"2023-10-09T10:48:38.732002+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:48:38.732002+0000","last_undegraded":"2023-10-09T10:48:38.732002+0000","last_fullsized":"2023-10-09T10:48:38.732002+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":114,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.073519+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:48:46.073519+0000","last_peered":"2023-10-09T10:48:46.073519+0000","last_clean":"2023-10-09T10:48:46.073519+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:48:46.073519+0000","last_undegraded":"2023-10-09T10:48:46.073519+0000","last_fullsized":"2023-10-09T10:48:46.073519+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"187'7602","reported_seq":15737,"reported_epoch":187,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:51.401308+0000","last_change":"2023-10-09T10:46:03.683196+0000","last_active":"2023-10-09T10:48:51.401308+0000","last_peered":"2023-10-09T10:48:51.401308+0000","last_clean":"2023-10-09T10:46:02.177309+0000","last_became_active":"2023-10-09T10:46:03.681058+0000","last_became_peered":"2023-10-09T10:46:03.681058+0000","last_unstale":"2023-10-09T10:48:51.401308+0000","last_undegraded":"2023-10-09T10:48:51.401308+0000","last_fullsized":"2023-10-09T10:48:51.401308+0000","mapping_epoch":143,"log_start":"103'376","ondisk_log_start":"103'376","created":107,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7226,"log_dups_size":0,"ondisk_log_size":7226,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15171584,"num_objects":232,"num_object_clones":0,"num_object_copies":696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":232,"num_objects_unfound":0,"num_objects_dirty":232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6556,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[1,5,6],"avail_no_missing":["1","2","3","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":232}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":109,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.057360+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:48:46.057360+0000","last_peered":"2023-10-09T10:48:46.057360+0000","last_clean":"2023-10-09T10:48:46.057360+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:48:46.057360+0000","last_undegraded":"2023-10-09T10:48:46.057360+0000","last_fullsized":"2023-10-09T10:48:46.057360+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":148,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.739447+0000","last_change":"2023-10-09T10:46:32.249921+0000","last_active":"2023-10-09T10:48:38.739447+0000","last_peered":"2023-10-09T10:48:38.739447+0000","last_clean":"2023-10-09T10:48:38.739447+0000","last_became_active":"2023-10-09T10:46:32.249590+0000","last_became_peered":"2023-10-09T10:46:32.249590+0000","last_unstale":"2023-10-09T10:48:38.739447+0000","last_undegraded":"2023-10-09T10:48:38.739447+0000","last_fullsized":"2023-10-09T10:48:38.739447+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:25.169752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"188'7430","reported_seq":15662,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.579662+0000","last_change":"2023-10-09T10:47:48.412928+0000","last_active":"2023-10-09T10:48:54.579662+0000","last_peered":"2023-10-09T10:48:54.579662+0000","last_clean":"2023-10-09T10:48:54.579662+0000","last_became_active":"2023-10-09T10:47:48.398813+0000","last_became_peered":"2023-10-09T10:47:48.398813+0000","last_unstale":"2023-10-09T10:48:54.579662+0000","last_undegraded":"2023-10-09T10:48:54.579662+0000","last_fullsized":"2023-10-09T10:48:54.579662+0000","mapping_epoch":177,"log_start":"104'1513","ondisk_log_start":"104'1513","created":107,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5917,"log_dups_size":0,"ondisk_log_size":5917,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9863168,"num_objects":151,"num_object_clones":0,"num_object_copies":453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":151,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6423,"num_write_kb":210336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2691,"num_bytes_recovered":176357376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":157,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.734505+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:48:38.734505+0000","last_peered":"2023-10-09T10:48:38.734505+0000","last_clean":"2023-10-09T10:48:38.734505+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:48:38.734505+0000","last_undegraded":"2023-10-09T10:48:38.734505+0000","last_fullsized":"2023-10-09T10:48:38.734505+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.733906+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:48:38.733906+0000","last_peered":"2023-10-09T10:48:38.733906+0000","last_clean":"2023-10-09T10:48:38.733906+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:48:38.733906+0000","last_undegraded":"2023-10-09T10:48:38.733906+0000","last_fullsized":"2023-10-09T10:48:38.733906+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":521,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.715604+0000","last_change":"2023-10-09T10:46:05.618276+0000","last_active":"2023-10-09T10:48:38.715604+0000","last_peered":"2023-10-09T10:48:38.715604+0000","last_clean":"2023-10-09T10:48:38.715604+0000","last_became_active":"2023-10-09T10:46:05.616509+0000","last_became_peered":"2023-10-09T10:46:05.616509+0000","last_unstale":"2023-10-09T10:48:38.715604+0000","last_undegraded":"2023-10-09T10:48:38.715604+0000","last_fullsized":"2023-10-09T10:48:38.715604+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.16","version":"188'7443","reported_seq":7556,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.526127+0000","last_change":"2023-10-09T10:46:03.592682+0000","last_active":"2023-10-09T10:48:54.526127+0000","last_peered":"2023-10-09T10:48:54.526127+0000","last_clean":"2023-10-09T10:48:54.526127+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:48:54.526127+0000","last_undegraded":"2023-10-09T10:48:54.526127+0000","last_fullsized":"2023-10-09T10:48:54.526127+0000","mapping_epoch":107,"log_start":"104'1455","ondisk_log_start":"104'1455","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5988,"log_dups_size":0,"ondisk_log_size":5988,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12189696,"num_objects":186,"num_object_clones":0,"num_object_copies":558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6400,"num_write_kb":210752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":169,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.736433+0000","last_change":"2023-10-09T10:46:32.250256+0000","last_active":"2023-10-09T10:48:38.736433+0000","last_peered":"2023-10-09T10:48:38.736433+0000","last_clean":"2023-10-09T10:48:38.736433+0000","last_became_active":"2023-10-09T10:46:32.243312+0000","last_became_peered":"2023-10-09T10:46:32.243312+0000","last_unstale":"2023-10-09T10:48:38.736433+0000","last_undegraded":"2023-10-09T10:48:38.736433+0000","last_fullsized":"2023-10-09T10:48:38.736433+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:08.059777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":180,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.060403+0000","last_change":"2023-10-09T10:46:32.237577+0000","last_active":"2023-10-09T10:48:46.060403+0000","last_peered":"2023-10-09T10:48:46.060403+0000","last_clean":"2023-10-09T10:48:46.060403+0000","last_became_active":"2023-10-09T10:46:32.237316+0000","last_became_peered":"2023-10-09T10:46:32.237316+0000","last_unstale":"2023-10-09T10:48:46.060403+0000","last_undegraded":"2023-10-09T10:48:46.060403+0000","last_fullsized":"2023-10-09T10:48:46.060403+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:39.684602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":522,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.710133+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:48:38.710133+0000","last_peered":"2023-10-09T10:48:38.710133+0000","last_clean":"2023-10-09T10:48:38.710133+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:48:38.710133+0000","last_undegraded":"2023-10-09T10:48:38.710133+0000","last_fullsized":"2023-10-09T10:48:38.710133+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.17","version":"187'7298","reported_seq":7404,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:51.109782+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:48:51.109782+0000","last_peered":"2023-10-09T10:48:51.109782+0000","last_clean":"2023-10-09T10:48:51.109782+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:48:51.109782+0000","last_undegraded":"2023-10-09T10:48:51.109782+0000","last_fullsized":"2023-10-09T10:48:51.109782+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7298,"log_dups_size":0,"ondisk_log_size":7298,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14155776,"num_objects":216,"num_object_clones":0,"num_object_copies":648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6290,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":56,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.711864+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:48:38.711864+0000","last_peered":"2023-10-09T10:48:38.711864+0000","last_clean":"2023-10-09T10:48:38.711864+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:48:38.711864+0000","last_undegraded":"2023-10-09T10:48:38.711864+0000","last_fullsized":"2023-10-09T10:48:38.711864+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723345+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:48:38.723345+0000","last_peered":"2023-10-09T10:48:38.723345+0000","last_clean":"2023-10-09T10:48:38.723345+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:48:38.723345+0000","last_undegraded":"2023-10-09T10:48:38.723345+0000","last_fullsized":"2023-10-09T10:48:38.723345+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":522,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.711999+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:48:38.711999+0000","last_peered":"2023-10-09T10:48:38.711999+0000","last_clean":"2023-10-09T10:48:38.711999+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:48:38.711999+0000","last_undegraded":"2023-10-09T10:48:38.711999+0000","last_fullsized":"2023-10-09T10:48:38.711999+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"187'7524","reported_seq":7635,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:51.496491+0000","last_change":"2023-10-09T10:46:33.243540+0000","last_active":"2023-10-09T10:48:51.496491+0000","last_peered":"2023-10-09T10:48:51.496491+0000","last_clean":"2023-10-09T10:48:51.496491+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:48:51.496491+0000","last_undegraded":"2023-10-09T10:48:51.496491+0000","last_fullsized":"2023-10-09T10:48:51.496491+0000","mapping_epoch":107,"log_start":"103'234","ondisk_log_start":"103'234","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7290,"log_dups_size":0,"ondisk_log_size":7290,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13336576,"num_objects":204,"num_object_clones":0,"num_object_copies":612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6470,"num_write_kb":213536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":97,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.733887+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:48:38.733887+0000","last_peered":"2023-10-09T10:48:38.733887+0000","last_clean":"2023-10-09T10:48:38.733887+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:48:38.733887+0000","last_undegraded":"2023-10-09T10:48:38.733887+0000","last_fullsized":"2023-10-09T10:48:38.733887+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":56,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.713459+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:48:38.713459+0000","last_peered":"2023-10-09T10:48:38.713459+0000","last_clean":"2023-10-09T10:48:38.713459+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:48:38.713459+0000","last_undegraded":"2023-10-09T10:48:38.713459+0000","last_fullsized":"2023-10-09T10:48:38.713459+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":522,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.711666+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:48:38.711666+0000","last_peered":"2023-10-09T10:48:38.711666+0000","last_clean":"2023-10-09T10:48:38.711666+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:48:38.711666+0000","last_undegraded":"2023-10-09T10:48:38.711666+0000","last_fullsized":"2023-10-09T10:48:38.711666+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"188'7605","reported_seq":7727,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.576645+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:48:54.576645+0000","last_peered":"2023-10-09T10:48:54.576645+0000","last_clean":"2023-10-09T10:48:54.576645+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:48:54.576645+0000","last_undegraded":"2023-10-09T10:48:54.576645+0000","last_fullsized":"2023-10-09T10:48:54.576645+0000","mapping_epoch":115,"log_start":"105'1768","ondisk_log_start":"105'1768","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5837,"log_dups_size":0,"ondisk_log_size":5837,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8126464,"num_objects":124,"num_object_clones":0,"num_object_copies":372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6544,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":66,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.710048+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:48:38.710048+0000","last_peered":"2023-10-09T10:48:38.710048+0000","last_clean":"2023-10-09T10:48:38.710048+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:48:38.710048+0000","last_undegraded":"2023-10-09T10:48:38.710048+0000","last_fullsized":"2023-10-09T10:48:38.710048+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":157,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.058039+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:48:46.058039+0000","last_peered":"2023-10-09T10:48:46.058039+0000","last_clean":"2023-10-09T10:48:46.058039+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:48:46.058039+0000","last_undegraded":"2023-10-09T10:48:46.058039+0000","last_fullsized":"2023-10-09T10:48:46.058039+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":522,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.713008+0000","last_change":"2023-10-09T10:46:05.620574+0000","last_active":"2023-10-09T10:48:38.713008+0000","last_peered":"2023-10-09T10:48:38.713008+0000","last_clean":"2023-10-09T10:48:38.713008+0000","last_became_active":"2023-10-09T10:46:05.620212+0000","last_became_peered":"2023-10-09T10:46:05.620212+0000","last_unstale":"2023-10-09T10:48:38.713008+0000","last_undegraded":"2023-10-09T10:48:38.713008+0000","last_fullsized":"2023-10-09T10:48:38.713008+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"187'13150","reported_seq":17312,"reported_epoch":187,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:51.146776+0000","last_change":"2023-10-09T10:46:32.247903+0000","last_active":"2023-10-09T10:48:51.146776+0000","last_peered":"2023-10-09T10:48:51.146776+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:46:02.739355+0000","last_became_peered":"2023-10-09T10:46:02.739355+0000","last_unstale":"2023-10-09T10:48:51.146776+0000","last_undegraded":"2023-10-09T10:46:02.608805+0000","last_fullsized":"2023-10-09T10:46:02.583589+0000","mapping_epoch":142,"log_start":"124'4300","ondisk_log_start":"124'4300","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8850,"log_dups_size":0,"ondisk_log_size":8850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:32.899634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31588352,"num_objects":482,"num_object_clones":0,"num_object_copies":1446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13150,"num_write_kb":436224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2022,"num_bytes_recovered":132513792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":479},{"shards":"5,6","objects":-144}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":180,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.737577+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:48:38.737577+0000","last_peered":"2023-10-09T10:48:38.737577+0000","last_clean":"2023-10-09T10:48:38.737577+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:48:38.737577+0000","last_undegraded":"2023-10-09T10:48:38.737577+0000","last_fullsized":"2023-10-09T10:48:38.737577+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":134,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723111+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:48:38.723111+0000","last_peered":"2023-10-09T10:48:38.723111+0000","last_clean":"2023-10-09T10:48:38.723111+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:48:38.723111+0000","last_undegraded":"2023-10-09T10:48:38.723111+0000","last_fullsized":"2023-10-09T10:48:38.723111+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":502,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.712808+0000","last_change":"2023-10-09T10:46:02.631378+0000","last_active":"2023-10-09T10:48:38.712808+0000","last_peered":"2023-10-09T10:48:38.712808+0000","last_clean":"2023-10-09T10:48:38.712808+0000","last_became_active":"2023-10-09T10:46:02.629142+0000","last_became_peered":"2023-10-09T10:46:02.629142+0000","last_unstale":"2023-10-09T10:48:38.712808+0000","last_undegraded":"2023-10-09T10:48:38.712808+0000","last_fullsized":"2023-10-09T10:48:38.712808+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.b","version":"188'12833","reported_seq":12933,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.639049+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:48:54.639049+0000","last_peered":"2023-10-09T10:48:54.639049+0000","last_clean":"2023-10-09T10:48:54.639049+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:48:54.639049+0000","last_undegraded":"2023-10-09T10:48:54.639049+0000","last_fullsized":"2023-10-09T10:48:54.639049+0000","mapping_epoch":98,"log_start":"150'7125","ondisk_log_start":"150'7125","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5708,"log_dups_size":0,"ondisk_log_size":5708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22740992,"num_objects":347,"num_object_clones":0,"num_object_copies":1041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12833,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":153,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.737131+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:48:38.737131+0000","last_peered":"2023-10-09T10:48:38.737131+0000","last_clean":"2023-10-09T10:48:38.737131+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:48:38.737131+0000","last_undegraded":"2023-10-09T10:48:38.737131+0000","last_fullsized":"2023-10-09T10:48:38.737131+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":188,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.733815+0000","last_change":"2023-10-09T10:46:32.251035+0000","last_active":"2023-10-09T10:48:38.733815+0000","last_peered":"2023-10-09T10:48:38.733815+0000","last_clean":"2023-10-09T10:48:38.733815+0000","last_became_active":"2023-10-09T10:46:32.250293+0000","last_became_peered":"2023-10-09T10:46:32.250293+0000","last_unstale":"2023-10-09T10:48:38.733815+0000","last_undegraded":"2023-10-09T10:48:38.733815+0000","last_fullsized":"2023-10-09T10:48:38.733815+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.787766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":522,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.713652+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:48:38.713652+0000","last_peered":"2023-10-09T10:48:38.713652+0000","last_clean":"2023-10-09T10:48:38.713652+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:48:38.713652+0000","last_undegraded":"2023-10-09T10:48:38.713652+0000","last_fullsized":"2023-10-09T10:48:38.713652+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"188'7555","reported_seq":7676,"reported_epoch":188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:54.442888+0000","last_change":"2023-10-09T10:46:33.540719+0000","last_active":"2023-10-09T10:48:54.442888+0000","last_peered":"2023-10-09T10:48:54.442888+0000","last_clean":"2023-10-09T10:46:31.137696+0000","last_became_active":"2023-10-09T10:46:33.527650+0000","last_became_peered":"2023-10-09T10:46:33.527650+0000","last_unstale":"2023-10-09T10:48:54.442888+0000","last_undegraded":"2023-10-09T10:46:33.285468+0000","last_fullsized":"2023-10-09T10:46:33.235683+0000","mapping_epoch":154,"log_start":"105'1635","ondisk_log_start":"105'1635","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5920,"log_dups_size":0,"ondisk_log_size":5920,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12877824,"num_objects":196,"num_object_clones":0,"num_object_copies":588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6490,"num_write_kb":213984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":184},{"shards":"3,7","objects":-2613}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":108,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723223+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:48:38.723223+0000","last_peered":"2023-10-09T10:48:38.723223+0000","last_clean":"2023-10-09T10:48:38.723223+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:48:38.723223+0000","last_undegraded":"2023-10-09T10:48:38.723223+0000","last_fullsized":"2023-10-09T10:48:38.723223+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":125,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.739705+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:48:38.739705+0000","last_peered":"2023-10-09T10:48:38.739705+0000","last_clean":"2023-10-09T10:48:38.739705+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:48:38.739705+0000","last_undegraded":"2023-10-09T10:48:38.739705+0000","last_fullsized":"2023-10-09T10:48:38.739705+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":522,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.713047+0000","last_change":"2023-10-09T10:46:05.620030+0000","last_active":"2023-10-09T10:48:38.713047+0000","last_peered":"2023-10-09T10:48:38.713047+0000","last_clean":"2023-10-09T10:48:38.713047+0000","last_became_active":"2023-10-09T10:46:05.619694+0000","last_became_peered":"2023-10-09T10:46:05.619694+0000","last_unstale":"2023-10-09T10:48:38.713047+0000","last_undegraded":"2023-10-09T10:48:38.713047+0000","last_fullsized":"2023-10-09T10:48:38.713047+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.9","version":"187'7429","reported_seq":11434,"reported_epoch":187,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:52.826565+0000","last_change":"2023-10-09T10:48:15.517098+0000","last_active":"2023-10-09T10:48:52.826565+0000","last_peered":"2023-10-09T10:48:52.826565+0000","last_clean":"2023-10-09T10:46:31.190603+0000","last_became_active":"2023-10-09T10:46:34.396448+0000","last_became_peered":"2023-10-09T10:46:34.396448+0000","last_unstale":"2023-10-09T10:48:52.826565+0000","last_undegraded":"2023-10-09T10:46:33.246949+0000","last_fullsized":"2023-10-09T10:46:33.237968+0000","mapping_epoch":154,"log_start":"108'2203","ondisk_log_start":"108'2203","created":98,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5226,"log_dups_size":0,"ondisk_log_size":5226,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14876672,"num_objects":227,"num_object_clones":0,"num_object_copies":681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":67,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6387,"num_write_kb":211648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3240,"num_bytes_recovered":85458944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,7","objects":160}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":152,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723838+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:48:38.723838+0000","last_peered":"2023-10-09T10:48:38.723838+0000","last_clean":"2023-10-09T10:48:38.723838+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:48:38.723838+0000","last_undegraded":"2023-10-09T10:48:38.723838+0000","last_fullsized":"2023-10-09T10:48:38.723838+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":56,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.712415+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:48:38.712415+0000","last_peered":"2023-10-09T10:48:38.712415+0000","last_clean":"2023-10-09T10:48:38.712415+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:48:38.712415+0000","last_undegraded":"2023-10-09T10:48:38.712415+0000","last_fullsized":"2023-10-09T10:48:38.712415+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":513,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.715458+0000","last_change":"2023-10-09T10:46:05.618134+0000","last_active":"2023-10-09T10:48:38.715458+0000","last_peered":"2023-10-09T10:48:38.715458+0000","last_clean":"2023-10-09T10:48:38.715458+0000","last_became_active":"2023-10-09T10:46:05.616057+0000","last_became_peered":"2023-10-09T10:46:05.616057+0000","last_unstale":"2023-10-09T10:48:38.715458+0000","last_undegraded":"2023-10-09T10:48:38.715458+0000","last_fullsized":"2023-10-09T10:48:38.715458+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"187'12728","reported_seq":12834,"reported_epoch":187,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:51.088832+0000","last_change":"2023-10-09T10:46:33.576530+0000","last_active":"2023-10-09T10:48:51.088832+0000","last_peered":"2023-10-09T10:48:51.088832+0000","last_clean":"2023-10-09T10:46:31.183481+0000","last_became_active":"2023-10-09T10:46:33.542304+0000","last_became_peered":"2023-10-09T10:46:33.542304+0000","last_unstale":"2023-10-09T10:48:51.088832+0000","last_undegraded":"2023-10-09T10:46:33.300581+0000","last_fullsized":"2023-10-09T10:46:33.246047+0000","mapping_epoch":154,"log_start":"134'5377","ondisk_log_start":"134'5377","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7351,"log_dups_size":0,"ondisk_log_size":7351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:26.739488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33161216,"num_objects":506,"num_object_clones":0,"num_object_copies":1518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6617,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12728,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":506},{"shards":"1,4","objects":-5055}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":181,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.061487+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:48:46.061487+0000","last_peered":"2023-10-09T10:48:46.061487+0000","last_clean":"2023-10-09T10:48:46.061487+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:48:46.061487+0000","last_undegraded":"2023-10-09T10:48:46.061487+0000","last_fullsized":"2023-10-09T10:48:46.061487+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.734398+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:48:38.734398+0000","last_peered":"2023-10-09T10:48:38.734398+0000","last_clean":"2023-10-09T10:48:38.734398+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:48:38.734398+0000","last_undegraded":"2023-10-09T10:48:38.734398+0000","last_fullsized":"2023-10-09T10:48:38.734398+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":502,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.710194+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:48:38.710194+0000","last_peered":"2023-10-09T10:48:38.710194+0000","last_clean":"2023-10-09T10:48:38.710194+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:48:38.710194+0000","last_undegraded":"2023-10-09T10:48:38.710194+0000","last_fullsized":"2023-10-09T10:48:38.710194+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.f","version":"188'12668","reported_seq":12775,"reported_epoch":188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:54.595056+0000","last_change":"2023-10-09T10:46:33.651756+0000","last_active":"2023-10-09T10:48:54.595056+0000","last_peered":"2023-10-09T10:48:54.595056+0000","last_clean":"2023-10-09T10:46:31.156458+0000","last_became_active":"2023-10-09T10:46:33.621003+0000","last_became_peered":"2023-10-09T10:46:33.621003+0000","last_unstale":"2023-10-09T10:48:54.595056+0000","last_undegraded":"2023-10-09T10:46:33.309813+0000","last_fullsized":"2023-10-09T10:46:33.236508+0000","mapping_epoch":154,"log_start":"149'6910","ondisk_log_start":"149'6910","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5758,"log_dups_size":0,"ondisk_log_size":5758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:13.704346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23199744,"num_objects":354,"num_object_clones":0,"num_object_copies":1062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12668,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":354},{"shards":"3,4","objects":-5098}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":126,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.737597+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:48:38.737597+0000","last_peered":"2023-10-09T10:48:38.737597+0000","last_clean":"2023-10-09T10:48:38.737597+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:48:38.737597+0000","last_undegraded":"2023-10-09T10:48:38.737597+0000","last_fullsized":"2023-10-09T10:48:38.737597+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":178,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.734524+0000","last_change":"2023-10-09T10:46:32.245583+0000","last_active":"2023-10-09T10:48:38.734524+0000","last_peered":"2023-10-09T10:48:38.734524+0000","last_clean":"2023-10-09T10:48:38.734524+0000","last_became_active":"2023-10-09T10:46:32.245271+0000","last_became_peered":"2023-10-09T10:46:32.245271+0000","last_unstale":"2023-10-09T10:48:38.734524+0000","last_undegraded":"2023-10-09T10:48:38.734524+0000","last_fullsized":"2023-10-09T10:48:38.734524+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.025051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":522,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.712163+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:48:38.712163+0000","last_peered":"2023-10-09T10:48:38.712163+0000","last_clean":"2023-10-09T10:48:38.712163+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:48:38.712163+0000","last_undegraded":"2023-10-09T10:48:38.712163+0000","last_fullsized":"2023-10-09T10:48:38.712163+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"188'12930","reported_seq":13030,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.633335+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:48:54.633335+0000","last_peered":"2023-10-09T10:48:54.633335+0000","last_clean":"2023-10-09T10:48:54.633335+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:48:54.633335+0000","last_undegraded":"2023-10-09T10:48:54.633335+0000","last_fullsized":"2023-10-09T10:48:54.633335+0000","mapping_epoch":98,"log_start":"150'7178","ondisk_log_start":"150'7178","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5752,"log_dups_size":0,"ondisk_log_size":5752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23199744,"num_objects":354,"num_object_clones":0,"num_object_copies":1062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12930,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":162,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.733486+0000","last_change":"2023-10-09T10:46:32.245685+0000","last_active":"2023-10-09T10:48:38.733486+0000","last_peered":"2023-10-09T10:48:38.733486+0000","last_clean":"2023-10-09T10:48:38.733486+0000","last_became_active":"2023-10-09T10:46:32.245314+0000","last_became_peered":"2023-10-09T10:46:32.245314+0000","last_unstale":"2023-10-09T10:48:38.733486+0000","last_undegraded":"2023-10-09T10:48:38.733486+0000","last_fullsized":"2023-10-09T10:48:38.733486+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:51.716860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.735061+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:48:38.735061+0000","last_peered":"2023-10-09T10:48:38.735061+0000","last_clean":"2023-10-09T10:48:38.735061+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:48:38.735061+0000","last_undegraded":"2023-10-09T10:48:38.735061+0000","last_fullsized":"2023-10-09T10:48:38.735061+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":513,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.711707+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:48:38.711707+0000","last_peered":"2023-10-09T10:48:38.711707+0000","last_clean":"2023-10-09T10:48:38.711707+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:48:38.711707+0000","last_undegraded":"2023-10-09T10:48:38.711707+0000","last_fullsized":"2023-10-09T10:48:38.711707+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"187'12886","reported_seq":13003,"reported_epoch":187,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:51.126702+0000","last_change":"2023-10-09T10:46:02.754507+0000","last_active":"2023-10-09T10:48:51.126702+0000","last_peered":"2023-10-09T10:48:51.126702+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:46:02.747510+0000","last_became_peered":"2023-10-09T10:46:02.747510+0000","last_unstale":"2023-10-09T10:48:51.126702+0000","last_undegraded":"2023-10-09T10:46:02.605987+0000","last_fullsized":"2023-10-09T10:46:02.585248+0000","mapping_epoch":142,"log_start":"123'4000","ondisk_log_start":"123'4000","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8886,"log_dups_size":0,"ondisk_log_size":8886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:59.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33357848,"num_objects":510,"num_object_clones":0,"num_object_copies":1530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":510,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12886,"num_write_kb":428609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,2,5","objects":509},{"shards":"1,5","objects":-2208}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":158,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.056764+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:48:46.056764+0000","last_peered":"2023-10-09T10:48:46.056764+0000","last_clean":"2023-10-09T10:48:46.056764+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:48:46.056764+0000","last_undegraded":"2023-10-09T10:48:46.056764+0000","last_fullsized":"2023-10-09T10:48:46.056764+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":139,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.744170+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:48:38.744170+0000","last_peered":"2023-10-09T10:48:38.744170+0000","last_clean":"2023-10-09T10:48:38.744170+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:48:38.744170+0000","last_undegraded":"2023-10-09T10:48:38.744170+0000","last_fullsized":"2023-10-09T10:48:38.744170+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":522,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.713809+0000","last_change":"2023-10-09T10:46:05.617983+0000","last_active":"2023-10-09T10:48:38.713809+0000","last_peered":"2023-10-09T10:48:38.713809+0000","last_clean":"2023-10-09T10:48:38.713809+0000","last_became_active":"2023-10-09T10:46:05.616021+0000","last_became_peered":"2023-10-09T10:46:05.616021+0000","last_unstale":"2023-10-09T10:48:38.713809+0000","last_undegraded":"2023-10-09T10:48:38.713809+0000","last_fullsized":"2023-10-09T10:48:38.713809+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":183,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.045571+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:48:46.045571+0000","last_peered":"2023-10-09T10:48:46.045571+0000","last_clean":"2023-10-09T10:48:46.045571+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:48:46.045571+0000","last_undegraded":"2023-10-09T10:48:46.045571+0000","last_fullsized":"2023-10-09T10:48:46.045571+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":179,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723418+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:48:38.723418+0000","last_peered":"2023-10-09T10:48:38.723418+0000","last_clean":"2023-10-09T10:48:38.723418+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:48:38.723418+0000","last_undegraded":"2023-10-09T10:48:38.723418+0000","last_fullsized":"2023-10-09T10:48:38.723418+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":125,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.043194+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:48:46.043194+0000","last_peered":"2023-10-09T10:48:46.043194+0000","last_clean":"2023-10-09T10:48:46.043194+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:48:46.043194+0000","last_undegraded":"2023-10-09T10:48:46.043194+0000","last_fullsized":"2023-10-09T10:48:46.043194+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"188'7483","reported_seq":11528,"reported_epoch":188,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:54.629593+0000","last_change":"2023-10-09T10:46:32.248218+0000","last_active":"2023-10-09T10:48:54.629593+0000","last_peered":"2023-10-09T10:48:54.629593+0000","last_clean":"2023-10-09T10:46:02.181912+0000","last_became_active":"2023-10-09T10:46:32.226237+0000","last_became_peered":"2023-10-09T10:46:32.226237+0000","last_unstale":"2023-10-09T10:48:54.629593+0000","last_undegraded":"2023-10-09T10:48:54.629593+0000","last_fullsized":"2023-10-09T10:48:54.629593+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7483,"log_dups_size":0,"ondisk_log_size":7483,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13664256,"num_objects":208,"num_object_clones":0,"num_object_copies":624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":208,"num_objects_unfound":0,"num_objects_dirty":208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6436,"num_write_kb":212640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":208}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":521,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.711517+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:48:38.711517+0000","last_peered":"2023-10-09T10:48:38.711517+0000","last_clean":"2023-10-09T10:48:38.711517+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:48:38.711517+0000","last_undegraded":"2023-10-09T10:48:38.711517+0000","last_fullsized":"2023-10-09T10:48:38.711517+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"188'7622","reported_seq":7813,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.578815+0000","last_change":"2023-10-09T10:46:02.760963+0000","last_active":"2023-10-09T10:48:54.578815+0000","last_peered":"2023-10-09T10:48:54.578815+0000","last_clean":"2023-10-09T10:48:54.578815+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:48:54.578815+0000","last_undegraded":"2023-10-09T10:48:54.578815+0000","last_fullsized":"2023-10-09T10:48:54.578815+0000","mapping_epoch":98,"log_start":"105'1844","ondisk_log_start":"105'1844","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"141'4416","last_scrub_stamp":"2023-10-09T10:46:02.760810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:46:02.760810+0000","objects_scrubbed":3378,"log_size":5778,"log_dups_size":0,"ondisk_log_size":5778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:35.863002+0000","scrub_duration":0.47769323899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11272192,"num_objects":172,"num_object_clones":0,"num_object_copies":516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3206,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":156,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723866+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:48:38.723866+0000","last_peered":"2023-10-09T10:48:38.723866+0000","last_clean":"2023-10-09T10:48:38.723866+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:48:38.723866+0000","last_undegraded":"2023-10-09T10:48:38.723866+0000","last_fullsized":"2023-10-09T10:48:38.723866+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":172,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.734678+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:48:38.734678+0000","last_peered":"2023-10-09T10:48:38.734678+0000","last_clean":"2023-10-09T10:48:38.734678+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:48:38.734678+0000","last_undegraded":"2023-10-09T10:48:38.734678+0000","last_fullsized":"2023-10-09T10:48:38.734678+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":164,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.045645+0000","last_change":"2023-10-09T10:46:32.245608+0000","last_active":"2023-10-09T10:48:46.045645+0000","last_peered":"2023-10-09T10:48:46.045645+0000","last_clean":"2023-10-09T10:48:46.045645+0000","last_became_active":"2023-10-09T10:46:32.245290+0000","last_became_peered":"2023-10-09T10:46:32.245290+0000","last_unstale":"2023-10-09T10:48:46.045645+0000","last_undegraded":"2023-10-09T10:48:46.045645+0000","last_fullsized":"2023-10-09T10:48:46.045645+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:48.832515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":522,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.711785+0000","last_change":"2023-10-09T10:46:05.620057+0000","last_active":"2023-10-09T10:48:38.711785+0000","last_peered":"2023-10-09T10:48:38.711785+0000","last_clean":"2023-10-09T10:48:38.711785+0000","last_became_active":"2023-10-09T10:46:05.619703+0000","last_became_peered":"2023-10-09T10:46:05.619703+0000","last_unstale":"2023-10-09T10:48:38.711785+0000","last_undegraded":"2023-10-09T10:48:38.711785+0000","last_fullsized":"2023-10-09T10:48:38.711785+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"188'7552","reported_seq":7677,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.576517+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:48:54.576517+0000","last_peered":"2023-10-09T10:48:54.576517+0000","last_clean":"2023-10-09T10:48:54.576517+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:48:54.576517+0000","last_undegraded":"2023-10-09T10:48:54.576517+0000","last_fullsized":"2023-10-09T10:48:54.576517+0000","mapping_epoch":115,"log_start":"105'1658","ondisk_log_start":"105'1658","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5894,"log_dups_size":0,"ondisk_log_size":5894,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12386304,"num_objects":189,"num_object_clones":0,"num_object_copies":567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6491,"num_write_kb":213760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":186,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.073660+0000","last_change":"2023-10-09T10:46:32.241041+0000","last_active":"2023-10-09T10:48:46.073660+0000","last_peered":"2023-10-09T10:48:46.073660+0000","last_clean":"2023-10-09T10:48:46.073660+0000","last_became_active":"2023-10-09T10:46:32.238683+0000","last_became_peered":"2023-10-09T10:46:32.238683+0000","last_unstale":"2023-10-09T10:48:46.073660+0000","last_undegraded":"2023-10-09T10:48:46.073660+0000","last_fullsized":"2023-10-09T10:48:46.073660+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:25.817713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":149,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.728288+0000","last_change":"2023-10-09T10:46:32.250104+0000","last_active":"2023-10-09T10:48:38.728288+0000","last_peered":"2023-10-09T10:48:38.728288+0000","last_clean":"2023-10-09T10:48:38.728288+0000","last_became_active":"2023-10-09T10:46:32.249805+0000","last_became_peered":"2023-10-09T10:46:32.249805+0000","last_unstale":"2023-10-09T10:48:38.728288+0000","last_undegraded":"2023-10-09T10:48:38.728288+0000","last_fullsized":"2023-10-09T10:48:38.728288+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:07.230672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":177,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.744261+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:48:38.744261+0000","last_peered":"2023-10-09T10:48:38.744261+0000","last_clean":"2023-10-09T10:48:38.744261+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:48:38.744261+0000","last_undegraded":"2023-10-09T10:48:38.744261+0000","last_fullsized":"2023-10-09T10:48:38.744261+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"187'7362","reported_seq":7473,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:51.109654+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:48:51.109654+0000","last_peered":"2023-10-09T10:48:51.109654+0000","last_clean":"2023-10-09T10:48:51.109654+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:48:51.109654+0000","last_undegraded":"2023-10-09T10:48:51.109654+0000","last_fullsized":"2023-10-09T10:48:51.109654+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7362,"log_dups_size":0,"ondisk_log_size":7362,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19398656,"num_objects":296,"num_object_clones":0,"num_object_copies":888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6354,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":166,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723071+0000","last_change":"2023-10-09T10:46:32.235346+0000","last_active":"2023-10-09T10:48:38.723071+0000","last_peered":"2023-10-09T10:48:38.723071+0000","last_clean":"2023-10-09T10:48:38.723071+0000","last_became_active":"2023-10-09T10:46:32.234906+0000","last_became_peered":"2023-10-09T10:46:32.234906+0000","last_unstale":"2023-10-09T10:48:38.723071+0000","last_undegraded":"2023-10-09T10:48:38.723071+0000","last_fullsized":"2023-10-09T10:48:38.723071+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.199251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":522,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.709901+0000","last_change":"2023-10-09T10:46:05.617146+0000","last_active":"2023-10-09T10:48:38.709901+0000","last_peered":"2023-10-09T10:48:38.709901+0000","last_clean":"2023-10-09T10:48:38.709901+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:48:38.709901+0000","last_undegraded":"2023-10-09T10:48:38.709901+0000","last_fullsized":"2023-10-09T10:48:38.709901+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":116,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.056995+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:48:46.056995+0000","last_peered":"2023-10-09T10:48:46.056995+0000","last_clean":"2023-10-09T10:48:46.056995+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:48:46.056995+0000","last_undegraded":"2023-10-09T10:48:46.056995+0000","last_fullsized":"2023-10-09T10:48:46.056995+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":84,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.734325+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:48:38.734325+0000","last_peered":"2023-10-09T10:48:38.734325+0000","last_clean":"2023-10-09T10:48:38.734325+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:48:38.734325+0000","last_undegraded":"2023-10-09T10:48:38.734325+0000","last_fullsized":"2023-10-09T10:48:38.734325+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":522,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.712082+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:48:38.712082+0000","last_peered":"2023-10-09T10:48:38.712082+0000","last_clean":"2023-10-09T10:48:38.712082+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:48:38.712082+0000","last_undegraded":"2023-10-09T10:48:38.712082+0000","last_fullsized":"2023-10-09T10:48:38.712082+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":179,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.728411+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:48:38.728411+0000","last_peered":"2023-10-09T10:48:38.728411+0000","last_clean":"2023-10-09T10:48:38.728411+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:48:38.728411+0000","last_undegraded":"2023-10-09T10:48:38.728411+0000","last_fullsized":"2023-10-09T10:48:38.728411+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":65,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.717271+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:48:38.717271+0000","last_peered":"2023-10-09T10:48:38.717271+0000","last_clean":"2023-10-09T10:48:38.717271+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:48:38.717271+0000","last_undegraded":"2023-10-09T10:48:38.717271+0000","last_fullsized":"2023-10-09T10:48:38.717271+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":65,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.713387+0000","last_change":"2023-10-09T10:46:32.236202+0000","last_active":"2023-10-09T10:48:38.713387+0000","last_peered":"2023-10-09T10:48:38.713387+0000","last_clean":"2023-10-09T10:48:38.713387+0000","last_became_active":"2023-10-09T10:46:32.235901+0000","last_became_peered":"2023-10-09T10:46:32.235901+0000","last_unstale":"2023-10-09T10:48:38.713387+0000","last_undegraded":"2023-10-09T10:48:38.713387+0000","last_fullsized":"2023-10-09T10:48:38.713387+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:48.260403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"187'7495","reported_seq":7654,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:51.492873+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:48:51.492873+0000","last_peered":"2023-10-09T10:48:51.492873+0000","last_clean":"2023-10-09T10:48:51.492873+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:48:51.492873+0000","last_undegraded":"2023-10-09T10:48:51.492873+0000","last_fullsized":"2023-10-09T10:48:51.492873+0000","mapping_epoch":98,"log_start":"101'48","ondisk_log_start":"101'48","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":7447,"log_dups_size":0,"ondisk_log_size":7447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18546688,"num_objects":283,"num_object_clones":0,"num_object_copies":849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4465,"num_write_kb":89664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":522,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.713137+0000","last_change":"2023-10-09T10:46:05.620764+0000","last_active":"2023-10-09T10:48:38.713137+0000","last_peered":"2023-10-09T10:48:38.713137+0000","last_clean":"2023-10-09T10:48:38.713137+0000","last_became_active":"2023-10-09T10:46:05.619870+0000","last_became_peered":"2023-10-09T10:46:05.619870+0000","last_unstale":"2023-10-09T10:48:38.713137+0000","last_undegraded":"2023-10-09T10:48:38.713137+0000","last_fullsized":"2023-10-09T10:48:38.713137+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":175,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.738131+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:48:38.738131+0000","last_peered":"2023-10-09T10:48:38.738131+0000","last_clean":"2023-10-09T10:48:38.738131+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:48:38.738131+0000","last_undegraded":"2023-10-09T10:48:38.738131+0000","last_fullsized":"2023-10-09T10:48:38.738131+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":149,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.045725+0000","last_change":"2023-10-09T10:46:32.251450+0000","last_active":"2023-10-09T10:48:46.045725+0000","last_peered":"2023-10-09T10:48:46.045725+0000","last_clean":"2023-10-09T10:48:46.045725+0000","last_became_active":"2023-10-09T10:46:32.250933+0000","last_became_peered":"2023-10-09T10:46:32.250933+0000","last_unstale":"2023-10-09T10:48:46.045725+0000","last_undegraded":"2023-10-09T10:48:46.045725+0000","last_fullsized":"2023-10-09T10:48:46.045725+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:04.116178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":56,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.712255+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:48:38.712255+0000","last_peered":"2023-10-09T10:48:38.712255+0000","last_clean":"2023-10-09T10:48:38.712255+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:48:38.712255+0000","last_undegraded":"2023-10-09T10:48:38.712255+0000","last_fullsized":"2023-10-09T10:48:38.712255+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"187'7355","reported_seq":14059,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:51.402436+0000","last_change":"2023-10-09T10:47:02.269289+0000","last_active":"2023-10-09T10:48:51.402436+0000","last_peered":"2023-10-09T10:48:51.402436+0000","last_clean":"2023-10-09T10:48:51.402436+0000","last_became_active":"2023-10-09T10:47:02.257854+0000","last_became_peered":"2023-10-09T10:47:02.257854+0000","last_unstale":"2023-10-09T10:48:51.402436+0000","last_undegraded":"2023-10-09T10:48:51.402436+0000","last_fullsized":"2023-10-09T10:48:51.402436+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7355,"log_dups_size":0,"ondisk_log_size":7355,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16744448,"num_objects":255,"num_object_clones":0,"num_object_copies":765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6347,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3287,"num_bytes_recovered":181927936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":522,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.711942+0000","last_change":"2023-10-09T10:46:05.618443+0000","last_active":"2023-10-09T10:48:38.711942+0000","last_peered":"2023-10-09T10:48:38.711942+0000","last_clean":"2023-10-09T10:48:38.711942+0000","last_became_active":"2023-10-09T10:46:05.616588+0000","last_became_peered":"2023-10-09T10:46:05.616588+0000","last_unstale":"2023-10-09T10:48:38.711942+0000","last_undegraded":"2023-10-09T10:48:38.711942+0000","last_fullsized":"2023-10-09T10:48:38.711942+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":181,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.057973+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:48:46.057973+0000","last_peered":"2023-10-09T10:48:46.057973+0000","last_clean":"2023-10-09T10:48:46.057973+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:48:46.057973+0000","last_undegraded":"2023-10-09T10:48:46.057973+0000","last_fullsized":"2023-10-09T10:48:46.057973+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":124,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.743551+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:48:38.743551+0000","last_peered":"2023-10-09T10:48:38.743551+0000","last_clean":"2023-10-09T10:48:38.743551+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:48:38.743551+0000","last_undegraded":"2023-10-09T10:48:38.743551+0000","last_fullsized":"2023-10-09T10:48:38.743551+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":170,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.056831+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:48:46.056831+0000","last_peered":"2023-10-09T10:48:46.056831+0000","last_clean":"2023-10-09T10:48:46.056831+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:48:46.056831+0000","last_undegraded":"2023-10-09T10:48:46.056831+0000","last_fullsized":"2023-10-09T10:48:46.056831+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"187'7457","reported_seq":14300,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:50.930612+0000","last_change":"2023-10-09T10:48:17.301310+0000","last_active":"2023-10-09T10:48:50.930612+0000","last_peered":"2023-10-09T10:48:50.930612+0000","last_clean":"2023-10-09T10:48:50.930612+0000","last_became_active":"2023-10-09T10:48:17.283042+0000","last_became_peered":"2023-10-09T10:48:17.283042+0000","last_unstale":"2023-10-09T10:48:50.930612+0000","last_undegraded":"2023-10-09T10:48:50.930612+0000","last_fullsized":"2023-10-09T10:48:50.930612+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7457,"log_dups_size":0,"ondisk_log_size":7457,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16023552,"num_objects":244,"num_object_clones":0,"num_object_copies":732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6420,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3337,"num_bytes_recovered":129564672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":521,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.712558+0000","last_change":"2023-10-09T10:46:05.617589+0000","last_active":"2023-10-09T10:48:38.712558+0000","last_peered":"2023-10-09T10:48:38.712558+0000","last_clean":"2023-10-09T10:48:38.712558+0000","last_became_active":"2023-10-09T10:46:05.616400+0000","last_became_peered":"2023-10-09T10:46:05.616400+0000","last_unstale":"2023-10-09T10:48:38.712558+0000","last_undegraded":"2023-10-09T10:48:38.712558+0000","last_fullsized":"2023-10-09T10:48:38.712558+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":46,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.057761+0000","last_change":"2023-10-09T10:46:32.245106+0000","last_active":"2023-10-09T10:48:46.057761+0000","last_peered":"2023-10-09T10:48:46.057761+0000","last_clean":"2023-10-09T10:48:46.057761+0000","last_became_active":"2023-10-09T10:46:32.244826+0000","last_became_peered":"2023-10-09T10:46:32.244826+0000","last_unstale":"2023-10-09T10:48:46.057761+0000","last_undegraded":"2023-10-09T10:48:46.057761+0000","last_fullsized":"2023-10-09T10:48:46.057761+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:30.322834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":60,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.713313+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:48:38.713313+0000","last_peered":"2023-10-09T10:48:38.713313+0000","last_clean":"2023-10-09T10:48:38.713313+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:48:38.713313+0000","last_undegraded":"2023-10-09T10:48:38.713313+0000","last_fullsized":"2023-10-09T10:48:38.713313+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":134,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.733317+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:48:38.733317+0000","last_peered":"2023-10-09T10:48:38.733317+0000","last_clean":"2023-10-09T10:48:38.733317+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:48:38.733317+0000","last_undegraded":"2023-10-09T10:48:38.733317+0000","last_fullsized":"2023-10-09T10:48:38.733317+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"187'7369","reported_seq":14185,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:50.975209+0000","last_change":"2023-10-09T10:47:31.271196+0000","last_active":"2023-10-09T10:48:50.975209+0000","last_peered":"2023-10-09T10:48:50.975209+0000","last_clean":"2023-10-09T10:48:50.975209+0000","last_became_active":"2023-10-09T10:47:31.229504+0000","last_became_peered":"2023-10-09T10:47:31.229504+0000","last_unstale":"2023-10-09T10:48:50.975209+0000","last_undegraded":"2023-10-09T10:48:50.975209+0000","last_fullsized":"2023-10-09T10:48:50.975209+0000","mapping_epoch":172,"log_start":"114'2615","ondisk_log_start":"114'2615","created":98,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":4754,"log_dups_size":0,"ondisk_log_size":4754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:26.533476+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16842752,"num_objects":257,"num_object_clones":0,"num_object_copies":771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4724,"num_write_kb":107264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3305,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":56,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.712339+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:48:38.712339+0000","last_peered":"2023-10-09T10:48:38.712339+0000","last_clean":"2023-10-09T10:48:38.712339+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:48:38.712339+0000","last_undegraded":"2023-10-09T10:48:38.712339+0000","last_fullsized":"2023-10-09T10:48:38.712339+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":118,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.745312+0000","last_change":"2023-10-09T10:46:32.251242+0000","last_active":"2023-10-09T10:48:38.745312+0000","last_peered":"2023-10-09T10:48:38.745312+0000","last_clean":"2023-10-09T10:48:38.745312+0000","last_became_active":"2023-10-09T10:46:32.250534+0000","last_became_peered":"2023-10-09T10:46:32.250534+0000","last_unstale":"2023-10-09T10:48:38.745312+0000","last_undegraded":"2023-10-09T10:48:38.745312+0000","last_fullsized":"2023-10-09T10:48:38.745312+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:33.726041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"187'7498","reported_seq":13863,"reported_epoch":187,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:52.772184+0000","last_change":"2023-10-09T10:46:34.514479+0000","last_active":"2023-10-09T10:48:52.772184+0000","last_peered":"2023-10-09T10:48:52.772184+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:46:34.409667+0000","last_became_peered":"2023-10-09T10:46:34.409667+0000","last_unstale":"2023-10-09T10:48:52.772184+0000","last_undegraded":"2023-10-09T10:46:34.376635+0000","last_fullsized":"2023-10-09T10:46:33.237998+0000","mapping_epoch":154,"log_start":"108'2297","ondisk_log_start":"108'2297","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5201,"log_dups_size":0,"ondisk_log_size":5201,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14286848,"num_objects":218,"num_object_clones":0,"num_object_copies":654,"num_objects_missing_on_primary":219,"num_objects_missing":219,"num_objects_degraded":729,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6456,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3119,"num_bytes_recovered":204406784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,7","objects":-1},{"shards":"2,3,7","objects":219}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":154,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723973+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:48:38.723973+0000","last_peered":"2023-10-09T10:48:38.723973+0000","last_clean":"2023-10-09T10:48:38.723973+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:48:38.723973+0000","last_undegraded":"2023-10-09T10:48:38.723973+0000","last_fullsized":"2023-10-09T10:48:38.723973+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":168,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.734803+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:48:38.734803+0000","last_peered":"2023-10-09T10:48:38.734803+0000","last_clean":"2023-10-09T10:48:38.734803+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:48:38.734803+0000","last_undegraded":"2023-10-09T10:48:38.734803+0000","last_fullsized":"2023-10-09T10:48:38.734803+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"188'7429","reported_seq":7549,"reported_epoch":188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:54.596559+0000","last_change":"2023-10-09T10:46:33.546991+0000","last_active":"2023-10-09T10:48:54.596559+0000","last_peered":"2023-10-09T10:48:54.596559+0000","last_clean":"2023-10-09T10:46:31.009484+0000","last_became_active":"2023-10-09T10:46:33.535665+0000","last_became_peered":"2023-10-09T10:46:33.535665+0000","last_unstale":"2023-10-09T10:48:54.596559+0000","last_undegraded":"2023-10-09T10:46:33.277168+0000","last_fullsized":"2023-10-09T10:46:33.235038+0000","mapping_epoch":154,"log_start":"104'1422","ondisk_log_start":"104'1422","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":6007,"log_dups_size":0,"ondisk_log_size":6007,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10649600,"num_objects":163,"num_object_clones":0,"num_object_copies":489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6365,"num_write_kb":208864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":175},{"shards":"3,7","objects":-2595}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":137,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.723528+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:48:38.723528+0000","last_peered":"2023-10-09T10:48:38.723528+0000","last_clean":"2023-10-09T10:48:38.723528+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:48:38.723528+0000","last_undegraded":"2023-10-09T10:48:38.723528+0000","last_fullsized":"2023-10-09T10:48:38.723528+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":35,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.737526+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:48:38.737526+0000","last_peered":"2023-10-09T10:48:38.737526+0000","last_clean":"2023-10-09T10:48:38.737526+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:48:38.737526+0000","last_undegraded":"2023-10-09T10:48:38.737526+0000","last_fullsized":"2023-10-09T10:48:38.737526+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":172,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.058317+0000","last_change":"2023-10-09T10:46:32.244607+0000","last_active":"2023-10-09T10:48:46.058317+0000","last_peered":"2023-10-09T10:48:46.058317+0000","last_clean":"2023-10-09T10:48:46.058317+0000","last_became_active":"2023-10-09T10:46:32.242628+0000","last_became_peered":"2023-10-09T10:46:32.242628+0000","last_unstale":"2023-10-09T10:48:46.058317+0000","last_undegraded":"2023-10-09T10:48:46.058317+0000","last_fullsized":"2023-10-09T10:48:46.058317+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:10.738819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":203,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T10:48:46.073452+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:48:46.073452+0000","last_peered":"2023-10-09T10:48:46.073452+0000","last_clean":"2023-10-09T10:48:46.073452+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:48:46.073452+0000","last_undegraded":"2023-10-09T10:48:46.073452+0000","last_fullsized":"2023-10-09T10:48:46.073452+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":77,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.713730+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:48:38.713730+0000","last_peered":"2023-10-09T10:48:38.713730+0000","last_clean":"2023-10-09T10:48:38.713730+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:48:38.713730+0000","last_undegraded":"2023-10-09T10:48:38.713730+0000","last_fullsized":"2023-10-09T10:48:38.713730+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T10:48:38.733257+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:48:38.733257+0000","last_peered":"2023-10-09T10:48:38.733257+0000","last_clean":"2023-10-09T10:48:38.733257+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:48:38.733257+0000","last_undegraded":"2023-10-09T10:48:38.733257+0000","last_fullsized":"2023-10-09T10:48:38.733257+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":453476376,"num_objects":6921,"num_object_clones":0,"num_object_copies":20763,"num_objects_missing_on_primary":219,"num_objects_missing":219,"num_objects_degraded":24927,"num_objects_misplaced":713,"num_objects_unfound":0,"num_objects_dirty":6921,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":197287,"num_write_kb":6258529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30548,"num_bytes_recovered":1611071488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3886559232,"data_stored":3886547016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170089,"ondisk_log_size":170089,"up":78,"acting":69,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1186,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332267,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":542764,"kb_used_data":241380,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93202388,"statfs":{"total":95995035648,"available":95439245312,"internally_reserved":0,"allocated":247173120,"data_stored":246461029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":1.155,"15min":0.76600000000000001},"min":{"1min":0.23499999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.476,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":3.4729999999999999},{"interface":"front","average":{"1min":3.1789999999999998,"5min":1.5029999999999999,"15min":0.92100000000000004},"min":{"1min":0.248,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.332000000000001,"5min":11.332000000000001,"15min":11.332000000000001},"last":3.27}]},{"osd":1,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":3.4359999999999999,"5min":1.909,"15min":0.997},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":11.367000000000001,"5min":11.826000000000001,"15min":11.826000000000001},"last":4.5999999999999996},{"interface":"front","average":{"1min":2.1739999999999999,"5min":1.484,"15min":0.84899999999999998},"min":{"1min":0.26100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":7.556,"5min":11.84,"15min":11.84},"last":7.556}]},{"osd":2,"last update":"Mon Oct 9 10:48:18 2023","interfaces":[{"interface":"back","average":{"1min":2.165,"5min":1.4810000000000001,"15min":0.83499999999999996},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.1419999999999995,"5min":9.1419999999999995,"15min":9.1419999999999995},"last":1.9970000000000001},{"interface":"front","average":{"1min":2.1970000000000001,"5min":1.1910000000000001,"15min":0.747},"min":{"1min":0.34999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":7.8120000000000003,"5min":7.8120000000000003,"15min":7.8120000000000003},"last":3.5139999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":3.903,"5min":1.893,"15min":1.0209999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":11.523,"5min":11.523,"15min":11.523},"last":3.2999999999999998},{"interface":"front","average":{"1min":4.2789999999999999,"5min":1.835,"15min":1.0109999999999999},"min":{"1min":0.35299999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":19.234999999999999,"5min":19.234999999999999,"15min":19.234999999999999},"last":2.0680000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":1.1579999999999999,"15min":0.70499999999999996},"min":{"1min":0.36399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":9.4900000000000002,"5min":9.4900000000000002,"15min":9.4900000000000002},"last":4.6349999999999998},{"interface":"front","average":{"1min":2.327,"5min":1.018,"15min":0.66100000000000003},"min":{"1min":0.35399999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.0999999999999996,"5min":9.0999999999999996,"15min":9.0999999999999996},"last":5.5990000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.0249999999999999,"5min":1.0609999999999999,"15min":0.69599999999999995},"min":{"1min":0.314,"5min":0.247,"15min":0.247},"max":{"1min":12.064,"5min":12.064,"15min":12.064},"last":5.173},{"interface":"front","average":{"1min":3.8839999999999999,"5min":1.4770000000000001,"15min":0.85299999999999998},"min":{"1min":0.435,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":11.741,"5min":11.741,"15min":11.741},"last":3.2320000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2999999999999998,"5min":1.3720000000000001,"15min":0.83199999999999996},"min":{"1min":0.33700000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.029,"5min":13.029,"15min":13.029},"last":5.8650000000000002},{"interface":"front","average":{"1min":2.6869999999999998,"5min":1.079,"15min":0.72699999999999998},"min":{"1min":0.41299999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":19.187999999999999,"5min":19.187999999999999,"15min":19.187999999999999},"last":4.8810000000000002}]}]},{"osd":6,"up_from":44,"seq":188978561198,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":483112,"kb_used_data":181728,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93262040,"statfs":{"total":95995035648,"available":95500328960,"internally_reserved":0,"allocated":186089472,"data_stored":185381919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":3.0499999999999998,"5min":1.4330000000000001,"15min":0.88},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.882,"5min":10.882,"15min":10.882},"last":0.442},{"interface":"front","average":{"1min":2.4540000000000002,"5min":1.367,"15min":0.86199999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.8710000000000004,"5min":7.8710000000000004,"15min":7.8710000000000004},"last":0.36499999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.157,"15min":0.77900000000000003},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.245},"max":{"1min":6.7560000000000002,"5min":16.747,"15min":16.747},"last":2.2010000000000001},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.284,"15min":0.79500000000000004},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.3579999999999997,"5min":16.734000000000002,"15min":16.734000000000002},"last":4.4729999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:48:20 2023","interfaces":[{"interface":"back","average":{"1min":2.798,"5min":1.411,"15min":0.82599999999999996},"min":{"1min":0.29699999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.208,"5min":13.208,"15min":13.208},"last":4.7599999999999998},{"interface":"front","average":{"1min":2.9420000000000002,"5min":1.619,"15min":0.91300000000000003},"min":{"1min":0.309,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.6749999999999998,"5min":12.571,"15min":12.571},"last":0.40000000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":1.496,"15min":0.85699999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":7.7649999999999997,"5min":7.7649999999999997,"15min":7.7649999999999997},"last":1.377},{"interface":"front","average":{"1min":2.254,"5min":1.7270000000000001,"15min":0.93999999999999995},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":13.179,"5min":13.179,"15min":13.179},"last":1.4590000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.0629999999999999,"15min":0.65000000000000002},"min":{"1min":0.29899999999999999,"5min":0.255,"15min":0.255},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.6579999999999999,"5min":0.89600000000000002,"15min":0.623},"min":{"1min":0.34100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.1260000000000003,"5min":5.1260000000000003,"15min":5.1260000000000003},"last":0.33100000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":1.1719999999999999,"15min":0.71099999999999997},"min":{"1min":0.35899999999999999,"5min":0.29499999999999998,"15min":0.27800000000000002},"max":{"1min":10.497,"5min":10.497,"15min":10.497},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.589,"5min":0.96599999999999997,"15min":0.67100000000000004},"min":{"1min":0.40699999999999997,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":6.8419999999999996,"5min":6.8419999999999996,"15min":6.8419999999999996},"last":0.39000000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:48:26 2023","interfaces":[{"interface":"back","average":{"1min":3.915,"5min":1.369,"15min":0.80600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":23.358000000000001,"5min":23.358000000000001,"15min":23.358000000000001},"last":0.376},{"interface":"front","average":{"1min":2.6840000000000002,"5min":1.25,"15min":0.79700000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34000000000000002},"max":{"1min":12.404,"5min":12.404,"15min":12.404},"last":0.42399999999999999}]}]},{"osd":1,"up_from":85,"seq":365072220237,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":440112,"kb_used_data":296168,"kb_used_omap":9,"kb_used_meta":143926,"kb_avail":93305040,"statfs":{"total":95995035648,"available":95544360960,"internally_reserved":0,"allocated":303276032,"data_stored":302329445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":147381225},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":2.1419999999999999,"15min":0.95599999999999996},"min":{"1min":0.249,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":18.027000000000001,"5min":18.027000000000001,"15min":18.027000000000001},"last":3.996},{"interface":"front","average":{"1min":2.6920000000000002,"5min":2.0209999999999999,"15min":0.93899999999999995},"min":{"1min":0.23599999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.6530000000000005,"5min":14.465,"15min":14.465},"last":6.0609999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:48:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":1.492,"15min":0.74199999999999999},"min":{"1min":0.221,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":15.356,"5min":15.356,"15min":15.356},"last":5.6790000000000003},{"interface":"front","average":{"1min":1.488,"5min":1.6890000000000001,"15min":0.81899999999999995},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.4130000000000003,"5min":14.234,"15min":14.234},"last":3.7469999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.6829999999999998,"5min":2.169,"15min":1.0369999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":18.274999999999999,"5min":18.274999999999999,"15min":18.274999999999999},"last":3.8479999999999999},{"interface":"front","average":{"1min":2.774,"5min":1.915,"15min":0.86799999999999999},"min":{"1min":0.313,"5min":0.251,"15min":0.251},"max":{"1min":8.5999999999999996,"5min":15.468999999999999,"15min":15.468999999999999},"last":1.9159999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":1.5289999999999999,"15min":0.90500000000000003},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.167999999999999,"5min":12.167999999999999,"15min":12.167999999999999},"last":2.419},{"interface":"front","average":{"1min":3.4399999999999999,"5min":1.7989999999999999,"15min":1.0129999999999999},"min":{"1min":0.309,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":14.228999999999999,"5min":14.228999999999999,"15min":14.228999999999999},"last":3.7879999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":1.4359999999999999,"15min":0.89700000000000002},"min":{"1min":0.34399999999999997,"5min":0.253,"15min":0.253},"max":{"1min":9.4809999999999999,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":1.9710000000000001},{"interface":"front","average":{"1min":3.7909999999999999,"5min":1.9219999999999999,"15min":1.0720000000000001},"min":{"1min":0.318,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":17.085000000000001,"5min":17.085000000000001,"15min":17.085000000000001},"last":2.3820000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":2.8559999999999999,"5min":1.6479999999999999,"15min":0.96999999999999997},"min":{"1min":0.38300000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.9179999999999993,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":1.9450000000000001},{"interface":"front","average":{"1min":2.819,"5min":1.7290000000000001,"15min":1.0069999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":12.231,"5min":12.231,"15min":12.231},"last":2.476}]},{"osd":7,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":3.1219999999999999,"5min":1.714,"15min":1.004},"min":{"1min":0.42799999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":12.249000000000001,"5min":12.249000000000001,"15min":12.249000000000001},"last":2.4489999999999998},{"interface":"front","average":{"1min":3.7250000000000001,"5min":1.907,"15min":1.0660000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":14.273999999999999,"5min":14.273999999999999,"15min":14.273999999999999},"last":3.8210000000000002}]}]},{"osd":0,"up_from":80,"seq":343597383771,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":305104,"kb_used_data":201288,"kb_used_omap":28,"kb_used_meta":103779,"kb_avail":93440048,"statfs":{"total":95995035648,"available":95682609152,"internally_reserved":0,"allocated":206118912,"data_stored":205139686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":106270544},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":3.931,"5min":2.258,"15min":0.998},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":11.869,"5min":25.475000000000001,"15min":25.475000000000001},"last":1.341},{"interface":"front","average":{"1min":3.702,"5min":2.2610000000000001,"15min":0.997},"min":{"1min":0.31,"5min":0.188,"15min":0.188},"max":{"1min":12.276,"5min":20.832999999999998,"15min":20.832999999999998},"last":0.55700000000000005}]},{"osd":2,"last update":"Mon Oct 9 10:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":1.4750000000000001,"15min":0.72399999999999998},"min":{"1min":0.249,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.0890000000000004,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":1.306},{"interface":"front","average":{"1min":2.1120000000000001,"5min":1.889,"15min":0.85799999999999998},"min":{"1min":0.27100000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.0329999999999995,"5min":14.391999999999999,"15min":14.391999999999999},"last":0.92300000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.3359999999999999,"15min":1.212},"min":{"1min":0.307,"5min":0.215,"15min":0.215},"max":{"1min":12.307,"5min":14.926,"15min":14.926},"last":0.45000000000000001},{"interface":"front","average":{"1min":3.5539999999999998,"5min":2.1960000000000002,"15min":1.093},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":11.885999999999999,"5min":11.885999999999999,"15min":11.885999999999999},"last":1.367}]},{"osd":4,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":2.8370000000000002,"5min":1.671,"15min":0.96999999999999997},"min":{"1min":0.434,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.0920000000000005,"5min":8.0920000000000005,"15min":8.0920000000000005},"last":0.58799999999999997},{"interface":"front","average":{"1min":3.2679999999999998,"5min":1.9370000000000001,"15min":1.0569999999999999},"min":{"1min":0.36099999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.318,"5min":12.318,"15min":12.318},"last":0.41999999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6179999999999999,"5min":1.8400000000000001,"15min":1.0289999999999999},"min":{"1min":0.35899999999999999,"5min":0.318,"15min":0.318},"max":{"1min":12.157,"5min":12.157,"15min":12.157},"last":1.573},{"interface":"front","average":{"1min":3.181,"5min":1.677,"15min":0.98999999999999999},"min":{"1min":0.40500000000000003,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":12.781000000000001,"5min":12.781000000000001,"15min":12.781000000000001},"last":1.2749999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":3.4649999999999999,"5min":1.96,"15min":1.073},"min":{"1min":0.33600000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":12.207000000000001,"5min":12.207000000000001,"15min":12.207000000000001},"last":2.387},{"interface":"front","average":{"1min":2.1269999999999998,"5min":1.3740000000000001,"15min":0.89100000000000001},"min":{"1min":0.41299999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":9.8889999999999993,"5min":9.8889999999999993,"15min":9.8889999999999993},"last":2.4649999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":3.6280000000000001,"5min":1.7509999999999999,"15min":1.0169999999999999},"min":{"1min":0.48799999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":12.727,"5min":12.727,"15min":12.727},"last":1.395},{"interface":"front","average":{"1min":2.843,"5min":1.8660000000000001,"15min":1.071},"min":{"1min":0.47599999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.6150000000000002,"5min":14.394,"15min":14.394},"last":0.623}]}]},{"osd":2,"up_from":0,"seq":386547056712,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893186,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":617920,"kb_used_data":360184,"kb_used_omap":7,"kb_used_meta":257464,"kb_avail":93127232,"statfs":{"total":95995035648,"available":95362285568,"internally_reserved":0,"allocated":368828416,"data_stored":367863831,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":263643752},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3159999999999998,"5min":1.913,"15min":0.85099999999999998},"min":{"1min":0.248,"5min":0.183,"15min":0.183},"max":{"1min":15.255000000000001,"5min":15.255000000000001,"15min":15.255000000000001},"last":7.5819999999999999},{"interface":"front","average":{"1min":2.9830000000000001,"5min":1.6539999999999999,"15min":0.82199999999999995},"min":{"1min":0.21099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":8.6470000000000002,"5min":8.6470000000000002,"15min":8.6470000000000002},"last":8.7560000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.6400000000000001,"5min":2.359,"15min":1.006},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":10.821,"5min":14.707000000000001,"15min":14.707000000000001},"last":2.3700000000000001},{"interface":"front","average":{"1min":3.4780000000000002,"5min":1.8100000000000001,"15min":0.85199999999999998},"min":{"1min":0.28999999999999998,"5min":0.193,"15min":0.193},"max":{"1min":15.443,"5min":15.443,"15min":15.443},"last":10.59}]},{"osd":2,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":2.0630000000000002,"15min":0.93200000000000005},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.375,"5min":13.853,"15min":13.853},"last":1.8680000000000001},{"interface":"front","average":{"1min":3.988,"5min":2.2309999999999999,"15min":1.0069999999999999},"min":{"1min":0.28699999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.557,"5min":10.557,"15min":10.557},"last":8.8569999999999993}]},{"osd":4,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":4.0739999999999998,"5min":1.9219999999999999,"15min":1.022},"min":{"1min":0.33900000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":20.388000000000002,"5min":20.388000000000002,"15min":20.388000000000002},"last":5.298},{"interface":"front","average":{"1min":4.4299999999999997,"5min":2.0609999999999999,"15min":1.0920000000000001},"min":{"1min":0.38700000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":19.085999999999999,"5min":19.085999999999999,"15min":19.085999999999999},"last":6.2990000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.7599999999999998,"5min":2.4369999999999998,"15min":1.2030000000000001},"min":{"1min":0.35399999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.3550000000000004,"5min":51.509999999999998,"15min":51.509999999999998},"last":6.335},{"interface":"front","average":{"1min":2.6240000000000001,"5min":2.7189999999999999,"15min":1.2809999999999999},"min":{"1min":0.316,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.7449999999999992,"5min":55.338999999999999,"15min":55.338999999999999},"last":1.901}]},{"osd":6,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":1.841,"15min":1.0049999999999999},"min":{"1min":0.43099999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.361000000000001,"5min":12.361000000000001,"15min":12.361000000000001},"last":8.5960000000000001},{"interface":"front","average":{"1min":2.8140000000000001,"5min":2.2189999999999999,"15min":1.139},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.5839999999999996,"5min":13.891999999999999,"15min":13.891999999999999},"last":3.6899999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.2269999999999999,"5min":2.8679999999999999,"15min":1.3500000000000001},"min":{"1min":0.34000000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.734,"5min":51.744,"15min":51.744},"last":2.069},{"interface":"front","average":{"1min":3.927,"5min":2.54,"15min":1.2290000000000001},"min":{"1min":0.46100000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":15.212999999999999,"5min":53.399000000000001,"15min":53.399000000000001},"last":5.2610000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019061,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":476392,"kb_used_data":170848,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93268760,"statfs":{"total":95995035648,"available":95507210240,"internally_reserved":0,"allocated":174948352,"data_stored":174240066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.9850000000000001,"5min":1.302,"15min":0.80500000000000005},"min":{"1min":0.216,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.7160000000000002,"5min":10.387,"15min":10.387},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.9780000000000002,"5min":1.6659999999999999,"15min":0.94299999999999995},"min":{"1min":0.48499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.0129999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":4.1970000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":2.351,"5min":1.512,"15min":0.872},"min":{"1min":0.254,"5min":0.249,"15min":0.249},"max":{"1min":7.5350000000000001,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":2.931,"5min":1.7010000000000001,"15min":0.98099999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.2919999999999998,"5min":8.7669999999999995,"15min":8.7669999999999995},"last":0.56699999999999995}]},{"osd":2,"last update":"Mon Oct 9 10:48:13 2023","interfaces":[{"interface":"back","average":{"1min":3.1520000000000001,"5min":1.617,"15min":0.91900000000000004},"min":{"1min":0.52100000000000002,"5min":0.253,"15min":0.253},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":4.2619999999999996},{"interface":"front","average":{"1min":4.016,"5min":1.855,"15min":0.99299999999999999},"min":{"1min":0.36799999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":20.545999999999999,"5min":20.545999999999999,"15min":20.545999999999999},"last":4.327}]},{"osd":3,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":3.2549999999999999,"5min":1.8420000000000001,"15min":1.0880000000000001},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.0269999999999992,"5min":9.0269999999999992,"15min":9.0269999999999992},"last":0.54400000000000004},{"interface":"front","average":{"1min":3.9809999999999999,"5min":2.0379999999999998,"15min":1.071},"min":{"1min":0.79400000000000004,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.6029999999999998,"5min":8.9809999999999999,"15min":8.9809999999999999},"last":4.234}]},{"osd":5,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":5.0960000000000001,"5min":1.8140000000000001,"15min":0.91000000000000003},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":24.149999999999999,"5min":24.149999999999999,"15min":24.149999999999999},"last":9.0020000000000007},{"interface":"front","average":{"1min":4.173,"5min":1.6699999999999999,"15min":0.86399999999999999},"min":{"1min":0.373,"5min":0.252,"15min":0.252},"max":{"1min":20.422000000000001,"5min":20.422000000000001,"15min":20.422000000000001},"last":15.071999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":1.1319999999999999,"15min":0.69399999999999995},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":12.391999999999999,"5min":12.391999999999999,"15min":12.391999999999999},"last":4.1260000000000003},{"interface":"front","average":{"1min":3.1200000000000001,"5min":1.2130000000000001,"15min":0.78100000000000003},"min":{"1min":0.374,"5min":0.311,"15min":0.311},"max":{"1min":12.539,"5min":12.539,"15min":12.539},"last":4.1020000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:48:21 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":1.2929999999999999,"15min":0.80900000000000005},"min":{"1min":0.33300000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.7919999999999998,"5min":8.7919999999999998,"15min":8.7919999999999998},"last":0.52000000000000002},{"interface":"front","average":{"1min":3.2389999999999999,"5min":1.27,"15min":0.79000000000000004},"min":{"1min":0.55900000000000005,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.423,"5min":7.423,"15min":7.423},"last":4.2859999999999996}]}]},{"osd":5,"up_from":37,"seq":158913790130,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":532440,"kb_used_data":222864,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93212712,"statfs":{"total":95995035648,"available":95449817088,"internally_reserved":0,"allocated":228212736,"data_stored":227516279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:41 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":1.2210000000000001,"15min":0.80500000000000005},"min":{"1min":0.28899999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.648999999999999,"5min":11.648999999999999,"15min":11.648999999999999},"last":1.847},{"interface":"front","average":{"1min":2.8570000000000002,"5min":1.294,"15min":0.875},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.68,"5min":11.68,"15min":11.68},"last":1.8680000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":1.367,"15min":0.86099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":12.999000000000001,"5min":12.999000000000001,"15min":12.999000000000001},"last":0.91400000000000003},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.2370000000000001,"15min":0.80700000000000005},"min":{"1min":0.27600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.3730000000000002,"5min":7.3730000000000002,"15min":7.3730000000000002},"last":3.4510000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:48:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6669999999999998,"5min":1.4390000000000001,"15min":0.86099999999999999},"min":{"1min":0.34799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.3280000000000003,"5min":7.9000000000000004,"15min":7.9000000000000004},"last":1.004},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.2350000000000001,"15min":0.79400000000000004},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.3059999999999992,"5min":10.352,"15min":10.352},"last":2.165}]},{"osd":3,"last update":"Mon Oct 9 10:48:47 2023","interfaces":[{"interface":"back","average":{"1min":2.859,"5min":2.048,"15min":1.165},"min":{"1min":0.432,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":12.558999999999999,"5min":18.016999999999999,"15min":18.016999999999999},"last":6.7210000000000001},{"interface":"front","average":{"1min":2.6930000000000001,"5min":2.089,"15min":1.105},"min":{"1min":0.40300000000000002,"5min":0.317,"15min":0.317},"max":{"1min":7.9560000000000004,"5min":15.555999999999999,"15min":15.555999999999999},"last":6.6870000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:47:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":0.83799999999999997,"15min":0.57799999999999996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":19.152999999999999,"5min":19.152999999999999,"15min":19.152999999999999},"last":1.792},{"interface":"front","average":{"1min":1.986,"5min":0.96999999999999997,"15min":0.63500000000000001},"min":{"1min":0.26800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.3059999999999992,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":6.6470000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":1.127,"15min":0.69899999999999995},"min":{"1min":0.33100000000000002,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":7.7060000000000004,"5min":7.7060000000000004,"15min":7.7060000000000004},"last":0.877},{"interface":"front","average":{"1min":1.958,"5min":0.90400000000000003,"15min":0.64000000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.30299999999999999},"max":{"1min":8.0909999999999993,"5min":8.0909999999999993,"15min":8.0909999999999993},"last":20.256}]},{"osd":7,"last update":"Mon Oct 9 10:48:26 2023","interfaces":[{"interface":"back","average":{"1min":2.79,"5min":1.1160000000000001,"15min":0.74099999999999999},"min":{"1min":0.41899999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.3940000000000001,"5min":7.3940000000000001,"15min":7.3940000000000001},"last":7.5919999999999996},{"interface":"front","average":{"1min":2.2669999999999999,"5min":1.0369999999999999,"15min":0.71999999999999997},"min":{"1min":0.38500000000000001,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":9.0579999999999998,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":1.8240000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":202117120,"data_stored":202113048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":286265344,"data_stored":286261272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2256928768,"data_stored":2256928768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":339017728,"data_stored":339017728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":168689664,"data_stored":168689664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":217321472,"data_stored":217317400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":181207040,"data_stored":181207040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":235012096,"data_stored":235012096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:48:56.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T10:48:57.066 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:57.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:57.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:57.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:57.218 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:57.252 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:57.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:57.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:57.357 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:48:57.384 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:57.414 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:57.458 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:48:57.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:57.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:48:57.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:57 smithi155 ceph-mon[184854]: from='client.54892 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:57.485 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:48:57.485 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:48:57.485 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:48:57.485 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:48:57.485 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:48:57.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:57 smithi125 ceph-mon[195507]: from='client.54892 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:57.511 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:57 smithi125 ceph-mon[200197]: from='client.54892 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:48:57.567 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:57.630 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:57.653 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:57.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:57.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:57.759 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:48:57.805 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:48:57.823 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:57.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:48:57.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:57.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:48:57.914 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:48:57.964 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:48:58.022 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:58.045 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:48:58.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:58.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:48:58.158 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:58.229 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:58.577 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:58 smithi125 ceph-mon[195507]: pgmap v422: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 329 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 24467/15852 objects degraded (154.346%); 481/15852 objects misplaced (3.034%); 1.1 MiB/s, 174 objects/s recovering 2023-10-09T10:48:58.577 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:48:58 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 24467/15852 objects degraded (154.346%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:58.577 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:58 smithi125 ceph-mon[200197]: pgmap v422: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 329 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 24467/15852 objects degraded (154.346%); 481/15852 objects misplaced (3.034%); 1.1 MiB/s, 174 objects/s recovering 2023-10-09T10:48:58.578 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:48:58 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 24467/15852 objects degraded (154.346%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:58.708 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:58.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:58.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:58.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:58 smithi155 ceph-mon[184854]: pgmap v422: 119 pgs: 107 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped; 329 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 24467/15852 objects degraded (154.346%); 481/15852 objects misplaced (3.034%); 1.1 MiB/s, 174 objects/s recovering 2023-10-09T10:48:58.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:48:58 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 24467/15852 objects degraded (154.346%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:48:58.813 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:48:58.858 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:59.087 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:48:59.604 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:48:59.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:59.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:48:59.689 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:59.713 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:48:59.737 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:48:59.773 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:48:59.847 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:48:59.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:48:59.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:48:59.955 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:00.021 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:00.062 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:00.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:00.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:00.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:49:00.230 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:00.240 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:00.320 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:49:00.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:00.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:49:00.344 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:00 smithi155 ceph-mon[184854]: pgmap v423: 119 pgs: 1 active+recovering+undersized+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 316 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 23671/15198 objects degraded (155.751%); 481/15198 objects misplaced (3.165%); 2.8 MiB/s, 184 objects/s recovering 2023-10-09T10:49:00.344 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:00.344 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:49:00.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:00.480 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:00.517 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:00.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:00.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:00.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:00 smithi125 ceph-mon[195507]: pgmap v423: 119 pgs: 1 active+recovering+undersized+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 316 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 23671/15198 objects degraded (155.751%); 481/15198 objects misplaced (3.165%); 2.8 MiB/s, 184 objects/s recovering 2023-10-09T10:49:00.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:49:00.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:00 smithi125 ceph-mon[200197]: pgmap v423: 119 pgs: 1 active+recovering+undersized+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 316 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 23671/15198 objects degraded (155.751%); 481/15198 objects misplaced (3.165%); 2.8 MiB/s, 184 objects/s recovering 2023-10-09T10:49:00.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:00.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:49:00.628 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:00.676 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:00.715 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:00.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:00.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:00.819 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:00.866 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:00.907 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:00.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:00.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:00.960 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:49:00.960 INFO:teuthology.orchestra.run.smithi125.stdout:{"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 23671/15198 objects degraded (155.751%), 7 pgs degraded, 8 pgs undersized","count":15},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":526,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":189,"num_osds":8,"num_up_osds":8,"osd_up_since":1696848214,"num_in_osds":7,"osd_in_since":1696848391,"num_remapped_pgs":11},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":108},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":5066,"data_bytes":330930104,"bytes_used":3092430848,"bytes_avail":668872818688,"bytes_total":671965249536,"degraded_objects":23671,"degraded_total":15198,"degraded_ratio":1.5575075667851033,"misplaced_objects":481,"misplaced_total":15198,"misplaced_ratio":0.031648901171206736,"recovering_objects_per_sec":184,"recovering_bytes_per_sec":2946974,"recovering_keys_per_sec":0,"num_objects_recovered":1481,"num_bytes_recovered":23592960,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":607},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi125.satgiw","status":"up:active","gid":24575}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"}},"servicemap":{"epoch":20,"modified":"2023-10-09T10:44:35.008237+0000","services":{}},"progress_events":{"4db0080d-306f-4da6-a38c-4c702d14cb34":{"message":"Global Recovery Event (3m)\n [=========================...] (remaining: 26s)","progress":0.89915966987609863,"add_to_ceph_s":true},"585851ac-6e70-4462-9176-ac50987fab77":{"message":"Upgrade to 18.2.0-598-gb4fa663f (3m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T10:49:01.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:49:01.091 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:01.574 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:49:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:49:01] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.43.0" 2023-10-09T10:49:01.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1188723885' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:49:01.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1188723885' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:49:01.604 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T10:49:01.604 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:49:01.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1188723885' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T10:49:01.737 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:49:01.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:01.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:49:01.849 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:01.885 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:01.952 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:01.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:01.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:02.059 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:49:02.101 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:02.198 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:49:02.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:02.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:49:02.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:02.349 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:02.376 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:02.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:02.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:02.481 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:02.524 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:02.537 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:02.550 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:02.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:02.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:02.655 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:02.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:02 smithi125 ceph-mon[195507]: pgmap v424: 119 pgs: 1 active+recovering+undersized+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 178 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 23268/8583 objects degraded (271.094%); 265/8583 objects misplaced (3.087%); 1.7 MiB/s, 165 objects/s recovering 2023-10-09T10:49:02.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:02 smithi125 ceph-mon[200197]: pgmap v424: 119 pgs: 1 active+recovering+undersized+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 178 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 23268/8583 objects degraded (271.094%); 265/8583 objects misplaced (3.087%); 1.7 MiB/s, 165 objects/s recovering 2023-10-09T10:49:02.685 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:49:02.719 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:49:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:02 smithi155 ceph-mon[184854]: pgmap v424: 119 pgs: 1 active+recovering+undersized+remapped, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 178 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 23268/8583 objects degraded (271.094%); 265/8583 objects misplaced (3.087%); 1.7 MiB/s, 165 objects/s recovering 2023-10-09T10:49:02.723 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:02.740 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:49:02.766 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:49:02.792 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:49:02.795 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:02.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:02.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:02.819 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:49:02.841 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:49:02.868 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:49:02.869 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:49:02.870 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:49:02.870 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:49:02.870 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:49:02.870 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:49:02.870 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:49:02.870 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:49:02.870 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:49:02.899 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:02.926 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:02.945 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:02.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:02.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:02.992 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:49:02.993 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:49:03.049 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:03.076 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:03.097 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:03.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:03.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:03.202 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:03.230 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:03.249 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:03.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:03.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:03.352 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:03.380 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:03.399 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:03.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:03.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:03.415 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:49:03.455 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Removed 106440 objects 2023-10-09T10:49:03.455 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Clean up completed and total clean up time :175.119 2023-10-09T10:49:03.504 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:03.541 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[195507]: osdmap e190: 8 total, 8 up, 7 in 2023-10-09T10:49:03.541 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[195507]: failed to encode map e190 with expected crc 2023-10-09T10:49:03.541 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[195507]: failed to encode map e190 with expected crc 2023-10-09T10:49:03.541 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[195507]: failed to encode map e190 with expected crc 2023-10-09T10:49:03.541 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[195507]: failed to encode map e190 with expected crc 2023-10-09T10:49:03.541 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/719917708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:03.541 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 23268/8583 objects degraded (271.094%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:49:03.541 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3624153467' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:49:03.541 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:49:03.542 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[200197]: osdmap e190: 8 total, 8 up, 7 in 2023-10-09T10:49:03.542 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[200197]: failed to encode map e190 with expected crc 2023-10-09T10:49:03.542 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[200197]: failed to encode map e190 with expected crc 2023-10-09T10:49:03.542 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[200197]: failed to encode map e190 with expected crc 2023-10-09T10:49:03.542 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[200197]: failed to encode map e190 with expected crc 2023-10-09T10:49:03.542 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/719917708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:03.542 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 23268/8583 objects degraded (271.094%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:49:03.542 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3624153467' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:49:03.542 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:03 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:49:03.544 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:03.667 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:03.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:03.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:03 smithi155 ceph-mon[184854]: osdmap e190: 8 total, 8 up, 7 in 2023-10-09T10:49:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:03 smithi155 ceph-mon[184854]: failed to encode map e190 with expected crc 2023-10-09T10:49:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:03 smithi155 ceph-mon[184854]: failed to encode map e190 with expected crc 2023-10-09T10:49:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:03 smithi155 ceph-mon[184854]: failed to encode map e190 with expected crc 2023-10-09T10:49:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:03 smithi155 ceph-mon[184854]: failed to encode map e190 with expected crc 2023-10-09T10:49:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/719917708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:03 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 23268/8583 objects degraded (271.094%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:49:03.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3624153467' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:49:03.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:03 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:49:03.779 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:03.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:49:03.810 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:04.118 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:49:04.133 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:04.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:04.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:04.239 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:04.277 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:04.299 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:04.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:04.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:04.394 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[200197]: pgmap v426: 119 pgs: 1 peering, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 91 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 23268/4419 objects degraded (526.544%); 151/4419 objects misplaced (3.417%); 1.7 MiB/s, 207 objects/s recovering 2023-10-09T10:49:04.394 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:49:04.394 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[200197]: failed to encode map e191 with expected crc 2023-10-09T10:49:04.394 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[200197]: failed to encode map e191 with expected crc 2023-10-09T10:49:04.394 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:49:04.394 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[200197]: osdmap e191: 8 total, 8 up, 7 in 2023-10-09T10:49:04.395 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[200197]: failed to encode map e191 with expected crc 2023-10-09T10:49:04.395 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[200197]: failed to encode map e191 with expected crc 2023-10-09T10:49:04.403 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:04.439 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:04.513 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:04.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:04.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:04.549 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:04 smithi155 ceph-mon[184854]: pgmap v426: 119 pgs: 1 peering, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 91 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 23268/4419 objects degraded (526.544%); 151/4419 objects misplaced (3.417%); 1.7 MiB/s, 207 objects/s recovering 2023-10-09T10:49:04.549 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:04 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:49:04.549 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:04 smithi155 ceph-mon[184854]: failed to encode map e191 with expected crc 2023-10-09T10:49:04.549 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:04 smithi155 ceph-mon[184854]: failed to encode map e191 with expected crc 2023-10-09T10:49:04.549 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:04 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:49:04.549 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:04 smithi155 ceph-mon[184854]: osdmap e191: 8 total, 8 up, 7 in 2023-10-09T10:49:04.549 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:04 smithi155 ceph-mon[184854]: failed to encode map e191 with expected crc 2023-10-09T10:49:04.549 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:04 smithi155 ceph-mon[184854]: failed to encode map e191 with expected crc 2023-10-09T10:49:04.617 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:04.632 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:49:04.635 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:49:04.648 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:04.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[195507]: pgmap v426: 119 pgs: 1 peering, 108 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 91 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 23268/4419 objects degraded (526.544%); 151/4419 objects misplaced (3.417%); 1.7 MiB/s, 207 objects/s recovering 2023-10-09T10:49:04.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:49:04.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[195507]: failed to encode map e191 with expected crc 2023-10-09T10:49:04.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[195507]: failed to encode map e191 with expected crc 2023-10-09T10:49:04.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:49:04.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[195507]: osdmap e191: 8 total, 8 up, 7 in 2023-10-09T10:49:04.651 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[195507]: failed to encode map e191 with expected crc 2023-10-09T10:49:04.651 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:04 smithi125 ceph-mon[195507]: failed to encode map e191 with expected crc 2023-10-09T10:49:04.670 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:04.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:04.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:04.773 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:04.804 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:04.825 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:04.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:04.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:04.930 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:04.964 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:05.110 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:49:05.110 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:49:05.439 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:05.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:05.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:05.516 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:49:05.543 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:05.589 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:05.615 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:05.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:05.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:05.715 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:05 smithi125 ceph-mon[195507]: failed to encode map e191 with expected crc 2023-10-09T10:49:05.715 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:05 smithi125 ceph-mon[195507]: failed to encode map e192 with expected crc 2023-10-09T10:49:05.715 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:05 smithi125 ceph-mon[195507]: failed to encode map e192 with expected crc 2023-10-09T10:49:05.715 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:05 smithi125 ceph-mon[195507]: osdmap e192: 8 total, 8 up, 7 in 2023-10-09T10:49:05.715 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:05 smithi125 ceph-mon[200197]: failed to encode map e191 with expected crc 2023-10-09T10:49:05.715 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:05 smithi125 ceph-mon[200197]: failed to encode map e192 with expected crc 2023-10-09T10:49:05.715 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:05 smithi125 ceph-mon[200197]: failed to encode map e192 with expected crc 2023-10-09T10:49:05.716 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:05 smithi125 ceph-mon[200197]: osdmap e192: 8 total, 8 up, 7 in 2023-10-09T10:49:05.720 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:05.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:05 smithi155 ceph-mon[184854]: failed to encode map e191 with expected crc 2023-10-09T10:49:05.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:05 smithi155 ceph-mon[184854]: failed to encode map e192 with expected crc 2023-10-09T10:49:05.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:05 smithi155 ceph-mon[184854]: failed to encode map e192 with expected crc 2023-10-09T10:49:05.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:05 smithi155 ceph-mon[184854]: osdmap e192: 8 total, 8 up, 7 in 2023-10-09T10:49:05.751 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:05.776 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:05.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:05.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:05.828 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:49:05.881 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:05.911 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:05.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:05.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:05.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:06.045 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:49:06.085 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:49:06.118 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:06.408 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:06 smithi125 ceph-mon[195507]: from='client.44987 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:49:06.408 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:06 smithi125 ceph-mon[195507]: pgmap v429: 119 pgs: 1 peering, 108 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 36 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 756 op/s; 22746/1755 objects degraded (1296.068%); 73/1755 objects misplaced (4.160%); 0 B/s, 183 objects/s recovering 2023-10-09T10:49:06.526 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:49:06.608 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:06 smithi155 ceph-mon[184854]: from='client.44987 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:49:06.608 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:06 smithi155 ceph-mon[184854]: pgmap v429: 119 pgs: 1 peering, 108 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 36 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 756 op/s; 22746/1755 objects degraded (1296.068%); 73/1755 objects misplaced (4.160%); 0 B/s, 183 objects/s recovering 2023-10-09T10:49:06.797 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-10-09T10:49:06.797 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-10-09T10:49:06.810 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:49:06.811 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:49:06.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:06 smithi125 ceph-mon[200197]: from='client.44987 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:49:06.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:06 smithi125 ceph-mon[200197]: pgmap v429: 119 pgs: 1 peering, 108 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped; 36 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 756 op/s; 22746/1755 objects degraded (1296.068%); 73/1755 objects misplaced (4.160%); 0 B/s, 183 objects/s recovering 2023-10-09T10:49:07.175 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:49:07.227 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:49:07.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:07.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:49:07.255 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":428,"stamp":"2023-10-09T10:49:04.400968+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":95492024,"num_objects":1473,"num_object_clones":0,"num_object_copies":4419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23268,"num_objects_misplaced":151,"num_objects_unfound":0,"num_objects_dirty":1473,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":203067,"num_write_kb":6260974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32231,"num_bytes_recovered":1627196162,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173015,"ondisk_log_size":173015,"up":357,"acting":350,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2381792,"kb_used_data":620136,"kb_used_omap":44,"kb_used_meta":1761491,"kb_avail":653834272,"statfs":{"total":671965249536,"available":669526294528,"internally_reserved":0,"allocated":635019264,"data_stored":629223348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1803767201},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,3,6,10,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-249724928,"num_objects":-3811,"num_object_clones":0,"num_object_copies":-11433,"num_objects_missing_on_primary":-219,"num_objects_missing":-219,"num_objects_degraded":-1199,"num_objects_misplaced":-330,"num_objects_unfound":0,"num_objects_dirty":-3811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3819,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1199,"num_bytes_recovered":14352384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.222535"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":182,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.299068+0000","last_change":"2023-10-09T10:46:32.250979+0000","last_active":"2023-10-09T10:48:54.299068+0000","last_peered":"2023-10-09T10:48:54.299068+0000","last_clean":"2023-10-09T10:48:54.299068+0000","last_became_active":"2023-10-09T10:46:32.250625+0000","last_became_peered":"2023-10-09T10:46:32.250625+0000","last_unstale":"2023-10-09T10:48:54.299068+0000","last_undegraded":"2023-10-09T10:48:54.299068+0000","last_fullsized":"2023-10-09T10:48:54.299068+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:34.858348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":158,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.347917+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:48:54.347917+0000","last_peered":"2023-10-09T10:48:54.347917+0000","last_clean":"2023-10-09T10:48:54.347917+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:48:54.347917+0000","last_undegraded":"2023-10-09T10:48:54.347917+0000","last_fullsized":"2023-10-09T10:48:54.347917+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":141,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.297053+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:48:54.297053+0000","last_peered":"2023-10-09T10:48:54.297053+0000","last_clean":"2023-10-09T10:48:54.297053+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:48:54.297053+0000","last_undegraded":"2023-10-09T10:48:54.297053+0000","last_fullsized":"2023-10-09T10:48:54.297053+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":59,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.306748+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:48:55.306748+0000","last_peered":"2023-10-09T10:48:55.306748+0000","last_clean":"2023-10-09T10:48:55.306748+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:48:55.306748+0000","last_undegraded":"2023-10-09T10:48:55.306748+0000","last_fullsized":"2023-10-09T10:48:55.306748+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":194,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.331018+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:48:54.331018+0000","last_peered":"2023-10-09T10:48:54.331018+0000","last_clean":"2023-10-09T10:48:54.331018+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:48:54.331018+0000","last_undegraded":"2023-10-09T10:48:54.331018+0000","last_fullsized":"2023-10-09T10:48:54.331018+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":127,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.291626+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:48:54.291626+0000","last_peered":"2023-10-09T10:48:54.291626+0000","last_clean":"2023-10-09T10:48:54.291626+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:48:54.291626+0000","last_undegraded":"2023-10-09T10:48:54.291626+0000","last_fullsized":"2023-10-09T10:48:54.291626+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"189'7842","reported_seq":12005,"reported_epoch":189,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:49:01.049671+0000","last_change":"2023-10-09T10:46:33.521602+0000","last_active":"2023-10-09T10:49:01.049671+0000","last_peered":"2023-10-09T10:49:01.049671+0000","last_clean":"2023-10-09T10:46:31.055751+0000","last_became_active":"2023-10-09T10:46:33.364058+0000","last_became_peered":"2023-10-09T10:46:33.364058+0000","last_unstale":"2023-10-09T10:49:01.049671+0000","last_undegraded":"2023-10-09T10:49:01.049671+0000","last_fullsized":"2023-10-09T10:49:01.049671+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7842,"log_dups_size":0,"ondisk_log_size":7842,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3375104,"num_objects":52,"num_object_clones":0,"num_object_copies":156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":52,"num_objects_unfound":0,"num_objects_dirty":52,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6806,"num_write_kb":219424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2010,"num_bytes_recovered":131727360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":52}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":59,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.307502+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:48:55.307502+0000","last_peered":"2023-10-09T10:48:55.307502+0000","last_clean":"2023-10-09T10:48:55.307502+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:48:55.307502+0000","last_undegraded":"2023-10-09T10:48:55.307502+0000","last_fullsized":"2023-10-09T10:48:55.307502+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":158,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.331081+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:48:54.331081+0000","last_peered":"2023-10-09T10:48:54.331081+0000","last_clean":"2023-10-09T10:48:54.331081+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:48:54.331081+0000","last_undegraded":"2023-10-09T10:48:54.331081+0000","last_fullsized":"2023-10-09T10:48:54.331081+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":524,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.306373+0000","last_change":"2023-10-09T10:46:05.620365+0000","last_active":"2023-10-09T10:48:55.306373+0000","last_peered":"2023-10-09T10:48:55.306373+0000","last_clean":"2023-10-09T10:48:55.306373+0000","last_became_active":"2023-10-09T10:46:05.620024+0000","last_became_peered":"2023-10-09T10:46:05.620024+0000","last_unstale":"2023-10-09T10:48:55.306373+0000","last_undegraded":"2023-10-09T10:48:55.306373+0000","last_fullsized":"2023-10-09T10:48:55.306373+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.13","version":"189'7644","reported_seq":14092,"reported_epoch":189,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:49:01.054708+0000","last_change":"2023-10-09T10:48:15.511525+0000","last_active":"2023-10-09T10:49:01.054708+0000","last_peered":"2023-10-09T10:49:01.054708+0000","last_clean":"2023-10-09T10:46:31.195699+0000","last_became_active":"2023-10-09T10:46:33.455632+0000","last_became_peered":"2023-10-09T10:46:33.455632+0000","last_unstale":"2023-10-09T10:49:01.054708+0000","last_undegraded":"2023-10-09T10:46:33.269323+0000","last_fullsized":"2023-10-09T10:46:33.241226+0000","mapping_epoch":154,"log_start":"118'2875","ondisk_log_start":"118'2875","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"115'2715","last_scrub_stamp":"2023-10-09T10:45:07.544515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:07.544515+0000","objects_scrubbed":1673,"log_size":4769,"log_dups_size":0,"ondisk_log_size":4769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:07:12.547333+0000","scrub_duration":0.15319010899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1835008,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":189,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4929,"num_write_kb":105088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3126,"num_bytes_recovered":66060288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"6,7","objects":-161}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":154,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.297772+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:48:54.297772+0000","last_peered":"2023-10-09T10:48:54.297772+0000","last_clean":"2023-10-09T10:48:54.297772+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:48:54.297772+0000","last_undegraded":"2023-10-09T10:48:54.297772+0000","last_fullsized":"2023-10-09T10:48:54.297772+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":115,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.348839+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:48:54.348839+0000","last_peered":"2023-10-09T10:48:54.348839+0000","last_clean":"2023-10-09T10:48:54.348839+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:48:54.348839+0000","last_undegraded":"2023-10-09T10:48:54.348839+0000","last_fullsized":"2023-10-09T10:48:54.348839+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"189'7832","reported_seq":15969,"reported_epoch":189,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:49:01.521834+0000","last_change":"2023-10-09T10:46:03.683196+0000","last_active":"2023-10-09T10:49:01.521834+0000","last_peered":"2023-10-09T10:49:01.521834+0000","last_clean":"2023-10-09T10:46:02.177309+0000","last_became_active":"2023-10-09T10:46:03.681058+0000","last_became_peered":"2023-10-09T10:46:03.681058+0000","last_unstale":"2023-10-09T10:49:01.521834+0000","last_undegraded":"2023-10-09T10:49:01.521834+0000","last_fullsized":"2023-10-09T10:49:01.521834+0000","mapping_epoch":143,"log_start":"103'797","ondisk_log_start":"103'797","created":107,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7035,"log_dups_size":0,"ondisk_log_size":7035,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6786,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3321,"num_bytes_recovered":217645056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[1,5,6],"avail_no_missing":["1","2","3","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":110,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.347840+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:48:54.347840+0000","last_peered":"2023-10-09T10:48:54.347840+0000","last_clean":"2023-10-09T10:48:54.347840+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:48:54.347840+0000","last_undegraded":"2023-10-09T10:48:54.347840+0000","last_fullsized":"2023-10-09T10:48:54.347840+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":150,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.296393+0000","last_change":"2023-10-09T10:46:32.249921+0000","last_active":"2023-10-09T10:48:54.296393+0000","last_peered":"2023-10-09T10:48:54.296393+0000","last_clean":"2023-10-09T10:48:54.296393+0000","last_became_active":"2023-10-09T10:46:32.249590+0000","last_became_peered":"2023-10-09T10:46:32.249590+0000","last_unstale":"2023-10-09T10:48:54.296393+0000","last_undegraded":"2023-10-09T10:48:54.296393+0000","last_fullsized":"2023-10-09T10:48:54.296393+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:25.169752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"189'7532","reported_seq":15763,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:59.582884+0000","last_change":"2023-10-09T10:47:48.412928+0000","last_active":"2023-10-09T10:48:59.582884+0000","last_peered":"2023-10-09T10:48:59.582884+0000","last_clean":"2023-10-09T10:48:59.582884+0000","last_became_active":"2023-10-09T10:47:48.398813+0000","last_became_peered":"2023-10-09T10:47:48.398813+0000","last_unstale":"2023-10-09T10:48:59.582884+0000","last_undegraded":"2023-10-09T10:48:59.582884+0000","last_fullsized":"2023-10-09T10:48:59.582884+0000","mapping_epoch":177,"log_start":"105'1708","ondisk_log_start":"105'1708","created":107,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3178496,"num_objects":49,"num_object_clones":0,"num_object_copies":147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":49,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6525,"num_write_kb":210336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2691,"num_bytes_recovered":176357376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":159,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.299399+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:48:54.299399+0000","last_peered":"2023-10-09T10:48:54.299399+0000","last_clean":"2023-10-09T10:48:54.299399+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:48:54.299399+0000","last_undegraded":"2023-10-09T10:48:54.299399+0000","last_fullsized":"2023-10-09T10:48:54.299399+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":153,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.299049+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:48:54.299049+0000","last_peered":"2023-10-09T10:48:54.299049+0000","last_clean":"2023-10-09T10:48:54.299049+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:48:54.299049+0000","last_undegraded":"2023-10-09T10:48:54.299049+0000","last_fullsized":"2023-10-09T10:48:54.299049+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":524,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.308181+0000","last_change":"2023-10-09T10:46:05.618276+0000","last_active":"2023-10-09T10:48:55.308181+0000","last_peered":"2023-10-09T10:48:55.308181+0000","last_clean":"2023-10-09T10:48:55.308181+0000","last_became_active":"2023-10-09T10:46:05.616509+0000","last_became_peered":"2023-10-09T10:46:05.616509+0000","last_unstale":"2023-10-09T10:48:55.308181+0000","last_undegraded":"2023-10-09T10:48:55.308181+0000","last_fullsized":"2023-10-09T10:48:55.308181+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.16","version":"189'7555","reported_seq":7669,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:59.570143+0000","last_change":"2023-10-09T10:46:03.592682+0000","last_active":"2023-10-09T10:48:59.570143+0000","last_peered":"2023-10-09T10:48:59.570143+0000","last_clean":"2023-10-09T10:48:59.570143+0000","last_became_active":"2023-10-09T10:44:50.084978+0000","last_became_peered":"2023-10-09T10:44:50.084978+0000","last_unstale":"2023-10-09T10:48:59.570143+0000","last_undegraded":"2023-10-09T10:48:59.570143+0000","last_fullsized":"2023-10-09T10:48:59.570143+0000","mapping_epoch":107,"log_start":"105'1682","ondisk_log_start":"105'1682","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5873,"log_dups_size":0,"ondisk_log_size":5873,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4849664,"num_objects":74,"num_object_clones":0,"num_object_copies":222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6512,"num_write_kb":210752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":171,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.309653+0000","last_change":"2023-10-09T10:46:32.250256+0000","last_active":"2023-10-09T10:48:54.309653+0000","last_peered":"2023-10-09T10:48:54.309653+0000","last_clean":"2023-10-09T10:48:54.309653+0000","last_became_active":"2023-10-09T10:46:32.243312+0000","last_became_peered":"2023-10-09T10:46:32.243312+0000","last_unstale":"2023-10-09T10:48:54.309653+0000","last_undegraded":"2023-10-09T10:48:54.309653+0000","last_fullsized":"2023-10-09T10:48:54.309653+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:08.059777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":181,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.338489+0000","last_change":"2023-10-09T10:46:32.237577+0000","last_active":"2023-10-09T10:48:54.338489+0000","last_peered":"2023-10-09T10:48:54.338489+0000","last_clean":"2023-10-09T10:48:54.338489+0000","last_became_active":"2023-10-09T10:46:32.237316+0000","last_became_peered":"2023-10-09T10:46:32.237316+0000","last_unstale":"2023-10-09T10:48:54.338489+0000","last_undegraded":"2023-10-09T10:48:54.338489+0000","last_fullsized":"2023-10-09T10:48:54.338489+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:39.684602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":525,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.305501+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:48:55.305501+0000","last_peered":"2023-10-09T10:48:55.305501+0000","last_clean":"2023-10-09T10:48:55.305501+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:48:55.305501+0000","last_undegraded":"2023-10-09T10:48:55.305501+0000","last_fullsized":"2023-10-09T10:48:55.305501+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.17","version":"189'7516","reported_seq":7624,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:49:01.162859+0000","last_change":"2023-10-09T10:44:50.087507+0000","last_active":"2023-10-09T10:49:01.162859+0000","last_peered":"2023-10-09T10:49:01.162859+0000","last_clean":"2023-10-09T10:49:01.162859+0000","last_became_active":"2023-10-09T10:44:50.085252+0000","last_became_peered":"2023-10-09T10:44:50.085252+0000","last_unstale":"2023-10-09T10:49:01.162859+0000","last_undegraded":"2023-10-09T10:49:01.162859+0000","last_fullsized":"2023-10-09T10:49:01.162859+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7516,"log_dups_size":0,"ondisk_log_size":7516,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6508,"num_write_kb":208192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":59,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.307094+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:48:55.307094+0000","last_peered":"2023-10-09T10:48:55.307094+0000","last_clean":"2023-10-09T10:48:55.307094+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:48:55.307094+0000","last_undegraded":"2023-10-09T10:48:55.307094+0000","last_fullsized":"2023-10-09T10:48:55.307094+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":153,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.290594+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:48:54.290594+0000","last_peered":"2023-10-09T10:48:54.290594+0000","last_clean":"2023-10-09T10:48:54.290594+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:48:54.290594+0000","last_undegraded":"2023-10-09T10:48:54.290594+0000","last_fullsized":"2023-10-09T10:48:54.290594+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":525,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.305685+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:48:55.305685+0000","last_peered":"2023-10-09T10:48:55.305685+0000","last_clean":"2023-10-09T10:48:55.305685+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:48:55.305685+0000","last_undegraded":"2023-10-09T10:48:55.305685+0000","last_fullsized":"2023-10-09T10:48:55.305685+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"189'7734","reported_seq":7847,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:49:01.484320+0000","last_change":"2023-10-09T10:46:33.243540+0000","last_active":"2023-10-09T10:49:01.484320+0000","last_peered":"2023-10-09T10:49:01.484320+0000","last_clean":"2023-10-09T10:49:01.484320+0000","last_became_active":"2023-10-09T10:44:50.083539+0000","last_became_peered":"2023-10-09T10:44:50.083539+0000","last_unstale":"2023-10-09T10:49:01.484320+0000","last_undegraded":"2023-10-09T10:49:01.484320+0000","last_fullsized":"2023-10-09T10:49:01.484320+0000","mapping_epoch":107,"log_start":"103'624","ondisk_log_start":"103'624","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7110,"log_dups_size":0,"ondisk_log_size":7110,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6680,"num_write_kb":213536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":99,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.299023+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:48:54.299023+0000","last_peered":"2023-10-09T10:48:54.299023+0000","last_clean":"2023-10-09T10:48:54.299023+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:48:54.299023+0000","last_undegraded":"2023-10-09T10:48:54.299023+0000","last_fullsized":"2023-10-09T10:48:54.299023+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":59,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.306986+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:48:55.306986+0000","last_peered":"2023-10-09T10:48:55.306986+0000","last_clean":"2023-10-09T10:48:55.306986+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:48:55.306986+0000","last_undegraded":"2023-10-09T10:48:55.306986+0000","last_fullsized":"2023-10-09T10:48:55.306986+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":525,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.306960+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:48:55.306960+0000","last_peered":"2023-10-09T10:48:55.306960+0000","last_clean":"2023-10-09T10:48:55.306960+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:48:55.306960+0000","last_undegraded":"2023-10-09T10:48:55.306960+0000","last_fullsized":"2023-10-09T10:48:55.306960+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"189'7692","reported_seq":7815,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:59.620025+0000","last_change":"2023-10-09T10:45:07.623326+0000","last_active":"2023-10-09T10:48:59.620025+0000","last_peered":"2023-10-09T10:48:59.620025+0000","last_clean":"2023-10-09T10:48:59.620025+0000","last_became_active":"2023-10-09T10:45:07.621836+0000","last_became_peered":"2023-10-09T10:45:07.621836+0000","last_unstale":"2023-10-09T10:48:59.620025+0000","last_undegraded":"2023-10-09T10:48:59.620025+0000","last_fullsized":"2023-10-09T10:48:59.620025+0000","mapping_epoch":115,"log_start":"106'1935","ondisk_log_start":"106'1935","created":107,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5757,"log_dups_size":0,"ondisk_log_size":5757,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2424832,"num_objects":37,"num_object_clones":0,"num_object_copies":111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6631,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":69,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.305418+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:48:55.305418+0000","last_peered":"2023-10-09T10:48:55.305418+0000","last_clean":"2023-10-09T10:48:55.305418+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:48:55.305418+0000","last_undegraded":"2023-10-09T10:48:55.305418+0000","last_fullsized":"2023-10-09T10:48:55.305418+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":158,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.350538+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:48:54.350538+0000","last_peered":"2023-10-09T10:48:54.350538+0000","last_clean":"2023-10-09T10:48:54.350538+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:48:54.350538+0000","last_undegraded":"2023-10-09T10:48:54.350538+0000","last_fullsized":"2023-10-09T10:48:54.350538+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":525,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.305842+0000","last_change":"2023-10-09T10:46:05.620574+0000","last_active":"2023-10-09T10:48:55.305842+0000","last_peered":"2023-10-09T10:48:55.305842+0000","last_clean":"2023-10-09T10:48:55.305842+0000","last_became_active":"2023-10-09T10:46:05.620212+0000","last_became_peered":"2023-10-09T10:46:05.620212+0000","last_unstale":"2023-10-09T10:48:55.305842+0000","last_undegraded":"2023-10-09T10:48:55.305842+0000","last_fullsized":"2023-10-09T10:48:55.305842+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"189'13570","reported_seq":17734,"reported_epoch":189,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:49:01.176875+0000","last_change":"2023-10-09T10:46:32.247903+0000","last_active":"2023-10-09T10:49:01.176875+0000","last_peered":"2023-10-09T10:49:01.176875+0000","last_clean":"2023-10-09T10:45:14.377855+0000","last_became_active":"2023-10-09T10:46:02.739355+0000","last_became_peered":"2023-10-09T10:46:02.739355+0000","last_unstale":"2023-10-09T10:49:01.176875+0000","last_undegraded":"2023-10-09T10:46:02.608805+0000","last_fullsized":"2023-10-09T10:46:02.583589+0000","mapping_epoch":142,"log_start":"125'4700","ondisk_log_start":"125'4700","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8870,"log_dups_size":0,"ondisk_log_size":8870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:32.899634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4063232,"num_objects":62,"num_object_clones":0,"num_object_copies":186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13570,"num_write_kb":436224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2022,"num_bytes_recovered":132513792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":59},{"shards":"5,6","objects":-564}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":182,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.306742+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:48:54.306742+0000","last_peered":"2023-10-09T10:48:54.306742+0000","last_clean":"2023-10-09T10:48:54.306742+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:48:54.306742+0000","last_undegraded":"2023-10-09T10:48:54.306742+0000","last_fullsized":"2023-10-09T10:48:54.306742+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":136,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.289541+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:48:54.289541+0000","last_peered":"2023-10-09T10:48:54.289541+0000","last_clean":"2023-10-09T10:48:54.289541+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:48:54.289541+0000","last_undegraded":"2023-10-09T10:48:54.289541+0000","last_fullsized":"2023-10-09T10:48:54.289541+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":505,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.306504+0000","last_change":"2023-10-09T10:46:02.631378+0000","last_active":"2023-10-09T10:48:55.306504+0000","last_peered":"2023-10-09T10:48:55.306504+0000","last_clean":"2023-10-09T10:48:55.306504+0000","last_became_active":"2023-10-09T10:46:02.629142+0000","last_became_peered":"2023-10-09T10:46:02.629142+0000","last_unstale":"2023-10-09T10:48:55.306504+0000","last_undegraded":"2023-10-09T10:48:55.306504+0000","last_fullsized":"2023-10-09T10:48:55.306504+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.b","version":"189'13047","reported_seq":13148,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:59.623459+0000","last_change":"2023-10-09T10:44:27.571018+0000","last_active":"2023-10-09T10:48:59.623459+0000","last_peered":"2023-10-09T10:48:59.623459+0000","last_clean":"2023-10-09T10:48:59.623459+0000","last_became_active":"2023-10-09T10:44:27.570591+0000","last_became_peered":"2023-10-09T10:44:27.570591+0000","last_unstale":"2023-10-09T10:48:59.623459+0000","last_undegraded":"2023-10-09T10:48:59.623459+0000","last_fullsized":"2023-10-09T10:48:59.623459+0000","mapping_epoch":98,"log_start":"151'7325","ondisk_log_start":"151'7325","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5722,"log_dups_size":0,"ondisk_log_size":5722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:37.811555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8716288,"num_objects":133,"num_object_clones":0,"num_object_copies":399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13047,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":155,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.306657+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:48:54.306657+0000","last_peered":"2023-10-09T10:48:54.306657+0000","last_clean":"2023-10-09T10:48:54.306657+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:48:54.306657+0000","last_undegraded":"2023-10-09T10:48:54.306657+0000","last_fullsized":"2023-10-09T10:48:54.306657+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":190,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.298978+0000","last_change":"2023-10-09T10:46:32.251035+0000","last_active":"2023-10-09T10:48:54.298978+0000","last_peered":"2023-10-09T10:48:54.298978+0000","last_clean":"2023-10-09T10:48:54.298978+0000","last_became_active":"2023-10-09T10:46:32.250293+0000","last_became_peered":"2023-10-09T10:46:32.250293+0000","last_unstale":"2023-10-09T10:48:54.298978+0000","last_undegraded":"2023-10-09T10:48:54.298978+0000","last_fullsized":"2023-10-09T10:48:54.298978+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.787766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":525,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.307202+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:48:55.307202+0000","last_peered":"2023-10-09T10:48:55.307202+0000","last_clean":"2023-10-09T10:48:55.307202+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:48:55.307202+0000","last_undegraded":"2023-10-09T10:48:55.307202+0000","last_fullsized":"2023-10-09T10:48:55.307202+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.8","version":"189'7662","reported_seq":7784,"reported_epoch":189,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:59.642316+0000","last_change":"2023-10-09T10:46:33.540719+0000","last_active":"2023-10-09T10:48:59.642316+0000","last_peered":"2023-10-09T10:48:59.642316+0000","last_clean":"2023-10-09T10:46:31.137696+0000","last_became_active":"2023-10-09T10:46:33.527650+0000","last_became_peered":"2023-10-09T10:46:33.527650+0000","last_unstale":"2023-10-09T10:48:59.642316+0000","last_undegraded":"2023-10-09T10:46:33.285468+0000","last_fullsized":"2023-10-09T10:46:33.235683+0000","mapping_epoch":154,"log_start":"105'1865","ondisk_log_start":"105'1865","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5797,"log_dups_size":0,"ondisk_log_size":5797,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5865472,"num_objects":89,"num_object_clones":0,"num_object_copies":267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6597,"num_write_kb":213984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":77},{"shards":"3,7","objects":-2720}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":110,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.290549+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:48:54.290549+0000","last_peered":"2023-10-09T10:48:54.290549+0000","last_clean":"2023-10-09T10:48:54.290549+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:48:54.290549+0000","last_undegraded":"2023-10-09T10:48:54.290549+0000","last_fullsized":"2023-10-09T10:48:54.290549+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":127,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.296652+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:48:54.296652+0000","last_peered":"2023-10-09T10:48:54.296652+0000","last_clean":"2023-10-09T10:48:54.296652+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:48:54.296652+0000","last_undegraded":"2023-10-09T10:48:54.296652+0000","last_fullsized":"2023-10-09T10:48:54.296652+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":525,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.306594+0000","last_change":"2023-10-09T10:46:05.620030+0000","last_active":"2023-10-09T10:48:55.306594+0000","last_peered":"2023-10-09T10:48:55.306594+0000","last_clean":"2023-10-09T10:48:55.306594+0000","last_became_active":"2023-10-09T10:46:05.619694+0000","last_became_peered":"2023-10-09T10:46:05.619694+0000","last_unstale":"2023-10-09T10:48:55.306594+0000","last_undegraded":"2023-10-09T10:48:55.306594+0000","last_fullsized":"2023-10-09T10:48:55.306594+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.9","version":"190'7655","reported_seq":11808,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T10:49:02.688092+0000","last_change":"2023-10-09T10:48:55.343388+0000","last_active":"2023-10-09T10:49:02.688092+0000","last_peered":"2023-10-09T10:49:02.688092+0000","last_clean":"2023-10-09T10:49:02.688092+0000","last_became_active":"2023-10-09T10:48:55.324380+0000","last_became_peered":"2023-10-09T10:48:55.324380+0000","last_unstale":"2023-10-09T10:49:02.688092+0000","last_undegraded":"2023-10-09T10:49:02.688092+0000","last_fullsized":"2023-10-09T10:49:02.688092+0000","mapping_epoch":188,"log_start":"111'2403","ondisk_log_start":"111'2403","created":98,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5252,"log_dups_size":0,"ondisk_log_size":5252,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65536,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6613,"num_write_kb":211648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3307,"num_bytes_recovered":85458944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":154,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.290831+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:48:54.290831+0000","last_peered":"2023-10-09T10:48:54.290831+0000","last_clean":"2023-10-09T10:48:54.290831+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:48:54.290831+0000","last_undegraded":"2023-10-09T10:48:54.290831+0000","last_fullsized":"2023-10-09T10:48:54.290831+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":59,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.306085+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:48:55.306085+0000","last_peered":"2023-10-09T10:48:55.306085+0000","last_clean":"2023-10-09T10:48:55.306085+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:48:55.306085+0000","last_undegraded":"2023-10-09T10:48:55.306085+0000","last_fullsized":"2023-10-09T10:48:55.306085+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":516,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.308102+0000","last_change":"2023-10-09T10:46:05.618134+0000","last_active":"2023-10-09T10:48:55.308102+0000","last_peered":"2023-10-09T10:48:55.308102+0000","last_clean":"2023-10-09T10:48:55.308102+0000","last_became_active":"2023-10-09T10:46:05.616057+0000","last_became_peered":"2023-10-09T10:46:05.616057+0000","last_unstale":"2023-10-09T10:48:55.308102+0000","last_undegraded":"2023-10-09T10:48:55.308102+0000","last_fullsized":"2023-10-09T10:48:55.308102+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"189'13160","reported_seq":13269,"reported_epoch":189,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:49:01.075656+0000","last_change":"2023-10-09T10:46:33.576530+0000","last_active":"2023-10-09T10:49:01.075656+0000","last_peered":"2023-10-09T10:49:01.075656+0000","last_clean":"2023-10-09T10:46:31.183481+0000","last_became_active":"2023-10-09T10:46:33.542304+0000","last_became_peered":"2023-10-09T10:46:33.542304+0000","last_unstale":"2023-10-09T10:49:01.075656+0000","last_undegraded":"2023-10-09T10:46:33.300581+0000","last_fullsized":"2023-10-09T10:46:33.246047+0000","mapping_epoch":154,"log_start":"136'5777","ondisk_log_start":"136'5777","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7383,"log_dups_size":0,"ondisk_log_size":7383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:26.739488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4849664,"num_objects":74,"num_object_clones":0,"num_object_copies":222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6617,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13160,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":74},{"shards":"1,4","objects":-5487}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":182,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.347552+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:48:54.347552+0000","last_peered":"2023-10-09T10:48:54.347552+0000","last_clean":"2023-10-09T10:48:54.347552+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:48:54.347552+0000","last_undegraded":"2023-10-09T10:48:54.347552+0000","last_fullsized":"2023-10-09T10:48:54.347552+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":153,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.299358+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:48:54.299358+0000","last_peered":"2023-10-09T10:48:54.299358+0000","last_clean":"2023-10-09T10:48:54.299358+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:48:54.299358+0000","last_undegraded":"2023-10-09T10:48:54.299358+0000","last_fullsized":"2023-10-09T10:48:54.299358+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":505,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.305581+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:48:55.305581+0000","last_peered":"2023-10-09T10:48:55.305581+0000","last_clean":"2023-10-09T10:48:55.305581+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:48:55.305581+0000","last_undegraded":"2023-10-09T10:48:55.305581+0000","last_fullsized":"2023-10-09T10:48:55.305581+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.f","version":"189'12894","reported_seq":13001,"reported_epoch":189,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:59.604063+0000","last_change":"2023-10-09T10:46:33.651756+0000","last_active":"2023-10-09T10:48:59.604063+0000","last_peered":"2023-10-09T10:48:59.604063+0000","last_clean":"2023-10-09T10:46:31.156458+0000","last_became_active":"2023-10-09T10:46:33.621003+0000","last_became_peered":"2023-10-09T10:46:33.621003+0000","last_unstale":"2023-10-09T10:48:59.604063+0000","last_undegraded":"2023-10-09T10:46:33.309813+0000","last_fullsized":"2023-10-09T10:46:33.236508+0000","mapping_epoch":154,"log_start":"151'7210","ondisk_log_start":"151'7210","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5684,"log_dups_size":0,"ondisk_log_size":5684,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:11:13.704346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8388608,"num_objects":128,"num_object_clones":0,"num_object_copies":384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12894,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":128},{"shards":"3,4","objects":-5324}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":128,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.299125+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:48:54.299125+0000","last_peered":"2023-10-09T10:48:54.299125+0000","last_clean":"2023-10-09T10:48:54.299125+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:48:54.299125+0000","last_undegraded":"2023-10-09T10:48:54.299125+0000","last_fullsized":"2023-10-09T10:48:54.299125+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":180,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.299457+0000","last_change":"2023-10-09T10:46:32.245583+0000","last_active":"2023-10-09T10:48:54.299457+0000","last_peered":"2023-10-09T10:48:54.299457+0000","last_clean":"2023-10-09T10:48:54.299457+0000","last_became_active":"2023-10-09T10:46:32.245271+0000","last_became_peered":"2023-10-09T10:46:32.245271+0000","last_unstale":"2023-10-09T10:48:54.299457+0000","last_undegraded":"2023-10-09T10:48:54.299457+0000","last_fullsized":"2023-10-09T10:48:54.299457+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.025051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":525,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.305849+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:48:55.305849+0000","last_peered":"2023-10-09T10:48:55.305849+0000","last_clean":"2023-10-09T10:48:55.305849+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:48:55.305849+0000","last_undegraded":"2023-10-09T10:48:55.305849+0000","last_fullsized":"2023-10-09T10:48:55.305849+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"189'13151","reported_seq":13251,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:59.623697+0000","last_change":"2023-10-09T10:44:27.568159+0000","last_active":"2023-10-09T10:48:59.623697+0000","last_peered":"2023-10-09T10:48:59.623697+0000","last_clean":"2023-10-09T10:48:59.623697+0000","last_became_active":"2023-10-09T10:44:27.567657+0000","last_became_peered":"2023-10-09T10:44:27.567657+0000","last_unstale":"2023-10-09T10:48:59.623697+0000","last_undegraded":"2023-10-09T10:48:59.623697+0000","last_fullsized":"2023-10-09T10:48:59.623697+0000","mapping_epoch":98,"log_start":"152'7478","ondisk_log_start":"152'7478","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5673,"log_dups_size":0,"ondisk_log_size":5673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:18.422764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8716288,"num_objects":133,"num_object_clones":0,"num_object_copies":399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13151,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":164,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.298931+0000","last_change":"2023-10-09T10:46:32.245685+0000","last_active":"2023-10-09T10:48:54.298931+0000","last_peered":"2023-10-09T10:48:54.298931+0000","last_clean":"2023-10-09T10:48:54.298931+0000","last_became_active":"2023-10-09T10:46:32.245314+0000","last_became_peered":"2023-10-09T10:46:32.245314+0000","last_unstale":"2023-10-09T10:48:54.298931+0000","last_undegraded":"2023-10-09T10:48:54.298931+0000","last_fullsized":"2023-10-09T10:48:54.298931+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:51.716860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":153,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.299584+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:48:54.299584+0000","last_peered":"2023-10-09T10:48:54.299584+0000","last_clean":"2023-10-09T10:48:54.299584+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:48:54.299584+0000","last_undegraded":"2023-10-09T10:48:54.299584+0000","last_fullsized":"2023-10-09T10:48:54.299584+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":516,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.307007+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:48:55.307007+0000","last_peered":"2023-10-09T10:48:55.307007+0000","last_clean":"2023-10-09T10:48:55.307007+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:48:55.307007+0000","last_undegraded":"2023-10-09T10:48:55.307007+0000","last_fullsized":"2023-10-09T10:48:55.307007+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"189'13327","reported_seq":13446,"reported_epoch":189,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:49:01.179504+0000","last_change":"2023-10-09T10:46:02.754507+0000","last_active":"2023-10-09T10:49:01.179504+0000","last_peered":"2023-10-09T10:49:01.179504+0000","last_clean":"2023-10-09T10:45:14.390049+0000","last_became_active":"2023-10-09T10:46:02.747510+0000","last_became_peered":"2023-10-09T10:46:02.747510+0000","last_unstale":"2023-10-09T10:49:01.179504+0000","last_undegraded":"2023-10-09T10:46:02.605987+0000","last_fullsized":"2023-10-09T10:46:02.585248+0000","mapping_epoch":142,"log_start":"124'4500","ondisk_log_start":"124'4500","created":98,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":8827,"log_dups_size":0,"ondisk_log_size":8827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:59.606594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4456472,"num_objects":69,"num_object_clones":0,"num_object_copies":207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13327,"num_write_kb":428609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,2,5","objects":68},{"shards":"1,5","objects":-2649}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":159,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.330882+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:48:54.330882+0000","last_peered":"2023-10-09T10:48:54.330882+0000","last_clean":"2023-10-09T10:48:54.330882+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:48:54.330882+0000","last_undegraded":"2023-10-09T10:48:54.330882+0000","last_fullsized":"2023-10-09T10:48:54.330882+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":141,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.299431+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:48:54.299431+0000","last_peered":"2023-10-09T10:48:54.299431+0000","last_clean":"2023-10-09T10:48:54.299431+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:48:54.299431+0000","last_undegraded":"2023-10-09T10:48:54.299431+0000","last_fullsized":"2023-10-09T10:48:54.299431+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":525,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.307374+0000","last_change":"2023-10-09T10:46:05.617983+0000","last_active":"2023-10-09T10:48:55.307374+0000","last_peered":"2023-10-09T10:48:55.307374+0000","last_clean":"2023-10-09T10:48:55.307374+0000","last_became_active":"2023-10-09T10:46:05.616021+0000","last_became_peered":"2023-10-09T10:46:05.616021+0000","last_unstale":"2023-10-09T10:48:55.307374+0000","last_undegraded":"2023-10-09T10:48:55.307374+0000","last_fullsized":"2023-10-09T10:48:55.307374+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":185,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.300778+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:48:54.300778+0000","last_peered":"2023-10-09T10:48:54.300778+0000","last_clean":"2023-10-09T10:48:54.300778+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:48:54.300778+0000","last_undegraded":"2023-10-09T10:48:54.300778+0000","last_fullsized":"2023-10-09T10:48:54.300778+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":181,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.290636+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:48:54.290636+0000","last_peered":"2023-10-09T10:48:54.290636+0000","last_clean":"2023-10-09T10:48:54.290636+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:48:54.290636+0000","last_undegraded":"2023-10-09T10:48:54.290636+0000","last_fullsized":"2023-10-09T10:48:54.290636+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":127,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.296470+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:48:54.296470+0000","last_peered":"2023-10-09T10:48:54.296470+0000","last_clean":"2023-10-09T10:48:54.296470+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:48:54.296470+0000","last_undegraded":"2023-10-09T10:48:54.296470+0000","last_fullsized":"2023-10-09T10:48:54.296470+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"189'7594","reported_seq":11640,"reported_epoch":189,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:48:59.680525+0000","last_change":"2023-10-09T10:46:32.248218+0000","last_active":"2023-10-09T10:48:59.680525+0000","last_peered":"2023-10-09T10:48:59.680525+0000","last_clean":"2023-10-09T10:46:02.181912+0000","last_became_active":"2023-10-09T10:46:32.226237+0000","last_became_peered":"2023-10-09T10:46:32.226237+0000","last_unstale":"2023-10-09T10:48:59.680525+0000","last_undegraded":"2023-10-09T10:48:59.680525+0000","last_fullsized":"2023-10-09T10:48:59.680525+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7594,"log_dups_size":0,"ondisk_log_size":7594,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6389760,"num_objects":97,"num_object_clones":0,"num_object_copies":291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":97,"num_objects_unfound":0,"num_objects_dirty":97,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6547,"num_write_kb":212640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1951,"num_bytes_recovered":127860736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":97}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":524,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.305609+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:48:55.305609+0000","last_peered":"2023-10-09T10:48:55.305609+0000","last_clean":"2023-10-09T10:48:55.305609+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:48:55.305609+0000","last_undegraded":"2023-10-09T10:48:55.305609+0000","last_fullsized":"2023-10-09T10:48:55.305609+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"189'7721","reported_seq":7913,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:59.522647+0000","last_change":"2023-10-09T10:46:02.760963+0000","last_active":"2023-10-09T10:48:59.522647+0000","last_peered":"2023-10-09T10:48:59.522647+0000","last_clean":"2023-10-09T10:48:59.522647+0000","last_became_active":"2023-10-09T10:44:50.074981+0000","last_became_peered":"2023-10-09T10:44:50.074981+0000","last_unstale":"2023-10-09T10:48:59.522647+0000","last_undegraded":"2023-10-09T10:48:59.522647+0000","last_fullsized":"2023-10-09T10:48:59.522647+0000","mapping_epoch":98,"log_start":"106'1965","ondisk_log_start":"106'1965","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"141'4416","last_scrub_stamp":"2023-10-09T10:46:02.760810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:46:02.760810+0000","objects_scrubbed":3378,"log_size":5756,"log_dups_size":0,"ondisk_log_size":5756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:35.863002+0000","scrub_duration":0.47769323899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4784128,"num_objects":73,"num_object_clones":0,"num_object_copies":219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3305,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":158,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.293941+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:48:54.293941+0000","last_peered":"2023-10-09T10:48:54.293941+0000","last_clean":"2023-10-09T10:48:54.293941+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:48:54.293941+0000","last_undegraded":"2023-10-09T10:48:54.293941+0000","last_fullsized":"2023-10-09T10:48:54.293941+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":174,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.299166+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:48:54.299166+0000","last_peered":"2023-10-09T10:48:54.299166+0000","last_clean":"2023-10-09T10:48:54.299166+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:48:54.299166+0000","last_undegraded":"2023-10-09T10:48:54.299166+0000","last_fullsized":"2023-10-09T10:48:54.299166+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":166,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.300847+0000","last_change":"2023-10-09T10:46:32.245608+0000","last_active":"2023-10-09T10:48:54.300847+0000","last_peered":"2023-10-09T10:48:54.300847+0000","last_clean":"2023-10-09T10:48:54.300847+0000","last_became_active":"2023-10-09T10:46:32.245290+0000","last_became_peered":"2023-10-09T10:46:32.245290+0000","last_unstale":"2023-10-09T10:48:54.300847+0000","last_undegraded":"2023-10-09T10:48:54.300847+0000","last_fullsized":"2023-10-09T10:48:54.300847+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:48.832515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":525,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.307050+0000","last_change":"2023-10-09T10:46:05.620057+0000","last_active":"2023-10-09T10:48:55.307050+0000","last_peered":"2023-10-09T10:48:55.307050+0000","last_clean":"2023-10-09T10:48:55.307050+0000","last_became_active":"2023-10-09T10:46:05.619703+0000","last_became_peered":"2023-10-09T10:46:05.619703+0000","last_unstale":"2023-10-09T10:48:55.307050+0000","last_undegraded":"2023-10-09T10:48:55.307050+0000","last_fullsized":"2023-10-09T10:48:55.307050+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"189'7657","reported_seq":7783,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:59.615386+0000","last_change":"2023-10-09T10:45:07.623081+0000","last_active":"2023-10-09T10:48:59.615386+0000","last_peered":"2023-10-09T10:48:59.615386+0000","last_clean":"2023-10-09T10:48:59.615386+0000","last_became_active":"2023-10-09T10:45:07.621464+0000","last_became_peered":"2023-10-09T10:45:07.621464+0000","last_unstale":"2023-10-09T10:48:59.615386+0000","last_undegraded":"2023-10-09T10:48:59.615386+0000","last_fullsized":"2023-10-09T10:48:59.615386+0000","mapping_epoch":115,"log_start":"105'1861","ondisk_log_start":"105'1861","created":98,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5796,"log_dups_size":0,"ondisk_log_size":5796,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5505024,"num_objects":84,"num_object_clones":0,"num_object_copies":252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":84,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6596,"num_write_kb":213760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":187,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.348928+0000","last_change":"2023-10-09T10:46:32.241041+0000","last_active":"2023-10-09T10:48:54.348928+0000","last_peered":"2023-10-09T10:48:54.348928+0000","last_clean":"2023-10-09T10:48:54.348928+0000","last_became_active":"2023-10-09T10:46:32.238683+0000","last_became_peered":"2023-10-09T10:46:32.238683+0000","last_unstale":"2023-10-09T10:48:54.348928+0000","last_undegraded":"2023-10-09T10:48:54.348928+0000","last_fullsized":"2023-10-09T10:48:54.348928+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:25.817713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":151,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.298661+0000","last_change":"2023-10-09T10:46:32.250104+0000","last_active":"2023-10-09T10:48:54.298661+0000","last_peered":"2023-10-09T10:48:54.298661+0000","last_clean":"2023-10-09T10:48:54.298661+0000","last_became_active":"2023-10-09T10:46:32.249805+0000","last_became_peered":"2023-10-09T10:46:32.249805+0000","last_unstale":"2023-10-09T10:48:54.298661+0000","last_undegraded":"2023-10-09T10:48:54.298661+0000","last_fullsized":"2023-10-09T10:48:54.298661+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:07.230672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":179,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.299491+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:48:54.299491+0000","last_peered":"2023-10-09T10:48:54.299491+0000","last_clean":"2023-10-09T10:48:54.299491+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:48:54.299491+0000","last_undegraded":"2023-10-09T10:48:54.299491+0000","last_fullsized":"2023-10-09T10:48:54.299491+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"189'7583","reported_seq":7696,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:49:01.162759+0000","last_change":"2023-10-09T10:44:50.089798+0000","last_active":"2023-10-09T10:49:01.162759+0000","last_peered":"2023-10-09T10:49:01.162759+0000","last_clean":"2023-10-09T10:49:01.162759+0000","last_became_active":"2023-10-09T10:44:50.080431+0000","last_became_peered":"2023-10-09T10:44:50.080431+0000","last_unstale":"2023-10-09T10:49:01.162759+0000","last_undegraded":"2023-10-09T10:49:01.162759+0000","last_fullsized":"2023-10-09T10:49:01.162759+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7583,"log_dups_size":0,"ondisk_log_size":7583,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4915200,"num_objects":75,"num_object_clones":0,"num_object_copies":225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":75,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6575,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":168,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.289411+0000","last_change":"2023-10-09T10:46:32.235346+0000","last_active":"2023-10-09T10:48:54.289411+0000","last_peered":"2023-10-09T10:48:54.289411+0000","last_clean":"2023-10-09T10:48:54.289411+0000","last_became_active":"2023-10-09T10:46:32.234906+0000","last_became_peered":"2023-10-09T10:46:32.234906+0000","last_unstale":"2023-10-09T10:48:54.289411+0000","last_undegraded":"2023-10-09T10:48:54.289411+0000","last_fullsized":"2023-10-09T10:48:54.289411+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.199251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":525,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.305322+0000","last_change":"2023-10-09T10:46:05.617146+0000","last_active":"2023-10-09T10:48:55.305322+0000","last_peered":"2023-10-09T10:48:55.305322+0000","last_clean":"2023-10-09T10:48:55.305322+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:48:55.305322+0000","last_undegraded":"2023-10-09T10:48:55.305322+0000","last_fullsized":"2023-10-09T10:48:55.305322+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":117,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.346598+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:48:54.346598+0000","last_peered":"2023-10-09T10:48:54.346598+0000","last_clean":"2023-10-09T10:48:54.346598+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:48:54.346598+0000","last_undegraded":"2023-10-09T10:48:54.346598+0000","last_fullsized":"2023-10-09T10:48:54.346598+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":86,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.299380+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:48:54.299380+0000","last_peered":"2023-10-09T10:48:54.299380+0000","last_clean":"2023-10-09T10:48:54.299380+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:48:54.299380+0000","last_undegraded":"2023-10-09T10:48:54.299380+0000","last_fullsized":"2023-10-09T10:48:54.299380+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":525,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.305767+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:48:55.305767+0000","last_peered":"2023-10-09T10:48:55.305767+0000","last_clean":"2023-10-09T10:48:55.305767+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:48:55.305767+0000","last_undegraded":"2023-10-09T10:48:55.305767+0000","last_fullsized":"2023-10-09T10:48:55.305767+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":181,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.298775+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:48:54.298775+0000","last_peered":"2023-10-09T10:48:54.298775+0000","last_clean":"2023-10-09T10:48:54.298775+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:48:54.298775+0000","last_undegraded":"2023-10-09T10:48:54.298775+0000","last_fullsized":"2023-10-09T10:48:54.298775+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":68,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.305528+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:48:55.305528+0000","last_peered":"2023-10-09T10:48:55.305528+0000","last_clean":"2023-10-09T10:48:55.305528+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:48:55.305528+0000","last_undegraded":"2023-10-09T10:48:55.305528+0000","last_fullsized":"2023-10-09T10:48:55.305528+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.306910+0000","last_change":"2023-10-09T10:46:32.236202+0000","last_active":"2023-10-09T10:48:55.306910+0000","last_peered":"2023-10-09T10:48:55.306910+0000","last_clean":"2023-10-09T10:48:55.306910+0000","last_became_active":"2023-10-09T10:46:32.235901+0000","last_became_peered":"2023-10-09T10:46:32.235901+0000","last_unstale":"2023-10-09T10:48:55.306910+0000","last_undegraded":"2023-10-09T10:48:55.306910+0000","last_fullsized":"2023-10-09T10:48:55.306910+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:48.260403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"189'7745","reported_seq":7906,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:49:01.518520+0000","last_change":"2023-10-09T10:45:15.490490+0000","last_active":"2023-10-09T10:49:01.518520+0000","last_peered":"2023-10-09T10:49:01.518520+0000","last_clean":"2023-10-09T10:49:01.518520+0000","last_became_active":"2023-10-09T10:44:50.064166+0000","last_became_peered":"2023-10-09T10:44:50.064166+0000","last_unstale":"2023-10-09T10:49:01.518520+0000","last_undegraded":"2023-10-09T10:49:01.518520+0000","last_fullsized":"2023-10-09T10:49:01.518520+0000","mapping_epoch":98,"log_start":"103'585","ondisk_log_start":"103'585","created":98,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'3030","last_scrub_stamp":"2023-10-09T10:45:15.490365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:15.490365+0000","objects_scrubbed":1946,"log_size":7160,"log_dups_size":0,"ondisk_log_size":7160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:21:27.821717+0000","scrub_duration":0.194015084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2162688,"num_objects":33,"num_object_clones":0,"num_object_copies":99,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4715,"num_write_kb":89664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":525,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.306672+0000","last_change":"2023-10-09T10:46:05.620764+0000","last_active":"2023-10-09T10:48:55.306672+0000","last_peered":"2023-10-09T10:48:55.306672+0000","last_clean":"2023-10-09T10:48:55.306672+0000","last_became_active":"2023-10-09T10:46:05.619870+0000","last_became_peered":"2023-10-09T10:46:05.619870+0000","last_unstale":"2023-10-09T10:48:55.306672+0000","last_undegraded":"2023-10-09T10:48:55.306672+0000","last_fullsized":"2023-10-09T10:48:55.306672+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":177,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.306713+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:48:54.306713+0000","last_peered":"2023-10-09T10:48:54.306713+0000","last_clean":"2023-10-09T10:48:54.306713+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:48:54.306713+0000","last_undegraded":"2023-10-09T10:48:54.306713+0000","last_fullsized":"2023-10-09T10:48:54.306713+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":151,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.300931+0000","last_change":"2023-10-09T10:46:32.251450+0000","last_active":"2023-10-09T10:48:54.300931+0000","last_peered":"2023-10-09T10:48:54.300931+0000","last_clean":"2023-10-09T10:48:54.300931+0000","last_became_active":"2023-10-09T10:46:32.250933+0000","last_became_peered":"2023-10-09T10:46:32.250933+0000","last_unstale":"2023-10-09T10:48:54.300931+0000","last_undegraded":"2023-10-09T10:48:54.300931+0000","last_fullsized":"2023-10-09T10:48:54.300931+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:04.116178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":59,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.305926+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:48:55.305926+0000","last_peered":"2023-10-09T10:48:55.305926+0000","last_clean":"2023-10-09T10:48:55.305926+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:48:55.305926+0000","last_undegraded":"2023-10-09T10:48:55.305926+0000","last_fullsized":"2023-10-09T10:48:55.305926+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"189'7576","reported_seq":14282,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:49:01.529425+0000","last_change":"2023-10-09T10:47:02.269289+0000","last_active":"2023-10-09T10:49:01.529425+0000","last_peered":"2023-10-09T10:49:01.529425+0000","last_clean":"2023-10-09T10:49:01.529425+0000","last_became_active":"2023-10-09T10:47:02.257854+0000","last_became_peered":"2023-10-09T10:47:02.257854+0000","last_unstale":"2023-10-09T10:49:01.529425+0000","last_undegraded":"2023-10-09T10:49:01.529425+0000","last_fullsized":"2023-10-09T10:49:01.529425+0000","mapping_epoch":164,"log_start":"103'365","ondisk_log_start":"103'365","created":98,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7211,"log_dups_size":0,"ondisk_log_size":7211,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2260992,"num_objects":34,"num_object_clones":0,"num_object_copies":102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6568,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3287,"num_bytes_recovered":181927936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":525,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.305755+0000","last_change":"2023-10-09T10:46:05.618443+0000","last_active":"2023-10-09T10:48:55.305755+0000","last_peered":"2023-10-09T10:48:55.305755+0000","last_clean":"2023-10-09T10:48:55.305755+0000","last_became_active":"2023-10-09T10:46:05.616588+0000","last_became_peered":"2023-10-09T10:46:05.616588+0000","last_unstale":"2023-10-09T10:48:55.305755+0000","last_undegraded":"2023-10-09T10:48:55.305755+0000","last_fullsized":"2023-10-09T10:48:55.305755+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":182,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.350459+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:48:54.350459+0000","last_peered":"2023-10-09T10:48:54.350459+0000","last_clean":"2023-10-09T10:48:54.350459+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:48:54.350459+0000","last_undegraded":"2023-10-09T10:48:54.350459+0000","last_fullsized":"2023-10-09T10:48:54.350459+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":126,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.301919+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:48:54.301919+0000","last_peered":"2023-10-09T10:48:54.301919+0000","last_clean":"2023-10-09T10:48:54.301919+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:48:54.301919+0000","last_undegraded":"2023-10-09T10:48:54.301919+0000","last_fullsized":"2023-10-09T10:48:54.301919+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":171,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.330950+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:48:54.330950+0000","last_peered":"2023-10-09T10:48:54.330950+0000","last_clean":"2023-10-09T10:48:54.330950+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:48:54.330950+0000","last_undegraded":"2023-10-09T10:48:54.330950+0000","last_fullsized":"2023-10-09T10:48:54.330950+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"189'7674","reported_seq":14519,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:49:01.084237+0000","last_change":"2023-10-09T10:48:17.301310+0000","last_active":"2023-10-09T10:49:01.084237+0000","last_peered":"2023-10-09T10:49:01.084237+0000","last_clean":"2023-10-09T10:49:01.084237+0000","last_became_active":"2023-10-09T10:48:17.283042+0000","last_became_peered":"2023-10-09T10:48:17.283042+0000","last_unstale":"2023-10-09T10:49:01.084237+0000","last_undegraded":"2023-10-09T10:49:01.084237+0000","last_fullsized":"2023-10-09T10:49:01.084237+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":98,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":7674,"log_dups_size":0,"ondisk_log_size":7674,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1802240,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6637,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3337,"num_bytes_recovered":129564672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":524,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.306236+0000","last_change":"2023-10-09T10:46:05.617589+0000","last_active":"2023-10-09T10:48:55.306236+0000","last_peered":"2023-10-09T10:48:55.306236+0000","last_clean":"2023-10-09T10:48:55.306236+0000","last_became_active":"2023-10-09T10:46:05.616400+0000","last_became_peered":"2023-10-09T10:46:05.616400+0000","last_unstale":"2023-10-09T10:48:55.306236+0000","last_undegraded":"2023-10-09T10:48:55.306236+0000","last_fullsized":"2023-10-09T10:48:55.306236+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":47,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.350199+0000","last_change":"2023-10-09T10:46:32.245106+0000","last_active":"2023-10-09T10:48:54.350199+0000","last_peered":"2023-10-09T10:48:54.350199+0000","last_clean":"2023-10-09T10:48:54.350199+0000","last_became_active":"2023-10-09T10:46:32.244826+0000","last_became_peered":"2023-10-09T10:46:32.244826+0000","last_unstale":"2023-10-09T10:48:54.350199+0000","last_undegraded":"2023-10-09T10:48:54.350199+0000","last_fullsized":"2023-10-09T10:48:54.350199+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:30.322834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":63,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.306832+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:48:55.306832+0000","last_peered":"2023-10-09T10:48:55.306832+0000","last_clean":"2023-10-09T10:48:55.306832+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:48:55.306832+0000","last_undegraded":"2023-10-09T10:48:55.306832+0000","last_fullsized":"2023-10-09T10:48:55.306832+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":136,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.298895+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:48:54.298895+0000","last_peered":"2023-10-09T10:48:54.298895+0000","last_clean":"2023-10-09T10:48:54.298895+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:48:54.298895+0000","last_undegraded":"2023-10-09T10:48:54.298895+0000","last_fullsized":"2023-10-09T10:48:54.298895+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"189'7594","reported_seq":14412,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:49:00.884501+0000","last_change":"2023-10-09T10:47:31.271196+0000","last_active":"2023-10-09T10:49:00.884501+0000","last_peered":"2023-10-09T10:49:00.884501+0000","last_clean":"2023-10-09T10:49:00.884501+0000","last_became_active":"2023-10-09T10:47:31.229504+0000","last_became_peered":"2023-10-09T10:47:31.229504+0000","last_unstale":"2023-10-09T10:49:00.884501+0000","last_undegraded":"2023-10-09T10:49:00.884501+0000","last_fullsized":"2023-10-09T10:49:00.884501+0000","mapping_epoch":172,"log_start":"118'2815","ondisk_log_start":"118'2815","created":98,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"114'2652","last_scrub_stamp":"2023-10-09T10:45:05.770263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:45:05.770263+0000","objects_scrubbed":1629,"log_size":4779,"log_dups_size":0,"ondisk_log_size":4779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:18:26.533476+0000","scrub_duration":0.42541933500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2097152,"num_objects":32,"num_object_clones":0,"num_object_copies":96,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":32,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4949,"num_write_kb":107264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3305,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":59,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.306010+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:48:55.306010+0000","last_peered":"2023-10-09T10:48:55.306010+0000","last_clean":"2023-10-09T10:48:55.306010+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:48:55.306010+0000","last_undegraded":"2023-10-09T10:48:55.306010+0000","last_fullsized":"2023-10-09T10:48:55.306010+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":121,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.315181+0000","last_change":"2023-10-09T10:46:32.251242+0000","last_active":"2023-10-09T10:48:55.315181+0000","last_peered":"2023-10-09T10:48:55.315181+0000","last_clean":"2023-10-09T10:48:55.315181+0000","last_became_active":"2023-10-09T10:46:32.250534+0000","last_became_peered":"2023-10-09T10:46:32.250534+0000","last_unstale":"2023-10-09T10:48:55.315181+0000","last_undegraded":"2023-10-09T10:48:55.315181+0000","last_fullsized":"2023-10-09T10:48:55.315181+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:33.726041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.19","version":"189'7706","reported_seq":15535,"reported_epoch":190,"state":"peering","last_fresh":"2023-10-09T10:49:02.398937+0000","last_change":"2023-10-09T10:49:02.399047+0000","last_active":"2023-10-09T10:49:02.398937+0000","last_peered":"2023-10-09T10:49:02.370912+0000","last_clean":"2023-10-09T10:45:14.278222+0000","last_became_active":"2023-10-09T10:46:34.409667+0000","last_became_peered":"2023-10-09T10:46:34.409667+0000","last_unstale":"2023-10-09T10:49:02.398937+0000","last_undegraded":"2023-10-09T10:49:02.398937+0000","last_fullsized":"2023-10-09T10:49:02.398937+0000","mapping_epoch":190,"log_start":"113'2497","ondisk_log_start":"113'2497","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5209,"log_dups_size":0,"ondisk_log_size":5209,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":655360,"num_objects":10,"num_object_clones":0,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6664,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3848,"num_bytes_recovered":218759168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3,7],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":156,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.290932+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:48:54.290932+0000","last_peered":"2023-10-09T10:48:54.290932+0000","last_clean":"2023-10-09T10:48:54.290932+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:48:54.290932+0000","last_undegraded":"2023-10-09T10:48:54.290932+0000","last_fullsized":"2023-10-09T10:48:54.290932+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":170,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.299299+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:48:54.299299+0000","last_peered":"2023-10-09T10:48:54.299299+0000","last_clean":"2023-10-09T10:48:54.299299+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:48:54.299299+0000","last_undegraded":"2023-10-09T10:48:54.299299+0000","last_fullsized":"2023-10-09T10:48:54.299299+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"189'7539","reported_seq":7661,"reported_epoch":189,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:48:59.609311+0000","last_change":"2023-10-09T10:46:33.546991+0000","last_active":"2023-10-09T10:48:59.609311+0000","last_peered":"2023-10-09T10:48:59.609311+0000","last_clean":"2023-10-09T10:46:31.009484+0000","last_became_active":"2023-10-09T10:46:33.535665+0000","last_became_peered":"2023-10-09T10:46:33.535665+0000","last_unstale":"2023-10-09T10:48:59.609311+0000","last_undegraded":"2023-10-09T10:46:33.277168+0000","last_fullsized":"2023-10-09T10:46:33.235038+0000","mapping_epoch":154,"log_start":"105'1656","ondisk_log_start":"105'1656","created":107,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:44:26.547734+0000","last_clean_scrub_stamp":"2023-10-09T10:44:26.547734+0000","objects_scrubbed":0,"log_size":5883,"log_dups_size":0,"ondisk_log_size":5883,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3440640,"num_objects":53,"num_object_clones":0,"num_object_copies":159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":53,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6475,"num_write_kb":208864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":65},{"shards":"3,7","objects":-2705}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":139,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.290679+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:48:54.290679+0000","last_peered":"2023-10-09T10:48:54.290679+0000","last_clean":"2023-10-09T10:48:54.290679+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:48:54.290679+0000","last_undegraded":"2023-10-09T10:48:54.290679+0000","last_fullsized":"2023-10-09T10:48:54.290679+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":37,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.299069+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:48:54.299069+0000","last_peered":"2023-10-09T10:48:54.299069+0000","last_clean":"2023-10-09T10:48:54.299069+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:48:54.299069+0000","last_undegraded":"2023-10-09T10:48:54.299069+0000","last_fullsized":"2023-10-09T10:48:54.299069+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":173,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.339230+0000","last_change":"2023-10-09T10:46:32.244607+0000","last_active":"2023-10-09T10:48:54.339230+0000","last_peered":"2023-10-09T10:48:54.339230+0000","last_clean":"2023-10-09T10:48:54.339230+0000","last_became_active":"2023-10-09T10:46:32.242628+0000","last_became_peered":"2023-10-09T10:46:32.242628+0000","last_unstale":"2023-10-09T10:48:54.339230+0000","last_undegraded":"2023-10-09T10:48:54.339230+0000","last_fullsized":"2023-10-09T10:48:54.339230+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:10.738819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":204,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.348838+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:48:54.348838+0000","last_peered":"2023-10-09T10:48:54.348838+0000","last_clean":"2023-10-09T10:48:54.348838+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:48:54.348838+0000","last_undegraded":"2023-10-09T10:48:54.348838+0000","last_fullsized":"2023-10-09T10:48:54.348838+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":80,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T10:48:55.307294+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:48:55.307294+0000","last_peered":"2023-10-09T10:48:55.307294+0000","last_clean":"2023-10-09T10:48:55.307294+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:48:55.307294+0000","last_undegraded":"2023-10-09T10:48:55.307294+0000","last_fullsized":"2023-10-09T10:48:55.307294+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":153,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T10:48:54.298834+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:48:54.298834+0000","last_peered":"2023-10-09T10:48:54.298834+0000","last_clean":"2023-10-09T10:48:54.298834+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:48:54.298834+0000","last_undegraded":"2023-10-09T10:48:54.298834+0000","last_fullsized":"2023-10-09T10:48:54.298834+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":94896152,"num_objects":1449,"num_object_clones":0,"num_object_copies":4347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23268,"num_objects_misplaced":151,"num_objects_unfound":0,"num_objects_dirty":1449,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":202767,"num_write_kb":6258529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32207,"num_bytes_recovered":1625423872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2745315328,"data_stored":2745303112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":169579,"ondisk_log_size":169579,"up":78,"acting":71,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1186,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332269,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":343812,"kb_used_data":46524,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93401340,"statfs":{"total":95995035648,"available":95642972160,"internally_reserved":0,"allocated":47640576,"data_stored":46920715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":1.155,"15min":0.76600000000000001},"min":{"1min":0.23499999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.476,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":0.34000000000000002},{"interface":"front","average":{"1min":3.1789999999999998,"5min":1.5029999999999999,"15min":0.92100000000000004},"min":{"1min":0.248,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.332000000000001,"5min":11.332000000000001,"15min":11.332000000000001},"last":1.4039999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":3.4359999999999999,"5min":1.909,"15min":0.997},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":11.367000000000001,"5min":11.826000000000001,"15min":11.826000000000001},"last":1.3660000000000001},{"interface":"front","average":{"1min":2.1739999999999999,"5min":1.484,"15min":0.84899999999999998},"min":{"1min":0.26100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":7.556,"5min":11.84,"15min":11.84},"last":0.377}]},{"osd":2,"last update":"Mon Oct 9 10:48:18 2023","interfaces":[{"interface":"back","average":{"1min":2.165,"5min":1.4810000000000001,"15min":0.83499999999999996},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.1419999999999995,"5min":9.1419999999999995,"15min":9.1419999999999995},"last":1.4750000000000001},{"interface":"front","average":{"1min":2.1970000000000001,"5min":1.1910000000000001,"15min":0.747},"min":{"1min":0.34999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":7.8120000000000003,"5min":7.8120000000000003,"15min":7.8120000000000003},"last":0.39700000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":3.903,"5min":1.893,"15min":1.0209999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":11.523,"5min":11.523,"15min":11.523},"last":8.5739999999999998},{"interface":"front","average":{"1min":4.2789999999999999,"5min":1.835,"15min":1.0109999999999999},"min":{"1min":0.35299999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":19.234999999999999,"5min":19.234999999999999,"15min":19.234999999999999},"last":7.4180000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":1.1579999999999999,"15min":0.70499999999999996},"min":{"1min":0.36399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":9.4900000000000002,"5min":9.4900000000000002,"15min":9.4900000000000002},"last":1.8700000000000001},{"interface":"front","average":{"1min":2.327,"5min":1.018,"15min":0.66100000000000003},"min":{"1min":0.35399999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.0999999999999996,"5min":9.0999999999999996,"15min":9.0999999999999996},"last":1.952}]},{"osd":5,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.0249999999999999,"5min":1.0609999999999999,"15min":0.69599999999999995},"min":{"1min":0.314,"5min":0.247,"15min":0.247},"max":{"1min":12.064,"5min":12.064,"15min":12.064},"last":4.7450000000000001},{"interface":"front","average":{"1min":3.8839999999999999,"5min":1.4770000000000001,"15min":0.85299999999999998},"min":{"1min":0.435,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":11.741,"5min":11.741,"15min":11.741},"last":1.4399999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2999999999999998,"5min":1.3720000000000001,"15min":0.83199999999999996},"min":{"1min":0.33700000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.029,"5min":13.029,"15min":13.029},"last":1.903},{"interface":"front","average":{"1min":2.6869999999999998,"5min":1.079,"15min":0.72699999999999998},"min":{"1min":0.41299999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":19.187999999999999,"5min":19.187999999999999,"15min":19.187999999999999},"last":1.972}]}]},{"osd":6,"up_from":44,"seq":188978561200,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":335168,"kb_used_data":25592,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93409984,"statfs":{"total":95995035648,"available":95651823616,"internally_reserved":0,"allocated":26206208,"data_stored":25490885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":3.0499999999999998,"5min":1.4330000000000001,"15min":0.88},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.882,"5min":10.882,"15min":10.882},"last":0.51700000000000002},{"interface":"front","average":{"1min":2.4540000000000002,"5min":1.367,"15min":0.86199999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.8710000000000004,"5min":7.8710000000000004,"15min":7.8710000000000004},"last":2.9390000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:48:53 2023","interfaces":[{"interface":"back","average":{"1min":3.2109999999999999,"5min":1.681,"15min":0.95399999999999996},"min":{"1min":0.28999999999999998,"5min":0.245,"15min":0.245},"max":{"1min":15.502000000000001,"5min":16.747,"15min":16.747},"last":3.415},{"interface":"front","average":{"1min":3.0779999999999998,"5min":1.79,"15min":0.96399999999999997},"min":{"1min":0.33700000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.137,"5min":16.734000000000002,"15min":16.734000000000002},"last":3.4260000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:48:20 2023","interfaces":[{"interface":"back","average":{"1min":2.798,"5min":1.411,"15min":0.82599999999999996},"min":{"1min":0.29699999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.208,"5min":13.208,"15min":13.208},"last":1.0229999999999999},{"interface":"front","average":{"1min":2.9420000000000002,"5min":1.619,"15min":0.91300000000000003},"min":{"1min":0.309,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.6749999999999998,"5min":12.571,"15min":12.571},"last":0.89900000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":1.496,"15min":0.85699999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":7.7649999999999997,"5min":7.7649999999999997,"15min":7.7649999999999997},"last":1.7589999999999999},{"interface":"front","average":{"1min":2.254,"5min":1.7270000000000001,"15min":0.93999999999999995},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":13.179,"5min":13.179,"15min":13.179},"last":0.99299999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.0629999999999999,"15min":0.65000000000000002},"min":{"1min":0.29899999999999999,"5min":0.255,"15min":0.255},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":0.86499999999999999},{"interface":"front","average":{"1min":1.6579999999999999,"5min":0.89600000000000002,"15min":0.623},"min":{"1min":0.34100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.1260000000000003,"5min":5.1260000000000003,"15min":5.1260000000000003},"last":0.437}]},{"osd":5,"last update":"Mon Oct 9 10:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":1.1719999999999999,"15min":0.71099999999999997},"min":{"1min":0.35899999999999999,"5min":0.29499999999999998,"15min":0.27800000000000002},"max":{"1min":10.497,"5min":10.497,"15min":10.497},"last":0.82499999999999996},{"interface":"front","average":{"1min":1.589,"5min":0.96599999999999997,"15min":0.67100000000000004},"min":{"1min":0.40699999999999997,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":6.8419999999999996,"5min":6.8419999999999996,"15min":6.8419999999999996},"last":0.40500000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:48:26 2023","interfaces":[{"interface":"back","average":{"1min":3.915,"5min":1.369,"15min":0.80600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":23.358000000000001,"5min":23.358000000000001,"15min":23.358000000000001},"last":1.1579999999999999},{"interface":"front","average":{"1min":2.6840000000000002,"5min":1.25,"15min":0.79700000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34000000000000002},"max":{"1min":12.404,"5min":12.404,"15min":12.404},"last":0.61099999999999999}]}]},{"osd":1,"up_from":85,"seq":365072220239,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":245516,"kb_used_data":105668,"kb_used_omap":9,"kb_used_meta":139830,"kb_avail":93499636,"statfs":{"total":95995035648,"available":95743627264,"internally_reserved":0,"allocated":108204032,"data_stored":107245860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":143186921},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":2.1419999999999999,"15min":0.95599999999999996},"min":{"1min":0.249,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":18.027000000000001,"5min":18.027000000000001,"15min":18.027000000000001},"last":4.117},{"interface":"front","average":{"1min":2.6920000000000002,"5min":2.0209999999999999,"15min":0.93899999999999995},"min":{"1min":0.23599999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.6530000000000005,"5min":14.465,"15min":14.465},"last":0.36299999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:48:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":1.492,"15min":0.74199999999999999},"min":{"1min":0.221,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":15.356,"5min":15.356,"15min":15.356},"last":1.351},{"interface":"front","average":{"1min":1.488,"5min":1.6890000000000001,"15min":0.81899999999999995},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.4130000000000003,"5min":14.234,"15min":14.234},"last":4.0810000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.6829999999999998,"5min":2.169,"15min":1.0369999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":18.274999999999999,"5min":18.274999999999999,"15min":18.274999999999999},"last":13.885999999999999},{"interface":"front","average":{"1min":2.774,"5min":1.915,"15min":0.86799999999999999},"min":{"1min":0.313,"5min":0.251,"15min":0.251},"max":{"1min":8.5999999999999996,"5min":15.468999999999999,"15min":15.468999999999999},"last":5.1870000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":1.5289999999999999,"15min":0.90500000000000003},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.167999999999999,"5min":12.167999999999999,"15min":12.167999999999999},"last":3.7869999999999999},{"interface":"front","average":{"1min":3.4399999999999999,"5min":1.7989999999999999,"15min":1.0129999999999999},"min":{"1min":0.309,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":14.228999999999999,"5min":14.228999999999999,"15min":14.228999999999999},"last":5.4669999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":1.4359999999999999,"15min":0.89700000000000002},"min":{"1min":0.34399999999999997,"5min":0.253,"15min":0.253},"max":{"1min":9.4809999999999999,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":0.39400000000000002},{"interface":"front","average":{"1min":3.7909999999999999,"5min":1.9219999999999999,"15min":1.0720000000000001},"min":{"1min":0.318,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":17.085000000000001,"5min":17.085000000000001,"15min":17.085000000000001},"last":1.3069999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":2.8559999999999999,"5min":1.6479999999999999,"15min":0.96999999999999997},"min":{"1min":0.38300000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.9179999999999993,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.819,"5min":1.7290000000000001,"15min":1.0069999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":12.231,"5min":12.231,"15min":12.231},"last":7.7480000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":3.1219999999999999,"5min":1.714,"15min":1.004},"min":{"1min":0.42799999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":12.249000000000001,"5min":12.249000000000001,"15min":12.249000000000001},"last":1.821},{"interface":"front","average":{"1min":3.7250000000000001,"5min":1.907,"15min":1.0660000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":14.273999999999999,"5min":14.273999999999999,"15min":14.273999999999999},"last":4.1440000000000001}]}]},{"osd":0,"up_from":80,"seq":343597383773,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302976,"kb_used_data":172920,"kb_used_omap":28,"kb_used_meta":130019,"kb_avail":93442176,"statfs":{"total":95995035648,"available":95684788224,"internally_reserved":0,"allocated":177070080,"data_stored":176067767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":133140304},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":3.931,"5min":2.258,"15min":0.998},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":11.869,"5min":25.475000000000001,"15min":25.475000000000001},"last":0.70799999999999996},{"interface":"front","average":{"1min":3.702,"5min":2.2610000000000001,"15min":0.997},"min":{"1min":0.31,"5min":0.188,"15min":0.188},"max":{"1min":12.276,"5min":20.832999999999998,"15min":20.832999999999998},"last":0.44800000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":1.4750000000000001,"15min":0.72399999999999998},"min":{"1min":0.249,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.0890000000000004,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":0.82599999999999996},{"interface":"front","average":{"1min":2.1120000000000001,"5min":1.889,"15min":0.85799999999999998},"min":{"1min":0.27100000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.0329999999999995,"5min":14.391999999999999,"15min":14.391999999999999},"last":0.46600000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.3359999999999999,"15min":1.212},"min":{"1min":0.307,"5min":0.215,"15min":0.215},"max":{"1min":12.307,"5min":14.926,"15min":14.926},"last":1.3100000000000001},{"interface":"front","average":{"1min":3.5539999999999998,"5min":2.1960000000000002,"15min":1.093},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":11.885999999999999,"5min":11.885999999999999,"15min":11.885999999999999},"last":2.4700000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":2.8370000000000002,"5min":1.671,"15min":0.96999999999999997},"min":{"1min":0.434,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.0920000000000005,"5min":8.0920000000000005,"15min":8.0920000000000005},"last":3.2530000000000001},{"interface":"front","average":{"1min":3.2679999999999998,"5min":1.9370000000000001,"15min":1.0569999999999999},"min":{"1min":0.36099999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.318,"5min":12.318,"15min":12.318},"last":3.488}]},{"osd":5,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6179999999999999,"5min":1.8400000000000001,"15min":1.0289999999999999},"min":{"1min":0.35899999999999999,"5min":0.318,"15min":0.318},"max":{"1min":12.157,"5min":12.157,"15min":12.157},"last":3.915},{"interface":"front","average":{"1min":3.181,"5min":1.677,"15min":0.98999999999999999},"min":{"1min":0.40500000000000003,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":12.781000000000001,"5min":12.781000000000001,"15min":12.781000000000001},"last":4.0149999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":3.4649999999999999,"5min":1.96,"15min":1.073},"min":{"1min":0.33600000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":12.207000000000001,"5min":12.207000000000001,"15min":12.207000000000001},"last":3.6200000000000001},{"interface":"front","average":{"1min":2.1269999999999998,"5min":1.3740000000000001,"15min":0.89100000000000001},"min":{"1min":0.41299999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":9.8889999999999993,"5min":9.8889999999999993,"15min":9.8889999999999993},"last":2.4390000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":3.6280000000000001,"5min":1.7509999999999999,"15min":1.0169999999999999},"min":{"1min":0.48799999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":12.727,"5min":12.727,"15min":12.727},"last":0.74299999999999999},{"interface":"front","average":{"1min":2.843,"5min":1.8660000000000001,"15min":1.071},"min":{"1min":0.47599999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.6150000000000002,"5min":14.394,"15min":14.394},"last":0.50700000000000001}]}]},{"osd":2,"up_from":0,"seq":386547056713,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893187,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":466016,"kb_used_data":200280,"kb_used_omap":7,"kb_used_meta":265656,"kb_avail":93279136,"statfs":{"total":95995035648,"available":95517835264,"internally_reserved":0,"allocated":205086720,"data_stored":204106756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":272032360},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3159999999999998,"5min":1.913,"15min":0.85099999999999998},"min":{"1min":0.248,"5min":0.183,"15min":0.183},"max":{"1min":15.255000000000001,"5min":15.255000000000001,"15min":15.255000000000001},"last":5.6820000000000004},{"interface":"front","average":{"1min":2.9830000000000001,"5min":1.6539999999999999,"15min":0.82199999999999995},"min":{"1min":0.21099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":8.6470000000000002,"5min":8.6470000000000002,"15min":8.6470000000000002},"last":0.29099999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.6400000000000001,"5min":2.359,"15min":1.006},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":10.821,"5min":14.707000000000001,"15min":14.707000000000001},"last":11.757},{"interface":"front","average":{"1min":3.4780000000000002,"5min":1.8100000000000001,"15min":0.85199999999999998},"min":{"1min":0.28999999999999998,"5min":0.193,"15min":0.193},"max":{"1min":15.443,"5min":15.443,"15min":15.443},"last":5.6420000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":2.0630000000000002,"15min":0.93200000000000005},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.375,"5min":13.853,"15min":13.853},"last":11.795},{"interface":"front","average":{"1min":3.988,"5min":2.2309999999999999,"15min":1.0069999999999999},"min":{"1min":0.28699999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.557,"5min":10.557,"15min":10.557},"last":0.26200000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":4.0739999999999998,"5min":1.9219999999999999,"15min":1.022},"min":{"1min":0.33900000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":20.388000000000002,"5min":20.388000000000002,"15min":20.388000000000002},"last":5.7069999999999999},{"interface":"front","average":{"1min":4.4299999999999997,"5min":2.0609999999999999,"15min":1.0920000000000001},"min":{"1min":0.38700000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":19.085999999999999,"5min":19.085999999999999,"15min":19.085999999999999},"last":0.32700000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.7599999999999998,"5min":2.4369999999999998,"15min":1.2030000000000001},"min":{"1min":0.35399999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.3550000000000004,"5min":51.509999999999998,"15min":51.509999999999998},"last":0.71799999999999997},{"interface":"front","average":{"1min":2.6240000000000001,"5min":2.7189999999999999,"15min":1.2809999999999999},"min":{"1min":0.316,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.7449999999999992,"5min":55.338999999999999,"15min":55.338999999999999},"last":7.6139999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":1.841,"15min":1.0049999999999999},"min":{"1min":0.43099999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.361000000000001,"5min":12.361000000000001,"15min":12.361000000000001},"last":0.313},{"interface":"front","average":{"1min":2.8140000000000001,"5min":2.2189999999999999,"15min":1.139},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.5839999999999996,"5min":13.891999999999999,"15min":13.891999999999999},"last":7.569}]},{"osd":7,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.2269999999999999,"5min":2.8679999999999999,"15min":1.3500000000000001},"min":{"1min":0.34000000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.734,"5min":51.744,"15min":51.744},"last":11.819000000000001},{"interface":"front","average":{"1min":3.927,"5min":2.54,"15min":1.2290000000000001},"min":{"1min":0.46100000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":15.212999999999999,"5min":53.399000000000001,"15min":53.399000000000001},"last":5.6660000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019063,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":343680,"kb_used_data":30008,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":93401472,"statfs":{"total":95995035648,"available":95643107328,"internally_reserved":0,"allocated":30728192,"data_stored":30012136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.9850000000000001,"5min":1.302,"15min":0.80500000000000005},"min":{"1min":0.216,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.7160000000000002,"5min":10.387,"15min":10.387},"last":1.5009999999999999},{"interface":"front","average":{"1min":2.9780000000000002,"5min":1.6659999999999999,"15min":0.94299999999999995},"min":{"1min":0.48499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.0129999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.86499999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":2.351,"5min":1.512,"15min":0.872},"min":{"1min":0.254,"5min":0.249,"15min":0.249},"max":{"1min":7.5350000000000001,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":2.931,"5min":1.7010000000000001,"15min":0.98099999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.2919999999999998,"5min":8.7669999999999995,"15min":8.7669999999999995},"last":0.36899999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:48:13 2023","interfaces":[{"interface":"back","average":{"1min":3.1520000000000001,"5min":1.617,"15min":0.91900000000000004},"min":{"1min":0.52100000000000002,"5min":0.253,"15min":0.253},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":1.355},{"interface":"front","average":{"1min":4.016,"5min":1.855,"15min":0.99299999999999999},"min":{"1min":0.36799999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":20.545999999999999,"5min":20.545999999999999,"15min":20.545999999999999},"last":5.5659999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":3.2549999999999999,"5min":1.8420000000000001,"15min":1.0880000000000001},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.0269999999999992,"5min":9.0269999999999992,"15min":9.0269999999999992},"last":0.59999999999999998},{"interface":"front","average":{"1min":3.9809999999999999,"5min":2.0379999999999998,"15min":1.071},"min":{"1min":0.79400000000000004,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.6029999999999998,"5min":8.9809999999999999,"15min":8.9809999999999999},"last":0.68100000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":5.0960000000000001,"5min":1.8140000000000001,"15min":0.91000000000000003},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":24.149999999999999,"5min":24.149999999999999,"15min":24.149999999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":4.173,"5min":1.6699999999999999,"15min":0.86399999999999999},"min":{"1min":0.373,"5min":0.252,"15min":0.252},"max":{"1min":20.422000000000001,"5min":20.422000000000001,"15min":20.422000000000001},"last":0.29199999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":1.1319999999999999,"15min":0.69399999999999995},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":12.391999999999999,"5min":12.391999999999999,"15min":12.391999999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":3.1200000000000001,"5min":1.2130000000000001,"15min":0.78100000000000003},"min":{"1min":0.374,"5min":0.311,"15min":0.311},"max":{"1min":12.539,"5min":12.539,"15min":12.539},"last":0.56299999999999994}]},{"osd":7,"last update":"Mon Oct 9 10:48:21 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":1.2929999999999999,"15min":0.80900000000000005},"min":{"1min":0.33300000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.7919999999999998,"5min":8.7919999999999998,"15min":8.7919999999999998},"last":0.40100000000000002},{"interface":"front","average":{"1min":3.2389999999999999,"5min":1.27,"15min":0.79000000000000004},"min":{"1min":0.55900000000000005,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.423,"5min":7.423,"15min":7.423},"last":0.41299999999999998}]}]},{"osd":5,"up_from":37,"seq":158913790132,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":344624,"kb_used_data":39144,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93400528,"statfs":{"total":95995035648,"available":95642140672,"internally_reserved":0,"allocated":40083456,"data_stored":39379229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:41 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":1.2210000000000001,"15min":0.80500000000000005},"min":{"1min":0.28899999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.648999999999999,"5min":11.648999999999999,"15min":11.648999999999999},"last":1},{"interface":"front","average":{"1min":2.8570000000000002,"5min":1.294,"15min":0.875},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.68,"5min":11.68,"15min":11.68},"last":1.077}]},{"osd":1,"last update":"Mon Oct 9 10:48:51 2023","interfaces":[{"interface":"back","average":{"1min":2.673,"5min":1.78,"15min":0.998},"min":{"1min":0.41799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":9.6839999999999993,"5min":12.999000000000001,"15min":12.999000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.0680000000000001,"5min":1.532,"15min":0.90500000000000003},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.1120000000000001,"5min":7.3730000000000002,"15min":7.3730000000000002},"last":1.024}]},{"osd":2,"last update":"Mon Oct 9 10:48:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6669999999999998,"5min":1.4390000000000001,"15min":0.86099999999999999},"min":{"1min":0.34799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.3280000000000003,"5min":7.9000000000000004,"15min":7.9000000000000004},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.2350000000000001,"15min":0.79400000000000004},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.3059999999999992,"5min":10.352,"15min":10.352},"last":1.0940000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:48:47 2023","interfaces":[{"interface":"back","average":{"1min":2.859,"5min":2.048,"15min":1.165},"min":{"1min":0.432,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":12.558999999999999,"5min":18.016999999999999,"15min":18.016999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.6930000000000001,"5min":2.089,"15min":1.105},"min":{"1min":0.40300000000000002,"5min":0.317,"15min":0.317},"max":{"1min":7.9560000000000004,"5min":15.555999999999999,"15min":15.555999999999999},"last":0.58099999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:48:51 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":1.0629999999999999,"15min":0.66000000000000003},"min":{"1min":0.26800000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":6.093,"5min":19.152999999999999,"15min":19.152999999999999},"last":1.05},{"interface":"front","average":{"1min":2.5369999999999999,"5min":1.375,"15min":0.77800000000000002},"min":{"1min":0.32400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.8610000000000007,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":4.5389999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":1.127,"15min":0.69899999999999995},"min":{"1min":0.33100000000000002,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":7.7060000000000004,"5min":7.7060000000000004,"15min":7.7060000000000004},"last":0.79000000000000004},{"interface":"front","average":{"1min":1.958,"5min":0.90400000000000003,"15min":0.64000000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.30299999999999999},"max":{"1min":8.0909999999999993,"5min":8.0909999999999993,"15min":8.0909999999999993},"last":3.9129999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:48:26 2023","interfaces":[{"interface":"back","average":{"1min":2.79,"5min":1.1160000000000001,"15min":0.74099999999999999},"min":{"1min":0.41899999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.3940000000000001,"5min":7.3940000000000001,"15min":7.3940000000000001},"last":1.0600000000000001},{"interface":"front","average":{"1min":2.2669999999999999,"5min":1.0369999999999999,"15min":0.71999999999999997},"min":{"1min":0.38500000000000001,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":9.0579999999999998,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":6.9640000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":174133248,"data_stored":174129176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":91295744,"data_stored":91291672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2203713536,"data_stored":2203713536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":165019648,"data_stored":165019648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576000,"data_stored":24576000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":30216192,"data_stored":30212120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":23068672,"data_stored":23068672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":33292288,"data_stored":33292288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:49:07.301 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:07.315 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:07.332 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:07.362 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:07.398 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:07.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:07.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:07.444 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:07.479 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:49:07.508 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:07.538 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:08.191 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:08.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:08.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:08.300 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:08.334 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:08.404 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:08.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:08.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:08.493 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:49:08.493 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:49:08.508 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:08.536 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:08 smithi155 ceph-mon[184854]: pgmap v430: 119 pgs: 1 peering, 109 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 8.6 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 21721/453 objects degraded (4794.923%); 24/453 objects misplaced (5.298%); 0 B/s, 287 objects/s recovering 2023-10-09T10:49:08.536 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:08 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 21721/453 objects degraded (4794.923%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:49:08.538 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:08.762 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:08.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:08.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:08.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:08 smithi125 ceph-mon[195507]: pgmap v430: 119 pgs: 1 peering, 109 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 8.6 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 21721/453 objects degraded (4794.923%); 24/453 objects misplaced (5.298%); 0 B/s, 287 objects/s recovering 2023-10-09T10:49:08.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:08 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 21721/453 objects degraded (4794.923%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:49:08.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:08 smithi125 ceph-mon[200197]: pgmap v430: 119 pgs: 1 peering, 109 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 8.6 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 21721/453 objects degraded (4794.923%); 24/453 objects misplaced (5.298%); 0 B/s, 287 objects/s recovering 2023-10-09T10:49:08.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:08 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 21721/453 objects degraded (4794.923%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:49:08.862 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:49:08.866 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:08.895 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:09.065 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:09.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:09.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:09.160 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:49:09.170 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:49:09.201 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:09.974 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:49:09.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:09.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:49:10.077 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:10.109 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:10.129 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:10.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:10.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:10.153 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:49:10.153 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:49:10.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:10.264 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:49:10.266 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:10.291 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:10.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:10.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:10.397 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:10.429 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:10.449 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:10.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:10.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:10.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:10 smithi125 ceph-mon[195507]: pgmap v431: 119 pgs: 110 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 7.9 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 246 op/s; 21721/423 objects degraded (5134.988%); 24/423 objects misplaced (5.674%); 0 B/s, 253 objects/s recovering 2023-10-09T10:49:10.486 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:10 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/649296595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:10.487 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:10 smithi125 ceph-mon[200197]: pgmap v431: 119 pgs: 110 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 7.9 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 246 op/s; 21721/423 objects degraded (5134.988%); 24/423 objects misplaced (5.674%); 0 B/s, 253 objects/s recovering 2023-10-09T10:49:10.487 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:10 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/649296595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:10.528 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:49:10.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:10.589 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:49:10.589 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:49:10.589 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:49:10.589 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:49:10.589 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:49:10.589 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:49:10.589 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:49:10.589 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:49:10.590 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:49:10.590 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:49:10.590 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:49:10.590 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:49:10.590 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:49:10.590 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:49:10.590 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:49:10.590 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:49:10.597 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:10 smithi155 ceph-mon[184854]: pgmap v431: 119 pgs: 110 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 7.9 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 246 op/s; 21721/423 objects degraded (5134.988%); 24/423 objects misplaced (5.674%); 0 B/s, 253 objects/s recovering 2023-10-09T10:49:10.597 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:10 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/649296595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:10.599 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:10.625 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:10.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:10.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:10.729 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:10.763 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:10.821 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:49:11.145 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:11.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:11.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:11.329 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:11.358 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:11.429 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:11.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:11.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:11.461 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_0' removed 2023-10-09T10:49:11.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:11 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2165226581' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:49:11.494 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:49:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:49:11] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.43.0" 2023-10-09T10:49:11.497 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:11 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2165226581' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:49:11.533 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:11.558 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:11 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2165226581' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:49:11.561 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:11.681 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:11.690 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:11.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:11.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:11.793 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:11.821 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:11.897 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:49:11.897 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:49:11.897 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:49:11.897 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:49:11.897 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:49:12.282 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:49:12.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:12.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:12.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:12.516 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:12.547 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:12.586 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:49:12.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:12 smithi125 ceph-mon[195507]: pgmap v432: 119 pgs: 110 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 7.9 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 183 op/s; 20475/423 objects degraded (4840.426%); 24/423 objects misplaced (5.674%); 0 B/s, 349 objects/s recovering 2023-10-09T10:49:12.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:12 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2165226581' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:49:12.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:12 smithi125 ceph-mon[195507]: osdmap e193: 8 total, 8 up, 7 in 2023-10-09T10:49:12.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:12 smithi125 ceph-mon[195507]: failed to encode map e193 with expected crc 2023-10-09T10:49:12.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:12 smithi125 ceph-mon[195507]: failed to encode map e193 with expected crc 2023-10-09T10:49:12.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:12 smithi125 ceph-mon[195507]: failed to encode map e193 with expected crc 2023-10-09T10:49:12.631 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:12 smithi125 ceph-mon[200197]: pgmap v432: 119 pgs: 110 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 7.9 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 183 op/s; 20475/423 objects degraded (4840.426%); 24/423 objects misplaced (5.674%); 0 B/s, 349 objects/s recovering 2023-10-09T10:49:12.631 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:12 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2165226581' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:49:12.631 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:12 smithi125 ceph-mon[200197]: osdmap e193: 8 total, 8 up, 7 in 2023-10-09T10:49:12.631 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:12 smithi125 ceph-mon[200197]: failed to encode map e193 with expected crc 2023-10-09T10:49:12.632 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:12 smithi125 ceph-mon[200197]: failed to encode map e193 with expected crc 2023-10-09T10:49:12.632 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:12 smithi125 ceph-mon[200197]: failed to encode map e193 with expected crc 2023-10-09T10:49:12.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:12 smithi155 ceph-mon[184854]: pgmap v432: 119 pgs: 110 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 7.9 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 183 op/s; 20475/423 objects degraded (4840.426%); 24/423 objects misplaced (5.674%); 0 B/s, 349 objects/s recovering 2023-10-09T10:49:12.687 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:12 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2165226581' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:49:12.687 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:12 smithi155 ceph-mon[184854]: osdmap e193: 8 total, 8 up, 7 in 2023-10-09T10:49:12.687 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:12 smithi155 ceph-mon[184854]: failed to encode map e193 with expected crc 2023-10-09T10:49:12.687 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:12 smithi155 ceph-mon[184854]: failed to encode map e193 with expected crc 2023-10-09T10:49:12.687 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:12 smithi155 ceph-mon[184854]: failed to encode map e193 with expected crc 2023-10-09T10:49:12.767 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:12.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:12.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:12.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:12.902 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:13.102 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T10:49:13.103 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T10:49:13.104 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-10-09T10:49:13.104 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-10-09T10:49:13.331 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:13.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:13.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:13.435 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:13.461 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:13.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:49:13.531 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:13.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:13.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:13.552 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:49:13.552 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:49:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:49:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:49:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70217 6.d 6:b8b4c26d:::benchmark_data_smithi125_244951_object70216:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:49:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.226550+0000", 2023-10-09T10:49:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 230.32402083400001, 2023-10-09T10:49:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.064685005000000004, 2023-10-09T10:49:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70217 2023-10-09T10:49:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226550+0000", 2023-10-09T10:49:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226550+0000", 2023-10-09T10:49:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226550+0000", 2023-10-09T10:49:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226583+0000", 2023-10-09T10:49:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2808000000000002e-05 2023-10-09T10:49:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226590+0000", 2023-10-09T10:49:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8509999999999999e-06 2023-10-09T10:49:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226595+0000", 2023-10-09T10:49:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6290000000000001e-06 2023-10-09T10:49:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226637+0000", 2023-10-09T10:49:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1734999999999999e-05 2023-10-09T10:49:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226771+0000", 2023-10-09T10:49:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013447000000000001 2023-10-09T10:49:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:49:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226923+0000", 2023-10-09T10:49:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151977 2023-10-09T10:49:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.229528+0000", 2023-10-09T10:49:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026052530000000001 2023-10-09T10:49:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.241565+0000", 2023-10-09T10:49:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012036467 2023-10-09T10:49:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.288660+0000", 2023-10-09T10:49:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047095342999999998 2023-10-09T10:49:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.288720+0000", 2023-10-09T10:49:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0025000000000002e-05 2023-10-09T10:49:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.291235+0000", 2023-10-09T10:49:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0025149249999999999 2023-10-09T10:49:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70220 6.d 6:bb7bde6c:::benchmark_data_smithi125_244951_object70219:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:49:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.231764+0000", 2023-10-09T10:49:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 230.31880651099999, 2023-10-09T10:49:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.061391124999999998, 2023-10-09T10:49:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70220 2023-10-09T10:49:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231764+0000", 2023-10-09T10:49:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231764+0000", 2023-10-09T10:49:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231763+0000", 2023-10-09T10:49:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:49:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231801+0000", 2023-10-09T10:49:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7691999999999998e-05 2023-10-09T10:49:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231812+0000", 2023-10-09T10:49:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0569e-05 2023-10-09T10:49:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231817+0000", 2023-10-09T10:49:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0490000000000003e-06 2023-10-09T10:49:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231856+0000", 2023-10-09T10:49:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9002999999999999e-05 2023-10-09T10:49:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231960+0000", 2023-10-09T10:49:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010419100000000001 2023-10-09T10:49:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:49:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.232064+0000", 2023-10-09T10:49:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010409 2023-10-09T10:49:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.234485+0000", 2023-10-09T10:49:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024210680000000002 2023-10-09T10:49:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.251746+0000", 2023-10-09T10:49:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017260707 2023-10-09T10:49:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.291288+0000", 2023-10-09T10:49:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039542425999999999 2023-10-09T10:49:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.291380+0000", 2023-10-09T10:49:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1514999999999994e-05 2023-10-09T10:49:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.293156+0000", 2023-10-09T10:49:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001775878 2023-10-09T10:49:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106432 6.a 6:50fa9909:::benchmark_data_smithi125_244951_object106431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:49:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.249049+0000", 2023-10-09T10:49:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 191.30152221099999, 2023-10-09T10:49:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.49699488600000002, 2023-10-09T10:49:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T10:49:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106432 2023-10-09T10:49:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249049+0000", 2023-10-09T10:49:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249049+0000", 2023-10-09T10:49:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249047+0000", 2023-10-09T10:49:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:49:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249097+0000", 2023-10-09T10:49:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0584000000000002e-05 2023-10-09T10:49:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249117+0000", 2023-10-09T10:49:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9763999999999999e-05 2023-10-09T10:49:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249125+0000", 2023-10-09T10:49:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6040000000000004e-06 2023-10-09T10:49:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249189+0000", 2023-10-09T10:49:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3779e-05 2023-10-09T10:49:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249335+0000", 2023-10-09T10:49:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014603399999999999 2023-10-09T10:49:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:49:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249521+0000", 2023-10-09T10:49:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018619099999999999 2023-10-09T10:49:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.253194+0000", 2023-10-09T10:49:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003673103 2023-10-09T10:49:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.253298+0000", 2023-10-09T10:49:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103828 2023-10-09T10:49:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.271832+0000", 2023-10-09T10:49:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018534658999999998 2023-10-09T10:49:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:49:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.271843+0000", 2023-10-09T10:49:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0995000000000001e-05 2023-10-09T10:49:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.745982+0000", 2023-10-09T10:49:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.47413843 2023-10-09T10:49:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746044+0000", 2023-10-09T10:49:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1832e-05 2023-10-09T10:49:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106439 6.a 6:5c3c51c3:::benchmark_data_smithi125_244951_object106438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-10-09T10:49:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.284123+0000", 2023-10-09T10:49:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 191.26644800599999, 2023-10-09T10:49:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.47113701699999999, 2023-10-09T10:49:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106439 2023-10-09T10:49:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284123+0000", 2023-10-09T10:49:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284123+0000", 2023-10-09T10:49:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284122+0000", 2023-10-09T10:49:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284170+0000", 2023-10-09T10:49:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7929999999999997e-05 2023-10-09T10:49:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284181+0000", 2023-10-09T10:49:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1292e-05 2023-10-09T10:49:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284186+0000", 2023-10-09T10:49:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1089999999999997e-06 2023-10-09T10:49:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284266+0000", 2023-10-09T10:49:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0311999999999999e-05 2023-10-09T10:49:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:49:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284287+0000", 2023-10-09T10:49:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0704e-05 2023-10-09T10:49:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746068+0000", 2023-10-09T10:49:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.46178057300000003 2023-10-09T10:49:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746295+0000", 2023-10-09T10:49:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000227335 2023-10-09T10:49:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:49:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746417+0000", 2023-10-09T10:49:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000121816 2023-10-09T10:49:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.752322+0000", 2023-10-09T10:49:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059055230000000002 2023-10-09T10:49:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754107+0000", 2023-10-09T10:49:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0017852149999999999 2023-10-09T10:49:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754378+0000", 2023-10-09T10:49:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000270547 2023-10-09T10:49:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754422+0000", 2023-10-09T10:49:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4063000000000001e-05 2023-10-09T10:49:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.755260+0000", 2023-10-09T10:49:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00083801000000000004 2023-10-09T10:49:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106440 6.a 6:5e7f7caf:::benchmark_data_smithi125_244951_object106439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-10-09T10:49:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.284197+0000", 2023-10-09T10:49:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 191.26637447799999, 2023-10-09T10:49:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.47186083699999998, 2023-10-09T10:49:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106440 2023-10-09T10:49:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284197+0000", 2023-10-09T10:49:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284197+0000", 2023-10-09T10:49:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284196+0000", 2023-10-09T10:49:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284284+0000", 2023-10-09T10:49:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7646999999999996e-05 2023-10-09T10:49:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284292+0000", 2023-10-09T10:49:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2689999999999999e-06 2023-10-09T10:49:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284297+0000", 2023-10-09T10:49:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4419999999999997e-06 2023-10-09T10:49:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284344+0000", 2023-10-09T10:49:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6940000000000001e-05 2023-10-09T10:49:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:49:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284352+0000", 2023-10-09T10:49:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0749999999999998e-06 2023-10-09T10:49:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.749219+0000", 2023-10-09T10:49:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.46486643799999999 2023-10-09T10:49:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.749378+0000", 2023-10-09T10:49:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015880000000000001 2023-10-09T10:49:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:49:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.749563+0000", 2023-10-09T10:49:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018555399999999999 2023-10-09T10:49:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754147+0000", 2023-10-09T10:49:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045842050000000001 2023-10-09T10:49:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754155+0000", 2023-10-09T10:49:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2080000000000003e-06 2023-10-09T10:49:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.755321+0000", 2023-10-09T10:49:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011663839999999999 2023-10-09T10:49:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.755360+0000", 2023-10-09T10:49:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8943000000000003e-05 2023-10-09T10:49:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.756057+0000", 2023-10-09T10:49:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00069742899999999995 2023-10-09T10:49:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:152994 6.7 6:e184a006:::benchmark_data_smithi125_244951_object46551:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:49:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:35.519171+0000", 2023-10-09T10:49:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 98.031400035999994, 2023-10-09T10:49:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.084211253, 2023-10-09T10:49:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 152994 2023-10-09T10:49:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519171+0000", 2023-10-09T10:49:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519171+0000", 2023-10-09T10:49:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519170+0000", 2023-10-09T10:49:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519181+0000", 2023-10-09T10:49:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1884e-05 2023-10-09T10:49:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519196+0000", 2023-10-09T10:49:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.448e-05 2023-10-09T10:49:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519202+0000", 2023-10-09T10:49:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3670000000000002e-06 2023-10-09T10:49:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.522571+0000", 2023-10-09T10:49:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0033686639999999999 2023-10-09T10:49:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.522712+0000", 2023-10-09T10:49:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014130800000000001 2023-10-09T10:49:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:49:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.522828+0000", 2023-10-09T10:49:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115808 2023-10-09T10:49:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.527082+0000", 2023-10-09T10:49:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042537520000000004 2023-10-09T10:49:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.528250+0000", 2023-10-09T10:49:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011679310000000001 2023-10-09T10:49:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.603293+0000", 2023-10-09T10:49:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.075043377999999994 2023-10-09T10:49:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.603336+0000", 2023-10-09T10:49:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2774000000000001e-05 2023-10-09T10:49:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.603382+0000", 2023-10-09T10:49:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6279999999999997e-05 2023-10-09T10:49:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:153730 6.d 6:b4bba254:::benchmark_data_smithi125_244951_object47287:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:49:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:36.580818+0000", 2023-10-09T10:49:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 96.969753263000001, 2023-10-09T10:49:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.060300754999999998, 2023-10-09T10:49:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 153730 2023-10-09T10:49:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.580818+0000", 2023-10-09T10:49:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.580818+0000", 2023-10-09T10:49:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.580817+0000", 2023-10-09T10:49:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.580821+0000", 2023-10-09T10:49:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2270000000000002e-06 2023-10-09T10:49:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.580827+0000", 2023-10-09T10:49:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7410000000000003e-06 2023-10-09T10:49:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.580830+0000", 2023-10-09T10:49:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0469999999999998e-06 2023-10-09T10:49:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.585573+0000", 2023-10-09T10:49:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00474311 2023-10-09T10:49:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.585727+0000", 2023-10-09T10:49:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015385 2023-10-09T10:49:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:49:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.585876+0000", 2023-10-09T10:49:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014922700000000001 2023-10-09T10:49:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.591228+0000", 2023-10-09T10:49:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0053518940000000003 2023-10-09T10:49:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.609925+0000", 2023-10-09T10:49:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018696833999999999 2023-10-09T10:49:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.639420+0000", 2023-10-09T10:49:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029494933000000001 2023-10-09T10:49:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.639454+0000", 2023-10-09T10:49:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3986000000000003e-05 2023-10-09T10:49:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.641118+0000", 2023-10-09T10:49:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0016643680000000001 2023-10-09T10:49:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:153733 6.d 6:b575b55b:::benchmark_data_smithi125_244951_object47290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:49:13.608 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:36.588580+0000", 2023-10-09T10:49:13.608 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 96.961991159999997, 2023-10-09T10:49:13.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.071524196999999998, 2023-10-09T10:49:13.608 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.608 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.608 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:13.608 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:13.608 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:13.608 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 153733 2023-10-09T10:49:13.608 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.608 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.608 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588580+0000", 2023-10-09T10:49:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588580+0000", 2023-10-09T10:49:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588578+0000", 2023-10-09T10:49:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588585+0000", 2023-10-09T10:49:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.139e-06 2023-10-09T10:49:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588596+0000", 2023-10-09T10:49:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.163e-05 2023-10-09T10:49:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588602+0000", 2023-10-09T10:49:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6869999999999999e-06 2023-10-09T10:49:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.595518+0000", 2023-10-09T10:49:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0069160599999999999 2023-10-09T10:49:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.595663+0000", 2023-10-09T10:49:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014470299999999999 2023-10-09T10:49:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:49:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.595807+0000", 2023-10-09T10:49:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014449800000000001 2023-10-09T10:49:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.600067+0000", 2023-10-09T10:49:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042602200000000003 2023-10-09T10:49:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.641151+0000", 2023-10-09T10:49:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.041083736000000003 2023-10-09T10:49:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.657793+0000", 2023-10-09T10:49:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016641420000000001 2023-10-09T10:49:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.657848+0000", 2023-10-09T10:49:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5611e-05 2023-10-09T10:49:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.660104+0000", 2023-10-09T10:49:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022558880000000002 2023-10-09T10:49:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:162477 6.a 6:52278cc7:::benchmark_data_smithi125_244951_object56034:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-10-09T10:49:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:49.307801+0000", 2023-10-09T10:49:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 84.242770003000004, 2023-10-09T10:49:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.061925675999999999, 2023-10-09T10:49:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162477 2023-10-09T10:49:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.307801+0000", 2023-10-09T10:49:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.307801+0000", 2023-10-09T10:49:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.307801+0000", 2023-10-09T10:49:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967296 2023-10-09T10:49:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.307802+0000", 2023-10-09T10:49:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.579e-06 2023-10-09T10:49:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.307805+0000", 2023-10-09T10:49:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8059999999999999e-06 2023-10-09T10:49:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.307807+0000", 2023-10-09T10:49:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2359999999999999e-06 2023-10-09T10:49:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.328037+0000", 2023-10-09T10:49:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020229737000000001 2023-10-09T10:49:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.328177+0000", 2023-10-09T10:49:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013998099999999999 2023-10-09T10:49:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:49:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.328323+0000", 2023-10-09T10:49:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000146024 2023-10-09T10:49:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.344032+0000", 2023-10-09T10:49:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015708751999999999 2023-10-09T10:49:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.356007+0000", 2023-10-09T10:49:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011975507 2023-10-09T10:49:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.369010+0000", 2023-10-09T10:49:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013002796 2023-10-09T10:49:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.369069+0000", 2023-10-09T10:49:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8711000000000001e-05 2023-10-09T10:49:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.369727+0000", 2023-10-09T10:49:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00065775299999999996 2023-10-09T10:49:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:162483 6.a 6:5efc2291:::benchmark_data_smithi125_244951_object56040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-10-09T10:49:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:49.308048+0000", 2023-10-09T10:49:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 84.242523482999999, 2023-10-09T10:49:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.062044714000000001, 2023-10-09T10:49:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162483 2023-10-09T10:49:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.308048+0000", 2023-10-09T10:49:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.308048+0000", 2023-10-09T10:49:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.308047+0000", 2023-10-09T10:49:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:49:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.308052+0000", 2023-10-09T10:49:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7339999999999996e-06 2023-10-09T10:49:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.308065+0000", 2023-10-09T10:49:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2945000000000001e-05 2023-10-09T10:49:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.308072+0000", 2023-10-09T10:49:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.002e-06 2023-10-09T10:49:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.333025+0000", 2023-10-09T10:49:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024952407999999999 2023-10-09T10:49:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.333161+0000", 2023-10-09T10:49:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000135928 2023-10-09T10:49:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:49:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.333302+0000", 2023-10-09T10:49:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014128599999999999 2023-10-09T10:49:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.349518+0000", 2023-10-09T10:49:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016216147 2023-10-09T10:49:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.356145+0000", 2023-10-09T10:49:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.006627099 2023-10-09T10:49:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.369763+0000", 2023-10-09T10:49:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013617895 2023-10-09T10:49:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.369805+0000", 2023-10-09T10:49:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2206999999999998e-05 2023-10-09T10:49:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.370092+0000", 2023-10-09T10:49:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00028702799999999999 2023-10-09T10:49:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:189056 6.d 6:bfa083a6:::benchmark_data_smithi125_244951_object82613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2023-10-09T10:49:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:28.307602+0000", 2023-10-09T10:49:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 45.242968578999999, 2023-10-09T10:49:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.060809119000000002, 2023-10-09T10:49:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 189056 2023-10-09T10:49:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:28.307602+0000", 2023-10-09T10:49:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:28.307602+0000", 2023-10-09T10:49:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:28.307602+0000", 2023-10-09T10:49:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:28.307605+0000", 2023-10-09T10:49:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7439999999999999e-06 2023-10-09T10:49:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:28.307610+0000", 2023-10-09T10:49:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2249999999999999e-06 2023-10-09T10:49:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:28.307614+0000", 2023-10-09T10:49:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.399e-06 2023-10-09T10:49:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:28.327862+0000", 2023-10-09T10:49:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020248042000000001 2023-10-09T10:49:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:28.328029+0000", 2023-10-09T10:49:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016741400000000001 2023-10-09T10:49:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:49:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:28.328184+0000", 2023-10-09T10:49:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015525399999999999 2023-10-09T10:49:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:28.335823+0000", 2023-10-09T10:49:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0076390920000000001 2023-10-09T10:49:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:28.355199+0000", 2023-10-09T10:49:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019375488 2023-10-09T10:49:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:28.366991+0000", 2023-10-09T10:49:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011792410999999999 2023-10-09T10:49:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:28.367033+0000", 2023-10-09T10:49:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1411e-05 2023-10-09T10:49:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:28.368412+0000", 2023-10-09T10:49:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001378883 2023-10-09T10:49:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:190320 6.d 6:b39af7cc:::benchmark_data_smithi125_244951_object83877:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2023-10-09T10:49:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:30.246705+0000", 2023-10-09T10:49:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 43.303866495999998, 2023-10-09T10:49:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.059646449999999997, 2023-10-09T10:49:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 190320 2023-10-09T10:49:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.246705+0000", 2023-10-09T10:49:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.246705+0000", 2023-10-09T10:49:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.246704+0000", 2023-10-09T10:49:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.246708+0000", 2023-10-09T10:49:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4089999999999996e-06 2023-10-09T10:49:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.246717+0000", 2023-10-09T10:49:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5660000000000007e-06 2023-10-09T10:49:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.246721+0000", 2023-10-09T10:49:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4859999999999999e-06 2023-10-09T10:49:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.264992+0000", 2023-10-09T10:49:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018270150999999998 2023-10-09T10:49:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.265140+0000", 2023-10-09T10:49:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014855099999999999 2023-10-09T10:49:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:49:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.265322+0000", 2023-10-09T10:49:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000182184 2023-10-09T10:49:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.274693+0000", 2023-10-09T10:49:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0093706169999999995 2023-10-09T10:49:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.288972+0000", 2023-10-09T10:49:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014279427000000001 2023-10-09T10:49:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.304422+0000", 2023-10-09T10:49:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01544918 2023-10-09T10:49:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.304478+0000", 2023-10-09T10:49:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6002999999999999e-05 2023-10-09T10:49:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.306351+0000", 2023-10-09T10:49:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018734540000000001 2023-10-09T10:49:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:190474 6.e e184/154 6:73a148c1:::benchmark_data_smithi125_244951_object84031:head v 184'11796, mlcod=104'1056)", 2023-10-09T10:49:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:30.473779+0000", 2023-10-09T10:49:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 43.076791804999999, 2023-10-09T10:49:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.065784969999999998, 2023-10-09T10:49:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.473779+0000", 2023-10-09T10:49:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.473779+0000", 2023-10-09T10:49:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.473778+0000", 2023-10-09T10:49:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.473789+0000", 2023-10-09T10:49:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0920999999999999e-05 2023-10-09T10:49:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.473803+0000", 2023-10-09T10:49:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4552999999999999e-05 2023-10-09T10:49:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.473810+0000", 2023-10-09T10:49:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5699999999999998e-06 2023-10-09T10:49:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.483312+0000", 2023-10-09T10:49:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0095017360000000002 2023-10-09T10:49:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.483337+0000", 2023-10-09T10:49:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5369000000000001e-05 2023-10-09T10:49:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.539532+0000", 2023-10-09T10:49:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.056195334 2023-10-09T10:49:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.539564+0000", 2023-10-09T10:49:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1739999999999998e-05 2023-10-09T10:49:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:190482 6.e e184/154 6:70754c6a:::benchmark_data_smithi125_244951_object84039:head v 184'11797, mlcod=104'1056)", 2023-10-09T10:49:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:30.478668+0000", 2023-10-09T10:49:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 43.071902813000001, 2023-10-09T10:49:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.060920808, 2023-10-09T10:49:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.478668+0000", 2023-10-09T10:49:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.478668+0000", 2023-10-09T10:49:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.478667+0000", 2023-10-09T10:49:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:49:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.478676+0000", 2023-10-09T10:49:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3499999999999997e-06 2023-10-09T10:49:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.478688+0000", 2023-10-09T10:49:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2099000000000001e-05 2023-10-09T10:49:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.478695+0000", 2023-10-09T10:49:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7850000000000003e-06 2023-10-09T10:49:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.495713+0000", 2023-10-09T10:49:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01701824 2023-10-09T10:49:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.495740+0000", 2023-10-09T10:49:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6835999999999999e-05 2023-10-09T10:49:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.539569+0000", 2023-10-09T10:49:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.043829385999999998 2023-10-09T10:49:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.539589+0000", 2023-10-09T10:49:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0030999999999998e-05 2023-10-09T10:49:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:191509 6.d 6:b42a920c:::benchmark_data_smithi125_244951_object85066:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2023-10-09T10:49:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:32.051748+0000", 2023-10-09T10:49:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 41.498823473999998, 2023-10-09T10:49:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.064273379000000005, 2023-10-09T10:49:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 191509 2023-10-09T10:49:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051748+0000", 2023-10-09T10:49:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051748+0000", 2023-10-09T10:49:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051747+0000", 2023-10-09T10:49:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051750+0000", 2023-10-09T10:49:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.402e-06 2023-10-09T10:49:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051755+0000", 2023-10-09T10:49:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0089999999999998e-06 2023-10-09T10:49:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051757+0000", 2023-10-09T10:49:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9179999999999998e-06 2023-10-09T10:49:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051798+0000", 2023-10-09T10:49:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0911000000000002e-05 2023-10-09T10:49:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051942+0000", 2023-10-09T10:49:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014350699999999999 2023-10-09T10:49:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:49:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.052082+0000", 2023-10-09T10:49:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014008 2023-10-09T10:49:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.057385+0000", 2023-10-09T10:49:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0053033250000000002 2023-10-09T10:49:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.082093+0000", 2023-10-09T10:49:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024707959000000002 2023-10-09T10:49:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.114170+0000", 2023-10-09T10:49:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.032076503999999999 2023-10-09T10:49:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.114226+0000", 2023-10-09T10:49:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6323000000000003e-05 2023-10-09T10:49:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.116021+0000", 2023-10-09T10:49:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001794805 2023-10-09T10:49:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:198775 6.e e186/154 6:70ffdb86:::benchmark_data_smithi125_244951_object92332:head v 186'12373, mlcod=104'1056)", 2023-10-09T10:49:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:43.027484+0000", 2023-10-09T10:49:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 30.523087268000001, 2023-10-09T10:49:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.066535034000000007, 2023-10-09T10:49:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.027484+0000", 2023-10-09T10:49:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.027484+0000", 2023-10-09T10:49:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.027483+0000", 2023-10-09T10:49:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:49:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.027493+0000", 2023-10-09T10:49:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0549e-05 2023-10-09T10:49:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.027509+0000", 2023-10-09T10:49:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6130999999999999e-05 2023-10-09T10:49:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.027518+0000", 2023-10-09T10:49:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3780000000000001e-06 2023-10-09T10:49:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.031517+0000", 2023-10-09T10:49:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003999379 2023-10-09T10:49:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.031540+0000", 2023-10-09T10:49:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2920000000000001e-05 2023-10-09T10:49:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.093982+0000", 2023-10-09T10:49:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.062441607000000003 2023-10-09T10:49:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.094019+0000", 2023-10-09T10:49:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7156999999999997e-05 2023-10-09T10:49:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:211949 6.e e189/154 6:700b4d98:::benchmark_data_smithi125_244951_object105506:head v 189'13175, mlcod=104'1056)", 2023-10-09T10:49:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:01.372020+0000", 2023-10-09T10:49:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 12.178551321, 2023-10-09T10:49:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.065605420999999997, 2023-10-09T10:49:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.372020+0000", 2023-10-09T10:49:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.372020+0000", 2023-10-09T10:49:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.372019+0000", 2023-10-09T10:49:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.372024+0000", 2023-10-09T10:49:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8319999999999996e-06 2023-10-09T10:49:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.372033+0000", 2023-10-09T10:49:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5890000000000003e-06 2023-10-09T10:49:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.372038+0000", 2023-10-09T10:49:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9479999999999999e-06 2023-10-09T10:49:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.401091+0000", 2023-10-09T10:49:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029053241 2023-10-09T10:49:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.401121+0000", 2023-10-09T10:49:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0014000000000001e-05 2023-10-09T10:49:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.437580+0000", 2023-10-09T10:49:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.036459566999999998 2023-10-09T10:49:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.437625+0000", 2023-10-09T10:49:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4719999999999999e-05 2023-10-09T10:49:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:211957 6.e e189/154 6:74f44ea0:::benchmark_data_smithi125_244951_object105514:head v 189'13176, mlcod=104'1056)", 2023-10-09T10:49:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:01.376382+0000", 2023-10-09T10:49:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 12.174189296, 2023-10-09T10:49:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.06126479, 2023-10-09T10:49:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.376382+0000", 2023-10-09T10:49:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.376382+0000", 2023-10-09T10:49:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.376381+0000", 2023-10-09T10:49:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.376386+0000", 2023-10-09T10:49:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5369999999999999e-06 2023-10-09T10:49:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.376393+0000", 2023-10-09T10:49:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.345e-06 2023-10-09T10:49:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.376398+0000", 2023-10-09T10:49:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8300000000000003e-06 2023-10-09T10:49:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.410955+0000", 2023-10-09T10:49:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.034556849000000001 2023-10-09T10:49:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.410984+0000", 2023-10-09T10:49:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9366e-05 2023-10-09T10:49:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.437629+0000", 2023-10-09T10:49:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026644404999999999 2023-10-09T10:49:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.437647+0000", 2023-10-09T10:49:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7876e-05 2023-10-09T10:49:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:212017 6.e e189/154 6:72d7ee3b:::benchmark_data_smithi125_244951_object105574:head v 189'13182, mlcod=104'1056)", 2023-10-09T10:49:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:01.461617+0000", 2023-10-09T10:49:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 12.088954230000001, 2023-10-09T10:49:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069220272999999999, 2023-10-09T10:49:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.461617+0000", 2023-10-09T10:49:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.461617+0000", 2023-10-09T10:49:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.461616+0000", 2023-10-09T10:49:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.461622+0000", 2023-10-09T10:49:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3009999999999997e-06 2023-10-09T10:49:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.461626+0000", 2023-10-09T10:49:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2520000000000001e-06 2023-10-09T10:49:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.461629+0000", 2023-10-09T10:49:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6579999999999998e-06 2023-10-09T10:49:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.467974+0000", 2023-10-09T10:49:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0063450989999999999 2023-10-09T10:49:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.467993+0000", 2023-10-09T10:49:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9394000000000001e-05 2023-10-09T10:49:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.530795+0000", 2023-10-09T10:49:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.062801642000000005 2023-10-09T10:49:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.530837+0000", 2023-10-09T10:49:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2338e-05 2023-10-09T10:49:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.680 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.680 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.680 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:212020 6.e e189/154 6:7c93ac91:::benchmark_data_smithi125_244951_object105577:head v 189'13183, mlcod=104'1056)", 2023-10-09T10:49:13.680 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:01.470082+0000", 2023-10-09T10:49:13.680 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 12.080489264000001, 2023-10-09T10:49:13.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.060778840000000001, 2023-10-09T10:49:13.680 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:13.680 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:13.680 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:13.680 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.680 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:13.680 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.470082+0000", 2023-10-09T10:49:13.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:13.681 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.470082+0000", 2023-10-09T10:49:13.681 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:13.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:13.681 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.470081+0000", 2023-10-09T10:49:13.681 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:13.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.682 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:13.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.470096+0000", 2023-10-09T10:49:13.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4739000000000001e-05 2023-10-09T10:49:13.682 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.682 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.682 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:13.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.470104+0000", 2023-10-09T10:49:13.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5140000000000004e-06 2023-10-09T10:49:13.682 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.682 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.682 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:13.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.470108+0000", 2023-10-09T10:49:13.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4569999999999998e-06 2023-10-09T10:49:13.683 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.683 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:13.683 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.478683+0000", 2023-10-09T10:49:13.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0085750010000000005 2023-10-09T10:49:13.683 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.683 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:13.683 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.478707+0000", 2023-10-09T10:49:13.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3842999999999999e-05 2023-10-09T10:49:13.683 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.684 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:13.684 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.530842+0000", 2023-10-09T10:49:13.684 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.052134947000000001 2023-10-09T10:49:13.684 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:13.684 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:13.684 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:13.684 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.530861+0000", 2023-10-09T10:49:13.684 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8680000000000001e-05 2023-10-09T10:49:13.684 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.684 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.684 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.684 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:13.684 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:13.685 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:49:13.685 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:13.686 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:13.689 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:13 smithi125 ceph-mon[195507]: failed to encode map e193 with expected crc 2023-10-09T10:49:13.689 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:13 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 20475/423 objects degraded (4840.426%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:49:13.689 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:13 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1776984049' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:49:13.689 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:13 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T10:49:13.689 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:13 smithi125 ceph-mon[200197]: failed to encode map e193 with expected crc 2023-10-09T10:49:13.690 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:13 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 20475/423 objects degraded (4840.426%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:49:13.690 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:13 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1776984049' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:49:13.690 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:13 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T10:49:13.705 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:13 smithi155 ceph-mon[184854]: failed to encode map e193 with expected crc 2023-10-09T10:49:13.705 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:13 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 20475/423 objects degraded (4840.426%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:49:13.705 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:13 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1776984049' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:49:13.705 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:13 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T10:49:13.719 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:13.724 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:49:13.926 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:49:13.999 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:14.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:14.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:14.103 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:14.133 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:14.154 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:14.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:14.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:14.198 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:49:14.257 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:49:14.258 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:49:14.258 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:14.288 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:14.307 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:14.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:14.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:14.412 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:14.441 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:14.461 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:14.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:14.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:14.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:14 smithi125 ceph-mon[200197]: pgmap v434: 93 pgs: 93 active+clean; 582 KiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:49:14.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:14 smithi125 ceph-mon[200197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20475/423 objects degraded (4840.426%), 6 pgs degraded, 6 pgs undersized) 2023-10-09T10:49:14.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:14 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:49:14.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:14 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1776984049' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:49:14.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T10:49:14.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:14 smithi125 ceph-mon[200197]: osdmap e194: 8 total, 8 up, 7 in 2023-10-09T10:49:14.533 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:14 smithi125 ceph-mon[200197]: failed to encode map e194 with expected crc 2023-10-09T10:49:14.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:14 smithi125 ceph-mon[195507]: pgmap v434: 93 pgs: 93 active+clean; 582 KiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:49:14.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:14 smithi125 ceph-mon[195507]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20475/423 objects degraded (4840.426%), 6 pgs degraded, 6 pgs undersized) 2023-10-09T10:49:14.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:14 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:49:14.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:14 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1776984049' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:49:14.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T10:49:14.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:14 smithi125 ceph-mon[195507]: osdmap e194: 8 total, 8 up, 7 in 2023-10-09T10:49:14.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:14 smithi125 ceph-mon[195507]: failed to encode map e194 with expected crc 2023-10-09T10:49:14.564 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:14.592 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:14.712 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:14.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:14.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:14 smithi155 ceph-mon[184854]: pgmap v434: 93 pgs: 93 active+clean; 582 KiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:49:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:14 smithi155 ceph-mon[184854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20475/423 objects degraded (4840.426%), 6 pgs degraded, 6 pgs undersized) 2023-10-09T10:49:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:14 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:49:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:14 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1776984049' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:49:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T10:49:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:14 smithi155 ceph-mon[184854]: osdmap e194: 8 total, 8 up, 7 in 2023-10-09T10:49:14.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:14 smithi155 ceph-mon[184854]: failed to encode map e194 with expected crc 2023-10-09T10:49:14.821 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:14.848 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:15.003 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":435,"stamp":"2023-10-09T10:49:13.707382+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":1772290,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3436,"ondisk_log_size":3436,"up":279,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2038120,"kb_used_data":292912,"kb_used_omap":44,"kb_used_meta":1745107,"kb_avail":654177944,"statfs":{"total":671965249536,"available":669878214656,"internally_reserved":0,"allocated":299941888,"data_stored":294052685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1786989985},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-36765720,"num_objects":-561,"num_object_clones":0,"num_object_copies":-1683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-22746,"num_objects_misplaced":-73,"num_objects_unfound":0,"num_objects_dirty":-561,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-203707,"num_write_kb":-6258529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-32571,"num_bytes_recovered":-1625423872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.523825"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":185,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.527024+0000","last_change":"2023-10-09T10:46:32.250979+0000","last_active":"2023-10-09T10:49:03.527024+0000","last_peered":"2023-10-09T10:49:03.527024+0000","last_clean":"2023-10-09T10:49:03.527024+0000","last_became_active":"2023-10-09T10:46:32.250625+0000","last_became_peered":"2023-10-09T10:46:32.250625+0000","last_unstale":"2023-10-09T10:49:03.527024+0000","last_undegraded":"2023-10-09T10:49:03.527024+0000","last_fullsized":"2023-10-09T10:49:03.527024+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:34.858348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":161,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.406489+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:49:03.406489+0000","last_peered":"2023-10-09T10:49:03.406489+0000","last_clean":"2023-10-09T10:49:03.406489+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:49:03.406489+0000","last_undegraded":"2023-10-09T10:49:03.406489+0000","last_fullsized":"2023-10-09T10:49:03.406489+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":145,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.402921+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:49:04.402921+0000","last_peered":"2023-10-09T10:49:04.402921+0000","last_clean":"2023-10-09T10:49:04.402921+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:49:04.402921+0000","last_undegraded":"2023-10-09T10:49:04.402921+0000","last_fullsized":"2023-10-09T10:49:04.402921+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":62,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830515+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:49:04.830515+0000","last_peered":"2023-10-09T10:49:04.830515+0000","last_clean":"2023-10-09T10:49:04.830515+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:49:04.830515+0000","last_undegraded":"2023-10-09T10:49:04.830515+0000","last_fullsized":"2023-10-09T10:49:04.830515+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":197,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.408853+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:49:03.408853+0000","last_peered":"2023-10-09T10:49:03.408853+0000","last_clean":"2023-10-09T10:49:03.408853+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:49:03.408853+0000","last_undegraded":"2023-10-09T10:49:03.408853+0000","last_fullsized":"2023-10-09T10:49:03.408853+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":130,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.420692+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:49:03.420692+0000","last_peered":"2023-10-09T10:49:03.420692+0000","last_clean":"2023-10-09T10:49:03.420692+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:49:03.420692+0000","last_undegraded":"2023-10-09T10:49:03.420692+0000","last_fullsized":"2023-10-09T10:49:03.420692+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":62,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830585+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:49:04.830585+0000","last_peered":"2023-10-09T10:49:04.830585+0000","last_clean":"2023-10-09T10:49:04.830585+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:49:04.830585+0000","last_undegraded":"2023-10-09T10:49:04.830585+0000","last_fullsized":"2023-10-09T10:49:04.830585+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":161,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.408925+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:49:03.408925+0000","last_peered":"2023-10-09T10:49:03.408925+0000","last_clean":"2023-10-09T10:49:03.408925+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:49:03.408925+0000","last_undegraded":"2023-10-09T10:49:03.408925+0000","last_fullsized":"2023-10-09T10:49:03.408925+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":527,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.829899+0000","last_change":"2023-10-09T10:46:05.620365+0000","last_active":"2023-10-09T10:49:04.829899+0000","last_peered":"2023-10-09T10:49:04.829899+0000","last_clean":"2023-10-09T10:49:04.829899+0000","last_became_active":"2023-10-09T10:46:05.620024+0000","last_became_peered":"2023-10-09T10:46:05.620024+0000","last_unstale":"2023-10-09T10:49:04.829899+0000","last_undegraded":"2023-10-09T10:49:04.829899+0000","last_fullsized":"2023-10-09T10:49:04.829899+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":157,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.420598+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:49:03.420598+0000","last_peered":"2023-10-09T10:49:03.420598+0000","last_clean":"2023-10-09T10:49:03.420598+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:49:03.420598+0000","last_undegraded":"2023-10-09T10:49:03.420598+0000","last_fullsized":"2023-10-09T10:49:03.420598+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":118,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.409087+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:49:03.409087+0000","last_peered":"2023-10-09T10:49:03.409087+0000","last_clean":"2023-10-09T10:49:03.409087+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:49:03.409087+0000","last_undegraded":"2023-10-09T10:49:03.409087+0000","last_fullsized":"2023-10-09T10:49:03.409087+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":113,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.406413+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:49:03.406413+0000","last_peered":"2023-10-09T10:49:03.406413+0000","last_clean":"2023-10-09T10:49:03.406413+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:49:03.406413+0000","last_undegraded":"2023-10-09T10:49:03.406413+0000","last_fullsized":"2023-10-09T10:49:03.406413+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":154,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.402047+0000","last_change":"2023-10-09T10:46:32.249921+0000","last_active":"2023-10-09T10:49:04.402047+0000","last_peered":"2023-10-09T10:49:04.402047+0000","last_clean":"2023-10-09T10:49:04.402047+0000","last_became_active":"2023-10-09T10:46:32.249590+0000","last_became_peered":"2023-10-09T10:46:32.249590+0000","last_unstale":"2023-10-09T10:49:04.402047+0000","last_undegraded":"2023-10-09T10:49:04.402047+0000","last_fullsized":"2023-10-09T10:49:04.402047+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:25.169752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":162,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.527097+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:49:03.527097+0000","last_peered":"2023-10-09T10:49:03.527097+0000","last_clean":"2023-10-09T10:49:03.527097+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:49:03.527097+0000","last_undegraded":"2023-10-09T10:49:03.527097+0000","last_fullsized":"2023-10-09T10:49:03.527097+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":156,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.527005+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:49:03.527005+0000","last_peered":"2023-10-09T10:49:03.527005+0000","last_clean":"2023-10-09T10:49:03.527005+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:49:03.527005+0000","last_undegraded":"2023-10-09T10:49:03.527005+0000","last_fullsized":"2023-10-09T10:49:03.527005+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":527,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.831105+0000","last_change":"2023-10-09T10:46:05.618276+0000","last_active":"2023-10-09T10:49:04.831105+0000","last_peered":"2023-10-09T10:49:04.831105+0000","last_clean":"2023-10-09T10:49:04.831105+0000","last_became_active":"2023-10-09T10:46:05.616509+0000","last_became_peered":"2023-10-09T10:46:05.616509+0000","last_unstale":"2023-10-09T10:49:04.831105+0000","last_undegraded":"2023-10-09T10:49:04.831105+0000","last_fullsized":"2023-10-09T10:49:04.831105+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":174,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.405586+0000","last_change":"2023-10-09T10:46:32.250256+0000","last_active":"2023-10-09T10:49:03.405586+0000","last_peered":"2023-10-09T10:49:03.405586+0000","last_clean":"2023-10-09T10:49:03.405586+0000","last_became_active":"2023-10-09T10:46:32.243312+0000","last_became_peered":"2023-10-09T10:46:32.243312+0000","last_unstale":"2023-10-09T10:49:03.405586+0000","last_undegraded":"2023-10-09T10:49:03.405586+0000","last_fullsized":"2023-10-09T10:49:03.405586+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:08.059777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":184,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.407563+0000","last_change":"2023-10-09T10:46:32.237577+0000","last_active":"2023-10-09T10:49:03.407563+0000","last_peered":"2023-10-09T10:49:03.407563+0000","last_clean":"2023-10-09T10:49:03.407563+0000","last_became_active":"2023-10-09T10:46:32.237316+0000","last_became_peered":"2023-10-09T10:46:32.237316+0000","last_unstale":"2023-10-09T10:49:03.407563+0000","last_undegraded":"2023-10-09T10:49:03.407563+0000","last_fullsized":"2023-10-09T10:49:03.407563+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:39.684602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830018+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:49:04.830018+0000","last_peered":"2023-10-09T10:49:04.830018+0000","last_clean":"2023-10-09T10:49:04.830018+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:49:04.830018+0000","last_undegraded":"2023-10-09T10:49:04.830018+0000","last_fullsized":"2023-10-09T10:49:04.830018+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":62,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.829968+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:49:04.829968+0000","last_peered":"2023-10-09T10:49:04.829968+0000","last_clean":"2023-10-09T10:49:04.829968+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:49:04.829968+0000","last_undegraded":"2023-10-09T10:49:04.829968+0000","last_fullsized":"2023-10-09T10:49:04.829968+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":156,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.420841+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:49:03.420841+0000","last_peered":"2023-10-09T10:49:03.420841+0000","last_clean":"2023-10-09T10:49:03.420841+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:49:03.420841+0000","last_undegraded":"2023-10-09T10:49:03.420841+0000","last_fullsized":"2023-10-09T10:49:03.420841+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830371+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:49:04.830371+0000","last_peered":"2023-10-09T10:49:04.830371+0000","last_clean":"2023-10-09T10:49:04.830371+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:49:04.830371+0000","last_undegraded":"2023-10-09T10:49:04.830371+0000","last_fullsized":"2023-10-09T10:49:04.830371+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":102,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.526886+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:49:03.526886+0000","last_peered":"2023-10-09T10:49:03.526886+0000","last_clean":"2023-10-09T10:49:03.526886+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:49:03.526886+0000","last_undegraded":"2023-10-09T10:49:03.526886+0000","last_fullsized":"2023-10-09T10:49:03.526886+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":62,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830757+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:49:04.830757+0000","last_peered":"2023-10-09T10:49:04.830757+0000","last_clean":"2023-10-09T10:49:04.830757+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:49:04.830757+0000","last_undegraded":"2023-10-09T10:49:04.830757+0000","last_fullsized":"2023-10-09T10:49:04.830757+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.829688+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:49:04.829688+0000","last_peered":"2023-10-09T10:49:04.829688+0000","last_clean":"2023-10-09T10:49:04.829688+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:49:04.829688+0000","last_undegraded":"2023-10-09T10:49:04.829688+0000","last_fullsized":"2023-10-09T10:49:04.829688+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":72,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.829934+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:49:04.829934+0000","last_peered":"2023-10-09T10:49:04.829934+0000","last_clean":"2023-10-09T10:49:04.829934+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:49:04.829934+0000","last_undegraded":"2023-10-09T10:49:04.829934+0000","last_fullsized":"2023-10-09T10:49:04.829934+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":161,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.407110+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:49:03.407110+0000","last_peered":"2023-10-09T10:49:03.407110+0000","last_clean":"2023-10-09T10:49:03.407110+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:49:03.407110+0000","last_undegraded":"2023-10-09T10:49:03.407110+0000","last_fullsized":"2023-10-09T10:49:03.407110+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.831415+0000","last_change":"2023-10-09T10:46:05.620574+0000","last_active":"2023-10-09T10:49:04.831415+0000","last_peered":"2023-10-09T10:49:04.831415+0000","last_clean":"2023-10-09T10:49:04.831415+0000","last_became_active":"2023-10-09T10:46:05.620212+0000","last_became_peered":"2023-10-09T10:46:05.620212+0000","last_unstale":"2023-10-09T10:49:04.831415+0000","last_undegraded":"2023-10-09T10:49:04.831415+0000","last_fullsized":"2023-10-09T10:49:04.831415+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":185,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.405905+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:49:03.405905+0000","last_peered":"2023-10-09T10:49:03.405905+0000","last_clean":"2023-10-09T10:49:03.405905+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:49:03.405905+0000","last_undegraded":"2023-10-09T10:49:03.405905+0000","last_fullsized":"2023-10-09T10:49:03.405905+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":139,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.420398+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:49:03.420398+0000","last_peered":"2023-10-09T10:49:03.420398+0000","last_clean":"2023-10-09T10:49:03.420398+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:49:03.420398+0000","last_undegraded":"2023-10-09T10:49:03.420398+0000","last_fullsized":"2023-10-09T10:49:03.420398+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":508,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830052+0000","last_change":"2023-10-09T10:46:02.631378+0000","last_active":"2023-10-09T10:49:04.830052+0000","last_peered":"2023-10-09T10:49:04.830052+0000","last_clean":"2023-10-09T10:49:04.830052+0000","last_became_active":"2023-10-09T10:46:02.629142+0000","last_became_peered":"2023-10-09T10:46:02.629142+0000","last_unstale":"2023-10-09T10:49:04.830052+0000","last_undegraded":"2023-10-09T10:49:04.830052+0000","last_fullsized":"2023-10-09T10:49:04.830052+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":158,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.406263+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:49:03.406263+0000","last_peered":"2023-10-09T10:49:03.406263+0000","last_clean":"2023-10-09T10:49:03.406263+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:49:03.406263+0000","last_undegraded":"2023-10-09T10:49:03.406263+0000","last_fullsized":"2023-10-09T10:49:03.406263+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":193,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.526881+0000","last_change":"2023-10-09T10:46:32.251035+0000","last_active":"2023-10-09T10:49:03.526881+0000","last_peered":"2023-10-09T10:49:03.526881+0000","last_clean":"2023-10-09T10:49:03.526881+0000","last_became_active":"2023-10-09T10:46:32.250293+0000","last_became_peered":"2023-10-09T10:46:32.250293+0000","last_unstale":"2023-10-09T10:49:03.526881+0000","last_undegraded":"2023-10-09T10:49:03.526881+0000","last_fullsized":"2023-10-09T10:49:03.526881+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.787766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830273+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:49:04.830273+0000","last_peered":"2023-10-09T10:49:04.830273+0000","last_clean":"2023-10-09T10:49:04.830273+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:49:04.830273+0000","last_undegraded":"2023-10-09T10:49:04.830273+0000","last_fullsized":"2023-10-09T10:49:04.830273+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":113,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.420763+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:49:03.420763+0000","last_peered":"2023-10-09T10:49:03.420763+0000","last_clean":"2023-10-09T10:49:03.420763+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:49:03.420763+0000","last_undegraded":"2023-10-09T10:49:03.420763+0000","last_fullsized":"2023-10-09T10:49:03.420763+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":131,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.402303+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:49:04.402303+0000","last_peered":"2023-10-09T10:49:04.402303+0000","last_clean":"2023-10-09T10:49:04.402303+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:49:04.402303+0000","last_undegraded":"2023-10-09T10:49:04.402303+0000","last_fullsized":"2023-10-09T10:49:04.402303+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830351+0000","last_change":"2023-10-09T10:46:05.620030+0000","last_active":"2023-10-09T10:49:04.830351+0000","last_peered":"2023-10-09T10:49:04.830351+0000","last_clean":"2023-10-09T10:49:04.830351+0000","last_became_active":"2023-10-09T10:46:05.619694+0000","last_became_peered":"2023-10-09T10:46:05.619694+0000","last_unstale":"2023-10-09T10:49:04.830351+0000","last_undegraded":"2023-10-09T10:49:04.830351+0000","last_fullsized":"2023-10-09T10:49:04.830351+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":157,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.420698+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:49:03.420698+0000","last_peered":"2023-10-09T10:49:03.420698+0000","last_clean":"2023-10-09T10:49:03.420698+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:49:03.420698+0000","last_undegraded":"2023-10-09T10:49:03.420698+0000","last_fullsized":"2023-10-09T10:49:03.420698+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":62,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830336+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:49:04.830336+0000","last_peered":"2023-10-09T10:49:04.830336+0000","last_clean":"2023-10-09T10:49:04.830336+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:49:04.830336+0000","last_undegraded":"2023-10-09T10:49:04.830336+0000","last_fullsized":"2023-10-09T10:49:04.830336+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":519,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.831031+0000","last_change":"2023-10-09T10:46:05.618134+0000","last_active":"2023-10-09T10:49:04.831031+0000","last_peered":"2023-10-09T10:49:04.831031+0000","last_clean":"2023-10-09T10:49:04.831031+0000","last_became_active":"2023-10-09T10:46:05.616057+0000","last_became_peered":"2023-10-09T10:46:05.616057+0000","last_unstale":"2023-10-09T10:49:04.831031+0000","last_undegraded":"2023-10-09T10:49:04.831031+0000","last_fullsized":"2023-10-09T10:49:04.831031+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":185,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.406556+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:49:03.406556+0000","last_peered":"2023-10-09T10:49:03.406556+0000","last_clean":"2023-10-09T10:49:03.406556+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:49:03.406556+0000","last_undegraded":"2023-10-09T10:49:03.406556+0000","last_fullsized":"2023-10-09T10:49:03.406556+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":156,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.526587+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:49:03.526587+0000","last_peered":"2023-10-09T10:49:03.526587+0000","last_clean":"2023-10-09T10:49:03.526587+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:49:03.526587+0000","last_undegraded":"2023-10-09T10:49:03.526587+0000","last_fullsized":"2023-10-09T10:49:03.526587+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":508,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830103+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:49:04.830103+0000","last_peered":"2023-10-09T10:49:04.830103+0000","last_clean":"2023-10-09T10:49:04.830103+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:49:04.830103+0000","last_undegraded":"2023-10-09T10:49:04.830103+0000","last_fullsized":"2023-10-09T10:49:04.830103+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":131,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.527106+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:49:03.527106+0000","last_peered":"2023-10-09T10:49:03.527106+0000","last_clean":"2023-10-09T10:49:03.527106+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:49:03.527106+0000","last_undegraded":"2023-10-09T10:49:03.527106+0000","last_fullsized":"2023-10-09T10:49:03.527106+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":183,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.527203+0000","last_change":"2023-10-09T10:46:32.245583+0000","last_active":"2023-10-09T10:49:03.527203+0000","last_peered":"2023-10-09T10:49:03.527203+0000","last_clean":"2023-10-09T10:49:03.527203+0000","last_became_active":"2023-10-09T10:46:32.245271+0000","last_became_peered":"2023-10-09T10:46:32.245271+0000","last_unstale":"2023-10-09T10:49:03.527203+0000","last_undegraded":"2023-10-09T10:49:03.527203+0000","last_fullsized":"2023-10-09T10:49:03.527203+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.025051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830102+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:49:04.830102+0000","last_peered":"2023-10-09T10:49:04.830102+0000","last_clean":"2023-10-09T10:49:04.830102+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:49:04.830102+0000","last_undegraded":"2023-10-09T10:49:04.830102+0000","last_fullsized":"2023-10-09T10:49:04.830102+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":167,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.526807+0000","last_change":"2023-10-09T10:46:32.245685+0000","last_active":"2023-10-09T10:49:03.526807+0000","last_peered":"2023-10-09T10:49:03.526807+0000","last_clean":"2023-10-09T10:49:03.526807+0000","last_became_active":"2023-10-09T10:46:32.245314+0000","last_became_peered":"2023-10-09T10:46:32.245314+0000","last_unstale":"2023-10-09T10:49:03.526807+0000","last_undegraded":"2023-10-09T10:49:03.526807+0000","last_fullsized":"2023-10-09T10:49:03.526807+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:51.716860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":156,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.527302+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:49:03.527302+0000","last_peered":"2023-10-09T10:49:03.527302+0000","last_clean":"2023-10-09T10:49:03.527302+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:49:03.527302+0000","last_undegraded":"2023-10-09T10:49:03.527302+0000","last_fullsized":"2023-10-09T10:49:03.527302+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":519,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.829777+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:49:04.829777+0000","last_peered":"2023-10-09T10:49:04.829777+0000","last_clean":"2023-10-09T10:49:04.829777+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:49:04.829777+0000","last_undegraded":"2023-10-09T10:49:04.829777+0000","last_fullsized":"2023-10-09T10:49:04.829777+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":162,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.408704+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:49:03.408704+0000","last_peered":"2023-10-09T10:49:03.408704+0000","last_clean":"2023-10-09T10:49:03.408704+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:49:03.408704+0000","last_undegraded":"2023-10-09T10:49:03.408704+0000","last_fullsized":"2023-10-09T10:49:03.408704+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":145,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.401854+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:49:04.401854+0000","last_peered":"2023-10-09T10:49:04.401854+0000","last_clean":"2023-10-09T10:49:04.401854+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:49:04.401854+0000","last_undegraded":"2023-10-09T10:49:04.401854+0000","last_fullsized":"2023-10-09T10:49:04.401854+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830448+0000","last_change":"2023-10-09T10:46:05.617983+0000","last_active":"2023-10-09T10:49:04.830448+0000","last_peered":"2023-10-09T10:49:04.830448+0000","last_clean":"2023-10-09T10:49:04.830448+0000","last_became_active":"2023-10-09T10:46:05.616021+0000","last_became_peered":"2023-10-09T10:46:05.616021+0000","last_unstale":"2023-10-09T10:49:04.830448+0000","last_undegraded":"2023-10-09T10:49:04.830448+0000","last_fullsized":"2023-10-09T10:49:04.830448+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":189,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.403194+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:49:04.403194+0000","last_peered":"2023-10-09T10:49:04.403194+0000","last_clean":"2023-10-09T10:49:04.403194+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:49:04.403194+0000","last_undegraded":"2023-10-09T10:49:04.403194+0000","last_fullsized":"2023-10-09T10:49:04.403194+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":184,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.420904+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:49:03.420904+0000","last_peered":"2023-10-09T10:49:03.420904+0000","last_clean":"2023-10-09T10:49:03.420904+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:49:03.420904+0000","last_undegraded":"2023-10-09T10:49:03.420904+0000","last_fullsized":"2023-10-09T10:49:03.420904+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":131,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.402111+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:49:04.402111+0000","last_peered":"2023-10-09T10:49:04.402111+0000","last_clean":"2023-10-09T10:49:04.402111+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:49:04.402111+0000","last_undegraded":"2023-10-09T10:49:04.402111+0000","last_fullsized":"2023-10-09T10:49:04.402111+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":527,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830905+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:49:04.830905+0000","last_peered":"2023-10-09T10:49:04.830905+0000","last_clean":"2023-10-09T10:49:04.830905+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:49:04.830905+0000","last_undegraded":"2023-10-09T10:49:04.830905+0000","last_fullsized":"2023-10-09T10:49:04.830905+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":161,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.420544+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:49:03.420544+0000","last_peered":"2023-10-09T10:49:03.420544+0000","last_clean":"2023-10-09T10:49:03.420544+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:49:03.420544+0000","last_undegraded":"2023-10-09T10:49:03.420544+0000","last_fullsized":"2023-10-09T10:49:03.420544+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":177,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.527055+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:49:03.527055+0000","last_peered":"2023-10-09T10:49:03.527055+0000","last_clean":"2023-10-09T10:49:03.527055+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:49:03.527055+0000","last_undegraded":"2023-10-09T10:49:03.527055+0000","last_fullsized":"2023-10-09T10:49:03.527055+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":170,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.403271+0000","last_change":"2023-10-09T10:46:32.245608+0000","last_active":"2023-10-09T10:49:04.403271+0000","last_peered":"2023-10-09T10:49:04.403271+0000","last_clean":"2023-10-09T10:49:04.403271+0000","last_became_active":"2023-10-09T10:46:32.245290+0000","last_became_peered":"2023-10-09T10:46:32.245290+0000","last_unstale":"2023-10-09T10:49:04.403271+0000","last_undegraded":"2023-10-09T10:49:04.403271+0000","last_fullsized":"2023-10-09T10:49:04.403271+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:48.832515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.829868+0000","last_change":"2023-10-09T10:46:05.620057+0000","last_active":"2023-10-09T10:49:04.829868+0000","last_peered":"2023-10-09T10:49:04.829868+0000","last_clean":"2023-10-09T10:49:04.829868+0000","last_became_active":"2023-10-09T10:46:05.619703+0000","last_became_peered":"2023-10-09T10:46:05.619703+0000","last_unstale":"2023-10-09T10:49:04.829868+0000","last_undegraded":"2023-10-09T10:49:04.829868+0000","last_fullsized":"2023-10-09T10:49:04.829868+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":190,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.409163+0000","last_change":"2023-10-09T10:46:32.241041+0000","last_active":"2023-10-09T10:49:03.409163+0000","last_peered":"2023-10-09T10:49:03.409163+0000","last_clean":"2023-10-09T10:49:03.409163+0000","last_became_active":"2023-10-09T10:46:32.238683+0000","last_became_peered":"2023-10-09T10:46:32.238683+0000","last_unstale":"2023-10-09T10:49:03.409163+0000","last_undegraded":"2023-10-09T10:49:03.409163+0000","last_fullsized":"2023-10-09T10:49:03.409163+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:25.817713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":155,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.401877+0000","last_change":"2023-10-09T10:46:32.250104+0000","last_active":"2023-10-09T10:49:04.401877+0000","last_peered":"2023-10-09T10:49:04.401877+0000","last_clean":"2023-10-09T10:49:04.401877+0000","last_became_active":"2023-10-09T10:46:32.249805+0000","last_became_peered":"2023-10-09T10:46:32.249805+0000","last_unstale":"2023-10-09T10:49:04.401877+0000","last_undegraded":"2023-10-09T10:49:04.401877+0000","last_fullsized":"2023-10-09T10:49:04.401877+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:07.230672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":184,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:06.556362+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:49:06.556362+0000","last_peered":"2023-10-09T10:49:06.556362+0000","last_clean":"2023-10-09T10:49:06.556362+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:49:06.556362+0000","last_undegraded":"2023-10-09T10:49:06.556362+0000","last_fullsized":"2023-10-09T10:49:06.556362+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":171,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.420488+0000","last_change":"2023-10-09T10:46:32.235346+0000","last_active":"2023-10-09T10:49:03.420488+0000","last_peered":"2023-10-09T10:49:03.420488+0000","last_clean":"2023-10-09T10:49:03.420488+0000","last_became_active":"2023-10-09T10:46:32.234906+0000","last_became_peered":"2023-10-09T10:46:32.234906+0000","last_unstale":"2023-10-09T10:49:03.420488+0000","last_undegraded":"2023-10-09T10:49:03.420488+0000","last_fullsized":"2023-10-09T10:49:03.420488+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.199251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.829782+0000","last_change":"2023-10-09T10:46:05.617146+0000","last_active":"2023-10-09T10:49:04.829782+0000","last_peered":"2023-10-09T10:49:04.829782+0000","last_clean":"2023-10-09T10:49:04.829782+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:49:04.829782+0000","last_undegraded":"2023-10-09T10:49:04.829782+0000","last_fullsized":"2023-10-09T10:49:04.829782+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":120,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.406167+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:49:03.406167+0000","last_peered":"2023-10-09T10:49:03.406167+0000","last_clean":"2023-10-09T10:49:03.406167+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:49:03.406167+0000","last_undegraded":"2023-10-09T10:49:03.406167+0000","last_fullsized":"2023-10-09T10:49:03.406167+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":89,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.527171+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:49:03.527171+0000","last_peered":"2023-10-09T10:49:03.527171+0000","last_clean":"2023-10-09T10:49:03.527171+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:49:03.527171+0000","last_undegraded":"2023-10-09T10:49:03.527171+0000","last_fullsized":"2023-10-09T10:49:03.527171+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830019+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:49:04.830019+0000","last_peered":"2023-10-09T10:49:04.830019+0000","last_clean":"2023-10-09T10:49:04.830019+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:49:04.830019+0000","last_undegraded":"2023-10-09T10:49:04.830019+0000","last_fullsized":"2023-10-09T10:49:04.830019+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":185,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.401990+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:49:04.401990+0000","last_peered":"2023-10-09T10:49:04.401990+0000","last_clean":"2023-10-09T10:49:04.401990+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:49:04.401990+0000","last_undegraded":"2023-10-09T10:49:04.401990+0000","last_fullsized":"2023-10-09T10:49:04.401990+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":71,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830862+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:49:04.830862+0000","last_peered":"2023-10-09T10:49:04.830862+0000","last_clean":"2023-10-09T10:49:04.830862+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:49:04.830862+0000","last_undegraded":"2023-10-09T10:49:04.830862+0000","last_fullsized":"2023-10-09T10:49:04.830862+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":71,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830680+0000","last_change":"2023-10-09T10:46:32.236202+0000","last_active":"2023-10-09T10:49:04.830680+0000","last_peered":"2023-10-09T10:49:04.830680+0000","last_clean":"2023-10-09T10:49:04.830680+0000","last_became_active":"2023-10-09T10:46:32.235901+0000","last_became_peered":"2023-10-09T10:46:32.235901+0000","last_unstale":"2023-10-09T10:49:04.830680+0000","last_undegraded":"2023-10-09T10:49:04.830680+0000","last_fullsized":"2023-10-09T10:49:04.830680+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:48.260403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830434+0000","last_change":"2023-10-09T10:46:05.620764+0000","last_active":"2023-10-09T10:49:04.830434+0000","last_peered":"2023-10-09T10:49:04.830434+0000","last_clean":"2023-10-09T10:49:04.830434+0000","last_became_active":"2023-10-09T10:46:05.619870+0000","last_became_peered":"2023-10-09T10:46:05.619870+0000","last_unstale":"2023-10-09T10:49:04.830434+0000","last_undegraded":"2023-10-09T10:49:04.830434+0000","last_fullsized":"2023-10-09T10:49:04.830434+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":180,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.406516+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:49:03.406516+0000","last_peered":"2023-10-09T10:49:03.406516+0000","last_clean":"2023-10-09T10:49:03.406516+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:49:03.406516+0000","last_undegraded":"2023-10-09T10:49:03.406516+0000","last_fullsized":"2023-10-09T10:49:03.406516+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":155,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.403326+0000","last_change":"2023-10-09T10:46:32.251450+0000","last_active":"2023-10-09T10:49:04.403326+0000","last_peered":"2023-10-09T10:49:04.403326+0000","last_clean":"2023-10-09T10:49:04.403326+0000","last_became_active":"2023-10-09T10:46:32.250933+0000","last_became_peered":"2023-10-09T10:46:32.250933+0000","last_unstale":"2023-10-09T10:49:04.403326+0000","last_undegraded":"2023-10-09T10:49:04.403326+0000","last_fullsized":"2023-10-09T10:49:04.403326+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:04.116178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":62,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830177+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:49:04.830177+0000","last_peered":"2023-10-09T10:49:04.830177+0000","last_clean":"2023-10-09T10:49:04.830177+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:49:04.830177+0000","last_undegraded":"2023-10-09T10:49:04.830177+0000","last_fullsized":"2023-10-09T10:49:04.830177+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830742+0000","last_change":"2023-10-09T10:46:05.618443+0000","last_active":"2023-10-09T10:49:04.830742+0000","last_peered":"2023-10-09T10:49:04.830742+0000","last_clean":"2023-10-09T10:49:04.830742+0000","last_became_active":"2023-10-09T10:46:05.616588+0000","last_became_peered":"2023-10-09T10:46:05.616588+0000","last_unstale":"2023-10-09T10:49:04.830742+0000","last_undegraded":"2023-10-09T10:49:04.830742+0000","last_fullsized":"2023-10-09T10:49:04.830742+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":185,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.407036+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:49:03.407036+0000","last_peered":"2023-10-09T10:49:03.407036+0000","last_clean":"2023-10-09T10:49:03.407036+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:49:03.407036+0000","last_undegraded":"2023-10-09T10:49:03.407036+0000","last_fullsized":"2023-10-09T10:49:03.407036+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":131,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:06.556329+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:49:06.556329+0000","last_peered":"2023-10-09T10:49:06.556329+0000","last_clean":"2023-10-09T10:49:06.556329+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:49:06.556329+0000","last_undegraded":"2023-10-09T10:49:06.556329+0000","last_fullsized":"2023-10-09T10:49:06.556329+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":174,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.408780+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:49:03.408780+0000","last_peered":"2023-10-09T10:49:03.408780+0000","last_clean":"2023-10-09T10:49:03.408780+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:49:03.408780+0000","last_undegraded":"2023-10-09T10:49:03.408780+0000","last_fullsized":"2023-10-09T10:49:03.408780+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":527,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.829745+0000","last_change":"2023-10-09T10:46:05.617589+0000","last_active":"2023-10-09T10:49:04.829745+0000","last_peered":"2023-10-09T10:49:04.829745+0000","last_clean":"2023-10-09T10:49:04.829745+0000","last_became_active":"2023-10-09T10:46:05.616400+0000","last_became_peered":"2023-10-09T10:46:05.616400+0000","last_unstale":"2023-10-09T10:49:04.829745+0000","last_undegraded":"2023-10-09T10:49:04.829745+0000","last_fullsized":"2023-10-09T10:49:04.829745+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":50,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.406776+0000","last_change":"2023-10-09T10:46:32.245106+0000","last_active":"2023-10-09T10:49:03.406776+0000","last_peered":"2023-10-09T10:49:03.406776+0000","last_clean":"2023-10-09T10:49:03.406776+0000","last_became_active":"2023-10-09T10:46:32.244826+0000","last_became_peered":"2023-10-09T10:46:32.244826+0000","last_unstale":"2023-10-09T10:49:03.406776+0000","last_undegraded":"2023-10-09T10:49:03.406776+0000","last_fullsized":"2023-10-09T10:49:03.406776+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:30.322834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":66,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830602+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:49:04.830602+0000","last_peered":"2023-10-09T10:49:04.830602+0000","last_clean":"2023-10-09T10:49:04.830602+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:49:04.830602+0000","last_undegraded":"2023-10-09T10:49:04.830602+0000","last_fullsized":"2023-10-09T10:49:04.830602+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":139,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.526797+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:49:03.526797+0000","last_peered":"2023-10-09T10:49:03.526797+0000","last_clean":"2023-10-09T10:49:03.526797+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:49:03.526797+0000","last_undegraded":"2023-10-09T10:49:03.526797+0000","last_fullsized":"2023-10-09T10:49:03.526797+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":62,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830262+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:49:04.830262+0000","last_peered":"2023-10-09T10:49:04.830262+0000","last_clean":"2023-10-09T10:49:04.830262+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:49:04.830262+0000","last_undegraded":"2023-10-09T10:49:04.830262+0000","last_fullsized":"2023-10-09T10:49:04.830262+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":125,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.403370+0000","last_change":"2023-10-09T10:46:32.251242+0000","last_active":"2023-10-09T10:49:04.403370+0000","last_peered":"2023-10-09T10:49:04.403370+0000","last_clean":"2023-10-09T10:49:04.403370+0000","last_became_active":"2023-10-09T10:46:32.250534+0000","last_became_peered":"2023-10-09T10:46:32.250534+0000","last_unstale":"2023-10-09T10:49:04.403370+0000","last_undegraded":"2023-10-09T10:49:04.403370+0000","last_fullsized":"2023-10-09T10:49:04.403370+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:33.726041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":159,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.420778+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:49:03.420778+0000","last_peered":"2023-10-09T10:49:03.420778+0000","last_clean":"2023-10-09T10:49:03.420778+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:49:03.420778+0000","last_undegraded":"2023-10-09T10:49:03.420778+0000","last_fullsized":"2023-10-09T10:49:03.420778+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":173,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.526566+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:49:03.526566+0000","last_peered":"2023-10-09T10:49:03.526566+0000","last_clean":"2023-10-09T10:49:03.526566+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:49:03.526566+0000","last_undegraded":"2023-10-09T10:49:03.526566+0000","last_fullsized":"2023-10-09T10:49:03.526566+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":142,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.420970+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:49:03.420970+0000","last_peered":"2023-10-09T10:49:03.420970+0000","last_clean":"2023-10-09T10:49:03.420970+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:49:03.420970+0000","last_undegraded":"2023-10-09T10:49:03.420970+0000","last_fullsized":"2023-10-09T10:49:03.420970+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":40,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.527053+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:49:03.527053+0000","last_peered":"2023-10-09T10:49:03.527053+0000","last_clean":"2023-10-09T10:49:03.527053+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:49:03.527053+0000","last_undegraded":"2023-10-09T10:49:03.527053+0000","last_fullsized":"2023-10-09T10:49:03.527053+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":176,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.406958+0000","last_change":"2023-10-09T10:46:32.244607+0000","last_active":"2023-10-09T10:49:03.406958+0000","last_peered":"2023-10-09T10:49:03.406958+0000","last_clean":"2023-10-09T10:49:03.406958+0000","last_became_active":"2023-10-09T10:46:32.242628+0000","last_became_peered":"2023-10-09T10:46:32.242628+0000","last_unstale":"2023-10-09T10:49:03.406958+0000","last_undegraded":"2023-10-09T10:49:03.406958+0000","last_fullsized":"2023-10-09T10:49:03.406958+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:10.738819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":207,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.409042+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:49:03.409042+0000","last_peered":"2023-10-09T10:49:03.409042+0000","last_clean":"2023-10-09T10:49:03.409042+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:49:03.409042+0000","last_undegraded":"2023-10-09T10:49:03.409042+0000","last_fullsized":"2023-10-09T10:49:03.409042+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":83,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830365+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:49:04.830365+0000","last_peered":"2023-10-09T10:49:04.830365+0000","last_clean":"2023-10-09T10:49:04.830365+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:49:04.830365+0000","last_undegraded":"2023-10-09T10:49:04.830365+0000","last_fullsized":"2023-10-09T10:49:04.830365+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":156,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T10:49:03.526645+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:49:03.526645+0000","last_peered":"2023-10-09T10:49:03.526645+0000","last_clean":"2023-10-09T10:49:03.526645+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:49:03.526645+0000","last_undegraded":"2023-10-09T10:49:03.526645+0000","last_fullsized":"2023-10-09T10:49:03.526645+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1186,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332271,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299880,"kb_used_data":2592,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93445272,"statfs":{"total":95995035648,"available":95687958528,"internally_reserved":0,"allocated":2654208,"data_stored":1922547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":1.155,"15min":0.76600000000000001},"min":{"1min":0.23499999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.476,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":0.40200000000000002},{"interface":"front","average":{"1min":3.1789999999999998,"5min":1.5029999999999999,"15min":0.92100000000000004},"min":{"1min":0.248,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.332000000000001,"5min":11.332000000000001,"15min":11.332000000000001},"last":0.28100000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":3.4359999999999999,"5min":1.909,"15min":0.997},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":11.367000000000001,"5min":11.826000000000001,"15min":11.826000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.1739999999999999,"5min":1.484,"15min":0.84899999999999998},"min":{"1min":0.26100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":7.556,"5min":11.84,"15min":11.84},"last":0.371}]},{"osd":2,"last update":"Mon Oct 9 10:48:18 2023","interfaces":[{"interface":"back","average":{"1min":2.165,"5min":1.4810000000000001,"15min":0.83499999999999996},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.1419999999999995,"5min":9.1419999999999995,"15min":9.1419999999999995},"last":0.31900000000000001},{"interface":"front","average":{"1min":2.1970000000000001,"5min":1.1910000000000001,"15min":0.747},"min":{"1min":0.34999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":7.8120000000000003,"5min":7.8120000000000003,"15min":7.8120000000000003},"last":0.438}]},{"osd":3,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":3.903,"5min":1.893,"15min":1.0209999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":11.523,"5min":11.523,"15min":11.523},"last":0.48599999999999999},{"interface":"front","average":{"1min":4.2789999999999999,"5min":1.835,"15min":1.0109999999999999},"min":{"1min":0.35299999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":19.234999999999999,"5min":19.234999999999999,"15min":19.234999999999999},"last":0.35999999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":1.1579999999999999,"15min":0.70499999999999996},"min":{"1min":0.36399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":9.4900000000000002,"5min":9.4900000000000002,"15min":9.4900000000000002},"last":0.34000000000000002},{"interface":"front","average":{"1min":2.327,"5min":1.018,"15min":0.66100000000000003},"min":{"1min":0.35399999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.0999999999999996,"5min":9.0999999999999996,"15min":9.0999999999999996},"last":0.41699999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.0249999999999999,"5min":1.0609999999999999,"15min":0.69599999999999995},"min":{"1min":0.314,"5min":0.247,"15min":0.247},"max":{"1min":12.064,"5min":12.064,"15min":12.064},"last":0.47499999999999998},{"interface":"front","average":{"1min":3.8839999999999999,"5min":1.4770000000000001,"15min":0.85299999999999998},"min":{"1min":0.435,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":11.741,"5min":11.741,"15min":11.741},"last":0.38200000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2999999999999998,"5min":1.3720000000000001,"15min":0.83199999999999996},"min":{"1min":0.33700000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.029,"5min":13.029,"15min":13.029},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.6869999999999998,"5min":1.079,"15min":0.72699999999999998},"min":{"1min":0.41299999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":19.187999999999999,"5min":19.187999999999999,"15min":19.187999999999999},"last":0.46700000000000003}]}]},{"osd":6,"up_from":44,"seq":188978561202,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316260,"kb_used_data":2588,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":93428892,"statfs":{"total":95995035648,"available":95671185408,"internally_reserved":0,"allocated":2650112,"data_stored":1922989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":3.0499999999999998,"5min":1.4330000000000001,"15min":0.88},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.882,"5min":10.882,"15min":10.882},"last":0.28699999999999998},{"interface":"front","average":{"1min":2.4540000000000002,"5min":1.367,"15min":0.86199999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.8710000000000004,"5min":7.8710000000000004,"15min":7.8710000000000004},"last":0.32400000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:48:53 2023","interfaces":[{"interface":"back","average":{"1min":3.2109999999999999,"5min":1.681,"15min":0.95399999999999996},"min":{"1min":0.28999999999999998,"5min":0.245,"15min":0.245},"max":{"1min":15.502000000000001,"5min":16.747,"15min":16.747},"last":0.35899999999999999},{"interface":"front","average":{"1min":3.0779999999999998,"5min":1.79,"15min":0.96399999999999997},"min":{"1min":0.33700000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.137,"5min":16.734000000000002,"15min":16.734000000000002},"last":0.36799999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:48:20 2023","interfaces":[{"interface":"back","average":{"1min":2.798,"5min":1.411,"15min":0.82599999999999996},"min":{"1min":0.29699999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.208,"5min":13.208,"15min":13.208},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.9420000000000002,"5min":1.619,"15min":0.91300000000000003},"min":{"1min":0.309,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.6749999999999998,"5min":12.571,"15min":12.571},"last":0.48999999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":1.496,"15min":0.85699999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":7.7649999999999997,"5min":7.7649999999999997,"15min":7.7649999999999997},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.254,"5min":1.7270000000000001,"15min":0.93999999999999995},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":13.179,"5min":13.179,"15min":13.179},"last":0.40100000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.0629999999999999,"15min":0.65000000000000002},"min":{"1min":0.29899999999999999,"5min":0.255,"15min":0.255},"max":{"1min":9.4060000000000006,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.6579999999999999,"5min":0.89600000000000002,"15min":0.623},"min":{"1min":0.34100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.1260000000000003,"5min":5.1260000000000003,"15min":5.1260000000000003},"last":0.38600000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":1.1719999999999999,"15min":0.71099999999999997},"min":{"1min":0.35899999999999999,"5min":0.29499999999999998,"15min":0.27800000000000002},"max":{"1min":10.497,"5min":10.497,"15min":10.497},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.589,"5min":0.96599999999999997,"15min":0.67100000000000004},"min":{"1min":0.40699999999999997,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":6.8419999999999996,"5min":6.8419999999999996,"15min":6.8419999999999996},"last":0.749}]},{"osd":7,"last update":"Mon Oct 9 10:48:26 2023","interfaces":[{"interface":"back","average":{"1min":3.915,"5min":1.369,"15min":0.80600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":23.358000000000001,"5min":23.358000000000001,"15min":23.358000000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":2.6840000000000002,"5min":1.25,"15min":0.79700000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34000000000000002},"max":{"1min":12.404,"5min":12.404,"15min":12.404},"last":0.42299999999999999}]}]},{"osd":1,"up_from":85,"seq":365072220241,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":200116,"kb_used_data":56172,"kb_used_omap":9,"kb_used_meta":143926,"kb_avail":93545036,"statfs":{"total":95995035648,"available":95790116864,"internally_reserved":0,"allocated":57520128,"data_stored":56546619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":147381225},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":2.1419999999999999,"15min":0.95599999999999996},"min":{"1min":0.249,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":18.027000000000001,"5min":18.027000000000001,"15min":18.027000000000001},"last":0.22700000000000001},{"interface":"front","average":{"1min":2.6920000000000002,"5min":2.0209999999999999,"15min":0.93899999999999995},"min":{"1min":0.23599999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.6530000000000005,"5min":14.465,"15min":14.465},"last":0.39100000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:48:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":1.492,"15min":0.74199999999999999},"min":{"1min":0.221,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":15.356,"5min":15.356,"15min":15.356},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.488,"5min":1.6890000000000001,"15min":0.81899999999999995},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.4130000000000003,"5min":14.234,"15min":14.234},"last":0.28799999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.6829999999999998,"5min":2.169,"15min":1.0369999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":18.274999999999999,"5min":18.274999999999999,"15min":18.274999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.774,"5min":1.915,"15min":0.86799999999999999},"min":{"1min":0.313,"5min":0.251,"15min":0.251},"max":{"1min":8.5999999999999996,"5min":15.468999999999999,"15min":15.468999999999999},"last":0.55900000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":1.5289999999999999,"15min":0.90500000000000003},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.167999999999999,"5min":12.167999999999999,"15min":12.167999999999999},"last":1.276},{"interface":"front","average":{"1min":3.4399999999999999,"5min":1.7989999999999999,"15min":1.0129999999999999},"min":{"1min":0.309,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":14.228999999999999,"5min":14.228999999999999,"15min":14.228999999999999},"last":0.52000000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":1.4359999999999999,"15min":0.89700000000000002},"min":{"1min":0.34399999999999997,"5min":0.253,"15min":0.253},"max":{"1min":9.4809999999999999,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":0.60799999999999998},{"interface":"front","average":{"1min":3.7909999999999999,"5min":1.9219999999999999,"15min":1.0720000000000001},"min":{"1min":0.318,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":17.085000000000001,"5min":17.085000000000001,"15min":17.085000000000001},"last":1.2350000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":2.8559999999999999,"5min":1.6479999999999999,"15min":0.96999999999999997},"min":{"1min":0.38300000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.9179999999999993,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":1.1080000000000001},{"interface":"front","average":{"1min":2.819,"5min":1.7290000000000001,"15min":1.0069999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":12.231,"5min":12.231,"15min":12.231},"last":0.61799999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":3.1219999999999999,"5min":1.714,"15min":1.004},"min":{"1min":0.42799999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":12.249000000000001,"5min":12.249000000000001,"15min":12.249000000000001},"last":1.0980000000000001},{"interface":"front","average":{"1min":3.7250000000000001,"5min":1.907,"15min":1.0660000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":14.273999999999999,"5min":14.273999999999999,"15min":14.273999999999999},"last":0.49099999999999999}]}]},{"osd":0,"up_from":80,"seq":343597383775,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":290664,"kb_used_data":164704,"kb_used_omap":28,"kb_used_meta":125923,"kb_avail":93454488,"statfs":{"total":95995035648,"available":95697395712,"internally_reserved":0,"allocated":168656896,"data_stored":167639141,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":128946000},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":3.931,"5min":2.258,"15min":0.998},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":11.869,"5min":25.475000000000001,"15min":25.475000000000001},"last":0.314},{"interface":"front","average":{"1min":3.702,"5min":2.2610000000000001,"15min":0.997},"min":{"1min":0.31,"5min":0.188,"15min":0.188},"max":{"1min":12.276,"5min":20.832999999999998,"15min":20.832999999999998},"last":0.28000000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":1.4750000000000001,"15min":0.72399999999999998},"min":{"1min":0.249,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.0890000000000004,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":0.26600000000000001},{"interface":"front","average":{"1min":2.1120000000000001,"5min":1.889,"15min":0.85799999999999998},"min":{"1min":0.27100000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.0329999999999995,"5min":14.391999999999999,"15min":14.391999999999999},"last":0.28999999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.3359999999999999,"15min":1.212},"min":{"1min":0.307,"5min":0.215,"15min":0.215},"max":{"1min":12.307,"5min":14.926,"15min":14.926},"last":0.25},{"interface":"front","average":{"1min":3.5539999999999998,"5min":2.1960000000000002,"15min":1.093},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":11.885999999999999,"5min":11.885999999999999,"15min":11.885999999999999},"last":0.23499999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":2.8370000000000002,"5min":1.671,"15min":0.96999999999999997},"min":{"1min":0.434,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.0920000000000005,"5min":8.0920000000000005,"15min":8.0920000000000005},"last":0.38500000000000001},{"interface":"front","average":{"1min":3.2679999999999998,"5min":1.9370000000000001,"15min":1.0569999999999999},"min":{"1min":0.36099999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.318,"5min":12.318,"15min":12.318},"last":0.33300000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6179999999999999,"5min":1.8400000000000001,"15min":1.0289999999999999},"min":{"1min":0.35899999999999999,"5min":0.318,"15min":0.318},"max":{"1min":12.157,"5min":12.157,"15min":12.157},"last":0.52100000000000002},{"interface":"front","average":{"1min":3.181,"5min":1.677,"15min":0.98999999999999999},"min":{"1min":0.40500000000000003,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":12.781000000000001,"5min":12.781000000000001,"15min":12.781000000000001},"last":0.34200000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":3.4649999999999999,"5min":1.96,"15min":1.073},"min":{"1min":0.33600000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":12.207000000000001,"5min":12.207000000000001,"15min":12.207000000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.1269999999999998,"5min":1.3740000000000001,"15min":0.89100000000000001},"min":{"1min":0.41299999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":9.8889999999999993,"5min":9.8889999999999993,"15min":9.8889999999999993},"last":0.35799999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":3.6280000000000001,"5min":1.7509999999999999,"15min":1.0169999999999999},"min":{"1min":0.48799999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":12.727,"5min":12.727,"15min":12.727},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.843,"5min":1.8660000000000001,"15min":1.071},"min":{"1min":0.47599999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.6150000000000002,"5min":14.394,"15min":14.394},"last":0.36599999999999999}]}]},{"osd":2,"up_from":0,"seq":386547056715,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893189,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":320332,"kb_used_data":62852,"kb_used_omap":7,"kb_used_meta":257464,"kb_avail":93424820,"statfs":{"total":95995035648,"available":95667015680,"internally_reserved":0,"allocated":64360448,"data_stored":63361104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":263643752},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3159999999999998,"5min":1.913,"15min":0.85099999999999998},"min":{"1min":0.248,"5min":0.183,"15min":0.183},"max":{"1min":15.255000000000001,"5min":15.255000000000001,"15min":15.255000000000001},"last":0.20899999999999999},{"interface":"front","average":{"1min":2.9830000000000001,"5min":1.6539999999999999,"15min":0.82199999999999995},"min":{"1min":0.21099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":8.6470000000000002,"5min":8.6470000000000002,"15min":8.6470000000000002},"last":0.23400000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.6400000000000001,"5min":2.359,"15min":1.006},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":10.821,"5min":14.707000000000001,"15min":14.707000000000001},"last":0.28699999999999998},{"interface":"front","average":{"1min":3.4780000000000002,"5min":1.8100000000000001,"15min":0.85199999999999998},"min":{"1min":0.28999999999999998,"5min":0.193,"15min":0.193},"max":{"1min":15.443,"5min":15.443,"15min":15.443},"last":0.308}]},{"osd":2,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":2.0630000000000002,"15min":0.93200000000000005},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.375,"5min":13.853,"15min":13.853},"last":0.32900000000000001},{"interface":"front","average":{"1min":3.988,"5min":2.2309999999999999,"15min":1.0069999999999999},"min":{"1min":0.28699999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.557,"5min":10.557,"15min":10.557},"last":0.38500000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":4.0739999999999998,"5min":1.9219999999999999,"15min":1.022},"min":{"1min":0.33900000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":20.388000000000002,"5min":20.388000000000002,"15min":20.388000000000002},"last":0.48299999999999998},{"interface":"front","average":{"1min":4.4299999999999997,"5min":2.0609999999999999,"15min":1.0920000000000001},"min":{"1min":0.38700000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":19.085999999999999,"5min":19.085999999999999,"15min":19.085999999999999},"last":0.54800000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.7599999999999998,"5min":2.4369999999999998,"15min":1.2030000000000001},"min":{"1min":0.35399999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.3550000000000004,"5min":51.509999999999998,"15min":51.509999999999998},"last":0.57599999999999996},{"interface":"front","average":{"1min":2.6240000000000001,"5min":2.7189999999999999,"15min":1.2809999999999999},"min":{"1min":0.316,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.7449999999999992,"5min":55.338999999999999,"15min":55.338999999999999},"last":0.65000000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":1.841,"15min":1.0049999999999999},"min":{"1min":0.43099999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.361000000000001,"5min":12.361000000000001,"15min":12.361000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.8140000000000001,"5min":2.2189999999999999,"15min":1.139},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.5839999999999996,"5min":13.891999999999999,"15min":13.891999999999999},"last":0.56299999999999994}]},{"osd":7,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.2269999999999999,"5min":2.8679999999999999,"15min":1.3500000000000001},"min":{"1min":0.34000000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.734,"5min":51.744,"15min":51.744},"last":0.53500000000000003},{"interface":"front","average":{"1min":3.927,"5min":2.54,"15min":1.2290000000000001},"min":{"1min":0.46100000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":15.212999999999999,"5min":53.399000000000001,"15min":53.399000000000001},"last":0.45300000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019065,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299300,"kb_used_data":2012,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93445852,"statfs":{"total":95995035648,"available":95688552448,"internally_reserved":0,"allocated":2060288,"data_stored":1332432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.9850000000000001,"5min":1.302,"15min":0.80500000000000005},"min":{"1min":0.216,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.7160000000000002,"5min":10.387,"15min":10.387},"last":0.29399999999999998},{"interface":"front","average":{"1min":2.9780000000000002,"5min":1.6659999999999999,"15min":0.94299999999999995},"min":{"1min":0.48499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.0129999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.39900000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":2.351,"5min":1.512,"15min":0.872},"min":{"1min":0.254,"5min":0.249,"15min":0.249},"max":{"1min":7.5350000000000001,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.931,"5min":1.7010000000000001,"15min":0.98099999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.2919999999999998,"5min":8.7669999999999995,"15min":8.7669999999999995},"last":0.34699999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:48:13 2023","interfaces":[{"interface":"back","average":{"1min":3.1520000000000001,"5min":1.617,"15min":0.91900000000000004},"min":{"1min":0.52100000000000002,"5min":0.253,"15min":0.253},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":0.37},{"interface":"front","average":{"1min":4.016,"5min":1.855,"15min":0.99299999999999999},"min":{"1min":0.36799999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":20.545999999999999,"5min":20.545999999999999,"15min":20.545999999999999},"last":0.49199999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":3.2549999999999999,"5min":1.8420000000000001,"15min":1.0880000000000001},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.0269999999999992,"5min":9.0269999999999992,"15min":9.0269999999999992},"last":0.32000000000000001},{"interface":"front","average":{"1min":3.9809999999999999,"5min":2.0379999999999998,"15min":1.071},"min":{"1min":0.79400000000000004,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.6029999999999998,"5min":8.9809999999999999,"15min":8.9809999999999999},"last":0.45000000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":5.0960000000000001,"5min":1.8140000000000001,"15min":0.91000000000000003},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":24.149999999999999,"5min":24.149999999999999,"15min":24.149999999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":4.173,"5min":1.6699999999999999,"15min":0.86399999999999999},"min":{"1min":0.373,"5min":0.252,"15min":0.252},"max":{"1min":20.422000000000001,"5min":20.422000000000001,"15min":20.422000000000001},"last":0.32900000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":1.1319999999999999,"15min":0.69399999999999995},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":12.391999999999999,"5min":12.391999999999999,"15min":12.391999999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":3.1200000000000001,"5min":1.2130000000000001,"15min":0.78100000000000003},"min":{"1min":0.374,"5min":0.311,"15min":0.311},"max":{"1min":12.539,"5min":12.539,"15min":12.539},"last":0.50600000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:48:21 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":1.2929999999999999,"15min":0.80900000000000005},"min":{"1min":0.33300000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.7919999999999998,"5min":8.7919999999999998,"15min":8.7919999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":3.2389999999999999,"5min":1.27,"15min":0.79000000000000004},"min":{"1min":0.55900000000000005,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.423,"5min":7.423,"15min":7.423},"last":0.42599999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790134,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311568,"kb_used_data":1992,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433584,"statfs":{"total":95995035648,"available":95675990016,"internally_reserved":0,"allocated":2039808,"data_stored":1327853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:41 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":1.2210000000000001,"15min":0.80500000000000005},"min":{"1min":0.28899999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.648999999999999,"5min":11.648999999999999,"15min":11.648999999999999},"last":0.23000000000000001},{"interface":"front","average":{"1min":2.8570000000000002,"5min":1.294,"15min":0.875},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.68,"5min":11.68,"15min":11.68},"last":0.40999999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:48:51 2023","interfaces":[{"interface":"back","average":{"1min":2.673,"5min":1.78,"15min":0.998},"min":{"1min":0.41799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":9.6839999999999993,"5min":12.999000000000001,"15min":12.999000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":2.0680000000000001,"5min":1.532,"15min":0.90500000000000003},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.1120000000000001,"5min":7.3730000000000002,"15min":7.3730000000000002},"last":0.35499999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:48:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6669999999999998,"5min":1.4390000000000001,"15min":0.86099999999999999},"min":{"1min":0.34799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.3280000000000003,"5min":7.9000000000000004,"15min":7.9000000000000004},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.2350000000000001,"15min":0.79400000000000004},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.3059999999999992,"5min":10.352,"15min":10.352},"last":0.47299999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:48:47 2023","interfaces":[{"interface":"back","average":{"1min":2.859,"5min":2.048,"15min":1.165},"min":{"1min":0.432,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":12.558999999999999,"5min":18.016999999999999,"15min":18.016999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":2.6930000000000001,"5min":2.089,"15min":1.105},"min":{"1min":0.40300000000000002,"5min":0.317,"15min":0.317},"max":{"1min":7.9560000000000004,"5min":15.555999999999999,"15min":15.555999999999999},"last":0.439}]},{"osd":4,"last update":"Mon Oct 9 10:48:51 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":1.0629999999999999,"15min":0.66000000000000003},"min":{"1min":0.26800000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":6.093,"5min":19.152999999999999,"15min":19.152999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.5369999999999999,"5min":1.375,"15min":0.77800000000000002},"min":{"1min":0.32400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.8610000000000007,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":0.45800000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:49:11 2023","interfaces":[{"interface":"back","average":{"1min":2.04,"5min":1.4359999999999999,"15min":0.81000000000000005},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":6.9100000000000001,"5min":7.7060000000000004,"15min":7.7060000000000004},"last":0.375},{"interface":"front","average":{"1min":2.609,"5min":1.3109999999999999,"15min":0.78500000000000003},"min":{"1min":0.34799999999999998,"5min":0.318,"15min":0.30299999999999999},"max":{"1min":20.256,"5min":20.256,"15min":20.256},"last":0.61099999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:48:26 2023","interfaces":[{"interface":"back","average":{"1min":2.79,"5min":1.1160000000000001,"15min":0.74099999999999999},"min":{"1min":0.41899999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.3940000000000001,"5min":7.3940000000000001,"15min":7.3940000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":2.2669999999999999,"5min":1.0369999999999999,"15min":0.71999999999999997},"min":{"1min":0.38500000000000001,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":9.0579999999999998,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":0.5}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:49:15.010 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T10:49:15.011 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:49:15.018 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:15.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:15.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:15.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:15.149 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:15.170 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:15.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:15.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:15.181 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:49:15.181 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:49:15.181 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T10:49:15.181 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:49:15.274 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:15.277 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:15.304 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:15 smithi155 ceph-mon[184854]: from='client.45011 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:49:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:15 smithi155 ceph-mon[184854]: failed to encode map e195 with expected crc 2023-10-09T10:49:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:15 smithi155 ceph-mon[184854]: failed to encode map e195 with expected crc 2023-10-09T10:49:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:15 smithi155 ceph-mon[184854]: osdmap e195: 8 total, 8 up, 7 in 2023-10-09T10:49:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:15 smithi155 ceph-mon[184854]: failed to encode map e195 with expected crc 2023-10-09T10:49:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:15 smithi155 ceph-mon[184854]: failed to encode map e195 with expected crc 2023-10-09T10:49:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:15 smithi155 ceph-mon[184854]: failed to encode map e195 with expected crc 2023-10-09T10:49:15.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:15.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:49:15.478 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:15.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:15.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:15.574 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:49:15.581 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:15.644 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:15.664 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:15.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:15.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:15.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:49:15.794 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[195507]: from='client.45011 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:49:15.795 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[195507]: failed to encode map e195 with expected crc 2023-10-09T10:49:15.795 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[195507]: failed to encode map e195 with expected crc 2023-10-09T10:49:15.795 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[195507]: osdmap e195: 8 total, 8 up, 7 in 2023-10-09T10:49:15.795 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[195507]: failed to encode map e195 with expected crc 2023-10-09T10:49:15.795 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[195507]: failed to encode map e195 with expected crc 2023-10-09T10:49:15.795 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[195507]: failed to encode map e195 with expected crc 2023-10-09T10:49:15.795 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:15.795 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:49:15.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[200197]: from='client.45011 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:49:15.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[200197]: failed to encode map e195 with expected crc 2023-10-09T10:49:15.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[200197]: failed to encode map e195 with expected crc 2023-10-09T10:49:15.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[200197]: osdmap e195: 8 total, 8 up, 7 in 2023-10-09T10:49:15.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[200197]: failed to encode map e195 with expected crc 2023-10-09T10:49:15.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[200197]: failed to encode map e195 with expected crc 2023-10-09T10:49:15.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[200197]: failed to encode map e195 with expected crc 2023-10-09T10:49:15.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:15.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:49:15.797 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:15.834 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_1' created 2023-10-09T10:49:15.853 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:49:16.118 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:49:16.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:16.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:49:16.221 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:16.250 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:16.269 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:16.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:16.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:16.369 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-10-09T10:49:16.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:16.402 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:16.471 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:16.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:16.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:16.508 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:49:16.575 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:16.596 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:49:16.602 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:16 smithi155 ceph-mon[184854]: pgmap v437: 93 pgs: 4 unknown, 89 active+clean; 468 KiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:49:16.602 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:16 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1270111233' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T10:49:16.602 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:16 smithi155 ceph-mon[184854]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.603 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:16 smithi155 ceph-mon[184854]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.603 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:16 smithi155 ceph-mon[184854]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.603 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:16 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1270111233' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T10:49:16.603 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:16 smithi155 ceph-mon[184854]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.603 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:16 smithi155 ceph-mon[184854]: osdmap e196: 8 total, 8 up, 7 in 2023-10-09T10:49:16.603 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:16 smithi155 ceph-mon[184854]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.603 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:16 smithi155 ceph-mon[184854]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.604 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:16.625 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:49:16.644 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:49:16.664 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:16.670 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:49:16.691 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:49:16.710 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:49:16.725 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:16.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:16.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:16.730 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:49:16.748 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:49:16.750 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:49:16.750 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:49:16.750 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:49:16.750 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:49:16.750 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:49:16.750 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:49:16.750 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:49:16.750 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:49:16.775 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[200197]: pgmap v437: 93 pgs: 4 unknown, 89 active+clean; 468 KiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:49:16.776 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1270111233' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T10:49:16.776 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[200197]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.776 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[200197]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.776 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[200197]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.776 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1270111233' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T10:49:16.776 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[200197]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.776 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[200197]: osdmap e196: 8 total, 8 up, 7 in 2023-10-09T10:49:16.776 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[200197]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.776 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[200197]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.776 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[195507]: pgmap v437: 93 pgs: 4 unknown, 89 active+clean; 468 KiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:49:16.777 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1270111233' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T10:49:16.777 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[195507]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.777 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[195507]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.777 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[195507]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.777 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1270111233' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T10:49:16.777 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[195507]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.777 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[195507]: osdmap e196: 8 total, 8 up, 7 in 2023-10-09T10:49:16.777 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[195507]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.777 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:16 smithi125 ceph-mon[195507]: failed to encode map e196 with expected crc 2023-10-09T10:49:16.829 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:16.867 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:16.887 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:16.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:16.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:16.941 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:49:16.941 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:49:16.990 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:17.019 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:17.037 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:17.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:17.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:17.141 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:17.170 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:17.300 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:49:17.347 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:17.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:17 smithi155 ceph-mon[184854]: failed to encode map e196 with expected crc 2023-10-09T10:49:17.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3935739742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:17.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:17 smithi155 ceph-mon[184854]: 1.0 scrub starts 2023-10-09T10:49:17.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:17 smithi155 ceph-mon[184854]: 1.0 scrub ok 2023-10-09T10:49:17.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:17 smithi155 ceph-mon[184854]: osdmap e197: 8 total, 8 up, 7 in 2023-10-09T10:49:17.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:17 smithi155 ceph-mon[184854]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:17 smithi155 ceph-mon[184854]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:17 smithi155 ceph-mon[184854]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:17 smithi155 ceph-mon[184854]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:17 smithi155 ceph-mon[184854]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:17 smithi155 ceph-mon[184854]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.490 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:17.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:17.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:17.562 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:49:17.595 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:17.622 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:17.685 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[195507]: failed to encode map e196 with expected crc 2023-10-09T10:49:17.685 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3935739742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:17.685 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[195507]: 1.0 scrub starts 2023-10-09T10:49:17.685 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[195507]: 1.0 scrub ok 2023-10-09T10:49:17.686 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[195507]: osdmap e197: 8 total, 8 up, 7 in 2023-10-09T10:49:17.686 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[195507]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.686 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[195507]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.686 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[195507]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.686 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[195507]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.686 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[195507]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.686 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[195507]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.686 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[200197]: failed to encode map e196 with expected crc 2023-10-09T10:49:17.686 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3935739742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:17.686 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[200197]: 1.0 scrub starts 2023-10-09T10:49:17.687 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[200197]: 1.0 scrub ok 2023-10-09T10:49:17.687 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[200197]: osdmap e197: 8 total, 8 up, 7 in 2023-10-09T10:49:17.687 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[200197]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.687 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[200197]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.687 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[200197]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.687 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[200197]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.687 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[200197]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.687 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:17 smithi125 ceph-mon[200197]: failed to encode map e197 with expected crc 2023-10-09T10:49:17.691 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:17.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:17.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:17.794 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:17.823 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:17.953 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:17.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:17.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:18.058 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:18.086 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:18.105 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:18.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:18.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:18.209 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:18.237 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:18.262 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:18.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:18.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:18.366 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:18.397 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:18.535 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:49:18.535 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:49:18.556 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:18 smithi155 ceph-mon[184854]: failed to encode map e197 with expected crc 2023-10-09T10:49:18.556 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:18 smithi155 ceph-mon[184854]: pgmap v440: 109 pgs: 2 active+undersized+remapped, 6 creating+peering, 14 unknown, 87 active+clean; 468 KiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:49:18.556 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:18 smithi155 ceph-mon[184854]: osdmap e198: 8 total, 8 up, 7 in 2023-10-09T10:49:18.556 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:18 smithi155 ceph-mon[184854]: failed to encode map e198 with expected crc 2023-10-09T10:49:18.556 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:18 smithi155 ceph-mon[184854]: failed to encode map e198 with expected crc 2023-10-09T10:49:18.556 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:18 smithi155 ceph-mon[184854]: failed to encode map e198 with expected crc 2023-10-09T10:49:18.557 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:18 smithi155 ceph-mon[184854]: failed to encode map e198 with expected crc 2023-10-09T10:49:18.567 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:18.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:18.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:18.671 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:18.701 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:18.714 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:18 smithi125 ceph-mon[195507]: failed to encode map e197 with expected crc 2023-10-09T10:49:18.714 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:18 smithi125 ceph-mon[195507]: pgmap v440: 109 pgs: 2 active+undersized+remapped, 6 creating+peering, 14 unknown, 87 active+clean; 468 KiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:49:18.715 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:18 smithi125 ceph-mon[195507]: osdmap e198: 8 total, 8 up, 7 in 2023-10-09T10:49:18.715 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:18 smithi125 ceph-mon[195507]: failed to encode map e198 with expected crc 2023-10-09T10:49:18.715 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:18 smithi125 ceph-mon[195507]: failed to encode map e198 with expected crc 2023-10-09T10:49:18.715 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:18 smithi125 ceph-mon[195507]: failed to encode map e198 with expected crc 2023-10-09T10:49:18.715 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:18 smithi125 ceph-mon[195507]: failed to encode map e198 with expected crc 2023-10-09T10:49:18.715 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:18 smithi125 ceph-mon[200197]: failed to encode map e197 with expected crc 2023-10-09T10:49:18.715 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:18 smithi125 ceph-mon[200197]: pgmap v440: 109 pgs: 2 active+undersized+remapped, 6 creating+peering, 14 unknown, 87 active+clean; 468 KiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:49:18.715 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:18 smithi125 ceph-mon[200197]: osdmap e198: 8 total, 8 up, 7 in 2023-10-09T10:49:18.716 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:18 smithi125 ceph-mon[200197]: failed to encode map e198 with expected crc 2023-10-09T10:49:18.716 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:18 smithi125 ceph-mon[200197]: failed to encode map e198 with expected crc 2023-10-09T10:49:18.716 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:18 smithi125 ceph-mon[200197]: failed to encode map e198 with expected crc 2023-10-09T10:49:18.716 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:18 smithi125 ceph-mon[200197]: failed to encode map e198 with expected crc 2023-10-09T10:49:18.822 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:18.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:18.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:18.908 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:49:18.926 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:49:18.956 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:18.975 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:49:19.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:19.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:49:19.112 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:19.142 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:19.162 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:19.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:19.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:19.175 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:49:19.266 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:49:19.294 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:19.312 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:49:19.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:19.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:49:19.376 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:49:19.416 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:19.447 INFO:teuthology.orchestra.run.smithi125.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-10-09T10:49:19.449 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:19.451 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:49:19.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:19.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:19.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[195507]: failed to encode map e198 with expected crc 2023-10-09T10:49:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[195507]: failed to encode map e198 with expected crc 2023-10-09T10:49:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[195507]: failed to encode map e198 with expected crc 2023-10-09T10:49:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2793802585' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:49:19.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:49:19.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[195507]: 1.11 deep-scrub starts 2023-10-09T10:49:19.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[195507]: 1.11 deep-scrub ok 2023-10-09T10:49:19.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[195507]: 1.12 scrub starts 2023-10-09T10:49:19.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[195507]: 1.12 scrub ok 2023-10-09T10:49:19.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[195507]: 1.5 scrub starts 2023-10-09T10:49:19.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[195507]: 1.5 scrub ok 2023-10-09T10:49:19.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/580905621' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:49:19.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[195507]: pgmap v442: 109 pgs: 2 active+undersized+remapped, 6 creating+peering, 8 unknown, 93 active+clean; 442 KiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:49:19.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[200197]: failed to encode map e198 with expected crc 2023-10-09T10:49:19.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[200197]: failed to encode map e198 with expected crc 2023-10-09T10:49:19.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[200197]: failed to encode map e198 with expected crc 2023-10-09T10:49:19.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2793802585' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:49:19.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:49:19.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[200197]: 1.11 deep-scrub starts 2023-10-09T10:49:19.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[200197]: 1.11 deep-scrub ok 2023-10-09T10:49:19.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[200197]: 1.12 scrub starts 2023-10-09T10:49:19.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[200197]: 1.12 scrub ok 2023-10-09T10:49:19.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[200197]: 1.5 scrub starts 2023-10-09T10:49:19.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[200197]: 1.5 scrub ok 2023-10-09T10:49:19.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/580905621' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:49:19.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:19 smithi125 ceph-mon[200197]: pgmap v442: 109 pgs: 2 active+undersized+remapped, 6 creating+peering, 8 unknown, 93 active+clean; 442 KiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:49:19.581 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:19.611 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:19.630 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:19.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:19.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:19.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:19 smithi155 ceph-mon[184854]: failed to encode map e198 with expected crc 2023-10-09T10:49:19.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:19 smithi155 ceph-mon[184854]: failed to encode map e198 with expected crc 2023-10-09T10:49:19.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:19 smithi155 ceph-mon[184854]: failed to encode map e198 with expected crc 2023-10-09T10:49:19.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:19 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2793802585' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:49:19.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:19 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:49:19.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:19 smithi155 ceph-mon[184854]: 1.11 deep-scrub starts 2023-10-09T10:49:19.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:19 smithi155 ceph-mon[184854]: 1.11 deep-scrub ok 2023-10-09T10:49:19.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:19 smithi155 ceph-mon[184854]: 1.12 scrub starts 2023-10-09T10:49:19.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:19 smithi155 ceph-mon[184854]: 1.12 scrub ok 2023-10-09T10:49:19.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:19 smithi155 ceph-mon[184854]: 1.5 scrub starts 2023-10-09T10:49:19.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:19 smithi155 ceph-mon[184854]: 1.5 scrub ok 2023-10-09T10:49:19.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:19 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/580905621' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:49:19.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:19 smithi155 ceph-mon[184854]: pgmap v442: 109 pgs: 2 active+undersized+remapped, 6 creating+peering, 8 unknown, 93 active+clean; 442 KiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T10:49:19.743 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:49:19.770 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:19.889 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:49:19.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:19.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:49:20.001 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:20.028 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:20.048 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:20.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:20.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:20.072 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:20.152 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:20.181 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:20.188 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:49:20.188 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:49:20.201 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:20.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:20.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:20.305 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:20.333 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:20.352 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:20.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:20.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:20.438 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:20 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:49:20.438 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:20 smithi125 ceph-mon[200197]: failed to encode map e199 with expected crc 2023-10-09T10:49:20.438 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:20 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:49:20.438 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:20 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/580905621' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:49:20.439 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:20 smithi125 ceph-mon[200197]: osdmap e199: 8 total, 8 up, 7 in 2023-10-09T10:49:20.456 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:20.484 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:20.504 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:20.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:20.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:20.584 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:49:20.608 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:49:20.635 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:20.704 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:49:20.705 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:20 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:49:20.706 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:20 smithi125 ceph-mon[195507]: failed to encode map e199 with expected crc 2023-10-09T10:49:20.706 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:20 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:49:20.706 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:20 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/580905621' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:49:20.706 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:20 smithi125 ceph-mon[195507]: osdmap e199: 8 total, 8 up, 7 in 2023-10-09T10:49:20.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:20.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:49:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:20 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:49:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:20 smithi155 ceph-mon[184854]: failed to encode map e199 with expected crc 2023-10-09T10:49:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:20 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:49:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:20 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/580905621' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:49:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:20 smithi155 ceph-mon[184854]: osdmap e199: 8 total, 8 up, 7 in 2023-10-09T10:49:20.808 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:20.835 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:20.846 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:49:20.920 DEBUG:teuthology.orchestra.run.smithi125:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-10-09T10:49:20.920 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2023-10-09T10:49:20.920 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:49:20.922 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T10:49:20.923 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T10:49:20.954 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:20.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:20.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:20.976 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:hints = 1 2023-10-09T10:49:20.976 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T10:49:20.976 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object prefix: benchmark_data_smithi125_295816 2023-10-09T10:49:20.976 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:49:20.976 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 0 0 0 0 0 0 - 0 2023-10-09T10:49:21.061 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:49:21.113 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:21.148 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:49:21.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:21.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:49:21.217 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:49:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:49:21] "GET /metrics HTTP/1.1" 200 34121 "" "Prometheus/2.43.0" 2023-10-09T10:49:21.254 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:21.263 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:49:21.290 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:21.305 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:49:21.324 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:21.352 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:49:21.383 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:49:21.416 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:21.418 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:49:21.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:21.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:21.450 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:49:21.484 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:49:21.490 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:21 smithi125 ceph-mon[195507]: pgmap v444: 109 pgs: 4 creating+peering, 105 active+clean; 2.8 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 18 KiB/s rd, 489 KiB/s wr, 21 op/s; 94 KiB/s, 0 objects/s recovering 2023-10-09T10:49:21.490 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:21 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3200647969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:21.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:21 smithi125 ceph-mon[200197]: pgmap v444: 109 pgs: 4 creating+peering, 105 active+clean; 2.8 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 18 KiB/s rd, 489 KiB/s wr, 21 op/s; 94 KiB/s, 0 objects/s recovering 2023-10-09T10:49:21.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:21 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3200647969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:21.519 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:49:21.522 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:21.531 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:49:21.531 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:49:21.532 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:49:21.532 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:49:21.532 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:49:21.532 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:49:21.532 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:49:21.532 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:49:21.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:21 smithi155 ceph-mon[184854]: pgmap v444: 109 pgs: 4 creating+peering, 105 active+clean; 2.8 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 18 KiB/s rd, 489 KiB/s wr, 21 op/s; 94 KiB/s, 0 objects/s recovering 2023-10-09T10:49:21.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:21 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3200647969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:21.563 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:21.641 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:21.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:21.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:21.746 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:21.797 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:21.957 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:49:21.957 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:49:21.957 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:49:21.958 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:49:21.958 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:49:21.976 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 1 16 2783 2767 172.928 172.938 0.00323431 0.00576574 2023-10-09T10:49:22.079 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:22.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:22.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:22.183 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:49:22.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:49:22.229 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:22.259 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:49:22.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:22.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:49:22.329 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:49:22.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:22.414 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:22.443 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:22.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:22.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:22.547 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:22.587 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:22.613 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:22.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:22.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:22.673 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:22.693 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:49:22.718 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:22.757 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:22.839 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:22.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:22.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:22.942 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:22.977 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 2 16 5239 5223 163.154 153.5 0.00356626 0.00611894 2023-10-09T10:49:22.984 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:23.011 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:23.016 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:49:23.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:23.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:23.028 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:49:23.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:49:23.176 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:23.789 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:49:23.789 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:49:23.790 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:49:23.790 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:49:23.790 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.790 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70444 6.3 6:c0306d38:::benchmark_data_smithi125_244951_object70443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:49:23.790 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.568354+0000", 2023-10-09T10:49:23.790 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 240.21687353600001, 2023-10-09T10:49:23.790 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047306648, 2023-10-09T10:49:23.790 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.790 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.790 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.790 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.791 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.791 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70444 2023-10-09T10:49:23.791 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.791 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568354+0000", 2023-10-09T10:49:23.791 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.791 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568354+0000", 2023-10-09T10:49:23.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.792 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568353+0000", 2023-10-09T10:49:23.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:23.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.792 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568387+0000", 2023-10-09T10:49:23.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4495000000000001e-05 2023-10-09T10:49:23.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.793 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.793 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568407+0000", 2023-10-09T10:49:23.793 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9729999999999999e-05 2023-10-09T10:49:23.793 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.793 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.793 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.793 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568412+0000", 2023-10-09T10:49:23.793 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4149999999999998e-06 2023-10-09T10:49:23.793 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.793 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.793 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.793 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568457+0000", 2023-10-09T10:49:23.794 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4270000000000001e-05 2023-10-09T10:49:23.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.794 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.794 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568587+0000", 2023-10-09T10:49:23.794 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013011399999999999 2023-10-09T10:49:23.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.794 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:49:23.794 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568753+0000", 2023-10-09T10:49:23.794 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016615099999999999 2023-10-09T10:49:23.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.795 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.570931+0000", 2023-10-09T10:49:23.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002177596 2023-10-09T10:49:23.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.795 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.570969+0000", 2023-10-09T10:49:23.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8661000000000001e-05 2023-10-09T10:49:23.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.796 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.796 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615586+0000", 2023-10-09T10:49:23.796 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.044616524999999997 2023-10-09T10:49:23.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.796 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.796 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615636+0000", 2023-10-09T10:49:23.796 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0009000000000002e-05 2023-10-09T10:49:23.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.797 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.797 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615661+0000", 2023-10-09T10:49:23.797 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.516e-05 2023-10-09T10:49:23.797 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.797 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.797 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.797 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.797 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.797 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70445 6.3 6:c48e8359:::benchmark_data_smithi125_244951_object70444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:49:23.797 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.568425+0000", 2023-10-09T10:49:23.797 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 240.216803277, 2023-10-09T10:49:23.798 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047372754000000003, 2023-10-09T10:49:23.798 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.798 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.798 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.798 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.798 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.798 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70445 2023-10-09T10:49:23.798 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.798 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.798 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.798 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.798 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568425+0000", 2023-10-09T10:49:23.799 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.799 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.799 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.799 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.799 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568425+0000", 2023-10-09T10:49:23.799 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.799 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.799 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.799 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.799 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568424+0000", 2023-10-09T10:49:23.799 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:23.799 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.800 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.800 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.800 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568448+0000", 2023-10-09T10:49:23.800 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3623000000000001e-05 2023-10-09T10:49:23.800 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.800 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.800 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.800 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568453+0000", 2023-10-09T10:49:23.800 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1880000000000001e-06 2023-10-09T10:49:23.800 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.800 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.801 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.801 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568457+0000", 2023-10-09T10:49:23.801 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1609999999999996e-06 2023-10-09T10:49:23.801 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.801 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.801 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.801 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.569823+0000", 2023-10-09T10:49:23.801 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001365996 2023-10-09T10:49:23.801 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.801 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.801 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.801 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.569913+0000", 2023-10-09T10:49:23.802 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9388000000000005e-05 2023-10-09T10:49:23.802 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.802 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.802 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:49:23.802 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.570017+0000", 2023-10-09T10:49:23.802 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103997 2023-10-09T10:49:23.802 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.802 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.802 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.802 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.571104+0000", 2023-10-09T10:49:23.802 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00108701 2023-10-09T10:49:23.803 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.803 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.803 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.803 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.571280+0000", 2023-10-09T10:49:23.803 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017659500000000001 2023-10-09T10:49:23.803 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.803 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.803 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.803 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615712+0000", 2023-10-09T10:49:23.803 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.044431813000000001 2023-10-09T10:49:23.804 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.804 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.804 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.804 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615755+0000", 2023-10-09T10:49:23.804 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2588e-05 2023-10-09T10:49:23.804 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.804 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.804 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.804 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615797+0000", 2023-10-09T10:49:23.804 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2756999999999997e-05 2023-10-09T10:49:23.804 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.805 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.805 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.805 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106416 6.3 6:c0486f03:::benchmark_data_smithi125_244951_object106415:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:49:23.805 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.234195+0000", 2023-10-09T10:49:23.805 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 201.55103303999999, 2023-10-09T10:49:23.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.048317987999999999, 2023-10-09T10:49:23.805 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.805 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.805 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.806 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.806 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.806 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106416 2023-10-09T10:49:23.806 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.806 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.806 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.806 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.806 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234195+0000", 2023-10-09T10:49:23.806 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.806 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.806 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.806 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.807 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234195+0000", 2023-10-09T10:49:23.807 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.807 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.807 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.807 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.807 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234193+0000", 2023-10-09T10:49:23.807 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:23.807 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.807 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.807 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.807 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234285+0000", 2023-10-09T10:49:23.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1546999999999996e-05 2023-10-09T10:49:23.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234305+0000", 2023-10-09T10:49:23.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0119000000000002e-05 2023-10-09T10:49:23.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234313+0000", 2023-10-09T10:49:23.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3590000000000008e-06 2023-10-09T10:49:23.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234382+0000", 2023-10-09T10:49:23.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8261000000000003e-05 2023-10-09T10:49:23.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234490+0000", 2023-10-09T10:49:23.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000108398 2023-10-09T10:49:23.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.810 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:49:23.810 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234588+0000", 2023-10-09T10:49:23.810 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7966000000000004e-05 2023-10-09T10:49:23.810 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.810 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.810 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.810 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.236668+0000", 2023-10-09T10:49:23.810 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020797540000000001 2023-10-09T10:49:23.810 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.810 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.811 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.811 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.236734+0000", 2023-10-09T10:49:23.811 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6644999999999996e-05 2023-10-09T10:49:23.811 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.811 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.811 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.811 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.282424+0000", 2023-10-09T10:49:23.811 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.045689354000000001 2023-10-09T10:49:23.811 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.811 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.812 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.812 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.282474+0000", 2023-10-09T10:49:23.812 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9938999999999997e-05 2023-10-09T10:49:23.812 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.812 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.812 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.812 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.282513+0000", 2023-10-09T10:49:23.812 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9168999999999997e-05 2023-10-09T10:49:23.812 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.812 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.812 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.813 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.813 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.813 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:137725 6.3 6:c0616549:::benchmark_data_smithi125_244951_object31282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:49:23.813 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:05.071869+0000", 2023-10-09T10:49:23.813 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 138.71335888799999, 2023-10-09T10:49:23.813 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038173246000000001, 2023-10-09T10:49:23.813 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.813 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.813 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.813 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.813 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.814 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 137725 2023-10-09T10:49:23.814 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.814 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.814 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.814 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.814 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071869+0000", 2023-10-09T10:49:23.814 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.814 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.814 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.814 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.814 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071869+0000", 2023-10-09T10:49:23.815 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.815 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.815 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.815 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.815 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071867+0000", 2023-10-09T10:49:23.815 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:23.815 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.815 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.815 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.815 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071875+0000", 2023-10-09T10:49:23.815 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6290000000000001e-06 2023-10-09T10:49:23.816 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.816 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.816 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.816 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071885+0000", 2023-10-09T10:49:23.816 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0397999999999999e-05 2023-10-09T10:49:23.816 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.816 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.816 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.816 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071892+0000", 2023-10-09T10:49:23.816 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6490000000000002e-06 2023-10-09T10:49:23.816 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.817 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.817 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.817 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.081942+0000", 2023-10-09T10:49:23.817 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010050325000000001 2023-10-09T10:49:23.817 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.817 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.817 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.817 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.082111+0000", 2023-10-09T10:49:23.817 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016843500000000001 2023-10-09T10:49:23.817 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.817 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.817 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:49:23.818 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.082600+0000", 2023-10-09T10:49:23.818 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00048894400000000005 2023-10-09T10:49:23.818 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.818 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.818 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.818 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.084543+0000", 2023-10-09T10:49:23.818 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019437110000000001 2023-10-09T10:49:23.818 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.818 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.818 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.818 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.086053+0000", 2023-10-09T10:49:23.818 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015096179999999999 2023-10-09T10:49:23.819 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.819 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.819 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.819 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.109298+0000", 2023-10-09T10:49:23.819 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023244813 2023-10-09T10:49:23.819 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.819 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.819 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.819 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.109336+0000", 2023-10-09T10:49:23.819 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.79e-05 2023-10-09T10:49:23.819 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.820 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.820 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.820 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.110042+0000", 2023-10-09T10:49:23.820 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00070647600000000004 2023-10-09T10:49:23.820 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.820 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.820 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.820 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.820 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.820 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:137790 6.13 6:c864fc16:::benchmark_data_smithi125_244951_object31347:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:49:23.821 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:05.180662+0000", 2023-10-09T10:49:23.821 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 138.60456568999999, 2023-10-09T10:49:23.821 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038056526, 2023-10-09T10:49:23.821 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.821 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.821 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.821 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.821 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.821 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 137790 2023-10-09T10:49:23.821 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.821 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.821 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.822 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.822 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180662+0000", 2023-10-09T10:49:23.822 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.822 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.822 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.822 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.822 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180662+0000", 2023-10-09T10:49:23.822 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.822 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.822 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.822 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.822 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180661+0000", 2023-10-09T10:49:23.823 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:23.823 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.823 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.823 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.823 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180667+0000", 2023-10-09T10:49:23.823 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4679999999999997e-06 2023-10-09T10:49:23.823 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.823 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.823 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.823 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180680+0000", 2023-10-09T10:49:23.823 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2431e-05 2023-10-09T10:49:23.824 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.824 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.824 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.824 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180685+0000", 2023-10-09T10:49:23.824 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1200000000000001e-06 2023-10-09T10:49:23.824 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.824 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.824 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.824 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182279+0000", 2023-10-09T10:49:23.824 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001594804 2023-10-09T10:49:23.824 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.824 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.825 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.825 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182481+0000", 2023-10-09T10:49:23.825 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00020201 2023-10-09T10:49:23.825 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.825 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.825 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:49:23.825 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182899+0000", 2023-10-09T10:49:23.825 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00041767300000000002 2023-10-09T10:49:23.825 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.825 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.826 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.826 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.185800+0000", 2023-10-09T10:49:23.826 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029009109999999999 2023-10-09T10:49:23.826 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.826 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.826 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.826 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.185815+0000", 2023-10-09T10:49:23.826 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4959e-05 2023-10-09T10:49:23.826 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.826 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.827 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.827 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.217393+0000", 2023-10-09T10:49:23.827 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.031578322999999998 2023-10-09T10:49:23.827 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.827 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.827 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.827 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.217443+0000", 2023-10-09T10:49:23.827 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9308999999999998e-05 2023-10-09T10:49:23.827 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.827 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.828 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.828 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.218719+0000", 2023-10-09T10:49:23.828 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001276016 2023-10-09T10:49:23.828 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.828 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.828 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.828 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.828 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.828 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:137793 6.3 6:c3f14b3e:::benchmark_data_smithi125_244951_object31350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:49:23.829 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:49:23.829 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 138.60453177599999, 2023-10-09T10:49:23.829 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.046213456, 2023-10-09T10:49:23.829 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.829 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.829 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.829 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.829 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.829 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 137793 2023-10-09T10:49:23.829 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.829 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.829 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.830 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.830 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:49:23.830 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.830 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.830 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.830 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.830 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:49:23.830 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.830 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.830 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.830 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.830 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:49:23.831 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:23.831 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.831 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.831 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.831 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180698+0000", 2023-10-09T10:49:23.831 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5009999999999999e-06 2023-10-09T10:49:23.831 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.831 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.831 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.831 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180702+0000", 2023-10-09T10:49:23.831 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2200000000000003e-06 2023-10-09T10:49:23.832 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.832 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.832 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.832 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180705+0000", 2023-10-09T10:49:23.832 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7779999999999999e-06 2023-10-09T10:49:23.832 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.832 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.832 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.832 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182790+0000", 2023-10-09T10:49:23.832 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020852819999999999 2023-10-09T10:49:23.832 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.832 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.833 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.833 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182925+0000", 2023-10-09T10:49:23.833 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013469099999999999 2023-10-09T10:49:23.833 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.833 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.833 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:49:23.833 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.183331+0000", 2023-10-09T10:49:23.833 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000405783 2023-10-09T10:49:23.833 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.833 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.833 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.834 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.185825+0000", 2023-10-09T10:49:23.834 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024937789999999998 2023-10-09T10:49:23.834 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.834 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.834 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.834 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.188205+0000", 2023-10-09T10:49:23.834 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023806790000000001 2023-10-09T10:49:23.834 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.834 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.834 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.834 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.225637+0000", 2023-10-09T10:49:23.835 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.037431091 2023-10-09T10:49:23.835 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.835 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.835 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.835 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.225680+0000", 2023-10-09T10:49:23.835 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3856999999999997e-05 2023-10-09T10:49:23.835 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.835 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.835 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.835 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.226909+0000", 2023-10-09T10:49:23.835 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001229129 2023-10-09T10:49:23.835 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.836 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.836 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.836 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.836 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.836 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:137805 6.3 6:c448070c:::benchmark_data_smithi125_244951_object31362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:49:23.836 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:05.197338+0000", 2023-10-09T10:49:23.836 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 138.58789017399999, 2023-10-09T10:49:23.836 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039251805000000001, 2023-10-09T10:49:23.836 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.836 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.836 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.836 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.836 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.837 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 137805 2023-10-09T10:49:23.837 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.837 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.837 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.837 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.837 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197338+0000", 2023-10-09T10:49:23.837 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.837 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.837 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.837 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.837 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197338+0000", 2023-10-09T10:49:23.837 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.837 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.838 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.838 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.838 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197336+0000", 2023-10-09T10:49:23.838 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:23.838 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.838 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.838 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.838 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197342+0000", 2023-10-09T10:49:23.838 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3990000000000001e-06 2023-10-09T10:49:23.838 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.838 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.838 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.839 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197354+0000", 2023-10-09T10:49:23.839 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1821000000000001e-05 2023-10-09T10:49:23.839 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.839 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.839 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.839 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197359+0000", 2023-10-09T10:49:23.839 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9960000000000001e-06 2023-10-09T10:49:23.839 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.839 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.839 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.839 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197413+0000", 2023-10-09T10:49:23.839 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3909999999999997e-05 2023-10-09T10:49:23.840 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.840 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.840 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.840 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197587+0000", 2023-10-09T10:49:23.840 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000173408 2023-10-09T10:49:23.840 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.840 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.840 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:49:23.840 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197988+0000", 2023-10-09T10:49:23.840 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00040095300000000001 2023-10-09T10:49:23.840 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.840 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.840 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.841 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.199873+0000", 2023-10-09T10:49:23.841 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018855790000000001 2023-10-09T10:49:23.841 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.841 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.841 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.841 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.200673+0000", 2023-10-09T10:49:23.841 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00079994400000000002 2023-10-09T10:49:23.841 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.841 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.841 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.841 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.235493+0000", 2023-10-09T10:49:23.841 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.034820028000000003 2023-10-09T10:49:23.841 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.842 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.842 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.842 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.235533+0000", 2023-10-09T10:49:23.842 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0139000000000003e-05 2023-10-09T10:49:23.842 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.842 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.842 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.842 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.236589+0000", 2023-10-09T10:49:23.842 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001056219 2023-10-09T10:49:23.842 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.842 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.842 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.843 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.843 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.843 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:152091 6.6 e174/107 6:64b0928f:::benchmark_data_smithi125_244951_object45648:head v 174'5840, mlcod=174'5838)", 2023-10-09T10:49:23.843 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.267910+0000", 2023-10-09T10:49:23.843 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 109.51731752000001, 2023-10-09T10:49:23.843 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038740213000000003, 2023-10-09T10:49:23.843 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.843 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.843 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.843 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.843 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.844 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267910+0000", 2023-10-09T10:49:23.844 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.844 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.844 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.844 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.844 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267910+0000", 2023-10-09T10:49:23.844 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.844 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.844 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.844 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.844 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267909+0000", 2023-10-09T10:49:23.844 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:49:23.845 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.845 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.845 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.845 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267917+0000", 2023-10-09T10:49:23.845 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1869999999999997e-06 2023-10-09T10:49:23.845 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.845 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.845 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.845 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267931+0000", 2023-10-09T10:49:23.845 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3239e-05 2023-10-09T10:49:23.845 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.845 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.846 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.846 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267936+0000", 2023-10-09T10:49:23.846 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.451e-06 2023-10-09T10:49:23.846 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.846 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.846 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.846 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267979+0000", 2023-10-09T10:49:23.846 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3096000000000002e-05 2023-10-09T10:49:23.846 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.846 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.846 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.846 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.268018+0000", 2023-10-09T10:49:23.846 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8532999999999999e-05 2023-10-09T10:49:23.847 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.847 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.847 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.847 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.306618+0000", 2023-10-09T10:49:23.847 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038600661000000001 2023-10-09T10:49:23.847 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.847 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.847 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.847 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.306651+0000", 2023-10-09T10:49:23.847 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2076000000000003e-05 2023-10-09T10:49:23.847 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.847 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.847 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.848 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.848 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.848 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:152093 6.13 6:cd7dbd99:::benchmark_data_smithi125_244951_object45650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:49:23.848 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.270440+0000", 2023-10-09T10:49:23.848 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 109.51478787400001, 2023-10-09T10:49:23.848 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040432893999999997, 2023-10-09T10:49:23.848 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.848 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.848 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.848 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.848 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.849 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 152093 2023-10-09T10:49:23.849 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.849 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.849 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.849 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.849 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270440+0000", 2023-10-09T10:49:23.849 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.849 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.849 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.849 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.849 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270440+0000", 2023-10-09T10:49:23.850 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.850 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.850 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.850 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.850 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270438+0000", 2023-10-09T10:49:23.850 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T10:49:23.850 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.850 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.850 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.850 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270450+0000", 2023-10-09T10:49:23.850 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1918e-05 2023-10-09T10:49:23.850 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.851 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.851 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.851 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270479+0000", 2023-10-09T10:49:23.851 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8911000000000002e-05 2023-10-09T10:49:23.851 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.851 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.851 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.851 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270488+0000", 2023-10-09T10:49:23.851 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6139999999999998e-06 2023-10-09T10:49:23.851 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.851 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.851 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.851 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270586+0000", 2023-10-09T10:49:23.852 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7823000000000002e-05 2023-10-09T10:49:23.852 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.852 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.852 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.852 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270758+0000", 2023-10-09T10:49:23.852 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000172103 2023-10-09T10:49:23.852 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.852 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.852 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:49:23.852 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.271250+0000", 2023-10-09T10:49:23.852 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000491688 2023-10-09T10:49:23.852 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.853 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.853 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.853 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.278619+0000", 2023-10-09T10:49:23.853 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0073690730000000003 2023-10-09T10:49:23.853 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.853 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.853 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.853 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.284528+0000", 2023-10-09T10:49:23.853 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059091789999999996 2023-10-09T10:49:23.853 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.853 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.853 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.854 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.309269+0000", 2023-10-09T10:49:23.854 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024740522000000001 2023-10-09T10:49:23.854 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.854 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.854 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.854 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.309319+0000", 2023-10-09T10:49:23.854 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0912999999999999e-05 2023-10-09T10:49:23.854 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.854 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.854 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.854 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.310873+0000", 2023-10-09T10:49:23.855 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001553369 2023-10-09T10:49:23.855 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.855 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.855 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.855 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.855 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.855 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:152095 6.1 e174/164 6:8109f342:::benchmark_data_smithi125_244951_object45652:head v 174'5685, mlcod=174'5683)", 2023-10-09T10:49:23.855 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.273512+0000", 2023-10-09T10:49:23.855 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 109.51171593399999, 2023-10-09T10:49:23.855 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040480956999999998, 2023-10-09T10:49:23.855 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.855 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.855 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.856 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.856 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.856 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273512+0000", 2023-10-09T10:49:23.856 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.856 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.856 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.856 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.856 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273512+0000", 2023-10-09T10:49:23.856 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.856 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.856 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.856 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.856 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273511+0000", 2023-10-09T10:49:23.857 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:49:23.857 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.857 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.857 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.857 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273520+0000", 2023-10-09T10:49:23.857 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.645e-06 2023-10-09T10:49:23.857 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.857 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.857 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.857 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273534+0000", 2023-10-09T10:49:23.857 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3875e-05 2023-10-09T10:49:23.857 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.857 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.858 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.858 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273541+0000", 2023-10-09T10:49:23.858 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4790000000000001e-06 2023-10-09T10:49:23.858 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.858 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.858 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.858 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273796+0000", 2023-10-09T10:49:23.858 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025495599999999998 2023-10-09T10:49:23.858 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.858 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.858 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.858 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273819+0000", 2023-10-09T10:49:23.859 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3175000000000001e-05 2023-10-09T10:49:23.859 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.859 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.859 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.859 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.313948+0000", 2023-10-09T10:49:23.859 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040129418 2023-10-09T10:49:23.859 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.859 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.859 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.859 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.313993+0000", 2023-10-09T10:49:23.859 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4549000000000003e-05 2023-10-09T10:49:23.859 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.859 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.860 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.860 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.860 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.860 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:161148 6.13 6:c8d28001:::benchmark_data_smithi125_244951_object54705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e176)", 2023-10-09T10:49:23.860 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:47.001392+0000", 2023-10-09T10:49:23.860 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 96.783836054000005, 2023-10-09T10:49:23.860 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038885253000000002, 2023-10-09T10:49:23.860 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.860 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.860 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.860 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.860 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.861 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 161148 2023-10-09T10:49:23.861 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.861 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.861 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.861 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.861 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001392+0000", 2023-10-09T10:49:23.861 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.861 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.861 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.861 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.861 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001392+0000", 2023-10-09T10:49:23.861 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.861 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.862 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.862 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.862 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001391+0000", 2023-10-09T10:49:23.862 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:49:23.862 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.862 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.862 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.862 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001397+0000", 2023-10-09T10:49:23.862 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0560000000000001e-06 2023-10-09T10:49:23.862 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.862 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.863 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.863 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001410+0000", 2023-10-09T10:49:23.863 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2695e-05 2023-10-09T10:49:23.863 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.863 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.863 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.863 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001416+0000", 2023-10-09T10:49:23.863 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9569999999999996e-06 2023-10-09T10:49:23.863 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.863 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.863 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.863 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.020851+0000", 2023-10-09T10:49:23.864 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019434414000000001 2023-10-09T10:49:23.864 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.864 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.864 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.864 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.021010+0000", 2023-10-09T10:49:23.864 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015947300000000001 2023-10-09T10:49:23.864 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.864 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.864 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:49:23.864 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.021509+0000", 2023-10-09T10:49:23.864 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00049899199999999997 2023-10-09T10:49:23.864 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.865 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.865 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.865 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.026203+0000", 2023-10-09T10:49:23.865 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0046932700000000003 2023-10-09T10:49:23.865 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.865 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.865 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.865 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.026305+0000", 2023-10-09T10:49:23.865 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000102744 2023-10-09T10:49:23.865 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.865 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.866 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.866 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.038789+0000", 2023-10-09T10:49:23.866 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012483823 2023-10-09T10:49:23.866 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.866 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.866 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.866 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.038863+0000", 2023-10-09T10:49:23.866 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3634999999999995e-05 2023-10-09T10:49:23.866 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.866 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.866 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.866 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.040277+0000", 2023-10-09T10:49:23.867 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014141900000000001 2023-10-09T10:49:23.867 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.867 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.867 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.867 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.867 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.867 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:162909 6.13 6:c8b03eff:::benchmark_data_smithi125_244951_object56466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-10-09T10:49:23.867 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:49.864994+0000", 2023-10-09T10:49:23.867 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 93.920233409999994, 2023-10-09T10:49:23.867 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.037068614, 2023-10-09T10:49:23.867 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.868 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.868 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.868 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.868 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.868 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162909 2023-10-09T10:49:23.868 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.868 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.868 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.868 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.868 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.864994+0000", 2023-10-09T10:49:23.868 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.868 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.868 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.869 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.869 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.864994+0000", 2023-10-09T10:49:23.869 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.869 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.869 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.869 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.869 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.864993+0000", 2023-10-09T10:49:23.869 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:49:23.869 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.869 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.869 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.869 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.864999+0000", 2023-10-09T10:49:23.870 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2809999999999999e-06 2023-10-09T10:49:23.870 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.870 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.870 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.870 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.865009+0000", 2023-10-09T10:49:23.870 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.039e-05 2023-10-09T10:49:23.870 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.870 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.870 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.870 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.865015+0000", 2023-10-09T10:49:23.870 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.592e-06 2023-10-09T10:49:23.871 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.871 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.871 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.871 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.865060+0000", 2023-10-09T10:49:23.871 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5311e-05 2023-10-09T10:49:23.871 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.871 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.871 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.871 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.865238+0000", 2023-10-09T10:49:23.871 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017833700000000001 2023-10-09T10:49:23.871 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.871 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.872 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:49:23.872 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.865775+0000", 2023-10-09T10:49:23.872 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00053696699999999996 2023-10-09T10:49:23.872 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.872 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.872 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.872 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.869743+0000", 2023-10-09T10:49:23.872 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039680150000000001 2023-10-09T10:49:23.872 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.872 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.872 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.872 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.870048+0000", 2023-10-09T10:49:23.872 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030474199999999999 2023-10-09T10:49:23.873 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.873 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.873 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.873 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.900766+0000", 2023-10-09T10:49:23.873 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030717874999999999 2023-10-09T10:49:23.873 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.873 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.873 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.873 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.900828+0000", 2023-10-09T10:49:23.873 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2466999999999997e-05 2023-10-09T10:49:23.873 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.874 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.874 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.874 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.902063+0000", 2023-10-09T10:49:23.874 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012347180000000001 2023-10-09T10:49:23.874 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.874 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.874 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.874 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.874 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.874 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:162969 6.13 6:cec0b93d:::benchmark_data_smithi125_244951_object56526:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-10-09T10:49:23.874 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:49.943081+0000", 2023-10-09T10:49:23.874 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 93.842147276999995, 2023-10-09T10:49:23.875 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039960117000000003, 2023-10-09T10:49:23.875 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.875 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.875 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.875 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.875 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.875 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162969 2023-10-09T10:49:23.875 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.875 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.875 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.875 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.876 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943081+0000", 2023-10-09T10:49:23.876 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.876 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.876 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.876 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.876 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943081+0000", 2023-10-09T10:49:23.876 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.876 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.876 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.876 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.876 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943079+0000", 2023-10-09T10:49:23.877 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:23.877 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.877 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.877 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.877 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943085+0000", 2023-10-09T10:49:23.877 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6890000000000001e-06 2023-10-09T10:49:23.877 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.877 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.877 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.877 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943097+0000", 2023-10-09T10:49:23.877 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2166e-05 2023-10-09T10:49:23.877 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.878 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.878 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.878 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943103+0000", 2023-10-09T10:49:23.878 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9150000000000001e-06 2023-10-09T10:49:23.878 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.878 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.878 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.878 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943207+0000", 2023-10-09T10:49:23.878 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104573 2023-10-09T10:49:23.878 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.879 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.879 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.879 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943393+0000", 2023-10-09T10:49:23.879 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018516299999999999 2023-10-09T10:49:23.879 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.879 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.879 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:49:23.879 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943913+0000", 2023-10-09T10:49:23.879 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00052048100000000005 2023-10-09T10:49:23.879 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.879 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.880 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.880 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.950165+0000", 2023-10-09T10:49:23.880 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062519439999999997 2023-10-09T10:49:23.880 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.880 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.880 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.880 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.956256+0000", 2023-10-09T10:49:23.880 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0060905630000000002 2023-10-09T10:49:23.880 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.880 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.880 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.880 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.981642+0000", 2023-10-09T10:49:23.881 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025386756999999999 2023-10-09T10:49:23.881 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.881 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.881 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.881 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.981697+0000", 2023-10-09T10:49:23.881 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4718e-05 2023-10-09T10:49:23.881 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.881 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.881 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.881 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.983041+0000", 2023-10-09T10:49:23.881 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001343639 2023-10-09T10:49:23.881 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.882 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.882 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.882 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.882 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.882 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:169526 6.13 6:c9d383e8:::benchmark_data_smithi125_244951_object63083:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-10-09T10:49:23.882 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:59.084841+0000", 2023-10-09T10:49:23.882 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 84.700386761999994, 2023-10-09T10:49:23.882 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.036194846000000003, 2023-10-09T10:49:23.882 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.882 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.882 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.883 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.883 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.883 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 169526 2023-10-09T10:49:23.883 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.883 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.883 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.883 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.883 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084841+0000", 2023-10-09T10:49:23.883 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.883 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.883 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.883 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.884 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084841+0000", 2023-10-09T10:49:23.884 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.884 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.884 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.884 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.884 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084841+0000", 2023-10-09T10:49:23.884 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:23.884 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.884 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.884 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.884 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084844+0000", 2023-10-09T10:49:23.885 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4860000000000002e-06 2023-10-09T10:49:23.885 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.885 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.885 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.885 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084851+0000", 2023-10-09T10:49:23.885 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7819999999999996e-06 2023-10-09T10:49:23.885 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.885 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.885 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.885 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084855+0000", 2023-10-09T10:49:23.885 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3719999999999998e-06 2023-10-09T10:49:23.885 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.886 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.886 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.886 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.087999+0000", 2023-10-09T10:49:23.886 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0031432880000000002 2023-10-09T10:49:23.886 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.886 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.886 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.886 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.088176+0000", 2023-10-09T10:49:23.886 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017714799999999999 2023-10-09T10:49:23.886 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.886 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.886 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:49:23.887 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.088664+0000", 2023-10-09T10:49:23.887 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00048842899999999997 2023-10-09T10:49:23.887 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.887 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.887 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.887 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.099515+0000", 2023-10-09T10:49:23.887 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010850966 2023-10-09T10:49:23.887 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.887 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.887 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.887 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.102453+0000", 2023-10-09T10:49:23.887 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002937598 2023-10-09T10:49:23.888 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.888 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.888 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.888 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.119544+0000", 2023-10-09T10:49:23.888 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017091733000000001 2023-10-09T10:49:23.888 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.888 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.888 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.888 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.119602+0000", 2023-10-09T10:49:23.888 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7642000000000001e-05 2023-10-09T10:49:23.888 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.888 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.889 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.889 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.121036+0000", 2023-10-09T10:49:23.889 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014338599999999999 2023-10-09T10:49:23.889 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.889 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.889 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.889 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.889 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.889 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:170177 6.3 6:c6f24c09:::benchmark_data_smithi125_244951_object63734:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-10-09T10:49:23.889 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:59.997605+0000", 2023-10-09T10:49:23.889 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 83.787623151999995, 2023-10-09T10:49:23.889 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038407996999999999, 2023-10-09T10:49:23.889 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.890 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.890 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.890 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.890 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.890 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 170177 2023-10-09T10:49:23.890 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.890 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.890 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.890 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.890 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997605+0000", 2023-10-09T10:49:23.890 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.890 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.891 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.891 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.891 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997605+0000", 2023-10-09T10:49:23.891 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.891 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.891 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.891 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.891 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997604+0000", 2023-10-09T10:49:23.891 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:23.891 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.891 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.891 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.892 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997607+0000", 2023-10-09T10:49:23.892 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6170000000000001e-06 2023-10-09T10:49:23.892 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.892 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.892 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.892 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997612+0000", 2023-10-09T10:49:23.892 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0490000000000003e-06 2023-10-09T10:49:23.892 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.892 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.892 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.892 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997616+0000", 2023-10-09T10:49:23.892 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5889999999999999e-06 2023-10-09T10:49:23.893 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.893 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.893 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.893 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.013316+0000", 2023-10-09T10:49:23.893 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015700823999999999 2023-10-09T10:49:23.893 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.893 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.893 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.893 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.013397+0000", 2023-10-09T10:49:23.893 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0259999999999994e-05 2023-10-09T10:49:23.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.894 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.894 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:49:23.894 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.013825+0000", 2023-10-09T10:49:23.894 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000428131 2023-10-09T10:49:23.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.894 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.894 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.894 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.017829+0000", 2023-10-09T10:49:23.894 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040045319999999999 2023-10-09T10:49:23.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.894 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.895 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.895 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.018823+0000", 2023-10-09T10:49:23.895 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00099334699999999989 2023-10-09T10:49:23.895 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.895 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.895 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.895 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.035922+0000", 2023-10-09T10:49:23.895 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017099487999999999 2023-10-09T10:49:23.895 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.895 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.895 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.895 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.035975+0000", 2023-10-09T10:49:23.895 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2741999999999997e-05 2023-10-09T10:49:23.896 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.896 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.896 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.896 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.036013+0000", 2023-10-09T10:49:23.896 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7812999999999999e-05 2023-10-09T10:49:23.896 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.896 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.896 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.896 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.896 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.896 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:189608 6.3 6:c6b3429c:::benchmark_data_smithi125_244951_object83165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2023-10-09T10:49:23.896 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:29.169892+0000", 2023-10-09T10:49:23.897 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 54.615335795, 2023-10-09T10:49:23.897 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039847636999999998, 2023-10-09T10:49:23.897 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.897 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.897 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.897 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.897 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.897 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 189608 2023-10-09T10:49:23.897 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.897 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.897 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169892+0000", 2023-10-09T10:49:23.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.898 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.898 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169892+0000", 2023-10-09T10:49:23.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.898 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.898 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169891+0000", 2023-10-09T10:49:23.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:23.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.899 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.899 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.899 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169896+0000", 2023-10-09T10:49:23.899 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7950000000000004e-06 2023-10-09T10:49:23.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.899 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.899 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.899 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169909+0000", 2023-10-09T10:49:23.899 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2062000000000001e-05 2023-10-09T10:49:23.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.899 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.900 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.900 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169914+0000", 2023-10-09T10:49:23.900 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3380000000000004e-06 2023-10-09T10:49:23.900 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.900 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.900 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.900 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169959+0000", 2023-10-09T10:49:23.900 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5392000000000002e-05 2023-10-09T10:49:23.900 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.900 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.900 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.900 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.170104+0000", 2023-10-09T10:49:23.900 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014468 2023-10-09T10:49:23.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.901 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.901 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:49:23.901 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.170424+0000", 2023-10-09T10:49:23.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00031972199999999999 2023-10-09T10:49:23.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.901 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.901 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.901 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.173247+0000", 2023-10-09T10:49:23.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028231710000000002 2023-10-09T10:49:23.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.901 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.901 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.902 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.186655+0000", 2023-10-09T10:49:23.902 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013408239000000001 2023-10-09T10:49:23.902 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.902 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.209647+0000", 2023-10-09T10:49:23.902 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022992404000000001 2023-10-09T10:49:23.902 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.902 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.209694+0000", 2023-10-09T10:49:23.902 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6822e-05 2023-10-09T10:49:23.903 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.903 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.903 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.903 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.209740+0000", 2023-10-09T10:49:23.903 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5361999999999997e-05 2023-10-09T10:49:23.903 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.903 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.903 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.903 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.903 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.903 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:192655 6.13 6:cee47a1c:::benchmark_data_smithi125_244951_object86212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-10-09T10:49:23.903 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:49:23.904 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 49.981733384000002, 2023-10-09T10:49:23.904 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039152566, 2023-10-09T10:49:23.904 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.904 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.904 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.904 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.904 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.904 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 192655 2023-10-09T10:49:23.904 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.904 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.904 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.904 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:49:23.905 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.905 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.905 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.905 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:49:23.905 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.905 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.905 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.905 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:49:23.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:23.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.906 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.906 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.906 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803498+0000", 2023-10-09T10:49:23.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4190000000000004e-06 2023-10-09T10:49:23.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.906 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.906 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.906 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803509+0000", 2023-10-09T10:49:23.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0801e-05 2023-10-09T10:49:23.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.907 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.907 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803515+0000", 2023-10-09T10:49:23.907 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.767e-06 2023-10-09T10:49:23.907 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.907 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.907 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.809697+0000", 2023-10-09T10:49:23.907 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061824799999999997 2023-10-09T10:49:23.907 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.907 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.908 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.809808+0000", 2023-10-09T10:49:23.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000110271 2023-10-09T10:49:23.908 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.908 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.908 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:49:23.908 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.809938+0000", 2023-10-09T10:49:23.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013000800000000001 2023-10-09T10:49:23.908 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.908 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.908 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.908 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.815553+0000", 2023-10-09T10:49:23.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0056156879999999998 2023-10-09T10:49:23.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.909 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.909 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.815567+0000", 2023-10-09T10:49:23.909 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4102e-05 2023-10-09T10:49:23.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.909 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.909 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.841982+0000", 2023-10-09T10:49:23.909 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026414812999999999 2023-10-09T10:49:23.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.910 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.910 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.842042+0000", 2023-10-09T10:49:23.910 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0198999999999997e-05 2023-10-09T10:49:23.910 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.910 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.910 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.842647+0000", 2023-10-09T10:49:23.910 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00060467599999999995 2023-10-09T10:49:23.910 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.910 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.910 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.911 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.911 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:193229 6.3 6:c7d3e5b6:::benchmark_data_smithi125_244951_object86786:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-10-09T10:49:23.911 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:34.764784+0000", 2023-10-09T10:49:23.911 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 49.020444024, 2023-10-09T10:49:23.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038137404999999999, 2023-10-09T10:49:23.911 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.911 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.911 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.911 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.911 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.911 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 193229 2023-10-09T10:49:23.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.912 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764784+0000", 2023-10-09T10:49:23.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764784+0000", 2023-10-09T10:49:23.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.913 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764782+0000", 2023-10-09T10:49:23.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:23.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.913 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764789+0000", 2023-10-09T10:49:23.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8020000000000003e-06 2023-10-09T10:49:23.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.914 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.914 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764800+0000", 2023-10-09T10:49:23.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.096e-05 2023-10-09T10:49:23.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.914 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.914 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.914 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764805+0000", 2023-10-09T10:49:23.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0969999999999996e-06 2023-10-09T10:49:23.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.914 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.914 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.915 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.767644+0000", 2023-10-09T10:49:23.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028391860000000001 2023-10-09T10:49:23.915 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.915 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.767769+0000", 2023-10-09T10:49:23.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012499599999999999 2023-10-09T10:49:23.915 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:49:23.915 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.767893+0000", 2023-10-09T10:49:23.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000123915 2023-10-09T10:49:23.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.916 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.773147+0000", 2023-10-09T10:49:23.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0052540110000000003 2023-10-09T10:49:23.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.916 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.778846+0000", 2023-10-09T10:49:23.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0056984519999999997 2023-10-09T10:49:23.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.916 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.917 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.802818+0000", 2023-10-09T10:49:23.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023972543999999998 2023-10-09T10:49:23.917 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.917 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.917 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.802876+0000", 2023-10-09T10:49:23.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7275000000000003e-05 2023-10-09T10:49:23.917 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.917 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.917 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.802921+0000", 2023-10-09T10:49:23.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5574000000000001e-05 2023-10-09T10:49:23.918 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.918 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.918 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.918 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.918 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.918 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:209514 6.13 6:c9bc4b21:::benchmark_data_smithi125_244951_object103071:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-10-09T10:49:23.918 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:58.156931+0000", 2023-10-09T10:49:23.918 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 25.628296826, 2023-10-09T10:49:23.918 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.037088755000000001, 2023-10-09T10:49:23.918 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.918 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.918 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.919 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.919 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.919 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 209514 2023-10-09T10:49:23.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.919 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.919 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.919 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156931+0000", 2023-10-09T10:49:23.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.919 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156931+0000", 2023-10-09T10:49:23.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156930+0000", 2023-10-09T10:49:23.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:23.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156935+0000", 2023-10-09T10:49:23.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5350000000000004e-06 2023-10-09T10:49:23.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156948+0000", 2023-10-09T10:49:23.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2842e-05 2023-10-09T10:49:23.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156954+0000", 2023-10-09T10:49:23.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.733e-06 2023-10-09T10:49:23.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.158522+0000", 2023-10-09T10:49:23.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015687570000000001 2023-10-09T10:49:23.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.158683+0000", 2023-10-09T10:49:23.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016043599999999999 2023-10-09T10:49:23.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:49:23.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.158851+0000", 2023-10-09T10:49:23.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016812899999999999 2023-10-09T10:49:23.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.167706+0000", 2023-10-09T10:49:23.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0088551359999999996 2023-10-09T10:49:23.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.167780+0000", 2023-10-09T10:49:23.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4383999999999998e-05 2023-10-09T10:49:23.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.924 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.193627+0000", 2023-10-09T10:49:23.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025846593000000001 2023-10-09T10:49:23.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.924 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.193689+0000", 2023-10-09T10:49:23.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1681999999999997e-05 2023-10-09T10:49:23.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.925 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.194020+0000", 2023-10-09T10:49:23.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00033098999999999997 2023-10-09T10:49:23.925 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.925 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.925 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.925 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:212706 6.13 6:c8228b1c:::benchmark_data_smithi125_244951_object106263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-10-09T10:49:23.925 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:02.392352+0000", 2023-10-09T10:49:23.925 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 21.392875903, 2023-10-09T10:49:23.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.041188625999999999, 2023-10-09T10:49:23.926 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:49:23.926 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:23.926 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:49:23.926 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:49:23.926 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:23.926 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212706 2023-10-09T10:49:23.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.926 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:49:23.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:49:23.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392352+0000", 2023-10-09T10:49:23.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:49:23.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392352+0000", 2023-10-09T10:49:23.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:49:23.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:49:23.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392350+0000", 2023-10-09T10:49:23.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:23.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:49:23.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392357+0000", 2023-10-09T10:49:23.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0380000000000002e-06 2023-10-09T10:49:23.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:49:23.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392369+0000", 2023-10-09T10:49:23.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1857e-05 2023-10-09T10:49:23.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:49:23.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392375+0000", 2023-10-09T10:49:23.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5219999999999998e-06 2023-10-09T10:49:23.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:49:23.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392421+0000", 2023-10-09T10:49:23.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5736999999999999e-05 2023-10-09T10:49:23.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:49:23.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392554+0000", 2023-10-09T10:49:23.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013335300000000001 2023-10-09T10:49:23.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:49:23.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392651+0000", 2023-10-09T10:49:23.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7156000000000003e-05 2023-10-09T10:49:23.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.397156+0000", 2023-10-09T10:49:23.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045045550000000004 2023-10-09T10:49:23.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:49:23.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.400165+0000", 2023-10-09T10:49:23.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030094319999999998 2023-10-09T10:49:23.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:23.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.433364+0000", 2023-10-09T10:49:23.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.033199472000000001 2023-10-09T10:49:23.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:49:23.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.433409+0000", 2023-10-09T10:49:23.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4070000000000003e-05 2023-10-09T10:49:23.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:49:23.932 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:49:23.932 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:49:23.932 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.433541+0000", 2023-10-09T10:49:23.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013196400000000001 2023-10-09T10:49:23.932 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.932 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.932 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.932 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:49:23.932 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:49:23.932 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:49:23.933 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:23 smithi155 ceph-mon[184854]: from='client.35199 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:49:23.933 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:23 smithi155 ceph-mon[184854]: pgmap v445: 109 pgs: 109 active+clean; 134 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 2.0 KiB/s rd, 21 MiB/s wr, 337 op/s; 95 KiB/s, 0 objects/s recovering 2023-10-09T10:49:23.977 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 3 16 8093 8077 168.221 178.375 0.00581084 0.00593698 2023-10-09T10:49:24.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:23 smithi125 ceph-mon[195507]: from='client.35199 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:49:24.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:23 smithi125 ceph-mon[195507]: pgmap v445: 109 pgs: 109 active+clean; 134 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 2.0 KiB/s rd, 21 MiB/s wr, 337 op/s; 95 KiB/s, 0 objects/s recovering 2023-10-09T10:49:24.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:23 smithi125 ceph-mon[200197]: from='client.35199 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:49:24.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:23 smithi125 ceph-mon[200197]: pgmap v445: 109 pgs: 109 active+clean; 134 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 2.0 KiB/s rd, 21 MiB/s wr, 337 op/s; 95 KiB/s, 0 objects/s recovering 2023-10-09T10:49:24.224 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:49:24.305 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:49:24.321 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:49:24.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:24.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:49:24.441 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:24.484 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:24.508 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:24.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:24.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:24.613 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:24.654 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:24.977 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 4 16 10680 10664 166.585 161.688 0.0060584 0.00599408 2023-10-09T10:49:25.978 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 5 16 13364 13348 166.788 167.75 0.00356456 0.00598699 2023-10-09T10:49:26.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:26 smithi125 ceph-mon[195507]: pgmap v446: 109 pgs: 109 active+clean; 283 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 1.6 KiB/s rd, 35 MiB/s wr, 566 op/s; 75 KiB/s, 0 objects/s recovering 2023-10-09T10:49:26.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:26 smithi125 ceph-mon[200197]: pgmap v446: 109 pgs: 109 active+clean; 283 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 1.6 KiB/s rd, 35 MiB/s wr, 566 op/s; 75 KiB/s, 0 objects/s recovering 2023-10-09T10:49:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:26 smithi155 ceph-mon[184854]: pgmap v446: 109 pgs: 109 active+clean; 283 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 1.6 KiB/s rd, 35 MiB/s wr, 566 op/s; 75 KiB/s, 0 objects/s recovering 2023-10-09T10:49:26.978 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 6 16 16037 16021 166.832 167.062 0.00522541 0.00598766 2023-10-09T10:49:27.304 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:27.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:27.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:27.344 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":444,"stamp":"2023-10-09T10:49:21.187567+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2967035,"num_objects":61,"num_object_clones":0,"num_object_copies":183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":61,"num_whiteouts":0,"num_read":443,"num_read_kb":830,"num_write":316,"num_write_kb":4594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25,"num_bytes_recovered":1938961,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3473,"ondisk_log_size":3473,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":312,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":1974368,"kb_used_data":237352,"kb_used_omap":44,"kb_used_meta":1736915,"kb_avail":654241696,"statfs":{"total":671965249536,"available":669943496704,"internally_reserved":0,"allocated":243048448,"data_stored":237034928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":1778601377},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2487448,"num_objects":37,"num_object_clones":0,"num_object_copies":111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37,"num_whiteouts":0,"num_read":52,"num_read_kb":97,"num_write":63,"num_write_kb":2625,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":515524,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.371399"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":192,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.836271+0000","last_change":"2023-10-09T10:46:32.250979+0000","last_active":"2023-10-09T10:49:17.836271+0000","last_peered":"2023-10-09T10:49:17.836271+0000","last_clean":"2023-10-09T10:49:17.836271+0000","last_became_active":"2023-10-09T10:46:32.250625+0000","last_became_peered":"2023-10-09T10:46:32.250625+0000","last_unstale":"2023-10-09T10:49:17.836271+0000","last_undegraded":"2023-10-09T10:49:17.836271+0000","last_fullsized":"2023-10-09T10:49:17.836271+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:34.858348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":168,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.826639+0000","last_change":"2023-10-09T10:43:35.490890+0000","last_active":"2023-10-09T10:49:17.826639+0000","last_peered":"2023-10-09T10:49:17.826639+0000","last_clean":"2023-10-09T10:49:17.826639+0000","last_became_active":"2023-10-09T10:43:35.490588+0000","last_became_peered":"2023-10-09T10:43:35.490588+0000","last_unstale":"2023-10-09T10:49:17.826639+0000","last_undegraded":"2023-10-09T10:49:17.826639+0000","last_fullsized":"2023-10-09T10:49:17.826639+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:51.142584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":148,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.715180+0000","last_change":"2023-10-09T10:43:35.491388+0000","last_active":"2023-10-09T10:49:14.715180+0000","last_peered":"2023-10-09T10:49:14.715180+0000","last_clean":"2023-10-09T10:49:14.715180+0000","last_became_active":"2023-10-09T10:43:35.491059+0000","last_became_peered":"2023-10-09T10:43:35.491059+0000","last_unstale":"2023-10-09T10:49:14.715180+0000","last_undegraded":"2023-10-09T10:49:14.715180+0000","last_fullsized":"2023-10-09T10:49:14.715180+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:22.298266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":67,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T10:49:16.827501+0000","last_change":"2023-10-09T10:46:02.640234+0000","last_active":"2023-10-09T10:49:16.827501+0000","last_peered":"2023-10-09T10:49:16.827501+0000","last_clean":"2023-10-09T10:49:16.827501+0000","last_became_active":"2023-10-09T10:46:02.639494+0000","last_became_peered":"2023-10-09T10:46:02.639494+0000","last_unstale":"2023-10-09T10:49:16.827501+0000","last_undegraded":"2023-10-09T10:49:16.827501+0000","last_fullsized":"2023-10-09T10:49:16.827501+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:46.415696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":204,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.826437+0000","last_change":"2023-10-09T10:46:02.635122+0000","last_active":"2023-10-09T10:49:17.826437+0000","last_peered":"2023-10-09T10:49:17.826437+0000","last_clean":"2023-10-09T10:49:17.826437+0000","last_became_active":"2023-10-09T10:46:02.634133+0000","last_became_peered":"2023-10-09T10:46:02.634133+0000","last_unstale":"2023-10-09T10:49:17.826437+0000","last_undegraded":"2023-10-09T10:49:17.826437+0000","last_fullsized":"2023-10-09T10:49:17.826437+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:19.745852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":134,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.716669+0000","last_change":"2023-10-09T10:45:07.619292+0000","last_active":"2023-10-09T10:49:14.716669+0000","last_peered":"2023-10-09T10:49:14.716669+0000","last_clean":"2023-10-09T10:49:14.716669+0000","last_became_active":"2023-10-09T10:45:07.618607+0000","last_became_peered":"2023-10-09T10:45:07.618607+0000","last_unstale":"2023-10-09T10:49:14.716669+0000","last_undegraded":"2023-10-09T10:49:14.716669+0000","last_fullsized":"2023-10-09T10:49:14.716669+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:43.548160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":67,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T10:49:16.827198+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:49:16.827198+0000","last_peered":"2023-10-09T10:49:16.827198+0000","last_clean":"2023-10-09T10:49:16.827198+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:49:16.827198+0000","last_undegraded":"2023-10-09T10:49:16.827198+0000","last_fullsized":"2023-10-09T10:49:16.827198+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":168,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.826454+0000","last_change":"2023-10-09T10:43:35.484250+0000","last_active":"2023-10-09T10:49:17.826454+0000","last_peered":"2023-10-09T10:49:17.826454+0000","last_clean":"2023-10-09T10:49:17.826454+0000","last_became_active":"2023-10-09T10:43:35.483980+0000","last_became_peered":"2023-10-09T10:43:35.483980+0000","last_unstale":"2023-10-09T10:49:17.826454+0000","last_undegraded":"2023-10-09T10:49:17.826454+0000","last_fullsized":"2023-10-09T10:49:17.826454+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:44.722390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":471,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.842096+0000","last_change":"2023-10-09T10:49:17.841479+0000","last_active":"2023-10-09T10:49:17.842096+0000","last_peered":"2023-10-09T10:49:17.842096+0000","last_clean":"2023-10-09T10:49:17.842096+0000","last_became_active":"2023-10-09T10:49:17.841079+0000","last_became_peered":"2023-10-09T10:49:17.841079+0000","last_unstale":"2023-10-09T10:49:17.842096+0000","last_undegraded":"2023-10-09T10:49:17.842096+0000","last_fullsized":"2023-10-09T10:49:17.842096+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":161,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.715937+0000","last_change":"2023-10-09T10:42:38.415778+0000","last_active":"2023-10-09T10:49:14.715937+0000","last_peered":"2023-10-09T10:49:14.715937+0000","last_clean":"2023-10-09T10:49:14.715937+0000","last_became_active":"2023-10-09T10:42:38.415445+0000","last_became_peered":"2023-10-09T10:42:38.415445+0000","last_unstale":"2023-10-09T10:49:14.715937+0000","last_undegraded":"2023-10-09T10:49:14.715937+0000","last_fullsized":"2023-10-09T10:49:14.715937+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:41.026179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":125,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.828100+0000","last_change":"2023-10-09T10:43:35.487546+0000","last_active":"2023-10-09T10:49:17.828100+0000","last_peered":"2023-10-09T10:49:17.828100+0000","last_clean":"2023-10-09T10:49:17.828100+0000","last_became_active":"2023-10-09T10:43:35.487037+0000","last_became_peered":"2023-10-09T10:43:35.487037+0000","last_unstale":"2023-10-09T10:49:17.828100+0000","last_undegraded":"2023-10-09T10:49:17.828100+0000","last_fullsized":"2023-10-09T10:49:17.828100+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:13.931163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":120,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.826561+0000","last_change":"2023-10-09T10:44:33.621433+0000","last_active":"2023-10-09T10:49:17.826561+0000","last_peered":"2023-10-09T10:49:17.826561+0000","last_clean":"2023-10-09T10:49:17.826561+0000","last_became_active":"2023-10-09T10:44:33.621031+0000","last_became_peered":"2023-10-09T10:44:33.621031+0000","last_unstale":"2023-10-09T10:49:17.826561+0000","last_undegraded":"2023-10-09T10:49:17.826561+0000","last_fullsized":"2023-10-09T10:49:17.826561+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:40.033977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":157,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.711932+0000","last_change":"2023-10-09T10:46:32.249921+0000","last_active":"2023-10-09T10:49:14.711932+0000","last_peered":"2023-10-09T10:49:14.711932+0000","last_clean":"2023-10-09T10:49:14.711932+0000","last_became_active":"2023-10-09T10:46:32.249590+0000","last_became_peered":"2023-10-09T10:46:32.249590+0000","last_unstale":"2023-10-09T10:49:14.711932+0000","last_undegraded":"2023-10-09T10:49:14.711932+0000","last_fullsized":"2023-10-09T10:49:14.711932+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:25.169752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":169,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.835757+0000","last_change":"2023-10-09T10:35:56.822450+0000","last_active":"2023-10-09T10:49:17.835757+0000","last_peered":"2023-10-09T10:49:17.835757+0000","last_clean":"2023-10-09T10:49:17.835757+0000","last_became_active":"2023-10-09T10:35:56.822135+0000","last_became_peered":"2023-10-09T10:35:56.822135+0000","last_unstale":"2023-10-09T10:49:17.835757+0000","last_undegraded":"2023-10-09T10:49:17.835757+0000","last_fullsized":"2023-10-09T10:49:17.835757+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:39.319410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":163,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.832777+0000","last_change":"2023-10-09T10:42:38.411348+0000","last_active":"2023-10-09T10:49:17.832777+0000","last_peered":"2023-10-09T10:49:17.832777+0000","last_clean":"2023-10-09T10:49:17.832777+0000","last_became_active":"2023-10-09T10:42:38.411050+0000","last_became_peered":"2023-10-09T10:42:38.411050+0000","last_unstale":"2023-10-09T10:49:17.832777+0000","last_undegraded":"2023-10-09T10:49:17.832777+0000","last_fullsized":"2023-10-09T10:49:17.832777+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":549,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.832759+0000","last_change":"2023-10-09T10:49:17.832759+0000","last_active":"2023-10-09T10:49:17.832759+0000","last_peered":"2023-10-09T10:49:17.832759+0000","last_clean":"2023-10-09T10:49:17.832759+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:49:17.832759+0000","last_undegraded":"2023-10-09T10:49:17.832759+0000","last_fullsized":"2023-10-09T10:49:17.832759+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":181,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.830769+0000","last_change":"2023-10-09T10:46:32.250256+0000","last_active":"2023-10-09T10:49:17.830769+0000","last_peered":"2023-10-09T10:49:17.830769+0000","last_clean":"2023-10-09T10:49:17.830769+0000","last_became_active":"2023-10-09T10:46:32.243312+0000","last_became_peered":"2023-10-09T10:46:32.243312+0000","last_unstale":"2023-10-09T10:49:17.830769+0000","last_undegraded":"2023-10-09T10:49:17.830769+0000","last_fullsized":"2023-10-09T10:49:17.830769+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:21:08.059777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":191,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.828894+0000","last_change":"2023-10-09T10:46:32.237577+0000","last_active":"2023-10-09T10:49:17.828894+0000","last_peered":"2023-10-09T10:49:17.828894+0000","last_clean":"2023-10-09T10:49:17.828894+0000","last_became_active":"2023-10-09T10:46:32.237316+0000","last_became_peered":"2023-10-09T10:46:32.237316+0000","last_unstale":"2023-10-09T10:49:17.828894+0000","last_undegraded":"2023-10-09T10:49:17.828894+0000","last_fullsized":"2023-10-09T10:49:17.828894+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:39.684602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830018+0000","last_change":"2023-10-09T10:46:05.619003+0000","last_active":"2023-10-09T10:49:04.830018+0000","last_peered":"2023-10-09T10:49:04.830018+0000","last_clean":"2023-10-09T10:49:04.830018+0000","last_became_active":"2023-10-09T10:46:05.618652+0000","last_became_peered":"2023-10-09T10:46:05.618652+0000","last_unstale":"2023-10-09T10:49:04.830018+0000","last_undegraded":"2023-10-09T10:49:04.830018+0000","last_fullsized":"2023-10-09T10:49:04.830018+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":67,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T10:49:16.825697+0000","last_change":"2023-10-09T10:46:02.646326+0000","last_active":"2023-10-09T10:49:16.825697+0000","last_peered":"2023-10-09T10:49:16.825697+0000","last_clean":"2023-10-09T10:49:16.825697+0000","last_became_active":"2023-10-09T10:46:02.645949+0000","last_became_peered":"2023-10-09T10:46:02.645949+0000","last_unstale":"2023-10-09T10:49:16.825697+0000","last_undegraded":"2023-10-09T10:49:16.825697+0000","last_fullsized":"2023-10-09T10:49:16.825697+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:39.876352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":160,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.715885+0000","last_change":"2023-10-09T10:43:35.486785+0000","last_active":"2023-10-09T10:49:14.715885+0000","last_peered":"2023-10-09T10:49:14.715885+0000","last_clean":"2023-10-09T10:49:14.715885+0000","last_became_active":"2023-10-09T10:43:35.486308+0000","last_became_peered":"2023-10-09T10:43:35.486308+0000","last_unstale":"2023-10-09T10:49:14.715885+0000","last_undegraded":"2023-10-09T10:49:14.715885+0000","last_fullsized":"2023-10-09T10:49:14.715885+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:58.876363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830371+0000","last_change":"2023-10-09T10:46:05.620129+0000","last_active":"2023-10-09T10:49:04.830371+0000","last_peered":"2023-10-09T10:49:04.830371+0000","last_clean":"2023-10-09T10:49:04.830371+0000","last_became_active":"2023-10-09T10:46:05.619788+0000","last_became_peered":"2023-10-09T10:46:05.619788+0000","last_unstale":"2023-10-09T10:49:04.830371+0000","last_undegraded":"2023-10-09T10:49:04.830371+0000","last_fullsized":"2023-10-09T10:49:04.830371+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":109,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.834322+0000","last_change":"2023-10-09T10:44:33.619505+0000","last_active":"2023-10-09T10:49:17.834322+0000","last_peered":"2023-10-09T10:49:17.834322+0000","last_clean":"2023-10-09T10:49:17.834322+0000","last_became_active":"2023-10-09T10:44:33.619147+0000","last_became_peered":"2023-10-09T10:44:33.619147+0000","last_unstale":"2023-10-09T10:49:17.834322+0000","last_undegraded":"2023-10-09T10:49:17.834322+0000","last_fullsized":"2023-10-09T10:49:17.834322+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:36.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":67,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T10:49:16.826117+0000","last_change":"2023-10-09T10:46:02.641981+0000","last_active":"2023-10-09T10:49:16.826117+0000","last_peered":"2023-10-09T10:49:16.826117+0000","last_clean":"2023-10-09T10:49:16.826117+0000","last_became_active":"2023-10-09T10:46:02.639904+0000","last_became_peered":"2023-10-09T10:46:02.639904+0000","last_unstale":"2023-10-09T10:49:16.826117+0000","last_undegraded":"2023-10-09T10:49:16.826117+0000","last_fullsized":"2023-10-09T10:49:16.826117+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:02.608155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.829688+0000","last_change":"2023-10-09T10:46:05.618525+0000","last_active":"2023-10-09T10:49:04.829688+0000","last_peered":"2023-10-09T10:49:04.829688+0000","last_clean":"2023-10-09T10:49:04.829688+0000","last_became_active":"2023-10-09T10:46:05.618018+0000","last_became_peered":"2023-10-09T10:46:05.618018+0000","last_unstale":"2023-10-09T10:49:04.829688+0000","last_undegraded":"2023-10-09T10:49:04.829688+0000","last_fullsized":"2023-10-09T10:49:04.829688+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":77,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T10:49:16.830623+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:49:16.830623+0000","last_peered":"2023-10-09T10:49:16.830623+0000","last_clean":"2023-10-09T10:49:16.830623+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:49:16.830623+0000","last_undegraded":"2023-10-09T10:49:16.830623+0000","last_fullsized":"2023-10-09T10:49:16.830623+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":168,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.827340+0000","last_change":"2023-10-09T10:43:35.487647+0000","last_active":"2023-10-09T10:49:17.827340+0000","last_peered":"2023-10-09T10:49:17.827340+0000","last_clean":"2023-10-09T10:49:17.827340+0000","last_became_active":"2023-10-09T10:43:35.486700+0000","last_became_peered":"2023-10-09T10:43:35.486700+0000","last_unstale":"2023-10-09T10:49:17.827340+0000","last_undegraded":"2023-10-09T10:49:17.827340+0000","last_fullsized":"2023-10-09T10:49:17.827340+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:37.983465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":1,"reported_epoch":196,"state":"creating+peering","last_fresh":"2023-10-09T10:49:15.821270+0000","last_change":"2023-10-09T10:49:15.821270+0000","last_active":"2023-10-09T10:49:15.811806+0000","last_peered":"2023-10-09T10:49:15.811806+0000","last_clean":"2023-10-09T10:49:15.811806+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T10:49:15.821270+0000","last_undegraded":"2023-10-09T10:49:15.821270+0000","last_fullsized":"2023-10-09T10:49:15.821270+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:46.339692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":471,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.842082+0000","last_change":"2023-10-09T10:49:17.841457+0000","last_active":"2023-10-09T10:49:17.842082+0000","last_peered":"2023-10-09T10:49:17.842082+0000","last_clean":"2023-10-09T10:49:17.842082+0000","last_became_active":"2023-10-09T10:49:17.841080+0000","last_became_peered":"2023-10-09T10:49:17.841080+0000","last_unstale":"2023-10-09T10:49:17.842082+0000","last_undegraded":"2023-10-09T10:49:17.842082+0000","last_fullsized":"2023-10-09T10:49:17.842082+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":192,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.830791+0000","last_change":"2023-10-09T10:46:02.651865+0000","last_active":"2023-10-09T10:49:17.830791+0000","last_peered":"2023-10-09T10:49:17.830791+0000","last_clean":"2023-10-09T10:49:17.830791+0000","last_became_active":"2023-10-09T10:46:02.629001+0000","last_became_peered":"2023-10-09T10:46:02.629001+0000","last_unstale":"2023-10-09T10:49:17.830791+0000","last_undegraded":"2023-10-09T10:49:17.830791+0000","last_fullsized":"2023-10-09T10:49:17.830791+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:49.411995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":143,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.716141+0000","last_change":"2023-10-09T10:35:58.711427+0000","last_active":"2023-10-09T10:49:14.716141+0000","last_peered":"2023-10-09T10:49:14.716141+0000","last_clean":"2023-10-09T10:49:14.716141+0000","last_became_active":"2023-10-09T10:35:58.711138+0000","last_became_peered":"2023-10-09T10:35:58.711138+0000","last_unstale":"2023-10-09T10:49:14.716141+0000","last_undegraded":"2023-10-09T10:49:14.716141+0000","last_fullsized":"2023-10-09T10:49:14.716141+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:04.891629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":12,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T10:49:19.442391+0000","last_change":"2023-10-09T10:49:16.836911+0000","last_active":"2023-10-09T10:49:19.442391+0000","last_peered":"2023-10-09T10:49:19.442391+0000","last_clean":"2023-10-09T10:49:19.442391+0000","last_became_active":"2023-10-09T10:49:16.836621+0000","last_became_peered":"2023-10-09T10:49:16.836621+0000","last_unstale":"2023-10-09T10:49:19.442391+0000","last_undegraded":"2023-10-09T10:49:19.442391+0000","last_fullsized":"2023-10-09T10:49:19.442391+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:04:45.956890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":522,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.845452+0000","last_change":"2023-10-09T10:49:17.845022+0000","last_active":"2023-10-09T10:49:17.845452+0000","last_peered":"2023-10-09T10:49:17.845452+0000","last_clean":"2023-10-09T10:49:17.845452+0000","last_became_active":"2023-10-09T10:49:17.844627+0000","last_became_peered":"2023-10-09T10:49:17.844627+0000","last_unstale":"2023-10-09T10:49:17.845452+0000","last_undegraded":"2023-10-09T10:49:17.845452+0000","last_fullsized":"2023-10-09T10:49:17.845452+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":381350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":165,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.830421+0000","last_change":"2023-10-09T10:42:38.417142+0000","last_active":"2023-10-09T10:49:17.830421+0000","last_peered":"2023-10-09T10:49:17.830421+0000","last_clean":"2023-10-09T10:49:17.830421+0000","last_became_active":"2023-10-09T10:42:38.416738+0000","last_became_peered":"2023-10-09T10:42:38.416738+0000","last_unstale":"2023-10-09T10:49:17.830421+0000","last_undegraded":"2023-10-09T10:49:17.830421+0000","last_fullsized":"2023-10-09T10:49:17.830421+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:36.033443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":200,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.833035+0000","last_change":"2023-10-09T10:46:32.251035+0000","last_active":"2023-10-09T10:49:17.833035+0000","last_peered":"2023-10-09T10:49:17.833035+0000","last_clean":"2023-10-09T10:49:17.833035+0000","last_became_active":"2023-10-09T10:46:32.250293+0000","last_became_peered":"2023-10-09T10:46:32.250293+0000","last_unstale":"2023-10-09T10:49:17.833035+0000","last_undegraded":"2023-10-09T10:49:17.833035+0000","last_fullsized":"2023-10-09T10:49:17.833035+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.787766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":11,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.828477+0000","last_change":"2023-10-09T10:49:16.844259+0000","last_active":"2023-10-09T10:49:17.828477+0000","last_peered":"2023-10-09T10:49:17.828477+0000","last_clean":"2023-10-09T10:49:17.828477+0000","last_became_active":"2023-10-09T10:49:16.843930+0000","last_became_peered":"2023-10-09T10:49:16.843930+0000","last_unstale":"2023-10-09T10:49:17.828477+0000","last_undegraded":"2023-10-09T10:49:17.828477+0000","last_fullsized":"2023-10-09T10:49:17.828477+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:55:15.698424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830273+0000","last_change":"2023-10-09T10:46:05.617470+0000","last_active":"2023-10-09T10:49:04.830273+0000","last_peered":"2023-10-09T10:49:04.830273+0000","last_clean":"2023-10-09T10:49:04.830273+0000","last_became_active":"2023-10-09T10:46:05.617124+0000","last_became_peered":"2023-10-09T10:46:05.617124+0000","last_unstale":"2023-10-09T10:49:04.830273+0000","last_undegraded":"2023-10-09T10:49:04.830273+0000","last_fullsized":"2023-10-09T10:49:04.830273+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":117,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.715797+0000","last_change":"2023-10-09T10:44:33.624970+0000","last_active":"2023-10-09T10:49:14.715797+0000","last_peered":"2023-10-09T10:49:14.715797+0000","last_clean":"2023-10-09T10:49:14.715797+0000","last_became_active":"2023-10-09T10:44:33.624626+0000","last_became_peered":"2023-10-09T10:44:33.624626+0000","last_unstale":"2023-10-09T10:49:14.715797+0000","last_undegraded":"2023-10-09T10:49:14.715797+0000","last_fullsized":"2023-10-09T10:49:14.715797+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:19.590017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":134,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.712287+0000","last_change":"2023-10-09T10:45:07.621753+0000","last_active":"2023-10-09T10:49:14.712287+0000","last_peered":"2023-10-09T10:49:14.712287+0000","last_clean":"2023-10-09T10:49:14.712287+0000","last_became_active":"2023-10-09T10:45:07.621410+0000","last_became_peered":"2023-10-09T10:45:07.621410+0000","last_unstale":"2023-10-09T10:49:14.712287+0000","last_undegraded":"2023-10-09T10:49:14.712287+0000","last_fullsized":"2023-10-09T10:49:14.712287+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:14.632587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"199'21","reported_seq":32,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T10:49:21.089590+0000","last_change":"2023-10-09T10:49:16.848252+0000","last_active":"2023-10-09T10:49:21.089590+0000","last_peered":"2023-10-09T10:49:21.089590+0000","last_clean":"2023-10-09T10:49:21.089590+0000","last_became_active":"2023-10-09T10:49:16.847969+0000","last_became_peered":"2023-10-09T10:49:16.847969+0000","last_unstale":"2023-10-09T10:49:21.089590+0000","last_undegraded":"2023-10-09T10:49:21.089590+0000","last_fullsized":"2023-10-09T10:49:21.089590+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":21,"log_dups_size":0,"ondisk_log_size":21,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:21.513105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1376256,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":21,"num_write_kb":1344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":544,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.842798+0000","last_change":"2023-10-09T10:49:17.841890+0000","last_active":"2023-10-09T10:49:17.842798+0000","last_peered":"2023-10-09T10:49:17.842798+0000","last_clean":"2023-10-09T10:49:17.842798+0000","last_became_active":"2023-10-09T10:49:17.841661+0000","last_became_peered":"2023-10-09T10:49:17.841661+0000","last_unstale":"2023-10-09T10:49:17.842798+0000","last_undegraded":"2023-10-09T10:49:17.842798+0000","last_fullsized":"2023-10-09T10:49:17.842798+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":161,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.716758+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:49:14.716758+0000","last_peered":"2023-10-09T10:49:14.716758+0000","last_clean":"2023-10-09T10:49:14.716758+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:49:14.716758+0000","last_undegraded":"2023-10-09T10:49:14.716758+0000","last_fullsized":"2023-10-09T10:49:14.716758+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":67,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T10:49:16.832077+0000","last_change":"2023-10-09T10:46:02.625109+0000","last_active":"2023-10-09T10:49:16.832077+0000","last_peered":"2023-10-09T10:49:16.832077+0000","last_clean":"2023-10-09T10:49:16.832077+0000","last_became_active":"2023-10-09T10:46:02.624814+0000","last_became_peered":"2023-10-09T10:46:02.624814+0000","last_unstale":"2023-10-09T10:49:16.832077+0000","last_undegraded":"2023-10-09T10:49:16.832077+0000","last_fullsized":"2023-10-09T10:49:16.832077+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:43.847868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":11,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.828547+0000","last_change":"2023-10-09T10:49:16.840737+0000","last_active":"2023-10-09T10:49:17.828547+0000","last_peered":"2023-10-09T10:49:17.828547+0000","last_clean":"2023-10-09T10:49:17.828547+0000","last_became_active":"2023-10-09T10:49:16.840589+0000","last_became_peered":"2023-10-09T10:49:16.840589+0000","last_unstale":"2023-10-09T10:49:17.828547+0000","last_undegraded":"2023-10-09T10:49:17.828547+0000","last_fullsized":"2023-10-09T10:49:17.828547+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:22.875755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":533,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.835466+0000","last_change":"2023-10-09T10:49:17.835166+0000","last_active":"2023-10-09T10:49:17.835466+0000","last_peered":"2023-10-09T10:49:17.835466+0000","last_clean":"2023-10-09T10:49:17.835466+0000","last_became_active":"2023-10-09T10:49:17.834795+0000","last_became_peered":"2023-10-09T10:49:17.834795+0000","last_unstale":"2023-10-09T10:49:17.835466+0000","last_undegraded":"2023-10-09T10:49:17.835466+0000","last_fullsized":"2023-10-09T10:49:17.835466+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":192,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.827780+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:49:17.827780+0000","last_peered":"2023-10-09T10:49:17.827780+0000","last_clean":"2023-10-09T10:49:17.827780+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:49:17.827780+0000","last_undegraded":"2023-10-09T10:49:17.827780+0000","last_fullsized":"2023-10-09T10:49:17.827780+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":163,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.835673+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:49:17.835673+0000","last_peered":"2023-10-09T10:49:17.835673+0000","last_clean":"2023-10-09T10:49:17.835673+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:49:17.835673+0000","last_undegraded":"2023-10-09T10:49:17.835673+0000","last_fullsized":"2023-10-09T10:49:17.835673+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2023-10-09T10:49:15.811806+0000","last_change":"2023-10-09T10:49:15.821458+0000","last_active":"2023-10-09T10:49:15.811806+0000","last_peered":"2023-10-09T10:49:15.811806+0000","last_clean":"2023-10-09T10:49:15.811806+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T10:49:15.811806+0000","last_undegraded":"2023-10-09T10:49:15.811806+0000","last_fullsized":"2023-10-09T10:49:15.811806+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:43:51.646394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":508,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830103+0000","last_change":"2023-10-09T10:46:02.635401+0000","last_active":"2023-10-09T10:49:04.830103+0000","last_peered":"2023-10-09T10:49:04.830103+0000","last_clean":"2023-10-09T10:49:04.830103+0000","last_became_active":"2023-10-09T10:46:02.634764+0000","last_became_peered":"2023-10-09T10:46:02.634764+0000","last_unstale":"2023-10-09T10:49:04.830103+0000","last_undegraded":"2023-10-09T10:49:04.830103+0000","last_fullsized":"2023-10-09T10:49:04.830103+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":138,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.836119+0000","last_change":"2023-10-09T10:45:07.622118+0000","last_active":"2023-10-09T10:49:17.836119+0000","last_peered":"2023-10-09T10:49:17.836119+0000","last_clean":"2023-10-09T10:49:17.836119+0000","last_became_active":"2023-10-09T10:45:07.621608+0000","last_became_peered":"2023-10-09T10:45:07.621608+0000","last_unstale":"2023-10-09T10:49:17.836119+0000","last_undegraded":"2023-10-09T10:49:17.836119+0000","last_fullsized":"2023-10-09T10:49:17.836119+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:04.411549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":190,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.835874+0000","last_change":"2023-10-09T10:46:32.245583+0000","last_active":"2023-10-09T10:49:17.835874+0000","last_peered":"2023-10-09T10:49:17.835874+0000","last_clean":"2023-10-09T10:49:17.835874+0000","last_became_active":"2023-10-09T10:46:32.245271+0000","last_became_peered":"2023-10-09T10:46:32.245271+0000","last_unstale":"2023-10-09T10:49:17.835874+0000","last_undegraded":"2023-10-09T10:49:17.835874+0000","last_fullsized":"2023-10-09T10:49:17.835874+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:18.025051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":1,"reported_epoch":196,"state":"creating+peering","last_fresh":"2023-10-09T10:49:15.821301+0000","last_change":"2023-10-09T10:49:15.821301+0000","last_active":"2023-10-09T10:49:15.811806+0000","last_peered":"2023-10-09T10:49:15.811806+0000","last_clean":"2023-10-09T10:49:15.811806+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T10:49:15.821301+0000","last_undegraded":"2023-10-09T10:49:15.821301+0000","last_fullsized":"2023-10-09T10:49:15.821301+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:11.280795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830102+0000","last_change":"2023-10-09T10:46:05.619400+0000","last_active":"2023-10-09T10:49:04.830102+0000","last_peered":"2023-10-09T10:49:04.830102+0000","last_clean":"2023-10-09T10:49:04.830102+0000","last_became_active":"2023-10-09T10:46:05.618703+0000","last_became_peered":"2023-10-09T10:46:05.618703+0000","last_unstale":"2023-10-09T10:49:04.830102+0000","last_undegraded":"2023-10-09T10:49:04.830102+0000","last_fullsized":"2023-10-09T10:49:04.830102+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":174,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.832949+0000","last_change":"2023-10-09T10:46:32.245685+0000","last_active":"2023-10-09T10:49:17.832949+0000","last_peered":"2023-10-09T10:49:17.832949+0000","last_clean":"2023-10-09T10:49:17.832949+0000","last_became_active":"2023-10-09T10:46:32.245314+0000","last_became_peered":"2023-10-09T10:46:32.245314+0000","last_unstale":"2023-10-09T10:49:17.832949+0000","last_undegraded":"2023-10-09T10:49:17.832949+0000","last_fullsized":"2023-10-09T10:49:17.832949+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:51.716860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":163,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.835747+0000","last_change":"2023-10-09T10:42:38.410357+0000","last_active":"2023-10-09T10:49:17.835747+0000","last_peered":"2023-10-09T10:49:17.835747+0000","last_clean":"2023-10-09T10:49:17.835747+0000","last_became_active":"2023-10-09T10:42:38.410106+0000","last_became_peered":"2023-10-09T10:42:38.410106+0000","last_unstale":"2023-10-09T10:49:17.835747+0000","last_undegraded":"2023-10-09T10:49:17.835747+0000","last_fullsized":"2023-10-09T10:49:17.835747+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:43:36.020686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":12,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T10:49:19.442571+0000","last_change":"2023-10-09T10:49:16.846123+0000","last_active":"2023-10-09T10:49:19.442571+0000","last_peered":"2023-10-09T10:49:19.442571+0000","last_clean":"2023-10-09T10:49:19.442571+0000","last_became_active":"2023-10-09T10:49:16.845863+0000","last_became_peered":"2023-10-09T10:49:16.845863+0000","last_unstale":"2023-10-09T10:49:19.442571+0000","last_undegraded":"2023-10-09T10:49:19.442571+0000","last_fullsized":"2023-10-09T10:49:19.442571+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:40.575517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":519,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.829777+0000","last_change":"2023-10-09T10:46:05.618336+0000","last_active":"2023-10-09T10:49:04.829777+0000","last_peered":"2023-10-09T10:49:04.829777+0000","last_clean":"2023-10-09T10:49:04.829777+0000","last_became_active":"2023-10-09T10:46:05.617668+0000","last_became_peered":"2023-10-09T10:46:05.617668+0000","last_unstale":"2023-10-09T10:49:04.829777+0000","last_undegraded":"2023-10-09T10:49:04.829777+0000","last_fullsized":"2023-10-09T10:49:04.829777+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":169,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.826353+0000","last_change":"2023-10-09T10:43:35.486030+0000","last_active":"2023-10-09T10:49:17.826353+0000","last_peered":"2023-10-09T10:49:17.826353+0000","last_clean":"2023-10-09T10:49:17.826353+0000","last_became_active":"2023-10-09T10:43:35.485646+0000","last_became_peered":"2023-10-09T10:43:35.485646+0000","last_unstale":"2023-10-09T10:49:17.826353+0000","last_undegraded":"2023-10-09T10:49:17.826353+0000","last_fullsized":"2023-10-09T10:49:17.826353+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:03.494269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":148,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.712538+0000","last_change":"2023-10-09T10:43:35.489126+0000","last_active":"2023-10-09T10:49:14.712538+0000","last_peered":"2023-10-09T10:49:14.712538+0000","last_clean":"2023-10-09T10:49:14.712538+0000","last_became_active":"2023-10-09T10:43:35.488832+0000","last_became_peered":"2023-10-09T10:43:35.488832+0000","last_unstale":"2023-10-09T10:49:14.712538+0000","last_undegraded":"2023-10-09T10:49:14.712538+0000","last_fullsized":"2023-10-09T10:49:14.712538+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:53.462382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"199'16","reported_seq":27,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T10:49:21.097249+0000","last_change":"2023-10-09T10:49:16.848544+0000","last_active":"2023-10-09T10:49:21.097249+0000","last_peered":"2023-10-09T10:49:21.097249+0000","last_clean":"2023-10-09T10:49:21.097249+0000","last_became_active":"2023-10-09T10:49:16.848251+0000","last_became_peered":"2023-10-09T10:49:16.848251+0000","last_unstale":"2023-10-09T10:49:21.097249+0000","last_undegraded":"2023-10-09T10:49:21.097249+0000","last_fullsized":"2023-10-09T10:49:21.097249+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:01.631428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1048576,"num_objects":16,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16,"num_write_kb":1024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":542,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.844421+0000","last_change":"2023-10-09T10:49:17.844151+0000","last_active":"2023-10-09T10:49:17.844421+0000","last_peered":"2023-10-09T10:49:17.844421+0000","last_clean":"2023-10-09T10:49:17.844421+0000","last_became_active":"2023-10-09T10:49:17.843856+0000","last_became_peered":"2023-10-09T10:49:17.843856+0000","last_unstale":"2023-10-09T10:49:17.844421+0000","last_undegraded":"2023-10-09T10:49:17.844421+0000","last_fullsized":"2023-10-09T10:49:17.844421+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":192,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.712247+0000","last_change":"2023-10-09T10:46:02.626658+0000","last_active":"2023-10-09T10:49:14.712247+0000","last_peered":"2023-10-09T10:49:14.712247+0000","last_clean":"2023-10-09T10:49:14.712247+0000","last_became_active":"2023-10-09T10:46:02.626358+0000","last_became_peered":"2023-10-09T10:46:02.626358+0000","last_unstale":"2023-10-09T10:49:14.712247+0000","last_undegraded":"2023-10-09T10:49:14.712247+0000","last_fullsized":"2023-10-09T10:49:14.712247+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:50:29.009406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":188,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.715966+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:49:14.715966+0000","last_peered":"2023-10-09T10:49:14.715966+0000","last_clean":"2023-10-09T10:49:14.715966+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:49:14.715966+0000","last_undegraded":"2023-10-09T10:49:14.715966+0000","last_fullsized":"2023-10-09T10:49:14.715966+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":134,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.712069+0000","last_change":"2023-10-09T10:42:38.407353+0000","last_active":"2023-10-09T10:49:14.712069+0000","last_peered":"2023-10-09T10:49:14.712069+0000","last_clean":"2023-10-09T10:49:14.712069+0000","last_became_active":"2023-10-09T10:42:38.406883+0000","last_became_peered":"2023-10-09T10:42:38.406883+0000","last_unstale":"2023-10-09T10:49:14.712069+0000","last_undegraded":"2023-10-09T10:49:14.712069+0000","last_fullsized":"2023-10-09T10:49:14.712069+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.265550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":11,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.828649+0000","last_change":"2023-10-09T10:49:16.843057+0000","last_active":"2023-10-09T10:49:17.828649+0000","last_peered":"2023-10-09T10:49:17.828649+0000","last_clean":"2023-10-09T10:49:17.828649+0000","last_became_active":"2023-10-09T10:49:16.842812+0000","last_became_peered":"2023-10-09T10:49:16.842812+0000","last_unstale":"2023-10-09T10:49:17.828649+0000","last_undegraded":"2023-10-09T10:49:17.828649+0000","last_fullsized":"2023-10-09T10:49:17.828649+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:37.333788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":527,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830905+0000","last_change":"2023-10-09T10:46:05.617980+0000","last_active":"2023-10-09T10:49:04.830905+0000","last_peered":"2023-10-09T10:49:04.830905+0000","last_clean":"2023-10-09T10:49:04.830905+0000","last_became_active":"2023-10-09T10:46:05.615591+0000","last_became_peered":"2023-10-09T10:46:05.615591+0000","last_unstale":"2023-10-09T10:49:04.830905+0000","last_undegraded":"2023-10-09T10:49:04.830905+0000","last_fullsized":"2023-10-09T10:49:04.830905+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":165,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.716579+0000","last_change":"2023-10-09T10:42:38.410684+0000","last_active":"2023-10-09T10:49:14.716579+0000","last_peered":"2023-10-09T10:49:14.716579+0000","last_clean":"2023-10-09T10:49:14.716579+0000","last_became_active":"2023-10-09T10:42:38.410401+0000","last_became_peered":"2023-10-09T10:42:38.410401+0000","last_unstale":"2023-10-09T10:49:14.716579+0000","last_undegraded":"2023-10-09T10:49:14.716579+0000","last_fullsized":"2023-10-09T10:49:14.716579+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:09.044320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":184,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.832933+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:49:17.832933+0000","last_peered":"2023-10-09T10:49:17.832933+0000","last_clean":"2023-10-09T10:49:17.832933+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:49:17.832933+0000","last_undegraded":"2023-10-09T10:49:17.832933+0000","last_fullsized":"2023-10-09T10:49:17.832933+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":173,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.712324+0000","last_change":"2023-10-09T10:46:32.245608+0000","last_active":"2023-10-09T10:49:14.712324+0000","last_peered":"2023-10-09T10:49:14.712324+0000","last_clean":"2023-10-09T10:49:14.712324+0000","last_became_active":"2023-10-09T10:46:32.245290+0000","last_became_peered":"2023-10-09T10:46:32.245290+0000","last_unstale":"2023-10-09T10:49:14.712324+0000","last_undegraded":"2023-10-09T10:49:14.712324+0000","last_fullsized":"2023-10-09T10:49:14.712324+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:48.832515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":11,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.828844+0000","last_change":"2023-10-09T10:49:16.844018+0000","last_active":"2023-10-09T10:49:17.828844+0000","last_peered":"2023-10-09T10:49:17.828844+0000","last_clean":"2023-10-09T10:49:17.828844+0000","last_became_active":"2023-10-09T10:49:16.843772+0000","last_became_peered":"2023-10-09T10:49:16.843772+0000","last_unstale":"2023-10-09T10:49:17.828844+0000","last_undegraded":"2023-10-09T10:49:17.828844+0000","last_fullsized":"2023-10-09T10:49:17.828844+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:00.175366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":550,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.839835+0000","last_change":"2023-10-09T10:49:17.839835+0000","last_active":"2023-10-09T10:49:17.839835+0000","last_peered":"2023-10-09T10:49:17.839835+0000","last_clean":"2023-10-09T10:49:17.839835+0000","last_became_active":"2023-10-09T10:49:17.839583+0000","last_became_peered":"2023-10-09T10:49:17.839583+0000","last_unstale":"2023-10-09T10:49:17.839835+0000","last_undegraded":"2023-10-09T10:49:17.839835+0000","last_fullsized":"2023-10-09T10:49:17.839835+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":197,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.828238+0000","last_change":"2023-10-09T10:46:32.241041+0000","last_active":"2023-10-09T10:49:17.828238+0000","last_peered":"2023-10-09T10:49:17.828238+0000","last_clean":"2023-10-09T10:49:17.828238+0000","last_became_active":"2023-10-09T10:46:32.238683+0000","last_became_peered":"2023-10-09T10:46:32.238683+0000","last_unstale":"2023-10-09T10:49:17.828238+0000","last_undegraded":"2023-10-09T10:49:17.828238+0000","last_fullsized":"2023-10-09T10:49:17.828238+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:25.817713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":158,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.712922+0000","last_change":"2023-10-09T10:46:32.250104+0000","last_active":"2023-10-09T10:49:14.712922+0000","last_peered":"2023-10-09T10:49:14.712922+0000","last_clean":"2023-10-09T10:49:14.712922+0000","last_became_active":"2023-10-09T10:46:32.249805+0000","last_became_peered":"2023-10-09T10:46:32.249805+0000","last_unstale":"2023-10-09T10:49:14.712922+0000","last_undegraded":"2023-10-09T10:49:14.712922+0000","last_fullsized":"2023-10-09T10:49:14.712922+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:07.230672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":187,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.712611+0000","last_change":"2023-10-09T10:46:02.623738+0000","last_active":"2023-10-09T10:49:14.712611+0000","last_peered":"2023-10-09T10:49:14.712611+0000","last_clean":"2023-10-09T10:49:14.712611+0000","last_became_active":"2023-10-09T10:46:02.623472+0000","last_became_peered":"2023-10-09T10:46:02.623472+0000","last_unstale":"2023-10-09T10:49:14.712611+0000","last_undegraded":"2023-10-09T10:49:14.712611+0000","last_fullsized":"2023-10-09T10:49:14.712611+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:41.397530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":11,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.828493+0000","last_change":"2023-10-09T10:49:16.841410+0000","last_active":"2023-10-09T10:49:17.828493+0000","last_peered":"2023-10-09T10:49:17.828493+0000","last_clean":"2023-10-09T10:49:17.828493+0000","last_became_active":"2023-10-09T10:49:16.840948+0000","last_became_peered":"2023-10-09T10:49:16.840948+0000","last_unstale":"2023-10-09T10:49:17.828493+0000","last_undegraded":"2023-10-09T10:49:17.828493+0000","last_fullsized":"2023-10-09T10:49:17.828493+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:11.725862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":175,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.715721+0000","last_change":"2023-10-09T10:46:32.235346+0000","last_active":"2023-10-09T10:49:14.715721+0000","last_peered":"2023-10-09T10:49:14.715721+0000","last_clean":"2023-10-09T10:49:14.715721+0000","last_became_active":"2023-10-09T10:46:32.234906+0000","last_became_peered":"2023-10-09T10:46:32.234906+0000","last_unstale":"2023-10-09T10:49:14.715721+0000","last_undegraded":"2023-10-09T10:49:14.715721+0000","last_fullsized":"2023-10-09T10:49:14.715721+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:07.199251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":545,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T10:49:16.825769+0000","last_change":"2023-10-09T10:49:16.685442+0000","last_active":"2023-10-09T10:49:16.825769+0000","last_peered":"2023-10-09T10:49:16.825769+0000","last_clean":"2023-10-09T10:49:16.825769+0000","last_became_active":"2023-10-09T10:46:05.616687+0000","last_became_peered":"2023-10-09T10:46:05.616687+0000","last_unstale":"2023-10-09T10:49:16.825769+0000","last_undegraded":"2023-10-09T10:49:16.825769+0000","last_fullsized":"2023-10-09T10:49:16.825769+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:42:57.493798+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":127,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.828195+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:49:17.828195+0000","last_peered":"2023-10-09T10:49:17.828195+0000","last_clean":"2023-10-09T10:49:17.828195+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:49:17.828195+0000","last_undegraded":"2023-10-09T10:49:17.828195+0000","last_fullsized":"2023-10-09T10:49:17.828195+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":96,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.835694+0000","last_change":"2023-10-09T10:45:07.621892+0000","last_active":"2023-10-09T10:49:17.835694+0000","last_peered":"2023-10-09T10:49:17.835694+0000","last_clean":"2023-10-09T10:49:17.835694+0000","last_became_active":"2023-10-09T10:45:07.621581+0000","last_became_peered":"2023-10-09T10:45:07.621581+0000","last_unstale":"2023-10-09T10:49:17.835694+0000","last_undegraded":"2023-10-09T10:49:17.835694+0000","last_fullsized":"2023-10-09T10:49:17.835694+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:11.770457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":11,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.829008+0000","last_change":"2023-10-09T10:49:16.846055+0000","last_active":"2023-10-09T10:49:17.829008+0000","last_peered":"2023-10-09T10:49:17.829008+0000","last_clean":"2023-10-09T10:49:17.829008+0000","last_became_active":"2023-10-09T10:49:16.845832+0000","last_became_peered":"2023-10-09T10:49:16.845832+0000","last_unstale":"2023-10-09T10:49:17.829008+0000","last_undegraded":"2023-10-09T10:49:17.829008+0000","last_fullsized":"2023-10-09T10:49:17.829008+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:14:13.300339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":528,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-10-09T10:49:04.830019+0000","last_change":"2023-10-09T10:46:05.619511+0000","last_active":"2023-10-09T10:49:04.830019+0000","last_peered":"2023-10-09T10:49:04.830019+0000","last_clean":"2023-10-09T10:49:04.830019+0000","last_became_active":"2023-10-09T10:46:05.619116+0000","last_became_peered":"2023-10-09T10:46:05.619116+0000","last_unstale":"2023-10-09T10:49:04.830019+0000","last_undegraded":"2023-10-09T10:49:04.830019+0000","last_fullsized":"2023-10-09T10:49:04.830019+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":188,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.713090+0000","last_change":"2023-10-09T10:46:02.628846+0000","last_active":"2023-10-09T10:49:14.713090+0000","last_peered":"2023-10-09T10:49:14.713090+0000","last_clean":"2023-10-09T10:49:14.713090+0000","last_became_active":"2023-10-09T10:46:02.628552+0000","last_became_peered":"2023-10-09T10:46:02.628552+0000","last_unstale":"2023-10-09T10:49:14.713090+0000","last_undegraded":"2023-10-09T10:49:14.713090+0000","last_fullsized":"2023-10-09T10:49:14.713090+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:55.132899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":76,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T10:49:16.832613+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:49:16.832613+0000","last_peered":"2023-10-09T10:49:16.832613+0000","last_clean":"2023-10-09T10:49:16.832613+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:49:16.832613+0000","last_undegraded":"2023-10-09T10:49:16.832613+0000","last_fullsized":"2023-10-09T10:49:16.832613+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":76,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T10:49:16.827336+0000","last_change":"2023-10-09T10:46:32.236202+0000","last_active":"2023-10-09T10:49:16.827336+0000","last_peered":"2023-10-09T10:49:16.827336+0000","last_clean":"2023-10-09T10:49:16.827336+0000","last_became_active":"2023-10-09T10:46:32.235901+0000","last_became_peered":"2023-10-09T10:46:32.235901+0000","last_unstale":"2023-10-09T10:49:16.827336+0000","last_undegraded":"2023-10-09T10:49:16.827336+0000","last_fullsized":"2023-10-09T10:49:16.827336+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:48.260403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2023-10-09T10:49:15.811806+0000","last_change":"2023-10-09T10:49:15.821509+0000","last_active":"2023-10-09T10:49:15.811806+0000","last_peered":"2023-10-09T10:49:15.811806+0000","last_clean":"2023-10-09T10:49:15.811806+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T10:49:15.811806+0000","last_undegraded":"2023-10-09T10:49:15.811806+0000","last_fullsized":"2023-10-09T10:49:15.811806+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:38:41.546326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,3],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":550,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.844984+0000","last_change":"2023-10-09T10:49:17.844984+0000","last_active":"2023-10-09T10:49:17.844984+0000","last_peered":"2023-10-09T10:49:17.844984+0000","last_clean":"2023-10-09T10:49:17.844984+0000","last_became_active":"2023-10-09T10:49:17.844634+0000","last_became_peered":"2023-10-09T10:49:17.844634+0000","last_unstale":"2023-10-09T10:49:17.844984+0000","last_undegraded":"2023-10-09T10:49:17.844984+0000","last_fullsized":"2023-10-09T10:49:17.844984+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":187,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.829605+0000","last_change":"2023-10-09T10:46:02.629522+0000","last_active":"2023-10-09T10:49:17.829605+0000","last_peered":"2023-10-09T10:49:17.829605+0000","last_clean":"2023-10-09T10:49:17.829605+0000","last_became_active":"2023-10-09T10:46:02.628992+0000","last_became_peered":"2023-10-09T10:46:02.628992+0000","last_unstale":"2023-10-09T10:49:17.829605+0000","last_undegraded":"2023-10-09T10:49:17.829605+0000","last_fullsized":"2023-10-09T10:49:17.829605+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:12.667310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":158,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.712435+0000","last_change":"2023-10-09T10:46:32.251450+0000","last_active":"2023-10-09T10:49:14.712435+0000","last_peered":"2023-10-09T10:49:14.712435+0000","last_clean":"2023-10-09T10:49:14.712435+0000","last_became_active":"2023-10-09T10:46:32.250933+0000","last_became_peered":"2023-10-09T10:46:32.250933+0000","last_unstale":"2023-10-09T10:49:14.712435+0000","last_undegraded":"2023-10-09T10:49:14.712435+0000","last_fullsized":"2023-10-09T10:49:14.712435+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:04.116178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":67,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T10:49:16.832029+0000","last_change":"2023-10-09T10:46:02.635319+0000","last_active":"2023-10-09T10:49:16.832029+0000","last_peered":"2023-10-09T10:49:16.832029+0000","last_clean":"2023-10-09T10:49:16.832029+0000","last_became_active":"2023-10-09T10:46:02.631996+0000","last_became_peered":"2023-10-09T10:46:02.631996+0000","last_unstale":"2023-10-09T10:49:16.832029+0000","last_undegraded":"2023-10-09T10:49:16.832029+0000","last_fullsized":"2023-10-09T10:49:16.832029+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.386062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":12,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T10:49:19.441897+0000","last_change":"2023-10-09T10:49:16.845103+0000","last_active":"2023-10-09T10:49:19.441897+0000","last_peered":"2023-10-09T10:49:19.441897+0000","last_clean":"2023-10-09T10:49:19.441897+0000","last_became_active":"2023-10-09T10:49:16.844812+0000","last_became_peered":"2023-10-09T10:49:16.844812+0000","last_unstale":"2023-10-09T10:49:19.441897+0000","last_undegraded":"2023-10-09T10:49:19.441897+0000","last_fullsized":"2023-10-09T10:49:19.441897+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:17:27.590153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":554,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:18.829214+0000","last_change":"2023-10-09T10:49:18.829156+0000","last_active":"2023-10-09T10:49:18.829214+0000","last_peered":"2023-10-09T10:49:18.829214+0000","last_clean":"2023-10-09T10:49:18.829214+0000","last_became_active":"2023-10-09T10:49:17.848031+0000","last_became_peered":"2023-10-09T10:49:17.848031+0000","last_unstale":"2023-10-09T10:49:18.829214+0000","last_undegraded":"2023-10-09T10:49:18.829214+0000","last_fullsized":"2023-10-09T10:49:18.829214+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.829081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.829081+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:25.174844+0000","scrub_duration":0.001277842,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":192,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.827257+0000","last_change":"2023-10-09T10:46:02.660805+0000","last_active":"2023-10-09T10:49:17.827257+0000","last_peered":"2023-10-09T10:49:17.827257+0000","last_clean":"2023-10-09T10:49:17.827257+0000","last_became_active":"2023-10-09T10:46:02.634560+0000","last_became_peered":"2023-10-09T10:46:02.634560+0000","last_unstale":"2023-10-09T10:49:17.827257+0000","last_undegraded":"2023-10-09T10:49:17.827257+0000","last_fullsized":"2023-10-09T10:49:17.827257+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:14.755691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":134,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.715718+0000","last_change":"2023-10-09T10:42:38.411082+0000","last_active":"2023-10-09T10:49:14.715718+0000","last_peered":"2023-10-09T10:49:14.715718+0000","last_clean":"2023-10-09T10:49:14.715718+0000","last_became_active":"2023-10-09T10:42:38.410795+0000","last_became_peered":"2023-10-09T10:42:38.410795+0000","last_unstale":"2023-10-09T10:49:14.715718+0000","last_undegraded":"2023-10-09T10:49:14.715718+0000","last_fullsized":"2023-10-09T10:49:14.715718+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:18.951217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":181,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.826387+0000","last_change":"2023-10-09T10:46:02.634411+0000","last_active":"2023-10-09T10:49:17.826387+0000","last_peered":"2023-10-09T10:49:17.826387+0000","last_clean":"2023-10-09T10:49:17.826387+0000","last_became_active":"2023-10-09T10:46:02.633521+0000","last_became_peered":"2023-10-09T10:46:02.633521+0000","last_unstale":"2023-10-09T10:49:17.826387+0000","last_undegraded":"2023-10-09T10:49:17.826387+0000","last_fullsized":"2023-10-09T10:49:17.826387+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:45.209038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":12,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-10-09T10:49:19.442337+0000","last_change":"2023-10-09T10:49:16.837451+0000","last_active":"2023-10-09T10:49:19.442337+0000","last_peered":"2023-10-09T10:49:19.442337+0000","last_clean":"2023-10-09T10:49:19.442337+0000","last_became_active":"2023-10-09T10:49:16.837132+0000","last_became_peered":"2023-10-09T10:49:16.837132+0000","last_unstale":"2023-10-09T10:49:19.442337+0000","last_undegraded":"2023-10-09T10:49:19.442337+0000","last_fullsized":"2023-10-09T10:49:19.442337+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:48:01.083335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":541,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.835318+0000","last_change":"2023-10-09T10:49:17.835010+0000","last_active":"2023-10-09T10:49:17.835318+0000","last_peered":"2023-10-09T10:49:17.835318+0000","last_clean":"2023-10-09T10:49:17.835318+0000","last_became_active":"2023-10-09T10:49:17.834675+0000","last_became_peered":"2023-10-09T10:49:17.834675+0000","last_unstale":"2023-10-09T10:49:17.835318+0000","last_undegraded":"2023-10-09T10:49:17.835318+0000","last_fullsized":"2023-10-09T10:49:17.835318+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":57,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.827123+0000","last_change":"2023-10-09T10:46:32.245106+0000","last_active":"2023-10-09T10:49:17.827123+0000","last_peered":"2023-10-09T10:49:17.827123+0000","last_clean":"2023-10-09T10:49:17.827123+0000","last_became_active":"2023-10-09T10:46:32.244826+0000","last_became_peered":"2023-10-09T10:46:32.244826+0000","last_unstale":"2023-10-09T10:49:17.827123+0000","last_undegraded":"2023-10-09T10:49:17.827123+0000","last_fullsized":"2023-10-09T10:49:17.827123+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:30.322834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":71,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T10:49:16.827420+0000","last_change":"2023-10-09T10:46:02.663582+0000","last_active":"2023-10-09T10:49:16.827420+0000","last_peered":"2023-10-09T10:49:16.827420+0000","last_clean":"2023-10-09T10:49:16.827420+0000","last_became_active":"2023-10-09T10:46:02.624873+0000","last_became_peered":"2023-10-09T10:46:02.624873+0000","last_unstale":"2023-10-09T10:49:16.827420+0000","last_undegraded":"2023-10-09T10:49:16.827420+0000","last_fullsized":"2023-10-09T10:49:16.827420+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:33:48.447336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":146,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.832860+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:49:17.832860+0000","last_peered":"2023-10-09T10:49:17.832860+0000","last_clean":"2023-10-09T10:49:17.832860+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:49:17.832860+0000","last_undegraded":"2023-10-09T10:49:17.832860+0000","last_fullsized":"2023-10-09T10:49:17.832860+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":67,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T10:49:16.832039+0000","last_change":"2023-10-09T10:46:02.635334+0000","last_active":"2023-10-09T10:49:16.832039+0000","last_peered":"2023-10-09T10:49:16.832039+0000","last_clean":"2023-10-09T10:49:16.832039+0000","last_became_active":"2023-10-09T10:46:02.631976+0000","last_became_peered":"2023-10-09T10:46:02.631976+0000","last_unstale":"2023-10-09T10:49:16.832039+0000","last_undegraded":"2023-10-09T10:49:16.832039+0000","last_fullsized":"2023-10-09T10:49:16.832039+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:40.350181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":128,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.712537+0000","last_change":"2023-10-09T10:46:32.251242+0000","last_active":"2023-10-09T10:49:14.712537+0000","last_peered":"2023-10-09T10:49:14.712537+0000","last_clean":"2023-10-09T10:49:14.712537+0000","last_became_active":"2023-10-09T10:46:32.250534+0000","last_became_peered":"2023-10-09T10:46:32.250534+0000","last_unstale":"2023-10-09T10:49:14.712537+0000","last_undegraded":"2023-10-09T10:49:14.712537+0000","last_fullsized":"2023-10-09T10:49:14.712537+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:33.726041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":163,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.716668+0000","last_change":"2023-10-09T10:42:38.414989+0000","last_active":"2023-10-09T10:49:14.716668+0000","last_peered":"2023-10-09T10:49:14.716668+0000","last_clean":"2023-10-09T10:49:14.716668+0000","last_became_active":"2023-10-09T10:42:38.414660+0000","last_became_peered":"2023-10-09T10:42:38.414660+0000","last_unstale":"2023-10-09T10:49:14.716668+0000","last_undegraded":"2023-10-09T10:49:14.716668+0000","last_fullsized":"2023-10-09T10:49:14.716668+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:30.148282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":180,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.835470+0000","last_change":"2023-10-09T10:43:35.492737+0000","last_active":"2023-10-09T10:49:17.835470+0000","last_peered":"2023-10-09T10:49:17.835470+0000","last_clean":"2023-10-09T10:49:17.835470+0000","last_became_active":"2023-10-09T10:43:35.492393+0000","last_became_peered":"2023-10-09T10:43:35.492393+0000","last_unstale":"2023-10-09T10:49:17.835470+0000","last_undegraded":"2023-10-09T10:49:17.835470+0000","last_fullsized":"2023-10-09T10:49:17.835470+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:36.930579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":146,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-10-09T10:49:14.716051+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:49:14.716051+0000","last_peered":"2023-10-09T10:49:14.716051+0000","last_clean":"2023-10-09T10:49:14.716051+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:49:14.716051+0000","last_undegraded":"2023-10-09T10:49:14.716051+0000","last_fullsized":"2023-10-09T10:49:14.716051+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":47,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.836220+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:49:17.836220+0000","last_peered":"2023-10-09T10:49:17.836220+0000","last_clean":"2023-10-09T10:49:17.836220+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:49:17.836220+0000","last_undegraded":"2023-10-09T10:49:17.836220+0000","last_fullsized":"2023-10-09T10:49:17.836220+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":183,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.827040+0000","last_change":"2023-10-09T10:46:32.244607+0000","last_active":"2023-10-09T10:49:17.827040+0000","last_peered":"2023-10-09T10:49:17.827040+0000","last_clean":"2023-10-09T10:49:17.827040+0000","last_became_active":"2023-10-09T10:46:32.242628+0000","last_became_peered":"2023-10-09T10:46:32.242628+0000","last_unstale":"2023-10-09T10:49:17.827040+0000","last_undegraded":"2023-10-09T10:49:17.827040+0000","last_fullsized":"2023-10-09T10:49:17.827040+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:10.738819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":214,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.828026+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:49:17.828026+0000","last_peered":"2023-10-09T10:49:17.828026+0000","last_clean":"2023-10-09T10:49:17.828026+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:49:17.828026+0000","last_undegraded":"2023-10-09T10:49:17.828026+0000","last_fullsized":"2023-10-09T10:49:17.828026+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":88,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T10:49:16.826640+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:49:16.826640+0000","last_peered":"2023-10-09T10:49:16.826640+0000","last_clean":"2023-10-09T10:49:16.826640+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:49:16.826640+0000","last_undegraded":"2023-10-09T10:49:16.826640+0000","last_fullsized":"2023-10-09T10:49:16.826640+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":163,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T10:49:17.832933+0000","last_change":"2023-10-09T10:42:38.411257+0000","last_active":"2023-10-09T10:49:17.832933+0000","last_peered":"2023-10-09T10:49:17.832933+0000","last_clean":"2023-10-09T10:49:17.832933+0000","last_became_active":"2023-10-09T10:42:38.410926+0000","last_became_peered":"2023-10-09T10:42:38.410926+0000","last_unstale":"2023-10-09T10:49:17.832933+0000","last_undegraded":"2023-10-09T10:49:17.832933+0000","last_fullsized":"2023-10-09T10:49:17.832933+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:12.964122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2424832,"num_objects":37,"num_object_clones":0,"num_object_copies":111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":37,"num_write_kb":2368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21233664,"data_stored":21233664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":37,"ondisk_log_size":37,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1186,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":536699,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":443,"num_read_kb":830,"num_write":218,"num_write_kb":2197,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":1937775,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332273,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303464,"kb_used_data":2080,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93441688,"statfs":{"total":95995035648,"available":95684288512,"internally_reserved":0,"allocated":2129920,"data_stored":1389110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":1.155,"15min":0.76600000000000001},"min":{"1min":0.23499999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.476,"5min":6.4349999999999996,"15min":6.4349999999999996},"last":0.52800000000000002},{"interface":"front","average":{"1min":3.1789999999999998,"5min":1.5029999999999999,"15min":0.92100000000000004},"min":{"1min":0.248,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.332000000000001,"5min":11.332000000000001,"15min":11.332000000000001},"last":0.65500000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":3.4359999999999999,"5min":1.909,"15min":0.997},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":11.367000000000001,"5min":11.826000000000001,"15min":11.826000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.1739999999999999,"5min":1.484,"15min":0.84899999999999998},"min":{"1min":0.26100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":7.556,"5min":11.84,"15min":11.84},"last":0.626}]},{"osd":2,"last update":"Mon Oct 9 10:49:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":1.7370000000000001,"15min":0.92000000000000004},"min":{"1min":0.31,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":11.058,"5min":11.058,"15min":11.058},"last":0.79500000000000004},{"interface":"front","average":{"1min":1.5449999999999999,"5min":1.395,"15min":0.81499999999999995},"min":{"1min":0.28699999999999998,"5min":0.251,"15min":0.251},"max":{"1min":7.2949999999999999,"5min":7.8120000000000003,"15min":7.8120000000000003},"last":0.34399999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":3.903,"5min":1.893,"15min":1.0209999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":11.523,"5min":11.523,"15min":11.523},"last":0.85299999999999998},{"interface":"front","average":{"1min":4.2789999999999999,"5min":1.835,"15min":1.0109999999999999},"min":{"1min":0.35299999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":19.234999999999999,"5min":19.234999999999999,"15min":19.234999999999999},"last":0.51000000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":1.1579999999999999,"15min":0.70499999999999996},"min":{"1min":0.36399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":9.4900000000000002,"5min":9.4900000000000002,"15min":9.4900000000000002},"last":0.56799999999999995},{"interface":"front","average":{"1min":2.327,"5min":1.018,"15min":0.66100000000000003},"min":{"1min":0.35399999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.0999999999999996,"5min":9.0999999999999996,"15min":9.0999999999999996},"last":0.60499999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.0249999999999999,"5min":1.0609999999999999,"15min":0.69599999999999995},"min":{"1min":0.314,"5min":0.247,"15min":0.247},"max":{"1min":12.064,"5min":12.064,"15min":12.064},"last":0.71299999999999997},{"interface":"front","average":{"1min":3.8839999999999999,"5min":1.4770000000000001,"15min":0.85299999999999998},"min":{"1min":0.435,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":11.741,"5min":11.741,"15min":11.741},"last":0.77500000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2999999999999998,"5min":1.3720000000000001,"15min":0.83199999999999996},"min":{"1min":0.33700000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.029,"5min":13.029,"15min":13.029},"last":0.68500000000000005},{"interface":"front","average":{"1min":2.6869999999999998,"5min":1.079,"15min":0.72699999999999998},"min":{"1min":0.41299999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":19.187999999999999,"5min":19.187999999999999,"15min":19.187999999999999},"last":0.75}]}]},{"osd":6,"up_from":44,"seq":188978561204,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303272,"kb_used_data":5984,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93441880,"statfs":{"total":95995035648,"available":95684485120,"internally_reserved":0,"allocated":6127616,"data_stored":5387792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":3.0499999999999998,"5min":1.4330000000000001,"15min":0.88},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.882,"5min":10.882,"15min":10.882},"last":0.63700000000000001},{"interface":"front","average":{"1min":2.4540000000000002,"5min":1.367,"15min":0.86199999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.8710000000000004,"5min":7.8710000000000004,"15min":7.8710000000000004},"last":0.54100000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:48:53 2023","interfaces":[{"interface":"back","average":{"1min":3.2109999999999999,"5min":1.681,"15min":0.95399999999999996},"min":{"1min":0.28999999999999998,"5min":0.245,"15min":0.245},"max":{"1min":15.502000000000001,"5min":16.747,"15min":16.747},"last":0.66000000000000003},{"interface":"front","average":{"1min":3.0779999999999998,"5min":1.79,"15min":0.96399999999999997},"min":{"1min":0.33700000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.137,"5min":16.734000000000002,"15min":16.734000000000002},"last":0.64700000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:48:20 2023","interfaces":[{"interface":"back","average":{"1min":2.798,"5min":1.411,"15min":0.82599999999999996},"min":{"1min":0.29699999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.208,"5min":13.208,"15min":13.208},"last":0.60299999999999998},{"interface":"front","average":{"1min":2.9420000000000002,"5min":1.619,"15min":0.91300000000000003},"min":{"1min":0.309,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.6749999999999998,"5min":12.571,"15min":12.571},"last":0.626}]},{"osd":3,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":1.496,"15min":0.85699999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":7.7649999999999997,"5min":7.7649999999999997,"15min":7.7649999999999997},"last":0.61699999999999999},{"interface":"front","average":{"1min":2.254,"5min":1.7270000000000001,"15min":0.93999999999999995},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":13.179,"5min":13.179,"15min":13.179},"last":0.55300000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:49:14 2023","interfaces":[{"interface":"back","average":{"1min":1.641,"5min":1.2969999999999999,"15min":0.73099999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":6.742,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":0.52000000000000002},{"interface":"front","average":{"1min":2.8639999999999999,"5min":1.363,"15min":0.78000000000000003},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":12.481999999999999,"5min":12.481999999999999,"15min":12.481999999999999},"last":0.57899999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:49:14 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":1.417,"15min":0.79800000000000004},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.343,"5min":10.497,"15min":10.497},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.6709999999999998,"5min":1.381,"15min":0.81399999999999995},"min":{"1min":0.32100000000000001,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":12.456,"5min":12.456,"15min":12.456},"last":0.47999999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:48:26 2023","interfaces":[{"interface":"back","average":{"1min":3.915,"5min":1.369,"15min":0.80600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":23.358000000000001,"5min":23.358000000000001,"15min":23.358000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":2.6840000000000002,"5min":1.25,"15min":0.79700000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34000000000000002},"max":{"1min":12.404,"5min":12.404,"15min":12.404},"last":0.68000000000000005}]}]},{"osd":1,"up_from":85,"seq":365072220242,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":190640,"kb_used_data":46696,"kb_used_omap":9,"kb_used_meta":143926,"kb_avail":93554512,"statfs":{"total":95995035648,"available":95799820288,"internally_reserved":0,"allocated":47816704,"data_stored":46817159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":147381225},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":2.1419999999999999,"15min":0.95599999999999996},"min":{"1min":0.249,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":18.027000000000001,"5min":18.027000000000001,"15min":18.027000000000001},"last":0.29499999999999998},{"interface":"front","average":{"1min":2.6920000000000002,"5min":2.0209999999999999,"15min":0.93899999999999995},"min":{"1min":0.23599999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.6530000000000005,"5min":14.465,"15min":14.465},"last":0.40100000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:49:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":1.829,"15min":0.85499999999999998},"min":{"1min":0.223,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":9.141,"5min":15.356,"15min":15.356},"last":0.27500000000000002},{"interface":"front","average":{"1min":2.8879999999999999,"5min":2.1899999999999999,"15min":0.98599999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":14.186999999999999,"5min":14.234,"15min":14.234},"last":0.35599999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.6829999999999998,"5min":2.169,"15min":1.0369999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":18.274999999999999,"5min":18.274999999999999,"15min":18.274999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.774,"5min":1.915,"15min":0.86799999999999999},"min":{"1min":0.313,"5min":0.251,"15min":0.251},"max":{"1min":8.5999999999999996,"5min":15.468999999999999,"15min":15.468999999999999},"last":0.36699999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":1.5289999999999999,"15min":0.90500000000000003},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.167999999999999,"5min":12.167999999999999,"15min":12.167999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":3.4399999999999999,"5min":1.7989999999999999,"15min":1.0129999999999999},"min":{"1min":0.309,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":14.228999999999999,"5min":14.228999999999999,"15min":14.228999999999999},"last":0.46800000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":1.4359999999999999,"15min":0.89700000000000002},"min":{"1min":0.34399999999999997,"5min":0.253,"15min":0.253},"max":{"1min":9.4809999999999999,"5min":9.7720000000000002,"15min":9.7720000000000002},"last":0.498},{"interface":"front","average":{"1min":3.7909999999999999,"5min":1.9219999999999999,"15min":1.0720000000000001},"min":{"1min":0.318,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":17.085000000000001,"5min":17.085000000000001,"15min":17.085000000000001},"last":0.45600000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":2.8559999999999999,"5min":1.6479999999999999,"15min":0.96999999999999997},"min":{"1min":0.38300000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.9179999999999993,"5min":9.7379999999999995,"15min":9.7379999999999995},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.819,"5min":1.7290000000000001,"15min":1.0069999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":12.231,"5min":12.231,"15min":12.231},"last":0.51400000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:48:49 2023","interfaces":[{"interface":"back","average":{"1min":3.1219999999999999,"5min":1.714,"15min":1.004},"min":{"1min":0.42799999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":12.249000000000001,"5min":12.249000000000001,"15min":12.249000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":3.7250000000000001,"5min":1.907,"15min":1.0660000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":14.273999999999999,"5min":14.273999999999999,"15min":14.273999999999999},"last":0.52400000000000002}]}]},{"osd":0,"up_from":80,"seq":343597383776,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":275612,"kb_used_data":145556,"kb_used_omap":28,"kb_used_meta":130019,"kb_avail":93469540,"statfs":{"total":95995035648,"available":95712808960,"internally_reserved":0,"allocated":149049344,"data_stored":148013360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":133140304},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":3.931,"5min":2.258,"15min":0.998},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":11.869,"5min":25.475000000000001,"15min":25.475000000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":3.702,"5min":2.2610000000000001,"15min":0.997},"min":{"1min":0.31,"5min":0.188,"15min":0.188},"max":{"1min":12.276,"5min":20.832999999999998,"15min":20.832999999999998},"last":0.36899999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":1.4750000000000001,"15min":0.72399999999999998},"min":{"1min":0.249,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.0890000000000004,"5min":8.9420000000000002,"15min":8.9420000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":2.1120000000000001,"5min":1.889,"15min":0.85799999999999998},"min":{"1min":0.27100000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.0329999999999995,"5min":14.391999999999999,"15min":14.391999999999999},"last":0.35099999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:48:45 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.3359999999999999,"15min":1.212},"min":{"1min":0.307,"5min":0.215,"15min":0.215},"max":{"1min":12.307,"5min":14.926,"15min":14.926},"last":0.317},{"interface":"front","average":{"1min":3.5539999999999998,"5min":2.1960000000000002,"15min":1.093},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":11.885999999999999,"5min":11.885999999999999,"15min":11.885999999999999},"last":0.29999999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":2.8370000000000002,"5min":1.671,"15min":0.96999999999999997},"min":{"1min":0.434,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.0920000000000005,"5min":8.0920000000000005,"15min":8.0920000000000005},"last":0.52900000000000003},{"interface":"front","average":{"1min":3.2679999999999998,"5min":1.9370000000000001,"15min":1.0569999999999999},"min":{"1min":0.36099999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.318,"5min":12.318,"15min":12.318},"last":0.66200000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6179999999999999,"5min":1.8400000000000001,"15min":1.0289999999999999},"min":{"1min":0.35899999999999999,"5min":0.318,"15min":0.318},"max":{"1min":12.157,"5min":12.157,"15min":12.157},"last":0.55500000000000005},{"interface":"front","average":{"1min":3.181,"5min":1.677,"15min":0.98999999999999999},"min":{"1min":0.40500000000000003,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":12.781000000000001,"5min":12.781000000000001,"15min":12.781000000000001},"last":0.47599999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":3.4649999999999999,"5min":1.96,"15min":1.073},"min":{"1min":0.33600000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":12.207000000000001,"5min":12.207000000000001,"15min":12.207000000000001},"last":0.63400000000000001},{"interface":"front","average":{"1min":2.1269999999999998,"5min":1.3740000000000001,"15min":0.89100000000000001},"min":{"1min":0.41299999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":9.8889999999999993,"5min":9.8889999999999993,"15min":9.8889999999999993},"last":0.53900000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:48:34 2023","interfaces":[{"interface":"back","average":{"1min":3.6280000000000001,"5min":1.7509999999999999,"15min":1.0169999999999999},"min":{"1min":0.48799999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":12.727,"5min":12.727,"15min":12.727},"last":0.504},{"interface":"front","average":{"1min":2.843,"5min":1.8660000000000001,"15min":1.071},"min":{"1min":0.47599999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.6150000000000002,"5min":14.394,"15min":14.394},"last":0.67100000000000004}]}]},{"osd":2,"up_from":0,"seq":386547056717,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893191,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":290412,"kb_used_data":32932,"kb_used_omap":7,"kb_used_meta":257464,"kb_avail":93454740,"statfs":{"total":95995035648,"available":95697653760,"internally_reserved":0,"allocated":33722368,"data_stored":32686217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":263643752},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3159999999999998,"5min":1.913,"15min":0.85099999999999998},"min":{"1min":0.248,"5min":0.183,"15min":0.183},"max":{"1min":15.255000000000001,"5min":15.255000000000001,"15min":15.255000000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":2.9830000000000001,"5min":1.6539999999999999,"15min":0.82199999999999995},"min":{"1min":0.21099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":8.6470000000000002,"5min":8.6470000000000002,"15min":8.6470000000000002},"last":0.61099999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.6400000000000001,"5min":2.359,"15min":1.006},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":10.821,"5min":14.707000000000001,"15min":14.707000000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":3.4780000000000002,"5min":1.8100000000000001,"15min":0.85199999999999998},"min":{"1min":0.28999999999999998,"5min":0.193,"15min":0.193},"max":{"1min":15.443,"5min":15.443,"15min":15.443},"last":0.46700000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":2.0630000000000002,"15min":0.93200000000000005},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.375,"5min":13.853,"15min":13.853},"last":0.57999999999999996},{"interface":"front","average":{"1min":3.988,"5min":2.2309999999999999,"15min":1.0069999999999999},"min":{"1min":0.28699999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.557,"5min":10.557,"15min":10.557},"last":0.755}]},{"osd":4,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":4.0739999999999998,"5min":1.9219999999999999,"15min":1.022},"min":{"1min":0.33900000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":20.388000000000002,"5min":20.388000000000002,"15min":20.388000000000002},"last":0.433},{"interface":"front","average":{"1min":4.4299999999999997,"5min":2.0609999999999999,"15min":1.0920000000000001},"min":{"1min":0.38700000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":19.085999999999999,"5min":19.085999999999999,"15min":19.085999999999999},"last":0.69899999999999995}]},{"osd":5,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.7599999999999998,"5min":2.4369999999999998,"15min":1.2030000000000001},"min":{"1min":0.35399999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.3550000000000004,"5min":51.509999999999998,"15min":51.509999999999998},"last":0.66700000000000004},{"interface":"front","average":{"1min":2.6240000000000001,"5min":2.7189999999999999,"15min":1.2809999999999999},"min":{"1min":0.316,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.7449999999999992,"5min":55.338999999999999,"15min":55.338999999999999},"last":0.45000000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":1.841,"15min":1.0049999999999999},"min":{"1min":0.43099999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.361000000000001,"5min":12.361000000000001,"15min":12.361000000000001},"last":0.72799999999999998},{"interface":"front","average":{"1min":2.8140000000000001,"5min":2.2189999999999999,"15min":1.139},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.5839999999999996,"5min":13.891999999999999,"15min":13.891999999999999},"last":0.51100000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":3.2269999999999999,"5min":2.8679999999999999,"15min":1.3500000000000001},"min":{"1min":0.34000000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.734,"5min":51.744,"15min":51.744},"last":0.48299999999999998},{"interface":"front","average":{"1min":3.927,"5min":2.54,"15min":1.2290000000000001},"min":{"1min":0.46100000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":15.212999999999999,"5min":53.399000000000001,"15min":53.399000000000001},"last":0.5}]}]},{"osd":4,"up_from":30,"seq":128849019067,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299368,"kb_used_data":2080,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93445784,"statfs":{"total":95995035648,"available":95688482816,"internally_reserved":0,"allocated":2129920,"data_stored":1389363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.9850000000000001,"5min":1.302,"15min":0.80500000000000005},"min":{"1min":0.216,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.7160000000000002,"5min":10.387,"15min":10.387},"last":0.497},{"interface":"front","average":{"1min":2.9780000000000002,"5min":1.6659999999999999,"15min":0.94299999999999995},"min":{"1min":0.48499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.0129999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.60399999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":2.351,"5min":1.512,"15min":0.872},"min":{"1min":0.254,"5min":0.249,"15min":0.249},"max":{"1min":7.5350000000000001,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.56899999999999995},{"interface":"front","average":{"1min":2.931,"5min":1.7010000000000001,"15min":0.98099999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.2919999999999998,"5min":8.7669999999999995,"15min":8.7669999999999995},"last":0.59299999999999997}]},{"osd":2,"last update":"Mon Oct 9 10:49:16 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.7749999999999999,"15min":0.97199999999999998},"min":{"1min":0.27500000000000002,"5min":0.253,"15min":0.253},"max":{"1min":4.2619999999999996,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.1800000000000002,"5min":2.1779999999999999,"15min":1.101},"min":{"1min":0.34799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.9870000000000001,"5min":20.545999999999999,"15min":20.545999999999999},"last":0.58099999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:48:38 2023","interfaces":[{"interface":"back","average":{"1min":3.2549999999999999,"5min":1.8420000000000001,"15min":1.0880000000000001},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.0269999999999992,"5min":9.0269999999999992,"15min":9.0269999999999992},"last":0.55900000000000005},{"interface":"front","average":{"1min":3.9809999999999999,"5min":2.0379999999999998,"15min":1.071},"min":{"1min":0.79400000000000004,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.6029999999999998,"5min":8.9809999999999999,"15min":8.9809999999999999},"last":0.42299999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:48:44 2023","interfaces":[{"interface":"back","average":{"1min":5.0960000000000001,"5min":1.8140000000000001,"15min":0.91000000000000003},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":24.149999999999999,"5min":24.149999999999999,"15min":24.149999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":4.173,"5min":1.6699999999999999,"15min":0.86399999999999999},"min":{"1min":0.373,"5min":0.252,"15min":0.252},"max":{"1min":20.422000000000001,"5min":20.422000000000001,"15min":20.422000000000001},"last":0.36499999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:49:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5819999999999999,"5min":1.544,"15min":0.83699999999999997},"min":{"1min":0.26000000000000001,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":14.897,"5min":14.897,"15min":14.897},"last":0.438},{"interface":"front","average":{"1min":1.9590000000000001,"5min":1.484,"15min":0.876},"min":{"1min":0.32100000000000001,"5min":0.311,"15min":0.311},"max":{"1min":10.329000000000001,"5min":12.539,"15min":12.539},"last":0.44900000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:48:21 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":1.2929999999999999,"15min":0.80900000000000005},"min":{"1min":0.33300000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.7919999999999998,"5min":8.7919999999999998,"15min":8.7919999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":3.2389999999999999,"5min":1.27,"15min":0.79000000000000004},"min":{"1min":0.55900000000000005,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.423,"5min":7.423,"15min":7.423},"last":0.47599999999999998}]}]},{"osd":5,"up_from":37,"seq":158913790135,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311600,"kb_used_data":2024,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433552,"statfs":{"total":95995035648,"available":95675957248,"internally_reserved":0,"allocated":2072576,"data_stored":1351927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:48:41 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":1.2210000000000001,"15min":0.80500000000000005},"min":{"1min":0.28899999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.648999999999999,"5min":11.648999999999999,"15min":11.648999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.8570000000000002,"5min":1.294,"15min":0.875},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.68,"5min":11.68,"15min":11.68},"last":0.55000000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:48:51 2023","interfaces":[{"interface":"back","average":{"1min":2.673,"5min":1.78,"15min":0.998},"min":{"1min":0.41799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":9.6839999999999993,"5min":12.999000000000001,"15min":12.999000000000001},"last":0.441},{"interface":"front","average":{"1min":2.0680000000000001,"5min":1.532,"15min":0.90500000000000003},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.1120000000000001,"5min":7.3730000000000002,"15min":7.3730000000000002},"last":0.42299999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:48:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6669999999999998,"5min":1.4390000000000001,"15min":0.86099999999999999},"min":{"1min":0.34799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.3280000000000003,"5min":7.9000000000000004,"15min":7.9000000000000004},"last":0.65000000000000002},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.2350000000000001,"15min":0.79400000000000004},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.3059999999999992,"5min":10.352,"15min":10.352},"last":0.58099999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:48:47 2023","interfaces":[{"interface":"back","average":{"1min":2.859,"5min":2.048,"15min":1.165},"min":{"1min":0.432,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":12.558999999999999,"5min":18.016999999999999,"15min":18.016999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":2.6930000000000001,"5min":2.089,"15min":1.105},"min":{"1min":0.40300000000000002,"5min":0.317,"15min":0.317},"max":{"1min":7.9560000000000004,"5min":15.555999999999999,"15min":15.555999999999999},"last":0.56899999999999995}]},{"osd":4,"last update":"Mon Oct 9 10:48:51 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":1.0629999999999999,"15min":0.66000000000000003},"min":{"1min":0.26800000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":6.093,"5min":19.152999999999999,"15min":19.152999999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.5369999999999999,"5min":1.375,"15min":0.77800000000000002},"min":{"1min":0.32400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.8610000000000007,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":0.60099999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:49:11 2023","interfaces":[{"interface":"back","average":{"1min":2.04,"5min":1.4359999999999999,"15min":0.81000000000000005},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":6.9100000000000001,"5min":7.7060000000000004,"15min":7.7060000000000004},"last":0.69399999999999995},{"interface":"front","average":{"1min":2.609,"5min":1.3109999999999999,"15min":0.78500000000000003},"min":{"1min":0.34799999999999998,"5min":0.318,"15min":0.30299999999999999},"max":{"1min":20.256,"5min":20.256,"15min":20.256},"last":0.63800000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:48:26 2023","interfaces":[{"interface":"back","average":{"1min":2.79,"5min":1.1160000000000001,"15min":0.74099999999999999},"min":{"1min":0.41899999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.3940000000000001,"5min":7.3940000000000001,"15min":7.3940000000000001},"last":0.72499999999999998},{"interface":"front","average":{"1min":2.2669999999999999,"5min":1.0369999999999999,"15min":0.71999999999999997},"min":{"1min":0.38500000000000001,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":9.0579999999999998,"5min":9.0579999999999998,"15min":9.0579999999999998},"last":0.66900000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":6881280,"data_stored":6881280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":9109504,"data_stored":9109504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":5242880,"data_stored":5242880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:49:27.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:49:27.458 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:27.588 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:49:27.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:27.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:49:27.615 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:27.656 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:27.693 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:49:27.758 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:27.978 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 7 16 18217 18201 162.462 136.25 0.00572401 0.00614888 2023-10-09T10:49:28.344 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:49:28.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:28.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:49:28.450 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:49:28.512 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:28.576 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:28 smithi125 ceph-mon[195507]: pgmap v447: 109 pgs: 109 active+clean; 599 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 1.4 KiB/s rd, 64 MiB/s wr, 1.02k op/s; 64 KiB/s, 0 objects/s recovering 2023-10-09T10:49:28.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:28 smithi125 ceph-mon[200197]: pgmap v447: 109 pgs: 109 active+clean; 599 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 1.4 KiB/s rd, 64 MiB/s wr, 1.02k op/s; 64 KiB/s, 0 objects/s recovering 2023-10-09T10:49:28.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:28 smithi155 ceph-mon[184854]: pgmap v447: 109 pgs: 109 active+clean; 599 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 1.4 KiB/s rd, 64 MiB/s wr, 1.02k op/s; 64 KiB/s, 0 objects/s recovering 2023-10-09T10:49:28.979 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 8 16 20440 20424 159.52 138.938 0.0035327 0.006262 2023-10-09T10:49:29.348 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:49:29.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:29.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:49:29.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:29.488 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:29.514 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:29.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:29.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:29.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:29.672 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:29.809 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:29.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:29.915 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:29.952 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:29.979 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:29.979 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 9 16 22273 22257 154.517 114.562 0.00597097 0.00646585 2023-10-09T10:49:29.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:29.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:30.083 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:30.121 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:30.146 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:30.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:30.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:30.251 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:30.287 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:30.306 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:49:30.311 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:30.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:30.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:30.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:30 smithi125 ceph-mon[200197]: pgmap v448: 109 pgs: 109 active+clean; 904 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 5.4 KiB/s rd, 90 MiB/s wr, 1.45k op/s; 68 KiB/s, 0 objects/s recovering 2023-10-09T10:49:30.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:30 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2832808839' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:49:30.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:30.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:49:30.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:30 smithi125 ceph-mon[195507]: pgmap v448: 109 pgs: 109 active+clean; 904 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 5.4 KiB/s rd, 90 MiB/s wr, 1.45k op/s; 68 KiB/s, 0 objects/s recovering 2023-10-09T10:49:30.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:30 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2832808839' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:49:30.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:30.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:49:30.417 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:30.451 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:30.472 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:30.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:30.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:30.575 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:30.607 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:30 smithi155 ceph-mon[184854]: pgmap v448: 109 pgs: 109 active+clean; 904 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 5.4 KiB/s rd, 90 MiB/s wr, 1.45k op/s; 68 KiB/s, 0 objects/s recovering 2023-10-09T10:49:30.607 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:30 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2832808839' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:49:30.607 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:30.607 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:49:30.610 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:30.633 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:30.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:30.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:30.666 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:49:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:49:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:23406 7.6 7:6ef508d9:::benchmark_data_smithi125_295816_object23405:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2023-10-09T10:49:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:49:30.648070+0000", 2023-10-09T10:49:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.016254866, 2023-10-09T10:49:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.016288540000000001, 2023-10-09T10:49:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:49:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:49:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:49:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 23406 2023-10-09T10:49:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648070+0000", 2023-10-09T10:49:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648070+0000", 2023-10-09T10:49:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648068+0000", 2023-10-09T10:49:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:49:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648094+0000", 2023-10-09T10:49:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6483e-05 2023-10-09T10:49:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648112+0000", 2023-10-09T10:49:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7176e-05 2023-10-09T10:49:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648120+0000", 2023-10-09T10:49:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3639999999999999e-06 2023-10-09T10:49:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648171+0000", 2023-10-09T10:49:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1477999999999997e-05 2023-10-09T10:49:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648317+0000", 2023-10-09T10:49:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014532299999999999 2023-10-09T10:49:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:49:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648461+0000", 2023-10-09T10:49:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014373399999999999 2023-10-09T10:49:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.656277+0000", 2023-10-09T10:49:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.007816524 2023-10-09T10:49:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.657213+0000", 2023-10-09T10:49:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00093575799999999997 2023-10-09T10:49:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:23407 7.6 7:63bff06a:::benchmark_data_smithi125_295816_object23406:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2023-10-09T10:49:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:49:30.648133+0000", 2023-10-09T10:49:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.016191130000000001, 2023-10-09T10:49:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.016318902999999999, 2023-10-09T10:49:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:49:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:49:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:49:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 23407 2023-10-09T10:49:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648133+0000", 2023-10-09T10:49:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648133+0000", 2023-10-09T10:49:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648133+0000", 2023-10-09T10:49:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648156+0000", 2023-10-09T10:49:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3521999999999999e-05 2023-10-09T10:49:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648165+0000", 2023-10-09T10:49:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7250000000000003e-06 2023-10-09T10:49:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648172+0000", 2023-10-09T10:49:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5930000000000002e-06 2023-10-09T10:49:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.650980+0000", 2023-10-09T10:49:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002808016 2023-10-09T10:49:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.651148+0000", 2023-10-09T10:49:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000168104 2023-10-09T10:49:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:49:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.651298+0000", 2023-10-09T10:49:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015020100000000001 2023-10-09T10:49:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.656343+0000", 2023-10-09T10:49:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0050452379999999996 2023-10-09T10:49:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.657278+0000", 2023-10-09T10:49:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00093468700000000002 2023-10-09T10:49:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:23409 7.6 7:653230b9:::benchmark_data_smithi125_295816_object23408:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2023-10-09T10:49:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:49:30.648178+0000", 2023-10-09T10:49:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.016146863000000001, 2023-10-09T10:49:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.016364022999999998, 2023-10-09T10:49:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:49:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:49:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:49:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 23409 2023-10-09T10:49:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648178+0000", 2023-10-09T10:49:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648178+0000", 2023-10-09T10:49:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648177+0000", 2023-10-09T10:49:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648200+0000", 2023-10-09T10:49:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2518999999999999e-05 2023-10-09T10:49:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648235+0000", 2023-10-09T10:49:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4894999999999997e-05 2023-10-09T10:49:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.648241+0000", 2023-10-09T10:49:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4010000000000002e-06 2023-10-09T10:49:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.652295+0000", 2023-10-09T10:49:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040542929999999996 2023-10-09T10:49:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.652412+0000", 2023-10-09T10:49:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011649100000000001 2023-10-09T10:49:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:49:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.652535+0000", 2023-10-09T10:49:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000123168 2023-10-09T10:49:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.656350+0000", 2023-10-09T10:49:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038148420000000001 2023-10-09T10:49:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.657338+0000", 2023-10-09T10:49:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00098789600000000004 2023-10-09T10:49:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:23428 7.7 7:eba6eb23:::benchmark_data_smithi125_295816_object23427:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2023-10-09T10:49:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:49:30.663008+0000", 2023-10-09T10:49:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.001316591, 2023-10-09T10:49:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001624213, 2023-10-09T10:49:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:49:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:49:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:49:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 23428 2023-10-09T10:49:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663008+0000", 2023-10-09T10:49:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663008+0000", 2023-10-09T10:49:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663007+0000", 2023-10-09T10:49:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663032+0000", 2023-10-09T10:49:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4977000000000002e-05 2023-10-09T10:49:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663041+0000", 2023-10-09T10:49:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5189999999999998e-06 2023-10-09T10:49:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663048+0000", 2023-10-09T10:49:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9469999999999997e-06 2023-10-09T10:49:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663095+0000", 2023-10-09T10:49:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6863999999999997e-05 2023-10-09T10:49:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663252+0000", 2023-10-09T10:49:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015733899999999999 2023-10-09T10:49:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:49:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663400+0000", 2023-10-09T10:49:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014731799999999999 2023-10-09T10:49:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:23429 7.9 7:9685b730:::benchmark_data_smithi125_295816_object23428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e200)", 2023-10-09T10:49:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:49:30.663067+0000", 2023-10-09T10:49:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0012577829999999999, 2023-10-09T10:49:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001658315, 2023-10-09T10:49:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:49:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:49:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:49:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 23429 2023-10-09T10:49:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663067+0000", 2023-10-09T10:49:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663067+0000", 2023-10-09T10:49:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663066+0000", 2023-10-09T10:49:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663088+0000", 2023-10-09T10:49:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2189999999999999e-05 2023-10-09T10:49:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663097+0000", 2023-10-09T10:49:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6130000000000004e-06 2023-10-09T10:49:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663103+0000", 2023-10-09T10:49:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0990000000000004e-06 2023-10-09T10:49:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663156+0000", 2023-10-09T10:49:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3235000000000001e-05 2023-10-09T10:49:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663285+0000", 2023-10-09T10:49:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012830299999999999 2023-10-09T10:49:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T10:49:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.663405+0000", 2023-10-09T10:49:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011992700000000001 2023-10-09T10:49:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:30.664691+0000", 2023-10-09T10:49:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012867460000000001 2023-10-09T10:49:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:49:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 5 2023-10-09T10:49:30.696 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:49:30.737 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:49:30.771 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:49:30.797 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:30.813 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:49:30.851 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:49:30.897 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:49:30.928 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:49:30.963 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:49:30.980 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:49:30.980 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 10 16 23951 23935 149.536 104.875 0.0104305 0.00667986 2023-10-09T10:49:30.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:30.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:49:30.994 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:49:31.038 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:49:31.043 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:49:31.043 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:49:31.044 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:49:31.044 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:49:31.044 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:49:31.044 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:49:31.044 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:49:31.044 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:49:31.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:31.127 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:31.152 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:31.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:31.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:31.256 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:31.294 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:31.319 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:31.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:31.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:31.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:31 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:49:31.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2832808839' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:49:31.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:31 smithi125 ceph-mon[200197]: osdmap e200: 8 total, 8 up, 7 in 2023-10-09T10:49:31.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:31 smithi125 ceph-mon[200197]: failed to encode map e200 with expected crc 2023-10-09T10:49:31.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:31 smithi125 ceph-mon[200197]: failed to encode map e200 with expected crc 2023-10-09T10:49:31.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:31 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:31.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3314212030' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:31.346 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:49:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:49:31] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T10:49:31.346 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:31 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:49:31.346 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2832808839' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:49:31.347 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:31 smithi125 ceph-mon[195507]: osdmap e200: 8 total, 8 up, 7 in 2023-10-09T10:49:31.347 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:31 smithi125 ceph-mon[195507]: failed to encode map e200 with expected crc 2023-10-09T10:49:31.347 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:31 smithi125 ceph-mon[195507]: failed to encode map e200 with expected crc 2023-10-09T10:49:31.347 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:31 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:31.347 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3314212030' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:31.380 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:49:31.424 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:31.456 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:31 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:49:31.456 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2832808839' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:49:31.456 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:31 smithi155 ceph-mon[184854]: osdmap e200: 8 total, 8 up, 7 in 2023-10-09T10:49:31.456 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:31 smithi155 ceph-mon[184854]: failed to encode map e200 with expected crc 2023-10-09T10:49:31.456 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:31 smithi155 ceph-mon[184854]: failed to encode map e200 with expected crc 2023-10-09T10:49:31.456 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:31 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:31.456 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3314212030' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:31.458 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:31.485 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:31.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:31.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:31.589 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:31.622 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:31.647 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:31.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:31.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:31.753 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:31.803 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:31.838 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:31.839 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:31.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:31.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:31.946 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:31.981 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:31.981 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 11 16 25804 25788 146.46 115.812 0.00994027 0.00682083 2023-10-09T10:49:32.005 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:32.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:32.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:32.110 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:32.158 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:32.188 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:32.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:32.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:32.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:32.330 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:32.352 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:49:32.352 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:49:32.352 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [1, 3, 4, 5, 6, 7, 0] 2023-10-09T10:49:32.353 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-09T10:49:32.355 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:32.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:32.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:32.464 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:32.522 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:32.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:32 smithi125 ceph-mon[195507]: pgmap v450: 109 pgs: 109 active+clean; 1.1 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s; 9.9 KiB/s, 0 objects/s recovering 2023-10-09T10:49:32.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:32 smithi125 ceph-mon[200197]: pgmap v450: 109 pgs: 109 active+clean; 1.1 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s; 9.9 KiB/s, 0 objects/s recovering 2023-10-09T10:49:32.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:32 smithi155 ceph-mon[184854]: pgmap v450: 109 pgs: 109 active+clean; 1.1 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s; 9.9 KiB/s, 0 objects/s recovering 2023-10-09T10:49:32.981 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 12 16 27626 27610 143.745 113.875 0.00835167 0.006952 2023-10-09T10:49:33.011 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:49:33.631 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:49:33.923 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:33 smithi125 ceph-mon[195507]: pgmap v451: 109 pgs: 109 active+clean; 1.5 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 142 MiB/s wr, 2.28k op/s 2023-10-09T10:49:33.923 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:33 smithi125 ceph-mon[200197]: pgmap v451: 109 pgs: 109 active+clean; 1.5 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 142 MiB/s wr, 2.28k op/s 2023-10-09T10:49:33.947 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:33.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:33.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:33 smithi155 ceph-mon[184854]: pgmap v451: 109 pgs: 109 active+clean; 1.5 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 142 MiB/s wr, 2.28k op/s 2023-10-09T10:49:33.981 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 13 16 29401 29385 141.217 110.938 0.00566314 0.00707325 2023-10-09T10:49:33.999 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:34.055 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:34.088 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:34.114 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:34.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:34.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:34.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:34.259 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:34.264 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:34.843 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:34.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:34.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:34.949 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:34.982 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 14 16 31099 31083 138.711 106.125 0.0202338 0.00720216 2023-10-09T10:49:34.998 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:35.118 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:49:35.118 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:49:35.132 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:35.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:35.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:35.244 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:49:35.324 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:35.451 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:49:35.451 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:49:35.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:49:35.982 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 15 16 32166 32150 133.91 66.6875 0.0060481 0.00746311 2023-10-09T10:49:36.164 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:49:36.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:36.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:49:36.271 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:36.320 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:36.389 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:36 smithi155 ceph-mon[184854]: pgmap v452: 109 pgs: 109 active+clean; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-10-09T10:49:36.437 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:36 smithi125 ceph-mon[195507]: pgmap v452: 109 pgs: 109 active+clean; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-10-09T10:49:36.437 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:36 smithi125 ceph-mon[200197]: pgmap v452: 109 pgs: 109 active+clean; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-10-09T10:49:36.455 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:36.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:36.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:36.559 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:36.597 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:36.611 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:49:36.677 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:36.732 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:36.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:36.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:36.836 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:36.888 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:36.980 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:36.982 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 16 16 33654 33638 131.353 93 0.0106189 0.00760959 2023-10-09T10:49:37.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:37.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:37.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:49:37.158 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:37.201 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:49:37.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:37.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:49:37.305 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:37.354 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:37.381 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:37.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:37.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:37.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:49:37.519 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:37.566 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:37.846 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:49:37.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:37.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:49:37.950 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:37.982 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 17 16 35077 35061 128.856 88.9375 0.00529949 0.00775523 2023-10-09T10:49:38.030 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:38.113 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:38.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:38.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:38.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:38.272 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:38.283 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:49:38.450 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:38 smithi125 ceph-mon[195507]: pgmap v453: 109 pgs: 109 active+clean; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T10:49:38.451 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:38 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2259964871' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:49:38.451 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:38 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:49:38.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:38 smithi125 ceph-mon[200197]: pgmap v453: 109 pgs: 109 active+clean; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T10:49:38.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:38 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2259964871' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:49:38.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:38 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:49:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:38 smithi155 ceph-mon[184854]: pgmap v453: 109 pgs: 109 active+clean; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T10:49:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:38 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2259964871' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:49:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:38 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:49:38.982 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 18 16 36490 36474 126.604 88.3125 0.010829 0.00789274 2023-10-09T10:49:39.260 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:38 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T10:49:38.975+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e201 definitely_dead 0 2023-10-09T10:49:39.260 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:49:39.260 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:49:39.260 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: osdmap e201: 8 total, 8 up, 7 in 2023-10-09T10:49:39.260 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: failed to encode map e201 with expected crc 2023-10-09T10:49:39.260 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: failed to encode map e201 with expected crc 2023-10-09T10:49:39.260 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: failed to encode map e201 with expected crc 2023-10-09T10:49:39.260 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: 1.6 scrub starts 2023-10-09T10:49:39.261 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: 1.6 scrub ok 2023-10-09T10:49:39.261 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: 1.7 scrub starts 2023-10-09T10:49:39.261 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: 1.7 scrub ok 2023-10-09T10:49:39.261 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: 1.1 scrub starts 2023-10-09T10:49:39.261 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: 1.1 scrub ok 2023-10-09T10:49:39.261 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:39.261 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:39.261 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2009194918' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:49:39.261 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:49:39.261 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[195507]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T10:49:39.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:49:39.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:49:39.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: osdmap e201: 8 total, 8 up, 7 in 2023-10-09T10:49:39.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: failed to encode map e201 with expected crc 2023-10-09T10:49:39.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: failed to encode map e201 with expected crc 2023-10-09T10:49:39.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: failed to encode map e201 with expected crc 2023-10-09T10:49:39.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: 1.6 scrub starts 2023-10-09T10:49:39.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: 1.6 scrub ok 2023-10-09T10:49:39.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: 1.7 scrub starts 2023-10-09T10:49:39.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: 1.7 scrub ok 2023-10-09T10:49:39.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: 1.1 scrub starts 2023-10-09T10:49:39.263 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: 1.1 scrub ok 2023-10-09T10:49:39.263 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:39.263 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:39.263 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2009194918' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:49:39.263 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:49:39.263 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:39 smithi125 ceph-mon[200197]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T10:49:39.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:49:39.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:49:39.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: osdmap e201: 8 total, 8 up, 7 in 2023-10-09T10:49:39.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: failed to encode map e201 with expected crc 2023-10-09T10:49:39.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: failed to encode map e201 with expected crc 2023-10-09T10:49:39.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: failed to encode map e201 with expected crc 2023-10-09T10:49:39.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: 1.6 scrub starts 2023-10-09T10:49:39.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: 1.6 scrub ok 2023-10-09T10:49:39.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: 1.7 scrub starts 2023-10-09T10:49:39.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: 1.7 scrub ok 2023-10-09T10:49:39.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: 1.1 scrub starts 2023-10-09T10:49:39.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: 1.1 scrub ok 2023-10-09T10:49:39.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:39.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:39.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2009194918' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:49:39.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:49:39.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:39 smithi155 ceph-mon[184854]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T10:49:39.982 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 19 16 38142 38126 125.374 103.25 0.00210838 0.00795739 2023-10-09T10:49:40.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[200197]: failed to encode map e201 with expected crc 2023-10-09T10:49:40.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[200197]: failed to encode map e201 with expected crc 2023-10-09T10:49:40.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[200197]: 1.b scrub starts 2023-10-09T10:49:40.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[200197]: 1.b scrub ok 2023-10-09T10:49:40.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[200197]: pgmap v455: 109 pgs: 109 active+clean; 2.0 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T10:49:40.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[200197]: 1.10 scrub starts 2023-10-09T10:49:40.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[200197]: 1.10 scrub ok 2023-10-09T10:49:40.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[200197]: failed to encode map e202 with expected crc 2023-10-09T10:49:40.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T10:49:40.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[200197]: osdmap e202: 8 total, 8 up, 6 in 2023-10-09T10:49:40.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[195507]: failed to encode map e201 with expected crc 2023-10-09T10:49:40.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[195507]: failed to encode map e201 with expected crc 2023-10-09T10:49:40.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[195507]: 1.b scrub starts 2023-10-09T10:49:40.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[195507]: 1.b scrub ok 2023-10-09T10:49:40.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[195507]: pgmap v455: 109 pgs: 109 active+clean; 2.0 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T10:49:40.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[195507]: 1.10 scrub starts 2023-10-09T10:49:40.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[195507]: 1.10 scrub ok 2023-10-09T10:49:40.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[195507]: failed to encode map e202 with expected crc 2023-10-09T10:49:40.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T10:49:40.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:40 smithi125 ceph-mon[195507]: osdmap e202: 8 total, 8 up, 6 in 2023-10-09T10:49:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:40 smithi155 ceph-mon[184854]: failed to encode map e201 with expected crc 2023-10-09T10:49:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:40 smithi155 ceph-mon[184854]: failed to encode map e201 with expected crc 2023-10-09T10:49:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:40 smithi155 ceph-mon[184854]: 1.b scrub starts 2023-10-09T10:49:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:40 smithi155 ceph-mon[184854]: 1.b scrub ok 2023-10-09T10:49:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:40 smithi155 ceph-mon[184854]: pgmap v455: 109 pgs: 109 active+clean; 2.0 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T10:49:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:40 smithi155 ceph-mon[184854]: 1.10 scrub starts 2023-10-09T10:49:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:40 smithi155 ceph-mon[184854]: 1.10 scrub ok 2023-10-09T10:49:40.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:40 smithi155 ceph-mon[184854]: failed to encode map e202 with expected crc 2023-10-09T10:49:40.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:40 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T10:49:40.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:40 smithi155 ceph-mon[184854]: osdmap e202: 8 total, 8 up, 6 in 2023-10-09T10:49:40.983 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:49:40.983615+0000 min lat: 0.00131548 max lat: 0.289207 avg lat: 0.00795739 2023-10-09T10:49:40.983 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:49:40.983 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 20 16 38142 38126 119.107 0 - 0.00795739 2023-10-09T10:49:41.050 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:49:41.051 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:49:41.051 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:49:41.051 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:49:41.051 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:49:41.051 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:49:41.052 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:49:41.052 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:49:41.052 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:49:41.052 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:49:41.052 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:49:41.052 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:49:41.052 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:49:41.053 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:49:41.053 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:49:41.053 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:49:41.079 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:41.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:41.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:41.183 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:41.212 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:41.231 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:41.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:41.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:41.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:41 smithi125 ceph-mon[195507]: failed to encode map e202 with expected crc 2023-10-09T10:49:41.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:41 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/286972826' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:41.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:41 smithi125 ceph-mon[195507]: osdmap e203: 8 total, 8 up, 6 in 2023-10-09T10:49:41.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:41 smithi125 ceph-mon[195507]: failed to encode map e203 with expected crc 2023-10-09T10:49:41.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:41 smithi125 ceph-mon[195507]: failed to encode map e203 with expected crc 2023-10-09T10:49:41.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:49:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:49:41] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T10:49:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:41 smithi125 ceph-mon[200197]: failed to encode map e202 with expected crc 2023-10-09T10:49:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:41 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/286972826' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:41 smithi125 ceph-mon[200197]: osdmap e203: 8 total, 8 up, 6 in 2023-10-09T10:49:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:41 smithi125 ceph-mon[200197]: failed to encode map e203 with expected crc 2023-10-09T10:49:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:41 smithi125 ceph-mon[200197]: failed to encode map e203 with expected crc 2023-10-09T10:49:41.335 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:49:41.364 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:41.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:41 smithi155 ceph-mon[184854]: failed to encode map e202 with expected crc 2023-10-09T10:49:41.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:41 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/286972826' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:41.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:41 smithi155 ceph-mon[184854]: osdmap e203: 8 total, 8 up, 6 in 2023-10-09T10:49:41.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:41 smithi155 ceph-mon[184854]: failed to encode map e203 with expected crc 2023-10-09T10:49:41.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:41 smithi155 ceph-mon[184854]: failed to encode map e203 with expected crc 2023-10-09T10:49:41.717 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.7. 2023-10-09T10:49:41.983 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 21 16 38142 38126 113.436 0 - 0.00795739 2023-10-09T10:49:42.036 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:49:42.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:42.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:49:42.141 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:42.178 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:42.208 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:42.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:42.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:42.312 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:42.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[200197]: failed to encode map e203 with expected crc 2023-10-09T10:49:42.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[200197]: failed to encode map e203 with expected crc 2023-10-09T10:49:42.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[200197]: failed to encode map e203 with expected crc 2023-10-09T10:49:42.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[200197]: failed to encode map e203 with expected crc 2023-10-09T10:49:42.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[200197]: pgmap v458: 109 pgs: 4 remapped+peering, 105 active+clean; 2.2 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T10:49:42.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[200197]: 1.c scrub starts 2023-10-09T10:49:42.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[200197]: 1.c scrub ok 2023-10-09T10:49:42.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:49:42.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[200197]: failed to encode map e204 with expected crc 2023-10-09T10:49:42.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[200197]: osdmap e204: 8 total, 8 up, 6 in 2023-10-09T10:49:42.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[200197]: failed to encode map e204 with expected crc 2023-10-09T10:49:42.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[195507]: failed to encode map e203 with expected crc 2023-10-09T10:49:42.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[195507]: failed to encode map e203 with expected crc 2023-10-09T10:49:42.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[195507]: failed to encode map e203 with expected crc 2023-10-09T10:49:42.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[195507]: failed to encode map e203 with expected crc 2023-10-09T10:49:42.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[195507]: pgmap v458: 109 pgs: 4 remapped+peering, 105 active+clean; 2.2 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T10:49:42.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[195507]: 1.c scrub starts 2023-10-09T10:49:42.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[195507]: 1.c scrub ok 2023-10-09T10:49:42.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:49:42.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[195507]: failed to encode map e204 with expected crc 2023-10-09T10:49:42.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[195507]: osdmap e204: 8 total, 8 up, 6 in 2023-10-09T10:49:42.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:42 smithi125 ceph-mon[195507]: failed to encode map e204 with expected crc 2023-10-09T10:49:42.345 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:42.371 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:42.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:42.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:42 smithi155 ceph-mon[184854]: failed to encode map e203 with expected crc 2023-10-09T10:49:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:42 smithi155 ceph-mon[184854]: failed to encode map e203 with expected crc 2023-10-09T10:49:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:42 smithi155 ceph-mon[184854]: failed to encode map e203 with expected crc 2023-10-09T10:49:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:42 smithi155 ceph-mon[184854]: failed to encode map e203 with expected crc 2023-10-09T10:49:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:42 smithi155 ceph-mon[184854]: pgmap v458: 109 pgs: 4 remapped+peering, 105 active+clean; 2.2 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T10:49:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:42 smithi155 ceph-mon[184854]: 1.c scrub starts 2023-10-09T10:49:42.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:42 smithi155 ceph-mon[184854]: 1.c scrub ok 2023-10-09T10:49:42.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:42 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:49:42.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:42 smithi155 ceph-mon[184854]: failed to encode map e204 with expected crc 2023-10-09T10:49:42.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:42 smithi155 ceph-mon[184854]: osdmap e204: 8 total, 8 up, 6 in 2023-10-09T10:49:42.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:42 smithi155 ceph-mon[184854]: failed to encode map e204 with expected crc 2023-10-09T10:49:42.475 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:42.506 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:42.534 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:42.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:42.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:42.641 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:42.677 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:42.713 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:42.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:42.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:42.819 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:42.852 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:42.876 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:42.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:42.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:42.981 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:49:42.984 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 22 16 39137 39121 111.107 20.7292 0.0161751 0.00899535 2023-10-09T10:49:43.073 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[195507]: failed to encode map e204 with expected crc 2023-10-09T10:49:43.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[195507]: failed to encode map e204 with expected crc 2023-10-09T10:49:43.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[195507]: failed to encode map e204 with expected crc 2023-10-09T10:49:43.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[195507]: failed to encode map e204 with expected crc 2023-10-09T10:49:43.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[195507]: failed to encode map e204 with expected crc 2023-10-09T10:49:43.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[195507]: osdmap e205: 8 total, 8 up, 6 in 2023-10-09T10:49:43.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[195507]: failed to encode map e205 with expected crc 2023-10-09T10:49:43.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[195507]: failed to encode map e205 with expected crc 2023-10-09T10:49:43.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[195507]: failed to encode map e205 with expected crc 2023-10-09T10:49:43.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[200197]: failed to encode map e204 with expected crc 2023-10-09T10:49:43.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[200197]: failed to encode map e204 with expected crc 2023-10-09T10:49:43.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[200197]: failed to encode map e204 with expected crc 2023-10-09T10:49:43.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[200197]: failed to encode map e204 with expected crc 2023-10-09T10:49:43.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[200197]: failed to encode map e204 with expected crc 2023-10-09T10:49:43.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[200197]: osdmap e205: 8 total, 8 up, 6 in 2023-10-09T10:49:43.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[200197]: failed to encode map e205 with expected crc 2023-10-09T10:49:43.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[200197]: failed to encode map e205 with expected crc 2023-10-09T10:49:43.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:43 smithi125 ceph-mon[200197]: failed to encode map e205 with expected crc 2023-10-09T10:49:43.403 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:49:43.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:43.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:49:43.508 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:43.536 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:43 smithi155 ceph-mon[184854]: failed to encode map e204 with expected crc 2023-10-09T10:49:43.536 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:43 smithi155 ceph-mon[184854]: failed to encode map e204 with expected crc 2023-10-09T10:49:43.537 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:43 smithi155 ceph-mon[184854]: failed to encode map e204 with expected crc 2023-10-09T10:49:43.537 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:43 smithi155 ceph-mon[184854]: failed to encode map e204 with expected crc 2023-10-09T10:49:43.537 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:43 smithi155 ceph-mon[184854]: failed to encode map e204 with expected crc 2023-10-09T10:49:43.537 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:43 smithi155 ceph-mon[184854]: osdmap e205: 8 total, 8 up, 6 in 2023-10-09T10:49:43.537 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:43 smithi155 ceph-mon[184854]: failed to encode map e205 with expected crc 2023-10-09T10:49:43.537 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:43 smithi155 ceph-mon[184854]: failed to encode map e205 with expected crc 2023-10-09T10:49:43.537 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:43 smithi155 ceph-mon[184854]: failed to encode map e205 with expected crc 2023-10-09T10:49:43.539 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:43.562 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:43.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:43.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:43.666 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:43.713 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:49:43.713 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:49:43.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:49:43.713 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:49:43.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.714 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42384 6.10 6:0c7bd3cf:::benchmark_data_smithi125_244951_object42383:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.714 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:49:43.714 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.59412675800002, 2023-10-09T10:49:43.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0577403670000001, 2023-10-09T10:49:43.714 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.714 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.714 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.714 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.714 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.714 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42384 2023-10-09T10:49:43.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.715 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:49:43.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:49:43.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117185+0000", 2023-10-09T10:49:43.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:43.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117225+0000", 2023-10-09T10:49:43.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0364000000000002e-05 2023-10-09T10:49:43.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117240+0000", 2023-10-09T10:49:43.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4425e-05 2023-10-09T10:49:43.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117248+0000", 2023-10-09T10:49:43.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4999999999999999e-06 2023-10-09T10:49:43.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117301+0000", 2023-10-09T10:49:43.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2685999999999997e-05 2023-10-09T10:49:43.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.718 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.718 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117312+0000", 2023-10-09T10:49:43.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1273e-05 2023-10-09T10:49:43.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.718 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.718 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103469+0000", 2023-10-09T10:49:43.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98615701300000003 2023-10-09T10:49:43.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103483+0000", 2023-10-09T10:49:43.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3207000000000001e-05 2023-10-09T10:49:43.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.148764+0000", 2023-10-09T10:49:43.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0452812789999999 2023-10-09T10:49:43.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.148882+0000", 2023-10-09T10:49:43.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118095 2023-10-09T10:49:43.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.149057+0000", 2023-10-09T10:49:43.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017516400000000001 2023-10-09T10:49:43.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174105+0000", 2023-10-09T10:49:43.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.025048237000000001 2023-10-09T10:49:43.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174158+0000", 2023-10-09T10:49:43.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2330999999999998e-05 2023-10-09T10:49:43.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174325+0000", 2023-10-09T10:49:43.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016690100000000001 2023-10-09T10:49:43.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174367+0000", 2023-10-09T10:49:43.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2759000000000002e-05 2023-10-09T10:49:43.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174927+0000", 2023-10-09T10:49:43.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00055942800000000005 2023-10-09T10:49:43.722 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.722 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.722 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.723 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42398 6.10 6:0867fefc:::benchmark_data_smithi125_244951_object42397:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.723 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:49:43.723 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.583347989, 2023-10-09T10:49:43.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0510695769999998, 2023-10-09T10:49:43.723 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.723 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.723 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.723 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.723 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.723 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42398 2023-10-09T10:49:43.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.723 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:49:43.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:49:43.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127964+0000", 2023-10-09T10:49:43.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:43.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127989+0000", 2023-10-09T10:49:43.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4811e-05 2023-10-09T10:49:43.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128005+0000", 2023-10-09T10:49:43.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6436999999999999e-05 2023-10-09T10:49:43.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128014+0000", 2023-10-09T10:49:43.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5990000000000002e-06 2023-10-09T10:49:43.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128072+0000", 2023-10-09T10:49:43.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7980999999999999e-05 2023-10-09T10:49:43.727 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.727 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.727 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.727 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128083+0000", 2023-10-09T10:49:43.727 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1051e-05 2023-10-09T10:49:43.727 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.727 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.727 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.727 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103714+0000", 2023-10-09T10:49:43.727 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97563124099999998 2023-10-09T10:49:43.727 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.727 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103725+0000", 2023-10-09T10:49:43.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1121000000000001e-05 2023-10-09T10:49:43.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174971+0000", 2023-10-09T10:49:43.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0712457310000001 2023-10-09T10:49:43.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.175149+0000", 2023-10-09T10:49:43.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000178584 2023-10-09T10:49:43.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.175358+0000", 2023-10-09T10:49:43.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000208848 2023-10-09T10:49:43.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178406+0000", 2023-10-09T10:49:43.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030478049999999998 2023-10-09T10:49:43.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178418+0000", 2023-10-09T10:49:43.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1499e-05 2023-10-09T10:49:43.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178466+0000", 2023-10-09T10:49:43.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8715999999999999e-05 2023-10-09T10:49:43.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178522+0000", 2023-10-09T10:49:43.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5952999999999998e-05 2023-10-09T10:49:43.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179035+0000", 2023-10-09T10:49:43.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051254299999999996 2023-10-09T10:49:43.731 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.731 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.731 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.732 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42518 6.10 6:0fbcc8bc:::benchmark_data_smithi125_244951_object42517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.732 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:49:43.732 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.51695066399998, 2023-10-09T10:49:43.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9852423510000001, 2023-10-09T10:49:43.732 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.732 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.732 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.732 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.732 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.732 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42518 2023-10-09T10:49:43.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.733 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:49:43.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:49:43.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194361+0000", 2023-10-09T10:49:43.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:43.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194385+0000", 2023-10-09T10:49:43.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4017000000000001e-05 2023-10-09T10:49:43.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194401+0000", 2023-10-09T10:49:43.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5841e-05 2023-10-09T10:49:43.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194408+0000", 2023-10-09T10:49:43.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0450000000000003e-06 2023-10-09T10:49:43.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194460+0000", 2023-10-09T10:49:43.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2015000000000003e-05 2023-10-09T10:49:43.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194474+0000", 2023-10-09T10:49:43.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4486e-05 2023-10-09T10:49:43.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103750+0000", 2023-10-09T10:49:43.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.90927595699999997 2023-10-09T10:49:43.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103758+0000", 2023-10-09T10:49:43.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9000000000000006e-06 2023-10-09T10:49:43.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176100+0000", 2023-10-09T10:49:43.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.07234187 2023-10-09T10:49:43.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176198+0000", 2023-10-09T10:49:43.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7408e-05 2023-10-09T10:49:43.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176317+0000", 2023-10-09T10:49:43.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011998900000000001 2023-10-09T10:49:43.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178425+0000", 2023-10-09T10:49:43.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021072 2023-10-09T10:49:43.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179085+0000", 2023-10-09T10:49:43.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00066011800000000001 2023-10-09T10:49:43.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179122+0000", 2023-10-09T10:49:43.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6831e-05 2023-10-09T10:49:43.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179155+0000", 2023-10-09T10:49:43.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3080000000000002e-05 2023-10-09T10:49:43.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179605+0000", 2023-10-09T10:49:43.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000450121 2023-10-09T10:49:43.740 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.740 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.740 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.741 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42546 6.10 6:0889293a:::benchmark_data_smithi125_244951_object42545:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.741 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:49:43.741 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.502521231, 2023-10-09T10:49:43.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.971488017, 2023-10-09T10:49:43.741 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.741 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.741 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.741 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.741 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.741 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42546 2023-10-09T10:49:43.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.742 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:49:43.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:49:43.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208790+0000", 2023-10-09T10:49:43.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:49:43.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208815+0000", 2023-10-09T10:49:43.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5103e-05 2023-10-09T10:49:43.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208831+0000", 2023-10-09T10:49:43.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5235000000000001e-05 2023-10-09T10:49:43.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208838+0000", 2023-10-09T10:49:43.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6510000000000001e-06 2023-10-09T10:49:43.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208895+0000", 2023-10-09T10:49:43.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6817999999999997e-05 2023-10-09T10:49:43.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208906+0000", 2023-10-09T10:49:43.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0550000000000001e-05 2023-10-09T10:49:43.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103780+0000", 2023-10-09T10:49:43.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.89487466699999996 2023-10-09T10:49:43.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103788+0000", 2023-10-09T10:49:43.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0970000000000006e-06 2023-10-09T10:49:43.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176855+0000", 2023-10-09T10:49:43.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0730664299999999 2023-10-09T10:49:43.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176993+0000", 2023-10-09T10:49:43.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013810999999999999 2023-10-09T10:49:43.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.177166+0000", 2023-10-09T10:49:43.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017289900000000001 2023-10-09T10:49:43.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179093+0000", 2023-10-09T10:49:43.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019276969999999999 2023-10-09T10:49:43.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179674+0000", 2023-10-09T10:49:43.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00058078000000000003 2023-10-09T10:49:43.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179730+0000", 2023-10-09T10:49:43.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5418999999999999e-05 2023-10-09T10:49:43.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179788+0000", 2023-10-09T10:49:43.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8510000000000001e-05 2023-10-09T10:49:43.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180280+0000", 2023-10-09T10:49:43.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00049179400000000004 2023-10-09T10:49:43.751 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.751 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.751 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.751 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42570 6.10 6:0f6ea5a0:::benchmark_data_smithi125_244951_object42569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.751 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:49:43.751 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.48843673800002, 2023-10-09T10:49:43.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.969653487, 2023-10-09T10:49:43.751 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.751 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.752 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.752 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.752 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.752 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42570 2023-10-09T10:49:43.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.752 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:49:43.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:49:43.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222875+0000", 2023-10-09T10:49:43.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:43.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222897+0000", 2023-10-09T10:49:43.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1678000000000001e-05 2023-10-09T10:49:43.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222906+0000", 2023-10-09T10:49:43.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4190000000000008e-06 2023-10-09T10:49:43.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222913+0000", 2023-10-09T10:49:43.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2659999999999998e-06 2023-10-09T10:49:43.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222958+0000", 2023-10-09T10:49:43.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5979000000000001e-05 2023-10-09T10:49:43.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222965+0000", 2023-10-09T10:49:43.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0099999999999998e-06 2023-10-09T10:49:43.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103823+0000", 2023-10-09T10:49:43.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.880857891 2023-10-09T10:49:43.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103833+0000", 2023-10-09T10:49:43.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8910000000000007e-06 2023-10-09T10:49:43.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180446+0000", 2023-10-09T10:49:43.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0766125440000001 2023-10-09T10:49:43.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180594+0000", 2023-10-09T10:49:43.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014846600000000001 2023-10-09T10:49:43.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180775+0000", 2023-10-09T10:49:43.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018071999999999999 2023-10-09T10:49:43.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183638+0000", 2023-10-09T10:49:43.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028626440000000001 2023-10-09T10:49:43.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183677+0000", 2023-10-09T10:49:43.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9545000000000002e-05 2023-10-09T10:49:43.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191583+0000", 2023-10-09T10:49:43.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0079059569999999999 2023-10-09T10:49:43.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191643+0000", 2023-10-09T10:49:43.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9590999999999999e-05 2023-10-09T10:49:43.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192530+0000", 2023-10-09T10:49:43.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00088716399999999996 2023-10-09T10:49:43.760 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.760 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.760 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.760 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42586 6.10 6:090d6b8d:::benchmark_data_smithi125_244951_object42585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.760 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:49:43.760 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.48184219699999, 2023-10-09T10:49:43.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.963875493, 2023-10-09T10:49:43.761 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.761 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.761 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.761 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.761 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.761 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42586 2023-10-09T10:49:43.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.761 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:49:43.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:49:43.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229470+0000", 2023-10-09T10:49:43.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:49:43.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229492+0000", 2023-10-09T10:49:43.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1716999999999999e-05 2023-10-09T10:49:43.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229501+0000", 2023-10-09T10:49:43.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0929999999999992e-06 2023-10-09T10:49:43.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229507+0000", 2023-10-09T10:49:43.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6760000000000001e-06 2023-10-09T10:49:43.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229565+0000", 2023-10-09T10:49:43.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7450000000000001e-05 2023-10-09T10:49:43.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229572+0000", 2023-10-09T10:49:43.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0820000000000001e-06 2023-10-09T10:49:43.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103865+0000", 2023-10-09T10:49:43.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.87429282200000003 2023-10-09T10:49:43.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103873+0000", 2023-10-09T10:49:43.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2090000000000005e-06 2023-10-09T10:49:43.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181656+0000", 2023-10-09T10:49:43.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.077783462 2023-10-09T10:49:43.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181791+0000", 2023-10-09T10:49:43.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000134351 2023-10-09T10:49:43.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181948+0000", 2023-10-09T10:49:43.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015678600000000001 2023-10-09T10:49:43.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183685+0000", 2023-10-09T10:49:43.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00173705 2023-10-09T10:49:43.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183719+0000", 2023-10-09T10:49:43.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4301999999999998e-05 2023-10-09T10:49:43.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192640+0000", 2023-10-09T10:49:43.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0089210899999999996 2023-10-09T10:49:43.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192696+0000", 2023-10-09T10:49:43.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5733000000000003e-05 2023-10-09T10:49:43.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193346+0000", 2023-10-09T10:49:43.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00065071200000000001 2023-10-09T10:49:43.769 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.769 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.769 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.769 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42606 6.10 6:0826a955:::benchmark_data_smithi125_244951_object42605:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.769 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:49:43.770 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.47308260199998, 2023-10-09T10:49:43.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.955680458, 2023-10-09T10:49:43.770 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.770 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.770 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.770 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.770 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.770 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42606 2023-10-09T10:49:43.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.770 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:49:43.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:49:43.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238230+0000", 2023-10-09T10:49:43.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:49:43.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238257+0000", 2023-10-09T10:49:43.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7551000000000001e-05 2023-10-09T10:49:43.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238267+0000", 2023-10-09T10:49:43.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5820000000000008e-06 2023-10-09T10:49:43.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238273+0000", 2023-10-09T10:49:43.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5490000000000003e-06 2023-10-09T10:49:43.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238322+0000", 2023-10-09T10:49:43.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8467999999999997e-05 2023-10-09T10:49:43.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238332+0000", 2023-10-09T10:49:43.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9000000000000001e-06 2023-10-09T10:49:43.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103898+0000", 2023-10-09T10:49:43.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.86556675699999996 2023-10-09T10:49:43.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103906+0000", 2023-10-09T10:49:43.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7710000000000006e-06 2023-10-09T10:49:43.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.182699+0000", 2023-10-09T10:49:43.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0787928490000001 2023-10-09T10:49:43.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.182847+0000", 2023-10-09T10:49:43.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014758600000000001 2023-10-09T10:49:43.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183016+0000", 2023-10-09T10:49:43.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000169326 2023-10-09T10:49:43.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186922+0000", 2023-10-09T10:49:43.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039060269999999998 2023-10-09T10:49:43.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186980+0000", 2023-10-09T10:49:43.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8313999999999998e-05 2023-10-09T10:49:43.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193472+0000", 2023-10-09T10:49:43.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0064917020000000002 2023-10-09T10:49:43.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193522+0000", 2023-10-09T10:49:43.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9672999999999997e-05 2023-10-09T10:49:43.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193911+0000", 2023-10-09T10:49:43.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00038944099999999997 2023-10-09T10:49:43.778 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.778 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.778 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.779 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42608 6.10 6:0c1c992a:::benchmark_data_smithi125_244951_object42607:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.779 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:49:43.779 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.46999090200001, 2023-10-09T10:49:43.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9589951289999998, 2023-10-09T10:49:43.779 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.779 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.779 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.779 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.780 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.780 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42608 2023-10-09T10:49:43.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.780 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:49:43.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:49:43.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241321+0000", 2023-10-09T10:49:43.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:49:43.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241343+0000", 2023-10-09T10:49:43.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1987000000000001e-05 2023-10-09T10:49:43.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.782 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241355+0000", 2023-10-09T10:49:43.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1395e-05 2023-10-09T10:49:43.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.782 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241362+0000", 2023-10-09T10:49:43.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7759999999999997e-06 2023-10-09T10:49:43.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241406+0000", 2023-10-09T10:49:43.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3189999999999998e-05 2023-10-09T10:49:43.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241413+0000", 2023-10-09T10:49:43.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5800000000000003e-06 2023-10-09T10:49:43.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103930+0000", 2023-10-09T10:49:43.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.86251731399999998 2023-10-09T10:49:43.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103938+0000", 2023-10-09T10:49:43.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7349999999999996e-06 2023-10-09T10:49:43.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183741+0000", 2023-10-09T10:49:43.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0798032179999999 2023-10-09T10:49:43.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183865+0000", 2023-10-09T10:49:43.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012327200000000001 2023-10-09T10:49:43.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.184011+0000", 2023-10-09T10:49:43.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014617499999999999 2023-10-09T10:49:43.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186989+0000", 2023-10-09T10:49:43.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029777340000000001 2023-10-09T10:49:43.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187029+0000", 2023-10-09T10:49:43.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0513000000000003e-05 2023-10-09T10:49:43.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199414+0000", 2023-10-09T10:49:43.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012385022000000001 2023-10-09T10:49:43.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199472+0000", 2023-10-09T10:49:43.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7998999999999998e-05 2023-10-09T10:49:43.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200317+0000", 2023-10-09T10:49:43.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000845245 2023-10-09T10:49:43.788 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.788 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.788 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.788 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42717 6.10 6:0e29612e:::benchmark_data_smithi125_244951_object42716:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.788 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:49:43.788 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.40386244400003, 2023-10-09T10:49:43.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8935023019999999, 2023-10-09T10:49:43.788 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.789 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.789 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.789 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.789 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.789 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42717 2023-10-09T10:49:43.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.789 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:49:43.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:49:43.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307449+0000", 2023-10-09T10:49:43.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:43.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307473+0000", 2023-10-09T10:49:43.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3349e-05 2023-10-09T10:49:43.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307490+0000", 2023-10-09T10:49:43.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7133e-05 2023-10-09T10:49:43.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307496+0000", 2023-10-09T10:49:43.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7349999999999999e-06 2023-10-09T10:49:43.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.792 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.792 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307561+0000", 2023-10-09T10:49:43.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4226999999999993e-05 2023-10-09T10:49:43.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.792 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.792 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307575+0000", 2023-10-09T10:49:43.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4504e-05 2023-10-09T10:49:43.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103963+0000", 2023-10-09T10:49:43.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.79638768500000001 2023-10-09T10:49:43.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103971+0000", 2023-10-09T10:49:43.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9629999999999998e-06 2023-10-09T10:49:43.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.185776+0000", 2023-10-09T10:49:43.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0818051479999999 2023-10-09T10:49:43.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.185949+0000", 2023-10-09T10:49:43.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000172659 2023-10-09T10:49:43.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186122+0000", 2023-10-09T10:49:43.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017296799999999999 2023-10-09T10:49:43.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189418+0000", 2023-10-09T10:49:43.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032959679999999998 2023-10-09T10:49:43.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189457+0000", 2023-10-09T10:49:43.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9669000000000003e-05 2023-10-09T10:49:43.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200372+0000", 2023-10-09T10:49:43.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010914633999999999 2023-10-09T10:49:43.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200421+0000", 2023-10-09T10:49:43.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9047e-05 2023-10-09T10:49:43.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200953+0000", 2023-10-09T10:49:43.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00053212500000000005 2023-10-09T10:49:43.797 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.797 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.797 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.797 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42722 6.10 6:0ee7a776:::benchmark_data_smithi125_244951_object42721:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.797 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:49:43.797 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.40380423599998, 2023-10-09T10:49:43.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8972440939999999, 2023-10-09T10:49:43.798 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.798 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.798 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.798 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.798 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.798 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42722 2023-10-09T10:49:43.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.798 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.798 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.798 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:49:43.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.799 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:49:43.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.799 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307508+0000", 2023-10-09T10:49:43.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:43.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307529+0000", 2023-10-09T10:49:43.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0866999999999999e-05 2023-10-09T10:49:43.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307539+0000", 2023-10-09T10:49:43.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2690000000000005e-06 2023-10-09T10:49:43.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.801 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307544+0000", 2023-10-09T10:49:43.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7930000000000003e-06 2023-10-09T10:49:43.801 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.801 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307599+0000", 2023-10-09T10:49:43.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4876000000000001e-05 2023-10-09T10:49:43.801 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.801 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.802 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307607+0000", 2023-10-09T10:49:43.802 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8760000000000005e-06 2023-10-09T10:49:43.802 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.802 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.802 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.802 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103995+0000", 2023-10-09T10:49:43.802 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.79638814099999999 2023-10-09T10:49:43.802 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.802 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.802 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.802 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104003+0000", 2023-10-09T10:49:43.803 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7819999999999994e-06 2023-10-09T10:49:43.803 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.803 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.803 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.803 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187058+0000", 2023-10-09T10:49:43.803 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.083054712 2023-10-09T10:49:43.803 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.803 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.803 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.803 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187234+0000", 2023-10-09T10:49:43.803 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017615799999999999 2023-10-09T10:49:43.803 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.804 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187416+0000", 2023-10-09T10:49:43.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018243 2023-10-09T10:49:43.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.804 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189466+0000", 2023-10-09T10:49:43.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020493210000000002 2023-10-09T10:49:43.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189504+0000", 2023-10-09T10:49:43.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8287000000000001e-05 2023-10-09T10:49:43.805 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.805 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.201001+0000", 2023-10-09T10:49:43.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011496632999999999 2023-10-09T10:49:43.805 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.805 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.201044+0000", 2023-10-09T10:49:43.806 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3078000000000003e-05 2023-10-09T10:49:43.806 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.806 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.806 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.806 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204753+0000", 2023-10-09T10:49:43.806 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037093209999999998 2023-10-09T10:49:43.806 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.806 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.806 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.806 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.806 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.807 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42757 6.10 6:08e1bcf5:::benchmark_data_smithi125_244951_object42756:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.807 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.333229+0000", 2023-10-09T10:49:43.807 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.37808407699998, 2023-10-09T10:49:43.807 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.872111463, 2023-10-09T10:49:43.807 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.807 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.807 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.807 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.807 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.807 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42757 2023-10-09T10:49:43.807 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.807 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.808 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.808 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.808 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333229+0000", 2023-10-09T10:49:43.808 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.808 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.808 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.808 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.808 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333229+0000", 2023-10-09T10:49:43.808 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.808 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.808 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.808 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.809 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333228+0000", 2023-10-09T10:49:43.809 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:49:43.809 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.809 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.809 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.809 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333280+0000", 2023-10-09T10:49:43.809 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1999999999999997e-05 2023-10-09T10:49:43.809 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.809 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.809 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.809 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333291+0000", 2023-10-09T10:49:43.809 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0738e-05 2023-10-09T10:49:43.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.810 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.810 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.810 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333298+0000", 2023-10-09T10:49:43.810 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7730000000000002e-06 2023-10-09T10:49:43.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.810 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.810 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.810 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333360+0000", 2023-10-09T10:49:43.810 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2230000000000006e-05 2023-10-09T10:49:43.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333371+0000", 2023-10-09T10:49:43.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0669999999999999e-05 2023-10-09T10:49:43.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104027+0000", 2023-10-09T10:49:43.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.77065640400000002 2023-10-09T10:49:43.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104035+0000", 2023-10-09T10:49:43.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7400000000000004e-06 2023-10-09T10:49:43.812 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.812 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.188337+0000", 2023-10-09T10:49:43.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.084302281 2023-10-09T10:49:43.812 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.812 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.188495+0000", 2023-10-09T10:49:43.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015774699999999999 2023-10-09T10:49:43.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.813 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.813 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.813 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.188674+0000", 2023-10-09T10:49:43.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000178729 2023-10-09T10:49:43.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.813 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.813 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.813 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191542+0000", 2023-10-09T10:49:43.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028680799999999998 2023-10-09T10:49:43.814 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.814 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.814 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.814 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192592+0000", 2023-10-09T10:49:43.814 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010502140000000001 2023-10-09T10:49:43.814 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.814 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.814 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.814 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204808+0000", 2023-10-09T10:49:43.814 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012216595 2023-10-09T10:49:43.815 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.815 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.815 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.815 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204867+0000", 2023-10-09T10:49:43.815 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8251999999999997e-05 2023-10-09T10:49:43.815 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.815 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.815 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.815 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205341+0000", 2023-10-09T10:49:43.815 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047385999999999999 2023-10-09T10:49:43.815 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.816 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.816 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.816 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.816 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.816 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42857 6.10 6:0e113cab:::benchmark_data_smithi125_244951_object42856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.816 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.409599+0000", 2023-10-09T10:49:43.816 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.30171450799998, 2023-10-09T10:49:43.816 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.796322089, 2023-10-09T10:49:43.816 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.816 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.816 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.816 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.817 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.817 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42857 2023-10-09T10:49:43.817 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.817 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.817 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.817 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.817 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409599+0000", 2023-10-09T10:49:43.817 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.817 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.817 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.817 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.817 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409599+0000", 2023-10-09T10:49:43.818 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.818 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.818 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.818 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.818 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409597+0000", 2023-10-09T10:49:43.818 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:43.818 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.818 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.818 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.818 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409619+0000", 2023-10-09T10:49:43.818 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1202000000000001e-05 2023-10-09T10:49:43.819 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.819 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.819 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.819 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409631+0000", 2023-10-09T10:49:43.819 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2546e-05 2023-10-09T10:49:43.819 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.819 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.819 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.819 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409638+0000", 2023-10-09T10:49:43.819 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9269999999999999e-06 2023-10-09T10:49:43.819 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.820 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.820 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.820 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409694+0000", 2023-10-09T10:49:43.820 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6025e-05 2023-10-09T10:49:43.820 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.820 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.820 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.820 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409706+0000", 2023-10-09T10:49:43.820 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.164e-05 2023-10-09T10:49:43.820 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.820 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.820 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.820 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104059+0000", 2023-10-09T10:49:43.821 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.69435323100000002 2023-10-09T10:49:43.821 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.821 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.821 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.821 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104067+0000", 2023-10-09T10:49:43.821 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9060000000000002e-06 2023-10-09T10:49:43.821 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.821 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.821 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.821 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189529+0000", 2023-10-09T10:49:43.821 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.085462189 2023-10-09T10:49:43.822 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.822 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.822 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.822 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189675+0000", 2023-10-09T10:49:43.822 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014598599999999999 2023-10-09T10:49:43.822 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.822 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.822 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.822 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189846+0000", 2023-10-09T10:49:43.822 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017139799999999999 2023-10-09T10:49:43.822 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.822 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.823 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.823 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192547+0000", 2023-10-09T10:49:43.823 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027008760000000001 2023-10-09T10:49:43.823 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.823 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.823 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.823 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193395+0000", 2023-10-09T10:49:43.823 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00084811200000000004 2023-10-09T10:49:43.823 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.823 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.823 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.823 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205386+0000", 2023-10-09T10:49:43.824 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011990763999999999 2023-10-09T10:49:43.824 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.824 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.824 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.824 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205435+0000", 2023-10-09T10:49:43.824 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8467999999999997e-05 2023-10-09T10:49:43.824 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.824 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.824 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.824 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205921+0000", 2023-10-09T10:49:43.824 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00048609599999999999 2023-10-09T10:49:43.825 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.825 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.825 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.825 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.825 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.825 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42860 6.10 6:08b59805:::benchmark_data_smithi125_244951_object42859:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.825 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.411356+0000", 2023-10-09T10:49:43.825 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.29995674499997, 2023-10-09T10:49:43.825 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7952376650000001, 2023-10-09T10:49:43.825 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.825 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.825 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.826 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.826 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.826 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42860 2023-10-09T10:49:43.826 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.826 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.826 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.826 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.826 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411356+0000", 2023-10-09T10:49:43.826 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.826 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.826 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.827 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.827 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411356+0000", 2023-10-09T10:49:43.827 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.827 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.827 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.827 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.827 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411355+0000", 2023-10-09T10:49:43.827 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:43.827 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.827 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.827 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.827 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411383+0000", 2023-10-09T10:49:43.828 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7348999999999999e-05 2023-10-09T10:49:43.828 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.828 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.828 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.828 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411398+0000", 2023-10-09T10:49:43.828 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5424e-05 2023-10-09T10:49:43.828 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.828 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.828 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.828 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411407+0000", 2023-10-09T10:49:43.828 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2429999999999994e-06 2023-10-09T10:49:43.829 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.829 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.829 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.829 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411465+0000", 2023-10-09T10:49:43.829 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7892000000000001e-05 2023-10-09T10:49:43.829 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.829 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.829 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.829 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411475+0000", 2023-10-09T10:49:43.829 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0066e-05 2023-10-09T10:49:43.829 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.829 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.830 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.830 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104091+0000", 2023-10-09T10:49:43.830 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.69261584300000001 2023-10-09T10:49:43.830 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.830 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.830 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.830 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104099+0000", 2023-10-09T10:49:43.830 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8760000000000005e-06 2023-10-09T10:49:43.830 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.830 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.830 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.831 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.190559+0000", 2023-10-09T10:49:43.831 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.086459659 2023-10-09T10:49:43.831 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.831 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.831 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.831 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.190698+0000", 2023-10-09T10:49:43.831 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013982400000000001 2023-10-09T10:49:43.831 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.831 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.831 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.831 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.190864+0000", 2023-10-09T10:49:43.832 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016535200000000001 2023-10-09T10:49:43.832 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.832 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.832 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.832 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192601+0000", 2023-10-09T10:49:43.832 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0017371999999999999 2023-10-09T10:49:43.832 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.832 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.832 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.832 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193434+0000", 2023-10-09T10:49:43.832 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00083267900000000001 2023-10-09T10:49:43.832 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.832 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.833 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.833 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206008+0000", 2023-10-09T10:49:43.833 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012574058000000001 2023-10-09T10:49:43.833 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.833 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.833 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.833 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206060+0000", 2023-10-09T10:49:43.833 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2710999999999997e-05 2023-10-09T10:49:43.833 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.833 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.833 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.834 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206594+0000", 2023-10-09T10:49:43.834 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00053372399999999996 2023-10-09T10:49:43.834 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.834 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.834 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.834 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.834 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.834 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42870 6.10 6:0e8b4417:::benchmark_data_smithi125_244951_object42869:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.834 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.417961+0000", 2023-10-09T10:49:43.834 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.29335205500001, 2023-10-09T10:49:43.834 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8001529010000001, 2023-10-09T10:49:43.834 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.835 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.835 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.835 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.835 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.835 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42870 2023-10-09T10:49:43.835 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.835 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.835 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.835 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.835 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417961+0000", 2023-10-09T10:49:43.835 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.835 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.836 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.836 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417961+0000", 2023-10-09T10:49:43.836 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.836 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.836 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.836 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417960+0000", 2023-10-09T10:49:43.836 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:43.836 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.836 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.837 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417982+0000", 2023-10-09T10:49:43.837 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2585000000000001e-05 2023-10-09T10:49:43.837 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.837 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.837 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.837 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417999+0000", 2023-10-09T10:49:43.837 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6711999999999999e-05 2023-10-09T10:49:43.837 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.837 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.837 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.837 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.418005+0000", 2023-10-09T10:49:43.837 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3180000000000003e-06 2023-10-09T10:49:43.838 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.838 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.838 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.418044+0000", 2023-10-09T10:49:43.838 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9171999999999997e-05 2023-10-09T10:49:43.838 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.838 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.838 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.418052+0000", 2023-10-09T10:49:43.838 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9880000000000004e-06 2023-10-09T10:49:43.839 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.839 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.839 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.839 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104123+0000", 2023-10-09T10:49:43.839 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.68607037299999996 2023-10-09T10:49:43.839 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.839 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.839 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.839 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104131+0000", 2023-10-09T10:49:43.839 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0730000000000005e-06 2023-10-09T10:49:43.839 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.839 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.840 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.840 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193936+0000", 2023-10-09T10:49:43.840 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0898054349999999 2023-10-09T10:49:43.840 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.840 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.840 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.840 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.194029+0000", 2023-10-09T10:49:43.840 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2313999999999997e-05 2023-10-09T10:49:43.840 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.840 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.840 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.840 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.194131+0000", 2023-10-09T10:49:43.841 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000102384 2023-10-09T10:49:43.841 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.841 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.841 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.841 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199366+0000", 2023-10-09T10:49:43.841 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0052347640000000003 2023-10-09T10:49:43.841 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.841 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.841 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.841 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205969+0000", 2023-10-09T10:49:43.841 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066036840000000003 2023-10-09T10:49:43.841 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.842 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.842 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.217437+0000", 2023-10-09T10:49:43.842 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011467252000000001 2023-10-09T10:49:43.842 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.842 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.842 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.217474+0000", 2023-10-09T10:49:43.842 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7177e-05 2023-10-09T10:49:43.842 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.842 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.843 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218114+0000", 2023-10-09T10:49:43.843 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000640231 2023-10-09T10:49:43.843 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.843 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.843 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.843 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.843 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.843 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42902 6.10 6:0d48f256:::benchmark_data_smithi125_244951_object42901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.843 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.448090+0000", 2023-10-09T10:49:43.843 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.26322359699998, 2023-10-09T10:49:43.843 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7704487879999999, 2023-10-09T10:49:43.843 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.844 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.844 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.844 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.844 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.844 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42902 2023-10-09T10:49:43.844 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.844 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.844 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.844 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.844 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448090+0000", 2023-10-09T10:49:43.844 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.845 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.845 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.845 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.845 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448090+0000", 2023-10-09T10:49:43.845 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.845 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.845 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.845 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.845 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448088+0000", 2023-10-09T10:49:43.845 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:49:43.845 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.845 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.846 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.846 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448108+0000", 2023-10-09T10:49:43.846 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9888999999999999e-05 2023-10-09T10:49:43.846 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.846 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.846 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.846 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448124+0000", 2023-10-09T10:49:43.846 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5529999999999999e-05 2023-10-09T10:49:43.846 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.846 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.846 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.847 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448131+0000", 2023-10-09T10:49:43.847 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9500000000000004e-06 2023-10-09T10:49:43.847 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.847 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.847 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.847 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448189+0000", 2023-10-09T10:49:43.847 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8010000000000002e-05 2023-10-09T10:49:43.847 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.847 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.847 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.847 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448201+0000", 2023-10-09T10:49:43.847 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2368000000000001e-05 2023-10-09T10:49:43.848 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.848 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.848 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.848 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104155+0000", 2023-10-09T10:49:43.848 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.65595361500000005 2023-10-09T10:49:43.848 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.848 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.848 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.848 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104163+0000", 2023-10-09T10:49:43.848 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9759999999999995e-06 2023-10-09T10:49:43.848 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.848 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.849 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.849 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.194936+0000", 2023-10-09T10:49:43.849 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.090773378 2023-10-09T10:49:43.849 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.849 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.849 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.849 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.195074+0000", 2023-10-09T10:49:43.849 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000138298 2023-10-09T10:49:43.849 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.849 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.849 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.850 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.196676+0000", 2023-10-09T10:49:43.850 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001601327 2023-10-09T10:49:43.850 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.850 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.850 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.850 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199373+0000", 2023-10-09T10:49:43.850 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026971320000000001 2023-10-09T10:49:43.850 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.850 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.850 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.850 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206643+0000", 2023-10-09T10:49:43.851 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0072700519999999999 2023-10-09T10:49:43.851 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.851 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.851 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.851 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218147+0000", 2023-10-09T10:49:43.851 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011503904000000001 2023-10-09T10:49:43.851 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.851 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.851 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.851 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218182+0000", 2023-10-09T10:49:43.851 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4752999999999997e-05 2023-10-09T10:49:43.851 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.851 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.852 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.852 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218538+0000", 2023-10-09T10:49:43.852 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035677400000000002 2023-10-09T10:49:43.852 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.852 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.852 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.852 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.852 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.852 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42906 6.10 6:0945c66e:::benchmark_data_smithi125_244951_object42905:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:49:43.852 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.453858+0000", 2023-10-09T10:49:43.852 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 288.25745548399999, 2023-10-09T10:49:43.852 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7650414240000001, 2023-10-09T10:49:43.853 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.853 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.853 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.853 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.853 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.853 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42906 2023-10-09T10:49:43.853 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.853 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.853 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.853 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.853 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453858+0000", 2023-10-09T10:49:43.853 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.854 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.854 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.854 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.854 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453858+0000", 2023-10-09T10:49:43.854 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.854 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.854 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.854 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.854 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453857+0000", 2023-10-09T10:49:43.854 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:49:43.854 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.855 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.855 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.855 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453877+0000", 2023-10-09T10:49:43.855 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0857999999999999e-05 2023-10-09T10:49:43.855 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.855 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.855 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.855 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453891+0000", 2023-10-09T10:49:43.855 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3249e-05 2023-10-09T10:49:43.855 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.855 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.855 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.856 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453897+0000", 2023-10-09T10:49:43.856 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1129999999999997e-06 2023-10-09T10:49:43.856 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.856 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.856 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.856 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453938+0000", 2023-10-09T10:49:43.856 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0799e-05 2023-10-09T10:49:43.856 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.856 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.856 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.856 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453945+0000", 2023-10-09T10:49:43.856 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2849999999999997e-06 2023-10-09T10:49:43.857 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.857 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.857 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.857 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104187+0000", 2023-10-09T10:49:43.857 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.65024259699999998 2023-10-09T10:49:43.857 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.857 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.857 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.857 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104195+0000", 2023-10-09T10:49:43.857 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9710000000000004e-06 2023-10-09T10:49:43.857 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.857 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.858 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.858 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.197298+0000", 2023-10-09T10:49:43.858 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.093102646 2023-10-09T10:49:43.858 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.858 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.858 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.858 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.197461+0000", 2023-10-09T10:49:43.858 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016253599999999999 2023-10-09T10:49:43.858 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.858 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.858 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.859 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.197621+0000", 2023-10-09T10:49:43.859 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000160178 2023-10-09T10:49:43.859 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.859 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.859 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.859 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200332+0000", 2023-10-09T10:49:43.859 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027108029999999999 2023-10-09T10:49:43.859 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.859 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.859 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.859 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206690+0000", 2023-10-09T10:49:43.859 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0063584990000000001 2023-10-09T10:49:43.860 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.860 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.860 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.860 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218563+0000", 2023-10-09T10:49:43.860 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011872562 2023-10-09T10:49:43.860 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.860 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.860 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.860 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218587+0000", 2023-10-09T10:49:43.860 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4586000000000001e-05 2023-10-09T10:49:43.861 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.861 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.861 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.861 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218899+0000", 2023-10-09T10:49:43.861 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000311883 2023-10-09T10:49:43.861 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.861 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.861 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.861 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.861 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:56713 6.10 6:086e6a33:::benchmark_data_smithi125_244951_object56712:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-10-09T10:49:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:08.647257+0000", 2023-10-09T10:49:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 275.06405630699999, 2023-10-09T10:49:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0123530810000001, 2023-10-09T10:49:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:49:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 56713 2023-10-09T10:49:43.862 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.863 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.863 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.863 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.863 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647257+0000", 2023-10-09T10:49:43.863 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.863 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.863 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.864 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.864 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647257+0000", 2023-10-09T10:49:43.864 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.864 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.864 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.864 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.864 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647256+0000", 2023-10-09T10:49:43.864 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:49:43.864 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.864 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.864 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.865 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647273+0000", 2023-10-09T10:49:43.865 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6075000000000001e-05 2023-10-09T10:49:43.865 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.865 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.865 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.865 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647281+0000", 2023-10-09T10:49:43.865 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6589999999999996e-06 2023-10-09T10:49:43.865 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.865 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.865 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.865 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647287+0000", 2023-10-09T10:49:43.865 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4940000000000003e-06 2023-10-09T10:49:43.866 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.866 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.866 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.866 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647464+0000", 2023-10-09T10:49:43.866 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017730999999999999 2023-10-09T10:49:43.866 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.866 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.866 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.866 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647667+0000", 2023-10-09T10:49:43.866 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020269 2023-10-09T10:49:43.866 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.867 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.867 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:49:43.867 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647835+0000", 2023-10-09T10:49:43.867 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000168677 2023-10-09T10:49:43.867 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.867 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.867 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.867 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.649455+0000", 2023-10-09T10:49:43.867 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016191999999999999 2023-10-09T10:49:43.867 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.867 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.867 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.868 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.655385+0000", 2023-10-09T10:49:43.868 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0059305549999999997 2023-10-09T10:49:43.868 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.868 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.868 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.868 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.658642+0000", 2023-10-09T10:49:43.868 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032573540000000001 2023-10-09T10:49:43.868 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.868 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.868 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:49:43.868 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.658657+0000", 2023-10-09T10:49:43.869 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4354e-05 2023-10-09T10:49:43.869 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.869 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.869 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.869 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.659536+0000", 2023-10-09T10:49:43.869 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0008789929999999 2023-10-09T10:49:43.869 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.869 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.869 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.869 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.659610+0000", 2023-10-09T10:49:43.869 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4134999999999994e-05 2023-10-09T10:49:43.869 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.870 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.870 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.870 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.870 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.870 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPush(6.11 155/143 [PushOp(6:8b7ae795:::benchmark_data_smithi125_244951_object18078:head, version: 104'1108, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:8b7ae795:::benchmark_data_smithi125_244951_object18078:head@104'1108, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:49:43.870 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:33.349257+0000", 2023-10-09T10:49:43.870 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 190.36205566699999, 2023-10-09T10:49:43.870 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1771969579999999, 2023-10-09T10:49:43.870 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.870 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:49:43.870 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.871 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.871 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.871 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349257+0000", 2023-10-09T10:49:43.871 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.871 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.871 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.871 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.871 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349257+0000", 2023-10-09T10:49:43.871 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.871 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.872 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.872 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.872 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349256+0000", 2023-10-09T10:49:43.872 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:43.872 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.872 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.872 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.872 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349302+0000", 2023-10-09T10:49:43.872 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5671999999999999e-05 2023-10-09T10:49:43.872 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.872 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.873 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.873 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349333+0000", 2023-10-09T10:49:43.873 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1256e-05 2023-10-09T10:49:43.873 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.873 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.873 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.873 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349351+0000", 2023-10-09T10:49:43.873 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8303000000000001e-05 2023-10-09T10:49:43.873 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.873 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.874 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.874 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.525188+0000", 2023-10-09T10:49:43.874 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.175836713 2023-10-09T10:49:43.874 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.874 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.874 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.874 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.525199+0000", 2023-10-09T10:49:43.874 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0458e-05 2023-10-09T10:49:43.874 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.874 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.874 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.875 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.526454+0000", 2023-10-09T10:49:43.875 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001255927 2023-10-09T10:49:43.875 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.875 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.875 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.875 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.875 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.875 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPush(6.11 155/143 [PushOp(6:8ff63630:::benchmark_data_smithi125_244951_object18084:head, version: 104'1109, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:8ff63630:::benchmark_data_smithi125_244951_object18084:head@104'1109, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:49:43.875 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:33.403563+0000", 2023-10-09T10:49:43.875 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 190.30775041499999, 2023-10-09T10:49:43.875 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1637633279999999, 2023-10-09T10:49:43.875 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.876 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:49:43.876 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.876 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.876 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.876 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403563+0000", 2023-10-09T10:49:43.876 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.876 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.876 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.876 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.876 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403563+0000", 2023-10-09T10:49:43.876 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.876 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.877 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.877 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.877 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403561+0000", 2023-10-09T10:49:43.877 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:49:43.877 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.877 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.877 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.877 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403594+0000", 2023-10-09T10:49:43.877 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2743999999999997e-05 2023-10-09T10:49:43.877 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.877 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.878 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.878 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403612+0000", 2023-10-09T10:49:43.878 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8389e-05 2023-10-09T10:49:43.878 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.878 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.878 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.878 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403626+0000", 2023-10-09T10:49:43.878 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3458999999999999e-05 2023-10-09T10:49:43.878 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.878 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.878 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.878 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.565733+0000", 2023-10-09T10:49:43.878 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.162107429 2023-10-09T10:49:43.879 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.879 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.879 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.879 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.565745+0000", 2023-10-09T10:49:43.879 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1355000000000001e-05 2023-10-09T10:49:43.879 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.879 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.879 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.879 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.567326+0000", 2023-10-09T10:49:43.879 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015813979999999999 2023-10-09T10:49:43.879 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.880 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.880 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.880 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.880 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.880 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:123629 6.9 6:932bc8aa:::benchmark_data_smithi125_244951_object17186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-10-09T10:49:43.880 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:33.543425+0000", 2023-10-09T10:49:43.880 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 190.16788782200001, 2023-10-09T10:49:43.880 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.002534984, 2023-10-09T10:49:43.880 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:43.880 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:49:43.880 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:43.881 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:49:43.881 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:49:43.881 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 123629 2023-10-09T10:49:43.881 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.881 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:43.881 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.881 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:43.881 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543425+0000", 2023-10-09T10:49:43.881 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.881 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.881 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.881 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:43.882 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543425+0000", 2023-10-09T10:49:43.882 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:43.882 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.882 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.882 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:43.882 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543424+0000", 2023-10-09T10:49:43.882 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:49:43.882 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.882 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.882 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:43.883 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543432+0000", 2023-10-09T10:49:43.883 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1389999999999995e-06 2023-10-09T10:49:43.883 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.883 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.883 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:43.883 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543448+0000", 2023-10-09T10:49:43.883 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5979000000000001e-05 2023-10-09T10:49:43.883 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.883 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.883 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:43.883 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543458+0000", 2023-10-09T10:49:43.884 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.859e-06 2023-10-09T10:49:43.884 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.884 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.884 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:43.884 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.520483+0000", 2023-10-09T10:49:43.884 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97702594300000001 2023-10-09T10:49:43.884 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.884 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.884 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:43.884 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.520610+0000", 2023-10-09T10:49:43.884 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012663799999999999 2023-10-09T10:49:43.884 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.885 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.885 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:49:43.885 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.520719+0000", 2023-10-09T10:49:43.885 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010933999999999999 2023-10-09T10:49:43.885 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.885 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.885 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.885 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.525132+0000", 2023-10-09T10:49:43.885 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0044124419999999999 2023-10-09T10:49:43.886 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.886 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.886 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:43.886 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.534848+0000", 2023-10-09T10:49:43.886 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0097164620000000004 2023-10-09T10:49:43.886 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.886 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.886 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:49:43.886 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.544664+0000", 2023-10-09T10:49:43.886 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0098156279999999999 2023-10-09T10:49:43.886 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.887 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.887 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:49:43.887 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.544725+0000", 2023-10-09T10:49:43.887 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1042000000000003e-05 2023-10-09T10:49:43.887 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:43.887 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:43.887 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:49:43.887 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.545960+0000", 2023-10-09T10:49:43.887 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001235311 2023-10-09T10:49:43.887 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.887 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.888 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.888 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:43.888 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:43.888 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:49:43.888 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:43.975 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:43.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:43.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:43.983 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 23 16 40000 39984 108.622 53.9375 0.0140144 0.00920053 2023-10-09T10:49:44.083 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:44.135 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:44.263 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:49:44.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:44.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:44.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:44.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:49:44.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:44 smithi125 ceph-mon[195507]: failed to encode map e205 with expected crc 2023-10-09T10:49:44.581 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:44 smithi125 ceph-mon[195507]: pgmap v461: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 peering, 5 remapped+peering, 99 active+clean; 2.3 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 83 MiB/s wr, 1.32k op/s; 4780/115023 objects degraded (4.156%); 2266/115023 objects misplaced (1.970%); 645 KiB/s, 12 objects/s recovering 2023-10-09T10:49:44.581 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:44 smithi125 ceph-mon[195507]: Health check failed: Degraded data redundancy: 4780/115023 objects degraded (4.156%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:49:44.581 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:44 smithi125 ceph-mon[195507]: osdmap e206: 8 total, 8 up, 6 in 2023-10-09T10:49:44.581 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:44 smithi125 ceph-mon[195507]: failed to encode map e206 with expected crc 2023-10-09T10:49:44.581 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:44 smithi125 ceph-mon[195507]: failed to encode map e206 with expected crc 2023-10-09T10:49:44.582 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:44 smithi125 ceph-mon[200197]: failed to encode map e205 with expected crc 2023-10-09T10:49:44.582 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:44 smithi125 ceph-mon[200197]: pgmap v461: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 peering, 5 remapped+peering, 99 active+clean; 2.3 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 83 MiB/s wr, 1.32k op/s; 4780/115023 objects degraded (4.156%); 2266/115023 objects misplaced (1.970%); 645 KiB/s, 12 objects/s recovering 2023-10-09T10:49:44.582 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:44 smithi125 ceph-mon[200197]: Health check failed: Degraded data redundancy: 4780/115023 objects degraded (4.156%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:49:44.582 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:44 smithi125 ceph-mon[200197]: osdmap e206: 8 total, 8 up, 6 in 2023-10-09T10:49:44.582 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:44 smithi125 ceph-mon[200197]: failed to encode map e206 with expected crc 2023-10-09T10:49:44.582 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:44 smithi125 ceph-mon[200197]: failed to encode map e206 with expected crc 2023-10-09T10:49:44.588 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:44.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:44 smithi155 ceph-mon[184854]: failed to encode map e205 with expected crc 2023-10-09T10:49:44.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:44 smithi155 ceph-mon[184854]: pgmap v461: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 peering, 5 remapped+peering, 99 active+clean; 2.3 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 83 MiB/s wr, 1.32k op/s; 4780/115023 objects degraded (4.156%); 2266/115023 objects misplaced (1.970%); 645 KiB/s, 12 objects/s recovering 2023-10-09T10:49:44.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:44 smithi155 ceph-mon[184854]: Health check failed: Degraded data redundancy: 4780/115023 objects degraded (4.156%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T10:49:44.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:44 smithi155 ceph-mon[184854]: osdmap e206: 8 total, 8 up, 6 in 2023-10-09T10:49:44.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:44 smithi155 ceph-mon[184854]: failed to encode map e206 with expected crc 2023-10-09T10:49:44.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:44 smithi155 ceph-mon[184854]: failed to encode map e206 with expected crc 2023-10-09T10:49:44.727 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:49:44.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:44.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:49:44.832 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:44.905 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:44.929 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:44.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:44.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:44.983 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 24 16 40957 40941 106.588 59.8125 0.0170832 0.00937447 2023-10-09T10:49:45.032 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:45.037 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:49:45.111 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:45.111 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:45.153 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:45.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:45.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:45.261 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:45.276 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:49:45.291 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:45.312 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:45.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:45.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:45.416 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:45.449 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:45.470 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:45.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:45.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:45.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:45 smithi125 ceph-mon[195507]: failed to encode map e206 with expected crc 2023-10-09T10:49:45.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:45 smithi125 ceph-mon[195507]: failed to encode map e206 with expected crc 2023-10-09T10:49:45.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:45.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:49:45.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:45 smithi125 ceph-mon[200197]: failed to encode map e206 with expected crc 2023-10-09T10:49:45.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:45 smithi125 ceph-mon[200197]: failed to encode map e206 with expected crc 2023-10-09T10:49:45.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:45.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:49:45.574 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:45.601 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:45 smithi155 ceph-mon[184854]: failed to encode map e206 with expected crc 2023-10-09T10:49:45.601 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:45 smithi155 ceph-mon[184854]: failed to encode map e206 with expected crc 2023-10-09T10:49:45.602 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:45.602 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:49:45.604 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:45.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:45.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:45.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:45.732 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:49:45.788 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:45.983 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 25 16 41811 41795 104.46 53.375 0.0149061 0.00956668 2023-10-09T10:49:46.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:46 smithi125 ceph-mon[195507]: 1.13 scrub starts 2023-10-09T10:49:46.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:46 smithi125 ceph-mon[195507]: pgmap v463: 109 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 peering, 5 remapped+peering, 97 active+clean; 2.4 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 63 MiB/s wr, 997 op/s; 7249/117243 objects degraded (6.183%); 4498/117243 objects misplaced (3.836%); 2.2 MiB/s, 38 objects/s recovering 2023-10-09T10:49:46.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:46 smithi125 ceph-mon[200197]: 1.13 scrub starts 2023-10-09T10:49:46.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:46 smithi125 ceph-mon[200197]: pgmap v463: 109 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 peering, 5 remapped+peering, 97 active+clean; 2.4 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 63 MiB/s wr, 997 op/s; 7249/117243 objects degraded (6.183%); 4498/117243 objects misplaced (3.836%); 2.2 MiB/s, 38 objects/s recovering 2023-10-09T10:49:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:46 smithi155 ceph-mon[184854]: 1.13 scrub starts 2023-10-09T10:49:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:46 smithi155 ceph-mon[184854]: pgmap v463: 109 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 peering, 5 remapped+peering, 97 active+clean; 2.4 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 63 MiB/s wr, 997 op/s; 7249/117243 objects degraded (6.183%); 4498/117243 objects misplaced (3.836%); 2.2 MiB/s, 38 objects/s recovering 2023-10-09T10:49:46.875 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:46.983 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 26 16 42966 42950 103.219 72.1875 0.0119051 0.00968221 2023-10-09T10:49:47.468 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 0] out_osds: [2, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:49:47.468 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:49:47.469 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [1, 3, 4, 5, 6, 0] 2023-10-09T10:49:47.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd out 1 2023-10-09T10:49:47.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:47 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T10:49:47.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:47 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T10:49:47.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:47 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T10:49:47.983 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 27 16 44056 44040 101.918 68.125 0.0140837 0.00980612 2023-10-09T10:49:48.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:48 smithi155 ceph-mon[184854]: pgmap v464: 109 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 100 active+clean; 2.5 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 53 MiB/s wr, 843 op/s; 9657/121851 objects degraded (7.925%); 4498/121851 objects misplaced (3.691%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T10:49:48.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:48 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:48.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:48 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:48.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:48 smithi125 ceph-mon[195507]: pgmap v464: 109 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 100 active+clean; 2.5 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 53 MiB/s wr, 843 op/s; 9657/121851 objects degraded (7.925%); 4498/121851 objects misplaced (3.691%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T10:49:48.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:48 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:48.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:48 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:48.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:48 smithi125 ceph-mon[200197]: pgmap v464: 109 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 100 active+clean; 2.5 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 53 MiB/s wr, 843 op/s; 9657/121851 objects degraded (7.925%); 4498/121851 objects misplaced (3.691%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T10:49:48.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:48 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:48.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:48 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:48.984 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 28 16 45095 45079 100.598 64.9375 0.00868069 0.00993413 2023-10-09T10:49:49.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:49 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 9657/127509 objects degraded (7.574%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:49:49.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:49 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 9657/127509 objects degraded (7.574%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:49:49.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:49 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 9657/127509 objects degraded (7.574%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:49:49.984 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 29 16 46152 46136 99.4066 66.0625 0.0206286 0.0100523 2023-10-09T10:49:50.012 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:49:50.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:50.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:49:50.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:50.172 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:50.200 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:50.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:50.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:50.258 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:50.309 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:50.342 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:50.371 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:50.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:50.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:50.476 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:49:50.553 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:50 smithi155 ceph-mon[184854]: pgmap v465: 109 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.6 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 49 MiB/s wr, 775 op/s; 9657/127509 objects degraded (7.574%); 4203/127509 objects misplaced (3.296%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T10:49:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:50 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:49:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:50 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:49:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:50 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:50.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:50 smithi125 ceph-mon[195507]: pgmap v465: 109 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.6 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 49 MiB/s wr, 775 op/s; 9657/127509 objects degraded (7.574%); 4203/127509 objects misplaced (3.296%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T10:49:50.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:50 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:49:50.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:50 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:49:50.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:50 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:50.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:50 smithi125 ceph-mon[200197]: pgmap v465: 109 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.6 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 49 MiB/s wr, 775 op/s; 9657/127509 objects degraded (7.574%); 4203/127509 objects misplaced (3.296%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T10:49:50.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:50 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:49:50.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:50 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:49:50.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:50 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:50.838 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:49:50.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:50.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:49:50.944 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:50.979 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:50.985 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 30 16 47156 47140 98.1795 62.75 0.0101507 0.0101778 2023-10-09T10:49:51.002 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:51.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:51.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:51.110 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:49:51.154 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:51.241 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:49:51.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:51.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:49:51.345 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:51.378 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:51.398 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:51.403 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:51.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:51.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:51.509 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:51.527 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:49:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:49:51] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.43.0" 2023-10-09T10:49:51.542 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:51.571 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:51.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:51.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:51.674 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:51.742 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:51.745 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:49:51.746 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:49:51.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:51.746 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:47748 7.d 7:ba7c5374:::benchmark_data_smithi125_295816_object47747:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-10-09T10:49:51.746 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:49:51.641200+0000", 2023-10-09T10:49:51.746 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0030373790000000002, 2023-10-09T10:49:51.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003074411, 2023-10-09T10:49:51.746 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:49:51.746 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:49:51.746 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:49:51.746 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:49:51.747 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:49:51.747 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 47748 2023-10-09T10:49:51.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:51.747 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:49:51.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:51.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:49:51.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:51.641200+0000", 2023-10-09T10:49:51.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:51.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:51.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:51.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:49:51.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:51.641200+0000", 2023-10-09T10:49:51.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:49:51.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:51.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:51.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:49:51.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:51.641198+0000", 2023-10-09T10:49:51.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:49:51.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:51.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:51.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:49:51.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:51.641232+0000", 2023-10-09T10:49:51.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3933000000000002e-05 2023-10-09T10:49:51.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:51.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:51.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:49:51.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:51.641249+0000", 2023-10-09T10:49:51.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7116999999999999e-05 2023-10-09T10:49:51.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:51.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:51.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:49:51.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:51.641259+0000", 2023-10-09T10:49:51.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8660000000000001e-06 2023-10-09T10:49:51.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:51.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:51.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:49:51.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:51.641300+0000", 2023-10-09T10:49:51.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1012e-05 2023-10-09T10:49:51.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:51.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:51.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:49:51.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:51.641426+0000", 2023-10-09T10:49:51.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012659600000000001 2023-10-09T10:49:51.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:51.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:51.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:49:51.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:51.641562+0000", 2023-10-09T10:49:51.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013589299999999999 2023-10-09T10:49:51.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:49:51.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:49:51.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:49:51.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:49:51.643533+0000", 2023-10-09T10:49:51.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001970954 2023-10-09T10:49:51.751 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:51.751 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:49:51.751 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:51.751 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:49:51.751 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:49:51.751 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 1 2023-10-09T10:49:51.751 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:49:51.822 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:51.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:51.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:51.927 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:51.971 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:51.985 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 31 16 48092 48076 96.8994 58.5 0.0426827 0.0103153 2023-10-09T10:49:52.367 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:52.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:52.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:52.473 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:52.507 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:52.533 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:52.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:52.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:52.638 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:52.669 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:52.692 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:52.695 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:52 smithi155 ceph-mon[184854]: 1.3 scrub starts 2023-10-09T10:49:52.695 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:52 smithi155 ceph-mon[184854]: pgmap v466: 109 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 61 MiB/s wr, 967 op/s; 9657/135219 objects degraded (7.142%); 3975/135219 objects misplaced (2.940%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T10:49:52.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:52.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:52.722 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:52 smithi125 ceph-mon[195507]: 1.3 scrub starts 2023-10-09T10:49:52.722 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:52 smithi125 ceph-mon[195507]: pgmap v466: 109 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 61 MiB/s wr, 967 op/s; 9657/135219 objects degraded (7.142%); 3975/135219 objects misplaced (2.940%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T10:49:52.723 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:52 smithi125 ceph-mon[200197]: 1.3 scrub starts 2023-10-09T10:49:52.723 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:52 smithi125 ceph-mon[200197]: pgmap v466: 109 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 61 MiB/s wr, 967 op/s; 9657/135219 objects degraded (7.142%); 3975/135219 objects misplaced (2.940%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T10:49:52.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:52.830 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:52.853 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:52.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:52.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:52.957 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:52.986 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 32 16 48957 48941 95.5588 54.0625 0.0292121 0.0104578 2023-10-09T10:49:52.988 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:53.011 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:53.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:53.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:53.115 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:49:53.153 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:53.184 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:49:53.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:53.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:49:53.259 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:49:53.289 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:49:53.321 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:53.342 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:49:53.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:53.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:49:53.447 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:53.477 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:53.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:53.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:53.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:53.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:53.634 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:49:53.636 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:53.660 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:53.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:53.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:53.682 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:49:53.722 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:49:53.763 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:49:53.774 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:49:53.787 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:49:53.814 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:49:53.822 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:53.870 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:49:53.882 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:49:53.928 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:49:53.947 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:49:53.947 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:49:53.947 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:49:53.948 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:49:53.948 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:49:53.948 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:49:53.948 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:49:53.948 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:49:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:53 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/591460042' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:49:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:53 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:49:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:53 smithi155 ceph-mon[184854]: pgmap v467: 109 pgs: 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 2.9 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 58 MiB/s wr, 922 op/s; 9657/142770 objects degraded (6.764%); 3647/142770 objects misplaced (2.554%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T10:49:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:53 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3037976557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:53.979 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:53 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/591460042' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:49:53.979 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:53 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:49:53.979 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:53 smithi125 ceph-mon[195507]: pgmap v467: 109 pgs: 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 2.9 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 58 MiB/s wr, 922 op/s; 9657/142770 objects degraded (6.764%); 3647/142770 objects misplaced (2.554%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T10:49:53.979 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:53 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3037976557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:53.982 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:53 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/591460042' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:49:53.982 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:53 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:49:53.982 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:53 smithi125 ceph-mon[200197]: pgmap v467: 109 pgs: 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 2.9 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 58 MiB/s wr, 922 op/s; 9657/142770 objects degraded (6.764%); 3647/142770 objects misplaced (2.554%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T10:49:53.982 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:53 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3037976557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:49:53.986 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 33 16 49751 49735 94.1671 49.625 0.015351 0.0106151 2023-10-09T10:49:54.101 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:49:54.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:54.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:49:54.211 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:54.245 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:54.268 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:54.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:54.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:54.373 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:49:54.410 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:54.438 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:49:54.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:54.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:49:54.541 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:54.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T10:49:54.338+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e207 definitely_dead 0 2023-10-09T10:49:54.597 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:54.610 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:54.666 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:54.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:54.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:54.770 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:54.796 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.1. 2023-10-09T10:49:54.811 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:54.835 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:54.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:54.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:54.848 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[195507]: 1.13 scrub starts 2023-10-09T10:49:54.848 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:49:54.848 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[195507]: failed to encode map e207 with expected crc 2023-10-09T10:49:54.848 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:49:54.848 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[195507]: osdmap e207: 8 total, 8 up, 6 in 2023-10-09T10:49:54.848 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[195507]: failed to encode map e207 with expected crc 2023-10-09T10:49:54.848 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3743474002' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:49:54.849 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:49:54.849 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[195507]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T10:49:54.849 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[200197]: 1.13 scrub starts 2023-10-09T10:49:54.849 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:49:54.849 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[200197]: failed to encode map e207 with expected crc 2023-10-09T10:49:54.849 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:49:54.849 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[200197]: osdmap e207: 8 total, 8 up, 6 in 2023-10-09T10:49:54.849 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[200197]: failed to encode map e207 with expected crc 2023-10-09T10:49:54.849 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3743474002' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:49:54.849 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:49:54.850 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:54 smithi125 ceph-mon[200197]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T10:49:54.940 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:49:54.972 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:54.986 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 34 16 50506 50490 92.7855 47.1875 0.00911114 0.0107178 2023-10-09T10:49:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:54 smithi155 ceph-mon[184854]: 1.13 scrub starts 2023-10-09T10:49:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:54 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:49:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:54 smithi155 ceph-mon[184854]: failed to encode map e207 with expected crc 2023-10-09T10:49:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:54 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:49:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:54 smithi155 ceph-mon[184854]: osdmap e207: 8 total, 8 up, 6 in 2023-10-09T10:49:55.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:54 smithi155 ceph-mon[184854]: failed to encode map e207 with expected crc 2023-10-09T10:49:55.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:54 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3743474002' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:49:55.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:54 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:49:55.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:54 smithi155 ceph-mon[184854]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T10:49:55.986 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 35 16 50506 50490 90.135 0 - 0.0107178 2023-10-09T10:49:56.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T10:49:56.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[195507]: osdmap e208: 8 total, 8 up, 5 in 2023-10-09T10:49:56.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[195507]: failed to encode map e208 with expected crc 2023-10-09T10:49:56.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[195507]: failed to encode map e208 with expected crc 2023-10-09T10:49:56.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[195507]: failed to encode map e208 with expected crc 2023-10-09T10:49:56.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[195507]: failed to encode map e208 with expected crc 2023-10-09T10:49:56.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[195507]: pgmap v470: 109 pgs: 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 3.0 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s; 9657/146163 objects degraded (6.607%); 3435/146163 objects misplaced (2.350%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T10:49:56.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:56.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T10:49:56.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[200197]: osdmap e208: 8 total, 8 up, 5 in 2023-10-09T10:49:56.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[200197]: failed to encode map e208 with expected crc 2023-10-09T10:49:56.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[200197]: failed to encode map e208 with expected crc 2023-10-09T10:49:56.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[200197]: failed to encode map e208 with expected crc 2023-10-09T10:49:56.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[200197]: failed to encode map e208 with expected crc 2023-10-09T10:49:56.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[200197]: pgmap v470: 109 pgs: 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 3.0 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s; 9657/146163 objects degraded (6.607%); 3435/146163 objects misplaced (2.350%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T10:49:56.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:55 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:56.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:55 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T10:49:56.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:55 smithi155 ceph-mon[184854]: osdmap e208: 8 total, 8 up, 5 in 2023-10-09T10:49:56.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:55 smithi155 ceph-mon[184854]: failed to encode map e208 with expected crc 2023-10-09T10:49:56.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:55 smithi155 ceph-mon[184854]: failed to encode map e208 with expected crc 2023-10-09T10:49:56.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:55 smithi155 ceph-mon[184854]: failed to encode map e208 with expected crc 2023-10-09T10:49:56.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:55 smithi155 ceph-mon[184854]: failed to encode map e208 with expected crc 2023-10-09T10:49:56.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:55 smithi155 ceph-mon[184854]: pgmap v470: 109 pgs: 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 3.0 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s; 9657/146163 objects degraded (6.607%); 3435/146163 objects misplaced (2.350%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T10:49:56.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:55 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:49:56.339 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:49:56.637 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:49:56.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 36 16 50509 50493 87.637 0.09375 2.17688 0.0108466 2023-10-09T10:49:57.007 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:56 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 9657/146163 objects degraded (6.607%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:49:57.007 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:56 smithi125 ceph-mon[200197]: failed to encode map e209 with expected crc 2023-10-09T10:49:57.007 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:56 smithi125 ceph-mon[200197]: failed to encode map e209 with expected crc 2023-10-09T10:49:57.007 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:56 smithi125 ceph-mon[200197]: failed to encode map e209 with expected crc 2023-10-09T10:49:57.007 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:56 smithi125 ceph-mon[200197]: osdmap e209: 8 total, 8 up, 5 in 2023-10-09T10:49:57.007 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:56 smithi125 ceph-mon[200197]: failed to encode map e209 with expected crc 2023-10-09T10:49:57.007 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:56 smithi125 ceph-mon[200197]: failed to encode map e209 with expected crc 2023-10-09T10:49:57.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:56 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 9657/146163 objects degraded (6.607%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:49:57.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:56 smithi125 ceph-mon[195507]: failed to encode map e209 with expected crc 2023-10-09T10:49:57.008 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:56 smithi125 ceph-mon[195507]: failed to encode map e209 with expected crc 2023-10-09T10:49:57.008 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:56 smithi125 ceph-mon[195507]: failed to encode map e209 with expected crc 2023-10-09T10:49:57.008 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:56 smithi125 ceph-mon[195507]: osdmap e209: 8 total, 8 up, 5 in 2023-10-09T10:49:57.008 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:56 smithi125 ceph-mon[195507]: failed to encode map e209 with expected crc 2023-10-09T10:49:57.008 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:56 smithi125 ceph-mon[195507]: failed to encode map e209 with expected crc 2023-10-09T10:49:57.038 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:49:57.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:57.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:49:57.143 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:49:57.190 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:56 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 9657/146163 objects degraded (6.607%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:49:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:56 smithi155 ceph-mon[184854]: failed to encode map e209 with expected crc 2023-10-09T10:49:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:56 smithi155 ceph-mon[184854]: failed to encode map e209 with expected crc 2023-10-09T10:49:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:56 smithi155 ceph-mon[184854]: failed to encode map e209 with expected crc 2023-10-09T10:49:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:56 smithi155 ceph-mon[184854]: osdmap e209: 8 total, 8 up, 5 in 2023-10-09T10:49:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:56 smithi155 ceph-mon[184854]: failed to encode map e209 with expected crc 2023-10-09T10:49:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:56 smithi155 ceph-mon[184854]: failed to encode map e209 with expected crc 2023-10-09T10:49:57.813 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[200197]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.813 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[200197]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.813 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[200197]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.813 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[200197]: osdmap e210: 8 total, 8 up, 5 in 2023-10-09T10:49:57.813 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[200197]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.814 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[200197]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.814 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[200197]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.814 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[200197]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.814 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[200197]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.814 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[200197]: pgmap v473: 109 pgs: 7 remapped+peering, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.0 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 33 MiB/s wr, 524 op/s; 7251/149061 objects degraded (4.864%); 5794/149061 objects misplaced (3.887%); 4.0 MiB/s, 65 objects/s recovering 2023-10-09T10:49:57.814 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[200197]: osdmap e211: 8 total, 8 up, 5 in 2023-10-09T10:49:57.815 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[195507]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.815 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[195507]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.815 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[195507]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.815 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[195507]: osdmap e210: 8 total, 8 up, 5 in 2023-10-09T10:49:57.815 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[195507]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[195507]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[195507]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[195507]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[195507]: failed to encode map e210 with expected crc 2023-10-09T10:49:57.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[195507]: pgmap v473: 109 pgs: 7 remapped+peering, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.0 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 33 MiB/s wr, 524 op/s; 7251/149061 objects degraded (4.864%); 5794/149061 objects misplaced (3.887%); 4.0 MiB/s, 65 objects/s recovering 2023-10-09T10:49:57.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:57 smithi125 ceph-mon[195507]: osdmap e211: 8 total, 8 up, 5 in 2023-10-09T10:49:57.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 37 16 51399 51383 86.7718 55.625 0.016412 0.0115177 2023-10-09T10:49:58.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:57 smithi155 ceph-mon[184854]: failed to encode map e210 with expected crc 2023-10-09T10:49:58.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:57 smithi155 ceph-mon[184854]: failed to encode map e210 with expected crc 2023-10-09T10:49:58.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:57 smithi155 ceph-mon[184854]: failed to encode map e210 with expected crc 2023-10-09T10:49:58.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:57 smithi155 ceph-mon[184854]: osdmap e210: 8 total, 8 up, 5 in 2023-10-09T10:49:58.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:57 smithi155 ceph-mon[184854]: failed to encode map e210 with expected crc 2023-10-09T10:49:58.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:57 smithi155 ceph-mon[184854]: failed to encode map e210 with expected crc 2023-10-09T10:49:58.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:57 smithi155 ceph-mon[184854]: failed to encode map e210 with expected crc 2023-10-09T10:49:58.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:57 smithi155 ceph-mon[184854]: failed to encode map e210 with expected crc 2023-10-09T10:49:58.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:57 smithi155 ceph-mon[184854]: failed to encode map e210 with expected crc 2023-10-09T10:49:58.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:57 smithi155 ceph-mon[184854]: pgmap v473: 109 pgs: 7 remapped+peering, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.0 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 33 MiB/s wr, 524 op/s; 7251/149061 objects degraded (4.864%); 5794/149061 objects misplaced (3.887%); 4.0 MiB/s, 65 objects/s recovering 2023-10-09T10:49:58.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:57 smithi155 ceph-mon[184854]: osdmap e211: 8 total, 8 up, 5 in 2023-10-09T10:49:58.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 38 16 52389 52373 86.1166 61.875 0.0187813 0.0116068 2023-10-09T10:49:59.019 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:59.066 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:49:59.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[195507]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[195507]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[195507]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[195507]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[195507]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[195507]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[195507]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[195507]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[195507]: osdmap e212: 8 total, 8 up, 5 in 2023-10-09T10:49:59.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[195507]: failed to encode map e212 with expected crc 2023-10-09T10:49:59.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[200197]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[200197]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[200197]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[200197]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[200197]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[200197]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[200197]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[200197]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[200197]: osdmap e212: 8 total, 8 up, 5 in 2023-10-09T10:49:59.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:58 smithi125 ceph-mon[200197]: failed to encode map e212 with expected crc 2023-10-09T10:49:59.100 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:49:59.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:59.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:49:59.210 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:49:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:58 smithi155 ceph-mon[184854]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:58 smithi155 ceph-mon[184854]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:58 smithi155 ceph-mon[184854]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:58 smithi155 ceph-mon[184854]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:58 smithi155 ceph-mon[184854]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:58 smithi155 ceph-mon[184854]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:58 smithi155 ceph-mon[184854]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:58 smithi155 ceph-mon[184854]: failed to encode map e211 with expected crc 2023-10-09T10:49:59.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:58 smithi155 ceph-mon[184854]: osdmap e212: 8 total, 8 up, 5 in 2023-10-09T10:49:59.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:58 smithi155 ceph-mon[184854]: failed to encode map e212 with expected crc 2023-10-09T10:49:59.267 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:59.751 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:49:59.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:59.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:49:59.855 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:49:59.912 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:49:59.945 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:49:59.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:49:59.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:49:59.985 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:59 smithi125 ceph-mon[195507]: failed to encode map e212 with expected crc 2023-10-09T10:49:59.985 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:59 smithi125 ceph-mon[195507]: failed to encode map e212 with expected crc 2023-10-09T10:49:59.985 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:59 smithi125 ceph-mon[195507]: failed to encode map e212 with expected crc 2023-10-09T10:49:59.986 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:59 smithi125 ceph-mon[195507]: failed to encode map e212 with expected crc 2023-10-09T10:49:59.986 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:59 smithi125 ceph-mon[195507]: failed to encode map e212 with expected crc 2023-10-09T10:49:59.986 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:49:59 smithi125 ceph-mon[195507]: pgmap v476: 109 pgs: 2 peering, 7 remapped+peering, 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 7.0 KiB/s rd, 33 MiB/s wr, 540 op/s; 21181/152586 objects degraded (13.881%); 4151/152586 objects misplaced (2.720%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T10:49:59.986 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:59 smithi125 ceph-mon[200197]: failed to encode map e212 with expected crc 2023-10-09T10:49:59.986 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:59 smithi125 ceph-mon[200197]: failed to encode map e212 with expected crc 2023-10-09T10:49:59.986 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:59 smithi125 ceph-mon[200197]: failed to encode map e212 with expected crc 2023-10-09T10:49:59.986 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:59 smithi125 ceph-mon[200197]: failed to encode map e212 with expected crc 2023-10-09T10:49:59.986 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:59 smithi125 ceph-mon[200197]: failed to encode map e212 with expected crc 2023-10-09T10:49:59.987 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:49:59 smithi125 ceph-mon[200197]: pgmap v476: 109 pgs: 2 peering, 7 remapped+peering, 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 7.0 KiB/s rd, 33 MiB/s wr, 540 op/s; 21181/152586 objects degraded (13.881%); 4151/152586 objects misplaced (2.720%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T10:49:59.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 39 16 53293 53277 85.3572 56.5 0.0197742 0.0117093 2023-10-09T10:50:00.060 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:00.111 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:59 smithi155 ceph-mon[184854]: failed to encode map e212 with expected crc 2023-10-09T10:50:00.111 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:59 smithi155 ceph-mon[184854]: failed to encode map e212 with expected crc 2023-10-09T10:50:00.111 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:59 smithi155 ceph-mon[184854]: failed to encode map e212 with expected crc 2023-10-09T10:50:00.111 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:59 smithi155 ceph-mon[184854]: failed to encode map e212 with expected crc 2023-10-09T10:50:00.111 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:59 smithi155 ceph-mon[184854]: failed to encode map e212 with expected crc 2023-10-09T10:50:00.111 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:49:59 smithi155 ceph-mon[184854]: pgmap v476: 109 pgs: 2 peering, 7 remapped+peering, 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 7.0 KiB/s rd, 33 MiB/s wr, 540 op/s; 21181/152586 objects degraded (13.881%); 4151/152586 objects misplaced (2.720%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T10:50:00.115 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:00.156 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:00.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:00.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:00.265 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:00.335 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:00.369 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:00.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:00.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:00.480 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:00.557 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:00.934 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:50:00.934 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:50:00.952 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:00.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:00.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:00.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:50:00.988233+0000 min lat: 0.00131548 max lat: 2.28015 avg lat: 0.0118116 2023-10-09T10:50:00.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:50:00.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 40 16 54189 54173 84.6224 56 0.0174239 0.0118116 2023-10-09T10:50:01.056 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:01.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[195507]: Health detail: HEALTH_WARN Degraded data redundancy: 21181/152586 objects degraded (13.881%), 8 pgs degraded 2023-10-09T10:50:01.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[195507]: [WRN] PG_DEGRADED: Degraded data redundancy: 21181/152586 objects degraded (13.881%), 8 pgs degraded 2023-10-09T10:50:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[195507]: pg 7.2 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2023-10-09T10:50:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[195507]: pg 7.4 is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-10-09T10:50:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[195507]: pg 7.5 is active+recovery_wait+undersized+degraded+remapped, acting [0,6] 2023-10-09T10:50:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[195507]: pg 7.6 is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-10-09T10:50:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[195507]: pg 7.8 is active+recovering+undersized+degraded+remapped, acting [4,3] 2023-10-09T10:50:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[195507]: pg 7.9 is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-10-09T10:50:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[195507]: pg 7.d is active+recovery_wait+undersized+degraded+remapped, acting [0,6] 2023-10-09T10:50:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[195507]: pg 7.f is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-10-09T10:50:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:50:01.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[200197]: Health detail: HEALTH_WARN Degraded data redundancy: 21181/152586 objects degraded (13.881%), 8 pgs degraded 2023-10-09T10:50:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[200197]: [WRN] PG_DEGRADED: Degraded data redundancy: 21181/152586 objects degraded (13.881%), 8 pgs degraded 2023-10-09T10:50:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[200197]: pg 7.2 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2023-10-09T10:50:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[200197]: pg 7.4 is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-10-09T10:50:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[200197]: pg 7.5 is active+recovery_wait+undersized+degraded+remapped, acting [0,6] 2023-10-09T10:50:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[200197]: pg 7.6 is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-10-09T10:50:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[200197]: pg 7.8 is active+recovering+undersized+degraded+remapped, acting [4,3] 2023-10-09T10:50:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[200197]: pg 7.9 is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-10-09T10:50:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[200197]: pg 7.d is active+recovery_wait+undersized+degraded+remapped, acting [0,6] 2023-10-09T10:50:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[200197]: pg 7.f is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-10-09T10:50:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:50:01.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:00 smithi155 ceph-mon[184854]: Health detail: HEALTH_WARN Degraded data redundancy: 21181/152586 objects degraded (13.881%), 8 pgs degraded 2023-10-09T10:50:01.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:00 smithi155 ceph-mon[184854]: [WRN] PG_DEGRADED: Degraded data redundancy: 21181/152586 objects degraded (13.881%), 8 pgs degraded 2023-10-09T10:50:01.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:00 smithi155 ceph-mon[184854]: pg 7.2 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2023-10-09T10:50:01.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:00 smithi155 ceph-mon[184854]: pg 7.4 is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-10-09T10:50:01.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:00 smithi155 ceph-mon[184854]: pg 7.5 is active+recovery_wait+undersized+degraded+remapped, acting [0,6] 2023-10-09T10:50:01.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:00 smithi155 ceph-mon[184854]: pg 7.6 is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-10-09T10:50:01.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:00 smithi155 ceph-mon[184854]: pg 7.8 is active+recovering+undersized+degraded+remapped, acting [4,3] 2023-10-09T10:50:01.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:00 smithi155 ceph-mon[184854]: pg 7.9 is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-10-09T10:50:01.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:00 smithi155 ceph-mon[184854]: pg 7.d is active+recovery_wait+undersized+degraded+remapped, acting [0,6] 2023-10-09T10:50:01.103 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:00 smithi155 ceph-mon[184854]: pg 7.f is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-10-09T10:50:01.103 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:01.103 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:50:01.103 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:01.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:01.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:01.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:01.250 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:01.308 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:01.341 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:50:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:50:01] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.43.0" 2023-10-09T10:50:01.582 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:01.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:01.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:01.621 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:50:01.624 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:50:01.652 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:50:01.692 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:50:01.695 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:01.741 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:50:01.742 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:01.772 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:01.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:01.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:01.785 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:50:01.843 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:50:01.879 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:01.897 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:50:01.926 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:01.936 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:50:01.951 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:50:01.951 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:50:01.951 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:50:01.951 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:50:01.952 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:50:01.952 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:50:01.952 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:50:01.952 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:50:01.956 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:01.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:01.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:01.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 41 16 55019 55003 83.8237 51.875 0.0330933 0.0119237 2023-10-09T10:50:02.031 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0] out_osds: [2, 7, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:50:02.031 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:50:02.031 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2023-10-09T10:50:02.031 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:50:02.064 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:02.114 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:01 smithi155 ceph-mon[184854]: pgmap v477: 109 pgs: 1 active+recovering+undersized+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.2 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 5.2 KiB/s rd, 46 MiB/s wr, 738 op/s; 33867/158118 objects degraded (21.419%); 3177/158118 objects misplaced (2.009%); 16 MiB/s, 250 objects/s recovering 2023-10-09T10:50:02.115 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/54843937' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:02.115 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:02.154 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:02.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:02.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:02.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:01 smithi125 ceph-mon[195507]: pgmap v477: 109 pgs: 1 active+recovering+undersized+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.2 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 5.2 KiB/s rd, 46 MiB/s wr, 738 op/s; 33867/158118 objects degraded (21.419%); 3177/158118 objects misplaced (2.009%); 16 MiB/s, 250 objects/s recovering 2023-10-09T10:50:02.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/54843937' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:02.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:01 smithi125 ceph-mon[200197]: pgmap v477: 109 pgs: 1 active+recovering+undersized+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.2 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 5.2 KiB/s rd, 46 MiB/s wr, 738 op/s; 33867/158118 objects degraded (21.419%); 3177/158118 objects misplaced (2.009%); 16 MiB/s, 250 objects/s recovering 2023-10-09T10:50:02.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/54843937' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:02.260 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:02.341 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:02.437 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:02.439 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:02.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:02.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:02.547 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:02.603 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:02.645 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:02.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:02.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:02.754 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:02.811 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:02.895 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:50:02.904 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:02.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:02.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:02.920 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:02.988 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 42 16 55833 55817 83.0374 50.875 0.0307814 0.012037 2023-10-09T10:50:03.014 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:03.065 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:03.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:02 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 33867/158118 objects degraded (21.419%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:03.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:02 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/315855563' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:50:03.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:02 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:50:03.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:02 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 33867/158118 objects degraded (21.419%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:03.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:02 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/315855563' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:50:03.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:02 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:50:03.095 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:03.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:03.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:03.202 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:03.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:02 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 33867/158118 objects degraded (21.419%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:03.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:02 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/315855563' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:50:03.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:02 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:50:03.249 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:03.297 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:03.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:03.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:03.400 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:03.463 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:03.491 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:03.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:03.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:03.597 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:03.666 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:03.916 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:03.916 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:50:03.916 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[195507]: osdmap e213: 8 total, 8 up, 5 in 2023-10-09T10:50:03.916 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[195507]: failed to encode map e213 with expected crc 2023-10-09T10:50:03.916 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[195507]: failed to encode map e213 with expected crc 2023-10-09T10:50:03.917 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[195507]: failed to encode map e213 with expected crc 2023-10-09T10:50:03.917 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[195507]: failed to encode map e213 with expected crc 2023-10-09T10:50:03.917 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[195507]: failed to encode map e213 with expected crc 2023-10-09T10:50:03.917 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[195507]: failed to encode map e213 with expected crc 2023-10-09T10:50:03.917 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[195507]: pgmap v479: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 4.7 KiB/s rd, 52 MiB/s wr, 830 op/s; 33867/163977 objects degraded (20.654%); 3177/163977 objects misplaced (1.937%); 14 MiB/s, 218 objects/s recovering 2023-10-09T10:50:03.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:03.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:50:03.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[200197]: osdmap e213: 8 total, 8 up, 5 in 2023-10-09T10:50:03.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[200197]: failed to encode map e213 with expected crc 2023-10-09T10:50:03.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[200197]: failed to encode map e213 with expected crc 2023-10-09T10:50:03.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[200197]: failed to encode map e213 with expected crc 2023-10-09T10:50:03.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[200197]: failed to encode map e213 with expected crc 2023-10-09T10:50:03.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[200197]: failed to encode map e213 with expected crc 2023-10-09T10:50:03.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[200197]: failed to encode map e213 with expected crc 2023-10-09T10:50:03.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:03 smithi125 ceph-mon[200197]: pgmap v479: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 4.7 KiB/s rd, 52 MiB/s wr, 830 op/s; 33867/163977 objects degraded (20.654%); 3177/163977 objects misplaced (1.937%); 14 MiB/s, 218 objects/s recovering 2023-10-09T10:50:03.988 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 43 16 56704 56688 82.3723 54.4375 0.0276231 0.0121328 2023-10-09T10:50:04.151 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:50:04.151 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:50:04.151 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:50:04.151 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:50:04.151 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.151 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42934 6.11 6:8a3972ec:::benchmark_data_smithi125_244951_object42933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.151 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.549876+0000", 2023-10-09T10:50:04.151 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.59723772199999, 2023-10-09T10:50:04.151 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.229929598, 2023-10-09T10:50:04.152 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.152 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.152 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.152 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.152 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.152 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42934 2023-10-09T10:50:04.152 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.152 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.152 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.152 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.152 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549876+0000", 2023-10-09T10:50:04.152 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.153 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.153 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.153 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.153 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549876+0000", 2023-10-09T10:50:04.153 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.153 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.153 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.153 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.153 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549874+0000", 2023-10-09T10:50:04.153 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:50:04.153 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.153 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.154 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.154 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549907+0000", 2023-10-09T10:50:04.154 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2579000000000001e-05 2023-10-09T10:50:04.154 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.154 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.154 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.154 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549922+0000", 2023-10-09T10:50:04.154 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5420000000000001e-05 2023-10-09T10:50:04.154 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.154 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.154 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.154 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549934+0000", 2023-10-09T10:50:04.155 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1724e-05 2023-10-09T10:50:04.155 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.155 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.155 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.155 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550289+0000", 2023-10-09T10:50:04.155 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035548900000000001 2023-10-09T10:50:04.155 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.155 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.155 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op must wait for map", 2023-10-09T10:50:04.155 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550297+0000", 2023-10-09T10:50:04.155 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8869999999999992e-06 2023-10-09T10:50:04.155 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.156 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.156 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.156 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551339+0000", 2023-10-09T10:50:04.156 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010423069999999999 2023-10-09T10:50:04.156 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.156 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.156 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.156 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551349+0000", 2023-10-09T10:50:04.156 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8060000000000006e-06 2023-10-09T10:50:04.156 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.156 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.156 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.157 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.776033+0000", 2023-10-09T10:50:04.157 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.224684135 2023-10-09T10:50:04.157 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.157 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.157 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.157 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.776159+0000", 2023-10-09T10:50:04.157 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012583900000000001 2023-10-09T10:50:04.157 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.157 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.157 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.157 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.776290+0000", 2023-10-09T10:50:04.157 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000130705 2023-10-09T10:50:04.158 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.158 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.158 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.158 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778810+0000", 2023-10-09T10:50:04.158 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025205549999999998 2023-10-09T10:50:04.158 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.158 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.158 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.158 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778863+0000", 2023-10-09T10:50:04.158 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2821000000000001e-05 2023-10-09T10:50:04.158 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.158 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.158 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.159 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778971+0000", 2023-10-09T10:50:04.159 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108244 2023-10-09T10:50:04.159 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.159 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.159 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.159 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779012+0000", 2023-10-09T10:50:04.159 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0612999999999999e-05 2023-10-09T10:50:04.159 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.159 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.159 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.159 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779058+0000", 2023-10-09T10:50:04.159 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5948999999999997e-05 2023-10-09T10:50:04.160 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.160 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.160 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.160 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779805+0000", 2023-10-09T10:50:04.160 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00074730900000000004 2023-10-09T10:50:04.160 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.160 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.160 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.160 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.160 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.160 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42936 6.11 6:8ce9d1a1:::benchmark_data_smithi125_244951_object42935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.160 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.549950+0000", 2023-10-09T10:50:04.160 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.59716370899997, 2023-10-09T10:50:04.161 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.235619359, 2023-10-09T10:50:04.161 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.161 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.161 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.161 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.161 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.161 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42936 2023-10-09T10:50:04.161 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.161 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.161 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.161 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.161 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549950+0000", 2023-10-09T10:50:04.162 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.162 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.162 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.162 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.162 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549950+0000", 2023-10-09T10:50:04.162 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.162 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.162 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.162 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.162 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549949+0000", 2023-10-09T10:50:04.162 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:04.162 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.163 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.163 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.163 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549976+0000", 2023-10-09T10:50:04.163 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7223999999999999e-05 2023-10-09T10:50:04.163 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.163 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.163 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.163 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549987+0000", 2023-10-09T10:50:04.163 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0375e-05 2023-10-09T10:50:04.163 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.163 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.163 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.164 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549996+0000", 2023-10-09T10:50:04.164 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4010000000000003e-06 2023-10-09T10:50:04.164 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.164 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.164 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.164 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550347+0000", 2023-10-09T10:50:04.164 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035085200000000002 2023-10-09T10:50:04.164 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.164 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.164 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:50:04.164 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550352+0000", 2023-10-09T10:50:04.164 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2079999999999999e-06 2023-10-09T10:50:04.164 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.165 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.165 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.165 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551460+0000", 2023-10-09T10:50:04.165 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001107762 2023-10-09T10:50:04.165 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.165 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.165 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.165 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551467+0000", 2023-10-09T10:50:04.165 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0350000000000004e-06 2023-10-09T10:50:04.165 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.165 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.165 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.165 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.777841+0000", 2023-10-09T10:50:04.166 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2263741750000001 2023-10-09T10:50:04.166 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.166 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.166 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.166 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778003+0000", 2023-10-09T10:50:04.166 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016172699999999999 2023-10-09T10:50:04.166 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.166 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.166 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.166 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778128+0000", 2023-10-09T10:50:04.166 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012541000000000001 2023-10-09T10:50:04.167 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.167 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.167 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.167 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780770+0000", 2023-10-09T10:50:04.167 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026419500000000001 2023-10-09T10:50:04.167 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.167 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.167 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.167 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780795+0000", 2023-10-09T10:50:04.167 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4363e-05 2023-10-09T10:50:04.167 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.167 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.167 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.168 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784699+0000", 2023-10-09T10:50:04.168 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039046739999999999 2023-10-09T10:50:04.168 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.168 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.168 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.168 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784714+0000", 2023-10-09T10:50:04.168 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4956999999999999e-05 2023-10-09T10:50:04.168 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.168 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.168 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.168 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784774+0000", 2023-10-09T10:50:04.168 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9994999999999997e-05 2023-10-09T10:50:04.169 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.169 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.169 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.169 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785569+0000", 2023-10-09T10:50:04.169 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00079485999999999999 2023-10-09T10:50:04.169 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.169 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.169 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.169 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.169 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.169 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42950 6.11 6:8b5bb41c:::benchmark_data_smithi125_244951_object42949:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.169 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:50:04.169 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.59711131400002, 2023-10-09T10:50:04.170 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2390020909999999, 2023-10-09T10:50:04.170 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.170 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.170 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.170 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.170 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.170 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42950 2023-10-09T10:50:04.170 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.170 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.170 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.170 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.170 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:50:04.171 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.171 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.171 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.171 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.171 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:50:04.171 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.171 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.171 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.171 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.171 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:50:04.171 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:04.171 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.171 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.172 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.172 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550021+0000", 2023-10-09T10:50:04.172 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8941999999999999e-05 2023-10-09T10:50:04.172 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.172 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.172 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.172 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550034+0000", 2023-10-09T10:50:04.172 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3267e-05 2023-10-09T10:50:04.172 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.172 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.172 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.172 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550040+0000", 2023-10-09T10:50:04.173 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5799999999999997e-06 2023-10-09T10:50:04.173 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.173 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.173 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.173 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550377+0000", 2023-10-09T10:50:04.173 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00033659800000000002 2023-10-09T10:50:04.173 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.173 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.173 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:50:04.173 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550382+0000", 2023-10-09T10:50:04.173 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6589999999999998e-06 2023-10-09T10:50:04.173 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.174 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.174 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.174 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551483+0000", 2023-10-09T10:50:04.174 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011012210000000001 2023-10-09T10:50:04.174 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.174 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.174 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.174 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551489+0000", 2023-10-09T10:50:04.174 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4910000000000002e-06 2023-10-09T10:50:04.174 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.174 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.174 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.174 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779851+0000", 2023-10-09T10:50:04.175 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.22836144 2023-10-09T10:50:04.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.175 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.175 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779959+0000", 2023-10-09T10:50:04.175 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108546 2023-10-09T10:50:04.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.175 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.175 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780061+0000", 2023-10-09T10:50:04.175 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010196600000000001 2023-10-09T10:50:04.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.176 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.176 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.176 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782707+0000", 2023-10-09T10:50:04.176 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002646074 2023-10-09T10:50:04.176 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.176 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.176 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.176 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785603+0000", 2023-10-09T10:50:04.176 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028958410000000001 2023-10-09T10:50:04.176 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.176 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.176 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.176 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785626+0000", 2023-10-09T10:50:04.177 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2371e-05 2023-10-09T10:50:04.177 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.177 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.177 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.177 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.787983+0000", 2023-10-09T10:50:04.177 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023577619999999998 2023-10-09T10:50:04.177 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.177 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.177 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.177 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.788053+0000", 2023-10-09T10:50:04.177 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9797999999999998e-05 2023-10-09T10:50:04.177 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.177 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.178 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.178 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789004+0000", 2023-10-09T10:50:04.178 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00095101500000000004 2023-10-09T10:50:04.178 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.178 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.178 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.178 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.178 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.178 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42962 6.11 6:8af38cb4:::benchmark_data_smithi125_244951_object42961:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.178 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550048+0000", 2023-10-09T10:50:04.178 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.59706578499998, 2023-10-09T10:50:04.179 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2398821879999999, 2023-10-09T10:50:04.179 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.179 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.179 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.179 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.179 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.179 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42962 2023-10-09T10:50:04.179 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.179 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.179 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.179 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.179 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550048+0000", 2023-10-09T10:50:04.180 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.180 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.180 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.180 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.180 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550048+0000", 2023-10-09T10:50:04.180 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.180 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.180 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.180 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.180 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550047+0000", 2023-10-09T10:50:04.180 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:04.181 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.181 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.181 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.181 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550067+0000", 2023-10-09T10:50:04.181 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0228999999999999e-05 2023-10-09T10:50:04.181 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.181 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.181 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.181 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550075+0000", 2023-10-09T10:50:04.181 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0979999999999996e-06 2023-10-09T10:50:04.182 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.182 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.182 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.182 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550080+0000", 2023-10-09T10:50:04.182 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7780000000000004e-06 2023-10-09T10:50:04.182 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.182 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.182 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.182 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550404+0000", 2023-10-09T10:50:04.182 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000323648 2023-10-09T10:50:04.182 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.183 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.183 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:50:04.183 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550409+0000", 2023-10-09T10:50:04.183 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6630000000000001e-06 2023-10-09T10:50:04.183 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.183 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.183 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.183 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551523+0000", 2023-10-09T10:50:04.183 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001113953 2023-10-09T10:50:04.183 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.184 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.184 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.184 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551531+0000", 2023-10-09T10:50:04.184 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5860000000000005e-06 2023-10-09T10:50:04.184 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.184 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.184 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.184 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780812+0000", 2023-10-09T10:50:04.184 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2292809120000001 2023-10-09T10:50:04.184 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.184 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.185 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.185 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780947+0000", 2023-10-09T10:50:04.185 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000135265 2023-10-09T10:50:04.185 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.185 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.185 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.185 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.781098+0000", 2023-10-09T10:50:04.185 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015082499999999999 2023-10-09T10:50:04.185 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.185 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.185 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.185 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782749+0000", 2023-10-09T10:50:04.185 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001651057 2023-10-09T10:50:04.186 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.186 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.186 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.186 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785607+0000", 2023-10-09T10:50:04.186 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028579569999999999 2023-10-09T10:50:04.186 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.186 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.186 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.186 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785661+0000", 2023-10-09T10:50:04.186 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4151999999999999e-05 2023-10-09T10:50:04.186 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.186 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.187 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.187 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789184+0000", 2023-10-09T10:50:04.187 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035224309999999999 2023-10-09T10:50:04.187 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.187 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.187 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.187 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789241+0000", 2023-10-09T10:50:04.187 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6962000000000001e-05 2023-10-09T10:50:04.187 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.187 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.187 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.187 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789930+0000", 2023-10-09T10:50:04.188 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000689148 2023-10-09T10:50:04.188 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.188 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.188 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.188 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.188 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.188 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43001 6.11 6:8f0a81da:::benchmark_data_smithi125_244951_object43000:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.188 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:50:04.188 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.59702807600002, 2023-10-09T10:50:04.188 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.240501734, 2023-10-09T10:50:04.188 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.188 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.189 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.189 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.189 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.189 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43001 2023-10-09T10:50:04.189 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.189 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.189 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.189 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.189 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:50:04.189 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.189 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.189 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.190 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.190 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:50:04.190 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.190 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.190 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.190 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.190 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:50:04.190 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:04.190 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.190 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.190 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.190 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550104+0000", 2023-10-09T10:50:04.191 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.912e-05 2023-10-09T10:50:04.191 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.191 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.191 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.191 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550110+0000", 2023-10-09T10:50:04.191 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3779999999999998e-06 2023-10-09T10:50:04.191 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.191 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.191 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.191 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550116+0000", 2023-10-09T10:50:04.191 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.609e-06 2023-10-09T10:50:04.191 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.192 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.192 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.192 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550429+0000", 2023-10-09T10:50:04.192 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00031264800000000001 2023-10-09T10:50:04.192 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.192 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.192 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:50:04.192 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550433+0000", 2023-10-09T10:50:04.192 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8420000000000004e-06 2023-10-09T10:50:04.192 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.192 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.192 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.192 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551574+0000", 2023-10-09T10:50:04.193 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011402210000000001 2023-10-09T10:50:04.193 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.193 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.193 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.193 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551584+0000", 2023-10-09T10:50:04.193 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9690000000000006e-06 2023-10-09T10:50:04.193 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.193 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.193 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.193 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.781792+0000", 2023-10-09T10:50:04.193 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.230208454 2023-10-09T10:50:04.193 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.194 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.194 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.194 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.781903+0000", 2023-10-09T10:50:04.194 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110733 2023-10-09T10:50:04.194 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.194 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.194 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.194 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782036+0000", 2023-10-09T10:50:04.194 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000133545 2023-10-09T10:50:04.194 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.194 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.194 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.195 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785598+0000", 2023-10-09T10:50:04.195 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035616100000000002 2023-10-09T10:50:04.195 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.195 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.195 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785679+0000", 2023-10-09T10:50:04.195 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0624999999999995e-05 2023-10-09T10:50:04.195 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.195 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.195 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785703+0000", 2023-10-09T10:50:04.195 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4150000000000001e-05 2023-10-09T10:50:04.196 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.196 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.196 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790058+0000", 2023-10-09T10:50:04.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043551919999999999 2023-10-09T10:50:04.196 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.196 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.196 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790110+0000", 2023-10-09T10:50:04.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2278000000000003e-05 2023-10-09T10:50:04.196 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.197 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.197 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.197 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790587+0000", 2023-10-09T10:50:04.197 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047690599999999998 2023-10-09T10:50:04.197 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.197 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.197 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.197 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.197 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.197 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43002 6.11 6:8bd791a2:::benchmark_data_smithi125_244951_object43001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.197 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550121+0000", 2023-10-09T10:50:04.197 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.59699278199997, 2023-10-09T10:50:04.198 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.241166963, 2023-10-09T10:50:04.198 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.198 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.198 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.198 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.198 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.198 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43002 2023-10-09T10:50:04.198 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.198 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.198 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.198 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550121+0000", 2023-10-09T10:50:04.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.199 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550121+0000", 2023-10-09T10:50:04.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.199 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550120+0000", 2023-10-09T10:50:04.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:04.200 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.200 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.200 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.200 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550139+0000", 2023-10-09T10:50:04.200 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8644999999999999e-05 2023-10-09T10:50:04.200 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.200 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.200 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.200 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550145+0000", 2023-10-09T10:50:04.200 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5180000000000001e-06 2023-10-09T10:50:04.200 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.200 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.200 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.201 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550151+0000", 2023-10-09T10:50:04.201 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0399999999999998e-06 2023-10-09T10:50:04.201 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.201 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.201 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.201 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551616+0000", 2023-10-09T10:50:04.201 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001464449 2023-10-09T10:50:04.201 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.201 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.201 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.201 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551623+0000", 2023-10-09T10:50:04.201 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0090000000000001e-06 2023-10-09T10:50:04.201 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.202 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.202 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.202 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782780+0000", 2023-10-09T10:50:04.202 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2311575050000001 2023-10-09T10:50:04.202 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.202 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.202 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.202 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782892+0000", 2023-10-09T10:50:04.202 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111826 2023-10-09T10:50:04.202 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.202 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.202 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.203 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782998+0000", 2023-10-09T10:50:04.203 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010603399999999999 2023-10-09T10:50:04.203 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.203 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.203 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.203 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785644+0000", 2023-10-09T10:50:04.203 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026456489999999999 2023-10-09T10:50:04.203 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.203 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.203 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.203 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785699+0000", 2023-10-09T10:50:04.203 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4778999999999998e-05 2023-10-09T10:50:04.203 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.204 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.204 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.204 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786762+0000", 2023-10-09T10:50:04.204 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001063172 2023-10-09T10:50:04.204 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.204 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.204 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.204 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790708+0000", 2023-10-09T10:50:04.204 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039466409999999999 2023-10-09T10:50:04.204 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.204 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.204 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.205 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790775+0000", 2023-10-09T10:50:04.205 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6155000000000006e-05 2023-10-09T10:50:04.205 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.205 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.205 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.205 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791288+0000", 2023-10-09T10:50:04.205 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051300599999999999 2023-10-09T10:50:04.205 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.205 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.205 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.205 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.205 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.206 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43015 6.11 6:8d7a9802:::benchmark_data_smithi125_244951_object43014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.206 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:50:04.206 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.596957084, 2023-10-09T10:50:04.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.241685028, 2023-10-09T10:50:04.206 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.206 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.206 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.206 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.206 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.206 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43015 2023-10-09T10:50:04.206 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.206 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.206 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.207 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:50:04.207 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.207 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:50:04.207 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.207 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:50:04.207 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:04.208 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.208 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.208 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550174+0000", 2023-10-09T10:50:04.208 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8289000000000001e-05 2023-10-09T10:50:04.208 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.208 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.208 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550181+0000", 2023-10-09T10:50:04.208 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.024e-06 2023-10-09T10:50:04.208 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.208 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.209 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.209 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550187+0000", 2023-10-09T10:50:04.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6659999999999996e-06 2023-10-09T10:50:04.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.209 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.209 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.209 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551652+0000", 2023-10-09T10:50:04.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001465406 2023-10-09T10:50:04.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.209 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.209 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.209 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551659+0000", 2023-10-09T10:50:04.210 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1130000000000003e-06 2023-10-09T10:50:04.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.210 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.783679+0000", 2023-10-09T10:50:04.210 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.232019406 2023-10-09T10:50:04.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.210 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.783831+0000", 2023-10-09T10:50:04.210 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000151769 2023-10-09T10:50:04.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.211 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.211 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784019+0000", 2023-10-09T10:50:04.211 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018824400000000001 2023-10-09T10:50:04.211 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.211 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.211 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786769+0000", 2023-10-09T10:50:04.211 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027502529999999998 2023-10-09T10:50:04.211 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.211 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.212 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789091+0000", 2023-10-09T10:50:04.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023215200000000001 2023-10-09T10:50:04.212 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.212 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.212 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.212 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789131+0000", 2023-10-09T10:50:04.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0766999999999998e-05 2023-10-09T10:50:04.212 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.212 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.212 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.212 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791345+0000", 2023-10-09T10:50:04.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022138349999999999 2023-10-09T10:50:04.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791404+0000", 2023-10-09T10:50:04.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8301000000000003e-05 2023-10-09T10:50:04.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791841+0000", 2023-10-09T10:50:04.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00043785599999999998 2023-10-09T10:50:04.213 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.213 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.214 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.214 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.214 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43040 6.11 6:892fed64:::benchmark_data_smithi125_244951_object43039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.214 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550192+0000", 2023-10-09T10:50:04.214 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.596921705, 2023-10-09T10:50:04.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2421168730000001, 2023-10-09T10:50:04.214 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.214 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.214 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.214 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.215 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.215 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43040 2023-10-09T10:50:04.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.215 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.215 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.215 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550192+0000", 2023-10-09T10:50:04.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.215 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.215 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550192+0000", 2023-10-09T10:50:04.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550191+0000", 2023-10-09T10:50:04.216 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:04.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550265+0000", 2023-10-09T10:50:04.216 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3668e-05 2023-10-09T10:50:04.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550273+0000", 2023-10-09T10:50:04.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2230000000000007e-06 2023-10-09T10:50:04.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550280+0000", 2023-10-09T10:50:04.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3729999999999998e-06 2023-10-09T10:50:04.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551688+0000", 2023-10-09T10:50:04.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001408011 2023-10-09T10:50:04.218 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.218 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551694+0000", 2023-10-09T10:50:04.218 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8120000000000002e-06 2023-10-09T10:50:04.218 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.218 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785718+0000", 2023-10-09T10:50:04.218 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2340238640000001 2023-10-09T10:50:04.218 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785825+0000", 2023-10-09T10:50:04.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000107161 2023-10-09T10:50:04.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.219 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785964+0000", 2023-10-09T10:50:04.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013889800000000001 2023-10-09T10:50:04.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.219 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789139+0000", 2023-10-09T10:50:04.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0031749249999999999 2023-10-09T10:50:04.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.220 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789979+0000", 2023-10-09T10:50:04.220 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00083957300000000003 2023-10-09T10:50:04.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.220 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790018+0000", 2023-10-09T10:50:04.220 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8816999999999998e-05 2023-10-09T10:50:04.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791894+0000", 2023-10-09T10:50:04.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018764160000000001 2023-10-09T10:50:04.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.221 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791933+0000", 2023-10-09T10:50:04.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8889e-05 2023-10-09T10:50:04.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.221 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792309+0000", 2023-10-09T10:50:04.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00037573100000000003 2023-10-09T10:50:04.222 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.222 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.222 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.222 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43116 6.11 6:8866ecf5:::benchmark_data_smithi125_244951_object43115:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.222 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550285+0000", 2023-10-09T10:50:04.222 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.59682855800003, 2023-10-09T10:50:04.222 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2425320870000001, 2023-10-09T10:50:04.222 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.222 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.223 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.223 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.223 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.223 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43116 2023-10-09T10:50:04.223 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.223 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.223 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.223 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.223 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550285+0000", 2023-10-09T10:50:04.223 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.223 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.223 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.224 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.224 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550285+0000", 2023-10-09T10:50:04.224 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.224 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.224 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.224 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.224 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550284+0000", 2023-10-09T10:50:04.224 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:04.224 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.224 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.224 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.224 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550303+0000", 2023-10-09T10:50:04.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8817e-05 2023-10-09T10:50:04.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550310+0000", 2023-10-09T10:50:04.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7889999999999997e-06 2023-10-09T10:50:04.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550315+0000", 2023-10-09T10:50:04.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4539999999999998e-06 2023-10-09T10:50:04.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551718+0000", 2023-10-09T10:50:04.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001402585 2023-10-09T10:50:04.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551725+0000", 2023-10-09T10:50:04.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9319999999999999e-06 2023-10-09T10:50:04.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786811+0000", 2023-10-09T10:50:04.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.23508555 2023-10-09T10:50:04.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786947+0000", 2023-10-09T10:50:04.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013601499999999999 2023-10-09T10:50:04.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.787112+0000", 2023-10-09T10:50:04.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016519000000000001 2023-10-09T10:50:04.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.228 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.228 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789147+0000", 2023-10-09T10:50:04.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020350899999999998 2023-10-09T10:50:04.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.228 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.228 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790634+0000", 2023-10-09T10:50:04.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014872209999999999 2023-10-09T10:50:04.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790670+0000", 2023-10-09T10:50:04.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6395e-05 2023-10-09T10:50:04.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792355+0000", 2023-10-09T10:50:04.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016850179999999999 2023-10-09T10:50:04.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792391+0000", 2023-10-09T10:50:04.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5484000000000001e-05 2023-10-09T10:50:04.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792817+0000", 2023-10-09T10:50:04.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042605300000000002 2023-10-09T10:50:04.230 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.230 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.230 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.231 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43152 6.11 6:8f5a8f11:::benchmark_data_smithi125_244951_object43151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.231 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:50:04.231 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.59679221599998, 2023-10-09T10:50:04.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.254750676, 2023-10-09T10:50:04.231 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.231 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.231 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.231 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.231 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.231 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43152 2023-10-09T10:50:04.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.232 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:50:04.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:50:04.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:50:04.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:04.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550339+0000", 2023-10-09T10:50:04.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7940000000000001e-05 2023-10-09T10:50:04.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550346+0000", 2023-10-09T10:50:04.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4789999999999999e-06 2023-10-09T10:50:04.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550353+0000", 2023-10-09T10:50:04.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7510000000000002e-06 2023-10-09T10:50:04.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551748+0000", 2023-10-09T10:50:04.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013946119999999999 2023-10-09T10:50:04.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551754+0000", 2023-10-09T10:50:04.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8460000000000002e-06 2023-10-09T10:50:04.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792855+0000", 2023-10-09T10:50:04.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.241101091 2023-10-09T10:50:04.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792993+0000", 2023-10-09T10:50:04.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013794 2023-10-09T10:50:04.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.793143+0000", 2023-10-09T10:50:04.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014960799999999999 2023-10-09T10:50:04.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797271+0000", 2023-10-09T10:50:04.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041283810000000004 2023-10-09T10:50:04.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797282+0000", 2023-10-09T10:50:04.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0543999999999999e-05 2023-10-09T10:50:04.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799296+0000", 2023-10-09T10:50:04.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020138999999999999 2023-10-09T10:50:04.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803962+0000", 2023-10-09T10:50:04.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046658840000000004 2023-10-09T10:50:04.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.238 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.804015+0000", 2023-10-09T10:50:04.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3229999999999997e-05 2023-10-09T10:50:04.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.238 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805072+0000", 2023-10-09T10:50:04.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010569690000000001 2023-10-09T10:50:04.239 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.239 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.239 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.239 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43224 6.11 6:8b0bc017:::benchmark_data_smithi125_244951_object43223:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.239 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550374+0000", 2023-10-09T10:50:04.239 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.59673982800001, 2023-10-09T10:50:04.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2556577980000001, 2023-10-09T10:50:04.239 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.239 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.239 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.239 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.239 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.240 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43224 2023-10-09T10:50:04.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.240 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550374+0000", 2023-10-09T10:50:04.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550374+0000", 2023-10-09T10:50:04.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550373+0000", 2023-10-09T10:50:04.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:04.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550388+0000", 2023-10-09T10:50:04.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.453e-05 2023-10-09T10:50:04.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550393+0000", 2023-10-09T10:50:04.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7119999999999999e-06 2023-10-09T10:50:04.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550397+0000", 2023-10-09T10:50:04.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6789999999999996e-06 2023-10-09T10:50:04.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551777+0000", 2023-10-09T10:50:04.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001380196 2023-10-09T10:50:04.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551784+0000", 2023-10-09T10:50:04.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0369999999999997e-06 2023-10-09T10:50:04.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.793904+0000", 2023-10-09T10:50:04.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.242119346 2023-10-09T10:50:04.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.794042+0000", 2023-10-09T10:50:04.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013829700000000001 2023-10-09T10:50:04.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.244 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.794190+0000", 2023-10-09T10:50:04.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014784600000000001 2023-10-09T10:50:04.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.244 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797289+0000", 2023-10-09T10:50:04.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003099152 2023-10-09T10:50:04.245 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.245 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.245 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.245 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799339+0000", 2023-10-09T10:50:04.245 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020501059999999999 2023-10-09T10:50:04.245 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.245 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.245 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.245 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803853+0000", 2023-10-09T10:50:04.245 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045136179999999996 2023-10-09T10:50:04.245 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.245 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.245 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.246 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805211+0000", 2023-10-09T10:50:04.246 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001358349 2023-10-09T10:50:04.246 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.246 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.246 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805266+0000", 2023-10-09T10:50:04.246 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5297999999999998e-05 2023-10-09T10:50:04.246 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.246 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.246 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806031+0000", 2023-10-09T10:50:04.246 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00076497100000000003 2023-10-09T10:50:04.247 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.247 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.247 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.247 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.247 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.247 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43255 6.11 6:88dd5493:::benchmark_data_smithi125_244951_object43254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.247 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:50:04.247 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.59671330899999, 2023-10-09T10:50:04.247 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2563999859999999, 2023-10-09T10:50:04.247 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.247 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.248 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.248 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.248 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.248 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43255 2023-10-09T10:50:04.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.248 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:50:04.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:50:04.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:50:04.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:04.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550413+0000", 2023-10-09T10:50:04.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3052e-05 2023-10-09T10:50:04.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550417+0000", 2023-10-09T10:50:04.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5999999999999998e-06 2023-10-09T10:50:04.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550420+0000", 2023-10-09T10:50:04.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6519999999999999e-06 2023-10-09T10:50:04.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551807+0000", 2023-10-09T10:50:04.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001386998 2023-10-09T10:50:04.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551814+0000", 2023-10-09T10:50:04.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3960000000000003e-06 2023-10-09T10:50:04.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.794932+0000", 2023-10-09T10:50:04.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2431179880000001 2023-10-09T10:50:04.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.795097+0000", 2023-10-09T10:50:04.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000165248 2023-10-09T10:50:04.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.795276+0000", 2023-10-09T10:50:04.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017957699999999999 2023-10-09T10:50:04.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799383+0000", 2023-10-09T10:50:04.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004106569 2023-10-09T10:50:04.253 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.253 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.253 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.253 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801446+0000", 2023-10-09T10:50:04.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020628370000000001 2023-10-09T10:50:04.253 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.253 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.253 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.253 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803918+0000", 2023-10-09T10:50:04.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024717239999999998 2023-10-09T10:50:04.253 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.254 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.254 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.254 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806076+0000", 2023-10-09T10:50:04.254 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002158395 2023-10-09T10:50:04.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.254 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.254 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.254 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806122+0000", 2023-10-09T10:50:04.254 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5807999999999999e-05 2023-10-09T10:50:04.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.254 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.254 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.254 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806800+0000", 2023-10-09T10:50:04.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00067843299999999999 2023-10-09T10:50:04.255 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.255 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.255 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.255 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.255 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43315 6.11 6:8fb75c47:::benchmark_data_smithi125_244951_object43314:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.255 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550436+0000", 2023-10-09T10:50:04.255 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.59667781500002, 2023-10-09T10:50:04.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2571070289999999, 2023-10-09T10:50:04.255 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.255 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.256 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.256 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.256 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.256 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43315 2023-10-09T10:50:04.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.256 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550436+0000", 2023-10-09T10:50:04.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550436+0000", 2023-10-09T10:50:04.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550435+0000", 2023-10-09T10:50:04.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:04.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550453+0000", 2023-10-09T10:50:04.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7504999999999999e-05 2023-10-09T10:50:04.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550460+0000", 2023-10-09T10:50:04.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2470000000000002e-06 2023-10-09T10:50:04.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550466+0000", 2023-10-09T10:50:04.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8699999999999997e-06 2023-10-09T10:50:04.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551836+0000", 2023-10-09T10:50:04.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001370664 2023-10-09T10:50:04.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551843+0000", 2023-10-09T10:50:04.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5320000000000003e-06 2023-10-09T10:50:04.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.796128+0000", 2023-10-09T10:50:04.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2442850169999999 2023-10-09T10:50:04.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.796286+0000", 2023-10-09T10:50:04.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015828500000000001 2023-10-09T10:50:04.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.796448+0000", 2023-10-09T10:50:04.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016164200000000001 2023-10-09T10:50:04.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.261 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801434+0000", 2023-10-09T10:50:04.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049856279999999998 2023-10-09T10:50:04.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.261 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801530+0000", 2023-10-09T10:50:04.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.645e-05 2023-10-09T10:50:04.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.261 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805133+0000", 2023-10-09T10:50:04.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036025850000000002 2023-10-09T10:50:04.262 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.262 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.262 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.262 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806935+0000", 2023-10-09T10:50:04.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018020410000000001 2023-10-09T10:50:04.262 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.262 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.262 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.262 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.807002+0000", 2023-10-09T10:50:04.263 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6883000000000003e-05 2023-10-09T10:50:04.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.263 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.263 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.807543+0000", 2023-10-09T10:50:04.263 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000541115 2023-10-09T10:50:04.263 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.263 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.263 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.263 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43318 6.11 6:8f0da7e4:::benchmark_data_smithi125_244951_object43317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.263 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550471+0000", 2023-10-09T10:50:04.264 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.59664293700001, 2023-10-09T10:50:04.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2582329400000001, 2023-10-09T10:50:04.264 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.264 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.264 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.264 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.264 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.264 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43318 2023-10-09T10:50:04.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.264 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550471+0000", 2023-10-09T10:50:04.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550471+0000", 2023-10-09T10:50:04.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550470+0000", 2023-10-09T10:50:04.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:04.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550518+0000", 2023-10-09T10:50:04.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8108000000000001e-05 2023-10-09T10:50:04.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550526+0000", 2023-10-09T10:50:04.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0199999999999994e-06 2023-10-09T10:50:04.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550532+0000", 2023-10-09T10:50:04.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9109999999999998e-06 2023-10-09T10:50:04.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551864+0000", 2023-10-09T10:50:04.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001332317 2023-10-09T10:50:04.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551870+0000", 2023-10-09T10:50:04.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6489999999999996e-06 2023-10-09T10:50:04.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797329+0000", 2023-10-09T10:50:04.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.245458701 2023-10-09T10:50:04.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797476+0000", 2023-10-09T10:50:04.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014769199999999999 2023-10-09T10:50:04.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797644+0000", 2023-10-09T10:50:04.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000167481 2023-10-09T10:50:04.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801478+0000", 2023-10-09T10:50:04.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038341780000000002 2023-10-09T10:50:04.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801537+0000", 2023-10-09T10:50:04.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8879000000000003e-05 2023-10-09T10:50:04.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806859+0000", 2023-10-09T10:50:04.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0053216640000000003 2023-10-09T10:50:04.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.807979+0000", 2023-10-09T10:50:04.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011203770000000001 2023-10-09T10:50:04.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808056+0000", 2023-10-09T10:50:04.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6568000000000003e-05 2023-10-09T10:50:04.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808703+0000", 2023-10-09T10:50:04.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00064785600000000004 2023-10-09T10:50:04.272 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.272 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.272 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.272 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43334 6.11 6:8b093292:::benchmark_data_smithi125_244951_object43333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.272 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:50:04.272 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.59657526799998, 2023-10-09T10:50:04.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.266648716, 2023-10-09T10:50:04.272 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.272 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.273 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.273 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.273 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.273 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43334 2023-10-09T10:50:04.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.273 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:50:04.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:50:04.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:50:04.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:04.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550579+0000", 2023-10-09T10:50:04.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1733000000000002e-05 2023-10-09T10:50:04.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550587+0000", 2023-10-09T10:50:04.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6260000000000003e-06 2023-10-09T10:50:04.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550592+0000", 2023-10-09T10:50:04.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4160000000000003e-06 2023-10-09T10:50:04.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551890+0000", 2023-10-09T10:50:04.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001297702 2023-10-09T10:50:04.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551896+0000", 2023-10-09T10:50:04.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4559999999999999e-06 2023-10-09T10:50:04.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.798375+0000", 2023-10-09T10:50:04.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2464794800000001 2023-10-09T10:50:04.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.798514+0000", 2023-10-09T10:50:04.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000139159 2023-10-09T10:50:04.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.798661+0000", 2023-10-09T10:50:04.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000146312 2023-10-09T10:50:04.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801519+0000", 2023-10-09T10:50:04.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028583720000000001 2023-10-09T10:50:04.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803865+0000", 2023-10-09T10:50:04.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002345752 2023-10-09T10:50:04.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808789+0000", 2023-10-09T10:50:04.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049239790000000002 2023-10-09T10:50:04.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.816450+0000", 2023-10-09T10:50:04.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076614550000000002 2023-10-09T10:50:04.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.816501+0000", 2023-10-09T10:50:04.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1298000000000003e-05 2023-10-09T10:50:04.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817187+0000", 2023-10-09T10:50:04.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00068546500000000001 2023-10-09T10:50:04.279 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.280 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.280 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.280 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43385 6.11 6:8e6a343d:::benchmark_data_smithi125_244951_object43384:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:50:04.280 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.636508+0000", 2023-10-09T10:50:04.280 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 306.51060536, 2023-10-09T10:50:04.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1811192319999999, 2023-10-09T10:50:04.280 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.280 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.280 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.280 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.280 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.280 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43385 2023-10-09T10:50:04.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.281 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636508+0000", 2023-10-09T10:50:04.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636508+0000", 2023-10-09T10:50:04.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636507+0000", 2023-10-09T10:50:04.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:04.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636527+0000", 2023-10-09T10:50:04.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9748000000000001e-05 2023-10-09T10:50:04.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636539+0000", 2023-10-09T10:50:04.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2197e-05 2023-10-09T10:50:04.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636545+0000", 2023-10-09T10:50:04.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1999999999999999e-06 2023-10-09T10:50:04.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636582+0000", 2023-10-09T10:50:04.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6791000000000001e-05 2023-10-09T10:50:04.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636590+0000", 2023-10-09T10:50:04.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4309999999999994e-06 2023-10-09T10:50:04.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799411+0000", 2023-10-09T10:50:04.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1628212659999999 2023-10-09T10:50:04.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799550+0000", 2023-10-09T10:50:04.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013822300000000001 2023-10-09T10:50:04.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:04.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799660+0000", 2023-10-09T10:50:04.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110701 2023-10-09T10:50:04.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801589+0000", 2023-10-09T10:50:04.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019285470000000001 2023-10-09T10:50:04.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803871+0000", 2023-10-09T10:50:04.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022818180000000001 2023-10-09T10:50:04.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808835+0000", 2023-10-09T10:50:04.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049646200000000003 2023-10-09T10:50:04.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817231+0000", 2023-10-09T10:50:04.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083956959999999994 2023-10-09T10:50:04.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817263+0000", 2023-10-09T10:50:04.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1899000000000001e-05 2023-10-09T10:50:04.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817627+0000", 2023-10-09T10:50:04.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00036434500000000002 2023-10-09T10:50:04.287 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.287 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.287 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.288 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106397 6.1 6:856288a5:::benchmark_data_smithi125_244951_object106396:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:50:04.288 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.204308+0000", 2023-10-09T10:50:04.288 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 241.942805346, 2023-10-09T10:50:04.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.075713931999999998, 2023-10-09T10:50:04.288 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.288 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.288 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.288 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.288 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.288 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106397 2023-10-09T10:50:04.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.289 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204308+0000", 2023-10-09T10:50:04.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204308+0000", 2023-10-09T10:50:04.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204307+0000", 2023-10-09T10:50:04.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:50:04.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204330+0000", 2023-10-09T10:50:04.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2852000000000001e-05 2023-10-09T10:50:04.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204340+0000", 2023-10-09T10:50:04.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3989999999999993e-06 2023-10-09T10:50:04.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204346+0000", 2023-10-09T10:50:04.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5989999999999998e-06 2023-10-09T10:50:04.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.207434+0000", 2023-10-09T10:50:04.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030874600000000002 2023-10-09T10:50:04.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227524+0000", 2023-10-09T10:50:04.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.020090545000000001 2023-10-09T10:50:04.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:50:04.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227667+0000", 2023-10-09T10:50:04.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014324699999999999 2023-10-09T10:50:04.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.233515+0000", 2023-10-09T10:50:04.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0058477219999999996 2023-10-09T10:50:04.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.234252+0000", 2023-10-09T10:50:04.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00073691800000000003 2023-10-09T10:50:04.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279940+0000", 2023-10-09T10:50:04.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.045688232000000002 2023-10-09T10:50:04.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279985+0000", 2023-10-09T10:50:04.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4916000000000002e-05 2023-10-09T10:50:04.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.280022+0000", 2023-10-09T10:50:04.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6871e-05 2023-10-09T10:50:04.294 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.294 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.294 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.294 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106414 6.11 6:8c9e6868:::benchmark_data_smithi125_244951_object106413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:50:04.295 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.227135+0000", 2023-10-09T10:50:04.295 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 241.919978657, 2023-10-09T10:50:04.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.37275067699999997, 2023-10-09T10:50:04.295 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.295 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:50:04.295 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.295 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.295 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.295 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106414 2023-10-09T10:50:04.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.295 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227135+0000", 2023-10-09T10:50:04.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227135+0000", 2023-10-09T10:50:04.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227134+0000", 2023-10-09T10:50:04.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:04.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227156+0000", 2023-10-09T10:50:04.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2059999999999999e-05 2023-10-09T10:50:04.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227163+0000", 2023-10-09T10:50:04.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4169999999999997e-06 2023-10-09T10:50:04.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227168+0000", 2023-10-09T10:50:04.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9509999999999997e-06 2023-10-09T10:50:04.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.229108+0000", 2023-10-09T10:50:04.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001940396 2023-10-09T10:50:04.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.229279+0000", 2023-10-09T10:50:04.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017114100000000001 2023-10-09T10:50:04.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T10:50:04.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.229487+0000", 2023-10-09T10:50:04.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000207977 2023-10-09T10:50:04.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.233575+0000", 2023-10-09T10:50:04.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040875140000000004 2023-10-09T10:50:04.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.240998+0000", 2023-10-09T10:50:04.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074230249999999998 2023-10-09T10:50:04.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.245203+0000", 2023-10-09T10:50:04.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042052039999999997 2023-10-09T10:50:04.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274084+0000", 2023-10-09T10:50:04.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.028880626999999999 2023-10-09T10:50:04.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274097+0000", 2023-10-09T10:50:04.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.344e-05 2023-10-09T10:50:04.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599869+0000", 2023-10-09T10:50:04.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32577187699999999 2023-10-09T10:50:04.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599885+0000", 2023-10-09T10:50:04.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6515e-05 2023-10-09T10:50:04.302 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.302 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.302 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.302 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106433 6.11 6:8d28bd42:::benchmark_data_smithi125_244951_object106432:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:50:04.303 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.251731+0000", 2023-10-09T10:50:04.303 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 241.895382189, 2023-10-09T10:50:04.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.34819989800000001, 2023-10-09T10:50:04.303 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.303 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:50:04.303 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.303 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.303 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.303 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106433 2023-10-09T10:50:04.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.303 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251731+0000", 2023-10-09T10:50:04.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251731+0000", 2023-10-09T10:50:04.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251729+0000", 2023-10-09T10:50:04.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:50:04.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251758+0000", 2023-10-09T10:50:04.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8909000000000001e-05 2023-10-09T10:50:04.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251780+0000", 2023-10-09T10:50:04.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2011999999999999e-05 2023-10-09T10:50:04.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251791+0000", 2023-10-09T10:50:04.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0147e-05 2023-10-09T10:50:04.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.253110+0000", 2023-10-09T10:50:04.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013193739999999999 2023-10-09T10:50:04.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.253287+0000", 2023-10-09T10:50:04.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017752800000000001 2023-10-09T10:50:04.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T10:50:04.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.253484+0000", 2023-10-09T10:50:04.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000196622 2023-10-09T10:50:04.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.257296+0000", 2023-10-09T10:50:04.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038119550000000001 2023-10-09T10:50:04.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.257345+0000", 2023-10-09T10:50:04.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8745000000000002e-05 2023-10-09T10:50:04.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279630+0000", 2023-10-09T10:50:04.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.02228538 2023-10-09T10:50:04.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279642+0000", 2023-10-09T10:50:04.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2328e-05 2023-10-09T10:50:04.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599919+0000", 2023-10-09T10:50:04.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32027628800000002 2023-10-09T10:50:04.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599931+0000", 2023-10-09T10:50:04.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2439000000000001e-05 2023-10-09T10:50:04.310 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.310 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.310 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.310 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:136053 6.1 6:832e3320:::benchmark_data_smithi125_244951_object29610:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-10-09T10:50:04.310 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:02.144996+0000", 2023-10-09T10:50:04.310 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 182.002117777, 2023-10-09T10:50:04.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.155053459, 2023-10-09T10:50:04.310 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:04.310 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:04.310 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:04.310 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:04.311 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:04.311 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 136053 2023-10-09T10:50:04.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.311 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:04.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:04.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.144996+0000", 2023-10-09T10:50:04.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:04.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.144996+0000", 2023-10-09T10:50:04.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:04.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:04.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.144994+0000", 2023-10-09T10:50:04.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:04.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:04.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145002+0000", 2023-10-09T10:50:04.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8429999999999993e-06 2023-10-09T10:50:04.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:04.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145014+0000", 2023-10-09T10:50:04.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2163e-05 2023-10-09T10:50:04.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:04.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145024+0000", 2023-10-09T10:50:04.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7610000000000002e-06 2023-10-09T10:50:04.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145083+0000", 2023-10-09T10:50:04.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8882999999999998e-05 2023-10-09T10:50:04.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:04.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145095+0000", 2023-10-09T10:50:04.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2255999999999999e-05 2023-10-09T10:50:04.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:04.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.287386+0000", 2023-10-09T10:50:04.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.14229140700000001 2023-10-09T10:50:04.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:04.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.287495+0000", 2023-10-09T10:50:04.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108651 2023-10-09T10:50:04.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:50:04.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.287593+0000", 2023-10-09T10:50:04.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8348000000000001e-05 2023-10-09T10:50:04.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.295087+0000", 2023-10-09T10:50:04.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074939550000000001 2023-10-09T10:50:04.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:04.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.295761+0000", 2023-10-09T10:50:04.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00067362100000000001 2023-10-09T10:50:04.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:04.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.299957+0000", 2023-10-09T10:50:04.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041957419999999997 2023-10-09T10:50:04.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:04.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.300009+0000", 2023-10-09T10:50:04.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2193e-05 2023-10-09T10:50:04.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:04.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:04.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:04.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.300049+0000", 2023-10-09T10:50:04.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0156e-05 2023-10-09T10:50:04.317 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.317 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.317 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.318 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:04.318 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:04.318 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:50:04.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:03 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:04.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:03 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:50:04.319 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:03 smithi155 ceph-mon[184854]: osdmap e213: 8 total, 8 up, 5 in 2023-10-09T10:50:04.319 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:03 smithi155 ceph-mon[184854]: failed to encode map e213 with expected crc 2023-10-09T10:50:04.319 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:03 smithi155 ceph-mon[184854]: failed to encode map e213 with expected crc 2023-10-09T10:50:04.319 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:03 smithi155 ceph-mon[184854]: failed to encode map e213 with expected crc 2023-10-09T10:50:04.319 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:03 smithi155 ceph-mon[184854]: failed to encode map e213 with expected crc 2023-10-09T10:50:04.319 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:03 smithi155 ceph-mon[184854]: failed to encode map e213 with expected crc 2023-10-09T10:50:04.319 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:03 smithi155 ceph-mon[184854]: failed to encode map e213 with expected crc 2023-10-09T10:50:04.319 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:03 smithi155 ceph-mon[184854]: pgmap v479: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 4.7 KiB/s rd, 52 MiB/s wr, 830 op/s; 33867/163977 objects degraded (20.654%); 3177/163977 objects misplaced (1.937%); 14 MiB/s, 218 objects/s recovering 2023-10-09T10:50:04.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:04.320 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:04.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:04.421 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:04.486 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:50:04.495 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:04.524 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:04.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:04.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:04.630 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:04.710 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:04.745 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:50:04.748 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:04.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:04.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:04.851 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:04.894 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:04.922 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:04.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:04.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:04.977 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:04.988 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 44 16 57485 57469 81.6096 48.8125 0.0265391 0.0122453 2023-10-09T10:50:05.026 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:05.076 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:05.109 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:05.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:05.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:05.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:05.278 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:05.318 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:05.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:05.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:05.337 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:50:05.339 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:50:05.424 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:05.475 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:05.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:05.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:05.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:05.607 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:05.657 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:05.689 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:05.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:05.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:05.794 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:05.841 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:05.871 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:05.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:05.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:05.947 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:50:05.975 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:05.989 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 45 16 58313 58297 80.9461 51.75 0.0182596 0.0123463 2023-10-09T10:50:06.030 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:06.081 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:06.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:06.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:06.194 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:06.250 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:06.274 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":480,"stamp":"2023-10-09T10:50:05.220272+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3649211175,"num_objects":55695,"num_object_clones":0,"num_object_copies":167085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33867,"num_objects_misplaced":2805,"num_objects_unfound":0,"num_objects_dirty":55695,"num_whiteouts":0,"num_read":294,"num_read_kb":554,"num_write":55877,"num_write_kb":3564500,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2986,"num_bytes_recovered":194804437,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":59108,"ondisk_log_size":59108,"up":327,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":9674572,"kb_used_data":8212260,"kb_used_omap":35,"kb_used_meta":1462236,"kb_avail":459051188,"statfs":{"total":479975178240,"available":470068416512,"internally_reserved":0,"allocated":8409354240,"data_stored":8404774641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36424,"internal_metadata":1497330104},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,4,4,9,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":393758178,"num_objects":6008,"num_object_clones":0,"num_object_copies":18024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26616,"num_objects_misplaced":-2989,"num_objects_unfound":0,"num_objects_dirty":6008,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":6016,"num_write_kb":384586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1683,"num_bytes_recovered":110219686,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.478943"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":215,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.761053+0000","last_change":"2023-10-09T10:49:55.830252+0000","last_active":"2023-10-09T10:49:57.761053+0000","last_peered":"2023-10-09T10:49:57.761053+0000","last_clean":"2023-10-09T10:49:57.761053+0000","last_became_active":"2023-10-09T10:49:55.829651+0000","last_became_peered":"2023-10-09T10:49:55.829651+0000","last_unstale":"2023-10-09T10:49:57.761053+0000","last_undegraded":"2023-10-09T10:49:57.761053+0000","last_fullsized":"2023-10-09T10:49:57.761053+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:34:08.701860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":191,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.777425+0000","last_change":"2023-10-09T10:49:55.811543+0000","last_active":"2023-10-09T10:49:58.777425+0000","last_peered":"2023-10-09T10:49:58.777425+0000","last_clean":"2023-10-09T10:49:58.777425+0000","last_became_active":"2023-10-09T10:49:55.811225+0000","last_became_peered":"2023-10-09T10:49:55.811225+0000","last_unstale":"2023-10-09T10:49:58.777425+0000","last_undegraded":"2023-10-09T10:49:58.777425+0000","last_fullsized":"2023-10-09T10:49:58.777425+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:56.330889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:59.698968+0000","last_change":"2023-10-09T10:49:55.813068+0000","last_active":"2023-10-09T10:49:59.698968+0000","last_peered":"2023-10-09T10:49:59.698968+0000","last_clean":"2023-10-09T10:49:59.698968+0000","last_became_active":"2023-10-09T10:49:55.811847+0000","last_became_peered":"2023-10-09T10:49:55.811847+0000","last_unstale":"2023-10-09T10:49:59.698968+0000","last_undegraded":"2023-10-09T10:49:59.698968+0000","last_fullsized":"2023-10-09T10:49:59.698968+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:35.889323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":100,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.765964+0000","last_change":"2023-10-09T10:49:55.838051+0000","last_active":"2023-10-09T10:49:58.765964+0000","last_peered":"2023-10-09T10:49:58.765964+0000","last_clean":"2023-10-09T10:49:58.765964+0000","last_became_active":"2023-10-09T10:49:55.836999+0000","last_became_peered":"2023-10-09T10:49:55.836999+0000","last_unstale":"2023-10-09T10:49:58.765964+0000","last_undegraded":"2023-10-09T10:49:58.765964+0000","last_fullsized":"2023-10-09T10:49:58.765964+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:00:35.849494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":227,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.773380+0000","last_change":"2023-10-09T10:49:55.820195+0000","last_active":"2023-10-09T10:49:58.773380+0000","last_peered":"2023-10-09T10:49:58.773380+0000","last_clean":"2023-10-09T10:49:58.773380+0000","last_became_active":"2023-10-09T10:49:55.819612+0000","last_became_peered":"2023-10-09T10:49:55.819612+0000","last_unstale":"2023-10-09T10:49:58.773380+0000","last_undegraded":"2023-10-09T10:49:58.773380+0000","last_fullsized":"2023-10-09T10:49:58.773380+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:40.995492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":160,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.753944+0000","last_change":"2023-10-09T10:49:55.824327+0000","last_active":"2023-10-09T10:49:57.753944+0000","last_peered":"2023-10-09T10:49:57.753944+0000","last_clean":"2023-10-09T10:49:57.753944+0000","last_became_active":"2023-10-09T10:49:55.824049+0000","last_became_peered":"2023-10-09T10:49:55.824049+0000","last_unstale":"2023-10-09T10:49:57.753944+0000","last_undegraded":"2023-10-09T10:49:57.753944+0000","last_fullsized":"2023-10-09T10:49:57.753944+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:10:51.148237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":81,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.748737+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:49:57.748737+0000","last_peered":"2023-10-09T10:49:57.748737+0000","last_clean":"2023-10-09T10:49:57.748737+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:49:57.748737+0000","last_undegraded":"2023-10-09T10:49:57.748737+0000","last_fullsized":"2023-10-09T10:49:57.748737+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":191,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.773448+0000","last_change":"2023-10-09T10:49:55.811228+0000","last_active":"2023-10-09T10:49:58.773448+0000","last_peered":"2023-10-09T10:49:58.773448+0000","last_clean":"2023-10-09T10:49:58.773448+0000","last_became_active":"2023-10-09T10:49:55.810914+0000","last_became_peered":"2023-10-09T10:49:55.810914+0000","last_unstale":"2023-10-09T10:49:58.773448+0000","last_undegraded":"2023-10-09T10:49:58.773448+0000","last_fullsized":"2023-10-09T10:49:58.773448+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:35.753543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":503,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.785371+0000","last_change":"2023-10-09T10:49:58.785371+0000","last_active":"2023-10-09T10:49:58.785371+0000","last_peered":"2023-10-09T10:49:58.785371+0000","last_clean":"2023-10-09T10:49:58.785371+0000","last_became_active":"2023-10-09T10:49:58.785190+0000","last_became_peered":"2023-10-09T10:49:58.785190+0000","last_unstale":"2023-10-09T10:49:58.785371+0000","last_undegraded":"2023-10-09T10:49:58.785371+0000","last_fullsized":"2023-10-09T10:49:58.785371+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":197,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.753480+0000","last_change":"2023-10-09T10:49:55.824836+0000","last_active":"2023-10-09T10:49:57.753480+0000","last_peered":"2023-10-09T10:49:57.753480+0000","last_clean":"2023-10-09T10:49:57.753480+0000","last_became_active":"2023-10-09T10:49:55.824581+0000","last_became_peered":"2023-10-09T10:49:55.824581+0000","last_unstale":"2023-10-09T10:49:57.753480+0000","last_undegraded":"2023-10-09T10:49:57.753480+0000","last_fullsized":"2023-10-09T10:49:57.753480+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:12:05.320238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":148,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.773453+0000","last_change":"2023-10-09T10:49:40.951753+0000","last_active":"2023-10-09T10:49:58.773453+0000","last_peered":"2023-10-09T10:49:58.773453+0000","last_clean":"2023-10-09T10:49:58.773453+0000","last_became_active":"2023-10-09T10:49:40.951441+0000","last_became_peered":"2023-10-09T10:49:40.951441+0000","last_unstale":"2023-10-09T10:49:58.773453+0000","last_undegraded":"2023-10-09T10:49:58.773453+0000","last_fullsized":"2023-10-09T10:49:58.773453+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:13.222334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":150,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.777356+0000","last_change":"2023-10-09T10:49:40.968706+0000","last_active":"2023-10-09T10:49:58.777356+0000","last_peered":"2023-10-09T10:49:58.777356+0000","last_clean":"2023-10-09T10:49:58.777356+0000","last_became_active":"2023-10-09T10:49:40.947149+0000","last_became_peered":"2023-10-09T10:49:40.947149+0000","last_unstale":"2023-10-09T10:49:58.777356+0000","last_undegraded":"2023-10-09T10:49:58.777356+0000","last_fullsized":"2023-10-09T10:49:58.777356+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:57:27.340414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":121,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.760861+0000","last_change":"2023-10-09T10:49:55.836846+0000","last_active":"2023-10-09T10:49:58.760861+0000","last_peered":"2023-10-09T10:49:58.760861+0000","last_clean":"2023-10-09T10:49:58.760861+0000","last_became_active":"2023-10-09T10:49:55.836134+0000","last_became_peered":"2023-10-09T10:49:55.836134+0000","last_unstale":"2023-10-09T10:49:58.760861+0000","last_undegraded":"2023-10-09T10:49:58.760861+0000","last_fullsized":"2023-10-09T10:49:58.760861+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:13:48.026449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":202,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.760939+0000","last_change":"2023-10-09T10:49:40.976212+0000","last_active":"2023-10-09T10:49:57.760939+0000","last_peered":"2023-10-09T10:49:57.760939+0000","last_clean":"2023-10-09T10:49:57.760939+0000","last_became_active":"2023-10-09T10:49:40.950978+0000","last_became_peered":"2023-10-09T10:49:40.950978+0000","last_unstale":"2023-10-09T10:49:57.760939+0000","last_undegraded":"2023-10-09T10:49:57.760939+0000","last_fullsized":"2023-10-09T10:49:57.760939+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:28.789737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":186,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.761942+0000","last_change":"2023-10-09T10:49:55.818343+0000","last_active":"2023-10-09T10:49:57.761942+0000","last_peered":"2023-10-09T10:49:57.761942+0000","last_clean":"2023-10-09T10:49:57.761942+0000","last_became_active":"2023-10-09T10:49:55.818010+0000","last_became_peered":"2023-10-09T10:49:55.818010+0000","last_unstale":"2023-10-09T10:49:57.761942+0000","last_undegraded":"2023-10-09T10:49:57.761942+0000","last_fullsized":"2023-10-09T10:49:57.761942+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:21.520307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":576,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.761283+0000","last_change":"2023-10-09T10:49:18.708370+0000","last_active":"2023-10-09T10:49:58.761283+0000","last_peered":"2023-10-09T10:49:58.761283+0000","last_clean":"2023-10-09T10:49:58.761283+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:49:58.761283+0000","last_undegraded":"2023-10-09T10:49:58.761283+0000","last_fullsized":"2023-10-09T10:49:58.761283+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:57.808410+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":39,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.761968+0000","last_change":"2023-10-09T10:49:40.980682+0000","last_active":"2023-10-09T10:49:57.761968+0000","last_peered":"2023-10-09T10:49:57.761968+0000","last_clean":"2023-10-09T10:49:57.761968+0000","last_became_active":"2023-10-09T10:49:40.951002+0000","last_became_peered":"2023-10-09T10:49:40.951002+0000","last_unstale":"2023-10-09T10:49:57.761968+0000","last_undegraded":"2023-10-09T10:49:57.761968+0000","last_fullsized":"2023-10-09T10:49:57.761968+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:54:07.300479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":223,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.774838+0000","last_change":"2023-10-09T10:49:55.817307+0000","last_active":"2023-10-09T10:49:58.774838+0000","last_peered":"2023-10-09T10:49:58.774838+0000","last_clean":"2023-10-09T10:49:58.774838+0000","last_became_active":"2023-10-09T10:49:55.815385+0000","last_became_peered":"2023-10-09T10:49:55.815385+0000","last_unstale":"2023-10-09T10:49:58.774838+0000","last_undegraded":"2023-10-09T10:49:58.774838+0000","last_fullsized":"2023-10-09T10:49:58.774838+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:51:11.536505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":587,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.752555+0000","last_change":"2023-10-09T10:49:43.769673+0000","last_active":"2023-10-09T10:49:57.752555+0000","last_peered":"2023-10-09T10:49:57.752555+0000","last_clean":"2023-10-09T10:49:57.752555+0000","last_became_active":"2023-10-09T10:49:43.769323+0000","last_became_peered":"2023-10-09T10:49:43.769323+0000","last_unstale":"2023-10-09T10:49:57.752555+0000","last_undegraded":"2023-10-09T10:49:57.752555+0000","last_fullsized":"2023-10-09T10:49:57.752555+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:23.224244+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":100,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.761067+0000","last_change":"2023-10-09T10:49:55.837023+0000","last_active":"2023-10-09T10:49:58.761067+0000","last_peered":"2023-10-09T10:49:58.761067+0000","last_clean":"2023-10-09T10:49:58.761067+0000","last_became_active":"2023-10-09T10:49:55.836546+0000","last_became_peered":"2023-10-09T10:49:55.836546+0000","last_unstale":"2023-10-09T10:49:58.761067+0000","last_undegraded":"2023-10-09T10:49:58.761067+0000","last_fullsized":"2023-10-09T10:49:58.761067+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:30.585420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":186,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.753541+0000","last_change":"2023-10-09T10:49:40.952783+0000","last_active":"2023-10-09T10:49:57.753541+0000","last_peered":"2023-10-09T10:49:57.753541+0000","last_clean":"2023-10-09T10:49:57.753541+0000","last_became_active":"2023-10-09T10:49:40.952421+0000","last_became_peered":"2023-10-09T10:49:40.952421+0000","last_unstale":"2023-10-09T10:49:57.753541+0000","last_undegraded":"2023-10-09T10:49:57.753541+0000","last_fullsized":"2023-10-09T10:49:57.753541+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:08:57.147258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":553,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:59.699024+0000","last_change":"2023-10-09T10:49:58.797203+0000","last_active":"2023-10-09T10:49:59.699024+0000","last_peered":"2023-10-09T10:49:59.699024+0000","last_clean":"2023-10-09T10:49:59.699024+0000","last_became_active":"2023-10-09T10:49:58.796656+0000","last_became_peered":"2023-10-09T10:49:58.796656+0000","last_unstale":"2023-10-09T10:49:59.699024+0000","last_undegraded":"2023-10-09T10:49:59.699024+0000","last_fullsized":"2023-10-09T10:49:59.699024+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":132,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.761588+0000","last_change":"2023-10-09T10:49:40.952105+0000","last_active":"2023-10-09T10:49:57.761588+0000","last_peered":"2023-10-09T10:49:57.761588+0000","last_clean":"2023-10-09T10:49:57.761588+0000","last_became_active":"2023-10-09T10:49:40.950553+0000","last_became_peered":"2023-10-09T10:49:40.950553+0000","last_unstale":"2023-10-09T10:49:57.761588+0000","last_undegraded":"2023-10-09T10:49:57.761588+0000","last_fullsized":"2023-10-09T10:49:57.761588+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:51:40.501639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":90,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.748712+0000","last_change":"2023-10-09T10:49:40.949501+0000","last_active":"2023-10-09T10:49:57.748712+0000","last_peered":"2023-10-09T10:49:57.748712+0000","last_clean":"2023-10-09T10:49:57.748712+0000","last_became_active":"2023-10-09T10:49:40.949158+0000","last_became_peered":"2023-10-09T10:49:40.949158+0000","last_unstale":"2023-10-09T10:49:57.748712+0000","last_undegraded":"2023-10-09T10:49:57.748712+0000","last_fullsized":"2023-10-09T10:49:57.748712+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:07.598560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":587,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.776174+0000","last_change":"2023-10-09T10:49:58.776174+0000","last_active":"2023-10-09T10:49:58.776174+0000","last_peered":"2023-10-09T10:49:58.776174+0000","last_clean":"2023-10-09T10:49:58.776174+0000","last_became_active":"2023-10-09T10:49:58.775795+0000","last_became_peered":"2023-10-09T10:49:58.775795+0000","last_unstale":"2023-10-09T10:49:58.776174+0000","last_undegraded":"2023-10-09T10:49:58.776174+0000","last_fullsized":"2023-10-09T10:49:58.776174+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:47.161519+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":92,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.760926+0000","last_change":"2023-10-09T10:46:02.682780+0000","last_active":"2023-10-09T10:49:58.760926+0000","last_peered":"2023-10-09T10:49:58.760926+0000","last_clean":"2023-10-09T10:49:58.760926+0000","last_became_active":"2023-10-09T10:46:02.638653+0000","last_became_peered":"2023-10-09T10:46:02.638653+0000","last_unstale":"2023-10-09T10:49:58.760926+0000","last_undegraded":"2023-10-09T10:49:58.760926+0000","last_fullsized":"2023-10-09T10:49:58.760926+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:21.967771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":200,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.778605+0000","last_change":"2023-10-09T10:49:55.814525+0000","last_active":"2023-10-09T10:49:58.778605+0000","last_peered":"2023-10-09T10:49:58.778605+0000","last_clean":"2023-10-09T10:49:58.778605+0000","last_became_active":"2023-10-09T10:49:55.814228+0000","last_became_peered":"2023-10-09T10:49:55.814228+0000","last_unstale":"2023-10-09T10:49:58.778605+0000","last_undegraded":"2023-10-09T10:49:58.778605+0000","last_fullsized":"2023-10-09T10:49:58.778605+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:19.868347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"212'3403","reported_seq":3437,"reported_epoch":212,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:01.181933+0000","last_change":"2023-10-09T10:49:56.964145+0000","last_active":"2023-10-09T10:50:01.181933+0000","last_peered":"2023-10-09T10:50:01.181933+0000","last_clean":"2023-10-09T10:49:54.739089+0000","last_became_active":"2023-10-09T10:49:56.954628+0000","last_became_peered":"2023-10-09T10:49:56.954628+0000","last_unstale":"2023-10-09T10:50:01.181933+0000","last_undegraded":"2023-10-09T10:49:56.831316+0000","last_fullsized":"2023-10-09T10:49:56.808257+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3403,"log_dups_size":0,"ondisk_log_size":3403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:22.039147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223019008,"num_objects":3403,"num_object_clones":0,"num_object_copies":10209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3165,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3403,"num_write_kb":217792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":3165},{"shards":"5,6","objects":238}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":503,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.790502+0000","last_change":"2023-10-09T10:49:58.790502+0000","last_active":"2023-10-09T10:49:58.790502+0000","last_peered":"2023-10-09T10:49:58.790502+0000","last_clean":"2023-10-09T10:49:58.790502+0000","last_became_active":"2023-10-09T10:49:58.790151+0000","last_became_peered":"2023-10-09T10:49:58.790151+0000","last_unstale":"2023-10-09T10:49:58.790502+0000","last_undegraded":"2023-10-09T10:49:58.790502+0000","last_fullsized":"2023-10-09T10:49:58.790502+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":51,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.766991+0000","last_change":"2023-10-09T10:49:55.846025+0000","last_active":"2023-10-09T10:49:57.766991+0000","last_peered":"2023-10-09T10:49:57.766991+0000","last_clean":"2023-10-09T10:49:57.766991+0000","last_became_active":"2023-10-09T10:49:55.821145+0000","last_became_peered":"2023-10-09T10:49:55.821145+0000","last_unstale":"2023-10-09T10:49:57.766991+0000","last_undegraded":"2023-10-09T10:49:57.766991+0000","last_fullsized":"2023-10-09T10:49:57.766991+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:50.726370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":169,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.752319+0000","last_change":"2023-10-09T10:49:40.955088+0000","last_active":"2023-10-09T10:49:57.752319+0000","last_peered":"2023-10-09T10:49:57.752319+0000","last_clean":"2023-10-09T10:49:57.752319+0000","last_became_active":"2023-10-09T10:49:40.954716+0000","last_became_peered":"2023-10-09T10:49:40.954716+0000","last_unstale":"2023-10-09T10:49:57.752319+0000","last_undegraded":"2023-10-09T10:49:57.752319+0000","last_fullsized":"2023-10-09T10:49:57.752319+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:39.090107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"213'3574","reported_seq":3608,"reported_epoch":213,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:04.691997+0000","last_change":"2023-10-09T10:49:57.045491+0000","last_active":"2023-10-09T10:50:04.691997+0000","last_peered":"2023-10-09T10:50:04.691997+0000","last_clean":"2023-10-09T10:49:54.734240+0000","last_became_active":"2023-10-09T10:49:57.036090+0000","last_became_peered":"2023-10-09T10:49:57.036090+0000","last_unstale":"2023-10-09T10:50:04.691997+0000","last_undegraded":"2023-10-09T10:49:56.833807+0000","last_fullsized":"2023-10-09T10:49:56.805766+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3574,"log_dups_size":0,"ondisk_log_size":3574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:14.052065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234225664,"num_objects":3574,"num_object_clones":0,"num_object_copies":10722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3125,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3574,"num_write_kb":228736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,5","objects":3125},{"shards":"3,5","objects":449}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":548,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.774742+0000","last_change":"2023-10-09T10:49:41.638077+0000","last_active":"2023-10-09T10:49:58.774742+0000","last_peered":"2023-10-09T10:49:58.774742+0000","last_clean":"2023-10-09T10:49:58.774742+0000","last_became_active":"2023-10-09T10:49:17.844627+0000","last_became_peered":"2023-10-09T10:49:17.844627+0000","last_unstale":"2023-10-09T10:49:58.774742+0000","last_undegraded":"2023-10-09T10:49:58.774742+0000","last_fullsized":"2023-10-09T10:49:58.774742+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:41.637984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:41.637984+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:39:39.594702+0000","scrub_duration":0.001804833,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":51,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.763234+0000","last_change":"2023-10-09T10:49:55.829966+0000","last_active":"2023-10-09T10:49:57.763234+0000","last_peered":"2023-10-09T10:49:57.763234+0000","last_clean":"2023-10-09T10:49:57.763234+0000","last_became_active":"2023-10-09T10:49:55.820924+0000","last_became_peered":"2023-10-09T10:49:55.820924+0000","last_unstale":"2023-10-09T10:49:57.763234+0000","last_undegraded":"2023-10-09T10:49:57.763234+0000","last_fullsized":"2023-10-09T10:49:57.763234+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:53.558978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":223,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.761505+0000","last_change":"2023-10-09T10:49:55.810641+0000","last_active":"2023-10-09T10:49:57.761505+0000","last_peered":"2023-10-09T10:49:57.761505+0000","last_clean":"2023-10-09T10:49:57.761505+0000","last_became_active":"2023-10-09T10:49:55.810297+0000","last_became_peered":"2023-10-09T10:49:55.810297+0000","last_unstale":"2023-10-09T10:49:57.761505+0000","last_undegraded":"2023-10-09T10:49:57.761505+0000","last_fullsized":"2023-10-09T10:49:57.761505+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:58.168147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"212'3616","reported_seq":3648,"reported_epoch":212,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:02.887228+0000","last_change":"2023-10-09T10:49:56.964132+0000","last_active":"2023-10-09T10:50:02.887228+0000","last_peered":"2023-10-09T10:50:02.887228+0000","last_clean":"2023-10-09T10:49:54.793473+0000","last_became_active":"2023-10-09T10:49:56.956688+0000","last_became_peered":"2023-10-09T10:49:56.956688+0000","last_unstale":"2023-10-09T10:50:02.887228+0000","last_undegraded":"2023-10-09T10:49:56.831390+0000","last_fullsized":"2023-10-09T10:49:56.801929+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3616,"log_dups_size":0,"ondisk_log_size":3616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:06:54.970299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236978176,"num_objects":3616,"num_object_clones":0,"num_object_copies":10848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3263,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3616,"num_write_kb":231424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3263},{"shards":"0,4","objects":353}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":557,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.753837+0000","last_change":"2023-10-09T10:49:17.848065+0000","last_active":"2023-10-09T10:49:57.753837+0000","last_peered":"2023-10-09T10:49:57.753837+0000","last_clean":"2023-10-09T10:49:57.753837+0000","last_became_active":"2023-10-09T10:49:17.847694+0000","last_became_peered":"2023-10-09T10:49:17.847694+0000","last_unstale":"2023-10-09T10:49:57.753837+0000","last_undegraded":"2023-10-09T10:49:57.753837+0000","last_fullsized":"2023-10-09T10:49:57.753837+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":147,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.753454+0000","last_change":"2023-10-09T10:49:40.958240+0000","last_active":"2023-10-09T10:49:57.753454+0000","last_peered":"2023-10-09T10:49:57.753454+0000","last_clean":"2023-10-09T10:49:57.753454+0000","last_became_active":"2023-10-09T10:49:40.952355+0000","last_became_peered":"2023-10-09T10:49:40.952355+0000","last_unstale":"2023-10-09T10:49:57.753454+0000","last_undegraded":"2023-10-09T10:49:57.753454+0000","last_fullsized":"2023-10-09T10:49:57.753454+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:30.412691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":158,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.753203+0000","last_change":"2023-10-09T10:49:55.847259+0000","last_active":"2023-10-09T10:49:57.753203+0000","last_peered":"2023-10-09T10:49:57.753203+0000","last_clean":"2023-10-09T10:49:57.753203+0000","last_became_active":"2023-10-09T10:49:55.847042+0000","last_became_peered":"2023-10-09T10:49:55.847042+0000","last_unstale":"2023-10-09T10:49:57.753203+0000","last_undegraded":"2023-10-09T10:49:57.753203+0000","last_fullsized":"2023-10-09T10:49:57.753203+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:08.261645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.8","version":"212'3433","reported_seq":4305,"reported_epoch":212,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:01.119880+0000","last_change":"2023-10-09T10:49:55.850868+0000","last_active":"2023-10-09T10:50:01.119880+0000","last_peered":"2023-10-09T10:50:01.119880+0000","last_clean":"2023-10-09T10:49:39.925425+0000","last_became_active":"2023-10-09T10:49:42.043307+0000","last_became_peered":"2023-10-09T10:49:42.043307+0000","last_unstale":"2023-10-09T10:50:01.119880+0000","last_undegraded":"2023-10-09T10:49:41.952596+0000","last_fullsized":"2023-10-09T10:49:41.940762+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3433,"log_dups_size":0,"ondisk_log_size":3433,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:56:14.382421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224985088,"num_objects":3433,"num_object_clones":0,"num_object_copies":10299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":1986,"num_objects_unfound":0,"num_objects_dirty":3433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3433,"num_write_kb":219712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":420,"num_bytes_recovered":27525120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4","objects":1445},{"shards":"3,4,7","objects":1986}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":597,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.784989+0000","last_change":"2023-10-09T10:49:58.784989+0000","last_active":"2023-10-09T10:49:58.784989+0000","last_peered":"2023-10-09T10:49:58.784989+0000","last_clean":"2023-10-09T10:49:58.784989+0000","last_became_active":"2023-10-09T10:49:58.784715+0000","last_became_peered":"2023-10-09T10:49:58.784715+0000","last_unstale":"2023-10-09T10:49:58.784989+0000","last_undegraded":"2023-10-09T10:49:58.784989+0000","last_fullsized":"2023-10-09T10:49:58.784989+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":177,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.753300+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:49:57.753300+0000","last_peered":"2023-10-09T10:49:57.753300+0000","last_clean":"2023-10-09T10:49:57.753300+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:49:57.753300+0000","last_undegraded":"2023-10-09T10:49:57.753300+0000","last_fullsized":"2023-10-09T10:49:57.753300+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":91,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.763148+0000","last_change":"2023-10-09T10:49:55.830400+0000","last_active":"2023-10-09T10:49:58.763148+0000","last_peered":"2023-10-09T10:49:58.763148+0000","last_clean":"2023-10-09T10:49:58.763148+0000","last_became_active":"2023-10-09T10:49:55.829982+0000","last_became_peered":"2023-10-09T10:49:55.829982+0000","last_unstale":"2023-10-09T10:49:58.763148+0000","last_undegraded":"2023-10-09T10:49:58.763148+0000","last_fullsized":"2023-10-09T10:49:58.763148+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:51.352143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"212'3506","reported_seq":3539,"reported_epoch":212,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:02.869309+0000","last_change":"2023-10-09T10:49:42.062469+0000","last_active":"2023-10-09T10:50:02.869309+0000","last_peered":"2023-10-09T10:50:02.869309+0000","last_clean":"2023-10-09T10:49:39.916756+0000","last_became_active":"2023-10-09T10:49:42.056867+0000","last_became_peered":"2023-10-09T10:49:42.056867+0000","last_unstale":"2023-10-09T10:50:02.869309+0000","last_undegraded":"2023-10-09T10:49:41.970254+0000","last_fullsized":"2023-10-09T10:49:41.933564+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3506,"log_dups_size":0,"ondisk_log_size":3506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:59:10.773024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229769216,"num_objects":3506,"num_object_clones":0,"num_object_copies":10518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2419,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3506,"num_write_kb":224384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":1087},{"shards":"0,3,7","objects":2419}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":564,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.773270+0000","last_change":"2023-10-09T10:49:43.769990+0000","last_active":"2023-10-09T10:49:58.773270+0000","last_peered":"2023-10-09T10:49:58.773270+0000","last_clean":"2023-10-09T10:49:58.773270+0000","last_became_active":"2023-10-09T10:49:43.769783+0000","last_became_peered":"2023-10-09T10:49:43.769783+0000","last_unstale":"2023-10-09T10:49:58.773270+0000","last_undegraded":"2023-10-09T10:49:58.773270+0000","last_fullsized":"2023-10-09T10:49:58.773270+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":206,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.773479+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:49:58.773479+0000","last_peered":"2023-10-09T10:49:58.773479+0000","last_clean":"2023-10-09T10:49:58.773479+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:49:58.773479+0000","last_undegraded":"2023-10-09T10:49:58.773479+0000","last_fullsized":"2023-10-09T10:49:58.773479+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":176,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.760974+0000","last_change":"2023-10-09T10:43:35.493287+0000","last_active":"2023-10-09T10:49:57.760974+0000","last_peered":"2023-10-09T10:49:57.760974+0000","last_clean":"2023-10-09T10:49:57.760974+0000","last_became_active":"2023-10-09T10:43:35.492918+0000","last_became_peered":"2023-10-09T10:43:35.492918+0000","last_unstale":"2023-10-09T10:49:57.760974+0000","last_undegraded":"2023-10-09T10:49:57.760974+0000","last_fullsized":"2023-10-09T10:49:57.760974+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:36.586199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"212'3400","reported_seq":3424,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:50:01.170806+0000","last_change":"2023-10-09T10:49:16.843562+0000","last_active":"2023-10-09T10:50:01.170806+0000","last_peered":"2023-10-09T10:50:01.170806+0000","last_clean":"2023-10-09T10:50:01.170806+0000","last_became_active":"2023-10-09T10:49:16.843246+0000","last_became_peered":"2023-10-09T10:49:16.843246+0000","last_unstale":"2023-10-09T10:50:01.170806+0000","last_undegraded":"2023-10-09T10:50:01.170806+0000","last_fullsized":"2023-10-09T10:50:01.170806+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3400,"log_dups_size":0,"ondisk_log_size":3400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:47.911382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222822400,"num_objects":3400,"num_object_clones":0,"num_object_copies":10200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3400,"num_write_kb":217600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":537,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.752539+0000","last_change":"2023-10-09T10:49:17.836306+0000","last_active":"2023-10-09T10:49:57.752539+0000","last_peered":"2023-10-09T10:49:57.752539+0000","last_clean":"2023-10-09T10:49:57.752539+0000","last_became_active":"2023-10-09T10:49:17.836052+0000","last_became_peered":"2023-10-09T10:49:17.836052+0000","last_unstale":"2023-10-09T10:49:57.752539+0000","last_undegraded":"2023-10-09T10:49:57.752539+0000","last_fullsized":"2023-10-09T10:49:57.752539+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":161,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.760811+0000","last_change":"2023-10-09T10:49:55.814556+0000","last_active":"2023-10-09T10:49:57.760811+0000","last_peered":"2023-10-09T10:49:57.760811+0000","last_clean":"2023-10-09T10:49:57.760811+0000","last_became_active":"2023-10-09T10:49:55.814193+0000","last_became_peered":"2023-10-09T10:49:55.814193+0000","last_unstale":"2023-10-09T10:49:57.760811+0000","last_undegraded":"2023-10-09T10:49:57.760811+0000","last_fullsized":"2023-10-09T10:49:57.760811+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:13:19.537756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":213,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.762656+0000","last_change":"2023-10-09T10:49:55.822597+0000","last_active":"2023-10-09T10:49:57.762656+0000","last_peered":"2023-10-09T10:49:57.762656+0000","last_clean":"2023-10-09T10:49:57.762656+0000","last_became_active":"2023-10-09T10:49:55.822289+0000","last_became_peered":"2023-10-09T10:49:55.822289+0000","last_unstale":"2023-10-09T10:49:57.762656+0000","last_undegraded":"2023-10-09T10:49:57.762656+0000","last_fullsized":"2023-10-09T10:49:57.762656+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:55.211374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"212'3464","reported_seq":3496,"reported_epoch":212,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:02.846090+0000","last_change":"2023-10-09T10:49:57.010520+0000","last_active":"2023-10-09T10:50:02.846090+0000","last_peered":"2023-10-09T10:50:02.846090+0000","last_clean":"2023-10-09T10:49:54.776483+0000","last_became_active":"2023-10-09T10:49:56.978281+0000","last_became_peered":"2023-10-09T10:49:56.978281+0000","last_unstale":"2023-10-09T10:50:02.846090+0000","last_undegraded":"2023-10-09T10:49:56.833097+0000","last_fullsized":"2023-10-09T10:49:56.802064+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3464,"log_dups_size":0,"ondisk_log_size":3464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:22:25.484896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227016704,"num_objects":3464,"num_object_clones":0,"num_object_copies":10392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3118,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3464,"num_write_kb":221696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":3118},{"shards":"0,6","objects":346}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":558,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.783707+0000","last_change":"2023-10-09T10:49:58.781265+0000","last_active":"2023-10-09T10:49:58.783707+0000","last_peered":"2023-10-09T10:49:58.783707+0000","last_clean":"2023-10-09T10:49:58.783707+0000","last_became_active":"2023-10-09T10:49:58.780915+0000","last_became_peered":"2023-10-09T10:49:58.780915+0000","last_unstale":"2023-10-09T10:49:58.783707+0000","last_undegraded":"2023-10-09T10:49:58.783707+0000","last_fullsized":"2023-10-09T10:49:58.783707+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.892515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.892515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":207,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.761413+0000","last_change":"2023-10-09T10:49:55.812746+0000","last_active":"2023-10-09T10:49:57.761413+0000","last_peered":"2023-10-09T10:49:57.761413+0000","last_clean":"2023-10-09T10:49:57.761413+0000","last_became_active":"2023-10-09T10:49:55.812438+0000","last_became_peered":"2023-10-09T10:49:55.812438+0000","last_unstale":"2023-10-09T10:49:57.761413+0000","last_undegraded":"2023-10-09T10:49:57.761413+0000","last_fullsized":"2023-10-09T10:49:57.761413+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:44:14.045846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":186,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.763148+0000","last_change":"2023-10-09T10:49:55.813906+0000","last_active":"2023-10-09T10:49:57.763148+0000","last_peered":"2023-10-09T10:49:57.763148+0000","last_clean":"2023-10-09T10:49:57.763148+0000","last_became_active":"2023-10-09T10:49:55.813599+0000","last_became_peered":"2023-10-09T10:49:55.813599+0000","last_unstale":"2023-10-09T10:49:57.763148+0000","last_undegraded":"2023-10-09T10:49:57.763148+0000","last_fullsized":"2023-10-09T10:49:57.763148+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:15.418370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"213'3506","reported_seq":6600,"reported_epoch":213,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:50:04.712551+0000","last_change":"2023-10-09T10:49:55.857052+0000","last_active":"2023-10-09T10:50:04.712551+0000","last_peered":"2023-10-09T10:50:04.712551+0000","last_clean":"2023-10-09T10:49:39.921112+0000","last_became_active":"2023-10-09T10:49:42.019409+0000","last_became_peered":"2023-10-09T10:49:42.019409+0000","last_unstale":"2023-10-09T10:50:04.712551+0000","last_undegraded":"2023-10-09T10:50:04.712551+0000","last_fullsized":"2023-10-09T10:49:41.933019+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3506,"log_dups_size":0,"ondisk_log_size":3506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:05.671978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229769216,"num_objects":3506,"num_object_clones":0,"num_object_copies":10518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":819,"num_objects_unfound":0,"num_objects_dirty":3506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3506,"num_write_kb":224384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1528,"num_bytes_recovered":100139008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6","objects":2687},{"shards":"3,6,7","objects":819}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":548,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.753601+0000","last_change":"2023-10-09T10:49:17.847894+0000","last_active":"2023-10-09T10:49:57.753601+0000","last_peered":"2023-10-09T10:49:57.753601+0000","last_clean":"2023-10-09T10:49:57.753601+0000","last_became_active":"2023-10-09T10:49:17.847624+0000","last_became_peered":"2023-10-09T10:49:17.847624+0000","last_unstale":"2023-10-09T10:49:57.753601+0000","last_undegraded":"2023-10-09T10:49:57.753601+0000","last_fullsized":"2023-10-09T10:49:57.753601+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":202,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T10:50:02.896823+0000","last_change":"2023-10-09T10:49:55.819881+0000","last_active":"2023-10-09T10:50:02.896823+0000","last_peered":"2023-10-09T10:50:02.896823+0000","last_clean":"2023-10-09T10:50:02.896823+0000","last_became_active":"2023-10-09T10:49:55.818533+0000","last_became_peered":"2023-10-09T10:49:55.818533+0000","last_unstale":"2023-10-09T10:50:02.896823+0000","last_undegraded":"2023-10-09T10:50:02.896823+0000","last_fullsized":"2023-10-09T10:50:02.896823+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:20.512152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":54,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:59.699192+0000","last_change":"2023-10-09T10:49:55.815880+0000","last_active":"2023-10-09T10:49:59.699192+0000","last_peered":"2023-10-09T10:49:59.699192+0000","last_clean":"2023-10-09T10:49:59.699192+0000","last_became_active":"2023-10-09T10:49:55.815594+0000","last_became_peered":"2023-10-09T10:49:55.815594+0000","last_unstale":"2023-10-09T10:49:59.699192+0000","last_undegraded":"2023-10-09T10:49:59.699192+0000","last_fullsized":"2023-10-09T10:49:59.699192+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:39.210709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"212'3280","reported_seq":3304,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:50:01.097183+0000","last_change":"2023-10-09T10:49:16.848544+0000","last_active":"2023-10-09T10:50:01.097183+0000","last_peered":"2023-10-09T10:50:01.097183+0000","last_clean":"2023-10-09T10:50:01.097183+0000","last_became_active":"2023-10-09T10:49:16.848251+0000","last_became_peered":"2023-10-09T10:49:16.848251+0000","last_unstale":"2023-10-09T10:50:01.097183+0000","last_undegraded":"2023-10-09T10:50:01.097183+0000","last_fullsized":"2023-10-09T10:50:01.097183+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3280,"log_dups_size":0,"ondisk_log_size":3280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:01.631428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214958080,"num_objects":3280,"num_object_clones":0,"num_object_copies":9840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3280,"num_write_kb":209920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":585,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.778100+0000","last_change":"2023-10-09T10:49:43.786891+0000","last_active":"2023-10-09T10:49:58.778100+0000","last_peered":"2023-10-09T10:49:58.778100+0000","last_clean":"2023-10-09T10:49:58.778100+0000","last_became_active":"2023-10-09T10:49:43.786665+0000","last_became_peered":"2023-10-09T10:49:43.786665+0000","last_unstale":"2023-10-09T10:49:58.778100+0000","last_undegraded":"2023-10-09T10:49:58.778100+0000","last_fullsized":"2023-10-09T10:49:58.778100+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.687290+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.687290+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:36.786558+0000","scrub_duration":0.012763172999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":214,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.891898+0000","last_change":"2023-10-09T10:49:55.825069+0000","last_active":"2023-10-09T10:49:57.891898+0000","last_peered":"2023-10-09T10:49:57.891898+0000","last_clean":"2023-10-09T10:49:57.891898+0000","last_became_active":"2023-10-09T10:49:55.824755+0000","last_became_peered":"2023-10-09T10:49:55.824755+0000","last_unstale":"2023-10-09T10:49:57.891898+0000","last_undegraded":"2023-10-09T10:49:57.891898+0000","last_fullsized":"2023-10-09T10:49:57.891898+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:04:28.113626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":204,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.753617+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:49:57.753617+0000","last_peered":"2023-10-09T10:49:57.753617+0000","last_clean":"2023-10-09T10:49:57.753617+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:49:57.753617+0000","last_undegraded":"2023-10-09T10:49:57.753617+0000","last_fullsized":"2023-10-09T10:49:57.753617+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":29,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.753461+0000","last_change":"2023-10-09T10:49:55.828773+0000","last_active":"2023-10-09T10:49:57.753461+0000","last_peered":"2023-10-09T10:49:57.753461+0000","last_clean":"2023-10-09T10:49:57.753461+0000","last_became_active":"2023-10-09T10:49:55.828458+0000","last_became_peered":"2023-10-09T10:49:55.828458+0000","last_unstale":"2023-10-09T10:49:57.753461+0000","last_undegraded":"2023-10-09T10:49:57.753461+0000","last_fullsized":"2023-10-09T10:49:57.753461+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:30.196079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"212'3579","reported_seq":3604,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:50:02.878170+0000","last_change":"2023-10-09T10:49:16.843057+0000","last_active":"2023-10-09T10:50:02.878170+0000","last_peered":"2023-10-09T10:50:02.878170+0000","last_clean":"2023-10-09T10:50:02.878170+0000","last_became_active":"2023-10-09T10:49:16.842812+0000","last_became_peered":"2023-10-09T10:49:16.842812+0000","last_unstale":"2023-10-09T10:50:02.878170+0000","last_undegraded":"2023-10-09T10:50:02.878170+0000","last_fullsized":"2023-10-09T10:50:02.878170+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3579,"log_dups_size":0,"ondisk_log_size":3579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:37.333788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234553344,"num_objects":3579,"num_object_clones":0,"num_object_copies":10737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3579,"num_write_kb":229056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":566,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.752638+0000","last_change":"2023-10-09T10:49:38.736950+0000","last_active":"2023-10-09T10:49:57.752638+0000","last_peered":"2023-10-09T10:49:57.752638+0000","last_clean":"2023-10-09T10:49:57.752638+0000","last_became_active":"2023-10-09T10:49:17.837821+0000","last_became_peered":"2023-10-09T10:49:17.837821+0000","last_unstale":"2023-10-09T10:49:57.752638+0000","last_undegraded":"2023-10-09T10:49:57.752638+0000","last_fullsized":"2023-10-09T10:49:57.752638+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.736834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.736834+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:17:16.734495+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":191,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.753392+0000","last_change":"2023-10-09T10:49:55.847216+0000","last_active":"2023-10-09T10:49:57.753392+0000","last_peered":"2023-10-09T10:49:57.753392+0000","last_clean":"2023-10-09T10:49:57.753392+0000","last_became_active":"2023-10-09T10:49:55.847002+0000","last_became_peered":"2023-10-09T10:49:55.847002+0000","last_unstale":"2023-10-09T10:49:57.753392+0000","last_undegraded":"2023-10-09T10:49:57.753392+0000","last_fullsized":"2023-10-09T10:49:57.753392+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:41.845106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":197,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.762026+0000","last_change":"2023-10-09T10:46:02.616625+0000","last_active":"2023-10-09T10:49:57.762026+0000","last_peered":"2023-10-09T10:49:57.762026+0000","last_clean":"2023-10-09T10:49:57.762026+0000","last_became_active":"2023-10-09T10:46:02.616265+0000","last_became_peered":"2023-10-09T10:46:02.616265+0000","last_unstale":"2023-10-09T10:49:57.762026+0000","last_undegraded":"2023-10-09T10:49:57.762026+0000","last_fullsized":"2023-10-09T10:49:57.762026+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:33.995126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":136,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:59.699125+0000","last_change":"2023-10-09T10:49:55.816187+0000","last_active":"2023-10-09T10:49:59.699125+0000","last_peered":"2023-10-09T10:49:59.699125+0000","last_clean":"2023-10-09T10:49:59.699125+0000","last_became_active":"2023-10-09T10:49:55.815345+0000","last_became_peered":"2023-10-09T10:49:55.815345+0000","last_unstale":"2023-10-09T10:49:59.699125+0000","last_undegraded":"2023-10-09T10:49:59.699125+0000","last_fullsized":"2023-10-09T10:49:59.699125+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:43.036104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.4","version":"212'3394","reported_seq":3431,"reported_epoch":212,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:02.864011+0000","last_change":"2023-10-09T10:49:57.003555+0000","last_active":"2023-10-09T10:50:02.864011+0000","last_peered":"2023-10-09T10:50:02.864011+0000","last_clean":"2023-10-09T10:49:54.777504+0000","last_became_active":"2023-10-09T10:49:56.956699+0000","last_became_peered":"2023-10-09T10:49:56.956699+0000","last_unstale":"2023-10-09T10:50:02.864011+0000","last_undegraded":"2023-10-09T10:49:56.938617+0000","last_fullsized":"2023-10-09T10:49:56.806441+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3394,"log_dups_size":0,"ondisk_log_size":3394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:51.048643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222429184,"num_objects":3394,"num_object_clones":0,"num_object_copies":10182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3084,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3394,"num_write_kb":217216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3083},{"shards":"0,4","objects":310}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":582,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.760931+0000","last_change":"2023-10-09T10:49:43.780951+0000","last_active":"2023-10-09T10:49:58.760931+0000","last_peered":"2023-10-09T10:49:58.760931+0000","last_clean":"2023-10-09T10:49:58.760931+0000","last_became_active":"2023-10-09T10:49:43.780564+0000","last_became_peered":"2023-10-09T10:49:43.780564+0000","last_unstale":"2023-10-09T10:49:58.760931+0000","last_undegraded":"2023-10-09T10:49:58.760931+0000","last_fullsized":"2023-10-09T10:49:58.760931+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":220,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.773640+0000","last_change":"2023-10-09T10:49:55.814470+0000","last_active":"2023-10-09T10:49:58.773640+0000","last_peered":"2023-10-09T10:49:58.773640+0000","last_clean":"2023-10-09T10:49:58.773640+0000","last_became_active":"2023-10-09T10:49:55.814154+0000","last_became_peered":"2023-10-09T10:49:55.814154+0000","last_unstale":"2023-10-09T10:49:58.773640+0000","last_undegraded":"2023-10-09T10:49:58.773640+0000","last_fullsized":"2023-10-09T10:49:58.773640+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:03:57.326294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":121,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.760374+0000","last_change":"2023-10-09T10:49:55.837979+0000","last_active":"2023-10-09T10:49:58.760374+0000","last_peered":"2023-10-09T10:49:58.760374+0000","last_clean":"2023-10-09T10:49:58.760374+0000","last_became_active":"2023-10-09T10:49:55.837682+0000","last_became_peered":"2023-10-09T10:49:55.837682+0000","last_unstale":"2023-10-09T10:49:58.760374+0000","last_undegraded":"2023-10-09T10:49:58.760374+0000","last_fullsized":"2023-10-09T10:49:58.760374+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:30:17.578000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":139,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.754015+0000","last_change":"2023-10-09T10:49:55.822332+0000","last_active":"2023-10-09T10:49:57.754015+0000","last_peered":"2023-10-09T10:49:57.754015+0000","last_clean":"2023-10-09T10:49:57.754015+0000","last_became_active":"2023-10-09T10:49:55.822068+0000","last_became_peered":"2023-10-09T10:49:55.822068+0000","last_unstale":"2023-10-09T10:49:57.754015+0000","last_undegraded":"2023-10-09T10:49:57.754015+0000","last_fullsized":"2023-10-09T10:49:57.754015+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:41:57.905151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"212'3402","reported_seq":5438,"reported_epoch":212,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:02.792347+0000","last_change":"2023-10-09T10:49:55.838303+0000","last_active":"2023-10-09T10:50:02.792347+0000","last_peered":"2023-10-09T10:50:02.792347+0000","last_clean":"2023-10-09T10:49:39.898166+0000","last_became_active":"2023-10-09T10:49:42.040841+0000","last_became_peered":"2023-10-09T10:49:42.040841+0000","last_unstale":"2023-10-09T10:50:02.792347+0000","last_undegraded":"2023-10-09T10:49:55.835709+0000","last_fullsized":"2023-10-09T10:49:41.933738+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3402,"log_dups_size":0,"ondisk_log_size":3402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:02.626804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222953472,"num_objects":3402,"num_object_clones":0,"num_object_copies":10206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1311,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3402,"num_write_kb":217728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1000,"num_bytes_recovered":65536000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":2091},{"shards":"0,3,7","objects":1311}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":201,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.754827+0000","last_change":"2023-10-09T10:49:40.951536+0000","last_active":"2023-10-09T10:49:57.754827+0000","last_peered":"2023-10-09T10:49:57.754827+0000","last_clean":"2023-10-09T10:49:57.754827+0000","last_became_active":"2023-10-09T10:49:40.951152+0000","last_became_peered":"2023-10-09T10:49:40.951152+0000","last_unstale":"2023-10-09T10:49:57.754827+0000","last_undegraded":"2023-10-09T10:49:57.754827+0000","last_fullsized":"2023-10-09T10:49:57.754827+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:29.134747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":595,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T10:50:02.896652+0000","last_change":"2023-10-09T10:49:58.796806+0000","last_active":"2023-10-09T10:50:02.896652+0000","last_peered":"2023-10-09T10:50:02.896652+0000","last_clean":"2023-10-09T10:50:02.896652+0000","last_became_active":"2023-10-09T10:49:58.796478+0000","last_became_peered":"2023-10-09T10:49:58.796478+0000","last_unstale":"2023-10-09T10:50:02.896652+0000","last_undegraded":"2023-10-09T10:50:02.896652+0000","last_fullsized":"2023-10-09T10:50:02.896652+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:44:35.791052+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":142,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.777745+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:49:58.777745+0000","last_peered":"2023-10-09T10:49:58.777745+0000","last_clean":"2023-10-09T10:49:58.777745+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:49:58.777745+0000","last_undegraded":"2023-10-09T10:49:58.777745+0000","last_fullsized":"2023-10-09T10:49:58.777745+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":119,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.762479+0000","last_change":"2023-10-09T10:49:40.951576+0000","last_active":"2023-10-09T10:49:57.762479+0000","last_peered":"2023-10-09T10:49:57.762479+0000","last_clean":"2023-10-09T10:49:57.762479+0000","last_became_active":"2023-10-09T10:49:40.951242+0000","last_became_peered":"2023-10-09T10:49:40.951242+0000","last_unstale":"2023-10-09T10:49:57.762479+0000","last_undegraded":"2023-10-09T10:49:57.762479+0000","last_fullsized":"2023-10-09T10:49:57.762479+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:05:01.483490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"212'3501","reported_seq":3546,"reported_epoch":212,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:02.864880+0000","last_change":"2023-10-09T10:49:57.053554+0000","last_active":"2023-10-09T10:50:02.864880+0000","last_peered":"2023-10-09T10:50:02.864880+0000","last_clean":"2023-10-09T10:49:39.922158+0000","last_became_active":"2023-10-09T10:49:57.045910+0000","last_became_peered":"2023-10-09T10:49:57.045910+0000","last_unstale":"2023-10-09T10:50:02.864880+0000","last_undegraded":"2023-10-09T10:49:56.942718+0000","last_fullsized":"2023-10-09T10:49:56.803232+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3501,"log_dups_size":0,"ondisk_log_size":3501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:46.585592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229441536,"num_objects":3501,"num_object_clones":0,"num_object_copies":10503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5515,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3501,"num_write_kb":224064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,6","objects":793},{"shards":"0,1,7","objects":2361},{"shards":"0,6","objects":347}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":555,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.793084+0000","last_change":"2023-10-09T10:49:58.793084+0000","last_active":"2023-10-09T10:49:58.793084+0000","last_peered":"2023-10-09T10:49:58.793084+0000","last_clean":"2023-10-09T10:49:58.793084+0000","last_became_active":"2023-10-09T10:49:58.792524+0000","last_became_peered":"2023-10-09T10:49:58.792524+0000","last_unstale":"2023-10-09T10:49:58.793084+0000","last_undegraded":"2023-10-09T10:49:58.793084+0000","last_fullsized":"2023-10-09T10:49:58.793084+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":211,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.760837+0000","last_change":"2023-10-09T10:49:55.837610+0000","last_active":"2023-10-09T10:49:58.760837+0000","last_peered":"2023-10-09T10:49:58.760837+0000","last_clean":"2023-10-09T10:49:58.760837+0000","last_became_active":"2023-10-09T10:49:55.837314+0000","last_became_peered":"2023-10-09T10:49:55.837314+0000","last_unstale":"2023-10-09T10:49:58.760837+0000","last_undegraded":"2023-10-09T10:49:58.760837+0000","last_fullsized":"2023-10-09T10:49:58.760837+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:01.185522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":91,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.761162+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:49:58.761162+0000","last_peered":"2023-10-09T10:49:58.761162+0000","last_clean":"2023-10-09T10:49:58.761162+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:49:58.761162+0000","last_undegraded":"2023-10-09T10:49:58.761162+0000","last_fullsized":"2023-10-09T10:49:58.761162+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":108,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.754286+0000","last_change":"2023-10-09T10:49:55.838212+0000","last_active":"2023-10-09T10:49:57.754286+0000","last_peered":"2023-10-09T10:49:57.754286+0000","last_clean":"2023-10-09T10:49:57.754286+0000","last_became_active":"2023-10-09T10:49:55.837374+0000","last_became_peered":"2023-10-09T10:49:55.837374+0000","last_unstale":"2023-10-09T10:49:57.754286+0000","last_undegraded":"2023-10-09T10:49:57.754286+0000","last_fullsized":"2023-10-09T10:49:57.754286+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:54.825127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"212'3420","reported_seq":3457,"reported_epoch":212,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:01.193994+0000","last_change":"2023-10-09T10:49:56.979655+0000","last_active":"2023-10-09T10:50:01.193994+0000","last_peered":"2023-10-09T10:50:01.193994+0000","last_clean":"2023-10-09T10:49:54.774195+0000","last_became_active":"2023-10-09T10:49:56.969698+0000","last_became_peered":"2023-10-09T10:49:56.969698+0000","last_unstale":"2023-10-09T10:50:01.193994+0000","last_undegraded":"2023-10-09T10:49:56.911686+0000","last_fullsized":"2023-10-09T10:49:56.808103+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3420,"log_dups_size":0,"ondisk_log_size":3420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:47.019837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224133120,"num_objects":3420,"num_object_clones":0,"num_object_copies":10260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3175,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3420,"num_write_kb":218880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":3175},{"shards":"3,5","objects":245}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":576,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.752990+0000","last_change":"2023-10-09T10:49:38.475553+0000","last_active":"2023-10-09T10:49:57.752990+0000","last_peered":"2023-10-09T10:49:57.752990+0000","last_clean":"2023-10-09T10:49:57.752990+0000","last_became_active":"2023-10-09T10:49:17.844634+0000","last_became_peered":"2023-10-09T10:49:17.844634+0000","last_unstale":"2023-10-09T10:49:57.752990+0000","last_undegraded":"2023-10-09T10:49:57.752990+0000","last_fullsized":"2023-10-09T10:49:57.752990+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.475429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.475429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:25:52.361095+0000","scrub_duration":0.001591812,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":152,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.746378+0000","last_change":"2023-10-09T10:49:55.820384+0000","last_active":"2023-10-09T10:49:57.746378+0000","last_peered":"2023-10-09T10:49:57.746378+0000","last_clean":"2023-10-09T10:49:57.746378+0000","last_became_active":"2023-10-09T10:49:55.820044+0000","last_became_peered":"2023-10-09T10:49:55.820044+0000","last_unstale":"2023-10-09T10:49:57.746378+0000","last_undegraded":"2023-10-09T10:49:57.746378+0000","last_fullsized":"2023-10-09T10:49:57.746378+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:58.595200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":121,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.754079+0000","last_change":"2023-10-09T10:49:55.824348+0000","last_active":"2023-10-09T10:49:57.754079+0000","last_peered":"2023-10-09T10:49:57.754079+0000","last_clean":"2023-10-09T10:49:57.754079+0000","last_became_active":"2023-10-09T10:49:55.823990+0000","last_became_peered":"2023-10-09T10:49:55.823990+0000","last_unstale":"2023-10-09T10:49:57.754079+0000","last_undegraded":"2023-10-09T10:49:57.754079+0000","last_fullsized":"2023-10-09T10:49:57.754079+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:33.050849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":100,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.762917+0000","last_change":"2023-10-09T10:49:55.837070+0000","last_active":"2023-10-09T10:49:58.762917+0000","last_peered":"2023-10-09T10:49:58.762917+0000","last_clean":"2023-10-09T10:49:58.762917+0000","last_became_active":"2023-10-09T10:49:55.836747+0000","last_became_peered":"2023-10-09T10:49:55.836747+0000","last_unstale":"2023-10-09T10:49:58.762917+0000","last_undegraded":"2023-10-09T10:49:58.762917+0000","last_fullsized":"2023-10-09T10:49:58.762917+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:45.683756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"213'3578","reported_seq":3604,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T10:50:04.645620+0000","last_change":"2023-10-09T10:49:16.845103+0000","last_active":"2023-10-09T10:50:04.645620+0000","last_peered":"2023-10-09T10:50:04.645620+0000","last_clean":"2023-10-09T10:50:04.645620+0000","last_became_active":"2023-10-09T10:49:16.844812+0000","last_became_peered":"2023-10-09T10:49:16.844812+0000","last_unstale":"2023-10-09T10:50:04.645620+0000","last_undegraded":"2023-10-09T10:50:04.645620+0000","last_fullsized":"2023-10-09T10:50:04.645620+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3578,"log_dups_size":0,"ondisk_log_size":3578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:17:27.590153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234487808,"num_objects":3578,"num_object_clones":0,"num_object_copies":10734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3578,"num_write_kb":228992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":567,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.762405+0000","last_change":"2023-10-09T10:49:18.829156+0000","last_active":"2023-10-09T10:49:57.762405+0000","last_peered":"2023-10-09T10:49:57.762405+0000","last_clean":"2023-10-09T10:49:57.762405+0000","last_became_active":"2023-10-09T10:49:17.848031+0000","last_became_peered":"2023-10-09T10:49:17.848031+0000","last_unstale":"2023-10-09T10:49:57.762405+0000","last_undegraded":"2023-10-09T10:49:57.762405+0000","last_fullsized":"2023-10-09T10:49:57.762405+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.829081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.829081+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:25.174844+0000","scrub_duration":0.001277842,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":220,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T10:50:02.897522+0000","last_change":"2023-10-09T10:49:40.959630+0000","last_active":"2023-10-09T10:50:02.897522+0000","last_peered":"2023-10-09T10:50:02.897522+0000","last_clean":"2023-10-09T10:50:02.897522+0000","last_became_active":"2023-10-09T10:49:40.945271+0000","last_became_peered":"2023-10-09T10:49:40.945271+0000","last_unstale":"2023-10-09T10:50:02.897522+0000","last_undegraded":"2023-10-09T10:50:02.897522+0000","last_fullsized":"2023-10-09T10:50:02.897522+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:37.426204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":29,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.762999+0000","last_change":"2023-10-09T10:49:55.813949+0000","last_active":"2023-10-09T10:49:57.762999+0000","last_peered":"2023-10-09T10:49:57.762999+0000","last_clean":"2023-10-09T10:49:57.762999+0000","last_became_active":"2023-10-09T10:49:55.813656+0000","last_became_peered":"2023-10-09T10:49:55.813656+0000","last_unstale":"2023-10-09T10:49:57.762999+0000","last_undegraded":"2023-10-09T10:49:57.762999+0000","last_fullsized":"2023-10-09T10:49:57.762999+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:55.530174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":214,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.773316+0000","last_change":"2023-10-09T10:49:55.820039+0000","last_active":"2023-10-09T10:49:58.773316+0000","last_peered":"2023-10-09T10:49:58.773316+0000","last_clean":"2023-10-09T10:49:58.773316+0000","last_became_active":"2023-10-09T10:49:55.819236+0000","last_became_peered":"2023-10-09T10:49:55.819236+0000","last_unstale":"2023-10-09T10:49:58.773316+0000","last_undegraded":"2023-10-09T10:49:58.773316+0000","last_fullsized":"2023-10-09T10:49:58.773316+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:47:19.590311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"213'3616","reported_seq":3657,"reported_epoch":213,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:04.624291+0000","last_change":"2023-10-09T10:49:57.045487+0000","last_active":"2023-10-09T10:50:04.624291+0000","last_peered":"2023-10-09T10:50:04.624291+0000","last_clean":"2023-10-09T10:49:39.930672+0000","last_became_active":"2023-10-09T10:49:57.038487+0000","last_became_peered":"2023-10-09T10:49:57.038487+0000","last_unstale":"2023-10-09T10:50:04.624291+0000","last_undegraded":"2023-10-09T10:49:56.923222+0000","last_fullsized":"2023-10-09T10:49:56.809367+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":3616,"log_dups_size":0,"ondisk_log_size":3616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:29:54.702472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236978176,"num_objects":3616,"num_object_clones":0,"num_object_copies":10848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5690,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3616,"num_write_kb":231424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":1},{"shards":"1,3,6","objects":752},{"shards":"1,3,7","objects":2468},{"shards":"3,6","objects":395}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":572,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.774793+0000","last_change":"2023-10-09T10:49:43.787701+0000","last_active":"2023-10-09T10:49:58.774793+0000","last_peered":"2023-10-09T10:49:58.774793+0000","last_clean":"2023-10-09T10:49:58.774793+0000","last_became_active":"2023-10-09T10:49:43.787366+0000","last_became_peered":"2023-10-09T10:49:43.787366+0000","last_unstale":"2023-10-09T10:49:58.774793+0000","last_undegraded":"2023-10-09T10:49:58.774793+0000","last_fullsized":"2023-10-09T10:49:58.774793+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":89,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.778244+0000","last_change":"2023-10-09T10:49:55.811456+0000","last_active":"2023-10-09T10:49:58.778244+0000","last_peered":"2023-10-09T10:49:58.778244+0000","last_clean":"2023-10-09T10:49:58.778244+0000","last_became_active":"2023-10-09T10:49:55.811134+0000","last_became_peered":"2023-10-09T10:49:55.811134+0000","last_unstale":"2023-10-09T10:49:58.778244+0000","last_undegraded":"2023-10-09T10:49:58.778244+0000","last_fullsized":"2023-10-09T10:49:58.778244+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:28.708781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":98,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.752748+0000","last_change":"2023-10-09T10:49:55.850829+0000","last_active":"2023-10-09T10:49:57.752748+0000","last_peered":"2023-10-09T10:49:57.752748+0000","last_clean":"2023-10-09T10:49:57.752748+0000","last_became_active":"2023-10-09T10:49:55.837771+0000","last_became_peered":"2023-10-09T10:49:55.837771+0000","last_unstale":"2023-10-09T10:49:57.752748+0000","last_undegraded":"2023-10-09T10:49:57.752748+0000","last_fullsized":"2023-10-09T10:49:57.752748+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:07.101559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":159,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.761648+0000","last_change":"2023-10-09T10:35:58.711478+0000","last_active":"2023-10-09T10:49:57.761648+0000","last_peered":"2023-10-09T10:49:57.761648+0000","last_clean":"2023-10-09T10:49:57.761648+0000","last_became_active":"2023-10-09T10:35:58.711183+0000","last_became_peered":"2023-10-09T10:35:58.711183+0000","last_unstale":"2023-10-09T10:49:57.761648+0000","last_undegraded":"2023-10-09T10:49:57.761648+0000","last_fullsized":"2023-10-09T10:49:57.761648+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:04.407457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":100,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.763037+0000","last_change":"2023-10-09T10:49:55.830269+0000","last_active":"2023-10-09T10:49:58.763037+0000","last_peered":"2023-10-09T10:49:58.763037+0000","last_clean":"2023-10-09T10:49:58.763037+0000","last_became_active":"2023-10-09T10:49:55.829637+0000","last_became_peered":"2023-10-09T10:49:55.829637+0000","last_unstale":"2023-10-09T10:49:58.763037+0000","last_undegraded":"2023-10-09T10:49:58.763037+0000","last_fullsized":"2023-10-09T10:49:58.763037+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:39.579608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:59.699104+0000","last_change":"2023-10-09T10:49:55.817112+0000","last_active":"2023-10-09T10:49:59.699104+0000","last_peered":"2023-10-09T10:49:59.699104+0000","last_clean":"2023-10-09T10:49:59.699104+0000","last_became_active":"2023-10-09T10:49:55.815889+0000","last_became_peered":"2023-10-09T10:49:55.815889+0000","last_unstale":"2023-10-09T10:49:59.699104+0000","last_undegraded":"2023-10-09T10:49:59.699104+0000","last_fullsized":"2023-10-09T10:49:59.699104+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:41.374680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":193,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.753222+0000","last_change":"2023-10-09T10:49:55.831015+0000","last_active":"2023-10-09T10:49:57.753222+0000","last_peered":"2023-10-09T10:49:57.753222+0000","last_clean":"2023-10-09T10:49:57.753222+0000","last_became_active":"2023-10-09T10:49:55.816720+0000","last_became_peered":"2023-10-09T10:49:55.816720+0000","last_unstale":"2023-10-09T10:49:57.753222+0000","last_undegraded":"2023-10-09T10:49:57.753222+0000","last_fullsized":"2023-10-09T10:49:57.753222+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:05:36.526668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":203,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.762103+0000","last_change":"2023-10-09T10:49:55.820737+0000","last_active":"2023-10-09T10:49:57.762103+0000","last_peered":"2023-10-09T10:49:57.762103+0000","last_clean":"2023-10-09T10:49:57.762103+0000","last_became_active":"2023-10-09T10:49:55.820390+0000","last_became_peered":"2023-10-09T10:49:55.820390+0000","last_unstale":"2023-10-09T10:49:57.762103+0000","last_undegraded":"2023-10-09T10:49:57.762103+0000","last_fullsized":"2023-10-09T10:49:57.762103+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:02:34.191035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":162,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.753679+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:49:57.753679+0000","last_peered":"2023-10-09T10:49:57.753679+0000","last_clean":"2023-10-09T10:49:57.753679+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:49:57.753679+0000","last_undegraded":"2023-10-09T10:49:57.753679+0000","last_fullsized":"2023-10-09T10:49:57.753679+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":60,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.760963+0000","last_change":"2023-10-09T10:46:57.193190+0000","last_active":"2023-10-09T10:49:57.760963+0000","last_peered":"2023-10-09T10:49:57.760963+0000","last_clean":"2023-10-09T10:49:57.760963+0000","last_became_active":"2023-10-09T10:46:57.192855+0000","last_became_peered":"2023-10-09T10:46:57.192855+0000","last_unstale":"2023-10-09T10:49:57.760963+0000","last_undegraded":"2023-10-09T10:49:57.760963+0000","last_fullsized":"2023-10-09T10:49:57.760963+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:55.201073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":207,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.776903+0000","last_change":"2023-10-09T10:49:55.821623+0000","last_active":"2023-10-09T10:49:58.776903+0000","last_peered":"2023-10-09T10:49:58.776903+0000","last_clean":"2023-10-09T10:49:58.776903+0000","last_became_active":"2023-10-09T10:49:55.821367+0000","last_became_peered":"2023-10-09T10:49:55.821367+0000","last_unstale":"2023-10-09T10:49:58.776903+0000","last_undegraded":"2023-10-09T10:49:58.776903+0000","last_fullsized":"2023-10-09T10:49:58.776903+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:09.753526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":228,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T10:49:58.773389+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:49:58.773389+0000","last_peered":"2023-10-09T10:49:58.773389+0000","last_clean":"2023-10-09T10:49:58.773389+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:49:58.773389+0000","last_undegraded":"2023-10-09T10:49:58.773389+0000","last_fullsized":"2023-10-09T10:49:58.773389+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":102,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.748595+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:49:57.748595+0000","last_peered":"2023-10-09T10:49:57.748595+0000","last_clean":"2023-10-09T10:49:57.748595+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:49:57.748595+0000","last_undegraded":"2023-10-09T10:49:57.748595+0000","last_fullsized":"2023-10-09T10:49:57.748595+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":186,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T10:49:57.761864+0000","last_change":"2023-10-09T10:49:55.814865+0000","last_active":"2023-10-09T10:49:57.761864+0000","last_peered":"2023-10-09T10:49:57.761864+0000","last_clean":"2023-10-09T10:49:57.761864+0000","last_became_active":"2023-10-09T10:49:55.814453+0000","last_became_peered":"2023-10-09T10:49:55.814453+0000","last_unstale":"2023-10-09T10:49:57.761864+0000","last_undegraded":"2023-10-09T10:49:57.761864+0000","last_fullsized":"2023-10-09T10:49:57.761864+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:05:10.291518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":3648520192,"num_objects":55672,"num_object_clones":0,"num_object_copies":167016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33867,"num_objects_misplaced":2805,"num_objects_unfound":0,"num_objects_dirty":55672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":55672,"num_write_kb":3563008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2952,"num_bytes_recovered":193462272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11065294848,"data_stored":11065294848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55672,"ondisk_log_size":55672,"up":48,"acting":36,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":6080,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":685479,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":294,"num_read_kb":554,"num_write":144,"num_write_kb":1463,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1336085,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":219043332281,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":44,"seq":188978561212,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1740600,"kb_used_data":1439216,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92004552,"statfs":{"total":95995035648,"available":94212661248,"internally_reserved":0,"allocated":1473757184,"data_stored":1472978117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:49:47 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":1.6339999999999999,"15min":0.94099999999999995},"min":{"1min":0.27000000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.288,"5min":12.288,"15min":12.288},"last":6.7469999999999999},{"interface":"front","average":{"1min":1.4630000000000001,"5min":1.5509999999999999,"15min":0.91900000000000004},"min":{"1min":0.25,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":12.369999999999999,"5min":12.369999999999999,"15min":12.369999999999999},"last":6.6040000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:49:54 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.7749999999999999,"15min":0.98399999999999999},"min":{"1min":0.26500000000000001,"5min":0.245,"15min":0.245},"max":{"1min":3.415,"5min":16.747,"15min":16.747},"last":7.4210000000000003},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.873,"15min":0.99099999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.4260000000000002,"5min":16.734000000000002,"15min":16.734000000000002},"last":0.42399999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:49:22 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":1.601,"15min":0.88900000000000001},"min":{"1min":0.29899999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.7599999999999998,"5min":13.208,"15min":13.208},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.7769999999999999,"5min":1.8620000000000001,"15min":0.99399999999999999},"min":{"1min":0.35199999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.186,"5min":12.571,"15min":12.571},"last":4.0170000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:49:47 2023","interfaces":[{"interface":"back","average":{"1min":1.478,"5min":1.6839999999999999,"15min":0.92000000000000004},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.255},"max":{"1min":12.391,"5min":12.391,"15min":12.391},"last":6.6349999999999998},{"interface":"front","average":{"1min":1.5089999999999999,"5min":1.919,"15min":1.004},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.9429999999999996,"5min":13.179,"15min":13.179},"last":0.51900000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:49:14 2023","interfaces":[{"interface":"back","average":{"1min":1.641,"5min":1.2969999999999999,"15min":0.73099999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":6.742,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":5.4569999999999999},{"interface":"front","average":{"1min":2.8639999999999999,"5min":1.363,"15min":0.78000000000000003},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":12.481999999999999,"5min":12.481999999999999,"15min":12.481999999999999},"last":6.6970000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:49:14 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":1.417,"15min":0.79800000000000004},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.343,"5min":10.497,"15min":10.497},"last":3.9460000000000002},{"interface":"front","average":{"1min":2.6709999999999998,"5min":1.381,"15min":0.81399999999999995},"min":{"1min":0.32100000000000001,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":12.456,"5min":12.456,"15min":12.456},"last":6.6710000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:49:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2559999999999998,"5min":1.702,"15min":0.92100000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":12.007,"5min":23.358000000000001,"15min":23.358000000000001},"last":3.9820000000000002},{"interface":"front","average":{"1min":1.95,"5min":1.514,"15min":0.88800000000000001},"min":{"1min":0.38700000000000001,"5min":0.35099999999999998,"15min":0.34000000000000002},"max":{"1min":12.422000000000001,"5min":12.422000000000001,"15min":12.422000000000001},"last":6.7210000000000001}]}]},{"osd":1,"up_from":0,"seq":365072220251,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":80,"seq":343597383785,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2141176,"kb_used_data":1956976,"kb_used_omap":28,"kb_used_meta":184163,"kb_avail":91603976,"statfs":{"total":95995035648,"available":93802471424,"internally_reserved":0,"allocated":2003943424,"data_stored":2002816906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":188583760},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,2,5,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:49:50 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":2.512,"15min":1.087},"min":{"1min":0.25,"5min":0.191,"15min":0.191},"max":{"1min":14.311,"5min":25.475000000000001,"15min":25.475000000000001},"last":4.431},{"interface":"front","average":{"1min":1.2749999999999999,"5min":2.347,"15min":1.0569999999999999},"min":{"1min":0.24099999999999999,"5min":0.188,"15min":0.188},"max":{"1min":6.8109999999999999,"5min":20.832999999999998,"15min":20.832999999999998},"last":4.718}]},{"osd":2,"last update":"Mon Oct 9 10:49:22 2023","interfaces":[{"interface":"back","average":{"1min":2.766,"5min":1.9590000000000001,"15min":0.88500000000000001},"min":{"1min":0.25600000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":11.837,"5min":11.837,"15min":11.837},"last":5.0430000000000001},{"interface":"front","average":{"1min":2.133,"5min":2.2469999999999999,"15min":0.97699999999999998},"min":{"1min":0.25700000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.1319999999999997,"5min":14.391999999999999,"15min":14.391999999999999},"last":2.9830000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:49:50 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":2.5459999999999998,"15min":1.282},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":6.782,"5min":14.926,"15min":14.926},"last":1.968},{"interface":"front","average":{"1min":2.1459999999999999,"5min":2.5169999999999999,"15min":1.2},"min":{"1min":0.23499999999999999,"5min":0.224,"15min":0.224},"max":{"1min":14.391999999999999,"5min":14.391999999999999,"15min":14.391999999999999},"last":5.0839999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:49:37 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":1.9259999999999999,"15min":1.0489999999999999},"min":{"1min":0.318,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.1720000000000006,"5min":9.1720000000000006,"15min":9.1720000000000006},"last":3.024},{"interface":"front","average":{"1min":2.145,"5min":2.2509999999999999,"15min":1.159},"min":{"1min":0.33300000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.4870000000000001,"5min":12.318,"15min":12.318},"last":1.9430000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:49:37 2023","interfaces":[{"interface":"back","average":{"1min":2.2160000000000002,"5min":2.1699999999999999,"15min":1.135},"min":{"1min":0.498,"5min":0.318,"15min":0.318},"max":{"1min":8.4260000000000002,"5min":12.157,"15min":12.157},"last":4.6820000000000004},{"interface":"front","average":{"1min":1.869,"5min":1.9339999999999999,"15min":1.071},"min":{"1min":0.34200000000000003,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.5199999999999996,"5min":12.781000000000001,"15min":12.781000000000001},"last":4.4889999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:49:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4089999999999998,"5min":2.327,"15min":1.1910000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.5839999999999996,"5min":12.207000000000001,"15min":12.207000000000001},"last":1.9079999999999999},{"interface":"front","average":{"1min":1.7210000000000001,"5min":1.601,"15min":0.96299999999999997},"min":{"1min":0.35799999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.5660000000000007,"5min":9.8889999999999993,"15min":9.8889999999999993},"last":4.9610000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:49:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1669999999999998,"5min":2.069,"15min":1.1180000000000001},"min":{"1min":0.42499999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":8.5449999999999999,"5min":12.727,"15min":12.727},"last":4.5339999999999998},{"interface":"front","average":{"1min":2.0249999999999999,"5min":2.153,"15min":1.161},"min":{"1min":0.36599999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.2050000000000001,"5min":14.394,"15min":14.394},"last":2.9529999999999998}]}]},{"osd":2,"up_from":0,"seq":386547056726,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893200,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2918840,"kb_used_data":2548976,"kb_used_omap":7,"kb_used_meta":369848,"kb_avail":90826312,"statfs":{"total":95995035648,"available":93006143488,"internally_reserved":0,"allocated":2610151424,"data_stored":2609028735,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":378724968},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,1,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:49:46 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":2.3730000000000002,"15min":1.0049999999999999},"min":{"1min":0.20899999999999999,"5min":0.183,"15min":0.183},"max":{"1min":8.4339999999999993,"5min":15.255000000000001,"15min":15.255000000000001},"last":0.219},{"interface":"front","average":{"1min":3.1120000000000001,"5min":2.1949999999999998,"15min":1.0029999999999999},"min":{"1min":0.23200000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":14.798,"5min":14.798,"15min":14.798},"last":4.6230000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:49:46 2023","interfaces":[{"interface":"back","average":{"1min":2.661,"5min":2.8260000000000001,"15min":1.1619999999999999},"min":{"1min":0.21099999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":11.757,"5min":14.707000000000001,"15min":14.707000000000001},"last":1.6719999999999999},{"interface":"front","average":{"1min":3.056,"5min":2.3460000000000001,"15min":1.0309999999999999},"min":{"1min":0.30199999999999999,"5min":0.193,"15min":0.193},"max":{"1min":10.59,"5min":15.443,"15min":15.443},"last":1.6579999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:49:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7280000000000002,"5min":2.5350000000000001,"15min":1.089},"min":{"1min":0.24099999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.795,"5min":13.853,"15min":13.853},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.964,"5min":2.5449999999999999,"15min":1.1120000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.8569999999999993,"5min":10.557,"15min":10.557},"last":2.4460000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:49:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4340000000000002,"5min":2.294,"15min":1.1459999999999999},"min":{"1min":0.34799999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.976,"5min":20.388000000000002,"15min":20.388000000000002},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.972,"5min":2.3340000000000001,"15min":1.1830000000000001},"min":{"1min":0.32100000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.9560000000000004,"5min":19.085999999999999,"15min":19.085999999999999},"last":0.59199999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:49:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2629999999999999,"5min":2.7730000000000001,"15min":1.3149999999999999},"min":{"1min":0.35099999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.619999999999999,"5min":51.509999999999998,"15min":51.509999999999998},"last":4.6559999999999997},{"interface":"front","average":{"1min":2.347,"5min":3.0750000000000002,"15min":1.3999999999999999},"min":{"1min":0.45000000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":10.798999999999999,"5min":55.338999999999999,"15min":55.338999999999999},"last":3.8620000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:49:46 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":2.0979999999999999,"15min":1.091},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":8.5960000000000001,"5min":12.361000000000001,"15min":12.361000000000001},"last":2.492},{"interface":"front","average":{"1min":2.2229999999999999,"5min":2.544,"15min":1.248},"min":{"1min":0.45000000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.569,"5min":13.891999999999999,"15min":13.891999999999999},"last":2.5030000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:49:46 2023","interfaces":[{"interface":"back","average":{"1min":2.6040000000000001,"5min":3.27,"15min":1.484},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":11.819000000000001,"5min":51.744,"15min":51.744},"last":1.27},{"interface":"front","average":{"1min":2.3490000000000002,"5min":2.895,"15min":1.347},"min":{"1min":0.39200000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.1900000000000004,"5min":53.399000000000001,"15min":53.399000000000001},"last":4.2039999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019075,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1485176,"kb_used_data":1183792,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92259976,"statfs":{"total":95995035648,"available":94474215424,"internally_reserved":0,"allocated":1212203008,"data_stored":1211423208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:49:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.403,"15min":0.83399999999999996},"min":{"1min":0.25700000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.056,"5min":10.387,"15min":10.387},"last":0.78800000000000003},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.762,"15min":0.97199999999999998},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":4.1970000000000001,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.89200000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:49:44 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.5329999999999999,"15min":0.89600000000000002},"min":{"1min":0.26400000000000001,"5min":0.249,"15min":0.249},"max":{"1min":3.7919999999999998,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.81599999999999995},{"interface":"front","average":{"1min":1.2210000000000001,"5min":1.79,"15min":1.0209999999999999},"min":{"1min":0.33500000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.6269999999999998,"5min":8.7669999999999995,"15min":8.7669999999999995},"last":0.33700000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:49:16 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.7749999999999999,"15min":0.97199999999999998},"min":{"1min":0.27500000000000002,"5min":0.253,"15min":0.253},"max":{"1min":4.2619999999999996,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":0.96499999999999997},{"interface":"front","average":{"1min":2.1800000000000002,"5min":2.1779999999999999,"15min":1.101},"min":{"1min":0.34799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.9870000000000001,"5min":20.545999999999999,"15min":20.545999999999999},"last":0.93899999999999995}]},{"osd":3,"last update":"Mon Oct 9 10:49:41 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":1.8580000000000001,"15min":1.0940000000000001},"min":{"1min":0.32000000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.718,"5min":9.0269999999999992,"15min":9.0269999999999992},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.9179999999999999,"5min":2.3039999999999998,"15min":1.1599999999999999},"min":{"1min":0.34200000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":9.6929999999999996,"5min":9.6929999999999996,"15min":9.6929999999999996},"last":0.91600000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:49:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.919,"15min":0.94999999999999996},"min":{"1min":0.27900000000000003,"5min":0.224,"15min":0.224},"max":{"1min":9.0020000000000007,"5min":24.149999999999999,"15min":24.149999999999999},"last":1.5469999999999999},{"interface":"front","average":{"1min":1.579,"5min":1.8720000000000001,"15min":0.94199999999999995},"min":{"1min":0.29199999999999998,"5min":0.252,"15min":0.252},"max":{"1min":15.071999999999999,"5min":20.422000000000001,"15min":20.422000000000001},"last":1.198}]},{"osd":6,"last update":"Mon Oct 9 10:49:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5819999999999999,"5min":1.544,"15min":0.83699999999999997},"min":{"1min":0.26000000000000001,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":14.897,"5min":14.897,"15min":14.897},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.9590000000000001,"5min":1.484,"15min":0.876},"min":{"1min":0.32100000000000001,"5min":0.311,"15min":0.311},"max":{"1min":10.329000000000001,"5min":12.539,"15min":12.539},"last":0.377}]},{"osd":7,"last update":"Mon Oct 9 10:49:21 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.452,"15min":0.86399999999999999},"min":{"1min":0.40100000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.3360000000000003,"5min":9.3360000000000003,"15min":9.3360000000000003},"last":1.627},{"interface":"front","average":{"1min":1.228,"5min":1.387,"15min":0.83699999999999997},"min":{"1min":0.314,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.6520000000000001,"5min":7.423,"15min":7.423},"last":5.5129999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790144,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1388780,"kb_used_data":1083300,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92356372,"statfs":{"total":95995035648,"available":94572924928,"internally_reserved":0,"allocated":1109299200,"data_stored":1108527675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:49:47 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.2849999999999999,"15min":0.82499999999999996},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.5230000000000001,"5min":11.648999999999999,"15min":11.648999999999999},"last":1.1200000000000001},{"interface":"front","average":{"1min":0.94099999999999995,"5min":1.3540000000000001,"15min":0.89300000000000002},"min":{"1min":0.30399999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.5459999999999998,"5min":11.68,"15min":11.68},"last":1.1519999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:49:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.8819999999999999,"15min":1.0289999999999999},"min":{"1min":0.26500000000000001,"5min":0.251,"15min":0.251},"max":{"1min":6.609,"5min":12.999000000000001,"15min":12.999000000000001},"last":1.629},{"interface":"front","average":{"1min":0.73499999999999999,"5min":1.512,"15min":0.91500000000000004},"min":{"1min":0.254,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":2.391,"5min":7.3730000000000002,"15min":7.3730000000000002},"last":0.28499999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:49:17 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":1.7110000000000001,"15min":0.95099999999999996},"min":{"1min":0.32500000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":11.393000000000001,"5min":11.393000000000001,"15min":11.393000000000001},"last":1.7050000000000001},{"interface":"front","average":{"1min":1.4770000000000001,"5min":1.415,"15min":0.85399999999999998},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.0380000000000003,"5min":10.352,"15min":10.352},"last":0.32200000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:49:51 2023","interfaces":[{"interface":"back","average":{"1min":1.256,"5min":2.1539999999999999,"15min":1.2},"min":{"1min":0.36099999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.7000000000000002,"5min":18.016999999999999,"15min":18.016999999999999},"last":0.309},{"interface":"front","average":{"1min":1.714,"5min":2.3090000000000002,"15min":1.1779999999999999},"min":{"1min":0.34000000000000002,"5min":0.317,"15min":0.317},"max":{"1min":9.2490000000000006,"5min":15.555999999999999,"15min":15.555999999999999},"last":0.36299999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:49:54 2023","interfaces":[{"interface":"back","average":{"1min":6.6059999999999999,"5min":2.29,"15min":1.075},"min":{"1min":0.24099999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":123.241,"5min":123.241,"15min":123.241},"last":0.35199999999999998},{"interface":"front","average":{"1min":7.2119999999999997,"5min":2.7029999999999998,"15min":1.232},"min":{"1min":0.26700000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":123.318,"5min":123.318,"15min":123.318},"last":0.39600000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:49:11 2023","interfaces":[{"interface":"back","average":{"1min":2.04,"5min":1.4359999999999999,"15min":0.81000000000000005},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":6.9100000000000001,"5min":7.7060000000000004,"15min":7.7060000000000004},"last":5.0129999999999999},{"interface":"front","average":{"1min":2.609,"5min":1.3109999999999999,"15min":0.78500000000000003},"min":{"1min":0.34799999999999998,"5min":0.318,"15min":0.30299999999999999},"max":{"1min":20.256,"5min":20.256,"15min":20.256},"last":0.57199999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:49:27 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.329,"15min":0.81699999999999995},"min":{"1min":0.315,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.5919999999999996,"5min":7.5919999999999996,"15min":7.5919999999999996},"last":1.597},{"interface":"front","average":{"1min":1.9670000000000001,"5min":1.3029999999999999,"15min":0.81899999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.31900000000000001},"max":{"1min":14.154,"5min":14.154,"15min":14.154},"last":0.38100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2011955200,"data_stored":2011955200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1658322944,"data_stored":1658322944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2607742976,"data_stored":2607742976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1223360512,"data_stored":1223360512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1120403456,"data_stored":1120403456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1505558528,"data_stored":1505558528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":937951232,"data_stored":937951232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:50:06.281 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_1 pgp_num 16 2023-10-09T10:50:06.282 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:06.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:06.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:06.294 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:06 smithi155 ceph-mon[184854]: pgmap v480: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 3.7 KiB/s rd, 50 MiB/s wr, 806 op/s; 33867/167085 objects degraded (20.269%); 2805/167085 objects misplaced (1.679%); 14 MiB/s, 225 objects/s recovering 2023-10-09T10:50:06.294 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:06 smithi155 ceph-mon[184854]: from='client.45098 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:50:06.294 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:06 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:06.394 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:06.442 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:06.474 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:06.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:06.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:06.530 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:06.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:06.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:06 smithi125 ceph-mon[195507]: pgmap v480: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 3.7 KiB/s rd, 50 MiB/s wr, 806 op/s; 33867/167085 objects degraded (20.269%); 2805/167085 objects misplaced (1.679%); 14 MiB/s, 225 objects/s recovering 2023-10-09T10:50:06.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:06 smithi125 ceph-mon[195507]: from='client.45098 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:50:06.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:06 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:06.580 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:06 smithi125 ceph-mon[200197]: pgmap v480: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 3.7 KiB/s rd, 50 MiB/s wr, 806 op/s; 33867/167085 objects degraded (20.269%); 2805/167085 objects misplaced (1.679%); 14 MiB/s, 225 objects/s recovering 2023-10-09T10:50:06.580 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:06 smithi125 ceph-mon[200197]: from='client.45098 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:50:06.580 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:06 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:06.584 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:06.623 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:06.648 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:06.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:06.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:06.755 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:06.805 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:06.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:06.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:06.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:06.954 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:06.989 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 46 16 59090 59074 80.2411 48.5625 0.0220041 0.0124559 2023-10-09T10:50:07.010 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:07.051 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:07.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:07.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:07.064 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:50:07.064 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:50:07.071 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:50:07.106 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:50:07.136 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:50:07.155 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:07.177 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:50:07.206 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:50:07.236 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:07.262 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:50:07.269 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:50:07.270 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:50:07.270 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:50:07.270 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:50:07.270 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:50:07.270 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:50:07.270 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:50:07.270 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:50:07.475 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:07 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1018561435' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:07.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:07 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1018561435' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:07.668 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:07.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:07.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:07.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:07 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1018561435' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:07.773 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:07.820 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:07.990 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 47 16 59814 59798 79.4958 45.25 0.0285314 0.0125739 2023-10-09T10:50:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:08 smithi125 ceph-mon[195507]: pgmap v481: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.5 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 45 MiB/s wr, 725 op/s; 33867/171015 objects degraded (19.804%); 2478/171015 objects misplaced (1.449%); 12 MiB/s, 197 objects/s recovering 2023-10-09T10:50:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:08 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 33867/171015 objects degraded (19.804%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:08.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:08 smithi125 ceph-mon[200197]: pgmap v481: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.5 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 45 MiB/s wr, 725 op/s; 33867/171015 objects degraded (19.804%); 2478/171015 objects misplaced (1.449%); 12 MiB/s, 197 objects/s recovering 2023-10-09T10:50:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:08 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 33867/171015 objects degraded (19.804%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:08.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:08 smithi155 ceph-mon[184854]: pgmap v481: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.5 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 45 MiB/s wr, 725 op/s; 33867/171015 objects degraded (19.804%); 2478/171015 objects misplaced (1.449%); 12 MiB/s, 197 objects/s recovering 2023-10-09T10:50:08.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:08 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 33867/171015 objects degraded (19.804%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:08.991 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 48 16 60759 60743 79.07 59.0625 0.012089 0.0126412 2023-10-09T10:50:09.990 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 49 16 61695 61679 78.6502 58.5 0.0153132 0.0127095 2023-10-09T10:50:10.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:10 smithi125 ceph-mon[195507]: pgmap v482: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 49 MiB/s wr, 790 op/s; 33867/176304 objects degraded (19.209%); 2478/176304 objects misplaced (1.406%); 10 MiB/s, 167 objects/s recovering 2023-10-09T10:50:10.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:10 smithi125 ceph-mon[200197]: pgmap v482: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 49 MiB/s wr, 790 op/s; 33867/176304 objects degraded (19.209%); 2478/176304 objects misplaced (1.406%); 10 MiB/s, 167 objects/s recovering 2023-10-09T10:50:10.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:10 smithi155 ceph-mon[184854]: pgmap v482: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 49 MiB/s wr, 790 op/s; 33867/176304 objects degraded (19.209%); 2478/176304 objects misplaced (1.406%); 10 MiB/s, 167 objects/s recovering 2023-10-09T10:50:10.990 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 50 16 62602 62586 78.2109 56.6875 0.0199852 0.0127788 2023-10-09T10:50:11.151 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:11.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:11.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:11.171 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:11.259 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:11.278 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:50:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:50:11] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.43.0" 2023-10-09T10:50:11.314 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:11.368 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:11.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:11.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:11.475 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:11.537 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:11.583 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:11.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:11.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:11.693 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:50:11.693 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:50:11.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:11.747 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:11.781 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:11.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:11.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:11.886 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:11.950 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:11.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:11.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:11.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:11.992 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 51 16 63376 63360 77.6231 48.375 0.0195381 0.0128758 2023-10-09T10:50:12.085 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:12.141 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:12.173 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:12.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:12.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:12.279 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:12.323 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:12.345 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:50:12.348 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:50:12.469 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:12.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:12 smithi155 ceph-mon[184854]: pgmap v483: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 52 MiB/s wr, 834 op/s; 33867/183180 objects degraded (18.488%); 1674/183180 objects misplaced (0.914%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T10:50:12.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:12 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2700052311' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:12.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:12 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:12.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:12.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:12.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:12 smithi125 ceph-mon[195507]: pgmap v483: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 52 MiB/s wr, 834 op/s; 33867/183180 objects degraded (18.488%); 1674/183180 objects misplaced (0.914%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T10:50:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:12 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2700052311' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:12 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:12.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:12 smithi125 ceph-mon[200197]: pgmap v483: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 52 MiB/s wr, 834 op/s; 33867/183180 objects degraded (18.488%); 1674/183180 objects misplaced (0.914%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T10:50:12.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:12 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2700052311' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:12.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:12 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:12.575 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:12.630 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:12.670 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:12.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:12.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:12.775 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:12.812 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:12.887 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:12.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:12.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:12.992 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 52 16 64189 64173 77.1076 50.8125 0.0153537 0.0129633 2023-10-09T10:50:12.997 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:13.014 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:50:13.045 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:13.090 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:13.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:13.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:13.198 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:13.207 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:13.281 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:13.409 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:13.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:13.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:13.518 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:13.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:13.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[195507]: failed to encode map e214 with expected crc 2023-10-09T10:50:13.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:50:13.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[195507]: osdmap e214: 8 total, 8 up, 5 in 2023-10-09T10:50:13.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[195507]: failed to encode map e214 with expected crc 2023-10-09T10:50:13.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[195507]: failed to encode map e214 with expected crc 2023-10-09T10:50:13.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[195507]: failed to encode map e214 with expected crc 2023-10-09T10:50:13.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 33867/183180 objects degraded (18.488%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:13.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:13.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[200197]: failed to encode map e214 with expected crc 2023-10-09T10:50:13.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:50:13.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[200197]: osdmap e214: 8 total, 8 up, 5 in 2023-10-09T10:50:13.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[200197]: failed to encode map e214 with expected crc 2023-10-09T10:50:13.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[200197]: failed to encode map e214 with expected crc 2023-10-09T10:50:13.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[200197]: failed to encode map e214 with expected crc 2023-10-09T10:50:13.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:13 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 33867/183180 objects degraded (18.488%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:13.596 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:13.638 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:13.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:13.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:13.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:13 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:13.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:13 smithi155 ceph-mon[184854]: failed to encode map e214 with expected crc 2023-10-09T10:50:13.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:13 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:50:13.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:13 smithi155 ceph-mon[184854]: osdmap e214: 8 total, 8 up, 5 in 2023-10-09T10:50:13.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:13 smithi155 ceph-mon[184854]: failed to encode map e214 with expected crc 2023-10-09T10:50:13.647 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:13 smithi155 ceph-mon[184854]: failed to encode map e214 with expected crc 2023-10-09T10:50:13.647 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:13 smithi155 ceph-mon[184854]: failed to encode map e214 with expected crc 2023-10-09T10:50:13.647 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:13 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 33867/183180 objects degraded (18.488%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:13.746 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:13.779 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 7 pgp_num to 16 2023-10-09T10:50:13.781 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:13.806 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:13.841 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:13.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:13.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:13.948 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:13.986 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:13.992 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 53 16 64915 64899 76.5089 45.375 0.00813927 0.0130488 2023-10-09T10:50:14.165 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:14.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:14.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:14.271 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:14.306 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:14.330 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:14.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:14.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:14.434 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:14.450 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:14 smithi125 ceph-mon[195507]: pgmap v485: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 52 MiB/s wr, 831 op/s; 33867/188952 objects degraded (17.924%); 1674/188952 objects misplaced (0.886%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T10:50:14.450 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:14 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3334817321' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:50:14.450 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:14 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:50:14.450 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:14 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T10:50:14.450 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:14 smithi125 ceph-mon[195507]: osdmap e215: 8 total, 8 up, 5 in 2023-10-09T10:50:14.450 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:14 smithi125 ceph-mon[195507]: failed to encode map e215 with expected crc 2023-10-09T10:50:14.450 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:14 smithi125 ceph-mon[195507]: failed to encode map e215 with expected crc 2023-10-09T10:50:14.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:14 smithi125 ceph-mon[200197]: pgmap v485: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 52 MiB/s wr, 831 op/s; 33867/188952 objects degraded (17.924%); 1674/188952 objects misplaced (0.886%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T10:50:14.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:14 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3334817321' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:50:14.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:14 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:50:14.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:14 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T10:50:14.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:14 smithi125 ceph-mon[200197]: osdmap e215: 8 total, 8 up, 5 in 2023-10-09T10:50:14.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:14 smithi125 ceph-mon[200197]: failed to encode map e215 with expected crc 2023-10-09T10:50:14.451 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:14 smithi125 ceph-mon[200197]: failed to encode map e215 with expected crc 2023-10-09T10:50:14.463 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:14.582 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:14.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:14.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:14.670 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:50:14.670 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:50:14.686 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:14.714 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:14 smithi155 ceph-mon[184854]: pgmap v485: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 52 MiB/s wr, 831 op/s; 33867/188952 objects degraded (17.924%); 1674/188952 objects misplaced (0.886%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T10:50:14.714 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:14 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3334817321' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:50:14.714 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:14 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:50:14.714 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:14 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T10:50:14.714 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:14 smithi155 ceph-mon[184854]: osdmap e215: 8 total, 8 up, 5 in 2023-10-09T10:50:14.714 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:14 smithi155 ceph-mon[184854]: failed to encode map e215 with expected crc 2023-10-09T10:50:14.714 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:14 smithi155 ceph-mon[184854]: failed to encode map e215 with expected crc 2023-10-09T10:50:14.717 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:14.738 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:14.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:14.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:14.841 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:14.885 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:50:14.900 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:14.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:14.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:14.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:14.995 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 54 16 65278 65262 75.5091 22.6875 0.0187561 0.0132369 2023-10-09T10:50:15.025 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:15.091 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:15.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:15.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:15.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:15.249 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:15.298 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:50:15.319 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:15.506 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:15.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:15.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:15 smithi125 ceph-mon[195507]: failed to encode map e215 with expected crc 2023-10-09T10:50:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:15 smithi125 ceph-mon[195507]: osdmap e216: 8 total, 8 up, 5 in 2023-10-09T10:50:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:15 smithi125 ceph-mon[195507]: failed to encode map e216 with expected crc 2023-10-09T10:50:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:15 smithi125 ceph-mon[195507]: failed to encode map e216 with expected crc 2023-10-09T10:50:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:50:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:15 smithi125 ceph-mon[200197]: failed to encode map e215 with expected crc 2023-10-09T10:50:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:15 smithi125 ceph-mon[200197]: osdmap e216: 8 total, 8 up, 5 in 2023-10-09T10:50:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:15 smithi125 ceph-mon[200197]: failed to encode map e216 with expected crc 2023-10-09T10:50:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:15 smithi125 ceph-mon[200197]: failed to encode map e216 with expected crc 2023-10-09T10:50:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:50:15.602 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:15.613 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:15.660 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:15 smithi155 ceph-mon[184854]: failed to encode map e215 with expected crc 2023-10-09T10:50:15.660 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:15 smithi155 ceph-mon[184854]: osdmap e216: 8 total, 8 up, 5 in 2023-10-09T10:50:15.660 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:15 smithi155 ceph-mon[184854]: failed to encode map e216 with expected crc 2023-10-09T10:50:15.660 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:15 smithi155 ceph-mon[184854]: failed to encode map e216 with expected crc 2023-10-09T10:50:15.660 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:15.660 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:50:15.667 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:15.705 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:15.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:15.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:15.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:15.888 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:15.918 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:15.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:15.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:15.995 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 55 16 66049 66033 75.0124 48.1875 0.0308493 0.0133253 2023-10-09T10:50:16.029 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:16.075 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:16.564 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:16.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:16.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:16.599 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:50:16.625 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:50:16.653 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:50:16.670 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:16.682 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:50:16.716 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:50:16.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:16 smithi155 ceph-mon[184854]: failed to encode map e216 with expected crc 2023-10-09T10:50:16.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:16 smithi155 ceph-mon[184854]: pgmap v488: 109 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 54 MiB/s wr, 856 op/s; 33867/191748 objects degraded (17.662%); 1311/191748 objects misplaced (0.684%); 12 MiB/s, 194 objects/s recovering 2023-10-09T10:50:16.730 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:16 smithi125 ceph-mon[195507]: failed to encode map e216 with expected crc 2023-10-09T10:50:16.730 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:16 smithi125 ceph-mon[195507]: pgmap v488: 109 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 54 MiB/s wr, 856 op/s; 33867/191748 objects degraded (17.662%); 1311/191748 objects misplaced (0.684%); 12 MiB/s, 194 objects/s recovering 2023-10-09T10:50:16.730 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:16 smithi125 ceph-mon[200197]: failed to encode map e216 with expected crc 2023-10-09T10:50:16.730 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:16 smithi125 ceph-mon[200197]: pgmap v488: 109 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 54 MiB/s wr, 856 op/s; 33867/191748 objects degraded (17.662%); 1311/191748 objects misplaced (0.684%); 12 MiB/s, 194 objects/s recovering 2023-10-09T10:50:16.739 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:16.746 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:50:16.769 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:16.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:16.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:16.779 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:50:16.822 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:50:16.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:50:16.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:50:16.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:50:16.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:50:16.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:50:16.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:50:16.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:50:16.833 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:50:16.875 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:16.900 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:16.926 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:16.962 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:16.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:16.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:16.995 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 56 16 66859 66843 74.5769 50.625 0.0183901 0.0134028 2023-10-09T10:50:17.072 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:17.125 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:17.150 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:17.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:17.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:17.258 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:17.322 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:17.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2694682038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:17.450 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:17.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:17.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:17.559 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:17.629 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:17.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:17.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:17.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:17.658 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2694682038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:17.758 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:17.766 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2694682038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:17.823 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:17.852 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:17.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:17.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:17.956 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:17.995 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 57 16 67668 67652 74.1554 50.5625 0.0250998 0.013479 2023-10-09T10:50:18.031 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:18.069 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:18.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:18.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:18.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:18.235 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:18.262 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:18.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:18.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:18.334 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:50:18.334 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:50:18.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:50:18.334 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:50:18.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.335 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1070 6.8 6:1676aaf7:::benchmark_data_smithi125_244951_object1069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:50:18.335 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.621927+0000", 2023-10-09T10:50:18.335 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 345.69870390900002, 2023-10-09T10:50:18.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99674418099999995, 2023-10-09T10:50:18.335 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.335 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:50:18.335 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.335 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.335 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.335 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1070 2023-10-09T10:50:18.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.336 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621927+0000", 2023-10-09T10:50:18.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621927+0000", 2023-10-09T10:50:18.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621926+0000", 2023-10-09T10:50:18.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:18.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621950+0000", 2023-10-09T10:50:18.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3706999999999999e-05 2023-10-09T10:50:18.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621964+0000", 2023-10-09T10:50:18.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4589999999999999e-05 2023-10-09T10:50:18.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.621972+0000", 2023-10-09T10:50:18.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3510000000000002e-06 2023-10-09T10:50:18.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.622022+0000", 2023-10-09T10:50:18.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9088000000000002e-05 2023-10-09T10:50:18.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:18.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.622034+0000", 2023-10-09T10:50:18.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2584e-05 2023-10-09T10:50:18.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.618557+0000", 2023-10-09T10:50:18.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99652308199999995 2023-10-09T10:50:18.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.618672+0000", 2023-10-09T10:50:18.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011433 2023-10-09T10:50:18.340 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.341 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.341 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.341 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1095 6.8 6:164b39e5:::benchmark_data_smithi125_244951_object1094:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:50:18.341 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.623092+0000", 2023-10-09T10:50:18.341 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 345.69753970800002, 2023-10-09T10:50:18.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99981824900000005, 2023-10-09T10:50:18.341 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.341 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:18.341 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.341 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.342 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.342 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1095 2023-10-09T10:50:18.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.342 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623092+0000", 2023-10-09T10:50:18.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623092+0000", 2023-10-09T10:50:18.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623090+0000", 2023-10-09T10:50:18.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:50:18.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.344 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623113+0000", 2023-10-09T10:50:18.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2691e-05 2023-10-09T10:50:18.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.344 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.344 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623130+0000", 2023-10-09T10:50:18.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7594000000000001e-05 2023-10-09T10:50:18.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.344 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.344 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623139+0000", 2023-10-09T10:50:18.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5359999999999992e-06 2023-10-09T10:50:18.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623187+0000", 2023-10-09T10:50:18.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7874000000000003e-05 2023-10-09T10:50:18.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:18.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.623197+0000", 2023-10-09T10:50:18.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0889e-05 2023-10-09T10:50:18.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.618984+0000", 2023-10-09T10:50:18.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99578652199999995 2023-10-09T10:50:18.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.619093+0000", 2023-10-09T10:50:18.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108948 2023-10-09T10:50:18.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:50:18.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.619248+0000", 2023-10-09T10:50:18.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000154747 2023-10-09T10:50:18.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620532+0000", 2023-10-09T10:50:18.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012847729999999999 2023-10-09T10:50:18.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:18.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621605+0000", 2023-10-09T10:50:18.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010723569999999999 2023-10-09T10:50:18.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622824+0000", 2023-10-09T10:50:18.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012196939999999999 2023-10-09T10:50:18.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:18.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622872+0000", 2023-10-09T10:50:18.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7573e-05 2023-10-09T10:50:18.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622910+0000", 2023-10-09T10:50:18.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7790999999999998e-05 2023-10-09T10:50:18.349 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.349 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.349 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.350 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1097 6.8 6:1cc6c81b:::benchmark_data_smithi125_244951_object1096:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:50:18.350 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.625080+0000", 2023-10-09T10:50:18.350 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 345.69555109200002, 2023-10-09T10:50:18.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.997987073, 2023-10-09T10:50:18.350 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.350 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:18.350 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.350 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.350 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.350 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1097 2023-10-09T10:50:18.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.351 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625080+0000", 2023-10-09T10:50:18.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625080+0000", 2023-10-09T10:50:18.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.352 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625078+0000", 2023-10-09T10:50:18.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T10:50:18.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.352 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.352 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625108+0000", 2023-10-09T10:50:18.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9626000000000001e-05 2023-10-09T10:50:18.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.352 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.352 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625132+0000", 2023-10-09T10:50:18.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4236999999999999e-05 2023-10-09T10:50:18.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.353 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.353 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625145+0000", 2023-10-09T10:50:18.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2814e-05 2023-10-09T10:50:18.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.353 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.353 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625215+0000", 2023-10-09T10:50:18.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0779999999999997e-05 2023-10-09T10:50:18.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:18.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.625228+0000", 2023-10-09T10:50:18.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2989e-05 2023-10-09T10:50:18.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.619577+0000", 2023-10-09T10:50:18.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99434825299999996 2023-10-09T10:50:18.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.619653+0000", 2023-10-09T10:50:18.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.682e-05 2023-10-09T10:50:18.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:50:18.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.619810+0000", 2023-10-09T10:50:18.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000156737 2023-10-09T10:50:18.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:18.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621611+0000", 2023-10-09T10:50:18.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001800472 2023-10-09T10:50:18.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621664+0000", 2023-10-09T10:50:18.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2967999999999998e-05 2023-10-09T10:50:18.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622991+0000", 2023-10-09T10:50:18.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001327122 2023-10-09T10:50:18.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:18.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623034+0000", 2023-10-09T10:50:18.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2772000000000003e-05 2023-10-09T10:50:18.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623067+0000", 2023-10-09T10:50:18.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3767999999999999e-05 2023-10-09T10:50:18.357 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.357 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.357 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.358 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1100 6.8 6:1c8c7e3c:::benchmark_data_smithi125_244951_object1099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:50:18.358 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.627382+0000", 2023-10-09T10:50:18.358 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 345.693249326, 2023-10-09T10:50:18.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99583719999999998, 2023-10-09T10:50:18.358 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.358 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:18.358 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.358 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.358 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.358 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1100 2023-10-09T10:50:18.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.358 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627382+0000", 2023-10-09T10:50:18.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627382+0000", 2023-10-09T10:50:18.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627380+0000", 2023-10-09T10:50:18.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:50:18.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627405+0000", 2023-10-09T10:50:18.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5058e-05 2023-10-09T10:50:18.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627419+0000", 2023-10-09T10:50:18.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3431999999999999e-05 2023-10-09T10:50:18.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627426+0000", 2023-10-09T10:50:18.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.34e-06 2023-10-09T10:50:18.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627534+0000", 2023-10-09T10:50:18.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010761599999999999 2023-10-09T10:50:18.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:18.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.627543+0000", 2023-10-09T10:50:18.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2660000000000007e-06 2023-10-09T10:50:18.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.362 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620079+0000", 2023-10-09T10:50:18.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99253606000000005 2023-10-09T10:50:18.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.362 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620154+0000", 2023-10-09T10:50:18.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5461000000000001e-05 2023-10-09T10:50:18.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:50:18.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620263+0000", 2023-10-09T10:50:18.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010842599999999999 2023-10-09T10:50:18.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621714+0000", 2023-10-09T10:50:18.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001451265 2023-10-09T10:50:18.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:18.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621722+0000", 2023-10-09T10:50:18.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2740000000000007e-06 2023-10-09T10:50:18.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623143+0000", 2023-10-09T10:50:18.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014206570000000001 2023-10-09T10:50:18.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:18.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623183+0000", 2023-10-09T10:50:18.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9660999999999999e-05 2023-10-09T10:50:18.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623219+0000", 2023-10-09T10:50:18.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6433000000000002e-05 2023-10-09T10:50:18.365 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.365 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.365 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.365 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1136 6.8 6:193c408d:::benchmark_data_smithi125_244951_object1135:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:50:18.366 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.650385+0000", 2023-10-09T10:50:18.366 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 345.67024669900002, 2023-10-09T10:50:18.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97294367199999998, 2023-10-09T10:50:18.366 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.366 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:18.366 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.366 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.366 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.366 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1136 2023-10-09T10:50:18.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.366 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650385+0000", 2023-10-09T10:50:18.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650385+0000", 2023-10-09T10:50:18.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650383+0000", 2023-10-09T10:50:18.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:18.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650411+0000", 2023-10-09T10:50:18.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7837000000000001e-05 2023-10-09T10:50:18.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650421+0000", 2023-10-09T10:50:18.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8779999999999993e-06 2023-10-09T10:50:18.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650427+0000", 2023-10-09T10:50:18.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7270000000000001e-06 2023-10-09T10:50:18.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650504+0000", 2023-10-09T10:50:18.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6800999999999999e-05 2023-10-09T10:50:18.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:18.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.650514+0000", 2023-10-09T10:50:18.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9490000000000008e-06 2023-10-09T10:50:18.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620561+0000", 2023-10-09T10:50:18.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97004676599999995 2023-10-09T10:50:18.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620656+0000", 2023-10-09T10:50:18.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5345000000000003e-05 2023-10-09T10:50:18.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:50:18.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.620791+0000", 2023-10-09T10:50:18.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013480999999999999 2023-10-09T10:50:18.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:18.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621728+0000", 2023-10-09T10:50:18.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00093716100000000005 2023-10-09T10:50:18.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621777+0000", 2023-10-09T10:50:18.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8569999999999997e-05 2023-10-09T10:50:18.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623256+0000", 2023-10-09T10:50:18.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014793569999999999 2023-10-09T10:50:18.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:18.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623297+0000", 2023-10-09T10:50:18.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1012e-05 2023-10-09T10:50:18.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623328+0000", 2023-10-09T10:50:18.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1044999999999998e-05 2023-10-09T10:50:18.373 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.373 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.373 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.373 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1140 6.8 6:1bbcb053:::benchmark_data_smithi125_244951_object1139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:50:18.373 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.651480+0000", 2023-10-09T10:50:18.373 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 345.669151769, 2023-10-09T10:50:18.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97196035599999997, 2023-10-09T10:50:18.374 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.374 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:18.374 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.374 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.374 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.374 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1140 2023-10-09T10:50:18.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.374 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651480+0000", 2023-10-09T10:50:18.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651480+0000", 2023-10-09T10:50:18.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651479+0000", 2023-10-09T10:50:18.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:18.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651498+0000", 2023-10-09T10:50:18.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9128999999999999e-05 2023-10-09T10:50:18.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651507+0000", 2023-10-09T10:50:18.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0410000000000003e-06 2023-10-09T10:50:18.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651513+0000", 2023-10-09T10:50:18.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4990000000000002e-06 2023-10-09T10:50:18.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.377 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.377 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651606+0000", 2023-10-09T10:50:18.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3546000000000002e-05 2023-10-09T10:50:18.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.377 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:18.377 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651618+0000", 2023-10-09T10:50:18.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1600000000000001e-05 2023-10-09T10:50:18.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.377 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621048+0000", 2023-10-09T10:50:18.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96943005599999998 2023-10-09T10:50:18.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621174+0000", 2023-10-09T10:50:18.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012588899999999999 2023-10-09T10:50:18.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:50:18.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621311+0000", 2023-10-09T10:50:18.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000137415 2023-10-09T10:50:18.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.379 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:18.379 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622787+0000", 2023-10-09T10:50:18.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014754880000000001 2023-10-09T10:50:18.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.379 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.379 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622958+0000", 2023-10-09T10:50:18.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017176800000000001 2023-10-09T10:50:18.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623365+0000", 2023-10-09T10:50:18.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00040624999999999998 2023-10-09T10:50:18.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:18.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623405+0000", 2023-10-09T10:50:18.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0680999999999999e-05 2023-10-09T10:50:18.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623440+0000", 2023-10-09T10:50:18.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4474000000000002e-05 2023-10-09T10:50:18.381 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.381 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.381 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.381 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1141 6.8 6:1ec53f00:::benchmark_data_smithi125_244951_object1140:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:50:18.381 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.651524+0000", 2023-10-09T10:50:18.381 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 345.66910737900002, 2023-10-09T10:50:18.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97203539800000005, 2023-10-09T10:50:18.381 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.381 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:18.382 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.382 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.382 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.382 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1141 2023-10-09T10:50:18.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.382 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651524+0000", 2023-10-09T10:50:18.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651524+0000", 2023-10-09T10:50:18.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651523+0000", 2023-10-09T10:50:18.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:18.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651544+0000", 2023-10-09T10:50:18.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.014e-05 2023-10-09T10:50:18.384 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.384 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.384 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.384 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651552+0000", 2023-10-09T10:50:18.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6179999999999995e-06 2023-10-09T10:50:18.384 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.384 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.384 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.384 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651561+0000", 2023-10-09T10:50:18.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7860000000000002e-06 2023-10-09T10:50:18.384 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651639+0000", 2023-10-09T10:50:18.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8362999999999995e-05 2023-10-09T10:50:18.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:18.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.651646+0000", 2023-10-09T10:50:18.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6610000000000003e-06 2023-10-09T10:50:18.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621816+0000", 2023-10-09T10:50:18.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97017014700000004 2023-10-09T10:50:18.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.621928+0000", 2023-10-09T10:50:18.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111427 2023-10-09T10:50:18.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:50:18.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622053+0000", 2023-10-09T10:50:18.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012548999999999999 2023-10-09T10:50:18.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:18.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622921+0000", 2023-10-09T10:50:18.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00086811799999999997 2023-10-09T10:50:18.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623114+0000", 2023-10-09T10:50:18.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019289700000000001 2023-10-09T10:50:18.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623487+0000", 2023-10-09T10:50:18.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00037276399999999998 2023-10-09T10:50:18.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:18.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623528+0000", 2023-10-09T10:50:18.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1179e-05 2023-10-09T10:50:18.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623559+0000", 2023-10-09T10:50:18.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1245999999999999e-05 2023-10-09T10:50:18.389 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.389 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.389 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.389 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1151 6.8 6:15b7e58c:::benchmark_data_smithi125_244951_object1150:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:50:18.389 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.658348+0000", 2023-10-09T10:50:18.389 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 345.66228369700002, 2023-10-09T10:50:18.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96534045700000004, 2023-10-09T10:50:18.390 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.390 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:18.390 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.390 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.390 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.390 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1151 2023-10-09T10:50:18.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.390 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658348+0000", 2023-10-09T10:50:18.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658348+0000", 2023-10-09T10:50:18.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658347+0000", 2023-10-09T10:50:18.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:18.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658366+0000", 2023-10-09T10:50:18.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8757e-05 2023-10-09T10:50:18.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.392 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.392 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.392 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658378+0000", 2023-10-09T10:50:18.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.216e-05 2023-10-09T10:50:18.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.392 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.392 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.392 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658384+0000", 2023-10-09T10:50:18.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0170000000000002e-06 2023-10-09T10:50:18.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.392 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.393 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.393 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658460+0000", 2023-10-09T10:50:18.393 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5543999999999994e-05 2023-10-09T10:50:18.393 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.393 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:18.393 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.658481+0000", 2023-10-09T10:50:18.393 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1916000000000002e-05 2023-10-09T10:50:18.393 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.393 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.393 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622329+0000", 2023-10-09T10:50:18.394 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96384757600000004 2023-10-09T10:50:18.394 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.394 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622438+0000", 2023-10-09T10:50:18.394 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108985 2023-10-09T10:50:18.394 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.394 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:50:18.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.622573+0000", 2023-10-09T10:50:18.394 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013480499999999999 2023-10-09T10:50:18.394 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.395 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:18.395 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623082+0000", 2023-10-09T10:50:18.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00050925299999999998 2023-10-09T10:50:18.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.395 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.395 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623595+0000", 2023-10-09T10:50:18.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051302300000000004 2023-10-09T10:50:18.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.395 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623625+0000", 2023-10-09T10:50:18.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0309000000000001e-05 2023-10-09T10:50:18.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:18.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623657+0000", 2023-10-09T10:50:18.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2027999999999999e-05 2023-10-09T10:50:18.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623688+0000", 2023-10-09T10:50:18.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0593000000000003e-05 2023-10-09T10:50:18.396 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.397 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.397 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.397 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:1184 6.8 6:157bd3fc:::benchmark_data_smithi125_244951_object1183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T10:50:18.397 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:32.696219+0000", 2023-10-09T10:50:18.397 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 345.62441271599999, 2023-10-09T10:50:18.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.92991913699999995, 2023-10-09T10:50:18.397 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.397 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:18.397 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.397 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.398 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.398 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1184 2023-10-09T10:50:18.398 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.398 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.398 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.398 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696219+0000", 2023-10-09T10:50:18.398 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.398 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.398 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696219+0000", 2023-10-09T10:50:18.399 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.399 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.399 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.399 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696217+0000", 2023-10-09T10:50:18.399 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:18.399 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.399 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.399 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696237+0000", 2023-10-09T10:50:18.399 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9983000000000002e-05 2023-10-09T10:50:18.400 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.400 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.400 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.400 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696249+0000", 2023-10-09T10:50:18.400 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2215000000000001e-05 2023-10-09T10:50:18.400 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.400 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.400 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.400 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696257+0000", 2023-10-09T10:50:18.400 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.072e-06 2023-10-09T10:50:18.400 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.400 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.400 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.401 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696297+0000", 2023-10-09T10:50:18.401 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9852999999999999e-05 2023-10-09T10:50:18.401 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.401 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.401 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:18.401 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:32.696305+0000", 2023-10-09T10:50:18.401 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7870000000000002e-06 2023-10-09T10:50:18.401 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.401 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.401 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.401 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623715+0000", 2023-10-09T10:50:18.401 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.92741050999999997 2023-10-09T10:50:18.401 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.402 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.402 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.402 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623814+0000", 2023-10-09T10:50:18.402 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8057e-05 2023-10-09T10:50:18.402 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.402 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.402 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:50:18.402 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.623938+0000", 2023-10-09T10:50:18.402 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012472899999999999 2023-10-09T10:50:18.402 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.402 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.402 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:18.402 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.625153+0000", 2023-10-09T10:50:18.403 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012150970000000001 2023-10-09T10:50:18.403 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.403 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.403 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.403 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.625631+0000", 2023-10-09T10:50:18.403 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047751299999999999 2023-10-09T10:50:18.403 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.403 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.403 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.403 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.626058+0000", 2023-10-09T10:50:18.403 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042723499999999998 2023-10-09T10:50:18.403 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.404 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.404 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:18.404 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.626104+0000", 2023-10-09T10:50:18.404 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5569999999999999e-05 2023-10-09T10:50:18.404 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.404 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.404 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.404 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:33.626138+0000", 2023-10-09T10:50:18.404 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4069999999999997e-05 2023-10-09T10:50:18.404 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.404 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.404 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.405 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.405 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.405 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:70301 6.19 6:98815d50:::benchmark_data_smithi125_244951_object70300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:50:18.405 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:23.364524+0000", 2023-10-09T10:50:18.405 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 294.95610764499997, 2023-10-09T10:50:18.405 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.054642890999999999, 2023-10-09T10:50:18.405 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.405 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:18.405 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.405 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.405 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.405 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 70301 2023-10-09T10:50:18.405 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.406 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.406 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.406 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.406 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364524+0000", 2023-10-09T10:50:18.406 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.406 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.406 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.406 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.406 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364524+0000", 2023-10-09T10:50:18.406 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.406 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.407 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.407 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.407 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364522+0000", 2023-10-09T10:50:18.407 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:50:18.407 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.407 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.407 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.407 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364544+0000", 2023-10-09T10:50:18.407 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1990000000000001e-05 2023-10-09T10:50:18.407 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.407 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.407 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.408 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364556+0000", 2023-10-09T10:50:18.408 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2181000000000001e-05 2023-10-09T10:50:18.408 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.408 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.408 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.408 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364564+0000", 2023-10-09T10:50:18.408 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0499999999999992e-06 2023-10-09T10:50:18.408 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.408 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.408 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.408 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364698+0000", 2023-10-09T10:50:18.408 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013398800000000001 2023-10-09T10:50:18.408 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.408 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.409 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.409 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364826+0000", 2023-10-09T10:50:18.409 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012756099999999999 2023-10-09T10:50:18.409 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.409 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.409 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:50:18.409 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.364936+0000", 2023-10-09T10:50:18.409 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011065000000000001 2023-10-09T10:50:18.409 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.409 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.409 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.409 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.366921+0000", 2023-10-09T10:50:18.409 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019843679999999998 2023-10-09T10:50:18.410 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.410 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.410 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:18.410 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.366945+0000", 2023-10-09T10:50:18.410 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3995999999999999e-05 2023-10-09T10:50:18.410 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.410 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.410 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.410 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.418241+0000", 2023-10-09T10:50:18.410 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.051296003 2023-10-09T10:50:18.410 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.411 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.411 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:18.411 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.418305+0000", 2023-10-09T10:50:18.411 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4815999999999997e-05 2023-10-09T10:50:18.411 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.411 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.411 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.411 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.419167+0000", 2023-10-09T10:50:18.411 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00086109200000000004 2023-10-09T10:50:18.411 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.411 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.411 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.412 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.412 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.412 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:70319 6.18 6:1f556d03:::benchmark_data_smithi125_244951_object70318:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:50:18.412 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:23.385122+0000", 2023-10-09T10:50:18.412 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 294.93550974200002, 2023-10-09T10:50:18.412 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.059906651999999998, 2023-10-09T10:50:18.412 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.412 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:18.412 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.412 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.412 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.413 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 70319 2023-10-09T10:50:18.413 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.413 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.413 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.413 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.413 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385122+0000", 2023-10-09T10:50:18.413 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.413 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.413 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.413 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.413 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385122+0000", 2023-10-09T10:50:18.413 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.414 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.414 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.414 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.414 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385120+0000", 2023-10-09T10:50:18.414 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:18.414 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.414 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.414 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.414 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385145+0000", 2023-10-09T10:50:18.414 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4414e-05 2023-10-09T10:50:18.414 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.415 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385155+0000", 2023-10-09T10:50:18.415 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0805000000000001e-05 2023-10-09T10:50:18.415 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.415 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385163+0000", 2023-10-09T10:50:18.415 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0330000000000008e-06 2023-10-09T10:50:18.415 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.416 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385219+0000", 2023-10-09T10:50:18.416 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5894999999999999e-05 2023-10-09T10:50:18.416 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.416 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.416 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385382+0000", 2023-10-09T10:50:18.416 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016238299999999999 2023-10-09T10:50:18.416 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.416 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.416 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:50:18.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.385553+0000", 2023-10-09T10:50:18.417 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017152199999999999 2023-10-09T10:50:18.417 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.417 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.417 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.417 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.399377+0000", 2023-10-09T10:50:18.417 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013823291 2023-10-09T10:50:18.417 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.417 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.417 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:18.417 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.400837+0000", 2023-10-09T10:50:18.417 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014599890000000001 2023-10-09T10:50:18.417 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.418 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.418 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.418 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.444954+0000", 2023-10-09T10:50:18.418 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.044117351999999999 2023-10-09T10:50:18.418 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.418 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.418 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:18.418 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.445001+0000", 2023-10-09T10:50:18.418 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7105999999999999e-05 2023-10-09T10:50:18.418 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.418 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.418 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.419 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:23.445028+0000", 2023-10-09T10:50:18.419 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7172e-05 2023-10-09T10:50:18.419 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.419 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.419 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.419 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.419 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.419 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106423 6.9 6:930bb25f:::benchmark_data_smithi125_244951_object106422:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:50:18.419 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.239301+0000", 2023-10-09T10:50:18.419 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 256.08133076899998, 2023-10-09T10:50:18.419 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.46007959199999998, 2023-10-09T10:50:18.419 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.419 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:50:18.420 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.420 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.420 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.420 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106423 2023-10-09T10:50:18.420 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.420 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.420 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.420 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.420 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239301+0000", 2023-10-09T10:50:18.420 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.420 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.420 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.421 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.421 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239301+0000", 2023-10-09T10:50:18.421 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.421 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.421 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.421 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.421 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239299+0000", 2023-10-09T10:50:18.421 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:50:18.421 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.421 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.421 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.422 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239340+0000", 2023-10-09T10:50:18.422 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0917000000000001e-05 2023-10-09T10:50:18.422 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.422 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.422 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.422 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239360+0000", 2023-10-09T10:50:18.422 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0649000000000002e-05 2023-10-09T10:50:18.422 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.422 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.422 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.422 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239371+0000", 2023-10-09T10:50:18.423 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0244000000000001e-05 2023-10-09T10:50:18.423 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.423 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.423 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.423 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.244269+0000", 2023-10-09T10:50:18.423 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0048985590000000002 2023-10-09T10:50:18.423 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.423 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.423 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.423 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.244451+0000", 2023-10-09T10:50:18.423 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000181526 2023-10-09T10:50:18.424 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.424 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.424 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:50:18.424 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.244646+0000", 2023-10-09T10:50:18.424 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000195133 2023-10-09T10:50:18.424 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.424 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.424 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.424 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.271437+0000", 2023-10-09T10:50:18.424 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.026791150999999999 2023-10-09T10:50:18.424 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.424 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.424 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.425 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274828+0000", 2023-10-09T10:50:18.425 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003391088 2023-10-09T10:50:18.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:18.425 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274837+0000", 2023-10-09T10:50:18.425 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6899999999999998e-06 2023-10-09T10:50:18.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.425 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.699260+0000", 2023-10-09T10:50:18.425 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.42442315200000003 2023-10-09T10:50:18.426 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.426 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.426 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.699380+0000", 2023-10-09T10:50:18.426 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120189 2023-10-09T10:50:18.426 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.426 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.426 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.426 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.426 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.426 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106425 6.9 6:966db102:::benchmark_data_smithi125_244951_object106424:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:50:18.426 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.239425+0000", 2023-10-09T10:50:18.427 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 256.08120609600002, 2023-10-09T10:50:18.427 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.46673965099999998, 2023-10-09T10:50:18.427 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.427 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:50:18.427 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.427 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.427 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.427 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106425 2023-10-09T10:50:18.427 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.427 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.427 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.427 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.428 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239425+0000", 2023-10-09T10:50:18.428 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.428 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.428 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.428 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.428 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239425+0000", 2023-10-09T10:50:18.428 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.428 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.428 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.428 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.428 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239425+0000", 2023-10-09T10:50:18.428 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:18.429 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.429 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.429 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.429 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239473+0000", 2023-10-09T10:50:18.429 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8177000000000003e-05 2023-10-09T10:50:18.429 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.429 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.429 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.429 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239480+0000", 2023-10-09T10:50:18.429 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0559999999999999e-06 2023-10-09T10:50:18.429 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.429 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.430 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.430 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.239485+0000", 2023-10-09T10:50:18.430 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.242e-06 2023-10-09T10:50:18.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.430 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.430 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.430 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.249603+0000", 2023-10-09T10:50:18.430 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010118121000000001 2023-10-09T10:50:18.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.430 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.430 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.249760+0000", 2023-10-09T10:50:18.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000156331 2023-10-09T10:50:18.431 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:50:18.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.249953+0000", 2023-10-09T10:50:18.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019313500000000001 2023-10-09T10:50:18.431 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274855+0000", 2023-10-09T10:50:18.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.024902173999999999 2023-10-09T10:50:18.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.432 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:18.432 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274860+0000", 2023-10-09T10:50:18.432 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8380000000000001e-06 2023-10-09T10:50:18.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.432 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.432 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.700402+0000", 2023-10-09T10:50:18.432 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.42554259900000002 2023-10-09T10:50:18.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:50:18.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.701238+0000", 2023-10-09T10:50:18.433 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000835825 2023-10-09T10:50:18.433 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.706109+0000", 2023-10-09T10:50:18.433 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0048705440000000001 2023-10-09T10:50:18.433 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.434 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.706165+0000", 2023-10-09T10:50:18.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6017999999999998e-05 2023-10-09T10:50:18.434 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.434 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.434 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.434 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.434 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.434 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106422 6.19 6:9914323d:::benchmark_data_smithi125_244951_object106421:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T10:50:18.434 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.601105+0000", 2023-10-09T10:50:18.434 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 255.71952638499999, 2023-10-09T10:50:18.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0560851339999999, 2023-10-09T10:50:18.434 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.435 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:50:18.435 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.435 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.435 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.435 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106422 2023-10-09T10:50:18.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.435 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.435 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601105+0000", 2023-10-09T10:50:18.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601105+0000", 2023-10-09T10:50:18.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601103+0000", 2023-10-09T10:50:18.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:50:18.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601130+0000", 2023-10-09T10:50:18.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7222000000000001e-05 2023-10-09T10:50:18.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.437 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.437 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601147+0000", 2023-10-09T10:50:18.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6600999999999999e-05 2023-10-09T10:50:18.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.437 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.437 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601162+0000", 2023-10-09T10:50:18.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4929000000000001e-05 2023-10-09T10:50:18.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.438 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.438 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.438 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.643541+0000", 2023-10-09T10:50:18.438 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.042379130000000001 2023-10-09T10:50:18.438 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.438 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.438 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:18.438 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.643555+0000", 2023-10-09T10:50:18.438 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3465000000000001e-05 2023-10-09T10:50:18.438 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.438 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.438 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.439 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:03.657089+0000", 2023-10-09T10:50:18.439 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0135346009999999 2023-10-09T10:50:18.439 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.439 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.439 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.439 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:03.657190+0000", 2023-10-09T10:50:18.439 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000100945 2023-10-09T10:50:18.439 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.439 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.439 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.439 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.440 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.440 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106426 6.19 6:9f6a94cf:::benchmark_data_smithi125_244951_object106425:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T10:50:18.440 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.601170+0000", 2023-10-09T10:50:18.440 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 255.719460876, 2023-10-09T10:50:18.440 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.057006321, 2023-10-09T10:50:18.440 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.440 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:50:18.440 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.440 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.440 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.440 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106426 2023-10-09T10:50:18.440 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.440 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.441 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601170+0000", 2023-10-09T10:50:18.441 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.441 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601170+0000", 2023-10-09T10:50:18.441 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601170+0000", 2023-10-09T10:50:18.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:18.442 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.442 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601191+0000", 2023-10-09T10:50:18.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0746000000000001e-05 2023-10-09T10:50:18.442 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.442 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601199+0000", 2023-10-09T10:50:18.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8019999999999998e-06 2023-10-09T10:50:18.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.601222+0000", 2023-10-09T10:50:18.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2801000000000001e-05 2023-10-09T10:50:18.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.643580+0000", 2023-10-09T10:50:18.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.042357842 2023-10-09T10:50:18.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.444 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:50:18.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.643589+0000", 2023-10-09T10:50:18.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1749999999999994e-06 2023-10-09T10:50:18.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.444 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:03.658076+0000", 2023-10-09T10:50:18.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0144863019999999 2023-10-09T10:50:18.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.444 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:03.658177+0000", 2023-10-09T10:50:18.445 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000101201 2023-10-09T10:50:18.445 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.445 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.445 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.445 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.445 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.445 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:115958 6.f 6:fa100500:::benchmark_data_smithi125_244951_object9515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-10-09T10:50:18.445 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:20.075222+0000", 2023-10-09T10:50:18.445 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.24540952800001, 2023-10-09T10:50:18.445 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.084394099, 2023-10-09T10:50:18.445 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.445 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:18.446 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.446 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.446 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.446 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115958 2023-10-09T10:50:18.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.446 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.446 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.446 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075222+0000", 2023-10-09T10:50:18.446 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.447 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.447 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075222+0000", 2023-10-09T10:50:18.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.447 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.447 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.447 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.447 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075220+0000", 2023-10-09T10:50:18.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:50:18.447 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.447 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.447 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.447 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075229+0000", 2023-10-09T10:50:18.448 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7949999999999996e-06 2023-10-09T10:50:18.448 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075244+0000", 2023-10-09T10:50:18.448 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5364000000000001e-05 2023-10-09T10:50:18.448 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.075254+0000", 2023-10-09T10:50:18.448 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4609999999999997e-06 2023-10-09T10:50:18.448 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.449 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.449 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.076549+0000", 2023-10-09T10:50:18.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012954900000000001 2023-10-09T10:50:18.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.449 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.449 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.076690+0000", 2023-10-09T10:50:18.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014043200000000001 2023-10-09T10:50:18.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.449 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:50:18.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.076818+0000", 2023-10-09T10:50:18.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012810400000000001 2023-10-09T10:50:18.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:18.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.085804+0000", 2023-10-09T10:50:18.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0089861170000000001 2023-10-09T10:50:18.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.087330+0000", 2023-10-09T10:50:18.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001526108 2023-10-09T10:50:18.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159461+0000", 2023-10-09T10:50:18.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.072131120000000007 2023-10-09T10:50:18.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:18.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159562+0000", 2023-10-09T10:50:18.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000101475 2023-10-09T10:50:18.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159616+0000", 2023-10-09T10:50:18.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3387000000000002e-05 2023-10-09T10:50:18.452 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.452 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.452 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.452 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:115960 6.f 6:f825ff14:::benchmark_data_smithi125_244951_object9517:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-10-09T10:50:18.452 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:20.081466+0000", 2023-10-09T10:50:18.452 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.239165066, 2023-10-09T10:50:18.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.078315183999999996, 2023-10-09T10:50:18.453 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.453 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:18.453 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.453 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.453 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.453 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115960 2023-10-09T10:50:18.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.453 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081466+0000", 2023-10-09T10:50:18.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081466+0000", 2023-10-09T10:50:18.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081464+0000", 2023-10-09T10:50:18.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:50:18.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081473+0000", 2023-10-09T10:50:18.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3960000000000006e-06 2023-10-09T10:50:18.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081490+0000", 2023-10-09T10:50:18.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7730000000000001e-05 2023-10-09T10:50:18.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.081502+0000", 2023-10-09T10:50:18.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1612e-05 2023-10-09T10:50:18.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.082766+0000", 2023-10-09T10:50:18.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012642350000000001 2023-10-09T10:50:18.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.082875+0000", 2023-10-09T10:50:18.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010849999999999999 2023-10-09T10:50:18.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:50:18.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.082969+0000", 2023-10-09T10:50:18.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4368000000000002e-05 2023-10-09T10:50:18.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.091397+0000", 2023-10-09T10:50:18.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0084276379999999994 2023-10-09T10:50:18.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:18.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.105493+0000", 2023-10-09T10:50:18.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014096630000000001 2023-10-09T10:50:18.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159659+0000", 2023-10-09T10:50:18.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.05416588 2023-10-09T10:50:18.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:18.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159740+0000", 2023-10-09T10:50:18.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0735999999999994e-05 2023-10-09T10:50:18.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159781+0000", 2023-10-09T10:50:18.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1458999999999997e-05 2023-10-09T10:50:18.459 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.459 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.459 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.460 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:115962 6.f 6:f37830ff:::benchmark_data_smithi125_244951_object9519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-10-09T10:50:18.460 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:20.095380+0000", 2023-10-09T10:50:18.460 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.225251327, 2023-10-09T10:50:18.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.064565055999999996, 2023-10-09T10:50:18.460 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.460 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:18.460 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.460 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.460 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.460 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:18.461 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115962 2023-10-09T10:50:18.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.461 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095380+0000", 2023-10-09T10:50:18.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.462 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.462 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.462 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095380+0000", 2023-10-09T10:50:18.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.462 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.462 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.462 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095377+0000", 2023-10-09T10:50:18.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T10:50:18.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.463 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095391+0000", 2023-10-09T10:50:18.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3073e-05 2023-10-09T10:50:18.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.463 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095412+0000", 2023-10-09T10:50:18.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1406000000000001e-05 2023-10-09T10:50:18.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.464 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095423+0000", 2023-10-09T10:50:18.464 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0764000000000001e-05 2023-10-09T10:50:18.464 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.464 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.464 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.464 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.097612+0000", 2023-10-09T10:50:18.464 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021888530000000002 2023-10-09T10:50:18.464 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.464 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.464 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.464 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.097704+0000", 2023-10-09T10:50:18.464 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2219000000000006e-05 2023-10-09T10:50:18.465 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.465 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:50:18.465 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.097775+0000", 2023-10-09T10:50:18.465 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1330999999999999e-05 2023-10-09T10:50:18.465 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.465 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.465 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.101037+0000", 2023-10-09T10:50:18.465 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032613899999999999 2023-10-09T10:50:18.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.466 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:18.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.122053+0000", 2023-10-09T10:50:18.466 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.021016743000000001 2023-10-09T10:50:18.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.466 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159823+0000", 2023-10-09T10:50:18.466 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.037769619999999997 2023-10-09T10:50:18.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.467 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:18.467 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159902+0000", 2023-10-09T10:50:18.467 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9357999999999995e-05 2023-10-09T10:50:18.467 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.467 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.467 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159945+0000", 2023-10-09T10:50:18.467 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2772999999999999e-05 2023-10-09T10:50:18.467 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.467 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.467 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.467 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.468 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.468 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:115966 6.f 6:f2b073c2:::benchmark_data_smithi125_244951_object9523:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-10-09T10:50:18.468 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:20.095468+0000", 2023-10-09T10:50:18.468 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.22516352100001, 2023-10-09T10:50:18.468 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.067415613999999999, 2023-10-09T10:50:18.468 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.468 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:18.468 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.468 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.468 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.468 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115966 2023-10-09T10:50:18.468 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.469 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.469 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.469 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095468+0000", 2023-10-09T10:50:18.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.469 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.469 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.469 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095468+0000", 2023-10-09T10:50:18.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.469 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.469 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.469 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.470 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095468+0000", 2023-10-09T10:50:18.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:18.470 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.470 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095470+0000", 2023-10-09T10:50:18.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2520000000000002e-06 2023-10-09T10:50:18.470 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.470 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095475+0000", 2023-10-09T10:50:18.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8919999999999999e-06 2023-10-09T10:50:18.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.471 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.471 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.095479+0000", 2023-10-09T10:50:18.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.476e-06 2023-10-09T10:50:18.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.471 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.471 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.101070+0000", 2023-10-09T10:50:18.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0055904309999999999 2023-10-09T10:50:18.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.472 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.472 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.472 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.101187+0000", 2023-10-09T10:50:18.472 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000117111 2023-10-09T10:50:18.472 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.472 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.472 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:50:18.472 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.101301+0000", 2023-10-09T10:50:18.472 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000113842 2023-10-09T10:50:18.472 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.472 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.472 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.473 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.111354+0000", 2023-10-09T10:50:18.473 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010053945 2023-10-09T10:50:18.473 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.473 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.473 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:18.473 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.122063+0000", 2023-10-09T10:50:18.473 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010709000999999999 2023-10-09T10:50:18.473 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.473 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.473 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.473 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.159984+0000", 2023-10-09T10:50:18.473 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.037920483999999997 2023-10-09T10:50:18.474 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.474 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.474 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:18.474 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.162830+0000", 2023-10-09T10:50:18.474 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002845888 2023-10-09T10:50:18.474 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.474 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.474 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.474 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:20.162883+0000", 2023-10-09T10:50:18.474 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3554999999999998e-05 2023-10-09T10:50:18.474 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.475 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.475 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.475 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.475 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.475 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:153008 6.0 6:05373277:::benchmark_data_smithi125_244951_object46565:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:50:18.475 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:35.545330+0000", 2023-10-09T10:50:18.475 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 162.77530085800001, 2023-10-09T10:50:18.475 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.057702479000000001, 2023-10-09T10:50:18.475 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:18.475 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:18.475 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:18.475 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:50:18.475 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:18.476 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 153008 2023-10-09T10:50:18.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.476 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:18.476 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.476 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:18.476 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545330+0000", 2023-10-09T10:50:18.476 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.476 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.476 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:18.476 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545330+0000", 2023-10-09T10:50:18.476 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:18.477 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.477 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.477 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:18.477 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545329+0000", 2023-10-09T10:50:18.477 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:50:18.477 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.477 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.477 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:18.477 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545338+0000", 2023-10-09T10:50:18.477 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3919999999999992e-06 2023-10-09T10:50:18.477 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.477 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.478 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:18.478 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545359+0000", 2023-10-09T10:50:18.478 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0851000000000001e-05 2023-10-09T10:50:18.478 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.478 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.478 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:18.478 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545368+0000", 2023-10-09T10:50:18.478 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7679999999999997e-06 2023-10-09T10:50:18.478 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.478 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.478 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:18.478 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545452+0000", 2023-10-09T10:50:18.478 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3881000000000005e-05 2023-10-09T10:50:18.479 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.479 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.479 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:18.479 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545599+0000", 2023-10-09T10:50:18.479 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014761900000000001 2023-10-09T10:50:18.479 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.479 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.479 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:50:18.479 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.545725+0000", 2023-10-09T10:50:18.479 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000126148 2023-10-09T10:50:18.479 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.479 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.479 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.480 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.547777+0000", 2023-10-09T10:50:18.480 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020514560000000001 2023-10-09T10:50:18.480 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.480 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.480 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:18.480 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.547844+0000", 2023-10-09T10:50:18.480 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7282999999999999e-05 2023-10-09T10:50:18.480 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.480 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.480 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.480 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.553741+0000", 2023-10-09T10:50:18.480 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0058972629999999998 2023-10-09T10:50:18.481 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.481 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.481 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:18.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.602869+0000", 2023-10-09T10:50:18.481 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.049127951000000003 2023-10-09T10:50:18.481 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.481 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.481 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:50:18.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.602922+0000", 2023-10-09T10:50:18.481 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2329e-05 2023-10-09T10:50:18.481 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:18.481 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:18.481 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:18.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:35.603033+0000", 2023-10-09T10:50:18.482 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011130599999999999 2023-10-09T10:50:18.482 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.482 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.482 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.482 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:18.482 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:18.482 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:50:18.528 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:18.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:18 smithi125 ceph-mon[195507]: pgmap v489: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 42 MiB/s wr, 669 op/s; 30702/195240 objects degraded (15.725%); 3862/195240 objects misplaced (1.978%); 10 MiB/s, 162 objects/s recovering 2023-10-09T10:50:18.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:18 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30702/195240 objects degraded (15.725%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:18.660 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:18 smithi125 ceph-mon[200197]: pgmap v489: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 42 MiB/s wr, 669 op/s; 30702/195240 objects degraded (15.725%); 3862/195240 objects misplaced (1.978%); 10 MiB/s, 162 objects/s recovering 2023-10-09T10:50:18.661 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:18 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30702/195240 objects degraded (15.725%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:18.661 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:18.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:18.684 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:50:18.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:18 smithi155 ceph-mon[184854]: pgmap v489: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 42 MiB/s wr, 669 op/s; 30702/195240 objects degraded (15.725%); 3862/195240 objects misplaced (1.978%); 10 MiB/s, 162 objects/s recovering 2023-10-09T10:50:18.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:18 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30702/195240 objects degraded (15.725%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:18.764 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:18.807 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:18.842 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:18.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:18.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:18.946 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:18.996 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 58 15 68469 68454 73.7399 50.125 0.0259583 0.0135555 2023-10-09T10:50:19.016 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:19.052 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:19.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:19.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:19.131 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:19.156 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:19.201 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:19.281 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:19.281 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:50:19.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:19.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:19.330 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0] out_osds: [2, 7, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:50:19.330 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:50:19.330 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:50:19.330 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.813220 2023-10-09T10:50:19.330 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 0.8132203687403816 2023-10-09T10:50:19.388 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:19.421 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:19 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1355441621' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:50:19.421 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:19 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:50:19.466 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:50:19.476 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:19 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1355441621' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:50:19.477 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:19 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:50:19.477 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:19 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1355441621' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:50:19.477 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:19 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:50:19.477 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:19.562 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:19.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:19.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:19.666 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:19.717 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:19.752 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:19.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:19.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:19.860 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:19.918 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:19.944 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:19.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:19.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:19.996 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 59 16 69237 69221 73.3026 47.9375 0.010605 0.0136359 2023-10-09T10:50:20.053 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:20.109 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:20.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:20.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:20.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:20.216 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:20.251 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:20.317 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:20.353 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:20.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:20.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:20.359 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:20.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:20.510 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:20.553 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:20.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:20.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:20.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:20.720 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:20 smithi155 ceph-mon[184854]: pgmap v490: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 50 MiB/s wr, 798 op/s; 30702/199713 objects degraded (15.373%); 3862/199713 objects misplaced (1.934%); 8.9 MiB/s, 141 objects/s recovering 2023-10-09T10:50:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:20 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:20.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:20 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:50:20.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:20 smithi155 ceph-mon[184854]: osdmap e217: 8 total, 8 up, 5 in 2023-10-09T10:50:20.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:20 smithi155 ceph-mon[184854]: failed to encode map e217 with expected crc 2023-10-09T10:50:20.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:20 smithi155 ceph-mon[184854]: failed to encode map e217 with expected crc 2023-10-09T10:50:20.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:20 smithi155 ceph-mon[184854]: failed to encode map e217 with expected crc 2023-10-09T10:50:20.729 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:50:20.730 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:50:20.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.730 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:69786 7.3 7:ce42eaea:::benchmark_data_smithi125_295816_object69785:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-10-09T10:50:20.730 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:50:20.715475+0000", 2023-10-09T10:50:20.730 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.011925544999999999, 2023-10-09T10:50:20.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011961774, 2023-10-09T10:50:20.730 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:20.730 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:50:20.730 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:20.730 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:50:20.731 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:50:20.731 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 69786 2023-10-09T10:50:20.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.731 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:20.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:20.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715475+0000", 2023-10-09T10:50:20.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:20.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715475+0000", 2023-10-09T10:50:20.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:20.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715474+0000", 2023-10-09T10:50:20.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:20.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:20.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715501+0000", 2023-10-09T10:50:20.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6794000000000001e-05 2023-10-09T10:50:20.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:20.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715520+0000", 2023-10-09T10:50:20.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9103999999999999e-05 2023-10-09T10:50:20.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:20.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715528+0000", 2023-10-09T10:50:20.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2479999999999996e-06 2023-10-09T10:50:20.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:20.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715586+0000", 2023-10-09T10:50:20.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7887000000000003e-05 2023-10-09T10:50:20.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:20.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715770+0000", 2023-10-09T10:50:20.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018362200000000001 2023-10-09T10:50:20.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:50:20.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715941+0000", 2023-10-09T10:50:20.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017114300000000001 2023-10-09T10:50:20.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:20.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.721405+0000", 2023-10-09T10:50:20.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0054640579999999999 2023-10-09T10:50:20.735 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.735 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:20.735 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.736 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:69787 7.3 7:c83136ac:::benchmark_data_smithi125_295816_object69786:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-10-09T10:50:20.736 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:50:20.715544+0000", 2023-10-09T10:50:20.736 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.011857479000000001, 2023-10-09T10:50:20.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011981448, 2023-10-09T10:50:20.736 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:20.736 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:50:20.736 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:20.736 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:50:20.736 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:50:20.736 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 69787 2023-10-09T10:50:20.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.736 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:20.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:20.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715544+0000", 2023-10-09T10:50:20.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:20.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715544+0000", 2023-10-09T10:50:20.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:20.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715543+0000", 2023-10-09T10:50:20.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:20.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:20.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715561+0000", 2023-10-09T10:50:20.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8216e-05 2023-10-09T10:50:20.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:20.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715567+0000", 2023-10-09T10:50:20.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2389999999999999e-06 2023-10-09T10:50:20.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:20.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715573+0000", 2023-10-09T10:50:20.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1240000000000002e-06 2023-10-09T10:50:20.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:20.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.718555+0000", 2023-10-09T10:50:20.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029812929999999999 2023-10-09T10:50:20.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:20.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.718724+0000", 2023-10-09T10:50:20.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016958699999999999 2023-10-09T10:50:20.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:50:20.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.718890+0000", 2023-10-09T10:50:20.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016544699999999999 2023-10-09T10:50:20.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:20.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.721683+0000", 2023-10-09T10:50:20.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027931259999999999 2023-10-09T10:50:20.741 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.741 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:20.741 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.741 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:69793 7.c 7:3cca2a07:::benchmark_data_smithi125_295816_object69792:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-10-09T10:50:20.741 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:50:20.715579+0000", 2023-10-09T10:50:20.741 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.011822373000000001, 2023-10-09T10:50:20.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012024770000000001, 2023-10-09T10:50:20.741 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:20.741 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:50:20.742 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:20.742 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:50:20.742 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:50:20.742 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 69793 2023-10-09T10:50:20.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.742 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:20.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:20.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715579+0000", 2023-10-09T10:50:20.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:20.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715579+0000", 2023-10-09T10:50:20.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:20.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715578+0000", 2023-10-09T10:50:20.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:20.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:20.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715598+0000", 2023-10-09T10:50:20.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0159000000000001e-05 2023-10-09T10:50:20.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:20.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715604+0000", 2023-10-09T10:50:20.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9220000000000002e-06 2023-10-09T10:50:20.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:20.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715611+0000", 2023-10-09T10:50:20.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7100000000000001e-06 2023-10-09T10:50:20.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:20.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715663+0000", 2023-10-09T10:50:20.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2383000000000003e-05 2023-10-09T10:50:20.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:20.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715805+0000", 2023-10-09T10:50:20.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014158500000000001 2023-10-09T10:50:20.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:50:20.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715941+0000", 2023-10-09T10:50:20.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000136138 2023-10-09T10:50:20.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:20.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725873+0000", 2023-10-09T10:50:20.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0099323710000000006 2023-10-09T10:50:20.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:20.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727565+0000", 2023-10-09T10:50:20.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016915669999999999 2023-10-09T10:50:20.747 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.747 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:20.747 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.747 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:69794 7.2 7:49580436:::benchmark_data_smithi125_295816_object69793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-10-09T10:50:20.747 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:50:20.715622+0000", 2023-10-09T10:50:20.747 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.011778557, 2023-10-09T10:50:20.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012034189000000001, 2023-10-09T10:50:20.747 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:20.747 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:50:20.747 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:50:20.748 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:50:20.748 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:50:20.748 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 69794 2023-10-09T10:50:20.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.748 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:20.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:20.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715622+0000", 2023-10-09T10:50:20.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:20.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715622+0000", 2023-10-09T10:50:20.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:20.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715622+0000", 2023-10-09T10:50:20.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:20.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:20.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715640+0000", 2023-10-09T10:50:20.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8088e-05 2023-10-09T10:50:20.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:20.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715646+0000", 2023-10-09T10:50:20.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3450000000000002e-06 2023-10-09T10:50:20.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:20.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715651+0000", 2023-10-09T10:50:20.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.69e-06 2023-10-09T10:50:20.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:20.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715714+0000", 2023-10-09T10:50:20.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2606000000000003e-05 2023-10-09T10:50:20.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:20.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.715857+0000", 2023-10-09T10:50:20.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014358 2023-10-09T10:50:20.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:50:20.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.716023+0000", 2023-10-09T10:50:20.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000165556 2023-10-09T10:50:20.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:50:20.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.721278+0000", 2023-10-09T10:50:20.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0052546479999999998 2023-10-09T10:50:20.752 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.752 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:20.752 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.753 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35187.0:69783 7.f e217/203 7:f2568996:::benchmark_data_smithi125_295816_object69782:head v 217'4332, mlcod=0'0)", 2023-10-09T10:50:20.753 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:50:20.725328+0000", 2023-10-09T10:50:20.753 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0020726500000000001, 2023-10-09T10:50:20.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002373932, 2023-10-09T10:50:20.753 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:20.753 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:50:20.753 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:20.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:20.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725328+0000", 2023-10-09T10:50:20.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:20.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725328+0000", 2023-10-09T10:50:20.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:20.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725327+0000", 2023-10-09T10:50:20.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:50:20.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:20.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725361+0000", 2023-10-09T10:50:20.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3204000000000003e-05 2023-10-09T10:50:20.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:20.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725370+0000", 2023-10-09T10:50:20.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5259999999999999e-06 2023-10-09T10:50:20.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:20.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725376+0000", 2023-10-09T10:50:20.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5010000000000004e-06 2023-10-09T10:50:20.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:20.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725413+0000", 2023-10-09T10:50:20.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7357999999999997e-05 2023-10-09T10:50:20.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:20.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725439+0000", 2023-10-09T10:50:20.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6041e-05 2023-10-09T10:50:20.757 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.757 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:20.757 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.757 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35187.0:69789 7.7 e217/196 7:edb5110c:::benchmark_data_smithi125_295816_object69788:head v 217'4430, mlcod=217'4428)", 2023-10-09T10:50:20.757 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:50:20.725403+0000", 2023-10-09T10:50:20.757 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0019976709999999999, 2023-10-09T10:50:20.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023338310000000002, 2023-10-09T10:50:20.758 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:20.758 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:50:20.758 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:20.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:20.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725403+0000", 2023-10-09T10:50:20.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:20.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725403+0000", 2023-10-09T10:50:20.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:20.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725403+0000", 2023-10-09T10:50:20.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:20.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:20.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725441+0000", 2023-10-09T10:50:20.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8343999999999997e-05 2023-10-09T10:50:20.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:20.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725448+0000", 2023-10-09T10:50:20.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5119999999999997e-06 2023-10-09T10:50:20.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:20.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725453+0000", 2023-10-09T10:50:20.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4330000000000003e-06 2023-10-09T10:50:20.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:20.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725608+0000", 2023-10-09T10:50:20.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015452400000000001 2023-10-09T10:50:20.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:20.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725625+0000", 2023-10-09T10:50:20.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7232999999999999e-05 2023-10-09T10:50:20.762 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.763 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:20.763 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.763 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35187.0:69791 7.7 e217/196)", 2023-10-09T10:50:20.763 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:50:20.725459+0000", 2023-10-09T10:50:20.763 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.001942343, 2023-10-09T10:50:20.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023104079999999999, 2023-10-09T10:50:20.763 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:20.763 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:50:20.763 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:20.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:20.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725459+0000", 2023-10-09T10:50:20.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:20.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725459+0000", 2023-10-09T10:50:20.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:20.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725458+0000", 2023-10-09T10:50:20.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:20.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:20.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725480+0000", 2023-10-09T10:50:20.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2010000000000001e-05 2023-10-09T10:50:20.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:20.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725487+0000", 2023-10-09T10:50:20.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2709999999999998e-06 2023-10-09T10:50:20.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:20.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725491+0000", 2023-10-09T10:50:20.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7129999999999996e-06 2023-10-09T10:50:20.766 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.766 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:20.766 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.766 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35187.0:69795 7.f e217/203)", 2023-10-09T10:50:20.767 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:50:20.725496+0000", 2023-10-09T10:50:20.767 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0019052979999999999, 2023-10-09T10:50:20.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023012190000000002, 2023-10-09T10:50:20.767 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:20.767 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:50:20.767 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:20.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:20.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725496+0000", 2023-10-09T10:50:20.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:20.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725496+0000", 2023-10-09T10:50:20.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:20.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725495+0000", 2023-10-09T10:50:20.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:20.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:20.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725514+0000", 2023-10-09T10:50:20.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8703e-05 2023-10-09T10:50:20.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:20.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725520+0000", 2023-10-09T10:50:20.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9109999999999998e-06 2023-10-09T10:50:20.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:20.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725525+0000", 2023-10-09T10:50:20.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5159999999999996e-06 2023-10-09T10:50:20.769 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.769 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:20.770 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.770 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35187.0:69796 7.7 e217/196)", 2023-10-09T10:50:20.770 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:50:20.725529+0000", 2023-10-09T10:50:20.770 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0018724200000000001, 2023-10-09T10:50:20.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002305255, 2023-10-09T10:50:20.770 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:20.770 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:50:20.770 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:20.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:20.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725529+0000", 2023-10-09T10:50:20.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:20.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725529+0000", 2023-10-09T10:50:20.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:20.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725528+0000", 2023-10-09T10:50:20.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:20.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:20.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725548+0000", 2023-10-09T10:50:20.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9539999999999999e-05 2023-10-09T10:50:20.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:20.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725553+0000", 2023-10-09T10:50:20.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5999999999999997e-06 2023-10-09T10:50:20.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:20.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.725557+0000", 2023-10-09T10:50:20.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9840000000000003e-06 2023-10-09T10:50:20.773 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.773 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:20.773 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.773 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35187.0:69797 7.7 e217/196)", 2023-10-09T10:50:20.773 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:50:20.727069+0000", 2023-10-09T10:50:20.773 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.000332393, 2023-10-09T10:50:20.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00079670299999999995, 2023-10-09T10:50:20.773 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:20.774 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:50:20.774 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:20.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:20.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727069+0000", 2023-10-09T10:50:20.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:20.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727069+0000", 2023-10-09T10:50:20.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:20.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727068+0000", 2023-10-09T10:50:20.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:50:20.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:20.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727094+0000", 2023-10-09T10:50:20.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6214e-05 2023-10-09T10:50:20.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:20.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727109+0000", 2023-10-09T10:50:20.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5446000000000001e-05 2023-10-09T10:50:20.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:20.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727118+0000", 2023-10-09T10:50:20.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4740000000000005e-06 2023-10-09T10:50:20.776 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.776 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:20.776 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.777 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.35187.0:69786 7.3 e217/196)", 2023-10-09T10:50:20.777 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:50:20.727848+0000", 2023-10-09T10:50:20.777 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4294967295.9995532, 2023-10-09T10:50:20.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9670000000000002e-05, 2023-10-09T10:50:20.777 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:20.777 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:50:20.777 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:20.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:20.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727848+0000", 2023-10-09T10:50:20.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:20.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727848+0000", 2023-10-09T10:50:20.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:20.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727847+0000", 2023-10-09T10:50:20.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:20.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:20.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727853+0000", 2023-10-09T10:50:20.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8340000000000004e-06 2023-10-09T10:50:20.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:20.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727864+0000", 2023-10-09T10:50:20.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0621999999999999e-05 2023-10-09T10:50:20.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:20.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727870+0000", 2023-10-09T10:50:20.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5690000000000001e-06 2023-10-09T10:50:20.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:20.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727907+0000", 2023-10-09T10:50:20.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6742999999999997e-05 2023-10-09T10:50:20.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:20.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727914+0000", 2023-10-09T10:50:20.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6350000000000006e-06 2023-10-09T10:50:20.780 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.780 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:20.781 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.781 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.35187.0:69787 7.3 e217/196)", 2023-10-09T10:50:20.781 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:50:20.727882+0000", 2023-10-09T10:50:20.781 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4294967295.9995189, 2023-10-09T10:50:20.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010639199999999999, 2023-10-09T10:50:20.781 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:50:20.781 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:50:20.781 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:50:20.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:50:20.782 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727882+0000", 2023-10-09T10:50:20.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:50:20.782 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727882+0000", 2023-10-09T10:50:20.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:50:20.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:50:20.782 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727882+0000", 2023-10-09T10:50:20.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:20.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:50:20.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727884+0000", 2023-10-09T10:50:20.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3070000000000001e-06 2023-10-09T10:50:20.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:50:20.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727890+0000", 2023-10-09T10:50:20.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7450000000000001e-06 2023-10-09T10:50:20.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:50:20.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727894+0000", 2023-10-09T10:50:20.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4179999999999999e-06 2023-10-09T10:50:20.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:50:20.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727974+0000", 2023-10-09T10:50:20.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.996e-05 2023-10-09T10:50:20.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:50:20.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727981+0000", 2023-10-09T10:50:20.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1640000000000004e-06 2023-10-09T10:50:20.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:50:20.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:50:20.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:50:20.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:50:20.727991+0000", 2023-10-09T10:50:20.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1050000000000001e-06 2023-10-09T10:50:20.785 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.785 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:50:20.785 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.785 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:50:20.785 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:50:20.786 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 12 2023-10-09T10:50:20.786 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:50:20.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:20.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:20 smithi125 ceph-mon[195507]: pgmap v490: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 50 MiB/s wr, 798 op/s; 30702/199713 objects degraded (15.373%); 3862/199713 objects misplaced (1.934%); 8.9 MiB/s, 141 objects/s recovering 2023-10-09T10:50:20.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:20 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:20.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:20 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:50:20.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:20 smithi125 ceph-mon[195507]: osdmap e217: 8 total, 8 up, 5 in 2023-10-09T10:50:20.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:20 smithi125 ceph-mon[195507]: failed to encode map e217 with expected crc 2023-10-09T10:50:20.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:20 smithi125 ceph-mon[195507]: failed to encode map e217 with expected crc 2023-10-09T10:50:20.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:20 smithi125 ceph-mon[195507]: failed to encode map e217 with expected crc 2023-10-09T10:50:20.787 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:20 smithi125 ceph-mon[200197]: pgmap v490: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 50 MiB/s wr, 798 op/s; 30702/199713 objects degraded (15.373%); 3862/199713 objects misplaced (1.934%); 8.9 MiB/s, 141 objects/s recovering 2023-10-09T10:50:20.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:20 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:20.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:20 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:50:20.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:20 smithi125 ceph-mon[200197]: osdmap e217: 8 total, 8 up, 5 in 2023-10-09T10:50:20.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:20 smithi125 ceph-mon[200197]: failed to encode map e217 with expected crc 2023-10-09T10:50:20.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:20 smithi125 ceph-mon[200197]: failed to encode map e217 with expected crc 2023-10-09T10:50:20.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:20 smithi125 ceph-mon[200197]: failed to encode map e217 with expected crc 2023-10-09T10:50:20.788 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:20.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:20.889 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:20.951 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:20.994 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:20.996 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:50:20.997410+0000 min lat: 0.00131548 max lat: 2.28015 avg lat: 0.0137162 2023-10-09T10:50:20.996 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:50:20.997 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 60 16 69994 69978 72.8695 47.3125 0.0199379 0.0137162 2023-10-09T10:50:21.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:21.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:21.101 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:21.149 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:21.284 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:50:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:50:21] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.43.0" 2023-10-09T10:50:21.318 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:50:21.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:21 smithi125 ceph-mon[195507]: failed to encode map e217 with expected crc 2023-10-09T10:50:21.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:21 smithi125 ceph-mon[195507]: failed to encode map e217 with expected crc 2023-10-09T10:50:21.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:21 smithi125 ceph-mon[200197]: failed to encode map e217 with expected crc 2023-10-09T10:50:21.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:21 smithi125 ceph-mon[200197]: failed to encode map e217 with expected crc 2023-10-09T10:50:21.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:21 smithi155 ceph-mon[184854]: failed to encode map e217 with expected crc 2023-10-09T10:50:21.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:21 smithi155 ceph-mon[184854]: failed to encode map e217 with expected crc 2023-10-09T10:50:21.997 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 61 16 70813 70797 72.514 51.1875 0.019198 0.0137845 2023-10-09T10:50:22.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:50:22.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:22 smithi125 ceph-mon[195507]: pgmap v492: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 43 MiB/s wr, 686 op/s; 30702/204324 objects degraded (15.026%); 3506/204324 objects misplaced (1.716%); 11 MiB/s, 178 objects/s recovering 2023-10-09T10:50:22.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:22 smithi125 ceph-mon[200197]: pgmap v492: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 43 MiB/s wr, 686 op/s; 30702/204324 objects degraded (15.026%); 3506/204324 objects misplaced (1.716%); 11 MiB/s, 178 objects/s recovering 2023-10-09T10:50:22.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:22 smithi155 ceph-mon[184854]: pgmap v492: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 43 MiB/s wr, 686 op/s; 30702/204324 objects degraded (15.026%); 3506/204324 objects misplaced (1.716%); 11 MiB/s, 178 objects/s recovering 2023-10-09T10:50:22.997 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 62 16 71675 71659 72.2134 53.875 0.0132891 0.0138405 2023-10-09T10:50:23.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:23 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30702/204324 objects degraded (15.026%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:23.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:23 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30702/204324 objects degraded (15.026%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:23.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:23 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30702/204324 objects degraded (15.026%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:23.997 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 63 16 72523 72507 71.9084 53 0.010551 0.0139011 2023-10-09T10:50:24.525 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:24.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:24.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:24.634 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:24.744 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:24 smithi125 ceph-mon[195507]: pgmap v493: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 49 MiB/s wr, 780 op/s; 30702/211581 objects degraded (14.511%); 3299/211581 objects misplaced (1.559%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T10:50:24.744 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:24 smithi125 ceph-mon[200197]: pgmap v493: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 49 MiB/s wr, 780 op/s; 30702/211581 objects degraded (14.511%); 3299/211581 objects misplaced (1.559%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T10:50:24.745 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:24.774 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:24.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:24.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:24.796 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:24.832 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:50:24.878 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:24.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:24.897 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:50:24.929 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:50:24.968 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:24.971 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:50:24.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:24 smithi155 ceph-mon[184854]: pgmap v493: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 49 MiB/s wr, 780 op/s; 30702/211581 objects degraded (14.511%); 3299/211581 objects misplaced (1.559%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T10:50:24.997 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 64 16 73270 73254 71.5144 46.6875 0.0432184 0.0139745 2023-10-09T10:50:25.019 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:50:25.059 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:50:25.091 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:50:25.127 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:50:25.148 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:50:25.148 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:50:25.148 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:50:25.148 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:50:25.149 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:50:25.149 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:50:25.149 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:50:25.149 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:50:25.363 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:25.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:25.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:25.467 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:25.512 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:25.517 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.6 primary-affinity to 0.81322 (8532952) 2023-10-09T10:50:25.645 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:25.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:25.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:25.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3890811471' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:25.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2982863242' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.8132203687403816}]: dispatch 2023-10-09T10:50:25.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:25 smithi125 ceph-mon[195507]: pgmap v494: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 48 MiB/s wr, 763 op/s; 30702/214659 objects degraded (14.303%); 3299/214659 objects misplaced (1.537%); 7.4 MiB/s, 117 objects/s recovering 2023-10-09T10:50:25.650 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3890811471' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:25.650 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2982863242' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.8132203687403816}]: dispatch 2023-10-09T10:50:25.650 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:25 smithi125 ceph-mon[200197]: pgmap v494: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 48 MiB/s wr, 763 op/s; 30702/214659 objects degraded (14.303%); 3299/214659 objects misplaced (1.537%); 7.4 MiB/s, 117 objects/s recovering 2023-10-09T10:50:25.750 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:25.778 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:25.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3890811471' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:25.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2982863242' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.8132203687403816}]: dispatch 2023-10-09T10:50:25.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:25 smithi155 ceph-mon[184854]: pgmap v494: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 48 MiB/s wr, 763 op/s; 30702/214659 objects degraded (14.303%); 3299/214659 objects misplaced (1.537%); 7.4 MiB/s, 117 objects/s recovering 2023-10-09T10:50:25.998 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 65 16 73788 73772 70.9118 32.375 0.00456484 0.0140172 2023-10-09T10:50:26.553 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:26.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:26.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:26.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:26 smithi125 ceph-mon[195507]: failed to encode map e218 with expected crc 2023-10-09T10:50:26.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:26 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2982863242' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.8132203687403816}]': finished 2023-10-09T10:50:26.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:26 smithi125 ceph-mon[195507]: osdmap e218: 8 total, 8 up, 5 in 2023-10-09T10:50:26.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:26 smithi125 ceph-mon[195507]: failed to encode map e218 with expected crc 2023-10-09T10:50:26.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:26 smithi125 ceph-mon[195507]: failed to encode map e218 with expected crc 2023-10-09T10:50:26.560 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:26 smithi125 ceph-mon[195507]: failed to encode map e218 with expected crc 2023-10-09T10:50:26.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:26 smithi125 ceph-mon[200197]: failed to encode map e218 with expected crc 2023-10-09T10:50:26.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:26 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2982863242' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.8132203687403816}]': finished 2023-10-09T10:50:26.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:26 smithi125 ceph-mon[200197]: osdmap e218: 8 total, 8 up, 5 in 2023-10-09T10:50:26.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:26 smithi125 ceph-mon[200197]: failed to encode map e218 with expected crc 2023-10-09T10:50:26.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:26 smithi125 ceph-mon[200197]: failed to encode map e218 with expected crc 2023-10-09T10:50:26.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:26 smithi125 ceph-mon[200197]: failed to encode map e218 with expected crc 2023-10-09T10:50:26.660 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:26.690 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:26.691 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:50:26.691 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:50:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:26 smithi155 ceph-mon[184854]: failed to encode map e218 with expected crc 2023-10-09T10:50:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:26 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2982863242' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.8132203687403816}]': finished 2023-10-09T10:50:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:26 smithi155 ceph-mon[184854]: osdmap e218: 8 total, 8 up, 5 in 2023-10-09T10:50:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:26 smithi155 ceph-mon[184854]: failed to encode map e218 with expected crc 2023-10-09T10:50:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:26 smithi155 ceph-mon[184854]: failed to encode map e218 with expected crc 2023-10-09T10:50:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:26 smithi155 ceph-mon[184854]: failed to encode map e218 with expected crc 2023-10-09T10:50:26.997 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 66 16 73829 73813 69.8764 2.5625 0.00345995 0.0140997 2023-10-09T10:50:27.060 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:27.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:27.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:27.163 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:27.192 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:27.212 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:27.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:27.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:27.315 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:27.340 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:50:27.343 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:27.362 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:27.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:27.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:27.466 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:27.493 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:27.516 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:50:27.522 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:27.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:27.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:27.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:27 smithi125 ceph-mon[195507]: failed to encode map e219 with expected crc 2023-10-09T10:50:27.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:27 smithi125 ceph-mon[195507]: failed to encode map e219 with expected crc 2023-10-09T10:50:27.540 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:27 smithi125 ceph-mon[195507]: osdmap e219: 8 total, 8 up, 5 in 2023-10-09T10:50:27.540 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:27 smithi125 ceph-mon[195507]: failed to encode map e219 with expected crc 2023-10-09T10:50:27.540 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:27 smithi125 ceph-mon[195507]: failed to encode map e219 with expected crc 2023-10-09T10:50:27.540 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:27 smithi125 ceph-mon[195507]: pgmap v497: 109 pgs: 1 active+recovering+undersized+remapped, 13 peering, 1 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 11 KiB/s rd, 50 MiB/s wr, 804 op/s; 30700/218328 objects degraded (14.061%); 2519/218328 objects misplaced (1.154%); 11 MiB/s, 172 objects/s recovering 2023-10-09T10:50:27.540 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:27 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3024090542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:27.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:27 smithi125 ceph-mon[200197]: failed to encode map e219 with expected crc 2023-10-09T10:50:27.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:27 smithi125 ceph-mon[200197]: failed to encode map e219 with expected crc 2023-10-09T10:50:27.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:27 smithi125 ceph-mon[200197]: osdmap e219: 8 total, 8 up, 5 in 2023-10-09T10:50:27.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:27 smithi125 ceph-mon[200197]: failed to encode map e219 with expected crc 2023-10-09T10:50:27.541 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:27 smithi125 ceph-mon[200197]: failed to encode map e219 with expected crc 2023-10-09T10:50:27.541 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:27 smithi125 ceph-mon[200197]: pgmap v497: 109 pgs: 1 active+recovering+undersized+remapped, 13 peering, 1 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 11 KiB/s rd, 50 MiB/s wr, 804 op/s; 30700/218328 objects degraded (14.061%); 2519/218328 objects misplaced (1.154%); 11 MiB/s, 172 objects/s recovering 2023-10-09T10:50:27.541 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:27 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3024090542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:27.626 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:27.695 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:27.734 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:27.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:27.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:27.837 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:27.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:27 smithi155 ceph-mon[184854]: failed to encode map e219 with expected crc 2023-10-09T10:50:27.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:27 smithi155 ceph-mon[184854]: failed to encode map e219 with expected crc 2023-10-09T10:50:27.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:27 smithi155 ceph-mon[184854]: osdmap e219: 8 total, 8 up, 5 in 2023-10-09T10:50:27.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:27 smithi155 ceph-mon[184854]: failed to encode map e219 with expected crc 2023-10-09T10:50:27.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:27 smithi155 ceph-mon[184854]: failed to encode map e219 with expected crc 2023-10-09T10:50:27.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:27 smithi155 ceph-mon[184854]: pgmap v497: 109 pgs: 1 active+recovering+undersized+remapped, 13 peering, 1 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 11 KiB/s rd, 50 MiB/s wr, 804 op/s; 30700/218328 objects degraded (14.061%); 2519/218328 objects misplaced (1.154%); 11 MiB/s, 172 objects/s recovering 2023-10-09T10:50:27.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:27 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3024090542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:27.893 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:27.897 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:27.940 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:27.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:27.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:27.997 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 67 16 74163 74147 69.1452 20.875 0.015182 0.0144555 2023-10-09T10:50:28.045 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:28.085 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:28.109 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:28.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:28.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:28.215 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:28.252 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:28.330 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:28.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:28.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:28.437 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:28.485 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:28.770 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:28.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:28.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:28.778 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 3 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T10:50:28.778 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:50:28.778 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[195507]: failed to encode map e220 with expected crc 2023-10-09T10:50:28.778 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[195507]: failed to encode map e220 with expected crc 2023-10-09T10:50:28.778 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3024090542' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:50:28.778 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[195507]: osdmap e220: 8 total, 8 up, 5 in 2023-10-09T10:50:28.778 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[195507]: failed to encode map e220 with expected crc 2023-10-09T10:50:28.778 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30700/218328 objects degraded (14.061%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:28.779 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 3 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T10:50:28.779 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:50:28.779 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[200197]: failed to encode map e220 with expected crc 2023-10-09T10:50:28.779 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[200197]: failed to encode map e220 with expected crc 2023-10-09T10:50:28.779 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3024090542' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:50:28.779 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[200197]: osdmap e220: 8 total, 8 up, 5 in 2023-10-09T10:50:28.779 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[200197]: failed to encode map e220 with expected crc 2023-10-09T10:50:28.779 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:28 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30700/218328 objects degraded (14.061%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:28.879 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:28.921 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:28.955 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:28.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:28.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:28.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:28 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 3 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T10:50:28.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:28 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:50:28.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:28 smithi155 ceph-mon[184854]: failed to encode map e220 with expected crc 2023-10-09T10:50:28.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:28 smithi155 ceph-mon[184854]: failed to encode map e220 with expected crc 2023-10-09T10:50:28.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:28 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3024090542' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:50:28.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:28 smithi155 ceph-mon[184854]: osdmap e220: 8 total, 8 up, 5 in 2023-10-09T10:50:28.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:28 smithi155 ceph-mon[184854]: failed to encode map e220 with expected crc 2023-10-09T10:50:28.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:28 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30700/218328 objects degraded (14.061%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:28.998 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 68 16 75012 74996 68.9087 53.0625 0.0149046 0.0145063 2023-10-09T10:50:28.999 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:50:29.064 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:29.117 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:29.149 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:29.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:29.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:29.257 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:29.260 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:50:29.260 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:50:29.325 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:29.350 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:29.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:29.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:29.454 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:29.536 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:29.571 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:29.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:29.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:29.681 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:29.729 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:29.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:29 smithi125 ceph-mon[195507]: pgmap v499: 109 pgs: 1 active+recovering+undersized+remapped, 13 peering, 1 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 14 KiB/s rd, 32 MiB/s wr, 520 op/s; 30700/220752 objects degraded (13.907%); 2519/220752 objects misplaced (1.141%); 8.2 MiB/s, 129 objects/s recovering 2023-10-09T10:50:29.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:29 smithi125 ceph-mon[200197]: pgmap v499: 109 pgs: 1 active+recovering+undersized+remapped, 13 peering, 1 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 14 KiB/s rd, 32 MiB/s wr, 520 op/s; 30700/220752 objects degraded (13.907%); 2519/220752 objects misplaced (1.141%); 8.2 MiB/s, 129 objects/s recovering 2023-10-09T10:50:29.920 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:50:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:29 smithi155 ceph-mon[184854]: pgmap v499: 109 pgs: 1 active+recovering+undersized+remapped, 13 peering, 1 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 14 KiB/s rd, 32 MiB/s wr, 520 op/s; 30700/220752 objects degraded (13.907%); 2519/220752 objects misplaced (1.141%); 8.2 MiB/s, 129 objects/s recovering 2023-10-09T10:50:29.998 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 69 16 75832 75816 68.6527 51.25 0.0157016 0.014561 2023-10-09T10:50:30.315 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:30.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:30.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:30.384 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:30.420 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:30.421 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:50:30.505 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:30.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:30.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:30.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:30.665 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:30.733 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:30.777 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:30.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:30.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:30.885 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:30.908 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:50:30.932 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:30.998 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 70 16 76626 76610 68.3809 49.625 0.0325076 0.0146187 2023-10-09T10:50:31.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:31 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:31.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:31 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:50:31.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:31 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:31.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:31 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:50:31.263 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:31.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:31.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:31.295 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:31 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:31.295 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:31 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:50:31.367 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:31.390 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0] out_osds: [2, 7, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:50:31.390 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:50:31.390 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T10:50:31.390 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:50:31.390 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:50:31.391 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T10:50:31.457 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:31.473 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:31.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:50:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:50:31] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.43.0" 2023-10-09T10:50:31.695 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:31.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:31.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:31.800 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:31.839 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:31.885 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:31.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:31.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:31.989 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:31.998 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 71 16 77366 77350 68.0692 46.25 0.0264371 0.0146856 2023-10-09T10:50:32.060 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:32.195 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:50:32.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:32.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:32 smithi155 ceph-mon[184854]: pgmap v500: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 14 KiB/s rd, 37 MiB/s wr, 605 op/s; 30700/225369 objects degraded (13.622%); 2519/225369 objects misplaced (1.118%); 8.2 MiB/s, 129 objects/s recovering 2023-10-09T10:50:32.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:32.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:32.241 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:32 smithi125 ceph-mon[195507]: pgmap v500: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 14 KiB/s rd, 37 MiB/s wr, 605 op/s; 30700/225369 objects degraded (13.622%); 2519/225369 objects misplaced (1.118%); 8.2 MiB/s, 129 objects/s recovering 2023-10-09T10:50:32.241 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:32 smithi125 ceph-mon[200197]: pgmap v500: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 14 KiB/s rd, 37 MiB/s wr, 605 op/s; 30700/225369 objects degraded (13.622%); 2519/225369 objects misplaced (1.118%); 8.2 MiB/s, 129 objects/s recovering 2023-10-09T10:50:32.253 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:50:32.254 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:50:32.334 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:32.393 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:32.536 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:50:32.577 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:32.578 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:50:32.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:32.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:32.608 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:50:32.639 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:50:32.676 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:50:32.688 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:32.708 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:50:32.721 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:50:32.736 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:32.762 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:32.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:32.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:32.782 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:50:32.822 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:50:32.824 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:50:32.824 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:50:32.824 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:50:32.824 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:50:32.824 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:50:32.824 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:50:32.824 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:50:32.824 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:50:32.867 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:32.939 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:33.003 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 72 16 78088 78072 67.7475 45.125 0.0259321 0.0147547 2023-10-09T10:50:33.064 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:33 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 8 pgs peering) 2023-10-09T10:50:33.065 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:33 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3516839931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:33.065 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:33 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30700/225369 objects degraded (13.622%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:33.065 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:33 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 8 pgs peering) 2023-10-09T10:50:33.065 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:33 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3516839931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:33.065 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:33 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30700/225369 objects degraded (13.622%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:33.239 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:33.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:33 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 8 pgs peering) 2023-10-09T10:50:33.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:33 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3516839931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:33.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:33 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30700/225369 objects degraded (13.622%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:33.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:33.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:33.346 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:33.458 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:33.464 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:50:33.550 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:50:33.550 INFO:teuthology.orchestra.run.smithi155.stdout: "success": "" 2023-10-09T10:50:33.550 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:50:33.674 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:33.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:33.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:33.780 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:33.861 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:33.996 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:34.001 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 73 16 78824 78808 67.4496 46 0.0162534 0.0148193 2023-10-09T10:50:34.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:34.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:34.060 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:50:34.106 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:34.147 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:34.177 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:34 smithi155 ceph-mon[184854]: pgmap v501: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 11 KiB/s rd, 43 MiB/s wr, 702 op/s; 30700/230775 objects degraded (13.303%); 2164/230775 objects misplaced (0.938%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T10:50:34.177 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:34 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2536556165' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:50:34.177 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:34 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:50:34.281 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:34 smithi125 ceph-mon[195507]: pgmap v501: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 11 KiB/s rd, 43 MiB/s wr, 702 op/s; 30700/230775 objects degraded (13.303%); 2164/230775 objects misplaced (0.938%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T10:50:34.281 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:34 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2536556165' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:50:34.281 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:34 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:50:34.283 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:34 smithi125 ceph-mon[200197]: pgmap v501: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 11 KiB/s rd, 43 MiB/s wr, 702 op/s; 30700/230775 objects degraded (13.303%); 2164/230775 objects misplaced (0.938%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T10:50:34.283 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:34 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2536556165' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:50:34.283 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:34 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:50:34.424 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:34.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:34.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:34.530 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:34.586 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:34.620 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:34.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:34.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:34.730 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:34.764 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:34.785 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:34.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:34.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:34.892 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:34.925 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:50:34.926 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:50:34.926 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:50:34.926 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:50:34.926 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:50:34.994 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:35.001 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 74 16 79600 79584 67.1935 48.5 0.0214683 0.0148763 2023-10-09T10:50:35.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:35 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:35.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:35 smithi125 ceph-mon[195507]: failed to encode map e221 with expected crc 2023-10-09T10:50:35.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:35 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:50:35.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:35 smithi125 ceph-mon[195507]: osdmap e221: 8 total, 8 up, 5 in 2023-10-09T10:50:35.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:35 smithi125 ceph-mon[195507]: failed to encode map e221 with expected crc 2023-10-09T10:50:35.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:35 smithi125 ceph-mon[195507]: failed to encode map e221 with expected crc 2023-10-09T10:50:35.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:35 smithi125 ceph-mon[195507]: failed to encode map e221 with expected crc 2023-10-09T10:50:35.098 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:35.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:35.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:35.204 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:35.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:35 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:35.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:35 smithi155 ceph-mon[184854]: failed to encode map e221 with expected crc 2023-10-09T10:50:35.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:35 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:50:35.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:35 smithi155 ceph-mon[184854]: osdmap e221: 8 total, 8 up, 5 in 2023-10-09T10:50:35.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:35 smithi155 ceph-mon[184854]: failed to encode map e221 with expected crc 2023-10-09T10:50:35.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:35 smithi155 ceph-mon[184854]: failed to encode map e221 with expected crc 2023-10-09T10:50:35.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:35 smithi155 ceph-mon[184854]: failed to encode map e221 with expected crc 2023-10-09T10:50:35.241 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:50:35.249 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:35.281 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:35.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:35.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:35.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:35 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:35.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:35 smithi125 ceph-mon[200197]: failed to encode map e221 with expected crc 2023-10-09T10:50:35.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:35 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:50:35.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:35 smithi125 ceph-mon[200197]: osdmap e221: 8 total, 8 up, 5 in 2023-10-09T10:50:35.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:35 smithi125 ceph-mon[200197]: failed to encode map e221 with expected crc 2023-10-09T10:50:35.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:35 smithi125 ceph-mon[200197]: failed to encode map e221 with expected crc 2023-10-09T10:50:35.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:35 smithi125 ceph-mon[200197]: failed to encode map e221 with expected crc 2023-10-09T10:50:35.369 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:50:35.388 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:35.447 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:35.490 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:35.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:35.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:35.595 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:35.646 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:35.673 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:35.677 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:35.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:35.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:35.786 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:35.839 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:35.862 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:50:35.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:35.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:35.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:35.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:36.002 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 75 16 80354 80338 66.926 47.125 0.0252088 0.0149357 2023-10-09T10:50:36.040 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:36.077 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:36.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:36.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:36.186 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:36.229 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:36.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:36 smithi125 ceph-mon[195507]: failed to encode map e221 with expected crc 2023-10-09T10:50:36.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:36 smithi125 ceph-mon[195507]: pgmap v503: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 40 MiB/s wr, 636 op/s; 30700/233601 objects degraded (13.142%); 2164/233601 objects misplaced (0.926%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T10:50:36.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:36 smithi125 ceph-mon[200197]: failed to encode map e221 with expected crc 2023-10-09T10:50:36.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:36 smithi125 ceph-mon[200197]: pgmap v503: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 40 MiB/s wr, 636 op/s; 30700/233601 objects degraded (13.142%); 2164/233601 objects misplaced (0.926%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T10:50:36.423 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:36.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:36.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:36.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:36 smithi155 ceph-mon[184854]: failed to encode map e221 with expected crc 2023-10-09T10:50:36.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:36 smithi155 ceph-mon[184854]: pgmap v503: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 40 MiB/s wr, 636 op/s; 30700/233601 objects degraded (13.142%); 2164/233601 objects misplaced (0.926%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T10:50:36.501 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:50:36.531 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:36.599 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:36.692 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:36.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:36.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:36.799 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:36.858 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:37.002 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 76 16 81117 81101 66.6728 47.6875 0.0213761 0.0149928 2023-10-09T10:50:37.068 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:37 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1080303433' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:37.159 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:50:37.159 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:50:37.159 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:50:37.159 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:50:37.159 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:50:37.159 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:50:37.187 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:50:37.200 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:37.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:37.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:37.229 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:50:37.235 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:50:37.235 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:50:37.235 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:50:37.235 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:50:37.235 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:50:37.235 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:50:37.235 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:50:37.235 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:50:37.272 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:37.315 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:37.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:37 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1080303433' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:37.361 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:37.450 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:37.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:37.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:37.468 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:37 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1080303433' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:37.557 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:37.609 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:37.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:37.663 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:37.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:37.764 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:37.837 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:37.935 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:37.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:37.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:38.002 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 77 16 81897 81881 66.4401 48.75 0.0115959 0.0150452 2023-10-09T10:50:38.041 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:38.139 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:38 smithi155 ceph-mon[184854]: pgmap v504: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 43 MiB/s wr, 685 op/s; 30700/238353 objects degraded (12.880%); 1958/238353 objects misplaced (0.821%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T10:50:38.139 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:38 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30700/233601 objects degraded (13.142%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:38.143 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:38.217 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:38.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:38.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:38.291 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:50:38.291 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:50:38.324 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:38.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:38 smithi125 ceph-mon[195507]: pgmap v504: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 43 MiB/s wr, 685 op/s; 30700/238353 objects degraded (12.880%); 1958/238353 objects misplaced (0.821%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T10:50:38.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:38 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30700/233601 objects degraded (13.142%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:38.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:38 smithi125 ceph-mon[200197]: pgmap v504: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 43 MiB/s wr, 685 op/s; 30700/238353 objects degraded (12.880%); 1958/238353 objects misplaced (0.821%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T10:50:38.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:38 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30700/233601 objects degraded (13.142%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:38.366 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:38.448 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:38.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:38.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:38.552 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:38.602 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:38.706 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:50:38.781 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:38.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:38.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:38.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:38.930 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:39.002 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 78 16 82644 82628 66.1868 46.6875 0.0265057 0.0151047 2023-10-09T10:50:39.062 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:39.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:39.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:39.168 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:39.177 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0] out_osds: [2, 7, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:50:39.178 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:50:39.178 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:50:39.178 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.352548 2023-10-09T10:50:39.178 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 0.3525476660843889 2023-10-09T10:50:39.190 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:50:39.253 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:39.349 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:39.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:39.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:39.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:39.519 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:39.550 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:39.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:39.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:39.601 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:39.656 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:39.732 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:39.863 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:39.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:39.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:39.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:40.004 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 79 16 83370 83354 65.9219 45.375 0.0225149 0.0151643 2023-10-09T10:50:40.034 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:40.069 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:40.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:40.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:40.171 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:40.211 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:40.312 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:50:40.433 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:50:40.434 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:50:40.438 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:40.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:40.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:40.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:40 smithi155 ceph-mon[184854]: pgmap v505: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 44 MiB/s wr, 707 op/s; 30700/241983 objects degraded (12.687%); 1958/241983 objects misplaced (0.809%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T10:50:40.463 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:40 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2618883438' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:40.463 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:40 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:40.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:40.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:40 smithi125 ceph-mon[195507]: pgmap v505: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 44 MiB/s wr, 707 op/s; 30700/241983 objects degraded (12.687%); 1958/241983 objects misplaced (0.809%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T10:50:40.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:40 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2618883438' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:40.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:40 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:40.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:40 smithi125 ceph-mon[200197]: pgmap v505: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 44 MiB/s wr, 707 op/s; 30700/241983 objects degraded (12.687%); 1958/241983 objects misplaced (0.809%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T10:50:40.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:40 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2618883438' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:40.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:40 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:40.622 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:40.814 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:50:40.820 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:50:40.852 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:40.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:40.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:40.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:41.008 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:50:41.006916+0000 min lat: 0.00131548 max lat: 2.28015 avg lat: 0.0152176 2023-10-09T10:50:41.008 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:50:41.008 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 80 16 84132 84116 65.6914 47.625 0.0258802 0.0152176 2023-10-09T10:50:41.038 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:41.084 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:41.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:41.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:41.194 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:41.225 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:50:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:50:41] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.43.0" 2023-10-09T10:50:41.278 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:50:41.284 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:41.287 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:41.320 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:41.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:41.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:41.433 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:41.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:41 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:41.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:41 smithi125 ceph-mon[195507]: failed to encode map e222 with expected crc 2023-10-09T10:50:41.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:41 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:50:41.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:41 smithi125 ceph-mon[195507]: osdmap e222: 8 total, 8 up, 5 in 2023-10-09T10:50:41.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:41 smithi125 ceph-mon[195507]: failed to encode map e222 with expected crc 2023-10-09T10:50:41.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:41 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:41.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:41 smithi125 ceph-mon[200197]: failed to encode map e222 with expected crc 2023-10-09T10:50:41.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:41 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:50:41.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:41 smithi125 ceph-mon[200197]: osdmap e222: 8 total, 8 up, 5 in 2023-10-09T10:50:41.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:41 smithi125 ceph-mon[200197]: failed to encode map e222 with expected crc 2023-10-09T10:50:41.508 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:41.535 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:41 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:41.535 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:41 smithi155 ceph-mon[184854]: failed to encode map e222 with expected crc 2023-10-09T10:50:41.535 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:41 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:50:41.535 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:41 smithi155 ceph-mon[184854]: osdmap e222: 8 total, 8 up, 5 in 2023-10-09T10:50:41.535 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:41 smithi155 ceph-mon[184854]: failed to encode map e222 with expected crc 2023-10-09T10:50:41.556 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:41.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:41.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:41.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:41.705 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:41.784 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:41.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:41.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:41.893 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:41.936 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:42.010 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 81 16 84881 84865 65.4553 46.8125 0.0145197 0.0152719 2023-10-09T10:50:42.015 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:42.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:42.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:42.119 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:42.210 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:42.300 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:42.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:42.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:42.362 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.4 primary-affinity to 0.352548 (8231042) 2023-10-09T10:50:42.403 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:42.455 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:42 smithi125 ceph-mon[195507]: pgmap v507: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 44 MiB/s wr, 702 op/s; 30700/247638 objects degraded (12.397%); 1958/247638 objects misplaced (0.791%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:50:42.455 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:42 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1140408271' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.3525476660843889}]: dispatch 2023-10-09T10:50:42.456 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:42 smithi125 ceph-mon[200197]: pgmap v507: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 44 MiB/s wr, 702 op/s; 30700/247638 objects degraded (12.397%); 1958/247638 objects misplaced (0.791%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:50:42.456 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:42 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1140408271' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.3525476660843889}]: dispatch 2023-10-09T10:50:42.459 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:42.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:42 smithi155 ceph-mon[184854]: pgmap v507: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 44 MiB/s wr, 702 op/s; 30700/247638 objects degraded (12.397%); 1958/247638 objects misplaced (0.791%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:50:42.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:42 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1140408271' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.3525476660843889}]: dispatch 2023-10-09T10:50:42.487 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:42.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:42.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:42.507 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:50:42.538 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:50:42.569 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:50:42.593 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:42.601 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:50:42.607 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:50:42.607 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:50:42.635 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:50:42.657 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:42.663 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:50:42.695 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:50:42.724 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:50:42.732 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:50:42.732 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:50:42.732 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:50:42.733 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:50:42.733 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:50:42.733 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:50:42.733 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:50:42.733 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:50:42.750 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:42.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:42.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:42.810 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:50:42.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:42.894 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:43.010 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 82 16 85449 85433 65.09 35.5 0.00520928 0.0152944 2023-10-09T10:50:43.066 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:43.067 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:50:43.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:43.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:43.108 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:43.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:43.214 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:43.283 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:43.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:43.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:43.353 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:50:43.371 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:43 smithi155 ceph-mon[184854]: failed to encode map e223 with expected crc 2023-10-09T10:50:43.371 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:43 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1140408271' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.3525476660843889}]': finished 2023-10-09T10:50:43.371 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:43 smithi155 ceph-mon[184854]: osdmap e223: 8 total, 8 up, 5 in 2023-10-09T10:50:43.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:43 smithi155 ceph-mon[184854]: failed to encode map e223 with expected crc 2023-10-09T10:50:43.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:43 smithi155 ceph-mon[184854]: failed to encode map e223 with expected crc 2023-10-09T10:50:43.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:43 smithi155 ceph-mon[184854]: failed to encode map e223 with expected crc 2023-10-09T10:50:43.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:43 smithi155 ceph-mon[184854]: failed to encode map e223 with expected crc 2023-10-09T10:50:43.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:43 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1665642997' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:43.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:43 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30700/247638 objects degraded (12.397%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:43.386 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:43.425 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:43.536 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:43.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:43.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:43.647 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:43.690 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[195507]: failed to encode map e223 with expected crc 2023-10-09T10:50:43.690 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1140408271' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.3525476660843889}]': finished 2023-10-09T10:50:43.691 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[195507]: osdmap e223: 8 total, 8 up, 5 in 2023-10-09T10:50:43.691 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[195507]: failed to encode map e223 with expected crc 2023-10-09T10:50:43.691 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[195507]: failed to encode map e223 with expected crc 2023-10-09T10:50:43.691 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[195507]: failed to encode map e223 with expected crc 2023-10-09T10:50:43.691 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[195507]: failed to encode map e223 with expected crc 2023-10-09T10:50:43.691 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1665642997' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:43.691 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30700/247638 objects degraded (12.397%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:43.691 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[200197]: failed to encode map e223 with expected crc 2023-10-09T10:50:43.691 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1140408271' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.3525476660843889}]': finished 2023-10-09T10:50:43.691 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[200197]: osdmap e223: 8 total, 8 up, 5 in 2023-10-09T10:50:43.692 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[200197]: failed to encode map e223 with expected crc 2023-10-09T10:50:43.692 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[200197]: failed to encode map e223 with expected crc 2023-10-09T10:50:43.692 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[200197]: failed to encode map e223 with expected crc 2023-10-09T10:50:43.692 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[200197]: failed to encode map e223 with expected crc 2023-10-09T10:50:43.692 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1665642997' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:43.692 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:43 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30700/247638 objects degraded (12.397%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:50:43.694 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:44.010 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 83 16 85929 85913 64.6674 30 0.0244557 0.0154579 2023-10-09T10:50:44.083 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:44.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:44.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:44.190 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:44.254 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:44.374 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[195507]: pgmap v509: 109 pgs: 3 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 51 MiB/s wr, 812 op/s; 30700/253119 objects degraded (12.129%); 1701/253119 objects misplaced (0.672%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T10:50:44.374 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[195507]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.374 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[195507]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[195507]: osdmap e224: 8 total, 8 up, 5 in 2023-10-09T10:50:44.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[195507]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[195507]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[195507]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[195507]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T10:50:44.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[200197]: pgmap v509: 109 pgs: 3 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 51 MiB/s wr, 812 op/s; 30700/253119 objects degraded (12.129%); 1701/253119 objects misplaced (0.672%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T10:50:44.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[200197]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[200197]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.376 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[200197]: osdmap e224: 8 total, 8 up, 5 in 2023-10-09T10:50:44.376 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[200197]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.376 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[200197]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.376 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[200197]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.376 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[200197]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.376 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:44 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T10:50:44.388 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:44.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:44.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:44.501 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:44.532 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:44 smithi155 ceph-mon[184854]: pgmap v509: 109 pgs: 3 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 51 MiB/s wr, 812 op/s; 30700/253119 objects degraded (12.129%); 1701/253119 objects misplaced (0.672%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T10:50:44.532 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:44 smithi155 ceph-mon[184854]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.532 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:44 smithi155 ceph-mon[184854]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.532 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:44 smithi155 ceph-mon[184854]: osdmap e224: 8 total, 8 up, 5 in 2023-10-09T10:50:44.532 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:44 smithi155 ceph-mon[184854]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.532 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:44 smithi155 ceph-mon[184854]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.532 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:44 smithi155 ceph-mon[184854]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.532 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:44 smithi155 ceph-mon[184854]: failed to encode map e224 with expected crc 2023-10-09T10:50:44.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:44 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T10:50:44.593 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:44.643 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:44.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:44.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:44.746 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:44.793 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:45.010 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 84 16 86727 86711 64.4913 49.875 0.0133118 0.0155005 2023-10-09T10:50:45.148 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:50:45.643 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:50:45.643 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:50:45.643 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:50:45.644 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:50:45.644 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.644 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70217 6.d 6:b8b4c26d:::benchmark_data_smithi125_244951_object70216:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:50:45.644 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.226550+0000", 2023-10-09T10:50:45.644 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 322.40694923400002, 2023-10-09T10:50:45.644 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.064685005000000004, 2023-10-09T10:50:45.644 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.644 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.644 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:45.644 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:45.644 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:45.644 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70217 2023-10-09T10:50:45.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.645 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.645 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226550+0000", 2023-10-09T10:50:45.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.645 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226550+0000", 2023-10-09T10:50:45.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226550+0000", 2023-10-09T10:50:45.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:45.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226583+0000", 2023-10-09T10:50:45.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2808000000000002e-05 2023-10-09T10:50:45.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.647 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226590+0000", 2023-10-09T10:50:45.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8509999999999999e-06 2023-10-09T10:50:45.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.647 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.647 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226595+0000", 2023-10-09T10:50:45.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6290000000000001e-06 2023-10-09T10:50:45.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.647 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.647 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226637+0000", 2023-10-09T10:50:45.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1734999999999999e-05 2023-10-09T10:50:45.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226771+0000", 2023-10-09T10:50:45.648 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013447000000000001 2023-10-09T10:50:45.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:50:45.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.226923+0000", 2023-10-09T10:50:45.648 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151977 2023-10-09T10:50:45.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:45.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.229528+0000", 2023-10-09T10:50:45.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026052530000000001 2023-10-09T10:50:45.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.649 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.241565+0000", 2023-10-09T10:50:45.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012036467 2023-10-09T10:50:45.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.649 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.288660+0000", 2023-10-09T10:50:45.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047095342999999998 2023-10-09T10:50:45.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.650 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.650 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.288720+0000", 2023-10-09T10:50:45.650 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0025000000000002e-05 2023-10-09T10:50:45.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.650 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.650 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.291235+0000", 2023-10-09T10:50:45.650 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0025149249999999999 2023-10-09T10:50:45.650 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.650 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.651 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.651 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.651 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.651 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70220 6.d 6:bb7bde6c:::benchmark_data_smithi125_244951_object70219:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:50:45.651 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.231764+0000", 2023-10-09T10:50:45.651 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 322.40173491100001, 2023-10-09T10:50:45.651 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.061391124999999998, 2023-10-09T10:50:45.651 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.651 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.651 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:45.651 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:45.651 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:45.652 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70220 2023-10-09T10:50:45.652 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.652 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.652 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.652 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.652 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231764+0000", 2023-10-09T10:50:45.652 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.652 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.652 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.652 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.652 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231764+0000", 2023-10-09T10:50:45.652 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.653 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.653 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231763+0000", 2023-10-09T10:50:45.653 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:50:45.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.653 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.653 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231801+0000", 2023-10-09T10:50:45.653 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7691999999999998e-05 2023-10-09T10:50:45.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231812+0000", 2023-10-09T10:50:45.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0569e-05 2023-10-09T10:50:45.654 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231817+0000", 2023-10-09T10:50:45.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0490000000000003e-06 2023-10-09T10:50:45.654 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231856+0000", 2023-10-09T10:50:45.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9002999999999999e-05 2023-10-09T10:50:45.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.655 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.655 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.231960+0000", 2023-10-09T10:50:45.655 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010419100000000001 2023-10-09T10:50:45.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.655 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:50:45.655 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.232064+0000", 2023-10-09T10:50:45.655 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010409 2023-10-09T10:50:45.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:45.656 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.234485+0000", 2023-10-09T10:50:45.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024210680000000002 2023-10-09T10:50:45.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.656 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.251746+0000", 2023-10-09T10:50:45.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017260707 2023-10-09T10:50:45.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.656 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.291288+0000", 2023-10-09T10:50:45.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039542425999999999 2023-10-09T10:50:45.657 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.657 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.657 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.291380+0000", 2023-10-09T10:50:45.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1514999999999994e-05 2023-10-09T10:50:45.657 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.657 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.657 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.293156+0000", 2023-10-09T10:50:45.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001775878 2023-10-09T10:50:45.657 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.657 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.657 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.658 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106432 6.a 6:50fa9909:::benchmark_data_smithi125_244951_object106431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:50:45.658 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.249049+0000", 2023-10-09T10:50:45.658 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 283.38445061099998, 2023-10-09T10:50:45.658 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.49699488600000002, 2023-10-09T10:50:45.658 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.658 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T10:50:45.658 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:45.658 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:45.658 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:45.659 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106432 2023-10-09T10:50:45.659 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.659 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.659 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249049+0000", 2023-10-09T10:50:45.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.659 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249049+0000", 2023-10-09T10:50:45.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.660 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249047+0000", 2023-10-09T10:50:45.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:50:45.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.660 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249097+0000", 2023-10-09T10:50:45.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0584000000000002e-05 2023-10-09T10:50:45.661 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.661 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.661 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249117+0000", 2023-10-09T10:50:45.661 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9763999999999999e-05 2023-10-09T10:50:45.661 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.661 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.661 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249125+0000", 2023-10-09T10:50:45.661 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6040000000000004e-06 2023-10-09T10:50:45.661 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.662 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.662 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249189+0000", 2023-10-09T10:50:45.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3779e-05 2023-10-09T10:50:45.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.662 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.662 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.662 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249335+0000", 2023-10-09T10:50:45.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014603399999999999 2023-10-09T10:50:45.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.662 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.663 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:50:45.663 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249521+0000", 2023-10-09T10:50:45.663 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018619099999999999 2023-10-09T10:50:45.663 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.663 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.663 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.253194+0000", 2023-10-09T10:50:45.663 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003673103 2023-10-09T10:50:45.663 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.664 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:45.664 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.253298+0000", 2023-10-09T10:50:45.664 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103828 2023-10-09T10:50:45.664 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.664 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.664 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.664 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.271832+0000", 2023-10-09T10:50:45.664 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018534658999999998 2023-10-09T10:50:45.664 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.664 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.664 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:50:45.665 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.271843+0000", 2023-10-09T10:50:45.665 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0995000000000001e-05 2023-10-09T10:50:45.665 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.665 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.665 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.665 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.745982+0000", 2023-10-09T10:50:45.665 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.47413843 2023-10-09T10:50:45.665 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.665 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.665 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.665 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746044+0000", 2023-10-09T10:50:45.665 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1832e-05 2023-10-09T10:50:45.665 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.666 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.666 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.666 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106439 6.a 6:5c3c51c3:::benchmark_data_smithi125_244951_object106438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-10-09T10:50:45.666 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.284123+0000", 2023-10-09T10:50:45.666 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 283.34937640599998, 2023-10-09T10:50:45.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.47113701699999999, 2023-10-09T10:50:45.666 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.666 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.666 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:45.666 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:45.666 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:45.667 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106439 2023-10-09T10:50:45.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.667 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284123+0000", 2023-10-09T10:50:45.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284123+0000", 2023-10-09T10:50:45.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284122+0000", 2023-10-09T10:50:45.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:45.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284170+0000", 2023-10-09T10:50:45.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7929999999999997e-05 2023-10-09T10:50:45.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284181+0000", 2023-10-09T10:50:45.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1292e-05 2023-10-09T10:50:45.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284186+0000", 2023-10-09T10:50:45.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1089999999999997e-06 2023-10-09T10:50:45.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284266+0000", 2023-10-09T10:50:45.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0311999999999999e-05 2023-10-09T10:50:45.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:50:45.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284287+0000", 2023-10-09T10:50:45.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0704e-05 2023-10-09T10:50:45.670 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746068+0000", 2023-10-09T10:50:45.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.46178057300000003 2023-10-09T10:50:45.670 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746295+0000", 2023-10-09T10:50:45.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000227335 2023-10-09T10:50:45.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.671 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.671 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:50:45.671 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746417+0000", 2023-10-09T10:50:45.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000121816 2023-10-09T10:50:45.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.671 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.671 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:45.671 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.752322+0000", 2023-10-09T10:50:45.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059055230000000002 2023-10-09T10:50:45.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.672 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.672 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754107+0000", 2023-10-09T10:50:45.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0017852149999999999 2023-10-09T10:50:45.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.672 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.672 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754378+0000", 2023-10-09T10:50:45.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000270547 2023-10-09T10:50:45.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.672 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754422+0000", 2023-10-09T10:50:45.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4063000000000001e-05 2023-10-09T10:50:45.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.755260+0000", 2023-10-09T10:50:45.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00083801000000000004 2023-10-09T10:50:45.673 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.673 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.673 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.674 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106440 6.a 6:5e7f7caf:::benchmark_data_smithi125_244951_object106439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-10-09T10:50:45.674 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.284197+0000", 2023-10-09T10:50:45.674 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 283.349302878, 2023-10-09T10:50:45.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.47186083699999998, 2023-10-09T10:50:45.674 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.674 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.674 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:45.674 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:45.674 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:45.674 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106440 2023-10-09T10:50:45.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.675 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284197+0000", 2023-10-09T10:50:45.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284197+0000", 2023-10-09T10:50:45.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284196+0000", 2023-10-09T10:50:45.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:45.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284284+0000", 2023-10-09T10:50:45.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7646999999999996e-05 2023-10-09T10:50:45.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284292+0000", 2023-10-09T10:50:45.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2689999999999999e-06 2023-10-09T10:50:45.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284297+0000", 2023-10-09T10:50:45.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4419999999999997e-06 2023-10-09T10:50:45.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.679 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.679 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284344+0000", 2023-10-09T10:50:45.679 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6940000000000001e-05 2023-10-09T10:50:45.679 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.679 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:50:45.679 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284352+0000", 2023-10-09T10:50:45.679 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0749999999999998e-06 2023-10-09T10:50:45.679 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.679 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.680 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.749219+0000", 2023-10-09T10:50:45.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.46486643799999999 2023-10-09T10:50:45.680 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.680 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.680 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.680 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.749378+0000", 2023-10-09T10:50:45.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015880000000000001 2023-10-09T10:50:45.680 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.680 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.680 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:50:45.680 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.749563+0000", 2023-10-09T10:50:45.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018555399999999999 2023-10-09T10:50:45.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.681 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754147+0000", 2023-10-09T10:50:45.681 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045842050000000001 2023-10-09T10:50:45.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:45.681 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754155+0000", 2023-10-09T10:50:45.681 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2080000000000003e-06 2023-10-09T10:50:45.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.682 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.682 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.755321+0000", 2023-10-09T10:50:45.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011663839999999999 2023-10-09T10:50:45.682 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.682 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.682 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.755360+0000", 2023-10-09T10:50:45.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8943000000000003e-05 2023-10-09T10:50:45.682 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.682 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.682 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.683 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.756057+0000", 2023-10-09T10:50:45.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00069742899999999995 2023-10-09T10:50:45.683 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.683 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.683 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.683 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.683 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:152994 6.7 6:e184a006:::benchmark_data_smithi125_244951_object46551:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:50:45.683 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:35.519171+0000", 2023-10-09T10:50:45.683 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 190.11432843599999, 2023-10-09T10:50:45.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.084211253, 2023-10-09T10:50:45.683 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.684 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.684 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:45.684 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:45.684 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:45.684 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 152994 2023-10-09T10:50:45.684 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.684 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.684 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.684 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.684 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519171+0000", 2023-10-09T10:50:45.684 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.684 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.685 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.685 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.685 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519171+0000", 2023-10-09T10:50:45.685 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.685 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.685 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.685 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.685 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519170+0000", 2023-10-09T10:50:45.685 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:45.685 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.685 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.685 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.685 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519181+0000", 2023-10-09T10:50:45.685 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1884e-05 2023-10-09T10:50:45.686 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.686 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.686 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.686 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519196+0000", 2023-10-09T10:50:45.686 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.448e-05 2023-10-09T10:50:45.686 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.686 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.686 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.686 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519202+0000", 2023-10-09T10:50:45.686 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3670000000000002e-06 2023-10-09T10:50:45.686 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.686 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.686 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.686 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.522571+0000", 2023-10-09T10:50:45.687 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0033686639999999999 2023-10-09T10:50:45.687 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.687 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.687 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.687 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.522712+0000", 2023-10-09T10:50:45.687 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014130800000000001 2023-10-09T10:50:45.687 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.687 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.687 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:50:45.687 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.522828+0000", 2023-10-09T10:50:45.687 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115808 2023-10-09T10:50:45.687 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.687 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.688 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:45.688 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.527082+0000", 2023-10-09T10:50:45.688 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042537520000000004 2023-10-09T10:50:45.688 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.688 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.688 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.688 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.528250+0000", 2023-10-09T10:50:45.688 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011679310000000001 2023-10-09T10:50:45.688 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.688 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.688 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.688 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.603293+0000", 2023-10-09T10:50:45.688 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.075043377999999994 2023-10-09T10:50:45.689 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.689 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.689 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.689 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.603336+0000", 2023-10-09T10:50:45.689 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2774000000000001e-05 2023-10-09T10:50:45.689 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.689 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.689 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.689 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.603382+0000", 2023-10-09T10:50:45.689 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6279999999999997e-05 2023-10-09T10:50:45.689 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.689 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.689 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.689 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.690 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.690 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:153733 6.d 6:b575b55b:::benchmark_data_smithi125_244951_object47290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:50:45.690 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:36.588580+0000", 2023-10-09T10:50:45.690 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 189.04491956000001, 2023-10-09T10:50:45.690 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.071524196999999998, 2023-10-09T10:50:45.690 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.690 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.690 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:45.690 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:45.690 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:45.690 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 153733 2023-10-09T10:50:45.690 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.691 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.691 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.691 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.691 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588580+0000", 2023-10-09T10:50:45.691 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.691 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.691 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.691 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.691 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588580+0000", 2023-10-09T10:50:45.691 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.691 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.692 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.692 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.692 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588578+0000", 2023-10-09T10:50:45.692 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:45.692 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.692 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.692 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.692 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588585+0000", 2023-10-09T10:50:45.692 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.139e-06 2023-10-09T10:50:45.692 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.692 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.692 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.692 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588596+0000", 2023-10-09T10:50:45.693 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.163e-05 2023-10-09T10:50:45.693 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.693 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.693 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.693 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.588602+0000", 2023-10-09T10:50:45.693 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6869999999999999e-06 2023-10-09T10:50:45.693 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.693 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.693 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.693 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.595518+0000", 2023-10-09T10:50:45.693 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0069160599999999999 2023-10-09T10:50:45.693 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.694 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.694 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.694 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.595663+0000", 2023-10-09T10:50:45.694 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014470299999999999 2023-10-09T10:50:45.694 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.694 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.694 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:50:45.694 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.595807+0000", 2023-10-09T10:50:45.694 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014449800000000001 2023-10-09T10:50:45.694 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.694 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.694 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:45.695 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.600067+0000", 2023-10-09T10:50:45.695 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042602200000000003 2023-10-09T10:50:45.695 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.695 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.695 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.695 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.641151+0000", 2023-10-09T10:50:45.695 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.041083736000000003 2023-10-09T10:50:45.695 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.695 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.695 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.695 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.657793+0000", 2023-10-09T10:50:45.695 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016641420000000001 2023-10-09T10:50:45.695 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.696 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.696 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.696 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.657848+0000", 2023-10-09T10:50:45.696 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5611e-05 2023-10-09T10:50:45.696 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.696 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.696 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.696 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:36.660104+0000", 2023-10-09T10:50:45.696 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022558880000000002 2023-10-09T10:50:45.696 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.696 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.696 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.697 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.697 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.697 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:162477 6.a 6:52278cc7:::benchmark_data_smithi125_244951_object56034:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-10-09T10:50:45.697 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:49.307801+0000", 2023-10-09T10:50:45.697 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 176.32569840299999, 2023-10-09T10:50:45.697 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.061925675999999999, 2023-10-09T10:50:45.697 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.697 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.697 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:45.697 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:45.697 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:45.697 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162477 2023-10-09T10:50:45.698 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.698 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.698 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.698 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.698 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.307801+0000", 2023-10-09T10:50:45.698 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.698 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.698 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.698 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.698 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.307801+0000", 2023-10-09T10:50:45.698 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.698 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.699 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.699 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.699 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.307801+0000", 2023-10-09T10:50:45.699 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967296 2023-10-09T10:50:45.699 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.699 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.699 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.699 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.307802+0000", 2023-10-09T10:50:45.699 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.579e-06 2023-10-09T10:50:45.699 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.699 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.699 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.699 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.307805+0000", 2023-10-09T10:50:45.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8059999999999999e-06 2023-10-09T10:50:45.700 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.700 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.307807+0000", 2023-10-09T10:50:45.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2359999999999999e-06 2023-10-09T10:50:45.700 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.700 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.328037+0000", 2023-10-09T10:50:45.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020229737000000001 2023-10-09T10:50:45.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.701 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.701 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.701 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.328177+0000", 2023-10-09T10:50:45.701 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013998099999999999 2023-10-09T10:50:45.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.701 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.701 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:50:45.701 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.328323+0000", 2023-10-09T10:50:45.701 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000146024 2023-10-09T10:50:45.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.701 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.701 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:45.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.344032+0000", 2023-10-09T10:50:45.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015708751999999999 2023-10-09T10:50:45.702 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.356007+0000", 2023-10-09T10:50:45.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011975507 2023-10-09T10:50:45.702 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.369010+0000", 2023-10-09T10:50:45.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013002796 2023-10-09T10:50:45.702 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.369069+0000", 2023-10-09T10:50:45.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8711000000000001e-05 2023-10-09T10:50:45.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.369727+0000", 2023-10-09T10:50:45.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00065775299999999996 2023-10-09T10:50:45.703 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.703 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.704 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.704 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.704 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.704 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:162483 6.a 6:5efc2291:::benchmark_data_smithi125_244951_object56040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-10-09T10:50:45.704 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:49.308048+0000", 2023-10-09T10:50:45.704 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 176.325451883, 2023-10-09T10:50:45.704 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.062044714000000001, 2023-10-09T10:50:45.704 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.704 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.704 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:45.704 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:45.705 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:45.705 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162483 2023-10-09T10:50:45.705 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.705 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.705 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.705 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.308048+0000", 2023-10-09T10:50:45.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.705 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.705 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.705 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.308048+0000", 2023-10-09T10:50:45.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.706 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.706 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.706 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.308047+0000", 2023-10-09T10:50:45.706 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:50:45.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.706 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.706 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.706 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.308052+0000", 2023-10-09T10:50:45.706 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7339999999999996e-06 2023-10-09T10:50:45.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.706 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.707 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.707 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.308065+0000", 2023-10-09T10:50:45.707 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2945000000000001e-05 2023-10-09T10:50:45.707 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.707 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.707 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.707 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.308072+0000", 2023-10-09T10:50:45.707 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.002e-06 2023-10-09T10:50:45.707 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.707 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.707 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.333025+0000", 2023-10-09T10:50:45.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024952407999999999 2023-10-09T10:50:45.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.708 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.708 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.333161+0000", 2023-10-09T10:50:45.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000135928 2023-10-09T10:50:45.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.708 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.708 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:50:45.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.333302+0000", 2023-10-09T10:50:45.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014128599999999999 2023-10-09T10:50:45.709 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:45.709 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.349518+0000", 2023-10-09T10:50:45.709 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016216147 2023-10-09T10:50:45.709 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.709 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.356145+0000", 2023-10-09T10:50:45.709 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.006627099 2023-10-09T10:50:45.709 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.710 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.369763+0000", 2023-10-09T10:50:45.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013617895 2023-10-09T10:50:45.710 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.710 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.710 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.710 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.369805+0000", 2023-10-09T10:50:45.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2206999999999998e-05 2023-10-09T10:50:45.710 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.710 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.710 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.711 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.370092+0000", 2023-10-09T10:50:45.711 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00028702799999999999 2023-10-09T10:50:45.711 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.711 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.711 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.711 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.711 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.711 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:190474 6.e e184/154 6:73a148c1:::benchmark_data_smithi125_244951_object84031:head v 184'11796, mlcod=104'1056)", 2023-10-09T10:50:45.711 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:30.473779+0000", 2023-10-09T10:50:45.711 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 135.15972020500001, 2023-10-09T10:50:45.711 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.065784969999999998, 2023-10-09T10:50:45.712 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.712 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.712 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.712 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.712 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.473779+0000", 2023-10-09T10:50:45.712 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.712 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.712 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.712 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.473779+0000", 2023-10-09T10:50:45.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.713 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.713 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.713 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.713 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.473778+0000", 2023-10-09T10:50:45.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:45.713 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.713 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.713 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.713 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.473789+0000", 2023-10-09T10:50:45.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0920999999999999e-05 2023-10-09T10:50:45.713 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.714 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.714 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.714 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.473803+0000", 2023-10-09T10:50:45.714 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4552999999999999e-05 2023-10-09T10:50:45.714 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.714 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.714 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.714 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.473810+0000", 2023-10-09T10:50:45.714 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5699999999999998e-06 2023-10-09T10:50:45.714 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.714 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.483312+0000", 2023-10-09T10:50:45.715 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0095017360000000002 2023-10-09T10:50:45.715 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.483337+0000", 2023-10-09T10:50:45.715 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5369000000000001e-05 2023-10-09T10:50:45.715 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.539532+0000", 2023-10-09T10:50:45.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.056195334 2023-10-09T10:50:45.716 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.716 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:30.539564+0000", 2023-10-09T10:50:45.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1739999999999998e-05 2023-10-09T10:50:45.716 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.716 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.716 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.716 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.716 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.716 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:191509 6.d 6:b42a920c:::benchmark_data_smithi125_244951_object85066:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2023-10-09T10:50:45.717 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:32.051748+0000", 2023-10-09T10:50:45.717 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 133.58175187399999, 2023-10-09T10:50:45.717 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.064273379000000005, 2023-10-09T10:50:45.717 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.717 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.717 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:45.717 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:45.717 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:45.717 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 191509 2023-10-09T10:50:45.717 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.717 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.717 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.717 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.718 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051748+0000", 2023-10-09T10:50:45.718 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.718 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.718 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.718 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.718 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051748+0000", 2023-10-09T10:50:45.718 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.718 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.718 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.718 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051747+0000", 2023-10-09T10:50:45.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:45.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.719 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.719 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051750+0000", 2023-10-09T10:50:45.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.402e-06 2023-10-09T10:50:45.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.719 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.719 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051755+0000", 2023-10-09T10:50:45.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0089999999999998e-06 2023-10-09T10:50:45.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.720 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051757+0000", 2023-10-09T10:50:45.720 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9179999999999998e-06 2023-10-09T10:50:45.720 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.720 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051798+0000", 2023-10-09T10:50:45.720 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0911000000000002e-05 2023-10-09T10:50:45.720 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.721 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.051942+0000", 2023-10-09T10:50:45.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014350699999999999 2023-10-09T10:50:45.721 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.721 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.721 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T10:50:45.721 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.052082+0000", 2023-10-09T10:50:45.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014008 2023-10-09T10:50:45.721 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.721 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.721 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:45.721 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.057385+0000", 2023-10-09T10:50:45.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0053033250000000002 2023-10-09T10:50:45.721 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.722 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.722 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.722 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.082093+0000", 2023-10-09T10:50:45.722 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024707959000000002 2023-10-09T10:50:45.722 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.722 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.722 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.722 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.114170+0000", 2023-10-09T10:50:45.722 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.032076503999999999 2023-10-09T10:50:45.722 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.722 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.722 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.114226+0000", 2023-10-09T10:50:45.723 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6323000000000003e-05 2023-10-09T10:50:45.723 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.723 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.723 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:32.116021+0000", 2023-10-09T10:50:45.723 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001794805 2023-10-09T10:50:45.723 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.723 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.723 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.723 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.723 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.724 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:198775 6.e e186/154 6:70ffdb86:::benchmark_data_smithi125_244951_object92332:head v 186'12373, mlcod=104'1056)", 2023-10-09T10:50:45.724 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:43.027484+0000", 2023-10-09T10:50:45.724 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 122.606015668, 2023-10-09T10:50:45.724 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.066535034000000007, 2023-10-09T10:50:45.724 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.724 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.724 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.724 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.724 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.724 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.027484+0000", 2023-10-09T10:50:45.724 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.724 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.725 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.725 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.725 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.027484+0000", 2023-10-09T10:50:45.725 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.725 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.725 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.725 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.725 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.027483+0000", 2023-10-09T10:50:45.725 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:50:45.725 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.725 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.726 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.726 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.027493+0000", 2023-10-09T10:50:45.726 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0549e-05 2023-10-09T10:50:45.726 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.726 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.726 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.726 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.027509+0000", 2023-10-09T10:50:45.726 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6130999999999999e-05 2023-10-09T10:50:45.726 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.726 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.726 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.726 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.027518+0000", 2023-10-09T10:50:45.727 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3780000000000001e-06 2023-10-09T10:50:45.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.727 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.727 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.031517+0000", 2023-10-09T10:50:45.727 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003999379 2023-10-09T10:50:45.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.727 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.727 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.031540+0000", 2023-10-09T10:50:45.727 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2920000000000001e-05 2023-10-09T10:50:45.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.093982+0000", 2023-10-09T10:50:45.728 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.062441607000000003 2023-10-09T10:50:45.728 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.728 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:43.094019+0000", 2023-10-09T10:50:45.728 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7156999999999997e-05 2023-10-09T10:50:45.728 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.728 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.728 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.728 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.729 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.729 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:211949 6.e e189/154 6:700b4d98:::benchmark_data_smithi125_244951_object105506:head v 189'13175, mlcod=104'1056)", 2023-10-09T10:50:45.729 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:01.372020+0000", 2023-10-09T10:50:45.729 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 104.261479721, 2023-10-09T10:50:45.729 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.065605420999999997, 2023-10-09T10:50:45.729 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.729 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.729 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.729 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.729 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.729 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.372020+0000", 2023-10-09T10:50:45.729 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.729 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.730 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.730 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.372020+0000", 2023-10-09T10:50:45.730 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.730 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.730 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.730 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.372019+0000", 2023-10-09T10:50:45.730 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:45.730 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.730 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.731 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.372024+0000", 2023-10-09T10:50:45.731 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8319999999999996e-06 2023-10-09T10:50:45.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.731 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.731 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.731 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.372033+0000", 2023-10-09T10:50:45.731 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5890000000000003e-06 2023-10-09T10:50:45.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.731 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.731 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.731 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.372038+0000", 2023-10-09T10:50:45.731 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9479999999999999e-06 2023-10-09T10:50:45.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.401091+0000", 2023-10-09T10:50:45.732 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029053241 2023-10-09T10:50:45.732 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.401121+0000", 2023-10-09T10:50:45.732 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0014000000000001e-05 2023-10-09T10:50:45.732 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.437580+0000", 2023-10-09T10:50:45.733 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.036459566999999998 2023-10-09T10:50:45.733 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.733 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.733 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.733 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.437625+0000", 2023-10-09T10:50:45.733 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4719999999999999e-05 2023-10-09T10:50:45.733 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.733 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.733 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.733 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.733 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.733 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:211957 6.e e189/154 6:74f44ea0:::benchmark_data_smithi125_244951_object105514:head v 189'13176, mlcod=104'1056)", 2023-10-09T10:50:45.734 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:01.376382+0000", 2023-10-09T10:50:45.734 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 104.25711769599999, 2023-10-09T10:50:45.734 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.06126479, 2023-10-09T10:50:45.734 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.734 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.734 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.734 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.734 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.734 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.376382+0000", 2023-10-09T10:50:45.734 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.734 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.734 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.734 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.376382+0000", 2023-10-09T10:50:45.735 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.735 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.735 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.376381+0000", 2023-10-09T10:50:45.735 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:45.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.735 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.735 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.376386+0000", 2023-10-09T10:50:45.735 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5369999999999999e-06 2023-10-09T10:50:45.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.736 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.736 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.376393+0000", 2023-10-09T10:50:45.736 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.345e-06 2023-10-09T10:50:45.736 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.736 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.736 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.376398+0000", 2023-10-09T10:50:45.736 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8300000000000003e-06 2023-10-09T10:50:45.736 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.736 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.736 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.410955+0000", 2023-10-09T10:50:45.737 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.034556849000000001 2023-10-09T10:50:45.737 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.737 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.737 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.737 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.410984+0000", 2023-10-09T10:50:45.737 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9366e-05 2023-10-09T10:50:45.737 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.737 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.737 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.737 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.437629+0000", 2023-10-09T10:50:45.737 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026644404999999999 2023-10-09T10:50:45.737 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.737 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.738 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.738 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.437647+0000", 2023-10-09T10:50:45.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7876e-05 2023-10-09T10:50:45.738 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.738 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.738 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.738 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.738 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.738 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:212017 6.e e189/154 6:72d7ee3b:::benchmark_data_smithi125_244951_object105574:head v 189'13182, mlcod=104'1056)", 2023-10-09T10:50:45.738 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:01.461617+0000", 2023-10-09T10:50:45.738 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 104.17188263, 2023-10-09T10:50:45.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069220272999999999, 2023-10-09T10:50:45.739 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.739 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.739 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.739 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.739 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.739 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.461617+0000", 2023-10-09T10:50:45.739 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.739 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.739 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.739 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.739 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.461617+0000", 2023-10-09T10:50:45.739 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.740 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.740 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.740 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.740 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.461616+0000", 2023-10-09T10:50:45.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:45.740 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.740 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.740 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.740 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.461622+0000", 2023-10-09T10:50:45.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3009999999999997e-06 2023-10-09T10:50:45.740 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.740 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.741 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.741 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.461626+0000", 2023-10-09T10:50:45.741 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2520000000000001e-06 2023-10-09T10:50:45.741 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.741 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.741 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.741 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.461629+0000", 2023-10-09T10:50:45.741 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6579999999999998e-06 2023-10-09T10:50:45.741 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.741 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.741 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.741 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.467974+0000", 2023-10-09T10:50:45.741 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0063450989999999999 2023-10-09T10:50:45.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.742 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.742 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.742 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.467993+0000", 2023-10-09T10:50:45.742 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9394000000000001e-05 2023-10-09T10:50:45.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.742 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.742 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.742 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.530795+0000", 2023-10-09T10:50:45.742 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.062801642000000005 2023-10-09T10:50:45.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.743 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.743 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.743 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.530837+0000", 2023-10-09T10:50:45.743 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2338e-05 2023-10-09T10:50:45.743 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.743 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.743 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.743 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.743 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.743 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:21936 7.0 7:015131e0:::benchmark_data_smithi125_295816_object21935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T10:50:45.743 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:29.765153+0000", 2023-10-09T10:50:45.743 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 75.868346505999995, 2023-10-09T10:50:45.743 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.087158461000000007, 2023-10-09T10:50:45.744 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.744 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.744 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:45.744 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:50:45.744 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:50:45.744 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 21936 2023-10-09T10:50:45.744 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.744 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.744 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.744 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.744 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765153+0000", 2023-10-09T10:50:45.744 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.744 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.745 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765153+0000", 2023-10-09T10:50:45.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.745 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765152+0000", 2023-10-09T10:50:45.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:45.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.745 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765187+0000", 2023-10-09T10:50:45.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4075999999999997e-05 2023-10-09T10:50:45.746 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.746 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765194+0000", 2023-10-09T10:50:45.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7230000000000004e-06 2023-10-09T10:50:45.746 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.746 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765199+0000", 2023-10-09T10:50:45.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7879999999999997e-06 2023-10-09T10:50:45.746 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.747 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.747 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765234+0000", 2023-10-09T10:50:45.747 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4699000000000001e-05 2023-10-09T10:50:45.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.747 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.747 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765367+0000", 2023-10-09T10:50:45.747 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013368399999999999 2023-10-09T10:50:45.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.747 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:50:45.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765505+0000", 2023-10-09T10:50:45.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013755399999999999 2023-10-09T10:50:45.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.748 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.748 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:45.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.766674+0000", 2023-10-09T10:50:45.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011688899999999999 2023-10-09T10:50:45.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.748 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.748 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.771590+0000", 2023-10-09T10:50:45.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0049158149999999996 2023-10-09T10:50:45.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852204+0000", 2023-10-09T10:50:45.749 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.080614665000000002 2023-10-09T10:50:45.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852265+0000", 2023-10-09T10:50:45.749 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0838999999999998e-05 2023-10-09T10:50:45.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852311+0000", 2023-10-09T10:50:45.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6202000000000003e-05 2023-10-09T10:50:45.750 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.750 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.750 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.750 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.750 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.750 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:21948 7.0 7:0b124c27:::benchmark_data_smithi125_295816_object21947:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T10:50:45.750 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:29.773164+0000", 2023-10-09T10:50:45.750 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 75.860335008999996, 2023-10-09T10:50:45.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.079256148999999998, 2023-10-09T10:50:45.750 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.750 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.751 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:45.751 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:50:45.751 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:50:45.751 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 21948 2023-10-09T10:50:45.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.751 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773164+0000", 2023-10-09T10:50:45.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773164+0000", 2023-10-09T10:50:45.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773163+0000", 2023-10-09T10:50:45.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:50:45.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773231+0000", 2023-10-09T10:50:45.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7830000000000006e-05 2023-10-09T10:50:45.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773246+0000", 2023-10-09T10:50:45.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4868000000000001e-05 2023-10-09T10:50:45.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773253+0000", 2023-10-09T10:50:45.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6479999999999997e-06 2023-10-09T10:50:45.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773296+0000", 2023-10-09T10:50:45.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3763000000000001e-05 2023-10-09T10:50:45.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.754 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773431+0000", 2023-10-09T10:50:45.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013489400000000001 2023-10-09T10:50:45.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:50:45.754 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773595+0000", 2023-10-09T10:50:45.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000163544 2023-10-09T10:50:45.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.755 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:45.755 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.774460+0000", 2023-10-09T10:50:45.755 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00086458399999999995 2023-10-09T10:50:45.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.755 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.755 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.783614+0000", 2023-10-09T10:50:45.755 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0091545570000000007 2023-10-09T10:50:45.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.755 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852352+0000", 2023-10-09T10:50:45.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.068737856 2023-10-09T10:50:45.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852383+0000", 2023-10-09T10:50:45.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1560000000000003e-05 2023-10-09T10:50:45.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852421+0000", 2023-10-09T10:50:45.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7061999999999999e-05 2023-10-09T10:50:45.757 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.757 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.757 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.757 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.757 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:21952 7.0 7:0825f8d2:::benchmark_data_smithi125_295816_object21951:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T10:50:45.757 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:29.778597+0000", 2023-10-09T10:50:45.757 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 75.854902835000004, 2023-10-09T10:50:45.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.073959639999999993, 2023-10-09T10:50:45.757 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.757 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.758 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:45.758 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:50:45.758 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:50:45.758 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 21952 2023-10-09T10:50:45.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.758 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778597+0000", 2023-10-09T10:50:45.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778597+0000", 2023-10-09T10:50:45.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778595+0000", 2023-10-09T10:50:45.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:45.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778636+0000", 2023-10-09T10:50:45.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1343e-05 2023-10-09T10:50:45.760 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.760 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.760 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.760 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778647+0000", 2023-10-09T10:50:45.760 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0971999999999999e-05 2023-10-09T10:50:45.760 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.760 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.760 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.760 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778653+0000", 2023-10-09T10:50:45.760 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5509999999999999e-06 2023-10-09T10:50:45.760 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.760 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.760 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.761 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778701+0000", 2023-10-09T10:50:45.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8219e-05 2023-10-09T10:50:45.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.761 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.761 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.761 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778789+0000", 2023-10-09T10:50:45.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7466999999999995e-05 2023-10-09T10:50:45.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.761 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.761 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:50:45.761 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778920+0000", 2023-10-09T10:50:45.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013081 2023-10-09T10:50:45.762 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:45.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.779741+0000", 2023-10-09T10:50:45.762 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00082173999999999995 2023-10-09T10:50:45.762 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.783915+0000", 2023-10-09T10:50:45.762 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041741879999999997 2023-10-09T10:50:45.762 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.763 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.763 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852486+0000", 2023-10-09T10:50:45.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.068570476000000005 2023-10-09T10:50:45.763 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.763 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.763 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.763 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852533+0000", 2023-10-09T10:50:45.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7379000000000002e-05 2023-10-09T10:50:45.763 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.763 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.763 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.763 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852556+0000", 2023-10-09T10:50:45.764 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2918999999999999e-05 2023-10-09T10:50:45.764 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.764 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.764 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.764 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.764 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.764 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:21963 7.0 7:03b87752:::benchmark_data_smithi125_295816_object21962:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T10:50:45.764 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:29.784939+0000", 2023-10-09T10:50:45.764 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 75.848560434000007, 2023-10-09T10:50:45.764 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.067721730999999993, 2023-10-09T10:50:45.764 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.764 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.764 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:45.765 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:50:45.765 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:50:45.765 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 21963 2023-10-09T10:50:45.765 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.765 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.765 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.765 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.765 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.784939+0000", 2023-10-09T10:50:45.765 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.765 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.765 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.765 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.765 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.784939+0000", 2023-10-09T10:50:45.766 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.766 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.766 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.766 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.766 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.784938+0000", 2023-10-09T10:50:45.766 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:50:45.766 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.766 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.766 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.766 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.785013+0000", 2023-10-09T10:50:45.766 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4683000000000003e-05 2023-10-09T10:50:45.766 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.766 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.767 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.767 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.785024+0000", 2023-10-09T10:50:45.767 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1711e-05 2023-10-09T10:50:45.767 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.767 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.767 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.767 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.785032+0000", 2023-10-09T10:50:45.767 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5599999999999996e-06 2023-10-09T10:50:45.767 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.767 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.767 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.785070+0000", 2023-10-09T10:50:45.768 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8383999999999997e-05 2023-10-09T10:50:45.768 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.768 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.785201+0000", 2023-10-09T10:50:45.768 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000130278 2023-10-09T10:50:45.768 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.768 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:50:45.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.785358+0000", 2023-10-09T10:50:45.768 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015768499999999999 2023-10-09T10:50:45.768 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.769 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.769 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:45.769 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.786167+0000", 2023-10-09T10:50:45.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00080891000000000003 2023-10-09T10:50:45.769 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.769 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.770 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.770 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.787492+0000", 2023-10-09T10:50:45.770 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001324804 2023-10-09T10:50:45.770 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.770 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.770 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:45.770 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852602+0000", 2023-10-09T10:50:45.770 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.065109912000000006 2023-10-09T10:50:45.770 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.770 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.770 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852638+0000", 2023-10-09T10:50:45.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6625000000000003e-05 2023-10-09T10:50:45.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852661+0000", 2023-10-09T10:50:45.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2212999999999999e-05 2023-10-09T10:50:45.771 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.771 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.771 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.772 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35187.0:57478 7.a e213/209 7:5edffc77:::benchmark_data_smithi125_295816_object57477:head v 213'3586, mlcod=0'0)", 2023-10-09T10:50:45.772 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:04.981156+0000", 2023-10-09T10:50:45.772 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 40.652343514999998, 2023-10-09T10:50:45.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.061196533999999997, 2023-10-09T10:50:45.772 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:45.772 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:45.772 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:45.772 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.772 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:45.772 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:04.981156+0000", 2023-10-09T10:50:45.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.772 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:45.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:04.981156+0000", 2023-10-09T10:50:45.773 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:45.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:45.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:04.981154+0000", 2023-10-09T10:50:45.773 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:45.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:45.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:04.981210+0000", 2023-10-09T10:50:45.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5636000000000001e-05 2023-10-09T10:50:45.774 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.774 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.774 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:45.774 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:05.031950+0000", 2023-10-09T10:50:45.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.050740250000000001 2023-10-09T10:50:45.774 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.774 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.774 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:45.774 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:05.031959+0000", 2023-10-09T10:50:45.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2619999999999998e-06 2023-10-09T10:50:45.774 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:45.775 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:05.032141+0000", 2023-10-09T10:50:45.775 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018193899999999999 2023-10-09T10:50:45.775 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:45.775 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:05.032169+0000", 2023-10-09T10:50:45.775 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.832e-05 2023-10-09T10:50:45.775 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:45.776 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:05.042302+0000", 2023-10-09T10:50:45.776 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01013331 2023-10-09T10:50:45.776 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:45.776 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:45.776 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:45.776 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:05.042352+0000", 2023-10-09T10:50:45.776 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0260000000000003e-05 2023-10-09T10:50:45.776 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.776 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.776 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.776 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:45.776 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:45.777 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:50:45.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:45.777 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:45.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:45.878 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:45.921 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:46.003 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:46.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:46.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:46.010 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 85 16 87506 87490 64.3054 48.6875 0.0119844 0.0155451 2023-10-09T10:50:46.077 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:50:46.107 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:46.161 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:46.172 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:46.209 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:46.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:46.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:46.314 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:46 smithi125 ceph-mon[195507]: failed to encode map e224 with expected crc 2023-10-09T10:50:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:50:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:46 smithi125 ceph-mon[195507]: pgmap v511: 109 pgs: 3 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 49 MiB/s wr, 781 op/s; 30700/256050 objects degraded (11.990%); 1701/256050 objects misplaced (0.664%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T10:50:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:46 smithi125 ceph-mon[200197]: failed to encode map e224 with expected crc 2023-10-09T10:50:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:50:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:46 smithi125 ceph-mon[200197]: pgmap v511: 109 pgs: 3 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 49 MiB/s wr, 781 op/s; 30700/256050 objects degraded (11.990%); 1701/256050 objects misplaced (0.664%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T10:50:46.380 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:46.411 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:46.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:46.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:46.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:46 smithi155 ceph-mon[184854]: failed to encode map e224 with expected crc 2023-10-09T10:50:46.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:46.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:50:46.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:46 smithi155 ceph-mon[184854]: pgmap v511: 109 pgs: 3 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 49 MiB/s wr, 781 op/s; 30700/256050 objects degraded (11.990%); 1701/256050 objects misplaced (0.664%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T10:50:46.516 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:46.542 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:50:46.605 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:46.701 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:46.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:46.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:46.805 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:46.867 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:47.010 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 86 16 88236 88220 64.0882 45.625 0.017209 0.015598 2023-10-09T10:50:47.020 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:47.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:47.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:47.072 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:50:47.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:47.168 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:47.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:47 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2510987377' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:50:47.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:47 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:50:47.246 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:47.248 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2510987377' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:50:47.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:47 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:50:47.249 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2510987377' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:50:47.249 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:47 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:50:47.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:47.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:47.350 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:47.398 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:47.430 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:47.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:47.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:47.538 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:47.584 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:47.717 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:47.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:47.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:47.777 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0] out_osds: [2, 7, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:50:47.777 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:50:47.777 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T10:50:47.777 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:50:47.824 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:47.876 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:50:47.902 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:47.909 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:50:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:50:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:88925 7.d 7:b6b73209:::benchmark_data_smithi125_295816_object88924:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2023-10-09T10:50:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:47.893406+0000", 2023-10-09T10:50:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.012200103, 2023-10-09T10:50:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012241831999999999, 2023-10-09T10:50:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:50:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:50:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:50:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 88925 2023-10-09T10:50:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.893406+0000", 2023-10-09T10:50:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.893406+0000", 2023-10-09T10:50:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.893405+0000", 2023-10-09T10:50:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:50:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.893436+0000", 2023-10-09T10:50:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1689000000000001e-05 2023-10-09T10:50:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.893450+0000", 2023-10-09T10:50:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.361e-05 2023-10-09T10:50:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.893458+0000", 2023-10-09T10:50:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1319999999999994e-06 2023-10-09T10:50:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.893612+0000", 2023-10-09T10:50:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015370100000000001 2023-10-09T10:50:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.893752+0000", 2023-10-09T10:50:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014038399999999999 2023-10-09T10:50:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:50:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.893919+0000", 2023-10-09T10:50:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000166935 2023-10-09T10:50:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.899381+0000", 2023-10-09T10:50:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0054618970000000003 2023-10-09T10:50:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35187.0:88926 7.9 e225/209 7:96d3c40a:::benchmark_data_smithi125_295816_object88925:head v 225'5692, mlcod=0'0)", 2023-10-09T10:50:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:47.896892+0000", 2023-10-09T10:50:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0087145290000000004, 2023-10-09T10:50:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0088430409999999994, 2023-10-09T10:50:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:50:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.896892+0000", 2023-10-09T10:50:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.896892+0000", 2023-10-09T10:50:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.896891+0000", 2023-10-09T10:50:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:50:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.896930+0000", 2023-10-09T10:50:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9116000000000003e-05 2023-10-09T10:50:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.896940+0000", 2023-10-09T10:50:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3510000000000008e-06 2023-10-09T10:50:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.896945+0000", 2023-10-09T10:50:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4020000000000001e-06 2023-10-09T10:50:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.896984+0000", 2023-10-09T10:50:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8699000000000003e-05 2023-10-09T10:50:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.897008+0000", 2023-10-09T10:50:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3757999999999999e-05 2023-10-09T10:50:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:88928 7.7 7:ec576614:::benchmark_data_smithi125_295816_object88927:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2023-10-09T10:50:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:47.897018+0000", 2023-10-09T10:50:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0085881210000000006, 2023-10-09T10:50:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0087807719999999992, 2023-10-09T10:50:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:50:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:50:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:50:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 88928 2023-10-09T10:50:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.897018+0000", 2023-10-09T10:50:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.897018+0000", 2023-10-09T10:50:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.897017+0000", 2023-10-09T10:50:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.897058+0000", 2023-10-09T10:50:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0479999999999999e-05 2023-10-09T10:50:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.897069+0000", 2023-10-09T10:50:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1561e-05 2023-10-09T10:50:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.897075+0000", 2023-10-09T10:50:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7089999999999999e-06 2023-10-09T10:50:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.897277+0000", 2023-10-09T10:50:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00020261300000000001 2023-10-09T10:50:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.897420+0000", 2023-10-09T10:50:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014228 2023-10-09T10:50:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:50:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.897577+0000", 2023-10-09T10:50:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015742100000000001 2023-10-09T10:50:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.900180+0000", 2023-10-09T10:50:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026031190000000001 2023-10-09T10:50:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35187.0:88930 7.3 e225/196 7:cdd5a26c:::benchmark_data_smithi125_295816_object88929:head v 225'5532, mlcod=225'5530)", 2023-10-09T10:50:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:47.903358+0000", 2023-10-09T10:50:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0022483080000000001, 2023-10-09T10:50:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002517118, 2023-10-09T10:50:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:50:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903358+0000", 2023-10-09T10:50:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903358+0000", 2023-10-09T10:50:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903357+0000", 2023-10-09T10:50:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903403+0000", 2023-10-09T10:50:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6365000000000001e-05 2023-10-09T10:50:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903416+0000", 2023-10-09T10:50:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2529e-05 2023-10-09T10:50:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903422+0000", 2023-10-09T10:50:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6660000000000002e-06 2023-10-09T10:50:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903517+0000", 2023-10-09T10:50:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4581999999999997e-05 2023-10-09T10:50:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903539+0000", 2023-10-09T10:50:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2527e-05 2023-10-09T10:50:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop_reply(client.35187.0:88925 7.d e225/219)", 2023-10-09T10:50:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:47.903436+0000", 2023-10-09T10:50:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0021707710000000002, 2023-10-09T10:50:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024933440000000002, 2023-10-09T10:50:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "queued for pg", 2023-10-09T10:50:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903436+0000", 2023-10-09T10:50:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903436+0000", 2023-10-09T10:50:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903435+0000", 2023-10-09T10:50:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903439+0000", 2023-10-09T10:50:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1099999999999999e-06 2023-10-09T10:50:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903444+0000", 2023-10-09T10:50:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6219999999999997e-06 2023-10-09T10:50:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903448+0000", 2023-10-09T10:50:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5190000000000001e-06 2023-10-09T10:50:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35187.0:88927 7.0 e225/209 7:0788fe64:::benchmark_data_smithi125_295816_object88926:head v 225'5570, mlcod=0'0)", 2023-10-09T10:50:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:47.903653+0000", 2023-10-09T10:50:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0019533950000000001, 2023-10-09T10:50:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002322902, 2023-10-09T10:50:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:50:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903653+0000", 2023-10-09T10:50:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903653+0000", 2023-10-09T10:50:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903653+0000", 2023-10-09T10:50:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903673+0000", 2023-10-09T10:50:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0778999999999999e-05 2023-10-09T10:50:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903683+0000", 2023-10-09T10:50:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2250000000000006e-06 2023-10-09T10:50:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903687+0000", 2023-10-09T10:50:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2230000000000001e-06 2023-10-09T10:50:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903729+0000", 2023-10-09T10:50:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1832000000000002e-05 2023-10-09T10:50:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.903753+0000", 2023-10-09T10:50:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4266000000000001e-05 2023-10-09T10:50:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35187.0:88922 7.4 e225/209 7:25267e03:::benchmark_data_smithi125_295816_object88921:head v 225'5600, mlcod=0'0)", 2023-10-09T10:50:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:47.904284+0000", 2023-10-09T10:50:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.001322504, 2023-10-09T10:50:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0017494839999999999, 2023-10-09T10:50:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:50:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.904284+0000", 2023-10-09T10:50:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.904284+0000", 2023-10-09T10:50:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.904283+0000", 2023-10-09T10:50:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:50:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.904317+0000", 2023-10-09T10:50:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3454000000000002e-05 2023-10-09T10:50:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.904331+0000", 2023-10-09T10:50:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4104000000000001e-05 2023-10-09T10:50:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.904336+0000", 2023-10-09T10:50:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9300000000000002e-06 2023-10-09T10:50:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.904390+0000", 2023-10-09T10:50:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3999999999999998e-05 2023-10-09T10:50:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:47.904410+0000", 2023-10-09T10:50:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0483999999999999e-05 2023-10-09T10:50:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T10:50:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 7 2023-10-09T10:50:47.942 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:50:47.943 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:47.943 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:50:47.943 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:50:47.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:47.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:47.976 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:50:48.010 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 87 16 89005 88989 63.904 48.0625 0.0384302 0.0156436 2023-10-09T10:50:48.029 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:50:48.051 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:48.077 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:50:48.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:48.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:50:48.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[195507]: osdmap e225: 8 total, 8 up, 5 in 2023-10-09T10:50:48.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[195507]: failed to encode map e225 with expected crc 2023-10-09T10:50:48.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[195507]: failed to encode map e225 with expected crc 2023-10-09T10:50:48.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[195507]: failed to encode map e225 with expected crc 2023-10-09T10:50:48.122 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[195507]: failed to encode map e225 with expected crc 2023-10-09T10:50:48.122 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[195507]: failed to encode map e225 with expected crc 2023-10-09T10:50:48.122 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[195507]: pgmap v513: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 peering, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 42 MiB/s wr, 679 op/s; 30700/259869 objects degraded (11.814%); 1384/259869 objects misplaced (0.533%); 6.0 MiB/s, 94 objects/s recovering 2023-10-09T10:50:48.122 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30700/256050 objects degraded (11.990%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:50:48.122 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2242034498' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:48.122 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:48.122 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:50:48.125 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[200197]: osdmap e225: 8 total, 8 up, 5 in 2023-10-09T10:50:48.125 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[200197]: failed to encode map e225 with expected crc 2023-10-09T10:50:48.125 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[200197]: failed to encode map e225 with expected crc 2023-10-09T10:50:48.125 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[200197]: failed to encode map e225 with expected crc 2023-10-09T10:50:48.125 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[200197]: failed to encode map e225 with expected crc 2023-10-09T10:50:48.125 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[200197]: failed to encode map e225 with expected crc 2023-10-09T10:50:48.125 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[200197]: pgmap v513: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 peering, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 42 MiB/s wr, 679 op/s; 30700/259869 objects degraded (11.814%); 1384/259869 objects misplaced (0.533%); 6.0 MiB/s, 94 objects/s recovering 2023-10-09T10:50:48.125 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30700/256050 objects degraded (11.990%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:50:48.126 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:48 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2242034498' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:48.126 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:50:48.126 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:48.162 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:48.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:48.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:48.168 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:50:48.173 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:50:48.173 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:50:48.173 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:50:48.173 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:50:48.173 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:50:48.173 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:50:48.173 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:50:48.173 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:50:48.190 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:48 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:50:48.190 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:48 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:50:48.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:48 smithi155 ceph-mon[184854]: osdmap e225: 8 total, 8 up, 5 in 2023-10-09T10:50:48.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:48 smithi155 ceph-mon[184854]: failed to encode map e225 with expected crc 2023-10-09T10:50:48.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:48 smithi155 ceph-mon[184854]: failed to encode map e225 with expected crc 2023-10-09T10:50:48.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:48 smithi155 ceph-mon[184854]: failed to encode map e225 with expected crc 2023-10-09T10:50:48.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:48 smithi155 ceph-mon[184854]: failed to encode map e225 with expected crc 2023-10-09T10:50:48.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:48 smithi155 ceph-mon[184854]: failed to encode map e225 with expected crc 2023-10-09T10:50:48.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:48 smithi155 ceph-mon[184854]: pgmap v513: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 peering, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 42 MiB/s wr, 679 op/s; 30700/259869 objects degraded (11.814%); 1384/259869 objects misplaced (0.533%); 6.0 MiB/s, 94 objects/s recovering 2023-10-09T10:50:48.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:48 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30700/256050 objects degraded (11.990%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:50:48.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:48 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2242034498' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:48.205 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:48.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:48.330 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:50:48.365 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:48.406 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:48.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:48.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:48.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:48.561 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:48.792 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:50:48.889 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:48.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:48.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:48.992 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:49.016 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 88 16 89755 89739 63.7063 46.875 0.0162493 0.0156912 2023-10-09T10:50:49.041 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:49.183 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:49.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:49.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:49 smithi155 ceph-mon[184854]: failed to encode map e225 with expected crc 2023-10-09T10:50:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:49 smithi155 ceph-mon[184854]: failed to encode map e225 with expected crc 2023-10-09T10:50:49.297 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:49.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:49 smithi125 ceph-mon[195507]: failed to encode map e225 with expected crc 2023-10-09T10:50:49.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:49 smithi125 ceph-mon[195507]: failed to encode map e225 with expected crc 2023-10-09T10:50:49.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:49 smithi125 ceph-mon[200197]: failed to encode map e225 with expected crc 2023-10-09T10:50:49.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:49 smithi125 ceph-mon[200197]: failed to encode map e225 with expected crc 2023-10-09T10:50:49.335 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:49.417 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:49.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:49.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:49.521 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:49.598 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:49.648 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:49.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:49.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:49.661 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:50:49.755 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:49.825 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:49.888 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:49.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:49.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:50.000 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:50.016 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 89 16 90537 90521 63.5396 48.875 0.0142356 0.0157332 2023-10-09T10:50:50.072 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:50.079 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:50:50.079 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:50:50.090 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:50:50.106 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:50:50.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:50.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:50.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:50.144 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:50.222 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:50.275 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:50.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:50 smithi125 ceph-mon[195507]: pgmap v514: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 41 MiB/s wr, 653 op/s; 30700/263184 objects degraded (11.665%); 1384/263184 objects misplaced (0.526%); 2.9 MiB/s, 44 objects/s recovering 2023-10-09T10:50:50.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:50 smithi125 ceph-mon[200197]: pgmap v514: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 41 MiB/s wr, 653 op/s; 30700/263184 objects degraded (11.665%); 1384/263184 objects misplaced (0.526%); 2.9 MiB/s, 44 objects/s recovering 2023-10-09T10:50:50.364 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:50.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:50.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:50.426 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:50 smithi155 ceph-mon[184854]: pgmap v514: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 41 MiB/s wr, 653 op/s; 30700/263184 objects degraded (11.665%); 1384/263184 objects misplaced (0.526%); 2.9 MiB/s, 44 objects/s recovering 2023-10-09T10:50:50.472 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:50.477 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:50:50.560 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:50.631 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":514,"stamp":"2023-10-09T10:50:49.244365+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5748579533,"num_objects":87728,"num_object_clones":0,"num_object_copies":263184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30700,"num_objects_misplaced":1384,"num_objects_unfound":0,"num_objects_dirty":87728,"num_whiteouts":0,"num_read":338,"num_read_kb":636,"num_write":87932,"num_write_kb":5614832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7562,"num_bytes_recovered":495639845,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77600,"ondisk_log_size":77600,"up":327,"acting":318,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":16278312,"kb_used_data":14678528,"kb_used_omap":35,"kb_used_meta":1599708,"kb_avail":452447448,"statfs":{"total":479975178240,"available":463306186752,"internally_reserved":0,"allocated":15030812672,"data_stored":15025923704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36424,"internal_metadata":1638101432},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,4,2,6,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":295698432,"num_objects":4512,"num_object_clones":0,"num_object_copies":13536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-317,"num_objects_unfound":0,"num_objects_dirty":4512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4512,"num_write_kb":288768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":309,"num_bytes_recovered":20770144,"num_keys_recovered":-10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.899205"},"pg_stats":[{"pgid":"3.1a","version":"64'1","reported_seq":151,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.389534+0000","last_change":"2023-10-09T10:50:43.385056+0000","last_active":"2023-10-09T10:50:43.389534+0000","last_peered":"2023-10-09T10:50:43.389534+0000","last_clean":"2023-10-09T10:50:43.389534+0000","last_became_active":"2023-10-09T10:50:43.384646+0000","last_became_peered":"2023-10-09T10:50:43.384646+0000","last_unstale":"2023-10-09T10:50:43.389534+0000","last_undegraded":"2023-10-09T10:50:43.389534+0000","last_fullsized":"2023-10-09T10:50:43.389534+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:42:25.339628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":202,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.355009+0000","last_change":"2023-10-09T10:49:55.811543+0000","last_active":"2023-10-09T10:50:42.355009+0000","last_peered":"2023-10-09T10:50:42.355009+0000","last_clean":"2023-10-09T10:50:42.355009+0000","last_became_active":"2023-10-09T10:49:55.811225+0000","last_became_peered":"2023-10-09T10:49:55.811225+0000","last_unstale":"2023-10-09T10:50:42.355009+0000","last_undegraded":"2023-10-09T10:50:42.355009+0000","last_fullsized":"2023-10-09T10:50:42.355009+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:56.330889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":65,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.354430+0000","last_change":"2023-10-09T10:49:55.813068+0000","last_active":"2023-10-09T10:50:42.354430+0000","last_peered":"2023-10-09T10:50:42.354430+0000","last_clean":"2023-10-09T10:50:42.354430+0000","last_became_active":"2023-10-09T10:49:55.811847+0000","last_became_peered":"2023-10-09T10:49:55.811847+0000","last_unstale":"2023-10-09T10:50:42.354430+0000","last_undegraded":"2023-10-09T10:50:42.354430+0000","last_fullsized":"2023-10-09T10:50:42.354430+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:35.889323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":113,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.368817+0000","last_change":"2023-10-09T10:49:55.838051+0000","last_active":"2023-10-09T10:50:43.368817+0000","last_peered":"2023-10-09T10:50:43.368817+0000","last_clean":"2023-10-09T10:50:43.368817+0000","last_became_active":"2023-10-09T10:49:55.836999+0000","last_became_peered":"2023-10-09T10:49:55.836999+0000","last_unstale":"2023-10-09T10:50:43.368817+0000","last_undegraded":"2023-10-09T10:50:43.368817+0000","last_fullsized":"2023-10-09T10:50:43.368817+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:00:35.849494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":238,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.363579+0000","last_change":"2023-10-09T10:49:55.820195+0000","last_active":"2023-10-09T10:50:42.363579+0000","last_peered":"2023-10-09T10:50:42.363579+0000","last_clean":"2023-10-09T10:50:42.363579+0000","last_became_active":"2023-10-09T10:49:55.819612+0000","last_became_peered":"2023-10-09T10:49:55.819612+0000","last_unstale":"2023-10-09T10:50:42.363579+0000","last_undegraded":"2023-10-09T10:50:42.363579+0000","last_fullsized":"2023-10-09T10:50:42.363579+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:40.995492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":166,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.369996+0000","last_change":"2023-10-09T10:50:26.524004+0000","last_active":"2023-10-09T10:50:42.369996+0000","last_peered":"2023-10-09T10:50:42.369996+0000","last_clean":"2023-10-09T10:50:42.369996+0000","last_became_active":"2023-10-09T10:50:26.523004+0000","last_became_peered":"2023-10-09T10:50:26.523004+0000","last_unstale":"2023-10-09T10:50:42.369996+0000","last_undegraded":"2023-10-09T10:50:42.369996+0000","last_fullsized":"2023-10-09T10:50:42.369996+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:30.758858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":94,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.371005+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:50:43.371005+0000","last_peered":"2023-10-09T10:50:43.371005+0000","last_clean":"2023-10-09T10:50:43.371005+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:50:43.371005+0000","last_undegraded":"2023-10-09T10:50:43.371005+0000","last_fullsized":"2023-10-09T10:50:43.371005+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":203,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.363595+0000","last_change":"2023-10-09T10:49:55.811228+0000","last_active":"2023-10-09T10:50:42.363595+0000","last_peered":"2023-10-09T10:50:42.363595+0000","last_clean":"2023-10-09T10:50:42.363595+0000","last_became_active":"2023-10-09T10:49:55.810914+0000","last_became_peered":"2023-10-09T10:49:55.810914+0000","last_unstale":"2023-10-09T10:50:42.363595+0000","last_undegraded":"2023-10-09T10:50:42.363595+0000","last_fullsized":"2023-10-09T10:50:42.363595+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:35.753543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":471,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.380879+0000","last_change":"2023-10-09T10:50:43.380879+0000","last_active":"2023-10-09T10:50:43.380879+0000","last_peered":"2023-10-09T10:50:43.380879+0000","last_clean":"2023-10-09T10:50:43.380879+0000","last_became_active":"2023-10-09T10:50:43.380602+0000","last_became_peered":"2023-10-09T10:50:43.380602+0000","last_unstale":"2023-10-09T10:50:43.380879+0000","last_undegraded":"2023-10-09T10:50:43.380879+0000","last_fullsized":"2023-10-09T10:50:43.380879+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":209,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.364679+0000","last_change":"2023-10-09T10:49:55.824836+0000","last_active":"2023-10-09T10:50:42.364679+0000","last_peered":"2023-10-09T10:50:42.364679+0000","last_clean":"2023-10-09T10:50:42.364679+0000","last_became_active":"2023-10-09T10:49:55.824581+0000","last_became_peered":"2023-10-09T10:49:55.824581+0000","last_unstale":"2023-10-09T10:50:42.364679+0000","last_undegraded":"2023-10-09T10:50:42.364679+0000","last_fullsized":"2023-10-09T10:50:42.364679+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:12:05.320238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":160,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.354354+0000","last_change":"2023-10-09T10:49:40.951753+0000","last_active":"2023-10-09T10:50:42.354354+0000","last_peered":"2023-10-09T10:50:42.354354+0000","last_clean":"2023-10-09T10:50:42.354354+0000","last_became_active":"2023-10-09T10:49:40.951441+0000","last_became_peered":"2023-10-09T10:49:40.951441+0000","last_unstale":"2023-10-09T10:50:42.354354+0000","last_undegraded":"2023-10-09T10:50:42.354354+0000","last_fullsized":"2023-10-09T10:50:42.354354+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:13.222334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":161,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.354943+0000","last_change":"2023-10-09T10:49:40.968706+0000","last_active":"2023-10-09T10:50:42.354943+0000","last_peered":"2023-10-09T10:50:42.354943+0000","last_clean":"2023-10-09T10:50:42.354943+0000","last_became_active":"2023-10-09T10:49:40.947149+0000","last_became_peered":"2023-10-09T10:49:40.947149+0000","last_unstale":"2023-10-09T10:50:42.354943+0000","last_undegraded":"2023-10-09T10:50:42.354943+0000","last_fullsized":"2023-10-09T10:50:42.354943+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:57:27.340414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":125,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.370396+0000","last_change":"2023-10-09T10:50:26.513643+0000","last_active":"2023-10-09T10:50:42.370396+0000","last_peered":"2023-10-09T10:50:42.370396+0000","last_clean":"2023-10-09T10:50:42.370396+0000","last_became_active":"2023-10-09T10:50:26.513180+0000","last_became_peered":"2023-10-09T10:50:26.513180+0000","last_unstale":"2023-10-09T10:50:42.370396+0000","last_undegraded":"2023-10-09T10:50:42.370396+0000","last_fullsized":"2023-10-09T10:50:42.370396+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:50.672619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":57,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.381114+0000","last_change":"2023-10-09T10:50:43.381114+0000","last_active":"2023-10-09T10:50:43.381114+0000","last_peered":"2023-10-09T10:50:43.381114+0000","last_clean":"2023-10-09T10:50:43.381114+0000","last_became_active":"2023-10-09T10:50:43.380772+0000","last_became_peered":"2023-10-09T10:50:43.380772+0000","last_unstale":"2023-10-09T10:50:43.381114+0000","last_undegraded":"2023-10-09T10:50:43.381114+0000","last_fullsized":"2023-10-09T10:50:43.381114+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:58.953007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":198,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.362030+0000","last_change":"2023-10-09T10:49:55.818343+0000","last_active":"2023-10-09T10:50:42.362030+0000","last_peered":"2023-10-09T10:50:42.362030+0000","last_clean":"2023-10-09T10:50:42.362030+0000","last_became_active":"2023-10-09T10:49:55.818010+0000","last_became_peered":"2023-10-09T10:49:55.818010+0000","last_unstale":"2023-10-09T10:50:42.362030+0000","last_undegraded":"2023-10-09T10:50:42.362030+0000","last_fullsized":"2023-10-09T10:50:42.362030+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:21.520307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":589,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.370024+0000","last_change":"2023-10-09T10:49:18.708370+0000","last_active":"2023-10-09T10:50:43.370024+0000","last_peered":"2023-10-09T10:50:43.370024+0000","last_clean":"2023-10-09T10:50:43.370024+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:50:43.370024+0000","last_undegraded":"2023-10-09T10:50:43.370024+0000","last_fullsized":"2023-10-09T10:50:43.370024+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:57.808410+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":134,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.385407+0000","last_change":"2023-10-09T10:50:43.385407+0000","last_active":"2023-10-09T10:50:43.385407+0000","last_peered":"2023-10-09T10:50:43.385407+0000","last_clean":"2023-10-09T10:50:43.385407+0000","last_became_active":"2023-10-09T10:50:43.385168+0000","last_became_peered":"2023-10-09T10:50:43.385168+0000","last_unstale":"2023-10-09T10:50:43.385407+0000","last_undegraded":"2023-10-09T10:50:43.385407+0000","last_fullsized":"2023-10-09T10:50:43.385407+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:58.387406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":235,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.357695+0000","last_change":"2023-10-09T10:49:55.817307+0000","last_active":"2023-10-09T10:50:42.357695+0000","last_peered":"2023-10-09T10:50:42.357695+0000","last_clean":"2023-10-09T10:50:42.357695+0000","last_became_active":"2023-10-09T10:49:55.815385+0000","last_became_peered":"2023-10-09T10:49:55.815385+0000","last_unstale":"2023-10-09T10:50:42.357695+0000","last_undegraded":"2023-10-09T10:50:42.357695+0000","last_fullsized":"2023-10-09T10:50:42.357695+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:51:11.536505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":599,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.360227+0000","last_change":"2023-10-09T10:49:43.769673+0000","last_active":"2023-10-09T10:50:42.360227+0000","last_peered":"2023-10-09T10:50:42.360227+0000","last_clean":"2023-10-09T10:50:42.360227+0000","last_became_active":"2023-10-09T10:49:43.769323+0000","last_became_peered":"2023-10-09T10:49:43.769323+0000","last_unstale":"2023-10-09T10:50:42.360227+0000","last_undegraded":"2023-10-09T10:50:42.360227+0000","last_fullsized":"2023-10-09T10:50:42.360227+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:23.224244+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":113,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.366300+0000","last_change":"2023-10-09T10:49:55.837023+0000","last_active":"2023-10-09T10:50:43.366300+0000","last_peered":"2023-10-09T10:50:43.366300+0000","last_clean":"2023-10-09T10:50:43.366300+0000","last_became_active":"2023-10-09T10:49:55.836546+0000","last_became_peered":"2023-10-09T10:49:55.836546+0000","last_unstale":"2023-10-09T10:50:43.366300+0000","last_undegraded":"2023-10-09T10:50:43.366300+0000","last_fullsized":"2023-10-09T10:50:43.366300+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:30.585420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":198,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.360059+0000","last_change":"2023-10-09T10:49:40.952783+0000","last_active":"2023-10-09T10:50:42.360059+0000","last_peered":"2023-10-09T10:50:42.360059+0000","last_clean":"2023-10-09T10:50:42.360059+0000","last_became_active":"2023-10-09T10:49:40.952421+0000","last_became_peered":"2023-10-09T10:49:40.952421+0000","last_unstale":"2023-10-09T10:50:42.360059+0000","last_undegraded":"2023-10-09T10:50:42.360059+0000","last_fullsized":"2023-10-09T10:50:42.360059+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:08:57.147258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":564,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.354729+0000","last_change":"2023-10-09T10:49:58.797203+0000","last_active":"2023-10-09T10:50:42.354729+0000","last_peered":"2023-10-09T10:50:42.354729+0000","last_clean":"2023-10-09T10:50:42.354729+0000","last_became_active":"2023-10-09T10:49:58.796656+0000","last_became_peered":"2023-10-09T10:49:58.796656+0000","last_unstale":"2023-10-09T10:50:42.354729+0000","last_undegraded":"2023-10-09T10:50:42.354729+0000","last_fullsized":"2023-10-09T10:50:42.354729+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":144,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.371055+0000","last_change":"2023-10-09T10:49:40.952105+0000","last_active":"2023-10-09T10:50:42.371055+0000","last_peered":"2023-10-09T10:50:42.371055+0000","last_clean":"2023-10-09T10:50:42.371055+0000","last_became_active":"2023-10-09T10:49:40.950553+0000","last_became_peered":"2023-10-09T10:49:40.950553+0000","last_unstale":"2023-10-09T10:50:42.371055+0000","last_undegraded":"2023-10-09T10:50:42.371055+0000","last_fullsized":"2023-10-09T10:50:42.371055+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:51:40.501639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":103,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.368521+0000","last_change":"2023-10-09T10:49:40.949501+0000","last_active":"2023-10-09T10:50:43.368521+0000","last_peered":"2023-10-09T10:50:43.368521+0000","last_clean":"2023-10-09T10:50:43.368521+0000","last_became_active":"2023-10-09T10:49:40.949158+0000","last_became_peered":"2023-10-09T10:49:40.949158+0000","last_unstale":"2023-10-09T10:50:43.368521+0000","last_undegraded":"2023-10-09T10:50:43.368521+0000","last_fullsized":"2023-10-09T10:50:43.368521+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:07.598560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":599,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.359731+0000","last_change":"2023-10-09T10:49:58.776174+0000","last_active":"2023-10-09T10:50:42.359731+0000","last_peered":"2023-10-09T10:50:42.359731+0000","last_clean":"2023-10-09T10:50:42.359731+0000","last_became_active":"2023-10-09T10:49:58.775795+0000","last_became_peered":"2023-10-09T10:49:58.775795+0000","last_unstale":"2023-10-09T10:50:42.359731+0000","last_undegraded":"2023-10-09T10:50:42.359731+0000","last_fullsized":"2023-10-09T10:50:42.359731+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:47.161519+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":103,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.370952+0000","last_change":"2023-10-09T10:50:26.516984+0000","last_active":"2023-10-09T10:50:42.370952+0000","last_peered":"2023-10-09T10:50:42.370952+0000","last_clean":"2023-10-09T10:50:42.370952+0000","last_became_active":"2023-10-09T10:50:26.516657+0000","last_became_peered":"2023-10-09T10:50:26.516657+0000","last_unstale":"2023-10-09T10:50:42.370952+0000","last_undegraded":"2023-10-09T10:50:42.370952+0000","last_fullsized":"2023-10-09T10:50:42.370952+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:13:43.766371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":211,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.356391+0000","last_change":"2023-10-09T10:49:55.814525+0000","last_active":"2023-10-09T10:50:42.356391+0000","last_peered":"2023-10-09T10:50:42.356391+0000","last_clean":"2023-10-09T10:50:42.356391+0000","last_became_active":"2023-10-09T10:49:55.814228+0000","last_became_peered":"2023-10-09T10:49:55.814228+0000","last_unstale":"2023-10-09T10:50:42.356391+0000","last_undegraded":"2023-10-09T10:50:42.356391+0000","last_fullsized":"2023-10-09T10:50:42.356391+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:19.868347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"224'5505","reported_seq":9113,"reported_epoch":224,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:50:46.292155+0000","last_change":"2023-10-09T10:50:13.000988+0000","last_active":"2023-10-09T10:50:46.292155+0000","last_peered":"2023-10-09T10:50:46.292155+0000","last_clean":"2023-10-09T10:49:54.739089+0000","last_became_active":"2023-10-09T10:49:56.954628+0000","last_became_peered":"2023-10-09T10:49:56.954628+0000","last_unstale":"2023-10-09T10:50:46.292155+0000","last_undegraded":"2023-10-09T10:50:46.292155+0000","last_fullsized":"2023-10-09T10:49:56.808257+0000","mapping_epoch":209,"log_start":"199'600","ondisk_log_start":"199'600","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4905,"log_dups_size":0,"ondisk_log_size":4905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:22.039147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360775680,"num_objects":5505,"num_object_clones":0,"num_object_copies":16515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1384,"num_objects_unfound":0,"num_objects_dirty":5505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5505,"num_write_kb":352320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1781,"num_bytes_recovered":116719616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1384},{"shards":"5,6","objects":4121}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":471,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.381903+0000","last_change":"2023-10-09T10:50:43.381903+0000","last_active":"2023-10-09T10:50:43.381903+0000","last_peered":"2023-10-09T10:50:43.381903+0000","last_clean":"2023-10-09T10:50:43.381903+0000","last_became_active":"2023-10-09T10:50:43.381360+0000","last_became_peered":"2023-10-09T10:50:43.381360+0000","last_unstale":"2023-10-09T10:50:43.381903+0000","last_undegraded":"2023-10-09T10:50:43.381903+0000","last_fullsized":"2023-10-09T10:50:43.381903+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":63,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.361829+0000","last_change":"2023-10-09T10:49:55.846025+0000","last_active":"2023-10-09T10:50:42.361829+0000","last_peered":"2023-10-09T10:50:42.361829+0000","last_clean":"2023-10-09T10:50:42.361829+0000","last_became_active":"2023-10-09T10:49:55.821145+0000","last_became_peered":"2023-10-09T10:49:55.821145+0000","last_unstale":"2023-10-09T10:50:42.361829+0000","last_undegraded":"2023-10-09T10:50:42.361829+0000","last_fullsized":"2023-10-09T10:50:42.361829+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:50.726370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":181,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.359153+0000","last_change":"2023-10-09T10:49:40.955088+0000","last_active":"2023-10-09T10:50:42.359153+0000","last_peered":"2023-10-09T10:50:42.359153+0000","last_clean":"2023-10-09T10:50:42.359153+0000","last_became_active":"2023-10-09T10:49:40.954716+0000","last_became_peered":"2023-10-09T10:49:40.954716+0000","last_unstale":"2023-10-09T10:50:42.359153+0000","last_undegraded":"2023-10-09T10:50:42.359153+0000","last_fullsized":"2023-10-09T10:50:42.359153+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:39.090107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"224'5412","reported_seq":5457,"reported_epoch":224,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:44.739063+0000","last_change":"2023-10-09T10:49:57.045491+0000","last_active":"2023-10-09T10:50:44.739063+0000","last_peered":"2023-10-09T10:50:44.739063+0000","last_clean":"2023-10-09T10:49:54.734240+0000","last_became_active":"2023-10-09T10:49:57.036090+0000","last_became_peered":"2023-10-09T10:49:57.036090+0000","last_unstale":"2023-10-09T10:50:44.739063+0000","last_undegraded":"2023-10-09T10:49:56.833807+0000","last_fullsized":"2023-10-09T10:49:56.805766+0000","mapping_epoch":209,"log_start":"199'800","ondisk_log_start":"199'800","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4612,"log_dups_size":0,"ondisk_log_size":4612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:14.052065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354680832,"num_objects":5412,"num_object_clones":0,"num_object_copies":16236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3125,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5412,"num_write_kb":346368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,5","objects":3125},{"shards":"3,5","objects":2287}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":559,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.357535+0000","last_change":"2023-10-09T10:49:41.638077+0000","last_active":"2023-10-09T10:50:42.357535+0000","last_peered":"2023-10-09T10:50:42.357535+0000","last_clean":"2023-10-09T10:50:42.357535+0000","last_became_active":"2023-10-09T10:49:17.844627+0000","last_became_peered":"2023-10-09T10:49:17.844627+0000","last_unstale":"2023-10-09T10:50:42.357535+0000","last_undegraded":"2023-10-09T10:50:42.357535+0000","last_fullsized":"2023-10-09T10:50:42.357535+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:41.637984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:41.637984+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:39:39.594702+0000","scrub_duration":0.001804833,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":28,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.389537+0000","last_change":"2023-10-09T10:50:43.385172+0000","last_active":"2023-10-09T10:50:43.389537+0000","last_peered":"2023-10-09T10:50:43.389537+0000","last_clean":"2023-10-09T10:50:43.389537+0000","last_became_active":"2023-10-09T10:50:43.384888+0000","last_became_peered":"2023-10-09T10:50:43.384888+0000","last_unstale":"2023-10-09T10:50:43.389537+0000","last_undegraded":"2023-10-09T10:50:43.389537+0000","last_fullsized":"2023-10-09T10:50:43.389537+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:29.784932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":223,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.392704+0000","last_change":"2023-10-09T10:50:43.384962+0000","last_active":"2023-10-09T10:50:43.392704+0000","last_peered":"2023-10-09T10:50:43.392704+0000","last_clean":"2023-10-09T10:50:43.392704+0000","last_became_active":"2023-10-09T10:50:43.384645+0000","last_became_peered":"2023-10-09T10:50:43.384645+0000","last_unstale":"2023-10-09T10:50:43.392704+0000","last_undegraded":"2023-10-09T10:50:43.392704+0000","last_fullsized":"2023-10-09T10:50:43.392704+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:45.825019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.9","version":"225'5693","reported_seq":5738,"reported_epoch":225,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:47.927810+0000","last_change":"2023-10-09T10:49:56.964132+0000","last_active":"2023-10-09T10:50:47.927810+0000","last_peered":"2023-10-09T10:50:47.927810+0000","last_clean":"2023-10-09T10:49:54.793473+0000","last_became_active":"2023-10-09T10:49:56.956688+0000","last_became_peered":"2023-10-09T10:49:56.956688+0000","last_unstale":"2023-10-09T10:50:47.927810+0000","last_undegraded":"2023-10-09T10:49:56.831390+0000","last_fullsized":"2023-10-09T10:49:56.801929+0000","mapping_epoch":209,"log_start":"199'1100","ondisk_log_start":"199'1100","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4593,"log_dups_size":0,"ondisk_log_size":4593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:06:54.970299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373096448,"num_objects":5693,"num_object_clones":0,"num_object_copies":17079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3263,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5693,"num_write_kb":364352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3263},{"shards":"0,4","objects":2430}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":420,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.370217+0000","last_change":"2023-10-09T10:50:26.524499+0000","last_active":"2023-10-09T10:50:42.370217+0000","last_peered":"2023-10-09T10:50:42.370217+0000","last_clean":"2023-10-09T10:50:42.370217+0000","last_became_active":"2023-10-09T10:50:26.523494+0000","last_became_peered":"2023-10-09T10:50:26.523494+0000","last_unstale":"2023-10-09T10:50:42.370217+0000","last_undegraded":"2023-10-09T10:50:42.370217+0000","last_fullsized":"2023-10-09T10:50:42.370217+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":159,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.359978+0000","last_change":"2023-10-09T10:49:40.958240+0000","last_active":"2023-10-09T10:50:42.359978+0000","last_peered":"2023-10-09T10:50:42.359978+0000","last_clean":"2023-10-09T10:50:42.359978+0000","last_became_active":"2023-10-09T10:49:40.952355+0000","last_became_peered":"2023-10-09T10:49:40.952355+0000","last_unstale":"2023-10-09T10:50:42.359978+0000","last_undegraded":"2023-10-09T10:50:42.359978+0000","last_fullsized":"2023-10-09T10:50:42.359978+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:30.412691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":164,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.370206+0000","last_change":"2023-10-09T10:50:26.517403+0000","last_active":"2023-10-09T10:50:42.370206+0000","last_peered":"2023-10-09T10:50:42.370206+0000","last_clean":"2023-10-09T10:50:42.370206+0000","last_became_active":"2023-10-09T10:50:26.517133+0000","last_became_peered":"2023-10-09T10:50:26.517133+0000","last_unstale":"2023-10-09T10:50:42.370206+0000","last_undegraded":"2023-10-09T10:50:42.370206+0000","last_fullsized":"2023-10-09T10:50:42.370206+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:51:57.400824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"224'5509","reported_seq":10381,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:46.128890+0000","last_change":"2023-10-09T10:50:27.531707+0000","last_active":"2023-10-09T10:50:46.128890+0000","last_peered":"2023-10-09T10:50:46.128890+0000","last_clean":"2023-10-09T10:50:46.128890+0000","last_became_active":"2023-10-09T10:50:27.518955+0000","last_became_peered":"2023-10-09T10:50:27.518955+0000","last_unstale":"2023-10-09T10:50:46.128890+0000","last_undegraded":"2023-10-09T10:50:46.128890+0000","last_fullsized":"2023-10-09T10:50:46.128890+0000","mapping_epoch":219,"log_start":"199'600","ondisk_log_start":"199'600","created":196,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4909,"log_dups_size":0,"ondisk_log_size":4909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:34.331554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361037824,"num_objects":5509,"num_object_clones":0,"num_object_copies":16527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5509,"num_write_kb":352576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2408,"num_bytes_recovered":157810688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":609,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.362531+0000","last_change":"2023-10-09T10:49:58.784989+0000","last_active":"2023-10-09T10:50:42.362531+0000","last_peered":"2023-10-09T10:50:42.362531+0000","last_clean":"2023-10-09T10:50:42.362531+0000","last_became_active":"2023-10-09T10:49:58.784715+0000","last_became_peered":"2023-10-09T10:49:58.784715+0000","last_unstale":"2023-10-09T10:50:42.362531+0000","last_undegraded":"2023-10-09T10:50:42.362531+0000","last_fullsized":"2023-10-09T10:50:42.362531+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":189,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.359931+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:50:42.359931+0000","last_peered":"2023-10-09T10:50:42.359931+0000","last_clean":"2023-10-09T10:50:42.359931+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:50:42.359931+0000","last_undegraded":"2023-10-09T10:50:42.359931+0000","last_fullsized":"2023-10-09T10:50:42.359931+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":104,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.366118+0000","last_change":"2023-10-09T10:49:55.830400+0000","last_active":"2023-10-09T10:50:43.366118+0000","last_peered":"2023-10-09T10:50:43.366118+0000","last_clean":"2023-10-09T10:50:43.366118+0000","last_became_active":"2023-10-09T10:49:55.829982+0000","last_became_peered":"2023-10-09T10:49:55.829982+0000","last_unstale":"2023-10-09T10:50:43.366118+0000","last_undegraded":"2023-10-09T10:50:43.366118+0000","last_fullsized":"2023-10-09T10:50:43.366118+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:51.352143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"225'5526","reported_seq":5572,"reported_epoch":225,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:47.950694+0000","last_change":"2023-10-09T10:49:42.062469+0000","last_active":"2023-10-09T10:50:47.950694+0000","last_peered":"2023-10-09T10:50:47.950694+0000","last_clean":"2023-10-09T10:49:39.916756+0000","last_became_active":"2023-10-09T10:49:42.056867+0000","last_became_peered":"2023-10-09T10:49:42.056867+0000","last_unstale":"2023-10-09T10:50:47.950694+0000","last_undegraded":"2023-10-09T10:49:41.970254+0000","last_fullsized":"2023-10-09T10:49:41.933564+0000","mapping_epoch":203,"log_start":"199'900","ondisk_log_start":"199'900","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4626,"log_dups_size":0,"ondisk_log_size":4626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:59:10.773024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362151936,"num_objects":5526,"num_object_clones":0,"num_object_copies":16578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2419,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5526,"num_write_kb":353664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":3107},{"shards":"0,3,7","objects":2419}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":575,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.347279+0000","last_change":"2023-10-09T10:49:43.769990+0000","last_active":"2023-10-09T10:50:42.347279+0000","last_peered":"2023-10-09T10:50:42.347279+0000","last_clean":"2023-10-09T10:50:42.347279+0000","last_became_active":"2023-10-09T10:49:43.769783+0000","last_became_peered":"2023-10-09T10:49:43.769783+0000","last_unstale":"2023-10-09T10:50:42.347279+0000","last_undegraded":"2023-10-09T10:50:42.347279+0000","last_fullsized":"2023-10-09T10:50:42.347279+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":218,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.347538+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:50:42.347538+0000","last_peered":"2023-10-09T10:50:42.347538+0000","last_clean":"2023-10-09T10:50:42.347538+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:50:42.347538+0000","last_undegraded":"2023-10-09T10:50:42.347538+0000","last_fullsized":"2023-10-09T10:50:42.347538+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":63,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.387419+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T10:50:43.387419+0000","last_peered":"2023-10-09T10:50:43.387419+0000","last_clean":"2023-10-09T10:50:43.387419+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T10:50:43.387419+0000","last_undegraded":"2023-10-09T10:50:43.387419+0000","last_fullsized":"2023-10-09T10:50:43.387419+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"224'5397","reported_seq":5433,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:46.252775+0000","last_change":"2023-10-09T10:49:16.843562+0000","last_active":"2023-10-09T10:50:46.252775+0000","last_peered":"2023-10-09T10:50:46.252775+0000","last_clean":"2023-10-09T10:50:46.252775+0000","last_became_active":"2023-10-09T10:49:16.843246+0000","last_became_peered":"2023-10-09T10:49:16.843246+0000","last_unstale":"2023-10-09T10:50:46.252775+0000","last_undegraded":"2023-10-09T10:50:46.252775+0000","last_fullsized":"2023-10-09T10:50:46.252775+0000","mapping_epoch":196,"log_start":"199'500","ondisk_log_start":"199'500","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4897,"log_dups_size":0,"ondisk_log_size":4897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:47.911382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353697792,"num_objects":5397,"num_object_clones":0,"num_object_copies":16191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5397,"num_write_kb":345408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":549,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.359347+0000","last_change":"2023-10-09T10:49:17.836306+0000","last_active":"2023-10-09T10:50:42.359347+0000","last_peered":"2023-10-09T10:50:42.359347+0000","last_clean":"2023-10-09T10:50:42.359347+0000","last_became_active":"2023-10-09T10:49:17.836052+0000","last_became_peered":"2023-10-09T10:49:17.836052+0000","last_unstale":"2023-10-09T10:50:42.359347+0000","last_undegraded":"2023-10-09T10:50:42.359347+0000","last_fullsized":"2023-10-09T10:50:42.359347+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":173,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.361111+0000","last_change":"2023-10-09T10:49:55.814556+0000","last_active":"2023-10-09T10:50:42.361111+0000","last_peered":"2023-10-09T10:50:42.361111+0000","last_clean":"2023-10-09T10:50:42.361111+0000","last_became_active":"2023-10-09T10:49:55.814193+0000","last_became_peered":"2023-10-09T10:49:55.814193+0000","last_unstale":"2023-10-09T10:50:42.361111+0000","last_undegraded":"2023-10-09T10:50:42.361111+0000","last_fullsized":"2023-10-09T10:50:42.361111+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:13:19.537756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":214,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.381687+0000","last_change":"2023-10-09T10:50:43.381687+0000","last_active":"2023-10-09T10:50:43.381687+0000","last_peered":"2023-10-09T10:50:43.381687+0000","last_clean":"2023-10-09T10:50:43.381687+0000","last_became_active":"2023-10-09T10:50:43.380313+0000","last_became_peered":"2023-10-09T10:50:43.380313+0000","last_unstale":"2023-10-09T10:50:43.381687+0000","last_undegraded":"2023-10-09T10:50:43.381687+0000","last_fullsized":"2023-10-09T10:50:43.381687+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:08.068017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.d","version":"224'5370","reported_seq":5421,"reported_epoch":224,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:50:46.081320+0000","last_change":"2023-10-09T10:50:27.589727+0000","last_active":"2023-10-09T10:50:46.081320+0000","last_peered":"2023-10-09T10:50:46.081320+0000","last_clean":"2023-10-09T10:49:54.776483+0000","last_became_active":"2023-10-09T10:50:27.528000+0000","last_became_peered":"2023-10-09T10:50:27.528000+0000","last_unstale":"2023-10-09T10:50:46.081320+0000","last_undegraded":"2023-10-09T10:50:27.519640+0000","last_fullsized":"2023-10-09T10:50:46.081320+0000","mapping_epoch":219,"log_start":"199'1098","ondisk_log_start":"199'1098","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4272,"log_dups_size":0,"ondisk_log_size":4272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:45:43.017712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351928320,"num_objects":5370,"num_object_clones":0,"num_object_copies":16110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3118,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5370,"num_write_kb":343680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":["6","0","1"],"object_location_counts":[{"shards":"0,1,6","objects":3118},{"shards":"0,3,6","objects":2252}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":571,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.366033+0000","last_change":"2023-10-09T10:49:58.781265+0000","last_active":"2023-10-09T10:50:43.366033+0000","last_peered":"2023-10-09T10:50:43.366033+0000","last_clean":"2023-10-09T10:50:43.366033+0000","last_became_active":"2023-10-09T10:49:58.780915+0000","last_became_peered":"2023-10-09T10:49:58.780915+0000","last_unstale":"2023-10-09T10:50:43.366033+0000","last_undegraded":"2023-10-09T10:50:43.366033+0000","last_fullsized":"2023-10-09T10:50:43.366033+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.892515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.892515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":219,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.371007+0000","last_change":"2023-10-09T10:49:55.812746+0000","last_active":"2023-10-09T10:50:42.371007+0000","last_peered":"2023-10-09T10:50:42.371007+0000","last_clean":"2023-10-09T10:50:42.371007+0000","last_became_active":"2023-10-09T10:49:55.812438+0000","last_became_peered":"2023-10-09T10:49:55.812438+0000","last_unstale":"2023-10-09T10:50:42.371007+0000","last_undegraded":"2023-10-09T10:50:42.371007+0000","last_fullsized":"2023-10-09T10:50:42.371007+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:44:14.045846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":187,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.380032+0000","last_change":"2023-10-09T10:50:43.380032+0000","last_active":"2023-10-09T10:50:43.380032+0000","last_peered":"2023-10-09T10:50:43.380032+0000","last_clean":"2023-10-09T10:50:43.380032+0000","last_became_active":"2023-10-09T10:50:43.379709+0000","last_became_peered":"2023-10-09T10:50:43.379709+0000","last_unstale":"2023-10-09T10:50:43.380032+0000","last_undegraded":"2023-10-09T10:50:43.380032+0000","last_fullsized":"2023-10-09T10:50:43.380032+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:04.284489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.c","version":"224'5356","reported_seq":10108,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:44.727895+0000","last_change":"2023-10-09T10:50:14.783907+0000","last_active":"2023-10-09T10:50:44.727895+0000","last_peered":"2023-10-09T10:50:44.727895+0000","last_clean":"2023-10-09T10:50:44.727895+0000","last_became_active":"2023-10-09T10:50:14.780725+0000","last_became_peered":"2023-10-09T10:50:14.780725+0000","last_unstale":"2023-10-09T10:50:44.727895+0000","last_undegraded":"2023-10-09T10:50:44.727895+0000","last_fullsized":"2023-10-09T10:50:44.727895+0000","mapping_epoch":215,"log_start":"199'800","ondisk_log_start":"199'800","created":196,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4556,"log_dups_size":0,"ondisk_log_size":4556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:11:40.081379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351010816,"num_objects":5356,"num_object_clones":0,"num_object_copies":16068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5356,"num_write_kb":342784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2347,"num_bytes_recovered":153812992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":560,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.359683+0000","last_change":"2023-10-09T10:49:17.847894+0000","last_active":"2023-10-09T10:50:42.359683+0000","last_peered":"2023-10-09T10:50:42.359683+0000","last_clean":"2023-10-09T10:50:42.359683+0000","last_became_active":"2023-10-09T10:49:17.847624+0000","last_became_peered":"2023-10-09T10:49:17.847624+0000","last_unstale":"2023-10-09T10:50:42.359683+0000","last_undegraded":"2023-10-09T10:50:42.359683+0000","last_fullsized":"2023-10-09T10:50:42.359683+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":213,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.363497+0000","last_change":"2023-10-09T10:49:55.819881+0000","last_active":"2023-10-09T10:50:42.363497+0000","last_peered":"2023-10-09T10:50:42.363497+0000","last_clean":"2023-10-09T10:50:42.363497+0000","last_became_active":"2023-10-09T10:49:55.818533+0000","last_became_peered":"2023-10-09T10:49:55.818533+0000","last_unstale":"2023-10-09T10:50:42.363497+0000","last_undegraded":"2023-10-09T10:50:42.363497+0000","last_fullsized":"2023-10-09T10:50:42.363497+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:20.512152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":65,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.357237+0000","last_change":"2023-10-09T10:49:55.815880+0000","last_active":"2023-10-09T10:50:42.357237+0000","last_peered":"2023-10-09T10:50:42.357237+0000","last_clean":"2023-10-09T10:50:42.357237+0000","last_became_active":"2023-10-09T10:49:55.815594+0000","last_became_peered":"2023-10-09T10:49:55.815594+0000","last_unstale":"2023-10-09T10:50:42.357237+0000","last_undegraded":"2023-10-09T10:50:42.357237+0000","last_fullsized":"2023-10-09T10:50:42.357237+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:39.210709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"224'5323","reported_seq":5365,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:44.734217+0000","last_change":"2023-10-09T10:50:43.392944+0000","last_active":"2023-10-09T10:50:44.734217+0000","last_peered":"2023-10-09T10:50:44.734217+0000","last_clean":"2023-10-09T10:50:44.734217+0000","last_became_active":"2023-10-09T10:50:43.384716+0000","last_became_peered":"2023-10-09T10:50:43.384716+0000","last_unstale":"2023-10-09T10:50:44.734217+0000","last_undegraded":"2023-10-09T10:50:44.734217+0000","last_fullsized":"2023-10-09T10:50:44.734217+0000","mapping_epoch":223,"log_start":"199'695","ondisk_log_start":"199'695","created":196,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4628,"log_dups_size":0,"ondisk_log_size":4628,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:19.728024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348848128,"num_objects":5323,"num_object_clones":0,"num_object_copies":15969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5323,"num_write_kb":340672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":420,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.370075+0000","last_change":"2023-10-09T10:50:26.524184+0000","last_active":"2023-10-09T10:50:42.370075+0000","last_peered":"2023-10-09T10:50:42.370075+0000","last_clean":"2023-10-09T10:50:42.370075+0000","last_became_active":"2023-10-09T10:50:26.523039+0000","last_became_peered":"2023-10-09T10:50:26.523039+0000","last_unstale":"2023-10-09T10:50:42.370075+0000","last_undegraded":"2023-10-09T10:50:42.370075+0000","last_fullsized":"2023-10-09T10:50:42.370075+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.687290+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.687290+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":227,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.373900+0000","last_change":"2023-10-09T10:49:55.825069+0000","last_active":"2023-10-09T10:50:43.373900+0000","last_peered":"2023-10-09T10:50:43.373900+0000","last_clean":"2023-10-09T10:50:43.373900+0000","last_became_active":"2023-10-09T10:49:55.824755+0000","last_became_peered":"2023-10-09T10:49:55.824755+0000","last_unstale":"2023-10-09T10:50:43.373900+0000","last_undegraded":"2023-10-09T10:50:43.373900+0000","last_fullsized":"2023-10-09T10:50:43.373900+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:04:28.113626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":216,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.360079+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:50:42.360079+0000","last_peered":"2023-10-09T10:50:42.360079+0000","last_clean":"2023-10-09T10:50:42.360079+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:50:42.360079+0000","last_undegraded":"2023-10-09T10:50:42.360079+0000","last_fullsized":"2023-10-09T10:50:42.360079+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":41,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.359574+0000","last_change":"2023-10-09T10:49:55.828773+0000","last_active":"2023-10-09T10:50:42.359574+0000","last_peered":"2023-10-09T10:50:42.359574+0000","last_clean":"2023-10-09T10:50:42.359574+0000","last_became_active":"2023-10-09T10:49:55.828458+0000","last_became_peered":"2023-10-09T10:49:55.828458+0000","last_unstale":"2023-10-09T10:50:42.359574+0000","last_undegraded":"2023-10-09T10:50:42.359574+0000","last_fullsized":"2023-10-09T10:50:42.359574+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:30.196079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"224'5550","reported_seq":5593,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:46.084402+0000","last_change":"2023-10-09T10:50:26.523779+0000","last_active":"2023-10-09T10:50:46.084402+0000","last_peered":"2023-10-09T10:50:46.084402+0000","last_clean":"2023-10-09T10:50:46.084402+0000","last_became_active":"2023-10-09T10:50:26.513738+0000","last_became_peered":"2023-10-09T10:50:26.513738+0000","last_unstale":"2023-10-09T10:50:46.084402+0000","last_undegraded":"2023-10-09T10:50:46.084402+0000","last_fullsized":"2023-10-09T10:50:46.084402+0000","mapping_epoch":218,"log_start":"199'1248","ondisk_log_start":"199'1248","created":196,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4302,"log_dups_size":0,"ondisk_log_size":4302,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:55.027791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363724800,"num_objects":5550,"num_object_clones":0,"num_object_copies":16650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5550,"num_write_kb":355200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":578,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.359691+0000","last_change":"2023-10-09T10:49:38.736950+0000","last_active":"2023-10-09T10:50:42.359691+0000","last_peered":"2023-10-09T10:50:42.359691+0000","last_clean":"2023-10-09T10:50:42.359691+0000","last_became_active":"2023-10-09T10:49:17.837821+0000","last_became_peered":"2023-10-09T10:49:17.837821+0000","last_unstale":"2023-10-09T10:50:42.359691+0000","last_undegraded":"2023-10-09T10:50:42.359691+0000","last_fullsized":"2023-10-09T10:50:42.359691+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.736834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.736834+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:17:16.734495+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":203,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.359499+0000","last_change":"2023-10-09T10:49:55.847216+0000","last_active":"2023-10-09T10:50:42.359499+0000","last_peered":"2023-10-09T10:50:42.359499+0000","last_clean":"2023-10-09T10:50:42.359499+0000","last_became_active":"2023-10-09T10:49:55.847002+0000","last_became_peered":"2023-10-09T10:49:55.847002+0000","last_unstale":"2023-10-09T10:50:42.359499+0000","last_undegraded":"2023-10-09T10:50:42.359499+0000","last_fullsized":"2023-10-09T10:50:42.359499+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:41.845106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":132,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.389610+0000","last_change":"2023-10-09T10:50:43.388615+0000","last_active":"2023-10-09T10:50:43.389610+0000","last_peered":"2023-10-09T10:50:43.389610+0000","last_clean":"2023-10-09T10:50:43.389610+0000","last_became_active":"2023-10-09T10:50:43.388176+0000","last_became_peered":"2023-10-09T10:50:43.388176+0000","last_unstale":"2023-10-09T10:50:43.389610+0000","last_undegraded":"2023-10-09T10:50:43.389610+0000","last_fullsized":"2023-10-09T10:50:43.389610+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:41:17.550111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":147,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.358559+0000","last_change":"2023-10-09T10:49:55.816187+0000","last_active":"2023-10-09T10:50:42.358559+0000","last_peered":"2023-10-09T10:50:42.358559+0000","last_clean":"2023-10-09T10:50:42.358559+0000","last_became_active":"2023-10-09T10:49:55.815345+0000","last_became_peered":"2023-10-09T10:49:55.815345+0000","last_unstale":"2023-10-09T10:50:42.358559+0000","last_undegraded":"2023-10-09T10:50:42.358559+0000","last_fullsized":"2023-10-09T10:50:42.358559+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:43.036104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.4","version":"225'5603","reported_seq":5653,"reported_epoch":225,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:47.927770+0000","last_change":"2023-10-09T10:49:57.003555+0000","last_active":"2023-10-09T10:50:47.927770+0000","last_peered":"2023-10-09T10:50:47.927770+0000","last_clean":"2023-10-09T10:49:54.777504+0000","last_became_active":"2023-10-09T10:49:56.956699+0000","last_became_peered":"2023-10-09T10:49:56.956699+0000","last_unstale":"2023-10-09T10:50:47.927770+0000","last_undegraded":"2023-10-09T10:49:56.938617+0000","last_fullsized":"2023-10-09T10:49:56.806441+0000","mapping_epoch":209,"log_start":"199'1000","ondisk_log_start":"199'1000","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4603,"log_dups_size":0,"ondisk_log_size":4603,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:51.048643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367198208,"num_objects":5603,"num_object_clones":0,"num_object_copies":16809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3084,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5603,"num_write_kb":358592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3083},{"shards":"0,4","objects":2519}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":595,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.366151+0000","last_change":"2023-10-09T10:49:43.780951+0000","last_active":"2023-10-09T10:50:43.366151+0000","last_peered":"2023-10-09T10:50:43.366151+0000","last_clean":"2023-10-09T10:50:43.366151+0000","last_became_active":"2023-10-09T10:49:43.780564+0000","last_became_peered":"2023-10-09T10:49:43.780564+0000","last_unstale":"2023-10-09T10:50:43.366151+0000","last_undegraded":"2023-10-09T10:50:43.366151+0000","last_fullsized":"2023-10-09T10:50:43.366151+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":232,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.362808+0000","last_change":"2023-10-09T10:49:55.814470+0000","last_active":"2023-10-09T10:50:43.362808+0000","last_peered":"2023-10-09T10:50:43.362808+0000","last_clean":"2023-10-09T10:50:43.362808+0000","last_became_active":"2023-10-09T10:49:55.814154+0000","last_became_peered":"2023-10-09T10:49:55.814154+0000","last_unstale":"2023-10-09T10:50:43.362808+0000","last_undegraded":"2023-10-09T10:50:43.362808+0000","last_fullsized":"2023-10-09T10:50:43.362808+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:03:57.326294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":125,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.371090+0000","last_change":"2023-10-09T10:50:26.516824+0000","last_active":"2023-10-09T10:50:42.371090+0000","last_peered":"2023-10-09T10:50:42.371090+0000","last_clean":"2023-10-09T10:50:42.371090+0000","last_became_active":"2023-10-09T10:50:26.516535+0000","last_became_peered":"2023-10-09T10:50:26.516535+0000","last_unstale":"2023-10-09T10:50:42.371090+0000","last_undegraded":"2023-10-09T10:50:42.371090+0000","last_fullsized":"2023-10-09T10:50:42.371090+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:21.223491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":151,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.363255+0000","last_change":"2023-10-09T10:49:55.822332+0000","last_active":"2023-10-09T10:50:42.363255+0000","last_peered":"2023-10-09T10:50:42.363255+0000","last_clean":"2023-10-09T10:50:42.363255+0000","last_became_active":"2023-10-09T10:49:55.822068+0000","last_became_peered":"2023-10-09T10:49:55.822068+0000","last_unstale":"2023-10-09T10:50:42.363255+0000","last_undegraded":"2023-10-09T10:50:42.363255+0000","last_fullsized":"2023-10-09T10:50:42.363255+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:41:57.905151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"225'5527","reported_seq":7576,"reported_epoch":225,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:47.894280+0000","last_change":"2023-10-09T10:49:55.838303+0000","last_active":"2023-10-09T10:50:47.894280+0000","last_peered":"2023-10-09T10:50:47.894280+0000","last_clean":"2023-10-09T10:49:39.898166+0000","last_became_active":"2023-10-09T10:49:42.040841+0000","last_became_peered":"2023-10-09T10:49:42.040841+0000","last_unstale":"2023-10-09T10:50:47.894280+0000","last_undegraded":"2023-10-09T10:49:55.835709+0000","last_fullsized":"2023-10-09T10:49:41.933738+0000","mapping_epoch":203,"log_start":"199'900","ondisk_log_start":"199'900","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4627,"log_dups_size":0,"ondisk_log_size":4627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:02.626804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362217472,"num_objects":5527,"num_object_clones":0,"num_object_copies":16581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1311,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5527,"num_write_kb":353728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1000,"num_bytes_recovered":65536000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":4216},{"shards":"0,3,7","objects":1311}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":213,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.362357+0000","last_change":"2023-10-09T10:49:40.951536+0000","last_active":"2023-10-09T10:50:42.362357+0000","last_peered":"2023-10-09T10:50:42.362357+0000","last_clean":"2023-10-09T10:50:42.362357+0000","last_became_active":"2023-10-09T10:49:40.951152+0000","last_became_peered":"2023-10-09T10:49:40.951152+0000","last_unstale":"2023-10-09T10:50:42.362357+0000","last_undegraded":"2023-10-09T10:50:42.362357+0000","last_fullsized":"2023-10-09T10:50:42.362357+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:29.134747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":607,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.365237+0000","last_change":"2023-10-09T10:49:58.796806+0000","last_active":"2023-10-09T10:50:43.365237+0000","last_peered":"2023-10-09T10:50:43.365237+0000","last_clean":"2023-10-09T10:50:43.365237+0000","last_became_active":"2023-10-09T10:49:58.796478+0000","last_became_peered":"2023-10-09T10:49:58.796478+0000","last_unstale":"2023-10-09T10:50:43.365237+0000","last_undegraded":"2023-10-09T10:50:43.365237+0000","last_fullsized":"2023-10-09T10:50:43.365237+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:44:35.791052+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":153,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.366901+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:50:42.366901+0000","last_peered":"2023-10-09T10:50:42.366901+0000","last_clean":"2023-10-09T10:50:42.366901+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:50:42.366901+0000","last_undegraded":"2023-10-09T10:50:42.366901+0000","last_fullsized":"2023-10-09T10:50:42.366901+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":119,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.380681+0000","last_change":"2023-10-09T10:50:26.518090+0000","last_active":"2023-10-09T10:50:42.380681+0000","last_peered":"2023-10-09T10:50:42.380681+0000","last_clean":"2023-10-09T10:50:42.380681+0000","last_became_active":"2023-10-09T10:50:26.517729+0000","last_became_peered":"2023-10-09T10:50:26.517729+0000","last_unstale":"2023-10-09T10:50:42.380681+0000","last_undegraded":"2023-10-09T10:50:42.380681+0000","last_fullsized":"2023-10-09T10:50:42.380681+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:51.254682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"225'5571","reported_seq":5629,"reported_epoch":225,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:47.964869+0000","last_change":"2023-10-09T10:49:57.053554+0000","last_active":"2023-10-09T10:50:47.964869+0000","last_peered":"2023-10-09T10:50:47.964869+0000","last_clean":"2023-10-09T10:49:39.922158+0000","last_became_active":"2023-10-09T10:49:57.045910+0000","last_became_peered":"2023-10-09T10:49:57.045910+0000","last_unstale":"2023-10-09T10:50:47.964869+0000","last_undegraded":"2023-10-09T10:49:56.942718+0000","last_fullsized":"2023-10-09T10:49:56.803232+0000","mapping_epoch":209,"log_start":"199'1000","ondisk_log_start":"199'1000","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4571,"log_dups_size":0,"ondisk_log_size":4571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:46.585592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365101056,"num_objects":5571,"num_object_clones":0,"num_object_copies":16713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5515,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5571,"num_write_kb":356544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,6","objects":793},{"shards":"0,1,7","objects":2361},{"shards":"0,6","objects":2417}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":420,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.384348+0000","last_change":"2023-10-09T10:50:26.517775+0000","last_active":"2023-10-09T10:50:42.384348+0000","last_peered":"2023-10-09T10:50:42.384348+0000","last_clean":"2023-10-09T10:50:42.384348+0000","last_became_active":"2023-10-09T10:50:26.517440+0000","last_became_peered":"2023-10-09T10:50:26.517440+0000","last_unstale":"2023-10-09T10:50:42.384348+0000","last_undegraded":"2023-10-09T10:50:42.384348+0000","last_fullsized":"2023-10-09T10:50:42.384348+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":223,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.365288+0000","last_change":"2023-10-09T10:49:55.837610+0000","last_active":"2023-10-09T10:50:43.365288+0000","last_peered":"2023-10-09T10:50:43.365288+0000","last_clean":"2023-10-09T10:50:43.365288+0000","last_became_active":"2023-10-09T10:49:55.837314+0000","last_became_peered":"2023-10-09T10:49:55.837314+0000","last_unstale":"2023-10-09T10:50:43.365288+0000","last_undegraded":"2023-10-09T10:50:43.365288+0000","last_fullsized":"2023-10-09T10:50:43.365288+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:01.185522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":104,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.369910+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:50:43.369910+0000","last_peered":"2023-10-09T10:50:43.369910+0000","last_clean":"2023-10-09T10:50:43.369910+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:50:43.369910+0000","last_undegraded":"2023-10-09T10:50:43.369910+0000","last_fullsized":"2023-10-09T10:50:43.369910+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":122,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.368593+0000","last_change":"2023-10-09T10:49:55.838212+0000","last_active":"2023-10-09T10:50:43.368593+0000","last_peered":"2023-10-09T10:50:43.368593+0000","last_clean":"2023-10-09T10:50:43.368593+0000","last_became_active":"2023-10-09T10:49:55.837374+0000","last_became_peered":"2023-10-09T10:49:55.837374+0000","last_unstale":"2023-10-09T10:50:43.368593+0000","last_undegraded":"2023-10-09T10:50:43.368593+0000","last_fullsized":"2023-10-09T10:50:43.368593+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:54.825127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"224'5497","reported_seq":5545,"reported_epoch":224,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:46.220571+0000","last_change":"2023-10-09T10:49:56.979655+0000","last_active":"2023-10-09T10:50:46.220571+0000","last_peered":"2023-10-09T10:50:46.220571+0000","last_clean":"2023-10-09T10:49:54.774195+0000","last_became_active":"2023-10-09T10:49:56.969698+0000","last_became_peered":"2023-10-09T10:49:56.969698+0000","last_unstale":"2023-10-09T10:50:46.220571+0000","last_undegraded":"2023-10-09T10:49:56.911686+0000","last_fullsized":"2023-10-09T10:49:56.808103+0000","mapping_epoch":209,"log_start":"199'600","ondisk_log_start":"199'600","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4897,"log_dups_size":0,"ondisk_log_size":4897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:47.019837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360251392,"num_objects":5497,"num_object_clones":0,"num_object_copies":16491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3175,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5497,"num_write_kb":351808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":3175},{"shards":"3,5","objects":2322}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":420,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.371455+0000","last_change":"2023-10-09T10:50:26.514252+0000","last_active":"2023-10-09T10:50:42.371455+0000","last_peered":"2023-10-09T10:50:42.371455+0000","last_clean":"2023-10-09T10:50:42.371455+0000","last_became_active":"2023-10-09T10:50:26.513924+0000","last_became_peered":"2023-10-09T10:50:26.513924+0000","last_unstale":"2023-10-09T10:50:42.371455+0000","last_undegraded":"2023-10-09T10:50:42.371455+0000","last_fullsized":"2023-10-09T10:50:42.371455+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.475429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.475429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":165,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.366477+0000","last_change":"2023-10-09T10:49:55.820384+0000","last_active":"2023-10-09T10:50:43.366477+0000","last_peered":"2023-10-09T10:50:43.366477+0000","last_clean":"2023-10-09T10:50:43.366477+0000","last_became_active":"2023-10-09T10:49:55.820044+0000","last_became_peered":"2023-10-09T10:49:55.820044+0000","last_unstale":"2023-10-09T10:50:43.366477+0000","last_undegraded":"2023-10-09T10:50:43.366477+0000","last_fullsized":"2023-10-09T10:50:43.366477+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:58.595200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":133,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.359998+0000","last_change":"2023-10-09T10:49:55.824348+0000","last_active":"2023-10-09T10:50:42.359998+0000","last_peered":"2023-10-09T10:50:42.359998+0000","last_clean":"2023-10-09T10:50:42.359998+0000","last_became_active":"2023-10-09T10:49:55.823990+0000","last_became_peered":"2023-10-09T10:49:55.823990+0000","last_unstale":"2023-10-09T10:50:42.359998+0000","last_undegraded":"2023-10-09T10:50:42.359998+0000","last_fullsized":"2023-10-09T10:50:42.359998+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:33.050849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":113,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.366012+0000","last_change":"2023-10-09T10:49:55.837070+0000","last_active":"2023-10-09T10:50:43.366012+0000","last_peered":"2023-10-09T10:50:43.366012+0000","last_clean":"2023-10-09T10:50:43.366012+0000","last_became_active":"2023-10-09T10:49:55.836747+0000","last_became_peered":"2023-10-09T10:49:55.836747+0000","last_unstale":"2023-10-09T10:50:43.366012+0000","last_undegraded":"2023-10-09T10:50:43.366012+0000","last_fullsized":"2023-10-09T10:50:43.366012+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:45.683756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"224'5387","reported_seq":5424,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:44.699018+0000","last_change":"2023-10-09T10:49:16.845103+0000","last_active":"2023-10-09T10:50:44.699018+0000","last_peered":"2023-10-09T10:50:44.699018+0000","last_clean":"2023-10-09T10:50:44.699018+0000","last_became_active":"2023-10-09T10:49:16.844812+0000","last_became_peered":"2023-10-09T10:49:16.844812+0000","last_unstale":"2023-10-09T10:50:44.699018+0000","last_undegraded":"2023-10-09T10:50:44.699018+0000","last_fullsized":"2023-10-09T10:50:44.699018+0000","mapping_epoch":196,"log_start":"199'800","ondisk_log_start":"199'800","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4587,"log_dups_size":0,"ondisk_log_size":4587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:17:27.590153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353042432,"num_objects":5387,"num_object_clones":0,"num_object_copies":16161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5387,"num_write_kb":344768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":579,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.363896+0000","last_change":"2023-10-09T10:49:18.829156+0000","last_active":"2023-10-09T10:50:42.363896+0000","last_peered":"2023-10-09T10:50:42.363896+0000","last_clean":"2023-10-09T10:50:42.363896+0000","last_became_active":"2023-10-09T10:49:17.848031+0000","last_became_peered":"2023-10-09T10:49:17.848031+0000","last_unstale":"2023-10-09T10:50:42.363896+0000","last_undegraded":"2023-10-09T10:50:42.363896+0000","last_fullsized":"2023-10-09T10:50:42.363896+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.829081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.829081+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:25.174844+0000","scrub_duration":0.001277842,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":231,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.356315+0000","last_change":"2023-10-09T10:49:40.959630+0000","last_active":"2023-10-09T10:50:42.356315+0000","last_peered":"2023-10-09T10:50:42.356315+0000","last_clean":"2023-10-09T10:50:42.356315+0000","last_became_active":"2023-10-09T10:49:40.945271+0000","last_became_peered":"2023-10-09T10:49:40.945271+0000","last_unstale":"2023-10-09T10:50:42.356315+0000","last_undegraded":"2023-10-09T10:50:42.356315+0000","last_fullsized":"2023-10-09T10:50:42.356315+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:37.426204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":35,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.371623+0000","last_change":"2023-10-09T10:50:26.519006+0000","last_active":"2023-10-09T10:50:42.371623+0000","last_peered":"2023-10-09T10:50:42.371623+0000","last_clean":"2023-10-09T10:50:42.371623+0000","last_became_active":"2023-10-09T10:50:26.518686+0000","last_became_peered":"2023-10-09T10:50:26.518686+0000","last_unstale":"2023-10-09T10:50:42.371623+0000","last_undegraded":"2023-10-09T10:50:42.371623+0000","last_fullsized":"2023-10-09T10:50:42.371623+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:42:44.442153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":225,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.363500+0000","last_change":"2023-10-09T10:49:55.820039+0000","last_active":"2023-10-09T10:50:42.363500+0000","last_peered":"2023-10-09T10:50:42.363500+0000","last_clean":"2023-10-09T10:50:42.363500+0000","last_became_active":"2023-10-09T10:49:55.819236+0000","last_became_peered":"2023-10-09T10:49:55.819236+0000","last_unstale":"2023-10-09T10:50:42.363500+0000","last_undegraded":"2023-10-09T10:50:42.363500+0000","last_fullsized":"2023-10-09T10:50:42.363500+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:47:19.590311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"224'5479","reported_seq":5531,"reported_epoch":224,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:50:44.716041+0000","last_change":"2023-10-09T10:49:57.045487+0000","last_active":"2023-10-09T10:50:44.716041+0000","last_peered":"2023-10-09T10:50:44.716041+0000","last_clean":"2023-10-09T10:49:39.930672+0000","last_became_active":"2023-10-09T10:49:57.038487+0000","last_became_peered":"2023-10-09T10:49:57.038487+0000","last_unstale":"2023-10-09T10:50:44.716041+0000","last_undegraded":"2023-10-09T10:49:56.923222+0000","last_fullsized":"2023-10-09T10:49:56.809367+0000","mapping_epoch":209,"log_start":"199'900","ondisk_log_start":"199'900","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4579,"log_dups_size":0,"ondisk_log_size":4579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:29:54.702472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359071744,"num_objects":5479,"num_object_clones":0,"num_object_copies":16437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5690,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5479,"num_write_kb":350656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":1},{"shards":"1,3,6","objects":752},{"shards":"1,3,7","objects":2468},{"shards":"3,6","objects":2258}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":583,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.357618+0000","last_change":"2023-10-09T10:49:43.787701+0000","last_active":"2023-10-09T10:50:42.357618+0000","last_peered":"2023-10-09T10:50:42.357618+0000","last_clean":"2023-10-09T10:50:42.357618+0000","last_became_active":"2023-10-09T10:49:43.787366+0000","last_became_peered":"2023-10-09T10:49:43.787366+0000","last_unstale":"2023-10-09T10:50:42.357618+0000","last_undegraded":"2023-10-09T10:50:42.357618+0000","last_fullsized":"2023-10-09T10:50:42.357618+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":99,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.370145+0000","last_change":"2023-10-09T10:50:26.524463+0000","last_active":"2023-10-09T10:50:42.370145+0000","last_peered":"2023-10-09T10:50:42.370145+0000","last_clean":"2023-10-09T10:50:42.370145+0000","last_became_active":"2023-10-09T10:50:26.523476+0000","last_became_peered":"2023-10-09T10:50:26.523476+0000","last_unstale":"2023-10-09T10:50:42.370145+0000","last_undegraded":"2023-10-09T10:50:42.370145+0000","last_fullsized":"2023-10-09T10:50:42.370145+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:43.514374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":112,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.368681+0000","last_change":"2023-10-09T10:49:55.850829+0000","last_active":"2023-10-09T10:50:43.368681+0000","last_peered":"2023-10-09T10:50:43.368681+0000","last_clean":"2023-10-09T10:50:43.368681+0000","last_became_active":"2023-10-09T10:49:55.837771+0000","last_became_peered":"2023-10-09T10:49:55.837771+0000","last_unstale":"2023-10-09T10:50:43.368681+0000","last_undegraded":"2023-10-09T10:50:43.368681+0000","last_fullsized":"2023-10-09T10:50:43.368681+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:07.101559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.384204+0000","last_change":"2023-10-09T10:50:43.384204+0000","last_active":"2023-10-09T10:50:43.384204+0000","last_peered":"2023-10-09T10:50:43.384204+0000","last_clean":"2023-10-09T10:50:43.384204+0000","last_became_active":"2023-10-09T10:50:43.383948+0000","last_became_peered":"2023-10-09T10:50:43.383948+0000","last_unstale":"2023-10-09T10:50:43.384204+0000","last_undegraded":"2023-10-09T10:50:43.384204+0000","last_fullsized":"2023-10-09T10:50:43.384204+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:13:15.523453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":112,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.366110+0000","last_change":"2023-10-09T10:49:55.830269+0000","last_active":"2023-10-09T10:50:43.366110+0000","last_peered":"2023-10-09T10:50:43.366110+0000","last_clean":"2023-10-09T10:50:43.366110+0000","last_became_active":"2023-10-09T10:49:55.829637+0000","last_became_peered":"2023-10-09T10:49:55.829637+0000","last_unstale":"2023-10-09T10:50:43.366110+0000","last_undegraded":"2023-10-09T10:50:43.366110+0000","last_fullsized":"2023-10-09T10:50:43.366110+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:39.579608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":103,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.357770+0000","last_change":"2023-10-09T10:49:55.817112+0000","last_active":"2023-10-09T10:50:42.357770+0000","last_peered":"2023-10-09T10:50:42.357770+0000","last_clean":"2023-10-09T10:50:42.357770+0000","last_became_active":"2023-10-09T10:49:55.815889+0000","last_became_peered":"2023-10-09T10:49:55.815889+0000","last_unstale":"2023-10-09T10:50:42.357770+0000","last_undegraded":"2023-10-09T10:50:42.357770+0000","last_fullsized":"2023-10-09T10:50:42.357770+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:41.374680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":205,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.359882+0000","last_change":"2023-10-09T10:49:55.831015+0000","last_active":"2023-10-09T10:50:42.359882+0000","last_peered":"2023-10-09T10:50:42.359882+0000","last_clean":"2023-10-09T10:50:42.359882+0000","last_became_active":"2023-10-09T10:49:55.816720+0000","last_became_peered":"2023-10-09T10:49:55.816720+0000","last_unstale":"2023-10-09T10:50:42.359882+0000","last_undegraded":"2023-10-09T10:50:42.359882+0000","last_fullsized":"2023-10-09T10:50:42.359882+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:05:36.526668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":203,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.389434+0000","last_change":"2023-10-09T10:50:43.388454+0000","last_active":"2023-10-09T10:50:43.389434+0000","last_peered":"2023-10-09T10:50:43.389434+0000","last_clean":"2023-10-09T10:50:43.389434+0000","last_became_active":"2023-10-09T10:50:43.388148+0000","last_became_peered":"2023-10-09T10:50:43.388148+0000","last_unstale":"2023-10-09T10:50:43.389434+0000","last_undegraded":"2023-10-09T10:50:43.389434+0000","last_fullsized":"2023-10-09T10:50:43.389434+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:49:07.119296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":174,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.360146+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:50:42.360146+0000","last_peered":"2023-10-09T10:50:42.360146+0000","last_clean":"2023-10-09T10:50:42.360146+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:50:42.360146+0000","last_undegraded":"2023-10-09T10:50:42.360146+0000","last_fullsized":"2023-10-09T10:50:42.360146+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":166,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.387422+0000","last_change":"2023-10-09T10:50:43.385150+0000","last_active":"2023-10-09T10:50:43.387422+0000","last_peered":"2023-10-09T10:50:43.387422+0000","last_clean":"2023-10-09T10:50:43.387422+0000","last_became_active":"2023-10-09T10:50:43.384788+0000","last_became_peered":"2023-10-09T10:50:43.384788+0000","last_unstale":"2023-10-09T10:50:43.387422+0000","last_undegraded":"2023-10-09T10:50:43.387422+0000","last_fullsized":"2023-10-09T10:50:43.387422+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:36.085910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":218,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.355984+0000","last_change":"2023-10-09T10:49:55.821623+0000","last_active":"2023-10-09T10:50:42.355984+0000","last_peered":"2023-10-09T10:50:42.355984+0000","last_clean":"2023-10-09T10:50:42.355984+0000","last_became_active":"2023-10-09T10:49:55.821367+0000","last_became_peered":"2023-10-09T10:49:55.821367+0000","last_unstale":"2023-10-09T10:50:42.355984+0000","last_undegraded":"2023-10-09T10:50:42.355984+0000","last_fullsized":"2023-10-09T10:50:42.355984+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:09.753526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":239,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.348023+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:50:42.348023+0000","last_peered":"2023-10-09T10:50:42.348023+0000","last_clean":"2023-10-09T10:50:42.348023+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:50:42.348023+0000","last_undegraded":"2023-10-09T10:50:42.348023+0000","last_fullsized":"2023-10-09T10:50:42.348023+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":116,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-10-09T10:50:43.370857+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:50:43.370857+0000","last_peered":"2023-10-09T10:50:43.370857+0000","last_clean":"2023-10-09T10:50:43.370857+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:50:43.370857+0000","last_undegraded":"2023-10-09T10:50:43.370857+0000","last_fullsized":"2023-10-09T10:50:43.370857+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":198,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T10:50:42.361760+0000","last_change":"2023-10-09T10:49:55.814865+0000","last_active":"2023-10-09T10:50:42.361760+0000","last_peered":"2023-10-09T10:50:42.361760+0000","last_clean":"2023-10-09T10:50:42.361760+0000","last_became_active":"2023-10-09T10:49:55.814453+0000","last_became_peered":"2023-10-09T10:49:55.814453+0000","last_unstale":"2023-10-09T10:50:42.361760+0000","last_undegraded":"2023-10-09T10:50:42.361760+0000","last_fullsized":"2023-10-09T10:50:42.361760+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:05:10.291518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":5747834880,"num_objects":87705,"num_object_clones":0,"num_object_copies":263115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30700,"num_objects_misplaced":1384,"num_objects_unfound":0,"num_objects_dirty":87705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":87705,"num_write_kb":5613120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7540,"num_bytes_recovered":494141440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17623941120,"data_stored":17623941120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74164,"ondisk_log_size":74164,"up":48,"acting":39,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1312,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":739149,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":338,"num_read_kb":636,"num_write":166,"num_write_kb":1683,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1497093,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":219043332290,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":44,"seq":188978561221,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3240840,"kb_used_data":2927168,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":90504312,"statfs":{"total":95995035648,"available":92676415488,"internally_reserved":0,"allocated":2997420032,"data_stored":2996593563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:49:47 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":1.6339999999999999,"15min":0.94099999999999995},"min":{"1min":0.27000000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.288,"5min":12.288,"15min":12.288},"last":6.5970000000000004},{"interface":"front","average":{"1min":1.4630000000000001,"5min":1.5509999999999999,"15min":0.91900000000000004},"min":{"1min":0.25,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":12.369999999999999,"5min":12.369999999999999,"15min":12.369999999999999},"last":11.845000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:49:54 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.7749999999999999,"15min":0.98399999999999999},"min":{"1min":0.26500000000000001,"5min":0.245,"15min":0.245},"max":{"1min":3.415,"5min":16.747,"15min":16.747},"last":7.2629999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.873,"15min":0.99099999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.4260000000000002,"5min":16.734000000000002,"15min":16.734000000000002},"last":6.3799999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:50:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.696,"15min":0.92600000000000005},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.3479999999999999,"5min":13.208,"15min":13.208},"last":6.4610000000000003},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.903,"15min":1.0369999999999999},"min":{"1min":0.29999999999999999,"5min":0.28599999999999998,"15min":0.28399999999999997},"max":{"1min":5.2539999999999996,"5min":12.571,"15min":12.571},"last":5.173}]},{"osd":3,"last update":"Mon Oct 9 10:49:47 2023","interfaces":[{"interface":"back","average":{"1min":1.478,"5min":1.6839999999999999,"15min":0.92000000000000004},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.255},"max":{"1min":12.391,"5min":12.391,"15min":12.391},"last":7.8250000000000002},{"interface":"front","average":{"1min":1.5089999999999999,"5min":1.919,"15min":1.004},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.9429999999999996,"5min":13.179,"15min":13.179},"last":6.5060000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:50:14 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.3440000000000001,"15min":0.751},"min":{"1min":0.27300000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.4569999999999999,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":4.6180000000000003},{"interface":"front","average":{"1min":1.3069999999999999,"5min":1.514,"15min":0.83399999999999996},"min":{"1min":0.28799999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.6970000000000001,"5min":12.481999999999999,"15min":12.481999999999999},"last":6.5430000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:50:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":1.4770000000000001,"15min":0.82499999999999996},"min":{"1min":0.29999999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.9589999999999996,"5min":10.497,"15min":10.497},"last":4.0140000000000002},{"interface":"front","average":{"1min":1.05,"5min":1.47,"15min":0.84899999999999998},"min":{"1min":0.318,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":6.6710000000000003,"5min":12.456,"15min":12.456},"last":6.5739999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":1.712,"15min":0.93899999999999995},"min":{"1min":0.34100000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.9820000000000002,"5min":23.358000000000001,"15min":23.358000000000001},"last":3.9769999999999999},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.603,"15min":0.93100000000000005},"min":{"1min":0.36299999999999999,"5min":0.35099999999999998,"15min":0.34000000000000002},"max":{"1min":6.7210000000000001,"5min":12.422000000000001,"15min":12.422000000000001},"last":6.6319999999999997}]}]},{"osd":1,"up_from":0,"seq":365072220260,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":80,"seq":343597383794,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3580988,"kb_used_data":3329844,"kb_used_omap":28,"kb_used_meta":251107,"kb_avail":90164164,"statfs":{"total":95995035648,"available":92328103936,"internally_reserved":0,"allocated":3409760256,"data_stored":3408544323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":257134416},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,0,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:49:50 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":2.512,"15min":1.087},"min":{"1min":0.25,"5min":0.191,"15min":0.191},"max":{"1min":14.311,"5min":25.475000000000001,"15min":25.475000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":1.2749999999999999,"5min":2.347,"15min":1.0569999999999999},"min":{"1min":0.24099999999999999,"5min":0.188,"15min":0.188},"max":{"1min":6.8109999999999999,"5min":20.832999999999998,"15min":20.832999999999998},"last":0.78700000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:50:22 2023","interfaces":[{"interface":"back","average":{"1min":2.6059999999999999,"5min":2.1280000000000001,"15min":1.036},"min":{"1min":0.23999999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":17.927,"5min":17.927,"15min":17.927},"last":0.315},{"interface":"front","average":{"1min":2.117,"5min":2.3809999999999998,"15min":1.0960000000000001},"min":{"1min":0.28799999999999998,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.8089999999999993,"5min":14.391999999999999,"15min":14.391999999999999},"last":1.5349999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:49:50 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":2.5459999999999998,"15min":1.282},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":6.782,"5min":14.926,"15min":14.926},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.1459999999999999,"5min":2.5169999999999999,"15min":1.2},"min":{"1min":0.23499999999999999,"5min":0.224,"15min":0.224},"max":{"1min":14.391999999999999,"5min":14.391999999999999,"15min":14.391999999999999},"last":0.29299999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":2.2770000000000001,"5min":2.0030000000000001,"15min":1.159},"min":{"1min":0.34699999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.6129999999999995,"5min":9.6129999999999995,"15min":9.6129999999999995},"last":1.6220000000000001},{"interface":"front","average":{"1min":2.9470000000000001,"5min":2.4489999999999998,"15min":1.3140000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.1790000000000003,"5min":12.318,"15min":12.318},"last":1.754}]},{"osd":5,"last update":"Mon Oct 9 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":2.4129999999999998,"15min":1.304},"min":{"1min":0.35399999999999998,"5min":0.318,"15min":0.318},"max":{"1min":7.1840000000000002,"5min":12.157,"15min":12.157},"last":0.49199999999999999},{"interface":"front","average":{"1min":2.9700000000000002,"5min":2.177,"15min":1.226},"min":{"1min":0.317,"5min":0.317,"15min":0.29499999999999998},"max":{"1min":14.446,"5min":14.446,"15min":14.446},"last":6.3099999999999996}]},{"osd":6,"last update":"Mon Oct 9 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":3.0619999999999998,"5min":2.548,"15min":1.3540000000000001},"min":{"1min":0.41499999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.734,"5min":12.207000000000001,"15min":12.207000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.738,"5min":1.768,"15min":1.1020000000000001},"min":{"1min":0.34300000000000003,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":14.420999999999999,"5min":14.420999999999999,"15min":14.420999999999999},"last":0.64600000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":3.2269999999999999,"5min":2.3490000000000002,"15min":1.2889999999999999},"min":{"1min":0.45600000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":14.48,"5min":14.48,"15min":14.48},"last":0.53800000000000003},{"interface":"front","average":{"1min":2.0720000000000001,"5min":2.1469999999999998,"15min":1.254},"min":{"1min":0.438,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.6440000000000001,"5min":14.394,"15min":14.394},"last":3.3759999999999999}]}]},{"osd":2,"up_from":0,"seq":386547056734,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893208,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4330784,"kb_used_data":3906776,"kb_used_omap":7,"kb_used_meta":423992,"kb_avail":89414368,"statfs":{"total":95995035648,"available":91560312832,"internally_reserved":0,"allocated":4000538624,"data_stored":3999338187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":434168424},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:49:46 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":2.3730000000000002,"15min":1.0049999999999999},"min":{"1min":0.20899999999999999,"5min":0.183,"15min":0.183},"max":{"1min":8.4339999999999993,"5min":15.255000000000001,"15min":15.255000000000001},"last":0.25},{"interface":"front","average":{"1min":3.1120000000000001,"5min":2.1949999999999998,"15min":1.0029999999999999},"min":{"1min":0.23200000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":14.798,"5min":14.798,"15min":14.798},"last":0.23300000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:49:46 2023","interfaces":[{"interface":"back","average":{"1min":2.661,"5min":2.8260000000000001,"15min":1.1619999999999999},"min":{"1min":0.21099999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":11.757,"5min":14.707000000000001,"15min":14.707000000000001},"last":0.317},{"interface":"front","average":{"1min":3.056,"5min":2.3460000000000001,"15min":1.0309999999999999},"min":{"1min":0.30199999999999999,"5min":0.193,"15min":0.193},"max":{"1min":10.59,"5min":15.443,"15min":15.443},"last":0.34100000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:49:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7280000000000002,"5min":2.5350000000000001,"15min":1.089},"min":{"1min":0.24099999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.795,"5min":13.853,"15min":13.853},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.964,"5min":2.5449999999999999,"15min":1.1120000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.8569999999999993,"5min":10.557,"15min":10.557},"last":0.28000000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:49:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4340000000000002,"5min":2.294,"15min":1.1459999999999999},"min":{"1min":0.34799999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.976,"5min":20.388000000000002,"15min":20.388000000000002},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.972,"5min":2.3340000000000001,"15min":1.1830000000000001},"min":{"1min":0.32100000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.9560000000000004,"5min":19.085999999999999,"15min":19.085999999999999},"last":0.30299999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:49:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2629999999999999,"5min":2.7730000000000001,"15min":1.3149999999999999},"min":{"1min":0.35099999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.619999999999999,"5min":51.509999999999998,"15min":51.509999999999998},"last":7.9640000000000004},{"interface":"front","average":{"1min":2.347,"5min":3.0750000000000002,"15min":1.3999999999999999},"min":{"1min":0.45000000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":10.798999999999999,"5min":55.338999999999999,"15min":55.338999999999999},"last":2.6389999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:49:46 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":2.0979999999999999,"15min":1.091},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":8.5960000000000001,"5min":12.361000000000001,"15min":12.361000000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":2.2229999999999999,"5min":2.544,"15min":1.248},"min":{"1min":0.45000000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.569,"5min":13.891999999999999,"15min":13.891999999999999},"last":7.4800000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:49:46 2023","interfaces":[{"interface":"back","average":{"1min":2.6040000000000001,"5min":3.27,"15min":1.484},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":11.819000000000001,"5min":51.744,"15min":51.744},"last":3.2949999999999999},{"interface":"front","average":{"1min":2.3490000000000002,"5min":2.895,"15min":1.347},"min":{"1min":0.39200000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.1900000000000004,"5min":53.399000000000001,"15min":53.399000000000001},"last":0.505}]}]},{"osd":4,"up_from":30,"seq":128849019084,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2810952,"kb_used_data":2501376,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":90934200,"statfs":{"total":95995035648,"available":93116620800,"internally_reserved":0,"allocated":2561409024,"data_stored":2560581822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:50:41 2023","interfaces":[{"interface":"back","average":{"1min":2.3319999999999999,"5min":1.6399999999999999,"15min":0.95299999999999996},"min":{"1min":0.217,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.4329999999999998,"5min":10.387,"15min":10.387},"last":2.6240000000000001},{"interface":"front","average":{"1min":2.2090000000000001,"5min":1.8660000000000001,"15min":1.081},"min":{"1min":0.28999999999999998,"5min":0.217,"15min":0.217},"max":{"1min":9.2889999999999997,"5min":10.311999999999999,"15min":10.311999999999999},"last":1.22}]},{"osd":1,"last update":"Mon Oct 9 10:49:44 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.5329999999999999,"15min":0.89600000000000002},"min":{"1min":0.26400000000000001,"5min":0.249,"15min":0.249},"max":{"1min":3.7919999999999998,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.2210000000000001,"5min":1.79,"15min":1.0209999999999999},"min":{"1min":0.33500000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.6269999999999998,"5min":8.7669999999999995,"15min":8.7669999999999995},"last":0.625}]},{"osd":2,"last update":"Mon Oct 9 10:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.7669999999999999,"15min":1.018},"min":{"1min":0.28599999999999998,"5min":0.253,"15min":0.253},"max":{"1min":5.335,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":1.3380000000000001},{"interface":"front","average":{"1min":1.629,"5min":2.2770000000000001,"15min":1.1719999999999999},"min":{"1min":0.35399999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.077,"5min":20.545999999999999,"15min":20.545999999999999},"last":2.2149999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:50:41 2023","interfaces":[{"interface":"back","average":{"1min":2.6669999999999998,"5min":2.0960000000000001,"15min":1.224},"min":{"1min":0.29299999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.9000000000000004,"5min":9.0269999999999992,"15min":9.0269999999999992},"last":1.879},{"interface":"front","average":{"1min":1.9259999999999999,"5min":2.4790000000000001,"15min":1.2490000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":9.2249999999999996,"5min":9.6929999999999996,"15min":9.6929999999999996},"last":1.1899999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:49:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.919,"15min":0.94999999999999996},"min":{"1min":0.27900000000000003,"5min":0.224,"15min":0.224},"max":{"1min":9.0020000000000007,"5min":24.149999999999999,"15min":24.149999999999999},"last":7.6710000000000003},{"interface":"front","average":{"1min":1.579,"5min":1.8720000000000001,"15min":0.94199999999999995},"min":{"1min":0.29199999999999998,"5min":0.252,"15min":0.252},"max":{"1min":15.071999999999999,"5min":20.422000000000001,"15min":20.422000000000001},"last":7.3150000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:50:14 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.6259999999999999,"15min":0.86799999999999999},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":2.9289999999999998,"5min":14.897,"15min":14.897},"last":0.24099999999999999},{"interface":"front","average":{"1min":1.7809999999999999,"5min":1.712,"15min":0.95899999999999996},"min":{"1min":0.36299999999999999,"5min":0.311,"15min":0.311},"max":{"1min":7.8499999999999996,"5min":12.539,"15min":12.539},"last":0.54300000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:50:25 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":1.6850000000000001,"15min":0.95499999999999996},"min":{"1min":0.33700000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.7240000000000002,"5min":9.3360000000000003,"15min":9.3360000000000003},"last":2.012},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.464,"15min":0.875},"min":{"1min":0.33900000000000002,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":5.5129999999999999,"5min":7.423,"15min":7.423},"last":2.097}]}]},{"osd":5,"up_from":37,"seq":158913790153,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2314748,"kb_used_data":2013364,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91430404,"statfs":{"total":95995035648,"available":93624733696,"internally_reserved":0,"allocated":2061684736,"data_stored":2060865809,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:49:47 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.2849999999999999,"15min":0.82499999999999996},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.5230000000000001,"5min":11.648999999999999,"15min":11.648999999999999},"last":2.9769999999999999},{"interface":"front","average":{"1min":0.94099999999999995,"5min":1.3540000000000001,"15min":0.89300000000000002},"min":{"1min":0.30399999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.5459999999999998,"5min":11.68,"15min":11.68},"last":2.9409999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:49:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.8819999999999999,"15min":1.0289999999999999},"min":{"1min":0.26500000000000001,"5min":0.251,"15min":0.251},"max":{"1min":6.609,"5min":12.999000000000001,"15min":12.999000000000001},"last":4.0730000000000004},{"interface":"front","average":{"1min":0.73499999999999999,"5min":1.512,"15min":0.91500000000000004},"min":{"1min":0.254,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":2.391,"5min":7.3730000000000002,"15min":7.3730000000000002},"last":2.8500000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:50:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":1.8799999999999999,"15min":1.0349999999999999},"min":{"1min":0.377,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.5529999999999999,"5min":11.393000000000001,"15min":11.393000000000001},"last":4.1059999999999999},{"interface":"front","average":{"1min":0.78900000000000003,"5min":1.4390000000000001,"15min":0.86799999999999999},"min":{"1min":0.32200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.2799999999999998,"5min":10.352,"15min":10.352},"last":0.40500000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:49:51 2023","interfaces":[{"interface":"back","average":{"1min":1.256,"5min":2.1539999999999999,"15min":1.2},"min":{"1min":0.36099999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.7000000000000002,"5min":18.016999999999999,"15min":18.016999999999999},"last":1.403},{"interface":"front","average":{"1min":1.714,"5min":2.3090000000000002,"15min":1.1779999999999999},"min":{"1min":0.34000000000000002,"5min":0.317,"15min":0.317},"max":{"1min":9.2490000000000006,"5min":15.555999999999999,"15min":15.555999999999999},"last":4.2640000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:49:54 2023","interfaces":[{"interface":"back","average":{"1min":6.6059999999999999,"5min":2.29,"15min":1.075},"min":{"1min":0.24099999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":123.241,"5min":123.241,"15min":123.241},"last":3.9199999999999999},{"interface":"front","average":{"1min":7.2119999999999997,"5min":2.7029999999999998,"15min":1.232},"min":{"1min":0.26700000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":123.318,"5min":123.318,"15min":123.318},"last":4.3029999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:50:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.569,"15min":0.86399999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28100000000000003},"max":{"1min":5.0129999999999999,"5min":7.7060000000000004,"15min":7.7060000000000004},"last":6.7990000000000004},{"interface":"front","average":{"1min":0.52000000000000002,"5min":1.288,"15min":0.79100000000000004},"min":{"1min":0.33800000000000002,"5min":0.318,"15min":0.30299999999999999},"max":{"1min":0.89800000000000002,"5min":20.256,"15min":20.256},"last":0.88100000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:50:31 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":1.615,"15min":0.92600000000000005},"min":{"1min":0.32300000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":10.739000000000001,"5min":10.739000000000001,"15min":10.739000000000001},"last":4.125},{"interface":"front","average":{"1min":1.014,"5min":1.3380000000000001,"15min":0.85399999999999998},"min":{"1min":0.35299999999999998,"5min":0.33200000000000002,"15min":0.31900000000000001},"max":{"1min":3.694,"5min":14.154,"15min":14.154},"last":0.44}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3435331584,"data_stored":3435331584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1658322944,"data_stored":1658322944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4035182592,"data_stored":4035182592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2575958016,"data_stored":2575958016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2067464192,"data_stored":2067464192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3013869568,"data_stored":3013869568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":837812224,"data_stored":837812224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:50:50.638 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:50:50.638 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 31 2023-10-09T10:50:50.661 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:50.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:50.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:50.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:50.831 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:51.017 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total time run: 90.0132 2023-10-09T10:50:51.017 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total writes made: 91134 2023-10-09T10:50:51.017 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Write size: 65536 2023-10-09T10:50:51.017 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object size: 65536 2023-10-09T10:50:51.017 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Bandwidth (MB/sec): 63.2782 2023-10-09T10:50:51.017 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Bandwidth: 38.5631 2023-10-09T10:50:51.017 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max bandwidth (MB/sec): 178.375 2023-10-09T10:50:51.017 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min bandwidth (MB/sec): 0 2023-10-09T10:50:51.017 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average IOPS: 1012 2023-10-09T10:50:51.017 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev IOPS: 617.027 2023-10-09T10:50:51.018 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max IOPS: 2854 2023-10-09T10:50:51.018 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min IOPS: 0 2023-10-09T10:50:51.018 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average Latency(s): 0.015799 2023-10-09T10:50:51.018 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Latency(s): 0.0495256 2023-10-09T10:50:51.018 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max latency(s): 2.28015 2023-10-09T10:50:51.018 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min latency(s): 0.00131548 2023-10-09T10:50:51.028 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T10:50:51.124 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:50:51.199 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:50:51.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:51 smithi155 ceph-mon[184854]: from='client.35286 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:50:51.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:51 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T10:50:51.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:51 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:50:51.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:51 smithi125 ceph-mon[195507]: from='client.35286 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:50:51.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:51 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T10:50:51.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:51 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:50:51.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:51 smithi125 ceph-mon[200197]: from='client.35286 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:50:51.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:51 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T10:50:51.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:51 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:50:51.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:50:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:50:51] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.43.0" 2023-10-09T10:50:52.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:52 smithi155 ceph-mon[184854]: pgmap v515: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 127 B/s rd, 41 MiB/s wr, 658 op/s; 30694/268770 objects degraded (11.420%); 1384/268770 objects misplaced (0.515%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:50:52.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:52 smithi125 ceph-mon[200197]: pgmap v515: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 127 B/s rd, 41 MiB/s wr, 658 op/s; 30694/268770 objects degraded (11.420%); 1384/268770 objects misplaced (0.515%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:50:52.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:52 smithi125 ceph-mon[195507]: pgmap v515: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 127 B/s rd, 41 MiB/s wr, 658 op/s; 30694/268770 objects degraded (11.420%); 1384/268770 objects misplaced (0.515%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:50:52.655 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:50:52.655 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:50:53.135 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:50:53.200 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:53 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30694/268770 objects degraded (11.420%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:50:53.420 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:53 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30694/268770 objects degraded (11.420%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:50:53.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:53 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30694/268770 objects degraded (11.420%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:50:53.506 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:53.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:53.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:53.616 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:53.662 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:53.688 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:53.844 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:50:54.266 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:54 smithi155 ceph-mon[184854]: pgmap v516: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 103 B/s rd, 41 MiB/s wr, 706 op/s; 30540/271401 objects degraded (11.253%); 1084/271401 objects misplaced (0.399%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:50:54.426 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:54.453 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:54 smithi125 ceph-mon[195507]: pgmap v516: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 103 B/s rd, 41 MiB/s wr, 706 op/s; 30540/271401 objects degraded (11.253%); 1084/271401 objects misplaced (0.399%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:50:54.454 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:54 smithi125 ceph-mon[200197]: pgmap v516: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 103 B/s rd, 41 MiB/s wr, 706 op/s; 30540/271401 objects degraded (11.253%); 1084/271401 objects misplaced (0.399%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:50:54.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:54.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:54.557 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:54.618 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:54.711 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:54.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:54.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:54.772 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:54.815 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:50:54.881 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:54.916 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:50:54.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:54.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:50:55.020 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:55.091 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:55.115 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:55.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:55.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:55.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:55 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/938682816' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:55.184 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:50:55.223 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:55.246 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:50:55.246 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:50:55.268 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:55.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:55 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/938682816' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:55.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:55 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/938682816' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:50:55.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:55.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:55.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:55.449 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:55.495 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:55.667 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:50:55.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:55.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:55.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:55.826 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:50:55.896 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:55.933 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:50:55.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:55.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:50:56.037 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:56.110 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:56.136 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:50:56.310 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:56.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:56.315 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:50:56.315 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/938682816' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:50:56.315 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[195507]: osdmap e226: 8 total, 8 up, 5 in 2023-10-09T10:50:56.315 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[195507]: failed to encode map e226 with expected crc 2023-10-09T10:50:56.315 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[195507]: failed to encode map e226 with expected crc 2023-10-09T10:50:56.315 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[195507]: failed to encode map e226 with expected crc 2023-10-09T10:50:56.315 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:56.316 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[195507]: failed to encode map e226 with expected crc 2023-10-09T10:50:56.316 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:56.316 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[195507]: pgmap v518: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 124 B/s rd, 35 MiB/s wr, 699 op/s; 30210/269874 objects degraded (11.194%); 1084/269874 objects misplaced (0.402%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:50:56.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:56.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:50:56.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/938682816' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:50:56.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[200197]: osdmap e226: 8 total, 8 up, 5 in 2023-10-09T10:50:56.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[200197]: failed to encode map e226 with expected crc 2023-10-09T10:50:56.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[200197]: failed to encode map e226 with expected crc 2023-10-09T10:50:56.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[200197]: failed to encode map e226 with expected crc 2023-10-09T10:50:56.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:56.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[200197]: failed to encode map e226 with expected crc 2023-10-09T10:50:56.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:56.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:56 smithi125 ceph-mon[200197]: pgmap v518: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 124 B/s rd, 35 MiB/s wr, 699 op/s; 30210/269874 objects degraded (11.194%); 1084/269874 objects misplaced (0.402%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:50:56.417 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:56.445 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:56 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:50:56.445 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/938682816' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:50:56.445 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:56 smithi155 ceph-mon[184854]: osdmap e226: 8 total, 8 up, 5 in 2023-10-09T10:50:56.446 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:56 smithi155 ceph-mon[184854]: failed to encode map e226 with expected crc 2023-10-09T10:50:56.446 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:56 smithi155 ceph-mon[184854]: failed to encode map e226 with expected crc 2023-10-09T10:50:56.446 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:56 smithi155 ceph-mon[184854]: failed to encode map e226 with expected crc 2023-10-09T10:50:56.446 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:56 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:56.446 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:56 smithi155 ceph-mon[184854]: failed to encode map e226 with expected crc 2023-10-09T10:50:56.446 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:56 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:50:56.446 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:56 smithi155 ceph-mon[184854]: pgmap v518: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 124 B/s rd, 35 MiB/s wr, 699 op/s; 30210/269874 objects degraded (11.194%); 1084/269874 objects misplaced (0.402%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:50:56.471 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:56.702 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:56.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:56.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:56.812 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:56.864 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:50:56.888 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:56.904 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:50:56.936 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:50:56.966 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:50:57.018 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:50:57.022 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:57.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:57.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:57.066 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:50:57.096 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:50:57.132 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:57.132 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:50:57.160 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:50:57.160 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:50:57.161 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:50:57.161 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:50:57.161 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:50:57.161 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:50:57.161 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:50:57.161 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:50:57.195 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:57.239 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:57.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:57.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:57.303 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:50:57.303 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:50:57.303 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:50:57.303 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:50:57.304 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:50:57.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:57 smithi125 ceph-mon[195507]: failed to encode map e226 with expected crc 2023-10-09T10:50:57.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:57 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/392693015' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:57.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:57 smithi125 ceph-mon[200197]: failed to encode map e226 with expected crc 2023-10-09T10:50:57.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:57 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/392693015' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:57.343 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:50:57.409 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:57 smithi155 ceph-mon[184854]: failed to encode map e226 with expected crc 2023-10-09T10:50:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:57 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/392693015' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:50:57.538 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:50:57.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:57.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:50:57.645 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:50:57.714 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:50:57.718 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:57.747 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:50:57.750 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:50:57.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:57.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:50:57.861 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:57.919 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:58.002 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:50:58.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:58.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:50:58.111 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:58.141 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:50:58.174 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:58.207 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:58.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:58.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:58.245 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 1 pg_num to 31 2023-10-09T10:50:58.316 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:58.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:58 smithi125 ceph-mon[195507]: pgmap v519: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 102 B/s rd, 28 MiB/s wr, 696 op/s; 30019/266169 objects degraded (11.278%); 755/266169 objects misplaced (0.284%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T10:50:58.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:58 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3210918307' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T10:50:58.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:58 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T10:50:58.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:58 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30210/269874 objects degraded (11.194%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:50:58.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:58 smithi125 ceph-mon[200197]: pgmap v519: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 102 B/s rd, 28 MiB/s wr, 696 op/s; 30019/266169 objects degraded (11.278%); 755/266169 objects misplaced (0.284%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T10:50:58.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:58 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3210918307' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T10:50:58.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:58 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T10:50:58.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:58 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30210/269874 objects degraded (11.194%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:50:58.376 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:58.405 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:50:58.407 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:58.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:58.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:58.418 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:58 smithi155 ceph-mon[184854]: pgmap v519: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 102 B/s rd, 28 MiB/s wr, 696 op/s; 30019/266169 objects degraded (11.278%); 755/266169 objects misplaced (0.284%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T10:50:58.419 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:58 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3210918307' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T10:50:58.419 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:58 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T10:50:58.419 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:58 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30210/269874 objects degraded (11.194%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:50:58.513 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:58.556 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:58.783 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:58.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:58.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:58.888 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:50:58.959 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:58.993 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:50:59.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:59.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:50:59.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:50:59.151 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:59.281 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:50:59.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:59.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:50:59.386 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:50:59.394 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:50:59.395 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:50:59.395 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:50:59.395 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:50:59.395 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.395 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70444 6.3 6:c0306d38:::benchmark_data_smithi125_244951_object70443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:50:59.395 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.568354+0000", 2023-10-09T10:50:59.395 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 335.82230886100001, 2023-10-09T10:50:59.395 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047306648, 2023-10-09T10:50:59.395 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.395 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.395 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.396 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.396 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.396 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70444 2023-10-09T10:50:59.396 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.396 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.396 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.396 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.396 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568354+0000", 2023-10-09T10:50:59.396 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.396 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.396 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.396 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.397 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568354+0000", 2023-10-09T10:50:59.397 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.397 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.397 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.397 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.397 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568353+0000", 2023-10-09T10:50:59.397 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:59.397 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.397 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.397 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.398 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568387+0000", 2023-10-09T10:50:59.398 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4495000000000001e-05 2023-10-09T10:50:59.398 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.398 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.398 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.398 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568407+0000", 2023-10-09T10:50:59.398 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9729999999999999e-05 2023-10-09T10:50:59.398 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.398 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.398 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.399 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568412+0000", 2023-10-09T10:50:59.399 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4149999999999998e-06 2023-10-09T10:50:59.399 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.399 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.399 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.399 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568457+0000", 2023-10-09T10:50:59.399 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4270000000000001e-05 2023-10-09T10:50:59.399 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.399 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.399 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.399 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568587+0000", 2023-10-09T10:50:59.400 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013011399999999999 2023-10-09T10:50:59.400 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.400 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.400 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:50:59.400 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568753+0000", 2023-10-09T10:50:59.400 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016615099999999999 2023-10-09T10:50:59.400 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.400 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.400 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.400 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.570931+0000", 2023-10-09T10:50:59.400 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002177596 2023-10-09T10:50:59.401 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.401 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.401 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.401 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.570969+0000", 2023-10-09T10:50:59.401 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8661000000000001e-05 2023-10-09T10:50:59.401 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.401 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.401 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.401 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615586+0000", 2023-10-09T10:50:59.401 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.044616524999999997 2023-10-09T10:50:59.401 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.402 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.402 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.402 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615636+0000", 2023-10-09T10:50:59.402 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0009000000000002e-05 2023-10-09T10:50:59.402 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.402 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.402 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.402 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615661+0000", 2023-10-09T10:50:59.402 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.516e-05 2023-10-09T10:50:59.402 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.402 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.403 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.403 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.403 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.403 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70445 6.3 6:c48e8359:::benchmark_data_smithi125_244951_object70444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:50:59.403 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.568425+0000", 2023-10-09T10:50:59.403 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 335.82223860200003, 2023-10-09T10:50:59.403 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047372754000000003, 2023-10-09T10:50:59.403 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.403 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.403 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.403 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.403 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.404 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70445 2023-10-09T10:50:59.404 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.404 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.404 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.404 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.404 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568425+0000", 2023-10-09T10:50:59.404 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.404 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.404 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.404 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.404 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568425+0000", 2023-10-09T10:50:59.404 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.405 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.405 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.405 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.405 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568424+0000", 2023-10-09T10:50:59.405 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:59.405 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.405 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.405 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.405 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568448+0000", 2023-10-09T10:50:59.405 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3623000000000001e-05 2023-10-09T10:50:59.405 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.405 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.406 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.406 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568453+0000", 2023-10-09T10:50:59.406 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1880000000000001e-06 2023-10-09T10:50:59.406 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.406 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.406 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.406 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568457+0000", 2023-10-09T10:50:59.406 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1609999999999996e-06 2023-10-09T10:50:59.406 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.406 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.406 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.407 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.569823+0000", 2023-10-09T10:50:59.407 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001365996 2023-10-09T10:50:59.407 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.407 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.407 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.407 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.569913+0000", 2023-10-09T10:50:59.407 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9388000000000005e-05 2023-10-09T10:50:59.407 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.407 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.407 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:50:59.407 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.570017+0000", 2023-10-09T10:50:59.407 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103997 2023-10-09T10:50:59.408 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.408 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.408 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.408 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.571104+0000", 2023-10-09T10:50:59.408 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00108701 2023-10-09T10:50:59.408 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.408 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.408 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.408 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.571280+0000", 2023-10-09T10:50:59.408 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017659500000000001 2023-10-09T10:50:59.408 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.408 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.409 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.409 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615712+0000", 2023-10-09T10:50:59.409 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.044431813000000001 2023-10-09T10:50:59.409 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.409 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.409 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.409 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615755+0000", 2023-10-09T10:50:59.409 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2588e-05 2023-10-09T10:50:59.409 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.409 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.409 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.410 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615797+0000", 2023-10-09T10:50:59.410 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2756999999999997e-05 2023-10-09T10:50:59.410 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.410 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.410 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.410 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.410 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.410 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106416 6.3 6:c0486f03:::benchmark_data_smithi125_244951_object106415:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:50:59.410 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.234195+0000", 2023-10-09T10:50:59.410 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 297.15646836500002, 2023-10-09T10:50:59.410 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.048317987999999999, 2023-10-09T10:50:59.410 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.411 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.411 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.411 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.411 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.411 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106416 2023-10-09T10:50:59.411 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.411 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.411 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.411 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.411 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234195+0000", 2023-10-09T10:50:59.411 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.411 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.411 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.412 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.412 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234195+0000", 2023-10-09T10:50:59.412 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.412 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.412 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.412 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.412 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234193+0000", 2023-10-09T10:50:59.412 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:59.412 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.412 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.412 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.412 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234285+0000", 2023-10-09T10:50:59.412 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1546999999999996e-05 2023-10-09T10:50:59.413 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.413 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.413 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.413 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234305+0000", 2023-10-09T10:50:59.413 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0119000000000002e-05 2023-10-09T10:50:59.413 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.413 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.413 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.413 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234313+0000", 2023-10-09T10:50:59.413 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3590000000000008e-06 2023-10-09T10:50:59.413 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.413 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.414 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.414 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234382+0000", 2023-10-09T10:50:59.414 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8261000000000003e-05 2023-10-09T10:50:59.414 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.414 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.414 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.414 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234490+0000", 2023-10-09T10:50:59.414 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000108398 2023-10-09T10:50:59.414 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.414 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.414 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:50:59.414 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234588+0000", 2023-10-09T10:50:59.415 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7966000000000004e-05 2023-10-09T10:50:59.415 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.415 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.415 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.415 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.236668+0000", 2023-10-09T10:50:59.415 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020797540000000001 2023-10-09T10:50:59.415 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.415 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.415 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.415 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.236734+0000", 2023-10-09T10:50:59.415 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6644999999999996e-05 2023-10-09T10:50:59.415 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.416 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.416 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.416 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.282424+0000", 2023-10-09T10:50:59.416 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.045689354000000001 2023-10-09T10:50:59.416 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.416 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.416 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.416 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.282474+0000", 2023-10-09T10:50:59.416 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9938999999999997e-05 2023-10-09T10:50:59.416 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.416 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.416 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.416 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.282513+0000", 2023-10-09T10:50:59.417 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9168999999999997e-05 2023-10-09T10:50:59.417 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.417 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.417 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.417 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.417 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.417 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:137725 6.3 6:c0616549:::benchmark_data_smithi125_244951_object31282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:50:59.417 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:05.071869+0000", 2023-10-09T10:50:59.417 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 234.31879421299999, 2023-10-09T10:50:59.417 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038173246000000001, 2023-10-09T10:50:59.417 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.417 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.418 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.418 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.418 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.418 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 137725 2023-10-09T10:50:59.418 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.418 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.418 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.418 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.418 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071869+0000", 2023-10-09T10:50:59.418 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.418 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.418 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.419 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.419 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071869+0000", 2023-10-09T10:50:59.419 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.419 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.419 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.419 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.419 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071867+0000", 2023-10-09T10:50:59.419 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:59.419 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.419 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.419 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.419 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071875+0000", 2023-10-09T10:50:59.419 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6290000000000001e-06 2023-10-09T10:50:59.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.420 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.420 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.420 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071885+0000", 2023-10-09T10:50:59.420 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0397999999999999e-05 2023-10-09T10:50:59.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.420 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.420 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.420 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071892+0000", 2023-10-09T10:50:59.420 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6490000000000002e-06 2023-10-09T10:50:59.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.420 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.421 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.081942+0000", 2023-10-09T10:50:59.421 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010050325000000001 2023-10-09T10:50:59.421 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.421 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.421 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.082111+0000", 2023-10-09T10:50:59.421 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016843500000000001 2023-10-09T10:50:59.421 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.421 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:50:59.421 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.082600+0000", 2023-10-09T10:50:59.421 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00048894400000000005 2023-10-09T10:50:59.422 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.422 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.422 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.422 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.084543+0000", 2023-10-09T10:50:59.422 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019437110000000001 2023-10-09T10:50:59.422 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.422 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.422 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.422 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.086053+0000", 2023-10-09T10:50:59.422 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015096179999999999 2023-10-09T10:50:59.422 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.422 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.423 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.423 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.109298+0000", 2023-10-09T10:50:59.423 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023244813 2023-10-09T10:50:59.423 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.423 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.423 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.423 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.109336+0000", 2023-10-09T10:50:59.423 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.79e-05 2023-10-09T10:50:59.423 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.423 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.423 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.423 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.110042+0000", 2023-10-09T10:50:59.423 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00070647600000000004 2023-10-09T10:50:59.424 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.424 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.424 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.424 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.424 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.424 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:137790 6.13 6:c864fc16:::benchmark_data_smithi125_244951_object31347:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:50:59.424 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:05.180662+0000", 2023-10-09T10:50:59.424 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 234.21000101499999, 2023-10-09T10:50:59.424 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038056526, 2023-10-09T10:50:59.424 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.424 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.424 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.425 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.425 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.425 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 137790 2023-10-09T10:50:59.425 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.425 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.425 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.425 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.425 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180662+0000", 2023-10-09T10:50:59.425 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.425 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.425 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.425 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.426 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180662+0000", 2023-10-09T10:50:59.426 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.426 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.426 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.426 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.426 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180661+0000", 2023-10-09T10:50:59.426 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:59.426 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.426 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.426 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.426 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180667+0000", 2023-10-09T10:50:59.426 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4679999999999997e-06 2023-10-09T10:50:59.427 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.427 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.427 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.427 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180680+0000", 2023-10-09T10:50:59.427 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2431e-05 2023-10-09T10:50:59.427 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.427 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.427 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.427 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180685+0000", 2023-10-09T10:50:59.427 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1200000000000001e-06 2023-10-09T10:50:59.427 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.428 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.429 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.429 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182279+0000", 2023-10-09T10:50:59.429 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001594804 2023-10-09T10:50:59.429 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.429 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.429 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.429 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182481+0000", 2023-10-09T10:50:59.429 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00020201 2023-10-09T10:50:59.429 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.430 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.430 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:50:59.430 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182899+0000", 2023-10-09T10:50:59.430 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00041767300000000002 2023-10-09T10:50:59.430 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.430 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.430 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.430 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.185800+0000", 2023-10-09T10:50:59.430 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029009109999999999 2023-10-09T10:50:59.430 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.430 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.430 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.431 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.185815+0000", 2023-10-09T10:50:59.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4959e-05 2023-10-09T10:50:59.431 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.431 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.431 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.431 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.217393+0000", 2023-10-09T10:50:59.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.031578322999999998 2023-10-09T10:50:59.431 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.431 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.431 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.431 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.217443+0000", 2023-10-09T10:50:59.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9308999999999998e-05 2023-10-09T10:50:59.432 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.432 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.432 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.432 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.218719+0000", 2023-10-09T10:50:59.432 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001276016 2023-10-09T10:50:59.432 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.432 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.432 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.432 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.432 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.432 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:137793 6.3 6:c3f14b3e:::benchmark_data_smithi125_244951_object31350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:50:59.432 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:50:59.432 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 234.20996710099999, 2023-10-09T10:50:59.433 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.046213456, 2023-10-09T10:50:59.433 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.433 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.433 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.433 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.433 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.433 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 137793 2023-10-09T10:50:59.433 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.433 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.433 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.433 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.433 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:50:59.433 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.434 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.434 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.434 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.434 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:50:59.434 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.434 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.434 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.434 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.434 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:50:59.434 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:59.434 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.435 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180698+0000", 2023-10-09T10:50:59.435 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5009999999999999e-06 2023-10-09T10:50:59.435 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.435 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180702+0000", 2023-10-09T10:50:59.435 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2200000000000003e-06 2023-10-09T10:50:59.435 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.435 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180705+0000", 2023-10-09T10:50:59.435 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7779999999999999e-06 2023-10-09T10:50:59.436 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.436 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.436 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.436 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182790+0000", 2023-10-09T10:50:59.436 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020852819999999999 2023-10-09T10:50:59.436 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.436 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.436 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.436 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182925+0000", 2023-10-09T10:50:59.436 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013469099999999999 2023-10-09T10:50:59.436 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.436 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.436 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:50:59.437 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.183331+0000", 2023-10-09T10:50:59.437 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000405783 2023-10-09T10:50:59.437 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.437 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.437 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.437 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.185825+0000", 2023-10-09T10:50:59.437 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024937789999999998 2023-10-09T10:50:59.437 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.437 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.437 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.437 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.188205+0000", 2023-10-09T10:50:59.437 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023806790000000001 2023-10-09T10:50:59.438 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.438 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.438 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.438 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.225637+0000", 2023-10-09T10:50:59.438 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.037431091 2023-10-09T10:50:59.438 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.438 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.438 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.438 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.225680+0000", 2023-10-09T10:50:59.438 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3856999999999997e-05 2023-10-09T10:50:59.438 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.438 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.438 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.438 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.226909+0000", 2023-10-09T10:50:59.439 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001229129 2023-10-09T10:50:59.439 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.439 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.439 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.439 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.439 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.439 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:137805 6.3 6:c448070c:::benchmark_data_smithi125_244951_object31362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:50:59.439 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:05.197338+0000", 2023-10-09T10:50:59.439 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 234.193325499, 2023-10-09T10:50:59.439 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039251805000000001, 2023-10-09T10:50:59.439 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.439 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.440 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.440 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.440 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.440 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 137805 2023-10-09T10:50:59.440 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.440 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.440 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.440 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.440 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197338+0000", 2023-10-09T10:50:59.440 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.440 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.440 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.441 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.441 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197338+0000", 2023-10-09T10:50:59.441 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.441 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.441 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.441 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.441 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197336+0000", 2023-10-09T10:50:59.441 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:59.441 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.441 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.441 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.441 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197342+0000", 2023-10-09T10:50:59.441 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3990000000000001e-06 2023-10-09T10:50:59.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197354+0000", 2023-10-09T10:50:59.442 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1821000000000001e-05 2023-10-09T10:50:59.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197359+0000", 2023-10-09T10:50:59.442 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9960000000000001e-06 2023-10-09T10:50:59.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.443 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197413+0000", 2023-10-09T10:50:59.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3909999999999997e-05 2023-10-09T10:50:59.443 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.443 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.443 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.443 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197587+0000", 2023-10-09T10:50:59.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000173408 2023-10-09T10:50:59.443 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.443 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.443 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:50:59.443 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197988+0000", 2023-10-09T10:50:59.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00040095300000000001 2023-10-09T10:50:59.444 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.444 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.444 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.444 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.199873+0000", 2023-10-09T10:50:59.444 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018855790000000001 2023-10-09T10:50:59.444 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.444 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.444 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.444 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.200673+0000", 2023-10-09T10:50:59.444 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00079994400000000002 2023-10-09T10:50:59.444 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.444 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.444 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.235493+0000", 2023-10-09T10:50:59.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.034820028000000003 2023-10-09T10:50:59.445 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.445 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.235533+0000", 2023-10-09T10:50:59.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0139000000000003e-05 2023-10-09T10:50:59.445 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.445 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.236589+0000", 2023-10-09T10:50:59.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001056219 2023-10-09T10:50:59.445 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.446 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.446 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.446 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.446 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.446 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:152091 6.6 e174/107 6:64b0928f:::benchmark_data_smithi125_244951_object45648:head v 174'5840, mlcod=174'5838)", 2023-10-09T10:50:59.446 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.267910+0000", 2023-10-09T10:50:59.446 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 205.12275284500001, 2023-10-09T10:50:59.446 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038740213000000003, 2023-10-09T10:50:59.446 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.446 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.446 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.446 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.447 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.447 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267910+0000", 2023-10-09T10:50:59.447 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.447 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.447 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.447 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.447 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267910+0000", 2023-10-09T10:50:59.447 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.447 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.447 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.447 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.447 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267909+0000", 2023-10-09T10:50:59.447 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:50:59.448 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.448 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.448 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.448 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267917+0000", 2023-10-09T10:50:59.448 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1869999999999997e-06 2023-10-09T10:50:59.448 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.448 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.448 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.448 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267931+0000", 2023-10-09T10:50:59.448 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3239e-05 2023-10-09T10:50:59.448 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.448 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.448 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.448 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267936+0000", 2023-10-09T10:50:59.449 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.451e-06 2023-10-09T10:50:59.449 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.449 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.449 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267979+0000", 2023-10-09T10:50:59.449 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3096000000000002e-05 2023-10-09T10:50:59.449 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.449 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.449 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.268018+0000", 2023-10-09T10:50:59.449 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8532999999999999e-05 2023-10-09T10:50:59.449 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.450 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.450 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.450 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.306618+0000", 2023-10-09T10:50:59.450 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038600661000000001 2023-10-09T10:50:59.450 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.450 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.450 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.450 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.306651+0000", 2023-10-09T10:50:59.450 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2076000000000003e-05 2023-10-09T10:50:59.450 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.450 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.450 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.450 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.451 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:152093 6.13 6:cd7dbd99:::benchmark_data_smithi125_244951_object45650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:50:59.451 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.270440+0000", 2023-10-09T10:50:59.451 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 205.12022319900001, 2023-10-09T10:50:59.451 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040432893999999997, 2023-10-09T10:50:59.451 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.451 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.451 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.451 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.451 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.451 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 152093 2023-10-09T10:50:59.451 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.452 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270440+0000", 2023-10-09T10:50:59.452 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270440+0000", 2023-10-09T10:50:59.452 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270438+0000", 2023-10-09T10:50:59.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T10:50:59.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.453 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.453 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270450+0000", 2023-10-09T10:50:59.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1918e-05 2023-10-09T10:50:59.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.453 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.453 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270479+0000", 2023-10-09T10:50:59.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8911000000000002e-05 2023-10-09T10:50:59.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.454 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.454 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.454 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270488+0000", 2023-10-09T10:50:59.454 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6139999999999998e-06 2023-10-09T10:50:59.454 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.454 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.454 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.454 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270586+0000", 2023-10-09T10:50:59.454 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7823000000000002e-05 2023-10-09T10:50:59.454 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.454 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.455 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.455 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270758+0000", 2023-10-09T10:50:59.455 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000172103 2023-10-09T10:50:59.455 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.455 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.455 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:50:59.455 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.271250+0000", 2023-10-09T10:50:59.455 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000491688 2023-10-09T10:50:59.455 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.455 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.455 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.455 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.278619+0000", 2023-10-09T10:50:59.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0073690730000000003 2023-10-09T10:50:59.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.456 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.456 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.456 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.284528+0000", 2023-10-09T10:50:59.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059091789999999996 2023-10-09T10:50:59.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.456 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.456 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.456 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.309269+0000", 2023-10-09T10:50:59.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024740522000000001 2023-10-09T10:50:59.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.456 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.457 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.309319+0000", 2023-10-09T10:50:59.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0912999999999999e-05 2023-10-09T10:50:59.457 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.457 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.457 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.310873+0000", 2023-10-09T10:50:59.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001553369 2023-10-09T10:50:59.457 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.457 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.457 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.457 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.458 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:152095 6.1 e174/164 6:8109f342:::benchmark_data_smithi125_244951_object45652:head v 174'5685, mlcod=174'5683)", 2023-10-09T10:50:59.458 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.273512+0000", 2023-10-09T10:50:59.458 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 205.117151259, 2023-10-09T10:50:59.458 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040480956999999998, 2023-10-09T10:50:59.458 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.458 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.458 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.458 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.458 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273512+0000", 2023-10-09T10:50:59.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.459 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273512+0000", 2023-10-09T10:50:59.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.459 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273511+0000", 2023-10-09T10:50:59.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:50:59.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273520+0000", 2023-10-09T10:50:59.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.645e-06 2023-10-09T10:50:59.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273534+0000", 2023-10-09T10:50:59.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3875e-05 2023-10-09T10:50:59.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273541+0000", 2023-10-09T10:50:59.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4790000000000001e-06 2023-10-09T10:50:59.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273796+0000", 2023-10-09T10:50:59.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025495599999999998 2023-10-09T10:50:59.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273819+0000", 2023-10-09T10:50:59.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3175000000000001e-05 2023-10-09T10:50:59.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.313948+0000", 2023-10-09T10:50:59.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040129418 2023-10-09T10:50:59.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.313993+0000", 2023-10-09T10:50:59.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4549000000000003e-05 2023-10-09T10:50:59.462 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.462 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.462 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.463 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:161148 6.13 6:c8d28001:::benchmark_data_smithi125_244951_object54705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e176)", 2023-10-09T10:50:59.463 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:47.001392+0000", 2023-10-09T10:50:59.463 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 192.38927137900001, 2023-10-09T10:50:59.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038885253000000002, 2023-10-09T10:50:59.463 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.463 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.463 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.463 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.463 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.463 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 161148 2023-10-09T10:50:59.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.463 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001392+0000", 2023-10-09T10:50:59.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001392+0000", 2023-10-09T10:50:59.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001391+0000", 2023-10-09T10:50:59.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:50:59.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.465 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001397+0000", 2023-10-09T10:50:59.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0560000000000001e-06 2023-10-09T10:50:59.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.465 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001410+0000", 2023-10-09T10:50:59.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2695e-05 2023-10-09T10:50:59.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.466 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.466 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001416+0000", 2023-10-09T10:50:59.466 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9569999999999996e-06 2023-10-09T10:50:59.466 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.466 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.466 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.020851+0000", 2023-10-09T10:50:59.466 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019434414000000001 2023-10-09T10:50:59.466 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.466 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.467 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.021010+0000", 2023-10-09T10:50:59.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015947300000000001 2023-10-09T10:50:59.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:50:59.467 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.021509+0000", 2023-10-09T10:50:59.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00049899199999999997 2023-10-09T10:50:59.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.467 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.026203+0000", 2023-10-09T10:50:59.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0046932700000000003 2023-10-09T10:50:59.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.026305+0000", 2023-10-09T10:50:59.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000102744 2023-10-09T10:50:59.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.038789+0000", 2023-10-09T10:50:59.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012483823 2023-10-09T10:50:59.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.038863+0000", 2023-10-09T10:50:59.469 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3634999999999995e-05 2023-10-09T10:50:59.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.469 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.469 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.040277+0000", 2023-10-09T10:50:59.469 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014141900000000001 2023-10-09T10:50:59.469 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.469 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.469 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.469 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:162909 6.13 6:c8b03eff:::benchmark_data_smithi125_244951_object56466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-10-09T10:50:59.469 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:49.864994+0000", 2023-10-09T10:50:59.470 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 189.52566873500001, 2023-10-09T10:50:59.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.037068614, 2023-10-09T10:50:59.470 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.470 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.470 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.470 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.470 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.470 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162909 2023-10-09T10:50:59.470 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.470 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.470 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.470 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.864994+0000", 2023-10-09T10:50:59.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.471 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.864994+0000", 2023-10-09T10:50:59.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.471 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.864993+0000", 2023-10-09T10:50:59.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:50:59.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.864999+0000", 2023-10-09T10:50:59.472 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2809999999999999e-06 2023-10-09T10:50:59.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.865009+0000", 2023-10-09T10:50:59.472 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.039e-05 2023-10-09T10:50:59.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.865015+0000", 2023-10-09T10:50:59.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.592e-06 2023-10-09T10:50:59.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.865060+0000", 2023-10-09T10:50:59.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5311e-05 2023-10-09T10:50:59.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.865238+0000", 2023-10-09T10:50:59.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017833700000000001 2023-10-09T10:50:59.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:50:59.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.865775+0000", 2023-10-09T10:50:59.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00053696699999999996 2023-10-09T10:50:59.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.869743+0000", 2023-10-09T10:50:59.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039680150000000001 2023-10-09T10:50:59.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.870048+0000", 2023-10-09T10:50:59.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030474199999999999 2023-10-09T10:50:59.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.475 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.475 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.900766+0000", 2023-10-09T10:50:59.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030717874999999999 2023-10-09T10:50:59.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.475 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.475 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.900828+0000", 2023-10-09T10:50:59.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2466999999999997e-05 2023-10-09T10:50:59.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.476 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.476 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.902063+0000", 2023-10-09T10:50:59.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012347180000000001 2023-10-09T10:50:59.476 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.476 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.476 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.476 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:162969 6.13 6:cec0b93d:::benchmark_data_smithi125_244951_object56526:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-10-09T10:50:59.476 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:49.943081+0000", 2023-10-09T10:50:59.476 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 189.44758260200001, 2023-10-09T10:50:59.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039960117000000003, 2023-10-09T10:50:59.476 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.477 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.477 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.477 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.477 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.477 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162969 2023-10-09T10:50:59.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.477 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943081+0000", 2023-10-09T10:50:59.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943081+0000", 2023-10-09T10:50:59.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943079+0000", 2023-10-09T10:50:59.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:59.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943085+0000", 2023-10-09T10:50:59.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6890000000000001e-06 2023-10-09T10:50:59.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943097+0000", 2023-10-09T10:50:59.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2166e-05 2023-10-09T10:50:59.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943103+0000", 2023-10-09T10:50:59.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9150000000000001e-06 2023-10-09T10:50:59.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943207+0000", 2023-10-09T10:50:59.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104573 2023-10-09T10:50:59.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943393+0000", 2023-10-09T10:50:59.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018516299999999999 2023-10-09T10:50:59.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:50:59.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943913+0000", 2023-10-09T10:50:59.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00052048100000000005 2023-10-09T10:50:59.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.950165+0000", 2023-10-09T10:50:59.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062519439999999997 2023-10-09T10:50:59.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.956256+0000", 2023-10-09T10:50:59.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0060905630000000002 2023-10-09T10:50:59.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.981642+0000", 2023-10-09T10:50:59.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025386756999999999 2023-10-09T10:50:59.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.981697+0000", 2023-10-09T10:50:59.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4718e-05 2023-10-09T10:50:59.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.983041+0000", 2023-10-09T10:50:59.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001343639 2023-10-09T10:50:59.483 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.483 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.483 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.483 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:169526 6.13 6:c9d383e8:::benchmark_data_smithi125_244951_object63083:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-10-09T10:50:59.483 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:59.084841+0000", 2023-10-09T10:50:59.483 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 180.305822087, 2023-10-09T10:50:59.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.036194846000000003, 2023-10-09T10:50:59.483 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.483 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.483 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.483 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.484 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.484 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 169526 2023-10-09T10:50:59.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.484 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084841+0000", 2023-10-09T10:50:59.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084841+0000", 2023-10-09T10:50:59.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084841+0000", 2023-10-09T10:50:59.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:59.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084844+0000", 2023-10-09T10:50:59.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4860000000000002e-06 2023-10-09T10:50:59.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084851+0000", 2023-10-09T10:50:59.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7819999999999996e-06 2023-10-09T10:50:59.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084855+0000", 2023-10-09T10:50:59.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3719999999999998e-06 2023-10-09T10:50:59.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.087999+0000", 2023-10-09T10:50:59.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0031432880000000002 2023-10-09T10:50:59.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.088176+0000", 2023-10-09T10:50:59.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017714799999999999 2023-10-09T10:50:59.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:50:59.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.088664+0000", 2023-10-09T10:50:59.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00048842899999999997 2023-10-09T10:50:59.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.099515+0000", 2023-10-09T10:50:59.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010850966 2023-10-09T10:50:59.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.102453+0000", 2023-10-09T10:50:59.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002937598 2023-10-09T10:50:59.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.119544+0000", 2023-10-09T10:50:59.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017091733000000001 2023-10-09T10:50:59.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.119602+0000", 2023-10-09T10:50:59.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7642000000000001e-05 2023-10-09T10:50:59.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.121036+0000", 2023-10-09T10:50:59.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014338599999999999 2023-10-09T10:50:59.489 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.489 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.489 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.490 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.490 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:170177 6.3 6:c6f24c09:::benchmark_data_smithi125_244951_object63734:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-10-09T10:50:59.490 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:59.997605+0000", 2023-10-09T10:50:59.490 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 179.39305847700001, 2023-10-09T10:50:59.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038407996999999999, 2023-10-09T10:50:59.490 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.490 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.490 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.490 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.490 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.490 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 170177 2023-10-09T10:50:59.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.491 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997605+0000", 2023-10-09T10:50:59.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997605+0000", 2023-10-09T10:50:59.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997604+0000", 2023-10-09T10:50:59.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:59.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997607+0000", 2023-10-09T10:50:59.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6170000000000001e-06 2023-10-09T10:50:59.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997612+0000", 2023-10-09T10:50:59.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0490000000000003e-06 2023-10-09T10:50:59.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997616+0000", 2023-10-09T10:50:59.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5889999999999999e-06 2023-10-09T10:50:59.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.013316+0000", 2023-10-09T10:50:59.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015700823999999999 2023-10-09T10:50:59.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.013397+0000", 2023-10-09T10:50:59.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0259999999999994e-05 2023-10-09T10:50:59.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:50:59.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.013825+0000", 2023-10-09T10:50:59.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000428131 2023-10-09T10:50:59.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.017829+0000", 2023-10-09T10:50:59.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040045319999999999 2023-10-09T10:50:59.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.018823+0000", 2023-10-09T10:50:59.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00099334699999999989 2023-10-09T10:50:59.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.035922+0000", 2023-10-09T10:50:59.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017099487999999999 2023-10-09T10:50:59.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.035975+0000", 2023-10-09T10:50:59.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2741999999999997e-05 2023-10-09T10:50:59.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.036013+0000", 2023-10-09T10:50:59.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7812999999999999e-05 2023-10-09T10:50:59.496 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.496 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.496 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.497 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:189608 6.3 6:c6b3429c:::benchmark_data_smithi125_244951_object83165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2023-10-09T10:50:59.497 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:29.169892+0000", 2023-10-09T10:50:59.497 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 150.22077111999999, 2023-10-09T10:50:59.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039847636999999998, 2023-10-09T10:50:59.497 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.497 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.497 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.497 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.497 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.497 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 189608 2023-10-09T10:50:59.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.498 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.498 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.498 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169892+0000", 2023-10-09T10:50:59.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.498 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.498 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169892+0000", 2023-10-09T10:50:59.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169891+0000", 2023-10-09T10:50:59.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:59.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169896+0000", 2023-10-09T10:50:59.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7950000000000004e-06 2023-10-09T10:50:59.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169909+0000", 2023-10-09T10:50:59.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2062000000000001e-05 2023-10-09T10:50:59.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169914+0000", 2023-10-09T10:50:59.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3380000000000004e-06 2023-10-09T10:50:59.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169959+0000", 2023-10-09T10:50:59.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5392000000000002e-05 2023-10-09T10:50:59.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.170104+0000", 2023-10-09T10:50:59.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014468 2023-10-09T10:50:59.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:50:59.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.170424+0000", 2023-10-09T10:50:59.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00031972199999999999 2023-10-09T10:50:59.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.173247+0000", 2023-10-09T10:50:59.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028231710000000002 2023-10-09T10:50:59.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.186655+0000", 2023-10-09T10:50:59.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013408239000000001 2023-10-09T10:50:59.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.209647+0000", 2023-10-09T10:50:59.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022992404000000001 2023-10-09T10:50:59.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.209694+0000", 2023-10-09T10:50:59.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6822e-05 2023-10-09T10:50:59.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.209740+0000", 2023-10-09T10:50:59.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5361999999999997e-05 2023-10-09T10:50:59.503 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.503 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.503 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.504 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:192655 6.13 6:cee47a1c:::benchmark_data_smithi125_244951_object86212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-10-09T10:50:59.504 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:50:59.504 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 145.587168709, 2023-10-09T10:50:59.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039152566, 2023-10-09T10:50:59.504 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.504 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.504 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.504 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.504 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.504 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 192655 2023-10-09T10:50:59.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.504 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.505 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.505 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:50:59.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.505 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.505 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:50:59.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.505 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.506 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:50:59.506 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:50:59.506 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.506 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.506 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.506 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803498+0000", 2023-10-09T10:50:59.506 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4190000000000004e-06 2023-10-09T10:50:59.506 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.506 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.506 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.506 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803509+0000", 2023-10-09T10:50:59.506 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0801e-05 2023-10-09T10:50:59.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.507 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.507 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803515+0000", 2023-10-09T10:50:59.507 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.767e-06 2023-10-09T10:50:59.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.507 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.507 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.809697+0000", 2023-10-09T10:50:59.507 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061824799999999997 2023-10-09T10:50:59.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.507 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.508 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.809808+0000", 2023-10-09T10:50:59.508 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000110271 2023-10-09T10:50:59.508 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.508 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.508 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:50:59.508 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.809938+0000", 2023-10-09T10:50:59.508 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013000800000000001 2023-10-09T10:50:59.508 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.508 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.508 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.508 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.815553+0000", 2023-10-09T10:50:59.508 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0056156879999999998 2023-10-09T10:50:59.509 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.509 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.815567+0000", 2023-10-09T10:50:59.509 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4102e-05 2023-10-09T10:50:59.509 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.509 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.841982+0000", 2023-10-09T10:50:59.509 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026414812999999999 2023-10-09T10:50:59.509 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.510 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.842042+0000", 2023-10-09T10:50:59.510 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0198999999999997e-05 2023-10-09T10:50:59.510 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.510 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.510 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.510 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.842647+0000", 2023-10-09T10:50:59.510 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00060467599999999995 2023-10-09T10:50:59.510 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.510 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.510 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.510 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.510 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.511 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:193229 6.3 6:c7d3e5b6:::benchmark_data_smithi125_244951_object86786:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-10-09T10:50:59.511 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:34.764784+0000", 2023-10-09T10:50:59.511 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 144.625879349, 2023-10-09T10:50:59.511 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038137404999999999, 2023-10-09T10:50:59.511 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.511 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.511 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.511 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.511 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.511 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 193229 2023-10-09T10:50:59.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.511 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764784+0000", 2023-10-09T10:50:59.512 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.512 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764784+0000", 2023-10-09T10:50:59.512 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.512 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764782+0000", 2023-10-09T10:50:59.513 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:59.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.513 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.513 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764789+0000", 2023-10-09T10:50:59.513 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8020000000000003e-06 2023-10-09T10:50:59.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.513 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.513 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764800+0000", 2023-10-09T10:50:59.513 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.096e-05 2023-10-09T10:50:59.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764805+0000", 2023-10-09T10:50:59.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0969999999999996e-06 2023-10-09T10:50:59.514 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.514 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.767644+0000", 2023-10-09T10:50:59.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028391860000000001 2023-10-09T10:50:59.514 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.514 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.767769+0000", 2023-10-09T10:50:59.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012499599999999999 2023-10-09T10:50:59.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.515 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:50:59.515 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.767893+0000", 2023-10-09T10:50:59.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000123915 2023-10-09T10:50:59.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.515 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.515 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.773147+0000", 2023-10-09T10:50:59.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0052540110000000003 2023-10-09T10:50:59.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.516 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.516 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.778846+0000", 2023-10-09T10:50:59.516 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0056984519999999997 2023-10-09T10:50:59.516 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.516 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.516 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.516 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.802818+0000", 2023-10-09T10:50:59.516 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023972543999999998 2023-10-09T10:50:59.516 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.516 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.516 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.516 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.802876+0000", 2023-10-09T10:50:59.517 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7275000000000003e-05 2023-10-09T10:50:59.517 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.517 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.517 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.802921+0000", 2023-10-09T10:50:59.517 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5574000000000001e-05 2023-10-09T10:50:59.517 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.517 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.517 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.517 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.517 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:209514 6.13 6:c9bc4b21:::benchmark_data_smithi125_244951_object103071:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-10-09T10:50:59.517 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:58.156931+0000", 2023-10-09T10:50:59.518 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 121.233732151, 2023-10-09T10:50:59.518 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.037088755000000001, 2023-10-09T10:50:59.518 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.518 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.518 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.518 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.518 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.518 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 209514 2023-10-09T10:50:59.518 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.518 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.518 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.519 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.519 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156931+0000", 2023-10-09T10:50:59.519 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.519 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.519 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.519 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.519 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156931+0000", 2023-10-09T10:50:59.519 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.519 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.519 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.519 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.519 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156930+0000", 2023-10-09T10:50:59.519 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:59.520 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.520 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.520 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.520 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156935+0000", 2023-10-09T10:50:59.520 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5350000000000004e-06 2023-10-09T10:50:59.520 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.520 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.520 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.520 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156948+0000", 2023-10-09T10:50:59.520 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2842e-05 2023-10-09T10:50:59.520 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.520 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.521 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156954+0000", 2023-10-09T10:50:59.521 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.733e-06 2023-10-09T10:50:59.521 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.521 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.158522+0000", 2023-10-09T10:50:59.521 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015687570000000001 2023-10-09T10:50:59.521 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.521 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.158683+0000", 2023-10-09T10:50:59.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016043599999999999 2023-10-09T10:50:59.522 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.522 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.522 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:50:59.522 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.158851+0000", 2023-10-09T10:50:59.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016812899999999999 2023-10-09T10:50:59.522 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.522 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.522 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.522 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.167706+0000", 2023-10-09T10:50:59.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0088551359999999996 2023-10-09T10:50:59.522 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.523 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.167780+0000", 2023-10-09T10:50:59.523 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4383999999999998e-05 2023-10-09T10:50:59.523 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.523 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.193627+0000", 2023-10-09T10:50:59.523 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025846593000000001 2023-10-09T10:50:59.523 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.524 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.193689+0000", 2023-10-09T10:50:59.524 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1681999999999997e-05 2023-10-09T10:50:59.524 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.524 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.524 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.524 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.194020+0000", 2023-10-09T10:50:59.524 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00033098999999999997 2023-10-09T10:50:59.524 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.524 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.524 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.524 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.524 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.524 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:212706 6.13 6:c8228b1c:::benchmark_data_smithi125_244951_object106263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-10-09T10:50:59.525 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:02.392352+0000", 2023-10-09T10:50:59.525 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 116.99831122800001, 2023-10-09T10:50:59.525 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.041188625999999999, 2023-10-09T10:50:59.525 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:50:59.525 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:50:59.525 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:50:59.525 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:50:59.525 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:50:59.525 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212706 2023-10-09T10:50:59.525 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.525 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:50:59.525 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.526 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:50:59.526 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392352+0000", 2023-10-09T10:50:59.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.526 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.526 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.526 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:50:59.526 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392352+0000", 2023-10-09T10:50:59.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:50:59.526 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.526 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.526 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:50:59.526 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392350+0000", 2023-10-09T10:50:59.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:50:59.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.527 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.527 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:50:59.527 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392357+0000", 2023-10-09T10:50:59.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0380000000000002e-06 2023-10-09T10:50:59.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.527 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.527 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:50:59.527 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392369+0000", 2023-10-09T10:50:59.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1857e-05 2023-10-09T10:50:59.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.528 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:50:59.528 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392375+0000", 2023-10-09T10:50:59.528 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5219999999999998e-06 2023-10-09T10:50:59.528 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.528 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:50:59.528 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392421+0000", 2023-10-09T10:50:59.528 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5736999999999999e-05 2023-10-09T10:50:59.528 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.528 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:50:59.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392554+0000", 2023-10-09T10:50:59.529 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013335300000000001 2023-10-09T10:50:59.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.529 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:50:59.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392651+0000", 2023-10-09T10:50:59.529 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7156000000000003e-05 2023-10-09T10:50:59.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.529 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.397156+0000", 2023-10-09T10:50:59.529 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045045550000000004 2023-10-09T10:50:59.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.530 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:50:59.530 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.400165+0000", 2023-10-09T10:50:59.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030094319999999998 2023-10-09T10:50:59.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.530 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:50:59.530 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.433364+0000", 2023-10-09T10:50:59.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.033199472000000001 2023-10-09T10:50:59.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.530 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:50:59.530 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.433409+0000", 2023-10-09T10:50:59.531 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4070000000000003e-05 2023-10-09T10:50:59.531 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:50:59.531 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:50:59.531 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:50:59.531 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.433541+0000", 2023-10-09T10:50:59.531 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013196400000000001 2023-10-09T10:50:59.531 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.531 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.531 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.531 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:50:59.531 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:50:59.531 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:50:59.532 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:59 smithi125 ceph-mon[195507]: failed to encode map e227 with expected crc 2023-10-09T10:50:59.532 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:59 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-10-09T10:50:59.532 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:59 smithi125 ceph-mon[195507]: osdmap e227: 8 total, 8 up, 5 in 2023-10-09T10:50:59.532 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:50:59 smithi125 ceph-mon[195507]: failed to encode map e227 with expected crc 2023-10-09T10:50:59.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:59 smithi155 ceph-mon[184854]: failed to encode map e227 with expected crc 2023-10-09T10:50:59.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:59 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-10-09T10:50:59.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:59 smithi155 ceph-mon[184854]: osdmap e227: 8 total, 8 up, 5 in 2023-10-09T10:50:59.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:50:59 smithi155 ceph-mon[184854]: failed to encode map e227 with expected crc 2023-10-09T10:50:59.533 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:59 smithi125 ceph-mon[200197]: failed to encode map e227 with expected crc 2023-10-09T10:50:59.533 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:59 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-10-09T10:50:59.533 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:59 smithi125 ceph-mon[200197]: osdmap e227: 8 total, 8 up, 5 in 2023-10-09T10:50:59.533 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:50:59 smithi125 ceph-mon[200197]: failed to encode map e227 with expected crc 2023-10-09T10:50:59.533 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:59.534 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:50:59.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:50:59.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:50:59.637 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:50:59.712 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:50:59.787 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:50:59.791 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:51:00.013 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:00.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:00.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:00.121 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:00.160 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:00.186 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:00.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:00.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:00.226 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:00.280 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:00.290 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:00.297 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:51:00.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[200197]: failed to encode map e227 with expected crc 2023-10-09T10:51:00.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[200197]: failed to encode map e227 with expected crc 2023-10-09T10:51:00.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[200197]: pgmap v521: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 12 MiB/s wr, 619 op/s; 29597/263145 objects degraded (11.247%); 755/263145 objects misplaced (0.287%); 9.9 MiB/s, 159 objects/s recovering 2023-10-09T10:51:00.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-10-09T10:51:00.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:00.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:00.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:00.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:51:00.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1159177027' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:51:00.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:51:00.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[195507]: failed to encode map e227 with expected crc 2023-10-09T10:51:00.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[195507]: failed to encode map e227 with expected crc 2023-10-09T10:51:00.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[195507]: pgmap v521: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 12 MiB/s wr, 619 op/s; 29597/263145 objects degraded (11.247%); 755/263145 objects misplaced (0.287%); 9.9 MiB/s, 159 objects/s recovering 2023-10-09T10:51:00.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-10-09T10:51:00.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:00.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:00.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:00.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:51:00.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1159177027' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:51:00.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:00 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:51:00.332 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:00.367 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:00.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:00.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:00 smithi155 ceph-mon[184854]: failed to encode map e227 with expected crc 2023-10-09T10:51:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:00 smithi155 ceph-mon[184854]: failed to encode map e227 with expected crc 2023-10-09T10:51:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:00 smithi155 ceph-mon[184854]: pgmap v521: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 12 MiB/s wr, 619 op/s; 29597/263145 objects degraded (11.247%); 755/263145 objects misplaced (0.287%); 9.9 MiB/s, 159 objects/s recovering 2023-10-09T10:51:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-10-09T10:51:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:00.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:00.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:00.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:51:00.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:00 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1159177027' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:51:00.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:00 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:51:00.474 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:00.513 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:00.536 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:00.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:00.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:00.640 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:00.687 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:00.873 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:00.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:00.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:00.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:01.025 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:01.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:01.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:01.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:01.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:01.265 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:01.301 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:01.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:01.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:01.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:01.479 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:51:01] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2023-10-09T10:51:01.480 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:01.480 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-10-09T10:51:01.480 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:51:01.480 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[200197]: osdmap e228: 8 total, 8 up, 5 in 2023-10-09T10:51:01.480 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[200197]: failed to encode map e228 with expected crc 2023-10-09T10:51:01.480 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:51:01.481 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:51:01.481 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:01.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:01.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-10-09T10:51:01.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:51:01.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[195507]: osdmap e228: 8 total, 8 up, 5 in 2023-10-09T10:51:01.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[195507]: failed to encode map e228 with expected crc 2023-10-09T10:51:01.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:51:01.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:51:01.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:01.496 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:01.525 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:01.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:01.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:01.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:01 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:01.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-10-09T10:51:01.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:01 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:51:01.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:01 smithi155 ceph-mon[184854]: osdmap e228: 8 total, 8 up, 5 in 2023-10-09T10:51:01.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:01 smithi155 ceph-mon[184854]: failed to encode map e228 with expected crc 2023-10-09T10:51:01.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:51:01.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:51:01.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:01.633 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:01.683 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:01.708 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:01.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:01.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:01.817 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:01.864 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:01.911 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:01.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:01.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:02.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:02.058 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:02.092 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:02.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:02.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:02.199 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:02.258 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:02.275 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:51:02.275 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:51:02.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.275 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPushReply(7.5 229/209 [PushReplyOp(7:a42fe69d:::benchmark_data_smithi125_295816_object7213:head)])", 2023-10-09T10:51:02.275 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:02.243957+0000", 2023-10-09T10:51:02.275 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.019164533000000001, 2023-10-09T10:51:02.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.019198453000000001, 2023-10-09T10:51:02.275 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:02.276 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:51:02.276 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:02.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:02.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.243957+0000", 2023-10-09T10:51:02.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:02.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.243957+0000", 2023-10-09T10:51:02.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:02.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.243957+0000", 2023-10-09T10:51:02.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967296 2023-10-09T10:51:02.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:02.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.243960+0000", 2023-10-09T10:51:02.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4779999999999998e-06 2023-10-09T10:51:02.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:02.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.243966+0000", 2023-10-09T10:51:02.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1140000000000002e-06 2023-10-09T10:51:02.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:02.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.243971+0000", 2023-10-09T10:51:02.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0300000000000001e-06 2023-10-09T10:51:02.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:02.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.261023+0000", 2023-10-09T10:51:02.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.017052395000000001 2023-10-09T10:51:02.279 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.279 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:02.279 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.279 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPushReply(7.5 229/209 [PushReplyOp(7:acd94a00:::benchmark_data_smithi125_295816_object7214:head)])", 2023-10-09T10:51:02.279 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:02.244974+0000", 2023-10-09T10:51:02.279 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.018147927000000001, 2023-10-09T10:51:02.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018263061000000001, 2023-10-09T10:51:02.279 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:02.279 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:51:02.279 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:02.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:02.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.244974+0000", 2023-10-09T10:51:02.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:02.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.244974+0000", 2023-10-09T10:51:02.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:02.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.244974+0000", 2023-10-09T10:51:02.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:02.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:02.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.244978+0000", 2023-10-09T10:51:02.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.967e-06 2023-10-09T10:51:02.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:02.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.244987+0000", 2023-10-09T10:51:02.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4339999999999992e-06 2023-10-09T10:51:02.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:02.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.244994+0000", 2023-10-09T10:51:02.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8079999999999999e-06 2023-10-09T10:51:02.282 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.282 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:02.282 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.282 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:98343 7.5 7:a95f7884:::benchmark_data_smithi125_295816_object7206:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:51:02.282 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:02.216131+0000", 2023-10-09T10:51:02.282 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.046990659999999997, 2023-10-09T10:51:02.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.047160963, 2023-10-09T10:51:02.283 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:02.283 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:51:02.283 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:02.283 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:51:02.283 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:51:02.283 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 98343 2023-10-09T10:51:02.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.283 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:02.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:02.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216131+0000", 2023-10-09T10:51:02.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:02.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216131+0000", 2023-10-09T10:51:02.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:02.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216130+0000", 2023-10-09T10:51:02.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:02.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:02.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216136+0000", 2023-10-09T10:51:02.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9599999999999997e-06 2023-10-09T10:51:02.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:02.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216145+0000", 2023-10-09T10:51:02.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4790000000000002e-06 2023-10-09T10:51:02.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:02.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216152+0000", 2023-10-09T10:51:02.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.827e-06 2023-10-09T10:51:02.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:02.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.225611+0000", 2023-10-09T10:51:02.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0094588369999999995 2023-10-09T10:51:02.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:51:02.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.226327+0000", 2023-10-09T10:51:02.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00071592800000000001 2023-10-09T10:51:02.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:02.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.252729+0000", 2023-10-09T10:51:02.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.026402500999999998 2023-10-09T10:51:02.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:02.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.252824+0000", 2023-10-09T10:51:02.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5112999999999996e-05 2023-10-09T10:51:02.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:51:02.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.252928+0000", 2023-10-09T10:51:02.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000103349 2023-10-09T10:51:02.288 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.288 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:02.288 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.288 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:98345 7.5 7:ad3dab50:::benchmark_data_smithi125_295816_object7208:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:51:02.288 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:02.216182+0000", 2023-10-09T10:51:02.288 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.046939509999999997, 2023-10-09T10:51:02.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.047200704000000003, 2023-10-09T10:51:02.288 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:02.289 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:51:02.289 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:02.289 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:51:02.289 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:51:02.289 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 98345 2023-10-09T10:51:02.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.289 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:02.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:02.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216182+0000", 2023-10-09T10:51:02.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:02.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216182+0000", 2023-10-09T10:51:02.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:02.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216182+0000", 2023-10-09T10:51:02.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:02.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:02.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216185+0000", 2023-10-09T10:51:02.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8389999999999998e-06 2023-10-09T10:51:02.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:02.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216191+0000", 2023-10-09T10:51:02.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2750000000000001e-06 2023-10-09T10:51:02.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:02.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216196+0000", 2023-10-09T10:51:02.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0549999999999999e-06 2023-10-09T10:51:02.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:02.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.226364+0000", 2023-10-09T10:51:02.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010167851 2023-10-09T10:51:02.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:51:02.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.226973+0000", 2023-10-09T10:51:02.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00060932 2023-10-09T10:51:02.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:02.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.258928+0000", 2023-10-09T10:51:02.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.031954455 2023-10-09T10:51:02.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:02.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.259009+0000", 2023-10-09T10:51:02.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0934999999999997e-05 2023-10-09T10:51:02.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:51:02.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.259101+0000", 2023-10-09T10:51:02.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2730000000000002e-05 2023-10-09T10:51:02.294 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.294 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:02.294 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.294 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:98350 7.5 7:a42fe69d:::benchmark_data_smithi125_295816_object7213:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:51:02.294 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:02.216558+0000", 2023-10-09T10:51:02.294 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.046563767999999998, 2023-10-09T10:51:02.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.046932478, 2023-10-09T10:51:02.294 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:02.294 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "delayed", 2023-10-09T10:51:02.295 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:02.295 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:51:02.295 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:51:02.295 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 98350 2023-10-09T10:51:02.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.295 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:02.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:02.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216558+0000", 2023-10-09T10:51:02.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:02.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216558+0000", 2023-10-09T10:51:02.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:02.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216557+0000", 2023-10-09T10:51:02.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:51:02.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:02.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216562+0000", 2023-10-09T10:51:02.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1970000000000004e-06 2023-10-09T10:51:02.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:02.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216574+0000", 2023-10-09T10:51:02.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1728e-05 2023-10-09T10:51:02.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:02.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216581+0000", 2023-10-09T10:51:02.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2590000000000003e-06 2023-10-09T10:51:02.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:02.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.227010+0000", 2023-10-09T10:51:02.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010428972999999999 2023-10-09T10:51:02.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:51:02.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.227607+0000", 2023-10-09T10:51:02.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00059629099999999999 2023-10-09T10:51:02.298 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.298 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:02.298 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.299 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:98351 7.5 7:acd94a00:::benchmark_data_smithi125_295816_object7214:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:51:02.299 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:02.216588+0000", 2023-10-09T10:51:02.299 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.046533603999999999, 2023-10-09T10:51:02.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.046972882000000001, 2023-10-09T10:51:02.299 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:02.299 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "delayed", 2023-10-09T10:51:02.299 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:02.299 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:51:02.299 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:51:02.299 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 98351 2023-10-09T10:51:02.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.300 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:02.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:02.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216588+0000", 2023-10-09T10:51:02.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:02.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216588+0000", 2023-10-09T10:51:02.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:02.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216588+0000", 2023-10-09T10:51:02.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:02.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:02.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216590+0000", 2023-10-09T10:51:02.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1919999999999999e-06 2023-10-09T10:51:02.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:02.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216596+0000", 2023-10-09T10:51:02.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5929999999999996e-06 2023-10-09T10:51:02.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:02.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.216600+0000", 2023-10-09T10:51:02.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6890000000000003e-06 2023-10-09T10:51:02.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:02.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.227630+0000", 2023-10-09T10:51:02.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011029727 2023-10-09T10:51:02.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:51:02.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.228351+0000", 2023-10-09T10:51:02.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00072128600000000002 2023-10-09T10:51:02.303 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.303 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:02.303 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.303 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:98375 7.4 7:2956c374:::benchmark_data_smithi125_295816_object7238:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:51:02.304 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:02.250529+0000", 2023-10-09T10:51:02.304 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.012592496999999999, 2023-10-09T10:51:02.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013103647, 2023-10-09T10:51:02.304 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:02.304 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:51:02.304 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:02.304 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:51:02.304 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:51:02.304 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 98375 2023-10-09T10:51:02.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.304 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:02.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:02.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.250529+0000", 2023-10-09T10:51:02.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:02.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.250529+0000", 2023-10-09T10:51:02.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:02.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.250528+0000", 2023-10-09T10:51:02.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:02.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:02.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.250534+0000", 2023-10-09T10:51:02.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2820000000000002e-06 2023-10-09T10:51:02.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:02.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.250545+0000", 2023-10-09T10:51:02.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0916e-05 2023-10-09T10:51:02.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:02.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.250553+0000", 2023-10-09T10:51:02.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.644e-06 2023-10-09T10:51:02.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:02.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.255935+0000", 2023-10-09T10:51:02.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0053823550000000001 2023-10-09T10:51:02.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:02.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.256097+0000", 2023-10-09T10:51:02.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016189700000000001 2023-10-09T10:51:02.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:51:02.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.256265+0000", 2023-10-09T10:51:02.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000167675 2023-10-09T10:51:02.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:02.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.258768+0000", 2023-10-09T10:51:02.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002503164 2023-10-09T10:51:02.309 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.309 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:02.309 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.309 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35187.0:98357 7.d e229/219)", 2023-10-09T10:51:02.309 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:02.255714+0000", 2023-10-09T10:51:02.309 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0074079369999999999, 2023-10-09T10:51:02.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0080093430000000004, 2023-10-09T10:51:02.309 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:02.309 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:51:02.309 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:02.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:02.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.255714+0000", 2023-10-09T10:51:02.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:02.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.255714+0000", 2023-10-09T10:51:02.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:02.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.255713+0000", 2023-10-09T10:51:02.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:51:02.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:02.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.255723+0000", 2023-10-09T10:51:02.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0112000000000001e-05 2023-10-09T10:51:02.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:02.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.255736+0000", 2023-10-09T10:51:02.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2883e-05 2023-10-09T10:51:02.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:02.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.255743+0000", 2023-10-09T10:51:02.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5050000000000001e-06 2023-10-09T10:51:02.312 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.312 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:02.312 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.312 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.35187.0:98343 7.5 e229/209)", 2023-10-09T10:51:02.312 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:02.258773+0000", 2023-10-09T10:51:02.312 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0043483480000000001, 2023-10-09T10:51:02.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0050001150000000003, 2023-10-09T10:51:02.313 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:02.313 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:51:02.313 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:02.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:02.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.258773+0000", 2023-10-09T10:51:02.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:02.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.258773+0000", 2023-10-09T10:51:02.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:02.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.258773+0000", 2023-10-09T10:51:02.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:51:02.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:02.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.258779+0000", 2023-10-09T10:51:02.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.669e-06 2023-10-09T10:51:02.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:02.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.258790+0000", 2023-10-09T10:51:02.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0845e-05 2023-10-09T10:51:02.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:02.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.258797+0000", 2023-10-09T10:51:02.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2130000000000002e-06 2023-10-09T10:51:02.315 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.315 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:02.315 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.316 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:98378 7.5 7.d5d6d915 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:51:02.316 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:02.258877+0000", 2023-10-09T10:51:02.316 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0042451579999999997, 2023-10-09T10:51:02.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.005032173, 2023-10-09T10:51:02.316 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:02.316 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:51:02.316 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:02.316 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:51:02.316 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:51:02.316 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 98378 2023-10-09T10:51:02.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.317 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:02.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:02.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.258877+0000", 2023-10-09T10:51:02.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:02.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.258877+0000", 2023-10-09T10:51:02.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:02.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.258875+0000", 2023-10-09T10:51:02.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:51:02.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:02.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.258879+0000", 2023-10-09T10:51:02.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5619999999999997e-06 2023-10-09T10:51:02.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:02.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.258887+0000", 2023-10-09T10:51:02.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3490000000000003e-06 2023-10-09T10:51:02.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:02.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.258893+0000", 2023-10-09T10:51:02.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7780000000000004e-06 2023-10-09T10:51:02.319 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.319 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:02.319 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.320 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:98380 7.5 7.4c862765 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:51:02.320 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:02.261441+0000", 2023-10-09T10:51:02.320 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0016811739999999999, 2023-10-09T10:51:02.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025267559999999998, 2023-10-09T10:51:02.320 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:02.320 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:51:02.320 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:02.320 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:51:02.320 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:51:02.320 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 98380 2023-10-09T10:51:02.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.320 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:02.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:02.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.261441+0000", 2023-10-09T10:51:02.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:02.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.261441+0000", 2023-10-09T10:51:02.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:02.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:02.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.261440+0000", 2023-10-09T10:51:02.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:51:02.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:02.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.261444+0000", 2023-10-09T10:51:02.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8330000000000001e-06 2023-10-09T10:51:02.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:02.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.261461+0000", 2023-10-09T10:51:02.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6385999999999999e-05 2023-10-09T10:51:02.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:02.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:02.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:02.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:02.261469+0000", 2023-10-09T10:51:02.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2609999999999993e-06 2023-10-09T10:51:02.323 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.323 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:02.323 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.323 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:02.323 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:51:02.323 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 11 2023-10-09T10:51:02.323 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:51:02.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:02.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:02.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:02.425 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:02.473 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:02.501 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:02.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:02.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:02.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[195507]: pgmap v523: 119 pgs: 1 active+recovery_wait+degraded, 5 peering, 10 unknown, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.3 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 28963/258459 objects degraded (11.206%); 755/258459 objects misplaced (0.292%); 12 MiB/s, 189 objects/s recovering 2023-10-09T10:51:02.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[195507]: osdmap e229: 8 total, 8 up, 5 in 2023-10-09T10:51:02.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[195507]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[195507]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[195507]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[195507]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[195507]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[195507]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[200197]: pgmap v523: 119 pgs: 1 active+recovery_wait+degraded, 5 peering, 10 unknown, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.3 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 28963/258459 objects degraded (11.206%); 755/258459 objects misplaced (0.292%); 12 MiB/s, 189 objects/s recovering 2023-10-09T10:51:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[200197]: osdmap e229: 8 total, 8 up, 5 in 2023-10-09T10:51:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[200197]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[200197]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[200197]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[200197]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[200197]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:02 smithi125 ceph-mon[200197]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.597 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:51:02.614 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:02.632 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:51:02.655 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:02 smithi155 ceph-mon[184854]: pgmap v523: 119 pgs: 1 active+recovery_wait+degraded, 5 peering, 10 unknown, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.3 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 28963/258459 objects degraded (11.206%); 755/258459 objects misplaced (0.292%); 12 MiB/s, 189 objects/s recovering 2023-10-09T10:51:02.655 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:02 smithi155 ceph-mon[184854]: osdmap e229: 8 total, 8 up, 5 in 2023-10-09T10:51:02.655 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:02 smithi155 ceph-mon[184854]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.655 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:02 smithi155 ceph-mon[184854]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.656 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:02 smithi155 ceph-mon[184854]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.656 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:02 smithi155 ceph-mon[184854]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.656 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:02 smithi155 ceph-mon[184854]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.656 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:02 smithi155 ceph-mon[184854]: failed to encode map e229 with expected crc 2023-10-09T10:51:02.657 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:02.666 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:51:02.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:02.696 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:02.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:02.697 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:51:02.730 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:51:02.751 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:51:02.764 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:51:02.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:02.799 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:51:02.844 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:02.847 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:51:02.850 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:51:02.851 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:51:02.851 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:51:02.851 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:51:02.851 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:51:02.851 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:51:02.851 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:51:02.851 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:51:02.868 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:02.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:02.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:02.898 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:51:02.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:03.040 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:03.212 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:03.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:03.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:03.275 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3043909726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:03.275 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:03 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 28963/258459 objects degraded (11.206%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:03.316 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:03.341 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:03.352 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:03.384 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:03.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:03.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:03.399 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:03.493 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:03.531 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3043909726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:03.531 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:03 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 28963/258459 objects degraded (11.206%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:03.542 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3043909726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:03.543 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:03 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 28963/258459 objects degraded (11.206%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:03.546 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:03.578 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:03.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:03.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:03.682 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:03.728 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:03.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:03.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:03.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:03.823 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0] out_osds: [2, 7, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:51:03.823 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:51:03.823 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:51:03.823 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.000000 2023-10-09T10:51:03.824 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 0 2023-10-09T10:51:03.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:03.917 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:04.058 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:04.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:04.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:04.162 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:04.191 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:04.214 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:04.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:04.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:04.318 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:04.345 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:04.364 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:04.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:04.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:04.468 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:04 smithi155 ceph-mon[184854]: pgmap v525: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating, 1 active+recovery_wait+degraded, 5 peering, 7 unknown, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 699 op/s; 28543/253494 objects degraded (11.260%); 200/253494 objects misplaced (0.079%); 15 MiB/s, 232 objects/s recovering 2023-10-09T10:51:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:04 smithi155 ceph-mon[184854]: osdmap e230: 8 total, 8 up, 5 in 2023-10-09T10:51:04.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:04 smithi155 ceph-mon[184854]: failed to encode map e230 with expected crc 2023-10-09T10:51:04.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:04 smithi155 ceph-mon[184854]: failed to encode map e230 with expected crc 2023-10-09T10:51:04.485 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:04.494 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:04.512 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:04.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:04.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:04.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:04 smithi125 ceph-mon[195507]: pgmap v525: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating, 1 active+recovery_wait+degraded, 5 peering, 7 unknown, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 699 op/s; 28543/253494 objects degraded (11.260%); 200/253494 objects misplaced (0.079%); 15 MiB/s, 232 objects/s recovering 2023-10-09T10:51:04.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:04 smithi125 ceph-mon[195507]: osdmap e230: 8 total, 8 up, 5 in 2023-10-09T10:51:04.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:04 smithi125 ceph-mon[195507]: failed to encode map e230 with expected crc 2023-10-09T10:51:04.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:04 smithi125 ceph-mon[195507]: failed to encode map e230 with expected crc 2023-10-09T10:51:04.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:04 smithi125 ceph-mon[200197]: pgmap v525: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating, 1 active+recovery_wait+degraded, 5 peering, 7 unknown, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 699 op/s; 28543/253494 objects degraded (11.260%); 200/253494 objects misplaced (0.079%); 15 MiB/s, 232 objects/s recovering 2023-10-09T10:51:04.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:04 smithi125 ceph-mon[200197]: osdmap e230: 8 total, 8 up, 5 in 2023-10-09T10:51:04.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:04 smithi125 ceph-mon[200197]: failed to encode map e230 with expected crc 2023-10-09T10:51:04.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:04 smithi125 ceph-mon[200197]: failed to encode map e230 with expected crc 2023-10-09T10:51:04.616 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:04.643 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:04.661 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:04.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:04.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:04.765 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:04.799 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:05.231 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:51:05.231 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:51:05.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:51:05.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:05 smithi125 ceph-mon[195507]: failed to encode map e230 with expected crc 2023-10-09T10:51:05.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:05 smithi125 ceph-mon[195507]: osdmap e231: 8 total, 8 up, 5 in 2023-10-09T10:51:05.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:05 smithi125 ceph-mon[195507]: failed to encode map e231 with expected crc 2023-10-09T10:51:05.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:05 smithi125 ceph-mon[200197]: failed to encode map e230 with expected crc 2023-10-09T10:51:05.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:05 smithi125 ceph-mon[200197]: osdmap e231: 8 total, 8 up, 5 in 2023-10-09T10:51:05.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:05 smithi125 ceph-mon[200197]: failed to encode map e231 with expected crc 2023-10-09T10:51:05.428 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:05.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:05.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:05.538 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:05.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:05 smithi155 ceph-mon[184854]: failed to encode map e230 with expected crc 2023-10-09T10:51:05.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:05 smithi155 ceph-mon[184854]: osdmap e231: 8 total, 8 up, 5 in 2023-10-09T10:51:05.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:05 smithi155 ceph-mon[184854]: failed to encode map e231 with expected crc 2023-10-09T10:51:05.597 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:05.630 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:05.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:05.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:05.739 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:05.809 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:05.844 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:05.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:05.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:05.948 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:05.998 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:06.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:06 smithi125 ceph-mon[195507]: failed to encode map e231 with expected crc 2023-10-09T10:51:06.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:06 smithi125 ceph-mon[195507]: pgmap v528: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating, 1 active+recovery_wait+degraded, 5 peering, 5 unknown, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 619 op/s; 23610/250947 objects degraded (9.408%); 4651/250947 objects misplaced (1.853%); 15 MiB/s, 242 objects/s recovering 2023-10-09T10:51:06.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:06 smithi125 ceph-mon[195507]: failed to encode map e231 with expected crc 2023-10-09T10:51:06.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:06 smithi125 ceph-mon[200197]: failed to encode map e231 with expected crc 2023-10-09T10:51:06.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:06 smithi125 ceph-mon[200197]: pgmap v528: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating, 1 active+recovery_wait+degraded, 5 peering, 5 unknown, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 619 op/s; 23610/250947 objects degraded (9.408%); 4651/250947 objects misplaced (1.853%); 15 MiB/s, 242 objects/s recovering 2023-10-09T10:51:06.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:06 smithi125 ceph-mon[200197]: failed to encode map e231 with expected crc 2023-10-09T10:51:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:06 smithi155 ceph-mon[184854]: failed to encode map e231 with expected crc 2023-10-09T10:51:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:06 smithi155 ceph-mon[184854]: pgmap v528: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating, 1 active+recovery_wait+degraded, 5 peering, 5 unknown, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 619 op/s; 23610/250947 objects degraded (9.408%); 4651/250947 objects misplaced (1.853%); 15 MiB/s, 242 objects/s recovering 2023-10-09T10:51:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:06 smithi155 ceph-mon[184854]: failed to encode map e231 with expected crc 2023-10-09T10:51:07.326 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:51:07.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:07 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/991545892' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:07.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:07 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/991545892' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:07.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:07 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/991545892' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:08 smithi125 ceph-mon[195507]: pgmap v529: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 633 op/s; 23420/247254 objects degraded (9.472%); 4451/247254 objects misplaced (1.800%); 15 MiB/s, 232 objects/s recovering 2023-10-09T10:51:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:08 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:08 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/991545892' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:51:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:08 smithi125 ceph-mon[195507]: osdmap e232: 8 total, 8 up, 5 in 2023-10-09T10:51:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:08 smithi125 ceph-mon[195507]: failed to encode map e232 with expected crc 2023-10-09T10:51:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:08 smithi125 ceph-mon[195507]: failed to encode map e232 with expected crc 2023-10-09T10:51:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:08 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 23420/247254 objects degraded (9.472%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:08 smithi125 ceph-mon[200197]: pgmap v529: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 633 op/s; 23420/247254 objects degraded (9.472%); 4451/247254 objects misplaced (1.800%); 15 MiB/s, 232 objects/s recovering 2023-10-09T10:51:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:08 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:08 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/991545892' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:51:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:08 smithi125 ceph-mon[200197]: osdmap e232: 8 total, 8 up, 5 in 2023-10-09T10:51:08.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:08 smithi125 ceph-mon[200197]: failed to encode map e232 with expected crc 2023-10-09T10:51:08.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:08 smithi125 ceph-mon[200197]: failed to encode map e232 with expected crc 2023-10-09T10:51:08.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:08 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 23420/247254 objects degraded (9.472%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:08.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:08 smithi155 ceph-mon[184854]: pgmap v529: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 633 op/s; 23420/247254 objects degraded (9.472%); 4451/247254 objects misplaced (1.800%); 15 MiB/s, 232 objects/s recovering 2023-10-09T10:51:08.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:08 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:08.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:08 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/991545892' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:51:08.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:08 smithi155 ceph-mon[184854]: osdmap e232: 8 total, 8 up, 5 in 2023-10-09T10:51:08.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:08 smithi155 ceph-mon[184854]: failed to encode map e232 with expected crc 2023-10-09T10:51:08.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:08 smithi155 ceph-mon[184854]: failed to encode map e232 with expected crc 2023-10-09T10:51:08.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:08 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 23420/247254 objects degraded (9.472%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:09.122 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:09.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:09.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:09.149 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:51:09.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:09.276 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:09.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:09.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:09.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:10.072 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:10.114 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:10.124 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:10.144 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:10.146 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:10.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:10.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:10.250 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:10.317 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:10.414 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:10.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:10.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:10.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:10 smithi155 ceph-mon[184854]: pgmap v531: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 537 op/s; 22972/244014 objects degraded (9.414%); 4451/244014 objects misplaced (1.824%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T10:51:10.531 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:10.554 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:10 smithi125 ceph-mon[195507]: pgmap v531: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 537 op/s; 22972/244014 objects degraded (9.414%); 4451/244014 objects misplaced (1.824%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T10:51:10.555 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:10 smithi125 ceph-mon[200197]: pgmap v531: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 537 op/s; 22972/244014 objects degraded (9.414%); 4451/244014 objects misplaced (1.824%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T10:51:10.590 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:11.352 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.6 primary-affinity to 0 (802) 2023-10-09T10:51:11.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:11 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3402524244' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T10:51:11.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:11 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T10:51:11.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:11 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3402524244' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T10:51:11.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:11 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T10:51:11.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:51:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:51:11] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2023-10-09T10:51:11.481 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:11.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:11.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:11.590 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:11.614 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:11 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3402524244' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T10:51:11.614 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:11 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-09T10:51:11.616 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:11.635 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:11.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:11.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:11.739 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:11.771 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:11.841 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:11.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:11.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:11.946 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:11.972 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:11.975 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:51:11.990 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:11.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:11.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:11.995 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:51:12.003 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:51:12.024 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:51:12.059 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:51:12.081 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:51:12.094 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:12.108 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:51:12.121 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:12.135 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:51:12.139 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:12.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:12.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:12.161 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:51:12.163 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:51:12.163 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:51:12.163 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:51:12.163 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:51:12.163 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:51:12.163 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:51:12.163 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:51:12.164 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:51:12.184 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:51:12.184 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:51:12.185 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:51:12.185 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:51:12.185 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.185 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42384 6.10 6:0c7bd3cf:::benchmark_data_smithi125_244951_object42383:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.185 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:51:12.185 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 377.06452382700002, 2023-10-09T10:51:12.185 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0577403670000001, 2023-10-09T10:51:12.185 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.185 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.185 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.186 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.186 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.186 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42384 2023-10-09T10:51:12.186 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.186 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.186 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.186 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.186 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:51:12.186 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.186 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.187 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.187 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.187 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:51:12.187 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.187 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.187 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.187 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.187 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117185+0000", 2023-10-09T10:51:12.187 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:12.187 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.187 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.188 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.188 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117225+0000", 2023-10-09T10:51:12.188 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0364000000000002e-05 2023-10-09T10:51:12.188 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.188 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.188 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.188 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117240+0000", 2023-10-09T10:51:12.188 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4425e-05 2023-10-09T10:51:12.188 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.188 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.188 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.189 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117248+0000", 2023-10-09T10:51:12.189 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4999999999999999e-06 2023-10-09T10:51:12.189 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.189 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.189 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.189 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117301+0000", 2023-10-09T10:51:12.189 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2685999999999997e-05 2023-10-09T10:51:12.189 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.189 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.189 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.189 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117312+0000", 2023-10-09T10:51:12.189 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1273e-05 2023-10-09T10:51:12.190 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.190 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.190 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.190 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103469+0000", 2023-10-09T10:51:12.190 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98615701300000003 2023-10-09T10:51:12.190 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.190 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.190 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.190 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103483+0000", 2023-10-09T10:51:12.190 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3207000000000001e-05 2023-10-09T10:51:12.190 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.190 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.191 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.191 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.148764+0000", 2023-10-09T10:51:12.191 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0452812789999999 2023-10-09T10:51:12.191 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.191 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.191 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.191 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.148882+0000", 2023-10-09T10:51:12.191 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118095 2023-10-09T10:51:12.191 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.191 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.191 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.191 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.149057+0000", 2023-10-09T10:51:12.192 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017516400000000001 2023-10-09T10:51:12.192 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.192 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.192 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.192 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174105+0000", 2023-10-09T10:51:12.192 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.025048237000000001 2023-10-09T10:51:12.192 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.192 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.192 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.192 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174158+0000", 2023-10-09T10:51:12.192 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2330999999999998e-05 2023-10-09T10:51:12.192 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.192 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.193 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.193 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174325+0000", 2023-10-09T10:51:12.193 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016690100000000001 2023-10-09T10:51:12.193 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.193 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.193 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.193 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174367+0000", 2023-10-09T10:51:12.193 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2759000000000002e-05 2023-10-09T10:51:12.193 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.193 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.193 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.194 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174927+0000", 2023-10-09T10:51:12.194 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00055942800000000005 2023-10-09T10:51:12.194 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.194 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.194 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.194 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.194 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.194 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42398 6.10 6:0867fefc:::benchmark_data_smithi125_244951_object42397:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.194 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:51:12.194 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 377.053745058, 2023-10-09T10:51:12.194 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0510695769999998, 2023-10-09T10:51:12.194 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.194 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.195 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.195 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.195 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.195 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42398 2023-10-09T10:51:12.195 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.195 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.195 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.195 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:51:12.195 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.195 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.196 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:51:12.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.196 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.196 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.196 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127964+0000", 2023-10-09T10:51:12.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:12.196 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.196 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.197 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.197 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127989+0000", 2023-10-09T10:51:12.197 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4811e-05 2023-10-09T10:51:12.197 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.197 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.197 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.197 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128005+0000", 2023-10-09T10:51:12.197 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6436999999999999e-05 2023-10-09T10:51:12.197 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.197 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.197 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.197 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128014+0000", 2023-10-09T10:51:12.198 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5990000000000002e-06 2023-10-09T10:51:12.198 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.198 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.198 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.198 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128072+0000", 2023-10-09T10:51:12.198 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7980999999999999e-05 2023-10-09T10:51:12.198 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.198 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.198 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.198 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128083+0000", 2023-10-09T10:51:12.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1051e-05 2023-10-09T10:51:12.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.199 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103714+0000", 2023-10-09T10:51:12.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97563124099999998 2023-10-09T10:51:12.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.199 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103725+0000", 2023-10-09T10:51:12.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1121000000000001e-05 2023-10-09T10:51:12.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.200 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.200 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.200 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174971+0000", 2023-10-09T10:51:12.200 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0712457310000001 2023-10-09T10:51:12.200 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.200 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.200 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.200 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.175149+0000", 2023-10-09T10:51:12.200 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000178584 2023-10-09T10:51:12.200 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.200 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.200 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.200 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.175358+0000", 2023-10-09T10:51:12.201 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000208848 2023-10-09T10:51:12.201 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.201 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.201 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.201 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178406+0000", 2023-10-09T10:51:12.201 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030478049999999998 2023-10-09T10:51:12.201 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.201 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.201 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.201 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178418+0000", 2023-10-09T10:51:12.202 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1499e-05 2023-10-09T10:51:12.202 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.202 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.202 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.202 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178466+0000", 2023-10-09T10:51:12.202 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8715999999999999e-05 2023-10-09T10:51:12.202 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.202 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.202 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.202 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178522+0000", 2023-10-09T10:51:12.202 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5952999999999998e-05 2023-10-09T10:51:12.203 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.203 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.203 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.203 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179035+0000", 2023-10-09T10:51:12.203 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051254299999999996 2023-10-09T10:51:12.203 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.203 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.203 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.203 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.203 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.203 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42518 6.10 6:0fbcc8bc:::benchmark_data_smithi125_244951_object42517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.203 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:51:12.204 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 376.98734773299998, 2023-10-09T10:51:12.204 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9852423510000001, 2023-10-09T10:51:12.204 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.204 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.204 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.204 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.204 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.204 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42518 2023-10-09T10:51:12.204 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.204 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.204 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.204 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.204 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:51:12.204 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.205 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.205 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.205 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.205 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:51:12.205 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.205 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.205 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.205 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.205 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194361+0000", 2023-10-09T10:51:12.205 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:12.205 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.205 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.205 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.205 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194385+0000", 2023-10-09T10:51:12.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4017000000000001e-05 2023-10-09T10:51:12.206 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.206 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.206 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.206 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194401+0000", 2023-10-09T10:51:12.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5841e-05 2023-10-09T10:51:12.206 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.206 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.206 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.206 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194408+0000", 2023-10-09T10:51:12.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0450000000000003e-06 2023-10-09T10:51:12.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.207 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194460+0000", 2023-10-09T10:51:12.207 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2015000000000003e-05 2023-10-09T10:51:12.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.207 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194474+0000", 2023-10-09T10:51:12.207 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4486e-05 2023-10-09T10:51:12.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103750+0000", 2023-10-09T10:51:12.208 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.90927595699999997 2023-10-09T10:51:12.208 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.208 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.208 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103758+0000", 2023-10-09T10:51:12.208 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9000000000000006e-06 2023-10-09T10:51:12.208 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.208 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.208 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176100+0000", 2023-10-09T10:51:12.208 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.07234187 2023-10-09T10:51:12.208 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.208 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.209 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.209 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176198+0000", 2023-10-09T10:51:12.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7408e-05 2023-10-09T10:51:12.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.209 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.209 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.209 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176317+0000", 2023-10-09T10:51:12.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011998900000000001 2023-10-09T10:51:12.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.209 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.209 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.209 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178425+0000", 2023-10-09T10:51:12.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021072 2023-10-09T10:51:12.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.210 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179085+0000", 2023-10-09T10:51:12.210 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00066011800000000001 2023-10-09T10:51:12.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.210 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179122+0000", 2023-10-09T10:51:12.210 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6831e-05 2023-10-09T10:51:12.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.210 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179155+0000", 2023-10-09T10:51:12.211 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3080000000000002e-05 2023-10-09T10:51:12.211 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.211 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.211 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179605+0000", 2023-10-09T10:51:12.211 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000450121 2023-10-09T10:51:12.211 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.211 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.211 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.211 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.211 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42546 6.10 6:0889293a:::benchmark_data_smithi125_244951_object42545:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.211 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:51:12.212 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 376.9729183, 2023-10-09T10:51:12.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.971488017, 2023-10-09T10:51:12.212 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.212 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.212 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.212 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.212 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.212 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42546 2023-10-09T10:51:12.212 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.212 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.212 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.212 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.212 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:51:12.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:51:12.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208790+0000", 2023-10-09T10:51:12.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:51:12.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.214 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.214 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208815+0000", 2023-10-09T10:51:12.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5103e-05 2023-10-09T10:51:12.214 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.214 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.214 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208831+0000", 2023-10-09T10:51:12.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5235000000000001e-05 2023-10-09T10:51:12.214 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.214 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.214 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208838+0000", 2023-10-09T10:51:12.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6510000000000001e-06 2023-10-09T10:51:12.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.215 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.215 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208895+0000", 2023-10-09T10:51:12.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6817999999999997e-05 2023-10-09T10:51:12.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.215 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.215 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208906+0000", 2023-10-09T10:51:12.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0550000000000001e-05 2023-10-09T10:51:12.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.215 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103780+0000", 2023-10-09T10:51:12.216 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.89487466699999996 2023-10-09T10:51:12.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103788+0000", 2023-10-09T10:51:12.216 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0970000000000006e-06 2023-10-09T10:51:12.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176855+0000", 2023-10-09T10:51:12.216 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0730664299999999 2023-10-09T10:51:12.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176993+0000", 2023-10-09T10:51:12.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013810999999999999 2023-10-09T10:51:12.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.177166+0000", 2023-10-09T10:51:12.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017289900000000001 2023-10-09T10:51:12.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.218 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179093+0000", 2023-10-09T10:51:12.218 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019276969999999999 2023-10-09T10:51:12.218 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.218 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179674+0000", 2023-10-09T10:51:12.218 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00058078000000000003 2023-10-09T10:51:12.218 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.219 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179730+0000", 2023-10-09T10:51:12.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5418999999999999e-05 2023-10-09T10:51:12.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.219 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179788+0000", 2023-10-09T10:51:12.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8510000000000001e-05 2023-10-09T10:51:12.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.219 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180280+0000", 2023-10-09T10:51:12.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00049179400000000004 2023-10-09T10:51:12.220 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.220 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.220 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.220 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42570 6.10 6:0f6ea5a0:::benchmark_data_smithi125_244951_object42569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.220 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:51:12.220 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 376.95883380700002, 2023-10-09T10:51:12.220 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.969653487, 2023-10-09T10:51:12.220 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.220 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.220 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.221 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.221 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.221 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42570 2023-10-09T10:51:12.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.221 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.221 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:51:12.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.221 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.222 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:51:12.222 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.222 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.222 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222875+0000", 2023-10-09T10:51:12.222 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:12.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.222 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.222 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222897+0000", 2023-10-09T10:51:12.222 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1678000000000001e-05 2023-10-09T10:51:12.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.223 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.223 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.223 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222906+0000", 2023-10-09T10:51:12.223 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4190000000000008e-06 2023-10-09T10:51:12.223 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.223 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.223 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.223 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222913+0000", 2023-10-09T10:51:12.223 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2659999999999998e-06 2023-10-09T10:51:12.223 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.223 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.223 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.223 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222958+0000", 2023-10-09T10:51:12.224 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5979000000000001e-05 2023-10-09T10:51:12.224 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.224 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.224 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.224 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222965+0000", 2023-10-09T10:51:12.224 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0099999999999998e-06 2023-10-09T10:51:12.224 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.224 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.224 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.224 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103823+0000", 2023-10-09T10:51:12.224 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.880857891 2023-10-09T10:51:12.224 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.224 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.224 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103833+0000", 2023-10-09T10:51:12.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8910000000000007e-06 2023-10-09T10:51:12.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180446+0000", 2023-10-09T10:51:12.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0766125440000001 2023-10-09T10:51:12.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180594+0000", 2023-10-09T10:51:12.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014846600000000001 2023-10-09T10:51:12.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180775+0000", 2023-10-09T10:51:12.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018071999999999999 2023-10-09T10:51:12.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183638+0000", 2023-10-09T10:51:12.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028626440000000001 2023-10-09T10:51:12.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183677+0000", 2023-10-09T10:51:12.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9545000000000002e-05 2023-10-09T10:51:12.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191583+0000", 2023-10-09T10:51:12.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0079059569999999999 2023-10-09T10:51:12.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191643+0000", 2023-10-09T10:51:12.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9590999999999999e-05 2023-10-09T10:51:12.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.228 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.228 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192530+0000", 2023-10-09T10:51:12.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00088716399999999996 2023-10-09T10:51:12.228 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.228 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.228 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.228 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42586 6.10 6:090d6b8d:::benchmark_data_smithi125_244951_object42585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.228 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:51:12.228 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 376.95223926599999, 2023-10-09T10:51:12.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.963875493, 2023-10-09T10:51:12.229 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.229 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.229 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.229 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.229 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.229 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42586 2023-10-09T10:51:12.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.229 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:51:12.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:51:12.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229470+0000", 2023-10-09T10:51:12.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:51:12.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229492+0000", 2023-10-09T10:51:12.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1716999999999999e-05 2023-10-09T10:51:12.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229501+0000", 2023-10-09T10:51:12.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0929999999999992e-06 2023-10-09T10:51:12.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229507+0000", 2023-10-09T10:51:12.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6760000000000001e-06 2023-10-09T10:51:12.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229565+0000", 2023-10-09T10:51:12.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7450000000000001e-05 2023-10-09T10:51:12.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229572+0000", 2023-10-09T10:51:12.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0820000000000001e-06 2023-10-09T10:51:12.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103865+0000", 2023-10-09T10:51:12.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.87429282200000003 2023-10-09T10:51:12.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103873+0000", 2023-10-09T10:51:12.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2090000000000005e-06 2023-10-09T10:51:12.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181656+0000", 2023-10-09T10:51:12.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.077783462 2023-10-09T10:51:12.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181791+0000", 2023-10-09T10:51:12.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000134351 2023-10-09T10:51:12.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181948+0000", 2023-10-09T10:51:12.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015678600000000001 2023-10-09T10:51:12.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183685+0000", 2023-10-09T10:51:12.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00173705 2023-10-09T10:51:12.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183719+0000", 2023-10-09T10:51:12.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4301999999999998e-05 2023-10-09T10:51:12.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192640+0000", 2023-10-09T10:51:12.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0089210899999999996 2023-10-09T10:51:12.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192696+0000", 2023-10-09T10:51:12.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5733000000000003e-05 2023-10-09T10:51:12.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193346+0000", 2023-10-09T10:51:12.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00065071200000000001 2023-10-09T10:51:12.238 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.238 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.238 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.238 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42606 6.10 6:0826a955:::benchmark_data_smithi125_244951_object42605:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.238 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:51:12.238 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 376.94347967099998, 2023-10-09T10:51:12.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.955680458, 2023-10-09T10:51:12.239 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.239 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.239 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.239 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.239 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.239 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42606 2023-10-09T10:51:12.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.239 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:51:12.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:51:12.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238230+0000", 2023-10-09T10:51:12.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:51:12.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238257+0000", 2023-10-09T10:51:12.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7551000000000001e-05 2023-10-09T10:51:12.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238267+0000", 2023-10-09T10:51:12.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5820000000000008e-06 2023-10-09T10:51:12.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238273+0000", 2023-10-09T10:51:12.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5490000000000003e-06 2023-10-09T10:51:12.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238322+0000", 2023-10-09T10:51:12.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8467999999999997e-05 2023-10-09T10:51:12.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238332+0000", 2023-10-09T10:51:12.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9000000000000001e-06 2023-10-09T10:51:12.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103898+0000", 2023-10-09T10:51:12.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.86556675699999996 2023-10-09T10:51:12.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103906+0000", 2023-10-09T10:51:12.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7710000000000006e-06 2023-10-09T10:51:12.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.182699+0000", 2023-10-09T10:51:12.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0787928490000001 2023-10-09T10:51:12.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.182847+0000", 2023-10-09T10:51:12.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014758600000000001 2023-10-09T10:51:12.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.244 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183016+0000", 2023-10-09T10:51:12.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000169326 2023-10-09T10:51:12.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.244 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186922+0000", 2023-10-09T10:51:12.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039060269999999998 2023-10-09T10:51:12.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.245 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.245 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186980+0000", 2023-10-09T10:51:12.245 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8313999999999998e-05 2023-10-09T10:51:12.245 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.245 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.245 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.245 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193472+0000", 2023-10-09T10:51:12.245 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0064917020000000002 2023-10-09T10:51:12.245 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.245 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.245 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.245 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193522+0000", 2023-10-09T10:51:12.245 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9672999999999997e-05 2023-10-09T10:51:12.246 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.246 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.246 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193911+0000", 2023-10-09T10:51:12.246 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00038944099999999997 2023-10-09T10:51:12.246 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.246 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.246 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.246 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.246 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42608 6.10 6:0c1c992a:::benchmark_data_smithi125_244951_object42607:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.246 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:51:12.247 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 376.94038797100001, 2023-10-09T10:51:12.247 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9589951289999998, 2023-10-09T10:51:12.247 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.247 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.247 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.247 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.247 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.247 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42608 2023-10-09T10:51:12.247 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.247 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:51:12.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:51:12.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241321+0000", 2023-10-09T10:51:12.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:51:12.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241343+0000", 2023-10-09T10:51:12.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1987000000000001e-05 2023-10-09T10:51:12.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241355+0000", 2023-10-09T10:51:12.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1395e-05 2023-10-09T10:51:12.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241362+0000", 2023-10-09T10:51:12.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7759999999999997e-06 2023-10-09T10:51:12.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241406+0000", 2023-10-09T10:51:12.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3189999999999998e-05 2023-10-09T10:51:12.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241413+0000", 2023-10-09T10:51:12.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5800000000000003e-06 2023-10-09T10:51:12.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103930+0000", 2023-10-09T10:51:12.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.86251731399999998 2023-10-09T10:51:12.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103938+0000", 2023-10-09T10:51:12.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7349999999999996e-06 2023-10-09T10:51:12.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183741+0000", 2023-10-09T10:51:12.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0798032179999999 2023-10-09T10:51:12.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183865+0000", 2023-10-09T10:51:12.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012327200000000001 2023-10-09T10:51:12.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.184011+0000", 2023-10-09T10:51:12.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014617499999999999 2023-10-09T10:51:12.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.253 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.253 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186989+0000", 2023-10-09T10:51:12.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029777340000000001 2023-10-09T10:51:12.253 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.253 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.253 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.253 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187029+0000", 2023-10-09T10:51:12.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0513000000000003e-05 2023-10-09T10:51:12.253 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.253 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.253 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.253 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199414+0000", 2023-10-09T10:51:12.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012385022000000001 2023-10-09T10:51:12.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.254 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.254 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.254 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199472+0000", 2023-10-09T10:51:12.254 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7998999999999998e-05 2023-10-09T10:51:12.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.254 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.254 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.254 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200317+0000", 2023-10-09T10:51:12.254 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000845245 2023-10-09T10:51:12.254 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.254 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.254 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.255 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.255 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42717 6.10 6:0e29612e:::benchmark_data_smithi125_244951_object42716:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.255 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:51:12.255 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 376.87425951300003, 2023-10-09T10:51:12.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8935023019999999, 2023-10-09T10:51:12.255 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.255 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.255 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.255 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.255 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.255 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42717 2023-10-09T10:51:12.255 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.256 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:51:12.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:51:12.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307449+0000", 2023-10-09T10:51:12.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:12.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307473+0000", 2023-10-09T10:51:12.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3349e-05 2023-10-09T10:51:12.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307490+0000", 2023-10-09T10:51:12.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7133e-05 2023-10-09T10:51:12.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307496+0000", 2023-10-09T10:51:12.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7349999999999999e-06 2023-10-09T10:51:12.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307561+0000", 2023-10-09T10:51:12.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4226999999999993e-05 2023-10-09T10:51:12.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307575+0000", 2023-10-09T10:51:12.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4504e-05 2023-10-09T10:51:12.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103963+0000", 2023-10-09T10:51:12.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.79638768500000001 2023-10-09T10:51:12.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103971+0000", 2023-10-09T10:51:12.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9629999999999998e-06 2023-10-09T10:51:12.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.185776+0000", 2023-10-09T10:51:12.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0818051479999999 2023-10-09T10:51:12.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.185949+0000", 2023-10-09T10:51:12.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000172659 2023-10-09T10:51:12.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186122+0000", 2023-10-09T10:51:12.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017296799999999999 2023-10-09T10:51:12.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.261 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189418+0000", 2023-10-09T10:51:12.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032959679999999998 2023-10-09T10:51:12.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.261 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189457+0000", 2023-10-09T10:51:12.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9669000000000003e-05 2023-10-09T10:51:12.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.261 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200372+0000", 2023-10-09T10:51:12.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010914633999999999 2023-10-09T10:51:12.262 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.262 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.262 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.262 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200421+0000", 2023-10-09T10:51:12.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9047e-05 2023-10-09T10:51:12.262 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.262 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.262 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.262 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200953+0000", 2023-10-09T10:51:12.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00053212500000000005 2023-10-09T10:51:12.262 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.262 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.263 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.263 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42722 6.10 6:0ee7a776:::benchmark_data_smithi125_244951_object42721:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.263 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:51:12.263 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 376.87420130499999, 2023-10-09T10:51:12.263 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8972440939999999, 2023-10-09T10:51:12.263 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.263 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.263 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.263 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.263 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.264 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42722 2023-10-09T10:51:12.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.264 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:51:12.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:51:12.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307508+0000", 2023-10-09T10:51:12.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:12.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307529+0000", 2023-10-09T10:51:12.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0866999999999999e-05 2023-10-09T10:51:12.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307539+0000", 2023-10-09T10:51:12.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2690000000000005e-06 2023-10-09T10:51:12.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307544+0000", 2023-10-09T10:51:12.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7930000000000003e-06 2023-10-09T10:51:12.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307599+0000", 2023-10-09T10:51:12.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4876000000000001e-05 2023-10-09T10:51:12.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307607+0000", 2023-10-09T10:51:12.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8760000000000005e-06 2023-10-09T10:51:12.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103995+0000", 2023-10-09T10:51:12.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.79638814099999999 2023-10-09T10:51:12.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104003+0000", 2023-10-09T10:51:12.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7819999999999994e-06 2023-10-09T10:51:12.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187058+0000", 2023-10-09T10:51:12.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.083054712 2023-10-09T10:51:12.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187234+0000", 2023-10-09T10:51:12.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017615799999999999 2023-10-09T10:51:12.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187416+0000", 2023-10-09T10:51:12.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018243 2023-10-09T10:51:12.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189466+0000", 2023-10-09T10:51:12.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020493210000000002 2023-10-09T10:51:12.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189504+0000", 2023-10-09T10:51:12.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8287000000000001e-05 2023-10-09T10:51:12.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.201001+0000", 2023-10-09T10:51:12.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011496632999999999 2023-10-09T10:51:12.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.272 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.272 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.201044+0000", 2023-10-09T10:51:12.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3078000000000003e-05 2023-10-09T10:51:12.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.272 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.272 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204753+0000", 2023-10-09T10:51:12.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037093209999999998 2023-10-09T10:51:12.272 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.272 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.273 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.273 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42757 6.10 6:08e1bcf5:::benchmark_data_smithi125_244951_object42756:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.273 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.333229+0000", 2023-10-09T10:51:12.273 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 376.84848114599998, 2023-10-09T10:51:12.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.872111463, 2023-10-09T10:51:12.273 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.273 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.273 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.273 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.273 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.274 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42757 2023-10-09T10:51:12.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.274 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333229+0000", 2023-10-09T10:51:12.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333229+0000", 2023-10-09T10:51:12.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333228+0000", 2023-10-09T10:51:12.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:51:12.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333280+0000", 2023-10-09T10:51:12.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1999999999999997e-05 2023-10-09T10:51:12.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333291+0000", 2023-10-09T10:51:12.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0738e-05 2023-10-09T10:51:12.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333298+0000", 2023-10-09T10:51:12.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7730000000000002e-06 2023-10-09T10:51:12.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333360+0000", 2023-10-09T10:51:12.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2230000000000006e-05 2023-10-09T10:51:12.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333371+0000", 2023-10-09T10:51:12.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0669999999999999e-05 2023-10-09T10:51:12.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104027+0000", 2023-10-09T10:51:12.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.77065640400000002 2023-10-09T10:51:12.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104035+0000", 2023-10-09T10:51:12.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7400000000000004e-06 2023-10-09T10:51:12.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.188337+0000", 2023-10-09T10:51:12.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.084302281 2023-10-09T10:51:12.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.188495+0000", 2023-10-09T10:51:12.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015774699999999999 2023-10-09T10:51:12.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.188674+0000", 2023-10-09T10:51:12.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000178729 2023-10-09T10:51:12.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191542+0000", 2023-10-09T10:51:12.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028680799999999998 2023-10-09T10:51:12.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192592+0000", 2023-10-09T10:51:12.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010502140000000001 2023-10-09T10:51:12.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204808+0000", 2023-10-09T10:51:12.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012216595 2023-10-09T10:51:12.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204867+0000", 2023-10-09T10:51:12.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8251999999999997e-05 2023-10-09T10:51:12.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205341+0000", 2023-10-09T10:51:12.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047385999999999999 2023-10-09T10:51:12.281 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.281 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.281 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.281 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42857 6.10 6:0e113cab:::benchmark_data_smithi125_244951_object42856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.281 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.409599+0000", 2023-10-09T10:51:12.281 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 376.77211157699998, 2023-10-09T10:51:12.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.796322089, 2023-10-09T10:51:12.282 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.282 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.282 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.282 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.282 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.282 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42857 2023-10-09T10:51:12.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.282 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409599+0000", 2023-10-09T10:51:12.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409599+0000", 2023-10-09T10:51:12.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409597+0000", 2023-10-09T10:51:12.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:12.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409619+0000", 2023-10-09T10:51:12.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1202000000000001e-05 2023-10-09T10:51:12.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409631+0000", 2023-10-09T10:51:12.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2546e-05 2023-10-09T10:51:12.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409638+0000", 2023-10-09T10:51:12.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9269999999999999e-06 2023-10-09T10:51:12.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409694+0000", 2023-10-09T10:51:12.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6025e-05 2023-10-09T10:51:12.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.409706+0000", 2023-10-09T10:51:12.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.164e-05 2023-10-09T10:51:12.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104059+0000", 2023-10-09T10:51:12.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.69435323100000002 2023-10-09T10:51:12.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104067+0000", 2023-10-09T10:51:12.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9060000000000002e-06 2023-10-09T10:51:12.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189529+0000", 2023-10-09T10:51:12.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.085462189 2023-10-09T10:51:12.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189675+0000", 2023-10-09T10:51:12.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014598599999999999 2023-10-09T10:51:12.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189846+0000", 2023-10-09T10:51:12.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017139799999999999 2023-10-09T10:51:12.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192547+0000", 2023-10-09T10:51:12.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027008760000000001 2023-10-09T10:51:12.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.288 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193395+0000", 2023-10-09T10:51:12.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00084811200000000004 2023-10-09T10:51:12.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.288 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205386+0000", 2023-10-09T10:51:12.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011990763999999999 2023-10-09T10:51:12.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205435+0000", 2023-10-09T10:51:12.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8467999999999997e-05 2023-10-09T10:51:12.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205921+0000", 2023-10-09T10:51:12.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00048609599999999999 2023-10-09T10:51:12.289 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.289 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.289 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.290 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42860 6.10 6:08b59805:::benchmark_data_smithi125_244951_object42859:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.290 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.411356+0000", 2023-10-09T10:51:12.290 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 376.77035381399998, 2023-10-09T10:51:12.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7952376650000001, 2023-10-09T10:51:12.290 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.290 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.290 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.290 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.290 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.291 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42860 2023-10-09T10:51:12.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.291 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411356+0000", 2023-10-09T10:51:12.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411356+0000", 2023-10-09T10:51:12.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411355+0000", 2023-10-09T10:51:12.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:12.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411383+0000", 2023-10-09T10:51:12.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7348999999999999e-05 2023-10-09T10:51:12.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411398+0000", 2023-10-09T10:51:12.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5424e-05 2023-10-09T10:51:12.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411407+0000", 2023-10-09T10:51:12.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2429999999999994e-06 2023-10-09T10:51:12.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411465+0000", 2023-10-09T10:51:12.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7892000000000001e-05 2023-10-09T10:51:12.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.411475+0000", 2023-10-09T10:51:12.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0066e-05 2023-10-09T10:51:12.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104091+0000", 2023-10-09T10:51:12.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.69261584300000001 2023-10-09T10:51:12.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104099+0000", 2023-10-09T10:51:12.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8760000000000005e-06 2023-10-09T10:51:12.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.190559+0000", 2023-10-09T10:51:12.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.086459659 2023-10-09T10:51:12.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.190698+0000", 2023-10-09T10:51:12.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013982400000000001 2023-10-09T10:51:12.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.190864+0000", 2023-10-09T10:51:12.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016535200000000001 2023-10-09T10:51:12.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192601+0000", 2023-10-09T10:51:12.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0017371999999999999 2023-10-09T10:51:12.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193434+0000", 2023-10-09T10:51:12.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00083267900000000001 2023-10-09T10:51:12.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206008+0000", 2023-10-09T10:51:12.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012574058000000001 2023-10-09T10:51:12.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206060+0000", 2023-10-09T10:51:12.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2710999999999997e-05 2023-10-09T10:51:12.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206594+0000", 2023-10-09T10:51:12.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00053372399999999996 2023-10-09T10:51:12.298 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.298 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.298 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.298 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42870 6.10 6:0e8b4417:::benchmark_data_smithi125_244951_object42869:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.298 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.417961+0000", 2023-10-09T10:51:12.298 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 376.76374912400001, 2023-10-09T10:51:12.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8001529010000001, 2023-10-09T10:51:12.299 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.299 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.299 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.299 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.299 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.299 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42870 2023-10-09T10:51:12.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.299 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417961+0000", 2023-10-09T10:51:12.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417961+0000", 2023-10-09T10:51:12.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417960+0000", 2023-10-09T10:51:12.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:12.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417982+0000", 2023-10-09T10:51:12.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2585000000000001e-05 2023-10-09T10:51:12.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.417999+0000", 2023-10-09T10:51:12.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6711999999999999e-05 2023-10-09T10:51:12.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.418005+0000", 2023-10-09T10:51:12.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3180000000000003e-06 2023-10-09T10:51:12.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.418044+0000", 2023-10-09T10:51:12.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9171999999999997e-05 2023-10-09T10:51:12.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.418052+0000", 2023-10-09T10:51:12.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9880000000000004e-06 2023-10-09T10:51:12.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104123+0000", 2023-10-09T10:51:12.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.68607037299999996 2023-10-09T10:51:12.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104131+0000", 2023-10-09T10:51:12.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0730000000000005e-06 2023-10-09T10:51:12.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193936+0000", 2023-10-09T10:51:12.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0898054349999999 2023-10-09T10:51:12.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.194029+0000", 2023-10-09T10:51:12.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2313999999999997e-05 2023-10-09T10:51:12.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.194131+0000", 2023-10-09T10:51:12.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000102384 2023-10-09T10:51:12.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199366+0000", 2023-10-09T10:51:12.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0052347640000000003 2023-10-09T10:51:12.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205969+0000", 2023-10-09T10:51:12.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066036840000000003 2023-10-09T10:51:12.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.217437+0000", 2023-10-09T10:51:12.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011467252000000001 2023-10-09T10:51:12.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.217474+0000", 2023-10-09T10:51:12.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7177e-05 2023-10-09T10:51:12.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218114+0000", 2023-10-09T10:51:12.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000640231 2023-10-09T10:51:12.306 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.306 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.306 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.307 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42902 6.10 6:0d48f256:::benchmark_data_smithi125_244951_object42901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.307 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.448090+0000", 2023-10-09T10:51:12.307 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 376.73362066599998, 2023-10-09T10:51:12.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7704487879999999, 2023-10-09T10:51:12.307 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.307 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.307 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.307 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.307 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.307 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42902 2023-10-09T10:51:12.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.307 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448090+0000", 2023-10-09T10:51:12.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448090+0000", 2023-10-09T10:51:12.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448088+0000", 2023-10-09T10:51:12.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:51:12.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448108+0000", 2023-10-09T10:51:12.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9888999999999999e-05 2023-10-09T10:51:12.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448124+0000", 2023-10-09T10:51:12.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5529999999999999e-05 2023-10-09T10:51:12.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448131+0000", 2023-10-09T10:51:12.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9500000000000004e-06 2023-10-09T10:51:12.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448189+0000", 2023-10-09T10:51:12.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8010000000000002e-05 2023-10-09T10:51:12.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.448201+0000", 2023-10-09T10:51:12.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2368000000000001e-05 2023-10-09T10:51:12.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104155+0000", 2023-10-09T10:51:12.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.65595361500000005 2023-10-09T10:51:12.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104163+0000", 2023-10-09T10:51:12.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9759999999999995e-06 2023-10-09T10:51:12.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.194936+0000", 2023-10-09T10:51:12.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.090773378 2023-10-09T10:51:12.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.195074+0000", 2023-10-09T10:51:12.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000138298 2023-10-09T10:51:12.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.196676+0000", 2023-10-09T10:51:12.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001601327 2023-10-09T10:51:12.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199373+0000", 2023-10-09T10:51:12.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026971320000000001 2023-10-09T10:51:12.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206643+0000", 2023-10-09T10:51:12.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0072700519999999999 2023-10-09T10:51:12.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218147+0000", 2023-10-09T10:51:12.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011503904000000001 2023-10-09T10:51:12.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218182+0000", 2023-10-09T10:51:12.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4752999999999997e-05 2023-10-09T10:51:12.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218538+0000", 2023-10-09T10:51:12.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035677400000000002 2023-10-09T10:51:12.315 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.315 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.315 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.315 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42906 6.10 6:0945c66e:::benchmark_data_smithi125_244951_object42905:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:51:12.315 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.453858+0000", 2023-10-09T10:51:12.315 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 376.72785255299999, 2023-10-09T10:51:12.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7650414240000001, 2023-10-09T10:51:12.315 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.315 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.315 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.316 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.316 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.316 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42906 2023-10-09T10:51:12.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.316 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453858+0000", 2023-10-09T10:51:12.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453858+0000", 2023-10-09T10:51:12.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453857+0000", 2023-10-09T10:51:12.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:51:12.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453877+0000", 2023-10-09T10:51:12.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0857999999999999e-05 2023-10-09T10:51:12.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453891+0000", 2023-10-09T10:51:12.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3249e-05 2023-10-09T10:51:12.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453897+0000", 2023-10-09T10:51:12.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1129999999999997e-06 2023-10-09T10:51:12.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453938+0000", 2023-10-09T10:51:12.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0799e-05 2023-10-09T10:51:12.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.453945+0000", 2023-10-09T10:51:12.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2849999999999997e-06 2023-10-09T10:51:12.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104187+0000", 2023-10-09T10:51:12.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.65024259699999998 2023-10-09T10:51:12.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104195+0000", 2023-10-09T10:51:12.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9710000000000004e-06 2023-10-09T10:51:12.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.197298+0000", 2023-10-09T10:51:12.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.093102646 2023-10-09T10:51:12.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.197461+0000", 2023-10-09T10:51:12.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016253599999999999 2023-10-09T10:51:12.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.197621+0000", 2023-10-09T10:51:12.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000160178 2023-10-09T10:51:12.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200332+0000", 2023-10-09T10:51:12.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027108029999999999 2023-10-09T10:51:12.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.206690+0000", 2023-10-09T10:51:12.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0063584990000000001 2023-10-09T10:51:12.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218563+0000", 2023-10-09T10:51:12.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011872562 2023-10-09T10:51:12.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218587+0000", 2023-10-09T10:51:12.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4586000000000001e-05 2023-10-09T10:51:12.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.218899+0000", 2023-10-09T10:51:12.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000311883 2023-10-09T10:51:12.323 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.323 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.323 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.323 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:56713 6.10 6:086e6a33:::benchmark_data_smithi125_244951_object56712:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-10-09T10:51:12.324 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:45:08.647257+0000", 2023-10-09T10:51:12.324 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 363.53445337599999, 2023-10-09T10:51:12.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0123530810000001, 2023-10-09T10:51:12.324 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.324 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:51:12.324 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.324 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.324 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.324 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 56713 2023-10-09T10:51:12.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.324 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647257+0000", 2023-10-09T10:51:12.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647257+0000", 2023-10-09T10:51:12.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647256+0000", 2023-10-09T10:51:12.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:12.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647273+0000", 2023-10-09T10:51:12.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6075000000000001e-05 2023-10-09T10:51:12.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647281+0000", 2023-10-09T10:51:12.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6589999999999996e-06 2023-10-09T10:51:12.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647287+0000", 2023-10-09T10:51:12.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4940000000000003e-06 2023-10-09T10:51:12.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647464+0000", 2023-10-09T10:51:12.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017730999999999999 2023-10-09T10:51:12.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647667+0000", 2023-10-09T10:51:12.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020269 2023-10-09T10:51:12.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:51:12.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.647835+0000", 2023-10-09T10:51:12.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000168677 2023-10-09T10:51:12.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.649455+0000", 2023-10-09T10:51:12.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016191999999999999 2023-10-09T10:51:12.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.655385+0000", 2023-10-09T10:51:12.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0059305549999999997 2023-10-09T10:51:12.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.658642+0000", 2023-10-09T10:51:12.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032573540000000001 2023-10-09T10:51:12.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:12.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:08.658657+0000", 2023-10-09T10:51:12.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4354e-05 2023-10-09T10:51:12.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.330 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.659536+0000", 2023-10-09T10:51:12.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0008789929999999 2023-10-09T10:51:12.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.330 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:45:09.659610+0000", 2023-10-09T10:51:12.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4134999999999994e-05 2023-10-09T10:51:12.330 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.330 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.330 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.331 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPush(6.11 155/143 [PushOp(6:8b7ae795:::benchmark_data_smithi125_244951_object18078:head, version: 104'1108, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:8b7ae795:::benchmark_data_smithi125_244951_object18078:head@104'1108, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:51:12.331 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:33.349257+0000", 2023-10-09T10:51:12.331 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 278.83245273599999, 2023-10-09T10:51:12.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1771969579999999, 2023-10-09T10:51:12.331 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.331 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:51:12.331 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349257+0000", 2023-10-09T10:51:12.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349257+0000", 2023-10-09T10:51:12.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349256+0000", 2023-10-09T10:51:12.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:12.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349302+0000", 2023-10-09T10:51:12.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5671999999999999e-05 2023-10-09T10:51:12.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349333+0000", 2023-10-09T10:51:12.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1256e-05 2023-10-09T10:51:12.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.349351+0000", 2023-10-09T10:51:12.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8303000000000001e-05 2023-10-09T10:51:12.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.525188+0000", 2023-10-09T10:51:12.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.175836713 2023-10-09T10:51:12.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.525199+0000", 2023-10-09T10:51:12.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0458e-05 2023-10-09T10:51:12.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.335 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.526454+0000", 2023-10-09T10:51:12.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001255927 2023-10-09T10:51:12.335 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.335 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.335 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.335 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPush(6.11 155/143 [PushOp(6:8ff63630:::benchmark_data_smithi125_244951_object18084:head, version: 104'1109, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:8ff63630:::benchmark_data_smithi125_244951_object18084:head@104'1109, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:51:12.335 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:33.403563+0000", 2023-10-09T10:51:12.335 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 278.77814748399999, 2023-10-09T10:51:12.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1637633279999999, 2023-10-09T10:51:12.336 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.336 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:51:12.336 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403563+0000", 2023-10-09T10:51:12.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403563+0000", 2023-10-09T10:51:12.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403561+0000", 2023-10-09T10:51:12.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:12.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403594+0000", 2023-10-09T10:51:12.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2743999999999997e-05 2023-10-09T10:51:12.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403612+0000", 2023-10-09T10:51:12.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8389e-05 2023-10-09T10:51:12.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.403626+0000", 2023-10-09T10:51:12.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3458999999999999e-05 2023-10-09T10:51:12.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.565733+0000", 2023-10-09T10:51:12.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.162107429 2023-10-09T10:51:12.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.565745+0000", 2023-10-09T10:51:12.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1355000000000001e-05 2023-10-09T10:51:12.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.567326+0000", 2023-10-09T10:51:12.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015813979999999999 2023-10-09T10:51:12.339 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.339 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.339 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.340 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:123629 6.9 6:932bc8aa:::benchmark_data_smithi125_244951_object17186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-10-09T10:51:12.340 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:33.543425+0000", 2023-10-09T10:51:12.340 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 278.63828489100001, 2023-10-09T10:51:12.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.002534984, 2023-10-09T10:51:12.340 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:12.340 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:12.340 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:12.340 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:12.340 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:12.340 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 123629 2023-10-09T10:51:12.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.341 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:12.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:12.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543425+0000", 2023-10-09T10:51:12.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:12.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543425+0000", 2023-10-09T10:51:12.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:12.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:12.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543424+0000", 2023-10-09T10:51:12.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:51:12.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:12.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543432+0000", 2023-10-09T10:51:12.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1389999999999995e-06 2023-10-09T10:51:12.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:12.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543448+0000", 2023-10-09T10:51:12.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5979000000000001e-05 2023-10-09T10:51:12.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:12.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:33.543458+0000", 2023-10-09T10:51:12.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.859e-06 2023-10-09T10:51:12.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:12.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.520483+0000", 2023-10-09T10:51:12.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97702594300000001 2023-10-09T10:51:12.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.344 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:12.344 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.520610+0000", 2023-10-09T10:51:12.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012663799999999999 2023-10-09T10:51:12.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.344 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:51:12.344 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.520719+0000", 2023-10-09T10:51:12.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010933999999999999 2023-10-09T10:51:12.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.344 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.525132+0000", 2023-10-09T10:51:12.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0044124419999999999 2023-10-09T10:51:12.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:12.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.534848+0000", 2023-10-09T10:51:12.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0097164620000000004 2023-10-09T10:51:12.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:12.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.544664+0000", 2023-10-09T10:51:12.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0098156279999999999 2023-10-09T10:51:12.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:12.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.544725+0000", 2023-10-09T10:51:12.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1042000000000003e-05 2023-10-09T10:51:12.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:12.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:12.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:12.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:34.545960+0000", 2023-10-09T10:51:12.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001235311 2023-10-09T10:51:12.346 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.346 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.347 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.347 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:12.347 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:12.347 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:51:12.347 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:12.383 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:12.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:12.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:12.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:12.517 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:12.575 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:12.653 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:12.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:12 smithi125 ceph-mon[195507]: pgmap v532: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 11 KiB/s rd, 29 KiB/s wr, 702 op/s; 22723/237981 objects degraded (9.548%); 4033/237981 objects misplaced (1.695%); 11 MiB/s, 176 objects/s recovering 2023-10-09T10:51:12.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:12 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-10-09T10:51:12.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:12 smithi125 ceph-mon[195507]: osdmap e233: 8 total, 8 up, 5 in 2023-10-09T10:51:12.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:12 smithi125 ceph-mon[195507]: failed to encode map e233 with expected crc 2023-10-09T10:51:12.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:12 smithi125 ceph-mon[195507]: failed to encode map e233 with expected crc 2023-10-09T10:51:12.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:12 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1114429424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:12.657 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:12 smithi125 ceph-mon[200197]: pgmap v532: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 11 KiB/s rd, 29 KiB/s wr, 702 op/s; 22723/237981 objects degraded (9.548%); 4033/237981 objects misplaced (1.695%); 11 MiB/s, 176 objects/s recovering 2023-10-09T10:51:12.657 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:12 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-10-09T10:51:12.657 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:12 smithi125 ceph-mon[200197]: osdmap e233: 8 total, 8 up, 5 in 2023-10-09T10:51:12.657 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:12 smithi125 ceph-mon[200197]: failed to encode map e233 with expected crc 2023-10-09T10:51:12.657 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:12 smithi125 ceph-mon[200197]: failed to encode map e233 with expected crc 2023-10-09T10:51:12.657 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:12 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1114429424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:12.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:12.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:12 smithi155 ceph-mon[184854]: pgmap v532: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 11 KiB/s rd, 29 KiB/s wr, 702 op/s; 22723/237981 objects degraded (9.548%); 4033/237981 objects misplaced (1.695%); 11 MiB/s, 176 objects/s recovering 2023-10-09T10:51:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:12 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-10-09T10:51:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:12 smithi155 ceph-mon[184854]: osdmap e233: 8 total, 8 up, 5 in 2023-10-09T10:51:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:12 smithi155 ceph-mon[184854]: failed to encode map e233 with expected crc 2023-10-09T10:51:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:12 smithi155 ceph-mon[184854]: failed to encode map e233 with expected crc 2023-10-09T10:51:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:12 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1114429424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:12.759 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:12.813 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:12.897 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:12.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:12.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:12.936 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:51:12.956 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:13.009 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:13.054 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:13.079 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:13.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:13.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:13.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:13.232 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:13.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:13.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:13.485 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:13 smithi125 ceph-mon[195507]: failed to encode map e234 with expected crc 2023-10-09T10:51:13.485 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:13 smithi125 ceph-mon[195507]: failed to encode map e234 with expected crc 2023-10-09T10:51:13.485 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:13 smithi125 ceph-mon[195507]: osdmap e234: 8 total, 8 up, 5 in 2023-10-09T10:51:13.485 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:13 smithi125 ceph-mon[195507]: failed to encode map e234 with expected crc 2023-10-09T10:51:13.485 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:13 smithi125 ceph-mon[195507]: failed to encode map e234 with expected crc 2023-10-09T10:51:13.485 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:13 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 22723/237981 objects degraded (9.548%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:13.486 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:13 smithi125 ceph-mon[200197]: failed to encode map e234 with expected crc 2023-10-09T10:51:13.486 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:13 smithi125 ceph-mon[200197]: failed to encode map e234 with expected crc 2023-10-09T10:51:13.486 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:13 smithi125 ceph-mon[200197]: osdmap e234: 8 total, 8 up, 5 in 2023-10-09T10:51:13.486 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:13 smithi125 ceph-mon[200197]: failed to encode map e234 with expected crc 2023-10-09T10:51:13.486 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:13 smithi125 ceph-mon[200197]: failed to encode map e234 with expected crc 2023-10-09T10:51:13.486 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:13 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 22723/237981 objects degraded (9.548%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:13.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:13.588 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:13.620 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:13 smithi155 ceph-mon[184854]: failed to encode map e234 with expected crc 2023-10-09T10:51:13.620 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:13 smithi155 ceph-mon[184854]: failed to encode map e234 with expected crc 2023-10-09T10:51:13.620 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:13 smithi155 ceph-mon[184854]: osdmap e234: 8 total, 8 up, 5 in 2023-10-09T10:51:13.620 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:13 smithi155 ceph-mon[184854]: failed to encode map e234 with expected crc 2023-10-09T10:51:13.620 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:13 smithi155 ceph-mon[184854]: failed to encode map e234 with expected crc 2023-10-09T10:51:13.620 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:13 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 22723/237981 objects degraded (9.548%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:13.622 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:13.660 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:13.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:13.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:13.765 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:13.810 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:13.816 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:13.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:13.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:13.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:13.947 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:13.995 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:14.020 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:14.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:14.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:14.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:14.217 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:14.355 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:14 smithi155 ceph-mon[184854]: pgmap v535: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 824 op/s; 22286/232362 objects degraded (9.591%); 4033/232362 objects misplaced (1.736%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T10:51:14.473 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:14.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:14.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:14.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:14 smithi125 ceph-mon[195507]: pgmap v535: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 824 op/s; 22286/232362 objects degraded (9.591%); 4033/232362 objects misplaced (1.736%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T10:51:14.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:14 smithi125 ceph-mon[200197]: pgmap v535: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 824 op/s; 22286/232362 objects degraded (9.591%); 4033/232362 objects misplaced (1.736%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T10:51:14.580 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:14.628 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:14.659 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:14.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:14.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:14.765 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:14.797 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:14.823 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:14.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:14.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:14.928 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:14.966 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:51:14.972 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:15.004 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:15.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:15.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:15.107 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:15.167 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:15.193 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:15.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:15.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:15.300 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:15.356 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:15.406 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:51:15.680 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:15 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1327902406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:51:15.680 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:15 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:51:15.680 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:15.680 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:51:15.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:15 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1327902406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:51:15.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:15 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:51:15.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:15.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:51:15.689 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:15.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:15.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:15 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1327902406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:51:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:15 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:51:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:51:15.796 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:15.861 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:16.242 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:16.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:16.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:16.346 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:16.397 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:51:16.397 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:51:16.397 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 0 2023-10-09T10:51:16.398 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:51:16.413 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:16.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:16 smithi125 ceph-mon[195507]: pgmap v536: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 743 op/s; 22286/229473 objects degraded (9.712%); 3511/229473 objects misplaced (1.530%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T10:51:16.498 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:16 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:16.498 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:16 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:51:16.498 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:16 smithi125 ceph-mon[195507]: osdmap e235: 8 total, 8 up, 5 in 2023-10-09T10:51:16.498 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:16 smithi125 ceph-mon[195507]: failed to encode map e235 with expected crc 2023-10-09T10:51:16.500 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:16 smithi125 ceph-mon[200197]: pgmap v536: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 743 op/s; 22286/229473 objects degraded (9.712%); 3511/229473 objects misplaced (1.530%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T10:51:16.500 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:16 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:16.500 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:16 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:51:16.500 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:16 smithi125 ceph-mon[200197]: osdmap e235: 8 total, 8 up, 5 in 2023-10-09T10:51:16.500 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:16 smithi125 ceph-mon[200197]: failed to encode map e235 with expected crc 2023-10-09T10:51:16.502 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:16.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:16.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:16.613 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:16.670 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:16 smithi155 ceph-mon[184854]: pgmap v536: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 743 op/s; 22286/229473 objects degraded (9.712%); 3511/229473 objects misplaced (1.530%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T10:51:16.670 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:16 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:16.670 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:16 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:51:16.670 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:16 smithi155 ceph-mon[184854]: osdmap e235: 8 total, 8 up, 5 in 2023-10-09T10:51:16.670 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:16 smithi155 ceph-mon[184854]: failed to encode map e235 with expected crc 2023-10-09T10:51:16.672 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:16.703 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:16.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:16.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:16.745 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:16.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:16.828 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0] out_osds: [2, 7, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:51:16.828 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:51:16.828 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T10:51:16.829 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:51:16.855 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:16.884 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:16.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:16.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:16.993 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:17.071 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:17.182 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:51:17.207 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:17.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:17.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:17.313 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:17.376 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:17.467 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:17.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:17.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:17.570 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:17.596 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:17.603 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:17.631 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:17.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:17.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:17.735 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:17.786 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:17.877 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:17.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:17.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:17.983 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:18.029 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:18.044 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:51:18.059 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:18.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:18.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:18.085 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:51:18.123 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:51:18.136 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:18.163 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:18.182 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:51:18.208 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:18.213 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:51:18.252 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:51:18.285 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:51:18.323 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:51:18.334 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:51:18.334 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:51:18.334 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:51:18.334 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:51:18.335 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:51:18.335 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:51:18.335 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:51:18.335 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:51:18.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:18 smithi155 ceph-mon[184854]: pgmap v538: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.6 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 592 op/s; 22286/227238 objects degraded (9.807%); 3511/227238 objects misplaced (1.545%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T10:51:18.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:18 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 22286/227238 objects degraded (9.807%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:18.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1658767971' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:18.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:51:18.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:18 smithi125 ceph-mon[195507]: pgmap v538: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.6 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 592 op/s; 22286/227238 objects degraded (9.807%); 3511/227238 objects misplaced (1.545%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T10:51:18.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:18 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 22286/227238 objects degraded (9.807%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:18.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:18 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1658767971' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:18.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:18 smithi125 ceph-mon[200197]: pgmap v538: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.6 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 592 op/s; 22286/227238 objects degraded (9.807%); 3511/227238 objects misplaced (1.545%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T10:51:18.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:18 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 22286/227238 objects degraded (9.807%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:18.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:18 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1658767971' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:18.597 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:18.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:18.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:18.702 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:18.786 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:18.819 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:18.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:18.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:18.925 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:18.959 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:18.994 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:19.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:19.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:19.104 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:19.158 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:19.502 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:19.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:19.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:19.517 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:19.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:19.645 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:19.667 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:19.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:19.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:19.771 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:19.846 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:19.924 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:51:19.962 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:51:20.082 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:20.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:20.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:20.190 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:20.263 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:20.300 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:51:20.301 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:51:20.534 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:20 smithi125 ceph-mon[200197]: pgmap v539: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 552 op/s; 21587/222405 objects degraded (9.706%); 3511/222405 objects misplaced (1.579%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T10:51:20.536 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:20 smithi125 ceph-mon[195507]: pgmap v539: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 552 op/s; 21587/222405 objects degraded (9.706%); 3511/222405 objects misplaced (1.579%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T10:51:20.566 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":539,"stamp":"2023-10-09T10:51:19.262024+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4857629503,"num_objects":74135,"num_object_clones":0,"num_object_copies":222405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21587,"num_objects_misplaced":3511,"num_objects_unfound":0,"num_objects_dirty":74135,"num_whiteouts":0,"num_read":295,"num_read_kb":555,"num_write":108363,"num_write_kb":5834069,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12312,"num_bytes_recovered":806512693,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73731,"ondisk_log_size":73731,"up":357,"acting":349,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":14712348,"kb_used_data":13074676,"kb_used_omap":35,"kb_used_meta":1637468,"kb_avail":454013412,"statfs":{"total":479975178240,"available":464909733888,"internally_reserved":0,"allocated":13388468224,"data_stored":13383313615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36424,"internal_metadata":1676767672},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,6,1,5,4,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-288739750,"num_objects":-4405,"num_object_clones":0,"num_object_copies":-13215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1136,"num_objects_misplaced":-522,"num_objects_unfound":0,"num_objects_dirty":-4405,"num_whiteouts":0,"num_read":-44,"num_read_kb":-82,"num_write":4383,"num_write_kb":-220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":972,"num_bytes_recovered":63277840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.931846"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":570,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.353697+0000","last_change":"2023-10-09T10:51:01.313699+0000","last_active":"2023-10-09T10:51:12.353697+0000","last_peered":"2023-10-09T10:51:12.353697+0000","last_clean":"2023-10-09T10:51:12.353697+0000","last_became_active":"2023-10-09T10:51:01.313365+0000","last_became_peered":"2023-10-09T10:51:01.313365+0000","last_unstale":"2023-10-09T10:51:12.353697+0000","last_undegraded":"2023-10-09T10:51:12.353697+0000","last_fullsized":"2023-10-09T10:51:12.353697+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":211849,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":161,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.354292+0000","last_change":"2023-10-09T10:50:43.385056+0000","last_active":"2023-10-09T10:51:12.354292+0000","last_peered":"2023-10-09T10:51:12.354292+0000","last_clean":"2023-10-09T10:51:12.354292+0000","last_became_active":"2023-10-09T10:50:43.384646+0000","last_became_peered":"2023-10-09T10:50:43.384646+0000","last_unstale":"2023-10-09T10:51:12.354292+0000","last_undegraded":"2023-10-09T10:51:12.354292+0000","last_fullsized":"2023-10-09T10:51:12.354292+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:42:25.339628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":213,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.365824+0000","last_change":"2023-10-09T10:49:55.811543+0000","last_active":"2023-10-09T10:51:11.365824+0000","last_peered":"2023-10-09T10:51:11.365824+0000","last_clean":"2023-10-09T10:51:11.365824+0000","last_became_active":"2023-10-09T10:49:55.811225+0000","last_became_peered":"2023-10-09T10:49:55.811225+0000","last_unstale":"2023-10-09T10:51:11.365824+0000","last_undegraded":"2023-10-09T10:51:11.365824+0000","last_fullsized":"2023-10-09T10:51:11.365824+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:56.330889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":590,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.352538+0000","last_change":"2023-10-09T10:51:01.307749+0000","last_active":"2023-10-09T10:51:12.352538+0000","last_peered":"2023-10-09T10:51:12.352538+0000","last_clean":"2023-10-09T10:51:12.352538+0000","last_became_active":"2023-10-09T10:51:01.307513+0000","last_became_peered":"2023-10-09T10:51:01.307513+0000","last_unstale":"2023-10-09T10:51:12.352538+0000","last_undegraded":"2023-10-09T10:51:12.352538+0000","last_fullsized":"2023-10-09T10:51:12.352538+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.892515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.892515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":75,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.364653+0000","last_change":"2023-10-09T10:49:55.813068+0000","last_active":"2023-10-09T10:51:11.364653+0000","last_peered":"2023-10-09T10:51:11.364653+0000","last_clean":"2023-10-09T10:51:11.364653+0000","last_became_active":"2023-10-09T10:49:55.811847+0000","last_became_peered":"2023-10-09T10:49:55.811847+0000","last_unstale":"2023-10-09T10:51:11.364653+0000","last_undegraded":"2023-10-09T10:51:11.364653+0000","last_fullsized":"2023-10-09T10:51:11.364653+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:35.889323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":123,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.353997+0000","last_change":"2023-10-09T10:49:55.838051+0000","last_active":"2023-10-09T10:51:12.353997+0000","last_peered":"2023-10-09T10:51:12.353997+0000","last_clean":"2023-10-09T10:51:12.353997+0000","last_became_active":"2023-10-09T10:49:55.836999+0000","last_became_peered":"2023-10-09T10:49:55.836999+0000","last_unstale":"2023-10-09T10:51:12.353997+0000","last_undegraded":"2023-10-09T10:51:12.353997+0000","last_fullsized":"2023-10-09T10:51:12.353997+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:00:35.849494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":580,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.357041+0000","last_change":"2023-10-09T10:51:01.313091+0000","last_active":"2023-10-09T10:51:12.357041+0000","last_peered":"2023-10-09T10:51:12.357041+0000","last_clean":"2023-10-09T10:51:12.357041+0000","last_became_active":"2023-10-09T10:51:01.312840+0000","last_became_peered":"2023-10-09T10:51:01.312840+0000","last_unstale":"2023-10-09T10:51:12.357041+0000","last_undegraded":"2023-10-09T10:51:12.357041+0000","last_fullsized":"2023-10-09T10:51:12.357041+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":249,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.368242+0000","last_change":"2023-10-09T10:49:55.820195+0000","last_active":"2023-10-09T10:51:11.368242+0000","last_peered":"2023-10-09T10:51:11.368242+0000","last_clean":"2023-10-09T10:51:11.368242+0000","last_became_active":"2023-10-09T10:49:55.819612+0000","last_became_peered":"2023-10-09T10:49:55.819612+0000","last_unstale":"2023-10-09T10:51:11.368242+0000","last_undegraded":"2023-10-09T10:51:11.368242+0000","last_fullsized":"2023-10-09T10:51:11.368242+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:40.995492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":178,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.362970+0000","last_change":"2023-10-09T10:51:12.360637+0000","last_active":"2023-10-09T10:51:12.362970+0000","last_peered":"2023-10-09T10:51:12.362970+0000","last_clean":"2023-10-09T10:51:12.362970+0000","last_became_active":"2023-10-09T10:51:12.360339+0000","last_became_peered":"2023-10-09T10:51:12.360339+0000","last_unstale":"2023-10-09T10:51:12.362970+0000","last_undegraded":"2023-10-09T10:51:12.362970+0000","last_fullsized":"2023-10-09T10:51:12.362970+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:03.903563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":599,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.360282+0000","last_change":"2023-10-09T10:51:01.315336+0000","last_active":"2023-10-09T10:51:12.360282+0000","last_peered":"2023-10-09T10:51:12.360282+0000","last_clean":"2023-10-09T10:51:12.360282+0000","last_became_active":"2023-10-09T10:51:01.314959+0000","last_became_peered":"2023-10-09T10:51:01.314959+0000","last_unstale":"2023-10-09T10:51:12.360282+0000","last_undegraded":"2023-10-09T10:51:12.360282+0000","last_fullsized":"2023-10-09T10:51:12.360282+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.829081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.829081+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001277842,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":105,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.358825+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:51:12.358825+0000","last_peered":"2023-10-09T10:51:12.358825+0000","last_clean":"2023-10-09T10:51:12.358825+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:51:12.358825+0000","last_undegraded":"2023-10-09T10:51:12.358825+0000","last_fullsized":"2023-10-09T10:51:12.358825+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":213,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.368323+0000","last_change":"2023-10-09T10:49:55.811228+0000","last_active":"2023-10-09T10:51:11.368323+0000","last_peered":"2023-10-09T10:51:11.368323+0000","last_clean":"2023-10-09T10:51:11.368323+0000","last_became_active":"2023-10-09T10:49:55.810914+0000","last_became_peered":"2023-10-09T10:49:55.810914+0000","last_unstale":"2023-10-09T10:51:11.368323+0000","last_undegraded":"2023-10-09T10:51:11.368323+0000","last_fullsized":"2023-10-09T10:51:11.368323+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:05:35.753543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":497,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:13.060918+0000","last_change":"2023-10-09T10:51:12.365654+0000","last_active":"2023-10-09T10:51:13.060918+0000","last_peered":"2023-10-09T10:51:13.060918+0000","last_clean":"2023-10-09T10:51:13.060918+0000","last_became_active":"2023-10-09T10:51:12.365446+0000","last_became_peered":"2023-10-09T10:51:12.365446+0000","last_unstale":"2023-10-09T10:51:13.060918+0000","last_undegraded":"2023-10-09T10:51:13.060918+0000","last_fullsized":"2023-10-09T10:51:13.060918+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":220,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.354233+0000","last_change":"2023-10-09T10:49:55.824836+0000","last_active":"2023-10-09T10:51:12.354233+0000","last_peered":"2023-10-09T10:51:12.354233+0000","last_clean":"2023-10-09T10:51:12.354233+0000","last_became_active":"2023-10-09T10:49:55.824581+0000","last_became_peered":"2023-10-09T10:49:55.824581+0000","last_unstale":"2023-10-09T10:51:12.354233+0000","last_undegraded":"2023-10-09T10:51:12.354233+0000","last_fullsized":"2023-10-09T10:51:12.354233+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:12:05.320238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":170,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.364579+0000","last_change":"2023-10-09T10:49:40.951753+0000","last_active":"2023-10-09T10:51:11.364579+0000","last_peered":"2023-10-09T10:51:11.364579+0000","last_clean":"2023-10-09T10:51:11.364579+0000","last_became_active":"2023-10-09T10:49:40.951441+0000","last_became_peered":"2023-10-09T10:49:40.951441+0000","last_unstale":"2023-10-09T10:51:11.364579+0000","last_undegraded":"2023-10-09T10:51:11.364579+0000","last_fullsized":"2023-10-09T10:51:11.364579+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:13.222334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":600,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.367586+0000","last_change":"2023-10-09T10:51:12.365018+0000","last_active":"2023-10-09T10:51:12.367586+0000","last_peered":"2023-10-09T10:51:12.367586+0000","last_clean":"2023-10-09T10:51:12.367586+0000","last_became_active":"2023-10-09T10:51:12.364541+0000","last_became_peered":"2023-10-09T10:51:12.364541+0000","last_unstale":"2023-10-09T10:51:12.367586+0000","last_undegraded":"2023-10-09T10:51:12.367586+0000","last_fullsized":"2023-10-09T10:51:12.367586+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.687290+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.687290+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012763172999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":171,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.366698+0000","last_change":"2023-10-09T10:49:40.968706+0000","last_active":"2023-10-09T10:51:11.366698+0000","last_peered":"2023-10-09T10:51:11.366698+0000","last_clean":"2023-10-09T10:51:11.366698+0000","last_became_active":"2023-10-09T10:49:40.947149+0000","last_became_peered":"2023-10-09T10:49:40.947149+0000","last_unstale":"2023-10-09T10:51:11.366698+0000","last_undegraded":"2023-10-09T10:51:11.366698+0000","last_fullsized":"2023-10-09T10:51:11.366698+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:57:27.340414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":136,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.368728+0000","last_change":"2023-10-09T10:51:12.367099+0000","last_active":"2023-10-09T10:51:12.368728+0000","last_peered":"2023-10-09T10:51:12.368728+0000","last_clean":"2023-10-09T10:51:12.368728+0000","last_became_active":"2023-10-09T10:51:12.366803+0000","last_became_peered":"2023-10-09T10:51:12.366803+0000","last_unstale":"2023-10-09T10:51:12.368728+0000","last_undegraded":"2023-10-09T10:51:12.368728+0000","last_fullsized":"2023-10-09T10:51:12.368728+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:08.419462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":594,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:13.068601+0000","last_change":"2023-10-09T10:51:12.367124+0000","last_active":"2023-10-09T10:51:13.068601+0000","last_peered":"2023-10-09T10:51:13.068601+0000","last_clean":"2023-10-09T10:51:13.068601+0000","last_became_active":"2023-10-09T10:51:12.366405+0000","last_became_peered":"2023-10-09T10:51:12.366405+0000","last_unstale":"2023-10-09T10:51:13.068601+0000","last_undegraded":"2023-10-09T10:51:13.068601+0000","last_fullsized":"2023-10-09T10:51:13.068601+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.475429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.475429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001591812,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":187,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:13.068478+0000","last_change":"2023-10-09T10:51:12.366005+0000","last_active":"2023-10-09T10:51:13.068478+0000","last_peered":"2023-10-09T10:51:13.068478+0000","last_clean":"2023-10-09T10:51:13.068478+0000","last_became_active":"2023-10-09T10:51:12.365722+0000","last_became_peered":"2023-10-09T10:51:12.365722+0000","last_unstale":"2023-10-09T10:51:13.068478+0000","last_undegraded":"2023-10-09T10:51:13.068478+0000","last_fullsized":"2023-10-09T10:51:13.068478+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:31:04.499929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":209,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.359785+0000","last_change":"2023-10-09T10:49:55.818343+0000","last_active":"2023-10-09T10:51:12.359785+0000","last_peered":"2023-10-09T10:51:12.359785+0000","last_clean":"2023-10-09T10:51:12.359785+0000","last_became_active":"2023-10-09T10:49:55.818010+0000","last_became_peered":"2023-10-09T10:49:55.818010+0000","last_unstale":"2023-10-09T10:51:12.359785+0000","last_undegraded":"2023-10-09T10:51:12.359785+0000","last_fullsized":"2023-10-09T10:51:12.359785+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:21.520307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":599,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.356298+0000","last_change":"2023-10-09T10:49:18.708370+0000","last_active":"2023-10-09T10:51:12.356298+0000","last_peered":"2023-10-09T10:51:12.356298+0000","last_clean":"2023-10-09T10:51:12.356298+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:51:12.356298+0000","last_undegraded":"2023-10-09T10:51:12.356298+0000","last_fullsized":"2023-10-09T10:51:12.356298+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:57.808410+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":28,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:13.060837+0000","last_change":"2023-10-09T10:51:12.366055+0000","last_active":"2023-10-09T10:51:13.060837+0000","last_peered":"2023-10-09T10:51:13.060837+0000","last_clean":"2023-10-09T10:51:13.060837+0000","last_became_active":"2023-10-09T10:51:12.365754+0000","last_became_peered":"2023-10-09T10:51:12.365754+0000","last_unstale":"2023-10-09T10:51:13.060837+0000","last_undegraded":"2023-10-09T10:51:13.060837+0000","last_fullsized":"2023-10-09T10:51:13.060837+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:04.062342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":245,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.364636+0000","last_change":"2023-10-09T10:49:55.817307+0000","last_active":"2023-10-09T10:51:11.364636+0000","last_peered":"2023-10-09T10:51:11.364636+0000","last_clean":"2023-10-09T10:51:11.364636+0000","last_became_active":"2023-10-09T10:49:55.815385+0000","last_became_peered":"2023-10-09T10:49:55.815385+0000","last_unstale":"2023-10-09T10:51:11.364636+0000","last_undegraded":"2023-10-09T10:51:11.364636+0000","last_fullsized":"2023-10-09T10:51:11.364636+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:51:11.536505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":610,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.354318+0000","last_change":"2023-10-09T10:49:43.769673+0000","last_active":"2023-10-09T10:51:12.354318+0000","last_peered":"2023-10-09T10:51:12.354318+0000","last_clean":"2023-10-09T10:51:12.354318+0000","last_became_active":"2023-10-09T10:49:43.769323+0000","last_became_peered":"2023-10-09T10:49:43.769323+0000","last_unstale":"2023-10-09T10:51:12.354318+0000","last_undegraded":"2023-10-09T10:51:12.354318+0000","last_fullsized":"2023-10-09T10:51:12.354318+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:23.224244+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":123,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.354905+0000","last_change":"2023-10-09T10:49:55.837023+0000","last_active":"2023-10-09T10:51:12.354905+0000","last_peered":"2023-10-09T10:51:12.354905+0000","last_clean":"2023-10-09T10:51:12.354905+0000","last_became_active":"2023-10-09T10:49:55.836546+0000","last_became_peered":"2023-10-09T10:49:55.836546+0000","last_unstale":"2023-10-09T10:51:12.354905+0000","last_undegraded":"2023-10-09T10:51:12.354905+0000","last_fullsized":"2023-10-09T10:51:12.354905+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:30.585420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":209,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.355068+0000","last_change":"2023-10-09T10:49:40.952783+0000","last_active":"2023-10-09T10:51:12.355068+0000","last_peered":"2023-10-09T10:51:12.355068+0000","last_clean":"2023-10-09T10:51:12.355068+0000","last_became_active":"2023-10-09T10:49:40.952421+0000","last_became_peered":"2023-10-09T10:49:40.952421+0000","last_unstale":"2023-10-09T10:51:12.355068+0000","last_undegraded":"2023-10-09T10:51:12.355068+0000","last_fullsized":"2023-10-09T10:51:12.355068+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:08:57.147258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":574,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.366335+0000","last_change":"2023-10-09T10:49:58.797203+0000","last_active":"2023-10-09T10:51:11.366335+0000","last_peered":"2023-10-09T10:51:11.366335+0000","last_clean":"2023-10-09T10:51:11.366335+0000","last_became_active":"2023-10-09T10:49:58.796656+0000","last_became_peered":"2023-10-09T10:49:58.796656+0000","last_unstale":"2023-10-09T10:51:11.366335+0000","last_undegraded":"2023-10-09T10:51:11.366335+0000","last_fullsized":"2023-10-09T10:51:11.366335+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":155,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.358453+0000","last_change":"2023-10-09T10:49:40.952105+0000","last_active":"2023-10-09T10:51:12.358453+0000","last_peered":"2023-10-09T10:51:12.358453+0000","last_clean":"2023-10-09T10:51:12.358453+0000","last_became_active":"2023-10-09T10:49:40.950553+0000","last_became_peered":"2023-10-09T10:49:40.950553+0000","last_unstale":"2023-10-09T10:51:12.358453+0000","last_undegraded":"2023-10-09T10:51:12.358453+0000","last_fullsized":"2023-10-09T10:51:12.358453+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:51:40.501639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":114,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.353723+0000","last_change":"2023-10-09T10:49:40.949501+0000","last_active":"2023-10-09T10:51:12.353723+0000","last_peered":"2023-10-09T10:51:12.353723+0000","last_clean":"2023-10-09T10:51:12.353723+0000","last_became_active":"2023-10-09T10:49:40.949158+0000","last_became_peered":"2023-10-09T10:49:40.949158+0000","last_unstale":"2023-10-09T10:51:12.353723+0000","last_undegraded":"2023-10-09T10:51:12.353723+0000","last_fullsized":"2023-10-09T10:51:12.353723+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:07.598560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":610,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.358183+0000","last_change":"2023-10-09T10:49:58.776174+0000","last_active":"2023-10-09T10:51:12.358183+0000","last_peered":"2023-10-09T10:51:12.358183+0000","last_clean":"2023-10-09T10:51:12.358183+0000","last_became_active":"2023-10-09T10:49:58.775795+0000","last_became_peered":"2023-10-09T10:49:58.775795+0000","last_unstale":"2023-10-09T10:51:12.358183+0000","last_undegraded":"2023-10-09T10:51:12.358183+0000","last_fullsized":"2023-10-09T10:51:12.358183+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:47.161519+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":108,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:13.060731+0000","last_change":"2023-10-09T10:51:12.366910+0000","last_active":"2023-10-09T10:51:13.060731+0000","last_peered":"2023-10-09T10:51:13.060731+0000","last_clean":"2023-10-09T10:51:13.060731+0000","last_became_active":"2023-10-09T10:51:12.366565+0000","last_became_peered":"2023-10-09T10:51:12.366565+0000","last_unstale":"2023-10-09T10:51:13.060731+0000","last_undegraded":"2023-10-09T10:51:13.060731+0000","last_fullsized":"2023-10-09T10:51:13.060731+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:53.464251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":222,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.373015+0000","last_change":"2023-10-09T10:49:55.814525+0000","last_active":"2023-10-09T10:51:11.373015+0000","last_peered":"2023-10-09T10:51:11.373015+0000","last_clean":"2023-10-09T10:51:11.373015+0000","last_became_active":"2023-10-09T10:49:55.814228+0000","last_became_peered":"2023-10-09T10:49:55.814228+0000","last_unstale":"2023-10-09T10:51:11.373015+0000","last_undegraded":"2023-10-09T10:51:11.373015+0000","last_fullsized":"2023-10-09T10:51:11.373015+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:19.868347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"235'6817","reported_seq":13215,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T10:51:16.249055+0000","last_change":"2023-10-09T10:51:04.839204+0000","last_active":"2023-10-09T10:51:16.249055+0000","last_peered":"2023-10-09T10:51:16.249055+0000","last_clean":"2023-10-09T10:51:16.249055+0000","last_became_active":"2023-10-09T10:51:04.806380+0000","last_became_peered":"2023-10-09T10:51:04.806380+0000","last_unstale":"2023-10-09T10:51:16.249055+0000","last_undegraded":"2023-10-09T10:51:16.249055+0000","last_fullsized":"2023-10-09T10:51:16.249055+0000","mapping_epoch":230,"log_start":"204'2401","ondisk_log_start":"204'2401","created":196,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4416,"log_dups_size":0,"ondisk_log_size":4416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:52:58.428666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302186496,"num_objects":4611,"num_object_clones":0,"num_object_copies":13833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6817,"num_write_kb":365696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3165,"num_bytes_recovered":207421440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":497,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:13.068530+0000","last_change":"2023-10-09T10:51:12.365858+0000","last_active":"2023-10-09T10:51:13.068530+0000","last_peered":"2023-10-09T10:51:13.068530+0000","last_clean":"2023-10-09T10:51:13.068530+0000","last_became_active":"2023-10-09T10:51:12.365123+0000","last_became_peered":"2023-10-09T10:51:12.365123+0000","last_unstale":"2023-10-09T10:51:13.068530+0000","last_undegraded":"2023-10-09T10:51:13.068530+0000","last_fullsized":"2023-10-09T10:51:13.068530+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":74,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.360603+0000","last_change":"2023-10-09T10:49:55.846025+0000","last_active":"2023-10-09T10:51:12.360603+0000","last_peered":"2023-10-09T10:51:12.360603+0000","last_clean":"2023-10-09T10:51:12.360603+0000","last_became_active":"2023-10-09T10:49:55.821145+0000","last_became_peered":"2023-10-09T10:49:55.821145+0000","last_unstale":"2023-10-09T10:51:12.360603+0000","last_undegraded":"2023-10-09T10:51:12.360603+0000","last_fullsized":"2023-10-09T10:51:12.360603+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:50.726370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":192,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.353976+0000","last_change":"2023-10-09T10:49:40.955088+0000","last_active":"2023-10-09T10:51:12.353976+0000","last_peered":"2023-10-09T10:51:12.353976+0000","last_clean":"2023-10-09T10:51:12.353976+0000","last_became_active":"2023-10-09T10:49:40.954716+0000","last_became_peered":"2023-10-09T10:49:40.954716+0000","last_unstale":"2023-10-09T10:51:12.353976+0000","last_undegraded":"2023-10-09T10:51:12.353976+0000","last_fullsized":"2023-10-09T10:51:12.353976+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:39.090107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"234'6659","reported_seq":6714,"reported_epoch":234,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:51:14.759456+0000","last_change":"2023-10-09T10:49:57.045491+0000","last_active":"2023-10-09T10:51:14.759456+0000","last_peered":"2023-10-09T10:51:14.759456+0000","last_clean":"2023-10-09T10:49:54.734240+0000","last_became_active":"2023-10-09T10:49:57.036090+0000","last_became_peered":"2023-10-09T10:49:57.036090+0000","last_unstale":"2023-10-09T10:51:14.759456+0000","last_undegraded":"2023-10-09T10:49:56.833807+0000","last_fullsized":"2023-10-09T10:49:56.805766+0000","mapping_epoch":209,"log_start":"206'2490","ondisk_log_start":"206'2490","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4169,"log_dups_size":0,"ondisk_log_size":4169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:14.052065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310181888,"num_objects":4733,"num_object_clones":0,"num_object_copies":14199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3125,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6659,"num_write_kb":364544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,5","objects":2162},{"shards":"3,5","objects":1608}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":579,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.364459+0000","last_change":"2023-10-09T10:51:01.296071+0000","last_active":"2023-10-09T10:51:11.364459+0000","last_peered":"2023-10-09T10:51:11.364459+0000","last_clean":"2023-10-09T10:51:11.364459+0000","last_became_active":"2023-10-09T10:51:01.295474+0000","last_became_peered":"2023-10-09T10:51:01.295474+0000","last_unstale":"2023-10-09T10:51:11.364459+0000","last_undegraded":"2023-10-09T10:51:11.364459+0000","last_fullsized":"2023-10-09T10:51:11.364459+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:41.637984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:41.637984+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001804833,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":38,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.353152+0000","last_change":"2023-10-09T10:50:43.385172+0000","last_active":"2023-10-09T10:51:12.353152+0000","last_peered":"2023-10-09T10:51:12.353152+0000","last_clean":"2023-10-09T10:51:12.353152+0000","last_became_active":"2023-10-09T10:50:43.384888+0000","last_became_peered":"2023-10-09T10:50:43.384888+0000","last_unstale":"2023-10-09T10:51:12.353152+0000","last_undegraded":"2023-10-09T10:51:12.353152+0000","last_fullsized":"2023-10-09T10:51:12.353152+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:29.784932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":233,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.364439+0000","last_change":"2023-10-09T10:50:43.384962+0000","last_active":"2023-10-09T10:51:11.364439+0000","last_peered":"2023-10-09T10:51:11.364439+0000","last_clean":"2023-10-09T10:51:11.364439+0000","last_became_active":"2023-10-09T10:50:43.384645+0000","last_became_peered":"2023-10-09T10:50:43.384645+0000","last_unstale":"2023-10-09T10:51:11.364439+0000","last_undegraded":"2023-10-09T10:51:11.364439+0000","last_fullsized":"2023-10-09T10:51:11.364439+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:45.825019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.9","version":"235'6955","reported_seq":7010,"reported_epoch":235,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:51:18.006608+0000","last_change":"2023-10-09T10:49:56.964132+0000","last_active":"2023-10-09T10:51:18.006608+0000","last_peered":"2023-10-09T10:51:18.006608+0000","last_clean":"2023-10-09T10:49:54.793473+0000","last_became_active":"2023-10-09T10:49:56.956688+0000","last_became_peered":"2023-10-09T10:49:56.956688+0000","last_unstale":"2023-10-09T10:51:18.006608+0000","last_undegraded":"2023-10-09T10:49:56.831390+0000","last_fullsized":"2023-10-09T10:49:56.801929+0000","mapping_epoch":209,"log_start":"206'2632","ondisk_log_start":"206'2632","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4323,"log_dups_size":0,"ondisk_log_size":4323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:06:54.970299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306249728,"num_objects":4673,"num_object_clones":0,"num_object_copies":14019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3263,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6955,"num_write_kb":372096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2122},{"shards":"0,4","objects":1410}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":575,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.363985+0000","last_change":"2023-10-09T10:51:12.362883+0000","last_active":"2023-10-09T10:51:12.363985+0000","last_peered":"2023-10-09T10:51:12.363985+0000","last_clean":"2023-10-09T10:51:12.363985+0000","last_became_active":"2023-10-09T10:51:12.362553+0000","last_became_peered":"2023-10-09T10:51:12.362553+0000","last_unstale":"2023-10-09T10:51:12.363985+0000","last_undegraded":"2023-10-09T10:51:12.363985+0000","last_fullsized":"2023-10-09T10:51:12.363985+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":170,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.354994+0000","last_change":"2023-10-09T10:49:40.958240+0000","last_active":"2023-10-09T10:51:12.354994+0000","last_peered":"2023-10-09T10:51:12.354994+0000","last_clean":"2023-10-09T10:51:12.354994+0000","last_became_active":"2023-10-09T10:49:40.952355+0000","last_became_peered":"2023-10-09T10:49:40.952355+0000","last_unstale":"2023-10-09T10:51:12.354994+0000","last_undegraded":"2023-10-09T10:51:12.354994+0000","last_fullsized":"2023-10-09T10:51:12.354994+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:41:30.412691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":176,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.363748+0000","last_change":"2023-10-09T10:51:12.363348+0000","last_active":"2023-10-09T10:51:12.363748+0000","last_peered":"2023-10-09T10:51:12.363748+0000","last_clean":"2023-10-09T10:51:12.363748+0000","last_became_active":"2023-10-09T10:51:12.362712+0000","last_became_peered":"2023-10-09T10:51:12.362712+0000","last_unstale":"2023-10-09T10:51:12.363748+0000","last_undegraded":"2023-10-09T10:51:12.363748+0000","last_fullsized":"2023-10-09T10:51:12.363748+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:13.272752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.8","version":"235'6835","reported_seq":11718,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T10:51:16.098831+0000","last_change":"2023-10-09T10:50:27.531707+0000","last_active":"2023-10-09T10:51:16.098831+0000","last_peered":"2023-10-09T10:51:16.098831+0000","last_clean":"2023-10-09T10:51:16.098831+0000","last_became_active":"2023-10-09T10:50:27.518955+0000","last_became_peered":"2023-10-09T10:50:27.518955+0000","last_unstale":"2023-10-09T10:51:16.098831+0000","last_undegraded":"2023-10-09T10:51:16.098831+0000","last_fullsized":"2023-10-09T10:51:16.098831+0000","mapping_epoch":219,"log_start":"201'2279","ondisk_log_start":"201'2279","created":196,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4556,"log_dups_size":0,"ondisk_log_size":4556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:34.331554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307167232,"num_objects":4687,"num_object_clones":0,"num_object_copies":14061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6835,"num_write_kb":368704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2408,"num_bytes_recovered":157810688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":630,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.361568+0000","last_change":"2023-10-09T10:51:01.294632+0000","last_active":"2023-10-09T10:51:12.361568+0000","last_peered":"2023-10-09T10:51:12.361568+0000","last_clean":"2023-10-09T10:51:12.361568+0000","last_became_active":"2023-10-09T10:51:01.294424+0000","last_became_peered":"2023-10-09T10:51:01.294424+0000","last_unstale":"2023-10-09T10:51:12.361568+0000","last_undegraded":"2023-10-09T10:51:12.361568+0000","last_fullsized":"2023-10-09T10:51:12.361568+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":200,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.354743+0000","last_change":"2023-10-09T10:43:35.487326+0000","last_active":"2023-10-09T10:51:12.354743+0000","last_peered":"2023-10-09T10:51:12.354743+0000","last_clean":"2023-10-09T10:51:12.354743+0000","last_became_active":"2023-10-09T10:43:35.486986+0000","last_became_peered":"2023-10-09T10:43:35.486986+0000","last_unstale":"2023-10-09T10:51:12.354743+0000","last_undegraded":"2023-10-09T10:51:12.354743+0000","last_fullsized":"2023-10-09T10:51:12.354743+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:50:35.429713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":114,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.353854+0000","last_change":"2023-10-09T10:49:55.830400+0000","last_active":"2023-10-09T10:51:12.353854+0000","last_peered":"2023-10-09T10:51:12.353854+0000","last_clean":"2023-10-09T10:51:12.353854+0000","last_became_active":"2023-10-09T10:49:55.829982+0000","last_became_peered":"2023-10-09T10:49:55.829982+0000","last_unstale":"2023-10-09T10:51:12.353854+0000","last_undegraded":"2023-10-09T10:51:12.353854+0000","last_fullsized":"2023-10-09T10:51:12.353854+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:51.352143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"235'6804","reported_seq":6860,"reported_epoch":235,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:51:18.008718+0000","last_change":"2023-10-09T10:49:42.062469+0000","last_active":"2023-10-09T10:51:18.008718+0000","last_peered":"2023-10-09T10:51:18.008718+0000","last_clean":"2023-10-09T10:49:39.916756+0000","last_became_active":"2023-10-09T10:49:42.056867+0000","last_became_peered":"2023-10-09T10:49:42.056867+0000","last_unstale":"2023-10-09T10:51:18.008718+0000","last_undegraded":"2023-10-09T10:49:41.970254+0000","last_fullsized":"2023-10-09T10:49:41.933564+0000","mapping_epoch":203,"log_start":"205'2464","ondisk_log_start":"205'2464","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4340,"log_dups_size":0,"ondisk_log_size":4340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:59:10.773024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295305216,"num_objects":4506,"num_object_clones":0,"num_object_copies":13518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2419,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6804,"num_write_kb":361920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":2087},{"shards":"0,3,7","objects":1270}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":594,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.364369+0000","last_change":"2023-10-09T10:51:01.305574+0000","last_active":"2023-10-09T10:51:11.364369+0000","last_peered":"2023-10-09T10:51:11.364369+0000","last_clean":"2023-10-09T10:51:11.364369+0000","last_became_active":"2023-10-09T10:51:01.305381+0000","last_became_peered":"2023-10-09T10:51:01.305381+0000","last_unstale":"2023-10-09T10:51:11.364369+0000","last_undegraded":"2023-10-09T10:51:11.364369+0000","last_fullsized":"2023-10-09T10:51:11.364369+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":228,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.366008+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:51:11.366008+0000","last_peered":"2023-10-09T10:51:11.366008+0000","last_clean":"2023-10-09T10:51:11.366008+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:51:11.366008+0000","last_undegraded":"2023-10-09T10:51:11.366008+0000","last_fullsized":"2023-10-09T10:51:11.366008+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":73,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.364796+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T10:51:11.364796+0000","last_peered":"2023-10-09T10:51:11.364796+0000","last_clean":"2023-10-09T10:51:11.364796+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T10:51:11.364796+0000","last_undegraded":"2023-10-09T10:51:11.364796+0000","last_fullsized":"2023-10-09T10:51:11.364796+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"235'6671","reported_seq":6718,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T10:51:16.302285+0000","last_change":"2023-10-09T10:49:16.843562+0000","last_active":"2023-10-09T10:51:16.302285+0000","last_peered":"2023-10-09T10:51:16.302285+0000","last_clean":"2023-10-09T10:51:16.302285+0000","last_became_active":"2023-10-09T10:49:16.843246+0000","last_became_peered":"2023-10-09T10:49:16.843246+0000","last_unstale":"2023-10-09T10:51:16.302285+0000","last_undegraded":"2023-10-09T10:51:16.302285+0000","last_fullsized":"2023-10-09T10:51:16.302285+0000","mapping_epoch":196,"log_start":"201'2253","ondisk_log_start":"201'2253","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4418,"log_dups_size":0,"ondisk_log_size":4418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:47.911382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299171840,"num_objects":4565,"num_object_clones":0,"num_object_copies":13695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6671,"num_write_kb":359552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":570,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.354232+0000","last_change":"2023-10-09T10:51:01.308143+0000","last_active":"2023-10-09T10:51:12.354232+0000","last_peered":"2023-10-09T10:51:12.354232+0000","last_clean":"2023-10-09T10:51:12.354232+0000","last_became_active":"2023-10-09T10:51:01.307860+0000","last_became_peered":"2023-10-09T10:51:01.307860+0000","last_unstale":"2023-10-09T10:51:12.354232+0000","last_undegraded":"2023-10-09T10:51:12.354232+0000","last_fullsized":"2023-10-09T10:51:12.354232+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":211848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":184,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.359784+0000","last_change":"2023-10-09T10:49:55.814556+0000","last_active":"2023-10-09T10:51:12.359784+0000","last_peered":"2023-10-09T10:51:12.359784+0000","last_clean":"2023-10-09T10:51:12.359784+0000","last_became_active":"2023-10-09T10:49:55.814193+0000","last_became_peered":"2023-10-09T10:49:55.814193+0000","last_unstale":"2023-10-09T10:51:12.359784+0000","last_undegraded":"2023-10-09T10:51:12.359784+0000","last_fullsized":"2023-10-09T10:51:12.359784+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:13:19.537756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":218,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:13.068548+0000","last_change":"2023-10-09T10:51:12.366102+0000","last_active":"2023-10-09T10:51:13.068548+0000","last_peered":"2023-10-09T10:51:13.068548+0000","last_clean":"2023-10-09T10:51:13.068548+0000","last_became_active":"2023-10-09T10:51:12.365345+0000","last_became_peered":"2023-10-09T10:51:12.365345+0000","last_unstale":"2023-10-09T10:51:13.068548+0000","last_undegraded":"2023-10-09T10:51:13.068548+0000","last_fullsized":"2023-10-09T10:51:13.068548+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:32:03.306059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.d","version":"235'6715","reported_seq":9030,"reported_epoch":235,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T10:51:18.064520+0000","last_change":"2023-10-09T10:51:12.405688+0000","last_active":"2023-10-09T10:51:18.064520+0000","last_peered":"2023-10-09T10:51:18.064520+0000","last_clean":"2023-10-09T10:49:54.776483+0000","last_became_active":"2023-10-09T10:51:12.367871+0000","last_became_peered":"2023-10-09T10:51:12.367871+0000","last_unstale":"2023-10-09T10:51:18.064520+0000","last_undegraded":"2023-10-09T10:51:12.361366+0000","last_fullsized":"2023-10-09T10:51:18.064520+0000","mapping_epoch":233,"log_start":"206'2652","ondisk_log_start":"206'2652","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4063,"log_dups_size":0,"ondisk_log_size":4063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:56.598741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292683800,"num_objects":4467,"num_object_clones":0,"num_object_copies":13401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1994,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4467,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6715,"num_write_kb":357761,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1124,"num_bytes_recovered":73662464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":1994},{"shards":"0,3,6","objects":2473}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":590,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.353614+0000","last_change":"2023-10-09T10:51:01.307425+0000","last_active":"2023-10-09T10:51:12.353614+0000","last_peered":"2023-10-09T10:51:12.353614+0000","last_clean":"2023-10-09T10:51:12.353614+0000","last_became_active":"2023-10-09T10:51:01.307151+0000","last_became_peered":"2023-10-09T10:51:01.307151+0000","last_unstale":"2023-10-09T10:51:12.353614+0000","last_undegraded":"2023-10-09T10:51:12.353614+0000","last_fullsized":"2023-10-09T10:51:12.353614+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.892515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.892515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":230,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.358319+0000","last_change":"2023-10-09T10:49:55.812746+0000","last_active":"2023-10-09T10:51:12.358319+0000","last_peered":"2023-10-09T10:51:12.358319+0000","last_clean":"2023-10-09T10:51:12.358319+0000","last_became_active":"2023-10-09T10:49:55.812438+0000","last_became_peered":"2023-10-09T10:49:55.812438+0000","last_unstale":"2023-10-09T10:51:12.358319+0000","last_undegraded":"2023-10-09T10:51:12.358319+0000","last_fullsized":"2023-10-09T10:51:12.358319+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:44:14.045846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":190,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.362006+0000","last_change":"2023-10-09T10:51:12.360772+0000","last_active":"2023-10-09T10:51:12.362006+0000","last_peered":"2023-10-09T10:51:12.362006+0000","last_clean":"2023-10-09T10:51:12.362006+0000","last_became_active":"2023-10-09T10:51:12.360507+0000","last_became_peered":"2023-10-09T10:51:12.360507+0000","last_unstale":"2023-10-09T10:51:12.362006+0000","last_undegraded":"2023-10-09T10:51:12.362006+0000","last_fullsized":"2023-10-09T10:51:12.362006+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:38:52.488339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.c","version":"234'6567","reported_seq":11329,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:14.792322+0000","last_change":"2023-10-09T10:50:14.783907+0000","last_active":"2023-10-09T10:51:14.792322+0000","last_peered":"2023-10-09T10:51:14.792322+0000","last_clean":"2023-10-09T10:51:14.792322+0000","last_became_active":"2023-10-09T10:50:14.780725+0000","last_became_peered":"2023-10-09T10:50:14.780725+0000","last_unstale":"2023-10-09T10:51:14.792322+0000","last_undegraded":"2023-10-09T10:51:14.792322+0000","last_fullsized":"2023-10-09T10:51:14.792322+0000","mapping_epoch":215,"log_start":"201'2311","ondisk_log_start":"201'2311","created":196,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4256,"log_dups_size":0,"ondisk_log_size":4256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:11:40.081379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308477952,"num_objects":4707,"num_object_clones":0,"num_object_copies":14121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6567,"num_write_kb":360768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2347,"num_bytes_recovered":153812992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":581,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.358089+0000","last_change":"2023-10-09T10:51:01.303234+0000","last_active":"2023-10-09T10:51:12.358089+0000","last_peered":"2023-10-09T10:51:12.358089+0000","last_clean":"2023-10-09T10:51:12.358089+0000","last_became_active":"2023-10-09T10:51:01.302898+0000","last_became_peered":"2023-10-09T10:51:01.302898+0000","last_unstale":"2023-10-09T10:51:12.358089+0000","last_undegraded":"2023-10-09T10:51:12.358089+0000","last_fullsized":"2023-10-09T10:51:12.358089+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":223,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.368055+0000","last_change":"2023-10-09T10:49:55.819881+0000","last_active":"2023-10-09T10:51:11.368055+0000","last_peered":"2023-10-09T10:51:11.368055+0000","last_clean":"2023-10-09T10:51:11.368055+0000","last_became_active":"2023-10-09T10:49:55.818533+0000","last_became_peered":"2023-10-09T10:49:55.818533+0000","last_unstale":"2023-10-09T10:51:11.368055+0000","last_undegraded":"2023-10-09T10:51:11.368055+0000","last_fullsized":"2023-10-09T10:51:11.368055+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:20.512152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":75,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.372841+0000","last_change":"2023-10-09T10:49:55.815880+0000","last_active":"2023-10-09T10:51:11.372841+0000","last_peered":"2023-10-09T10:51:11.372841+0000","last_clean":"2023-10-09T10:51:11.372841+0000","last_became_active":"2023-10-09T10:49:55.815594+0000","last_became_peered":"2023-10-09T10:49:55.815594+0000","last_unstale":"2023-10-09T10:51:11.372841+0000","last_undegraded":"2023-10-09T10:51:11.372841+0000","last_fullsized":"2023-10-09T10:51:11.372841+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:39.210709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"234'6500","reported_seq":6552,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:14.734399+0000","last_change":"2023-10-09T10:50:43.392944+0000","last_active":"2023-10-09T10:51:14.734399+0000","last_peered":"2023-10-09T10:51:14.734399+0000","last_clean":"2023-10-09T10:51:14.734399+0000","last_became_active":"2023-10-09T10:50:43.384716+0000","last_became_peered":"2023-10-09T10:50:43.384716+0000","last_unstale":"2023-10-09T10:51:14.734399+0000","last_undegraded":"2023-10-09T10:51:14.734399+0000","last_fullsized":"2023-10-09T10:51:14.734399+0000","mapping_epoch":223,"log_start":"201'2273","ondisk_log_start":"201'2273","created":196,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4227,"log_dups_size":0,"ondisk_log_size":4227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:19.728024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308412416,"num_objects":4706,"num_object_clones":0,"num_object_copies":14118,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4706,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6500,"num_write_kb":358592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":599,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.364831+0000","last_change":"2023-10-09T10:51:12.364831+0000","last_active":"2023-10-09T10:51:12.364831+0000","last_peered":"2023-10-09T10:51:12.364831+0000","last_clean":"2023-10-09T10:51:12.364831+0000","last_became_active":"2023-10-09T10:51:12.364197+0000","last_became_peered":"2023-10-09T10:51:12.364197+0000","last_unstale":"2023-10-09T10:51:12.364831+0000","last_undegraded":"2023-10-09T10:51:12.364831+0000","last_fullsized":"2023-10-09T10:51:12.364831+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.687290+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.687290+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012763172999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":237,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.355366+0000","last_change":"2023-10-09T10:49:55.825069+0000","last_active":"2023-10-09T10:51:12.355366+0000","last_peered":"2023-10-09T10:51:12.355366+0000","last_clean":"2023-10-09T10:51:12.355366+0000","last_became_active":"2023-10-09T10:49:55.824755+0000","last_became_peered":"2023-10-09T10:49:55.824755+0000","last_unstale":"2023-10-09T10:51:12.355366+0000","last_undegraded":"2023-10-09T10:51:12.355366+0000","last_fullsized":"2023-10-09T10:51:12.355366+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:04:28.113626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":227,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.355212+0000","last_change":"2023-10-09T10:46:32.242336+0000","last_active":"2023-10-09T10:51:12.355212+0000","last_peered":"2023-10-09T10:51:12.355212+0000","last_clean":"2023-10-09T10:51:12.355212+0000","last_became_active":"2023-10-09T10:46:32.242008+0000","last_became_peered":"2023-10-09T10:46:32.242008+0000","last_unstale":"2023-10-09T10:51:12.355212+0000","last_undegraded":"2023-10-09T10:51:12.355212+0000","last_fullsized":"2023-10-09T10:51:12.355212+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:21.796289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":52,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.357948+0000","last_change":"2023-10-09T10:49:55.828773+0000","last_active":"2023-10-09T10:51:12.357948+0000","last_peered":"2023-10-09T10:51:12.357948+0000","last_clean":"2023-10-09T10:51:12.357948+0000","last_became_active":"2023-10-09T10:49:55.828458+0000","last_became_peered":"2023-10-09T10:49:55.828458+0000","last_unstale":"2023-10-09T10:51:12.357948+0000","last_undegraded":"2023-10-09T10:51:12.357948+0000","last_fullsized":"2023-10-09T10:51:12.357948+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:30.196079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"235'6930","reported_seq":6988,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T10:51:18.035085+0000","last_change":"2023-10-09T10:51:12.374840+0000","last_active":"2023-10-09T10:51:18.035085+0000","last_peered":"2023-10-09T10:51:18.035085+0000","last_clean":"2023-10-09T10:51:18.035085+0000","last_became_active":"2023-10-09T10:51:12.366993+0000","last_became_peered":"2023-10-09T10:51:12.366993+0000","last_unstale":"2023-10-09T10:51:18.035085+0000","last_undegraded":"2023-10-09T10:51:18.035085+0000","last_fullsized":"2023-10-09T10:51:18.035085+0000","mapping_epoch":233,"log_start":"206'2863","ondisk_log_start":"206'2863","created":196,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4067,"log_dups_size":0,"ondisk_log_size":4067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:47.959952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304742400,"num_objects":4650,"num_object_clones":0,"num_object_copies":13950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6930,"num_write_kb":370560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":589,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.354237+0000","last_change":"2023-10-09T10:49:38.736950+0000","last_active":"2023-10-09T10:51:12.354237+0000","last_peered":"2023-10-09T10:51:12.354237+0000","last_clean":"2023-10-09T10:51:12.354237+0000","last_became_active":"2023-10-09T10:49:17.837821+0000","last_became_peered":"2023-10-09T10:49:17.837821+0000","last_unstale":"2023-10-09T10:51:12.354237+0000","last_undegraded":"2023-10-09T10:51:12.354237+0000","last_fullsized":"2023-10-09T10:51:12.354237+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.736834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.736834+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:17:16.734495+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":214,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.357874+0000","last_change":"2023-10-09T10:49:55.847216+0000","last_active":"2023-10-09T10:51:12.357874+0000","last_peered":"2023-10-09T10:51:12.357874+0000","last_clean":"2023-10-09T10:51:12.357874+0000","last_became_active":"2023-10-09T10:49:55.847002+0000","last_became_peered":"2023-10-09T10:49:55.847002+0000","last_unstale":"2023-10-09T10:51:12.357874+0000","last_undegraded":"2023-10-09T10:51:12.357874+0000","last_fullsized":"2023-10-09T10:51:12.357874+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:41.845106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":142,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.353528+0000","last_change":"2023-10-09T10:50:43.388615+0000","last_active":"2023-10-09T10:51:12.353528+0000","last_peered":"2023-10-09T10:51:12.353528+0000","last_clean":"2023-10-09T10:51:12.353528+0000","last_became_active":"2023-10-09T10:50:43.388176+0000","last_became_peered":"2023-10-09T10:50:43.388176+0000","last_unstale":"2023-10-09T10:51:12.353528+0000","last_undegraded":"2023-10-09T10:51:12.353528+0000","last_fullsized":"2023-10-09T10:51:12.353528+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:41:17.550111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":157,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.364936+0000","last_change":"2023-10-09T10:49:55.816187+0000","last_active":"2023-10-09T10:51:11.364936+0000","last_peered":"2023-10-09T10:51:11.364936+0000","last_clean":"2023-10-09T10:51:11.364936+0000","last_became_active":"2023-10-09T10:49:55.815345+0000","last_became_peered":"2023-10-09T10:49:55.815345+0000","last_unstale":"2023-10-09T10:51:11.364936+0000","last_undegraded":"2023-10-09T10:51:11.364936+0000","last_fullsized":"2023-10-09T10:51:11.364936+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:43.036104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.4","version":"235'6899","reported_seq":6959,"reported_epoch":235,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:51:18.058377+0000","last_change":"2023-10-09T10:49:57.003555+0000","last_active":"2023-10-09T10:51:18.058377+0000","last_peered":"2023-10-09T10:51:18.058377+0000","last_clean":"2023-10-09T10:49:54.777504+0000","last_became_active":"2023-10-09T10:49:56.956699+0000","last_became_peered":"2023-10-09T10:49:56.956699+0000","last_unstale":"2023-10-09T10:51:18.058377+0000","last_undegraded":"2023-10-09T10:49:56.938617+0000","last_fullsized":"2023-10-09T10:49:56.806441+0000","mapping_epoch":209,"log_start":"206'2534","ondisk_log_start":"206'2534","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4365,"log_dups_size":0,"ondisk_log_size":4365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:47:51.048643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300351488,"num_objects":4583,"num_object_clones":0,"num_object_copies":13749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3084,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6899,"num_write_kb":367424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":1925},{"shards":"0,4","objects":1499}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":605,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.354769+0000","last_change":"2023-10-09T10:49:43.780951+0000","last_active":"2023-10-09T10:51:12.354769+0000","last_peered":"2023-10-09T10:51:12.354769+0000","last_clean":"2023-10-09T10:51:12.354769+0000","last_became_active":"2023-10-09T10:49:43.780564+0000","last_became_peered":"2023-10-09T10:49:43.780564+0000","last_unstale":"2023-10-09T10:51:12.354769+0000","last_undegraded":"2023-10-09T10:51:12.354769+0000","last_fullsized":"2023-10-09T10:51:12.354769+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":242,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.364721+0000","last_change":"2023-10-09T10:49:55.814470+0000","last_active":"2023-10-09T10:51:11.364721+0000","last_peered":"2023-10-09T10:51:11.364721+0000","last_clean":"2023-10-09T10:51:11.364721+0000","last_became_active":"2023-10-09T10:49:55.814154+0000","last_became_peered":"2023-10-09T10:49:55.814154+0000","last_unstale":"2023-10-09T10:51:11.364721+0000","last_undegraded":"2023-10-09T10:51:11.364721+0000","last_fullsized":"2023-10-09T10:51:11.364721+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:03:57.326294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":136,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.368557+0000","last_change":"2023-10-09T10:51:12.365160+0000","last_active":"2023-10-09T10:51:12.368557+0000","last_peered":"2023-10-09T10:51:12.368557+0000","last_clean":"2023-10-09T10:51:12.368557+0000","last_became_active":"2023-10-09T10:51:12.364780+0000","last_became_peered":"2023-10-09T10:51:12.364780+0000","last_unstale":"2023-10-09T10:51:12.368557+0000","last_undegraded":"2023-10-09T10:51:12.368557+0000","last_fullsized":"2023-10-09T10:51:12.368557+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:43.765287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":162,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.356762+0000","last_change":"2023-10-09T10:49:55.822332+0000","last_active":"2023-10-09T10:51:12.356762+0000","last_peered":"2023-10-09T10:51:12.356762+0000","last_clean":"2023-10-09T10:51:12.356762+0000","last_became_active":"2023-10-09T10:49:55.822068+0000","last_became_peered":"2023-10-09T10:49:55.822068+0000","last_unstale":"2023-10-09T10:51:12.356762+0000","last_undegraded":"2023-10-09T10:51:12.356762+0000","last_fullsized":"2023-10-09T10:51:12.356762+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:41:57.905151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"235'6735","reported_seq":8794,"reported_epoch":235,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:51:18.032473+0000","last_change":"2023-10-09T10:49:55.838303+0000","last_active":"2023-10-09T10:51:18.032473+0000","last_peered":"2023-10-09T10:51:18.032473+0000","last_clean":"2023-10-09T10:49:39.898166+0000","last_became_active":"2023-10-09T10:49:42.040841+0000","last_became_peered":"2023-10-09T10:49:42.040841+0000","last_unstale":"2023-10-09T10:51:18.032473+0000","last_undegraded":"2023-10-09T10:49:55.835709+0000","last_fullsized":"2023-10-09T10:49:41.933738+0000","mapping_epoch":203,"log_start":"206'2420","ondisk_log_start":"206'2420","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4315,"log_dups_size":0,"ondisk_log_size":4315,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:02.626804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300351488,"num_objects":4583,"num_object_clones":0,"num_object_copies":13749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1311,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6735,"num_write_kb":362176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1000,"num_bytes_recovered":65536000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":3272},{"shards":"0,3,7","objects":1235}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":224,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.355093+0000","last_change":"2023-10-09T10:49:40.951536+0000","last_active":"2023-10-09T10:51:12.355093+0000","last_peered":"2023-10-09T10:51:12.355093+0000","last_clean":"2023-10-09T10:51:12.355093+0000","last_became_active":"2023-10-09T10:49:40.951152+0000","last_became_peered":"2023-10-09T10:49:40.951152+0000","last_unstale":"2023-10-09T10:51:12.355093+0000","last_undegraded":"2023-10-09T10:51:12.355093+0000","last_fullsized":"2023-10-09T10:51:12.355093+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:29.134747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":617,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.352640+0000","last_change":"2023-10-09T10:49:58.796806+0000","last_active":"2023-10-09T10:51:12.352640+0000","last_peered":"2023-10-09T10:51:12.352640+0000","last_clean":"2023-10-09T10:51:12.352640+0000","last_became_active":"2023-10-09T10:49:58.796478+0000","last_became_peered":"2023-10-09T10:49:58.796478+0000","last_unstale":"2023-10-09T10:51:12.352640+0000","last_undegraded":"2023-10-09T10:51:12.352640+0000","last_fullsized":"2023-10-09T10:51:12.352640+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:44:35.791052+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":163,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.366870+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:51:11.366870+0000","last_peered":"2023-10-09T10:51:11.366870+0000","last_clean":"2023-10-09T10:51:11.366870+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:51:11.366870+0000","last_undegraded":"2023-10-09T10:51:11.366870+0000","last_fullsized":"2023-10-09T10:51:11.366870+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":137,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.364704+0000","last_change":"2023-10-09T10:51:12.363829+0000","last_active":"2023-10-09T10:51:12.364704+0000","last_peered":"2023-10-09T10:51:12.364704+0000","last_clean":"2023-10-09T10:51:12.364704+0000","last_became_active":"2023-10-09T10:51:12.363534+0000","last_became_peered":"2023-10-09T10:51:12.363534+0000","last_unstale":"2023-10-09T10:51:12.364704+0000","last_undegraded":"2023-10-09T10:51:12.364704+0000","last_fullsized":"2023-10-09T10:51:12.364704+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:59.472228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"235'6861","reported_seq":10378,"reported_epoch":235,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:51:18.082021+0000","last_change":"2023-10-09T10:49:57.053554+0000","last_active":"2023-10-09T10:51:18.082021+0000","last_peered":"2023-10-09T10:51:18.082021+0000","last_clean":"2023-10-09T10:49:39.922158+0000","last_became_active":"2023-10-09T10:49:57.045910+0000","last_became_peered":"2023-10-09T10:49:57.045910+0000","last_unstale":"2023-10-09T10:51:18.082021+0000","last_undegraded":"2023-10-09T10:49:56.942718+0000","last_fullsized":"2023-10-09T10:49:56.803232+0000","mapping_epoch":209,"log_start":"206'2501","ondisk_log_start":"206'2501","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4360,"log_dups_size":0,"ondisk_log_size":4360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:46.585592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299171840,"num_objects":4565,"num_object_clones":0,"num_object_copies":13695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3215,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6861,"num_write_kb":365632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1150,"num_bytes_recovered":75366400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,6","objects":793},{"shards":"0,1,6,7","objects":2},{"shards":"0,1,7","objects":1210},{"shards":"0,6","objects":2560}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":573,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.364519+0000","last_change":"2023-10-09T10:51:12.363631+0000","last_active":"2023-10-09T10:51:12.364519+0000","last_peered":"2023-10-09T10:51:12.364519+0000","last_clean":"2023-10-09T10:51:12.364519+0000","last_became_active":"2023-10-09T10:51:12.363416+0000","last_became_peered":"2023-10-09T10:51:12.363416+0000","last_unstale":"2023-10-09T10:51:12.364519+0000","last_undegraded":"2023-10-09T10:51:12.364519+0000","last_fullsized":"2023-10-09T10:51:12.364519+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":233,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.352725+0000","last_change":"2023-10-09T10:49:55.837610+0000","last_active":"2023-10-09T10:51:12.352725+0000","last_peered":"2023-10-09T10:51:12.352725+0000","last_clean":"2023-10-09T10:51:12.352725+0000","last_became_active":"2023-10-09T10:49:55.837314+0000","last_became_peered":"2023-10-09T10:49:55.837314+0000","last_unstale":"2023-10-09T10:51:12.352725+0000","last_undegraded":"2023-10-09T10:51:12.352725+0000","last_fullsized":"2023-10-09T10:51:12.352725+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:01.185522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":114,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.356159+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:51:12.356159+0000","last_peered":"2023-10-09T10:51:12.356159+0000","last_clean":"2023-10-09T10:51:12.356159+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:51:12.356159+0000","last_undegraded":"2023-10-09T10:51:12.356159+0000","last_fullsized":"2023-10-09T10:51:12.356159+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":132,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.353795+0000","last_change":"2023-10-09T10:49:55.838212+0000","last_active":"2023-10-09T10:51:12.353795+0000","last_peered":"2023-10-09T10:51:12.353795+0000","last_clean":"2023-10-09T10:51:12.353795+0000","last_became_active":"2023-10-09T10:49:55.837374+0000","last_became_peered":"2023-10-09T10:49:55.837374+0000","last_unstale":"2023-10-09T10:51:12.353795+0000","last_undegraded":"2023-10-09T10:51:12.353795+0000","last_fullsized":"2023-10-09T10:51:12.353795+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:54.825127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"235'6784","reported_seq":6844,"reported_epoch":235,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:51:16.299864+0000","last_change":"2023-10-09T10:49:56.979655+0000","last_active":"2023-10-09T10:51:16.299864+0000","last_peered":"2023-10-09T10:51:16.299864+0000","last_clean":"2023-10-09T10:49:54.774195+0000","last_became_active":"2023-10-09T10:49:56.969698+0000","last_became_peered":"2023-10-09T10:49:56.969698+0000","last_unstale":"2023-10-09T10:51:16.299864+0000","last_undegraded":"2023-10-09T10:49:56.911686+0000","last_fullsized":"2023-10-09T10:49:56.808103+0000","mapping_epoch":209,"log_start":"201'2370","ondisk_log_start":"201'2370","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4414,"log_dups_size":0,"ondisk_log_size":4414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:47.019837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302514176,"num_objects":4616,"num_object_clones":0,"num_object_copies":13848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3175,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6784,"num_write_kb":364800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":2091},{"shards":"3,5","objects":1441}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":593,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.368671+0000","last_change":"2023-10-09T10:51:12.367319+0000","last_active":"2023-10-09T10:51:12.368671+0000","last_peered":"2023-10-09T10:51:12.368671+0000","last_clean":"2023-10-09T10:51:12.368671+0000","last_became_active":"2023-10-09T10:51:12.366828+0000","last_became_peered":"2023-10-09T10:51:12.366828+0000","last_unstale":"2023-10-09T10:51:12.368671+0000","last_undegraded":"2023-10-09T10:51:12.368671+0000","last_fullsized":"2023-10-09T10:51:12.368671+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.475429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.475429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001591812,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":175,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.353790+0000","last_change":"2023-10-09T10:49:55.820384+0000","last_active":"2023-10-09T10:51:12.353790+0000","last_peered":"2023-10-09T10:51:12.353790+0000","last_clean":"2023-10-09T10:51:12.353790+0000","last_became_active":"2023-10-09T10:49:55.820044+0000","last_became_peered":"2023-10-09T10:49:55.820044+0000","last_unstale":"2023-10-09T10:51:12.353790+0000","last_undegraded":"2023-10-09T10:51:12.353790+0000","last_fullsized":"2023-10-09T10:51:12.353790+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:58.595200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":144,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.354833+0000","last_change":"2023-10-09T10:49:55.824348+0000","last_active":"2023-10-09T10:51:12.354833+0000","last_peered":"2023-10-09T10:51:12.354833+0000","last_clean":"2023-10-09T10:51:12.354833+0000","last_became_active":"2023-10-09T10:49:55.823990+0000","last_became_peered":"2023-10-09T10:49:55.823990+0000","last_unstale":"2023-10-09T10:51:12.354833+0000","last_undegraded":"2023-10-09T10:51:12.354833+0000","last_fullsized":"2023-10-09T10:51:12.354833+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:33.050849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":123,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.353691+0000","last_change":"2023-10-09T10:49:55.837070+0000","last_active":"2023-10-09T10:51:12.353691+0000","last_peered":"2023-10-09T10:51:12.353691+0000","last_clean":"2023-10-09T10:51:12.353691+0000","last_became_active":"2023-10-09T10:49:55.836747+0000","last_became_peered":"2023-10-09T10:49:55.836747+0000","last_unstale":"2023-10-09T10:51:12.353691+0000","last_undegraded":"2023-10-09T10:51:12.353691+0000","last_fullsized":"2023-10-09T10:51:12.353691+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:45.683756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"234'6639","reported_seq":6686,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:14.700738+0000","last_change":"2023-10-09T10:49:16.845103+0000","last_active":"2023-10-09T10:51:14.700738+0000","last_peered":"2023-10-09T10:51:14.700738+0000","last_clean":"2023-10-09T10:51:14.700738+0000","last_became_active":"2023-10-09T10:49:16.844812+0000","last_became_peered":"2023-10-09T10:49:16.844812+0000","last_unstale":"2023-10-09T10:51:14.700738+0000","last_undegraded":"2023-10-09T10:51:14.700738+0000","last_fullsized":"2023-10-09T10:51:14.700738+0000","mapping_epoch":196,"log_start":"205'2442","ondisk_log_start":"205'2442","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4197,"log_dups_size":0,"ondisk_log_size":4197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:17:27.590153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308871168,"num_objects":4713,"num_object_clones":0,"num_object_copies":14139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6639,"num_write_kb":363264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":600,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.360416+0000","last_change":"2023-10-09T10:51:01.315582+0000","last_active":"2023-10-09T10:51:12.360416+0000","last_peered":"2023-10-09T10:51:12.360416+0000","last_clean":"2023-10-09T10:51:12.360416+0000","last_became_active":"2023-10-09T10:51:01.314820+0000","last_became_peered":"2023-10-09T10:51:01.314820+0000","last_unstale":"2023-10-09T10:51:12.360416+0000","last_undegraded":"2023-10-09T10:51:12.360416+0000","last_fullsized":"2023-10-09T10:51:12.360416+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.829081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.829081+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001277842,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":241,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.372934+0000","last_change":"2023-10-09T10:49:40.959630+0000","last_active":"2023-10-09T10:51:11.372934+0000","last_peered":"2023-10-09T10:51:11.372934+0000","last_clean":"2023-10-09T10:51:11.372934+0000","last_became_active":"2023-10-09T10:49:40.945271+0000","last_became_peered":"2023-10-09T10:49:40.945271+0000","last_unstale":"2023-10-09T10:51:11.372934+0000","last_undegraded":"2023-10-09T10:51:11.372934+0000","last_fullsized":"2023-10-09T10:51:11.372934+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:37.426204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":47,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.361626+0000","last_change":"2023-10-09T10:51:12.360090+0000","last_active":"2023-10-09T10:51:12.361626+0000","last_peered":"2023-10-09T10:51:12.361626+0000","last_clean":"2023-10-09T10:51:12.361626+0000","last_became_active":"2023-10-09T10:51:12.359768+0000","last_became_peered":"2023-10-09T10:51:12.359768+0000","last_unstale":"2023-10-09T10:51:12.361626+0000","last_undegraded":"2023-10-09T10:51:12.361626+0000","last_fullsized":"2023-10-09T10:51:12.361626+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:43:03.756758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":235,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.368136+0000","last_change":"2023-10-09T10:49:55.820039+0000","last_active":"2023-10-09T10:51:11.368136+0000","last_peered":"2023-10-09T10:51:11.368136+0000","last_clean":"2023-10-09T10:51:11.368136+0000","last_became_active":"2023-10-09T10:49:55.819236+0000","last_became_peered":"2023-10-09T10:49:55.819236+0000","last_unstale":"2023-10-09T10:51:11.368136+0000","last_undegraded":"2023-10-09T10:51:11.368136+0000","last_fullsized":"2023-10-09T10:51:11.368136+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:47:19.590311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"234'6787","reported_seq":10117,"reported_epoch":234,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:51:14.776507+0000","last_change":"2023-10-09T10:51:02.785715+0000","last_active":"2023-10-09T10:51:14.776507+0000","last_peered":"2023-10-09T10:51:14.776507+0000","last_clean":"2023-10-09T10:49:39.930672+0000","last_became_active":"2023-10-09T10:49:57.038487+0000","last_became_peered":"2023-10-09T10:49:57.038487+0000","last_unstale":"2023-10-09T10:51:14.776507+0000","last_undegraded":"2023-10-09T10:49:56.923222+0000","last_fullsized":"2023-10-09T10:49:56.809367+0000","mapping_epoch":209,"log_start":"206'2588","ondisk_log_start":"206'2588","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4199,"log_dups_size":0,"ondisk_log_size":4199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:29:54.702472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311099392,"num_objects":4747,"num_object_clones":0,"num_object_copies":14241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":3511,"num_objects_unfound":0,"num_objects_dirty":4747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6787,"num_write_kb":369088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1089,"num_bytes_recovered":71368704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":1},{"shards":"1,3,6","objects":752},{"shards":"1,3,7","objects":1379},{"shards":"3,6","objects":2615}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":593,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.364549+0000","last_change":"2023-10-09T10:49:43.787701+0000","last_active":"2023-10-09T10:51:11.364549+0000","last_peered":"2023-10-09T10:51:11.364549+0000","last_clean":"2023-10-09T10:51:11.364549+0000","last_became_active":"2023-10-09T10:49:43.787366+0000","last_became_peered":"2023-10-09T10:49:43.787366+0000","last_unstale":"2023-10-09T10:51:11.364549+0000","last_undegraded":"2023-10-09T10:51:11.364549+0000","last_fullsized":"2023-10-09T10:51:11.364549+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":104,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.362009+0000","last_change":"2023-10-09T10:51:12.360973+0000","last_active":"2023-10-09T10:51:12.362009+0000","last_peered":"2023-10-09T10:51:12.362009+0000","last_clean":"2023-10-09T10:51:12.362009+0000","last_became_active":"2023-10-09T10:51:12.360690+0000","last_became_peered":"2023-10-09T10:51:12.360690+0000","last_unstale":"2023-10-09T10:51:12.362009+0000","last_undegraded":"2023-10-09T10:51:12.362009+0000","last_fullsized":"2023-10-09T10:51:12.362009+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:34.857608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":122,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.353875+0000","last_change":"2023-10-09T10:49:55.850829+0000","last_active":"2023-10-09T10:51:12.353875+0000","last_peered":"2023-10-09T10:51:12.353875+0000","last_clean":"2023-10-09T10:51:12.353875+0000","last_became_active":"2023-10-09T10:49:55.837771+0000","last_became_peered":"2023-10-09T10:49:55.837771+0000","last_unstale":"2023-10-09T10:51:12.353875+0000","last_undegraded":"2023-10-09T10:51:12.353875+0000","last_fullsized":"2023-10-09T10:51:12.353875+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:07.101559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":25,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.354023+0000","last_change":"2023-10-09T10:50:43.384204+0000","last_active":"2023-10-09T10:51:12.354023+0000","last_peered":"2023-10-09T10:51:12.354023+0000","last_clean":"2023-10-09T10:51:12.354023+0000","last_became_active":"2023-10-09T10:50:43.383948+0000","last_became_peered":"2023-10-09T10:50:43.383948+0000","last_unstale":"2023-10-09T10:51:12.354023+0000","last_undegraded":"2023-10-09T10:51:12.354023+0000","last_fullsized":"2023-10-09T10:51:12.354023+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:13:15.523453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":594,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.364076+0000","last_change":"2023-10-09T10:51:01.305767+0000","last_active":"2023-10-09T10:51:11.364076+0000","last_peered":"2023-10-09T10:51:11.364076+0000","last_clean":"2023-10-09T10:51:11.364076+0000","last_became_active":"2023-10-09T10:51:01.305503+0000","last_became_peered":"2023-10-09T10:51:01.305503+0000","last_unstale":"2023-10-09T10:51:11.364076+0000","last_undegraded":"2023-10-09T10:51:11.364076+0000","last_fullsized":"2023-10-09T10:51:11.364076+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":123,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.353779+0000","last_change":"2023-10-09T10:49:55.830269+0000","last_active":"2023-10-09T10:51:12.353779+0000","last_peered":"2023-10-09T10:51:12.353779+0000","last_clean":"2023-10-09T10:51:12.353779+0000","last_became_active":"2023-10-09T10:49:55.829637+0000","last_became_peered":"2023-10-09T10:49:55.829637+0000","last_unstale":"2023-10-09T10:51:12.353779+0000","last_undegraded":"2023-10-09T10:51:12.353779+0000","last_fullsized":"2023-10-09T10:51:12.353779+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:39.579608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":113,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.364719+0000","last_change":"2023-10-09T10:49:55.817112+0000","last_active":"2023-10-09T10:51:11.364719+0000","last_peered":"2023-10-09T10:51:11.364719+0000","last_clean":"2023-10-09T10:51:11.364719+0000","last_became_active":"2023-10-09T10:49:55.815889+0000","last_became_peered":"2023-10-09T10:49:55.815889+0000","last_unstale":"2023-10-09T10:51:11.364719+0000","last_undegraded":"2023-10-09T10:51:11.364719+0000","last_fullsized":"2023-10-09T10:51:11.364719+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:41.374680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":629,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.360427+0000","last_change":"2023-10-09T10:51:01.310288+0000","last_active":"2023-10-09T10:51:12.360427+0000","last_peered":"2023-10-09T10:51:12.360427+0000","last_clean":"2023-10-09T10:51:12.360427+0000","last_became_active":"2023-10-09T10:51:01.309938+0000","last_became_peered":"2023-10-09T10:51:01.309938+0000","last_unstale":"2023-10-09T10:51:12.360427+0000","last_undegraded":"2023-10-09T10:51:12.360427+0000","last_fullsized":"2023-10-09T10:51:12.360427+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":216,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.354653+0000","last_change":"2023-10-09T10:49:55.831015+0000","last_active":"2023-10-09T10:51:12.354653+0000","last_peered":"2023-10-09T10:51:12.354653+0000","last_clean":"2023-10-09T10:51:12.354653+0000","last_became_active":"2023-10-09T10:49:55.816720+0000","last_became_peered":"2023-10-09T10:49:55.816720+0000","last_unstale":"2023-10-09T10:51:12.354653+0000","last_undegraded":"2023-10-09T10:51:12.354653+0000","last_fullsized":"2023-10-09T10:51:12.354653+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:05:36.526668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":213,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.366100+0000","last_change":"2023-10-09T10:50:43.388454+0000","last_active":"2023-10-09T10:51:11.366100+0000","last_peered":"2023-10-09T10:51:11.366100+0000","last_clean":"2023-10-09T10:51:11.366100+0000","last_became_active":"2023-10-09T10:50:43.388148+0000","last_became_peered":"2023-10-09T10:50:43.388148+0000","last_unstale":"2023-10-09T10:51:11.366100+0000","last_undegraded":"2023-10-09T10:51:11.366100+0000","last_fullsized":"2023-10-09T10:51:11.366100+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:49:07.119296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":185,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.355289+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:51:12.355289+0000","last_peered":"2023-10-09T10:51:12.355289+0000","last_clean":"2023-10-09T10:51:12.355289+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:51:12.355289+0000","last_undegraded":"2023-10-09T10:51:12.355289+0000","last_fullsized":"2023-10-09T10:51:12.355289+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":176,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.368934+0000","last_change":"2023-10-09T10:50:43.385150+0000","last_active":"2023-10-09T10:51:11.368934+0000","last_peered":"2023-10-09T10:51:11.368934+0000","last_clean":"2023-10-09T10:51:11.368934+0000","last_became_active":"2023-10-09T10:50:43.384788+0000","last_became_peered":"2023-10-09T10:50:43.384788+0000","last_unstale":"2023-10-09T10:51:11.368934+0000","last_undegraded":"2023-10-09T10:51:11.368934+0000","last_fullsized":"2023-10-09T10:51:11.368934+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:36.085910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":578,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.364216+0000","last_change":"2023-10-09T10:51:01.296405+0000","last_active":"2023-10-09T10:51:11.364216+0000","last_peered":"2023-10-09T10:51:11.364216+0000","last_clean":"2023-10-09T10:51:11.364216+0000","last_became_active":"2023-10-09T10:51:01.295997+0000","last_became_peered":"2023-10-09T10:51:01.295997+0000","last_unstale":"2023-10-09T10:51:11.364216+0000","last_undegraded":"2023-10-09T10:51:11.364216+0000","last_fullsized":"2023-10-09T10:51:11.364216+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:41.637984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:41.637984+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001804833,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":228,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.369063+0000","last_change":"2023-10-09T10:49:55.821623+0000","last_active":"2023-10-09T10:51:11.369063+0000","last_peered":"2023-10-09T10:51:11.369063+0000","last_clean":"2023-10-09T10:51:11.369063+0000","last_became_active":"2023-10-09T10:49:55.821367+0000","last_became_peered":"2023-10-09T10:49:55.821367+0000","last_unstale":"2023-10-09T10:51:11.369063+0000","last_undegraded":"2023-10-09T10:51:11.369063+0000","last_fullsized":"2023-10-09T10:51:11.369063+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:09.753526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":250,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T10:51:11.364511+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:51:11.364511+0000","last_peered":"2023-10-09T10:51:11.364511+0000","last_clean":"2023-10-09T10:51:11.364511+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:51:11.364511+0000","last_undegraded":"2023-10-09T10:51:11.364511+0000","last_fullsized":"2023-10-09T10:51:11.364511+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":497,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:13.068511+0000","last_change":"2023-10-09T10:51:12.365645+0000","last_active":"2023-10-09T10:51:13.068511+0000","last_peered":"2023-10-09T10:51:13.068511+0000","last_clean":"2023-10-09T10:51:13.068511+0000","last_became_active":"2023-10-09T10:51:12.364873+0000","last_became_peered":"2023-10-09T10:51:12.364873+0000","last_unstale":"2023-10-09T10:51:13.068511+0000","last_undegraded":"2023-10-09T10:51:13.068511+0000","last_fullsized":"2023-10-09T10:51:13.068511+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":126,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.358703+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:51:12.358703+0000","last_peered":"2023-10-09T10:51:12.358703+0000","last_clean":"2023-10-09T10:51:12.358703+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:51:12.358703+0000","last_undegraded":"2023-10-09T10:51:12.358703+0000","last_fullsized":"2023-10-09T10:51:12.358703+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":209,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T10:51:12.357693+0000","last_change":"2023-10-09T10:49:55.814865+0000","last_active":"2023-10-09T10:51:12.357693+0000","last_peered":"2023-10-09T10:51:12.357693+0000","last_clean":"2023-10-09T10:51:12.357693+0000","last_became_active":"2023-10-09T10:49:55.814453+0000","last_became_peered":"2023-10-09T10:49:55.814453+0000","last_unstale":"2023-10-09T10:51:12.357693+0000","last_undegraded":"2023-10-09T10:51:12.357693+0000","last_fullsized":"2023-10-09T10:51:12.357693+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:05:10.291518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4856938520,"num_objects":74112,"num_object_clones":0,"num_object_copies":222336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21587,"num_objects_misplaced":3511,"num_objects_unfound":0,"num_objects_dirty":74112,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":108158,"num_write_kb":5832577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12286,"num_bytes_recovered":805175296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15662985216,"data_stored":15662973000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":68685,"ondisk_log_size":68685,"up":48,"acting":40,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":1312,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":685479,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":294,"num_read_kb":554,"num_write":144,"num_write_kb":1463,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1336085,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":219043332296,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":44,"seq":188978561227,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3026192,"kb_used_data":2716616,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":90718960,"statfs":{"total":95995035648,"available":92896215040,"internally_reserved":0,"allocated":2781814784,"data_stored":2780940253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:50:48 2023","interfaces":[{"interface":"back","average":{"1min":1.9730000000000001,"5min":1.821,"15min":1.0309999999999999},"min":{"1min":0.23300000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":7.0170000000000003,"5min":12.288,"15min":12.288},"last":0.624},{"interface":"front","average":{"1min":2.2869999999999999,"5min":1.8480000000000001,"15min":1.0309999999999999},"min":{"1min":0.35399999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.845000000000001,"5min":12.369999999999999,"15min":12.369999999999999},"last":0.42299999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:50:56 2023","interfaces":[{"interface":"back","average":{"1min":2.2770000000000001,"5min":1.875,"15min":1.0960000000000001},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":8.2650000000000006,"5min":15.502000000000001,"15min":16.747},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.536,"5min":1.786,"15min":1.0569999999999999},"min":{"1min":0.34300000000000003,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.3799999999999999,"5min":13.137,"15min":16.734000000000002},"last":0.40400000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:50:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.696,"15min":0.92600000000000005},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.3479999999999999,"5min":13.208,"15min":13.208},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.903,"15min":1.0369999999999999},"min":{"1min":0.29999999999999999,"5min":0.28599999999999998,"15min":0.28399999999999997},"max":{"1min":5.2539999999999996,"5min":12.571,"15min":12.571},"last":0.60699999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:50:48 2023","interfaces":[{"interface":"back","average":{"1min":2.3519999999999999,"5min":1.8700000000000001,"15min":1.04},"min":{"1min":0.35899999999999999,"5min":0.255,"15min":0.255},"max":{"1min":7.8250000000000002,"5min":12.391,"15min":12.391},"last":0.67700000000000005},{"interface":"front","average":{"1min":2.8479999999999999,"5min":2.234,"15min":1.157},"min":{"1min":0.39400000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.1879999999999997,"5min":13.179,"15min":13.179},"last":0.48599999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:50:14 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.3440000000000001,"15min":0.751},"min":{"1min":0.27300000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.4569999999999999,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.3069999999999999,"5min":1.514,"15min":0.83399999999999996},"min":{"1min":0.28799999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.6970000000000001,"5min":12.481999999999999,"15min":12.481999999999999},"last":0.71899999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:50:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":1.4770000000000001,"15min":0.82499999999999996},"min":{"1min":0.29999999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.9589999999999996,"5min":10.497,"15min":10.497},"last":0.65100000000000002},{"interface":"front","average":{"1min":1.05,"5min":1.47,"15min":0.84899999999999998},"min":{"1min":0.318,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":6.6710000000000003,"5min":12.456,"15min":12.456},"last":0.70199999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":1.712,"15min":0.93899999999999995},"min":{"1min":0.34100000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.9820000000000002,"5min":23.358000000000001,"15min":23.358000000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.603,"15min":0.93100000000000005},"min":{"1min":0.36299999999999999,"5min":0.35099999999999998,"15min":0.34000000000000002},"max":{"1min":6.7210000000000001,"5min":12.422000000000001,"15min":12.422000000000001},"last":0.73699999999999999}]}]},{"osd":1,"up_from":0,"seq":365072220266,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":80,"seq":343597383800,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3095628,"kb_used_data":2812420,"kb_used_omap":28,"kb_used_meta":283107,"kb_avail":90649524,"statfs":{"total":95995035648,"available":92825112576,"internally_reserved":0,"allocated":2879918080,"data_stored":2878639259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":289902416},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,6,0,3,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:50:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":2.1520000000000001,"15min":1.143},"min":{"1min":0.19500000000000001,"5min":0.191,"15min":0.191},"max":{"1min":4.4909999999999997,"5min":14.311,"15min":25.475000000000001},"last":4.9930000000000003},{"interface":"front","average":{"1min":1.407,"5min":2.153,"15min":1.127},"min":{"1min":0.249,"5min":0.188,"15min":0.188},"max":{"1min":6.6849999999999996,"5min":12.276,"15min":20.832999999999998},"last":4.8470000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:50:22 2023","interfaces":[{"interface":"back","average":{"1min":2.6059999999999999,"5min":2.1280000000000001,"15min":1.036},"min":{"1min":0.23999999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":17.927,"5min":17.927,"15min":17.927},"last":0.26700000000000002},{"interface":"front","average":{"1min":2.117,"5min":2.3809999999999998,"15min":1.0960000000000001},"min":{"1min":0.28799999999999998,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.8089999999999993,"5min":14.391999999999999,"15min":14.391999999999999},"last":1.4710000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:50:55 2023","interfaces":[{"interface":"back","average":{"1min":2.2389999999999999,"5min":2.4460000000000002,"15min":1.3879999999999999},"min":{"1min":0.218,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":11.237,"5min":12.307,"15min":14.926},"last":12.090999999999999},{"interface":"front","average":{"1min":1.986,"5min":2.3860000000000001,"15min":1.296},"min":{"1min":0.29099999999999998,"5min":0.224,"15min":0.224},"max":{"1min":10.462999999999999,"5min":14.391999999999999,"15min":14.391999999999999},"last":4.5709999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":2.2770000000000001,"5min":2.0030000000000001,"15min":1.159},"min":{"1min":0.34699999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.6129999999999995,"5min":9.6129999999999995,"15min":9.6129999999999995},"last":1.5089999999999999},{"interface":"front","average":{"1min":2.9470000000000001,"5min":2.4489999999999998,"15min":1.3140000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.1790000000000003,"5min":12.318,"15min":12.318},"last":4.9340000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":2.4129999999999998,"15min":1.304},"min":{"1min":0.35399999999999998,"5min":0.318,"15min":0.318},"max":{"1min":7.1840000000000002,"5min":12.157,"15min":12.157},"last":4.96},{"interface":"front","average":{"1min":2.9700000000000002,"5min":2.177,"15min":1.226},"min":{"1min":0.317,"5min":0.317,"15min":0.29499999999999998},"max":{"1min":14.446,"5min":14.446,"15min":14.446},"last":4.5330000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":3.0619999999999998,"5min":2.548,"15min":1.3540000000000001},"min":{"1min":0.41499999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.734,"5min":12.207000000000001,"15min":12.207000000000001},"last":4.8940000000000001},{"interface":"front","average":{"1min":2.738,"5min":1.768,"15min":1.1020000000000001},"min":{"1min":0.34300000000000003,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":14.420999999999999,"5min":14.420999999999999,"15min":14.420999999999999},"last":1.1100000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":3.2269999999999999,"5min":2.3490000000000002,"15min":1.2889999999999999},"min":{"1min":0.45600000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":14.48,"5min":14.48,"15min":14.48},"last":0.99099999999999999},{"interface":"front","average":{"1min":2.0720000000000001,"5min":2.1469999999999998,"15min":1.254},"min":{"1min":0.438,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.6440000000000001,"5min":14.394,"15min":14.394},"last":1.5740000000000001}]}]},{"osd":2,"up_from":0,"seq":386547056740,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893214,"num_pgs":72,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3953204,"kb_used_data":3506988,"kb_used_omap":7,"kb_used_meta":446136,"kb_avail":89791948,"statfs":{"total":95995035648,"available":91946954752,"internally_reserved":0,"allocated":3591155712,"data_stored":3589888564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":456843880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:50:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1520000000000001,"5min":2.548,"15min":1.127},"min":{"1min":0.20100000000000001,"5min":0.183,"15min":0.183},"max":{"1min":10.388999999999999,"5min":15.255000000000001,"15min":15.255000000000001},"last":1.3320000000000001},{"interface":"front","average":{"1min":2.9079999999999999,"5min":2.355,"15min":1.169},"min":{"1min":0.20300000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":13.266,"5min":14.798,"15min":14.798},"last":0.27200000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:50:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1960000000000002,"5min":2.508,"15min":1.286},"min":{"1min":0.23699999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":8.2880000000000003,"5min":13.967000000000001,"15min":14.707000000000001},"last":0.81499999999999995},{"interface":"front","average":{"1min":2.5110000000000001,"5min":2.5150000000000001,"15min":1.1739999999999999},"min":{"1min":0.34100000000000003,"5min":0.19400000000000001,"15min":0.193},"max":{"1min":7.9059999999999997,"5min":15.443,"15min":15.443},"last":0.51900000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:50:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3079999999999998,"5min":2.3149999999999999,"15min":1.2190000000000001},"min":{"1min":0.26700000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.9510000000000005,"5min":13.853,"15min":13.853},"last":0.85999999999999999},{"interface":"front","average":{"1min":2.1549999999999998,"5min":2.4359999999999999,"15min":1.2290000000000001},"min":{"1min":0.28000000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.1859999999999999,"5min":10.557,"15min":10.557},"last":0.33600000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:50:50 2023","interfaces":[{"interface":"back","average":{"1min":1.7549999999999999,"5min":2.3500000000000001,"15min":1.2250000000000001},"min":{"1min":0.38400000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.0019999999999998,"5min":20.388000000000002,"15min":20.388000000000002},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.7090000000000001,"5min":2.3380000000000001,"15min":1.256},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.2530000000000001,"5min":19.085999999999999,"15min":19.085999999999999},"last":0.46100000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:50:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0750000000000002,"5min":3.0430000000000001,"15min":1.4810000000000001},"min":{"1min":0.29899999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.6790000000000003,"5min":51.509999999999998,"15min":51.509999999999998},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.8130000000000002,"5min":3.0139999999999998,"15min":1.55},"min":{"1min":0.36899999999999999,"5min":0.316,"15min":0.27500000000000002},"max":{"1min":11.082000000000001,"5min":55.338999999999999,"15min":55.338999999999999},"last":1.1759999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:50:50 2023","interfaces":[{"interface":"back","average":{"1min":2.165,"5min":2.1179999999999999,"15min":1.196},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.3740000000000006,"5min":12.361000000000001,"15min":12.361000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.8010000000000002,"5min":2.4449999999999998,"15min":1.3939999999999999},"min":{"1min":0.42299999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":11.24,"5min":11.470000000000001,"15min":13.891999999999999},"last":1.113}]},{"osd":7,"last update":"Mon Oct 9 10:50:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2429999999999999,"5min":3.0760000000000001,"15min":1.5940000000000001},"min":{"1min":0.433,"5min":0.31,"15min":0.31},"max":{"1min":8.2650000000000006,"5min":51.744,"15min":51.744},"last":1.1599999999999999},{"interface":"front","average":{"1min":2.7130000000000001,"5min":3.1150000000000002,"15min":1.49},"min":{"1min":0.505,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.8070000000000004,"5min":53.399000000000001,"15min":53.399000000000001},"last":0.83499999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019090,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2640588,"kb_used_data":2339204,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91104564,"statfs":{"total":95995035648,"available":93291073536,"internally_reserved":0,"allocated":2395344896,"data_stored":2394473704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:50:41 2023","interfaces":[{"interface":"back","average":{"1min":2.3319999999999999,"5min":1.6399999999999999,"15min":0.95299999999999996},"min":{"1min":0.217,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.4329999999999998,"5min":10.387,"15min":10.387},"last":0.253},{"interface":"front","average":{"1min":2.2090000000000001,"5min":1.8660000000000001,"15min":1.081},"min":{"1min":0.28999999999999998,"5min":0.217,"15min":0.217},"max":{"1min":9.2889999999999997,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.30599999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:50:46 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.462,"15min":0.94499999999999995},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.1239999999999997,"5min":7.5350000000000001,"15min":7.5350000000000001},"last":3.2360000000000002},{"interface":"front","average":{"1min":2.0960000000000001,"5min":1.946,"15min":1.119},"min":{"1min":0.33700000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.3179999999999996,"5min":8.7669999999999995,"15min":8.7669999999999995},"last":0.57599999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.7669999999999999,"15min":1.018},"min":{"1min":0.28599999999999998,"5min":0.253,"15min":0.253},"max":{"1min":5.335,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.629,"5min":2.2770000000000001,"15min":1.1719999999999999},"min":{"1min":0.35399999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.077,"5min":20.545999999999999,"15min":20.545999999999999},"last":0.373}]},{"osd":3,"last update":"Mon Oct 9 10:50:41 2023","interfaces":[{"interface":"back","average":{"1min":2.6669999999999998,"5min":2.0960000000000001,"15min":1.224},"min":{"1min":0.29299999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.9000000000000004,"5min":9.0269999999999992,"15min":9.0269999999999992},"last":1.835},{"interface":"front","average":{"1min":1.9259999999999999,"5min":2.4790000000000001,"15min":1.2490000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":9.2249999999999996,"5min":9.6929999999999996,"15min":9.6929999999999996},"last":3.423}]},{"osd":5,"last update":"Mon Oct 9 10:50:46 2023","interfaces":[{"interface":"back","average":{"1min":3.081,"5min":2.2570000000000001,"15min":1.129},"min":{"1min":0.26100000000000001,"5min":0.224,"15min":0.224},"max":{"1min":10.263999999999999,"5min":24.149999999999999,"15min":24.149999999999999},"last":6.1020000000000003},{"interface":"front","average":{"1min":1.952,"5min":2.069,"15min":1.046},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":7.3150000000000004,"5min":20.422000000000001,"15min":20.422000000000001},"last":6.0419999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:51:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3809999999999998,"5min":1.8700000000000001,"15min":0.997},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":12.279,"5min":14.897,"15min":14.897},"last":0.47899999999999998},{"interface":"front","average":{"1min":3.625,"5min":2.242,"15min":1.1639999999999999},"min":{"1min":0.40000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":14.272,"5min":14.272,"15min":14.272},"last":0.49399999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:50:25 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":1.6850000000000001,"15min":0.95499999999999996},"min":{"1min":0.33700000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.7240000000000002,"5min":9.3360000000000003,"15min":9.3360000000000003},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.464,"15min":0.875},"min":{"1min":0.33900000000000002,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":5.5129999999999999,"5min":7.423,"15min":7.423},"last":3.3839999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790159,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1996736,"kb_used_data":1699448,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91748416,"statfs":{"total":95995035648,"available":93950377984,"internally_reserved":0,"allocated":1740234752,"data_stored":1739371835,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:50:48 2023","interfaces":[{"interface":"back","average":{"1min":3.0899999999999999,"5min":1.6870000000000001,"15min":0.99099999999999999},"min":{"1min":0.23300000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":10.997,"5min":11.648999999999999,"15min":11.648999999999999},"last":3.2549999999999999},{"interface":"front","average":{"1min":3.1200000000000001,"5min":1.752,"15min":1.0569999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":11.031000000000001,"5min":11.68,"15min":11.68},"last":3.282}]},{"osd":1,"last update":"Mon Oct 9 10:50:56 2023","interfaces":[{"interface":"back","average":{"1min":3.2730000000000001,"5min":2.3250000000000002,"15min":1.2070000000000001},"min":{"1min":0.39000000000000001,"5min":0.251,"15min":0.251},"max":{"1min":10.682,"5min":12.999000000000001,"15min":12.999000000000001},"last":6.2690000000000001},{"interface":"front","average":{"1min":2.3380000000000001,"5min":1.6759999999999999,"15min":1.0309999999999999},"min":{"1min":0.28499999999999998,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":12.109,"5min":12.109,"15min":12.109},"last":2.613}]},{"osd":2,"last update":"Mon Oct 9 10:50:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":1.8799999999999999,"15min":1.0349999999999999},"min":{"1min":0.377,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.5529999999999999,"5min":11.393000000000001,"15min":11.393000000000001},"last":2.5379999999999998},{"interface":"front","average":{"1min":0.78900000000000003,"5min":1.4390000000000001,"15min":0.86799999999999999},"min":{"1min":0.32200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.2799999999999998,"5min":10.352,"15min":10.352},"last":2.637}]},{"osd":3,"last update":"Mon Oct 9 10:50:51 2023","interfaces":[{"interface":"back","average":{"1min":1.9950000000000001,"5min":2.2400000000000002,"15min":1.2849999999999999},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.1369999999999996,"5min":18.016999999999999,"15min":18.016999999999999},"last":2.6970000000000001},{"interface":"front","average":{"1min":2.7509999999999999,"5min":2.5539999999999998,"15min":1.321},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":11.067,"5min":15.555999999999999,"15min":15.555999999999999},"last":4.1829999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:50:56 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":2.5569999999999999,"15min":1.1799999999999999},"min":{"1min":0.34200000000000003,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.3890000000000002,"5min":123.241,"15min":123.241},"last":3.093},{"interface":"front","average":{"1min":2.698,"5min":3.0960000000000001,"15min":1.3859999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.111000000000001,"5min":123.318,"15min":123.318},"last":3.3719999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:51:14 2023","interfaces":[{"interface":"back","average":{"1min":3.0920000000000001,"5min":2.024,"15min":1.044},"min":{"1min":0.34599999999999997,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":11.098000000000001,"5min":11.098000000000001,"15min":11.098000000000001},"last":2.5710000000000002},{"interface":"front","average":{"1min":3.2690000000000001,"5min":1.8340000000000001,"15min":0.97999999999999998},"min":{"1min":0.46700000000000003,"5min":0.318,"15min":0.318},"max":{"1min":15.16,"5min":20.256,"15min":20.256},"last":2.9529999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:50:31 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":1.615,"15min":0.92600000000000005},"min":{"1min":0.32300000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":10.739000000000001,"5min":10.739000000000001,"15min":10.739000000000001},"last":6.2279999999999998},{"interface":"front","average":{"1min":1.014,"5min":1.3380000000000001,"15min":0.85399999999999998},"min":{"1min":0.35299999999999998,"5min":0.33200000000000002,"15min":0.31900000000000001},"max":{"1min":3.694,"5min":14.154,"15min":14.154},"last":2.669}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2859470848,"data_stored":2859466776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1634795520,"data_stored":1634795520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3560443904,"data_stored":3560439832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2384789504,"data_stored":2384789504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1736179712,"data_stored":1736179712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2767458304,"data_stored":2767454232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":719847424,"data_stored":719847424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:51:20.574 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:51:20.575 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 41 2023-10-09T10:51:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:20 smithi155 ceph-mon[184854]: pgmap v539: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 552 op/s; 21587/222405 objects degraded (9.706%); 3511/222405 objects misplaced (1.579%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T10:51:20.881 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:20.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:20.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:20.893 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:51:20.994 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:21.070 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:21.154 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:21.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:21.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:21.260 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:21.298 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:21.331 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:21.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:21.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:21.406 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:51:21.435 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:21 smithi125 ceph-mon[195507]: from='client.55132 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:51:21.436 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:51:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:51:21] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.43.0" 2023-10-09T10:51:21.438 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:21.488 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:21.533 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:21.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:21.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:21.637 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:21.707 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:21 smithi125 ceph-mon[200197]: from='client.55132 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:51:21.717 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:21 smithi155 ceph-mon[184854]: from='client.55132 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:51:22.379 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:22.454 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:22.509 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:22.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:22.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:22.528 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:22.622 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:22.666 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:22.699 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:22.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:22.709 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:22 smithi155 ceph-mon[184854]: pgmap v540: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 650 op/s; 21587/218151 objects degraded (9.895%); 3161/218151 objects misplaced (1.449%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T10:51:22.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:22.810 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:22.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:22 smithi125 ceph-mon[195507]: pgmap v540: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 650 op/s; 21587/218151 objects degraded (9.895%); 3161/218151 objects misplaced (1.449%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T10:51:22.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:22 smithi125 ceph-mon[200197]: pgmap v540: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 650 op/s; 21587/218151 objects degraded (9.895%); 3161/218151 objects misplaced (1.449%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T10:51:22.872 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:22.900 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:22.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:22.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:23.004 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:23.049 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:23.079 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:23.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:23.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:23.182 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:23.235 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:23.322 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:23.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:23.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:23.435 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:23.438 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:23 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3356465191' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:23.438 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:23 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:23.438 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:23 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 21587/218151 objects degraded (9.895%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:23.465 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:51:23.470 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:23.500 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:23 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3356465191' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:23.500 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:23 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:23.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:23 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 21587/218151 objects degraded (9.895%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:23.501 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:23 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3356465191' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:23.501 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:23 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:23.501 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:23 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 21587/218151 objects degraded (9.895%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:23.502 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:23.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:23.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:23.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:23.647 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:23.676 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:23.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:23.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:23.784 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:23.831 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:23.862 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:23.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:23.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:23.971 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:24.019 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:24.212 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:24.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:24.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:24.319 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:24.364 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:24.387 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:24.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:24.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:24.491 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:24.541 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:24.680 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[200197]: pgmap v541: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 702 op/s; 20907/211230 objects degraded (9.898%); 3161/211230 objects misplaced (1.496%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T10:51:24.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:51:24.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[200197]: failed to encode map e236 with expected crc 2023-10-09T10:51:24.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:51:24.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[200197]: osdmap e236: 8 total, 8 up, 5 in 2023-10-09T10:51:24.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[200197]: failed to encode map e236 with expected crc 2023-10-09T10:51:24.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[200197]: failed to encode map e236 with expected crc 2023-10-09T10:51:24.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[200197]: failed to encode map e236 with expected crc 2023-10-09T10:51:24.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[200197]: failed to encode map e236 with expected crc 2023-10-09T10:51:24.681 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[195507]: pgmap v541: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 702 op/s; 20907/211230 objects degraded (9.898%); 3161/211230 objects misplaced (1.496%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T10:51:24.682 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:51:24.682 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[195507]: failed to encode map e236 with expected crc 2023-10-09T10:51:24.682 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:51:24.682 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[195507]: osdmap e236: 8 total, 8 up, 5 in 2023-10-09T10:51:24.682 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[195507]: failed to encode map e236 with expected crc 2023-10-09T10:51:24.682 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[195507]: failed to encode map e236 with expected crc 2023-10-09T10:51:24.682 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[195507]: failed to encode map e236 with expected crc 2023-10-09T10:51:24.682 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:24 smithi125 ceph-mon[195507]: failed to encode map e236 with expected crc 2023-10-09T10:51:24.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:24 smithi155 ceph-mon[184854]: pgmap v541: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 702 op/s; 20907/211230 objects degraded (9.898%); 3161/211230 objects misplaced (1.496%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T10:51:24.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:24 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:51:24.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:24 smithi155 ceph-mon[184854]: failed to encode map e236 with expected crc 2023-10-09T10:51:24.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:24 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:51:24.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:24 smithi155 ceph-mon[184854]: osdmap e236: 8 total, 8 up, 5 in 2023-10-09T10:51:24.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:24 smithi155 ceph-mon[184854]: failed to encode map e236 with expected crc 2023-10-09T10:51:24.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:24 smithi155 ceph-mon[184854]: failed to encode map e236 with expected crc 2023-10-09T10:51:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:24 smithi155 ceph-mon[184854]: failed to encode map e236 with expected crc 2023-10-09T10:51:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:24 smithi155 ceph-mon[184854]: failed to encode map e236 with expected crc 2023-10-09T10:51:24.931 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:24.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:24.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:25.038 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:25.089 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:25.484 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:25.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:25.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:25.594 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:25.654 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:25.869 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:25.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:25.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:25.920 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:51:25.955 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:51:25.974 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:26.005 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:51:26.040 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:26.072 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:51:26.088 INFO:teuthology.orchestra.run.smithi125.stderr:Error EINVAL: specified pg_num 41 < pg_num_max 32 2023-10-09T10:51:26.103 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:26.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:26.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:26.120 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:51:26.169 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:51:26.212 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:26.226 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:51:26.226 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:51:26.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:51:26.226 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:51:26.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.226 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42934 6.11 6:8a3972ec:::benchmark_data_smithi125_244951_object42933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.226 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.549876+0000", 2023-10-09T10:51:26.226 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.66967790699999, 2023-10-09T10:51:26.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.229929598, 2023-10-09T10:51:26.227 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.227 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.227 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.227 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.227 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.227 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42934 2023-10-09T10:51:26.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.227 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549876+0000", 2023-10-09T10:51:26.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.228 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.228 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549876+0000", 2023-10-09T10:51:26.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.228 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.228 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549874+0000", 2023-10-09T10:51:26.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:51:26.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549907+0000", 2023-10-09T10:51:26.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2579000000000001e-05 2023-10-09T10:51:26.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549922+0000", 2023-10-09T10:51:26.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5420000000000001e-05 2023-10-09T10:51:26.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549934+0000", 2023-10-09T10:51:26.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1724e-05 2023-10-09T10:51:26.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550289+0000", 2023-10-09T10:51:26.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035548900000000001 2023-10-09T10:51:26.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op must wait for map", 2023-10-09T10:51:26.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550297+0000", 2023-10-09T10:51:26.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8869999999999992e-06 2023-10-09T10:51:26.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551339+0000", 2023-10-09T10:51:26.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010423069999999999 2023-10-09T10:51:26.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551349+0000", 2023-10-09T10:51:26.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8060000000000006e-06 2023-10-09T10:51:26.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.776033+0000", 2023-10-09T10:51:26.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.224684135 2023-10-09T10:51:26.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.776159+0000", 2023-10-09T10:51:26.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012583900000000001 2023-10-09T10:51:26.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.776290+0000", 2023-10-09T10:51:26.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000130705 2023-10-09T10:51:26.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778810+0000", 2023-10-09T10:51:26.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025205549999999998 2023-10-09T10:51:26.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778863+0000", 2023-10-09T10:51:26.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2821000000000001e-05 2023-10-09T10:51:26.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778971+0000", 2023-10-09T10:51:26.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108244 2023-10-09T10:51:26.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779012+0000", 2023-10-09T10:51:26.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0612999999999999e-05 2023-10-09T10:51:26.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779058+0000", 2023-10-09T10:51:26.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5948999999999997e-05 2023-10-09T10:51:26.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779805+0000", 2023-10-09T10:51:26.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00074730900000000004 2023-10-09T10:51:26.235 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.235 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.235 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.235 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42936 6.11 6:8ce9d1a1:::benchmark_data_smithi125_244951_object42935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.235 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.549950+0000", 2023-10-09T10:51:26.235 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.66960389399998, 2023-10-09T10:51:26.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.235619359, 2023-10-09T10:51:26.236 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.236 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.236 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.236 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.236 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.236 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42936 2023-10-09T10:51:26.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.236 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549950+0000", 2023-10-09T10:51:26.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549950+0000", 2023-10-09T10:51:26.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549949+0000", 2023-10-09T10:51:26.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:26.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549976+0000", 2023-10-09T10:51:26.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7223999999999999e-05 2023-10-09T10:51:26.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.238 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549987+0000", 2023-10-09T10:51:26.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0375e-05 2023-10-09T10:51:26.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.238 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549996+0000", 2023-10-09T10:51:26.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4010000000000003e-06 2023-10-09T10:51:26.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550347+0000", 2023-10-09T10:51:26.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035085200000000002 2023-10-09T10:51:26.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:51:26.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550352+0000", 2023-10-09T10:51:26.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2079999999999999e-06 2023-10-09T10:51:26.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551460+0000", 2023-10-09T10:51:26.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001107762 2023-10-09T10:51:26.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551467+0000", 2023-10-09T10:51:26.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0350000000000004e-06 2023-10-09T10:51:26.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.777841+0000", 2023-10-09T10:51:26.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2263741750000001 2023-10-09T10:51:26.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778003+0000", 2023-10-09T10:51:26.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016172699999999999 2023-10-09T10:51:26.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778128+0000", 2023-10-09T10:51:26.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012541000000000001 2023-10-09T10:51:26.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780770+0000", 2023-10-09T10:51:26.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026419500000000001 2023-10-09T10:51:26.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780795+0000", 2023-10-09T10:51:26.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4363e-05 2023-10-09T10:51:26.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784699+0000", 2023-10-09T10:51:26.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039046739999999999 2023-10-09T10:51:26.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784714+0000", 2023-10-09T10:51:26.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4956999999999999e-05 2023-10-09T10:51:26.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784774+0000", 2023-10-09T10:51:26.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9994999999999997e-05 2023-10-09T10:51:26.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.244 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785569+0000", 2023-10-09T10:51:26.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00079485999999999999 2023-10-09T10:51:26.244 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.244 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.244 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.244 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42950 6.11 6:8b5bb41c:::benchmark_data_smithi125_244951_object42949:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.244 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:51:26.244 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.66955149900002, 2023-10-09T10:51:26.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2390020909999999, 2023-10-09T10:51:26.245 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.245 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.245 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.245 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.245 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.245 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42950 2023-10-09T10:51:26.245 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.245 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.245 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.245 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.245 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:51:26.245 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.245 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.246 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.246 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:51:26.246 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.246 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.246 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.246 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:51:26.246 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:26.246 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.246 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.247 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550021+0000", 2023-10-09T10:51:26.247 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8941999999999999e-05 2023-10-09T10:51:26.247 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.247 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.247 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.247 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550034+0000", 2023-10-09T10:51:26.247 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3267e-05 2023-10-09T10:51:26.247 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.247 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.247 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.247 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550040+0000", 2023-10-09T10:51:26.247 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5799999999999997e-06 2023-10-09T10:51:26.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550377+0000", 2023-10-09T10:51:26.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00033659800000000002 2023-10-09T10:51:26.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:51:26.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550382+0000", 2023-10-09T10:51:26.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6589999999999998e-06 2023-10-09T10:51:26.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551483+0000", 2023-10-09T10:51:26.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011012210000000001 2023-10-09T10:51:26.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551489+0000", 2023-10-09T10:51:26.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4910000000000002e-06 2023-10-09T10:51:26.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779851+0000", 2023-10-09T10:51:26.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.22836144 2023-10-09T10:51:26.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779959+0000", 2023-10-09T10:51:26.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108546 2023-10-09T10:51:26.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780061+0000", 2023-10-09T10:51:26.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010196600000000001 2023-10-09T10:51:26.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782707+0000", 2023-10-09T10:51:26.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002646074 2023-10-09T10:51:26.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785603+0000", 2023-10-09T10:51:26.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028958410000000001 2023-10-09T10:51:26.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785626+0000", 2023-10-09T10:51:26.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2371e-05 2023-10-09T10:51:26.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.787983+0000", 2023-10-09T10:51:26.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023577619999999998 2023-10-09T10:51:26.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.788053+0000", 2023-10-09T10:51:26.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9797999999999998e-05 2023-10-09T10:51:26.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.253 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789004+0000", 2023-10-09T10:51:26.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00095101500000000004 2023-10-09T10:51:26.253 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.253 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.253 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.253 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.253 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.253 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42962 6.11 6:8af38cb4:::benchmark_data_smithi125_244951_object42961:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.253 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550048+0000", 2023-10-09T10:51:26.253 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.66950596999999, 2023-10-09T10:51:26.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2398821879999999, 2023-10-09T10:51:26.253 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.254 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.254 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.254 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.254 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.254 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42962 2023-10-09T10:51:26.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.254 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.254 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.254 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.254 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550048+0000", 2023-10-09T10:51:26.254 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.255 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.255 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550048+0000", 2023-10-09T10:51:26.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.255 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.255 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.255 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550047+0000", 2023-10-09T10:51:26.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:26.255 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.255 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.255 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550067+0000", 2023-10-09T10:51:26.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0228999999999999e-05 2023-10-09T10:51:26.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550075+0000", 2023-10-09T10:51:26.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0979999999999996e-06 2023-10-09T10:51:26.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550080+0000", 2023-10-09T10:51:26.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7780000000000004e-06 2023-10-09T10:51:26.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550404+0000", 2023-10-09T10:51:26.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000323648 2023-10-09T10:51:26.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:51:26.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550409+0000", 2023-10-09T10:51:26.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6630000000000001e-06 2023-10-09T10:51:26.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551523+0000", 2023-10-09T10:51:26.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001113953 2023-10-09T10:51:26.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551531+0000", 2023-10-09T10:51:26.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5860000000000005e-06 2023-10-09T10:51:26.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780812+0000", 2023-10-09T10:51:26.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2292809120000001 2023-10-09T10:51:26.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780947+0000", 2023-10-09T10:51:26.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000135265 2023-10-09T10:51:26.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.781098+0000", 2023-10-09T10:51:26.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015082499999999999 2023-10-09T10:51:26.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782749+0000", 2023-10-09T10:51:26.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001651057 2023-10-09T10:51:26.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785607+0000", 2023-10-09T10:51:26.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028579569999999999 2023-10-09T10:51:26.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785661+0000", 2023-10-09T10:51:26.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4151999999999999e-05 2023-10-09T10:51:26.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.261 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789184+0000", 2023-10-09T10:51:26.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035224309999999999 2023-10-09T10:51:26.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.261 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789241+0000", 2023-10-09T10:51:26.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6962000000000001e-05 2023-10-09T10:51:26.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.262 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.262 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789930+0000", 2023-10-09T10:51:26.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000689148 2023-10-09T10:51:26.262 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.262 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.262 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.262 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.262 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.262 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43001 6.11 6:8f0a81da:::benchmark_data_smithi125_244951_object43000:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.262 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:51:26.262 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.66946826100002, 2023-10-09T10:51:26.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.240501734, 2023-10-09T10:51:26.263 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.263 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.263 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.263 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.263 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.263 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43001 2023-10-09T10:51:26.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.263 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.263 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.263 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:51:26.263 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:51:26.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:51:26.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:26.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550104+0000", 2023-10-09T10:51:26.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.912e-05 2023-10-09T10:51:26.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550110+0000", 2023-10-09T10:51:26.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3779999999999998e-06 2023-10-09T10:51:26.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550116+0000", 2023-10-09T10:51:26.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.609e-06 2023-10-09T10:51:26.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550429+0000", 2023-10-09T10:51:26.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00031264800000000001 2023-10-09T10:51:26.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:51:26.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550433+0000", 2023-10-09T10:51:26.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8420000000000004e-06 2023-10-09T10:51:26.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551574+0000", 2023-10-09T10:51:26.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011402210000000001 2023-10-09T10:51:26.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551584+0000", 2023-10-09T10:51:26.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9690000000000006e-06 2023-10-09T10:51:26.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.781792+0000", 2023-10-09T10:51:26.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.230208454 2023-10-09T10:51:26.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.781903+0000", 2023-10-09T10:51:26.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110733 2023-10-09T10:51:26.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782036+0000", 2023-10-09T10:51:26.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000133545 2023-10-09T10:51:26.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785598+0000", 2023-10-09T10:51:26.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035616100000000002 2023-10-09T10:51:26.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785679+0000", 2023-10-09T10:51:26.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0624999999999995e-05 2023-10-09T10:51:26.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785703+0000", 2023-10-09T10:51:26.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4150000000000001e-05 2023-10-09T10:51:26.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790058+0000", 2023-10-09T10:51:26.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043551919999999999 2023-10-09T10:51:26.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790110+0000", 2023-10-09T10:51:26.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2278000000000003e-05 2023-10-09T10:51:26.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790587+0000", 2023-10-09T10:51:26.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047690599999999998 2023-10-09T10:51:26.271 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.271 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.271 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.271 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43002 6.11 6:8bd791a2:::benchmark_data_smithi125_244951_object43001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.271 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550121+0000", 2023-10-09T10:51:26.271 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.66943296699998, 2023-10-09T10:51:26.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.241166963, 2023-10-09T10:51:26.271 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.271 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.272 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.272 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.272 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.272 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43002 2023-10-09T10:51:26.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.272 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.272 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.272 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550121+0000", 2023-10-09T10:51:26.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.272 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550121+0000", 2023-10-09T10:51:26.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550120+0000", 2023-10-09T10:51:26.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:26.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550139+0000", 2023-10-09T10:51:26.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8644999999999999e-05 2023-10-09T10:51:26.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550145+0000", 2023-10-09T10:51:26.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5180000000000001e-06 2023-10-09T10:51:26.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550151+0000", 2023-10-09T10:51:26.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0399999999999998e-06 2023-10-09T10:51:26.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551616+0000", 2023-10-09T10:51:26.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001464449 2023-10-09T10:51:26.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551623+0000", 2023-10-09T10:51:26.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0090000000000001e-06 2023-10-09T10:51:26.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782780+0000", 2023-10-09T10:51:26.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2311575050000001 2023-10-09T10:51:26.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782892+0000", 2023-10-09T10:51:26.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111826 2023-10-09T10:51:26.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782998+0000", 2023-10-09T10:51:26.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010603399999999999 2023-10-09T10:51:26.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785644+0000", 2023-10-09T10:51:26.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026456489999999999 2023-10-09T10:51:26.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785699+0000", 2023-10-09T10:51:26.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4778999999999998e-05 2023-10-09T10:51:26.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786762+0000", 2023-10-09T10:51:26.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001063172 2023-10-09T10:51:26.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790708+0000", 2023-10-09T10:51:26.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039466409999999999 2023-10-09T10:51:26.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790775+0000", 2023-10-09T10:51:26.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6155000000000006e-05 2023-10-09T10:51:26.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791288+0000", 2023-10-09T10:51:26.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051300599999999999 2023-10-09T10:51:26.279 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.279 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.279 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.279 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43015 6.11 6:8d7a9802:::benchmark_data_smithi125_244951_object43014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.279 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:51:26.279 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.669397269, 2023-10-09T10:51:26.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.241685028, 2023-10-09T10:51:26.280 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.280 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.280 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.280 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.280 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.280 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43015 2023-10-09T10:51:26.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.280 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:51:26.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:51:26.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:51:26.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:26.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550174+0000", 2023-10-09T10:51:26.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8289000000000001e-05 2023-10-09T10:51:26.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550181+0000", 2023-10-09T10:51:26.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.024e-06 2023-10-09T10:51:26.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550187+0000", 2023-10-09T10:51:26.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6659999999999996e-06 2023-10-09T10:51:26.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551652+0000", 2023-10-09T10:51:26.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001465406 2023-10-09T10:51:26.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551659+0000", 2023-10-09T10:51:26.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1130000000000003e-06 2023-10-09T10:51:26.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.783679+0000", 2023-10-09T10:51:26.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.232019406 2023-10-09T10:51:26.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.783831+0000", 2023-10-09T10:51:26.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000151769 2023-10-09T10:51:26.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784019+0000", 2023-10-09T10:51:26.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018824400000000001 2023-10-09T10:51:26.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786769+0000", 2023-10-09T10:51:26.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027502529999999998 2023-10-09T10:51:26.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789091+0000", 2023-10-09T10:51:26.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023215200000000001 2023-10-09T10:51:26.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789131+0000", 2023-10-09T10:51:26.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0766999999999998e-05 2023-10-09T10:51:26.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791345+0000", 2023-10-09T10:51:26.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022138349999999999 2023-10-09T10:51:26.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791404+0000", 2023-10-09T10:51:26.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8301000000000003e-05 2023-10-09T10:51:26.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791841+0000", 2023-10-09T10:51:26.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00043785599999999998 2023-10-09T10:51:26.287 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.287 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.287 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.288 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43040 6.11 6:892fed64:::benchmark_data_smithi125_244951_object43039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.288 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550192+0000", 2023-10-09T10:51:26.288 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.66936189, 2023-10-09T10:51:26.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2421168730000001, 2023-10-09T10:51:26.288 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.288 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.288 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.288 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.288 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.288 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43040 2023-10-09T10:51:26.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.288 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550192+0000", 2023-10-09T10:51:26.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550192+0000", 2023-10-09T10:51:26.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550191+0000", 2023-10-09T10:51:26.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:26.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550265+0000", 2023-10-09T10:51:26.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3668e-05 2023-10-09T10:51:26.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550273+0000", 2023-10-09T10:51:26.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2230000000000007e-06 2023-10-09T10:51:26.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550280+0000", 2023-10-09T10:51:26.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3729999999999998e-06 2023-10-09T10:51:26.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551688+0000", 2023-10-09T10:51:26.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001408011 2023-10-09T10:51:26.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551694+0000", 2023-10-09T10:51:26.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8120000000000002e-06 2023-10-09T10:51:26.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785718+0000", 2023-10-09T10:51:26.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2340238640000001 2023-10-09T10:51:26.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785825+0000", 2023-10-09T10:51:26.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000107161 2023-10-09T10:51:26.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785964+0000", 2023-10-09T10:51:26.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013889800000000001 2023-10-09T10:51:26.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789139+0000", 2023-10-09T10:51:26.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0031749249999999999 2023-10-09T10:51:26.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789979+0000", 2023-10-09T10:51:26.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00083957300000000003 2023-10-09T10:51:26.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790018+0000", 2023-10-09T10:51:26.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8816999999999998e-05 2023-10-09T10:51:26.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791894+0000", 2023-10-09T10:51:26.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018764160000000001 2023-10-09T10:51:26.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791933+0000", 2023-10-09T10:51:26.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8889e-05 2023-10-09T10:51:26.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792309+0000", 2023-10-09T10:51:26.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00037573100000000003 2023-10-09T10:51:26.295 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.296 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.296 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.296 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43116 6.11 6:8866ecf5:::benchmark_data_smithi125_244951_object43115:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.296 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550285+0000", 2023-10-09T10:51:26.296 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.66926874299998, 2023-10-09T10:51:26.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2425320870000001, 2023-10-09T10:51:26.296 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.296 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.296 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.296 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.297 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.297 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43116 2023-10-09T10:51:26.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.297 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550285+0000", 2023-10-09T10:51:26.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550285+0000", 2023-10-09T10:51:26.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550284+0000", 2023-10-09T10:51:26.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:26.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550303+0000", 2023-10-09T10:51:26.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8817e-05 2023-10-09T10:51:26.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550310+0000", 2023-10-09T10:51:26.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7889999999999997e-06 2023-10-09T10:51:26.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550315+0000", 2023-10-09T10:51:26.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4539999999999998e-06 2023-10-09T10:51:26.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551718+0000", 2023-10-09T10:51:26.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001402585 2023-10-09T10:51:26.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551725+0000", 2023-10-09T10:51:26.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9319999999999999e-06 2023-10-09T10:51:26.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786811+0000", 2023-10-09T10:51:26.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.23508555 2023-10-09T10:51:26.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786947+0000", 2023-10-09T10:51:26.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013601499999999999 2023-10-09T10:51:26.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.787112+0000", 2023-10-09T10:51:26.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016519000000000001 2023-10-09T10:51:26.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789147+0000", 2023-10-09T10:51:26.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020350899999999998 2023-10-09T10:51:26.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790634+0000", 2023-10-09T10:51:26.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014872209999999999 2023-10-09T10:51:26.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790670+0000", 2023-10-09T10:51:26.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6395e-05 2023-10-09T10:51:26.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792355+0000", 2023-10-09T10:51:26.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016850179999999999 2023-10-09T10:51:26.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792391+0000", 2023-10-09T10:51:26.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5484000000000001e-05 2023-10-09T10:51:26.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792817+0000", 2023-10-09T10:51:26.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042605300000000002 2023-10-09T10:51:26.304 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.304 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.304 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.304 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43152 6.11 6:8f5a8f11:::benchmark_data_smithi125_244951_object43151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.304 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:51:26.304 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.66923240099999, 2023-10-09T10:51:26.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.254750676, 2023-10-09T10:51:26.304 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.304 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.305 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.305 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.305 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.305 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43152 2023-10-09T10:51:26.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.305 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:51:26.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:51:26.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:51:26.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:26.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550339+0000", 2023-10-09T10:51:26.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7940000000000001e-05 2023-10-09T10:51:26.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550346+0000", 2023-10-09T10:51:26.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4789999999999999e-06 2023-10-09T10:51:26.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550353+0000", 2023-10-09T10:51:26.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7510000000000002e-06 2023-10-09T10:51:26.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551748+0000", 2023-10-09T10:51:26.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013946119999999999 2023-10-09T10:51:26.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551754+0000", 2023-10-09T10:51:26.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8460000000000002e-06 2023-10-09T10:51:26.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792855+0000", 2023-10-09T10:51:26.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.241101091 2023-10-09T10:51:26.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792993+0000", 2023-10-09T10:51:26.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013794 2023-10-09T10:51:26.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.793143+0000", 2023-10-09T10:51:26.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014960799999999999 2023-10-09T10:51:26.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797271+0000", 2023-10-09T10:51:26.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041283810000000004 2023-10-09T10:51:26.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797282+0000", 2023-10-09T10:51:26.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0543999999999999e-05 2023-10-09T10:51:26.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799296+0000", 2023-10-09T10:51:26.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020138999999999999 2023-10-09T10:51:26.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803962+0000", 2023-10-09T10:51:26.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046658840000000004 2023-10-09T10:51:26.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.804015+0000", 2023-10-09T10:51:26.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3229999999999997e-05 2023-10-09T10:51:26.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805072+0000", 2023-10-09T10:51:26.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010569690000000001 2023-10-09T10:51:26.312 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.312 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.312 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.312 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43224 6.11 6:8b0bc017:::benchmark_data_smithi125_244951_object43223:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.312 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550374+0000", 2023-10-09T10:51:26.312 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.66918001300002, 2023-10-09T10:51:26.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2556577980000001, 2023-10-09T10:51:26.312 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.312 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.313 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.313 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.313 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.313 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43224 2023-10-09T10:51:26.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.313 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550374+0000", 2023-10-09T10:51:26.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550374+0000", 2023-10-09T10:51:26.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550373+0000", 2023-10-09T10:51:26.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:26.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550388+0000", 2023-10-09T10:51:26.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.453e-05 2023-10-09T10:51:26.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550393+0000", 2023-10-09T10:51:26.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7119999999999999e-06 2023-10-09T10:51:26.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550397+0000", 2023-10-09T10:51:26.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6789999999999996e-06 2023-10-09T10:51:26.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551777+0000", 2023-10-09T10:51:26.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001380196 2023-10-09T10:51:26.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551784+0000", 2023-10-09T10:51:26.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0369999999999997e-06 2023-10-09T10:51:26.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.793904+0000", 2023-10-09T10:51:26.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.242119346 2023-10-09T10:51:26.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.794042+0000", 2023-10-09T10:51:26.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013829700000000001 2023-10-09T10:51:26.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.794190+0000", 2023-10-09T10:51:26.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014784600000000001 2023-10-09T10:51:26.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797289+0000", 2023-10-09T10:51:26.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003099152 2023-10-09T10:51:26.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799339+0000", 2023-10-09T10:51:26.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020501059999999999 2023-10-09T10:51:26.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803853+0000", 2023-10-09T10:51:26.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045136179999999996 2023-10-09T10:51:26.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805211+0000", 2023-10-09T10:51:26.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001358349 2023-10-09T10:51:26.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805266+0000", 2023-10-09T10:51:26.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5297999999999998e-05 2023-10-09T10:51:26.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806031+0000", 2023-10-09T10:51:26.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00076497100000000003 2023-10-09T10:51:26.320 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.320 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.320 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.320 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43255 6.11 6:88dd5493:::benchmark_data_smithi125_244951_object43254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.320 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:51:26.321 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.669153494, 2023-10-09T10:51:26.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2563999859999999, 2023-10-09T10:51:26.321 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.321 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.321 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.321 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.321 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.321 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43255 2023-10-09T10:51:26.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.321 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:51:26.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:51:26.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:51:26.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:26.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550413+0000", 2023-10-09T10:51:26.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3052e-05 2023-10-09T10:51:26.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550417+0000", 2023-10-09T10:51:26.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5999999999999998e-06 2023-10-09T10:51:26.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550420+0000", 2023-10-09T10:51:26.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6519999999999999e-06 2023-10-09T10:51:26.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.324 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551807+0000", 2023-10-09T10:51:26.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001386998 2023-10-09T10:51:26.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.324 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551814+0000", 2023-10-09T10:51:26.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3960000000000003e-06 2023-10-09T10:51:26.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.794932+0000", 2023-10-09T10:51:26.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2431179880000001 2023-10-09T10:51:26.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.795097+0000", 2023-10-09T10:51:26.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000165248 2023-10-09T10:51:26.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.795276+0000", 2023-10-09T10:51:26.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017957699999999999 2023-10-09T10:51:26.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799383+0000", 2023-10-09T10:51:26.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004106569 2023-10-09T10:51:26.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801446+0000", 2023-10-09T10:51:26.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020628370000000001 2023-10-09T10:51:26.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803918+0000", 2023-10-09T10:51:26.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024717239999999998 2023-10-09T10:51:26.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806076+0000", 2023-10-09T10:51:26.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002158395 2023-10-09T10:51:26.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806122+0000", 2023-10-09T10:51:26.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5807999999999999e-05 2023-10-09T10:51:26.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806800+0000", 2023-10-09T10:51:26.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00067843299999999999 2023-10-09T10:51:26.328 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.328 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.328 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.329 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43315 6.11 6:8fb75c47:::benchmark_data_smithi125_244951_object43314:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.329 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550436+0000", 2023-10-09T10:51:26.329 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.66911800000003, 2023-10-09T10:51:26.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2571070289999999, 2023-10-09T10:51:26.329 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.329 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.329 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.329 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.329 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.329 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43315 2023-10-09T10:51:26.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.330 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.330 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550436+0000", 2023-10-09T10:51:26.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.330 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550436+0000", 2023-10-09T10:51:26.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550435+0000", 2023-10-09T10:51:26.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:26.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550453+0000", 2023-10-09T10:51:26.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7504999999999999e-05 2023-10-09T10:51:26.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550460+0000", 2023-10-09T10:51:26.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2470000000000002e-06 2023-10-09T10:51:26.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550466+0000", 2023-10-09T10:51:26.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8699999999999997e-06 2023-10-09T10:51:26.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551836+0000", 2023-10-09T10:51:26.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001370664 2023-10-09T10:51:26.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551843+0000", 2023-10-09T10:51:26.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5320000000000003e-06 2023-10-09T10:51:26.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.796128+0000", 2023-10-09T10:51:26.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2442850169999999 2023-10-09T10:51:26.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.796286+0000", 2023-10-09T10:51:26.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015828500000000001 2023-10-09T10:51:26.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.796448+0000", 2023-10-09T10:51:26.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016164200000000001 2023-10-09T10:51:26.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801434+0000", 2023-10-09T10:51:26.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049856279999999998 2023-10-09T10:51:26.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.335 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801530+0000", 2023-10-09T10:51:26.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.645e-05 2023-10-09T10:51:26.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.335 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805133+0000", 2023-10-09T10:51:26.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036025850000000002 2023-10-09T10:51:26.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806935+0000", 2023-10-09T10:51:26.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018020410000000001 2023-10-09T10:51:26.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.807002+0000", 2023-10-09T10:51:26.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6883000000000003e-05 2023-10-09T10:51:26.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.807543+0000", 2023-10-09T10:51:26.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000541115 2023-10-09T10:51:26.337 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.337 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.337 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.337 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43318 6.11 6:8f0da7e4:::benchmark_data_smithi125_244951_object43317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.337 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550471+0000", 2023-10-09T10:51:26.337 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.66908312200002, 2023-10-09T10:51:26.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2582329400000001, 2023-10-09T10:51:26.337 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.337 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.338 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.338 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.338 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.338 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43318 2023-10-09T10:51:26.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.338 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550471+0000", 2023-10-09T10:51:26.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550471+0000", 2023-10-09T10:51:26.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550470+0000", 2023-10-09T10:51:26.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:26.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550518+0000", 2023-10-09T10:51:26.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8108000000000001e-05 2023-10-09T10:51:26.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550526+0000", 2023-10-09T10:51:26.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0199999999999994e-06 2023-10-09T10:51:26.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550532+0000", 2023-10-09T10:51:26.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9109999999999998e-06 2023-10-09T10:51:26.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551864+0000", 2023-10-09T10:51:26.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001332317 2023-10-09T10:51:26.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551870+0000", 2023-10-09T10:51:26.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6489999999999996e-06 2023-10-09T10:51:26.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797329+0000", 2023-10-09T10:51:26.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.245458701 2023-10-09T10:51:26.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797476+0000", 2023-10-09T10:51:26.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014769199999999999 2023-10-09T10:51:26.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797644+0000", 2023-10-09T10:51:26.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000167481 2023-10-09T10:51:26.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801478+0000", 2023-10-09T10:51:26.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038341780000000002 2023-10-09T10:51:26.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801537+0000", 2023-10-09T10:51:26.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8879000000000003e-05 2023-10-09T10:51:26.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806859+0000", 2023-10-09T10:51:26.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0053216640000000003 2023-10-09T10:51:26.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.344 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.344 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.807979+0000", 2023-10-09T10:51:26.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011203770000000001 2023-10-09T10:51:26.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.344 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.344 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808056+0000", 2023-10-09T10:51:26.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6568000000000003e-05 2023-10-09T10:51:26.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808703+0000", 2023-10-09T10:51:26.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00064785600000000004 2023-10-09T10:51:26.345 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.345 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.345 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.345 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43334 6.11 6:8b093292:::benchmark_data_smithi125_244951_object43333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.345 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:51:26.345 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.66901545299999, 2023-10-09T10:51:26.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.266648716, 2023-10-09T10:51:26.345 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.346 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.346 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.346 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.346 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.346 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43334 2023-10-09T10:51:26.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.346 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:51:26.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:51:26.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:51:26.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:26.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550579+0000", 2023-10-09T10:51:26.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1733000000000002e-05 2023-10-09T10:51:26.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550587+0000", 2023-10-09T10:51:26.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6260000000000003e-06 2023-10-09T10:51:26.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550592+0000", 2023-10-09T10:51:26.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4160000000000003e-06 2023-10-09T10:51:26.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551890+0000", 2023-10-09T10:51:26.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001297702 2023-10-09T10:51:26.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551896+0000", 2023-10-09T10:51:26.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4559999999999999e-06 2023-10-09T10:51:26.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.798375+0000", 2023-10-09T10:51:26.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2464794800000001 2023-10-09T10:51:26.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.798514+0000", 2023-10-09T10:51:26.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000139159 2023-10-09T10:51:26.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.798661+0000", 2023-10-09T10:51:26.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000146312 2023-10-09T10:51:26.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801519+0000", 2023-10-09T10:51:26.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028583720000000001 2023-10-09T10:51:26.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803865+0000", 2023-10-09T10:51:26.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002345752 2023-10-09T10:51:26.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.352 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.352 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808789+0000", 2023-10-09T10:51:26.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049239790000000002 2023-10-09T10:51:26.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.352 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.352 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.816450+0000", 2023-10-09T10:51:26.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076614550000000002 2023-10-09T10:51:26.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.352 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.352 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.816501+0000", 2023-10-09T10:51:26.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1298000000000003e-05 2023-10-09T10:51:26.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.353 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.353 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817187+0000", 2023-10-09T10:51:26.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00068546500000000001 2023-10-09T10:51:26.353 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.353 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.353 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.353 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43385 6.11 6:8e6a343d:::benchmark_data_smithi125_244951_object43384:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:51:26.353 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.636508+0000", 2023-10-09T10:51:26.354 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.583045545, 2023-10-09T10:51:26.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1811192319999999, 2023-10-09T10:51:26.354 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.354 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.354 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.354 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.354 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.354 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43385 2023-10-09T10:51:26.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.354 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636508+0000", 2023-10-09T10:51:26.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636508+0000", 2023-10-09T10:51:26.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636507+0000", 2023-10-09T10:51:26.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:26.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636527+0000", 2023-10-09T10:51:26.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9748000000000001e-05 2023-10-09T10:51:26.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636539+0000", 2023-10-09T10:51:26.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2197e-05 2023-10-09T10:51:26.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636545+0000", 2023-10-09T10:51:26.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1999999999999999e-06 2023-10-09T10:51:26.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636582+0000", 2023-10-09T10:51:26.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6791000000000001e-05 2023-10-09T10:51:26.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636590+0000", 2023-10-09T10:51:26.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4309999999999994e-06 2023-10-09T10:51:26.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799411+0000", 2023-10-09T10:51:26.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1628212659999999 2023-10-09T10:51:26.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799550+0000", 2023-10-09T10:51:26.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013822300000000001 2023-10-09T10:51:26.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:51:26.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799660+0000", 2023-10-09T10:51:26.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110701 2023-10-09T10:51:26.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801589+0000", 2023-10-09T10:51:26.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019285470000000001 2023-10-09T10:51:26.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803871+0000", 2023-10-09T10:51:26.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022818180000000001 2023-10-09T10:51:26.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808835+0000", 2023-10-09T10:51:26.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049646200000000003 2023-10-09T10:51:26.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817231+0000", 2023-10-09T10:51:26.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083956959999999994 2023-10-09T10:51:26.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817263+0000", 2023-10-09T10:51:26.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1899000000000001e-05 2023-10-09T10:51:26.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817627+0000", 2023-10-09T10:51:26.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00036434500000000002 2023-10-09T10:51:26.361 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.361 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.361 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.362 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106397 6.1 6:856288a5:::benchmark_data_smithi125_244951_object106396:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:51:26.362 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.204308+0000", 2023-10-09T10:51:26.362 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 324.01524553100001, 2023-10-09T10:51:26.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.075713931999999998, 2023-10-09T10:51:26.362 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.362 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.362 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.362 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.362 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.362 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106397 2023-10-09T10:51:26.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.362 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204308+0000", 2023-10-09T10:51:26.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204308+0000", 2023-10-09T10:51:26.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204307+0000", 2023-10-09T10:51:26.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:51:26.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204330+0000", 2023-10-09T10:51:26.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2852000000000001e-05 2023-10-09T10:51:26.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204340+0000", 2023-10-09T10:51:26.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3989999999999993e-06 2023-10-09T10:51:26.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.204346+0000", 2023-10-09T10:51:26.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5989999999999998e-06 2023-10-09T10:51:26.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.207434+0000", 2023-10-09T10:51:26.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030874600000000002 2023-10-09T10:51:26.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227524+0000", 2023-10-09T10:51:26.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.020090545000000001 2023-10-09T10:51:26.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:51:26.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227667+0000", 2023-10-09T10:51:26.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014324699999999999 2023-10-09T10:51:26.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.233515+0000", 2023-10-09T10:51:26.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0058477219999999996 2023-10-09T10:51:26.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.234252+0000", 2023-10-09T10:51:26.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00073691800000000003 2023-10-09T10:51:26.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279940+0000", 2023-10-09T10:51:26.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.045688232000000002 2023-10-09T10:51:26.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279985+0000", 2023-10-09T10:51:26.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4916000000000002e-05 2023-10-09T10:51:26.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.280022+0000", 2023-10-09T10:51:26.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6871e-05 2023-10-09T10:51:26.368 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.368 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.368 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.369 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106414 6.11 6:8c9e6868:::benchmark_data_smithi125_244951_object106413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:51:26.369 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.227135+0000", 2023-10-09T10:51:26.369 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 323.99241884200001, 2023-10-09T10:51:26.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.37275067699999997, 2023-10-09T10:51:26.369 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.369 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:51:26.369 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.369 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.369 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.369 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106414 2023-10-09T10:51:26.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.370 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227135+0000", 2023-10-09T10:51:26.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227135+0000", 2023-10-09T10:51:26.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227134+0000", 2023-10-09T10:51:26.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:26.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227156+0000", 2023-10-09T10:51:26.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2059999999999999e-05 2023-10-09T10:51:26.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227163+0000", 2023-10-09T10:51:26.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4169999999999997e-06 2023-10-09T10:51:26.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227168+0000", 2023-10-09T10:51:26.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9509999999999997e-06 2023-10-09T10:51:26.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.229108+0000", 2023-10-09T10:51:26.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001940396 2023-10-09T10:51:26.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.229279+0000", 2023-10-09T10:51:26.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017114100000000001 2023-10-09T10:51:26.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T10:51:26.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.229487+0000", 2023-10-09T10:51:26.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000207977 2023-10-09T10:51:26.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.233575+0000", 2023-10-09T10:51:26.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040875140000000004 2023-10-09T10:51:26.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.240998+0000", 2023-10-09T10:51:26.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074230249999999998 2023-10-09T10:51:26.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.245203+0000", 2023-10-09T10:51:26.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042052039999999997 2023-10-09T10:51:26.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274084+0000", 2023-10-09T10:51:26.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.028880626999999999 2023-10-09T10:51:26.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274097+0000", 2023-10-09T10:51:26.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.344e-05 2023-10-09T10:51:26.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599869+0000", 2023-10-09T10:51:26.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32577187699999999 2023-10-09T10:51:26.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599885+0000", 2023-10-09T10:51:26.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6515e-05 2023-10-09T10:51:26.376 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.376 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.376 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.376 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106433 6.11 6:8d28bd42:::benchmark_data_smithi125_244951_object106432:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:51:26.376 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.251731+0000", 2023-10-09T10:51:26.377 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 323.96782237399998, 2023-10-09T10:51:26.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.34819989800000001, 2023-10-09T10:51:26.377 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.377 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:51:26.377 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.377 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.377 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.377 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106433 2023-10-09T10:51:26.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.377 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.377 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251731+0000", 2023-10-09T10:51:26.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251731+0000", 2023-10-09T10:51:26.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251729+0000", 2023-10-09T10:51:26.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:51:26.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.379 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.379 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251758+0000", 2023-10-09T10:51:26.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8909000000000001e-05 2023-10-09T10:51:26.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.379 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.379 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251780+0000", 2023-10-09T10:51:26.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2011999999999999e-05 2023-10-09T10:51:26.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.379 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.251791+0000", 2023-10-09T10:51:26.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0147e-05 2023-10-09T10:51:26.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.253110+0000", 2023-10-09T10:51:26.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013193739999999999 2023-10-09T10:51:26.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.253287+0000", 2023-10-09T10:51:26.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017752800000000001 2023-10-09T10:51:26.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T10:51:26.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.253484+0000", 2023-10-09T10:51:26.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000196622 2023-10-09T10:51:26.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.257296+0000", 2023-10-09T10:51:26.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038119550000000001 2023-10-09T10:51:26.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.257345+0000", 2023-10-09T10:51:26.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8745000000000002e-05 2023-10-09T10:51:26.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279630+0000", 2023-10-09T10:51:26.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.02228538 2023-10-09T10:51:26.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.279642+0000", 2023-10-09T10:51:26.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2328e-05 2023-10-09T10:51:26.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599919+0000", 2023-10-09T10:51:26.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32027628800000002 2023-10-09T10:51:26.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599931+0000", 2023-10-09T10:51:26.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2439000000000001e-05 2023-10-09T10:51:26.383 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.383 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.383 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.384 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.384 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:136053 6.1 6:832e3320:::benchmark_data_smithi125_244951_object29610:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-10-09T10:51:26.384 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:47:02.144996+0000", 2023-10-09T10:51:26.384 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 264.07455796199997, 2023-10-09T10:51:26.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.155053459, 2023-10-09T10:51:26.384 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:26.384 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:51:26.384 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:26.384 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:51:26.384 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:51:26.384 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 136053 2023-10-09T10:51:26.384 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.385 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:26.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:26.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.144996+0000", 2023-10-09T10:51:26.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:26.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.144996+0000", 2023-10-09T10:51:26.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:26.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:26.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.144994+0000", 2023-10-09T10:51:26.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:26.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:26.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145002+0000", 2023-10-09T10:51:26.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8429999999999993e-06 2023-10-09T10:51:26.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:26.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145014+0000", 2023-10-09T10:51:26.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2163e-05 2023-10-09T10:51:26.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:26.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145024+0000", 2023-10-09T10:51:26.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7610000000000002e-06 2023-10-09T10:51:26.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145083+0000", 2023-10-09T10:51:26.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8882999999999998e-05 2023-10-09T10:51:26.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:51:26.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.145095+0000", 2023-10-09T10:51:26.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2255999999999999e-05 2023-10-09T10:51:26.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:26.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.287386+0000", 2023-10-09T10:51:26.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.14229140700000001 2023-10-09T10:51:26.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:26.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.287495+0000", 2023-10-09T10:51:26.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108651 2023-10-09T10:51:26.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 6,7", 2023-10-09T10:51:26.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.287593+0000", 2023-10-09T10:51:26.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8348000000000001e-05 2023-10-09T10:51:26.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.295087+0000", 2023-10-09T10:51:26.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074939550000000001 2023-10-09T10:51:26.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:26.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.295761+0000", 2023-10-09T10:51:26.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00067362100000000001 2023-10-09T10:51:26.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:26.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.299957+0000", 2023-10-09T10:51:26.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041957419999999997 2023-10-09T10:51:26.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:51:26.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.300009+0000", 2023-10-09T10:51:26.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2193e-05 2023-10-09T10:51:26.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:26.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:26.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:51:26.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:47:02.300049+0000", 2023-10-09T10:51:26.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0156e-05 2023-10-09T10:51:26.391 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.391 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.391 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.391 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:26.391 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:26.392 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:51:26.392 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:51:26.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:26.393 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:51:26.393 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:51:26.394 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:51:26.394 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:51:26.394 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:51:26.394 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:51:26.394 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:51:26.394 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:51:26.394 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:51:26.394 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:51:26.395 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:26.395 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:26.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:26.495 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:26.557 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T10:51:26.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:26 smithi125 ceph-mon[195507]: pgmap v543: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 735 op/s; 20907/207648 objects degraded (10.068%); 2672/207648 objects misplaced (1.287%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T10:51:26.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:26 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2116006999' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:26.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:26 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/628623603' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T10:51:26.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:26 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T10:51:26.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:26 smithi125 ceph-mon[200197]: pgmap v543: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 735 op/s; 20907/207648 objects degraded (10.068%); 2672/207648 objects misplaced (1.287%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T10:51:26.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:26 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2116006999' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:26.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:26 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/628623603' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T10:51:26.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:26 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T10:51:26.628 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:26 smithi155 ceph-mon[184854]: pgmap v543: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 735 op/s; 20907/207648 objects degraded (10.068%); 2672/207648 objects misplaced (1.287%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T10:51:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:26 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2116006999' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:26 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/628623603' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T10:51:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:26 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T10:51:26.769 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:26.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:26.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:26.874 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:26.932 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:26.966 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:26.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:26.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:27.072 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:27.113 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:27.145 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:27.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:27.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:27.257 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:27.312 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:27.346 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:27.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:27.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:27.450 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:27.489 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:27.534 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:27.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:27.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:27.637 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:27.689 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:27.717 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:27.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:27.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:27.825 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:27.870 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:27.904 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:27.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:27.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:28.011 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:28.054 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:28.096 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:28.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:28.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:28.208 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:28.254 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:28.291 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:28.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:28.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:28.394 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:28.439 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:28.468 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:28.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:28.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:28.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:28.625 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:28.663 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:28.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:28.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:28.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:28 smithi155 ceph-mon[184854]: pgmap v544: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 747 op/s; 20907/204789 objects degraded (10.209%); 2672/204789 objects misplaced (1.305%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T10:51:28.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:28 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 20907/204789 objects degraded (10.209%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:28.769 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:28.813 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:28 smithi125 ceph-mon[195507]: pgmap v544: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 747 op/s; 20907/204789 objects degraded (10.209%); 2672/204789 objects misplaced (1.305%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T10:51:28.813 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:28 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 20907/204789 objects degraded (10.209%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:28.814 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:28 smithi125 ceph-mon[200197]: pgmap v544: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 747 op/s; 20907/204789 objects degraded (10.209%); 2672/204789 objects misplaced (1.305%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T10:51:28.814 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:28 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 20907/204789 objects degraded (10.209%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:28.817 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:28.949 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:51:28.957 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:28.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:28.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:29.028 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:29.068 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:29.107 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:29.136 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:29.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:29.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:29.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:29.292 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:29.638 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:29.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:29.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:29.743 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:29.803 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:29.840 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:29.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:29.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:29.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:29.993 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:30.006 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:30.034 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:30.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:30.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:30.144 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:30.221 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:30.253 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:30.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:30.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:30.359 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:30.437 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:30.557 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:30 smithi125 ceph-mon[195507]: pgmap v545: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 768 op/s; 20123/199329 objects degraded (10.095%); 2672/199329 objects misplaced (1.340%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T10:51:30.557 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:30.557 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:51:30.558 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:30 smithi125 ceph-mon[200197]: pgmap v545: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 768 op/s; 20123/199329 objects degraded (10.095%); 2672/199329 objects misplaced (1.340%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T10:51:30.558 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:30.558 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:51:30.692 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:51:30.778 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:30.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:30.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:30.885 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:30 smithi155 ceph-mon[184854]: pgmap v545: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 768 op/s; 20123/199329 objects degraded (10.095%); 2672/199329 objects misplaced (1.340%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T10:51:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:51:30.979 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:31.079 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:31.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:31.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:31.184 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:31.193 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:31.217 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:51:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:51:31] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T10:51:31.261 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:31.339 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:31.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:31.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:31.443 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:31.479 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2633331186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:51:31.514 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:31.526 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:51:31.552 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:31.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:31.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:31.558 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0] out_osds: [2, 7, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:51:31.558 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:51:31.558 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-10-09T10:51:31.558 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:51:31.558 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:51:31.559 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-10-09T10:51:31.658 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:31.664 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:51:31.665 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:51:31.665 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 0 2023-10-09T10:51:31.665 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:51:31.727 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:31.802 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:31.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:31.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:31.808 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2633331186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:51:31.909 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2633331186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:51:31.975 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:32.155 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:32.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:32.236 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:51:32.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:32.338 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:32.402 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:32.490 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:32.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:32.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:32.597 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:32.642 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:32.684 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:32.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:32.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:32.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:32 smithi125 ceph-mon[195507]: pgmap v546: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 782 op/s; 20123/194676 objects degraded (10.337%); 2141/194676 objects misplaced (1.100%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T10:51:32.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:32 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:32.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:32 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2633331186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:51:32.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:32 smithi125 ceph-mon[195507]: osdmap e237: 8 total, 8 up, 5 in 2023-10-09T10:51:32.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:32 smithi125 ceph-mon[195507]: failed to encode map e237 with expected crc 2023-10-09T10:51:32.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:32 smithi125 ceph-mon[200197]: pgmap v546: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 782 op/s; 20123/194676 objects degraded (10.337%); 2141/194676 objects misplaced (1.100%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T10:51:32.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:32 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:32.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:32 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2633331186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:51:32.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:32 smithi125 ceph-mon[200197]: osdmap e237: 8 total, 8 up, 5 in 2023-10-09T10:51:32.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:32 smithi125 ceph-mon[200197]: failed to encode map e237 with expected crc 2023-10-09T10:51:32.773 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:32.794 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:32.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:32 smithi155 ceph-mon[184854]: pgmap v546: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 782 op/s; 20123/194676 objects degraded (10.337%); 2141/194676 objects misplaced (1.100%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T10:51:32.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:32 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:32.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:32 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2633331186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:51:32.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:32 smithi155 ceph-mon[184854]: osdmap e237: 8 total, 8 up, 5 in 2023-10-09T10:51:32.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:32 smithi155 ceph-mon[184854]: failed to encode map e237 with expected crc 2023-10-09T10:51:32.836 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:32.872 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:32.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:32.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:32.978 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:33.021 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:33.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:33 smithi125 ceph-mon[195507]: failed to encode map e237 with expected crc 2023-10-09T10:51:33.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:33 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 20123/194676 objects degraded (10.337%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:33.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:33 smithi125 ceph-mon[200197]: failed to encode map e237 with expected crc 2023-10-09T10:51:33.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:33 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 20123/194676 objects degraded (10.337%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:33.965 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:33.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:33 smithi155 ceph-mon[184854]: failed to encode map e237 with expected crc 2023-10-09T10:51:33.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:33 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 20123/194676 objects degraded (10.337%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:34.099 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:51:34.136 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:51:34.175 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:51:34.223 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:51:34.257 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:51:34.297 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:51:34.328 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:51:34.330 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:34.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:34.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:34.355 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:51:34.360 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:51:34.360 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:51:34.360 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:51:34.360 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:51:34.360 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:51:34.360 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:51:34.360 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:51:34.360 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:51:34.384 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:51:34.450 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:34.511 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:34.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:34 smithi125 ceph-mon[195507]: pgmap v548: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 813 op/s; 19419/187260 objects degraded (10.370%); 2141/187260 objects misplaced (1.143%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T10:51:34.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:34 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3575649375' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:34.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:34 smithi125 ceph-mon[200197]: pgmap v548: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 813 op/s; 19419/187260 objects degraded (10.370%); 2141/187260 objects misplaced (1.143%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T10:51:34.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:34 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3575649375' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:34 smithi155 ceph-mon[184854]: pgmap v548: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 813 op/s; 19419/187260 objects degraded (10.370%); 2141/187260 objects misplaced (1.143%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T10:51:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:34 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3575649375' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:36.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:36 smithi125 ceph-mon[195507]: pgmap v549: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 799 op/s; 19419/183642 objects degraded (10.574%); 1621/183642 objects misplaced (0.883%); 9.4 MiB/s, 151 objects/s recovering 2023-10-09T10:51:36.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:36 smithi125 ceph-mon[200197]: pgmap v549: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 799 op/s; 19419/183642 objects degraded (10.574%); 1621/183642 objects misplaced (0.883%); 9.4 MiB/s, 151 objects/s recovering 2023-10-09T10:51:36.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:36 smithi155 ceph-mon[184854]: pgmap v549: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 799 op/s; 19419/183642 objects degraded (10.574%); 1621/183642 objects misplaced (0.883%); 9.4 MiB/s, 151 objects/s recovering 2023-10-09T10:51:37.057 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:51:37.057 INFO:teuthology.orchestra.run.smithi125.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T10:51:37.057 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:51:37.712 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:37.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:37.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:37.818 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:37.872 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:37.903 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:37.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:37.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:38.010 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:38.016 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:38.121 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:38.207 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:38.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:38.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:38.317 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:38.383 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:38.518 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:38.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:38.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:38.625 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:38 smithi125 ceph-mon[195507]: pgmap v550: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 807 op/s; 19419/180558 objects degraded (10.755%); 1621/180558 objects misplaced (0.898%); 9.4 MiB/s, 151 objects/s recovering 2023-10-09T10:51:38.625 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:38 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 19419/180558 objects degraded (10.755%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:38.625 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:38 smithi125 ceph-mon[200197]: pgmap v550: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 807 op/s; 19419/180558 objects degraded (10.755%); 1621/180558 objects misplaced (0.898%); 9.4 MiB/s, 151 objects/s recovering 2023-10-09T10:51:38.625 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:38 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 19419/180558 objects degraded (10.755%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:38.628 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:38.634 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:51:38.634 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:51:38.696 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:38.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:38 smithi155 ceph-mon[184854]: pgmap v550: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 807 op/s; 19419/180558 objects degraded (10.755%); 1621/180558 objects misplaced (0.898%); 9.4 MiB/s, 151 objects/s recovering 2023-10-09T10:51:38.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:38 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 19419/180558 objects degraded (10.755%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:39.286 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:39.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:39.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:39.346 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:51:39.391 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:39.473 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:39.556 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:39.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:39.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:39.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:51:39.663 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:39.700 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:39.724 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:39.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:39.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:39.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:39 smithi125 ceph-mon[195507]: pgmap v551: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 823 op/s; 18556/174606 objects degraded (10.627%); 1621/174606 objects misplaced (0.928%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T10:51:39.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:39 smithi125 ceph-mon[200197]: pgmap v551: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 823 op/s; 18556/174606 objects degraded (10.627%); 1621/174606 objects misplaced (0.928%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T10:51:39.823 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:39.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:39.915 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:39.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:39 smithi155 ceph-mon[184854]: pgmap v551: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 823 op/s; 18556/174606 objects degraded (10.627%); 1621/174606 objects misplaced (0.928%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T10:51:40.002 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:40.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:40.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:40.108 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:40.155 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:40.340 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:40.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:40.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:40.370 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:40.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:40.493 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:40.571 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:40.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:40.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:40.609 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:51:40.683 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:40.745 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:40 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4179053540' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:40.745 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:40 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:40.746 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:40 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4179053540' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:40.746 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:40 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:40.746 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:40.869 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:40.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:40.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:40.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:40 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4179053540' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:40.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:40 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:40.972 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:41.011 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:41.060 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:41.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:41.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:41.166 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:41.237 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:41.277 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:41.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:41.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:41.387 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:41.465 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:51:41] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T10:51:41.474 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:41.560 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:51:41.560 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:51:41.569 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:41.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:41.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:41.674 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:41.719 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:41.719 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:51:41.719 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[195507]: osdmap e238: 8 total, 8 up, 5 in 2023-10-09T10:51:41.719 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[195507]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.719 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[195507]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.719 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[195507]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.720 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[195507]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.720 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[195507]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.720 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[195507]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.720 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[195507]: pgmap v553: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 786 op/s; 18556/169344 objects degraded (10.958%); 1055/169344 objects misplaced (0.623%); 10 MiB/s, 162 objects/s recovering 2023-10-09T10:51:41.720 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:41.720 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:51:41.720 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[200197]: osdmap e238: 8 total, 8 up, 5 in 2023-10-09T10:51:41.720 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[200197]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.720 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[200197]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.720 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[200197]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.721 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[200197]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.721 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[200197]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.721 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[200197]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.721 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:41 smithi125 ceph-mon[200197]: pgmap v553: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 786 op/s; 18556/169344 objects degraded (10.958%); 1055/169344 objects misplaced (0.623%); 10 MiB/s, 162 objects/s recovering 2023-10-09T10:51:41.724 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:41.924 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:41.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:41.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:41.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:41 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:41.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:41 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:51:41.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:41 smithi155 ceph-mon[184854]: osdmap e238: 8 total, 8 up, 5 in 2023-10-09T10:51:41.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:41 smithi155 ceph-mon[184854]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:41 smithi155 ceph-mon[184854]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:41 smithi155 ceph-mon[184854]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:41 smithi155 ceph-mon[184854]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.977 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:41 smithi155 ceph-mon[184854]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.977 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:41 smithi155 ceph-mon[184854]: failed to encode map e238 with expected crc 2023-10-09T10:51:41.977 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:41 smithi155 ceph-mon[184854]: pgmap v553: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 786 op/s; 18556/169344 objects degraded (10.958%); 1055/169344 objects misplaced (0.623%); 10 MiB/s, 162 objects/s recovering 2023-10-09T10:51:42.029 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:42.080 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:42.160 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:42.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:42.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:42.176 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:51:42.268 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:42.326 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:42.355 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:42.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:42.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:42.459 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:42.506 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:42.535 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:42.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:42.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:42.639 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:42.690 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:42.717 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:42.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:42.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:42.822 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:42.882 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:42 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 18556/169344 objects degraded (10.958%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:42.999 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0] out_osds: [2, 7, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:51:42.999 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:51:42.999 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:51:42.999 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.000000 2023-10-09T10:51:42.999 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 0 2023-10-09T10:51:43.216 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:51:43.216 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:51:43.236 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:51:43.262 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:43.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:43.269 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:42 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 18556/169344 objects degraded (10.958%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:43.269 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:42 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 18556/169344 objects degraded (10.958%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:43.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:43.327 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:43.327 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:51:43.364 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:51:43.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:43.386 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:51:43.396 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:51:43.407 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:43.437 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:51:43.437 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:43.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:43.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:43.467 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:51:43.487 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:51:43.487 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:51:43.487 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:51:43.487 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:51:43.487 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:51:43.487 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:51:43.487 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:51:43.487 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:51:43.521 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:43.544 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:43.571 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:43.590 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:43.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:43.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:43.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:43.732 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:44.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:43 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2546619346' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:44.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:43 smithi125 ceph-mon[195507]: pgmap v554: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 864 op/s; 17886/161286 objects degraded (11.090%); 1055/161286 objects misplaced (0.654%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T10:51:44.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:43 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2546619346' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:44.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:43 smithi125 ceph-mon[200197]: pgmap v554: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 864 op/s; 17886/161286 objects degraded (11.090%); 1055/161286 objects misplaced (0.654%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T10:51:44.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:43 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2546619346' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:44.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:43 smithi155 ceph-mon[184854]: pgmap v554: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 864 op/s; 17886/161286 objects degraded (11.090%); 1055/161286 objects misplaced (0.654%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T10:51:45.478 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:45.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:45.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:45.593 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:45.643 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:45.881 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:45.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:45.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:45.987 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:46.022 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:51:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:46 smithi125 ceph-mon[195507]: pgmap v555: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 881 op/s; 17886/157161 objects degraded (11.381%); 570/157161 objects misplaced (0.363%); 10 MiB/s, 166 objects/s recovering 2023-10-09T10:51:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:51:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:46 smithi125 ceph-mon[200197]: pgmap v555: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 881 op/s; 17886/157161 objects degraded (11.381%); 570/157161 objects misplaced (0.363%); 10 MiB/s, 166 objects/s recovering 2023-10-09T10:51:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:51:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:51:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:46 smithi155 ceph-mon[184854]: pgmap v555: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 881 op/s; 17886/157161 objects degraded (11.381%); 570/157161 objects misplaced (0.363%); 10 MiB/s, 166 objects/s recovering 2023-10-09T10:51:46.565 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:46.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:46.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:46.669 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:46.730 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:46.756 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:46.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:46.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:46.862 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:46.919 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:47.011 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:47.317 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:51:47.318 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:51:47.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.318 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:133902 7.1 7:8e44a344:::benchmark_data_smithi125_295816_object42765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e238)", 2023-10-09T10:51:47.318 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:47.311478+0000", 2023-10-09T10:51:47.318 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0056308549999999997, 2023-10-09T10:51:47.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.005653534, 2023-10-09T10:51:47.318 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:47.318 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:51:47.318 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:47.318 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:51:47.319 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:51:47.319 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133902 2023-10-09T10:51:47.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.319 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:47.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:47.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311478+0000", 2023-10-09T10:51:47.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:47.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311478+0000", 2023-10-09T10:51:47.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:47.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311476+0000", 2023-10-09T10:51:47.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:47.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:47.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311483+0000", 2023-10-09T10:51:47.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2960000000000004e-06 2023-10-09T10:51:47.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:47.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311495+0000", 2023-10-09T10:51:47.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2112e-05 2023-10-09T10:51:47.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:47.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311502+0000", 2023-10-09T10:51:47.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4710000000000001e-06 2023-10-09T10:51:47.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:47.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311542+0000", 2023-10-09T10:51:47.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9920999999999999e-05 2023-10-09T10:51:47.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:47.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311662+0000", 2023-10-09T10:51:47.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119746 2023-10-09T10:51:47.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:51:47.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311788+0000", 2023-10-09T10:51:47.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012591499999999999 2023-10-09T10:51:47.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:47.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.314421+0000", 2023-10-09T10:51:47.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002633481 2023-10-09T10:51:47.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:51:47.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.314748+0000", 2023-10-09T10:51:47.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00032652199999999999 2023-10-09T10:51:47.324 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.324 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:47.324 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.324 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:133903 7.a 7:59446cd4:::benchmark_data_smithi125_295816_object42766:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e238)", 2023-10-09T10:51:47.324 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:47.311515+0000", 2023-10-09T10:51:47.324 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0055936079999999999, 2023-10-09T10:51:47.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0056766840000000004, 2023-10-09T10:51:47.324 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:47.324 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:51:47.325 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:47.325 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:51:47.325 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:51:47.325 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133903 2023-10-09T10:51:47.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.325 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:47.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:47.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311515+0000", 2023-10-09T10:51:47.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:47.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311515+0000", 2023-10-09T10:51:47.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:47.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311514+0000", 2023-10-09T10:51:47.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:47.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:47.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311518+0000", 2023-10-09T10:51:47.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.275e-06 2023-10-09T10:51:47.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:47.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311525+0000", 2023-10-09T10:51:47.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1019999999999999e-06 2023-10-09T10:51:47.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:47.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311530+0000", 2023-10-09T10:51:47.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7379999999999999e-06 2023-10-09T10:51:47.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:47.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311574+0000", 2023-10-09T10:51:47.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3952000000000002e-05 2023-10-09T10:51:47.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:47.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311743+0000", 2023-10-09T10:51:47.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016850200000000001 2023-10-09T10:51:47.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,5", 2023-10-09T10:51:47.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.311939+0000", 2023-10-09T10:51:47.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019651599999999999 2023-10-09T10:51:47.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:51:47.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316133+0000", 2023-10-09T10:51:47.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041937229999999999 2023-10-09T10:51:47.329 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.329 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:47.329 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.330 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35187.0:133904 7.f e238/203 7:f2d62be9:::benchmark_data_smithi125_295816_object42767:head v 238'8387, mlcod=0'0)", 2023-10-09T10:51:47.330 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:47.314761+0000", 2023-10-09T10:51:47.330 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0023473769999999999, 2023-10-09T10:51:47.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024935320000000001, 2023-10-09T10:51:47.330 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:47.330 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:51:47.330 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:47.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.330 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:47.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.314761+0000", 2023-10-09T10:51:47.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:47.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.314761+0000", 2023-10-09T10:51:47.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:47.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.314761+0000", 2023-10-09T10:51:47.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:47.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:47.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.314766+0000", 2023-10-09T10:51:47.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5389999999999998e-06 2023-10-09T10:51:47.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:47.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.314775+0000", 2023-10-09T10:51:47.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3999999999999992e-06 2023-10-09T10:51:47.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:47.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.314780+0000", 2023-10-09T10:51:47.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4530000000000001e-06 2023-10-09T10:51:47.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:47.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.314827+0000", 2023-10-09T10:51:47.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6619000000000002e-05 2023-10-09T10:51:47.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:47.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.314849+0000", 2023-10-09T10:51:47.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2226999999999999e-05 2023-10-09T10:51:47.333 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.333 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:47.334 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.334 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35187.0:133901 7.0 e238/209 7:0a2afc1a:::benchmark_data_smithi125_295816_object42764:head v 238'8393, mlcod=0'0)", 2023-10-09T10:51:47.334 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:47.315236+0000", 2023-10-09T10:51:47.334 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.001872711, 2023-10-09T10:51:47.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020573940000000001, 2023-10-09T10:51:47.334 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:47.334 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:51:47.334 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:47.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:47.335 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.315236+0000", 2023-10-09T10:51:47.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:47.335 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.315236+0000", 2023-10-09T10:51:47.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:47.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.315235+0000", 2023-10-09T10:51:47.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:47.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:47.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.315242+0000", 2023-10-09T10:51:47.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3629999999999999e-06 2023-10-09T10:51:47.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:47.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.315250+0000", 2023-10-09T10:51:47.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7960000000000001e-06 2023-10-09T10:51:47.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:47.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.315257+0000", 2023-10-09T10:51:47.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5309999999999998e-06 2023-10-09T10:51:47.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:47.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.315319+0000", 2023-10-09T10:51:47.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1821000000000003e-05 2023-10-09T10:51:47.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:47.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.315341+0000", 2023-10-09T10:51:47.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2484e-05 2023-10-09T10:51:47.338 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.338 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:47.338 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.338 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:133905 7.1 7.72608e01 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e238)", 2023-10-09T10:51:47.339 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:47.316494+0000", 2023-10-09T10:51:47.339 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.00061502300000000002, 2023-10-09T10:51:47.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00084474799999999998, 2023-10-09T10:51:47.339 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:47.339 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:51:47.339 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:47.339 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:51:47.339 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:51:47.339 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133905 2023-10-09T10:51:47.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.339 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:47.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:47.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316494+0000", 2023-10-09T10:51:47.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:47.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316494+0000", 2023-10-09T10:51:47.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:47.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316492+0000", 2023-10-09T10:51:47.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:51:47.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:47.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316499+0000", 2023-10-09T10:51:47.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9380000000000003e-06 2023-10-09T10:51:47.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:47.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316513+0000", 2023-10-09T10:51:47.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4053000000000001e-05 2023-10-09T10:51:47.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:47.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316521+0000", 2023-10-09T10:51:47.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6890000000000004e-06 2023-10-09T10:51:47.342 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.342 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:47.342 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.342 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:133907 7.2 7:43c74b35:::benchmark_data_smithi125_295816_object42770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e238)", 2023-10-09T10:51:47.342 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:47.316536+0000", 2023-10-09T10:51:47.343 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.00057286900000000005, 2023-10-09T10:51:47.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00083817099999999997, 2023-10-09T10:51:47.343 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:47.343 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:51:47.343 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:47.343 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:51:47.343 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:51:47.343 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133907 2023-10-09T10:51:47.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.343 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:47.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:47.344 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316536+0000", 2023-10-09T10:51:47.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.344 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:47.344 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316536+0000", 2023-10-09T10:51:47.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.344 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:47.344 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316535+0000", 2023-10-09T10:51:47.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:47.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:47.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316538+0000", 2023-10-09T10:51:47.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.666e-06 2023-10-09T10:51:47.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:47.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316545+0000", 2023-10-09T10:51:47.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1579999999999999e-06 2023-10-09T10:51:47.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:47.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316550+0000", 2023-10-09T10:51:47.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8520000000000003e-06 2023-10-09T10:51:47.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:47.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316800+0000", 2023-10-09T10:51:47.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025051899999999999 2023-10-09T10:51:47.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:47.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316927+0000", 2023-10-09T10:51:47.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000126344 2023-10-09T10:51:47.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:51:47.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.317060+0000", 2023-10-09T10:51:47.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013330699999999999 2023-10-09T10:51:47.347 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.347 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:47.347 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.348 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:133909 7.c 7:37a25a7e:::benchmark_data_smithi125_295816_object42772:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e238)", 2023-10-09T10:51:47.348 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:47.316558+0000", 2023-10-09T10:51:47.348 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.00055041300000000003, 2023-10-09T10:51:47.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00086240299999999998, 2023-10-09T10:51:47.348 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:47.348 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:51:47.348 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:47.348 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:51:47.348 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:51:47.348 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133909 2023-10-09T10:51:47.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.349 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:47.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:47.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316558+0000", 2023-10-09T10:51:47.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:47.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316558+0000", 2023-10-09T10:51:47.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:47.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316558+0000", 2023-10-09T10:51:47.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:47.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:47.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316561+0000", 2023-10-09T10:51:47.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8839999999999998e-06 2023-10-09T10:51:47.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:47.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316572+0000", 2023-10-09T10:51:47.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0932e-05 2023-10-09T10:51:47.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:47.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316576+0000", 2023-10-09T10:51:47.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.639e-06 2023-10-09T10:51:47.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:51:47.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316639+0000", 2023-10-09T10:51:47.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2677999999999999e-05 2023-10-09T10:51:47.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.352 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:51:47.352 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316779+0000", 2023-10-09T10:51:47.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000140462 2023-10-09T10:51:47.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.352 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:51:47.352 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316897+0000", 2023-10-09T10:51:47.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000117488 2023-10-09T10:51:47.352 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.352 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:47.353 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.353 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:133910 7.2 7.547ce7f2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e238)", 2023-10-09T10:51:47.353 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:51:47.316587+0000", 2023-10-09T10:51:47.353 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.00052142099999999999, 2023-10-09T10:51:47.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00087692499999999997, 2023-10-09T10:51:47.353 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:51:47.353 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:51:47.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:51:47.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:51:47.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:51:47.354 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133910 2023-10-09T10:51:47.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.354 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:51:47.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:51:47.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316587+0000", 2023-10-09T10:51:47.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:51:47.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316587+0000", 2023-10-09T10:51:47.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:51:47.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:51:47.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316587+0000", 2023-10-09T10:51:47.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:51:47.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:51:47.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316589+0000", 2023-10-09T10:51:47.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2879999999999999e-06 2023-10-09T10:51:47.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:51:47.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316595+0000", 2023-10-09T10:51:47.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4229999999999996e-06 2023-10-09T10:51:47.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:51:47.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:51:47.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:51:47.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:51:47.316599+0000", 2023-10-09T10:51:47.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4530000000000004e-06 2023-10-09T10:51:47.356 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.356 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:51:47.356 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.357 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:51:47.357 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:51:47.357 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 8 2023-10-09T10:51:47.357 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:51:47.666 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:47.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:47.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:47.772 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:47.838 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:47.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:47.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:47.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:47.980 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:48.033 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:48.063 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:48.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:48.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:48.142 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:51:48.167 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:48.242 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:48.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:48.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:48.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:48.399 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:48.437 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:51:48.439 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:48.467 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:48.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:48.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:48.485 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:48 smithi125 ceph-mon[195507]: pgmap v556: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 902 op/s; 17886/153450 objects degraded (11.656%); 570/153450 objects misplaced (0.371%); 10 MiB/s, 166 objects/s recovering 2023-10-09T10:51:48.485 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:48 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 17886/153450 objects degraded (11.656%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:48.485 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:48 smithi125 ceph-mon[200197]: pgmap v556: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 902 op/s; 17886/153450 objects degraded (11.656%); 570/153450 objects misplaced (0.371%); 10 MiB/s, 166 objects/s recovering 2023-10-09T10:51:48.485 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:48 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 17886/153450 objects degraded (11.656%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:48.572 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:48.609 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:48 smithi155 ceph-mon[184854]: pgmap v556: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 902 op/s; 17886/153450 objects degraded (11.656%); 570/153450 objects misplaced (0.371%); 10 MiB/s, 166 objects/s recovering 2023-10-09T10:51:48.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:48 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 17886/153450 objects degraded (11.656%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:48.612 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:48.644 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:48.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:48.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:48.747 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:48.804 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:48.841 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:48.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:48.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:48.945 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:49.005 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:49.106 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:49.132 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:49.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:49.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:49.235 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:49.275 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:49.300 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:49.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:49.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:49.386 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.5 primary-affinity to 0 (802) 2023-10-09T10:51:49.404 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:49.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:49 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1095399478' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T10:51:49.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:49 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1095399478' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T10:51:49.450 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:49.575 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:49.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:49.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:49.686 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:49.715 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:49.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:49 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1095399478' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T10:51:49.886 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:49.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:49.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:49.926 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:49.991 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:50.020 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:50.190 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:50.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:50.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:50.293 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:50.322 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:50.367 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:51:50.392 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:50.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:50.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:50.497 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:50.525 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:50.544 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:50.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:50.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:50.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[200197]: pgmap v557: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 916 op/s; 17605/147072 objects degraded (11.970%); 570/147072 objects misplaced (0.388%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T10:51:50.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1095399478' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T10:51:50.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[200197]: osdmap e239: 8 total, 8 up, 5 in 2023-10-09T10:51:50.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[200197]: failed to encode map e239 with expected crc 2023-10-09T10:51:50.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/561660118' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:51:50.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:51:50.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:50.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:51:50.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[200197]: osdmap e240: 8 total, 8 up, 5 in 2023-10-09T10:51:50.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[195507]: pgmap v557: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 916 op/s; 17605/147072 objects degraded (11.970%); 570/147072 objects misplaced (0.388%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T10:51:50.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1095399478' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T10:51:50.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[195507]: osdmap e239: 8 total, 8 up, 5 in 2023-10-09T10:51:50.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[195507]: failed to encode map e239 with expected crc 2023-10-09T10:51:50.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/561660118' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:51:50.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:51:50.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:50.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:51:50.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:50 smithi125 ceph-mon[195507]: osdmap e240: 8 total, 8 up, 5 in 2023-10-09T10:51:50.648 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:50.679 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:50 smithi155 ceph-mon[184854]: pgmap v557: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 916 op/s; 17605/147072 objects degraded (11.970%); 570/147072 objects misplaced (0.388%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T10:51:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:50 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1095399478' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T10:51:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:50 smithi155 ceph-mon[184854]: osdmap e239: 8 total, 8 up, 5 in 2023-10-09T10:51:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:50 smithi155 ceph-mon[184854]: failed to encode map e239 with expected crc 2023-10-09T10:51:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:50 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/561660118' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:51:50.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:50 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:51:50.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:50 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:51:50.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:50 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:51:50.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:50 smithi155 ceph-mon[184854]: osdmap e240: 8 total, 8 up, 5 in 2023-10-09T10:51:50.903 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:50.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:50.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:50.947 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:51:50.947 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:51:50.947 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:51:50.948 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:51:50.948 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:51:51.006 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:51.033 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:51.052 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:51.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:51.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:51.155 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:51.184 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:51.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:51:51] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.43.0" 2023-10-09T10:51:51.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:51 smithi155 ceph-mon[184854]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:51 smithi155 ceph-mon[184854]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:51 smithi155 ceph-mon[184854]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:51 smithi155 ceph-mon[184854]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:51 smithi155 ceph-mon[184854]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:51 smithi155 ceph-mon[184854]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:51 smithi155 ceph-mon[184854]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:51 smithi155 ceph-mon[184854]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:51 smithi155 ceph-mon[184854]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:51 smithi155 ceph-mon[184854]: osdmap e241: 8 total, 8 up, 5 in 2023-10-09T10:51:51.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[195507]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[195507]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[195507]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[195507]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[195507]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[195507]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[195507]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[195507]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[195507]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[195507]: osdmap e241: 8 total, 8 up, 5 in 2023-10-09T10:51:51.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[200197]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[200197]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[200197]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[200197]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[200197]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[200197]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[200197]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[200197]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[200197]: failed to encode map e240 with expected crc 2023-10-09T10:51:51.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:51 smithi125 ceph-mon[200197]: osdmap e241: 8 total, 8 up, 5 in 2023-10-09T10:51:52.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:52 smithi155 ceph-mon[184854]: pgmap v560: 119 pgs: 8 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 780 op/s; 17605/142551 objects degraded (12.350%); 53/142551 objects misplaced (0.037%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T10:51:52.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:52 smithi155 ceph-mon[184854]: failed to encode map e241 with expected crc 2023-10-09T10:51:52.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:52 smithi155 ceph-mon[184854]: failed to encode map e241 with expected crc 2023-10-09T10:51:52.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:52 smithi155 ceph-mon[184854]: failed to encode map e241 with expected crc 2023-10-09T10:51:52.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:52 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T10:51:52.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:52 smithi155 ceph-mon[184854]: failed to encode map e241 with expected crc 2023-10-09T10:51:52.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:52 smithi155 ceph-mon[184854]: failed to encode map e241 with expected crc 2023-10-09T10:51:52.779 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:51:52.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:52 smithi125 ceph-mon[200197]: pgmap v560: 119 pgs: 8 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 780 op/s; 17605/142551 objects degraded (12.350%); 53/142551 objects misplaced (0.037%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T10:51:52.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:52 smithi125 ceph-mon[200197]: failed to encode map e241 with expected crc 2023-10-09T10:51:52.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:52 smithi125 ceph-mon[200197]: failed to encode map e241 with expected crc 2023-10-09T10:51:52.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:52 smithi125 ceph-mon[200197]: failed to encode map e241 with expected crc 2023-10-09T10:51:52.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:52 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T10:51:52.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:52 smithi125 ceph-mon[200197]: failed to encode map e241 with expected crc 2023-10-09T10:51:52.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:52 smithi125 ceph-mon[200197]: failed to encode map e241 with expected crc 2023-10-09T10:51:52.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:52 smithi125 ceph-mon[195507]: pgmap v560: 119 pgs: 8 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 780 op/s; 17605/142551 objects degraded (12.350%); 53/142551 objects misplaced (0.037%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T10:51:52.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:52 smithi125 ceph-mon[195507]: failed to encode map e241 with expected crc 2023-10-09T10:51:52.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:52 smithi125 ceph-mon[195507]: failed to encode map e241 with expected crc 2023-10-09T10:51:52.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:52 smithi125 ceph-mon[195507]: failed to encode map e241 with expected crc 2023-10-09T10:51:52.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:52 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T10:51:52.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:52 smithi125 ceph-mon[195507]: failed to encode map e241 with expected crc 2023-10-09T10:51:52.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:52 smithi125 ceph-mon[195507]: failed to encode map e241 with expected crc 2023-10-09T10:51:53.081 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:53.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:53.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:53.188 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:53.255 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:53.287 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:53.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:53.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:53.394 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:53.460 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:53.490 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:53.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:53.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:53.598 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:53.642 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:53.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:53 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 17605/142551 objects degraded (12.350%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:53.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:53.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:53.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:53.774 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:53.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:53 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 17605/142551 objects degraded (12.350%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:53.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:53 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 17605/142551 objects degraded (12.350%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:53.835 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:53.876 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:53.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:53.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:53.982 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:54.031 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:54.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:54 smithi155 ceph-mon[184854]: pgmap v562: 119 pgs: 1 active+recovery_wait+degraded, 8 peering, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 900 op/s; 17068/137337 objects degraded (12.428%); 106/137337 objects misplaced (0.077%); 13 MiB/s, 213 objects/s recovering 2023-10-09T10:51:54.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:54 smithi125 ceph-mon[195507]: pgmap v562: 119 pgs: 1 active+recovery_wait+degraded, 8 peering, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 900 op/s; 17068/137337 objects degraded (12.428%); 106/137337 objects misplaced (0.077%); 13 MiB/s, 213 objects/s recovering 2023-10-09T10:51:54.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:54 smithi125 ceph-mon[200197]: pgmap v562: 119 pgs: 1 active+recovery_wait+degraded, 8 peering, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 900 op/s; 17068/137337 objects degraded (12.428%); 106/137337 objects misplaced (0.077%); 13 MiB/s, 213 objects/s recovering 2023-10-09T10:51:54.897 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0] out_osds: [2, 7, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:51:54.897 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:51:54.897 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T10:51:54.897 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:51:54.897 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:51:54.898 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T10:51:55.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:51:55.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:55.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:55.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:55.305 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:55.339 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:55.483 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:55 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2023-10-09T10:51:55.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:55 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2023-10-09T10:51:55.821 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:55.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:55.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:55.930 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:51:55.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:55 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2023-10-09T10:51:55.985 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:56.027 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:51:56.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:56.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:51:56.132 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:56.169 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:56.171 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:56.194 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:56.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:56.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:56.227 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:56.300 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:56.303 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:51:56.334 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:56.343 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:51:56.362 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:56.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:56.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:56.385 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:51:56.437 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:51:56.466 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:56.471 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:51:56.506 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:51:56.531 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:56.541 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:51:56.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:56 smithi125 ceph-mon[200197]: pgmap v563: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.7 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 799 op/s; 16848/132771 objects degraded (12.690%); 53/132771 objects misplaced (0.040%); 13 MiB/s, 211 objects/s recovering 2023-10-09T10:51:56.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1229491824' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:56.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:56 smithi125 ceph-mon[195507]: pgmap v563: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.7 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 799 op/s; 16848/132771 objects degraded (12.690%); 53/132771 objects misplaced (0.040%); 13 MiB/s, 211 objects/s recovering 2023-10-09T10:51:56.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1229491824' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:56.573 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:51:56.576 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:51:56.577 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:51:56.577 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:51:56.577 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:51:56.577 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:51:56.577 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:51:56.577 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:51:56.577 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:51:56.604 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:56.621 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:56.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:56.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:56.732 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:51:56.791 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:56 smithi155 ceph-mon[184854]: pgmap v563: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.7 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 799 op/s; 16848/132771 objects degraded (12.690%); 53/132771 objects misplaced (0.040%); 13 MiB/s, 211 objects/s recovering 2023-10-09T10:51:56.791 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1229491824' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:51:56.794 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:56.831 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:51:56.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:56.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:51:56.937 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:56.990 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:57.475 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:57.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:57.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:57.586 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:57.649 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:58.094 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:58.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:58.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:58.201 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:51:58.250 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:58.278 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:51:58.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:58.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:51:58.384 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:51:58.434 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:58.466 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:51:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:58.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:51:58.571 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:58.572 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:51:58.572 INFO:teuthology.orchestra.run.smithi125.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T10:51:58.572 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:51:58.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:58 smithi125 ceph-mon[195507]: pgmap v564: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.7 GiB data, 9.5 GiB used, 437 GiB / 447 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 678 op/s; 16848/131031 objects degraded (12.858%); 53/131031 objects misplaced (0.040%); 10 MiB/s, 160 objects/s recovering 2023-10-09T10:51:58.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:58 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 16848/131031 objects degraded (12.858%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:58.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:58 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1998310152' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:58.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:58 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:58.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:58 smithi125 ceph-mon[200197]: pgmap v564: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.7 GiB data, 9.5 GiB used, 437 GiB / 447 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 678 op/s; 16848/131031 objects degraded (12.858%); 53/131031 objects misplaced (0.040%); 10 MiB/s, 160 objects/s recovering 2023-10-09T10:51:58.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:58 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 16848/131031 objects degraded (12.858%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:58.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:58 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1998310152' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:58.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:58 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:58.619 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:58.648 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:58.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:58.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:58.753 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:51:58.798 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:58 smithi155 ceph-mon[184854]: pgmap v564: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.7 GiB data, 9.5 GiB used, 437 GiB / 447 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 678 op/s; 16848/131031 objects degraded (12.858%); 53/131031 objects misplaced (0.040%); 10 MiB/s, 160 objects/s recovering 2023-10-09T10:51:58.798 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:58 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 16848/131031 objects degraded (12.858%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:51:58.798 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:58 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1998310152' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:58.798 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:58 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:51:58.803 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:58.842 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:51:58.844 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:51:58.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:58.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:51:58.946 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:51:58.951 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:51:58.951 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:51:58.990 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:59.164 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:51:59.325 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:51:59.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:59.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:51:59.438 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:59.487 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:59.525 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:59.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:59.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:51:59.573 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:51:59.629 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:51:59.669 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:59.700 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:51:59.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:59.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:51:59.705 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:59 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:51:59.705 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:59 smithi125 ceph-mon[195507]: failed to encode map e242 with expected crc 2023-10-09T10:51:59.705 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:59 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:51:59.705 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:59 smithi125 ceph-mon[195507]: osdmap e242: 8 total, 8 up, 5 in 2023-10-09T10:51:59.705 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:59 smithi125 ceph-mon[195507]: failed to encode map e242 with expected crc 2023-10-09T10:51:59.705 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:59 smithi125 ceph-mon[195507]: failed to encode map e242 with expected crc 2023-10-09T10:51:59.705 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:51:59 smithi125 ceph-mon[195507]: failed to encode map e242 with expected crc 2023-10-09T10:51:59.705 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:59 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:51:59.706 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:59 smithi125 ceph-mon[200197]: failed to encode map e242 with expected crc 2023-10-09T10:51:59.706 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:59 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:51:59.706 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:59 smithi125 ceph-mon[200197]: osdmap e242: 8 total, 8 up, 5 in 2023-10-09T10:51:59.706 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:59 smithi125 ceph-mon[200197]: failed to encode map e242 with expected crc 2023-10-09T10:51:59.706 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:59 smithi125 ceph-mon[200197]: failed to encode map e242 with expected crc 2023-10-09T10:51:59.706 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:51:59 smithi125 ceph-mon[200197]: failed to encode map e242 with expected crc 2023-10-09T10:51:59.804 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:51:59.859 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:59 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:51:59.859 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:59 smithi155 ceph-mon[184854]: failed to encode map e242 with expected crc 2023-10-09T10:51:59.859 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:59 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:51:59.859 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:59 smithi155 ceph-mon[184854]: osdmap e242: 8 total, 8 up, 5 in 2023-10-09T10:51:59.859 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:59 smithi155 ceph-mon[184854]: failed to encode map e242 with expected crc 2023-10-09T10:51:59.860 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:59 smithi155 ceph-mon[184854]: failed to encode map e242 with expected crc 2023-10-09T10:51:59.860 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:51:59 smithi155 ceph-mon[184854]: failed to encode map e242 with expected crc 2023-10-09T10:51:59.862 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:51:59.863 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:51:59.903 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:51:59.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:51:59.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:00.019 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:52:00.028 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:52:00.077 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:00.169 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:52:00.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:00.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:52:00.276 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:00.337 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:00.369 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:00.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:00.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:00.477 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:00.530 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:00.611 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:00.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:00.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:00.715 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:52:00.784 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:00 smithi125 ceph-mon[195507]: pgmap v566: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.5 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 5.1 KiB/s rd, 14 KiB/s wr, 771 op/s; 16359/124125 objects degraded (13.179%); 8.3 MiB/s, 162 objects/s recovering 2023-10-09T10:52:00.785 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:00.785 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:52:00.786 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:00 smithi125 ceph-mon[200197]: pgmap v566: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.5 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 5.1 KiB/s rd, 14 KiB/s wr, 771 op/s; 16359/124125 objects degraded (13.179%); 8.3 MiB/s, 162 objects/s recovering 2023-10-09T10:52:00.786 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:00.786 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:52:00.790 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:00.838 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:52:00.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:00.847 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:00 smithi155 ceph-mon[184854]: pgmap v566: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.5 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 5.1 KiB/s rd, 14 KiB/s wr, 771 op/s; 16359/124125 objects degraded (13.179%); 8.3 MiB/s, 162 objects/s recovering 2023-10-09T10:52:00.848 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:00.848 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:52:00.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:52:00.949 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:52:01.007 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:01.046 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:52:01.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:01.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:52:01.103 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:52:01.137 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:52:01.155 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:01.172 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:52:01.208 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:52:01.221 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:01.258 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:52:01.266 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:01.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:01.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:01.291 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:52:01.291 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:52:01.291 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T10:52:01.291 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:52:01.293 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:52:01.322 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:52:01.322 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:52:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:52:01] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.43.0" 2023-10-09T10:52:01.359 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:52:01.360 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:52:01.360 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:52:01.360 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:52:01.361 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:52:01.361 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:52:01.361 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:52:01.361 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:52:01.361 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:52:01.377 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:01.430 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:01.694 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:52:01.803 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:52:01.803 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/975710408' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:01.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:52:01.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/975710408' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:01.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:52:01.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/975710408' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:01.843 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:52:02.288 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:52:02.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:02 smithi125 ceph-mon[200197]: pgmap v567: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.4 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 4.1 KiB/s rd, 11 KiB/s wr, 800 op/s; 16196/118242 objects degraded (13.697%); 7.7 MiB/s, 147 objects/s recovering 2023-10-09T10:52:02.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:02 smithi125 ceph-mon[195507]: pgmap v567: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.4 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 4.1 KiB/s rd, 11 KiB/s wr, 800 op/s; 16196/118242 objects degraded (13.697%); 7.7 MiB/s, 147 objects/s recovering 2023-10-09T10:52:02.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:02 smithi155 ceph-mon[184854]: pgmap v567: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.4 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 4.1 KiB/s rd, 11 KiB/s wr, 800 op/s; 16196/118242 objects degraded (13.697%); 7.7 MiB/s, 147 objects/s recovering 2023-10-09T10:52:03.783 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:52:03.783 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:52:03.783 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:52:03.783 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:52:03.783 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:52:03.805 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:03 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 16196/118242 objects degraded (13.697%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:52:03.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:03 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 16196/118242 objects degraded (13.697%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:52:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:03 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 16196/118242 objects degraded (13.697%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:52:04.185 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0] out_osds: [2, 7, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:52:04.185 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:52:04.185 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-10-09T10:52:04.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd in 1 2023-10-09T10:52:04.304 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:52:04.554 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:04 smithi125 ceph-mon[200197]: pgmap v568: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.2 GiB data, 8.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 897 op/s; 15955/110418 objects degraded (14.450%); 4.6 MiB/s, 121 objects/s recovering 2023-10-09T10:52:04.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:04 smithi125 ceph-mon[195507]: pgmap v568: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.2 GiB data, 8.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 897 op/s; 15955/110418 objects degraded (14.450%); 4.6 MiB/s, 121 objects/s recovering 2023-10-09T10:52:04.950 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:52:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:04 smithi155 ceph-mon[184854]: pgmap v568: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.2 GiB data, 8.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 897 op/s; 15955/110418 objects degraded (14.450%); 4.6 MiB/s, 121 objects/s recovering 2023-10-09T10:52:05.181 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:05.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:05.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:05.281 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:05.286 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:05.348 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:05.483 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:05.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:05.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:05.589 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:05.636 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:05.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:05.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:05.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:05.777 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:52:05.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:05 smithi125 ceph-mon[200197]: pgmap v569: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.1 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 920 op/s; 15955/105162 objects degraded (15.172%); 2.9 MiB/s, 94 objects/s recovering 2023-10-09T10:52:05.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:05 smithi125 ceph-mon[195507]: pgmap v569: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.1 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 920 op/s; 15955/105162 objects degraded (15.172%); 2.9 MiB/s, 94 objects/s recovering 2023-10-09T10:52:05.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:05 smithi155 ceph-mon[184854]: pgmap v569: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.1 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 920 op/s; 15955/105162 objects degraded (15.172%); 2.9 MiB/s, 94 objects/s recovering 2023-10-09T10:52:05.843 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:05.883 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:52:05.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:05.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:52:05.989 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:06.037 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:06.113 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:06.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:06.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:06.148 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:06.220 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:52:06.267 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:06.355 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:52:06.355 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:52:06.355 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:52:06.355 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:52:06.356 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.356 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:159662 6.e 6:771f0157:::benchmark_data_smithi125_244951_object53219:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e176)", 2023-10-09T10:52:06.356 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:44.940337+0000", 2023-10-09T10:52:06.356 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 261.41164019000001, 2023-10-09T10:52:06.356 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.078880612000000003, 2023-10-09T10:52:06.356 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.356 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.356 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.356 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.356 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.356 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 159662 2023-10-09T10:52:06.356 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.357 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.357 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.357 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.357 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.940337+0000", 2023-10-09T10:52:06.357 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.357 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.357 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.357 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.357 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.940337+0000", 2023-10-09T10:52:06.357 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.357 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.357 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.358 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.358 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.940336+0000", 2023-10-09T10:52:06.358 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:06.358 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.358 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.358 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.358 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.940341+0000", 2023-10-09T10:52:06.358 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0440000000000003e-06 2023-10-09T10:52:06.358 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.358 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.358 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.358 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.940351+0000", 2023-10-09T10:52:06.359 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3479999999999993e-06 2023-10-09T10:52:06.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.359 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.359 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.359 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.940356+0000", 2023-10-09T10:52:06.359 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.101e-06 2023-10-09T10:52:06.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.359 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.359 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.359 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.948489+0000", 2023-10-09T10:52:06.359 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0081332190000000006 2023-10-09T10:52:06.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.360 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.360 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.360 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.948629+0000", 2023-10-09T10:52:06.360 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013946499999999999 2023-10-09T10:52:06.360 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.360 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.360 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:52:06.360 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.948773+0000", 2023-10-09T10:52:06.360 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014470299999999999 2023-10-09T10:52:06.360 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.360 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.360 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.360 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.953060+0000", 2023-10-09T10:52:06.361 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042871899999999998 2023-10-09T10:52:06.361 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.361 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.361 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.361 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:44.962396+0000", 2023-10-09T10:52:06.361 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0093354849999999993 2023-10-09T10:52:06.361 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.361 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.361 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.361 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:45.015699+0000", 2023-10-09T10:52:06.361 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.053302714000000001 2023-10-09T10:52:06.361 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.361 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.362 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.362 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:45.015759+0000", 2023-10-09T10:52:06.362 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0767000000000003e-05 2023-10-09T10:52:06.362 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.362 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.362 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.362 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:45.019218+0000", 2023-10-09T10:52:06.362 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034584030000000001 2023-10-09T10:52:06.362 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.362 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.362 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.362 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.362 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.363 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181597 6.2 6:42bf0766:::benchmark_data_smithi125_244951_object75154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:52:06.363 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.261648+0000", 2023-10-09T10:52:06.363 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 230.090329569, 2023-10-09T10:52:06.363 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.045766977, 2023-10-09T10:52:06.363 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.363 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.363 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.363 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.363 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.363 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181597 2023-10-09T10:52:06.363 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.363 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.363 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.364 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.364 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261648+0000", 2023-10-09T10:52:06.364 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.364 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.364 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.364 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.364 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261648+0000", 2023-10-09T10:52:06.364 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.364 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.364 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.364 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.364 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261646+0000", 2023-10-09T10:52:06.365 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:52:06.365 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.365 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.365 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.365 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261657+0000", 2023-10-09T10:52:06.365 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1413000000000001e-05 2023-10-09T10:52:06.365 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.365 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.365 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.365 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261674+0000", 2023-10-09T10:52:06.365 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6969e-05 2023-10-09T10:52:06.365 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.366 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.366 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.366 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261682+0000", 2023-10-09T10:52:06.366 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.357e-06 2023-10-09T10:52:06.366 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.366 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.366 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.366 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261737+0000", 2023-10-09T10:52:06.366 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5757999999999997e-05 2023-10-09T10:52:06.366 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.366 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.366 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op must wait for map", 2023-10-09T10:52:06.366 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261745+0000", 2023-10-09T10:52:06.367 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.576e-06 2023-10-09T10:52:06.367 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.367 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.367 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.367 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.268199+0000", 2023-10-09T10:52:06.367 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064538019999999998 2023-10-09T10:52:06.367 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.367 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.367 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:06.367 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.268208+0000", 2023-10-09T10:52:06.367 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1880000000000008e-06 2023-10-09T10:52:06.367 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.368 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.368 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.368 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.298495+0000", 2023-10-09T10:52:06.368 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0302869029999999 2023-10-09T10:52:06.368 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.368 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.368 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.368 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.298621+0000", 2023-10-09T10:52:06.368 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012632400000000001 2023-10-09T10:52:06.368 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.368 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.368 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:06.368 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.298695+0000", 2023-10-09T10:52:06.369 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3521000000000003e-05 2023-10-09T10:52:06.369 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.369 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.369 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.369 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.301440+0000", 2023-10-09T10:52:06.369 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002744879 2023-10-09T10:52:06.369 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.369 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.369 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.369 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.304506+0000", 2023-10-09T10:52:06.369 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030660570000000001 2023-10-09T10:52:06.369 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.369 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.370 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.370 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307330+0000", 2023-10-09T10:52:06.370 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028240029999999998 2023-10-09T10:52:06.370 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.370 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.370 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.370 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307381+0000", 2023-10-09T10:52:06.370 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1792000000000002e-05 2023-10-09T10:52:06.370 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.370 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.370 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.370 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307415+0000", 2023-10-09T10:52:06.371 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3429000000000001e-05 2023-10-09T10:52:06.371 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.371 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.371 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.371 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.371 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.371 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181636 6.2 6:40ad50b0:::benchmark_data_smithi125_244951_object75193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:52:06.371 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.320597+0000", 2023-10-09T10:52:06.371 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 230.03138089999999, 2023-10-09T10:52:06.371 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98701349400000005, 2023-10-09T10:52:06.371 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.371 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.371 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.372 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.372 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.372 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181636 2023-10-09T10:52:06.372 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.372 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.372 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.372 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.372 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320597+0000", 2023-10-09T10:52:06.372 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.372 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.372 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.372 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.373 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320597+0000", 2023-10-09T10:52:06.373 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.373 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.373 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.373 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.373 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320595+0000", 2023-10-09T10:52:06.373 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:52:06.373 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.373 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.373 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.373 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320604+0000", 2023-10-09T10:52:06.373 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8580000000000006e-06 2023-10-09T10:52:06.373 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.374 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.374 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.374 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320623+0000", 2023-10-09T10:52:06.374 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9894999999999998e-05 2023-10-09T10:52:06.374 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.374 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.374 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.374 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320631+0000", 2023-10-09T10:52:06.374 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7389999999999999e-06 2023-10-09T10:52:06.374 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.374 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.374 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.375 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320699+0000", 2023-10-09T10:52:06.375 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7858e-05 2023-10-09T10:52:06.375 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.375 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.375 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:06.375 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320709+0000", 2023-10-09T10:52:06.375 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0225e-05 2023-10-09T10:52:06.375 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.375 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.375 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.375 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.301476+0000", 2023-10-09T10:52:06.375 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98076654399999996 2023-10-09T10:52:06.375 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.376 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.376 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.376 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.301576+0000", 2023-10-09T10:52:06.376 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010008700000000001 2023-10-09T10:52:06.376 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.376 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.376 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:06.376 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.301679+0000", 2023-10-09T10:52:06.376 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000102806 2023-10-09T10:52:06.376 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.376 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.376 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.377 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307490+0000", 2023-10-09T10:52:06.377 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0058112210000000001 2023-10-09T10:52:06.377 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.377 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.377 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.377 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307500+0000", 2023-10-09T10:52:06.377 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9000000000000001e-06 2023-10-09T10:52:06.377 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.377 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.377 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.377 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307534+0000", 2023-10-09T10:52:06.377 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4637000000000001e-05 2023-10-09T10:52:06.377 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.378 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.378 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.378 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307577+0000", 2023-10-09T10:52:06.378 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2453999999999997e-05 2023-10-09T10:52:06.378 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.378 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.378 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.378 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307610+0000", 2023-10-09T10:52:06.378 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3167000000000003e-05 2023-10-09T10:52:06.378 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.378 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.378 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.379 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.379 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.379 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181651 6.2 6:46858d5d:::benchmark_data_smithi125_244951_object75208:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:52:06.379 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.342675+0000", 2023-10-09T10:52:06.379 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 230.00930244700001, 2023-10-09T10:52:06.379 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98594575900000003, 2023-10-09T10:52:06.379 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.379 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.379 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.379 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.379 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.379 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181651 2023-10-09T10:52:06.379 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.380 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.380 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.380 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.380 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342675+0000", 2023-10-09T10:52:06.380 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.380 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.380 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.380 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.380 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342675+0000", 2023-10-09T10:52:06.380 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.380 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.380 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.381 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.381 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342674+0000", 2023-10-09T10:52:06.381 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:06.381 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.381 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.381 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.381 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342681+0000", 2023-10-09T10:52:06.381 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5189999999999998e-06 2023-10-09T10:52:06.381 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.381 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.381 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.381 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342690+0000", 2023-10-09T10:52:06.381 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6029999999999994e-06 2023-10-09T10:52:06.382 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.382 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.382 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.382 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342694+0000", 2023-10-09T10:52:06.382 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9979999999999997e-06 2023-10-09T10:52:06.382 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.382 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.382 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.382 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342738+0000", 2023-10-09T10:52:06.382 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4175000000000003e-05 2023-10-09T10:52:06.382 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.382 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.382 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:06.383 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342749+0000", 2023-10-09T10:52:06.383 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0664e-05 2023-10-09T10:52:06.383 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.383 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.383 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.383 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.304532+0000", 2023-10-09T10:52:06.383 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.96178257899999997 2023-10-09T10:52:06.383 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.383 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.383 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.383 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.304626+0000", 2023-10-09T10:52:06.383 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4790999999999994e-05 2023-10-09T10:52:06.384 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.384 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.384 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:06.384 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.304730+0000", 2023-10-09T10:52:06.384 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103824 2023-10-09T10:52:06.384 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.384 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.384 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.384 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.310700+0000", 2023-10-09T10:52:06.384 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059698260000000001 2023-10-09T10:52:06.384 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.384 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.384 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.385 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313386+0000", 2023-10-09T10:52:06.385 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026858849999999998 2023-10-09T10:52:06.385 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.385 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.385 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.385 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328535+0000", 2023-10-09T10:52:06.385 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015149249 2023-10-09T10:52:06.385 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.385 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.385 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.385 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328576+0000", 2023-10-09T10:52:06.385 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1075999999999998e-05 2023-10-09T10:52:06.385 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.386 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.386 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.386 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328621+0000", 2023-10-09T10:52:06.386 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4577999999999999e-05 2023-10-09T10:52:06.386 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.386 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.386 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.386 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.386 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.386 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181659 6.2 6:43df335a:::benchmark_data_smithi125_244951_object75216:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:52:06.386 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.355928+0000", 2023-10-09T10:52:06.386 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 229.99604926500001, 2023-10-09T10:52:06.387 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97280544800000002, 2023-10-09T10:52:06.387 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.387 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.387 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.387 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.387 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.387 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181659 2023-10-09T10:52:06.387 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.387 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.387 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.387 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.387 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355928+0000", 2023-10-09T10:52:06.388 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.388 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.388 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.388 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.388 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355928+0000", 2023-10-09T10:52:06.388 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.388 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.388 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.388 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.388 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355926+0000", 2023-10-09T10:52:06.388 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:52:06.388 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.388 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.389 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.389 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355939+0000", 2023-10-09T10:52:06.389 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2756e-05 2023-10-09T10:52:06.389 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.389 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.389 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.389 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355957+0000", 2023-10-09T10:52:06.389 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8128999999999999e-05 2023-10-09T10:52:06.389 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.389 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.389 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.389 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355965+0000", 2023-10-09T10:52:06.389 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5320000000000001e-06 2023-10-09T10:52:06.390 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.390 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.390 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.390 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.356031+0000", 2023-10-09T10:52:06.390 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6012000000000004e-05 2023-10-09T10:52:06.390 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.390 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.390 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:06.390 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.356044+0000", 2023-10-09T10:52:06.390 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2897e-05 2023-10-09T10:52:06.390 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.390 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.391 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.391 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307650+0000", 2023-10-09T10:52:06.391 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95160661899999999 2023-10-09T10:52:06.391 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.391 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.391 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.391 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307782+0000", 2023-10-09T10:52:06.391 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013221500000000001 2023-10-09T10:52:06.391 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.391 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.391 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:06.391 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307905+0000", 2023-10-09T10:52:06.391 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000122192 2023-10-09T10:52:06.392 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.392 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.392 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.392 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313393+0000", 2023-10-09T10:52:06.392 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0054883229999999998 2023-10-09T10:52:06.392 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.392 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.392 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.392 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313435+0000", 2023-10-09T10:52:06.392 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2500999999999998e-05 2023-10-09T10:52:06.392 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.392 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.392 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.393 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328675+0000", 2023-10-09T10:52:06.393 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015239948999999999 2023-10-09T10:52:06.393 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.393 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.393 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.393 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328701+0000", 2023-10-09T10:52:06.393 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5860999999999999e-05 2023-10-09T10:52:06.393 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.393 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.393 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.393 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328734+0000", 2023-10-09T10:52:06.393 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2398999999999999e-05 2023-10-09T10:52:06.394 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.394 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.394 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.394 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.394 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.394 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181668 6.2 6:47615e5b:::benchmark_data_smithi125_244951_object75225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:52:06.394 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.371758+0000", 2023-10-09T10:52:06.394 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 229.98021915999999, 2023-10-09T10:52:06.394 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95707596299999997, 2023-10-09T10:52:06.394 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.394 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.394 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.395 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.395 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.395 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181668 2023-10-09T10:52:06.395 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.395 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.395 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.395 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.395 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371758+0000", 2023-10-09T10:52:06.395 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.395 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.395 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.395 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.395 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371758+0000", 2023-10-09T10:52:06.396 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.396 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.396 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.396 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.396 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371757+0000", 2023-10-09T10:52:06.396 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:06.396 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.396 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.396 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.396 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371764+0000", 2023-10-09T10:52:06.396 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3379999999999999e-06 2023-10-09T10:52:06.396 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.396 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.397 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.397 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371780+0000", 2023-10-09T10:52:06.397 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5863999999999999e-05 2023-10-09T10:52:06.397 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.397 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.397 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.397 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371789+0000", 2023-10-09T10:52:06.397 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2050000000000002e-06 2023-10-09T10:52:06.397 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.397 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.397 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.397 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371836+0000", 2023-10-09T10:52:06.398 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7939000000000003e-05 2023-10-09T10:52:06.398 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.398 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.398 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:06.398 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371845+0000", 2023-10-09T10:52:06.398 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2619999999999998e-06 2023-10-09T10:52:06.398 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.398 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.398 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.398 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.310735+0000", 2023-10-09T10:52:06.398 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93888995500000005 2023-10-09T10:52:06.398 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.399 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.399 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.399 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.310868+0000", 2023-10-09T10:52:06.399 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000132867 2023-10-09T10:52:06.399 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.399 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.399 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:06.399 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.310995+0000", 2023-10-09T10:52:06.399 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012735700000000001 2023-10-09T10:52:06.399 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.399 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.399 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.399 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.319588+0000", 2023-10-09T10:52:06.400 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0085930829999999996 2023-10-09T10:52:06.400 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.400 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.400 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.400 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322662+0000", 2023-10-09T10:52:06.400 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030742059999999999 2023-10-09T10:52:06.400 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.400 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.400 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.400 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328782+0000", 2023-10-09T10:52:06.400 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061201789999999999 2023-10-09T10:52:06.400 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.401 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.401 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.401 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328812+0000", 2023-10-09T10:52:06.401 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9555000000000001e-05 2023-10-09T10:52:06.401 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.401 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.401 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.401 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328834+0000", 2023-10-09T10:52:06.401 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2314999999999999e-05 2023-10-09T10:52:06.401 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.401 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.401 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.401 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.402 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.402 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181725 6.2 6:4079eac3:::benchmark_data_smithi125_244951_object75282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:52:06.402 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.472093+0000", 2023-10-09T10:52:06.402 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 229.87988434299999, 2023-10-09T10:52:06.402 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.85682714000000004, 2023-10-09T10:52:06.402 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.402 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.402 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.402 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.402 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.402 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181725 2023-10-09T10:52:06.402 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.403 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.403 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.403 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.403 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472093+0000", 2023-10-09T10:52:06.403 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.403 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.403 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.403 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.403 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472093+0000", 2023-10-09T10:52:06.403 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.403 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.403 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.403 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.404 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472092+0000", 2023-10-09T10:52:06.404 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:06.404 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.404 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.404 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.404 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472100+0000", 2023-10-09T10:52:06.404 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8759999999999994e-06 2023-10-09T10:52:06.404 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.404 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.404 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.404 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472118+0000", 2023-10-09T10:52:06.404 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7863000000000002e-05 2023-10-09T10:52:06.405 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.405 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.405 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.405 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472125+0000", 2023-10-09T10:52:06.405 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5209999999999999e-06 2023-10-09T10:52:06.405 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.405 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.405 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.405 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472170+0000", 2023-10-09T10:52:06.405 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4774999999999997e-05 2023-10-09T10:52:06.405 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.405 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.406 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:06.406 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472178+0000", 2023-10-09T10:52:06.406 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1410000000000005e-06 2023-10-09T10:52:06.406 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.406 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.406 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.406 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313459+0000", 2023-10-09T10:52:06.406 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.84128165899999996 2023-10-09T10:52:06.406 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.406 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.406 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.406 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313598+0000", 2023-10-09T10:52:06.407 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013813200000000001 2023-10-09T10:52:06.407 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.407 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.407 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:06.407 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313729+0000", 2023-10-09T10:52:06.407 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013123599999999999 2023-10-09T10:52:06.407 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.407 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.407 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.407 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322622+0000", 2023-10-09T10:52:06.407 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0088927280000000008 2023-10-09T10:52:06.407 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.408 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.408 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.408 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322695+0000", 2023-10-09T10:52:06.408 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3676000000000004e-05 2023-10-09T10:52:06.408 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.408 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.408 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.408 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328866+0000", 2023-10-09T10:52:06.408 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.006170432 2023-10-09T10:52:06.408 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.408 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.408 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.408 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328890+0000", 2023-10-09T10:52:06.408 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4579999999999998e-05 2023-10-09T10:52:06.409 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.409 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.409 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.409 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328920+0000", 2023-10-09T10:52:06.409 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0015999999999999e-05 2023-10-09T10:52:06.409 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.409 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.409 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.409 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.409 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.409 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181731 6.2 6:47d350c6:::benchmark_data_smithi125_244951_object75288:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:52:06.409 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.487013+0000", 2023-10-09T10:52:06.410 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 229.86496462100001, 2023-10-09T10:52:06.410 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.84197810100000003, 2023-10-09T10:52:06.410 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.410 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.410 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.410 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.410 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.410 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181731 2023-10-09T10:52:06.410 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.410 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.410 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.410 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.411 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487013+0000", 2023-10-09T10:52:06.411 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.411 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.411 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.411 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.411 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487013+0000", 2023-10-09T10:52:06.411 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.411 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.411 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.411 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.411 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487011+0000", 2023-10-09T10:52:06.411 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:06.411 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.412 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.412 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.412 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487020+0000", 2023-10-09T10:52:06.412 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1850000000000004e-06 2023-10-09T10:52:06.412 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.412 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.412 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.412 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487036+0000", 2023-10-09T10:52:06.412 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6529e-05 2023-10-09T10:52:06.412 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.412 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.412 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.413 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487043+0000", 2023-10-09T10:52:06.413 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0940000000000002e-06 2023-10-09T10:52:06.413 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.413 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.413 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.413 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487118+0000", 2023-10-09T10:52:06.413 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5085000000000004e-05 2023-10-09T10:52:06.413 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.413 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.413 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:06.413 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487127+0000", 2023-10-09T10:52:06.413 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7819999999999999e-06 2023-10-09T10:52:06.414 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.414 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.414 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.414 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.316596+0000", 2023-10-09T10:52:06.414 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.82946907199999997 2023-10-09T10:52:06.414 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.414 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.414 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.414 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.316692+0000", 2023-10-09T10:52:06.414 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5668000000000006e-05 2023-10-09T10:52:06.414 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.414 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.414 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:06.415 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.316798+0000", 2023-10-09T10:52:06.415 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000105782 2023-10-09T10:52:06.415 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.415 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.415 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.415 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328499+0000", 2023-10-09T10:52:06.415 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011701549 2023-10-09T10:52:06.415 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.415 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.415 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.415 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328633+0000", 2023-10-09T10:52:06.415 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013372199999999999 2023-10-09T10:52:06.416 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.416 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.416 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.416 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328950+0000", 2023-10-09T10:52:06.416 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00031750600000000001 2023-10-09T10:52:06.416 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.416 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.416 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.416 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328973+0000", 2023-10-09T10:52:06.416 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2549999999999999e-05 2023-10-09T10:52:06.416 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.416 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.416 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.417 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328991+0000", 2023-10-09T10:52:06.417 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8042000000000001e-05 2023-10-09T10:52:06.417 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.417 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.417 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.417 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.417 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.417 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181758 6.2 6:418c688b:::benchmark_data_smithi125_244951_object75315:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:52:06.417 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.547191+0000", 2023-10-09T10:52:06.417 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 229.80478691900001, 2023-10-09T10:52:06.417 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.79446401099999997, 2023-10-09T10:52:06.417 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.417 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.418 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.418 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.418 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.418 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181758 2023-10-09T10:52:06.418 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.418 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.418 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.418 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.418 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547191+0000", 2023-10-09T10:52:06.418 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.418 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.418 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.419 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.419 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547191+0000", 2023-10-09T10:52:06.419 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.419 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.419 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.419 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.419 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547189+0000", 2023-10-09T10:52:06.419 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:52:06.419 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.419 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.419 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.419 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547198+0000", 2023-10-09T10:52:06.419 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7680000000000003e-06 2023-10-09T10:52:06.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.420 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.420 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.420 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547219+0000", 2023-10-09T10:52:06.420 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0837000000000001e-05 2023-10-09T10:52:06.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.420 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.420 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.420 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547228+0000", 2023-10-09T10:52:06.420 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6610000000000006e-06 2023-10-09T10:52:06.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.420 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.420 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.421 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547285+0000", 2023-10-09T10:52:06.421 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6617999999999999e-05 2023-10-09T10:52:06.421 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.421 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:06.421 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547296+0000", 2023-10-09T10:52:06.421 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1141999999999999e-05 2023-10-09T10:52:06.421 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.421 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.421 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.319617+0000", 2023-10-09T10:52:06.421 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.77232171000000005 2023-10-09T10:52:06.422 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.422 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.422 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.422 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.319712+0000", 2023-10-09T10:52:06.422 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4128000000000004e-05 2023-10-09T10:52:06.422 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.422 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.422 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:06.422 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.319799+0000", 2023-10-09T10:52:06.422 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7299999999999994e-05 2023-10-09T10:52:06.422 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.422 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.422 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.423 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328638+0000", 2023-10-09T10:52:06.423 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0088386809999999993 2023-10-09T10:52:06.423 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.423 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.423 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.423 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341523+0000", 2023-10-09T10:52:06.423 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012885369000000001 2023-10-09T10:52:06.423 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.423 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.423 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.423 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341553+0000", 2023-10-09T10:52:06.423 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9666999999999999e-05 2023-10-09T10:52:06.423 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.424 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.424 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.424 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341610+0000", 2023-10-09T10:52:06.424 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7250999999999998e-05 2023-10-09T10:52:06.424 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.424 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.424 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.424 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341655+0000", 2023-10-09T10:52:06.424 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4746999999999997e-05 2023-10-09T10:52:06.424 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.424 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.424 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.424 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.425 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.425 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181784 6.2 6:472c8d43:::benchmark_data_smithi125_244951_object75341:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:52:06.425 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.604167+0000", 2023-10-09T10:52:06.425 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 229.74781010500001, 2023-10-09T10:52:06.425 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.74181532500000003, 2023-10-09T10:52:06.425 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.425 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.425 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.425 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.425 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.425 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181784 2023-10-09T10:52:06.425 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.426 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.426 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.426 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.426 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604167+0000", 2023-10-09T10:52:06.426 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.426 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.426 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.426 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.426 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604167+0000", 2023-10-09T10:52:06.426 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.426 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.426 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.427 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.427 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604165+0000", 2023-10-09T10:52:06.427 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:52:06.427 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.427 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.427 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.427 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604175+0000", 2023-10-09T10:52:06.427 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5910000000000002e-06 2023-10-09T10:52:06.427 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.427 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.427 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.427 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604193+0000", 2023-10-09T10:52:06.427 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8117e-05 2023-10-09T10:52:06.428 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.428 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.428 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.428 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604201+0000", 2023-10-09T10:52:06.428 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9689999999999994e-06 2023-10-09T10:52:06.428 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.428 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.428 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.428 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604264+0000", 2023-10-09T10:52:06.428 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2731000000000006e-05 2023-10-09T10:52:06.428 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.428 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.429 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:06.429 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604275+0000", 2023-10-09T10:52:06.429 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.117e-05 2023-10-09T10:52:06.429 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.429 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.429 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.429 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322716+0000", 2023-10-09T10:52:06.429 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.71844131899999997 2023-10-09T10:52:06.429 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.429 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.429 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.429 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322828+0000", 2023-10-09T10:52:06.429 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112049 2023-10-09T10:52:06.430 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.430 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.430 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:06.430 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322941+0000", 2023-10-09T10:52:06.430 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112274 2023-10-09T10:52:06.430 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.430 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.430 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.430 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328642+0000", 2023-10-09T10:52:06.430 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0057018609999999999 2023-10-09T10:52:06.430 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.430 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.431 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.431 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341689+0000", 2023-10-09T10:52:06.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01304662 2023-10-09T10:52:06.431 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.431 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.431 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.431 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.345883+0000", 2023-10-09T10:52:06.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041934269999999996 2023-10-09T10:52:06.431 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.431 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.431 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.431 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.345938+0000", 2023-10-09T10:52:06.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5408000000000002e-05 2023-10-09T10:52:06.432 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.432 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.432 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.432 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.345983+0000", 2023-10-09T10:52:06.432 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4749999999999997e-05 2023-10-09T10:52:06.432 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.432 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.432 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.432 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.432 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.432 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181846 6.2 6:4620c9a0:::benchmark_data_smithi125_244951_object75403:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:52:06.432 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.736404+0000", 2023-10-09T10:52:06.433 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 229.61557314999999, 2023-10-09T10:52:06.433 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.60969460900000005, 2023-10-09T10:52:06.433 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.433 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.433 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.433 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.433 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.433 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181846 2023-10-09T10:52:06.433 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.433 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.433 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.433 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.433 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736404+0000", 2023-10-09T10:52:06.434 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.434 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.434 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.434 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.434 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736404+0000", 2023-10-09T10:52:06.434 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.434 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.434 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.434 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.434 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736403+0000", 2023-10-09T10:52:06.434 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:06.434 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.435 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736409+0000", 2023-10-09T10:52:06.435 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1940000000000003e-06 2023-10-09T10:52:06.435 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.435 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736419+0000", 2023-10-09T10:52:06.435 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0188e-05 2023-10-09T10:52:06.435 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.436 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736426+0000", 2023-10-09T10:52:06.436 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1770000000000003e-06 2023-10-09T10:52:06.436 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.436 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.436 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.436 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736485+0000", 2023-10-09T10:52:06.436 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9590999999999999e-05 2023-10-09T10:52:06.436 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.436 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.436 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:06.436 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736494+0000", 2023-10-09T10:52:06.436 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3180000000000007e-06 2023-10-09T10:52:06.436 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.436 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.437 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.437 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.325746+0000", 2023-10-09T10:52:06.437 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.589252691 2023-10-09T10:52:06.437 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.437 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.437 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.437 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.325846+0000", 2023-10-09T10:52:06.437 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000100058 2023-10-09T10:52:06.437 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.437 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.437 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:06.437 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.325944+0000", 2023-10-09T10:52:06.438 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7664999999999994e-05 2023-10-09T10:52:06.438 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.438 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.438 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.438 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.331507+0000", 2023-10-09T10:52:06.438 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0055631719999999999 2023-10-09T10:52:06.438 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.438 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.438 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.438 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341718+0000", 2023-10-09T10:52:06.438 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010211291000000001 2023-10-09T10:52:06.438 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.439 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.439 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.439 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346032+0000", 2023-10-09T10:52:06.439 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043136299999999997 2023-10-09T10:52:06.439 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.439 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.439 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.439 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346069+0000", 2023-10-09T10:52:06.439 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6520000000000003e-05 2023-10-09T10:52:06.439 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.439 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.439 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.439 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346099+0000", 2023-10-09T10:52:06.440 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0341999999999999e-05 2023-10-09T10:52:06.440 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.440 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.440 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.440 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.440 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.440 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181882 6.2 6:44b14788:::benchmark_data_smithi125_244951_object75439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:52:06.440 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.816228+0000", 2023-10-09T10:52:06.440 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 229.53574950500001, 2023-10-09T10:52:06.440 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.529971305, 2023-10-09T10:52:06.440 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.440 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.440 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.441 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.441 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.441 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181882 2023-10-09T10:52:06.441 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.441 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.441 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.441 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.441 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816228+0000", 2023-10-09T10:52:06.441 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.441 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.441 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.441 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816228+0000", 2023-10-09T10:52:06.442 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816226+0000", 2023-10-09T10:52:06.442 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:52:06.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816234+0000", 2023-10-09T10:52:06.442 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6620000000000005e-06 2023-10-09T10:52:06.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.443 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.443 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.443 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816249+0000", 2023-10-09T10:52:06.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5097e-05 2023-10-09T10:52:06.443 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.443 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.443 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.443 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816256+0000", 2023-10-09T10:52:06.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7000000000000002e-06 2023-10-09T10:52:06.443 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.443 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.443 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.443 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816311+0000", 2023-10-09T10:52:06.444 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5797000000000001e-05 2023-10-09T10:52:06.444 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.444 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.444 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:06.444 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816322+0000", 2023-10-09T10:52:06.444 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0219e-05 2023-10-09T10:52:06.444 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.444 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.444 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.444 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.329013+0000", 2023-10-09T10:52:06.444 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.51269123999999999 2023-10-09T10:52:06.444 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.444 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.329098+0000", 2023-10-09T10:52:06.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4586999999999995e-05 2023-10-09T10:52:06.445 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.445 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:06.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.329172+0000", 2023-10-09T10:52:06.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4690000000000005e-05 2023-10-09T10:52:06.445 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.445 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.338822+0000", 2023-10-09T10:52:06.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0096501520000000004 2023-10-09T10:52:06.446 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.446 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.446 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.446 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.343192+0000", 2023-10-09T10:52:06.446 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043696409999999996 2023-10-09T10:52:06.446 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.446 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.446 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.446 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346136+0000", 2023-10-09T10:52:06.446 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029442629999999999 2023-10-09T10:52:06.446 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.446 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.447 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.447 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346170+0000", 2023-10-09T10:52:06.447 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3294999999999999e-05 2023-10-09T10:52:06.447 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.447 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.447 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.447 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346199+0000", 2023-10-09T10:52:06.447 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9674000000000001e-05 2023-10-09T10:52:06.447 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.447 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.447 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.447 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.448 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.448 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181958 6.2 6:44748d34:::benchmark_data_smithi125_244951_object75515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:52:06.448 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:17.015707+0000", 2023-10-09T10:52:06.448 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 229.336270052, 2023-10-09T10:52:06.448 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.33061554999999998, 2023-10-09T10:52:06.448 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.448 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.448 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.448 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.448 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.448 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181958 2023-10-09T10:52:06.448 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.449 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.449 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.449 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015707+0000", 2023-10-09T10:52:06.449 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.449 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.449 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.449 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015707+0000", 2023-10-09T10:52:06.449 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.449 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.449 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.450 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015706+0000", 2023-10-09T10:52:06.450 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:06.450 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.450 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.450 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.450 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015717+0000", 2023-10-09T10:52:06.450 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0558e-05 2023-10-09T10:52:06.450 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.450 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.450 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.450 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015733+0000", 2023-10-09T10:52:06.451 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6203999999999999e-05 2023-10-09T10:52:06.451 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.451 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.451 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015739+0000", 2023-10-09T10:52:06.451 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.088e-06 2023-10-09T10:52:06.451 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.451 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.451 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015793+0000", 2023-10-09T10:52:06.451 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4046999999999999e-05 2023-10-09T10:52:06.451 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:06.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015802+0000", 2023-10-09T10:52:06.452 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.772e-06 2023-10-09T10:52:06.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.331544+0000", 2023-10-09T10:52:06.452 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.31574197300000001 2023-10-09T10:52:06.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.331667+0000", 2023-10-09T10:52:06.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012380900000000001 2023-10-09T10:52:06.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.453 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.453 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:06.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.331748+0000", 2023-10-09T10:52:06.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0389000000000006e-05 2023-10-09T10:52:06.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.453 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.453 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.338838+0000", 2023-10-09T10:52:06.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007089962 2023-10-09T10:52:06.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.453 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.454 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.454 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.343219+0000", 2023-10-09T10:52:06.454 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043816970000000004 2023-10-09T10:52:06.454 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.454 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.454 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.454 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346261+0000", 2023-10-09T10:52:06.454 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030411969999999998 2023-10-09T10:52:06.454 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.454 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.454 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.454 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346294+0000", 2023-10-09T10:52:06.455 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3383000000000002e-05 2023-10-09T10:52:06.455 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.455 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.455 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.455 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346323+0000", 2023-10-09T10:52:06.455 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8864999999999999e-05 2023-10-09T10:52:06.455 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.455 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.455 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.455 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.455 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.455 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:182031 6.2 6:424d2b82:::benchmark_data_smithi125_244951_object75588:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:52:06.455 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:17.198389+0000", 2023-10-09T10:52:06.456 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 229.15358823299999, 2023-10-09T10:52:06.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15372846300000001, 2023-10-09T10:52:06.456 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.456 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.456 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.456 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.456 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.456 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 182031 2023-10-09T10:52:06.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.456 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.456 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.456 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.456 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198389+0000", 2023-10-09T10:52:06.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.457 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.457 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.457 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198389+0000", 2023-10-09T10:52:06.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.457 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.457 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.457 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198388+0000", 2023-10-09T10:52:06.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:06.457 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.458 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.458 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198396+0000", 2023-10-09T10:52:06.458 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7409999999999992e-06 2023-10-09T10:52:06.458 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.458 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.458 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198409+0000", 2023-10-09T10:52:06.458 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.343e-05 2023-10-09T10:52:06.458 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.458 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.458 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198415+0000", 2023-10-09T10:52:06.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.314e-06 2023-10-09T10:52:06.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.459 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198483+0000", 2023-10-09T10:52:06.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7677999999999998e-05 2023-10-09T10:52:06.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.459 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:06.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198494+0000", 2023-10-09T10:52:06.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1236000000000001e-05 2023-10-09T10:52:06.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.335697+0000", 2023-10-09T10:52:06.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.137203295 2023-10-09T10:52:06.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.335802+0000", 2023-10-09T10:52:06.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104095 2023-10-09T10:52:06.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:06.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.335901+0000", 2023-10-09T10:52:06.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9179999999999996e-05 2023-10-09T10:52:06.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.343158+0000", 2023-10-09T10:52:06.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0072576120000000001 2023-10-09T10:52:06.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346383+0000", 2023-10-09T10:52:06.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003224312 2023-10-09T10:52:06.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.352022+0000", 2023-10-09T10:52:06.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0056397219999999998 2023-10-09T10:52:06.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.352075+0000", 2023-10-09T10:52:06.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2369e-05 2023-10-09T10:52:06.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.352118+0000", 2023-10-09T10:52:06.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2911999999999998e-05 2023-10-09T10:52:06.462 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.463 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.463 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.463 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:211949 6.e 6:700b4d98:::benchmark_data_smithi125_244951_object105506:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-10-09T10:52:06.463 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:01.361232+0000", 2023-10-09T10:52:06.463 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 184.99074496899999, 2023-10-09T10:52:06.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.087977252000000006, 2023-10-09T10:52:06.463 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.463 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.463 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.463 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.463 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.464 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 211949 2023-10-09T10:52:06.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.464 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.361232+0000", 2023-10-09T10:52:06.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.361232+0000", 2023-10-09T10:52:06.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.465 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.361231+0000", 2023-10-09T10:52:06.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:52:06.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.465 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.361238+0000", 2023-10-09T10:52:06.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5939999999999996e-06 2023-10-09T10:52:06.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.466 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.361253+0000", 2023-10-09T10:52:06.466 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4741e-05 2023-10-09T10:52:06.466 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.466 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.466 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.361260+0000", 2023-10-09T10:52:06.466 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0770000000000002e-06 2023-10-09T10:52:06.466 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.466 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.466 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.371532+0000", 2023-10-09T10:52:06.466 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010272106 2023-10-09T10:52:06.466 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.467 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.371720+0000", 2023-10-09T10:52:06.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018771400000000001 2023-10-09T10:52:06.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:52:06.467 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.371907+0000", 2023-10-09T10:52:06.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018719 2023-10-09T10:52:06.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.380057+0000", 2023-10-09T10:52:06.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0081502099999999997 2023-10-09T10:52:06.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.388092+0000", 2023-10-09T10:52:06.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0080344240000000001 2023-10-09T10:52:06.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.447054+0000", 2023-10-09T10:52:06.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.058962522000000003 2023-10-09T10:52:06.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.469 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.469 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.447110+0000", 2023-10-09T10:52:06.469 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5661999999999997e-05 2023-10-09T10:52:06.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.469 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.469 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.449210+0000", 2023-10-09T10:52:06.469 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020998459999999998 2023-10-09T10:52:06.469 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.469 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.469 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.470 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.470 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.470 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:212017 6.e 6:72d7ee3b:::benchmark_data_smithi125_244951_object105574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-10-09T10:52:06.470 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:01.454495+0000", 2023-10-09T10:52:06.470 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 184.897482376, 2023-10-09T10:52:06.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.079011253000000004, 2023-10-09T10:52:06.470 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.470 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.470 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.470 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.470 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.470 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212017 2023-10-09T10:52:06.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.471 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.471 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.454495+0000", 2023-10-09T10:52:06.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.471 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.454495+0000", 2023-10-09T10:52:06.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.454493+0000", 2023-10-09T10:52:06.472 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:06.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.454501+0000", 2023-10-09T10:52:06.472 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0890000000000002e-06 2023-10-09T10:52:06.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.454514+0000", 2023-10-09T10:52:06.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.293e-05 2023-10-09T10:52:06.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.454520+0000", 2023-10-09T10:52:06.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0279999999999997e-06 2023-10-09T10:52:06.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.459436+0000", 2023-10-09T10:52:06.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0049160360000000004 2023-10-09T10:52:06.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.459625+0000", 2023-10-09T10:52:06.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018896000000000001 2023-10-09T10:52:06.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:52:06.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.459825+0000", 2023-10-09T10:52:06.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00020007 2023-10-09T10:52:06.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.468866+0000", 2023-10-09T10:52:06.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0090409570000000005 2023-10-09T10:52:06.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.475 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.475 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.473194+0000", 2023-10-09T10:52:06.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043287170000000002 2023-10-09T10:52:06.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.475 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.475 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.530992+0000", 2023-10-09T10:52:06.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.057798119000000002 2023-10-09T10:52:06.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.476 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.476 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.531071+0000", 2023-10-09T10:52:06.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8489e-05 2023-10-09T10:52:06.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.476 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.476 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.533506+0000", 2023-10-09T10:52:06.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024354390000000002 2023-10-09T10:52:06.476 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.476 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.476 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.477 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:212020 6.e 6:7c93ac91:::benchmark_data_smithi125_244951_object105577:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-10-09T10:52:06.477 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:01.454657+0000", 2023-10-09T10:52:06.477 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 184.89732079800001, 2023-10-09T10:52:06.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.081475798000000002, 2023-10-09T10:52:06.477 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.477 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.477 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.477 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:06.477 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:06.477 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212020 2023-10-09T10:52:06.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.478 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.454657+0000", 2023-10-09T10:52:06.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.454657+0000", 2023-10-09T10:52:06.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.454656+0000", 2023-10-09T10:52:06.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:52:06.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.454660+0000", 2023-10-09T10:52:06.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0749999999999999e-06 2023-10-09T10:52:06.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.454669+0000", 2023-10-09T10:52:06.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6629999999999999e-06 2023-10-09T10:52:06.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.454673+0000", 2023-10-09T10:52:06.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3059999999999999e-06 2023-10-09T10:52:06.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.468941+0000", 2023-10-09T10:52:06.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014268147 2023-10-09T10:52:06.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.469105+0000", 2023-10-09T10:52:06.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016396100000000001 2023-10-09T10:52:06.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:52:06.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.469269+0000", 2023-10-09T10:52:06.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000163621 2023-10-09T10:52:06.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.488018+0000", 2023-10-09T10:52:06.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018749393999999999 2023-10-09T10:52:06.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.488120+0000", 2023-10-09T10:52:06.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010183600000000001 2023-10-09T10:52:06.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.533553+0000", 2023-10-09T10:52:06.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.045432605000000001 2023-10-09T10:52:06.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.483 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.483 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.533600+0000", 2023-10-09T10:52:06.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7151999999999998e-05 2023-10-09T10:52:06.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.483 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.483 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:01.536132+0000", 2023-10-09T10:52:06.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002532697 2023-10-09T10:52:06.483 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.483 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.483 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.484 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:21941 7.8 7:1e642e31:::benchmark_data_smithi125_295816_object21940:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T10:52:06.484 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:29.771418+0000", 2023-10-09T10:52:06.484 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 156.58055935199999, 2023-10-09T10:52:06.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.080887202000000005, 2023-10-09T10:52:06.484 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.484 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.484 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.484 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:06.484 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:06.484 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 21941 2023-10-09T10:52:06.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.484 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.771418+0000", 2023-10-09T10:52:06.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.771418+0000", 2023-10-09T10:52:06.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.771417+0000", 2023-10-09T10:52:06.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:06.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.771449+0000", 2023-10-09T10:52:06.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1816000000000002e-05 2023-10-09T10:52:06.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.771461+0000", 2023-10-09T10:52:06.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2098e-05 2023-10-09T10:52:06.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.771473+0000", 2023-10-09T10:52:06.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2085e-05 2023-10-09T10:52:06.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.771512+0000", 2023-10-09T10:52:06.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9501e-05 2023-10-09T10:52:06.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.771630+0000", 2023-10-09T10:52:06.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118099 2023-10-09T10:52:06.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:52:06.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.771762+0000", 2023-10-09T10:52:06.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000131521 2023-10-09T10:52:06.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.772712+0000", 2023-10-09T10:52:06.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00095052999999999999 2023-10-09T10:52:06.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773165+0000", 2023-10-09T10:52:06.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00045247899999999999 2023-10-09T10:52:06.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852196+0000", 2023-10-09T10:52:06.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.079031004000000002 2023-10-09T10:52:06.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.490 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852256+0000", 2023-10-09T10:52:06.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0297000000000002e-05 2023-10-09T10:52:06.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.490 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.490 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.490 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852305+0000", 2023-10-09T10:52:06.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9082999999999997e-05 2023-10-09T10:52:06.490 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.490 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.490 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.490 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.491 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:21943 7.8 7:1c85ee25:::benchmark_data_smithi125_295816_object21942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T10:52:06.491 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:29.773573+0000", 2023-10-09T10:52:06.491 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 156.57840493200001, 2023-10-09T10:52:06.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.078882089000000002, 2023-10-09T10:52:06.491 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.491 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.491 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.491 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:06.491 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:06.491 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 21943 2023-10-09T10:52:06.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.491 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773573+0000", 2023-10-09T10:52:06.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773573+0000", 2023-10-09T10:52:06.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773572+0000", 2023-10-09T10:52:06.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:06.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773630+0000", 2023-10-09T10:52:06.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8307000000000003e-05 2023-10-09T10:52:06.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773641+0000", 2023-10-09T10:52:06.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0865e-05 2023-10-09T10:52:06.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773645+0000", 2023-10-09T10:52:06.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5349999999999998e-06 2023-10-09T10:52:06.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773678+0000", 2023-10-09T10:52:06.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2811000000000001e-05 2023-10-09T10:52:06.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773777+0000", 2023-10-09T10:52:06.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8658000000000003e-05 2023-10-09T10:52:06.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,7", 2023-10-09T10:52:06.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773859+0000", 2023-10-09T10:52:06.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2698e-05 2023-10-09T10:52:06.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.774789+0000", 2023-10-09T10:52:06.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00092949199999999997 2023-10-09T10:52:06.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.774947+0000", 2023-10-09T10:52:06.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015804999999999999 2023-10-09T10:52:06.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852363+0000", 2023-10-09T10:52:06.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.077415805000000004 2023-10-09T10:52:06.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852414+0000", 2023-10-09T10:52:06.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1167e-05 2023-10-09T10:52:06.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852455+0000", 2023-10-09T10:52:06.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0632e-05 2023-10-09T10:52:06.497 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.497 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.497 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.497 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:130164 7.8 7:1f41fbaa:::benchmark_data_smithi125_295816_object39027:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e238)", 2023-10-09T10:52:06.497 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:43.530226+0000", 2023-10-09T10:52:06.497 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 22.821751720000002, 2023-10-09T10:52:06.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.091038162000000006, 2023-10-09T10:52:06.498 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:06.498 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:06.498 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:06.498 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:06.498 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:06.498 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 130164 2023-10-09T10:52:06.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.498 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:06.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.498 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:06.498 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.530226+0000", 2023-10-09T10:52:06.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:06.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.530226+0000", 2023-10-09T10:52:06.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:06.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:06.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.530225+0000", 2023-10-09T10:52:06.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:06.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:06.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.530230+0000", 2023-10-09T10:52:06.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9010000000000002e-06 2023-10-09T10:52:06.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:06.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.530238+0000", 2023-10-09T10:52:06.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6149999999999997e-06 2023-10-09T10:52:06.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:06.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.530242+0000", 2023-10-09T10:52:06.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0879999999999997e-06 2023-10-09T10:52:06.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:06.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.530296+0000", 2023-10-09T10:52:06.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3718000000000003e-05 2023-10-09T10:52:06.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:06.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.530422+0000", 2023-10-09T10:52:06.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012543600000000001 2023-10-09T10:52:06.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T10:52:06.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.530533+0000", 2023-10-09T10:52:06.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000110998 2023-10-09T10:52:06.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:06.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.532905+0000", 2023-10-09T10:52:06.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002372674 2023-10-09T10:52:06.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.535226+0000", 2023-10-09T10:52:06.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002320903 2023-10-09T10:52:06.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:06.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.621185+0000", 2023-10-09T10:52:06.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.085959252 2023-10-09T10:52:06.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:06.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.621222+0000", 2023-10-09T10:52:06.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6174000000000003e-05 2023-10-09T10:52:06.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:06.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:06.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:06.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.621264+0000", 2023-10-09T10:52:06.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2313e-05 2023-10-09T10:52:06.504 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.504 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.504 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.504 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:06.504 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:06.504 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:52:06.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:06.505 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:52:06.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:52:06.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:06.670 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:06.719 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:06.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:06.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:06.827 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:06.899 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:06.906 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:52:06.931 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:06.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:06.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:07.037 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:07.096 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:07.204 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:52:07.336 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:52:07.371 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:07 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:07.371 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:07 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:07.371 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:07 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2468656531' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:52:07.371 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:07 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:52:07.372 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:07 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:07.372 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:07 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:07.372 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:07 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2468656531' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:52:07.372 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:07 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:52:07.387 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:52:07.648 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:07 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:07.649 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:07 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:07.649 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:07 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2468656531' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:52:07.649 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:07 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:52:07.953 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:07.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:07.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:08.059 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:52:08.107 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:08.233 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:52:08.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:08.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:52:08.340 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:08.395 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:08.423 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:08.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:08.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:08.513 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:52:08.513 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:52:08.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:08.513 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35187.0:151808 7.6 e243/203 7:61e72ce1:::benchmark_data_smithi125_295816_object60671:head v 243'9368, mlcod=199'1000)", 2023-10-09T10:52:08.513 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:52:08.511705+0000", 2023-10-09T10:52:08.514 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.001219569, 2023-10-09T10:52:08.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012397370000000001, 2023-10-09T10:52:08.514 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:08.514 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:52:08.514 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:08.514 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:08.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:08.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:08.511705+0000", 2023-10-09T10:52:08.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:08.514 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:08.514 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:08.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:08.515 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:08.511705+0000", 2023-10-09T10:52:08.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:08.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:08.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:08.515 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:08.515 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:08.511703+0000", 2023-10-09T10:52:08.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:08.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:08.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:08.515 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:08.515 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:08.511714+0000", 2023-10-09T10:52:08.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0961999999999999e-05 2023-10-09T10:52:08.516 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:08.516 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:08.516 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:08.516 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:08.511728+0000", 2023-10-09T10:52:08.516 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3699999999999999e-05 2023-10-09T10:52:08.516 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:08.516 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:08.516 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:08.516 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:08.511736+0000", 2023-10-09T10:52:08.516 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4989999999999997e-06 2023-10-09T10:52:08.516 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:08.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:08.517 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:08.517 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:08.511775+0000", 2023-10-09T10:52:08.517 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9279000000000001e-05 2023-10-09T10:52:08.517 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:08.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:08.517 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:08.517 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:08.511799+0000", 2023-10-09T10:52:08.517 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4139e-05 2023-10-09T10:52:08.517 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:08.517 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:08.517 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:08.517 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:08.518 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T10:52:08.518 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 1 2023-10-09T10:52:08.518 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:52:08.527 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:08.541 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:08 smithi155 ceph-mon[184854]: pgmap v570: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.1 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 920 op/s; 15955/103404 objects degraded (15.430%); 2.9 MiB/s, 94 objects/s recovering 2023-10-09T10:52:08.541 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:08 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:52:08.541 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:08 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:52:08.541 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:08 smithi155 ceph-mon[184854]: osdmap e243: 8 total, 8 up, 5 in 2023-10-09T10:52:08.541 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:08 smithi155 ceph-mon[184854]: failed to encode map e243 with expected crc 2023-10-09T10:52:08.541 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:08 smithi155 ceph-mon[184854]: failed to encode map e243 with expected crc 2023-10-09T10:52:08.541 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:08 smithi155 ceph-mon[184854]: failed to encode map e243 with expected crc 2023-10-09T10:52:08.541 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:08 smithi155 ceph-mon[184854]: failed to encode map e243 with expected crc 2023-10-09T10:52:08.541 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:08 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 15955/103404 objects degraded (15.430%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:52:08.545 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:08.571 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:08.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[195507]: pgmap v570: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.1 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 920 op/s; 15955/103404 objects degraded (15.430%); 2.9 MiB/s, 94 objects/s recovering 2023-10-09T10:52:08.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:52:08.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:52:08.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[195507]: osdmap e243: 8 total, 8 up, 5 in 2023-10-09T10:52:08.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[195507]: failed to encode map e243 with expected crc 2023-10-09T10:52:08.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[195507]: failed to encode map e243 with expected crc 2023-10-09T10:52:08.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[195507]: failed to encode map e243 with expected crc 2023-10-09T10:52:08.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[195507]: failed to encode map e243 with expected crc 2023-10-09T10:52:08.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 15955/103404 objects degraded (15.430%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:52:08.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[200197]: pgmap v570: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.1 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 920 op/s; 15955/103404 objects degraded (15.430%); 2.9 MiB/s, 94 objects/s recovering 2023-10-09T10:52:08.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:52:08.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:52:08.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[200197]: osdmap e243: 8 total, 8 up, 5 in 2023-10-09T10:52:08.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[200197]: failed to encode map e243 with expected crc 2023-10-09T10:52:08.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[200197]: failed to encode map e243 with expected crc 2023-10-09T10:52:08.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[200197]: failed to encode map e243 with expected crc 2023-10-09T10:52:08.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[200197]: failed to encode map e243 with expected crc 2023-10-09T10:52:08.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 15955/103404 objects degraded (15.430%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T10:52:08.649 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:08.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:08.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:08.753 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:08.806 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:08.943 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:52:08.946 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:08.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:08.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:08.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:08 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T10:52:08.639+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e243 definitely_dead 0 2023-10-09T10:52:09.049 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:09.118 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:09.170 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:09.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:09.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:09.283 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:09.344 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:09.378 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.1. 2023-10-09T10:52:09.401 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:52:09.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:09 smithi155 ceph-mon[184854]: failed to encode map e243 with expected crc 2023-10-09T10:52:09.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:09 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1578589281' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T10:52:09.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:09 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T10:52:09.496 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:09.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:09.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:09.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:09.627 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:09 smithi125 ceph-mon[195507]: failed to encode map e243 with expected crc 2023-10-09T10:52:09.627 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:09 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1578589281' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T10:52:09.627 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:09 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T10:52:09.627 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:09 smithi125 ceph-mon[200197]: failed to encode map e243 with expected crc 2023-10-09T10:52:09.627 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:09 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1578589281' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T10:52:09.627 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:09 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T10:52:09.629 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:09.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:52:10.254 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:10.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:10.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:10.357 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:10.388 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:52:10.389 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:52:10.389 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:52:10.389 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:52:10.389 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:52:10.407 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:10 smithi155 ceph-mon[184854]: pgmap v572: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.0 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 927 op/s; 15729/96279 objects degraded (16.337%); 1.0 MiB/s, 62 objects/s recovering 2023-10-09T10:52:10.407 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:10 smithi155 ceph-mon[184854]: failed to encode map e244 with expected crc 2023-10-09T10:52:10.407 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:10 smithi155 ceph-mon[184854]: failed to encode map e244 with expected crc 2023-10-09T10:52:10.407 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:10 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T10:52:10.407 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:10 smithi155 ceph-mon[184854]: osdmap e244: 8 total, 8 up, 6 in 2023-10-09T10:52:10.410 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:10.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:10.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:10.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:10.538 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:52:10.567 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:10.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:10 smithi125 ceph-mon[195507]: pgmap v572: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.0 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 927 op/s; 15729/96279 objects degraded (16.337%); 1.0 MiB/s, 62 objects/s recovering 2023-10-09T10:52:10.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:10 smithi125 ceph-mon[195507]: failed to encode map e244 with expected crc 2023-10-09T10:52:10.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:10 smithi125 ceph-mon[195507]: failed to encode map e244 with expected crc 2023-10-09T10:52:10.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:10 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T10:52:10.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:10 smithi125 ceph-mon[195507]: osdmap e244: 8 total, 8 up, 6 in 2023-10-09T10:52:10.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:10 smithi125 ceph-mon[200197]: pgmap v572: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.0 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 927 op/s; 15729/96279 objects degraded (16.337%); 1.0 MiB/s, 62 objects/s recovering 2023-10-09T10:52:10.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:10 smithi125 ceph-mon[200197]: failed to encode map e244 with expected crc 2023-10-09T10:52:10.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:10 smithi125 ceph-mon[200197]: failed to encode map e244 with expected crc 2023-10-09T10:52:10.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:10 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T10:52:10.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:10 smithi125 ceph-mon[200197]: osdmap e244: 8 total, 8 up, 6 in 2023-10-09T10:52:10.689 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:52:10.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:10.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:52:10.758 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:52:10.794 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:10.821 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:11.032 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:52:11.461 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:11 smithi155 ceph-mon[184854]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.461 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:11 smithi155 ceph-mon[184854]: osdmap e245: 8 total, 8 up, 6 in 2023-10-09T10:52:11.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:11 smithi155 ceph-mon[184854]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:11 smithi155 ceph-mon[184854]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:11 smithi155 ceph-mon[184854]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:11 smithi155 ceph-mon[184854]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:11 smithi155 ceph-mon[184854]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:11 smithi155 ceph-mon[184854]: 7.5 starting backfill to osd.1 from (0'0,207'3154] MAX to 243'9568 2023-10-09T10:52:11.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[195507]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[195507]: osdmap e245: 8 total, 8 up, 6 in 2023-10-09T10:52:11.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[195507]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[195507]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[195507]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[195507]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[195507]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[195507]: 7.5 starting backfill to osd.1 from (0'0,207'3154] MAX to 243'9568 2023-10-09T10:52:11.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:52:11] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.43.0" 2023-10-09T10:52:11.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[200197]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[200197]: osdmap e245: 8 total, 8 up, 6 in 2023-10-09T10:52:11.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[200197]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[200197]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[200197]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[200197]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[200197]: failed to encode map e245 with expected crc 2023-10-09T10:52:11.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:11 smithi125 ceph-mon[200197]: 7.5 starting backfill to osd.1 from (0'0,207'3154] MAX to 243'9568 2023-10-09T10:52:12.039 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:52:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:52:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:52:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:52:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106432 6.a 6:50fa9909:::benchmark_data_smithi125_244951_object106431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:52:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.249049+0000", 2023-10-09T10:52:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 369.78748665099999, 2023-10-09T10:52:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.49699488600000002, 2023-10-09T10:52:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T10:52:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106432 2023-10-09T10:52:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249049+0000", 2023-10-09T10:52:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249049+0000", 2023-10-09T10:52:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249047+0000", 2023-10-09T10:52:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:52:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249097+0000", 2023-10-09T10:52:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0584000000000002e-05 2023-10-09T10:52:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249117+0000", 2023-10-09T10:52:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9763999999999999e-05 2023-10-09T10:52:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249125+0000", 2023-10-09T10:52:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6040000000000004e-06 2023-10-09T10:52:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249189+0000", 2023-10-09T10:52:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3779e-05 2023-10-09T10:52:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249335+0000", 2023-10-09T10:52:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014603399999999999 2023-10-09T10:52:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:52:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.249521+0000", 2023-10-09T10:52:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018619099999999999 2023-10-09T10:52:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.253194+0000", 2023-10-09T10:52:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003673103 2023-10-09T10:52:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.253298+0000", 2023-10-09T10:52:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103828 2023-10-09T10:52:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.271832+0000", 2023-10-09T10:52:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018534658999999998 2023-10-09T10:52:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.271843+0000", 2023-10-09T10:52:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0995000000000001e-05 2023-10-09T10:52:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.745982+0000", 2023-10-09T10:52:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.47413843 2023-10-09T10:52:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746044+0000", 2023-10-09T10:52:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1832e-05 2023-10-09T10:52:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106439 6.a 6:5c3c51c3:::benchmark_data_smithi125_244951_object106438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-10-09T10:52:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.284123+0000", 2023-10-09T10:52:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 369.75241244599999, 2023-10-09T10:52:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.47113701699999999, 2023-10-09T10:52:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106439 2023-10-09T10:52:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284123+0000", 2023-10-09T10:52:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284123+0000", 2023-10-09T10:52:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284122+0000", 2023-10-09T10:52:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284170+0000", 2023-10-09T10:52:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7929999999999997e-05 2023-10-09T10:52:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284181+0000", 2023-10-09T10:52:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1292e-05 2023-10-09T10:52:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284186+0000", 2023-10-09T10:52:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1089999999999997e-06 2023-10-09T10:52:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284266+0000", 2023-10-09T10:52:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0311999999999999e-05 2023-10-09T10:52:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284287+0000", 2023-10-09T10:52:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0704e-05 2023-10-09T10:52:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746068+0000", 2023-10-09T10:52:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.46178057300000003 2023-10-09T10:52:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746295+0000", 2023-10-09T10:52:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000227335 2023-10-09T10:52:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:52:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.746417+0000", 2023-10-09T10:52:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000121816 2023-10-09T10:52:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.752322+0000", 2023-10-09T10:52:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059055230000000002 2023-10-09T10:52:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754107+0000", 2023-10-09T10:52:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0017852149999999999 2023-10-09T10:52:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754378+0000", 2023-10-09T10:52:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000270547 2023-10-09T10:52:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754422+0000", 2023-10-09T10:52:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4063000000000001e-05 2023-10-09T10:52:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.755260+0000", 2023-10-09T10:52:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00083801000000000004 2023-10-09T10:52:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106440 6.a 6:5e7f7caf:::benchmark_data_smithi125_244951_object106439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-10-09T10:52:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.284197+0000", 2023-10-09T10:52:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 369.75233891800002, 2023-10-09T10:52:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.47186083699999998, 2023-10-09T10:52:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106440 2023-10-09T10:52:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284197+0000", 2023-10-09T10:52:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284197+0000", 2023-10-09T10:52:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.057 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.057 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.057 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284196+0000", 2023-10-09T10:52:12.057 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:52:12.057 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.057 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.057 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284284+0000", 2023-10-09T10:52:12.057 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7646999999999996e-05 2023-10-09T10:52:12.057 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.058 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284292+0000", 2023-10-09T10:52:12.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2689999999999999e-06 2023-10-09T10:52:12.058 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.058 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.058 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284297+0000", 2023-10-09T10:52:12.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4419999999999997e-06 2023-10-09T10:52:12.058 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.058 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.059 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284344+0000", 2023-10-09T10:52:12.059 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6940000000000001e-05 2023-10-09T10:52:12.059 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.059 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.059 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:12.059 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.284352+0000", 2023-10-09T10:52:12.059 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0749999999999998e-06 2023-10-09T10:52:12.059 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.059 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.059 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.059 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.749219+0000", 2023-10-09T10:52:12.059 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.46486643799999999 2023-10-09T10:52:12.059 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.060 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.060 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:12.060 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.749378+0000", 2023-10-09T10:52:12.060 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015880000000000001 2023-10-09T10:52:12.060 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.060 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.060 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:52:12.060 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.749563+0000", 2023-10-09T10:52:12.060 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018555399999999999 2023-10-09T10:52:12.060 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.060 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.060 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.060 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754147+0000", 2023-10-09T10:52:12.061 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045842050000000001 2023-10-09T10:52:12.061 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.061 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.061 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:12.061 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.754155+0000", 2023-10-09T10:52:12.061 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2080000000000003e-06 2023-10-09T10:52:12.061 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.061 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.061 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.061 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.755321+0000", 2023-10-09T10:52:12.061 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011663839999999999 2023-10-09T10:52:12.061 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.062 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.062 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:12.062 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.755360+0000", 2023-10-09T10:52:12.062 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8943000000000003e-05 2023-10-09T10:52:12.062 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.062 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.062 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.062 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.756057+0000", 2023-10-09T10:52:12.062 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00069742899999999995 2023-10-09T10:52:12.062 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.062 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.062 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.063 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.063 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.063 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:152994 6.7 6:e184a006:::benchmark_data_smithi125_244951_object46551:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:52:12.063 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:35.519171+0000", 2023-10-09T10:52:12.063 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 276.51736447600001, 2023-10-09T10:52:12.063 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.084211253, 2023-10-09T10:52:12.063 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.063 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:12.063 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.063 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:12.063 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:12.063 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 152994 2023-10-09T10:52:12.064 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.064 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.064 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.064 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.064 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519171+0000", 2023-10-09T10:52:12.064 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.064 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.064 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.064 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.064 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519171+0000", 2023-10-09T10:52:12.064 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.064 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.064 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.065 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.065 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519170+0000", 2023-10-09T10:52:12.065 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:12.065 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.065 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.065 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.065 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519181+0000", 2023-10-09T10:52:12.065 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1884e-05 2023-10-09T10:52:12.065 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.065 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.065 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.065 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519196+0000", 2023-10-09T10:52:12.065 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.448e-05 2023-10-09T10:52:12.066 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.066 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.066 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.066 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.519202+0000", 2023-10-09T10:52:12.066 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3670000000000002e-06 2023-10-09T10:52:12.066 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.066 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.066 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.066 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.522571+0000", 2023-10-09T10:52:12.066 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0033686639999999999 2023-10-09T10:52:12.066 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.066 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.067 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:12.067 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.522712+0000", 2023-10-09T10:52:12.067 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014130800000000001 2023-10-09T10:52:12.067 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.067 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.067 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:52:12.067 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.522828+0000", 2023-10-09T10:52:12.067 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115808 2023-10-09T10:52:12.067 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.067 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.067 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:12.067 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.527082+0000", 2023-10-09T10:52:12.067 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042537520000000004 2023-10-09T10:52:12.068 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.068 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.068 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.068 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.528250+0000", 2023-10-09T10:52:12.068 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011679310000000001 2023-10-09T10:52:12.068 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.068 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.068 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.068 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.603293+0000", 2023-10-09T10:52:12.068 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.075043377999999994 2023-10-09T10:52:12.068 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.068 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.068 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:12.069 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.603336+0000", 2023-10-09T10:52:12.069 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2774000000000001e-05 2023-10-09T10:52:12.069 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.069 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.069 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.069 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:35.603382+0000", 2023-10-09T10:52:12.069 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6279999999999997e-05 2023-10-09T10:52:12.069 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.069 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.069 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.069 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.069 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.070 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:21936 7.0 7:015131e0:::benchmark_data_smithi125_295816_object21935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T10:52:12.070 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:29.765153+0000", 2023-10-09T10:52:12.070 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 162.27138254600001, 2023-10-09T10:52:12.070 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.087158461000000007, 2023-10-09T10:52:12.070 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.070 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:12.070 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.070 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.070 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.070 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 21936 2023-10-09T10:52:12.070 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.071 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.071 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.071 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.071 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765153+0000", 2023-10-09T10:52:12.071 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.071 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.071 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.071 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.071 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765153+0000", 2023-10-09T10:52:12.071 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.071 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.071 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.072 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.072 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765152+0000", 2023-10-09T10:52:12.072 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:52:12.072 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.072 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.072 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.072 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765187+0000", 2023-10-09T10:52:12.072 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4075999999999997e-05 2023-10-09T10:52:12.072 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.072 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.072 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.072 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765194+0000", 2023-10-09T10:52:12.073 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7230000000000004e-06 2023-10-09T10:52:12.073 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.073 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.073 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.073 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765199+0000", 2023-10-09T10:52:12.073 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7879999999999997e-06 2023-10-09T10:52:12.073 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.073 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.073 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.073 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765234+0000", 2023-10-09T10:52:12.073 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4699000000000001e-05 2023-10-09T10:52:12.073 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.073 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.074 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:12.074 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765367+0000", 2023-10-09T10:52:12.074 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013368399999999999 2023-10-09T10:52:12.074 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.074 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.074 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:52:12.074 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.765505+0000", 2023-10-09T10:52:12.074 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013755399999999999 2023-10-09T10:52:12.074 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.074 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.074 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:12.074 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.766674+0000", 2023-10-09T10:52:12.075 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011688899999999999 2023-10-09T10:52:12.075 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.075 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.075 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.075 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.771590+0000", 2023-10-09T10:52:12.075 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0049158149999999996 2023-10-09T10:52:12.075 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.075 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.075 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.075 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852204+0000", 2023-10-09T10:52:12.075 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.080614665000000002 2023-10-09T10:52:12.075 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.076 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.076 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:12.076 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852265+0000", 2023-10-09T10:52:12.076 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0838999999999998e-05 2023-10-09T10:52:12.076 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.076 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.076 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.076 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852311+0000", 2023-10-09T10:52:12.076 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6202000000000003e-05 2023-10-09T10:52:12.076 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.076 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.076 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.076 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.077 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.077 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:21948 7.0 7:0b124c27:::benchmark_data_smithi125_295816_object21947:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T10:52:12.077 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:29.773164+0000", 2023-10-09T10:52:12.077 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 162.263371049, 2023-10-09T10:52:12.077 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.079256148999999998, 2023-10-09T10:52:12.077 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.077 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:12.077 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.077 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.077 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.077 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 21948 2023-10-09T10:52:12.077 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.078 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.078 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.078 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.078 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773164+0000", 2023-10-09T10:52:12.078 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.078 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.078 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.078 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.078 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773164+0000", 2023-10-09T10:52:12.078 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.078 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.078 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.079 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.079 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773163+0000", 2023-10-09T10:52:12.079 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:12.079 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.079 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.079 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.079 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773231+0000", 2023-10-09T10:52:12.079 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7830000000000006e-05 2023-10-09T10:52:12.079 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.079 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.079 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.079 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773246+0000", 2023-10-09T10:52:12.079 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4868000000000001e-05 2023-10-09T10:52:12.080 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.080 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.080 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.080 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773253+0000", 2023-10-09T10:52:12.080 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6479999999999997e-06 2023-10-09T10:52:12.080 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.080 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.080 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.080 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773296+0000", 2023-10-09T10:52:12.080 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3763000000000001e-05 2023-10-09T10:52:12.080 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.080 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.081 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:12.081 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773431+0000", 2023-10-09T10:52:12.081 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013489400000000001 2023-10-09T10:52:12.081 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.081 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.081 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:52:12.081 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.773595+0000", 2023-10-09T10:52:12.081 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000163544 2023-10-09T10:52:12.081 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.081 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.081 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:12.081 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.774460+0000", 2023-10-09T10:52:12.081 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00086458399999999995 2023-10-09T10:52:12.082 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.082 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.082 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.082 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.783614+0000", 2023-10-09T10:52:12.082 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0091545570000000007 2023-10-09T10:52:12.082 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.082 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.082 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.082 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852352+0000", 2023-10-09T10:52:12.082 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.068737856 2023-10-09T10:52:12.082 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.082 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.083 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:12.083 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852383+0000", 2023-10-09T10:52:12.083 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1560000000000003e-05 2023-10-09T10:52:12.083 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.083 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.083 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.083 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852421+0000", 2023-10-09T10:52:12.083 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7061999999999999e-05 2023-10-09T10:52:12.083 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.083 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.083 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.083 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.084 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.084 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:21952 7.0 7:0825f8d2:::benchmark_data_smithi125_295816_object21951:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T10:52:12.084 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:29.778597+0000", 2023-10-09T10:52:12.084 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 162.25793887500001, 2023-10-09T10:52:12.084 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.073959639999999993, 2023-10-09T10:52:12.084 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.084 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:12.084 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.084 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.084 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.084 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 21952 2023-10-09T10:52:12.084 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.084 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.085 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.085 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.085 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778597+0000", 2023-10-09T10:52:12.085 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.085 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.085 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.085 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.085 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778597+0000", 2023-10-09T10:52:12.085 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.085 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.085 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.085 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.086 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778595+0000", 2023-10-09T10:52:12.086 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:12.086 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.086 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.086 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.086 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778636+0000", 2023-10-09T10:52:12.086 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1343e-05 2023-10-09T10:52:12.086 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.086 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.086 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.086 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778647+0000", 2023-10-09T10:52:12.086 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0971999999999999e-05 2023-10-09T10:52:12.087 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.087 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.087 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.087 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778653+0000", 2023-10-09T10:52:12.087 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5509999999999999e-06 2023-10-09T10:52:12.087 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.087 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.087 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.087 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778701+0000", 2023-10-09T10:52:12.087 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8219e-05 2023-10-09T10:52:12.087 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.087 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.088 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:12.088 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778789+0000", 2023-10-09T10:52:12.088 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7466999999999995e-05 2023-10-09T10:52:12.088 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.088 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.088 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:52:12.088 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.778920+0000", 2023-10-09T10:52:12.088 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013081 2023-10-09T10:52:12.088 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.088 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.088 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:12.088 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.779741+0000", 2023-10-09T10:52:12.088 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00082173999999999995 2023-10-09T10:52:12.089 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.089 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.089 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.089 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.783915+0000", 2023-10-09T10:52:12.089 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041741879999999997 2023-10-09T10:52:12.089 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.089 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.089 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.089 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852486+0000", 2023-10-09T10:52:12.089 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.068570476000000005 2023-10-09T10:52:12.089 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.089 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.090 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:12.090 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852533+0000", 2023-10-09T10:52:12.090 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7379000000000002e-05 2023-10-09T10:52:12.090 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.090 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.090 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.090 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:29.852556+0000", 2023-10-09T10:52:12.090 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2918999999999999e-05 2023-10-09T10:52:12.090 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.090 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.090 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.090 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.091 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.091 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96830 7.0 7:01f63212:::benchmark_data_smithi125_295816_object5693:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:52:12.091 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:59.910344+0000", 2023-10-09T10:52:12.091 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 72.126191864999996, 2023-10-09T10:52:12.091 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.111260499, 2023-10-09T10:52:12.091 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.091 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:12.091 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.091 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.091 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.091 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96830 2023-10-09T10:52:12.091 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.092 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.092 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.092 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.092 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910344+0000", 2023-10-09T10:52:12.092 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.092 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.092 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.092 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.092 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910344+0000", 2023-10-09T10:52:12.092 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.092 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.092 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.092 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.093 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910342+0000", 2023-10-09T10:52:12.093 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:12.093 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.093 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.093 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.093 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910348+0000", 2023-10-09T10:52:12.093 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9689999999999999e-06 2023-10-09T10:52:12.093 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.093 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.093 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.093 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910363+0000", 2023-10-09T10:52:12.093 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4228e-05 2023-10-09T10:52:12.094 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.094 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.094 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.094 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910369+0000", 2023-10-09T10:52:12.094 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2870000000000001e-06 2023-10-09T10:52:12.094 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.094 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.094 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.094 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910421+0000", 2023-10-09T10:52:12.094 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1722999999999999e-05 2023-10-09T10:52:12.094 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.094 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.095 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:12.095 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910605+0000", 2023-10-09T10:52:12.095 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000184296 2023-10-09T10:52:12.095 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.095 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.095 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:12.095 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910776+0000", 2023-10-09T10:52:12.095 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017150299999999999 2023-10-09T10:52:12.095 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.095 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.095 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:12.095 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.913518+0000", 2023-10-09T10:52:12.095 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0027416620000000002 2023-10-09T10:52:12.096 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.096 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.096 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.096 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.915768+0000", 2023-10-09T10:52:12.096 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022495950000000001 2023-10-09T10:52:12.096 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.096 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.096 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.096 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.019926+0000", 2023-10-09T10:52:12.096 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.104158473 2023-10-09T10:52:12.096 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.096 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.097 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:12.097 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.019973+0000", 2023-10-09T10:52:12.097 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6959999999999998e-05 2023-10-09T10:52:12.097 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.097 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.097 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.097 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.021604+0000", 2023-10-09T10:52:12.097 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0016310230000000001 2023-10-09T10:52:12.097 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.097 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.097 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.097 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.098 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.098 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:107471 7.0 7:0ef8873b:::benchmark_data_smithi125_295816_object16334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-10-09T10:52:12.098 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:15.564834+0000", 2023-10-09T10:52:12.098 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 56.471701150999998, 2023-10-09T10:52:12.098 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.097912184999999999, 2023-10-09T10:52:12.098 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.098 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:12.098 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.098 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.098 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.098 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 107471 2023-10-09T10:52:12.098 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.099 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.099 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.099 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.099 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:15.564834+0000", 2023-10-09T10:52:12.099 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.099 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.099 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.099 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.099 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:15.564834+0000", 2023-10-09T10:52:12.099 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.099 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.099 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.100 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.100 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:15.564833+0000", 2023-10-09T10:52:12.100 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:12.100 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.100 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.100 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.100 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:15.564842+0000", 2023-10-09T10:52:12.100 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3530000000000001e-06 2023-10-09T10:52:12.100 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.100 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.100 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.100 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:15.564859+0000", 2023-10-09T10:52:12.100 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7186000000000001e-05 2023-10-09T10:52:12.101 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.101 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.101 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.101 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:15.564867+0000", 2023-10-09T10:52:12.101 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5909999999999998e-06 2023-10-09T10:52:12.101 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.101 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.101 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.101 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:15.564945+0000", 2023-10-09T10:52:12.101 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8101000000000003e-05 2023-10-09T10:52:12.101 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.101 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.102 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:12.102 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:15.565099+0000", 2023-10-09T10:52:12.102 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000154494 2023-10-09T10:52:12.102 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.102 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.102 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:12.102 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:15.565214+0000", 2023-10-09T10:52:12.102 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011493300000000001 2023-10-09T10:52:12.102 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.102 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.102 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:12.102 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:15.567486+0000", 2023-10-09T10:52:12.103 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022712269999999998 2023-10-09T10:52:12.103 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.103 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.103 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.103 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:15.568886+0000", 2023-10-09T10:52:12.103 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001400311 2023-10-09T10:52:12.103 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.103 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.103 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.103 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:15.660577+0000", 2023-10-09T10:52:12.103 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.091691139000000005 2023-10-09T10:52:12.103 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.104 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.104 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:12.104 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:15.660640+0000", 2023-10-09T10:52:12.104 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3273000000000002e-05 2023-10-09T10:52:12.104 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.104 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.104 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.104 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:15.662746+0000", 2023-10-09T10:52:12.104 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021061859999999999 2023-10-09T10:52:12.104 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.104 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.104 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.104 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.105 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.105 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:130169 7.e 7:703cbc83:::benchmark_data_smithi125_295816_object39032:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e238)", 2023-10-09T10:52:12.105 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:43.539183+0000", 2023-10-09T10:52:12.105 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 28.497352486, 2023-10-09T10:52:12.105 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.083955833999999993, 2023-10-09T10:52:12.105 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.105 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:12.105 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.105 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.105 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.105 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 130169 2023-10-09T10:52:12.105 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.105 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.106 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.106 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.106 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.539183+0000", 2023-10-09T10:52:12.106 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.106 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.106 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.106 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.106 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.539183+0000", 2023-10-09T10:52:12.106 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.106 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.106 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.107 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.107 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.539182+0000", 2023-10-09T10:52:12.107 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:12.107 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.107 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.107 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.107 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.539187+0000", 2023-10-09T10:52:12.107 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7940000000000002e-06 2023-10-09T10:52:12.107 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.107 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.107 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.107 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.539195+0000", 2023-10-09T10:52:12.107 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0609999999999996e-06 2023-10-09T10:52:12.108 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.108 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.108 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.108 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.539199+0000", 2023-10-09T10:52:12.108 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1520000000000002e-06 2023-10-09T10:52:12.108 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.108 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.108 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.108 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.539230+0000", 2023-10-09T10:52:12.108 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0712e-05 2023-10-09T10:52:12.108 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.108 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.109 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:12.109 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.539310+0000", 2023-10-09T10:52:12.109 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0568999999999994e-05 2023-10-09T10:52:12.109 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.109 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.109 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:52:12.109 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.539380+0000", 2023-10-09T10:52:12.109 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9727000000000005e-05 2023-10-09T10:52:12.109 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.109 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.109 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:12.109 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.541330+0000", 2023-10-09T10:52:12.110 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019498499999999999 2023-10-09T10:52:12.110 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.110 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.110 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.110 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.571754+0000", 2023-10-09T10:52:12.110 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030424244999999999 2023-10-09T10:52:12.110 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.110 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.110 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.110 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.623083+0000", 2023-10-09T10:52:12.110 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0513283 2023-10-09T10:52:12.110 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.111 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.111 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:12.111 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.623115+0000", 2023-10-09T10:52:12.111 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2072000000000001e-05 2023-10-09T10:52:12.111 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.111 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.111 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.111 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:43.623139+0000", 2023-10-09T10:52:12.111 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4221000000000001e-05 2023-10-09T10:52:12.111 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.111 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.111 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.111 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.112 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.112 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:135658 7.0 7:043f6893:::benchmark_data_smithi125_295816_object44521:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e238)", 2023-10-09T10:52:12.112 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:49.348823+0000", 2023-10-09T10:52:12.112 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 22.687712663999999, 2023-10-09T10:52:12.112 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0192536720000001, 2023-10-09T10:52:12.112 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.112 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T10:52:12.112 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.112 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.112 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.112 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135658 2023-10-09T10:52:12.112 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.113 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.113 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.113 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.113 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.348823+0000", 2023-10-09T10:52:12.113 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.113 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.113 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.113 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.113 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.348823+0000", 2023-10-09T10:52:12.113 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.113 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.113 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.113 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.114 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.348822+0000", 2023-10-09T10:52:12.114 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:12.114 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.114 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.114 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.114 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.348828+0000", 2023-10-09T10:52:12.114 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1199999999999999e-06 2023-10-09T10:52:12.114 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.114 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.114 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.114 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.348839+0000", 2023-10-09T10:52:12.114 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1522000000000001e-05 2023-10-09T10:52:12.115 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.115 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.115 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.115 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.348846+0000", 2023-10-09T10:52:12.115 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.297e-06 2023-10-09T10:52:12.115 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.115 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.115 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.115 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.349013+0000", 2023-10-09T10:52:12.115 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016718200000000001 2023-10-09T10:52:12.115 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.115 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.115 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:12.116 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.349181+0000", 2023-10-09T10:52:12.116 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000168171 2023-10-09T10:52:12.116 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.116 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.116 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:52:12.116 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.349346+0000", 2023-10-09T10:52:12.116 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000165518 2023-10-09T10:52:12.116 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.116 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.116 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:12.116 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.351916+0000", 2023-10-09T10:52:12.116 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002569572 2023-10-09T10:52:12.117 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.117 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.117 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:12.117 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.354528+0000", 2023-10-09T10:52:12.117 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026118399999999998 2023-10-09T10:52:12.117 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.117 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.117 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.117 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.365760+0000", 2023-10-09T10:52:12.117 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011232205 2023-10-09T10:52:12.117 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.117 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.118 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:12.118 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.365771+0000", 2023-10-09T10:52:12.118 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1326e-05 2023-10-09T10:52:12.118 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.118 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.118 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.118 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368062+0000", 2023-10-09T10:52:12.118 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.002290425 2023-10-09T10:52:12.118 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.118 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.118 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.118 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368076+0000", 2023-10-09T10:52:12.119 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4696e-05 2023-10-09T10:52:12.119 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.119 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.119 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.119 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.119 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.119 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:135667 7.0 7.37f62310 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e238)", 2023-10-09T10:52:12.119 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:49.359431+0000", 2023-10-09T10:52:12.119 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 22.677104242999999, 2023-10-09T10:52:12.119 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0086783500000001, 2023-10-09T10:52:12.119 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.119 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T10:52:12.120 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.120 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.120 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.120 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135667 2023-10-09T10:52:12.120 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.120 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.120 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.120 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.120 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.359431+0000", 2023-10-09T10:52:12.120 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.120 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.120 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.120 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.121 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.359431+0000", 2023-10-09T10:52:12.121 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.121 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.121 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.121 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.121 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.359430+0000", 2023-10-09T10:52:12.121 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:12.121 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.121 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.121 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.121 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.359437+0000", 2023-10-09T10:52:12.121 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0029999999999997e-06 2023-10-09T10:52:12.121 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.122 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.122 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.122 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.360049+0000", 2023-10-09T10:52:12.122 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00061261099999999999 2023-10-09T10:52:12.122 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.122 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.122 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.122 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.360058+0000", 2023-10-09T10:52:12.122 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6000000000000007e-06 2023-10-09T10:52:12.122 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.122 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.122 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.123 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.366286+0000", 2023-10-09T10:52:12.123 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062284719999999997 2023-10-09T10:52:12.123 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.123 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.123 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:12.123 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.366295+0000", 2023-10-09T10:52:12.123 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7879999999999995e-06 2023-10-09T10:52:12.123 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.123 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.123 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.123 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368101+0000", 2023-10-09T10:52:12.123 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.001805447 2023-10-09T10:52:12.123 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.124 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.124 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.124 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368110+0000", 2023-10-09T10:52:12.124 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9309999999999997e-06 2023-10-09T10:52:12.124 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.124 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.124 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.124 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.124 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.124 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:135669 7.0 7.d6038d50 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e238)", 2023-10-09T10:52:12.124 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:49.365933+0000", 2023-10-09T10:52:12.124 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 22.670602506000002, 2023-10-09T10:52:12.124 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.00220737, 2023-10-09T10:52:12.125 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.125 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T10:52:12.125 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.125 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.125 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.125 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135669 2023-10-09T10:52:12.125 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.125 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.125 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.125 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.125 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.365933+0000", 2023-10-09T10:52:12.125 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.125 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.126 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.126 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.126 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.365933+0000", 2023-10-09T10:52:12.126 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.126 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.126 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.126 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.126 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.365931+0000", 2023-10-09T10:52:12.126 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:12.126 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.126 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.126 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.127 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.365939+0000", 2023-10-09T10:52:12.127 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6009999999999997e-06 2023-10-09T10:52:12.127 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.127 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.127 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.127 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.365952+0000", 2023-10-09T10:52:12.127 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2677000000000001e-05 2023-10-09T10:52:12.127 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.127 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.127 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.127 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.365959+0000", 2023-10-09T10:52:12.127 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2810000000000003e-06 2023-10-09T10:52:12.127 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.128 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.128 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.128 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.366454+0000", 2023-10-09T10:52:12.128 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00049474899999999995 2023-10-09T10:52:12.128 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.128 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.128 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:12.128 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.366461+0000", 2023-10-09T10:52:12.128 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4490000000000002e-06 2023-10-09T10:52:12.128 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.128 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.128 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.128 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368131+0000", 2023-10-09T10:52:12.129 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0016699739999999 2023-10-09T10:52:12.129 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.129 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.129 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.129 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368140+0000", 2023-10-09T10:52:12.129 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1749999999999994e-06 2023-10-09T10:52:12.129 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.129 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.129 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.129 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.129 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.129 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:135680 7.0 7.4df1cea0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e239)", 2023-10-09T10:52:12.130 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:49.390019+0000", 2023-10-09T10:52:12.130 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 22.646516257999998, 2023-10-09T10:52:12.130 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97814997199999998, 2023-10-09T10:52:12.130 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.130 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T10:52:12.130 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.130 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.130 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.130 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135680 2023-10-09T10:52:12.130 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.130 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.130 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.131 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.131 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.390019+0000", 2023-10-09T10:52:12.131 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.131 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.131 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.131 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.131 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.390019+0000", 2023-10-09T10:52:12.131 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.131 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.131 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.131 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.131 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.390018+0000", 2023-10-09T10:52:12.131 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:12.132 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.132 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.132 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.132 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.390023+0000", 2023-10-09T10:52:12.132 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5709999999999997e-06 2023-10-09T10:52:12.132 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.132 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.132 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.132 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.390035+0000", 2023-10-09T10:52:12.132 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1915e-05 2023-10-09T10:52:12.132 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.132 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.133 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.133 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.390041+0000", 2023-10-09T10:52:12.133 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3240000000000002e-06 2023-10-09T10:52:12.133 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.133 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.133 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.133 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.390080+0000", 2023-10-09T10:52:12.133 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9147999999999999e-05 2023-10-09T10:52:12.133 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.133 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.133 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:12.133 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.390087+0000", 2023-10-09T10:52:12.133 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.447e-06 2023-10-09T10:52:12.134 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.134 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.134 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.134 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368160+0000", 2023-10-09T10:52:12.134 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.978072469 2023-10-09T10:52:12.134 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.134 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.134 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.134 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368169+0000", 2023-10-09T10:52:12.134 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3680000000000008e-06 2023-10-09T10:52:12.134 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.134 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.135 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.135 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.135 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.135 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:135681 7.0 7.513fa580 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e239)", 2023-10-09T10:52:12.135 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:49.392515+0000", 2023-10-09T10:52:12.135 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 22.644020545, 2023-10-09T10:52:12.135 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97568450299999998, 2023-10-09T10:52:12.135 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.135 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T10:52:12.135 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.135 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.135 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.135 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135681 2023-10-09T10:52:12.136 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.136 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.136 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.136 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.136 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.392515+0000", 2023-10-09T10:52:12.136 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.136 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.136 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.136 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.136 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.392515+0000", 2023-10-09T10:52:12.136 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.136 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.137 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.137 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.137 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.392514+0000", 2023-10-09T10:52:12.137 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:52:12.137 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.137 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.137 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.137 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.392519+0000", 2023-10-09T10:52:12.137 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7099999999999998e-06 2023-10-09T10:52:12.137 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.137 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.138 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.138 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.392530+0000", 2023-10-09T10:52:12.138 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.066e-05 2023-10-09T10:52:12.138 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.138 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.138 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.138 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.392535+0000", 2023-10-09T10:52:12.138 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6330000000000001e-06 2023-10-09T10:52:12.138 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.138 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.138 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.138 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.392574+0000", 2023-10-09T10:52:12.139 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9286000000000003e-05 2023-10-09T10:52:12.139 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.139 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.139 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:12.139 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.392581+0000", 2023-10-09T10:52:12.139 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6510000000000004e-06 2023-10-09T10:52:12.139 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.139 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.139 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.139 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368192+0000", 2023-10-09T10:52:12.139 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.975610541 2023-10-09T10:52:12.139 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.140 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.140 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.140 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368199+0000", 2023-10-09T10:52:12.140 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7330000000000003e-06 2023-10-09T10:52:12.140 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.140 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.140 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.140 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.140 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.140 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:135714 7.0 7.2eb43610 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e239)", 2023-10-09T10:52:12.140 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:49.447405+0000", 2023-10-09T10:52:12.140 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 22.589129961000001, 2023-10-09T10:52:12.140 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92082156000000004, 2023-10-09T10:52:12.141 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.141 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T10:52:12.141 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.141 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.141 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.141 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135714 2023-10-09T10:52:12.141 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.141 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.141 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.141 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.141 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.447405+0000", 2023-10-09T10:52:12.141 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.142 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.142 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.142 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.142 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.447405+0000", 2023-10-09T10:52:12.142 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.142 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.142 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.142 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.142 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.447404+0000", 2023-10-09T10:52:12.142 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:12.142 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.142 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.143 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.143 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.447414+0000", 2023-10-09T10:52:12.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5599999999999999e-06 2023-10-09T10:52:12.143 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.143 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.143 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.143 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.447433+0000", 2023-10-09T10:52:12.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9647999999999999e-05 2023-10-09T10:52:12.143 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.143 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.143 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.143 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.447442+0000", 2023-10-09T10:52:12.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.456e-06 2023-10-09T10:52:12.144 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.144 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.144 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.144 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.447521+0000", 2023-10-09T10:52:12.144 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9751000000000003e-05 2023-10-09T10:52:12.144 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.144 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.144 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:12.144 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.447533+0000", 2023-10-09T10:52:12.144 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1762e-05 2023-10-09T10:52:12.144 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.144 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.144 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.145 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368219+0000", 2023-10-09T10:52:12.145 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92068601900000002 2023-10-09T10:52:12.145 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.145 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.145 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.145 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368227+0000", 2023-10-09T10:52:12.145 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8849999999999999e-06 2023-10-09T10:52:12.145 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.145 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.145 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.145 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.145 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.146 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:135716 7.0 7.9ac401e0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e239)", 2023-10-09T10:52:12.146 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:49.456288+0000", 2023-10-09T10:52:12.146 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 22.580247436000001, 2023-10-09T10:52:12.146 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.91196445900000001, 2023-10-09T10:52:12.146 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.146 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T10:52:12.146 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.146 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.146 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.146 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135716 2023-10-09T10:52:12.146 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.146 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.146 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.147 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.147 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.456288+0000", 2023-10-09T10:52:12.147 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.147 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.147 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.147 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.147 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.456288+0000", 2023-10-09T10:52:12.147 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.147 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.147 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.147 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.147 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.456287+0000", 2023-10-09T10:52:12.148 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:12.148 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.148 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.148 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.148 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.456294+0000", 2023-10-09T10:52:12.148 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0330000000000003e-06 2023-10-09T10:52:12.148 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.148 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.148 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.148 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.456308+0000", 2023-10-09T10:52:12.148 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4575e-05 2023-10-09T10:52:12.148 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.149 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.149 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.149 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.456317+0000", 2023-10-09T10:52:12.149 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5420000000000005e-06 2023-10-09T10:52:12.149 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.149 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.149 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.149 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.456378+0000", 2023-10-09T10:52:12.149 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0959999999999999e-05 2023-10-09T10:52:12.149 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.149 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.149 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:12.150 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.456386+0000", 2023-10-09T10:52:12.150 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7500000000000003e-06 2023-10-09T10:52:12.150 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.150 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.150 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.150 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368244+0000", 2023-10-09T10:52:12.150 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.91185882900000004 2023-10-09T10:52:12.150 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.150 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.150 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.150 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368252+0000", 2023-10-09T10:52:12.150 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9840000000000001e-06 2023-10-09T10:52:12.150 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.151 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.151 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.151 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.151 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.151 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:135718 7.0 7.a783deb0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e239)", 2023-10-09T10:52:12.151 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:49.459686+0000", 2023-10-09T10:52:12.151 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 22.57684939, 2023-10-09T10:52:12.151 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.90859148300000003, 2023-10-09T10:52:12.151 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.151 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T10:52:12.151 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.151 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.152 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.152 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135718 2023-10-09T10:52:12.152 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.152 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.152 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.152 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.152 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.459686+0000", 2023-10-09T10:52:12.152 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.152 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.152 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.152 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.152 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.459686+0000", 2023-10-09T10:52:12.152 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.153 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.153 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.153 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.153 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.459685+0000", 2023-10-09T10:52:12.153 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:12.153 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.153 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.153 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.153 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.459690+0000", 2023-10-09T10:52:12.153 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5770000000000004e-06 2023-10-09T10:52:12.153 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.153 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.154 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.154 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.459699+0000", 2023-10-09T10:52:12.154 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4609999999999997e-06 2023-10-09T10:52:12.154 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.154 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.154 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.154 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.459703+0000", 2023-10-09T10:52:12.154 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2200000000000003e-06 2023-10-09T10:52:12.154 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.154 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.154 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.154 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.459757+0000", 2023-10-09T10:52:12.155 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3866999999999998e-05 2023-10-09T10:52:12.155 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.155 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.155 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:12.155 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.459763+0000", 2023-10-09T10:52:12.155 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.818e-06 2023-10-09T10:52:12.155 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.155 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.155 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.155 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368269+0000", 2023-10-09T10:52:12.155 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.908506178 2023-10-09T10:52:12.155 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.155 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.156 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.156 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368278+0000", 2023-10-09T10:52:12.156 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1810000000000001e-06 2023-10-09T10:52:12.156 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.156 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.156 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.156 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.156 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.156 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:135732 7.0 7.19aca990 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e239)", 2023-10-09T10:52:12.156 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:49.493582+0000", 2023-10-09T10:52:12.156 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 22.542953921999999, 2023-10-09T10:52:12.156 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.87472585899999999, 2023-10-09T10:52:12.156 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.157 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T10:52:12.157 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.157 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.157 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.157 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135732 2023-10-09T10:52:12.157 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.157 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.157 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.157 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.157 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.493582+0000", 2023-10-09T10:52:12.157 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.157 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.158 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.158 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.158 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.493582+0000", 2023-10-09T10:52:12.158 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.158 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.158 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.158 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.158 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.493580+0000", 2023-10-09T10:52:12.158 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:12.158 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.158 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.158 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.158 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.493585+0000", 2023-10-09T10:52:12.159 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.865e-06 2023-10-09T10:52:12.159 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.159 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.159 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.159 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.493596+0000", 2023-10-09T10:52:12.159 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0608000000000001e-05 2023-10-09T10:52:12.159 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.159 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.159 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.159 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.493601+0000", 2023-10-09T10:52:12.159 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2440000000000001e-06 2023-10-09T10:52:12.159 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.160 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.160 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.160 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.493647+0000", 2023-10-09T10:52:12.160 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5788000000000003e-05 2023-10-09T10:52:12.160 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.160 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.160 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:12.160 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.493654+0000", 2023-10-09T10:52:12.160 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.968e-06 2023-10-09T10:52:12.160 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.160 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.160 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.161 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368299+0000", 2023-10-09T10:52:12.161 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.874644957 2023-10-09T10:52:12.161 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.161 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.161 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.161 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368307+0000", 2023-10-09T10:52:12.161 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4940000000000003e-06 2023-10-09T10:52:12.161 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.161 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.161 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.161 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.161 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.162 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:135741 7.0 7.b05b7660 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e239)", 2023-10-09T10:52:12.162 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:49.529222+0000", 2023-10-09T10:52:12.162 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 22.507313692, 2023-10-09T10:52:12.162 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.83911106800000002, 2023-10-09T10:52:12.162 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:12.162 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T10:52:12.162 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:12.162 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:52:12.162 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:12.162 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135741 2023-10-09T10:52:12.162 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.162 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:12.163 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.163 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:12.163 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.529222+0000", 2023-10-09T10:52:12.163 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.163 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.163 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.163 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:12.163 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.529222+0000", 2023-10-09T10:52:12.163 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:12.163 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.163 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.163 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:12.164 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.529221+0000", 2023-10-09T10:52:12.164 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:12.164 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.164 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.164 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:12.164 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.529226+0000", 2023-10-09T10:52:12.164 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8740000000000003e-06 2023-10-09T10:52:12.164 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.164 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.164 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:12.164 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.529237+0000", 2023-10-09T10:52:12.164 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.093e-05 2023-10-09T10:52:12.165 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.165 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.165 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:12.165 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.529241+0000", 2023-10-09T10:52:12.165 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.476e-06 2023-10-09T10:52:12.165 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.165 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.165 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.165 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.529293+0000", 2023-10-09T10:52:12.165 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1906000000000001e-05 2023-10-09T10:52:12.165 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.165 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.165 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:52:12.166 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:49.529304+0000", 2023-10-09T10:52:12.166 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0978999999999999e-05 2023-10-09T10:52:12.166 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.166 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.166 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:12.166 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368324+0000", 2023-10-09T10:52:12.166 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.83902029099999997 2023-10-09T10:52:12.166 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:12.166 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:12.166 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:12.166 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:50.368333+0000", 2023-10-09T10:52:12.166 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6279999999999994e-06 2023-10-09T10:52:12.167 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.167 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.167 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.167 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:12.167 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:12.167 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:52:12.202 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:12.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:12.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:12.315 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:12.363 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:12.405 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:12.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:12.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:12.432 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:52:12.514 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:52:12.563 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:12.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[195507]: pgmap v575: 119 pgs: 5 remapped+peering, 12 peering, 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.9 GiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 754 op/s; 12604/91572 objects degraded (13.764%); 3 B/s, 37 objects/s recovering 2023-10-09T10:52:12.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[195507]: failed to encode map e246 with expected crc 2023-10-09T10:52:12.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[195507]: 7.0 starting backfill to osd.1 from (0'0,207'3175] MAX to 243'9570 2023-10-09T10:52:12.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[195507]: failed to encode map e246 with expected crc 2023-10-09T10:52:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[195507]: 7.2 starting backfill to osd.1 from (0'0,207'3221] MIN to 243'9609 2023-10-09T10:52:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[195507]: osdmap e246: 8 total, 8 up, 6 in 2023-10-09T10:52:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[195507]: 7.a starting backfill to osd.1 from (0'0,207'3125] MAX to 243'9509 2023-10-09T10:52:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[195507]: 7.9 starting backfill to osd.1 from (0'0,207'3263] MAX to 243'9814 2023-10-09T10:52:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[195507]: failed to encode map e246 with expected crc 2023-10-09T10:52:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[195507]: 7.d starting backfill to osd.1 from (0'0,207'3118] MAX to 243'9414 2023-10-09T10:52:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[195507]: 7.4 starting backfill to osd.1 from (0'0,207'3083] MAX to 243'9541 2023-10-09T10:52:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[195507]: 7.b starting backfill to osd.1 from (0'0,207'3165] MIN to 243'9558 2023-10-09T10:52:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:12.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[200197]: pgmap v575: 119 pgs: 5 remapped+peering, 12 peering, 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.9 GiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 754 op/s; 12604/91572 objects degraded (13.764%); 3 B/s, 37 objects/s recovering 2023-10-09T10:52:12.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[200197]: failed to encode map e246 with expected crc 2023-10-09T10:52:12.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[200197]: 7.0 starting backfill to osd.1 from (0'0,207'3175] MAX to 243'9570 2023-10-09T10:52:12.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[200197]: failed to encode map e246 with expected crc 2023-10-09T10:52:12.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[200197]: 7.2 starting backfill to osd.1 from (0'0,207'3221] MIN to 243'9609 2023-10-09T10:52:12.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[200197]: osdmap e246: 8 total, 8 up, 6 in 2023-10-09T10:52:12.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[200197]: 7.a starting backfill to osd.1 from (0'0,207'3125] MAX to 243'9509 2023-10-09T10:52:12.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[200197]: 7.9 starting backfill to osd.1 from (0'0,207'3263] MAX to 243'9814 2023-10-09T10:52:12.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[200197]: failed to encode map e246 with expected crc 2023-10-09T10:52:12.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[200197]: 7.d starting backfill to osd.1 from (0'0,207'3118] MAX to 243'9414 2023-10-09T10:52:12.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[200197]: 7.4 starting backfill to osd.1 from (0'0,207'3083] MAX to 243'9541 2023-10-09T10:52:12.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[200197]: 7.b starting backfill to osd.1 from (0'0,207'3165] MIN to 243'9558 2023-10-09T10:52:12.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:12.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:12 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:12.605 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:52:12.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:12.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:52:12.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:12 smithi155 ceph-mon[184854]: pgmap v575: 119 pgs: 5 remapped+peering, 12 peering, 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.9 GiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 754 op/s; 12604/91572 objects degraded (13.764%); 3 B/s, 37 objects/s recovering 2023-10-09T10:52:12.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:12 smithi155 ceph-mon[184854]: failed to encode map e246 with expected crc 2023-10-09T10:52:12.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:12 smithi155 ceph-mon[184854]: 7.0 starting backfill to osd.1 from (0'0,207'3175] MAX to 243'9570 2023-10-09T10:52:12.610 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:12 smithi155 ceph-mon[184854]: failed to encode map e246 with expected crc 2023-10-09T10:52:12.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:12 smithi155 ceph-mon[184854]: 7.2 starting backfill to osd.1 from (0'0,207'3221] MIN to 243'9609 2023-10-09T10:52:12.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:12 smithi155 ceph-mon[184854]: osdmap e246: 8 total, 8 up, 6 in 2023-10-09T10:52:12.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:12 smithi155 ceph-mon[184854]: 7.a starting backfill to osd.1 from (0'0,207'3125] MAX to 243'9509 2023-10-09T10:52:12.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:12 smithi155 ceph-mon[184854]: 7.9 starting backfill to osd.1 from (0'0,207'3263] MAX to 243'9814 2023-10-09T10:52:12.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:12 smithi155 ceph-mon[184854]: failed to encode map e246 with expected crc 2023-10-09T10:52:12.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:12 smithi155 ceph-mon[184854]: 7.d starting backfill to osd.1 from (0'0,207'3118] MAX to 243'9414 2023-10-09T10:52:12.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:12 smithi155 ceph-mon[184854]: 7.4 starting backfill to osd.1 from (0'0,207'3083] MAX to 243'9541 2023-10-09T10:52:12.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:12 smithi155 ceph-mon[184854]: 7.b starting backfill to osd.1 from (0'0,207'3165] MIN to 243'9558 2023-10-09T10:52:12.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:12 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:12.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:12 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:12.711 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:12.762 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:12.792 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:12.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:12.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:12.898 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:52:12.910 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:52:12.952 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:12.995 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:52:12.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:13.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:52:13.100 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:13.158 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:13.192 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:13.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:13.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:13.296 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:52:13.369 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:13.603 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:13 smithi155 ceph-mon[184854]: failed to encode map e246 with expected crc 2023-10-09T10:52:13.603 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:13 smithi155 ceph-mon[184854]: osdmap e247: 8 total, 8 up, 6 in 2023-10-09T10:52:13.603 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:13 smithi155 ceph-mon[184854]: failed to encode map e247 with expected crc 2023-10-09T10:52:13.603 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:13 smithi155 ceph-mon[184854]: failed to encode map e247 with expected crc 2023-10-09T10:52:13.603 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:13 smithi155 ceph-mon[184854]: failed to encode map e247 with expected crc 2023-10-09T10:52:13.603 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:13 smithi155 ceph-mon[184854]: failed to encode map e247 with expected crc 2023-10-09T10:52:13.603 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:13 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 12604/91572 objects degraded (13.764%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T10:52:13.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:13 smithi125 ceph-mon[195507]: failed to encode map e246 with expected crc 2023-10-09T10:52:13.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:13 smithi125 ceph-mon[195507]: osdmap e247: 8 total, 8 up, 6 in 2023-10-09T10:52:13.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:13 smithi125 ceph-mon[195507]: failed to encode map e247 with expected crc 2023-10-09T10:52:13.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:13 smithi125 ceph-mon[195507]: failed to encode map e247 with expected crc 2023-10-09T10:52:13.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:13 smithi125 ceph-mon[195507]: failed to encode map e247 with expected crc 2023-10-09T10:52:13.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:13 smithi125 ceph-mon[195507]: failed to encode map e247 with expected crc 2023-10-09T10:52:13.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:13 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 12604/91572 objects degraded (13.764%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T10:52:13.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:13 smithi125 ceph-mon[200197]: failed to encode map e246 with expected crc 2023-10-09T10:52:13.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:13 smithi125 ceph-mon[200197]: osdmap e247: 8 total, 8 up, 6 in 2023-10-09T10:52:13.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:13 smithi125 ceph-mon[200197]: failed to encode map e247 with expected crc 2023-10-09T10:52:13.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:13 smithi125 ceph-mon[200197]: failed to encode map e247 with expected crc 2023-10-09T10:52:13.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:13 smithi125 ceph-mon[200197]: failed to encode map e247 with expected crc 2023-10-09T10:52:13.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:13 smithi125 ceph-mon[200197]: failed to encode map e247 with expected crc 2023-10-09T10:52:13.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:13 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 12604/91572 objects degraded (13.764%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T10:52:14.194 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:52:14.194 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:52:14.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.195 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35187.0:154520 7.b e248/245 7:db8910c9:::benchmark_data_smithi125_295816_object63383:head v 248'9680, mlcod=248'9678)", 2023-10-09T10:52:14.195 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:52:14.190378+0000", 2023-10-09T10:52:14.195 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.003150791, 2023-10-09T10:52:14.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0031785860000000002, 2023-10-09T10:52:14.195 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:14.195 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:52:14.195 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:14.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:14.195 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.190378+0000", 2023-10-09T10:52:14.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:14.196 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.196 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.196 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:14.196 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.190378+0000", 2023-10-09T10:52:14.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:14.196 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.196 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.196 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:14.196 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.190378+0000", 2023-10-09T10:52:14.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:52:14.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.197 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:14.197 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.190383+0000", 2023-10-09T10:52:14.197 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8409999999999999e-06 2023-10-09T10:52:14.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.197 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:14.197 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.190393+0000", 2023-10-09T10:52:14.197 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0553e-05 2023-10-09T10:52:14.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.198 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:14.198 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.190398+0000", 2023-10-09T10:52:14.198 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9450000000000001e-06 2023-10-09T10:52:14.198 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.198 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.198 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:14.198 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.192503+0000", 2023-10-09T10:52:14.198 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002104308 2023-10-09T10:52:14.198 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.198 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.198 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:14.198 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.192539+0000", 2023-10-09T10:52:14.199 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6319999999999998e-05 2023-10-09T10:52:14.199 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:14.199 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:14.199 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:14.199 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.199 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.199 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35187.0:154516 7.c e248/215 7:36d09d9e:::benchmark_data_smithi125_295816_object63379:head v 248'9528, mlcod=248'9525)", 2023-10-09T10:52:14.199 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:52:14.191704+0000", 2023-10-09T10:52:14.199 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0018247859999999999, 2023-10-09T10:52:14.199 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001918054, 2023-10-09T10:52:14.199 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:14.199 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:52:14.200 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:14.200 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.200 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:14.200 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.191704+0000", 2023-10-09T10:52:14.200 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:14.200 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.200 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.200 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:14.200 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.191704+0000", 2023-10-09T10:52:14.200 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:14.200 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.200 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:14.201 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.191704+0000", 2023-10-09T10:52:14.201 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:14.201 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:14.201 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.191710+0000", 2023-10-09T10:52:14.201 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.392e-06 2023-10-09T10:52:14.201 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:14.201 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.191724+0000", 2023-10-09T10:52:14.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3869999999999999e-05 2023-10-09T10:52:14.202 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.202 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.202 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:14.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.191730+0000", 2023-10-09T10:52:14.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0299999999999999e-06 2023-10-09T10:52:14.202 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.202 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.202 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:14.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.192814+0000", 2023-10-09T10:52:14.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010843599999999999 2023-10-09T10:52:14.203 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.203 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.203 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:14.203 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.192833+0000", 2023-10-09T10:52:14.203 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8567999999999999e-05 2023-10-09T10:52:14.203 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:14.203 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:14.203 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:14.203 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.203 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.203 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35187.0:154522 7.b e248/245)", 2023-10-09T10:52:14.203 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:52:14.193150+0000", 2023-10-09T10:52:14.204 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.00037878899999999998, 2023-10-09T10:52:14.204 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000527412, 2023-10-09T10:52:14.204 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:14.204 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "queued for pg", 2023-10-09T10:52:14.204 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:14.204 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.204 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:14.204 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.193150+0000", 2023-10-09T10:52:14.204 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:14.204 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.204 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.204 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:14.205 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.193150+0000", 2023-10-09T10:52:14.205 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:14.205 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.205 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.205 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:14.205 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.193150+0000", 2023-10-09T10:52:14.205 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:52:14.205 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.205 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.205 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:14.205 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.193157+0000", 2023-10-09T10:52:14.205 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7499999999999997e-06 2023-10-09T10:52:14.206 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.206 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.206 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:14.206 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.193167+0000", 2023-10-09T10:52:14.206 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0139000000000001e-05 2023-10-09T10:52:14.206 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:14.206 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:14.206 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:14.206 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:14.193172+0000", 2023-10-09T10:52:14.206 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6869999999999999e-06 2023-10-09T10:52:14.206 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:14.206 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:14.207 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:14.207 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:14.207 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T10:52:14.207 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 3 2023-10-09T10:52:14.207 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:52:14.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:14 smithi155 ceph-mon[184854]: pgmap v578: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 5 remapped+peering, 13 peering, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 94 active+clean; 1.8 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 647 op/s; 10921/88506 objects degraded (12.339%); 1709/88506 objects misplaced (1.931%); 5 B/s, 277 objects/s recovering 2023-10-09T10:52:14.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:14 smithi155 ceph-mon[184854]: osdmap e248: 8 total, 8 up, 6 in 2023-10-09T10:52:14.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:14 smithi155 ceph-mon[184854]: failed to encode map e248 with expected crc 2023-10-09T10:52:14.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:14 smithi155 ceph-mon[184854]: failed to encode map e248 with expected crc 2023-10-09T10:52:14.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:14 smithi155 ceph-mon[184854]: failed to encode map e248 with expected crc 2023-10-09T10:52:14.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:14 smithi155 ceph-mon[184854]: failed to encode map e248 with expected crc 2023-10-09T10:52:14.585 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:52:14.687 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:52:14.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:14.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:52:14.694 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:14.697 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:14 smithi125 ceph-mon[195507]: pgmap v578: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 5 remapped+peering, 13 peering, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 94 active+clean; 1.8 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 647 op/s; 10921/88506 objects degraded (12.339%); 1709/88506 objects misplaced (1.931%); 5 B/s, 277 objects/s recovering 2023-10-09T10:52:14.697 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:14 smithi125 ceph-mon[195507]: osdmap e248: 8 total, 8 up, 6 in 2023-10-09T10:52:14.697 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:14 smithi125 ceph-mon[195507]: failed to encode map e248 with expected crc 2023-10-09T10:52:14.697 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:14 smithi125 ceph-mon[195507]: failed to encode map e248 with expected crc 2023-10-09T10:52:14.697 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:14 smithi125 ceph-mon[195507]: failed to encode map e248 with expected crc 2023-10-09T10:52:14.697 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:14 smithi125 ceph-mon[195507]: failed to encode map e248 with expected crc 2023-10-09T10:52:14.697 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:14 smithi125 ceph-mon[200197]: pgmap v578: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 5 remapped+peering, 13 peering, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 94 active+clean; 1.8 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 647 op/s; 10921/88506 objects degraded (12.339%); 1709/88506 objects misplaced (1.931%); 5 B/s, 277 objects/s recovering 2023-10-09T10:52:14.698 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:14 smithi125 ceph-mon[200197]: osdmap e248: 8 total, 8 up, 6 in 2023-10-09T10:52:14.698 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:14 smithi125 ceph-mon[200197]: failed to encode map e248 with expected crc 2023-10-09T10:52:14.698 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:14 smithi125 ceph-mon[200197]: failed to encode map e248 with expected crc 2023-10-09T10:52:14.698 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:14 smithi125 ceph-mon[200197]: failed to encode map e248 with expected crc 2023-10-09T10:52:14.698 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:14 smithi125 ceph-mon[200197]: failed to encode map e248 with expected crc 2023-10-09T10:52:14.726 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-10-09T10:52:14.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:14.856 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:14.894 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:14.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:14.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:14.997 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:52:15.030 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:52:15.059 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:15.092 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:52:15.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:15.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:52:15.198 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:52:15.263 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:15.447 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:52:15.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:15.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:52:15.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:52:15.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:52:15.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:52:15.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:15.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:15.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:52:15.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:52:15.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:52:15.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:15.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:15.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:52:15.610 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:15.690 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:52:15.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:15.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:52:15.696 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:52:15.696 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:52:15.697 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:15.697 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:15.697 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:52:15.796 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:15.845 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:15.891 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:15.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:15.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:15.996 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:16.021 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:52:16.054 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:52:16.068 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:16.096 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:52:16.141 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:52:16.181 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:52:16.185 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:52:16.185 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:52:16.185 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:52:16.185 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:52:16.185 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:52:16.206 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:52:16.236 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:52:16.264 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:52:16.277 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:52:16.277 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:52:16.277 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:52:16.277 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:52:16.278 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:52:16.278 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:52:16.278 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:52:16.278 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:52:16.415 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:16.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:16.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:16.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:16 smithi155 ceph-mon[184854]: pgmap v580: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 4 remapped+peering, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 105 active+clean; 1.7 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 538 op/s; 11511/84693 objects degraded (13.591%); 3457/84693 objects misplaced (4.082%); 4 B/s, 224 objects/s recovering 2023-10-09T10:52:16.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:16 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/519055367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:16.518 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:16.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:52:16.566 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:16 smithi125 ceph-mon[195507]: pgmap v580: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 4 remapped+peering, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 105 active+clean; 1.7 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 538 op/s; 11511/84693 objects degraded (13.591%); 3457/84693 objects misplaced (4.082%); 4 B/s, 224 objects/s recovering 2023-10-09T10:52:16.566 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:16 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/519055367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:16.567 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:16 smithi125 ceph-mon[200197]: pgmap v580: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 4 remapped+peering, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 105 active+clean; 1.7 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 538 op/s; 11511/84693 objects degraded (13.591%); 3457/84693 objects misplaced (4.082%); 4 B/s, 224 objects/s recovering 2023-10-09T10:52:16.567 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:16 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/519055367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:16.568 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:16.696 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:16.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:16.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:16.800 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:16.852 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:16.905 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:16.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:16.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:16.979 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:52:17.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:17.067 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:17.108 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:17.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:17.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:17.262 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:17.326 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:17.362 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:17.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:17.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:17.466 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:52:17.510 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:17.551 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:52:17.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:17.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:52:17.656 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:52:17.710 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:17.739 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:52:17.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:17.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:52:17.851 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:52:17.926 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:17.956 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:52:17.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:17.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:52:18.065 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:18.118 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:18.163 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:18.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:18.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:18.267 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:18.328 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:18.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:18.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:18.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:18.459 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:52:18.459 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:52:18.522 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:18.559 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:52:18.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:18.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:52:18.663 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:18.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:18 smithi155 ceph-mon[184854]: pgmap v581: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 1.7 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 464 op/s; 11511/83211 objects degraded (13.834%); 5074/83211 objects misplaced (6.098%); 0 B/s, 184 objects/s recovering 2023-10-09T10:52:18.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2914694424' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:52:18.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:18 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:52:18.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:18 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 11511/83211 objects degraded (13.834%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:52:18.730 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:18.773 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:18.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:18.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:18.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:18 smithi125 ceph-mon[195507]: pgmap v581: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 1.7 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 464 op/s; 11511/83211 objects degraded (13.834%); 5074/83211 objects misplaced (6.098%); 0 B/s, 184 objects/s recovering 2023-10-09T10:52:18.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:18 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2914694424' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:52:18.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:18 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:52:18.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:18 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 11511/83211 objects degraded (13.834%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:52:18.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:18 smithi125 ceph-mon[200197]: pgmap v581: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 1.7 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 464 op/s; 11511/83211 objects degraded (13.834%); 5074/83211 objects misplaced (6.098%); 0 B/s, 184 objects/s recovering 2023-10-09T10:52:18.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:18 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2914694424' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:52:18.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:18 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:52:18.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:18 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 11511/83211 objects degraded (13.834%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:52:18.864 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:52:18.876 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:18.997 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:52:18.997 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:52:19.018 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:19.070 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:19.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:19.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:19.172 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:19.216 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:19.309 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:19.351 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:19.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:19.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:19.382 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:52:19.463 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:19.528 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:19.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:19.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:19.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:19.681 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:19.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:19 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:52:19.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:19 smithi155 ceph-mon[184854]: failed to encode map e249 with expected crc 2023-10-09T10:52:19.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:19 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:52:19.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:19 smithi155 ceph-mon[184854]: osdmap e249: 8 total, 8 up, 6 in 2023-10-09T10:52:19.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:19 smithi155 ceph-mon[184854]: failed to encode map e249 with expected crc 2023-10-09T10:52:19.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:19 smithi155 ceph-mon[184854]: failed to encode map e249 with expected crc 2023-10-09T10:52:19.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:19 smithi155 ceph-mon[184854]: failed to encode map e249 with expected crc 2023-10-09T10:52:19.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:19 smithi155 ceph-mon[184854]: failed to encode map e249 with expected crc 2023-10-09T10:52:19.727 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 1] out_osds: [2, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:52:19.727 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:52:19.727 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T10:52:19.727 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:52:19.727 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:52:19.727 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T10:52:19.744 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:19.773 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:19.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:19.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:19.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:52:19.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[195507]: failed to encode map e249 with expected crc 2023-10-09T10:52:19.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:52:19.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[195507]: osdmap e249: 8 total, 8 up, 6 in 2023-10-09T10:52:19.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[195507]: failed to encode map e249 with expected crc 2023-10-09T10:52:19.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[195507]: failed to encode map e249 with expected crc 2023-10-09T10:52:19.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[195507]: failed to encode map e249 with expected crc 2023-10-09T10:52:19.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[195507]: failed to encode map e249 with expected crc 2023-10-09T10:52:19.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:52:19.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[200197]: failed to encode map e249 with expected crc 2023-10-09T10:52:19.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:52:19.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[200197]: osdmap e249: 8 total, 8 up, 6 in 2023-10-09T10:52:19.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[200197]: failed to encode map e249 with expected crc 2023-10-09T10:52:19.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[200197]: failed to encode map e249 with expected crc 2023-10-09T10:52:19.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[200197]: failed to encode map e249 with expected crc 2023-10-09T10:52:19.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:19 smithi125 ceph-mon[200197]: failed to encode map e249 with expected crc 2023-10-09T10:52:19.877 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:52:19.878 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:19.924 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:20.008 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:20.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:20.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:20.113 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:20.147 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:20.173 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:20.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:20.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:20.278 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:20.298 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:52:20.309 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:20.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:20.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:20.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:20.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:20.462 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:20 smithi125 ceph-mon[195507]: pgmap v583: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 109 active+clean; 1.6 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 729 op/s; 10243/76467 objects degraded (13.395%); 4811/76467 objects misplaced (6.292%); 0 B/s, 220 objects/s recovering 2023-10-09T10:52:20.462 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:20 smithi125 ceph-mon[195507]: osdmap e250: 8 total, 8 up, 6 in 2023-10-09T10:52:20.462 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:20 smithi125 ceph-mon[195507]: failed to encode map e250 with expected crc 2023-10-09T10:52:20.463 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:20 smithi125 ceph-mon[195507]: failed to encode map e250 with expected crc 2023-10-09T10:52:20.463 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:20 smithi125 ceph-mon[195507]: failed to encode map e250 with expected crc 2023-10-09T10:52:20.463 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:20 smithi125 ceph-mon[195507]: failed to encode map e250 with expected crc 2023-10-09T10:52:20.464 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:20.589 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:20.594 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:52:20.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:20 smithi155 ceph-mon[184854]: pgmap v583: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 109 active+clean; 1.6 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 729 op/s; 10243/76467 objects degraded (13.395%); 4811/76467 objects misplaced (6.292%); 0 B/s, 220 objects/s recovering 2023-10-09T10:52:20.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:20 smithi155 ceph-mon[184854]: osdmap e250: 8 total, 8 up, 6 in 2023-10-09T10:52:20.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:20 smithi155 ceph-mon[184854]: failed to encode map e250 with expected crc 2023-10-09T10:52:20.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:20 smithi155 ceph-mon[184854]: failed to encode map e250 with expected crc 2023-10-09T10:52:20.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:20 smithi155 ceph-mon[184854]: failed to encode map e250 with expected crc 2023-10-09T10:52:20.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:20 smithi155 ceph-mon[184854]: failed to encode map e250 with expected crc 2023-10-09T10:52:20.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:20.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:20.628 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:52:20.665 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:52:20.702 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:20.707 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:52:20.743 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:52:20.752 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:20.773 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:52:20.781 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:20.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:20.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:20.808 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:52:20.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:20 smithi125 ceph-mon[200197]: pgmap v583: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 109 active+clean; 1.6 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 729 op/s; 10243/76467 objects degraded (13.395%); 4811/76467 objects misplaced (6.292%); 0 B/s, 220 objects/s recovering 2023-10-09T10:52:20.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:20 smithi125 ceph-mon[200197]: osdmap e250: 8 total, 8 up, 6 in 2023-10-09T10:52:20.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:20 smithi125 ceph-mon[200197]: failed to encode map e250 with expected crc 2023-10-09T10:52:20.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:20 smithi125 ceph-mon[200197]: failed to encode map e250 with expected crc 2023-10-09T10:52:20.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:20 smithi125 ceph-mon[200197]: failed to encode map e250 with expected crc 2023-10-09T10:52:20.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:20 smithi125 ceph-mon[200197]: failed to encode map e250 with expected crc 2023-10-09T10:52:20.848 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:52:20.853 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:52:20.853 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:52:20.854 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:52:20.854 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:52:20.854 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:52:20.854 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:52:20.854 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:52:20.854 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:52:20.885 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:20.902 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:52:20.931 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:20.971 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:20.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:20.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:21.082 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:21.165 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:21.222 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:52:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:52:21] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2023-10-09T10:52:21.253 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:52:21.253 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:52:21.253 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T10:52:21.253 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:52:21.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:21.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:21.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:21.397 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:21.445 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:21.552 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:21 smithi155 ceph-mon[184854]: failed to encode map e251 with expected crc 2023-10-09T10:52:21.552 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:21 smithi155 ceph-mon[184854]: osdmap e251: 8 total, 8 up, 6 in 2023-10-09T10:52:21.552 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:21 smithi155 ceph-mon[184854]: failed to encode map e251 with expected crc 2023-10-09T10:52:21.552 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:21 smithi155 ceph-mon[184854]: failed to encode map e251 with expected crc 2023-10-09T10:52:21.552 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:21 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/882335905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:21.601 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:52:21.601 INFO:teuthology.orchestra.run.smithi155.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T10:52:21.601 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:52:21.731 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:52:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:21 smithi125 ceph-mon[195507]: failed to encode map e251 with expected crc 2023-10-09T10:52:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:21 smithi125 ceph-mon[195507]: osdmap e251: 8 total, 8 up, 6 in 2023-10-09T10:52:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:21 smithi125 ceph-mon[195507]: failed to encode map e251 with expected crc 2023-10-09T10:52:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:21 smithi125 ceph-mon[195507]: failed to encode map e251 with expected crc 2023-10-09T10:52:21.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:21 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/882335905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:21.826 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:21 smithi125 ceph-mon[200197]: failed to encode map e251 with expected crc 2023-10-09T10:52:21.827 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:21 smithi125 ceph-mon[200197]: osdmap e251: 8 total, 8 up, 6 in 2023-10-09T10:52:21.827 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:21 smithi125 ceph-mon[200197]: failed to encode map e251 with expected crc 2023-10-09T10:52:21.827 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:21 smithi125 ceph-mon[200197]: failed to encode map e251 with expected crc 2023-10-09T10:52:21.827 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:21 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/882335905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:21.887 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:21.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:21.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:21.927 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:21.997 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:22.078 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:22.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:22.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:22.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:22.198 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:52:22.226 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:22.279 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:22.364 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:22.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:22.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:22.470 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:52:22.545 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:22.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:22 smithi125 ceph-mon[200197]: pgmap v586: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 109 active+clean; 1.4 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 816 op/s; 8289/69996 objects degraded (11.842%); 4266/69996 objects misplaced (6.095%); 0 B/s, 82 objects/s recovering 2023-10-09T10:52:22.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:22 smithi125 ceph-mon[195507]: pgmap v586: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 109 active+clean; 1.4 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 816 op/s; 8289/69996 objects degraded (11.842%); 4266/69996 objects misplaced (6.095%); 0 B/s, 82 objects/s recovering 2023-10-09T10:52:22.851 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:22 smithi155 ceph-mon[184854]: pgmap v586: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 109 active+clean; 1.4 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 816 op/s; 8289/69996 objects degraded (11.842%); 4266/69996 objects misplaced (6.095%); 0 B/s, 82 objects/s recovering 2023-10-09T10:52:23.393 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:52:23.393 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:52:23.393 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:52:23.393 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:52:23.393 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:52:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:23 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 8289/69996 objects degraded (11.842%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:52:23.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:23 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 8289/69996 objects degraded (11.842%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:52:23.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:23 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 8289/69996 objects degraded (11.842%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:52:23.846 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:52:23.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:23.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:52:23.992 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:52:24.061 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:24.438 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:52:24.550 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:52:24.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:24.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:52:24.655 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:24.705 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:24.728 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:24.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:24.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:24.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:24 smithi125 ceph-mon[195507]: pgmap v587: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 1.3 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 7208/64614 objects degraded (11.155%); 4045/64614 objects misplaced (6.260%); 0 B/s, 146 objects/s recovering 2023-10-09T10:52:24.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:24 smithi125 ceph-mon[200197]: pgmap v587: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 1.3 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 7208/64614 objects degraded (11.155%); 4045/64614 objects misplaced (6.260%); 0 B/s, 146 objects/s recovering 2023-10-09T10:52:24.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:52:24.866 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:52:24.900 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:24 smithi155 ceph-mon[184854]: pgmap v587: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 1.3 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 7208/64614 objects degraded (11.155%); 4045/64614 objects misplaced (6.260%); 0 B/s, 146 objects/s recovering 2023-10-09T10:52:24.904 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:24.991 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:52:24.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:24.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:52:25.098 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:25.148 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:25.340 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:25.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:25.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:25.445 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:52:25.505 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:25.594 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:52:25.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:25.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:52:25.707 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:25.770 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:25.807 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:25.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:25.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:25.871 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:52:25.910 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:52:25.992 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:26.010 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:52:26.010 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:52:26.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:52:26.010 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:52:26.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.010 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70444 6.3 6:c0306d38:::benchmark_data_smithi125_244951_object70443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:52:26.011 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.568354+0000", 2023-10-09T10:52:26.011 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 422.43660483000002, 2023-10-09T10:52:26.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047306648, 2023-10-09T10:52:26.011 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.011 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.011 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.011 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.011 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.011 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70444 2023-10-09T10:52:26.012 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.012 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568354+0000", 2023-10-09T10:52:26.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.012 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568354+0000", 2023-10-09T10:52:26.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568353+0000", 2023-10-09T10:52:26.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:26.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568387+0000", 2023-10-09T10:52:26.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4495000000000001e-05 2023-10-09T10:52:26.014 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568407+0000", 2023-10-09T10:52:26.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9729999999999999e-05 2023-10-09T10:52:26.014 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.015 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568412+0000", 2023-10-09T10:52:26.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4149999999999998e-06 2023-10-09T10:52:26.015 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.015 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.015 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.015 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568457+0000", 2023-10-09T10:52:26.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4270000000000001e-05 2023-10-09T10:52:26.015 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.015 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.016 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.016 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568587+0000", 2023-10-09T10:52:26.016 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013011399999999999 2023-10-09T10:52:26.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.016 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:52:26.016 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568753+0000", 2023-10-09T10:52:26.016 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016615099999999999 2023-10-09T10:52:26.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.016 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.016 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.570931+0000", 2023-10-09T10:52:26.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002177596 2023-10-09T10:52:26.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.570969+0000", 2023-10-09T10:52:26.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8661000000000001e-05 2023-10-09T10:52:26.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615586+0000", 2023-10-09T10:52:26.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.044616524999999997 2023-10-09T10:52:26.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615636+0000", 2023-10-09T10:52:26.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0009000000000002e-05 2023-10-09T10:52:26.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.018 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615661+0000", 2023-10-09T10:52:26.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.516e-05 2023-10-09T10:52:26.018 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.018 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.018 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.019 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70445 6.3 6:c48e8359:::benchmark_data_smithi125_244951_object70444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:52:26.019 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.568425+0000", 2023-10-09T10:52:26.019 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 422.43653457099998, 2023-10-09T10:52:26.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047372754000000003, 2023-10-09T10:52:26.019 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.019 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.019 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.019 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.019 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.019 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70445 2023-10-09T10:52:26.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.020 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.020 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.020 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568425+0000", 2023-10-09T10:52:26.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.020 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.020 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568425+0000", 2023-10-09T10:52:26.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568424+0000", 2023-10-09T10:52:26.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:52:26.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568448+0000", 2023-10-09T10:52:26.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3623000000000001e-05 2023-10-09T10:52:26.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.022 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.022 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568453+0000", 2023-10-09T10:52:26.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1880000000000001e-06 2023-10-09T10:52:26.022 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.022 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.022 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.022 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568457+0000", 2023-10-09T10:52:26.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1609999999999996e-06 2023-10-09T10:52:26.022 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.022 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.022 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.022 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.569823+0000", 2023-10-09T10:52:26.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001365996 2023-10-09T10:52:26.023 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.023 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.023 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.023 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.569913+0000", 2023-10-09T10:52:26.023 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9388000000000005e-05 2023-10-09T10:52:26.023 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.023 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.023 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:52:26.023 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.570017+0000", 2023-10-09T10:52:26.023 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103997 2023-10-09T10:52:26.023 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.023 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.023 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.571104+0000", 2023-10-09T10:52:26.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00108701 2023-10-09T10:52:26.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.024 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.571280+0000", 2023-10-09T10:52:26.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017659500000000001 2023-10-09T10:52:26.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.024 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615712+0000", 2023-10-09T10:52:26.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.044431813000000001 2023-10-09T10:52:26.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.025 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.025 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615755+0000", 2023-10-09T10:52:26.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2588e-05 2023-10-09T10:52:26.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.025 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.025 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615797+0000", 2023-10-09T10:52:26.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2756999999999997e-05 2023-10-09T10:52:26.025 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.026 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.026 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.026 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106416 6.3 6:c0486f03:::benchmark_data_smithi125_244951_object106415:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:52:26.026 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.234195+0000", 2023-10-09T10:52:26.026 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 383.77076433399998, 2023-10-09T10:52:26.026 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.048317987999999999, 2023-10-09T10:52:26.026 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.026 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.026 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.026 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.027 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.027 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106416 2023-10-09T10:52:26.027 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.027 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.027 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.027 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.027 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234195+0000", 2023-10-09T10:52:26.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.027 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.027 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.027 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234195+0000", 2023-10-09T10:52:26.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234193+0000", 2023-10-09T10:52:26.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:26.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234285+0000", 2023-10-09T10:52:26.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1546999999999996e-05 2023-10-09T10:52:26.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.029 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234305+0000", 2023-10-09T10:52:26.029 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0119000000000002e-05 2023-10-09T10:52:26.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.029 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234313+0000", 2023-10-09T10:52:26.029 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3590000000000008e-06 2023-10-09T10:52:26.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.030 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.030 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234382+0000", 2023-10-09T10:52:26.030 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8261000000000003e-05 2023-10-09T10:52:26.030 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.030 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.030 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.030 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234490+0000", 2023-10-09T10:52:26.030 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000108398 2023-10-09T10:52:26.030 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.030 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.030 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:52:26.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234588+0000", 2023-10-09T10:52:26.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7966000000000004e-05 2023-10-09T10:52:26.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.236668+0000", 2023-10-09T10:52:26.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020797540000000001 2023-10-09T10:52:26.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.236734+0000", 2023-10-09T10:52:26.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6644999999999996e-05 2023-10-09T10:52:26.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.282424+0000", 2023-10-09T10:52:26.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.045689354000000001 2023-10-09T10:52:26.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.282474+0000", 2023-10-09T10:52:26.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9938999999999997e-05 2023-10-09T10:52:26.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.282513+0000", 2023-10-09T10:52:26.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9168999999999997e-05 2023-10-09T10:52:26.033 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.033 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.033 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.033 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:137725 6.3 6:c0616549:::benchmark_data_smithi125_244951_object31282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:52:26.033 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:05.071869+0000", 2023-10-09T10:52:26.033 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 320.933090182, 2023-10-09T10:52:26.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038173246000000001, 2023-10-09T10:52:26.034 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.034 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.034 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.034 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.034 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.034 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 137725 2023-10-09T10:52:26.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.034 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.034 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071869+0000", 2023-10-09T10:52:26.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.035 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.035 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.035 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.035 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071869+0000", 2023-10-09T10:52:26.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.035 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.035 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.035 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.035 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071867+0000", 2023-10-09T10:52:26.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:26.035 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.035 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071875+0000", 2023-10-09T10:52:26.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6290000000000001e-06 2023-10-09T10:52:26.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071885+0000", 2023-10-09T10:52:26.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0397999999999999e-05 2023-10-09T10:52:26.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.071892+0000", 2023-10-09T10:52:26.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6490000000000002e-06 2023-10-09T10:52:26.037 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.037 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.037 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.081942+0000", 2023-10-09T10:52:26.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010050325000000001 2023-10-09T10:52:26.037 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.037 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.037 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.082111+0000", 2023-10-09T10:52:26.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016843500000000001 2023-10-09T10:52:26.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:52:26.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.082600+0000", 2023-10-09T10:52:26.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00048894400000000005 2023-10-09T10:52:26.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.084543+0000", 2023-10-09T10:52:26.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019437110000000001 2023-10-09T10:52:26.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.086053+0000", 2023-10-09T10:52:26.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015096179999999999 2023-10-09T10:52:26.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.109298+0000", 2023-10-09T10:52:26.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023244813 2023-10-09T10:52:26.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.040 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.109336+0000", 2023-10-09T10:52:26.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.79e-05 2023-10-09T10:52:26.040 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.040 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.040 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.040 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.110042+0000", 2023-10-09T10:52:26.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00070647600000000004 2023-10-09T10:52:26.040 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.040 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.040 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.040 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.040 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.041 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:137790 6.13 6:c864fc16:::benchmark_data_smithi125_244951_object31347:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:52:26.041 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:05.180662+0000", 2023-10-09T10:52:26.041 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 320.824296984, 2023-10-09T10:52:26.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038056526, 2023-10-09T10:52:26.041 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.041 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.041 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.041 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.041 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.041 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 137790 2023-10-09T10:52:26.041 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.041 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.042 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180662+0000", 2023-10-09T10:52:26.042 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.042 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.042 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180662+0000", 2023-10-09T10:52:26.042 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.042 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180661+0000", 2023-10-09T10:52:26.043 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:26.043 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180667+0000", 2023-10-09T10:52:26.043 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4679999999999997e-06 2023-10-09T10:52:26.043 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180680+0000", 2023-10-09T10:52:26.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2431e-05 2023-10-09T10:52:26.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.044 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.044 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180685+0000", 2023-10-09T10:52:26.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1200000000000001e-06 2023-10-09T10:52:26.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.044 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.044 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182279+0000", 2023-10-09T10:52:26.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001594804 2023-10-09T10:52:26.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.045 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182481+0000", 2023-10-09T10:52:26.045 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00020201 2023-10-09T10:52:26.045 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.045 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:52:26.045 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182899+0000", 2023-10-09T10:52:26.045 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00041767300000000002 2023-10-09T10:52:26.045 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.045 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.045 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.185800+0000", 2023-10-09T10:52:26.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029009109999999999 2023-10-09T10:52:26.046 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.046 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.046 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.046 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.185815+0000", 2023-10-09T10:52:26.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4959e-05 2023-10-09T10:52:26.046 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.046 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.046 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.046 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.217393+0000", 2023-10-09T10:52:26.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.031578322999999998 2023-10-09T10:52:26.047 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.047 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.047 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.047 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.217443+0000", 2023-10-09T10:52:26.047 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9308999999999998e-05 2023-10-09T10:52:26.047 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.047 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.047 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.047 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.218719+0000", 2023-10-09T10:52:26.047 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001276016 2023-10-09T10:52:26.047 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.047 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.048 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.048 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.048 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.048 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:137793 6.3 6:c3f14b3e:::benchmark_data_smithi125_244951_object31350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:52:26.048 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:52:26.048 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 320.82426306999997, 2023-10-09T10:52:26.048 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.046213456, 2023-10-09T10:52:26.048 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.048 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.048 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.048 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.048 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.049 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 137793 2023-10-09T10:52:26.049 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.049 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.049 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.049 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.049 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:52:26.049 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.049 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.049 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.049 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.049 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:52:26.049 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:52:26.050 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:52:26.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180698+0000", 2023-10-09T10:52:26.050 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5009999999999999e-06 2023-10-09T10:52:26.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.051 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.051 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180702+0000", 2023-10-09T10:52:26.051 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2200000000000003e-06 2023-10-09T10:52:26.051 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.051 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.051 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.051 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180705+0000", 2023-10-09T10:52:26.051 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7779999999999999e-06 2023-10-09T10:52:26.051 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.051 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.051 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.051 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182790+0000", 2023-10-09T10:52:26.051 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020852819999999999 2023-10-09T10:52:26.052 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.052 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.052 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.052 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182925+0000", 2023-10-09T10:52:26.052 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013469099999999999 2023-10-09T10:52:26.052 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.052 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.052 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:52:26.052 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.183331+0000", 2023-10-09T10:52:26.052 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000405783 2023-10-09T10:52:26.052 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.052 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.052 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.185825+0000", 2023-10-09T10:52:26.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024937789999999998 2023-10-09T10:52:26.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.053 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.188205+0000", 2023-10-09T10:52:26.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023806790000000001 2023-10-09T10:52:26.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.053 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.225637+0000", 2023-10-09T10:52:26.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.037431091 2023-10-09T10:52:26.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.225680+0000", 2023-10-09T10:52:26.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3856999999999997e-05 2023-10-09T10:52:26.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.226909+0000", 2023-10-09T10:52:26.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001229129 2023-10-09T10:52:26.054 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.054 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.055 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.055 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:137805 6.3 6:c448070c:::benchmark_data_smithi125_244951_object31362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:52:26.055 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:05.197338+0000", 2023-10-09T10:52:26.055 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 320.80762146799998, 2023-10-09T10:52:26.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039251805000000001, 2023-10-09T10:52:26.055 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.055 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.055 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.055 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.056 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.056 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 137805 2023-10-09T10:52:26.056 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.056 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.056 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.056 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.056 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197338+0000", 2023-10-09T10:52:26.056 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.056 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.056 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.056 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.056 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197338+0000", 2023-10-09T10:52:26.056 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.057 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.057 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.057 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197336+0000", 2023-10-09T10:52:26.057 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:26.057 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.057 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.057 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197342+0000", 2023-10-09T10:52:26.057 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3990000000000001e-06 2023-10-09T10:52:26.057 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.058 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197354+0000", 2023-10-09T10:52:26.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1821000000000001e-05 2023-10-09T10:52:26.058 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.058 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.058 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197359+0000", 2023-10-09T10:52:26.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9960000000000001e-06 2023-10-09T10:52:26.058 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.058 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.058 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197413+0000", 2023-10-09T10:52:26.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3909999999999997e-05 2023-10-09T10:52:26.059 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.059 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.059 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.059 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197587+0000", 2023-10-09T10:52:26.059 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000173408 2023-10-09T10:52:26.059 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.059 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.059 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:52:26.059 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197988+0000", 2023-10-09T10:52:26.059 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00040095300000000001 2023-10-09T10:52:26.059 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.059 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.060 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.060 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.199873+0000", 2023-10-09T10:52:26.060 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018855790000000001 2023-10-09T10:52:26.060 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.060 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.060 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.060 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.200673+0000", 2023-10-09T10:52:26.060 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00079994400000000002 2023-10-09T10:52:26.060 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.060 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.060 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.061 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.235493+0000", 2023-10-09T10:52:26.061 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.034820028000000003 2023-10-09T10:52:26.061 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.061 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.061 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.061 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.235533+0000", 2023-10-09T10:52:26.061 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0139000000000003e-05 2023-10-09T10:52:26.061 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.061 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.061 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.061 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.236589+0000", 2023-10-09T10:52:26.061 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001056219 2023-10-09T10:52:26.061 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.062 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.062 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.062 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.062 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.062 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:152091 6.6 e174/107 6:64b0928f:::benchmark_data_smithi125_244951_object45648:head v 174'5840, mlcod=174'5838)", 2023-10-09T10:52:26.062 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.267910+0000", 2023-10-09T10:52:26.062 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 291.73704881399999, 2023-10-09T10:52:26.062 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038740213000000003, 2023-10-09T10:52:26.062 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.062 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.062 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.063 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.063 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.063 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267910+0000", 2023-10-09T10:52:26.063 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.063 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.063 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.063 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.063 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267910+0000", 2023-10-09T10:52:26.063 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.063 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.063 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.063 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.064 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267909+0000", 2023-10-09T10:52:26.064 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:26.064 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.064 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.064 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.064 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267917+0000", 2023-10-09T10:52:26.064 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1869999999999997e-06 2023-10-09T10:52:26.064 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.064 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.064 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.064 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267931+0000", 2023-10-09T10:52:26.065 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3239e-05 2023-10-09T10:52:26.065 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.065 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.065 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.065 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267936+0000", 2023-10-09T10:52:26.065 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.451e-06 2023-10-09T10:52:26.065 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.065 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.065 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.065 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267979+0000", 2023-10-09T10:52:26.065 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3096000000000002e-05 2023-10-09T10:52:26.065 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.065 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.066 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.066 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.268018+0000", 2023-10-09T10:52:26.066 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8532999999999999e-05 2023-10-09T10:52:26.066 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.066 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.066 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.066 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.306618+0000", 2023-10-09T10:52:26.066 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038600661000000001 2023-10-09T10:52:26.066 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.066 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.066 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.067 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.306651+0000", 2023-10-09T10:52:26.067 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2076000000000003e-05 2023-10-09T10:52:26.067 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.067 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.067 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.067 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.067 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.067 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:152093 6.13 6:cd7dbd99:::benchmark_data_smithi125_244951_object45650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:52:26.067 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.270440+0000", 2023-10-09T10:52:26.067 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 291.73451916800002, 2023-10-09T10:52:26.067 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040432893999999997, 2023-10-09T10:52:26.067 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.068 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.068 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.068 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.068 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.068 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 152093 2023-10-09T10:52:26.068 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.068 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.068 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.068 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.068 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270440+0000", 2023-10-09T10:52:26.068 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.068 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.069 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.069 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.069 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270440+0000", 2023-10-09T10:52:26.069 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.069 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.069 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.069 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.069 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270438+0000", 2023-10-09T10:52:26.069 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T10:52:26.069 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.069 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.070 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.070 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270450+0000", 2023-10-09T10:52:26.070 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1918e-05 2023-10-09T10:52:26.070 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.070 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.070 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.070 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270479+0000", 2023-10-09T10:52:26.070 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8911000000000002e-05 2023-10-09T10:52:26.070 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.070 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.070 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.071 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270488+0000", 2023-10-09T10:52:26.071 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6139999999999998e-06 2023-10-09T10:52:26.071 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.071 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.071 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.071 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270586+0000", 2023-10-09T10:52:26.071 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7823000000000002e-05 2023-10-09T10:52:26.071 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.071 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.071 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.071 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270758+0000", 2023-10-09T10:52:26.071 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000172103 2023-10-09T10:52:26.072 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.072 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.072 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:52:26.072 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.271250+0000", 2023-10-09T10:52:26.072 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000491688 2023-10-09T10:52:26.072 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.072 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.072 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.072 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.278619+0000", 2023-10-09T10:52:26.072 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0073690730000000003 2023-10-09T10:52:26.072 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.072 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.073 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.073 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.284528+0000", 2023-10-09T10:52:26.073 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059091789999999996 2023-10-09T10:52:26.073 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.073 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.073 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.073 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.309269+0000", 2023-10-09T10:52:26.073 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024740522000000001 2023-10-09T10:52:26.073 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.073 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.073 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.074 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.309319+0000", 2023-10-09T10:52:26.074 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0912999999999999e-05 2023-10-09T10:52:26.074 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.074 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.074 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.074 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.310873+0000", 2023-10-09T10:52:26.074 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001553369 2023-10-09T10:52:26.074 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.074 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.074 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.074 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.075 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.075 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:152095 6.1 e174/164 6:8109f342:::benchmark_data_smithi125_244951_object45652:head v 174'5685, mlcod=174'5683)", 2023-10-09T10:52:26.075 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.273512+0000", 2023-10-09T10:52:26.075 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 291.73144722799998, 2023-10-09T10:52:26.075 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040480956999999998, 2023-10-09T10:52:26.075 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.075 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.075 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.075 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.075 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.075 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273512+0000", 2023-10-09T10:52:26.075 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.076 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.076 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.076 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.076 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273512+0000", 2023-10-09T10:52:26.076 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.076 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.076 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.076 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.076 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273511+0000", 2023-10-09T10:52:26.076 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:26.076 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.076 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.076 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.077 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273520+0000", 2023-10-09T10:52:26.077 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.645e-06 2023-10-09T10:52:26.077 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.077 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.077 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.077 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273534+0000", 2023-10-09T10:52:26.077 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3875e-05 2023-10-09T10:52:26.077 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.077 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.077 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.077 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273541+0000", 2023-10-09T10:52:26.077 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4790000000000001e-06 2023-10-09T10:52:26.078 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.078 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.078 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.078 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273796+0000", 2023-10-09T10:52:26.078 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025495599999999998 2023-10-09T10:52:26.078 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.078 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.078 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.078 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273819+0000", 2023-10-09T10:52:26.078 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3175000000000001e-05 2023-10-09T10:52:26.078 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.078 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.079 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.079 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.313948+0000", 2023-10-09T10:52:26.079 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040129418 2023-10-09T10:52:26.079 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.079 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.079 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.079 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.313993+0000", 2023-10-09T10:52:26.079 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4549000000000003e-05 2023-10-09T10:52:26.079 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.079 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.079 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.080 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.080 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.080 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:161148 6.13 6:c8d28001:::benchmark_data_smithi125_244951_object54705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e176)", 2023-10-09T10:52:26.080 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:47.001392+0000", 2023-10-09T10:52:26.080 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 279.00356734799999, 2023-10-09T10:52:26.080 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038885253000000002, 2023-10-09T10:52:26.080 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.080 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.080 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.080 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.080 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.080 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 161148 2023-10-09T10:52:26.081 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.081 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.081 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.081 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.081 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001392+0000", 2023-10-09T10:52:26.081 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.081 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.081 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.081 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.081 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001392+0000", 2023-10-09T10:52:26.081 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.082 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.082 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.082 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.082 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001391+0000", 2023-10-09T10:52:26.082 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:26.082 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.082 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.082 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.082 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001397+0000", 2023-10-09T10:52:26.082 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0560000000000001e-06 2023-10-09T10:52:26.082 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.082 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.083 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.083 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001410+0000", 2023-10-09T10:52:26.083 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2695e-05 2023-10-09T10:52:26.083 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.083 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.083 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.083 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001416+0000", 2023-10-09T10:52:26.083 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9569999999999996e-06 2023-10-09T10:52:26.083 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.083 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.083 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.083 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.020851+0000", 2023-10-09T10:52:26.084 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019434414000000001 2023-10-09T10:52:26.084 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.084 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.084 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.084 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.021010+0000", 2023-10-09T10:52:26.084 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015947300000000001 2023-10-09T10:52:26.084 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.084 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.084 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:52:26.084 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.021509+0000", 2023-10-09T10:52:26.084 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00049899199999999997 2023-10-09T10:52:26.084 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.084 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.084 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.085 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.026203+0000", 2023-10-09T10:52:26.085 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0046932700000000003 2023-10-09T10:52:26.085 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.085 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.085 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.085 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.026305+0000", 2023-10-09T10:52:26.085 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000102744 2023-10-09T10:52:26.085 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.085 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.085 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.085 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.038789+0000", 2023-10-09T10:52:26.086 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012483823 2023-10-09T10:52:26.086 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.086 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.086 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.086 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.038863+0000", 2023-10-09T10:52:26.086 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3634999999999995e-05 2023-10-09T10:52:26.086 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.086 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.086 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.086 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.040277+0000", 2023-10-09T10:52:26.086 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014141900000000001 2023-10-09T10:52:26.086 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.087 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.087 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.087 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.087 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.087 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:162909 6.13 6:c8b03eff:::benchmark_data_smithi125_244951_object56466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-10-09T10:52:26.087 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:49.864994+0000", 2023-10-09T10:52:26.087 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 276.13996470400002, 2023-10-09T10:52:26.087 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.037068614, 2023-10-09T10:52:26.087 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.087 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.087 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.087 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.088 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.088 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162909 2023-10-09T10:52:26.088 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.088 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.088 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.088 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.088 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.864994+0000", 2023-10-09T10:52:26.088 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.088 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.088 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.088 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.088 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.864994+0000", 2023-10-09T10:52:26.089 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.089 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.089 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.089 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.089 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.864993+0000", 2023-10-09T10:52:26.089 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:52:26.089 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.089 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.089 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.089 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.864999+0000", 2023-10-09T10:52:26.089 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2809999999999999e-06 2023-10-09T10:52:26.089 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.090 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.090 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.090 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.865009+0000", 2023-10-09T10:52:26.090 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.039e-05 2023-10-09T10:52:26.090 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.090 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.090 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.090 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.865015+0000", 2023-10-09T10:52:26.090 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.592e-06 2023-10-09T10:52:26.090 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.090 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.091 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.091 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.865060+0000", 2023-10-09T10:52:26.091 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5311e-05 2023-10-09T10:52:26.091 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.091 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.091 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.091 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.865238+0000", 2023-10-09T10:52:26.091 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017833700000000001 2023-10-09T10:52:26.091 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.091 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.091 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:52:26.091 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.865775+0000", 2023-10-09T10:52:26.092 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00053696699999999996 2023-10-09T10:52:26.092 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.092 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.092 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.092 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.869743+0000", 2023-10-09T10:52:26.092 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039680150000000001 2023-10-09T10:52:26.092 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.092 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.092 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.092 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.870048+0000", 2023-10-09T10:52:26.092 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030474199999999999 2023-10-09T10:52:26.092 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.092 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.093 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.093 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.900766+0000", 2023-10-09T10:52:26.093 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030717874999999999 2023-10-09T10:52:26.093 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.093 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.093 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.093 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.900828+0000", 2023-10-09T10:52:26.093 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2466999999999997e-05 2023-10-09T10:52:26.093 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.093 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.093 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.093 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.902063+0000", 2023-10-09T10:52:26.094 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012347180000000001 2023-10-09T10:52:26.094 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.094 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.094 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.094 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.094 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.094 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:162969 6.13 6:cec0b93d:::benchmark_data_smithi125_244951_object56526:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-10-09T10:52:26.094 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:49.943081+0000", 2023-10-09T10:52:26.094 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 276.06187857100002, 2023-10-09T10:52:26.094 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039960117000000003, 2023-10-09T10:52:26.094 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.094 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.094 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.095 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.095 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.095 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162969 2023-10-09T10:52:26.095 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.095 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.095 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.095 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.095 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943081+0000", 2023-10-09T10:52:26.095 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.095 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.095 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.096 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.096 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943081+0000", 2023-10-09T10:52:26.096 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.096 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.096 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.096 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.096 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943079+0000", 2023-10-09T10:52:26.096 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:26.096 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.096 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.096 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.096 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943085+0000", 2023-10-09T10:52:26.096 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6890000000000001e-06 2023-10-09T10:52:26.097 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.097 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.097 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.097 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943097+0000", 2023-10-09T10:52:26.097 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2166e-05 2023-10-09T10:52:26.097 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.097 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.097 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.097 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943103+0000", 2023-10-09T10:52:26.097 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9150000000000001e-06 2023-10-09T10:52:26.097 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.097 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.098 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.098 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943207+0000", 2023-10-09T10:52:26.098 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104573 2023-10-09T10:52:26.098 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.098 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.098 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.098 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943393+0000", 2023-10-09T10:52:26.098 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018516299999999999 2023-10-09T10:52:26.098 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.098 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.098 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:52:26.098 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943913+0000", 2023-10-09T10:52:26.099 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00052048100000000005 2023-10-09T10:52:26.099 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.099 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.099 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.099 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.950165+0000", 2023-10-09T10:52:26.099 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062519439999999997 2023-10-09T10:52:26.099 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.099 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.099 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.099 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.956256+0000", 2023-10-09T10:52:26.099 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0060905630000000002 2023-10-09T10:52:26.099 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.100 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.100 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.100 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.981642+0000", 2023-10-09T10:52:26.100 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025386756999999999 2023-10-09T10:52:26.100 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.100 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.100 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.100 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.981697+0000", 2023-10-09T10:52:26.100 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4718e-05 2023-10-09T10:52:26.100 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.100 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.100 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.100 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.983041+0000", 2023-10-09T10:52:26.101 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001343639 2023-10-09T10:52:26.101 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.101 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.101 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.101 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.101 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.101 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:169526 6.13 6:c9d383e8:::benchmark_data_smithi125_244951_object63083:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-10-09T10:52:26.101 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:59.084841+0000", 2023-10-09T10:52:26.101 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 266.92011805599998, 2023-10-09T10:52:26.101 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.036194846000000003, 2023-10-09T10:52:26.101 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.101 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.102 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.102 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.102 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.102 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 169526 2023-10-09T10:52:26.102 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.102 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.102 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.102 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.102 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084841+0000", 2023-10-09T10:52:26.102 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.102 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.103 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.103 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.103 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084841+0000", 2023-10-09T10:52:26.103 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.103 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.103 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.103 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.105 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084841+0000", 2023-10-09T10:52:26.105 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:52:26.105 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.105 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.105 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.105 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084844+0000", 2023-10-09T10:52:26.105 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4860000000000002e-06 2023-10-09T10:52:26.105 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.106 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.106 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.106 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084851+0000", 2023-10-09T10:52:26.106 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7819999999999996e-06 2023-10-09T10:52:26.106 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.106 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.106 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.106 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.084855+0000", 2023-10-09T10:52:26.106 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3719999999999998e-06 2023-10-09T10:52:26.106 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.106 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.107 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.107 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.087999+0000", 2023-10-09T10:52:26.107 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0031432880000000002 2023-10-09T10:52:26.107 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.107 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.107 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.107 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.088176+0000", 2023-10-09T10:52:26.107 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017714799999999999 2023-10-09T10:52:26.107 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.107 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.107 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:52:26.107 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.088664+0000", 2023-10-09T10:52:26.108 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00048842899999999997 2023-10-09T10:52:26.108 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.108 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.108 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.108 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.099515+0000", 2023-10-09T10:52:26.108 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010850966 2023-10-09T10:52:26.108 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.108 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.108 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.108 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.102453+0000", 2023-10-09T10:52:26.108 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002937598 2023-10-09T10:52:26.108 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.109 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.109 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.109 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.119544+0000", 2023-10-09T10:52:26.109 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017091733000000001 2023-10-09T10:52:26.109 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.109 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.109 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.109 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.119602+0000", 2023-10-09T10:52:26.109 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7642000000000001e-05 2023-10-09T10:52:26.109 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.109 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.109 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.110 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.121036+0000", 2023-10-09T10:52:26.110 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014338599999999999 2023-10-09T10:52:26.110 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.110 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.110 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.110 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.110 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.110 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:170177 6.3 6:c6f24c09:::benchmark_data_smithi125_244951_object63734:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-10-09T10:52:26.110 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:59.997605+0000", 2023-10-09T10:52:26.110 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 266.00735444600002, 2023-10-09T10:52:26.110 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038407996999999999, 2023-10-09T10:52:26.111 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.111 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.111 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.111 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.111 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.111 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 170177 2023-10-09T10:52:26.111 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.111 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.111 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.111 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.111 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997605+0000", 2023-10-09T10:52:26.111 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.112 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.112 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.112 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.112 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997605+0000", 2023-10-09T10:52:26.112 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.112 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.112 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.112 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.112 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997604+0000", 2023-10-09T10:52:26.112 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:52:26.112 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.112 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.112 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.113 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997607+0000", 2023-10-09T10:52:26.113 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6170000000000001e-06 2023-10-09T10:52:26.113 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.113 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.113 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.113 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997612+0000", 2023-10-09T10:52:26.113 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0490000000000003e-06 2023-10-09T10:52:26.113 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.113 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.113 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.113 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:59.997616+0000", 2023-10-09T10:52:26.113 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5889999999999999e-06 2023-10-09T10:52:26.114 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.114 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.114 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.114 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.013316+0000", 2023-10-09T10:52:26.114 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015700823999999999 2023-10-09T10:52:26.114 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.114 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.114 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.114 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.013397+0000", 2023-10-09T10:52:26.114 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0259999999999994e-05 2023-10-09T10:52:26.114 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.114 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.115 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:52:26.115 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.013825+0000", 2023-10-09T10:52:26.115 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000428131 2023-10-09T10:52:26.115 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.115 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.115 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.115 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.017829+0000", 2023-10-09T10:52:26.115 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040045319999999999 2023-10-09T10:52:26.115 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.115 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.115 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.115 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.018823+0000", 2023-10-09T10:52:26.116 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00099334699999999989 2023-10-09T10:52:26.116 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.116 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.116 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.116 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.035922+0000", 2023-10-09T10:52:26.116 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017099487999999999 2023-10-09T10:52:26.116 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.116 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.116 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.116 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.035975+0000", 2023-10-09T10:52:26.116 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2741999999999997e-05 2023-10-09T10:52:26.116 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.117 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.117 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.117 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:00.036013+0000", 2023-10-09T10:52:26.117 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7812999999999999e-05 2023-10-09T10:52:26.117 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.117 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.117 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.117 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.117 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.117 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:189608 6.3 6:c6b3429c:::benchmark_data_smithi125_244951_object83165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2023-10-09T10:52:26.117 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:29.169892+0000", 2023-10-09T10:52:26.117 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 236.83506708900001, 2023-10-09T10:52:26.117 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039847636999999998, 2023-10-09T10:52:26.118 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.118 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.118 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.118 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.118 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.118 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 189608 2023-10-09T10:52:26.118 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.118 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.118 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.118 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.118 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169892+0000", 2023-10-09T10:52:26.119 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.119 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.119 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.119 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.119 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169892+0000", 2023-10-09T10:52:26.119 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.119 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.119 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.119 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.119 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169891+0000", 2023-10-09T10:52:26.119 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:26.119 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.120 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.120 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.120 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169896+0000", 2023-10-09T10:52:26.120 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7950000000000004e-06 2023-10-09T10:52:26.120 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.120 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.120 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.120 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169909+0000", 2023-10-09T10:52:26.120 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2062000000000001e-05 2023-10-09T10:52:26.120 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.120 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.120 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.121 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169914+0000", 2023-10-09T10:52:26.121 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3380000000000004e-06 2023-10-09T10:52:26.121 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.121 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.121 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.121 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169959+0000", 2023-10-09T10:52:26.121 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5392000000000002e-05 2023-10-09T10:52:26.121 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.121 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.121 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.121 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.170104+0000", 2023-10-09T10:52:26.122 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014468 2023-10-09T10:52:26.122 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.122 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.122 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:52:26.122 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.170424+0000", 2023-10-09T10:52:26.122 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00031972199999999999 2023-10-09T10:52:26.122 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.122 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.122 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.122 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.173247+0000", 2023-10-09T10:52:26.122 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028231710000000002 2023-10-09T10:52:26.122 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.123 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.123 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.123 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.186655+0000", 2023-10-09T10:52:26.123 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013408239000000001 2023-10-09T10:52:26.123 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.123 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.123 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.123 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.209647+0000", 2023-10-09T10:52:26.123 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022992404000000001 2023-10-09T10:52:26.123 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.123 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.123 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.124 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.209694+0000", 2023-10-09T10:52:26.124 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6822e-05 2023-10-09T10:52:26.124 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.124 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.124 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.124 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.209740+0000", 2023-10-09T10:52:26.124 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5361999999999997e-05 2023-10-09T10:52:26.124 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.124 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.124 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.124 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.124 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.125 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:192655 6.13 6:cee47a1c:::benchmark_data_smithi125_244951_object86212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-10-09T10:52:26.125 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:52:26.125 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 232.20146467800001, 2023-10-09T10:52:26.125 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039152566, 2023-10-09T10:52:26.125 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.125 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.125 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.125 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.125 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.125 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 192655 2023-10-09T10:52:26.125 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.125 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.126 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.126 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.126 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:52:26.126 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.126 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.126 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.126 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.126 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:52:26.126 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.126 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.126 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.127 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.127 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:52:26.127 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:52:26.127 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.127 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.127 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.127 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803498+0000", 2023-10-09T10:52:26.127 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4190000000000004e-06 2023-10-09T10:52:26.127 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.127 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.127 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.127 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803509+0000", 2023-10-09T10:52:26.128 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0801e-05 2023-10-09T10:52:26.128 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.128 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.128 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.128 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803515+0000", 2023-10-09T10:52:26.128 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.767e-06 2023-10-09T10:52:26.128 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.128 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.128 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.128 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.809697+0000", 2023-10-09T10:52:26.128 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061824799999999997 2023-10-09T10:52:26.128 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.129 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.129 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.129 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.809808+0000", 2023-10-09T10:52:26.129 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000110271 2023-10-09T10:52:26.129 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.129 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.129 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:52:26.129 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.809938+0000", 2023-10-09T10:52:26.129 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013000800000000001 2023-10-09T10:52:26.129 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.129 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.129 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.129 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.815553+0000", 2023-10-09T10:52:26.130 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0056156879999999998 2023-10-09T10:52:26.130 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.130 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.130 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.130 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.815567+0000", 2023-10-09T10:52:26.130 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4102e-05 2023-10-09T10:52:26.130 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.130 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.130 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.130 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.841982+0000", 2023-10-09T10:52:26.130 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026414812999999999 2023-10-09T10:52:26.130 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.131 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.131 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.131 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.842042+0000", 2023-10-09T10:52:26.131 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0198999999999997e-05 2023-10-09T10:52:26.131 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.131 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.131 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.131 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.842647+0000", 2023-10-09T10:52:26.131 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00060467599999999995 2023-10-09T10:52:26.131 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.131 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.131 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.131 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.132 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.132 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:193229 6.3 6:c7d3e5b6:::benchmark_data_smithi125_244951_object86786:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-10-09T10:52:26.132 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:34.764784+0000", 2023-10-09T10:52:26.132 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 231.24017531800001, 2023-10-09T10:52:26.132 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038137404999999999, 2023-10-09T10:52:26.132 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.132 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.132 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.132 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.132 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.132 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 193229 2023-10-09T10:52:26.132 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.133 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.133 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.133 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.133 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764784+0000", 2023-10-09T10:52:26.133 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.133 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.133 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.133 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.133 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764784+0000", 2023-10-09T10:52:26.133 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.133 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.133 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.134 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.134 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764782+0000", 2023-10-09T10:52:26.134 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:26.134 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.134 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.134 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.134 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764789+0000", 2023-10-09T10:52:26.134 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8020000000000003e-06 2023-10-09T10:52:26.134 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.134 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.134 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.134 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764800+0000", 2023-10-09T10:52:26.135 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.096e-05 2023-10-09T10:52:26.135 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.135 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.135 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.135 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.764805+0000", 2023-10-09T10:52:26.135 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0969999999999996e-06 2023-10-09T10:52:26.135 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.135 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.135 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.135 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.767644+0000", 2023-10-09T10:52:26.135 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028391860000000001 2023-10-09T10:52:26.135 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.136 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.136 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.136 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.767769+0000", 2023-10-09T10:52:26.136 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012499599999999999 2023-10-09T10:52:26.136 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.136 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.136 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:52:26.136 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.767893+0000", 2023-10-09T10:52:26.136 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000123915 2023-10-09T10:52:26.136 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.136 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.136 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.137 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.773147+0000", 2023-10-09T10:52:26.137 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0052540110000000003 2023-10-09T10:52:26.137 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.137 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.137 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.137 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.778846+0000", 2023-10-09T10:52:26.137 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0056984519999999997 2023-10-09T10:52:26.137 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.137 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.137 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.137 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.802818+0000", 2023-10-09T10:52:26.137 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023972543999999998 2023-10-09T10:52:26.138 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.138 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.138 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.138 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.802876+0000", 2023-10-09T10:52:26.138 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7275000000000003e-05 2023-10-09T10:52:26.138 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.138 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.138 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.138 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:34.802921+0000", 2023-10-09T10:52:26.138 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5574000000000001e-05 2023-10-09T10:52:26.138 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.138 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.139 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.139 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.139 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.139 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:209514 6.13 6:c9bc4b21:::benchmark_data_smithi125_244951_object103071:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-10-09T10:52:26.139 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:58.156931+0000", 2023-10-09T10:52:26.139 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 207.84802812000001, 2023-10-09T10:52:26.139 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.037088755000000001, 2023-10-09T10:52:26.139 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.139 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.139 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.139 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.139 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.140 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 209514 2023-10-09T10:52:26.140 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.140 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.140 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.140 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.140 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156931+0000", 2023-10-09T10:52:26.140 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.140 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.140 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.140 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.140 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156931+0000", 2023-10-09T10:52:26.140 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.141 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.141 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.141 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.141 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156930+0000", 2023-10-09T10:52:26.141 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:26.141 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.141 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.141 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.141 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156935+0000", 2023-10-09T10:52:26.141 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5350000000000004e-06 2023-10-09T10:52:26.141 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.141 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.142 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.142 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156948+0000", 2023-10-09T10:52:26.142 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2842e-05 2023-10-09T10:52:26.142 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.142 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.142 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.142 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.156954+0000", 2023-10-09T10:52:26.142 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.733e-06 2023-10-09T10:52:26.142 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.142 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.142 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.143 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.158522+0000", 2023-10-09T10:52:26.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015687570000000001 2023-10-09T10:52:26.143 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.143 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.143 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.143 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.158683+0000", 2023-10-09T10:52:26.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016043599999999999 2023-10-09T10:52:26.143 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.143 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.143 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:52:26.143 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.158851+0000", 2023-10-09T10:52:26.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016812899999999999 2023-10-09T10:52:26.144 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.144 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.144 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.144 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.167706+0000", 2023-10-09T10:52:26.144 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0088551359999999996 2023-10-09T10:52:26.144 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.144 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.144 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.144 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.167780+0000", 2023-10-09T10:52:26.144 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4383999999999998e-05 2023-10-09T10:52:26.144 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.144 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.145 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.145 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.193627+0000", 2023-10-09T10:52:26.145 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025846593000000001 2023-10-09T10:52:26.145 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.145 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.145 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.145 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.193689+0000", 2023-10-09T10:52:26.145 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1681999999999997e-05 2023-10-09T10:52:26.145 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.145 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.145 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.145 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:58.194020+0000", 2023-10-09T10:52:26.146 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00033098999999999997 2023-10-09T10:52:26.146 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.146 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.146 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.146 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.146 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.146 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:212706 6.13 6:c8228b1c:::benchmark_data_smithi125_244951_object106263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-10-09T10:52:26.146 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:02.392352+0000", 2023-10-09T10:52:26.146 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 203.61260719699999, 2023-10-09T10:52:26.146 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.041188625999999999, 2023-10-09T10:52:26.146 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:52:26.147 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:26.147 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:52:26.147 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:52:26.147 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:26.147 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212706 2023-10-09T10:52:26.147 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.147 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:52:26.147 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.147 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:52:26.147 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392352+0000", 2023-10-09T10:52:26.147 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.147 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.148 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.148 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:52:26.148 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392352+0000", 2023-10-09T10:52:26.148 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:52:26.148 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.148 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.148 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:52:26.148 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392350+0000", 2023-10-09T10:52:26.148 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:26.148 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.148 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.148 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:52:26.149 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392357+0000", 2023-10-09T10:52:26.149 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0380000000000002e-06 2023-10-09T10:52:26.149 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.149 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.149 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:52:26.149 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392369+0000", 2023-10-09T10:52:26.149 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1857e-05 2023-10-09T10:52:26.149 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.149 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.149 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:52:26.149 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392375+0000", 2023-10-09T10:52:26.149 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5219999999999998e-06 2023-10-09T10:52:26.150 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.150 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.150 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:52:26.150 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392421+0000", 2023-10-09T10:52:26.150 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5736999999999999e-05 2023-10-09T10:52:26.150 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.150 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.150 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:52:26.150 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392554+0000", 2023-10-09T10:52:26.150 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013335300000000001 2023-10-09T10:52:26.150 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.150 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.151 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:52:26.151 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392651+0000", 2023-10-09T10:52:26.151 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7156000000000003e-05 2023-10-09T10:52:26.151 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.151 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.151 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.151 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.397156+0000", 2023-10-09T10:52:26.151 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045045550000000004 2023-10-09T10:52:26.151 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.151 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.151 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:52:26.151 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.400165+0000", 2023-10-09T10:52:26.152 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030094319999999998 2023-10-09T10:52:26.152 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.152 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.152 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:26.152 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.433364+0000", 2023-10-09T10:52:26.152 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.033199472000000001 2023-10-09T10:52:26.152 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.152 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.152 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:52:26.152 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.433409+0000", 2023-10-09T10:52:26.152 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4070000000000003e-05 2023-10-09T10:52:26.152 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:52:26.152 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:52:26.153 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:52:26.153 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.433541+0000", 2023-10-09T10:52:26.153 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013196400000000001 2023-10-09T10:52:26.153 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.153 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.153 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.153 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:52:26.153 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:52:26.153 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:52:26.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:26.155 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:52:26.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:52:26.255 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:26.293 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:26.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:26.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:26.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:26.345 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:52:26.346 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:26.439 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:26.505 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:26.544 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:26.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:26.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:26.558 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:52:26.652 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:26.685 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:26.714 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:26.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:26.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:26.812 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:26 smithi125 ceph-mon[195507]: pgmap v588: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 1.2 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 6983/60219 objects degraded (11.596%); 3784/60219 objects misplaced (6.284%); 0 B/s, 119 objects/s recovering 2023-10-09T10:52:26.812 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:26 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1022835140' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:52:26.812 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:26 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:52:26.812 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:26 smithi125 ceph-mon[200197]: pgmap v588: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 1.2 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 6983/60219 objects degraded (11.596%); 3784/60219 objects misplaced (6.284%); 0 B/s, 119 objects/s recovering 2023-10-09T10:52:26.812 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:26 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1022835140' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:52:26.813 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:26 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:52:26.823 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:26.869 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:26 smithi155 ceph-mon[184854]: pgmap v588: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 1.2 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 6983/60219 objects degraded (11.596%); 3784/60219 objects misplaced (6.284%); 0 B/s, 119 objects/s recovering 2023-10-09T10:52:26.870 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:26 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1022835140' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:52:26.870 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:26 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:52:26.870 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:26.895 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:26.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:26.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:27.001 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:27.059 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:27.089 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 1] out_osds: [2, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:52:27.089 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:52:27.089 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2023-10-09T10:52:27.089 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:52:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:27 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:52:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:27 smithi125 ceph-mon[195507]: failed to encode map e252 with expected crc 2023-10-09T10:52:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:27 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:52:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:27 smithi125 ceph-mon[195507]: osdmap e252: 8 total, 8 up, 6 in 2023-10-09T10:52:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:27 smithi125 ceph-mon[195507]: failed to encode map e252 with expected crc 2023-10-09T10:52:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:27 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:52:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:27 smithi125 ceph-mon[200197]: failed to encode map e252 with expected crc 2023-10-09T10:52:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:27 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:52:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:27 smithi125 ceph-mon[200197]: osdmap e252: 8 total, 8 up, 6 in 2023-10-09T10:52:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:27 smithi125 ceph-mon[200197]: failed to encode map e252 with expected crc 2023-10-09T10:52:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:27 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:52:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:27 smithi155 ceph-mon[184854]: failed to encode map e252 with expected crc 2023-10-09T10:52:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:27 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:52:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:27 smithi155 ceph-mon[184854]: osdmap e252: 8 total, 8 up, 6 in 2023-10-09T10:52:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:27 smithi155 ceph-mon[184854]: failed to encode map e252 with expected crc 2023-10-09T10:52:28.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:28 smithi125 ceph-mon[195507]: pgmap v590: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 1.2 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 759 op/s; 6983/58614 objects degraded (11.914%); 3515/58614 objects misplaced (5.997%); 0 B/s, 105 objects/s recovering 2023-10-09T10:52:28.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:28 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 6983/58614 objects degraded (11.914%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:52:28.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:28 smithi125 ceph-mon[200197]: pgmap v590: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 1.2 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 759 op/s; 6983/58614 objects degraded (11.914%); 3515/58614 objects misplaced (5.997%); 0 B/s, 105 objects/s recovering 2023-10-09T10:52:28.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:28 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 6983/58614 objects degraded (11.914%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:52:28.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:28 smithi155 ceph-mon[184854]: pgmap v590: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 1.2 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 759 op/s; 6983/58614 objects degraded (11.914%); 3515/58614 objects misplaced (5.997%); 0 B/s, 105 objects/s recovering 2023-10-09T10:52:28.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:28 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 6983/58614 objects degraded (11.914%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:52:29.234 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:29.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:29.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:29.339 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:29.366 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:29.385 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:29.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:29.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:29.488 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:52:29.547 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:29.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:29 smithi155 ceph-mon[184854]: pgmap v591: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 1.1 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 722 op/s; 5829/52371 objects degraded (11.130%); 3292/52371 objects misplaced (6.286%); 0 B/s, 126 objects/s recovering 2023-10-09T10:52:29.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:29 smithi125 ceph-mon[195507]: pgmap v591: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 1.1 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 722 op/s; 5829/52371 objects degraded (11.130%); 3292/52371 objects misplaced (6.286%); 0 B/s, 126 objects/s recovering 2023-10-09T10:52:29.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:29 smithi125 ceph-mon[200197]: pgmap v591: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 1.1 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 722 op/s; 5829/52371 objects degraded (11.130%); 3292/52371 objects misplaced (6.286%); 0 B/s, 126 objects/s recovering 2023-10-09T10:52:30.642 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:52:30.646 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:30.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:30.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:52:30.672 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:30.751 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:30.814 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:30.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:52:30.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:52:30.840 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:30.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:30.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:30.845 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:52:30.944 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:52:31.004 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:31.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:52:31.304 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:52:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:52:31] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-10-09T10:52:31.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:31 smithi125 ceph-mon[195507]: pgmap v592: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 959 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 797 op/s; 5570/46050 objects degraded (12.096%); 2779/46050 objects misplaced (6.035%); 0 B/s, 146 objects/s recovering 2023-10-09T10:52:31.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:31 smithi125 ceph-mon[200197]: pgmap v592: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 959 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 797 op/s; 5570/46050 objects degraded (12.096%); 2779/46050 objects misplaced (6.035%); 0 B/s, 146 objects/s recovering 2023-10-09T10:52:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:31 smithi155 ceph-mon[184854]: pgmap v592: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 959 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 797 op/s; 5570/46050 objects degraded (12.096%); 2779/46050 objects misplaced (6.035%); 0 B/s, 146 objects/s recovering 2023-10-09T10:52:32.001 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:52:32.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:32.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:52:32.111 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:32.158 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:32.195 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:32.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:32.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:32.256 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:52:32.300 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:32.309 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:52:32.339 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:52:32.374 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:32.395 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:52:32.433 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:52:32.466 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:52:32.522 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:52:32.553 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:52:32.559 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:52:32.559 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:52:32.559 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:52:32.559 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:52:32.559 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:52:32.560 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:52:32.560 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:52:32.560 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:52:32.693 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:32.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:32.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:32.802 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:32.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:32 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3034545655' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:32.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:32 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3034545655' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:32.858 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:32.891 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:32.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:32 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3034545655' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:32.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:32.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:32.997 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:33.063 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:33.204 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:33.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:33.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:33.308 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:52:33.369 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:33.928 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:33 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 5570/46050 objects degraded (12.096%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:52:34.171 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:33 smithi125 ceph-mon[195507]: pgmap v593: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 826 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 830 op/s; 4352/39690 objects degraded (10.965%); 2512/39690 objects misplaced (6.329%); 0 B/s, 149 objects/s recovering 2023-10-09T10:52:34.172 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:33 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 5570/46050 objects degraded (12.096%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:52:34.172 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:33 smithi125 ceph-mon[200197]: pgmap v593: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 826 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 830 op/s; 4352/39690 objects degraded (10.965%); 2512/39690 objects misplaced (6.329%); 0 B/s, 149 objects/s recovering 2023-10-09T10:52:34.172 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:52:34.172 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:52:34.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:34.174 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:52:34.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:52:34.174 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:33 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 5570/46050 objects degraded (12.096%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:52:34.174 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:33 smithi155 ceph-mon[184854]: pgmap v593: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 826 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 830 op/s; 4352/39690 objects degraded (10.965%); 2512/39690 objects misplaced (6.329%); 0 B/s, 149 objects/s recovering 2023-10-09T10:52:34.275 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:34.304 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:34.337 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:34.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:34.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:34.445 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:34.504 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:34.636 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:34.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:34.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:34.741 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:52:34.784 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:34.809 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:52:34.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:34.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:52:34.916 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:34.956 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:34.987 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:34.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:34.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:35.094 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:35.178 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:35.411 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:35.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:35.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:35.502 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:52:35.519 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:52:35.613 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:35.733 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:52:35.741 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:52:35.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:35.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:52:35.769 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:52:35.850 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:35.915 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:35.949 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:35.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:35.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:36.053 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:52:36.115 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:36.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:36 smithi155 ceph-mon[184854]: from='client.45281 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:52:36.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:36 smithi155 ceph-mon[184854]: pgmap v594: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 729 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 838 op/s; 4112/35046 objects degraded (11.733%); 2258/35046 objects misplaced (6.443%); 0 B/s, 152 objects/s recovering 2023-10-09T10:52:36.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:36 smithi125 ceph-mon[200197]: from='client.45281 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:52:36.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:36 smithi125 ceph-mon[200197]: pgmap v594: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 729 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 838 op/s; 4112/35046 objects degraded (11.733%); 2258/35046 objects misplaced (6.443%); 0 B/s, 152 objects/s recovering 2023-10-09T10:52:36.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:36 smithi125 ceph-mon[195507]: from='client.45281 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:52:36.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:36 smithi125 ceph-mon[195507]: pgmap v594: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 729 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 838 op/s; 4112/35046 objects degraded (11.733%); 2258/35046 objects misplaced (6.443%); 0 B/s, 152 objects/s recovering 2023-10-09T10:52:37.779 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:52:37.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:37.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:52:37.883 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:37.926 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:37.951 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:37.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:37.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:38.055 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:38.106 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:38.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:38.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:38.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:38.217 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:52:38.247 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:38.306 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:38.453 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:38 smithi125 ceph-mon[195507]: pgmap v595: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 700 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 772 op/s; 4112/33654 objects degraded (12.218%); 2044/33654 objects misplaced (6.074%); 0 B/s, 141 objects/s recovering 2023-10-09T10:52:38.453 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:38 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 4112/33654 objects degraded (12.218%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:52:38.453 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:38 smithi125 ceph-mon[200197]: pgmap v595: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 700 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 772 op/s; 4112/33654 objects degraded (12.218%); 2044/33654 objects misplaced (6.074%); 0 B/s, 141 objects/s recovering 2023-10-09T10:52:38.454 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:38 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 4112/33654 objects degraded (12.218%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:52:38.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:38 smithi155 ceph-mon[184854]: pgmap v595: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 700 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 772 op/s; 4112/33654 objects degraded (12.218%); 2044/33654 objects misplaced (6.074%); 0 B/s, 141 objects/s recovering 2023-10-09T10:52:38.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:38 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 4112/33654 objects degraded (12.218%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:52:38.841 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":593,"stamp":"2023-10-09T10:52:33.302844+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":866186258,"num_objects":13230,"num_object_clones":0,"num_object_copies":39690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4352,"num_objects_misplaced":2512,"num_objects_unfound":0,"num_objects_dirty":13230,"num_whiteouts":0,"num_read":317,"num_read_kb":596,"num_write":169279,"num_write_kb":5834179,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22572,"num_bytes_recovered":1204265054,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":81675,"ondisk_log_size":81675,"up":357,"acting":355,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":373,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":6189184,"kb_used_data":4223312,"kb_used_omap":44,"kb_used_meta":1965715,"kb_avail":556281728,"statfs":{"total":575970213888,"available":569632489472,"internally_reserved":0,"allocated":4324671488,"data_stored":4317766403,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":2012892577},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,10,3,8,4,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-544473088,"num_objects":-8308,"num_object_clones":0,"num_object_copies":-24924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2856,"num_objects_misplaced":-1533,"num_objects_unfound":0,"num_objects_dirty":-8308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8308,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1492,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007311"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":533,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.472657+0000","last_change":"2023-10-09T10:51:50.374063+0000","last_active":"2023-10-09T10:52:20.472657+0000","last_peered":"2023-10-09T10:52:20.472657+0000","last_clean":"2023-10-09T10:52:20.472657+0000","last_became_active":"2023-10-09T10:51:50.373610+0000","last_became_peered":"2023-10-09T10:51:50.373610+0000","last_unstale":"2023-10-09T10:52:20.472657+0000","last_undegraded":"2023-10-09T10:52:20.472657+0000","last_fullsized":"2023-10-09T10:52:20.472657+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.539709+0000","last_change":"2023-10-09T10:52:10.406855+0000","last_active":"2023-10-09T10:52:20.539709+0000","last_peered":"2023-10-09T10:52:20.539709+0000","last_clean":"2023-10-09T10:52:20.539709+0000","last_became_active":"2023-10-09T10:52:10.406435+0000","last_became_peered":"2023-10-09T10:52:10.406435+0000","last_unstale":"2023-10-09T10:52:20.539709+0000","last_undegraded":"2023-10-09T10:52:20.539709+0000","last_fullsized":"2023-10-09T10:52:20.539709+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:59:34.082764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":240,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.472852+0000","last_change":"2023-10-09T10:52:10.379409+0000","last_active":"2023-10-09T10:52:20.472852+0000","last_peered":"2023-10-09T10:52:20.472852+0000","last_clean":"2023-10-09T10:52:20.472852+0000","last_became_active":"2023-10-09T10:52:10.379097+0000","last_became_peered":"2023-10-09T10:52:10.379097+0000","last_unstale":"2023-10-09T10:52:20.472852+0000","last_undegraded":"2023-10-09T10:52:20.472852+0000","last_fullsized":"2023-10-09T10:52:20.472852+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:24:31.678460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":616,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.539162+0000","last_change":"2023-10-09T10:52:13.416805+0000","last_active":"2023-10-09T10:52:20.539162+0000","last_peered":"2023-10-09T10:52:20.539162+0000","last_clean":"2023-10-09T10:52:20.539162+0000","last_became_active":"2023-10-09T10:52:13.416463+0000","last_became_peered":"2023-10-09T10:52:13.416463+0000","last_unstale":"2023-10-09T10:52:20.539162+0000","last_undegraded":"2023-10-09T10:52:20.539162+0000","last_fullsized":"2023-10-09T10:52:20.539162+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.892515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.892515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.537944+0000","last_change":"2023-10-09T10:52:10.405482+0000","last_active":"2023-10-09T10:52:20.537944+0000","last_peered":"2023-10-09T10:52:20.537944+0000","last_clean":"2023-10-09T10:52:20.537944+0000","last_became_active":"2023-10-09T10:52:10.404851+0000","last_became_peered":"2023-10-09T10:52:10.404851+0000","last_unstale":"2023-10-09T10:52:20.537944+0000","last_undegraded":"2023-10-09T10:52:20.537944+0000","last_fullsized":"2023-10-09T10:52:20.537944+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:48.412641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":150,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.471601+0000","last_change":"2023-10-09T10:52:10.386651+0000","last_active":"2023-10-09T10:52:20.471601+0000","last_peered":"2023-10-09T10:52:20.471601+0000","last_clean":"2023-10-09T10:52:20.471601+0000","last_became_active":"2023-10-09T10:52:10.386412+0000","last_became_peered":"2023-10-09T10:52:10.386412+0000","last_unstale":"2023-10-09T10:52:20.471601+0000","last_undegraded":"2023-10-09T10:52:20.471601+0000","last_fullsized":"2023-10-09T10:52:20.471601+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:49.116500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":545,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.480583+0000","last_change":"2023-10-09T10:51:50.372232+0000","last_active":"2023-10-09T10:52:20.480583+0000","last_peered":"2023-10-09T10:52:20.480583+0000","last_clean":"2023-10-09T10:52:20.480583+0000","last_became_active":"2023-10-09T10:51:50.371900+0000","last_became_peered":"2023-10-09T10:51:50.371900+0000","last_unstale":"2023-10-09T10:52:20.480583+0000","last_undegraded":"2023-10-09T10:52:20.480583+0000","last_fullsized":"2023-10-09T10:52:20.480583+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":276,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473100+0000","last_change":"2023-10-09T10:52:10.379917+0000","last_active":"2023-10-09T10:52:20.473100+0000","last_peered":"2023-10-09T10:52:20.473100+0000","last_clean":"2023-10-09T10:52:20.473100+0000","last_became_active":"2023-10-09T10:52:10.379056+0000","last_became_peered":"2023-10-09T10:52:10.379056+0000","last_unstale":"2023-10-09T10:52:20.473100+0000","last_undegraded":"2023-10-09T10:52:20.473100+0000","last_fullsized":"2023-10-09T10:52:20.473100+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:34.556823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.542464+0000","last_change":"2023-10-09T10:52:10.372537+0000","last_active":"2023-10-09T10:52:20.542464+0000","last_peered":"2023-10-09T10:52:20.542464+0000","last_clean":"2023-10-09T10:52:20.542464+0000","last_became_active":"2023-10-09T10:52:10.372058+0000","last_became_peered":"2023-10-09T10:52:10.372058+0000","last_unstale":"2023-10-09T10:52:20.542464+0000","last_undegraded":"2023-10-09T10:52:20.542464+0000","last_fullsized":"2023-10-09T10:52:20.542464+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:46:35.708276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":616,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.480944+0000","last_change":"2023-10-09T10:51:01.315336+0000","last_active":"2023-10-09T10:52:20.480944+0000","last_peered":"2023-10-09T10:52:20.480944+0000","last_clean":"2023-10-09T10:52:20.480944+0000","last_became_active":"2023-10-09T10:51:01.314959+0000","last_became_peered":"2023-10-09T10:51:01.314959+0000","last_unstale":"2023-10-09T10:52:20.480944+0000","last_undegraded":"2023-10-09T10:52:20.480944+0000","last_fullsized":"2023-10-09T10:52:20.480944+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.829081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.829081+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001277842,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":122,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.472420+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:52:20.472420+0000","last_peered":"2023-10-09T10:52:20.472420+0000","last_clean":"2023-10-09T10:52:20.472420+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:52:20.472420+0000","last_undegraded":"2023-10-09T10:52:20.472420+0000","last_fullsized":"2023-10-09T10:52:20.472420+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":240,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473180+0000","last_change":"2023-10-09T10:52:10.380048+0000","last_active":"2023-10-09T10:52:20.473180+0000","last_peered":"2023-10-09T10:52:20.473180+0000","last_clean":"2023-10-09T10:52:20.473180+0000","last_became_active":"2023-10-09T10:52:10.379450+0000","last_became_peered":"2023-10-09T10:52:10.379450+0000","last_unstale":"2023-10-09T10:52:20.473180+0000","last_undegraded":"2023-10-09T10:52:20.473180+0000","last_fullsized":"2023-10-09T10:52:20.473180+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:36.402930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":523,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.538174+0000","last_change":"2023-10-09T10:52:13.413178+0000","last_active":"2023-10-09T10:52:20.538174+0000","last_peered":"2023-10-09T10:52:20.538174+0000","last_clean":"2023-10-09T10:52:20.538174+0000","last_became_active":"2023-10-09T10:52:13.412838+0000","last_became_peered":"2023-10-09T10:52:13.412838+0000","last_unstale":"2023-10-09T10:52:20.538174+0000","last_undegraded":"2023-10-09T10:52:20.538174+0000","last_fullsized":"2023-10-09T10:52:20.538174+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.540396+0000","last_change":"2023-10-09T10:52:10.383226+0000","last_active":"2023-10-09T10:52:20.540396+0000","last_peered":"2023-10-09T10:52:20.540396+0000","last_clean":"2023-10-09T10:52:20.540396+0000","last_became_active":"2023-10-09T10:52:10.382825+0000","last_became_peered":"2023-10-09T10:52:10.382825+0000","last_unstale":"2023-10-09T10:52:20.540396+0000","last_undegraded":"2023-10-09T10:52:20.540396+0000","last_fullsized":"2023-10-09T10:52:20.540396+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:51:38.458718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":188,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473771+0000","last_change":"2023-10-09T10:49:40.951753+0000","last_active":"2023-10-09T10:52:20.473771+0000","last_peered":"2023-10-09T10:52:20.473771+0000","last_clean":"2023-10-09T10:52:20.473771+0000","last_became_active":"2023-10-09T10:49:40.951441+0000","last_became_peered":"2023-10-09T10:49:40.951441+0000","last_unstale":"2023-10-09T10:52:20.473771+0000","last_undegraded":"2023-10-09T10:52:20.473771+0000","last_fullsized":"2023-10-09T10:52:20.473771+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:13.222334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":618,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473267+0000","last_change":"2023-10-09T10:51:12.365018+0000","last_active":"2023-10-09T10:52:20.473267+0000","last_peered":"2023-10-09T10:52:20.473267+0000","last_clean":"2023-10-09T10:52:20.473267+0000","last_became_active":"2023-10-09T10:51:12.364541+0000","last_became_peered":"2023-10-09T10:51:12.364541+0000","last_unstale":"2023-10-09T10:52:20.473267+0000","last_undegraded":"2023-10-09T10:52:20.473267+0000","last_fullsized":"2023-10-09T10:52:20.473267+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.687290+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.687290+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012763172999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":190,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.472788+0000","last_change":"2023-10-09T10:49:40.968706+0000","last_active":"2023-10-09T10:52:20.472788+0000","last_peered":"2023-10-09T10:52:20.472788+0000","last_clean":"2023-10-09T10:52:20.472788+0000","last_became_active":"2023-10-09T10:49:40.947149+0000","last_became_peered":"2023-10-09T10:49:40.947149+0000","last_unstale":"2023-10-09T10:52:20.472788+0000","last_undegraded":"2023-10-09T10:52:20.472788+0000","last_fullsized":"2023-10-09T10:52:20.472788+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:57:27.340414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.539854+0000","last_change":"2023-10-09T10:52:10.392814+0000","last_active":"2023-10-09T10:52:20.539854+0000","last_peered":"2023-10-09T10:52:20.539854+0000","last_clean":"2023-10-09T10:52:20.539854+0000","last_became_active":"2023-10-09T10:52:10.392416+0000","last_became_peered":"2023-10-09T10:52:10.392416+0000","last_unstale":"2023-10-09T10:52:20.539854+0000","last_undegraded":"2023-10-09T10:52:20.539854+0000","last_fullsized":"2023-10-09T10:52:20.539854+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:30.563121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":611,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.470769+0000","last_change":"2023-10-09T10:51:12.367124+0000","last_active":"2023-10-09T10:52:20.470769+0000","last_peered":"2023-10-09T10:52:20.470769+0000","last_clean":"2023-10-09T10:52:20.470769+0000","last_became_active":"2023-10-09T10:51:12.366405+0000","last_became_peered":"2023-10-09T10:51:12.366405+0000","last_unstale":"2023-10-09T10:52:20.470769+0000","last_undegraded":"2023-10-09T10:52:20.470769+0000","last_fullsized":"2023-10-09T10:52:20.470769+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.475429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.475429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001591812,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":204,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.470511+0000","last_change":"2023-10-09T10:51:12.366005+0000","last_active":"2023-10-09T10:52:20.470511+0000","last_peered":"2023-10-09T10:52:20.470511+0000","last_clean":"2023-10-09T10:52:20.470511+0000","last_became_active":"2023-10-09T10:51:12.365722+0000","last_became_peered":"2023-10-09T10:51:12.365722+0000","last_unstale":"2023-10-09T10:52:20.470511+0000","last_undegraded":"2023-10-09T10:52:20.470511+0000","last_fullsized":"2023-10-09T10:52:20.470511+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:31:04.499929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":236,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.481315+0000","last_change":"2023-10-09T10:52:10.392499+0000","last_active":"2023-10-09T10:52:20.481315+0000","last_peered":"2023-10-09T10:52:20.481315+0000","last_clean":"2023-10-09T10:52:20.481315+0000","last_became_active":"2023-10-09T10:52:10.392193+0000","last_became_peered":"2023-10-09T10:52:10.392193+0000","last_unstale":"2023-10-09T10:52:20.481315+0000","last_undegraded":"2023-10-09T10:52:20.481315+0000","last_fullsized":"2023-10-09T10:52:20.481315+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:39.058086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":616,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.470046+0000","last_change":"2023-10-09T10:49:18.708370+0000","last_active":"2023-10-09T10:52:20.470046+0000","last_peered":"2023-10-09T10:52:20.470046+0000","last_clean":"2023-10-09T10:52:20.470046+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:52:20.470046+0000","last_undegraded":"2023-10-09T10:52:20.470046+0000","last_fullsized":"2023-10-09T10:52:20.470046+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:57.808410+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":45,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.470104+0000","last_change":"2023-10-09T10:51:12.366055+0000","last_active":"2023-10-09T10:52:20.470104+0000","last_peered":"2023-10-09T10:52:20.470104+0000","last_clean":"2023-10-09T10:52:20.470104+0000","last_became_active":"2023-10-09T10:51:12.365754+0000","last_became_peered":"2023-10-09T10:51:12.365754+0000","last_unstale":"2023-10-09T10:52:20.470104+0000","last_undegraded":"2023-10-09T10:52:20.470104+0000","last_fullsized":"2023-10-09T10:52:20.470104+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:04.062342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":272,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473723+0000","last_change":"2023-10-09T10:52:10.381725+0000","last_active":"2023-10-09T10:52:20.473723+0000","last_peered":"2023-10-09T10:52:20.473723+0000","last_clean":"2023-10-09T10:52:20.473723+0000","last_became_active":"2023-10-09T10:52:10.378515+0000","last_became_peered":"2023-10-09T10:52:10.378515+0000","last_unstale":"2023-10-09T10:52:20.473723+0000","last_undegraded":"2023-10-09T10:52:20.473723+0000","last_fullsized":"2023-10-09T10:52:20.473723+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:04:30.121336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":587,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.480261+0000","last_change":"2023-10-09T10:51:50.372663+0000","last_active":"2023-10-09T10:52:20.480261+0000","last_peered":"2023-10-09T10:52:20.480261+0000","last_clean":"2023-10-09T10:52:20.480261+0000","last_became_active":"2023-10-09T10:51:50.371722+0000","last_became_peered":"2023-10-09T10:51:50.371722+0000","last_unstale":"2023-10-09T10:52:20.480261+0000","last_undegraded":"2023-10-09T10:52:20.480261+0000","last_fullsized":"2023-10-09T10:52:20.480261+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:08:34.850824+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":149,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.470045+0000","last_change":"2023-10-09T10:52:10.391042+0000","last_active":"2023-10-09T10:52:20.470045+0000","last_peered":"2023-10-09T10:52:20.470045+0000","last_clean":"2023-10-09T10:52:20.470045+0000","last_became_active":"2023-10-09T10:52:10.390784+0000","last_became_peered":"2023-10-09T10:52:10.390784+0000","last_unstale":"2023-10-09T10:52:20.470045+0000","last_undegraded":"2023-10-09T10:52:20.470045+0000","last_fullsized":"2023-10-09T10:52:20.470045+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:57.404148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":192,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.472968+0000","last_change":"2023-10-09T10:51:50.373536+0000","last_active":"2023-10-09T10:52:20.472968+0000","last_peered":"2023-10-09T10:52:20.472968+0000","last_clean":"2023-10-09T10:52:20.472968+0000","last_became_active":"2023-10-09T10:51:50.373032+0000","last_became_peered":"2023-10-09T10:51:50.373032+0000","last_unstale":"2023-10-09T10:52:20.472968+0000","last_undegraded":"2023-10-09T10:52:20.472968+0000","last_fullsized":"2023-10-09T10:52:20.472968+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:15:53.653399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":552,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.539016+0000","last_change":"2023-10-09T10:52:13.424402+0000","last_active":"2023-10-09T10:52:20.539016+0000","last_peered":"2023-10-09T10:52:20.539016+0000","last_clean":"2023-10-09T10:52:20.539016+0000","last_became_active":"2023-10-09T10:52:13.424095+0000","last_became_peered":"2023-10-09T10:52:13.424095+0000","last_unstale":"2023-10-09T10:52:20.539016+0000","last_undegraded":"2023-10-09T10:52:20.539016+0000","last_fullsized":"2023-10-09T10:52:20.539016+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":172,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.480608+0000","last_change":"2023-10-09T10:49:40.952105+0000","last_active":"2023-10-09T10:52:20.480608+0000","last_peered":"2023-10-09T10:52:20.480608+0000","last_clean":"2023-10-09T10:52:20.480608+0000","last_became_active":"2023-10-09T10:49:40.950553+0000","last_became_peered":"2023-10-09T10:49:40.950553+0000","last_unstale":"2023-10-09T10:52:20.480608+0000","last_undegraded":"2023-10-09T10:52:20.480608+0000","last_fullsized":"2023-10-09T10:52:20.480608+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:51:40.501639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":131,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.471143+0000","last_change":"2023-10-09T10:49:40.949501+0000","last_active":"2023-10-09T10:52:20.471143+0000","last_peered":"2023-10-09T10:52:20.471143+0000","last_clean":"2023-10-09T10:52:20.471143+0000","last_became_active":"2023-10-09T10:49:40.949158+0000","last_became_peered":"2023-10-09T10:49:40.949158+0000","last_unstale":"2023-10-09T10:52:20.471143+0000","last_undegraded":"2023-10-09T10:52:20.471143+0000","last_fullsized":"2023-10-09T10:52:20.471143+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:07.598560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":601,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.539568+0000","last_change":"2023-10-09T10:52:13.413332+0000","last_active":"2023-10-09T10:52:20.539568+0000","last_peered":"2023-10-09T10:52:20.539568+0000","last_clean":"2023-10-09T10:52:20.539568+0000","last_became_active":"2023-10-09T10:52:13.412975+0000","last_became_peered":"2023-10-09T10:52:13.412975+0000","last_unstale":"2023-10-09T10:52:20.539568+0000","last_undegraded":"2023-10-09T10:52:20.539568+0000","last_fullsized":"2023-10-09T10:52:20.539568+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:25.778974+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":125,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.470026+0000","last_change":"2023-10-09T10:51:12.366910+0000","last_active":"2023-10-09T10:52:20.470026+0000","last_peered":"2023-10-09T10:52:20.470026+0000","last_clean":"2023-10-09T10:52:20.470026+0000","last_became_active":"2023-10-09T10:51:12.366565+0000","last_became_peered":"2023-10-09T10:51:12.366565+0000","last_unstale":"2023-10-09T10:52:20.470026+0000","last_undegraded":"2023-10-09T10:52:20.470026+0000","last_fullsized":"2023-10-09T10:52:20.470026+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:53.464251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":249,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473605+0000","last_change":"2023-10-09T10:52:10.378688+0000","last_active":"2023-10-09T10:52:20.473605+0000","last_peered":"2023-10-09T10:52:20.473605+0000","last_clean":"2023-10-09T10:52:20.473605+0000","last_became_active":"2023-10-09T10:52:10.378377+0000","last_became_peered":"2023-10-09T10:52:10.378377+0000","last_unstale":"2023-10-09T10:52:20.473605+0000","last_undegraded":"2023-10-09T10:52:20.473605+0000","last_fullsized":"2023-10-09T10:52:20.473605+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:13:45.982238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"252'10590","reported_seq":17009,"reported_epoch":252,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:52:31.108688+0000","last_change":"2023-10-09T10:52:12.174623+0000","last_active":"2023-10-09T10:52:31.108688+0000","last_peered":"2023-10-09T10:52:31.108688+0000","last_clean":"2023-10-09T10:52:09.294816+0000","last_became_active":"2023-10-09T10:52:12.138455+0000","last_became_peered":"2023-10-09T10:52:12.138455+0000","last_unstale":"2023-10-09T10:52:31.108688+0000","last_undegraded":"2023-10-09T10:52:31.108688+0000","last_fullsized":"2023-10-09T10:52:31.108688+0000","mapping_epoch":245,"log_start":"224'5501","ondisk_log_start":"224'5501","created":196,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":5089,"log_dups_size":0,"ondisk_log_size":5089,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:49:20.911922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54919168,"num_objects":838,"num_object_clones":0,"num_object_copies":2514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":838,"num_objects_unfound":0,"num_objects_dirty":838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10590,"num_write_kb":365696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3165,"num_bytes_recovered":207421440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[4,5,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":838}],"blocked_by":[],"up_primary":1,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":523,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.539990+0000","last_change":"2023-10-09T10:52:13.414704+0000","last_active":"2023-10-09T10:52:20.539990+0000","last_peered":"2023-10-09T10:52:20.539990+0000","last_clean":"2023-10-09T10:52:20.539990+0000","last_became_active":"2023-10-09T10:52:13.414277+0000","last_became_peered":"2023-10-09T10:52:13.414277+0000","last_unstale":"2023-10-09T10:52:20.539990+0000","last_undegraded":"2023-10-09T10:52:20.539990+0000","last_fullsized":"2023-10-09T10:52:20.539990+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":105,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.481572+0000","last_change":"2023-10-09T10:52:10.444698+0000","last_active":"2023-10-09T10:52:20.481572+0000","last_peered":"2023-10-09T10:52:20.481572+0000","last_clean":"2023-10-09T10:52:20.481572+0000","last_became_active":"2023-10-09T10:52:10.394194+0000","last_became_peered":"2023-10-09T10:52:10.394194+0000","last_unstale":"2023-10-09T10:52:20.481572+0000","last_undegraded":"2023-10-09T10:52:20.481572+0000","last_fullsized":"2023-10-09T10:52:20.481572+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:58.599587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":209,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.475296+0000","last_change":"2023-10-09T10:49:40.955088+0000","last_active":"2023-10-09T10:52:20.475296+0000","last_peered":"2023-10-09T10:52:20.475296+0000","last_clean":"2023-10-09T10:52:20.475296+0000","last_became_active":"2023-10-09T10:49:40.954716+0000","last_became_peered":"2023-10-09T10:49:40.954716+0000","last_unstale":"2023-10-09T10:52:20.475296+0000","last_undegraded":"2023-10-09T10:52:20.475296+0000","last_fullsized":"2023-10-09T10:52:20.475296+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:39.090107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"252'10447","reported_seq":10531,"reported_epoch":252,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:52:29.911163+0000","last_change":"2023-10-09T10:52:12.159689+0000","last_active":"2023-10-09T10:52:29.911163+0000","last_peered":"2023-10-09T10:52:29.911163+0000","last_clean":"2023-10-09T10:49:54.734240+0000","last_became_active":"2023-10-09T10:52:12.121140+0000","last_became_peered":"2023-10-09T10:52:12.121140+0000","last_unstale":"2023-10-09T10:52:29.911163+0000","last_undegraded":"2023-10-09T10:52:11.469334+0000","last_fullsized":"2023-10-09T10:52:29.911163+0000","mapping_epoch":245,"log_start":"225'5690","ondisk_log_start":"225'5690","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4757,"log_dups_size":0,"ondisk_log_size":4757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:48.795406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61931520,"num_objects":945,"num_object_clones":0,"num_object_copies":2835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":945,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10447,"num_write_kb":364544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,4],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":945}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":598,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473543+0000","last_change":"2023-10-09T10:51:01.296071+0000","last_active":"2023-10-09T10:52:20.473543+0000","last_peered":"2023-10-09T10:52:20.473543+0000","last_clean":"2023-10-09T10:52:20.473543+0000","last_became_active":"2023-10-09T10:51:01.295474+0000","last_became_peered":"2023-10-09T10:51:01.295474+0000","last_unstale":"2023-10-09T10:52:20.473543+0000","last_undegraded":"2023-10-09T10:52:20.473543+0000","last_fullsized":"2023-10-09T10:52:20.473543+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:41.637984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:41.637984+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001804833,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":22,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.540241+0000","last_change":"2023-10-09T10:52:10.442531+0000","last_active":"2023-10-09T10:52:20.540241+0000","last_peered":"2023-10-09T10:52:20.540241+0000","last_clean":"2023-10-09T10:52:20.540241+0000","last_became_active":"2023-10-09T10:52:10.411619+0000","last_became_peered":"2023-10-09T10:52:10.411619+0000","last_unstale":"2023-10-09T10:52:20.540241+0000","last_undegraded":"2023-10-09T10:52:20.540241+0000","last_fullsized":"2023-10-09T10:52:20.540241+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:11.313709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.538017+0000","last_change":"2023-10-09T10:52:10.405631+0000","last_active":"2023-10-09T10:52:20.538017+0000","last_peered":"2023-10-09T10:52:20.538017+0000","last_clean":"2023-10-09T10:52:20.538017+0000","last_became_active":"2023-10-09T10:52:10.405228+0000","last_became_peered":"2023-10-09T10:52:10.405228+0000","last_unstale":"2023-10-09T10:52:20.538017+0000","last_undegraded":"2023-10-09T10:52:20.538017+0000","last_fullsized":"2023-10-09T10:52:20.538017+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:53.535609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"252'10885","reported_seq":10966,"reported_epoch":252,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:52:33.191598+0000","last_change":"2023-10-09T10:52:12.178041+0000","last_active":"2023-10-09T10:52:33.191598+0000","last_peered":"2023-10-09T10:52:33.191598+0000","last_clean":"2023-10-09T10:49:54.793473+0000","last_became_active":"2023-10-09T10:52:12.144470+0000","last_became_peered":"2023-10-09T10:52:12.144470+0000","last_unstale":"2023-10-09T10:52:33.191598+0000","last_undegraded":"2023-10-09T10:52:11.465780+0000","last_fullsized":"2023-10-09T10:52:33.191598+0000","mapping_epoch":245,"log_start":"227'6132","ondisk_log_start":"227'6132","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4753,"log_dups_size":0,"ondisk_log_size":4753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:46:19.131705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48693248,"num_objects":743,"num_object_clones":0,"num_object_copies":2229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":743,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10885,"num_write_kb":372096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,4,6],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4,6","objects":743}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":554,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.481525+0000","last_change":"2023-10-09T10:51:50.371393+0000","last_active":"2023-10-09T10:52:20.481525+0000","last_peered":"2023-10-09T10:52:20.481525+0000","last_clean":"2023-10-09T10:52:20.481525+0000","last_became_active":"2023-10-09T10:51:50.371032+0000","last_became_peered":"2023-10-09T10:51:50.371032+0000","last_unstale":"2023-10-09T10:52:20.481525+0000","last_undegraded":"2023-10-09T10:52:20.481525+0000","last_fullsized":"2023-10-09T10:52:20.481525+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":149,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.472618+0000","last_change":"2023-10-09T10:51:50.373832+0000","last_active":"2023-10-09T10:52:20.472618+0000","last_peered":"2023-10-09T10:52:20.472618+0000","last_clean":"2023-10-09T10:52:20.472618+0000","last_became_active":"2023-10-09T10:51:50.373476+0000","last_became_peered":"2023-10-09T10:51:50.373476+0000","last_unstale":"2023-10-09T10:52:20.472618+0000","last_undegraded":"2023-10-09T10:52:20.472618+0000","last_fullsized":"2023-10-09T10:52:20.472618+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:18:58.804710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.539635+0000","last_change":"2023-10-09T10:52:10.373157+0000","last_active":"2023-10-09T10:52:20.539635+0000","last_peered":"2023-10-09T10:52:20.539635+0000","last_clean":"2023-10-09T10:52:20.539635+0000","last_became_active":"2023-10-09T10:52:10.372722+0000","last_became_peered":"2023-10-09T10:52:10.372722+0000","last_unstale":"2023-10-09T10:52:20.539635+0000","last_undegraded":"2023-10-09T10:52:20.539635+0000","last_fullsized":"2023-10-09T10:52:20.539635+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:56.520627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"252'10660","reported_seq":15560,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T10:52:31.191216+0000","last_change":"2023-10-09T10:50:27.531707+0000","last_active":"2023-10-09T10:52:31.191216+0000","last_peered":"2023-10-09T10:52:31.191216+0000","last_clean":"2023-10-09T10:52:31.191216+0000","last_became_active":"2023-10-09T10:50:27.518955+0000","last_became_peered":"2023-10-09T10:50:27.518955+0000","last_unstale":"2023-10-09T10:52:31.191216+0000","last_undegraded":"2023-10-09T10:52:31.191216+0000","last_fullsized":"2023-10-09T10:52:31.191216+0000","mapping_epoch":219,"log_start":"224'5479","ondisk_log_start":"224'5479","created":196,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":5181,"log_dups_size":0,"ondisk_log_size":5181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:29:34.331554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56492032,"num_objects":862,"num_object_clones":0,"num_object_copies":2586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10660,"num_write_kb":368704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2408,"num_bytes_recovered":157810688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":667,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.481180+0000","last_change":"2023-10-09T10:52:13.418596+0000","last_active":"2023-10-09T10:52:20.481180+0000","last_peered":"2023-10-09T10:52:20.481180+0000","last_clean":"2023-10-09T10:52:20.481180+0000","last_became_active":"2023-10-09T10:52:13.418103+0000","last_became_peered":"2023-10-09T10:52:13.418103+0000","last_unstale":"2023-10-09T10:52:20.481180+0000","last_undegraded":"2023-10-09T10:52:20.481180+0000","last_fullsized":"2023-10-09T10:52:20.481180+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":76,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473984+0000","last_change":"2023-10-09T10:51:50.374527+0000","last_active":"2023-10-09T10:52:20.473984+0000","last_peered":"2023-10-09T10:52:20.473984+0000","last_clean":"2023-10-09T10:52:20.473984+0000","last_became_active":"2023-10-09T10:51:50.374297+0000","last_became_peered":"2023-10-09T10:51:50.374297+0000","last_unstale":"2023-10-09T10:52:20.473984+0000","last_undegraded":"2023-10-09T10:52:20.473984+0000","last_fullsized":"2023-10-09T10:52:20.473984+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:31:49.358307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":140,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.470386+0000","last_change":"2023-10-09T10:52:10.385820+0000","last_active":"2023-10-09T10:52:20.470386+0000","last_peered":"2023-10-09T10:52:20.470386+0000","last_clean":"2023-10-09T10:52:20.470386+0000","last_became_active":"2023-10-09T10:52:10.385498+0000","last_became_peered":"2023-10-09T10:52:10.385498+0000","last_unstale":"2023-10-09T10:52:20.470386+0000","last_undegraded":"2023-10-09T10:52:20.470386+0000","last_fullsized":"2023-10-09T10:52:20.470386+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:18.341986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"252'10563","reported_seq":13285,"reported_epoch":252,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:52:33.199948+0000","last_change":"2023-10-09T10:52:18.805046+0000","last_active":"2023-10-09T10:52:33.199948+0000","last_peered":"2023-10-09T10:52:33.199948+0000","last_clean":"2023-10-09T10:49:39.916756+0000","last_became_active":"2023-10-09T10:49:42.056867+0000","last_became_peered":"2023-10-09T10:49:42.056867+0000","last_unstale":"2023-10-09T10:52:33.199948+0000","last_undegraded":"2023-10-09T10:49:41.970254+0000","last_fullsized":"2023-10-09T10:49:41.933564+0000","mapping_epoch":203,"log_start":"226'5864","ondisk_log_start":"226'5864","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4699,"log_dups_size":0,"ondisk_log_size":4699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:59:10.773024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48955392,"num_objects":747,"num_object_clones":0,"num_object_copies":2241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1097,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10563,"num_write_kb":361920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1322,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-350}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":613,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473573+0000","last_change":"2023-10-09T10:51:01.305574+0000","last_active":"2023-10-09T10:52:20.473573+0000","last_peered":"2023-10-09T10:52:20.473573+0000","last_clean":"2023-10-09T10:52:20.473573+0000","last_became_active":"2023-10-09T10:51:01.305381+0000","last_became_peered":"2023-10-09T10:51:01.305381+0000","last_unstale":"2023-10-09T10:52:20.473573+0000","last_undegraded":"2023-10-09T10:52:20.473573+0000","last_fullsized":"2023-10-09T10:52:20.473573+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":247,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T10:52:26.556791+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:52:26.556791+0000","last_peered":"2023-10-09T10:52:26.556791+0000","last_clean":"2023-10-09T10:52:26.556791+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:52:26.556791+0000","last_undegraded":"2023-10-09T10:52:26.556791+0000","last_fullsized":"2023-10-09T10:52:26.556791+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":91,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473864+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T10:52:20.473864+0000","last_peered":"2023-10-09T10:52:20.473864+0000","last_clean":"2023-10-09T10:52:20.473864+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T10:52:20.473864+0000","last_undegraded":"2023-10-09T10:52:20.473864+0000","last_fullsized":"2023-10-09T10:52:20.473864+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"252'10513","reported_seq":10584,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T10:52:33.133270+0000","last_change":"2023-10-09T10:51:50.399646+0000","last_active":"2023-10-09T10:52:33.133270+0000","last_peered":"2023-10-09T10:52:33.133270+0000","last_clean":"2023-10-09T10:52:33.133270+0000","last_became_active":"2023-10-09T10:51:50.376297+0000","last_became_peered":"2023-10-09T10:51:50.376297+0000","last_unstale":"2023-10-09T10:52:33.133270+0000","last_undegraded":"2023-10-09T10:52:33.133270+0000","last_fullsized":"2023-10-09T10:52:33.133270+0000","mapping_epoch":239,"log_start":"225'5756","ondisk_log_start":"225'5756","created":196,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4757,"log_dups_size":0,"ondisk_log_size":4757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:25.950800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47382528,"num_objects":723,"num_object_clones":0,"num_object_copies":2169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10513,"num_write_kb":359552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":533,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.472829+0000","last_change":"2023-10-09T10:51:50.373921+0000","last_active":"2023-10-09T10:52:20.472829+0000","last_peered":"2023-10-09T10:52:20.472829+0000","last_clean":"2023-10-09T10:52:20.472829+0000","last_became_active":"2023-10-09T10:51:50.373185+0000","last_became_peered":"2023-10-09T10:51:50.373185+0000","last_unstale":"2023-10-09T10:52:20.472829+0000","last_undegraded":"2023-10-09T10:52:20.472829+0000","last_fullsized":"2023-10-09T10:52:20.472829+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":211,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.480660+0000","last_change":"2023-10-09T10:52:10.392827+0000","last_active":"2023-10-09T10:52:20.480660+0000","last_peered":"2023-10-09T10:52:20.480660+0000","last_clean":"2023-10-09T10:52:20.480660+0000","last_became_active":"2023-10-09T10:52:10.392082+0000","last_became_peered":"2023-10-09T10:52:10.392082+0000","last_unstale":"2023-10-09T10:52:20.480660+0000","last_undegraded":"2023-10-09T10:52:20.480660+0000","last_fullsized":"2023-10-09T10:52:20.480660+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:48.702641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.539833+0000","last_change":"2023-10-09T10:52:10.409941+0000","last_active":"2023-10-09T10:52:20.539833+0000","last_peered":"2023-10-09T10:52:20.539833+0000","last_clean":"2023-10-09T10:52:20.539833+0000","last_became_active":"2023-10-09T10:52:10.409687+0000","last_became_peered":"2023-10-09T10:52:10.409687+0000","last_unstale":"2023-10-09T10:52:20.539833+0000","last_undegraded":"2023-10-09T10:52:20.539833+0000","last_fullsized":"2023-10-09T10:52:20.539833+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:23:37.794696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"252'10447","reported_seq":16774,"reported_epoch":252,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:52:33.191889+0000","last_change":"2023-10-09T10:52:12.168794+0000","last_active":"2023-10-09T10:52:33.191889+0000","last_peered":"2023-10-09T10:52:33.191889+0000","last_clean":"2023-10-09T10:49:54.776483+0000","last_became_active":"2023-10-09T10:52:12.139812+0000","last_became_peered":"2023-10-09T10:52:12.139812+0000","last_unstale":"2023-10-09T10:52:33.191889+0000","last_undegraded":"2023-10-09T10:52:33.191889+0000","last_fullsized":"2023-10-09T10:52:33.191889+0000","mapping_epoch":245,"log_start":"226'5752","ondisk_log_start":"226'5752","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4695,"log_dups_size":0,"ondisk_log_size":4695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:25.794876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48103448,"num_objects":735,"num_object_clones":0,"num_object_copies":2205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":735,"num_objects_unfound":0,"num_objects_dirty":735,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10447,"num_write_kb":357761,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3118,"num_bytes_recovered":204341248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[0,6,3],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":735}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":616,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.539090+0000","last_change":"2023-10-09T10:52:13.417673+0000","last_active":"2023-10-09T10:52:20.539090+0000","last_peered":"2023-10-09T10:52:20.539090+0000","last_clean":"2023-10-09T10:52:20.539090+0000","last_became_active":"2023-10-09T10:52:13.417290+0000","last_became_peered":"2023-10-09T10:52:13.417290+0000","last_unstale":"2023-10-09T10:52:20.539090+0000","last_undegraded":"2023-10-09T10:52:20.539090+0000","last_fullsized":"2023-10-09T10:52:20.539090+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.892515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.892515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":257,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.480733+0000","last_change":"2023-10-09T10:52:10.385085+0000","last_active":"2023-10-09T10:52:20.480733+0000","last_peered":"2023-10-09T10:52:20.480733+0000","last_clean":"2023-10-09T10:52:20.480733+0000","last_became_active":"2023-10-09T10:52:10.384815+0000","last_became_peered":"2023-10-09T10:52:10.384815+0000","last_unstale":"2023-10-09T10:52:20.480733+0000","last_undegraded":"2023-10-09T10:52:20.480733+0000","last_fullsized":"2023-10-09T10:52:20.480733+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:27.373292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.540153+0000","last_change":"2023-10-09T10:52:10.412518+0000","last_active":"2023-10-09T10:52:20.540153+0000","last_peered":"2023-10-09T10:52:20.540153+0000","last_clean":"2023-10-09T10:52:20.540153+0000","last_became_active":"2023-10-09T10:52:10.411988+0000","last_became_peered":"2023-10-09T10:52:10.411988+0000","last_unstale":"2023-10-09T10:52:20.540153+0000","last_undegraded":"2023-10-09T10:52:20.540153+0000","last_fullsized":"2023-10-09T10:52:20.540153+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:13.191079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.c","version":"252'10356","reported_seq":15135,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T10:52:29.939657+0000","last_change":"2023-10-09T10:50:14.783907+0000","last_active":"2023-10-09T10:52:29.939657+0000","last_peered":"2023-10-09T10:52:29.939657+0000","last_clean":"2023-10-09T10:52:29.939657+0000","last_became_active":"2023-10-09T10:50:14.780725+0000","last_became_peered":"2023-10-09T10:50:14.780725+0000","last_unstale":"2023-10-09T10:52:29.939657+0000","last_undegraded":"2023-10-09T10:52:29.939657+0000","last_fullsized":"2023-10-09T10:52:29.939657+0000","mapping_epoch":215,"log_start":"225'5611","ondisk_log_start":"225'5611","created":196,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4745,"log_dups_size":0,"ondisk_log_size":4745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:11:40.081379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60162048,"num_objects":918,"num_object_clones":0,"num_object_copies":2754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10356,"num_write_kb":360768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2347,"num_bytes_recovered":153812992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":545,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.480591+0000","last_change":"2023-10-09T10:51:50.372344+0000","last_active":"2023-10-09T10:52:20.480591+0000","last_peered":"2023-10-09T10:52:20.480591+0000","last_clean":"2023-10-09T10:52:20.480591+0000","last_became_active":"2023-10-09T10:51:50.372038+0000","last_became_peered":"2023-10-09T10:51:50.372038+0000","last_unstale":"2023-10-09T10:52:20.480591+0000","last_undegraded":"2023-10-09T10:52:20.480591+0000","last_fullsized":"2023-10-09T10:52:20.480591+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":251,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.472921+0000","last_change":"2023-10-09T10:52:10.382147+0000","last_active":"2023-10-09T10:52:20.472921+0000","last_peered":"2023-10-09T10:52:20.472921+0000","last_clean":"2023-10-09T10:52:20.472921+0000","last_became_active":"2023-10-09T10:52:10.381778+0000","last_became_peered":"2023-10-09T10:52:10.381778+0000","last_unstale":"2023-10-09T10:52:20.472921+0000","last_undegraded":"2023-10-09T10:52:20.472921+0000","last_fullsized":"2023-10-09T10:52:20.472921+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:05.494868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.542593+0000","last_change":"2023-10-09T10:52:10.407797+0000","last_active":"2023-10-09T10:52:20.542593+0000","last_peered":"2023-10-09T10:52:20.542593+0000","last_clean":"2023-10-09T10:52:20.542593+0000","last_became_active":"2023-10-09T10:52:10.407481+0000","last_became_peered":"2023-10-09T10:52:10.407481+0000","last_unstale":"2023-10-09T10:52:20.542593+0000","last_undegraded":"2023-10-09T10:52:20.542593+0000","last_fullsized":"2023-10-09T10:52:20.542593+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:54.829995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"252'10317","reported_seq":10387,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T10:52:29.920509+0000","last_change":"2023-10-09T10:50:43.392944+0000","last_active":"2023-10-09T10:52:29.920509+0000","last_peered":"2023-10-09T10:52:29.920509+0000","last_clean":"2023-10-09T10:52:29.920509+0000","last_became_active":"2023-10-09T10:50:43.384716+0000","last_became_peered":"2023-10-09T10:50:43.384716+0000","last_unstale":"2023-10-09T10:52:29.920509+0000","last_undegraded":"2023-10-09T10:52:29.920509+0000","last_fullsized":"2023-10-09T10:52:29.920509+0000","mapping_epoch":223,"log_start":"225'5573","ondisk_log_start":"225'5573","created":196,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4744,"log_dups_size":0,"ondisk_log_size":4744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:19.728024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58261504,"num_objects":889,"num_object_clones":0,"num_object_copies":2667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10317,"num_write_kb":358592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":618,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473173+0000","last_change":"2023-10-09T10:51:12.364831+0000","last_active":"2023-10-09T10:52:20.473173+0000","last_peered":"2023-10-09T10:52:20.473173+0000","last_clean":"2023-10-09T10:52:20.473173+0000","last_became_active":"2023-10-09T10:51:12.364197+0000","last_became_peered":"2023-10-09T10:51:12.364197+0000","last_unstale":"2023-10-09T10:52:20.473173+0000","last_undegraded":"2023-10-09T10:52:20.473173+0000","last_fullsized":"2023-10-09T10:52:20.473173+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.687290+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.687290+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012763172999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.538096+0000","last_change":"2023-10-09T10:52:10.391327+0000","last_active":"2023-10-09T10:52:20.538096+0000","last_peered":"2023-10-09T10:52:20.538096+0000","last_clean":"2023-10-09T10:52:20.538096+0000","last_became_active":"2023-10-09T10:52:10.390819+0000","last_became_peered":"2023-10-09T10:52:10.390819+0000","last_unstale":"2023-10-09T10:52:20.538096+0000","last_undegraded":"2023-10-09T10:52:20.538096+0000","last_fullsized":"2023-10-09T10:52:20.538096+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:30.331038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":161,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.472705+0000","last_change":"2023-10-09T10:51:50.373359+0000","last_active":"2023-10-09T10:52:20.472705+0000","last_peered":"2023-10-09T10:52:20.472705+0000","last_clean":"2023-10-09T10:52:20.472705+0000","last_became_active":"2023-10-09T10:51:50.373009+0000","last_became_peered":"2023-10-09T10:51:50.373009+0000","last_unstale":"2023-10-09T10:52:20.472705+0000","last_undegraded":"2023-10-09T10:52:20.472705+0000","last_fullsized":"2023-10-09T10:52:20.472705+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:11.509198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.539498+0000","last_change":"2023-10-09T10:52:10.406697+0000","last_active":"2023-10-09T10:52:20.539498+0000","last_peered":"2023-10-09T10:52:20.539498+0000","last_clean":"2023-10-09T10:52:20.539498+0000","last_became_active":"2023-10-09T10:52:10.405905+0000","last_became_peered":"2023-10-09T10:52:10.405905+0000","last_unstale":"2023-10-09T10:52:20.539498+0000","last_undegraded":"2023-10-09T10:52:20.539498+0000","last_fullsized":"2023-10-09T10:52:20.539498+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:27:39.691503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"252'10815","reported_seq":10891,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T10:52:33.180978+0000","last_change":"2023-10-09T10:51:12.374840+0000","last_active":"2023-10-09T10:52:33.180978+0000","last_peered":"2023-10-09T10:52:33.180978+0000","last_clean":"2023-10-09T10:52:33.180978+0000","last_became_active":"2023-10-09T10:51:12.366993+0000","last_became_peered":"2023-10-09T10:51:12.366993+0000","last_unstale":"2023-10-09T10:52:33.180978+0000","last_undegraded":"2023-10-09T10:52:33.180978+0000","last_fullsized":"2023-10-09T10:52:33.180978+0000","mapping_epoch":233,"log_start":"226'6063","ondisk_log_start":"226'6063","created":196,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4752,"log_dups_size":0,"ondisk_log_size":4752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:47.959952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50135040,"num_objects":765,"num_object_clones":0,"num_object_copies":2295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10815,"num_write_kb":370560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":558,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.469962+0000","last_change":"2023-10-09T10:51:50.376468+0000","last_active":"2023-10-09T10:52:20.469962+0000","last_peered":"2023-10-09T10:52:20.469962+0000","last_clean":"2023-10-09T10:52:20.469962+0000","last_became_active":"2023-10-09T10:51:50.376078+0000","last_became_peered":"2023-10-09T10:51:50.376078+0000","last_unstale":"2023-10-09T10:52:20.469962+0000","last_undegraded":"2023-10-09T10:52:20.469962+0000","last_fullsized":"2023-10-09T10:52:20.469962+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.736834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.736834+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.539435+0000","last_change":"2023-10-09T10:52:10.372993+0000","last_active":"2023-10-09T10:52:20.539435+0000","last_peered":"2023-10-09T10:52:20.539435+0000","last_clean":"2023-10-09T10:52:20.539435+0000","last_became_active":"2023-10-09T10:52:10.372346+0000","last_became_peered":"2023-10-09T10:52:10.372346+0000","last_unstale":"2023-10-09T10:52:20.539435+0000","last_undegraded":"2023-10-09T10:52:20.539435+0000","last_fullsized":"2023-10-09T10:52:20.539435+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:55.012253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":159,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.470201+0000","last_change":"2023-10-09T10:50:43.388615+0000","last_active":"2023-10-09T10:52:20.470201+0000","last_peered":"2023-10-09T10:52:20.470201+0000","last_clean":"2023-10-09T10:52:20.470201+0000","last_became_active":"2023-10-09T10:50:43.388176+0000","last_became_peered":"2023-10-09T10:50:43.388176+0000","last_unstale":"2023-10-09T10:52:20.470201+0000","last_undegraded":"2023-10-09T10:52:20.470201+0000","last_fullsized":"2023-10-09T10:52:20.470201+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:41:17.550111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.538253+0000","last_change":"2023-10-09T10:52:10.380162+0000","last_active":"2023-10-09T10:52:20.538253+0000","last_peered":"2023-10-09T10:52:20.538253+0000","last_clean":"2023-10-09T10:52:20.538253+0000","last_became_active":"2023-10-09T10:52:10.379844+0000","last_became_peered":"2023-10-09T10:52:10.379844+0000","last_unstale":"2023-10-09T10:52:20.538253+0000","last_undegraded":"2023-10-09T10:52:20.538253+0000","last_fullsized":"2023-10-09T10:52:20.538253+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:04:28.650194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"252'10691","reported_seq":13015,"reported_epoch":252,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:52:33.179184+0000","last_change":"2023-10-09T10:52:12.155121+0000","last_active":"2023-10-09T10:52:33.179184+0000","last_peered":"2023-10-09T10:52:33.179184+0000","last_clean":"2023-10-09T10:49:54.777504+0000","last_became_active":"2023-10-09T10:52:12.120937+0000","last_became_peered":"2023-10-09T10:52:12.120937+0000","last_unstale":"2023-10-09T10:52:33.179184+0000","last_undegraded":"2023-10-09T10:52:11.454769+0000","last_fullsized":"2023-10-09T10:52:33.179184+0000","mapping_epoch":245,"log_start":"226'5934","ondisk_log_start":"226'5934","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4757,"log_dups_size":0,"ondisk_log_size":4757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:08:33.874181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51838976,"num_objects":791,"num_object_clones":0,"num_object_copies":2373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":791,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10691,"num_write_kb":367424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1122,"num_bytes_recovered":23330816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,6],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4,6","objects":791}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":623,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.470036+0000","last_change":"2023-10-09T10:49:43.780951+0000","last_active":"2023-10-09T10:52:20.470036+0000","last_peered":"2023-10-09T10:52:20.470036+0000","last_clean":"2023-10-09T10:52:20.470036+0000","last_became_active":"2023-10-09T10:49:43.780564+0000","last_became_peered":"2023-10-09T10:49:43.780564+0000","last_unstale":"2023-10-09T10:52:20.470036+0000","last_undegraded":"2023-10-09T10:52:20.470036+0000","last_fullsized":"2023-10-09T10:52:20.470036+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":269,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473897+0000","last_change":"2023-10-09T10:52:10.378818+0000","last_active":"2023-10-09T10:52:20.473897+0000","last_peered":"2023-10-09T10:52:20.473897+0000","last_clean":"2023-10-09T10:52:20.473897+0000","last_became_active":"2023-10-09T10:52:10.378248+0000","last_became_peered":"2023-10-09T10:52:10.378248+0000","last_unstale":"2023-10-09T10:52:20.473897+0000","last_undegraded":"2023-10-09T10:52:20.473897+0000","last_fullsized":"2023-10-09T10:52:20.473897+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:27.613093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.538543+0000","last_change":"2023-10-09T10:52:10.376143+0000","last_active":"2023-10-09T10:52:20.538543+0000","last_peered":"2023-10-09T10:52:20.538543+0000","last_clean":"2023-10-09T10:52:20.538543+0000","last_became_active":"2023-10-09T10:52:10.375861+0000","last_became_peered":"2023-10-09T10:52:10.375861+0000","last_unstale":"2023-10-09T10:52:20.538543+0000","last_undegraded":"2023-10-09T10:52:20.538543+0000","last_fullsized":"2023-10-09T10:52:20.538543+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:24.689310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.542530+0000","last_change":"2023-10-09T10:52:10.390172+0000","last_active":"2023-10-09T10:52:20.542530+0000","last_peered":"2023-10-09T10:52:20.542530+0000","last_clean":"2023-10-09T10:52:20.542530+0000","last_became_active":"2023-10-09T10:52:10.389909+0000","last_became_peered":"2023-10-09T10:52:10.389909+0000","last_unstale":"2023-10-09T10:52:20.542530+0000","last_undegraded":"2023-10-09T10:52:20.542530+0000","last_fullsized":"2023-10-09T10:52:20.542530+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:38.723747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"252'10582","reported_seq":15290,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T10:52:33.212881+0000","last_change":"2023-10-09T10:52:20.507454+0000","last_active":"2023-10-09T10:52:33.212881+0000","last_peered":"2023-10-09T10:52:33.212881+0000","last_clean":"2023-10-09T10:52:33.212881+0000","last_became_active":"2023-10-09T10:52:20.476678+0000","last_became_peered":"2023-10-09T10:52:20.476678+0000","last_unstale":"2023-10-09T10:52:33.212881+0000","last_undegraded":"2023-10-09T10:52:33.212881+0000","last_fullsized":"2023-10-09T10:52:33.212881+0000","mapping_epoch":250,"log_start":"226'5820","ondisk_log_start":"226'5820","created":196,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4762,"log_dups_size":0,"ondisk_log_size":4762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:14.680309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48234496,"num_objects":736,"num_object_clones":0,"num_object_copies":2208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10582,"num_write_kb":362176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2311,"num_bytes_recovered":65536000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":207,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473833+0000","last_change":"2023-10-09T10:51:50.374067+0000","last_active":"2023-10-09T10:52:20.473833+0000","last_peered":"2023-10-09T10:52:20.473833+0000","last_clean":"2023-10-09T10:52:20.473833+0000","last_became_active":"2023-10-09T10:51:50.373877+0000","last_became_peered":"2023-10-09T10:51:50.373877+0000","last_unstale":"2023-10-09T10:52:20.473833+0000","last_undegraded":"2023-10-09T10:52:20.473833+0000","last_fullsized":"2023-10-09T10:52:20.473833+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:10.499049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":652,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.469885+0000","last_change":"2023-10-09T10:52:13.416041+0000","last_active":"2023-10-09T10:52:20.469885+0000","last_peered":"2023-10-09T10:52:20.469885+0000","last_clean":"2023-10-09T10:52:20.469885+0000","last_became_active":"2023-10-09T10:52:13.415750+0000","last_became_peered":"2023-10-09T10:52:13.415750+0000","last_unstale":"2023-10-09T10:52:20.469885+0000","last_undegraded":"2023-10-09T10:52:20.469885+0000","last_fullsized":"2023-10-09T10:52:20.469885+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:23:49.791297+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":182,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.472891+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:52:20.472891+0000","last_peered":"2023-10-09T10:52:20.472891+0000","last_clean":"2023-10-09T10:52:20.472891+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:52:20.472891+0000","last_undegraded":"2023-10-09T10:52:20.472891+0000","last_fullsized":"2023-10-09T10:52:20.472891+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":154,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.481105+0000","last_change":"2023-10-09T10:51:12.363829+0000","last_active":"2023-10-09T10:52:20.481105+0000","last_peered":"2023-10-09T10:52:20.481105+0000","last_clean":"2023-10-09T10:52:20.481105+0000","last_became_active":"2023-10-09T10:51:12.363534+0000","last_became_peered":"2023-10-09T10:51:12.363534+0000","last_unstale":"2023-10-09T10:52:20.481105+0000","last_undegraded":"2023-10-09T10:52:20.481105+0000","last_fullsized":"2023-10-09T10:52:20.481105+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:59.472228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"252'10650","reported_seq":17826,"reported_epoch":252,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-10-09T10:52:33.169948+0000","last_change":"2023-10-09T10:52:10.485355+0000","last_active":"2023-10-09T10:52:33.169948+0000","last_peered":"2023-10-09T10:52:33.169948+0000","last_clean":"2023-10-09T10:49:39.922158+0000","last_became_active":"2023-10-09T10:52:10.460053+0000","last_became_peered":"2023-10-09T10:52:10.460053+0000","last_unstale":"2023-10-09T10:52:33.169948+0000","last_undegraded":"2023-10-09T10:52:10.392014+0000","last_fullsized":"2023-10-09T10:52:10.365737+0000","mapping_epoch":244,"log_start":"226'5901","ondisk_log_start":"226'5901","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4749,"log_dups_size":0,"ondisk_log_size":4749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:51.219612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50855936,"num_objects":776,"num_object_clones":0,"num_object_copies":2328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":776,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10650,"num_write_kb":365632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2362,"num_bytes_recovered":154796032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,6],"avail_no_missing":["0","4","6","7"],"object_location_counts":[{"shards":"0,6","objects":776}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":601,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.538934+0000","last_change":"2023-10-09T10:52:13.412193+0000","last_active":"2023-10-09T10:52:20.538934+0000","last_peered":"2023-10-09T10:52:20.538934+0000","last_clean":"2023-10-09T10:52:20.538934+0000","last_became_active":"2023-10-09T10:52:13.411872+0000","last_became_peered":"2023-10-09T10:52:13.411872+0000","last_unstale":"2023-10-09T10:52:20.538934+0000","last_undegraded":"2023-10-09T10:52:20.538934+0000","last_fullsized":"2023-10-09T10:52:20.538934+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.538429+0000","last_change":"2023-10-09T10:52:10.375998+0000","last_active":"2023-10-09T10:52:20.538429+0000","last_peered":"2023-10-09T10:52:20.538429+0000","last_clean":"2023-10-09T10:52:20.538429+0000","last_became_active":"2023-10-09T10:52:10.375707+0000","last_became_peered":"2023-10-09T10:52:10.375707+0000","last_unstale":"2023-10-09T10:52:20.538429+0000","last_undegraded":"2023-10-09T10:52:20.538429+0000","last_fullsized":"2023-10-09T10:52:20.538429+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:13:25.396329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":132,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.469868+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:52:20.469868+0000","last_peered":"2023-10-09T10:52:20.469868+0000","last_clean":"2023-10-09T10:52:20.469868+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:52:20.469868+0000","last_undegraded":"2023-10-09T10:52:20.469868+0000","last_fullsized":"2023-10-09T10:52:20.469868+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":159,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T10:52:26.556095+0000","last_change":"2023-10-09T10:52:10.387740+0000","last_active":"2023-10-09T10:52:26.556095+0000","last_peered":"2023-10-09T10:52:26.556095+0000","last_clean":"2023-10-09T10:52:26.556095+0000","last_became_active":"2023-10-09T10:52:10.386971+0000","last_became_peered":"2023-10-09T10:52:10.386971+0000","last_unstale":"2023-10-09T10:52:26.556095+0000","last_undegraded":"2023-10-09T10:52:26.556095+0000","last_fullsized":"2023-10-09T10:52:26.556095+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:49.303931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"252'10479","reported_seq":12905,"reported_epoch":252,"state":"active+recovering+remapped","last_fresh":"2023-10-09T10:52:29.936109+0000","last_change":"2023-10-09T10:52:18.818685+0000","last_active":"2023-10-09T10:52:29.936109+0000","last_peered":"2023-10-09T10:52:29.936109+0000","last_clean":"2023-10-09T10:49:54.774195+0000","last_became_active":"2023-10-09T10:52:12.110110+0000","last_became_peered":"2023-10-09T10:52:12.110110+0000","last_unstale":"2023-10-09T10:52:29.936109+0000","last_undegraded":"2023-10-09T10:52:29.936109+0000","last_fullsized":"2023-10-09T10:52:29.936109+0000","mapping_epoch":245,"log_start":"225'5728","ondisk_log_start":"225'5728","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4751,"log_dups_size":0,"ondisk_log_size":4751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:18.464425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60358656,"num_objects":921,"num_object_clones":0,"num_object_copies":2763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10479,"num_write_kb":364800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1169,"num_bytes_recovered":25034752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5,6],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5,6","objects":921}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":611,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T10:52:26.556422+0000","last_change":"2023-10-09T10:51:12.367319+0000","last_active":"2023-10-09T10:52:26.556422+0000","last_peered":"2023-10-09T10:52:26.556422+0000","last_clean":"2023-10-09T10:52:26.556422+0000","last_became_active":"2023-10-09T10:51:12.366828+0000","last_became_peered":"2023-10-09T10:51:12.366828+0000","last_unstale":"2023-10-09T10:52:26.556422+0000","last_undegraded":"2023-10-09T10:52:26.556422+0000","last_fullsized":"2023-10-09T10:52:26.556422+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.475429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.475429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001591812,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":202,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.470536+0000","last_change":"2023-10-09T10:52:10.402104+0000","last_active":"2023-10-09T10:52:20.470536+0000","last_peered":"2023-10-09T10:52:20.470536+0000","last_clean":"2023-10-09T10:52:20.470536+0000","last_became_active":"2023-10-09T10:52:10.393660+0000","last_became_peered":"2023-10-09T10:52:10.393660+0000","last_unstale":"2023-10-09T10:52:20.470536+0000","last_undegraded":"2023-10-09T10:52:20.470536+0000","last_fullsized":"2023-10-09T10:52:20.470536+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:12.269565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.538026+0000","last_change":"2023-10-09T10:52:10.394775+0000","last_active":"2023-10-09T10:52:20.538026+0000","last_peered":"2023-10-09T10:52:20.538026+0000","last_clean":"2023-10-09T10:52:20.538026+0000","last_became_active":"2023-10-09T10:52:10.390932+0000","last_became_peered":"2023-10-09T10:52:10.390932+0000","last_unstale":"2023-10-09T10:52:20.538026+0000","last_undegraded":"2023-10-09T10:52:20.538026+0000","last_fullsized":"2023-10-09T10:52:20.538026+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:00.176413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":150,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.470294+0000","last_change":"2023-10-09T10:52:10.387653+0000","last_active":"2023-10-09T10:52:20.470294+0000","last_peered":"2023-10-09T10:52:20.470294+0000","last_clean":"2023-10-09T10:52:20.470294+0000","last_became_active":"2023-10-09T10:52:10.387357+0000","last_became_peered":"2023-10-09T10:52:10.387357+0000","last_unstale":"2023-10-09T10:52:20.470294+0000","last_undegraded":"2023-10-09T10:52:20.470294+0000","last_fullsized":"2023-10-09T10:52:20.470294+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:47.503866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"252'10473","reported_seq":10539,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T10:52:29.920374+0000","last_change":"2023-10-09T10:49:16.845103+0000","last_active":"2023-10-09T10:52:29.920374+0000","last_peered":"2023-10-09T10:52:29.920374+0000","last_clean":"2023-10-09T10:52:29.920374+0000","last_became_active":"2023-10-09T10:49:16.844812+0000","last_became_peered":"2023-10-09T10:49:16.844812+0000","last_unstale":"2023-10-09T10:52:29.920374+0000","last_undegraded":"2023-10-09T10:52:29.920374+0000","last_fullsized":"2023-10-09T10:52:29.920374+0000","mapping_epoch":196,"log_start":"225'5742","ondisk_log_start":"225'5742","created":196,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4731,"log_dups_size":0,"ondisk_log_size":4731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:17:27.590153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57606144,"num_objects":879,"num_object_clones":0,"num_object_copies":2637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10473,"num_write_kb":363264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":617,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.481065+0000","last_change":"2023-10-09T10:51:01.315582+0000","last_active":"2023-10-09T10:52:20.481065+0000","last_peered":"2023-10-09T10:52:20.481065+0000","last_clean":"2023-10-09T10:52:20.481065+0000","last_became_active":"2023-10-09T10:51:01.314820+0000","last_became_peered":"2023-10-09T10:51:01.314820+0000","last_unstale":"2023-10-09T10:52:20.481065+0000","last_undegraded":"2023-10-09T10:52:20.481065+0000","last_fullsized":"2023-10-09T10:52:20.481065+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.829081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.829081+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001277842,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":260,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473529+0000","last_change":"2023-10-09T10:49:40.959630+0000","last_active":"2023-10-09T10:52:20.473529+0000","last_peered":"2023-10-09T10:52:20.473529+0000","last_clean":"2023-10-09T10:52:20.473529+0000","last_became_active":"2023-10-09T10:49:40.945271+0000","last_became_peered":"2023-10-09T10:49:40.945271+0000","last_unstale":"2023-10-09T10:52:20.473529+0000","last_undegraded":"2023-10-09T10:52:20.473529+0000","last_fullsized":"2023-10-09T10:52:20.473529+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:37.426204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.540317+0000","last_change":"2023-10-09T10:52:10.412601+0000","last_active":"2023-10-09T10:52:20.540317+0000","last_peered":"2023-10-09T10:52:20.540317+0000","last_clean":"2023-10-09T10:52:20.540317+0000","last_became_active":"2023-10-09T10:52:10.412352+0000","last_became_peered":"2023-10-09T10:52:10.412352+0000","last_unstale":"2023-10-09T10:52:20.540317+0000","last_undegraded":"2023-10-09T10:52:20.540317+0000","last_fullsized":"2023-10-09T10:52:20.540317+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:48.842030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":263,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473006+0000","last_change":"2023-10-09T10:52:10.379786+0000","last_active":"2023-10-09T10:52:20.473006+0000","last_peered":"2023-10-09T10:52:20.473006+0000","last_clean":"2023-10-09T10:52:20.473006+0000","last_became_active":"2023-10-09T10:52:10.378532+0000","last_became_peered":"2023-10-09T10:52:10.378532+0000","last_unstale":"2023-10-09T10:52:20.473006+0000","last_undegraded":"2023-10-09T10:52:20.473006+0000","last_fullsized":"2023-10-09T10:52:20.473006+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:02.855358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"252'10595","reported_seq":19606,"reported_epoch":252,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:52:29.910290+0000","last_change":"2023-10-09T10:52:12.159145+0000","last_active":"2023-10-09T10:52:29.910290+0000","last_peered":"2023-10-09T10:52:29.910290+0000","last_clean":"2023-10-09T10:52:09.344937+0000","last_became_active":"2023-10-09T10:52:12.120743+0000","last_became_peered":"2023-10-09T10:52:12.120743+0000","last_unstale":"2023-10-09T10:52:29.910290+0000","last_undegraded":"2023-10-09T10:52:29.910290+0000","last_fullsized":"2023-10-09T10:52:29.910290+0000","mapping_epoch":245,"log_start":"225'5888","ondisk_log_start":"225'5888","created":196,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:49:15.811806+0000","last_clean_scrub_stamp":"2023-10-09T10:49:15.811806+0000","objects_scrubbed":0,"log_size":4707,"log_dups_size":0,"ondisk_log_size":4707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:35.390933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61538304,"num_objects":939,"num_object_clones":0,"num_object_copies":2817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":939,"num_objects_unfound":0,"num_objects_dirty":939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10595,"num_write_kb":369088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3221,"num_bytes_recovered":211025920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,6,0],"avail_no_missing":["3","0","6","7"],"object_location_counts":[{"shards":"0,3,6","objects":939}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":612,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473636+0000","last_change":"2023-10-09T10:49:43.787701+0000","last_active":"2023-10-09T10:52:20.473636+0000","last_peered":"2023-10-09T10:52:20.473636+0000","last_clean":"2023-10-09T10:52:20.473636+0000","last_became_active":"2023-10-09T10:49:43.787366+0000","last_became_peered":"2023-10-09T10:49:43.787366+0000","last_unstale":"2023-10-09T10:52:20.473636+0000","last_undegraded":"2023-10-09T10:52:20.473636+0000","last_fullsized":"2023-10-09T10:52:20.473636+0000","mapping_epoch":205,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":131,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473341+0000","last_change":"2023-10-09T10:52:10.377456+0000","last_active":"2023-10-09T10:52:20.473341+0000","last_peered":"2023-10-09T10:52:20.473341+0000","last_clean":"2023-10-09T10:52:20.473341+0000","last_became_active":"2023-10-09T10:52:10.377137+0000","last_became_peered":"2023-10-09T10:52:10.377137+0000","last_unstale":"2023-10-09T10:52:20.473341+0000","last_undegraded":"2023-10-09T10:52:20.473341+0000","last_fullsized":"2023-10-09T10:52:20.473341+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:56.801560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":153,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-09T10:52:26.556173+0000","last_change":"2023-10-09T10:52:10.421736+0000","last_active":"2023-10-09T10:52:26.556173+0000","last_peered":"2023-10-09T10:52:26.556173+0000","last_clean":"2023-10-09T10:52:26.556173+0000","last_became_active":"2023-10-09T10:52:10.387400+0000","last_became_peered":"2023-10-09T10:52:10.387400+0000","last_unstale":"2023-10-09T10:52:26.556173+0000","last_undegraded":"2023-10-09T10:52:26.556173+0000","last_fullsized":"2023-10-09T10:52:26.556173+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:23.931527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":193,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.480530+0000","last_change":"2023-10-09T10:51:50.371282+0000","last_active":"2023-10-09T10:52:20.480530+0000","last_peered":"2023-10-09T10:52:20.480530+0000","last_clean":"2023-10-09T10:52:20.480530+0000","last_became_active":"2023-10-09T10:51:50.370973+0000","last_became_peered":"2023-10-09T10:51:50.370973+0000","last_unstale":"2023-10-09T10:52:20.480530+0000","last_undegraded":"2023-10-09T10:52:20.480530+0000","last_fullsized":"2023-10-09T10:52:20.480530+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:15.603164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":612,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473346+0000","last_change":"2023-10-09T10:51:01.305767+0000","last_active":"2023-10-09T10:52:20.473346+0000","last_peered":"2023-10-09T10:52:20.473346+0000","last_clean":"2023-10-09T10:52:20.473346+0000","last_became_active":"2023-10-09T10:51:01.305503+0000","last_became_peered":"2023-10-09T10:51:01.305503+0000","last_unstale":"2023-10-09T10:52:20.473346+0000","last_undegraded":"2023-10-09T10:52:20.473346+0000","last_fullsized":"2023-10-09T10:52:20.473346+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":149,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.470343+0000","last_change":"2023-10-09T10:52:10.385987+0000","last_active":"2023-10-09T10:52:20.470343+0000","last_peered":"2023-10-09T10:52:20.470343+0000","last_clean":"2023-10-09T10:52:20.470343+0000","last_became_active":"2023-10-09T10:52:10.385656+0000","last_became_peered":"2023-10-09T10:52:10.385656+0000","last_unstale":"2023-10-09T10:52:20.470343+0000","last_undegraded":"2023-10-09T10:52:20.470343+0000","last_fullsized":"2023-10-09T10:52:20.470343+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:31:22.947732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":18,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.537944+0000","last_change":"2023-10-09T10:52:10.391133+0000","last_active":"2023-10-09T10:52:20.537944+0000","last_peered":"2023-10-09T10:52:20.537944+0000","last_clean":"2023-10-09T10:52:20.537944+0000","last_became_active":"2023-10-09T10:52:10.390586+0000","last_became_peered":"2023-10-09T10:52:10.390586+0000","last_unstale":"2023-10-09T10:52:20.537944+0000","last_undegraded":"2023-10-09T10:52:20.537944+0000","last_fullsized":"2023-10-09T10:52:20.537944+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:12.394654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":666,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.481216+0000","last_change":"2023-10-09T10:52:13.418412+0000","last_active":"2023-10-09T10:52:20.481216+0000","last_peered":"2023-10-09T10:52:20.481216+0000","last_clean":"2023-10-09T10:52:20.481216+0000","last_became_active":"2023-10-09T10:52:13.417667+0000","last_became_peered":"2023-10-09T10:52:13.417667+0000","last_unstale":"2023-10-09T10:52:20.481216+0000","last_undegraded":"2023-10-09T10:52:20.481216+0000","last_fullsized":"2023-10-09T10:52:20.481216+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":50,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.480991+0000","last_change":"2023-10-09T10:52:10.438843+0000","last_active":"2023-10-09T10:52:20.480991+0000","last_peered":"2023-10-09T10:52:20.480991+0000","last_clean":"2023-10-09T10:52:20.480991+0000","last_became_active":"2023-10-09T10:52:10.384006+0000","last_became_peered":"2023-10-09T10:52:10.384006+0000","last_unstale":"2023-10-09T10:52:20.480991+0000","last_undegraded":"2023-10-09T10:52:20.480991+0000","last_fullsized":"2023-10-09T10:52:20.480991+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:47.904749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":240,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473715+0000","last_change":"2023-10-09T10:52:10.382952+0000","last_active":"2023-10-09T10:52:20.473715+0000","last_peered":"2023-10-09T10:52:20.473715+0000","last_clean":"2023-10-09T10:52:20.473715+0000","last_became_active":"2023-10-09T10:52:10.382639+0000","last_became_peered":"2023-10-09T10:52:10.382639+0000","last_unstale":"2023-10-09T10:52:20.473715+0000","last_undegraded":"2023-10-09T10:52:20.473715+0000","last_fullsized":"2023-10-09T10:52:20.473715+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:47:26.158048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":202,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.475913+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:52:20.475913+0000","last_peered":"2023-10-09T10:52:20.475913+0000","last_clean":"2023-10-09T10:52:20.475913+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:52:20.475913+0000","last_undegraded":"2023-10-09T10:52:20.475913+0000","last_fullsized":"2023-10-09T10:52:20.475913+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":194,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.472969+0000","last_change":"2023-10-09T10:50:43.385150+0000","last_active":"2023-10-09T10:52:20.472969+0000","last_peered":"2023-10-09T10:52:20.472969+0000","last_clean":"2023-10-09T10:52:20.472969+0000","last_became_active":"2023-10-09T10:50:43.384788+0000","last_became_peered":"2023-10-09T10:50:43.384788+0000","last_unstale":"2023-10-09T10:52:20.472969+0000","last_undegraded":"2023-10-09T10:52:20.472969+0000","last_fullsized":"2023-10-09T10:52:20.472969+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:36.085910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":596,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473297+0000","last_change":"2023-10-09T10:51:01.296405+0000","last_active":"2023-10-09T10:52:20.473297+0000","last_peered":"2023-10-09T10:52:20.473297+0000","last_clean":"2023-10-09T10:52:20.473297+0000","last_became_active":"2023-10-09T10:51:01.295997+0000","last_became_peered":"2023-10-09T10:51:01.295997+0000","last_unstale":"2023-10-09T10:52:20.473297+0000","last_undegraded":"2023-10-09T10:52:20.473297+0000","last_fullsized":"2023-10-09T10:52:20.473297+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:41.637984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:41.637984+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001804833,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":256,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.474369+0000","last_change":"2023-10-09T10:52:10.379003+0000","last_active":"2023-10-09T10:52:20.474369+0000","last_peered":"2023-10-09T10:52:20.474369+0000","last_clean":"2023-10-09T10:52:20.474369+0000","last_became_active":"2023-10-09T10:52:10.378671+0000","last_became_peered":"2023-10-09T10:52:10.378671+0000","last_unstale":"2023-10-09T10:52:20.474369+0000","last_undegraded":"2023-10-09T10:52:20.474369+0000","last_fullsized":"2023-10-09T10:52:20.474369+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:07.193433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":268,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.473694+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:52:20.473694+0000","last_peered":"2023-10-09T10:52:20.473694+0000","last_clean":"2023-10-09T10:52:20.473694+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:52:20.473694+0000","last_undegraded":"2023-10-09T10:52:20.473694+0000","last_fullsized":"2023-10-09T10:52:20.473694+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":523,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.540009+0000","last_change":"2023-10-09T10:52:13.414592+0000","last_active":"2023-10-09T10:52:20.540009+0000","last_peered":"2023-10-09T10:52:20.540009+0000","last_clean":"2023-10-09T10:52:20.540009+0000","last_became_active":"2023-10-09T10:52:13.413923+0000","last_became_peered":"2023-10-09T10:52:13.413923+0000","last_unstale":"2023-10-09T10:52:20.540009+0000","last_undegraded":"2023-10-09T10:52:20.540009+0000","last_fullsized":"2023-10-09T10:52:20.540009+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":144,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.472281+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:52:20.472281+0000","last_peered":"2023-10-09T10:52:20.472281+0000","last_clean":"2023-10-09T10:52:20.472281+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:52:20.472281+0000","last_undegraded":"2023-10-09T10:52:20.472281+0000","last_fullsized":"2023-10-09T10:52:20.472281+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":236,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-09T10:52:20.479887+0000","last_change":"2023-10-09T10:52:10.392260+0000","last_active":"2023-10-09T10:52:20.479887+0000","last_peered":"2023-10-09T10:52:20.479887+0000","last_clean":"2023-10-09T10:52:20.479887+0000","last_became_active":"2023-10-09T10:52:10.391955+0000","last_became_peered":"2023-10-09T10:52:10.391955+0000","last_unstale":"2023-10-09T10:52:20.479887+0000","last_undegraded":"2023-10-09T10:52:20.479887+0000","last_fullsized":"2023-10-09T10:52:20.479887+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:51.768262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":865468440,"num_objects":13207,"num_object_clones":0,"num_object_copies":39621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4352,"num_objects_misplaced":2512,"num_objects_unfound":0,"num_objects_dirty":13207,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":169063,"num_write_kb":5832577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22545,"num_bytes_recovered":1203109888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4782895104,"data_stored":4782882888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76629,"ondisk_log_size":76629,"up":48,"acting":46,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":1264,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":712314,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":316,"num_read_kb":595,"num_write":155,"num_write_kb":1573,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1153902,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":219043332311,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":44,"seq":188978561242,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":923612,"kb_used_data":622228,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92821540,"statfs":{"total":95995035648,"available":95049256960,"internally_reserved":0,"allocated":637161472,"data_stored":636219709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:51:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":1.679,"15min":1.0620000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":5.6790000000000003,"5min":12.288,"15min":12.288},"last":5.0490000000000004},{"interface":"front","average":{"1min":1.615,"5min":1.7669999999999999,"15min":1.0980000000000001},"min":{"1min":0.26700000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":8.9299999999999997,"5min":12.369999999999999,"15min":12.369999999999999},"last":0.41299999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:51:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":1.806,"15min":1.155},"min":{"1min":0.30399999999999999,"5min":0.25700000000000001,"15min":0.245},"max":{"1min":8.4730000000000008,"5min":15.502000000000001,"15min":16.747},"last":0.60399999999999998},{"interface":"front","average":{"1min":2.3290000000000002,"5min":1.875,"15min":1.175},"min":{"1min":0.34399999999999997,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":8.0579999999999998,"5min":13.137,"15min":16.734000000000002},"last":0.56100000000000005}]},{"osd":2,"last update":"Mon Oct 9 10:52:29 2023","interfaces":[{"interface":"back","average":{"1min":2.5510000000000002,"5min":2.0960000000000001,"15min":1.196},"min":{"1min":0.34000000000000002,"5min":0.28899999999999998,"15min":0.26800000000000002},"max":{"1min":7.3849999999999998,"5min":13.208,"15min":13.208},"last":1.1559999999999999},{"interface":"front","average":{"1min":1.5649999999999999,"5min":2.2650000000000001,"15min":1.323},"min":{"1min":0.315,"5min":0.29999999999999999,"15min":0.28399999999999997},"max":{"1min":4.2720000000000002,"5min":10.186,"15min":12.571},"last":0.433}]},{"osd":3,"last update":"Mon Oct 9 10:51:49 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.9810000000000001,"15min":1.1379999999999999},"min":{"1min":0.29799999999999999,"5min":0.255,"15min":0.255},"max":{"1min":6.9660000000000002,"5min":12.391,"15min":12.391},"last":1.353},{"interface":"front","average":{"1min":2.3170000000000002,"5min":2.21,"15min":1.2749999999999999},"min":{"1min":0.30099999999999999,"5min":0.29999999999999999,"15min":0.28100000000000003},"max":{"1min":6.9299999999999997,"5min":13.179,"15min":13.179},"last":1.5880000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:52:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.0449999999999999,"15min":1.0509999999999999},"min":{"1min":0.27700000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":9.0600000000000005,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.9470000000000001,"15min":1.03},"min":{"1min":0.32500000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.2380000000000004,"5min":12.481999999999999,"15min":12.481999999999999},"last":0.377}]},{"osd":5,"last update":"Mon Oct 9 10:52:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2480000000000002,"5min":2.0209999999999999,"15min":1.087},"min":{"1min":0.34200000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.0069999999999997,"5min":10.497,"15min":10.497},"last":0.505},{"interface":"front","average":{"1min":1.958,"5min":2.0019999999999998,"15min":1.093},"min":{"1min":0.33200000000000002,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":9.6769999999999996,"5min":12.456,"15min":12.456},"last":0.45200000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:51:33 2023","interfaces":[{"interface":"back","average":{"1min":2.984,"5min":2.1309999999999998,"15min":1.103},"min":{"1min":0.35499999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.0340000000000007,"5min":23.358000000000001,"15min":23.358000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.5129999999999999,"5min":1.6819999999999999,"15min":0.99299999999999999},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.34000000000000002},"max":{"1min":7.0880000000000001,"5min":12.422000000000001,"15min":12.422000000000001},"last":0.55100000000000005}]}]},{"osd":1,"up_from":85,"seq":365072220281,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1689268,"kb_used_data":1463340,"kb_used_omap":9,"kb_used_meta":225910,"kb_avail":92055884,"statfs":{"total":95995035648,"available":94265225216,"internally_reserved":0,"allocated":1498460160,"data_stored":1497147755,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":231332841},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:51:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.9299999999999999,"15min":1.2030000000000001},"min":{"1min":0.32200000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":4.6130000000000004,"5min":18.027000000000001,"15min":18.027000000000001},"last":8.1750000000000007},{"interface":"front","average":{"1min":1.452,"5min":1.8480000000000001,"15min":1.171},"min":{"1min":0.25800000000000001,"5min":0.219,"15min":0.219},"max":{"1min":7.4749999999999996,"5min":10.08,"15min":14.465},"last":10.436999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.573,"15min":1.02},"min":{"1min":0.222,"5min":0.221,"15min":0.19700000000000001},"max":{"1min":3.774,"5min":15.356,"15min":15.356},"last":4.3600000000000003},{"interface":"front","average":{"1min":1.607,"5min":1.7470000000000001,"15min":1.2},"min":{"1min":0.26900000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":10.281000000000001,"5min":14.186999999999999,"15min":14.234},"last":0.42799999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:51:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9199999999999999,"5min":2.4700000000000002,"15min":1.3799999999999999},"min":{"1min":0.223,"5min":0.218,"15min":0.218},"max":{"1min":6.4180000000000001,"5min":18.274999999999999,"15min":18.274999999999999},"last":3.3690000000000002},{"interface":"front","average":{"1min":1.9770000000000001,"5min":2.113,"15min":1.1879999999999999},"min":{"1min":0.25,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.4290000000000003,"5min":15.468999999999999,"15min":15.468999999999999},"last":10.595000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:51:56 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.569,"15min":1.083},"min":{"1min":0.33600000000000002,"5min":0.255,"15min":0.255},"max":{"1min":3.8919999999999999,"5min":12.167999999999999,"15min":12.167999999999999},"last":0.622},{"interface":"front","average":{"1min":1.5740000000000001,"5min":2.1880000000000002,"15min":1.2769999999999999},"min":{"1min":0.35199999999999998,"5min":0.30299999999999999,"15min":0.30099999999999999},"max":{"1min":4.0469999999999997,"5min":14.228999999999999,"15min":14.228999999999999},"last":0.46400000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:51:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.655,"15min":1.0800000000000001},"min":{"1min":0.34100000000000003,"5min":0.28100000000000003,"15min":0.253},"max":{"1min":7.3739999999999997,"5min":9.4809999999999999,"15min":9.7720000000000002},"last":1.091},{"interface":"front","average":{"1min":1.5489999999999999,"5min":2.2389999999999999,"15min":1.3400000000000001},"min":{"1min":0.41499999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.992,"5min":17.085000000000001,"15min":17.085000000000001},"last":0.66000000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:51:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":1.9450000000000001,"15min":1.1830000000000001},"min":{"1min":0.38600000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.4210000000000003,"5min":10.425000000000001,"15min":10.425000000000001},"last":1.121},{"interface":"front","average":{"1min":1.226,"5min":1.835,"15min":1.2190000000000001},"min":{"1min":0.372,"5min":0.313,"15min":0.313},"max":{"1min":3.9399999999999999,"5min":12.231,"15min":12.231},"last":0.69499999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:51:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.788,"15min":1.198},"min":{"1min":0.48099999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.9180000000000001,"5min":12.249000000000001,"15min":12.249000000000001},"last":0.72199999999999998},{"interface":"front","average":{"1min":1.012,"5min":2.125,"15min":1.264},"min":{"1min":0.45100000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.0259999999999998,"5min":14.273999999999999,"15min":14.273999999999999},"last":3.3330000000000002}]}]},{"osd":0,"up_from":80,"seq":343597383815,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":814928,"kb_used_data":503880,"kb_used_omap":28,"kb_used_meta":311011,"kb_avail":92930224,"statfs":{"total":95995035648,"available":95160549376,"internally_reserved":0,"allocated":515973120,"data_stored":514574155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":318476112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,10,0,2,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:51:55 2023","interfaces":[{"interface":"back","average":{"1min":2.4089999999999998,"5min":2.302,"15min":1.2789999999999999},"min":{"1min":0.20599999999999999,"5min":0.191,"15min":0.191},"max":{"1min":9.125,"5min":14.311,"15min":25.475000000000001},"last":1.2250000000000001},{"interface":"front","average":{"1min":2.3860000000000001,"5min":2.2280000000000002,"15min":1.262},"min":{"1min":0.29399999999999998,"5min":0.188,"15min":0.188},"max":{"1min":9.359,"5min":12.276,"15min":20.832999999999998},"last":1.196}]},{"osd":2,"last update":"Mon Oct 9 10:52:29 2023","interfaces":[{"interface":"back","average":{"1min":2.3820000000000001,"5min":2.1080000000000001,"15min":1.2310000000000001},"min":{"1min":0.25800000000000001,"5min":0.23999999999999999,"15min":0.19700000000000001},"max":{"1min":9.1460000000000008,"5min":17.927,"15min":17.927},"last":0.68899999999999995},{"interface":"front","average":{"1min":2.1080000000000001,"5min":2.1459999999999999,"15min":1.341},"min":{"1min":0.29399999999999998,"5min":0.25700000000000001,"15min":0.20399999999999999},"max":{"1min":7.4790000000000001,"5min":14.111000000000001,"15min":14.391999999999999},"last":2.5649999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:51:55 2023","interfaces":[{"interface":"back","average":{"1min":3.0449999999999999,"5min":2.4620000000000002,"15min":1.548},"min":{"1min":0.22500000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":12.090999999999999,"5min":12.307,"15min":14.926},"last":1.1679999999999999},{"interface":"front","average":{"1min":3.093,"5min":2.601,"15min":1.466},"min":{"1min":0.25900000000000001,"5min":0.224,"15min":0.224},"max":{"1min":10.726000000000001,"5min":14.391999999999999,"15min":14.391999999999999},"last":1.252}]},{"osd":4,"last update":"Mon Oct 9 10:51:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2170000000000001,"5min":2.2280000000000002,"15min":1.266},"min":{"1min":0.39800000000000002,"5min":0.318,"15min":0.26200000000000001},"max":{"1min":7.1890000000000001,"5min":9.6129999999999995,"15min":9.6129999999999995},"last":2.665},{"interface":"front","average":{"1min":2.145,"5min":2.5790000000000002,"15min":1.4159999999999999},"min":{"1min":0.33400000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.9340000000000002,"5min":12.318,"15min":12.318},"last":1.7210000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:51:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":2.46,"15min":1.4019999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.96,"5min":12.157,"15min":12.157},"last":2.262},{"interface":"front","average":{"1min":2.2829999999999999,"5min":2.403,"15min":1.335},"min":{"1min":0.32100000000000001,"5min":0.317,"15min":0.29499999999999998},"max":{"1min":8.2119999999999997,"5min":14.446,"15min":14.446},"last":1.752}]},{"osd":6,"last update":"Mon Oct 9 10:51:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":2.6240000000000001,"15min":1.4490000000000001},"min":{"1min":0.36399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.9539999999999997,"5min":12.207000000000001,"15min":12.207000000000001},"last":1.327},{"interface":"front","average":{"1min":1.843,"5min":1.909,"15min":1.181},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":11.041,"5min":14.420999999999999,"15min":14.420999999999999},"last":1.2749999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:51:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":2.5219999999999998,"15min":1.387},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.901,"5min":14.48,"15min":14.48},"last":1.3},{"interface":"front","average":{"1min":2.948,"5min":2.5030000000000001,"15min":1.405},"min":{"1min":0.46800000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.2130000000000001,"5min":14.394,"15min":14.394},"last":2.7010000000000001}]}]},{"osd":2,"up_from":0,"seq":386547056755,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893229,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1246528,"kb_used_data":734072,"kb_used_omap":7,"kb_used_meta":512376,"kb_avail":92498624,"statfs":{"total":95995035648,"available":94718590976,"internally_reserved":0,"allocated":751689728,"data_stored":750302275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":524673640},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:51:52 2023","interfaces":[{"interface":"back","average":{"1min":1.2869999999999999,"5min":2.2589999999999999,"15min":1.1910000000000001},"min":{"1min":0.19500000000000001,"5min":0.192,"15min":0.183},"max":{"1min":3.9420000000000002,"5min":15.255000000000001,"15min":15.255000000000001},"last":0.69299999999999995},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.609,"15min":1.3169999999999999},"min":{"1min":0.193,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":10.874000000000001,"5min":14.798,"15min":14.798},"last":0.255}]},{"osd":1,"last update":"Mon Oct 9 10:51:52 2023","interfaces":[{"interface":"back","average":{"1min":2.6949999999999998,"5min":2.5840000000000001,"15min":1.444},"min":{"1min":0.23899999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":8.0540000000000003,"5min":11.757,"15min":14.707000000000001},"last":0.60099999999999998},{"interface":"front","average":{"1min":2.625,"5min":2.6579999999999999,"15min":1.3240000000000001},"min":{"1min":0.249,"5min":0.19400000000000001,"15min":0.193},"max":{"1min":10.613,"5min":15.443,"15min":15.443},"last":0.625}]},{"osd":2,"last update":"Mon Oct 9 10:51:52 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":2.2949999999999999,"15min":1.3300000000000001},"min":{"1min":0.24199999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.1820000000000004,"5min":11.795,"15min":13.853},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.6910000000000001,"5min":2.4710000000000001,"15min":1.3149999999999999},"min":{"1min":0.33600000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.9640000000000004,"5min":10.557,"15min":10.557},"last":0.40200000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:51:52 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":2.3519999999999999,"15min":1.3149999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":7.0499999999999998,"5min":20.388000000000002,"15min":20.388000000000002},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.5609999999999999,"5min":2.383,"15min":1.3200000000000001},"min":{"1min":0.29899999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.9349999999999996,"5min":19.085999999999999,"15min":19.085999999999999},"last":0.46400000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:51:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6950000000000001,"5min":2.5099999999999998,"15min":1.5549999999999999},"min":{"1min":0.29399999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.8559999999999999,"5min":10.619999999999999,"15min":51.509999999999998},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.6879999999999999,"5min":2.323,"15min":1.625},"min":{"1min":0.311,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":7,"5min":11.082000000000001,"15min":55.338999999999999},"last":0.67600000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:51:52 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":2.2130000000000001,"15min":1.264},"min":{"1min":0.35399999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.9000000000000004,"5min":12.361000000000001,"15min":12.361000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.784,"5min":2.399,"15min":1.4730000000000001},"min":{"1min":0.35399999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.8680000000000003,"5min":11.24,"15min":13.891999999999999},"last":0.71199999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:51:52 2023","interfaces":[{"interface":"back","average":{"1min":1.9770000000000001,"5min":2.472,"15min":1.6870000000000001},"min":{"1min":0.38500000000000001,"5min":0.31,"15min":0.31},"max":{"1min":7.6740000000000004,"5min":11.819000000000001,"15min":51.744},"last":0.74099999999999999},{"interface":"front","average":{"1min":2.4700000000000002,"5min":2.6739999999999999,"15min":1.6160000000000001},"min":{"1min":0.39900000000000002,"5min":0.33000000000000002,"15min":0.31900000000000001},"max":{"1min":11.379,"5min":15.212999999999999,"15min":53.399000000000001},"last":0.58199999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019105,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":881880,"kb_used_data":572304,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92863272,"statfs":{"total":95995035648,"available":95091990528,"internally_reserved":0,"allocated":586039296,"data_stored":585100872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:51:44 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.4750000000000001,"15min":0.97899999999999998},"min":{"1min":0.247,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":4.3470000000000004,"5min":9.4329999999999998,"15min":10.387},"last":0.23699999999999999},{"interface":"front","average":{"1min":2.4670000000000001,"5min":2.052,"15min":1.206},"min":{"1min":0.23300000000000001,"5min":0.217,"15min":0.217},"max":{"1min":12.58,"5min":12.58,"15min":12.58},"last":0.46999999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:51:46 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":1.609,"15min":1.0489999999999999},"min":{"1min":0.246,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":14.099,"5min":14.099,"15min":14.099},"last":0.27100000000000002},{"interface":"front","average":{"1min":2.0499999999999998,"5min":2.0310000000000001,"15min":1.2150000000000001},"min":{"1min":0.315,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":14.089,"5min":14.089,"15min":14.089},"last":0.42799999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:52:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":2.1139999999999999,"15min":1.262},"min":{"1min":0.29699999999999999,"5min":0.27500000000000002,"15min":0.253},"max":{"1min":12.204000000000001,"5min":12.204000000000001,"15min":12.204000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.79,"5min":2.3450000000000002,"15min":1.357},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26300000000000001},"max":{"1min":6.8970000000000002,"5min":20.545999999999999,"15min":20.545999999999999},"last":0.53000000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:51:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4299999999999999,"5min":1.9970000000000001,"15min":1.272},"min":{"1min":0.38400000000000001,"5min":0.29299999999999998,"15min":0.27400000000000002},"max":{"1min":6.9219999999999997,"5min":9.0269999999999992,"15min":9.0269999999999992},"last":0.54100000000000004},{"interface":"front","average":{"1min":2.6890000000000001,"5min":2.5750000000000002,"15min":1.389},"min":{"1min":0.41099999999999998,"5min":0.318,"15min":0.318},"max":{"1min":8.4760000000000009,"5min":9.6929999999999996,"15min":9.6929999999999996},"last":0.44500000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:51:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":2.431,"15min":1.2090000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.224},"max":{"1min":6.8410000000000002,"5min":24.149999999999999,"15min":24.149999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.702,"5min":2.2120000000000002,"15min":1.1299999999999999},"min":{"1min":0.28999999999999998,"5min":0.27100000000000002,"15min":0.252},"max":{"1min":6.0419999999999998,"5min":20.422000000000001,"15min":20.422000000000001},"last":0.32900000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:52:15 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":2.032,"15min":1.0840000000000001},"min":{"1min":0.28000000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.5279999999999996,"5min":14.897,"15min":14.897},"last":0.34200000000000003},{"interface":"front","average":{"1min":2.4020000000000001,"5min":2.577,"15min":1.286},"min":{"1min":0.35699999999999998,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":10.423999999999999,"5min":14.272,"15min":14.272},"last":0.51500000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":2.198,"15min":1.1870000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":10.385999999999999,"5min":10.385999999999999,"15min":10.385999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.6779999999999999,"5min":1.8280000000000001,"15min":1.0409999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":6.9710000000000001,"5min":7.423,"15min":7.423},"last":0.502}]}]},{"osd":5,"up_from":37,"seq":158913790174,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":632968,"kb_used_data":327488,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93112184,"statfs":{"total":95995035648,"available":95346876416,"internally_reserved":0,"allocated":335347712,"data_stored":334421637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:51:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0739999999999998,"5min":1.95,"15min":1.0900000000000001},"min":{"1min":0.24299999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":14.837,"5min":14.837,"15min":14.837},"last":2.1680000000000001},{"interface":"front","average":{"1min":2.129,"5min":2.012,"15min":1.1539999999999999},"min":{"1min":0.30199999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":14.863,"5min":14.863,"15min":14.863},"last":2.198}]},{"osd":1,"last update":"Mon Oct 9 10:51:57 2023","interfaces":[{"interface":"back","average":{"1min":2.7559999999999998,"5min":2.5859999999999999,"15min":1.3500000000000001},"min":{"1min":0.26000000000000001,"5min":0.251,"15min":0.251},"max":{"1min":6.585,"5min":12.999000000000001,"15min":12.999000000000001},"last":2.496},{"interface":"front","average":{"1min":1.5069999999999999,"5min":1.754,"15min":1.0920000000000001},"min":{"1min":0.28000000000000003,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":4.2199999999999998,"5min":12.109,"15min":12.109},"last":0.30599999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:52:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":2.4630000000000001,"15min":1.351},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.0590000000000002,"5min":11.393000000000001,"15min":11.393000000000001},"last":1.179},{"interface":"front","average":{"1min":3.3519999999999999,"5min":2.0539999999999998,"15min":1.177},"min":{"1min":0.314,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":15.087999999999999,"5min":15.087999999999999,"15min":15.087999999999999},"last":0.40999999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:51:52 2023","interfaces":[{"interface":"back","average":{"1min":5.5999999999999996,"5min":2.8410000000000002,"15min":1.6100000000000001},"min":{"1min":0.42999999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":63.508000000000003,"5min":63.508000000000003,"15min":63.508000000000003},"last":2.0489999999999999},{"interface":"front","average":{"1min":5.242,"5min":3.0720000000000001,"15min":1.629},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":62.418999999999997,"5min":62.418999999999997,"15min":62.418999999999997},"last":1.623}]},{"osd":4,"last update":"Mon Oct 9 10:51:57 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":2.7120000000000002,"15min":1.2410000000000001},"min":{"1min":0.247,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.71,"5min":123.241,"15min":123.241},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.5780000000000001,"5min":3.202,"15min":1.458},"min":{"1min":0.28000000000000003,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.8450000000000002,"5min":123.318,"15min":123.318},"last":0.311}]},{"osd":6,"last update":"Mon Oct 9 10:52:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1150000000000002,"5min":2.2570000000000001,"15min":1.143},"min":{"1min":0.30199999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":10.762,"5min":11.098000000000001,"15min":11.098000000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.4239999999999999,"5min":2.1560000000000001,"15min":1.1040000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.0980000000000008,"5min":20.256,"15min":20.256},"last":0.44400000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:51:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4129999999999998,"5min":1.9259999999999999,"15min":1.054},"min":{"1min":0.41199999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.2279999999999998,"5min":10.739000000000001,"15min":10.739000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.962,"5min":1.605,"15min":0.95299999999999996},"min":{"1min":0.40999999999999998,"5min":0.33200000000000002,"15min":0.31900000000000001},"max":{"1min":6.9640000000000004,"5min":14.154,"15min":14.154},"last":0.38800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":492703744,"data_stored":492699672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1495269376,"data_stored":1495269376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":718213120,"data_stored":718209048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":567672832,"data_stored":567672832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":330694656,"data_stored":330694656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":608768000,"data_stored":608763928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":569573376,"data_stored":569573376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:52:38.849 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_1 pgp_num 16 2023-10-09T10:52:38.850 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:38.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:38.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:38.918 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:38.956 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:39.009 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:39.040 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:39.040 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:39.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:39.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:39.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:39.255 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:39.321 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:39.656 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:39.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:39.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:39.762 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:52:39.828 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:40.192 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:52:40.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:40.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:52:40.297 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:52:40.347 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:40.381 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:52:40.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:40.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:52:40.490 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:40.525 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:40.558 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:40.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:40.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:40.663 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:40.688 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:40 smithi125 ceph-mon[195507]: pgmap v596: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 563 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 875 op/s; 2796/27078 objects degraded (10.326%); 1773/27078 objects misplaced (6.548%); 0 B/s, 168 objects/s recovering 2023-10-09T10:52:40.688 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:40 smithi125 ceph-mon[200197]: pgmap v596: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 563 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 875 op/s; 2796/27078 objects degraded (10.326%); 1773/27078 objects misplaced (6.548%); 0 B/s, 168 objects/s recovering 2023-10-09T10:52:40.705 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:40 smithi155 ceph-mon[184854]: pgmap v596: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 563 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 875 op/s; 2796/27078 objects degraded (10.326%); 1773/27078 objects misplaced (6.548%); 0 B/s, 168 objects/s recovering 2023-10-09T10:52:40.710 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:40.741 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:40.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:40.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:40.850 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:40.864 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:40.888 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:40.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:40.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:40.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:41.026 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:41.070 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:41.110 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:41.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:41.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:41.214 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:41.219 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:52:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:52:41] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-10-09T10:52:41.255 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:41.284 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:41.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:41.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:41.291 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:52:41.333 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:52:41.381 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:52:41.381 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:52:41.382 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:52:41.382 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:52:41.382 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:52:41.390 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:41.404 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:52:41.424 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:41.458 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:41.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:41.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:41.465 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:52:41.508 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:52:41.540 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:52:41.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:41 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2744297358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:41.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:41 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2744297358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:41.565 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:52:41.600 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:52:41.618 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:41.647 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:52:41.649 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:52:41.649 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:52:41.649 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:52:41.650 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:52:41.650 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:52:41.650 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:52:41.650 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:52:41.650 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:52:41.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:41 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2744297358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:42.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:42 smithi155 ceph-mon[184854]: pgmap v597: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 430 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 879 op/s; 2494/20697 objects degraded (12.050%); 1207/20697 objects misplaced (5.832%); 0 B/s, 171 objects/s recovering 2023-10-09T10:52:42.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:42 smithi125 ceph-mon[195507]: pgmap v597: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 430 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 879 op/s; 2494/20697 objects degraded (12.050%); 1207/20697 objects misplaced (5.832%); 0 B/s, 171 objects/s recovering 2023-10-09T10:52:42.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:42 smithi125 ceph-mon[200197]: pgmap v597: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 430 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 879 op/s; 2494/20697 objects degraded (12.050%); 1207/20697 objects misplaced (5.832%); 0 B/s, 171 objects/s recovering 2023-10-09T10:52:43.115 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:52:43.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:43.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:52:43.221 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:52:43.281 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:43.420 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:43 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3526897676' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:52:43.420 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:43 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 2494/20697 objects degraded (12.050%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:52:43.457 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:52:43.720 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:52:43.720 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:43 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3526897676' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:52:43.720 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:43 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 2494/20697 objects degraded (12.050%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:52:43.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:43 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3526897676' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:52:43.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:43 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 2494/20697 objects degraded (12.050%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:52:43.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:43.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:52:43.827 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:52:43.893 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:43.937 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:52:43.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:43.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:52:44.045 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:44.098 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:44.134 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:44.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:44.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:44.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:52:44.310 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:44.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:44 smithi125 ceph-mon[195507]: pgmap v598: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 303 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 873 op/s; 1140/14586 objects degraded (7.816%); 965/14586 objects misplaced (6.616%); 0 B/s, 185 objects/s recovering 2023-10-09T10:52:44.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:44 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:52:44.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:44 smithi125 ceph-mon[195507]: failed to encode map e253 with expected crc 2023-10-09T10:52:44.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:44 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3526897676' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:52:44.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:44 smithi125 ceph-mon[195507]: osdmap e253: 8 total, 8 up, 6 in 2023-10-09T10:52:44.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:44 smithi125 ceph-mon[195507]: failed to encode map e253 with expected crc 2023-10-09T10:52:44.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:44 smithi125 ceph-mon[195507]: failed to encode map e253 with expected crc 2023-10-09T10:52:44.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:44 smithi125 ceph-mon[200197]: pgmap v598: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 303 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 873 op/s; 1140/14586 objects degraded (7.816%); 965/14586 objects misplaced (6.616%); 0 B/s, 185 objects/s recovering 2023-10-09T10:52:44.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:44 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:52:44.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:44 smithi125 ceph-mon[200197]: failed to encode map e253 with expected crc 2023-10-09T10:52:44.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:44 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3526897676' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:52:44.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:44 smithi125 ceph-mon[200197]: osdmap e253: 8 total, 8 up, 6 in 2023-10-09T10:52:44.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:44 smithi125 ceph-mon[200197]: failed to encode map e253 with expected crc 2023-10-09T10:52:44.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:44 smithi125 ceph-mon[200197]: failed to encode map e253 with expected crc 2023-10-09T10:52:44.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:44 smithi155 ceph-mon[184854]: pgmap v598: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 303 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 873 op/s; 1140/14586 objects degraded (7.816%); 965/14586 objects misplaced (6.616%); 0 B/s, 185 objects/s recovering 2023-10-09T10:52:44.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:44 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:52:44.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:44 smithi155 ceph-mon[184854]: failed to encode map e253 with expected crc 2023-10-09T10:52:44.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:44 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3526897676' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:52:44.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:44 smithi155 ceph-mon[184854]: osdmap e253: 8 total, 8 up, 6 in 2023-10-09T10:52:44.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:44 smithi155 ceph-mon[184854]: failed to encode map e253 with expected crc 2023-10-09T10:52:44.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:44 smithi155 ceph-mon[184854]: failed to encode map e253 with expected crc 2023-10-09T10:52:46.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:45 smithi125 ceph-mon[200197]: osdmap e254: 8 total, 8 up, 6 in 2023-10-09T10:52:46.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:45 smithi125 ceph-mon[200197]: failed to encode map e254 with expected crc 2023-10-09T10:52:46.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:45 smithi125 ceph-mon[200197]: failed to encode map e254 with expected crc 2023-10-09T10:52:46.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:45 smithi125 ceph-mon[200197]: failed to encode map e254 with expected crc 2023-10-09T10:52:46.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:46.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:52:46.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:45 smithi125 ceph-mon[200197]: pgmap v601: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 203 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 993 op/s; 867/9807 objects degraded (8.841%); 686/9807 objects misplaced (6.995%); 0 B/s, 231 objects/s recovering 2023-10-09T10:52:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:45 smithi125 ceph-mon[195507]: osdmap e254: 8 total, 8 up, 6 in 2023-10-09T10:52:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:45 smithi125 ceph-mon[195507]: failed to encode map e254 with expected crc 2023-10-09T10:52:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:45 smithi125 ceph-mon[195507]: failed to encode map e254 with expected crc 2023-10-09T10:52:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:45 smithi125 ceph-mon[195507]: failed to encode map e254 with expected crc 2023-10-09T10:52:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:52:46.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:45 smithi125 ceph-mon[195507]: pgmap v601: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 203 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 993 op/s; 867/9807 objects degraded (8.841%); 686/9807 objects misplaced (6.995%); 0 B/s, 231 objects/s recovering 2023-10-09T10:52:46.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:45 smithi155 ceph-mon[184854]: osdmap e254: 8 total, 8 up, 6 in 2023-10-09T10:52:46.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:45 smithi155 ceph-mon[184854]: failed to encode map e254 with expected crc 2023-10-09T10:52:46.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:45 smithi155 ceph-mon[184854]: failed to encode map e254 with expected crc 2023-10-09T10:52:46.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:45 smithi155 ceph-mon[184854]: failed to encode map e254 with expected crc 2023-10-09T10:52:46.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:52:46.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:52:46.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:45 smithi155 ceph-mon[184854]: pgmap v601: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 203 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 993 op/s; 867/9807 objects degraded (8.841%); 686/9807 objects misplaced (6.995%); 0 B/s, 231 objects/s recovering 2023-10-09T10:52:46.846 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:52:46.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:46.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:52:46.992 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:47.045 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:47.082 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:47.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:47.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:47.131 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:46 smithi125 ceph-mon[195507]: failed to encode map e255 with expected crc 2023-10-09T10:52:47.131 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:46 smithi125 ceph-mon[195507]: osdmap e255: 8 total, 8 up, 6 in 2023-10-09T10:52:47.132 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:46 smithi125 ceph-mon[195507]: failed to encode map e255 with expected crc 2023-10-09T10:52:47.132 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:46 smithi125 ceph-mon[200197]: failed to encode map e255 with expected crc 2023-10-09T10:52:47.132 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:46 smithi125 ceph-mon[200197]: osdmap e255: 8 total, 8 up, 6 in 2023-10-09T10:52:47.132 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:46 smithi125 ceph-mon[200197]: failed to encode map e255 with expected crc 2023-10-09T10:52:47.188 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:47.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:46 smithi155 ceph-mon[184854]: failed to encode map e255 with expected crc 2023-10-09T10:52:47.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:46 smithi155 ceph-mon[184854]: osdmap e255: 8 total, 8 up, 6 in 2023-10-09T10:52:47.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:46 smithi155 ceph-mon[184854]: failed to encode map e255 with expected crc 2023-10-09T10:52:47.240 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:47.271 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:47.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:47.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:47.376 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:47.423 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:47.854 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Removed 91134 objects 2023-10-09T10:52:47.855 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Clean up completed and total clean up time :116.826 2023-10-09T10:52:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:47 smithi155 ceph-mon[184854]: pgmap v603: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 172 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 688 op/s; 867/8298 objects degraded (10.448%); 445/8298 objects misplaced (5.363%); 0 B/s, 157 objects/s recovering 2023-10-09T10:52:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:47 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 867/9807 objects degraded (8.841%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:52:48.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:47 smithi125 ceph-mon[195507]: pgmap v603: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 172 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 688 op/s; 867/8298 objects degraded (10.448%); 445/8298 objects misplaced (5.363%); 0 B/s, 157 objects/s recovering 2023-10-09T10:52:48.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:47 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 867/9807 objects degraded (8.841%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:52:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:47 smithi125 ceph-mon[200197]: pgmap v603: 119 pgs: 1 active+recovering+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 172 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 688 op/s; 867/8298 objects degraded (10.448%); 445/8298 objects misplaced (5.363%); 0 B/s, 157 objects/s recovering 2023-10-09T10:52:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:47 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 867/9807 objects degraded (8.841%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:52:48.903 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:52:49.532 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:49.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:49.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:49.636 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:49.663 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:49.682 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:49.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:49.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:49.786 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:49.812 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:49.830 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:49.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:49.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:49.934 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:52:49.961 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:49.979 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:52:49.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:49.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:52:50.082 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:52:50.111 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:50 smithi155 ceph-mon[184854]: pgmap v604: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 111 active+clean; 60 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 646 op/s; 130/2937 objects degraded (4.426%); 223/2937 objects misplaced (7.593%); 0 B/s, 85 objects/s recovering 2023-10-09T10:52:50.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:50 smithi125 ceph-mon[195507]: pgmap v604: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 111 active+clean; 60 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 646 op/s; 130/2937 objects degraded (4.426%); 223/2937 objects misplaced (7.593%); 0 B/s, 85 objects/s recovering 2023-10-09T10:52:50.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:50 smithi125 ceph-mon[200197]: pgmap v604: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 111 active+clean; 60 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 646 op/s; 130/2937 objects degraded (4.426%); 223/2937 objects misplaced (7.593%); 0 B/s, 85 objects/s recovering 2023-10-09T10:52:51.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:52:51.130 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:52:51.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:51.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:52:51.266 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:51.306 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:51.333 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:51.352 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:51.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:51.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:51.365 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:52:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:52:51] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2023-10-09T10:52:51.456 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:51.483 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:51.501 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:51.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:51.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:51.605 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:51.630 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:51 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T10:52:51.632 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:51.651 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:51.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:51.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:51.755 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:52:51.780 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:51 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T10:52:51.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:51 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T10:52:51.783 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:51.903 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:52:51.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:51.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:52:52.008 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:52.035 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:52.054 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:52.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:52.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:52.158 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:52:52.186 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:52.356 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:52:52.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:52.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:52:52.461 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:52:52.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:52 smithi125 ceph-mon[200197]: pgmap v605: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 701 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 0 B/s, 163 objects/s recovering 2023-10-09T10:52:52.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:52 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-10-09T10:52:52.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:52 smithi125 ceph-mon[200197]: osdmap e256: 8 total, 8 up, 6 in 2023-10-09T10:52:52.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:52 smithi125 ceph-mon[200197]: failed to encode map e256 with expected crc 2023-10-09T10:52:52.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:52 smithi125 ceph-mon[200197]: failed to encode map e256 with expected crc 2023-10-09T10:52:52.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:52 smithi125 ceph-mon[200197]: failed to encode map e256 with expected crc 2023-10-09T10:52:52.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:52 smithi125 ceph-mon[200197]: failed to encode map e256 with expected crc 2023-10-09T10:52:52.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:52 smithi125 ceph-mon[195507]: pgmap v605: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 701 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 0 B/s, 163 objects/s recovering 2023-10-09T10:52:52.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:52 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-10-09T10:52:52.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:52 smithi125 ceph-mon[195507]: osdmap e256: 8 total, 8 up, 6 in 2023-10-09T10:52:52.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:52 smithi125 ceph-mon[195507]: failed to encode map e256 with expected crc 2023-10-09T10:52:52.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:52 smithi125 ceph-mon[195507]: failed to encode map e256 with expected crc 2023-10-09T10:52:52.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:52 smithi125 ceph-mon[195507]: failed to encode map e256 with expected crc 2023-10-09T10:52:52.494 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:52 smithi125 ceph-mon[195507]: failed to encode map e256 with expected crc 2023-10-09T10:52:52.494 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:52.564 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:52:52.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:52.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:52:52.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:52.693 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:52 smithi155 ceph-mon[184854]: pgmap v605: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 111 active+clean; 701 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 0 B/s, 163 objects/s recovering 2023-10-09T10:52:52.694 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:52 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-10-09T10:52:52.694 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:52 smithi155 ceph-mon[184854]: osdmap e256: 8 total, 8 up, 6 in 2023-10-09T10:52:52.694 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:52 smithi155 ceph-mon[184854]: failed to encode map e256 with expected crc 2023-10-09T10:52:52.694 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:52 smithi155 ceph-mon[184854]: failed to encode map e256 with expected crc 2023-10-09T10:52:52.694 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:52 smithi155 ceph-mon[184854]: failed to encode map e256 with expected crc 2023-10-09T10:52:52.694 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:52 smithi155 ceph-mon[184854]: failed to encode map e256 with expected crc 2023-10-09T10:52:52.695 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:52.697 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:52.716 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:52.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:52.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:52.819 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:52.846 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:52.865 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:52.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:52.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:52.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:52.997 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:53.114 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:52:53.267 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:53.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:53.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:53.371 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:53.401 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:53.408 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 7 pgp_num to 16 2023-10-09T10:52:53.423 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:53.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:53.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:53.526 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:53.567 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:53 smithi155 ceph-mon[184854]: osdmap e257: 8 total, 8 up, 6 in 2023-10-09T10:52:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:53 smithi155 ceph-mon[184854]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:53 smithi155 ceph-mon[184854]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:53 smithi155 ceph-mon[184854]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:53 smithi155 ceph-mon[184854]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:53 smithi155 ceph-mon[184854]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:53 smithi155 ceph-mon[184854]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:53 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1220329924' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:52:53.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:53 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:52:53.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:53 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-10-09T10:52:53.730 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[195507]: osdmap e257: 8 total, 8 up, 6 in 2023-10-09T10:52:53.730 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[195507]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.730 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[195507]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.730 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[195507]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.730 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[195507]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.730 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[195507]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.731 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[195507]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.731 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1220329924' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:52:53.731 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:52:53.731 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-10-09T10:52:53.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[200197]: osdmap e257: 8 total, 8 up, 6 in 2023-10-09T10:52:53.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[200197]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[200197]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[200197]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[200197]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[200197]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.732 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[200197]: failed to encode map e257 with expected crc 2023-10-09T10:52:53.732 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1220329924' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:52:53.732 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T10:52:53.732 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:53 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-10-09T10:52:53.737 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:53.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:53.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:53.842 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:52:53.867 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-10-09T10:52:53.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-10-09T10:52:53.870 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:53.885 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:52:53.885 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:52:53.886 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:52:53.886 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:52:53.886 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.886 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42384 6.10 6:0c7bd3cf:::benchmark_data_smithi125_244951_object42383:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:52:53.886 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:52:53.886 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 478.67092885099999, 2023-10-09T10:52:53.886 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0577403670000001, 2023-10-09T10:52:53.886 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:53.886 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:53.886 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:53.887 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:52:53.887 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:53.887 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42384 2023-10-09T10:52:53.887 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.887 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:53.887 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.887 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:53.887 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:52:53.887 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.887 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.887 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.887 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:53.888 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:52:53.888 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.888 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.888 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.888 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:53.888 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117185+0000", 2023-10-09T10:52:53.888 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:53.888 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.888 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.888 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:53.888 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117225+0000", 2023-10-09T10:52:53.888 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0364000000000002e-05 2023-10-09T10:52:53.889 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.889 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.889 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:53.889 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117240+0000", 2023-10-09T10:52:53.889 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4425e-05 2023-10-09T10:52:53.889 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.889 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.889 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:53.889 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117248+0000", 2023-10-09T10:52:53.889 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4999999999999999e-06 2023-10-09T10:52:53.889 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.889 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.890 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.890 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117301+0000", 2023-10-09T10:52:53.890 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2685999999999997e-05 2023-10-09T10:52:53.890 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.890 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.890 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.890 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117312+0000", 2023-10-09T10:52:53.890 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1273e-05 2023-10-09T10:52:53.890 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.890 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.890 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.890 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103469+0000", 2023-10-09T10:52:53.891 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98615701300000003 2023-10-09T10:52:53.891 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.891 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.891 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.891 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103483+0000", 2023-10-09T10:52:53.891 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3207000000000001e-05 2023-10-09T10:52:53.891 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.891 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.891 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.891 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.148764+0000", 2023-10-09T10:52:53.891 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0452812789999999 2023-10-09T10:52:53.891 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.892 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.892 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:53.892 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.148882+0000", 2023-10-09T10:52:53.892 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118095 2023-10-09T10:52:53.892 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.892 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.892 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:52:53.892 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.149057+0000", 2023-10-09T10:52:53.892 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017516400000000001 2023-10-09T10:52:53.892 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.892 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.892 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:53.893 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174105+0000", 2023-10-09T10:52:53.893 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.025048237000000001 2023-10-09T10:52:53.893 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.893 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.893 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.893 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174158+0000", 2023-10-09T10:52:53.893 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2330999999999998e-05 2023-10-09T10:52:53.893 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.893 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.893 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.893 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174325+0000", 2023-10-09T10:52:53.893 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016690100000000001 2023-10-09T10:52:53.893 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.894 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.894 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:52:53.894 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174367+0000", 2023-10-09T10:52:53.894 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2759000000000002e-05 2023-10-09T10:52:53.894 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.894 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.894 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:53.894 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174927+0000", 2023-10-09T10:52:53.894 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00055942800000000005 2023-10-09T10:52:53.894 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.894 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:53.894 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.895 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.895 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.895 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42398 6.10 6:0867fefc:::benchmark_data_smithi125_244951_object42397:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:52:53.895 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:52:53.895 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 478.66015008199997, 2023-10-09T10:52:53.895 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0510695769999998, 2023-10-09T10:52:53.895 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:53.895 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:53.895 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:53.895 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:52:53.895 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:53.895 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42398 2023-10-09T10:52:53.895 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.896 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:53.896 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.896 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:53.896 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:52:53.896 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.896 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.896 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.896 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:53.896 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:52:53.896 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.896 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.896 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.897 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:53.897 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127964+0000", 2023-10-09T10:52:53.897 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:53.897 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.897 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.897 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:53.897 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127989+0000", 2023-10-09T10:52:53.897 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4811e-05 2023-10-09T10:52:53.897 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.897 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.897 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:53.897 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128005+0000", 2023-10-09T10:52:53.897 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6436999999999999e-05 2023-10-09T10:52:53.898 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.898 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.898 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:53.898 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128014+0000", 2023-10-09T10:52:53.898 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5990000000000002e-06 2023-10-09T10:52:53.898 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.898 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.898 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.898 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128072+0000", 2023-10-09T10:52:53.898 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7980999999999999e-05 2023-10-09T10:52:53.898 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.898 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.899 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.899 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128083+0000", 2023-10-09T10:52:53.899 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1051e-05 2023-10-09T10:52:53.899 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.899 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.899 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.899 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103714+0000", 2023-10-09T10:52:53.899 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97563124099999998 2023-10-09T10:52:53.899 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.899 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.899 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.899 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103725+0000", 2023-10-09T10:52:53.899 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1121000000000001e-05 2023-10-09T10:52:53.900 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.900 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.900 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.900 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174971+0000", 2023-10-09T10:52:53.900 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0712457310000001 2023-10-09T10:52:53.900 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.900 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.900 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:53.900 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.175149+0000", 2023-10-09T10:52:53.900 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000178584 2023-10-09T10:52:53.900 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.900 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.900 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:52:53.901 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.175358+0000", 2023-10-09T10:52:53.901 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000208848 2023-10-09T10:52:53.901 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.901 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.901 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.901 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178406+0000", 2023-10-09T10:52:53.901 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030478049999999998 2023-10-09T10:52:53.901 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.901 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.901 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:53.901 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178418+0000", 2023-10-09T10:52:53.901 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1499e-05 2023-10-09T10:52:53.902 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.902 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.902 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.902 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178466+0000", 2023-10-09T10:52:53.902 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8715999999999999e-05 2023-10-09T10:52:53.902 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.902 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.902 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:52:53.902 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178522+0000", 2023-10-09T10:52:53.902 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5952999999999998e-05 2023-10-09T10:52:53.902 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.902 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.903 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:53.903 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179035+0000", 2023-10-09T10:52:53.903 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051254299999999996 2023-10-09T10:52:53.903 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.903 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:53.903 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.903 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.903 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.903 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42518 6.10 6:0fbcc8bc:::benchmark_data_smithi125_244951_object42517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:52:53.903 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:52:53.903 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 478.593752757, 2023-10-09T10:52:53.903 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9852423510000001, 2023-10-09T10:52:53.904 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:53.904 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:53.904 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:53.904 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:52:53.904 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:53.904 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42518 2023-10-09T10:52:53.904 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.904 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:53.904 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.904 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:53.904 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:52:53.904 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.904 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.905 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.905 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:53.905 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:52:53.905 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.905 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.905 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.905 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:53.905 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194361+0000", 2023-10-09T10:52:53.905 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:53.905 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.905 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.905 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:53.906 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194385+0000", 2023-10-09T10:52:53.906 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4017000000000001e-05 2023-10-09T10:52:53.906 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.906 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.906 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:53.906 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194401+0000", 2023-10-09T10:52:53.906 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5841e-05 2023-10-09T10:52:53.906 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.906 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.906 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:53.906 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194408+0000", 2023-10-09T10:52:53.906 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0450000000000003e-06 2023-10-09T10:52:53.907 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.907 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.907 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.907 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194460+0000", 2023-10-09T10:52:53.907 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2015000000000003e-05 2023-10-09T10:52:53.907 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.907 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.907 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.907 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194474+0000", 2023-10-09T10:52:53.907 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4486e-05 2023-10-09T10:52:53.907 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.907 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.907 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.908 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103750+0000", 2023-10-09T10:52:53.908 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.90927595699999997 2023-10-09T10:52:53.908 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.908 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.908 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.908 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103758+0000", 2023-10-09T10:52:53.908 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9000000000000006e-06 2023-10-09T10:52:53.908 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.908 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.908 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.908 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176100+0000", 2023-10-09T10:52:53.908 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.07234187 2023-10-09T10:52:53.908 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.909 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.909 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:53.909 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176198+0000", 2023-10-09T10:52:53.909 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7408e-05 2023-10-09T10:52:53.909 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.909 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.909 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:52:53.909 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176317+0000", 2023-10-09T10:52:53.909 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011998900000000001 2023-10-09T10:52:53.909 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.909 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.909 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:53.909 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178425+0000", 2023-10-09T10:52:53.910 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021072 2023-10-09T10:52:53.910 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.910 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.910 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.910 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179085+0000", 2023-10-09T10:52:53.910 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00066011800000000001 2023-10-09T10:52:53.910 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.910 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.910 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.910 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179122+0000", 2023-10-09T10:52:53.910 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6831e-05 2023-10-09T10:52:53.910 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.911 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.911 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:52:53.911 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179155+0000", 2023-10-09T10:52:53.911 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3080000000000002e-05 2023-10-09T10:52:53.911 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.911 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.911 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:53.911 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179605+0000", 2023-10-09T10:52:53.911 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000450121 2023-10-09T10:52:53.911 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.911 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:53.911 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.911 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.912 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.912 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42546 6.10 6:0889293a:::benchmark_data_smithi125_244951_object42545:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:52:53.912 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:52:53.912 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 478.57932332399997, 2023-10-09T10:52:53.912 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.971488017, 2023-10-09T10:52:53.912 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:53.912 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:53.912 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:53.912 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:52:53.912 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:53.912 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42546 2023-10-09T10:52:53.912 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.912 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:53.913 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.913 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:53.913 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:52:53.913 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.913 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.913 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.913 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:53.913 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:52:53.913 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.913 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.913 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.913 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:53.914 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208790+0000", 2023-10-09T10:52:53.914 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:52:53.914 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.914 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.914 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:53.914 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208815+0000", 2023-10-09T10:52:53.914 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5103e-05 2023-10-09T10:52:53.914 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.914 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.914 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:53.914 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208831+0000", 2023-10-09T10:52:53.915 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5235000000000001e-05 2023-10-09T10:52:53.915 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.915 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.915 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:53.915 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208838+0000", 2023-10-09T10:52:53.915 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6510000000000001e-06 2023-10-09T10:52:53.915 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.915 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.915 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.915 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208895+0000", 2023-10-09T10:52:53.915 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6817999999999997e-05 2023-10-09T10:52:53.915 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.916 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.916 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.916 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208906+0000", 2023-10-09T10:52:53.916 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0550000000000001e-05 2023-10-09T10:52:53.916 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.916 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.916 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.916 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103780+0000", 2023-10-09T10:52:53.916 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.89487466699999996 2023-10-09T10:52:53.916 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.916 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.916 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.917 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103788+0000", 2023-10-09T10:52:53.917 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0970000000000006e-06 2023-10-09T10:52:53.917 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.917 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.917 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.917 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176855+0000", 2023-10-09T10:52:53.917 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0730664299999999 2023-10-09T10:52:53.917 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.917 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.917 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:53.917 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176993+0000", 2023-10-09T10:52:53.917 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013810999999999999 2023-10-09T10:52:53.918 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.918 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.918 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:52:53.918 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.177166+0000", 2023-10-09T10:52:53.918 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017289900000000001 2023-10-09T10:52:53.918 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.918 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.918 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:53.918 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179093+0000", 2023-10-09T10:52:53.918 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019276969999999999 2023-10-09T10:52:53.918 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.918 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.919 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.919 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179674+0000", 2023-10-09T10:52:53.919 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00058078000000000003 2023-10-09T10:52:53.919 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.919 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.919 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.919 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179730+0000", 2023-10-09T10:52:53.919 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5418999999999999e-05 2023-10-09T10:52:53.919 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.919 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.919 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:52:53.919 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179788+0000", 2023-10-09T10:52:53.920 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8510000000000001e-05 2023-10-09T10:52:53.920 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.920 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.920 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:53.920 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180280+0000", 2023-10-09T10:52:53.920 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00049179400000000004 2023-10-09T10:52:53.920 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.920 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:53.920 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.920 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.920 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.920 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42570 6.10 6:0f6ea5a0:::benchmark_data_smithi125_244951_object42569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:52:53.921 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:52:53.921 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 478.56523883099999, 2023-10-09T10:52:53.921 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.969653487, 2023-10-09T10:52:53.921 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:53.921 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:53.921 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:53.921 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:52:53.921 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:53.921 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42570 2023-10-09T10:52:53.921 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.921 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:53.921 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.921 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:53.922 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:52:53.922 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.922 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.922 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.922 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:53.922 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:52:53.922 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.922 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.922 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.922 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:53.922 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222875+0000", 2023-10-09T10:52:53.922 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:53.923 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.923 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.923 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:53.923 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222897+0000", 2023-10-09T10:52:53.923 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1678000000000001e-05 2023-10-09T10:52:53.923 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.923 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.923 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:53.923 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222906+0000", 2023-10-09T10:52:53.923 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4190000000000008e-06 2023-10-09T10:52:53.923 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.923 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.924 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:53.924 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222913+0000", 2023-10-09T10:52:53.924 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2659999999999998e-06 2023-10-09T10:52:53.924 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.924 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.924 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.924 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222958+0000", 2023-10-09T10:52:53.924 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5979000000000001e-05 2023-10-09T10:52:53.924 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.924 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.924 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.924 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222965+0000", 2023-10-09T10:52:53.925 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0099999999999998e-06 2023-10-09T10:52:53.925 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.925 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.925 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.925 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103823+0000", 2023-10-09T10:52:53.925 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.880857891 2023-10-09T10:52:53.925 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.925 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.925 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.925 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103833+0000", 2023-10-09T10:52:53.925 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8910000000000007e-06 2023-10-09T10:52:53.925 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.926 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.926 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.926 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180446+0000", 2023-10-09T10:52:53.926 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0766125440000001 2023-10-09T10:52:53.926 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.926 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.926 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:53.926 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180594+0000", 2023-10-09T10:52:53.926 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014846600000000001 2023-10-09T10:52:53.926 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.926 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.926 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:52:53.927 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180775+0000", 2023-10-09T10:52:53.927 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018071999999999999 2023-10-09T10:52:53.927 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.927 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.927 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:53.927 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183638+0000", 2023-10-09T10:52:53.927 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028626440000000001 2023-10-09T10:52:53.927 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.927 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.927 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.927 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183677+0000", 2023-10-09T10:52:53.927 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9545000000000002e-05 2023-10-09T10:52:53.928 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.928 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.928 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.928 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191583+0000", 2023-10-09T10:52:53.928 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0079059569999999999 2023-10-09T10:52:53.928 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.928 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.928 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:52:53.928 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191643+0000", 2023-10-09T10:52:53.928 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9590999999999999e-05 2023-10-09T10:52:53.928 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.928 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.929 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:53.929 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192530+0000", 2023-10-09T10:52:53.929 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00088716399999999996 2023-10-09T10:52:53.929 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.929 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:53.929 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.929 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.929 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.929 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42586 6.10 6:090d6b8d:::benchmark_data_smithi125_244951_object42585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:52:53.929 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:52:53.929 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 478.55864429000002, 2023-10-09T10:52:53.930 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.963875493, 2023-10-09T10:52:53.930 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:53.930 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:53.930 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:53.930 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:52:53.930 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:53.930 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42586 2023-10-09T10:52:53.930 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.930 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:53.930 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.930 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:53.930 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:52:53.931 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.931 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.931 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.931 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:53.931 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:52:53.931 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.931 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.931 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.931 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:53.931 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229470+0000", 2023-10-09T10:52:53.931 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:52:53.931 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.932 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.932 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:53.932 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229492+0000", 2023-10-09T10:52:53.932 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1716999999999999e-05 2023-10-09T10:52:53.932 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.932 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.932 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:53.932 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229501+0000", 2023-10-09T10:52:53.932 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0929999999999992e-06 2023-10-09T10:52:53.932 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.932 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.932 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:53.933 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229507+0000", 2023-10-09T10:52:53.933 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6760000000000001e-06 2023-10-09T10:52:53.933 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.933 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.933 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.933 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229565+0000", 2023-10-09T10:52:53.933 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7450000000000001e-05 2023-10-09T10:52:53.933 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.933 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.933 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.933 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229572+0000", 2023-10-09T10:52:53.933 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0820000000000001e-06 2023-10-09T10:52:53.934 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.934 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.934 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.934 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103865+0000", 2023-10-09T10:52:53.934 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.87429282200000003 2023-10-09T10:52:53.934 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.934 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.934 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.934 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103873+0000", 2023-10-09T10:52:53.934 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2090000000000005e-06 2023-10-09T10:52:53.934 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.934 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.935 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.935 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181656+0000", 2023-10-09T10:52:53.935 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.077783462 2023-10-09T10:52:53.935 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.935 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.935 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:53.935 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181791+0000", 2023-10-09T10:52:53.935 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000134351 2023-10-09T10:52:53.935 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.935 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.935 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:52:53.935 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181948+0000", 2023-10-09T10:52:53.935 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015678600000000001 2023-10-09T10:52:53.936 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.936 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.936 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:53.936 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183685+0000", 2023-10-09T10:52:53.936 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00173705 2023-10-09T10:52:53.936 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.936 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.936 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.936 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183719+0000", 2023-10-09T10:52:53.936 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4301999999999998e-05 2023-10-09T10:52:53.936 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.936 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.936 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.937 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192640+0000", 2023-10-09T10:52:53.937 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0089210899999999996 2023-10-09T10:52:53.937 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.937 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.937 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:52:53.937 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192696+0000", 2023-10-09T10:52:53.937 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5733000000000003e-05 2023-10-09T10:52:53.937 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.937 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.937 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:53.937 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193346+0000", 2023-10-09T10:52:53.937 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00065071200000000001 2023-10-09T10:52:53.938 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.938 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:53.938 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.938 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.938 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.938 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42606 6.10 6:0826a955:::benchmark_data_smithi125_244951_object42605:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:52:53.938 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:52:53.938 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 478.549884695, 2023-10-09T10:52:53.938 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.955680458, 2023-10-09T10:52:53.938 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:53.938 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:53.938 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:53.939 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:52:53.939 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:53.939 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42606 2023-10-09T10:52:53.939 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.939 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:53.939 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.939 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:53.939 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:52:53.939 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.939 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.939 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.939 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:53.940 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:52:53.940 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.940 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.940 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.940 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:53.940 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238230+0000", 2023-10-09T10:52:53.940 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:52:53.940 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.940 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.940 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:53.940 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238257+0000", 2023-10-09T10:52:53.940 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7551000000000001e-05 2023-10-09T10:52:53.940 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.941 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.941 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:53.941 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238267+0000", 2023-10-09T10:52:53.941 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5820000000000008e-06 2023-10-09T10:52:53.941 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.941 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.941 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:53.941 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238273+0000", 2023-10-09T10:52:53.941 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5490000000000003e-06 2023-10-09T10:52:53.941 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.941 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.941 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.942 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238322+0000", 2023-10-09T10:52:53.942 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8467999999999997e-05 2023-10-09T10:52:53.942 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.942 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.942 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.942 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238332+0000", 2023-10-09T10:52:53.942 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9000000000000001e-06 2023-10-09T10:52:53.942 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.942 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.942 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.942 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103898+0000", 2023-10-09T10:52:53.942 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.86556675699999996 2023-10-09T10:52:53.943 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.943 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.943 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.943 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103906+0000", 2023-10-09T10:52:53.943 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7710000000000006e-06 2023-10-09T10:52:53.943 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.943 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.943 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.943 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.182699+0000", 2023-10-09T10:52:53.943 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0787928490000001 2023-10-09T10:52:53.943 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.944 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.944 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:53.944 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.182847+0000", 2023-10-09T10:52:53.944 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014758600000000001 2023-10-09T10:52:53.944 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.944 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.944 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:52:53.944 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183016+0000", 2023-10-09T10:52:53.944 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000169326 2023-10-09T10:52:53.944 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.944 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.944 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:53.945 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186922+0000", 2023-10-09T10:52:53.945 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039060269999999998 2023-10-09T10:52:53.945 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.945 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.945 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.945 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186980+0000", 2023-10-09T10:52:53.945 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8313999999999998e-05 2023-10-09T10:52:53.945 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.945 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.945 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.945 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193472+0000", 2023-10-09T10:52:53.945 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0064917020000000002 2023-10-09T10:52:53.945 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.946 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.946 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:52:53.946 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193522+0000", 2023-10-09T10:52:53.946 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9672999999999997e-05 2023-10-09T10:52:53.946 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.946 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.946 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:53.946 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193911+0000", 2023-10-09T10:52:53.946 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00038944099999999997 2023-10-09T10:52:53.946 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.946 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:53.946 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.947 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.947 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.947 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42608 6.10 6:0c1c992a:::benchmark_data_smithi125_244951_object42607:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:52:53.947 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:52:53.947 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 478.54679299499998, 2023-10-09T10:52:53.947 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9589951289999998, 2023-10-09T10:52:53.947 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:53.947 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:53.947 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:53.947 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:52:53.947 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:53.948 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42608 2023-10-09T10:52:53.948 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.948 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:53.948 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.948 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:53.948 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:52:53.948 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.948 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.948 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.948 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:53.948 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:52:53.948 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.948 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.949 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.949 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:53.949 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241321+0000", 2023-10-09T10:52:53.949 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:52:53.949 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.949 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.949 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:53.949 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241343+0000", 2023-10-09T10:52:53.949 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1987000000000001e-05 2023-10-09T10:52:53.949 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.949 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.950 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:53.950 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241355+0000", 2023-10-09T10:52:53.950 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1395e-05 2023-10-09T10:52:53.950 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.950 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.950 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:53.950 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241362+0000", 2023-10-09T10:52:53.950 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7759999999999997e-06 2023-10-09T10:52:53.950 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.950 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.950 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.950 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241406+0000", 2023-10-09T10:52:53.950 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3189999999999998e-05 2023-10-09T10:52:53.951 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.951 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.951 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.951 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241413+0000", 2023-10-09T10:52:53.951 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5800000000000003e-06 2023-10-09T10:52:53.951 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.951 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.951 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.951 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103930+0000", 2023-10-09T10:52:53.951 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.86251731399999998 2023-10-09T10:52:53.951 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.951 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.952 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.952 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103938+0000", 2023-10-09T10:52:53.952 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7349999999999996e-06 2023-10-09T10:52:53.952 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.952 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.952 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.952 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183741+0000", 2023-10-09T10:52:53.952 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0798032179999999 2023-10-09T10:52:53.952 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.952 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.952 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:53.952 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183865+0000", 2023-10-09T10:52:53.953 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012327200000000001 2023-10-09T10:52:53.953 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.953 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.953 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:52:53.953 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.184011+0000", 2023-10-09T10:52:53.953 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014617499999999999 2023-10-09T10:52:53.953 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.953 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.953 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:53.953 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186989+0000", 2023-10-09T10:52:53.953 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029777340000000001 2023-10-09T10:52:53.953 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.954 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.954 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.954 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187029+0000", 2023-10-09T10:52:53.954 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0513000000000003e-05 2023-10-09T10:52:53.954 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.954 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.954 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.954 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199414+0000", 2023-10-09T10:52:53.954 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012385022000000001 2023-10-09T10:52:53.954 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.954 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.954 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:52:53.955 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199472+0000", 2023-10-09T10:52:53.955 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7998999999999998e-05 2023-10-09T10:52:53.955 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.955 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.955 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:53.955 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200317+0000", 2023-10-09T10:52:53.955 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000845245 2023-10-09T10:52:53.955 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.955 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:53.955 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.955 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.955 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.955 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42717 6.10 6:0e29612e:::benchmark_data_smithi125_244951_object42716:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:52:53.956 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:52:53.956 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 478.480664537, 2023-10-09T10:52:53.956 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8935023019999999, 2023-10-09T10:52:53.956 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:53.956 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:53.956 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:53.956 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:52:53.956 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:53.956 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42717 2023-10-09T10:52:53.956 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.956 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:53.956 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.957 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:53.957 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:52:53.957 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.957 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.957 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.957 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:53.957 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:52:53.957 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.957 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.957 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.957 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:53.957 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307449+0000", 2023-10-09T10:52:53.957 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:53.958 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.958 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.958 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:53.958 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307473+0000", 2023-10-09T10:52:53.958 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3349e-05 2023-10-09T10:52:53.958 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.958 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.958 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:53.958 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307490+0000", 2023-10-09T10:52:53.958 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7133e-05 2023-10-09T10:52:53.958 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.958 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.958 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:53.959 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307496+0000", 2023-10-09T10:52:53.959 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7349999999999999e-06 2023-10-09T10:52:53.959 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.959 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.959 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.959 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307561+0000", 2023-10-09T10:52:53.959 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4226999999999993e-05 2023-10-09T10:52:53.959 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.959 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.959 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.959 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307575+0000", 2023-10-09T10:52:53.959 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4504e-05 2023-10-09T10:52:53.960 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.960 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.960 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.960 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103963+0000", 2023-10-09T10:52:53.960 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.79638768500000001 2023-10-09T10:52:53.960 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.960 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.960 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.960 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103971+0000", 2023-10-09T10:52:53.960 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9629999999999998e-06 2023-10-09T10:52:53.960 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.960 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.960 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.961 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.185776+0000", 2023-10-09T10:52:53.961 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0818051479999999 2023-10-09T10:52:53.961 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.961 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.961 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:53.961 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.185949+0000", 2023-10-09T10:52:53.961 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000172659 2023-10-09T10:52:53.961 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.961 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.961 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:52:53.961 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186122+0000", 2023-10-09T10:52:53.961 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017296799999999999 2023-10-09T10:52:53.962 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.962 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.962 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:53.962 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189418+0000", 2023-10-09T10:52:53.962 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032959679999999998 2023-10-09T10:52:53.962 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.962 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.962 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.962 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189457+0000", 2023-10-09T10:52:53.962 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9669000000000003e-05 2023-10-09T10:52:53.962 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.962 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.963 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.963 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200372+0000", 2023-10-09T10:52:53.963 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010914633999999999 2023-10-09T10:52:53.963 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.963 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.963 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:52:53.963 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200421+0000", 2023-10-09T10:52:53.963 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9047e-05 2023-10-09T10:52:53.963 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.963 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.963 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:53.963 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200953+0000", 2023-10-09T10:52:53.964 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00053212500000000005 2023-10-09T10:52:53.964 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.964 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:53.964 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.964 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.964 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.964 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42722 6.10 6:0ee7a776:::benchmark_data_smithi125_244951_object42721:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:52:53.964 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:52:53.964 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 478.48060632900001, 2023-10-09T10:52:53.964 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8972440939999999, 2023-10-09T10:52:53.964 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:53.965 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:53.965 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:53.965 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:52:53.965 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:53.965 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42722 2023-10-09T10:52:53.965 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.965 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:53.965 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.965 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:53.965 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:52:53.965 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.965 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.965 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.966 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:53.966 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:52:53.966 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.966 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.966 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.966 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:53.966 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307508+0000", 2023-10-09T10:52:53.966 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:52:53.966 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.966 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.966 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:53.966 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307529+0000", 2023-10-09T10:52:53.967 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0866999999999999e-05 2023-10-09T10:52:53.967 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.967 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.967 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:53.967 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307539+0000", 2023-10-09T10:52:53.967 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2690000000000005e-06 2023-10-09T10:52:53.967 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.967 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.967 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:53.967 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307544+0000", 2023-10-09T10:52:53.967 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7930000000000003e-06 2023-10-09T10:52:53.967 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.967 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.968 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.968 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307599+0000", 2023-10-09T10:52:53.968 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4876000000000001e-05 2023-10-09T10:52:53.968 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.968 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.968 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.968 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307607+0000", 2023-10-09T10:52:53.968 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8760000000000005e-06 2023-10-09T10:52:53.968 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.968 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.968 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.968 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103995+0000", 2023-10-09T10:52:53.969 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.79638814099999999 2023-10-09T10:52:53.969 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.969 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.969 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.969 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104003+0000", 2023-10-09T10:52:53.969 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7819999999999994e-06 2023-10-09T10:52:53.969 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.969 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.969 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.969 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187058+0000", 2023-10-09T10:52:53.969 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.083054712 2023-10-09T10:52:53.969 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.969 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.970 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:53.970 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187234+0000", 2023-10-09T10:52:53.970 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017615799999999999 2023-10-09T10:52:53.970 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.970 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.970 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:52:53.970 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187416+0000", 2023-10-09T10:52:53.970 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018243 2023-10-09T10:52:53.970 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.970 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.970 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:53.970 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189466+0000", 2023-10-09T10:52:53.970 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020493210000000002 2023-10-09T10:52:53.971 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.971 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.971 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.971 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189504+0000", 2023-10-09T10:52:53.971 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8287000000000001e-05 2023-10-09T10:52:53.971 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.971 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.971 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.971 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.201001+0000", 2023-10-09T10:52:53.971 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011496632999999999 2023-10-09T10:52:53.971 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.971 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.971 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:52:53.972 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.201044+0000", 2023-10-09T10:52:53.972 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3078000000000003e-05 2023-10-09T10:52:53.972 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.972 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.972 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:53.972 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204753+0000", 2023-10-09T10:52:53.972 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037093209999999998 2023-10-09T10:52:53.972 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.972 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:53.972 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.972 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.972 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.973 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42757 6.10 6:08e1bcf5:::benchmark_data_smithi125_244951_object42756:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:52:53.973 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.333229+0000", 2023-10-09T10:52:53.973 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 478.45488617000001, 2023-10-09T10:52:53.973 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.872111463, 2023-10-09T10:52:53.973 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:53.973 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:53.973 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:53.973 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:52:53.973 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:52:53.973 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42757 2023-10-09T10:52:53.973 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.973 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:53.974 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.974 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:53.974 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333229+0000", 2023-10-09T10:52:53.974 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.974 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.974 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.974 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:53.974 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333229+0000", 2023-10-09T10:52:53.974 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.974 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.974 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.974 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:53.975 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333228+0000", 2023-10-09T10:52:53.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:52:53.975 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.975 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:53.975 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333280+0000", 2023-10-09T10:52:53.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1999999999999997e-05 2023-10-09T10:52:53.975 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.975 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:53.975 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333291+0000", 2023-10-09T10:52:53.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0738e-05 2023-10-09T10:52:53.975 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.976 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:53.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333298+0000", 2023-10-09T10:52:53.976 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7730000000000002e-06 2023-10-09T10:52:53.976 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.976 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333360+0000", 2023-10-09T10:52:53.976 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2230000000000006e-05 2023-10-09T10:52:53.976 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.976 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.333371+0000", 2023-10-09T10:52:53.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0669999999999999e-05 2023-10-09T10:52:53.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.977 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.977 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104027+0000", 2023-10-09T10:52:53.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.77065640400000002 2023-10-09T10:52:53.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.977 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.977 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104035+0000", 2023-10-09T10:52:53.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7400000000000004e-06 2023-10-09T10:52:53.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.978 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.978 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.978 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.188337+0000", 2023-10-09T10:52:53.978 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.084302281 2023-10-09T10:52:53.978 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.978 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.978 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:53.978 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.188495+0000", 2023-10-09T10:52:53.978 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015774699999999999 2023-10-09T10:52:53.978 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.978 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.978 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:52:53.979 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.188674+0000", 2023-10-09T10:52:53.979 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000178729 2023-10-09T10:52:53.979 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.979 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.979 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:53.979 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191542+0000", 2023-10-09T10:52:53.979 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028680799999999998 2023-10-09T10:52:53.979 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.979 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.979 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.979 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192592+0000", 2023-10-09T10:52:53.979 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010502140000000001 2023-10-09T10:52:53.979 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.980 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.980 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.980 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204808+0000", 2023-10-09T10:52:53.980 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012216595 2023-10-09T10:52:53.980 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.980 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.980 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:52:53.980 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204867+0000", 2023-10-09T10:52:53.980 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8251999999999997e-05 2023-10-09T10:52:53.980 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.980 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.980 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:53.981 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.205341+0000", 2023-10-09T10:52:53.981 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047385999999999999 2023-10-09T10:52:53.981 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.981 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:53.981 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.981 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.981 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.981 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152582 7.4 7:222afaba:::benchmark_data_smithi125_295816_object61445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:52:53.981 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.340929+0000", 2023-10-09T10:52:53.981 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 44.447186440999999, 2023-10-09T10:52:53.981 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8140581390000001, 2023-10-09T10:52:53.982 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:53.982 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:52:53.982 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:53.982 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:52:53.982 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:53.982 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152582 2023-10-09T10:52:53.982 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.982 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:53.982 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.982 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:53.982 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340929+0000", 2023-10-09T10:52:53.982 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.983 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.983 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.983 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:53.983 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340929+0000", 2023-10-09T10:52:53.983 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.983 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.983 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.983 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:53.983 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340928+0000", 2023-10-09T10:52:53.983 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:52:53.983 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.983 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.983 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:53.984 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340931+0000", 2023-10-09T10:52:53.984 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1329999999999999e-06 2023-10-09T10:52:53.984 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.984 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.984 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:53.984 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340948+0000", 2023-10-09T10:52:53.984 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6178999999999999e-05 2023-10-09T10:52:53.984 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.984 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.984 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:53.984 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340956+0000", 2023-10-09T10:52:53.984 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9980000000000003e-06 2023-10-09T10:52:53.985 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.985 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.985 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.985 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.341000+0000", 2023-10-09T10:52:53.985 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4508000000000001e-05 2023-10-09T10:52:53.985 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.985 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.985 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:53.985 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.341113+0000", 2023-10-09T10:52:53.985 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000113288 2023-10-09T10:52:53.985 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.985 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.986 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:52:53.986 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.341196+0000", 2023-10-09T10:52:53.986 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2163000000000006e-05 2023-10-09T10:52:53.986 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.986 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.986 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:53.986 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.344533+0000", 2023-10-09T10:52:53.986 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0033370129999999998 2023-10-09T10:52:53.986 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.986 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.986 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.986 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.346664+0000", 2023-10-09T10:52:53.986 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002131001 2023-10-09T10:52:53.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.987 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.987 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.987 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375125+0000", 2023-10-09T10:52:53.987 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.028460961999999999 2023-10-09T10:52:53.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.987 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.987 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.987 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375138+0000", 2023-10-09T10:52:53.987 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3223e-05 2023-10-09T10:52:53.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.987 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.987 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366624+0000", 2023-10-09T10:52:53.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99148582100000004 2023-10-09T10:52:53.988 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.988 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366632+0000", 2023-10-09T10:52:53.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5620000000000004e-06 2023-10-09T10:52:53.988 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.988 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.154916+0000", 2023-10-09T10:52:53.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.788283748 2023-10-09T10:52:53.988 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.989 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:53.989 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.154987+0000", 2023-10-09T10:52:53.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1076999999999997e-05 2023-10-09T10:52:53.989 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.989 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:53.989 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.989 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152588 7.9 7:96c4bba8:::benchmark_data_smithi125_295816_object61451:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:52:53.989 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.348147+0000", 2023-10-09T10:52:53.989 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 44.439968567000001, 2023-10-09T10:52:53.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8297705579999999, 2023-10-09T10:52:53.990 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:53.990 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:52:53.990 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:53.990 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:52:53.990 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:53.990 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152588 2023-10-09T10:52:53.990 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.990 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:53.990 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.990 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:53.990 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348147+0000", 2023-10-09T10:52:53.990 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.991 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.991 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.991 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:53.991 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348147+0000", 2023-10-09T10:52:53.991 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.991 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.991 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.991 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:53.991 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348146+0000", 2023-10-09T10:52:53.991 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:52:53.991 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.991 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.991 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:53.992 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348151+0000", 2023-10-09T10:52:53.992 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1150000000000002e-06 2023-10-09T10:52:53.992 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.992 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.992 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:53.992 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348158+0000", 2023-10-09T10:52:53.992 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6850000000000001e-06 2023-10-09T10:52:53.992 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.992 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.992 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:53.992 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348163+0000", 2023-10-09T10:52:53.992 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5979999999999999e-06 2023-10-09T10:52:53.992 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.993 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.993 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.993 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348227+0000", 2023-10-09T10:52:53.993 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3596000000000006e-05 2023-10-09T10:52:53.993 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.993 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.993 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:53.993 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348357+0000", 2023-10-09T10:52:53.993 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012999100000000001 2023-10-09T10:52:53.993 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.993 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.993 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:52:53.994 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348485+0000", 2023-10-09T10:52:53.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012842499999999999 2023-10-09T10:52:53.994 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.994 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.994 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:53.994 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350944+0000", 2023-10-09T10:52:53.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024594769999999998 2023-10-09T10:52:53.994 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.994 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.994 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:53.994 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.353449+0000", 2023-10-09T10:52:53.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025044780000000001 2023-10-09T10:52:53.995 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.995 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.995 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.376073+0000", 2023-10-09T10:52:53.995 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.022624136 2023-10-09T10:52:53.995 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.995 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.995 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.376084+0000", 2023-10-09T10:52:53.995 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0803e-05 2023-10-09T10:52:53.995 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.995 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.996 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.364523+0000", 2023-10-09T10:52:53.996 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98843897800000002 2023-10-09T10:52:53.996 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.996 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.996 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:53.996 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.364533+0000", 2023-10-09T10:52:53.996 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0458999999999999e-05 2023-10-09T10:52:53.996 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.996 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.996 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:53.996 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.177846+0000", 2023-10-09T10:52:53.996 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.81331285 2023-10-09T10:52:53.996 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.997 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:53.997 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.177917+0000", 2023-10-09T10:52:53.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1117000000000004e-05 2023-10-09T10:52:53.997 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.997 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:53.997 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:53.997 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.997 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152590 7.4 7:284d5455:::benchmark_data_smithi125_295816_object61453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:52:53.997 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.350331+0000", 2023-10-09T10:52:53.997 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 44.437784624000003, 2023-10-09T10:52:53.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.805048094, 2023-10-09T10:52:53.998 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:53.998 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:52:53.998 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:53.998 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:52:53.998 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:53.998 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152590 2023-10-09T10:52:53.998 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.998 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:53.998 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.998 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:53.998 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350331+0000", 2023-10-09T10:52:53.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.999 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:53.999 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350331+0000", 2023-10-09T10:52:53.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:53.999 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:53.999 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350329+0000", 2023-10-09T10:52:53.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:53.999 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:53.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:53.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:54.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350335+0000", 2023-10-09T10:52:54.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4639999999999997e-06 2023-10-09T10:52:54.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.000 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:54.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350352+0000", 2023-10-09T10:52:54.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7269e-05 2023-10-09T10:52:54.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.000 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:54.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350361+0000", 2023-10-09T10:52:54.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.388e-06 2023-10-09T10:52:54.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350415+0000", 2023-10-09T10:52:54.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4784000000000003e-05 2023-10-09T10:52:54.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:54.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350517+0000", 2023-10-09T10:52:54.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000101724 2023-10-09T10:52:54.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:52:54.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350653+0000", 2023-10-09T10:52:54.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013574099999999999 2023-10-09T10:52:54.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.002 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:54.002 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.353798+0000", 2023-10-09T10:52:54.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003145002 2023-10-09T10:52:54.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.002 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:54.002 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360490+0000", 2023-10-09T10:52:54.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066923240000000004 2023-10-09T10:52:54.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.003 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.003 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375171+0000", 2023-10-09T10:52:54.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014681339999999999 2023-10-09T10:52:54.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.003 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:54.003 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375182+0000", 2023-10-09T10:52:54.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0129000000000001e-05 2023-10-09T10:52:54.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.003 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366657+0000", 2023-10-09T10:52:54.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99147512100000001 2023-10-09T10:52:54.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:54.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366664+0000", 2023-10-09T10:52:54.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9369999999999998e-06 2023-10-09T10:52:54.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155313+0000", 2023-10-09T10:52:54.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.788649556 2023-10-09T10:52:54.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:54.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155379+0000", 2023-10-09T10:52:54.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5518000000000005e-05 2023-10-09T10:52:54.005 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:54.005 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:54.005 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:54.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.005 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152591 7.9 7:9f5cec69:::benchmark_data_smithi125_295816_object61454:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:52:54.005 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.354683+0000", 2023-10-09T10:52:54.005 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 44.433432013999997, 2023-10-09T10:52:54.006 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8235682930000001, 2023-10-09T10:52:54.006 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:54.006 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:52:54.006 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:54.006 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:52:54.006 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:54.006 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152591 2023-10-09T10:52:54.006 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.006 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:54.006 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.006 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:54.006 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354683+0000", 2023-10-09T10:52:54.007 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:54.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.007 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:54.007 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354683+0000", 2023-10-09T10:52:54.007 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:54.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.007 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:54.007 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354682+0000", 2023-10-09T10:52:54.007 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:54.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:54.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354688+0000", 2023-10-09T10:52:54.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7810000000000002e-06 2023-10-09T10:52:54.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:54.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354699+0000", 2023-10-09T10:52:54.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1556000000000001e-05 2023-10-09T10:52:54.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:54.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354707+0000", 2023-10-09T10:52:54.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3799999999999996e-06 2023-10-09T10:52:54.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354752+0000", 2023-10-09T10:52:54.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5756e-05 2023-10-09T10:52:54.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:54.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354885+0000", 2023-10-09T10:52:54.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000132397 2023-10-09T10:52:54.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.010 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:52:54.010 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.355009+0000", 2023-10-09T10:52:54.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000123714 2023-10-09T10:52:54.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.010 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.010 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:54.010 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357282+0000", 2023-10-09T10:52:54.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022739679999999999 2023-10-09T10:52:54.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.010 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.010 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.010 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.376279+0000", 2023-10-09T10:52:54.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018997012000000001 2023-10-09T10:52:54.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:54.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.376289+0000", 2023-10-09T10:52:54.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.781e-06 2023-10-09T10:52:54.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.364560+0000", 2023-10-09T10:52:54.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98827064099999995 2023-10-09T10:52:54.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.012 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:54.012 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365610+0000", 2023-10-09T10:52:54.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001049822 2023-10-09T10:52:54.012 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.012 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.012 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.012 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.178188+0000", 2023-10-09T10:52:54.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8125778029999999 2023-10-09T10:52:54.012 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.012 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.012 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:54.012 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.178252+0000", 2023-10-09T10:52:54.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4000999999999999e-05 2023-10-09T10:52:54.013 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:54.013 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:54.013 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:54.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.013 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152593 7.4 7:21dbc744:::benchmark_data_smithi125_295816_object61456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:52:54.013 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.354718+0000", 2023-10-09T10:52:54.013 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 44.433396782000003, 2023-10-09T10:52:54.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8007503599999999, 2023-10-09T10:52:54.013 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:54.013 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:52:54.013 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:54.013 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:52:54.014 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:54.014 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152593 2023-10-09T10:52:54.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.014 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:54.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:54.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354718+0000", 2023-10-09T10:52:54.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:54.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:54.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354718+0000", 2023-10-09T10:52:54.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:54.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:54.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354718+0000", 2023-10-09T10:52:54.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:52:54.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:54.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354721+0000", 2023-10-09T10:52:54.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8200000000000001e-06 2023-10-09T10:52:54.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:54.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354728+0000", 2023-10-09T10:52:54.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7309999999999996e-06 2023-10-09T10:52:54.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:54.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354734+0000", 2023-10-09T10:52:54.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1009999999999997e-06 2023-10-09T10:52:54.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354780+0000", 2023-10-09T10:52:54.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6464000000000001e-05 2023-10-09T10:52:54.017 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.017 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:54.017 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354913+0000", 2023-10-09T10:52:54.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000132569 2023-10-09T10:52:54.017 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.017 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:52:54.017 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.355043+0000", 2023-10-09T10:52:54.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013061799999999999 2023-10-09T10:52:54.017 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.018 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:54.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.359660+0000", 2023-10-09T10:52:54.018 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046166929999999998 2023-10-09T10:52:54.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.018 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375223+0000", 2023-10-09T10:52:54.018 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015563420999999999 2023-10-09T10:52:54.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.018 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:54.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375233+0000", 2023-10-09T10:52:54.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7999999999999993e-06 2023-10-09T10:52:54.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.019 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366685+0000", 2023-10-09T10:52:54.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99145126699999997 2023-10-09T10:52:54.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:54.019 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366692+0000", 2023-10-09T10:52:54.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.498e-06 2023-10-09T10:52:54.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.020 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155410+0000", 2023-10-09T10:52:54.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.788717651 2023-10-09T10:52:54.020 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.020 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:54.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155469+0000", 2023-10-09T10:52:54.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9157000000000003e-05 2023-10-09T10:52:54.020 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:54.020 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:54.020 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:54.020 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.021 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.021 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152597 7.4 7:23416840:::benchmark_data_smithi125_295816_object61460:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:52:54.021 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.357109+0000", 2023-10-09T10:52:54.021 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 44.431006605999997, 2023-10-09T10:52:54.021 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7984508359999998, 2023-10-09T10:52:54.021 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:54.021 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:52:54.021 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:54.021 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:52:54.021 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:54.021 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152597 2023-10-09T10:52:54.021 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.022 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:54.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:54.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357109+0000", 2023-10-09T10:52:54.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:54.022 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:54.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357109+0000", 2023-10-09T10:52:54.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:54.022 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:54.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357108+0000", 2023-10-09T10:52:54.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:52:54.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:54.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357112+0000", 2023-10-09T10:52:54.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.673e-06 2023-10-09T10:52:54.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:54.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357123+0000", 2023-10-09T10:52:54.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0567999999999999e-05 2023-10-09T10:52:54.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:54.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357130+0000", 2023-10-09T10:52:54.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4599999999999997e-06 2023-10-09T10:52:54.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357336+0000", 2023-10-09T10:52:54.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020526499999999999 2023-10-09T10:52:54.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:54.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357462+0000", 2023-10-09T10:52:54.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012642400000000001 2023-10-09T10:52:54.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:52:54.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357581+0000", 2023-10-09T10:52:54.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118647 2023-10-09T10:52:54.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:54.026 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.359832+0000", 2023-10-09T10:52:54.026 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002251121 2023-10-09T10:52:54.026 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.026 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.026 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375269+0000", 2023-10-09T10:52:54.026 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015436699999999999 2023-10-09T10:52:54.026 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.026 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:54.026 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375278+0000", 2023-10-09T10:52:54.026 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8199999999999992e-06 2023-10-09T10:52:54.027 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.027 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.027 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366716+0000", 2023-10-09T10:52:54.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99143730600000002 2023-10-09T10:52:54.027 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.027 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.027 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:54.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366725+0000", 2023-10-09T10:52:54.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8759999999999994e-06 2023-10-09T10:52:54.027 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.027 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.028 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.028 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155501+0000", 2023-10-09T10:52:54.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7887762249999999 2023-10-09T10:52:54.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.028 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:54.028 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155559+0000", 2023-10-09T10:52:54.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8628000000000002e-05 2023-10-09T10:52:54.028 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:54.028 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:54.028 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:54.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.029 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.029 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152599 7.4 7:26957a0e:::benchmark_data_smithi125_295816_object61462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:52:54.029 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.360294+0000", 2023-10-09T10:52:54.029 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 44.427821322, 2023-10-09T10:52:54.029 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7970287539999998, 2023-10-09T10:52:54.029 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:54.029 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:52:54.029 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:54.029 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:52:54.029 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:54.029 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152599 2023-10-09T10:52:54.029 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.030 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:54.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:54.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360294+0000", 2023-10-09T10:52:54.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:54.030 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:54.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360294+0000", 2023-10-09T10:52:54.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:54.030 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.031 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:54.031 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360292+0000", 2023-10-09T10:52:54.031 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:52:54.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.031 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.031 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:54.031 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360297+0000", 2023-10-09T10:52:54.031 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0170000000000004e-06 2023-10-09T10:52:54.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.031 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.031 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:54.032 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360318+0000", 2023-10-09T10:52:54.032 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0460999999999999e-05 2023-10-09T10:52:54.032 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.032 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.032 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:54.032 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360325+0000", 2023-10-09T10:52:54.032 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2119999999999997e-06 2023-10-09T10:52:54.032 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.032 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.032 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.032 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360517+0000", 2023-10-09T10:52:54.032 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000192487 2023-10-09T10:52:54.032 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:54.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360676+0000", 2023-10-09T10:52:54.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015863499999999999 2023-10-09T10:52:54.033 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:52:54.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360826+0000", 2023-10-09T10:52:54.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015037199999999999 2023-10-09T10:52:54.033 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.034 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.034 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.378926+0000", 2023-10-09T10:52:54.034 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018099725000000001 2023-10-09T10:52:54.034 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.034 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.034 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:54.034 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.378938+0000", 2023-10-09T10:52:54.034 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2191000000000001e-05 2023-10-09T10:52:54.034 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.034 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.034 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.034 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366754+0000", 2023-10-09T10:52:54.035 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98781607800000004 2023-10-09T10:52:54.035 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.035 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.035 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:54.035 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366763+0000", 2023-10-09T10:52:54.035 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6179999999999995e-06 2023-10-09T10:52:54.035 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.035 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.035 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.035 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.157249+0000", 2023-10-09T10:52:54.035 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.790486386 2023-10-09T10:52:54.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.036 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:54.036 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.157323+0000", 2023-10-09T10:52:54.036 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3258999999999998e-05 2023-10-09T10:52:54.036 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:54.036 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:54.036 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:54.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.036 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152606 7.9 7:912bbd31:::benchmark_data_smithi125_295816_object61469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T10:52:54.036 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.394803+0000", 2023-10-09T10:52:54.036 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 44.393312039000001, 2023-10-09T10:52:54.037 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7960543420000001, 2023-10-09T10:52:54.037 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:54.037 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:54.037 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:54.037 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:52:54.037 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:54.037 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152606 2023-10-09T10:52:54.037 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.037 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:54.037 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.037 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:54.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394803+0000", 2023-10-09T10:52:54.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:54.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:54.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394803+0000", 2023-10-09T10:52:54.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:54.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:54.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394802+0000", 2023-10-09T10:52:54.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:54.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.039 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:54.039 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394808+0000", 2023-10-09T10:52:54.039 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5220000000000004e-06 2023-10-09T10:52:54.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.039 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:54.039 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394821+0000", 2023-10-09T10:52:54.039 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2325e-05 2023-10-09T10:52:54.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.039 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:54.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394829+0000", 2023-10-09T10:52:54.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3820000000000004e-06 2023-10-09T10:52:54.040 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.040 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.040 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394972+0000", 2023-10-09T10:52:54.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000142788 2023-10-09T10:52:54.040 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.040 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.040 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:54.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394982+0000", 2023-10-09T10:52:54.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0336000000000001e-05 2023-10-09T10:52:54.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365844+0000", 2023-10-09T10:52:54.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97086172500000001 2023-10-09T10:52:54.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:54.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365853+0000", 2023-10-09T10:52:54.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5230000000000001e-06 2023-10-09T10:52:54.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.042 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.042 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.178273+0000", 2023-10-09T10:52:54.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8124192300000002 2023-10-09T10:52:54.042 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.042 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.042 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:54.042 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.178376+0000", 2023-10-09T10:52:54.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000102841 2023-10-09T10:52:54.042 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.042 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.042 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T10:52:54.043 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.178485+0000", 2023-10-09T10:52:54.043 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000109516 2023-10-09T10:52:54.043 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.043 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.043 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:54.043 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.183468+0000", 2023-10-09T10:52:54.043 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049829920000000003 2023-10-09T10:52:54.043 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.043 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.043 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:54.043 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.186132+0000", 2023-10-09T10:52:54.044 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002663538 2023-10-09T10:52:54.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:54.044 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.186288+0000", 2023-10-09T10:52:54.044 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015653499999999999 2023-10-09T10:52:54.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:54.044 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.190512+0000", 2023-10-09T10:52:54.044 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042239850000000004 2023-10-09T10:52:54.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:52:54.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.190562+0000", 2023-10-09T10:52:54.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0169e-05 2023-10-09T10:52:54.045 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.045 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:54.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.190858+0000", 2023-10-09T10:52:54.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00029527200000000001 2023-10-09T10:52:54.045 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:54.045 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:54.046 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:54.046 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.046 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.046 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152607 7.9 7:9c2ec87f:::benchmark_data_smithi125_295816_object61470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T10:52:54.046 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.397825+0000", 2023-10-09T10:52:54.046 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 44.390290090999997, 2023-10-09T10:52:54.046 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.811002282, 2023-10-09T10:52:54.046 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:52:54.046 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:52:54.046 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:52:54.046 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:52:54.047 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:52:54.047 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152607 2023-10-09T10:52:54.047 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.047 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:52:54.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.047 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:52:54.047 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397825+0000", 2023-10-09T10:52:54.047 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:54.047 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.047 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:52:54.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397825+0000", 2023-10-09T10:52:54.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:52:54.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.048 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:52:54.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397824+0000", 2023-10-09T10:52:54.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:52:54.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.048 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:52:54.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397830+0000", 2023-10-09T10:52:54.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9199999999999998e-06 2023-10-09T10:52:54.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:52:54.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397844+0000", 2023-10-09T10:52:54.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3648999999999999e-05 2023-10-09T10:52:54.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:52:54.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397852+0000", 2023-10-09T10:52:54.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9009999999999994e-06 2023-10-09T10:52:54.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397895+0000", 2023-10-09T10:52:54.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3269999999999997e-05 2023-10-09T10:52:54.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:54.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397907+0000", 2023-10-09T10:52:54.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1507e-05 2023-10-09T10:52:54.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365878+0000", 2023-10-09T10:52:54.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96797074800000005 2023-10-09T10:52:54.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:52:54.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365886+0000", 2023-10-09T10:52:54.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0909999999999993e-06 2023-10-09T10:52:54.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:52:54.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.180824+0000", 2023-10-09T10:52:54.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8149384070000001 2023-10-09T10:52:54.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:52:54.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.180944+0000", 2023-10-09T10:52:54.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120256 2023-10-09T10:52:54.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T10:52:54.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.181120+0000", 2023-10-09T10:52:54.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017529399999999999 2023-10-09T10:52:54.053 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.053 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:52:54.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.188514+0000", 2023-10-09T10:52:54.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0073948709999999999 2023-10-09T10:52:54.053 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.053 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:54.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.190897+0000", 2023-10-09T10:52:54.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023828120000000002 2023-10-09T10:52:54.053 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.054 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:54.054 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.192009+0000", 2023-10-09T10:52:54.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011115719999999999 2023-10-09T10:52:54.054 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.054 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.054 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:52:54.054 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.208460+0000", 2023-10-09T10:52:54.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.016451364999999999 2023-10-09T10:52:54.054 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.054 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.054 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:52:54.054 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.208512+0000", 2023-10-09T10:52:54.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2067999999999997e-05 2023-10-09T10:52:54.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:52:54.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:52:54.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:52:54.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.208827+0000", 2023-10-09T10:52:54.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00031514599999999998 2023-10-09T10:52:54.055 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:54.055 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:54.055 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:54.055 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:52:54.055 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:52:54.055 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:52:54.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:54.057 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:52:54.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:52:54.138 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:52:54.156 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:52:54.159 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:52:54.183 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:52:54.186 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:54.205 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:52:54.205 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:52:54.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:54.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:52:54.213 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:52:54.215 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:54.230 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:52:54.254 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:52:54.280 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:52:54.299 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:52:54.300 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:52:54.300 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:52:54.301 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:52:54.301 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:52:54.301 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:52:54.301 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:52:54.301 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:52:54.301 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:52:54.309 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:52:54.341 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:54.359 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:52:54.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:54.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:52:54.462 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:52:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:54 smithi155 ceph-mon[184854]: pgmap v608: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 remapped+peering, 1 peering, 3 unknown, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 105 active+clean; 524 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 433 op/s; 14 KiB/s, 327 objects/s recovering 2023-10-09T10:52:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:54 smithi155 ceph-mon[184854]: failed to encode map e258 with expected crc 2023-10-09T10:52:54.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:54 smithi155 ceph-mon[184854]: failed to encode map e257 with expected crc 2023-10-09T10:52:54.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:54 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T10:52:54.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:54 smithi155 ceph-mon[184854]: osdmap e258: 8 total, 8 up, 6 in 2023-10-09T10:52:54.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:54 smithi155 ceph-mon[184854]: failed to encode map e258 with expected crc 2023-10-09T10:52:54.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:54 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:52:54.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:54 smithi155 ceph-mon[184854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2023-10-09T10:52:54.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:54 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/354911004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:54.490 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[195507]: pgmap v608: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 remapped+peering, 1 peering, 3 unknown, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 105 active+clean; 524 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 433 op/s; 14 KiB/s, 327 objects/s recovering 2023-10-09T10:52:54.491 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[195507]: failed to encode map e258 with expected crc 2023-10-09T10:52:54.491 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[195507]: failed to encode map e257 with expected crc 2023-10-09T10:52:54.491 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T10:52:54.491 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[195507]: osdmap e258: 8 total, 8 up, 6 in 2023-10-09T10:52:54.491 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[195507]: failed to encode map e258 with expected crc 2023-10-09T10:52:54.491 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:52:54.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[195507]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2023-10-09T10:52:54.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/354911004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:54.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[200197]: pgmap v608: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 remapped+peering, 1 peering, 3 unknown, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 105 active+clean; 524 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 433 op/s; 14 KiB/s, 327 objects/s recovering 2023-10-09T10:52:54.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[200197]: failed to encode map e258 with expected crc 2023-10-09T10:52:54.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[200197]: failed to encode map e257 with expected crc 2023-10-09T10:52:54.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T10:52:54.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[200197]: osdmap e258: 8 total, 8 up, 6 in 2023-10-09T10:52:54.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[200197]: failed to encode map e258 with expected crc 2023-10-09T10:52:54.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:52:54.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[200197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2023-10-09T10:52:54.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:54 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/354911004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:52:54.494 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:54.612 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:52:54.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:54.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:52:54.716 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:54.729 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:54.804 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:54.829 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:54.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:54.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:54.851 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:52:54.932 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:52:54.963 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:55.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:52:55.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:55.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:52:55.137 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:52:55.166 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:55.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:55 smithi125 ceph-mon[195507]: failed to encode map e259 with expected crc 2023-10-09T10:52:55.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:55 smithi125 ceph-mon[195507]: osdmap e259: 8 total, 8 up, 6 in 2023-10-09T10:52:55.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:55 smithi125 ceph-mon[195507]: failed to encode map e259 with expected crc 2023-10-09T10:52:55.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:55 smithi125 ceph-mon[200197]: failed to encode map e259 with expected crc 2023-10-09T10:52:55.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:55 smithi125 ceph-mon[200197]: osdmap e259: 8 total, 8 up, 6 in 2023-10-09T10:52:55.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:55 smithi125 ceph-mon[200197]: failed to encode map e259 with expected crc 2023-10-09T10:52:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:55 smithi155 ceph-mon[184854]: failed to encode map e259 with expected crc 2023-10-09T10:52:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:55 smithi155 ceph-mon[184854]: osdmap e259: 8 total, 8 up, 6 in 2023-10-09T10:52:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:55 smithi155 ceph-mon[184854]: failed to encode map e259 with expected crc 2023-10-09T10:52:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:56 smithi155 ceph-mon[184854]: pgmap v611: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 remapped+peering, 3 peering, 2 unknown, 2 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 103 active+clean; 550 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 2 op/s; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%); 47 KiB/s, 455 objects/s recovering 2023-10-09T10:52:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:56 smithi155 ceph-mon[184854]: Health check failed: Degraded data redundancy: 1/66 objects degraded (1.515%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:52:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:56 smithi155 ceph-mon[184854]: failed to encode map e260 with expected crc 2023-10-09T10:52:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:56 smithi155 ceph-mon[184854]: failed to encode map e260 with expected crc 2023-10-09T10:52:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:56 smithi155 ceph-mon[184854]: osdmap e260: 8 total, 8 up, 6 in 2023-10-09T10:52:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:56 smithi155 ceph-mon[184854]: failed to encode map e260 with expected crc 2023-10-09T10:52:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:56 smithi155 ceph-mon[184854]: failed to encode map e260 with expected crc 2023-10-09T10:52:56.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:56 smithi155 ceph-mon[184854]: failed to encode map e260 with expected crc 2023-10-09T10:52:56.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[200197]: pgmap v611: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 remapped+peering, 3 peering, 2 unknown, 2 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 103 active+clean; 550 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 2 op/s; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%); 47 KiB/s, 455 objects/s recovering 2023-10-09T10:52:56.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[200197]: Health check failed: Degraded data redundancy: 1/66 objects degraded (1.515%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:52:56.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[200197]: failed to encode map e260 with expected crc 2023-10-09T10:52:56.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[200197]: failed to encode map e260 with expected crc 2023-10-09T10:52:56.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[200197]: osdmap e260: 8 total, 8 up, 6 in 2023-10-09T10:52:56.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[200197]: failed to encode map e260 with expected crc 2023-10-09T10:52:56.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[200197]: failed to encode map e260 with expected crc 2023-10-09T10:52:56.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[200197]: failed to encode map e260 with expected crc 2023-10-09T10:52:56.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[195507]: pgmap v611: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 remapped+peering, 3 peering, 2 unknown, 2 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 103 active+clean; 550 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 2 op/s; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%); 47 KiB/s, 455 objects/s recovering 2023-10-09T10:52:56.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[195507]: Health check failed: Degraded data redundancy: 1/66 objects degraded (1.515%), 1 pg degraded (PG_DEGRADED) 2023-10-09T10:52:56.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[195507]: failed to encode map e260 with expected crc 2023-10-09T10:52:56.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[195507]: failed to encode map e260 with expected crc 2023-10-09T10:52:56.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[195507]: osdmap e260: 8 total, 8 up, 6 in 2023-10-09T10:52:56.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[195507]: failed to encode map e260 with expected crc 2023-10-09T10:52:56.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[195507]: failed to encode map e260 with expected crc 2023-10-09T10:52:56.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:56 smithi125 ceph-mon[195507]: failed to encode map e260 with expected crc 2023-10-09T10:52:56.827 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:52:57.134 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:52:57.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:52:57.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:52:57.279 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:52:57.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:57 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:52:57.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:57 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:52:57.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:57 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:52:58.260 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:52:58.521 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:52:58 smithi125 ceph-mon[195507]: pgmap v613: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 remapped+peering, 2 peering, 2 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 107 active+clean; 550 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 11 KiB/s rd, 30 KiB/s wr, 9 op/s; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%); 54 KiB/s, 369 objects/s recovering 2023-10-09T10:52:58.521 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:52:58 smithi125 ceph-mon[200197]: pgmap v613: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 remapped+peering, 2 peering, 2 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 107 active+clean; 550 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 11 KiB/s rd, 30 KiB/s wr, 9 op/s; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%); 54 KiB/s, 369 objects/s recovering 2023-10-09T10:52:58.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:52:58 smithi155 ceph-mon[184854]: pgmap v613: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 remapped+peering, 2 peering, 2 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 107 active+clean; 550 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 11 KiB/s rd, 30 KiB/s wr, 9 op/s; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%); 54 KiB/s, 369 objects/s recovering 2023-10-09T10:52:58.868 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 1] out_osds: [2, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:52:58.868 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:52:58.868 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-09T10:52:58.869 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-09T10:53:00.220 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:52:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=sqlstore.transactions t=2023-10-09T10:52:59.882359225Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T10:53:00.437 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:53:00.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:00.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:53:00.470 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:53:00.540 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:53:00.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:00 smithi125 ceph-mon[200197]: pgmap v614: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 peering, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 550 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 5 op/s; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%); 13 KiB/s, 100 objects/s recovering 2023-10-09T10:53:00.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:00 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4159796191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:53:00.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:53:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:00 smithi125 ceph-mon[195507]: pgmap v614: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 peering, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 550 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 5 op/s; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%); 13 KiB/s, 100 objects/s recovering 2023-10-09T10:53:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:00 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4159796191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:53:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:53:00.578 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:00.613 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:53:00.613 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:53:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:00 smithi155 ceph-mon[184854]: pgmap v614: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 peering, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 108 active+clean; 550 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 5 op/s; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%); 13 KiB/s, 100 objects/s recovering 2023-10-09T10:53:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:00 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4159796191' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:53:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:53:01.472 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:53:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:53:01] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T10:53:01.482 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_1' removed 2023-10-09T10:53:01.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:01 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:53:01.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4159796191' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:53:01.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:01 smithi125 ceph-mon[195507]: osdmap e261: 8 total, 8 up, 6 in 2023-10-09T10:53:01.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:01 smithi125 ceph-mon[195507]: failed to encode map e261 with expected crc 2023-10-09T10:53:01.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:01 smithi125 ceph-mon[195507]: failed to encode map e261 with expected crc 2023-10-09T10:53:01.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/812104842' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:53:01.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:01 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:53:01.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:01 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:53:01.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4159796191' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:53:01.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:01 smithi125 ceph-mon[200197]: osdmap e261: 8 total, 8 up, 6 in 2023-10-09T10:53:01.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:01 smithi125 ceph-mon[200197]: failed to encode map e261 with expected crc 2023-10-09T10:53:01.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:01 smithi125 ceph-mon[200197]: failed to encode map e261 with expected crc 2023-10-09T10:53:01.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/812104842' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:53:01.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:01 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:53:01.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:01 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:53:01.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4159796191' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:53:01.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:01 smithi155 ceph-mon[184854]: osdmap e261: 8 total, 8 up, 6 in 2023-10-09T10:53:01.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:01 smithi155 ceph-mon[184854]: failed to encode map e261 with expected crc 2023-10-09T10:53:01.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:01 smithi155 ceph-mon[184854]: failed to encode map e261 with expected crc 2023-10-09T10:53:01.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/812104842' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:53:01.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:01 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:53:02.258 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:53:02.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:02.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:53:02.361 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:53:02.390 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:02.527 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:02 smithi125 ceph-mon[195507]: pgmap v616: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 110 active+clean; 550 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 6.1 KiB/s rd, 16 KiB/s wr, 4 op/s; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%); 11 KiB/s, 334 objects/s recovering 2023-10-09T10:53:02.527 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:02 smithi125 ceph-mon[195507]: failed to encode map e262 with expected crc 2023-10-09T10:53:02.527 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:02 smithi125 ceph-mon[195507]: failed to encode map e262 with expected crc 2023-10-09T10:53:02.528 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:02 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:53:02.528 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:02 smithi125 ceph-mon[195507]: osdmap e262: 8 total, 8 up, 6 in 2023-10-09T10:53:02.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:02 smithi125 ceph-mon[200197]: pgmap v616: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 110 active+clean; 550 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 6.1 KiB/s rd, 16 KiB/s wr, 4 op/s; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%); 11 KiB/s, 334 objects/s recovering 2023-10-09T10:53:02.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:02 smithi125 ceph-mon[200197]: failed to encode map e262 with expected crc 2023-10-09T10:53:02.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:02 smithi125 ceph-mon[200197]: failed to encode map e262 with expected crc 2023-10-09T10:53:02.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:02 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:53:02.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:02 smithi125 ceph-mon[200197]: osdmap e262: 8 total, 8 up, 6 in 2023-10-09T10:53:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:02 smithi155 ceph-mon[184854]: pgmap v616: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 110 active+clean; 550 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 6.1 KiB/s rd, 16 KiB/s wr, 4 op/s; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%); 11 KiB/s, 334 objects/s recovering 2023-10-09T10:53:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:02 smithi155 ceph-mon[184854]: failed to encode map e262 with expected crc 2023-10-09T10:53:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:02 smithi155 ceph-mon[184854]: failed to encode map e262 with expected crc 2023-10-09T10:53:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:02 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:53:02.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:02 smithi155 ceph-mon[184854]: osdmap e262: 8 total, 8 up, 6 in 2023-10-09T10:53:03.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:03 smithi125 ceph-mon[195507]: failed to encode map e263 with expected crc 2023-10-09T10:53:03.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:03 smithi125 ceph-mon[195507]: osdmap e263: 8 total, 8 up, 6 in 2023-10-09T10:53:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:03 smithi125 ceph-mon[200197]: failed to encode map e263 with expected crc 2023-10-09T10:53:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:03 smithi125 ceph-mon[200197]: osdmap e263: 8 total, 8 up, 6 in 2023-10-09T10:53:03.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:03 smithi155 ceph-mon[184854]: failed to encode map e263 with expected crc 2023-10-09T10:53:03.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:03 smithi155 ceph-mon[184854]: osdmap e263: 8 total, 8 up, 6 in 2023-10-09T10:53:04.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:04 smithi125 ceph-mon[195507]: pgmap v619: 103 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 550 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%) 2023-10-09T10:53:04.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:04 smithi125 ceph-mon[195507]: failed to encode map e264 with expected crc 2023-10-09T10:53:04.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:04 smithi125 ceph-mon[195507]: osdmap e264: 8 total, 8 up, 6 in 2023-10-09T10:53:04.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:04 smithi125 ceph-mon[195507]: failed to encode map e264 with expected crc 2023-10-09T10:53:04.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:04 smithi125 ceph-mon[195507]: failed to encode map e264 with expected crc 2023-10-09T10:53:04.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:04 smithi125 ceph-mon[200197]: pgmap v619: 103 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 550 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%) 2023-10-09T10:53:04.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:04 smithi125 ceph-mon[200197]: failed to encode map e264 with expected crc 2023-10-09T10:53:04.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:04 smithi125 ceph-mon[200197]: osdmap e264: 8 total, 8 up, 6 in 2023-10-09T10:53:04.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:04 smithi125 ceph-mon[200197]: failed to encode map e264 with expected crc 2023-10-09T10:53:04.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:04 smithi125 ceph-mon[200197]: failed to encode map e264 with expected crc 2023-10-09T10:53:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:04 smithi155 ceph-mon[184854]: pgmap v619: 103 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 550 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%) 2023-10-09T10:53:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:04 smithi155 ceph-mon[184854]: failed to encode map e264 with expected crc 2023-10-09T10:53:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:04 smithi155 ceph-mon[184854]: osdmap e264: 8 total, 8 up, 6 in 2023-10-09T10:53:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:04 smithi155 ceph-mon[184854]: failed to encode map e264 with expected crc 2023-10-09T10:53:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:04 smithi155 ceph-mon[184854]: failed to encode map e264 with expected crc 2023-10-09T10:53:05.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:05 smithi125 ceph-mon[195507]: failed to encode map e264 with expected crc 2023-10-09T10:53:05.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:05 smithi125 ceph-mon[200197]: failed to encode map e264 with expected crc 2023-10-09T10:53:05.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:05 smithi155 ceph-mon[184854]: failed to encode map e264 with expected crc 2023-10-09T10:53:06.097 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:53:06.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:06.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:53:06.391 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:53:06.424 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:06.553 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:53:06.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:06 smithi125 ceph-mon[195507]: pgmap v621: 103 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 550 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%) 2023-10-09T10:53:06.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:06 smithi125 ceph-mon[200197]: pgmap v621: 103 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 550 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%) 2023-10-09T10:53:06.700 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T10:53:06.701 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T10:53:06.701 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-10-09T10:53:06.702 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-10-09T10:53:06.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:06 smithi155 ceph-mon[184854]: pgmap v621: 103 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 550 KiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 1/66 objects degraded (1.515%); 1/66 objects misplaced (1.515%) 2023-10-09T10:53:07.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:07 smithi125 ceph-mon[195507]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/66 objects degraded (1.515%), 1 pg degraded) 2023-10-09T10:53:07.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:07 smithi125 ceph-mon[200197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/66 objects degraded (1.515%), 1 pg degraded) 2023-10-09T10:53:07.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:07 smithi155 ceph-mon[184854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/66 objects degraded (1.515%), 1 pg degraded) 2023-10-09T10:53:08.068 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:53:08.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:08.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:53:08.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:08.206 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:08.427 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:53:08.581 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:08.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:08.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:08.643 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:08.643 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:08.643 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:08.696 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:53:08.721 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:08 smithi125 ceph-mon[195507]: pgmap v622: 103 pgs: 103 active+clean; 550 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail 2023-10-09T10:53:08.722 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:08 smithi125 ceph-mon[200197]: pgmap v622: 103 pgs: 103 active+clean; 550 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail 2023-10-09T10:53:08.723 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:08.786 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:53:08.787 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:08.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:08.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:53:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:08 smithi155 ceph-mon[184854]: pgmap v622: 103 pgs: 103 active+clean; 550 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail 2023-10-09T10:53:08.972 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:53:08.999 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:10.172 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:53:10.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:10.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:53:10.276 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:53:10.311 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:10.335 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:53:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:10.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:53:10.438 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:10.467 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:10.537 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:10.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:10.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:10.641 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:10.672 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:10 smithi155 ceph-mon[184854]: pgmap v623: 103 pgs: 103 active+clean; 550 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 33 KiB/s, 0 objects/s recovering 2023-10-09T10:53:10.675 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:10.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:10 smithi125 ceph-mon[195507]: pgmap v623: 103 pgs: 103 active+clean; 550 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 33 KiB/s, 0 objects/s recovering 2023-10-09T10:53:10.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:10 smithi125 ceph-mon[200197]: pgmap v623: 103 pgs: 103 active+clean; 550 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 33 KiB/s, 0 objects/s recovering 2023-10-09T10:53:10.695 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:10.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:10.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:10.799 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:10.827 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:10.946 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:10.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:10.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:11.050 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:53:11.083 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:11.104 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:53:11.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:11.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:53:11.208 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:53:11.239 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:11.259 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:53:11.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:11.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:53:11.344 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:53:11.362 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:53:11.363 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:53:11.381 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:53:11.391 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:11.400 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:53:11.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:53:11.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:11.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:53:11.420 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:53:11.441 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:53:11.463 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:53:11.482 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:53:11.484 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:53:11.484 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:53:11.484 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:53:11.484 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:53:11.484 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:53:11.484 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:53:11.484 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:53:11.484 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:53:11.504 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:53:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:53:11] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T10:53:11.513 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:11.542 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:11.665 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:11.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:11.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:11.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:11 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/680053121' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:53:11.769 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:11.798 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:11.818 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:11.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:11 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/680053121' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:53:11.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:11 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/680053121' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:53:11.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:11.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:11.922 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:53:11.950 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:11.968 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:53:11.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:11.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:53:12.072 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:12.105 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:12.174 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:12.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:12.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:12.278 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:12.309 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:12.332 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:12.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:12.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:12.436 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:12.464 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:12.483 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:12.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:12.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:12.587 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:53:12.615 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:12.634 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:53:12.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:12.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:53:12.737 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:53:12.767 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:12.785 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:53:12.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:12.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:53:12.808 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:12.815 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_2' created 2023-10-09T10:53:12.890 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:12.918 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:12.936 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:12.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:12.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:12.992 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:12 smithi125 ceph-mon[195507]: pgmap v624: 103 pgs: 103 active+clean; 550 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 29 KiB/s, 0 objects/s recovering 2023-10-09T10:53:12.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:12 smithi125 ceph-mon[200197]: pgmap v624: 103 pgs: 103 active+clean; 550 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 29 KiB/s, 0 objects/s recovering 2023-10-09T10:53:13.040 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:53:13.068 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:13.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:53:13.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:13.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:53:13.193 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:13.218 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:12 smithi155 ceph-mon[184854]: pgmap v624: 103 pgs: 103 active+clean; 550 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 29 KiB/s, 0 objects/s recovering 2023-10-09T10:53:13.220 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:13.238 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:13.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:13.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:13.308 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T10:53:13.054+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e265 definitely_dead 0 2023-10-09T10:53:13.342 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:13.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-10-09T10:53:13.371 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:13.492 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:13.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:13.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:13.605 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:53:13.633 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:13.691 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:13.702 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:53:13.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:13.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:53:13.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:13.817 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.7. 2023-10-09T10:53:13.819 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:53:13.856 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3535893568' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T10:53:13.856 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T10:53:13.857 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[200197]: failed to encode map e265 with expected crc 2023-10-09T10:53:13.857 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T10:53:13.857 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[200197]: osdmap e265: 8 total, 8 up, 6 in 2023-10-09T10:53:13.857 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[200197]: failed to encode map e265 with expected crc 2023-10-09T10:53:13.857 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[200197]: failed to encode map e265 with expected crc 2023-10-09T10:53:13.857 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[200197]: failed to encode map e265 with expected crc 2023-10-09T10:53:13.857 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[200197]: failed to encode map e265 with expected crc 2023-10-09T10:53:13.857 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/247068762' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:53:13.857 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4121616918' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T10:53:13.857 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T10:53:13.857 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[200197]: pgmap v626: 119 pgs: 5 creating+peering, 11 unknown, 103 active+clean; 550 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 26 KiB/s, 0 objects/s recovering 2023-10-09T10:53:13.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3535893568' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T10:53:13.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T10:53:13.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[195507]: failed to encode map e265 with expected crc 2023-10-09T10:53:13.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T10:53:13.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[195507]: osdmap e265: 8 total, 8 up, 6 in 2023-10-09T10:53:13.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[195507]: failed to encode map e265 with expected crc 2023-10-09T10:53:13.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[195507]: failed to encode map e265 with expected crc 2023-10-09T10:53:13.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[195507]: failed to encode map e265 with expected crc 2023-10-09T10:53:13.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[195507]: failed to encode map e265 with expected crc 2023-10-09T10:53:13.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/247068762' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:53:13.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4121616918' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T10:53:13.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T10:53:13.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:13 smithi125 ceph-mon[195507]: pgmap v626: 119 pgs: 5 creating+peering, 11 unknown, 103 active+clean; 550 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 26 KiB/s, 0 objects/s recovering 2023-10-09T10:53:13.866 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:13.993 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:13.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:13.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:14.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:53:14.128 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:13 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3535893568' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T10:53:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:13 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T10:53:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:13 smithi155 ceph-mon[184854]: failed to encode map e265 with expected crc 2023-10-09T10:53:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:13 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T10:53:14.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:13 smithi155 ceph-mon[184854]: osdmap e265: 8 total, 8 up, 6 in 2023-10-09T10:53:14.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:13 smithi155 ceph-mon[184854]: failed to encode map e265 with expected crc 2023-10-09T10:53:14.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:13 smithi155 ceph-mon[184854]: failed to encode map e265 with expected crc 2023-10-09T10:53:14.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:13 smithi155 ceph-mon[184854]: failed to encode map e265 with expected crc 2023-10-09T10:53:14.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:13 smithi155 ceph-mon[184854]: failed to encode map e265 with expected crc 2023-10-09T10:53:14.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:13 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/247068762' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:53:14.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:13 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4121616918' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T10:53:14.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:13 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T10:53:14.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:13 smithi155 ceph-mon[184854]: pgmap v626: 119 pgs: 5 creating+peering, 11 unknown, 103 active+clean; 550 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 26 KiB/s, 0 objects/s recovering 2023-10-09T10:53:14.225 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:53:14.226 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:53:14.226 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:53:14.226 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:53:14.226 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:53:14.448 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:53:14.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:14.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:53:14.551 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:14.582 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:14.601 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:14.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:14.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:14.706 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:14.732 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:53:14.734 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:14.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:53:14.933 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:53:14.934 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[195507]: failed to encode map e266 with expected crc 2023-10-09T10:53:14.934 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/247068762' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:53:14.934 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T10:53:14.934 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[195507]: osdmap e266: 8 total, 8 up, 7 in 2023-10-09T10:53:14.934 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[195507]: failed to encode map e266 with expected crc 2023-10-09T10:53:14.934 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[195507]: failed to encode map e266 with expected crc 2023-10-09T10:53:14.934 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[195507]: failed to encode map e266 with expected crc 2023-10-09T10:53:14.934 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:53:14.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[200197]: failed to encode map e266 with expected crc 2023-10-09T10:53:14.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/247068762' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:53:14.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T10:53:14.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[200197]: osdmap e266: 8 total, 8 up, 7 in 2023-10-09T10:53:14.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[200197]: failed to encode map e266 with expected crc 2023-10-09T10:53:14.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[200197]: failed to encode map e266 with expected crc 2023-10-09T10:53:14.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:14 smithi125 ceph-mon[200197]: failed to encode map e266 with expected crc 2023-10-09T10:53:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:14 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:53:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:14 smithi155 ceph-mon[184854]: failed to encode map e266 with expected crc 2023-10-09T10:53:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:14 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/247068762' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:53:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:14 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T10:53:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:14 smithi155 ceph-mon[184854]: osdmap e266: 8 total, 8 up, 7 in 2023-10-09T10:53:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:14 smithi155 ceph-mon[184854]: failed to encode map e266 with expected crc 2023-10-09T10:53:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:14 smithi155 ceph-mon[184854]: failed to encode map e266 with expected crc 2023-10-09T10:53:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:14 smithi155 ceph-mon[184854]: failed to encode map e266 with expected crc 2023-10-09T10:53:15.064 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-09T10:53:15.261 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:15.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:15.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:15.349 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:15.365 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:53:15.366 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:15.396 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:15.466 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:53:15.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:15.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:53:15.570 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:53:15.598 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:15.616 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:53:15.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:15.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:53:15.720 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:53:15.750 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:15.870 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:53:15.872 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: osdmap e267: 8 total, 8 up, 7 in 2023-10-09T10:53:15.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:53:15.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:53:15.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: pgmap v629: 119 pgs: 4 remapped+peering, 5 creating+peering, 7 unknown, 103 active+clean; 550 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 14 B/s, 0 objects/s recovering 2023-10-09T10:53:15.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:15.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:53:15.976 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:15.979 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.979 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: osdmap e267: 8 total, 8 up, 7 in 2023-10-09T10:53:15.979 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.979 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.979 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.979 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.979 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.980 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.980 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:53:15.980 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.980 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:53:15.980 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.980 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.980 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: pgmap v629: 119 pgs: 4 remapped+peering, 5 creating+peering, 7 unknown, 103 active+clean; 550 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 14 B/s, 0 objects/s recovering 2023-10-09T10:53:15.980 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.980 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.980 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.981 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.981 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: osdmap e267: 8 total, 8 up, 7 in 2023-10-09T10:53:15.981 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.981 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.981 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.981 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.981 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.981 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: failed to encode map e267 with expected crc 2023-10-09T10:53:15.981 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:53:15.981 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.981 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:53:15.982 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.982 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.982 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: pgmap v629: 119 pgs: 4 remapped+peering, 5 creating+peering, 7 unknown, 103 active+clean; 550 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 14 B/s, 0 objects/s recovering 2023-10-09T10:53:15.982 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.982 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:15.982 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:16.003 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:16.071 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:16.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:16.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:16.175 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:53:16.205 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:16.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:53:16.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:16.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:53:16.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:53:16.456 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:16.479 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:53:16.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:16.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:53:16.582 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:16.610 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:16.630 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:16.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:16.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:16.679 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:53:16.734 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:53:16.764 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:16.840 INFO:teuthology.orchestra.run.smithi125.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-10-09T10:53:16.924 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:53:16.924 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:53:16.924 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:53:16.924 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:53:16.924 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.924 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42934 6.11 6:8a3972ec:::benchmark_data_smithi125_244951_object42933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:16.924 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.549876+0000", 2023-10-09T10:53:16.924 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.371190829, 2023-10-09T10:53:16.924 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.229929598, 2023-10-09T10:53:16.924 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:16.925 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:16.925 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:16.925 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:16.925 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:16.925 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42934 2023-10-09T10:53:16.925 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.925 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:16.925 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.925 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:16.925 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549876+0000", 2023-10-09T10:53:16.925 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.925 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.926 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.926 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:16.926 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549876+0000", 2023-10-09T10:53:16.926 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.926 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.926 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.926 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:16.926 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549874+0000", 2023-10-09T10:53:16.926 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:53:16.926 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.926 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.926 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:16.927 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549907+0000", 2023-10-09T10:53:16.927 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2579000000000001e-05 2023-10-09T10:53:16.927 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.927 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.927 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:16.927 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549922+0000", 2023-10-09T10:53:16.927 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5420000000000001e-05 2023-10-09T10:53:16.927 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.927 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.927 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:16.927 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549934+0000", 2023-10-09T10:53:16.927 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1724e-05 2023-10-09T10:53:16.927 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.928 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.928 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.928 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550289+0000", 2023-10-09T10:53:16.928 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035548900000000001 2023-10-09T10:53:16.928 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.928 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.928 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op must wait for map", 2023-10-09T10:53:16.928 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550297+0000", 2023-10-09T10:53:16.928 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8869999999999992e-06 2023-10-09T10:53:16.928 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.928 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.928 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.929 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551339+0000", 2023-10-09T10:53:16.929 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010423069999999999 2023-10-09T10:53:16.929 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.929 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.929 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:16.929 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551349+0000", 2023-10-09T10:53:16.929 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8060000000000006e-06 2023-10-09T10:53:16.929 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.929 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.929 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.929 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.776033+0000", 2023-10-09T10:53:16.929 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.224684135 2023-10-09T10:53:16.929 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.930 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.930 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:16.930 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.776159+0000", 2023-10-09T10:53:16.930 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012583900000000001 2023-10-09T10:53:16.930 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.930 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.930 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:16.930 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.776290+0000", 2023-10-09T10:53:16.930 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000130705 2023-10-09T10:53:16.930 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.931 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.931 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.931 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778810+0000", 2023-10-09T10:53:16.931 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025205549999999998 2023-10-09T10:53:16.931 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.931 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.931 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.931 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778863+0000", 2023-10-09T10:53:16.931 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2821000000000001e-05 2023-10-09T10:53:16.931 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.931 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.931 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:16.932 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778971+0000", 2023-10-09T10:53:16.932 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108244 2023-10-09T10:53:16.932 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.932 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.932 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.932 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779012+0000", 2023-10-09T10:53:16.932 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0612999999999999e-05 2023-10-09T10:53:16.932 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.932 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.932 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:16.932 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779058+0000", 2023-10-09T10:53:16.932 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5948999999999997e-05 2023-10-09T10:53:16.932 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.933 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.933 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:16.933 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779805+0000", 2023-10-09T10:53:16.933 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00074730900000000004 2023-10-09T10:53:16.933 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.933 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:16.933 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.933 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.933 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.933 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42936 6.11 6:8ce9d1a1:::benchmark_data_smithi125_244951_object42935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:16.933 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.549950+0000", 2023-10-09T10:53:16.933 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.37111681599998, 2023-10-09T10:53:16.934 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.235619359, 2023-10-09T10:53:16.934 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:16.934 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:16.934 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:16.934 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:16.934 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:16.934 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42936 2023-10-09T10:53:16.934 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.934 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:16.934 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.934 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:16.934 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549950+0000", 2023-10-09T10:53:16.935 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.935 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.935 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.935 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:16.935 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549950+0000", 2023-10-09T10:53:16.935 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.935 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.935 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.935 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:16.935 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549949+0000", 2023-10-09T10:53:16.935 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:53:16.935 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.935 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.936 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:16.936 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549976+0000", 2023-10-09T10:53:16.937 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7223999999999999e-05 2023-10-09T10:53:16.937 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.937 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.937 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:16.937 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549987+0000", 2023-10-09T10:53:16.937 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0375e-05 2023-10-09T10:53:16.938 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.938 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.938 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:16.938 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.549996+0000", 2023-10-09T10:53:16.938 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4010000000000003e-06 2023-10-09T10:53:16.938 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.938 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.938 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.938 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550347+0000", 2023-10-09T10:53:16.938 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035085200000000002 2023-10-09T10:53:16.938 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.938 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.939 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:53:16.939 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550352+0000", 2023-10-09T10:53:16.939 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2079999999999999e-06 2023-10-09T10:53:16.939 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.939 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.939 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.939 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551460+0000", 2023-10-09T10:53:16.939 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001107762 2023-10-09T10:53:16.939 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.939 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.939 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:16.939 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551467+0000", 2023-10-09T10:53:16.940 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0350000000000004e-06 2023-10-09T10:53:16.940 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.940 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.940 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.940 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.777841+0000", 2023-10-09T10:53:16.940 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2263741750000001 2023-10-09T10:53:16.940 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.940 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.940 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:16.940 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778003+0000", 2023-10-09T10:53:16.940 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016172699999999999 2023-10-09T10:53:16.940 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.941 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.941 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:16.941 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.778128+0000", 2023-10-09T10:53:16.941 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012541000000000001 2023-10-09T10:53:16.941 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.941 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.941 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.941 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780770+0000", 2023-10-09T10:53:16.941 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026419500000000001 2023-10-09T10:53:16.941 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.941 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.941 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.942 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780795+0000", 2023-10-09T10:53:16.942 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4363e-05 2023-10-09T10:53:16.942 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.942 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.942 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.942 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784699+0000", 2023-10-09T10:53:16.942 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039046739999999999 2023-10-09T10:53:16.942 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.942 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.942 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:16.942 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784714+0000", 2023-10-09T10:53:16.942 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4956999999999999e-05 2023-10-09T10:53:16.943 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.943 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.943 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:16.943 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784774+0000", 2023-10-09T10:53:16.943 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9994999999999997e-05 2023-10-09T10:53:16.943 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.943 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.943 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:16.943 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785569+0000", 2023-10-09T10:53:16.943 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00079485999999999999 2023-10-09T10:53:16.943 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.943 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:16.943 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.944 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.944 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.944 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42950 6.11 6:8b5bb41c:::benchmark_data_smithi125_244951_object42949:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:16.944 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:53:16.944 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.37106442100003, 2023-10-09T10:53:16.944 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2390020909999999, 2023-10-09T10:53:16.944 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:16.944 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:16.944 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:16.944 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:16.944 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:16.945 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42950 2023-10-09T10:53:16.945 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.945 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:16.945 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.945 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:16.945 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:53:16.945 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.945 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.945 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.945 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:16.945 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:53:16.946 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.946 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.946 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.946 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:16.946 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550002+0000", 2023-10-09T10:53:16.946 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:53:16.946 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.946 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.946 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:16.946 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550021+0000", 2023-10-09T10:53:16.946 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8941999999999999e-05 2023-10-09T10:53:16.946 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.946 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.947 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:16.947 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550034+0000", 2023-10-09T10:53:16.947 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3267e-05 2023-10-09T10:53:16.947 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.947 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.947 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:16.947 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550040+0000", 2023-10-09T10:53:16.947 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5799999999999997e-06 2023-10-09T10:53:16.947 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.947 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.947 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.947 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550377+0000", 2023-10-09T10:53:16.947 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00033659800000000002 2023-10-09T10:53:16.948 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.948 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.948 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:53:16.948 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550382+0000", 2023-10-09T10:53:16.948 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6589999999999998e-06 2023-10-09T10:53:16.948 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.948 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.948 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.948 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551483+0000", 2023-10-09T10:53:16.948 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011012210000000001 2023-10-09T10:53:16.948 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.948 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.948 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:16.949 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551489+0000", 2023-10-09T10:53:16.949 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4910000000000002e-06 2023-10-09T10:53:16.949 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.949 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.949 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.949 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779851+0000", 2023-10-09T10:53:16.949 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.22836144 2023-10-09T10:53:16.949 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.949 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.949 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:16.949 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.779959+0000", 2023-10-09T10:53:16.949 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108546 2023-10-09T10:53:16.949 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.950 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.950 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:16.950 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780061+0000", 2023-10-09T10:53:16.950 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010196600000000001 2023-10-09T10:53:16.950 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.950 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.950 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.950 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782707+0000", 2023-10-09T10:53:16.950 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002646074 2023-10-09T10:53:16.950 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.950 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.950 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:16.950 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785603+0000", 2023-10-09T10:53:16.951 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028958410000000001 2023-10-09T10:53:16.951 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.951 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.951 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.951 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785626+0000", 2023-10-09T10:53:16.951 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2371e-05 2023-10-09T10:53:16.951 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.951 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.951 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.951 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.787983+0000", 2023-10-09T10:53:16.951 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023577619999999998 2023-10-09T10:53:16.952 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.952 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.952 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:16.952 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.788053+0000", 2023-10-09T10:53:16.952 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9797999999999998e-05 2023-10-09T10:53:16.952 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.952 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.952 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:16.952 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789004+0000", 2023-10-09T10:53:16.952 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00095101500000000004 2023-10-09T10:53:16.952 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.953 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:16.953 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.953 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.953 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.953 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42962 6.11 6:8af38cb4:::benchmark_data_smithi125_244951_object42961:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:16.953 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550048+0000", 2023-10-09T10:53:16.953 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.371018892, 2023-10-09T10:53:16.953 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2398821879999999, 2023-10-09T10:53:16.953 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:16.953 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:16.953 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:16.954 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:16.954 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:16.954 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42962 2023-10-09T10:53:16.954 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.954 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:16.954 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.954 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:16.954 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550048+0000", 2023-10-09T10:53:16.954 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.954 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.954 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.955 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:16.955 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550048+0000", 2023-10-09T10:53:16.955 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.955 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.955 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.955 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:16.955 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550047+0000", 2023-10-09T10:53:16.955 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:53:16.955 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.955 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.955 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:16.955 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550067+0000", 2023-10-09T10:53:16.956 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0228999999999999e-05 2023-10-09T10:53:16.956 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.956 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.956 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:16.956 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550075+0000", 2023-10-09T10:53:16.956 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0979999999999996e-06 2023-10-09T10:53:16.956 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.956 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.956 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:16.956 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550080+0000", 2023-10-09T10:53:16.956 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7780000000000004e-06 2023-10-09T10:53:16.957 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.957 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.957 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.957 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550404+0000", 2023-10-09T10:53:16.957 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000323648 2023-10-09T10:53:16.957 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.957 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.957 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:53:16.957 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550409+0000", 2023-10-09T10:53:16.957 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6630000000000001e-06 2023-10-09T10:53:16.957 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.957 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.957 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.958 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551523+0000", 2023-10-09T10:53:16.958 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001113953 2023-10-09T10:53:16.958 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.958 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.958 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:16.958 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551531+0000", 2023-10-09T10:53:16.958 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5860000000000005e-06 2023-10-09T10:53:16.958 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.958 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.958 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.958 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780812+0000", 2023-10-09T10:53:16.958 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2292809120000001 2023-10-09T10:53:16.958 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.959 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.959 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:16.959 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.780947+0000", 2023-10-09T10:53:16.959 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000135265 2023-10-09T10:53:16.959 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.959 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.959 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:16.959 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.781098+0000", 2023-10-09T10:53:16.959 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015082499999999999 2023-10-09T10:53:16.959 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.959 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.959 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.959 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782749+0000", 2023-10-09T10:53:16.960 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001651057 2023-10-09T10:53:16.960 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.960 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.960 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:16.960 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785607+0000", 2023-10-09T10:53:16.960 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028579569999999999 2023-10-09T10:53:16.960 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.960 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.960 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.960 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785661+0000", 2023-10-09T10:53:16.960 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4151999999999999e-05 2023-10-09T10:53:16.960 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.960 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.961 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.961 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789184+0000", 2023-10-09T10:53:16.961 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035224309999999999 2023-10-09T10:53:16.961 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.961 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.961 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:16.961 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789241+0000", 2023-10-09T10:53:16.961 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6962000000000001e-05 2023-10-09T10:53:16.961 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.961 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.961 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:16.961 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789930+0000", 2023-10-09T10:53:16.961 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000689148 2023-10-09T10:53:16.962 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.962 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:16.962 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.962 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.962 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.962 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43001 6.11 6:8f0a81da:::benchmark_data_smithi125_244951_object43000:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:16.962 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:53:16.962 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.37098118300003, 2023-10-09T10:53:16.962 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.240501734, 2023-10-09T10:53:16.962 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:16.962 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:16.962 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:16.962 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:16.963 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:16.963 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43001 2023-10-09T10:53:16.963 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.963 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:16.963 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.963 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:16.963 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:53:16.963 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.963 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.963 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.963 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:16.963 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:53:16.964 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.964 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.964 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.964 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:16.964 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550085+0000", 2023-10-09T10:53:16.964 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:53:16.964 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.964 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.964 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:16.964 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550104+0000", 2023-10-09T10:53:16.964 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.912e-05 2023-10-09T10:53:16.964 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.964 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.965 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:16.965 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550110+0000", 2023-10-09T10:53:16.965 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3779999999999998e-06 2023-10-09T10:53:16.965 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.965 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.965 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:16.965 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550116+0000", 2023-10-09T10:53:16.965 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.609e-06 2023-10-09T10:53:16.965 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.965 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.965 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.965 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550429+0000", 2023-10-09T10:53:16.965 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00031264800000000001 2023-10-09T10:53:16.966 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.966 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.966 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting_for_map not empty", 2023-10-09T10:53:16.966 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550433+0000", 2023-10-09T10:53:16.966 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8420000000000004e-06 2023-10-09T10:53:16.966 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.966 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.966 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.966 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551574+0000", 2023-10-09T10:53:16.966 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011402210000000001 2023-10-09T10:53:16.966 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.966 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.967 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:16.967 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551584+0000", 2023-10-09T10:53:16.967 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9690000000000006e-06 2023-10-09T10:53:16.967 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.967 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.967 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.967 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.781792+0000", 2023-10-09T10:53:16.967 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.230208454 2023-10-09T10:53:16.967 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.967 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.967 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:16.967 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.781903+0000", 2023-10-09T10:53:16.967 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110733 2023-10-09T10:53:16.968 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.968 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.968 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:16.968 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782036+0000", 2023-10-09T10:53:16.968 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000133545 2023-10-09T10:53:16.968 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.968 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.968 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.968 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785598+0000", 2023-10-09T10:53:16.968 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035616100000000002 2023-10-09T10:53:16.968 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.968 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.968 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.969 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785679+0000", 2023-10-09T10:53:16.969 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0624999999999995e-05 2023-10-09T10:53:16.969 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.969 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.969 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:16.969 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785703+0000", 2023-10-09T10:53:16.969 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4150000000000001e-05 2023-10-09T10:53:16.969 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.969 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.969 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.969 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790058+0000", 2023-10-09T10:53:16.969 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043551919999999999 2023-10-09T10:53:16.969 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.970 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.970 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:16.970 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790110+0000", 2023-10-09T10:53:16.970 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2278000000000003e-05 2023-10-09T10:53:16.970 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.970 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.970 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:16.970 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790587+0000", 2023-10-09T10:53:16.970 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047690599999999998 2023-10-09T10:53:16.970 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.970 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:16.970 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.971 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.971 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.971 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43002 6.11 6:8bd791a2:::benchmark_data_smithi125_244951_object43001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:16.971 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550121+0000", 2023-10-09T10:53:16.971 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.37094588899998, 2023-10-09T10:53:16.971 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.241166963, 2023-10-09T10:53:16.971 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:16.971 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:16.971 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:16.971 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:16.971 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:16.971 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43002 2023-10-09T10:53:16.972 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.972 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:16.972 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.972 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:16.972 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550121+0000", 2023-10-09T10:53:16.972 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.972 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.972 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.972 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:16.972 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550121+0000", 2023-10-09T10:53:16.972 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.972 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.972 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.973 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:16.973 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550120+0000", 2023-10-09T10:53:16.973 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:53:16.973 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.973 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.973 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:16.973 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550139+0000", 2023-10-09T10:53:16.973 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8644999999999999e-05 2023-10-09T10:53:16.973 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.973 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.973 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:16.973 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550145+0000", 2023-10-09T10:53:16.974 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5180000000000001e-06 2023-10-09T10:53:16.974 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.974 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.974 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:16.974 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550151+0000", 2023-10-09T10:53:16.974 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0399999999999998e-06 2023-10-09T10:53:16.974 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.974 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.974 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.974 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551616+0000", 2023-10-09T10:53:16.974 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001464449 2023-10-09T10:53:16.974 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.975 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:16.975 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551623+0000", 2023-10-09T10:53:16.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0090000000000001e-06 2023-10-09T10:53:16.975 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.975 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.975 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782780+0000", 2023-10-09T10:53:16.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2311575050000001 2023-10-09T10:53:16.975 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:16.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782892+0000", 2023-10-09T10:53:16.976 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111826 2023-10-09T10:53:16.976 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.976 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:16.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.782998+0000", 2023-10-09T10:53:16.976 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010603399999999999 2023-10-09T10:53:16.976 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.976 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785644+0000", 2023-10-09T10:53:16.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026456489999999999 2023-10-09T10:53:16.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.977 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.977 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785699+0000", 2023-10-09T10:53:16.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4778999999999998e-05 2023-10-09T10:53:16.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.977 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:16.977 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786762+0000", 2023-10-09T10:53:16.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001063172 2023-10-09T10:53:16.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.978 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.978 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790708+0000", 2023-10-09T10:53:16.978 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039466409999999999 2023-10-09T10:53:16.978 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.978 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.978 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:16.978 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790775+0000", 2023-10-09T10:53:16.978 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6155000000000006e-05 2023-10-09T10:53:16.978 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.978 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.978 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:16.978 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791288+0000", 2023-10-09T10:53:16.979 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051300599999999999 2023-10-09T10:53:16.979 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.979 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:16.979 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.979 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.979 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.979 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43015 6.11 6:8d7a9802:::benchmark_data_smithi125_244951_object43014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:16.979 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:53:16.979 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.37091019100001, 2023-10-09T10:53:16.979 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.241685028, 2023-10-09T10:53:16.979 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:16.980 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:16.980 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:16.980 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:16.980 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:16.980 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43015 2023-10-09T10:53:16.980 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.980 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:16.980 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.980 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:16.980 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:53:16.980 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.980 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.980 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.981 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:16.981 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:53:16.981 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.981 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.981 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.981 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:16.981 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550156+0000", 2023-10-09T10:53:16.981 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:53:16.981 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.981 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.981 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:16.981 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550174+0000", 2023-10-09T10:53:16.981 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8289000000000001e-05 2023-10-09T10:53:16.981 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.982 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.982 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:16.982 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550181+0000", 2023-10-09T10:53:16.982 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.024e-06 2023-10-09T10:53:16.982 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.982 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.982 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:16.982 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550187+0000", 2023-10-09T10:53:16.982 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6659999999999996e-06 2023-10-09T10:53:16.982 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.982 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.982 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.982 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551652+0000", 2023-10-09T10:53:16.983 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001465406 2023-10-09T10:53:16.983 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.983 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.983 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:16.983 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551659+0000", 2023-10-09T10:53:16.983 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1130000000000003e-06 2023-10-09T10:53:16.983 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.983 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.983 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.983 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.783679+0000", 2023-10-09T10:53:16.983 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.232019406 2023-10-09T10:53:16.983 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.983 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.984 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:16.984 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.783831+0000", 2023-10-09T10:53:16.984 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000151769 2023-10-09T10:53:16.984 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.984 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.984 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:16.984 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.784019+0000", 2023-10-09T10:53:16.984 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018824400000000001 2023-10-09T10:53:16.984 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.984 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.984 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:16.984 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786769+0000", 2023-10-09T10:53:16.985 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027502529999999998 2023-10-09T10:53:16.985 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.985 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.985 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.985 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789091+0000", 2023-10-09T10:53:16.985 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023215200000000001 2023-10-09T10:53:16.985 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.985 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.985 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.985 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789131+0000", 2023-10-09T10:53:16.985 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0766999999999998e-05 2023-10-09T10:53:16.985 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.985 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.986 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.986 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791345+0000", 2023-10-09T10:53:16.986 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022138349999999999 2023-10-09T10:53:16.986 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.986 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.986 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:16.986 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791404+0000", 2023-10-09T10:53:16.986 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8301000000000003e-05 2023-10-09T10:53:16.986 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.986 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.986 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:16.986 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791841+0000", 2023-10-09T10:53:16.987 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00043785599999999998 2023-10-09T10:53:16.987 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.987 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:16.987 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.987 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.987 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43040 6.11 6:892fed64:::benchmark_data_smithi125_244951_object43039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:16.987 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550192+0000", 2023-10-09T10:53:16.987 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.37087481200001, 2023-10-09T10:53:16.987 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2421168730000001, 2023-10-09T10:53:16.987 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:16.987 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:16.987 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:16.988 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:16.988 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:16.988 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43040 2023-10-09T10:53:16.988 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.988 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:16.988 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:16.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550192+0000", 2023-10-09T10:53:16.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.988 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.988 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:16.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550192+0000", 2023-10-09T10:53:16.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.989 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:16.989 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550191+0000", 2023-10-09T10:53:16.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:53:16.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.989 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:16.989 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550265+0000", 2023-10-09T10:53:16.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3668e-05 2023-10-09T10:53:16.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.989 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:16.990 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550273+0000", 2023-10-09T10:53:16.990 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2230000000000007e-06 2023-10-09T10:53:16.990 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.990 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.990 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:16.990 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550280+0000", 2023-10-09T10:53:16.990 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3729999999999998e-06 2023-10-09T10:53:16.990 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.990 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.990 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.990 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551688+0000", 2023-10-09T10:53:16.990 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001408011 2023-10-09T10:53:16.990 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.991 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.991 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:16.991 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551694+0000", 2023-10-09T10:53:16.991 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8120000000000002e-06 2023-10-09T10:53:16.991 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.991 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.991 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.991 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785718+0000", 2023-10-09T10:53:16.991 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2340238640000001 2023-10-09T10:53:16.991 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.991 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.991 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:16.992 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785825+0000", 2023-10-09T10:53:16.992 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000107161 2023-10-09T10:53:16.992 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.992 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.992 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:16.992 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.785964+0000", 2023-10-09T10:53:16.992 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013889800000000001 2023-10-09T10:53:16.992 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.992 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.992 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:16.992 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789139+0000", 2023-10-09T10:53:16.992 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0031749249999999999 2023-10-09T10:53:16.993 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.993 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.993 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.993 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789979+0000", 2023-10-09T10:53:16.993 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00083957300000000003 2023-10-09T10:53:16.993 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.993 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.993 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.993 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790018+0000", 2023-10-09T10:53:16.993 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8816999999999998e-05 2023-10-09T10:53:16.993 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.993 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.993 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:16.994 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791894+0000", 2023-10-09T10:53:16.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018764160000000001 2023-10-09T10:53:16.994 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.994 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.994 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:16.994 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.791933+0000", 2023-10-09T10:53:16.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8889e-05 2023-10-09T10:53:16.994 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.994 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.994 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:16.994 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792309+0000", 2023-10-09T10:53:16.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00037573100000000003 2023-10-09T10:53:16.995 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.995 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:16.995 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:16.995 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.995 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43116 6.11 6:8866ecf5:::benchmark_data_smithi125_244951_object43115:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:16.995 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550285+0000", 2023-10-09T10:53:16.995 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.37078166499998, 2023-10-09T10:53:16.995 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2425320870000001, 2023-10-09T10:53:16.995 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:16.995 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:16.995 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:16.996 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:16.996 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:16.996 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43116 2023-10-09T10:53:16.996 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.996 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:16.996 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.996 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:16.996 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550285+0000", 2023-10-09T10:53:16.996 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.996 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.996 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.996 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:16.997 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550285+0000", 2023-10-09T10:53:16.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:16.997 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.997 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:16.997 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550284+0000", 2023-10-09T10:53:16.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:53:16.997 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.997 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:16.997 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550303+0000", 2023-10-09T10:53:16.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8817e-05 2023-10-09T10:53:16.998 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.998 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.998 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:16.998 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550310+0000", 2023-10-09T10:53:16.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7889999999999997e-06 2023-10-09T10:53:16.998 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.998 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.998 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:16.998 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550315+0000", 2023-10-09T10:53:16.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4539999999999998e-06 2023-10-09T10:53:16.998 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.999 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551718+0000", 2023-10-09T10:53:16.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001402585 2023-10-09T10:53:16.999 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:16.999 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551725+0000", 2023-10-09T10:53:16.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9319999999999999e-06 2023-10-09T10:53:16.999 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:16.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:16.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:16.999 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786811+0000", 2023-10-09T10:53:16.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.23508555 2023-10-09T10:53:17.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.000 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:17.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.786947+0000", 2023-10-09T10:53:17.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013601499999999999 2023-10-09T10:53:17.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.000 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:17.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.787112+0000", 2023-10-09T10:53:17.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016519000000000001 2023-10-09T10:53:17.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.000 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:17.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.789147+0000", 2023-10-09T10:53:17.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020350899999999998 2023-10-09T10:53:17.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790634+0000", 2023-10-09T10:53:17.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014872209999999999 2023-10-09T10:53:17.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.790670+0000", 2023-10-09T10:53:17.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6395e-05 2023-10-09T10:53:17.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.002 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.002 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792355+0000", 2023-10-09T10:53:17.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016850179999999999 2023-10-09T10:53:17.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.002 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:17.002 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792391+0000", 2023-10-09T10:53:17.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5484000000000001e-05 2023-10-09T10:53:17.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.002 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:17.002 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792817+0000", 2023-10-09T10:53:17.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042605300000000002 2023-10-09T10:53:17.003 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.003 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:17.003 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.003 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.003 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43152 6.11 6:8f5a8f11:::benchmark_data_smithi125_244951_object43151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:17.003 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:53:17.003 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.37074532299999, 2023-10-09T10:53:17.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.254750676, 2023-10-09T10:53:17.003 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:17.003 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:17.004 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:17.004 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:17.004 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:17.004 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43152 2023-10-09T10:53:17.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.004 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:17.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:17.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:53:17.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:17.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:53:17.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:17.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550321+0000", 2023-10-09T10:53:17.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:53:17.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.006 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:17.006 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550339+0000", 2023-10-09T10:53:17.006 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7940000000000001e-05 2023-10-09T10:53:17.006 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.006 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.006 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:17.006 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550346+0000", 2023-10-09T10:53:17.006 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4789999999999999e-06 2023-10-09T10:53:17.006 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.006 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.006 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:17.006 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550353+0000", 2023-10-09T10:53:17.007 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7510000000000002e-06 2023-10-09T10:53:17.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.007 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.007 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551748+0000", 2023-10-09T10:53:17.007 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013946119999999999 2023-10-09T10:53:17.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.007 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:17.007 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551754+0000", 2023-10-09T10:53:17.007 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8460000000000002e-06 2023-10-09T10:53:17.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792855+0000", 2023-10-09T10:53:17.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.241101091 2023-10-09T10:53:17.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:17.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.792993+0000", 2023-10-09T10:53:17.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013794 2023-10-09T10:53:17.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:17.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.793143+0000", 2023-10-09T10:53:17.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014960799999999999 2023-10-09T10:53:17.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797271+0000", 2023-10-09T10:53:17.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041283810000000004 2023-10-09T10:53:17.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:17.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797282+0000", 2023-10-09T10:53:17.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0543999999999999e-05 2023-10-09T10:53:17.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.010 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.010 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.010 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799296+0000", 2023-10-09T10:53:17.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020138999999999999 2023-10-09T10:53:17.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.010 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.010 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.010 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803962+0000", 2023-10-09T10:53:17.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046658840000000004 2023-10-09T10:53:17.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.010 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:17.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.804015+0000", 2023-10-09T10:53:17.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3229999999999997e-05 2023-10-09T10:53:17.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:17.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805072+0000", 2023-10-09T10:53:17.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010569690000000001 2023-10-09T10:53:17.011 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.011 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:17.011 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.012 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43224 6.11 6:8b0bc017:::benchmark_data_smithi125_244951_object43223:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:17.012 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550374+0000", 2023-10-09T10:53:17.012 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.37069293500002, 2023-10-09T10:53:17.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2556577980000001, 2023-10-09T10:53:17.012 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:17.012 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:17.012 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:17.012 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:17.012 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:17.012 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43224 2023-10-09T10:53:17.012 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.012 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:17.012 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.013 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:17.013 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550374+0000", 2023-10-09T10:53:17.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.013 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:17.013 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550374+0000", 2023-10-09T10:53:17.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.013 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:17.013 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550373+0000", 2023-10-09T10:53:17.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:53:17.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:17.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550388+0000", 2023-10-09T10:53:17.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.453e-05 2023-10-09T10:53:17.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:17.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550393+0000", 2023-10-09T10:53:17.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7119999999999999e-06 2023-10-09T10:53:17.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:17.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550397+0000", 2023-10-09T10:53:17.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6789999999999996e-06 2023-10-09T10:53:17.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551777+0000", 2023-10-09T10:53:17.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001380196 2023-10-09T10:53:17.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:17.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551784+0000", 2023-10-09T10:53:17.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0369999999999997e-06 2023-10-09T10:53:17.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.793904+0000", 2023-10-09T10:53:17.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.242119346 2023-10-09T10:53:17.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:17.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.794042+0000", 2023-10-09T10:53:17.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013829700000000001 2023-10-09T10:53:17.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.017 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:17.017 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.794190+0000", 2023-10-09T10:53:17.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014784600000000001 2023-10-09T10:53:17.017 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.017 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:17.017 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797289+0000", 2023-10-09T10:53:17.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003099152 2023-10-09T10:53:17.017 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799339+0000", 2023-10-09T10:53:17.018 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020501059999999999 2023-10-09T10:53:17.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.018 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803853+0000", 2023-10-09T10:53:17.018 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045136179999999996 2023-10-09T10:53:17.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.018 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805211+0000", 2023-10-09T10:53:17.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001358349 2023-10-09T10:53:17.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:17.019 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805266+0000", 2023-10-09T10:53:17.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5297999999999998e-05 2023-10-09T10:53:17.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:17.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806031+0000", 2023-10-09T10:53:17.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00076497100000000003 2023-10-09T10:53:17.020 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.020 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:17.020 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.020 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.020 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43255 6.11 6:88dd5493:::benchmark_data_smithi125_244951_object43254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:17.020 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:53:17.020 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.37066641600001, 2023-10-09T10:53:17.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2563999859999999, 2023-10-09T10:53:17.020 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:17.021 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:17.021 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:17.021 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:17.021 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:17.021 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43255 2023-10-09T10:53:17.021 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.021 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:17.021 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.021 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:17.021 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:53:17.021 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.021 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.021 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:17.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:53:17.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.022 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:17.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550400+0000", 2023-10-09T10:53:17.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:53:17.022 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:17.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550413+0000", 2023-10-09T10:53:17.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3052e-05 2023-10-09T10:53:17.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:17.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550417+0000", 2023-10-09T10:53:17.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5999999999999998e-06 2023-10-09T10:53:17.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:17.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550420+0000", 2023-10-09T10:53:17.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6519999999999999e-06 2023-10-09T10:53:17.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551807+0000", 2023-10-09T10:53:17.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001386998 2023-10-09T10:53:17.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:17.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551814+0000", 2023-10-09T10:53:17.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3960000000000003e-06 2023-10-09T10:53:17.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.794932+0000", 2023-10-09T10:53:17.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2431179880000001 2023-10-09T10:53:17.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:17.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.795097+0000", 2023-10-09T10:53:17.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000165248 2023-10-09T10:53:17.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:17.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.795276+0000", 2023-10-09T10:53:17.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017957699999999999 2023-10-09T10:53:17.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.026 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.026 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799383+0000", 2023-10-09T10:53:17.026 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004106569 2023-10-09T10:53:17.026 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.026 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:17.026 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801446+0000", 2023-10-09T10:53:17.026 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020628370000000001 2023-10-09T10:53:17.026 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.026 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803918+0000", 2023-10-09T10:53:17.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024717239999999998 2023-10-09T10:53:17.027 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.027 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.027 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806076+0000", 2023-10-09T10:53:17.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002158395 2023-10-09T10:53:17.027 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.027 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.027 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:17.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806122+0000", 2023-10-09T10:53:17.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5807999999999999e-05 2023-10-09T10:53:17.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.028 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:17.028 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806800+0000", 2023-10-09T10:53:17.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00067843299999999999 2023-10-09T10:53:17.028 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.028 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:17.028 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.028 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43315 6.11 6:8fb75c47:::benchmark_data_smithi125_244951_object43314:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:17.029 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550436+0000", 2023-10-09T10:53:17.029 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.37063092199998, 2023-10-09T10:53:17.029 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2571070289999999, 2023-10-09T10:53:17.029 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:17.029 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:17.029 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:17.029 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:17.029 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:17.029 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43315 2023-10-09T10:53:17.029 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.029 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:17.029 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.029 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:17.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550436+0000", 2023-10-09T10:53:17.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.030 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:17.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550436+0000", 2023-10-09T10:53:17.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.030 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:17.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550435+0000", 2023-10-09T10:53:17.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:53:17.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.031 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.031 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:17.031 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550453+0000", 2023-10-09T10:53:17.031 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7504999999999999e-05 2023-10-09T10:53:17.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.031 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.031 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:17.031 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550460+0000", 2023-10-09T10:53:17.031 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2470000000000002e-06 2023-10-09T10:53:17.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.031 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.032 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:17.032 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550466+0000", 2023-10-09T10:53:17.032 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8699999999999997e-06 2023-10-09T10:53:17.032 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.032 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.032 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.032 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551836+0000", 2023-10-09T10:53:17.032 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001370664 2023-10-09T10:53:17.032 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.032 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.032 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:17.032 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551843+0000", 2023-10-09T10:53:17.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5320000000000003e-06 2023-10-09T10:53:17.033 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.796128+0000", 2023-10-09T10:53:17.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2442850169999999 2023-10-09T10:53:17.033 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:17.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.796286+0000", 2023-10-09T10:53:17.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015828500000000001 2023-10-09T10:53:17.034 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.034 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.034 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:17.034 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.796448+0000", 2023-10-09T10:53:17.034 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016164200000000001 2023-10-09T10:53:17.034 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.034 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.034 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.034 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801434+0000", 2023-10-09T10:53:17.034 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049856279999999998 2023-10-09T10:53:17.034 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.034 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.035 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:17.035 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801530+0000", 2023-10-09T10:53:17.035 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.645e-05 2023-10-09T10:53:17.035 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.035 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.035 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.035 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.805133+0000", 2023-10-09T10:53:17.035 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036025850000000002 2023-10-09T10:53:17.035 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.035 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.035 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.035 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806935+0000", 2023-10-09T10:53:17.036 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018020410000000001 2023-10-09T10:53:17.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.036 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:17.036 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.807002+0000", 2023-10-09T10:53:17.036 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6883000000000003e-05 2023-10-09T10:53:17.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.036 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:17.036 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.807543+0000", 2023-10-09T10:53:17.036 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000541115 2023-10-09T10:53:17.036 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.037 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:17.037 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.037 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.037 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.037 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43318 6.11 6:8f0da7e4:::benchmark_data_smithi125_244951_object43317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:17.037 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550471+0000", 2023-10-09T10:53:17.037 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.37059604400002, 2023-10-09T10:53:17.037 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2582329400000001, 2023-10-09T10:53:17.037 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:17.037 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:17.037 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:17.037 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:17.037 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:17.038 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43318 2023-10-09T10:53:17.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.038 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:17.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:17.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550471+0000", 2023-10-09T10:53:17.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:17.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550471+0000", 2023-10-09T10:53:17.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.039 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:17.039 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550470+0000", 2023-10-09T10:53:17.039 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:53:17.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.039 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:17.039 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550518+0000", 2023-10-09T10:53:17.039 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8108000000000001e-05 2023-10-09T10:53:17.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.040 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.040 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:17.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550526+0000", 2023-10-09T10:53:17.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0199999999999994e-06 2023-10-09T10:53:17.040 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.040 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.040 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:17.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550532+0000", 2023-10-09T10:53:17.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9109999999999998e-06 2023-10-09T10:53:17.040 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551864+0000", 2023-10-09T10:53:17.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001332317 2023-10-09T10:53:17.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:17.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551870+0000", 2023-10-09T10:53:17.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6489999999999996e-06 2023-10-09T10:53:17.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.042 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.042 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797329+0000", 2023-10-09T10:53:17.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.245458701 2023-10-09T10:53:17.042 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.042 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.042 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:17.042 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797476+0000", 2023-10-09T10:53:17.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014769199999999999 2023-10-09T10:53:17.042 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.042 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.043 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:17.043 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.797644+0000", 2023-10-09T10:53:17.043 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000167481 2023-10-09T10:53:17.043 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.043 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.043 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.043 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801478+0000", 2023-10-09T10:53:17.043 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038341780000000002 2023-10-09T10:53:17.043 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.043 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.043 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:17.044 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801537+0000", 2023-10-09T10:53:17.044 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8879000000000003e-05 2023-10-09T10:53:17.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.044 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.806859+0000", 2023-10-09T10:53:17.044 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0053216640000000003 2023-10-09T10:53:17.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.807979+0000", 2023-10-09T10:53:17.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011203770000000001 2023-10-09T10:53:17.045 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.045 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:17.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808056+0000", 2023-10-09T10:53:17.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6568000000000003e-05 2023-10-09T10:53:17.045 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.045 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:17.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808703+0000", 2023-10-09T10:53:17.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00064785600000000004 2023-10-09T10:53:17.046 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.046 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:17.046 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.046 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.046 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.046 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43334 6.11 6:8b093292:::benchmark_data_smithi125_244951_object43333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:17.046 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:53:17.046 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.37052837499999, 2023-10-09T10:53:17.046 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.266648716, 2023-10-09T10:53:17.046 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:17.046 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:17.047 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:17.047 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:17.047 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:17.047 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43334 2023-10-09T10:53:17.047 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.047 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:17.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.047 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:17.047 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:53:17.047 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.047 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.047 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:17.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:53:17.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.048 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:17.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550538+0000", 2023-10-09T10:53:17.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:53:17.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.048 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:17.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550579+0000", 2023-10-09T10:53:17.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1733000000000002e-05 2023-10-09T10:53:17.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:17.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550587+0000", 2023-10-09T10:53:17.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6260000000000003e-06 2023-10-09T10:53:17.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:17.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.550592+0000", 2023-10-09T10:53:17.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4160000000000003e-06 2023-10-09T10:53:17.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551890+0000", 2023-10-09T10:53:17.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001297702 2023-10-09T10:53:17.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:17.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.551896+0000", 2023-10-09T10:53:17.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4559999999999999e-06 2023-10-09T10:53:17.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.798375+0000", 2023-10-09T10:53:17.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2464794800000001 2023-10-09T10:53:17.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:17.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.798514+0000", 2023-10-09T10:53:17.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000139159 2023-10-09T10:53:17.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:17.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.798661+0000", 2023-10-09T10:53:17.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000146312 2023-10-09T10:53:17.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801519+0000", 2023-10-09T10:53:17.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028583720000000001 2023-10-09T10:53:17.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:17.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803865+0000", 2023-10-09T10:53:17.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002345752 2023-10-09T10:53:17.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808789+0000", 2023-10-09T10:53:17.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049239790000000002 2023-10-09T10:53:17.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.816450+0000", 2023-10-09T10:53:17.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076614550000000002 2023-10-09T10:53:17.053 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.053 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:17.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.816501+0000", 2023-10-09T10:53:17.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1298000000000003e-05 2023-10-09T10:53:17.053 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.053 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:17.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817187+0000", 2023-10-09T10:53:17.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00068546500000000001 2023-10-09T10:53:17.054 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.054 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:17.054 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.054 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.054 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.054 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:43385 6.11 6:8e6a343d:::benchmark_data_smithi125_244951_object43384:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2023-10-09T10:53:17.054 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:57.636508+0000", 2023-10-09T10:53:17.054 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 499.28455846700001, 2023-10-09T10:53:17.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1811192319999999, 2023-10-09T10:53:17.054 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:17.055 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:53:17.055 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:17.055 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:17.055 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:17.055 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 43385 2023-10-09T10:53:17.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.055 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:17.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:17.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636508+0000", 2023-10-09T10:53:17.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.056 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:17.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636508+0000", 2023-10-09T10:53:17.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.056 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:17.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636507+0000", 2023-10-09T10:53:17.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:53:17.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.056 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:17.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636527+0000", 2023-10-09T10:53:17.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9748000000000001e-05 2023-10-09T10:53:17.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:17.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636539+0000", 2023-10-09T10:53:17.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2197e-05 2023-10-09T10:53:17.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:17.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636545+0000", 2023-10-09T10:53:17.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1999999999999999e-06 2023-10-09T10:53:17.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636582+0000", 2023-10-09T10:53:17.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6791000000000001e-05 2023-10-09T10:53:17.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:17.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.636590+0000", 2023-10-09T10:53:17.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4309999999999994e-06 2023-10-09T10:53:17.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799411+0000", 2023-10-09T10:53:17.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1628212659999999 2023-10-09T10:53:17.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.059 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.059 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:17.059 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799550+0000", 2023-10-09T10:53:17.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013822300000000001 2023-10-09T10:53:17.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.059 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.059 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T10:53:17.059 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.799660+0000", 2023-10-09T10:53:17.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110701 2023-10-09T10:53:17.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.059 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.060 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.801589+0000", 2023-10-09T10:53:17.060 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019285470000000001 2023-10-09T10:53:17.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:17.060 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.803871+0000", 2023-10-09T10:53:17.060 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022818180000000001 2023-10-09T10:53:17.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.060 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.808835+0000", 2023-10-09T10:53:17.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049646200000000003 2023-10-09T10:53:17.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817231+0000", 2023-10-09T10:53:17.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083956959999999994 2023-10-09T10:53:17.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:53:17.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817263+0000", 2023-10-09T10:53:17.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1899000000000001e-05 2023-10-09T10:53:17.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:17.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:58.817627+0000", 2023-10-09T10:53:17.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00036434500000000002 2023-10-09T10:53:17.062 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.062 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:17.062 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.062 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:106414 6.11 6:8c9e6868:::benchmark_data_smithi125_244951_object106413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:53:17.062 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:46:02.227135+0000", 2023-10-09T10:53:17.062 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 434.69393176400001, 2023-10-09T10:53:17.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.37275067699999997, 2023-10-09T10:53:17.063 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:17.063 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:53:17.063 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:17.063 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:53:17.063 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:53:17.063 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 106414 2023-10-09T10:53:17.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.063 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:17.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:17.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227135+0000", 2023-10-09T10:53:17.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.064 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:17.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227135+0000", 2023-10-09T10:53:17.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.064 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:17.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227134+0000", 2023-10-09T10:53:17.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:53:17.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:17.065 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227156+0000", 2023-10-09T10:53:17.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2059999999999999e-05 2023-10-09T10:53:17.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:17.065 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227163+0000", 2023-10-09T10:53:17.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4169999999999997e-06 2023-10-09T10:53:17.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:17.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.227168+0000", 2023-10-09T10:53:17.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9509999999999997e-06 2023-10-09T10:53:17.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.229108+0000", 2023-10-09T10:53:17.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001940396 2023-10-09T10:53:17.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:53:17.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.229279+0000", 2023-10-09T10:53:17.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017114100000000001 2023-10-09T10:53:17.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.067 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5", 2023-10-09T10:53:17.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.229487+0000", 2023-10-09T10:53:17.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000207977 2023-10-09T10:53:17.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.067 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:53:17.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.233575+0000", 2023-10-09T10:53:17.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040875140000000004 2023-10-09T10:53:17.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.068 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.240998+0000", 2023-10-09T10:53:17.068 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074230249999999998 2023-10-09T10:53:17.068 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:53:17.068 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.245203+0000", 2023-10-09T10:53:17.068 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042052039999999997 2023-10-09T10:53:17.068 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.068 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274084+0000", 2023-10-09T10:53:17.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.028880626999999999 2023-10-09T10:53:17.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.069 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:17.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.274097+0000", 2023-10-09T10:53:17.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.344e-05 2023-10-09T10:53:17.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.069 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599869+0000", 2023-10-09T10:53:17.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32577187699999999 2023-10-09T10:53:17.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.070 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:17.070 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:46:02.599885+0000", 2023-10-09T10:53:17.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6515e-05 2023-10-09T10:53:17.070 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.070 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:17.070 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.070 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.070 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152595 7.d 7.864309dd (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T10:53:17.070 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.372988+0000", 2023-10-09T10:53:17.070 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 67.548078507, 2023-10-09T10:53:17.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98918339399999999, 2023-10-09T10:53:17.071 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:17.071 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:53:17.071 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:17.071 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:53:17.071 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:53:17.071 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152595 2023-10-09T10:53:17.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.071 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:17.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.071 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:17.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.372988+0000", 2023-10-09T10:53:17.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:17.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.372988+0000", 2023-10-09T10:53:17.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:17.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.372986+0000", 2023-10-09T10:53:17.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:53:17.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:17.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.372996+0000", 2023-10-09T10:53:17.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8579999999999995e-06 2023-10-09T10:53:17.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:17.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.373013+0000", 2023-10-09T10:53:17.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6504999999999999e-05 2023-10-09T10:53:17.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:17.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.373024+0000", 2023-10-09T10:53:17.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1617000000000001e-05 2023-10-09T10:53:17.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.373661+0000", 2023-10-09T10:53:17.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00063733599999999996 2023-10-09T10:53:17.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:17.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.373673+0000", 2023-10-09T10:53:17.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1973e-05 2023-10-09T10:53:17.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.362159+0000", 2023-10-09T10:53:17.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98848516099999995 2023-10-09T10:53:17.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:17.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.362171+0000", 2023-10-09T10:53:17.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2843000000000001e-05 2023-10-09T10:53:17.076 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.076 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:17.076 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.076 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152608 7.b 7.db54bf0b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T10:53:17.076 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.399603+0000", 2023-10-09T10:53:17.076 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 67.521463733999994, 2023-10-09T10:53:17.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96599031099999999, 2023-10-09T10:53:17.076 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:17.076 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:53:17.076 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:17.077 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:53:17.077 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:53:17.077 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152608 2023-10-09T10:53:17.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.077 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:17.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:17.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.399603+0000", 2023-10-09T10:53:17.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:17.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.399603+0000", 2023-10-09T10:53:17.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.078 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:17.078 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.399601+0000", 2023-10-09T10:53:17.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:53:17.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.078 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:17.078 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.399609+0000", 2023-10-09T10:53:17.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4499999999999998e-06 2023-10-09T10:53:17.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:17.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.399620+0000", 2023-10-09T10:53:17.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1747e-05 2023-10-09T10:53:17.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:17.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.399629+0000", 2023-10-09T10:53:17.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8820000000000007e-06 2023-10-09T10:53:17.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.401924+0000", 2023-10-09T10:53:17.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022950850000000001 2023-10-09T10:53:17.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.080 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:17.080 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.401936+0000", 2023-10-09T10:53:17.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1788e-05 2023-10-09T10:53:17.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.080 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.080 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365580+0000", 2023-10-09T10:53:17.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96364352600000003 2023-10-09T10:53:17.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.080 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:17.081 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365593+0000", 2023-10-09T10:53:17.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3399e-05 2023-10-09T10:53:17.081 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.081 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:17.081 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.081 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.081 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152609 7.0 7.ecbeda60 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T10:53:17.081 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.399805+0000", 2023-10-09T10:53:17.081 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 67.521261967000001, 2023-10-09T10:53:17.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95907162000000001, 2023-10-09T10:53:17.081 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:53:17.081 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:53:17.081 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:53:17.082 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:53:17.082 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:53:17.082 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152609 2023-10-09T10:53:17.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.082 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:53:17.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:53:17.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.399805+0000", 2023-10-09T10:53:17.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:53:17.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.399805+0000", 2023-10-09T10:53:17.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:53:17.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.083 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:53:17.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.399804+0000", 2023-10-09T10:53:17.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:53:17.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.083 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:53:17.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.399809+0000", 2023-10-09T10:53:17.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8350000000000003e-06 2023-10-09T10:53:17.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:53:17.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.399817+0000", 2023-10-09T10:53:17.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4489999999999999e-06 2023-10-09T10:53:17.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:53:17.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.399824+0000", 2023-10-09T10:53:17.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.34e-06 2023-10-09T10:53:17.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.399869+0000", 2023-10-09T10:53:17.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4947000000000002e-05 2023-10-09T10:53:17.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:53:17.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.399882+0000", 2023-10-09T10:53:17.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2213e-05 2023-10-09T10:53:17.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:53:17.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.358860+0000", 2023-10-09T10:53:17.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95897841399999995 2023-10-09T10:53:17.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:53:17.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:53:17.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:53:17.086 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.358876+0000", 2023-10-09T10:53:17.086 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6242000000000001e-05 2023-10-09T10:53:17.086 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.086 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:17.086 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.086 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:53:17.086 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:53:17.086 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:53:17.087 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[195507]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.087 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[195507]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.087 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[195507]: osdmap e268: 8 total, 8 up, 7 in 2023-10-09T10:53:17.087 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[195507]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.087 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[195507]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.087 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[195507]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.087 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[195507]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.087 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1962515284' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:53:17.088 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[200197]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.088 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[200197]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.088 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[200197]: osdmap e268: 8 total, 8 up, 7 in 2023-10-09T10:53:17.088 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[200197]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.088 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[200197]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.088 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[200197]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.088 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[200197]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.088 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:16 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1962515284' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:53:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:17.089 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:53:17.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:53:17.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:16 smithi155 ceph-mon[184854]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:16 smithi155 ceph-mon[184854]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:16 smithi155 ceph-mon[184854]: osdmap e268: 8 total, 8 up, 7 in 2023-10-09T10:53:17.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:16 smithi155 ceph-mon[184854]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:16 smithi155 ceph-mon[184854]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:16 smithi155 ceph-mon[184854]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:16 smithi155 ceph-mon[184854]: failed to encode map e268 with expected crc 2023-10-09T10:53:17.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:16 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1962515284' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:53:17.190 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:17.220 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:17.377 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:53:17.395 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:53:17.414 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:53:17.434 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:53:17.439 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:17.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:17.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:17.455 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:53:17.473 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:53:17.492 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:53:17.511 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:53:17.512 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:53:17.512 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:53:17.512 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:53:17.512 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:53:17.512 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:53:17.512 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:53:17.512 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:53:17.512 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:53:17.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:17.570 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:17.890 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:17.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:17.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:17.995 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:18.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1962515284' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:53:18.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[200197]: osdmap e269: 8 total, 8 up, 7 in 2023-10-09T10:53:18.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[200197]: failed to encode map e269 with expected crc 2023-10-09T10:53:18.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[200197]: failed to encode map e269 with expected crc 2023-10-09T10:53:18.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[200197]: failed to encode map e269 with expected crc 2023-10-09T10:53:18.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[200197]: failed to encode map e269 with expected crc 2023-10-09T10:53:18.021 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[200197]: failed to encode map e269 with expected crc 2023-10-09T10:53:18.021 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[200197]: pgmap v632: 119 pgs: 7 active+undersized+remapped, 4 remapped+peering, 4 creating+peering, 1 active+remapped, 103 active+clean; 550 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 65 KiB/s, 2 keys/s, 2 objects/s recovering 2023-10-09T10:53:18.021 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/355376432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:53:18.021 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:18.021 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:18.021 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1962515284' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:53:18.021 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[195507]: osdmap e269: 8 total, 8 up, 7 in 2023-10-09T10:53:18.021 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[195507]: failed to encode map e269 with expected crc 2023-10-09T10:53:18.021 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[195507]: failed to encode map e269 with expected crc 2023-10-09T10:53:18.021 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[195507]: failed to encode map e269 with expected crc 2023-10-09T10:53:18.022 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[195507]: failed to encode map e269 with expected crc 2023-10-09T10:53:18.022 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[195507]: failed to encode map e269 with expected crc 2023-10-09T10:53:18.022 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[195507]: pgmap v632: 119 pgs: 7 active+undersized+remapped, 4 remapped+peering, 4 creating+peering, 1 active+remapped, 103 active+clean; 550 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 65 KiB/s, 2 keys/s, 2 objects/s recovering 2023-10-09T10:53:18.022 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/355376432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:53:18.022 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:18.022 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:17 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:18.022 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:18.141 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:18.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:18.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:18.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1962515284' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:53:18.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:17 smithi155 ceph-mon[184854]: osdmap e269: 8 total, 8 up, 7 in 2023-10-09T10:53:18.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:17 smithi155 ceph-mon[184854]: failed to encode map e269 with expected crc 2023-10-09T10:53:18.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:17 smithi155 ceph-mon[184854]: failed to encode map e269 with expected crc 2023-10-09T10:53:18.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:17 smithi155 ceph-mon[184854]: failed to encode map e269 with expected crc 2023-10-09T10:53:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:17 smithi155 ceph-mon[184854]: failed to encode map e269 with expected crc 2023-10-09T10:53:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:17 smithi155 ceph-mon[184854]: failed to encode map e269 with expected crc 2023-10-09T10:53:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:17 smithi155 ceph-mon[184854]: pgmap v632: 119 pgs: 7 active+undersized+remapped, 4 remapped+peering, 4 creating+peering, 1 active+remapped, 103 active+clean; 550 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 65 KiB/s, 2 keys/s, 2 objects/s recovering 2023-10-09T10:53:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/355376432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:53:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:17 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:18.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:17 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:18.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:18.272 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:18.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:18.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:18.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:18.397 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:53:18.427 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:53:18.427 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:18.445 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:53:18.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:18.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:53:18.452 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:18.550 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:53:18.577 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:18.702 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:53:18.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:18.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:53:18.737 DEBUG:teuthology.orchestra.run.smithi125:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-10-09T10:53:18.748 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T10:53:18.749 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T10:53:18.806 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:18.837 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:18.857 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:18.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:18.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:18.930 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:hints = 1 2023-10-09T10:53:18.930 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T10:53:18.930 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object prefix: benchmark_data_smithi125_329074 2023-10-09T10:53:18.931 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:53:18.931 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 0 0 0 0 0 0 - 0 2023-10-09T10:53:18.962 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:19.005 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:19.024 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:19.032 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:19.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:19.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:19.137 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:53:19.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[195507]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[195507]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[195507]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[195507]: osdmap e270: 8 total, 8 up, 7 in 2023-10-09T10:53:19.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[195507]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[195507]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[195507]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[195507]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[195507]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[195507]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[200197]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[200197]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[200197]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[200197]: osdmap e270: 8 total, 8 up, 7 in 2023-10-09T10:53:19.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[200197]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[200197]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[200197]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[200197]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[200197]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:18 smithi125 ceph-mon[200197]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.185 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:18 smithi155 ceph-mon[184854]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:18 smithi155 ceph-mon[184854]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:18 smithi155 ceph-mon[184854]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:18 smithi155 ceph-mon[184854]: osdmap e270: 8 total, 8 up, 7 in 2023-10-09T10:53:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:18 smithi155 ceph-mon[184854]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:18 smithi155 ceph-mon[184854]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:18 smithi155 ceph-mon[184854]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:18 smithi155 ceph-mon[184854]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:18 smithi155 ceph-mon[184854]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:18 smithi155 ceph-mon[184854]: failed to encode map e270 with expected crc 2023-10-09T10:53:19.314 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:53:19.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:19.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:53:19.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:19.456 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:19.482 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:19.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:19.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:19.586 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:19.623 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:19.649 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:19.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:19.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:19.753 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:19.795 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:19.820 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:19.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:19.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:19.924 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:19.931 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 1 16 3232 3216 200.929 201 0.00704097 0.00495032 2023-10-09T10:53:19.974 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:20.065 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 1, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:53:20.066 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:53:20.066 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T10:53:20.067 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:53:20.110 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:19 smithi125 ceph-mon[195507]: pgmap v634: 119 pgs: 7 active+undersized+remapped, 4 remapped+peering, 1 active+remapped, 107 active+clean; 550 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 58 KiB/s, 2 keys/s, 2 objects/s recovering 2023-10-09T10:53:20.110 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:19 smithi125 ceph-mon[200197]: pgmap v634: 119 pgs: 7 active+undersized+remapped, 4 remapped+peering, 1 active+remapped, 107 active+clean; 550 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 58 KiB/s, 2 keys/s, 2 objects/s recovering 2023-10-09T10:53:20.162 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:53:20.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:19 smithi155 ceph-mon[184854]: pgmap v634: 119 pgs: 7 active+undersized+remapped, 4 remapped+peering, 1 active+remapped, 107 active+clean; 550 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 58 KiB/s, 2 keys/s, 2 objects/s recovering 2023-10-09T10:53:20.464 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:20.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:20.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:20.570 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:20.661 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:20.692 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:20.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:20.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:20.795 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:20.833 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:20.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:20.865 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:53:20.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:20.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:20.896 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:53:20.896 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:53:20.911 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:20.912 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:20.931 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 2 16 5537 5521 172.493 144.062 0.0078028 0.00578639 2023-10-09T10:53:20.966 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:21.026 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:21.217 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:20 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1181552572' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:53:21.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:20 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:53:21.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:20 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1181552572' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:53:21.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:20 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:53:21.218 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:53:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:53:21] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.43.0" 2023-10-09T10:53:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:20 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1181552572' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:53:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:20 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:53:21.931 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 3 16 8675 8659 180.358 196.125 0.00334226 0.00553537 2023-10-09T10:53:22.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:21 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:53:22.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:21 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:53:22.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:21 smithi125 ceph-mon[195507]: osdmap e271: 8 total, 8 up, 7 in 2023-10-09T10:53:22.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:21 smithi125 ceph-mon[195507]: failed to encode map e271 with expected crc 2023-10-09T10:53:22.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:21 smithi125 ceph-mon[195507]: failed to encode map e271 with expected crc 2023-10-09T10:53:22.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:21 smithi125 ceph-mon[195507]: failed to encode map e271 with expected crc 2023-10-09T10:53:22.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:21 smithi125 ceph-mon[195507]: pgmap v636: 119 pgs: 2 active+undersized+remapped, 117 active+clean; 129 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 23 MiB/s wr, 373 op/s; 47 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T10:53:22.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:21 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:53:22.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:21 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:53:22.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:21 smithi125 ceph-mon[200197]: osdmap e271: 8 total, 8 up, 7 in 2023-10-09T10:53:22.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:21 smithi125 ceph-mon[200197]: failed to encode map e271 with expected crc 2023-10-09T10:53:22.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:21 smithi125 ceph-mon[200197]: failed to encode map e271 with expected crc 2023-10-09T10:53:22.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:21 smithi125 ceph-mon[200197]: failed to encode map e271 with expected crc 2023-10-09T10:53:22.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:21 smithi125 ceph-mon[200197]: pgmap v636: 119 pgs: 2 active+undersized+remapped, 117 active+clean; 129 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 23 MiB/s wr, 373 op/s; 47 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T10:53:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:21 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:53:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:21 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:53:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:21 smithi155 ceph-mon[184854]: osdmap e271: 8 total, 8 up, 7 in 2023-10-09T10:53:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:21 smithi155 ceph-mon[184854]: failed to encode map e271 with expected crc 2023-10-09T10:53:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:21 smithi155 ceph-mon[184854]: failed to encode map e271 with expected crc 2023-10-09T10:53:22.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:21 smithi155 ceph-mon[184854]: failed to encode map e271 with expected crc 2023-10-09T10:53:22.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:21 smithi155 ceph-mon[184854]: pgmap v636: 119 pgs: 2 active+undersized+remapped, 117 active+clean; 129 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 23 MiB/s wr, 373 op/s; 47 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-09T10:53:22.932 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 4 16 11707 11691 182.608 189.5 0.00792022 0.00547061 2023-10-09T10:53:23.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:22 smithi125 ceph-mon[195507]: 1.14 scrub starts 2023-10-09T10:53:23.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:22 smithi125 ceph-mon[195507]: 1.14 scrub ok 2023-10-09T10:53:23.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:22 smithi125 ceph-mon[200197]: 1.14 scrub starts 2023-10-09T10:53:23.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:22 smithi125 ceph-mon[200197]: 1.14 scrub ok 2023-10-09T10:53:23.154 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:53:23.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:22 smithi155 ceph-mon[184854]: 1.14 scrub starts 2023-10-09T10:53:23.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:22 smithi155 ceph-mon[184854]: 1.14 scrub ok 2023-10-09T10:53:23.337 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:23.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:23.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:23.447 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:53:23.498 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:23.532 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:53:23.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:23.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:53:23.637 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:53:23.678 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:23.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:53:23.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:23.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:53:23.811 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:23.848 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:23.874 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:23.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:23.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:23.932 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 5 16 14144 14128 176.548 152.312 0.00483167 0.00565497 2023-10-09T10:53:23.948 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:23 smithi125 ceph-mon[195507]: 1.3 scrub starts 2023-10-09T10:53:23.948 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:23 smithi125 ceph-mon[195507]: 1.3 scrub ok 2023-10-09T10:53:23.948 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:23 smithi125 ceph-mon[195507]: 1.15 scrub starts 2023-10-09T10:53:23.948 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:23 smithi125 ceph-mon[195507]: 1.15 scrub ok 2023-10-09T10:53:23.948 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:23 smithi125 ceph-mon[195507]: 1.5 scrub starts 2023-10-09T10:53:23.949 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:23 smithi125 ceph-mon[195507]: 1.5 scrub ok 2023-10-09T10:53:23.949 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:23 smithi125 ceph-mon[195507]: pgmap v637: 119 pgs: 119 active+clean; 468 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T10:53:23.949 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:23 smithi125 ceph-mon[200197]: 1.3 scrub starts 2023-10-09T10:53:23.949 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:23 smithi125 ceph-mon[200197]: 1.3 scrub ok 2023-10-09T10:53:23.949 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:23 smithi125 ceph-mon[200197]: 1.15 scrub starts 2023-10-09T10:53:23.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:23 smithi125 ceph-mon[200197]: 1.15 scrub ok 2023-10-09T10:53:23.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:23 smithi125 ceph-mon[200197]: 1.5 scrub starts 2023-10-09T10:53:23.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:23 smithi125 ceph-mon[200197]: 1.5 scrub ok 2023-10-09T10:53:23.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:23 smithi125 ceph-mon[200197]: pgmap v637: 119 pgs: 119 active+clean; 468 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T10:53:23.979 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:24.024 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:24.051 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:24.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:24.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:24.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:53:24.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:23 smithi155 ceph-mon[184854]: 1.3 scrub starts 2023-10-09T10:53:24.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:23 smithi155 ceph-mon[184854]: 1.3 scrub ok 2023-10-09T10:53:24.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:23 smithi155 ceph-mon[184854]: 1.15 scrub starts 2023-10-09T10:53:24.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:23 smithi155 ceph-mon[184854]: 1.15 scrub ok 2023-10-09T10:53:24.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:23 smithi155 ceph-mon[184854]: 1.5 scrub starts 2023-10-09T10:53:24.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:23 smithi155 ceph-mon[184854]: 1.5 scrub ok 2023-10-09T10:53:24.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:23 smithi155 ceph-mon[184854]: pgmap v637: 119 pgs: 119 active+clean; 468 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T10:53:24.193 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:24.211 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:53:24.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:24.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:53:24.314 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:24.371 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:24.498 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:24.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:24.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:24.639 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:53:24.653 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:53:24.660 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:53:24.660 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:53:24.719 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:24.740 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:53:24.752 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:53:24.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:24.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:53:24.781 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:53:24.813 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:53:24.849 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:53:24.857 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:24.882 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:53:24.901 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:24.914 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:53:24.929 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:24.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:24.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:24.933 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 6 16 16239 16223 168.932 130.938 0.00619621 0.00591077 2023-10-09T10:53:24.961 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:53:24.996 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:53:25.005 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:53:25.005 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:53:25.005 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:53:25.005 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:53:25.006 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:53:25.006 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:53:25.006 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:53:25.006 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:53:25.033 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:53:25.077 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:25.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:24 smithi155 ceph-mon[184854]: 1.b scrub starts 2023-10-09T10:53:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:24 smithi155 ceph-mon[184854]: 1.b scrub ok 2023-10-09T10:53:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:24 smithi155 ceph-mon[184854]: 1.d scrub starts 2023-10-09T10:53:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:24 smithi155 ceph-mon[184854]: 1.d scrub ok 2023-10-09T10:53:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:24 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2181583764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:53:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:24 smithi125 ceph-mon[195507]: 1.b scrub starts 2023-10-09T10:53:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:24 smithi125 ceph-mon[195507]: 1.b scrub ok 2023-10-09T10:53:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:24 smithi125 ceph-mon[195507]: 1.d scrub starts 2023-10-09T10:53:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:24 smithi125 ceph-mon[195507]: 1.d scrub ok 2023-10-09T10:53:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:24 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2181583764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:53:25.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:24 smithi125 ceph-mon[200197]: 1.b scrub starts 2023-10-09T10:53:25.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:24 smithi125 ceph-mon[200197]: 1.b scrub ok 2023-10-09T10:53:25.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:24 smithi125 ceph-mon[200197]: 1.d scrub starts 2023-10-09T10:53:25.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:24 smithi125 ceph-mon[200197]: 1.d scrub ok 2023-10-09T10:53:25.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:24 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2181583764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:53:25.661 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:53:25.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:25.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:53:25.765 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:53:25.816 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:25.844 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:53:25.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:25.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:53:25.933 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 7 16 18550 18534 165.409 144.438 0.0113723 0.00603866 2023-10-09T10:53:25.948 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:25.994 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:26.028 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:26.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:26.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:26.135 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:53:26.187 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:26.220 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:53:26.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:26.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:53:26.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:26.365 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:26.402 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:26.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:26.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:26.506 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:53:26.569 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:26.621 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:26.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:26 smithi125 ceph-mon[195507]: from='client.35439 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:53:26.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:26 smithi125 ceph-mon[195507]: pgmap v638: 119 pgs: 119 active+clean; 672 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T10:53:26.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:26 smithi125 ceph-mon[195507]: 1.c scrub starts 2023-10-09T10:53:26.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:26 smithi125 ceph-mon[195507]: 1.c scrub ok 2023-10-09T10:53:26.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:26 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:26.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:26 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:26.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:26 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:26 smithi125 ceph-mon[200197]: from='client.35439 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:53:26.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:26 smithi125 ceph-mon[200197]: pgmap v638: 119 pgs: 119 active+clean; 672 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T10:53:26.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:26 smithi125 ceph-mon[200197]: 1.c scrub starts 2023-10-09T10:53:26.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:26 smithi125 ceph-mon[200197]: 1.c scrub ok 2023-10-09T10:53:26.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:26 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:26.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:26 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:26.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:26 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:26.933 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 8 16 20863 20847 162.802 144.562 0.00887569 0.00613768 2023-10-09T10:53:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:26 smithi155 ceph-mon[184854]: from='client.35439 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:53:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:26 smithi155 ceph-mon[184854]: pgmap v638: 119 pgs: 119 active+clean; 672 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T10:53:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:26 smithi155 ceph-mon[184854]: 1.c scrub starts 2023-10-09T10:53:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:26 smithi155 ceph-mon[184854]: 1.c scrub ok 2023-10-09T10:53:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:26 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:26 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:26 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:27.571 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:53:27.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:27.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:53:27.677 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:27.720 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:27 smithi125 ceph-mon[195507]: 1.8 scrub starts 2023-10-09T10:53:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:27 smithi125 ceph-mon[195507]: 1.8 scrub ok 2023-10-09T10:53:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:27 smithi125 ceph-mon[195507]: 1.a scrub starts 2023-10-09T10:53:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:27 smithi125 ceph-mon[195507]: 1.a scrub ok 2023-10-09T10:53:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:27 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:53:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:27 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:53:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:27 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:27 smithi125 ceph-mon[200197]: 1.8 scrub starts 2023-10-09T10:53:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:27 smithi125 ceph-mon[200197]: 1.8 scrub ok 2023-10-09T10:53:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:27 smithi125 ceph-mon[200197]: 1.a scrub starts 2023-10-09T10:53:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:27 smithi125 ceph-mon[200197]: 1.a scrub ok 2023-10-09T10:53:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:27 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:53:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:27 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:53:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:27 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:27.934 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 9 16 23203 23187 160.943 146.25 0.00492281 0.00620856 2023-10-09T10:53:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:27 smithi155 ceph-mon[184854]: 1.8 scrub starts 2023-10-09T10:53:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:27 smithi155 ceph-mon[184854]: 1.8 scrub ok 2023-10-09T10:53:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:27 smithi155 ceph-mon[184854]: 1.a scrub starts 2023-10-09T10:53:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:27 smithi155 ceph-mon[184854]: 1.a scrub ok 2023-10-09T10:53:27.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:27 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:53:27.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:27 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:53:27.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:27 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:28.558 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:28 smithi125 ceph-mon[200197]: 1.13 scrub starts 2023-10-09T10:53:28.559 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:28 smithi125 ceph-mon[200197]: 1.13 scrub ok 2023-10-09T10:53:28.559 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:28 smithi125 ceph-mon[200197]: pgmap v639: 119 pgs: 119 active+clean; 1.0 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T10:53:28.559 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:28 smithi125 ceph-mon[200197]: 1.6 scrub starts 2023-10-09T10:53:28.559 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:28 smithi125 ceph-mon[200197]: 1.6 scrub ok 2023-10-09T10:53:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:28 smithi125 ceph-mon[195507]: 1.13 scrub starts 2023-10-09T10:53:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:28 smithi125 ceph-mon[195507]: 1.13 scrub ok 2023-10-09T10:53:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:28 smithi125 ceph-mon[195507]: pgmap v639: 119 pgs: 119 active+clean; 1.0 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T10:53:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:28 smithi125 ceph-mon[195507]: 1.6 scrub starts 2023-10-09T10:53:28.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:28 smithi125 ceph-mon[195507]: 1.6 scrub ok 2023-10-09T10:53:28.935 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 10 16 25580 25564 159.704 148.562 0.00452024 0.00625714 2023-10-09T10:53:28.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:28 smithi155 ceph-mon[184854]: 1.13 scrub starts 2023-10-09T10:53:28.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:28 smithi155 ceph-mon[184854]: 1.13 scrub ok 2023-10-09T10:53:28.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:28 smithi155 ceph-mon[184854]: pgmap v639: 119 pgs: 119 active+clean; 1.0 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T10:53:28.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:28 smithi155 ceph-mon[184854]: 1.6 scrub starts 2023-10-09T10:53:28.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:28 smithi155 ceph-mon[184854]: 1.6 scrub ok 2023-10-09T10:53:29.935 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 11 16 27886 27870 158.287 144.125 0.00804058 0.00631257 2023-10-09T10:53:30.117 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":637,"stamp":"2023-10-09T10:53:23.320480+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":490444523,"num_objects":7497,"num_object_clones":0,"num_object_copies":22491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7497,"num_whiteouts":0,"num_read":324,"num_read_kb":609,"num_write":7695,"num_write_kb":480039,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":1876123,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":12521,"ondisk_log_size":12521,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5034180,"kb_used_data":2680844,"kb_used_omap":44,"kb_used_meta":2353235,"kb_avail":651181884,"statfs":{"total":671965249536,"available":666810249216,"internally_reserved":0,"allocated":2745184256,"data_stored":2736865777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45663,"internal_metadata":2409713057},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":489881600,"num_objects":7475,"num_object_clones":0,"num_object_copies":22425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7475,"num_write_kb":478400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.490983"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":577,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.836982+0000","last_change":"2023-10-09T10:53:17.836709+0000","last_active":"2023-10-09T10:53:17.836982+0000","last_peered":"2023-10-09T10:53:17.836982+0000","last_clean":"2023-10-09T10:53:17.836982+0000","last_became_active":"2023-10-09T10:53:17.836424+0000","last_became_peered":"2023-10-09T10:53:17.836424+0000","last_unstale":"2023-10-09T10:53:17.836982+0000","last_undegraded":"2023-10-09T10:53:17.836982+0000","last_fullsized":"2023-10-09T10:53:17.836982+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":605881,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.828900+0000","last_change":"2023-10-09T10:52:10.406855+0000","last_active":"2023-10-09T10:53:17.828900+0000","last_peered":"2023-10-09T10:53:17.828900+0000","last_clean":"2023-10-09T10:53:17.828900+0000","last_became_active":"2023-10-09T10:52:10.406435+0000","last_became_peered":"2023-10-09T10:52:10.406435+0000","last_unstale":"2023-10-09T10:53:17.828900+0000","last_undegraded":"2023-10-09T10:53:17.828900+0000","last_fullsized":"2023-10-09T10:53:17.828900+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:59:34.082764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":258,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.827561+0000","last_change":"2023-10-09T10:52:10.379409+0000","last_active":"2023-10-09T10:53:16.827561+0000","last_peered":"2023-10-09T10:53:16.827561+0000","last_clean":"2023-10-09T10:53:16.827561+0000","last_became_active":"2023-10-09T10:52:10.379097+0000","last_became_peered":"2023-10-09T10:52:10.379097+0000","last_unstale":"2023-10-09T10:53:16.827561+0000","last_undegraded":"2023-10-09T10:53:16.827561+0000","last_fullsized":"2023-10-09T10:53:16.827561+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:24:31.678460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":638,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.830847+0000","last_change":"2023-10-09T10:52:55.415402+0000","last_active":"2023-10-09T10:53:17.830847+0000","last_peered":"2023-10-09T10:53:17.830847+0000","last_clean":"2023-10-09T10:53:17.830847+0000","last_became_active":"2023-10-09T10:52:55.414995+0000","last_became_peered":"2023-10-09T10:52:55.414995+0000","last_unstale":"2023-10-09T10:53:17.830847+0000","last_undegraded":"2023-10-09T10:53:17.830847+0000","last_fullsized":"2023-10-09T10:53:17.830847+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.892515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.892515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.826913+0000","last_change":"2023-10-09T10:52:10.405482+0000","last_active":"2023-10-09T10:53:17.826913+0000","last_peered":"2023-10-09T10:53:17.826913+0000","last_clean":"2023-10-09T10:53:17.826913+0000","last_became_active":"2023-10-09T10:52:10.404851+0000","last_became_peered":"2023-10-09T10:52:10.404851+0000","last_unstale":"2023-10-09T10:53:17.826913+0000","last_undegraded":"2023-10-09T10:53:17.826913+0000","last_fullsized":"2023-10-09T10:53:17.826913+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:48.412641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":178,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.831435+0000","last_change":"2023-10-09T10:53:14.823051+0000","last_active":"2023-10-09T10:53:17.831435+0000","last_peered":"2023-10-09T10:53:17.831435+0000","last_clean":"2023-10-09T10:53:17.831435+0000","last_became_active":"2023-10-09T10:53:14.822334+0000","last_became_peered":"2023-10-09T10:53:14.822334+0000","last_unstale":"2023-10-09T10:53:17.831435+0000","last_undegraded":"2023-10-09T10:53:17.831435+0000","last_fullsized":"2023-10-09T10:53:17.831435+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:46.573381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":604,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.841825+0000","last_change":"2023-10-09T10:53:17.841135+0000","last_active":"2023-10-09T10:53:17.841825+0000","last_peered":"2023-10-09T10:53:17.841825+0000","last_clean":"2023-10-09T10:53:17.841825+0000","last_became_active":"2023-10-09T10:53:17.840761+0000","last_became_peered":"2023-10-09T10:53:17.840761+0000","last_unstale":"2023-10-09T10:53:17.841825+0000","last_undegraded":"2023-10-09T10:53:17.841825+0000","last_fullsized":"2023-10-09T10:53:17.841825+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":295,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.825118+0000","last_change":"2023-10-09T10:52:10.379917+0000","last_active":"2023-10-09T10:53:16.825118+0000","last_peered":"2023-10-09T10:53:16.825118+0000","last_clean":"2023-10-09T10:53:16.825118+0000","last_became_active":"2023-10-09T10:52:10.379056+0000","last_became_peered":"2023-10-09T10:52:10.379056+0000","last_unstale":"2023-10-09T10:53:16.825118+0000","last_undegraded":"2023-10-09T10:53:16.825118+0000","last_fullsized":"2023-10-09T10:53:16.825118+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:17:34.556823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.827259+0000","last_change":"2023-10-09T10:52:10.372537+0000","last_active":"2023-10-09T10:53:17.827259+0000","last_peered":"2023-10-09T10:53:17.827259+0000","last_clean":"2023-10-09T10:53:17.827259+0000","last_became_active":"2023-10-09T10:52:10.372058+0000","last_became_peered":"2023-10-09T10:52:10.372058+0000","last_unstale":"2023-10-09T10:53:17.827259+0000","last_undegraded":"2023-10-09T10:53:17.827259+0000","last_fullsized":"2023-10-09T10:53:17.827259+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:46:35.708276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":655,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.839908+0000","last_change":"2023-10-09T10:53:17.839597+0000","last_active":"2023-10-09T10:53:17.839908+0000","last_peered":"2023-10-09T10:53:17.839908+0000","last_clean":"2023-10-09T10:53:17.839908+0000","last_became_active":"2023-10-09T10:53:17.839221+0000","last_became_peered":"2023-10-09T10:53:17.839221+0000","last_unstale":"2023-10-09T10:53:17.839908+0000","last_undegraded":"2023-10-09T10:53:17.839908+0000","last_fullsized":"2023-10-09T10:53:17.839908+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.829081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.829081+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001277842,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":142,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:18.235462+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:53:18.235462+0000","last_peered":"2023-10-09T10:53:18.235462+0000","last_clean":"2023-10-09T10:53:18.235462+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:53:18.235462+0000","last_undegraded":"2023-10-09T10:53:18.235462+0000","last_fullsized":"2023-10-09T10:53:18.235462+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":259,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.825194+0000","last_change":"2023-10-09T10:52:10.380048+0000","last_active":"2023-10-09T10:53:16.825194+0000","last_peered":"2023-10-09T10:53:16.825194+0000","last_clean":"2023-10-09T10:53:16.825194+0000","last_became_active":"2023-10-09T10:52:10.379450+0000","last_became_peered":"2023-10-09T10:52:10.379450+0000","last_unstale":"2023-10-09T10:53:16.825194+0000","last_undegraded":"2023-10-09T10:53:16.825194+0000","last_fullsized":"2023-10-09T10:53:16.825194+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:36.402930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":552,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.841387+0000","last_change":"2023-10-09T10:53:17.840544+0000","last_active":"2023-10-09T10:53:17.841387+0000","last_peered":"2023-10-09T10:53:17.841387+0000","last_clean":"2023-10-09T10:53:17.841387+0000","last_became_active":"2023-10-09T10:53:17.840167+0000","last_became_peered":"2023-10-09T10:53:17.840167+0000","last_unstale":"2023-10-09T10:53:17.841387+0000","last_undegraded":"2023-10-09T10:53:17.841387+0000","last_fullsized":"2023-10-09T10:53:17.841387+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":46,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.828052+0000","last_change":"2023-10-09T10:53:14.828754+0000","last_active":"2023-10-09T10:53:17.828052+0000","last_peered":"2023-10-09T10:53:17.828052+0000","last_clean":"2023-10-09T10:53:17.828052+0000","last_became_active":"2023-10-09T10:53:14.828480+0000","last_became_peered":"2023-10-09T10:53:14.828480+0000","last_unstale":"2023-10-09T10:53:17.828052+0000","last_undegraded":"2023-10-09T10:53:17.828052+0000","last_fullsized":"2023-10-09T10:53:17.828052+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:42.254336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":216,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.826780+0000","last_change":"2023-10-09T10:53:14.838859+0000","last_active":"2023-10-09T10:53:16.826780+0000","last_peered":"2023-10-09T10:53:16.826780+0000","last_clean":"2023-10-09T10:53:16.826780+0000","last_became_active":"2023-10-09T10:53:14.838536+0000","last_became_peered":"2023-10-09T10:53:14.838536+0000","last_unstale":"2023-10-09T10:53:16.826780+0000","last_undegraded":"2023-10-09T10:53:16.826780+0000","last_fullsized":"2023-10-09T10:53:16.826780+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:44:19.980568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":646,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.831311+0000","last_change":"2023-10-09T10:52:55.415145+0000","last_active":"2023-10-09T10:53:17.831311+0000","last_peered":"2023-10-09T10:53:17.831311+0000","last_clean":"2023-10-09T10:53:17.831311+0000","last_became_active":"2023-10-09T10:52:55.414818+0000","last_became_peered":"2023-10-09T10:52:55.414818+0000","last_unstale":"2023-10-09T10:53:17.831311+0000","last_undegraded":"2023-10-09T10:53:17.831311+0000","last_fullsized":"2023-10-09T10:53:17.831311+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.687290+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.687290+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012763172999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":223,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.827657+0000","last_change":"2023-10-09T10:53:14.847647+0000","last_active":"2023-10-09T10:53:16.827657+0000","last_peered":"2023-10-09T10:53:16.827657+0000","last_clean":"2023-10-09T10:53:16.827657+0000","last_became_active":"2023-10-09T10:53:14.832089+0000","last_became_peered":"2023-10-09T10:53:14.832089+0000","last_unstale":"2023-10-09T10:53:16.827657+0000","last_undegraded":"2023-10-09T10:53:16.827657+0000","last_fullsized":"2023-10-09T10:53:16.827657+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:34:45.371102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.828975+0000","last_change":"2023-10-09T10:52:10.392814+0000","last_active":"2023-10-09T10:53:17.828975+0000","last_peered":"2023-10-09T10:53:17.828975+0000","last_clean":"2023-10-09T10:53:17.828975+0000","last_became_active":"2023-10-09T10:52:10.392416+0000","last_became_peered":"2023-10-09T10:52:10.392416+0000","last_unstale":"2023-10-09T10:53:17.828975+0000","last_undegraded":"2023-10-09T10:53:17.828975+0000","last_fullsized":"2023-10-09T10:53:17.828975+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:30.563121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":638,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.827509+0000","last_change":"2023-10-09T10:52:55.416844+0000","last_active":"2023-10-09T10:53:17.827509+0000","last_peered":"2023-10-09T10:53:17.827509+0000","last_clean":"2023-10-09T10:53:17.827509+0000","last_became_active":"2023-10-09T10:52:55.416467+0000","last_became_peered":"2023-10-09T10:52:55.416467+0000","last_unstale":"2023-10-09T10:53:17.827509+0000","last_undegraded":"2023-10-09T10:53:17.827509+0000","last_fullsized":"2023-10-09T10:53:17.827509+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.475429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.475429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001591812,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":24,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.833217+0000","last_change":"2023-10-09T10:53:14.869000+0000","last_active":"2023-10-09T10:53:17.833217+0000","last_peered":"2023-10-09T10:53:17.833217+0000","last_clean":"2023-10-09T10:53:17.833217+0000","last_became_active":"2023-10-09T10:53:14.831904+0000","last_became_peered":"2023-10-09T10:53:14.831904+0000","last_unstale":"2023-10-09T10:53:17.833217+0000","last_undegraded":"2023-10-09T10:53:17.833217+0000","last_fullsized":"2023-10-09T10:53:17.833217+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:38.571980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":255,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.836324+0000","last_change":"2023-10-09T10:52:10.392499+0000","last_active":"2023-10-09T10:53:17.836324+0000","last_peered":"2023-10-09T10:53:17.836324+0000","last_clean":"2023-10-09T10:53:17.836324+0000","last_became_active":"2023-10-09T10:52:10.392193+0000","last_became_peered":"2023-10-09T10:52:10.392193+0000","last_unstale":"2023-10-09T10:53:17.836324+0000","last_undegraded":"2023-10-09T10:53:17.836324+0000","last_fullsized":"2023-10-09T10:53:17.836324+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:39.058086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":636,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.831931+0000","last_change":"2023-10-09T10:49:18.708370+0000","last_active":"2023-10-09T10:53:17.831931+0000","last_peered":"2023-10-09T10:53:17.831931+0000","last_clean":"2023-10-09T10:53:17.831931+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:53:17.831931+0000","last_undegraded":"2023-10-09T10:53:17.831931+0000","last_fullsized":"2023-10-09T10:53:17.831931+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:57.808410+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":20,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.832800+0000","last_change":"2023-10-09T10:53:14.857682+0000","last_active":"2023-10-09T10:53:17.832800+0000","last_peered":"2023-10-09T10:53:17.832800+0000","last_clean":"2023-10-09T10:53:17.832800+0000","last_became_active":"2023-10-09T10:53:14.831181+0000","last_became_peered":"2023-10-09T10:53:14.831181+0000","last_unstale":"2023-10-09T10:53:17.832800+0000","last_undegraded":"2023-10-09T10:53:17.832800+0000","last_fullsized":"2023-10-09T10:53:17.832800+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:13:30.921632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":300,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.827101+0000","last_change":"2023-10-09T10:53:14.836782+0000","last_active":"2023-10-09T10:53:16.827101+0000","last_peered":"2023-10-09T10:53:16.827101+0000","last_clean":"2023-10-09T10:53:16.827101+0000","last_became_active":"2023-10-09T10:53:14.836453+0000","last_became_peered":"2023-10-09T10:53:14.836453+0000","last_unstale":"2023-10-09T10:53:16.827101+0000","last_undegraded":"2023-10-09T10:53:16.827101+0000","last_fullsized":"2023-10-09T10:53:16.827101+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:43.183879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":618,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.846755+0000","last_change":"2023-10-09T10:53:17.846464+0000","last_active":"2023-10-09T10:53:17.846755+0000","last_peered":"2023-10-09T10:53:17.846755+0000","last_clean":"2023-10-09T10:53:17.846755+0000","last_became_active":"2023-10-09T10:53:17.845979+0000","last_became_peered":"2023-10-09T10:53:17.845979+0000","last_unstale":"2023-10-09T10:53:17.846755+0000","last_undegraded":"2023-10-09T10:53:17.846755+0000","last_fullsized":"2023-10-09T10:53:17.846755+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:56.295299+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":178,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.831635+0000","last_change":"2023-10-09T10:53:14.821961+0000","last_active":"2023-10-09T10:53:17.831635+0000","last_peered":"2023-10-09T10:53:17.831635+0000","last_clean":"2023-10-09T10:53:17.831635+0000","last_became_active":"2023-10-09T10:53:14.821625+0000","last_became_peered":"2023-10-09T10:53:14.821625+0000","last_unstale":"2023-10-09T10:53:17.831635+0000","last_undegraded":"2023-10-09T10:53:17.831635+0000","last_fullsized":"2023-10-09T10:53:17.831635+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:29.305952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":219,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.827469+0000","last_change":"2023-10-09T10:53:14.832387+0000","last_active":"2023-10-09T10:53:16.827469+0000","last_peered":"2023-10-09T10:53:16.827469+0000","last_clean":"2023-10-09T10:53:16.827469+0000","last_became_active":"2023-10-09T10:53:14.831672+0000","last_became_peered":"2023-10-09T10:53:14.831672+0000","last_unstale":"2023-10-09T10:53:16.827469+0000","last_undegraded":"2023-10-09T10:53:16.827469+0000","last_fullsized":"2023-10-09T10:53:16.827469+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:59.194806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":571,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.828499+0000","last_change":"2023-10-09T10:52:13.424402+0000","last_active":"2023-10-09T10:53:17.828499+0000","last_peered":"2023-10-09T10:53:17.828499+0000","last_clean":"2023-10-09T10:53:17.828499+0000","last_became_active":"2023-10-09T10:52:13.424095+0000","last_became_peered":"2023-10-09T10:52:13.424095+0000","last_unstale":"2023-10-09T10:53:17.828499+0000","last_undegraded":"2023-10-09T10:53:17.828499+0000","last_fullsized":"2023-10-09T10:53:17.828499+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":201,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.836561+0000","last_change":"2023-10-09T10:53:14.835799+0000","last_active":"2023-10-09T10:53:17.836561+0000","last_peered":"2023-10-09T10:53:17.836561+0000","last_clean":"2023-10-09T10:53:17.836561+0000","last_became_active":"2023-10-09T10:53:14.835476+0000","last_became_peered":"2023-10-09T10:53:14.835476+0000","last_unstale":"2023-10-09T10:53:17.836561+0000","last_undegraded":"2023-10-09T10:53:17.836561+0000","last_fullsized":"2023-10-09T10:53:17.836561+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:29.248961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":160,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:18.235413+0000","last_change":"2023-10-09T10:53:14.823485+0000","last_active":"2023-10-09T10:53:18.235413+0000","last_peered":"2023-10-09T10:53:18.235413+0000","last_clean":"2023-10-09T10:53:18.235413+0000","last_became_active":"2023-10-09T10:53:14.822756+0000","last_became_peered":"2023-10-09T10:53:14.822756+0000","last_unstale":"2023-10-09T10:53:18.235413+0000","last_undegraded":"2023-10-09T10:53:18.235413+0000","last_fullsized":"2023-10-09T10:53:18.235413+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:05.359313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":620,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.828746+0000","last_change":"2023-10-09T10:52:13.413332+0000","last_active":"2023-10-09T10:53:17.828746+0000","last_peered":"2023-10-09T10:53:17.828746+0000","last_clean":"2023-10-09T10:53:17.828746+0000","last_became_active":"2023-10-09T10:52:13.412975+0000","last_became_peered":"2023-10-09T10:52:13.412975+0000","last_unstale":"2023-10-09T10:53:17.828746+0000","last_undegraded":"2023-10-09T10:53:17.828746+0000","last_fullsized":"2023-10-09T10:53:17.828746+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:25.778974+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":144,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.830898+0000","last_change":"2023-10-09T10:51:12.366910+0000","last_active":"2023-10-09T10:53:17.830898+0000","last_peered":"2023-10-09T10:53:17.830898+0000","last_clean":"2023-10-09T10:53:17.830898+0000","last_became_active":"2023-10-09T10:51:12.366565+0000","last_became_peered":"2023-10-09T10:51:12.366565+0000","last_unstale":"2023-10-09T10:53:17.830898+0000","last_undegraded":"2023-10-09T10:53:17.830898+0000","last_fullsized":"2023-10-09T10:53:17.830898+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:53.464251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":277,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.829032+0000","last_change":"2023-10-09T10:53:14.836845+0000","last_active":"2023-10-09T10:53:16.829032+0000","last_peered":"2023-10-09T10:53:16.829032+0000","last_clean":"2023-10-09T10:53:16.829032+0000","last_became_active":"2023-10-09T10:53:14.836614+0000","last_became_peered":"2023-10-09T10:53:14.836614+0000","last_unstale":"2023-10-09T10:53:16.829032+0000","last_undegraded":"2023-10-09T10:53:16.829032+0000","last_fullsized":"2023-10-09T10:53:16.829032+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:21:33.381735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"271'407","reported_seq":421,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-10-09T10:53:21.202816+0000","last_change":"2023-10-09T10:53:13.820083+0000","last_active":"2023-10-09T10:53:21.202816+0000","last_peered":"2023-10-09T10:53:21.202816+0000","last_clean":"2023-10-09T10:53:21.202816+0000","last_became_active":"2023-10-09T10:53:13.819794+0000","last_became_peered":"2023-10-09T10:53:13.819794+0000","last_unstale":"2023-10-09T10:53:21.202816+0000","last_undegraded":"2023-10-09T10:53:21.202816+0000","last_fullsized":"2023-10-09T10:53:21.202816+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":407,"log_dups_size":0,"ondisk_log_size":407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:53.397574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26673152,"num_objects":407,"num_object_clones":0,"num_object_copies":1221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":407,"num_write_kb":26048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":552,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.843832+0000","last_change":"2023-10-09T10:53:17.842283+0000","last_active":"2023-10-09T10:53:17.843832+0000","last_peered":"2023-10-09T10:53:17.843832+0000","last_clean":"2023-10-09T10:53:17.843832+0000","last_became_active":"2023-10-09T10:53:17.841926+0000","last_became_peered":"2023-10-09T10:53:17.841926+0000","last_unstale":"2023-10-09T10:53:17.843832+0000","last_undegraded":"2023-10-09T10:53:17.843832+0000","last_fullsized":"2023-10-09T10:53:17.843832+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":18,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.834266+0000","last_change":"2023-10-09T10:53:14.854904+0000","last_active":"2023-10-09T10:53:17.834266+0000","last_peered":"2023-10-09T10:53:17.834266+0000","last_clean":"2023-10-09T10:53:17.834266+0000","last_became_active":"2023-10-09T10:53:14.829611+0000","last_became_peered":"2023-10-09T10:53:14.829611+0000","last_unstale":"2023-10-09T10:53:17.834266+0000","last_undegraded":"2023-10-09T10:53:17.834266+0000","last_fullsized":"2023-10-09T10:53:17.834266+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:26.064630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":238,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.835442+0000","last_change":"2023-10-09T10:53:14.831930+0000","last_active":"2023-10-09T10:53:17.835442+0000","last_peered":"2023-10-09T10:53:17.835442+0000","last_clean":"2023-10-09T10:53:17.835442+0000","last_became_active":"2023-10-09T10:53:14.831600+0000","last_became_peered":"2023-10-09T10:53:14.831600+0000","last_unstale":"2023-10-09T10:53:17.835442+0000","last_undegraded":"2023-10-09T10:53:17.835442+0000","last_fullsized":"2023-10-09T10:53:17.835442+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:16.104668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"271'382","reported_seq":395,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-10-09T10:53:21.093242+0000","last_change":"2023-10-09T10:53:14.832791+0000","last_active":"2023-10-09T10:53:21.093242+0000","last_peered":"2023-10-09T10:53:21.093242+0000","last_clean":"2023-10-09T10:53:21.093242+0000","last_became_active":"2023-10-09T10:53:14.832491+0000","last_became_peered":"2023-10-09T10:53:14.832491+0000","last_unstale":"2023-10-09T10:53:21.093242+0000","last_undegraded":"2023-10-09T10:53:21.093242+0000","last_fullsized":"2023-10-09T10:53:21.093242+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":382,"log_dups_size":0,"ondisk_log_size":382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:51.903123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25034752,"num_objects":382,"num_object_clones":0,"num_object_copies":1146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":382,"num_write_kb":24448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":616,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.827021+0000","last_change":"2023-10-09T10:51:01.296071+0000","last_active":"2023-10-09T10:53:16.827021+0000","last_peered":"2023-10-09T10:53:16.827021+0000","last_clean":"2023-10-09T10:53:16.827021+0000","last_became_active":"2023-10-09T10:51:01.295474+0000","last_became_peered":"2023-10-09T10:51:01.295474+0000","last_unstale":"2023-10-09T10:53:16.827021+0000","last_undegraded":"2023-10-09T10:53:16.827021+0000","last_fullsized":"2023-10-09T10:53:16.827021+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:41.637984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:41.637984+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001804833,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":18,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.833019+0000","last_change":"2023-10-09T10:53:14.851091+0000","last_active":"2023-10-09T10:53:17.833019+0000","last_peered":"2023-10-09T10:53:17.833019+0000","last_clean":"2023-10-09T10:53:17.833019+0000","last_became_active":"2023-10-09T10:53:14.830375+0000","last_became_peered":"2023-10-09T10:53:14.830375+0000","last_unstale":"2023-10-09T10:53:17.833019+0000","last_undegraded":"2023-10-09T10:53:17.833019+0000","last_fullsized":"2023-10-09T10:53:17.833019+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:49.351116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.826992+0000","last_change":"2023-10-09T10:52:10.405631+0000","last_active":"2023-10-09T10:53:17.826992+0000","last_peered":"2023-10-09T10:53:17.826992+0000","last_clean":"2023-10-09T10:53:17.826992+0000","last_became_active":"2023-10-09T10:52:10.405228+0000","last_became_peered":"2023-10-09T10:52:10.405228+0000","last_unstale":"2023-10-09T10:53:17.826992+0000","last_undegraded":"2023-10-09T10:53:17.826992+0000","last_fullsized":"2023-10-09T10:53:17.826992+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:53.535609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"271'793","reported_seq":809,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-10-09T10:53:23.234243+0000","last_change":"2023-10-09T10:53:14.826832+0000","last_active":"2023-10-09T10:53:23.234243+0000","last_peered":"2023-10-09T10:53:23.234243+0000","last_clean":"2023-10-09T10:53:23.234243+0000","last_became_active":"2023-10-09T10:53:14.826507+0000","last_became_peered":"2023-10-09T10:53:14.826507+0000","last_unstale":"2023-10-09T10:53:23.234243+0000","last_undegraded":"2023-10-09T10:53:23.234243+0000","last_fullsized":"2023-10-09T10:53:23.234243+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":793,"log_dups_size":0,"ondisk_log_size":793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:06:41.609418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51970048,"num_objects":793,"num_object_clones":0,"num_object_copies":2379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":793,"num_write_kb":50752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":573,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.837607+0000","last_change":"2023-10-09T10:51:50.371393+0000","last_active":"2023-10-09T10:53:17.837607+0000","last_peered":"2023-10-09T10:53:17.837607+0000","last_clean":"2023-10-09T10:53:17.837607+0000","last_became_active":"2023-10-09T10:51:50.371032+0000","last_became_peered":"2023-10-09T10:51:50.371032+0000","last_unstale":"2023-10-09T10:53:17.837607+0000","last_undegraded":"2023-10-09T10:53:17.837607+0000","last_fullsized":"2023-10-09T10:53:17.837607+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":18,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.838604+0000","last_change":"2023-10-09T10:53:14.841343+0000","last_active":"2023-10-09T10:53:17.838604+0000","last_peered":"2023-10-09T10:53:17.838604+0000","last_clean":"2023-10-09T10:53:17.838604+0000","last_became_active":"2023-10-09T10:53:14.828213+0000","last_became_peered":"2023-10-09T10:53:14.828213+0000","last_unstale":"2023-10-09T10:53:17.838604+0000","last_undegraded":"2023-10-09T10:53:17.838604+0000","last_fullsized":"2023-10-09T10:53:17.838604+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:28.137803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":46,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.828835+0000","last_change":"2023-10-09T10:53:14.820650+0000","last_active":"2023-10-09T10:53:17.828835+0000","last_peered":"2023-10-09T10:53:17.828835+0000","last_clean":"2023-10-09T10:53:17.828835+0000","last_became_active":"2023-10-09T10:53:14.820332+0000","last_became_peered":"2023-10-09T10:53:14.820332+0000","last_unstale":"2023-10-09T10:53:17.828835+0000","last_undegraded":"2023-10-09T10:53:17.828835+0000","last_fullsized":"2023-10-09T10:53:17.828835+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:59.838629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"271'451","reported_seq":466,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-10-09T10:53:21.599308+0000","last_change":"2023-10-09T10:53:13.834477+0000","last_active":"2023-10-09T10:53:21.599308+0000","last_peered":"2023-10-09T10:53:21.599308+0000","last_clean":"2023-10-09T10:53:21.599308+0000","last_became_active":"2023-10-09T10:53:13.833978+0000","last_became_peered":"2023-10-09T10:53:13.833978+0000","last_unstale":"2023-10-09T10:53:21.599308+0000","last_undegraded":"2023-10-09T10:53:21.599308+0000","last_fullsized":"2023-10-09T10:53:21.599308+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":451,"log_dups_size":0,"ondisk_log_size":451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:00:09.470321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29556736,"num_objects":451,"num_object_clones":0,"num_object_copies":1353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":451,"num_write_kb":28864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":706,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.842400+0000","last_change":"2023-10-09T10:53:17.840913+0000","last_active":"2023-10-09T10:53:17.842400+0000","last_peered":"2023-10-09T10:53:17.842400+0000","last_clean":"2023-10-09T10:53:17.842400+0000","last_became_active":"2023-10-09T10:53:17.840478+0000","last_became_peered":"2023-10-09T10:53:17.840478+0000","last_unstale":"2023-10-09T10:53:17.842400+0000","last_undegraded":"2023-10-09T10:53:17.842400+0000","last_fullsized":"2023-10-09T10:53:17.842400+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":94,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.827327+0000","last_change":"2023-10-09T10:51:50.374527+0000","last_active":"2023-10-09T10:53:16.827327+0000","last_peered":"2023-10-09T10:53:16.827327+0000","last_clean":"2023-10-09T10:53:16.827327+0000","last_became_active":"2023-10-09T10:51:50.374297+0000","last_became_peered":"2023-10-09T10:51:50.374297+0000","last_unstale":"2023-10-09T10:53:16.827327+0000","last_undegraded":"2023-10-09T10:53:16.827327+0000","last_fullsized":"2023-10-09T10:53:16.827327+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:31:49.358307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":160,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.831283+0000","last_change":"2023-10-09T10:52:10.385820+0000","last_active":"2023-10-09T10:53:17.831283+0000","last_peered":"2023-10-09T10:53:17.831283+0000","last_clean":"2023-10-09T10:53:17.831283+0000","last_became_active":"2023-10-09T10:52:10.385498+0000","last_became_peered":"2023-10-09T10:52:10.385498+0000","last_unstale":"2023-10-09T10:53:17.831283+0000","last_undegraded":"2023-10-09T10:53:17.831283+0000","last_fullsized":"2023-10-09T10:53:17.831283+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:18.341986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"270'218","reported_seq":232,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:19.970132+0000","last_change":"2023-10-09T10:53:13.825865+0000","last_active":"2023-10-09T10:53:19.970132+0000","last_peered":"2023-10-09T10:53:19.970132+0000","last_clean":"2023-10-09T10:53:19.970132+0000","last_became_active":"2023-10-09T10:53:13.825545+0000","last_became_peered":"2023-10-09T10:53:13.825545+0000","last_unstale":"2023-10-09T10:53:19.970132+0000","last_undegraded":"2023-10-09T10:53:19.970132+0000","last_fullsized":"2023-10-09T10:53:19.970132+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":218,"log_dups_size":0,"ondisk_log_size":218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.958559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14286848,"num_objects":218,"num_object_clones":0,"num_object_copies":654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":218,"num_write_kb":13952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":648,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.840772+0000","last_change":"2023-10-09T10:53:17.840772+0000","last_active":"2023-10-09T10:53:17.840772+0000","last_peered":"2023-10-09T10:53:17.840772+0000","last_clean":"2023-10-09T10:53:17.840772+0000","last_became_active":"2023-10-09T10:53:17.840335+0000","last_became_peered":"2023-10-09T10:53:17.840335+0000","last_unstale":"2023-10-09T10:53:17.840772+0000","last_undegraded":"2023-10-09T10:53:17.840772+0000","last_fullsized":"2023-10-09T10:53:17.840772+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":265,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.827454+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:53:16.827454+0000","last_peered":"2023-10-09T10:53:16.827454+0000","last_clean":"2023-10-09T10:53:16.827454+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:53:16.827454+0000","last_undegraded":"2023-10-09T10:53:16.827454+0000","last_fullsized":"2023-10-09T10:53:16.827454+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":110,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.827241+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T10:53:16.827241+0000","last_peered":"2023-10-09T10:53:16.827241+0000","last_clean":"2023-10-09T10:53:16.827241+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T10:53:16.827241+0000","last_undegraded":"2023-10-09T10:53:16.827241+0000","last_fullsized":"2023-10-09T10:53:16.827241+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"271'495","reported_seq":510,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-10-09T10:53:21.615918+0000","last_change":"2023-10-09T10:53:13.832896+0000","last_active":"2023-10-09T10:53:21.615918+0000","last_peered":"2023-10-09T10:53:21.615918+0000","last_clean":"2023-10-09T10:53:21.615918+0000","last_became_active":"2023-10-09T10:53:13.832683+0000","last_became_peered":"2023-10-09T10:53:13.832683+0000","last_unstale":"2023-10-09T10:53:21.615918+0000","last_undegraded":"2023-10-09T10:53:21.615918+0000","last_fullsized":"2023-10-09T10:53:21.615918+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":495,"log_dups_size":0,"ondisk_log_size":495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:49.467415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32440320,"num_objects":495,"num_object_clones":0,"num_object_copies":1485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":495,"num_write_kb":31680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":551,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.824808+0000","last_change":"2023-10-09T10:51:50.373921+0000","last_active":"2023-10-09T10:53:16.824808+0000","last_peered":"2023-10-09T10:53:16.824808+0000","last_clean":"2023-10-09T10:53:16.824808+0000","last_became_active":"2023-10-09T10:51:50.373185+0000","last_became_peered":"2023-10-09T10:51:50.373185+0000","last_unstale":"2023-10-09T10:53:16.824808+0000","last_undegraded":"2023-10-09T10:53:16.824808+0000","last_fullsized":"2023-10-09T10:53:16.824808+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":230,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.836400+0000","last_change":"2023-10-09T10:52:10.392827+0000","last_active":"2023-10-09T10:53:17.836400+0000","last_peered":"2023-10-09T10:53:17.836400+0000","last_clean":"2023-10-09T10:53:17.836400+0000","last_became_active":"2023-10-09T10:52:10.392082+0000","last_became_peered":"2023-10-09T10:52:10.392082+0000","last_unstale":"2023-10-09T10:53:17.836400+0000","last_undegraded":"2023-10-09T10:53:17.836400+0000","last_fullsized":"2023-10-09T10:53:17.836400+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:48.702641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.828886+0000","last_change":"2023-10-09T10:52:10.409941+0000","last_active":"2023-10-09T10:53:17.828886+0000","last_peered":"2023-10-09T10:53:17.828886+0000","last_clean":"2023-10-09T10:53:17.828886+0000","last_became_active":"2023-10-09T10:52:10.409687+0000","last_became_peered":"2023-10-09T10:52:10.409687+0000","last_unstale":"2023-10-09T10:53:17.828886+0000","last_undegraded":"2023-10-09T10:53:17.828886+0000","last_fullsized":"2023-10-09T10:53:17.828886+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:23:37.794696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"271'416","reported_seq":430,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-10-09T10:53:21.221200+0000","last_change":"2023-10-09T10:53:13.831715+0000","last_active":"2023-10-09T10:53:21.221200+0000","last_peered":"2023-10-09T10:53:21.221200+0000","last_clean":"2023-10-09T10:53:21.221200+0000","last_became_active":"2023-10-09T10:53:13.831447+0000","last_became_peered":"2023-10-09T10:53:13.831447+0000","last_unstale":"2023-10-09T10:53:21.221200+0000","last_undegraded":"2023-10-09T10:53:21.221200+0000","last_fullsized":"2023-10-09T10:53:21.221200+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":416,"log_dups_size":0,"ondisk_log_size":416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:27.579880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27262976,"num_objects":416,"num_object_clones":0,"num_object_copies":1248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":416,"num_write_kb":26624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":635,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.828569+0000","last_change":"2023-10-09T10:52:13.417673+0000","last_active":"2023-10-09T10:53:17.828569+0000","last_peered":"2023-10-09T10:53:17.828569+0000","last_clean":"2023-10-09T10:53:17.828569+0000","last_became_active":"2023-10-09T10:52:13.417290+0000","last_became_peered":"2023-10-09T10:52:13.417290+0000","last_unstale":"2023-10-09T10:53:17.828569+0000","last_undegraded":"2023-10-09T10:53:17.828569+0000","last_fullsized":"2023-10-09T10:53:17.828569+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.892515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.892515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":286,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.836381+0000","last_change":"2023-10-09T10:53:14.835835+0000","last_active":"2023-10-09T10:53:17.836381+0000","last_peered":"2023-10-09T10:53:17.836381+0000","last_clean":"2023-10-09T10:53:17.836381+0000","last_became_active":"2023-10-09T10:53:14.835528+0000","last_became_peered":"2023-10-09T10:53:14.835528+0000","last_unstale":"2023-10-09T10:53:17.836381+0000","last_undegraded":"2023-10-09T10:53:17.836381+0000","last_fullsized":"2023-10-09T10:53:17.836381+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:31:49.025740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.827864+0000","last_change":"2023-10-09T10:52:10.412518+0000","last_active":"2023-10-09T10:53:17.827864+0000","last_peered":"2023-10-09T10:53:17.827864+0000","last_clean":"2023-10-09T10:53:17.827864+0000","last_became_active":"2023-10-09T10:52:10.411988+0000","last_became_peered":"2023-10-09T10:52:10.411988+0000","last_unstale":"2023-10-09T10:53:17.827864+0000","last_undegraded":"2023-10-09T10:53:17.827864+0000","last_fullsized":"2023-10-09T10:53:17.827864+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:13.191079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"271'492","reported_seq":507,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-10-09T10:53:21.609786+0000","last_change":"2023-10-09T10:53:13.822820+0000","last_active":"2023-10-09T10:53:21.609786+0000","last_peered":"2023-10-09T10:53:21.609786+0000","last_clean":"2023-10-09T10:53:21.609786+0000","last_became_active":"2023-10-09T10:53:13.822464+0000","last_became_peered":"2023-10-09T10:53:13.822464+0000","last_unstale":"2023-10-09T10:53:21.609786+0000","last_undegraded":"2023-10-09T10:53:21.609786+0000","last_fullsized":"2023-10-09T10:53:21.609786+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":492,"log_dups_size":0,"ondisk_log_size":492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:58.388496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32243712,"num_objects":492,"num_object_clones":0,"num_object_copies":1476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":492,"num_write_kb":31488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":564,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.836278+0000","last_change":"2023-10-09T10:51:50.372344+0000","last_active":"2023-10-09T10:53:17.836278+0000","last_peered":"2023-10-09T10:53:17.836278+0000","last_clean":"2023-10-09T10:53:17.836278+0000","last_became_active":"2023-10-09T10:51:50.372038+0000","last_became_peered":"2023-10-09T10:51:50.372038+0000","last_unstale":"2023-10-09T10:53:17.836278+0000","last_undegraded":"2023-10-09T10:53:17.836278+0000","last_fullsized":"2023-10-09T10:53:17.836278+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":278,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.824959+0000","last_change":"2023-10-09T10:53:14.843091+0000","last_active":"2023-10-09T10:53:16.824959+0000","last_peered":"2023-10-09T10:53:16.824959+0000","last_clean":"2023-10-09T10:53:16.824959+0000","last_became_active":"2023-10-09T10:53:14.842519+0000","last_became_peered":"2023-10-09T10:53:14.842519+0000","last_unstale":"2023-10-09T10:53:16.824959+0000","last_undegraded":"2023-10-09T10:53:16.824959+0000","last_fullsized":"2023-10-09T10:53:16.824959+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:15.192067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.827409+0000","last_change":"2023-10-09T10:52:10.407797+0000","last_active":"2023-10-09T10:53:17.827409+0000","last_peered":"2023-10-09T10:53:17.827409+0000","last_clean":"2023-10-09T10:53:17.827409+0000","last_became_active":"2023-10-09T10:52:10.407481+0000","last_became_peered":"2023-10-09T10:52:10.407481+0000","last_unstale":"2023-10-09T10:53:17.827409+0000","last_undegraded":"2023-10-09T10:53:17.827409+0000","last_fullsized":"2023-10-09T10:53:17.827409+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:54.829995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"271'759","reported_seq":775,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-10-09T10:53:23.224348+0000","last_change":"2023-10-09T10:53:14.823717+0000","last_active":"2023-10-09T10:53:23.224348+0000","last_peered":"2023-10-09T10:53:23.224348+0000","last_clean":"2023-10-09T10:53:23.224348+0000","last_became_active":"2023-10-09T10:53:14.823392+0000","last_became_peered":"2023-10-09T10:53:14.823392+0000","last_unstale":"2023-10-09T10:53:23.224348+0000","last_undegraded":"2023-10-09T10:53:23.224348+0000","last_fullsized":"2023-10-09T10:53:23.224348+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":759,"log_dups_size":0,"ondisk_log_size":759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:03.561050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49741824,"num_objects":759,"num_object_clones":0,"num_object_copies":2277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":759,"num_write_kb":48576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":653,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.842380+0000","last_change":"2023-10-09T10:53:17.842380+0000","last_active":"2023-10-09T10:53:17.842380+0000","last_peered":"2023-10-09T10:53:17.842380+0000","last_clean":"2023-10-09T10:53:17.842380+0000","last_became_active":"2023-10-09T10:53:17.841970+0000","last_became_peered":"2023-10-09T10:53:17.841970+0000","last_unstale":"2023-10-09T10:53:17.842380+0000","last_undegraded":"2023-10-09T10:53:17.842380+0000","last_fullsized":"2023-10-09T10:53:17.842380+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.687290+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.687290+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012763172999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":46,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.829681+0000","last_change":"2023-10-09T10:53:14.825425+0000","last_active":"2023-10-09T10:53:17.829681+0000","last_peered":"2023-10-09T10:53:17.829681+0000","last_clean":"2023-10-09T10:53:17.829681+0000","last_became_active":"2023-10-09T10:53:14.825102+0000","last_became_peered":"2023-10-09T10:53:14.825102+0000","last_unstale":"2023-10-09T10:53:17.829681+0000","last_undegraded":"2023-10-09T10:53:17.829681+0000","last_fullsized":"2023-10-09T10:53:17.829681+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:53:22.289226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":179,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.827743+0000","last_change":"2023-10-09T10:51:50.373359+0000","last_active":"2023-10-09T10:53:16.827743+0000","last_peered":"2023-10-09T10:53:16.827743+0000","last_clean":"2023-10-09T10:53:16.827743+0000","last_became_active":"2023-10-09T10:51:50.373009+0000","last_became_peered":"2023-10-09T10:51:50.373009+0000","last_unstale":"2023-10-09T10:53:16.827743+0000","last_undegraded":"2023-10-09T10:53:16.827743+0000","last_fullsized":"2023-10-09T10:53:16.827743+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:11.509198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.828580+0000","last_change":"2023-10-09T10:52:10.406697+0000","last_active":"2023-10-09T10:53:17.828580+0000","last_peered":"2023-10-09T10:53:17.828580+0000","last_clean":"2023-10-09T10:53:17.828580+0000","last_became_active":"2023-10-09T10:52:10.405905+0000","last_became_peered":"2023-10-09T10:52:10.405905+0000","last_unstale":"2023-10-09T10:53:17.828580+0000","last_undegraded":"2023-10-09T10:53:17.828580+0000","last_fullsized":"2023-10-09T10:53:17.828580+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:27:39.691503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"270'217","reported_seq":232,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:19.977298+0000","last_change":"2023-10-09T10:53:14.843269+0000","last_active":"2023-10-09T10:53:19.977298+0000","last_peered":"2023-10-09T10:53:19.977298+0000","last_clean":"2023-10-09T10:53:19.977298+0000","last_became_active":"2023-10-09T10:53:14.842845+0000","last_became_peered":"2023-10-09T10:53:14.842845+0000","last_unstale":"2023-10-09T10:53:19.977298+0000","last_undegraded":"2023-10-09T10:53:19.977298+0000","last_fullsized":"2023-10-09T10:53:19.977298+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":217,"log_dups_size":0,"ondisk_log_size":217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:29.867741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14221312,"num_objects":217,"num_object_clones":0,"num_object_copies":651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":217,"num_write_kb":13888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":577,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.831846+0000","last_change":"2023-10-09T10:51:50.376468+0000","last_active":"2023-10-09T10:53:17.831846+0000","last_peered":"2023-10-09T10:53:17.831846+0000","last_clean":"2023-10-09T10:53:17.831846+0000","last_became_active":"2023-10-09T10:51:50.376078+0000","last_became_peered":"2023-10-09T10:51:50.376078+0000","last_unstale":"2023-10-09T10:53:17.831846+0000","last_undegraded":"2023-10-09T10:53:17.831846+0000","last_fullsized":"2023-10-09T10:53:17.831846+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.736834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.736834+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.828492+0000","last_change":"2023-10-09T10:52:10.372993+0000","last_active":"2023-10-09T10:53:17.828492+0000","last_peered":"2023-10-09T10:53:17.828492+0000","last_clean":"2023-10-09T10:53:17.828492+0000","last_became_active":"2023-10-09T10:52:10.372346+0000","last_became_peered":"2023-10-09T10:52:10.372346+0000","last_unstale":"2023-10-09T10:53:17.828492+0000","last_undegraded":"2023-10-09T10:53:17.828492+0000","last_fullsized":"2023-10-09T10:53:17.828492+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:55.012253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":179,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.830995+0000","last_change":"2023-10-09T10:50:43.388615+0000","last_active":"2023-10-09T10:53:17.830995+0000","last_peered":"2023-10-09T10:53:17.830995+0000","last_clean":"2023-10-09T10:53:17.830995+0000","last_became_active":"2023-10-09T10:50:43.388176+0000","last_became_peered":"2023-10-09T10:50:43.388176+0000","last_unstale":"2023-10-09T10:53:17.830995+0000","last_undegraded":"2023-10-09T10:53:17.830995+0000","last_fullsized":"2023-10-09T10:53:17.830995+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:41:17.550111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.829867+0000","last_change":"2023-10-09T10:52:10.380162+0000","last_active":"2023-10-09T10:53:17.829867+0000","last_peered":"2023-10-09T10:53:17.829867+0000","last_clean":"2023-10-09T10:53:17.829867+0000","last_became_active":"2023-10-09T10:52:10.379844+0000","last_became_peered":"2023-10-09T10:52:10.379844+0000","last_unstale":"2023-10-09T10:53:17.829867+0000","last_undegraded":"2023-10-09T10:53:17.829867+0000","last_fullsized":"2023-10-09T10:53:17.829867+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:04:28.650194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"270'210","reported_seq":224,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:19.969607+0000","last_change":"2023-10-09T10:53:13.840703+0000","last_active":"2023-10-09T10:53:19.969607+0000","last_peered":"2023-10-09T10:53:19.969607+0000","last_clean":"2023-10-09T10:53:19.969607+0000","last_became_active":"2023-10-09T10:53:13.840488+0000","last_became_peered":"2023-10-09T10:53:13.840488+0000","last_unstale":"2023-10-09T10:53:19.969607+0000","last_undegraded":"2023-10-09T10:53:19.969607+0000","last_fullsized":"2023-10-09T10:53:19.969607+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":210,"log_dups_size":0,"ondisk_log_size":210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:22:24.617041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13762560,"num_objects":210,"num_object_clones":0,"num_object_copies":630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":210,"num_write_kb":13440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":659,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.842862+0000","last_change":"2023-10-09T10:53:17.842587+0000","last_active":"2023-10-09T10:53:17.842862+0000","last_peered":"2023-10-09T10:53:17.842862+0000","last_clean":"2023-10-09T10:53:17.842862+0000","last_became_active":"2023-10-09T10:53:17.842229+0000","last_became_peered":"2023-10-09T10:53:17.842229+0000","last_unstale":"2023-10-09T10:53:17.842862+0000","last_undegraded":"2023-10-09T10:53:17.842862+0000","last_fullsized":"2023-10-09T10:53:17.842862+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":288,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.826893+0000","last_change":"2023-10-09T10:52:10.378818+0000","last_active":"2023-10-09T10:53:16.826893+0000","last_peered":"2023-10-09T10:53:16.826893+0000","last_clean":"2023-10-09T10:53:16.826893+0000","last_became_active":"2023-10-09T10:52:10.378248+0000","last_became_peered":"2023-10-09T10:52:10.378248+0000","last_unstale":"2023-10-09T10:53:16.826893+0000","last_undegraded":"2023-10-09T10:53:16.826893+0000","last_fullsized":"2023-10-09T10:53:16.826893+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:27.613093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.826333+0000","last_change":"2023-10-09T10:52:10.376143+0000","last_active":"2023-10-09T10:53:17.826333+0000","last_peered":"2023-10-09T10:53:17.826333+0000","last_clean":"2023-10-09T10:53:17.826333+0000","last_became_active":"2023-10-09T10:52:10.375861+0000","last_became_peered":"2023-10-09T10:52:10.375861+0000","last_unstale":"2023-10-09T10:53:17.826333+0000","last_undegraded":"2023-10-09T10:53:17.826333+0000","last_fullsized":"2023-10-09T10:53:17.826333+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:24.689310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.827332+0000","last_change":"2023-10-09T10:52:10.390172+0000","last_active":"2023-10-09T10:53:17.827332+0000","last_peered":"2023-10-09T10:53:17.827332+0000","last_clean":"2023-10-09T10:53:17.827332+0000","last_became_active":"2023-10-09T10:52:10.389909+0000","last_became_peered":"2023-10-09T10:52:10.389909+0000","last_unstale":"2023-10-09T10:53:17.827332+0000","last_undegraded":"2023-10-09T10:53:17.827332+0000","last_fullsized":"2023-10-09T10:53:17.827332+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:38.723747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"271'773","reported_seq":788,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-10-09T10:53:23.216777+0000","last_change":"2023-10-09T10:53:13.829514+0000","last_active":"2023-10-09T10:53:23.216777+0000","last_peered":"2023-10-09T10:53:23.216777+0000","last_clean":"2023-10-09T10:53:23.216777+0000","last_became_active":"2023-10-09T10:53:13.829236+0000","last_became_peered":"2023-10-09T10:53:13.829236+0000","last_unstale":"2023-10-09T10:53:23.216777+0000","last_undegraded":"2023-10-09T10:53:23.216777+0000","last_fullsized":"2023-10-09T10:53:23.216777+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":773,"log_dups_size":0,"ondisk_log_size":773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:38.141857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50659328,"num_objects":773,"num_object_clones":0,"num_object_copies":2319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":773,"num_write_kb":49472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":14,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.832885+0000","last_change":"2023-10-09T10:53:14.833623+0000","last_active":"2023-10-09T10:53:17.832885+0000","last_peered":"2023-10-09T10:53:17.832885+0000","last_clean":"2023-10-09T10:53:17.832885+0000","last_became_active":"2023-10-09T10:53:14.833349+0000","last_became_peered":"2023-10-09T10:53:14.833349+0000","last_unstale":"2023-10-09T10:53:17.832885+0000","last_undegraded":"2023-10-09T10:53:17.832885+0000","last_fullsized":"2023-10-09T10:53:17.832885+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:35:38.536919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":688,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.844577+0000","last_change":"2023-10-09T10:53:17.844346+0000","last_active":"2023-10-09T10:53:17.844577+0000","last_peered":"2023-10-09T10:53:17.844577+0000","last_clean":"2023-10-09T10:53:17.844577+0000","last_became_active":"2023-10-09T10:53:17.844056+0000","last_became_peered":"2023-10-09T10:53:17.844056+0000","last_unstale":"2023-10-09T10:53:17.844577+0000","last_undegraded":"2023-10-09T10:53:17.844577+0000","last_fullsized":"2023-10-09T10:53:17.844577+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:27:59.443732+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":200,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.826922+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:53:16.826922+0000","last_peered":"2023-10-09T10:53:16.826922+0000","last_clean":"2023-10-09T10:53:16.826922+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:53:16.826922+0000","last_undegraded":"2023-10-09T10:53:16.826922+0000","last_fullsized":"2023-10-09T10:53:16.826922+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":183,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.836601+0000","last_change":"2023-10-09T10:53:14.836523+0000","last_active":"2023-10-09T10:53:17.836601+0000","last_peered":"2023-10-09T10:53:17.836601+0000","last_clean":"2023-10-09T10:53:17.836601+0000","last_became_active":"2023-10-09T10:53:14.836134+0000","last_became_peered":"2023-10-09T10:53:14.836134+0000","last_unstale":"2023-10-09T10:53:17.836601+0000","last_undegraded":"2023-10-09T10:53:17.836601+0000","last_fullsized":"2023-10-09T10:53:17.836601+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:05:24.615198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"271'467","reported_seq":482,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-10-09T10:53:21.612630+0000","last_change":"2023-10-09T10:53:13.833563+0000","last_active":"2023-10-09T10:53:21.612630+0000","last_peered":"2023-10-09T10:53:21.612630+0000","last_clean":"2023-10-09T10:53:21.612630+0000","last_became_active":"2023-10-09T10:53:13.833298+0000","last_became_peered":"2023-10-09T10:53:13.833298+0000","last_unstale":"2023-10-09T10:53:21.612630+0000","last_undegraded":"2023-10-09T10:53:21.612630+0000","last_fullsized":"2023-10-09T10:53:21.612630+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":467,"log_dups_size":0,"ondisk_log_size":467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:50:25.787312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30605312,"num_objects":467,"num_object_clones":0,"num_object_copies":1401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":467,"num_write_kb":29888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":620,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.828424+0000","last_change":"2023-10-09T10:52:13.412193+0000","last_active":"2023-10-09T10:53:17.828424+0000","last_peered":"2023-10-09T10:53:17.828424+0000","last_clean":"2023-10-09T10:53:17.828424+0000","last_became_active":"2023-10-09T10:52:13.411872+0000","last_became_peered":"2023-10-09T10:52:13.411872+0000","last_unstale":"2023-10-09T10:53:17.828424+0000","last_undegraded":"2023-10-09T10:53:17.828424+0000","last_fullsized":"2023-10-09T10:53:17.828424+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":46,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.826179+0000","last_change":"2023-10-09T10:53:14.818885+0000","last_active":"2023-10-09T10:53:17.826179+0000","last_peered":"2023-10-09T10:53:17.826179+0000","last_clean":"2023-10-09T10:53:17.826179+0000","last_became_active":"2023-10-09T10:53:14.818541+0000","last_became_peered":"2023-10-09T10:53:14.818541+0000","last_unstale":"2023-10-09T10:53:17.826179+0000","last_undegraded":"2023-10-09T10:53:17.826179+0000","last_fullsized":"2023-10-09T10:53:17.826179+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:51.649543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":151,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.831762+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:53:17.831762+0000","last_peered":"2023-10-09T10:53:17.831762+0000","last_clean":"2023-10-09T10:53:17.831762+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:53:17.831762+0000","last_undegraded":"2023-10-09T10:53:17.831762+0000","last_fullsized":"2023-10-09T10:53:17.831762+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":187,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.831276+0000","last_change":"2023-10-09T10:53:14.819286+0000","last_active":"2023-10-09T10:53:17.831276+0000","last_peered":"2023-10-09T10:53:17.831276+0000","last_clean":"2023-10-09T10:53:17.831276+0000","last_became_active":"2023-10-09T10:53:14.818822+0000","last_became_peered":"2023-10-09T10:53:14.818822+0000","last_unstale":"2023-10-09T10:53:17.831276+0000","last_undegraded":"2023-10-09T10:53:17.831276+0000","last_fullsized":"2023-10-09T10:53:17.831276+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:47:53.257016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"270'206","reported_seq":220,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:19.962038+0000","last_change":"2023-10-09T10:53:13.832229+0000","last_active":"2023-10-09T10:53:19.962038+0000","last_peered":"2023-10-09T10:53:19.962038+0000","last_clean":"2023-10-09T10:53:19.962038+0000","last_became_active":"2023-10-09T10:53:13.831969+0000","last_became_peered":"2023-10-09T10:53:13.831969+0000","last_unstale":"2023-10-09T10:53:19.962038+0000","last_undegraded":"2023-10-09T10:53:19.962038+0000","last_fullsized":"2023-10-09T10:53:19.962038+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":206,"log_dups_size":0,"ondisk_log_size":206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:22.724023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13500416,"num_objects":206,"num_object_clones":0,"num_object_copies":618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":206,"num_write_kb":13184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":630,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.831520+0000","last_change":"2023-10-09T10:51:12.367319+0000","last_active":"2023-10-09T10:53:17.831520+0000","last_peered":"2023-10-09T10:53:17.831520+0000","last_clean":"2023-10-09T10:53:17.831520+0000","last_became_active":"2023-10-09T10:51:12.366828+0000","last_became_peered":"2023-10-09T10:51:12.366828+0000","last_unstale":"2023-10-09T10:53:17.831520+0000","last_undegraded":"2023-10-09T10:53:17.831520+0000","last_fullsized":"2023-10-09T10:53:17.831520+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:38.475429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:38.475429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001591812,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.838494+0000","last_change":"2023-10-09T10:53:14.833073+0000","last_active":"2023-10-09T10:53:17.838494+0000","last_peered":"2023-10-09T10:53:17.838494+0000","last_clean":"2023-10-09T10:53:17.838494+0000","last_became_active":"2023-10-09T10:53:14.832736+0000","last_became_peered":"2023-10-09T10:53:14.832736+0000","last_unstale":"2023-10-09T10:53:17.838494+0000","last_undegraded":"2023-10-09T10:53:17.838494+0000","last_fullsized":"2023-10-09T10:53:17.838494+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:27.550707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.829612+0000","last_change":"2023-10-09T10:52:10.394775+0000","last_active":"2023-10-09T10:53:17.829612+0000","last_peered":"2023-10-09T10:53:17.829612+0000","last_clean":"2023-10-09T10:53:17.829612+0000","last_became_active":"2023-10-09T10:52:10.390932+0000","last_became_peered":"2023-10-09T10:52:10.390932+0000","last_unstale":"2023-10-09T10:53:17.829612+0000","last_undegraded":"2023-10-09T10:53:17.829612+0000","last_fullsized":"2023-10-09T10:53:17.829612+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:00.176413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":178,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.831121+0000","last_change":"2023-10-09T10:53:14.828291+0000","last_active":"2023-10-09T10:53:17.831121+0000","last_peered":"2023-10-09T10:53:17.831121+0000","last_clean":"2023-10-09T10:53:17.831121+0000","last_became_active":"2023-10-09T10:53:14.827991+0000","last_became_peered":"2023-10-09T10:53:14.827991+0000","last_unstale":"2023-10-09T10:53:17.831121+0000","last_undegraded":"2023-10-09T10:53:17.831121+0000","last_fullsized":"2023-10-09T10:53:17.831121+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:08:14.283112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"271'798","reported_seq":814,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-10-09T10:53:23.235053+0000","last_change":"2023-10-09T10:53:14.822279+0000","last_active":"2023-10-09T10:53:23.235053+0000","last_peered":"2023-10-09T10:53:23.235053+0000","last_clean":"2023-10-09T10:53:23.235053+0000","last_became_active":"2023-10-09T10:53:14.821952+0000","last_became_peered":"2023-10-09T10:53:14.821952+0000","last_unstale":"2023-10-09T10:53:23.235053+0000","last_undegraded":"2023-10-09T10:53:23.235053+0000","last_fullsized":"2023-10-09T10:53:23.235053+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":798,"log_dups_size":0,"ondisk_log_size":798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:37:14.124652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52297728,"num_objects":798,"num_object_clones":0,"num_object_copies":2394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":798,"num_write_kb":51072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":636,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.836484+0000","last_change":"2023-10-09T10:51:01.315582+0000","last_active":"2023-10-09T10:53:17.836484+0000","last_peered":"2023-10-09T10:53:17.836484+0000","last_clean":"2023-10-09T10:53:17.836484+0000","last_became_active":"2023-10-09T10:51:01.314820+0000","last_became_peered":"2023-10-09T10:51:01.314820+0000","last_unstale":"2023-10-09T10:53:17.836484+0000","last_undegraded":"2023-10-09T10:53:17.836484+0000","last_fullsized":"2023-10-09T10:53:17.836484+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.829081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.829081+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001277842,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":291,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.828958+0000","last_change":"2023-10-09T10:53:14.841466+0000","last_active":"2023-10-09T10:53:16.828958+0000","last_peered":"2023-10-09T10:53:16.828958+0000","last_clean":"2023-10-09T10:53:16.828958+0000","last_became_active":"2023-10-09T10:53:14.831674+0000","last_became_peered":"2023-10-09T10:53:14.831674+0000","last_unstale":"2023-10-09T10:53:16.828958+0000","last_undegraded":"2023-10-09T10:53:16.828958+0000","last_fullsized":"2023-10-09T10:53:16.828958+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:02.612069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.827984+0000","last_change":"2023-10-09T10:52:10.412601+0000","last_active":"2023-10-09T10:53:17.827984+0000","last_peered":"2023-10-09T10:53:17.827984+0000","last_clean":"2023-10-09T10:53:17.827984+0000","last_became_active":"2023-10-09T10:52:10.412352+0000","last_became_peered":"2023-10-09T10:52:10.412352+0000","last_unstale":"2023-10-09T10:53:17.827984+0000","last_undegraded":"2023-10-09T10:53:17.827984+0000","last_fullsized":"2023-10-09T10:53:17.827984+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:48.842030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":290,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.825038+0000","last_change":"2023-10-09T10:53:14.835841+0000","last_active":"2023-10-09T10:53:16.825038+0000","last_peered":"2023-10-09T10:53:16.825038+0000","last_clean":"2023-10-09T10:53:16.825038+0000","last_became_active":"2023-10-09T10:53:14.835551+0000","last_became_peered":"2023-10-09T10:53:14.835551+0000","last_unstale":"2023-10-09T10:53:16.825038+0000","last_undegraded":"2023-10-09T10:53:16.825038+0000","last_fullsized":"2023-10-09T10:53:16.825038+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:21.498099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"271'391","reported_seq":404,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-10-09T10:53:21.374932+0000","last_change":"2023-10-09T10:53:14.832326+0000","last_active":"2023-10-09T10:53:21.374932+0000","last_peered":"2023-10-09T10:53:21.374932+0000","last_clean":"2023-10-09T10:53:21.374932+0000","last_became_active":"2023-10-09T10:53:14.832006+0000","last_became_peered":"2023-10-09T10:53:14.832006+0000","last_unstale":"2023-10-09T10:53:21.374932+0000","last_undegraded":"2023-10-09T10:53:21.374932+0000","last_fullsized":"2023-10-09T10:53:21.374932+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":391,"log_dups_size":0,"ondisk_log_size":391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:47.681235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25624576,"num_objects":391,"num_object_clones":0,"num_object_copies":1173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":391,"num_write_kb":25024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":647,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.841386+0000","last_change":"2023-10-09T10:53:17.841386+0000","last_active":"2023-10-09T10:53:17.841386+0000","last_peered":"2023-10-09T10:53:17.841386+0000","last_clean":"2023-10-09T10:53:17.841386+0000","last_became_active":"2023-10-09T10:53:17.840928+0000","last_became_peered":"2023-10-09T10:53:17.840928+0000","last_unstale":"2023-10-09T10:53:17.841386+0000","last_undegraded":"2023-10-09T10:53:17.841386+0000","last_fullsized":"2023-10-09T10:53:17.841386+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":158,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.828756+0000","last_change":"2023-10-09T10:53:14.838808+0000","last_active":"2023-10-09T10:53:16.828756+0000","last_peered":"2023-10-09T10:53:16.828756+0000","last_clean":"2023-10-09T10:53:16.828756+0000","last_became_active":"2023-10-09T10:53:14.838545+0000","last_became_peered":"2023-10-09T10:53:14.838545+0000","last_unstale":"2023-10-09T10:53:16.828756+0000","last_undegraded":"2023-10-09T10:53:16.828756+0000","last_fullsized":"2023-10-09T10:53:16.828756+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:12:37.675602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":172,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.831357+0000","last_change":"2023-10-09T10:52:10.421736+0000","last_active":"2023-10-09T10:53:17.831357+0000","last_peered":"2023-10-09T10:53:17.831357+0000","last_clean":"2023-10-09T10:53:17.831357+0000","last_became_active":"2023-10-09T10:52:10.387400+0000","last_became_peered":"2023-10-09T10:52:10.387400+0000","last_unstale":"2023-10-09T10:53:17.831357+0000","last_undegraded":"2023-10-09T10:53:17.831357+0000","last_fullsized":"2023-10-09T10:53:17.831357+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:23.931527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":212,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.836280+0000","last_change":"2023-10-09T10:51:50.371282+0000","last_active":"2023-10-09T10:53:17.836280+0000","last_peered":"2023-10-09T10:53:17.836280+0000","last_clean":"2023-10-09T10:53:17.836280+0000","last_became_active":"2023-10-09T10:51:50.370973+0000","last_became_peered":"2023-10-09T10:51:50.370973+0000","last_unstale":"2023-10-09T10:53:17.836280+0000","last_undegraded":"2023-10-09T10:53:17.836280+0000","last_fullsized":"2023-10-09T10:53:17.836280+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:15.603164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":647,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.824939+0000","last_change":"2023-10-09T10:52:55.416707+0000","last_active":"2023-10-09T10:53:16.824939+0000","last_peered":"2023-10-09T10:53:16.824939+0000","last_clean":"2023-10-09T10:53:16.824939+0000","last_became_active":"2023-10-09T10:52:55.416357+0000","last_became_peered":"2023-10-09T10:52:55.416357+0000","last_unstale":"2023-10-09T10:53:16.824939+0000","last_undegraded":"2023-10-09T10:53:16.824939+0000","last_fullsized":"2023-10-09T10:53:16.824939+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":177,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.831201+0000","last_change":"2023-10-09T10:53:14.823731+0000","last_active":"2023-10-09T10:53:17.831201+0000","last_peered":"2023-10-09T10:53:17.831201+0000","last_clean":"2023-10-09T10:53:17.831201+0000","last_became_active":"2023-10-09T10:53:14.823470+0000","last_became_peered":"2023-10-09T10:53:14.823470+0000","last_unstale":"2023-10-09T10:53:17.831201+0000","last_undegraded":"2023-10-09T10:53:17.831201+0000","last_fullsized":"2023-10-09T10:53:17.831201+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:36.927599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":37,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.829534+0000","last_change":"2023-10-09T10:52:10.391133+0000","last_active":"2023-10-09T10:53:17.829534+0000","last_peered":"2023-10-09T10:53:17.829534+0000","last_clean":"2023-10-09T10:53:17.829534+0000","last_became_active":"2023-10-09T10:52:10.390586+0000","last_became_peered":"2023-10-09T10:52:10.390586+0000","last_unstale":"2023-10-09T10:53:17.829534+0000","last_undegraded":"2023-10-09T10:53:17.829534+0000","last_fullsized":"2023-10-09T10:53:17.829534+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:42:12.394654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":579,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.843346+0000","last_change":"2023-10-09T10:53:17.843346+0000","last_active":"2023-10-09T10:53:17.843346+0000","last_peered":"2023-10-09T10:53:17.843346+0000","last_clean":"2023-10-09T10:53:17.843346+0000","last_became_active":"2023-10-09T10:53:17.842952+0000","last_became_peered":"2023-10-09T10:53:17.842952+0000","last_unstale":"2023-10-09T10:53:17.843346+0000","last_undegraded":"2023-10-09T10:53:17.843346+0000","last_fullsized":"2023-10-09T10:53:17.843346+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":69,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.836793+0000","last_change":"2023-10-09T10:52:10.438843+0000","last_active":"2023-10-09T10:53:17.836793+0000","last_peered":"2023-10-09T10:53:17.836793+0000","last_clean":"2023-10-09T10:53:17.836793+0000","last_became_active":"2023-10-09T10:52:10.384006+0000","last_became_peered":"2023-10-09T10:52:10.384006+0000","last_unstale":"2023-10-09T10:53:17.836793+0000","last_undegraded":"2023-10-09T10:53:17.836793+0000","last_fullsized":"2023-10-09T10:53:17.836793+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:47.904749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":259,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.827540+0000","last_change":"2023-10-09T10:52:10.382952+0000","last_active":"2023-10-09T10:53:16.827540+0000","last_peered":"2023-10-09T10:53:16.827540+0000","last_clean":"2023-10-09T10:53:16.827540+0000","last_became_active":"2023-10-09T10:52:10.382639+0000","last_became_peered":"2023-10-09T10:52:10.382639+0000","last_unstale":"2023-10-09T10:53:16.827540+0000","last_undegraded":"2023-10-09T10:53:16.827540+0000","last_fullsized":"2023-10-09T10:53:16.827540+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:47:26.158048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":221,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.836311+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:53:17.836311+0000","last_peered":"2023-10-09T10:53:17.836311+0000","last_clean":"2023-10-09T10:53:17.836311+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:53:17.836311+0000","last_undegraded":"2023-10-09T10:53:17.836311+0000","last_fullsized":"2023-10-09T10:53:17.836311+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":213,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.827091+0000","last_change":"2023-10-09T10:50:43.385150+0000","last_active":"2023-10-09T10:53:16.827091+0000","last_peered":"2023-10-09T10:53:16.827091+0000","last_clean":"2023-10-09T10:53:16.827091+0000","last_became_active":"2023-10-09T10:50:43.384788+0000","last_became_peered":"2023-10-09T10:50:43.384788+0000","last_unstale":"2023-10-09T10:53:16.827091+0000","last_undegraded":"2023-10-09T10:53:16.827091+0000","last_fullsized":"2023-10-09T10:53:16.827091+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:36.085910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":562,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.843186+0000","last_change":"2023-10-09T10:53:17.842907+0000","last_active":"2023-10-09T10:53:17.843186+0000","last_peered":"2023-10-09T10:53:17.843186+0000","last_clean":"2023-10-09T10:53:17.843186+0000","last_became_active":"2023-10-09T10:53:17.842371+0000","last_became_peered":"2023-10-09T10:53:17.842371+0000","last_unstale":"2023-10-09T10:53:17.843186+0000","last_undegraded":"2023-10-09T10:53:17.843186+0000","last_fullsized":"2023-10-09T10:53:17.843186+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:41.637984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:41.637984+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":274,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.830930+0000","last_change":"2023-10-09T10:52:10.379003+0000","last_active":"2023-10-09T10:53:16.830930+0000","last_peered":"2023-10-09T10:53:16.830930+0000","last_clean":"2023-10-09T10:53:16.830930+0000","last_became_active":"2023-10-09T10:52:10.378671+0000","last_became_peered":"2023-10-09T10:52:10.378671+0000","last_unstale":"2023-10-09T10:53:16.830930+0000","last_undegraded":"2023-10-09T10:53:16.830930+0000","last_fullsized":"2023-10-09T10:53:16.830930+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:07.193433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":287,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T10:53:16.826739+0000","last_change":"2023-10-09T10:46:32.227393+0000","last_active":"2023-10-09T10:53:16.826739+0000","last_peered":"2023-10-09T10:53:16.826739+0000","last_clean":"2023-10-09T10:53:16.826739+0000","last_became_active":"2023-10-09T10:46:32.227102+0000","last_became_peered":"2023-10-09T10:46:32.227102+0000","last_unstale":"2023-10-09T10:53:16.826739+0000","last_undegraded":"2023-10-09T10:53:16.826739+0000","last_fullsized":"2023-10-09T10:53:16.826739+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:53.171009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":559,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.829118+0000","last_change":"2023-10-09T10:52:55.417998+0000","last_active":"2023-10-09T10:53:17.829118+0000","last_peered":"2023-10-09T10:53:17.829118+0000","last_clean":"2023-10-09T10:53:17.829118+0000","last_became_active":"2023-10-09T10:52:55.417620+0000","last_became_peered":"2023-10-09T10:52:55.417620+0000","last_unstale":"2023-10-09T10:53:17.829118+0000","last_undegraded":"2023-10-09T10:53:17.829118+0000","last_fullsized":"2023-10-09T10:53:17.829118+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:33:35.022165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":163,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.831540+0000","last_change":"2023-10-09T10:46:32.262531+0000","last_active":"2023-10-09T10:53:17.831540+0000","last_peered":"2023-10-09T10:53:17.831540+0000","last_clean":"2023-10-09T10:53:17.831540+0000","last_became_active":"2023-10-09T10:46:32.241031+0000","last_became_peered":"2023-10-09T10:46:32.241031+0000","last_unstale":"2023-10-09T10:53:17.831540+0000","last_undegraded":"2023-10-09T10:53:17.831540+0000","last_fullsized":"2023-10-09T10:53:17.831540+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:45.776278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":255,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T10:53:17.837007+0000","last_change":"2023-10-09T10:52:10.392260+0000","last_active":"2023-10-09T10:53:17.837007+0000","last_peered":"2023-10-09T10:53:17.837007+0000","last_clean":"2023-10-09T10:53:17.837007+0000","last_became_active":"2023-10-09T10:52:10.391955+0000","last_became_peered":"2023-10-09T10:52:10.391955+0000","last_unstale":"2023-10-09T10:53:17.837007+0000","last_undegraded":"2023-10-09T10:53:17.837007+0000","last_fullsized":"2023-10-09T10:53:17.837007+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:51.768262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":6170,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":489881600,"num_objects":7475,"num_object_clones":0,"num_object_copies":22425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7475,"num_write_kb":478400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1367998464,"data_stored":1367998464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":7475,"ondisk_log_size":7475,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":557419,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":324,"num_read_kb":609,"num_write":159,"num_write_kb":1610,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1869953,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332321,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":746460,"kb_used_data":449172,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92998692,"statfs":{"total":95995035648,"available":95230660608,"internally_reserved":0,"allocated":459952128,"data_stored":458961196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:52:52 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":1.8480000000000001,"15min":1.073},"min":{"1min":0.28699999999999998,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":3.75,"5min":9.6379999999999999,"15min":9.6379999999999999},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.5900000000000001,"5min":2.133,"15min":1.2490000000000001},"min":{"1min":0.27900000000000003,"5min":0.248,"15min":0.24299999999999999},"max":{"1min":6.2300000000000004,"5min":16.731000000000002,"15min":16.731000000000002},"last":0.78400000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:52:56 2023","interfaces":[{"interface":"back","average":{"1min":3.3100000000000001,"5min":2.3180000000000001,"15min":1.3959999999999999},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":34.756,"5min":34.756,"15min":34.756},"last":0.32100000000000001},{"interface":"front","average":{"1min":2.863,"5min":1.9259999999999999,"15min":1.204},"min":{"1min":0.28999999999999998,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":35.658999999999999,"5min":35.658999999999999,"15min":35.658999999999999},"last":0.38300000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:52:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.954,"15min":1.2050000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.26400000000000001},"max":{"1min":4.4189999999999996,"5min":11.058,"15min":11.058},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.3500000000000001,"5min":1.7270000000000001,"15min":1.036},"min":{"1min":0.35199999999999998,"5min":0.28699999999999998,"15min":0.251},"max":{"1min":4.79,"5min":7.8120000000000003,"15min":7.8120000000000003},"last":0.57599999999999996}]},{"osd":3,"last update":"Mon Oct 9 10:52:52 2023","interfaces":[{"interface":"back","average":{"1min":1.4670000000000001,"5min":2.4020000000000001,"15min":1.4059999999999999},"min":{"1min":0.39500000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.2649999999999997,"5min":11.523,"15min":11.523},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.234,"5min":2.2330000000000001,"15min":1.3109999999999999},"min":{"1min":0.35599999999999998,"5min":0.34399999999999997,"15min":0.32800000000000001},"max":{"1min":4.4459999999999997,"5min":19.234999999999999,"15min":19.234999999999999},"last":1.2050000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:52:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":2.1219999999999999,"15min":1.1120000000000001},"min":{"1min":0.34300000000000003,"5min":0.27700000000000002,"15min":0.255},"max":{"1min":4.1820000000000004,"5min":16.695,"15min":16.695},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.96,"15min":1.0329999999999999},"min":{"1min":0.32200000000000001,"5min":0.28999999999999998,"15min":0.26200000000000001},"max":{"1min":4.375,"5min":11.109999999999999,"15min":11.109999999999999},"last":0.40699999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:52:30 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":2.1920000000000002,"15min":1.0900000000000001},"min":{"1min":0.32900000000000001,"5min":0.29699999999999999,"15min":0.247},"max":{"1min":4.7619999999999996,"5min":13.262,"15min":13.262},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.444,"5min":2.3969999999999998,"15min":1.25},"min":{"1min":0.39500000000000002,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":6.0300000000000002,"5min":11.741,"15min":11.741},"last":0.497}]},{"osd":6,"last update":"Mon Oct 9 10:52:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":2.5720000000000001,"15min":1.3220000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.28199999999999997},"max":{"1min":7.6660000000000004,"5min":14.913,"15min":14.913},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.7310000000000001,"5min":2.3809999999999998,"15min":1.198},"min":{"1min":0.39400000000000002,"5min":0.313,"15min":0.29399999999999998},"max":{"1min":12.268000000000001,"5min":19.187999999999999,"15min":19.187999999999999},"last":0.46500000000000002}]}]},{"osd":6,"up_from":44,"seq":188978561252,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":438932,"kb_used_data":133452,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93306220,"statfs":{"total":95995035648,"available":95545569280,"internally_reserved":0,"allocated":136654848,"data_stored":135672006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:52:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.7569999999999999,"15min":1.1000000000000001},"min":{"1min":0.222,"5min":0.216,"15min":0.216},"max":{"1min":5.0490000000000004,"5min":12.288,"15min":12.288},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.6579999999999999,"5min":1.895,"15min":1.1679999999999999},"min":{"1min":0.29699999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.5209999999999999,"5min":12.369999999999999,"15min":12.369999999999999},"last":0.78800000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:53:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.8009999999999999,"15min":1.1830000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":5.0819999999999999,"5min":15.502000000000001,"15min":16.747},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.7949999999999999,"15min":1.21},"min":{"1min":0.23599999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.8929999999999998,"5min":13.137,"15min":16.734000000000002},"last":0.36399999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:52:29 2023","interfaces":[{"interface":"back","average":{"1min":2.5510000000000002,"5min":2.0960000000000001,"15min":1.196},"min":{"1min":0.34000000000000002,"5min":0.28899999999999998,"15min":0.26800000000000002},"max":{"1min":7.3849999999999998,"5min":13.208,"15min":13.208},"last":0.80900000000000005},{"interface":"front","average":{"1min":1.5649999999999999,"5min":2.2650000000000001,"15min":1.323},"min":{"1min":0.315,"5min":0.29999999999999999,"15min":0.28399999999999997},"max":{"1min":4.2720000000000002,"5min":10.186,"15min":12.571},"last":0.626}]},{"osd":3,"last update":"Mon Oct 9 10:52:51 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":2.036,"15min":1.23},"min":{"1min":0.41199999999999998,"5min":0.255,"15min":0.255},"max":{"1min":7.4720000000000004,"5min":12.391,"15min":12.391},"last":0.83199999999999996},{"interface":"front","average":{"1min":2.6520000000000001,"5min":2.3159999999999998,"15min":1.4159999999999999},"min":{"1min":0.307,"5min":0.29999999999999999,"15min":0.28100000000000003},"max":{"1min":7.327,"5min":13.179,"15min":13.179},"last":3.024}]},{"osd":4,"last update":"Mon Oct 9 10:52:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.0449999999999999,"15min":1.0509999999999999},"min":{"1min":0.27700000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":9.0600000000000005,"5min":9.4060000000000006,"15min":9.4060000000000006},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.9470000000000001,"15min":1.03},"min":{"1min":0.32500000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.2380000000000004,"5min":12.481999999999999,"15min":12.481999999999999},"last":0.52000000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:52:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2480000000000002,"5min":2.0209999999999999,"15min":1.087},"min":{"1min":0.34200000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.0069999999999997,"5min":10.497,"15min":10.497},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.958,"5min":2.0019999999999998,"15min":1.093},"min":{"1min":0.33200000000000002,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":9.6769999999999996,"5min":12.456,"15min":12.456},"last":0.60799999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:52:35 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":2.3109999999999999,"15min":1.175},"min":{"1min":0.435,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.4359999999999999,"5min":23.358000000000001,"15min":23.358000000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.679,"15min":1.0229999999999999},"min":{"1min":0.39100000000000001,"5min":0.35099999999999998,"15min":0.34000000000000002},"max":{"1min":4.9569999999999999,"5min":12.422000000000001,"15min":12.422000000000001},"last":0.63900000000000001}]}]},{"osd":1,"up_from":85,"seq":365072220291,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1564728,"kb_used_data":1338800,"kb_used_omap":9,"kb_used_meta":225910,"kb_avail":92180424,"statfs":{"total":95995035648,"available":94392754176,"internally_reserved":0,"allocated":1370931200,"data_stored":1369516932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":231332841},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:52:58 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.986,"15min":1.3100000000000001},"min":{"1min":0.22,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":8.1750000000000007,"5min":18.027000000000001,"15min":18.027000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":2.9049999999999998,"5min":2.052,"15min":1.3380000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":16.739000000000001,"5min":16.739000000000001,"15min":16.739000000000001},"last":0.247}]},{"osd":2,"last update":"Mon Oct 9 10:53:21 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.429,"15min":1.0960000000000001},"min":{"1min":0.23699999999999999,"5min":0.222,"15min":0.19700000000000001},"max":{"1min":6.9530000000000003,"5min":9.141,"15min":15.356},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.6279999999999999,"15min":1.234},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.9950000000000001,"5min":14.186999999999999,"15min":14.234},"last":0.436}]},{"osd":3,"last update":"Mon Oct 9 10:52:51 2023","interfaces":[{"interface":"back","average":{"1min":2.5539999999999998,"5min":2.5590000000000002,"15min":1.5189999999999999},"min":{"1min":0.252,"5min":0.218,"15min":0.218},"max":{"1min":6.9089999999999998,"5min":18.274999999999999,"15min":18.274999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":3.5840000000000001,"5min":2.4390000000000001,"15min":1.4039999999999999},"min":{"1min":0.26100000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":14.169,"5min":14.169,"15min":15.468999999999999},"last":0.27700000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:52:58 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":1.736,"15min":1.165},"min":{"1min":0.26500000000000001,"5min":0.255,"15min":0.255},"max":{"1min":5.8559999999999999,"5min":12.167999999999999,"15min":12.167999999999999},"last":1.22},{"interface":"front","average":{"1min":1.5960000000000001,"5min":2.173,"15min":1.3420000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.6600000000000001,"5min":14.228999999999999,"15min":14.228999999999999},"last":0.47399999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:52:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1920000000000002,"5min":1.8140000000000001,"15min":1.1839999999999999},"min":{"1min":0.35199999999999998,"5min":0.28100000000000003,"15min":0.253},"max":{"1min":7.9299999999999997,"5min":9.4809999999999999,"15min":9.7720000000000002},"last":0.96399999999999997},{"interface":"front","average":{"1min":1.9710000000000001,"5min":2.3460000000000001,"15min":1.429},"min":{"1min":0.34100000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.915,"5min":17.085000000000001,"15min":17.085000000000001},"last":2.0630000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:52:58 2023","interfaces":[{"interface":"back","average":{"1min":2.4769999999999999,"5min":2.0819999999999999,"15min":1.306},"min":{"1min":0.32800000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.4049999999999994,"5min":10.425000000000001,"15min":10.425000000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":2.056,"5min":1.9970000000000001,"15min":1.3129999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":5.9039999999999999,"5min":12.231,"15min":12.231},"last":1.0609999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:52:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5739999999999998,"5min":2.1080000000000001,"15min":1.3260000000000001},"min":{"1min":0.36299999999999999,"5min":0.34300000000000003,"15min":0.34000000000000002},"max":{"1min":16.312000000000001,"5min":16.312000000000001,"15min":16.312000000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":2.194,"5min":2.1659999999999999,"15min":1.367},"min":{"1min":0.35799999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.8529999999999998,"5min":14.273999999999999,"15min":14.273999999999999},"last":0.45800000000000002}]}]},{"osd":0,"up_from":80,"seq":343597383825,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":650428,"kb_used_data":263092,"kb_used_omap":28,"kb_used_meta":387299,"kb_avail":93094724,"statfs":{"total":95995035648,"available":95328997376,"internally_reserved":0,"allocated":269406208,"data_stored":267923714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":396595024},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:52:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":2.1379999999999999,"15min":1.3500000000000001},"min":{"1min":0.215,"5min":0.191,"15min":0.191},"max":{"1min":4.8769999999999998,"5min":14.311,"15min":25.475000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.8799999999999999,"5min":2.1299999999999999,"15min":1.363},"min":{"1min":0.29699999999999999,"5min":0.24099999999999999,"15min":0.188},"max":{"1min":8.8100000000000005,"5min":12.276,"15min":20.832999999999998},"last":0.36799999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:52:29 2023","interfaces":[{"interface":"back","average":{"1min":2.3820000000000001,"5min":2.1080000000000001,"15min":1.2310000000000001},"min":{"1min":0.25800000000000001,"5min":0.23999999999999999,"15min":0.19700000000000001},"max":{"1min":9.1460000000000008,"5min":17.927,"15min":17.927},"last":0.51300000000000001},{"interface":"front","average":{"1min":2.1080000000000001,"5min":2.1459999999999999,"15min":1.341},"min":{"1min":0.29399999999999998,"5min":0.25700000000000001,"15min":0.20399999999999999},"max":{"1min":7.4790000000000001,"5min":14.111000000000001,"15min":14.391999999999999},"last":0.44800000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:52:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1040000000000001,"5min":2.3879999999999999,"15min":1.645},"min":{"1min":0.29199999999999998,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":13.166,"5min":13.166,"15min":14.926},"last":2.3570000000000002},{"interface":"front","average":{"1min":1.6519999999999999,"5min":2.4860000000000002,"15min":1.54},"min":{"1min":0.27800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":12.228999999999999,"5min":14.391999999999999,"15min":14.391999999999999},"last":2.9510000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:52:45 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":2.1970000000000001,"15min":1.3480000000000001},"min":{"1min":0.34999999999999998,"5min":0.318,"15min":0.26200000000000001},"max":{"1min":7.673,"5min":9.6129999999999995,"15min":9.6129999999999995},"last":1.21},{"interface":"front","average":{"1min":2.2799999999999998,"5min":2.5569999999999999,"15min":1.5269999999999999},"min":{"1min":0.37,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.3119999999999994,"5min":12.318,"15min":12.318},"last":0.41699999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:52:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3999999999999999,"5min":2.4980000000000002,"15min":1.52},"min":{"1min":0.35399999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.3360000000000003,"5min":12.157,"15min":12.157},"last":0.65600000000000003},{"interface":"front","average":{"1min":2.004,"5min":2.4609999999999999,"15min":1.425},"min":{"1min":0.33900000000000002,"5min":0.317,"15min":0.29499999999999998},"max":{"1min":9.0109999999999992,"5min":14.446,"15min":14.446},"last":0.71599999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:52:45 2023","interfaces":[{"interface":"back","average":{"1min":2.7509999999999999,"5min":2.7490000000000001,"15min":1.5900000000000001},"min":{"1min":0.51100000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.2850000000000001,"5min":12.207000000000001,"15min":12.207000000000001},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.9019999999999999,"5min":2.0659999999999998,"15min":1.2649999999999999},"min":{"1min":0.39200000000000002,"5min":0.307,"15min":0.307},"max":{"1min":9.0449999999999999,"5min":14.420999999999999,"15min":14.420999999999999},"last":0.74099999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:52:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":2.6920000000000002,"15min":1.498},"min":{"1min":0.374,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.0709999999999997,"5min":14.48,"15min":14.48},"last":1.1859999999999999},{"interface":"front","average":{"1min":1.962,"5min":2.3700000000000001,"15min":1.4910000000000001},"min":{"1min":0.38400000000000001,"5min":0.36599999999999999,"15min":0.32200000000000001},"max":{"1min":7.702,"5min":9.6440000000000001,"15min":14.394},"last":1.325}]}]},{"osd":2,"up_from":0,"seq":386547056765,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":95,"seq":408021893239,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591416,"kb_used_data":60976,"kb_used_omap":7,"kb_used_meta":530424,"kb_avail":93153736,"statfs":{"total":95995035648,"available":95389425664,"internally_reserved":0,"allocated":62439424,"data_stored":60953535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":543154792},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:52:57 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":2.1659999999999999,"15min":1.2669999999999999},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":10.179,"5min":15.255000000000001,"15min":15.255000000000001},"last":0.71799999999999997},{"interface":"front","average":{"1min":3.1349999999999998,"5min":2.9529999999999998,"15min":1.4990000000000001},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":13.324999999999999,"5min":14.798,"15min":14.798},"last":0.373}]},{"osd":1,"last update":"Mon Oct 9 10:52:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6259999999999999,"5min":2.7629999999999999,"15min":1.597},"min":{"1min":0.23100000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":11.712999999999999,"5min":11.757,"15min":14.707000000000001},"last":0.70799999999999996},{"interface":"front","average":{"1min":1.988,"5min":2.7309999999999999,"15min":1.431},"min":{"1min":0.29299999999999998,"5min":0.249,"15min":0.193},"max":{"1min":11.741,"5min":15.443,"15min":15.443},"last":0.77600000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:52:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6910000000000001,"5min":2.2130000000000001,"15min":1.4179999999999999},"min":{"1min":0.28199999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.9399999999999995,"5min":11.795,"15min":13.853},"last":1.5409999999999999},{"interface":"front","average":{"1min":1.8029999999999999,"5min":2.3199999999999998,"15min":1.409},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":8.7490000000000006,"5min":10.557,"15min":10.557},"last":1.5589999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:52:57 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":2.3370000000000002,"15min":1.3759999999999999},"min":{"1min":0.33900000000000002,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":8.859,"5min":20.388000000000002,"15min":20.388000000000002},"last":0.76200000000000001},{"interface":"front","average":{"1min":1.4590000000000001,"5min":2.226,"15min":1.377},"min":{"1min":0.28399999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.992,"5min":19.085999999999999,"15min":19.085999999999999},"last":0.54500000000000004}]},{"osd":5,"last update":"Mon Oct 9 10:52:57 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":2.4790000000000001,"15min":1.623},"min":{"1min":0.32200000000000001,"5min":0.29399999999999998,"15min":0.29199999999999998},"max":{"1min":4.157,"5min":10.619999999999999,"15min":51.509999999999998},"last":0.51800000000000002},{"interface":"front","average":{"1min":2.1000000000000001,"5min":2.3140000000000001,"15min":1.728},"min":{"1min":0.32400000000000001,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":10.193,"5min":11.082000000000001,"15min":55.338999999999999},"last":0.35799999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:52:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8240000000000001,"5min":2.0899999999999999,"15min":1.3460000000000001},"min":{"1min":0.29899999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.8490000000000002,"5min":12.361000000000001,"15min":12.361000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.548,"5min":2.4340000000000002,"15min":1.603},"min":{"1min":0.39300000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":10.148999999999999,"5min":11.24,"15min":13.891999999999999},"last":0.40500000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:52:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8240000000000001,"5min":2.375,"15min":1.7689999999999999},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.31},"max":{"1min":10.222,"5min":11.819000000000001,"15min":51.744},"last":0.435},{"interface":"front","average":{"1min":1.589,"5min":2.609,"15min":1.6839999999999999},"min":{"1min":0.39700000000000002,"5min":0.39200000000000002,"15min":0.31900000000000001},"max":{"1min":8.8160000000000007,"5min":15.212999999999999,"15min":53.399000000000001},"last":0.746}]}]},{"osd":4,"up_from":30,"seq":128849019115,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":521048,"kb_used_data":215568,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93224104,"statfs":{"total":95995035648,"available":95461482496,"internally_reserved":0,"allocated":220741632,"data_stored":219754505,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:52:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":1.5509999999999999,"15min":1.04},"min":{"1min":0.217,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":6.4660000000000002,"5min":9.4329999999999998,"15min":10.387},"last":0.83799999999999997},{"interface":"front","average":{"1min":3.1030000000000002,"5min":2.3559999999999999,"15min":1.3740000000000001},"min":{"1min":0.30199999999999999,"5min":0.217,"15min":0.217},"max":{"1min":10.847,"5min":12.58,"15min":12.58},"last":0.32800000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:52:49 2023","interfaces":[{"interface":"back","average":{"1min":2.032,"5min":1.7390000000000001,"15min":1.1479999999999999},"min":{"1min":0.26900000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.8689999999999998,"5min":14.099,"15min":14.099},"last":0.45200000000000001},{"interface":"front","average":{"1min":2.8700000000000001,"5min":2.2330000000000001,"15min":1.3640000000000001},"min":{"1min":0.31,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":10.313000000000001,"5min":14.089,"15min":14.089},"last":0.47699999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:52:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":2.1139999999999999,"15min":1.262},"min":{"1min":0.29699999999999999,"5min":0.27500000000000002,"15min":0.253},"max":{"1min":12.204000000000001,"5min":12.204000000000001,"15min":12.204000000000001},"last":0.374},{"interface":"front","average":{"1min":1.79,"5min":2.3450000000000002,"15min":1.357},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26300000000000001},"max":{"1min":6.8970000000000002,"5min":20.545999999999999,"15min":20.545999999999999},"last":0.46500000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:52:46 2023","interfaces":[{"interface":"back","average":{"1min":2.6659999999999999,"5min":2.1619999999999999,"15min":1.4019999999999999},"min":{"1min":0.29599999999999999,"5min":0.29299999999999998,"15min":0.27400000000000002},"max":{"1min":10.266999999999999,"5min":10.266999999999999,"15min":10.266999999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.2050000000000001,"5min":2.5430000000000001,"15min":1.4970000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.8369999999999997,"5min":9.6929999999999996,"15min":9.6929999999999996},"last":0.49299999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:52:49 2023","interfaces":[{"interface":"back","average":{"1min":2.7799999999999998,"5min":2.7229999999999999,"15min":1.3600000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":10.529999999999999,"5min":24.149999999999999,"15min":24.149999999999999},"last":0.313},{"interface":"front","average":{"1min":1.5940000000000001,"5min":2.2000000000000002,"15min":1.202},"min":{"1min":0.32900000000000001,"5min":0.27100000000000002,"15min":0.252},"max":{"1min":6.9210000000000003,"5min":20.422000000000001,"15min":20.422000000000001},"last":0.42599999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.7010000000000001,"15min":1.1080000000000001},"min":{"1min":0.29399999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.2880000000000003,"5min":14.897,"15min":14.897},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.1060000000000001,"5min":2.1739999999999999,"15min":1.323},"min":{"1min":0.375,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":4.3520000000000003,"5min":14.272,"15min":14.272},"last":0.40899999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:52:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":2.198,"15min":1.1870000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":10.385999999999999,"5min":10.385999999999999,"15min":10.385999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.6779999999999999,"5min":1.8280000000000001,"15min":1.0409999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":6.9710000000000001,"5min":7.423,"15min":7.423},"last":0.53100000000000003}]}]},{"osd":5,"up_from":37,"seq":158913790184,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":521168,"kb_used_data":219784,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93223984,"statfs":{"total":95995035648,"available":95461359616,"internally_reserved":0,"allocated":225058816,"data_stored":224083889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:52:54 2023","interfaces":[{"interface":"back","average":{"1min":2.407,"5min":2.258,"15min":1.21},"min":{"1min":0.245,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.5830000000000002,"5min":14.837,"15min":14.837},"last":0.61199999999999999},{"interface":"front","average":{"1min":2.4630000000000001,"5min":2.302,"15min":1.2729999999999999},"min":{"1min":0.315,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.6909999999999998,"5min":14.863,"15min":14.863},"last":0.628}]},{"osd":1,"last update":"Mon Oct 9 10:53:00 2023","interfaces":[{"interface":"back","average":{"1min":1.9870000000000001,"5min":2.351,"15min":1.4419999999999999},"min":{"1min":0.32200000000000001,"5min":0.26000000000000001,"15min":0.251},"max":{"1min":5.7839999999999998,"5min":10.682,"15min":12.999000000000001},"last":0.56899999999999995},{"interface":"front","average":{"1min":2.7229999999999999,"5min":1.8740000000000001,"15min":1.234},"min":{"1min":0.28199999999999997,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":7.6059999999999999,"5min":12.109,"15min":12.109},"last":0.46000000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:52:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":2.4630000000000001,"15min":1.351},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.0590000000000002,"5min":11.393000000000001,"15min":11.393000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":3.3519999999999999,"5min":2.0539999999999998,"15min":1.177},"min":{"1min":0.314,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":15.087999999999999,"5min":15.087999999999999,"15min":15.087999999999999},"last":0.65100000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:52:54 2023","interfaces":[{"interface":"back","average":{"1min":2.456,"5min":2.8330000000000002,"15min":1.726},"min":{"1min":0.35299999999999998,"5min":0.28599999999999998,"15min":0.25700000000000001},"max":{"1min":4.8689999999999998,"5min":63.508000000000003,"15min":63.508000000000003},"last":0.434},{"interface":"front","average":{"1min":2.4369999999999998,"5min":2.9670000000000001,"15min":1.7509999999999999},"min":{"1min":0.32600000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.0679999999999996,"5min":62.418999999999997,"15min":62.418999999999997},"last":0.55700000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:53:00 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":2.6600000000000001,"15min":1.3240000000000001},"min":{"1min":0.27200000000000002,"5min":0.24099999999999999,"15min":0.23200000000000001},"max":{"1min":6.6120000000000001,"5min":123.241,"15min":123.241},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.97799999999999998,"5min":3,"15min":1.486},"min":{"1min":0.311,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.5090000000000003,"5min":123.318,"15min":123.318},"last":0.47199999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:53:16 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":1.841,"15min":1.163},"min":{"1min":0.34399999999999997,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.5430000000000001,"5min":11.098000000000001,"15min":11.098000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.105,"5min":1.9850000000000001,"15min":1.145},"min":{"1min":0.35099999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.2969999999999997,"5min":20.256,"15min":20.256},"last":0.58399999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:52:38 2023","interfaces":[{"interface":"back","average":{"1min":1.988,"5min":2.1949999999999998,"15min":1.1419999999999999},"min":{"1min":0.35899999999999999,"5min":0.315,"15min":0.28699999999999998},"max":{"1min":6.9710000000000001,"5min":10.739000000000001,"15min":10.739000000000001},"last":0.59899999999999998},{"interface":"front","average":{"1min":2.194,"5min":1.8799999999999999,"15min":1.0549999999999999},"min":{"1min":0.34699999999999998,"5min":0.33200000000000002,"15min":0.32900000000000001},"max":{"1min":10.858000000000001,"5min":14.154,"15min":14.154},"last":0.54500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":309329920,"data_stored":309329920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":220659712,"data_stored":220659712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":94502912,"data_stored":94502912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":161218560,"data_stored":161218560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":250281984,"data_stored":250281984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":187039744,"data_stored":187039744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":144965632,"data_stored":144965632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:53:30.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 41 2023-10-09T10:53:30.179 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:30.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:30.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:30.284 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:30.328 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:30.363 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:30.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:30.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:30.473 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:30.518 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:30.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:30 smithi155 ceph-mon[184854]: pgmap v640: 119 pgs: 119 active+clean; 1.2 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-10-09T10:53:30.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:30 smithi155 ceph-mon[184854]: 1.19 scrub starts 2023-10-09T10:53:30.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:30 smithi155 ceph-mon[184854]: 1.19 scrub ok 2023-10-09T10:53:30.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:53:30.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:30 smithi125 ceph-mon[195507]: pgmap v640: 119 pgs: 119 active+clean; 1.2 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-10-09T10:53:30.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:30 smithi125 ceph-mon[195507]: 1.19 scrub starts 2023-10-09T10:53:30.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:30 smithi125 ceph-mon[195507]: 1.19 scrub ok 2023-10-09T10:53:30.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:53:30.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:30 smithi125 ceph-mon[200197]: pgmap v640: 119 pgs: 119 active+clean; 1.2 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-10-09T10:53:30.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:30 smithi125 ceph-mon[200197]: 1.19 scrub starts 2023-10-09T10:53:30.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:30 smithi125 ceph-mon[200197]: 1.19 scrub ok 2023-10-09T10:53:30.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:53:30.935 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 12 16 29928 29912 155.732 127.625 0.0117444 0.00641529 2023-10-09T10:53:31.000 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:53:31.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:53:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:53:31] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.43.0" 2023-10-09T10:53:31.935 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 13 16 31973 31957 153.584 127.812 0.00654014 0.00650669 2023-10-09T10:53:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:31 smithi155 ceph-mon[184854]: 1.1 scrub starts 2023-10-09T10:53:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:31 smithi155 ceph-mon[184854]: 1.1 scrub ok 2023-10-09T10:53:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:31 smithi155 ceph-mon[184854]: 1.4 scrub starts 2023-10-09T10:53:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:31 smithi155 ceph-mon[184854]: 1.4 scrub ok 2023-10-09T10:53:32.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:31 smithi125 ceph-mon[195507]: 1.1 scrub starts 2023-10-09T10:53:32.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:31 smithi125 ceph-mon[195507]: 1.1 scrub ok 2023-10-09T10:53:32.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:31 smithi125 ceph-mon[195507]: 1.4 scrub starts 2023-10-09T10:53:32.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:31 smithi125 ceph-mon[195507]: 1.4 scrub ok 2023-10-09T10:53:32.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:31 smithi125 ceph-mon[200197]: 1.1 scrub starts 2023-10-09T10:53:32.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:31 smithi125 ceph-mon[200197]: 1.1 scrub ok 2023-10-09T10:53:32.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:31 smithi125 ceph-mon[200197]: 1.4 scrub starts 2023-10-09T10:53:32.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:31 smithi125 ceph-mon[200197]: 1.4 scrub ok 2023-10-09T10:53:32.111 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:32.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:32.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:32.119 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:32.209 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:32.217 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:32.275 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:32.308 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:32.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:32.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:32.322 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:32.416 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:53:32.465 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:32.501 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:53:32.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:32.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:53:32.605 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:32.654 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:32.687 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:32.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:32.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:32.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:32 smithi155 ceph-mon[184854]: pgmap v641: 119 pgs: 119 active+clean; 1.5 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T10:53:32.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:32 smithi155 ceph-mon[184854]: 1.9 scrub starts 2023-10-09T10:53:32.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:32 smithi155 ceph-mon[184854]: 1.9 scrub ok 2023-10-09T10:53:32.795 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:32.844 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:32.876 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:32.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:32.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:32.935 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 14 15 33831 33816 150.913 116.188 0.00855803 0.00662281 2023-10-09T10:53:32.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:33.030 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:33.065 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:33.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:33.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:33.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:32 smithi125 ceph-mon[195507]: pgmap v641: 119 pgs: 119 active+clean; 1.5 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T10:53:33.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:32 smithi125 ceph-mon[195507]: 1.9 scrub starts 2023-10-09T10:53:33.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:32 smithi125 ceph-mon[195507]: 1.9 scrub ok 2023-10-09T10:53:33.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:32 smithi125 ceph-mon[200197]: pgmap v641: 119 pgs: 119 active+clean; 1.5 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T10:53:33.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:32 smithi125 ceph-mon[200197]: 1.9 scrub starts 2023-10-09T10:53:33.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:32 smithi125 ceph-mon[200197]: 1.9 scrub ok 2023-10-09T10:53:33.171 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:33.214 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:33.292 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:33.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:33.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:33.462 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:33.505 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:33.935 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 15 16 35639 35623 148.381 112.938 0.00647905 0.00673502 2023-10-09T10:53:34.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:33 smithi125 ceph-mon[195507]: 1.17 scrub starts 2023-10-09T10:53:34.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:33 smithi125 ceph-mon[195507]: 1.17 scrub ok 2023-10-09T10:53:34.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:33 smithi125 ceph-mon[195507]: pgmap v642: 119 pgs: 119 active+clean; 1.9 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 151 MiB/s wr, 2.41k op/s 2023-10-09T10:53:34.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:33 smithi125 ceph-mon[200197]: 1.17 scrub starts 2023-10-09T10:53:34.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:33 smithi125 ceph-mon[200197]: 1.17 scrub ok 2023-10-09T10:53:34.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:33 smithi125 ceph-mon[200197]: pgmap v642: 119 pgs: 119 active+clean; 1.9 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 151 MiB/s wr, 2.41k op/s 2023-10-09T10:53:34.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:33 smithi155 ceph-mon[184854]: 1.17 scrub starts 2023-10-09T10:53:34.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:33 smithi155 ceph-mon[184854]: 1.17 scrub ok 2023-10-09T10:53:34.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:33 smithi155 ceph-mon[184854]: pgmap v642: 119 pgs: 119 active+clean; 1.9 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 151 MiB/s wr, 2.41k op/s 2023-10-09T10:53:34.509 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:53:34.510 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:53:34.510 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:53:34.510 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:53:34.510 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:53:34.936 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 16 16 37435 37419 146.122 112.25 0.0088468 0.00683836 2023-10-09T10:53:35.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:34 smithi155 ceph-mon[184854]: 1.7 scrub starts 2023-10-09T10:53:35.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:34 smithi155 ceph-mon[184854]: 1.7 scrub ok 2023-10-09T10:53:35.300 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:35.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:35.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:35.307 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:34 smithi125 ceph-mon[195507]: 1.7 scrub starts 2023-10-09T10:53:35.307 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:34 smithi125 ceph-mon[195507]: 1.7 scrub ok 2023-10-09T10:53:35.307 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:34 smithi125 ceph-mon[200197]: 1.7 scrub starts 2023-10-09T10:53:35.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:34 smithi125 ceph-mon[200197]: 1.7 scrub ok 2023-10-09T10:53:35.408 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:35.478 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:35.935 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 17 16 39236 39220 144.147 112.562 0.00379753 0.00691939 2023-10-09T10:53:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:35 smithi155 ceph-mon[184854]: 1.1d scrub starts 2023-10-09T10:53:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:35 smithi155 ceph-mon[184854]: 1.1d scrub ok 2023-10-09T10:53:36.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:35 smithi155 ceph-mon[184854]: 1.1b scrub starts 2023-10-09T10:53:36.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:35 smithi155 ceph-mon[184854]: 1.1b scrub ok 2023-10-09T10:53:36.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:35 smithi155 ceph-mon[184854]: pgmap v643: 119 pgs: 119 active+clean; 2.0 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-10-09T10:53:36.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:35 smithi125 ceph-mon[195507]: 1.1d scrub starts 2023-10-09T10:53:36.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:35 smithi125 ceph-mon[195507]: 1.1d scrub ok 2023-10-09T10:53:36.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:35 smithi125 ceph-mon[195507]: 1.1b scrub starts 2023-10-09T10:53:36.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:35 smithi125 ceph-mon[195507]: 1.1b scrub ok 2023-10-09T10:53:36.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:35 smithi125 ceph-mon[195507]: pgmap v643: 119 pgs: 119 active+clean; 2.0 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-10-09T10:53:36.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:35 smithi125 ceph-mon[200197]: 1.1d scrub starts 2023-10-09T10:53:36.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:35 smithi125 ceph-mon[200197]: 1.1d scrub ok 2023-10-09T10:53:36.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:35 smithi125 ceph-mon[200197]: 1.1b scrub starts 2023-10-09T10:53:36.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:35 smithi125 ceph-mon[200197]: 1.1b scrub ok 2023-10-09T10:53:36.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:35 smithi125 ceph-mon[200197]: pgmap v643: 119 pgs: 119 active+clean; 2.0 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-10-09T10:53:36.935 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 18 16 40755 40739 141.414 94.9375 0.00341186 0.0070662 2023-10-09T10:53:37.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:36 smithi155 ceph-mon[184854]: 1.2 scrub starts 2023-10-09T10:53:37.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:36 smithi155 ceph-mon[184854]: 1.2 scrub ok 2023-10-09T10:53:37.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:36 smithi155 ceph-mon[184854]: 1.1e scrub starts 2023-10-09T10:53:37.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:36 smithi155 ceph-mon[184854]: 1.1e scrub ok 2023-10-09T10:53:37.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:36 smithi125 ceph-mon[195507]: 1.2 scrub starts 2023-10-09T10:53:37.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:36 smithi125 ceph-mon[195507]: 1.2 scrub ok 2023-10-09T10:53:37.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:36 smithi125 ceph-mon[195507]: 1.1e scrub starts 2023-10-09T10:53:37.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:36 smithi125 ceph-mon[195507]: 1.1e scrub ok 2023-10-09T10:53:37.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:36 smithi125 ceph-mon[200197]: 1.2 scrub starts 2023-10-09T10:53:37.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:36 smithi125 ceph-mon[200197]: 1.2 scrub ok 2023-10-09T10:53:37.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:36 smithi125 ceph-mon[200197]: 1.1e scrub starts 2023-10-09T10:53:37.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:36 smithi125 ceph-mon[200197]: 1.1e scrub ok 2023-10-09T10:53:37.936 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 19 16 42455 42439 139.563 106.25 0.0142722 0.00716035 2023-10-09T10:53:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:37 smithi125 ceph-mon[195507]: 1.f scrub starts 2023-10-09T10:53:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:37 smithi125 ceph-mon[195507]: 1.f scrub ok 2023-10-09T10:53:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:37 smithi125 ceph-mon[195507]: pgmap v644: 119 pgs: 119 active+clean; 2.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-09T10:53:38.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:37 smithi125 ceph-mon[200197]: 1.f scrub starts 2023-10-09T10:53:38.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:37 smithi125 ceph-mon[200197]: 1.f scrub ok 2023-10-09T10:53:38.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:37 smithi125 ceph-mon[200197]: pgmap v644: 119 pgs: 119 active+clean; 2.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-09T10:53:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:37 smithi155 ceph-mon[184854]: 1.f scrub starts 2023-10-09T10:53:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:37 smithi155 ceph-mon[184854]: 1.f scrub ok 2023-10-09T10:53:38.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:37 smithi155 ceph-mon[184854]: pgmap v644: 119 pgs: 119 active+clean; 2.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-09T10:53:38.937 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:53:38.938016+0000 min lat: 0.00119012 max lat: 0.134094 avg lat: 0.00724262 2023-10-09T10:53:38.937 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:53:38.937 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 20 16 44190 44174 137.996 108.438 0.00543562 0.00724262 2023-10-09T10:53:39.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:38 smithi125 ceph-mon[200197]: 1.16 scrub starts 2023-10-09T10:53:39.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:38 smithi125 ceph-mon[200197]: 1.16 scrub ok 2023-10-09T10:53:39.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:38 smithi125 ceph-mon[200197]: 1.1a scrub starts 2023-10-09T10:53:39.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:38 smithi125 ceph-mon[200197]: 1.1a scrub ok 2023-10-09T10:53:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:38 smithi125 ceph-mon[195507]: 1.16 scrub starts 2023-10-09T10:53:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:38 smithi125 ceph-mon[195507]: 1.16 scrub ok 2023-10-09T10:53:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:38 smithi125 ceph-mon[195507]: 1.1a scrub starts 2023-10-09T10:53:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:38 smithi125 ceph-mon[195507]: 1.1a scrub ok 2023-10-09T10:53:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:38 smithi155 ceph-mon[184854]: 1.16 scrub starts 2023-10-09T10:53:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:38 smithi155 ceph-mon[184854]: 1.16 scrub ok 2023-10-09T10:53:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:38 smithi155 ceph-mon[184854]: 1.1a scrub starts 2023-10-09T10:53:39.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:38 smithi155 ceph-mon[184854]: 1.1a scrub ok 2023-10-09T10:53:39.937 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 21 16 45872 45856 136.431 105.125 0.0124617 0.00732558 2023-10-09T10:53:40.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:39 smithi125 ceph-mon[195507]: 1.e scrub starts 2023-10-09T10:53:40.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:39 smithi125 ceph-mon[195507]: 1.e scrub ok 2023-10-09T10:53:40.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:39 smithi125 ceph-mon[195507]: pgmap v645: 119 pgs: 119 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-09T10:53:40.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:39 smithi125 ceph-mon[200197]: 1.e scrub starts 2023-10-09T10:53:40.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:39 smithi125 ceph-mon[200197]: 1.e scrub ok 2023-10-09T10:53:40.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:39 smithi125 ceph-mon[200197]: pgmap v645: 119 pgs: 119 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-09T10:53:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:39 smithi155 ceph-mon[184854]: 1.e scrub starts 2023-10-09T10:53:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:39 smithi155 ceph-mon[184854]: 1.e scrub ok 2023-10-09T10:53:40.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:39 smithi155 ceph-mon[184854]: pgmap v645: 119 pgs: 119 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-09T10:53:40.501 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:53:40.938 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 22 16 47464 47448 134.747 99.5 0.00767562 0.00741698 2023-10-09T10:53:41.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:41 smithi125 ceph-mon[195507]: 1.18 scrub starts 2023-10-09T10:53:41.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:41 smithi125 ceph-mon[195507]: 1.18 scrub ok 2023-10-09T10:53:41.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:53:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:53:41] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.43.0" 2023-10-09T10:53:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:41 smithi125 ceph-mon[200197]: 1.18 scrub starts 2023-10-09T10:53:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:41 smithi125 ceph-mon[200197]: 1.18 scrub ok 2023-10-09T10:53:41.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:41 smithi155 ceph-mon[184854]: 1.18 scrub starts 2023-10-09T10:53:41.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:41 smithi155 ceph-mon[184854]: 1.18 scrub ok 2023-10-09T10:53:41.938 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 23 16 49016 49000 133.106 97 0.0136749 0.00750822 2023-10-09T10:53:42.134 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:42.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:42.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:42.239 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:53:42.262 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:42 smithi125 ceph-mon[195507]: pgmap v646: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T10:53:42.263 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:42 smithi125 ceph-mon[200197]: pgmap v646: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T10:53:42.289 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:42.320 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:53:42.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:42.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:53:42.326 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:42 smithi155 ceph-mon[184854]: pgmap v646: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T10:53:42.426 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:42.462 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:42.494 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:42.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:42.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:42.583 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:42.598 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:42.601 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:53:42.643 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:53:42.682 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:42.693 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:53:42.721 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:42.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:42.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:42.744 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:53:42.801 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:53:42.830 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:53:42.840 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:53:42.845 INFO:teuthology.orchestra.run.smithi125.stderr:Error EINVAL: specified pgp_num 41 > pg_num 31 2023-10-09T10:53:42.873 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:53:42.897 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:42.906 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:53:42.920 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:53:42.920 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:53:42.920 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:53:42.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:53:42.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:53:42.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:53:42.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:53:42.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:53:42.924 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:53:42.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:42.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:53:42.938 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 24 16 50207 50191 130.662 74.4375 0.00672139 0.00764873 2023-10-09T10:53:43.029 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:43.091 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:43.096 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:53:43.127 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:43.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:43.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:43 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2221331256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:53:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:43 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1836850985' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:53:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:43 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:53:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:43 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/949138332' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-10-09T10:53:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:43 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-10-09T10:53:43.236 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:43.262 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:43.281 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:43.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:43.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:43.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:43 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2221331256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:53:43.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:43 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1836850985' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:53:43.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:43 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:53:43.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:43 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/949138332' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-10-09T10:53:43.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:43 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-10-09T10:53:43.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:43 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2221331256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:53:43.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:43 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1836850985' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:53:43.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:43 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:53:43.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:43 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/949138332' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-10-09T10:53:43.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:43 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-10-09T10:53:43.383 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:53:43.441 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:43.483 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:53:43.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:43.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:53:43.588 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:43.643 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:43.938 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 25 16 51545 51529 128.781 83.625 0.0134523 0.00776 2023-10-09T10:53:44.328 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:44.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:44.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:44.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:44 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:53:44.345 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:44 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T10:53:44.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:44 smithi125 ceph-mon[200197]: failed to encode map e272 with expected crc 2023-10-09T10:53:44.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:44 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:53:44.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:44 smithi125 ceph-mon[200197]: osdmap e272: 8 total, 8 up, 7 in 2023-10-09T10:53:44.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:44 smithi125 ceph-mon[200197]: failed to encode map e272 with expected crc 2023-10-09T10:53:44.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:44 smithi125 ceph-mon[200197]: pgmap v648: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T10:53:44.346 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:44 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:53:44.346 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:44 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T10:53:44.346 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:44 smithi125 ceph-mon[195507]: failed to encode map e272 with expected crc 2023-10-09T10:53:44.346 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:44 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:53:44.346 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:44 smithi125 ceph-mon[195507]: osdmap e272: 8 total, 8 up, 7 in 2023-10-09T10:53:44.347 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:44 smithi125 ceph-mon[195507]: failed to encode map e272 with expected crc 2023-10-09T10:53:44.347 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:44 smithi125 ceph-mon[195507]: pgmap v648: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T10:53:44.432 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:44.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:44 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:53:44.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:44 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T10:53:44.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:44 smithi155 ceph-mon[184854]: failed to encode map e272 with expected crc 2023-10-09T10:53:44.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:44 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:53:44.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:44 smithi155 ceph-mon[184854]: osdmap e272: 8 total, 8 up, 7 in 2023-10-09T10:53:44.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:44 smithi155 ceph-mon[184854]: failed to encode map e272 with expected crc 2023-10-09T10:53:44.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:44 smithi155 ceph-mon[184854]: pgmap v648: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T10:53:44.483 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:44.517 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:44.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:44.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:44.628 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:44.684 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:44.703 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T10:53:44.719 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:44.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:44.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:44.825 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:44.870 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:44.901 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:44.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:44.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:44.938 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 26 16 53005 52989 127.337 91.25 0.00920959 0.00784711 2023-10-09T10:53:45.005 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:45.062 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:45.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:45.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:45.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:45.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:45.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 7]}]: dispatch 2023-10-09T10:53:45.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2023-10-09T10:53:45.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:45.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:53:45.277 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:45.309 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:45.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:45.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:45.416 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:53:45.463 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 7]}]: dispatch 2023-10-09T10:53:45.464 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2023-10-09T10:53:45.464 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:45.464 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:53:45.464 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 7]}]: dispatch 2023-10-09T10:53:45.464 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2023-10-09T10:53:45.464 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:53:45.464 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:53:45.470 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:45.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:53:45.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:45.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:53:45.605 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:53:45.675 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:45.719 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:53:45.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:45.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:53:45.779 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:53:45.780 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:53:45.825 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:45.872 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:45.899 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:45.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:45.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:45.938 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 27 16 54357 54341 125.751 84.5 0.00786003 0.00794773 2023-10-09T10:53:46.008 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:46.054 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:46.139 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:46 smithi155 ceph-mon[184854]: failed to encode map e273 with expected crc 2023-10-09T10:53:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 7]}]': finished 2023-10-09T10:53:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2023-10-09T10:53:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:46 smithi155 ceph-mon[184854]: osdmap e273: 8 total, 8 up, 7 in 2023-10-09T10:53:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:46 smithi155 ceph-mon[184854]: failed to encode map e273 with expected crc 2023-10-09T10:53:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:46 smithi155 ceph-mon[184854]: pgmap v650: 119 pgs: 119 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-10-09T10:53:46.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:46.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:46.159 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:53:46.231 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:46 smithi125 ceph-mon[195507]: failed to encode map e273 with expected crc 2023-10-09T10:53:46.231 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 7]}]': finished 2023-10-09T10:53:46.231 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2023-10-09T10:53:46.232 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:46 smithi125 ceph-mon[195507]: osdmap e273: 8 total, 8 up, 7 in 2023-10-09T10:53:46.232 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:46 smithi125 ceph-mon[195507]: failed to encode map e273 with expected crc 2023-10-09T10:53:46.232 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:46 smithi125 ceph-mon[195507]: pgmap v650: 119 pgs: 119 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-10-09T10:53:46.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:46 smithi125 ceph-mon[200197]: failed to encode map e273 with expected crc 2023-10-09T10:53:46.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 7]}]': finished 2023-10-09T10:53:46.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2023-10-09T10:53:46.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:46 smithi125 ceph-mon[200197]: osdmap e273: 8 total, 8 up, 7 in 2023-10-09T10:53:46.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:46 smithi125 ceph-mon[200197]: failed to encode map e273 with expected crc 2023-10-09T10:53:46.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:46 smithi125 ceph-mon[200197]: pgmap v650: 119 pgs: 119 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-10-09T10:53:46.247 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:46.311 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:46.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:46.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:46.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:46.451 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:46.496 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:46.527 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:46.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:46.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:46.594 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:46.635 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:53:46.681 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:46.765 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:53:46.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:46.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:53:46.872 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:53:46.911 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:46.939 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 28 16 55686 55670 124.226 83.0625 0.00569019 0.00804631 2023-10-09T10:53:46.946 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:53:46.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:46.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:53:46.959 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:53:47.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:47.139 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:47.142 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:53:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:47 smithi155 ceph-mon[184854]: failed to encode map e274 with expected crc 2023-10-09T10:53:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:47 smithi155 ceph-mon[184854]: osdmap e274: 8 total, 8 up, 7 in 2023-10-09T10:53:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:47 smithi155 ceph-mon[184854]: failed to encode map e274 with expected crc 2023-10-09T10:53:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:47 smithi155 ceph-mon[184854]: 1.1c scrub starts 2023-10-09T10:53:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:47 smithi155 ceph-mon[184854]: 1.1c scrub ok 2023-10-09T10:53:47.261 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:47 smithi125 ceph-mon[195507]: failed to encode map e274 with expected crc 2023-10-09T10:53:47.262 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:47 smithi125 ceph-mon[195507]: osdmap e274: 8 total, 8 up, 7 in 2023-10-09T10:53:47.262 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:47 smithi125 ceph-mon[195507]: failed to encode map e274 with expected crc 2023-10-09T10:53:47.262 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:47 smithi125 ceph-mon[195507]: 1.1c scrub starts 2023-10-09T10:53:47.262 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:47 smithi125 ceph-mon[195507]: 1.1c scrub ok 2023-10-09T10:53:47.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:47 smithi125 ceph-mon[200197]: failed to encode map e274 with expected crc 2023-10-09T10:53:47.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:47 smithi125 ceph-mon[200197]: osdmap e274: 8 total, 8 up, 7 in 2023-10-09T10:53:47.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:47 smithi125 ceph-mon[200197]: failed to encode map e274 with expected crc 2023-10-09T10:53:47.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:47 smithi125 ceph-mon[200197]: 1.1c scrub starts 2023-10-09T10:53:47.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:47 smithi125 ceph-mon[200197]: 1.1c scrub ok 2023-10-09T10:53:47.379 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:47.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:47.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:47.409 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:47.483 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:47.529 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:47.561 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:47.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:47.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:47.667 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:47.708 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:47.744 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:47.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:47.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:47.852 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:47.886 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:53:47.886 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:53:47.919 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:47.939 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 29 16 56854 56838 122.46 73 0.0148991 0.00815984 2023-10-09T10:53:48.364 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:48 smithi125 ceph-mon[195507]: pgmap v652: 119 pgs: 119 active+clean; 3.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T10:53:48.365 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:48 smithi125 ceph-mon[200197]: pgmap v652: 119 pgs: 119 active+clean; 3.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T10:53:48.400 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:48.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:48.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:48.419 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:48 smithi155 ceph-mon[184854]: pgmap v652: 119 pgs: 119 active+clean; 3.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T10:53:48.508 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:48.556 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:48.597 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:48.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:48.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:48.703 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:53:48.751 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:48.774 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:53:48.778 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:53:48.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:48.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:53:48.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:48.900 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:53:48.926 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:53:48.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 30 16 58167 58151 121.103 82.0625 0.0261304 0.00825375 2023-10-09T10:53:48.955 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:48.987 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:53:48.994 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:49.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:49.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:49.009 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:53:49.057 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:53:49.085 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:53:49.103 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:49.121 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:53:49.150 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:49.175 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:53:49.178 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:53:49.178 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:53:49.178 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:53:49.179 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:53:49.179 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:53:49.179 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:53:49.179 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:53:49.179 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:53:49.185 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:49.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:49.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:49.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:49 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/254583856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:53:49.203 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:49 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/254583856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:53:49.291 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:53:49.344 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:49.377 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:53:49.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:49.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:53:49.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:49 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/254583856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:53:49.483 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:53:49.546 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:49.571 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:53:49.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:49.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:53:49.678 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:49.704 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 1, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:53:49.704 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:53:49.704 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [3, 4, 5, 6, 0, 1, 7] 2023-10-09T10:53:49.704 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd out 1 2023-10-09T10:53:49.731 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:49.771 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:49.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:49.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:49.879 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:49.898 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:49.923 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:49.941 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 31 16 59363 59347 119.608 74.75 0.0113292 0.00835609 2023-10-09T10:53:49.950 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:49.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:49.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:50.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:53:50.138 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:50.185 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:53:50.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:50.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:53:50.196 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:50 smithi155 ceph-mon[184854]: pgmap v653: 119 pgs: 119 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T10:53:50.290 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:53:50.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:50 smithi125 ceph-mon[195507]: pgmap v653: 119 pgs: 119 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T10:53:50.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:50 smithi125 ceph-mon[200197]: pgmap v653: 119 pgs: 119 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T10:53:50.347 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:50.431 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:53:50.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:50.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:53:50.514 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:50.538 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:53:50.599 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:50.628 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:53:50.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:50.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:53:50.735 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:53:50.788 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:50.820 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:53:50.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:50.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:53:50.932 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:53:50.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 32 16 60535 60519 118.159 73.25 0.00550773 0.00845882 2023-10-09T10:53:50.983 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:51.023 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:53:51.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:51.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:53:51.131 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:51.176 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:51.224 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:53:51.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3107854742' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:53:51.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:51 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:53:51.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3107854742' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:53:51.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:51 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:53:51.575 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:53:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:53:51] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2023-10-09T10:53:51.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3107854742' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:53:51.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:51 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:53:51.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 33 16 61796 61780 116.967 78.8125 0.00770573 0.00854472 2023-10-09T10:53:51.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:53:52.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:52 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:53:52.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:52 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:53:52.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:52 smithi125 ceph-mon[195507]: osdmap e275: 8 total, 8 up, 7 in 2023-10-09T10:53:52.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:52 smithi125 ceph-mon[195507]: failed to encode map e275 with expected crc 2023-10-09T10:53:52.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:52 smithi125 ceph-mon[195507]: failed to encode map e275 with expected crc 2023-10-09T10:53:52.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:52 smithi125 ceph-mon[195507]: pgmap v655: 119 pgs: 119 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T10:53:52.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:52 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:53:52.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:52 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:53:52.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:52 smithi125 ceph-mon[200197]: osdmap e275: 8 total, 8 up, 7 in 2023-10-09T10:53:52.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:52 smithi125 ceph-mon[200197]: failed to encode map e275 with expected crc 2023-10-09T10:53:52.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:52 smithi125 ceph-mon[200197]: failed to encode map e275 with expected crc 2023-10-09T10:53:52.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:52 smithi125 ceph-mon[200197]: pgmap v655: 119 pgs: 119 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T10:53:52.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:52 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:53:52.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:52 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:53:52.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:52 smithi155 ceph-mon[184854]: osdmap e275: 8 total, 8 up, 7 in 2023-10-09T10:53:52.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:52 smithi155 ceph-mon[184854]: failed to encode map e275 with expected crc 2023-10-09T10:53:52.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:52 smithi155 ceph-mon[184854]: failed to encode map e275 with expected crc 2023-10-09T10:53:52.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:52 smithi155 ceph-mon[184854]: pgmap v655: 119 pgs: 119 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T10:53:52.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 34 16 63114 63098 115.95 82.375 0.00876 0.00862032 2023-10-09T10:53:53.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 35 16 64471 64455 115.06 84.8125 0.0100321 0.00868632 2023-10-09T10:53:54.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:53 smithi155 ceph-mon[184854]: pgmap v656: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T10:53:54.222 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:54.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:53 smithi125 ceph-mon[195507]: pgmap v656: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T10:53:54.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:53 smithi125 ceph-mon[200197]: pgmap v656: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T10:53:54.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:54.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:54.527 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:53:54.594 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:54.630 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:53:54.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:54.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:53:54.741 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:53:54.791 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:54.843 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:53:54.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:54.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:53:54.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 36 16 65773 65757 114.124 81.375 0.00810896 0.00875755 2023-10-09T10:53:54.955 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:53:55.009 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:55.194 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:53:55.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:55.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:53:55.429 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:55.489 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:55.778 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:55.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:55.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:55.828 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:53:55.828 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:53:55.828 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:53:55.828 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:53:55.828 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:53:55.890 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:53:55.935 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:55.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 37 16 66925 66909 112.986 72 0.00346267 0.00884501 2023-10-09T10:53:56.119 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:53:56.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:56.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:53:56.228 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:53:56.302 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:56.332 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:53:56.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:56.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:53:56.437 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:56.475 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:56.612 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:53:56.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:56 smithi125 ceph-mon[195507]: pgmap v657: 119 pgs: 119 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T10:53:56.650 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:56 smithi125 ceph-mon[200197]: pgmap v657: 119 pgs: 119 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T10:53:56.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:56 smithi155 ceph-mon[184854]: pgmap v657: 119 pgs: 119 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T10:53:56.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 38 15 68125 68110 111.988 75.0625 0.0168146 0.00892493 2023-10-09T10:53:57.619 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:53:57.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:57.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:53:57.723 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:53:57.767 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:57.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 39 16 69311 69295 111.016 74.0625 0.010615 0.00900313 2023-10-09T10:53:58.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:53:58 smithi125 ceph-mon[195507]: pgmap v658: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T10:53:58.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:53:58 smithi125 ceph-mon[200197]: pgmap v658: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T10:53:58.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:53:58 smithi155 ceph-mon[184854]: pgmap v658: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T10:53:58.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:53:58.943235+0000 min lat: 0.00119012 max lat: 0.134094 avg lat: 0.00907001 2023-10-09T10:53:58.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:53:58.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 40 16 70568 70552 110.204 78.5625 0.0083328 0.00907001 2023-10-09T10:53:59.507 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:53:59.749 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:53:59.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:53:59.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:53:59.772 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:53:59.853 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:53:59.916 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:53:59.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 41 16 71756 71740 109.327 74.25 0.0218714 0.00914051 2023-10-09T10:54:00.548 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:54:00.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:00.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:54:00.661 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:00.708 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:00 smithi125 ceph-mon[195507]: pgmap v659: 119 pgs: 119 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T10:54:00.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:00.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:54:00.709 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:00 smithi125 ceph-mon[200197]: pgmap v659: 119 pgs: 119 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T10:54:00.709 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:00.709 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:54:00.717 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:00 smithi155 ceph-mon[184854]: pgmap v659: 119 pgs: 119 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T10:54:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:54:00.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 42 16 72921 72905 108.458 72.8125 0.0138604 0.00921542 2023-10-09T10:54:01.094 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:01.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:01.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:01.200 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:54:01.260 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:54:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:54:01] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2023-10-09T10:54:01.263 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:01.358 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:01.698 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:54:01.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:01.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:54:01.804 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:01.869 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:01.912 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:01.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:01.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:01.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 43 16 74027 74011 107.543 69.125 0.0124518 0.00929447 2023-10-09T10:54:02.017 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:02.051 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T10:54:01.727+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e275 definitely_dead 0 2023-10-09T10:54:02.057 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:02.344 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:02.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:02.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:02.448 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:02.468 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.1. 2023-10-09T10:54:02.491 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:02.509 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:02.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:02.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:02.612 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:02.638 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:02.644 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:02 smithi125 ceph-mon[200197]: pgmap v660: 119 pgs: 119 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T10:54:02.644 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:02 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/464165502' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:54:02.644 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:02 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:54:02.644 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:02 smithi125 ceph-mon[200197]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T10:54:02.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:02 smithi125 ceph-mon[195507]: pgmap v660: 119 pgs: 119 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T10:54:02.645 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:02 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/464165502' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:54:02.645 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:02 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:54:02.645 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:02 smithi125 ceph-mon[195507]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T10:54:02.657 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:02.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:02.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:02 smithi155 ceph-mon[184854]: pgmap v660: 119 pgs: 119 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T10:54:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:02 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/464165502' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:54:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:02 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T10:54:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:02 smithi155 ceph-mon[184854]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T10:54:02.761 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:02.787 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:02.805 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:02.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:02.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:02.908 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:02.938 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:02.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 44 16 74619 74603 105.94 37 0.00342399 0.00932934 2023-10-09T10:54:03.412 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:03.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:03.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:03.486 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:03.517 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:03.546 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:03.564 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:03.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:03.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T10:54:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[195507]: osdmap e276: 8 total, 8 up, 6 in 2023-10-09T10:54:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[195507]: failed to encode map e276 with expected crc 2023-10-09T10:54:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[195507]: failed to encode map e276 with expected crc 2023-10-09T10:54:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[195507]: failed to encode map e276 with expected crc 2023-10-09T10:54:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[195507]: failed to encode map e276 with expected crc 2023-10-09T10:54:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[195507]: failed to encode map e277 with expected crc 2023-10-09T10:54:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[195507]: osdmap e277: 8 total, 8 up, 6 in 2023-10-09T10:54:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[195507]: failed to encode map e277 with expected crc 2023-10-09T10:54:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[195507]: failed to encode map e277 with expected crc 2023-10-09T10:54:03.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T10:54:03.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[200197]: osdmap e276: 8 total, 8 up, 6 in 2023-10-09T10:54:03.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[200197]: failed to encode map e276 with expected crc 2023-10-09T10:54:03.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[200197]: failed to encode map e276 with expected crc 2023-10-09T10:54:03.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[200197]: failed to encode map e276 with expected crc 2023-10-09T10:54:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[200197]: failed to encode map e276 with expected crc 2023-10-09T10:54:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[200197]: failed to encode map e277 with expected crc 2023-10-09T10:54:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[200197]: osdmap e277: 8 total, 8 up, 6 in 2023-10-09T10:54:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[200197]: failed to encode map e277 with expected crc 2023-10-09T10:54:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:03 smithi125 ceph-mon[200197]: failed to encode map e277 with expected crc 2023-10-09T10:54:03.636 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:54:03.658 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:54:03.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:03.678 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:54:03.696 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:03.697 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:54:03.714 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:03.715 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:54:03.716 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:03 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T10:54:03.716 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:03 smithi155 ceph-mon[184854]: osdmap e276: 8 total, 8 up, 6 in 2023-10-09T10:54:03.716 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:03 smithi155 ceph-mon[184854]: failed to encode map e276 with expected crc 2023-10-09T10:54:03.716 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:03 smithi155 ceph-mon[184854]: failed to encode map e276 with expected crc 2023-10-09T10:54:03.716 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:03 smithi155 ceph-mon[184854]: failed to encode map e276 with expected crc 2023-10-09T10:54:03.716 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:03 smithi155 ceph-mon[184854]: failed to encode map e276 with expected crc 2023-10-09T10:54:03.717 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:03 smithi155 ceph-mon[184854]: failed to encode map e277 with expected crc 2023-10-09T10:54:03.717 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:03 smithi155 ceph-mon[184854]: osdmap e277: 8 total, 8 up, 6 in 2023-10-09T10:54:03.717 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:03 smithi155 ceph-mon[184854]: failed to encode map e277 with expected crc 2023-10-09T10:54:03.717 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:03 smithi155 ceph-mon[184854]: failed to encode map e277 with expected crc 2023-10-09T10:54:03.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:03.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:03.734 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:54:03.754 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:54:03.775 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:54:03.777 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:54:03.777 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:54:03.777 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:54:03.777 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:54:03.777 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:54:03.778 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:54:03.778 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:54:03.778 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:54:03.818 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:03.846 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:03.869 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:03.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:03.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:03.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 45 16 74619 74603 103.585 0 - 0.00932934 2023-10-09T10:54:03.973 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:54:04.023 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:04.614 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:54:04.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:04.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:54:04.718 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:04.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:04 smithi155 ceph-mon[184854]: failed to encode map e277 with expected crc 2023-10-09T10:54:04.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:04 smithi155 ceph-mon[184854]: pgmap v663: 119 pgs: 3 remapped+peering, 116 active+clean; 4.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 1.7 KiB/s rd, 82 MiB/s wr, 1.31k op/s; 3.3 KiB/s, 0 objects/s recovering 2023-10-09T10:54:04.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:04 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/740147020' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:04.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:04 smithi155 ceph-mon[184854]: failed to encode map e278 with expected crc 2023-10-09T10:54:04.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:04 smithi155 ceph-mon[184854]: failed to encode map e278 with expected crc 2023-10-09T10:54:04.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:04 smithi155 ceph-mon[184854]: osdmap e278: 8 total, 8 up, 6 in 2023-10-09T10:54:04.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:04 smithi155 ceph-mon[184854]: failed to encode map e278 with expected crc 2023-10-09T10:54:04.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:04 smithi155 ceph-mon[184854]: failed to encode map e278 with expected crc 2023-10-09T10:54:04.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:04 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:54:04.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[200197]: failed to encode map e277 with expected crc 2023-10-09T10:54:04.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[200197]: pgmap v663: 119 pgs: 3 remapped+peering, 116 active+clean; 4.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 1.7 KiB/s rd, 82 MiB/s wr, 1.31k op/s; 3.3 KiB/s, 0 objects/s recovering 2023-10-09T10:54:04.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/740147020' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:04.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[200197]: failed to encode map e278 with expected crc 2023-10-09T10:54:04.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[200197]: failed to encode map e278 with expected crc 2023-10-09T10:54:04.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[200197]: osdmap e278: 8 total, 8 up, 6 in 2023-10-09T10:54:04.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[200197]: failed to encode map e278 with expected crc 2023-10-09T10:54:04.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[200197]: failed to encode map e278 with expected crc 2023-10-09T10:54:04.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:54:04.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[195507]: failed to encode map e277 with expected crc 2023-10-09T10:54:04.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[195507]: pgmap v663: 119 pgs: 3 remapped+peering, 116 active+clean; 4.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 1.7 KiB/s rd, 82 MiB/s wr, 1.31k op/s; 3.3 KiB/s, 0 objects/s recovering 2023-10-09T10:54:04.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/740147020' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:04.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[195507]: failed to encode map e278 with expected crc 2023-10-09T10:54:04.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[195507]: failed to encode map e278 with expected crc 2023-10-09T10:54:04.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[195507]: osdmap e278: 8 total, 8 up, 6 in 2023-10-09T10:54:04.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[195507]: failed to encode map e278 with expected crc 2023-10-09T10:54:04.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[195507]: failed to encode map e278 with expected crc 2023-10-09T10:54:04.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:04 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:54:04.787 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:04.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 46 16 75170 75154 102.082 17.2188 0.0184099 0.00977924 2023-10-09T10:54:05.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[195507]: failed to encode map e278 with expected crc 2023-10-09T10:54:05.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[195507]: failed to encode map e278 with expected crc 2023-10-09T10:54:05.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[195507]: osdmap e279: 8 total, 8 up, 6 in 2023-10-09T10:54:05.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[195507]: failed to encode map e279 with expected crc 2023-10-09T10:54:05.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[195507]: failed to encode map e279 with expected crc 2023-10-09T10:54:05.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[195507]: failed to encode map e279 with expected crc 2023-10-09T10:54:05.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3362250482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:05.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:05.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[200197]: failed to encode map e278 with expected crc 2023-10-09T10:54:05.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[200197]: failed to encode map e278 with expected crc 2023-10-09T10:54:05.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[200197]: osdmap e279: 8 total, 8 up, 6 in 2023-10-09T10:54:05.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[200197]: failed to encode map e279 with expected crc 2023-10-09T10:54:05.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[200197]: failed to encode map e279 with expected crc 2023-10-09T10:54:05.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[200197]: failed to encode map e279 with expected crc 2023-10-09T10:54:05.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3362250482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:05.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:05 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:05.909 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:54:05.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 47 16 75932 75916 100.924 47.625 0.0222504 0.00990228 2023-10-09T10:54:05.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:05 smithi155 ceph-mon[184854]: failed to encode map e278 with expected crc 2023-10-09T10:54:05.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:05 smithi155 ceph-mon[184854]: failed to encode map e278 with expected crc 2023-10-09T10:54:05.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:05 smithi155 ceph-mon[184854]: osdmap e279: 8 total, 8 up, 6 in 2023-10-09T10:54:05.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:05 smithi155 ceph-mon[184854]: failed to encode map e279 with expected crc 2023-10-09T10:54:05.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:05 smithi155 ceph-mon[184854]: failed to encode map e279 with expected crc 2023-10-09T10:54:05.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:05 smithi155 ceph-mon[184854]: failed to encode map e279 with expected crc 2023-10-09T10:54:05.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:05 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3362250482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:05.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:05 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:06.752 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[200197]: failed to encode map e279 with expected crc 2023-10-09T10:54:06.753 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[200197]: failed to encode map e279 with expected crc 2023-10-09T10:54:06.753 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[200197]: failed to encode map e279 with expected crc 2023-10-09T10:54:06.753 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[200197]: pgmap v666: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 remapped+peering, 109 active+clean; 4.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 16 KiB/s rd, 65 MiB/s wr, 1.06k op/s; 18762/223413 objects degraded (8.398%); 4599/223413 objects misplaced (2.059%); 493 KiB/s, 7 objects/s recovering 2023-10-09T10:54:06.753 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[200197]: Health check failed: Degraded data redundancy: 18762/223413 objects degraded (8.398%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:06.754 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:54:06.754 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[200197]: failed to encode map e280 with expected crc 2023-10-09T10:54:06.754 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:54:06.754 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[200197]: osdmap e280: 8 total, 8 up, 6 in 2023-10-09T10:54:06.754 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[200197]: failed to encode map e280 with expected crc 2023-10-09T10:54:06.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[195507]: failed to encode map e279 with expected crc 2023-10-09T10:54:06.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[195507]: failed to encode map e279 with expected crc 2023-10-09T10:54:06.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[195507]: failed to encode map e279 with expected crc 2023-10-09T10:54:06.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[195507]: pgmap v666: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 remapped+peering, 109 active+clean; 4.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 16 KiB/s rd, 65 MiB/s wr, 1.06k op/s; 18762/223413 objects degraded (8.398%); 4599/223413 objects misplaced (2.059%); 493 KiB/s, 7 objects/s recovering 2023-10-09T10:54:06.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[195507]: Health check failed: Degraded data redundancy: 18762/223413 objects degraded (8.398%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:06.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:54:06.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[195507]: failed to encode map e280 with expected crc 2023-10-09T10:54:06.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:54:06.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[195507]: osdmap e280: 8 total, 8 up, 6 in 2023-10-09T10:54:06.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:06 smithi125 ceph-mon[195507]: failed to encode map e280 with expected crc 2023-10-09T10:54:06.944 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 48 16 76697 76681 99.8173 47.8125 0.0113689 0.0100131 2023-10-09T10:54:06.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:06 smithi155 ceph-mon[184854]: failed to encode map e279 with expected crc 2023-10-09T10:54:06.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:06 smithi155 ceph-mon[184854]: failed to encode map e279 with expected crc 2023-10-09T10:54:06.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:06 smithi155 ceph-mon[184854]: failed to encode map e279 with expected crc 2023-10-09T10:54:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:06 smithi155 ceph-mon[184854]: pgmap v666: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 remapped+peering, 109 active+clean; 4.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 16 KiB/s rd, 65 MiB/s wr, 1.06k op/s; 18762/223413 objects degraded (8.398%); 4599/223413 objects misplaced (2.059%); 493 KiB/s, 7 objects/s recovering 2023-10-09T10:54:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:06 smithi155 ceph-mon[184854]: Health check failed: Degraded data redundancy: 18762/223413 objects degraded (8.398%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:06 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:54:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:06 smithi155 ceph-mon[184854]: failed to encode map e280 with expected crc 2023-10-09T10:54:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:06 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:54:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:06 smithi155 ceph-mon[184854]: osdmap e280: 8 total, 8 up, 6 in 2023-10-09T10:54:06.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:06 smithi155 ceph-mon[184854]: failed to encode map e280 with expected crc 2023-10-09T10:54:07.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:07 smithi125 ceph-mon[195507]: failed to encode map e280 with expected crc 2023-10-09T10:54:07.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:07 smithi125 ceph-mon[200197]: failed to encode map e280 with expected crc 2023-10-09T10:54:07.944 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 49 16 77520 77504 98.8301 51.4375 0.021228 0.0101116 2023-10-09T10:54:07.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:07 smithi155 ceph-mon[184854]: failed to encode map e280 with expected crc 2023-10-09T10:54:08.321 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:08.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:08.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:08.433 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:08.493 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:08 smithi125 ceph-mon[195507]: pgmap v668: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 remapped+peering, 108 active+clean; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 23 KiB/s rd, 45 MiB/s wr, 730 op/s; 18762/226053 objects degraded (8.300%); 9229/226053 objects misplaced (4.083%); 493 KiB/s, 7 objects/s recovering 2023-10-09T10:54:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:08 smithi125 ceph-mon[200197]: pgmap v668: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 remapped+peering, 108 active+clean; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 23 KiB/s rd, 45 MiB/s wr, 730 op/s; 18762/226053 objects degraded (8.300%); 9229/226053 objects misplaced (4.083%); 493 KiB/s, 7 objects/s recovering 2023-10-09T10:54:08.938 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:08.944 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 50 16 78260 78244 97.7786 46.25 0.0258827 0.0102208 2023-10-09T10:54:08.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:08.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:08 smithi155 ceph-mon[184854]: pgmap v668: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 remapped+peering, 108 active+clean; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 23 KiB/s rd, 45 MiB/s wr, 730 op/s; 18762/226053 objects degraded (8.300%); 9229/226053 objects misplaced (4.083%); 493 KiB/s, 7 objects/s recovering 2023-10-09T10:54:09.033 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:54:09.033 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:54:09.047 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:09.114 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:09.151 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:09.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:09.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:09.257 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:09.314 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:09.439 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:09.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:09.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:09.460 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:54:09.543 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:09.575 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:09 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T10:54:09.623 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:09.658 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:09.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:09.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:09.770 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:09.823 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:09.848 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:09.853 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:09 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T10:54:09.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:09.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:09.925 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:54:09.944 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 51 16 78970 78954 96.7316 44.375 0.0174415 0.0103318 2023-10-09T10:54:09.955 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:09.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:09 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T10:54:10.041 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:10.076 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:10.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:10.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:10.187 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:10.246 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:10.335 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:54:10.795 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:10.800 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:10.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:10.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:10.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:10 smithi125 ceph-mon[195507]: pgmap v669: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 110 active+clean; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 15 KiB/s rd, 31 MiB/s wr, 506 op/s; 23393/229149 objects degraded (10.209%); 9229/229149 objects misplaced (4.028%); 407 KiB/s, 5 objects/s recovering 2023-10-09T10:54:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:10 smithi125 ceph-mon[200197]: pgmap v669: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 110 active+clean; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 15 KiB/s rd, 31 MiB/s wr, 506 op/s; 23393/229149 objects degraded (10.209%); 9229/229149 objects misplaced (4.028%); 407 KiB/s, 5 objects/s recovering 2023-10-09T10:54:10.837 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:54:10.852 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:10 smithi155 ceph-mon[184854]: pgmap v669: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 110 active+clean; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 15 KiB/s rd, 31 MiB/s wr, 506 op/s; 23393/229149 objects degraded (10.209%); 9229/229149 objects misplaced (4.028%); 407 KiB/s, 5 objects/s recovering 2023-10-09T10:54:10.906 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:10.945 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 52 16 79724 79708 95.7751 47.125 0.0362253 0.0104356 2023-10-09T10:54:10.976 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:11.131 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:11.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:11.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:11.237 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:11.296 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:11.492 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:11.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:11.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:11.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:54:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:54:11] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2023-10-09T10:54:11.601 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:11.637 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:11.704 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:11 smithi155 ceph-mon[184854]: pgmap v670: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 12 KiB/s rd, 43 MiB/s wr, 691 op/s; 23393/235569 objects degraded (9.930%); 9086/235569 objects misplaced (3.857%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:54:11.704 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:11 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 23393/235569 objects degraded (9.930%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:11.946 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 53 16 80429 80413 94.7995 44.0625 0.0103639 0.0105436 2023-10-09T10:54:11.980 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:11.980 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:11 smithi125 ceph-mon[195507]: pgmap v670: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 12 KiB/s rd, 43 MiB/s wr, 691 op/s; 23393/235569 objects degraded (9.930%); 9086/235569 objects misplaced (3.857%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:54:11.981 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:11 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 23393/235569 objects degraded (9.930%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:11.982 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:11 smithi125 ceph-mon[200197]: pgmap v670: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 12 KiB/s rd, 43 MiB/s wr, 691 op/s; 23393/235569 objects degraded (9.930%); 9086/235569 objects misplaced (3.857%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:54:11.982 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:11 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 23393/235569 objects degraded (9.930%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:11.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:11.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:12.004 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:54:12.004 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:54:12.084 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:12.154 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:12.401 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:12.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:12.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:12.438 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:54:12.510 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:12.549 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:12.573 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:12.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:12.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:12.688 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:12.806 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:12.888 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:12.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:12.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:12.941 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:54:12.946 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 54 16 81107 81091 93.8288 42.375 0.0112015 0.0106522 2023-10-09T10:54:12.996 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:13.074 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:13.117 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:13.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:13.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:13.226 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:13.276 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:13.391 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 7] out_osds: [2, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:54:13.391 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:54:13.391 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-09T10:54:13.392 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-09T10:54:13.685 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:13.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:13.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:13.719 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:13.793 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:13.862 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:13.946 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 55 16 81807 81791 92.9184 43.75 0.0196961 0.0107573 2023-10-09T10:54:13.976 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:13 smithi125 ceph-mon[200197]: pgmap v671: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 11 KiB/s rd, 49 MiB/s wr, 798 op/s; 23393/240300 objects degraded (9.735%); 9086/240300 objects misplaced (3.781%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:54:13.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:13 smithi125 ceph-mon[195507]: pgmap v671: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 11 KiB/s rd, 49 MiB/s wr, 798 op/s; 23393/240300 objects degraded (9.735%); 9086/240300 objects misplaced (3.781%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:54:14.098 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:14.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:14.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:14.203 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:14.204 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:14.211 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:54:14.211 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:54:14.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:13 smithi155 ceph-mon[184854]: pgmap v671: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 11 KiB/s rd, 49 MiB/s wr, 798 op/s; 23393/240300 objects degraded (9.735%); 9086/240300 objects misplaced (3.781%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:54:14.290 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:14.375 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:14.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:14.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:14.430 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:54:14.464 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:54:14.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:14.493 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:54:14.524 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:14.540 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:54:14.578 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:54:14.627 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:54:14.641 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:54:14.661 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:54:14.695 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:54:14.704 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:54:14.704 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:54:14.705 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:54:14.705 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:54:14.705 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:54:14.705 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:54:14.705 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:54:14.705 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:54:14.897 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:14 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1715937608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:14.946 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 56 16 82485 82469 92.0159 42.375 0.00703958 0.010862 2023-10-09T10:54:15.008 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:15.015 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:14 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1715937608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:15.016 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:14 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1715937608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:15.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:15.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:15.117 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:15.159 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:54:15.192 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:15.363 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:15.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:15.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:15.468 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:15.528 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:15.672 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:15.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:15.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:15.775 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:15.814 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:15.947 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 57 16 83181 83165 91.1648 43.5 0.0248463 0.0109638 2023-10-09T10:54:15.958 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:15.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:15.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:16.063 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:16.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:16.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:54:16.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:16 smithi155 ceph-mon[184854]: pgmap v672: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 4.1 KiB/s rd, 44 MiB/s wr, 700 op/s; 23393/244323 objects degraded (9.575%); 8945/244323 objects misplaced (3.661%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:54:16.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:16.164 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:16.220 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:16.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:16.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:16.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:16.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:54:16.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:16 smithi125 ceph-mon[195507]: pgmap v672: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 4.1 KiB/s rd, 44 MiB/s wr, 700 op/s; 23393/244323 objects degraded (9.575%); 8945/244323 objects misplaced (3.661%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:54:16.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:16.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:16.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:54:16.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:16 smithi125 ceph-mon[200197]: pgmap v672: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 4.1 KiB/s rd, 44 MiB/s wr, 700 op/s; 23393/244323 objects degraded (9.575%); 8945/244323 objects misplaced (3.661%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:54:16.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:16.324 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:16.410 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:16.477 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:16.480 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:54:16.480 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:54:16.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:54:16.480 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:54:16.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.481 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181597 6.2 6:42bf0766:::benchmark_data_smithi125_244951_object75154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:54:16.481 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.261648+0000", 2023-10-09T10:54:16.481 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 360.21237576099998, 2023-10-09T10:54:16.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.045766977, 2023-10-09T10:54:16.481 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.481 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.481 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.481 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:16.481 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:16.481 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181597 2023-10-09T10:54:16.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.481 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261648+0000", 2023-10-09T10:54:16.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261648+0000", 2023-10-09T10:54:16.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.483 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.483 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261646+0000", 2023-10-09T10:54:16.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:54:16.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.483 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.483 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261657+0000", 2023-10-09T10:54:16.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1413000000000001e-05 2023-10-09T10:54:16.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.483 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.483 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261674+0000", 2023-10-09T10:54:16.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6969e-05 2023-10-09T10:54:16.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261682+0000", 2023-10-09T10:54:16.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.357e-06 2023-10-09T10:54:16.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261737+0000", 2023-10-09T10:54:16.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5757999999999997e-05 2023-10-09T10:54:16.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op must wait for map", 2023-10-09T10:54:16.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261745+0000", 2023-10-09T10:54:16.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.576e-06 2023-10-09T10:54:16.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.268199+0000", 2023-10-09T10:54:16.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064538019999999998 2023-10-09T10:54:16.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:54:16.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.268208+0000", 2023-10-09T10:54:16.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1880000000000008e-06 2023-10-09T10:54:16.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.298495+0000", 2023-10-09T10:54:16.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0302869029999999 2023-10-09T10:54:16.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.298621+0000", 2023-10-09T10:54:16.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012632400000000001 2023-10-09T10:54:16.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:54:16.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.298695+0000", 2023-10-09T10:54:16.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3521000000000003e-05 2023-10-09T10:54:16.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.301440+0000", 2023-10-09T10:54:16.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002744879 2023-10-09T10:54:16.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.304506+0000", 2023-10-09T10:54:16.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030660570000000001 2023-10-09T10:54:16.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307330+0000", 2023-10-09T10:54:16.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028240029999999998 2023-10-09T10:54:16.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307381+0000", 2023-10-09T10:54:16.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1792000000000002e-05 2023-10-09T10:54:16.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307415+0000", 2023-10-09T10:54:16.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3429000000000001e-05 2023-10-09T10:54:16.489 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.489 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.489 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.490 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181636 6.2 6:40ad50b0:::benchmark_data_smithi125_244951_object75193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:54:16.490 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.320597+0000", 2023-10-09T10:54:16.490 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 360.15342709200002, 2023-10-09T10:54:16.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98701349400000005, 2023-10-09T10:54:16.490 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.490 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.490 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.490 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:16.490 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:16.490 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181636 2023-10-09T10:54:16.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.491 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320597+0000", 2023-10-09T10:54:16.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320597+0000", 2023-10-09T10:54:16.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320595+0000", 2023-10-09T10:54:16.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:54:16.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320604+0000", 2023-10-09T10:54:16.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8580000000000006e-06 2023-10-09T10:54:16.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320623+0000", 2023-10-09T10:54:16.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9894999999999998e-05 2023-10-09T10:54:16.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320631+0000", 2023-10-09T10:54:16.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7389999999999999e-06 2023-10-09T10:54:16.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320699+0000", 2023-10-09T10:54:16.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7858e-05 2023-10-09T10:54:16.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:54:16.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320709+0000", 2023-10-09T10:54:16.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0225e-05 2023-10-09T10:54:16.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.301476+0000", 2023-10-09T10:54:16.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98076654399999996 2023-10-09T10:54:16.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.301576+0000", 2023-10-09T10:54:16.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010008700000000001 2023-10-09T10:54:16.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:54:16.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.301679+0000", 2023-10-09T10:54:16.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000102806 2023-10-09T10:54:16.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307490+0000", 2023-10-09T10:54:16.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0058112210000000001 2023-10-09T10:54:16.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307500+0000", 2023-10-09T10:54:16.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9000000000000001e-06 2023-10-09T10:54:16.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307534+0000", 2023-10-09T10:54:16.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4637000000000001e-05 2023-10-09T10:54:16.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307577+0000", 2023-10-09T10:54:16.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2453999999999997e-05 2023-10-09T10:54:16.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307610+0000", 2023-10-09T10:54:16.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3167000000000003e-05 2023-10-09T10:54:16.497 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.497 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.497 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.498 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181651 6.2 6:46858d5d:::benchmark_data_smithi125_244951_object75208:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:54:16.498 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.342675+0000", 2023-10-09T10:54:16.498 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 360.13134863900001, 2023-10-09T10:54:16.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98594575900000003, 2023-10-09T10:54:16.498 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.498 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.498 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.498 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:16.498 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:16.499 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181651 2023-10-09T10:54:16.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.499 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342675+0000", 2023-10-09T10:54:16.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342675+0000", 2023-10-09T10:54:16.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342674+0000", 2023-10-09T10:54:16.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:54:16.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342681+0000", 2023-10-09T10:54:16.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5189999999999998e-06 2023-10-09T10:54:16.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342690+0000", 2023-10-09T10:54:16.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6029999999999994e-06 2023-10-09T10:54:16.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342694+0000", 2023-10-09T10:54:16.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9979999999999997e-06 2023-10-09T10:54:16.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342738+0000", 2023-10-09T10:54:16.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4175000000000003e-05 2023-10-09T10:54:16.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:54:16.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342749+0000", 2023-10-09T10:54:16.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0664e-05 2023-10-09T10:54:16.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.304532+0000", 2023-10-09T10:54:16.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.96178257899999997 2023-10-09T10:54:16.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.304626+0000", 2023-10-09T10:54:16.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4790999999999994e-05 2023-10-09T10:54:16.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:54:16.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.304730+0000", 2023-10-09T10:54:16.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103824 2023-10-09T10:54:16.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.310700+0000", 2023-10-09T10:54:16.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059698260000000001 2023-10-09T10:54:16.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313386+0000", 2023-10-09T10:54:16.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026858849999999998 2023-10-09T10:54:16.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328535+0000", 2023-10-09T10:54:16.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015149249 2023-10-09T10:54:16.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.505 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.505 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328576+0000", 2023-10-09T10:54:16.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1075999999999998e-05 2023-10-09T10:54:16.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.505 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.505 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328621+0000", 2023-10-09T10:54:16.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4577999999999999e-05 2023-10-09T10:54:16.505 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.506 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.506 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.506 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.506 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.506 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181659 6.2 6:43df335a:::benchmark_data_smithi125_244951_object75216:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:54:16.506 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.355928+0000", 2023-10-09T10:54:16.506 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 360.11809545699998, 2023-10-09T10:54:16.506 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97280544800000002, 2023-10-09T10:54:16.506 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.506 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.506 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.506 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:16.507 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:16.507 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181659 2023-10-09T10:54:16.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.507 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.507 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.507 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355928+0000", 2023-10-09T10:54:16.507 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.507 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.507 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355928+0000", 2023-10-09T10:54:16.508 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.508 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.508 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.508 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.508 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355926+0000", 2023-10-09T10:54:16.508 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:54:16.508 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.508 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.508 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.508 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355939+0000", 2023-10-09T10:54:16.508 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2756e-05 2023-10-09T10:54:16.508 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.509 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355957+0000", 2023-10-09T10:54:16.509 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8128999999999999e-05 2023-10-09T10:54:16.509 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.509 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355965+0000", 2023-10-09T10:54:16.509 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5320000000000001e-06 2023-10-09T10:54:16.509 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.510 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.356031+0000", 2023-10-09T10:54:16.510 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6012000000000004e-05 2023-10-09T10:54:16.510 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.510 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.510 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:54:16.510 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.356044+0000", 2023-10-09T10:54:16.510 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2897e-05 2023-10-09T10:54:16.510 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.510 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.510 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.510 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307650+0000", 2023-10-09T10:54:16.511 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95160661899999999 2023-10-09T10:54:16.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.511 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.511 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.511 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307782+0000", 2023-10-09T10:54:16.511 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013221500000000001 2023-10-09T10:54:16.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.511 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.511 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:54:16.511 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307905+0000", 2023-10-09T10:54:16.511 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000122192 2023-10-09T10:54:16.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313393+0000", 2023-10-09T10:54:16.512 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0054883229999999998 2023-10-09T10:54:16.512 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313435+0000", 2023-10-09T10:54:16.512 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2500999999999998e-05 2023-10-09T10:54:16.512 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.513 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.513 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328675+0000", 2023-10-09T10:54:16.513 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015239948999999999 2023-10-09T10:54:16.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.513 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.513 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328701+0000", 2023-10-09T10:54:16.513 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5860999999999999e-05 2023-10-09T10:54:16.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.513 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.513 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328734+0000", 2023-10-09T10:54:16.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2398999999999999e-05 2023-10-09T10:54:16.514 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.514 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.514 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.514 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.514 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.514 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181668 6.2 6:47615e5b:::benchmark_data_smithi125_244951_object75225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:54:16.514 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.371758+0000", 2023-10-09T10:54:16.514 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 360.10226535200002, 2023-10-09T10:54:16.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95707596299999997, 2023-10-09T10:54:16.514 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.515 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.515 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.515 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:16.515 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:16.515 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181668 2023-10-09T10:54:16.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.515 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.515 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.515 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371758+0000", 2023-10-09T10:54:16.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.516 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.516 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.516 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371758+0000", 2023-10-09T10:54:16.516 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.516 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.516 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.516 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.516 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371757+0000", 2023-10-09T10:54:16.516 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:54:16.516 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.516 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.516 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.516 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371764+0000", 2023-10-09T10:54:16.517 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3379999999999999e-06 2023-10-09T10:54:16.517 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.517 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.517 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371780+0000", 2023-10-09T10:54:16.517 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5863999999999999e-05 2023-10-09T10:54:16.517 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.517 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.517 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371789+0000", 2023-10-09T10:54:16.517 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2050000000000002e-06 2023-10-09T10:54:16.518 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.518 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.518 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.518 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371836+0000", 2023-10-09T10:54:16.518 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7939000000000003e-05 2023-10-09T10:54:16.518 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.518 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.518 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:54:16.518 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371845+0000", 2023-10-09T10:54:16.518 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2619999999999998e-06 2023-10-09T10:54:16.518 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.518 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.519 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.519 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.310735+0000", 2023-10-09T10:54:16.519 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93888995500000005 2023-10-09T10:54:16.519 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.519 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.519 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.519 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.310868+0000", 2023-10-09T10:54:16.519 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000132867 2023-10-09T10:54:16.519 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.519 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.519 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:54:16.519 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.310995+0000", 2023-10-09T10:54:16.520 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012735700000000001 2023-10-09T10:54:16.520 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.520 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.520 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.520 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.319588+0000", 2023-10-09T10:54:16.520 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0085930829999999996 2023-10-09T10:54:16.520 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.520 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.520 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.520 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322662+0000", 2023-10-09T10:54:16.520 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030742059999999999 2023-10-09T10:54:16.520 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.521 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328782+0000", 2023-10-09T10:54:16.521 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061201789999999999 2023-10-09T10:54:16.521 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.521 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328812+0000", 2023-10-09T10:54:16.521 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9555000000000001e-05 2023-10-09T10:54:16.521 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.522 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328834+0000", 2023-10-09T10:54:16.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2314999999999999e-05 2023-10-09T10:54:16.522 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.522 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.522 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.522 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.522 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.522 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181725 6.2 6:4079eac3:::benchmark_data_smithi125_244951_object75282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:54:16.522 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.472093+0000", 2023-10-09T10:54:16.522 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 360.00193053499999, 2023-10-09T10:54:16.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.85682714000000004, 2023-10-09T10:54:16.523 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.523 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.523 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.523 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:16.523 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:16.523 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181725 2023-10-09T10:54:16.523 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.523 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.523 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472093+0000", 2023-10-09T10:54:16.523 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.524 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.524 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.524 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.524 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472093+0000", 2023-10-09T10:54:16.524 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.524 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.524 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.524 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.524 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472092+0000", 2023-10-09T10:54:16.524 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:16.524 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.524 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.525 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.525 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472100+0000", 2023-10-09T10:54:16.525 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8759999999999994e-06 2023-10-09T10:54:16.525 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.525 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.525 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.525 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472118+0000", 2023-10-09T10:54:16.525 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7863000000000002e-05 2023-10-09T10:54:16.525 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.525 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.525 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.525 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472125+0000", 2023-10-09T10:54:16.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5209999999999999e-06 2023-10-09T10:54:16.526 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.526 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.526 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.526 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472170+0000", 2023-10-09T10:54:16.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4774999999999997e-05 2023-10-09T10:54:16.526 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.526 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.526 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:54:16.526 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472178+0000", 2023-10-09T10:54:16.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1410000000000005e-06 2023-10-09T10:54:16.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.527 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.527 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.527 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313459+0000", 2023-10-09T10:54:16.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.84128165899999996 2023-10-09T10:54:16.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.527 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.527 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.527 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313598+0000", 2023-10-09T10:54:16.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013813200000000001 2023-10-09T10:54:16.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.528 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:54:16.528 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313729+0000", 2023-10-09T10:54:16.528 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013123599999999999 2023-10-09T10:54:16.528 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.528 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.528 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322622+0000", 2023-10-09T10:54:16.528 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0088927280000000008 2023-10-09T10:54:16.528 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.528 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322695+0000", 2023-10-09T10:54:16.529 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3676000000000004e-05 2023-10-09T10:54:16.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.529 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328866+0000", 2023-10-09T10:54:16.529 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.006170432 2023-10-09T10:54:16.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.529 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328890+0000", 2023-10-09T10:54:16.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4579999999999998e-05 2023-10-09T10:54:16.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.530 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.530 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328920+0000", 2023-10-09T10:54:16.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0015999999999999e-05 2023-10-09T10:54:16.530 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.530 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.530 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.530 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181731 6.2 6:47d350c6:::benchmark_data_smithi125_244951_object75288:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:54:16.531 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.487013+0000", 2023-10-09T10:54:16.531 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 359.98701081299998, 2023-10-09T10:54:16.531 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.84197810100000003, 2023-10-09T10:54:16.531 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.531 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.531 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.531 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:16.531 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:16.531 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181731 2023-10-09T10:54:16.531 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.531 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.531 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.532 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.532 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487013+0000", 2023-10-09T10:54:16.532 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.532 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.532 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.532 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.532 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487013+0000", 2023-10-09T10:54:16.532 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.532 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.532 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.532 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.532 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487011+0000", 2023-10-09T10:54:16.533 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:16.533 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.533 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.533 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.533 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487020+0000", 2023-10-09T10:54:16.533 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1850000000000004e-06 2023-10-09T10:54:16.533 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.533 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.533 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.533 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487036+0000", 2023-10-09T10:54:16.533 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6529e-05 2023-10-09T10:54:16.533 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.534 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.534 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.534 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487043+0000", 2023-10-09T10:54:16.534 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0940000000000002e-06 2023-10-09T10:54:16.534 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.534 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.534 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.534 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487118+0000", 2023-10-09T10:54:16.534 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5085000000000004e-05 2023-10-09T10:54:16.534 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.534 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.535 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:54:16.535 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487127+0000", 2023-10-09T10:54:16.535 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7819999999999999e-06 2023-10-09T10:54:16.535 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.535 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.535 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.535 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.316596+0000", 2023-10-09T10:54:16.535 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.82946907199999997 2023-10-09T10:54:16.535 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.535 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.535 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.535 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.316692+0000", 2023-10-09T10:54:16.536 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5668000000000006e-05 2023-10-09T10:54:16.536 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.536 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.536 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:54:16.536 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.316798+0000", 2023-10-09T10:54:16.536 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000105782 2023-10-09T10:54:16.536 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.536 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.536 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.536 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328499+0000", 2023-10-09T10:54:16.536 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011701549 2023-10-09T10:54:16.536 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.537 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328633+0000", 2023-10-09T10:54:16.537 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013372199999999999 2023-10-09T10:54:16.537 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.537 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328950+0000", 2023-10-09T10:54:16.537 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00031750600000000001 2023-10-09T10:54:16.537 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328973+0000", 2023-10-09T10:54:16.538 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2549999999999999e-05 2023-10-09T10:54:16.538 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.538 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.538 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328991+0000", 2023-10-09T10:54:16.538 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8042000000000001e-05 2023-10-09T10:54:16.538 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.538 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.538 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.538 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.538 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.539 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181758 6.2 6:418c688b:::benchmark_data_smithi125_244951_object75315:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:54:16.539 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.547191+0000", 2023-10-09T10:54:16.539 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 359.92683311100001, 2023-10-09T10:54:16.539 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.79446401099999997, 2023-10-09T10:54:16.539 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.539 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.539 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.539 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:16.539 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:16.539 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181758 2023-10-09T10:54:16.539 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.539 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.540 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.540 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547191+0000", 2023-10-09T10:54:16.540 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.540 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.540 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.540 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547191+0000", 2023-10-09T10:54:16.540 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.540 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.540 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547189+0000", 2023-10-09T10:54:16.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:54:16.541 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.541 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.541 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547198+0000", 2023-10-09T10:54:16.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7680000000000003e-06 2023-10-09T10:54:16.541 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.541 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.541 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547219+0000", 2023-10-09T10:54:16.542 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0837000000000001e-05 2023-10-09T10:54:16.542 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.542 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.542 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.542 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547228+0000", 2023-10-09T10:54:16.542 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6610000000000006e-06 2023-10-09T10:54:16.542 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.542 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.542 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.542 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547285+0000", 2023-10-09T10:54:16.542 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6617999999999999e-05 2023-10-09T10:54:16.542 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:54:16.543 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547296+0000", 2023-10-09T10:54:16.543 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1141999999999999e-05 2023-10-09T10:54:16.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.543 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.319617+0000", 2023-10-09T10:54:16.543 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.77232171000000005 2023-10-09T10:54:16.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.319712+0000", 2023-10-09T10:54:16.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4128000000000004e-05 2023-10-09T10:54:16.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:54:16.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.319799+0000", 2023-10-09T10:54:16.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7299999999999994e-05 2023-10-09T10:54:16.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328638+0000", 2023-10-09T10:54:16.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0088386809999999993 2023-10-09T10:54:16.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341523+0000", 2023-10-09T10:54:16.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012885369000000001 2023-10-09T10:54:16.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341553+0000", 2023-10-09T10:54:16.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9666999999999999e-05 2023-10-09T10:54:16.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.546 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.546 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341610+0000", 2023-10-09T10:54:16.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7250999999999998e-05 2023-10-09T10:54:16.546 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.546 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.546 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341655+0000", 2023-10-09T10:54:16.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4746999999999997e-05 2023-10-09T10:54:16.546 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.546 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.547 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.547 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.547 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181784 6.2 6:472c8d43:::benchmark_data_smithi125_244951_object75341:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:54:16.547 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.604167+0000", 2023-10-09T10:54:16.547 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 359.86985629700001, 2023-10-09T10:54:16.547 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.74181532500000003, 2023-10-09T10:54:16.547 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.547 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.547 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.547 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:16.547 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:16.548 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181784 2023-10-09T10:54:16.548 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.548 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.548 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.548 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.548 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604167+0000", 2023-10-09T10:54:16.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.548 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.548 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.548 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.548 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604167+0000", 2023-10-09T10:54:16.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.549 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.549 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.549 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.549 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604165+0000", 2023-10-09T10:54:16.549 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:54:16.549 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.549 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.549 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.549 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604175+0000", 2023-10-09T10:54:16.549 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5910000000000002e-06 2023-10-09T10:54:16.549 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.550 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.550 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.550 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604193+0000", 2023-10-09T10:54:16.550 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8117e-05 2023-10-09T10:54:16.550 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.550 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.550 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.550 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604201+0000", 2023-10-09T10:54:16.550 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9689999999999994e-06 2023-10-09T10:54:16.550 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.550 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.550 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.551 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604264+0000", 2023-10-09T10:54:16.551 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2731000000000006e-05 2023-10-09T10:54:16.551 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.551 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.551 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:54:16.551 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604275+0000", 2023-10-09T10:54:16.551 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.117e-05 2023-10-09T10:54:16.551 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.551 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.551 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.551 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322716+0000", 2023-10-09T10:54:16.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.71844131899999997 2023-10-09T10:54:16.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.552 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322828+0000", 2023-10-09T10:54:16.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112049 2023-10-09T10:54:16.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:54:16.552 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322941+0000", 2023-10-09T10:54:16.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112274 2023-10-09T10:54:16.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328642+0000", 2023-10-09T10:54:16.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0057018609999999999 2023-10-09T10:54:16.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341689+0000", 2023-10-09T10:54:16.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01304662 2023-10-09T10:54:16.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.345883+0000", 2023-10-09T10:54:16.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041934269999999996 2023-10-09T10:54:16.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.345938+0000", 2023-10-09T10:54:16.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5408000000000002e-05 2023-10-09T10:54:16.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.345983+0000", 2023-10-09T10:54:16.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4749999999999997e-05 2023-10-09T10:54:16.555 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.555 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.555 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.555 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181846 6.2 6:4620c9a0:::benchmark_data_smithi125_244951_object75403:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:54:16.555 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.736404+0000", 2023-10-09T10:54:16.555 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 359.73761934200002, 2023-10-09T10:54:16.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.60969460900000005, 2023-10-09T10:54:16.555 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.555 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.556 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.556 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:16.556 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:16.556 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181846 2023-10-09T10:54:16.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.556 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736404+0000", 2023-10-09T10:54:16.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736404+0000", 2023-10-09T10:54:16.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.557 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736403+0000", 2023-10-09T10:54:16.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:16.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.557 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736409+0000", 2023-10-09T10:54:16.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1940000000000003e-06 2023-10-09T10:54:16.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736419+0000", 2023-10-09T10:54:16.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0188e-05 2023-10-09T10:54:16.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736426+0000", 2023-10-09T10:54:16.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1770000000000003e-06 2023-10-09T10:54:16.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736485+0000", 2023-10-09T10:54:16.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9590999999999999e-05 2023-10-09T10:54:16.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:54:16.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736494+0000", 2023-10-09T10:54:16.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3180000000000007e-06 2023-10-09T10:54:16.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.325746+0000", 2023-10-09T10:54:16.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.589252691 2023-10-09T10:54:16.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.325846+0000", 2023-10-09T10:54:16.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000100058 2023-10-09T10:54:16.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:54:16.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.325944+0000", 2023-10-09T10:54:16.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7664999999999994e-05 2023-10-09T10:54:16.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.331507+0000", 2023-10-09T10:54:16.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0055631719999999999 2023-10-09T10:54:16.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341718+0000", 2023-10-09T10:54:16.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010211291000000001 2023-10-09T10:54:16.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.562 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.562 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346032+0000", 2023-10-09T10:54:16.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043136299999999997 2023-10-09T10:54:16.562 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.562 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.562 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346069+0000", 2023-10-09T10:54:16.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6520000000000003e-05 2023-10-09T10:54:16.562 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.562 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346099+0000", 2023-10-09T10:54:16.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0341999999999999e-05 2023-10-09T10:54:16.563 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.563 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.563 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.563 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181882 6.2 6:44b14788:::benchmark_data_smithi125_244951_object75439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:54:16.563 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.816228+0000", 2023-10-09T10:54:16.563 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 359.65779569699998, 2023-10-09T10:54:16.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.529971305, 2023-10-09T10:54:16.564 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.564 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.564 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.564 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:16.564 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:16.564 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181882 2023-10-09T10:54:16.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.564 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816228+0000", 2023-10-09T10:54:16.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816228+0000", 2023-10-09T10:54:16.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816226+0000", 2023-10-09T10:54:16.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:54:16.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816234+0000", 2023-10-09T10:54:16.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6620000000000005e-06 2023-10-09T10:54:16.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816249+0000", 2023-10-09T10:54:16.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5097e-05 2023-10-09T10:54:16.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816256+0000", 2023-10-09T10:54:16.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7000000000000002e-06 2023-10-09T10:54:16.567 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.567 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.567 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.567 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816311+0000", 2023-10-09T10:54:16.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5797000000000001e-05 2023-10-09T10:54:16.567 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.567 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.567 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:54:16.567 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816322+0000", 2023-10-09T10:54:16.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0219e-05 2023-10-09T10:54:16.567 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.329013+0000", 2023-10-09T10:54:16.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.51269123999999999 2023-10-09T10:54:16.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.329098+0000", 2023-10-09T10:54:16.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4586999999999995e-05 2023-10-09T10:54:16.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:54:16.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.329172+0000", 2023-10-09T10:54:16.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4690000000000005e-05 2023-10-09T10:54:16.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.338822+0000", 2023-10-09T10:54:16.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0096501520000000004 2023-10-09T10:54:16.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.343192+0000", 2023-10-09T10:54:16.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043696409999999996 2023-10-09T10:54:16.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346136+0000", 2023-10-09T10:54:16.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029442629999999999 2023-10-09T10:54:16.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346170+0000", 2023-10-09T10:54:16.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3294999999999999e-05 2023-10-09T10:54:16.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346199+0000", 2023-10-09T10:54:16.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9674000000000001e-05 2023-10-09T10:54:16.571 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.571 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.571 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.571 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181958 6.2 6:44748d34:::benchmark_data_smithi125_244951_object75515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:54:16.571 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:17.015707+0000", 2023-10-09T10:54:16.571 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 359.458316244, 2023-10-09T10:54:16.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.33061554999999998, 2023-10-09T10:54:16.572 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.572 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.572 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.572 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:16.572 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:16.572 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181958 2023-10-09T10:54:16.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.572 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015707+0000", 2023-10-09T10:54:16.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015707+0000", 2023-10-09T10:54:16.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015706+0000", 2023-10-09T10:54:16.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:16.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.574 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015717+0000", 2023-10-09T10:54:16.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0558e-05 2023-10-09T10:54:16.574 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.574 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.574 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015733+0000", 2023-10-09T10:54:16.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6203999999999999e-05 2023-10-09T10:54:16.574 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.574 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.574 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015739+0000", 2023-10-09T10:54:16.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.088e-06 2023-10-09T10:54:16.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.575 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015793+0000", 2023-10-09T10:54:16.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4046999999999999e-05 2023-10-09T10:54:16.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:54:16.575 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015802+0000", 2023-10-09T10:54:16.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.772e-06 2023-10-09T10:54:16.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.331544+0000", 2023-10-09T10:54:16.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.31574197300000001 2023-10-09T10:54:16.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.576 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.331667+0000", 2023-10-09T10:54:16.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012380900000000001 2023-10-09T10:54:16.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.576 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:54:16.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.331748+0000", 2023-10-09T10:54:16.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0389000000000006e-05 2023-10-09T10:54:16.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.338838+0000", 2023-10-09T10:54:16.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007089962 2023-10-09T10:54:16.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.343219+0000", 2023-10-09T10:54:16.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043816970000000004 2023-10-09T10:54:16.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346261+0000", 2023-10-09T10:54:16.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030411969999999998 2023-10-09T10:54:16.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346294+0000", 2023-10-09T10:54:16.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3383000000000002e-05 2023-10-09T10:54:16.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.579 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.579 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346323+0000", 2023-10-09T10:54:16.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8864999999999999e-05 2023-10-09T10:54:16.579 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.579 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.579 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.579 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:182031 6.2 6:424d2b82:::benchmark_data_smithi125_244951_object75588:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:54:16.579 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:17.198389+0000", 2023-10-09T10:54:16.579 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 359.27563442500002, 2023-10-09T10:54:16.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15372846300000001, 2023-10-09T10:54:16.581 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.581 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.581 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.581 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:16.581 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:16.581 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 182031 2023-10-09T10:54:16.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.582 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.582 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.582 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.582 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198389+0000", 2023-10-09T10:54:16.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.582 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.582 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.582 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.582 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198389+0000", 2023-10-09T10:54:16.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.582 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.583 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.583 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198388+0000", 2023-10-09T10:54:16.583 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:16.583 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.583 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.583 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198396+0000", 2023-10-09T10:54:16.583 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7409999999999992e-06 2023-10-09T10:54:16.583 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.584 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.584 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198409+0000", 2023-10-09T10:54:16.584 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.343e-05 2023-10-09T10:54:16.584 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.584 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.584 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.584 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198415+0000", 2023-10-09T10:54:16.584 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.314e-06 2023-10-09T10:54:16.584 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.584 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.584 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198483+0000", 2023-10-09T10:54:16.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7677999999999998e-05 2023-10-09T10:54:16.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:54:16.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198494+0000", 2023-10-09T10:54:16.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1236000000000001e-05 2023-10-09T10:54:16.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.335697+0000", 2023-10-09T10:54:16.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.137203295 2023-10-09T10:54:16.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.335802+0000", 2023-10-09T10:54:16.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104095 2023-10-09T10:54:16.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:54:16.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.335901+0000", 2023-10-09T10:54:16.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9179999999999996e-05 2023-10-09T10:54:16.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.343158+0000", 2023-10-09T10:54:16.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0072576120000000001 2023-10-09T10:54:16.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346383+0000", 2023-10-09T10:54:16.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003224312 2023-10-09T10:54:16.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.352022+0000", 2023-10-09T10:54:16.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0056397219999999998 2023-10-09T10:54:16.588 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.588 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.588 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.352075+0000", 2023-10-09T10:54:16.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2369e-05 2023-10-09T10:54:16.588 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.588 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.588 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.352118+0000", 2023-10-09T10:54:16.589 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2911999999999998e-05 2023-10-09T10:54:16.589 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.589 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.589 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.589 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.589 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.589 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:152608 7.b 7:d0fd2adb:::benchmark_data_smithi125_295816_object61471:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e245)", 2023-10-09T10:54:16.589 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:52:10.363211+0000", 2023-10-09T10:54:16.589 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 126.110812618, 2023-10-09T10:54:16.589 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8185187439999999, 2023-10-09T10:54:16.589 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.589 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.590 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.590 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:54:16.590 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:54:16.590 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 152608 2023-10-09T10:54:16.590 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.590 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363211+0000", 2023-10-09T10:54:16.590 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.591 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.591 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363211+0000", 2023-10-09T10:54:16.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.591 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.591 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363209+0000", 2023-10-09T10:54:16.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:54:16.592 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.592 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363218+0000", 2023-10-09T10:54:16.592 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8470000000000002e-06 2023-10-09T10:54:16.592 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.592 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363236+0000", 2023-10-09T10:54:16.592 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8062000000000001e-05 2023-10-09T10:54:16.592 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.593 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.593 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363244+0000", 2023-10-09T10:54:16.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2560000000000002e-06 2023-10-09T10:54:16.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.593 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.593 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.365179+0000", 2023-10-09T10:54:16.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001934755 2023-10-09T10:54:16.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.594 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op must wait for map", 2023-10-09T10:54:16.594 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.365188+0000", 2023-10-09T10:54:16.594 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.132e-06 2023-10-09T10:54:16.594 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.594 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.594 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.594 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.380926+0000", 2023-10-09T10:54:16.594 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015737860999999999 2023-10-09T10:54:16.594 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.594 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.594 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:54:16.595 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.380936+0000", 2023-10-09T10:54:16.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.679e-06 2023-10-09T10:54:16.595 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.595 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.595 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.595 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.172400+0000", 2023-10-09T10:54:16.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7914640509999999 2023-10-09T10:54:16.595 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.595 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.595 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.595 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.172506+0000", 2023-10-09T10:54:16.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106677 2023-10-09T10:54:16.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.596 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.596 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T10:54:16.596 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.172597+0000", 2023-10-09T10:54:16.596 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0718999999999997e-05 2023-10-09T10:54:16.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.596 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.596 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.596 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.176419+0000", 2023-10-09T10:54:16.596 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0038220480000000002 2023-10-09T10:54:16.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.176443+0000", 2023-10-09T10:54:16.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3583000000000001e-05 2023-10-09T10:54:16.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.176459+0000", 2023-10-09T10:54:16.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5815999999999999e-05 2023-10-09T10:54:16.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.181617+0000", 2023-10-09T10:54:16.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051585210000000001 2023-10-09T10:54:16.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.181654+0000", 2023-10-09T10:54:16.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6573999999999999e-05 2023-10-09T10:54:16.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.181730+0000", 2023-10-09T10:54:16.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6070999999999997e-05 2023-10-09T10:54:16.599 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.599 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.599 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.599 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.599 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.599 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:39213 8.2 8:4e7be898:::benchmark_data_smithi125_329074_object39212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T10:54:16.599 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:53:35.895072+0000", 2023-10-09T10:54:16.599 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 40.578952018000003, 2023-10-09T10:54:16.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.128363015, 2023-10-09T10:54:16.599 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.599 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.599 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.600 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:54:16.600 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:54:16.600 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39213 2023-10-09T10:54:16.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.600 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.600 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895072+0000", 2023-10-09T10:54:16.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895072+0000", 2023-10-09T10:54:16.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895070+0000", 2023-10-09T10:54:16.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:54:16.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895204+0000", 2023-10-09T10:54:16.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000133561 2023-10-09T10:54:16.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895215+0000", 2023-10-09T10:54:16.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0925e-05 2023-10-09T10:54:16.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895220+0000", 2023-10-09T10:54:16.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4949999999999999e-06 2023-10-09T10:54:16.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.603 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895798+0000", 2023-10-09T10:54:16.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00057733700000000003 2023-10-09T10:54:16.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.603 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895926+0000", 2023-10-09T10:54:16.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012778600000000001 2023-10-09T10:54:16.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:54:16.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.896072+0000", 2023-10-09T10:54:16.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014615599999999999 2023-10-09T10:54:16.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.897973+0000", 2023-10-09T10:54:16.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019008510000000001 2023-10-09T10:54:16.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.899072+0000", 2023-10-09T10:54:16.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010995239999999999 2023-10-09T10:54:16.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023353+0000", 2023-10-09T10:54:16.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.124281323 2023-10-09T10:54:16.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023405+0000", 2023-10-09T10:54:16.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1158e-05 2023-10-09T10:54:16.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023435+0000", 2023-10-09T10:54:16.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.006e-05 2023-10-09T10:54:16.606 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.606 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.606 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.606 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:39226 8.2 8:44bd6854:::benchmark_data_smithi125_329074_object39225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T10:54:16.606 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:53:35.900703+0000", 2023-10-09T10:54:16.607 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 40.573320981000002, 2023-10-09T10:54:16.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.122919798, 2023-10-09T10:54:16.607 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.607 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.607 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.607 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:54:16.607 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:54:16.607 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39226 2023-10-09T10:54:16.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.607 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.607 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.900703+0000", 2023-10-09T10:54:16.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.608 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.608 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.608 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.608 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.900703+0000", 2023-10-09T10:54:16.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.608 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.608 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.608 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.608 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.900702+0000", 2023-10-09T10:54:16.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:54:16.608 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.900806+0000", 2023-10-09T10:54:16.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104288 2023-10-09T10:54:16.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.900818+0000", 2023-10-09T10:54:16.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2063e-05 2023-10-09T10:54:16.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.900823+0000", 2023-10-09T10:54:16.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.916e-06 2023-10-09T10:54:16.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.900866+0000", 2023-10-09T10:54:16.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2997999999999997e-05 2023-10-09T10:54:16.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.901022+0000", 2023-10-09T10:54:16.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000155728 2023-10-09T10:54:16.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:54:16.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.901165+0000", 2023-10-09T10:54:16.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000143406 2023-10-09T10:54:16.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.902667+0000", 2023-10-09T10:54:16.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001501893 2023-10-09T10:54:16.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.903304+0000", 2023-10-09T10:54:16.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00063756399999999997 2023-10-09T10:54:16.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023536+0000", 2023-10-09T10:54:16.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.12023191 2023-10-09T10:54:16.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.613 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023587+0000", 2023-10-09T10:54:16.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0411999999999998e-05 2023-10-09T10:54:16.613 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.613 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.613 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.613 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023622+0000", 2023-10-09T10:54:16.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5750000000000002e-05 2023-10-09T10:54:16.613 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.613 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.613 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.613 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.614 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.614 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:39229 8.2 8:481c8ff5:::benchmark_data_smithi125_329074_object39228:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T10:54:16.614 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:53:35.903946+0000", 2023-10-09T10:54:16.614 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 40.570077740000002, 2023-10-09T10:54:16.614 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.119790275, 2023-10-09T10:54:16.614 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.614 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.614 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.614 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:54:16.614 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:54:16.614 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39229 2023-10-09T10:54:16.614 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.615 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.615 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.615 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.615 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.903946+0000", 2023-10-09T10:54:16.615 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.615 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.615 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.615 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.615 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.903946+0000", 2023-10-09T10:54:16.615 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.615 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.616 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.903944+0000", 2023-10-09T10:54:16.616 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:54:16.616 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.616 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.904004+0000", 2023-10-09T10:54:16.616 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9895000000000002e-05 2023-10-09T10:54:16.616 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.904038+0000", 2023-10-09T10:54:16.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4507e-05 2023-10-09T10:54:16.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.904055+0000", 2023-10-09T10:54:16.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7172000000000001e-05 2023-10-09T10:54:16.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.904183+0000", 2023-10-09T10:54:16.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012730200000000001 2023-10-09T10:54:16.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.904404+0000", 2023-10-09T10:54:16.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022130199999999999 2023-10-09T10:54:16.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:54:16.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.904499+0000", 2023-10-09T10:54:16.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4823999999999999e-05 2023-10-09T10:54:16.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.619 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.906412+0000", 2023-10-09T10:54:16.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019136839999999999 2023-10-09T10:54:16.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.619 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.906462+0000", 2023-10-09T10:54:16.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9450999999999998e-05 2023-10-09T10:54:16.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.620 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023666+0000", 2023-10-09T10:54:16.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11720370300000001 2023-10-09T10:54:16.620 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.620 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.620 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.620 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023710+0000", 2023-10-09T10:54:16.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3934999999999999e-05 2023-10-09T10:54:16.620 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.620 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.620 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.620 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023736+0000", 2023-10-09T10:54:16.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6593000000000001e-05 2023-10-09T10:54:16.621 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.621 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.621 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.621 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.621 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.621 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:39230 8.2 8:49dd141d:::benchmark_data_smithi125_329074_object39229:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T10:54:16.621 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:53:35.904084+0000", 2023-10-09T10:54:16.621 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 40.569940041999999, 2023-10-09T10:54:16.621 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11972348300000001, 2023-10-09T10:54:16.621 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.621 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.621 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.622 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:54:16.622 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:54:16.622 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39230 2023-10-09T10:54:16.622 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.622 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.622 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.622 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.622 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.904084+0000", 2023-10-09T10:54:16.622 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.622 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.622 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.623 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.623 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.904084+0000", 2023-10-09T10:54:16.623 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.623 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.623 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.623 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.623 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.904082+0000", 2023-10-09T10:54:16.623 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:16.623 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.623 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.623 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.623 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.904143+0000", 2023-10-09T10:54:16.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0798999999999998e-05 2023-10-09T10:54:16.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.624 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.624 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.904163+0000", 2023-10-09T10:54:16.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9738999999999999e-05 2023-10-09T10:54:16.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.624 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.624 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.904175+0000", 2023-10-09T10:54:16.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2567e-05 2023-10-09T10:54:16.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.905469+0000", 2023-10-09T10:54:16.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012937319999999999 2023-10-09T10:54:16.625 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.905589+0000", 2023-10-09T10:54:16.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000120326 2023-10-09T10:54:16.625 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:54:16.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.905724+0000", 2023-10-09T10:54:16.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000134399 2023-10-09T10:54:16.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.626 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.906591+0000", 2023-10-09T10:54:16.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000867583 2023-10-09T10:54:16.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.626 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.906785+0000", 2023-10-09T10:54:16.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00019417499999999999 2023-10-09T10:54:16.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023759+0000", 2023-10-09T10:54:16.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.116973222 2023-10-09T10:54:16.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023789+0000", 2023-10-09T10:54:16.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9833999999999999e-05 2023-10-09T10:54:16.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.628 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023807+0000", 2023-10-09T10:54:16.628 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8527000000000001e-05 2023-10-09T10:54:16.628 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.628 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.628 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.628 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.628 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:69148 8.2 8:488aeec9:::benchmark_data_smithi125_329074_object69147:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-10-09T10:54:16.628 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:53:57.740349+0000", 2023-10-09T10:54:16.628 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 18.733674902000001, 2023-10-09T10:54:16.628 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.103684918, 2023-10-09T10:54:16.628 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.629 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.629 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.629 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:54:16.629 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:54:16.629 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 69148 2023-10-09T10:54:16.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.629 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.629 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.629 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.629 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.740349+0000", 2023-10-09T10:54:16.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.630 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.630 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.740349+0000", 2023-10-09T10:54:16.630 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.630 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.630 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.740347+0000", 2023-10-09T10:54:16.630 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:16.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.740401+0000", 2023-10-09T10:54:16.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3609999999999997e-05 2023-10-09T10:54:16.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.740413+0000", 2023-10-09T10:54:16.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.257e-05 2023-10-09T10:54:16.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.740419+0000", 2023-10-09T10:54:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8780000000000003e-06 2023-10-09T10:54:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.740457+0000", 2023-10-09T10:54:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7913000000000001e-05 2023-10-09T10:54:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.740584+0000", 2023-10-09T10:54:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012719700000000001 2023-10-09T10:54:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:54:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.740679+0000", 2023-10-09T10:54:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4877000000000007e-05 2023-10-09T10:54:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.744475+0000", 2023-10-09T10:54:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0037954019999999998 2023-10-09T10:54:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.754560+0000", 2023-10-09T10:54:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010085724000000001 2023-10-09T10:54:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.843933+0000", 2023-10-09T10:54:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.089372280999999998 2023-10-09T10:54:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.843987+0000", 2023-10-09T10:54:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4132999999999998e-05 2023-10-09T10:54:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.844034+0000", 2023-10-09T10:54:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6894000000000002e-05 2023-10-09T10:54:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:69160 8.2 8:40690474:::benchmark_data_smithi125_329074_object69159:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-10-09T10:54:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:53:57.746763+0000", 2023-10-09T10:54:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 18.727260176000001, 2023-10-09T10:54:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.097460890999999994, 2023-10-09T10:54:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:54:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:54:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 69160 2023-10-09T10:54:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.746763+0000", 2023-10-09T10:54:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.746763+0000", 2023-10-09T10:54:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.746763+0000", 2023-10-09T10:54:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:54:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.746847+0000", 2023-10-09T10:54:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4172000000000007e-05 2023-10-09T10:54:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.746856+0000", 2023-10-09T10:54:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7080000000000003e-06 2023-10-09T10:54:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.746860+0000", 2023-10-09T10:54:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4669999999999998e-06 2023-10-09T10:54:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.746898+0000", 2023-10-09T10:54:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7840999999999999e-05 2023-10-09T10:54:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.747010+0000", 2023-10-09T10:54:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000111508 2023-10-09T10:54:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:54:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.747102+0000", 2023-10-09T10:54:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2607000000000003e-05 2023-10-09T10:54:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.754587+0000", 2023-10-09T10:54:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0074848120000000004 2023-10-09T10:54:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.755554+0000", 2023-10-09T10:54:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00096673500000000003 2023-10-09T10:54:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.844143+0000", 2023-10-09T10:54:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.088589755000000006 2023-10-09T10:54:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.844184+0000", 2023-10-09T10:54:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0760000000000003e-05 2023-10-09T10:54:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:57.844224+0000", 2023-10-09T10:54:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0160999999999997e-05 2023-10-09T10:54:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.643 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:16.643 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:16.643 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:54:16.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:16.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:16.744 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:16.795 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:16 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T10:54:16.494+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e280 definitely_dead 0 2023-10-09T10:54:16.798 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:16.830 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:16.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:16.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:16.901 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:54:16.935 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:16.946 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 58 16 83857 83841 90.3216 42.25 0.0206347 0.0110658 2023-10-09T10:54:17.020 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:17.051 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:17.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:17.070 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.2. 2023-10-09T10:54:17.070 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:54:17.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:17.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1580084273' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:54:17.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3574271622' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T10:54:17.184 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:17.204 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:17.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:17.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:17.259 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:54:17.308 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:17.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1580084273' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:54:17.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3574271622' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T10:54:17.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1580084273' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:54:17.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3574271622' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T10:54:17.336 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:17.401 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:54:17.406 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:17.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:17.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:17.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:54:17.537 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:17.557 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:54:17.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:17.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:54:17.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:17.691 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:17.860 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:17.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:17.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:17.951 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 59 16 83975 83959 88.9079 7.375 0.00320569 0.0110822 2023-10-09T10:54:17.965 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:17.993 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:18.061 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:18.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:18.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:18.082 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:18 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:54:18.082 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1580084273' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:54:18.082 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3574271622' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T10:54:18.082 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:18 smithi155 ceph-mon[184854]: osdmap e281: 8 total, 8 up, 7 in 2023-10-09T10:54:18.082 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:18 smithi155 ceph-mon[184854]: failed to encode map e281 with expected crc 2023-10-09T10:54:18.082 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:18 smithi155 ceph-mon[184854]: failed to encode map e281 with expected crc 2023-10-09T10:54:18.082 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:18 smithi155 ceph-mon[184854]: pgmap v674: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 45 MiB/s wr, 716 op/s; 23393/247569 objects degraded (9.449%); 8945/247569 objects misplaced (3.613%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:54:18.082 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:18 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 23393/244323 objects degraded (9.575%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:18.082 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:18 smithi155 ceph-mon[184854]: osdmap e282: 8 total, 8 up, 7 in 2023-10-09T10:54:18.083 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:18 smithi155 ceph-mon[184854]: failed to encode map e282 with expected crc 2023-10-09T10:54:18.083 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:18 smithi155 ceph-mon[184854]: failed to encode map e282 with expected crc 2023-10-09T10:54:18.165 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:18.255 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:54:18.256 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:54:18.256 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T10:54:18.256 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:54:18.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:54:18.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1580084273' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:54:18.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3574271622' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T10:54:18.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[200197]: osdmap e281: 8 total, 8 up, 7 in 2023-10-09T10:54:18.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[200197]: failed to encode map e281 with expected crc 2023-10-09T10:54:18.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[200197]: failed to encode map e281 with expected crc 2023-10-09T10:54:18.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[200197]: pgmap v674: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 45 MiB/s wr, 716 op/s; 23393/247569 objects degraded (9.449%); 8945/247569 objects misplaced (3.613%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:54:18.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 23393/244323 objects degraded (9.575%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:18.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[200197]: osdmap e282: 8 total, 8 up, 7 in 2023-10-09T10:54:18.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[200197]: failed to encode map e282 with expected crc 2023-10-09T10:54:18.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[200197]: failed to encode map e282 with expected crc 2023-10-09T10:54:18.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:54:18.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1580084273' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:54:18.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3574271622' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T10:54:18.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[195507]: osdmap e281: 8 total, 8 up, 7 in 2023-10-09T10:54:18.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[195507]: failed to encode map e281 with expected crc 2023-10-09T10:54:18.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[195507]: failed to encode map e281 with expected crc 2023-10-09T10:54:18.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[195507]: pgmap v674: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 45 MiB/s wr, 716 op/s; 23393/247569 objects degraded (9.449%); 8945/247569 objects misplaced (3.613%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T10:54:18.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 23393/244323 objects degraded (9.575%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:18.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[195507]: osdmap e282: 8 total, 8 up, 7 in 2023-10-09T10:54:18.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[195507]: failed to encode map e282 with expected crc 2023-10-09T10:54:18.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:18 smithi125 ceph-mon[195507]: failed to encode map e282 with expected crc 2023-10-09T10:54:18.320 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:18.610 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:18.612 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:54:18.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:18.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:18.714 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:18.741 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:18.813 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:18.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:18.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:18.917 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:18.935 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:54:18.944 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:18.951 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:54:18.952310+0000 min lat: 0.00119012 max lat: 1.796 avg lat: 0.0110939 2023-10-09T10:54:18.951 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:54:18.951 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 60 16 83979 83963 87.4307 0.25 0.00312201 0.0110939 2023-10-09T10:54:19.258 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[195507]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.258 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[195507]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.258 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[195507]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.258 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[195507]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.258 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[195507]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[195507]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[195507]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[195507]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[195507]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[195507]: osdmap e283: 8 total, 8 up, 7 in 2023-10-09T10:54:19.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[195507]: 8.b starting backfill to osd.2 from (0'0,0'0] MAX to 280'5303 2023-10-09T10:54:19.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[195507]: 8.a starting backfill to osd.2 from (0'0,0'0] MAX to 280'5188 2023-10-09T10:54:19.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[195507]: 8.f starting backfill to osd.2 from (0'0,0'0] MAX to 280'5319 2023-10-09T10:54:19.259 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[200197]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.259 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[200197]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[200197]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[200197]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[200197]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[200197]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[200197]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[200197]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[200197]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[200197]: osdmap e283: 8 total, 8 up, 7 in 2023-10-09T10:54:19.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[200197]: 8.b starting backfill to osd.2 from (0'0,0'0] MAX to 280'5303 2023-10-09T10:54:19.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[200197]: 8.a starting backfill to osd.2 from (0'0,0'0] MAX to 280'5188 2023-10-09T10:54:19.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:19 smithi125 ceph-mon[200197]: 8.f starting backfill to osd.2 from (0'0,0'0] MAX to 280'5319 2023-10-09T10:54:19.425 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:19 smithi155 ceph-mon[184854]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.425 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:19 smithi155 ceph-mon[184854]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.425 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:19 smithi155 ceph-mon[184854]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.425 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:19 smithi155 ceph-mon[184854]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.425 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:19 smithi155 ceph-mon[184854]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.426 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:19 smithi155 ceph-mon[184854]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.426 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:19 smithi155 ceph-mon[184854]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.426 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:19 smithi155 ceph-mon[184854]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.426 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:19 smithi155 ceph-mon[184854]: failed to encode map e282 with expected crc 2023-10-09T10:54:19.426 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:19 smithi155 ceph-mon[184854]: osdmap e283: 8 total, 8 up, 7 in 2023-10-09T10:54:19.426 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:19 smithi155 ceph-mon[184854]: 8.b starting backfill to osd.2 from (0'0,0'0] MAX to 280'5303 2023-10-09T10:54:19.426 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:19 smithi155 ceph-mon[184854]: 8.a starting backfill to osd.2 from (0'0,0'0] MAX to 280'5188 2023-10-09T10:54:19.426 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:19 smithi155 ceph-mon[184854]: 8.f starting backfill to osd.2 from (0'0,0'0] MAX to 280'5319 2023-10-09T10:54:19.952 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 61 16 84353 84337 86.3805 23.375 0.0583007 0.0115695 2023-10-09T10:54:19.953 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:19.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:19.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:20.061 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:20.135 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:54:20.135 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:54:20.135 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:54:20.135 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:54:20.135 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:54:20.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:20 smithi155 ceph-mon[184854]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:20 smithi155 ceph-mon[184854]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:20 smithi155 ceph-mon[184854]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:20 smithi155 ceph-mon[184854]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:20 smithi155 ceph-mon[184854]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:20 smithi155 ceph-mon[184854]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:20 smithi155 ceph-mon[184854]: 8.4 starting backfill to osd.2 from (0'0,0'0] MAX to 280'5270 2023-10-09T10:54:20.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:20 smithi155 ceph-mon[184854]: 8.7 starting backfill to osd.2 from (0'0,0'0] MAX to 280'5196 2023-10-09T10:54:20.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:20 smithi155 ceph-mon[184854]: pgmap v677: 119 pgs: 4 remapped+peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 34 MiB/s wr, 538 op/s; 23966/249990 objects degraded (9.587%); 8945/249990 objects misplaced (3.578%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:54:20.156 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:20.189 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:20.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:20.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:20.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[195507]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[195507]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[195507]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[195507]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[195507]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.221 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[195507]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.221 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[195507]: 8.4 starting backfill to osd.2 from (0'0,0'0] MAX to 280'5270 2023-10-09T10:54:20.221 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[195507]: 8.7 starting backfill to osd.2 from (0'0,0'0] MAX to 280'5196 2023-10-09T10:54:20.221 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[195507]: pgmap v677: 119 pgs: 4 remapped+peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 34 MiB/s wr, 538 op/s; 23966/249990 objects degraded (9.587%); 8945/249990 objects misplaced (3.578%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:54:20.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[200197]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[200197]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[200197]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[200197]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[200197]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[200197]: failed to encode map e283 with expected crc 2023-10-09T10:54:20.222 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[200197]: 8.4 starting backfill to osd.2 from (0'0,0'0] MAX to 280'5270 2023-10-09T10:54:20.222 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[200197]: 8.7 starting backfill to osd.2 from (0'0,0'0] MAX to 280'5196 2023-10-09T10:54:20.222 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:20 smithi125 ceph-mon[200197]: pgmap v677: 119 pgs: 4 remapped+peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 34 MiB/s wr, 538 op/s; 23966/249990 objects degraded (9.587%); 8945/249990 objects misplaced (3.578%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T10:54:20.259 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-09T10:54:20.281 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:20.293 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:20.377 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:20.471 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:20.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:20.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:20.524 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:54:20.575 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:20.627 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:20.662 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:20.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:20.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:20.770 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:20.843 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:20.888 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:20.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:20.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:20.952 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 62 16 84960 84944 85.5993 37.9375 0.0242326 0.0116783 2023-10-09T10:54:20.976 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:54:21.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:21.056 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:21.136 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:21.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:21.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:21.215 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:21 smithi125 ceph-mon[195507]: failed to encode map e284 with expected crc 2023-10-09T10:54:21.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:21 smithi125 ceph-mon[195507]: failed to encode map e284 with expected crc 2023-10-09T10:54:21.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:21 smithi125 ceph-mon[195507]: failed to encode map e284 with expected crc 2023-10-09T10:54:21.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:21 smithi125 ceph-mon[195507]: osdmap e284: 8 total, 8 up, 7 in 2023-10-09T10:54:21.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:21 smithi125 ceph-mon[195507]: failed to encode map e284 with expected crc 2023-10-09T10:54:21.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:21 smithi125 ceph-mon[200197]: failed to encode map e284 with expected crc 2023-10-09T10:54:21.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:21 smithi125 ceph-mon[200197]: failed to encode map e284 with expected crc 2023-10-09T10:54:21.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:21 smithi125 ceph-mon[200197]: failed to encode map e284 with expected crc 2023-10-09T10:54:21.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:21 smithi125 ceph-mon[200197]: osdmap e284: 8 total, 8 up, 7 in 2023-10-09T10:54:21.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:21 smithi125 ceph-mon[200197]: failed to encode map e284 with expected crc 2023-10-09T10:54:21.218 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:54:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:54:21] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.43.0" 2023-10-09T10:54:21.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:21.307 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:21.413 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:21.418 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:21 smithi155 ceph-mon[184854]: failed to encode map e284 with expected crc 2023-10-09T10:54:21.418 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:21 smithi155 ceph-mon[184854]: failed to encode map e284 with expected crc 2023-10-09T10:54:21.419 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:21 smithi155 ceph-mon[184854]: failed to encode map e284 with expected crc 2023-10-09T10:54:21.419 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:21 smithi155 ceph-mon[184854]: osdmap e284: 8 total, 8 up, 7 in 2023-10-09T10:54:21.419 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:21 smithi155 ceph-mon[184854]: failed to encode map e284 with expected crc 2023-10-09T10:54:21.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:21.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:21.520 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:21.564 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:21.692 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:21.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:21.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:21.777 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:54:21.797 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:21.835 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:54:21.884 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:54:21.885 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:21.907 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:21.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:21.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:21.914 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:54:21.952 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 63 16 85561 85545 84.8369 37.5625 0.0146898 0.0117818 2023-10-09T10:54:21.960 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:54:21.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:54:21.992 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:54:22.014 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:22.042 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:54:22.065 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:22.070 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:54:22.086 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:54:22.087 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:54:22.087 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:54:22.087 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:54:22.087 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:54:22.087 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:54:22.087 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:54:22.087 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:54:22.105 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:22.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:22.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:22.209 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:54:22.213 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:54:22.213 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:54:22.304 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:22.384 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:22 smithi155 ceph-mon[184854]: 1.1c scrub starts 2023-10-09T10:54:22.384 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:22 smithi155 ceph-mon[184854]: osdmap e285: 8 total, 8 up, 7 in 2023-10-09T10:54:22.384 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:22 smithi155 ceph-mon[184854]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.384 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:22 smithi155 ceph-mon[184854]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.384 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:22 smithi155 ceph-mon[184854]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.384 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:22 smithi155 ceph-mon[184854]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.385 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:22 smithi155 ceph-mon[184854]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.385 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:22 smithi155 ceph-mon[184854]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.385 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:22 smithi155 ceph-mon[184854]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.385 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:22 smithi155 ceph-mon[184854]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.385 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:22 smithi155 ceph-mon[184854]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.385 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:22 smithi155 ceph-mon[184854]: pgmap v680: 119 pgs: 1 activating, 2 peering, 4 remapped+peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 104 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 28 MiB/s wr, 448 op/s; 23520/252948 objects degraded (9.298%); 17330/252948 objects misplaced (6.851%); 3.5 MiB/s, 2 keys/s, 56 objects/s recovering 2023-10-09T10:54:22.385 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:22 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2053087129' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:22.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[195507]: 1.1c scrub starts 2023-10-09T10:54:22.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[195507]: osdmap e285: 8 total, 8 up, 7 in 2023-10-09T10:54:22.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[195507]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[195507]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[195507]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[195507]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[195507]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[195507]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[195507]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[195507]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[195507]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[195507]: pgmap v680: 119 pgs: 1 activating, 2 peering, 4 remapped+peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 104 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 28 MiB/s wr, 448 op/s; 23520/252948 objects degraded (9.298%); 17330/252948 objects misplaced (6.851%); 3.5 MiB/s, 2 keys/s, 56 objects/s recovering 2023-10-09T10:54:22.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2053087129' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:22.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[200197]: 1.1c scrub starts 2023-10-09T10:54:22.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[200197]: osdmap e285: 8 total, 8 up, 7 in 2023-10-09T10:54:22.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[200197]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[200197]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[200197]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[200197]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[200197]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[200197]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[200197]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[200197]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[200197]: failed to encode map e285 with expected crc 2023-10-09T10:54:22.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[200197]: pgmap v680: 119 pgs: 1 activating, 2 peering, 4 remapped+peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 104 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 28 MiB/s wr, 448 op/s; 23520/252948 objects degraded (9.298%); 17330/252948 objects misplaced (6.851%); 3.5 MiB/s, 2 keys/s, 56 objects/s recovering 2023-10-09T10:54:22.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:22 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2053087129' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:22.583 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:54:22.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:22.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:54:22.598 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:54:22.690 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:22.724 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:22.765 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:22.820 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:22.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:22.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:22.928 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:22.952 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 64 16 86170 86154 84.1062 38.0625 0.0442819 0.0118832 2023-10-09T10:54:22.972 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:23.019 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:23.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:23.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:23.041 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:54:23.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:23.186 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:23.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:23 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 23520/252948 objects degraded (9.298%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:23.321 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:23.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:23.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:23.424 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:23.464 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:23 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 23520/252948 objects degraded (9.298%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:23.464 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:23 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 23520/252948 objects degraded (9.298%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:23.465 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:23.598 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:23.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:23.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:23.708 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:23.763 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:23.896 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:23.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:23.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:23.952 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 65 16 86794 86778 83.4124 39 0.0281139 0.0119834 2023-10-09T10:54:24.003 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:24.066 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:24.148 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:24.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:24.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:24.235 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:54:24.235 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:54:24.258 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:24.317 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:24.361 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:24.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:24.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:24.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:54:24.525 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:24 smithi125 ceph-mon[195507]: 1.1c scrub starts 2023-10-09T10:54:24.525 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:24 smithi125 ceph-mon[195507]: pgmap v681: 119 pgs: 1 activating, 2 peering, 2 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 105 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 32 MiB/s wr, 510 op/s; 33957/255969 objects degraded (13.266%); 17879/255969 objects misplaced (6.985%); 2.6 MiB/s, 1 keys/s, 41 objects/s recovering 2023-10-09T10:54:24.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:24 smithi125 ceph-mon[200197]: 1.1c scrub starts 2023-10-09T10:54:24.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:24 smithi125 ceph-mon[200197]: pgmap v681: 119 pgs: 1 activating, 2 peering, 2 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 105 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 32 MiB/s wr, 510 op/s; 33957/255969 objects degraded (13.266%); 17879/255969 objects misplaced (6.985%); 2.6 MiB/s, 1 keys/s, 41 objects/s recovering 2023-10-09T10:54:24.529 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:24.555 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:24 smithi155 ceph-mon[184854]: 1.1c scrub starts 2023-10-09T10:54:24.555 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:24 smithi155 ceph-mon[184854]: pgmap v681: 119 pgs: 1 activating, 2 peering, 2 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 105 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 32 MiB/s wr, 510 op/s; 33957/255969 objects degraded (13.266%); 17879/255969 objects misplaced (6.985%); 2.6 MiB/s, 1 keys/s, 41 objects/s recovering 2023-10-09T10:54:24.569 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:54:24.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:24.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:54:24.606 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:54:24.618 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:54:24.679 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:24.746 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:24.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:24.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:24.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:24.893 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:24.952 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 66 16 87397 87381 82.7197 37.6875 0.0222356 0.0120831 2023-10-09T10:54:24.965 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:25.016 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:25.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:25.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:25.066 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:25.097 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:54:25.123 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:25.181 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:25.260 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 7, 2] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:54:25.260 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:54:25.261 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_2 2023-10-09T10:54:25.261 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:54:25.261 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:25.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:25.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:25.287 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:54:25.303 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:25 smithi125 ceph-mon[195507]: 1.1c scrub starts 2023-10-09T10:54:25.303 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3047063696' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:25.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:25 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:25.304 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:25 smithi125 ceph-mon[200197]: 1.1c scrub starts 2023-10-09T10:54:25.304 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3047063696' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:25.304 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:25 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:25.368 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:25.396 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:25.574 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:25 smithi155 ceph-mon[184854]: 1.1c scrub starts 2023-10-09T10:54:25.575 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3047063696' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:25.575 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:25 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:25.952 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 67 16 87781 87765 81.8435 24 0.0266792 0.0122116 2023-10-09T10:54:26.348 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:54:26.349 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:54:26.380 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:26 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:54:26.380 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:26 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:54:26.380 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:26 smithi155 ceph-mon[184854]: osdmap e286: 8 total, 8 up, 7 in 2023-10-09T10:54:26.380 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:26 smithi155 ceph-mon[184854]: failed to encode map e286 with expected crc 2023-10-09T10:54:26.380 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:26 smithi155 ceph-mon[184854]: failed to encode map e286 with expected crc 2023-10-09T10:54:26.381 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:26 smithi155 ceph-mon[184854]: failed to encode map e286 with expected crc 2023-10-09T10:54:26.381 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:26 smithi155 ceph-mon[184854]: failed to encode map e286 with expected crc 2023-10-09T10:54:26.381 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:26 smithi155 ceph-mon[184854]: pgmap v683: 119 pgs: 1 activating, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 107 active+clean; 5.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 4.7 KiB/s rd, 32 MiB/s wr, 523 op/s; 33957/259347 objects degraded (13.093%); 18500/259347 objects misplaced (7.133%); 3.7 MiB/s, 1 keys/s, 59 objects/s recovering 2023-10-09T10:54:26.381 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:26 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:26.568 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:54:26.568 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:54:26.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[195507]: osdmap e286: 8 total, 8 up, 7 in 2023-10-09T10:54:26.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[195507]: failed to encode map e286 with expected crc 2023-10-09T10:54:26.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[195507]: failed to encode map e286 with expected crc 2023-10-09T10:54:26.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[195507]: failed to encode map e286 with expected crc 2023-10-09T10:54:26.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[195507]: failed to encode map e286 with expected crc 2023-10-09T10:54:26.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[195507]: pgmap v683: 119 pgs: 1 activating, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 107 active+clean; 5.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 4.7 KiB/s rd, 32 MiB/s wr, 523 op/s; 33957/259347 objects degraded (13.093%); 18500/259347 objects misplaced (7.133%); 3.7 MiB/s, 1 keys/s, 59 objects/s recovering 2023-10-09T10:54:26.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:26.569 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:54:26.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:54:26.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[200197]: osdmap e286: 8 total, 8 up, 7 in 2023-10-09T10:54:26.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[200197]: failed to encode map e286 with expected crc 2023-10-09T10:54:26.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[200197]: failed to encode map e286 with expected crc 2023-10-09T10:54:26.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[200197]: failed to encode map e286 with expected crc 2023-10-09T10:54:26.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[200197]: failed to encode map e286 with expected crc 2023-10-09T10:54:26.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[200197]: pgmap v683: 119 pgs: 1 activating, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 107 active+clean; 5.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 4.7 KiB/s rd, 32 MiB/s wr, 523 op/s; 33957/259347 objects degraded (13.093%); 18500/259347 objects misplaced (7.133%); 3.7 MiB/s, 1 keys/s, 59 objects/s recovering 2023-10-09T10:54:26.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:26 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:26.763 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:54:26.933 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:26.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:26.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:26.953 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 68 16 88439 88423 81.2434 41.125 0.0255565 0.0123021 2023-10-09T10:54:27.039 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:27.094 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:27.128 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:27.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:27.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:27.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:27.249 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:54:27.323 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:27.352 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:27.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:27.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:27.399 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:27.404 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:27 smithi155 ceph-mon[184854]: 1.1c scrub starts 2023-10-09T10:54:27.404 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:27 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:54:27.460 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:27.534 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:27.537 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:27 smithi125 ceph-mon[200197]: 1.1c scrub starts 2023-10-09T10:54:27.537 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:27 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:54:27.537 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:27 smithi125 ceph-mon[195507]: 1.1c scrub starts 2023-10-09T10:54:27.537 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:27 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:54:27.622 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:27.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:27.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:27.726 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:27.797 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:27.934 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:27.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:27.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:27.953 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 69 16 88984 88968 80.5598 34.0625 0.0483858 0.0124067 2023-10-09T10:54:28.044 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:28.130 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:28.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:28.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:28.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:28.373 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:28.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:28 smithi155 ceph-mon[184854]: 1.1c scrub starts 2023-10-09T10:54:28.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:28 smithi155 ceph-mon[184854]: pgmap v684: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.9 KiB/s rd, 35 MiB/s wr, 559 op/s; 33957/262068 objects degraded (12.957%); 18675/262068 objects misplaced (7.126%); 3.1 MiB/s, 1 keys/s, 49 objects/s recovering 2023-10-09T10:54:28.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:28 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 33957/259347 objects degraded (13.093%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:28.459 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:28.547 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:54:28.552 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:54:28.552 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:54:28.560 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:28.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:28.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:28.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:28 smithi125 ceph-mon[200197]: 1.1c scrub starts 2023-10-09T10:54:28.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:28 smithi125 ceph-mon[200197]: pgmap v684: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.9 KiB/s rd, 35 MiB/s wr, 559 op/s; 33957/262068 objects degraded (12.957%); 18675/262068 objects misplaced (7.126%); 3.1 MiB/s, 1 keys/s, 49 objects/s recovering 2023-10-09T10:54:28.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:28 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 33957/259347 objects degraded (13.093%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:28.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:28 smithi125 ceph-mon[195507]: 1.1c scrub starts 2023-10-09T10:54:28.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:28 smithi125 ceph-mon[195507]: pgmap v684: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.9 KiB/s rd, 35 MiB/s wr, 559 op/s; 33957/262068 objects degraded (12.957%); 18675/262068 objects misplaced (7.126%); 3.1 MiB/s, 1 keys/s, 49 objects/s recovering 2023-10-09T10:54:28.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:28 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 33957/259347 objects degraded (13.093%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:28.576 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:54:28.613 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:54:28.650 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:54:28.667 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:28.694 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:54:28.729 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:28.763 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:54:28.765 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:28.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:28.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:28.833 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:54:28.871 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:28.887 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:54:28.904 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:54:28.904 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:54:28.904 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:54:28.904 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:54:28.904 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:54:28.904 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:54:28.904 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:54:28.904 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:54:28.937 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:28.953 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 70 16 89518 89502 79.8858 33.375 0.0478325 0.0125117 2023-10-09T10:54:29.046 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:54:29.067 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:54:29.411 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:29.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:29.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:29.448 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:54:29.464 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:54:29.477 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:29 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2170862360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:29.517 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:29.555 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:54:29.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:29 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2170862360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:29.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:29 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2170862360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:29.580 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:29.954 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 71 16 90080 90064 79.2555 35.125 0.0276104 0.0126108 2023-10-09T10:54:30.300 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:30 smithi155 ceph-mon[184854]: pgmap v685: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.4 KiB/s rd, 32 MiB/s wr, 519 op/s; 33957/264714 objects degraded (12.828%); 18675/264714 objects misplaced (7.055%); 1.0 MiB/s, 1 keys/s, 16 objects/s recovering 2023-10-09T10:54:30.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:30 smithi155 ceph-mon[184854]: from='client.45413 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:54:30.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:30.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:54:30.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:30 smithi125 ceph-mon[195507]: pgmap v685: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.4 KiB/s rd, 32 MiB/s wr, 519 op/s; 33957/264714 objects degraded (12.828%); 18675/264714 objects misplaced (7.055%); 1.0 MiB/s, 1 keys/s, 16 objects/s recovering 2023-10-09T10:54:30.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:30 smithi125 ceph-mon[195507]: from='client.45413 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:54:30.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:30.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:54:30.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:30 smithi125 ceph-mon[200197]: pgmap v685: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.4 KiB/s rd, 32 MiB/s wr, 519 op/s; 33957/264714 objects degraded (12.828%); 18675/264714 objects misplaced (7.055%); 1.0 MiB/s, 1 keys/s, 16 objects/s recovering 2023-10-09T10:54:30.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:30 smithi125 ceph-mon[200197]: from='client.45413 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:54:30.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:30.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:54:30.955 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 72 16 90704 90688 78.6949 39 0.0100311 0.0127024 2023-10-09T10:54:31.037 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:31.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:31.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:31.143 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:31.216 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:31.223 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:54:31.223 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:54:31.223 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.223 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:90834 8.b e286/282 8:d5c437d9:::benchmark_data_smithi125_329074_object90833:head v 286'5701, mlcod=0'0)", 2023-10-09T10:54:31.223 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:31.208165+0000", 2023-10-09T10:54:31.223 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0077904469999999998, 2023-10-09T10:54:31.223 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0078220129999999992, 2023-10-09T10:54:31.223 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:31.223 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:54:31.223 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:31.223 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.223 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:31.224 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.208165+0000", 2023-10-09T10:54:31.224 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:31.224 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.224 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.224 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:31.224 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.208165+0000", 2023-10-09T10:54:31.224 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:31.224 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.224 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.224 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:31.224 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.208164+0000", 2023-10-09T10:54:31.224 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:31.225 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.225 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.225 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:31.225 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.208215+0000", 2023-10-09T10:54:31.225 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1246999999999999e-05 2023-10-09T10:54:31.225 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.225 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.225 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:31.225 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.208235+0000", 2023-10-09T10:54:31.225 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0183999999999999e-05 2023-10-09T10:54:31.225 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.225 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.225 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:31.226 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.208243+0000", 2023-10-09T10:54:31.226 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9500000000000001e-06 2023-10-09T10:54:31.226 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.226 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.226 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:31.226 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.208986+0000", 2023-10-09T10:54:31.226 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00074329300000000001 2023-10-09T10:54:31.226 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.226 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.226 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:31.226 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.209009+0000", 2023-10-09T10:54:31.226 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3068e-05 2023-10-09T10:54:31.226 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:31.227 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:31.227 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:31.227 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.227 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.227 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:90849 8.b e286/282 8:ddc84ce4:::benchmark_data_smithi125_329074_object90848:head v 286'5702, mlcod=0'0)", 2023-10-09T10:54:31.227 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:31.208896+0000", 2023-10-09T10:54:31.227 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0070591689999999997, 2023-10-09T10:54:31.227 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0071568159999999999, 2023-10-09T10:54:31.227 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:31.227 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:54:31.227 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:31.227 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.227 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:31.228 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.208896+0000", 2023-10-09T10:54:31.228 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:31.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.228 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.228 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:31.228 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.208896+0000", 2023-10-09T10:54:31.228 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:31.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.228 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.228 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:31.228 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.208896+0000", 2023-10-09T10:54:31.228 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:54:31.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:31.229 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.208922+0000", 2023-10-09T10:54:31.229 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6358000000000001e-05 2023-10-09T10:54:31.229 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:31.229 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.208932+0000", 2023-10-09T10:54:31.229 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0074999999999999e-05 2023-10-09T10:54:31.229 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:31.229 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.208938+0000", 2023-10-09T10:54:31.230 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5110000000000003e-06 2023-10-09T10:54:31.230 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.230 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.230 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:31.230 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.215086+0000", 2023-10-09T10:54:31.230 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061481060000000004 2023-10-09T10:54:31.230 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.230 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.230 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:31.230 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.215112+0000", 2023-10-09T10:54:31.230 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6123999999999999e-05 2023-10-09T10:54:31.230 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:31.230 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:31.231 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:31.231 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.231 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.231 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:90851 8.b e286/282)", 2023-10-09T10:54:31.231 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:31.214956+0000", 2023-10-09T10:54:31.231 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.00099975200000000002, 2023-10-09T10:54:31.231 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00115245, 2023-10-09T10:54:31.231 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:31.231 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "queued for pg", 2023-10-09T10:54:31.231 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:31.231 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.231 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:31.231 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.214956+0000", 2023-10-09T10:54:31.232 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:31.232 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.232 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.232 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:31.232 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.214956+0000", 2023-10-09T10:54:31.232 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:31.232 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.232 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.232 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:31.232 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.214955+0000", 2023-10-09T10:54:31.232 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:54:31.232 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.232 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.233 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:31.233 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.214997+0000", 2023-10-09T10:54:31.233 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2144000000000002e-05 2023-10-09T10:54:31.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.233 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.233 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:31.233 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.215010+0000", 2023-10-09T10:54:31.233 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2815e-05 2023-10-09T10:54:31.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:31.233 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:31.233 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:31.233 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.215016+0000", 2023-10-09T10:54:31.234 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8739999999999997e-06 2023-10-09T10:54:31.234 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:31.234 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:31.234 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:31.234 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:31.234 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T10:54:31.234 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 3 2023-10-09T10:54:31.234 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:54:31.324 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:54:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:54:31] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T10:54:31.352 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:31.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:31.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:31.464 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:31.531 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:31.646 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:31.672 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:54:31.672 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:31.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:31.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:31.731 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":685,"stamp":"2023-10-09T10:54:29.348482+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5781873554,"num_objects":88238,"num_object_clones":0,"num_object_copies":264714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33957,"num_objects_misplaced":18675,"num_objects_unfound":0,"num_objects_dirty":88238,"num_whiteouts":0,"num_read":71,"num_read_kb":132,"num_write":88310,"num_write_kb":5646073,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":713,"num_bytes_recovered":44597339,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89360,"ondisk_log_size":89360,"up":357,"acting":353,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17600104,"kb_used_data":14820016,"kb_used_omap":117,"kb_used_meta":2779914,"kb_avail":638615960,"statfs":{"total":671965249536,"available":653942743040,"internally_reserved":0,"allocated":15175696384,"data_stored":15167114851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120174,"internal_metadata":2846632594},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":16,"apply_latency_ms":16,"commit_latency_ns":16000000,"apply_latency_ns":16000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":278742498,"num_objects":4253,"num_object_clones":0,"num_object_copies":12759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10437,"num_objects_misplaced":6681,"num_objects_unfound":0,"num_objects_dirty":4253,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":4260,"num_write_kb":272265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":135,"num_bytes_recovered":8769992,"num_keys_recovered":10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.223284"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":619,"reported_epoch":285,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:54:21.134006+0000","last_change":"2023-10-09T10:54:19.094257+0000","last_active":"2023-10-09T10:54:21.134006+0000","last_peered":"2023-10-09T10:54:21.134006+0000","last_clean":"2023-10-09T10:54:08.295760+0000","last_became_active":"2023-10-09T10:54:19.093928+0000","last_became_peered":"2023-10-09T10:54:19.093928+0000","last_unstale":"2023-10-09T10:54:21.134006+0000","last_undegraded":"2023-10-09T10:54:21.134006+0000","last_fullsized":"2023-10-09T10:54:21.134006+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:40.174975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:40.174975+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:38:44.409770+0000","scrub_duration":0.010654205,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[0,5,7],"avail_no_missing":["0","1","5","7"],"object_location_counts":[{"shards":"0,1,5,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":14,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.119683+0000","last_change":"2023-10-09T10:54:17.875178+0000","last_active":"2023-10-09T10:54:20.119683+0000","last_peered":"2023-10-09T10:54:20.119683+0000","last_clean":"2023-10-09T10:54:20.119683+0000","last_became_active":"2023-10-09T10:54:17.874821+0000","last_became_peered":"2023-10-09T10:54:17.874821+0000","last_unstale":"2023-10-09T10:54:20.119683+0000","last_undegraded":"2023-10-09T10:54:20.119683+0000","last_fullsized":"2023-10-09T10:54:20.119683+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:25.057745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":283,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.108650+0000","last_change":"2023-10-09T10:54:02.863751+0000","last_active":"2023-10-09T10:54:20.108650+0000","last_peered":"2023-10-09T10:54:20.108650+0000","last_clean":"2023-10-09T10:54:20.108650+0000","last_became_active":"2023-10-09T10:54:02.863405+0000","last_became_peered":"2023-10-09T10:54:02.863405+0000","last_unstale":"2023-10-09T10:54:20.108650+0000","last_undegraded":"2023-10-09T10:54:20.108650+0000","last_fullsized":"2023-10-09T10:54:20.108650+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:11:13.015855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":666,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.140770+0000","last_change":"2023-10-09T10:53:34.537094+0000","last_active":"2023-10-09T10:54:21.140770+0000","last_peered":"2023-10-09T10:54:21.140770+0000","last_clean":"2023-10-09T10:54:21.140770+0000","last_became_active":"2023-10-09T10:52:55.414995+0000","last_became_peered":"2023-10-09T10:52:55.414995+0000","last_unstale":"2023-10-09T10:54:21.140770+0000","last_undegraded":"2023-10-09T10:54:21.140770+0000","last_fullsized":"2023-10-09T10:54:21.140770+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.536962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.536962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:30.197404+0000","scrub_duration":0.0053325960000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":103,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.112951+0000","last_change":"2023-10-09T10:54:02.873362+0000","last_active":"2023-10-09T10:54:20.112951+0000","last_peered":"2023-10-09T10:54:20.112951+0000","last_clean":"2023-10-09T10:54:20.112951+0000","last_became_active":"2023-10-09T10:54:02.873017+0000","last_became_peered":"2023-10-09T10:54:02.873017+0000","last_unstale":"2023-10-09T10:54:20.112951+0000","last_undegraded":"2023-10-09T10:54:20.112951+0000","last_fullsized":"2023-10-09T10:54:20.112951+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:04:34.827345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":194,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.159794+0000","last_change":"2023-10-09T10:53:14.823051+0000","last_active":"2023-10-09T10:54:21.159794+0000","last_peered":"2023-10-09T10:54:21.159794+0000","last_clean":"2023-10-09T10:54:21.159794+0000","last_became_active":"2023-10-09T10:53:14.822334+0000","last_became_peered":"2023-10-09T10:53:14.822334+0000","last_unstale":"2023-10-09T10:54:21.159794+0000","last_undegraded":"2023-10-09T10:54:21.159794+0000","last_fullsized":"2023-10-09T10:54:21.159794+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:46.573381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":649,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.143394+0000","last_change":"2023-10-09T10:54:05.914198+0000","last_active":"2023-10-09T10:54:21.143394+0000","last_peered":"2023-10-09T10:54:21.143394+0000","last_clean":"2023-10-09T10:54:21.143394+0000","last_became_active":"2023-10-09T10:54:05.913869+0000","last_became_peered":"2023-10-09T10:54:05.913869+0000","last_unstale":"2023-10-09T10:54:21.143394+0000","last_undegraded":"2023-10-09T10:54:21.143394+0000","last_fullsized":"2023-10-09T10:54:21.143394+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:28:43.317418+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":328,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.143101+0000","last_change":"2023-10-09T10:54:17.911379+0000","last_active":"2023-10-09T10:54:20.143101+0000","last_peered":"2023-10-09T10:54:20.143101+0000","last_clean":"2023-10-09T10:54:20.143101+0000","last_became_active":"2023-10-09T10:54:17.911063+0000","last_became_peered":"2023-10-09T10:54:17.911063+0000","last_unstale":"2023-10-09T10:54:20.143101+0000","last_undegraded":"2023-10-09T10:54:20.143101+0000","last_fullsized":"2023-10-09T10:54:20.143101+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:37.876290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":20,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.143628+0000","last_change":"2023-10-09T10:54:02.865193+0000","last_active":"2023-10-09T10:54:21.143628+0000","last_peered":"2023-10-09T10:54:21.143628+0000","last_clean":"2023-10-09T10:54:21.143628+0000","last_became_active":"2023-10-09T10:54:02.864906+0000","last_became_peered":"2023-10-09T10:54:02.864906+0000","last_unstale":"2023-10-09T10:54:21.143628+0000","last_undegraded":"2023-10-09T10:54:21.143628+0000","last_fullsized":"2023-10-09T10:54:21.143628+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:15:31.356702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":700,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.142913+0000","last_change":"2023-10-09T10:54:05.917242+0000","last_active":"2023-10-09T10:54:21.142913+0000","last_peered":"2023-10-09T10:54:21.142913+0000","last_clean":"2023-10-09T10:54:21.142913+0000","last_became_active":"2023-10-09T10:54:05.916918+0000","last_became_peered":"2023-10-09T10:54:05.916918+0000","last_unstale":"2023-10-09T10:54:21.142913+0000","last_undegraded":"2023-10-09T10:54:21.142913+0000","last_fullsized":"2023-10-09T10:54:21.142913+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.787782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.787782+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:54:02.577841+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":157,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.144119+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:54:21.144119+0000","last_peered":"2023-10-09T10:54:21.144119+0000","last_clean":"2023-10-09T10:54:21.144119+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:54:21.144119+0000","last_undegraded":"2023-10-09T10:54:21.144119+0000","last_fullsized":"2023-10-09T10:54:21.144119+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":283,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.143143+0000","last_change":"2023-10-09T10:54:02.864501+0000","last_active":"2023-10-09T10:54:20.143143+0000","last_peered":"2023-10-09T10:54:20.143143+0000","last_clean":"2023-10-09T10:54:20.143143+0000","last_became_active":"2023-10-09T10:54:02.864305+0000","last_became_peered":"2023-10-09T10:54:02.864305+0000","last_unstale":"2023-10-09T10:54:20.143143+0000","last_undegraded":"2023-10-09T10:54:20.143143+0000","last_fullsized":"2023-10-09T10:54:20.143143+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:00.225241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":529,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.165662+0000","last_change":"2023-10-09T10:54:21.161539+0000","last_active":"2023-10-09T10:54:21.165662+0000","last_peered":"2023-10-09T10:54:21.165662+0000","last_clean":"2023-10-09T10:54:21.165662+0000","last_became_active":"2023-10-09T10:54:21.161106+0000","last_became_peered":"2023-10-09T10:54:21.161106+0000","last_unstale":"2023-10-09T10:54:21.165662+0000","last_undegraded":"2023-10-09T10:54:21.165662+0000","last_fullsized":"2023-10-09T10:54:21.165662+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:21.824982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:21.824982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":19,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.108372+0000","last_change":"2023-10-09T10:54:02.858554+0000","last_active":"2023-10-09T10:54:20.108372+0000","last_peered":"2023-10-09T10:54:20.108372+0000","last_clean":"2023-10-09T10:54:20.108372+0000","last_became_active":"2023-10-09T10:54:02.858197+0000","last_became_peered":"2023-10-09T10:54:02.858197+0000","last_unstale":"2023-10-09T10:54:20.108372+0000","last_undegraded":"2023-10-09T10:54:20.108372+0000","last_fullsized":"2023-10-09T10:54:20.108372+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:06.405267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":231,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.112829+0000","last_change":"2023-10-09T10:53:14.838859+0000","last_active":"2023-10-09T10:54:20.112829+0000","last_peered":"2023-10-09T10:54:20.112829+0000","last_clean":"2023-10-09T10:54:20.112829+0000","last_became_active":"2023-10-09T10:53:14.838536+0000","last_became_peered":"2023-10-09T10:53:14.838536+0000","last_unstale":"2023-10-09T10:54:20.112829+0000","last_undegraded":"2023-10-09T10:54:20.112829+0000","last_fullsized":"2023-10-09T10:54:20.112829+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:44:19.980568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":673,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.143719+0000","last_change":"2023-10-09T10:53:32.540768+0000","last_active":"2023-10-09T10:54:21.143719+0000","last_peered":"2023-10-09T10:54:21.143719+0000","last_clean":"2023-10-09T10:54:21.143719+0000","last_became_active":"2023-10-09T10:52:55.414818+0000","last_became_peered":"2023-10-09T10:52:55.414818+0000","last_unstale":"2023-10-09T10:54:21.143719+0000","last_undegraded":"2023-10-09T10:54:21.143719+0000","last_fullsized":"2023-10-09T10:54:21.143719+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:32.540610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:32.540610+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:47.374383+0000","scrub_duration":0.0064251300000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":239,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.108725+0000","last_change":"2023-10-09T10:53:14.847647+0000","last_active":"2023-10-09T10:54:20.108725+0000","last_peered":"2023-10-09T10:54:20.108725+0000","last_clean":"2023-10-09T10:54:20.108725+0000","last_became_active":"2023-10-09T10:53:14.832089+0000","last_became_peered":"2023-10-09T10:53:14.832089+0000","last_unstale":"2023-10-09T10:54:20.108725+0000","last_undegraded":"2023-10-09T10:54:20.108725+0000","last_fullsized":"2023-10-09T10:54:20.108725+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:34:45.371102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.120277+0000","last_change":"2023-10-09T10:54:17.879013+0000","last_active":"2023-10-09T10:54:20.120277+0000","last_peered":"2023-10-09T10:54:20.120277+0000","last_clean":"2023-10-09T10:54:20.120277+0000","last_became_active":"2023-10-09T10:54:17.878650+0000","last_became_peered":"2023-10-09T10:54:17.878650+0000","last_unstale":"2023-10-09T10:54:20.120277+0000","last_undegraded":"2023-10-09T10:54:20.120277+0000","last_fullsized":"2023-10-09T10:54:20.120277+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:40.449383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":650,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.159493+0000","last_change":"2023-10-09T10:54:05.924596+0000","last_active":"2023-10-09T10:54:21.159493+0000","last_peered":"2023-10-09T10:54:21.159493+0000","last_clean":"2023-10-09T10:54:21.159493+0000","last_became_active":"2023-10-09T10:54:05.924286+0000","last_became_peered":"2023-10-09T10:54:05.924286+0000","last_unstale":"2023-10-09T10:54:21.159493+0000","last_undegraded":"2023-10-09T10:54:21.159493+0000","last_fullsized":"2023-10-09T10:54:21.159493+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.200574+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.200574+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001591812,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":39,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.143976+0000","last_change":"2023-10-09T10:53:14.869000+0000","last_active":"2023-10-09T10:54:21.143976+0000","last_peered":"2023-10-09T10:54:21.143976+0000","last_clean":"2023-10-09T10:54:21.143976+0000","last_became_active":"2023-10-09T10:53:14.831904+0000","last_became_peered":"2023-10-09T10:53:14.831904+0000","last_unstale":"2023-10-09T10:54:21.143976+0000","last_undegraded":"2023-10-09T10:54:21.143976+0000","last_fullsized":"2023-10-09T10:54:21.143976+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:38.571980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":280,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.147319+0000","last_change":"2023-10-09T10:54:02.875497+0000","last_active":"2023-10-09T10:54:21.147319+0000","last_peered":"2023-10-09T10:54:21.147319+0000","last_clean":"2023-10-09T10:54:21.147319+0000","last_became_active":"2023-10-09T10:54:02.873941+0000","last_became_peered":"2023-10-09T10:54:02.873941+0000","last_unstale":"2023-10-09T10:54:21.147319+0000","last_undegraded":"2023-10-09T10:54:21.147319+0000","last_fullsized":"2023-10-09T10:54:21.147319+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:08:24.583780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":651,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.142758+0000","last_change":"2023-10-09T10:49:18.708370+0000","last_active":"2023-10-09T10:54:21.142758+0000","last_peered":"2023-10-09T10:54:21.142758+0000","last_clean":"2023-10-09T10:54:21.142758+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:54:21.142758+0000","last_undegraded":"2023-10-09T10:54:21.142758+0000","last_fullsized":"2023-10-09T10:54:21.142758+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:57.808410+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":51,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.149938+0000","last_change":"2023-10-09T10:54:18.080112+0000","last_active":"2023-10-09T10:54:21.149938+0000","last_peered":"2023-10-09T10:54:21.149938+0000","last_clean":"2023-10-09T10:54:21.149938+0000","last_became_active":"2023-10-09T10:54:17.878848+0000","last_became_peered":"2023-10-09T10:54:17.878848+0000","last_unstale":"2023-10-09T10:54:21.149938+0000","last_undegraded":"2023-10-09T10:54:21.149938+0000","last_fullsized":"2023-10-09T10:54:21.149938+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:12.314073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":333,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.107828+0000","last_change":"2023-10-09T10:54:17.884326+0000","last_active":"2023-10-09T10:54:20.107828+0000","last_peered":"2023-10-09T10:54:20.107828+0000","last_clean":"2023-10-09T10:54:20.107828+0000","last_became_active":"2023-10-09T10:54:17.883720+0000","last_became_peered":"2023-10-09T10:54:17.883720+0000","last_unstale":"2023-10-09T10:54:20.107828+0000","last_undegraded":"2023-10-09T10:54:20.107828+0000","last_fullsized":"2023-10-09T10:54:20.107828+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:49:25.299060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":633,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.147802+0000","last_change":"2023-10-09T10:53:17.846464+0000","last_active":"2023-10-09T10:54:21.147802+0000","last_peered":"2023-10-09T10:54:21.147802+0000","last_clean":"2023-10-09T10:54:21.147802+0000","last_became_active":"2023-10-09T10:53:17.845979+0000","last_became_peered":"2023-10-09T10:53:17.845979+0000","last_unstale":"2023-10-09T10:54:21.147802+0000","last_undegraded":"2023-10-09T10:54:21.147802+0000","last_fullsized":"2023-10-09T10:54:21.147802+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:56.295299+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":193,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.141520+0000","last_change":"2023-10-09T10:53:14.821961+0000","last_active":"2023-10-09T10:54:21.141520+0000","last_peered":"2023-10-09T10:54:21.141520+0000","last_clean":"2023-10-09T10:54:21.141520+0000","last_became_active":"2023-10-09T10:53:14.821625+0000","last_became_peered":"2023-10-09T10:53:14.821625+0000","last_unstale":"2023-10-09T10:54:21.141520+0000","last_undegraded":"2023-10-09T10:54:21.141520+0000","last_fullsized":"2023-10-09T10:54:21.141520+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:29.305952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":235,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.108586+0000","last_change":"2023-10-09T10:53:14.832387+0000","last_active":"2023-10-09T10:54:20.108586+0000","last_peered":"2023-10-09T10:54:20.108586+0000","last_clean":"2023-10-09T10:54:20.108586+0000","last_became_active":"2023-10-09T10:53:14.831672+0000","last_became_peered":"2023-10-09T10:53:14.831672+0000","last_unstale":"2023-10-09T10:54:20.108586+0000","last_undegraded":"2023-10-09T10:54:20.108586+0000","last_fullsized":"2023-10-09T10:54:20.108586+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:59.194806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":610,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.117171+0000","last_change":"2023-10-09T10:54:05.928993+0000","last_active":"2023-10-09T10:54:20.117171+0000","last_peered":"2023-10-09T10:54:20.117171+0000","last_clean":"2023-10-09T10:54:20.117171+0000","last_became_active":"2023-10-09T10:54:05.928687+0000","last_became_peered":"2023-10-09T10:54:05.928687+0000","last_unstale":"2023-10-09T10:54:20.117171+0000","last_undegraded":"2023-10-09T10:54:20.117171+0000","last_fullsized":"2023-10-09T10:54:20.117171+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.077788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.077788+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":216,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.142718+0000","last_change":"2023-10-09T10:53:14.835799+0000","last_active":"2023-10-09T10:54:21.142718+0000","last_peered":"2023-10-09T10:54:21.142718+0000","last_clean":"2023-10-09T10:54:21.142718+0000","last_became_active":"2023-10-09T10:53:14.835476+0000","last_became_peered":"2023-10-09T10:53:14.835476+0000","last_unstale":"2023-10-09T10:54:21.142718+0000","last_undegraded":"2023-10-09T10:54:21.142718+0000","last_fullsized":"2023-10-09T10:54:21.142718+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:29.248961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":175,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.159571+0000","last_change":"2023-10-09T10:53:14.823485+0000","last_active":"2023-10-09T10:54:21.159571+0000","last_peered":"2023-10-09T10:54:21.159571+0000","last_clean":"2023-10-09T10:54:21.159571+0000","last_became_active":"2023-10-09T10:53:14.822756+0000","last_became_peered":"2023-10-09T10:53:14.822756+0000","last_unstale":"2023-10-09T10:54:21.159571+0000","last_undegraded":"2023-10-09T10:54:21.159571+0000","last_fullsized":"2023-10-09T10:54:21.159571+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:05.359313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":645,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.142183+0000","last_change":"2023-10-09T10:54:05.916105+0000","last_active":"2023-10-09T10:54:21.142183+0000","last_peered":"2023-10-09T10:54:21.142183+0000","last_clean":"2023-10-09T10:54:21.142183+0000","last_became_active":"2023-10-09T10:54:05.915781+0000","last_became_peered":"2023-10-09T10:54:05.915781+0000","last_unstale":"2023-10-09T10:54:21.142183+0000","last_undegraded":"2023-10-09T10:54:21.142183+0000","last_fullsized":"2023-10-09T10:54:21.142183+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:14.863915+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":160,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.134478+0000","last_change":"2023-10-09T10:51:12.366910+0000","last_active":"2023-10-09T10:54:21.134478+0000","last_peered":"2023-10-09T10:54:21.134478+0000","last_clean":"2023-10-09T10:54:21.134478+0000","last_became_active":"2023-10-09T10:51:12.366565+0000","last_became_peered":"2023-10-09T10:51:12.366565+0000","last_unstale":"2023-10-09T10:54:21.134478+0000","last_undegraded":"2023-10-09T10:54:21.134478+0000","last_fullsized":"2023-10-09T10:54:21.134478+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:53.464251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":310,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.126672+0000","last_change":"2023-10-09T10:54:17.885566+0000","last_active":"2023-10-09T10:54:20.126672+0000","last_peered":"2023-10-09T10:54:20.126672+0000","last_clean":"2023-10-09T10:54:20.126672+0000","last_became_active":"2023-10-09T10:54:17.885234+0000","last_became_peered":"2023-10-09T10:54:17.885234+0000","last_unstale":"2023-10-09T10:54:20.126672+0000","last_undegraded":"2023-10-09T10:54:20.126672+0000","last_fullsized":"2023-10-09T10:54:20.126672+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:30.481658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"286'5511","reported_seq":5552,"reported_epoch":286,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:54:26.240804+0000","last_change":"2023-10-09T10:54:19.219180+0000","last_active":"2023-10-09T10:54:26.240804+0000","last_peered":"2023-10-09T10:54:26.240804+0000","last_clean":"2023-10-09T10:54:17.036595+0000","last_became_active":"2023-10-09T10:54:19.209139+0000","last_became_peered":"2023-10-09T10:54:19.209139+0000","last_unstale":"2023-10-09T10:54:26.240804+0000","last_undegraded":"2023-10-09T10:54:26.240804+0000","last_fullsized":"2023-10-09T10:54:26.240804+0000","mapping_epoch":282,"log_start":"270'100","ondisk_log_start":"270'100","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5411,"log_dups_size":0,"ondisk_log_size":5411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:44.976899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361168896,"num_objects":5511,"num_object_clones":0,"num_object_copies":16533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5511,"num_objects_unfound":0,"num_objects_dirty":5511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5511,"num_write_kb":352704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":5511}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":617,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.169799+0000","last_change":"2023-10-09T10:54:21.166098+0000","last_active":"2023-10-09T10:54:21.169799+0000","last_peered":"2023-10-09T10:54:21.169799+0000","last_clean":"2023-10-09T10:54:21.169799+0000","last_became_active":"2023-10-09T10:54:21.165706+0000","last_became_peered":"2023-10-09T10:54:21.165706+0000","last_unstale":"2023-10-09T10:54:21.169799+0000","last_undegraded":"2023-10-09T10:54:21.169799+0000","last_fullsized":"2023-10-09T10:54:21.169799+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:18.764311+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":33,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.144273+0000","last_change":"2023-10-09T10:53:14.854904+0000","last_active":"2023-10-09T10:54:21.144273+0000","last_peered":"2023-10-09T10:54:21.144273+0000","last_clean":"2023-10-09T10:54:21.144273+0000","last_became_active":"2023-10-09T10:53:14.829611+0000","last_became_peered":"2023-10-09T10:53:14.829611+0000","last_unstale":"2023-10-09T10:54:21.144273+0000","last_undegraded":"2023-10-09T10:54:21.144273+0000","last_fullsized":"2023-10-09T10:54:21.144273+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:26.064630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":253,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.135098+0000","last_change":"2023-10-09T10:53:14.831930+0000","last_active":"2023-10-09T10:54:21.135098+0000","last_peered":"2023-10-09T10:54:21.135098+0000","last_clean":"2023-10-09T10:54:21.135098+0000","last_became_active":"2023-10-09T10:53:14.831600+0000","last_became_peered":"2023-10-09T10:53:14.831600+0000","last_unstale":"2023-10-09T10:54:21.135098+0000","last_undegraded":"2023-10-09T10:54:21.135098+0000","last_fullsized":"2023-10-09T10:54:21.135098+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:16.104668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"286'5475","reported_seq":5502,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T10:54:26.134593+0000","last_change":"2023-10-09T10:53:14.832791+0000","last_active":"2023-10-09T10:54:26.134593+0000","last_peered":"2023-10-09T10:54:26.134593+0000","last_clean":"2023-10-09T10:54:26.134593+0000","last_became_active":"2023-10-09T10:53:14.832491+0000","last_became_peered":"2023-10-09T10:53:14.832491+0000","last_unstale":"2023-10-09T10:54:26.134593+0000","last_undegraded":"2023-10-09T10:54:26.134593+0000","last_fullsized":"2023-10-09T10:54:26.134593+0000","mapping_epoch":266,"log_start":"270'300","ondisk_log_start":"270'300","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5175,"log_dups_size":0,"ondisk_log_size":5175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:51.903123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358809600,"num_objects":5475,"num_object_clones":0,"num_object_copies":16425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5475,"num_write_kb":350400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":644,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.107670+0000","last_change":"2023-10-09T10:53:25.540494+0000","last_active":"2023-10-09T10:54:20.107670+0000","last_peered":"2023-10-09T10:54:20.107670+0000","last_clean":"2023-10-09T10:54:20.107670+0000","last_became_active":"2023-10-09T10:51:01.295474+0000","last_became_peered":"2023-10-09T10:51:01.295474+0000","last_unstale":"2023-10-09T10:54:20.107670+0000","last_undegraded":"2023-10-09T10:54:20.107670+0000","last_fullsized":"2023-10-09T10:54:20.107670+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:39.069143+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":47,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.146488+0000","last_change":"2023-10-09T10:54:02.892130+0000","last_active":"2023-10-09T10:54:21.146488+0000","last_peered":"2023-10-09T10:54:21.146488+0000","last_clean":"2023-10-09T10:54:21.146488+0000","last_became_active":"2023-10-09T10:54:02.869738+0000","last_became_peered":"2023-10-09T10:54:02.869738+0000","last_unstale":"2023-10-09T10:54:21.146488+0000","last_undegraded":"2023-10-09T10:54:21.146488+0000","last_fullsized":"2023-10-09T10:54:21.146488+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:14.524316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":261,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.112681+0000","last_change":"2023-10-09T10:54:02.863541+0000","last_active":"2023-10-09T10:54:20.112681+0000","last_peered":"2023-10-09T10:54:20.112681+0000","last_clean":"2023-10-09T10:54:20.112681+0000","last_became_active":"2023-10-09T10:54:02.863178+0000","last_became_peered":"2023-10-09T10:54:02.863178+0000","last_unstale":"2023-10-09T10:54:20.112681+0000","last_undegraded":"2023-10-09T10:54:20.112681+0000","last_fullsized":"2023-10-09T10:54:20.112681+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:08:59.424126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.6","version":"286'5583","reported_seq":5614,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T10:54:28.262539+0000","last_change":"2023-10-09T10:53:14.826832+0000","last_active":"2023-10-09T10:54:28.262539+0000","last_peered":"2023-10-09T10:54:28.262539+0000","last_clean":"2023-10-09T10:54:28.262539+0000","last_became_active":"2023-10-09T10:53:14.826507+0000","last_became_peered":"2023-10-09T10:53:14.826507+0000","last_unstale":"2023-10-09T10:54:28.262539+0000","last_undegraded":"2023-10-09T10:54:28.262539+0000","last_fullsized":"2023-10-09T10:54:28.262539+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5583,"log_dups_size":0,"ondisk_log_size":5583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:06:41.609418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365887488,"num_objects":5583,"num_object_clones":0,"num_object_copies":16749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5583,"num_write_kb":357312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":598,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.147130+0000","last_change":"2023-10-09T10:53:36.680150+0000","last_active":"2023-10-09T10:54:21.147130+0000","last_peered":"2023-10-09T10:54:21.147130+0000","last_clean":"2023-10-09T10:54:21.147130+0000","last_became_active":"2023-10-09T10:51:50.371032+0000","last_became_peered":"2023-10-09T10:51:50.371032+0000","last_unstale":"2023-10-09T10:54:21.147130+0000","last_undegraded":"2023-10-09T10:54:21.147130+0000","last_fullsized":"2023-10-09T10:54:21.147130+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.680090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.680090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:08:28.952827+0000","scrub_duration":0.005577172,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":33,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.142829+0000","last_change":"2023-10-09T10:53:14.841343+0000","last_active":"2023-10-09T10:54:21.142829+0000","last_peered":"2023-10-09T10:54:21.142829+0000","last_clean":"2023-10-09T10:54:21.142829+0000","last_became_active":"2023-10-09T10:53:14.828213+0000","last_became_peered":"2023-10-09T10:53:14.828213+0000","last_unstale":"2023-10-09T10:54:21.142829+0000","last_undegraded":"2023-10-09T10:54:21.142829+0000","last_fullsized":"2023-10-09T10:54:21.142829+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:28.137803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":206,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.150543+0000","last_change":"2023-10-09T10:54:02.857163+0000","last_active":"2023-10-09T10:54:21.150543+0000","last_peered":"2023-10-09T10:54:21.150543+0000","last_clean":"2023-10-09T10:54:21.150543+0000","last_became_active":"2023-10-09T10:54:02.856813+0000","last_became_peered":"2023-10-09T10:54:02.856813+0000","last_unstale":"2023-10-09T10:54:21.150543+0000","last_undegraded":"2023-10-09T10:54:21.150543+0000","last_fullsized":"2023-10-09T10:54:21.150543+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:25:41.790684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"285'5445","reported_seq":5490,"reported_epoch":285,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:54:25.169784+0000","last_change":"2023-10-09T10:54:19.427578+0000","last_active":"2023-10-09T10:54:25.169784+0000","last_peered":"2023-10-09T10:54:25.169784+0000","last_clean":"2023-10-09T10:54:02.391425+0000","last_became_active":"2023-10-09T10:54:19.377070+0000","last_became_peered":"2023-10-09T10:54:19.377070+0000","last_unstale":"2023-10-09T10:54:25.169784+0000","last_undegraded":"2023-10-09T10:54:19.303250+0000","last_fullsized":"2023-10-09T10:54:25.169784+0000","mapping_epoch":282,"log_start":"271'500","ondisk_log_start":"271'500","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":4945,"log_dups_size":0,"ondisk_log_size":4945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:01.697673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356843520,"num_objects":5445,"num_object_clones":0,"num_object_copies":16335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4602,"num_objects_misplaced":843,"num_objects_unfound":0,"num_objects_dirty":5445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5445,"num_write_kb":348480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,4],"avail_no_missing":["3","1","6"],"object_location_counts":[{"shards":"1,3,6","objects":4602},{"shards":"3,4,6","objects":843}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":731,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.145948+0000","last_change":"2023-10-09T10:53:38.681891+0000","last_active":"2023-10-09T10:54:21.145948+0000","last_peered":"2023-10-09T10:54:21.145948+0000","last_clean":"2023-10-09T10:54:21.145948+0000","last_became_active":"2023-10-09T10:53:17.840478+0000","last_became_peered":"2023-10-09T10:53:17.840478+0000","last_unstale":"2023-10-09T10:54:21.145948+0000","last_undegraded":"2023-10-09T10:54:21.145948+0000","last_fullsized":"2023-10-09T10:54:21.145948+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:31:14.758036+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.108024+0000","last_change":"2023-10-09T10:51:50.374527+0000","last_active":"2023-10-09T10:54:20.108024+0000","last_peered":"2023-10-09T10:54:20.108024+0000","last_clean":"2023-10-09T10:54:20.108024+0000","last_became_active":"2023-10-09T10:51:50.374297+0000","last_became_peered":"2023-10-09T10:51:50.374297+0000","last_unstale":"2023-10-09T10:54:20.108024+0000","last_undegraded":"2023-10-09T10:54:20.108024+0000","last_fullsized":"2023-10-09T10:54:20.108024+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:31:49.358307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":184,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.141106+0000","last_change":"2023-10-09T10:54:02.867825+0000","last_active":"2023-10-09T10:54:21.141106+0000","last_peered":"2023-10-09T10:54:21.141106+0000","last_clean":"2023-10-09T10:54:21.141106+0000","last_became_active":"2023-10-09T10:54:02.867539+0000","last_became_peered":"2023-10-09T10:54:02.867539+0000","last_unstale":"2023-10-09T10:54:21.141106+0000","last_undegraded":"2023-10-09T10:54:21.141106+0000","last_fullsized":"2023-10-09T10:54:21.141106+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:13:48.047792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"285'5490","reported_seq":6133,"reported_epoch":285,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:54:25.156646+0000","last_change":"2023-10-09T10:54:18.089367+0000","last_active":"2023-10-09T10:54:25.156646+0000","last_peered":"2023-10-09T10:54:25.156646+0000","last_clean":"2023-10-09T10:54:02.435878+0000","last_became_active":"2023-10-09T10:54:04.139742+0000","last_became_peered":"2023-10-09T10:54:04.139742+0000","last_unstale":"2023-10-09T10:54:25.156646+0000","last_undegraded":"2023-10-09T10:54:04.058828+0000","last_fullsized":"2023-10-09T10:54:03.850075+0000","mapping_epoch":277,"log_start":"271'600","ondisk_log_start":"271'600","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":4890,"log_dups_size":0,"ondisk_log_size":4890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:16.828540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359792640,"num_objects":5490,"num_object_clones":0,"num_object_copies":16470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":4403,"num_objects_unfound":0,"num_objects_dirty":5490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5490,"num_write_kb":351360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":302,"num_bytes_recovered":19791872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,5","objects":4403},{"shards":"3,5","objects":1086}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":693,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.165674+0000","last_change":"2023-10-09T10:54:21.165674+0000","last_active":"2023-10-09T10:54:21.165674+0000","last_peered":"2023-10-09T10:54:21.165674+0000","last_clean":"2023-10-09T10:54:21.165674+0000","last_became_active":"2023-10-09T10:54:21.165317+0000","last_became_peered":"2023-10-09T10:54:21.165317+0000","last_unstale":"2023-10-09T10:54:21.165674+0000","last_undegraded":"2023-10-09T10:54:21.165674+0000","last_fullsized":"2023-10-09T10:54:21.165674+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:34:22.533546+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":280,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.110556+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:54:20.110556+0000","last_peered":"2023-10-09T10:54:20.110556+0000","last_clean":"2023-10-09T10:54:20.110556+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:54:20.110556+0000","last_undegraded":"2023-10-09T10:54:20.110556+0000","last_fullsized":"2023-10-09T10:54:20.110556+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":125,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.107946+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T10:54:20.107946+0000","last_peered":"2023-10-09T10:54:20.107946+0000","last_clean":"2023-10-09T10:54:20.107946+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T10:54:20.107946+0000","last_undegraded":"2023-10-09T10:54:20.107946+0000","last_fullsized":"2023-10-09T10:54:20.107946+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"286'5532","reported_seq":5576,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:54:28.235707+0000","last_change":"2023-10-09T10:54:18.104295+0000","last_active":"2023-10-09T10:54:28.235707+0000","last_peered":"2023-10-09T10:54:28.235707+0000","last_clean":"2023-10-09T10:54:02.426241+0000","last_became_active":"2023-10-09T10:54:18.092994+0000","last_became_peered":"2023-10-09T10:54:18.092994+0000","last_unstale":"2023-10-09T10:54:28.235707+0000","last_undegraded":"2023-10-09T10:54:17.900122+0000","last_fullsized":"2023-10-09T10:54:17.861561+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5532,"log_dups_size":0,"ondisk_log_size":5532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:30.321489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362545152,"num_objects":5532,"num_object_clones":0,"num_object_copies":16596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5204,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5532,"num_write_kb":354048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,5],"avail_no_missing":["0","1","5"],"object_location_counts":[{"shards":"0,1,5","objects":4631},{"shards":"0,3,5","objects":573},{"shards":"0,5","objects":328}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":588,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.180999+0000","last_change":"2023-10-09T10:54:21.173078+0000","last_active":"2023-10-09T10:54:21.180999+0000","last_peered":"2023-10-09T10:54:21.180999+0000","last_clean":"2023-10-09T10:54:21.180999+0000","last_became_active":"2023-10-09T10:54:21.172767+0000","last_became_peered":"2023-10-09T10:54:21.172767+0000","last_unstale":"2023-10-09T10:54:21.180999+0000","last_undegraded":"2023-10-09T10:54:21.180999+0000","last_fullsized":"2023-10-09T10:54:21.180999+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.532890+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.532890+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:32.335841+0000","scrub_duration":0.0045290130000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":255,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.143629+0000","last_change":"2023-10-09T10:54:02.878463+0000","last_active":"2023-10-09T10:54:21.143629+0000","last_peered":"2023-10-09T10:54:21.143629+0000","last_clean":"2023-10-09T10:54:21.143629+0000","last_became_active":"2023-10-09T10:54:02.878112+0000","last_became_peered":"2023-10-09T10:54:02.878112+0000","last_unstale":"2023-10-09T10:54:21.143629+0000","last_undegraded":"2023-10-09T10:54:21.143629+0000","last_fullsized":"2023-10-09T10:54:21.143629+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:56:16.603654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.122361+0000","last_change":"2023-10-09T10:54:17.879424+0000","last_active":"2023-10-09T10:54:20.122361+0000","last_peered":"2023-10-09T10:54:20.122361+0000","last_clean":"2023-10-09T10:54:20.122361+0000","last_became_active":"2023-10-09T10:54:17.879091+0000","last_became_peered":"2023-10-09T10:54:17.879091+0000","last_unstale":"2023-10-09T10:54:20.122361+0000","last_undegraded":"2023-10-09T10:54:20.122361+0000","last_fullsized":"2023-10-09T10:54:20.122361+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:07.947911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.2","version":"286'5587","reported_seq":5616,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T10:54:26.221319+0000","last_change":"2023-10-09T10:53:13.831715+0000","last_active":"2023-10-09T10:54:26.221319+0000","last_peered":"2023-10-09T10:54:26.221319+0000","last_clean":"2023-10-09T10:54:26.221319+0000","last_became_active":"2023-10-09T10:53:13.831447+0000","last_became_peered":"2023-10-09T10:53:13.831447+0000","last_unstale":"2023-10-09T10:54:26.221319+0000","last_undegraded":"2023-10-09T10:54:26.221319+0000","last_fullsized":"2023-10-09T10:54:26.221319+0000","mapping_epoch":265,"log_start":"270'200","ondisk_log_start":"270'200","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5387,"log_dups_size":0,"ondisk_log_size":5387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:27.579880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366149632,"num_objects":5587,"num_object_clones":0,"num_object_copies":16761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5587,"num_write_kb":357568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":651,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.155095+0000","last_change":"2023-10-09T10:54:21.154379+0000","last_active":"2023-10-09T10:54:21.155095+0000","last_peered":"2023-10-09T10:54:21.155095+0000","last_clean":"2023-10-09T10:54:21.155095+0000","last_became_active":"2023-10-09T10:54:21.153953+0000","last_became_peered":"2023-10-09T10:54:21.153953+0000","last_unstale":"2023-10-09T10:54:21.155095+0000","last_undegraded":"2023-10-09T10:54:21.155095+0000","last_fullsized":"2023-10-09T10:54:21.155095+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.103750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.103750+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":321,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.142585+0000","last_change":"2023-10-09T10:54:17.876795+0000","last_active":"2023-10-09T10:54:21.142585+0000","last_peered":"2023-10-09T10:54:21.142585+0000","last_clean":"2023-10-09T10:54:21.142585+0000","last_became_active":"2023-10-09T10:54:17.876504+0000","last_became_peered":"2023-10-09T10:54:17.876504+0000","last_unstale":"2023-10-09T10:54:21.142585+0000","last_undegraded":"2023-10-09T10:54:21.142585+0000","last_fullsized":"2023-10-09T10:54:21.142585+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:04:48.657295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":19,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.108866+0000","last_change":"2023-10-09T10:54:02.873719+0000","last_active":"2023-10-09T10:54:20.108866+0000","last_peered":"2023-10-09T10:54:20.108866+0000","last_clean":"2023-10-09T10:54:20.108866+0000","last_became_active":"2023-10-09T10:54:02.873238+0000","last_became_peered":"2023-10-09T10:54:02.873238+0000","last_unstale":"2023-10-09T10:54:20.108866+0000","last_undegraded":"2023-10-09T10:54:20.108866+0000","last_fullsized":"2023-10-09T10:54:20.108866+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:17:35.208798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.3","version":"286'5466","reported_seq":5508,"reported_epoch":286,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:54:26.594261+0000","last_change":"2023-10-09T10:54:18.097699+0000","last_active":"2023-10-09T10:54:26.594261+0000","last_peered":"2023-10-09T10:54:26.594261+0000","last_clean":"2023-10-09T10:54:02.411042+0000","last_became_active":"2023-10-09T10:54:18.085941+0000","last_became_peered":"2023-10-09T10:54:18.085941+0000","last_unstale":"2023-10-09T10:54:26.594261+0000","last_undegraded":"2023-10-09T10:54:26.594261+0000","last_fullsized":"2023-10-09T10:54:26.594261+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5466,"log_dups_size":0,"ondisk_log_size":5466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:07.136452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358219776,"num_objects":5466,"num_object_clones":0,"num_object_copies":16398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5178,"num_objects_unfound":0,"num_objects_dirty":5466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5466,"num_write_kb":349824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[1,5,6],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,3,5,6","objects":548},{"shards":"1,5,6","objects":288}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":599,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.158325+0000","last_change":"2023-10-09T10:54:21.152148+0000","last_active":"2023-10-09T10:54:21.158325+0000","last_peered":"2023-10-09T10:54:21.158325+0000","last_clean":"2023-10-09T10:54:21.158325+0000","last_became_active":"2023-10-09T10:54:21.151809+0000","last_became_peered":"2023-10-09T10:54:21.151809+0000","last_unstale":"2023-10-09T10:54:21.158325+0000","last_undegraded":"2023-10-09T10:54:21.158325+0000","last_fullsized":"2023-10-09T10:54:21.158325+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.701128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.701128+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:27.021145+0000","scrub_duration":0.0025071830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":303,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.136435+0000","last_change":"2023-10-09T10:54:02.866530+0000","last_active":"2023-10-09T10:54:21.136435+0000","last_peered":"2023-10-09T10:54:21.136435+0000","last_clean":"2023-10-09T10:54:21.136435+0000","last_became_active":"2023-10-09T10:54:02.866266+0000","last_became_peered":"2023-10-09T10:54:02.866266+0000","last_unstale":"2023-10-09T10:54:21.136435+0000","last_undegraded":"2023-10-09T10:54:21.136435+0000","last_fullsized":"2023-10-09T10:54:21.136435+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:50.217838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":103,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.126535+0000","last_change":"2023-10-09T10:54:02.873800+0000","last_active":"2023-10-09T10:54:20.126535+0000","last_peered":"2023-10-09T10:54:20.126535+0000","last_clean":"2023-10-09T10:54:20.126535+0000","last_became_active":"2023-10-09T10:54:02.873488+0000","last_became_peered":"2023-10-09T10:54:02.873488+0000","last_unstale":"2023-10-09T10:54:20.126535+0000","last_undegraded":"2023-10-09T10:54:20.126535+0000","last_fullsized":"2023-10-09T10:54:20.126535+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:58.095614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.e","version":"286'5608","reported_seq":5646,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:54:28.254573+0000","last_change":"2023-10-09T10:54:19.338847+0000","last_active":"2023-10-09T10:54:28.254573+0000","last_peered":"2023-10-09T10:54:28.254573+0000","last_clean":"2023-10-09T10:54:17.076172+0000","last_became_active":"2023-10-09T10:54:19.327879+0000","last_became_peered":"2023-10-09T10:54:19.327879+0000","last_unstale":"2023-10-09T10:54:28.254573+0000","last_undegraded":"2023-10-09T10:54:18.890724+0000","last_fullsized":"2023-10-09T10:54:18.868702+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5608,"log_dups_size":0,"ondisk_log_size":5608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:49.273305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367525888,"num_objects":5608,"num_object_clones":0,"num_object_copies":16824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5292,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5608,"num_write_kb":358912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":316},{"shards":"0,4,7","objects":5292}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":681,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.126405+0000","last_change":"2023-10-09T10:53:34.602531+0000","last_active":"2023-10-09T10:54:20.126405+0000","last_peered":"2023-10-09T10:54:20.126405+0000","last_clean":"2023-10-09T10:54:20.126405+0000","last_became_active":"2023-10-09T10:53:17.841970+0000","last_became_peered":"2023-10-09T10:53:17.841970+0000","last_unstale":"2023-10-09T10:54:20.126405+0000","last_undegraded":"2023-10-09T10:54:20.126405+0000","last_fullsized":"2023-10-09T10:54:20.126405+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:04.601861+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":275,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.150278+0000","last_change":"2023-10-09T10:54:17.877481+0000","last_active":"2023-10-09T10:54:21.150278+0000","last_peered":"2023-10-09T10:54:21.150278+0000","last_clean":"2023-10-09T10:54:21.150278+0000","last_became_active":"2023-10-09T10:54:17.877116+0000","last_became_peered":"2023-10-09T10:54:17.877116+0000","last_unstale":"2023-10-09T10:54:21.150278+0000","last_undegraded":"2023-10-09T10:54:21.150278+0000","last_fullsized":"2023-10-09T10:54:21.150278+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:19.084003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":204,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.108793+0000","last_change":"2023-10-09T10:54:17.886921+0000","last_active":"2023-10-09T10:54:20.108793+0000","last_peered":"2023-10-09T10:54:20.108793+0000","last_clean":"2023-10-09T10:54:20.108793+0000","last_became_active":"2023-10-09T10:54:17.886584+0000","last_became_peered":"2023-10-09T10:54:17.886584+0000","last_unstale":"2023-10-09T10:54:20.108793+0000","last_undegraded":"2023-10-09T10:54:20.108793+0000","last_fullsized":"2023-10-09T10:54:20.108793+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:15.848951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":54,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.143801+0000","last_change":"2023-10-09T10:54:02.875620+0000","last_active":"2023-10-09T10:54:21.143801+0000","last_peered":"2023-10-09T10:54:21.143801+0000","last_clean":"2023-10-09T10:54:21.143801+0000","last_became_active":"2023-10-09T10:54:02.873790+0000","last_became_peered":"2023-10-09T10:54:02.873790+0000","last_unstale":"2023-10-09T10:54:21.143801+0000","last_undegraded":"2023-10-09T10:54:21.143801+0000","last_fullsized":"2023-10-09T10:54:21.143801+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:25.125501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.8","version":"285'5438","reported_seq":5468,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:25.139358+0000","last_change":"2023-10-09T10:53:14.843269+0000","last_active":"2023-10-09T10:54:25.139358+0000","last_peered":"2023-10-09T10:54:25.139358+0000","last_clean":"2023-10-09T10:54:25.139358+0000","last_became_active":"2023-10-09T10:53:14.842845+0000","last_became_peered":"2023-10-09T10:53:14.842845+0000","last_unstale":"2023-10-09T10:54:25.139358+0000","last_undegraded":"2023-10-09T10:54:25.139358+0000","last_fullsized":"2023-10-09T10:54:25.139358+0000","mapping_epoch":266,"log_start":"271'500","ondisk_log_start":"271'500","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":4938,"log_dups_size":0,"ondisk_log_size":4938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:29.867741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356384768,"num_objects":5438,"num_object_clones":0,"num_object_copies":16314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5438,"num_write_kb":348032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":622,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.171741+0000","last_change":"2023-10-09T10:54:21.163072+0000","last_active":"2023-10-09T10:54:21.171741+0000","last_peered":"2023-10-09T10:54:21.171741+0000","last_clean":"2023-10-09T10:54:21.171741+0000","last_became_active":"2023-10-09T10:54:21.162710+0000","last_became_peered":"2023-10-09T10:54:21.162710+0000","last_unstale":"2023-10-09T10:54:21.171741+0000","last_undegraded":"2023-10-09T10:54:21.171741+0000","last_fullsized":"2023-10-09T10:54:21.171741+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:15.932806+0000","scrub_duration":0.0019146790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":20,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.141456+0000","last_change":"2023-10-09T10:54:02.873632+0000","last_active":"2023-10-09T10:54:21.141456+0000","last_peered":"2023-10-09T10:54:21.141456+0000","last_clean":"2023-10-09T10:54:21.141456+0000","last_became_active":"2023-10-09T10:54:02.872935+0000","last_became_peered":"2023-10-09T10:54:02.872935+0000","last_unstale":"2023-10-09T10:54:21.141456+0000","last_undegraded":"2023-10-09T10:54:21.141456+0000","last_fullsized":"2023-10-09T10:54:21.141456+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:54.337774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":194,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.140897+0000","last_change":"2023-10-09T10:50:43.388615+0000","last_active":"2023-10-09T10:54:21.140897+0000","last_peered":"2023-10-09T10:54:21.140897+0000","last_clean":"2023-10-09T10:54:21.140897+0000","last_became_active":"2023-10-09T10:50:43.388176+0000","last_became_peered":"2023-10-09T10:50:43.388176+0000","last_unstale":"2023-10-09T10:54:21.140897+0000","last_undegraded":"2023-10-09T10:54:21.140897+0000","last_fullsized":"2023-10-09T10:54:21.140897+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:41:17.550111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.120615+0000","last_change":"2023-10-09T10:54:17.879373+0000","last_active":"2023-10-09T10:54:20.120615+0000","last_peered":"2023-10-09T10:54:20.120615+0000","last_clean":"2023-10-09T10:54:20.120615+0000","last_became_active":"2023-10-09T10:54:17.878866+0000","last_became_peered":"2023-10-09T10:54:17.878866+0000","last_unstale":"2023-10-09T10:54:20.120615+0000","last_undegraded":"2023-10-09T10:54:20.120615+0000","last_fullsized":"2023-10-09T10:54:20.120615+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:05.180566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"285'5513","reported_seq":5562,"reported_epoch":285,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:54:25.090801+0000","last_change":"2023-10-09T10:54:19.229497+0000","last_active":"2023-10-09T10:54:25.090801+0000","last_peered":"2023-10-09T10:54:25.090801+0000","last_clean":"2023-10-09T10:54:02.429830+0000","last_became_active":"2023-10-09T10:54:19.214421+0000","last_became_peered":"2023-10-09T10:54:19.214421+0000","last_unstale":"2023-10-09T10:54:25.090801+0000","last_undegraded":"2023-10-09T10:54:19.095304+0000","last_fullsized":"2023-10-09T10:54:25.090801+0000","mapping_epoch":282,"log_start":"271'600","ondisk_log_start":"271'600","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":4913,"log_dups_size":0,"ondisk_log_size":4913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:13.901103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361299968,"num_objects":5513,"num_object_clones":0,"num_object_copies":16539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4699,"num_objects_misplaced":814,"num_objects_unfound":0,"num_objects_dirty":5513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5513,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,7],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":4698},{"shards":"3,4","objects":1},{"shards":"3,4,7","objects":814}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":687,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:23.302182+0000","last_change":"2023-10-09T10:53:35.501606+0000","last_active":"2023-10-09T10:54:23.302182+0000","last_peered":"2023-10-09T10:54:23.302182+0000","last_clean":"2023-10-09T10:54:23.302182+0000","last_became_active":"2023-10-09T10:53:17.842229+0000","last_became_peered":"2023-10-09T10:53:17.842229+0000","last_unstale":"2023-10-09T10:54:23.302182+0000","last_undegraded":"2023-10-09T10:54:23.302182+0000","last_fullsized":"2023-10-09T10:54:23.302182+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:36.948816+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.120073+0000","last_change":"2023-10-09T10:54:17.885030+0000","last_active":"2023-10-09T10:54:20.120073+0000","last_peered":"2023-10-09T10:54:20.120073+0000","last_clean":"2023-10-09T10:54:20.120073+0000","last_became_active":"2023-10-09T10:54:17.884752+0000","last_became_peered":"2023-10-09T10:54:17.884752+0000","last_unstale":"2023-10-09T10:54:20.120073+0000","last_undegraded":"2023-10-09T10:54:20.120073+0000","last_fullsized":"2023-10-09T10:54:20.120073+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:38.548132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.118828+0000","last_change":"2023-10-09T10:54:17.879189+0000","last_active":"2023-10-09T10:54:20.118828+0000","last_peered":"2023-10-09T10:54:20.118828+0000","last_clean":"2023-10-09T10:54:20.118828+0000","last_became_active":"2023-10-09T10:54:17.878814+0000","last_became_peered":"2023-10-09T10:54:17.878814+0000","last_unstale":"2023-10-09T10:54:20.118828+0000","last_undegraded":"2023-10-09T10:54:20.118828+0000","last_fullsized":"2023-10-09T10:54:20.118828+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:07.846903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":174,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.144031+0000","last_change":"2023-10-09T10:54:02.869829+0000","last_active":"2023-10-09T10:54:21.144031+0000","last_peered":"2023-10-09T10:54:21.144031+0000","last_clean":"2023-10-09T10:54:21.144031+0000","last_became_active":"2023-10-09T10:54:02.869488+0000","last_became_peered":"2023-10-09T10:54:02.869488+0000","last_unstale":"2023-10-09T10:54:21.144031+0000","last_undegraded":"2023-10-09T10:54:21.144031+0000","last_fullsized":"2023-10-09T10:54:21.144031+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:34.722248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.9","version":"286'5579","reported_seq":5609,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T10:54:28.207750+0000","last_change":"2023-10-09T10:53:13.829514+0000","last_active":"2023-10-09T10:54:28.207750+0000","last_peered":"2023-10-09T10:54:28.207750+0000","last_clean":"2023-10-09T10:54:28.207750+0000","last_became_active":"2023-10-09T10:53:13.829236+0000","last_became_peered":"2023-10-09T10:53:13.829236+0000","last_unstale":"2023-10-09T10:54:28.207750+0000","last_undegraded":"2023-10-09T10:54:28.207750+0000","last_fullsized":"2023-10-09T10:54:28.207750+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5579,"log_dups_size":0,"ondisk_log_size":5579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:38.141857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365625344,"num_objects":5579,"num_object_clones":0,"num_object_copies":16737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5579,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":14,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.123431+0000","last_change":"2023-10-09T10:54:17.884876+0000","last_active":"2023-10-09T10:54:20.123431+0000","last_peered":"2023-10-09T10:54:20.123431+0000","last_clean":"2023-10-09T10:54:20.123431+0000","last_became_active":"2023-10-09T10:54:17.884402+0000","last_became_peered":"2023-10-09T10:54:17.884402+0000","last_unstale":"2023-10-09T10:54:20.123431+0000","last_undegraded":"2023-10-09T10:54:20.123431+0000","last_fullsized":"2023-10-09T10:54:20.123431+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:18.679950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":704,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.134336+0000","last_change":"2023-10-09T10:53:17.844346+0000","last_active":"2023-10-09T10:54:21.134336+0000","last_peered":"2023-10-09T10:54:21.134336+0000","last_clean":"2023-10-09T10:54:21.134336+0000","last_became_active":"2023-10-09T10:53:17.844056+0000","last_became_peered":"2023-10-09T10:53:17.844056+0000","last_unstale":"2023-10-09T10:54:21.134336+0000","last_undegraded":"2023-10-09T10:54:21.134336+0000","last_fullsized":"2023-10-09T10:54:21.134336+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:27:59.443732+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":216,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.107405+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:54:20.107405+0000","last_peered":"2023-10-09T10:54:20.107405+0000","last_clean":"2023-10-09T10:54:20.107405+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:54:20.107405+0000","last_undegraded":"2023-10-09T10:54:20.107405+0000","last_fullsized":"2023-10-09T10:54:20.107405+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":198,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.146512+0000","last_change":"2023-10-09T10:53:14.836523+0000","last_active":"2023-10-09T10:54:21.146512+0000","last_peered":"2023-10-09T10:54:21.146512+0000","last_clean":"2023-10-09T10:54:21.146512+0000","last_became_active":"2023-10-09T10:53:14.836134+0000","last_became_peered":"2023-10-09T10:53:14.836134+0000","last_unstale":"2023-10-09T10:54:21.146512+0000","last_undegraded":"2023-10-09T10:54:21.146512+0000","last_fullsized":"2023-10-09T10:54:21.146512+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:05:24.615198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"285'5407","reported_seq":6182,"reported_epoch":285,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:54:25.175181+0000","last_change":"2023-10-09T10:54:19.232675+0000","last_active":"2023-10-09T10:54:25.175181+0000","last_peered":"2023-10-09T10:54:25.175181+0000","last_clean":"2023-10-09T10:54:02.408036+0000","last_became_active":"2023-10-09T10:54:19.222444+0000","last_became_peered":"2023-10-09T10:54:19.222444+0000","last_unstale":"2023-10-09T10:54:25.175181+0000","last_undegraded":"2023-10-09T10:54:19.082530+0000","last_fullsized":"2023-10-09T10:54:25.175181+0000","mapping_epoch":282,"log_start":"271'500","ondisk_log_start":"271'500","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":4907,"log_dups_size":0,"ondisk_log_size":4907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:34.001687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354353152,"num_objects":5407,"num_object_clones":0,"num_object_copies":16221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4258,"num_objects_misplaced":1149,"num_objects_unfound":0,"num_objects_dirty":5407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5407,"num_write_kb":346048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":366,"num_bytes_recovered":23986176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[3,5,4],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,5","objects":4258},{"shards":"3,4,5","objects":1149}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":622,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.143028+0000","last_change":"2023-10-09T10:54:05.916167+0000","last_active":"2023-10-09T10:54:21.143028+0000","last_peered":"2023-10-09T10:54:21.143028+0000","last_clean":"2023-10-09T10:54:21.143028+0000","last_became_active":"2023-10-09T10:54:05.915827+0000","last_became_peered":"2023-10-09T10:54:05.915827+0000","last_unstale":"2023-10-09T10:54:21.143028+0000","last_undegraded":"2023-10-09T10:54:21.143028+0000","last_fullsized":"2023-10-09T10:54:21.143028+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.070026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.070026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":262,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.134405+0000","last_change":"2023-10-09T10:54:02.863813+0000","last_active":"2023-10-09T10:54:21.134405+0000","last_peered":"2023-10-09T10:54:21.134405+0000","last_clean":"2023-10-09T10:54:21.134405+0000","last_became_active":"2023-10-09T10:54:02.863618+0000","last_became_peered":"2023-10-09T10:54:02.863618+0000","last_unstale":"2023-10-09T10:54:21.134405+0000","last_undegraded":"2023-10-09T10:54:21.134405+0000","last_fullsized":"2023-10-09T10:54:21.134405+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:41.499614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":167,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:23.302070+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:54:23.302070+0000","last_peered":"2023-10-09T10:54:23.302070+0000","last_clean":"2023-10-09T10:54:23.302070+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:54:23.302070+0000","last_undegraded":"2023-10-09T10:54:23.302070+0000","last_fullsized":"2023-10-09T10:54:23.302070+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":220,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.159640+0000","last_change":"2023-10-09T10:54:17.880468+0000","last_active":"2023-10-09T10:54:21.159640+0000","last_peered":"2023-10-09T10:54:21.159640+0000","last_clean":"2023-10-09T10:54:21.159640+0000","last_became_active":"2023-10-09T10:54:17.880120+0000","last_became_peered":"2023-10-09T10:54:17.880120+0000","last_unstale":"2023-10-09T10:54:21.159640+0000","last_undegraded":"2023-10-09T10:54:21.159640+0000","last_fullsized":"2023-10-09T10:54:21.159640+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:31:41.489513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"285'5532","reported_seq":5581,"reported_epoch":285,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T10:54:25.091331+0000","last_change":"2023-10-09T10:54:19.429806+0000","last_active":"2023-10-09T10:54:25.091331+0000","last_peered":"2023-10-09T10:54:25.091331+0000","last_clean":"2023-10-09T10:54:02.435746+0000","last_became_active":"2023-10-09T10:54:19.373132+0000","last_became_peered":"2023-10-09T10:54:19.373132+0000","last_unstale":"2023-10-09T10:54:25.091331+0000","last_undegraded":"2023-10-09T10:54:19.096924+0000","last_fullsized":"2023-10-09T10:54:25.091331+0000","mapping_epoch":282,"log_start":"271'600","ondisk_log_start":"271'600","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":4932,"log_dups_size":0,"ondisk_log_size":4932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:46.106518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362545152,"num_objects":5532,"num_object_clones":0,"num_object_copies":16596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4756,"num_objects_misplaced":776,"num_objects_unfound":0,"num_objects_dirty":5532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5532,"num_write_kb":354048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,4,6],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":4755},{"shards":"3,4","objects":1},{"shards":"3,4,6","objects":776}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":657,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.159869+0000","last_change":"2023-10-09T10:53:27.625180+0000","last_active":"2023-10-09T10:54:21.159869+0000","last_peered":"2023-10-09T10:54:21.159869+0000","last_clean":"2023-10-09T10:54:21.159869+0000","last_became_active":"2023-10-09T10:51:12.366828+0000","last_became_peered":"2023-10-09T10:51:12.366828+0000","last_unstale":"2023-10-09T10:54:21.159869+0000","last_undegraded":"2023-10-09T10:54:21.159869+0000","last_fullsized":"2023-10-09T10:54:21.159869+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.625079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.625079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:59:41.253976+0000","scrub_duration":0.0041067969999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":29,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.142730+0000","last_change":"2023-10-09T10:53:14.833073+0000","last_active":"2023-10-09T10:54:21.142730+0000","last_peered":"2023-10-09T10:54:21.142730+0000","last_clean":"2023-10-09T10:54:21.142730+0000","last_became_active":"2023-10-09T10:53:14.832736+0000","last_became_peered":"2023-10-09T10:53:14.832736+0000","last_unstale":"2023-10-09T10:54:21.142730+0000","last_undegraded":"2023-10-09T10:54:21.142730+0000","last_fullsized":"2023-10-09T10:54:21.142730+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:27.550707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.121188+0000","last_change":"2023-10-09T10:54:17.884550+0000","last_active":"2023-10-09T10:54:20.121188+0000","last_peered":"2023-10-09T10:54:20.121188+0000","last_clean":"2023-10-09T10:54:20.121188+0000","last_became_active":"2023-10-09T10:54:17.883702+0000","last_became_peered":"2023-10-09T10:54:17.883702+0000","last_unstale":"2023-10-09T10:54:20.121188+0000","last_undegraded":"2023-10-09T10:54:20.121188+0000","last_fullsized":"2023-10-09T10:54:20.121188+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:16.550811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":194,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:23.302263+0000","last_change":"2023-10-09T10:53:14.828291+0000","last_active":"2023-10-09T10:54:23.302263+0000","last_peered":"2023-10-09T10:54:23.302263+0000","last_clean":"2023-10-09T10:54:23.302263+0000","last_became_active":"2023-10-09T10:53:14.827991+0000","last_became_peered":"2023-10-09T10:53:14.827991+0000","last_unstale":"2023-10-09T10:54:23.302263+0000","last_undegraded":"2023-10-09T10:54:23.302263+0000","last_fullsized":"2023-10-09T10:54:23.302263+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:08:14.283112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"286'5623","reported_seq":5654,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T10:54:28.291292+0000","last_change":"2023-10-09T10:53:14.822279+0000","last_active":"2023-10-09T10:54:28.291292+0000","last_peered":"2023-10-09T10:54:28.291292+0000","last_clean":"2023-10-09T10:54:28.291292+0000","last_became_active":"2023-10-09T10:53:14.821952+0000","last_became_peered":"2023-10-09T10:53:14.821952+0000","last_unstale":"2023-10-09T10:54:28.291292+0000","last_undegraded":"2023-10-09T10:54:28.291292+0000","last_fullsized":"2023-10-09T10:54:28.291292+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5623,"log_dups_size":0,"ondisk_log_size":5623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:37:14.124652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368508928,"num_objects":5623,"num_object_clones":0,"num_object_copies":16869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5623,"num_write_kb":359872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":681,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.168132+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T10:54:21.168132+0000","last_peered":"2023-10-09T10:54:21.168132+0000","last_clean":"2023-10-09T10:54:21.168132+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T10:54:21.168132+0000","last_undegraded":"2023-10-09T10:54:21.168132+0000","last_fullsized":"2023-10-09T10:54:21.168132+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":307,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.146467+0000","last_change":"2023-10-09T10:53:14.841466+0000","last_active":"2023-10-09T10:54:21.146467+0000","last_peered":"2023-10-09T10:54:21.146467+0000","last_clean":"2023-10-09T10:54:21.146467+0000","last_became_active":"2023-10-09T10:53:14.831674+0000","last_became_peered":"2023-10-09T10:53:14.831674+0000","last_unstale":"2023-10-09T10:54:21.146467+0000","last_undegraded":"2023-10-09T10:54:21.146467+0000","last_fullsized":"2023-10-09T10:54:21.146467+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:02.612069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":76,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.146031+0000","last_change":"2023-10-09T10:54:02.868744+0000","last_active":"2023-10-09T10:54:21.146031+0000","last_peered":"2023-10-09T10:54:21.146031+0000","last_clean":"2023-10-09T10:54:21.146031+0000","last_became_active":"2023-10-09T10:54:02.866451+0000","last_became_peered":"2023-10-09T10:54:02.866451+0000","last_unstale":"2023-10-09T10:54:21.146031+0000","last_undegraded":"2023-10-09T10:54:21.146031+0000","last_fullsized":"2023-10-09T10:54:21.146031+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:51.707788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":306,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.143059+0000","last_change":"2023-10-09T10:53:14.835841+0000","last_active":"2023-10-09T10:54:20.143059+0000","last_peered":"2023-10-09T10:54:20.143059+0000","last_clean":"2023-10-09T10:54:20.143059+0000","last_became_active":"2023-10-09T10:53:14.835551+0000","last_became_peered":"2023-10-09T10:53:14.835551+0000","last_unstale":"2023-10-09T10:54:20.143059+0000","last_undegraded":"2023-10-09T10:54:20.143059+0000","last_fullsized":"2023-10-09T10:54:20.143059+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:21.498099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"286'5425","reported_seq":5460,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:54:26.344011+0000","last_change":"2023-10-09T10:54:19.338855+0000","last_active":"2023-10-09T10:54:26.344011+0000","last_peered":"2023-10-09T10:54:26.344011+0000","last_clean":"2023-10-09T10:54:17.082254+0000","last_became_active":"2023-10-09T10:54:19.319875+0000","last_became_peered":"2023-10-09T10:54:19.319875+0000","last_unstale":"2023-10-09T10:54:26.344011+0000","last_undegraded":"2023-10-09T10:54:18.902108+0000","last_fullsized":"2023-10-09T10:54:18.874134+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5425,"log_dups_size":0,"ondisk_log_size":5425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:54.083267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355532800,"num_objects":5425,"num_object_clones":0,"num_object_copies":16275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5145,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5425,"num_write_kb":347200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[5,7],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6,7","objects":5145},{"shards":"5,7","objects":280}],"blocked_by":[],"up_primary":2,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":675,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.107753+0000","last_change":"2023-10-09T10:53:30.567189+0000","last_active":"2023-10-09T10:54:20.107753+0000","last_peered":"2023-10-09T10:54:20.107753+0000","last_clean":"2023-10-09T10:54:20.107753+0000","last_became_active":"2023-10-09T10:53:17.840928+0000","last_became_peered":"2023-10-09T10:53:17.840928+0000","last_unstale":"2023-10-09T10:54:20.107753+0000","last_undegraded":"2023-10-09T10:54:20.107753+0000","last_fullsized":"2023-10-09T10:54:20.107753+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:52.176229+0000","scrub_duration":0.0053707060000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":14,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.121806+0000","last_change":"2023-10-09T10:54:17.879705+0000","last_active":"2023-10-09T10:54:20.121806+0000","last_peered":"2023-10-09T10:54:20.121806+0000","last_clean":"2023-10-09T10:54:20.121806+0000","last_became_active":"2023-10-09T10:54:17.879390+0000","last_became_peered":"2023-10-09T10:54:17.879390+0000","last_unstale":"2023-10-09T10:54:20.121806+0000","last_undegraded":"2023-10-09T10:54:20.121806+0000","last_fullsized":"2023-10-09T10:54:20.121806+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:24.062107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":200,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.159724+0000","last_change":"2023-10-09T10:54:02.880580+0000","last_active":"2023-10-09T10:54:21.159724+0000","last_peered":"2023-10-09T10:54:21.159724+0000","last_clean":"2023-10-09T10:54:21.159724+0000","last_became_active":"2023-10-09T10:54:02.867519+0000","last_became_peered":"2023-10-09T10:54:02.867519+0000","last_unstale":"2023-10-09T10:54:21.159724+0000","last_undegraded":"2023-10-09T10:54:21.159724+0000","last_fullsized":"2023-10-09T10:54:21.159724+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:49:35.915644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":227,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.142504+0000","last_change":"2023-10-09T10:51:50.371282+0000","last_active":"2023-10-09T10:54:21.142504+0000","last_peered":"2023-10-09T10:54:21.142504+0000","last_clean":"2023-10-09T10:54:21.142504+0000","last_became_active":"2023-10-09T10:51:50.370973+0000","last_became_peered":"2023-10-09T10:51:50.370973+0000","last_unstale":"2023-10-09T10:54:21.142504+0000","last_undegraded":"2023-10-09T10:54:21.142504+0000","last_fullsized":"2023-10-09T10:54:21.142504+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:15.603164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":675,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.111890+0000","last_change":"2023-10-09T10:53:29.568199+0000","last_active":"2023-10-09T10:54:20.111890+0000","last_peered":"2023-10-09T10:54:20.111890+0000","last_clean":"2023-10-09T10:54:20.111890+0000","last_became_active":"2023-10-09T10:52:55.416357+0000","last_became_peered":"2023-10-09T10:52:55.416357+0000","last_unstale":"2023-10-09T10:54:20.111890+0000","last_undegraded":"2023-10-09T10:54:20.111890+0000","last_fullsized":"2023-10-09T10:54:20.111890+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:44.364650+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":193,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.141043+0000","last_change":"2023-10-09T10:53:14.823731+0000","last_active":"2023-10-09T10:54:21.141043+0000","last_peered":"2023-10-09T10:54:21.141043+0000","last_clean":"2023-10-09T10:54:21.141043+0000","last_became_active":"2023-10-09T10:53:14.823470+0000","last_became_peered":"2023-10-09T10:53:14.823470+0000","last_unstale":"2023-10-09T10:54:21.141043+0000","last_undegraded":"2023-10-09T10:54:21.141043+0000","last_fullsized":"2023-10-09T10:54:21.141043+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:36.927599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.121258+0000","last_change":"2023-10-09T10:54:17.876079+0000","last_active":"2023-10-09T10:54:20.121258+0000","last_peered":"2023-10-09T10:54:20.121258+0000","last_clean":"2023-10-09T10:54:20.121258+0000","last_became_active":"2023-10-09T10:54:17.875802+0000","last_became_peered":"2023-10-09T10:54:17.875802+0000","last_unstale":"2023-10-09T10:54:20.121258+0000","last_undegraded":"2023-10-09T10:54:20.121258+0000","last_fullsized":"2023-10-09T10:54:20.121258+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:36.106833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":606,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.108935+0000","last_change":"2023-10-09T10:53:36.563918+0000","last_active":"2023-10-09T10:54:20.108935+0000","last_peered":"2023-10-09T10:54:20.108935+0000","last_clean":"2023-10-09T10:54:20.108935+0000","last_became_active":"2023-10-09T10:53:17.842952+0000","last_became_peered":"2023-10-09T10:53:17.842952+0000","last_unstale":"2023-10-09T10:54:20.108935+0000","last_undegraded":"2023-10-09T10:54:20.108935+0000","last_fullsized":"2023-10-09T10:54:20.108935+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:53.185917+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":98,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.143484+0000","last_change":"2023-10-09T10:54:02.882437+0000","last_active":"2023-10-09T10:54:21.143484+0000","last_peered":"2023-10-09T10:54:21.143484+0000","last_clean":"2023-10-09T10:54:21.143484+0000","last_became_active":"2023-10-09T10:54:02.873916+0000","last_became_peered":"2023-10-09T10:54:02.873916+0000","last_unstale":"2023-10-09T10:54:21.143484+0000","last_undegraded":"2023-10-09T10:54:21.143484+0000","last_fullsized":"2023-10-09T10:54:21.143484+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:23:13.358333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":292,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.137773+0000","last_change":"2023-10-09T10:54:17.878148+0000","last_active":"2023-10-09T10:54:20.137773+0000","last_peered":"2023-10-09T10:54:20.137773+0000","last_clean":"2023-10-09T10:54:20.137773+0000","last_became_active":"2023-10-09T10:54:17.877903+0000","last_became_peered":"2023-10-09T10:54:17.877903+0000","last_unstale":"2023-10-09T10:54:20.137773+0000","last_undegraded":"2023-10-09T10:54:20.137773+0000","last_fullsized":"2023-10-09T10:54:20.137773+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:19.072462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":236,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.139923+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:54:21.139923+0000","last_peered":"2023-10-09T10:54:21.139923+0000","last_clean":"2023-10-09T10:54:21.139923+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:54:21.139923+0000","last_undegraded":"2023-10-09T10:54:21.139923+0000","last_fullsized":"2023-10-09T10:54:21.139923+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":228,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.112723+0000","last_change":"2023-10-09T10:50:43.385150+0000","last_active":"2023-10-09T10:54:20.112723+0000","last_peered":"2023-10-09T10:54:20.112723+0000","last_clean":"2023-10-09T10:54:20.112723+0000","last_became_active":"2023-10-09T10:50:43.384788+0000","last_became_peered":"2023-10-09T10:50:43.384788+0000","last_unstale":"2023-10-09T10:54:20.112723+0000","last_undegraded":"2023-10-09T10:54:20.112723+0000","last_fullsized":"2023-10-09T10:54:20.112723+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:36.085910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":587,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T10:54:25.498908+0000","last_change":"2023-10-09T10:54:25.498908+0000","last_active":"2023-10-09T10:54:25.498908+0000","last_peered":"2023-10-09T10:54:25.498908+0000","last_clean":"2023-10-09T10:54:25.498908+0000","last_became_active":"2023-10-09T10:54:05.916015+0000","last_became_peered":"2023-10-09T10:54:05.916015+0000","last_unstale":"2023-10-09T10:54:25.498908+0000","last_undegraded":"2023-10-09T10:54:25.498908+0000","last_fullsized":"2023-10-09T10:54:25.498908+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:46.305288+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:46.305288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":308,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.109253+0000","last_change":"2023-10-09T10:54:17.886158+0000","last_active":"2023-10-09T10:54:20.109253+0000","last_peered":"2023-10-09T10:54:20.109253+0000","last_clean":"2023-10-09T10:54:20.109253+0000","last_became_active":"2023-10-09T10:54:17.885816+0000","last_became_peered":"2023-10-09T10:54:17.885816+0000","last_unstale":"2023-10-09T10:54:20.109253+0000","last_undegraded":"2023-10-09T10:54:20.109253+0000","last_fullsized":"2023-10-09T10:54:20.109253+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:05.270825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":311,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T10:54:20.112753+0000","last_change":"2023-10-09T10:54:17.957784+0000","last_active":"2023-10-09T10:54:20.112753+0000","last_peered":"2023-10-09T10:54:20.112753+0000","last_clean":"2023-10-09T10:54:20.112753+0000","last_became_active":"2023-10-09T10:54:17.954788+0000","last_became_peered":"2023-10-09T10:54:17.954788+0000","last_unstale":"2023-10-09T10:54:20.112753+0000","last_undegraded":"2023-10-09T10:54:20.112753+0000","last_fullsized":"2023-10-09T10:54:20.112753+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:48.709390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":575,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.159922+0000","last_change":"2023-10-09T10:54:21.159922+0000","last_active":"2023-10-09T10:54:21.159922+0000","last_peered":"2023-10-09T10:54:21.159922+0000","last_clean":"2023-10-09T10:54:21.159922+0000","last_became_active":"2023-10-09T10:54:21.159581+0000","last_became_peered":"2023-10-09T10:54:21.159581+0000","last_unstale":"2023-10-09T10:54:21.159922+0000","last_undegraded":"2023-10-09T10:54:21.159922+0000","last_fullsized":"2023-10-09T10:54:21.159922+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.070492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.070492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":194,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:23.302400+0000","last_change":"2023-10-09T10:54:18.089053+0000","last_active":"2023-10-09T10:54:23.302400+0000","last_peered":"2023-10-09T10:54:23.302400+0000","last_clean":"2023-10-09T10:54:23.302400+0000","last_became_active":"2023-10-09T10:54:17.886070+0000","last_became_peered":"2023-10-09T10:54:17.886070+0000","last_unstale":"2023-10-09T10:54:23.302400+0000","last_undegraded":"2023-10-09T10:54:23.302400+0000","last_fullsized":"2023-10-09T10:54:23.302400+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:35.628407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":280,"reported_epoch":285,"state":"active+clean","last_fresh":"2023-10-09T10:54:21.143124+0000","last_change":"2023-10-09T10:54:02.878622+0000","last_active":"2023-10-09T10:54:21.143124+0000","last_peered":"2023-10-09T10:54:21.143124+0000","last_clean":"2023-10-09T10:54:21.143124+0000","last_became_active":"2023-10-09T10:54:02.877012+0000","last_became_peered":"2023-10-09T10:54:02.877012+0000","last_unstale":"2023-10-09T10:54:21.143124+0000","last_undegraded":"2023-10-09T10:54:21.143124+0000","last_fullsized":"2023-10-09T10:54:21.143124+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:41:10.034332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":6758,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5781192704,"num_objects":88214,"num_object_clones":0,"num_object_copies":264642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33957,"num_objects_misplaced":18674,"num_objects_unfound":0,"num_objects_dirty":88214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88214,"num_write_kb":5645696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":670,"num_bytes_recovered":43909120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17407934464,"data_stored":17407934464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84314,"ondisk_log_size":84314,"up":48,"acting":44,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":675346,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":71,"num_read_kb":132,"num_write":35,"num_write_kb":348,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":681461,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332334,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2443588,"kb_used_data":2142204,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91301564,"statfs":{"total":95995035648,"available":93492801536,"internally_reserved":0,"allocated":2193616896,"data_stored":2192586806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:53:52 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.6060000000000001,"15min":1.1080000000000001},"min":{"1min":0.311,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":4.3049999999999997,"5min":9.6379999999999999,"15min":9.6379999999999999},"last":6.2859999999999996},{"interface":"front","average":{"1min":1.7689999999999999,"5min":1.851,"15min":1.3240000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.24299999999999999},"max":{"1min":8.3859999999999992,"5min":16.731000000000002,"15min":16.731000000000002},"last":7.0990000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.98,"5min":2.0270000000000001,"15min":1.492},"min":{"1min":0.32100000000000001,"5min":0.27000000000000002,"15min":0.25700000000000001},"max":{"1min":8.4109999999999996,"5min":34.756,"15min":34.756},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.097,"5min":1.7110000000000001,"15min":1.242},"min":{"1min":0.28999999999999998,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":4.1719999999999997,"5min":35.658999999999999,"15min":35.658999999999999},"last":2.0030000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:53:26 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":1.629,"15min":1.2070000000000001},"min":{"1min":0.33800000000000002,"5min":0.30399999999999999,"15min":0.26400000000000001},"max":{"1min":0.96299999999999997,"5min":11.058,"15min":11.058},"last":2.1349999999999998},{"interface":"front","average":{"1min":0.84499999999999997,"5min":1.4570000000000001,"15min":1.0580000000000001},"min":{"1min":0.38200000000000001,"5min":0.28699999999999998,"15min":0.251},"max":{"1min":3.4990000000000001,"5min":7.2949999999999999,"15min":7.8120000000000003},"last":0.41599999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:53:52 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":1.9770000000000001,"15min":1.4850000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":8.3350000000000009,"5min":8.8339999999999996,"15min":11.523},"last":10.069000000000001},{"interface":"front","average":{"1min":0.89300000000000002,"5min":1.556,"15min":1.3300000000000001},"min":{"1min":0.39500000000000002,"5min":0.34399999999999997,"15min":0.32800000000000001},"max":{"1min":3.1819999999999999,"5min":9.9329999999999998,"15min":19.234999999999999},"last":2.1989999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:53:33 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.798,"15min":1.147},"min":{"1min":0.29699999999999999,"5min":0.27700000000000002,"15min":0.255},"max":{"1min":4.8440000000000003,"5min":16.695,"15min":16.695},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.627,"15min":1.0469999999999999},"min":{"1min":0.40000000000000002,"5min":0.28999999999999998,"15min":0.26200000000000001},"max":{"1min":1.6819999999999999,"5min":11.109999999999999,"15min":11.109999999999999},"last":0.376}]},{"osd":5,"last update":"Mon Oct 9 10:53:33 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":1.734,"15min":1.105},"min":{"1min":0.33600000000000002,"5min":0.29699999999999999,"15min":0.247},"max":{"1min":2.3069999999999999,"5min":13.262,"15min":13.262},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.83099999999999996,"5min":1.7869999999999999,"15min":1.272},"min":{"1min":0.39500000000000002,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":4.7930000000000001,"5min":11.494,"15min":11.741},"last":1.159}]},{"osd":6,"last update":"Mon Oct 9 10:53:33 2023","interfaces":[{"interface":"back","average":{"1min":1.256,"5min":2.1640000000000001,"15min":1.3720000000000001},"min":{"1min":0.39700000000000002,"5min":0.317,"15min":0.28199999999999997},"max":{"1min":7.3499999999999996,"5min":14.913,"15min":14.913},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.79100000000000004,"5min":2.0019999999999998,"15min":1.218},"min":{"1min":0.33300000000000002,"5min":0.313,"15min":0.29399999999999998},"max":{"1min":2.9159999999999999,"5min":12.268000000000001,"15min":19.187999999999999},"last":2.1699999999999999}]}]},{"osd":6,"up_from":44,"seq":188978561265,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3074868,"kb_used_data":2481004,"kb_used_omap":0,"kb_used_meta":593856,"kb_avail":90670284,"statfs":{"total":95995035648,"available":92846370816,"internally_reserved":0,"allocated":2540548096,"data_stored":2539533700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608108544},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:53:54 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":1.3180000000000001,"15min":1.1160000000000001},"min":{"1min":0.24399999999999999,"5min":0.216,"15min":0.216},"max":{"1min":2.4350000000000001,"5min":12.288,"15min":12.288},"last":0.252},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.609,"15min":1.196},"min":{"1min":0.29799999999999999,"5min":0.25,"15min":0.22800000000000001},"max":{"1min":3.5110000000000001,"5min":12.369999999999999,"15min":12.369999999999999},"last":0.40500000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:54:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.3660000000000001,"15min":1.2130000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.0230000000000001,"5min":8.4730000000000008,"15min":16.747},"last":3.7290000000000001},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.4159999999999999,"15min":1.252},"min":{"1min":0.26400000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.7869999999999999,"5min":8.0579999999999998,"15min":16.734000000000002},"last":3.7589999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:53:32 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":1.706,"15min":1.2170000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":3.202,"5min":8.0060000000000002,"15min":13.208},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.1910000000000001,"5min":1.9139999999999999,"15min":1.365},"min":{"1min":0.35599999999999998,"5min":0.29999999999999999,"15min":0.28399999999999997},"max":{"1min":3.1709999999999998,"5min":10.186,"15min":12.571},"last":1.5429999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:53:54 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.833,"15min":1.2889999999999999},"min":{"1min":0.40600000000000003,"5min":0.27900000000000003,"15min":0.255},"max":{"1min":4.8760000000000003,"5min":12.391,"15min":12.391},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.3049999999999999,"5min":2.1259999999999999,"15min":1.466},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":3.5990000000000002,"5min":7.327,"15min":13.179},"last":3.8130000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:54:26 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.5620000000000001,"15min":1.1060000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.23200000000000001},"max":{"1min":5.968,"5min":9.0600000000000005,"15min":9.4060000000000006},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.54,"5min":1.47,"15min":1.113},"min":{"1min":0.308,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":4.7930000000000001,"5min":12.061999999999999,"15min":12.481999999999999},"last":0.46100000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:54:26 2023","interfaces":[{"interface":"back","average":{"1min":1.8979999999999999,"5min":1.802,"15min":1.244},"min":{"1min":0.30299999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.5589999999999993,"5min":9.3290000000000006,"15min":10.497},"last":0.375},{"interface":"front","average":{"1min":1.5920000000000001,"5min":1.599,"15min":1.1759999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.27600000000000002},"max":{"1min":6.7519999999999998,"5min":12.103999999999999,"15min":12.456},"last":0.495}]},{"osd":7,"last update":"Mon Oct 9 10:53:39 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":1.724,"15min":1.208},"min":{"1min":0.34200000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.0710000000000002,"5min":12.007,"15min":23.358000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.81399999999999995,"5min":1.3049999999999999,"15min":1.044},"min":{"1min":0.435,"5min":0.36299999999999999,"15min":0.34100000000000003},"max":{"1min":1.6919999999999999,"5min":12.422000000000001,"15min":12.422000000000001},"last":0.56100000000000005}]}]},{"osd":1,"up_from":0,"seq":365072220304,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":80,"seq":343597383838,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2570888,"kb_used_data":2146240,"kb_used_omap":28,"kb_used_meta":424611,"kb_avail":91174264,"statfs":{"total":95995035648,"available":93362446336,"internally_reserved":0,"allocated":2197749760,"data_stored":2196189311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":434802512},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.538,"15min":1.3879999999999999},"min":{"1min":0.20399999999999999,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":3.9849999999999999,"5min":14.311,"15min":25.475000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.85799999999999998,"5min":1.5609999999999999,"15min":1.3959999999999999},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":3.7269999999999999,"5min":9.359,"15min":20.832999999999998},"last":0.26000000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:53:31 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":2.0430000000000001,"15min":1.2889999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.19700000000000001},"max":{"1min":6.6630000000000003,"5min":17.927,"15min":17.927},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.823,"5min":2.0880000000000001,"15min":1.4399999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.20399999999999999},"max":{"1min":5.7560000000000002,"5min":14.111000000000001,"15min":14.391999999999999},"last":0.52700000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.9970000000000001,"15min":1.661},"min":{"1min":0.23599999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.698,"5min":13.166,"15min":14.926},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.1919999999999999,"5min":2.0129999999999999,"15min":1.5840000000000001},"min":{"1min":0.25800000000000001,"5min":0.23499999999999999,"15min":0.224},"max":{"1min":3.8820000000000001,"5min":14.391999999999999,"15min":14.391999999999999},"last":2.6309999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":1.7969999999999999,"15min":1.3620000000000001},"min":{"1min":0.33800000000000002,"5min":0.318,"15min":0.26200000000000001},"max":{"1min":3.9860000000000002,"5min":9.6129999999999995,"15min":9.6129999999999995},"last":2.1909999999999998},{"interface":"front","average":{"1min":1.0700000000000001,"5min":2.117,"15min":1.5569999999999999},"min":{"1min":0.28899999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.661,"5min":9.3119999999999994,"15min":12.318},"last":2.3999999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":2.173,"15min":1.544},"min":{"1min":0.41099999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.5910000000000002,"5min":9.3360000000000003,"15min":12.157},"last":4.4189999999999996},{"interface":"front","average":{"1min":0.93700000000000006,"5min":2.012,"15min":1.444},"min":{"1min":0.38600000000000001,"5min":0.317,"15min":0.29499999999999998},"max":{"1min":3.8029999999999999,"5min":14.446,"15min":14.446},"last":2.6600000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":2.2490000000000001,"15min":1.6120000000000001},"min":{"1min":0.34200000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.6339999999999999,"5min":9.2850000000000001,"15min":12.207000000000001},"last":6.2039999999999997},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.849,"15min":1.2909999999999999},"min":{"1min":0.41299999999999998,"5min":0.307,"15min":0.307},"max":{"1min":3.911,"5min":14.420999999999999,"15min":14.420999999999999},"last":2.7080000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:53:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":2.1789999999999998,"15min":1.526},"min":{"1min":0.45500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.7719999999999998,"5min":14.48,"15min":14.48},"last":2.6869999999999998},{"interface":"front","average":{"1min":0.89000000000000001,"5min":1.9790000000000001,"15min":1.5049999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32200000000000001},"max":{"1min":3.9409999999999998,"5min":9.6440000000000001,"15min":14.394},"last":3.1890000000000001}]}]},{"osd":2,"up_from":90,"seq":386547056778,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":263808,"kb_used_data":57400,"kb_used_omap":81,"kb_used_meta":206318,"kb_avail":93481344,"statfs":{"total":95995035648,"available":95724896256,"internally_reserved":0,"allocated":58777600,"data_stored":57394172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":211269850},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:53:29 2023","interfaces":[{"interface":"back","average":{"1min":1.494,"5min":1.7090000000000001,"15min":1.212},"min":{"1min":0.247,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":14.843,"5min":18.512,"15min":18.512},"last":3.7970000000000002},{"interface":"front","average":{"1min":1.7989999999999999,"5min":1.891,"15min":1.3400000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.19900000000000001},"max":{"1min":15.595000000000001,"5min":15.952999999999999,"15min":17.975000000000001},"last":3.782}]},{"osd":1,"last update":"Mon Oct 9 10:53:29 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.708,"15min":1.3129999999999999},"min":{"1min":0.313,"5min":0.23200000000000001,"15min":0.20200000000000001},"max":{"1min":4.9829999999999997,"5min":11.504,"15min":15.247999999999999},"last":3.8220000000000001},{"interface":"front","average":{"1min":0.77000000000000002,"5min":1.367,"15min":1.0960000000000001},"min":{"1min":0.29599999999999999,"5min":0.253,"15min":0.20699999999999999},"max":{"1min":3.3519999999999999,"5min":16.454999999999998,"15min":16.454999999999998},"last":1.077}]},{"osd":3,"last update":"Mon Oct 9 10:53:55 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":1.393,"15min":1.2110000000000001},"min":{"1min":0.27800000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":2.0030000000000001,"5min":8.1639999999999997,"15min":14.113},"last":4.3449999999999998},{"interface":"front","average":{"1min":1.216,"5min":1.8320000000000001,"15min":1.5429999999999999},"min":{"1min":0.30399999999999999,"5min":0.25,"15min":0.20200000000000001},"max":{"1min":7.0309999999999997,"5min":15.561,"15min":15.561},"last":0.39100000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:53:29 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.7270000000000001,"15min":1.387},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.254},"max":{"1min":3.899,"5min":16.231999999999999,"15min":16.231999999999999},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.262,"5min":1.6890000000000001,"15min":1.369},"min":{"1min":0.31,"5min":0.29199999999999998,"15min":0.26300000000000001},"max":{"1min":3.8809999999999998,"5min":9.7460000000000004,"15min":15.51},"last":3.879}]},{"osd":5,"last update":"Mon Oct 9 10:53:29 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":1.639,"15min":1.3680000000000001},"min":{"1min":0.34399999999999997,"5min":0.31900000000000001,"15min":0.245},"max":{"1min":2.9369999999999998,"5min":8.2270000000000003,"15min":17.920000000000002},"last":3.8620000000000001},{"interface":"front","average":{"1min":1.3100000000000001,"5min":1.5940000000000001,"15min":1.3180000000000001},"min":{"1min":0.40400000000000003,"5min":0.35799999999999998,"15min":0.30599999999999999},"max":{"1min":5.2050000000000001,"5min":16.187999999999999,"15min":17.882999999999999},"last":2.742}]},{"osd":6,"last update":"Mon Oct 9 10:53:29 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.7230000000000001,"15min":1.2250000000000001},"min":{"1min":0.34699999999999998,"5min":0.30499999999999999,"15min":0.247},"max":{"1min":4.7990000000000004,"5min":10.443,"15min":14.029999999999999},"last":3.9140000000000001},{"interface":"front","average":{"1min":1.099,"5min":1.671,"15min":1.349},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.29199999999999998},"max":{"1min":4.3470000000000004,"5min":8.8620000000000001,"15min":12.862},"last":0.44600000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:53:29 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":1.4690000000000001,"15min":1.232},"min":{"1min":0.371,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":3.2730000000000001,"5min":7.0609999999999999,"15min":13.999000000000001},"last":3.9399999999999999},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.655,"15min":1.3819999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.307},"max":{"1min":2.919,"5min":8.8970000000000002,"15min":12.826000000000001},"last":3.754}]}]},{"osd":3,"up_from":95,"seq":408021893252,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3162104,"kb_used_data":2515184,"kb_used_omap":7,"kb_used_meta":646904,"kb_avail":90583048,"statfs":{"total":95995035648,"available":92757041152,"internally_reserved":0,"allocated":2575548416,"data_stored":2573995467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":662430312},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":1.762,"15min":1.3320000000000001},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":9.6099999999999994,"5min":10.388999999999999,"15min":15.255000000000001},"last":2.3439999999999999},{"interface":"front","average":{"1min":2.496,"5min":2.855,"15min":1.6379999999999999},"min":{"1min":0.26900000000000002,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":17.312000000000001,"5min":17.312000000000001,"15min":17.312000000000001},"last":3.0249999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":2.278,"15min":1.655},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.19900000000000001},"max":{"1min":4.5510000000000002,"5min":11.757,"15min":14.707000000000001},"last":2.0800000000000001},{"interface":"front","average":{"1min":1.7290000000000001,"5min":2.3809999999999998,"15min":1.522},"min":{"1min":0.26300000000000001,"5min":0.249,"15min":0.193},"max":{"1min":9.5790000000000006,"5min":11.741,"15min":15.443},"last":2.3730000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.9890000000000001,"15min":1.4730000000000001},"min":{"1min":0.32600000000000001,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":4.5910000000000002,"5min":11.795,"15min":13.853},"last":0.42599999999999999},{"interface":"front","average":{"1min":2.5,"5min":2.0219999999999998,"15min":1.55},"min":{"1min":0.315,"5min":0.248,"15min":0.248},"max":{"1min":17.268000000000001,"5min":17.268000000000001,"15min":17.268000000000001},"last":2.319}]},{"osd":4,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.8140000000000001,"15min":1.4359999999999999},"min":{"1min":0.36199999999999999,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":9.2270000000000003,"5min":9.2270000000000003,"15min":20.388000000000002},"last":7.3959999999999999},{"interface":"front","average":{"1min":1.9419999999999999,"5min":1.728,"15min":1.4650000000000001},"min":{"1min":0.41399999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.680999999999999,"5min":11.680999999999999,"15min":19.085999999999999},"last":2.266}]},{"osd":5,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":2.1120000000000001,"15min":1.712},"min":{"1min":0.438,"5min":0.29399999999999998,"15min":0.29199999999999998},"max":{"1min":11.755000000000001,"5min":11.755000000000001,"15min":51.509999999999998},"last":2.2959999999999998},{"interface":"front","average":{"1min":1.5669999999999999,"5min":2.1030000000000002,"15min":1.7949999999999999},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":5.3289999999999997,"5min":11.082000000000001,"15min":55.338999999999999},"last":4.1310000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":2.0550000000000002,"5min":1.905,"15min":1.444},"min":{"1min":0.38400000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.723000000000001,"5min":11.723000000000001,"15min":12.361000000000001},"last":2.3639999999999999},{"interface":"front","average":{"1min":1.415,"5min":2.1539999999999999,"15min":1.657},"min":{"1min":0.35999999999999999,"5min":0.35399999999999998,"15min":0.33800000000000002},"max":{"1min":4.8120000000000003,"5min":11.24,"15min":13.891999999999999},"last":2.0059999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7130000000000001,"5min":2.0720000000000001,"15min":1.8440000000000001},"min":{"1min":0.38900000000000001,"5min":0.31,"15min":0.31},"max":{"1min":5.2229999999999999,"5min":11.819000000000001,"15min":51.744},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.542,"5min":2.1320000000000001,"15min":1.748},"min":{"1min":0.39400000000000002,"5min":0.39200000000000002,"15min":0.31900000000000001},"max":{"1min":9.2799999999999994,"5min":11.379,"15min":53.399000000000001},"last":7.3369999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019128,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2616064,"kb_used_data":2318776,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91129088,"statfs":{"total":95995035648,"available":93316186112,"internally_reserved":0,"allocated":2374426624,"data_stored":2373400339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:53:47 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":1.2869999999999999,"15min":1.048},"min":{"1min":0.27600000000000002,"5min":0.217,"15min":0.20999999999999999},"max":{"1min":1.861,"5min":9.4329999999999998,"15min":10.387},"last":213.88999999999999},{"interface":"front","average":{"1min":0.68000000000000005,"5min":1.8959999999999999,"15min":1.3799999999999999},"min":{"1min":0.26000000000000001,"5min":0.217,"15min":0.217},"max":{"1min":2.5310000000000001,"5min":12.58,"15min":12.58},"last":214.09800000000001}]},{"osd":1,"last update":"Mon Oct 9 10:53:52 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.383,"15min":1.149},"min":{"1min":0.24399999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.897,"5min":14.099,"15min":14.099},"last":206.50399999999999},{"interface":"front","average":{"1min":0.70299999999999996,"5min":1.788,"15min":1.3700000000000001},"min":{"1min":0.37,"5min":0.31,"15min":0.28000000000000003},"max":{"1min":2.0680000000000001,"5min":14.089,"15min":14.089},"last":206.529}]},{"osd":2,"last update":"Mon Oct 9 10:53:27 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.7030000000000001,"15min":1.2969999999999999},"min":{"1min":0.33100000000000002,"5min":0.27500000000000002,"15min":0.253},"max":{"1min":3.8700000000000001,"5min":12.204000000000001,"15min":12.204000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.091,"5min":1.76,"15min":1.3919999999999999},"min":{"1min":0.39400000000000002,"5min":0.29599999999999999,"15min":0.26300000000000001},"max":{"1min":3.927,"5min":9.077,"15min":20.545999999999999},"last":0.44700000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:53:47 2023","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":1.673,"15min":1.409},"min":{"1min":0.35599999999999998,"5min":0.29299999999999998,"15min":0.27400000000000002},"max":{"1min":2.5129999999999999,"5min":10.266999999999999,"15min":10.266999999999999},"last":209.03700000000001},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.887,"15min":1.504},"min":{"1min":0.35799999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.468,"5min":9.6929999999999996,"15min":9.6929999999999996},"last":208.80199999999999}]},{"osd":5,"last update":"Mon Oct 9 10:53:52 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.8560000000000001,"15min":1.381},"min":{"1min":0.28199999999999997,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":5.274,"5min":10.529999999999999,"15min":24.149999999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.77200000000000002,"5min":1.5189999999999999,"15min":1.224},"min":{"1min":0.29899999999999999,"5min":0.27100000000000002,"15min":0.252},"max":{"1min":2.8029999999999999,"5min":15.071999999999999,"15min":20.422000000000001},"last":0.312}]},{"osd":6,"last update":"Mon Oct 9 10:54:25 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":1.617,"15min":1.2230000000000001},"min":{"1min":0.29999999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":15.739000000000001,"5min":15.739000000000001,"15min":15.739000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":2.3650000000000002,"5min":2.2549999999999999,"15min":1.4450000000000001},"min":{"1min":0.371,"5min":0.35699999999999998,"15min":0.311},"max":{"1min":14.093999999999999,"5min":14.272,"15min":14.272},"last":0.371}]},{"osd":7,"last update":"Mon Oct 9 10:53:27 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.7869999999999999,"15min":1.218},"min":{"1min":0.375,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":3.0840000000000001,"5min":10.385999999999999,"15min":10.385999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.3740000000000001,"15min":1.071},"min":{"1min":0.33800000000000002,"5min":0.255,"15min":0.255},"max":{"1min":3.8969999999999998,"5min":6.9710000000000001,"15min":7.423},"last":0.40400000000000003}]}]},{"osd":5,"up_from":37,"seq":158913790197,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3468784,"kb_used_data":3159208,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":90276368,"statfs":{"total":95995035648,"available":92443000832,"internally_reserved":0,"allocated":3235028992,"data_stored":3234015056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:53:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.9399999999999999,"15min":1.2529999999999999},"min":{"1min":0.41099999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.923,"5min":14.837,"15min":14.837},"last":4.1449999999999996},{"interface":"front","average":{"1min":1.282,"5min":1.9870000000000001,"15min":1.3140000000000001},"min":{"1min":0.45700000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.008,"5min":14.863,"15min":14.863},"last":4.2000000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":2.024,"15min":1.4710000000000001},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":3.306,"5min":10.682,"15min":12.999000000000001},"last":8.2629999999999999},{"interface":"front","average":{"1min":1.353,"5min":1.7310000000000001,"15min":1.2849999999999999},"min":{"1min":0.27900000000000003,"5min":0.254,"15min":0.24299999999999999},"max":{"1min":4.8150000000000004,"5min":12.109,"15min":12.109},"last":8.7690000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:53:25 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":2.1040000000000001,"15min":1.3720000000000001},"min":{"1min":0.28699999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.2469999999999999,"5min":11.393000000000001,"15min":11.393000000000001},"last":10.052},{"interface":"front","average":{"1min":0.64700000000000002,"5min":1.738,"15min":1.1819999999999999},"min":{"1min":0.36399999999999999,"5min":0.28399999999999997,"15min":0.26200000000000001},"max":{"1min":2.0640000000000001,"5min":15.087999999999999,"15min":15.087999999999999},"last":8.7880000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:53:55 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":2.4630000000000001,"15min":1.7450000000000001},"min":{"1min":0.32800000000000001,"5min":0.28599999999999998,"15min":0.25700000000000001},"max":{"1min":4.7640000000000002,"5min":63.508000000000003,"15min":63.508000000000003},"last":3.1480000000000001},{"interface":"front","average":{"1min":0.84299999999999997,"5min":2.597,"15min":1.766},"min":{"1min":0.34499999999999997,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.9990000000000001,"5min":62.418999999999997,"15min":62.418999999999997},"last":3.2109999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":2.5310000000000001,"15min":1.357},"min":{"1min":0.27100000000000002,"5min":0.24099999999999999,"15min":0.23200000000000001},"max":{"1min":4.4989999999999997,"5min":123.241,"15min":123.241},"last":4.3369999999999997},{"interface":"front","average":{"1min":1.0680000000000001,"5min":2.706,"15min":1.5269999999999999},"min":{"1min":0.30299999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.6379999999999999,"5min":123.318,"15min":123.318},"last":3.1850000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:54:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.7250000000000001,"15min":1.2310000000000001},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.5750000000000002,"5min":11.098000000000001,"15min":11.098000000000001},"last":8.9049999999999994},{"interface":"front","average":{"1min":1.2490000000000001,"5min":1.7130000000000001,"15min":1.1970000000000001},"min":{"1min":0.36399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.7210000000000001,"5min":15.16,"15min":20.256},"last":4.1779999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:53:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.853,"15min":1.1799999999999999},"min":{"1min":0.38900000000000001,"5min":0.315,"15min":0.28699999999999998},"max":{"1min":3.3450000000000002,"5min":10.739000000000001,"15min":10.739000000000001},"last":8.468},{"interface":"front","average":{"1min":0.745,"5min":1.5760000000000001,"15min":1.0700000000000001},"min":{"1min":0.35899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.173,"5min":14.154,"15min":14.154},"last":8.734}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2201092096,"data_stored":2201092096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2194210816,"data_stored":2194210816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":57212928,"data_stored":57212928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2586902528,"data_stored":2586902528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2385117184,"data_stored":2385117184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3236757504,"data_stored":3236757504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2549415936,"data_stored":2549415936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2197225472,"data_stored":2197225472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:54:31.738 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:54:31.739 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_2 pg_num 26 2023-10-09T10:54:31.777 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:31.856 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:31.906 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:31.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:31.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:31.955 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 73 16 91266 91250 78.0982 35.125 0.021017 0.0127995 2023-10-09T10:54:32.014 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:32.067 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:32.205 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:32.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:32.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:32.243 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:54:32.316 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:32.359 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:32.401 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:32.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:32.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:32.506 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:32.551 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:32.570 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:32.637 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:32 smithi155 ceph-mon[184854]: pgmap v686: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 2.8 KiB/s rd, 34 MiB/s wr, 545 op/s; 33957/269256 objects degraded (12.611%); 19373/269256 objects misplaced (7.195%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:54:32.637 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:32 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:32.637 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:32 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:32.755 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:32.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:32.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:32.795 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:32 smithi125 ceph-mon[195507]: pgmap v686: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 2.8 KiB/s rd, 34 MiB/s wr, 545 op/s; 33957/269256 objects degraded (12.611%); 19373/269256 objects misplaced (7.195%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:54:32.795 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:32 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:32.796 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:32 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:32.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:32 smithi125 ceph-mon[200197]: pgmap v686: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 2.8 KiB/s rd, 34 MiB/s wr, 545 op/s; 33957/269256 objects degraded (12.611%); 19373/269256 objects misplaced (7.195%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:54:32.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:32 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:32.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:32 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:32.859 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:32.913 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:32.958 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 74 16 91792 91776 77.484 32.875 0.035463 0.0128984 2023-10-09T10:54:33.006 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:33.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:33.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:33.111 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:33.192 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:33.379 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:54:33.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:33 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 33957/269256 objects degraded (12.611%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:33 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 33957/269256 objects degraded (12.611%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:33.478 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:54:33.479 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:54:33.479 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:54:33.479 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:54:33.479 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:54:33.591 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:33.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:33.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:33.697 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:33.741 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:33.778 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:33.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:33.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:33.812 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:33 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 33957/269256 objects degraded (12.611%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:33.817 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:54:33.882 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:33.939 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:33.959 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 75 16 92310 92294 76.8822 32.375 0.0185315 0.0130011 2023-10-09T10:54:34.303 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:54:34.680 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:34.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:34.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:34.692 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:34 smithi125 ceph-mon[195507]: pgmap v687: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2.8 KiB/s rd, 35 MiB/s wr, 567 op/s; 33957/272958 objects degraded (12.440%); 19373/272958 objects misplaced (7.097%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:54:34.692 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:34 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2504919047' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:54:34.692 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:34 smithi125 ceph-mon[200197]: pgmap v687: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2.8 KiB/s rd, 35 MiB/s wr, 567 op/s; 33957/272958 objects degraded (12.440%); 19373/272958 objects misplaced (7.097%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:54:34.692 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:34 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2504919047' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:54:34.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:34 smithi155 ceph-mon[184854]: pgmap v687: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2.8 KiB/s rd, 35 MiB/s wr, 567 op/s; 33957/272958 objects degraded (12.440%); 19373/272958 objects misplaced (7.097%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:54:34.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:34 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2504919047' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:54:34.793 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:34.855 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:34.933 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:54:34.966 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 76 16 92926 92910 76.3771 38.5 0.043224 0.0130868 2023-10-09T10:54:35.027 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:35.050 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:35.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:35.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:35.165 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:35.227 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:35.267 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:35.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:35.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:35.371 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:35.430 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:35.540 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:54:35.541 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:54:35.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:54:35.541 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:54:35.541 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.541 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70444 6.3 6:c0306d38:::benchmark_data_smithi125_244951_object70443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:54:35.541 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.568354+0000", 2023-10-09T10:54:35.541 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 551.96478924899998, 2023-10-09T10:54:35.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047306648, 2023-10-09T10:54:35.541 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.541 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.541 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.542 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:35.542 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:35.542 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70444 2023-10-09T10:54:35.542 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.542 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.542 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.542 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.542 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568354+0000", 2023-10-09T10:54:35.542 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.542 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.542 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.543 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568354+0000", 2023-10-09T10:54:35.543 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.543 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568353+0000", 2023-10-09T10:54:35.543 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:35.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.543 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568387+0000", 2023-10-09T10:54:35.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4495000000000001e-05 2023-10-09T10:54:35.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568407+0000", 2023-10-09T10:54:35.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9729999999999999e-05 2023-10-09T10:54:35.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568412+0000", 2023-10-09T10:54:35.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4149999999999998e-06 2023-10-09T10:54:35.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568457+0000", 2023-10-09T10:54:35.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4270000000000001e-05 2023-10-09T10:54:35.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568587+0000", 2023-10-09T10:54:35.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013011399999999999 2023-10-09T10:54:35.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:54:35.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568753+0000", 2023-10-09T10:54:35.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016615099999999999 2023-10-09T10:54:35.546 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.546 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.546 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.570931+0000", 2023-10-09T10:54:35.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002177596 2023-10-09T10:54:35.546 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.546 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.546 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.570969+0000", 2023-10-09T10:54:35.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8661000000000001e-05 2023-10-09T10:54:35.547 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.547 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.547 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615586+0000", 2023-10-09T10:54:35.547 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.044616524999999997 2023-10-09T10:54:35.547 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.547 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.547 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615636+0000", 2023-10-09T10:54:35.547 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0009000000000002e-05 2023-10-09T10:54:35.547 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.548 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.548 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615661+0000", 2023-10-09T10:54:35.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.516e-05 2023-10-09T10:54:35.548 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.548 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.548 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.548 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.548 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.548 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:70445 6.3 6:c48e8359:::benchmark_data_smithi125_244951_object70444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T10:54:35.548 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:45:23.568425+0000", 2023-10-09T10:54:35.548 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 551.96471899000005, 2023-10-09T10:54:35.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047372754000000003, 2023-10-09T10:54:35.549 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.549 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.549 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.549 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:35.549 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:35.549 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70445 2023-10-09T10:54:35.549 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.549 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.549 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.549 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.549 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568425+0000", 2023-10-09T10:54:35.549 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.550 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.550 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.550 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.550 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568425+0000", 2023-10-09T10:54:35.550 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.550 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.550 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.550 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.550 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568424+0000", 2023-10-09T10:54:35.550 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:54:35.550 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.550 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.551 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.551 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568448+0000", 2023-10-09T10:54:35.551 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3623000000000001e-05 2023-10-09T10:54:35.551 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.551 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.551 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.551 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568453+0000", 2023-10-09T10:54:35.551 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1880000000000001e-06 2023-10-09T10:54:35.551 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.551 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.551 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.551 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.568457+0000", 2023-10-09T10:54:35.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1609999999999996e-06 2023-10-09T10:54:35.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.552 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.569823+0000", 2023-10-09T10:54:35.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001365996 2023-10-09T10:54:35.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.552 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.569913+0000", 2023-10-09T10:54:35.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9388000000000005e-05 2023-10-09T10:54:35.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:54:35.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.570017+0000", 2023-10-09T10:54:35.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103997 2023-10-09T10:54:35.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.571104+0000", 2023-10-09T10:54:35.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00108701 2023-10-09T10:54:35.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.571280+0000", 2023-10-09T10:54:35.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017659500000000001 2023-10-09T10:54:35.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615712+0000", 2023-10-09T10:54:35.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.044431813000000001 2023-10-09T10:54:35.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615755+0000", 2023-10-09T10:54:35.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2588e-05 2023-10-09T10:54:35.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:45:23.615797+0000", 2023-10-09T10:54:35.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2756999999999997e-05 2023-10-09T10:54:35.555 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.555 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.555 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.555 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106416 6.3 6:c0486f03:::benchmark_data_smithi125_244951_object106415:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:54:35.555 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.234195+0000", 2023-10-09T10:54:35.555 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 513.29894875299999, 2023-10-09T10:54:35.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.048317987999999999, 2023-10-09T10:54:35.556 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.556 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.556 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.556 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:35.556 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:35.556 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106416 2023-10-09T10:54:35.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.556 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234195+0000", 2023-10-09T10:54:35.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.557 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234195+0000", 2023-10-09T10:54:35.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.557 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234193+0000", 2023-10-09T10:54:35.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:35.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234285+0000", 2023-10-09T10:54:35.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1546999999999996e-05 2023-10-09T10:54:35.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234305+0000", 2023-10-09T10:54:35.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0119000000000002e-05 2023-10-09T10:54:35.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234313+0000", 2023-10-09T10:54:35.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3590000000000008e-06 2023-10-09T10:54:35.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234382+0000", 2023-10-09T10:54:35.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8261000000000003e-05 2023-10-09T10:54:35.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234490+0000", 2023-10-09T10:54:35.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000108398 2023-10-09T10:54:35.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:54:35.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234588+0000", 2023-10-09T10:54:35.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7966000000000004e-05 2023-10-09T10:54:35.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.236668+0000", 2023-10-09T10:54:35.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020797540000000001 2023-10-09T10:54:35.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.236734+0000", 2023-10-09T10:54:35.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6644999999999996e-05 2023-10-09T10:54:35.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.282424+0000", 2023-10-09T10:54:35.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.045689354000000001 2023-10-09T10:54:35.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.282474+0000", 2023-10-09T10:54:35.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9938999999999997e-05 2023-10-09T10:54:35.562 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.562 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.562 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.282513+0000", 2023-10-09T10:54:35.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9168999999999997e-05 2023-10-09T10:54:35.562 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.562 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.562 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.562 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.562 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:137793 6.3 6:c3f14b3e:::benchmark_data_smithi125_244951_object31350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:54:35.563 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:54:35.563 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 450.35244748899999, 2023-10-09T10:54:35.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.046213456, 2023-10-09T10:54:35.563 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.563 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.563 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.563 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:35.563 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:35.563 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 137793 2023-10-09T10:54:35.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.563 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:54:35.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:54:35.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:54:35.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:54:35.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180698+0000", 2023-10-09T10:54:35.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5009999999999999e-06 2023-10-09T10:54:35.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180702+0000", 2023-10-09T10:54:35.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2200000000000003e-06 2023-10-09T10:54:35.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180705+0000", 2023-10-09T10:54:35.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7779999999999999e-06 2023-10-09T10:54:35.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182790+0000", 2023-10-09T10:54:35.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020852819999999999 2023-10-09T10:54:35.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.567 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182925+0000", 2023-10-09T10:54:35.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013469099999999999 2023-10-09T10:54:35.567 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.567 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.567 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:54:35.567 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.183331+0000", 2023-10-09T10:54:35.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000405783 2023-10-09T10:54:35.567 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.567 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.567 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.567 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.185825+0000", 2023-10-09T10:54:35.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024937789999999998 2023-10-09T10:54:35.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.188205+0000", 2023-10-09T10:54:35.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023806790000000001 2023-10-09T10:54:35.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.225637+0000", 2023-10-09T10:54:35.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.037431091 2023-10-09T10:54:35.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.225680+0000", 2023-10-09T10:54:35.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3856999999999997e-05 2023-10-09T10:54:35.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.226909+0000", 2023-10-09T10:54:35.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001229129 2023-10-09T10:54:35.569 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.569 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.569 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.570 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:137805 6.3 6:c448070c:::benchmark_data_smithi125_244951_object31362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:54:35.570 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:05.197338+0000", 2023-10-09T10:54:35.570 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 450.33580588699999, 2023-10-09T10:54:35.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039251805000000001, 2023-10-09T10:54:35.570 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.570 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.570 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.570 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:35.570 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:35.570 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 137805 2023-10-09T10:54:35.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.570 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197338+0000", 2023-10-09T10:54:35.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197338+0000", 2023-10-09T10:54:35.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197336+0000", 2023-10-09T10:54:35.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:35.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197342+0000", 2023-10-09T10:54:35.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3990000000000001e-06 2023-10-09T10:54:35.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197354+0000", 2023-10-09T10:54:35.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1821000000000001e-05 2023-10-09T10:54:35.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197359+0000", 2023-10-09T10:54:35.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9960000000000001e-06 2023-10-09T10:54:35.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197413+0000", 2023-10-09T10:54:35.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3909999999999997e-05 2023-10-09T10:54:35.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.574 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197587+0000", 2023-10-09T10:54:35.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000173408 2023-10-09T10:54:35.574 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.574 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:54:35.574 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.197988+0000", 2023-10-09T10:54:35.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00040095300000000001 2023-10-09T10:54:35.574 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.574 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.574 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.199873+0000", 2023-10-09T10:54:35.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018855790000000001 2023-10-09T10:54:35.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.575 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.200673+0000", 2023-10-09T10:54:35.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00079994400000000002 2023-10-09T10:54:35.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.575 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.235493+0000", 2023-10-09T10:54:35.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.034820028000000003 2023-10-09T10:54:35.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.576 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.235533+0000", 2023-10-09T10:54:35.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0139000000000003e-05 2023-10-09T10:54:35.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.576 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.236589+0000", 2023-10-09T10:54:35.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001056219 2023-10-09T10:54:35.576 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.576 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.576 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.577 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:152091 6.6 e174/107 6:64b0928f:::benchmark_data_smithi125_244951_object45648:head v 174'5840, mlcod=174'5838)", 2023-10-09T10:54:35.577 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.267910+0000", 2023-10-09T10:54:35.577 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 421.265233233, 2023-10-09T10:54:35.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038740213000000003, 2023-10-09T10:54:35.577 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.577 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.577 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267910+0000", 2023-10-09T10:54:35.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267910+0000", 2023-10-09T10:54:35.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267909+0000", 2023-10-09T10:54:35.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:54:35.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.579 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.579 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267917+0000", 2023-10-09T10:54:35.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1869999999999997e-06 2023-10-09T10:54:35.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.579 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.579 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267931+0000", 2023-10-09T10:54:35.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3239e-05 2023-10-09T10:54:35.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.579 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.579 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267936+0000", 2023-10-09T10:54:35.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.451e-06 2023-10-09T10:54:35.580 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.580 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.267979+0000", 2023-10-09T10:54:35.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3096000000000002e-05 2023-10-09T10:54:35.580 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.580 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.268018+0000", 2023-10-09T10:54:35.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8532999999999999e-05 2023-10-09T10:54:35.580 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.581 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.581 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.581 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.306618+0000", 2023-10-09T10:54:35.581 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038600661000000001 2023-10-09T10:54:35.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.581 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.581 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.581 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.306651+0000", 2023-10-09T10:54:35.581 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2076000000000003e-05 2023-10-09T10:54:35.581 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.581 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.581 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.582 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.582 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.582 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:152093 6.13 6:cd7dbd99:::benchmark_data_smithi125_244951_object45650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:54:35.582 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.270440+0000", 2023-10-09T10:54:35.582 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 421.26270358699998, 2023-10-09T10:54:35.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040432893999999997, 2023-10-09T10:54:35.582 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.582 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.582 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.582 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:35.582 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:35.582 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 152093 2023-10-09T10:54:35.583 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.583 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.583 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.583 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270440+0000", 2023-10-09T10:54:35.583 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.583 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.583 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.583 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270440+0000", 2023-10-09T10:54:35.583 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.583 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.584 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.584 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270438+0000", 2023-10-09T10:54:35.584 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T10:54:35.584 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.584 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.584 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.584 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270450+0000", 2023-10-09T10:54:35.584 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1918e-05 2023-10-09T10:54:35.584 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.584 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.584 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270479+0000", 2023-10-09T10:54:35.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8911000000000002e-05 2023-10-09T10:54:35.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270488+0000", 2023-10-09T10:54:35.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6139999999999998e-06 2023-10-09T10:54:35.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270586+0000", 2023-10-09T10:54:35.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7823000000000002e-05 2023-10-09T10:54:35.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270758+0000", 2023-10-09T10:54:35.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000172103 2023-10-09T10:54:35.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:54:35.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.271250+0000", 2023-10-09T10:54:35.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000491688 2023-10-09T10:54:35.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.278619+0000", 2023-10-09T10:54:35.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0073690730000000003 2023-10-09T10:54:35.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.284528+0000", 2023-10-09T10:54:35.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059091789999999996 2023-10-09T10:54:35.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.309269+0000", 2023-10-09T10:54:35.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024740522000000001 2023-10-09T10:54:35.588 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.588 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.588 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.309319+0000", 2023-10-09T10:54:35.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0912999999999999e-05 2023-10-09T10:54:35.588 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.588 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.588 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.310873+0000", 2023-10-09T10:54:35.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001553369 2023-10-09T10:54:35.588 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.589 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.589 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.589 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.589 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.589 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:152095 6.1 e174/164 6:8109f342:::benchmark_data_smithi125_244951_object45652:head v 174'5685, mlcod=174'5683)", 2023-10-09T10:54:35.589 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.273512+0000", 2023-10-09T10:54:35.589 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 421.25963164699999, 2023-10-09T10:54:35.589 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040480956999999998, 2023-10-09T10:54:35.589 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.589 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.589 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.589 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273512+0000", 2023-10-09T10:54:35.590 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.590 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273512+0000", 2023-10-09T10:54:35.590 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.590 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.591 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273511+0000", 2023-10-09T10:54:35.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:54:35.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.591 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.591 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273520+0000", 2023-10-09T10:54:35.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.645e-06 2023-10-09T10:54:35.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.591 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.591 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273534+0000", 2023-10-09T10:54:35.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3875e-05 2023-10-09T10:54:35.592 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.592 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273541+0000", 2023-10-09T10:54:35.592 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4790000000000001e-06 2023-10-09T10:54:35.592 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.592 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273796+0000", 2023-10-09T10:54:35.592 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025495599999999998 2023-10-09T10:54:35.592 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.592 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.593 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.593 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273819+0000", 2023-10-09T10:54:35.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3175000000000001e-05 2023-10-09T10:54:35.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.593 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.593 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.313948+0000", 2023-10-09T10:54:35.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040129418 2023-10-09T10:54:35.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.593 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.594 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.313993+0000", 2023-10-09T10:54:35.594 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4549000000000003e-05 2023-10-09T10:54:35.594 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.594 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.594 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.594 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.594 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.594 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:161148 6.13 6:c8d28001:::benchmark_data_smithi125_244951_object54705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e176)", 2023-10-09T10:54:35.594 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:47.001392+0000", 2023-10-09T10:54:35.594 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 408.531751767, 2023-10-09T10:54:35.594 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.038885253000000002, 2023-10-09T10:54:35.594 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.595 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.595 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.595 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:35.595 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:35.595 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 161148 2023-10-09T10:54:35.595 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.595 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.595 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.595 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.595 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001392+0000", 2023-10-09T10:54:35.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.595 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.596 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.596 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.596 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001392+0000", 2023-10-09T10:54:35.596 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.596 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.596 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.596 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001391+0000", 2023-10-09T10:54:35.596 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:54:35.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.596 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.596 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001397+0000", 2023-10-09T10:54:35.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0560000000000001e-06 2023-10-09T10:54:35.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001410+0000", 2023-10-09T10:54:35.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2695e-05 2023-10-09T10:54:35.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.001416+0000", 2023-10-09T10:54:35.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9569999999999996e-06 2023-10-09T10:54:35.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.020851+0000", 2023-10-09T10:54:35.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019434414000000001 2023-10-09T10:54:35.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.021010+0000", 2023-10-09T10:54:35.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015947300000000001 2023-10-09T10:54:35.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.599 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:54:35.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.021509+0000", 2023-10-09T10:54:35.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00049899199999999997 2023-10-09T10:54:35.599 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.599 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.599 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.026203+0000", 2023-10-09T10:54:35.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0046932700000000003 2023-10-09T10:54:35.599 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.599 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.599 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.026305+0000", 2023-10-09T10:54:35.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000102744 2023-10-09T10:54:35.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.600 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.038789+0000", 2023-10-09T10:54:35.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012483823 2023-10-09T10:54:35.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.600 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.038863+0000", 2023-10-09T10:54:35.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3634999999999995e-05 2023-10-09T10:54:35.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:47.040277+0000", 2023-10-09T10:54:35.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014141900000000001 2023-10-09T10:54:35.601 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.601 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.601 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.601 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:162969 6.13 6:cec0b93d:::benchmark_data_smithi125_244951_object56526:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-10-09T10:54:35.601 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:49.943081+0000", 2023-10-09T10:54:35.601 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 405.59006298999998, 2023-10-09T10:54:35.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039960117000000003, 2023-10-09T10:54:35.602 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.602 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.602 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.602 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:35.602 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:35.602 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162969 2023-10-09T10:54:35.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.602 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943081+0000", 2023-10-09T10:54:35.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.603 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943081+0000", 2023-10-09T10:54:35.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.603 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943079+0000", 2023-10-09T10:54:35.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:35.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943085+0000", 2023-10-09T10:54:35.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6890000000000001e-06 2023-10-09T10:54:35.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943097+0000", 2023-10-09T10:54:35.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2166e-05 2023-10-09T10:54:35.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943103+0000", 2023-10-09T10:54:35.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9150000000000001e-06 2023-10-09T10:54:35.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943207+0000", 2023-10-09T10:54:35.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104573 2023-10-09T10:54:35.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943393+0000", 2023-10-09T10:54:35.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018516299999999999 2023-10-09T10:54:35.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:54:35.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943913+0000", 2023-10-09T10:54:35.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00052048100000000005 2023-10-09T10:54:35.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.950165+0000", 2023-10-09T10:54:35.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062519439999999997 2023-10-09T10:54:35.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.956256+0000", 2023-10-09T10:54:35.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0060905630000000002 2023-10-09T10:54:35.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.607 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.981642+0000", 2023-10-09T10:54:35.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025386756999999999 2023-10-09T10:54:35.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.607 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.981697+0000", 2023-10-09T10:54:35.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4718e-05 2023-10-09T10:54:35.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.608 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.608 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.983041+0000", 2023-10-09T10:54:35.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001343639 2023-10-09T10:54:35.608 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.608 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.608 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.608 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.608 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.608 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:189608 6.3 6:c6b3429c:::benchmark_data_smithi125_244951_object83165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2023-10-09T10:54:35.608 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:29.169892+0000", 2023-10-09T10:54:35.608 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 366.36325150800002, 2023-10-09T10:54:35.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039847636999999998, 2023-10-09T10:54:35.609 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.609 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.609 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.609 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:35.609 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:35.609 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 189608 2023-10-09T10:54:35.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.609 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169892+0000", 2023-10-09T10:54:35.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169892+0000", 2023-10-09T10:54:35.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169891+0000", 2023-10-09T10:54:35.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:35.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169896+0000", 2023-10-09T10:54:35.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7950000000000004e-06 2023-10-09T10:54:35.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169909+0000", 2023-10-09T10:54:35.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2062000000000001e-05 2023-10-09T10:54:35.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169914+0000", 2023-10-09T10:54:35.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3380000000000004e-06 2023-10-09T10:54:35.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.169959+0000", 2023-10-09T10:54:35.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5392000000000002e-05 2023-10-09T10:54:35.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.170104+0000", 2023-10-09T10:54:35.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014468 2023-10-09T10:54:35.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.613 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:54:35.613 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.170424+0000", 2023-10-09T10:54:35.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00031972199999999999 2023-10-09T10:54:35.613 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.613 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.613 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.613 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.173247+0000", 2023-10-09T10:54:35.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028231710000000002 2023-10-09T10:54:35.613 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.613 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.613 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.613 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.186655+0000", 2023-10-09T10:54:35.614 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013408239000000001 2023-10-09T10:54:35.614 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.614 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.614 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.614 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.209647+0000", 2023-10-09T10:54:35.614 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022992404000000001 2023-10-09T10:54:35.614 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.614 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.614 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.614 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.209694+0000", 2023-10-09T10:54:35.614 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6822e-05 2023-10-09T10:54:35.614 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.615 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.615 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.615 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:29.209740+0000", 2023-10-09T10:54:35.615 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5361999999999997e-05 2023-10-09T10:54:35.615 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.615 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.615 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.615 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.615 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.615 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:192655 6.13 6:cee47a1c:::benchmark_data_smithi125_244951_object86212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-10-09T10:54:35.615 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:54:35.615 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 361.72964909699999, 2023-10-09T10:54:35.615 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039152566, 2023-10-09T10:54:35.616 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.616 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.616 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.616 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:35.616 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:35.616 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 192655 2023-10-09T10:54:35.616 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.616 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.616 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:54:35.616 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:54:35.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803494+0000", 2023-10-09T10:54:35.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:54:35.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803498+0000", 2023-10-09T10:54:35.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4190000000000004e-06 2023-10-09T10:54:35.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803509+0000", 2023-10-09T10:54:35.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0801e-05 2023-10-09T10:54:35.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.803515+0000", 2023-10-09T10:54:35.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.767e-06 2023-10-09T10:54:35.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.619 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.809697+0000", 2023-10-09T10:54:35.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061824799999999997 2023-10-09T10:54:35.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.619 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.809808+0000", 2023-10-09T10:54:35.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000110271 2023-10-09T10:54:35.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.620 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.620 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:54:35.620 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.809938+0000", 2023-10-09T10:54:35.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013000800000000001 2023-10-09T10:54:35.620 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.620 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.620 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.620 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.815553+0000", 2023-10-09T10:54:35.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0056156879999999998 2023-10-09T10:54:35.620 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.620 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.620 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.621 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.815567+0000", 2023-10-09T10:54:35.621 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4102e-05 2023-10-09T10:54:35.621 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.621 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.621 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.621 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.841982+0000", 2023-10-09T10:54:35.621 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026414812999999999 2023-10-09T10:54:35.621 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.621 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.621 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.621 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.842042+0000", 2023-10-09T10:54:35.621 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0198999999999997e-05 2023-10-09T10:54:35.621 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.622 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.622 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.622 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:33.842647+0000", 2023-10-09T10:54:35.622 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00060467599999999995 2023-10-09T10:54:35.622 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.622 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.622 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.622 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.622 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.622 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:212706 6.13 6:c8228b1c:::benchmark_data_smithi125_244951_object106263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-10-09T10:54:35.622 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:02.392352+0000", 2023-10-09T10:54:35.622 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 333.140791616, 2023-10-09T10:54:35.623 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.041188625999999999, 2023-10-09T10:54:35.623 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.623 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.623 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.623 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:54:35.623 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:35.623 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212706 2023-10-09T10:54:35.623 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.623 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.623 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.623 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.623 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392352+0000", 2023-10-09T10:54:35.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.624 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.624 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392352+0000", 2023-10-09T10:54:35.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.624 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.624 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392350+0000", 2023-10-09T10:54:35.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:35.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392357+0000", 2023-10-09T10:54:35.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0380000000000002e-06 2023-10-09T10:54:35.625 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392369+0000", 2023-10-09T10:54:35.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1857e-05 2023-10-09T10:54:35.625 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392375+0000", 2023-10-09T10:54:35.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5219999999999998e-06 2023-10-09T10:54:35.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.626 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392421+0000", 2023-10-09T10:54:35.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5736999999999999e-05 2023-10-09T10:54:35.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.626 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392554+0000", 2023-10-09T10:54:35.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013335300000000001 2023-10-09T10:54:35.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:54:35.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392651+0000", 2023-10-09T10:54:35.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7156000000000003e-05 2023-10-09T10:54:35.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.397156+0000", 2023-10-09T10:54:35.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045045550000000004 2023-10-09T10:54:35.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.628 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.628 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.400165+0000", 2023-10-09T10:54:35.628 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030094319999999998 2023-10-09T10:54:35.628 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.628 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.628 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.433364+0000", 2023-10-09T10:54:35.628 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.033199472000000001 2023-10-09T10:54:35.628 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.628 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.629 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.433409+0000", 2023-10-09T10:54:35.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4070000000000003e-05 2023-10-09T10:54:35.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.629 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.629 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.629 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.433541+0000", 2023-10-09T10:54:35.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013196400000000001 2023-10-09T10:54:35.629 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.629 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.629 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.629 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.630 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:47412 8.5 8:a7ec256e:::benchmark_data_smithi125_329074_object47411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T10:54:35.630 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:53:40.881628+0000", 2023-10-09T10:54:35.630 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 54.651515779999997, 2023-10-09T10:54:35.630 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.046307181000000003, 2023-10-09T10:54:35.630 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.630 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.630 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.630 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:54:35.630 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:54:35.630 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 47412 2023-10-09T10:54:35.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.631 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.881628+0000", 2023-10-09T10:54:35.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.881628+0000", 2023-10-09T10:54:35.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.632 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.632 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.881626+0000", 2023-10-09T10:54:35.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:35.632 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.632 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.632 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.881660+0000", 2023-10-09T10:54:35.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3698000000000002e-05 2023-10-09T10:54:35.632 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.632 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.632 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.881670+0000", 2023-10-09T10:54:35.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7450000000000007e-06 2023-10-09T10:54:35.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.881676+0000", 2023-10-09T10:54:35.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6579999999999999e-06 2023-10-09T10:54:35.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.883172+0000", 2023-10-09T10:54:35.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014959229999999999 2023-10-09T10:54:35.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.883302+0000", 2023-10-09T10:54:35.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013064700000000001 2023-10-09T10:54:35.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:54:35.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.883524+0000", 2023-10-09T10:54:35.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000221766 2023-10-09T10:54:35.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.885395+0000", 2023-10-09T10:54:35.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018711450000000001 2023-10-09T10:54:35.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.635 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.888824+0000", 2023-10-09T10:54:35.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034284419999999999 2023-10-09T10:54:35.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.635 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.927840+0000", 2023-10-09T10:54:35.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039016119000000002 2023-10-09T10:54:35.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.927895+0000", 2023-10-09T10:54:35.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5723999999999997e-05 2023-10-09T10:54:35.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.927935+0000", 2023-10-09T10:54:35.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9578e-05 2023-10-09T10:54:35.636 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.637 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.637 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.637 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83924 8.5 8:a629301e:::benchmark_data_smithi125_329074_object83923:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2023-10-09T10:54:35.637 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.041702+0000", 2023-10-09T10:54:35.637 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 18.491441218999999, 2023-10-09T10:54:35.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.043059291999999999, 2023-10-09T10:54:35.637 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.637 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.637 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.637 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:54:35.638 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:54:35.638 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83924 2023-10-09T10:54:35.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.638 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041702+0000", 2023-10-09T10:54:35.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041702+0000", 2023-10-09T10:54:35.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041701+0000", 2023-10-09T10:54:35.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:35.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041743+0000", 2023-10-09T10:54:35.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2562999999999999e-05 2023-10-09T10:54:35.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.640 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.640 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041768+0000", 2023-10-09T10:54:35.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4527000000000001e-05 2023-10-09T10:54:35.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.640 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.640 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041778+0000", 2023-10-09T10:54:35.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0559e-05 2023-10-09T10:54:35.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.640 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041849+0000", 2023-10-09T10:54:35.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0878999999999997e-05 2023-10-09T10:54:35.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041955+0000", 2023-10-09T10:54:35.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000105648 2023-10-09T10:54:35.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:54:35.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.042088+0000", 2023-10-09T10:54:35.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000133182 2023-10-09T10:54:35.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.642 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.047791+0000", 2023-10-09T10:54:35.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0057031039999999996 2023-10-09T10:54:35.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.642 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.056116+0000", 2023-10-09T10:54:35.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0083248560000000003 2023-10-09T10:54:35.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.643 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.643 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.084671+0000", 2023-10-09T10:54:35.643 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028554619 2023-10-09T10:54:35.643 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.643 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.643 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.643 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.084720+0000", 2023-10-09T10:54:35.643 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9555000000000002e-05 2023-10-09T10:54:35.643 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.643 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.643 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.643 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.084762+0000", 2023-10-09T10:54:35.644 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1335000000000003e-05 2023-10-09T10:54:35.644 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.644 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.644 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.644 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.644 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.644 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83927 8.5 8:a1c8355b:::benchmark_data_smithi125_329074_object83926:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2023-10-09T10:54:35.644 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.041807+0000", 2023-10-09T10:54:35.644 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 18.491336125, 2023-10-09T10:54:35.644 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.043092544000000003, 2023-10-09T10:54:35.644 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.644 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.645 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.645 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:54:35.645 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:54:35.645 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83927 2023-10-09T10:54:35.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.645 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.645 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041807+0000", 2023-10-09T10:54:35.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041807+0000", 2023-10-09T10:54:35.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041807+0000", 2023-10-09T10:54:35.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:54:35.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.647 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041867+0000", 2023-10-09T10:54:35.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0238999999999997e-05 2023-10-09T10:54:35.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.647 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.647 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041882+0000", 2023-10-09T10:54:35.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4769999999999999e-05 2023-10-09T10:54:35.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.647 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.647 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041887+0000", 2023-10-09T10:54:35.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6559999999999997e-06 2023-10-09T10:54:35.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.044590+0000", 2023-10-09T10:54:35.648 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0027030079999999998 2023-10-09T10:54:35.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.044682+0000", 2023-10-09T10:54:35.648 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2077999999999995e-05 2023-10-09T10:54:35.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.649 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:54:35.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.044795+0000", 2023-10-09T10:54:35.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011243099999999999 2023-10-09T10:54:35.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.649 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.047799+0000", 2023-10-09T10:54:35.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030036529999999998 2023-10-09T10:54:35.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.649 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.056142+0000", 2023-10-09T10:54:35.650 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0083437849999999994 2023-10-09T10:54:35.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.650 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.650 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.084829+0000", 2023-10-09T10:54:35.650 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028686832999999998 2023-10-09T10:54:35.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.650 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.650 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.084867+0000", 2023-10-09T10:54:35.650 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7938999999999997e-05 2023-10-09T10:54:35.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.651 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.651 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.084900+0000", 2023-10-09T10:54:35.651 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2867999999999998e-05 2023-10-09T10:54:35.651 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.651 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.651 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.651 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.651 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.651 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:84464 8.5 8:a7d25aec:::benchmark_data_smithi125_329074_object84463:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2023-10-09T10:54:35.651 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:20.139339+0000", 2023-10-09T10:54:35.651 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 15.393804159, 2023-10-09T10:54:35.651 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039776732000000002, 2023-10-09T10:54:35.652 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.652 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.652 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.652 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:54:35.652 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:54:35.652 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 84464 2023-10-09T10:54:35.652 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.652 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.652 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.652 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.652 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:20.139339+0000", 2023-10-09T10:54:35.652 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.653 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.653 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:20.139339+0000", 2023-10-09T10:54:35.653 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.653 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.653 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:20.139337+0000", 2023-10-09T10:54:35.653 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T10:54:35.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:20.139383+0000", 2023-10-09T10:54:35.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6433000000000001e-05 2023-10-09T10:54:35.654 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:20.139411+0000", 2023-10-09T10:54:35.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7866000000000001e-05 2023-10-09T10:54:35.654 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:20.139420+0000", 2023-10-09T10:54:35.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8689999999999993e-06 2023-10-09T10:54:35.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.655 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.655 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:20.139557+0000", 2023-10-09T10:54:35.655 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000137169 2023-10-09T10:54:35.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.655 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.655 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:20.139718+0000", 2023-10-09T10:54:35.655 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000161051 2023-10-09T10:54:35.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:54:35.656 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:20.139885+0000", 2023-10-09T10:54:35.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016716699999999999 2023-10-09T10:54:35.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.656 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:20.146936+0000", 2023-10-09T10:54:35.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0070507670000000003 2023-10-09T10:54:35.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:20.149888+0000", 2023-10-09T10:54:35.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002951403 2023-10-09T10:54:35.657 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.657 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.657 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:20.179026+0000", 2023-10-09T10:54:35.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029138535 2023-10-09T10:54:35.657 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.657 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.657 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:20.179078+0000", 2023-10-09T10:54:35.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2142000000000003e-05 2023-10-09T10:54:35.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.658 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.658 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:20.179116+0000", 2023-10-09T10:54:35.658 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7790999999999998e-05 2023-10-09T10:54:35.658 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.658 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.658 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.658 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:90373 8.5 8:ab1aeb30:::benchmark_data_smithi125_329074_object90372:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-10-09T10:54:35.658 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:30.432480+0000", 2023-10-09T10:54:35.659 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 5.1006631909999998, 2023-10-09T10:54:35.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040172680000000002, 2023-10-09T10:54:35.659 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.659 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.659 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.659 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:54:35.659 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:54:35.659 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 90373 2023-10-09T10:54:35.659 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.659 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432480+0000", 2023-10-09T10:54:35.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.660 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432480+0000", 2023-10-09T10:54:35.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.660 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432480+0000", 2023-10-09T10:54:35.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:54:35.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.661 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.661 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432515+0000", 2023-10-09T10:54:35.661 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5957999999999997e-05 2023-10-09T10:54:35.661 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.661 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.661 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432525+0000", 2023-10-09T10:54:35.661 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1309999999999995e-06 2023-10-09T10:54:35.661 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.661 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.662 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432530+0000", 2023-10-09T10:54:35.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3299999999999998e-06 2023-10-09T10:54:35.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.662 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.662 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.662 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.437580+0000", 2023-10-09T10:54:35.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0050505460000000004 2023-10-09T10:54:35.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.662 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.662 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.662 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.437732+0000", 2023-10-09T10:54:35.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151889 2023-10-09T10:54:35.663 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.663 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:54:35.663 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.437890+0000", 2023-10-09T10:54:35.663 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015802500000000001 2023-10-09T10:54:35.663 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.663 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.663 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.444270+0000", 2023-10-09T10:54:35.663 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0063798969999999998 2023-10-09T10:54:35.663 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.664 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.664 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.664 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.451841+0000", 2023-10-09T10:54:35.664 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0075704609999999997 2023-10-09T10:54:35.664 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.664 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.664 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.664 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.472550+0000", 2023-10-09T10:54:35.664 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020708998999999999 2023-10-09T10:54:35.664 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.664 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.664 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.664 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.472605+0000", 2023-10-09T10:54:35.665 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4846999999999999e-05 2023-10-09T10:54:35.665 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.665 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.665 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.665 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.472653+0000", 2023-10-09T10:54:35.665 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8395e-05 2023-10-09T10:54:35.665 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.665 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.665 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.665 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.665 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.665 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:90375 8.5 8:aff9469c:::benchmark_data_smithi125_329074_object90374:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-10-09T10:54:35.666 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:30.432536+0000", 2023-10-09T10:54:35.666 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 5.1006076619999998, 2023-10-09T10:54:35.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040255572000000003, 2023-10-09T10:54:35.666 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.666 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.666 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:54:35.667 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:54:35.667 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:54:35.667 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 90375 2023-10-09T10:54:35.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.667 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432536+0000", 2023-10-09T10:54:35.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432536+0000", 2023-10-09T10:54:35.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432535+0000", 2023-10-09T10:54:35.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:54:35.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432561+0000", 2023-10-09T10:54:35.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6033e-05 2023-10-09T10:54:35.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432569+0000", 2023-10-09T10:54:35.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4070000000000004e-06 2023-10-09T10:54:35.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432573+0000", 2023-10-09T10:54:35.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4410000000000003e-06 2023-10-09T10:54:35.670 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.440907+0000", 2023-10-09T10:54:35.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0083333880000000006 2023-10-09T10:54:35.670 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.441049+0000", 2023-10-09T10:54:35.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000142657 2023-10-09T10:54:35.670 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.671 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.671 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:54:35.671 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.441204+0000", 2023-10-09T10:54:35.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000154726 2023-10-09T10:54:35.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.671 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.671 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:54:35.671 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.448176+0000", 2023-10-09T10:54:35.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0069721679999999999 2023-10-09T10:54:35.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.671 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.671 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.672 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.451868+0000", 2023-10-09T10:54:35.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0036913610000000002 2023-10-09T10:54:35.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.672 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:35.672 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.472699+0000", 2023-10-09T10:54:35.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.02083115 2023-10-09T10:54:35.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.672 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.672 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.472752+0000", 2023-10-09T10:54:35.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3489000000000002e-05 2023-10-09T10:54:35.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.472791+0000", 2023-10-09T10:54:35.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9277999999999999e-05 2023-10-09T10:54:35.673 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.673 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.673 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.673 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:91038 8.b e286/282 8:dc68fccf:::benchmark_data_smithi125_329074_object91037:head v 286'5712, mlcod=0'0)", 2023-10-09T10:54:35.673 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:31.540617+0000", 2023-10-09T10:54:35.674 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 3.9925266939999999, 2023-10-09T10:54:35.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039446645000000002, 2023-10-09T10:54:35.674 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:54:35.674 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:35.674 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:54:35.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:54:35.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.540617+0000", 2023-10-09T10:54:35.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:54:35.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.540617+0000", 2023-10-09T10:54:35.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:54:35.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:54:35.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.540616+0000", 2023-10-09T10:54:35.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:54:35.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:54:35.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.540647+0000", 2023-10-09T10:54:35.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0171e-05 2023-10-09T10:54:35.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:54:35.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.540666+0000", 2023-10-09T10:54:35.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9069e-05 2023-10-09T10:54:35.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:54:35.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.540672+0000", 2023-10-09T10:54:35.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3770000000000001e-06 2023-10-09T10:54:35.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:54:35.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.550795+0000", 2023-10-09T10:54:35.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010122878 2023-10-09T10:54:35.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:54:35.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.550823+0000", 2023-10-09T10:54:35.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8104999999999999e-05 2023-10-09T10:54:35.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:54:35.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.580018+0000", 2023-10-09T10:54:35.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029195256999999999 2023-10-09T10:54:35.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:54:35.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:54:35.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:54:35.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:31.580063+0000", 2023-10-09T10:54:35.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.515e-05 2023-10-09T10:54:35.678 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.678 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.678 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.678 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:54:35.678 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:54:35.678 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:54:35.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:35 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:54:35.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:35 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2504919047' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:54:35.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:35 smithi155 ceph-mon[184854]: osdmap e287: 8 total, 8 up, 7 in 2023-10-09T10:54:35.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:35 smithi155 ceph-mon[184854]: failed to encode map e287 with expected crc 2023-10-09T10:54:35.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:35 smithi155 ceph-mon[184854]: failed to encode map e287 with expected crc 2023-10-09T10:54:35.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:35 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:54:35.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:35 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2504919047' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:54:35.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:35 smithi125 ceph-mon[195507]: osdmap e287: 8 total, 8 up, 7 in 2023-10-09T10:54:35.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:35 smithi125 ceph-mon[195507]: failed to encode map e287 with expected crc 2023-10-09T10:54:35.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:35 smithi125 ceph-mon[195507]: failed to encode map e287 with expected crc 2023-10-09T10:54:35.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:35 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:54:35.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:35 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2504919047' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:54:35.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:35 smithi125 ceph-mon[200197]: osdmap e287: 8 total, 8 up, 7 in 2023-10-09T10:54:35.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:35 smithi125 ceph-mon[200197]: failed to encode map e287 with expected crc 2023-10-09T10:54:35.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:35 smithi125 ceph-mon[200197]: failed to encode map e287 with expected crc 2023-10-09T10:54:35.938 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:54:35.960 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 77 16 93505 93489 75.8547 36.1875 0.00923242 0.0131777 2023-10-09T10:54:36.260 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:36.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:36.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:36.366 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:36.430 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:36.462 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:36.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:36.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:36.571 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:36.619 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:36.659 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:36.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:36.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:36.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:36 smithi155 ceph-mon[184854]: 1.1c scrub starts 2023-10-09T10:54:36.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:36 smithi155 ceph-mon[184854]: failed to encode map e287 with expected crc 2023-10-09T10:54:36.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:36 smithi155 ceph-mon[184854]: failed to encode map e287 with expected crc 2023-10-09T10:54:36.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:36 smithi155 ceph-mon[184854]: pgmap v689: 119 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 36 MiB/s wr, 568 op/s; 33957/276420 objects degraded (12.285%); 19996/276420 objects misplaced (7.234%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:54:36.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:36 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:36.756 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:36.769 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:54:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:36 smithi125 ceph-mon[200197]: 1.1c scrub starts 2023-10-09T10:54:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:36 smithi125 ceph-mon[200197]: failed to encode map e287 with expected crc 2023-10-09T10:54:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:36 smithi125 ceph-mon[200197]: failed to encode map e287 with expected crc 2023-10-09T10:54:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:36 smithi125 ceph-mon[200197]: pgmap v689: 119 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 36 MiB/s wr, 568 op/s; 33957/276420 objects degraded (12.285%); 19996/276420 objects misplaced (7.234%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:54:36.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:36 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:36.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:36 smithi125 ceph-mon[195507]: 1.1c scrub starts 2023-10-09T10:54:36.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:36 smithi125 ceph-mon[195507]: failed to encode map e287 with expected crc 2023-10-09T10:54:36.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:36 smithi125 ceph-mon[195507]: failed to encode map e287 with expected crc 2023-10-09T10:54:36.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:36 smithi125 ceph-mon[195507]: pgmap v689: 119 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 36 MiB/s wr, 568 op/s; 33957/276420 objects degraded (12.285%); 19996/276420 objects misplaced (7.234%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:54:36.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:36 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:36.867 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:36.965 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 78 16 94044 94028 75.3097 33.6875 0.0340291 0.0132711 2023-10-09T10:54:37.006 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:54:37.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:37.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:54:37.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:37.160 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:37.191 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:37.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:37.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:37.298 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:37.353 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:37.395 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:37.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:37.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:37.500 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:37.527 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 8 pg_num to 26 2023-10-09T10:54:37.570 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:37.814 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:37.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:37 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2572543187' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:54:37.859 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:37 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2572543187' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:54:37.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:37.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:37.864 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:54:37.909 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:54:37.937 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:54:37.961 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:37.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:37 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2572543187' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:54:37.971 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 79 16 94622 94606 74.8075 36.125 0.0377584 0.0133603 2023-10-09T10:54:38.030 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:38.031 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:54:38.078 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:54:38.113 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:54:38.118 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:54:38.153 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:54:38.199 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:54:38.207 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:54:38.207 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:54:38.207 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:54:38.207 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:54:38.207 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:54:38.207 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:54:38.207 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:54:38.208 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:54:38.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[195507]: pgmap v690: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 559 op/s; 33957/278880 objects degraded (12.176%); 20172/278880 objects misplaced (7.233%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:54:38.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2572543187' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:54:38.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[195507]: osdmap e288: 8 total, 8 up, 7 in 2023-10-09T10:54:38.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[195507]: failed to encode map e288 with expected crc 2023-10-09T10:54:38.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:38.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:38.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3667769718' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:38.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 33957/278880 objects degraded (12.176%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:38.827 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[200197]: pgmap v690: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 559 op/s; 33957/278880 objects degraded (12.176%); 20172/278880 objects misplaced (7.233%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:54:38.827 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2572543187' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:54:38.827 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[200197]: osdmap e288: 8 total, 8 up, 7 in 2023-10-09T10:54:38.827 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[200197]: failed to encode map e288 with expected crc 2023-10-09T10:54:38.828 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:38.828 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:38.828 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3667769718' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:38.828 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:38 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 33957/278880 objects degraded (12.176%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:38.971 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:54:38.972409+0000 min lat: 0.00119012 max lat: 2.36975 avg lat: 0.0134462 2023-10-09T10:54:38.971 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:54:38.972 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 80 16 95220 95204 74.3398 37.375 0.0480768 0.0134462 2023-10-09T10:54:38.975 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:38 smithi155 ceph-mon[184854]: pgmap v690: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 559 op/s; 33957/278880 objects degraded (12.176%); 20172/278880 objects misplaced (7.233%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T10:54:38.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:38 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2572543187' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:54:38.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:38 smithi155 ceph-mon[184854]: osdmap e288: 8 total, 8 up, 7 in 2023-10-09T10:54:38.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:38 smithi155 ceph-mon[184854]: failed to encode map e288 with expected crc 2023-10-09T10:54:38.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:38 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:38.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:38 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:38.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:38 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3667769718' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:38.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:38 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 33957/278880 objects degraded (12.176%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:39.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:39.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:39.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:39.673 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:39.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:39 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:54:39.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:39 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:54:39.903 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:39.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:39 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:54:39.971 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 81 16 95593 95577 73.71 23.3125 0.0621609 0.0135054 2023-10-09T10:54:40.227 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:40.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:40.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:40.331 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:54:40.358 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:40.428 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:54:40.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:40.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:54:40.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:40.621 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:40.655 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:40.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:40.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:40.750 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:40.762 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:40.785 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[195507]: pgmap v692: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 32 MiB/s wr, 508 op/s; 33957/281484 objects degraded (12.064%); 20172/281484 objects misplaced (7.166%); 1023 KiB/s, 15 objects/s recovering 2023-10-09T10:54:40.785 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:54:40.785 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[195507]: osdmap e289: 8 total, 8 up, 7 in 2023-10-09T10:54:40.785 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[195507]: failed to encode map e289 with expected crc 2023-10-09T10:54:40.785 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:54:40.785 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[195507]: failed to encode map e289 with expected crc 2023-10-09T10:54:40.785 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:54:40.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:40.786 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[200197]: pgmap v692: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 32 MiB/s wr, 508 op/s; 33957/281484 objects degraded (12.064%); 20172/281484 objects misplaced (7.166%); 1023 KiB/s, 15 objects/s recovering 2023-10-09T10:54:40.786 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:54:40.786 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[200197]: osdmap e289: 8 total, 8 up, 7 in 2023-10-09T10:54:40.786 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[200197]: failed to encode map e289 with expected crc 2023-10-09T10:54:40.786 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:54:40.786 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[200197]: failed to encode map e289 with expected crc 2023-10-09T10:54:40.786 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:54:40.787 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:40 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:40.790 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:40 smithi155 ceph-mon[184854]: pgmap v692: 119 pgs: 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 32 MiB/s wr, 508 op/s; 33957/281484 objects degraded (12.064%); 20172/281484 objects misplaced (7.166%); 1023 KiB/s, 15 objects/s recovering 2023-10-09T10:54:40.790 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:40 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:54:40.790 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:40 smithi155 ceph-mon[184854]: osdmap e289: 8 total, 8 up, 7 in 2023-10-09T10:54:40.790 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:40 smithi155 ceph-mon[184854]: failed to encode map e289 with expected crc 2023-10-09T10:54:40.790 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:40 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:54:40.790 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:40 smithi155 ceph-mon[184854]: failed to encode map e289 with expected crc 2023-10-09T10:54:40.790 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:40 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:54:40.791 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:40 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:40.792 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:40.813 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:40.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:40.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:40.844 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:54:40.844 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:54:40.917 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:40.945 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:40.964 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:40.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:40.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:40.972 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 82 16 95661 95645 72.8633 4.25 0.00667231 0.013624 2023-10-09T10:54:41.068 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:41.098 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:41.269 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:41.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:41.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:41.319 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:54:41] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T10:54:41.320 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:54:41.373 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:41.400 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:41.419 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:41.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:41.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:41.523 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:54:41.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[195507]: failed to encode map e289 with expected crc 2023-10-09T10:54:41.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[195507]: failed to encode map e290 with expected crc 2023-10-09T10:54:41.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[195507]: failed to encode map e290 with expected crc 2023-10-09T10:54:41.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[195507]: osdmap e290: 8 total, 8 up, 7 in 2023-10-09T10:54:41.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[195507]: failed to encode map e290 with expected crc 2023-10-09T10:54:41.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[195507]: failed to encode map e290 with expected crc 2023-10-09T10:54:41.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[195507]: 8.7 continuing backfill to osd.2 from (271'1000,288'5953] MIN to 288'5953 2023-10-09T10:54:41.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[195507]: 8.4 continuing backfill to osd.2 from (271'600,288'5971] MIN to 288'5971 2023-10-09T10:54:41.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[200197]: failed to encode map e289 with expected crc 2023-10-09T10:54:41.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[200197]: failed to encode map e290 with expected crc 2023-10-09T10:54:41.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[200197]: failed to encode map e290 with expected crc 2023-10-09T10:54:41.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[200197]: osdmap e290: 8 total, 8 up, 7 in 2023-10-09T10:54:41.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[200197]: failed to encode map e290 with expected crc 2023-10-09T10:54:41.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[200197]: failed to encode map e290 with expected crc 2023-10-09T10:54:41.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[200197]: 8.7 continuing backfill to osd.2 from (271'1000,288'5953] MIN to 288'5953 2023-10-09T10:54:41.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:41 smithi125 ceph-mon[200197]: 8.4 continuing backfill to osd.2 from (271'600,288'5971] MIN to 288'5971 2023-10-09T10:54:41.577 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:41.654 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:54:41.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:41.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:54:41.713 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:41.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:41 smithi155 ceph-mon[184854]: failed to encode map e289 with expected crc 2023-10-09T10:54:41.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:41 smithi155 ceph-mon[184854]: failed to encode map e290 with expected crc 2023-10-09T10:54:41.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:41 smithi155 ceph-mon[184854]: failed to encode map e290 with expected crc 2023-10-09T10:54:41.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:41 smithi155 ceph-mon[184854]: osdmap e290: 8 total, 8 up, 7 in 2023-10-09T10:54:41.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:41 smithi155 ceph-mon[184854]: failed to encode map e290 with expected crc 2023-10-09T10:54:41.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:41 smithi155 ceph-mon[184854]: failed to encode map e290 with expected crc 2023-10-09T10:54:41.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:41 smithi155 ceph-mon[184854]: 8.7 continuing backfill to osd.2 from (271'1000,288'5953] MIN to 288'5953 2023-10-09T10:54:41.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:41 smithi155 ceph-mon[184854]: 8.4 continuing backfill to osd.2 from (271'600,288'5971] MIN to 288'5971 2023-10-09T10:54:41.759 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:41.786 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:41.805 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:41.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:41.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:41.909 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:41.945 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:41.972 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 83 16 95664 95648 71.9881 0.1875 0.00311472 0.013652 2023-10-09T10:54:42.120 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:42.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:42.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:42.224 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:42.259 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:42.336 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:42.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:42.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:42.440 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:42.471 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:42.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:42 smithi125 ceph-mon[200197]: pgmap v695: 129 pgs: 2 peering, 2 remapped+peering, 7 unknown, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 109 active+clean; 6.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s; 29354/294309 objects degraded (9.974%); 12385/294309 objects misplaced (4.208%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:54:42.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:42 smithi125 ceph-mon[200197]: failed to encode map e291 with expected crc 2023-10-09T10:54:42.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:42 smithi125 ceph-mon[200197]: failed to encode map e291 with expected crc 2023-10-09T10:54:42.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:42 smithi125 ceph-mon[200197]: osdmap e291: 8 total, 8 up, 7 in 2023-10-09T10:54:42.532 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:42 smithi125 ceph-mon[195507]: pgmap v695: 129 pgs: 2 peering, 2 remapped+peering, 7 unknown, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 109 active+clean; 6.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s; 29354/294309 objects degraded (9.974%); 12385/294309 objects misplaced (4.208%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:54:42.532 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:42 smithi125 ceph-mon[195507]: failed to encode map e291 with expected crc 2023-10-09T10:54:42.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:42 smithi125 ceph-mon[195507]: failed to encode map e291 with expected crc 2023-10-09T10:54:42.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:42 smithi125 ceph-mon[195507]: osdmap e291: 8 total, 8 up, 7 in 2023-10-09T10:54:42.598 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:42.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:42.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:42.703 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:42.736 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:42.761 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:42.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:42.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:42.788 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:54:42.865 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:42.900 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:42.923 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:42.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:42.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:42.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:42 smithi155 ceph-mon[184854]: pgmap v695: 129 pgs: 2 peering, 2 remapped+peering, 7 unknown, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 109 active+clean; 6.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s; 29354/294309 objects degraded (9.974%); 12385/294309 objects misplaced (4.208%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T10:54:42.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:42 smithi155 ceph-mon[184854]: failed to encode map e291 with expected crc 2023-10-09T10:54:42.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:42 smithi155 ceph-mon[184854]: failed to encode map e291 with expected crc 2023-10-09T10:54:42.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:42 smithi155 ceph-mon[184854]: osdmap e291: 8 total, 8 up, 7 in 2023-10-09T10:54:42.972 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 84 16 95740 95724 71.1879 4.75 0.00449331 0.0137756 2023-10-09T10:54:43.018 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:54:43.018 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:54:43.028 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:43.055 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:43.075 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:43.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:43.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:43.108 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 7, 2] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:54:43.109 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:54:43.109 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-10-09T10:54:43.109 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd in 1 2023-10-09T10:54:43.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:43.207 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:43.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:43.233 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:43.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:43.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:43.337 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:43.364 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:43.383 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:43.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:43.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:43.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:43.520 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:43.550 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:54:43.708 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3953500025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:43.708 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[195507]: failed to encode map e292 with expected crc 2023-10-09T10:54:43.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[195507]: failed to encode map e292 with expected crc 2023-10-09T10:54:43.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[195507]: osdmap e292: 8 total, 8 up, 7 in 2023-10-09T10:54:43.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[195507]: failed to encode map e292 with expected crc 2023-10-09T10:54:43.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[195507]: 8.17 continuing backfill to osd.2 from (271'1000,288'5953] MIN to 288'5953 2023-10-09T10:54:43.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[195507]: 8.14 continuing backfill to osd.2 from (271'600,288'5971] MIN to 288'5971 2023-10-09T10:54:43.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[195507]: failed to encode map e292 with expected crc 2023-10-09T10:54:43.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 29354/294309 objects degraded (9.974%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:43.709 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3953500025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:43.709 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[200197]: failed to encode map e292 with expected crc 2023-10-09T10:54:43.709 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[200197]: failed to encode map e292 with expected crc 2023-10-09T10:54:43.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[200197]: osdmap e292: 8 total, 8 up, 7 in 2023-10-09T10:54:43.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[200197]: failed to encode map e292 with expected crc 2023-10-09T10:54:43.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[200197]: 8.17 continuing backfill to osd.2 from (271'1000,288'5953] MIN to 288'5953 2023-10-09T10:54:43.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[200197]: 8.14 continuing backfill to osd.2 from (271'600,288'5971] MIN to 288'5971 2023-10-09T10:54:43.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[200197]: failed to encode map e292 with expected crc 2023-10-09T10:54:43.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:43 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 29354/294309 objects degraded (9.974%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:43.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:43 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3953500025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:43.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:43 smithi155 ceph-mon[184854]: failed to encode map e292 with expected crc 2023-10-09T10:54:43.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:43 smithi155 ceph-mon[184854]: failed to encode map e292 with expected crc 2023-10-09T10:54:43.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:43 smithi155 ceph-mon[184854]: osdmap e292: 8 total, 8 up, 7 in 2023-10-09T10:54:43.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:43 smithi155 ceph-mon[184854]: failed to encode map e292 with expected crc 2023-10-09T10:54:43.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:43 smithi155 ceph-mon[184854]: 8.17 continuing backfill to osd.2 from (271'1000,288'5953] MIN to 288'5953 2023-10-09T10:54:43.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:43 smithi155 ceph-mon[184854]: 8.14 continuing backfill to osd.2 from (271'600,288'5971] MIN to 288'5971 2023-10-09T10:54:43.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:43 smithi155 ceph-mon[184854]: failed to encode map e292 with expected crc 2023-10-09T10:54:43.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:43 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 29354/294309 objects degraded (9.974%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:43.774 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:54:43.972 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 85 16 95740 95724 70.3508 0 - 0.0137756 2023-10-09T10:54:44.101 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:44.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:44.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:44.206 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:44.234 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:44.253 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:44.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:44.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:44.357 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:54:44.394 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:44.407 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:44.574 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:54:44.594 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:54:44.613 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:54:44.632 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:54:44.652 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:54:44.671 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:54:44.690 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:54:44.707 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:44 smithi125 ceph-mon[195507]: pgmap v698: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 2 remapped+peering, 4 unknown, 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 2 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 109 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 24209/295584 objects degraded (8.190%); 17414/295584 objects misplaced (5.891%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T10:54:44.708 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:44 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:54:44.708 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:44 smithi125 ceph-mon[195507]: failed to encode map e293 with expected crc 2023-10-09T10:54:44.708 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:44 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3953500025' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:54:44.708 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:44 smithi125 ceph-mon[195507]: osdmap e293: 8 total, 8 up, 7 in 2023-10-09T10:54:44.708 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:44 smithi125 ceph-mon[195507]: failed to encode map e293 with expected crc 2023-10-09T10:54:44.708 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:44 smithi125 ceph-mon[200197]: pgmap v698: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 2 remapped+peering, 4 unknown, 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 2 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 109 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 24209/295584 objects degraded (8.190%); 17414/295584 objects misplaced (5.891%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T10:54:44.708 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:44 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:54:44.708 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:44 smithi125 ceph-mon[200197]: failed to encode map e293 with expected crc 2023-10-09T10:54:44.709 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:44 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3953500025' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:54:44.709 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:44 smithi125 ceph-mon[200197]: osdmap e293: 8 total, 8 up, 7 in 2023-10-09T10:54:44.709 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:44 smithi125 ceph-mon[200197]: failed to encode map e293 with expected crc 2023-10-09T10:54:44.709 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:44.715 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:54:44.718 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:54:44.718 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:54:44.718 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:54:44.718 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:54:44.718 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:54:44.718 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:54:44.718 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:54:44.718 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:54:44.719 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:54:44.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:44.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:54:44.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:44.865 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:44.940 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:44.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:44.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:44.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:44 smithi155 ceph-mon[184854]: pgmap v698: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 2 remapped+peering, 4 unknown, 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 2 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 109 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 24209/295584 objects degraded (8.190%); 17414/295584 objects misplaced (5.891%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T10:54:44.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:44 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:54:44.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:44 smithi155 ceph-mon[184854]: failed to encode map e293 with expected crc 2023-10-09T10:54:44.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:44 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3953500025' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:54:44.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:44 smithi155 ceph-mon[184854]: osdmap e293: 8 total, 8 up, 7 in 2023-10-09T10:54:44.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:44 smithi155 ceph-mon[184854]: failed to encode map e293 with expected crc 2023-10-09T10:54:44.972 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 86 16 95740 95724 69.533 0 - 0.0137756 2023-10-09T10:54:45.044 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:45.071 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:45.093 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:45.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:45.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:45.197 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:45.227 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:45.247 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:45.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:45.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:45.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:45.385 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:45.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:45.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:45.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:45.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:45.540 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:45.738 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:45 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3475983873' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:45.738 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:45.738 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:54:45.738 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:45 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3475983873' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:45.738 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:45.738 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:54:45.763 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:45.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:45.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:45.866 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:45.906 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:45.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:45 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3475983873' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:45.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:45.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:54:45.972 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 87 16 95832 95816 68.8002 1.91667 0.00688831 0.0143326 2023-10-09T10:54:46.183 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:46.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:46.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:46.288 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:46.324 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:46.349 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:46.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:46.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:46.454 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:46.528 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:46.555 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:46 smithi125 ceph-mon[195507]: failed to encode map e293 with expected crc 2023-10-09T10:54:46.555 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:46 smithi125 ceph-mon[195507]: pgmap v700: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 unknown, 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 111 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 11 MiB/s wr, 179 op/s; 32843/295728 objects degraded (11.106%); 18795/295728 objects misplaced (6.356%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:54:46.555 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:46.577 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:46.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:46.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:46.632 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:54:46.632 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:54:46.632 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:54:46.632 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:54:46.632 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.632 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42384 6.10 6:0c7bd3cf:::benchmark_data_smithi125_244951_object42383:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:54:46.632 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:54:46.632 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 591.51122748, 2023-10-09T10:54:46.632 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0577403670000001, 2023-10-09T10:54:46.632 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.633 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:46.633 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.633 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:54:46.633 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:46.633 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42384 2023-10-09T10:54:46.633 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.633 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.633 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.633 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:54:46.633 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.633 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117186+0000", 2023-10-09T10:54:46.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117185+0000", 2023-10-09T10:54:46.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:46.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117225+0000", 2023-10-09T10:54:46.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0364000000000002e-05 2023-10-09T10:54:46.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117240+0000", 2023-10-09T10:54:46.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4425e-05 2023-10-09T10:54:46.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117248+0000", 2023-10-09T10:54:46.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4999999999999999e-06 2023-10-09T10:54:46.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117301+0000", 2023-10-09T10:54:46.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2685999999999997e-05 2023-10-09T10:54:46.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.117312+0000", 2023-10-09T10:54:46.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1273e-05 2023-10-09T10:54:46.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103469+0000", 2023-10-09T10:54:46.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98615701300000003 2023-10-09T10:54:46.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103483+0000", 2023-10-09T10:54:46.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3207000000000001e-05 2023-10-09T10:54:46.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.148764+0000", 2023-10-09T10:54:46.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0452812789999999 2023-10-09T10:54:46.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.638 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.148882+0000", 2023-10-09T10:54:46.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118095 2023-10-09T10:54:46.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.638 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:54:46.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.149057+0000", 2023-10-09T10:54:46.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017516400000000001 2023-10-09T10:54:46.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174105+0000", 2023-10-09T10:54:46.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.025048237000000001 2023-10-09T10:54:46.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174158+0000", 2023-10-09T10:54:46.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2330999999999998e-05 2023-10-09T10:54:46.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.640 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.640 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174325+0000", 2023-10-09T10:54:46.640 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016690100000000001 2023-10-09T10:54:46.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.640 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:54:46.640 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174367+0000", 2023-10-09T10:54:46.640 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2759000000000002e-05 2023-10-09T10:54:46.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.640 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.640 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174927+0000", 2023-10-09T10:54:46.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00055942800000000005 2023-10-09T10:54:46.641 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.641 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.641 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.641 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42398 6.10 6:0867fefc:::benchmark_data_smithi125_244951_object42397:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:54:46.641 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:54:46.641 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 591.50044871099999, 2023-10-09T10:54:46.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0510695769999998, 2023-10-09T10:54:46.641 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.641 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:46.642 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.642 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:54:46.642 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:46.642 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42398 2023-10-09T10:54:46.642 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.642 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.642 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.642 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.642 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:54:46.642 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.642 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.642 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.643 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127965+0000", 2023-10-09T10:54:46.643 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.643 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.643 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.643 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127964+0000", 2023-10-09T10:54:46.643 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:46.643 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.643 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.643 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.127989+0000", 2023-10-09T10:54:46.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4811e-05 2023-10-09T10:54:46.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128005+0000", 2023-10-09T10:54:46.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6436999999999999e-05 2023-10-09T10:54:46.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128014+0000", 2023-10-09T10:54:46.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5990000000000002e-06 2023-10-09T10:54:46.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.645 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.645 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128072+0000", 2023-10-09T10:54:46.645 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7980999999999999e-05 2023-10-09T10:54:46.645 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.645 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.645 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.645 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.128083+0000", 2023-10-09T10:54:46.645 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1051e-05 2023-10-09T10:54:46.645 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.645 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.645 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.645 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103714+0000", 2023-10-09T10:54:46.646 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97563124099999998 2023-10-09T10:54:46.646 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.646 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.646 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.646 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103725+0000", 2023-10-09T10:54:46.646 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1121000000000001e-05 2023-10-09T10:54:46.646 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.646 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.646 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.646 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.174971+0000", 2023-10-09T10:54:46.646 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0712457310000001 2023-10-09T10:54:46.646 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.646 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.647 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.647 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.175149+0000", 2023-10-09T10:54:46.647 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000178584 2023-10-09T10:54:46.647 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.647 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.647 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:54:46.647 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.175358+0000", 2023-10-09T10:54:46.647 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000208848 2023-10-09T10:54:46.647 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.647 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.647 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.648 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178406+0000", 2023-10-09T10:54:46.648 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030478049999999998 2023-10-09T10:54:46.648 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.648 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.648 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.648 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178418+0000", 2023-10-09T10:54:46.648 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1499e-05 2023-10-09T10:54:46.648 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.648 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.648 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.648 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178466+0000", 2023-10-09T10:54:46.648 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8715999999999999e-05 2023-10-09T10:54:46.649 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:54:46.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178522+0000", 2023-10-09T10:54:46.649 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5952999999999998e-05 2023-10-09T10:54:46.649 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179035+0000", 2023-10-09T10:54:46.649 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051254299999999996 2023-10-09T10:54:46.649 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.649 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.650 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.650 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.650 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.650 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42518 6.10 6:0fbcc8bc:::benchmark_data_smithi125_244951_object42517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:54:46.650 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:54:46.650 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 591.43405138599996, 2023-10-09T10:54:46.650 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9852423510000001, 2023-10-09T10:54:46.650 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.650 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:46.650 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.650 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:54:46.651 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:46.651 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42518 2023-10-09T10:54:46.651 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.651 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.651 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.651 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.651 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:54:46.651 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.651 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.651 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.651 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.651 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194362+0000", 2023-10-09T10:54:46.651 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.652 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.652 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.652 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.652 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194361+0000", 2023-10-09T10:54:46.652 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:46.652 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.652 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.652 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.652 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194385+0000", 2023-10-09T10:54:46.652 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4017000000000001e-05 2023-10-09T10:54:46.652 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.652 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.653 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.653 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194401+0000", 2023-10-09T10:54:46.653 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5841e-05 2023-10-09T10:54:46.653 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.653 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.653 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.653 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194408+0000", 2023-10-09T10:54:46.653 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0450000000000003e-06 2023-10-09T10:54:46.653 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.653 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.653 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.653 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194460+0000", 2023-10-09T10:54:46.654 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2015000000000003e-05 2023-10-09T10:54:46.654 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.654 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.654 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.654 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.194474+0000", 2023-10-09T10:54:46.654 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4486e-05 2023-10-09T10:54:46.654 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.654 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.654 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.654 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103750+0000", 2023-10-09T10:54:46.654 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.90927595699999997 2023-10-09T10:54:46.654 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.655 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.655 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103758+0000", 2023-10-09T10:54:46.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9000000000000006e-06 2023-10-09T10:54:46.655 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.655 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.655 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176100+0000", 2023-10-09T10:54:46.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.07234187 2023-10-09T10:54:46.655 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.656 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.656 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176198+0000", 2023-10-09T10:54:46.656 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7408e-05 2023-10-09T10:54:46.656 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.656 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.656 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:54:46.656 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176317+0000", 2023-10-09T10:54:46.656 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011998900000000001 2023-10-09T10:54:46.656 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.656 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.656 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.656 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.178425+0000", 2023-10-09T10:54:46.656 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021072 2023-10-09T10:54:46.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.657 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179085+0000", 2023-10-09T10:54:46.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00066011800000000001 2023-10-09T10:54:46.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.657 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179122+0000", 2023-10-09T10:54:46.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6831e-05 2023-10-09T10:54:46.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.658 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:54:46.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179155+0000", 2023-10-09T10:54:46.658 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3080000000000002e-05 2023-10-09T10:54:46.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.658 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179605+0000", 2023-10-09T10:54:46.658 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000450121 2023-10-09T10:54:46.658 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.658 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.658 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.659 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.659 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42546 6.10 6:0889293a:::benchmark_data_smithi125_244951_object42545:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:54:46.659 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:54:46.659 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 591.41962195300005, 2023-10-09T10:54:46.659 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.971488017, 2023-10-09T10:54:46.659 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.659 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:46.659 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.659 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:54:46.659 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:46.659 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42546 2023-10-09T10:54:46.659 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.659 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.660 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.660 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.660 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:54:46.660 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.660 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.660 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.660 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.660 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208792+0000", 2023-10-09T10:54:46.660 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.660 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.660 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.660 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.661 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208790+0000", 2023-10-09T10:54:46.661 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:54:46.661 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.661 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.661 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.661 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208815+0000", 2023-10-09T10:54:46.661 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5103e-05 2023-10-09T10:54:46.661 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.661 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.661 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.661 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208831+0000", 2023-10-09T10:54:46.661 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5235000000000001e-05 2023-10-09T10:54:46.662 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.662 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.662 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.662 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208838+0000", 2023-10-09T10:54:46.662 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6510000000000001e-06 2023-10-09T10:54:46.662 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.662 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.662 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.662 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208895+0000", 2023-10-09T10:54:46.662 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6817999999999997e-05 2023-10-09T10:54:46.662 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.662 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.662 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.663 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.208906+0000", 2023-10-09T10:54:46.663 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0550000000000001e-05 2023-10-09T10:54:46.663 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.663 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.663 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.663 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103780+0000", 2023-10-09T10:54:46.663 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.89487466699999996 2023-10-09T10:54:46.663 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.663 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.663 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.663 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103788+0000", 2023-10-09T10:54:46.663 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0970000000000006e-06 2023-10-09T10:54:46.664 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.664 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.664 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.664 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176855+0000", 2023-10-09T10:54:46.664 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0730664299999999 2023-10-09T10:54:46.664 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.664 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.664 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.664 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.176993+0000", 2023-10-09T10:54:46.664 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013810999999999999 2023-10-09T10:54:46.664 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.664 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.665 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:54:46.665 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.177166+0000", 2023-10-09T10:54:46.665 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017289900000000001 2023-10-09T10:54:46.665 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.665 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.665 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.665 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179093+0000", 2023-10-09T10:54:46.665 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019276969999999999 2023-10-09T10:54:46.665 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.665 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.665 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.665 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179674+0000", 2023-10-09T10:54:46.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00058078000000000003 2023-10-09T10:54:46.666 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.666 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.666 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.666 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179730+0000", 2023-10-09T10:54:46.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5418999999999999e-05 2023-10-09T10:54:46.666 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.666 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.666 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:54:46.666 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.179788+0000", 2023-10-09T10:54:46.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8510000000000001e-05 2023-10-09T10:54:46.666 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.666 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.667 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.667 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180280+0000", 2023-10-09T10:54:46.667 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00049179400000000004 2023-10-09T10:54:46.667 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.667 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.667 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.667 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.667 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.667 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42570 6.10 6:0f6ea5a0:::benchmark_data_smithi125_244951_object42569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:54:46.667 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:54:46.667 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 591.40553746000001, 2023-10-09T10:54:46.667 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.969653487, 2023-10-09T10:54:46.668 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.668 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:46.668 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.668 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:54:46.668 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:46.668 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42570 2023-10-09T10:54:46.668 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.668 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.668 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.668 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.668 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:54:46.668 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.668 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.669 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.669 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222876+0000", 2023-10-09T10:54:46.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.669 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.669 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222875+0000", 2023-10-09T10:54:46.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:46.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.669 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.669 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222897+0000", 2023-10-09T10:54:46.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1678000000000001e-05 2023-10-09T10:54:46.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.670 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.670 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222906+0000", 2023-10-09T10:54:46.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4190000000000008e-06 2023-10-09T10:54:46.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.670 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.670 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222913+0000", 2023-10-09T10:54:46.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2659999999999998e-06 2023-10-09T10:54:46.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.670 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222958+0000", 2023-10-09T10:54:46.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5979000000000001e-05 2023-10-09T10:54:46.671 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.222965+0000", 2023-10-09T10:54:46.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0099999999999998e-06 2023-10-09T10:54:46.671 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103823+0000", 2023-10-09T10:54:46.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.880857891 2023-10-09T10:54:46.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103833+0000", 2023-10-09T10:54:46.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8910000000000007e-06 2023-10-09T10:54:46.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180446+0000", 2023-10-09T10:54:46.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0766125440000001 2023-10-09T10:54:46.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180594+0000", 2023-10-09T10:54:46.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014846600000000001 2023-10-09T10:54:46.673 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.673 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.673 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:54:46.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.180775+0000", 2023-10-09T10:54:46.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018071999999999999 2023-10-09T10:54:46.673 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.673 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.673 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183638+0000", 2023-10-09T10:54:46.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028626440000000001 2023-10-09T10:54:46.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.674 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.674 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183677+0000", 2023-10-09T10:54:46.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9545000000000002e-05 2023-10-09T10:54:46.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.674 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.674 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191583+0000", 2023-10-09T10:54:46.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0079059569999999999 2023-10-09T10:54:46.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.675 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:54:46.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.191643+0000", 2023-10-09T10:54:46.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9590999999999999e-05 2023-10-09T10:54:46.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.675 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.675 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192530+0000", 2023-10-09T10:54:46.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00088716399999999996 2023-10-09T10:54:46.675 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.675 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.675 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.675 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.676 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42586 6.10 6:090d6b8d:::benchmark_data_smithi125_244951_object42585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:54:46.676 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:54:46.676 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 591.39894291899998, 2023-10-09T10:54:46.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.963875493, 2023-10-09T10:54:46.676 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.676 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:46.676 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.676 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:54:46.676 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:46.676 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42586 2023-10-09T10:54:46.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.676 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.677 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:54:46.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.677 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229471+0000", 2023-10-09T10:54:46.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.677 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229470+0000", 2023-10-09T10:54:46.678 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:54:46.678 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.678 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.678 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229492+0000", 2023-10-09T10:54:46.678 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1716999999999999e-05 2023-10-09T10:54:46.678 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.678 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.678 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229501+0000", 2023-10-09T10:54:46.678 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0929999999999992e-06 2023-10-09T10:54:46.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229507+0000", 2023-10-09T10:54:46.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6760000000000001e-06 2023-10-09T10:54:46.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229565+0000", 2023-10-09T10:54:46.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7450000000000001e-05 2023-10-09T10:54:46.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.229572+0000", 2023-10-09T10:54:46.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0820000000000001e-06 2023-10-09T10:54:46.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103865+0000", 2023-10-09T10:54:46.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.87429282200000003 2023-10-09T10:54:46.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103873+0000", 2023-10-09T10:54:46.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2090000000000005e-06 2023-10-09T10:54:46.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.681 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181656+0000", 2023-10-09T10:54:46.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.077783462 2023-10-09T10:54:46.681 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.681 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181791+0000", 2023-10-09T10:54:46.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000134351 2023-10-09T10:54:46.681 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.681 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:54:46.682 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.181948+0000", 2023-10-09T10:54:46.682 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015678600000000001 2023-10-09T10:54:46.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.682 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.682 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183685+0000", 2023-10-09T10:54:46.682 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00173705 2023-10-09T10:54:46.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.682 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.682 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183719+0000", 2023-10-09T10:54:46.682 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4301999999999998e-05 2023-10-09T10:54:46.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192640+0000", 2023-10-09T10:54:46.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0089210899999999996 2023-10-09T10:54:46.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:54:46.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.192696+0000", 2023-10-09T10:54:46.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5733000000000003e-05 2023-10-09T10:54:46.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193346+0000", 2023-10-09T10:54:46.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00065071200000000001 2023-10-09T10:54:46.684 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.684 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.684 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.684 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.684 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.684 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42606 6.10 6:0826a955:::benchmark_data_smithi125_244951_object42605:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:54:46.684 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:54:46.684 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 591.39018332399996, 2023-10-09T10:54:46.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.955680458, 2023-10-09T10:54:46.684 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.684 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:46.684 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.685 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:54:46.685 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:46.685 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42606 2023-10-09T10:54:46.685 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.685 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.685 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.685 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:54:46.685 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.685 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.685 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238231+0000", 2023-10-09T10:54:46.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238230+0000", 2023-10-09T10:54:46.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:54:46.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238257+0000", 2023-10-09T10:54:46.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7551000000000001e-05 2023-10-09T10:54:46.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238267+0000", 2023-10-09T10:54:46.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5820000000000008e-06 2023-10-09T10:54:46.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238273+0000", 2023-10-09T10:54:46.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5490000000000003e-06 2023-10-09T10:54:46.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238322+0000", 2023-10-09T10:54:46.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8467999999999997e-05 2023-10-09T10:54:46.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.688 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.238332+0000", 2023-10-09T10:54:46.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9000000000000001e-06 2023-10-09T10:54:46.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.688 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103898+0000", 2023-10-09T10:54:46.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.86556675699999996 2023-10-09T10:54:46.689 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.689 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.689 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103906+0000", 2023-10-09T10:54:46.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7710000000000006e-06 2023-10-09T10:54:46.689 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.689 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.689 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.182699+0000", 2023-10-09T10:54:46.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0787928490000001 2023-10-09T10:54:46.689 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.689 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.182847+0000", 2023-10-09T10:54:46.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014758600000000001 2023-10-09T10:54:46.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:54:46.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183016+0000", 2023-10-09T10:54:46.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000169326 2023-10-09T10:54:46.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186922+0000", 2023-10-09T10:54:46.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039060269999999998 2023-10-09T10:54:46.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186980+0000", 2023-10-09T10:54:46.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8313999999999998e-05 2023-10-09T10:54:46.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193472+0000", 2023-10-09T10:54:46.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0064917020000000002 2023-10-09T10:54:46.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:54:46.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193522+0000", 2023-10-09T10:54:46.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9672999999999997e-05 2023-10-09T10:54:46.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.193911+0000", 2023-10-09T10:54:46.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00038944099999999997 2023-10-09T10:54:46.692 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.692 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.692 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.693 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42608 6.10 6:0c1c992a:::benchmark_data_smithi125_244951_object42607:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:54:46.693 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:54:46.693 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 591.38709162400005, 2023-10-09T10:54:46.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9589951289999998, 2023-10-09T10:54:46.693 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.693 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:46.693 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.693 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:54:46.693 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:46.693 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42608 2023-10-09T10:54:46.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.693 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:54:46.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241322+0000", 2023-10-09T10:54:46.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241321+0000", 2023-10-09T10:54:46.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:54:46.695 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241343+0000", 2023-10-09T10:54:46.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1987000000000001e-05 2023-10-09T10:54:46.695 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241355+0000", 2023-10-09T10:54:46.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1395e-05 2023-10-09T10:54:46.696 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.696 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.696 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.696 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241362+0000", 2023-10-09T10:54:46.696 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7759999999999997e-06 2023-10-09T10:54:46.696 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.696 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.696 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.696 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241406+0000", 2023-10-09T10:54:46.696 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3189999999999998e-05 2023-10-09T10:54:46.696 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.696 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.696 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.697 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.241413+0000", 2023-10-09T10:54:46.697 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5800000000000003e-06 2023-10-09T10:54:46.697 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.697 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.697 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.697 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103930+0000", 2023-10-09T10:54:46.697 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.86251731399999998 2023-10-09T10:54:46.697 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.697 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.697 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.697 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103938+0000", 2023-10-09T10:54:46.697 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7349999999999996e-06 2023-10-09T10:54:46.698 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.698 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.698 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.698 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183741+0000", 2023-10-09T10:54:46.698 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0798032179999999 2023-10-09T10:54:46.698 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.698 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.698 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.698 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.183865+0000", 2023-10-09T10:54:46.698 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012327200000000001 2023-10-09T10:54:46.698 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.698 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.699 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:54:46.699 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.184011+0000", 2023-10-09T10:54:46.699 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014617499999999999 2023-10-09T10:54:46.699 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.699 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.699 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.699 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186989+0000", 2023-10-09T10:54:46.699 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029777340000000001 2023-10-09T10:54:46.699 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.699 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.699 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.699 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187029+0000", 2023-10-09T10:54:46.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0513000000000003e-05 2023-10-09T10:54:46.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199414+0000", 2023-10-09T10:54:46.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012385022000000001 2023-10-09T10:54:46.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:54:46.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.199472+0000", 2023-10-09T10:54:46.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7998999999999998e-05 2023-10-09T10:54:46.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.701 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.701 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200317+0000", 2023-10-09T10:54:46.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000845245 2023-10-09T10:54:46.701 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.701 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.701 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.701 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42717 6.10 6:0e29612e:::benchmark_data_smithi125_244951_object42716:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:54:46.701 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:54:46.701 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 591.32096316599996, 2023-10-09T10:54:46.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8935023019999999, 2023-10-09T10:54:46.702 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.702 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:46.702 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.702 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:54:46.702 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:46.702 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42717 2023-10-09T10:54:46.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.702 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.702 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:54:46.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307451+0000", 2023-10-09T10:54:46.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.703 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307449+0000", 2023-10-09T10:54:46.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:46.703 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307473+0000", 2023-10-09T10:54:46.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3349e-05 2023-10-09T10:54:46.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307490+0000", 2023-10-09T10:54:46.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7133e-05 2023-10-09T10:54:46.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307496+0000", 2023-10-09T10:54:46.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7349999999999999e-06 2023-10-09T10:54:46.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307561+0000", 2023-10-09T10:54:46.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4226999999999993e-05 2023-10-09T10:54:46.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307575+0000", 2023-10-09T10:54:46.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4504e-05 2023-10-09T10:54:46.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103963+0000", 2023-10-09T10:54:46.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.79638768500000001 2023-10-09T10:54:46.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.706 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103971+0000", 2023-10-09T10:54:46.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9629999999999998e-06 2023-10-09T10:54:46.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.706 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.185776+0000", 2023-10-09T10:54:46.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0818051479999999 2023-10-09T10:54:46.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.185949+0000", 2023-10-09T10:54:46.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000172659 2023-10-09T10:54:46.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.707 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.707 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:54:46.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.186122+0000", 2023-10-09T10:54:46.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017296799999999999 2023-10-09T10:54:46.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.707 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.707 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189418+0000", 2023-10-09T10:54:46.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032959679999999998 2023-10-09T10:54:46.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.708 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189457+0000", 2023-10-09T10:54:46.708 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9669000000000003e-05 2023-10-09T10:54:46.708 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.708 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200372+0000", 2023-10-09T10:54:46.708 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010914633999999999 2023-10-09T10:54:46.708 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:54:46.709 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200421+0000", 2023-10-09T10:54:46.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9047e-05 2023-10-09T10:54:46.709 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.709 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.709 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.709 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.200953+0000", 2023-10-09T10:54:46.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00053212500000000005 2023-10-09T10:54:46.709 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.709 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.709 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.709 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.709 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.709 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44570.0:42722 6.10 6:0ee7a776:::benchmark_data_smithi125_244951_object42721:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T10:54:46.710 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:54:46.710 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 591.32090495800003, 2023-10-09T10:54:46.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8972440939999999, 2023-10-09T10:54:46.710 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.710 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:46.710 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.710 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44570", 2023-10-09T10:54:46.710 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:54:46.710 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 42722 2023-10-09T10:54:46.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.710 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:54:46.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307509+0000", 2023-10-09T10:54:46.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307508+0000", 2023-10-09T10:54:46.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:54:46.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307529+0000", 2023-10-09T10:54:46.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0866999999999999e-05 2023-10-09T10:54:46.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307539+0000", 2023-10-09T10:54:46.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2690000000000005e-06 2023-10-09T10:54:46.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307544+0000", 2023-10-09T10:54:46.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7930000000000003e-06 2023-10-09T10:54:46.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307599+0000", 2023-10-09T10:54:46.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4876000000000001e-05 2023-10-09T10:54:46.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:55.307607+0000", 2023-10-09T10:54:46.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8760000000000005e-06 2023-10-09T10:54:46.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.103995+0000", 2023-10-09T10:54:46.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.79638814099999999 2023-10-09T10:54:46.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:56.104003+0000", 2023-10-09T10:54:46.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7819999999999994e-06 2023-10-09T10:54:46.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187058+0000", 2023-10-09T10:54:46.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.083054712 2023-10-09T10:54:46.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187234+0000", 2023-10-09T10:54:46.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017615799999999999 2023-10-09T10:54:46.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:54:46.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.187416+0000", 2023-10-09T10:54:46.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018243 2023-10-09T10:54:46.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189466+0000", 2023-10-09T10:54:46.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020493210000000002 2023-10-09T10:54:46.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.189504+0000", 2023-10-09T10:54:46.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8287000000000001e-05 2023-10-09T10:54:46.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.201001+0000", 2023-10-09T10:54:46.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011496632999999999 2023-10-09T10:54:46.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:54:46.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.201044+0000", 2023-10-09T10:54:46.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3078000000000003e-05 2023-10-09T10:54:46.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:44:57.204753+0000", 2023-10-09T10:54:46.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037093209999999998 2023-10-09T10:54:46.717 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.717 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.717 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.718 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152582 7.4 7:222afaba:::benchmark_data_smithi125_295816_object61445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:54:46.718 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.340929+0000", 2023-10-09T10:54:46.718 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 157.28748507, 2023-10-09T10:54:46.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8140581390000001, 2023-10-09T10:54:46.718 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.718 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:54:46.718 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.718 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:54:46.718 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:54:46.718 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152582 2023-10-09T10:54:46.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.719 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340929+0000", 2023-10-09T10:54:46.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340929+0000", 2023-10-09T10:54:46.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340928+0000", 2023-10-09T10:54:46.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:54:46.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340931+0000", 2023-10-09T10:54:46.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1329999999999999e-06 2023-10-09T10:54:46.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340948+0000", 2023-10-09T10:54:46.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6178999999999999e-05 2023-10-09T10:54:46.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340956+0000", 2023-10-09T10:54:46.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9980000000000003e-06 2023-10-09T10:54:46.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.341000+0000", 2023-10-09T10:54:46.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4508000000000001e-05 2023-10-09T10:54:46.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.341113+0000", 2023-10-09T10:54:46.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000113288 2023-10-09T10:54:46.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:54:46.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.341196+0000", 2023-10-09T10:54:46.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2163000000000006e-05 2023-10-09T10:54:46.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.344533+0000", 2023-10-09T10:54:46.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0033370129999999998 2023-10-09T10:54:46.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.346664+0000", 2023-10-09T10:54:46.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002131001 2023-10-09T10:54:46.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375125+0000", 2023-10-09T10:54:46.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.028460961999999999 2023-10-09T10:54:46.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375138+0000", 2023-10-09T10:54:46.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3223e-05 2023-10-09T10:54:46.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366624+0000", 2023-10-09T10:54:46.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99148582100000004 2023-10-09T10:54:46.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366632+0000", 2023-10-09T10:54:46.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5620000000000004e-06 2023-10-09T10:54:46.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.154916+0000", 2023-10-09T10:54:46.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.788283748 2023-10-09T10:54:46.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.154987+0000", 2023-10-09T10:54:46.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1076999999999997e-05 2023-10-09T10:54:46.725 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.725 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.726 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.726 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152588 7.9 7:96c4bba8:::benchmark_data_smithi125_295816_object61451:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:54:46.726 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.348147+0000", 2023-10-09T10:54:46.726 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 157.28026719600001, 2023-10-09T10:54:46.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8297705579999999, 2023-10-09T10:54:46.726 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.726 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:54:46.726 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.726 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:54:46.727 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:54:46.727 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152588 2023-10-09T10:54:46.727 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.727 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.727 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.727 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.727 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348147+0000", 2023-10-09T10:54:46.727 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.727 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.727 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.727 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.727 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348147+0000", 2023-10-09T10:54:46.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348146+0000", 2023-10-09T10:54:46.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:54:46.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348151+0000", 2023-10-09T10:54:46.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1150000000000002e-06 2023-10-09T10:54:46.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348158+0000", 2023-10-09T10:54:46.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6850000000000001e-06 2023-10-09T10:54:46.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348163+0000", 2023-10-09T10:54:46.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5979999999999999e-06 2023-10-09T10:54:46.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348227+0000", 2023-10-09T10:54:46.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3596000000000006e-05 2023-10-09T10:54:46.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348357+0000", 2023-10-09T10:54:46.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012999100000000001 2023-10-09T10:54:46.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:54:46.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348485+0000", 2023-10-09T10:54:46.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012842499999999999 2023-10-09T10:54:46.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350944+0000", 2023-10-09T10:54:46.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024594769999999998 2023-10-09T10:54:46.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.353449+0000", 2023-10-09T10:54:46.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025044780000000001 2023-10-09T10:54:46.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.376073+0000", 2023-10-09T10:54:46.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.022624136 2023-10-09T10:54:46.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.376084+0000", 2023-10-09T10:54:46.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0803e-05 2023-10-09T10:54:46.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.364523+0000", 2023-10-09T10:54:46.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98843897800000002 2023-10-09T10:54:46.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.364533+0000", 2023-10-09T10:54:46.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0458999999999999e-05 2023-10-09T10:54:46.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.177846+0000", 2023-10-09T10:54:46.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.81331285 2023-10-09T10:54:46.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.177917+0000", 2023-10-09T10:54:46.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1117000000000004e-05 2023-10-09T10:54:46.733 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.734 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.734 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.734 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152590 7.4 7:284d5455:::benchmark_data_smithi125_295816_object61453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:54:46.734 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.350331+0000", 2023-10-09T10:54:46.734 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 157.27808325300001, 2023-10-09T10:54:46.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.805048094, 2023-10-09T10:54:46.734 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.734 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:54:46.734 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.734 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:54:46.735 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:54:46.735 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152590 2023-10-09T10:54:46.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.735 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350331+0000", 2023-10-09T10:54:46.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350331+0000", 2023-10-09T10:54:46.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350329+0000", 2023-10-09T10:54:46.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:46.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350335+0000", 2023-10-09T10:54:46.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4639999999999997e-06 2023-10-09T10:54:46.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350352+0000", 2023-10-09T10:54:46.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7269e-05 2023-10-09T10:54:46.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350361+0000", 2023-10-09T10:54:46.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.388e-06 2023-10-09T10:54:46.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350415+0000", 2023-10-09T10:54:46.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4784000000000003e-05 2023-10-09T10:54:46.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350517+0000", 2023-10-09T10:54:46.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000101724 2023-10-09T10:54:46.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:54:46.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350653+0000", 2023-10-09T10:54:46.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013574099999999999 2023-10-09T10:54:46.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.353798+0000", 2023-10-09T10:54:46.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003145002 2023-10-09T10:54:46.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360490+0000", 2023-10-09T10:54:46.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066923240000000004 2023-10-09T10:54:46.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375171+0000", 2023-10-09T10:54:46.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014681339999999999 2023-10-09T10:54:46.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375182+0000", 2023-10-09T10:54:46.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0129000000000001e-05 2023-10-09T10:54:46.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366657+0000", 2023-10-09T10:54:46.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99147512100000001 2023-10-09T10:54:46.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366664+0000", 2023-10-09T10:54:46.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9369999999999998e-06 2023-10-09T10:54:46.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155313+0000", 2023-10-09T10:54:46.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.788649556 2023-10-09T10:54:46.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155379+0000", 2023-10-09T10:54:46.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5518000000000005e-05 2023-10-09T10:54:46.741 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.741 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.741 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.742 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152591 7.9 7:9f5cec69:::benchmark_data_smithi125_295816_object61454:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:54:46.742 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.354683+0000", 2023-10-09T10:54:46.742 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 157.27373064299999, 2023-10-09T10:54:46.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8235682930000001, 2023-10-09T10:54:46.742 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.742 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:54:46.742 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.742 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:54:46.742 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:54:46.742 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152591 2023-10-09T10:54:46.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.743 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354683+0000", 2023-10-09T10:54:46.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354683+0000", 2023-10-09T10:54:46.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354682+0000", 2023-10-09T10:54:46.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:46.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354688+0000", 2023-10-09T10:54:46.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7810000000000002e-06 2023-10-09T10:54:46.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354699+0000", 2023-10-09T10:54:46.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1556000000000001e-05 2023-10-09T10:54:46.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354707+0000", 2023-10-09T10:54:46.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3799999999999996e-06 2023-10-09T10:54:46.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354752+0000", 2023-10-09T10:54:46.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5756e-05 2023-10-09T10:54:46.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354885+0000", 2023-10-09T10:54:46.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000132397 2023-10-09T10:54:46.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:54:46.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.355009+0000", 2023-10-09T10:54:46.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000123714 2023-10-09T10:54:46.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357282+0000", 2023-10-09T10:54:46.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022739679999999999 2023-10-09T10:54:46.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.376279+0000", 2023-10-09T10:54:46.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018997012000000001 2023-10-09T10:54:46.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.376289+0000", 2023-10-09T10:54:46.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.781e-06 2023-10-09T10:54:46.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.364560+0000", 2023-10-09T10:54:46.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98827064099999995 2023-10-09T10:54:46.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365610+0000", 2023-10-09T10:54:46.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001049822 2023-10-09T10:54:46.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.178188+0000", 2023-10-09T10:54:46.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8125778029999999 2023-10-09T10:54:46.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.178252+0000", 2023-10-09T10:54:46.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4000999999999999e-05 2023-10-09T10:54:46.749 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.749 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.749 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.749 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152593 7.4 7:21dbc744:::benchmark_data_smithi125_295816_object61456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:54:46.749 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.354718+0000", 2023-10-09T10:54:46.749 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 157.27369541100001, 2023-10-09T10:54:46.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8007503599999999, 2023-10-09T10:54:46.750 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.750 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:54:46.750 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.750 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:54:46.750 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:54:46.750 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152593 2023-10-09T10:54:46.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.750 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354718+0000", 2023-10-09T10:54:46.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354718+0000", 2023-10-09T10:54:46.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354718+0000", 2023-10-09T10:54:46.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:54:46.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354721+0000", 2023-10-09T10:54:46.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8200000000000001e-06 2023-10-09T10:54:46.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354728+0000", 2023-10-09T10:54:46.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7309999999999996e-06 2023-10-09T10:54:46.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354734+0000", 2023-10-09T10:54:46.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1009999999999997e-06 2023-10-09T10:54:46.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354780+0000", 2023-10-09T10:54:46.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6464000000000001e-05 2023-10-09T10:54:46.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354913+0000", 2023-10-09T10:54:46.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000132569 2023-10-09T10:54:46.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:54:46.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.355043+0000", 2023-10-09T10:54:46.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013061799999999999 2023-10-09T10:54:46.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.359660+0000", 2023-10-09T10:54:46.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046166929999999998 2023-10-09T10:54:46.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375223+0000", 2023-10-09T10:54:46.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015563420999999999 2023-10-09T10:54:46.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375233+0000", 2023-10-09T10:54:46.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7999999999999993e-06 2023-10-09T10:54:46.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366685+0000", 2023-10-09T10:54:46.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99145126699999997 2023-10-09T10:54:46.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366692+0000", 2023-10-09T10:54:46.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.498e-06 2023-10-09T10:54:46.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155410+0000", 2023-10-09T10:54:46.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.788717651 2023-10-09T10:54:46.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155469+0000", 2023-10-09T10:54:46.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9157000000000003e-05 2023-10-09T10:54:46.757 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.757 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.757 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.757 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152597 7.4 7:23416840:::benchmark_data_smithi125_295816_object61460:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:54:46.757 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.357109+0000", 2023-10-09T10:54:46.757 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 157.271305235, 2023-10-09T10:54:46.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7984508359999998, 2023-10-09T10:54:46.758 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.758 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:54:46.758 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.758 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:54:46.758 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:54:46.758 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152597 2023-10-09T10:54:46.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.758 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357109+0000", 2023-10-09T10:54:46.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357109+0000", 2023-10-09T10:54:46.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357108+0000", 2023-10-09T10:54:46.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:54:46.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357112+0000", 2023-10-09T10:54:46.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.673e-06 2023-10-09T10:54:46.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357123+0000", 2023-10-09T10:54:46.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0567999999999999e-05 2023-10-09T10:54:46.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357130+0000", 2023-10-09T10:54:46.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4599999999999997e-06 2023-10-09T10:54:46.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357336+0000", 2023-10-09T10:54:46.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020526499999999999 2023-10-09T10:54:46.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357462+0000", 2023-10-09T10:54:46.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012642400000000001 2023-10-09T10:54:46.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:54:46.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357581+0000", 2023-10-09T10:54:46.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118647 2023-10-09T10:54:46.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.359832+0000", 2023-10-09T10:54:46.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002251121 2023-10-09T10:54:46.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375269+0000", 2023-10-09T10:54:46.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015436699999999999 2023-10-09T10:54:46.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375278+0000", 2023-10-09T10:54:46.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8199999999999992e-06 2023-10-09T10:54:46.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366716+0000", 2023-10-09T10:54:46.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99143730600000002 2023-10-09T10:54:46.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366725+0000", 2023-10-09T10:54:46.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8759999999999994e-06 2023-10-09T10:54:46.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155501+0000", 2023-10-09T10:54:46.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7887762249999999 2023-10-09T10:54:46.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155559+0000", 2023-10-09T10:54:46.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8628000000000002e-05 2023-10-09T10:54:46.764 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.764 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.764 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.765 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152599 7.4 7:26957a0e:::benchmark_data_smithi125_295816_object61462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:54:46.765 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.360294+0000", 2023-10-09T10:54:46.765 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 157.26811995099999, 2023-10-09T10:54:46.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7970287539999998, 2023-10-09T10:54:46.765 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.765 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:54:46.765 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.765 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:54:46.765 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:54:46.765 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152599 2023-10-09T10:54:46.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.766 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360294+0000", 2023-10-09T10:54:46.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360294+0000", 2023-10-09T10:54:46.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360292+0000", 2023-10-09T10:54:46.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:54:46.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360297+0000", 2023-10-09T10:54:46.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0170000000000004e-06 2023-10-09T10:54:46.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360318+0000", 2023-10-09T10:54:46.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0460999999999999e-05 2023-10-09T10:54:46.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360325+0000", 2023-10-09T10:54:46.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2119999999999997e-06 2023-10-09T10:54:46.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360517+0000", 2023-10-09T10:54:46.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000192487 2023-10-09T10:54:46.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360676+0000", 2023-10-09T10:54:46.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015863499999999999 2023-10-09T10:54:46.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:54:46.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360826+0000", 2023-10-09T10:54:46.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015037199999999999 2023-10-09T10:54:46.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.378926+0000", 2023-10-09T10:54:46.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018099725000000001 2023-10-09T10:54:46.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.378938+0000", 2023-10-09T10:54:46.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2191000000000001e-05 2023-10-09T10:54:46.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366754+0000", 2023-10-09T10:54:46.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98781607800000004 2023-10-09T10:54:46.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366763+0000", 2023-10-09T10:54:46.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6179999999999995e-06 2023-10-09T10:54:46.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.157249+0000", 2023-10-09T10:54:46.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.790486386 2023-10-09T10:54:46.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.157323+0000", 2023-10-09T10:54:46.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3258999999999998e-05 2023-10-09T10:54:46.772 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.772 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.772 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.772 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152606 7.9 7:912bbd31:::benchmark_data_smithi125_295816_object61469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T10:54:46.772 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.394803+0000", 2023-10-09T10:54:46.772 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 157.23361066800001, 2023-10-09T10:54:46.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7960543420000001, 2023-10-09T10:54:46.772 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.772 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:46.772 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.772 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:54:46.772 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:54:46.773 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152606 2023-10-09T10:54:46.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.773 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394803+0000", 2023-10-09T10:54:46.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394803+0000", 2023-10-09T10:54:46.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394802+0000", 2023-10-09T10:54:46.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:46.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394808+0000", 2023-10-09T10:54:46.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5220000000000004e-06 2023-10-09T10:54:46.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394821+0000", 2023-10-09T10:54:46.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2325e-05 2023-10-09T10:54:46.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394829+0000", 2023-10-09T10:54:46.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3820000000000004e-06 2023-10-09T10:54:46.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394972+0000", 2023-10-09T10:54:46.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000142788 2023-10-09T10:54:46.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394982+0000", 2023-10-09T10:54:46.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0336000000000001e-05 2023-10-09T10:54:46.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365844+0000", 2023-10-09T10:54:46.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97086172500000001 2023-10-09T10:54:46.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365853+0000", 2023-10-09T10:54:46.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5230000000000001e-06 2023-10-09T10:54:46.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.178273+0000", 2023-10-09T10:54:46.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8124192300000002 2023-10-09T10:54:46.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.178376+0000", 2023-10-09T10:54:46.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000102841 2023-10-09T10:54:46.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T10:54:46.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.178485+0000", 2023-10-09T10:54:46.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000109516 2023-10-09T10:54:46.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.183468+0000", 2023-10-09T10:54:46.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049829920000000003 2023-10-09T10:54:46.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.186132+0000", 2023-10-09T10:54:46.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002663538 2023-10-09T10:54:46.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.186288+0000", 2023-10-09T10:54:46.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015653499999999999 2023-10-09T10:54:46.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.190512+0000", 2023-10-09T10:54:46.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042239850000000004 2023-10-09T10:54:46.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:54:46.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.190562+0000", 2023-10-09T10:54:46.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0169e-05 2023-10-09T10:54:46.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.190858+0000", 2023-10-09T10:54:46.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00029527200000000001 2023-10-09T10:54:46.781 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.781 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.781 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.781 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152607 7.9 7:9c2ec87f:::benchmark_data_smithi125_295816_object61470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T10:54:46.781 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.397825+0000", 2023-10-09T10:54:46.781 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 157.23058871999999, 2023-10-09T10:54:46.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.811002282, 2023-10-09T10:54:46.781 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.781 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:54:46.781 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.782 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:54:46.782 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:54:46.782 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152607 2023-10-09T10:54:46.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.782 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.782 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397825+0000", 2023-10-09T10:54:46.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397825+0000", 2023-10-09T10:54:46.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397824+0000", 2023-10-09T10:54:46.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:54:46.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397830+0000", 2023-10-09T10:54:46.784 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:54:46.784 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:54:46.785 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:46 smithi155 ceph-mon[184854]: failed to encode map e293 with expected crc 2023-10-09T10:54:46.785 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:46 smithi155 ceph-mon[184854]: pgmap v700: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 unknown, 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 111 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 11 MiB/s wr, 179 op/s; 32843/295728 objects degraded (11.106%); 18795/295728 objects misplaced (6.356%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:54:46.785 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:46.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9199999999999998e-06 2023-10-09T10:54:46.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397844+0000", 2023-10-09T10:54:46.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3648999999999999e-05 2023-10-09T10:54:46.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397852+0000", 2023-10-09T10:54:46.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9009999999999994e-06 2023-10-09T10:54:46.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397895+0000", 2023-10-09T10:54:46.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3269999999999997e-05 2023-10-09T10:54:46.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397907+0000", 2023-10-09T10:54:46.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1507e-05 2023-10-09T10:54:46.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365878+0000", 2023-10-09T10:54:46.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96797074800000005 2023-10-09T10:54:46.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365886+0000", 2023-10-09T10:54:46.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0909999999999993e-06 2023-10-09T10:54:46.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.180824+0000", 2023-10-09T10:54:46.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8149384070000001 2023-10-09T10:54:46.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.180944+0000", 2023-10-09T10:54:46.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120256 2023-10-09T10:54:46.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T10:54:46.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.181120+0000", 2023-10-09T10:54:46.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017529399999999999 2023-10-09T10:54:46.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.188514+0000", 2023-10-09T10:54:46.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0073948709999999999 2023-10-09T10:54:46.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.190897+0000", 2023-10-09T10:54:46.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023828120000000002 2023-10-09T10:54:46.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.192009+0000", 2023-10-09T10:54:46.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011115719999999999 2023-10-09T10:54:46.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.208460+0000", 2023-10-09T10:54:46.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.016451364999999999 2023-10-09T10:54:46.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:54:46.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.208512+0000", 2023-10-09T10:54:46.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2067999999999997e-05 2023-10-09T10:54:46.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.208827+0000", 2023-10-09T10:54:46.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00031514599999999998 2023-10-09T10:54:46.791 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.792 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.792 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.792 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:83945 8.e 8:7f009390:::benchmark_data_smithi125_329074_object83944:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2023-10-09T10:54:46.792 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:17.076397+0000", 2023-10-09T10:54:46.792 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 29.552016569999999, 2023-10-09T10:54:46.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2623378889999999, 2023-10-09T10:54:46.792 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:54:46.792 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:54:46.792 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:54:46.792 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:54:46.792 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:54:46.793 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 83945 2023-10-09T10:54:46.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.793 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:54:46.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:54:46.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076397+0000", 2023-10-09T10:54:46.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:54:46.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076397+0000", 2023-10-09T10:54:46.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:54:46.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:54:46.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076396+0000", 2023-10-09T10:54:46.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:54:46.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:54:46.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076429+0000", 2023-10-09T10:54:46.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2443000000000001e-05 2023-10-09T10:54:46.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:54:46.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076441+0000", 2023-10-09T10:54:46.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2639999999999999e-05 2023-10-09T10:54:46.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:54:46.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076451+0000", 2023-10-09T10:54:46.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1789999999999997e-06 2023-10-09T10:54:46.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076490+0000", 2023-10-09T10:54:46.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9607000000000002e-05 2023-10-09T10:54:46.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:54:46.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076606+0000", 2023-10-09T10:54:46.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011632 2023-10-09T10:54:46.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,7", 2023-10-09T10:54:46.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076713+0000", 2023-10-09T10:54:46.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000106542 2023-10-09T10:54:46.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.798 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.798 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:54:46.798 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.091575+0000", 2023-10-09T10:54:46.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014862176 2023-10-09T10:54:46.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.798 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.798 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:54:46.798 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.095602+0000", 2023-10-09T10:54:46.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040271650000000001 2023-10-09T10:54:46.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.100859+0000", 2023-10-09T10:54:46.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0052565240000000003 2023-10-09T10:54:46.799 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.100873+0000", 2023-10-09T10:54:46.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4364e-05 2023-10-09T10:54:46.799 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.869642+0000", 2023-10-09T10:54:46.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.76876907000000005 2023-10-09T10:54:46.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:54:46.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.869658+0000", 2023-10-09T10:54:46.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5216e-05 2023-10-09T10:54:46.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:54:46.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.338629+0000", 2023-10-09T10:54:46.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.468971483 2023-10-09T10:54:46.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:54:46.801 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:54:46.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:54:46.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.338735+0000", 2023-10-09T10:54:46.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000106165 2023-10-09T10:54:46.801 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.801 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.801 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.801 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:54:46.801 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:54:46.801 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:54:46.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:46 smithi125 ceph-mon[200197]: failed to encode map e293 with expected crc 2023-10-09T10:54:46.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:46 smithi125 ceph-mon[200197]: pgmap v700: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 unknown, 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 111 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 11 MiB/s wr, 179 op/s; 32843/295728 objects degraded (11.106%); 18795/295728 objects misplaced (6.356%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T10:54:46.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:46.900 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:46.972 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 88 16 96286 96270 68.3409 28.375 0.0190807 0.014624 2023-10-09T10:54:47.007 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:54:47.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:47.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:54:47.066 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:54:47.109 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:46 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T10:54:46.834+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e293 definitely_dead 0 2023-10-09T10:54:47.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:47.150 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:47.172 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:47.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:47.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:47.256 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:47.276 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:47.329 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:47.436 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:47.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:47.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:47.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:47.582 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:47.642 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.1. 2023-10-09T10:54:47.696 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3212971162' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T10:54:47.696 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:47 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T10:54:47.696 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3212971162' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T10:54:47.696 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:47 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T10:54:47.763 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:54:47.864 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:47.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:47.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:47.968 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:47.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:47 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3212971162' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T10:54:47.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:47 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T10:54:47.974 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 89 16 96641 96625 67.8211 22.1875 0.00868775 0.0146714 2023-10-09T10:54:47.996 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:48.016 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:48.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:48.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:48.121 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:48.148 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:48.167 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-10-09T10:54:48.168 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:48.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:48.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:48.272 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:48.301 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:48.328 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:48.472 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:48.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:48.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:48.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:48.606 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:48.680 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:48.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:48.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:48.721 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:48.786 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:48.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[195507]: pgmap v701: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 5 active+recovery_wait+degraded+remapped, 111 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 38021/287373 objects degraded (13.231%); 16249/287373 objects misplaced (5.654%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T10:54:48.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[195507]: failed to encode map e294 with expected crc 2023-10-09T10:54:48.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T10:54:48.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[195507]: osdmap e294: 8 total, 8 up, 8 in 2023-10-09T10:54:48.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 38021/287373 objects degraded (13.231%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:48.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[195507]: failed to encode map e295 with expected crc 2023-10-09T10:54:48.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[195507]: failed to encode map e295 with expected crc 2023-10-09T10:54:48.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[195507]: failed to encode map e295 with expected crc 2023-10-09T10:54:48.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[195507]: osdmap e295: 8 total, 8 up, 8 in 2023-10-09T10:54:48.817 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[200197]: pgmap v701: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 5 active+recovery_wait+degraded+remapped, 111 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 38021/287373 objects degraded (13.231%); 16249/287373 objects misplaced (5.654%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T10:54:48.817 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[200197]: failed to encode map e294 with expected crc 2023-10-09T10:54:48.817 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T10:54:48.817 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[200197]: osdmap e294: 8 total, 8 up, 8 in 2023-10-09T10:54:48.817 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 38021/287373 objects degraded (13.231%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:48.817 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[200197]: failed to encode map e295 with expected crc 2023-10-09T10:54:48.817 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[200197]: failed to encode map e295 with expected crc 2023-10-09T10:54:48.817 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[200197]: failed to encode map e295 with expected crc 2023-10-09T10:54:48.817 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:48 smithi125 ceph-mon[200197]: osdmap e295: 8 total, 8 up, 8 in 2023-10-09T10:54:48.819 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:48.939 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:48.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:48.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:48.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:48 smithi155 ceph-mon[184854]: pgmap v701: 129 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 5 active+recovery_wait+degraded+remapped, 111 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 38021/287373 objects degraded (13.231%); 16249/287373 objects misplaced (5.654%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T10:54:48.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:48 smithi155 ceph-mon[184854]: failed to encode map e294 with expected crc 2023-10-09T10:54:48.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:48 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T10:54:48.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:48 smithi155 ceph-mon[184854]: osdmap e294: 8 total, 8 up, 8 in 2023-10-09T10:54:48.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:48 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 38021/287373 objects degraded (13.231%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:48.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:48 smithi155 ceph-mon[184854]: failed to encode map e295 with expected crc 2023-10-09T10:54:48.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:48 smithi155 ceph-mon[184854]: failed to encode map e295 with expected crc 2023-10-09T10:54:48.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:48 smithi155 ceph-mon[184854]: failed to encode map e295 with expected crc 2023-10-09T10:54:48.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:48 smithi155 ceph-mon[184854]: osdmap e295: 8 total, 8 up, 8 in 2023-10-09T10:54:48.974 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 90 16 96651 96635 67.0747 0.625 0.00293609 0.0146747 2023-10-09T10:54:49.045 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:49.074 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:49.094 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:49.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:49.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:49.198 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:49.232 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:49.255 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:49.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:49.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:49.359 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:49.386 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:49.405 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:49.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:49.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:49.509 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:49.536 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:49.555 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:49.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:49.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:49.659 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:49.691 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:49.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:49 smithi155 ceph-mon[184854]: failed to encode map e295 with expected crc 2023-10-09T10:54:49.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:49 smithi155 ceph-mon[184854]: osdmap e296: 8 total, 8 up, 8 in 2023-10-09T10:54:49.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:49 smithi155 ceph-mon[184854]: failed to encode map e296 with expected crc 2023-10-09T10:54:49.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:49 smithi155 ceph-mon[184854]: failed to encode map e296 with expected crc 2023-10-09T10:54:49.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:49 smithi155 ceph-mon[184854]: failed to encode map e296 with expected crc 2023-10-09T10:54:49.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:49 smithi155 ceph-mon[184854]: 8.a continuing backfill to osd.2 from (271'1372,293'5978] MIN to 293'5978 2023-10-09T10:54:49.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:49 smithi155 ceph-mon[184854]: 8.f continuing backfill to osd.2 from (271'1471,293'6085] MIN to 293'6085 2023-10-09T10:54:49.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:49 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2221367957' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:54:49.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:49 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:54:49.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[195507]: failed to encode map e295 with expected crc 2023-10-09T10:54:49.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[195507]: osdmap e296: 8 total, 8 up, 8 in 2023-10-09T10:54:49.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[195507]: failed to encode map e296 with expected crc 2023-10-09T10:54:49.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[195507]: failed to encode map e296 with expected crc 2023-10-09T10:54:49.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[195507]: failed to encode map e296 with expected crc 2023-10-09T10:54:49.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[195507]: 8.a continuing backfill to osd.2 from (271'1372,293'5978] MIN to 293'5978 2023-10-09T10:54:49.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[195507]: 8.f continuing backfill to osd.2 from (271'1471,293'6085] MIN to 293'6085 2023-10-09T10:54:49.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2221367957' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:54:49.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:54:49.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[200197]: failed to encode map e295 with expected crc 2023-10-09T10:54:49.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[200197]: osdmap e296: 8 total, 8 up, 8 in 2023-10-09T10:54:49.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[200197]: failed to encode map e296 with expected crc 2023-10-09T10:54:49.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[200197]: failed to encode map e296 with expected crc 2023-10-09T10:54:49.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[200197]: failed to encode map e296 with expected crc 2023-10-09T10:54:49.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[200197]: 8.a continuing backfill to osd.2 from (271'1372,293'5978] MIN to 293'5978 2023-10-09T10:54:49.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[200197]: 8.f continuing backfill to osd.2 from (271'1471,293'6085] MIN to 293'6085 2023-10-09T10:54:49.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2221367957' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:54:49.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:49 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:54:49.915 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:54:49.948 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:54:49.948 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:54:49.948 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 0 2023-10-09T10:54:49.948 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:54:49.974 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total time run: 90.2665 2023-10-09T10:54:49.975 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total writes made: 96651 2023-10-09T10:54:49.975 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Write size: 65536 2023-10-09T10:54:49.975 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object size: 65536 2023-10-09T10:54:49.975 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Bandwidth (MB/sec): 66.9206 2023-10-09T10:54:49.975 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Bandwidth: 47.6502 2023-10-09T10:54:49.975 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max bandwidth (MB/sec): 201 2023-10-09T10:54:49.975 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min bandwidth (MB/sec): 0 2023-10-09T10:54:49.975 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average IOPS: 1070 2023-10-09T10:54:49.975 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev IOPS: 762.42 2023-10-09T10:54:49.975 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max IOPS: 3216 2023-10-09T10:54:49.975 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min IOPS: 0 2023-10-09T10:54:49.975 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average Latency(s): 0.0149365 2023-10-09T10:54:49.976 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Latency(s): 0.0768352 2023-10-09T10:54:49.976 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max latency(s): 6.38455 2023-10-09T10:54:49.976 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min latency(s): 0.00119012 2023-10-09T10:54:49.993 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T10:54:50.167 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:50.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:50.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:50.275 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:50.305 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:50.328 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:50.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:50.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:50.431 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:54:50.466 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:54:50.467 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:50.590 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:54:50.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:50.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:54:50.695 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:50.723 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[195507]: failed to encode map e296 with expected crc 2023-10-09T10:54:50.723 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[195507]: pgmap v705: 129 pgs: 5 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 5 active+recovery_wait+degraded+remapped, 108 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 32817/288303 objects degraded (11.383%); 16249/288303 objects misplaced (5.636%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T10:54:50.723 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:54:50.723 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:54:50.723 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:54:50.724 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[195507]: osdmap e297: 8 total, 8 up, 8 in 2023-10-09T10:54:50.724 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[195507]: failed to encode map e297 with expected crc 2023-10-09T10:54:50.724 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[195507]: failed to encode map e297 with expected crc 2023-10-09T10:54:50.724 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[195507]: failed to encode map e297 with expected crc 2023-10-09T10:54:50.724 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[195507]: failed to encode map e297 with expected crc 2023-10-09T10:54:50.724 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[200197]: failed to encode map e296 with expected crc 2023-10-09T10:54:50.724 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[200197]: pgmap v705: 129 pgs: 5 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 5 active+recovery_wait+degraded+remapped, 108 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 32817/288303 objects degraded (11.383%); 16249/288303 objects misplaced (5.636%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T10:54:50.724 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:54:50.724 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:54:50.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:54:50.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[200197]: osdmap e297: 8 total, 8 up, 8 in 2023-10-09T10:54:50.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[200197]: failed to encode map e297 with expected crc 2023-10-09T10:54:50.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[200197]: failed to encode map e297 with expected crc 2023-10-09T10:54:50.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[200197]: failed to encode map e297 with expected crc 2023-10-09T10:54:50.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:50 smithi125 ceph-mon[200197]: failed to encode map e297 with expected crc 2023-10-09T10:54:50.726 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:50.899 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:50.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:50.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:50.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:50 smithi155 ceph-mon[184854]: failed to encode map e296 with expected crc 2023-10-09T10:54:50.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:50 smithi155 ceph-mon[184854]: pgmap v705: 129 pgs: 5 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 5 active+recovery_wait+degraded+remapped, 108 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 32817/288303 objects degraded (11.383%); 16249/288303 objects misplaced (5.636%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T10:54:50.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:50 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:54:50.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:50 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:54:50.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:50 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:54:50.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:50 smithi155 ceph-mon[184854]: osdmap e297: 8 total, 8 up, 8 in 2023-10-09T10:54:50.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:50 smithi155 ceph-mon[184854]: failed to encode map e297 with expected crc 2023-10-09T10:54:50.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:50 smithi155 ceph-mon[184854]: failed to encode map e297 with expected crc 2023-10-09T10:54:50.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:50 smithi155 ceph-mon[184854]: failed to encode map e297 with expected crc 2023-10-09T10:54:50.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:50 smithi155 ceph-mon[184854]: failed to encode map e297 with expected crc 2023-10-09T10:54:51.008 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:51.015 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:54:51.024 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:51.040 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:51.061 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:54:51.062 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:51.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:51.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:51.106 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:54:51.152 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:54:51.166 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:51.196 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:51.198 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:54:51.216 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:51.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:51.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:51.247 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:54:51.286 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:54:51.321 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:51.352 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:54:51.391 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:54:51.392 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:54:51.392 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:54:51.392 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:54:51.392 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:54:51.392 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:54:51.392 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:54:51.392 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:54:51.402 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:51.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:54:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:54:51] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.43.0" 2023-10-09T10:54:51.637 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:51.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:51.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:51.744 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:51.827 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:51.854 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:51 smithi125 ceph-mon[195507]: failed to encode map e297 with expected crc 2023-10-09T10:54:51.854 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:51 smithi125 ceph-mon[195507]: failed to encode map e298 with expected crc 2023-10-09T10:54:51.854 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:51 smithi125 ceph-mon[195507]: failed to encode map e298 with expected crc 2023-10-09T10:54:51.854 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:51 smithi125 ceph-mon[195507]: osdmap e298: 8 total, 8 up, 8 in 2023-10-09T10:54:51.854 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:51 smithi125 ceph-mon[195507]: failed to encode map e298 with expected crc 2023-10-09T10:54:51.854 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3550057593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:51.855 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:51 smithi125 ceph-mon[200197]: failed to encode map e297 with expected crc 2023-10-09T10:54:51.855 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:51 smithi125 ceph-mon[200197]: failed to encode map e298 with expected crc 2023-10-09T10:54:51.855 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:51 smithi125 ceph-mon[200197]: failed to encode map e298 with expected crc 2023-10-09T10:54:51.855 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:51 smithi125 ceph-mon[200197]: osdmap e298: 8 total, 8 up, 8 in 2023-10-09T10:54:51.855 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:51 smithi125 ceph-mon[200197]: failed to encode map e298 with expected crc 2023-10-09T10:54:51.855 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3550057593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:51.873 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:51.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:51.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:51.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:51 smithi155 ceph-mon[184854]: failed to encode map e297 with expected crc 2023-10-09T10:54:51.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:51 smithi155 ceph-mon[184854]: failed to encode map e298 with expected crc 2023-10-09T10:54:51.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:51 smithi155 ceph-mon[184854]: failed to encode map e298 with expected crc 2023-10-09T10:54:51.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:51 smithi155 ceph-mon[184854]: osdmap e298: 8 total, 8 up, 8 in 2023-10-09T10:54:51.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:51 smithi155 ceph-mon[184854]: failed to encode map e298 with expected crc 2023-10-09T10:54:51.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3550057593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:51.981 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:52.024 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:52.219 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:52.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:52.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:52.328 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:52.359 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:52.380 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:52.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:52.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:52.483 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:52.513 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:52.538 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:52.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:52.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:52.643 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:52.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:52 smithi125 ceph-mon[195507]: failed to encode map e298 with expected crc 2023-10-09T10:54:52.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:52 smithi125 ceph-mon[195507]: pgmap v708: 129 pgs: 5 remapped+peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 108 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 13 MiB/s wr, 249 op/s; 16350/289239 objects degraded (5.653%); 23199/289239 objects misplaced (8.021%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T10:54:52.706 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:52 smithi125 ceph-mon[200197]: failed to encode map e298 with expected crc 2023-10-09T10:54:52.706 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:52 smithi125 ceph-mon[200197]: pgmap v708: 129 pgs: 5 remapped+peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 108 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 13 MiB/s wr, 249 op/s; 16350/289239 objects degraded (5.653%); 23199/289239 objects misplaced (8.021%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T10:54:52.707 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:52.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:52 smithi155 ceph-mon[184854]: failed to encode map e298 with expected crc 2023-10-09T10:54:52.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:52 smithi155 ceph-mon[184854]: pgmap v708: 129 pgs: 5 remapped+peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 108 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 13 MiB/s wr, 249 op/s; 16350/289239 objects degraded (5.653%); 23199/289239 objects misplaced (8.021%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T10:54:52.726 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:54:52.726 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:54:52.740 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:52.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:52.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:52.791 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:54:52.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:52.878 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:52.900 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:52.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:52.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:53.003 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:53.042 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:53.072 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:53.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:53.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:53.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:54:53.167 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 7, 2, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:54:53.167 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:54:53.168 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_2 2023-10-09T10:54:53.168 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:54:53.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:53.211 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:53.231 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:53.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:53.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:53.303 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:53.335 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:53.367 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:53.389 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:53.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:53.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:53.495 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:53.526 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:53.547 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:53.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:53.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:53.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:53.694 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:53.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:53 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 16350/289239 objects degraded (5.653%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:53.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:53 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 16350/289239 objects degraded (5.653%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:53.889 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:53.894 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:53.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:53.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:53.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:54:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:53 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 16350/289239 objects degraded (5.653%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:53.999 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:54.079 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:54.366 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:54.480 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:54.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:54.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:54.587 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:54.620 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:54.643 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:54.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:54.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:54.746 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:54.749 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:54 smithi125 ceph-mon[195507]: pgmap v709: 129 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 186 B/s rd, 10 MiB/s wr, 320 op/s; 17602/287412 objects degraded (6.124%); 22898/287412 objects misplaced (7.967%); 11 MiB/s, 180 objects/s recovering 2023-10-09T10:54:54.749 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:54 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:54:54.750 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:54 smithi125 ceph-mon[200197]: pgmap v709: 129 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 186 B/s rd, 10 MiB/s wr, 320 op/s; 17602/287412 objects degraded (6.124%); 22898/287412 objects misplaced (7.967%); 11 MiB/s, 180 objects/s recovering 2023-10-09T10:54:54.750 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:54 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:54:54.782 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:54.807 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:54.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:54.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:54.911 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:54.942 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:54 smithi155 ceph-mon[184854]: pgmap v709: 129 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 186 B/s rd, 10 MiB/s wr, 320 op/s; 17602/287412 objects degraded (6.124%); 22898/287412 objects misplaced (7.967%); 11 MiB/s, 180 objects/s recovering 2023-10-09T10:54:54.942 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:54 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:54:54.944 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:54.971 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:54.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:54.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:55.075 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:55.106 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:55.128 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:55.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:55.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:55.233 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:55.268 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:55.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:55.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:55.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:55.398 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:55.428 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:55.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:55.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:55.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:55.559 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:55.596 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:55.628 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:55.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:55.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:55.663 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:54:55.686 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:55 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1969358554' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:55.687 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:55 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:55.687 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:55 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1969358554' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:55.687 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:55 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:55.718 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:55.732 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:55.762 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:55.789 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:55.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:55.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:55.892 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:55.920 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:55 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1969358554' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:55.921 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:55 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:54:55.923 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:55.942 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:55.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:55.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:56.047 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:54:56.077 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:56.099 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:54:56.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:56.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:54:56.163 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:54:56.163 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:54:56.203 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:56.233 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:56.259 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:56.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:56.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:56.362 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:56.398 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:56.474 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:56.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:56.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:56.579 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:56.609 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:56.704 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:56.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:56.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:56.743 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:54:56.743 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[195507]: pgmap v710: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 157 B/s rd, 5.5 MiB/s wr, 403 op/s; 17602/283905 objects degraded (6.200%); 22017/283905 objects misplaced (7.755%); 14 MiB/s, 227 objects/s recovering 2023-10-09T10:54:56.743 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:54:56.743 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[195507]: failed to encode map e299 with expected crc 2023-10-09T10:54:56.743 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:54:56.743 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[195507]: osdmap e299: 8 total, 8 up, 8 in 2023-10-09T10:54:56.743 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[195507]: failed to encode map e299 with expected crc 2023-10-09T10:54:56.744 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[195507]: failed to encode map e299 with expected crc 2023-10-09T10:54:56.744 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[195507]: failed to encode map e299 with expected crc 2023-10-09T10:54:56.744 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:56.744 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[200197]: pgmap v710: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 157 B/s rd, 5.5 MiB/s wr, 403 op/s; 17602/283905 objects degraded (6.200%); 22017/283905 objects misplaced (7.755%); 14 MiB/s, 227 objects/s recovering 2023-10-09T10:54:56.744 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:54:56.744 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[200197]: failed to encode map e299 with expected crc 2023-10-09T10:54:56.744 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:54:56.744 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[200197]: osdmap e299: 8 total, 8 up, 8 in 2023-10-09T10:54:56.744 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[200197]: failed to encode map e299 with expected crc 2023-10-09T10:54:56.744 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[200197]: failed to encode map e299 with expected crc 2023-10-09T10:54:56.745 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[200197]: failed to encode map e299 with expected crc 2023-10-09T10:54:56.745 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:56 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:56.815 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:56.919 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:56.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:56 smithi155 ceph-mon[184854]: pgmap v710: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 157 B/s rd, 5.5 MiB/s wr, 403 op/s; 17602/283905 objects degraded (6.200%); 22017/283905 objects misplaced (7.755%); 14 MiB/s, 227 objects/s recovering 2023-10-09T10:54:56.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:56 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:54:56.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:56 smithi155 ceph-mon[184854]: failed to encode map e299 with expected crc 2023-10-09T10:54:56.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:56 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:54:56.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:56 smithi155 ceph-mon[184854]: osdmap e299: 8 total, 8 up, 8 in 2023-10-09T10:54:56.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:56 smithi155 ceph-mon[184854]: failed to encode map e299 with expected crc 2023-10-09T10:54:56.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:56 smithi155 ceph-mon[184854]: failed to encode map e299 with expected crc 2023-10-09T10:54:56.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:56 smithi155 ceph-mon[184854]: failed to encode map e299 with expected crc 2023-10-09T10:54:56.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:56 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:54:57.027 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:57.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:57.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:57.136 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:57.201 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:57.279 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:54:57.297 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:54:57.451 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:57.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:57.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:57.557 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:54:57.601 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:57.622 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:54:57.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:57.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:54:57.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:57 smithi125 ceph-mon[195507]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:57 smithi125 ceph-mon[195507]: osdmap e300: 8 total, 8 up, 8 in 2023-10-09T10:54:57.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:57 smithi125 ceph-mon[195507]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:57 smithi125 ceph-mon[195507]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:57 smithi125 ceph-mon[195507]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:57 smithi125 ceph-mon[195507]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:57 smithi125 ceph-mon[195507]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.728 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:54:57.778 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:57.829 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:54:57.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:57.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:54:57.923 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:57 smithi125 ceph-mon[200197]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.923 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:57 smithi125 ceph-mon[200197]: osdmap e300: 8 total, 8 up, 8 in 2023-10-09T10:54:57.923 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:57 smithi125 ceph-mon[200197]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.923 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:57 smithi125 ceph-mon[200197]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.923 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:57 smithi125 ceph-mon[200197]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.924 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:57 smithi125 ceph-mon[200197]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.924 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:57 smithi125 ceph-mon[200197]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.933 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:57 smithi155 ceph-mon[184854]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:57 smithi155 ceph-mon[184854]: osdmap e300: 8 total, 8 up, 8 in 2023-10-09T10:54:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:57 smithi155 ceph-mon[184854]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:57 smithi155 ceph-mon[184854]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:57 smithi155 ceph-mon[184854]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:57 smithi155 ceph-mon[184854]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:57 smithi155 ceph-mon[184854]: failed to encode map e300 with expected crc 2023-10-09T10:54:57.995 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:58.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:54:58.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:58.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:54:58.109 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":712,"stamp":"2023-10-09T10:54:56.622336+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6142714794,"num_objects":93745,"num_object_clones":0,"num_object_copies":281235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17602,"num_objects_misplaced":21593,"num_objects_unfound":0,"num_objects_dirty":93745,"num_whiteouts":0,"num_read":72,"num_read_kb":133,"num_write":99679,"num_write_kb":6186042,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3237,"num_bytes_recovered":209813595,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":146116,"ondisk_log_size":146116,"up":387,"acting":376,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":434,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21858740,"kb_used_data":18318516,"kb_used_omap":126,"kb_used_meta":3539777,"kb_avail":728102476,"statfs":{"total":767960285184,"available":745576935424,"internally_reserved":0,"allocated":18758160384,"data_stored":18747566193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":3624732283},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,0,0,0,2,4,9],"upper_bound":1024},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-154468328,"num_objects":-2356,"num_object_clones":0,"num_object_copies":-7068,"num_objects_missing_on_primary":-5178,"num_objects_missing":-5178,"num_objects_degraded":-15215,"num_objects_misplaced":5344,"num_objects_unfound":0,"num_objects_dirty":-2356,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":3506,"num_write_kb":36737,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1656,"num_bytes_recovered":108396544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.730948"},"pg_stats":[{"pgid":"8.11","version":"298'6023","reported_seq":6230,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:54:53.302858+0000","last_change":"2023-10-09T10:54:49.220300+0000","last_active":"2023-10-09T10:54:53.302858+0000","last_peered":"2023-10-09T10:54:53.302858+0000","last_clean":"2023-10-09T10:54:02.426241+0000","last_became_active":"2023-10-09T10:54:49.215481+0000","last_became_peered":"2023-10-09T10:54:49.215481+0000","last_unstale":"2023-10-09T10:54:53.302858+0000","last_undegraded":"2023-10-09T10:54:49.026934+0000","last_fullsized":"2023-10-09T10:54:48.893501+0000","mapping_epoch":295,"log_start":"270'200","ondisk_log_start":"270'200","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5823,"log_dups_size":0,"ondisk_log_size":5823,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191758336,"num_objects":2926,"num_object_clones":0,"num_object_copies":8778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3223,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3062,"num_write_kb":191616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":4521984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,1,5","objects":2254},{"shards":"0,2","objects":-9},{"shards":"0,2,5","objects":393},{"shards":"0,3,5","objects":288}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.18","version":"76'161","reported_seq":631,"reported_epoch":297,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T10:54:49.897635+0000","last_change":"2023-10-09T10:54:19.094257+0000","last_active":"2023-10-09T10:54:49.897635+0000","last_peered":"2023-10-09T10:54:49.897635+0000","last_clean":"2023-10-09T10:54:08.295760+0000","last_became_active":"2023-10-09T10:54:19.093928+0000","last_became_peered":"2023-10-09T10:54:19.093928+0000","last_unstale":"2023-10-09T10:54:49.897635+0000","last_undegraded":"2023-10-09T10:54:49.897635+0000","last_fullsized":"2023-10-09T10:54:49.897635+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:40.174975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:40.174975+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:38:44.409770+0000","scrub_duration":0.010654205,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[0,5,7],"avail_no_missing":["0","1","5","7"],"object_location_counts":[{"shards":"0,1,5,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.898369+0000","last_change":"2023-10-09T10:54:47.925818+0000","last_active":"2023-10-09T10:54:50.898369+0000","last_peered":"2023-10-09T10:54:50.898369+0000","last_clean":"2023-10-09T10:54:50.898369+0000","last_became_active":"2023-10-09T10:54:47.925375+0000","last_became_peered":"2023-10-09T10:54:47.925375+0000","last_unstale":"2023-10-09T10:54:50.898369+0000","last_undegraded":"2023-10-09T10:54:50.898369+0000","last_fullsized":"2023-10-09T10:54:50.898369+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:27.635383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":305,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.905922+0000","last_change":"2023-10-09T10:54:47.945782+0000","last_active":"2023-10-09T10:54:49.905922+0000","last_peered":"2023-10-09T10:54:49.905922+0000","last_clean":"2023-10-09T10:54:49.905922+0000","last_became_active":"2023-10-09T10:54:47.945460+0000","last_became_peered":"2023-10-09T10:54:47.945460+0000","last_unstale":"2023-10-09T10:54:49.905922+0000","last_undegraded":"2023-10-09T10:54:49.905922+0000","last_fullsized":"2023-10-09T10:54:49.905922+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:50.537469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"299'6234","reported_seq":6291,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:54:56.224521+0000","last_change":"2023-10-09T10:54:40.626105+0000","last_active":"2023-10-09T10:54:56.224521+0000","last_peered":"2023-10-09T10:54:56.224521+0000","last_clean":"2023-10-09T10:54:56.224521+0000","last_became_active":"2023-10-09T10:54:40.609428+0000","last_became_peered":"2023-10-09T10:54:40.609428+0000","last_unstale":"2023-10-09T10:54:56.224521+0000","last_undegraded":"2023-10-09T10:54:56.224521+0000","last_fullsized":"2023-10-09T10:54:56.224521+0000","mapping_epoch":289,"log_start":"271'700","ondisk_log_start":"271'700","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5534,"log_dups_size":0,"ondisk_log_size":5534,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192086016,"num_objects":2931,"num_object_clones":0,"num_object_copies":8793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3199,"num_write_kb":196160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":678,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.900278+0000","last_change":"2023-10-09T10:53:34.537094+0000","last_active":"2023-10-09T10:54:49.900278+0000","last_peered":"2023-10-09T10:54:49.900278+0000","last_clean":"2023-10-09T10:54:49.900278+0000","last_became_active":"2023-10-09T10:52:55.414995+0000","last_became_peered":"2023-10-09T10:52:55.414995+0000","last_unstale":"2023-10-09T10:54:49.900278+0000","last_undegraded":"2023-10-09T10:54:49.900278+0000","last_fullsized":"2023-10-09T10:54:49.900278+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.536962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.536962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:30.197404+0000","scrub_duration":0.0053325960000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.898004+0000","last_change":"2023-10-09T10:54:47.944768+0000","last_active":"2023-10-09T10:54:50.898004+0000","last_peered":"2023-10-09T10:54:50.898004+0000","last_clean":"2023-10-09T10:54:50.898004+0000","last_became_active":"2023-10-09T10:54:47.943888+0000","last_became_peered":"2023-10-09T10:54:47.943888+0000","last_unstale":"2023-10-09T10:54:50.898004+0000","last_undegraded":"2023-10-09T10:54:50.898004+0000","last_fullsized":"2023-10-09T10:54:50.898004+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:11.046762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":206,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.901544+0000","last_change":"2023-10-09T10:53:14.823051+0000","last_active":"2023-10-09T10:54:49.901544+0000","last_peered":"2023-10-09T10:54:49.901544+0000","last_clean":"2023-10-09T10:54:49.901544+0000","last_became_active":"2023-10-09T10:53:14.822334+0000","last_became_peered":"2023-10-09T10:53:14.822334+0000","last_unstale":"2023-10-09T10:54:49.901544+0000","last_undegraded":"2023-10-09T10:54:49.901544+0000","last_fullsized":"2023-10-09T10:54:49.901544+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:46.573381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"298'5972","reported_seq":6069,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:51.668620+0000","last_change":"2023-10-09T10:54:50.916172+0000","last_active":"2023-10-09T10:54:51.668620+0000","last_peered":"2023-10-09T10:54:51.668620+0000","last_clean":"2023-10-09T10:54:51.668620+0000","last_became_active":"2023-10-09T10:54:50.913596+0000","last_became_peered":"2023-10-09T10:54:50.913596+0000","last_unstale":"2023-10-09T10:54:51.668620+0000","last_undegraded":"2023-10-09T10:54:51.668620+0000","last_fullsized":"2023-10-09T10:54:51.668620+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5972,"log_dups_size":0,"ondisk_log_size":5972,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194019328,"num_objects":2961,"num_object_clones":0,"num_object_copies":8883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3015,"num_write_kb":191200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1835008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":682,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.913824+0000","last_change":"2023-10-09T10:54:50.913546+0000","last_active":"2023-10-09T10:54:50.913824+0000","last_peered":"2023-10-09T10:54:50.913824+0000","last_clean":"2023-10-09T10:54:50.913824+0000","last_became_active":"2023-10-09T10:54:50.913190+0000","last_became_peered":"2023-10-09T10:54:50.913190+0000","last_unstale":"2023-10-09T10:54:50.913824+0000","last_undegraded":"2023-10-09T10:54:50.913824+0000","last_fullsized":"2023-10-09T10:54:50.913824+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:23.805498+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":350,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.900841+0000","last_change":"2023-10-09T10:54:48.038948+0000","last_active":"2023-10-09T10:54:49.900841+0000","last_peered":"2023-10-09T10:54:49.900841+0000","last_clean":"2023-10-09T10:54:49.900841+0000","last_became_active":"2023-10-09T10:54:48.001556+0000","last_became_peered":"2023-10-09T10:54:48.001556+0000","last_unstale":"2023-10-09T10:54:49.900841+0000","last_undegraded":"2023-10-09T10:54:49.900841+0000","last_fullsized":"2023-10-09T10:54:49.900841+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:23.415040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.900707+0000","last_change":"2023-10-09T10:54:47.914742+0000","last_active":"2023-10-09T10:54:50.900707+0000","last_peered":"2023-10-09T10:54:50.900707+0000","last_clean":"2023-10-09T10:54:50.900707+0000","last_became_active":"2023-10-09T10:54:47.914551+0000","last_became_peered":"2023-10-09T10:54:47.914551+0000","last_unstale":"2023-10-09T10:54:50.900707+0000","last_undegraded":"2023-10-09T10:54:50.900707+0000","last_fullsized":"2023-10-09T10:54:50.900707+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:16.067994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":733,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.916061+0000","last_change":"2023-10-09T10:54:50.915690+0000","last_active":"2023-10-09T10:54:50.916061+0000","last_peered":"2023-10-09T10:54:50.916061+0000","last_clean":"2023-10-09T10:54:50.916061+0000","last_became_active":"2023-10-09T10:54:50.915412+0000","last_became_peered":"2023-10-09T10:54:50.915412+0000","last_unstale":"2023-10-09T10:54:50.916061+0000","last_undegraded":"2023-10-09T10:54:50.916061+0000","last_fullsized":"2023-10-09T10:54:50.916061+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.787782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.787782+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:20:13.339672+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":169,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.901423+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:54:49.901423+0000","last_peered":"2023-10-09T10:54:49.901423+0000","last_clean":"2023-10-09T10:54:49.901423+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:54:49.901423+0000","last_undegraded":"2023-10-09T10:54:49.901423+0000","last_fullsized":"2023-10-09T10:54:49.901423+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":305,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.900877+0000","last_change":"2023-10-09T10:54:48.039275+0000","last_active":"2023-10-09T10:54:49.900877+0000","last_peered":"2023-10-09T10:54:49.900877+0000","last_clean":"2023-10-09T10:54:49.900877+0000","last_became_active":"2023-10-09T10:54:48.017645+0000","last_became_peered":"2023-10-09T10:54:48.017645+0000","last_unstale":"2023-10-09T10:54:49.900877+0000","last_undegraded":"2023-10-09T10:54:49.900877+0000","last_fullsized":"2023-10-09T10:54:49.900877+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:55.986791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":543,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.923063+0000","last_change":"2023-10-09T10:54:21.161539+0000","last_active":"2023-10-09T10:54:50.923063+0000","last_peered":"2023-10-09T10:54:50.923063+0000","last_clean":"2023-10-09T10:54:50.923063+0000","last_became_active":"2023-10-09T10:54:21.161106+0000","last_became_peered":"2023-10-09T10:54:21.161106+0000","last_unstale":"2023-10-09T10:54:50.923063+0000","last_undegraded":"2023-10-09T10:54:50.923063+0000","last_fullsized":"2023-10-09T10:54:50.923063+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:21.824982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:21.824982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.900622+0000","last_change":"2023-10-09T10:54:47.944178+0000","last_active":"2023-10-09T10:54:50.900622+0000","last_peered":"2023-10-09T10:54:50.900622+0000","last_clean":"2023-10-09T10:54:50.900622+0000","last_became_active":"2023-10-09T10:54:47.943913+0000","last_became_peered":"2023-10-09T10:54:47.943913+0000","last_unstale":"2023-10-09T10:54:50.900622+0000","last_undegraded":"2023-10-09T10:54:50.900622+0000","last_fullsized":"2023-10-09T10:54:50.900622+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:06.448201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":244,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.907521+0000","last_change":"2023-10-09T10:53:14.838859+0000","last_active":"2023-10-09T10:54:49.907521+0000","last_peered":"2023-10-09T10:54:49.907521+0000","last_clean":"2023-10-09T10:54:49.907521+0000","last_became_active":"2023-10-09T10:53:14.838536+0000","last_became_peered":"2023-10-09T10:53:14.838536+0000","last_unstale":"2023-10-09T10:54:49.907521+0000","last_undegraded":"2023-10-09T10:54:49.907521+0000","last_fullsized":"2023-10-09T10:54:49.907521+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:44:19.980568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":685,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.901266+0000","last_change":"2023-10-09T10:53:32.540768+0000","last_active":"2023-10-09T10:54:49.901266+0000","last_peered":"2023-10-09T10:54:49.901266+0000","last_clean":"2023-10-09T10:54:49.901266+0000","last_became_active":"2023-10-09T10:52:55.414818+0000","last_became_peered":"2023-10-09T10:52:55.414818+0000","last_unstale":"2023-10-09T10:54:49.901266+0000","last_undegraded":"2023-10-09T10:54:49.901266+0000","last_fullsized":"2023-10-09T10:54:49.901266+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:32.540610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:32.540610+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:47.374383+0000","scrub_duration":0.0064251300000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":252,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.905989+0000","last_change":"2023-10-09T10:53:14.847647+0000","last_active":"2023-10-09T10:54:49.905989+0000","last_peered":"2023-10-09T10:54:49.905989+0000","last_clean":"2023-10-09T10:54:49.905989+0000","last_became_active":"2023-10-09T10:53:14.832089+0000","last_became_peered":"2023-10-09T10:53:14.832089+0000","last_unstale":"2023-10-09T10:54:49.905989+0000","last_undegraded":"2023-10-09T10:54:49.905989+0000","last_fullsized":"2023-10-09T10:54:49.905989+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:34:45.371102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.898456+0000","last_change":"2023-10-09T10:54:47.925654+0000","last_active":"2023-10-09T10:54:50.898456+0000","last_peered":"2023-10-09T10:54:50.898456+0000","last_clean":"2023-10-09T10:54:50.898456+0000","last_became_active":"2023-10-09T10:54:47.924989+0000","last_became_peered":"2023-10-09T10:54:47.924989+0000","last_unstale":"2023-10-09T10:54:50.898456+0000","last_undegraded":"2023-10-09T10:54:50.898456+0000","last_fullsized":"2023-10-09T10:54:50.898456+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:02:49.755988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":671,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.918222+0000","last_change":"2023-10-09T10:54:50.913652+0000","last_active":"2023-10-09T10:54:50.918222+0000","last_peered":"2023-10-09T10:54:50.918222+0000","last_clean":"2023-10-09T10:54:50.918222+0000","last_became_active":"2023-10-09T10:54:50.913330+0000","last_became_peered":"2023-10-09T10:54:50.913330+0000","last_unstale":"2023-10-09T10:54:50.918222+0000","last_undegraded":"2023-10-09T10:54:50.918222+0000","last_fullsized":"2023-10-09T10:54:50.918222+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.200574+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.200574+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001591812,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":52,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.901042+0000","last_change":"2023-10-09T10:53:14.869000+0000","last_active":"2023-10-09T10:54:50.901042+0000","last_peered":"2023-10-09T10:54:50.901042+0000","last_clean":"2023-10-09T10:54:50.901042+0000","last_became_active":"2023-10-09T10:53:14.831904+0000","last_became_peered":"2023-10-09T10:53:14.831904+0000","last_unstale":"2023-10-09T10:54:50.901042+0000","last_undegraded":"2023-10-09T10:54:50.901042+0000","last_fullsized":"2023-10-09T10:54:50.901042+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:38.571980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":303,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.899948+0000","last_change":"2023-10-09T10:54:47.908531+0000","last_active":"2023-10-09T10:54:50.899948+0000","last_peered":"2023-10-09T10:54:50.899948+0000","last_clean":"2023-10-09T10:54:50.899948+0000","last_became_active":"2023-10-09T10:54:47.908232+0000","last_became_peered":"2023-10-09T10:54:47.908232+0000","last_unstale":"2023-10-09T10:54:50.899948+0000","last_undegraded":"2023-10-09T10:54:50.899948+0000","last_fullsized":"2023-10-09T10:54:50.899948+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:20.617543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"298'6124","reported_seq":6176,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:55.218911+0000","last_change":"2023-10-09T10:54:40.649728+0000","last_active":"2023-10-09T10:54:55.218911+0000","last_peered":"2023-10-09T10:54:55.218911+0000","last_clean":"2023-10-09T10:54:55.218911+0000","last_became_active":"2023-10-09T10:54:40.647392+0000","last_became_peered":"2023-10-09T10:54:40.647392+0000","last_unstale":"2023-10-09T10:54:55.218911+0000","last_undegraded":"2023-10-09T10:54:55.218911+0000","last_fullsized":"2023-10-09T10:54:55.218911+0000","mapping_epoch":289,"log_start":"271'1000","ondisk_log_start":"271'1000","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5124,"log_dups_size":0,"ondisk_log_size":5124,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190840832,"num_objects":2912,"num_object_clones":0,"num_object_copies":8736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3134,"num_write_kb":193472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":663,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.904054+0000","last_change":"2023-10-09T10:49:18.708370+0000","last_active":"2023-10-09T10:54:49.904054+0000","last_peered":"2023-10-09T10:54:49.904054+0000","last_clean":"2023-10-09T10:54:49.904054+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:54:49.904054+0000","last_undegraded":"2023-10-09T10:54:49.904054+0000","last_fullsized":"2023-10-09T10:54:49.904054+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:57.808410+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":64,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.900346+0000","last_change":"2023-10-09T10:54:18.080112+0000","last_active":"2023-10-09T10:54:50.900346+0000","last_peered":"2023-10-09T10:54:50.900346+0000","last_clean":"2023-10-09T10:54:50.900346+0000","last_became_active":"2023-10-09T10:54:17.878848+0000","last_became_peered":"2023-10-09T10:54:17.878848+0000","last_unstale":"2023-10-09T10:54:50.900346+0000","last_undegraded":"2023-10-09T10:54:50.900346+0000","last_fullsized":"2023-10-09T10:54:50.900346+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:12.314073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":355,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.904330+0000","last_change":"2023-10-09T10:54:47.941670+0000","last_active":"2023-10-09T10:54:49.904330+0000","last_peered":"2023-10-09T10:54:49.904330+0000","last_clean":"2023-10-09T10:54:49.904330+0000","last_became_active":"2023-10-09T10:54:47.941341+0000","last_became_peered":"2023-10-09T10:54:47.941341+0000","last_unstale":"2023-10-09T10:54:49.904330+0000","last_undegraded":"2023-10-09T10:54:49.904330+0000","last_fullsized":"2023-10-09T10:54:49.904330+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:31.570728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.19","version":"298'6065","reported_seq":6116,"reported_epoch":298,"state":"active+recovery_wait","last_fresh":"2023-10-09T10:54:53.220821+0000","last_change":"2023-10-09T10:54:40.731544+0000","last_active":"2023-10-09T10:54:53.220821+0000","last_peered":"2023-10-09T10:54:53.220821+0000","last_clean":"2023-10-09T10:54:39.518652+0000","last_became_active":"2023-10-09T10:54:40.598732+0000","last_became_peered":"2023-10-09T10:54:40.598732+0000","last_unstale":"2023-10-09T10:54:53.220821+0000","last_undegraded":"2023-10-09T10:54:53.220821+0000","last_fullsized":"2023-10-09T10:54:53.220821+0000","mapping_epoch":289,"log_start":"270'300","ondisk_log_start":"270'300","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5765,"log_dups_size":0,"ondisk_log_size":5765,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194772992,"num_objects":2972,"num_object_clones":0,"num_object_copies":8916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3080,"num_write_kb":193664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":2972}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":646,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.900868+0000","last_change":"2023-10-09T10:53:17.846464+0000","last_active":"2023-10-09T10:54:50.900868+0000","last_peered":"2023-10-09T10:54:50.900868+0000","last_clean":"2023-10-09T10:54:50.900868+0000","last_became_active":"2023-10-09T10:53:17.845979+0000","last_became_peered":"2023-10-09T10:53:17.845979+0000","last_unstale":"2023-10-09T10:54:50.900868+0000","last_undegraded":"2023-10-09T10:54:50.900868+0000","last_fullsized":"2023-10-09T10:54:50.900868+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:56.295299+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":205,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.899801+0000","last_change":"2023-10-09T10:53:14.821961+0000","last_active":"2023-10-09T10:54:49.899801+0000","last_peered":"2023-10-09T10:54:49.899801+0000","last_clean":"2023-10-09T10:54:49.899801+0000","last_became_active":"2023-10-09T10:53:14.821625+0000","last_became_peered":"2023-10-09T10:53:14.821625+0000","last_unstale":"2023-10-09T10:54:49.899801+0000","last_undegraded":"2023-10-09T10:54:49.899801+0000","last_fullsized":"2023-10-09T10:54:49.899801+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:29.305952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":248,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.905874+0000","last_change":"2023-10-09T10:53:14.832387+0000","last_active":"2023-10-09T10:54:49.905874+0000","last_peered":"2023-10-09T10:54:49.905874+0000","last_clean":"2023-10-09T10:54:49.905874+0000","last_became_active":"2023-10-09T10:53:14.831672+0000","last_became_peered":"2023-10-09T10:53:14.831672+0000","last_unstale":"2023-10-09T10:54:49.905874+0000","last_undegraded":"2023-10-09T10:54:49.905874+0000","last_fullsized":"2023-10-09T10:54:49.905874+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:59.194806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":602,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.920367+0000","last_change":"2023-10-09T10:54:50.916062+0000","last_active":"2023-10-09T10:54:50.920367+0000","last_peered":"2023-10-09T10:54:50.920367+0000","last_clean":"2023-10-09T10:54:50.920367+0000","last_became_active":"2023-10-09T10:54:50.915189+0000","last_became_peered":"2023-10-09T10:54:50.915189+0000","last_unstale":"2023-10-09T10:54:50.920367+0000","last_undegraded":"2023-10-09T10:54:50.920367+0000","last_fullsized":"2023-10-09T10:54:50.920367+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.077788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.077788+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":229,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.899812+0000","last_change":"2023-10-09T10:53:14.835799+0000","last_active":"2023-10-09T10:54:50.899812+0000","last_peered":"2023-10-09T10:54:50.899812+0000","last_clean":"2023-10-09T10:54:50.899812+0000","last_became_active":"2023-10-09T10:53:14.835476+0000","last_became_peered":"2023-10-09T10:53:14.835476+0000","last_unstale":"2023-10-09T10:54:50.899812+0000","last_undegraded":"2023-10-09T10:54:50.899812+0000","last_fullsized":"2023-10-09T10:54:50.899812+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:29.248961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":187,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.901284+0000","last_change":"2023-10-09T10:53:14.823485+0000","last_active":"2023-10-09T10:54:49.901284+0000","last_peered":"2023-10-09T10:54:49.901284+0000","last_clean":"2023-10-09T10:54:49.901284+0000","last_became_active":"2023-10-09T10:53:14.822756+0000","last_became_peered":"2023-10-09T10:53:14.822756+0000","last_unstale":"2023-10-09T10:54:49.901284+0000","last_undegraded":"2023-10-09T10:54:49.901284+0000","last_fullsized":"2023-10-09T10:54:49.901284+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:05.359313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":629,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.915744+0000","last_change":"2023-10-09T10:54:50.910027+0000","last_active":"2023-10-09T10:54:50.915744+0000","last_peered":"2023-10-09T10:54:50.915744+0000","last_clean":"2023-10-09T10:54:50.915744+0000","last_became_active":"2023-10-09T10:54:50.909698+0000","last_became_peered":"2023-10-09T10:54:50.909698+0000","last_unstale":"2023-10-09T10:54:50.915744+0000","last_undegraded":"2023-10-09T10:54:50.915744+0000","last_fullsized":"2023-10-09T10:54:50.915744+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:01.637196+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":172,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.898066+0000","last_change":"2023-10-09T10:51:12.366910+0000","last_active":"2023-10-09T10:54:49.898066+0000","last_peered":"2023-10-09T10:54:49.898066+0000","last_clean":"2023-10-09T10:54:49.898066+0000","last_became_active":"2023-10-09T10:51:12.366565+0000","last_became_peered":"2023-10-09T10:51:12.366565+0000","last_unstale":"2023-10-09T10:54:49.898066+0000","last_undegraded":"2023-10-09T10:54:49.898066+0000","last_fullsized":"2023-10-09T10:54:49.898066+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:53.464251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":332,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.908191+0000","last_change":"2023-10-09T10:54:48.017646+0000","last_active":"2023-10-09T10:54:49.908191+0000","last_peered":"2023-10-09T10:54:49.908191+0000","last_clean":"2023-10-09T10:54:49.908191+0000","last_became_active":"2023-10-09T10:54:47.994162+0000","last_became_peered":"2023-10-09T10:54:47.994162+0000","last_unstale":"2023-10-09T10:54:49.908191+0000","last_undegraded":"2023-10-09T10:54:49.908191+0000","last_fullsized":"2023-10-09T10:54:49.908191+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:11:59.205546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"299'6148","reported_seq":6213,"reported_epoch":299,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:54:56.228063+0000","last_change":"2023-10-09T10:54:40.551123+0000","last_active":"2023-10-09T10:54:56.228063+0000","last_peered":"2023-10-09T10:54:56.228063+0000","last_clean":"2023-10-09T10:54:17.036595+0000","last_became_active":"2023-10-09T10:54:40.548039+0000","last_became_peered":"2023-10-09T10:54:40.548039+0000","last_unstale":"2023-10-09T10:54:56.228063+0000","last_undegraded":"2023-10-09T10:54:56.228063+0000","last_fullsized":"2023-10-09T10:54:56.228063+0000","mapping_epoch":282,"log_start":"271'600","ondisk_log_start":"271'600","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5548,"log_dups_size":0,"ondisk_log_size":5548,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188383232,"num_objects":2874,"num_object_clones":0,"num_object_copies":8622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2874,"num_objects_unfound":0,"num_objects_dirty":2874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3162,"num_write_kb":193184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":2874}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":650,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.910169+0000","last_change":"2023-10-09T10:54:50.909873+0000","last_active":"2023-10-09T10:54:50.910169+0000","last_peered":"2023-10-09T10:54:50.910169+0000","last_clean":"2023-10-09T10:54:50.910169+0000","last_became_active":"2023-10-09T10:54:50.909533+0000","last_became_peered":"2023-10-09T10:54:50.909533+0000","last_unstale":"2023-10-09T10:54:50.910169+0000","last_undegraded":"2023-10-09T10:54:50.910169+0000","last_fullsized":"2023-10-09T10:54:50.910169+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:45.045013+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":46,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.900615+0000","last_change":"2023-10-09T10:53:14.854904+0000","last_active":"2023-10-09T10:54:50.900615+0000","last_peered":"2023-10-09T10:54:50.900615+0000","last_clean":"2023-10-09T10:54:50.900615+0000","last_became_active":"2023-10-09T10:53:14.829611+0000","last_became_peered":"2023-10-09T10:53:14.829611+0000","last_unstale":"2023-10-09T10:54:50.900615+0000","last_undegraded":"2023-10-09T10:54:50.900615+0000","last_fullsized":"2023-10-09T10:54:50.900615+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:26.064630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":266,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.895988+0000","last_change":"2023-10-09T10:53:14.831930+0000","last_active":"2023-10-09T10:54:50.895988+0000","last_peered":"2023-10-09T10:54:50.895988+0000","last_clean":"2023-10-09T10:54:50.895988+0000","last_became_active":"2023-10-09T10:53:14.831600+0000","last_became_peered":"2023-10-09T10:53:14.831600+0000","last_unstale":"2023-10-09T10:54:50.895988+0000","last_undegraded":"2023-10-09T10:54:50.895988+0000","last_fullsized":"2023-10-09T10:54:50.895988+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:36:16.104668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"299'6074","reported_seq":6124,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:54:56.127726+0000","last_change":"2023-10-09T10:54:40.557441+0000","last_active":"2023-10-09T10:54:56.127726+0000","last_peered":"2023-10-09T10:54:56.127726+0000","last_clean":"2023-10-09T10:54:56.127726+0000","last_became_active":"2023-10-09T10:54:40.554360+0000","last_became_peered":"2023-10-09T10:54:40.554360+0000","last_unstale":"2023-10-09T10:54:56.127726+0000","last_undegraded":"2023-10-09T10:54:56.127726+0000","last_fullsized":"2023-10-09T10:54:56.127726+0000","mapping_epoch":266,"log_start":"271'400","ondisk_log_start":"271'400","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188284928,"num_objects":2873,"num_object_clones":0,"num_object_copies":8619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3113,"num_write_kb":191552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":657,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.904160+0000","last_change":"2023-10-09T10:53:25.540494+0000","last_active":"2023-10-09T10:54:49.904160+0000","last_peered":"2023-10-09T10:54:49.904160+0000","last_clean":"2023-10-09T10:54:49.904160+0000","last_became_active":"2023-10-09T10:51:01.295474+0000","last_became_peered":"2023-10-09T10:51:01.295474+0000","last_unstale":"2023-10-09T10:54:49.904160+0000","last_undegraded":"2023-10-09T10:54:49.904160+0000","last_fullsized":"2023-10-09T10:54:49.904160+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:39.069143+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":74,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.901004+0000","last_change":"2023-10-09T10:54:48.000482+0000","last_active":"2023-10-09T10:54:50.901004+0000","last_peered":"2023-10-09T10:54:50.901004+0000","last_clean":"2023-10-09T10:54:50.901004+0000","last_became_active":"2023-10-09T10:54:47.905689+0000","last_became_peered":"2023-10-09T10:54:47.905689+0000","last_unstale":"2023-10-09T10:54:50.901004+0000","last_undegraded":"2023-10-09T10:54:50.901004+0000","last_fullsized":"2023-10-09T10:54:50.901004+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.666594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.897845+0000","last_change":"2023-10-09T10:54:47.944930+0000","last_active":"2023-10-09T10:54:50.897845+0000","last_peered":"2023-10-09T10:54:50.897845+0000","last_clean":"2023-10-09T10:54:50.897845+0000","last_became_active":"2023-10-09T10:54:47.944485+0000","last_became_peered":"2023-10-09T10:54:47.944485+0000","last_unstale":"2023-10-09T10:54:50.897845+0000","last_undegraded":"2023-10-09T10:54:50.897845+0000","last_fullsized":"2023-10-09T10:54:50.897845+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:11.213129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"298'6086","reported_seq":6138,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:53.201724+0000","last_change":"2023-10-09T10:54:40.586495+0000","last_active":"2023-10-09T10:54:53.201724+0000","last_peered":"2023-10-09T10:54:53.201724+0000","last_clean":"2023-10-09T10:54:53.201724+0000","last_became_active":"2023-10-09T10:54:40.577656+0000","last_became_peered":"2023-10-09T10:54:40.577656+0000","last_unstale":"2023-10-09T10:54:53.201724+0000","last_undegraded":"2023-10-09T10:54:53.201724+0000","last_fullsized":"2023-10-09T10:54:53.201724+0000","mapping_epoch":266,"log_start":"270'300","ondisk_log_start":"270'300","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5786,"log_dups_size":0,"ondisk_log_size":5786,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194740224,"num_objects":2971,"num_object_clones":0,"num_object_copies":8913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3091,"num_write_kb":194016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":611,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.899948+0000","last_change":"2023-10-09T10:53:36.680150+0000","last_active":"2023-10-09T10:54:50.899948+0000","last_peered":"2023-10-09T10:54:50.899948+0000","last_clean":"2023-10-09T10:54:50.899948+0000","last_became_active":"2023-10-09T10:51:50.371032+0000","last_became_peered":"2023-10-09T10:51:50.371032+0000","last_unstale":"2023-10-09T10:54:50.899948+0000","last_undegraded":"2023-10-09T10:54:50.899948+0000","last_fullsized":"2023-10-09T10:54:50.899948+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.680090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.680090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:08:28.952827+0000","scrub_duration":0.005577172,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":46,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.901642+0000","last_change":"2023-10-09T10:53:14.841343+0000","last_active":"2023-10-09T10:54:50.901642+0000","last_peered":"2023-10-09T10:54:50.901642+0000","last_clean":"2023-10-09T10:54:50.901642+0000","last_became_active":"2023-10-09T10:53:14.828213+0000","last_became_peered":"2023-10-09T10:53:14.828213+0000","last_unstale":"2023-10-09T10:54:50.901642+0000","last_undegraded":"2023-10-09T10:54:50.901642+0000","last_fullsized":"2023-10-09T10:54:50.901642+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:28.137803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.899911+0000","last_change":"2023-10-09T10:54:47.942716+0000","last_active":"2023-10-09T10:54:50.899911+0000","last_peered":"2023-10-09T10:54:50.899911+0000","last_clean":"2023-10-09T10:54:50.899911+0000","last_became_active":"2023-10-09T10:54:47.942459+0000","last_became_peered":"2023-10-09T10:54:47.942459+0000","last_unstale":"2023-10-09T10:54:50.899911+0000","last_undegraded":"2023-10-09T10:54:50.899911+0000","last_fullsized":"2023-10-09T10:54:50.899911+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:07.790165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"298'6085","reported_seq":6163,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:54:55.160251+0000","last_change":"2023-10-09T10:54:49.183581+0000","last_active":"2023-10-09T10:54:55.160251+0000","last_peered":"2023-10-09T10:54:55.160251+0000","last_clean":"2023-10-09T10:54:02.391425+0000","last_became_active":"2023-10-09T10:54:49.111868+0000","last_became_peered":"2023-10-09T10:54:49.111868+0000","last_unstale":"2023-10-09T10:54:55.160251+0000","last_undegraded":"2023-10-09T10:54:48.912262+0000","last_fullsized":"2023-10-09T10:54:48.907341+0000","mapping_epoch":295,"log_start":"271'1000","ondisk_log_start":"271'1000","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5085,"log_dups_size":0,"ondisk_log_size":5085,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191397888,"num_objects":2920,"num_object_clones":0,"num_object_copies":8760,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":697,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2920,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3108,"num_write_kb":192928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"3,4,6","objects":697},{"shards":"3,6","objects":2223}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":744,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.899715+0000","last_change":"2023-10-09T10:53:38.681891+0000","last_active":"2023-10-09T10:54:50.899715+0000","last_peered":"2023-10-09T10:54:50.899715+0000","last_clean":"2023-10-09T10:54:50.899715+0000","last_became_active":"2023-10-09T10:53:17.840478+0000","last_became_peered":"2023-10-09T10:53:17.840478+0000","last_unstale":"2023-10-09T10:54:50.899715+0000","last_undegraded":"2023-10-09T10:54:50.899715+0000","last_fullsized":"2023-10-09T10:54:50.899715+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:31:14.758036+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":123,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.904882+0000","last_change":"2023-10-09T10:51:50.374527+0000","last_active":"2023-10-09T10:54:49.904882+0000","last_peered":"2023-10-09T10:54:49.904882+0000","last_clean":"2023-10-09T10:54:49.904882+0000","last_became_active":"2023-10-09T10:51:50.374297+0000","last_became_peered":"2023-10-09T10:51:50.374297+0000","last_unstale":"2023-10-09T10:54:49.904882+0000","last_undegraded":"2023-10-09T10:54:49.904882+0000","last_fullsized":"2023-10-09T10:54:49.904882+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:31:49.358307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":205,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.901152+0000","last_change":"2023-10-09T10:54:47.939771+0000","last_active":"2023-10-09T10:54:49.901152+0000","last_peered":"2023-10-09T10:54:49.901152+0000","last_clean":"2023-10-09T10:54:49.901152+0000","last_became_active":"2023-10-09T10:54:47.939440+0000","last_became_peered":"2023-10-09T10:54:47.939440+0000","last_unstale":"2023-10-09T10:54:49.901152+0000","last_undegraded":"2023-10-09T10:54:49.901152+0000","last_fullsized":"2023-10-09T10:54:49.901152+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:58.335536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"298'6167","reported_seq":8098,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:54:55.230851+0000","last_change":"2023-10-09T10:54:49.158909+0000","last_active":"2023-10-09T10:54:55.230851+0000","last_peered":"2023-10-09T10:54:55.230851+0000","last_clean":"2023-10-09T10:54:02.435878+0000","last_became_active":"2023-10-09T10:54:48.018286+0000","last_became_peered":"2023-10-09T10:54:48.018286+0000","last_unstale":"2023-10-09T10:54:55.230851+0000","last_undegraded":"2023-10-09T10:54:49.152454+0000","last_fullsized":"2023-10-09T10:54:47.891421+0000","mapping_epoch":294,"log_start":"271'1100","ondisk_log_start":"271'1100","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5067,"log_dups_size":0,"ondisk_log_size":5067,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193167360,"num_objects":2947,"num_object_clones":0,"num_object_copies":8841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":407,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3149,"num_write_kb":195104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":592,"num_bytes_recovered":38830080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":407},{"shards":"3,5","objects":2540}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":706,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.900083+0000","last_change":"2023-10-09T10:54:21.165674+0000","last_active":"2023-10-09T10:54:49.900083+0000","last_peered":"2023-10-09T10:54:49.900083+0000","last_clean":"2023-10-09T10:54:49.900083+0000","last_became_active":"2023-10-09T10:54:21.165317+0000","last_became_peered":"2023-10-09T10:54:21.165317+0000","last_unstale":"2023-10-09T10:54:49.900083+0000","last_undegraded":"2023-10-09T10:54:49.900083+0000","last_fullsized":"2023-10-09T10:54:49.900083+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:34:22.533546+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":294,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.903660+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:54:49.903660+0000","last_peered":"2023-10-09T10:54:49.903660+0000","last_clean":"2023-10-09T10:54:49.903660+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:54:49.903660+0000","last_undegraded":"2023-10-09T10:54:49.903660+0000","last_fullsized":"2023-10-09T10:54:49.903660+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":138,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.904413+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T10:54:49.904413+0000","last_peered":"2023-10-09T10:54:49.904413+0000","last_clean":"2023-10-09T10:54:49.904413+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T10:54:49.904413+0000","last_undegraded":"2023-10-09T10:54:49.904413+0000","last_fullsized":"2023-10-09T10:54:49.904413+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"298'6021","reported_seq":6219,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:54:53.269008+0000","last_change":"2023-10-09T10:54:49.220251+0000","last_active":"2023-10-09T10:54:53.269008+0000","last_peered":"2023-10-09T10:54:53.269008+0000","last_clean":"2023-10-09T10:54:02.426241+0000","last_became_active":"2023-10-09T10:54:49.172055+0000","last_became_peered":"2023-10-09T10:54:49.172055+0000","last_unstale":"2023-10-09T10:54:53.269008+0000","last_undegraded":"2023-10-09T10:54:49.015836+0000","last_fullsized":"2023-10-09T10:54:48.896714+0000","mapping_epoch":295,"log_start":"270'200","ondisk_log_start":"270'200","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5821,"log_dups_size":0,"ondisk_log_size":5821,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192151552,"num_objects":2932,"num_object_clones":0,"num_object_copies":8796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3207,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3060,"num_write_kb":191744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,1,5","objects":2244},{"shards":"0,2","objects":10},{"shards":"0,2,5","objects":393},{"shards":"0,3,5","objects":285}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":602,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.923338+0000","last_change":"2023-10-09T10:54:21.173078+0000","last_active":"2023-10-09T10:54:50.923338+0000","last_peered":"2023-10-09T10:54:50.923338+0000","last_clean":"2023-10-09T10:54:50.923338+0000","last_became_active":"2023-10-09T10:54:21.172767+0000","last_became_peered":"2023-10-09T10:54:21.172767+0000","last_unstale":"2023-10-09T10:54:50.923338+0000","last_undegraded":"2023-10-09T10:54:50.923338+0000","last_fullsized":"2023-10-09T10:54:50.923338+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.532890+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.532890+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:32.335841+0000","scrub_duration":0.0045290130000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":278,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.900164+0000","last_change":"2023-10-09T10:54:47.908506+0000","last_active":"2023-10-09T10:54:50.900164+0000","last_peered":"2023-10-09T10:54:50.900164+0000","last_clean":"2023-10-09T10:54:50.900164+0000","last_became_active":"2023-10-09T10:54:47.908110+0000","last_became_peered":"2023-10-09T10:54:47.908110+0000","last_unstale":"2023-10-09T10:54:50.900164+0000","last_undegraded":"2023-10-09T10:54:50.900164+0000","last_fullsized":"2023-10-09T10:54:50.900164+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:38.808943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":37,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.924241+0000","last_change":"2023-10-09T10:54:47.948732+0000","last_active":"2023-10-09T10:54:50.924241+0000","last_peered":"2023-10-09T10:54:50.924241+0000","last_clean":"2023-10-09T10:54:50.924241+0000","last_became_active":"2023-10-09T10:54:47.948414+0000","last_became_peered":"2023-10-09T10:54:47.948414+0000","last_unstale":"2023-10-09T10:54:50.924241+0000","last_undegraded":"2023-10-09T10:54:50.924241+0000","last_fullsized":"2023-10-09T10:54:50.924241+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:38.020807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.2","version":"299'6225","reported_seq":6276,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:54:56.256682+0000","last_change":"2023-10-09T10:54:40.570138+0000","last_active":"2023-10-09T10:54:56.256682+0000","last_peered":"2023-10-09T10:54:56.256682+0000","last_clean":"2023-10-09T10:54:56.256682+0000","last_became_active":"2023-10-09T10:54:40.567891+0000","last_became_peered":"2023-10-09T10:54:40.567891+0000","last_unstale":"2023-10-09T10:54:56.256682+0000","last_undegraded":"2023-10-09T10:54:56.256682+0000","last_fullsized":"2023-10-09T10:54:56.256682+0000","mapping_epoch":265,"log_start":"271'700","ondisk_log_start":"271'700","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5525,"log_dups_size":0,"ondisk_log_size":5525,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191496192,"num_objects":2922,"num_object_clones":0,"num_object_copies":8766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3190,"num_write_kb":195584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":672,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.920163+0000","last_change":"2023-10-09T10:54:50.914888+0000","last_active":"2023-10-09T10:54:50.920163+0000","last_peered":"2023-10-09T10:54:50.920163+0000","last_clean":"2023-10-09T10:54:50.920163+0000","last_became_active":"2023-10-09T10:54:50.909639+0000","last_became_peered":"2023-10-09T10:54:50.909639+0000","last_unstale":"2023-10-09T10:54:50.920163+0000","last_undegraded":"2023-10-09T10:54:50.920163+0000","last_fullsized":"2023-10-09T10:54:50.920163+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.103750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.103750+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":334,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.899681+0000","last_change":"2023-10-09T10:54:17.876795+0000","last_active":"2023-10-09T10:54:50.899681+0000","last_peered":"2023-10-09T10:54:50.899681+0000","last_clean":"2023-10-09T10:54:50.899681+0000","last_became_active":"2023-10-09T10:54:17.876504+0000","last_became_peered":"2023-10-09T10:54:17.876504+0000","last_unstale":"2023-10-09T10:54:50.899681+0000","last_undegraded":"2023-10-09T10:54:50.899681+0000","last_fullsized":"2023-10-09T10:54:50.899681+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:04:48.657295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.898597+0000","last_change":"2023-10-09T10:54:47.913826+0000","last_active":"2023-10-09T10:54:50.898597+0000","last_peered":"2023-10-09T10:54:50.898597+0000","last_clean":"2023-10-09T10:54:50.898597+0000","last_became_active":"2023-10-09T10:54:47.913524+0000","last_became_peered":"2023-10-09T10:54:47.913524+0000","last_unstale":"2023-10-09T10:54:50.898597+0000","last_undegraded":"2023-10-09T10:54:50.898597+0000","last_fullsized":"2023-10-09T10:54:50.898597+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:56:23.699552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"298'5975","reported_seq":6080,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:51.580617+0000","last_change":"2023-10-09T10:54:50.915546+0000","last_active":"2023-10-09T10:54:51.580617+0000","last_peered":"2023-10-09T10:54:51.580617+0000","last_clean":"2023-10-09T10:54:51.580617+0000","last_became_active":"2023-10-09T10:54:50.910131+0000","last_became_peered":"2023-10-09T10:54:50.910131+0000","last_unstale":"2023-10-09T10:54:51.580617+0000","last_undegraded":"2023-10-09T10:54:51.580617+0000","last_fullsized":"2023-10-09T10:54:51.580617+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5975,"log_dups_size":0,"ondisk_log_size":5975,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194215936,"num_objects":2963,"num_object_clones":0,"num_object_copies":8889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3017,"num_write_kb":191392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":2031616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":613,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.923680+0000","last_change":"2023-10-09T10:54:21.152148+0000","last_active":"2023-10-09T10:54:50.923680+0000","last_peered":"2023-10-09T10:54:50.923680+0000","last_clean":"2023-10-09T10:54:50.923680+0000","last_became_active":"2023-10-09T10:54:21.151809+0000","last_became_peered":"2023-10-09T10:54:21.151809+0000","last_unstale":"2023-10-09T10:54:50.923680+0000","last_undegraded":"2023-10-09T10:54:50.923680+0000","last_fullsized":"2023-10-09T10:54:50.923680+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.701128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.701128+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:27.021145+0000","scrub_duration":0.0025071830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":325,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.900715+0000","last_change":"2023-10-09T10:54:48.038946+0000","last_active":"2023-10-09T10:54:49.900715+0000","last_peered":"2023-10-09T10:54:49.900715+0000","last_clean":"2023-10-09T10:54:49.900715+0000","last_became_active":"2023-10-09T10:54:48.001493+0000","last_became_peered":"2023-10-09T10:54:48.001493+0000","last_unstale":"2023-10-09T10:54:49.900715+0000","last_undegraded":"2023-10-09T10:54:49.900715+0000","last_fullsized":"2023-10-09T10:54:49.900715+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:25.197466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.900779+0000","last_change":"2023-10-09T10:54:48.012411+0000","last_active":"2023-10-09T10:54:50.900779+0000","last_peered":"2023-10-09T10:54:50.900779+0000","last_clean":"2023-10-09T10:54:50.900779+0000","last_became_active":"2023-10-09T10:54:48.011894+0000","last_became_peered":"2023-10-09T10:54:48.011894+0000","last_unstale":"2023-10-09T10:54:50.900779+0000","last_undegraded":"2023-10-09T10:54:50.900779+0000","last_fullsized":"2023-10-09T10:54:50.900779+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:28.067337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"298'6226","reported_seq":6276,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:54:53.297013+0000","last_change":"2023-10-09T10:54:19.338847+0000","last_active":"2023-10-09T10:54:53.297013+0000","last_peered":"2023-10-09T10:54:53.297013+0000","last_clean":"2023-10-09T10:54:17.076172+0000","last_became_active":"2023-10-09T10:54:19.327879+0000","last_became_peered":"2023-10-09T10:54:19.327879+0000","last_unstale":"2023-10-09T10:54:53.297013+0000","last_undegraded":"2023-10-09T10:54:18.890724+0000","last_fullsized":"2023-10-09T10:54:18.868702+0000","mapping_epoch":282,"log_start":"271'952","ondisk_log_start":"271'952","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5274,"log_dups_size":0,"ondisk_log_size":5274,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:49.273305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389677056,"num_objects":5946,"num_object_clones":0,"num_object_copies":17838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5292,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6226,"num_write_kb":389504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":654},{"shards":"0,4,7","objects":5152}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":694,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.907123+0000","last_change":"2023-10-09T10:53:34.602531+0000","last_active":"2023-10-09T10:54:49.907123+0000","last_peered":"2023-10-09T10:54:49.907123+0000","last_clean":"2023-10-09T10:54:49.907123+0000","last_became_active":"2023-10-09T10:53:17.841970+0000","last_became_peered":"2023-10-09T10:53:17.841970+0000","last_unstale":"2023-10-09T10:54:49.907123+0000","last_undegraded":"2023-10-09T10:54:49.907123+0000","last_fullsized":"2023-10-09T10:54:49.907123+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:04.601861+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.898910+0000","last_change":"2023-10-09T10:54:47.952216+0000","last_active":"2023-10-09T10:54:50.898910+0000","last_peered":"2023-10-09T10:54:50.898910+0000","last_clean":"2023-10-09T10:54:50.898910+0000","last_became_active":"2023-10-09T10:54:47.941070+0000","last_became_peered":"2023-10-09T10:54:47.941070+0000","last_unstale":"2023-10-09T10:54:50.898910+0000","last_undegraded":"2023-10-09T10:54:50.898910+0000","last_fullsized":"2023-10-09T10:54:50.898910+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:09.075266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":217,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.906046+0000","last_change":"2023-10-09T10:54:17.886921+0000","last_active":"2023-10-09T10:54:49.906046+0000","last_peered":"2023-10-09T10:54:49.906046+0000","last_clean":"2023-10-09T10:54:49.906046+0000","last_became_active":"2023-10-09T10:54:17.886584+0000","last_became_peered":"2023-10-09T10:54:17.886584+0000","last_unstale":"2023-10-09T10:54:49.906046+0000","last_undegraded":"2023-10-09T10:54:49.906046+0000","last_fullsized":"2023-10-09T10:54:49.906046+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:15.848951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.899743+0000","last_change":"2023-10-09T10:54:47.908294+0000","last_active":"2023-10-09T10:54:50.899743+0000","last_peered":"2023-10-09T10:54:50.899743+0000","last_clean":"2023-10-09T10:54:50.899743+0000","last_became_active":"2023-10-09T10:54:47.908019+0000","last_became_peered":"2023-10-09T10:54:47.908019+0000","last_unstale":"2023-10-09T10:54:50.899743+0000","last_undegraded":"2023-10-09T10:54:50.899743+0000","last_fullsized":"2023-10-09T10:54:50.899743+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:48.662764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"298'6119","reported_seq":6171,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:55.150335+0000","last_change":"2023-10-09T10:54:40.642047+0000","last_active":"2023-10-09T10:54:55.150335+0000","last_peered":"2023-10-09T10:54:55.150335+0000","last_clean":"2023-10-09T10:54:55.150335+0000","last_became_active":"2023-10-09T10:54:40.639305+0000","last_became_peered":"2023-10-09T10:54:40.639305+0000","last_unstale":"2023-10-09T10:54:55.150335+0000","last_undegraded":"2023-10-09T10:54:55.150335+0000","last_fullsized":"2023-10-09T10:54:55.150335+0000","mapping_epoch":266,"log_start":"271'1000","ondisk_log_start":"271'1000","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5119,"log_dups_size":0,"ondisk_log_size":5119,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191430656,"num_objects":2921,"num_object_clones":0,"num_object_copies":8763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3129,"num_write_kb":193600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":634,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.903980+0000","last_change":"2023-10-09T10:54:21.163072+0000","last_active":"2023-10-09T10:54:49.903980+0000","last_peered":"2023-10-09T10:54:49.903980+0000","last_clean":"2023-10-09T10:54:49.903980+0000","last_became_active":"2023-10-09T10:54:21.162710+0000","last_became_peered":"2023-10-09T10:54:21.162710+0000","last_unstale":"2023-10-09T10:54:49.903980+0000","last_undegraded":"2023-10-09T10:54:49.903980+0000","last_fullsized":"2023-10-09T10:54:49.903980+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:15.932806+0000","scrub_duration":0.0019146790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.899817+0000","last_change":"2023-10-09T10:54:47.899967+0000","last_active":"2023-10-09T10:54:50.899817+0000","last_peered":"2023-10-09T10:54:50.899817+0000","last_clean":"2023-10-09T10:54:50.899817+0000","last_became_active":"2023-10-09T10:54:47.899584+0000","last_became_peered":"2023-10-09T10:54:47.899584+0000","last_unstale":"2023-10-09T10:54:50.899817+0000","last_undegraded":"2023-10-09T10:54:50.899817+0000","last_fullsized":"2023-10-09T10:54:50.899817+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:43.973868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":207,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.910076+0000","last_change":"2023-10-09T10:50:43.388615+0000","last_active":"2023-10-09T10:54:50.910076+0000","last_peered":"2023-10-09T10:54:50.910076+0000","last_clean":"2023-10-09T10:54:50.910076+0000","last_became_active":"2023-10-09T10:50:43.388176+0000","last_became_peered":"2023-10-09T10:50:43.388176+0000","last_unstale":"2023-10-09T10:54:50.910076+0000","last_undegraded":"2023-10-09T10:54:50.910076+0000","last_fullsized":"2023-10-09T10:54:50.910076+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:41:17.550111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.898830+0000","last_change":"2023-10-09T10:54:47.950462+0000","last_active":"2023-10-09T10:54:50.898830+0000","last_peered":"2023-10-09T10:54:50.898830+0000","last_clean":"2023-10-09T10:54:50.898830+0000","last_became_active":"2023-10-09T10:54:47.940606+0000","last_became_peered":"2023-10-09T10:54:47.940606+0000","last_unstale":"2023-10-09T10:54:50.898830+0000","last_undegraded":"2023-10-09T10:54:50.898830+0000","last_fullsized":"2023-10-09T10:54:50.898830+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:50.557850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"298'6274","reported_seq":6343,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:54:55.231560+0000","last_change":"2023-10-09T10:54:49.182381+0000","last_active":"2023-10-09T10:54:55.231560+0000","last_peered":"2023-10-09T10:54:55.231560+0000","last_clean":"2023-10-09T10:54:02.429830+0000","last_became_active":"2023-10-09T10:54:49.176829+0000","last_became_peered":"2023-10-09T10:54:49.176829+0000","last_unstale":"2023-10-09T10:54:55.231560+0000","last_undegraded":"2023-10-09T10:54:48.956857+0000","last_fullsized":"2023-10-09T10:54:48.892711+0000","mapping_epoch":295,"log_start":"271'1455","ondisk_log_start":"271'1455","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":4819,"log_dups_size":0,"ondisk_log_size":4819,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:04.678256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383778816,"num_objects":5856,"num_object_clones":0,"num_object_copies":17568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1366,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6274,"num_write_kb":388160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"3,4","objects":4490},{"shards":"3,4,7","objects":1365}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":699,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.899665+0000","last_change":"2023-10-09T10:53:35.501606+0000","last_active":"2023-10-09T10:54:49.899665+0000","last_peered":"2023-10-09T10:54:49.899665+0000","last_clean":"2023-10-09T10:54:49.899665+0000","last_became_active":"2023-10-09T10:53:17.842229+0000","last_became_peered":"2023-10-09T10:53:17.842229+0000","last_unstale":"2023-10-09T10:54:49.899665+0000","last_undegraded":"2023-10-09T10:54:49.899665+0000","last_fullsized":"2023-10-09T10:54:49.899665+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:36.948816+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.923626+0000","last_change":"2023-10-09T10:54:17.885030+0000","last_active":"2023-10-09T10:54:50.923626+0000","last_peered":"2023-10-09T10:54:50.923626+0000","last_clean":"2023-10-09T10:54:50.923626+0000","last_became_active":"2023-10-09T10:54:17.884752+0000","last_became_peered":"2023-10-09T10:54:17.884752+0000","last_unstale":"2023-10-09T10:54:50.923626+0000","last_undegraded":"2023-10-09T10:54:50.923626+0000","last_fullsized":"2023-10-09T10:54:50.923626+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:38.548132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.899032+0000","last_change":"2023-10-09T10:54:47.927822+0000","last_active":"2023-10-09T10:54:50.899032+0000","last_peered":"2023-10-09T10:54:50.899032+0000","last_clean":"2023-10-09T10:54:50.899032+0000","last_became_active":"2023-10-09T10:54:47.927455+0000","last_became_peered":"2023-10-09T10:54:47.927455+0000","last_unstale":"2023-10-09T10:54:50.899032+0000","last_undegraded":"2023-10-09T10:54:50.899032+0000","last_fullsized":"2023-10-09T10:54:50.899032+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:11.818203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.900731+0000","last_change":"2023-10-09T10:54:48.012629+0000","last_active":"2023-10-09T10:54:50.900731+0000","last_peered":"2023-10-09T10:54:50.900731+0000","last_clean":"2023-10-09T10:54:50.900731+0000","last_became_active":"2023-10-09T10:54:48.012148+0000","last_became_peered":"2023-10-09T10:54:48.012148+0000","last_unstale":"2023-10-09T10:54:50.900731+0000","last_undegraded":"2023-10-09T10:54:50.900731+0000","last_fullsized":"2023-10-09T10:54:50.900731+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:22.894111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"298'6064","reported_seq":6115,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:53.155835+0000","last_change":"2023-10-09T10:54:40.559886+0000","last_active":"2023-10-09T10:54:53.155835+0000","last_peered":"2023-10-09T10:54:53.155835+0000","last_clean":"2023-10-09T10:54:53.155835+0000","last_became_active":"2023-10-09T10:54:40.557153+0000","last_became_peered":"2023-10-09T10:54:40.557153+0000","last_unstale":"2023-10-09T10:54:53.155835+0000","last_undegraded":"2023-10-09T10:54:53.155835+0000","last_fullsized":"2023-10-09T10:54:53.155835+0000","mapping_epoch":265,"log_start":"270'300","ondisk_log_start":"270'300","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5764,"log_dups_size":0,"ondisk_log_size":5764,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194052096,"num_objects":2961,"num_object_clones":0,"num_object_copies":8883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3079,"num_write_kb":193280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":28,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.922924+0000","last_change":"2023-10-09T10:54:17.884876+0000","last_active":"2023-10-09T10:54:50.922924+0000","last_peered":"2023-10-09T10:54:50.922924+0000","last_clean":"2023-10-09T10:54:50.922924+0000","last_became_active":"2023-10-09T10:54:17.884402+0000","last_became_peered":"2023-10-09T10:54:17.884402+0000","last_unstale":"2023-10-09T10:54:50.922924+0000","last_undegraded":"2023-10-09T10:54:50.922924+0000","last_fullsized":"2023-10-09T10:54:50.922924+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:18.679950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":716,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.898011+0000","last_change":"2023-10-09T10:53:17.844346+0000","last_active":"2023-10-09T10:54:49.898011+0000","last_peered":"2023-10-09T10:54:49.898011+0000","last_clean":"2023-10-09T10:54:49.898011+0000","last_became_active":"2023-10-09T10:53:17.844056+0000","last_became_peered":"2023-10-09T10:53:17.844056+0000","last_unstale":"2023-10-09T10:54:49.898011+0000","last_undegraded":"2023-10-09T10:54:49.898011+0000","last_fullsized":"2023-10-09T10:54:49.898011+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:27:59.443732+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":229,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.900454+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:54:49.900454+0000","last_peered":"2023-10-09T10:54:49.900454+0000","last_clean":"2023-10-09T10:54:49.900454+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:54:49.900454+0000","last_undegraded":"2023-10-09T10:54:49.900454+0000","last_fullsized":"2023-10-09T10:54:49.900454+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":211,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.899562+0000","last_change":"2023-10-09T10:53:14.836523+0000","last_active":"2023-10-09T10:54:50.899562+0000","last_peered":"2023-10-09T10:54:50.899562+0000","last_clean":"2023-10-09T10:54:50.899562+0000","last_became_active":"2023-10-09T10:53:14.836134+0000","last_became_peered":"2023-10-09T10:53:14.836134+0000","last_unstale":"2023-10-09T10:54:50.899562+0000","last_undegraded":"2023-10-09T10:54:50.899562+0000","last_fullsized":"2023-10-09T10:54:50.899562+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:05:24.615198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"298'6185","reported_seq":6985,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:54:55.195161+0000","last_change":"2023-10-09T10:54:49.078027+0000","last_active":"2023-10-09T10:54:55.195161+0000","last_peered":"2023-10-09T10:54:55.195161+0000","last_clean":"2023-10-09T10:54:02.408036+0000","last_became_active":"2023-10-09T10:54:49.069949+0000","last_became_peered":"2023-10-09T10:54:49.069949+0000","last_unstale":"2023-10-09T10:54:55.195161+0000","last_undegraded":"2023-10-09T10:54:49.050557+0000","last_fullsized":"2023-10-09T10:54:48.898155+0000","mapping_epoch":295,"log_start":"271'1372","ondisk_log_start":"271'1372","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":4813,"log_dups_size":0,"ondisk_log_size":4813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:38.092075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378339328,"num_objects":5773,"num_object_clones":0,"num_object_copies":17319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1353,"num_objects_misplaced":5773,"num_objects_unfound":0,"num_objects_dirty":5773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6185,"num_write_kb":382656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":366,"num_bytes_recovered":23986176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":1353},{"shards":"3,5","objects":4420}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":645,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.920314+0000","last_change":"2023-10-09T10:54:50.915855+0000","last_active":"2023-10-09T10:54:50.920314+0000","last_peered":"2023-10-09T10:54:50.920314+0000","last_clean":"2023-10-09T10:54:50.920314+0000","last_became_active":"2023-10-09T10:54:50.914536+0000","last_became_peered":"2023-10-09T10:54:50.914536+0000","last_unstale":"2023-10-09T10:54:50.920314+0000","last_undegraded":"2023-10-09T10:54:50.920314+0000","last_fullsized":"2023-10-09T10:54:50.920314+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.070026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.070026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.899108+0000","last_change":"2023-10-09T10:54:47.997773+0000","last_active":"2023-10-09T10:54:50.899108+0000","last_peered":"2023-10-09T10:54:50.899108+0000","last_clean":"2023-10-09T10:54:50.899108+0000","last_became_active":"2023-10-09T10:54:47.997412+0000","last_became_peered":"2023-10-09T10:54:47.997412+0000","last_unstale":"2023-10-09T10:54:50.899108+0000","last_undegraded":"2023-10-09T10:54:50.899108+0000","last_fullsized":"2023-10-09T10:54:50.899108+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:28:00.547000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":179,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.903757+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:54:49.903757+0000","last_peered":"2023-10-09T10:54:49.903757+0000","last_clean":"2023-10-09T10:54:49.903757+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:54:49.903757+0000","last_undegraded":"2023-10-09T10:54:49.903757+0000","last_fullsized":"2023-10-09T10:54:49.903757+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":242,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.901371+0000","last_change":"2023-10-09T10:54:47.939756+0000","last_active":"2023-10-09T10:54:49.901371+0000","last_peered":"2023-10-09T10:54:49.901371+0000","last_clean":"2023-10-09T10:54:49.901371+0000","last_became_active":"2023-10-09T10:54:47.938928+0000","last_became_peered":"2023-10-09T10:54:47.938928+0000","last_unstale":"2023-10-09T10:54:49.901371+0000","last_undegraded":"2023-10-09T10:54:49.901371+0000","last_fullsized":"2023-10-09T10:54:49.901371+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:05.310929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"298'6290","reported_seq":6361,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:54:55.179152+0000","last_change":"2023-10-09T10:54:49.192234+0000","last_active":"2023-10-09T10:54:55.179152+0000","last_peered":"2023-10-09T10:54:55.179152+0000","last_clean":"2023-10-09T10:54:02.435746+0000","last_became_active":"2023-10-09T10:54:49.186762+0000","last_became_peered":"2023-10-09T10:54:49.186762+0000","last_unstale":"2023-10-09T10:54:55.179152+0000","last_undegraded":"2023-10-09T10:54:49.038902+0000","last_fullsized":"2023-10-09T10:54:48.912839+0000","mapping_epoch":295,"log_start":"271'1471","ondisk_log_start":"271'1471","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":4819,"log_dups_size":0,"ondisk_log_size":4819,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:48.327483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385351680,"num_objects":5880,"num_object_clones":0,"num_object_copies":17640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1330,"num_objects_misplaced":5880,"num_objects_unfound":0,"num_objects_dirty":5880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6290,"num_write_kb":389440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":4550},{"shards":"3,4,6","objects":1328}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":670,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.901632+0000","last_change":"2023-10-09T10:53:27.625180+0000","last_active":"2023-10-09T10:54:49.901632+0000","last_peered":"2023-10-09T10:54:49.901632+0000","last_clean":"2023-10-09T10:54:49.901632+0000","last_became_active":"2023-10-09T10:51:12.366828+0000","last_became_peered":"2023-10-09T10:51:12.366828+0000","last_unstale":"2023-10-09T10:54:49.901632+0000","last_undegraded":"2023-10-09T10:54:49.901632+0000","last_fullsized":"2023-10-09T10:54:49.901632+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.625079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.625079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:59:41.253976+0000","scrub_duration":0.0041067969999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":42,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.901453+0000","last_change":"2023-10-09T10:53:14.833073+0000","last_active":"2023-10-09T10:54:50.901453+0000","last_peered":"2023-10-09T10:54:50.901453+0000","last_clean":"2023-10-09T10:54:50.901453+0000","last_became_active":"2023-10-09T10:53:14.832736+0000","last_became_peered":"2023-10-09T10:53:14.832736+0000","last_unstale":"2023-10-09T10:54:50.901453+0000","last_undegraded":"2023-10-09T10:54:50.901453+0000","last_fullsized":"2023-10-09T10:54:50.901453+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:27.550707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.899062+0000","last_change":"2023-10-09T10:54:47.925711+0000","last_active":"2023-10-09T10:54:50.899062+0000","last_peered":"2023-10-09T10:54:50.899062+0000","last_clean":"2023-10-09T10:54:50.899062+0000","last_became_active":"2023-10-09T10:54:47.924286+0000","last_became_peered":"2023-10-09T10:54:47.924286+0000","last_unstale":"2023-10-09T10:54:50.899062+0000","last_undegraded":"2023-10-09T10:54:50.899062+0000","last_fullsized":"2023-10-09T10:54:50.899062+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:58.379080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":206,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.901013+0000","last_change":"2023-10-09T10:53:14.828291+0000","last_active":"2023-10-09T10:54:49.901013+0000","last_peered":"2023-10-09T10:54:49.901013+0000","last_clean":"2023-10-09T10:54:49.901013+0000","last_became_active":"2023-10-09T10:53:14.827991+0000","last_became_peered":"2023-10-09T10:53:14.827991+0000","last_unstale":"2023-10-09T10:54:49.901013+0000","last_undegraded":"2023-10-09T10:54:49.901013+0000","last_fullsized":"2023-10-09T10:54:49.901013+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:08:14.283112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"298'6199","reported_seq":6242,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:53.209361+0000","last_change":"2023-10-09T10:53:14.822279+0000","last_active":"2023-10-09T10:54:53.209361+0000","last_peered":"2023-10-09T10:54:53.209361+0000","last_clean":"2023-10-09T10:54:53.209361+0000","last_became_active":"2023-10-09T10:53:14.821952+0000","last_became_peered":"2023-10-09T10:53:14.821952+0000","last_unstale":"2023-10-09T10:54:53.209361+0000","last_undegraded":"2023-10-09T10:54:53.209361+0000","last_fullsized":"2023-10-09T10:54:53.209361+0000","mapping_epoch":266,"log_start":"271'841","ondisk_log_start":"271'841","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5358,"log_dups_size":0,"ondisk_log_size":5358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:37:14.124652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390266880,"num_objects":5955,"num_object_clones":0,"num_object_copies":17865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6199,"num_write_kb":388928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":694,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.899485+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T10:54:50.899485+0000","last_peered":"2023-10-09T10:54:50.899485+0000","last_clean":"2023-10-09T10:54:50.899485+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T10:54:50.899485+0000","last_undegraded":"2023-10-09T10:54:50.899485+0000","last_fullsized":"2023-10-09T10:54:50.899485+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":320,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.907348+0000","last_change":"2023-10-09T10:53:14.841466+0000","last_active":"2023-10-09T10:54:49.907348+0000","last_peered":"2023-10-09T10:54:49.907348+0000","last_clean":"2023-10-09T10:54:49.907348+0000","last_became_active":"2023-10-09T10:53:14.831674+0000","last_became_peered":"2023-10-09T10:53:14.831674+0000","last_unstale":"2023-10-09T10:54:49.907348+0000","last_undegraded":"2023-10-09T10:54:49.907348+0000","last_fullsized":"2023-10-09T10:54:49.907348+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:02.612069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":14,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.898734+0000","last_change":"2023-10-09T10:54:47.908305+0000","last_active":"2023-10-09T10:54:50.898734+0000","last_peered":"2023-10-09T10:54:50.898734+0000","last_clean":"2023-10-09T10:54:50.898734+0000","last_became_active":"2023-10-09T10:54:47.907999+0000","last_became_peered":"2023-10-09T10:54:47.907999+0000","last_unstale":"2023-10-09T10:54:50.898734+0000","last_undegraded":"2023-10-09T10:54:50.898734+0000","last_fullsized":"2023-10-09T10:54:50.898734+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:48.405124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":319,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.900776+0000","last_change":"2023-10-09T10:53:14.835841+0000","last_active":"2023-10-09T10:54:49.900776+0000","last_peered":"2023-10-09T10:54:49.900776+0000","last_clean":"2023-10-09T10:54:49.900776+0000","last_became_active":"2023-10-09T10:53:14.835551+0000","last_became_peered":"2023-10-09T10:53:14.835551+0000","last_unstale":"2023-10-09T10:54:49.900776+0000","last_undegraded":"2023-10-09T10:54:49.900776+0000","last_fullsized":"2023-10-09T10:54:49.900776+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:21.498099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"299'6221","reported_seq":8209,"reported_epoch":299,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T10:54:56.461713+0000","last_change":"2023-10-09T10:54:39.800544+0000","last_active":"2023-10-09T10:54:56.461713+0000","last_peered":"2023-10-09T10:54:56.461713+0000","last_clean":"2023-10-09T10:54:17.082254+0000","last_became_active":"2023-10-09T10:54:19.319875+0000","last_became_peered":"2023-10-09T10:54:19.319875+0000","last_unstale":"2023-10-09T10:54:56.461713+0000","last_undegraded":"2023-10-09T10:54:56.461713+0000","last_fullsized":"2023-10-09T10:54:18.874134+0000","mapping_epoch":282,"log_start":"271'949","ondisk_log_start":"271'949","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5272,"log_dups_size":0,"ondisk_log_size":5272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:54.083267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375259160,"num_objects":5727,"num_object_clones":0,"num_object_copies":17181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4176,"num_objects_unfound":0,"num_objects_dirty":5727,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6221,"num_write_kb":382273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":969,"num_bytes_recovered":63504384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[5,7],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6,7","objects":4176},{"shards":"5,7","objects":1551}],"blocked_by":[],"up_primary":2,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":688,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.904250+0000","last_change":"2023-10-09T10:53:30.567189+0000","last_active":"2023-10-09T10:54:49.904250+0000","last_peered":"2023-10-09T10:54:49.904250+0000","last_clean":"2023-10-09T10:54:49.904250+0000","last_became_active":"2023-10-09T10:53:17.840928+0000","last_became_peered":"2023-10-09T10:53:17.840928+0000","last_unstale":"2023-10-09T10:54:49.904250+0000","last_undegraded":"2023-10-09T10:54:49.904250+0000","last_fullsized":"2023-10-09T10:54:49.904250+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:52.176229+0000","scrub_duration":0.0053707060000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.925716+0000","last_change":"2023-10-09T10:54:17.879705+0000","last_active":"2023-10-09T10:54:50.925716+0000","last_peered":"2023-10-09T10:54:50.925716+0000","last_clean":"2023-10-09T10:54:50.925716+0000","last_became_active":"2023-10-09T10:54:17.879390+0000","last_became_peered":"2023-10-09T10:54:17.879390+0000","last_unstale":"2023-10-09T10:54:50.925716+0000","last_undegraded":"2023-10-09T10:54:50.925716+0000","last_fullsized":"2023-10-09T10:54:50.925716+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:24.062107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":226,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.901463+0000","last_change":"2023-10-09T10:54:48.016159+0000","last_active":"2023-10-09T10:54:49.901463+0000","last_peered":"2023-10-09T10:54:49.901463+0000","last_clean":"2023-10-09T10:54:49.901463+0000","last_became_active":"2023-10-09T10:54:47.939409+0000","last_became_peered":"2023-10-09T10:54:47.939409+0000","last_unstale":"2023-10-09T10:54:49.901463+0000","last_undegraded":"2023-10-09T10:54:49.901463+0000","last_fullsized":"2023-10-09T10:54:49.901463+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:11:15.175207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":240,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.899602+0000","last_change":"2023-10-09T10:51:50.371282+0000","last_active":"2023-10-09T10:54:50.899602+0000","last_peered":"2023-10-09T10:54:50.899602+0000","last_clean":"2023-10-09T10:54:50.899602+0000","last_became_active":"2023-10-09T10:51:50.370973+0000","last_became_peered":"2023-10-09T10:51:50.370973+0000","last_unstale":"2023-10-09T10:54:50.899602+0000","last_undegraded":"2023-10-09T10:54:50.899602+0000","last_fullsized":"2023-10-09T10:54:50.899602+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:15.603164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"298'6180","reported_seq":9047,"reported_epoch":298,"state":"active+undersized+remapped","last_fresh":"2023-10-09T10:54:55.138275+0000","last_change":"2023-10-09T10:54:54.967936+0000","last_active":"2023-10-09T10:54:55.138275+0000","last_peered":"2023-10-09T10:54:55.138275+0000","last_clean":"2023-10-09T10:54:02.435878+0000","last_became_active":"2023-10-09T10:54:48.040150+0000","last_became_peered":"2023-10-09T10:54:48.040150+0000","last_unstale":"2023-10-09T10:54:55.138275+0000","last_undegraded":"2023-10-09T10:54:55.138275+0000","last_fullsized":"2023-10-09T10:54:47.890793+0000","mapping_epoch":294,"log_start":"271'1100","ondisk_log_start":"271'1100","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5080,"log_dups_size":0,"ondisk_log_size":5080,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191922176,"num_objects":2929,"num_object_clones":0,"num_object_copies":8787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3163,"num_write_kb":194912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1061,"num_bytes_recovered":69500928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"3,5","objects":2929}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":688,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.899900+0000","last_change":"2023-10-09T10:53:29.568199+0000","last_active":"2023-10-09T10:54:49.899900+0000","last_peered":"2023-10-09T10:54:49.899900+0000","last_clean":"2023-10-09T10:54:49.899900+0000","last_became_active":"2023-10-09T10:52:55.416357+0000","last_became_peered":"2023-10-09T10:52:55.416357+0000","last_unstale":"2023-10-09T10:54:49.899900+0000","last_undegraded":"2023-10-09T10:54:49.899900+0000","last_fullsized":"2023-10-09T10:54:49.899900+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:44.364650+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":205,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.901098+0000","last_change":"2023-10-09T10:53:14.823731+0000","last_active":"2023-10-09T10:54:49.901098+0000","last_peered":"2023-10-09T10:54:49.901098+0000","last_clean":"2023-10-09T10:54:49.901098+0000","last_became_active":"2023-10-09T10:53:14.823470+0000","last_became_peered":"2023-10-09T10:53:14.823470+0000","last_unstale":"2023-10-09T10:54:49.901098+0000","last_undegraded":"2023-10-09T10:54:49.901098+0000","last_fullsized":"2023-10-09T10:54:49.901098+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:36.927599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":37,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.923218+0000","last_change":"2023-10-09T10:54:47.944380+0000","last_active":"2023-10-09T10:54:50.923218+0000","last_peered":"2023-10-09T10:54:50.923218+0000","last_clean":"2023-10-09T10:54:50.923218+0000","last_became_active":"2023-10-09T10:54:47.943766+0000","last_became_peered":"2023-10-09T10:54:47.943766+0000","last_unstale":"2023-10-09T10:54:50.923218+0000","last_undegraded":"2023-10-09T10:54:50.923218+0000","last_fullsized":"2023-10-09T10:54:50.923218+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:16:11.869603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"298'6072","reported_seq":6149,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:54:55.121696+0000","last_change":"2023-10-09T10:54:49.184288+0000","last_active":"2023-10-09T10:54:55.121696+0000","last_peered":"2023-10-09T10:54:55.121696+0000","last_clean":"2023-10-09T10:54:02.391425+0000","last_became_active":"2023-10-09T10:54:49.118164+0000","last_became_peered":"2023-10-09T10:54:49.118164+0000","last_unstale":"2023-10-09T10:54:55.121696+0000","last_undegraded":"2023-10-09T10:54:48.969517+0000","last_fullsized":"2023-10-09T10:54:48.892012+0000","mapping_epoch":295,"log_start":"271'1000","ondisk_log_start":"271'1000","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5072,"log_dups_size":0,"ondisk_log_size":5072,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192118784,"num_objects":2932,"num_object_clones":0,"num_object_copies":8796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":727,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3096,"num_write_kb":192864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"3,4,6","objects":727},{"shards":"3,6","objects":2205}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":620,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.906530+0000","last_change":"2023-10-09T10:53:36.563918+0000","last_active":"2023-10-09T10:54:50.906530+0000","last_peered":"2023-10-09T10:54:50.906530+0000","last_clean":"2023-10-09T10:54:50.906530+0000","last_became_active":"2023-10-09T10:53:17.842952+0000","last_became_peered":"2023-10-09T10:53:17.842952+0000","last_unstale":"2023-10-09T10:54:50.906530+0000","last_undegraded":"2023-10-09T10:54:50.906530+0000","last_fullsized":"2023-10-09T10:54:50.906530+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:53.185917+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":125,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.900208+0000","last_change":"2023-10-09T10:54:47.923386+0000","last_active":"2023-10-09T10:54:50.900208+0000","last_peered":"2023-10-09T10:54:50.900208+0000","last_clean":"2023-10-09T10:54:50.900208+0000","last_became_active":"2023-10-09T10:54:47.914257+0000","last_became_peered":"2023-10-09T10:54:47.914257+0000","last_unstale":"2023-10-09T10:54:50.900208+0000","last_undegraded":"2023-10-09T10:54:50.900208+0000","last_fullsized":"2023-10-09T10:54:50.900208+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:24:18.190715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":314,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.903728+0000","last_change":"2023-10-09T10:54:47.977518+0000","last_active":"2023-10-09T10:54:49.903728+0000","last_peered":"2023-10-09T10:54:49.903728+0000","last_clean":"2023-10-09T10:54:49.903728+0000","last_became_active":"2023-10-09T10:54:47.977332+0000","last_became_peered":"2023-10-09T10:54:47.977332+0000","last_unstale":"2023-10-09T10:54:49.903728+0000","last_undegraded":"2023-10-09T10:54:49.903728+0000","last_fullsized":"2023-10-09T10:54:49.903728+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:29.620839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.16","version":"298'6077","reported_seq":6129,"reported_epoch":298,"state":"active+recovery_wait","last_fresh":"2023-10-09T10:54:53.093981+0000","last_change":"2023-10-09T10:54:40.600782+0000","last_active":"2023-10-09T10:54:53.093981+0000","last_peered":"2023-10-09T10:54:53.093981+0000","last_clean":"2023-10-09T10:54:39.522361+0000","last_became_active":"2023-10-09T10:54:40.590969+0000","last_became_peered":"2023-10-09T10:54:40.590969+0000","last_unstale":"2023-10-09T10:54:53.093981+0000","last_undegraded":"2023-10-09T10:54:53.093981+0000","last_fullsized":"2023-10-09T10:54:53.093981+0000","mapping_epoch":289,"log_start":"270'300","ondisk_log_start":"270'300","created":289,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5777,"log_dups_size":0,"ondisk_log_size":5777,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195592192,"num_objects":2985,"num_object_clones":0,"num_object_copies":8955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3083,"num_write_kb":194144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2985}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":249,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.896847+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:54:50.896847+0000","last_peered":"2023-10-09T10:54:50.896847+0000","last_clean":"2023-10-09T10:54:50.896847+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:54:50.896847+0000","last_undegraded":"2023-10-09T10:54:50.896847+0000","last_fullsized":"2023-10-09T10:54:50.896847+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":241,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.902134+0000","last_change":"2023-10-09T10:50:43.385150+0000","last_active":"2023-10-09T10:54:49.902134+0000","last_peered":"2023-10-09T10:54:49.902134+0000","last_clean":"2023-10-09T10:54:49.902134+0000","last_became_active":"2023-10-09T10:50:43.384788+0000","last_became_peered":"2023-10-09T10:50:43.384788+0000","last_unstale":"2023-10-09T10:54:49.902134+0000","last_undegraded":"2023-10-09T10:54:49.902134+0000","last_fullsized":"2023-10-09T10:54:49.902134+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:36.085910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.15","version":"299'6078","reported_seq":6133,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-10-09T10:54:56.074047+0000","last_change":"2023-10-09T10:54:40.578264+0000","last_active":"2023-10-09T10:54:56.074047+0000","last_peered":"2023-10-09T10:54:56.074047+0000","last_clean":"2023-10-09T10:54:56.074047+0000","last_became_active":"2023-10-09T10:54:40.568498+0000","last_became_peered":"2023-10-09T10:54:40.568498+0000","last_unstale":"2023-10-09T10:54:56.074047+0000","last_undegraded":"2023-10-09T10:54:56.074047+0000","last_fullsized":"2023-10-09T10:54:56.074047+0000","mapping_epoch":289,"log_start":"271'400","ondisk_log_start":"271'400","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5678,"log_dups_size":0,"ondisk_log_size":5678,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187629568,"num_objects":2863,"num_object_clones":0,"num_object_copies":8589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3117,"num_write_kb":191360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":567,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.916407+0000","last_change":"2023-10-09T10:54:50.910602+0000","last_active":"2023-10-09T10:54:50.916407+0000","last_peered":"2023-10-09T10:54:50.916407+0000","last_clean":"2023-10-09T10:54:50.916407+0000","last_became_active":"2023-10-09T10:54:50.910103+0000","last_became_peered":"2023-10-09T10:54:50.910103+0000","last_unstale":"2023-10-09T10:54:50.916407+0000","last_undegraded":"2023-10-09T10:54:50.916407+0000","last_fullsized":"2023-10-09T10:54:50.916407+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:46.305288+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:46.305288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":321,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.908077+0000","last_change":"2023-10-09T10:54:17.886158+0000","last_active":"2023-10-09T10:54:49.908077+0000","last_peered":"2023-10-09T10:54:49.908077+0000","last_clean":"2023-10-09T10:54:49.908077+0000","last_became_active":"2023-10-09T10:54:17.885816+0000","last_became_peered":"2023-10-09T10:54:17.885816+0000","last_unstale":"2023-10-09T10:54:49.908077+0000","last_undegraded":"2023-10-09T10:54:49.908077+0000","last_fullsized":"2023-10-09T10:54:49.908077+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:05.270825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":324,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.907449+0000","last_change":"2023-10-09T10:54:17.957784+0000","last_active":"2023-10-09T10:54:49.907449+0000","last_peered":"2023-10-09T10:54:49.907449+0000","last_clean":"2023-10-09T10:54:49.907449+0000","last_became_active":"2023-10-09T10:54:17.954788+0000","last_became_peered":"2023-10-09T10:54:17.954788+0000","last_unstale":"2023-10-09T10:54:49.907449+0000","last_undegraded":"2023-10-09T10:54:49.907449+0000","last_fullsized":"2023-10-09T10:54:49.907449+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:48.709390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"299'6126","reported_seq":6192,"reported_epoch":299,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:54:56.253870+0000","last_change":"2023-10-09T10:54:42.558777+0000","last_active":"2023-10-09T10:54:56.253870+0000","last_peered":"2023-10-09T10:54:56.253870+0000","last_clean":"2023-10-09T10:54:17.036595+0000","last_became_active":"2023-10-09T10:54:42.553632+0000","last_became_peered":"2023-10-09T10:54:42.553632+0000","last_unstale":"2023-10-09T10:54:56.253870+0000","last_undegraded":"2023-10-09T10:54:56.253870+0000","last_fullsized":"2023-10-09T10:54:56.253870+0000","mapping_epoch":291,"log_start":"271'600","ondisk_log_start":"271'600","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5526,"log_dups_size":0,"ondisk_log_size":5526,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189300736,"num_objects":2889,"num_object_clones":0,"num_object_copies":8667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2889,"num_objects_unfound":0,"num_objects_dirty":2889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3141,"num_write_kb":192928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":2889}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":572,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.916504+0000","last_change":"2023-10-09T10:54:50.914473+0000","last_active":"2023-10-09T10:54:50.916504+0000","last_peered":"2023-10-09T10:54:50.916504+0000","last_clean":"2023-10-09T10:54:50.916504+0000","last_became_active":"2023-10-09T10:54:50.914005+0000","last_became_peered":"2023-10-09T10:54:50.914005+0000","last_unstale":"2023-10-09T10:54:50.916504+0000","last_undegraded":"2023-10-09T10:54:50.916504+0000","last_fullsized":"2023-10-09T10:54:50.916504+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.070492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.070492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":206,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T10:54:49.898591+0000","last_change":"2023-10-09T10:54:18.089053+0000","last_active":"2023-10-09T10:54:49.898591+0000","last_peered":"2023-10-09T10:54:49.898591+0000","last_clean":"2023-10-09T10:54:49.898591+0000","last_became_active":"2023-10-09T10:54:17.886070+0000","last_became_peered":"2023-10-09T10:54:17.886070+0000","last_unstale":"2023-10-09T10:54:49.898591+0000","last_undegraded":"2023-10-09T10:54:49.898591+0000","last_fullsized":"2023-10-09T10:54:49.898591+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:35.628407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":303,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T10:54:50.899099+0000","last_change":"2023-10-09T10:54:47.908059+0000","last_active":"2023-10-09T10:54:50.899099+0000","last_peered":"2023-10-09T10:54:50.899099+0000","last_clean":"2023-10-09T10:54:50.899099+0000","last_became_active":"2023-10-09T10:54:47.907780+0000","last_became_peered":"2023-10-09T10:54:47.907780+0000","last_unstale":"2023-10-09T10:54:50.899099+0000","last_undegraded":"2023-10-09T10:54:50.899099+0000","last_fullsized":"2023-10-09T10:54:50.899099+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:55:59.064617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":6758,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":6142033944,"num_objects":93721,"num_object_clones":0,"num_object_copies":281163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17602,"num_objects_misplaced":21592,"num_objects_unfound":0,"num_objects_dirty":93721,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":99583,"num_write_kb":6185665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3191,"num_bytes_recovered":209125376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18648674304,"data_stored":18648662088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141070,"ondisk_log_size":141070,"up":78,"acting":67,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":675346,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":71,"num_read_kb":132,"num_write":35,"num_write_kb":348,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":681461,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332340,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2592344,"kb_used_data":2290960,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91152808,"statfs":{"total":95995035648,"available":93340475392,"internally_reserved":0,"allocated":2345943040,"data_stored":2344863184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:54:53 2023","interfaces":[{"interface":"back","average":{"1min":2.8740000000000001,"5min":1.913,"15min":1.2609999999999999},"min":{"1min":0.318,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":9.6259999999999994,"5min":9.6379999999999999,"15min":9.6379999999999999},"last":0.318},{"interface":"front","average":{"1min":2.3100000000000001,"5min":2.0059999999999998,"15min":1.4350000000000001},"min":{"1min":0.30099999999999999,"5min":0.246,"15min":0.24299999999999999},"max":{"1min":7.7859999999999996,"5min":16.731000000000002,"15min":16.731000000000002},"last":2.7160000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.98,"5min":2.0270000000000001,"15min":1.492},"min":{"1min":0.32100000000000001,"5min":0.27000000000000002,"15min":0.25700000000000001},"max":{"1min":8.4109999999999996,"5min":34.756,"15min":34.756},"last":1.4019999999999999},{"interface":"front","average":{"1min":1.097,"5min":1.7110000000000001,"15min":1.242},"min":{"1min":0.28999999999999998,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":4.1719999999999997,"5min":35.658999999999999,"15min":35.658999999999999},"last":1.482}]},{"osd":2,"last update":"Mon Oct 9 10:54:29 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":1.512,"15min":1.2529999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.3889999999999998,"5min":8.3620000000000001,"15min":11.058},"last":2.5369999999999999},{"interface":"front","average":{"1min":2.27,"5min":1.6020000000000001,"15min":1.1739999999999999},"min":{"1min":0.34499999999999997,"5min":0.32400000000000001,"15min":0.251},"max":{"1min":11.579000000000001,"5min":11.579000000000001,"15min":11.579000000000001},"last":0.26600000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:54:53 2023","interfaces":[{"interface":"back","average":{"1min":2.4169999999999998,"5min":2.1230000000000002,"15min":1.607},"min":{"1min":0.36099999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":10.069000000000001,"5min":10.069000000000001,"15min":11.523},"last":1.298},{"interface":"front","average":{"1min":2.1429999999999998,"5min":1.7110000000000001,"15min":1.4330000000000001},"min":{"1min":0.38400000000000001,"5min":0.34399999999999997,"15min":0.32800000000000001},"max":{"1min":8.6300000000000008,"5min":9.9329999999999998,"15min":19.234999999999999},"last":2.415}]},{"osd":4,"last update":"Mon Oct 9 10:54:33 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":1.6950000000000001,"15min":1.2090000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":8.2940000000000005,"5min":16.695,"15min":16.695},"last":0.24099999999999999},{"interface":"front","average":{"1min":1.123,"5min":1.4750000000000001,"15min":1.089},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26200000000000001},"max":{"1min":7.9320000000000004,"5min":9.0459999999999994,"15min":11.109999999999999},"last":0.28699999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:54:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.5840000000000001,"15min":1.1539999999999999},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.247},"max":{"1min":4.5999999999999996,"5min":13.262,"15min":13.262},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.4670000000000001,"5min":1.6799999999999999,"15min":1.3340000000000001},"min":{"1min":0.318,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":8.4339999999999993,"5min":8.8659999999999997,"15min":11.741},"last":0.34200000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:54:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":1.958,"15min":1.423},"min":{"1min":0.34499999999999997,"5min":0.317,"15min":0.28199999999999997},"max":{"1min":8.3610000000000007,"5min":14.913,"15min":14.913},"last":0.371},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.73,"15min":1.262},"min":{"1min":0.317,"5min":0.317,"15min":0.29399999999999998},"max":{"1min":7.9809999999999999,"5min":12.268000000000001,"15min":19.187999999999999},"last":0.32900000000000001}]}]},{"osd":6,"up_from":44,"seq":188978561271,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3540688,"kb_used_data":2640520,"kb_used_omap":0,"kb_used_meta":900160,"kb_avail":90204464,"statfs":{"total":95995035648,"available":92369371136,"internally_reserved":0,"allocated":2703892480,"data_stored":2702828534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921763840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:54:54 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":1.331,"15min":1.181},"min":{"1min":0.252,"5min":0.216,"15min":0.216},"max":{"1min":5.2140000000000004,"5min":7.0170000000000003,"15min":12.288},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.603,"15min":1.2509999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.22800000000000001},"max":{"1min":4.8689999999999998,"5min":11.845000000000001,"15min":12.369999999999999},"last":0.85899999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:54:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.3660000000000001,"15min":1.2130000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.0230000000000001,"5min":8.4730000000000008,"15min":16.747},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.4159999999999999,"15min":1.252},"min":{"1min":0.26400000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.7869999999999999,"5min":8.0579999999999998,"15min":16.734000000000002},"last":0.41299999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:54:32 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":1.7490000000000001,"15min":1.294},"min":{"1min":0.30099999999999999,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":8.6389999999999993,"5min":8.6389999999999993,"15min":13.208},"last":6.4109999999999996},{"interface":"front","average":{"1min":2.1629999999999998,"5min":1.992,"15min":1.4710000000000001},"min":{"1min":0.31,"5min":0.29999999999999999,"15min":0.28399999999999997},"max":{"1min":12.83,"5min":12.83,"15min":12.83},"last":4.798}]},{"osd":3,"last update":"Mon Oct 9 10:54:54 2023","interfaces":[{"interface":"back","average":{"1min":1.667,"5min":1.871,"15min":1.3640000000000001},"min":{"1min":0.39500000000000002,"5min":0.29799999999999999,"15min":0.255},"max":{"1min":4.8380000000000001,"5min":7.8250000000000002,"15min":12.391},"last":1.8280000000000001},{"interface":"front","average":{"1min":1.905,"5min":2.2050000000000001,"15min":1.5569999999999999},"min":{"1min":0.35099999999999998,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":5.0430000000000001,"5min":7.327,"15min":13.179},"last":0.72399999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:54:26 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.5620000000000001,"15min":1.1060000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.23200000000000001},"max":{"1min":5.968,"5min":9.0600000000000005,"15min":9.4060000000000006},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.54,"5min":1.47,"15min":1.113},"min":{"1min":0.308,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":4.7930000000000001,"5min":12.061999999999999,"15min":12.481999999999999},"last":0.42699999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:54:26 2023","interfaces":[{"interface":"back","average":{"1min":1.8979999999999999,"5min":1.802,"15min":1.244},"min":{"1min":0.30299999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.5589999999999993,"5min":9.3290000000000006,"15min":10.497},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.5920000000000001,"5min":1.599,"15min":1.1759999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.27600000000000002},"max":{"1min":6.7519999999999998,"5min":12.103999999999999,"15min":12.456},"last":0.38700000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:54:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3759999999999999,"5min":1.748,"15min":1.335},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.26600000000000001},"max":{"1min":12.882,"5min":12.882,"15min":23.358000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.2569999999999999,"15min":1.123},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.34100000000000003},"max":{"1min":6.1029999999999998,"5min":7.0880000000000001,"15min":12.422000000000001},"last":0.46800000000000003}]}]},{"osd":1,"up_from":85,"seq":365072220309,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2574260,"kb_used_data":2217900,"kb_used_omap":9,"kb_used_meta":356342,"kb_avail":91170892,"statfs":{"total":95995035648,"available":93358993408,"internally_reserved":0,"allocated":2271129600,"data_stored":2269562489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":364895209},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,0,0,0,2,4,9],"upper_bound":1024},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.575,"15min":1.3600000000000001},"min":{"1min":0.20999999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":8.093,"5min":8.7469999999999999,"15min":18.027000000000001},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.7070000000000001,"15min":1.3759999999999999},"min":{"1min":0.247,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":2.8820000000000001,"5min":16.739000000000001,"15min":16.739000000000001},"last":0.63500000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:54:24 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":1.383,"15min":1.1930000000000001},"min":{"1min":0.223,"5min":0.222,"15min":0.19700000000000001},"max":{"1min":10.032999999999999,"5min":10.032999999999999,"15min":15.356},"last":0.505},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.2629999999999999,"15min":1.2789999999999999},"min":{"1min":0.255,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.6349999999999998,"5min":10.281000000000001,"15min":14.234},"last":0.41099999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:53:55 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":1.9550000000000001,"15min":1.532},"min":{"1min":0.249,"5min":0.223,"15min":0.218},"max":{"1min":3.6960000000000002,"5min":13.885999999999999,"15min":18.274999999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.0389999999999999,"5min":2.0920000000000001,"15min":1.45},"min":{"1min":0.25,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.2429999999999999,"5min":14.169,"15min":15.468999999999999},"last":0.90400000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":1.486,"15min":1.2030000000000001},"min":{"1min":0.33900000000000002,"5min":0.255,"15min":0.255},"max":{"1min":5.133,"5min":7.2750000000000004,"15min":12.167999999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.681,"15min":1.3660000000000001},"min":{"1min":0.36299999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.7839999999999998,"5min":8.9969999999999999,"15min":14.228999999999999},"last":0.60299999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.5449999999999999,"15min":1.2030000000000001},"min":{"1min":0.32700000000000001,"5min":0.28100000000000003,"15min":0.253},"max":{"1min":2.589,"5min":7.9299999999999997,"15min":9.7720000000000002},"last":0.872},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.8260000000000001,"15min":1.4650000000000001},"min":{"1min":0.30299999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.1989999999999998,"5min":9.1820000000000004,"15min":17.085000000000001},"last":0.54300000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":1.6870000000000001,"15min":1.3220000000000001},"min":{"1min":0.34799999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.8620000000000001,"5min":10.425000000000001,"15min":10.425000000000001},"last":0.83699999999999997},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.6699999999999999,"15min":1.349},"min":{"1min":0.41299999999999998,"5min":0.312,"15min":0.312},"max":{"1min":5.173,"5min":7.7480000000000002,"15min":12.231},"last":0.61399999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.498,"5min":1.7829999999999999,"15min":1.3819999999999999},"min":{"1min":0.38500000000000001,"5min":0.34300000000000003,"15min":0.34000000000000002},"max":{"1min":5.0609999999999999,"5min":16.312000000000001,"15min":16.312000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.6319999999999999,"15min":1.395},"min":{"1min":0.38500000000000001,"5min":0.35799999999999998,"15min":0.32500000000000001},"max":{"1min":5.7549999999999999,"5min":8.6959999999999997,"15min":14.273999999999999},"last":0.85399999999999998}]}]},{"osd":0,"up_from":80,"seq":343597383843,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2769472,"kb_used_data":2296248,"kb_used_omap":28,"kb_used_meta":472995,"kb_avail":90975680,"statfs":{"total":95995035648,"available":93159096320,"internally_reserved":0,"allocated":2351357952,"data_stored":2349715916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":484347728},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.538,"15min":1.3879999999999999},"min":{"1min":0.20399999999999999,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":3.9849999999999999,"5min":14.311,"15min":25.475000000000001},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.85799999999999998,"5min":1.5609999999999999,"15min":1.3959999999999999},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":3.7269999999999999,"5min":9.359,"15min":20.832999999999998},"last":0.317}]},{"osd":2,"last update":"Mon Oct 9 10:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":1.837,"15min":1.381},"min":{"1min":0.252,"5min":0.23100000000000001,"15min":0.19700000000000001},"max":{"1min":12.609999999999999,"5min":17.927,"15min":17.927},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.8620000000000001,"5min":2.0339999999999998,"15min":1.5409999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.20399999999999999},"max":{"1min":8.7119999999999997,"5min":14.111000000000001,"15min":14.391999999999999},"last":0.34300000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.9970000000000001,"15min":1.661},"min":{"1min":0.23599999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.698,"5min":13.166,"15min":14.926},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.1919999999999999,"5min":2.0129999999999999,"15min":1.5840000000000001},"min":{"1min":0.25800000000000001,"5min":0.23499999999999999,"15min":0.224},"max":{"1min":3.8820000000000001,"5min":14.391999999999999,"15min":14.391999999999999},"last":0.51000000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:54:50 2023","interfaces":[{"interface":"back","average":{"1min":6.3310000000000004,"5min":2.702,"15min":1.7430000000000001},"min":{"1min":0.38300000000000001,"5min":0.33800000000000002,"15min":0.26200000000000001},"max":{"1min":77.177999999999997,"5min":77.177999999999997,"15min":77.177999999999997},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.716,"5min":2.0310000000000001,"15min":1.6299999999999999},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.427,"5min":10.427,"15min":12.318},"last":0.76400000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:54:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4329999999999998,"5min":2.2160000000000002,"15min":1.665},"min":{"1min":0.35299999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.478,"5min":10.478,"15min":12.157},"last":0.79400000000000004},{"interface":"front","average":{"1min":2.5779999999999998,"5min":2.1539999999999999,"15min":1.573},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.29499999999999998},"max":{"1min":24.754999999999999,"5min":24.754999999999999,"15min":24.754999999999999},"last":0.85699999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:54:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":2.2930000000000001,"15min":1.746},"min":{"1min":0.38200000000000001,"5min":0.34200000000000003,"15min":0.30399999999999999},"max":{"1min":10.384,"5min":10.384,"15min":12.207000000000001},"last":0.82499999999999996},{"interface":"front","average":{"1min":3.3180000000000001,"5min":2.1680000000000001,"15min":1.4690000000000001},"min":{"1min":0.48399999999999999,"5min":0.307,"15min":0.307},"max":{"1min":24.800000000000001,"5min":24.800000000000001,"15min":24.800000000000001},"last":0.91600000000000004}]},{"osd":7,"last update":"Mon Oct 9 10:54:50 2023","interfaces":[{"interface":"back","average":{"1min":2.379,"5min":2.2210000000000001,"15min":1.641},"min":{"1min":0.48499999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":13.225,"5min":14.48,"15min":14.48},"last":0.88800000000000001},{"interface":"front","average":{"1min":6.2060000000000004,"5min":2.8149999999999999,"15min":1.873},"min":{"1min":0.42799999999999999,"5min":0.32600000000000001,"15min":0.32200000000000001},"max":{"1min":77.228999999999999,"5min":77.228999999999999,"15min":77.228999999999999},"last":0.745}]}]},{"osd":2,"up_from":90,"seq":386547056784,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":476544,"kb_used_data":239352,"kb_used_omap":81,"kb_used_meta":237102,"kb_avail":93268608,"statfs":{"total":95995035648,"available":95507054592,"internally_reserved":0,"allocated":245096448,"data_stored":243630396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":242792666},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.595,"15min":1.2589999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":5.4039999999999999,"5min":18.512,"15min":18.512},"last":5.6159999999999997},{"interface":"front","average":{"1min":1.49,"5min":1.8240000000000001,"15min":1.4139999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":6.7560000000000002,"5min":15.952999999999999,"15min":17.975000000000001},"last":11.167}]},{"osd":1,"last update":"Mon Oct 9 10:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":1.546,"15min":1.389},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20200000000000001},"max":{"1min":6.8179999999999996,"5min":8.4740000000000002,"15min":15.247999999999999},"last":11.204000000000001},{"interface":"front","average":{"1min":1.55,"5min":1.1180000000000001,"15min":1.175},"min":{"1min":0.25600000000000001,"5min":0.253,"15min":0.20699999999999999},"max":{"1min":4.6020000000000003,"5min":6.3789999999999996,"15min":16.454999999999998},"last":10.361000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:54:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":1.3460000000000001,"15min":1.2949999999999999},"min":{"1min":0.28100000000000003,"5min":0.26900000000000002,"15min":0.22500000000000001},"max":{"1min":7.9669999999999996,"5min":7.9669999999999996,"15min":14.113},"last":1.78},{"interface":"front","average":{"1min":1.292,"5min":1.7709999999999999,"15min":1.6000000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.20200000000000001},"max":{"1min":3.9529999999999998,"5min":15.561,"15min":15.561},"last":3.9529999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":1.7829999999999999,"15min":1.532},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.254},"max":{"1min":15.672000000000001,"5min":15.672000000000001,"15min":16.231999999999999},"last":10.289},{"interface":"front","average":{"1min":2.371,"5min":1.8600000000000001,"15min":1.4870000000000001},"min":{"1min":0.307,"5min":0.29899999999999999,"15min":0.26300000000000001},"max":{"1min":15.696999999999999,"5min":15.696999999999999,"15min":15.696999999999999},"last":0.38500000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1240000000000001,"5min":1.659,"15min":1.47},"min":{"1min":0.34399999999999997,"5min":0.31900000000000001,"15min":0.245},"max":{"1min":12.128,"5min":12.128,"15min":17.920000000000002},"last":4.0170000000000003},{"interface":"front","average":{"1min":3.0489999999999999,"5min":1.571,"15min":1.4810000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":16.597000000000001,"5min":16.597000000000001,"15min":17.882999999999999},"last":10.336}]},{"osd":6,"last update":"Mon Oct 9 10:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.609,"15min":1.278},"min":{"1min":0.34000000000000002,"5min":0.30499999999999999,"15min":0.247},"max":{"1min":5.3490000000000002,"5min":10.443,"15min":14.029999999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.9550000000000001,"5min":1.627,"15min":1.4399999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.29199999999999998},"max":{"1min":6.9500000000000002,"5min":6.9500000000000002,"15min":12.862},"last":3.9849999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3580000000000001,"5min":1.484,"15min":1.2829999999999999},"min":{"1min":0.38600000000000001,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":7.6109999999999998,"5min":7.6109999999999998,"15min":13.999000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.1960000000000002,"5min":1.7110000000000001,"15min":1.49},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.307},"max":{"1min":9.4830000000000005,"5min":9.4830000000000005,"15min":12.826000000000001},"last":4.0019999999999998}]}]},{"osd":3,"up_from":95,"seq":408021893258,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3376824,"kb_used_data":2711792,"kb_used_omap":7,"kb_used_meta":664952,"kb_avail":90368328,"statfs":{"total":95995035648,"available":92537167872,"internally_reserved":0,"allocated":2776875008,"data_stored":2775239355,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":680911464},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":1.762,"15min":1.3320000000000001},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":9.6099999999999994,"5min":10.388999999999999,"15min":15.255000000000001},"last":4.7569999999999997},{"interface":"front","average":{"1min":2.496,"5min":2.855,"15min":1.6379999999999999},"min":{"1min":0.26900000000000002,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":17.312000000000001,"5min":17.312000000000001,"15min":17.312000000000001},"last":1.127}]},{"osd":1,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":2.278,"15min":1.655},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.19900000000000001},"max":{"1min":4.5510000000000002,"5min":11.757,"15min":14.707000000000001},"last":0.19400000000000001},{"interface":"front","average":{"1min":1.7290000000000001,"5min":2.3809999999999998,"15min":1.522},"min":{"1min":0.26300000000000001,"5min":0.249,"15min":0.193},"max":{"1min":9.5790000000000006,"5min":11.741,"15min":15.443},"last":12.442}]},{"osd":2,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.9890000000000001,"15min":1.4730000000000001},"min":{"1min":0.32600000000000001,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":4.5910000000000002,"5min":11.795,"15min":13.853},"last":0.29999999999999999},{"interface":"front","average":{"1min":2.5,"5min":2.0219999999999998,"15min":1.55},"min":{"1min":0.315,"5min":0.248,"15min":0.248},"max":{"1min":17.268000000000001,"5min":17.268000000000001,"15min":17.268000000000001},"last":0.33700000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.8140000000000001,"15min":1.4359999999999999},"min":{"1min":0.36199999999999999,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":9.2270000000000003,"5min":9.2270000000000003,"15min":20.388000000000002},"last":4.7839999999999998},{"interface":"front","average":{"1min":1.9419999999999999,"5min":1.728,"15min":1.4650000000000001},"min":{"1min":0.41399999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.680999999999999,"5min":11.680999999999999,"15min":19.085999999999999},"last":0.49399999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":2.1120000000000001,"15min":1.712},"min":{"1min":0.438,"5min":0.29399999999999998,"15min":0.29199999999999998},"max":{"1min":11.755000000000001,"5min":11.755000000000001,"15min":51.509999999999998},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.5669999999999999,"5min":2.1030000000000002,"15min":1.7949999999999999},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":5.3289999999999997,"5min":11.082000000000001,"15min":55.338999999999999},"last":0.32600000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":2.0550000000000002,"5min":1.905,"15min":1.444},"min":{"1min":0.38400000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.723000000000001,"5min":11.723000000000001,"15min":12.361000000000001},"last":0.439},{"interface":"front","average":{"1min":1.415,"5min":2.1539999999999999,"15min":1.657},"min":{"1min":0.35999999999999999,"5min":0.35399999999999998,"15min":0.33800000000000002},"max":{"1min":4.8120000000000003,"5min":11.24,"15min":13.891999999999999},"last":4.3339999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:53:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7130000000000001,"5min":2.0720000000000001,"15min":1.8440000000000001},"min":{"1min":0.38900000000000001,"5min":0.31,"15min":0.31},"max":{"1min":5.2229999999999999,"5min":11.819000000000001,"15min":51.744},"last":4.3899999999999997},{"interface":"front","average":{"1min":1.542,"5min":2.1320000000000001,"15min":1.748},"min":{"1min":0.39400000000000002,"5min":0.39200000000000002,"15min":0.31900000000000001},"max":{"1min":9.2799999999999994,"5min":11.379,"15min":53.399000000000001},"last":4.7220000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019134,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2883024,"kb_used_data":2577544,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":90862128,"statfs":{"total":95995035648,"available":93042819072,"internally_reserved":0,"allocated":2639405056,"data_stored":2638333077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:54:52 2023","interfaces":[{"interface":"back","average":{"1min":13.236000000000001,"5min":3.7330000000000001,"15min":1.893},"min":{"1min":0.23899999999999999,"5min":0.217,"15min":0.20999999999999999},"max":{"1min":213.88999999999999,"5min":213.88999999999999,"15min":213.88999999999999},"last":2.04},{"interface":"front","average":{"1min":13.113,"5min":4.3140000000000001,"15min":2.2160000000000002},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.217},"max":{"1min":214.09800000000001,"5min":214.09800000000001,"15min":214.09800000000001},"last":0.26000000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:53:52 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.383,"15min":1.149},"min":{"1min":0.24399999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.897,"5min":14.099,"15min":14.099},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.70299999999999996,"5min":1.788,"15min":1.3700000000000001},"min":{"1min":0.37,"5min":0.31,"15min":0.28000000000000003},"max":{"1min":2.0680000000000001,"5min":14.089,"15min":14.089},"last":0.38700000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:54:32 2023","interfaces":[{"interface":"back","average":{"1min":2.2240000000000002,"5min":1.8740000000000001,"15min":1.407},"min":{"1min":0.316,"5min":0.28599999999999998,"15min":0.253},"max":{"1min":15.446999999999999,"5min":15.446999999999999,"15min":15.446999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":2.5790000000000002,"5min":1.8400000000000001,"15min":1.5269999999999999},"min":{"1min":0.29699999999999999,"5min":0.29599999999999999,"15min":0.26300000000000001},"max":{"1min":15.56,"5min":15.56,"15min":20.545999999999999},"last":1.603}]},{"osd":3,"last update":"Mon Oct 9 10:54:52 2023","interfaces":[{"interface":"back","average":{"1min":13.404,"5min":4.1950000000000003,"15min":2.2549999999999999},"min":{"1min":0.38700000000000001,"5min":0.29299999999999998,"15min":0.27400000000000002},"max":{"1min":209.03700000000001,"5min":209.03700000000001,"15min":209.03700000000001},"last":1.9379999999999999},{"interface":"front","average":{"1min":14.557,"5min":4.415,"15min":2.4350000000000001},"min":{"1min":0.34799999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":208.80199999999999,"5min":208.80199999999999,"15min":208.80199999999999},"last":0.43099999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:53:52 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.8560000000000001,"15min":1.381},"min":{"1min":0.28199999999999997,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":5.274,"5min":10.529999999999999,"15min":24.149999999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.77200000000000002,"5min":1.5189999999999999,"15min":1.224},"min":{"1min":0.29899999999999999,"5min":0.27100000000000002,"15min":0.252},"max":{"1min":2.8029999999999999,"5min":15.071999999999999,"15min":20.422000000000001},"last":0.47099999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:54:25 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":1.617,"15min":1.2230000000000001},"min":{"1min":0.29999999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":15.739000000000001,"5min":15.739000000000001,"15min":15.739000000000001},"last":0.55700000000000005},{"interface":"front","average":{"1min":2.3650000000000002,"5min":2.2549999999999999,"15min":1.4450000000000001},"min":{"1min":0.371,"5min":0.35699999999999998,"15min":0.311},"max":{"1min":14.093999999999999,"5min":14.272,"15min":14.272},"last":0.60999999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:54:32 2023","interfaces":[{"interface":"back","average":{"1min":2.157,"5min":1.9410000000000001,"15min":1.327},"min":{"1min":0.40600000000000003,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":14.458,"5min":14.458,"15min":14.458},"last":0.66400000000000003},{"interface":"front","average":{"1min":2.6920000000000002,"5min":1.667,"15min":1.216},"min":{"1min":0.379,"5min":0.255,"15min":0.255},"max":{"1min":15.592000000000001,"5min":15.592000000000001,"15min":15.592000000000001},"last":0.58599999999999997}]}]},{"osd":5,"up_from":37,"seq":158913790203,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3645584,"kb_used_data":3344200,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":90099568,"statfs":{"total":95995035648,"available":92261957632,"internally_reserved":0,"allocated":3424460800,"data_stored":3423393242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:53:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.9399999999999999,"15min":1.2529999999999999},"min":{"1min":0.41099999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.923,"5min":14.837,"15min":14.837},"last":4.5510000000000002},{"interface":"front","average":{"1min":1.282,"5min":1.9870000000000001,"15min":1.3140000000000001},"min":{"1min":0.45700000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.008,"5min":14.863,"15min":14.863},"last":4.5830000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:54:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":2.024,"15min":1.4710000000000001},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":3.306,"5min":10.682,"15min":12.999000000000001},"last":0.876},{"interface":"front","average":{"1min":1.353,"5min":1.7310000000000001,"15min":1.2849999999999999},"min":{"1min":0.27900000000000003,"5min":0.254,"15min":0.24299999999999999},"max":{"1min":4.8150000000000004,"5min":12.109,"15min":12.109},"last":1.6799999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:54:26 2023","interfaces":[{"interface":"back","average":{"1min":2.081,"5min":2.1339999999999999,"15min":1.472},"min":{"1min":0.29899999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":10.052,"5min":11.368,"15min":11.393000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.8410000000000002,"5min":2.0110000000000001,"15min":1.333},"min":{"1min":0.32100000000000001,"5min":0.314,"15min":0.26200000000000001},"max":{"1min":9.0269999999999992,"5min":15.087999999999999,"15min":15.087999999999999},"last":0.45900000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:53:55 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":2.4630000000000001,"15min":1.7450000000000001},"min":{"1min":0.32800000000000001,"5min":0.28599999999999998,"15min":0.25700000000000001},"max":{"1min":4.7640000000000002,"5min":63.508000000000003,"15min":63.508000000000003},"last":1.988},{"interface":"front","average":{"1min":0.84299999999999997,"5min":2.597,"15min":1.766},"min":{"1min":0.34499999999999997,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.9990000000000001,"5min":62.418999999999997,"15min":62.418999999999997},"last":2.117}]},{"osd":4,"last update":"Mon Oct 9 10:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":2.5310000000000001,"15min":1.357},"min":{"1min":0.27100000000000002,"5min":0.24099999999999999,"15min":0.23200000000000001},"max":{"1min":4.4989999999999997,"5min":123.241,"15min":123.241},"last":0.29799999999999999},{"interface":"front","average":{"1min":1.0680000000000001,"5min":2.706,"15min":1.5269999999999999},"min":{"1min":0.30299999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.6379999999999999,"5min":123.318,"15min":123.318},"last":0.42599999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:54:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.7250000000000001,"15min":1.2310000000000001},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.5750000000000002,"5min":11.098000000000001,"15min":11.098000000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.2490000000000001,"5min":1.7130000000000001,"15min":1.1970000000000001},"min":{"1min":0.36399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.7210000000000001,"5min":15.16,"15min":20.256},"last":0.36699999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:54:41 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":1.917,"15min":1.2769999999999999},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.28699999999999998},"max":{"1min":8.468,"5min":10.739000000000001,"15min":10.739000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.47,"15min":1.129},"min":{"1min":0.35999999999999999,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":8.734,"5min":10.858000000000001,"15min":14.154},"last":0.41099999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2334916608,"data_stored":2334916608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2253914112,"data_stored":2253914112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":237309952,"data_stored":237305880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2756509696,"data_stored":2756509696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2621898752,"data_stored":2621898752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3419607040,"data_stored":3419602968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2693267456,"data_stored":2693267456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2331250688,"data_stored":2331246616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:54:58.117 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_2 pgp_num 26 2023-10-09T10:54:58.196 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:58.225 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:58.228 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:58.255 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:58.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:58.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:58.324 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:54:58.358 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:58.401 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:54:58.437 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:54:58.470 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:58.484 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:54:58.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:58.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:58.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:58.586 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:54:58.616 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:54:58.652 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:54:58.653 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:54:58.687 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:54:58.724 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:54:58.737 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:54:58.737 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:54:58.737 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:54:58.737 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:54:58.737 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:54:58.738 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:54:58.738 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:54:58.738 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:54:58.788 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:58 smithi125 ceph-mon[195507]: from='client.35529 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:54:58.788 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:58 smithi125 ceph-mon[195507]: pgmap v713: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 158 B/s rd, 2.2 MiB/s wr, 510 op/s; 17602/280602 objects degraded (6.273%); 21593/280602 objects misplaced (7.695%); 11 MiB/s, 179 objects/s recovering 2023-10-09T10:54:58.788 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:58 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 17602/280602 objects degraded (6.273%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:58.788 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:58 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/399194320' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:58.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:58 smithi125 ceph-mon[200197]: from='client.35529 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:54:58.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:58 smithi125 ceph-mon[200197]: pgmap v713: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 158 B/s rd, 2.2 MiB/s wr, 510 op/s; 17602/280602 objects degraded (6.273%); 21593/280602 objects misplaced (7.695%); 11 MiB/s, 179 objects/s recovering 2023-10-09T10:54:58.789 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:58 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 17602/280602 objects degraded (6.273%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:58.789 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:58 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/399194320' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:58.802 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:58.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:58 smithi155 ceph-mon[184854]: from='client.35529 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:54:58.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:58 smithi155 ceph-mon[184854]: pgmap v713: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 158 B/s rd, 2.2 MiB/s wr, 510 op/s; 17602/280602 objects degraded (6.273%); 21593/280602 objects misplaced (7.695%); 11 MiB/s, 179 objects/s recovering 2023-10-09T10:54:58.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:58 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 17602/280602 objects degraded (6.273%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:54:58.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:58 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/399194320' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:54:58.973 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:54:58.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:58.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:54:59.083 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:54:59.132 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:59.159 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:54:59.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:59.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:54:59.251 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:59.263 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:54:59.294 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:59.317 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:54:59.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:59.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:54:59.351 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:54:59.421 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:54:59.472 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:59.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:54:59.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:54:59.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:54:59.664 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:54:59.719 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:54:59 smithi125 ceph-mon[195507]: pgmap v714: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 799 KiB/s wr, 521 op/s; 17382/277311 objects degraded (6.268%); 21593/277311 objects misplaced (7.787%); 11 MiB/s, 172 objects/s recovering 2023-10-09T10:54:59.724 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:54:59.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:54:59 smithi155 ceph-mon[184854]: pgmap v714: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 799 KiB/s wr, 521 op/s; 17382/277311 objects degraded (6.268%); 21593/277311 objects misplaced (7.787%); 11 MiB/s, 172 objects/s recovering 2023-10-09T10:55:00.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:54:59 smithi125 ceph-mon[200197]: pgmap v714: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 799 KiB/s wr, 521 op/s; 17382/277311 objects degraded (6.268%); 21593/277311 objects misplaced (7.787%); 11 MiB/s, 172 objects/s recovering 2023-10-09T10:55:00.127 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:00.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:00.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:00.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:00.287 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:00.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:00.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:00.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:00.582 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:55:00.623 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:00.647 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:55:00.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:00.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:55:00.751 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:55:00.785 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:00.815 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:55:00.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:00.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:55:00.921 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:00.995 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:01.115 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:55:01.115 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:55:01.115 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.115 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPush(8.d 300/282 [PushOp(8:b285c4c0:::benchmark_data_smithi125_329074_object19414:head, version: 271'1148, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b285c4c0:::benchmark_data_smithi125_329074_object19414:head@271'1148, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:55:01.115 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:01.099339+0000", 2023-10-09T10:55:01.115 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.014663654, 2023-10-09T10:55:01.115 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014715116, 2023-10-09T10:55:01.115 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:55:01.116 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:55:01.116 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:55:01.116 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.116 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:55:01.116 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.099339+0000", 2023-10-09T10:55:01.116 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:01.116 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.116 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.116 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:55:01.116 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.099339+0000", 2023-10-09T10:55:01.117 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:01.117 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.117 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.117 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:55:01.117 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.099339+0000", 2023-10-09T10:55:01.117 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:55:01.117 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.117 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.117 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:55:01.117 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.099388+0000", 2023-10-09T10:55:01.117 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8507000000000002e-05 2023-10-09T10:55:01.117 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.118 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.118 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:55:01.118 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.099402+0000", 2023-10-09T10:55:01.118 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4504e-05 2023-10-09T10:55:01.118 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.118 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.118 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:55:01.118 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.099416+0000", 2023-10-09T10:55:01.118 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4381e-05 2023-10-09T10:55:01.118 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:01.119 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:55:01.119 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:01.119 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.119 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.119 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:104178 8.d e300/282 8:b7214f72:::benchmark_data_smithi125_329074_object7524:head v 300'6405, mlcod=271'1143)", 2023-10-09T10:55:01.119 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:01.099435+0000", 2023-10-09T10:55:01.119 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.014567916, 2023-10-09T10:55:01.119 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014655616999999999, 2023-10-09T10:55:01.119 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:55:01.119 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:55:01.119 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:55:01.120 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.120 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:55:01.120 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.099435+0000", 2023-10-09T10:55:01.120 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:01.120 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.120 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.120 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:55:01.120 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.099435+0000", 2023-10-09T10:55:01.120 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:01.120 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.120 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.121 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:55:01.121 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.099435+0000", 2023-10-09T10:55:01.121 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:55:01.121 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.121 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.121 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:55:01.121 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.099439+0000", 2023-10-09T10:55:01.121 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1150000000000004e-06 2023-10-09T10:55:01.121 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.121 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.122 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:55:01.122 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.099445+0000", 2023-10-09T10:55:01.122 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7180000000000001e-06 2023-10-09T10:55:01.122 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.122 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.122 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:55:01.122 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.099449+0000", 2023-10-09T10:55:01.122 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.369e-06 2023-10-09T10:55:01.122 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.122 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.122 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:55:01.123 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.101754+0000", 2023-10-09T10:55:01.123 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023051539999999998 2023-10-09T10:55:01.123 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.123 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.123 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:55:01.123 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.101780+0000", 2023-10-09T10:55:01.123 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5728000000000002e-05 2023-10-09T10:55:01.123 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:01.123 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:55:01.123 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:01.123 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.124 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.124 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:104184 8.d e300/282 8:b01e1ca4:::benchmark_data_smithi125_329074_object7530:head v 300'6406, mlcod=271'1143)", 2023-10-09T10:55:01.124 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:01.106837+0000", 2023-10-09T10:55:01.124 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0071664470000000003, 2023-10-09T10:55:01.124 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0072910079999999999, 2023-10-09T10:55:01.124 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:55:01.124 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:55:01.124 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:55:01.124 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.124 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:55:01.124 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.106837+0000", 2023-10-09T10:55:01.124 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:01.125 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.125 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.125 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:55:01.125 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.106837+0000", 2023-10-09T10:55:01.125 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:01.125 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.125 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.125 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:55:01.125 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.106836+0000", 2023-10-09T10:55:01.125 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:55:01.125 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.126 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.126 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:55:01.126 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.106844+0000", 2023-10-09T10:55:01.126 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6899999999999992e-06 2023-10-09T10:55:01.126 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.126 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.126 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:55:01.126 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.106854+0000", 2023-10-09T10:55:01.126 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0481e-05 2023-10-09T10:55:01.126 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.126 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.126 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:55:01.127 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.106862+0000", 2023-10-09T10:55:01.127 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8439999999999998e-06 2023-10-09T10:55:01.127 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.127 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.127 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:55:01.127 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.107542+0000", 2023-10-09T10:55:01.127 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00068028999999999995 2023-10-09T10:55:01.127 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.127 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.127 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:55:01.128 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.107570+0000", 2023-10-09T10:55:01.128 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7613000000000001e-05 2023-10-09T10:55:01.128 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:01.128 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:55:01.128 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:01.128 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.128 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.128 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:104176 8.f e300/295 8:f4b2b2e8:::benchmark_data_smithi125_329074_object7522:head v 300'6575, mlcod=275'4755)", 2023-10-09T10:55:01.128 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:01.110327+0000", 2023-10-09T10:55:01.128 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0036756459999999999, 2023-10-09T10:55:01.128 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038368320000000001, 2023-10-09T10:55:01.128 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:55:01.129 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:55:01.129 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:55:01.129 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.129 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:55:01.129 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.110327+0000", 2023-10-09T10:55:01.129 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:01.129 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.129 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.129 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:55:01.129 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.110327+0000", 2023-10-09T10:55:01.129 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:01.129 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.130 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.130 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:55:01.130 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.110326+0000", 2023-10-09T10:55:01.130 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:55:01.130 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.130 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.130 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:55:01.130 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.110335+0000", 2023-10-09T10:55:01.130 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.242e-06 2023-10-09T10:55:01.130 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.130 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.130 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:55:01.131 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.110348+0000", 2023-10-09T10:55:01.131 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3387000000000001e-05 2023-10-09T10:55:01.131 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.131 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.131 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:55:01.131 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.110356+0000", 2023-10-09T10:55:01.131 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0909999999999993e-06 2023-10-09T10:55:01.131 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.131 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.131 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:55:01.131 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.110404+0000", 2023-10-09T10:55:01.131 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7624999999999999e-05 2023-10-09T10:55:01.132 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.132 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.132 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:55:01.132 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.110427+0000", 2023-10-09T10:55:01.132 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3604999999999998e-05 2023-10-09T10:55:01.132 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:01.132 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:55:01.132 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:01.132 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.132 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.132 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:104183 8.a e300/295)", 2023-10-09T10:55:01.132 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:01.110371+0000", 2023-10-09T10:55:01.133 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0036321550000000002, 2023-10-09T10:55:01.133 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038264750000000002, 2023-10-09T10:55:01.133 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:55:01.133 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:55:01.133 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:55:01.133 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.133 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:55:01.133 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.110371+0000", 2023-10-09T10:55:01.133 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:01.133 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.133 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.134 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:55:01.134 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.110371+0000", 2023-10-09T10:55:01.134 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:01.134 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.134 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.134 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:55:01.134 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.110371+0000", 2023-10-09T10:55:01.134 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:55:01.134 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.134 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.134 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:55:01.134 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.110375+0000", 2023-10-09T10:55:01.135 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3170000000000003e-06 2023-10-09T10:55:01.135 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.135 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.135 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:55:01.135 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.110382+0000", 2023-10-09T10:55:01.135 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8700000000000003e-06 2023-10-09T10:55:01.135 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:01.135 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:01.135 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:55:01.135 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:01.110387+0000", 2023-10-09T10:55:01.135 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8729999999999998e-06 2023-10-09T10:55:01.135 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:01.136 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:55:01.136 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:01.136 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:01.136 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:55:01.136 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 5 2023-10-09T10:55:01.136 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:55:01.258 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:55:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:55:01] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.43.0" 2023-10-09T10:55:01.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:01.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:55:01.259 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:01.259 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:55:01.328 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:01.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:01.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:01.432 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:01.466 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:01.467 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:55:01.469 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:01.498 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:01.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:01.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:01.560 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:55:01.602 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:01.677 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:02.177 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:55:02.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:02 smithi155 ceph-mon[184854]: pgmap v715: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 17382/271398 objects degraded (6.405%); 20938/271398 objects misplaced (7.715%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T10:55:02.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:02 smithi125 ceph-mon[195507]: pgmap v715: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 17382/271398 objects degraded (6.405%); 20938/271398 objects misplaced (7.715%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T10:55:02.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:02 smithi125 ceph-mon[200197]: pgmap v715: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 17382/271398 objects degraded (6.405%); 20938/271398 objects misplaced (7.715%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T10:55:02.583 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:02.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:02.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:02.689 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:02.723 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:02.750 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:02.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:02.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:02.854 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:02.911 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:02.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:02.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:02.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:03.070 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:03.107 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 8 pgp_num to 26 2023-10-09T10:55:03.158 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3429862622' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:55:03.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:03 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:55:03.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:03 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 17382/271398 objects degraded (6.405%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:55:03.159 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3429862622' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:55:03.159 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:03 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:55:03.159 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:03 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 17382/271398 objects degraded (6.405%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:55:03.159 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:03.410 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:03.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:03.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3429862622' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:55:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:03 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:55:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:03 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 17382/271398 objects degraded (6.405%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:55:03.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:03.575 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:03.973 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:03.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:03.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:04.079 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:55:04.116 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:04.137 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:04.137 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:04.142 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:55:04.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:04.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:55:04.151 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:55:04.245 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:55:04.276 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:04.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:55:04.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:04.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:55:04.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:04 smithi125 ceph-mon[195507]: failed to encode map e301 with expected crc 2023-10-09T10:55:04.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:04 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T10:55:04.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:04 smithi125 ceph-mon[195507]: osdmap e301: 8 total, 8 up, 8 in 2023-10-09T10:55:04.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:04 smithi125 ceph-mon[195507]: failed to encode map e301 with expected crc 2023-10-09T10:55:04.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:04 smithi125 ceph-mon[195507]: pgmap v717: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 17153/266526 objects degraded (6.436%); 20748/266526 objects misplaced (7.785%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T10:55:04.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:04 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/212929972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:55:04.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:04 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:55:04.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:04 smithi125 ceph-mon[200197]: failed to encode map e301 with expected crc 2023-10-09T10:55:04.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:04 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T10:55:04.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:04 smithi125 ceph-mon[200197]: osdmap e301: 8 total, 8 up, 8 in 2023-10-09T10:55:04.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:04 smithi125 ceph-mon[200197]: failed to encode map e301 with expected crc 2023-10-09T10:55:04.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:04 smithi125 ceph-mon[200197]: pgmap v717: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 17153/266526 objects degraded (6.436%); 20748/266526 objects misplaced (7.785%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T10:55:04.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:04 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/212929972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:55:04.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:04 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:55:04.407 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:04 smithi155 ceph-mon[184854]: failed to encode map e301 with expected crc 2023-10-09T10:55:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:04 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T10:55:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:04 smithi155 ceph-mon[184854]: osdmap e301: 8 total, 8 up, 8 in 2023-10-09T10:55:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:04 smithi155 ceph-mon[184854]: failed to encode map e301 with expected crc 2023-10-09T10:55:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:04 smithi155 ceph-mon[184854]: pgmap v717: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 17153/266526 objects degraded (6.436%); 20748/266526 objects misplaced (7.785%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T10:55:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:04 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/212929972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:55:04.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:04 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:55:04.475 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:05 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:05 smithi155 ceph-mon[184854]: failed to encode map e302 with expected crc 2023-10-09T10:55:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:05 smithi155 ceph-mon[184854]: failed to encode map e302 with expected crc 2023-10-09T10:55:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:05 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:55:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:05 smithi155 ceph-mon[184854]: osdmap e302: 8 total, 8 up, 8 in 2023-10-09T10:55:05.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:05 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:05.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:05 smithi125 ceph-mon[195507]: failed to encode map e302 with expected crc 2023-10-09T10:55:05.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:05 smithi125 ceph-mon[195507]: failed to encode map e302 with expected crc 2023-10-09T10:55:05.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:05 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:55:05.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:05 smithi125 ceph-mon[195507]: osdmap e302: 8 total, 8 up, 8 in 2023-10-09T10:55:05.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:05 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:05.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:05 smithi125 ceph-mon[200197]: failed to encode map e302 with expected crc 2023-10-09T10:55:05.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:05 smithi125 ceph-mon[200197]: failed to encode map e302 with expected crc 2023-10-09T10:55:05.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:05 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:55:05.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:05 smithi125 ceph-mon[200197]: osdmap e302: 8 total, 8 up, 8 in 2023-10-09T10:55:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:06 smithi155 ceph-mon[184854]: pgmap v719: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 17153/261894 objects degraded (6.550%); 20251/261894 objects misplaced (7.733%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T10:55:06.576 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:06 smithi125 ceph-mon[195507]: pgmap v719: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 17153/261894 objects degraded (6.550%); 20251/261894 objects misplaced (7.733%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T10:55:06.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:06 smithi125 ceph-mon[200197]: pgmap v719: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 17153/261894 objects degraded (6.550%); 20251/261894 objects misplaced (7.733%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T10:55:07.012 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:07.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:07.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:07.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:07.174 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:07.682 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:07.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:07.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:07.786 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:07.841 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:07.929 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:07.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:07.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:08.037 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:08.102 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:08.139 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:08.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:08.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:08.244 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:08.305 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:08.495 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:08.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:08.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:08 smithi125 ceph-mon[195507]: pgmap v720: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 17153/257793 objects degraded (6.654%); 19771/257793 objects misplaced (7.669%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T10:55:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:08 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 17153/257793 objects degraded (6.654%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:55:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:08 smithi125 ceph-mon[200197]: pgmap v720: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 17153/257793 objects degraded (6.654%); 19771/257793 objects misplaced (7.669%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T10:55:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:08 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 17153/257793 objects degraded (6.654%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:55:08.605 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:08.677 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:08.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:08 smithi155 ceph-mon[184854]: pgmap v720: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 17153/257793 objects degraded (6.654%); 19771/257793 objects misplaced (7.669%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T10:55:08.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:08 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 17153/257793 objects degraded (6.654%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:55:08.878 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:08.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:08.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:08.952 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:55:08.982 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:55:08.982 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:55:08.982 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:55:08.982 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:55:08.988 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:09.067 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:09.094 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:09.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:09.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:09.165 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:55:09.200 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:09.204 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:55:09.250 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:55:09.261 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:09.302 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:55:09.336 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:55:09.338 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:09.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:09.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:09.374 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:55:09.377 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:55:09.422 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:55:09.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:09.457 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:55:09.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:55:09.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:55:09.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:55:09.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:55:09.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:55:09.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:55:09.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:55:09.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:55:09.480 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:09 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2738241845' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:09.480 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:09 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2738241845' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:09.510 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:09.600 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:09.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:09.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:09.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:55:09.710 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:09.741 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:09 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2738241845' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:09.745 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:09.776 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:09.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:09.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:09.880 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:09.912 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:09.936 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:09.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:09.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:10.005 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:10.042 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:10.090 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:10.118 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:10.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:10.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:10.227 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:10.292 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:10.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:10.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:10.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:10.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:10.449 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:10.460 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:10.485 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:10.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:10.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:10.590 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:10.631 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:10.661 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:10.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:10.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:10.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:10 smithi155 ceph-mon[184854]: pgmap v721: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 16873/253749 objects degraded (6.649%); 19771/253749 objects misplaced (7.792%); 6.9 MiB/s, 111 objects/s recovering 2023-10-09T10:55:10.766 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:10.804 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:10.813 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:10 smithi125 ceph-mon[195507]: pgmap v721: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 16873/253749 objects degraded (6.649%); 19771/253749 objects misplaced (7.792%); 6.9 MiB/s, 111 objects/s recovering 2023-10-09T10:55:10.817 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:10 smithi125 ceph-mon[200197]: pgmap v721: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 16873/253749 objects degraded (6.649%); 19771/253749 objects misplaced (7.792%); 6.9 MiB/s, 111 objects/s recovering 2023-10-09T10:55:10.830 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:10.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:10.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:10.937 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:11.010 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:11.195 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:11.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:11.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:11.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:55:11.341 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:11.365 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:55:11.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:11.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:55:11.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:11.495 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:55:11.495 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:55:11.526 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:55:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:55:11] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.43.0" 2023-10-09T10:55:11.538 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:11.731 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:11.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:11.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:11.837 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:55:11.872 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:11.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:55:11.896 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:55:11.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:11.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:55:12.004 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:12.035 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:12.059 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:12.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:12.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:12.165 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:12.198 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:12.200 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:55:12.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:12.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:12.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:12.268 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 7, 2, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:55:12.268 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:55:12.268 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T10:55:12.268 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:55:12.268 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:55:12.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T10:55:12.329 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:12.419 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:12.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:12 smithi125 ceph-mon[195507]: pgmap v722: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 925 op/s; 16873/246984 objects degraded (6.832%); 19032/246984 objects misplaced (7.706%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T10:55:12.542 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:12.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:12.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:12.606 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:55:12.612 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:12.648 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:12.715 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:12.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:12 smithi125 ceph-mon[200197]: pgmap v722: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 925 op/s; 16873/246984 objects degraded (6.832%); 19032/246984 objects misplaced (7.706%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T10:55:12.850 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:12.850 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:12.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:12.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:12.954 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:55:12.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:12 smithi155 ceph-mon[184854]: pgmap v722: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 925 op/s; 16873/246984 objects degraded (6.832%); 19032/246984 objects misplaced (7.706%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T10:55:12.986 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:13.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:55:13.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:13.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:55:13.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:13.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:13 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 16873/246984 objects degraded (6.832%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:55:13.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:13 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2492754014' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:55:13.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:13 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:55:13.580 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:13 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 16873/246984 objects degraded (6.832%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:55:13.581 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:13 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2492754014' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:55:13.581 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:13 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:55:13.596 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:13.624 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:13.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:13.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:13.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:13 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 16873/246984 objects degraded (6.832%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:55:13.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:13 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2492754014' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:55:13.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:13 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:55:13.728 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:13.786 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:13.955 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:55:14.014 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:14.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:14.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:14.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:14.192 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:14.474 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:55:14.474 INFO:teuthology.orchestra.run.smithi155.stdout: "success": "" 2023-10-09T10:55:14.475 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:55:14.476 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:14.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:14.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:14.506 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:55:14.556 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:55:14.580 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:14.591 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:55:14.622 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:14.638 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:55:14.689 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:55:14.726 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:55:14.759 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:55:14.769 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:14 smithi155 ceph-mon[184854]: pgmap v723: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 16622/241869 objects degraded (6.872%); 18842/241869 objects misplaced (7.790%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:55:14.769 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:14 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:14.769 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:14 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:55:14.770 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:14 smithi155 ceph-mon[184854]: osdmap e303: 8 total, 8 up, 8 in 2023-10-09T10:55:14.770 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:14 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3500467955' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:14.795 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:55:14.797 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:14.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:14.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:14.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:14 smithi125 ceph-mon[195507]: pgmap v723: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 16622/241869 objects degraded (6.872%); 18842/241869 objects misplaced (7.790%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:55:14.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:14 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:14.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:14 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:55:14.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:14 smithi125 ceph-mon[195507]: osdmap e303: 8 total, 8 up, 8 in 2023-10-09T10:55:14.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:14 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3500467955' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:14.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:14 smithi125 ceph-mon[200197]: pgmap v723: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 16622/241869 objects degraded (6.872%); 18842/241869 objects misplaced (7.790%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:55:14.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:14 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:14.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:14 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:55:14.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:14 smithi125 ceph-mon[200197]: osdmap e303: 8 total, 8 up, 8 in 2023-10-09T10:55:14.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:14 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3500467955' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:14.839 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:55:14.839 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:55:14.839 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:55:14.839 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:55:14.839 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:55:14.839 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:55:14.839 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:55:14.840 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:55:14.901 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:14.993 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:14.998 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:55:14.999 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:55:14.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:14.999 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:115262 8.10 8:094abb49:::benchmark_data_smithi125_329074_object18608:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T10:55:14.999 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:14.979723+0000", 2023-10-09T10:55:14.999 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.015362077, 2023-10-09T10:55:14.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015395842999999999, 2023-10-09T10:55:14.999 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:55:14.999 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:55:14.999 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:55:14.999 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:55:15.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:15.000 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115262 2023-10-09T10:55:15.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.000 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:55:15.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.000 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:55:15.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.979723+0000", 2023-10-09T10:55:15.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:15.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:55:15.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.979723+0000", 2023-10-09T10:55:15.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:15.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.002 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:55:15.002 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.979721+0000", 2023-10-09T10:55:15.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:55:15.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:55:15.003 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.979729+0000", 2023-10-09T10:55:15.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1280000000000008e-06 2023-10-09T10:55:15.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.003 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:55:15.003 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.979742+0000", 2023-10-09T10:55:15.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3254999999999999e-05 2023-10-09T10:55:15.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.003 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:55:15.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.979751+0000", 2023-10-09T10:55:15.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4689999999999997e-06 2023-10-09T10:55:15.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:55:15.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.979877+0000", 2023-10-09T10:55:15.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012651200000000001 2023-10-09T10:55:15.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:55:15.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.980000+0000", 2023-10-09T10:55:15.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000123001 2023-10-09T10:55:15.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:55:15.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.980348+0000", 2023-10-09T10:55:15.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00034737199999999998 2023-10-09T10:55:15.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:15.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.991162+0000", 2023-10-09T10:55:15.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.01081414 2023-10-09T10:55:15.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.006 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:55:15.006 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.992763+0000", 2023-10-09T10:55:15.006 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016006379999999999 2023-10-09T10:55:15.006 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:15.006 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:55:15.006 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:15.006 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.006 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.006 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:115263 8.a 8:5a02c9e8:::benchmark_data_smithi125_329074_object18609:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T10:55:15.006 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:14.979769+0000", 2023-10-09T10:55:15.007 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.015315629000000001, 2023-10-09T10:55:15.007 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015461585, 2023-10-09T10:55:15.007 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:55:15.007 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:55:15.007 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:55:15.007 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:55:15.007 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:15.007 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115263 2023-10-09T10:55:15.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.007 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:55:15.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.007 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:55:15.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.979769+0000", 2023-10-09T10:55:15.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:15.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:55:15.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.979769+0000", 2023-10-09T10:55:15.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:15.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:55:15.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.979769+0000", 2023-10-09T10:55:15.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:55:15.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:55:15.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.979772+0000", 2023-10-09T10:55:15.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.185e-06 2023-10-09T10:55:15.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:55:15.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.979778+0000", 2023-10-09T10:55:15.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1630000000000001e-06 2023-10-09T10:55:15.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:55:15.010 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.979783+0000", 2023-10-09T10:55:15.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0780000000000003e-06 2023-10-09T10:55:15.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.010 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.010 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:55:15.010 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.981798+0000", 2023-10-09T10:55:15.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002014933 2023-10-09T10:55:15.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.010 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.010 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:55:15.010 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.981976+0000", 2023-10-09T10:55:15.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017801500000000001 2023-10-09T10:55:15.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2,5", 2023-10-09T10:55:15.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.982137+0000", 2023-10-09T10:55:15.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000160774 2023-10-09T10:55:15.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:55:15.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.993540+0000", 2023-10-09T10:55:15.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011402647 2023-10-09T10:55:15.011 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:15.011 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:55:15.012 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:15.012 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.012 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.012 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:115268 8.18 8:195e794f:::benchmark_data_smithi125_329074_object18614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T10:55:15.012 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:14.985365+0000", 2023-10-09T10:55:15.012 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0097195690000000008, 2023-10-09T10:55:15.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0099534670000000006, 2023-10-09T10:55:15.012 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:55:15.012 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:55:15.012 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:55:15.012 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:55:15.012 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:15.013 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115268 2023-10-09T10:55:15.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.013 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:55:15.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.013 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:55:15.013 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.985365+0000", 2023-10-09T10:55:15.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:15.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.013 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:55:15.013 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.985365+0000", 2023-10-09T10:55:15.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:15.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:55:15.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.985364+0000", 2023-10-09T10:55:15.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:55:15.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:55:15.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.985372+0000", 2023-10-09T10:55:15.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4910000000000005e-06 2023-10-09T10:55:15.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:55:15.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.985386+0000", 2023-10-09T10:55:15.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4205e-05 2023-10-09T10:55:15.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:55:15.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.985395+0000", 2023-10-09T10:55:15.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2700000000000004e-06 2023-10-09T10:55:15.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:55:15.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.991182+0000", 2023-10-09T10:55:15.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0057871989999999998 2023-10-09T10:55:15.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:55:15.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.991287+0000", 2023-10-09T10:55:15.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000105425 2023-10-09T10:55:15.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:55:15.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.991559+0000", 2023-10-09T10:55:15.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027167499999999998 2023-10-09T10:55:15.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.017 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:55:15.017 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.993309+0000", 2023-10-09T10:55:15.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0017505730000000001 2023-10-09T10:55:15.017 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:15.017 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:55:15.017 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:15.017 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.017 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:115273 8.a 8:5b039bbe:::benchmark_data_smithi125_329074_object18619:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T10:55:15.017 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:14.990315+0000", 2023-10-09T10:55:15.017 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0047700140000000004, 2023-10-09T10:55:15.018 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0050903839999999999, 2023-10-09T10:55:15.018 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:55:15.018 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:55:15.018 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:55:15.018 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:55:15.018 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:15.018 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 115273 2023-10-09T10:55:15.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.018 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:55:15.018 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:55:15.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.990315+0000", 2023-10-09T10:55:15.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:15.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:55:15.019 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.990315+0000", 2023-10-09T10:55:15.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:15.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:55:15.019 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.990313+0000", 2023-10-09T10:55:15.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:55:15.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.020 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:55:15.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.990320+0000", 2023-10-09T10:55:15.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8929999999999999e-06 2023-10-09T10:55:15.020 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.020 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:55:15.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.990336+0000", 2023-10-09T10:55:15.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5668999999999999e-05 2023-10-09T10:55:15.020 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.020 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:55:15.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.990346+0000", 2023-10-09T10:55:15.021 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5389999999999996e-06 2023-10-09T10:55:15.021 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.021 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.021 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:55:15.021 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.990454+0000", 2023-10-09T10:55:15.021 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010803599999999999 2023-10-09T10:55:15.021 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.021 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.021 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:55:15.021 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.990625+0000", 2023-10-09T10:55:15.021 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017185200000000001 2023-10-09T10:55:15.021 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2,5", 2023-10-09T10:55:15.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.990788+0000", 2023-10-09T10:55:15.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000163058 2023-10-09T10:55:15.022 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:15.022 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:55:15.022 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:15.022 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.022 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.35433.0:115263 8.a e303/295)", 2023-10-09T10:55:15.022 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:14.993739+0000", 2023-10-09T10:55:15.023 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.001345939, 2023-10-09T10:55:15.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0017371039999999999, 2023-10-09T10:55:15.023 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:55:15.023 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:55:15.023 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:55:15.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:55:15.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.993739+0000", 2023-10-09T10:55:15.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:15.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:55:15.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.993739+0000", 2023-10-09T10:55:15.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:15.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:55:15.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.993738+0000", 2023-10-09T10:55:15.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:55:15.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:55:15.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.993743+0000", 2023-10-09T10:55:15.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7380000000000002e-06 2023-10-09T10:55:15.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:55:15.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.993755+0000", 2023-10-09T10:55:15.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2189e-05 2023-10-09T10:55:15.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:55:15.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.993763+0000", 2023-10-09T10:55:15.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2139999999999996e-06 2023-10-09T10:55:15.025 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:15.025 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:55:15.026 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:15.026 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.026 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.35433.0:115268 8.18 e303/289)", 2023-10-09T10:55:15.026 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:14.995475+0000", 2023-10-09T10:55:15.026 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4294967295.9996104, 2023-10-09T10:55:15.026 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8986999999999997e-05, 2023-10-09T10:55:15.026 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:55:15.026 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:55:15.026 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:55:15.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.026 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:55:15.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.995475+0000", 2023-10-09T10:55:15.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:15.027 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.027 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.027 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:55:15.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.995475+0000", 2023-10-09T10:55:15.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:55:15.027 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.027 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.027 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:55:15.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.995474+0000", 2023-10-09T10:55:15.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:55:15.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.028 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:55:15.028 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.995479+0000", 2023-10-09T10:55:15.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0900000000000004e-06 2023-10-09T10:55:15.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.028 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:55:15.028 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.995494+0000", 2023-10-09T10:55:15.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4905999999999999e-05 2023-10-09T10:55:15.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:55:15.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:55:15.029 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:55:15.029 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:14.995502+0000", 2023-10-09T10:55:15.029 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5760000000000006e-06 2023-10-09T10:55:15.029 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:15.029 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:55:15.029 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:15.029 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:55:15.029 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:55:15.029 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 6 2023-10-09T10:55:15.029 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:55:15.433 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:15.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:15.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:15.538 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:15.600 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:15.693 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:15.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:15.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:15.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:55:15.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:15 smithi125 ceph-mon[195507]: failed to encode map e303 with expected crc 2023-10-09T10:55:15.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:15.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:55:15.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:15 smithi125 ceph-mon[200197]: failed to encode map e303 with expected crc 2023-10-09T10:55:15.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:15.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:55:15.798 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:15.866 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:15.905 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:15.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:15.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:15.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:15 smithi155 ceph-mon[184854]: failed to encode map e303 with expected crc 2023-10-09T10:55:15.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:15.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:55:16.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:16.075 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:16.108 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:16.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:16.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:16.216 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:16.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:16.295 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:16.320 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:16.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:16.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:16.424 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:16.470 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:16.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:16.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:16.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:16.583 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:16 smithi125 ceph-mon[195507]: pgmap v725: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 16622/237627 objects degraded (6.995%); 18389/237627 objects misplaced (7.739%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:55:16.583 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:16 smithi125 ceph-mon[200197]: pgmap v725: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 16622/237627 objects degraded (6.995%); 18389/237627 objects misplaced (7.739%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:55:16.603 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:16.638 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:16.663 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:16.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:16.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:16.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:16 smithi155 ceph-mon[184854]: pgmap v725: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 16622/237627 objects degraded (6.995%); 18389/237627 objects misplaced (7.739%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:55:16.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:16.840 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:16.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:55:16.877 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:16.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:16.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:16.988 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:17.078 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:17.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:17.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:17.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:17.275 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:17.305 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:17.328 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:17.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:17.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:17.424 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:55:17.432 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:17.472 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:17.494 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:17.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:17.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:17.599 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:17.627 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:17.646 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:17.654 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:17.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:17.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:17.758 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:17.795 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:55:17.795 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:55:17.795 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:55:17.795 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:55:17.795 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:55:17.882 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:17.978 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:17.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:17.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:18.083 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:18.113 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:18.139 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:18.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:18.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:18.242 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:18.303 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:18.493 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:18.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:18.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:18.523 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:55:18.601 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:18.639 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:18.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:18.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:18.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:18.769 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:18.804 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:18 smithi155 ceph-mon[184854]: pgmap v726: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 16622/234012 objects degraded (7.103%); 17941/234012 objects misplaced (7.667%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:55:18.804 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:18 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 16622/234012 objects degraded (7.103%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:55:18.806 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:18.807 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:18 smithi125 ceph-mon[195507]: pgmap v726: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 16622/234012 objects degraded (7.103%); 17941/234012 objects misplaced (7.667%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:55:18.807 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:18 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 16622/234012 objects degraded (7.103%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:55:18.807 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:18 smithi125 ceph-mon[200197]: pgmap v726: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 16622/234012 objects degraded (7.103%); 17941/234012 objects misplaced (7.667%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:55:18.807 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:18 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 16622/234012 objects degraded (7.103%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T10:55:18.807 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:18.825 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:18.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:18.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:18.929 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:18.967 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:18.995 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:18.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:18.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:19.099 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:19.136 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:19.160 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:19.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:19.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:19.264 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:19.296 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:19.322 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:19.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:19.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:19.426 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:19.459 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:19.486 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:19.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:19.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:19.592 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:19.627 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:19.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:19.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:19.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:19.758 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:19.832 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:19.837 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 7, 2, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:55:19.837 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:55:19.837 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:55:19.838 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 1.000000 2023-10-09T10:55:19.838 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-10-09T10:55:19.856 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:19.874 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:19.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:19.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:19.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:20.012 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:20.035 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:20.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:20.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:20.139 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:20.259 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:20.544 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:20.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:20.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:20.556 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:55:20.592 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:55:20.611 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:55:20.649 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:55:20.654 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:20.689 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:20.693 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:55:20.711 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:20.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:20.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:20.727 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:55:20.766 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:55:20.813 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:55:20.815 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:20.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:20 smithi125 ceph-mon[195507]: pgmap v727: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 16385/230520 objects degraded (7.108%); 17941/230520 objects misplaced (7.783%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T10:55:20.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:20 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3247759901' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:55:20.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:20 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:55:20.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:20 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1446215079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:20.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:20 smithi125 ceph-mon[200197]: pgmap v727: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 16385/230520 objects degraded (7.108%); 17941/230520 objects misplaced (7.783%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T10:55:20.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:20 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3247759901' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:55:20.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:20 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:55:20.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:20 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1446215079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:20.845 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:20 smithi155 ceph-mon[184854]: pgmap v727: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 16385/230520 objects degraded (7.108%); 17941/230520 objects misplaced (7.783%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T10:55:20.846 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:20 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3247759901' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:55:20.846 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:20 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:55:20.846 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:20 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1446215079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:20.848 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:20.859 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:55:20.870 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:55:20.871 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:55:20.871 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:55:20.871 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:55:20.871 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:55:20.871 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:55:20.871 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:55:20.871 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:55:20.881 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:20.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:20.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:20.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:21.063 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:21.374 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:21.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:21.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:21.499 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:21.533 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:21.546 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:55:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:55:21] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.43.0" 2023-10-09T10:55:21.563 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:21.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:21.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:21.669 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:21.727 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:21.806 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:21 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:21.807 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:21 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:55:21.807 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:21 smithi125 ceph-mon[195507]: osdmap e304: 8 total, 8 up, 8 in 2023-10-09T10:55:21.807 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:21 smithi125 ceph-mon[195507]: failed to encode map e304 with expected crc 2023-10-09T10:55:21.807 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:21 smithi125 ceph-mon[195507]: failed to encode map e304 with expected crc 2023-10-09T10:55:21.808 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:21 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:21.808 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:21 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:55:21.808 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:21 smithi125 ceph-mon[200197]: osdmap e304: 8 total, 8 up, 8 in 2023-10-09T10:55:21.808 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:21 smithi125 ceph-mon[200197]: failed to encode map e304 with expected crc 2023-10-09T10:55:21.808 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:21 smithi125 ceph-mon[200197]: failed to encode map e304 with expected crc 2023-10-09T10:55:21.812 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:21.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:21.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:21.921 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:21.922 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:21.949 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:21 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:21.949 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:21 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:55:21.949 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:21 smithi155 ceph-mon[184854]: osdmap e304: 8 total, 8 up, 8 in 2023-10-09T10:55:21.950 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:21 smithi155 ceph-mon[184854]: failed to encode map e304 with expected crc 2023-10-09T10:55:21.950 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:21 smithi155 ceph-mon[184854]: failed to encode map e304 with expected crc 2023-10-09T10:55:21.951 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:21.972 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:21.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:21.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:22.072 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:55:22.073 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:55:22.076 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:22.173 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:22.208 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:22.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:22.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:22.315 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:22.346 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:22.372 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:22.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:22.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:22.476 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:22.510 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:22.543 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:22.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:22.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:22.646 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:22.680 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:55:22.710 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:22 smithi125 ceph-mon[195507]: pgmap v729: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 16385/224208 objects degraded (7.308%); 17183/224208 objects misplaced (7.664%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T10:55:22.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:22 smithi125 ceph-mon[200197]: pgmap v729: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 16385/224208 objects degraded (7.308%); 17183/224208 objects misplaced (7.664%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T10:55:22.712 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:22 smithi155 ceph-mon[184854]: pgmap v729: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 16385/224208 objects degraded (7.308%); 17183/224208 objects misplaced (7.664%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T10:55:22.720 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:22.860 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:22.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:22.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:22.970 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:23.007 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:23.061 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:55:23.096 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:23.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:23.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:23.201 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:55:23.237 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:23.271 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:55:23.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:23.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:55:23.374 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:23.405 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:55:23.420 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:23.546 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:23.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:23.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:23.650 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:23.729 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:23 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 16385/224208 objects degraded (7.308%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:55:23.729 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:23 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 16385/224208 objects degraded (7.308%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:55:23.740 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:23.786 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:23.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:23.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:23.834 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:55:23.890 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:23.897 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:23.944 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:23.947 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:23 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 16385/224208 objects degraded (7.308%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T10:55:24.031 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:24.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:24.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:24.137 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:24.178 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:55:24.178 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:55:24.188 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:24.317 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:24.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:24.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:24.421 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:24.467 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:24.504 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:24.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:24.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:24.579 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:55:24.611 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:24.613 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:24 smithi125 ceph-mon[195507]: pgmap v730: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 16151/219162 objects degraded (7.369%); 16983/219162 objects misplaced (7.749%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:55:24.613 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:24 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2725114033' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:55:24.613 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:24 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:55:24.623 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-10-09T10:55:24.653 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:24.676 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:24.707 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:24.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:24.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:24.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:24 smithi155 ceph-mon[184854]: pgmap v730: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 16151/219162 objects degraded (7.369%); 16983/219162 objects misplaced (7.749%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:55:24.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:24 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2725114033' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:55:24.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:24 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:55:24.811 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:55:24.851 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:24.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:24 smithi125 ceph-mon[200197]: pgmap v730: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 16151/219162 objects degraded (7.369%); 16983/219162 objects misplaced (7.749%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:55:24.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:24 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2725114033' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:55:24.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:24 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:55:24.928 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:55:24.932 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:55:24.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:24.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:55:25.036 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:25.065 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:25.086 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:25.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:25.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:25.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:25.220 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:25.394 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:25.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:25.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:25.499 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:25.530 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:25.549 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:25.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:25.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:25.628 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:55:25.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:25.685 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:25.810 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[195507]: 8.2 scrub starts 2023-10-09T10:55:25.810 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:55:25.811 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[195507]: osdmap e305: 8 total, 8 up, 8 in 2023-10-09T10:55:25.811 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[195507]: failed to encode map e305 with expected crc 2023-10-09T10:55:25.811 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[195507]: failed to encode map e305 with expected crc 2023-10-09T10:55:25.811 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[195507]: failed to encode map e305 with expected crc 2023-10-09T10:55:25.811 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[195507]: failed to encode map e305 with expected crc 2023-10-09T10:55:25.811 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2963518498' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:55:25.811 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[200197]: 8.2 scrub starts 2023-10-09T10:55:25.812 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:55:25.812 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[200197]: osdmap e305: 8 total, 8 up, 8 in 2023-10-09T10:55:25.812 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[200197]: failed to encode map e305 with expected crc 2023-10-09T10:55:25.812 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[200197]: failed to encode map e305 with expected crc 2023-10-09T10:55:25.812 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[200197]: failed to encode map e305 with expected crc 2023-10-09T10:55:25.812 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[200197]: failed to encode map e305 with expected crc 2023-10-09T10:55:25.812 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2963518498' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:55:25.918 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:55:25.918 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:55:25.938 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:25 smithi155 ceph-mon[184854]: 8.2 scrub starts 2023-10-09T10:55:25.938 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:25 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:55:25.938 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:25 smithi155 ceph-mon[184854]: osdmap e305: 8 total, 8 up, 8 in 2023-10-09T10:55:25.938 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:25 smithi155 ceph-mon[184854]: failed to encode map e305 with expected crc 2023-10-09T10:55:25.938 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:25 smithi155 ceph-mon[184854]: failed to encode map e305 with expected crc 2023-10-09T10:55:25.939 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:25 smithi155 ceph-mon[184854]: failed to encode map e305 with expected crc 2023-10-09T10:55:25.939 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:25 smithi155 ceph-mon[184854]: failed to encode map e305 with expected crc 2023-10-09T10:55:25.939 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2963518498' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:55:26.258 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:26.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:26.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:26.285 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:55:26.362 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:26.391 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:26.461 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:26.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:26.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:26.546 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:55:26.566 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:26.598 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:26.623 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:26.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:26.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:26.731 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:26.741 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:55:26.776 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:26.778 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:55:26.811 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:26.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:26.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:26.829 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:55:26.887 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:55:26.917 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:26.936 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:55:26.958 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:26 smithi155 ceph-mon[184854]: pgmap v732: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 16151/215292 objects degraded (7.502%); 16556/215292 objects misplaced (7.690%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T10:55:26.958 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:26 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:55:26.958 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:26 smithi155 ceph-mon[184854]: failed to encode map e306 with expected crc 2023-10-09T10:55:26.958 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:26 smithi155 ceph-mon[184854]: failed to encode map e306 with expected crc 2023-10-09T10:55:26.958 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:26 smithi155 ceph-mon[184854]: failed to encode map e306 with expected crc 2023-10-09T10:55:26.959 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:26 smithi155 ceph-mon[184854]: failed to encode map e306 with expected crc 2023-10-09T10:55:26.959 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:26 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2963518498' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:55:26.959 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:26 smithi155 ceph-mon[184854]: osdmap e306: 8 total, 8 up, 8 in 2023-10-09T10:55:26.959 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:26 smithi155 ceph-mon[184854]: failed to encode map e306 with expected crc 2023-10-09T10:55:26.959 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:26 smithi155 ceph-mon[184854]: failed to encode map e306 with expected crc 2023-10-09T10:55:26.959 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:26 smithi155 ceph-mon[184854]: 8.5 deep-scrub starts 2023-10-09T10:55:26.961 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:26.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:26.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:26.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:26.992 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:55:27.029 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:55:27.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[195507]: pgmap v732: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 16151/215292 objects degraded (7.502%); 16556/215292 objects misplaced (7.690%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T10:55:27.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:55:27.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[195507]: failed to encode map e306 with expected crc 2023-10-09T10:55:27.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[195507]: failed to encode map e306 with expected crc 2023-10-09T10:55:27.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[195507]: failed to encode map e306 with expected crc 2023-10-09T10:55:27.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[195507]: failed to encode map e306 with expected crc 2023-10-09T10:55:27.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2963518498' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:55:27.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[195507]: osdmap e306: 8 total, 8 up, 8 in 2023-10-09T10:55:27.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[195507]: failed to encode map e306 with expected crc 2023-10-09T10:55:27.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[195507]: failed to encode map e306 with expected crc 2023-10-09T10:55:27.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[195507]: 8.5 deep-scrub starts 2023-10-09T10:55:27.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[200197]: pgmap v732: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 16151/215292 objects degraded (7.502%); 16556/215292 objects misplaced (7.690%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T10:55:27.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:55:27.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[200197]: failed to encode map e306 with expected crc 2023-10-09T10:55:27.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[200197]: failed to encode map e306 with expected crc 2023-10-09T10:55:27.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[200197]: failed to encode map e306 with expected crc 2023-10-09T10:55:27.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[200197]: failed to encode map e306 with expected crc 2023-10-09T10:55:27.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2963518498' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:55:27.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[200197]: osdmap e306: 8 total, 8 up, 8 in 2023-10-09T10:55:27.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[200197]: failed to encode map e306 with expected crc 2023-10-09T10:55:27.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[200197]: failed to encode map e306 with expected crc 2023-10-09T10:55:27.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:26 smithi125 ceph-mon[200197]: 8.5 deep-scrub starts 2023-10-09T10:55:27.078 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:55:27.092 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:27.113 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:55:27.113 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:55:27.113 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:55:27.113 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:55:27.113 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:55:27.113 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:55:27.113 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:55:27.113 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:55:27.145 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:27.233 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:27.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:27.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:27.346 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:27.433 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:27.565 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:27.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:27.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:27.595 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:55:27.595 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:55:27.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:55:27.595 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:55:27.595 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.595 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181597 6.2 6:42bf0766:::benchmark_data_smithi125_244951_object75154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:55:27.596 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.261648+0000", 2023-10-09T10:55:27.596 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 431.33014869900001, 2023-10-09T10:55:27.596 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.045766977, 2023-10-09T10:55:27.596 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.596 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.596 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.596 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:27.596 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:27.596 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181597 2023-10-09T10:55:27.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.596 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.596 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261648+0000", 2023-10-09T10:55:27.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261648+0000", 2023-10-09T10:55:27.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261646+0000", 2023-10-09T10:55:27.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:55:27.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261657+0000", 2023-10-09T10:55:27.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1413000000000001e-05 2023-10-09T10:55:27.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261674+0000", 2023-10-09T10:55:27.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6969e-05 2023-10-09T10:55:27.599 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.599 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.599 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261682+0000", 2023-10-09T10:55:27.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.357e-06 2023-10-09T10:55:27.599 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.599 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.599 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261737+0000", 2023-10-09T10:55:27.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5757999999999997e-05 2023-10-09T10:55:27.599 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.600 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op must wait for map", 2023-10-09T10:55:27.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.261745+0000", 2023-10-09T10:55:27.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.576e-06 2023-10-09T10:55:27.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.600 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.268199+0000", 2023-10-09T10:55:27.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064538019999999998 2023-10-09T10:55:27.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.600 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.268208+0000", 2023-10-09T10:55:27.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1880000000000008e-06 2023-10-09T10:55:27.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.298495+0000", 2023-10-09T10:55:27.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0302869029999999 2023-10-09T10:55:27.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.298621+0000", 2023-10-09T10:55:27.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012632400000000001 2023-10-09T10:55:27.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:55:27.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.298695+0000", 2023-10-09T10:55:27.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3521000000000003e-05 2023-10-09T10:55:27.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.301440+0000", 2023-10-09T10:55:27.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002744879 2023-10-09T10:55:27.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.603 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.304506+0000", 2023-10-09T10:55:27.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030660570000000001 2023-10-09T10:55:27.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.603 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307330+0000", 2023-10-09T10:55:27.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028240029999999998 2023-10-09T10:55:27.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.603 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307381+0000", 2023-10-09T10:55:27.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1792000000000002e-05 2023-10-09T10:55:27.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307415+0000", 2023-10-09T10:55:27.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3429000000000001e-05 2023-10-09T10:55:27.604 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.604 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.604 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.604 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181636 6.2 6:40ad50b0:::benchmark_data_smithi125_244951_object75193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:55:27.604 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.320597+0000", 2023-10-09T10:55:27.605 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 431.27120002999999, 2023-10-09T10:55:27.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98701349400000005, 2023-10-09T10:55:27.605 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.605 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.605 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.605 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:27.605 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:27.605 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181636 2023-10-09T10:55:27.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.605 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320597+0000", 2023-10-09T10:55:27.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320597+0000", 2023-10-09T10:55:27.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320595+0000", 2023-10-09T10:55:27.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:55:27.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.607 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320604+0000", 2023-10-09T10:55:27.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8580000000000006e-06 2023-10-09T10:55:27.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.607 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320623+0000", 2023-10-09T10:55:27.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9894999999999998e-05 2023-10-09T10:55:27.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.608 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.608 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.608 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320631+0000", 2023-10-09T10:55:27.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7389999999999999e-06 2023-10-09T10:55:27.608 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.608 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.608 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.608 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320699+0000", 2023-10-09T10:55:27.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7858e-05 2023-10-09T10:55:27.608 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.608 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.608 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.320709+0000", 2023-10-09T10:55:27.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0225e-05 2023-10-09T10:55:27.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.301476+0000", 2023-10-09T10:55:27.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98076654399999996 2023-10-09T10:55:27.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.301576+0000", 2023-10-09T10:55:27.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010008700000000001 2023-10-09T10:55:27.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:55:27.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.301679+0000", 2023-10-09T10:55:27.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000102806 2023-10-09T10:55:27.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307490+0000", 2023-10-09T10:55:27.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0058112210000000001 2023-10-09T10:55:27.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307500+0000", 2023-10-09T10:55:27.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9000000000000001e-06 2023-10-09T10:55:27.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307534+0000", 2023-10-09T10:55:27.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4637000000000001e-05 2023-10-09T10:55:27.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307577+0000", 2023-10-09T10:55:27.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2453999999999997e-05 2023-10-09T10:55:27.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307610+0000", 2023-10-09T10:55:27.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3167000000000003e-05 2023-10-09T10:55:27.612 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.612 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.612 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.613 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181651 6.2 6:46858d5d:::benchmark_data_smithi125_244951_object75208:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:55:27.613 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.342675+0000", 2023-10-09T10:55:27.613 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 431.24912157699998, 2023-10-09T10:55:27.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98594575900000003, 2023-10-09T10:55:27.613 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.613 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.613 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.613 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:27.613 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:27.613 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181651 2023-10-09T10:55:27.613 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.613 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.614 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.614 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.614 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342675+0000", 2023-10-09T10:55:27.614 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.614 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.614 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.614 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.615 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342675+0000", 2023-10-09T10:55:27.615 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.615 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.615 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.615 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.615 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342674+0000", 2023-10-09T10:55:27.615 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:55:27.615 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.615 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.615 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.615 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342681+0000", 2023-10-09T10:55:27.616 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5189999999999998e-06 2023-10-09T10:55:27.616 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.616 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342690+0000", 2023-10-09T10:55:27.616 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6029999999999994e-06 2023-10-09T10:55:27.616 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342694+0000", 2023-10-09T10:55:27.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9979999999999997e-06 2023-10-09T10:55:27.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342738+0000", 2023-10-09T10:55:27.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4175000000000003e-05 2023-10-09T10:55:27.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.342749+0000", 2023-10-09T10:55:27.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0664e-05 2023-10-09T10:55:27.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.304532+0000", 2023-10-09T10:55:27.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.96178257899999997 2023-10-09T10:55:27.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.304626+0000", 2023-10-09T10:55:27.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4790999999999994e-05 2023-10-09T10:55:27.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:55:27.619 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.304730+0000", 2023-10-09T10:55:27.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103824 2023-10-09T10:55:27.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.620 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.310700+0000", 2023-10-09T10:55:27.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059698260000000001 2023-10-09T10:55:27.620 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.620 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.620 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.620 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313386+0000", 2023-10-09T10:55:27.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026858849999999998 2023-10-09T10:55:27.620 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.620 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.620 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.621 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328535+0000", 2023-10-09T10:55:27.621 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015149249 2023-10-09T10:55:27.621 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.621 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.621 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.621 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328576+0000", 2023-10-09T10:55:27.621 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1075999999999998e-05 2023-10-09T10:55:27.621 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.621 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.621 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.621 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328621+0000", 2023-10-09T10:55:27.621 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4577999999999999e-05 2023-10-09T10:55:27.622 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.622 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.622 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.622 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.622 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.622 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181659 6.2 6:43df335a:::benchmark_data_smithi125_244951_object75216:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:55:27.622 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.355928+0000", 2023-10-09T10:55:27.622 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 431.23586839500001, 2023-10-09T10:55:27.622 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97280544800000002, 2023-10-09T10:55:27.623 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.623 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.623 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.623 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:27.623 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:27.623 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181659 2023-10-09T10:55:27.623 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.623 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.623 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.624 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.624 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355928+0000", 2023-10-09T10:55:27.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.624 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.624 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355928+0000", 2023-10-09T10:55:27.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.625 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355926+0000", 2023-10-09T10:55:27.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:55:27.625 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.626 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355939+0000", 2023-10-09T10:55:27.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2756e-05 2023-10-09T10:55:27.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.626 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355957+0000", 2023-10-09T10:55:27.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8128999999999999e-05 2023-10-09T10:55:27.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.355965+0000", 2023-10-09T10:55:27.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5320000000000001e-06 2023-10-09T10:55:27.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.356031+0000", 2023-10-09T10:55:27.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6012000000000004e-05 2023-10-09T10:55:27.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.628 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.628 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.356044+0000", 2023-10-09T10:55:27.628 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2897e-05 2023-10-09T10:55:27.628 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.628 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.628 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307650+0000", 2023-10-09T10:55:27.628 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95160661899999999 2023-10-09T10:55:27.628 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.628 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.628 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307782+0000", 2023-10-09T10:55:27.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013221500000000001 2023-10-09T10:55:27.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.629 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.629 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:55:27.629 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.307905+0000", 2023-10-09T10:55:27.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000122192 2023-10-09T10:55:27.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.629 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.629 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.629 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313393+0000", 2023-10-09T10:55:27.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0054883229999999998 2023-10-09T10:55:27.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.630 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.630 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313435+0000", 2023-10-09T10:55:27.630 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2500999999999998e-05 2023-10-09T10:55:27.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.630 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.630 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328675+0000", 2023-10-09T10:55:27.630 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015239948999999999 2023-10-09T10:55:27.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328701+0000", 2023-10-09T10:55:27.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5860999999999999e-05 2023-10-09T10:55:27.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328734+0000", 2023-10-09T10:55:27.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2398999999999999e-05 2023-10-09T10:55:27.631 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.631 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.631 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.632 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181668 6.2 6:47615e5b:::benchmark_data_smithi125_244951_object75225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:55:27.632 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.371758+0000", 2023-10-09T10:55:27.632 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 431.22003828999999, 2023-10-09T10:55:27.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95707596299999997, 2023-10-09T10:55:27.632 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.632 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.632 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.632 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:27.632 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:27.632 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181668 2023-10-09T10:55:27.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.633 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371758+0000", 2023-10-09T10:55:27.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371758+0000", 2023-10-09T10:55:27.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371757+0000", 2023-10-09T10:55:27.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:55:27.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371764+0000", 2023-10-09T10:55:27.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3379999999999999e-06 2023-10-09T10:55:27.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.635 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371780+0000", 2023-10-09T10:55:27.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5863999999999999e-05 2023-10-09T10:55:27.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.635 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371789+0000", 2023-10-09T10:55:27.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2050000000000002e-06 2023-10-09T10:55:27.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.635 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371836+0000", 2023-10-09T10:55:27.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7939000000000003e-05 2023-10-09T10:55:27.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.371845+0000", 2023-10-09T10:55:27.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2619999999999998e-06 2023-10-09T10:55:27.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.310735+0000", 2023-10-09T10:55:27.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93888995500000005 2023-10-09T10:55:27.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.310868+0000", 2023-10-09T10:55:27.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000132867 2023-10-09T10:55:27.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:55:27.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.310995+0000", 2023-10-09T10:55:27.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012735700000000001 2023-10-09T10:55:27.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.319588+0000", 2023-10-09T10:55:27.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0085930829999999996 2023-10-09T10:55:27.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322662+0000", 2023-10-09T10:55:27.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030742059999999999 2023-10-09T10:55:27.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328782+0000", 2023-10-09T10:55:27.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061201789999999999 2023-10-09T10:55:27.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.642 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328812+0000", 2023-10-09T10:55:27.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9555000000000001e-05 2023-10-09T10:55:27.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.642 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328834+0000", 2023-10-09T10:55:27.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2314999999999999e-05 2023-10-09T10:55:27.642 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.643 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.643 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.643 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.643 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.643 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181725 6.2 6:4079eac3:::benchmark_data_smithi125_244951_object75282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:55:27.643 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.472093+0000", 2023-10-09T10:55:27.643 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 431.11970347300002, 2023-10-09T10:55:27.643 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.85682714000000004, 2023-10-09T10:55:27.643 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.643 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.643 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.643 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:27.644 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:27.644 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181725 2023-10-09T10:55:27.644 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.644 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.644 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.644 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.644 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472093+0000", 2023-10-09T10:55:27.644 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.644 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.644 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.644 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.644 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472093+0000", 2023-10-09T10:55:27.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.645 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472092+0000", 2023-10-09T10:55:27.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:55:27.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.645 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472100+0000", 2023-10-09T10:55:27.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8759999999999994e-06 2023-10-09T10:55:27.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472118+0000", 2023-10-09T10:55:27.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7863000000000002e-05 2023-10-09T10:55:27.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472125+0000", 2023-10-09T10:55:27.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5209999999999999e-06 2023-10-09T10:55:27.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472170+0000", 2023-10-09T10:55:27.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4774999999999997e-05 2023-10-09T10:55:27.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.647 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.647 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.472178+0000", 2023-10-09T10:55:27.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1410000000000005e-06 2023-10-09T10:55:27.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.647 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.647 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313459+0000", 2023-10-09T10:55:27.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.84128165899999996 2023-10-09T10:55:27.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313598+0000", 2023-10-09T10:55:27.648 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013813200000000001 2023-10-09T10:55:27.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:55:27.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.313729+0000", 2023-10-09T10:55:27.648 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013123599999999999 2023-10-09T10:55:27.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322622+0000", 2023-10-09T10:55:27.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0088927280000000008 2023-10-09T10:55:27.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.649 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322695+0000", 2023-10-09T10:55:27.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3676000000000004e-05 2023-10-09T10:55:27.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.649 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328866+0000", 2023-10-09T10:55:27.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.006170432 2023-10-09T10:55:27.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.650 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.650 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328890+0000", 2023-10-09T10:55:27.650 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4579999999999998e-05 2023-10-09T10:55:27.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.651 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.651 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328920+0000", 2023-10-09T10:55:27.651 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0015999999999999e-05 2023-10-09T10:55:27.651 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.651 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.651 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.651 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.651 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.651 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181731 6.2 6:47d350c6:::benchmark_data_smithi125_244951_object75288:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:55:27.651 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.487013+0000", 2023-10-09T10:55:27.651 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 431.10478375100001, 2023-10-09T10:55:27.651 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.84197810100000003, 2023-10-09T10:55:27.652 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.652 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.652 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.652 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:27.652 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:27.652 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181731 2023-10-09T10:55:27.652 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.653 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.653 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.653 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487013+0000", 2023-10-09T10:55:27.653 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.653 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.653 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487013+0000", 2023-10-09T10:55:27.653 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487011+0000", 2023-10-09T10:55:27.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:55:27.654 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487020+0000", 2023-10-09T10:55:27.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1850000000000004e-06 2023-10-09T10:55:27.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.655 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.655 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487036+0000", 2023-10-09T10:55:27.655 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6529e-05 2023-10-09T10:55:27.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.655 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.655 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487043+0000", 2023-10-09T10:55:27.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0940000000000002e-06 2023-10-09T10:55:27.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.656 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487118+0000", 2023-10-09T10:55:27.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5085000000000004e-05 2023-10-09T10:55:27.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.656 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.487127+0000", 2023-10-09T10:55:27.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7819999999999999e-06 2023-10-09T10:55:27.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.657 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.657 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.316596+0000", 2023-10-09T10:55:27.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.82946907199999997 2023-10-09T10:55:27.657 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.657 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.657 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.316692+0000", 2023-10-09T10:55:27.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5668000000000006e-05 2023-10-09T10:55:27.657 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.657 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.657 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:55:27.658 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.316798+0000", 2023-10-09T10:55:27.658 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000105782 2023-10-09T10:55:27.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.658 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.658 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328499+0000", 2023-10-09T10:55:27.658 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011701549 2023-10-09T10:55:27.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.658 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.658 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328633+0000", 2023-10-09T10:55:27.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013372199999999999 2023-10-09T10:55:27.659 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.659 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328950+0000", 2023-10-09T10:55:27.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00031750600000000001 2023-10-09T10:55:27.659 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.659 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328973+0000", 2023-10-09T10:55:27.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2549999999999999e-05 2023-10-09T10:55:27.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.660 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328991+0000", 2023-10-09T10:55:27.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8042000000000001e-05 2023-10-09T10:55:27.660 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.660 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.660 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.660 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181758 6.2 6:418c688b:::benchmark_data_smithi125_244951_object75315:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:55:27.660 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.547191+0000", 2023-10-09T10:55:27.661 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 431.04460604899998, 2023-10-09T10:55:27.661 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.79446401099999997, 2023-10-09T10:55:27.661 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.661 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.661 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.661 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:27.661 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:27.661 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181758 2023-10-09T10:55:27.661 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.661 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.662 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.662 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547191+0000", 2023-10-09T10:55:27.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.662 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.662 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.662 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547191+0000", 2023-10-09T10:55:27.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.663 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.663 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547189+0000", 2023-10-09T10:55:27.663 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:55:27.663 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.663 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.664 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547198+0000", 2023-10-09T10:55:27.664 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7680000000000003e-06 2023-10-09T10:55:27.664 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.664 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.664 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.664 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547219+0000", 2023-10-09T10:55:27.664 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0837000000000001e-05 2023-10-09T10:55:27.664 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.665 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.665 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.665 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547228+0000", 2023-10-09T10:55:27.665 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6610000000000006e-06 2023-10-09T10:55:27.665 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.665 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.666 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.666 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547285+0000", 2023-10-09T10:55:27.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6617999999999999e-05 2023-10-09T10:55:27.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.666 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.666 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.547296+0000", 2023-10-09T10:55:27.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1141999999999999e-05 2023-10-09T10:55:27.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.319617+0000", 2023-10-09T10:55:27.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.77232171000000005 2023-10-09T10:55:27.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.319712+0000", 2023-10-09T10:55:27.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4128000000000004e-05 2023-10-09T10:55:27.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:55:27.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.319799+0000", 2023-10-09T10:55:27.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7299999999999994e-05 2023-10-09T10:55:27.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328638+0000", 2023-10-09T10:55:27.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0088386809999999993 2023-10-09T10:55:27.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341523+0000", 2023-10-09T10:55:27.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012885369000000001 2023-10-09T10:55:27.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341553+0000", 2023-10-09T10:55:27.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9666999999999999e-05 2023-10-09T10:55:27.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341610+0000", 2023-10-09T10:55:27.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7250999999999998e-05 2023-10-09T10:55:27.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341655+0000", 2023-10-09T10:55:27.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4746999999999997e-05 2023-10-09T10:55:27.670 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.670 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.670 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.670 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.670 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181784 6.2 6:472c8d43:::benchmark_data_smithi125_244951_object75341:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:55:27.670 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.604167+0000", 2023-10-09T10:55:27.670 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 430.98762923499999, 2023-10-09T10:55:27.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.74181532500000003, 2023-10-09T10:55:27.671 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.671 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.671 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.671 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:27.671 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:27.671 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181784 2023-10-09T10:55:27.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.671 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.671 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.671 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.671 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604167+0000", 2023-10-09T10:55:27.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.672 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.672 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604167+0000", 2023-10-09T10:55:27.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.672 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.672 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604165+0000", 2023-10-09T10:55:27.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:55:27.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604175+0000", 2023-10-09T10:55:27.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5910000000000002e-06 2023-10-09T10:55:27.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604193+0000", 2023-10-09T10:55:27.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8117e-05 2023-10-09T10:55:27.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604201+0000", 2023-10-09T10:55:27.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9689999999999994e-06 2023-10-09T10:55:27.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604264+0000", 2023-10-09T10:55:27.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2731000000000006e-05 2023-10-09T10:55:27.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.604275+0000", 2023-10-09T10:55:27.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.117e-05 2023-10-09T10:55:27.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322716+0000", 2023-10-09T10:55:27.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.71844131899999997 2023-10-09T10:55:27.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322828+0000", 2023-10-09T10:55:27.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112049 2023-10-09T10:55:27.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:55:27.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.322941+0000", 2023-10-09T10:55:27.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112274 2023-10-09T10:55:27.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.328642+0000", 2023-10-09T10:55:27.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0057018609999999999 2023-10-09T10:55:27.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341689+0000", 2023-10-09T10:55:27.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01304662 2023-10-09T10:55:27.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.345883+0000", 2023-10-09T10:55:27.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041934269999999996 2023-10-09T10:55:27.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.345938+0000", 2023-10-09T10:55:27.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5408000000000002e-05 2023-10-09T10:55:27.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.345983+0000", 2023-10-09T10:55:27.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4749999999999997e-05 2023-10-09T10:55:27.678 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.678 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.678 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.678 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181846 6.2 6:4620c9a0:::benchmark_data_smithi125_244951_object75403:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:55:27.678 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.736404+0000", 2023-10-09T10:55:27.678 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 430.85539227999999, 2023-10-09T10:55:27.679 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.60969460900000005, 2023-10-09T10:55:27.679 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.679 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.679 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.679 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:27.679 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:27.679 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181846 2023-10-09T10:55:27.679 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.679 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.679 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.679 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736404+0000", 2023-10-09T10:55:27.679 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.679 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.680 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.680 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.680 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736404+0000", 2023-10-09T10:55:27.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.680 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.680 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.680 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.680 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736403+0000", 2023-10-09T10:55:27.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:55:27.680 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.680 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.680 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.680 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736409+0000", 2023-10-09T10:55:27.681 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1940000000000003e-06 2023-10-09T10:55:27.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.681 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736419+0000", 2023-10-09T10:55:27.681 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0188e-05 2023-10-09T10:55:27.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.681 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736426+0000", 2023-10-09T10:55:27.681 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1770000000000003e-06 2023-10-09T10:55:27.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736485+0000", 2023-10-09T10:55:27.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9590999999999999e-05 2023-10-09T10:55:27.682 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.682 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.682 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.736494+0000", 2023-10-09T10:55:27.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3180000000000007e-06 2023-10-09T10:55:27.682 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.682 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.682 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.325746+0000", 2023-10-09T10:55:27.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.589252691 2023-10-09T10:55:27.683 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.683 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.683 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.325846+0000", 2023-10-09T10:55:27.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000100058 2023-10-09T10:55:27.683 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.683 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:55:27.683 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.325944+0000", 2023-10-09T10:55:27.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7664999999999994e-05 2023-10-09T10:55:27.683 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.683 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.684 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.331507+0000", 2023-10-09T10:55:27.684 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0055631719999999999 2023-10-09T10:55:27.684 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.684 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.684 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.684 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.341718+0000", 2023-10-09T10:55:27.684 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010211291000000001 2023-10-09T10:55:27.684 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.684 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.684 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.684 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346032+0000", 2023-10-09T10:55:27.684 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043136299999999997 2023-10-09T10:55:27.685 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.685 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.685 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.685 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346069+0000", 2023-10-09T10:55:27.685 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6520000000000003e-05 2023-10-09T10:55:27.685 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.685 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.685 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.685 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346099+0000", 2023-10-09T10:55:27.685 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0341999999999999e-05 2023-10-09T10:55:27.685 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.686 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.686 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.686 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.686 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.686 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181882 6.2 6:44b14788:::benchmark_data_smithi125_244951_object75439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:55:27.686 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:16.816228+0000", 2023-10-09T10:55:27.686 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 430.77556863500001, 2023-10-09T10:55:27.687 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.529971305, 2023-10-09T10:55:27.687 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.687 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.687 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.687 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:27.687 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:27.687 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181882 2023-10-09T10:55:27.687 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.687 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.687 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.687 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.688 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816228+0000", 2023-10-09T10:55:27.688 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.688 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.688 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.688 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.688 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816228+0000", 2023-10-09T10:55:27.688 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.688 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.688 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.689 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.689 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816226+0000", 2023-10-09T10:55:27.689 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:55:27.689 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.689 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.689 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.689 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816234+0000", 2023-10-09T10:55:27.689 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6620000000000005e-06 2023-10-09T10:55:27.689 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.689 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.690 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.690 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816249+0000", 2023-10-09T10:55:27.690 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5097e-05 2023-10-09T10:55:27.690 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.690 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.690 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.690 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816256+0000", 2023-10-09T10:55:27.690 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7000000000000002e-06 2023-10-09T10:55:27.690 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.691 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.691 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.691 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816311+0000", 2023-10-09T10:55:27.691 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5797000000000001e-05 2023-10-09T10:55:27.691 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.691 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.691 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.691 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:16.816322+0000", 2023-10-09T10:55:27.691 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0219e-05 2023-10-09T10:55:27.691 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.692 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.692 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.692 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.329013+0000", 2023-10-09T10:55:27.692 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.51269123999999999 2023-10-09T10:55:27.692 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.692 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.692 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.692 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.329098+0000", 2023-10-09T10:55:27.692 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4586999999999995e-05 2023-10-09T10:55:27.693 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.693 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.693 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:55:27.693 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.329172+0000", 2023-10-09T10:55:27.693 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4690000000000005e-05 2023-10-09T10:55:27.693 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.693 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.693 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.693 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.338822+0000", 2023-10-09T10:55:27.693 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0096501520000000004 2023-10-09T10:55:27.694 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.694 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.694 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.694 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.343192+0000", 2023-10-09T10:55:27.694 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043696409999999996 2023-10-09T10:55:27.694 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.694 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.694 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.695 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346136+0000", 2023-10-09T10:55:27.695 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029442629999999999 2023-10-09T10:55:27.695 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.695 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.695 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.695 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346170+0000", 2023-10-09T10:55:27.695 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3294999999999999e-05 2023-10-09T10:55:27.695 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.695 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.696 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.696 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346199+0000", 2023-10-09T10:55:27.696 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9674000000000001e-05 2023-10-09T10:55:27.696 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.696 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.696 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.696 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.697 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.697 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:181958 6.2 6:44748d34:::benchmark_data_smithi125_244951_object75515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:55:27.697 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:17.015707+0000", 2023-10-09T10:55:27.697 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 430.57608918199998, 2023-10-09T10:55:27.697 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.33061554999999998, 2023-10-09T10:55:27.697 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.697 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.697 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.698 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:27.698 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:27.698 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 181958 2023-10-09T10:55:27.698 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.698 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.698 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.698 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.698 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015707+0000", 2023-10-09T10:55:27.698 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.699 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.699 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.699 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.699 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015707+0000", 2023-10-09T10:55:27.699 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.699 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.699 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.699 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.699 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015706+0000", 2023-10-09T10:55:27.699 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:55:27.699 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.699 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015717+0000", 2023-10-09T10:55:27.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0558e-05 2023-10-09T10:55:27.700 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.700 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015733+0000", 2023-10-09T10:55:27.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6203999999999999e-05 2023-10-09T10:55:27.700 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.700 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015739+0000", 2023-10-09T10:55:27.701 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.088e-06 2023-10-09T10:55:27.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.701 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.701 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.701 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015793+0000", 2023-10-09T10:55:27.701 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4046999999999999e-05 2023-10-09T10:55:27.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.701 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.701 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.701 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.015802+0000", 2023-10-09T10:55:27.701 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.772e-06 2023-10-09T10:55:27.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.331544+0000", 2023-10-09T10:55:27.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.31574197300000001 2023-10-09T10:55:27.702 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.331667+0000", 2023-10-09T10:55:27.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012380900000000001 2023-10-09T10:55:27.702 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:55:27.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.331748+0000", 2023-10-09T10:55:27.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0389000000000006e-05 2023-10-09T10:55:27.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.338838+0000", 2023-10-09T10:55:27.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007089962 2023-10-09T10:55:27.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.704 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.343219+0000", 2023-10-09T10:55:27.704 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043816970000000004 2023-10-09T10:55:27.704 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.704 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.704 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.704 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346261+0000", 2023-10-09T10:55:27.704 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030411969999999998 2023-10-09T10:55:27.704 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.704 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.704 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346294+0000", 2023-10-09T10:55:27.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3383000000000002e-05 2023-10-09T10:55:27.705 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.705 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.705 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346323+0000", 2023-10-09T10:55:27.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8864999999999999e-05 2023-10-09T10:55:27.705 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.705 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.705 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.705 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.705 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.706 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:182031 6.2 6:424d2b82:::benchmark_data_smithi125_244951_object75588:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T10:55:27.706 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:48:17.198389+0000", 2023-10-09T10:55:27.706 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 430.39340736299999, 2023-10-09T10:55:27.706 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15372846300000001, 2023-10-09T10:55:27.706 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.706 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.706 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.706 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:27.706 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:27.706 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 182031 2023-10-09T10:55:27.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.706 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.707 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.707 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.707 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198389+0000", 2023-10-09T10:55:27.707 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.707 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.707 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.707 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.707 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198389+0000", 2023-10-09T10:55:27.707 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.707 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.707 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.708 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198388+0000", 2023-10-09T10:55:27.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:55:27.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.708 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.708 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198396+0000", 2023-10-09T10:55:27.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7409999999999992e-06 2023-10-09T10:55:27.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.708 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.708 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198409+0000", 2023-10-09T10:55:27.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.343e-05 2023-10-09T10:55:27.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.709 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198415+0000", 2023-10-09T10:55:27.709 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.314e-06 2023-10-09T10:55:27.709 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.709 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198483+0000", 2023-10-09T10:55:27.709 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7677999999999998e-05 2023-10-09T10:55:27.709 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.709 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.198494+0000", 2023-10-09T10:55:27.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1236000000000001e-05 2023-10-09T10:55:27.710 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.710 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.710 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.710 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.335697+0000", 2023-10-09T10:55:27.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.137203295 2023-10-09T10:55:27.710 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.710 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.710 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.710 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.335802+0000", 2023-10-09T10:55:27.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104095 2023-10-09T10:55:27.711 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.711 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.711 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T10:55:27.711 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.335901+0000", 2023-10-09T10:55:27.711 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9179999999999996e-05 2023-10-09T10:55:27.711 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.711 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.711 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.711 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.343158+0000", 2023-10-09T10:55:27.711 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0072576120000000001 2023-10-09T10:55:27.712 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.712 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.712 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.346383+0000", 2023-10-09T10:55:27.712 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003224312 2023-10-09T10:55:27.712 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.712 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.712 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.352022+0000", 2023-10-09T10:55:27.712 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0056397219999999998 2023-10-09T10:55:27.712 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.713 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.713 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.352075+0000", 2023-10-09T10:55:27.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2369e-05 2023-10-09T10:55:27.713 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.713 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.713 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.713 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:48:17.352118+0000", 2023-10-09T10:55:27.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2911999999999998e-05 2023-10-09T10:55:27.713 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.713 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.713 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.713 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.713 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.714 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:152608 7.b 7:d0fd2adb:::benchmark_data_smithi125_295816_object61471:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e245)", 2023-10-09T10:55:27.714 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:52:10.363211+0000", 2023-10-09T10:55:27.714 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 197.22858555600001, 2023-10-09T10:55:27.714 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8185187439999999, 2023-10-09T10:55:27.714 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.714 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.714 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.714 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:55:27.714 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:55:27.714 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 152608 2023-10-09T10:55:27.714 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.715 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363211+0000", 2023-10-09T10:55:27.715 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.715 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363211+0000", 2023-10-09T10:55:27.715 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.715 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363209+0000", 2023-10-09T10:55:27.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:55:27.716 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.716 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363218+0000", 2023-10-09T10:55:27.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8470000000000002e-06 2023-10-09T10:55:27.716 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.716 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363236+0000", 2023-10-09T10:55:27.717 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8062000000000001e-05 2023-10-09T10:55:27.717 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.717 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.717 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.717 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363244+0000", 2023-10-09T10:55:27.717 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2560000000000002e-06 2023-10-09T10:55:27.717 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.717 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.717 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.717 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.365179+0000", 2023-10-09T10:55:27.717 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001934755 2023-10-09T10:55:27.718 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.718 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.718 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op must wait for map", 2023-10-09T10:55:27.718 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.365188+0000", 2023-10-09T10:55:27.718 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.132e-06 2023-10-09T10:55:27.718 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.718 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.718 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.718 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.380926+0000", 2023-10-09T10:55:27.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015737860999999999 2023-10-09T10:55:27.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.719 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.719 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.380936+0000", 2023-10-09T10:55:27.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.679e-06 2023-10-09T10:55:27.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.719 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.719 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.172400+0000", 2023-10-09T10:55:27.720 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7914640509999999 2023-10-09T10:55:27.720 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.720 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.172506+0000", 2023-10-09T10:55:27.720 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106677 2023-10-09T10:55:27.720 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T10:55:27.720 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.172597+0000", 2023-10-09T10:55:27.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0718999999999997e-05 2023-10-09T10:55:27.721 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.721 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.721 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.721 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.176419+0000", 2023-10-09T10:55:27.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0038220480000000002 2023-10-09T10:55:27.721 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.721 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.721 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.721 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.176443+0000", 2023-10-09T10:55:27.722 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3583000000000001e-05 2023-10-09T10:55:27.722 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.722 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.722 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.722 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.176459+0000", 2023-10-09T10:55:27.722 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5815999999999999e-05 2023-10-09T10:55:27.722 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.722 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.722 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.722 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.181617+0000", 2023-10-09T10:55:27.722 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051585210000000001 2023-10-09T10:55:27.723 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.723 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.723 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.181654+0000", 2023-10-09T10:55:27.723 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6573999999999999e-05 2023-10-09T10:55:27.723 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.723 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.723 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.181730+0000", 2023-10-09T10:55:27.724 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6070999999999997e-05 2023-10-09T10:55:27.724 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.724 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.724 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.724 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.724 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.724 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:39213 8.2 8:4e7be898:::benchmark_data_smithi125_329074_object39212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T10:55:27.724 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:53:35.895072+0000", 2023-10-09T10:55:27.724 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 111.696724956, 2023-10-09T10:55:27.725 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.128363015, 2023-10-09T10:55:27.725 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.725 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.725 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.725 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:27.725 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:27.725 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39213 2023-10-09T10:55:27.725 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.725 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.725 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.726 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.726 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895072+0000", 2023-10-09T10:55:27.726 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.726 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.726 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.726 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.726 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895072+0000", 2023-10-09T10:55:27.726 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.727 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.727 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895070+0000", 2023-10-09T10:55:27.727 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:55:27.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.727 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.727 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895204+0000", 2023-10-09T10:55:27.727 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000133561 2023-10-09T10:55:27.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895215+0000", 2023-10-09T10:55:27.728 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0925e-05 2023-10-09T10:55:27.728 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.728 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895220+0000", 2023-10-09T10:55:27.728 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4949999999999999e-06 2023-10-09T10:55:27.728 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.728 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895798+0000", 2023-10-09T10:55:27.729 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00057733700000000003 2023-10-09T10:55:27.729 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.729 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.729 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.729 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895926+0000", 2023-10-09T10:55:27.729 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012778600000000001 2023-10-09T10:55:27.729 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.729 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.729 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:55:27.729 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.896072+0000", 2023-10-09T10:55:27.730 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014615599999999999 2023-10-09T10:55:27.730 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.730 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.730 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.897973+0000", 2023-10-09T10:55:27.730 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019008510000000001 2023-10-09T10:55:27.730 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.730 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.730 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.899072+0000", 2023-10-09T10:55:27.731 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010995239999999999 2023-10-09T10:55:27.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.731 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.731 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.731 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023353+0000", 2023-10-09T10:55:27.731 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.124281323 2023-10-09T10:55:27.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023405+0000", 2023-10-09T10:55:27.732 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1158e-05 2023-10-09T10:55:27.732 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023435+0000", 2023-10-09T10:55:27.733 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.006e-05 2023-10-09T10:55:27.733 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.733 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.733 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.733 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.733 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.733 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:39226 8.2 8:44bd6854:::benchmark_data_smithi125_329074_object39225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T10:55:27.734 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:53:35.900703+0000", 2023-10-09T10:55:27.734 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 111.691093919, 2023-10-09T10:55:27.734 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.122919798, 2023-10-09T10:55:27.734 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.734 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.734 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.734 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:27.734 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:27.734 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39226 2023-10-09T10:55:27.734 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.734 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.735 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.735 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.900703+0000", 2023-10-09T10:55:27.735 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.735 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.735 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.900703+0000", 2023-10-09T10:55:27.735 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.735 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.735 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.736 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.900702+0000", 2023-10-09T10:55:27.736 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:55:27.736 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.736 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.736 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.900806+0000", 2023-10-09T10:55:27.736 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104288 2023-10-09T10:55:27.736 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.736 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.736 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.900818+0000", 2023-10-09T10:55:27.736 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2063e-05 2023-10-09T10:55:27.736 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.737 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.737 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.737 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.900823+0000", 2023-10-09T10:55:27.737 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.916e-06 2023-10-09T10:55:27.737 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.737 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.737 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.737 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.900866+0000", 2023-10-09T10:55:27.737 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2997999999999997e-05 2023-10-09T10:55:27.737 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.737 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.737 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.738 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.901022+0000", 2023-10-09T10:55:27.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000155728 2023-10-09T10:55:27.738 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.738 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.738 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:55:27.738 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.901165+0000", 2023-10-09T10:55:27.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000143406 2023-10-09T10:55:27.738 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.738 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.738 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.739 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.902667+0000", 2023-10-09T10:55:27.739 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001501893 2023-10-09T10:55:27.739 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.739 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.739 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.739 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.903304+0000", 2023-10-09T10:55:27.739 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00063756399999999997 2023-10-09T10:55:27.739 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.739 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.739 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.739 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023536+0000", 2023-10-09T10:55:27.739 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.12023191 2023-10-09T10:55:27.739 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.740 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.740 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.740 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023587+0000", 2023-10-09T10:55:27.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0411999999999998e-05 2023-10-09T10:55:27.740 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.740 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.740 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.740 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023622+0000", 2023-10-09T10:55:27.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5750000000000002e-05 2023-10-09T10:55:27.740 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.740 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.740 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.741 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.741 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.741 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83954 8.4 8:2689e239:::benchmark_data_smithi125_329074_object83953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T10:55:27.741 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.108148+0000", 2023-10-09T10:55:27.741 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 70.483648803999998, 2023-10-09T10:55:27.741 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1275014759999999, 2023-10-09T10:55:27.741 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.741 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.741 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.741 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:27.741 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:27.741 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83954 2023-10-09T10:55:27.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.742 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.742 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.742 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.742 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108148+0000", 2023-10-09T10:55:27.742 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.742 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.742 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.742 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108148+0000", 2023-10-09T10:55:27.742 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.743 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.743 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.743 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108146+0000", 2023-10-09T10:55:27.743 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:55:27.743 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.743 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.743 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.743 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108191+0000", 2023-10-09T10:55:27.743 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4549000000000003e-05 2023-10-09T10:55:27.743 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.743 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.743 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.744 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108215+0000", 2023-10-09T10:55:27.744 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3994000000000001e-05 2023-10-09T10:55:27.744 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.744 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.744 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.744 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108222+0000", 2023-10-09T10:55:27.744 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4499999999999998e-06 2023-10-09T10:55:27.744 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.744 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.744 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.744 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108279+0000", 2023-10-09T10:55:27.744 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6643e-05 2023-10-09T10:55:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108288+0000", 2023-10-09T10:55:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5109999999999992e-06 2023-10-09T10:55:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.872892+0000", 2023-10-09T10:55:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.76460362800000004 2023-10-09T10:55:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.872903+0000", 2023-10-09T10:55:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1158000000000001e-05 2023-10-09T10:55:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213528+0000", 2023-10-09T10:55:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.340624501 2023-10-09T10:55:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213679+0000", 2023-10-09T10:55:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151578 2023-10-09T10:55:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T10:55:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213861+0000", 2023-10-09T10:55:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018154 2023-10-09T10:55:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219288+0000", 2023-10-09T10:55:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00542677 2023-10-09T10:55:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219340+0000", 2023-10-09T10:55:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2657000000000001e-05 2023-10-09T10:55:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219644+0000", 2023-10-09T10:55:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030375299999999998 2023-10-09T10:55:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235530+0000", 2023-10-09T10:55:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01588606 2023-10-09T10:55:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235575+0000", 2023-10-09T10:55:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5012999999999997e-05 2023-10-09T10:55:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235649+0000", 2023-10-09T10:55:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4252000000000007e-05 2023-10-09T10:55:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83959 8.4 8:27b6f913:::benchmark_data_smithi125_329074_object83958:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T10:55:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.112252+0000", 2023-10-09T10:55:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 70.479544774000004, 2023-10-09T10:55:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1235403750000001, 2023-10-09T10:55:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83959 2023-10-09T10:55:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112252+0000", 2023-10-09T10:55:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112252+0000", 2023-10-09T10:55:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112251+0000", 2023-10-09T10:55:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:55:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112312+0000", 2023-10-09T10:55:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0888999999999999e-05 2023-10-09T10:55:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112324+0000", 2023-10-09T10:55:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2585999999999999e-05 2023-10-09T10:55:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112329+0000", 2023-10-09T10:55:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9880000000000004e-06 2023-10-09T10:55:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112378+0000", 2023-10-09T10:55:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8905e-05 2023-10-09T10:55:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112389+0000", 2023-10-09T10:55:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0665e-05 2023-10-09T10:55:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.755 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.755 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.873769+0000", 2023-10-09T10:55:27.755 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.76138013699999996 2023-10-09T10:55:27.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.755 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:55:27.755 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.873776+0000", 2023-10-09T10:55:27.755 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.17e-06 2023-10-09T10:55:27.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.755 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216109+0000", 2023-10-09T10:55:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.342332699 2023-10-09T10:55:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216239+0000", 2023-10-09T10:55:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012997 2023-10-09T10:55:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T10:55:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216354+0000", 2023-10-09T10:55:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115116 2023-10-09T10:55:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219398+0000", 2023-10-09T10:55:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030438679999999999 2023-10-09T10:55:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219407+0000", 2023-10-09T10:55:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1870000000000003e-06 2023-10-09T10:55:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219682+0000", 2023-10-09T10:55:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00027478600000000001 2023-10-09T10:55:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235686+0000", 2023-10-09T10:55:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016004852 2023-10-09T10:55:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235725+0000", 2023-10-09T10:55:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8392000000000001e-05 2023-10-09T10:55:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235792+0000", 2023-10-09T10:55:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7311000000000007e-05 2023-10-09T10:55:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:93330 8.2 8:4f5a2bcc:::benchmark_data_smithi125_329074_object93329:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T10:55:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:35.625993+0000", 2023-10-09T10:55:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 51.965803893, 2023-10-09T10:55:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.12993565000000001, 2023-10-09T10:55:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 93330 2023-10-09T10:55:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625993+0000", 2023-10-09T10:55:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625993+0000", 2023-10-09T10:55:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625992+0000", 2023-10-09T10:55:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:55:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626043+0000", 2023-10-09T10:55:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0500000000000001e-05 2023-10-09T10:55:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626051+0000", 2023-10-09T10:55:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0509999999999997e-06 2023-10-09T10:55:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626056+0000", 2023-10-09T10:55:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9570000000000001e-06 2023-10-09T10:55:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626113+0000", 2023-10-09T10:55:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.736e-05 2023-10-09T10:55:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626283+0000", 2023-10-09T10:55:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017031699999999999 2023-10-09T10:55:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:55:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626463+0000", 2023-10-09T10:55:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018018 2023-10-09T10:55:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.634095+0000", 2023-10-09T10:55:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0076312469999999999 2023-10-09T10:55:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.637111+0000", 2023-10-09T10:55:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003016725 2023-10-09T10:55:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755853+0000", 2023-10-09T10:55:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11874143299999999 2023-10-09T10:55:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755887+0000", 2023-10-09T10:55:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4017999999999999e-05 2023-10-09T10:55:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755928+0000", 2023-10-09T10:55:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1452999999999998e-05 2023-10-09T10:55:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:93340 8.2 8:4653dd45:::benchmark_data_smithi125_329074_object93339:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T10:55:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:35.641334+0000", 2023-10-09T10:55:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 51.950462952999999, 2023-10-09T10:55:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.125653775, 2023-10-09T10:55:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 93340 2023-10-09T10:55:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641334+0000", 2023-10-09T10:55:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641334+0000", 2023-10-09T10:55:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641333+0000", 2023-10-09T10:55:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:55:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641362+0000", 2023-10-09T10:55:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9034999999999999e-05 2023-10-09T10:55:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641372+0000", 2023-10-09T10:55:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0597e-05 2023-10-09T10:55:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641377+0000", 2023-10-09T10:55:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7060000000000003e-06 2023-10-09T10:55:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641429+0000", 2023-10-09T10:55:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1721000000000002e-05 2023-10-09T10:55:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641529+0000", 2023-10-09T10:55:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000100602 2023-10-09T10:55:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:55:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641632+0000", 2023-10-09T10:55:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103099 2023-10-09T10:55:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.649088+0000", 2023-10-09T10:55:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0074551569999999996 2023-10-09T10:55:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.649159+0000", 2023-10-09T10:55:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1760000000000004e-05 2023-10-09T10:55:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766886+0000", 2023-10-09T10:55:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11772616499999999 2023-10-09T10:55:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766941+0000", 2023-10-09T10:55:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5186000000000003e-05 2023-10-09T10:55:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:27.777 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766987+0000", 2023-10-09T10:55:27.777 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6692e-05 2023-10-09T10:55:27.777 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.777 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.777 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.777 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:27.777 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:27.777 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:55:27.778 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:27 smithi125 ceph-mon[195507]: failed to encode map e307 with expected crc 2023-10-09T10:55:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:27 smithi125 ceph-mon[195507]: failed to encode map e307 with expected crc 2023-10-09T10:55:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:27 smithi125 ceph-mon[195507]: osdmap e307: 8 total, 8 up, 8 in 2023-10-09T10:55:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:27 smithi125 ceph-mon[195507]: failed to encode map e307 with expected crc 2023-10-09T10:55:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:27 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1515613387' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:27 smithi125 ceph-mon[200197]: failed to encode map e307 with expected crc 2023-10-09T10:55:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:27 smithi125 ceph-mon[200197]: failed to encode map e307 with expected crc 2023-10-09T10:55:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:27 smithi125 ceph-mon[200197]: osdmap e307: 8 total, 8 up, 8 in 2023-10-09T10:55:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:27 smithi125 ceph-mon[200197]: failed to encode map e307 with expected crc 2023-10-09T10:55:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:27 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1515613387' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:27.831 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:27.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:27.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:27.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:27.914 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:27 smithi155 ceph-mon[184854]: failed to encode map e307 with expected crc 2023-10-09T10:55:27.914 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:27 smithi155 ceph-mon[184854]: failed to encode map e307 with expected crc 2023-10-09T10:55:27.914 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:27 smithi155 ceph-mon[184854]: osdmap e307: 8 total, 8 up, 8 in 2023-10-09T10:55:27.914 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:27 smithi155 ceph-mon[184854]: failed to encode map e307 with expected crc 2023-10-09T10:55:27.914 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:27 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1515613387' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:27.974 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:55:27.983 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:28.064 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:28.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:28.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:28.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:28.210 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:28.260 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:55:28.277 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:28.321 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:28.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:28.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:28.332 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:55:28.427 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:28.485 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:28.524 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:28.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:28.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:28.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:28.676 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:28.720 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:28.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:28 smithi155 ceph-mon[184854]: pgmap v735: 129 pgs: 1 remapped+peering, 1 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 637 op/s; 16151/212913 objects degraded (7.586%); 13185/212913 objects misplaced (6.193%); 5.9 MiB/s, 92 objects/s recovering 2023-10-09T10:55:28.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:28 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 16151/212913 objects degraded (7.586%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:55:28.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:28 smithi125 ceph-mon[195507]: pgmap v735: 129 pgs: 1 remapped+peering, 1 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 637 op/s; 16151/212913 objects degraded (7.586%); 13185/212913 objects misplaced (6.193%); 5.9 MiB/s, 92 objects/s recovering 2023-10-09T10:55:28.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:28 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 16151/212913 objects degraded (7.586%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:55:28.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:28 smithi125 ceph-mon[200197]: pgmap v735: 129 pgs: 1 remapped+peering, 1 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 637 op/s; 16151/212913 objects degraded (7.586%); 13185/212913 objects misplaced (6.193%); 5.9 MiB/s, 92 objects/s recovering 2023-10-09T10:55:28.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:28 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 16151/212913 objects degraded (7.586%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:55:28.897 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:28.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:28.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:29.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:55:29.034 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:29.165 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:55:29.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:29.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:55:29.272 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:29.329 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:29.364 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:29.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:29.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:29.389 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:55:29.389 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:55:29.389 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T10:55:29.389 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:55:29.469 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:29.514 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:29.587 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:55:29.693 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:29.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:29.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:29.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:55:29.803 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:29.906 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:29.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:29.964 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:29.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:30.065 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:30.089 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:55:30.135 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:30.151 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 7, 2, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:55:30.151 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:55:30.151 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_2 2023-10-09T10:55:30.152 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:55:30.166 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:30.316 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:30.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:30.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:30.423 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:30.464 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:30.686 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:30.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:30.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:30.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:30 smithi155 ceph-mon[184854]: pgmap v736: 129 pgs: 1 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 483 op/s; 12309/210642 objects degraded (5.844%); 16704/210642 objects misplaced (7.930%); 4.7 MiB/s, 73 objects/s recovering 2023-10-09T10:55:30.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:30.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:55:30.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:30 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/502446029' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:55:30.790 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:30.838 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:30.868 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:30.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:30.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:30.934 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:30 smithi125 ceph-mon[195507]: pgmap v736: 129 pgs: 1 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 483 op/s; 12309/210642 objects degraded (5.844%); 16704/210642 objects misplaced (7.930%); 4.7 MiB/s, 73 objects/s recovering 2023-10-09T10:55:30.934 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:30.935 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:55:30.935 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:30 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/502446029' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:55:30.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:30 smithi125 ceph-mon[200197]: pgmap v736: 129 pgs: 1 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 483 op/s; 12309/210642 objects degraded (5.844%); 16704/210642 objects misplaced (7.930%); 4.7 MiB/s, 73 objects/s recovering 2023-10-09T10:55:30.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:30.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:55:30.935 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:30 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/502446029' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:55:30.973 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:31.013 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:31.055 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:31.068 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:55:31.069 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:55:31.069 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:55:31.069 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:55:31.069 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:55:31.070 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:55:31.219 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:55:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:55:31] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.43.0" 2023-10-09T10:55:31.270 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:31.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:31.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:31.375 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:55:31.415 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:31.430 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:55:31.439 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:55:31.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:31.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:55:31.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:31.617 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:31.699 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:31.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:31.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:31.780 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:55:31.806 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:31.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:31 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:31.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/502446029' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:55:31.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:31 smithi125 ceph-mon[195507]: osdmap e308: 8 total, 8 up, 8 in 2023-10-09T10:55:31.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:31 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:31.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/502446029' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:55:31.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:31 smithi125 ceph-mon[200197]: osdmap e308: 8 total, 8 up, 8 in 2023-10-09T10:55:31.839 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:31.914 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:31.917 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:31 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:31.917 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/502446029' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:55:31.917 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:31 smithi155 ceph-mon[184854]: osdmap e308: 8 total, 8 up, 8 in 2023-10-09T10:55:31.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:31.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:31.925 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:55:31.964 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:55:32.002 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:55:32.019 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:32.065 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:55:32.108 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:55:32.112 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:32.142 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:55:32.180 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:55:32.227 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:55:32.245 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:55:32.245 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:55:32.245 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:55:32.246 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:55:32.246 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:55:32.246 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:55:32.246 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:55:32.246 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:55:32.459 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:32.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:32.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:32.568 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:32.603 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:32.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:32 smithi125 ceph-mon[195507]: failed to encode map e308 with expected crc 2023-10-09T10:55:32.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:32 smithi125 ceph-mon[195507]: pgmap v738: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 572 op/s; 15399/205158 objects degraded (7.506%); 16160/205158 objects misplaced (7.877%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T10:55:32.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:32 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2581206111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:32.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:32 smithi125 ceph-mon[200197]: failed to encode map e308 with expected crc 2023-10-09T10:55:32.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:32 smithi125 ceph-mon[200197]: pgmap v738: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 572 op/s; 15399/205158 objects degraded (7.506%); 16160/205158 objects misplaced (7.877%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T10:55:32.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:32 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2581206111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:32.735 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:32.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:32.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:32.812 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:55:32.812 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:55:32.839 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:32.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:32 smithi155 ceph-mon[184854]: failed to encode map e308 with expected crc 2023-10-09T10:55:32.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:32 smithi155 ceph-mon[184854]: pgmap v738: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 572 op/s; 15399/205158 objects degraded (7.506%); 16160/205158 objects misplaced (7.877%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T10:55:32.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:32 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2581206111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:32.874 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:33.111 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:33.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:33.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:33.191 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:55:33.216 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:33.267 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:33.308 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:33.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:33.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:33.413 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:33.495 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:55:33.497 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:33.533 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:55:33.538 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:55:33.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:33.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:33.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:33.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:33.700 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:33.704 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:55:33.735 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:33.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:33.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:33.780 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:33 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 15399/205158 objects degraded (7.506%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:55:33.783 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:33 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 15399/205158 objects degraded (7.506%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:55:33.838 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:33.928 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:33.950 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:33.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:33.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:33 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 15399/205158 objects degraded (7.506%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:55:34.056 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:34.145 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:34.718 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:55:34.781 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:55:34.782 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:55:34.814 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:34 smithi155 ceph-mon[184854]: pgmap v739: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 11 KiB/s rd, 28 KiB/s wr, 609 op/s; 15190/201216 objects degraded (7.549%); 16030/201216 objects misplaced (7.967%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T10:55:34.814 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:34 smithi155 ceph-mon[184854]: from='client.55456 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:55:35.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:34 smithi125 ceph-mon[195507]: pgmap v739: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 11 KiB/s rd, 28 KiB/s wr, 609 op/s; 15190/201216 objects degraded (7.549%); 16030/201216 objects misplaced (7.967%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T10:55:35.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:34 smithi125 ceph-mon[195507]: from='client.55456 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:55:35.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:34 smithi125 ceph-mon[200197]: pgmap v739: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 11 KiB/s rd, 28 KiB/s wr, 609 op/s; 15190/201216 objects degraded (7.549%); 16030/201216 objects misplaced (7.967%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T10:55:35.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:34 smithi125 ceph-mon[200197]: from='client.55456 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:55:35.164 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:55:35.503 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:55:36.554 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:55:36.554 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:55:36.617 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:36.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:36.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:36.633 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":739,"stamp":"2023-10-09T10:55:33.417818+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4394726736,"num_objects":67072,"num_object_clones":0,"num_object_copies":201216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15190,"num_objects_misplaced":16030,"num_objects_unfound":0,"num_objects_dirty":67072,"num_whiteouts":0,"num_read":116,"num_read_kb":215,"num_write":126373,"num_write_kb":6186261,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6435,"num_bytes_recovered":419821330,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":163989,"ondisk_log_size":163989,"up":387,"acting":377,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":422,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17029608,"kb_used_data":13451176,"kb_used_omap":126,"kb_used_meta":3577857,"kb_avail":732931608,"statfs":{"total":767960285184,"available":750521966592,"internally_reserved":0,"allocated":13774004224,"data_stored":13763051373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":3663726203},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-307441242,"num_objects":-4692,"num_object_clones":0,"num_object_copies":-14076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-961,"num_objects_misplaced":-526,"num_objects_unfound":0,"num_objects_dirty":-4692,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":4713,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":784,"num_bytes_recovered":51803831,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.804419"},"pg_stats":[{"pgid":"8.11","version":"308'6944","reported_seq":9002,"reported_epoch":308,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:55:33.336185+0000","last_change":"2023-10-09T10:54:49.220300+0000","last_active":"2023-10-09T10:55:33.336185+0000","last_peered":"2023-10-09T10:55:33.336185+0000","last_clean":"2023-10-09T10:54:02.426241+0000","last_became_active":"2023-10-09T10:54:49.215481+0000","last_became_peered":"2023-10-09T10:54:49.215481+0000","last_unstale":"2023-10-09T10:55:33.336185+0000","last_undegraded":"2023-10-09T10:54:49.026934+0000","last_fullsized":"2023-10-09T10:54:48.893501+0000","mapping_epoch":295,"log_start":"270'200","ondisk_log_start":"270'200","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6744,"log_dups_size":0,"ondisk_log_size":6744,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131399680,"num_objects":2005,"num_object_clones":0,"num_object_copies":6015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2303,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3983,"num_write_kb":191616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":989,"num_bytes_recovered":64815104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,1,5","objects":1334},{"shards":"0,2","objects":-10},{"shards":"0,2,5","objects":393},{"shards":"0,3,5","objects":288}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.18","version":"76'161","reported_seq":621,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.628655+0000","last_change":"2023-10-09T10:55:26.628655+0000","last_active":"2023-10-09T10:55:26.628655+0000","last_peered":"2023-10-09T10:55:26.628655+0000","last_clean":"2023-10-09T10:55:26.628655+0000","last_became_active":"2023-10-09T10:55:26.628269+0000","last_became_peered":"2023-10-09T10:55:26.628269+0000","last_unstale":"2023-10-09T10:55:26.628655+0000","last_undegraded":"2023-10-09T10:55:26.628655+0000","last_fullsized":"2023-10-09T10:55:26.628655+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:40.174975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:40.174975+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:24:16.854746+0000","scrub_duration":0.010654205,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.645705+0000","last_change":"2023-10-09T10:54:47.925818+0000","last_active":"2023-10-09T10:55:26.645705+0000","last_peered":"2023-10-09T10:55:26.645705+0000","last_clean":"2023-10-09T10:55:26.645705+0000","last_became_active":"2023-10-09T10:54:47.925375+0000","last_became_peered":"2023-10-09T10:54:47.925375+0000","last_unstale":"2023-10-09T10:55:26.645705+0000","last_undegraded":"2023-10-09T10:55:26.645705+0000","last_fullsized":"2023-10-09T10:55:26.645705+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:27.635383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":314,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.621991+0000","last_change":"2023-10-09T10:54:47.945782+0000","last_active":"2023-10-09T10:55:25.621991+0000","last_peered":"2023-10-09T10:55:25.621991+0000","last_clean":"2023-10-09T10:55:25.621991+0000","last_became_active":"2023-10-09T10:54:47.945460+0000","last_became_peered":"2023-10-09T10:54:47.945460+0000","last_unstale":"2023-10-09T10:55:25.621991+0000","last_undegraded":"2023-10-09T10:55:25.621991+0000","last_fullsized":"2023-10-09T10:55:25.621991+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:50.537469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"308'7043","reported_seq":7109,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:55:31.156610+0000","last_change":"2023-10-09T10:54:40.626105+0000","last_active":"2023-10-09T10:55:31.156610+0000","last_peered":"2023-10-09T10:55:31.156610+0000","last_clean":"2023-10-09T10:55:31.156610+0000","last_became_active":"2023-10-09T10:54:40.609428+0000","last_became_peered":"2023-10-09T10:54:40.609428+0000","last_unstale":"2023-10-09T10:55:31.156610+0000","last_undegraded":"2023-10-09T10:55:31.156610+0000","last_fullsized":"2023-10-09T10:55:31.156610+0000","mapping_epoch":289,"log_start":"271'700","ondisk_log_start":"271'700","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6343,"log_dups_size":0,"ondisk_log_size":6343,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139067392,"num_objects":2122,"num_object_clones":0,"num_object_copies":6366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4008,"num_write_kb":196160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":688,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.638071+0000","last_change":"2023-10-09T10:53:34.537094+0000","last_active":"2023-10-09T10:55:26.638071+0000","last_peered":"2023-10-09T10:55:26.638071+0000","last_clean":"2023-10-09T10:55:26.638071+0000","last_became_active":"2023-10-09T10:52:55.414995+0000","last_became_peered":"2023-10-09T10:52:55.414995+0000","last_unstale":"2023-10-09T10:55:26.638071+0000","last_undegraded":"2023-10-09T10:55:26.638071+0000","last_fullsized":"2023-10-09T10:55:26.638071+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.536962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.536962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:30.197404+0000","scrub_duration":0.0053325960000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.645578+0000","last_change":"2023-10-09T10:54:47.944768+0000","last_active":"2023-10-09T10:55:26.645578+0000","last_peered":"2023-10-09T10:55:26.645578+0000","last_clean":"2023-10-09T10:55:26.645578+0000","last_became_active":"2023-10-09T10:54:47.943888+0000","last_became_peered":"2023-10-09T10:54:47.943888+0000","last_unstale":"2023-10-09T10:55:26.645578+0000","last_undegraded":"2023-10-09T10:55:26.645578+0000","last_fullsized":"2023-10-09T10:55:26.645578+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:11.046762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":216,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.641519+0000","last_change":"2023-10-09T10:53:14.823051+0000","last_active":"2023-10-09T10:55:26.641519+0000","last_peered":"2023-10-09T10:55:26.641519+0000","last_clean":"2023-10-09T10:55:26.641519+0000","last_became_active":"2023-10-09T10:53:14.822334+0000","last_became_peered":"2023-10-09T10:53:14.822334+0000","last_unstale":"2023-10-09T10:55:26.641519+0000","last_undegraded":"2023-10-09T10:55:26.641519+0000","last_fullsized":"2023-10-09T10:55:26.641519+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:46.573381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"308'6884","reported_seq":6991,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:55:31.629702+0000","last_change":"2023-10-09T10:54:50.916172+0000","last_active":"2023-10-09T10:55:31.629702+0000","last_peered":"2023-10-09T10:55:31.629702+0000","last_clean":"2023-10-09T10:55:31.629702+0000","last_became_active":"2023-10-09T10:54:50.913596+0000","last_became_peered":"2023-10-09T10:54:50.913596+0000","last_unstale":"2023-10-09T10:55:31.629702+0000","last_undegraded":"2023-10-09T10:55:31.629702+0000","last_fullsized":"2023-10-09T10:55:31.629702+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6884,"log_dups_size":0,"ondisk_log_size":6884,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134250496,"num_objects":2049,"num_object_clones":0,"num_object_copies":6147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3927,"num_write_kb":191200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1835008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":691,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.623342+0000","last_change":"2023-10-09T10:54:50.913546+0000","last_active":"2023-10-09T10:55:26.623342+0000","last_peered":"2023-10-09T10:55:26.623342+0000","last_clean":"2023-10-09T10:55:26.623342+0000","last_became_active":"2023-10-09T10:54:50.913190+0000","last_became_peered":"2023-10-09T10:54:50.913190+0000","last_unstale":"2023-10-09T10:55:26.623342+0000","last_undegraded":"2023-10-09T10:55:26.623342+0000","last_fullsized":"2023-10-09T10:55:26.623342+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:23.805498+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":359,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.621883+0000","last_change":"2023-10-09T10:54:48.038948+0000","last_active":"2023-10-09T10:55:25.621883+0000","last_peered":"2023-10-09T10:55:25.621883+0000","last_clean":"2023-10-09T10:55:25.621883+0000","last_became_active":"2023-10-09T10:54:48.001556+0000","last_became_peered":"2023-10-09T10:54:48.001556+0000","last_unstale":"2023-10-09T10:55:25.621883+0000","last_undegraded":"2023-10-09T10:55:25.621883+0000","last_fullsized":"2023-10-09T10:55:25.621883+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:23.415040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.646611+0000","last_change":"2023-10-09T10:54:47.914742+0000","last_active":"2023-10-09T10:55:26.646611+0000","last_peered":"2023-10-09T10:55:26.646611+0000","last_clean":"2023-10-09T10:55:26.646611+0000","last_became_active":"2023-10-09T10:54:47.914551+0000","last_became_peered":"2023-10-09T10:54:47.914551+0000","last_unstale":"2023-10-09T10:55:26.646611+0000","last_undegraded":"2023-10-09T10:55:26.646611+0000","last_fullsized":"2023-10-09T10:55:26.646611+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:16.067994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":742,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.634731+0000","last_change":"2023-10-09T10:54:50.915690+0000","last_active":"2023-10-09T10:55:26.634731+0000","last_peered":"2023-10-09T10:55:26.634731+0000","last_clean":"2023-10-09T10:55:26.634731+0000","last_became_active":"2023-10-09T10:54:50.915412+0000","last_became_peered":"2023-10-09T10:54:50.915412+0000","last_unstale":"2023-10-09T10:55:26.634731+0000","last_undegraded":"2023-10-09T10:55:26.634731+0000","last_fullsized":"2023-10-09T10:55:26.634731+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.787782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.787782+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:20:13.339672+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":179,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.638550+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:55:26.638550+0000","last_peered":"2023-10-09T10:55:26.638550+0000","last_clean":"2023-10-09T10:55:26.638550+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:55:26.638550+0000","last_undegraded":"2023-10-09T10:55:26.638550+0000","last_fullsized":"2023-10-09T10:55:26.638550+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":315,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.621945+0000","last_change":"2023-10-09T10:54:48.039275+0000","last_active":"2023-10-09T10:55:25.621945+0000","last_peered":"2023-10-09T10:55:25.621945+0000","last_clean":"2023-10-09T10:55:25.621945+0000","last_became_active":"2023-10-09T10:54:48.017645+0000","last_became_peered":"2023-10-09T10:54:48.017645+0000","last_unstale":"2023-10-09T10:55:25.621945+0000","last_undegraded":"2023-10-09T10:55:25.621945+0000","last_fullsized":"2023-10-09T10:55:25.621945+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:55.986791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":551,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.615598+0000","last_change":"2023-10-09T10:54:21.161539+0000","last_active":"2023-10-09T10:55:25.615598+0000","last_peered":"2023-10-09T10:55:25.615598+0000","last_clean":"2023-10-09T10:55:25.615598+0000","last_became_active":"2023-10-09T10:54:21.161106+0000","last_became_peered":"2023-10-09T10:54:21.161106+0000","last_unstale":"2023-10-09T10:55:25.615598+0000","last_undegraded":"2023-10-09T10:55:25.615598+0000","last_fullsized":"2023-10-09T10:55:25.615598+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:21.824982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:21.824982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":48,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625822+0000","last_change":"2023-10-09T10:55:25.632545+0000","last_active":"2023-10-09T10:55:26.625822+0000","last_peered":"2023-10-09T10:55:26.625822+0000","last_clean":"2023-10-09T10:55:26.625822+0000","last_became_active":"2023-10-09T10:55:25.632235+0000","last_became_peered":"2023-10-09T10:55:25.632235+0000","last_unstale":"2023-10-09T10:55:26.625822+0000","last_undegraded":"2023-10-09T10:55:26.625822+0000","last_fullsized":"2023-10-09T10:55:26.625822+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:11.982194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":254,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.620955+0000","last_change":"2023-10-09T10:53:14.838859+0000","last_active":"2023-10-09T10:55:25.620955+0000","last_peered":"2023-10-09T10:55:25.620955+0000","last_clean":"2023-10-09T10:55:25.620955+0000","last_became_active":"2023-10-09T10:53:14.838536+0000","last_became_peered":"2023-10-09T10:53:14.838536+0000","last_unstale":"2023-10-09T10:55:25.620955+0000","last_undegraded":"2023-10-09T10:55:25.620955+0000","last_fullsized":"2023-10-09T10:55:25.620955+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:44:19.980568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":696,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.638262+0000","last_change":"2023-10-09T10:53:32.540768+0000","last_active":"2023-10-09T10:55:26.638262+0000","last_peered":"2023-10-09T10:55:26.638262+0000","last_clean":"2023-10-09T10:55:26.638262+0000","last_became_active":"2023-10-09T10:52:55.414818+0000","last_became_peered":"2023-10-09T10:52:55.414818+0000","last_unstale":"2023-10-09T10:55:26.638262+0000","last_undegraded":"2023-10-09T10:55:26.638262+0000","last_fullsized":"2023-10-09T10:55:26.638262+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:32.540610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:32.540610+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:47.374383+0000","scrub_duration":0.0064251300000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":220,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.634528+0000","last_change":"2023-10-09T10:55:25.637189+0000","last_active":"2023-10-09T10:55:26.634528+0000","last_peered":"2023-10-09T10:55:26.634528+0000","last_clean":"2023-10-09T10:55:26.634528+0000","last_became_active":"2023-10-09T10:55:25.636844+0000","last_became_peered":"2023-10-09T10:55:25.636844+0000","last_unstale":"2023-10-09T10:55:26.634528+0000","last_undegraded":"2023-10-09T10:55:26.634528+0000","last_fullsized":"2023-10-09T10:55:26.634528+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:18.943805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.645796+0000","last_change":"2023-10-09T10:54:47.925654+0000","last_active":"2023-10-09T10:55:26.645796+0000","last_peered":"2023-10-09T10:55:26.645796+0000","last_clean":"2023-10-09T10:55:26.645796+0000","last_became_active":"2023-10-09T10:54:47.924989+0000","last_became_peered":"2023-10-09T10:54:47.924989+0000","last_unstale":"2023-10-09T10:55:26.645796+0000","last_undegraded":"2023-10-09T10:55:26.645796+0000","last_fullsized":"2023-10-09T10:55:26.645796+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:02:49.755988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":681,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.646231+0000","last_change":"2023-10-09T10:54:50.913652+0000","last_active":"2023-10-09T10:55:26.646231+0000","last_peered":"2023-10-09T10:55:26.646231+0000","last_clean":"2023-10-09T10:55:26.646231+0000","last_became_active":"2023-10-09T10:54:50.913330+0000","last_became_peered":"2023-10-09T10:54:50.913330+0000","last_unstale":"2023-10-09T10:55:26.646231+0000","last_undegraded":"2023-10-09T10:55:26.646231+0000","last_fullsized":"2023-10-09T10:55:26.646231+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.200574+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.200574+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001591812,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":61,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625929+0000","last_change":"2023-10-09T10:53:14.869000+0000","last_active":"2023-10-09T10:55:26.625929+0000","last_peered":"2023-10-09T10:55:26.625929+0000","last_clean":"2023-10-09T10:55:26.625929+0000","last_became_active":"2023-10-09T10:53:14.831904+0000","last_became_peered":"2023-10-09T10:53:14.831904+0000","last_unstale":"2023-10-09T10:55:26.625929+0000","last_undegraded":"2023-10-09T10:55:26.625929+0000","last_fullsized":"2023-10-09T10:55:26.625929+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:38.571980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":312,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.624853+0000","last_change":"2023-10-09T10:54:47.908531+0000","last_active":"2023-10-09T10:55:26.624853+0000","last_peered":"2023-10-09T10:55:26.624853+0000","last_clean":"2023-10-09T10:55:26.624853+0000","last_became_active":"2023-10-09T10:54:47.908232+0000","last_became_peered":"2023-10-09T10:54:47.908232+0000","last_unstale":"2023-10-09T10:55:26.624853+0000","last_undegraded":"2023-10-09T10:55:26.624853+0000","last_fullsized":"2023-10-09T10:55:26.624853+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:20.617543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"307'6908","reported_seq":6969,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:30.251323+0000","last_change":"2023-10-09T10:54:40.649728+0000","last_active":"2023-10-09T10:55:30.251323+0000","last_peered":"2023-10-09T10:55:30.251323+0000","last_clean":"2023-10-09T10:55:30.251323+0000","last_became_active":"2023-10-09T10:54:40.647392+0000","last_became_peered":"2023-10-09T10:54:40.647392+0000","last_unstale":"2023-10-09T10:55:30.251323+0000","last_undegraded":"2023-10-09T10:55:30.251323+0000","last_fullsized":"2023-10-09T10:55:30.251323+0000","mapping_epoch":289,"log_start":"271'1000","ondisk_log_start":"271'1000","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5908,"log_dups_size":0,"ondisk_log_size":5908,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139460608,"num_objects":2128,"num_object_clones":0,"num_object_copies":6384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3918,"num_write_kb":193472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":674,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.640622+0000","last_change":"2023-10-09T10:49:18.708370+0000","last_active":"2023-10-09T10:55:26.640622+0000","last_peered":"2023-10-09T10:55:26.640622+0000","last_clean":"2023-10-09T10:55:26.640622+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:55:26.640622+0000","last_undegraded":"2023-10-09T10:55:26.640622+0000","last_fullsized":"2023-10-09T10:55:26.640622+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:57.808410+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":73,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625679+0000","last_change":"2023-10-09T10:54:18.080112+0000","last_active":"2023-10-09T10:55:26.625679+0000","last_peered":"2023-10-09T10:55:26.625679+0000","last_clean":"2023-10-09T10:55:26.625679+0000","last_became_active":"2023-10-09T10:54:17.878848+0000","last_became_peered":"2023-10-09T10:54:17.878848+0000","last_unstale":"2023-10-09T10:55:26.625679+0000","last_undegraded":"2023-10-09T10:55:26.625679+0000","last_fullsized":"2023-10-09T10:55:26.625679+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:12.314073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":365,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.621116+0000","last_change":"2023-10-09T10:54:47.941670+0000","last_active":"2023-10-09T10:55:25.621116+0000","last_peered":"2023-10-09T10:55:25.621116+0000","last_clean":"2023-10-09T10:55:25.621116+0000","last_became_active":"2023-10-09T10:54:47.941341+0000","last_became_peered":"2023-10-09T10:54:47.941341+0000","last_unstale":"2023-10-09T10:55:25.621116+0000","last_undegraded":"2023-10-09T10:55:25.621116+0000","last_fullsized":"2023-10-09T10:55:25.621116+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:31.570728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.19","version":"308'6949","reported_seq":7010,"reported_epoch":308,"state":"active+recovery_wait","last_fresh":"2023-10-09T10:55:33.344381+0000","last_change":"2023-10-09T10:54:40.731544+0000","last_active":"2023-10-09T10:55:33.344381+0000","last_peered":"2023-10-09T10:55:33.344381+0000","last_clean":"2023-10-09T10:54:39.518652+0000","last_became_active":"2023-10-09T10:54:40.598732+0000","last_became_peered":"2023-10-09T10:54:40.598732+0000","last_unstale":"2023-10-09T10:55:33.344381+0000","last_undegraded":"2023-10-09T10:55:33.344381+0000","last_fullsized":"2023-10-09T10:55:33.344381+0000","mapping_epoch":289,"log_start":"270'300","ondisk_log_start":"270'300","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6649,"log_dups_size":0,"ondisk_log_size":6649,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136839168,"num_objects":2088,"num_object_clones":0,"num_object_copies":6264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3964,"num_write_kb":193664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":2088}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":655,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625679+0000","last_change":"2023-10-09T10:53:17.846464+0000","last_active":"2023-10-09T10:55:26.625679+0000","last_peered":"2023-10-09T10:55:26.625679+0000","last_clean":"2023-10-09T10:55:26.625679+0000","last_became_active":"2023-10-09T10:53:17.845979+0000","last_became_peered":"2023-10-09T10:53:17.845979+0000","last_unstale":"2023-10-09T10:55:26.625679+0000","last_undegraded":"2023-10-09T10:55:26.625679+0000","last_fullsized":"2023-10-09T10:55:26.625679+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:56.295299+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":216,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.640148+0000","last_change":"2023-10-09T10:53:14.821961+0000","last_active":"2023-10-09T10:55:26.640148+0000","last_peered":"2023-10-09T10:55:26.640148+0000","last_clean":"2023-10-09T10:55:26.640148+0000","last_became_active":"2023-10-09T10:53:14.821625+0000","last_became_peered":"2023-10-09T10:53:14.821625+0000","last_unstale":"2023-10-09T10:55:26.640148+0000","last_undegraded":"2023-10-09T10:55:26.640148+0000","last_fullsized":"2023-10-09T10:55:26.640148+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:29.305952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":222,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.634486+0000","last_change":"2023-10-09T10:55:25.629418+0000","last_active":"2023-10-09T10:55:26.634486+0000","last_peered":"2023-10-09T10:55:26.634486+0000","last_clean":"2023-10-09T10:55:26.634486+0000","last_became_active":"2023-10-09T10:55:25.629130+0000","last_became_peered":"2023-10-09T10:55:25.629130+0000","last_unstale":"2023-10-09T10:55:26.634486+0000","last_undegraded":"2023-10-09T10:55:26.634486+0000","last_fullsized":"2023-10-09T10:55:26.634486+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:51.186552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":612,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.646951+0000","last_change":"2023-10-09T10:54:50.916062+0000","last_active":"2023-10-09T10:55:26.646951+0000","last_peered":"2023-10-09T10:55:26.646951+0000","last_clean":"2023-10-09T10:55:26.646951+0000","last_became_active":"2023-10-09T10:54:50.915189+0000","last_became_peered":"2023-10-09T10:54:50.915189+0000","last_unstale":"2023-10-09T10:55:26.646951+0000","last_undegraded":"2023-10-09T10:55:26.646951+0000","last_fullsized":"2023-10-09T10:55:26.646951+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.077788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.077788+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":203,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625376+0000","last_change":"2023-10-09T10:55:25.633116+0000","last_active":"2023-10-09T10:55:26.625376+0000","last_peered":"2023-10-09T10:55:26.625376+0000","last_clean":"2023-10-09T10:55:26.625376+0000","last_became_active":"2023-10-09T10:55:25.632837+0000","last_became_peered":"2023-10-09T10:55:25.632837+0000","last_unstale":"2023-10-09T10:55:26.625376+0000","last_undegraded":"2023-10-09T10:55:26.625376+0000","last_fullsized":"2023-10-09T10:55:26.625376+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:46.192186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":197,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.641369+0000","last_change":"2023-10-09T10:53:14.823485+0000","last_active":"2023-10-09T10:55:26.641369+0000","last_peered":"2023-10-09T10:55:26.641369+0000","last_clean":"2023-10-09T10:55:26.641369+0000","last_became_active":"2023-10-09T10:53:14.822756+0000","last_became_peered":"2023-10-09T10:53:14.822756+0000","last_unstale":"2023-10-09T10:55:26.641369+0000","last_undegraded":"2023-10-09T10:55:26.641369+0000","last_fullsized":"2023-10-09T10:55:26.641369+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:05.359313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":639,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.646129+0000","last_change":"2023-10-09T10:54:50.910027+0000","last_active":"2023-10-09T10:55:26.646129+0000","last_peered":"2023-10-09T10:55:26.646129+0000","last_clean":"2023-10-09T10:55:26.646129+0000","last_became_active":"2023-10-09T10:54:50.909698+0000","last_became_peered":"2023-10-09T10:54:50.909698+0000","last_unstale":"2023-10-09T10:55:26.646129+0000","last_undegraded":"2023-10-09T10:55:26.646129+0000","last_fullsized":"2023-10-09T10:55:26.646129+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:01.637196+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":182,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.638113+0000","last_change":"2023-10-09T10:51:12.366910+0000","last_active":"2023-10-09T10:55:26.638113+0000","last_peered":"2023-10-09T10:55:26.638113+0000","last_clean":"2023-10-09T10:55:26.638113+0000","last_became_active":"2023-10-09T10:51:12.366565+0000","last_became_peered":"2023-10-09T10:51:12.366565+0000","last_unstale":"2023-10-09T10:55:26.638113+0000","last_undegraded":"2023-10-09T10:55:26.638113+0000","last_fullsized":"2023-10-09T10:55:26.638113+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:53.464251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":341,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.622687+0000","last_change":"2023-10-09T10:54:48.017646+0000","last_active":"2023-10-09T10:55:25.622687+0000","last_peered":"2023-10-09T10:55:25.622687+0000","last_clean":"2023-10-09T10:55:25.622687+0000","last_became_active":"2023-10-09T10:54:47.994162+0000","last_became_peered":"2023-10-09T10:54:47.994162+0000","last_unstale":"2023-10-09T10:55:25.622687+0000","last_undegraded":"2023-10-09T10:55:25.622687+0000","last_fullsized":"2023-10-09T10:55:25.622687+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:11:59.205546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"308'6927","reported_seq":7001,"reported_epoch":308,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:55:31.206391+0000","last_change":"2023-10-09T10:54:40.551123+0000","last_active":"2023-10-09T10:55:31.206391+0000","last_peered":"2023-10-09T10:55:31.206391+0000","last_clean":"2023-10-09T10:54:17.036595+0000","last_became_active":"2023-10-09T10:54:40.548039+0000","last_became_peered":"2023-10-09T10:54:40.548039+0000","last_unstale":"2023-10-09T10:55:31.206391+0000","last_undegraded":"2023-10-09T10:55:31.206391+0000","last_fullsized":"2023-10-09T10:55:31.206391+0000","mapping_epoch":282,"log_start":"271'600","ondisk_log_start":"271'600","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6327,"log_dups_size":0,"ondisk_log_size":6327,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137330688,"num_objects":2095,"num_object_clones":0,"num_object_copies":6285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2095,"num_objects_unfound":0,"num_objects_dirty":2095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3941,"num_write_kb":193184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":2095}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":659,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.634650+0000","last_change":"2023-10-09T10:54:50.909873+0000","last_active":"2023-10-09T10:55:26.634650+0000","last_peered":"2023-10-09T10:55:26.634650+0000","last_clean":"2023-10-09T10:55:26.634650+0000","last_became_active":"2023-10-09T10:54:50.909533+0000","last_became_peered":"2023-10-09T10:54:50.909533+0000","last_unstale":"2023-10-09T10:55:26.634650+0000","last_undegraded":"2023-10-09T10:55:26.634650+0000","last_fullsized":"2023-10-09T10:55:26.634650+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:45.045013+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":55,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625844+0000","last_change":"2023-10-09T10:53:14.854904+0000","last_active":"2023-10-09T10:55:26.625844+0000","last_peered":"2023-10-09T10:55:26.625844+0000","last_clean":"2023-10-09T10:55:26.625844+0000","last_became_active":"2023-10-09T10:53:14.829611+0000","last_became_peered":"2023-10-09T10:53:14.829611+0000","last_unstale":"2023-10-09T10:55:26.625844+0000","last_undegraded":"2023-10-09T10:55:26.625844+0000","last_fullsized":"2023-10-09T10:55:26.625844+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:26.064630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":240,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625956+0000","last_change":"2023-10-09T10:55:25.633485+0000","last_active":"2023-10-09T10:55:26.625956+0000","last_peered":"2023-10-09T10:55:26.625956+0000","last_clean":"2023-10-09T10:55:26.625956+0000","last_became_active":"2023-10-09T10:55:25.633028+0000","last_became_peered":"2023-10-09T10:55:25.633028+0000","last_unstale":"2023-10-09T10:55:26.625956+0000","last_undegraded":"2023-10-09T10:55:26.625956+0000","last_fullsized":"2023-10-09T10:55:26.625956+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:57.601747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"308'6917","reported_seq":6979,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:55:31.064105+0000","last_change":"2023-10-09T10:55:26.330497+0000","last_active":"2023-10-09T10:55:31.064105+0000","last_peered":"2023-10-09T10:55:31.064105+0000","last_clean":"2023-10-09T10:55:31.064105+0000","last_became_active":"2023-10-09T10:54:40.554360+0000","last_became_peered":"2023-10-09T10:54:40.554360+0000","last_unstale":"2023-10-09T10:55:31.064105+0000","last_undegraded":"2023-10-09T10:55:31.064105+0000","last_fullsized":"2023-10-09T10:55:31.064105+0000","mapping_epoch":266,"log_start":"271'400","ondisk_log_start":"271'400","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6517,"log_dups_size":0,"ondisk_log_size":6517,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133038080,"num_objects":2030,"num_object_clones":0,"num_object_copies":6090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3956,"num_write_kb":191552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":666,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.620959+0000","last_change":"2023-10-09T10:53:25.540494+0000","last_active":"2023-10-09T10:55:25.620959+0000","last_peered":"2023-10-09T10:55:25.620959+0000","last_clean":"2023-10-09T10:55:25.620959+0000","last_became_active":"2023-10-09T10:51:01.295474+0000","last_became_peered":"2023-10-09T10:51:01.295474+0000","last_unstale":"2023-10-09T10:55:25.620959+0000","last_undegraded":"2023-10-09T10:55:25.620959+0000","last_fullsized":"2023-10-09T10:55:25.620959+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:39.069143+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":83,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625971+0000","last_change":"2023-10-09T10:54:48.000482+0000","last_active":"2023-10-09T10:55:26.625971+0000","last_peered":"2023-10-09T10:55:26.625971+0000","last_clean":"2023-10-09T10:55:26.625971+0000","last_became_active":"2023-10-09T10:54:47.905689+0000","last_became_peered":"2023-10-09T10:54:47.905689+0000","last_unstale":"2023-10-09T10:55:26.625971+0000","last_undegraded":"2023-10-09T10:55:26.625971+0000","last_fullsized":"2023-10-09T10:55:26.625971+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.666594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.645422+0000","last_change":"2023-10-09T10:54:47.944930+0000","last_active":"2023-10-09T10:55:26.645422+0000","last_peered":"2023-10-09T10:55:26.645422+0000","last_clean":"2023-10-09T10:55:26.645422+0000","last_became_active":"2023-10-09T10:54:47.944485+0000","last_became_peered":"2023-10-09T10:54:47.944485+0000","last_unstale":"2023-10-09T10:55:26.645422+0000","last_undegraded":"2023-10-09T10:55:26.645422+0000","last_fullsized":"2023-10-09T10:55:26.645422+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:11.213129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"308'6988","reported_seq":7051,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:55:33.326636+0000","last_change":"2023-10-09T10:54:40.586495+0000","last_active":"2023-10-09T10:55:33.326636+0000","last_peered":"2023-10-09T10:55:33.326636+0000","last_clean":"2023-10-09T10:55:33.326636+0000","last_became_active":"2023-10-09T10:54:40.577656+0000","last_became_peered":"2023-10-09T10:54:40.577656+0000","last_unstale":"2023-10-09T10:55:33.326636+0000","last_undegraded":"2023-10-09T10:55:33.326636+0000","last_fullsized":"2023-10-09T10:55:33.326636+0000","mapping_epoch":266,"log_start":"270'300","ondisk_log_start":"270'300","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6688,"log_dups_size":0,"ondisk_log_size":6688,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135626752,"num_objects":2069,"num_object_clones":0,"num_object_copies":6207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3993,"num_write_kb":194016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":620,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625220+0000","last_change":"2023-10-09T10:53:36.680150+0000","last_active":"2023-10-09T10:55:26.625220+0000","last_peered":"2023-10-09T10:55:26.625220+0000","last_clean":"2023-10-09T10:55:26.625220+0000","last_became_active":"2023-10-09T10:51:50.371032+0000","last_became_peered":"2023-10-09T10:51:50.371032+0000","last_unstale":"2023-10-09T10:55:26.625220+0000","last_undegraded":"2023-10-09T10:55:26.625220+0000","last_fullsized":"2023-10-09T10:55:26.625220+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.680090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.680090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:08:28.952827+0000","scrub_duration":0.005577172,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":55,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.634703+0000","last_change":"2023-10-09T10:53:14.841343+0000","last_active":"2023-10-09T10:55:26.634703+0000","last_peered":"2023-10-09T10:55:26.634703+0000","last_clean":"2023-10-09T10:55:26.634703+0000","last_became_active":"2023-10-09T10:53:14.828213+0000","last_became_peered":"2023-10-09T10:53:14.828213+0000","last_unstale":"2023-10-09T10:55:26.634703+0000","last_undegraded":"2023-10-09T10:55:26.634703+0000","last_fullsized":"2023-10-09T10:55:26.634703+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:28.137803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.646042+0000","last_change":"2023-10-09T10:54:47.942716+0000","last_active":"2023-10-09T10:55:26.646042+0000","last_peered":"2023-10-09T10:55:26.646042+0000","last_clean":"2023-10-09T10:55:26.646042+0000","last_became_active":"2023-10-09T10:54:47.942459+0000","last_became_peered":"2023-10-09T10:54:47.942459+0000","last_unstale":"2023-10-09T10:55:26.646042+0000","last_undegraded":"2023-10-09T10:55:26.646042+0000","last_fullsized":"2023-10-09T10:55:26.646042+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:07.790165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"307'6918","reported_seq":7005,"reported_epoch":307,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:55:30.191032+0000","last_change":"2023-10-09T10:54:49.183581+0000","last_active":"2023-10-09T10:55:30.191032+0000","last_peered":"2023-10-09T10:55:30.191032+0000","last_clean":"2023-10-09T10:54:02.391425+0000","last_became_active":"2023-10-09T10:54:49.111868+0000","last_became_peered":"2023-10-09T10:54:49.111868+0000","last_unstale":"2023-10-09T10:55:30.191032+0000","last_undegraded":"2023-10-09T10:54:48.912262+0000","last_fullsized":"2023-10-09T10:54:48.907341+0000","mapping_epoch":295,"log_start":"271'1000","ondisk_log_start":"271'1000","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5918,"log_dups_size":0,"ondisk_log_size":5918,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136806400,"num_objects":2087,"num_object_clones":0,"num_object_copies":6261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":697,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3941,"num_write_kb":192928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"3,4,6","objects":697},{"shards":"3,6","objects":1390}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":753,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625504+0000","last_change":"2023-10-09T10:53:38.681891+0000","last_active":"2023-10-09T10:55:26.625504+0000","last_peered":"2023-10-09T10:55:26.625504+0000","last_clean":"2023-10-09T10:55:26.625504+0000","last_became_active":"2023-10-09T10:53:17.840478+0000","last_became_peered":"2023-10-09T10:53:17.840478+0000","last_unstale":"2023-10-09T10:55:26.625504+0000","last_undegraded":"2023-10-09T10:55:26.625504+0000","last_fullsized":"2023-10-09T10:55:26.625504+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:31:14.758036+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":132,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.621305+0000","last_change":"2023-10-09T10:51:50.374527+0000","last_active":"2023-10-09T10:55:25.621305+0000","last_peered":"2023-10-09T10:55:25.621305+0000","last_clean":"2023-10-09T10:55:25.621305+0000","last_became_active":"2023-10-09T10:51:50.374297+0000","last_became_peered":"2023-10-09T10:51:50.374297+0000","last_unstale":"2023-10-09T10:55:25.621305+0000","last_undegraded":"2023-10-09T10:55:25.621305+0000","last_fullsized":"2023-10-09T10:55:25.621305+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:31:49.358307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":216,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.638516+0000","last_change":"2023-10-09T10:54:47.939771+0000","last_active":"2023-10-09T10:55:26.638516+0000","last_peered":"2023-10-09T10:55:26.638516+0000","last_clean":"2023-10-09T10:55:26.638516+0000","last_became_active":"2023-10-09T10:54:47.939440+0000","last_became_peered":"2023-10-09T10:54:47.939440+0000","last_unstale":"2023-10-09T10:55:26.638516+0000","last_undegraded":"2023-10-09T10:55:26.638516+0000","last_fullsized":"2023-10-09T10:55:26.638516+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:58.335536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"307'6985","reported_seq":8926,"reported_epoch":307,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:55:30.081370+0000","last_change":"2023-10-09T10:54:49.158909+0000","last_active":"2023-10-09T10:55:30.081370+0000","last_peered":"2023-10-09T10:55:30.081370+0000","last_clean":"2023-10-09T10:54:02.435878+0000","last_became_active":"2023-10-09T10:54:48.018286+0000","last_became_peered":"2023-10-09T10:54:48.018286+0000","last_unstale":"2023-10-09T10:55:30.081370+0000","last_undegraded":"2023-10-09T10:54:49.152454+0000","last_fullsized":"2023-10-09T10:54:47.891421+0000","mapping_epoch":294,"log_start":"271'1100","ondisk_log_start":"271'1100","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5885,"log_dups_size":0,"ondisk_log_size":5885,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139558912,"num_objects":2129,"num_object_clones":0,"num_object_copies":6387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":407,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3967,"num_write_kb":195104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":592,"num_bytes_recovered":38830080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":407},{"shards":"3,5","objects":1722}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":645,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625549+0000","last_change":"2023-10-09T10:55:25.628214+0000","last_active":"2023-10-09T10:55:26.625549+0000","last_peered":"2023-10-09T10:55:26.625549+0000","last_clean":"2023-10-09T10:55:26.625549+0000","last_became_active":"2023-10-09T10:55:25.627822+0000","last_became_peered":"2023-10-09T10:55:25.627822+0000","last_unstale":"2023-10-09T10:55:26.625549+0000","last_undegraded":"2023-10-09T10:55:26.625549+0000","last_fullsized":"2023-10-09T10:55:26.625549+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":303,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.620668+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:55:25.620668+0000","last_peered":"2023-10-09T10:55:25.620668+0000","last_clean":"2023-10-09T10:55:25.620668+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:55:25.620668+0000","last_undegraded":"2023-10-09T10:55:25.620668+0000","last_fullsized":"2023-10-09T10:55:25.620668+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":148,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.621240+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T10:55:25.621240+0000","last_peered":"2023-10-09T10:55:25.621240+0000","last_clean":"2023-10-09T10:55:25.621240+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T10:55:25.621240+0000","last_undegraded":"2023-10-09T10:55:25.621240+0000","last_fullsized":"2023-10-09T10:55:25.621240+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"308'6927","reported_seq":8948,"reported_epoch":308,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:55:33.275863+0000","last_change":"2023-10-09T10:54:49.220251+0000","last_active":"2023-10-09T10:55:33.275863+0000","last_peered":"2023-10-09T10:55:33.275863+0000","last_clean":"2023-10-09T10:54:02.426241+0000","last_became_active":"2023-10-09T10:54:49.172055+0000","last_became_peered":"2023-10-09T10:54:49.172055+0000","last_unstale":"2023-10-09T10:55:33.275863+0000","last_undegraded":"2023-10-09T10:54:49.015836+0000","last_fullsized":"2023-10-09T10:54:48.896714+0000","mapping_epoch":295,"log_start":"270'200","ondisk_log_start":"270'200","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6727,"log_dups_size":0,"ondisk_log_size":6727,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132775936,"num_objects":2026,"num_object_clones":0,"num_object_copies":6078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2301,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3966,"num_write_kb":191744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":970,"num_bytes_recovered":63569920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,1,5","objects":1338},{"shards":"0,2","objects":10},{"shards":"0,2,5","objects":393},{"shards":"0,3,5","objects":285}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":610,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.614674+0000","last_change":"2023-10-09T10:54:21.173078+0000","last_active":"2023-10-09T10:55:25.614674+0000","last_peered":"2023-10-09T10:55:25.614674+0000","last_clean":"2023-10-09T10:55:25.614674+0000","last_became_active":"2023-10-09T10:54:21.172767+0000","last_became_peered":"2023-10-09T10:54:21.172767+0000","last_unstale":"2023-10-09T10:55:25.614674+0000","last_undegraded":"2023-10-09T10:55:25.614674+0000","last_fullsized":"2023-10-09T10:55:25.614674+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.532890+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.532890+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:32.335841+0000","scrub_duration":0.0045290130000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":287,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.623576+0000","last_change":"2023-10-09T10:54:47.908506+0000","last_active":"2023-10-09T10:55:26.623576+0000","last_peered":"2023-10-09T10:55:26.623576+0000","last_clean":"2023-10-09T10:55:26.623576+0000","last_became_active":"2023-10-09T10:54:47.908110+0000","last_became_peered":"2023-10-09T10:54:47.908110+0000","last_unstale":"2023-10-09T10:55:26.623576+0000","last_undegraded":"2023-10-09T10:55:26.623576+0000","last_fullsized":"2023-10-09T10:55:26.623576+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:38.808943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":45,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.615876+0000","last_change":"2023-10-09T10:54:47.948732+0000","last_active":"2023-10-09T10:55:25.615876+0000","last_peered":"2023-10-09T10:55:25.615876+0000","last_clean":"2023-10-09T10:55:25.615876+0000","last_became_active":"2023-10-09T10:54:47.948414+0000","last_became_peered":"2023-10-09T10:54:47.948414+0000","last_unstale":"2023-10-09T10:55:25.615876+0000","last_undegraded":"2023-10-09T10:55:25.615876+0000","last_fullsized":"2023-10-09T10:55:25.615876+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:38.020807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.2","version":"308'7009","reported_seq":7072,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:55:31.271252+0000","last_change":"2023-10-09T10:55:24.369167+0000","last_active":"2023-10-09T10:55:31.271252+0000","last_peered":"2023-10-09T10:55:31.271252+0000","last_clean":"2023-10-09T10:55:31.271252+0000","last_became_active":"2023-10-09T10:54:40.567891+0000","last_became_peered":"2023-10-09T10:54:40.567891+0000","last_unstale":"2023-10-09T10:55:31.271252+0000","last_undegraded":"2023-10-09T10:55:31.271252+0000","last_fullsized":"2023-10-09T10:55:31.271252+0000","mapping_epoch":265,"log_start":"271'700","ondisk_log_start":"271'700","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6309,"log_dups_size":0,"ondisk_log_size":6309,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140115968,"num_objects":2138,"num_object_clones":0,"num_object_copies":6414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3974,"num_write_kb":195584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":682,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.647117+0000","last_change":"2023-10-09T10:54:50.914888+0000","last_active":"2023-10-09T10:55:26.647117+0000","last_peered":"2023-10-09T10:55:26.647117+0000","last_clean":"2023-10-09T10:55:26.647117+0000","last_became_active":"2023-10-09T10:54:50.909639+0000","last_became_peered":"2023-10-09T10:54:50.909639+0000","last_unstale":"2023-10-09T10:55:26.647117+0000","last_undegraded":"2023-10-09T10:55:26.647117+0000","last_fullsized":"2023-10-09T10:55:26.647117+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.103750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.103750+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":343,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.623157+0000","last_change":"2023-10-09T10:54:17.876795+0000","last_active":"2023-10-09T10:55:26.623157+0000","last_peered":"2023-10-09T10:55:26.623157+0000","last_clean":"2023-10-09T10:55:26.623157+0000","last_became_active":"2023-10-09T10:54:17.876504+0000","last_became_peered":"2023-10-09T10:54:17.876504+0000","last_unstale":"2023-10-09T10:55:26.623157+0000","last_undegraded":"2023-10-09T10:55:26.623157+0000","last_fullsized":"2023-10-09T10:55:26.623157+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:04:48.657295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.646313+0000","last_change":"2023-10-09T10:54:47.913826+0000","last_active":"2023-10-09T10:55:26.646313+0000","last_peered":"2023-10-09T10:55:26.646313+0000","last_clean":"2023-10-09T10:55:26.646313+0000","last_became_active":"2023-10-09T10:54:47.913524+0000","last_became_peered":"2023-10-09T10:54:47.913524+0000","last_unstale":"2023-10-09T10:55:26.646313+0000","last_undegraded":"2023-10-09T10:55:26.646313+0000","last_fullsized":"2023-10-09T10:55:26.646313+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:56:23.699552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"308'6893","reported_seq":7009,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:55:31.613193+0000","last_change":"2023-10-09T10:54:50.915546+0000","last_active":"2023-10-09T10:55:31.613193+0000","last_peered":"2023-10-09T10:55:31.613193+0000","last_clean":"2023-10-09T10:55:31.613193+0000","last_became_active":"2023-10-09T10:54:50.910131+0000","last_became_peered":"2023-10-09T10:54:50.910131+0000","last_unstale":"2023-10-09T10:55:31.613193+0000","last_undegraded":"2023-10-09T10:55:31.613193+0000","last_fullsized":"2023-10-09T10:55:31.613193+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6893,"log_dups_size":0,"ondisk_log_size":6893,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134053888,"num_objects":2045,"num_object_clones":0,"num_object_copies":6135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3935,"num_write_kb":191392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":2031616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":621,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.614889+0000","last_change":"2023-10-09T10:54:21.152148+0000","last_active":"2023-10-09T10:55:25.614889+0000","last_peered":"2023-10-09T10:55:25.614889+0000","last_clean":"2023-10-09T10:55:25.614889+0000","last_became_active":"2023-10-09T10:54:21.151809+0000","last_became_peered":"2023-10-09T10:54:21.151809+0000","last_unstale":"2023-10-09T10:55:25.614889+0000","last_undegraded":"2023-10-09T10:55:25.614889+0000","last_fullsized":"2023-10-09T10:55:25.614889+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.701128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.701128+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:27.021145+0000","scrub_duration":0.0025071830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":334,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.621755+0000","last_change":"2023-10-09T10:54:48.038946+0000","last_active":"2023-10-09T10:55:25.621755+0000","last_peered":"2023-10-09T10:55:25.621755+0000","last_clean":"2023-10-09T10:55:25.621755+0000","last_became_active":"2023-10-09T10:54:48.001493+0000","last_became_peered":"2023-10-09T10:54:48.001493+0000","last_unstale":"2023-10-09T10:55:25.621755+0000","last_undegraded":"2023-10-09T10:55:25.621755+0000","last_fullsized":"2023-10-09T10:55:25.621755+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:25.197466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.646688+0000","last_change":"2023-10-09T10:54:48.012411+0000","last_active":"2023-10-09T10:55:26.646688+0000","last_peered":"2023-10-09T10:55:26.646688+0000","last_clean":"2023-10-09T10:55:26.646688+0000","last_became_active":"2023-10-09T10:54:48.011894+0000","last_became_peered":"2023-10-09T10:54:48.011894+0000","last_unstale":"2023-10-09T10:55:26.646688+0000","last_undegraded":"2023-10-09T10:55:26.646688+0000","last_fullsized":"2023-10-09T10:55:26.646688+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:28.067337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"308'7988","reported_seq":8624,"reported_epoch":308,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:55:33.324512+0000","last_change":"2023-10-09T10:55:24.700754+0000","last_active":"2023-10-09T10:55:33.324512+0000","last_peered":"2023-10-09T10:55:33.324512+0000","last_clean":"2023-10-09T10:54:17.076172+0000","last_became_active":"2023-10-09T10:54:19.327879+0000","last_became_peered":"2023-10-09T10:54:19.327879+0000","last_unstale":"2023-10-09T10:55:33.324512+0000","last_undegraded":"2023-10-09T10:54:18.890724+0000","last_fullsized":"2023-10-09T10:54:18.868702+0000","mapping_epoch":282,"log_start":"271'2652","ondisk_log_start":"271'2652","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5336,"log_dups_size":0,"ondisk_log_size":5336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:49.273305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274202624,"num_objects":4184,"num_object_clones":0,"num_object_copies":12552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1616,"num_objects_misplaced":3389,"num_objects_unfound":0,"num_objects_dirty":4184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7988,"num_write_kb":389504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":287,"num_bytes_recovered":18808832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-821},{"shards":"0,4,7","objects":3389}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":703,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.622422+0000","last_change":"2023-10-09T10:53:34.602531+0000","last_active":"2023-10-09T10:55:25.622422+0000","last_peered":"2023-10-09T10:55:25.622422+0000","last_clean":"2023-10-09T10:55:25.622422+0000","last_became_active":"2023-10-09T10:53:17.841970+0000","last_became_peered":"2023-10-09T10:53:17.841970+0000","last_unstale":"2023-10-09T10:55:25.622422+0000","last_undegraded":"2023-10-09T10:55:25.622422+0000","last_fullsized":"2023-10-09T10:55:25.622422+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:04.601861+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.645807+0000","last_change":"2023-10-09T10:54:47.952216+0000","last_active":"2023-10-09T10:55:26.645807+0000","last_peered":"2023-10-09T10:55:26.645807+0000","last_clean":"2023-10-09T10:55:26.645807+0000","last_became_active":"2023-10-09T10:54:47.941070+0000","last_became_peered":"2023-10-09T10:54:47.941070+0000","last_unstale":"2023-10-09T10:55:26.645807+0000","last_undegraded":"2023-10-09T10:55:26.645807+0000","last_fullsized":"2023-10-09T10:55:26.645807+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:09.075266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":226,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.622192+0000","last_change":"2023-10-09T10:54:17.886921+0000","last_active":"2023-10-09T10:55:25.622192+0000","last_peered":"2023-10-09T10:55:25.622192+0000","last_clean":"2023-10-09T10:55:25.622192+0000","last_became_active":"2023-10-09T10:54:17.886584+0000","last_became_peered":"2023-10-09T10:54:17.886584+0000","last_unstale":"2023-10-09T10:55:25.622192+0000","last_undegraded":"2023-10-09T10:55:25.622192+0000","last_fullsized":"2023-10-09T10:55:25.622192+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:15.848951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.645881+0000","last_change":"2023-10-09T10:54:47.908294+0000","last_active":"2023-10-09T10:55:26.645881+0000","last_peered":"2023-10-09T10:55:26.645881+0000","last_clean":"2023-10-09T10:55:26.645881+0000","last_became_active":"2023-10-09T10:54:47.908019+0000","last_became_peered":"2023-10-09T10:54:47.908019+0000","last_unstale":"2023-10-09T10:55:26.645881+0000","last_undegraded":"2023-10-09T10:55:26.645881+0000","last_fullsized":"2023-10-09T10:55:26.645881+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:48.662764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"307'6885","reported_seq":6947,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:30.214551+0000","last_change":"2023-10-09T10:54:40.642047+0000","last_active":"2023-10-09T10:55:30.214551+0000","last_peered":"2023-10-09T10:55:30.214551+0000","last_clean":"2023-10-09T10:55:30.214551+0000","last_became_active":"2023-10-09T10:54:40.639305+0000","last_became_peered":"2023-10-09T10:54:40.639305+0000","last_unstale":"2023-10-09T10:55:30.214551+0000","last_undegraded":"2023-10-09T10:55:30.214551+0000","last_fullsized":"2023-10-09T10:55:30.214551+0000","mapping_epoch":266,"log_start":"271'1000","ondisk_log_start":"271'1000","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5885,"log_dups_size":0,"ondisk_log_size":5885,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141230080,"num_objects":2155,"num_object_clones":0,"num_object_copies":6465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3895,"num_write_kb":193600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":644,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.640574+0000","last_change":"2023-10-09T10:54:21.163072+0000","last_active":"2023-10-09T10:55:26.640574+0000","last_peered":"2023-10-09T10:55:26.640574+0000","last_clean":"2023-10-09T10:55:26.640574+0000","last_became_active":"2023-10-09T10:54:21.162710+0000","last_became_peered":"2023-10-09T10:54:21.162710+0000","last_unstale":"2023-10-09T10:55:26.640574+0000","last_undegraded":"2023-10-09T10:55:26.640574+0000","last_fullsized":"2023-10-09T10:55:26.640574+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:15.932806+0000","scrub_duration":0.0019146790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.645960+0000","last_change":"2023-10-09T10:54:47.899967+0000","last_active":"2023-10-09T10:55:26.645960+0000","last_peered":"2023-10-09T10:55:26.645960+0000","last_clean":"2023-10-09T10:55:26.645960+0000","last_became_active":"2023-10-09T10:54:47.899584+0000","last_became_peered":"2023-10-09T10:54:47.899584+0000","last_unstale":"2023-10-09T10:55:26.645960+0000","last_undegraded":"2023-10-09T10:55:26.645960+0000","last_fullsized":"2023-10-09T10:55:26.645960+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:43.973868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":217,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.638268+0000","last_change":"2023-10-09T10:50:43.388615+0000","last_active":"2023-10-09T10:55:26.638268+0000","last_peered":"2023-10-09T10:55:26.638268+0000","last_clean":"2023-10-09T10:55:26.638268+0000","last_became_active":"2023-10-09T10:50:43.388176+0000","last_became_peered":"2023-10-09T10:50:43.388176+0000","last_unstale":"2023-10-09T10:55:26.638268+0000","last_undegraded":"2023-10-09T10:55:26.638268+0000","last_fullsized":"2023-10-09T10:55:26.638268+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:41:17.550111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.645733+0000","last_change":"2023-10-09T10:54:47.950462+0000","last_active":"2023-10-09T10:55:26.645733+0000","last_peered":"2023-10-09T10:55:26.645733+0000","last_clean":"2023-10-09T10:55:26.645733+0000","last_became_active":"2023-10-09T10:54:47.940606+0000","last_became_peered":"2023-10-09T10:54:47.940606+0000","last_unstale":"2023-10-09T10:55:26.645733+0000","last_undegraded":"2023-10-09T10:55:26.645733+0000","last_fullsized":"2023-10-09T10:55:26.645733+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:50.557850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"307'7846","reported_seq":7925,"reported_epoch":307,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:55:30.276122+0000","last_change":"2023-10-09T10:54:49.182381+0000","last_active":"2023-10-09T10:55:30.276122+0000","last_peered":"2023-10-09T10:55:30.276122+0000","last_clean":"2023-10-09T10:54:02.429830+0000","last_became_active":"2023-10-09T10:54:49.176829+0000","last_became_peered":"2023-10-09T10:54:49.176829+0000","last_unstale":"2023-10-09T10:55:30.276122+0000","last_undegraded":"2023-10-09T10:54:48.956857+0000","last_fullsized":"2023-10-09T10:54:48.892711+0000","mapping_epoch":295,"log_start":"271'2755","ondisk_log_start":"271'2755","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5091,"log_dups_size":0,"ondisk_log_size":5091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:04.678256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280756224,"num_objects":4284,"num_object_clones":0,"num_object_copies":12852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1366,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7846,"num_write_kb":388160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"3,4","objects":2918},{"shards":"3,4,7","objects":1365}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":655,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625791+0000","last_change":"2023-10-09T10:55:25.634848+0000","last_active":"2023-10-09T10:55:26.625791+0000","last_peered":"2023-10-09T10:55:26.625791+0000","last_clean":"2023-10-09T10:55:26.625791+0000","last_became_active":"2023-10-09T10:55:25.634509+0000","last_became_peered":"2023-10-09T10:55:25.634509+0000","last_unstale":"2023-10-09T10:55:26.625791+0000","last_undegraded":"2023-10-09T10:55:26.625791+0000","last_fullsized":"2023-10-09T10:55:26.625791+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":36,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.615290+0000","last_change":"2023-10-09T10:54:17.885030+0000","last_active":"2023-10-09T10:55:25.615290+0000","last_peered":"2023-10-09T10:55:25.615290+0000","last_clean":"2023-10-09T10:55:25.615290+0000","last_became_active":"2023-10-09T10:54:17.884752+0000","last_became_peered":"2023-10-09T10:54:17.884752+0000","last_unstale":"2023-10-09T10:55:25.615290+0000","last_undegraded":"2023-10-09T10:55:25.615290+0000","last_fullsized":"2023-10-09T10:55:25.615290+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:38.548132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.645557+0000","last_change":"2023-10-09T10:54:47.927822+0000","last_active":"2023-10-09T10:55:26.645557+0000","last_peered":"2023-10-09T10:55:26.645557+0000","last_clean":"2023-10-09T10:55:26.645557+0000","last_became_active":"2023-10-09T10:54:47.927455+0000","last_became_peered":"2023-10-09T10:54:47.927455+0000","last_unstale":"2023-10-09T10:55:26.645557+0000","last_undegraded":"2023-10-09T10:55:26.645557+0000","last_fullsized":"2023-10-09T10:55:26.645557+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:11.818203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.646650+0000","last_change":"2023-10-09T10:54:48.012629+0000","last_active":"2023-10-09T10:55:26.646650+0000","last_peered":"2023-10-09T10:55:26.646650+0000","last_clean":"2023-10-09T10:55:26.646650+0000","last_became_active":"2023-10-09T10:54:48.012148+0000","last_became_peered":"2023-10-09T10:54:48.012148+0000","last_unstale":"2023-10-09T10:55:26.646650+0000","last_undegraded":"2023-10-09T10:55:26.646650+0000","last_fullsized":"2023-10-09T10:55:26.646650+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:22.894111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"308'6998","reported_seq":7060,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:55:33.331406+0000","last_change":"2023-10-09T10:54:40.559886+0000","last_active":"2023-10-09T10:55:33.331406+0000","last_peered":"2023-10-09T10:55:33.331406+0000","last_clean":"2023-10-09T10:55:33.331406+0000","last_became_active":"2023-10-09T10:54:40.557153+0000","last_became_peered":"2023-10-09T10:54:40.557153+0000","last_unstale":"2023-10-09T10:55:33.331406+0000","last_undegraded":"2023-10-09T10:55:33.331406+0000","last_fullsized":"2023-10-09T10:55:33.331406+0000","mapping_epoch":265,"log_start":"270'300","ondisk_log_start":"270'300","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6698,"log_dups_size":0,"ondisk_log_size":6698,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132841472,"num_objects":2027,"num_object_clones":0,"num_object_copies":6081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4013,"num_write_kb":193280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":36,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.615580+0000","last_change":"2023-10-09T10:54:17.884876+0000","last_active":"2023-10-09T10:55:25.615580+0000","last_peered":"2023-10-09T10:55:25.615580+0000","last_clean":"2023-10-09T10:55:25.615580+0000","last_became_active":"2023-10-09T10:54:17.884402+0000","last_became_peered":"2023-10-09T10:54:17.884402+0000","last_unstale":"2023-10-09T10:55:25.615580+0000","last_undegraded":"2023-10-09T10:55:25.615580+0000","last_fullsized":"2023-10-09T10:55:25.615580+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:18.679950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":684,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625533+0000","last_change":"2023-10-09T10:55:25.633525+0000","last_active":"2023-10-09T10:55:26.625533+0000","last_peered":"2023-10-09T10:55:26.625533+0000","last_clean":"2023-10-09T10:55:26.625533+0000","last_became_active":"2023-10-09T10:55:25.633079+0000","last_became_peered":"2023-10-09T10:55:25.633079+0000","last_unstale":"2023-10-09T10:55:26.625533+0000","last_undegraded":"2023-10-09T10:55:26.625533+0000","last_fullsized":"2023-10-09T10:55:26.625533+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:37.378259+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":238,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.620529+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:55:25.620529+0000","last_peered":"2023-10-09T10:55:25.620529+0000","last_clean":"2023-10-09T10:55:25.620529+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:55:25.620529+0000","last_undegraded":"2023-10-09T10:55:25.620529+0000","last_fullsized":"2023-10-09T10:55:25.620529+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":220,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.624199+0000","last_change":"2023-10-09T10:53:14.836523+0000","last_active":"2023-10-09T10:55:26.624199+0000","last_peered":"2023-10-09T10:55:26.624199+0000","last_clean":"2023-10-09T10:55:26.624199+0000","last_became_active":"2023-10-09T10:53:14.836134+0000","last_became_peered":"2023-10-09T10:53:14.836134+0000","last_unstale":"2023-10-09T10:55:26.624199+0000","last_undegraded":"2023-10-09T10:55:26.624199+0000","last_fullsized":"2023-10-09T10:55:26.624199+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:05:24.615198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"307'7749","reported_seq":8558,"reported_epoch":307,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:55:30.059966+0000","last_change":"2023-10-09T10:54:49.078027+0000","last_active":"2023-10-09T10:55:30.059966+0000","last_peered":"2023-10-09T10:55:30.059966+0000","last_clean":"2023-10-09T10:54:02.408036+0000","last_became_active":"2023-10-09T10:54:49.069949+0000","last_became_peered":"2023-10-09T10:54:49.069949+0000","last_unstale":"2023-10-09T10:55:30.059966+0000","last_undegraded":"2023-10-09T10:54:49.050557+0000","last_fullsized":"2023-10-09T10:54:48.898155+0000","mapping_epoch":295,"log_start":"271'2672","ondisk_log_start":"271'2672","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5077,"log_dups_size":0,"ondisk_log_size":5077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:38.092075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275841024,"num_objects":4209,"num_object_clones":0,"num_object_copies":12627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1353,"num_objects_misplaced":4209,"num_objects_unfound":0,"num_objects_dirty":4209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7749,"num_write_kb":382656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":366,"num_bytes_recovered":23986176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":1353},{"shards":"3,5","objects":2856}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":655,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.646994+0000","last_change":"2023-10-09T10:54:50.915855+0000","last_active":"2023-10-09T10:55:26.646994+0000","last_peered":"2023-10-09T10:55:26.646994+0000","last_clean":"2023-10-09T10:55:26.646994+0000","last_became_active":"2023-10-09T10:54:50.914536+0000","last_became_peered":"2023-10-09T10:54:50.914536+0000","last_unstale":"2023-10-09T10:55:26.646994+0000","last_undegraded":"2023-10-09T10:55:26.646994+0000","last_fullsized":"2023-10-09T10:55:26.646994+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.070026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.070026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":258,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625439+0000","last_change":"2023-10-09T10:55:25.633308+0000","last_active":"2023-10-09T10:55:26.625439+0000","last_peered":"2023-10-09T10:55:26.625439+0000","last_clean":"2023-10-09T10:55:26.625439+0000","last_became_active":"2023-10-09T10:55:25.632646+0000","last_became_peered":"2023-10-09T10:55:25.632646+0000","last_unstale":"2023-10-09T10:55:26.625439+0000","last_undegraded":"2023-10-09T10:55:26.625439+0000","last_fullsized":"2023-10-09T10:55:26.625439+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:17:13.735888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":189,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.640423+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:55:26.640423+0000","last_peered":"2023-10-09T10:55:26.640423+0000","last_clean":"2023-10-09T10:55:26.640423+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:55:26.640423+0000","last_undegraded":"2023-10-09T10:55:26.640423+0000","last_fullsized":"2023-10-09T10:55:26.640423+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":252,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.641409+0000","last_change":"2023-10-09T10:54:47.939756+0000","last_active":"2023-10-09T10:55:26.641409+0000","last_peered":"2023-10-09T10:55:26.641409+0000","last_clean":"2023-10-09T10:55:26.641409+0000","last_became_active":"2023-10-09T10:54:47.938928+0000","last_became_peered":"2023-10-09T10:54:47.938928+0000","last_unstale":"2023-10-09T10:55:26.641409+0000","last_undegraded":"2023-10-09T10:55:26.641409+0000","last_fullsized":"2023-10-09T10:55:26.641409+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:05.310929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"307'7907","reported_seq":7988,"reported_epoch":307,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:55:30.285996+0000","last_change":"2023-10-09T10:54:49.192234+0000","last_active":"2023-10-09T10:55:30.285996+0000","last_peered":"2023-10-09T10:55:30.285996+0000","last_clean":"2023-10-09T10:54:02.435746+0000","last_became_active":"2023-10-09T10:54:49.186762+0000","last_became_peered":"2023-10-09T10:54:49.186762+0000","last_unstale":"2023-10-09T10:55:30.285996+0000","last_undegraded":"2023-10-09T10:54:49.038902+0000","last_fullsized":"2023-10-09T10:54:48.912839+0000","mapping_epoch":295,"log_start":"271'2871","ondisk_log_start":"271'2871","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5036,"log_dups_size":0,"ondisk_log_size":5036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:48.327483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279379968,"num_objects":4263,"num_object_clones":0,"num_object_copies":12789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1330,"num_objects_misplaced":4263,"num_objects_unfound":0,"num_objects_dirty":4263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7907,"num_write_kb":389440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":2933},{"shards":"3,4,6","objects":1328}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":680,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.641564+0000","last_change":"2023-10-09T10:53:27.625180+0000","last_active":"2023-10-09T10:55:26.641564+0000","last_peered":"2023-10-09T10:55:26.641564+0000","last_clean":"2023-10-09T10:55:26.641564+0000","last_became_active":"2023-10-09T10:51:12.366828+0000","last_became_peered":"2023-10-09T10:51:12.366828+0000","last_unstale":"2023-10-09T10:55:26.641564+0000","last_undegraded":"2023-10-09T10:55:26.641564+0000","last_fullsized":"2023-10-09T10:55:26.641564+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.625079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.625079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:59:41.253976+0000","scrub_duration":0.0041067969999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":51,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.634568+0000","last_change":"2023-10-09T10:53:14.833073+0000","last_active":"2023-10-09T10:55:26.634568+0000","last_peered":"2023-10-09T10:55:26.634568+0000","last_clean":"2023-10-09T10:55:26.634568+0000","last_became_active":"2023-10-09T10:53:14.832736+0000","last_became_peered":"2023-10-09T10:53:14.832736+0000","last_unstale":"2023-10-09T10:55:26.634568+0000","last_undegraded":"2023-10-09T10:55:26.634568+0000","last_fullsized":"2023-10-09T10:55:26.634568+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:27.550707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.645985+0000","last_change":"2023-10-09T10:54:47.925711+0000","last_active":"2023-10-09T10:55:26.645985+0000","last_peered":"2023-10-09T10:55:26.645985+0000","last_clean":"2023-10-09T10:55:26.645985+0000","last_became_active":"2023-10-09T10:54:47.924286+0000","last_became_peered":"2023-10-09T10:54:47.924286+0000","last_unstale":"2023-10-09T10:55:26.645985+0000","last_undegraded":"2023-10-09T10:55:26.645985+0000","last_fullsized":"2023-10-09T10:55:26.645985+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:58.379080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":216,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.638384+0000","last_change":"2023-10-09T10:53:14.828291+0000","last_active":"2023-10-09T10:55:26.638384+0000","last_peered":"2023-10-09T10:55:26.638384+0000","last_clean":"2023-10-09T10:55:26.638384+0000","last_became_active":"2023-10-09T10:53:14.827991+0000","last_became_peered":"2023-10-09T10:53:14.827991+0000","last_unstale":"2023-10-09T10:55:26.638384+0000","last_undegraded":"2023-10-09T10:55:26.638384+0000","last_fullsized":"2023-10-09T10:55:26.638384+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:08:14.283112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"308'7991","reported_seq":8045,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:55:33.275239+0000","last_change":"2023-10-09T10:53:14.822279+0000","last_active":"2023-10-09T10:55:33.275239+0000","last_peered":"2023-10-09T10:55:33.275239+0000","last_clean":"2023-10-09T10:55:33.275239+0000","last_became_active":"2023-10-09T10:53:14.821952+0000","last_became_peered":"2023-10-09T10:53:14.821952+0000","last_unstale":"2023-10-09T10:55:33.275239+0000","last_undegraded":"2023-10-09T10:55:33.275239+0000","last_fullsized":"2023-10-09T10:55:33.275239+0000","mapping_epoch":266,"log_start":"271'2641","ondisk_log_start":"271'2641","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5350,"log_dups_size":0,"ondisk_log_size":5350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:37:14.124652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272826368,"num_objects":4163,"num_object_clones":0,"num_object_copies":12489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7991,"num_write_kb":388928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":703,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.624134+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T10:55:26.624134+0000","last_peered":"2023-10-09T10:55:26.624134+0000","last_clean":"2023-10-09T10:55:26.624134+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T10:55:26.624134+0000","last_undegraded":"2023-10-09T10:55:26.624134+0000","last_fullsized":"2023-10-09T10:55:26.624134+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":329,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.622613+0000","last_change":"2023-10-09T10:53:14.841466+0000","last_active":"2023-10-09T10:55:25.622613+0000","last_peered":"2023-10-09T10:55:25.622613+0000","last_clean":"2023-10-09T10:55:25.622613+0000","last_became_active":"2023-10-09T10:53:14.831674+0000","last_became_peered":"2023-10-09T10:53:14.831674+0000","last_unstale":"2023-10-09T10:55:25.622613+0000","last_undegraded":"2023-10-09T10:55:25.622613+0000","last_fullsized":"2023-10-09T10:55:25.622613+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:02.612069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":24,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.646441+0000","last_change":"2023-10-09T10:54:47.908305+0000","last_active":"2023-10-09T10:55:26.646441+0000","last_peered":"2023-10-09T10:55:26.646441+0000","last_clean":"2023-10-09T10:55:26.646441+0000","last_became_active":"2023-10-09T10:54:47.907999+0000","last_became_peered":"2023-10-09T10:54:47.907999+0000","last_unstale":"2023-10-09T10:55:26.646441+0000","last_undegraded":"2023-10-09T10:55:26.646441+0000","last_fullsized":"2023-10-09T10:55:26.646441+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:48.405124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":328,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.621818+0000","last_change":"2023-10-09T10:53:14.835841+0000","last_active":"2023-10-09T10:55:25.621818+0000","last_peered":"2023-10-09T10:55:25.621818+0000","last_clean":"2023-10-09T10:55:25.621818+0000","last_became_active":"2023-10-09T10:53:14.835551+0000","last_became_peered":"2023-10-09T10:53:14.835551+0000","last_unstale":"2023-10-09T10:55:25.621818+0000","last_undegraded":"2023-10-09T10:55:25.621818+0000","last_fullsized":"2023-10-09T10:55:25.621818+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:21.498099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"308'7762","reported_seq":11938,"reported_epoch":308,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:55:31.103694+0000","last_change":"2023-10-09T10:55:26.696179+0000","last_active":"2023-10-09T10:55:31.103694+0000","last_peered":"2023-10-09T10:55:31.103694+0000","last_clean":"2023-10-09T10:54:17.082254+0000","last_became_active":"2023-10-09T10:55:26.641906+0000","last_became_peered":"2023-10-09T10:55:26.641906+0000","last_unstale":"2023-10-09T10:55:31.103694+0000","last_undegraded":"2023-10-09T10:55:26.633572+0000","last_fullsized":"2023-10-09T10:55:26.625249+0000","mapping_epoch":306,"log_start":"271'2249","ondisk_log_start":"271'2249","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5513,"log_dups_size":0,"ondisk_log_size":5513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:02.762689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274268184,"num_objects":4186,"num_object_clones":0,"num_object_copies":12558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3090,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4186,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7762,"num_write_kb":382273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2055,"num_bytes_recovered":134610944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5],"avail_no_missing":["7","5","6"],"object_location_counts":[{"shards":"5,6,7","objects":3090},{"shards":"5,7","objects":1096}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":698,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.626605+0000","last_change":"2023-10-09T10:53:30.567189+0000","last_active":"2023-10-09T10:55:26.626605+0000","last_peered":"2023-10-09T10:55:26.626605+0000","last_clean":"2023-10-09T10:55:26.626605+0000","last_became_active":"2023-10-09T10:53:17.840928+0000","last_became_peered":"2023-10-09T10:53:17.840928+0000","last_unstale":"2023-10-09T10:55:26.626605+0000","last_undegraded":"2023-10-09T10:55:26.626605+0000","last_fullsized":"2023-10-09T10:55:26.626605+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:52.176229+0000","scrub_duration":0.0053707060000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":161,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625628+0000","last_change":"2023-10-09T10:55:25.634235+0000","last_active":"2023-10-09T10:55:26.625628+0000","last_peered":"2023-10-09T10:55:26.625628+0000","last_clean":"2023-10-09T10:55:26.625628+0000","last_became_active":"2023-10-09T10:55:25.633940+0000","last_became_peered":"2023-10-09T10:55:25.633940+0000","last_unstale":"2023-10-09T10:55:26.625628+0000","last_undegraded":"2023-10-09T10:55:26.625628+0000","last_fullsized":"2023-10-09T10:55:26.625628+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:40.554244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":236,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.641474+0000","last_change":"2023-10-09T10:54:48.016159+0000","last_active":"2023-10-09T10:55:26.641474+0000","last_peered":"2023-10-09T10:55:26.641474+0000","last_clean":"2023-10-09T10:55:26.641474+0000","last_became_active":"2023-10-09T10:54:47.939409+0000","last_became_peered":"2023-10-09T10:54:47.939409+0000","last_unstale":"2023-10-09T10:55:26.641474+0000","last_undegraded":"2023-10-09T10:55:26.641474+0000","last_fullsized":"2023-10-09T10:55:26.641474+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:11:15.175207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":249,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.623112+0000","last_change":"2023-10-09T10:51:50.371282+0000","last_active":"2023-10-09T10:55:26.623112+0000","last_peered":"2023-10-09T10:55:26.623112+0000","last_clean":"2023-10-09T10:55:26.623112+0000","last_became_active":"2023-10-09T10:51:50.370973+0000","last_became_peered":"2023-10-09T10:51:50.370973+0000","last_unstale":"2023-10-09T10:55:26.623112+0000","last_undegraded":"2023-10-09T10:55:26.623112+0000","last_fullsized":"2023-10-09T10:55:26.623112+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:15.603164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"307'6959","reported_seq":9844,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:30.240597+0000","last_change":"2023-10-09T10:54:56.638107+0000","last_active":"2023-10-09T10:55:30.240597+0000","last_peered":"2023-10-09T10:55:30.240597+0000","last_clean":"2023-10-09T10:55:30.240597+0000","last_became_active":"2023-10-09T10:54:56.633188+0000","last_became_peered":"2023-10-09T10:54:56.633188+0000","last_unstale":"2023-10-09T10:55:30.240597+0000","last_undegraded":"2023-10-09T10:55:30.240597+0000","last_fullsized":"2023-10-09T10:55:30.240597+0000","mapping_epoch":299,"log_start":"271'1100","ondisk_log_start":"271'1100","created":289,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5859,"log_dups_size":0,"ondisk_log_size":5859,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140869632,"num_objects":2150,"num_object_clones":0,"num_object_copies":6450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3942,"num_write_kb":194912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1061,"num_bytes_recovered":69500928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":697,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.620525+0000","last_change":"2023-10-09T10:53:29.568199+0000","last_active":"2023-10-09T10:55:25.620525+0000","last_peered":"2023-10-09T10:55:25.620525+0000","last_clean":"2023-10-09T10:55:25.620525+0000","last_became_active":"2023-10-09T10:52:55.416357+0000","last_became_peered":"2023-10-09T10:52:55.416357+0000","last_unstale":"2023-10-09T10:55:25.620525+0000","last_undegraded":"2023-10-09T10:55:25.620525+0000","last_fullsized":"2023-10-09T10:55:25.620525+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:44.364650+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":215,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.638454+0000","last_change":"2023-10-09T10:53:14.823731+0000","last_active":"2023-10-09T10:55:26.638454+0000","last_peered":"2023-10-09T10:55:26.638454+0000","last_clean":"2023-10-09T10:55:26.638454+0000","last_became_active":"2023-10-09T10:53:14.823470+0000","last_became_peered":"2023-10-09T10:53:14.823470+0000","last_unstale":"2023-10-09T10:55:26.638454+0000","last_undegraded":"2023-10-09T10:55:26.638454+0000","last_fullsized":"2023-10-09T10:55:26.638454+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:36.927599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":45,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.615727+0000","last_change":"2023-10-09T10:54:47.944380+0000","last_active":"2023-10-09T10:55:25.615727+0000","last_peered":"2023-10-09T10:55:25.615727+0000","last_clean":"2023-10-09T10:55:25.615727+0000","last_became_active":"2023-10-09T10:54:47.943766+0000","last_became_peered":"2023-10-09T10:54:47.943766+0000","last_unstale":"2023-10-09T10:55:25.615727+0000","last_undegraded":"2023-10-09T10:55:25.615727+0000","last_fullsized":"2023-10-09T10:55:25.615727+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:16:11.869603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"307'6813","reported_seq":6899,"reported_epoch":307,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:55:30.265177+0000","last_change":"2023-10-09T10:54:49.184288+0000","last_active":"2023-10-09T10:55:30.265177+0000","last_peered":"2023-10-09T10:55:30.265177+0000","last_clean":"2023-10-09T10:54:02.391425+0000","last_became_active":"2023-10-09T10:54:49.118164+0000","last_became_peered":"2023-10-09T10:54:49.118164+0000","last_unstale":"2023-10-09T10:55:30.265177+0000","last_undegraded":"2023-10-09T10:54:48.969517+0000","last_fullsized":"2023-10-09T10:54:48.892012+0000","mapping_epoch":295,"log_start":"271'1000","ondisk_log_start":"271'1000","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5813,"log_dups_size":0,"ondisk_log_size":5813,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143556608,"num_objects":2191,"num_object_clones":0,"num_object_copies":6573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":727,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3837,"num_write_kb":192864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"3,4,6","objects":727},{"shards":"3,6","objects":1464}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":629,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.621919+0000","last_change":"2023-10-09T10:53:36.563918+0000","last_active":"2023-10-09T10:55:25.621919+0000","last_peered":"2023-10-09T10:55:25.621919+0000","last_clean":"2023-10-09T10:55:25.621919+0000","last_became_active":"2023-10-09T10:53:17.842952+0000","last_became_peered":"2023-10-09T10:53:17.842952+0000","last_unstale":"2023-10-09T10:55:25.621919+0000","last_undegraded":"2023-10-09T10:55:25.621919+0000","last_fullsized":"2023-10-09T10:55:25.621919+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:53.185917+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":134,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.624487+0000","last_change":"2023-10-09T10:54:47.923386+0000","last_active":"2023-10-09T10:55:26.624487+0000","last_peered":"2023-10-09T10:55:26.624487+0000","last_clean":"2023-10-09T10:55:26.624487+0000","last_became_active":"2023-10-09T10:54:47.914257+0000","last_became_peered":"2023-10-09T10:54:47.914257+0000","last_unstale":"2023-10-09T10:55:26.624487+0000","last_undegraded":"2023-10-09T10:55:26.624487+0000","last_fullsized":"2023-10-09T10:55:26.624487+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:24:18.190715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":323,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.620746+0000","last_change":"2023-10-09T10:54:47.977518+0000","last_active":"2023-10-09T10:55:25.620746+0000","last_peered":"2023-10-09T10:55:25.620746+0000","last_clean":"2023-10-09T10:55:25.620746+0000","last_became_active":"2023-10-09T10:54:47.977332+0000","last_became_peered":"2023-10-09T10:54:47.977332+0000","last_unstale":"2023-10-09T10:55:25.620746+0000","last_undegraded":"2023-10-09T10:55:25.620746+0000","last_fullsized":"2023-10-09T10:55:25.620746+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:29.620839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.16","version":"308'6974","reported_seq":7036,"reported_epoch":308,"state":"active+recovery_wait","last_fresh":"2023-10-09T10:55:33.274358+0000","last_change":"2023-10-09T10:54:40.600782+0000","last_active":"2023-10-09T10:55:33.274358+0000","last_peered":"2023-10-09T10:55:33.274358+0000","last_clean":"2023-10-09T10:54:39.522361+0000","last_became_active":"2023-10-09T10:54:40.590969+0000","last_became_peered":"2023-10-09T10:54:40.590969+0000","last_unstale":"2023-10-09T10:55:33.274358+0000","last_undegraded":"2023-10-09T10:55:33.274358+0000","last_fullsized":"2023-10-09T10:55:33.274358+0000","mapping_epoch":289,"log_start":"270'300","ondisk_log_start":"270'300","created":289,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6674,"log_dups_size":0,"ondisk_log_size":6674,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136806400,"num_objects":2088,"num_object_clones":0,"num_object_copies":6264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3980,"num_write_kb":194144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2088}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":258,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.623866+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:55:26.623866+0000","last_peered":"2023-10-09T10:55:26.623866+0000","last_clean":"2023-10-09T10:55:26.623866+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:55:26.623866+0000","last_undegraded":"2023-10-09T10:55:26.623866+0000","last_fullsized":"2023-10-09T10:55:26.623866+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":250,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.621521+0000","last_change":"2023-10-09T10:50:43.385150+0000","last_active":"2023-10-09T10:55:25.621521+0000","last_peered":"2023-10-09T10:55:25.621521+0000","last_clean":"2023-10-09T10:55:25.621521+0000","last_became_active":"2023-10-09T10:50:43.384788+0000","last_became_peered":"2023-10-09T10:50:43.384788+0000","last_unstale":"2023-10-09T10:55:25.621521+0000","last_undegraded":"2023-10-09T10:55:25.621521+0000","last_fullsized":"2023-10-09T10:55:25.621521+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:36.085910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.15","version":"308'6878","reported_seq":6942,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-10-09T10:55:31.108795+0000","last_change":"2023-10-09T10:54:40.578264+0000","last_active":"2023-10-09T10:55:31.108795+0000","last_peered":"2023-10-09T10:55:31.108795+0000","last_clean":"2023-10-09T10:55:31.108795+0000","last_became_active":"2023-10-09T10:54:40.568498+0000","last_became_peered":"2023-10-09T10:54:40.568498+0000","last_unstale":"2023-10-09T10:55:31.108795+0000","last_undegraded":"2023-10-09T10:55:31.108795+0000","last_fullsized":"2023-10-09T10:55:31.108795+0000","mapping_epoch":289,"log_start":"271'400","ondisk_log_start":"271'400","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6478,"log_dups_size":0,"ondisk_log_size":6478,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135200768,"num_objects":2063,"num_object_clones":0,"num_object_copies":6189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3917,"num_write_kb":191360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":614,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.625747+0000","last_change":"2023-10-09T10:55:25.635605+0000","last_active":"2023-10-09T10:55:26.625747+0000","last_peered":"2023-10-09T10:55:26.625747+0000","last_clean":"2023-10-09T10:55:26.625747+0000","last_became_active":"2023-10-09T10:55:25.635328+0000","last_became_peered":"2023-10-09T10:55:25.635328+0000","last_unstale":"2023-10-09T10:55:26.625747+0000","last_undegraded":"2023-10-09T10:55:26.625747+0000","last_fullsized":"2023-10-09T10:55:26.625747+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:46.305288+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:46.305288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":330,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.622310+0000","last_change":"2023-10-09T10:54:17.886158+0000","last_active":"2023-10-09T10:55:25.622310+0000","last_peered":"2023-10-09T10:55:25.622310+0000","last_clean":"2023-10-09T10:55:25.622310+0000","last_became_active":"2023-10-09T10:54:17.885816+0000","last_became_peered":"2023-10-09T10:54:17.885816+0000","last_unstale":"2023-10-09T10:55:25.622310+0000","last_undegraded":"2023-10-09T10:55:25.622310+0000","last_fullsized":"2023-10-09T10:55:25.622310+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:05.270825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":333,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-10-09T10:55:25.620887+0000","last_change":"2023-10-09T10:54:17.957784+0000","last_active":"2023-10-09T10:55:25.620887+0000","last_peered":"2023-10-09T10:55:25.620887+0000","last_clean":"2023-10-09T10:55:25.620887+0000","last_became_active":"2023-10-09T10:54:17.954788+0000","last_became_peered":"2023-10-09T10:54:17.954788+0000","last_unstale":"2023-10-09T10:55:25.620887+0000","last_undegraded":"2023-10-09T10:55:25.620887+0000","last_fullsized":"2023-10-09T10:55:25.620887+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:48.709390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"308'6941","reported_seq":7016,"reported_epoch":308,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:55:31.191504+0000","last_change":"2023-10-09T10:54:42.558777+0000","last_active":"2023-10-09T10:55:31.191504+0000","last_peered":"2023-10-09T10:55:31.191504+0000","last_clean":"2023-10-09T10:54:17.036595+0000","last_became_active":"2023-10-09T10:54:42.553632+0000","last_became_peered":"2023-10-09T10:54:42.553632+0000","last_unstale":"2023-10-09T10:55:31.191504+0000","last_undegraded":"2023-10-09T10:55:31.191504+0000","last_fullsized":"2023-10-09T10:55:31.191504+0000","mapping_epoch":291,"log_start":"271'600","ondisk_log_start":"271'600","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6341,"log_dups_size":0,"ondisk_log_size":6341,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135888896,"num_objects":2074,"num_object_clones":0,"num_object_copies":6222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2074,"num_objects_unfound":0,"num_objects_dirty":2074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3956,"num_write_kb":192928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":2074}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":582,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.646001+0000","last_change":"2023-10-09T10:54:50.914473+0000","last_active":"2023-10-09T10:55:26.646001+0000","last_peered":"2023-10-09T10:55:26.646001+0000","last_clean":"2023-10-09T10:55:26.646001+0000","last_became_active":"2023-10-09T10:54:50.914005+0000","last_became_peered":"2023-10-09T10:54:50.914005+0000","last_unstale":"2023-10-09T10:55:26.646001+0000","last_undegraded":"2023-10-09T10:55:26.646001+0000","last_fullsized":"2023-10-09T10:55:26.646001+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.070492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.070492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":216,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.638441+0000","last_change":"2023-10-09T10:54:18.089053+0000","last_active":"2023-10-09T10:55:26.638441+0000","last_peered":"2023-10-09T10:55:26.638441+0000","last_clean":"2023-10-09T10:55:26.638441+0000","last_became_active":"2023-10-09T10:54:17.886070+0000","last_became_peered":"2023-10-09T10:54:17.886070+0000","last_unstale":"2023-10-09T10:55:26.638441+0000","last_undegraded":"2023-10-09T10:55:26.638441+0000","last_fullsized":"2023-10-09T10:55:26.638441+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:35.628407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":312,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-10-09T10:55:26.623071+0000","last_change":"2023-10-09T10:54:47.908059+0000","last_active":"2023-10-09T10:55:26.623071+0000","last_peered":"2023-10-09T10:55:26.623071+0000","last_clean":"2023-10-09T10:55:26.623071+0000","last_became_active":"2023-10-09T10:54:47.907780+0000","last_became_peered":"2023-10-09T10:54:47.907780+0000","last_unstale":"2023-10-09T10:55:26.623071+0000","last_undegraded":"2023-10-09T10:55:26.623071+0000","last_fullsized":"2023-10-09T10:55:26.623071+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:55:59.064617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":6668,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":4393992216,"num_objects":67048,"num_object_clones":0,"num_object_copies":201144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15190,"num_objects_misplaced":16030,"num_objects_unfound":0,"num_objects_dirty":67048,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":126256,"num_write_kb":6185665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6390,"num_bytes_recovered":418709504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13671346176,"data_stored":13671333960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":158943,"ondisk_log_size":158943,"up":78,"acting":68,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":729016,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":115,"num_read_kb":214,"num_write":56,"num_write_kb":567,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1105158,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332347,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2084932,"kb_used_data":1787644,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91660220,"statfs":{"total":95995035648,"available":93860065280,"internally_reserved":0,"allocated":1830547456,"data_stored":1829432830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:54:53 2023","interfaces":[{"interface":"back","average":{"1min":2.8740000000000001,"5min":1.913,"15min":1.2609999999999999},"min":{"1min":0.318,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":9.6259999999999994,"5min":9.6379999999999999,"15min":9.6379999999999999},"last":1.544},{"interface":"front","average":{"1min":2.3100000000000001,"5min":2.0059999999999998,"15min":1.4350000000000001},"min":{"1min":0.30099999999999999,"5min":0.246,"15min":0.24299999999999999},"max":{"1min":7.7859999999999996,"5min":16.731000000000002,"15min":16.731000000000002},"last":1.3959999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:55:01 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":2.0539999999999998,"15min":1.534},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.25700000000000001},"max":{"1min":3.9860000000000002,"5min":34.756,"15min":34.756},"last":9.0719999999999992},{"interface":"front","average":{"1min":1.6830000000000001,"5min":1.8280000000000001,"15min":1.319},"min":{"1min":0.32100000000000001,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":6.641,"5min":35.658999999999999,"15min":35.658999999999999},"last":1.4119999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:55:29 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":1.3480000000000001,"15min":1.3089999999999999},"min":{"1min":0.32300000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.8580000000000001,"5min":8.3620000000000001,"15min":11.058},"last":1.3120000000000001},{"interface":"front","average":{"1min":1.401,"5min":1.5660000000000001,"15min":1.232},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":4.8890000000000002,"5min":11.579000000000001,"15min":11.579000000000001},"last":0.81799999999999995}]},{"osd":3,"last update":"Mon Oct 9 10:54:53 2023","interfaces":[{"interface":"back","average":{"1min":2.4169999999999998,"5min":2.1230000000000002,"15min":1.607},"min":{"1min":0.36099999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":10.069000000000001,"5min":10.069000000000001,"15min":11.523},"last":1.075},{"interface":"front","average":{"1min":2.1429999999999998,"5min":1.7110000000000001,"15min":1.4330000000000001},"min":{"1min":0.38400000000000001,"5min":0.34399999999999997,"15min":0.32800000000000001},"max":{"1min":8.6300000000000008,"5min":9.9329999999999998,"15min":19.234999999999999},"last":3.4670000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:54:33 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":1.6950000000000001,"15min":1.2090000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":8.2940000000000005,"5min":16.695,"15min":16.695},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.123,"5min":1.4750000000000001,"15min":1.089},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26200000000000001},"max":{"1min":7.9320000000000004,"5min":9.0459999999999994,"15min":11.109999999999999},"last":0.44700000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:54:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.5840000000000001,"15min":1.1539999999999999},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.247},"max":{"1min":4.5999999999999996,"5min":13.262,"15min":13.262},"last":0.379},{"interface":"front","average":{"1min":1.4670000000000001,"5min":1.6799999999999999,"15min":1.3340000000000001},"min":{"1min":0.318,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":8.4339999999999993,"5min":8.8659999999999997,"15min":11.741},"last":0.46800000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:54:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":1.958,"15min":1.423},"min":{"1min":0.34499999999999997,"5min":0.317,"15min":0.28199999999999997},"max":{"1min":8.3610000000000007,"5min":14.913,"15min":14.913},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.73,"15min":1.262},"min":{"1min":0.317,"5min":0.317,"15min":0.29399999999999998},"max":{"1min":7.9809999999999999,"5min":12.268000000000001,"15min":19.187999999999999},"last":0.497}]}]},{"osd":6,"up_from":44,"seq":188978561278,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2932400,"kb_used_data":2028072,"kb_used_omap":0,"kb_used_meta":904256,"kb_avail":90812752,"statfs":{"total":95995035648,"available":92992258048,"internally_reserved":0,"allocated":2076745728,"data_stored":2075650868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925958144},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:54:54 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":1.331,"15min":1.181},"min":{"1min":0.252,"5min":0.216,"15min":0.216},"max":{"1min":5.2140000000000004,"5min":7.0170000000000003,"15min":12.288},"last":9.2680000000000007},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.603,"15min":1.2509999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.22800000000000001},"max":{"1min":4.8689999999999998,"5min":11.845000000000001,"15min":12.369999999999999},"last":6.3959999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:55:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4410000000000001,"5min":1.448,"15min":1.2689999999999999},"min":{"1min":0.32800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.6820000000000004,"5min":8.4730000000000008,"15min":16.747},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.585,"5min":1.54,"15min":1.321},"min":{"1min":0.30499999999999999,"5min":0.23599999999999999,"15min":0.23100000000000001},"max":{"1min":7.5049999999999999,"5min":8.0579999999999998,"15min":16.734000000000002},"last":0.46000000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:54:32 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":1.7490000000000001,"15min":1.294},"min":{"1min":0.30099999999999999,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":8.6389999999999993,"5min":8.6389999999999993,"15min":13.208},"last":8.0589999999999993},{"interface":"front","average":{"1min":2.1629999999999998,"5min":1.992,"15min":1.4710000000000001},"min":{"1min":0.31,"5min":0.29999999999999999,"15min":0.28399999999999997},"max":{"1min":12.83,"5min":12.83,"15min":12.83},"last":0.42899999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:54:54 2023","interfaces":[{"interface":"back","average":{"1min":1.667,"5min":1.871,"15min":1.3640000000000001},"min":{"1min":0.39500000000000002,"5min":0.29799999999999999,"15min":0.255},"max":{"1min":4.8380000000000001,"5min":7.8250000000000002,"15min":12.391},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.905,"5min":2.2050000000000001,"15min":1.5569999999999999},"min":{"1min":0.35099999999999998,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":5.0430000000000001,"5min":7.327,"15min":13.179},"last":2.0209999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:55:28 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.7010000000000001,"15min":1.171},"min":{"1min":0.251,"5min":0.247,"15min":0.23200000000000001},"max":{"1min":16.978999999999999,"5min":16.978999999999999,"15min":16.978999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.69399999999999995,"5min":1.347,"15min":1.1279999999999999},"min":{"1min":0.311,"5min":0.308,"15min":0.26400000000000001},"max":{"1min":3.6709999999999998,"5min":12.061999999999999,"15min":12.481999999999999},"last":0.59499999999999997}]},{"osd":5,"last update":"Mon Oct 9 10:55:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":1.869,"15min":1.292},"min":{"1min":0.28599999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.923,"5min":12.923,"15min":12.923},"last":0.37},{"interface":"front","average":{"1min":0.71799999999999997,"5min":1.532,"15min":1.1910000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.3100000000000001,"5min":12.103999999999999,"15min":12.456},"last":0.623}]},{"osd":7,"last update":"Mon Oct 9 10:54:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3759999999999999,"5min":1.748,"15min":1.335},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.26600000000000001},"max":{"1min":12.882,"5min":12.882,"15min":23.358000000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.2569999999999999,"15min":1.123},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.34100000000000003},"max":{"1min":6.1029999999999998,"5min":7.0880000000000001,"15min":12.422000000000001},"last":0.747}]}]},{"osd":1,"up_from":85,"seq":365072220317,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1789232,"kb_used_data":1422888,"kb_used_omap":9,"kb_used_meta":366198,"kb_avail":91955920,"statfs":{"total":95995035648,"available":94162862080,"internally_reserved":0,"allocated":1457037312,"data_stored":1455419784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":374987753},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.3959999999999999,"15min":1.4510000000000001},"min":{"1min":0.20999999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":12.188000000000001,"5min":12.188000000000001,"15min":18.027000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.8540000000000001,"5min":1.9059999999999999,"15min":1.5389999999999999},"min":{"1min":0.23400000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":18.303000000000001,"5min":18.303000000000001,"15min":18.303000000000001},"last":3.6190000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:55:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5899999999999999,"5min":1.599,"15min":1.341},"min":{"1min":0.23499999999999999,"5min":0.222,"15min":0.19700000000000001},"max":{"1min":12.244999999999999,"5min":12.244999999999999,"15min":15.356},"last":3.742},{"interface":"front","average":{"1min":3.3220000000000001,"5min":1.587,"15min":1.4750000000000001},"min":{"1min":0.23899999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":18.004000000000001,"5min":18.004000000000001,"15min":18.004000000000001},"last":3.0249999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:54:56 2023","interfaces":[{"interface":"back","average":{"1min":1.347,"5min":1.6779999999999999,"15min":1.5900000000000001},"min":{"1min":0.247,"5min":0.223,"15min":0.218},"max":{"1min":9.1059999999999999,"5min":9.1059999999999999,"15min":18.274999999999999},"last":0.95799999999999996},{"interface":"front","average":{"1min":1.718,"5min":2.0760000000000001,"15min":1.542},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":13.769,"5min":14.169,"15min":15.468999999999999},"last":3.3490000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":1.4810000000000001,"15min":1.2689999999999999},"min":{"1min":0.29699999999999999,"5min":0.26500000000000001,"15min":0.255},"max":{"1min":5.423,"5min":5.8559999999999999,"15min":12.167999999999999},"last":0.438},{"interface":"front","average":{"1min":2.073,"5min":1.661,"15min":1.4630000000000001},"min":{"1min":0.36799999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.835,"5min":8.9969999999999999,"15min":14.228999999999999},"last":0.92800000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":2.4969999999999999,"5min":1.756,"15min":1.327},"min":{"1min":0.33700000000000002,"5min":0.28100000000000003,"15min":0.253},"max":{"1min":9.6289999999999996,"5min":9.6289999999999996,"15min":9.7720000000000002},"last":3.395},{"interface":"front","average":{"1min":2.6429999999999998,"5min":2.0310000000000001,"15min":1.5980000000000001},"min":{"1min":0.41199999999999998,"5min":0.30299999999999999,"15min":0.28899999999999998},"max":{"1min":9.2460000000000004,"5min":9.2460000000000004,"15min":17.085000000000001},"last":0.48899999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.845,"15min":1.4219999999999999},"min":{"1min":0.35299999999999998,"5min":0.32800000000000001,"15min":0.27400000000000002},"max":{"1min":7.1360000000000001,"5min":10.425000000000001,"15min":10.425000000000001},"last":3.3769999999999998},{"interface":"front","average":{"1min":1.665,"5min":1.639,"15min":1.4159999999999999},"min":{"1min":0.437,"5min":0.312,"15min":0.312},"max":{"1min":6.0739999999999998,"5min":7.718,"15min":12.231},"last":1.7569999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":2.1139999999999999,"5min":1.867,"15min":1.48},"min":{"1min":0.39300000000000002,"5min":0.36299999999999999,"15min":0.34000000000000002},"max":{"1min":6.8289999999999997,"5min":16.312000000000001,"15min":16.312000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.0009999999999999,"5min":1.5680000000000001,"15min":1.4850000000000001},"min":{"1min":0.377,"5min":0.35799999999999998,"15min":0.32500000000000001},"max":{"1min":8.8149999999999995,"5min":8.8149999999999995,"15min":14.273999999999999},"last":1.3120000000000001}]}]},{"osd":0,"up_from":80,"seq":343597383851,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2074572,"kb_used_data":1601540,"kb_used_omap":28,"kb_used_meta":472995,"kb_avail":91670580,"statfs":{"total":95995035648,"available":93870673920,"internally_reserved":0,"allocated":1639976960,"data_stored":1638277140,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":484347728},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:54:59 2023","interfaces":[{"interface":"back","average":{"1min":2.609,"5min":1.7190000000000001,"15min":1.5369999999999999},"min":{"1min":0.24199999999999999,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":13.263999999999999,"5min":13.263999999999999,"15min":25.475000000000001},"last":0.98699999999999999},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.633,"15min":1.48},"min":{"1min":0.26000000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":9.0860000000000003,"5min":9.359,"15min":20.832999999999998},"last":1.907}]},{"osd":2,"last update":"Mon Oct 9 10:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":1.837,"15min":1.381},"min":{"1min":0.252,"5min":0.23100000000000001,"15min":0.19700000000000001},"max":{"1min":12.609999999999999,"5min":17.927,"15min":17.927},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.8620000000000001,"5min":2.0339999999999998,"15min":1.5409999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.20399999999999999},"max":{"1min":8.7119999999999997,"5min":14.111000000000001,"15min":14.391999999999999},"last":0.67300000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:54:59 2023","interfaces":[{"interface":"back","average":{"1min":2.3940000000000001,"5min":2.1360000000000001,"15min":1.7769999999999999},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.20200000000000001},"max":{"1min":10.456,"5min":13.166,"15min":14.926},"last":4.242},{"interface":"front","average":{"1min":3.2130000000000001,"5min":2.2269999999999999,"15min":1.762},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.224},"max":{"1min":13.305,"5min":13.305,"15min":14.391999999999999},"last":5.6130000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:54:50 2023","interfaces":[{"interface":"back","average":{"1min":6.3310000000000004,"5min":2.702,"15min":1.7430000000000001},"min":{"1min":0.38300000000000001,"5min":0.33800000000000002,"15min":0.26200000000000001},"max":{"1min":77.177999999999997,"5min":77.177999999999997,"15min":77.177999999999997},"last":0.64800000000000002},{"interface":"front","average":{"1min":1.716,"5min":2.0310000000000001,"15min":1.6299999999999999},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.427,"5min":10.427,"15min":12.318},"last":1.9650000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:54:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4329999999999998,"5min":2.2160000000000002,"15min":1.665},"min":{"1min":0.35299999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.478,"5min":10.478,"15min":12.157},"last":2.3679999999999999},{"interface":"front","average":{"1min":2.5779999999999998,"5min":2.1539999999999999,"15min":1.573},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.29499999999999998},"max":{"1min":24.754999999999999,"5min":24.754999999999999,"15min":24.754999999999999},"last":0.48199999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:54:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":2.2930000000000001,"15min":1.746},"min":{"1min":0.38200000000000001,"5min":0.34200000000000003,"15min":0.30399999999999999},"max":{"1min":10.384,"5min":10.384,"15min":12.207000000000001},"last":2.7320000000000002},{"interface":"front","average":{"1min":3.3180000000000001,"5min":2.1680000000000001,"15min":1.4690000000000001},"min":{"1min":0.48399999999999999,"5min":0.307,"15min":0.307},"max":{"1min":24.800000000000001,"5min":24.800000000000001,"15min":24.800000000000001},"last":0.51000000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:54:50 2023","interfaces":[{"interface":"back","average":{"1min":2.379,"5min":2.2210000000000001,"15min":1.641},"min":{"1min":0.48499999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":13.225,"5min":14.48,"15min":14.48},"last":1.0089999999999999},{"interface":"front","average":{"1min":6.2060000000000004,"5min":2.8149999999999999,"15min":1.873},"min":{"1min":0.42799999999999999,"5min":0.32600000000000001,"15min":0.32200000000000001},"max":{"1min":77.228999999999999,"5min":77.228999999999999,"15min":77.228999999999999},"last":1.861}]}]},{"osd":2,"up_from":90,"seq":386547056791,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":437888,"kb_used_data":178552,"kb_used_omap":81,"kb_used_meta":259246,"kb_avail":93307264,"statfs":{"total":95995035648,"available":95546638336,"internally_reserved":0,"allocated":182837248,"data_stored":181317593,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":265468122},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.595,"15min":1.2589999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":5.4039999999999999,"5min":18.512,"15min":18.512},"last":1.528},{"interface":"front","average":{"1min":1.49,"5min":1.8240000000000001,"15min":1.4139999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":6.7560000000000002,"5min":15.952999999999999,"15min":17.975000000000001},"last":2.9380000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":1.546,"15min":1.389},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20200000000000001},"max":{"1min":6.8179999999999996,"5min":8.4740000000000002,"15min":15.247999999999999},"last":2.9900000000000002},{"interface":"front","average":{"1min":1.55,"5min":1.1180000000000001,"15min":1.175},"min":{"1min":0.25600000000000001,"5min":0.253,"15min":0.20699999999999999},"max":{"1min":4.6020000000000003,"5min":6.3789999999999996,"15min":16.454999999999998},"last":1.4379999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:54:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":1.3460000000000001,"15min":1.2949999999999999},"min":{"1min":0.28100000000000003,"5min":0.26900000000000002,"15min":0.22500000000000001},"max":{"1min":7.9669999999999996,"5min":7.9669999999999996,"15min":14.113},"last":1.607},{"interface":"front","average":{"1min":1.292,"5min":1.7709999999999999,"15min":1.6000000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.20200000000000001},"max":{"1min":3.9529999999999998,"5min":15.561,"15min":15.561},"last":5.8780000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":1.7829999999999999,"15min":1.532},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.254},"max":{"1min":15.672000000000001,"5min":15.672000000000001,"15min":16.231999999999999},"last":1.415},{"interface":"front","average":{"1min":2.371,"5min":1.8600000000000001,"15min":1.4870000000000001},"min":{"1min":0.307,"5min":0.29899999999999999,"15min":0.26300000000000001},"max":{"1min":15.696999999999999,"5min":15.696999999999999,"15min":15.696999999999999},"last":1.627}]},{"osd":5,"last update":"Mon Oct 9 10:54:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1240000000000001,"5min":1.659,"15min":1.47},"min":{"1min":0.34399999999999997,"5min":0.31900000000000001,"15min":0.245},"max":{"1min":12.128,"5min":12.128,"15min":17.920000000000002},"last":2.8969999999999998},{"interface":"front","average":{"1min":3.0489999999999999,"5min":1.571,"15min":1.4810000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":16.597000000000001,"5min":16.597000000000001,"15min":17.882999999999999},"last":1.381}]},{"osd":6,"last update":"Mon Oct 9 10:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.609,"15min":1.278},"min":{"1min":0.34000000000000002,"5min":0.30499999999999999,"15min":0.247},"max":{"1min":5.3490000000000002,"5min":10.443,"15min":14.029999999999999},"last":1.5640000000000001},{"interface":"front","average":{"1min":1.9550000000000001,"5min":1.627,"15min":1.4399999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.29199999999999998},"max":{"1min":6.9500000000000002,"5min":6.9500000000000002,"15min":12.862},"last":3.012}]},{"osd":7,"last update":"Mon Oct 9 10:54:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3580000000000001,"5min":1.484,"15min":1.2829999999999999},"min":{"1min":0.38600000000000001,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":7.6109999999999998,"5min":7.6109999999999998,"15min":13.999000000000001},"last":1.587},{"interface":"front","average":{"1min":2.1960000000000002,"5min":1.7110000000000001,"15min":1.49},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.307},"max":{"1min":9.4830000000000005,"5min":9.4830000000000005,"15min":12.826000000000001},"last":2.9649999999999999}]}]},{"osd":3,"up_from":95,"seq":408021893265,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2661824,"kb_used_data":1994680,"kb_used_omap":7,"kb_used_meta":666936,"kb_avail":91083328,"statfs":{"total":95995035648,"available":93269327872,"internally_reserved":0,"allocated":2042552320,"data_stored":2040855068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":682943080},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":5.9539999999999997,"5min":2.4279999999999999,"15min":1.7070000000000001},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":38.817999999999998,"5min":38.817999999999998,"15min":38.817999999999998},"last":3.3279999999999998},{"interface":"front","average":{"1min":3.5680000000000001,"5min":2.9470000000000001,"15min":1.849},"min":{"1min":0.22900000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":10.398,"5min":17.312000000000001,"15min":17.312000000000001},"last":3.5409999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":2.8980000000000001,"5min":2.3250000000000002,"15min":1.827},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":12.401,"5min":12.401,"15min":14.707000000000001},"last":18.806999999999999},{"interface":"front","average":{"1min":4.0979999999999999,"5min":2.5899999999999999,"15min":1.77},"min":{"1min":0.22,"5min":0.22,"15min":0.193},"max":{"1min":18.526,"5min":18.526,"15min":18.526},"last":3.3530000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":2.7799999999999998,"5min":1.9990000000000001,"15min":1.6339999999999999},"min":{"1min":0.29999999999999999,"5min":0.24199999999999999,"15min":0.23699999999999999},"max":{"1min":10.144,"5min":10.144,"15min":13.853},"last":18.774000000000001},{"interface":"front","average":{"1min":4.6900000000000004,"5min":2.5670000000000002,"15min":1.8360000000000001},"min":{"1min":0.33700000000000002,"5min":0.248,"15min":0.248},"max":{"1min":14.081,"5min":17.268000000000001,"15min":17.268000000000001},"last":3.8889999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":3.726,"5min":2.073,"15min":1.6459999999999999},"min":{"1min":0.36099999999999999,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":11.863,"5min":11.863,"15min":20.388000000000002},"last":3.2309999999999999},{"interface":"front","average":{"1min":3.1549999999999998,"5min":1.9650000000000001,"15min":1.635},"min":{"1min":0.28399999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.269,"5min":11.680999999999999,"15min":19.085999999999999},"last":3.488}]},{"osd":5,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":3.4700000000000002,"5min":2.3530000000000002,"15min":1.9039999999999999},"min":{"1min":0.32200000000000001,"5min":0.29399999999999998,"15min":0.29199999999999998},"max":{"1min":14.045,"5min":14.045,"15min":51.509999999999998},"last":3.4500000000000002},{"interface":"front","average":{"1min":1.897,"5min":2.0129999999999999,"15min":1.883},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":6.7380000000000004,"5min":11.082000000000001,"15min":55.338999999999999},"last":3.056}]},{"osd":6,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":3.2549999999999999,"5min":2.181,"15min":1.6220000000000001},"min":{"1min":0.35099999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.726,"5min":11.723000000000001,"15min":12.361000000000001},"last":3.5150000000000001},{"interface":"front","average":{"1min":2.4750000000000001,"5min":2.2040000000000002,"15min":1.782},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":11.132999999999999,"5min":11.24,"15min":13.891999999999999},"last":3.0899999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":2.2709999999999999,"5min":2.0049999999999999,"15min":1.956},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.31},"max":{"1min":9.5990000000000002,"5min":10.222,"15min":51.744},"last":18.722000000000001},{"interface":"front","average":{"1min":4.1420000000000003,"5min":2.4910000000000001,"15min":1.986},"min":{"1min":0.42799999999999999,"5min":0.39400000000000002,"15min":0.31900000000000001},"max":{"1min":13.337,"5min":13.337,"15min":53.399000000000001},"last":3.2869999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019141,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2211900,"kb_used_data":1902324,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91533252,"statfs":{"total":95995035648,"available":93730050048,"internally_reserved":0,"allocated":1947979776,"data_stored":1946873027,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:54:52 2023","interfaces":[{"interface":"back","average":{"1min":13.236000000000001,"5min":3.7330000000000001,"15min":1.893},"min":{"1min":0.23899999999999999,"5min":0.217,"15min":0.20999999999999999},"max":{"1min":213.88999999999999,"5min":213.88999999999999,"15min":213.88999999999999},"last":3.4750000000000001},{"interface":"front","average":{"1min":13.113,"5min":4.3140000000000001,"15min":2.2160000000000002},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.217},"max":{"1min":214.09800000000001,"5min":214.09800000000001,"15min":214.09800000000001},"last":1.419}]},{"osd":1,"last update":"Mon Oct 9 10:54:57 2023","interfaces":[{"interface":"back","average":{"1min":12.721,"5min":3.7450000000000001,"15min":1.96},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":206.50399999999999,"5min":206.50399999999999,"15min":206.50399999999999},"last":0.26400000000000001},{"interface":"front","average":{"1min":12.779,"5min":4.0990000000000002,"15min":2.1800000000000002},"min":{"1min":0.38300000000000001,"5min":0.31,"15min":0.28000000000000003},"max":{"1min":206.529,"5min":206.529,"15min":206.529},"last":0.311}]},{"osd":2,"last update":"Mon Oct 9 10:54:32 2023","interfaces":[{"interface":"back","average":{"1min":2.2240000000000002,"5min":1.8740000000000001,"15min":1.407},"min":{"1min":0.316,"5min":0.28599999999999998,"15min":0.253},"max":{"1min":15.446999999999999,"5min":15.446999999999999,"15min":15.446999999999999},"last":0.435},{"interface":"front","average":{"1min":2.5790000000000002,"5min":1.8400000000000001,"15min":1.5269999999999999},"min":{"1min":0.29699999999999999,"5min":0.29599999999999999,"15min":0.26300000000000001},"max":{"1min":15.56,"5min":15.56,"15min":20.545999999999999},"last":0.47899999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:54:52 2023","interfaces":[{"interface":"back","average":{"1min":13.404,"5min":4.1950000000000003,"15min":2.2549999999999999},"min":{"1min":0.38700000000000001,"5min":0.29299999999999998,"15min":0.27400000000000002},"max":{"1min":209.03700000000001,"5min":209.03700000000001,"15min":209.03700000000001},"last":1.8500000000000001},{"interface":"front","average":{"1min":14.557,"5min":4.415,"15min":2.4350000000000001},"min":{"1min":0.34799999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":208.80199999999999,"5min":208.80199999999999,"15min":208.80199999999999},"last":0.46100000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:54:57 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.2639999999999998,"15min":1.554},"min":{"1min":0.25700000000000001,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":19.068999999999999,"5min":19.068999999999999,"15min":24.149999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":2.7930000000000001,"5min":1.762,"15min":1.379},"min":{"1min":0.312,"5min":0.27100000000000002,"15min":0.252},"max":{"1min":15.535,"5min":15.535,"15min":20.422000000000001},"last":0.51500000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:55:28 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":1.524,"15min":1.22},"min":{"1min":0.28000000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.85299999999999998,"5min":15.739000000000001,"15min":15.739000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.86399999999999999,"5min":2.0720000000000001,"15min":1.4630000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":4.2329999999999997,"5min":14.272,"15min":14.272},"last":0.42199999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:54:32 2023","interfaces":[{"interface":"back","average":{"1min":2.157,"5min":1.9410000000000001,"15min":1.327},"min":{"1min":0.40600000000000003,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":14.458,"5min":14.458,"15min":14.458},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.6920000000000002,"5min":1.667,"15min":1.216},"min":{"1min":0.379,"5min":0.255,"15min":0.255},"max":{"1min":15.592000000000001,"5min":15.592000000000001,"15min":15.592000000000001},"last":0.64400000000000002}]}]},{"osd":5,"up_from":37,"seq":158913790210,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2836860,"kb_used_data":2535476,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":90908292,"statfs":{"total":95995035648,"available":93090091008,"internally_reserved":0,"allocated":2596327424,"data_stored":2595225063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:54:57 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":2.1880000000000002,"15min":1.355},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":16.449000000000002,"5min":16.449000000000002,"15min":16.449000000000002},"last":0.30499999999999999},{"interface":"front","average":{"1min":2.2050000000000001,"5min":2.2389999999999999,"15min":1.417},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":16.489000000000001,"5min":16.489000000000001,"15min":16.489000000000001},"last":0.50700000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:55:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2869999999999999,"5min":2.2679999999999998,"15min":1.5820000000000001},"min":{"1min":0.318,"5min":0.215,"15min":0.215},"max":{"1min":8.2629999999999999,"5min":10.682,"15min":12.999000000000001},"last":5.5549999999999997},{"interface":"front","average":{"1min":1.9850000000000001,"5min":1.9810000000000001,"15min":1.3779999999999999},"min":{"1min":0.29199999999999998,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":9.2699999999999996,"5min":12.109,"15min":12.109},"last":5.024}]},{"osd":2,"last update":"Mon Oct 9 10:55:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.266,"15min":1.6000000000000001},"min":{"1min":0.28000000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.257999999999999,"5min":11.368,"15min":11.393000000000001},"last":2.3359999999999999},{"interface":"front","average":{"1min":1.667,"5min":2.1859999999999999,"15min":1.4059999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.4249999999999998,"5min":15.087999999999999,"15min":15.087999999999999},"last":0.36699999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:54:57 2023","interfaces":[{"interface":"back","average":{"1min":2.4369999999999998,"5min":2.7000000000000002,"15min":1.859},"min":{"1min":0.314,"5min":0.28599999999999998,"15min":0.25700000000000001},"max":{"1min":14.006,"5min":63.508000000000003,"15min":63.508000000000003},"last":0.64500000000000002},{"interface":"front","average":{"1min":2.8839999999999999,"5min":2.831,"15min":1.917},"min":{"1min":0.40000000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":10.887,"5min":62.418999999999997,"15min":62.418999999999997},"last":0.82599999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:55:06 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":1.367,"15min":1.3799999999999999},"min":{"1min":0.26300000000000001,"5min":0.247,"15min":0.23200000000000001},"max":{"1min":4.3369999999999997,"5min":6.6120000000000001,"15min":123.241},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.4930000000000001,"15min":1.573},"min":{"1min":0.28999999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.149,"5min":11.111000000000001,"15min":123.318},"last":0.33200000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:55:20 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":1.79,"15min":1.296},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.9049999999999994,"5min":11.098000000000001,"15min":11.098000000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.83899999999999997,"5min":1.7769999999999999,"15min":1.2170000000000001},"min":{"1min":0.32000000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.1779999999999999,"5min":15.16,"15min":20.256},"last":0.56200000000000006}]},{"osd":7,"last update":"Mon Oct 9 10:54:41 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":1.917,"15min":1.2769999999999999},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.28699999999999998},"max":{"1min":8.468,"5min":10.739000000000001,"15min":10.739000000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.47,"15min":1.129},"min":{"1min":0.35999999999999999,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":8.734,"5min":10.858000000000001,"15min":14.154},"last":0.60099999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1626013696,"data_stored":1626013696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1440612352,"data_stored":1440612352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":178196480,"data_stored":178192408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2032205824,"data_stored":2032205824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1927479296,"data_stored":1927479296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2588348416,"data_stored":2588344344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2063794176,"data_stored":2063794176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1814695936,"data_stored":1814691864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:55:36.642 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_2 pgp_num 26 2023-10-09T10:55:36.667 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:36.705 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:36.729 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:36.770 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:36.893 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:36.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:36.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:36.939 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:55:36.966 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:36 smithi155 ceph-mon[184854]: pgmap v740: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 15190/197139 objects degraded (7.705%); 15564/197139 objects misplaced (7.895%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T10:55:36.997 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:37.055 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:36 smithi125 ceph-mon[195507]: pgmap v740: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 15190/197139 objects degraded (7.705%); 15564/197139 objects misplaced (7.895%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T10:55:37.055 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:36 smithi125 ceph-mon[200197]: pgmap v740: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 15190/197139 objects degraded (7.705%); 15564/197139 objects misplaced (7.895%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T10:55:37.057 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:37.272 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:55:38.143 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:38.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:38.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:38.249 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:38.315 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:38.341 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:38.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:38.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:38.359 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:55:38.360 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:55:38.368 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:38.448 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:55:38.484 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:38.666 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:55:38.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:38.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:55:38.715 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:55:38.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:38.818 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:38.843 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:38.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:38.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:38.950 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:38.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:38 smithi155 ceph-mon[184854]: pgmap v741: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 15190/193620 objects degraded (7.845%); 15329/193620 objects misplaced (7.917%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T10:55:38.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:38 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 15190/193620 objects degraded (7.845%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:55:39.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:38 smithi125 ceph-mon[195507]: pgmap v741: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 15190/193620 objects degraded (7.845%); 15329/193620 objects misplaced (7.917%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T10:55:39.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:38 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 15190/193620 objects degraded (7.845%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:55:39.003 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:38 smithi125 ceph-mon[200197]: pgmap v741: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 15190/193620 objects degraded (7.845%); 15329/193620 objects misplaced (7.917%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T10:55:39.003 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:38 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 15190/193620 objects degraded (7.845%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:55:39.009 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:39.070 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:55:40.792 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:40.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:40.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:40.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:40.969 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:41.075 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:40 smithi125 ceph-mon[195507]: pgmap v742: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 15072/189978 objects degraded (7.934%); 15077/189978 objects misplaced (7.936%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:55:41.075 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:40 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:55:41.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:40 smithi125 ceph-mon[200197]: pgmap v742: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 15072/189978 objects degraded (7.934%); 15077/189978 objects misplaced (7.936%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:55:41.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:40 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:55:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:40 smithi155 ceph-mon[184854]: pgmap v742: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 15072/189978 objects degraded (7.934%); 15077/189978 objects misplaced (7.936%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:55:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:40 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:55:41.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:55:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:55:41] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.43.0" 2023-10-09T10:55:42.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:41 smithi125 ceph-mon[195507]: pgmap v743: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 15072/182772 objects degraded (8.246%); 14353/182772 objects misplaced (7.853%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:55:42.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:41 smithi125 ceph-mon[200197]: pgmap v743: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 15072/182772 objects degraded (8.246%); 14353/182772 objects misplaced (7.853%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:55:42.085 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:41 smithi155 ceph-mon[184854]: pgmap v743: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 15072/182772 objects degraded (8.246%); 14353/182772 objects misplaced (7.853%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T10:55:42.224 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:42.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:42.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:42.328 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:42.404 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:42.968 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:42.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:42.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:43.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:43.180 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:42 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 15072/182772 objects degraded (8.246%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:55:43.244 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:42 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 15072/182772 objects degraded (8.246%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:55:43.245 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:42 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 15072/182772 objects degraded (8.246%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:55:43.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:43.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:43.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:43.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:43.449 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:55:43.450 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:55:43.498 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:43.596 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:43.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:43.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:43.649 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:55:43.704 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:43.709 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:55:43.749 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:43.751 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:55:43.809 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:55:43.873 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:55:43.924 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:55:43.926 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:43.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:43.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:43.953 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:55:44.005 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:55:44.027 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:55:44.031 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:55:44.048 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:55:44.078 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:44.082 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:55:44.082 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:55:44.082 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:55:44.082 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:55:44.082 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:55:44.082 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:55:44.083 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:55:44.083 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:55:44.106 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:55:44.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:44.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:55:44.211 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:44.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:44 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2748597615' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:55:44.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:44 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:55:44.258 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:44 smithi155 ceph-mon[184854]: pgmap v744: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 14930/178269 objects degraded (8.375%); 14120/178269 objects misplaced (7.921%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T10:55:44.259 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:44 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/138602489' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:44.260 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:44.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:44 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2748597615' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:55:44.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:44 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:55:44.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:44 smithi125 ceph-mon[195507]: pgmap v744: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 14930/178269 objects degraded (8.375%); 14120/178269 objects misplaced (7.921%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T10:55:44.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:44 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/138602489' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:44.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:44 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2748597615' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:55:44.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:44 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:55:44.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:44 smithi125 ceph-mon[200197]: pgmap v744: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 14930/178269 objects degraded (8.375%); 14120/178269 objects misplaced (7.921%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T10:55:44.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:44 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/138602489' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:44.388 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:44.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:44.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:44.491 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:44.521 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:44.644 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:44.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:44.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:44.710 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:55:44.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:44.835 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:44.938 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:44.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:44.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:45.040 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 8 pgp_num to 26 2023-10-09T10:55:45.043 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:45.083 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:45.163 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:45.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:45.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:45.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:45 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:45.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:45 smithi155 ceph-mon[184854]: failed to encode map e309 with expected crc 2023-10-09T10:55:45.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:45 smithi155 ceph-mon[184854]: failed to encode map e309 with expected crc 2023-10-09T10:55:45.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:45 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:55:45.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:45 smithi155 ceph-mon[184854]: osdmap e309: 8 total, 8 up, 8 in 2023-10-09T10:55:45.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:45 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2508145922' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:55:45.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:45 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:55:45.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:45.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:45 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:45.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:45 smithi125 ceph-mon[195507]: failed to encode map e309 with expected crc 2023-10-09T10:55:45.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:45 smithi125 ceph-mon[195507]: failed to encode map e309 with expected crc 2023-10-09T10:55:45.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:45 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:55:45.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:45 smithi125 ceph-mon[195507]: osdmap e309: 8 total, 8 up, 8 in 2023-10-09T10:55:45.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:45 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2508145922' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:55:45.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:45 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:55:45.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:45 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:45.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:45 smithi125 ceph-mon[200197]: failed to encode map e309 with expected crc 2023-10-09T10:55:45.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:45 smithi125 ceph-mon[200197]: failed to encode map e309 with expected crc 2023-10-09T10:55:45.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:45 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:55:45.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:45 smithi125 ceph-mon[200197]: osdmap e309: 8 total, 8 up, 8 in 2023-10-09T10:55:45.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:45 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2508145922' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:55:45.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:45 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:55:45.302 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:45.335 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:45.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:45.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:45.440 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:45.511 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:45.550 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:45.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:45.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:45.655 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:45.694 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:45.717 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:45.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:45.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:45.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:45.884 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:45.972 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:45.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:45.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:46.055 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:55:46.055 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:55:46.056 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:55:46.056 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:55:46.056 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:55:46.079 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:46.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:46 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T10:55:46.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:46 smithi125 ceph-mon[195507]: osdmap e310: 8 total, 8 up, 8 in 2023-10-09T10:55:46.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:46.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:55:46.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:46 smithi125 ceph-mon[195507]: failed to encode map e310 with expected crc 2023-10-09T10:55:46.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:46 smithi125 ceph-mon[195507]: pgmap v747: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 14930/174081 objects degraded (8.576%); 13681/174081 objects misplaced (7.859%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T10:55:46.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:46 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T10:55:46.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:46 smithi125 ceph-mon[200197]: osdmap e310: 8 total, 8 up, 8 in 2023-10-09T10:55:46.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:46.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:55:46.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:46 smithi125 ceph-mon[200197]: failed to encode map e310 with expected crc 2023-10-09T10:55:46.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:46 smithi125 ceph-mon[200197]: pgmap v747: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 14930/174081 objects degraded (8.576%); 13681/174081 objects misplaced (7.859%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T10:55:46.144 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:46.234 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:46.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:46.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:46.341 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:46.402 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:46.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:46 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T10:55:46.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:46 smithi155 ceph-mon[184854]: osdmap e310: 8 total, 8 up, 8 in 2023-10-09T10:55:46.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:46.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:55:46.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:46 smithi155 ceph-mon[184854]: failed to encode map e310 with expected crc 2023-10-09T10:55:46.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:46 smithi155 ceph-mon[184854]: pgmap v747: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 14930/174081 objects degraded (8.576%); 13681/174081 objects misplaced (7.859%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T10:55:46.470 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:55:46.579 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:46.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:46.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:46.649 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:55:46.688 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:46.775 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:46.824 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:55:46.852 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:46.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:46.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:46.955 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:55:46.956 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:47.062 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:47.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:47 smithi155 ceph-mon[184854]: failed to encode map e310 with expected crc 2023-10-09T10:55:47.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:47 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:47.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:47 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:47.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:47 smithi125 ceph-mon[200197]: failed to encode map e310 with expected crc 2023-10-09T10:55:47.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:47 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:47.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:47 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:47.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:47 smithi125 ceph-mon[195507]: failed to encode map e310 with expected crc 2023-10-09T10:55:47.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:47 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:47.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:47 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:47.875 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:55:47.876 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:55:47.876 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:55:47.876 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:55:47.876 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.876 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:106416 6.3 6:c0486f03:::benchmark_data_smithi125_244951_object106415:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T10:55:47.876 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:46:02.234195+0000", 2023-10-09T10:55:47.876 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 585.638959602, 2023-10-09T10:55:47.876 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.048317987999999999, 2023-10-09T10:55:47.876 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.876 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.877 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.877 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:47.877 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:47.877 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 106416 2023-10-09T10:55:47.877 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.877 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.877 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.877 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.877 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234195+0000", 2023-10-09T10:55:47.877 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.877 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.877 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.878 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.878 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234195+0000", 2023-10-09T10:55:47.878 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.878 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.878 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.878 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.878 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234193+0000", 2023-10-09T10:55:47.878 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:55:47.878 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.878 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.878 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.879 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234285+0000", 2023-10-09T10:55:47.879 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1546999999999996e-05 2023-10-09T10:55:47.879 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.879 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.879 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.879 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234305+0000", 2023-10-09T10:55:47.879 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0119000000000002e-05 2023-10-09T10:55:47.879 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.879 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.879 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.879 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234313+0000", 2023-10-09T10:55:47.879 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3590000000000008e-06 2023-10-09T10:55:47.879 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.880 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.880 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.880 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234382+0000", 2023-10-09T10:55:47.880 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8261000000000003e-05 2023-10-09T10:55:47.880 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.880 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.880 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.880 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234490+0000", 2023-10-09T10:55:47.880 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000108398 2023-10-09T10:55:47.880 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.880 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.880 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T10:55:47.881 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.234588+0000", 2023-10-09T10:55:47.881 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7966000000000004e-05 2023-10-09T10:55:47.881 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.881 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.881 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.881 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.236668+0000", 2023-10-09T10:55:47.881 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020797540000000001 2023-10-09T10:55:47.881 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.881 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.881 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.881 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.236734+0000", 2023-10-09T10:55:47.881 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6644999999999996e-05 2023-10-09T10:55:47.882 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.882 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.882 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.882 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.282424+0000", 2023-10-09T10:55:47.882 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.045689354000000001 2023-10-09T10:55:47.882 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.882 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.882 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.882 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.282474+0000", 2023-10-09T10:55:47.882 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9938999999999997e-05 2023-10-09T10:55:47.882 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.883 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.883 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.883 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:46:02.282513+0000", 2023-10-09T10:55:47.883 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9168999999999997e-05 2023-10-09T10:55:47.883 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.883 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.883 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.883 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.883 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.883 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:137793 6.3 6:c3f14b3e:::benchmark_data_smithi125_244951_object31350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T10:55:47.883 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:55:47.883 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 522.69245833800005, 2023-10-09T10:55:47.884 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.046213456, 2023-10-09T10:55:47.884 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.884 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.884 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.884 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:47.884 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:47.884 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 137793 2023-10-09T10:55:47.884 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.884 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.884 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.884 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.884 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:55:47.885 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.885 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.885 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.885 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.885 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:55:47.885 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.885 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.885 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.885 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.885 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180696+0000", 2023-10-09T10:55:47.885 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:55:47.885 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.886 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.886 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.886 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180698+0000", 2023-10-09T10:55:47.886 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5009999999999999e-06 2023-10-09T10:55:47.886 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.886 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.886 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.886 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180702+0000", 2023-10-09T10:55:47.886 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2200000000000003e-06 2023-10-09T10:55:47.886 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.886 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.886 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.887 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.180705+0000", 2023-10-09T10:55:47.887 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7779999999999999e-06 2023-10-09T10:55:47.887 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.887 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.887 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.887 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182790+0000", 2023-10-09T10:55:47.887 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020852819999999999 2023-10-09T10:55:47.887 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.887 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.887 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.887 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.182925+0000", 2023-10-09T10:55:47.887 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013469099999999999 2023-10-09T10:55:47.888 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.888 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.888 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:55:47.888 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.183331+0000", 2023-10-09T10:55:47.888 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000405783 2023-10-09T10:55:47.888 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.888 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.888 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.888 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.185825+0000", 2023-10-09T10:55:47.888 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024937789999999998 2023-10-09T10:55:47.888 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.888 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.888 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.889 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.188205+0000", 2023-10-09T10:55:47.889 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023806790000000001 2023-10-09T10:55:47.889 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.889 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.889 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.889 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.225637+0000", 2023-10-09T10:55:47.889 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.037431091 2023-10-09T10:55:47.889 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.889 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.889 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.889 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.225680+0000", 2023-10-09T10:55:47.889 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3856999999999997e-05 2023-10-09T10:55:47.889 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.890 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.890 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.890 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:05.226909+0000", 2023-10-09T10:55:47.890 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001229129 2023-10-09T10:55:47.890 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.890 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.890 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.890 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.890 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.890 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:152093 6.13 6:cd7dbd99:::benchmark_data_smithi125_244951_object45650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T10:55:47.890 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.270440+0000", 2023-10-09T10:55:47.890 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 493.60271443599999, 2023-10-09T10:55:47.891 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040432893999999997, 2023-10-09T10:55:47.891 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.891 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.891 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.891 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:47.891 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:47.891 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 152093 2023-10-09T10:55:47.891 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.891 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.891 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.891 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.891 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270440+0000", 2023-10-09T10:55:47.892 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.892 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.892 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.892 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.892 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270440+0000", 2023-10-09T10:55:47.892 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.892 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.892 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.892 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.892 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270438+0000", 2023-10-09T10:55:47.892 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T10:55:47.892 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.892 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.893 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.893 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270450+0000", 2023-10-09T10:55:47.893 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1918e-05 2023-10-09T10:55:47.893 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.893 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.893 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.893 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270479+0000", 2023-10-09T10:55:47.893 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8911000000000002e-05 2023-10-09T10:55:47.893 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.893 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.893 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.893 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270488+0000", 2023-10-09T10:55:47.894 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6139999999999998e-06 2023-10-09T10:55:47.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.894 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.894 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.894 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270586+0000", 2023-10-09T10:55:47.894 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7823000000000002e-05 2023-10-09T10:55:47.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.894 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.894 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.894 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.270758+0000", 2023-10-09T10:55:47.894 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000172103 2023-10-09T10:55:47.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.895 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.895 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:55:47.895 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.271250+0000", 2023-10-09T10:55:47.895 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000491688 2023-10-09T10:55:47.895 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.895 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.895 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.895 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.278619+0000", 2023-10-09T10:55:47.895 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0073690730000000003 2023-10-09T10:55:47.895 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.895 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.895 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.895 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.284528+0000", 2023-10-09T10:55:47.896 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059091789999999996 2023-10-09T10:55:47.896 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.896 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.896 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.896 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.309269+0000", 2023-10-09T10:55:47.896 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024740522000000001 2023-10-09T10:55:47.896 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.896 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.896 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.896 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.309319+0000", 2023-10-09T10:55:47.896 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0912999999999999e-05 2023-10-09T10:55:47.896 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.897 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.897 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.897 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.310873+0000", 2023-10-09T10:55:47.897 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001553369 2023-10-09T10:55:47.897 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.897 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.897 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.897 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.897 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.897 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44570.0:152095 6.1 e174/164 6:8109f342:::benchmark_data_smithi125_244951_object45652:head v 174'5685, mlcod=174'5683)", 2023-10-09T10:55:47.897 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:34.273512+0000", 2023-10-09T10:55:47.897 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 493.599642496, 2023-10-09T10:55:47.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040480956999999998, 2023-10-09T10:55:47.898 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.898 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.898 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.898 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273512+0000", 2023-10-09T10:55:47.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.898 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.898 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273512+0000", 2023-10-09T10:55:47.899 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.899 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.899 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.899 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273511+0000", 2023-10-09T10:55:47.899 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:55:47.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.899 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.899 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.899 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273520+0000", 2023-10-09T10:55:47.899 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.645e-06 2023-10-09T10:55:47.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.900 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.900 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.900 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273534+0000", 2023-10-09T10:55:47.900 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3875e-05 2023-10-09T10:55:47.900 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.900 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.900 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.900 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273541+0000", 2023-10-09T10:55:47.900 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4790000000000001e-06 2023-10-09T10:55:47.900 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.900 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.900 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.900 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273796+0000", 2023-10-09T10:55:47.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025495599999999998 2023-10-09T10:55:47.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.901 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.901 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.901 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.273819+0000", 2023-10-09T10:55:47.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3175000000000001e-05 2023-10-09T10:55:47.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.901 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.901 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.901 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.313948+0000", 2023-10-09T10:55:47.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040129418 2023-10-09T10:55:47.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.902 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:34.313993+0000", 2023-10-09T10:55:47.902 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4549000000000003e-05 2023-10-09T10:55:47.902 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.902 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.902 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.902 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.902 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:162969 6.13 6:cec0b93d:::benchmark_data_smithi125_244951_object56526:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-10-09T10:55:47.902 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:47:49.943081+0000", 2023-10-09T10:55:47.902 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 477.93007383899999, 2023-10-09T10:55:47.903 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039960117000000003, 2023-10-09T10:55:47.903 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.903 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.903 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.903 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:47.903 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:47.903 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162969 2023-10-09T10:55:47.903 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.903 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.903 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.903 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.903 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943081+0000", 2023-10-09T10:55:47.904 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.904 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.904 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.904 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.904 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943081+0000", 2023-10-09T10:55:47.904 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.904 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.904 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.904 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.904 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943079+0000", 2023-10-09T10:55:47.904 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:55:47.904 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.904 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.905 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943085+0000", 2023-10-09T10:55:47.905 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6890000000000001e-06 2023-10-09T10:55:47.905 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.905 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.905 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943097+0000", 2023-10-09T10:55:47.905 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2166e-05 2023-10-09T10:55:47.905 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.905 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.905 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943103+0000", 2023-10-09T10:55:47.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9150000000000001e-06 2023-10-09T10:55:47.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.906 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.906 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.906 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943207+0000", 2023-10-09T10:55:47.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104573 2023-10-09T10:55:47.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.906 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.906 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.906 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943393+0000", 2023-10-09T10:55:47.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018516299999999999 2023-10-09T10:55:47.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.907 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:55:47.907 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.943913+0000", 2023-10-09T10:55:47.907 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00052048100000000005 2023-10-09T10:55:47.907 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.907 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.907 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.950165+0000", 2023-10-09T10:55:47.907 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062519439999999997 2023-10-09T10:55:47.907 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.908 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.908 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.956256+0000", 2023-10-09T10:55:47.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0060905630000000002 2023-10-09T10:55:47.908 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.908 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.908 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.908 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.981642+0000", 2023-10-09T10:55:47.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025386756999999999 2023-10-09T10:55:47.908 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.908 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.908 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.908 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.981697+0000", 2023-10-09T10:55:47.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4718e-05 2023-10-09T10:55:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:47:49.983041+0000", 2023-10-09T10:55:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001343639 2023-10-09T10:55:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44570.0:212706 6.13 6:c8228b1c:::benchmark_data_smithi125_244951_object106263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2023-10-09T10:55:47.909 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:49:02.392352+0000", 2023-10-09T10:55:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 405.48080246500001, 2023-10-09T10:55:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.041188625999999999, 2023-10-09T10:55:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44570", 2023-10-09T10:55:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/370533575", 2023-10-09T10:55:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212706 2023-10-09T10:55:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392352+0000", 2023-10-09T10:55:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392352+0000", 2023-10-09T10:55:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392350+0000", 2023-10-09T10:55:47.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:55:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392357+0000", 2023-10-09T10:55:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0380000000000002e-06 2023-10-09T10:55:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392369+0000", 2023-10-09T10:55:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1857e-05 2023-10-09T10:55:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392375+0000", 2023-10-09T10:55:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5219999999999998e-06 2023-10-09T10:55:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392421+0000", 2023-10-09T10:55:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5736999999999999e-05 2023-10-09T10:55:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.913 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392554+0000", 2023-10-09T10:55:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013335300000000001 2023-10-09T10:55:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:55:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.392651+0000", 2023-10-09T10:55:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7156000000000003e-05 2023-10-09T10:55:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.397156+0000", 2023-10-09T10:55:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045045550000000004 2023-10-09T10:55:47.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.400165+0000", 2023-10-09T10:55:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030094319999999998 2023-10-09T10:55:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.433364+0000", 2023-10-09T10:55:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.033199472000000001 2023-10-09T10:55:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.433409+0000", 2023-10-09T10:55:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4070000000000003e-05 2023-10-09T10:55:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:49:02.433541+0000", 2023-10-09T10:55:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013196400000000001 2023-10-09T10:55:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.916 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:47412 8.5 8:a7ec256e:::benchmark_data_smithi125_329074_object47411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T10:55:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:53:40.881628+0000", 2023-10-09T10:55:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 126.99152662900001, 2023-10-09T10:55:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.046307181000000003, 2023-10-09T10:55:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 47412 2023-10-09T10:55:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.917 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.881628+0000", 2023-10-09T10:55:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.881628+0000", 2023-10-09T10:55:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.918 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.881626+0000", 2023-10-09T10:55:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:55:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.881660+0000", 2023-10-09T10:55:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3698000000000002e-05 2023-10-09T10:55:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.881670+0000", 2023-10-09T10:55:47.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7450000000000007e-06 2023-10-09T10:55:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.881676+0000", 2023-10-09T10:55:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6579999999999999e-06 2023-10-09T10:55:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.883172+0000", 2023-10-09T10:55:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014959229999999999 2023-10-09T10:55:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.883302+0000", 2023-10-09T10:55:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013064700000000001 2023-10-09T10:55:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:55:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.883524+0000", 2023-10-09T10:55:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000221766 2023-10-09T10:55:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.885395+0000", 2023-10-09T10:55:47.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018711450000000001 2023-10-09T10:55:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.888824+0000", 2023-10-09T10:55:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034284419999999999 2023-10-09T10:55:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.927840+0000", 2023-10-09T10:55:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039016119000000002 2023-10-09T10:55:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.927895+0000", 2023-10-09T10:55:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5723999999999997e-05 2023-10-09T10:55:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:40.927935+0000", 2023-10-09T10:55:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9578e-05 2023-10-09T10:55:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83924 8.5 8:a629301e:::benchmark_data_smithi125_329074_object83923:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2023-10-09T10:55:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.041702+0000", 2023-10-09T10:55:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 90.831452068000004, 2023-10-09T10:55:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.043059291999999999, 2023-10-09T10:55:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83924 2023-10-09T10:55:47.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041702+0000", 2023-10-09T10:55:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041702+0000", 2023-10-09T10:55:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041701+0000", 2023-10-09T10:55:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:55:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041743+0000", 2023-10-09T10:55:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2562999999999999e-05 2023-10-09T10:55:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041768+0000", 2023-10-09T10:55:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4527000000000001e-05 2023-10-09T10:55:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041778+0000", 2023-10-09T10:55:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0559e-05 2023-10-09T10:55:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041849+0000", 2023-10-09T10:55:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0878999999999997e-05 2023-10-09T10:55:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041955+0000", 2023-10-09T10:55:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000105648 2023-10-09T10:55:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:55:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.042088+0000", 2023-10-09T10:55:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000133182 2023-10-09T10:55:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.047791+0000", 2023-10-09T10:55:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0057031039999999996 2023-10-09T10:55:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.056116+0000", 2023-10-09T10:55:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0083248560000000003 2023-10-09T10:55:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.084671+0000", 2023-10-09T10:55:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028554619 2023-10-09T10:55:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.084720+0000", 2023-10-09T10:55:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9555000000000002e-05 2023-10-09T10:55:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.084762+0000", 2023-10-09T10:55:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1335000000000003e-05 2023-10-09T10:55:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83927 8.5 8:a1c8355b:::benchmark_data_smithi125_329074_object83926:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2023-10-09T10:55:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.041807+0000", 2023-10-09T10:55:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 90.831346973999999, 2023-10-09T10:55:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.043092544000000003, 2023-10-09T10:55:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83927 2023-10-09T10:55:47.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041807+0000", 2023-10-09T10:55:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041807+0000", 2023-10-09T10:55:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.932 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041807+0000", 2023-10-09T10:55:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:55:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041867+0000", 2023-10-09T10:55:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0238999999999997e-05 2023-10-09T10:55:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041882+0000", 2023-10-09T10:55:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4769999999999999e-05 2023-10-09T10:55:47.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.041887+0000", 2023-10-09T10:55:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6559999999999997e-06 2023-10-09T10:55:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.044590+0000", 2023-10-09T10:55:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0027030079999999998 2023-10-09T10:55:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.044682+0000", 2023-10-09T10:55:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2077999999999995e-05 2023-10-09T10:55:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:55:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.044795+0000", 2023-10-09T10:55:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011243099999999999 2023-10-09T10:55:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.047799+0000", 2023-10-09T10:55:47.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030036529999999998 2023-10-09T10:55:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.056142+0000", 2023-10-09T10:55:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0083437849999999994 2023-10-09T10:55:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.084829+0000", 2023-10-09T10:55:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028686832999999998 2023-10-09T10:55:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.084867+0000", 2023-10-09T10:55:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7938999999999997e-05 2023-10-09T10:55:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.084900+0000", 2023-10-09T10:55:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2867999999999998e-05 2023-10-09T10:55:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:90373 8.5 8:ab1aeb30:::benchmark_data_smithi125_329074_object90372:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-10-09T10:55:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:30.432480+0000", 2023-10-09T10:55:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 77.440674040000005, 2023-10-09T10:55:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040172680000000002, 2023-10-09T10:55:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:47.938 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 90373 2023-10-09T10:55:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432480+0000", 2023-10-09T10:55:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432480+0000", 2023-10-09T10:55:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432480+0000", 2023-10-09T10:55:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:55:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432515+0000", 2023-10-09T10:55:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5957999999999997e-05 2023-10-09T10:55:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432525+0000", 2023-10-09T10:55:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1309999999999995e-06 2023-10-09T10:55:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432530+0000", 2023-10-09T10:55:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3299999999999998e-06 2023-10-09T10:55:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.437580+0000", 2023-10-09T10:55:47.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0050505460000000004 2023-10-09T10:55:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.437732+0000", 2023-10-09T10:55:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151889 2023-10-09T10:55:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:55:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.437890+0000", 2023-10-09T10:55:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015802500000000001 2023-10-09T10:55:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.444270+0000", 2023-10-09T10:55:47.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0063798969999999998 2023-10-09T10:55:47.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.451841+0000", 2023-10-09T10:55:47.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0075704609999999997 2023-10-09T10:55:47.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.472550+0000", 2023-10-09T10:55:47.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020708998999999999 2023-10-09T10:55:47.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.944 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.944 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.472605+0000", 2023-10-09T10:55:47.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4846999999999999e-05 2023-10-09T10:55:47.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.944 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.944 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.472653+0000", 2023-10-09T10:55:47.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8395e-05 2023-10-09T10:55:47.944 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.944 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.945 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.945 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.945 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.945 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:90375 8.5 8:aff9469c:::benchmark_data_smithi125_329074_object90374:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-10-09T10:55:47.945 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:30.432536+0000", 2023-10-09T10:55:47.945 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 77.440618510999997, 2023-10-09T10:55:47.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.040255572000000003, 2023-10-09T10:55:47.945 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.945 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.945 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.945 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:47.945 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:47.945 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 90375 2023-10-09T10:55:47.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.946 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.946 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432536+0000", 2023-10-09T10:55:47.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.946 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432536+0000", 2023-10-09T10:55:47.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432535+0000", 2023-10-09T10:55:47.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:55:47.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432561+0000", 2023-10-09T10:55:47.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6033e-05 2023-10-09T10:55:47.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432569+0000", 2023-10-09T10:55:47.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4070000000000004e-06 2023-10-09T10:55:47.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.432573+0000", 2023-10-09T10:55:47.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4410000000000003e-06 2023-10-09T10:55:47.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.440907+0000", 2023-10-09T10:55:47.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0083333880000000006 2023-10-09T10:55:47.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.441049+0000", 2023-10-09T10:55:47.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000142657 2023-10-09T10:55:47.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:55:47.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.441204+0000", 2023-10-09T10:55:47.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000154726 2023-10-09T10:55:47.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.950 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.448176+0000", 2023-10-09T10:55:47.950 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0069721679999999999 2023-10-09T10:55:47.950 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.950 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.451868+0000", 2023-10-09T10:55:47.950 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0036913610000000002 2023-10-09T10:55:47.950 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.472699+0000", 2023-10-09T10:55:47.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.02083115 2023-10-09T10:55:47.951 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.951 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.951 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.472752+0000", 2023-10-09T10:55:47.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3489000000000002e-05 2023-10-09T10:55:47.951 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.951 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.951 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:30.472791+0000", 2023-10-09T10:55:47.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9277999999999999e-05 2023-10-09T10:55:47.952 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.952 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.952 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.952 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.952 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.952 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:125308 8.d 8:b403002a:::benchmark_data_smithi125_329074_object28654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T10:55:47.952 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:55:30.155182+0000", 2023-10-09T10:55:47.952 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 17.717972863, 2023-10-09T10:55:47.952 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.048649175000000003, 2023-10-09T10:55:47.952 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.952 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.952 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.952 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:47.953 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:47.953 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 125308 2023-10-09T10:55:47.953 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.953 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.953 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.953 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:30.155182+0000", 2023-10-09T10:55:47.953 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.953 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.953 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.953 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:30.155182+0000", 2023-10-09T10:55:47.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.954 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.954 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.954 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:30.155180+0000", 2023-10-09T10:55:47.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:55:47.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.954 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.954 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.954 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:30.155187+0000", 2023-10-09T10:55:47.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8310000000000003e-06 2023-10-09T10:55:47.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:30.155200+0000", 2023-10-09T10:55:47.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3044000000000001e-05 2023-10-09T10:55:47.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:30.155207+0000", 2023-10-09T10:55:47.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9759999999999998e-06 2023-10-09T10:55:47.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:30.155243+0000", 2023-10-09T10:55:47.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6248999999999998e-05 2023-10-09T10:55:47.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:30.155346+0000", 2023-10-09T10:55:47.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103174 2023-10-09T10:55:47.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:55:47.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:30.155432+0000", 2023-10-09T10:55:47.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5538000000000007e-05 2023-10-09T10:55:47.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:30.160497+0000", 2023-10-09T10:55:47.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0050645710000000004 2023-10-09T10:55:47.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:30.161646+0000", 2023-10-09T10:55:47.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011494020000000001 2023-10-09T10:55:47.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:30.201427+0000", 2023-10-09T10:55:47.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039780784999999999 2023-10-09T10:55:47.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:30.201490+0000", 2023-10-09T10:55:47.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3559000000000006e-05 2023-10-09T10:55:47.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:30.203831+0000", 2023-10-09T10:55:47.959 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023404010000000002 2023-10-09T10:55:47.959 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.959 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.959 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.959 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.959 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.959 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:135717 8.d 8:bb547382:::benchmark_data_smithi125_329074_object39063:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-10-09T10:55:47.959 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:55:44.082842+0000", 2023-10-09T10:55:47.959 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 3.790312175, 2023-10-09T10:55:47.959 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.042185752999999999, 2023-10-09T10:55:47.959 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.959 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.960 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.960 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:47.960 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:47.960 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135717 2023-10-09T10:55:47.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.960 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.960 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.960 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.960 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:44.082842+0000", 2023-10-09T10:55:47.960 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.960 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.960 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:44.082842+0000", 2023-10-09T10:55:47.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.961 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:44.082840+0000", 2023-10-09T10:55:47.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:55:47.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.961 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:44.082850+0000", 2023-10-09T10:55:47.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1239999999999994e-06 2023-10-09T10:55:47.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.962 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:44.082867+0000", 2023-10-09T10:55:47.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7333000000000002e-05 2023-10-09T10:55:47.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.962 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:44.082875+0000", 2023-10-09T10:55:47.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3310000000000004e-06 2023-10-09T10:55:47.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:44.082971+0000", 2023-10-09T10:55:47.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6287999999999997e-05 2023-10-09T10:55:47.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:44.083164+0000", 2023-10-09T10:55:47.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00019220500000000001 2023-10-09T10:55:47.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:55:47.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:44.083351+0000", 2023-10-09T10:55:47.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018773100000000001 2023-10-09T10:55:47.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:44.088657+0000", 2023-10-09T10:55:47.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0053052799999999999 2023-10-09T10:55:47.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:44.092687+0000", 2023-10-09T10:55:47.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040299890000000003 2023-10-09T10:55:47.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:44.122655+0000", 2023-10-09T10:55:47.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029967994000000001 2023-10-09T10:55:47.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:44.122715+0000", 2023-10-09T10:55:47.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0278999999999997e-05 2023-10-09T10:55:47.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:44.125028+0000", 2023-10-09T10:55:47.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023130020000000002 2023-10-09T10:55:47.966 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.966 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.966 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.966 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.966 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:136490 8.d 8:b13fca49:::benchmark_data_smithi125_329074_object39836:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T10:55:47.966 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:55:45.207557+0000", 2023-10-09T10:55:47.966 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 2.6655977630000001, 2023-10-09T10:55:47.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.041132158000000002, 2023-10-09T10:55:47.966 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.966 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.966 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.967 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:47.967 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:47.967 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 136490 2023-10-09T10:55:47.967 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.967 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.967 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.967 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.207557+0000", 2023-10-09T10:55:47.967 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.967 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.967 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.968 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.207557+0000", 2023-10-09T10:55:47.968 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.968 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.968 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.968 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.968 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.207556+0000", 2023-10-09T10:55:47.968 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:55:47.968 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.968 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.968 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.968 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.207561+0000", 2023-10-09T10:55:47.968 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3519999999999997e-06 2023-10-09T10:55:47.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.969 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.207572+0000", 2023-10-09T10:55:47.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0698e-05 2023-10-09T10:55:47.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.969 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.207577+0000", 2023-10-09T10:55:47.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2120000000000002e-06 2023-10-09T10:55:47.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.207623+0000", 2023-10-09T10:55:47.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6078999999999997e-05 2023-10-09T10:55:47.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.207800+0000", 2023-10-09T10:55:47.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000176894 2023-10-09T10:55:47.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:55:47.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.207980+0000", 2023-10-09T10:55:47.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018054799999999999 2023-10-09T10:55:47.971 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.214217+0000", 2023-10-09T10:55:47.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062365149999999998 2023-10-09T10:55:47.971 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.216290+0000", 2023-10-09T10:55:47.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020732400000000001 2023-10-09T10:55:47.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.972 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.972 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.246521+0000", 2023-10-09T10:55:47.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030230773999999998 2023-10-09T10:55:47.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.972 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.972 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.246581+0000", 2023-10-09T10:55:47.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9799000000000001e-05 2023-10-09T10:55:47.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.248689+0000", 2023-10-09T10:55:47.973 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002108127 2023-10-09T10:55:47.973 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.973 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.973 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.973 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.973 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:136827 8.d 8:b814e7f5:::benchmark_data_smithi125_329074_object40173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T10:55:47.973 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:55:45.677329+0000", 2023-10-09T10:55:47.973 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 2.195825428, 2023-10-09T10:55:47.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.045645552999999998, 2023-10-09T10:55:47.974 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.974 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.974 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.974 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:47.974 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:47.974 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 136827 2023-10-09T10:55:47.974 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.974 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.974 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.974 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.974 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.677329+0000", 2023-10-09T10:55:47.975 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.975 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.975 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.975 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.975 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.677329+0000", 2023-10-09T10:55:47.975 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.975 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.975 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.975 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.975 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.677329+0000", 2023-10-09T10:55:47.975 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:55:47.975 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.677332+0000", 2023-10-09T10:55:47.976 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0120000000000001e-06 2023-10-09T10:55:47.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.677337+0000", 2023-10-09T10:55:47.976 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7799999999999997e-06 2023-10-09T10:55:47.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.977 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.677341+0000", 2023-10-09T10:55:47.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4359999999999998e-06 2023-10-09T10:55:47.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.977 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.682192+0000", 2023-10-09T10:55:47.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0048509900000000003 2023-10-09T10:55:47.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.977 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.682365+0000", 2023-10-09T10:55:47.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017317600000000001 2023-10-09T10:55:47.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:55:47.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.682549+0000", 2023-10-09T10:55:47.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000184169 2023-10-09T10:55:47.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.694431+0000", 2023-10-09T10:55:47.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011881750999999999 2023-10-09T10:55:47.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.979 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.979 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.694484+0000", 2023-10-09T10:55:47.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3480999999999998e-05 2023-10-09T10:55:47.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.979 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.979 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.720577+0000", 2023-10-09T10:55:47.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026092917 2023-10-09T10:55:47.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.979 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.979 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.720626+0000", 2023-10-09T10:55:47.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9118e-05 2023-10-09T10:55:47.980 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.980 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.980 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.980 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.722974+0000", 2023-10-09T10:55:47.980 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002348087 2023-10-09T10:55:47.980 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.980 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.980 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.980 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.980 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.980 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:136828 8.d 8:b43e070d:::benchmark_data_smithi125_329074_object40174:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T10:55:47.980 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:55:45.680450+0000", 2023-10-09T10:55:47.980 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 2.1927046570000002, 2023-10-09T10:55:47.981 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.044465351, 2023-10-09T10:55:47.981 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.981 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.981 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.981 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:47.981 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:47.981 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 136828 2023-10-09T10:55:47.981 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.981 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.981 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.981 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.981 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680450+0000", 2023-10-09T10:55:47.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.982 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.982 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.982 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680450+0000", 2023-10-09T10:55:47.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.982 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.982 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.982 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680449+0000", 2023-10-09T10:55:47.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:55:47.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680453+0000", 2023-10-09T10:55:47.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2610000000000004e-06 2023-10-09T10:55:47.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680473+0000", 2023-10-09T10:55:47.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9820999999999999e-05 2023-10-09T10:55:47.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680480+0000", 2023-10-09T10:55:47.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.627e-06 2023-10-09T10:55:47.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.685422+0000", 2023-10-09T10:55:47.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0049412049999999997 2023-10-09T10:55:47.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.685594+0000", 2023-10-09T10:55:47.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017232799999999999 2023-10-09T10:55:47.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.985 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:55:47.985 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.686028+0000", 2023-10-09T10:55:47.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00043385599999999999 2023-10-09T10:55:47.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.985 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.985 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.694493+0000", 2023-10-09T10:55:47.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0084653039999999999 2023-10-09T10:55:47.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.694542+0000", 2023-10-09T10:55:47.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8498999999999998e-05 2023-10-09T10:55:47.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.723046+0000", 2023-10-09T10:55:47.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028504162999999999 2023-10-09T10:55:47.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.723103+0000", 2023-10-09T10:55:47.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7376000000000001e-05 2023-10-09T10:55:47.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.724915+0000", 2023-10-09T10:55:47.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018119919999999999 2023-10-09T10:55:47.987 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.987 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.987 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.988 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:136829 8.d 8:b777f9c3:::benchmark_data_smithi125_329074_object40175:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T10:55:47.988 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:55:45.680487+0000", 2023-10-09T10:55:47.988 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 2.1926674230000001, 2023-10-09T10:55:47.988 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.046326241999999997, 2023-10-09T10:55:47.988 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.988 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.988 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.988 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:47.988 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:47.988 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 136829 2023-10-09T10:55:47.988 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.989 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680487+0000", 2023-10-09T10:55:47.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680487+0000", 2023-10-09T10:55:47.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680486+0000", 2023-10-09T10:55:47.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:55:47.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680490+0000", 2023-10-09T10:55:47.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.191e-06 2023-10-09T10:55:47.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680497+0000", 2023-10-09T10:55:47.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6550000000000004e-06 2023-10-09T10:55:47.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680502+0000", 2023-10-09T10:55:47.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7029999999999997e-06 2023-10-09T10:55:47.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.691743+0000", 2023-10-09T10:55:47.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011241104 2023-10-09T10:55:47.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.992 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.691895+0000", 2023-10-09T10:55:47.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000152139 2023-10-09T10:55:47.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:55:47.992 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.692013+0000", 2023-10-09T10:55:47.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011788100000000001 2023-10-09T10:55:47.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.703930+0000", 2023-10-09T10:55:47.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01191732 2023-10-09T10:55:47.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:47.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.704537+0000", 2023-10-09T10:55:47.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00060678900000000005 2023-10-09T10:55:47.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.724970+0000", 2023-10-09T10:55:47.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020433020999999999 2023-10-09T10:55:47.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:47.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.725011+0000", 2023-10-09T10:55:47.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1056999999999997e-05 2023-10-09T10:55:47.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:47.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.726813+0000", 2023-10-09T10:55:47.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018018719999999999 2023-10-09T10:55:47.994 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.994 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:47.995 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:47.995 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.995 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.995 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:136830 8.d 8:b728c7b8:::benchmark_data_smithi125_329074_object40176:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T10:55:47.995 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:55:45.680506+0000", 2023-10-09T10:55:47.995 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 2.1926481, 2023-10-09T10:55:47.995 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047951427999999997, 2023-10-09T10:55:47.995 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:47.995 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:47.995 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:47.995 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:47.995 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:47.995 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 136830 2023-10-09T10:55:47.995 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.996 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:47.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.996 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:47.996 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680506+0000", 2023-10-09T10:55:47.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.996 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:47.996 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680506+0000", 2023-10-09T10:55:47.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:47.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:47.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680506+0000", 2023-10-09T10:55:47.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:55:47.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:47.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680508+0000", 2023-10-09T10:55:47.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.616e-06 2023-10-09T10:55:47.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:47.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680514+0000", 2023-10-09T10:55:47.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6439999999999997e-06 2023-10-09T10:55:47.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:47.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680518+0000", 2023-10-09T10:55:47.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1160000000000001e-06 2023-10-09T10:55:47.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:47.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.694568+0000", 2023-10-09T10:55:47.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014049378 2023-10-09T10:55:47.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:47.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.694730+0000", 2023-10-09T10:55:47.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000161986 2023-10-09T10:55:47.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:55:47.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.694906+0000", 2023-10-09T10:55:47.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017605499999999999 2023-10-09T10:55:47.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:47.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:47.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:47.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.704525+0000", 2023-10-09T10:55:48.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0096190000000000008 2023-10-09T10:55:48.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.000 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:48.000 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.704548+0000", 2023-10-09T10:55:48.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3161999999999999e-05 2023-10-09T10:55:48.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.000 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:48.000 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.726878+0000", 2023-10-09T10:55:48.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022330470000000002 2023-10-09T10:55:48.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:48.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.726940+0000", 2023-10-09T10:55:48.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2048e-05 2023-10-09T10:55:48.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:48.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.728458+0000", 2023-10-09T10:55:48.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001517343 2023-10-09T10:55:48.001 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:48.001 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:48.001 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:48.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.002 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:136833 8.d 8:bcfe2f85:::benchmark_data_smithi125_329074_object40179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T10:55:48.002 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:55:45.680522+0000", 2023-10-09T10:55:48.002 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 2.1926321820000001, 2023-10-09T10:55:48.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.053403137000000003, 2023-10-09T10:55:48.002 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:48.002 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:48.002 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:48.002 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:48.002 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:48.003 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 136833 2023-10-09T10:55:48.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.003 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:48.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:48.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680522+0000", 2023-10-09T10:55:48.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:48.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:48.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680522+0000", 2023-10-09T10:55:48.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:48.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:48.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680522+0000", 2023-10-09T10:55:48.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:55:48.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:48.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680524+0000", 2023-10-09T10:55:48.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.418e-06 2023-10-09T10:55:48.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:48.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680532+0000", 2023-10-09T10:55:48.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8490000000000006e-06 2023-10-09T10:55:48.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:48.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.680536+0000", 2023-10-09T10:55:48.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.109e-06 2023-10-09T10:55:48.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:48.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.697326+0000", 2023-10-09T10:55:48.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016789664999999999 2023-10-09T10:55:48.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:48.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.697530+0000", 2023-10-09T10:55:48.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000204002 2023-10-09T10:55:48.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:55:48.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.697695+0000", 2023-10-09T10:55:48.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016470699999999999 2023-10-09T10:55:48.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:48.007 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.704554+0000", 2023-10-09T10:55:48.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0068594629999999997 2023-10-09T10:55:48.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.007 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:48.007 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.706585+0000", 2023-10-09T10:55:48.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020310850000000002 2023-10-09T10:55:48.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.007 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:48.007 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.731872+0000", 2023-10-09T10:55:48.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025286798999999999 2023-10-09T10:55:48.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:48.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.731963+0000", 2023-10-09T10:55:48.008 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0896000000000005e-05 2023-10-09T10:55:48.008 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:48.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.733925+0000", 2023-10-09T10:55:48.008 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019624920000000001 2023-10-09T10:55:48.008 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:48.008 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:48.008 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:48.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.009 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.009 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:136838 8.d 8:b9a31146:::benchmark_data_smithi125_329074_object40184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T10:55:48.009 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:55:45.686288+0000", 2023-10-09T10:55:48.009 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 2.1868665539999999, 2023-10-09T10:55:48.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.049122289999999999, 2023-10-09T10:55:48.009 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:55:48.009 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:55:48.009 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:55:48.009 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:55:48.009 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:55:48.009 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 136838 2023-10-09T10:55:48.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.010 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:55:48.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.010 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:55:48.010 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.686288+0000", 2023-10-09T10:55:48.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:48.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.010 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:55:48.010 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.686288+0000", 2023-10-09T10:55:48.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:55:48.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:55:48.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.686287+0000", 2023-10-09T10:55:48.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:55:48.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:55:48.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.686292+0000", 2023-10-09T10:55:48.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.806e-06 2023-10-09T10:55:48.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:55:48.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.686304+0000", 2023-10-09T10:55:48.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1731e-05 2023-10-09T10:55:48.012 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:55:48.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.686310+0000", 2023-10-09T10:55:48.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1530000000000002e-06 2023-10-09T10:55:48.012 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:55:48.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.700926+0000", 2023-10-09T10:55:48.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014615642 2023-10-09T10:55:48.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:55:48.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.701094+0000", 2023-10-09T10:55:48.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016804299999999999 2023-10-09T10:55:48.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5", 2023-10-09T10:55:48.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.701260+0000", 2023-10-09T10:55:48.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000166102 2023-10-09T10:55:48.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:55:48.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.704561+0000", 2023-10-09T10:55:48.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0033013769999999999 2023-10-09T10:55:48.014 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:48.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.706749+0000", 2023-10-09T10:55:48.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021871709999999999 2023-10-09T10:55:48.014 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:55:48.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.733980+0000", 2023-10-09T10:55:48.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027231699000000002 2023-10-09T10:55:48.015 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.015 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.015 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:55:48.015 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.734040+0000", 2023-10-09T10:55:48.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0204000000000002e-05 2023-10-09T10:55:48.015 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:55:48.015 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:55:48.015 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:55:48.015 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:55:45.735410+0000", 2023-10-09T10:55:48.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0013696139999999999 2023-10-09T10:55:48.015 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:48.015 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:48.016 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:48.016 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:55:48.016 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:55:48.016 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:55:48.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:48.017 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:48.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:48.052 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:48.094 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:48.118 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:48.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:48 smithi155 ceph-mon[184854]: pgmap v748: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 14930/170826 objects degraded (8.740%); 13443/170826 objects misplaced (7.869%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T10:55:48.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:48 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 14930/174081 objects degraded (8.576%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:55:48.158 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:48.285 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:48.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:48.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:48.292 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:55:48.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:48 smithi125 ceph-mon[195507]: pgmap v748: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 14930/170826 objects degraded (8.740%); 13443/170826 objects misplaced (7.869%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T10:55:48.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:48 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 14930/174081 objects degraded (8.576%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:55:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:48 smithi125 ceph-mon[200197]: pgmap v748: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 14930/170826 objects degraded (8.740%); 13443/170826 objects misplaced (7.869%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T10:55:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:48 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 14930/174081 objects degraded (8.576%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T10:55:48.389 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:48.471 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:48.801 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:48.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:48.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:48.908 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:48.941 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:48.965 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:48.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:48.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:49.069 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:55:49.142 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:49 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:49 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:49.238 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:55:49.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:49.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:55:49.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:49 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:49.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:49 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:49.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:49 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:49.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:49 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:49.342 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:49.406 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:50.403 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:50.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:50.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:50.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:50 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:55:50.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:50 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:55:50.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:50 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:50.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:50 smithi125 ceph-mon[195507]: pgmap v749: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 14807/167628 objects degraded (8.833%); 13226/167628 objects misplaced (7.890%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T10:55:50.410 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:50 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:55:50.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:50 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:55:50.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:50 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:50.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:50 smithi125 ceph-mon[200197]: pgmap v749: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 14807/167628 objects degraded (8.833%); 13226/167628 objects misplaced (7.890%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T10:55:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:50 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:55:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:50 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:55:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:50 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:55:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:50 smithi155 ceph-mon[184854]: pgmap v749: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 14807/167628 objects degraded (8.833%); 13226/167628 objects misplaced (7.890%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T10:55:50.511 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:55:50.531 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 7, 2, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:55:50.531 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:55:50.532 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T10:55:50.532 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:55:50.532 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:55:50.532 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T10:55:50.543 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:50.571 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:55:50.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:50.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:55:50.674 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:50.713 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:50.735 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:50.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:50.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:50.839 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:55:50.886 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:50.919 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:55:50.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:55:50.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:50.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:55:51.072 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:55:51.113 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:51.147 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:55:51.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:51.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:55:51.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:51.314 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:51.573 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:55:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:55:51] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.43.0" 2023-10-09T10:55:51.945 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:55:51.945 INFO:teuthology.orchestra.run.smithi155.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T10:55:51.945 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:55:52.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:52 smithi155 ceph-mon[184854]: pgmap v750: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 14807/161004 objects degraded (9.197%); 12581/161004 objects misplaced (7.814%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T10:55:52.757 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:52 smithi125 ceph-mon[195507]: pgmap v750: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 14807/161004 objects degraded (9.197%); 12581/161004 objects misplaced (7.814%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T10:55:52.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:52 smithi125 ceph-mon[200197]: pgmap v750: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 14807/161004 objects degraded (9.197%); 12581/161004 objects misplaced (7.814%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T10:55:53.173 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:55:53.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:53 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 14807/161004 objects degraded (9.197%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:55:53.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:53 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 14807/161004 objects degraded (9.197%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:55:53.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:53 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 14807/161004 objects degraded (9.197%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:55:54.144 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:54.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:54.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:54.319 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:55:54.349 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:54.369 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:55:54.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:54.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:55:54.472 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:54.515 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:54 smithi125 ceph-mon[195507]: pgmap v751: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 14683/156855 objects degraded (9.361%); 12343/156855 objects misplaced (7.869%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T10:55:54.528 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:54.568 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:54.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:54.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:54.674 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:55:54.702 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:54 smithi155 ceph-mon[184854]: pgmap v751: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 14683/156855 objects degraded (9.361%); 12343/156855 objects misplaced (7.869%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T10:55:54.704 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:54.728 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:55:54.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:54.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:55:54.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:54 smithi125 ceph-mon[200197]: pgmap v751: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 14683/156855 objects degraded (9.361%); 12343/156855 objects misplaced (7.869%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T10:55:54.835 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:54.890 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:56.108 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:55:56.170 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:55:56.211 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:56.216 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:55:56.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:56.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:56.268 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:55:56.320 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:56.321 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:55:56.378 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:55:56.390 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:56.429 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:55:56.471 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:55:56.483 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:55:56.483 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:55:56.483 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:55:56.484 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:55:56.484 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:55:56.484 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:55:56.484 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:55:56.484 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:55:56.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:56 smithi125 ceph-mon[195507]: pgmap v752: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 14683/152307 objects degraded (9.640%); 11833/152307 objects misplaced (7.769%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T10:55:56.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3582466877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:56.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:56 smithi125 ceph-mon[200197]: pgmap v752: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 14683/152307 objects degraded (9.640%); 11833/152307 objects misplaced (7.769%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T10:55:56.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3582466877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:56.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:56 smithi155 ceph-mon[184854]: pgmap v752: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 14683/152307 objects degraded (9.640%); 11833/152307 objects misplaced (7.769%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T10:55:56.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3582466877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:55:57.353 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 7, 2, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:55:57.354 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:55:57.354 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_2 2023-10-09T10:55:57.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:55:57.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:55:57.583 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:55:57.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:57.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:55:57.690 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:55:57.723 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:57.743 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:55:57.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:57.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:55:57.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:57 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3279982650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:55:57.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:57 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:55:57.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:57 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3279982650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:55:57.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:57 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:55:57.848 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:55:57.897 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:57 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3279982650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:55:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:57 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:55:58.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:58 smithi155 ceph-mon[184854]: pgmap v753: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 14683/148413 objects degraded (9.893%); 11556/148413 objects misplaced (7.786%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T10:55:58.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:58 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:58.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:58 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:55:58.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:58 smithi155 ceph-mon[184854]: osdmap e311: 8 total, 8 up, 8 in 2023-10-09T10:55:58.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:58 smithi155 ceph-mon[184854]: failed to encode map e311 with expected crc 2023-10-09T10:55:58.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:58 smithi155 ceph-mon[184854]: failed to encode map e311 with expected crc 2023-10-09T10:55:58.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:55:58 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 14683/148413 objects degraded (9.893%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:55:59.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:58 smithi125 ceph-mon[195507]: pgmap v753: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 14683/148413 objects degraded (9.893%); 11556/148413 objects misplaced (7.786%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T10:55:59.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:58 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:59.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:58 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:55:59.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:58 smithi125 ceph-mon[195507]: osdmap e311: 8 total, 8 up, 8 in 2023-10-09T10:55:59.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:58 smithi125 ceph-mon[195507]: failed to encode map e311 with expected crc 2023-10-09T10:55:59.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:58 smithi125 ceph-mon[195507]: failed to encode map e311 with expected crc 2023-10-09T10:55:59.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:55:58 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 14683/148413 objects degraded (9.893%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:55:59.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:58 smithi125 ceph-mon[200197]: pgmap v753: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 14683/148413 objects degraded (9.893%); 11556/148413 objects misplaced (7.786%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T10:55:59.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:58 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:55:59.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:58 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:55:59.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:58 smithi125 ceph-mon[200197]: osdmap e311: 8 total, 8 up, 8 in 2023-10-09T10:55:59.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:58 smithi125 ceph-mon[200197]: failed to encode map e311 with expected crc 2023-10-09T10:55:59.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:58 smithi125 ceph-mon[200197]: failed to encode map e311 with expected crc 2023-10-09T10:55:59.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:55:58 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 14683/148413 objects degraded (9.893%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:55:59.679 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:55:59.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:55:59.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:55:59.787 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:55:59.851 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:55:59.904 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:55:59.904 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:56:00.089 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:00.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:00.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:00.195 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:00.227 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:00.261 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:00.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:00.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:00.368 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:00.404 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:56:00.473 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:00.488 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:00.501 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:00.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:00.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:00.594 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:00 smithi155 ceph-mon[184854]: pgmap v755: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 14587/144864 objects degraded (10.069%); 11322/144864 objects misplaced (7.816%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T10:56:00.594 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:56:00.613 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:00.674 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:00 smithi125 ceph-mon[195507]: pgmap v755: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 14587/144864 objects degraded (10.069%); 11322/144864 objects misplaced (7.816%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T10:56:00.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:56:00.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:00 smithi125 ceph-mon[200197]: pgmap v755: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 14587/144864 objects degraded (10.069%); 11322/144864 objects misplaced (7.816%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T10:56:00.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:56:00.678 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:00.740 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:00.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:00.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:00.847 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:00.915 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:01.056 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:01.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:01.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:01.161 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:01.235 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:01.373 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:01.382 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:01.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:01.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:01.486 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:01.531 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:56:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:56:01] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2023-10-09T10:56:01.537 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:01.581 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:01.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:01.690 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:01.722 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:01.746 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:01.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:01.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:01.851 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:01.913 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:02.010 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:02.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:02.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:02.122 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:02.157 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:02.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:56:02.183 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:02.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:02.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:02.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:56:02.287 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:02.316 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:02.339 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:02.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:02.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:02.438 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:56:02.443 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:02.460 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:56:02.470 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:02.498 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:02.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:02.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:02.602 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:02.673 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:02 smithi155 ceph-mon[184854]: pgmap v756: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 14587/137673 objects degraded (10.595%); 10594/137673 objects misplaced (7.695%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T10:56:03.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:02 smithi125 ceph-mon[195507]: pgmap v756: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 14587/137673 objects degraded (10.595%); 10594/137673 objects misplaced (7.695%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T10:56:03.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:02 smithi125 ceph-mon[200197]: pgmap v756: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 14587/137673 objects degraded (10.595%); 10594/137673 objects misplaced (7.695%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T10:56:03.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:03 smithi155 ceph-mon[184854]: from='client.45527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:56:03.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:03 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 14587/137673 objects degraded (10.595%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:04.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:03 smithi125 ceph-mon[200197]: from='client.45527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:56:04.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:03 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 14587/137673 objects degraded (10.595%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:04.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:03 smithi125 ceph-mon[195507]: from='client.45527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:56:04.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:03 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 14587/137673 objects degraded (10.595%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:04.187 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:04.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:04.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:04.294 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:04.384 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:04.516 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:04.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:04.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:04.621 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:04.682 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:04.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:04 smithi125 ceph-mon[195507]: pgmap v757: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 14587/136992 objects degraded (10.648%); 10594/136992 objects misplaced (7.733%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T10:56:04.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:04 smithi125 ceph-mon[200197]: pgmap v757: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 14587/136992 objects degraded (10.648%); 10594/136992 objects misplaced (7.733%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T10:56:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:04 smithi155 ceph-mon[184854]: pgmap v757: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 14587/136992 objects degraded (10.648%); 10594/136992 objects misplaced (7.733%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T10:56:06.434 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:06.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:06.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:06.451 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":756,"stamp":"2023-10-09T10:56:01.455788+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3006608720,"num_objects":45891,"num_object_clones":0,"num_object_copies":137673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14587,"num_objects_misplaced":10594,"num_objects_unfound":0,"num_objects_dirty":45891,"num_whiteouts":0,"num_read":116,"num_read_kb":215,"num_write":147554,"num_write_kb":6186261,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8212,"num_bytes_recovered":536278802,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":177170,"ondisk_log_size":177170,"up":387,"acting":377,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":421,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13627336,"kb_used_data":9676232,"kb_used_omap":126,"kb_used_meta":3950721,"kb_avail":736333880,"statfs":{"total":767960285184,"available":754005893120,"internally_reserved":0,"allocated":9908461568,"data_stored":9897385097,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":4045538939},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-509673472,"num_objects":-7777,"num_object_clones":0,"num_object_copies":-23331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-220,"num_objects_misplaced":-1987,"num_objects_unfound":0,"num_objects_dirty":-7777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7777,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":692,"num_bytes_recovered":45350912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.017501"},"pg_stats":[{"pgid":"8.11","version":"311'7512","reported_seq":10710,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:55:58.423781+0000","last_change":"2023-10-09T10:54:49.220300+0000","last_active":"2023-10-09T10:55:58.423781+0000","last_peered":"2023-10-09T10:55:58.423781+0000","last_clean":"2023-10-09T10:54:02.426241+0000","last_became_active":"2023-10-09T10:54:49.215481+0000","last_became_peered":"2023-10-09T10:54:49.215481+0000","last_unstale":"2023-10-09T10:55:58.423781+0000","last_undegraded":"2023-10-09T10:54:49.026934+0000","last_fullsized":"2023-10-09T10:54:48.893501+0000","mapping_epoch":295,"log_start":"270'200","ondisk_log_start":"270'200","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7312,"log_dups_size":0,"ondisk_log_size":7312,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94175232,"num_objects":1437,"num_object_clones":0,"num_object_copies":4311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1734,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4551,"num_write_kb":191616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1558,"num_bytes_recovered":102105088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,1,5","objects":765},{"shards":"0,2","objects":-9},{"shards":"0,2,5","objects":393},{"shards":"0,3,5","objects":288}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.18","version":"76'161","reported_seq":625,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.051015+0000","last_change":"2023-10-09T10:55:26.628655+0000","last_active":"2023-10-09T10:55:45.051015+0000","last_peered":"2023-10-09T10:55:45.051015+0000","last_clean":"2023-10-09T10:55:45.051015+0000","last_became_active":"2023-10-09T10:55:26.628269+0000","last_became_peered":"2023-10-09T10:55:26.628269+0000","last_unstale":"2023-10-09T10:55:45.051015+0000","last_undegraded":"2023-10-09T10:55:45.051015+0000","last_fullsized":"2023-10-09T10:55:45.051015+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:40.174975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:40.174975+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:24:16.854746+0000","scrub_duration":0.010654205,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.057548+0000","last_change":"2023-10-09T10:54:47.925818+0000","last_active":"2023-10-09T10:55:44.057548+0000","last_peered":"2023-10-09T10:55:44.057548+0000","last_clean":"2023-10-09T10:55:44.057548+0000","last_became_active":"2023-10-09T10:54:47.925375+0000","last_became_peered":"2023-10-09T10:54:47.925375+0000","last_unstale":"2023-10-09T10:55:44.057548+0000","last_undegraded":"2023-10-09T10:55:44.057548+0000","last_fullsized":"2023-10-09T10:55:44.057548+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:27.635383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":318,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.071045+0000","last_change":"2023-10-09T10:54:47.945782+0000","last_active":"2023-10-09T10:55:45.071045+0000","last_peered":"2023-10-09T10:55:45.071045+0000","last_clean":"2023-10-09T10:55:45.071045+0000","last_became_active":"2023-10-09T10:54:47.945460+0000","last_became_peered":"2023-10-09T10:54:47.945460+0000","last_unstale":"2023-10-09T10:55:45.071045+0000","last_undegraded":"2023-10-09T10:55:45.071045+0000","last_fullsized":"2023-10-09T10:55:45.071045+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:50.537469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"311'7751","reported_seq":7820,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:56:01.256499+0000","last_change":"2023-10-09T10:54:40.626105+0000","last_active":"2023-10-09T10:56:01.256499+0000","last_peered":"2023-10-09T10:56:01.256499+0000","last_clean":"2023-10-09T10:56:01.256499+0000","last_became_active":"2023-10-09T10:54:40.609428+0000","last_became_peered":"2023-10-09T10:54:40.609428+0000","last_unstale":"2023-10-09T10:56:01.256499+0000","last_undegraded":"2023-10-09T10:56:01.256499+0000","last_fullsized":"2023-10-09T10:56:01.256499+0000","mapping_epoch":289,"log_start":"271'700","ondisk_log_start":"271'700","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7051,"log_dups_size":0,"ondisk_log_size":7051,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92667904,"num_objects":1414,"num_object_clones":0,"num_object_copies":4242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4716,"num_write_kb":196160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":691,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.050199+0000","last_change":"2023-10-09T10:53:34.537094+0000","last_active":"2023-10-09T10:55:45.050199+0000","last_peered":"2023-10-09T10:55:45.050199+0000","last_clean":"2023-10-09T10:55:45.050199+0000","last_became_active":"2023-10-09T10:52:55.414995+0000","last_became_peered":"2023-10-09T10:52:55.414995+0000","last_unstale":"2023-10-09T10:55:45.050199+0000","last_undegraded":"2023-10-09T10:55:45.050199+0000","last_fullsized":"2023-10-09T10:55:45.050199+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.536962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.536962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:30.197404+0000","scrub_duration":0.0053325960000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.058360+0000","last_change":"2023-10-09T10:54:47.944768+0000","last_active":"2023-10-09T10:55:44.058360+0000","last_peered":"2023-10-09T10:55:44.058360+0000","last_clean":"2023-10-09T10:55:44.058360+0000","last_became_active":"2023-10-09T10:54:47.943888+0000","last_became_peered":"2023-10-09T10:54:47.943888+0000","last_unstale":"2023-10-09T10:55:44.058360+0000","last_undegraded":"2023-10-09T10:55:44.058360+0000","last_fullsized":"2023-10-09T10:55:44.058360+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:11.046762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":219,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.052632+0000","last_change":"2023-10-09T10:53:14.823051+0000","last_active":"2023-10-09T10:55:45.052632+0000","last_peered":"2023-10-09T10:55:45.052632+0000","last_clean":"2023-10-09T10:55:45.052632+0000","last_became_active":"2023-10-09T10:53:14.822334+0000","last_became_peered":"2023-10-09T10:53:14.822334+0000","last_unstale":"2023-10-09T10:55:45.052632+0000","last_undegraded":"2023-10-09T10:55:45.052632+0000","last_fullsized":"2023-10-09T10:55:45.052632+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:46.573381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"310'7403","reported_seq":7512,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:56.596687+0000","last_change":"2023-10-09T10:54:50.916172+0000","last_active":"2023-10-09T10:55:56.596687+0000","last_peered":"2023-10-09T10:55:56.596687+0000","last_clean":"2023-10-09T10:55:56.596687+0000","last_became_active":"2023-10-09T10:54:50.913596+0000","last_became_peered":"2023-10-09T10:54:50.913596+0000","last_unstale":"2023-10-09T10:55:56.596687+0000","last_undegraded":"2023-10-09T10:55:56.596687+0000","last_fullsized":"2023-10-09T10:55:56.596687+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7403,"log_dups_size":0,"ondisk_log_size":7403,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100237312,"num_objects":1530,"num_object_clones":0,"num_object_copies":4590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4446,"num_write_kb":191200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1835008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":694,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.039775+0000","last_change":"2023-10-09T10:54:50.913546+0000","last_active":"2023-10-09T10:55:45.039775+0000","last_peered":"2023-10-09T10:55:45.039775+0000","last_clean":"2023-10-09T10:55:45.039775+0000","last_became_active":"2023-10-09T10:54:50.913190+0000","last_became_peered":"2023-10-09T10:54:50.913190+0000","last_unstale":"2023-10-09T10:55:45.039775+0000","last_undegraded":"2023-10-09T10:55:45.039775+0000","last_fullsized":"2023-10-09T10:55:45.039775+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:23.805498+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":364,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.071111+0000","last_change":"2023-10-09T10:54:48.038948+0000","last_active":"2023-10-09T10:55:45.071111+0000","last_peered":"2023-10-09T10:55:45.071111+0000","last_clean":"2023-10-09T10:55:45.071111+0000","last_became_active":"2023-10-09T10:54:48.001556+0000","last_became_peered":"2023-10-09T10:54:48.001556+0000","last_unstale":"2023-10-09T10:55:45.071111+0000","last_undegraded":"2023-10-09T10:55:45.071111+0000","last_fullsized":"2023-10-09T10:55:45.071111+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:23.415040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.062170+0000","last_change":"2023-10-09T10:54:47.914742+0000","last_active":"2023-10-09T10:55:44.062170+0000","last_peered":"2023-10-09T10:55:44.062170+0000","last_clean":"2023-10-09T10:55:44.062170+0000","last_became_active":"2023-10-09T10:54:47.914551+0000","last_became_peered":"2023-10-09T10:54:47.914551+0000","last_unstale":"2023-10-09T10:55:44.062170+0000","last_undegraded":"2023-10-09T10:55:44.062170+0000","last_fullsized":"2023-10-09T10:55:44.062170+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:16.067994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":745,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.060869+0000","last_change":"2023-10-09T10:54:50.915690+0000","last_active":"2023-10-09T10:55:45.060869+0000","last_peered":"2023-10-09T10:55:45.060869+0000","last_clean":"2023-10-09T10:55:45.060869+0000","last_became_active":"2023-10-09T10:54:50.915412+0000","last_became_peered":"2023-10-09T10:54:50.915412+0000","last_unstale":"2023-10-09T10:55:45.060869+0000","last_undegraded":"2023-10-09T10:55:45.060869+0000","last_fullsized":"2023-10-09T10:55:45.060869+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.787782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.787782+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:20:13.339672+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":183,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.051240+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:55:45.051240+0000","last_peered":"2023-10-09T10:55:45.051240+0000","last_clean":"2023-10-09T10:55:45.051240+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:55:45.051240+0000","last_undegraded":"2023-10-09T10:55:45.051240+0000","last_fullsized":"2023-10-09T10:55:45.051240+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":319,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.071187+0000","last_change":"2023-10-09T10:54:48.039275+0000","last_active":"2023-10-09T10:55:45.071187+0000","last_peered":"2023-10-09T10:55:45.071187+0000","last_clean":"2023-10-09T10:55:45.071187+0000","last_became_active":"2023-10-09T10:54:48.017645+0000","last_became_peered":"2023-10-09T10:54:48.017645+0000","last_unstale":"2023-10-09T10:55:45.071187+0000","last_undegraded":"2023-10-09T10:55:45.071187+0000","last_fullsized":"2023-10-09T10:55:45.071187+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:55.986791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":555,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.054158+0000","last_change":"2023-10-09T10:54:21.161539+0000","last_active":"2023-10-09T10:55:45.054158+0000","last_peered":"2023-10-09T10:55:45.054158+0000","last_clean":"2023-10-09T10:55:45.054158+0000","last_became_active":"2023-10-09T10:54:21.161106+0000","last_became_peered":"2023-10-09T10:54:21.161106+0000","last_unstale":"2023-10-09T10:55:45.054158+0000","last_undegraded":"2023-10-09T10:55:45.054158+0000","last_fullsized":"2023-10-09T10:55:45.054158+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:21.824982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:21.824982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":51,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.059499+0000","last_change":"2023-10-09T10:55:25.632545+0000","last_active":"2023-10-09T10:55:45.059499+0000","last_peered":"2023-10-09T10:55:45.059499+0000","last_clean":"2023-10-09T10:55:45.059499+0000","last_became_active":"2023-10-09T10:55:25.632235+0000","last_became_peered":"2023-10-09T10:55:25.632235+0000","last_unstale":"2023-10-09T10:55:45.059499+0000","last_undegraded":"2023-10-09T10:55:45.059499+0000","last_fullsized":"2023-10-09T10:55:45.059499+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:11.982194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":258,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.070548+0000","last_change":"2023-10-09T10:53:14.838859+0000","last_active":"2023-10-09T10:55:45.070548+0000","last_peered":"2023-10-09T10:55:45.070548+0000","last_clean":"2023-10-09T10:55:45.070548+0000","last_became_active":"2023-10-09T10:53:14.838536+0000","last_became_peered":"2023-10-09T10:53:14.838536+0000","last_unstale":"2023-10-09T10:55:45.070548+0000","last_undegraded":"2023-10-09T10:55:45.070548+0000","last_fullsized":"2023-10-09T10:55:45.070548+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:44:19.980568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":699,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.050892+0000","last_change":"2023-10-09T10:53:32.540768+0000","last_active":"2023-10-09T10:55:45.050892+0000","last_peered":"2023-10-09T10:55:45.050892+0000","last_clean":"2023-10-09T10:55:45.050892+0000","last_became_active":"2023-10-09T10:52:55.414818+0000","last_became_peered":"2023-10-09T10:52:55.414818+0000","last_unstale":"2023-10-09T10:55:45.050892+0000","last_undegraded":"2023-10-09T10:55:45.050892+0000","last_fullsized":"2023-10-09T10:55:45.050892+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:32.540610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:32.540610+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:47.374383+0000","scrub_duration":0.0064251300000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":223,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.060597+0000","last_change":"2023-10-09T10:55:25.637189+0000","last_active":"2023-10-09T10:55:45.060597+0000","last_peered":"2023-10-09T10:55:45.060597+0000","last_clean":"2023-10-09T10:55:45.060597+0000","last_became_active":"2023-10-09T10:55:25.636844+0000","last_became_peered":"2023-10-09T10:55:25.636844+0000","last_unstale":"2023-10-09T10:55:45.060597+0000","last_undegraded":"2023-10-09T10:55:45.060597+0000","last_fullsized":"2023-10-09T10:55:45.060597+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:18.943805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.057586+0000","last_change":"2023-10-09T10:54:47.925654+0000","last_active":"2023-10-09T10:55:44.057586+0000","last_peered":"2023-10-09T10:55:44.057586+0000","last_clean":"2023-10-09T10:55:44.057586+0000","last_became_active":"2023-10-09T10:54:47.924989+0000","last_became_peered":"2023-10-09T10:54:47.924989+0000","last_unstale":"2023-10-09T10:55:44.057586+0000","last_undegraded":"2023-10-09T10:55:44.057586+0000","last_fullsized":"2023-10-09T10:55:44.057586+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:02:49.755988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":683,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.059851+0000","last_change":"2023-10-09T10:54:50.913652+0000","last_active":"2023-10-09T10:55:44.059851+0000","last_peered":"2023-10-09T10:55:44.059851+0000","last_clean":"2023-10-09T10:55:44.059851+0000","last_became_active":"2023-10-09T10:54:50.913330+0000","last_became_peered":"2023-10-09T10:54:50.913330+0000","last_unstale":"2023-10-09T10:55:44.059851+0000","last_undegraded":"2023-10-09T10:55:44.059851+0000","last_fullsized":"2023-10-09T10:55:44.059851+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.200574+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.200574+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001591812,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":64,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.059618+0000","last_change":"2023-10-09T10:53:14.869000+0000","last_active":"2023-10-09T10:55:45.059618+0000","last_peered":"2023-10-09T10:55:45.059618+0000","last_clean":"2023-10-09T10:55:45.059618+0000","last_became_active":"2023-10-09T10:53:14.831904+0000","last_became_peered":"2023-10-09T10:53:14.831904+0000","last_unstale":"2023-10-09T10:55:45.059618+0000","last_undegraded":"2023-10-09T10:55:45.059618+0000","last_fullsized":"2023-10-09T10:55:45.059618+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:38.571980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":315,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.039734+0000","last_change":"2023-10-09T10:54:47.908531+0000","last_active":"2023-10-09T10:55:45.039734+0000","last_peered":"2023-10-09T10:55:45.039734+0000","last_clean":"2023-10-09T10:55:45.039734+0000","last_became_active":"2023-10-09T10:54:47.908232+0000","last_became_peered":"2023-10-09T10:54:47.908232+0000","last_unstale":"2023-10-09T10:55:45.039734+0000","last_undegraded":"2023-10-09T10:55:45.039734+0000","last_fullsized":"2023-10-09T10:55:45.039734+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:20.617543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"311'7574","reported_seq":7639,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:56:00.238417+0000","last_change":"2023-10-09T10:54:40.649728+0000","last_active":"2023-10-09T10:56:00.238417+0000","last_peered":"2023-10-09T10:56:00.238417+0000","last_clean":"2023-10-09T10:56:00.238417+0000","last_became_active":"2023-10-09T10:54:40.647392+0000","last_became_peered":"2023-10-09T10:54:40.647392+0000","last_unstale":"2023-10-09T10:56:00.238417+0000","last_undegraded":"2023-10-09T10:56:00.238417+0000","last_fullsized":"2023-10-09T10:56:00.238417+0000","mapping_epoch":289,"log_start":"271'1000","ondisk_log_start":"271'1000","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6574,"log_dups_size":0,"ondisk_log_size":6574,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95813632,"num_objects":1462,"num_object_clones":0,"num_object_copies":4386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4584,"num_write_kb":193472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":677,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.053183+0000","last_change":"2023-10-09T10:49:18.708370+0000","last_active":"2023-10-09T10:55:45.053183+0000","last_peered":"2023-10-09T10:55:45.053183+0000","last_clean":"2023-10-09T10:55:45.053183+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:55:45.053183+0000","last_undegraded":"2023-10-09T10:55:45.053183+0000","last_fullsized":"2023-10-09T10:55:45.053183+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:57.808410+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":76,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.059170+0000","last_change":"2023-10-09T10:54:18.080112+0000","last_active":"2023-10-09T10:55:45.059170+0000","last_peered":"2023-10-09T10:55:45.059170+0000","last_clean":"2023-10-09T10:55:45.059170+0000","last_became_active":"2023-10-09T10:54:17.878848+0000","last_became_peered":"2023-10-09T10:54:17.878848+0000","last_unstale":"2023-10-09T10:55:45.059170+0000","last_undegraded":"2023-10-09T10:55:45.059170+0000","last_fullsized":"2023-10-09T10:55:45.059170+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:12.314073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":369,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.076321+0000","last_change":"2023-10-09T10:54:47.941670+0000","last_active":"2023-10-09T10:55:45.076321+0000","last_peered":"2023-10-09T10:55:45.076321+0000","last_clean":"2023-10-09T10:55:45.076321+0000","last_became_active":"2023-10-09T10:54:47.941341+0000","last_became_peered":"2023-10-09T10:54:47.941341+0000","last_unstale":"2023-10-09T10:55:45.076321+0000","last_undegraded":"2023-10-09T10:55:45.076321+0000","last_fullsized":"2023-10-09T10:55:45.076321+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:31.570728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.19","version":"311'7545","reported_seq":7609,"reported_epoch":311,"state":"active+recovery_wait","last_fresh":"2023-10-09T10:55:58.379150+0000","last_change":"2023-10-09T10:54:40.731544+0000","last_active":"2023-10-09T10:55:58.379150+0000","last_peered":"2023-10-09T10:55:58.379150+0000","last_clean":"2023-10-09T10:54:39.518652+0000","last_became_active":"2023-10-09T10:54:40.598732+0000","last_became_peered":"2023-10-09T10:54:40.598732+0000","last_unstale":"2023-10-09T10:55:58.379150+0000","last_undegraded":"2023-10-09T10:55:58.379150+0000","last_fullsized":"2023-10-09T10:55:58.379150+0000","mapping_epoch":289,"log_start":"270'300","ondisk_log_start":"270'300","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7245,"log_dups_size":0,"ondisk_log_size":7245,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97779712,"num_objects":1492,"num_object_clones":0,"num_object_copies":4476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4560,"num_write_kb":193664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":1492}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":658,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.059141+0000","last_change":"2023-10-09T10:53:17.846464+0000","last_active":"2023-10-09T10:55:45.059141+0000","last_peered":"2023-10-09T10:55:45.059141+0000","last_clean":"2023-10-09T10:55:45.059141+0000","last_became_active":"2023-10-09T10:53:17.845979+0000","last_became_peered":"2023-10-09T10:53:17.845979+0000","last_unstale":"2023-10-09T10:55:45.059141+0000","last_undegraded":"2023-10-09T10:55:45.059141+0000","last_fullsized":"2023-10-09T10:55:45.059141+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:56.295299+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":219,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.050252+0000","last_change":"2023-10-09T10:53:14.821961+0000","last_active":"2023-10-09T10:55:45.050252+0000","last_peered":"2023-10-09T10:55:45.050252+0000","last_clean":"2023-10-09T10:55:45.050252+0000","last_became_active":"2023-10-09T10:53:14.821625+0000","last_became_peered":"2023-10-09T10:53:14.821625+0000","last_unstale":"2023-10-09T10:55:45.050252+0000","last_undegraded":"2023-10-09T10:55:45.050252+0000","last_fullsized":"2023-10-09T10:55:45.050252+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:29.305952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":225,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.059369+0000","last_change":"2023-10-09T10:55:25.629418+0000","last_active":"2023-10-09T10:55:45.059369+0000","last_peered":"2023-10-09T10:55:45.059369+0000","last_clean":"2023-10-09T10:55:45.059369+0000","last_became_active":"2023-10-09T10:55:25.629130+0000","last_became_peered":"2023-10-09T10:55:25.629130+0000","last_unstale":"2023-10-09T10:55:45.059369+0000","last_undegraded":"2023-10-09T10:55:45.059369+0000","last_fullsized":"2023-10-09T10:55:45.059369+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:51.186552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":614,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.060735+0000","last_change":"2023-10-09T10:54:50.916062+0000","last_active":"2023-10-09T10:55:44.060735+0000","last_peered":"2023-10-09T10:55:44.060735+0000","last_clean":"2023-10-09T10:55:44.060735+0000","last_became_active":"2023-10-09T10:54:50.915189+0000","last_became_peered":"2023-10-09T10:54:50.915189+0000","last_unstale":"2023-10-09T10:55:44.060735+0000","last_undegraded":"2023-10-09T10:55:44.060735+0000","last_fullsized":"2023-10-09T10:55:44.060735+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.077788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.077788+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":206,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.058689+0000","last_change":"2023-10-09T10:55:25.633116+0000","last_active":"2023-10-09T10:55:45.058689+0000","last_peered":"2023-10-09T10:55:45.058689+0000","last_clean":"2023-10-09T10:55:45.058689+0000","last_became_active":"2023-10-09T10:55:25.632837+0000","last_became_peered":"2023-10-09T10:55:25.632837+0000","last_unstale":"2023-10-09T10:55:45.058689+0000","last_undegraded":"2023-10-09T10:55:45.058689+0000","last_fullsized":"2023-10-09T10:55:45.058689+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:46.192186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":201,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.052416+0000","last_change":"2023-10-09T10:53:14.823485+0000","last_active":"2023-10-09T10:55:45.052416+0000","last_peered":"2023-10-09T10:55:45.052416+0000","last_clean":"2023-10-09T10:55:45.052416+0000","last_became_active":"2023-10-09T10:53:14.822756+0000","last_became_peered":"2023-10-09T10:53:14.822756+0000","last_unstale":"2023-10-09T10:55:45.052416+0000","last_undegraded":"2023-10-09T10:55:45.052416+0000","last_fullsized":"2023-10-09T10:55:45.052416+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:05.359313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":641,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.057741+0000","last_change":"2023-10-09T10:54:50.910027+0000","last_active":"2023-10-09T10:55:44.057741+0000","last_peered":"2023-10-09T10:55:44.057741+0000","last_clean":"2023-10-09T10:55:44.057741+0000","last_became_active":"2023-10-09T10:54:50.909698+0000","last_became_peered":"2023-10-09T10:54:50.909698+0000","last_unstale":"2023-10-09T10:55:44.057741+0000","last_undegraded":"2023-10-09T10:55:44.057741+0000","last_fullsized":"2023-10-09T10:55:44.057741+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:01.637196+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":186,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:53.419600+0000","last_change":"2023-10-09T10:51:12.366910+0000","last_active":"2023-10-09T10:55:53.419600+0000","last_peered":"2023-10-09T10:55:53.419600+0000","last_clean":"2023-10-09T10:55:53.419600+0000","last_became_active":"2023-10-09T10:51:12.366565+0000","last_became_peered":"2023-10-09T10:51:12.366565+0000","last_unstale":"2023-10-09T10:55:53.419600+0000","last_undegraded":"2023-10-09T10:55:53.419600+0000","last_fullsized":"2023-10-09T10:55:53.419600+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:53.464251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":346,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.073880+0000","last_change":"2023-10-09T10:54:48.017646+0000","last_active":"2023-10-09T10:55:45.073880+0000","last_peered":"2023-10-09T10:55:45.073880+0000","last_clean":"2023-10-09T10:55:45.073880+0000","last_became_active":"2023-10-09T10:54:47.994162+0000","last_became_peered":"2023-10-09T10:54:47.994162+0000","last_unstale":"2023-10-09T10:55:45.073880+0000","last_undegraded":"2023-10-09T10:55:45.073880+0000","last_fullsized":"2023-10-09T10:55:45.073880+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:11:59.205546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"311'7650","reported_seq":7727,"reported_epoch":311,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:56:01.018635+0000","last_change":"2023-10-09T10:54:40.551123+0000","last_active":"2023-10-09T10:56:01.018635+0000","last_peered":"2023-10-09T10:56:01.018635+0000","last_clean":"2023-10-09T10:54:17.036595+0000","last_became_active":"2023-10-09T10:54:40.548039+0000","last_became_peered":"2023-10-09T10:54:40.548039+0000","last_unstale":"2023-10-09T10:56:01.018635+0000","last_undegraded":"2023-10-09T10:56:01.018635+0000","last_fullsized":"2023-10-09T10:56:01.018635+0000","mapping_epoch":282,"log_start":"271'600","ondisk_log_start":"271'600","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7050,"log_dups_size":0,"ondisk_log_size":7050,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89948160,"num_objects":1372,"num_object_clones":0,"num_object_copies":4116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1372,"num_objects_unfound":0,"num_objects_dirty":1372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4664,"num_write_kb":193184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1372}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":662,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.060776+0000","last_change":"2023-10-09T10:54:50.909873+0000","last_active":"2023-10-09T10:55:45.060776+0000","last_peered":"2023-10-09T10:55:45.060776+0000","last_clean":"2023-10-09T10:55:45.060776+0000","last_became_active":"2023-10-09T10:54:50.909533+0000","last_became_peered":"2023-10-09T10:54:50.909533+0000","last_unstale":"2023-10-09T10:55:45.060776+0000","last_undegraded":"2023-10-09T10:55:45.060776+0000","last_fullsized":"2023-10-09T10:55:45.060776+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:45.045013+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":58,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.060012+0000","last_change":"2023-10-09T10:53:14.854904+0000","last_active":"2023-10-09T10:55:45.060012+0000","last_peered":"2023-10-09T10:55:45.060012+0000","last_clean":"2023-10-09T10:55:45.060012+0000","last_became_active":"2023-10-09T10:53:14.829611+0000","last_became_peered":"2023-10-09T10:53:14.829611+0000","last_unstale":"2023-10-09T10:55:45.060012+0000","last_undegraded":"2023-10-09T10:55:45.060012+0000","last_fullsized":"2023-10-09T10:55:45.060012+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:26.064630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":243,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.058782+0000","last_change":"2023-10-09T10:55:25.633485+0000","last_active":"2023-10-09T10:55:45.058782+0000","last_peered":"2023-10-09T10:55:45.058782+0000","last_clean":"2023-10-09T10:55:45.058782+0000","last_became_active":"2023-10-09T10:55:25.633028+0000","last_became_peered":"2023-10-09T10:55:25.633028+0000","last_unstale":"2023-10-09T10:55:45.058782+0000","last_undegraded":"2023-10-09T10:55:45.058782+0000","last_fullsized":"2023-10-09T10:55:45.058782+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:57.601747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"311'7624","reported_seq":7689,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:56:01.150622+0000","last_change":"2023-10-09T10:55:26.330497+0000","last_active":"2023-10-09T10:56:01.150622+0000","last_peered":"2023-10-09T10:56:01.150622+0000","last_clean":"2023-10-09T10:56:01.150622+0000","last_became_active":"2023-10-09T10:54:40.554360+0000","last_became_peered":"2023-10-09T10:54:40.554360+0000","last_unstale":"2023-10-09T10:56:01.150622+0000","last_undegraded":"2023-10-09T10:56:01.150622+0000","last_fullsized":"2023-10-09T10:56:01.150622+0000","mapping_epoch":266,"log_start":"271'400","ondisk_log_start":"271'400","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7224,"log_dups_size":0,"ondisk_log_size":7224,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86704128,"num_objects":1323,"num_object_clones":0,"num_object_copies":3969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4663,"num_write_kb":191552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":671,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:55:57.587412+0000","last_change":"2023-10-09T10:53:25.540494+0000","last_active":"2023-10-09T10:55:57.587412+0000","last_peered":"2023-10-09T10:55:57.587412+0000","last_clean":"2023-10-09T10:55:57.587412+0000","last_became_active":"2023-10-09T10:51:01.295474+0000","last_became_peered":"2023-10-09T10:51:01.295474+0000","last_unstale":"2023-10-09T10:55:57.587412+0000","last_undegraded":"2023-10-09T10:55:57.587412+0000","last_fullsized":"2023-10-09T10:55:57.587412+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:39.069143+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":86,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.059690+0000","last_change":"2023-10-09T10:54:48.000482+0000","last_active":"2023-10-09T10:55:45.059690+0000","last_peered":"2023-10-09T10:55:45.059690+0000","last_clean":"2023-10-09T10:55:45.059690+0000","last_became_active":"2023-10-09T10:54:47.905689+0000","last_became_peered":"2023-10-09T10:54:47.905689+0000","last_unstale":"2023-10-09T10:55:45.059690+0000","last_undegraded":"2023-10-09T10:55:45.059690+0000","last_fullsized":"2023-10-09T10:55:45.059690+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.666594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.058222+0000","last_change":"2023-10-09T10:54:47.944930+0000","last_active":"2023-10-09T10:55:44.058222+0000","last_peered":"2023-10-09T10:55:44.058222+0000","last_clean":"2023-10-09T10:55:44.058222+0000","last_became_active":"2023-10-09T10:54:47.944485+0000","last_became_peered":"2023-10-09T10:54:47.944485+0000","last_unstale":"2023-10-09T10:55:44.058222+0000","last_undegraded":"2023-10-09T10:55:44.058222+0000","last_fullsized":"2023-10-09T10:55:44.058222+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:11.213129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"311'7565","reported_seq":7631,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:55:58.382183+0000","last_change":"2023-10-09T10:54:40.586495+0000","last_active":"2023-10-09T10:55:58.382183+0000","last_peered":"2023-10-09T10:55:58.382183+0000","last_clean":"2023-10-09T10:55:58.382183+0000","last_became_active":"2023-10-09T10:54:40.577656+0000","last_became_peered":"2023-10-09T10:54:40.577656+0000","last_unstale":"2023-10-09T10:55:58.382183+0000","last_undegraded":"2023-10-09T10:55:58.382183+0000","last_fullsized":"2023-10-09T10:55:58.382183+0000","mapping_epoch":266,"log_start":"270'300","ondisk_log_start":"270'300","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7265,"log_dups_size":0,"ondisk_log_size":7265,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97812480,"num_objects":1492,"num_object_clones":0,"num_object_copies":4476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4570,"num_write_kb":194016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":623,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.041021+0000","last_change":"2023-10-09T10:53:36.680150+0000","last_active":"2023-10-09T10:55:45.041021+0000","last_peered":"2023-10-09T10:55:45.041021+0000","last_clean":"2023-10-09T10:55:45.041021+0000","last_became_active":"2023-10-09T10:51:50.371032+0000","last_became_peered":"2023-10-09T10:51:50.371032+0000","last_unstale":"2023-10-09T10:55:45.041021+0000","last_undegraded":"2023-10-09T10:55:45.041021+0000","last_fullsized":"2023-10-09T10:55:45.041021+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.680090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.680090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:08:28.952827+0000","scrub_duration":0.005577172,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":58,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.060824+0000","last_change":"2023-10-09T10:53:14.841343+0000","last_active":"2023-10-09T10:55:45.060824+0000","last_peered":"2023-10-09T10:55:45.060824+0000","last_clean":"2023-10-09T10:55:45.060824+0000","last_became_active":"2023-10-09T10:53:14.828213+0000","last_became_peered":"2023-10-09T10:53:14.828213+0000","last_unstale":"2023-10-09T10:55:45.060824+0000","last_undegraded":"2023-10-09T10:55:45.060824+0000","last_fullsized":"2023-10-09T10:55:45.060824+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:28.137803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.057708+0000","last_change":"2023-10-09T10:54:47.942716+0000","last_active":"2023-10-09T10:55:44.057708+0000","last_peered":"2023-10-09T10:55:44.057708+0000","last_clean":"2023-10-09T10:55:44.057708+0000","last_became_active":"2023-10-09T10:54:47.942459+0000","last_became_peered":"2023-10-09T10:54:47.942459+0000","last_unstale":"2023-10-09T10:55:44.057708+0000","last_undegraded":"2023-10-09T10:55:44.057708+0000","last_fullsized":"2023-10-09T10:55:44.057708+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:07.790165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"311'7603","reported_seq":7694,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:00.032897+0000","last_change":"2023-10-09T10:54:49.183581+0000","last_active":"2023-10-09T10:56:00.032897+0000","last_peered":"2023-10-09T10:56:00.032897+0000","last_clean":"2023-10-09T10:54:02.391425+0000","last_became_active":"2023-10-09T10:54:49.111868+0000","last_became_peered":"2023-10-09T10:54:49.111868+0000","last_unstale":"2023-10-09T10:56:00.032897+0000","last_undegraded":"2023-10-09T10:54:48.912262+0000","last_fullsized":"2023-10-09T10:54:48.907341+0000","mapping_epoch":295,"log_start":"271'1000","ondisk_log_start":"271'1000","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6603,"log_dups_size":0,"ondisk_log_size":6603,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91914240,"num_objects":1402,"num_object_clones":0,"num_object_copies":4206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":697,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4626,"num_write_kb":192928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"3,4,6","objects":697},{"shards":"3,6","objects":705}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":756,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.040139+0000","last_change":"2023-10-09T10:53:38.681891+0000","last_active":"2023-10-09T10:55:45.040139+0000","last_peered":"2023-10-09T10:55:45.040139+0000","last_clean":"2023-10-09T10:55:45.040139+0000","last_became_active":"2023-10-09T10:53:17.840478+0000","last_became_peered":"2023-10-09T10:53:17.840478+0000","last_unstale":"2023-10-09T10:55:45.040139+0000","last_undegraded":"2023-10-09T10:55:45.040139+0000","last_fullsized":"2023-10-09T10:55:45.040139+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:31:14.758036+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":136,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.076520+0000","last_change":"2023-10-09T10:51:50.374527+0000","last_active":"2023-10-09T10:55:45.076520+0000","last_peered":"2023-10-09T10:55:45.076520+0000","last_clean":"2023-10-09T10:55:45.076520+0000","last_became_active":"2023-10-09T10:51:50.374297+0000","last_became_peered":"2023-10-09T10:51:50.374297+0000","last_unstale":"2023-10-09T10:55:45.076520+0000","last_undegraded":"2023-10-09T10:55:45.076520+0000","last_fullsized":"2023-10-09T10:55:45.076520+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:31:49.358307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":219,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.050655+0000","last_change":"2023-10-09T10:54:47.939771+0000","last_active":"2023-10-09T10:55:45.050655+0000","last_peered":"2023-10-09T10:55:45.050655+0000","last_clean":"2023-10-09T10:55:45.050655+0000","last_became_active":"2023-10-09T10:54:47.939440+0000","last_became_peered":"2023-10-09T10:54:47.939440+0000","last_unstale":"2023-10-09T10:55:45.050655+0000","last_undegraded":"2023-10-09T10:55:45.050655+0000","last_fullsized":"2023-10-09T10:55:45.050655+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:58.335536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"311'7691","reported_seq":9636,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:00.237425+0000","last_change":"2023-10-09T10:54:49.158909+0000","last_active":"2023-10-09T10:56:00.237425+0000","last_peered":"2023-10-09T10:56:00.237425+0000","last_clean":"2023-10-09T10:54:02.435878+0000","last_became_active":"2023-10-09T10:54:48.018286+0000","last_became_peered":"2023-10-09T10:54:48.018286+0000","last_unstale":"2023-10-09T10:56:00.237425+0000","last_undegraded":"2023-10-09T10:54:49.152454+0000","last_fullsized":"2023-10-09T10:54:47.891421+0000","mapping_epoch":294,"log_start":"271'1100","ondisk_log_start":"271'1100","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6591,"log_dups_size":0,"ondisk_log_size":6591,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93290496,"num_objects":1423,"num_object_clones":0,"num_object_copies":4269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":407,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4673,"num_write_kb":195104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":592,"num_bytes_recovered":38830080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":407},{"shards":"3,5","objects":1016}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":648,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.058873+0000","last_change":"2023-10-09T10:55:25.628214+0000","last_active":"2023-10-09T10:55:45.058873+0000","last_peered":"2023-10-09T10:55:45.058873+0000","last_clean":"2023-10-09T10:55:45.058873+0000","last_became_active":"2023-10-09T10:55:25.627822+0000","last_became_peered":"2023-10-09T10:55:25.627822+0000","last_unstale":"2023-10-09T10:55:45.058873+0000","last_undegraded":"2023-10-09T10:55:45.058873+0000","last_fullsized":"2023-10-09T10:55:45.058873+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":307,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.076786+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:55:45.076786+0000","last_peered":"2023-10-09T10:55:45.076786+0000","last_clean":"2023-10-09T10:55:45.076786+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:55:45.076786+0000","last_undegraded":"2023-10-09T10:55:45.076786+0000","last_fullsized":"2023-10-09T10:55:45.076786+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":152,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.076441+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T10:55:45.076441+0000","last_peered":"2023-10-09T10:55:45.076441+0000","last_clean":"2023-10-09T10:55:45.076441+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T10:55:45.076441+0000","last_undegraded":"2023-10-09T10:55:45.076441+0000","last_fullsized":"2023-10-09T10:55:45.076441+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"311'7502","reported_seq":10678,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:55:58.421971+0000","last_change":"2023-10-09T10:54:49.220251+0000","last_active":"2023-10-09T10:55:58.421971+0000","last_peered":"2023-10-09T10:55:58.421971+0000","last_clean":"2023-10-09T10:54:02.426241+0000","last_became_active":"2023-10-09T10:54:49.172055+0000","last_became_peered":"2023-10-09T10:54:49.172055+0000","last_unstale":"2023-10-09T10:55:58.421971+0000","last_undegraded":"2023-10-09T10:54:49.015836+0000","last_fullsized":"2023-10-09T10:54:48.896714+0000","mapping_epoch":295,"log_start":"270'200","ondisk_log_start":"270'200","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7302,"log_dups_size":0,"ondisk_log_size":7302,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95092736,"num_objects":1451,"num_object_clones":0,"num_object_copies":4353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1725,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4541,"num_write_kb":191744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1546,"num_bytes_recovered":101318656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,1,5","objects":762},{"shards":"0,2","objects":11},{"shards":"0,2,5","objects":393},{"shards":"0,3,5","objects":285}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":614,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.050935+0000","last_change":"2023-10-09T10:54:21.173078+0000","last_active":"2023-10-09T10:55:45.050935+0000","last_peered":"2023-10-09T10:55:45.050935+0000","last_clean":"2023-10-09T10:55:45.050935+0000","last_became_active":"2023-10-09T10:54:21.172767+0000","last_became_peered":"2023-10-09T10:54:21.172767+0000","last_unstale":"2023-10-09T10:55:45.050935+0000","last_undegraded":"2023-10-09T10:55:45.050935+0000","last_fullsized":"2023-10-09T10:55:45.050935+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.532890+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.532890+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:32.335841+0000","scrub_duration":0.0045290130000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":290,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.040010+0000","last_change":"2023-10-09T10:54:47.908506+0000","last_active":"2023-10-09T10:55:45.040010+0000","last_peered":"2023-10-09T10:55:45.040010+0000","last_clean":"2023-10-09T10:55:45.040010+0000","last_became_active":"2023-10-09T10:54:47.908110+0000","last_became_peered":"2023-10-09T10:54:47.908110+0000","last_unstale":"2023-10-09T10:55:45.040010+0000","last_undegraded":"2023-10-09T10:55:45.040010+0000","last_fullsized":"2023-10-09T10:55:45.040010+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:38.808943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":49,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.055457+0000","last_change":"2023-10-09T10:54:47.948732+0000","last_active":"2023-10-09T10:55:45.055457+0000","last_peered":"2023-10-09T10:55:45.055457+0000","last_clean":"2023-10-09T10:55:45.055457+0000","last_became_active":"2023-10-09T10:54:47.948414+0000","last_became_peered":"2023-10-09T10:54:47.948414+0000","last_unstale":"2023-10-09T10:55:45.055457+0000","last_undegraded":"2023-10-09T10:55:45.055457+0000","last_fullsized":"2023-10-09T10:55:45.055457+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:38.020807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.2","version":"311'7735","reported_seq":7801,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:56:01.200476+0000","last_change":"2023-10-09T10:55:24.369167+0000","last_active":"2023-10-09T10:56:01.200476+0000","last_peered":"2023-10-09T10:56:01.200476+0000","last_clean":"2023-10-09T10:56:01.200476+0000","last_became_active":"2023-10-09T10:54:40.567891+0000","last_became_peered":"2023-10-09T10:54:40.567891+0000","last_unstale":"2023-10-09T10:56:01.200476+0000","last_undegraded":"2023-10-09T10:56:01.200476+0000","last_fullsized":"2023-10-09T10:56:01.200476+0000","mapping_epoch":265,"log_start":"271'700","ondisk_log_start":"271'700","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7035,"log_dups_size":0,"ondisk_log_size":7035,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92536832,"num_objects":1412,"num_object_clones":0,"num_object_copies":4236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4700,"num_write_kb":195584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":684,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.061031+0000","last_change":"2023-10-09T10:54:50.914888+0000","last_active":"2023-10-09T10:55:44.061031+0000","last_peered":"2023-10-09T10:55:44.061031+0000","last_clean":"2023-10-09T10:55:44.061031+0000","last_became_active":"2023-10-09T10:54:50.909639+0000","last_became_peered":"2023-10-09T10:54:50.909639+0000","last_unstale":"2023-10-09T10:55:44.061031+0000","last_undegraded":"2023-10-09T10:55:44.061031+0000","last_fullsized":"2023-10-09T10:55:44.061031+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.103750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.103750+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":346,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.039434+0000","last_change":"2023-10-09T10:54:17.876795+0000","last_active":"2023-10-09T10:55:45.039434+0000","last_peered":"2023-10-09T10:55:45.039434+0000","last_clean":"2023-10-09T10:55:45.039434+0000","last_became_active":"2023-10-09T10:54:17.876504+0000","last_became_peered":"2023-10-09T10:54:17.876504+0000","last_unstale":"2023-10-09T10:55:45.039434+0000","last_undegraded":"2023-10-09T10:55:45.039434+0000","last_fullsized":"2023-10-09T10:55:45.039434+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:04:48.657295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.059916+0000","last_change":"2023-10-09T10:54:47.913826+0000","last_active":"2023-10-09T10:55:44.059916+0000","last_peered":"2023-10-09T10:55:44.059916+0000","last_clean":"2023-10-09T10:55:44.059916+0000","last_became_active":"2023-10-09T10:54:47.913524+0000","last_became_peered":"2023-10-09T10:54:47.913524+0000","last_unstale":"2023-10-09T10:55:44.059916+0000","last_undegraded":"2023-10-09T10:55:44.059916+0000","last_fullsized":"2023-10-09T10:55:44.059916+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:56:23.699552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"310'7481","reported_seq":7599,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:56.596758+0000","last_change":"2023-10-09T10:54:50.915546+0000","last_active":"2023-10-09T10:55:56.596758+0000","last_peered":"2023-10-09T10:55:56.596758+0000","last_clean":"2023-10-09T10:55:56.596758+0000","last_became_active":"2023-10-09T10:54:50.910131+0000","last_became_peered":"2023-10-09T10:54:50.910131+0000","last_unstale":"2023-10-09T10:55:56.596758+0000","last_undegraded":"2023-10-09T10:55:56.596758+0000","last_fullsized":"2023-10-09T10:55:56.596758+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7481,"log_dups_size":0,"ondisk_log_size":7481,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95518720,"num_objects":1457,"num_object_clones":0,"num_object_copies":4371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4523,"num_write_kb":191392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":2031616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":625,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.054076+0000","last_change":"2023-10-09T10:54:21.152148+0000","last_active":"2023-10-09T10:55:45.054076+0000","last_peered":"2023-10-09T10:55:45.054076+0000","last_clean":"2023-10-09T10:55:45.054076+0000","last_became_active":"2023-10-09T10:54:21.151809+0000","last_became_peered":"2023-10-09T10:54:21.151809+0000","last_unstale":"2023-10-09T10:55:45.054076+0000","last_undegraded":"2023-10-09T10:55:45.054076+0000","last_fullsized":"2023-10-09T10:55:45.054076+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.701128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.701128+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:27.021145+0000","scrub_duration":0.0025071830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":338,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.070951+0000","last_change":"2023-10-09T10:54:48.038946+0000","last_active":"2023-10-09T10:55:45.070951+0000","last_peered":"2023-10-09T10:55:45.070951+0000","last_clean":"2023-10-09T10:55:45.070951+0000","last_became_active":"2023-10-09T10:54:48.001493+0000","last_became_peered":"2023-10-09T10:54:48.001493+0000","last_unstale":"2023-10-09T10:55:45.070951+0000","last_undegraded":"2023-10-09T10:55:45.070951+0000","last_fullsized":"2023-10-09T10:55:45.070951+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:25.197466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.061935+0000","last_change":"2023-10-09T10:54:48.012411+0000","last_active":"2023-10-09T10:55:44.061935+0000","last_peered":"2023-10-09T10:55:44.061935+0000","last_clean":"2023-10-09T10:55:44.061935+0000","last_became_active":"2023-10-09T10:54:48.011894+0000","last_became_peered":"2023-10-09T10:54:48.011894+0000","last_unstale":"2023-10-09T10:55:44.061935+0000","last_undegraded":"2023-10-09T10:55:44.061935+0000","last_fullsized":"2023-10-09T10:55:44.061935+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:28.067337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"311'9160","reported_seq":11063,"reported_epoch":311,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:55:58.430646+0000","last_change":"2023-10-09T10:55:24.700754+0000","last_active":"2023-10-09T10:55:58.430646+0000","last_peered":"2023-10-09T10:55:58.430646+0000","last_clean":"2023-10-09T10:54:17.076172+0000","last_became_active":"2023-10-09T10:54:19.327879+0000","last_became_peered":"2023-10-09T10:54:19.327879+0000","last_unstale":"2023-10-09T10:55:58.430646+0000","last_undegraded":"2023-10-09T10:54:18.890724+0000","last_fullsized":"2023-10-09T10:54:18.868702+0000","mapping_epoch":282,"log_start":"275'3852","ondisk_log_start":"275'3852","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5308,"log_dups_size":0,"ondisk_log_size":5308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:49.273305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197394432,"num_objects":3012,"num_object_clones":0,"num_object_copies":9036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2158,"num_objects_misplaced":2215,"num_objects_unfound":0,"num_objects_dirty":3012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9160,"num_write_kb":389504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":919,"num_bytes_recovered":60227584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-1361},{"shards":"0,4,7","objects":2215}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":708,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.302477+0000","last_change":"2023-10-09T10:53:34.602531+0000","last_active":"2023-10-09T10:55:45.302477+0000","last_peered":"2023-10-09T10:55:45.302477+0000","last_clean":"2023-10-09T10:55:45.302477+0000","last_became_active":"2023-10-09T10:53:17.841970+0000","last_became_peered":"2023-10-09T10:53:17.841970+0000","last_unstale":"2023-10-09T10:55:45.302477+0000","last_undegraded":"2023-10-09T10:55:45.302477+0000","last_fullsized":"2023-10-09T10:55:45.302477+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:04.601861+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.059124+0000","last_change":"2023-10-09T10:54:47.952216+0000","last_active":"2023-10-09T10:55:44.059124+0000","last_peered":"2023-10-09T10:55:44.059124+0000","last_clean":"2023-10-09T10:55:44.059124+0000","last_became_active":"2023-10-09T10:54:47.941070+0000","last_became_peered":"2023-10-09T10:54:47.941070+0000","last_unstale":"2023-10-09T10:55:44.059124+0000","last_undegraded":"2023-10-09T10:55:44.059124+0000","last_fullsized":"2023-10-09T10:55:44.059124+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:09.075266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":231,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:55:57.587856+0000","last_change":"2023-10-09T10:54:17.886921+0000","last_active":"2023-10-09T10:55:57.587856+0000","last_peered":"2023-10-09T10:55:57.587856+0000","last_clean":"2023-10-09T10:55:57.587856+0000","last_became_active":"2023-10-09T10:54:17.886584+0000","last_became_peered":"2023-10-09T10:54:17.886584+0000","last_unstale":"2023-10-09T10:55:57.587856+0000","last_undegraded":"2023-10-09T10:55:57.587856+0000","last_fullsized":"2023-10-09T10:55:57.587856+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:15.848951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.057629+0000","last_change":"2023-10-09T10:54:47.908294+0000","last_active":"2023-10-09T10:55:44.057629+0000","last_peered":"2023-10-09T10:55:44.057629+0000","last_clean":"2023-10-09T10:55:44.057629+0000","last_became_active":"2023-10-09T10:54:47.908019+0000","last_became_peered":"2023-10-09T10:54:47.908019+0000","last_unstale":"2023-10-09T10:55:44.057629+0000","last_undegraded":"2023-10-09T10:55:44.057629+0000","last_fullsized":"2023-10-09T10:55:44.057629+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:48.662764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"311'7598","reported_seq":7664,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:56:00.221707+0000","last_change":"2023-10-09T10:54:40.642047+0000","last_active":"2023-10-09T10:56:00.221707+0000","last_peered":"2023-10-09T10:56:00.221707+0000","last_clean":"2023-10-09T10:56:00.221707+0000","last_became_active":"2023-10-09T10:54:40.639305+0000","last_became_peered":"2023-10-09T10:54:40.639305+0000","last_unstale":"2023-10-09T10:56:00.221707+0000","last_undegraded":"2023-10-09T10:56:00.221707+0000","last_fullsized":"2023-10-09T10:56:00.221707+0000","mapping_epoch":266,"log_start":"271'1000","ondisk_log_start":"271'1000","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6598,"log_dups_size":0,"ondisk_log_size":6598,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94502912,"num_objects":1442,"num_object_clones":0,"num_object_copies":4326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4608,"num_write_kb":193600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":647,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.053109+0000","last_change":"2023-10-09T10:54:21.163072+0000","last_active":"2023-10-09T10:55:45.053109+0000","last_peered":"2023-10-09T10:55:45.053109+0000","last_clean":"2023-10-09T10:55:45.053109+0000","last_became_active":"2023-10-09T10:54:21.162710+0000","last_became_peered":"2023-10-09T10:54:21.162710+0000","last_unstale":"2023-10-09T10:55:45.053109+0000","last_undegraded":"2023-10-09T10:55:45.053109+0000","last_fullsized":"2023-10-09T10:55:45.053109+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:15.932806+0000","scrub_duration":0.0019146790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.057661+0000","last_change":"2023-10-09T10:54:47.899967+0000","last_active":"2023-10-09T10:55:44.057661+0000","last_peered":"2023-10-09T10:55:44.057661+0000","last_clean":"2023-10-09T10:55:44.057661+0000","last_became_active":"2023-10-09T10:54:47.899584+0000","last_became_peered":"2023-10-09T10:54:47.899584+0000","last_unstale":"2023-10-09T10:55:44.057661+0000","last_undegraded":"2023-10-09T10:55:44.057661+0000","last_fullsized":"2023-10-09T10:55:44.057661+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:43.973868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":220,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.050358+0000","last_change":"2023-10-09T10:50:43.388615+0000","last_active":"2023-10-09T10:55:45.050358+0000","last_peered":"2023-10-09T10:55:45.050358+0000","last_clean":"2023-10-09T10:55:45.050358+0000","last_became_active":"2023-10-09T10:50:43.388176+0000","last_became_peered":"2023-10-09T10:50:43.388176+0000","last_unstale":"2023-10-09T10:55:45.050358+0000","last_undegraded":"2023-10-09T10:55:45.050358+0000","last_fullsized":"2023-10-09T10:55:45.050358+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:41:17.550111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.059062+0000","last_change":"2023-10-09T10:54:47.950462+0000","last_active":"2023-10-09T10:55:44.059062+0000","last_peered":"2023-10-09T10:55:44.059062+0000","last_clean":"2023-10-09T10:55:44.059062+0000","last_became_active":"2023-10-09T10:54:47.940606+0000","last_became_peered":"2023-10-09T10:54:47.940606+0000","last_unstale":"2023-10-09T10:55:44.059062+0000","last_undegraded":"2023-10-09T10:55:44.059062+0000","last_fullsized":"2023-10-09T10:55:44.059062+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:50.557850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"311'9316","reported_seq":9399,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:00.279657+0000","last_change":"2023-10-09T10:54:49.182381+0000","last_active":"2023-10-09T10:56:00.279657+0000","last_peered":"2023-10-09T10:56:00.279657+0000","last_clean":"2023-10-09T10:54:02.429830+0000","last_became_active":"2023-10-09T10:54:49.176829+0000","last_became_peered":"2023-10-09T10:54:49.176829+0000","last_unstale":"2023-10-09T10:56:00.279657+0000","last_undegraded":"2023-10-09T10:54:48.956857+0000","last_fullsized":"2023-10-09T10:54:48.892711+0000","mapping_epoch":295,"log_start":"275'4255","ondisk_log_start":"275'4255","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5061,"log_dups_size":0,"ondisk_log_size":5061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:04.678256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184418304,"num_objects":2814,"num_object_clones":0,"num_object_copies":8442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1366,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9316,"num_write_kb":388160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"3,4","objects":1448},{"shards":"3,4,7","objects":1365}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":658,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.058883+0000","last_change":"2023-10-09T10:55:25.634848+0000","last_active":"2023-10-09T10:55:45.058883+0000","last_peered":"2023-10-09T10:55:45.058883+0000","last_clean":"2023-10-09T10:55:45.058883+0000","last_became_active":"2023-10-09T10:55:25.634509+0000","last_became_peered":"2023-10-09T10:55:25.634509+0000","last_unstale":"2023-10-09T10:55:45.058883+0000","last_undegraded":"2023-10-09T10:55:45.058883+0000","last_fullsized":"2023-10-09T10:55:45.058883+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":40,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.053915+0000","last_change":"2023-10-09T10:54:17.885030+0000","last_active":"2023-10-09T10:55:45.053915+0000","last_peered":"2023-10-09T10:55:45.053915+0000","last_clean":"2023-10-09T10:55:45.053915+0000","last_became_active":"2023-10-09T10:54:17.884752+0000","last_became_peered":"2023-10-09T10:54:17.884752+0000","last_unstale":"2023-10-09T10:55:45.053915+0000","last_undegraded":"2023-10-09T10:55:45.053915+0000","last_fullsized":"2023-10-09T10:55:45.053915+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:38.548132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.056731+0000","last_change":"2023-10-09T10:54:47.927822+0000","last_active":"2023-10-09T10:55:44.056731+0000","last_peered":"2023-10-09T10:55:44.056731+0000","last_clean":"2023-10-09T10:55:44.056731+0000","last_became_active":"2023-10-09T10:54:47.927455+0000","last_became_peered":"2023-10-09T10:54:47.927455+0000","last_unstale":"2023-10-09T10:55:44.056731+0000","last_undegraded":"2023-10-09T10:55:44.056731+0000","last_fullsized":"2023-10-09T10:55:44.056731+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:11.818203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.062046+0000","last_change":"2023-10-09T10:54:48.012629+0000","last_active":"2023-10-09T10:55:44.062046+0000","last_peered":"2023-10-09T10:55:44.062046+0000","last_clean":"2023-10-09T10:55:44.062046+0000","last_became_active":"2023-10-09T10:54:48.012148+0000","last_became_peered":"2023-10-09T10:54:48.012148+0000","last_unstale":"2023-10-09T10:55:44.062046+0000","last_undegraded":"2023-10-09T10:55:44.062046+0000","last_fullsized":"2023-10-09T10:55:44.062046+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:22.894111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"311'7581","reported_seq":7646,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:55:58.348583+0000","last_change":"2023-10-09T10:54:40.559886+0000","last_active":"2023-10-09T10:55:58.348583+0000","last_peered":"2023-10-09T10:55:58.348583+0000","last_clean":"2023-10-09T10:55:58.348583+0000","last_became_active":"2023-10-09T10:54:40.557153+0000","last_became_peered":"2023-10-09T10:54:40.557153+0000","last_unstale":"2023-10-09T10:55:58.348583+0000","last_undegraded":"2023-10-09T10:55:58.348583+0000","last_fullsized":"2023-10-09T10:55:58.348583+0000","mapping_epoch":265,"log_start":"270'300","ondisk_log_start":"270'300","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7281,"log_dups_size":0,"ondisk_log_size":7281,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94633984,"num_objects":1444,"num_object_clones":0,"num_object_copies":4332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4596,"num_write_kb":193280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":40,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.053676+0000","last_change":"2023-10-09T10:54:17.884876+0000","last_active":"2023-10-09T10:55:45.053676+0000","last_peered":"2023-10-09T10:55:45.053676+0000","last_clean":"2023-10-09T10:55:45.053676+0000","last_became_active":"2023-10-09T10:54:17.884402+0000","last_became_peered":"2023-10-09T10:54:17.884402+0000","last_unstale":"2023-10-09T10:55:45.053676+0000","last_undegraded":"2023-10-09T10:55:45.053676+0000","last_fullsized":"2023-10-09T10:55:45.053676+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:18.679950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":687,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.058989+0000","last_change":"2023-10-09T10:55:25.633525+0000","last_active":"2023-10-09T10:55:45.058989+0000","last_peered":"2023-10-09T10:55:45.058989+0000","last_clean":"2023-10-09T10:55:45.058989+0000","last_became_active":"2023-10-09T10:55:25.633079+0000","last_became_peered":"2023-10-09T10:55:25.633079+0000","last_unstale":"2023-10-09T10:55:45.058989+0000","last_undegraded":"2023-10-09T10:55:45.058989+0000","last_fullsized":"2023-10-09T10:55:45.058989+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:37.378259+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":242,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.071350+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:55:45.071350+0000","last_peered":"2023-10-09T10:55:45.071350+0000","last_clean":"2023-10-09T10:55:45.071350+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:55:45.071350+0000","last_undegraded":"2023-10-09T10:55:45.071350+0000","last_fullsized":"2023-10-09T10:55:45.071350+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":223,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.040271+0000","last_change":"2023-10-09T10:53:14.836523+0000","last_active":"2023-10-09T10:55:45.040271+0000","last_peered":"2023-10-09T10:55:45.040271+0000","last_clean":"2023-10-09T10:55:45.040271+0000","last_became_active":"2023-10-09T10:53:14.836134+0000","last_became_peered":"2023-10-09T10:53:14.836134+0000","last_unstale":"2023-10-09T10:55:45.040271+0000","last_undegraded":"2023-10-09T10:55:45.040271+0000","last_fullsized":"2023-10-09T10:55:45.040271+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:05:24.615198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"311'9122","reported_seq":9935,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:00.220461+0000","last_change":"2023-10-09T10:54:49.078027+0000","last_active":"2023-10-09T10:56:00.220461+0000","last_peered":"2023-10-09T10:56:00.220461+0000","last_clean":"2023-10-09T10:54:02.408036+0000","last_became_active":"2023-10-09T10:54:49.069949+0000","last_became_peered":"2023-10-09T10:54:49.069949+0000","last_unstale":"2023-10-09T10:56:00.220461+0000","last_undegraded":"2023-10-09T10:54:49.050557+0000","last_fullsized":"2023-10-09T10:54:48.898155+0000","mapping_epoch":295,"log_start":"275'4072","ondisk_log_start":"275'4072","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5050,"log_dups_size":0,"ondisk_log_size":5050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:38.092075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185860096,"num_objects":2836,"num_object_clones":0,"num_object_copies":8508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1353,"num_objects_misplaced":2836,"num_objects_unfound":0,"num_objects_dirty":2836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9122,"num_write_kb":382656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":366,"num_bytes_recovered":23986176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":1353},{"shards":"3,5","objects":1483}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":657,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.060812+0000","last_change":"2023-10-09T10:54:50.915855+0000","last_active":"2023-10-09T10:55:44.060812+0000","last_peered":"2023-10-09T10:55:44.060812+0000","last_clean":"2023-10-09T10:55:44.060812+0000","last_became_active":"2023-10-09T10:54:50.914536+0000","last_became_peered":"2023-10-09T10:54:50.914536+0000","last_unstale":"2023-10-09T10:55:44.060812+0000","last_undegraded":"2023-10-09T10:55:44.060812+0000","last_fullsized":"2023-10-09T10:55:44.060812+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.070026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.070026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":261,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.058914+0000","last_change":"2023-10-09T10:55:25.633308+0000","last_active":"2023-10-09T10:55:45.058914+0000","last_peered":"2023-10-09T10:55:45.058914+0000","last_clean":"2023-10-09T10:55:45.058914+0000","last_became_active":"2023-10-09T10:55:25.632646+0000","last_became_peered":"2023-10-09T10:55:25.632646+0000","last_unstale":"2023-10-09T10:55:45.058914+0000","last_undegraded":"2023-10-09T10:55:45.058914+0000","last_fullsized":"2023-10-09T10:55:45.058914+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:17:13.735888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":192,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.052772+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:55:45.052772+0000","last_peered":"2023-10-09T10:55:45.052772+0000","last_clean":"2023-10-09T10:55:45.052772+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:55:45.052772+0000","last_undegraded":"2023-10-09T10:55:45.052772+0000","last_fullsized":"2023-10-09T10:55:45.052772+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":255,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.052485+0000","last_change":"2023-10-09T10:54:47.939756+0000","last_active":"2023-10-09T10:55:45.052485+0000","last_peered":"2023-10-09T10:55:45.052485+0000","last_clean":"2023-10-09T10:55:45.052485+0000","last_became_active":"2023-10-09T10:54:47.938928+0000","last_became_peered":"2023-10-09T10:54:47.938928+0000","last_unstale":"2023-10-09T10:55:45.052485+0000","last_undegraded":"2023-10-09T10:55:45.052485+0000","last_fullsized":"2023-10-09T10:55:45.052485+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:05.310929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"311'9352","reported_seq":9437,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:00.218531+0000","last_change":"2023-10-09T10:54:49.192234+0000","last_active":"2023-10-09T10:56:00.218531+0000","last_peered":"2023-10-09T10:56:00.218531+0000","last_clean":"2023-10-09T10:54:02.435746+0000","last_became_active":"2023-10-09T10:54:49.186762+0000","last_became_peered":"2023-10-09T10:54:49.186762+0000","last_unstale":"2023-10-09T10:56:00.218531+0000","last_undegraded":"2023-10-09T10:54:49.038902+0000","last_fullsized":"2023-10-09T10:54:48.912839+0000","mapping_epoch":295,"log_start":"275'4271","ondisk_log_start":"275'4271","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5081,"log_dups_size":0,"ondisk_log_size":5081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:48.327483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184680448,"num_objects":2818,"num_object_clones":0,"num_object_copies":8454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1330,"num_objects_misplaced":2818,"num_objects_unfound":0,"num_objects_dirty":2818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9352,"num_write_kb":389440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":1488},{"shards":"3,4,6","objects":1328}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":683,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.052703+0000","last_change":"2023-10-09T10:53:27.625180+0000","last_active":"2023-10-09T10:55:45.052703+0000","last_peered":"2023-10-09T10:55:45.052703+0000","last_clean":"2023-10-09T10:55:45.052703+0000","last_became_active":"2023-10-09T10:51:12.366828+0000","last_became_peered":"2023-10-09T10:51:12.366828+0000","last_unstale":"2023-10-09T10:55:45.052703+0000","last_undegraded":"2023-10-09T10:55:45.052703+0000","last_fullsized":"2023-10-09T10:55:45.052703+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.625079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.625079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:59:41.253976+0000","scrub_duration":0.0041067969999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":54,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.060667+0000","last_change":"2023-10-09T10:53:14.833073+0000","last_active":"2023-10-09T10:55:45.060667+0000","last_peered":"2023-10-09T10:55:45.060667+0000","last_clean":"2023-10-09T10:55:45.060667+0000","last_became_active":"2023-10-09T10:53:14.832736+0000","last_became_peered":"2023-10-09T10:53:14.832736+0000","last_unstale":"2023-10-09T10:55:45.060667+0000","last_undegraded":"2023-10-09T10:55:45.060667+0000","last_fullsized":"2023-10-09T10:55:45.060667+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:27.550707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.059289+0000","last_change":"2023-10-09T10:54:47.925711+0000","last_active":"2023-10-09T10:55:44.059289+0000","last_peered":"2023-10-09T10:55:44.059289+0000","last_clean":"2023-10-09T10:55:44.059289+0000","last_became_active":"2023-10-09T10:54:47.924286+0000","last_became_peered":"2023-10-09T10:54:47.924286+0000","last_unstale":"2023-10-09T10:55:44.059289+0000","last_undegraded":"2023-10-09T10:55:44.059289+0000","last_fullsized":"2023-10-09T10:55:44.059289+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:58.379080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":219,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.050491+0000","last_change":"2023-10-09T10:53:14.828291+0000","last_active":"2023-10-09T10:55:45.050491+0000","last_peered":"2023-10-09T10:55:45.050491+0000","last_clean":"2023-10-09T10:55:45.050491+0000","last_became_active":"2023-10-09T10:53:14.827991+0000","last_became_peered":"2023-10-09T10:53:14.827991+0000","last_unstale":"2023-10-09T10:55:45.050491+0000","last_undegraded":"2023-10-09T10:55:45.050491+0000","last_fullsized":"2023-10-09T10:55:45.050491+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:08:14.283112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"311'9231","reported_seq":9288,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:55:58.390911+0000","last_change":"2023-10-09T10:53:14.822279+0000","last_active":"2023-10-09T10:55:58.390911+0000","last_peered":"2023-10-09T10:55:58.390911+0000","last_clean":"2023-10-09T10:55:58.390911+0000","last_became_active":"2023-10-09T10:53:14.821952+0000","last_became_peered":"2023-10-09T10:53:14.821952+0000","last_unstale":"2023-10-09T10:55:58.390911+0000","last_undegraded":"2023-10-09T10:55:58.390911+0000","last_fullsized":"2023-10-09T10:55:58.390911+0000","mapping_epoch":266,"log_start":"275'3941","ondisk_log_start":"275'3941","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5290,"log_dups_size":0,"ondisk_log_size":5290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:37:14.124652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191561728,"num_objects":2923,"num_object_clones":0,"num_object_copies":8769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9231,"num_write_kb":388928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":706,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.042317+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T10:55:45.042317+0000","last_peered":"2023-10-09T10:55:45.042317+0000","last_clean":"2023-10-09T10:55:45.042317+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T10:55:45.042317+0000","last_undegraded":"2023-10-09T10:55:45.042317+0000","last_fullsized":"2023-10-09T10:55:45.042317+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":333,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.073799+0000","last_change":"2023-10-09T10:53:14.841466+0000","last_active":"2023-10-09T10:55:45.073799+0000","last_peered":"2023-10-09T10:55:45.073799+0000","last_clean":"2023-10-09T10:55:45.073799+0000","last_became_active":"2023-10-09T10:53:14.831674+0000","last_became_peered":"2023-10-09T10:53:14.831674+0000","last_unstale":"2023-10-09T10:55:45.073799+0000","last_undegraded":"2023-10-09T10:55:45.073799+0000","last_fullsized":"2023-10-09T10:55:45.073799+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:02.612069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.060028+0000","last_change":"2023-10-09T10:54:47.908305+0000","last_active":"2023-10-09T10:55:44.060028+0000","last_peered":"2023-10-09T10:55:44.060028+0000","last_clean":"2023-10-09T10:55:44.060028+0000","last_became_active":"2023-10-09T10:54:47.907999+0000","last_became_peered":"2023-10-09T10:54:47.907999+0000","last_unstale":"2023-10-09T10:55:44.060028+0000","last_undegraded":"2023-10-09T10:55:44.060028+0000","last_fullsized":"2023-10-09T10:55:44.060028+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:48.405124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":333,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.071031+0000","last_change":"2023-10-09T10:53:14.835841+0000","last_active":"2023-10-09T10:55:45.071031+0000","last_peered":"2023-10-09T10:55:45.071031+0000","last_clean":"2023-10-09T10:55:45.071031+0000","last_became_active":"2023-10-09T10:53:14.835551+0000","last_became_peered":"2023-10-09T10:53:14.835551+0000","last_unstale":"2023-10-09T10:55:45.071031+0000","last_undegraded":"2023-10-09T10:55:45.071031+0000","last_fullsized":"2023-10-09T10:55:45.071031+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:21.498099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"311'9185","reported_seq":13364,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:01.169259+0000","last_change":"2023-10-09T10:55:26.696179+0000","last_active":"2023-10-09T10:56:01.169259+0000","last_peered":"2023-10-09T10:56:01.169259+0000","last_clean":"2023-10-09T10:54:17.082254+0000","last_became_active":"2023-10-09T10:55:26.641906+0000","last_became_peered":"2023-10-09T10:55:26.641906+0000","last_unstale":"2023-10-09T10:56:01.169259+0000","last_undegraded":"2023-10-09T10:55:26.633572+0000","last_fullsized":"2023-10-09T10:55:26.625249+0000","mapping_epoch":306,"log_start":"274'3449","ondisk_log_start":"274'3449","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5736,"log_dups_size":0,"ondisk_log_size":5736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:02.762689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181010456,"num_objects":2763,"num_object_clones":0,"num_object_copies":8289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3090,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2763,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9185,"num_write_kb":382273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2055,"num_bytes_recovered":134610944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5],"avail_no_missing":["7","5","6"],"object_location_counts":[{"shards":"5,6,7","objects":1934},{"shards":"5,7","objects":-327}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":702,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.076248+0000","last_change":"2023-10-09T10:53:30.567189+0000","last_active":"2023-10-09T10:55:45.076248+0000","last_peered":"2023-10-09T10:55:45.076248+0000","last_clean":"2023-10-09T10:55:45.076248+0000","last_became_active":"2023-10-09T10:53:17.840928+0000","last_became_peered":"2023-10-09T10:53:17.840928+0000","last_unstale":"2023-10-09T10:55:45.076248+0000","last_undegraded":"2023-10-09T10:55:45.076248+0000","last_fullsized":"2023-10-09T10:55:45.076248+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:52.176229+0000","scrub_duration":0.0053707060000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":164,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.059906+0000","last_change":"2023-10-09T10:55:25.634235+0000","last_active":"2023-10-09T10:55:45.059906+0000","last_peered":"2023-10-09T10:55:45.059906+0000","last_clean":"2023-10-09T10:55:45.059906+0000","last_became_active":"2023-10-09T10:55:25.633940+0000","last_became_peered":"2023-10-09T10:55:25.633940+0000","last_unstale":"2023-10-09T10:55:45.059906+0000","last_undegraded":"2023-10-09T10:55:45.059906+0000","last_fullsized":"2023-10-09T10:55:45.059906+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:40.554244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":239,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.052564+0000","last_change":"2023-10-09T10:54:48.016159+0000","last_active":"2023-10-09T10:55:45.052564+0000","last_peered":"2023-10-09T10:55:45.052564+0000","last_clean":"2023-10-09T10:55:45.052564+0000","last_became_active":"2023-10-09T10:54:47.939409+0000","last_became_peered":"2023-10-09T10:54:47.939409+0000","last_unstale":"2023-10-09T10:55:45.052564+0000","last_undegraded":"2023-10-09T10:55:45.052564+0000","last_fullsized":"2023-10-09T10:55:45.052564+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:11:15.175207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":252,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.039432+0000","last_change":"2023-10-09T10:51:50.371282+0000","last_active":"2023-10-09T10:55:45.039432+0000","last_peered":"2023-10-09T10:55:45.039432+0000","last_clean":"2023-10-09T10:55:45.039432+0000","last_became_active":"2023-10-09T10:51:50.370973+0000","last_became_peered":"2023-10-09T10:51:50.370973+0000","last_unstale":"2023-10-09T10:55:45.039432+0000","last_undegraded":"2023-10-09T10:55:45.039432+0000","last_fullsized":"2023-10-09T10:55:45.039432+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:15.603164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"311'7682","reported_seq":10571,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:56:00.255723+0000","last_change":"2023-10-09T10:54:56.638107+0000","last_active":"2023-10-09T10:56:00.255723+0000","last_peered":"2023-10-09T10:56:00.255723+0000","last_clean":"2023-10-09T10:56:00.255723+0000","last_became_active":"2023-10-09T10:54:56.633188+0000","last_became_peered":"2023-10-09T10:54:56.633188+0000","last_unstale":"2023-10-09T10:56:00.255723+0000","last_undegraded":"2023-10-09T10:56:00.255723+0000","last_fullsized":"2023-10-09T10:56:00.255723+0000","mapping_epoch":299,"log_start":"271'1100","ondisk_log_start":"271'1100","created":289,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6582,"log_dups_size":0,"ondisk_log_size":6582,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93487104,"num_objects":1427,"num_object_clones":0,"num_object_copies":4281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4665,"num_write_kb":194912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1061,"num_bytes_recovered":69500928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":702,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:55.312029+0000","last_change":"2023-10-09T10:53:29.568199+0000","last_active":"2023-10-09T10:55:55.312029+0000","last_peered":"2023-10-09T10:55:55.312029+0000","last_clean":"2023-10-09T10:55:55.312029+0000","last_became_active":"2023-10-09T10:52:55.416357+0000","last_became_peered":"2023-10-09T10:52:55.416357+0000","last_unstale":"2023-10-09T10:55:55.312029+0000","last_undegraded":"2023-10-09T10:55:55.312029+0000","last_fullsized":"2023-10-09T10:55:55.312029+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:44.364650+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":218,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.050574+0000","last_change":"2023-10-09T10:53:14.823731+0000","last_active":"2023-10-09T10:55:45.050574+0000","last_peered":"2023-10-09T10:55:45.050574+0000","last_clean":"2023-10-09T10:55:45.050574+0000","last_became_active":"2023-10-09T10:53:14.823470+0000","last_became_peered":"2023-10-09T10:53:14.823470+0000","last_unstale":"2023-10-09T10:55:45.050574+0000","last_undegraded":"2023-10-09T10:55:45.050574+0000","last_fullsized":"2023-10-09T10:55:45.050574+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:36.927599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":49,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.054294+0000","last_change":"2023-10-09T10:54:47.944380+0000","last_active":"2023-10-09T10:55:45.054294+0000","last_peered":"2023-10-09T10:55:45.054294+0000","last_clean":"2023-10-09T10:55:45.054294+0000","last_became_active":"2023-10-09T10:54:47.943766+0000","last_became_peered":"2023-10-09T10:54:47.943766+0000","last_unstale":"2023-10-09T10:55:45.054294+0000","last_undegraded":"2023-10-09T10:55:45.054294+0000","last_fullsized":"2023-10-09T10:55:45.054294+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:16:11.869603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"311'7538","reported_seq":7628,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:00.245929+0000","last_change":"2023-10-09T10:54:49.184288+0000","last_active":"2023-10-09T10:56:00.245929+0000","last_peered":"2023-10-09T10:56:00.245929+0000","last_clean":"2023-10-09T10:54:02.391425+0000","last_became_active":"2023-10-09T10:54:49.118164+0000","last_became_peered":"2023-10-09T10:54:49.118164+0000","last_unstale":"2023-10-09T10:56:00.245929+0000","last_undegraded":"2023-10-09T10:54:48.969517+0000","last_fullsized":"2023-10-09T10:54:48.892012+0000","mapping_epoch":295,"log_start":"271'1000","ondisk_log_start":"271'1000","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6538,"log_dups_size":0,"ondisk_log_size":6538,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96043008,"num_objects":1466,"num_object_clones":0,"num_object_copies":4398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":727,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4562,"num_write_kb":192864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"3,4,6","objects":727},{"shards":"3,6","objects":739}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":633,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.070024+0000","last_change":"2023-10-09T10:53:36.563918+0000","last_active":"2023-10-09T10:55:45.070024+0000","last_peered":"2023-10-09T10:55:45.070024+0000","last_clean":"2023-10-09T10:55:45.070024+0000","last_became_active":"2023-10-09T10:53:17.842952+0000","last_became_peered":"2023-10-09T10:53:17.842952+0000","last_unstale":"2023-10-09T10:55:45.070024+0000","last_undegraded":"2023-10-09T10:55:45.070024+0000","last_fullsized":"2023-10-09T10:55:45.070024+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:53.185917+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":137,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.039986+0000","last_change":"2023-10-09T10:54:47.923386+0000","last_active":"2023-10-09T10:55:45.039986+0000","last_peered":"2023-10-09T10:55:45.039986+0000","last_clean":"2023-10-09T10:55:45.039986+0000","last_became_active":"2023-10-09T10:54:47.914257+0000","last_became_peered":"2023-10-09T10:54:47.914257+0000","last_unstale":"2023-10-09T10:55:45.039986+0000","last_undegraded":"2023-10-09T10:55:45.039986+0000","last_fullsized":"2023-10-09T10:55:45.039986+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:24:18.190715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":328,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.302345+0000","last_change":"2023-10-09T10:54:47.977518+0000","last_active":"2023-10-09T10:55:45.302345+0000","last_peered":"2023-10-09T10:55:45.302345+0000","last_clean":"2023-10-09T10:55:45.302345+0000","last_became_active":"2023-10-09T10:54:47.977332+0000","last_became_peered":"2023-10-09T10:54:47.977332+0000","last_unstale":"2023-10-09T10:55:45.302345+0000","last_undegraded":"2023-10-09T10:55:45.302345+0000","last_fullsized":"2023-10-09T10:55:45.302345+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:29.620839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.16","version":"311'7571","reported_seq":7636,"reported_epoch":311,"state":"active+recovery_wait","last_fresh":"2023-10-09T10:55:58.348436+0000","last_change":"2023-10-09T10:54:40.600782+0000","last_active":"2023-10-09T10:55:58.348436+0000","last_peered":"2023-10-09T10:55:58.348436+0000","last_clean":"2023-10-09T10:54:39.522361+0000","last_became_active":"2023-10-09T10:54:40.590969+0000","last_became_peered":"2023-10-09T10:54:40.590969+0000","last_unstale":"2023-10-09T10:55:58.348436+0000","last_undegraded":"2023-10-09T10:55:58.348436+0000","last_fullsized":"2023-10-09T10:55:58.348436+0000","mapping_epoch":289,"log_start":"270'300","ondisk_log_start":"270'300","created":289,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7271,"log_dups_size":0,"ondisk_log_size":7271,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97681408,"num_objects":1491,"num_object_clones":0,"num_object_copies":4473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4577,"num_write_kb":194144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":1491}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":260,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.048155+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:55:44.048155+0000","last_peered":"2023-10-09T10:55:44.048155+0000","last_clean":"2023-10-09T10:55:44.048155+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:55:44.048155+0000","last_undegraded":"2023-10-09T10:55:44.048155+0000","last_fullsized":"2023-10-09T10:55:44.048155+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":255,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.302320+0000","last_change":"2023-10-09T10:50:43.385150+0000","last_active":"2023-10-09T10:55:45.302320+0000","last_peered":"2023-10-09T10:55:45.302320+0000","last_clean":"2023-10-09T10:55:45.302320+0000","last_became_active":"2023-10-09T10:50:43.384788+0000","last_became_peered":"2023-10-09T10:50:43.384788+0000","last_unstale":"2023-10-09T10:55:45.302320+0000","last_undegraded":"2023-10-09T10:55:45.302320+0000","last_fullsized":"2023-10-09T10:55:45.302320+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:36.085910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.15","version":"311'7530","reported_seq":7597,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-10-09T10:56:01.101864+0000","last_change":"2023-10-09T10:54:40.578264+0000","last_active":"2023-10-09T10:56:01.101864+0000","last_peered":"2023-10-09T10:56:01.101864+0000","last_clean":"2023-10-09T10:56:01.101864+0000","last_became_active":"2023-10-09T10:54:40.568498+0000","last_became_peered":"2023-10-09T10:54:40.568498+0000","last_unstale":"2023-10-09T10:56:01.101864+0000","last_undegraded":"2023-10-09T10:56:01.101864+0000","last_fullsized":"2023-10-09T10:56:01.101864+0000","mapping_epoch":289,"log_start":"271'400","ondisk_log_start":"271'400","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7130,"log_dups_size":0,"ondisk_log_size":7130,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92471296,"num_objects":1411,"num_object_clones":0,"num_object_copies":4233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4569,"num_write_kb":191360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":617,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.059345+0000","last_change":"2023-10-09T10:55:25.635605+0000","last_active":"2023-10-09T10:55:45.059345+0000","last_peered":"2023-10-09T10:55:45.059345+0000","last_clean":"2023-10-09T10:55:45.059345+0000","last_became_active":"2023-10-09T10:55:25.635328+0000","last_became_peered":"2023-10-09T10:55:25.635328+0000","last_unstale":"2023-10-09T10:55:45.059345+0000","last_undegraded":"2023-10-09T10:55:45.059345+0000","last_fullsized":"2023-10-09T10:55:45.059345+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:46.305288+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:46.305288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":335,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.070370+0000","last_change":"2023-10-09T10:54:17.886158+0000","last_active":"2023-10-09T10:55:45.070370+0000","last_peered":"2023-10-09T10:55:45.070370+0000","last_clean":"2023-10-09T10:55:45.070370+0000","last_became_active":"2023-10-09T10:54:17.885816+0000","last_became_peered":"2023-10-09T10:54:17.885816+0000","last_unstale":"2023-10-09T10:55:45.070370+0000","last_undegraded":"2023-10-09T10:55:45.070370+0000","last_fullsized":"2023-10-09T10:55:45.070370+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:05.270825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":338,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.070840+0000","last_change":"2023-10-09T10:54:17.957784+0000","last_active":"2023-10-09T10:55:45.070840+0000","last_peered":"2023-10-09T10:55:45.070840+0000","last_clean":"2023-10-09T10:55:45.070840+0000","last_became_active":"2023-10-09T10:54:17.954788+0000","last_became_peered":"2023-10-09T10:54:17.954788+0000","last_unstale":"2023-10-09T10:55:45.070840+0000","last_undegraded":"2023-10-09T10:55:45.070840+0000","last_fullsized":"2023-10-09T10:55:45.070840+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:48.709390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"311'7662","reported_seq":7740,"reported_epoch":311,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:56:01.284283+0000","last_change":"2023-10-09T10:54:42.558777+0000","last_active":"2023-10-09T10:56:01.284283+0000","last_peered":"2023-10-09T10:56:01.284283+0000","last_clean":"2023-10-09T10:54:17.036595+0000","last_became_active":"2023-10-09T10:54:42.553632+0000","last_became_peered":"2023-10-09T10:54:42.553632+0000","last_unstale":"2023-10-09T10:56:01.284283+0000","last_undegraded":"2023-10-09T10:56:01.284283+0000","last_fullsized":"2023-10-09T10:56:01.284283+0000","mapping_epoch":291,"log_start":"271'600","ondisk_log_start":"271'600","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7062,"log_dups_size":0,"ondisk_log_size":7062,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88637440,"num_objects":1353,"num_object_clones":0,"num_object_copies":4059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1353,"num_objects_unfound":0,"num_objects_dirty":1353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4677,"num_write_kb":192928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":1353}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":584,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T10:55:44.059715+0000","last_change":"2023-10-09T10:54:50.914473+0000","last_active":"2023-10-09T10:55:44.059715+0000","last_peered":"2023-10-09T10:55:44.059715+0000","last_clean":"2023-10-09T10:55:44.059715+0000","last_became_active":"2023-10-09T10:54:50.914005+0000","last_became_peered":"2023-10-09T10:54:50.914005+0000","last_unstale":"2023-10-09T10:55:44.059715+0000","last_undegraded":"2023-10-09T10:55:44.059715+0000","last_fullsized":"2023-10-09T10:55:44.059715+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.070492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.070492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":219,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.051097+0000","last_change":"2023-10-09T10:54:18.089053+0000","last_active":"2023-10-09T10:55:45.051097+0000","last_peered":"2023-10-09T10:55:45.051097+0000","last_clean":"2023-10-09T10:55:45.051097+0000","last_became_active":"2023-10-09T10:54:17.886070+0000","last_became_peered":"2023-10-09T10:54:17.886070+0000","last_unstale":"2023-10-09T10:55:45.051097+0000","last_undegraded":"2023-10-09T10:55:45.051097+0000","last_fullsized":"2023-10-09T10:55:45.051097+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:35.628407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":315,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T10:55:45.039365+0000","last_change":"2023-10-09T10:54:47.908059+0000","last_active":"2023-10-09T10:55:45.039365+0000","last_peered":"2023-10-09T10:55:45.039365+0000","last_clean":"2023-10-09T10:55:45.039365+0000","last_became_active":"2023-10-09T10:54:47.907780+0000","last_became_peered":"2023-10-09T10:54:47.907780+0000","last_unstale":"2023-10-09T10:55:45.039365+0000","last_undegraded":"2023-10-09T10:55:45.039365+0000","last_fullsized":"2023-10-09T10:55:45.039365+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:55:59.064617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":6668,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":3005874200,"num_objects":45867,"num_object_clones":0,"num_object_copies":137601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14587,"num_objects_misplaced":10594,"num_objects_unfound":0,"num_objects_dirty":45867,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":147437,"num_write_kb":6185665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8167,"num_bytes_recovered":535166976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9815404544,"data_stored":9815392328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":172124,"ondisk_log_size":172124,"up":78,"acting":68,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":729016,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":115,"num_read_kb":214,"num_write":56,"num_write_kb":567,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1105158,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332353,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1647476,"kb_used_data":1337900,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92097676,"statfs":{"total":95995035648,"available":94308020224,"internally_reserved":0,"allocated":1370009600,"data_stored":1368879418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1429999999999998,"5min":1.9299999999999999,"15min":1.365},"min":{"1min":0.23599999999999999,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":7.4080000000000004,"5min":9.6259999999999994,"15min":9.6379999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.3650000000000002,"5min":1.8839999999999999,"15min":1.55},"min":{"1min":0.28699999999999998,"5min":0.246,"15min":0.24299999999999999},"max":{"1min":8.7739999999999991,"5min":8.7739999999999991,"15min":16.731000000000002},"last":3.4969999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:55:01 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":2.0539999999999998,"15min":1.534},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.25700000000000001},"max":{"1min":3.9860000000000002,"5min":34.756,"15min":34.756},"last":2.5270000000000001},{"interface":"front","average":{"1min":1.6830000000000001,"5min":1.8280000000000001,"15min":1.319},"min":{"1min":0.32100000000000001,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":6.641,"5min":35.658999999999999,"15min":35.658999999999999},"last":2.8999999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:55:29 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":1.3480000000000001,"15min":1.3089999999999999},"min":{"1min":0.32300000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.8580000000000001,"5min":8.3620000000000001,"15min":11.058},"last":0.25900000000000001},{"interface":"front","average":{"1min":1.401,"5min":1.5660000000000001,"15min":1.232},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":4.8890000000000002,"5min":11.579000000000001,"15min":11.579000000000001},"last":0.72699999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:55:55 2023","interfaces":[{"interface":"back","average":{"1min":2.056,"5min":1.9490000000000001,"15min":1.7050000000000001},"min":{"1min":0.32900000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":8.7260000000000009,"5min":10.069000000000001,"15min":11.523},"last":6.4989999999999997},{"interface":"front","average":{"1min":1.9930000000000001,"5min":1.52,"15min":1.526},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.585,"5min":8.6300000000000008,"15min":19.234999999999999},"last":3.7450000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:55:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.3819999999999999,"15min":1.214},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.0350000000000001,"5min":16.695,"15min":16.695},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.57499999999999996,"5min":1.1990000000000001,"15min":1.0940000000000001},"min":{"1min":0.28699999999999998,"5min":0.28199999999999997,"15min":0.26200000000000001},"max":{"1min":2.2480000000000002,"5min":9.0459999999999994,"15min":11.109999999999999},"last":0.45600000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:55:34 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.218,"15min":1.1519999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.247},"max":{"1min":1.919,"5min":13.262,"15min":13.262},"last":0.441},{"interface":"front","average":{"1min":0.49199999999999999,"5min":1.365,"15min":1.331},"min":{"1min":0.31900000000000001,"5min":0.318,"15min":0.29599999999999999},"max":{"1min":1.5649999999999999,"5min":8.8659999999999997,"15min":11.741},"last":0.378}]},{"osd":6,"last update":"Mon Oct 9 10:55:34 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":1.679,"15min":1.4259999999999999},"min":{"1min":0.34399999999999997,"5min":0.317,"15min":0.28199999999999997},"max":{"1min":1.9850000000000001,"5min":14.913,"15min":14.913},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.72399999999999998,"5min":1.4279999999999999,"15min":1.2709999999999999},"min":{"1min":0.32900000000000001,"5min":0.317,"15min":0.29399999999999998},"max":{"1min":3.3799999999999999,"5min":12.268000000000001,"15min":19.187999999999999},"last":0.41799999999999998}]}]},{"osd":6,"up_from":44,"seq":188978561284,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2399904,"kb_used_data":1495640,"kb_used_omap":0,"kb_used_meta":904256,"kb_avail":91345248,"statfs":{"total":95995035648,"available":93537533952,"internally_reserved":0,"allocated":1531535360,"data_stored":1530424944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925958144},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:55 2023","interfaces":[{"interface":"back","average":{"1min":5.0620000000000003,"5min":1.9490000000000001,"15min":1.4770000000000001},"min":{"1min":0.22800000000000001,"5min":0.216,"15min":0.216},"max":{"1min":54.743000000000002,"5min":54.743000000000002,"15min":54.743000000000002},"last":0.28399999999999997},{"interface":"front","average":{"1min":4.1219999999999999,"5min":1.97,"15min":1.4850000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.22800000000000001},"max":{"1min":53.701000000000001,"5min":53.701000000000001,"15min":53.701000000000001},"last":0.246}]},{"osd":1,"last update":"Mon Oct 9 10:55:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4410000000000001,"5min":1.448,"15min":1.2689999999999999},"min":{"1min":0.32800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.6820000000000004,"5min":8.4730000000000008,"15min":16.747},"last":2.8029999999999999},{"interface":"front","average":{"1min":1.585,"5min":1.54,"15min":1.321},"min":{"1min":0.30499999999999999,"5min":0.23599999999999999,"15min":0.23100000000000001},"max":{"1min":7.5049999999999999,"5min":8.0579999999999998,"15min":16.734000000000002},"last":2.8079999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:55:33 2023","interfaces":[{"interface":"back","average":{"1min":2.1309999999999998,"5min":1.958,"15min":1.401},"min":{"1min":0.29199999999999998,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":8.0589999999999993,"5min":8.6389999999999993,"15min":13.208},"last":2.9119999999999999},{"interface":"front","average":{"1min":1.8999999999999999,"5min":2.1309999999999998,"15min":1.5609999999999999},"min":{"1min":0.36299999999999999,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":8.6890000000000001,"5min":12.83,"15min":12.83},"last":1.5880000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:55:55 2023","interfaces":[{"interface":"back","average":{"1min":1.837,"5min":1.768,"15min":1.4510000000000001},"min":{"1min":0.32500000000000001,"5min":0.29799999999999999,"15min":0.255},"max":{"1min":5.8399999999999999,"5min":7.4720000000000004,"15min":12.391},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.1949999999999998,"5min":2.0739999999999998,"15min":1.667},"min":{"1min":0.38600000000000001,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":8.3759999999999994,"5min":8.3759999999999994,"15min":13.179},"last":5.5830000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:55:28 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.7010000000000001,"15min":1.171},"min":{"1min":0.251,"5min":0.247,"15min":0.23200000000000001},"max":{"1min":16.978999999999999,"5min":16.978999999999999,"15min":16.978999999999999},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.69399999999999995,"5min":1.347,"15min":1.1279999999999999},"min":{"1min":0.311,"5min":0.308,"15min":0.26400000000000001},"max":{"1min":3.6709999999999998,"5min":12.061999999999999,"15min":12.481999999999999},"last":0.32000000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:55:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":1.869,"15min":1.292},"min":{"1min":0.28599999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.923,"5min":12.923,"15min":12.923},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.71799999999999997,"5min":1.532,"15min":1.1910000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.3100000000000001,"5min":12.103999999999999,"15min":12.456},"last":0.40699999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:55:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":1.6950000000000001,"15min":1.3340000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.26600000000000001},"max":{"1min":2.0449999999999999,"5min":12.882,"15min":23.358000000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.70099999999999996,"5min":1.149,"15min":1.1299999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.7200000000000002,"5min":7.0880000000000001,"15min":12.422000000000001},"last":0.36599999999999999}]}]},{"osd":1,"up_from":85,"seq":365072220322,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1297496,"kb_used_data":904976,"kb_used_omap":9,"kb_used_meta":392438,"kb_avail":92447656,"statfs":{"total":95995035648,"available":94666399744,"internally_reserved":0,"allocated":926695424,"data_stored":925062717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":401857513},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.3959999999999999,"15min":1.4510000000000001},"min":{"1min":0.20999999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":12.188000000000001,"5min":12.188000000000001,"15min":18.027000000000001},"last":1.1499999999999999},{"interface":"front","average":{"1min":2.8540000000000001,"5min":1.9059999999999999,"15min":1.5389999999999999},"min":{"1min":0.23400000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":18.303000000000001,"5min":18.303000000000001,"15min":18.303000000000001},"last":0.373}]},{"osd":2,"last update":"Mon Oct 9 10:55:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5899999999999999,"5min":1.599,"15min":1.341},"min":{"1min":0.23499999999999999,"5min":0.222,"15min":0.19700000000000001},"max":{"1min":12.244999999999999,"5min":12.244999999999999,"15min":15.356},"last":14.878},{"interface":"front","average":{"1min":3.3220000000000001,"5min":1.587,"15min":1.4750000000000001},"min":{"1min":0.23899999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":18.004000000000001,"5min":18.004000000000001,"15min":18.004000000000001},"last":2.8130000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:55:56 2023","interfaces":[{"interface":"back","average":{"1min":4.4429999999999996,"5min":2.1859999999999999,"15min":1.855},"min":{"1min":0.22900000000000001,"5min":0.223,"15min":0.218},"max":{"1min":20.036000000000001,"5min":20.036000000000001,"15min":20.036000000000001},"last":1.4330000000000001},{"interface":"front","average":{"1min":6.359,"5min":2.9350000000000001,"15min":1.9430000000000001},"min":{"1min":0.42799999999999999,"5min":0.219,"15min":0.219},"max":{"1min":19.640999999999998,"5min":19.640999999999998,"15min":19.640999999999998},"last":2.3839999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":1.4810000000000001,"15min":1.2689999999999999},"min":{"1min":0.29699999999999999,"5min":0.26500000000000001,"15min":0.255},"max":{"1min":5.423,"5min":5.8559999999999999,"15min":12.167999999999999},"last":14.763},{"interface":"front","average":{"1min":2.073,"5min":1.661,"15min":1.4630000000000001},"min":{"1min":0.36799999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.835,"5min":8.9969999999999999,"15min":14.228999999999999},"last":1.1919999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":2.4969999999999999,"5min":1.756,"15min":1.327},"min":{"1min":0.33700000000000002,"5min":0.28100000000000003,"15min":0.253},"max":{"1min":9.6289999999999996,"5min":9.6289999999999996,"15min":9.7720000000000002},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.6429999999999998,"5min":2.0310000000000001,"15min":1.5980000000000001},"min":{"1min":0.41199999999999998,"5min":0.30299999999999999,"15min":0.28899999999999998},"max":{"1min":9.2460000000000004,"5min":9.2460000000000004,"15min":17.085000000000001},"last":14.833}]},{"osd":6,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.845,"15min":1.4219999999999999},"min":{"1min":0.35299999999999998,"5min":0.32800000000000001,"15min":0.27400000000000002},"max":{"1min":7.1360000000000001,"5min":10.425000000000001,"15min":10.425000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.665,"5min":1.639,"15min":1.4159999999999999},"min":{"1min":0.437,"5min":0.312,"15min":0.312},"max":{"1min":6.0739999999999998,"5min":7.718,"15min":12.231},"last":14.804}]},{"osd":7,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":2.1139999999999999,"5min":1.867,"15min":1.48},"min":{"1min":0.39300000000000002,"5min":0.36299999999999999,"15min":0.34000000000000002},"max":{"1min":6.8289999999999997,"5min":16.312000000000001,"15min":16.312000000000001},"last":14.856},{"interface":"front","average":{"1min":2.0009999999999999,"5min":1.5680000000000001,"15min":1.4850000000000001},"min":{"1min":0.377,"5min":0.35799999999999998,"15min":0.32500000000000001},"max":{"1min":8.8149999999999995,"5min":8.8149999999999995,"15min":14.273999999999999},"last":1.214}]}]},{"osd":0,"up_from":80,"seq":343597383856,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1631412,"kb_used_data":1148524,"kb_used_omap":28,"kb_used_meta":482851,"kb_avail":92113740,"statfs":{"total":95995035648,"available":94324469760,"internally_reserved":0,"allocated":1176088576,"data_stored":1174373577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":494440272},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:54:59 2023","interfaces":[{"interface":"back","average":{"1min":2.609,"5min":1.7190000000000001,"15min":1.5369999999999999},"min":{"1min":0.24199999999999999,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":13.263999999999999,"5min":13.263999999999999,"15min":25.475000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.633,"15min":1.48},"min":{"1min":0.26000000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":9.0860000000000003,"5min":9.359,"15min":20.832999999999998},"last":0.55000000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:55:34 2023","interfaces":[{"interface":"back","average":{"1min":2.996,"5min":1.915,"15min":1.5580000000000001},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":24.852,"5min":24.852,"15min":24.852},"last":0.434},{"interface":"front","average":{"1min":7.2240000000000002,"5min":3.0550000000000002,"15min":2},"min":{"1min":0.30599999999999999,"5min":0.24099999999999999,"15min":0.20399999999999999},"max":{"1min":77.256,"5min":77.256,"15min":77.256},"last":5.1950000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:54:59 2023","interfaces":[{"interface":"back","average":{"1min":2.3940000000000001,"5min":2.1360000000000001,"15min":1.7769999999999999},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.20200000000000001},"max":{"1min":10.456,"5min":13.166,"15min":14.926},"last":2.073},{"interface":"front","average":{"1min":3.2130000000000001,"5min":2.2269999999999999,"15min":1.762},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.224},"max":{"1min":13.305,"5min":13.305,"15min":14.391999999999999},"last":0.46500000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:55:52 2023","interfaces":[{"interface":"back","average":{"1min":2.9550000000000001,"5min":2.8380000000000001,"15min":1.8979999999999999},"min":{"1min":0.38500000000000001,"5min":0.33800000000000002,"15min":0.26200000000000001},"max":{"1min":9.4749999999999996,"5min":77.177999999999997,"15min":77.177999999999997},"last":1.901},{"interface":"front","average":{"1min":1.839,"5min":1.8100000000000001,"15min":1.7110000000000001},"min":{"1min":0.36599999999999999,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":9.032,"5min":10.427,"15min":12.318},"last":0.60399999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:55:52 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":1.966,"15min":1.7509999999999999},"min":{"1min":0.34300000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.9559999999999995,"5min":10.478,"15min":12.157},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.7929999999999999,"15min":1.607},"min":{"1min":0.30399999999999999,"5min":0.30199999999999999,"15min":0.29499999999999998},"max":{"1min":5.319,"5min":24.754999999999999,"15min":24.754999999999999},"last":0.65200000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:55:52 2023","interfaces":[{"interface":"back","average":{"1min":2.7040000000000002,"5min":2.2210000000000001,"15min":1.8839999999999999},"min":{"1min":0.38700000000000001,"5min":0.34200000000000003,"15min":0.30399999999999999},"max":{"1min":18.920000000000002,"5min":18.920000000000002,"15min":18.920000000000002},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.4119999999999999,"5min":1.903,"15min":1.5189999999999999},"min":{"1min":0.34200000000000003,"5min":0.307,"15min":0.307},"max":{"1min":5.2910000000000004,"5min":24.800000000000001,"15min":24.800000000000001},"last":0.68799999999999994}]},{"osd":7,"last update":"Mon Oct 9 10:55:52 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":1.917,"15min":1.7110000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.125,"5min":13.225,"15min":14.48},"last":0.71299999999999997},{"interface":"front","average":{"1min":3.2130000000000001,"5min":3.0430000000000001,"15min":2.0419999999999998},"min":{"1min":0.40000000000000002,"5min":0.32600000000000001,"15min":0.32200000000000001},"max":{"1min":10.819000000000001,"5min":77.228999999999999,"15min":77.228999999999999},"last":1.8680000000000001}]}]},{"osd":2,"up_from":90,"seq":386547056797,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":435700,"kb_used_data":158316,"kb_used_omap":81,"kb_used_meta":277294,"kb_avail":93309452,"statfs":{"total":95995035648,"available":95548878848,"internally_reserved":0,"allocated":162115584,"data_stored":160576521,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":283949274},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:35 2023","interfaces":[{"interface":"back","average":{"1min":2.4790000000000001,"5min":1.7310000000000001,"15min":1.403},"min":{"1min":0.18099999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":17.574000000000002,"5min":17.574000000000002,"15min":18.512},"last":7.6029999999999998},{"interface":"front","average":{"1min":2.1829999999999998,"5min":1.8420000000000001,"15min":1.5349999999999999},"min":{"1min":0.27700000000000002,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":11.167,"5min":15.595000000000001,"15min":17.975000000000001},"last":0.48799999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:55:35 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":1.571,"15min":1.52},"min":{"1min":0.26300000000000001,"5min":0.23300000000000001,"15min":0.20200000000000001},"max":{"1min":11.204000000000001,"5min":11.204000000000001,"15min":15.247999999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":2.1749999999999998,"5min":1.3200000000000001,"15min":1.296},"min":{"1min":0.317,"5min":0.253,"15min":0.20699999999999999},"max":{"1min":10.361000000000001,"5min":10.361000000000001,"15min":16.454999999999998},"last":0.42299999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:55:57 2023","interfaces":[{"interface":"back","average":{"1min":4.4539999999999997,"5min":1.954,"15min":1.5640000000000001},"min":{"1min":0.30299999999999999,"5min":0.27800000000000002,"15min":0.22500000000000001},"max":{"1min":19.539000000000001,"5min":19.539000000000001,"15min":19.539000000000001},"last":7.1859999999999999},{"interface":"front","average":{"1min":4.4400000000000004,"5min":2.327,"15min":1.867},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.20200000000000001},"max":{"1min":16.702999999999999,"5min":16.702999999999999,"15min":16.702999999999999},"last":0.52300000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:55:35 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":1.8200000000000001,"15min":1.5940000000000001},"min":{"1min":0.32800000000000001,"5min":0.29599999999999999,"15min":0.254},"max":{"1min":10.289,"5min":15.672000000000001,"15min":16.231999999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.679,"5min":1.9410000000000001,"15min":1.5589999999999999},"min":{"1min":0.311,"5min":0.307,"15min":0.26300000000000001},"max":{"1min":8.9139999999999997,"5min":15.696999999999999,"15min":15.696999999999999},"last":8.157}]},{"osd":5,"last update":"Mon Oct 9 10:55:35 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":1.508,"15min":1.532},"min":{"1min":0.374,"5min":0.31900000000000001,"15min":0.245},"max":{"1min":4.46,"5min":12.128,"15min":17.920000000000002},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.615,"5min":1.6839999999999999,"15min":1.5489999999999999},"min":{"1min":0.34999999999999998,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":10.336,"5min":16.597000000000001,"15min":17.882999999999999},"last":0.32200000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:55:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6240000000000001,"5min":1.6299999999999999,"15min":1.3480000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.247},"max":{"1min":8.8460000000000001,"5min":10.443,"15min":14.029999999999999},"last":7.2400000000000002},{"interface":"front","average":{"1min":1.4810000000000001,"5min":1.5840000000000001,"15min":1.5},"min":{"1min":0.42299999999999999,"5min":0.34799999999999998,"15min":0.29199999999999998},"max":{"1min":4.492,"5min":6.9500000000000002,"15min":12.862},"last":0.56899999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:55:35 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":1.601,"15min":1.369},"min":{"1min":0.40000000000000002,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":8.8879999999999999,"5min":8.8879999999999999,"15min":13.999000000000001},"last":7.2789999999999999},{"interface":"front","average":{"1min":1.837,"5min":1.6579999999999999,"15min":1.573},"min":{"1min":0.38,"5min":0.34000000000000002,"15min":0.307},"max":{"1min":5.6539999999999999,"5min":9.4830000000000005,"15min":12.826000000000001},"last":0.59299999999999997}]}]},{"osd":3,"up_from":95,"seq":408021893271,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2039092,"kb_used_data":1353964,"kb_used_omap":7,"kb_used_meta":684984,"kb_avail":91706060,"statfs":{"total":95995035648,"available":93907005440,"internally_reserved":0,"allocated":1386459136,"data_stored":1384742476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":701424232},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":5.9539999999999997,"5min":2.4279999999999999,"15min":1.7070000000000001},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":38.817999999999998,"5min":38.817999999999998,"15min":38.817999999999998},"last":0.98599999999999999},{"interface":"front","average":{"1min":3.5680000000000001,"5min":2.9470000000000001,"15min":1.849},"min":{"1min":0.22900000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":10.398,"5min":17.312000000000001,"15min":17.312000000000001},"last":1.006}]},{"osd":1,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":2.8980000000000001,"5min":2.3250000000000002,"15min":1.827},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":12.401,"5min":12.401,"15min":14.707000000000001},"last":0.81799999999999995},{"interface":"front","average":{"1min":4.0979999999999999,"5min":2.5899999999999999,"15min":1.77},"min":{"1min":0.22,"5min":0.22,"15min":0.193},"max":{"1min":18.526,"5min":18.526,"15min":18.526},"last":0.94799999999999995}]},{"osd":2,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":2.7799999999999998,"5min":1.9990000000000001,"15min":1.6339999999999999},"min":{"1min":0.29999999999999999,"5min":0.24199999999999999,"15min":0.23699999999999999},"max":{"1min":10.144,"5min":10.144,"15min":13.853},"last":7.3630000000000004},{"interface":"front","average":{"1min":4.6900000000000004,"5min":2.5670000000000002,"15min":1.8360000000000001},"min":{"1min":0.33700000000000002,"5min":0.248,"15min":0.248},"max":{"1min":14.081,"5min":17.268000000000001,"15min":17.268000000000001},"last":4.3490000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":3.726,"5min":2.073,"15min":1.6459999999999999},"min":{"1min":0.36099999999999999,"5min":0.311,"15min":0.28199999999999997},"max":{"1min":11.863,"5min":11.863,"15min":20.388000000000002},"last":1.0229999999999999},{"interface":"front","average":{"1min":3.1549999999999998,"5min":1.9650000000000001,"15min":1.635},"min":{"1min":0.28399999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.269,"5min":11.680999999999999,"15min":19.085999999999999},"last":0.316}]},{"osd":5,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":3.4700000000000002,"5min":2.3530000000000002,"15min":1.9039999999999999},"min":{"1min":0.32200000000000001,"5min":0.29399999999999998,"15min":0.29199999999999998},"max":{"1min":14.045,"5min":14.045,"15min":51.509999999999998},"last":0.26000000000000001},{"interface":"front","average":{"1min":1.897,"5min":2.0129999999999999,"15min":1.883},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":6.7380000000000004,"5min":11.082000000000001,"15min":55.338999999999999},"last":0.86899999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":3.2549999999999999,"5min":2.181,"15min":1.6220000000000001},"min":{"1min":0.35099999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.726,"5min":11.723000000000001,"15min":12.361000000000001},"last":0.29199999999999998},{"interface":"front","average":{"1min":2.4750000000000001,"5min":2.2040000000000002,"15min":1.782},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":11.132999999999999,"5min":11.24,"15min":13.891999999999999},"last":0.89700000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:55:03 2023","interfaces":[{"interface":"back","average":{"1min":2.2709999999999999,"5min":2.0049999999999999,"15min":1.956},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.31},"max":{"1min":9.5990000000000002,"5min":10.222,"15min":51.744},"last":0.92100000000000004},{"interface":"front","average":{"1min":4.1420000000000003,"5min":2.4910000000000001,"15min":1.986},"min":{"1min":0.42799999999999999,"5min":0.39400000000000002,"15min":0.31900000000000001},"max":{"1min":13.337,"5min":13.337,"15min":53.399000000000001},"last":1.046}]}]},{"osd":4,"up_from":30,"seq":128849019147,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1646828,"kb_used_data":1337252,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92098324,"statfs":{"total":95995035648,"available":94308683776,"internally_reserved":0,"allocated":1369346048,"data_stored":1368223743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:54 2023","interfaces":[{"interface":"back","average":{"1min":2.8079999999999998,"5min":3.8279999999999998,"15min":2.0430000000000001},"min":{"1min":0.23499999999999999,"5min":0.217,"15min":0.20999999999999999},"max":{"1min":12.332000000000001,"5min":213.88999999999999,"15min":213.88999999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":2.3330000000000002,"5min":4.3390000000000004,"15min":2.3319999999999999},"min":{"1min":0.24399999999999999,"5min":0.23300000000000001,"15min":0.217},"max":{"1min":8.9130000000000003,"5min":214.09800000000001,"15min":214.09800000000001},"last":0.44600000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:55:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":3.9249999999999998,"15min":2.0699999999999998},"min":{"1min":0.23799999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":9.0099999999999998,"5min":206.50399999999999,"15min":206.50399999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":2.1480000000000001,"5min":4.1100000000000003,"15min":2.282},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":8.4870000000000001,"5min":206.529,"15min":206.529},"last":0.29999999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:55:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5790000000000002,"5min":2.137,"15min":1.5409999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.253},"max":{"1min":8.4800000000000004,"5min":15.446999999999999,"15min":15.446999999999999},"last":2.6429999999999998},{"interface":"front","average":{"1min":3.8519999999999999,"5min":2.2850000000000001,"15min":1.746},"min":{"1min":0.30399999999999999,"5min":0.29599999999999999,"15min":0.26300000000000001},"max":{"1min":27.818000000000001,"5min":27.818000000000001,"15min":27.818000000000001},"last":6.5510000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:55:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":4.0410000000000004,"15min":2.3340000000000001},"min":{"1min":0.34599999999999997,"5min":0.29599999999999999,"15min":0.27400000000000002},"max":{"1min":6.0469999999999997,"5min":209.03700000000001,"15min":209.03700000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":2.2149999999999999,"5min":4.4720000000000004,"15min":2.544},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.5919999999999996,"5min":208.80199999999999,"15min":208.80199999999999},"last":2.6739999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:55:57 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":1.7649999999999999,"15min":1.5629999999999999},"min":{"1min":0.253,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":4.1210000000000004,"5min":19.068999999999999,"15min":24.149999999999999},"last":0.622},{"interface":"front","average":{"1min":0.53800000000000003,"5min":1.4790000000000001,"15min":1.385},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.252},"max":{"1min":1.917,"5min":15.535,"15min":20.422000000000001},"last":0.371}]},{"osd":6,"last update":"Mon Oct 9 10:55:28 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":1.524,"15min":1.22},"min":{"1min":0.28000000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.85299999999999998,"5min":15.739000000000001,"15min":15.739000000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.86399999999999999,"5min":2.0720000000000001,"15min":1.4630000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":4.2329999999999997,"5min":14.272,"15min":14.272},"last":0.434}]},{"osd":7,"last update":"Mon Oct 9 10:55:36 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":1.831,"15min":1.381},"min":{"1min":0.35499999999999998,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":4.9470000000000001,"5min":14.458,"15min":14.458},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.6120000000000001,"5min":1.7729999999999999,"15min":1.2869999999999999},"min":{"1min":0.36599999999999999,"5min":0.255,"15min":0.255},"max":{"1min":8.9589999999999996,"5min":15.592000000000001,"15min":15.592000000000001},"last":0.52300000000000002}]}]},{"osd":5,"up_from":37,"seq":158913790215,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2529428,"kb_used_data":1939660,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":91215724,"statfs":{"total":95995035648,"available":93404901376,"internally_reserved":0,"allocated":1986211840,"data_stored":1985101701,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:54:57 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":2.1880000000000002,"15min":1.355},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":16.449000000000002,"5min":16.449000000000002,"15min":16.449000000000002},"last":1.46},{"interface":"front","average":{"1min":2.2050000000000001,"5min":2.2389999999999999,"15min":1.417},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":16.489000000000001,"5min":16.489000000000001,"15min":16.489000000000001},"last":1.4890000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:55:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2869999999999999,"5min":2.2679999999999998,"15min":1.5820000000000001},"min":{"1min":0.318,"5min":0.215,"15min":0.215},"max":{"1min":8.2629999999999999,"5min":10.682,"15min":12.999000000000001},"last":3.004},{"interface":"front","average":{"1min":1.9850000000000001,"5min":1.9810000000000001,"15min":1.3779999999999999},"min":{"1min":0.29199999999999998,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":9.2699999999999996,"5min":12.109,"15min":12.109},"last":3.492}]},{"osd":2,"last update":"Mon Oct 9 10:55:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.266,"15min":1.6000000000000001},"min":{"1min":0.28000000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.257999999999999,"5min":11.368,"15min":11.393000000000001},"last":3.0579999999999998},{"interface":"front","average":{"1min":1.667,"5min":2.1859999999999999,"15min":1.4059999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.4249999999999998,"5min":15.087999999999999,"15min":15.087999999999999},"last":2.0110000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:54:57 2023","interfaces":[{"interface":"back","average":{"1min":2.4369999999999998,"5min":2.7000000000000002,"15min":1.859},"min":{"1min":0.314,"5min":0.28599999999999998,"15min":0.25700000000000001},"max":{"1min":14.006,"5min":63.508000000000003,"15min":63.508000000000003},"last":0.60399999999999998},{"interface":"front","average":{"1min":2.8839999999999999,"5min":2.831,"15min":1.917},"min":{"1min":0.40000000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":10.887,"5min":62.418999999999997,"15min":62.418999999999997},"last":6.1820000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:55:06 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":1.367,"15min":1.3799999999999999},"min":{"1min":0.26300000000000001,"5min":0.247,"15min":0.23200000000000001},"max":{"1min":4.3369999999999997,"5min":6.6120000000000001,"15min":123.241},"last":0.436},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.4930000000000001,"15min":1.573},"min":{"1min":0.28999999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.149,"5min":11.111000000000001,"15min":123.318},"last":2.702}]},{"osd":6,"last update":"Mon Oct 9 10:55:20 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":1.79,"15min":1.296},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.9049999999999994,"5min":11.098000000000001,"15min":11.098000000000001},"last":3.0299999999999998},{"interface":"front","average":{"1min":0.83899999999999997,"5min":1.7769999999999999,"15min":1.2170000000000001},"min":{"1min":0.32000000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.1779999999999999,"5min":15.16,"15min":20.256},"last":0.40600000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:55:46 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":1.6240000000000001,"15min":1.2869999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":2.911,"5min":8.468,"15min":10.739000000000001},"last":2.9689999999999999},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.3899999999999999,"15min":1.1319999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":2.2450000000000001,"5min":10.858000000000001,"15min":14.154},"last":0.48399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1159987200,"data_stored":1159987200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":904527872,"data_stored":904527872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":157880320,"data_stored":157876248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1380122624,"data_stored":1380122624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1364328448,"data_stored":1364328448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1975914496,"data_stored":1975910424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1517813760,"data_stored":1517813760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1354829824,"data_stored":1354825752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:56:06.458 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_2 pgp_num 26 2023-10-09T10:56:06.505 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:06.520 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:06.541 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:06.572 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:06.594 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:06.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:06.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:06.698 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:06.781 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:06 smithi125 ceph-mon[195507]: pgmap v758: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 14490/129243 objects degraded (11.211%); 9863/129243 objects misplaced (7.631%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:56:06.781 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:06 smithi125 ceph-mon[200197]: pgmap v758: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 14490/129243 objects degraded (11.211%); 9863/129243 objects misplaced (7.631%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:56:06.783 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:06.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:06 smithi155 ceph-mon[184854]: pgmap v758: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 14490/129243 objects degraded (11.211%); 9863/129243 objects misplaced (7.631%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T10:56:07.124 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:07.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:07.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:07.231 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:07.265 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:07.294 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:07.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:07.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:07.312 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:56:07.312 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:56:07.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:07.465 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:08.615 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:08.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:08.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:08.732 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:08.761 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:08.792 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:08.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:08.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:08.895 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:08.930 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:08 smithi155 ceph-mon[184854]: pgmap v759: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 14490/125787 objects degraded (11.519%); 9653/125787 objects misplaced (7.674%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T10:56:08.930 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:08 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 14490/125787 objects degraded (11.519%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:08.932 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:08.957 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:08.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:08.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:09.063 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:09.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:08 smithi125 ceph-mon[195507]: pgmap v759: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 14490/125787 objects degraded (11.519%); 9653/125787 objects misplaced (7.674%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T10:56:09.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:08 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 14490/125787 objects degraded (11.519%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:09.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:08 smithi125 ceph-mon[200197]: pgmap v759: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 14490/125787 objects degraded (11.519%); 9653/125787 objects misplaced (7.674%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T10:56:09.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:08 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 14490/125787 objects degraded (11.519%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:09.124 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:09.568 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:09.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:09.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:09.588 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:56:09.607 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:09.674 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:09.779 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:09.832 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:09.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:09.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:09.937 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:09.968 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:09.987 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:09.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:09.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:10.091 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:10.149 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:10.410 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:56:10.467 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:56:10.481 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:10.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:10.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:10.536 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:56:10.576 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:56:10.588 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:10.613 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:56:10.626 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:10.648 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:10.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:10.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:10.652 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:56:10.691 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:56:10.726 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:56:10.734 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:56:10.745 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:56:10.745 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:56:10.745 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:56:10.745 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:56:10.745 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:56:10.745 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:56:10.745 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:56:10.746 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:56:10.752 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:10.783 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:10.812 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:10.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:10.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:10.916 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:10.964 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:10 smithi155 ceph-mon[184854]: pgmap v760: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 14351/122094 objects degraded (11.754%); 9423/122094 objects misplaced (7.718%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T10:56:10.964 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:10 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3251798524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:10.964 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:10 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3203126054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:10.964 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:10 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:10.968 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:10.989 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:10.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:10.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:11.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:10 smithi125 ceph-mon[200197]: pgmap v760: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 14351/122094 objects degraded (11.754%); 9423/122094 objects misplaced (7.718%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T10:56:11.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:10 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3251798524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:11.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:10 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3203126054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:11.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:10 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:11.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:10 smithi125 ceph-mon[195507]: pgmap v760: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 14351/122094 objects degraded (11.754%); 9423/122094 objects misplaced (7.718%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T10:56:11.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:10 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3251798524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:11.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:10 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3203126054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:11.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:10 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:11.093 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:11.126 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:11.159 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:11.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:11.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:11.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:11.342 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:11.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:56:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:56:11] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2023-10-09T10:56:11.791 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:11.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:11.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:11.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:11.978 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:12.001 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:12.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:12.005 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:11 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:56:12.005 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:11 smithi125 ceph-mon[195507]: failed to encode map e312 with expected crc 2023-10-09T10:56:12.005 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:11 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:56:12.005 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:11 smithi125 ceph-mon[195507]: osdmap e312: 8 total, 8 up, 8 in 2023-10-09T10:56:12.005 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:11 smithi125 ceph-mon[195507]: failed to encode map e312 with expected crc 2023-10-09T10:56:12.005 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:11 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:56:12.005 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:11 smithi125 ceph-mon[200197]: failed to encode map e312 with expected crc 2023-10-09T10:56:12.005 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:11 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:56:12.006 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:11 smithi125 ceph-mon[200197]: osdmap e312: 8 total, 8 up, 8 in 2023-10-09T10:56:12.006 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:11 smithi125 ceph-mon[200197]: failed to encode map e312 with expected crc 2023-10-09T10:56:12.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:12.107 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:12.166 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:12.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:11 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:56:12.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:11 smithi155 ceph-mon[184854]: failed to encode map e312 with expected crc 2023-10-09T10:56:12.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:11 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:56:12.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:11 smithi155 ceph-mon[184854]: osdmap e312: 8 total, 8 up, 8 in 2023-10-09T10:56:12.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:11 smithi155 ceph-mon[184854]: failed to encode map e312 with expected crc 2023-10-09T10:56:12.287 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:12.414 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:12.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:12.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:12.522 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:12.584 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:12.626 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:12.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:12.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:12.730 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:12.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:12 smithi125 ceph-mon[195507]: pgmap v762: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 14351/115194 objects degraded (12.458%); 8724/115194 objects misplaced (7.573%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:56:12.798 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:12 smithi125 ceph-mon[200197]: pgmap v762: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 14351/115194 objects degraded (12.458%); 8724/115194 objects misplaced (7.573%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:56:12.801 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:13.104 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:13.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:13.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:13.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:13.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:12 smithi155 ceph-mon[184854]: pgmap v762: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 14351/115194 objects degraded (12.458%); 8724/115194 objects misplaced (7.573%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:56:13.296 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:13.398 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:13.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:13.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:13.503 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:13.566 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:13.619 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:13.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:13.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:13.688 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:56:13.688 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:56:13.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:56:13.689 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:56:13.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.689 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152582 7.4 7:222afaba:::benchmark_data_smithi125_295816_object61445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:56:13.689 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.340929+0000", 2023-10-09T10:56:13.689 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 244.34339712799999, 2023-10-09T10:56:13.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8140581390000001, 2023-10-09T10:56:13.689 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.689 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:56:13.689 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.689 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:56:13.690 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:56:13.690 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152582 2023-10-09T10:56:13.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.690 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340929+0000", 2023-10-09T10:56:13.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340929+0000", 2023-10-09T10:56:13.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340928+0000", 2023-10-09T10:56:13.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:13.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340931+0000", 2023-10-09T10:56:13.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1329999999999999e-06 2023-10-09T10:56:13.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340948+0000", 2023-10-09T10:56:13.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6178999999999999e-05 2023-10-09T10:56:13.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.340956+0000", 2023-10-09T10:56:13.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9980000000000003e-06 2023-10-09T10:56:13.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.341000+0000", 2023-10-09T10:56:13.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4508000000000001e-05 2023-10-09T10:56:13.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.341113+0000", 2023-10-09T10:56:13.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000113288 2023-10-09T10:56:13.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:56:13.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.341196+0000", 2023-10-09T10:56:13.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2163000000000006e-05 2023-10-09T10:56:13.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.344533+0000", 2023-10-09T10:56:13.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0033370129999999998 2023-10-09T10:56:13.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.346664+0000", 2023-10-09T10:56:13.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002131001 2023-10-09T10:56:13.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375125+0000", 2023-10-09T10:56:13.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.028460961999999999 2023-10-09T10:56:13.695 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375138+0000", 2023-10-09T10:56:13.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3223e-05 2023-10-09T10:56:13.695 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.696 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.696 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366624+0000", 2023-10-09T10:56:13.696 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99148582100000004 2023-10-09T10:56:13.696 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.696 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.696 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.696 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366632+0000", 2023-10-09T10:56:13.696 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5620000000000004e-06 2023-10-09T10:56:13.696 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.696 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.696 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.696 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.154916+0000", 2023-10-09T10:56:13.697 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.788283748 2023-10-09T10:56:13.697 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.697 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.697 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.697 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.154987+0000", 2023-10-09T10:56:13.697 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1076999999999997e-05 2023-10-09T10:56:13.697 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.697 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.697 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.697 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.697 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.698 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152588 7.9 7:96c4bba8:::benchmark_data_smithi125_295816_object61451:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:56:13.698 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.348147+0000", 2023-10-09T10:56:13.698 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 244.336179254, 2023-10-09T10:56:13.698 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8297705579999999, 2023-10-09T10:56:13.698 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.698 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:56:13.698 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.698 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:56:13.698 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:56:13.698 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152588 2023-10-09T10:56:13.698 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.698 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.699 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.699 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.699 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348147+0000", 2023-10-09T10:56:13.699 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.699 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.699 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.699 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.699 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348147+0000", 2023-10-09T10:56:13.699 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.699 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.699 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.699 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348146+0000", 2023-10-09T10:56:13.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:56:13.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348151+0000", 2023-10-09T10:56:13.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1150000000000002e-06 2023-10-09T10:56:13.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348158+0000", 2023-10-09T10:56:13.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6850000000000001e-06 2023-10-09T10:56:13.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.701 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.701 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348163+0000", 2023-10-09T10:56:13.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5979999999999999e-06 2023-10-09T10:56:13.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.701 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.701 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348227+0000", 2023-10-09T10:56:13.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3596000000000006e-05 2023-10-09T10:56:13.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.702 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348357+0000", 2023-10-09T10:56:13.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012999100000000001 2023-10-09T10:56:13.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:56:13.702 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.348485+0000", 2023-10-09T10:56:13.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012842499999999999 2023-10-09T10:56:13.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350944+0000", 2023-10-09T10:56:13.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024594769999999998 2023-10-09T10:56:13.703 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.353449+0000", 2023-10-09T10:56:13.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025044780000000001 2023-10-09T10:56:13.703 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.376073+0000", 2023-10-09T10:56:13.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.022624136 2023-10-09T10:56:13.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.376084+0000", 2023-10-09T10:56:13.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0803e-05 2023-10-09T10:56:13.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.364523+0000", 2023-10-09T10:56:13.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98843897800000002 2023-10-09T10:56:13.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.364533+0000", 2023-10-09T10:56:13.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0458999999999999e-05 2023-10-09T10:56:13.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.177846+0000", 2023-10-09T10:56:13.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.81331285 2023-10-09T10:56:13.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.177917+0000", 2023-10-09T10:56:13.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1117000000000004e-05 2023-10-09T10:56:13.706 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.706 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.706 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.706 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152590 7.4 7:284d5455:::benchmark_data_smithi125_295816_object61453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:56:13.706 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.350331+0000", 2023-10-09T10:56:13.706 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 244.333995311, 2023-10-09T10:56:13.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.805048094, 2023-10-09T10:56:13.706 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.706 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:56:13.707 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.707 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:56:13.707 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:56:13.707 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152590 2023-10-09T10:56:13.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.707 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.707 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.707 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350331+0000", 2023-10-09T10:56:13.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.708 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350331+0000", 2023-10-09T10:56:13.708 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.708 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.708 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350329+0000", 2023-10-09T10:56:13.708 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:13.708 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.709 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350335+0000", 2023-10-09T10:56:13.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4639999999999997e-06 2023-10-09T10:56:13.709 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.709 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.709 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.709 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350352+0000", 2023-10-09T10:56:13.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7269e-05 2023-10-09T10:56:13.709 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.709 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.709 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.709 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350361+0000", 2023-10-09T10:56:13.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.388e-06 2023-10-09T10:56:13.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.710 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.710 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350415+0000", 2023-10-09T10:56:13.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4784000000000003e-05 2023-10-09T10:56:13.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.710 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.710 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350517+0000", 2023-10-09T10:56:13.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000101724 2023-10-09T10:56:13.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:56:13.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.350653+0000", 2023-10-09T10:56:13.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013574099999999999 2023-10-09T10:56:13.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.353798+0000", 2023-10-09T10:56:13.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003145002 2023-10-09T10:56:13.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360490+0000", 2023-10-09T10:56:13.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066923240000000004 2023-10-09T10:56:13.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375171+0000", 2023-10-09T10:56:13.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014681339999999999 2023-10-09T10:56:13.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375182+0000", 2023-10-09T10:56:13.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0129000000000001e-05 2023-10-09T10:56:13.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366657+0000", 2023-10-09T10:56:13.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99147512100000001 2023-10-09T10:56:13.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366664+0000", 2023-10-09T10:56:13.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9369999999999998e-06 2023-10-09T10:56:13.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155313+0000", 2023-10-09T10:56:13.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.788649556 2023-10-09T10:56:13.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155379+0000", 2023-10-09T10:56:13.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5518000000000005e-05 2023-10-09T10:56:13.714 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.714 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.714 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.715 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152591 7.9 7:9f5cec69:::benchmark_data_smithi125_295816_object61454:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:56:13.715 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.354683+0000", 2023-10-09T10:56:13.715 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 244.32964270100001, 2023-10-09T10:56:13.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8235682930000001, 2023-10-09T10:56:13.715 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.715 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:56:13.715 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.715 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:56:13.715 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:56:13.715 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152591 2023-10-09T10:56:13.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.716 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354683+0000", 2023-10-09T10:56:13.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354683+0000", 2023-10-09T10:56:13.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354682+0000", 2023-10-09T10:56:13.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:13.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354688+0000", 2023-10-09T10:56:13.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7810000000000002e-06 2023-10-09T10:56:13.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354699+0000", 2023-10-09T10:56:13.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1556000000000001e-05 2023-10-09T10:56:13.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.718 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.718 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354707+0000", 2023-10-09T10:56:13.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3799999999999996e-06 2023-10-09T10:56:13.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.718 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.718 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354752+0000", 2023-10-09T10:56:13.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5756e-05 2023-10-09T10:56:13.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354885+0000", 2023-10-09T10:56:13.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000132397 2023-10-09T10:56:13.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:56:13.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.355009+0000", 2023-10-09T10:56:13.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000123714 2023-10-09T10:56:13.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357282+0000", 2023-10-09T10:56:13.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022739679999999999 2023-10-09T10:56:13.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.376279+0000", 2023-10-09T10:56:13.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018997012000000001 2023-10-09T10:56:13.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.376289+0000", 2023-10-09T10:56:13.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.781e-06 2023-10-09T10:56:13.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.364560+0000", 2023-10-09T10:56:13.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98827064099999995 2023-10-09T10:56:13.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365610+0000", 2023-10-09T10:56:13.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001049822 2023-10-09T10:56:13.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.178188+0000", 2023-10-09T10:56:13.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8125778029999999 2023-10-09T10:56:13.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.178252+0000", 2023-10-09T10:56:13.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4000999999999999e-05 2023-10-09T10:56:13.722 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.722 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.722 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.723 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152593 7.4 7:21dbc744:::benchmark_data_smithi125_295816_object61456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:56:13.723 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.354718+0000", 2023-10-09T10:56:13.723 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 244.329607469, 2023-10-09T10:56:13.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8007503599999999, 2023-10-09T10:56:13.723 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.723 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:56:13.723 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.723 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:56:13.723 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:56:13.724 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152593 2023-10-09T10:56:13.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.724 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354718+0000", 2023-10-09T10:56:13.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354718+0000", 2023-10-09T10:56:13.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354718+0000", 2023-10-09T10:56:13.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:13.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354721+0000", 2023-10-09T10:56:13.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8200000000000001e-06 2023-10-09T10:56:13.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354728+0000", 2023-10-09T10:56:13.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7309999999999996e-06 2023-10-09T10:56:13.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354734+0000", 2023-10-09T10:56:13.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1009999999999997e-06 2023-10-09T10:56:13.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.727 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354780+0000", 2023-10-09T10:56:13.727 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6464000000000001e-05 2023-10-09T10:56:13.727 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.727 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.727 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.727 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354913+0000", 2023-10-09T10:56:13.727 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000132569 2023-10-09T10:56:13.727 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.727 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.727 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:56:13.727 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.355043+0000", 2023-10-09T10:56:13.727 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013061799999999999 2023-10-09T10:56:13.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.359660+0000", 2023-10-09T10:56:13.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046166929999999998 2023-10-09T10:56:13.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375223+0000", 2023-10-09T10:56:13.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015563420999999999 2023-10-09T10:56:13.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375233+0000", 2023-10-09T10:56:13.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7999999999999993e-06 2023-10-09T10:56:13.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366685+0000", 2023-10-09T10:56:13.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99145126699999997 2023-10-09T10:56:13.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366692+0000", 2023-10-09T10:56:13.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.498e-06 2023-10-09T10:56:13.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155410+0000", 2023-10-09T10:56:13.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.788717651 2023-10-09T10:56:13.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155469+0000", 2023-10-09T10:56:13.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9157000000000003e-05 2023-10-09T10:56:13.730 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.731 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.731 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.731 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152597 7.4 7:23416840:::benchmark_data_smithi125_295816_object61460:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:56:13.731 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.357109+0000", 2023-10-09T10:56:13.731 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 244.32721729299999, 2023-10-09T10:56:13.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7984508359999998, 2023-10-09T10:56:13.731 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.731 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:56:13.731 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.731 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:56:13.732 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:56:13.732 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152597 2023-10-09T10:56:13.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.732 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357109+0000", 2023-10-09T10:56:13.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357109+0000", 2023-10-09T10:56:13.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357108+0000", 2023-10-09T10:56:13.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:56:13.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357112+0000", 2023-10-09T10:56:13.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.673e-06 2023-10-09T10:56:13.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357123+0000", 2023-10-09T10:56:13.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0567999999999999e-05 2023-10-09T10:56:13.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357130+0000", 2023-10-09T10:56:13.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4599999999999997e-06 2023-10-09T10:56:13.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357336+0000", 2023-10-09T10:56:13.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020526499999999999 2023-10-09T10:56:13.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357462+0000", 2023-10-09T10:56:13.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012642400000000001 2023-10-09T10:56:13.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:56:13.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357581+0000", 2023-10-09T10:56:13.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118647 2023-10-09T10:56:13.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.359832+0000", 2023-10-09T10:56:13.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002251121 2023-10-09T10:56:13.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375269+0000", 2023-10-09T10:56:13.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015436699999999999 2023-10-09T10:56:13.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.375278+0000", 2023-10-09T10:56:13.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8199999999999992e-06 2023-10-09T10:56:13.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366716+0000", 2023-10-09T10:56:13.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99143730600000002 2023-10-09T10:56:13.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366725+0000", 2023-10-09T10:56:13.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8759999999999994e-06 2023-10-09T10:56:13.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155501+0000", 2023-10-09T10:56:13.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7887762249999999 2023-10-09T10:56:13.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.155559+0000", 2023-10-09T10:56:13.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8628000000000002e-05 2023-10-09T10:56:13.738 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.738 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.739 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.739 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152599 7.4 7:26957a0e:::benchmark_data_smithi125_295816_object61462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:56:13.739 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.360294+0000", 2023-10-09T10:56:13.739 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 244.32403200900001, 2023-10-09T10:56:13.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7970287539999998, 2023-10-09T10:56:13.739 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.739 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:56:13.739 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.739 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:56:13.739 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:56:13.740 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152599 2023-10-09T10:56:13.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.740 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360294+0000", 2023-10-09T10:56:13.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360294+0000", 2023-10-09T10:56:13.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360292+0000", 2023-10-09T10:56:13.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:56:13.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360297+0000", 2023-10-09T10:56:13.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0170000000000004e-06 2023-10-09T10:56:13.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360318+0000", 2023-10-09T10:56:13.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0460999999999999e-05 2023-10-09T10:56:13.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360325+0000", 2023-10-09T10:56:13.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2119999999999997e-06 2023-10-09T10:56:13.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360517+0000", 2023-10-09T10:56:13.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000192487 2023-10-09T10:56:13.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360676+0000", 2023-10-09T10:56:13.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015863499999999999 2023-10-09T10:56:13.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:56:13.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.360826+0000", 2023-10-09T10:56:13.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015037199999999999 2023-10-09T10:56:13.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.378926+0000", 2023-10-09T10:56:13.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018099725000000001 2023-10-09T10:56:13.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.378938+0000", 2023-10-09T10:56:13.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2191000000000001e-05 2023-10-09T10:56:13.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366754+0000", 2023-10-09T10:56:13.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98781607800000004 2023-10-09T10:56:13.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.366763+0000", 2023-10-09T10:56:13.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6179999999999995e-06 2023-10-09T10:56:13.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.157249+0000", 2023-10-09T10:56:13.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.790486386 2023-10-09T10:56:13.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.157323+0000", 2023-10-09T10:56:13.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3258999999999998e-05 2023-10-09T10:56:13.746 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.746 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.746 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.746 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152606 7.9 7:912bbd31:::benchmark_data_smithi125_295816_object61469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T10:56:13.747 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.394803+0000", 2023-10-09T10:56:13.747 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 244.289522726, 2023-10-09T10:56:13.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7960543420000001, 2023-10-09T10:56:13.747 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.747 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:13.747 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.747 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:56:13.747 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:56:13.747 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152606 2023-10-09T10:56:13.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.747 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394803+0000", 2023-10-09T10:56:13.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394803+0000", 2023-10-09T10:56:13.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394802+0000", 2023-10-09T10:56:13.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:13.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394808+0000", 2023-10-09T10:56:13.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5220000000000004e-06 2023-10-09T10:56:13.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394821+0000", 2023-10-09T10:56:13.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2325e-05 2023-10-09T10:56:13.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394829+0000", 2023-10-09T10:56:13.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3820000000000004e-06 2023-10-09T10:56:13.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394972+0000", 2023-10-09T10:56:13.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000142788 2023-10-09T10:56:13.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.394982+0000", 2023-10-09T10:56:13.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0336000000000001e-05 2023-10-09T10:56:13.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365844+0000", 2023-10-09T10:56:13.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97086172500000001 2023-10-09T10:56:13.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365853+0000", 2023-10-09T10:56:13.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5230000000000001e-06 2023-10-09T10:56:13.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.178273+0000", 2023-10-09T10:56:13.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8124192300000002 2023-10-09T10:56:13.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.178376+0000", 2023-10-09T10:56:13.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000102841 2023-10-09T10:56:13.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T10:56:13.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.178485+0000", 2023-10-09T10:56:13.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000109516 2023-10-09T10:56:13.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.183468+0000", 2023-10-09T10:56:13.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049829920000000003 2023-10-09T10:56:13.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.186132+0000", 2023-10-09T10:56:13.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002663538 2023-10-09T10:56:13.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.186288+0000", 2023-10-09T10:56:13.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015653499999999999 2023-10-09T10:56:13.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.190512+0000", 2023-10-09T10:56:13.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042239850000000004 2023-10-09T10:56:13.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:13.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.190562+0000", 2023-10-09T10:56:13.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0169e-05 2023-10-09T10:56:13.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.190858+0000", 2023-10-09T10:56:13.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00029527200000000001 2023-10-09T10:56:13.756 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.756 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.756 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.756 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152607 7.9 7:9c2ec87f:::benchmark_data_smithi125_295816_object61470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T10:56:13.756 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.397825+0000", 2023-10-09T10:56:13.756 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 244.286500778, 2023-10-09T10:56:13.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.811002282, 2023-10-09T10:56:13.756 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.756 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:13.756 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.757 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:56:13.757 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:56:13.757 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152607 2023-10-09T10:56:13.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.757 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397825+0000", 2023-10-09T10:56:13.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397825+0000", 2023-10-09T10:56:13.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397824+0000", 2023-10-09T10:56:13.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:13.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397830+0000", 2023-10-09T10:56:13.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9199999999999998e-06 2023-10-09T10:56:13.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397844+0000", 2023-10-09T10:56:13.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3648999999999999e-05 2023-10-09T10:56:13.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397852+0000", 2023-10-09T10:56:13.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9009999999999994e-06 2023-10-09T10:56:13.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397895+0000", 2023-10-09T10:56:13.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3269999999999997e-05 2023-10-09T10:56:13.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.397907+0000", 2023-10-09T10:56:13.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1507e-05 2023-10-09T10:56:13.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365878+0000", 2023-10-09T10:56:13.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96797074800000005 2023-10-09T10:56:13.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.365886+0000", 2023-10-09T10:56:13.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0909999999999993e-06 2023-10-09T10:56:13.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.180824+0000", 2023-10-09T10:56:13.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8149384070000001 2023-10-09T10:56:13.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.180944+0000", 2023-10-09T10:56:13.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120256 2023-10-09T10:56:13.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T10:56:13.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.181120+0000", 2023-10-09T10:56:13.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017529399999999999 2023-10-09T10:56:13.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.188514+0000", 2023-10-09T10:56:13.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0073948709999999999 2023-10-09T10:56:13.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.190897+0000", 2023-10-09T10:56:13.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023828120000000002 2023-10-09T10:56:13.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.192009+0000", 2023-10-09T10:56:13.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011115719999999999 2023-10-09T10:56:13.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.208460+0000", 2023-10-09T10:56:13.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.016451364999999999 2023-10-09T10:56:13.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:13.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.208512+0000", 2023-10-09T10:56:13.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2067999999999997e-05 2023-10-09T10:56:13.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.208827+0000", 2023-10-09T10:56:13.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00031514599999999998 2023-10-09T10:56:13.765 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.765 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.765 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.765 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:83945 8.e 8:7f009390:::benchmark_data_smithi125_329074_object83944:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2023-10-09T10:56:13.765 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:17.076397+0000", 2023-10-09T10:56:13.765 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 116.607928628, 2023-10-09T10:56:13.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2623378889999999, 2023-10-09T10:56:13.766 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.766 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:56:13.766 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.766 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:13.766 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:13.766 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 83945 2023-10-09T10:56:13.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.766 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076397+0000", 2023-10-09T10:56:13.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076397+0000", 2023-10-09T10:56:13.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076396+0000", 2023-10-09T10:56:13.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:56:13.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076429+0000", 2023-10-09T10:56:13.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2443000000000001e-05 2023-10-09T10:56:13.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076441+0000", 2023-10-09T10:56:13.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2639999999999999e-05 2023-10-09T10:56:13.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076451+0000", 2023-10-09T10:56:13.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1789999999999997e-06 2023-10-09T10:56:13.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076490+0000", 2023-10-09T10:56:13.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9607000000000002e-05 2023-10-09T10:56:13.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076606+0000", 2023-10-09T10:56:13.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011632 2023-10-09T10:56:13.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,7", 2023-10-09T10:56:13.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.076713+0000", 2023-10-09T10:56:13.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000106542 2023-10-09T10:56:13.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.091575+0000", 2023-10-09T10:56:13.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014862176 2023-10-09T10:56:13.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.095602+0000", 2023-10-09T10:56:13.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040271650000000001 2023-10-09T10:56:13.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.100859+0000", 2023-10-09T10:56:13.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0052565240000000003 2023-10-09T10:56:13.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.100873+0000", 2023-10-09T10:56:13.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4364e-05 2023-10-09T10:56:13.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.869642+0000", 2023-10-09T10:56:13.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.76876907000000005 2023-10-09T10:56:13.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:13.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.869658+0000", 2023-10-09T10:56:13.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5216e-05 2023-10-09T10:56:13.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.338629+0000", 2023-10-09T10:56:13.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.468971483 2023-10-09T10:56:13.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.338735+0000", 2023-10-09T10:56:13.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000106165 2023-10-09T10:56:13.774 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.774 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.774 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.774 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:110482 8.11 8:8f3e1ed5:::benchmark_data_smithi125_329074_object13828:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-10-09T10:56:13.774 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:08.539863+0000", 2023-10-09T10:56:13.774 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 65.144462769, 2023-10-09T10:56:13.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.140803555, 2023-10-09T10:56:13.774 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.774 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:13.775 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.775 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:13.775 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:13.775 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 110482 2023-10-09T10:56:13.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.775 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.539863+0000", 2023-10-09T10:56:13.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.539863+0000", 2023-10-09T10:56:13.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.539863+0000", 2023-10-09T10:56:13.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:13.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.539866+0000", 2023-10-09T10:56:13.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3390000000000001e-06 2023-10-09T10:56:13.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.539872+0000", 2023-10-09T10:56:13.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4230000000000002e-06 2023-10-09T10:56:13.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.539878+0000", 2023-10-09T10:56:13.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2329999999999997e-06 2023-10-09T10:56:13.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.547202+0000", 2023-10-09T10:56:13.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0073243190000000001 2023-10-09T10:56:13.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:56:13.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.549113+0000", 2023-10-09T10:56:13.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001910929 2023-10-09T10:56:13.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.615615+0000", 2023-10-09T10:56:13.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.066502212000000005 2023-10-09T10:56:13.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.615727+0000", 2023-10-09T10:56:13.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011190499999999999 2023-10-09T10:56:13.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:56:13.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.616095+0000", 2023-10-09T10:56:13.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00036848300000000002 2023-10-09T10:56:13.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.619930+0000", 2023-10-09T10:56:13.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038347199999999998 2023-10-09T10:56:13.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.622951+0000", 2023-10-09T10:56:13.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030211840000000001 2023-10-09T10:56:13.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.678647+0000", 2023-10-09T10:56:13.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.055695785999999997 2023-10-09T10:56:13.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:13.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.678693+0000", 2023-10-09T10:56:13.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5609000000000003e-05 2023-10-09T10:56:13.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.782 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.680667+0000", 2023-10-09T10:56:13.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001974005 2023-10-09T10:56:13.782 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.782 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.782 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.782 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:110507 8.11 8:8e8694c4:::benchmark_data_smithi125_329074_object13853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-10-09T10:56:13.782 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:08.579090+0000", 2023-10-09T10:56:13.782 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 65.105236153999996, 2023-10-09T10:56:13.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.14939474899999999, 2023-10-09T10:56:13.783 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.783 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:13.783 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.783 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:13.783 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:13.783 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 110507 2023-10-09T10:56:13.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.783 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.579090+0000", 2023-10-09T10:56:13.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.579090+0000", 2023-10-09T10:56:13.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.579088+0000", 2023-10-09T10:56:13.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:13.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.579093+0000", 2023-10-09T10:56:13.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5700000000000003e-06 2023-10-09T10:56:13.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.579102+0000", 2023-10-09T10:56:13.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1810000000000007e-06 2023-10-09T10:56:13.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.579109+0000", 2023-10-09T10:56:13.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9129999999999997e-06 2023-10-09T10:56:13.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.617432+0000", 2023-10-09T10:56:13.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.038322380000000003 2023-10-09T10:56:13.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:56:13.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.618033+0000", 2023-10-09T10:56:13.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00060150799999999995 2023-10-09T10:56:13.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.686851+0000", 2023-10-09T10:56:13.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.068818162000000002 2023-10-09T10:56:13.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.686945+0000", 2023-10-09T10:56:13.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3445999999999999e-05 2023-10-09T10:56:13.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:56:13.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.687298+0000", 2023-10-09T10:56:13.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035372399999999998 2023-10-09T10:56:13.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.697969+0000", 2023-10-09T10:56:13.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010670962000000001 2023-10-09T10:56:13.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.705650+0000", 2023-10-09T10:56:13.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076810460000000004 2023-10-09T10:56:13.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.726320+0000", 2023-10-09T10:56:13.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.020669870999999999 2023-10-09T10:56:13.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:13.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.726373+0000", 2023-10-09T10:56:13.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2510999999999999e-05 2023-10-09T10:56:13.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.728485+0000", 2023-10-09T10:56:13.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021117890000000002 2023-10-09T10:56:13.790 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.790 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.790 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.790 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:110517 8.1 8:8233a494:::benchmark_data_smithi125_329074_object13863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-10-09T10:56:13.791 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:08.591319+0000", 2023-10-09T10:56:13.791 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 65.093006774000003, 2023-10-09T10:56:13.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12282760400000001, 2023-10-09T10:56:13.791 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.791 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:13.791 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.791 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:13.791 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:13.791 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 110517 2023-10-09T10:56:13.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.793 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.591319+0000", 2023-10-09T10:56:13.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.591319+0000", 2023-10-09T10:56:13.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.591317+0000", 2023-10-09T10:56:13.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T10:56:13.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.591325+0000", 2023-10-09T10:56:13.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.191e-06 2023-10-09T10:56:13.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.591337+0000", 2023-10-09T10:56:13.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1977999999999999e-05 2023-10-09T10:56:13.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.591344+0000", 2023-10-09T10:56:13.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2289999999999997e-06 2023-10-09T10:56:13.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.617414+0000", 2023-10-09T10:56:13.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.026069994999999999 2023-10-09T10:56:13.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:56:13.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.619240+0000", 2023-10-09T10:56:13.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018255700000000001 2023-10-09T10:56:13.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.686863+0000", 2023-10-09T10:56:13.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.067623529000000002 2023-10-09T10:56:13.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.686949+0000", 2023-10-09T10:56:13.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5978999999999998e-05 2023-10-09T10:56:13.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:56:13.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.687237+0000", 2023-10-09T10:56:13.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000288034 2023-10-09T10:56:13.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.798 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.798 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.798 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.697965+0000", 2023-10-09T10:56:13.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010727360999999999 2023-10-09T10:56:13.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.798 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.798 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.798 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.705608+0000", 2023-10-09T10:56:13.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076431260000000001 2023-10-09T10:56:13.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.798 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.798 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.712230+0000", 2023-10-09T10:56:13.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066218550000000003 2023-10-09T10:56:13.799 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:13.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.712292+0000", 2023-10-09T10:56:13.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2544000000000003e-05 2023-10-09T10:56:13.799 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.714147+0000", 2023-10-09T10:56:13.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001854566 2023-10-09T10:56:13.800 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.800 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.800 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.800 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:138367 8.e 8:755661a0:::benchmark_data_smithi125_329074_object41713:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T10:56:13.800 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:47.881139+0000", 2023-10-09T10:56:13.800 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 25.803186942, 2023-10-09T10:56:13.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12900935399999999, 2023-10-09T10:56:13.800 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.800 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:13.801 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.801 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:13.801 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:13.801 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 138367 2023-10-09T10:56:13.801 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.801 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.801 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.881139+0000", 2023-10-09T10:56:13.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.801 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.801 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.802 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.881139+0000", 2023-10-09T10:56:13.802 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.802 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.802 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.802 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.802 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.881138+0000", 2023-10-09T10:56:13.802 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:56:13.802 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.802 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.802 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.802 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.881144+0000", 2023-10-09T10:56:13.803 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.891e-06 2023-10-09T10:56:13.803 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.803 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.803 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.803 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.881153+0000", 2023-10-09T10:56:13.803 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1039999999999996e-06 2023-10-09T10:56:13.803 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.803 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.803 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.803 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.881158+0000", 2023-10-09T10:56:13.803 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3859999999999998e-06 2023-10-09T10:56:13.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.804 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.897497+0000", 2023-10-09T10:56:13.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.016338350000000001 2023-10-09T10:56:13.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.804 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for rw locks", 2023-10-09T10:56:13.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.897661+0000", 2023-10-09T10:56:13.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0001639 2023-10-09T10:56:13.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.966616+0000", 2023-10-09T10:56:13.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.068955214000000001 2023-10-09T10:56:13.805 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.805 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.966708+0000", 2023-10-09T10:56:13.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2087000000000001e-05 2023-10-09T10:56:13.805 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.805 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:56:13.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.966847+0000", 2023-10-09T10:56:13.806 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013905300000000001 2023-10-09T10:56:13.806 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.806 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.806 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.806 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.978459+0000", 2023-10-09T10:56:13.806 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011611628000000001 2023-10-09T10:56:13.806 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.806 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.806 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.806 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.991056+0000", 2023-10-09T10:56:13.806 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012597489 2023-10-09T10:56:13.806 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.807 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.807 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.807 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.007354+0000", 2023-10-09T10:56:13.807 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.016298284999999999 2023-10-09T10:56:13.807 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.807 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.807 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:13.807 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.007411+0000", 2023-10-09T10:56:13.807 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6153000000000003e-05 2023-10-09T10:56:13.807 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.807 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.808 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.808 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.010148+0000", 2023-10-09T10:56:13.808 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027377920000000002 2023-10-09T10:56:13.808 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.808 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.808 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.808 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.808 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.808 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:138747 8.e 8:70980004:::benchmark_data_smithi125_329074_object42093:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T10:56:13.808 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:48.454453+0000", 2023-10-09T10:56:13.808 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 25.229872951000001, 2023-10-09T10:56:13.808 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12374226000000001, 2023-10-09T10:56:13.809 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.809 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:13.809 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.809 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:13.809 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:13.809 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 138747 2023-10-09T10:56:13.809 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.809 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.809 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.809 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.809 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.454453+0000", 2023-10-09T10:56:13.809 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.810 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.810 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.810 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.454453+0000", 2023-10-09T10:56:13.810 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.810 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.810 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.810 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.454452+0000", 2023-10-09T10:56:13.810 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:13.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.454458+0000", 2023-10-09T10:56:13.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9970000000000004e-06 2023-10-09T10:56:13.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.454469+0000", 2023-10-09T10:56:13.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1769e-05 2023-10-09T10:56:13.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.454477+0000", 2023-10-09T10:56:13.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8299999999999996e-06 2023-10-09T10:56:13.812 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.812 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.466988+0000", 2023-10-09T10:56:13.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012510759999999999 2023-10-09T10:56:13.812 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.812 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for rw locks", 2023-10-09T10:56:13.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.467108+0000", 2023-10-09T10:56:13.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011983 2023-10-09T10:56:13.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.813 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.813 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.813 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.481672+0000", 2023-10-09T10:56:13.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014563904000000001 2023-10-09T10:56:13.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.813 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.813 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.813 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.481770+0000", 2023-10-09T10:56:13.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8465e-05 2023-10-09T10:56:13.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.814 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.814 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,4", 2023-10-09T10:56:13.814 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.481914+0000", 2023-10-09T10:56:13.814 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000143986 2023-10-09T10:56:13.814 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.814 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.814 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.814 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.574314+0000", 2023-10-09T10:56:13.814 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.092399853000000004 2023-10-09T10:56:13.814 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.814 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.814 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.815 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.574422+0000", 2023-10-09T10:56:13.815 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010758400000000001 2023-10-09T10:56:13.815 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.815 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.815 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.815 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.575123+0000", 2023-10-09T10:56:13.815 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00070190300000000003 2023-10-09T10:56:13.815 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.815 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.815 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:13.815 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.575169+0000", 2023-10-09T10:56:13.816 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5148000000000002e-05 2023-10-09T10:56:13.816 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.816 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.816 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.816 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:48.578195+0000", 2023-10-09T10:56:13.816 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003026599 2023-10-09T10:56:13.816 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.816 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.816 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.816 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.816 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.816 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:139460 8.11 8:8f02d346:::benchmark_data_smithi125_329074_object42806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T10:56:13.817 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:49.526952+0000", 2023-10-09T10:56:13.817 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 24.157374228999998, 2023-10-09T10:56:13.817 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.158705444, 2023-10-09T10:56:13.817 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.817 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:13.817 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.817 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:13.817 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:13.817 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 139460 2023-10-09T10:56:13.817 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.817 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.817 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.818 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.818 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.526952+0000", 2023-10-09T10:56:13.818 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.818 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.818 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.818 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.818 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.526952+0000", 2023-10-09T10:56:13.818 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.818 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.818 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.818 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.819 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.526950+0000", 2023-10-09T10:56:13.819 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:13.819 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.819 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.819 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.819 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.526958+0000", 2023-10-09T10:56:13.819 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7179999999999996e-06 2023-10-09T10:56:13.819 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.819 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.819 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.820 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:13.821 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.526975+0000", 2023-10-09T10:56:13.821 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6742e-05 2023-10-09T10:56:13.821 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.821 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.821 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.821 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.526983+0000", 2023-10-09T10:56:13.821 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9639999999999992e-06 2023-10-09T10:56:13.822 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.822 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.822 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.822 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.537030+0000", 2023-10-09T10:56:13.822 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010046500999999999 2023-10-09T10:56:13.822 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.822 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.822 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:56:13.822 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.537634+0000", 2023-10-09T10:56:13.822 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00060404299999999996 2023-10-09T10:56:13.822 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.823 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.823 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.823 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.573650+0000", 2023-10-09T10:56:13.823 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.036015903000000002 2023-10-09T10:56:13.823 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.823 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.823 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.823 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.573738+0000", 2023-10-09T10:56:13.823 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7602000000000006e-05 2023-10-09T10:56:13.823 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.823 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.824 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:56:13.824 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.574142+0000", 2023-10-09T10:56:13.824 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00040417900000000001 2023-10-09T10:56:13.824 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.824 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.824 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.824 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.681427+0000", 2023-10-09T10:56:13.824 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.107285591 2023-10-09T10:56:13.824 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.824 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.824 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.824 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.683967+0000", 2023-10-09T10:56:13.825 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002539253 2023-10-09T10:56:13.825 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.825 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.825 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.825 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.684248+0000", 2023-10-09T10:56:13.825 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00028192199999999999 2023-10-09T10:56:13.825 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.825 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.825 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:13.825 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.684285+0000", 2023-10-09T10:56:13.825 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6245999999999998e-05 2023-10-09T10:56:13.826 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.826 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.826 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.826 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.685657+0000", 2023-10-09T10:56:13.826 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00137243 2023-10-09T10:56:13.826 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.826 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.826 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.826 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.826 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.826 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:139472 8.11 8:8d7b4db7:::benchmark_data_smithi125_329074_object42818:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T10:56:13.827 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:49.546481+0000", 2023-10-09T10:56:13.827 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 24.137845338999998, 2023-10-09T10:56:13.827 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.140469225, 2023-10-09T10:56:13.827 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.827 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:13.827 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.827 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:13.827 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:13.827 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 139472 2023-10-09T10:56:13.827 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.827 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.828 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.828 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.828 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.546481+0000", 2023-10-09T10:56:13.828 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.828 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.828 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.828 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.828 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.546481+0000", 2023-10-09T10:56:13.828 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.828 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.828 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.829 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.829 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.546479+0000", 2023-10-09T10:56:13.829 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:56:13.829 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.829 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.829 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.829 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.546488+0000", 2023-10-09T10:56:13.829 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4739999999999994e-06 2023-10-09T10:56:13.829 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.829 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.829 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.830 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.546502+0000", 2023-10-09T10:56:13.830 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3540000000000001e-05 2023-10-09T10:56:13.830 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.830 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.830 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.830 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.546509+0000", 2023-10-09T10:56:13.830 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5970000000000003e-06 2023-10-09T10:56:13.830 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.830 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.830 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.830 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.549464+0000", 2023-10-09T10:56:13.830 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029542959999999999 2023-10-09T10:56:13.831 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.831 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.831 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:56:13.831 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.550130+0000", 2023-10-09T10:56:13.831 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00066620999999999998 2023-10-09T10:56:13.831 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.831 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.831 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.831 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.578653+0000", 2023-10-09T10:56:13.831 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.028522868 2023-10-09T10:56:13.831 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.831 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.831 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.832 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.578723+0000", 2023-10-09T10:56:13.832 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9855000000000001e-05 2023-10-09T10:56:13.832 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.832 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.832 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:56:13.832 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.579134+0000", 2023-10-09T10:56:13.832 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00041102899999999998 2023-10-09T10:56:13.832 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.832 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.832 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.832 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.684005+0000", 2023-10-09T10:56:13.832 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.104871704 2023-10-09T10:56:13.833 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.833 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.833 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.833 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.684074+0000", 2023-10-09T10:56:13.833 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8450999999999999e-05 2023-10-09T10:56:13.833 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.833 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.833 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.833 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.685666+0000", 2023-10-09T10:56:13.833 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001592273 2023-10-09T10:56:13.833 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.833 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.834 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:13.834 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.685706+0000", 2023-10-09T10:56:13.834 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9898999999999999e-05 2023-10-09T10:56:13.834 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.834 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.834 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.834 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.686950+0000", 2023-10-09T10:56:13.834 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001243781 2023-10-09T10:56:13.834 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.834 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.834 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.835 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.835 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.835 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:139473 8.11 8:889594f5:::benchmark_data_smithi125_329074_object42819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T10:56:13.835 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:49.546516+0000", 2023-10-09T10:56:13.835 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 24.137809671999999, 2023-10-09T10:56:13.835 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.14179746800000001, 2023-10-09T10:56:13.835 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.835 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:13.835 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.835 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:13.835 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:13.835 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 139473 2023-10-09T10:56:13.836 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.836 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.836 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.836 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.546516+0000", 2023-10-09T10:56:13.836 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.836 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.836 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.836 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.546516+0000", 2023-10-09T10:56:13.836 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.837 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.837 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.837 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.837 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.546516+0000", 2023-10-09T10:56:13.837 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:13.837 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.837 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.837 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.837 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.546519+0000", 2023-10-09T10:56:13.837 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8289999999999999e-06 2023-10-09T10:56:13.837 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.837 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.838 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.546524+0000", 2023-10-09T10:56:13.838 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1490000000000002e-06 2023-10-09T10:56:13.838 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.838 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.838 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.546529+0000", 2023-10-09T10:56:13.838 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8559999999999998e-06 2023-10-09T10:56:13.838 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.838 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.838 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.839 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.550175+0000", 2023-10-09T10:56:13.839 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036463120000000001 2023-10-09T10:56:13.839 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.839 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.839 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:56:13.839 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.550747+0000", 2023-10-09T10:56:13.839 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00057174799999999996 2023-10-09T10:56:13.839 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.839 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.839 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.839 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.583606+0000", 2023-10-09T10:56:13.839 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.032858763999999999 2023-10-09T10:56:13.840 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.840 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.840 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.840 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.583685+0000", 2023-10-09T10:56:13.840 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9039e-05 2023-10-09T10:56:13.840 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.840 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.840 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:56:13.840 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.584062+0000", 2023-10-09T10:56:13.840 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00037773499999999997 2023-10-09T10:56:13.840 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.840 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.841 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.841 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.684109+0000", 2023-10-09T10:56:13.841 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.100046382 2023-10-09T10:56:13.841 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.841 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.841 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.841 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.684152+0000", 2023-10-09T10:56:13.841 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3223999999999998e-05 2023-10-09T10:56:13.841 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.841 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.841 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.842 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.686959+0000", 2023-10-09T10:56:13.842 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028074609999999998 2023-10-09T10:56:13.842 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.842 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:13.842 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.687013+0000", 2023-10-09T10:56:13.842 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4039000000000002e-05 2023-10-09T10:56:13.842 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.842 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.842 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.688314+0000", 2023-10-09T10:56:13.843 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013003470000000001 2023-10-09T10:56:13.843 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.843 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.843 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.843 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.843 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.843 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:139494 8.11 8:8aca9d0a:::benchmark_data_smithi125_329074_object42840:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T10:56:13.843 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:49.578333+0000", 2023-10-09T10:56:13.843 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 24.105992529000002, 2023-10-09T10:56:13.843 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.131936894, 2023-10-09T10:56:13.843 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.843 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:13.844 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.844 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:13.844 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:13.844 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 139494 2023-10-09T10:56:13.844 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.844 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.844 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.844 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.844 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.578333+0000", 2023-10-09T10:56:13.844 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.844 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.845 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.845 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.845 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.578333+0000", 2023-10-09T10:56:13.845 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.845 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.845 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.845 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.845 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.578332+0000", 2023-10-09T10:56:13.845 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:13.845 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.845 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.845 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.846 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.578337+0000", 2023-10-09T10:56:13.846 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6920000000000001e-06 2023-10-09T10:56:13.846 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.846 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.846 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.846 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.578344+0000", 2023-10-09T10:56:13.846 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9719999999999992e-06 2023-10-09T10:56:13.846 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.846 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.846 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.846 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.578349+0000", 2023-10-09T10:56:13.847 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9880000000000004e-06 2023-10-09T10:56:13.847 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.847 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.847 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.847 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.681467+0000", 2023-10-09T10:56:13.847 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.103117059 2023-10-09T10:56:13.847 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.847 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.847 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:56:13.847 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.683155+0000", 2023-10-09T10:56:13.847 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001688286 2023-10-09T10:56:13.847 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.848 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.848 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.848 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.696291+0000", 2023-10-09T10:56:13.848 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013135859999999999 2023-10-09T10:56:13.848 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.848 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.848 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.848 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.696350+0000", 2023-10-09T10:56:13.848 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9243000000000002e-05 2023-10-09T10:56:13.848 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.848 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.848 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:56:13.849 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.696742+0000", 2023-10-09T10:56:13.849 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000392397 2023-10-09T10:56:13.849 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.849 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.849 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.849 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.700999+0000", 2023-10-09T10:56:13.849 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042562279999999999 2023-10-09T10:56:13.849 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.849 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.849 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.849 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.704614+0000", 2023-10-09T10:56:13.850 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036151220000000001 2023-10-09T10:56:13.850 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.850 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.850 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.850 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.708747+0000", 2023-10-09T10:56:13.850 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041334579999999996 2023-10-09T10:56:13.850 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.850 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.850 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:13.850 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.708806+0000", 2023-10-09T10:56:13.850 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8588999999999998e-05 2023-10-09T10:56:13.851 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.851 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.851 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.851 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:49.710270+0000", 2023-10-09T10:56:13.851 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014645719999999999 2023-10-09T10:56:13.851 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.851 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.851 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.851 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.851 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.851 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:143309 8.1 8:847684b2:::benchmark_data_smithi125_329074_object46655:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T10:56:13.852 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:54.248280+0000", 2023-10-09T10:56:13.852 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 19.436045543999999, 2023-10-09T10:56:13.852 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12605065700000001, 2023-10-09T10:56:13.852 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:13.852 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:13.852 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:13.852 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:13.852 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:13.852 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 143309 2023-10-09T10:56:13.852 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.852 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:13.852 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.853 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:13.853 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.248280+0000", 2023-10-09T10:56:13.853 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.853 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.853 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.853 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:13.853 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.248280+0000", 2023-10-09T10:56:13.853 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:13.853 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.853 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.853 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:13.853 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.248279+0000", 2023-10-09T10:56:13.853 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:56:13.854 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.854 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.854 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:13.854 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.248284+0000", 2023-10-09T10:56:13.854 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8770000000000001e-06 2023-10-09T10:56:13.854 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.854 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.854 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:13.854 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.248293+0000", 2023-10-09T10:56:13.854 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7569999999999993e-06 2023-10-09T10:56:13.854 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.855 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.855 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:13.855 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.248299+0000", 2023-10-09T10:56:13.855 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8939999999999998e-06 2023-10-09T10:56:13.855 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.855 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.855 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.855 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.248338+0000", 2023-10-09T10:56:13.855 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9076000000000003e-05 2023-10-09T10:56:13.855 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.855 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.855 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:56:13.855 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.251816+0000", 2023-10-09T10:56:13.856 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0034784410000000001 2023-10-09T10:56:13.856 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.856 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.856 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:13.856 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.262327+0000", 2023-10-09T10:56:13.856 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010510884 2023-10-09T10:56:13.856 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.856 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.856 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:13.856 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.262420+0000", 2023-10-09T10:56:13.856 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2647999999999998e-05 2023-10-09T10:56:13.856 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.856 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.857 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:56:13.857 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.262864+0000", 2023-10-09T10:56:13.857 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00044401400000000001 2023-10-09T10:56:13.857 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.857 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.857 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.857 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.269610+0000", 2023-10-09T10:56:13.857 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0067460360000000004 2023-10-09T10:56:13.857 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.857 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.857 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:13.857 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.270980+0000", 2023-10-09T10:56:13.858 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013698720000000001 2023-10-09T10:56:13.858 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.858 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.858 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:13.858 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.372797+0000", 2023-10-09T10:56:13.858 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.101816883 2023-10-09T10:56:13.858 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.858 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.858 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:13.858 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.372856+0000", 2023-10-09T10:56:13.858 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8799999999999999e-05 2023-10-09T10:56:13.859 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:13.859 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:13.859 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:13.859 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:54.374331+0000", 2023-10-09T10:56:13.859 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014754760000000001 2023-10-09T10:56:13.859 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.859 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.859 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.859 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:13.859 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:13.859 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:56:13.859 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 8 pgp_num to 26 2023-10-09T10:56:13.860 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:13.878 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:13.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:13.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:13.981 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:14.035 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:56:14.051 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:13 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3860403192' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:56:14.051 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:13 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:56:14.051 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:13 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 14351/115194 objects degraded (12.458%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:14.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:13 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3860403192' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:56:14.052 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:13 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:56:14.052 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:13 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 14351/115194 objects degraded (12.458%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:14.055 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:14.139 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:14.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:14.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:13 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3860403192' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:56:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:13 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T10:56:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:13 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 14351/115194 objects degraded (12.458%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:14.246 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:14.345 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:14.377 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:56:14.438 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:14.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:14.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:14.470 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:56:14.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:14.571 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:14.590 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:14.590 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:14.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:14.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:14.695 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:14.802 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:14.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:14.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:14.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:14.959 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:14 smithi155 ceph-mon[184854]: pgmap v763: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 14351/114504 objects degraded (12.533%); 8724/114504 objects misplaced (7.619%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:56:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:14 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T10:56:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:14 smithi155 ceph-mon[184854]: osdmap e313: 8 total, 8 up, 8 in 2023-10-09T10:56:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:14 smithi155 ceph-mon[184854]: failed to encode map e313 with expected crc 2023-10-09T10:56:14.988 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:15.015 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:15.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:15.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:15.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:14 smithi125 ceph-mon[195507]: pgmap v763: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 14351/114504 objects degraded (12.533%); 8724/114504 objects misplaced (7.619%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:56:15.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:14 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T10:56:15.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:14 smithi125 ceph-mon[195507]: osdmap e313: 8 total, 8 up, 8 in 2023-10-09T10:56:15.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:14 smithi125 ceph-mon[195507]: failed to encode map e313 with expected crc 2023-10-09T10:56:15.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:14 smithi125 ceph-mon[200197]: pgmap v763: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 14351/114504 objects degraded (12.533%); 8724/114504 objects misplaced (7.619%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T10:56:15.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:14 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T10:56:15.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:14 smithi125 ceph-mon[200197]: osdmap e313: 8 total, 8 up, 8 in 2023-10-09T10:56:15.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:14 smithi125 ceph-mon[200197]: failed to encode map e313 with expected crc 2023-10-09T10:56:15.123 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:15.187 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:15.214 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:15.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:15.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:15.323 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:15.359 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:15.378 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:15.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:15.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:15.481 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:15.514 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:15.545 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:15.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:15.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:15.599 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:15.628 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:15.649 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:15.729 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:16.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:56:16.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:56:16.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:16 smithi125 ceph-mon[200197]: pgmap v765: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 14221/111078 objects degraded (12.803%); 8490/111078 objects misplaced (7.643%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:56:16.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:56:16.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:56:16.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:16 smithi125 ceph-mon[195507]: pgmap v765: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 14221/111078 objects degraded (12.803%); 8490/111078 objects misplaced (7.643%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:56:16.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:56:16.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:56:16.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:16 smithi155 ceph-mon[184854]: pgmap v765: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 14221/111078 objects degraded (12.803%); 8490/111078 objects misplaced (7.643%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T10:56:16.968 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:16.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:16.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:17.095 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:17.128 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:17.150 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:17.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:17.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:17.254 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:17.289 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:17.311 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:17.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:17.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:17.417 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:17.448 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:17.470 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:17.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:17.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:17.575 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:17.643 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:17.824 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:17.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:17.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:17.935 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:17.976 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:18.003 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:18.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:18.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:18.107 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:18.141 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:18.166 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:18.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:18.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:18.271 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:18.347 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:18.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:18 smithi125 ceph-mon[200197]: pgmap v766: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 14221/103833 objects degraded (13.696%); 7823/103833 objects misplaced (7.534%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T10:56:18.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:18 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 14221/103833 objects degraded (13.696%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:18.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:18 smithi125 ceph-mon[195507]: pgmap v766: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 14221/103833 objects degraded (13.696%); 7823/103833 objects misplaced (7.534%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T10:56:18.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:18 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 14221/103833 objects degraded (13.696%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:18.648 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:18.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:18.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:18.754 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:18.783 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:18.784 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:18 smithi155 ceph-mon[184854]: pgmap v766: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 14221/103833 objects degraded (13.696%); 7823/103833 objects misplaced (7.534%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T10:56:18.784 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:18 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 14221/103833 objects degraded (13.696%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:18.810 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:18.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:18.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:18.915 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:18.973 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:18.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:19.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:19.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:19.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:19.189 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 7, 2, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:56:19.190 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:56:19.190 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T10:56:19.190 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:56:19.191 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:56:19.234 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:56:19.259 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:19.305 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:56:19.307 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:19.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:19.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:19.329 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:56:19.330 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:56:19.382 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:56:19.417 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:19.440 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:56:19.479 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:56:19.498 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:19.517 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:56:19.543 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:19.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:19.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:19.564 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:56:19.570 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:56:19.570 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:56:19.570 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:56:19.570 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:56:19.570 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:56:19.570 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:56:19.571 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:56:19.571 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:56:19.585 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:56:19.648 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:19.681 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:19.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:19 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2573043587' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:19.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:19 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2448354848' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:56:19.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:19 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2573043587' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:19.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:19 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2448354848' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:56:19.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:19 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2573043587' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:19.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:19 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2448354848' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:56:20.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:20 smithi125 ceph-mon[195507]: pgmap v767: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 14068/100446 objects degraded (14.006%); 7590/100446 objects misplaced (7.556%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T10:56:20.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:20 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:56:20.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:20 smithi125 ceph-mon[195507]: failed to encode map e314 with expected crc 2023-10-09T10:56:20.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:20 smithi125 ceph-mon[195507]: failed to encode map e314 with expected crc 2023-10-09T10:56:20.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:20 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2448354848' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:56:20.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:20 smithi125 ceph-mon[195507]: osdmap e314: 8 total, 8 up, 8 in 2023-10-09T10:56:20.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:20 smithi125 ceph-mon[200197]: pgmap v767: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 14068/100446 objects degraded (14.006%); 7590/100446 objects misplaced (7.556%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T10:56:20.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:20 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:56:20.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:20 smithi125 ceph-mon[200197]: failed to encode map e314 with expected crc 2023-10-09T10:56:20.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:20 smithi125 ceph-mon[200197]: failed to encode map e314 with expected crc 2023-10-09T10:56:20.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:20 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2448354848' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:56:20.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:20 smithi125 ceph-mon[200197]: osdmap e314: 8 total, 8 up, 8 in 2023-10-09T10:56:20.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:20.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:20.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:20.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:56:20.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:20 smithi155 ceph-mon[184854]: pgmap v767: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 14068/100446 objects degraded (14.006%); 7590/100446 objects misplaced (7.556%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T10:56:20.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:20 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:56:20.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:20 smithi155 ceph-mon[184854]: failed to encode map e314 with expected crc 2023-10-09T10:56:20.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:20 smithi155 ceph-mon[184854]: failed to encode map e314 with expected crc 2023-10-09T10:56:20.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:20 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2448354848' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:56:20.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:20 smithi155 ceph-mon[184854]: osdmap e314: 8 total, 8 up, 8 in 2023-10-09T10:56:21.029 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:21.062 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:21.132 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:21.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:21.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:21.222 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:56:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:56:21] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.43.0" 2023-10-09T10:56:21.239 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:21.379 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:21.412 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:21.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:21.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:21.499 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:21.517 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:21.549 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:21.569 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:21.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:21.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:21.674 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:21.766 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:21.801 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:21.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:21.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:21.910 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:21.965 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:22.003 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:22.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:22.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:22.112 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:22.142 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:22.172 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:22.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:22.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:22.249 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:22.277 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:22.370 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:22.420 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:22.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:22.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:22.528 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:22.634 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:22.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:22.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:22.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:22.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:22 smithi125 ceph-mon[195507]: pgmap v769: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 14068/94101 objects degraded (14.950%); 6962/94101 objects misplaced (7.398%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T10:56:22.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:22 smithi125 ceph-mon[200197]: pgmap v769: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 14068/94101 objects degraded (14.950%); 6962/94101 objects misplaced (7.398%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T10:56:22.825 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:22.900 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:22.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:22 smithi155 ceph-mon[184854]: pgmap v769: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 14068/94101 objects degraded (14.950%); 6962/94101 objects misplaced (7.398%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T10:56:23.256 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:56:23.269 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:56:23.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:56:23.458 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:56:23.932 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:23.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:23.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:23.936 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:23 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 14068/94101 objects degraded (14.950%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:23.938 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:23 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 14068/94101 objects degraded (14.950%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:23.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:23 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 14068/94101 objects degraded (14.950%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:24.036 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:24.104 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:24.291 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:24.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:24.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:24.397 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:24.465 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:24.542 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":769,"stamp":"2023-10-09T10:56:21.475600+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2054763856,"num_objects":31367,"num_object_clones":0,"num_object_copies":94101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14068,"num_objects_misplaced":6962,"num_objects_unfound":0,"num_objects_dirty":31367,"num_whiteouts":0,"num_read":116,"num_read_kb":215,"num_write":162078,"num_write_kb":6186261,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9672,"num_bytes_recovered":631961362,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":186294,"ondisk_log_size":186294,"up":387,"acting":377,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":421,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10975464,"kb_used_data":7061352,"kb_used_omap":126,"kb_used_meta":3913857,"kb_avail":738985752,"statfs":{"total":767960285184,"available":756721410048,"internally_reserved":0,"allocated":7230824448,"data_stored":7219647314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":4007790203},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,9,2,7,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-445710336,"num_objects":-6801,"num_object_clones":0,"num_object_copies":-20403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-283,"num_objects_misplaced":-1762,"num_objects_unfound":0,"num_objects_dirty":-6801,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6801,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":750,"num_bytes_recovered":49152000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.005291"},"pg_stats":[{"pgid":"8.11","version":"313'7958","reported_seq":12049,"reported_epoch":313,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:18.498951+0000","last_change":"2023-10-09T10:54:49.220300+0000","last_active":"2023-10-09T10:56:18.498951+0000","last_peered":"2023-10-09T10:56:18.498951+0000","last_clean":"2023-10-09T10:54:02.426241+0000","last_became_active":"2023-10-09T10:54:49.215481+0000","last_became_peered":"2023-10-09T10:54:49.215481+0000","last_unstale":"2023-10-09T10:56:18.498951+0000","last_undegraded":"2023-10-09T10:54:49.026934+0000","last_fullsized":"2023-10-09T10:54:48.893501+0000","mapping_epoch":295,"log_start":"270'200","ondisk_log_start":"270'200","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7758,"log_dups_size":0,"ondisk_log_size":7758,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64946176,"num_objects":991,"num_object_clones":0,"num_object_copies":2973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1289,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4997,"num_write_kb":191616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2003,"num_bytes_recovered":131268608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,1,5","objects":320},{"shards":"0,2","objects":-10},{"shards":"0,2,5","objects":393},{"shards":"0,3,5","objects":288}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.18","version":"76'161","reported_seq":628,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.790891+0000","last_change":"2023-10-09T10:55:26.628655+0000","last_active":"2023-10-09T10:56:13.790891+0000","last_peered":"2023-10-09T10:56:13.790891+0000","last_clean":"2023-10-09T10:56:13.790891+0000","last_became_active":"2023-10-09T10:55:26.628269+0000","last_became_peered":"2023-10-09T10:55:26.628269+0000","last_unstale":"2023-10-09T10:56:13.790891+0000","last_undegraded":"2023-10-09T10:56:13.790891+0000","last_fullsized":"2023-10-09T10:56:13.790891+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:40.174975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:40.174975+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:24:16.854746+0000","scrub_duration":0.010654205,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.732169+0000","last_change":"2023-10-09T10:54:47.925818+0000","last_active":"2023-10-09T10:56:10.732169+0000","last_peered":"2023-10-09T10:56:10.732169+0000","last_clean":"2023-10-09T10:56:10.732169+0000","last_became_active":"2023-10-09T10:54:47.925375+0000","last_became_peered":"2023-10-09T10:54:47.925375+0000","last_unstale":"2023-10-09T10:56:10.732169+0000","last_undegraded":"2023-10-09T10:56:10.732169+0000","last_fullsized":"2023-10-09T10:56:10.732169+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:27.635383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":322,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.764006+0000","last_change":"2023-10-09T10:54:47.945782+0000","last_active":"2023-10-09T10:56:13.764006+0000","last_peered":"2023-10-09T10:56:13.764006+0000","last_clean":"2023-10-09T10:56:13.764006+0000","last_became_active":"2023-10-09T10:54:47.945460+0000","last_became_peered":"2023-10-09T10:54:47.945460+0000","last_unstale":"2023-10-09T10:56:13.764006+0000","last_undegraded":"2023-10-09T10:56:13.764006+0000","last_fullsized":"2023-10-09T10:56:13.764006+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:50.537469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"314'8175","reported_seq":8247,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:21.220496+0000","last_change":"2023-10-09T10:54:40.626105+0000","last_active":"2023-10-09T10:56:21.220496+0000","last_peered":"2023-10-09T10:56:21.220496+0000","last_clean":"2023-10-09T10:56:21.220496+0000","last_became_active":"2023-10-09T10:54:40.609428+0000","last_became_peered":"2023-10-09T10:54:40.609428+0000","last_unstale":"2023-10-09T10:56:21.220496+0000","last_undegraded":"2023-10-09T10:56:21.220496+0000","last_fullsized":"2023-10-09T10:56:21.220496+0000","mapping_epoch":289,"log_start":"271'700","ondisk_log_start":"271'700","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7475,"log_dups_size":0,"ondisk_log_size":7475,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64880640,"num_objects":990,"num_object_clones":0,"num_object_copies":2970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5140,"num_write_kb":196160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":694,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.712048+0000","last_change":"2023-10-09T10:53:34.537094+0000","last_active":"2023-10-09T10:56:10.712048+0000","last_peered":"2023-10-09T10:56:10.712048+0000","last_clean":"2023-10-09T10:56:10.712048+0000","last_became_active":"2023-10-09T10:52:55.414995+0000","last_became_peered":"2023-10-09T10:52:55.414995+0000","last_unstale":"2023-10-09T10:56:10.712048+0000","last_undegraded":"2023-10-09T10:56:10.712048+0000","last_fullsized":"2023-10-09T10:56:10.712048+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.536962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.536962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:30.197404+0000","scrub_duration":0.0053325960000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.735145+0000","last_change":"2023-10-09T10:54:47.944768+0000","last_active":"2023-10-09T10:56:10.735145+0000","last_peered":"2023-10-09T10:56:10.735145+0000","last_clean":"2023-10-09T10:56:10.735145+0000","last_became_active":"2023-10-09T10:54:47.943888+0000","last_became_peered":"2023-10-09T10:54:47.943888+0000","last_unstale":"2023-10-09T10:56:10.735145+0000","last_undegraded":"2023-10-09T10:56:10.735145+0000","last_fullsized":"2023-10-09T10:56:10.735145+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:11.046762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":222,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.718583+0000","last_change":"2023-10-09T10:53:14.823051+0000","last_active":"2023-10-09T10:56:10.718583+0000","last_peered":"2023-10-09T10:56:10.718583+0000","last_clean":"2023-10-09T10:56:10.718583+0000","last_became_active":"2023-10-09T10:53:14.822334+0000","last_became_peered":"2023-10-09T10:53:14.822334+0000","last_unstale":"2023-10-09T10:56:10.718583+0000","last_undegraded":"2023-10-09T10:56:10.718583+0000","last_fullsized":"2023-10-09T10:56:10.718583+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:46.573381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"313'7854","reported_seq":7966,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:16.721353+0000","last_change":"2023-10-09T10:54:50.916172+0000","last_active":"2023-10-09T10:56:16.721353+0000","last_peered":"2023-10-09T10:56:16.721353+0000","last_clean":"2023-10-09T10:56:16.721353+0000","last_became_active":"2023-10-09T10:54:50.913596+0000","last_became_peered":"2023-10-09T10:54:50.913596+0000","last_unstale":"2023-10-09T10:56:16.721353+0000","last_undegraded":"2023-10-09T10:56:16.721353+0000","last_fullsized":"2023-10-09T10:56:16.721353+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7854,"log_dups_size":0,"ondisk_log_size":7854,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70680576,"num_objects":1079,"num_object_clones":0,"num_object_copies":3237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4897,"num_write_kb":191200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1835008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":697,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.758827+0000","last_change":"2023-10-09T10:54:50.913546+0000","last_active":"2023-10-09T10:56:13.758827+0000","last_peered":"2023-10-09T10:56:13.758827+0000","last_clean":"2023-10-09T10:56:13.758827+0000","last_became_active":"2023-10-09T10:54:50.913190+0000","last_became_peered":"2023-10-09T10:54:50.913190+0000","last_unstale":"2023-10-09T10:56:13.758827+0000","last_undegraded":"2023-10-09T10:56:13.758827+0000","last_fullsized":"2023-10-09T10:56:13.758827+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:23.805498+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":367,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.764826+0000","last_change":"2023-10-09T10:54:48.038948+0000","last_active":"2023-10-09T10:56:13.764826+0000","last_peered":"2023-10-09T10:56:13.764826+0000","last_clean":"2023-10-09T10:56:13.764826+0000","last_became_active":"2023-10-09T10:54:48.001556+0000","last_became_peered":"2023-10-09T10:54:48.001556+0000","last_unstale":"2023-10-09T10:56:13.764826+0000","last_undegraded":"2023-10-09T10:56:13.764826+0000","last_fullsized":"2023-10-09T10:56:13.764826+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:23.415040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.733881+0000","last_change":"2023-10-09T10:54:47.914742+0000","last_active":"2023-10-09T10:56:10.733881+0000","last_peered":"2023-10-09T10:56:10.733881+0000","last_clean":"2023-10-09T10:56:10.733881+0000","last_became_active":"2023-10-09T10:54:47.914551+0000","last_became_peered":"2023-10-09T10:54:47.914551+0000","last_unstale":"2023-10-09T10:56:10.733881+0000","last_undegraded":"2023-10-09T10:56:10.733881+0000","last_fullsized":"2023-10-09T10:56:10.733881+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:16.067994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":748,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.767005+0000","last_change":"2023-10-09T10:54:50.915690+0000","last_active":"2023-10-09T10:56:13.767005+0000","last_peered":"2023-10-09T10:56:13.767005+0000","last_clean":"2023-10-09T10:56:13.767005+0000","last_became_active":"2023-10-09T10:54:50.915412+0000","last_became_peered":"2023-10-09T10:54:50.915412+0000","last_unstale":"2023-10-09T10:56:13.767005+0000","last_undegraded":"2023-10-09T10:56:13.767005+0000","last_fullsized":"2023-10-09T10:56:13.767005+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.787782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.787782+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:20:13.339672+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":185,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.714403+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:56:10.714403+0000","last_peered":"2023-10-09T10:56:10.714403+0000","last_clean":"2023-10-09T10:56:10.714403+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:56:10.714403+0000","last_undegraded":"2023-10-09T10:56:10.714403+0000","last_fullsized":"2023-10-09T10:56:10.714403+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":322,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.764868+0000","last_change":"2023-10-09T10:54:48.039275+0000","last_active":"2023-10-09T10:56:13.764868+0000","last_peered":"2023-10-09T10:56:13.764868+0000","last_clean":"2023-10-09T10:56:13.764868+0000","last_became_active":"2023-10-09T10:54:48.017645+0000","last_became_peered":"2023-10-09T10:54:48.017645+0000","last_unstale":"2023-10-09T10:56:13.764868+0000","last_undegraded":"2023-10-09T10:56:13.764868+0000","last_fullsized":"2023-10-09T10:56:13.764868+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:55.986791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":558,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.791676+0000","last_change":"2023-10-09T10:54:21.161539+0000","last_active":"2023-10-09T10:56:13.791676+0000","last_peered":"2023-10-09T10:56:13.791676+0000","last_clean":"2023-10-09T10:56:13.791676+0000","last_became_active":"2023-10-09T10:54:21.161106+0000","last_became_peered":"2023-10-09T10:54:21.161106+0000","last_unstale":"2023-10-09T10:56:13.791676+0000","last_undegraded":"2023-10-09T10:56:13.791676+0000","last_fullsized":"2023-10-09T10:56:13.791676+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:21.824982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:21.824982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":54,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.765693+0000","last_change":"2023-10-09T10:55:25.632545+0000","last_active":"2023-10-09T10:56:13.765693+0000","last_peered":"2023-10-09T10:56:13.765693+0000","last_clean":"2023-10-09T10:56:13.765693+0000","last_became_active":"2023-10-09T10:55:25.632235+0000","last_became_peered":"2023-10-09T10:55:25.632235+0000","last_unstale":"2023-10-09T10:56:13.765693+0000","last_undegraded":"2023-10-09T10:56:13.765693+0000","last_fullsized":"2023-10-09T10:56:13.765693+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:11.982194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":261,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.763776+0000","last_change":"2023-10-09T10:53:14.838859+0000","last_active":"2023-10-09T10:56:13.763776+0000","last_peered":"2023-10-09T10:56:13.763776+0000","last_clean":"2023-10-09T10:56:13.763776+0000","last_became_active":"2023-10-09T10:53:14.838536+0000","last_became_peered":"2023-10-09T10:53:14.838536+0000","last_unstale":"2023-10-09T10:56:13.763776+0000","last_undegraded":"2023-10-09T10:56:13.763776+0000","last_fullsized":"2023-10-09T10:56:13.763776+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:44:19.980568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":701,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.714051+0000","last_change":"2023-10-09T10:53:32.540768+0000","last_active":"2023-10-09T10:56:10.714051+0000","last_peered":"2023-10-09T10:56:10.714051+0000","last_clean":"2023-10-09T10:56:10.714051+0000","last_became_active":"2023-10-09T10:52:55.414818+0000","last_became_peered":"2023-10-09T10:52:55.414818+0000","last_unstale":"2023-10-09T10:56:10.714051+0000","last_undegraded":"2023-10-09T10:56:10.714051+0000","last_fullsized":"2023-10-09T10:56:10.714051+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:32.540610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:32.540610+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:47.374383+0000","scrub_duration":0.0064251300000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":226,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.766557+0000","last_change":"2023-10-09T10:55:25.637189+0000","last_active":"2023-10-09T10:56:13.766557+0000","last_peered":"2023-10-09T10:56:13.766557+0000","last_clean":"2023-10-09T10:56:13.766557+0000","last_became_active":"2023-10-09T10:55:25.636844+0000","last_became_peered":"2023-10-09T10:55:25.636844+0000","last_unstale":"2023-10-09T10:56:13.766557+0000","last_undegraded":"2023-10-09T10:56:13.766557+0000","last_fullsized":"2023-10-09T10:56:13.766557+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:18.943805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.732253+0000","last_change":"2023-10-09T10:54:47.925654+0000","last_active":"2023-10-09T10:56:10.732253+0000","last_peered":"2023-10-09T10:56:10.732253+0000","last_clean":"2023-10-09T10:56:10.732253+0000","last_became_active":"2023-10-09T10:54:47.924989+0000","last_became_peered":"2023-10-09T10:54:47.924989+0000","last_unstale":"2023-10-09T10:56:10.732253+0000","last_undegraded":"2023-10-09T10:56:10.732253+0000","last_fullsized":"2023-10-09T10:56:10.732253+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:02:49.755988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":686,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.733178+0000","last_change":"2023-10-09T10:54:50.913652+0000","last_active":"2023-10-09T10:56:10.733178+0000","last_peered":"2023-10-09T10:56:10.733178+0000","last_clean":"2023-10-09T10:56:10.733178+0000","last_became_active":"2023-10-09T10:54:50.913330+0000","last_became_peered":"2023-10-09T10:54:50.913330+0000","last_unstale":"2023-10-09T10:56:10.733178+0000","last_undegraded":"2023-10-09T10:56:10.733178+0000","last_fullsized":"2023-10-09T10:56:10.733178+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.200574+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.200574+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001591812,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":67,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.766052+0000","last_change":"2023-10-09T10:53:14.869000+0000","last_active":"2023-10-09T10:56:13.766052+0000","last_peered":"2023-10-09T10:56:13.766052+0000","last_clean":"2023-10-09T10:56:13.766052+0000","last_became_active":"2023-10-09T10:53:14.831904+0000","last_became_peered":"2023-10-09T10:53:14.831904+0000","last_unstale":"2023-10-09T10:56:13.766052+0000","last_undegraded":"2023-10-09T10:56:13.766052+0000","last_fullsized":"2023-10-09T10:56:13.766052+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:38.571980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":318,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.759221+0000","last_change":"2023-10-09T10:54:47.908531+0000","last_active":"2023-10-09T10:56:13.759221+0000","last_peered":"2023-10-09T10:56:13.759221+0000","last_clean":"2023-10-09T10:56:13.759221+0000","last_became_active":"2023-10-09T10:54:47.908232+0000","last_became_peered":"2023-10-09T10:54:47.908232+0000","last_unstale":"2023-10-09T10:56:13.759221+0000","last_undegraded":"2023-10-09T10:56:13.759221+0000","last_fullsized":"2023-10-09T10:56:13.759221+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:20.617543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"314'8040","reported_seq":8108,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:20.262484+0000","last_change":"2023-10-09T10:54:40.649728+0000","last_active":"2023-10-09T10:56:20.262484+0000","last_peered":"2023-10-09T10:56:20.262484+0000","last_clean":"2023-10-09T10:56:20.262484+0000","last_became_active":"2023-10-09T10:54:40.647392+0000","last_became_peered":"2023-10-09T10:54:40.647392+0000","last_unstale":"2023-10-09T10:56:20.262484+0000","last_undegraded":"2023-10-09T10:56:20.262484+0000","last_fullsized":"2023-10-09T10:56:20.262484+0000","mapping_epoch":289,"log_start":"271'1000","ondisk_log_start":"271'1000","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7040,"log_dups_size":0,"ondisk_log_size":7040,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65273856,"num_objects":996,"num_object_clones":0,"num_object_copies":2988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5050,"num_write_kb":193472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":679,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.715738+0000","last_change":"2023-10-09T10:49:18.708370+0000","last_active":"2023-10-09T10:56:10.715738+0000","last_peered":"2023-10-09T10:56:10.715738+0000","last_clean":"2023-10-09T10:56:10.715738+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:56:10.715738+0000","last_undegraded":"2023-10-09T10:56:10.715738+0000","last_fullsized":"2023-10-09T10:56:10.715738+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:57.808410+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":79,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.765557+0000","last_change":"2023-10-09T10:54:18.080112+0000","last_active":"2023-10-09T10:56:13.765557+0000","last_peered":"2023-10-09T10:56:13.765557+0000","last_clean":"2023-10-09T10:56:13.765557+0000","last_became_active":"2023-10-09T10:54:17.878848+0000","last_became_peered":"2023-10-09T10:54:17.878848+0000","last_unstale":"2023-10-09T10:56:13.765557+0000","last_undegraded":"2023-10-09T10:56:13.765557+0000","last_fullsized":"2023-10-09T10:56:13.765557+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:12.314073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":372,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.764889+0000","last_change":"2023-10-09T10:54:47.941670+0000","last_active":"2023-10-09T10:56:13.764889+0000","last_peered":"2023-10-09T10:56:13.764889+0000","last_clean":"2023-10-09T10:56:13.764889+0000","last_became_active":"2023-10-09T10:54:47.941341+0000","last_became_peered":"2023-10-09T10:54:47.941341+0000","last_unstale":"2023-10-09T10:56:13.764889+0000","last_undegraded":"2023-10-09T10:56:13.764889+0000","last_fullsized":"2023-10-09T10:56:13.764889+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:31.570728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.19","version":"313'8023","reported_seq":8089,"reported_epoch":313,"state":"active+recovery_wait","last_fresh":"2023-10-09T10:56:18.416372+0000","last_change":"2023-10-09T10:54:40.731544+0000","last_active":"2023-10-09T10:56:18.416372+0000","last_peered":"2023-10-09T10:56:18.416372+0000","last_clean":"2023-10-09T10:54:39.518652+0000","last_became_active":"2023-10-09T10:54:40.598732+0000","last_became_peered":"2023-10-09T10:54:40.598732+0000","last_unstale":"2023-10-09T10:56:18.416372+0000","last_undegraded":"2023-10-09T10:56:18.416372+0000","last_fullsized":"2023-10-09T10:56:18.416372+0000","mapping_epoch":289,"log_start":"270'300","ondisk_log_start":"270'300","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7723,"log_dups_size":0,"ondisk_log_size":7723,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66453504,"num_objects":1014,"num_object_clones":0,"num_object_copies":3042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5038,"num_write_kb":193664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":1014}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":661,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.765358+0000","last_change":"2023-10-09T10:53:17.846464+0000","last_active":"2023-10-09T10:56:13.765358+0000","last_peered":"2023-10-09T10:56:13.765358+0000","last_clean":"2023-10-09T10:56:13.765358+0000","last_became_active":"2023-10-09T10:53:17.845979+0000","last_became_peered":"2023-10-09T10:53:17.845979+0000","last_unstale":"2023-10-09T10:56:13.765358+0000","last_undegraded":"2023-10-09T10:56:13.765358+0000","last_fullsized":"2023-10-09T10:56:13.765358+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:56.295299+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":221,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.714904+0000","last_change":"2023-10-09T10:53:14.821961+0000","last_active":"2023-10-09T10:56:10.714904+0000","last_peered":"2023-10-09T10:56:10.714904+0000","last_clean":"2023-10-09T10:56:10.714904+0000","last_became_active":"2023-10-09T10:53:14.821625+0000","last_became_peered":"2023-10-09T10:53:14.821625+0000","last_unstale":"2023-10-09T10:56:10.714904+0000","last_undegraded":"2023-10-09T10:56:10.714904+0000","last_fullsized":"2023-10-09T10:56:10.714904+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:29.305952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":228,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.765720+0000","last_change":"2023-10-09T10:55:25.629418+0000","last_active":"2023-10-09T10:56:13.765720+0000","last_peered":"2023-10-09T10:56:13.765720+0000","last_clean":"2023-10-09T10:56:13.765720+0000","last_became_active":"2023-10-09T10:55:25.629130+0000","last_became_peered":"2023-10-09T10:55:25.629130+0000","last_unstale":"2023-10-09T10:56:13.765720+0000","last_undegraded":"2023-10-09T10:56:13.765720+0000","last_fullsized":"2023-10-09T10:56:13.765720+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:51.186552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":617,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.739457+0000","last_change":"2023-10-09T10:54:50.916062+0000","last_active":"2023-10-09T10:56:10.739457+0000","last_peered":"2023-10-09T10:56:10.739457+0000","last_clean":"2023-10-09T10:56:10.739457+0000","last_became_active":"2023-10-09T10:54:50.915189+0000","last_became_peered":"2023-10-09T10:54:50.915189+0000","last_unstale":"2023-10-09T10:56:10.739457+0000","last_undegraded":"2023-10-09T10:56:10.739457+0000","last_fullsized":"2023-10-09T10:56:10.739457+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.077788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.077788+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":209,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.765048+0000","last_change":"2023-10-09T10:55:25.633116+0000","last_active":"2023-10-09T10:56:13.765048+0000","last_peered":"2023-10-09T10:56:13.765048+0000","last_clean":"2023-10-09T10:56:13.765048+0000","last_became_active":"2023-10-09T10:55:25.632837+0000","last_became_peered":"2023-10-09T10:55:25.632837+0000","last_unstale":"2023-10-09T10:56:13.765048+0000","last_undegraded":"2023-10-09T10:56:13.765048+0000","last_fullsized":"2023-10-09T10:56:13.765048+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:46.192186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":203,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.718370+0000","last_change":"2023-10-09T10:53:14.823485+0000","last_active":"2023-10-09T10:56:10.718370+0000","last_peered":"2023-10-09T10:56:10.718370+0000","last_clean":"2023-10-09T10:56:10.718370+0000","last_became_active":"2023-10-09T10:53:14.822756+0000","last_became_peered":"2023-10-09T10:53:14.822756+0000","last_unstale":"2023-10-09T10:56:10.718370+0000","last_undegraded":"2023-10-09T10:56:10.718370+0000","last_fullsized":"2023-10-09T10:56:10.718370+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:05.359313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":644,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.732548+0000","last_change":"2023-10-09T10:54:50.910027+0000","last_active":"2023-10-09T10:56:10.732548+0000","last_peered":"2023-10-09T10:56:10.732548+0000","last_clean":"2023-10-09T10:56:10.732548+0000","last_became_active":"2023-10-09T10:54:50.909698+0000","last_became_peered":"2023-10-09T10:54:50.909698+0000","last_unstale":"2023-10-09T10:56:10.732548+0000","last_undegraded":"2023-10-09T10:56:10.732548+0000","last_fullsized":"2023-10-09T10:56:10.732548+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:01.637196+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":188,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.711908+0000","last_change":"2023-10-09T10:51:12.366910+0000","last_active":"2023-10-09T10:56:10.711908+0000","last_peered":"2023-10-09T10:56:10.711908+0000","last_clean":"2023-10-09T10:56:10.711908+0000","last_became_active":"2023-10-09T10:51:12.366565+0000","last_became_peered":"2023-10-09T10:51:12.366565+0000","last_unstale":"2023-10-09T10:56:10.711908+0000","last_undegraded":"2023-10-09T10:56:10.711908+0000","last_fullsized":"2023-10-09T10:56:10.711908+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:53.464251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":349,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.768302+0000","last_change":"2023-10-09T10:54:48.017646+0000","last_active":"2023-10-09T10:56:13.768302+0000","last_peered":"2023-10-09T10:56:13.768302+0000","last_clean":"2023-10-09T10:56:13.768302+0000","last_became_active":"2023-10-09T10:54:47.994162+0000","last_became_peered":"2023-10-09T10:54:47.994162+0000","last_unstale":"2023-10-09T10:56:13.768302+0000","last_undegraded":"2023-10-09T10:56:13.768302+0000","last_fullsized":"2023-10-09T10:56:13.768302+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:11:59.205546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"314'8094","reported_seq":8174,"reported_epoch":314,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:56:21.295230+0000","last_change":"2023-10-09T10:54:40.551123+0000","last_active":"2023-10-09T10:56:21.295230+0000","last_peered":"2023-10-09T10:56:21.295230+0000","last_clean":"2023-10-09T10:54:17.036595+0000","last_became_active":"2023-10-09T10:54:40.548039+0000","last_became_peered":"2023-10-09T10:54:40.548039+0000","last_unstale":"2023-10-09T10:56:21.295230+0000","last_undegraded":"2023-10-09T10:56:21.295230+0000","last_fullsized":"2023-10-09T10:56:21.295230+0000","mapping_epoch":282,"log_start":"271'600","ondisk_log_start":"271'600","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7494,"log_dups_size":0,"ondisk_log_size":7494,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60850176,"num_objects":928,"num_object_clones":0,"num_object_copies":2784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":928,"num_objects_unfound":0,"num_objects_dirty":928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5108,"num_write_kb":193184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":928}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":665,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.766825+0000","last_change":"2023-10-09T10:54:50.909873+0000","last_active":"2023-10-09T10:56:13.766825+0000","last_peered":"2023-10-09T10:56:13.766825+0000","last_clean":"2023-10-09T10:56:13.766825+0000","last_became_active":"2023-10-09T10:54:50.909533+0000","last_became_peered":"2023-10-09T10:54:50.909533+0000","last_unstale":"2023-10-09T10:56:13.766825+0000","last_undegraded":"2023-10-09T10:56:13.766825+0000","last_fullsized":"2023-10-09T10:56:13.766825+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:45.045013+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":61,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.765968+0000","last_change":"2023-10-09T10:53:14.854904+0000","last_active":"2023-10-09T10:56:13.765968+0000","last_peered":"2023-10-09T10:56:13.765968+0000","last_clean":"2023-10-09T10:56:13.765968+0000","last_became_active":"2023-10-09T10:53:14.829611+0000","last_became_peered":"2023-10-09T10:53:14.829611+0000","last_unstale":"2023-10-09T10:56:13.765968+0000","last_undegraded":"2023-10-09T10:56:13.765968+0000","last_fullsized":"2023-10-09T10:56:13.765968+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:26.064630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":246,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.765056+0000","last_change":"2023-10-09T10:55:25.633485+0000","last_active":"2023-10-09T10:56:13.765056+0000","last_peered":"2023-10-09T10:56:13.765056+0000","last_clean":"2023-10-09T10:56:13.765056+0000","last_became_active":"2023-10-09T10:55:25.633028+0000","last_became_peered":"2023-10-09T10:55:25.633028+0000","last_unstale":"2023-10-09T10:56:13.765056+0000","last_undegraded":"2023-10-09T10:56:13.765056+0000","last_fullsized":"2023-10-09T10:56:13.765056+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:57.601747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"314'8048","reported_seq":8116,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:21.109240+0000","last_change":"2023-10-09T10:55:26.330497+0000","last_active":"2023-10-09T10:56:21.109240+0000","last_peered":"2023-10-09T10:56:21.109240+0000","last_clean":"2023-10-09T10:56:21.109240+0000","last_became_active":"2023-10-09T10:54:40.554360+0000","last_became_peered":"2023-10-09T10:54:40.554360+0000","last_unstale":"2023-10-09T10:56:21.109240+0000","last_undegraded":"2023-10-09T10:56:21.109240+0000","last_fullsized":"2023-10-09T10:56:21.109240+0000","mapping_epoch":266,"log_start":"271'400","ondisk_log_start":"271'400","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7648,"log_dups_size":0,"ondisk_log_size":7648,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58916864,"num_objects":899,"num_object_clones":0,"num_object_copies":2697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5087,"num_write_kb":191552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":674,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.764736+0000","last_change":"2023-10-09T10:53:25.540494+0000","last_active":"2023-10-09T10:56:13.764736+0000","last_peered":"2023-10-09T10:56:13.764736+0000","last_clean":"2023-10-09T10:56:13.764736+0000","last_became_active":"2023-10-09T10:51:01.295474+0000","last_became_peered":"2023-10-09T10:51:01.295474+0000","last_unstale":"2023-10-09T10:56:13.764736+0000","last_undegraded":"2023-10-09T10:56:13.764736+0000","last_fullsized":"2023-10-09T10:56:13.764736+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:39.069143+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":89,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.765871+0000","last_change":"2023-10-09T10:54:48.000482+0000","last_active":"2023-10-09T10:56:13.765871+0000","last_peered":"2023-10-09T10:56:13.765871+0000","last_clean":"2023-10-09T10:56:13.765871+0000","last_became_active":"2023-10-09T10:54:47.905689+0000","last_became_peered":"2023-10-09T10:54:47.905689+0000","last_unstale":"2023-10-09T10:56:13.765871+0000","last_undegraded":"2023-10-09T10:56:13.765871+0000","last_fullsized":"2023-10-09T10:56:13.765871+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.666594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.734985+0000","last_change":"2023-10-09T10:54:47.944930+0000","last_active":"2023-10-09T10:56:10.734985+0000","last_peered":"2023-10-09T10:56:10.734985+0000","last_clean":"2023-10-09T10:56:10.734985+0000","last_became_active":"2023-10-09T10:54:47.944485+0000","last_became_peered":"2023-10-09T10:54:47.944485+0000","last_unstale":"2023-10-09T10:56:10.734985+0000","last_undegraded":"2023-10-09T10:56:10.734985+0000","last_fullsized":"2023-10-09T10:56:10.734985+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:11.213129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"313'8016","reported_seq":8084,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:18.506863+0000","last_change":"2023-10-09T10:54:40.586495+0000","last_active":"2023-10-09T10:56:18.506863+0000","last_peered":"2023-10-09T10:56:18.506863+0000","last_clean":"2023-10-09T10:56:18.506863+0000","last_became_active":"2023-10-09T10:54:40.577656+0000","last_became_peered":"2023-10-09T10:54:40.577656+0000","last_unstale":"2023-10-09T10:56:18.506863+0000","last_undegraded":"2023-10-09T10:56:18.506863+0000","last_fullsized":"2023-10-09T10:56:18.506863+0000","mapping_epoch":266,"log_start":"270'300","ondisk_log_start":"270'300","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7716,"log_dups_size":0,"ondisk_log_size":7716,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68255744,"num_objects":1041,"num_object_clones":0,"num_object_copies":3123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5021,"num_write_kb":194016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":626,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.760723+0000","last_change":"2023-10-09T10:53:36.680150+0000","last_active":"2023-10-09T10:56:13.760723+0000","last_peered":"2023-10-09T10:56:13.760723+0000","last_clean":"2023-10-09T10:56:13.760723+0000","last_became_active":"2023-10-09T10:51:50.371032+0000","last_became_peered":"2023-10-09T10:51:50.371032+0000","last_unstale":"2023-10-09T10:56:13.760723+0000","last_undegraded":"2023-10-09T10:56:13.760723+0000","last_fullsized":"2023-10-09T10:56:13.760723+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.680090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.680090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:08:28.952827+0000","scrub_duration":0.005577172,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":61,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.766917+0000","last_change":"2023-10-09T10:53:14.841343+0000","last_active":"2023-10-09T10:56:13.766917+0000","last_peered":"2023-10-09T10:56:13.766917+0000","last_clean":"2023-10-09T10:56:13.766917+0000","last_became_active":"2023-10-09T10:53:14.828213+0000","last_became_peered":"2023-10-09T10:53:14.828213+0000","last_unstale":"2023-10-09T10:56:13.766917+0000","last_undegraded":"2023-10-09T10:56:13.766917+0000","last_fullsized":"2023-10-09T10:56:13.766917+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:28.137803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.732470+0000","last_change":"2023-10-09T10:54:47.942716+0000","last_active":"2023-10-09T10:56:10.732470+0000","last_peered":"2023-10-09T10:56:10.732470+0000","last_clean":"2023-10-09T10:56:10.732470+0000","last_became_active":"2023-10-09T10:54:47.942459+0000","last_became_peered":"2023-10-09T10:54:47.942459+0000","last_unstale":"2023-10-09T10:56:10.732470+0000","last_undegraded":"2023-10-09T10:56:10.732470+0000","last_fullsized":"2023-10-09T10:56:10.732470+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:07.790165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"314'8068","reported_seq":8163,"reported_epoch":314,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:20.436385+0000","last_change":"2023-10-09T10:54:49.183581+0000","last_active":"2023-10-09T10:56:20.436385+0000","last_peered":"2023-10-09T10:56:20.436385+0000","last_clean":"2023-10-09T10:54:02.391425+0000","last_became_active":"2023-10-09T10:54:49.111868+0000","last_became_peered":"2023-10-09T10:54:49.111868+0000","last_unstale":"2023-10-09T10:56:20.436385+0000","last_undegraded":"2023-10-09T10:54:48.912262+0000","last_fullsized":"2023-10-09T10:54:48.907341+0000","mapping_epoch":295,"log_start":"271'1000","ondisk_log_start":"271'1000","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7068,"log_dups_size":0,"ondisk_log_size":7068,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61440000,"num_objects":937,"num_object_clones":0,"num_object_copies":2811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":697,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5091,"num_write_kb":192928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"3,4,6","objects":697},{"shards":"3,6","objects":240}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":759,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.760120+0000","last_change":"2023-10-09T10:53:38.681891+0000","last_active":"2023-10-09T10:56:13.760120+0000","last_peered":"2023-10-09T10:56:13.760120+0000","last_clean":"2023-10-09T10:56:13.760120+0000","last_became_active":"2023-10-09T10:53:17.840478+0000","last_became_peered":"2023-10-09T10:53:17.840478+0000","last_unstale":"2023-10-09T10:56:13.760120+0000","last_undegraded":"2023-10-09T10:56:13.760120+0000","last_fullsized":"2023-10-09T10:56:13.760120+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:31:14.758036+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":140,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.588903+0000","last_change":"2023-10-09T10:51:50.374527+0000","last_active":"2023-10-09T10:56:19.588903+0000","last_peered":"2023-10-09T10:56:19.588903+0000","last_clean":"2023-10-09T10:56:19.588903+0000","last_became_active":"2023-10-09T10:51:50.374297+0000","last_became_peered":"2023-10-09T10:51:50.374297+0000","last_unstale":"2023-10-09T10:56:19.588903+0000","last_undegraded":"2023-10-09T10:56:19.588903+0000","last_fullsized":"2023-10-09T10:56:19.588903+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:31:49.358307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":221,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.712492+0000","last_change":"2023-10-09T10:54:47.939771+0000","last_active":"2023-10-09T10:56:10.712492+0000","last_peered":"2023-10-09T10:56:10.712492+0000","last_clean":"2023-10-09T10:56:10.712492+0000","last_became_active":"2023-10-09T10:54:47.939440+0000","last_became_peered":"2023-10-09T10:54:47.939440+0000","last_unstale":"2023-10-09T10:56:10.712492+0000","last_undegraded":"2023-10-09T10:56:10.712492+0000","last_fullsized":"2023-10-09T10:56:10.712492+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:58.335536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"314'8129","reported_seq":10077,"reported_epoch":314,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:20.411233+0000","last_change":"2023-10-09T10:54:49.158909+0000","last_active":"2023-10-09T10:56:20.411233+0000","last_peered":"2023-10-09T10:56:20.411233+0000","last_clean":"2023-10-09T10:54:02.435878+0000","last_became_active":"2023-10-09T10:54:48.018286+0000","last_became_peered":"2023-10-09T10:54:48.018286+0000","last_unstale":"2023-10-09T10:56:20.411233+0000","last_undegraded":"2023-10-09T10:54:49.152454+0000","last_fullsized":"2023-10-09T10:54:47.891421+0000","mapping_epoch":294,"log_start":"271'1100","ondisk_log_start":"271'1100","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7029,"log_dups_size":0,"ondisk_log_size":7029,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64585728,"num_objects":985,"num_object_clones":0,"num_object_copies":2955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":407,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5111,"num_write_kb":195104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":592,"num_bytes_recovered":38830080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":407},{"shards":"3,5","objects":578}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":651,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.765144+0000","last_change":"2023-10-09T10:55:25.628214+0000","last_active":"2023-10-09T10:56:13.765144+0000","last_peered":"2023-10-09T10:56:13.765144+0000","last_clean":"2023-10-09T10:56:13.765144+0000","last_became_active":"2023-10-09T10:55:25.627822+0000","last_became_peered":"2023-10-09T10:55:25.627822+0000","last_unstale":"2023-10-09T10:56:13.765144+0000","last_undegraded":"2023-10-09T10:56:13.765144+0000","last_fullsized":"2023-10-09T10:56:13.765144+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":310,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.766319+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:56:13.766319+0000","last_peered":"2023-10-09T10:56:13.766319+0000","last_clean":"2023-10-09T10:56:13.766319+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:56:13.766319+0000","last_undegraded":"2023-10-09T10:56:13.766319+0000","last_fullsized":"2023-10-09T10:56:13.766319+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":155,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.764990+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T10:56:13.764990+0000","last_peered":"2023-10-09T10:56:13.764990+0000","last_clean":"2023-10-09T10:56:13.764990+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T10:56:13.764990+0000","last_undegraded":"2023-10-09T10:56:13.764990+0000","last_fullsized":"2023-10-09T10:56:13.764990+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"313'7966","reported_seq":12071,"reported_epoch":313,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:18.483253+0000","last_change":"2023-10-09T10:54:49.220251+0000","last_active":"2023-10-09T10:56:18.483253+0000","last_peered":"2023-10-09T10:56:18.483253+0000","last_clean":"2023-10-09T10:54:02.426241+0000","last_became_active":"2023-10-09T10:54:49.172055+0000","last_became_peered":"2023-10-09T10:54:49.172055+0000","last_unstale":"2023-10-09T10:56:18.483253+0000","last_undegraded":"2023-10-09T10:54:49.015836+0000","last_fullsized":"2023-10-09T10:54:48.896714+0000","mapping_epoch":295,"log_start":"270'200","ondisk_log_start":"270'200","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7766,"log_dups_size":0,"ondisk_log_size":7766,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64684032,"num_objects":987,"num_object_clones":0,"num_object_copies":2961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1262,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5005,"num_write_kb":191744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2009,"num_bytes_recovered":131661824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,1,5","objects":299},{"shards":"0,2","objects":10},{"shards":"0,2,5","objects":393},{"shards":"0,3,5","objects":285}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":617,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.768997+0000","last_change":"2023-10-09T10:54:21.173078+0000","last_active":"2023-10-09T10:56:13.768997+0000","last_peered":"2023-10-09T10:56:13.768997+0000","last_clean":"2023-10-09T10:56:13.768997+0000","last_became_active":"2023-10-09T10:54:21.172767+0000","last_became_peered":"2023-10-09T10:54:21.172767+0000","last_unstale":"2023-10-09T10:56:13.768997+0000","last_undegraded":"2023-10-09T10:56:13.768997+0000","last_fullsized":"2023-10-09T10:56:13.768997+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.532890+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.532890+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:32.335841+0000","scrub_duration":0.0045290130000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":293,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.759065+0000","last_change":"2023-10-09T10:54:47.908506+0000","last_active":"2023-10-09T10:56:13.759065+0000","last_peered":"2023-10-09T10:56:13.759065+0000","last_clean":"2023-10-09T10:56:13.759065+0000","last_became_active":"2023-10-09T10:54:47.908110+0000","last_became_peered":"2023-10-09T10:54:47.908110+0000","last_unstale":"2023-10-09T10:56:13.759065+0000","last_undegraded":"2023-10-09T10:56:13.759065+0000","last_fullsized":"2023-10-09T10:56:13.759065+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:38.808943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":52,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.792822+0000","last_change":"2023-10-09T10:54:47.948732+0000","last_active":"2023-10-09T10:56:13.792822+0000","last_peered":"2023-10-09T10:56:13.792822+0000","last_clean":"2023-10-09T10:56:13.792822+0000","last_became_active":"2023-10-09T10:54:47.948414+0000","last_became_peered":"2023-10-09T10:54:47.948414+0000","last_unstale":"2023-10-09T10:56:13.792822+0000","last_undegraded":"2023-10-09T10:56:13.792822+0000","last_fullsized":"2023-10-09T10:56:13.792822+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:38.020807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.2","version":"314'8203","reported_seq":8272,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:21.167282+0000","last_change":"2023-10-09T10:55:24.369167+0000","last_active":"2023-10-09T10:56:21.167282+0000","last_peered":"2023-10-09T10:56:21.167282+0000","last_clean":"2023-10-09T10:56:21.167282+0000","last_became_active":"2023-10-09T10:54:40.567891+0000","last_became_peered":"2023-10-09T10:54:40.567891+0000","last_unstale":"2023-10-09T10:56:21.167282+0000","last_undegraded":"2023-10-09T10:56:21.167282+0000","last_fullsized":"2023-10-09T10:56:21.167282+0000","mapping_epoch":265,"log_start":"271'700","ondisk_log_start":"271'700","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7503,"log_dups_size":0,"ondisk_log_size":7503,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61865984,"num_objects":944,"num_object_clones":0,"num_object_copies":2832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5168,"num_write_kb":195584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":687,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.739782+0000","last_change":"2023-10-09T10:54:50.914888+0000","last_active":"2023-10-09T10:56:10.739782+0000","last_peered":"2023-10-09T10:56:10.739782+0000","last_clean":"2023-10-09T10:56:10.739782+0000","last_became_active":"2023-10-09T10:54:50.909639+0000","last_became_peered":"2023-10-09T10:54:50.909639+0000","last_unstale":"2023-10-09T10:56:10.739782+0000","last_undegraded":"2023-10-09T10:56:10.739782+0000","last_fullsized":"2023-10-09T10:56:10.739782+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.103750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.103750+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":349,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.758667+0000","last_change":"2023-10-09T10:54:17.876795+0000","last_active":"2023-10-09T10:56:13.758667+0000","last_peered":"2023-10-09T10:56:13.758667+0000","last_clean":"2023-10-09T10:56:13.758667+0000","last_became_active":"2023-10-09T10:54:17.876504+0000","last_became_peered":"2023-10-09T10:54:17.876504+0000","last_unstale":"2023-10-09T10:56:13.758667+0000","last_undegraded":"2023-10-09T10:56:13.758667+0000","last_fullsized":"2023-10-09T10:56:13.758667+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:04:48.657295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.733252+0000","last_change":"2023-10-09T10:54:47.913826+0000","last_active":"2023-10-09T10:56:10.733252+0000","last_peered":"2023-10-09T10:56:10.733252+0000","last_clean":"2023-10-09T10:56:10.733252+0000","last_became_active":"2023-10-09T10:54:47.913524+0000","last_became_peered":"2023-10-09T10:54:47.913524+0000","last_unstale":"2023-10-09T10:56:10.733252+0000","last_undegraded":"2023-10-09T10:56:10.733252+0000","last_fullsized":"2023-10-09T10:56:10.733252+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:56:23.699552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"313'7937","reported_seq":8056,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:16.729939+0000","last_change":"2023-10-09T10:54:50.915546+0000","last_active":"2023-10-09T10:56:16.729939+0000","last_peered":"2023-10-09T10:56:16.729939+0000","last_clean":"2023-10-09T10:56:16.729939+0000","last_became_active":"2023-10-09T10:54:50.910131+0000","last_became_peered":"2023-10-09T10:54:50.910131+0000","last_unstale":"2023-10-09T10:56:16.729939+0000","last_undegraded":"2023-10-09T10:56:16.729939+0000","last_fullsized":"2023-10-09T10:56:16.729939+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":265,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7937,"log_dups_size":0,"ondisk_log_size":7937,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65634304,"num_objects":1001,"num_object_clones":0,"num_object_copies":3003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1001,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4979,"num_write_kb":191392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":2031616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":628,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.792118+0000","last_change":"2023-10-09T10:54:21.152148+0000","last_active":"2023-10-09T10:56:13.792118+0000","last_peered":"2023-10-09T10:56:13.792118+0000","last_clean":"2023-10-09T10:56:13.792118+0000","last_became_active":"2023-10-09T10:54:21.151809+0000","last_became_peered":"2023-10-09T10:54:21.151809+0000","last_unstale":"2023-10-09T10:56:13.792118+0000","last_undegraded":"2023-10-09T10:56:13.792118+0000","last_fullsized":"2023-10-09T10:56:13.792118+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.701128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.701128+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:27.021145+0000","scrub_duration":0.0025071830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":341,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.764733+0000","last_change":"2023-10-09T10:54:48.038946+0000","last_active":"2023-10-09T10:56:13.764733+0000","last_peered":"2023-10-09T10:56:13.764733+0000","last_clean":"2023-10-09T10:56:13.764733+0000","last_became_active":"2023-10-09T10:54:48.001493+0000","last_became_peered":"2023-10-09T10:54:48.001493+0000","last_unstale":"2023-10-09T10:56:13.764733+0000","last_undegraded":"2023-10-09T10:56:13.764733+0000","last_fullsized":"2023-10-09T10:56:13.764733+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:25.197466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.734013+0000","last_change":"2023-10-09T10:54:48.012411+0000","last_active":"2023-10-09T10:56:10.734013+0000","last_peered":"2023-10-09T10:56:10.734013+0000","last_clean":"2023-10-09T10:56:10.734013+0000","last_became_active":"2023-10-09T10:54:48.011894+0000","last_became_peered":"2023-10-09T10:54:48.011894+0000","last_unstale":"2023-10-09T10:56:10.734013+0000","last_undegraded":"2023-10-09T10:56:10.734013+0000","last_fullsized":"2023-10-09T10:56:10.734013+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:28.067337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"313'10104","reported_seq":13113,"reported_epoch":313,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:18.498963+0000","last_change":"2023-10-09T10:55:24.700754+0000","last_active":"2023-10-09T10:56:18.498963+0000","last_peered":"2023-10-09T10:56:18.498963+0000","last_clean":"2023-10-09T10:54:17.076172+0000","last_became_active":"2023-10-09T10:54:19.327879+0000","last_became_peered":"2023-10-09T10:54:19.327879+0000","last_unstale":"2023-10-09T10:56:18.498963+0000","last_undegraded":"2023-10-09T10:54:18.890724+0000","last_fullsized":"2023-10-09T10:54:18.868702+0000","mapping_epoch":282,"log_start":"279'4752","ondisk_log_start":"279'4752","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5352,"log_dups_size":0,"ondisk_log_size":5352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:49.273305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135528448,"num_objects":2068,"num_object_clones":0,"num_object_copies":6204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2547,"num_objects_misplaced":1274,"num_objects_unfound":0,"num_objects_dirty":2068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10104,"num_write_kb":389504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1471,"num_bytes_recovered":96403456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-1753},{"shards":"0,4,7","objects":1274}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":711,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.767997+0000","last_change":"2023-10-09T10:53:34.602531+0000","last_active":"2023-10-09T10:56:13.767997+0000","last_peered":"2023-10-09T10:56:13.767997+0000","last_clean":"2023-10-09T10:56:13.767997+0000","last_became_active":"2023-10-09T10:53:17.841970+0000","last_became_peered":"2023-10-09T10:53:17.841970+0000","last_unstale":"2023-10-09T10:56:13.767997+0000","last_undegraded":"2023-10-09T10:56:13.767997+0000","last_fullsized":"2023-10-09T10:56:13.767997+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:04.601861+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.732530+0000","last_change":"2023-10-09T10:54:47.952216+0000","last_active":"2023-10-09T10:56:10.732530+0000","last_peered":"2023-10-09T10:56:10.732530+0000","last_clean":"2023-10-09T10:56:10.732530+0000","last_became_active":"2023-10-09T10:54:47.941070+0000","last_became_peered":"2023-10-09T10:54:47.941070+0000","last_unstale":"2023-10-09T10:56:10.732530+0000","last_undegraded":"2023-10-09T10:56:10.732530+0000","last_fullsized":"2023-10-09T10:56:10.732530+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:09.075266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":234,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.764125+0000","last_change":"2023-10-09T10:54:17.886921+0000","last_active":"2023-10-09T10:56:13.764125+0000","last_peered":"2023-10-09T10:56:13.764125+0000","last_clean":"2023-10-09T10:56:13.764125+0000","last_became_active":"2023-10-09T10:54:17.886584+0000","last_became_peered":"2023-10-09T10:54:17.886584+0000","last_unstale":"2023-10-09T10:56:13.764125+0000","last_undegraded":"2023-10-09T10:56:13.764125+0000","last_fullsized":"2023-10-09T10:56:13.764125+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:15.848951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.732333+0000","last_change":"2023-10-09T10:54:47.908294+0000","last_active":"2023-10-09T10:56:10.732333+0000","last_peered":"2023-10-09T10:56:10.732333+0000","last_clean":"2023-10-09T10:56:10.732333+0000","last_became_active":"2023-10-09T10:54:47.908019+0000","last_became_peered":"2023-10-09T10:54:47.908019+0000","last_unstale":"2023-10-09T10:56:10.732333+0000","last_undegraded":"2023-10-09T10:56:10.732333+0000","last_fullsized":"2023-10-09T10:56:10.732333+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:48.662764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"314'8046","reported_seq":8115,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:20.468388+0000","last_change":"2023-10-09T10:54:40.642047+0000","last_active":"2023-10-09T10:56:20.468388+0000","last_peered":"2023-10-09T10:56:20.468388+0000","last_clean":"2023-10-09T10:56:20.468388+0000","last_became_active":"2023-10-09T10:54:40.639305+0000","last_became_peered":"2023-10-09T10:54:40.639305+0000","last_unstale":"2023-10-09T10:56:20.468388+0000","last_undegraded":"2023-10-09T10:56:20.468388+0000","last_fullsized":"2023-10-09T10:56:20.468388+0000","mapping_epoch":266,"log_start":"271'1000","ondisk_log_start":"271'1000","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7046,"log_dups_size":0,"ondisk_log_size":7046,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65142784,"num_objects":994,"num_object_clones":0,"num_object_copies":2982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5056,"num_write_kb":193600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":650,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.715658+0000","last_change":"2023-10-09T10:54:21.163072+0000","last_active":"2023-10-09T10:56:10.715658+0000","last_peered":"2023-10-09T10:56:10.715658+0000","last_clean":"2023-10-09T10:56:10.715658+0000","last_became_active":"2023-10-09T10:54:21.162710+0000","last_became_peered":"2023-10-09T10:54:21.162710+0000","last_unstale":"2023-10-09T10:56:10.715658+0000","last_undegraded":"2023-10-09T10:56:10.715658+0000","last_fullsized":"2023-10-09T10:56:10.715658+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:15.932806+0000","scrub_duration":0.0019146790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.732402+0000","last_change":"2023-10-09T10:54:47.899967+0000","last_active":"2023-10-09T10:56:10.732402+0000","last_peered":"2023-10-09T10:56:10.732402+0000","last_clean":"2023-10-09T10:56:10.732402+0000","last_became_active":"2023-10-09T10:54:47.899584+0000","last_became_peered":"2023-10-09T10:54:47.899584+0000","last_unstale":"2023-10-09T10:56:10.732402+0000","last_undegraded":"2023-10-09T10:56:10.732402+0000","last_fullsized":"2023-10-09T10:56:10.732402+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:43.973868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":222,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.712222+0000","last_change":"2023-10-09T10:50:43.388615+0000","last_active":"2023-10-09T10:56:10.712222+0000","last_peered":"2023-10-09T10:56:10.712222+0000","last_clean":"2023-10-09T10:56:10.712222+0000","last_became_active":"2023-10-09T10:50:43.388176+0000","last_became_peered":"2023-10-09T10:50:43.388176+0000","last_unstale":"2023-10-09T10:56:10.712222+0000","last_undegraded":"2023-10-09T10:56:10.712222+0000","last_fullsized":"2023-10-09T10:56:10.712222+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:41:17.550111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.732459+0000","last_change":"2023-10-09T10:54:47.950462+0000","last_active":"2023-10-09T10:56:10.732459+0000","last_peered":"2023-10-09T10:56:10.732459+0000","last_clean":"2023-10-09T10:56:10.732459+0000","last_became_active":"2023-10-09T10:54:47.940606+0000","last_became_peered":"2023-10-09T10:54:47.940606+0000","last_unstale":"2023-10-09T10:56:10.732459+0000","last_undegraded":"2023-10-09T10:56:10.732459+0000","last_fullsized":"2023-10-09T10:56:10.732459+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:50.557850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"314'10217","reported_seq":10303,"reported_epoch":314,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:20.382730+0000","last_change":"2023-10-09T10:54:49.182381+0000","last_active":"2023-10-09T10:56:20.382730+0000","last_peered":"2023-10-09T10:56:20.382730+0000","last_clean":"2023-10-09T10:54:02.429830+0000","last_became_active":"2023-10-09T10:54:49.176829+0000","last_became_peered":"2023-10-09T10:54:49.176829+0000","last_unstale":"2023-10-09T10:56:20.382730+0000","last_undegraded":"2023-10-09T10:54:48.956857+0000","last_fullsized":"2023-10-09T10:54:48.892711+0000","mapping_epoch":295,"log_start":"280'5155","ondisk_log_start":"280'5155","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5062,"log_dups_size":0,"ondisk_log_size":5062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:04.678256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125370368,"num_objects":1913,"num_object_clones":0,"num_object_copies":5739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1366,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10217,"num_write_kb":388160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"3,4","objects":547},{"shards":"3,4,7","objects":1365}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":661,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.765124+0000","last_change":"2023-10-09T10:55:25.634848+0000","last_active":"2023-10-09T10:56:13.765124+0000","last_peered":"2023-10-09T10:56:13.765124+0000","last_clean":"2023-10-09T10:56:13.765124+0000","last_became_active":"2023-10-09T10:55:25.634509+0000","last_became_peered":"2023-10-09T10:55:25.634509+0000","last_unstale":"2023-10-09T10:56:13.765124+0000","last_undegraded":"2023-10-09T10:56:13.765124+0000","last_fullsized":"2023-10-09T10:56:13.765124+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":44,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.615596+0000","last_change":"2023-10-09T10:54:17.885030+0000","last_active":"2023-10-09T10:56:19.615596+0000","last_peered":"2023-10-09T10:56:19.615596+0000","last_clean":"2023-10-09T10:56:19.615596+0000","last_became_active":"2023-10-09T10:54:17.884752+0000","last_became_peered":"2023-10-09T10:54:17.884752+0000","last_unstale":"2023-10-09T10:56:19.615596+0000","last_undegraded":"2023-10-09T10:56:19.615596+0000","last_fullsized":"2023-10-09T10:56:19.615596+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:38.548132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.745518+0000","last_change":"2023-10-09T10:54:47.927822+0000","last_active":"2023-10-09T10:56:10.745518+0000","last_peered":"2023-10-09T10:56:10.745518+0000","last_clean":"2023-10-09T10:56:10.745518+0000","last_became_active":"2023-10-09T10:54:47.927455+0000","last_became_peered":"2023-10-09T10:54:47.927455+0000","last_unstale":"2023-10-09T10:56:10.745518+0000","last_undegraded":"2023-10-09T10:56:10.745518+0000","last_fullsized":"2023-10-09T10:56:10.745518+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:11.818203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.733949+0000","last_change":"2023-10-09T10:54:48.012629+0000","last_active":"2023-10-09T10:56:10.733949+0000","last_peered":"2023-10-09T10:56:10.733949+0000","last_clean":"2023-10-09T10:56:10.733949+0000","last_became_active":"2023-10-09T10:54:48.012148+0000","last_became_peered":"2023-10-09T10:54:48.012148+0000","last_unstale":"2023-10-09T10:56:10.733949+0000","last_undegraded":"2023-10-09T10:56:10.733949+0000","last_fullsized":"2023-10-09T10:56:10.733949+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:22.894111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"313'8025","reported_seq":8092,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:18.436081+0000","last_change":"2023-10-09T10:54:40.559886+0000","last_active":"2023-10-09T10:56:18.436081+0000","last_peered":"2023-10-09T10:56:18.436081+0000","last_clean":"2023-10-09T10:56:18.436081+0000","last_became_active":"2023-10-09T10:54:40.557153+0000","last_became_peered":"2023-10-09T10:54:40.557153+0000","last_unstale":"2023-10-09T10:56:18.436081+0000","last_undegraded":"2023-10-09T10:56:18.436081+0000","last_fullsized":"2023-10-09T10:56:18.436081+0000","mapping_epoch":265,"log_start":"270'300","ondisk_log_start":"270'300","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7725,"log_dups_size":0,"ondisk_log_size":7725,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65536000,"num_objects":1000,"num_object_clones":0,"num_object_copies":3000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5040,"num_write_kb":193280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":43,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.790650+0000","last_change":"2023-10-09T10:54:17.884876+0000","last_active":"2023-10-09T10:56:13.790650+0000","last_peered":"2023-10-09T10:56:13.790650+0000","last_clean":"2023-10-09T10:56:13.790650+0000","last_became_active":"2023-10-09T10:54:17.884402+0000","last_became_peered":"2023-10-09T10:54:17.884402+0000","last_unstale":"2023-10-09T10:56:13.790650+0000","last_undegraded":"2023-10-09T10:56:13.790650+0000","last_fullsized":"2023-10-09T10:56:13.790650+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:18.679950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":690,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.765212+0000","last_change":"2023-10-09T10:55:25.633525+0000","last_active":"2023-10-09T10:56:13.765212+0000","last_peered":"2023-10-09T10:56:13.765212+0000","last_clean":"2023-10-09T10:56:13.765212+0000","last_became_active":"2023-10-09T10:55:25.633079+0000","last_became_peered":"2023-10-09T10:55:25.633079+0000","last_unstale":"2023-10-09T10:56:13.765212+0000","last_undegraded":"2023-10-09T10:56:13.765212+0000","last_fullsized":"2023-10-09T10:56:13.765212+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:37.378259+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":246,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.765106+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:56:13.765106+0000","last_peered":"2023-10-09T10:56:13.765106+0000","last_clean":"2023-10-09T10:56:13.765106+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:56:13.765106+0000","last_undegraded":"2023-10-09T10:56:13.765106+0000","last_fullsized":"2023-10-09T10:56:13.765106+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":226,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.759168+0000","last_change":"2023-10-09T10:53:14.836523+0000","last_active":"2023-10-09T10:56:13.759168+0000","last_peered":"2023-10-09T10:56:13.759168+0000","last_clean":"2023-10-09T10:56:13.759168+0000","last_became_active":"2023-10-09T10:53:14.836134+0000","last_became_peered":"2023-10-09T10:53:14.836134+0000","last_unstale":"2023-10-09T10:56:13.759168+0000","last_undegraded":"2023-10-09T10:56:13.759168+0000","last_fullsized":"2023-10-09T10:56:13.759168+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:05:24.615198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"314'10027","reported_seq":10844,"reported_epoch":314,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:20.477578+0000","last_change":"2023-10-09T10:54:49.078027+0000","last_active":"2023-10-09T10:56:20.477578+0000","last_peered":"2023-10-09T10:56:20.477578+0000","last_clean":"2023-10-09T10:54:02.408036+0000","last_became_active":"2023-10-09T10:54:49.069949+0000","last_became_peered":"2023-10-09T10:54:49.069949+0000","last_unstale":"2023-10-09T10:56:20.477578+0000","last_undegraded":"2023-10-09T10:54:49.050557+0000","last_fullsized":"2023-10-09T10:54:48.898155+0000","mapping_epoch":295,"log_start":"280'4972","ondisk_log_start":"280'4972","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5055,"log_dups_size":0,"ondisk_log_size":5055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:38.092075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126550016,"num_objects":1931,"num_object_clones":0,"num_object_copies":5793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1353,"num_objects_misplaced":1931,"num_objects_unfound":0,"num_objects_dirty":1931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10027,"num_write_kb":382656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":366,"num_bytes_recovered":23986176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":1353},{"shards":"3,5","objects":578}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":660,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.739554+0000","last_change":"2023-10-09T10:54:50.915855+0000","last_active":"2023-10-09T10:56:10.739554+0000","last_peered":"2023-10-09T10:56:10.739554+0000","last_clean":"2023-10-09T10:56:10.739554+0000","last_became_active":"2023-10-09T10:54:50.914536+0000","last_became_peered":"2023-10-09T10:54:50.914536+0000","last_unstale":"2023-10-09T10:56:10.739554+0000","last_undegraded":"2023-10-09T10:56:10.739554+0000","last_fullsized":"2023-10-09T10:56:10.739554+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.070026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.070026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":264,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.765136+0000","last_change":"2023-10-09T10:55:25.633308+0000","last_active":"2023-10-09T10:56:13.765136+0000","last_peered":"2023-10-09T10:56:13.765136+0000","last_clean":"2023-10-09T10:56:13.765136+0000","last_became_active":"2023-10-09T10:55:25.632646+0000","last_became_peered":"2023-10-09T10:55:25.632646+0000","last_unstale":"2023-10-09T10:56:13.765136+0000","last_undegraded":"2023-10-09T10:56:13.765136+0000","last_fullsized":"2023-10-09T10:56:13.765136+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:17:13.735888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":194,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.715426+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:56:10.715426+0000","last_peered":"2023-10-09T10:56:10.715426+0000","last_clean":"2023-10-09T10:56:10.715426+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:56:10.715426+0000","last_undegraded":"2023-10-09T10:56:10.715426+0000","last_fullsized":"2023-10-09T10:56:10.715426+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":257,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.718437+0000","last_change":"2023-10-09T10:54:47.939756+0000","last_active":"2023-10-09T10:56:10.718437+0000","last_peered":"2023-10-09T10:56:10.718437+0000","last_clean":"2023-10-09T10:56:10.718437+0000","last_became_active":"2023-10-09T10:54:47.938928+0000","last_became_peered":"2023-10-09T10:54:47.938928+0000","last_unstale":"2023-10-09T10:56:10.718437+0000","last_undegraded":"2023-10-09T10:56:10.718437+0000","last_fullsized":"2023-10-09T10:56:10.718437+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:05.310929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"314'10268","reported_seq":10356,"reported_epoch":314,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:20.479599+0000","last_change":"2023-10-09T10:54:49.192234+0000","last_active":"2023-10-09T10:56:20.479599+0000","last_peered":"2023-10-09T10:56:20.479599+0000","last_clean":"2023-10-09T10:54:02.435746+0000","last_became_active":"2023-10-09T10:54:49.186762+0000","last_became_peered":"2023-10-09T10:54:49.186762+0000","last_unstale":"2023-10-09T10:56:20.479599+0000","last_undegraded":"2023-10-09T10:54:49.038902+0000","last_fullsized":"2023-10-09T10:54:48.912839+0000","mapping_epoch":295,"log_start":"280'5171","ondisk_log_start":"280'5171","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5097,"log_dups_size":0,"ondisk_log_size":5097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:48.327483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124649472,"num_objects":1902,"num_object_clones":0,"num_object_copies":5706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1330,"num_objects_misplaced":1902,"num_objects_unfound":0,"num_objects_dirty":1902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10268,"num_write_kb":389440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":572},{"shards":"3,4,6","objects":1328}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":685,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.718655+0000","last_change":"2023-10-09T10:53:27.625180+0000","last_active":"2023-10-09T10:56:10.718655+0000","last_peered":"2023-10-09T10:56:10.718655+0000","last_clean":"2023-10-09T10:56:10.718655+0000","last_became_active":"2023-10-09T10:51:12.366828+0000","last_became_peered":"2023-10-09T10:51:12.366828+0000","last_unstale":"2023-10-09T10:56:10.718655+0000","last_undegraded":"2023-10-09T10:56:10.718655+0000","last_fullsized":"2023-10-09T10:56:10.718655+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.625079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.625079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:59:41.253976+0000","scrub_duration":0.0041067969999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":57,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.766634+0000","last_change":"2023-10-09T10:53:14.833073+0000","last_active":"2023-10-09T10:56:13.766634+0000","last_peered":"2023-10-09T10:56:13.766634+0000","last_clean":"2023-10-09T10:56:13.766634+0000","last_became_active":"2023-10-09T10:53:14.832736+0000","last_became_peered":"2023-10-09T10:53:14.832736+0000","last_unstale":"2023-10-09T10:56:13.766634+0000","last_undegraded":"2023-10-09T10:56:13.766634+0000","last_fullsized":"2023-10-09T10:56:13.766634+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:27.550707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.732700+0000","last_change":"2023-10-09T10:54:47.925711+0000","last_active":"2023-10-09T10:56:10.732700+0000","last_peered":"2023-10-09T10:56:10.732700+0000","last_clean":"2023-10-09T10:56:10.732700+0000","last_became_active":"2023-10-09T10:54:47.924286+0000","last_became_peered":"2023-10-09T10:54:47.924286+0000","last_unstale":"2023-10-09T10:56:10.732700+0000","last_undegraded":"2023-10-09T10:56:10.732700+0000","last_fullsized":"2023-10-09T10:56:10.732700+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:58.379080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":221,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.712349+0000","last_change":"2023-10-09T10:53:14.828291+0000","last_active":"2023-10-09T10:56:10.712349+0000","last_peered":"2023-10-09T10:56:10.712349+0000","last_clean":"2023-10-09T10:56:10.712349+0000","last_became_active":"2023-10-09T10:53:14.827991+0000","last_became_peered":"2023-10-09T10:53:14.827991+0000","last_unstale":"2023-10-09T10:56:10.712349+0000","last_undegraded":"2023-10-09T10:56:10.712349+0000","last_fullsized":"2023-10-09T10:56:10.712349+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:08:14.283112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"313'10161","reported_seq":10220,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:18.493775+0000","last_change":"2023-10-09T10:53:14.822279+0000","last_active":"2023-10-09T10:56:18.493775+0000","last_peered":"2023-10-09T10:56:18.493775+0000","last_clean":"2023-10-09T10:56:18.493775+0000","last_became_active":"2023-10-09T10:53:14.821952+0000","last_became_peered":"2023-10-09T10:53:14.821952+0000","last_unstale":"2023-10-09T10:56:18.493775+0000","last_undegraded":"2023-10-09T10:56:18.493775+0000","last_fullsized":"2023-10-09T10:56:18.493775+0000","mapping_epoch":266,"log_start":"280'4841","ondisk_log_start":"280'4841","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5320,"log_dups_size":0,"ondisk_log_size":5320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:37:14.124652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130613248,"num_objects":1993,"num_object_clones":0,"num_object_copies":5979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1993,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10161,"num_write_kb":388928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":709,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.759095+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T10:56:13.759095+0000","last_peered":"2023-10-09T10:56:13.759095+0000","last_clean":"2023-10-09T10:56:13.759095+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T10:56:13.759095+0000","last_undegraded":"2023-10-09T10:56:13.759095+0000","last_fullsized":"2023-10-09T10:56:13.759095+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":336,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.768227+0000","last_change":"2023-10-09T10:53:14.841466+0000","last_active":"2023-10-09T10:56:13.768227+0000","last_peered":"2023-10-09T10:56:13.768227+0000","last_clean":"2023-10-09T10:56:13.768227+0000","last_became_active":"2023-10-09T10:53:14.831674+0000","last_became_peered":"2023-10-09T10:53:14.831674+0000","last_unstale":"2023-10-09T10:56:13.768227+0000","last_undegraded":"2023-10-09T10:56:13.768227+0000","last_fullsized":"2023-10-09T10:56:13.768227+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:02.612069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":29,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.733370+0000","last_change":"2023-10-09T10:54:47.908305+0000","last_active":"2023-10-09T10:56:10.733370+0000","last_peered":"2023-10-09T10:56:10.733370+0000","last_clean":"2023-10-09T10:56:10.733370+0000","last_became_active":"2023-10-09T10:54:47.907999+0000","last_became_peered":"2023-10-09T10:54:47.907999+0000","last_unstale":"2023-10-09T10:56:10.733370+0000","last_undegraded":"2023-10-09T10:56:10.733370+0000","last_fullsized":"2023-10-09T10:56:10.733370+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:48.405124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":336,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.764791+0000","last_change":"2023-10-09T10:53:14.835841+0000","last_active":"2023-10-09T10:56:13.764791+0000","last_peered":"2023-10-09T10:56:13.764791+0000","last_clean":"2023-10-09T10:56:13.764791+0000","last_became_active":"2023-10-09T10:53:14.835551+0000","last_became_peered":"2023-10-09T10:53:14.835551+0000","last_unstale":"2023-10-09T10:56:13.764791+0000","last_undegraded":"2023-10-09T10:56:13.764791+0000","last_fullsized":"2023-10-09T10:56:13.764791+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:21.498099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"314'10043","reported_seq":14225,"reported_epoch":314,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:21.150363+0000","last_change":"2023-10-09T10:55:26.696179+0000","last_active":"2023-10-09T10:56:21.150363+0000","last_peered":"2023-10-09T10:56:21.150363+0000","last_clean":"2023-10-09T10:54:17.082254+0000","last_became_active":"2023-10-09T10:55:26.641906+0000","last_became_peered":"2023-10-09T10:55:26.641906+0000","last_unstale":"2023-10-09T10:56:21.150363+0000","last_undegraded":"2023-10-09T10:55:26.633572+0000","last_fullsized":"2023-10-09T10:55:26.625249+0000","mapping_epoch":306,"log_start":"275'4349","ondisk_log_start":"275'4349","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5694,"log_dups_size":0,"ondisk_log_size":5694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:02.762689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124780568,"num_objects":1905,"num_object_clones":0,"num_object_copies":5715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3090,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1905,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10043,"num_write_kb":382273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2055,"num_bytes_recovered":134610944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5],"avail_no_missing":["7","5","6"],"object_location_counts":[{"shards":"5,6,7","objects":1076},{"shards":"5,7","objects":-1185}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":705,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.764821+0000","last_change":"2023-10-09T10:53:30.567189+0000","last_active":"2023-10-09T10:56:13.764821+0000","last_peered":"2023-10-09T10:56:13.764821+0000","last_clean":"2023-10-09T10:56:13.764821+0000","last_became_active":"2023-10-09T10:53:17.840928+0000","last_became_peered":"2023-10-09T10:53:17.840928+0000","last_unstale":"2023-10-09T10:56:13.764821+0000","last_undegraded":"2023-10-09T10:56:13.764821+0000","last_fullsized":"2023-10-09T10:56:13.764821+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:52.176229+0000","scrub_duration":0.0053707060000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":167,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.765765+0000","last_change":"2023-10-09T10:55:25.634235+0000","last_active":"2023-10-09T10:56:13.765765+0000","last_peered":"2023-10-09T10:56:13.765765+0000","last_clean":"2023-10-09T10:56:13.765765+0000","last_became_active":"2023-10-09T10:55:25.633940+0000","last_became_peered":"2023-10-09T10:55:25.633940+0000","last_unstale":"2023-10-09T10:56:13.765765+0000","last_undegraded":"2023-10-09T10:56:13.765765+0000","last_fullsized":"2023-10-09T10:56:13.765765+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:40.554244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":241,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.718516+0000","last_change":"2023-10-09T10:54:48.016159+0000","last_active":"2023-10-09T10:56:10.718516+0000","last_peered":"2023-10-09T10:56:10.718516+0000","last_clean":"2023-10-09T10:56:10.718516+0000","last_became_active":"2023-10-09T10:54:47.939409+0000","last_became_peered":"2023-10-09T10:54:47.939409+0000","last_unstale":"2023-10-09T10:56:10.718516+0000","last_undegraded":"2023-10-09T10:56:10.718516+0000","last_fullsized":"2023-10-09T10:56:10.718516+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:11:15.175207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":255,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.758627+0000","last_change":"2023-10-09T10:51:50.371282+0000","last_active":"2023-10-09T10:56:13.758627+0000","last_peered":"2023-10-09T10:56:13.758627+0000","last_clean":"2023-10-09T10:56:13.758627+0000","last_became_active":"2023-10-09T10:51:50.370973+0000","last_became_peered":"2023-10-09T10:51:50.370973+0000","last_unstale":"2023-10-09T10:56:13.758627+0000","last_undegraded":"2023-10-09T10:56:13.758627+0000","last_fullsized":"2023-10-09T10:56:13.758627+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:15.603164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"314'8136","reported_seq":11028,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:20.280743+0000","last_change":"2023-10-09T10:54:56.638107+0000","last_active":"2023-10-09T10:56:20.280743+0000","last_peered":"2023-10-09T10:56:20.280743+0000","last_clean":"2023-10-09T10:56:20.280743+0000","last_became_active":"2023-10-09T10:54:56.633188+0000","last_became_peered":"2023-10-09T10:54:56.633188+0000","last_unstale":"2023-10-09T10:56:20.280743+0000","last_undegraded":"2023-10-09T10:56:20.280743+0000","last_fullsized":"2023-10-09T10:56:20.280743+0000","mapping_epoch":299,"log_start":"271'1100","ondisk_log_start":"271'1100","created":289,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7036,"log_dups_size":0,"ondisk_log_size":7036,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63733760,"num_objects":973,"num_object_clones":0,"num_object_copies":2919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5119,"num_write_kb":194912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1061,"num_bytes_recovered":69500928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":705,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.763437+0000","last_change":"2023-10-09T10:53:29.568199+0000","last_active":"2023-10-09T10:56:13.763437+0000","last_peered":"2023-10-09T10:56:13.763437+0000","last_clean":"2023-10-09T10:56:13.763437+0000","last_became_active":"2023-10-09T10:52:55.416357+0000","last_became_peered":"2023-10-09T10:52:55.416357+0000","last_unstale":"2023-10-09T10:56:13.763437+0000","last_undegraded":"2023-10-09T10:56:13.763437+0000","last_fullsized":"2023-10-09T10:56:13.763437+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:44.364650+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":221,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.712425+0000","last_change":"2023-10-09T10:53:14.823731+0000","last_active":"2023-10-09T10:56:10.712425+0000","last_peered":"2023-10-09T10:56:10.712425+0000","last_clean":"2023-10-09T10:56:10.712425+0000","last_became_active":"2023-10-09T10:53:14.823470+0000","last_became_peered":"2023-10-09T10:53:14.823470+0000","last_unstale":"2023-10-09T10:56:10.712425+0000","last_undegraded":"2023-10-09T10:56:10.712425+0000","last_fullsized":"2023-10-09T10:56:10.712425+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:36.927599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":52,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.791793+0000","last_change":"2023-10-09T10:54:47.944380+0000","last_active":"2023-10-09T10:56:13.791793+0000","last_peered":"2023-10-09T10:56:13.791793+0000","last_clean":"2023-10-09T10:56:13.791793+0000","last_became_active":"2023-10-09T10:54:47.943766+0000","last_became_peered":"2023-10-09T10:54:47.943766+0000","last_unstale":"2023-10-09T10:56:13.791793+0000","last_undegraded":"2023-10-09T10:56:13.791793+0000","last_fullsized":"2023-10-09T10:56:13.791793+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:16:11.869603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"314'7989","reported_seq":8082,"reported_epoch":314,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:20.406109+0000","last_change":"2023-10-09T10:54:49.184288+0000","last_active":"2023-10-09T10:56:20.406109+0000","last_peered":"2023-10-09T10:56:20.406109+0000","last_clean":"2023-10-09T10:54:02.391425+0000","last_became_active":"2023-10-09T10:54:49.118164+0000","last_became_peered":"2023-10-09T10:54:49.118164+0000","last_unstale":"2023-10-09T10:56:20.406109+0000","last_undegraded":"2023-10-09T10:54:48.969517+0000","last_fullsized":"2023-10-09T10:54:48.892012+0000","mapping_epoch":295,"log_start":"271'1000","ondisk_log_start":"271'1000","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":6989,"log_dups_size":0,"ondisk_log_size":6989,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66486272,"num_objects":1015,"num_object_clones":0,"num_object_copies":3045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":727,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5013,"num_write_kb":192864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"3,4,6","objects":727},{"shards":"3,6","objects":288}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":636,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.764186+0000","last_change":"2023-10-09T10:53:36.563918+0000","last_active":"2023-10-09T10:56:13.764186+0000","last_peered":"2023-10-09T10:56:13.764186+0000","last_clean":"2023-10-09T10:56:13.764186+0000","last_became_active":"2023-10-09T10:53:17.842952+0000","last_became_peered":"2023-10-09T10:53:17.842952+0000","last_unstale":"2023-10-09T10:56:13.764186+0000","last_undegraded":"2023-10-09T10:56:13.764186+0000","last_fullsized":"2023-10-09T10:56:13.764186+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:53.185917+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":140,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.760721+0000","last_change":"2023-10-09T10:54:47.923386+0000","last_active":"2023-10-09T10:56:13.760721+0000","last_peered":"2023-10-09T10:56:13.760721+0000","last_clean":"2023-10-09T10:56:13.760721+0000","last_became_active":"2023-10-09T10:54:47.914257+0000","last_became_peered":"2023-10-09T10:54:47.914257+0000","last_unstale":"2023-10-09T10:56:13.760721+0000","last_undegraded":"2023-10-09T10:56:13.760721+0000","last_fullsized":"2023-10-09T10:56:13.760721+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:24:18.190715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":331,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.766367+0000","last_change":"2023-10-09T10:54:47.977518+0000","last_active":"2023-10-09T10:56:13.766367+0000","last_peered":"2023-10-09T10:56:13.766367+0000","last_clean":"2023-10-09T10:56:13.766367+0000","last_became_active":"2023-10-09T10:54:47.977332+0000","last_became_peered":"2023-10-09T10:54:47.977332+0000","last_unstale":"2023-10-09T10:56:13.766367+0000","last_undegraded":"2023-10-09T10:56:13.766367+0000","last_fullsized":"2023-10-09T10:56:13.766367+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:29.620839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.16","version":"313'8079","reported_seq":8146,"reported_epoch":313,"state":"active+recovery_wait","last_fresh":"2023-10-09T10:56:18.236394+0000","last_change":"2023-10-09T10:54:40.600782+0000","last_active":"2023-10-09T10:56:18.236394+0000","last_peered":"2023-10-09T10:56:18.236394+0000","last_clean":"2023-10-09T10:54:39.522361+0000","last_became_active":"2023-10-09T10:54:40.590969+0000","last_became_peered":"2023-10-09T10:54:40.590969+0000","last_unstale":"2023-10-09T10:56:18.236394+0000","last_undegraded":"2023-10-09T10:56:18.236394+0000","last_fullsized":"2023-10-09T10:56:18.236394+0000","mapping_epoch":289,"log_start":"270'300","ondisk_log_start":"270'300","created":289,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7779,"log_dups_size":0,"ondisk_log_size":7779,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64389120,"num_objects":983,"num_object_clones":0,"num_object_copies":2949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5085,"num_write_kb":194144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":983}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":263,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.697677+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:56:10.697677+0000","last_peered":"2023-10-09T10:56:10.697677+0000","last_clean":"2023-10-09T10:56:10.697677+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:56:10.697677+0000","last_undegraded":"2023-10-09T10:56:10.697677+0000","last_fullsized":"2023-10-09T10:56:10.697677+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":258,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.766885+0000","last_change":"2023-10-09T10:50:43.385150+0000","last_active":"2023-10-09T10:56:13.766885+0000","last_peered":"2023-10-09T10:56:13.766885+0000","last_clean":"2023-10-09T10:56:13.766885+0000","last_became_active":"2023-10-09T10:50:43.384788+0000","last_became_peered":"2023-10-09T10:50:43.384788+0000","last_unstale":"2023-10-09T10:56:13.766885+0000","last_undegraded":"2023-10-09T10:56:13.766885+0000","last_fullsized":"2023-10-09T10:56:13.766885+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:36.085910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.15","version":"314'7994","reported_seq":8064,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:21.151800+0000","last_change":"2023-10-09T10:54:40.578264+0000","last_active":"2023-10-09T10:56:21.151800+0000","last_peered":"2023-10-09T10:56:21.151800+0000","last_clean":"2023-10-09T10:56:21.151800+0000","last_became_active":"2023-10-09T10:54:40.568498+0000","last_became_peered":"2023-10-09T10:54:40.568498+0000","last_unstale":"2023-10-09T10:56:21.151800+0000","last_undegraded":"2023-10-09T10:56:21.151800+0000","last_fullsized":"2023-10-09T10:56:21.151800+0000","mapping_epoch":289,"log_start":"271'400","ondisk_log_start":"271'400","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7594,"log_dups_size":0,"ondisk_log_size":7594,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62062592,"num_objects":947,"num_object_clones":0,"num_object_copies":2841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5033,"num_write_kb":191360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":620,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.765776+0000","last_change":"2023-10-09T10:55:25.635605+0000","last_active":"2023-10-09T10:56:13.765776+0000","last_peered":"2023-10-09T10:56:13.765776+0000","last_clean":"2023-10-09T10:56:13.765776+0000","last_became_active":"2023-10-09T10:55:25.635328+0000","last_became_peered":"2023-10-09T10:55:25.635328+0000","last_unstale":"2023-10-09T10:56:13.765776+0000","last_undegraded":"2023-10-09T10:56:13.765776+0000","last_fullsized":"2023-10-09T10:56:13.765776+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:46.305288+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:46.305288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":338,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.764523+0000","last_change":"2023-10-09T10:54:17.886158+0000","last_active":"2023-10-09T10:56:13.764523+0000","last_peered":"2023-10-09T10:56:13.764523+0000","last_clean":"2023-10-09T10:56:13.764523+0000","last_became_active":"2023-10-09T10:54:17.885816+0000","last_became_peered":"2023-10-09T10:54:17.885816+0000","last_unstale":"2023-10-09T10:56:13.764523+0000","last_undegraded":"2023-10-09T10:56:13.764523+0000","last_fullsized":"2023-10-09T10:56:13.764523+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:05.270825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":341,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.763707+0000","last_change":"2023-10-09T10:54:17.957784+0000","last_active":"2023-10-09T10:56:13.763707+0000","last_peered":"2023-10-09T10:56:13.763707+0000","last_clean":"2023-10-09T10:56:13.763707+0000","last_became_active":"2023-10-09T10:54:17.954788+0000","last_became_peered":"2023-10-09T10:54:17.954788+0000","last_unstale":"2023-10-09T10:56:13.763707+0000","last_undegraded":"2023-10-09T10:56:13.763707+0000","last_fullsized":"2023-10-09T10:56:13.763707+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:48.709390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"314'8088","reported_seq":8169,"reported_epoch":314,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:56:21.264323+0000","last_change":"2023-10-09T10:54:42.558777+0000","last_active":"2023-10-09T10:56:21.264323+0000","last_peered":"2023-10-09T10:56:21.264323+0000","last_clean":"2023-10-09T10:54:17.036595+0000","last_became_active":"2023-10-09T10:54:42.553632+0000","last_became_peered":"2023-10-09T10:54:42.553632+0000","last_unstale":"2023-10-09T10:56:21.264323+0000","last_undegraded":"2023-10-09T10:56:21.264323+0000","last_fullsized":"2023-10-09T10:56:21.264323+0000","mapping_epoch":291,"log_start":"271'600","ondisk_log_start":"271'600","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7488,"log_dups_size":0,"ondisk_log_size":7488,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60719104,"num_objects":927,"num_object_clones":0,"num_object_copies":2781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":927,"num_objects_unfound":0,"num_objects_dirty":927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5103,"num_write_kb":192928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":927}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":587,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.732985+0000","last_change":"2023-10-09T10:54:50.914473+0000","last_active":"2023-10-09T10:56:10.732985+0000","last_peered":"2023-10-09T10:56:10.732985+0000","last_clean":"2023-10-09T10:56:10.732985+0000","last_became_active":"2023-10-09T10:54:50.914005+0000","last_became_peered":"2023-10-09T10:54:50.914005+0000","last_unstale":"2023-10-09T10:56:10.732985+0000","last_undegraded":"2023-10-09T10:56:10.732985+0000","last_fullsized":"2023-10-09T10:56:10.732985+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.070492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.070492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":221,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-10-09T10:56:10.714286+0000","last_change":"2023-10-09T10:54:18.089053+0000","last_active":"2023-10-09T10:56:10.714286+0000","last_peered":"2023-10-09T10:56:10.714286+0000","last_clean":"2023-10-09T10:56:10.714286+0000","last_became_active":"2023-10-09T10:54:17.886070+0000","last_became_peered":"2023-10-09T10:54:17.886070+0000","last_unstale":"2023-10-09T10:56:10.714286+0000","last_undegraded":"2023-10-09T10:56:10.714286+0000","last_fullsized":"2023-10-09T10:56:10.714286+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:35.628407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":318,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-10-09T10:56:13.758574+0000","last_change":"2023-10-09T10:54:47.908059+0000","last_active":"2023-10-09T10:56:13.758574+0000","last_peered":"2023-10-09T10:56:13.758574+0000","last_clean":"2023-10-09T10:56:13.758574+0000","last_became_active":"2023-10-09T10:54:47.907780+0000","last_became_peered":"2023-10-09T10:54:47.907780+0000","last_unstale":"2023-10-09T10:56:13.758574+0000","last_undegraded":"2023-10-09T10:56:13.758574+0000","last_fullsized":"2023-10-09T10:56:13.758574+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:55:59.064617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":6668,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":2054029336,"num_objects":31343,"num_object_clones":0,"num_object_copies":94029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14068,"num_objects_misplaced":6962,"num_objects_unfound":0,"num_objects_dirty":31343,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":161961,"num_write_kb":6185665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9627,"num_bytes_recovered":630849536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7130984448,"data_stored":7130972232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":181248,"ondisk_log_size":181248,"up":78,"acting":68,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":729016,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":115,"num_read_kb":214,"num_write":56,"num_write_kb":567,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1105158,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332357,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1356824,"kb_used_data":1047248,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92388328,"statfs":{"total":95995035648,"available":94605647872,"internally_reserved":0,"allocated":1072381952,"data_stored":1071240103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1429999999999998,"5min":1.9299999999999999,"15min":1.365},"min":{"1min":0.23599999999999999,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":7.4080000000000004,"5min":9.6259999999999994,"15min":9.6379999999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":2.3650000000000002,"5min":1.8839999999999999,"15min":1.55},"min":{"1min":0.28699999999999998,"5min":0.246,"15min":0.24299999999999999},"max":{"1min":8.7739999999999991,"5min":8.7739999999999991,"15min":16.731000000000002},"last":0.51700000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:56:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1099999999999999,"5min":2.2309999999999999,"15min":1.706},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.25700000000000001},"max":{"1min":9.2349999999999994,"5min":34.756,"15min":34.756},"last":0.47299999999999998},{"interface":"front","average":{"1min":2.4039999999999999,"5min":1.9750000000000001,"15min":1.444},"min":{"1min":0.26100000000000001,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":11.048,"5min":35.658999999999999,"15min":35.658999999999999},"last":5.5880000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:55:29 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":1.3480000000000001,"15min":1.3089999999999999},"min":{"1min":0.32300000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.8580000000000001,"5min":8.3620000000000001,"15min":11.058},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.401,"5min":1.5660000000000001,"15min":1.232},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":4.8890000000000002,"5min":11.579000000000001,"15min":11.579000000000001},"last":0.5}]},{"osd":3,"last update":"Mon Oct 9 10:55:55 2023","interfaces":[{"interface":"back","average":{"1min":2.056,"5min":1.9490000000000001,"15min":1.7050000000000001},"min":{"1min":0.32900000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":8.7260000000000009,"5min":10.069000000000001,"15min":11.523},"last":11.760999999999999},{"interface":"front","average":{"1min":1.9930000000000001,"5min":1.52,"15min":1.526},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.585,"5min":8.6300000000000008,"15min":19.234999999999999},"last":1.585}]},{"osd":4,"last update":"Mon Oct 9 10:55:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.3819999999999999,"15min":1.214},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.0350000000000001,"5min":16.695,"15min":16.695},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":1.1990000000000001,"15min":1.0940000000000001},"min":{"1min":0.28699999999999998,"5min":0.28199999999999997,"15min":0.26200000000000001},"max":{"1min":2.2480000000000002,"5min":9.0459999999999994,"15min":11.109999999999999},"last":0.48499999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:55:34 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.218,"15min":1.1519999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.247},"max":{"1min":1.919,"5min":13.262,"15min":13.262},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":1.365,"15min":1.331},"min":{"1min":0.31900000000000001,"5min":0.318,"15min":0.29599999999999999},"max":{"1min":1.5649999999999999,"5min":8.8659999999999997,"15min":11.741},"last":0.89700000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:55:34 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":1.679,"15min":1.4259999999999999},"min":{"1min":0.34399999999999997,"5min":0.317,"15min":0.28199999999999997},"max":{"1min":1.9850000000000001,"5min":14.913,"15min":14.913},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.72399999999999998,"5min":1.4279999999999999,"15min":1.2709999999999999},"min":{"1min":0.32900000000000001,"5min":0.317,"15min":0.29399999999999998},"max":{"1min":3.3799999999999999,"5min":12.268000000000001,"15min":19.187999999999999},"last":0.42699999999999999}]}]},{"osd":6,"up_from":44,"seq":188978561288,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2044676,"kb_used_data":1152700,"kb_used_omap":0,"kb_used_meta":891968,"kb_avail":91700476,"statfs":{"total":95995035648,"available":93901287424,"internally_reserved":0,"allocated":1180364800,"data_stored":1179242717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913375232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:55 2023","interfaces":[{"interface":"back","average":{"1min":5.0620000000000003,"5min":1.9490000000000001,"15min":1.4770000000000001},"min":{"1min":0.22800000000000001,"5min":0.216,"15min":0.216},"max":{"1min":54.743000000000002,"5min":54.743000000000002,"15min":54.743000000000002},"last":1.1319999999999999},{"interface":"front","average":{"1min":4.1219999999999999,"5min":1.97,"15min":1.4850000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.22800000000000001},"max":{"1min":53.701000000000001,"5min":53.701000000000001,"15min":53.701000000000001},"last":1.028}]},{"osd":1,"last update":"Mon Oct 9 10:56:13 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":1.2949999999999999,"15min":1.331},"min":{"1min":0.29199999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.2779999999999996,"5min":8.4730000000000008,"15min":16.747},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.488,"5min":1.53,"15min":1.3839999999999999},"min":{"1min":0.32000000000000001,"5min":0.23599999999999999,"15min":0.23100000000000001},"max":{"1min":4.2549999999999999,"5min":8.0579999999999998,"15min":16.734000000000002},"last":0.44800000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:55:33 2023","interfaces":[{"interface":"back","average":{"1min":2.1309999999999998,"5min":1.958,"15min":1.401},"min":{"1min":0.29199999999999998,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":8.0589999999999993,"5min":8.6389999999999993,"15min":13.208},"last":2.9089999999999998},{"interface":"front","average":{"1min":1.8999999999999999,"5min":2.1309999999999998,"15min":1.5609999999999999},"min":{"1min":0.36299999999999999,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":8.6890000000000001,"5min":12.83,"15min":12.83},"last":2.2679999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:55:55 2023","interfaces":[{"interface":"back","average":{"1min":1.837,"5min":1.768,"15min":1.4510000000000001},"min":{"1min":0.32500000000000001,"5min":0.29799999999999999,"15min":0.255},"max":{"1min":5.8399999999999999,"5min":7.4720000000000004,"15min":12.391},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.1949999999999998,"5min":2.0739999999999998,"15min":1.667},"min":{"1min":0.38600000000000001,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":8.3759999999999994,"5min":8.3759999999999994,"15min":13.179},"last":2.1200000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:55:28 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.7010000000000001,"15min":1.171},"min":{"1min":0.251,"5min":0.247,"15min":0.23200000000000001},"max":{"1min":16.978999999999999,"5min":16.978999999999999,"15min":16.978999999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.69399999999999995,"5min":1.347,"15min":1.1279999999999999},"min":{"1min":0.311,"5min":0.308,"15min":0.26400000000000001},"max":{"1min":3.6709999999999998,"5min":12.061999999999999,"15min":12.481999999999999},"last":0.41899999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:55:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":1.869,"15min":1.292},"min":{"1min":0.28599999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":12.923,"5min":12.923,"15min":12.923},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.71799999999999997,"5min":1.532,"15min":1.1910000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.3100000000000001,"5min":12.103999999999999,"15min":12.456},"last":0.439}]},{"osd":7,"last update":"Mon Oct 9 10:55:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":1.6950000000000001,"15min":1.3340000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.26600000000000001},"max":{"1min":2.0449999999999999,"5min":12.882,"15min":23.358000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.70099999999999996,"5min":1.149,"15min":1.1299999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.7200000000000002,"5min":7.0880000000000001,"15min":12.422000000000001},"last":0.55100000000000005}]}]},{"osd":1,"up_from":85,"seq":365072220326,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":872380,"kb_used_data":492212,"kb_used_omap":9,"kb_used_meta":380150,"kb_avail":92872772,"statfs":{"total":95995035648,"available":95101718528,"internally_reserved":0,"allocated":504025088,"data_stored":502380714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":389274601},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:56:07 2023","interfaces":[{"interface":"back","average":{"1min":4.6989999999999998,"5min":2.1320000000000001,"15min":1.74},"min":{"1min":0.26200000000000001,"5min":0.20999999999999999,"15min":0.19500000000000001},"max":{"1min":19.992999999999999,"5min":19.992999999999999,"15min":19.992999999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":4.9359999999999999,"5min":2.6230000000000002,"15min":1.8420000000000001},"min":{"1min":0.29399999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":18.140000000000001,"5min":18.303000000000001,"15min":18.303000000000001},"last":0.26700000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:55:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5899999999999999,"5min":1.599,"15min":1.341},"min":{"1min":0.23499999999999999,"5min":0.222,"15min":0.19700000000000001},"max":{"1min":12.244999999999999,"5min":12.244999999999999,"15min":15.356},"last":11.538},{"interface":"front","average":{"1min":3.3220000000000001,"5min":1.587,"15min":1.4750000000000001},"min":{"1min":0.23899999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":18.004000000000001,"5min":18.004000000000001,"15min":18.004000000000001},"last":11.481999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:55:56 2023","interfaces":[{"interface":"back","average":{"1min":4.4429999999999996,"5min":2.1859999999999999,"15min":1.855},"min":{"1min":0.22900000000000001,"5min":0.223,"15min":0.218},"max":{"1min":20.036000000000001,"5min":20.036000000000001,"15min":20.036000000000001},"last":0.78300000000000003},{"interface":"front","average":{"1min":6.359,"5min":2.9350000000000001,"15min":1.9430000000000001},"min":{"1min":0.42799999999999999,"5min":0.219,"15min":0.219},"max":{"1min":19.640999999999998,"5min":19.640999999999998,"15min":19.640999999999998},"last":0.307}]},{"osd":4,"last update":"Mon Oct 9 10:56:07 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":1.6339999999999999,"15min":1.397},"min":{"1min":0.32200000000000001,"5min":0.26500000000000001,"15min":0.255},"max":{"1min":14.763,"5min":14.763,"15min":14.763},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.78,"5min":1.6000000000000001,"15min":1.5409999999999999},"min":{"1min":0.30399999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.2759999999999998,"5min":6.835,"15min":14.228999999999999},"last":0.53700000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:56:07 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":1.8759999999999999,"15min":1.4490000000000001},"min":{"1min":0.42999999999999999,"5min":0.32700000000000001,"15min":0.253},"max":{"1min":8.423,"5min":9.6289999999999996,"15min":9.7720000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":2.6899999999999999,"5min":2.008,"15min":1.734},"min":{"1min":0.33100000000000002,"5min":0.30299999999999999,"15min":0.28899999999999998},"max":{"1min":14.833,"5min":14.833,"15min":17.085000000000001},"last":0.59899999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:56:07 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":1.877,"15min":1.534},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.27400000000000002},"max":{"1min":8.468,"5min":9.4049999999999994,"15min":10.425000000000001},"last":0.432},{"interface":"front","average":{"1min":2.6190000000000002,"5min":1.7490000000000001,"15min":1.548},"min":{"1min":0.35799999999999998,"5min":0.312,"15min":0.312},"max":{"1min":14.804,"5min":14.804,"15min":14.804},"last":0.64000000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:56:07 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":2.1059999999999999,"15min":1.639},"min":{"1min":0.38600000000000001,"5min":0.36299999999999999,"15min":0.34000000000000002},"max":{"1min":14.856,"5min":16.312000000000001,"15min":16.312000000000001},"last":0.67100000000000004},{"interface":"front","average":{"1min":1.794,"5min":1.611,"15min":1.5620000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.32500000000000001},"max":{"1min":6.2990000000000004,"5min":8.8149999999999995,"15min":14.273999999999999},"last":0.57199999999999995}]}]},{"osd":0,"up_from":80,"seq":343597383860,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1270032,"kb_used_data":787144,"kb_used_omap":28,"kb_used_meta":482851,"kb_avail":92475120,"statfs":{"total":95995035648,"available":94694522880,"internally_reserved":0,"allocated":806035456,"data_stored":804309167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":494440272},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,6,0,2,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.8400000000000001,"15min":1.6339999999999999},"min":{"1min":0.22800000000000001,"5min":0.20399999999999999,"15min":0.191},"max":{"1min":6.0919999999999996,"5min":13.263999999999999,"15min":25.475000000000001},"last":10.289999999999999},{"interface":"front","average":{"1min":2.3559999999999999,"5min":1.8220000000000001,"15min":1.613},"min":{"1min":0.28899999999999998,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":9.1219999999999999,"5min":9.359,"15min":20.832999999999998},"last":7.0419999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:55:34 2023","interfaces":[{"interface":"back","average":{"1min":2.996,"5min":1.915,"15min":1.5580000000000001},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":24.852,"5min":24.852,"15min":24.852},"last":6.601},{"interface":"front","average":{"1min":7.2240000000000002,"5min":3.0550000000000002,"15min":2},"min":{"1min":0.30599999999999999,"5min":0.24099999999999999,"15min":0.20399999999999999},"max":{"1min":77.256,"5min":77.256,"15min":77.256},"last":1.008}]},{"osd":3,"last update":"Mon Oct 9 10:55:59 2023","interfaces":[{"interface":"back","average":{"1min":3.3900000000000001,"5min":2.3660000000000001,"15min":1.96},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20200000000000001},"max":{"1min":18.972000000000001,"5min":18.972000000000001,"15min":18.972000000000001},"last":0.29699999999999999},{"interface":"front","average":{"1min":2.7050000000000001,"5min":2.371,"15min":1.9059999999999999},"min":{"1min":0.35999999999999999,"5min":0.25800000000000001,"15min":0.224},"max":{"1min":7.8380000000000001,"5min":13.305,"15min":14.391999999999999},"last":3.8399999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:55:52 2023","interfaces":[{"interface":"back","average":{"1min":2.9550000000000001,"5min":2.8380000000000001,"15min":1.8979999999999999},"min":{"1min":0.38500000000000001,"5min":0.33800000000000002,"15min":0.26200000000000001},"max":{"1min":9.4749999999999996,"5min":77.177999999999997,"15min":77.177999999999997},"last":1.038},{"interface":"front","average":{"1min":1.839,"5min":1.8100000000000001,"15min":1.7110000000000001},"min":{"1min":0.36599999999999999,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":9.032,"5min":10.427,"15min":12.318},"last":0.36099999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:55:52 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":1.966,"15min":1.7509999999999999},"min":{"1min":0.34300000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.9559999999999995,"5min":10.478,"15min":12.157},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.7929999999999999,"15min":1.607},"min":{"1min":0.30399999999999999,"5min":0.30199999999999999,"15min":0.29499999999999998},"max":{"1min":5.319,"5min":24.754999999999999,"15min":24.754999999999999},"last":3.6779999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:55:52 2023","interfaces":[{"interface":"back","average":{"1min":2.7040000000000002,"5min":2.2210000000000001,"15min":1.8839999999999999},"min":{"1min":0.38700000000000001,"5min":0.34200000000000003,"15min":0.30399999999999999},"max":{"1min":18.920000000000002,"5min":18.920000000000002,"15min":18.920000000000002},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.4119999999999999,"5min":1.903,"15min":1.5189999999999999},"min":{"1min":0.34200000000000003,"5min":0.307,"15min":0.307},"max":{"1min":5.2910000000000004,"5min":24.800000000000001,"15min":24.800000000000001},"last":3.6619999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:55:52 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":1.917,"15min":1.7110000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.125,"5min":13.225,"15min":14.48},"last":3.6309999999999998},{"interface":"front","average":{"1min":3.2130000000000001,"5min":3.0430000000000001,"15min":2.0419999999999998},"min":{"1min":0.40000000000000002,"5min":0.32600000000000001,"15min":0.32200000000000001},"max":{"1min":10.819000000000001,"5min":77.228999999999999,"15min":77.228999999999999},"last":1.0660000000000001}]}]},{"osd":2,"up_from":90,"seq":386547056801,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":440924,"kb_used_data":159444,"kb_used_omap":81,"kb_used_meta":281390,"kb_avail":93304228,"statfs":{"total":95995035648,"available":95543529472,"internally_reserved":0,"allocated":163270656,"data_stored":161716074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":288143578},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:35 2023","interfaces":[{"interface":"back","average":{"1min":2.4790000000000001,"5min":1.7310000000000001,"15min":1.403},"min":{"1min":0.18099999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":17.574000000000002,"5min":17.574000000000002,"15min":18.512},"last":4.7859999999999996},{"interface":"front","average":{"1min":2.1829999999999998,"5min":1.8420000000000001,"15min":1.5349999999999999},"min":{"1min":0.27700000000000002,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":11.167,"5min":15.595000000000001,"15min":17.975000000000001},"last":3.2690000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:55:35 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":1.571,"15min":1.52},"min":{"1min":0.26300000000000001,"5min":0.23300000000000001,"15min":0.20200000000000001},"max":{"1min":11.204000000000001,"5min":11.204000000000001,"15min":15.247999999999999},"last":3.0899999999999999},{"interface":"front","average":{"1min":2.1749999999999998,"5min":1.3200000000000001,"15min":1.296},"min":{"1min":0.317,"5min":0.253,"15min":0.20699999999999999},"max":{"1min":10.361000000000001,"5min":10.361000000000001,"15min":16.454999999999998},"last":3.5049999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:55:57 2023","interfaces":[{"interface":"back","average":{"1min":4.4539999999999997,"5min":1.954,"15min":1.5640000000000001},"min":{"1min":0.30299999999999999,"5min":0.27800000000000002,"15min":0.22500000000000001},"max":{"1min":19.539000000000001,"5min":19.539000000000001,"15min":19.539000000000001},"last":13.276},{"interface":"front","average":{"1min":4.4400000000000004,"5min":2.327,"15min":1.867},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.20200000000000001},"max":{"1min":16.702999999999999,"5min":16.702999999999999,"15min":16.702999999999999},"last":4.6360000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:55:35 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":1.8200000000000001,"15min":1.5940000000000001},"min":{"1min":0.32800000000000001,"5min":0.29599999999999999,"15min":0.254},"max":{"1min":10.289,"5min":15.672000000000001,"15min":16.231999999999999},"last":3.5699999999999998},{"interface":"front","average":{"1min":1.679,"5min":1.9410000000000001,"15min":1.5589999999999999},"min":{"1min":0.311,"5min":0.307,"15min":0.26300000000000001},"max":{"1min":8.9139999999999997,"5min":15.696999999999999,"15min":15.696999999999999},"last":4.8239999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:55:35 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":1.508,"15min":1.532},"min":{"1min":0.374,"5min":0.31900000000000001,"15min":0.245},"max":{"1min":4.46,"5min":12.128,"15min":17.920000000000002},"last":3.0670000000000002},{"interface":"front","average":{"1min":1.615,"5min":1.6839999999999999,"15min":1.5489999999999999},"min":{"1min":0.34999999999999998,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":10.336,"5min":16.597000000000001,"15min":17.882999999999999},"last":3.5449999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:55:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6240000000000001,"5min":1.6299999999999999,"15min":1.3480000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.247},"max":{"1min":8.8460000000000001,"5min":10.443,"15min":14.029999999999999},"last":4.8789999999999996},{"interface":"front","average":{"1min":1.4810000000000001,"5min":1.5840000000000001,"15min":1.5},"min":{"1min":0.42299999999999999,"5min":0.34799999999999998,"15min":0.29199999999999998},"max":{"1min":4.492,"5min":6.9500000000000002,"15min":12.862},"last":3.0390000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:55:35 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":1.601,"15min":1.369},"min":{"1min":0.40000000000000002,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":8.8879999999999999,"5min":8.8879999999999999,"15min":13.999000000000001},"last":4.8540000000000001},{"interface":"front","average":{"1min":1.837,"5min":1.6579999999999999,"15min":1.573},"min":{"1min":0.38,"5min":0.34000000000000002,"15min":0.307},"max":{"1min":5.6539999999999999,"5min":9.4830000000000005,"15min":12.826000000000001},"last":3.008}]}]},{"osd":3,"up_from":95,"seq":408021893275,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1638876,"kb_used_data":953876,"kb_used_omap":7,"kb_used_meta":684984,"kb_avail":92106276,"statfs":{"total":95995035648,"available":94316826624,"internally_reserved":0,"allocated":976769024,"data_stored":975036845,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":701424232},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:56:04 2023","interfaces":[{"interface":"back","average":{"1min":3.7389999999999999,"5min":2.7450000000000001,"15min":1.9350000000000001},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":11.819000000000001,"5min":38.817999999999998,"15min":38.817999999999998},"last":0.81399999999999995},{"interface":"front","average":{"1min":5.0810000000000004,"5min":3.3809999999999998,"15min":2.161},"min":{"1min":0.67800000000000005,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":13.351000000000001,"5min":17.312000000000001,"15min":17.312000000000001},"last":11.539}]},{"osd":1,"last update":"Mon Oct 9 10:56:04 2023","interfaces":[{"interface":"back","average":{"1min":3.0870000000000002,"5min":2.5030000000000001,"15min":2.0099999999999998},"min":{"1min":0.20200000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":18.806999999999999,"5min":18.806999999999999,"15min":18.806999999999999},"last":7.202},{"interface":"front","average":{"1min":3.5,"5min":2.7879999999999998,"15min":1.978},"min":{"1min":0.29899999999999999,"5min":0.22,"15min":0.193},"max":{"1min":11.928000000000001,"5min":18.526,"15min":18.526},"last":0.432}]},{"osd":2,"last update":"Mon Oct 9 10:56:04 2023","interfaces":[{"interface":"back","average":{"1min":4.0949999999999998,"5min":2.3559999999999999,"15min":1.8819999999999999},"min":{"1min":0.26300000000000001,"5min":0.24199999999999999,"15min":0.23699999999999999},"max":{"1min":18.774000000000001,"5min":18.774000000000001,"15min":18.774000000000001},"last":7.234},{"interface":"front","average":{"1min":4.5810000000000004,"5min":3.0529999999999999,"15min":2.1150000000000002},"min":{"1min":0.255,"5min":0.248,"15min":0.248},"max":{"1min":14.134,"5min":17.268000000000001,"15min":17.268000000000001},"last":7.25}]},{"osd":4,"last update":"Mon Oct 9 10:56:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4220000000000002,"5min":2.206,"15min":1.7689999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28199999999999997},"max":{"1min":9.0500000000000007,"5min":11.863,"15min":20.388000000000002},"last":0.46500000000000002},{"interface":"front","average":{"1min":2.3300000000000001,"5min":2.089,"15min":1.75},"min":{"1min":0.28599999999999998,"5min":0.28399999999999997,"15min":0.24299999999999999},"max":{"1min":10.462999999999999,"5min":11.680999999999999,"15min":19.085999999999999},"last":7.1289999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:56:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2010000000000001,"5min":2.1779999999999999,"15min":2.012},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.492000000000001,"5min":14.045,"15min":51.509999999999998},"last":7.2229999999999999},{"interface":"front","average":{"1min":1.52,"5min":1.754,"15min":1.9470000000000001},"min":{"1min":0.374,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":9.1140000000000008,"5min":10.193,"15min":55.338999999999999},"last":7.3070000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:56:04 2023","interfaces":[{"interface":"back","average":{"1min":2.379,"5min":2.2240000000000002,"15min":1.7410000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28000000000000003},"max":{"1min":10.411,"5min":11.723000000000001,"15min":12.361000000000001},"last":7.1920000000000002},{"interface":"front","average":{"1min":1.6739999999999999,"5min":1.9790000000000001,"15min":1.8540000000000001},"min":{"1min":0.39100000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":9.1969999999999992,"5min":11.132999999999999,"15min":13.891999999999999},"last":7.1710000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:56:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6480000000000001,"5min":2.0859999999999999,"15min":2.093},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.31},"max":{"1min":18.722000000000001,"5min":18.722000000000001,"15min":51.744},"last":4.218},{"interface":"front","average":{"1min":2.294,"5min":2.407,"15min":2.101},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.31900000000000001},"max":{"1min":9.0779999999999994,"5min":13.337,"15min":53.399000000000001},"last":3.097}]}]},{"osd":4,"up_from":30,"seq":128849019151,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1292048,"kb_used_data":994760,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92453104,"statfs":{"total":95995035648,"available":94671978496,"internally_reserved":0,"allocated":1018634240,"data_stored":1017500268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:54 2023","interfaces":[{"interface":"back","average":{"1min":2.8079999999999998,"5min":3.8279999999999998,"15min":2.0430000000000001},"min":{"1min":0.23499999999999999,"5min":0.217,"15min":0.20999999999999999},"max":{"1min":12.332000000000001,"5min":213.88999999999999,"15min":213.88999999999999},"last":2.9529999999999998},{"interface":"front","average":{"1min":2.3330000000000002,"5min":4.3390000000000004,"15min":2.3319999999999999},"min":{"1min":0.24399999999999999,"5min":0.23300000000000001,"15min":0.217},"max":{"1min":8.9130000000000003,"5min":214.09800000000001,"15min":214.09800000000001},"last":2.0369999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:55:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":3.9249999999999998,"15min":2.0699999999999998},"min":{"1min":0.23799999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":9.0099999999999998,"5min":206.50399999999999,"15min":206.50399999999999},"last":5.1139999999999999},{"interface":"front","average":{"1min":2.1480000000000001,"5min":4.1100000000000003,"15min":2.282},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":8.4870000000000001,"5min":206.529,"15min":206.529},"last":5.1539999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:55:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5790000000000002,"5min":2.137,"15min":1.5409999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.253},"max":{"1min":8.4800000000000004,"5min":15.446999999999999,"15min":15.446999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":3.8519999999999999,"5min":2.2850000000000001,"15min":1.746},"min":{"1min":0.30399999999999999,"5min":0.29599999999999999,"15min":0.26300000000000001},"max":{"1min":27.818000000000001,"5min":27.818000000000001,"15min":27.818000000000001},"last":1.294}]},{"osd":3,"last update":"Mon Oct 9 10:55:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":4.0410000000000004,"15min":2.3340000000000001},"min":{"1min":0.34599999999999997,"5min":0.29599999999999999,"15min":0.27400000000000002},"max":{"1min":6.0469999999999997,"5min":209.03700000000001,"15min":209.03700000000001},"last":11.372999999999999},{"interface":"front","average":{"1min":2.2149999999999999,"5min":4.4720000000000004,"15min":2.544},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.5919999999999996,"5min":208.80199999999999,"15min":208.80199999999999},"last":6.6349999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:55:57 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":1.7649999999999999,"15min":1.5629999999999999},"min":{"1min":0.253,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":4.1210000000000004,"5min":19.068999999999999,"15min":24.149999999999999},"last":0.254},{"interface":"front","average":{"1min":0.53800000000000003,"5min":1.4790000000000001,"15min":1.385},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.252},"max":{"1min":1.917,"5min":15.535,"15min":20.422000000000001},"last":0.34899999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:55:28 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":1.524,"15min":1.22},"min":{"1min":0.28000000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.85299999999999998,"5min":15.739000000000001,"15min":15.739000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.86399999999999999,"5min":2.0720000000000001,"15min":1.4630000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":4.2329999999999997,"5min":14.272,"15min":14.272},"last":0.433}]},{"osd":7,"last update":"Mon Oct 9 10:55:36 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":1.831,"15min":1.381},"min":{"1min":0.35499999999999998,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":4.9470000000000001,"5min":14.458,"15min":14.458},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.6120000000000001,"5min":1.7729999999999999,"15min":1.2869999999999999},"min":{"1min":0.36599999999999999,"5min":0.255,"15min":0.255},"max":{"1min":8.9589999999999996,"5min":15.592000000000001,"15min":15.592000000000001},"last":0.51100000000000001}]}]},{"osd":5,"up_from":37,"seq":158913790219,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2059704,"kb_used_data":1473968,"kb_used_omap":0,"kb_used_meta":585664,"kb_avail":91685448,"statfs":{"total":95995035648,"available":93885898752,"internally_reserved":0,"allocated":1509343232,"data_stored":1508221426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599719936},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:57 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":1.9419999999999999,"15min":1.4390000000000001},"min":{"1min":0.25600000000000001,"5min":0.215,"15min":0.215},"max":{"1min":11.683,"5min":16.449000000000002,"15min":16.449000000000002},"last":4.6769999999999996},{"interface":"front","average":{"1min":1.897,"5min":1.9950000000000001,"15min":1.498},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.653,"5min":16.489000000000001,"15min":16.489000000000001},"last":4.7050000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:56:10 2023","interfaces":[{"interface":"back","average":{"1min":2.9689999999999999,"5min":2.2080000000000002,"15min":1.74},"min":{"1min":0.247,"5min":0.215,"15min":0.215},"max":{"1min":6.6950000000000003,"5min":8.2629999999999999,"15min":12.999000000000001},"last":2.1299999999999999},{"interface":"front","average":{"1min":1.9510000000000001,"5min":1.903,"15min":1.468},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.24299999999999999},"max":{"1min":5.024,"5min":9.2699999999999996,"15min":12.109},"last":1.395}]},{"osd":2,"last update":"Mon Oct 9 10:55:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.266,"15min":1.6000000000000001},"min":{"1min":0.28000000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.257999999999999,"5min":11.368,"15min":11.393000000000001},"last":3.5470000000000002},{"interface":"front","average":{"1min":1.667,"5min":2.1859999999999999,"15min":1.4059999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.4249999999999998,"5min":15.087999999999999,"15min":15.087999999999999},"last":1.1040000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:55:57 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":2.754,"15min":1.962},"min":{"1min":0.34499999999999997,"5min":0.314,"15min":0.25700000000000001},"max":{"1min":6.6399999999999997,"5min":63.508000000000003,"15min":63.508000000000003},"last":1.268},{"interface":"front","average":{"1min":2.5030000000000001,"5min":2.7810000000000001,"15min":2.0430000000000001},"min":{"1min":0.39300000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.1820000000000004,"5min":62.418999999999997,"15min":62.418999999999997},"last":2.4660000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:56:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":1.081,"15min":1.385},"min":{"1min":0.28799999999999998,"5min":0.247,"15min":0.23200000000000001},"max":{"1min":1.6879999999999999,"5min":6.6120000000000001,"15min":123.241},"last":1.03},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.1299999999999999,"15min":1.6020000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.24199999999999999},"max":{"1min":2.8730000000000002,"5min":4.5090000000000003,"15min":123.318},"last":4.8550000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:55:20 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":1.79,"15min":1.296},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.9049999999999994,"5min":11.098000000000001,"15min":11.098000000000001},"last":3.7149999999999999},{"interface":"front","average":{"1min":0.83899999999999997,"5min":1.7769999999999999,"15min":1.2170000000000001},"min":{"1min":0.32000000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.1779999999999999,"5min":15.16,"15min":20.256},"last":1.0600000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:55:46 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":1.6240000000000001,"15min":1.2869999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":2.911,"5min":8.468,"15min":10.739000000000001},"last":2.0880000000000001},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.3899999999999999,"15min":1.1319999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":2.2450000000000001,"5min":10.858000000000001,"15min":14.154},"last":1.085}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":794558464,"data_stored":794558464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":481165312,"data_stored":481165312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":158732288,"data_stored":158728216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":961085440,"data_stored":961085440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1014693888,"data_stored":1014693888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1496846336,"data_stored":1496842264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1164640256,"data_stored":1164640256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1059262464,"data_stored":1059258392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:56:24.552 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:56:24.552 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 51 2023-10-09T10:56:24.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:24.554 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:24.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:24.655 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:24.725 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:24.759 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:24.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:24.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:24.764 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:24.801 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:24.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:24 smithi125 ceph-mon[195507]: from='client.55492 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:56:24.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:24 smithi125 ceph-mon[195507]: pgmap v770: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 14068/93480 objects degraded (15.049%); 6962/93480 objects misplaced (7.448%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:56:24.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:24 smithi125 ceph-mon[200197]: from='client.55492 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:56:24.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:24 smithi125 ceph-mon[200197]: pgmap v770: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 14068/93480 objects degraded (15.049%); 6962/93480 objects misplaced (7.448%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:56:24.863 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:24.879 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:56:24.879 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:56:24.941 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:24.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:24 smithi155 ceph-mon[184854]: from='client.55492 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:56:24.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:24 smithi155 ceph-mon[184854]: pgmap v770: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 14068/93480 objects degraded (15.049%); 6962/93480 objects misplaced (7.448%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T10:56:24.995 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:24.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:25.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:25.100 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:25.176 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:25.978 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:26.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:26.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:26.068 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:56:26.111 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:26.148 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:26.234 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:26.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:26.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:26.338 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:26.438 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:26.460 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:26.497 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:26.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:26.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:26.604 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:26.638 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:26.659 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:26.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:26.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:26.762 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:26.788 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:26 smithi125 ceph-mon[195507]: pgmap v771: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 13910/90378 objects degraded (15.391%); 6750/90378 objects misplaced (7.469%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T10:56:26.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:26 smithi125 ceph-mon[200197]: pgmap v771: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 13910/90378 objects degraded (15.391%); 6750/90378 objects misplaced (7.469%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T10:56:26.796 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:26.822 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:26.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:26.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:26.926 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:26.954 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:26 smithi155 ceph-mon[184854]: pgmap v771: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 13910/90378 objects degraded (15.391%); 6750/90378 objects misplaced (7.469%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T10:56:26.957 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:26.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:26.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:26.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:27.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:27.137 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:27.208 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:27.242 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:56:27.292 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:56:27.333 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:56:27.389 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:56:27.411 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:27.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:27.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:27.435 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:56:27.472 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:56:27.505 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:56:27.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:27.555 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:56:27.573 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:56:27.573 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:56:27.573 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:56:27.573 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:56:27.573 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:56:27.573 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:56:27.574 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:56:27.574 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:56:27.618 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:27.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:27 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/332672482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:27.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:27 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/332672482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:27.823 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:27.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:27.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:27.929 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:27.955 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:27 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/332672482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:27.957 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:27.984 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:27.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:27.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:28.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:28.120 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:28.147 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:28.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:28.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:28.252 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:28.287 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:28.313 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:28.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:28.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:28.425 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:28.458 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:28.483 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:28.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:28.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:28.590 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:28.640 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:28.677 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:28.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:28.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:28.785 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:28.826 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:28.860 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:28.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:28.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:28.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:28.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:28 smithi155 ceph-mon[184854]: pgmap v772: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 13910/83217 objects degraded (16.715%); 6105/83217 objects misplaced (7.336%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T10:56:28.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:28 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 13910/83217 objects degraded (16.715%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:28.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:28 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2453702338' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:28.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:28 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:28.994 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:56:29.017 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:28 smithi125 ceph-mon[195507]: pgmap v772: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 13910/83217 objects degraded (16.715%); 6105/83217 objects misplaced (7.336%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T10:56:29.017 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:28 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 13910/83217 objects degraded (16.715%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:29.017 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:28 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2453702338' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:29.017 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:28 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:29.017 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:28 smithi125 ceph-mon[200197]: pgmap v772: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.7 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 13910/83217 objects degraded (16.715%); 6105/83217 objects misplaced (7.336%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T10:56:29.018 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:28 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 13910/83217 objects degraded (16.715%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:29.018 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:28 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2453702338' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:29.018 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:28 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:29.038 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:29.333 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:29.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:29.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:29.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:29.476 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:29.509 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:29.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:29.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:29.613 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:29.626 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:29 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:56:29.626 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:29 smithi125 ceph-mon[195507]: failed to encode map e315 with expected crc 2023-10-09T10:56:29.626 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:29 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:56:29.626 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:29 smithi125 ceph-mon[195507]: osdmap e315: 8 total, 8 up, 8 in 2023-10-09T10:56:29.644 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:29.672 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:29.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:29.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:29.778 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:29.807 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:29 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:56:29.807 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:29 smithi155 ceph-mon[184854]: failed to encode map e315 with expected crc 2023-10-09T10:56:29.807 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:29 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:56:29.807 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:29 smithi155 ceph-mon[184854]: osdmap e315: 8 total, 8 up, 8 in 2023-10-09T10:56:29.809 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:29.839 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:29.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:29.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:29.942 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:29.975 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:29.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:29 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:56:29.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:29 smithi125 ceph-mon[200197]: failed to encode map e315 with expected crc 2023-10-09T10:56:29.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:29 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:56:29.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:29 smithi125 ceph-mon[200197]: osdmap e315: 8 total, 8 up, 8 in 2023-10-09T10:56:29.997 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:30.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:30.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:30.101 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:30.145 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:30.227 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:30.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:30.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:30.332 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:30.424 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:30.472 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:30.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:30.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:30.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:30.596 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:56:30.596 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:56:30.596 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:56:30.596 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:56:30.597 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.597 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:96531 8.13 8:cc58e7a9:::benchmark_data_smithi125_329074_object96530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e294)", 2023-10-09T10:56:30.597 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:47.587368+0000", 2023-10-09T10:56:30.597 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 103.005062854, 2023-10-09T10:56:30.597 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5657507750000002, 2023-10-09T10:56:30.597 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.597 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:56:30.597 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:30.597 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:30.597 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:30.597 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 96531 2023-10-09T10:56:30.597 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.598 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.598 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.598 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.598 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.587368+0000", 2023-10-09T10:56:30.598 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.598 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.598 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.598 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.598 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.587368+0000", 2023-10-09T10:56:30.598 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.598 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.599 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.599 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.599 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.587366+0000", 2023-10-09T10:56:30.599 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:56:30.599 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.599 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.599 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.599 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.587426+0000", 2023-10-09T10:56:30.599 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0009000000000001e-05 2023-10-09T10:56:30.599 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.599 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.599 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.599 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.587438+0000", 2023-10-09T10:56:30.600 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1769e-05 2023-10-09T10:56:30.600 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.600 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.600 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.600 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.587448+0000", 2023-10-09T10:56:30.600 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0166999999999999e-05 2023-10-09T10:56:30.600 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.600 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.600 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.600 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.593585+0000", 2023-10-09T10:56:30.600 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0061369060000000001 2023-10-09T10:56:30.600 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.601 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.601 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.601 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.593597+0000", 2023-10-09T10:56:30.601 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1707e-05 2023-10-09T10:56:30.601 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.601 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.601 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.601 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906276+0000", 2023-10-09T10:56:30.601 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.31267926400000001 2023-10-09T10:56:30.601 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.601 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.602 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.602 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906284+0000", 2023-10-09T10:56:30.602 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2300000000000008e-06 2023-10-09T10:56:30.602 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.602 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.602 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.602 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:48.954678+0000", 2023-10-09T10:56:30.602 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0483933839999999 2023-10-09T10:56:30.602 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.602 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.602 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:56:30.602 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:48.954868+0000", 2023-10-09T10:56:30.603 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000190726 2023-10-09T10:56:30.603 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.603 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.603 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.603 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.153008+0000", 2023-10-09T10:56:30.603 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19813940399999999 2023-10-09T10:56:30.603 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.603 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.603 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.603 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.153119+0000", 2023-10-09T10:56:30.603 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111295 2023-10-09T10:56:30.603 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.604 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.604 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.604 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.604 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.604 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:96541 8.3 8:c030ac3e:::benchmark_data_smithi125_329074_object96540:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e294)", 2023-10-09T10:56:30.604 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:47.591599+0000", 2023-10-09T10:56:30.604 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 103.000831699, 2023-10-09T10:56:30.604 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.573380974, 2023-10-09T10:56:30.604 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.604 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:56:30.604 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:30.604 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:30.604 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:30.605 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 96541 2023-10-09T10:56:30.605 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.605 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.605 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.605 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.605 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.591599+0000", 2023-10-09T10:56:30.605 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.605 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.605 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.605 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.605 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.591599+0000", 2023-10-09T10:56:30.605 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.606 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.606 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.606 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.606 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.591598+0000", 2023-10-09T10:56:30.606 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:30.606 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.606 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.606 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.606 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.591658+0000", 2023-10-09T10:56:30.606 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0265e-05 2023-10-09T10:56:30.606 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.606 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.607 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.607 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.593520+0000", 2023-10-09T10:56:30.607 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00186181 2023-10-09T10:56:30.607 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.607 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.607 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.607 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.593542+0000", 2023-10-09T10:56:30.607 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1968e-05 2023-10-09T10:56:30.607 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.607 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.607 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.607 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.599618+0000", 2023-10-09T10:56:30.608 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0060761599999999997 2023-10-09T10:56:30.608 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.608 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.608 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.608 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.599631+0000", 2023-10-09T10:56:30.608 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2194e-05 2023-10-09T10:56:30.608 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.608 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.608 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.608 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906106+0000", 2023-10-09T10:56:30.608 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.30647559000000002 2023-10-09T10:56:30.608 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.608 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.609 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.609 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906118+0000", 2023-10-09T10:56:30.609 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1813e-05 2023-10-09T10:56:30.609 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.609 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.609 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.609 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.153159+0000", 2023-10-09T10:56:30.609 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2470411399999999 2023-10-09T10:56:30.609 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.609 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.609 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:56:30.609 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.153336+0000", 2023-10-09T10:56:30.610 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017656799999999999 2023-10-09T10:56:30.610 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.610 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.610 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.610 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.164870+0000", 2023-10-09T10:56:30.610 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011534233 2023-10-09T10:56:30.610 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.610 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.610 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.610 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.164980+0000", 2023-10-09T10:56:30.610 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011043 2023-10-09T10:56:30.610 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.611 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.611 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.611 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.611 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.611 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:96549 8.3 8:c6214b70:::benchmark_data_smithi125_329074_object96548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e294)", 2023-10-09T10:56:30.611 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:47.596436+0000", 2023-10-09T10:56:30.611 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 102.995995063, 2023-10-09T10:56:30.611 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.568400507, 2023-10-09T10:56:30.611 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.611 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:56:30.611 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:30.611 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:30.611 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:30.612 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 96549 2023-10-09T10:56:30.612 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.612 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.612 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.612 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.612 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596436+0000", 2023-10-09T10:56:30.612 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.612 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.612 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.612 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.612 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596436+0000", 2023-10-09T10:56:30.612 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.613 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.613 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.613 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.613 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596435+0000", 2023-10-09T10:56:30.613 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:56:30.613 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.613 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.613 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.613 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596492+0000", 2023-10-09T10:56:30.613 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7383999999999998e-05 2023-10-09T10:56:30.613 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.613 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.613 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.614 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596504+0000", 2023-10-09T10:56:30.614 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1762e-05 2023-10-09T10:56:30.614 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.614 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.614 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.614 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596514+0000", 2023-10-09T10:56:30.614 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6549999999999999e-06 2023-10-09T10:56:30.614 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.614 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.614 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.614 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.599655+0000", 2023-10-09T10:56:30.614 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003141161 2023-10-09T10:56:30.615 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.615 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.615 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.615 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.599662+0000", 2023-10-09T10:56:30.615 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6450000000000005e-06 2023-10-09T10:56:30.615 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.615 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.615 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.615 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906142+0000", 2023-10-09T10:56:30.615 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.306479271 2023-10-09T10:56:30.615 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.615 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.616 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.616 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906150+0000", 2023-10-09T10:56:30.616 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0849999999999997e-06 2023-10-09T10:56:30.616 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.616 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.616 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.616 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.153411+0000", 2023-10-09T10:56:30.616 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.247261223 2023-10-09T10:56:30.616 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.616 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.616 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:56:30.616 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.153548+0000", 2023-10-09T10:56:30.616 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000136637 2023-10-09T10:56:30.617 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.617 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.617 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.617 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.164737+0000", 2023-10-09T10:56:30.617 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011188929 2023-10-09T10:56:30.617 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.617 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.617 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.617 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.164837+0000", 2023-10-09T10:56:30.617 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9942e-05 2023-10-09T10:56:30.617 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.617 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.618 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.618 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.618 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.618 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:96552 8.3 8:c5d8fcd5:::benchmark_data_smithi125_329074_object96551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e294)", 2023-10-09T10:56:30.618 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:47.596519+0000", 2023-10-09T10:56:30.618 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 102.995911781, 2023-10-09T10:56:30.618 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5831809479999999, 2023-10-09T10:56:30.618 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.618 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:56:30.618 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:30.618 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:30.618 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:30.619 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 96552 2023-10-09T10:56:30.619 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.619 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.619 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.619 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.619 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596519+0000", 2023-10-09T10:56:30.619 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.619 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.619 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.619 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.619 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596519+0000", 2023-10-09T10:56:30.619 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.619 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.620 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.620 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.620 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596519+0000", 2023-10-09T10:56:30.620 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:30.620 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.620 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.620 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.620 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596570+0000", 2023-10-09T10:56:30.620 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0989000000000003e-05 2023-10-09T10:56:30.620 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.620 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.620 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.621 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596577+0000", 2023-10-09T10:56:30.621 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5229999999999998e-06 2023-10-09T10:56:30.621 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.621 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.621 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.621 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596583+0000", 2023-10-09T10:56:30.621 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7359999999999998e-06 2023-10-09T10:56:30.621 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.621 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.621 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.621 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.599684+0000", 2023-10-09T10:56:30.622 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0031007780000000002 2023-10-09T10:56:30.622 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.622 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.622 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.622 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.599691+0000", 2023-10-09T10:56:30.622 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4240000000000004e-06 2023-10-09T10:56:30.622 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.622 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.622 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.622 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906172+0000", 2023-10-09T10:56:30.622 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.30648051100000001 2023-10-09T10:56:30.622 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.623 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.623 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.623 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906180+0000", 2023-10-09T10:56:30.623 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0520000000000002e-06 2023-10-09T10:56:30.623 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.623 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.623 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.623 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.153576+0000", 2023-10-09T10:56:30.623 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.247396304 2023-10-09T10:56:30.623 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.623 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.623 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:56:30.623 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.153717+0000", 2023-10-09T10:56:30.624 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014072299999999999 2023-10-09T10:56:30.624 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.624 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.624 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.624 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.164616+0000", 2023-10-09T10:56:30.624 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010899143 2023-10-09T10:56:30.624 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.624 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.624 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for degraded object", 2023-10-09T10:56:30.624 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.164702+0000", 2023-10-09T10:56:30.624 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6069999999999994e-05 2023-10-09T10:56:30.624 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.624 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.625 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.625 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.179586+0000", 2023-10-09T10:56:30.625 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014883498 2023-10-09T10:56:30.625 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.625 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.625 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.625 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.179700+0000", 2023-10-09T10:56:30.625 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011465599999999999 2023-10-09T10:56:30.625 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.625 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.625 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.625 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.625 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.626 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:96568 8.3 8:c7d467fd:::benchmark_data_smithi125_329074_object96567:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e294)", 2023-10-09T10:56:30.626 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:47.596588+0000", 2023-10-09T10:56:30.626 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 102.995843306, 2023-10-09T10:56:30.626 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5676355040000001, 2023-10-09T10:56:30.626 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.626 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.626 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:30.626 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:30.626 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:30.626 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 96568 2023-10-09T10:56:30.626 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.626 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.627 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.627 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.627 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596588+0000", 2023-10-09T10:56:30.627 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.627 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.627 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.627 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.627 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596588+0000", 2023-10-09T10:56:30.627 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.627 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.629 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.629 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.629 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596587+0000", 2023-10-09T10:56:30.629 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:30.629 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.629 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.629 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.629 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596638+0000", 2023-10-09T10:56:30.630 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0346000000000002e-05 2023-10-09T10:56:30.630 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.630 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.630 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.630 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596645+0000", 2023-10-09T10:56:30.630 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2540000000000003e-06 2023-10-09T10:56:30.630 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.630 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.630 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.630 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596650+0000", 2023-10-09T10:56:30.630 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3109999999999996e-06 2023-10-09T10:56:30.630 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.631 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.631 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.631 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.599712+0000", 2023-10-09T10:56:30.631 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030619459999999999 2023-10-09T10:56:30.631 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.631 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.631 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.631 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.599720+0000", 2023-10-09T10:56:30.631 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.306e-06 2023-10-09T10:56:30.631 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.631 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.631 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.632 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906202+0000", 2023-10-09T10:56:30.632 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.30648211800000003 2023-10-09T10:56:30.632 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.632 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.632 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.632 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906224+0000", 2023-10-09T10:56:30.632 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2350000000000001e-05 2023-10-09T10:56:30.632 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.632 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.632 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.632 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.153748+0000", 2023-10-09T10:56:30.632 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.247524072 2023-10-09T10:56:30.632 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.633 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.633 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.153927+0000", 2023-10-09T10:56:30.633 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017877299999999999 2023-10-09T10:56:30.633 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.633 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:30.633 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.154405+0000", 2023-10-09T10:56:30.633 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047855100000000002 2023-10-09T10:56:30.633 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.633 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:30.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.159817+0000", 2023-10-09T10:56:30.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0054113820000000002 2023-10-09T10:56:30.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:30.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.163919+0000", 2023-10-09T10:56:30.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041019660000000003 2023-10-09T10:56:30.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:30.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.163976+0000", 2023-10-09T10:56:30.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7695999999999998e-05 2023-10-09T10:56:30.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.164046+0000", 2023-10-09T10:56:30.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0007999999999998e-05 2023-10-09T10:56:30.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.164223+0000", 2023-10-09T10:56:30.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000176832 2023-10-09T10:56:30.635 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.635 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.636 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.636 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:96592 8.13 8:ca14b56d:::benchmark_data_smithi125_329074_object96591:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e294)", 2023-10-09T10:56:30.636 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:47.596655+0000", 2023-10-09T10:56:30.636 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 102.995776432, 2023-10-09T10:56:30.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5560402770000001, 2023-10-09T10:56:30.636 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.636 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.636 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:30.636 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:30.636 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:30.637 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 96592 2023-10-09T10:56:30.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.637 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596655+0000", 2023-10-09T10:56:30.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596655+0000", 2023-10-09T10:56:30.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.638 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596654+0000", 2023-10-09T10:56:30.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:30.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.638 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596697+0000", 2023-10-09T10:56:30.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2533000000000001e-05 2023-10-09T10:56:30.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596704+0000", 2023-10-09T10:56:30.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7229999999999998e-06 2023-10-09T10:56:30.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596709+0000", 2023-10-09T10:56:30.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4999999999999999e-06 2023-10-09T10:56:30.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.599747+0000", 2023-10-09T10:56:30.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030383699999999999 2023-10-09T10:56:30.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.640 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.640 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.599754+0000", 2023-10-09T10:56:30.640 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7469999999999999e-06 2023-10-09T10:56:30.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.640 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.640 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906306+0000", 2023-10-09T10:56:30.640 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.30655143000000001 2023-10-09T10:56:30.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.641 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906313+0000", 2023-10-09T10:56:30.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3889999999999999e-06 2023-10-09T10:56:30.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.641 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:48.954902+0000", 2023-10-09T10:56:30.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0485886760000001 2023-10-09T10:56:30.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.641 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:48.955096+0000", 2023-10-09T10:56:30.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000194153 2023-10-09T10:56:30.642 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.642 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.642 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:30.642 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:48.955540+0000", 2023-10-09T10:56:30.642 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00044427699999999997 2023-10-09T10:56:30.642 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.642 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.642 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:30.642 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.078123+0000", 2023-10-09T10:56:30.642 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12258316800000001 2023-10-09T10:56:30.642 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.642 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.643 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:30.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.152432+0000", 2023-10-09T10:56:30.643 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.074309119000000007 2023-10-09T10:56:30.643 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.643 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.643 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:30.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.152498+0000", 2023-10-09T10:56:30.643 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5864999999999994e-05 2023-10-09T10:56:30.643 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.643 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.643 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.152555+0000", 2023-10-09T10:56:30.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7117000000000002e-05 2023-10-09T10:56:30.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.152695+0000", 2023-10-09T10:56:30.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000139612 2023-10-09T10:56:30.644 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.644 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.644 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.644 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:96594 8.3 8:c5c4186d:::benchmark_data_smithi125_329074_object96593:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e294)", 2023-10-09T10:56:30.645 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:47.596714+0000", 2023-10-09T10:56:30.645 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 102.995717532, 2023-10-09T10:56:30.645 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5678602609999999, 2023-10-09T10:56:30.645 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.645 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.645 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:30.645 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:30.645 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:30.645 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 96594 2023-10-09T10:56:30.645 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.645 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.645 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.646 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.646 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596714+0000", 2023-10-09T10:56:30.646 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.646 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.646 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.646 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.646 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596714+0000", 2023-10-09T10:56:30.646 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.646 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.646 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.646 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.647 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596713+0000", 2023-10-09T10:56:30.647 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:30.647 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.647 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.647 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.647 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596757+0000", 2023-10-09T10:56:30.647 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3659000000000003e-05 2023-10-09T10:56:30.647 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.647 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.647 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.647 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596764+0000", 2023-10-09T10:56:30.647 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6579999999999996e-06 2023-10-09T10:56:30.647 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.648 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.648 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.648 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.596769+0000", 2023-10-09T10:56:30.648 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3770000000000004e-06 2023-10-09T10:56:30.648 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.648 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.648 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.648 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.599776+0000", 2023-10-09T10:56:30.648 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030069699999999999 2023-10-09T10:56:30.648 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.648 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.648 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.599784+0000", 2023-10-09T10:56:30.649 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7740000000000004e-06 2023-10-09T10:56:30.649 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906246+0000", 2023-10-09T10:56:30.649 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.30646272099999999 2023-10-09T10:56:30.649 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906254+0000", 2023-10-09T10:56:30.649 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8800000000000008e-06 2023-10-09T10:56:30.650 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.650 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.650 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.650 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.155926+0000", 2023-10-09T10:56:30.650 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2496721580000001 2023-10-09T10:56:30.650 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.650 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.650 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.650 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.156097+0000", 2023-10-09T10:56:30.650 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017074000000000001 2023-10-09T10:56:30.650 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.650 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.651 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:30.651 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.156414+0000", 2023-10-09T10:56:30.651 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00031734800000000001 2023-10-09T10:56:30.651 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.651 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.651 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:30.651 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.159828+0000", 2023-10-09T10:56:30.651 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0034136869999999999 2023-10-09T10:56:30.651 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.651 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.651 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:30.651 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.164274+0000", 2023-10-09T10:56:30.651 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0044460360000000004 2023-10-09T10:56:30.652 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.652 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.652 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:30.652 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.164310+0000", 2023-10-09T10:56:30.652 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.608e-05 2023-10-09T10:56:30.652 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.652 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.652 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.652 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.164388+0000", 2023-10-09T10:56:30.652 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8134999999999996e-05 2023-10-09T10:56:30.652 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.652 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.653 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.653 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.164574+0000", 2023-10-09T10:56:30.653 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018543499999999999 2023-10-09T10:56:30.653 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.653 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.653 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.653 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.653 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.653 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:96637 8.13 8:ca18ae91:::benchmark_data_smithi125_329074_object96636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2023-10-09T10:56:30.653 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:47.680055+0000", 2023-10-09T10:56:30.653 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 102.912376369, 2023-10-09T10:56:30.653 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.472911826, 2023-10-09T10:56:30.654 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.654 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.654 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:30.654 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:30.654 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:30.654 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 96637 2023-10-09T10:56:30.654 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.654 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.654 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.654 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.654 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.680055+0000", 2023-10-09T10:56:30.654 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.655 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.655 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.655 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.680055+0000", 2023-10-09T10:56:30.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.655 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.655 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.655 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.680052+0000", 2023-10-09T10:56:30.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T10:56:30.655 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.655 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.656 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.680110+0000", 2023-10-09T10:56:30.656 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8031000000000001e-05 2023-10-09T10:56:30.656 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.656 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.656 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.656 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.680124+0000", 2023-10-09T10:56:30.656 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3461e-05 2023-10-09T10:56:30.656 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.656 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.656 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.656 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.680134+0000", 2023-10-09T10:56:30.656 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0531e-05 2023-10-09T10:56:30.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.657 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.680199+0000", 2023-10-09T10:56:30.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5054000000000004e-05 2023-10-09T10:56:30.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.657 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.680222+0000", 2023-10-09T10:56:30.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2235999999999999e-05 2023-10-09T10:56:30.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.658 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906337+0000", 2023-10-09T10:56:30.658 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.22611497 2023-10-09T10:56:30.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.658 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:56:30.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.906349+0000", 2023-10-09T10:56:30.658 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.235e-05 2023-10-09T10:56:30.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.658 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:48.957239+0000", 2023-10-09T10:56:30.659 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0508904670000001 2023-10-09T10:56:30.659 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.659 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.659 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.659 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:48.957434+0000", 2023-10-09T10:56:30.659 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019483200000000001 2023-10-09T10:56:30.659 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.659 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.659 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:30.659 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:48.957819+0000", 2023-10-09T10:56:30.659 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00038458399999999999 2023-10-09T10:56:30.659 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.660 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.660 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:30.660 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.078133+0000", 2023-10-09T10:56:30.660 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.120314265 2023-10-09T10:56:30.660 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.660 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.660 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:30.660 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.152737+0000", 2023-10-09T10:56:30.660 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.074604132000000004 2023-10-09T10:56:30.660 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.660 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.660 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:30.661 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.152771+0000", 2023-10-09T10:56:30.661 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3939999999999997e-05 2023-10-09T10:56:30.661 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.661 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.661 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.661 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.152819+0000", 2023-10-09T10:56:30.661 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7790000000000002e-05 2023-10-09T10:56:30.661 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.661 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.661 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.661 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.152967+0000", 2023-10-09T10:56:30.661 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014764199999999999 2023-10-09T10:56:30.661 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.662 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.662 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.662 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.662 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.662 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:110769 8.f e302/295 8:f6972c7a:::benchmark_data_smithi125_329074_object14115:head v 302'6976, mlcod=275'4755)", 2023-10-09T10:56:30.662 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:08.953833+0000", 2023-10-09T10:56:30.662 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 81.638598216999995, 2023-10-09T10:56:30.662 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.04514688, 2023-10-09T10:56:30.662 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.662 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.662 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.662 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.663 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.663 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.953833+0000", 2023-10-09T10:56:30.663 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.663 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.663 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.663 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.663 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.953833+0000", 2023-10-09T10:56:30.663 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.663 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.663 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.663 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.663 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.953832+0000", 2023-10-09T10:56:30.664 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:30.664 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.664 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.664 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.664 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.953839+0000", 2023-10-09T10:56:30.664 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7519999999999996e-06 2023-10-09T10:56:30.664 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.664 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.664 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.664 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.953859+0000", 2023-10-09T10:56:30.664 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0007999999999999e-05 2023-10-09T10:56:30.664 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.665 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.665 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.665 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.953875+0000", 2023-10-09T10:56:30.665 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5324000000000001e-05 2023-10-09T10:56:30.665 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.665 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.665 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.665 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.968490+0000", 2023-10-09T10:56:30.665 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014615352 2023-10-09T10:56:30.665 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.665 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.665 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.665 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.968519+0000", 2023-10-09T10:56:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9369e-05 2023-10-09T10:56:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.998943+0000", 2023-10-09T10:56:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.03042359 2023-10-09T10:56:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.998980+0000", 2023-10-09T10:56:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6705999999999997e-05 2023-10-09T10:56:30.666 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:110773 8.f e302/295 8:fe843335:::benchmark_data_smithi125_329074_object14119:head v 302'6977, mlcod=275'4755)", 2023-10-09T10:56:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:08.958009+0000", 2023-10-09T10:56:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 81.634422119000007, 2023-10-09T10:56:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.041010304999999997, 2023-10-09T10:56:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.667 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.958009+0000", 2023-10-09T10:56:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.958009+0000", 2023-10-09T10:56:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.958008+0000", 2023-10-09T10:56:30.668 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.958020+0000", 2023-10-09T10:56:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1982e-05 2023-10-09T10:56:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.958031+0000", 2023-10-09T10:56:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1474999999999999e-05 2023-10-09T10:56:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.669 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.958041+0000", 2023-10-09T10:56:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6949999999999995e-06 2023-10-09T10:56:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.972091+0000", 2023-10-09T10:56:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014049997 2023-10-09T10:56:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.972118+0000", 2023-10-09T10:56:30.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7341000000000001e-05 2023-10-09T10:56:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.998983+0000", 2023-10-09T10:56:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.026864485 2023-10-09T10:56:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.999019+0000", 2023-10-09T10:56:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6678999999999999e-05 2023-10-09T10:56:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.671 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:110777 8.f e302/295 8:fc3c80be:::benchmark_data_smithi125_329074_object14123:head v 302'6978, mlcod=275'4755)", 2023-10-09T10:56:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:08.958075+0000", 2023-10-09T10:56:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 81.634355826999993, 2023-10-09T10:56:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.040965205999999997, 2023-10-09T10:56:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.958075+0000", 2023-10-09T10:56:30.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.958075+0000", 2023-10-09T10:56:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.958075+0000", 2023-10-09T10:56:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967296 2023-10-09T10:56:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.958081+0000", 2023-10-09T10:56:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0070000000000003e-06 2023-10-09T10:56:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.958086+0000", 2023-10-09T10:56:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1139999999999997e-06 2023-10-09T10:56:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.958090+0000", 2023-10-09T10:56:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8360000000000002e-06 2023-10-09T10:56:30.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.975766+0000", 2023-10-09T10:56:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.017676112000000001 2023-10-09T10:56:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.975790+0000", 2023-10-09T10:56:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3609999999999999e-05 2023-10-09T10:56:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.675 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.999022+0000", 2023-10-09T10:56:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.023231979 2023-10-09T10:56:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:08.999041+0000", 2023-10-09T10:56:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8751000000000001e-05 2023-10-09T10:56:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:125177 8.13 8:cf9ed410:::benchmark_data_smithi125_329074_object28523:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T10:56:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:29.926634+0000", 2023-10-09T10:56:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 60.665796747999998, 2023-10-09T10:56:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.043878526000000001, 2023-10-09T10:56:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 125177 2023-10-09T10:56:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.677 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:29.926634+0000", 2023-10-09T10:56:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:29.926634+0000", 2023-10-09T10:56:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.678 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:29.926634+0000", 2023-10-09T10:56:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:29.926637+0000", 2023-10-09T10:56:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2669999999999998e-06 2023-10-09T10:56:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:29.926644+0000", 2023-10-09T10:56:30.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1969999999999999e-06 2023-10-09T10:56:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:29.926651+0000", 2023-10-09T10:56:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3940000000000001e-06 2023-10-09T10:56:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:29.929041+0000", 2023-10-09T10:56:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002390433 2023-10-09T10:56:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:29.929188+0000", 2023-10-09T10:56:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014687200000000001 2023-10-09T10:56:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:29.939715+0000", 2023-10-09T10:56:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010526512999999999 2023-10-09T10:56:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:29.956383+0000", 2023-10-09T10:56:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.016668081000000001 2023-10-09T10:56:30.681 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:29.964471+0000", 2023-10-09T10:56:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0080884370000000004 2023-10-09T10:56:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:29.970415+0000", 2023-10-09T10:56:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0059439860000000001 2023-10-09T10:56:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.682 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:29.970468+0000", 2023-10-09T10:56:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3109999999999998e-05 2023-10-09T10:56:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:29.970513+0000", 2023-10-09T10:56:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4771000000000002e-05 2023-10-09T10:56:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:125381 8.a e307/295 8:5efa18af:::benchmark_data_smithi125_329074_object28727:head v 307'7751, mlcod=275'4625)", 2023-10-09T10:56:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:30.280519+0000", 2023-10-09T10:56:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 60.311911698000003, 2023-10-09T10:56:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.04259988, 2023-10-09T10:56:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:30.280519+0000", 2023-10-09T10:56:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.684 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:30.280519+0000", 2023-10-09T10:56:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:30.280519+0000", 2023-10-09T10:56:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.685 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:30.280524+0000", 2023-10-09T10:56:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6400000000000002e-06 2023-10-09T10:56:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:30.280534+0000", 2023-10-09T10:56:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2210000000000003e-06 2023-10-09T10:56:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:30.280540+0000", 2023-10-09T10:56:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.93e-06 2023-10-09T10:56:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:30.296264+0000", 2023-10-09T10:56:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015724360999999999 2023-10-09T10:56:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:30.296290+0000", 2023-10-09T10:56:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5965e-05 2023-10-09T10:56:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:30.323067+0000", 2023-10-09T10:56:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.026776898 2023-10-09T10:56:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:30.323119+0000", 2023-10-09T10:56:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2465e-05 2023-10-09T10:56:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:137828 8.a e310/295 8:58447bc4:::benchmark_data_smithi125_329074_object41174:head v 310'8525, mlcod=275'4625)", 2023-10-09T10:56:30.688 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:55:47.224054+0000", 2023-10-09T10:56:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 43.368377064000001, 2023-10-09T10:56:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.040498115000000001, 2023-10-09T10:56:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.224054+0000", 2023-10-09T10:56:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.224054+0000", 2023-10-09T10:56:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.224053+0000", 2023-10-09T10:56:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:56:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.224063+0000", 2023-10-09T10:56:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0339e-05 2023-10-09T10:56:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.224079+0000", 2023-10-09T10:56:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5860000000000001e-05 2023-10-09T10:56:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.224087+0000", 2023-10-09T10:56:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7100000000000007e-06 2023-10-09T10:56:30.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.251792+0000", 2023-10-09T10:56:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.027704909 2023-10-09T10:56:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.251826+0000", 2023-10-09T10:56:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3980000000000003e-05 2023-10-09T10:56:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.264522+0000", 2023-10-09T10:56:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012696107999999999 2023-10-09T10:56:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:55:47.264552+0000", 2023-10-09T10:56:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0341e-05 2023-10-09T10:56:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.693 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:158453 8.f e313/295 8:ff705f87:::benchmark_data_smithi125_329074_object61799:head v 313'10023, mlcod=275'4755)", 2023-10-09T10:56:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:14.773309+0000", 2023-10-09T10:56:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 15.819122457000001, 2023-10-09T10:56:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.042881305000000002, 2023-10-09T10:56:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.773309+0000", 2023-10-09T10:56:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.773309+0000", 2023-10-09T10:56:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.773308+0000", 2023-10-09T10:56:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:56:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.773318+0000", 2023-10-09T10:56:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.075e-05 2023-10-09T10:56:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.773338+0000", 2023-10-09T10:56:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9227000000000001e-05 2023-10-09T10:56:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.773347+0000", 2023-10-09T10:56:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7790000000000007e-06 2023-10-09T10:56:30.696 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.697 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.697 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.697 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.774336+0000", 2023-10-09T10:56:30.697 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00098902899999999999 2023-10-09T10:56:30.697 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.697 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.697 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.697 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.774361+0000", 2023-10-09T10:56:30.697 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5055e-05 2023-10-09T10:56:30.697 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.697 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.697 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.697 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.816138+0000", 2023-10-09T10:56:30.698 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.041776714 2023-10-09T10:56:30.698 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.698 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.698 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.698 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.816190+0000", 2023-10-09T10:56:30.698 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1901000000000003e-05 2023-10-09T10:56:30.698 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.698 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.698 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.698 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.698 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.698 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:158457 8.f e313/295 8:fb8fcfff:::benchmark_data_smithi125_329074_object61803:head v 313'10024, mlcod=275'4755)", 2023-10-09T10:56:30.699 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:14.784668+0000", 2023-10-09T10:56:30.699 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 15.807763211999999, 2023-10-09T10:56:30.699 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.059385282999999997, 2023-10-09T10:56:30.699 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.699 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.699 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.699 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.699 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.699 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.784668+0000", 2023-10-09T10:56:30.699 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.699 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.699 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.784668+0000", 2023-10-09T10:56:30.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.784668+0000", 2023-10-09T10:56:30.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:30.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.784672+0000", 2023-10-09T10:56:30.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2490000000000003e-06 2023-10-09T10:56:30.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.701 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.701 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.784677+0000", 2023-10-09T10:56:30.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1479999999999997e-06 2023-10-09T10:56:30.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.701 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.701 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.784681+0000", 2023-10-09T10:56:30.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.997e-06 2023-10-09T10:56:30.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.702 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.802550+0000", 2023-10-09T10:56:30.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.017868627000000002 2023-10-09T10:56:30.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.702 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.802585+0000", 2023-10-09T10:56:30.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4921e-05 2023-10-09T10:56:30.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.844012+0000", 2023-10-09T10:56:30.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.041427216000000003 2023-10-09T10:56:30.703 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.844053+0000", 2023-10-09T10:56:30.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1363999999999999e-05 2023-10-09T10:56:30.703 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.703 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.703 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.703 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.704 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:158456 8.17 e313/295 8:e9532416:::benchmark_data_smithi125_329074_object61802:head v 313'7867, mlcod=275'4602)", 2023-10-09T10:56:30.704 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:14.784728+0000", 2023-10-09T10:56:30.704 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 15.807703246999999, 2023-10-09T10:56:30.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.059557423999999998, 2023-10-09T10:56:30.704 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.704 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.704 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.784728+0000", 2023-10-09T10:56:30.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.784728+0000", 2023-10-09T10:56:30.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.784728+0000", 2023-10-09T10:56:30.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967296 2023-10-09T10:56:30.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.784734+0000", 2023-10-09T10:56:30.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6440000000000003e-06 2023-10-09T10:56:30.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.706 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.784739+0000", 2023-10-09T10:56:30.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6140000000000002e-06 2023-10-09T10:56:30.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.706 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.784743+0000", 2023-10-09T10:56:30.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6909999999999999e-06 2023-10-09T10:56:30.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.707 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.707 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.816248+0000", 2023-10-09T10:56:30.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.031505534000000002 2023-10-09T10:56:30.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.707 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.707 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.816276+0000", 2023-10-09T10:56:30.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7328999999999999e-05 2023-10-09T10:56:30.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.707 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.708 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.844056+0000", 2023-10-09T10:56:30.708 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.027780732999999998 2023-10-09T10:56:30.708 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.708 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.844285+0000", 2023-10-09T10:56:30.708 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000229058 2023-10-09T10:56:30.708 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.708 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.708 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.708 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.709 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.709 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:158460 8.f e313/295 8:fe05340b:::benchmark_data_smithi125_329074_object61806:head v 313'10025, mlcod=275'4755)", 2023-10-09T10:56:30.709 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:14.803508+0000", 2023-10-09T10:56:30.709 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 15.788922711, 2023-10-09T10:56:30.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.040831547000000003, 2023-10-09T10:56:30.709 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.709 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.709 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.709 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.709 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.709 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.803508+0000", 2023-10-09T10:56:30.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.709 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.710 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.710 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.803508+0000", 2023-10-09T10:56:30.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.710 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.710 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.803508+0000", 2023-10-09T10:56:30.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:30.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.710 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.803517+0000", 2023-10-09T10:56:30.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5759999999999995e-06 2023-10-09T10:56:30.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.803544+0000", 2023-10-09T10:56:30.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6733e-05 2023-10-09T10:56:30.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.803553+0000", 2023-10-09T10:56:30.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9220000000000003e-06 2023-10-09T10:56:30.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.819675+0000", 2023-10-09T10:56:30.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.016121694999999998 2023-10-09T10:56:30.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.819707+0000", 2023-10-09T10:56:30.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2221999999999997e-05 2023-10-09T10:56:30.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.844315+0000", 2023-10-09T10:56:30.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.024608310000000001 2023-10-09T10:56:30.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:14.844340+0000", 2023-10-09T10:56:30.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4762000000000001e-05 2023-10-09T10:56:30.713 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.713 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.713 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.714 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:169062 8.f e315/295 8:f533903d:::benchmark_data_smithi125_329074_object72408:head v 315'10712, mlcod=275'4755)", 2023-10-09T10:56:30.714 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:30.184706+0000", 2023-10-09T10:56:30.714 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.40772479499999997, 2023-10-09T10:56:30.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.040517926000000003, 2023-10-09T10:56:30.714 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.714 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.714 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.184706+0000", 2023-10-09T10:56:30.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.184706+0000", 2023-10-09T10:56:30.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.184705+0000", 2023-10-09T10:56:30.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:30.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.184716+0000", 2023-10-09T10:56:30.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1246999999999999e-05 2023-10-09T10:56:30.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.184737+0000", 2023-10-09T10:56:30.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.128e-05 2023-10-09T10:56:30.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.184754+0000", 2023-10-09T10:56:30.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6470999999999999e-05 2023-10-09T10:56:30.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.191471+0000", 2023-10-09T10:56:30.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0067176550000000003 2023-10-09T10:56:30.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.191501+0000", 2023-10-09T10:56:30.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9524999999999999e-05 2023-10-09T10:56:30.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.718 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.225178+0000", 2023-10-09T10:56:30.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.033677064 2023-10-09T10:56:30.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.718 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.718 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.225224+0000", 2023-10-09T10:56:30.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6264999999999998e-05 2023-10-09T10:56:30.718 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.718 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.718 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.719 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:169067 8.f e315/295 8:f9993d31:::benchmark_data_smithi125_329074_object72413:head v 315'10713, mlcod=275'4755)", 2023-10-09T10:56:30.719 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:30.186473+0000", 2023-10-09T10:56:30.719 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.40595778199999999, 2023-10-09T10:56:30.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.043476439999999998, 2023-10-09T10:56:30.719 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:30.719 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:30.719 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:30.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:30.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.186473+0000", 2023-10-09T10:56:30.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:30.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.186473+0000", 2023-10-09T10:56:30.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:30.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:30.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.186472+0000", 2023-10-09T10:56:30.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:56:30.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:30.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.186482+0000", 2023-10-09T10:56:30.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6779999999999995e-06 2023-10-09T10:56:30.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:30.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.186496+0000", 2023-10-09T10:56:30.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4214e-05 2023-10-09T10:56:30.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:30.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.186505+0000", 2023-10-09T10:56:30.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7479999999999999e-06 2023-10-09T10:56:30.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:30.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.198475+0000", 2023-10-09T10:56:30.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011970539000000001 2023-10-09T10:56:30.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:30.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.198497+0000", 2023-10-09T10:56:30.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1778000000000001e-05 2023-10-09T10:56:30.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:30.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.229904+0000", 2023-10-09T10:56:30.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.031406431999999998 2023-10-09T10:56:30.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:30.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:30.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:30.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:30.229950+0000", 2023-10-09T10:56:30.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6162999999999998e-05 2023-10-09T10:56:30.723 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.723 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.723 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.723 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:30.723 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:30.724 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:56:30.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:30.725 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:30.725 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:30.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:30.806 INFO:teuthology.orchestra.run.smithi125.stderr:Error EINVAL: specified pg_num 51 < pg_num_max 32 2023-10-09T10:56:30.826 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:30.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:30 smithi125 ceph-mon[195507]: failed to encode map e315 with expected crc 2023-10-09T10:56:30.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:30 smithi125 ceph-mon[195507]: pgmap v774: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.6 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 13796/80805 objects degraded (17.073%); 5935/80805 objects misplaced (7.345%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T10:56:30.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:56:30.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:56:30.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:30 smithi125 ceph-mon[200197]: failed to encode map e315 with expected crc 2023-10-09T10:56:30.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:30 smithi125 ceph-mon[200197]: pgmap v774: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.6 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 13796/80805 objects degraded (17.073%); 5935/80805 objects misplaced (7.345%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T10:56:30.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:56:30.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:56:30.893 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:30 smithi155 ceph-mon[184854]: failed to encode map e315 with expected crc 2023-10-09T10:56:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:30 smithi155 ceph-mon[184854]: pgmap v774: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.6 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 13796/80805 objects degraded (17.073%); 5935/80805 objects misplaced (7.345%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T10:56:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:56:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:56:30.995 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:31.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:31.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:31.027 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:56:31.105 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:31.187 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:31.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:31.333 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:31.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:31.361 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T10:56:31.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:31.476 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:31.507 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:31.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:31.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:31.575 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:56:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:56:31] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T10:56:31.579 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:31.610 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:31.643 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:31.667 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:56:31.675 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:31.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:31.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:31.781 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:31.810 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3724626525' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T10:56:31.810 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:31 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T10:56:31.813 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:31.835 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:31.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:31.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:31.941 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:31.977 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:32.000 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3724626525' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T10:56:32.000 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:31 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T10:56:32.001 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:32.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:32.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:32.004 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3724626525' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T10:56:32.004 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:31 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T10:56:32.007 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:56:32.104 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:32.141 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:32.163 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:32.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:32.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:32.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:32.300 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:32.326 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:32.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:32.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:32.428 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:32.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:32.513 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:32.545 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:32.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:32.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:32.649 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:32.707 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:32.753 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:32.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:32.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:32.858 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:32.891 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:32.916 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:32.918 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:32 smithi155 ceph-mon[184854]: pgmap v775: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 13796/74277 objects degraded (18.574%); 5261/74277 objects misplaced (7.083%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T10:56:32.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:32.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:33.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:33.034 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:33.051 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:33.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:32 smithi125 ceph-mon[195507]: pgmap v775: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 13796/74277 objects degraded (18.574%); 5261/74277 objects misplaced (7.083%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T10:56:33.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:32 smithi125 ceph-mon[200197]: pgmap v775: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 13796/74277 objects degraded (18.574%); 5261/74277 objects misplaced (7.083%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T10:56:33.073 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:33.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:33.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:33.103 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:56:33.103 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:56:33.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.103 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:170992 8.e e315/282 8:7bec26b2:::benchmark_data_smithi125_329074_object74338:head v 315'10744, mlcod=0'0)", 2023-10-09T10:56:33.103 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:33.090263+0000", 2023-10-09T10:56:33.103 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0066513409999999999, 2023-10-09T10:56:33.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066845330000000003, 2023-10-09T10:56:33.104 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:33.104 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:56:33.104 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:33.104 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.104 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:33.104 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.090263+0000", 2023-10-09T10:56:33.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:33.104 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:33.104 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.104 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:33.104 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.090263+0000", 2023-10-09T10:56:33.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:33.105 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:33.105 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.105 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:33.105 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.090262+0000", 2023-10-09T10:56:33.105 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:33.105 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:33.105 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.105 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:33.105 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.090272+0000", 2023-10-09T10:56:33.105 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6560000000000004e-06 2023-10-09T10:56:33.105 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:33.105 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.105 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:33.106 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.090283+0000", 2023-10-09T10:56:33.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1117999999999999e-05 2023-10-09T10:56:33.106 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:33.106 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.106 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:33.106 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.090290+0000", 2023-10-09T10:56:33.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4270000000000002e-06 2023-10-09T10:56:33.106 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:33.106 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.106 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:33.106 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.090492+0000", 2023-10-09T10:56:33.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000201936 2023-10-09T10:56:33.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:33.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:33.107 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.090520+0000", 2023-10-09T10:56:33.107 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7402999999999998e-05 2023-10-09T10:56:33.107 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:33.107 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:33.107 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:33.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:33.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.107 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:171041 8.e e315/282 8:7a7bdbda:::benchmark_data_smithi125_329074_object74387:head v 315'10745, mlcod=0'0)", 2023-10-09T10:56:33.107 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:33.092745+0000", 2023-10-09T10:56:33.108 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0041696349999999997, 2023-10-09T10:56:33.108 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042692229999999999, 2023-10-09T10:56:33.108 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:33.108 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:56:33.108 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:33.108 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.108 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:33.108 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.092745+0000", 2023-10-09T10:56:33.108 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:33.108 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:33.108 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.108 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:33.108 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.092745+0000", 2023-10-09T10:56:33.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:33.109 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:33.109 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.109 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:33.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.092744+0000", 2023-10-09T10:56:33.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:56:33.109 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:33.109 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.109 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:33.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.092752+0000", 2023-10-09T10:56:33.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2449999999999998e-06 2023-10-09T10:56:33.109 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:33.109 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:33.110 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.092766+0000", 2023-10-09T10:56:33.110 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3229e-05 2023-10-09T10:56:33.110 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:33.110 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:33.110 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.092773+0000", 2023-10-09T10:56:33.110 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3100000000000003e-06 2023-10-09T10:56:33.110 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:33.110 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:33.110 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.095429+0000", 2023-10-09T10:56:33.111 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002656303 2023-10-09T10:56:33.111 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:33.111 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:33.111 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:33.111 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:33.095454+0000", 2023-10-09T10:56:33.111 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4389e-05 2023-10-09T10:56:33.111 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:33.111 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:33.111 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:33.111 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:33.111 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:56:33.111 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 2 2023-10-09T10:56:33.111 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:56:33.179 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:33.207 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:33.234 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:33.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:33.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:33.338 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:33.370 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:33.396 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:33.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:33.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:33.500 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:33.540 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:33.567 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:33.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:33.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:33.673 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:33.713 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:33.742 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:33.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:33.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:33.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:33.918 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:33 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 13796/74277 objects degraded (18.574%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:33.920 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:33 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 13796/74277 objects degraded (18.574%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:33.921 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:33 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 13796/74277 objects degraded (18.574%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:34 smithi155 ceph-mon[184854]: pgmap v776: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 13657/70359 objects degraded (19.410%); 5031/70359 objects misplaced (7.150%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T10:56:35.031 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:34 smithi125 ceph-mon[195507]: pgmap v776: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 13657/70359 objects degraded (19.410%); 5031/70359 objects misplaced (7.150%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T10:56:35.031 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:34 smithi125 ceph-mon[200197]: pgmap v776: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 13657/70359 objects degraded (19.410%); 5031/70359 objects misplaced (7.150%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T10:56:35.063 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:35.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:35.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:35.167 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:35.198 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:35.223 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:35.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:35.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:35.335 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:35.383 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:35.781 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:56:35.977 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:35.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:35.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:36.085 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:36.115 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:36.139 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:36.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:36.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:36.242 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:36.272 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:36.294 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:36.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:36.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:36.363 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 7, 2, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:56:36.363 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:56:36.363 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T10:56:36.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:56:36.391 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:56:36.399 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:36.429 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:56:36.429 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:36.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:36.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:36.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:36.465 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:56:36.502 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:56:36.535 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:56:36.557 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:36.582 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:56:36.586 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:36.613 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:36.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:36.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:36.618 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:56:36.665 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:56:36.708 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:56:36.708 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:56:36.709 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:56:36.709 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:56:36.709 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:56:36.709 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:56:36.709 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:56:36.709 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:56:36.716 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:36.751 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:36.784 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:36.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:36.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:36.892 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:36.931 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:36.957 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:36.961 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:36 smithi155 ceph-mon[184854]: pgmap v777: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 13657/70359 objects degraded (19.410%); 5031/70359 objects misplaced (7.150%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T10:56:36.961 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:36 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/652355472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:36.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:36.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:37.063 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:37.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:36 smithi125 ceph-mon[195507]: pgmap v777: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 13657/70359 objects degraded (19.410%); 5031/70359 objects misplaced (7.150%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T10:56:37.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:36 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/652355472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:37.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:36 smithi125 ceph-mon[200197]: pgmap v777: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 13657/70359 objects degraded (19.410%); 5031/70359 objects misplaced (7.150%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T10:56:37.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:36 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/652355472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:37.118 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:37.691 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:37.708 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:37.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:37.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:37.814 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:37.843 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:37.867 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:37.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:37.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:37.973 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:38.039 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:38.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:38.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:38.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:38.195 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:38.233 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:38.256 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:38.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:38.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:38.296 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:38.360 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:38.395 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:38.419 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:38.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:38.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:38.526 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:38.592 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:38.806 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:38.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:38.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:38.877 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:38 smithi125 ceph-mon[195507]: pgmap v778: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.3 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 13467/63012 objects degraded (21.372%); 4571/63012 objects misplaced (7.254%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T10:56:38.878 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:38 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 13467/63012 objects degraded (21.372%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:38.878 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:38 smithi125 ceph-mon[200197]: pgmap v778: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.3 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 13467/63012 objects degraded (21.372%); 4571/63012 objects misplaced (7.254%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T10:56:38.878 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:38 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 13467/63012 objects degraded (21.372%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:38.918 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:38.946 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:38.972 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:38.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:38 smithi155 ceph-mon[184854]: pgmap v778: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.3 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 13467/63012 objects degraded (21.372%); 4571/63012 objects misplaced (7.254%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T10:56:38.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:38 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 13467/63012 objects degraded (21.372%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:38.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:38.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:39.077 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:39.105 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:39.127 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:39.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:39.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:39.231 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:39.314 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:39.409 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:39.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:39.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:39.504 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:56:39.504 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:56:39.515 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:39.588 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:39.750 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:39 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1711285914' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:56:39.751 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:39 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:56:39.794 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:56:40.050 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:56:40.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:39 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1711285914' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:56:40.075 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:39 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:56:40.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:39 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1711285914' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:56:40.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:39 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:56:41.000 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:41.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:41.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:41.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[195507]: pgmap v779: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 13130/59427 objects degraded (22.094%); 4320/59427 objects misplaced (7.269%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:56:41.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:56:41.008 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:56:41.008 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[195507]: osdmap e316: 8 total, 8 up, 8 in 2023-10-09T10:56:41.008 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[195507]: failed to encode map e316 with expected crc 2023-10-09T10:56:41.008 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[195507]: failed to encode map e316 with expected crc 2023-10-09T10:56:41.008 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[195507]: failed to encode map e316 with expected crc 2023-10-09T10:56:41.008 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[195507]: failed to encode map e316 with expected crc 2023-10-09T10:56:41.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[200197]: pgmap v779: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 13130/59427 objects degraded (22.094%); 4320/59427 objects misplaced (7.269%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:56:41.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:56:41.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:56:41.009 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[200197]: osdmap e316: 8 total, 8 up, 8 in 2023-10-09T10:56:41.009 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[200197]: failed to encode map e316 with expected crc 2023-10-09T10:56:41.009 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[200197]: failed to encode map e316 with expected crc 2023-10-09T10:56:41.009 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[200197]: failed to encode map e316 with expected crc 2023-10-09T10:56:41.009 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:40 smithi125 ceph-mon[200197]: failed to encode map e316 with expected crc 2023-10-09T10:56:41.106 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:41.176 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:40 smithi155 ceph-mon[184854]: pgmap v779: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 13130/59427 objects degraded (22.094%); 4320/59427 objects misplaced (7.269%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T10:56:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:40 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:56:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:40 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:56:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:40 smithi155 ceph-mon[184854]: osdmap e316: 8 total, 8 up, 8 in 2023-10-09T10:56:41.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:40 smithi155 ceph-mon[184854]: failed to encode map e316 with expected crc 2023-10-09T10:56:41.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:40 smithi155 ceph-mon[184854]: failed to encode map e316 with expected crc 2023-10-09T10:56:41.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:40 smithi155 ceph-mon[184854]: failed to encode map e316 with expected crc 2023-10-09T10:56:41.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:40 smithi155 ceph-mon[184854]: failed to encode map e316 with expected crc 2023-10-09T10:56:41.322 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:56:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:56:41] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T10:56:41.373 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:41.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:41.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:41.478 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:41.511 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:41.540 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:41.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:41.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:41.619 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:41.645 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:41.656 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:56:41.741 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:41.822 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:41.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:41.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:41.935 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:42.147 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:42.175 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:56:42.183 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:56:42.585 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:42.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:42.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:42.690 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:42.727 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:42.767 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:42.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:42.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:42.807 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:42.823 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":781,"stamp":"2023-10-09T10:56:41.488986+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1159673168,"num_objects":17709,"num_object_clones":0,"num_object_copies":53127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12725,"num_objects_misplaced":4095,"num_objects_unfound":0,"num_objects_dirty":17709,"num_whiteouts":0,"num_read":116,"num_read_kb":215,"num_write":175736,"num_write_kb":6186261,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11088,"num_bytes_recovered":724760338,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":193814,"ondisk_log_size":193814,"up":387,"acting":377,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":421,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8849840,"kb_used_data":4629232,"kb_used_omap":126,"kb_used_meta":4220289,"kb_avail":741111376,"statfs":{"total":767960285184,"available":758898049024,"internally_reserved":0,"allocated":4740333568,"data_stored":4729086506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":4321576571},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-462028800,"num_objects":-7050,"num_object_clones":0,"num_object_copies":-21150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1071,"num_objects_misplaced":-1166,"num_objects_unfound":0,"num_objects_dirty":-7050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7050,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":762,"num_bytes_recovered":49938432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008195"},"pg_stats":[{"pgid":"8.11","version":"315'8369","reported_seq":13378,"reported_epoch":315,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:37.535318+0000","last_change":"2023-10-09T10:54:49.220300+0000","last_active":"2023-10-09T10:56:37.535318+0000","last_peered":"2023-10-09T10:56:37.535318+0000","last_clean":"2023-10-09T10:54:02.426241+0000","last_became_active":"2023-10-09T10:54:49.215481+0000","last_became_peered":"2023-10-09T10:54:49.215481+0000","last_unstale":"2023-10-09T10:56:37.535318+0000","last_undegraded":"2023-10-09T10:54:49.026934+0000","last_fullsized":"2023-10-09T10:54:48.893501+0000","mapping_epoch":295,"log_start":"270'284","ondisk_log_start":"270'284","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":8085,"log_dups_size":0,"ondisk_log_size":8085,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38010880,"num_objects":580,"num_object_clones":0,"num_object_copies":1740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":785,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5408,"num_write_kb":191616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2415,"num_bytes_recovered":158269440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,2","objects":-9},{"shards":"0,2,5","objects":393},{"shards":"0,3,5","objects":196}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.18","version":"76'161","reported_seq":630,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.992579+0000","last_change":"2023-10-09T10:55:26.628655+0000","last_active":"2023-10-09T10:56:28.992579+0000","last_peered":"2023-10-09T10:56:28.992579+0000","last_clean":"2023-10-09T10:56:28.992579+0000","last_became_active":"2023-10-09T10:55:26.628269+0000","last_became_peered":"2023-10-09T10:55:26.628269+0000","last_unstale":"2023-10-09T10:56:28.992579+0000","last_undegraded":"2023-10-09T10:56:28.992579+0000","last_fullsized":"2023-10-09T10:56:28.992579+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:40.174975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:40.174975+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:24:16.854746+0000","scrub_duration":0.010654205,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.577119+0000","last_change":"2023-10-09T10:54:47.925818+0000","last_active":"2023-10-09T10:56:19.577119+0000","last_peered":"2023-10-09T10:56:19.577119+0000","last_clean":"2023-10-09T10:56:19.577119+0000","last_became_active":"2023-10-09T10:54:47.925375+0000","last_became_peered":"2023-10-09T10:54:47.925375+0000","last_unstale":"2023-10-09T10:56:19.577119+0000","last_undegraded":"2023-10-09T10:56:19.577119+0000","last_fullsized":"2023-10-09T10:56:19.577119+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:14:27.635383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":324,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.980669+0000","last_change":"2023-10-09T10:54:47.945782+0000","last_active":"2023-10-09T10:56:28.980669+0000","last_peered":"2023-10-09T10:56:28.980669+0000","last_clean":"2023-10-09T10:56:28.980669+0000","last_became_active":"2023-10-09T10:54:47.945460+0000","last_became_peered":"2023-10-09T10:54:47.945460+0000","last_unstale":"2023-10-09T10:56:28.980669+0000","last_undegraded":"2023-10-09T10:56:28.980669+0000","last_fullsized":"2023-10-09T10:56:28.980669+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:50.537469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"316'8645","reported_seq":8719,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T10:56:41.254746+0000","last_change":"2023-10-09T10:54:40.626105+0000","last_active":"2023-10-09T10:56:41.254746+0000","last_peered":"2023-10-09T10:56:41.254746+0000","last_clean":"2023-10-09T10:56:41.254746+0000","last_became_active":"2023-10-09T10:54:40.609428+0000","last_became_peered":"2023-10-09T10:54:40.609428+0000","last_unstale":"2023-10-09T10:56:41.254746+0000","last_undegraded":"2023-10-09T10:56:41.254746+0000","last_fullsized":"2023-10-09T10:56:41.254746+0000","mapping_epoch":289,"log_start":"271'700","ondisk_log_start":"271'700","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7945,"log_dups_size":0,"ondisk_log_size":7945,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34078720,"num_objects":520,"num_object_clones":0,"num_object_copies":1560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5610,"num_write_kb":196160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":696,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.592783+0000","last_change":"2023-10-09T10:53:34.537094+0000","last_active":"2023-10-09T10:56:19.592783+0000","last_peered":"2023-10-09T10:56:19.592783+0000","last_clean":"2023-10-09T10:56:19.592783+0000","last_became_active":"2023-10-09T10:52:55.414995+0000","last_became_peered":"2023-10-09T10:52:55.414995+0000","last_unstale":"2023-10-09T10:56:19.592783+0000","last_undegraded":"2023-10-09T10:56:19.592783+0000","last_fullsized":"2023-10-09T10:56:19.592783+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.536962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.536962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:30.197404+0000","scrub_duration":0.0053325960000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.575303+0000","last_change":"2023-10-09T10:54:47.944768+0000","last_active":"2023-10-09T10:56:19.575303+0000","last_peered":"2023-10-09T10:56:19.575303+0000","last_clean":"2023-10-09T10:56:19.575303+0000","last_became_active":"2023-10-09T10:54:47.943888+0000","last_became_peered":"2023-10-09T10:54:47.943888+0000","last_unstale":"2023-10-09T10:56:19.575303+0000","last_undegraded":"2023-10-09T10:56:19.575303+0000","last_fullsized":"2023-10-09T10:56:19.575303+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:11.046762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":224,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.595004+0000","last_change":"2023-10-09T10:53:14.823051+0000","last_active":"2023-10-09T10:56:19.595004+0000","last_peered":"2023-10-09T10:56:19.595004+0000","last_clean":"2023-10-09T10:56:19.595004+0000","last_became_active":"2023-10-09T10:53:14.822334+0000","last_became_peered":"2023-10-09T10:53:14.822334+0000","last_unstale":"2023-10-09T10:56:19.595004+0000","last_undegraded":"2023-10-09T10:56:19.595004+0000","last_fullsized":"2023-10-09T10:56:19.595004+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:46.573381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"315'8291","reported_seq":8405,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:36.554124+0000","last_change":"2023-10-09T10:54:50.916172+0000","last_active":"2023-10-09T10:56:36.554124+0000","last_peered":"2023-10-09T10:56:36.554124+0000","last_clean":"2023-10-09T10:56:36.554124+0000","last_became_active":"2023-10-09T10:54:50.913596+0000","last_became_peered":"2023-10-09T10:54:50.913596+0000","last_unstale":"2023-10-09T10:56:36.554124+0000","last_undegraded":"2023-10-09T10:56:36.554124+0000","last_fullsized":"2023-10-09T10:56:36.554124+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":289,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":8291,"log_dups_size":0,"ondisk_log_size":8291,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42041344,"num_objects":642,"num_object_clones":0,"num_object_copies":1926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5334,"num_write_kb":191200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1835008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":699,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.968834+0000","last_change":"2023-10-09T10:54:50.913546+0000","last_active":"2023-10-09T10:56:28.968834+0000","last_peered":"2023-10-09T10:56:28.968834+0000","last_clean":"2023-10-09T10:56:28.968834+0000","last_became_active":"2023-10-09T10:54:50.913190+0000","last_became_peered":"2023-10-09T10:54:50.913190+0000","last_unstale":"2023-10-09T10:56:28.968834+0000","last_undegraded":"2023-10-09T10:56:28.968834+0000","last_fullsized":"2023-10-09T10:56:28.968834+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:23.805498+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":369,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.979770+0000","last_change":"2023-10-09T10:54:48.038948+0000","last_active":"2023-10-09T10:56:28.979770+0000","last_peered":"2023-10-09T10:56:28.979770+0000","last_clean":"2023-10-09T10:56:28.979770+0000","last_became_active":"2023-10-09T10:54:48.001556+0000","last_became_peered":"2023-10-09T10:54:48.001556+0000","last_unstale":"2023-10-09T10:56:28.979770+0000","last_undegraded":"2023-10-09T10:56:28.979770+0000","last_fullsized":"2023-10-09T10:56:28.979770+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:23.415040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.577234+0000","last_change":"2023-10-09T10:54:47.914742+0000","last_active":"2023-10-09T10:56:19.577234+0000","last_peered":"2023-10-09T10:56:19.577234+0000","last_clean":"2023-10-09T10:56:19.577234+0000","last_became_active":"2023-10-09T10:54:47.914551+0000","last_became_peered":"2023-10-09T10:54:47.914551+0000","last_unstale":"2023-10-09T10:56:19.577234+0000","last_undegraded":"2023-10-09T10:56:19.577234+0000","last_fullsized":"2023-10-09T10:56:19.577234+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:16.067994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":750,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.983799+0000","last_change":"2023-10-09T10:54:50.915690+0000","last_active":"2023-10-09T10:56:28.983799+0000","last_peered":"2023-10-09T10:56:28.983799+0000","last_clean":"2023-10-09T10:56:28.983799+0000","last_became_active":"2023-10-09T10:54:50.915412+0000","last_became_peered":"2023-10-09T10:54:50.915412+0000","last_unstale":"2023-10-09T10:56:28.983799+0000","last_undegraded":"2023-10-09T10:56:28.983799+0000","last_fullsized":"2023-10-09T10:56:28.983799+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.787782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.787782+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:20:13.339672+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":187,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.593108+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:56:19.593108+0000","last_peered":"2023-10-09T10:56:19.593108+0000","last_clean":"2023-10-09T10:56:19.593108+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:56:19.593108+0000","last_undegraded":"2023-10-09T10:56:19.593108+0000","last_fullsized":"2023-10-09T10:56:19.593108+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":324,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.979844+0000","last_change":"2023-10-09T10:54:48.039275+0000","last_active":"2023-10-09T10:56:28.979844+0000","last_peered":"2023-10-09T10:56:28.979844+0000","last_clean":"2023-10-09T10:56:28.979844+0000","last_became_active":"2023-10-09T10:54:48.017645+0000","last_became_peered":"2023-10-09T10:54:48.017645+0000","last_unstale":"2023-10-09T10:56:28.979844+0000","last_undegraded":"2023-10-09T10:56:28.979844+0000","last_fullsized":"2023-10-09T10:56:28.979844+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:55.986791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":560,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.994959+0000","last_change":"2023-10-09T10:54:21.161539+0000","last_active":"2023-10-09T10:56:28.994959+0000","last_peered":"2023-10-09T10:56:28.994959+0000","last_clean":"2023-10-09T10:56:28.994959+0000","last_became_active":"2023-10-09T10:54:21.161106+0000","last_became_peered":"2023-10-09T10:54:21.161106+0000","last_unstale":"2023-10-09T10:56:28.994959+0000","last_undegraded":"2023-10-09T10:56:28.994959+0000","last_fullsized":"2023-10-09T10:56:28.994959+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:21.824982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:21.824982+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":56,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.981421+0000","last_change":"2023-10-09T10:55:25.632545+0000","last_active":"2023-10-09T10:56:28.981421+0000","last_peered":"2023-10-09T10:56:28.981421+0000","last_clean":"2023-10-09T10:56:28.981421+0000","last_became_active":"2023-10-09T10:55:25.632235+0000","last_became_peered":"2023-10-09T10:55:25.632235+0000","last_unstale":"2023-10-09T10:56:28.981421+0000","last_undegraded":"2023-10-09T10:56:28.981421+0000","last_fullsized":"2023-10-09T10:56:28.981421+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:11.982194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":263,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.978863+0000","last_change":"2023-10-09T10:53:14.838859+0000","last_active":"2023-10-09T10:56:28.978863+0000","last_peered":"2023-10-09T10:56:28.978863+0000","last_clean":"2023-10-09T10:56:28.978863+0000","last_became_active":"2023-10-09T10:53:14.838536+0000","last_became_peered":"2023-10-09T10:53:14.838536+0000","last_unstale":"2023-10-09T10:56:28.978863+0000","last_undegraded":"2023-10-09T10:56:28.978863+0000","last_fullsized":"2023-10-09T10:56:28.978863+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:44:19.980568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":703,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.592809+0000","last_change":"2023-10-09T10:53:32.540768+0000","last_active":"2023-10-09T10:56:19.592809+0000","last_peered":"2023-10-09T10:56:19.592809+0000","last_clean":"2023-10-09T10:56:19.592809+0000","last_became_active":"2023-10-09T10:52:55.414818+0000","last_became_peered":"2023-10-09T10:52:55.414818+0000","last_unstale":"2023-10-09T10:56:19.592809+0000","last_undegraded":"2023-10-09T10:56:19.592809+0000","last_fullsized":"2023-10-09T10:56:19.592809+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:32.540610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:32.540610+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:47.374383+0000","scrub_duration":0.0064251300000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":228,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.983431+0000","last_change":"2023-10-09T10:55:25.637189+0000","last_active":"2023-10-09T10:56:28.983431+0000","last_peered":"2023-10-09T10:56:28.983431+0000","last_clean":"2023-10-09T10:56:28.983431+0000","last_became_active":"2023-10-09T10:55:25.636844+0000","last_became_peered":"2023-10-09T10:55:25.636844+0000","last_unstale":"2023-10-09T10:56:28.983431+0000","last_undegraded":"2023-10-09T10:56:28.983431+0000","last_fullsized":"2023-10-09T10:56:28.983431+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:18.943805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.577191+0000","last_change":"2023-10-09T10:54:47.925654+0000","last_active":"2023-10-09T10:56:19.577191+0000","last_peered":"2023-10-09T10:56:19.577191+0000","last_clean":"2023-10-09T10:56:19.577191+0000","last_became_active":"2023-10-09T10:54:47.924989+0000","last_became_peered":"2023-10-09T10:54:47.924989+0000","last_unstale":"2023-10-09T10:56:19.577191+0000","last_undegraded":"2023-10-09T10:56:19.577191+0000","last_fullsized":"2023-10-09T10:56:19.577191+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:02:49.755988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":688,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.574837+0000","last_change":"2023-10-09T10:54:50.913652+0000","last_active":"2023-10-09T10:56:19.574837+0000","last_peered":"2023-10-09T10:56:19.574837+0000","last_clean":"2023-10-09T10:56:19.574837+0000","last_became_active":"2023-10-09T10:54:50.913330+0000","last_became_peered":"2023-10-09T10:54:50.913330+0000","last_unstale":"2023-10-09T10:56:19.574837+0000","last_undegraded":"2023-10-09T10:56:19.574837+0000","last_fullsized":"2023-10-09T10:56:19.574837+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.200574+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.200574+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001591812,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":69,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.982374+0000","last_change":"2023-10-09T10:53:14.869000+0000","last_active":"2023-10-09T10:56:28.982374+0000","last_peered":"2023-10-09T10:56:28.982374+0000","last_clean":"2023-10-09T10:56:28.982374+0000","last_became_active":"2023-10-09T10:53:14.831904+0000","last_became_peered":"2023-10-09T10:53:14.831904+0000","last_unstale":"2023-10-09T10:56:28.982374+0000","last_undegraded":"2023-10-09T10:56:28.982374+0000","last_fullsized":"2023-10-09T10:56:28.982374+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:38.571980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":320,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.969166+0000","last_change":"2023-10-09T10:54:47.908531+0000","last_active":"2023-10-09T10:56:28.969166+0000","last_peered":"2023-10-09T10:56:28.969166+0000","last_clean":"2023-10-09T10:56:28.969166+0000","last_became_active":"2023-10-09T10:54:47.908232+0000","last_became_peered":"2023-10-09T10:54:47.908232+0000","last_unstale":"2023-10-09T10:56:28.969166+0000","last_undegraded":"2023-10-09T10:56:28.969166+0000","last_fullsized":"2023-10-09T10:56:28.969166+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:20.617543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"316'8484","reported_seq":8554,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T10:56:40.505118+0000","last_change":"2023-10-09T10:54:40.649728+0000","last_active":"2023-10-09T10:56:40.505118+0000","last_peered":"2023-10-09T10:56:40.505118+0000","last_clean":"2023-10-09T10:56:40.505118+0000","last_became_active":"2023-10-09T10:54:40.647392+0000","last_became_peered":"2023-10-09T10:54:40.647392+0000","last_unstale":"2023-10-09T10:56:40.505118+0000","last_undegraded":"2023-10-09T10:56:40.505118+0000","last_fullsized":"2023-10-09T10:56:40.505118+0000","mapping_epoch":289,"log_start":"271'1000","ondisk_log_start":"271'1000","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7484,"log_dups_size":0,"ondisk_log_size":7484,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36175872,"num_objects":552,"num_object_clones":0,"num_object_copies":1656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5494,"num_write_kb":193472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":681,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.596114+0000","last_change":"2023-10-09T10:49:18.708370+0000","last_active":"2023-10-09T10:56:19.596114+0000","last_peered":"2023-10-09T10:56:19.596114+0000","last_clean":"2023-10-09T10:56:19.596114+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:56:19.596114+0000","last_undegraded":"2023-10-09T10:56:19.596114+0000","last_fullsized":"2023-10-09T10:56:19.596114+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:57.808410+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":81,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.981611+0000","last_change":"2023-10-09T10:54:18.080112+0000","last_active":"2023-10-09T10:56:28.981611+0000","last_peered":"2023-10-09T10:56:28.981611+0000","last_clean":"2023-10-09T10:56:28.981611+0000","last_became_active":"2023-10-09T10:54:17.878848+0000","last_became_peered":"2023-10-09T10:54:17.878848+0000","last_unstale":"2023-10-09T10:56:28.981611+0000","last_undegraded":"2023-10-09T10:56:28.981611+0000","last_fullsized":"2023-10-09T10:56:28.981611+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:12.314073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":374,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.981083+0000","last_change":"2023-10-09T10:54:47.941670+0000","last_active":"2023-10-09T10:56:28.981083+0000","last_peered":"2023-10-09T10:56:28.981083+0000","last_clean":"2023-10-09T10:56:28.981083+0000","last_became_active":"2023-10-09T10:54:47.941341+0000","last_became_peered":"2023-10-09T10:54:47.941341+0000","last_unstale":"2023-10-09T10:56:28.981083+0000","last_undegraded":"2023-10-09T10:56:28.981083+0000","last_fullsized":"2023-10-09T10:56:28.981083+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:31.570728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.19","version":"315'8418","reported_seq":8486,"reported_epoch":315,"state":"active+recovery_wait","last_fresh":"2023-10-09T10:56:37.467421+0000","last_change":"2023-10-09T10:54:40.731544+0000","last_active":"2023-10-09T10:56:37.467421+0000","last_peered":"2023-10-09T10:56:37.467421+0000","last_clean":"2023-10-09T10:54:39.518652+0000","last_became_active":"2023-10-09T10:54:40.598732+0000","last_became_peered":"2023-10-09T10:54:40.598732+0000","last_unstale":"2023-10-09T10:56:37.467421+0000","last_undegraded":"2023-10-09T10:56:37.467421+0000","last_fullsized":"2023-10-09T10:56:37.467421+0000","mapping_epoch":289,"log_start":"270'318","ondisk_log_start":"270'318","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":8100,"log_dups_size":0,"ondisk_log_size":8100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40566784,"num_objects":619,"num_object_clones":0,"num_object_copies":1857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5433,"num_write_kb":193664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":619}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":663,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.981482+0000","last_change":"2023-10-09T10:53:17.846464+0000","last_active":"2023-10-09T10:56:28.981482+0000","last_peered":"2023-10-09T10:56:28.981482+0000","last_clean":"2023-10-09T10:56:28.981482+0000","last_became_active":"2023-10-09T10:53:17.845979+0000","last_became_peered":"2023-10-09T10:53:17.845979+0000","last_unstale":"2023-10-09T10:56:28.981482+0000","last_undegraded":"2023-10-09T10:56:28.981482+0000","last_fullsized":"2023-10-09T10:56:28.981482+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:56.295299+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":223,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.592570+0000","last_change":"2023-10-09T10:53:14.821961+0000","last_active":"2023-10-09T10:56:19.592570+0000","last_peered":"2023-10-09T10:56:19.592570+0000","last_clean":"2023-10-09T10:56:19.592570+0000","last_became_active":"2023-10-09T10:53:14.821625+0000","last_became_peered":"2023-10-09T10:53:14.821625+0000","last_unstale":"2023-10-09T10:56:19.592570+0000","last_undegraded":"2023-10-09T10:56:19.592570+0000","last_fullsized":"2023-10-09T10:56:19.592570+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:29.305952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":230,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.982502+0000","last_change":"2023-10-09T10:55:25.629418+0000","last_active":"2023-10-09T10:56:28.982502+0000","last_peered":"2023-10-09T10:56:28.982502+0000","last_clean":"2023-10-09T10:56:28.982502+0000","last_became_active":"2023-10-09T10:55:25.629130+0000","last_became_peered":"2023-10-09T10:55:25.629130+0000","last_unstale":"2023-10-09T10:56:28.982502+0000","last_undegraded":"2023-10-09T10:56:28.982502+0000","last_fullsized":"2023-10-09T10:56:28.982502+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:51.186552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":619,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.577756+0000","last_change":"2023-10-09T10:54:50.916062+0000","last_active":"2023-10-09T10:56:19.577756+0000","last_peered":"2023-10-09T10:56:19.577756+0000","last_clean":"2023-10-09T10:56:19.577756+0000","last_became_active":"2023-10-09T10:54:50.915189+0000","last_became_peered":"2023-10-09T10:54:50.915189+0000","last_unstale":"2023-10-09T10:56:19.577756+0000","last_undegraded":"2023-10-09T10:56:19.577756+0000","last_fullsized":"2023-10-09T10:56:19.577756+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.077788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.077788+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":211,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.982901+0000","last_change":"2023-10-09T10:55:25.633116+0000","last_active":"2023-10-09T10:56:28.982901+0000","last_peered":"2023-10-09T10:56:28.982901+0000","last_clean":"2023-10-09T10:56:28.982901+0000","last_became_active":"2023-10-09T10:55:25.632837+0000","last_became_peered":"2023-10-09T10:55:25.632837+0000","last_unstale":"2023-10-09T10:56:28.982901+0000","last_undegraded":"2023-10-09T10:56:28.982901+0000","last_fullsized":"2023-10-09T10:56:28.982901+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:46.192186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":205,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.594770+0000","last_change":"2023-10-09T10:53:14.823485+0000","last_active":"2023-10-09T10:56:19.594770+0000","last_peered":"2023-10-09T10:56:19.594770+0000","last_clean":"2023-10-09T10:56:19.594770+0000","last_became_active":"2023-10-09T10:53:14.822756+0000","last_became_peered":"2023-10-09T10:53:14.822756+0000","last_unstale":"2023-10-09T10:56:19.594770+0000","last_undegraded":"2023-10-09T10:56:19.594770+0000","last_fullsized":"2023-10-09T10:56:19.594770+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:05.359313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":646,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.577505+0000","last_change":"2023-10-09T10:54:50.910027+0000","last_active":"2023-10-09T10:56:19.577505+0000","last_peered":"2023-10-09T10:56:19.577505+0000","last_clean":"2023-10-09T10:56:19.577505+0000","last_became_active":"2023-10-09T10:54:50.909698+0000","last_became_peered":"2023-10-09T10:54:50.909698+0000","last_unstale":"2023-10-09T10:56:19.577505+0000","last_undegraded":"2023-10-09T10:56:19.577505+0000","last_fullsized":"2023-10-09T10:56:19.577505+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:01.637196+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":190,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.592098+0000","last_change":"2023-10-09T10:51:12.366910+0000","last_active":"2023-10-09T10:56:19.592098+0000","last_peered":"2023-10-09T10:56:19.592098+0000","last_clean":"2023-10-09T10:56:19.592098+0000","last_became_active":"2023-10-09T10:51:12.366565+0000","last_became_peered":"2023-10-09T10:51:12.366565+0000","last_unstale":"2023-10-09T10:56:19.592098+0000","last_undegraded":"2023-10-09T10:56:19.592098+0000","last_fullsized":"2023-10-09T10:56:19.592098+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:53.464251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":351,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.984332+0000","last_change":"2023-10-09T10:54:48.017646+0000","last_active":"2023-10-09T10:56:28.984332+0000","last_peered":"2023-10-09T10:56:28.984332+0000","last_clean":"2023-10-09T10:56:28.984332+0000","last_became_active":"2023-10-09T10:54:47.994162+0000","last_became_peered":"2023-10-09T10:54:47.994162+0000","last_unstale":"2023-10-09T10:56:28.984332+0000","last_undegraded":"2023-10-09T10:56:28.984332+0000","last_fullsized":"2023-10-09T10:56:28.984332+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:11:59.205546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"316'8527","reported_seq":8609,"reported_epoch":316,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:56:41.295426+0000","last_change":"2023-10-09T10:54:40.551123+0000","last_active":"2023-10-09T10:56:41.295426+0000","last_peered":"2023-10-09T10:56:41.295426+0000","last_clean":"2023-10-09T10:54:17.036595+0000","last_became_active":"2023-10-09T10:54:40.548039+0000","last_became_peered":"2023-10-09T10:54:40.548039+0000","last_unstale":"2023-10-09T10:56:41.295426+0000","last_undegraded":"2023-10-09T10:56:41.295426+0000","last_fullsized":"2023-10-09T10:56:41.295426+0000","mapping_epoch":282,"log_start":"271'600","ondisk_log_start":"271'600","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7927,"log_dups_size":0,"ondisk_log_size":7927,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32473088,"num_objects":495,"num_object_clones":0,"num_object_copies":1485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":495,"num_objects_unfound":0,"num_objects_dirty":495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5541,"num_write_kb":193184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":495}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":667,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.983648+0000","last_change":"2023-10-09T10:54:50.909873+0000","last_active":"2023-10-09T10:56:28.983648+0000","last_peered":"2023-10-09T10:56:28.983648+0000","last_clean":"2023-10-09T10:56:28.983648+0000","last_became_active":"2023-10-09T10:54:50.909533+0000","last_became_peered":"2023-10-09T10:54:50.909533+0000","last_unstale":"2023-10-09T10:56:28.983648+0000","last_undegraded":"2023-10-09T10:56:28.983648+0000","last_fullsized":"2023-10-09T10:56:28.983648+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:45.045013+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":63,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.981971+0000","last_change":"2023-10-09T10:53:14.854904+0000","last_active":"2023-10-09T10:56:28.981971+0000","last_peered":"2023-10-09T10:56:28.981971+0000","last_clean":"2023-10-09T10:56:28.981971+0000","last_became_active":"2023-10-09T10:53:14.829611+0000","last_became_peered":"2023-10-09T10:53:14.829611+0000","last_unstale":"2023-10-09T10:56:28.981971+0000","last_undegraded":"2023-10-09T10:56:28.981971+0000","last_fullsized":"2023-10-09T10:56:28.981971+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:48:26.064630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":248,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.981273+0000","last_change":"2023-10-09T10:55:25.633485+0000","last_active":"2023-10-09T10:56:28.981273+0000","last_peered":"2023-10-09T10:56:28.981273+0000","last_clean":"2023-10-09T10:56:28.981273+0000","last_became_active":"2023-10-09T10:55:25.633028+0000","last_became_peered":"2023-10-09T10:55:25.633028+0000","last_unstale":"2023-10-09T10:56:28.981273+0000","last_undegraded":"2023-10-09T10:56:28.981273+0000","last_fullsized":"2023-10-09T10:56:28.981273+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:57.601747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"316'8448","reported_seq":8518,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T10:56:41.127406+0000","last_change":"2023-10-09T10:55:26.330497+0000","last_active":"2023-10-09T10:56:41.127406+0000","last_peered":"2023-10-09T10:56:41.127406+0000","last_clean":"2023-10-09T10:56:41.127406+0000","last_became_active":"2023-10-09T10:54:40.554360+0000","last_became_peered":"2023-10-09T10:54:40.554360+0000","last_unstale":"2023-10-09T10:56:41.127406+0000","last_undegraded":"2023-10-09T10:56:41.127406+0000","last_fullsized":"2023-10-09T10:56:41.127406+0000","mapping_epoch":266,"log_start":"271'400","ondisk_log_start":"271'400","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":8048,"log_dups_size":0,"ondisk_log_size":8048,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32702464,"num_objects":499,"num_object_clones":0,"num_object_copies":1497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5487,"num_write_kb":191552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":676,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.980913+0000","last_change":"2023-10-09T10:53:25.540494+0000","last_active":"2023-10-09T10:56:28.980913+0000","last_peered":"2023-10-09T10:56:28.980913+0000","last_clean":"2023-10-09T10:56:28.980913+0000","last_became_active":"2023-10-09T10:51:01.295474+0000","last_became_peered":"2023-10-09T10:51:01.295474+0000","last_unstale":"2023-10-09T10:56:28.980913+0000","last_undegraded":"2023-10-09T10:56:28.980913+0000","last_fullsized":"2023-10-09T10:56:28.980913+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:39.069143+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":91,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.981554+0000","last_change":"2023-10-09T10:54:48.000482+0000","last_active":"2023-10-09T10:56:28.981554+0000","last_peered":"2023-10-09T10:56:28.981554+0000","last_clean":"2023-10-09T10:56:28.981554+0000","last_became_active":"2023-10-09T10:54:47.905689+0000","last_became_peered":"2023-10-09T10:54:47.905689+0000","last_unstale":"2023-10-09T10:56:28.981554+0000","last_undegraded":"2023-10-09T10:56:28.981554+0000","last_fullsized":"2023-10-09T10:56:28.981554+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.666594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.575159+0000","last_change":"2023-10-09T10:54:47.944930+0000","last_active":"2023-10-09T10:56:19.575159+0000","last_peered":"2023-10-09T10:56:19.575159+0000","last_clean":"2023-10-09T10:56:19.575159+0000","last_became_active":"2023-10-09T10:54:47.944485+0000","last_became_peered":"2023-10-09T10:54:47.944485+0000","last_unstale":"2023-10-09T10:56:19.575159+0000","last_undegraded":"2023-10-09T10:56:19.575159+0000","last_fullsized":"2023-10-09T10:56:19.575159+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:11.213129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"315'8408","reported_seq":8478,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:37.473377+0000","last_change":"2023-10-09T10:54:40.586495+0000","last_active":"2023-10-09T10:56:37.473377+0000","last_peered":"2023-10-09T10:56:37.473377+0000","last_clean":"2023-10-09T10:56:37.473377+0000","last_became_active":"2023-10-09T10:54:40.577656+0000","last_became_peered":"2023-10-09T10:54:40.577656+0000","last_unstale":"2023-10-09T10:56:37.473377+0000","last_undegraded":"2023-10-09T10:56:37.473377+0000","last_fullsized":"2023-10-09T10:56:37.473377+0000","mapping_epoch":266,"log_start":"270'300","ondisk_log_start":"270'300","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":8108,"log_dups_size":0,"ondisk_log_size":8108,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42565632,"num_objects":649,"num_object_clones":0,"num_object_copies":1947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5413,"num_write_kb":194016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":628,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.969578+0000","last_change":"2023-10-09T10:53:36.680150+0000","last_active":"2023-10-09T10:56:28.969578+0000","last_peered":"2023-10-09T10:56:28.969578+0000","last_clean":"2023-10-09T10:56:28.969578+0000","last_became_active":"2023-10-09T10:51:50.371032+0000","last_became_peered":"2023-10-09T10:51:50.371032+0000","last_unstale":"2023-10-09T10:56:28.969578+0000","last_undegraded":"2023-10-09T10:56:28.969578+0000","last_fullsized":"2023-10-09T10:56:28.969578+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.680090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.680090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:08:28.952827+0000","scrub_duration":0.005577172,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":63,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.983735+0000","last_change":"2023-10-09T10:53:14.841343+0000","last_active":"2023-10-09T10:56:28.983735+0000","last_peered":"2023-10-09T10:56:28.983735+0000","last_clean":"2023-10-09T10:56:28.983735+0000","last_became_active":"2023-10-09T10:53:14.828213+0000","last_became_peered":"2023-10-09T10:53:14.828213+0000","last_unstale":"2023-10-09T10:56:28.983735+0000","last_undegraded":"2023-10-09T10:56:28.983735+0000","last_fullsized":"2023-10-09T10:56:28.983735+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:28.137803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.577422+0000","last_change":"2023-10-09T10:54:47.942716+0000","last_active":"2023-10-09T10:56:19.577422+0000","last_peered":"2023-10-09T10:56:19.577422+0000","last_clean":"2023-10-09T10:56:19.577422+0000","last_became_active":"2023-10-09T10:54:47.942459+0000","last_became_peered":"2023-10-09T10:54:47.942459+0000","last_unstale":"2023-10-09T10:56:19.577422+0000","last_undegraded":"2023-10-09T10:56:19.577422+0000","last_fullsized":"2023-10-09T10:56:19.577422+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:07.790165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"316'8505","reported_seq":8602,"reported_epoch":316,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:40.452368+0000","last_change":"2023-10-09T10:54:49.183581+0000","last_active":"2023-10-09T10:56:40.452368+0000","last_peered":"2023-10-09T10:56:40.452368+0000","last_clean":"2023-10-09T10:54:02.391425+0000","last_became_active":"2023-10-09T10:54:49.111868+0000","last_became_peered":"2023-10-09T10:54:49.111868+0000","last_unstale":"2023-10-09T10:56:40.452368+0000","last_undegraded":"2023-10-09T10:54:48.912262+0000","last_fullsized":"2023-10-09T10:54:48.907341+0000","mapping_epoch":295,"log_start":"271'1037","ondisk_log_start":"271'1037","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7468,"log_dups_size":0,"ondisk_log_size":7468,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32800768,"num_objects":500,"num_object_clones":0,"num_object_copies":1500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":697,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5528,"num_write_kb":192928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"3,4,6","objects":531},{"shards":"3,6","objects":-197}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":761,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.969459+0000","last_change":"2023-10-09T10:53:38.681891+0000","last_active":"2023-10-09T10:56:28.969459+0000","last_peered":"2023-10-09T10:56:28.969459+0000","last_clean":"2023-10-09T10:56:28.969459+0000","last_became_active":"2023-10-09T10:53:17.840478+0000","last_became_peered":"2023-10-09T10:53:17.840478+0000","last_unstale":"2023-10-09T10:56:28.969459+0000","last_undegraded":"2023-10-09T10:56:28.969459+0000","last_fullsized":"2023-10-09T10:56:28.969459+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:31:14.758036+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":142,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.981451+0000","last_change":"2023-10-09T10:51:50.374527+0000","last_active":"2023-10-09T10:56:28.981451+0000","last_peered":"2023-10-09T10:56:28.981451+0000","last_clean":"2023-10-09T10:56:28.981451+0000","last_became_active":"2023-10-09T10:51:50.374297+0000","last_became_peered":"2023-10-09T10:51:50.374297+0000","last_unstale":"2023-10-09T10:56:28.981451+0000","last_undegraded":"2023-10-09T10:56:28.981451+0000","last_fullsized":"2023-10-09T10:56:28.981451+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:31:49.358307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":223,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.593159+0000","last_change":"2023-10-09T10:54:47.939771+0000","last_active":"2023-10-09T10:56:19.593159+0000","last_peered":"2023-10-09T10:56:19.593159+0000","last_clean":"2023-10-09T10:56:19.593159+0000","last_became_active":"2023-10-09T10:54:47.939440+0000","last_became_peered":"2023-10-09T10:54:47.939440+0000","last_unstale":"2023-10-09T10:56:19.593159+0000","last_undegraded":"2023-10-09T10:56:19.593159+0000","last_fullsized":"2023-10-09T10:56:19.593159+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:58.335536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"316'8564","reported_seq":10514,"reported_epoch":316,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:40.480084+0000","last_change":"2023-10-09T10:54:49.158909+0000","last_active":"2023-10-09T10:56:40.480084+0000","last_peered":"2023-10-09T10:56:40.480084+0000","last_clean":"2023-10-09T10:54:02.435878+0000","last_became_active":"2023-10-09T10:54:48.018286+0000","last_became_peered":"2023-10-09T10:54:48.018286+0000","last_unstale":"2023-10-09T10:56:40.480084+0000","last_undegraded":"2023-10-09T10:54:49.152454+0000","last_fullsized":"2023-10-09T10:54:47.891421+0000","mapping_epoch":294,"log_start":"271'1100","ondisk_log_start":"271'1100","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7464,"log_dups_size":0,"ondisk_log_size":7464,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36077568,"num_objects":550,"num_object_clones":0,"num_object_copies":1650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":407,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5546,"num_write_kb":195104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":592,"num_bytes_recovered":38830080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":407},{"shards":"3,5","objects":143}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":653,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.983022+0000","last_change":"2023-10-09T10:55:25.628214+0000","last_active":"2023-10-09T10:56:28.983022+0000","last_peered":"2023-10-09T10:56:28.983022+0000","last_clean":"2023-10-09T10:56:28.983022+0000","last_became_active":"2023-10-09T10:55:25.627822+0000","last_became_peered":"2023-10-09T10:55:25.627822+0000","last_unstale":"2023-10-09T10:56:28.983022+0000","last_undegraded":"2023-10-09T10:56:28.983022+0000","last_fullsized":"2023-10-09T10:56:28.983022+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":312,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.979178+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:56:28.979178+0000","last_peered":"2023-10-09T10:56:28.979178+0000","last_clean":"2023-10-09T10:56:28.979178+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:56:28.979178+0000","last_undegraded":"2023-10-09T10:56:28.979178+0000","last_fullsized":"2023-10-09T10:56:28.979178+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":157,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.981280+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T10:56:28.981280+0000","last_peered":"2023-10-09T10:56:28.981280+0000","last_clean":"2023-10-09T10:56:28.981280+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T10:56:28.981280+0000","last_undegraded":"2023-10-09T10:56:28.981280+0000","last_fullsized":"2023-10-09T10:56:28.981280+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"315'8368","reported_seq":13382,"reported_epoch":315,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:37.524081+0000","last_change":"2023-10-09T10:54:49.220251+0000","last_active":"2023-10-09T10:56:37.524081+0000","last_peered":"2023-10-09T10:56:37.524081+0000","last_clean":"2023-10-09T10:54:02.426241+0000","last_became_active":"2023-10-09T10:54:49.172055+0000","last_became_peered":"2023-10-09T10:54:49.172055+0000","last_unstale":"2023-10-09T10:56:37.524081+0000","last_undegraded":"2023-10-09T10:54:49.015836+0000","last_fullsized":"2023-10-09T10:54:48.896714+0000","mapping_epoch":295,"log_start":"270'290","ondisk_log_start":"270'290","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":8078,"log_dups_size":0,"ondisk_log_size":8078,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38338560,"num_objects":585,"num_object_clones":0,"num_object_copies":1755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":757,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5407,"num_write_kb":191744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2411,"num_bytes_recovered":158007296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[0,2],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,2","objects":10},{"shards":"0,2,5","objects":393},{"shards":"0,3,5","objects":182}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":620,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:35.895287+0000","last_change":"2023-10-09T10:54:21.173078+0000","last_active":"2023-10-09T10:56:35.895287+0000","last_peered":"2023-10-09T10:56:35.895287+0000","last_clean":"2023-10-09T10:56:35.895287+0000","last_became_active":"2023-10-09T10:54:21.172767+0000","last_became_peered":"2023-10-09T10:54:21.172767+0000","last_unstale":"2023-10-09T10:56:35.895287+0000","last_undegraded":"2023-10-09T10:56:35.895287+0000","last_fullsized":"2023-10-09T10:56:35.895287+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.532890+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.532890+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:32.335841+0000","scrub_duration":0.0045290130000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":295,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.969061+0000","last_change":"2023-10-09T10:54:47.908506+0000","last_active":"2023-10-09T10:56:28.969061+0000","last_peered":"2023-10-09T10:56:28.969061+0000","last_clean":"2023-10-09T10:56:28.969061+0000","last_became_active":"2023-10-09T10:54:47.908110+0000","last_became_peered":"2023-10-09T10:54:47.908110+0000","last_unstale":"2023-10-09T10:56:28.969061+0000","last_undegraded":"2023-10-09T10:56:28.969061+0000","last_fullsized":"2023-10-09T10:56:28.969061+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:38.808943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":55,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:35.895387+0000","last_change":"2023-10-09T10:54:47.948732+0000","last_active":"2023-10-09T10:56:35.895387+0000","last_peered":"2023-10-09T10:56:35.895387+0000","last_clean":"2023-10-09T10:56:35.895387+0000","last_became_active":"2023-10-09T10:54:47.948414+0000","last_became_peered":"2023-10-09T10:54:47.948414+0000","last_unstale":"2023-10-09T10:56:35.895387+0000","last_undegraded":"2023-10-09T10:56:35.895387+0000","last_fullsized":"2023-10-09T10:56:35.895387+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:38.020807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.2","version":"316'8623","reported_seq":8694,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T10:56:41.267297+0000","last_change":"2023-10-09T10:55:24.369167+0000","last_active":"2023-10-09T10:56:41.267297+0000","last_peered":"2023-10-09T10:56:41.267297+0000","last_clean":"2023-10-09T10:56:41.267297+0000","last_became_active":"2023-10-09T10:54:40.567891+0000","last_became_peered":"2023-10-09T10:54:40.567891+0000","last_unstale":"2023-10-09T10:56:41.267297+0000","last_undegraded":"2023-10-09T10:56:41.267297+0000","last_fullsized":"2023-10-09T10:56:41.267297+0000","mapping_epoch":265,"log_start":"271'700","ondisk_log_start":"271'700","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7923,"log_dups_size":0,"ondisk_log_size":7923,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34340864,"num_objects":524,"num_object_clones":0,"num_object_copies":1572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5588,"num_write_kb":195584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":689,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.578097+0000","last_change":"2023-10-09T10:54:50.914888+0000","last_active":"2023-10-09T10:56:19.578097+0000","last_peered":"2023-10-09T10:56:19.578097+0000","last_clean":"2023-10-09T10:56:19.578097+0000","last_became_active":"2023-10-09T10:54:50.909639+0000","last_became_peered":"2023-10-09T10:54:50.909639+0000","last_unstale":"2023-10-09T10:56:19.578097+0000","last_undegraded":"2023-10-09T10:56:19.578097+0000","last_fullsized":"2023-10-09T10:56:19.578097+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.103750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.103750+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":351,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.968865+0000","last_change":"2023-10-09T10:54:17.876795+0000","last_active":"2023-10-09T10:56:28.968865+0000","last_peered":"2023-10-09T10:56:28.968865+0000","last_clean":"2023-10-09T10:56:28.968865+0000","last_became_active":"2023-10-09T10:54:17.876504+0000","last_became_peered":"2023-10-09T10:54:17.876504+0000","last_unstale":"2023-10-09T10:56:28.968865+0000","last_undegraded":"2023-10-09T10:56:28.968865+0000","last_fullsized":"2023-10-09T10:56:28.968865+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:04:48.657295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.574908+0000","last_change":"2023-10-09T10:54:47.913826+0000","last_active":"2023-10-09T10:56:19.574908+0000","last_peered":"2023-10-09T10:56:19.574908+0000","last_clean":"2023-10-09T10:56:19.574908+0000","last_became_active":"2023-10-09T10:54:47.913524+0000","last_became_peered":"2023-10-09T10:54:47.913524+0000","last_unstale":"2023-10-09T10:56:19.574908+0000","last_undegraded":"2023-10-09T10:56:19.574908+0000","last_fullsized":"2023-10-09T10:56:19.574908+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:56:23.699552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"315'8367","reported_seq":8490,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:36.686838+0000","last_change":"2023-10-09T10:54:50.915546+0000","last_active":"2023-10-09T10:56:36.686838+0000","last_peered":"2023-10-09T10:56:36.686838+0000","last_clean":"2023-10-09T10:56:36.686838+0000","last_became_active":"2023-10-09T10:54:50.910131+0000","last_became_peered":"2023-10-09T10:54:50.910131+0000","last_unstale":"2023-10-09T10:56:36.686838+0000","last_undegraded":"2023-10-09T10:56:36.686838+0000","last_fullsized":"2023-10-09T10:56:36.686838+0000","mapping_epoch":297,"log_start":"270'158","ondisk_log_start":"270'158","created":265,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":8209,"log_dups_size":0,"ondisk_log_size":8209,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37453824,"num_objects":571,"num_object_clones":0,"num_object_copies":1713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5409,"num_write_kb":191392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":2031616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":631,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:35.895334+0000","last_change":"2023-10-09T10:54:21.152148+0000","last_active":"2023-10-09T10:56:35.895334+0000","last_peered":"2023-10-09T10:56:35.895334+0000","last_clean":"2023-10-09T10:56:35.895334+0000","last_became_active":"2023-10-09T10:54:21.151809+0000","last_became_peered":"2023-10-09T10:54:21.151809+0000","last_unstale":"2023-10-09T10:56:35.895334+0000","last_undegraded":"2023-10-09T10:56:35.895334+0000","last_fullsized":"2023-10-09T10:56:35.895334+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.701128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.701128+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:27.021145+0000","scrub_duration":0.0025071830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":344,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:35.501738+0000","last_change":"2023-10-09T10:54:48.038946+0000","last_active":"2023-10-09T10:56:35.501738+0000","last_peered":"2023-10-09T10:56:35.501738+0000","last_clean":"2023-10-09T10:56:35.501738+0000","last_became_active":"2023-10-09T10:54:48.001493+0000","last_became_peered":"2023-10-09T10:54:48.001493+0000","last_unstale":"2023-10-09T10:56:35.501738+0000","last_undegraded":"2023-10-09T10:56:35.501738+0000","last_fullsized":"2023-10-09T10:56:35.501738+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:25.197466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.577374+0000","last_change":"2023-10-09T10:54:48.012411+0000","last_active":"2023-10-09T10:56:19.577374+0000","last_peered":"2023-10-09T10:56:19.577374+0000","last_clean":"2023-10-09T10:56:19.577374+0000","last_became_active":"2023-10-09T10:54:48.011894+0000","last_became_peered":"2023-10-09T10:54:48.011894+0000","last_unstale":"2023-10-09T10:56:19.577374+0000","last_undegraded":"2023-10-09T10:56:19.577374+0000","last_fullsized":"2023-10-09T10:56:19.577374+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:28.067337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"315'10966","reported_seq":15181,"reported_epoch":315,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:37.538838+0000","last_change":"2023-10-09T10:55:24.700754+0000","last_active":"2023-10-09T10:56:37.538838+0000","last_peered":"2023-10-09T10:56:37.538838+0000","last_clean":"2023-10-09T10:54:17.076172+0000","last_became_active":"2023-10-09T10:54:19.327879+0000","last_became_peered":"2023-10-09T10:54:19.327879+0000","last_unstale":"2023-10-09T10:56:37.538838+0000","last_undegraded":"2023-10-09T10:54:18.890724+0000","last_fullsized":"2023-10-09T10:54:18.868702+0000","mapping_epoch":282,"log_start":"286'5652","ondisk_log_start":"286'5652","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5314,"log_dups_size":0,"ondisk_log_size":5314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:49.273305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79036416,"num_objects":1206,"num_object_clones":0,"num_object_copies":3618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2808,"num_objects_misplaced":411,"num_objects_unfound":0,"num_objects_dirty":1206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10966,"num_write_kb":389504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2073,"num_bytes_recovered":135856128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-2013},{"shards":"0,4,7","objects":411}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":713,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.984159+0000","last_change":"2023-10-09T10:53:34.602531+0000","last_active":"2023-10-09T10:56:28.984159+0000","last_peered":"2023-10-09T10:56:28.984159+0000","last_clean":"2023-10-09T10:56:28.984159+0000","last_became_active":"2023-10-09T10:53:17.841970+0000","last_became_peered":"2023-10-09T10:53:17.841970+0000","last_unstale":"2023-10-09T10:56:28.984159+0000","last_undegraded":"2023-10-09T10:56:28.984159+0000","last_fullsized":"2023-10-09T10:56:28.984159+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:04.601861+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.574495+0000","last_change":"2023-10-09T10:54:47.952216+0000","last_active":"2023-10-09T10:56:19.574495+0000","last_peered":"2023-10-09T10:56:19.574495+0000","last_clean":"2023-10-09T10:56:19.574495+0000","last_became_active":"2023-10-09T10:54:47.941070+0000","last_became_peered":"2023-10-09T10:54:47.941070+0000","last_unstale":"2023-10-09T10:56:19.574495+0000","last_undegraded":"2023-10-09T10:56:19.574495+0000","last_fullsized":"2023-10-09T10:56:19.574495+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:09.075266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":236,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.980806+0000","last_change":"2023-10-09T10:54:17.886921+0000","last_active":"2023-10-09T10:56:28.980806+0000","last_peered":"2023-10-09T10:56:28.980806+0000","last_clean":"2023-10-09T10:56:28.980806+0000","last_became_active":"2023-10-09T10:54:17.886584+0000","last_became_peered":"2023-10-09T10:54:17.886584+0000","last_unstale":"2023-10-09T10:56:28.980806+0000","last_undegraded":"2023-10-09T10:56:28.980806+0000","last_fullsized":"2023-10-09T10:56:28.980806+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:15.848951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.577268+0000","last_change":"2023-10-09T10:54:47.908294+0000","last_active":"2023-10-09T10:56:19.577268+0000","last_peered":"2023-10-09T10:56:19.577268+0000","last_clean":"2023-10-09T10:56:19.577268+0000","last_became_active":"2023-10-09T10:54:47.908019+0000","last_became_peered":"2023-10-09T10:54:47.908019+0000","last_unstale":"2023-10-09T10:56:19.577268+0000","last_undegraded":"2023-10-09T10:56:19.577268+0000","last_fullsized":"2023-10-09T10:56:19.577268+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:48.662764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"316'8491","reported_seq":8562,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T10:56:40.458569+0000","last_change":"2023-10-09T10:54:40.642047+0000","last_active":"2023-10-09T10:56:40.458569+0000","last_peered":"2023-10-09T10:56:40.458569+0000","last_clean":"2023-10-09T10:56:40.458569+0000","last_became_active":"2023-10-09T10:54:40.639305+0000","last_became_peered":"2023-10-09T10:54:40.639305+0000","last_unstale":"2023-10-09T10:56:40.458569+0000","last_undegraded":"2023-10-09T10:56:40.458569+0000","last_fullsized":"2023-10-09T10:56:40.458569+0000","mapping_epoch":266,"log_start":"271'1000","ondisk_log_start":"271'1000","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7491,"log_dups_size":0,"ondisk_log_size":7491,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35979264,"num_objects":549,"num_object_clones":0,"num_object_copies":1647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5501,"num_write_kb":193600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":652,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.596021+0000","last_change":"2023-10-09T10:54:21.163072+0000","last_active":"2023-10-09T10:56:19.596021+0000","last_peered":"2023-10-09T10:56:19.596021+0000","last_clean":"2023-10-09T10:56:19.596021+0000","last_became_active":"2023-10-09T10:54:21.162710+0000","last_became_peered":"2023-10-09T10:54:21.162710+0000","last_unstale":"2023-10-09T10:56:19.596021+0000","last_undegraded":"2023-10-09T10:56:19.596021+0000","last_fullsized":"2023-10-09T10:56:19.596021+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:15.932806+0000","scrub_duration":0.0019146790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.577341+0000","last_change":"2023-10-09T10:54:47.899967+0000","last_active":"2023-10-09T10:56:19.577341+0000","last_peered":"2023-10-09T10:56:19.577341+0000","last_clean":"2023-10-09T10:56:19.577341+0000","last_became_active":"2023-10-09T10:54:47.899584+0000","last_became_peered":"2023-10-09T10:54:47.899584+0000","last_unstale":"2023-10-09T10:56:19.577341+0000","last_undegraded":"2023-10-09T10:56:19.577341+0000","last_fullsized":"2023-10-09T10:56:19.577341+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:43.973868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":224,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.593161+0000","last_change":"2023-10-09T10:50:43.388615+0000","last_active":"2023-10-09T10:56:19.593161+0000","last_peered":"2023-10-09T10:56:19.593161+0000","last_clean":"2023-10-09T10:56:19.593161+0000","last_became_active":"2023-10-09T10:50:43.388176+0000","last_became_peered":"2023-10-09T10:50:43.388176+0000","last_unstale":"2023-10-09T10:56:19.593161+0000","last_undegraded":"2023-10-09T10:56:19.593161+0000","last_fullsized":"2023-10-09T10:56:19.593161+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:41:17.550111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.574408+0000","last_change":"2023-10-09T10:54:47.950462+0000","last_active":"2023-10-09T10:56:19.574408+0000","last_peered":"2023-10-09T10:56:19.574408+0000","last_clean":"2023-10-09T10:56:19.574408+0000","last_became_active":"2023-10-09T10:54:47.940606+0000","last_became_peered":"2023-10-09T10:54:47.940606+0000","last_unstale":"2023-10-09T10:56:19.574408+0000","last_undegraded":"2023-10-09T10:56:19.574408+0000","last_fullsized":"2023-10-09T10:56:19.574408+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:50.557850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"316'11083","reported_seq":11171,"reported_epoch":316,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:40.477407+0000","last_change":"2023-10-09T10:54:49.182381+0000","last_active":"2023-10-09T10:56:40.477407+0000","last_peered":"2023-10-09T10:56:40.477407+0000","last_clean":"2023-10-09T10:54:02.429830+0000","last_became_active":"2023-10-09T10:54:49.176829+0000","last_became_peered":"2023-10-09T10:54:49.176829+0000","last_unstale":"2023-10-09T10:56:40.477407+0000","last_undegraded":"2023-10-09T10:54:48.956857+0000","last_fullsized":"2023-10-09T10:54:48.892711+0000","mapping_epoch":295,"log_start":"293'6055","ondisk_log_start":"293'6055","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5028,"log_dups_size":0,"ondisk_log_size":5028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:04.678256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68616192,"num_objects":1047,"num_object_clones":0,"num_object_copies":3141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1366,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11083,"num_write_kb":388160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"3,4","objects":-319},{"shards":"3,4,7","objects":1046}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":663,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.981210+0000","last_change":"2023-10-09T10:55:25.634848+0000","last_active":"2023-10-09T10:56:28.981210+0000","last_peered":"2023-10-09T10:56:28.981210+0000","last_clean":"2023-10-09T10:56:28.981210+0000","last_became_active":"2023-10-09T10:55:25.634509+0000","last_became_peered":"2023-10-09T10:55:25.634509+0000","last_unstale":"2023-10-09T10:56:28.981210+0000","last_undegraded":"2023-10-09T10:56:28.981210+0000","last_fullsized":"2023-10-09T10:56:28.981210+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":46,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.993985+0000","last_change":"2023-10-09T10:54:17.885030+0000","last_active":"2023-10-09T10:56:28.993985+0000","last_peered":"2023-10-09T10:56:28.993985+0000","last_clean":"2023-10-09T10:56:28.993985+0000","last_became_active":"2023-10-09T10:54:17.884752+0000","last_became_peered":"2023-10-09T10:54:17.884752+0000","last_unstale":"2023-10-09T10:56:28.993985+0000","last_undegraded":"2023-10-09T10:56:28.993985+0000","last_fullsized":"2023-10-09T10:56:28.993985+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:38.548132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.576680+0000","last_change":"2023-10-09T10:54:47.927822+0000","last_active":"2023-10-09T10:56:19.576680+0000","last_peered":"2023-10-09T10:56:19.576680+0000","last_clean":"2023-10-09T10:56:19.576680+0000","last_became_active":"2023-10-09T10:54:47.927455+0000","last_became_peered":"2023-10-09T10:54:47.927455+0000","last_unstale":"2023-10-09T10:56:19.576680+0000","last_undegraded":"2023-10-09T10:56:19.576680+0000","last_fullsized":"2023-10-09T10:56:19.576680+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:11.818203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.577307+0000","last_change":"2023-10-09T10:54:48.012629+0000","last_active":"2023-10-09T10:56:19.577307+0000","last_peered":"2023-10-09T10:56:19.577307+0000","last_clean":"2023-10-09T10:56:19.577307+0000","last_became_active":"2023-10-09T10:54:48.012148+0000","last_became_peered":"2023-10-09T10:54:48.012148+0000","last_unstale":"2023-10-09T10:56:19.577307+0000","last_undegraded":"2023-10-09T10:56:19.577307+0000","last_fullsized":"2023-10-09T10:56:19.577307+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:22.894111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"315'8439","reported_seq":8508,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:37.494193+0000","last_change":"2023-10-09T10:54:40.559886+0000","last_active":"2023-10-09T10:56:37.494193+0000","last_peered":"2023-10-09T10:56:37.494193+0000","last_clean":"2023-10-09T10:56:37.494193+0000","last_became_active":"2023-10-09T10:54:40.557153+0000","last_became_peered":"2023-10-09T10:54:40.557153+0000","last_unstale":"2023-10-09T10:56:37.494193+0000","last_undegraded":"2023-10-09T10:56:37.494193+0000","last_fullsized":"2023-10-09T10:56:37.494193+0000","mapping_epoch":265,"log_start":"271'403","ondisk_log_start":"271'403","created":265,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":8036,"log_dups_size":0,"ondisk_log_size":8036,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38404096,"num_objects":586,"num_object_clones":0,"num_object_copies":1758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5454,"num_write_kb":193280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":46,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.992752+0000","last_change":"2023-10-09T10:54:17.884876+0000","last_active":"2023-10-09T10:56:28.992752+0000","last_peered":"2023-10-09T10:56:28.992752+0000","last_clean":"2023-10-09T10:56:28.992752+0000","last_became_active":"2023-10-09T10:54:17.884402+0000","last_became_peered":"2023-10-09T10:54:17.884402+0000","last_unstale":"2023-10-09T10:56:28.992752+0000","last_undegraded":"2023-10-09T10:56:28.992752+0000","last_fullsized":"2023-10-09T10:56:28.992752+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:18.679950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":692,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.981378+0000","last_change":"2023-10-09T10:55:25.633525+0000","last_active":"2023-10-09T10:56:28.981378+0000","last_peered":"2023-10-09T10:56:28.981378+0000","last_clean":"2023-10-09T10:56:28.981378+0000","last_became_active":"2023-10-09T10:55:25.633079+0000","last_became_peered":"2023-10-09T10:55:25.633079+0000","last_unstale":"2023-10-09T10:56:28.981378+0000","last_undegraded":"2023-10-09T10:56:28.981378+0000","last_fullsized":"2023-10-09T10:56:28.981378+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:37.378259+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":248,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.978504+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:56:28.978504+0000","last_peered":"2023-10-09T10:56:28.978504+0000","last_clean":"2023-10-09T10:56:28.978504+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:56:28.978504+0000","last_undegraded":"2023-10-09T10:56:28.978504+0000","last_fullsized":"2023-10-09T10:56:28.978504+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":228,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.969032+0000","last_change":"2023-10-09T10:53:14.836523+0000","last_active":"2023-10-09T10:56:28.969032+0000","last_peered":"2023-10-09T10:56:28.969032+0000","last_clean":"2023-10-09T10:56:28.969032+0000","last_became_active":"2023-10-09T10:53:14.836134+0000","last_became_peered":"2023-10-09T10:53:14.836134+0000","last_unstale":"2023-10-09T10:56:28.969032+0000","last_undegraded":"2023-10-09T10:56:28.969032+0000","last_fullsized":"2023-10-09T10:56:28.969032+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:05:24.615198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"316'10906","reported_seq":11723,"reported_epoch":316,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:40.473362+0000","last_change":"2023-10-09T10:54:49.078027+0000","last_active":"2023-10-09T10:56:40.473362+0000","last_peered":"2023-10-09T10:56:40.473362+0000","last_clean":"2023-10-09T10:54:02.408036+0000","last_became_active":"2023-10-09T10:54:49.069949+0000","last_became_peered":"2023-10-09T10:54:49.069949+0000","last_unstale":"2023-10-09T10:56:40.473362+0000","last_undegraded":"2023-10-09T10:54:49.050557+0000","last_fullsized":"2023-10-09T10:54:48.898155+0000","mapping_epoch":295,"log_start":"288'5872","ondisk_log_start":"288'5872","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5034,"log_dups_size":0,"ondisk_log_size":5034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:38.092075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68943872,"num_objects":1052,"num_object_clones":0,"num_object_copies":3156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1052,"num_objects_misplaced":1353,"num_objects_unfound":0,"num_objects_dirty":1052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10906,"num_write_kb":382656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":366,"num_bytes_recovered":23986176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,4,5","objects":1051},{"shards":"3,5","objects":-301}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":662,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.577851+0000","last_change":"2023-10-09T10:54:50.915855+0000","last_active":"2023-10-09T10:56:19.577851+0000","last_peered":"2023-10-09T10:56:19.577851+0000","last_clean":"2023-10-09T10:56:19.577851+0000","last_became_active":"2023-10-09T10:54:50.914536+0000","last_became_peered":"2023-10-09T10:54:50.914536+0000","last_unstale":"2023-10-09T10:56:19.577851+0000","last_undegraded":"2023-10-09T10:56:19.577851+0000","last_fullsized":"2023-10-09T10:56:19.577851+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.070026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.070026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":266,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.981351+0000","last_change":"2023-10-09T10:55:25.633308+0000","last_active":"2023-10-09T10:56:28.981351+0000","last_peered":"2023-10-09T10:56:28.981351+0000","last_clean":"2023-10-09T10:56:28.981351+0000","last_became_active":"2023-10-09T10:55:25.632646+0000","last_became_peered":"2023-10-09T10:55:25.632646+0000","last_unstale":"2023-10-09T10:56:28.981351+0000","last_undegraded":"2023-10-09T10:56:28.981351+0000","last_fullsized":"2023-10-09T10:56:28.981351+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:17:13.735888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":197,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:27.494375+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:56:27.494375+0000","last_peered":"2023-10-09T10:56:27.494375+0000","last_clean":"2023-10-09T10:56:27.494375+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:56:27.494375+0000","last_undegraded":"2023-10-09T10:56:27.494375+0000","last_fullsized":"2023-10-09T10:56:27.494375+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":259,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.594843+0000","last_change":"2023-10-09T10:54:47.939756+0000","last_active":"2023-10-09T10:56:19.594843+0000","last_peered":"2023-10-09T10:56:19.594843+0000","last_clean":"2023-10-09T10:56:19.594843+0000","last_became_active":"2023-10-09T10:54:47.938928+0000","last_became_peered":"2023-10-09T10:54:47.938928+0000","last_unstale":"2023-10-09T10:56:19.594843+0000","last_undegraded":"2023-10-09T10:56:19.594843+0000","last_fullsized":"2023-10-09T10:56:19.594843+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:05.310929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"316'11134","reported_seq":11224,"reported_epoch":316,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:40.473131+0000","last_change":"2023-10-09T10:54:49.192234+0000","last_active":"2023-10-09T10:56:40.473131+0000","last_peered":"2023-10-09T10:56:40.473131+0000","last_clean":"2023-10-09T10:54:02.435746+0000","last_became_active":"2023-10-09T10:54:49.186762+0000","last_became_peered":"2023-10-09T10:54:49.186762+0000","last_unstale":"2023-10-09T10:56:40.473131+0000","last_undegraded":"2023-10-09T10:54:49.038902+0000","last_fullsized":"2023-10-09T10:54:48.912839+0000","mapping_epoch":295,"log_start":"293'6071","ondisk_log_start":"293'6071","created":265,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5063,"log_dups_size":0,"ondisk_log_size":5063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:48.327483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67895296,"num_objects":1036,"num_object_clones":0,"num_object_copies":3108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1036,"num_objects_misplaced":1330,"num_objects_unfound":0,"num_objects_dirty":1036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11134,"num_write_kb":389440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":-294},{"shards":"3,4,6","objects":1035}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":687,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.595086+0000","last_change":"2023-10-09T10:53:27.625180+0000","last_active":"2023-10-09T10:56:19.595086+0000","last_peered":"2023-10-09T10:56:19.595086+0000","last_clean":"2023-10-09T10:56:19.595086+0000","last_became_active":"2023-10-09T10:51:12.366828+0000","last_became_peered":"2023-10-09T10:51:12.366828+0000","last_unstale":"2023-10-09T10:56:19.595086+0000","last_undegraded":"2023-10-09T10:56:19.595086+0000","last_fullsized":"2023-10-09T10:56:19.595086+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.625079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.625079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:59:41.253976+0000","scrub_duration":0.0041067969999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":59,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.983521+0000","last_change":"2023-10-09T10:53:14.833073+0000","last_active":"2023-10-09T10:56:28.983521+0000","last_peered":"2023-10-09T10:56:28.983521+0000","last_clean":"2023-10-09T10:56:28.983521+0000","last_became_active":"2023-10-09T10:53:14.832736+0000","last_became_peered":"2023-10-09T10:53:14.832736+0000","last_unstale":"2023-10-09T10:56:28.983521+0000","last_undegraded":"2023-10-09T10:56:28.983521+0000","last_fullsized":"2023-10-09T10:56:28.983521+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:27.550707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.574710+0000","last_change":"2023-10-09T10:54:47.925711+0000","last_active":"2023-10-09T10:56:19.574710+0000","last_peered":"2023-10-09T10:56:19.574710+0000","last_clean":"2023-10-09T10:56:19.574710+0000","last_became_active":"2023-10-09T10:54:47.924286+0000","last_became_peered":"2023-10-09T10:54:47.924286+0000","last_unstale":"2023-10-09T10:56:19.574710+0000","last_undegraded":"2023-10-09T10:56:19.574710+0000","last_fullsized":"2023-10-09T10:56:19.574710+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:58.379080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":224,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:27.494446+0000","last_change":"2023-10-09T10:53:14.828291+0000","last_active":"2023-10-09T10:56:27.494446+0000","last_peered":"2023-10-09T10:56:27.494446+0000","last_clean":"2023-10-09T10:56:27.494446+0000","last_became_active":"2023-10-09T10:53:14.827991+0000","last_became_peered":"2023-10-09T10:53:14.827991+0000","last_unstale":"2023-10-09T10:56:27.494446+0000","last_undegraded":"2023-10-09T10:56:27.494446+0000","last_fullsized":"2023-10-09T10:56:27.494446+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:08:14.283112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"315'10959","reported_seq":11020,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:37.472756+0000","last_change":"2023-10-09T10:53:14.822279+0000","last_active":"2023-10-09T10:56:37.472756+0000","last_peered":"2023-10-09T10:56:37.472756+0000","last_clean":"2023-10-09T10:56:37.472756+0000","last_became_active":"2023-10-09T10:53:14.821952+0000","last_became_peered":"2023-10-09T10:53:14.821952+0000","last_unstale":"2023-10-09T10:56:37.472756+0000","last_undegraded":"2023-10-09T10:56:37.472756+0000","last_fullsized":"2023-10-09T10:56:37.472756+0000","mapping_epoch":266,"log_start":"286'5641","ondisk_log_start":"286'5641","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5318,"log_dups_size":0,"ondisk_log_size":5318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:37:14.124652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78315520,"num_objects":1195,"num_object_clones":0,"num_object_copies":3585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10959,"num_write_kb":388928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":711,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.968954+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T10:56:28.968954+0000","last_peered":"2023-10-09T10:56:28.968954+0000","last_clean":"2023-10-09T10:56:28.968954+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T10:56:28.968954+0000","last_undegraded":"2023-10-09T10:56:28.968954+0000","last_fullsized":"2023-10-09T10:56:28.968954+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":339,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:35.502002+0000","last_change":"2023-10-09T10:53:14.841466+0000","last_active":"2023-10-09T10:56:35.502002+0000","last_peered":"2023-10-09T10:56:35.502002+0000","last_clean":"2023-10-09T10:56:35.502002+0000","last_became_active":"2023-10-09T10:53:14.831674+0000","last_became_peered":"2023-10-09T10:53:14.831674+0000","last_unstale":"2023-10-09T10:56:35.502002+0000","last_undegraded":"2023-10-09T10:56:35.502002+0000","last_fullsized":"2023-10-09T10:56:35.502002+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:02.612069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":31,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.575027+0000","last_change":"2023-10-09T10:54:47.908305+0000","last_active":"2023-10-09T10:56:19.575027+0000","last_peered":"2023-10-09T10:56:19.575027+0000","last_clean":"2023-10-09T10:56:19.575027+0000","last_became_active":"2023-10-09T10:54:47.907999+0000","last_became_peered":"2023-10-09T10:54:47.907999+0000","last_unstale":"2023-10-09T10:56:19.575027+0000","last_undegraded":"2023-10-09T10:56:19.575027+0000","last_fullsized":"2023-10-09T10:56:19.575027+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:48.405124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":338,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.979698+0000","last_change":"2023-10-09T10:53:14.835841+0000","last_active":"2023-10-09T10:56:28.979698+0000","last_peered":"2023-10-09T10:56:28.979698+0000","last_clean":"2023-10-09T10:56:28.979698+0000","last_became_active":"2023-10-09T10:53:14.835551+0000","last_became_peered":"2023-10-09T10:53:14.835551+0000","last_unstale":"2023-10-09T10:56:28.979698+0000","last_undegraded":"2023-10-09T10:56:28.979698+0000","last_fullsized":"2023-10-09T10:56:28.979698+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:21.498099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"316'10893","reported_seq":15077,"reported_epoch":316,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:41.169121+0000","last_change":"2023-10-09T10:55:26.696179+0000","last_active":"2023-10-09T10:56:41.169121+0000","last_peered":"2023-10-09T10:56:41.169121+0000","last_clean":"2023-10-09T10:54:17.082254+0000","last_became_active":"2023-10-09T10:55:26.641906+0000","last_became_peered":"2023-10-09T10:55:26.641906+0000","last_unstale":"2023-10-09T10:56:41.169121+0000","last_undegraded":"2023-10-09T10:55:26.633572+0000","last_fullsized":"2023-10-09T10:55:26.625249+0000","mapping_epoch":306,"log_start":"283'5149","ondisk_log_start":"283'5149","created":265,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":5744,"log_dups_size":0,"ondisk_log_size":5744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:02.762689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69074968,"num_objects":1055,"num_object_clones":0,"num_object_copies":3165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3090,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1055,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10893,"num_write_kb":382273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2055,"num_bytes_recovered":134610944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5],"avail_no_missing":["7","5","6"],"object_location_counts":[{"shards":"5,6,7","objects":226},{"shards":"5,7","objects":-2035}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":707,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.981006+0000","last_change":"2023-10-09T10:53:30.567189+0000","last_active":"2023-10-09T10:56:28.981006+0000","last_peered":"2023-10-09T10:56:28.981006+0000","last_clean":"2023-10-09T10:56:28.981006+0000","last_became_active":"2023-10-09T10:53:17.840928+0000","last_became_peered":"2023-10-09T10:53:17.840928+0000","last_unstale":"2023-10-09T10:56:28.981006+0000","last_undegraded":"2023-10-09T10:56:28.981006+0000","last_fullsized":"2023-10-09T10:56:28.981006+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:52.176229+0000","scrub_duration":0.0053707060000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":169,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.981790+0000","last_change":"2023-10-09T10:55:25.634235+0000","last_active":"2023-10-09T10:56:28.981790+0000","last_peered":"2023-10-09T10:56:28.981790+0000","last_clean":"2023-10-09T10:56:28.981790+0000","last_became_active":"2023-10-09T10:55:25.633940+0000","last_became_peered":"2023-10-09T10:55:25.633940+0000","last_unstale":"2023-10-09T10:56:28.981790+0000","last_undegraded":"2023-10-09T10:56:28.981790+0000","last_fullsized":"2023-10-09T10:56:28.981790+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:40.554244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":243,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.594929+0000","last_change":"2023-10-09T10:54:48.016159+0000","last_active":"2023-10-09T10:56:19.594929+0000","last_peered":"2023-10-09T10:56:19.594929+0000","last_clean":"2023-10-09T10:56:19.594929+0000","last_became_active":"2023-10-09T10:54:47.939409+0000","last_became_peered":"2023-10-09T10:54:47.939409+0000","last_unstale":"2023-10-09T10:56:19.594929+0000","last_undegraded":"2023-10-09T10:56:19.594929+0000","last_fullsized":"2023-10-09T10:56:19.594929+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:11:15.175207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":257,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.968611+0000","last_change":"2023-10-09T10:51:50.371282+0000","last_active":"2023-10-09T10:56:28.968611+0000","last_peered":"2023-10-09T10:56:28.968611+0000","last_clean":"2023-10-09T10:56:28.968611+0000","last_became_active":"2023-10-09T10:51:50.370973+0000","last_became_peered":"2023-10-09T10:51:50.370973+0000","last_unstale":"2023-10-09T10:56:28.968611+0000","last_undegraded":"2023-10-09T10:56:28.968611+0000","last_fullsized":"2023-10-09T10:56:28.968611+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:15.603164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"316'8586","reported_seq":11480,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T10:56:40.480522+0000","last_change":"2023-10-09T10:54:56.638107+0000","last_active":"2023-10-09T10:56:40.480522+0000","last_peered":"2023-10-09T10:56:40.480522+0000","last_clean":"2023-10-09T10:56:40.480522+0000","last_became_active":"2023-10-09T10:54:56.633188+0000","last_became_peered":"2023-10-09T10:54:56.633188+0000","last_unstale":"2023-10-09T10:56:40.480522+0000","last_undegraded":"2023-10-09T10:56:40.480522+0000","last_fullsized":"2023-10-09T10:56:40.480522+0000","mapping_epoch":299,"log_start":"271'1212","ondisk_log_start":"271'1212","created":289,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7374,"log_dups_size":0,"ondisk_log_size":7374,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34242560,"num_objects":523,"num_object_clones":0,"num_object_copies":1569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5569,"num_write_kb":194912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1061,"num_bytes_recovered":69500928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":707,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.978501+0000","last_change":"2023-10-09T10:53:29.568199+0000","last_active":"2023-10-09T10:56:28.978501+0000","last_peered":"2023-10-09T10:56:28.978501+0000","last_clean":"2023-10-09T10:56:28.978501+0000","last_became_active":"2023-10-09T10:52:55.416357+0000","last_became_peered":"2023-10-09T10:52:55.416357+0000","last_unstale":"2023-10-09T10:56:28.978501+0000","last_undegraded":"2023-10-09T10:56:28.978501+0000","last_fullsized":"2023-10-09T10:56:28.978501+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:44.364650+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":223,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.593082+0000","last_change":"2023-10-09T10:53:14.823731+0000","last_active":"2023-10-09T10:56:19.593082+0000","last_peered":"2023-10-09T10:56:19.593082+0000","last_clean":"2023-10-09T10:56:19.593082+0000","last_became_active":"2023-10-09T10:53:14.823470+0000","last_became_peered":"2023-10-09T10:53:14.823470+0000","last_unstale":"2023-10-09T10:56:19.593082+0000","last_undegraded":"2023-10-09T10:56:19.593082+0000","last_fullsized":"2023-10-09T10:56:19.593082+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:58:36.927599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":54,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.995086+0000","last_change":"2023-10-09T10:54:47.944380+0000","last_active":"2023-10-09T10:56:28.995086+0000","last_peered":"2023-10-09T10:56:28.995086+0000","last_clean":"2023-10-09T10:56:28.995086+0000","last_became_active":"2023-10-09T10:54:47.943766+0000","last_became_peered":"2023-10-09T10:54:47.943766+0000","last_unstale":"2023-10-09T10:56:28.995086+0000","last_undegraded":"2023-10-09T10:56:28.995086+0000","last_fullsized":"2023-10-09T10:56:28.995086+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:16:11.869603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"316'8403","reported_seq":8498,"reported_epoch":316,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T10:56:40.497668+0000","last_change":"2023-10-09T10:54:49.184288+0000","last_active":"2023-10-09T10:56:40.497668+0000","last_peered":"2023-10-09T10:56:40.497668+0000","last_clean":"2023-10-09T10:54:02.391425+0000","last_became_active":"2023-10-09T10:54:49.118164+0000","last_became_peered":"2023-10-09T10:54:49.118164+0000","last_unstale":"2023-10-09T10:56:40.497668+0000","last_undegraded":"2023-10-09T10:54:48.969517+0000","last_fullsized":"2023-10-09T10:54:48.892012+0000","mapping_epoch":295,"log_start":"271'1000","ondisk_log_start":"271'1000","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7403,"log_dups_size":0,"ondisk_log_size":7403,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39354368,"num_objects":601,"num_object_clones":0,"num_object_copies":1803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":727,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5427,"num_write_kb":192864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"3,4,6","objects":568},{"shards":"3,6","objects":-126}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":638,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.980396+0000","last_change":"2023-10-09T10:53:36.563918+0000","last_active":"2023-10-09T10:56:28.980396+0000","last_peered":"2023-10-09T10:56:28.980396+0000","last_clean":"2023-10-09T10:56:28.980396+0000","last_became_active":"2023-10-09T10:53:17.842952+0000","last_became_peered":"2023-10-09T10:53:17.842952+0000","last_unstale":"2023-10-09T10:56:28.980396+0000","last_undegraded":"2023-10-09T10:56:28.980396+0000","last_fullsized":"2023-10-09T10:56:28.980396+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:18:53.185917+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":142,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.969146+0000","last_change":"2023-10-09T10:54:47.923386+0000","last_active":"2023-10-09T10:56:28.969146+0000","last_peered":"2023-10-09T10:56:28.969146+0000","last_clean":"2023-10-09T10:56:28.969146+0000","last_became_active":"2023-10-09T10:54:47.914257+0000","last_became_peered":"2023-10-09T10:54:47.914257+0000","last_unstale":"2023-10-09T10:56:28.969146+0000","last_undegraded":"2023-10-09T10:56:28.969146+0000","last_fullsized":"2023-10-09T10:56:28.969146+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:24:18.190715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":333,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.979293+0000","last_change":"2023-10-09T10:54:47.977518+0000","last_active":"2023-10-09T10:56:28.979293+0000","last_peered":"2023-10-09T10:56:28.979293+0000","last_clean":"2023-10-09T10:56:28.979293+0000","last_became_active":"2023-10-09T10:54:47.977332+0000","last_became_peered":"2023-10-09T10:54:47.977332+0000","last_unstale":"2023-10-09T10:56:28.979293+0000","last_undegraded":"2023-10-09T10:56:28.979293+0000","last_fullsized":"2023-10-09T10:56:28.979293+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:29.620839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.16","version":"315'8527","reported_seq":8596,"reported_epoch":315,"state":"active+recovery_wait","last_fresh":"2023-10-09T10:56:37.460989+0000","last_change":"2023-10-09T10:54:40.600782+0000","last_active":"2023-10-09T10:56:37.460989+0000","last_peered":"2023-10-09T10:56:37.460989+0000","last_clean":"2023-10-09T10:54:39.522361+0000","last_became_active":"2023-10-09T10:54:40.590969+0000","last_became_peered":"2023-10-09T10:54:40.590969+0000","last_unstale":"2023-10-09T10:56:37.460989+0000","last_undegraded":"2023-10-09T10:56:37.460989+0000","last_fullsized":"2023-10-09T10:56:37.460989+0000","mapping_epoch":289,"log_start":"271'636","ondisk_log_start":"271'636","created":289,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7891,"log_dups_size":0,"ondisk_log_size":7891,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35028992,"num_objects":535,"num_object_clones":0,"num_object_copies":1605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5533,"num_write_kb":194144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":535}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":265,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.567525+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:56:19.567525+0000","last_peered":"2023-10-09T10:56:19.567525+0000","last_clean":"2023-10-09T10:56:19.567525+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:56:19.567525+0000","last_undegraded":"2023-10-09T10:56:19.567525+0000","last_fullsized":"2023-10-09T10:56:19.567525+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":260,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.979543+0000","last_change":"2023-10-09T10:50:43.385150+0000","last_active":"2023-10-09T10:56:28.979543+0000","last_peered":"2023-10-09T10:56:28.979543+0000","last_clean":"2023-10-09T10:56:28.979543+0000","last_became_active":"2023-10-09T10:50:43.384788+0000","last_became_peered":"2023-10-09T10:50:43.384788+0000","last_unstale":"2023-10-09T10:56:28.979543+0000","last_undegraded":"2023-10-09T10:56:28.979543+0000","last_fullsized":"2023-10-09T10:56:28.979543+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:36.085910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.15","version":"316'8433","reported_seq":8505,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-10-09T10:56:41.146698+0000","last_change":"2023-10-09T10:54:40.578264+0000","last_active":"2023-10-09T10:56:41.146698+0000","last_peered":"2023-10-09T10:56:41.146698+0000","last_clean":"2023-10-09T10:56:41.146698+0000","last_became_active":"2023-10-09T10:54:40.568498+0000","last_became_peered":"2023-10-09T10:54:40.568498+0000","last_unstale":"2023-10-09T10:56:41.146698+0000","last_undegraded":"2023-10-09T10:56:41.146698+0000","last_fullsized":"2023-10-09T10:56:41.146698+0000","mapping_epoch":289,"log_start":"271'400","ondisk_log_start":"271'400","created":289,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":8033,"log_dups_size":0,"ondisk_log_size":8033,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33292288,"num_objects":508,"num_object_clones":0,"num_object_copies":1524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5472,"num_write_kb":191360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":622,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.982151+0000","last_change":"2023-10-09T10:55:25.635605+0000","last_active":"2023-10-09T10:56:28.982151+0000","last_peered":"2023-10-09T10:56:28.982151+0000","last_clean":"2023-10-09T10:56:28.982151+0000","last_became_active":"2023-10-09T10:55:25.635328+0000","last_became_peered":"2023-10-09T10:55:25.635328+0000","last_unstale":"2023-10-09T10:56:28.982151+0000","last_undegraded":"2023-10-09T10:56:28.982151+0000","last_fullsized":"2023-10-09T10:56:28.982151+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:46.305288+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:46.305288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":340,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.980617+0000","last_change":"2023-10-09T10:54:17.886158+0000","last_active":"2023-10-09T10:56:28.980617+0000","last_peered":"2023-10-09T10:56:28.980617+0000","last_clean":"2023-10-09T10:56:28.980617+0000","last_became_active":"2023-10-09T10:54:17.885816+0000","last_became_peered":"2023-10-09T10:54:17.885816+0000","last_unstale":"2023-10-09T10:56:28.980617+0000","last_undegraded":"2023-10-09T10:56:28.980617+0000","last_fullsized":"2023-10-09T10:56:28.980617+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:05.270825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":343,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.978800+0000","last_change":"2023-10-09T10:54:17.957784+0000","last_active":"2023-10-09T10:56:28.978800+0000","last_peered":"2023-10-09T10:56:28.978800+0000","last_clean":"2023-10-09T10:56:28.978800+0000","last_became_active":"2023-10-09T10:54:17.954788+0000","last_became_peered":"2023-10-09T10:54:17.954788+0000","last_unstale":"2023-10-09T10:56:28.978800+0000","last_undegraded":"2023-10-09T10:56:28.978800+0000","last_fullsized":"2023-10-09T10:56:28.978800+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:48.709390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"316'8509","reported_seq":8592,"reported_epoch":316,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T10:56:41.282108+0000","last_change":"2023-10-09T10:54:42.558777+0000","last_active":"2023-10-09T10:56:41.282108+0000","last_peered":"2023-10-09T10:56:41.282108+0000","last_clean":"2023-10-09T10:54:17.036595+0000","last_became_active":"2023-10-09T10:54:42.553632+0000","last_became_peered":"2023-10-09T10:54:42.553632+0000","last_unstale":"2023-10-09T10:56:41.282108+0000","last_undegraded":"2023-10-09T10:56:41.282108+0000","last_fullsized":"2023-10-09T10:56:41.282108+0000","mapping_epoch":291,"log_start":"271'600","ondisk_log_start":"271'600","created":289,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:53:12.795527+0000","last_clean_scrub_stamp":"2023-10-09T10:53:12.795527+0000","objects_scrubbed":0,"log_size":7909,"log_dups_size":0,"ondisk_log_size":7909,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33128448,"num_objects":506,"num_object_clones":0,"num_object_copies":1518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":506,"num_objects_unfound":0,"num_objects_dirty":506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5524,"num_write_kb":192928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":506}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":589,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:19.574630+0000","last_change":"2023-10-09T10:54:50.914473+0000","last_active":"2023-10-09T10:56:19.574630+0000","last_peered":"2023-10-09T10:56:19.574630+0000","last_clean":"2023-10-09T10:56:19.574630+0000","last_became_active":"2023-10-09T10:54:50.914005+0000","last_became_peered":"2023-10-09T10:54:50.914005+0000","last_unstale":"2023-10-09T10:56:19.574630+0000","last_undegraded":"2023-10-09T10:56:19.574630+0000","last_fullsized":"2023-10-09T10:56:19.574630+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.070492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.070492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":224,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-10-09T10:56:27.494838+0000","last_change":"2023-10-09T10:54:18.089053+0000","last_active":"2023-10-09T10:56:27.494838+0000","last_peered":"2023-10-09T10:56:27.494838+0000","last_clean":"2023-10-09T10:56:27.494838+0000","last_became_active":"2023-10-09T10:54:17.886070+0000","last_became_peered":"2023-10-09T10:54:17.886070+0000","last_unstale":"2023-10-09T10:56:27.494838+0000","last_undegraded":"2023-10-09T10:56:27.494838+0000","last_fullsized":"2023-10-09T10:56:27.494838+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:35.628407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":320,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-10-09T10:56:28.968513+0000","last_change":"2023-10-09T10:54:47.908059+0000","last_active":"2023-10-09T10:56:28.968513+0000","last_peered":"2023-10-09T10:56:28.968513+0000","last_clean":"2023-10-09T10:56:28.968513+0000","last_became_active":"2023-10-09T10:54:47.907780+0000","last_became_peered":"2023-10-09T10:54:47.907780+0000","last_unstale":"2023-10-09T10:56:28.968513+0000","last_undegraded":"2023-10-09T10:56:28.968513+0000","last_fullsized":"2023-10-09T10:56:28.968513+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:55:59.064617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":6668,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":1158938648,"num_objects":17685,"num_object_clones":0,"num_object_copies":53055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12725,"num_objects_misplaced":4095,"num_objects_unfound":0,"num_objects_dirty":17685,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":175619,"num_write_kb":6185665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11043,"num_bytes_recovered":723648512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4641730560,"data_stored":4641718344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":188768,"ondisk_log_size":188768,"up":78,"acting":68,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":729016,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":115,"num_read_kb":214,"num_write":56,"num_write_kb":567,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1105158,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332361,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1084144,"kb_used_data":774568,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92661008,"statfs":{"total":95995035648,"available":94884872192,"internally_reserved":0,"allocated":793157632,"data_stored":792008005,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1429999999999998,"5min":1.9299999999999999,"15min":1.365},"min":{"1min":0.23599999999999999,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":7.4080000000000004,"5min":9.6259999999999994,"15min":9.6379999999999999},"last":0.26300000000000001},{"interface":"front","average":{"1min":2.3650000000000002,"5min":1.8839999999999999,"15min":1.55},"min":{"1min":0.28699999999999998,"5min":0.246,"15min":0.24299999999999999},"max":{"1min":8.7739999999999991,"5min":8.7739999999999991,"15min":16.731000000000002},"last":0.40999999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:56:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1099999999999999,"5min":2.2309999999999999,"15min":1.706},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.25700000000000001},"max":{"1min":9.2349999999999994,"5min":34.756,"15min":34.756},"last":4.1790000000000003},{"interface":"front","average":{"1min":2.4039999999999999,"5min":1.9750000000000001,"15min":1.444},"min":{"1min":0.26100000000000001,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":11.048,"5min":35.658999999999999,"15min":35.658999999999999},"last":0.315}]},{"osd":2,"last update":"Mon Oct 9 10:56:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8610000000000002,"5min":1.415,"15min":1.4650000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.202,"5min":9.202,"15min":11.058},"last":3.931},{"interface":"front","average":{"1min":2.3250000000000002,"5min":1.6379999999999999,"15min":1.3520000000000001},"min":{"1min":0.29599999999999999,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":9.4280000000000008,"5min":11.579000000000001,"15min":11.579000000000001},"last":1.27}]},{"osd":3,"last update":"Mon Oct 9 10:55:55 2023","interfaces":[{"interface":"back","average":{"1min":2.056,"5min":1.9490000000000001,"15min":1.7050000000000001},"min":{"1min":0.32900000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":8.7260000000000009,"5min":10.069000000000001,"15min":11.523},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.9930000000000001,"5min":1.52,"15min":1.526},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.585,"5min":8.6300000000000008,"15min":19.234999999999999},"last":0.746}]},{"osd":4,"last update":"Mon Oct 9 10:56:36 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.94399999999999995,"15min":1.234},"min":{"1min":0.27200000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.569,"5min":8.2940000000000005,"15min":16.695},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.88800000000000001,"15min":1.1299999999999999},"min":{"1min":0.28699999999999998,"5min":0.28199999999999997,"15min":0.26200000000000001},"max":{"1min":4.1100000000000003,"5min":7.9320000000000004,"15min":11.109999999999999},"last":0.34100000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:56:36 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.84499999999999997,"15min":1.163},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":3.7330000000000001,"5min":4.7619999999999996,"15min":13.262},"last":0.37},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.98899999999999999,"15min":1.3440000000000001},"min":{"1min":0.33500000000000002,"5min":0.318,"15min":0.29599999999999999},"max":{"1min":2.363,"5min":8.4339999999999993,"15min":11.741},"last":0.46100000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:56:36 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.2070000000000001,"15min":1.46},"min":{"1min":0.315,"5min":0.315,"15min":0.28199999999999997},"max":{"1min":3.5139999999999998,"5min":8.3610000000000007,"15min":14.913},"last":0.496},{"interface":"front","average":{"1min":1.123,"5min":1.1120000000000001,"15min":1.3100000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.3259999999999996,"5min":12.268000000000001,"15min":19.187999999999999},"last":0.39800000000000002}]}]},{"osd":6,"up_from":44,"seq":188978561292,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1721052,"kb_used_data":824980,"kb_used_omap":0,"kb_used_meta":896064,"kb_avail":92024100,"statfs":{"total":95995035648,"available":94232678400,"internally_reserved":0,"allocated":844779520,"data_stored":843649659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917569536},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:55 2023","interfaces":[{"interface":"back","average":{"1min":5.0620000000000003,"5min":1.9490000000000001,"15min":1.4770000000000001},"min":{"1min":0.22800000000000001,"5min":0.216,"15min":0.216},"max":{"1min":54.743000000000002,"5min":54.743000000000002,"15min":54.743000000000002},"last":0.23499999999999999},{"interface":"front","average":{"1min":4.1219999999999999,"5min":1.97,"15min":1.4850000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.22800000000000001},"max":{"1min":53.701000000000001,"5min":53.701000000000001,"15min":53.701000000000001},"last":0.29099999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:56:13 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":1.2949999999999999,"15min":1.331},"min":{"1min":0.29199999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.2779999999999996,"5min":8.4730000000000008,"15min":16.747},"last":3.4830000000000001},{"interface":"front","average":{"1min":1.488,"5min":1.53,"15min":1.3839999999999999},"min":{"1min":0.32000000000000001,"5min":0.23599999999999999,"15min":0.23100000000000001},"max":{"1min":4.2549999999999999,"5min":8.0579999999999998,"15min":16.734000000000002},"last":3.46}]},{"osd":2,"last update":"Mon Oct 9 10:56:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1389999999999998,"5min":1.873,"15min":1.508},"min":{"1min":0.29799999999999999,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":6.2400000000000002,"5min":8.6389999999999993,"15min":13.208},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.6839999999999999,"5min":1.7,"15min":1.635},"min":{"1min":0.32500000000000001,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":5.6829999999999998,"5min":12.83,"15min":12.83},"last":0.35999999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:55:55 2023","interfaces":[{"interface":"back","average":{"1min":1.837,"5min":1.768,"15min":1.4510000000000001},"min":{"1min":0.32500000000000001,"5min":0.29799999999999999,"15min":0.255},"max":{"1min":5.8399999999999999,"5min":7.4720000000000004,"15min":12.391},"last":3.6360000000000001},{"interface":"front","average":{"1min":2.1949999999999998,"5min":2.0739999999999998,"15min":1.667},"min":{"1min":0.38600000000000001,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":8.3759999999999994,"5min":8.3759999999999994,"15min":13.179},"last":0.371}]},{"osd":4,"last update":"Mon Oct 9 10:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":1.157,"15min":1.1759999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.23200000000000001},"max":{"1min":2.0110000000000001,"5min":16.978999999999999,"15min":16.978999999999999},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.93799999999999994,"15min":1.145},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.867,"5min":5.2380000000000004,"15min":12.481999999999999},"last":0.82899999999999996}]},{"osd":5,"last update":"Mon Oct 9 10:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":1.4890000000000001,"15min":1.3069999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.5880000000000001,"5min":12.923,"15min":12.923},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.73299999999999998,"5min":1.1299999999999999,"15min":1.206},"min":{"1min":0.312,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.7719999999999998,"5min":9.6769999999999996,"15min":12.456},"last":0.495}]},{"osd":7,"last update":"Mon Oct 9 10:55:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":1.6950000000000001,"15min":1.3340000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.26600000000000001},"max":{"1min":2.0449999999999999,"5min":12.882,"15min":23.358000000000001},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.70099999999999996,"5min":1.149,"15min":1.1299999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.7200000000000002,"5min":7.0880000000000001,"15min":12.422000000000001},"last":1.9850000000000001}]}]},{"osd":1,"up_from":85,"seq":365072220330,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":523156,"kb_used_data":138892,"kb_used_omap":9,"kb_used_meta":384246,"kb_avail":93221996,"statfs":{"total":95995035648,"available":95459323904,"internally_reserved":0,"allocated":142225408,"data_stored":140573256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":393468905},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:56:07 2023","interfaces":[{"interface":"back","average":{"1min":4.6989999999999998,"5min":2.1320000000000001,"15min":1.74},"min":{"1min":0.26200000000000001,"5min":0.20999999999999999,"15min":0.19500000000000001},"max":{"1min":19.992999999999999,"5min":19.992999999999999,"15min":19.992999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":4.9359999999999999,"5min":2.6230000000000002,"15min":1.8420000000000001},"min":{"1min":0.29399999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":18.140000000000001,"5min":18.303000000000001,"15min":18.303000000000001},"last":10.901999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:56:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8149999999999999,"5min":1.9450000000000001,"15min":1.504},"min":{"1min":0.26100000000000001,"5min":0.222,"15min":0.19700000000000001},"max":{"1min":14.878,"5min":14.878,"15min":15.356},"last":0.40300000000000002},{"interface":"front","average":{"1min":3.0430000000000001,"5min":1.986,"15min":1.6519999999999999},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":11.481999999999999,"5min":18.004000000000001,"15min":18.004000000000001},"last":0.33900000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:55:56 2023","interfaces":[{"interface":"back","average":{"1min":4.4429999999999996,"5min":2.1859999999999999,"15min":1.855},"min":{"1min":0.22900000000000001,"5min":0.223,"15min":0.218},"max":{"1min":20.036000000000001,"5min":20.036000000000001,"15min":20.036000000000001},"last":1.593},{"interface":"front","average":{"1min":6.359,"5min":2.9350000000000001,"15min":1.9430000000000001},"min":{"1min":0.42799999999999999,"5min":0.219,"15min":0.219},"max":{"1min":19.640999999999998,"5min":19.640999999999998,"15min":19.640999999999998},"last":14.303000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:56:07 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":1.6339999999999999,"15min":1.397},"min":{"1min":0.32200000000000001,"5min":0.26500000000000001,"15min":0.255},"max":{"1min":14.763,"5min":14.763,"15min":14.763},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.78,"5min":1.6000000000000001,"15min":1.5409999999999999},"min":{"1min":0.30399999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.2759999999999998,"5min":6.835,"15min":14.228999999999999},"last":0.39500000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:56:07 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":1.8759999999999999,"15min":1.4490000000000001},"min":{"1min":0.42999999999999999,"5min":0.32700000000000001,"15min":0.253},"max":{"1min":8.423,"5min":9.6289999999999996,"15min":9.7720000000000002},"last":8.1519999999999992},{"interface":"front","average":{"1min":2.6899999999999999,"5min":2.008,"15min":1.734},"min":{"1min":0.33100000000000002,"5min":0.30299999999999999,"15min":0.28899999999999998},"max":{"1min":14.833,"5min":14.833,"15min":17.085000000000001},"last":0.434}]},{"osd":6,"last update":"Mon Oct 9 10:56:07 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":1.877,"15min":1.534},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.27400000000000002},"max":{"1min":8.468,"5min":9.4049999999999994,"15min":10.425000000000001},"last":8.1020000000000003},{"interface":"front","average":{"1min":2.6190000000000002,"5min":1.7490000000000001,"15min":1.548},"min":{"1min":0.35799999999999998,"5min":0.312,"15min":0.312},"max":{"1min":14.804,"5min":14.804,"15min":14.804},"last":0.48999999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:56:07 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":2.1059999999999999,"15min":1.639},"min":{"1min":0.38600000000000001,"5min":0.36299999999999999,"15min":0.34000000000000002},"max":{"1min":14.856,"5min":16.312000000000001,"15min":16.312000000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.794,"5min":1.611,"15min":1.5620000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.32500000000000001},"max":{"1min":6.2990000000000004,"5min":8.8149999999999995,"15min":14.273999999999999},"last":0.374}]}]},{"osd":0,"up_from":80,"seq":343597383864,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":951656,"kb_used_data":468768,"kb_used_omap":28,"kb_used_meta":482851,"kb_avail":92793496,"statfs":{"total":95995035648,"available":95020539904,"internally_reserved":0,"allocated":480018432,"data_stored":478284365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":494440272},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.8400000000000001,"15min":1.6339999999999999},"min":{"1min":0.22800000000000001,"5min":0.20399999999999999,"15min":0.191},"max":{"1min":6.0919999999999996,"5min":13.263999999999999,"15min":25.475000000000001},"last":5.6280000000000001},{"interface":"front","average":{"1min":2.3559999999999999,"5min":1.8220000000000001,"15min":1.613},"min":{"1min":0.28899999999999998,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":9.1219999999999999,"5min":9.359,"15min":20.832999999999998},"last":4.5019999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:56:37 2023","interfaces":[{"interface":"back","average":{"1min":4.1479999999999997,"5min":2.4969999999999999,"15min":1.8109999999999999},"min":{"1min":0.42199999999999999,"5min":0.189,"15min":0.189},"max":{"1min":13.976000000000001,"5min":24.852,"15min":24.852},"last":5.3949999999999996},{"interface":"front","average":{"1min":3.6400000000000001,"5min":3.331,"15min":2.2200000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.256,"5min":77.256,"15min":77.256},"last":1.522}]},{"osd":3,"last update":"Mon Oct 9 10:55:59 2023","interfaces":[{"interface":"back","average":{"1min":3.3900000000000001,"5min":2.3660000000000001,"15min":1.96},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20200000000000001},"max":{"1min":18.972000000000001,"5min":18.972000000000001,"15min":18.972000000000001},"last":1.7629999999999999},{"interface":"front","average":{"1min":2.7050000000000001,"5min":2.371,"15min":1.9059999999999999},"min":{"1min":0.35999999999999999,"5min":0.25800000000000001,"15min":0.224},"max":{"1min":7.8380000000000001,"5min":13.305,"15min":14.391999999999999},"last":1.8660000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:55:52 2023","interfaces":[{"interface":"back","average":{"1min":2.9550000000000001,"5min":2.8380000000000001,"15min":1.8979999999999999},"min":{"1min":0.38500000000000001,"5min":0.33800000000000002,"15min":0.26200000000000001},"max":{"1min":9.4749999999999996,"5min":77.177999999999997,"15min":77.177999999999997},"last":1.4930000000000001},{"interface":"front","average":{"1min":1.839,"5min":1.8100000000000001,"15min":1.7110000000000001},"min":{"1min":0.36599999999999999,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":9.032,"5min":10.427,"15min":12.318},"last":1.724}]},{"osd":5,"last update":"Mon Oct 9 10:55:52 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":1.966,"15min":1.7509999999999999},"min":{"1min":0.34300000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.9559999999999995,"5min":10.478,"15min":12.157},"last":1.7889999999999999},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.7929999999999999,"15min":1.607},"min":{"1min":0.30399999999999999,"5min":0.30199999999999999,"15min":0.29499999999999998},"max":{"1min":5.319,"5min":24.754999999999999,"15min":24.754999999999999},"last":5.4320000000000004}]},{"osd":6,"last update":"Mon Oct 9 10:55:52 2023","interfaces":[{"interface":"back","average":{"1min":2.7040000000000002,"5min":2.2210000000000001,"15min":1.8839999999999999},"min":{"1min":0.38700000000000001,"5min":0.34200000000000003,"15min":0.30399999999999999},"max":{"1min":18.920000000000002,"5min":18.920000000000002,"15min":18.920000000000002},"last":1.8280000000000001},{"interface":"front","average":{"1min":1.4119999999999999,"5min":1.903,"15min":1.5189999999999999},"min":{"1min":0.34200000000000003,"5min":0.307,"15min":0.307},"max":{"1min":5.2910000000000004,"5min":24.800000000000001,"15min":24.800000000000001},"last":1.9039999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:55:52 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":1.917,"15min":1.7110000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.125,"5min":13.225,"15min":14.48},"last":1.929},{"interface":"front","average":{"1min":3.2130000000000001,"5min":3.0430000000000001,"15min":2.0419999999999998},"min":{"1min":0.40000000000000002,"5min":0.32600000000000001,"15min":0.32200000000000001},"max":{"1min":10.819000000000001,"5min":77.228999999999999,"15min":77.228999999999999},"last":1.45}]}]},{"osd":2,"up_from":90,"seq":386547056805,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":445688,"kb_used_data":158448,"kb_used_omap":81,"kb_used_meta":287150,"kb_avail":93299464,"statfs":{"total":95995035648,"available":95538651136,"internally_reserved":0,"allocated":162250752,"data_stored":160684540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":294041818},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:56:37 2023","interfaces":[{"interface":"back","average":{"1min":3.0920000000000001,"5min":2.0150000000000001,"15min":1.5880000000000001},"min":{"1min":0.188,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":11.737,"5min":17.574000000000002,"15min":18.512},"last":5.6280000000000001},{"interface":"front","average":{"1min":3.1440000000000001,"5min":2.0870000000000002,"15min":1.7190000000000001},"min":{"1min":0.34100000000000003,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":10.18,"5min":15.595000000000001,"15min":17.975000000000001},"last":0.53200000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:56:37 2023","interfaces":[{"interface":"back","average":{"1min":2.6600000000000001,"5min":1.879,"15min":1.671},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":16.359000000000002,"5min":16.359000000000002,"15min":16.359000000000002},"last":0.499},{"interface":"front","average":{"1min":2.7440000000000002,"5min":1.633,"15min":1.4550000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.20699999999999999},"max":{"1min":12.693,"5min":12.693,"15min":16.454999999999998},"last":3.9129999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:55:57 2023","interfaces":[{"interface":"back","average":{"1min":4.4539999999999997,"5min":1.954,"15min":1.5640000000000001},"min":{"1min":0.30299999999999999,"5min":0.27800000000000002,"15min":0.22500000000000001},"max":{"1min":19.539000000000001,"5min":19.539000000000001,"15min":19.539000000000001},"last":6.149},{"interface":"front","average":{"1min":4.4400000000000004,"5min":2.327,"15min":1.867},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.20200000000000001},"max":{"1min":16.702999999999999,"5min":16.702999999999999,"15min":16.702999999999999},"last":3.2229999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:56:37 2023","interfaces":[{"interface":"back","average":{"1min":2.464,"5min":1.8859999999999999,"15min":1.7190000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.254},"max":{"1min":12.452999999999999,"5min":15.672000000000001,"15min":16.231999999999999},"last":4.0049999999999999},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.853,"15min":1.657},"min":{"1min":0.32100000000000001,"5min":0.307,"15min":0.26300000000000001},"max":{"1min":8.157,"5min":15.696999999999999,"15min":15.696999999999999},"last":5.6680000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:56:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":1.6579999999999999,"15min":1.6279999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.245},"max":{"1min":10.135999999999999,"5min":12.128,"15min":17.920000000000002},"last":0.55700000000000005},{"interface":"front","average":{"1min":2.3330000000000002,"5min":1.8520000000000001,"15min":1.665},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":12.484,"5min":16.597000000000001,"15min":17.882999999999999},"last":4.1040000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:56:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1459999999999999,"5min":1.6739999999999999,"15min":1.4530000000000001},"min":{"1min":0.40699999999999997,"5min":0.33400000000000002,"15min":0.247},"max":{"1min":7.7190000000000003,"5min":8.8460000000000001,"15min":14.029999999999999},"last":5.6929999999999996},{"interface":"front","average":{"1min":1.911,"5min":1.6779999999999999,"15min":1.5880000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.29199999999999998},"max":{"1min":10.111000000000001,"5min":10.111000000000001,"15min":12.862},"last":0.58399999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:56:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1070000000000002,"5min":1.6479999999999999,"15min":1.4710000000000001},"min":{"1min":0.42199999999999999,"5min":0.371,"15min":0.32500000000000001},"max":{"1min":7.7439999999999998,"5min":8.8879999999999999,"15min":13.999000000000001},"last":6.0860000000000003},{"interface":"front","average":{"1min":2.161,"5min":1.8899999999999999,"15min":1.679},"min":{"1min":0.378,"5min":0.34000000000000002,"15min":0.307},"max":{"1min":10.157999999999999,"5min":10.157999999999999,"15min":12.826000000000001},"last":0.60899999999999999}]}]},{"osd":3,"up_from":95,"seq":408021893279,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1236280,"kb_used_data":563440,"kb_used_omap":7,"kb_used_meta":672696,"kb_avail":92508872,"statfs":{"total":95995035648,"available":94729084928,"internally_reserved":0,"allocated":576962560,"data_stored":575218751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":688841320},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:56:04 2023","interfaces":[{"interface":"back","average":{"1min":3.7389999999999999,"5min":2.7450000000000001,"15min":1.9350000000000001},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":11.819000000000001,"5min":38.817999999999998,"15min":38.817999999999998},"last":8.8629999999999995},{"interface":"front","average":{"1min":5.0810000000000004,"5min":3.3809999999999998,"15min":2.161},"min":{"1min":0.67800000000000005,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":13.351000000000001,"5min":17.312000000000001,"15min":17.312000000000001},"last":14.263999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:56:04 2023","interfaces":[{"interface":"back","average":{"1min":3.0870000000000002,"5min":2.5030000000000001,"15min":2.0099999999999998},"min":{"1min":0.20200000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":18.806999999999999,"5min":18.806999999999999,"15min":18.806999999999999},"last":3.004},{"interface":"front","average":{"1min":3.5,"5min":2.7879999999999998,"15min":1.978},"min":{"1min":0.29899999999999999,"5min":0.22,"15min":0.193},"max":{"1min":11.928000000000001,"5min":18.526,"15min":18.526},"last":8.8300000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:56:04 2023","interfaces":[{"interface":"back","average":{"1min":4.0949999999999998,"5min":2.3559999999999999,"15min":1.8819999999999999},"min":{"1min":0.26300000000000001,"5min":0.24199999999999999,"15min":0.23699999999999999},"max":{"1min":18.774000000000001,"5min":18.774000000000001,"15min":18.774000000000001},"last":1.9810000000000001},{"interface":"front","average":{"1min":4.5810000000000004,"5min":3.0529999999999999,"15min":2.1150000000000002},"min":{"1min":0.255,"5min":0.248,"15min":0.248},"max":{"1min":14.134,"5min":17.268000000000001,"15min":17.268000000000001},"last":27.538}]},{"osd":4,"last update":"Mon Oct 9 10:56:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4220000000000002,"5min":2.206,"15min":1.7689999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28199999999999997},"max":{"1min":9.0500000000000007,"5min":11.863,"15min":20.388000000000002},"last":8.3490000000000002},{"interface":"front","average":{"1min":2.3300000000000001,"5min":2.089,"15min":1.75},"min":{"1min":0.28599999999999998,"5min":0.28399999999999997,"15min":0.24299999999999999},"max":{"1min":10.462999999999999,"5min":11.680999999999999,"15min":19.085999999999999},"last":4.9009999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:56:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2010000000000001,"5min":2.1779999999999999,"15min":2.012},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.492000000000001,"5min":14.045,"15min":51.509999999999998},"last":8.1519999999999992},{"interface":"front","average":{"1min":1.52,"5min":1.754,"15min":1.9470000000000001},"min":{"1min":0.374,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":9.1140000000000008,"5min":10.193,"15min":55.338999999999999},"last":1.8959999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:56:04 2023","interfaces":[{"interface":"back","average":{"1min":2.379,"5min":2.2240000000000002,"15min":1.7410000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28000000000000003},"max":{"1min":10.411,"5min":11.723000000000001,"15min":12.361000000000001},"last":4.9340000000000002},{"interface":"front","average":{"1min":1.6739999999999999,"5min":1.9790000000000001,"15min":1.8540000000000001},"min":{"1min":0.39100000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":9.1969999999999992,"5min":11.132999999999999,"15min":13.891999999999999},"last":1.9259999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:56:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6480000000000001,"5min":2.0859999999999999,"15min":2.093},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.31},"max":{"1min":18.722000000000001,"5min":18.722000000000001,"15min":51.744},"last":1.95},{"interface":"front","average":{"1min":2.294,"5min":2.407,"15min":2.101},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.31900000000000001},"max":{"1min":9.0779999999999994,"5min":13.337,"15min":53.399000000000001},"last":8.3840000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019155,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1263720,"kb_used_data":673952,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":92481432,"statfs":{"total":95995035648,"available":94700986368,"internally_reserved":0,"allocated":690126848,"data_stored":688985098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:54 2023","interfaces":[{"interface":"back","average":{"1min":2.8079999999999998,"5min":3.8279999999999998,"15min":2.0430000000000001},"min":{"1min":0.23499999999999999,"5min":0.217,"15min":0.20999999999999999},"max":{"1min":12.332000000000001,"5min":213.88999999999999,"15min":213.88999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.3330000000000002,"5min":4.3390000000000004,"15min":2.3319999999999999},"min":{"1min":0.24399999999999999,"5min":0.23300000000000001,"15min":0.217},"max":{"1min":8.9130000000000003,"5min":214.09800000000001,"15min":214.09800000000001},"last":0.58199999999999996}]},{"osd":1,"last update":"Mon Oct 9 10:55:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":3.9249999999999998,"15min":2.0699999999999998},"min":{"1min":0.23799999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":9.0099999999999998,"5min":206.50399999999999,"15min":206.50399999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.1480000000000001,"5min":4.1100000000000003,"15min":2.282},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":8.4870000000000001,"5min":206.529,"15min":206.529},"last":0.59699999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:56:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.9390000000000001,"15min":1.619},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.253},"max":{"1min":5.7089999999999996,"5min":15.446999999999999,"15min":15.446999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":2.141,"5min":2.29,"15min":1.851},"min":{"1min":0.316,"5min":0.29599999999999999,"15min":0.26300000000000001},"max":{"1min":6.5510000000000002,"5min":27.818000000000001,"15min":27.818000000000001},"last":0.98999999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:55:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":4.0410000000000004,"15min":2.3340000000000001},"min":{"1min":0.34599999999999997,"5min":0.29599999999999999,"15min":0.27400000000000002},"max":{"1min":6.0469999999999997,"5min":209.03700000000001,"15min":209.03700000000001},"last":2.266},{"interface":"front","average":{"1min":2.2149999999999999,"5min":4.4720000000000004,"15min":2.544},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.5919999999999996,"5min":208.80199999999999,"15min":208.80199999999999},"last":1.55}]},{"osd":5,"last update":"Mon Oct 9 10:55:57 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":1.7649999999999999,"15min":1.5629999999999999},"min":{"1min":0.253,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":4.1210000000000004,"5min":19.068999999999999,"15min":24.149999999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":1.4790000000000001,"15min":1.385},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.252},"max":{"1min":1.917,"5min":15.535,"15min":20.422000000000001},"last":0.48399999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:56:32 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":1.2070000000000001,"15min":1.242},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":3.0550000000000002,"5min":15.739000000000001,"15min":15.739000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.68700000000000006,"5min":1.484,"15min":1.4710000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.311},"max":{"1min":3.375,"5min":14.093999999999999,"15min":14.272},"last":0.623}]},{"osd":7,"last update":"Mon Oct 9 10:56:39 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.5149999999999999,"15min":1.3979999999999999},"min":{"1min":0.33400000000000002,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":4.7800000000000002,"5min":14.458,"15min":14.458},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.534,"15min":1.298},"min":{"1min":0.34499999999999997,"5min":0.255,"15min":0.255},"max":{"1min":4.4290000000000003,"5min":15.592000000000001,"15min":15.592000000000001},"last":0.54500000000000004}]}]},{"osd":5,"up_from":37,"seq":158913790223,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1624144,"kb_used_data":1026184,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":92121008,"statfs":{"total":95995035648,"available":94331912192,"internally_reserved":0,"allocated":1050812416,"data_stored":1049682832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:55:57 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":1.9419999999999999,"15min":1.4390000000000001},"min":{"1min":0.25600000000000001,"5min":0.215,"15min":0.215},"max":{"1min":11.683,"5min":16.449000000000002,"15min":16.449000000000002},"last":4.71},{"interface":"front","average":{"1min":1.897,"5min":1.9950000000000001,"15min":1.498},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.653,"5min":16.489000000000001,"15min":16.489000000000001},"last":4.7480000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:56:10 2023","interfaces":[{"interface":"back","average":{"1min":2.9689999999999999,"5min":2.2080000000000002,"15min":1.74},"min":{"1min":0.247,"5min":0.215,"15min":0.215},"max":{"1min":6.6950000000000003,"5min":8.2629999999999999,"15min":12.999000000000001},"last":3.609},{"interface":"front","average":{"1min":1.9510000000000001,"5min":1.903,"15min":1.468},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.24299999999999999},"max":{"1min":5.024,"5min":9.2699999999999996,"15min":12.109},"last":2.6560000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:56:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8380000000000001,"5min":2.105,"15min":1.7509999999999999},"min":{"1min":0.28299999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.7789999999999999,"5min":11.257999999999999,"15min":11.393000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":3.0259999999999998,"5min":2.306,"15min":1.569},"min":{"1min":0.28799999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.2569999999999997,"5min":15.087999999999999,"15min":15.087999999999999},"last":0.38300000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:55:57 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":2.754,"15min":1.962},"min":{"1min":0.34499999999999997,"5min":0.314,"15min":0.25700000000000001},"max":{"1min":6.6399999999999997,"5min":63.508000000000003,"15min":63.508000000000003},"last":8.7910000000000004},{"interface":"front","average":{"1min":2.5030000000000001,"5min":2.7810000000000001,"15min":2.0430000000000001},"min":{"1min":0.39300000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.1820000000000004,"5min":62.418999999999997,"15min":62.418999999999997},"last":0.69599999999999995}]},{"osd":4,"last update":"Mon Oct 9 10:56:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":1.081,"15min":1.385},"min":{"1min":0.28799999999999998,"5min":0.247,"15min":0.23200000000000001},"max":{"1min":1.6879999999999999,"5min":6.6120000000000001,"15min":123.241},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.1299999999999999,"15min":1.6020000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.24199999999999999},"max":{"1min":2.8730000000000002,"5min":4.5090000000000003,"15min":123.318},"last":0.35399999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:56:24 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.4239999999999999,"15min":1.347},"min":{"1min":0.30099999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.9580000000000002,"5min":10.762,"15min":11.098000000000001},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.63400000000000001,"5min":1.25,"15min":1.224},"min":{"1min":0.318,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.1360000000000001,"5min":9.0980000000000008,"15min":20.256},"last":0.78700000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:55:46 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":1.6240000000000001,"15min":1.2869999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":2.911,"5min":8.468,"15min":10.739000000000001},"last":0.372},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.3899999999999999,"15min":1.1319999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":2.2450000000000001,"5min":10.858000000000001,"15min":14.154},"last":0.51600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":469565440,"data_stored":469565440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":134152192,"data_stored":134152192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":157880320,"data_stored":157876248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":563412992,"data_stored":563412992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":668729344,"data_stored":668729344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1043337216,"data_stored":1043333144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":827981824,"data_stored":827981824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":776671232,"data_stored":776667160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:56:42.832 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:56:42.832 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 61 2023-10-09T10:56:42.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:42.896 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:42 smithi125 ceph-mon[195507]: pgmap v781: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 12725/53127 objects degraded (23.952%); 4095/53127 objects misplaced (7.708%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T10:56:42.896 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:42 smithi125 ceph-mon[200197]: pgmap v781: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 12725/53127 objects degraded (23.952%); 4095/53127 objects misplaced (7.708%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T10:56:42.940 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:43.049 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:43.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:43.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:43.155 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:42 smithi155 ceph-mon[184854]: pgmap v781: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 12725/53127 objects degraded (23.952%); 4095/53127 objects misplaced (7.708%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T10:56:43.242 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:43.311 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:56:43.332 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:43.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:43.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:43.364 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:56:43.408 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:56:43.413 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:56:43.442 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:43.468 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:56:43.499 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:43.509 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:56:43.541 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:56:43.572 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:56:43.594 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:43.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:43.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:43.638 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:56:43.657 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:56:43.658 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:56:43.658 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:56:43.658 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:56:43.658 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:56:43.658 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:56:43.658 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:56:43.658 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:56:43.675 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:43.710 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:43.743 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:43.771 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:43.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:43.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:43.790 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:43 smithi125 ceph-mon[195507]: from='client.55519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:56:43.790 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:43 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 12725/53127 objects degraded (23.952%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:43.790 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:43 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/841039620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:43.791 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:43 smithi125 ceph-mon[200197]: from='client.55519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:56:43.791 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:43 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 12725/53127 objects degraded (23.952%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:43.791 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:43 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/841039620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:43.875 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:43.909 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:43.933 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:43.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:43.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:44.037 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:44.063 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:43 smithi155 ceph-mon[184854]: from='client.55519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:56:44.063 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:43 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 12725/53127 objects degraded (23.952%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:44.063 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:43 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/841039620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:44.065 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:44.099 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:44.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:44.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:44.204 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:44.266 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:44.353 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:56:44.353 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:56:44.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:56:44.353 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:56:44.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.353 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95743 8.3 8:c55106e5:::benchmark_data_smithi125_329074_object95742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:56:44.353 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:45.760025+0000", 2023-10-09T10:56:44.353 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 118.58840156399999, 2023-10-09T10:56:44.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.35609172, 2023-10-09T10:56:44.354 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.354 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.354 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:44.354 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:44.354 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:44.354 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95743 2023-10-09T10:56:44.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.354 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.760025+0000", 2023-10-09T10:56:44.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.760025+0000", 2023-10-09T10:56:44.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.760023+0000", 2023-10-09T10:56:44.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:44.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.760052+0000", 2023-10-09T10:56:44.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8801000000000001e-05 2023-10-09T10:56:44.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.760066+0000", 2023-10-09T10:56:44.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3723999999999999e-05 2023-10-09T10:56:44.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.760075+0000", 2023-10-09T10:56:44.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.414e-06 2023-10-09T10:56:44.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.043017+0000", 2023-10-09T10:56:44.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.28294185300000002 2023-10-09T10:56:44.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.046876+0000", 2023-10-09T10:56:44.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038587830000000002 2023-10-09T10:56:44.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:44.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.052755+0000", 2023-10-09T10:56:44.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0058790969999999998 2023-10-09T10:56:44.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.060724+0000", 2023-10-09T10:56:44.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0079689440000000004 2023-10-09T10:56:44.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.061803+0000", 2023-10-09T10:56:44.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010790210000000001 2023-10-09T10:56:44.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:44.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.108203+0000", 2023-10-09T10:56:44.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.046400400000000001 2023-10-09T10:56:44.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.109669+0000", 2023-10-09T10:56:44.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014653159999999999 2023-10-09T10:56:44.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.116117+0000", 2023-10-09T10:56:44.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0064479610000000003 2023-10-09T10:56:44.360 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.360 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.360 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.360 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95750 8.13 8:cab0bd25:::benchmark_data_smithi125_329074_object95749:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:56:44.360 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:45.796219+0000", 2023-10-09T10:56:44.360 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 118.552207521, 2023-10-09T10:56:44.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.34625348099999997, 2023-10-09T10:56:44.361 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.361 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.361 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:44.361 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:44.361 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:44.361 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95750 2023-10-09T10:56:44.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.361 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.796219+0000", 2023-10-09T10:56:44.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.362 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.796219+0000", 2023-10-09T10:56:44.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.362 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.796217+0000", 2023-10-09T10:56:44.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:56:44.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.796248+0000", 2023-10-09T10:56:44.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0877999999999998e-05 2023-10-09T10:56:44.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.796263+0000", 2023-10-09T10:56:44.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5088e-05 2023-10-09T10:56:44.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.796271+0000", 2023-10-09T10:56:44.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2390000000000002e-06 2023-10-09T10:56:44.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.070667+0000", 2023-10-09T10:56:44.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.27439532799999999 2023-10-09T10:56:44.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.074852+0000", 2023-10-09T10:56:44.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041858870000000001 2023-10-09T10:56:44.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:44.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.081478+0000", 2023-10-09T10:56:44.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066251260000000003 2023-10-09T10:56:44.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.100468+0000", 2023-10-09T10:56:44.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018990145 2023-10-09T10:56:44.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.101474+0000", 2023-10-09T10:56:44.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001006018 2023-10-09T10:56:44.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:44.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.132307+0000", 2023-10-09T10:56:44.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.030833182000000001 2023-10-09T10:56:44.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.134406+0000", 2023-10-09T10:56:44.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020994569999999999 2023-10-09T10:56:44.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.142472+0000", 2023-10-09T10:56:44.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0080661329999999996 2023-10-09T10:56:44.367 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.367 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.367 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.367 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPush(8.13 293/289 [PushOp(8:cbe2b721:::benchmark_data_smithi125_329074_object29:head, version: 270'1, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:cbe2b721:::benchmark_data_smithi125_329074_object29:head@270'1, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:56:44.367 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:45.877250+0000", 2023-10-09T10:56:44.367 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 118.47117654100001, 2023-10-09T10:56:44.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.363915395, 2023-10-09T10:56:44.368 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.368 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:56:44.368 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877250+0000", 2023-10-09T10:56:44.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877250+0000", 2023-10-09T10:56:44.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877249+0000", 2023-10-09T10:56:44.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:44.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877303+0000", 2023-10-09T10:56:44.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3928999999999998e-05 2023-10-09T10:56:44.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877323+0000", 2023-10-09T10:56:44.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0276999999999999e-05 2023-10-09T10:56:44.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877339+0000", 2023-10-09T10:56:44.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6362999999999999e-05 2023-10-09T10:56:44.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.190367+0000", 2023-10-09T10:56:44.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3130274530000001 2023-10-09T10:56:44.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.190793+0000", 2023-10-09T10:56:44.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042582999999999998 2023-10-09T10:56:44.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.241165+0000", 2023-10-09T10:56:44.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.050372739 2023-10-09T10:56:44.371 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.371 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.372 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.372 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95836 8.3 8:c6d59e02:::benchmark_data_smithi125_329074_object95835:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:56:44.372 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:45.974376+0000", 2023-10-09T10:56:44.372 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 118.374050783, 2023-10-09T10:56:44.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.29878043500000001, 2023-10-09T10:56:44.372 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.372 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.372 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:44.372 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:44.372 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:44.373 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95836 2023-10-09T10:56:44.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.373 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.974376+0000", 2023-10-09T10:56:44.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.974376+0000", 2023-10-09T10:56:44.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.974374+0000", 2023-10-09T10:56:44.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:56:44.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.974399+0000", 2023-10-09T10:56:44.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5358e-05 2023-10-09T10:56:44.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.974410+0000", 2023-10-09T10:56:44.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0625000000000001e-05 2023-10-09T10:56:44.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.974416+0000", 2023-10-09T10:56:44.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5660000000000003e-06 2023-10-09T10:56:44.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.235361+0000", 2023-10-09T10:56:44.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.26094443099999998 2023-10-09T10:56:44.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.240275+0000", 2023-10-09T10:56:44.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049143030000000001 2023-10-09T10:56:44.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:44.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.247125+0000", 2023-10-09T10:56:44.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.006849997 2023-10-09T10:56:44.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.260446+0000", 2023-10-09T10:56:44.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013321319 2023-10-09T10:56:44.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.377 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.377 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.261732+0000", 2023-10-09T10:56:44.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012852300000000001 2023-10-09T10:56:44.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.377 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:44.377 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.262817+0000", 2023-10-09T10:56:44.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001085309 2023-10-09T10:56:44.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.264862+0000", 2023-10-09T10:56:44.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020456110000000001 2023-10-09T10:56:44.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.273156+0000", 2023-10-09T10:56:44.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0082937140000000006 2023-10-09T10:56:44.378 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.378 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.378 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.379 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95842 8.3 8:c3562596:::benchmark_data_smithi125_329074_object95841:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:56:44.379 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:45.992483+0000", 2023-10-09T10:56:44.379 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 118.355943463, 2023-10-09T10:56:44.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.34110989600000002, 2023-10-09T10:56:44.379 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.379 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.379 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:44.379 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:44.379 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:44.379 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95842 2023-10-09T10:56:44.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.379 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.992483+0000", 2023-10-09T10:56:44.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.992483+0000", 2023-10-09T10:56:44.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.992481+0000", 2023-10-09T10:56:44.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:44.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.992510+0000", 2023-10-09T10:56:44.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8849000000000001e-05 2023-10-09T10:56:44.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.992525+0000", 2023-10-09T10:56:44.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4632e-05 2023-10-09T10:56:44.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.992534+0000", 2023-10-09T10:56:44.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5889999999999992e-06 2023-10-09T10:56:44.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.273661+0000", 2023-10-09T10:56:44.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.281126449 2023-10-09T10:56:44.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.278225+0000", 2023-10-09T10:56:44.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045641459999999998 2023-10-09T10:56:44.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:44.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.288539+0000", 2023-10-09T10:56:44.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010314014999999999 2023-10-09T10:56:44.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.296771+0000", 2023-10-09T10:56:44.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0082322690000000004 2023-10-09T10:56:44.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.384 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.322322+0000", 2023-10-09T10:56:44.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.025550585000000001 2023-10-09T10:56:44.384 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.384 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.384 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:44.384 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.322748+0000", 2023-10-09T10:56:44.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042560800000000001 2023-10-09T10:56:44.384 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.384 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.384 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.384 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.325248+0000", 2023-10-09T10:56:44.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002500539 2023-10-09T10:56:44.384 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.333593+0000", 2023-10-09T10:56:44.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083448670000000006 2023-10-09T10:56:44.385 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.385 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.385 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.385 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95845 8.13 8:c92b0cb5:::benchmark_data_smithi125_329074_object95844:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:56:44.385 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.006226+0000", 2023-10-09T10:56:44.385 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 118.34220064199999, 2023-10-09T10:56:44.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.34395316799999998, 2023-10-09T10:56:44.386 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.386 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.386 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:44.386 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:44.386 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:44.386 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95845 2023-10-09T10:56:44.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.386 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.006226+0000", 2023-10-09T10:56:44.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.006226+0000", 2023-10-09T10:56:44.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.006224+0000", 2023-10-09T10:56:44.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:56:44.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.006246+0000", 2023-10-09T10:56:44.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1963999999999998e-05 2023-10-09T10:56:44.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.006260+0000", 2023-10-09T10:56:44.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3601000000000001e-05 2023-10-09T10:56:44.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.006270+0000", 2023-10-09T10:56:44.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0496000000000001e-05 2023-10-09T10:56:44.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.274272+0000", 2023-10-09T10:56:44.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.26800151500000002 2023-10-09T10:56:44.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.279959+0000", 2023-10-09T10:56:44.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.005687531 2023-10-09T10:56:44.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:44.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.297659+0000", 2023-10-09T10:56:44.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.017699719999999999 2023-10-09T10:56:44.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.320528+0000", 2023-10-09T10:56:44.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.022869356 2023-10-09T10:56:44.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.322039+0000", 2023-10-09T10:56:44.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00151096 2023-10-09T10:56:44.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:44.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.334688+0000", 2023-10-09T10:56:44.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012648609 2023-10-09T10:56:44.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.337196+0000", 2023-10-09T10:56:44.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025082479999999998 2023-10-09T10:56:44.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.392 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.350179+0000", 2023-10-09T10:56:44.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012982852 2023-10-09T10:56:44.392 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.392 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.392 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.392 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.392 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95854 8.3 8:c64ea022:::benchmark_data_smithi125_329074_object95853:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:56:44.392 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.037001+0000", 2023-10-09T10:56:44.392 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 118.31142521699999, 2023-10-09T10:56:44.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.36041259799999997, 2023-10-09T10:56:44.392 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.393 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.393 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:44.393 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:44.393 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:44.393 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95854 2023-10-09T10:56:44.393 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.393 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.393 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.393 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.037001+0000", 2023-10-09T10:56:44.393 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.393 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.037001+0000", 2023-10-09T10:56:44.394 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.394 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.394 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.036999+0000", 2023-10-09T10:56:44.394 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:56:44.394 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.394 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.037026+0000", 2023-10-09T10:56:44.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7047e-05 2023-10-09T10:56:44.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.395 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.395 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.037041+0000", 2023-10-09T10:56:44.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4082e-05 2023-10-09T10:56:44.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.395 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.395 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.037050+0000", 2023-10-09T10:56:44.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6879999999999994e-06 2023-10-09T10:56:44.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.297294+0000", 2023-10-09T10:56:44.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.26024395700000003 2023-10-09T10:56:44.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.302697+0000", 2023-10-09T10:56:44.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0054025640000000003 2023-10-09T10:56:44.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.397 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:44.397 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.312000+0000", 2023-10-09T10:56:44.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0093036569999999999 2023-10-09T10:56:44.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.397 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.397 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.342728+0000", 2023-10-09T10:56:44.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.030727616999999999 2023-10-09T10:56:44.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.398 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.398 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.344397+0000", 2023-10-09T10:56:44.398 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001668957 2023-10-09T10:56:44.398 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.398 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:44.398 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.388186+0000", 2023-10-09T10:56:44.398 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.043788541 2023-10-09T10:56:44.398 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.398 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.390218+0000", 2023-10-09T10:56:44.399 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020322959999999998 2023-10-09T10:56:44.399 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.399 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.399 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.397414+0000", 2023-10-09T10:56:44.399 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0071960519999999997 2023-10-09T10:56:44.399 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.399 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.399 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.399 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.399 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.399 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95858 8.3 8:c3410bce:::benchmark_data_smithi125_329074_object95857:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:56:44.400 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.044670+0000", 2023-10-09T10:56:44.400 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 118.303756994, 2023-10-09T10:56:44.400 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.36581522900000002, 2023-10-09T10:56:44.400 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.400 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.400 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:44.400 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:44.400 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:44.400 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95858 2023-10-09T10:56:44.400 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.400 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.400 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.400 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.401 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.044670+0000", 2023-10-09T10:56:44.401 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.401 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.401 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.401 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.401 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.044670+0000", 2023-10-09T10:56:44.401 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.401 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.401 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.401 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.401 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.044668+0000", 2023-10-09T10:56:44.402 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:56:44.402 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.402 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.402 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.402 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.044692+0000", 2023-10-09T10:56:44.402 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3935999999999999e-05 2023-10-09T10:56:44.402 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.402 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.402 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.402 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.044705+0000", 2023-10-09T10:56:44.402 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2911000000000001e-05 2023-10-09T10:56:44.402 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.403 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.403 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.403 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.044715+0000", 2023-10-09T10:56:44.403 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0413e-05 2023-10-09T10:56:44.403 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.403 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.403 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.403 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.350771+0000", 2023-10-09T10:56:44.403 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.30605545499999998 2023-10-09T10:56:44.403 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.403 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.403 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.404 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.355300+0000", 2023-10-09T10:56:44.404 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045296310000000001 2023-10-09T10:56:44.404 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.404 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.404 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:44.404 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.366313+0000", 2023-10-09T10:56:44.404 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011012921 2023-10-09T10:56:44.404 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.404 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.404 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.404 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.398739+0000", 2023-10-09T10:56:44.404 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.032425871000000002 2023-10-09T10:56:44.404 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.405 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.405 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.405 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.400018+0000", 2023-10-09T10:56:44.405 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001279213 2023-10-09T10:56:44.405 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.405 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.405 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:44.405 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.400451+0000", 2023-10-09T10:56:44.405 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00043300000000000001 2023-10-09T10:56:44.405 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.405 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.405 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.406 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.402357+0000", 2023-10-09T10:56:44.406 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001905885 2023-10-09T10:56:44.406 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.406 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.406 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.406 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.410485+0000", 2023-10-09T10:56:44.406 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0081277120000000005 2023-10-09T10:56:44.406 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.406 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.406 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.406 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.406 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.407 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95871 8.3 8:c48d05f2:::benchmark_data_smithi125_329074_object95870:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:56:44.407 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.078356+0000", 2023-10-09T10:56:44.407 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 118.27007043499999, 2023-10-09T10:56:44.407 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.35194249700000002, 2023-10-09T10:56:44.407 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.407 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.407 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:44.407 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:44.407 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:44.407 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95871 2023-10-09T10:56:44.407 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.407 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.408 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.408 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.408 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.078356+0000", 2023-10-09T10:56:44.408 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.408 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.408 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.408 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.408 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.078356+0000", 2023-10-09T10:56:44.408 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.408 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.408 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.408 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.408 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.078354+0000", 2023-10-09T10:56:44.409 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:56:44.409 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.409 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.409 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.409 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.078379+0000", 2023-10-09T10:56:44.409 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4553e-05 2023-10-09T10:56:44.409 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.409 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.409 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.409 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.078394+0000", 2023-10-09T10:56:44.409 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5007999999999999e-05 2023-10-09T10:56:44.410 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.410 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.410 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.410 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.078401+0000", 2023-10-09T10:56:44.410 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7610000000000007e-06 2023-10-09T10:56:44.410 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.410 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.410 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.410 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.371028+0000", 2023-10-09T10:56:44.410 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.29262697300000001 2023-10-09T10:56:44.410 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.410 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.411 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.411 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.374954+0000", 2023-10-09T10:56:44.411 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039256650000000001 2023-10-09T10:56:44.411 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.411 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.411 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:44.411 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.382362+0000", 2023-10-09T10:56:44.411 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074084119999999996 2023-10-09T10:56:44.411 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.411 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.411 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:44.411 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.410967+0000", 2023-10-09T10:56:44.411 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.028604319999999999 2023-10-09T10:56:44.412 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.412 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.412 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.412 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.412353+0000", 2023-10-09T10:56:44.412 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013863219999999999 2023-10-09T10:56:44.412 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.412 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.412 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.412 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.416616+0000", 2023-10-09T10:56:44.412 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042625199999999997 2023-10-09T10:56:44.412 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.412 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.412 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.413 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.418988+0000", 2023-10-09T10:56:44.413 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023724190000000002 2023-10-09T10:56:44.413 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.413 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.413 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.413 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.430299+0000", 2023-10-09T10:56:44.413 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011310534000000001 2023-10-09T10:56:44.413 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.413 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.413 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.413 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.414 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.414 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95875 8.3 8:c45f4548:::benchmark_data_smithi125_329074_object95874:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:56:44.414 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.104335+0000", 2023-10-09T10:56:44.414 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 118.24409200700001, 2023-10-09T10:56:44.414 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.39137140100000001, 2023-10-09T10:56:44.414 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.414 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.414 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:44.414 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:44.414 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:44.414 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95875 2023-10-09T10:56:44.414 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.415 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.415 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.104335+0000", 2023-10-09T10:56:44.415 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.415 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.415 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.104335+0000", 2023-10-09T10:56:44.415 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.415 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.104333+0000", 2023-10-09T10:56:44.416 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:44.416 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.416 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.416 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.104355+0000", 2023-10-09T10:56:44.416 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1704000000000001e-05 2023-10-09T10:56:44.416 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.416 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.416 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.104365+0000", 2023-10-09T10:56:44.416 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0842000000000001e-05 2023-10-09T10:56:44.416 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.417 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.417 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.417 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.104372+0000", 2023-10-09T10:56:44.417 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.139e-06 2023-10-09T10:56:44.417 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.417 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.417 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.417 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.430995+0000", 2023-10-09T10:56:44.417 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32662346599999997 2023-10-09T10:56:44.417 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.417 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.417 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.418 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.435804+0000", 2023-10-09T10:56:44.418 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004808545 2023-10-09T10:56:44.418 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.418 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.418 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:44.418 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.443158+0000", 2023-10-09T10:56:44.418 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0073543310000000004 2023-10-09T10:56:44.418 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.418 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.418 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.418 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.475849+0000", 2023-10-09T10:56:44.418 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.032691027999999997 2023-10-09T10:56:44.418 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.419 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.419 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:44.419 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.476399+0000", 2023-10-09T10:56:44.419 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00054972300000000001 2023-10-09T10:56:44.419 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.419 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.419 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.419 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.477978+0000", 2023-10-09T10:56:44.419 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015797599999999999 2023-10-09T10:56:44.419 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.419 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.419 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.419 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.484184+0000", 2023-10-09T10:56:44.420 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062053780000000001 2023-10-09T10:56:44.420 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.420 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.420 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.420 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.495706+0000", 2023-10-09T10:56:44.420 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011522045 2023-10-09T10:56:44.420 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.420 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.420 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.420 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.420 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.420 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95895 8.3 8:c64bfacc:::benchmark_data_smithi125_329074_object95894:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:56:44.421 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.161613+0000", 2023-10-09T10:56:44.421 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 118.186813986, 2023-10-09T10:56:44.421 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.35546433399999999, 2023-10-09T10:56:44.421 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.421 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.421 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:44.421 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:44.421 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:44.421 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95895 2023-10-09T10:56:44.421 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.421 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.421 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.422 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.422 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.161613+0000", 2023-10-09T10:56:44.422 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.422 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.422 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.422 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.422 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.161613+0000", 2023-10-09T10:56:44.422 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.422 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.422 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.422 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.422 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.161610+0000", 2023-10-09T10:56:44.423 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T10:56:44.423 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.423 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.423 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.423 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.161639+0000", 2023-10-09T10:56:44.423 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8572999999999999e-05 2023-10-09T10:56:44.423 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.423 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.423 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.423 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.161655+0000", 2023-10-09T10:56:44.423 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6172999999999999e-05 2023-10-09T10:56:44.423 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.424 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.424 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.424 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.161665+0000", 2023-10-09T10:56:44.424 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0304e-05 2023-10-09T10:56:44.424 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.424 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.424 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.424 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.449268+0000", 2023-10-09T10:56:44.424 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.28760287600000001 2023-10-09T10:56:44.424 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.424 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.424 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.424 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.453531+0000", 2023-10-09T10:56:44.425 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042631939999999997 2023-10-09T10:56:44.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:44.425 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.461412+0000", 2023-10-09T10:56:44.425 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0078809580000000004 2023-10-09T10:56:44.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:44.425 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.476830+0000", 2023-10-09T10:56:44.425 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015417854999999999 2023-10-09T10:56:44.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.426 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.497838+0000", 2023-10-09T10:56:44.426 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.021007978 2023-10-09T10:56:44.426 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.426 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.426 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.505361+0000", 2023-10-09T10:56:44.426 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0075223690000000001 2023-10-09T10:56:44.426 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.426 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.426 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.507459+0000", 2023-10-09T10:56:44.427 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020982729999999999 2023-10-09T10:56:44.427 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.427 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.427 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.427 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.517077+0000", 2023-10-09T10:56:44.427 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0096180799999999993 2023-10-09T10:56:44.427 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.427 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.427 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.427 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.427 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.427 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95902 8.13 8:c9dc5fe7:::benchmark_data_smithi125_329074_object95901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:56:44.428 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.181255+0000", 2023-10-09T10:56:44.428 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 118.167171313, 2023-10-09T10:56:44.428 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.46745452100000001, 2023-10-09T10:56:44.428 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.428 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.428 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:44.428 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:44.428 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:44.428 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95902 2023-10-09T10:56:44.428 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.428 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.428 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.429 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.429 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181255+0000", 2023-10-09T10:56:44.429 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.429 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.429 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.429 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.429 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181255+0000", 2023-10-09T10:56:44.429 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.429 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.429 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.429 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.429 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181253+0000", 2023-10-09T10:56:44.429 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:56:44.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.430 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.430 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.430 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181286+0000", 2023-10-09T10:56:44.430 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2928000000000001e-05 2023-10-09T10:56:44.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.430 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.430 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.430 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181306+0000", 2023-10-09T10:56:44.430 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9514999999999999e-05 2023-10-09T10:56:44.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.430 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181317+0000", 2023-10-09T10:56:44.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1070000000000001e-05 2023-10-09T10:56:44.431 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.503475+0000", 2023-10-09T10:56:44.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32215859400000002 2023-10-09T10:56:44.431 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.513376+0000", 2023-10-09T10:56:44.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0099005069999999994 2023-10-09T10:56:44.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.432 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:44.432 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.528220+0000", 2023-10-09T10:56:44.432 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014844346 2023-10-09T10:56:44.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.432 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.432 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.543824+0000", 2023-10-09T10:56:44.432 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015603361 2023-10-09T10:56:44.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.547461+0000", 2023-10-09T10:56:44.433 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036375349999999999 2023-10-09T10:56:44.433 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:44.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.633195+0000", 2023-10-09T10:56:44.433 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.085734224999999997 2023-10-09T10:56:44.433 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.636727+0000", 2023-10-09T10:56:44.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035320870000000002 2023-10-09T10:56:44.434 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.434 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.434 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.434 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.648710+0000", 2023-10-09T10:56:44.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011982244 2023-10-09T10:56:44.434 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.434 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.434 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.434 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.434 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.434 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95936 8.3 8:c568270b:::benchmark_data_smithi125_329074_object95935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:56:44.435 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.239203+0000", 2023-10-09T10:56:44.435 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 118.10922367800001, 2023-10-09T10:56:44.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.38808216600000001, 2023-10-09T10:56:44.435 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.435 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.435 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:44.435 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:44.435 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:44.435 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95936 2023-10-09T10:56:44.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.435 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.239203+0000", 2023-10-09T10:56:44.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.239203+0000", 2023-10-09T10:56:44.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.239201+0000", 2023-10-09T10:56:44.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:44.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.437 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.437 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.239231+0000", 2023-10-09T10:56:44.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9762999999999999e-05 2023-10-09T10:56:44.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.437 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.437 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.239244+0000", 2023-10-09T10:56:44.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3473e-05 2023-10-09T10:56:44.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.438 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.438 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.239254+0000", 2023-10-09T10:56:44.438 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0829999999999993e-06 2023-10-09T10:56:44.438 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.438 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.438 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.438 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.517531+0000", 2023-10-09T10:56:44.438 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.27827761899999998 2023-10-09T10:56:44.438 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.438 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.438 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.438 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.522622+0000", 2023-10-09T10:56:44.439 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0050906229999999998 2023-10-09T10:56:44.439 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.439 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.439 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:44.439 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.530722+0000", 2023-10-09T10:56:44.439 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0081005079999999993 2023-10-09T10:56:44.439 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.439 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.439 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.439 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.544008+0000", 2023-10-09T10:56:44.439 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013285732999999999 2023-10-09T10:56:44.439 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.440 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.440 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.440 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.545127+0000", 2023-10-09T10:56:44.440 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011191319999999999 2023-10-09T10:56:44.440 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.440 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.440 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:44.440 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.615780+0000", 2023-10-09T10:56:44.440 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.070652551999999993 2023-10-09T10:56:44.440 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.440 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.440 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.440 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.618009+0000", 2023-10-09T10:56:44.441 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022288569999999999 2023-10-09T10:56:44.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.441 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.627285+0000", 2023-10-09T10:56:44.441 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0092764270000000003 2023-10-09T10:56:44.441 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.441 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.441 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.441 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95942 8.3 8:c1416b96:::benchmark_data_smithi125_329074_object95941:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:56:44.442 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.246107+0000", 2023-10-09T10:56:44.442 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 118.102319495, 2023-10-09T10:56:44.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.41670001299999998, 2023-10-09T10:56:44.442 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.442 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.442 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:44.442 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:44.442 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:44.442 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95942 2023-10-09T10:56:44.442 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.442 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.246107+0000", 2023-10-09T10:56:44.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.246107+0000", 2023-10-09T10:56:44.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.246106+0000", 2023-10-09T10:56:44.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:44.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.444 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.246143+0000", 2023-10-09T10:56:44.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7042000000000002e-05 2023-10-09T10:56:44.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.444 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.246158+0000", 2023-10-09T10:56:44.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5378000000000001e-05 2023-10-09T10:56:44.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.445 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.445 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.246168+0000", 2023-10-09T10:56:44.445 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5440000000000004e-06 2023-10-09T10:56:44.445 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.445 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.445 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.445 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.549808+0000", 2023-10-09T10:56:44.445 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.30364068399999999 2023-10-09T10:56:44.445 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.445 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.445 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.445 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.555779+0000", 2023-10-09T10:56:44.445 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0059703960000000002 2023-10-09T10:56:44.445 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.446 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:44.446 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.562896+0000", 2023-10-09T10:56:44.446 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0071178830000000002 2023-10-09T10:56:44.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.446 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.446 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.571242+0000", 2023-10-09T10:56:44.446 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083458920000000006 2023-10-09T10:56:44.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.447 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.447 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.572716+0000", 2023-10-09T10:56:44.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014732759999999999 2023-10-09T10:56:44.447 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.447 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.447 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:44.447 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.653417+0000", 2023-10-09T10:56:44.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.080700999999999995 2023-10-09T10:56:44.447 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.447 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.655103+0000", 2023-10-09T10:56:44.448 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001686173 2023-10-09T10:56:44.448 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.662807+0000", 2023-10-09T10:56:44.448 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0077042059999999999 2023-10-09T10:56:44.448 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.448 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.448 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.448 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.449 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95991 8.3 8:c7234827:::benchmark_data_smithi125_329074_object95990:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:56:44.449 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.358750+0000", 2023-10-09T10:56:44.449 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 117.989676077, 2023-10-09T10:56:44.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.31338819600000001, 2023-10-09T10:56:44.449 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.449 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.449 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:44.449 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:44.449 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:44.449 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95991 2023-10-09T10:56:44.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.449 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.358750+0000", 2023-10-09T10:56:44.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.358750+0000", 2023-10-09T10:56:44.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.358749+0000", 2023-10-09T10:56:44.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:44.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.358772+0000", 2023-10-09T10:56:44.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3393000000000001e-05 2023-10-09T10:56:44.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.358785+0000", 2023-10-09T10:56:44.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2336e-05 2023-10-09T10:56:44.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.358794+0000", 2023-10-09T10:56:44.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8219999999999996e-06 2023-10-09T10:56:44.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.573131+0000", 2023-10-09T10:56:44.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.214337478 2023-10-09T10:56:44.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.579816+0000", 2023-10-09T10:56:44.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066854109999999996 2023-10-09T10:56:44.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:44.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.588464+0000", 2023-10-09T10:56:44.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0086475609999999998 2023-10-09T10:56:44.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.628692+0000", 2023-10-09T10:56:44.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.040228208000000001 2023-10-09T10:56:44.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.629893+0000", 2023-10-09T10:56:44.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012005519999999999 2023-10-09T10:56:44.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:44.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.663115+0000", 2023-10-09T10:56:44.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.033222139999999997 2023-10-09T10:56:44.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.664681+0000", 2023-10-09T10:56:44.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015656750000000001 2023-10-09T10:56:44.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.672139+0000", 2023-10-09T10:56:44.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074580239999999997 2023-10-09T10:56:44.455 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.455 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.455 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.455 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:96005 8.3 8:c030217b:::benchmark_data_smithi125_329074_object96004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:56:44.455 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.386683+0000", 2023-10-09T10:56:44.455 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 117.961743266, 2023-10-09T10:56:44.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.29650773000000002, 2023-10-09T10:56:44.456 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.456 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.456 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:44.456 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:44.456 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:44.456 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 96005 2023-10-09T10:56:44.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.456 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.386683+0000", 2023-10-09T10:56:44.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.386683+0000", 2023-10-09T10:56:44.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.386682+0000", 2023-10-09T10:56:44.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:44.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.386704+0000", 2023-10-09T10:56:44.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2478000000000001e-05 2023-10-09T10:56:44.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.386713+0000", 2023-10-09T10:56:44.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0580000000000004e-06 2023-10-09T10:56:44.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.386719+0000", 2023-10-09T10:56:44.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.784e-06 2023-10-09T10:56:44.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.595142+0000", 2023-10-09T10:56:44.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20842272100000001 2023-10-09T10:56:44.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.602128+0000", 2023-10-09T10:56:44.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0069863640000000001 2023-10-09T10:56:44.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:56:44.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.609782+0000", 2023-10-09T10:56:44.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076536219999999997 2023-10-09T10:56:44.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.630874+0000", 2023-10-09T10:56:44.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.021091682 2023-10-09T10:56:44.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:44.461 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.632323+0000", 2023-10-09T10:56:44.461 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001449868 2023-10-09T10:56:44.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:44.461 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.672484+0000", 2023-10-09T10:56:44.461 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.040160985000000003 2023-10-09T10:56:44.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.461 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.674513+0000", 2023-10-09T10:56:44.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020288659999999998 2023-10-09T10:56:44.462 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.462 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.462 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.683191+0000", 2023-10-09T10:56:44.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0086776709999999996 2023-10-09T10:56:44.462 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.462 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.462 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.462 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.462 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.463 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:157109 8.d e312/306 8:b7dc21c9:::benchmark_data_smithi125_329074_object60455:head v 312'9714, mlcod=0'0)", 2023-10-09T10:56:44.463 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:12.725322+0000", 2023-10-09T10:56:44.463 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 31.623104331, 2023-10-09T10:56:44.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.054966636999999999, 2023-10-09T10:56:44.463 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.463 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.463 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.463 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.725322+0000", 2023-10-09T10:56:44.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.464 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.464 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.464 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.725322+0000", 2023-10-09T10:56:44.464 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.464 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.464 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.464 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.464 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.725321+0000", 2023-10-09T10:56:44.464 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:56:44.464 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.464 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.464 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.464 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.725327+0000", 2023-10-09T10:56:44.465 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9499999999999998e-06 2023-10-09T10:56:44.465 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.465 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.465 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.725341+0000", 2023-10-09T10:56:44.465 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3392e-05 2023-10-09T10:56:44.465 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.465 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.465 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.725348+0000", 2023-10-09T10:56:44.465 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3490000000000003e-06 2023-10-09T10:56:44.465 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.725391+0000", 2023-10-09T10:56:44.466 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3065000000000002e-05 2023-10-09T10:56:44.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.466 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.725416+0000", 2023-10-09T10:56:44.466 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4464000000000002e-05 2023-10-09T10:56:44.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.466 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.780245+0000", 2023-10-09T10:56:44.467 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.054829180999999998 2023-10-09T10:56:44.467 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.467 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.467 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.780289+0000", 2023-10-09T10:56:44.467 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4035000000000001e-05 2023-10-09T10:56:44.467 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.467 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.467 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.467 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.467 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:157116 8.d e312/306 8:ba4a95d0:::benchmark_data_smithi125_329074_object60462:head v 312'9715, mlcod=0'0)", 2023-10-09T10:56:44.468 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:12.732201+0000", 2023-10-09T10:56:44.468 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 31.616225476, 2023-10-09T10:56:44.468 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.053835692999999997, 2023-10-09T10:56:44.468 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.468 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.468 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.468 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.468 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.468 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.732201+0000", 2023-10-09T10:56:44.468 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.468 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.468 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.468 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.732201+0000", 2023-10-09T10:56:44.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.469 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.469 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.469 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.732200+0000", 2023-10-09T10:56:44.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:56:44.469 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.469 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.469 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.732214+0000", 2023-10-09T10:56:44.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3732e-05 2023-10-09T10:56:44.469 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.470 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.732230+0000", 2023-10-09T10:56:44.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5797000000000001e-05 2023-10-09T10:56:44.470 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.470 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.732237+0000", 2023-10-09T10:56:44.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7440000000000007e-06 2023-10-09T10:56:44.470 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.733002+0000", 2023-10-09T10:56:44.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00076495999999999997 2023-10-09T10:56:44.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.471 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.471 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.733031+0000", 2023-10-09T10:56:44.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9068999999999999e-05 2023-10-09T10:56:44.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.471 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.471 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.786000+0000", 2023-10-09T10:56:44.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.052968952 2023-10-09T10:56:44.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.472 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.472 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.472 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.786037+0000", 2023-10-09T10:56:44.472 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6334000000000001e-05 2023-10-09T10:56:44.472 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.472 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.472 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.472 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.472 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.472 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:157121 8.d e312/306 8:b9f86f9e:::benchmark_data_smithi125_329074_object60467:head v 312'9716, mlcod=0'0)", 2023-10-09T10:56:44.472 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:12.737321+0000", 2023-10-09T10:56:44.472 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 31.611105486, 2023-10-09T10:56:44.472 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.048739092999999997, 2023-10-09T10:56:44.473 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.473 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.473 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.473 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.473 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.473 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.737321+0000", 2023-10-09T10:56:44.473 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.473 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.473 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.473 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.473 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.737321+0000", 2023-10-09T10:56:44.473 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.473 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.474 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.474 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.474 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.737320+0000", 2023-10-09T10:56:44.474 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:56:44.474 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.474 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.474 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.474 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.737328+0000", 2023-10-09T10:56:44.474 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7349999999999996e-06 2023-10-09T10:56:44.474 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.474 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.475 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.475 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.737344+0000", 2023-10-09T10:56:44.475 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6816999999999998e-05 2023-10-09T10:56:44.475 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.475 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.475 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.475 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.737355+0000", 2023-10-09T10:56:44.475 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0097e-05 2023-10-09T10:56:44.475 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.475 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.475 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.475 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.746040+0000", 2023-10-09T10:56:44.476 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0086859020000000006 2023-10-09T10:56:44.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.476 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.476 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.476 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.746071+0000", 2023-10-09T10:56:44.476 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0301999999999999e-05 2023-10-09T10:56:44.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.476 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.476 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.476 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.786039+0000", 2023-10-09T10:56:44.476 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.039968615999999998 2023-10-09T10:56:44.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.477 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.477 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.477 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:12.786060+0000", 2023-10-09T10:56:44.477 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0784e-05 2023-10-09T10:56:44.477 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.477 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.477 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.477 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.477 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.477 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.35433.0:173150 8.e e315/282 8:7d0ef7d9:::benchmark_data_smithi125_329074_object76496:head v 315'10892, mlcod=0'0)", 2023-10-09T10:56:44.477 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:35.848043+0000", 2023-10-09T10:56:44.477 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 8.5003832310000007, 2023-10-09T10:56:44.477 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.041088313000000001, 2023-10-09T10:56:44.478 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:44.478 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:44.478 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:44.478 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.478 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:44.478 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:35.848043+0000", 2023-10-09T10:56:44.478 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.478 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.478 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.478 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:44.478 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:35.848043+0000", 2023-10-09T10:56:44.478 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:44.479 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.479 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.479 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:44.479 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:35.848043+0000", 2023-10-09T10:56:44.479 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967296 2023-10-09T10:56:44.479 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.479 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.479 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:44.479 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:35.848046+0000", 2023-10-09T10:56:44.479 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1149999999999998e-06 2023-10-09T10:56:44.479 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.479 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.479 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:44.480 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:35.848049+0000", 2023-10-09T10:56:44.480 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.117e-06 2023-10-09T10:56:44.480 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.480 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.480 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:44.480 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:35.848052+0000", 2023-10-09T10:56:44.480 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4899999999999999e-06 2023-10-09T10:56:44.480 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.480 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.480 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:44.480 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:35.848174+0000", 2023-10-09T10:56:44.480 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012193600000000001 2023-10-09T10:56:44.480 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.481 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.481 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:44.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:35.848202+0000", 2023-10-09T10:56:44.481 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8586999999999999e-05 2023-10-09T10:56:44.481 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.481 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.481 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:44.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:35.889088+0000", 2023-10-09T10:56:44.481 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.040885489999999997 2023-10-09T10:56:44.481 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:44.481 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:44.481 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:56:44.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:35.889132+0000", 2023-10-09T10:56:44.482 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3767999999999998e-05 2023-10-09T10:56:44.482 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.482 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.482 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.482 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:44.482 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:44.482 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:56:44.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:44.483 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:44.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:44.584 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:44.628 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:44.653 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:44.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:44.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:44.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:44.789 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:44.817 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:44.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:44.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:44.907 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:44.922 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:44.948 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:44 smithi155 ceph-mon[184854]: pgmap v782: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 12360/49314 objects degraded (25.064%); 3861/49314 objects misplaced (7.829%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T10:56:44.951 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:44.979 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:44.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:44.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:44.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:56:44.997 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:44 smithi125 ceph-mon[200197]: pgmap v782: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 12360/49314 objects degraded (25.064%); 3861/49314 objects misplaced (7.829%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T10:56:44.998 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:44 smithi125 ceph-mon[195507]: pgmap v782: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 12360/49314 objects degraded (25.064%); 3861/49314 objects misplaced (7.829%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T10:56:45.085 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:45.116 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:45.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:45.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:45.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:45.242 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:45.314 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:45.553 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:45.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:45.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:45.656 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:45.725 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:45.805 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:45.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:45.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:45.809 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:56:45.910 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:45.983 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:46.025 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:46.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:46.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:46.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:56:46.099 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:46.128 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:56:46.133 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:46.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:56:46.163 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:46.189 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:46.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:46.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:46.293 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:46.325 INFO:teuthology.orchestra.run.smithi125.stderr:Error EINVAL: specified pg_num 61 < pg_num_max 32 2023-10-09T10:56:46.374 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:46.732 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:46.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:46.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:46.838 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:46.880 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:46.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:46.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:46.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:47.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:46 smithi125 ceph-mon[195507]: pgmap v783: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 12360/49314 objects degraded (25.064%); 3861/49314 objects misplaced (7.829%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T10:56:47.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:46 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2790375726' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T10:56:47.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:46 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T10:56:47.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:46 smithi125 ceph-mon[200197]: pgmap v783: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 12360/49314 objects degraded (25.064%); 3861/49314 objects misplaced (7.829%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T10:56:47.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:46 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2790375726' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T10:56:47.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:46 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T10:56:47.090 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:47.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:46 smithi155 ceph-mon[184854]: pgmap v783: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 12360/49314 objects degraded (25.064%); 3861/49314 objects misplaced (7.829%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T10:56:47.152 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:46 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2790375726' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T10:56:47.152 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:46 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T10:56:47.154 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:47.176 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:47.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:47.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:47.281 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:47.349 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:47.406 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:47.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:47.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:47.508 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:47.570 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:47.605 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:47.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:47.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:47.712 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:47.743 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:47.865 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:47.866 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:56:47.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:47.868 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/563151349' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:47.868 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:47 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:47.868 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/563151349' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:47.868 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:47 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:47.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:47.970 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:48.006 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:48.030 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:48.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:48.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:48.135 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:48.173 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:47 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/563151349' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:48.174 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:47 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:56:48.177 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:48.207 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:48.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:48.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:48.314 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:48.372 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:49.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:48 smithi155 ceph-mon[184854]: pgmap v784: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 881 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 11938/42339 objects degraded (28.196%); 3647/42339 objects misplaced (8.614%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T10:56:49.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:48 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:56:49.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:48 smithi155 ceph-mon[184854]: failed to encode map e317 with expected crc 2023-10-09T10:56:49.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:48 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:56:49.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:48 smithi155 ceph-mon[184854]: osdmap e317: 8 total, 8 up, 8 in 2023-10-09T10:56:49.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:48 smithi155 ceph-mon[184854]: failed to encode map e317 with expected crc 2023-10-09T10:56:49.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:48 smithi155 ceph-mon[184854]: failed to encode map e317 with expected crc 2023-10-09T10:56:49.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:48 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 11938/42339 objects degraded (28.196%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:49.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[195507]: pgmap v784: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 881 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 11938/42339 objects degraded (28.196%); 3647/42339 objects misplaced (8.614%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T10:56:49.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:56:49.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[195507]: failed to encode map e317 with expected crc 2023-10-09T10:56:49.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:56:49.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[195507]: osdmap e317: 8 total, 8 up, 8 in 2023-10-09T10:56:49.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[195507]: failed to encode map e317 with expected crc 2023-10-09T10:56:49.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[195507]: failed to encode map e317 with expected crc 2023-10-09T10:56:49.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 11938/42339 objects degraded (28.196%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:49.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[200197]: pgmap v784: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 881 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 11938/42339 objects degraded (28.196%); 3647/42339 objects misplaced (8.614%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T10:56:49.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:56:49.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[200197]: failed to encode map e317 with expected crc 2023-10-09T10:56:49.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:56:49.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[200197]: osdmap e317: 8 total, 8 up, 8 in 2023-10-09T10:56:49.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[200197]: failed to encode map e317 with expected crc 2023-10-09T10:56:49.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[200197]: failed to encode map e317 with expected crc 2023-10-09T10:56:49.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:48 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 11938/42339 objects degraded (28.196%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:49.580 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:49.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:49.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:49.695 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:56:49.722 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:49.858 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:49 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:56:49.858 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:49 smithi125 ceph-mon[200197]: pgmap v786: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 816 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 11608/39210 objects degraded (29.605%); 3470/39210 objects misplaced (8.850%); 4.5 MiB/s, 74 objects/s recovering 2023-10-09T10:56:49.901 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T10:56:49.960 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:56:49.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:49.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:56:50.067 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:50.110 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:49 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:56:50.110 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:49 smithi125 ceph-mon[195507]: pgmap v786: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 816 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 11608/39210 objects degraded (29.605%); 3470/39210 objects misplaced (8.850%); 4.5 MiB/s, 74 objects/s recovering 2023-10-09T10:56:50.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:49 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:56:50.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:49 smithi155 ceph-mon[184854]: pgmap v786: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 816 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 11608/39210 objects degraded (29.605%); 3470/39210 objects misplaced (8.850%); 4.5 MiB/s, 74 objects/s recovering 2023-10-09T10:56:50.172 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:50.204 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:50.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:50.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:50.314 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:50.389 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:50.441 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:50.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:50.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:50.543 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:50.546 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:50.660 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:50.708 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:56:50.708 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:56:50.709 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.709 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:183467 8.b 8:d52bf616:::benchmark_data_smithi125_329074_object86813:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T10:56:50.709 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:50.697294+0000", 2023-10-09T10:56:50.709 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0046332420000000001, 2023-10-09T10:56:50.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046646810000000004, 2023-10-09T10:56:50.709 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:50.709 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:56:50.709 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:50.709 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:50.709 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:50.709 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183467 2023-10-09T10:56:50.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.710 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:50.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.710 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:50.710 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697294+0000", 2023-10-09T10:56:50.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.710 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:50.710 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697294+0000", 2023-10-09T10:56:50.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:50.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697292+0000", 2023-10-09T10:56:50.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:56:50.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:50.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697302+0000", 2023-10-09T10:56:50.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7580000000000004e-06 2023-10-09T10:56:50.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:50.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697323+0000", 2023-10-09T10:56:50.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1007000000000001e-05 2023-10-09T10:56:50.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:50.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697334+0000", 2023-10-09T10:56:50.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1583000000000001e-05 2023-10-09T10:56:50.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:50.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697712+0000", 2023-10-09T10:56:50.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00037786699999999999 2023-10-09T10:56:50.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:50.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697897+0000", 2023-10-09T10:56:50.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000185291 2023-10-09T10:56:50.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T10:56:50.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.698089+0000", 2023-10-09T10:56:50.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019158900000000001 2023-10-09T10:56:50.713 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.714 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:50.714 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.714 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.35433.0:183464 8.7 e317/295)", 2023-10-09T10:56:50.714 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:50.697410+0000", 2023-10-09T10:56:50.714 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0045165300000000004, 2023-10-09T10:56:50.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004619527, 2023-10-09T10:56:50.714 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:50.714 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:56:50.714 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:50.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:50.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697410+0000", 2023-10-09T10:56:50.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:50.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697410+0000", 2023-10-09T10:56:50.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:50.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697410+0000", 2023-10-09T10:56:50.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:50.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:50.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697413+0000", 2023-10-09T10:56:50.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6780000000000002e-06 2023-10-09T10:56:50.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:50.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697423+0000", 2023-10-09T10:56:50.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7210000000000006e-06 2023-10-09T10:56:50.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:50.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697429+0000", 2023-10-09T10:56:50.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5430000000000001e-06 2023-10-09T10:56:50.717 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.717 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:50.717 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.717 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.35433.0:183460 8.f e317/295)", 2023-10-09T10:56:50.718 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:50.697439+0000", 2023-10-09T10:56:50.718 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0044873830000000002, 2023-10-09T10:56:50.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046323639999999999, 2023-10-09T10:56:50.718 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:50.718 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:56:50.718 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:50.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.718 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:50.718 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697439+0000", 2023-10-09T10:56:50.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:50.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697439+0000", 2023-10-09T10:56:50.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:50.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697439+0000", 2023-10-09T10:56:50.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967296 2023-10-09T10:56:50.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:50.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697451+0000", 2023-10-09T10:56:50.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1445e-05 2023-10-09T10:56:50.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:50.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697456+0000", 2023-10-09T10:56:50.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4009999999999996e-06 2023-10-09T10:56:50.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:50.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697460+0000", 2023-10-09T10:56:50.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2479999999999998e-06 2023-10-09T10:56:50.721 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.721 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:50.721 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.721 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.35433.0:183460 8.f e317/295)", 2023-10-09T10:56:50.721 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:50.697493+0000", 2023-10-09T10:56:50.721 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.004434042, 2023-10-09T10:56:50.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046187789999999999, 2023-10-09T10:56:50.721 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:50.721 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:56:50.721 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:50.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:50.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697493+0000", 2023-10-09T10:56:50.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:50.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697493+0000", 2023-10-09T10:56:50.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:50.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697492+0000", 2023-10-09T10:56:50.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:50.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:50.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697495+0000", 2023-10-09T10:56:50.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9289999999999998e-06 2023-10-09T10:56:50.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:50.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697502+0000", 2023-10-09T10:56:50.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6510000000000004e-06 2023-10-09T10:56:50.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:50.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697506+0000", 2023-10-09T10:56:50.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4819999999999996e-06 2023-10-09T10:56:50.724 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.724 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:50.724 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.725 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:183451 8.b 8:d0373928:::benchmark_data_smithi125_329074_object86797:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T10:56:50.725 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:50.667560+0000", 2023-10-09T10:56:50.725 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.034366340000000002, 2023-10-09T10:56:50.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.034597160000000002, 2023-10-09T10:56:50.725 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:50.725 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:50.725 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:50.725 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:50.725 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:50.725 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183451 2023-10-09T10:56:50.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.726 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:50.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:50.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.667560+0000", 2023-10-09T10:56:50.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:50.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.667560+0000", 2023-10-09T10:56:50.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.727 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:50.727 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.667559+0000", 2023-10-09T10:56:50.727 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:56:50.727 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.727 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.727 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:50.727 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.667567+0000", 2023-10-09T10:56:50.727 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4449999999999996e-06 2023-10-09T10:56:50.727 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.727 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.727 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:50.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.667581+0000", 2023-10-09T10:56:50.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4066000000000001e-05 2023-10-09T10:56:50.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:50.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.667589+0000", 2023-10-09T10:56:50.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6000000000000001e-06 2023-10-09T10:56:50.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:50.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.668485+0000", 2023-10-09T10:56:50.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00089628899999999998 2023-10-09T10:56:50.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:50.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.668650+0000", 2023-10-09T10:56:50.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000164835 2023-10-09T10:56:50.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T10:56:50.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.668824+0000", 2023-10-09T10:56:50.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017392899999999999 2023-10-09T10:56:50.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:50.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.672846+0000", 2023-10-09T10:56:50.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040218270000000004 2023-10-09T10:56:50.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:50.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.691628+0000", 2023-10-09T10:56:50.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018782266999999998 2023-10-09T10:56:50.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:50.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.701203+0000", 2023-10-09T10:56:50.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0095746830000000005 2023-10-09T10:56:50.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:50.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.701262+0000", 2023-10-09T10:56:50.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9756999999999997e-05 2023-10-09T10:56:50.731 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.731 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:50.731 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.731 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:183452 8.18 8:1d82d4c5:::benchmark_data_smithi125_329074_object86798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T10:56:50.732 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:50.668434+0000", 2023-10-09T10:56:50.732 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.033492517999999999, 2023-10-09T10:56:50.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.033834197000000003, 2023-10-09T10:56:50.732 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:50.732 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:56:50.732 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:50.732 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:50.732 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:50.732 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183452 2023-10-09T10:56:50.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.732 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:50.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:50.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.668434+0000", 2023-10-09T10:56:50.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:50.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.668434+0000", 2023-10-09T10:56:50.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:50.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.668434+0000", 2023-10-09T10:56:50.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:50.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:50.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.668438+0000", 2023-10-09T10:56:50.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2270000000000004e-06 2023-10-09T10:56:50.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:50.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.668447+0000", 2023-10-09T10:56:50.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2390000000000008e-06 2023-10-09T10:56:50.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:50.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.668454+0000", 2023-10-09T10:56:50.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6649999999999997e-06 2023-10-09T10:56:50.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:50.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.671903+0000", 2023-10-09T10:56:50.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0034496040000000002 2023-10-09T10:56:50.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:50.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.672031+0000", 2023-10-09T10:56:50.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000127222 2023-10-09T10:56:50.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:56:50.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.672581+0000", 2023-10-09T10:56:50.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00055055100000000004 2023-10-09T10:56:50.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:50.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.679878+0000", 2023-10-09T10:56:50.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0072965670000000003 2023-10-09T10:56:50.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:50.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682125+0000", 2023-10-09T10:56:50.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022468929999999998 2023-10-09T10:56:50.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:56:50.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.701136+0000", 2023-10-09T10:56:50.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.019010958000000001 2023-10-09T10:56:50.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:56:50.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.701187+0000", 2023-10-09T10:56:50.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1514999999999997e-05 2023-10-09T10:56:50.738 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.738 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:50.738 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.738 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.35433.0:183460 8.f e317/295)", 2023-10-09T10:56:50.738 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:50.698335+0000", 2023-10-09T10:56:50.738 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.003592095, 2023-10-09T10:56:50.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004017662, 2023-10-09T10:56:50.738 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:50.738 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:56:50.739 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:50.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:50.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.698335+0000", 2023-10-09T10:56:50.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:50.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.698335+0000", 2023-10-09T10:56:50.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:50.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.698334+0000", 2023-10-09T10:56:50.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:50.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:50.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.698337+0000", 2023-10-09T10:56:50.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5780000000000001e-06 2023-10-09T10:56:50.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:50.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.698343+0000", 2023-10-09T10:56:50.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4659999999999996e-06 2023-10-09T10:56:50.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:50.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.698349+0000", 2023-10-09T10:56:50.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.152e-06 2023-10-09T10:56:50.741 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.741 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:50.741 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.741 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.35433.0:183464 8.7 e317/295)", 2023-10-09T10:56:50.741 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:50.701795+0000", 2023-10-09T10:56:50.741 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.000132244, 2023-10-09T10:56:50.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00060129699999999999, 2023-10-09T10:56:50.742 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:50.742 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:56:50.742 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:50.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:50.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.701795+0000", 2023-10-09T10:56:50.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:50.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.701795+0000", 2023-10-09T10:56:50.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:50.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.701794+0000", 2023-10-09T10:56:50.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:50.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:50.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.701796+0000", 2023-10-09T10:56:50.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1660000000000001e-06 2023-10-09T10:56:50.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:50.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.701804+0000", 2023-10-09T10:56:50.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2799999999999998e-06 2023-10-09T10:56:50.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:50.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.701808+0000", 2023-10-09T10:56:50.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4499999999999997e-06 2023-10-09T10:56:50.744 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.744 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:50.744 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.745 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:183455 8.a 8:54537019:::benchmark_data_smithi125_329074_object86801:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T10:56:50.745 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:50.675127+0000", 2023-10-09T10:56:50.745 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.026799479000000001, 2023-10-09T10:56:50.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.027318960999999999, 2023-10-09T10:56:50.745 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:50.745 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:56:50.745 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:50.745 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:50.745 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:50.745 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183455 2023-10-09T10:56:50.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.746 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:50.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:50.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.675127+0000", 2023-10-09T10:56:50.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:50.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.675127+0000", 2023-10-09T10:56:50.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:50.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.675127+0000", 2023-10-09T10:56:50.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:50.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:50.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.675130+0000", 2023-10-09T10:56:50.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3859999999999999e-06 2023-10-09T10:56:50.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:50.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.675137+0000", 2023-10-09T10:56:50.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2790000000000001e-06 2023-10-09T10:56:50.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:50.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.675143+0000", 2023-10-09T10:56:50.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8760000000000002e-06 2023-10-09T10:56:50.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:50.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.680409+0000", 2023-10-09T10:56:50.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0052658679999999999 2023-10-09T10:56:50.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:50.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.680607+0000", 2023-10-09T10:56:50.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000197512 2023-10-09T10:56:50.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2,5", 2023-10-09T10:56:50.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.680805+0000", 2023-10-09T10:56:50.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019833000000000001 2023-10-09T10:56:50.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:50.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.687194+0000", 2023-10-09T10:56:50.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0063894140000000004 2023-10-09T10:56:50.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:50.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.692762+0000", 2023-10-09T10:56:50.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0055671389999999996 2023-10-09T10:56:50.750 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.750 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:50.750 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.750 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:183458 8.a 8:59df8116:::benchmark_data_smithi125_329074_object86804:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T10:56:50.751 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:50.682099+0000", 2023-10-09T10:56:50.751 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.019828278000000001, 2023-10-09T10:56:50.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.020424668999999999, 2023-10-09T10:56:50.751 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:50.751 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:56:50.751 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:50.751 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:50.751 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:50.751 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183458 2023-10-09T10:56:50.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.751 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:50.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:50.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682099+0000", 2023-10-09T10:56:50.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:50.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682099+0000", 2023-10-09T10:56:50.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:50.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682097+0000", 2023-10-09T10:56:50.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:50.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:50.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682102+0000", 2023-10-09T10:56:50.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0370000000000002e-06 2023-10-09T10:56:50.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:50.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682111+0000", 2023-10-09T10:56:50.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9260000000000006e-06 2023-10-09T10:56:50.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:50.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682118+0000", 2023-10-09T10:56:50.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2910000000000004e-06 2023-10-09T10:56:50.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:50.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.687237+0000", 2023-10-09T10:56:50.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0051191320000000002 2023-10-09T10:56:50.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:50.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.687443+0000", 2023-10-09T10:56:50.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020635099999999999 2023-10-09T10:56:50.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2,5", 2023-10-09T10:56:50.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.687642+0000", 2023-10-09T10:56:50.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000198672 2023-10-09T10:56:50.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:50.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.702287+0000", 2023-10-09T10:56:50.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014645128 2023-10-09T10:56:50.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:50.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.702327+0000", 2023-10-09T10:56:50.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9796999999999999e-05 2023-10-09T10:56:50.756 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.756 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:50.756 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.756 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:183459 8.b 8:d104e9ba:::benchmark_data_smithi125_329074_object86805:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T10:56:50.756 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:50.682131+0000", 2023-10-09T10:56:50.756 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.019796118000000001, 2023-10-09T10:56:50.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.020468415, 2023-10-09T10:56:50.757 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:50.757 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:56:50.757 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:50.757 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:50.757 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:50.757 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183459 2023-10-09T10:56:50.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.757 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:50.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:50.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682131+0000", 2023-10-09T10:56:50.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:50.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682131+0000", 2023-10-09T10:56:50.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:50.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682130+0000", 2023-10-09T10:56:50.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:50.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:50.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682133+0000", 2023-10-09T10:56:50.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6570000000000001e-06 2023-10-09T10:56:50.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:50.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682139+0000", 2023-10-09T10:56:50.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8529999999999997e-06 2023-10-09T10:56:50.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:50.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682142+0000", 2023-10-09T10:56:50.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8039999999999999e-06 2023-10-09T10:56:50.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:50.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.685831+0000", 2023-10-09T10:56:50.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036881330000000001 2023-10-09T10:56:50.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:50.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.685948+0000", 2023-10-09T10:56:50.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000117434 2023-10-09T10:56:50.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T10:56:50.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.686055+0000", 2023-10-09T10:56:50.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010689 2023-10-09T10:56:50.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:50.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697649+0000", 2023-10-09T10:56:50.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011593846999999999 2023-10-09T10:56:50.761 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.761 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:50.762 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.762 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:183460 8.f 8:f9933af4:::benchmark_data_smithi125_329074_object86806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T10:56:50.762 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:50.682150+0000", 2023-10-09T10:56:50.762 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.019776466999999999, 2023-10-09T10:56:50.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.020519083, 2023-10-09T10:56:50.762 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:50.762 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:56:50.762 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:50.762 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:50.762 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:50.763 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183460 2023-10-09T10:56:50.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.763 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:50.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:50.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682150+0000", 2023-10-09T10:56:50.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:50.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682150+0000", 2023-10-09T10:56:50.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:50.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682150+0000", 2023-10-09T10:56:50.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:50.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:50.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682152+0000", 2023-10-09T10:56:50.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7969999999999999e-06 2023-10-09T10:56:50.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:50.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682155+0000", 2023-10-09T10:56:50.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3230000000000002e-06 2023-10-09T10:56:50.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:50.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682158+0000", 2023-10-09T10:56:50.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7870000000000002e-06 2023-10-09T10:56:50.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:50.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682602+0000", 2023-10-09T10:56:50.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00044390199999999998 2023-10-09T10:56:50.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:50.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682721+0000", 2023-10-09T10:56:50.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119441 2023-10-09T10:56:50.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2,4", 2023-10-09T10:56:50.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682841+0000", 2023-10-09T10:56:50.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119255 2023-10-09T10:56:50.766 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.767 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:50.767 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.767 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:183462 8.b 8:de2a8810:::benchmark_data_smithi125_329074_object86808:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T10:56:50.767 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:50.682165+0000", 2023-10-09T10:56:50.767 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.019762021000000001, 2023-10-09T10:56:50.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.020570404, 2023-10-09T10:56:50.767 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:50.767 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:56:50.767 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:50.767 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:50.768 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:50.768 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183462 2023-10-09T10:56:50.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.768 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:50.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:50.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682165+0000", 2023-10-09T10:56:50.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:50.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682165+0000", 2023-10-09T10:56:50.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:50.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682165+0000", 2023-10-09T10:56:50.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:56:50.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:50.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682166+0000", 2023-10-09T10:56:50.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.562e-06 2023-10-09T10:56:50.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:50.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682169+0000", 2023-10-09T10:56:50.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9390000000000002e-06 2023-10-09T10:56:50.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:50.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.682172+0000", 2023-10-09T10:56:50.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.824e-06 2023-10-09T10:56:50.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:50.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.688230+0000", 2023-10-09T10:56:50.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0060581929999999999 2023-10-09T10:56:50.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:50.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.688325+0000", 2023-10-09T10:56:50.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5229999999999995e-05 2023-10-09T10:56:50.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T10:56:50.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.688415+0000", 2023-10-09T10:56:50.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9307000000000004e-05 2023-10-09T10:56:50.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:56:50.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.697686+0000", 2023-10-09T10:56:50.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0092710499999999994 2023-10-09T10:56:50.772 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.772 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:50.772 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.772 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:183464 8.7 8:e3e81359:::benchmark_data_smithi125_329074_object86810:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T10:56:50.772 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:56:50.687261+0000", 2023-10-09T10:56:50.772 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.014665635999999999, 2023-10-09T10:56:50.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015551776, 2023-10-09T10:56:50.773 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:56:50.773 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:56:50.773 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:56:50.773 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:56:50.773 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:56:50.773 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183464 2023-10-09T10:56:50.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.773 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:56:50.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:56:50.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.687261+0000", 2023-10-09T10:56:50.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:56:50.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.687261+0000", 2023-10-09T10:56:50.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:56:50.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:56:50.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.687259+0000", 2023-10-09T10:56:50.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:56:50.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:56:50.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.687269+0000", 2023-10-09T10:56:50.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.815e-06 2023-10-09T10:56:50.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:56:50.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.687284+0000", 2023-10-09T10:56:50.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4352000000000001e-05 2023-10-09T10:56:50.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:56:50.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.687294+0000", 2023-10-09T10:56:50.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0159999999999999e-05 2023-10-09T10:56:50.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:56:50.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.687353+0000", 2023-10-09T10:56:50.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8860000000000002e-05 2023-10-09T10:56:50.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:56:50.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.687509+0000", 2023-10-09T10:56:50.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000156735 2023-10-09T10:56:50.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:56:50.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:56:50.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,6", 2023-10-09T10:56:50.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:56:50.688019+0000", 2023-10-09T10:56:50.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00050963699999999996 2023-10-09T10:56:50.777 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.777 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:56:50.777 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.777 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:56:50.777 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:56:50.777 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 14 2023-10-09T10:56:50.777 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:56:50.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:50.779 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:50.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:50.880 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:50.961 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:51.402 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:56:51.445 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:51.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:51.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:51.477 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:56:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:56:51] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.43.0" 2023-10-09T10:56:51.552 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:51.595 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:51.668 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:51.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:51.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:51.772 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:51.809 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:51.829 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:51.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:51.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:51.934 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:51.993 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:52.079 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:52.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:52.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:52.128 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:52.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:52.221 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:56:52.269 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:56:52.278 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:52.311 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:52.313 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:56:52.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:52.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:52.357 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:56:52.406 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:56:52.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:52.444 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:56:52.454 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:56:52.498 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:56:52.517 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:52.541 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:56:52.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:56:52.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:56:52.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:56:52.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:56:52.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:56:52.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:56:52.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:56:52.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:56:52.565 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:52.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:52.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:52.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:52.720 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:52.804 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:52.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:52.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:52.809 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:52 smithi155 ceph-mon[184854]: pgmap v787: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 673 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 11165/32328 objects degraded (34.537%); 3254/32328 objects misplaced (10.066%); 4.4 MiB/s, 72 objects/s recovering 2023-10-09T10:56:52.809 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:52 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/87639161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:52.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:52 smithi125 ceph-mon[195507]: pgmap v787: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 673 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 11165/32328 objects degraded (34.537%); 3254/32328 objects misplaced (10.066%); 4.4 MiB/s, 72 objects/s recovering 2023-10-09T10:56:52.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:52 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/87639161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:52.828 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:52 smithi125 ceph-mon[200197]: pgmap v787: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 673 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 11165/32328 objects degraded (34.537%); 3254/32328 objects misplaced (10.066%); 4.4 MiB/s, 72 objects/s recovering 2023-10-09T10:56:52.828 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:52 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/87639161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:56:52.909 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:52.936 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:53.055 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:53.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:53.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:53.161 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:53.198 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:53.229 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:53.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:53.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:53.333 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:53.378 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:53.404 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:53.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:53.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:53.509 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:53.549 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:53.601 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:53.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:53.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:53.707 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:53 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 11165/32328 objects degraded (34.537%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:53 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:56:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:53 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:56:53.765 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:54.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:53 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 11165/32328 objects degraded (34.537%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:54.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:53 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:56:54.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:53 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:56:54.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:53 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 11165/32328 objects degraded (34.537%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:54.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:53 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:56:54.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:53 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:56:54.902 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 7, 2, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:56:54.903 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:56:54.903 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [3, 4, 5, 6, 0, 7, 2, 1] 2023-10-09T10:56:54.903 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-09T10:56:54.963 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:54 smithi125 ceph-mon[195507]: pgmap v788: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 583 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 10966/28020 objects degraded (39.136%); 3254/28020 objects misplaced (11.613%); 2.0 MiB/s, 53 objects/s recovering 2023-10-09T10:56:54.964 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:54 smithi125 ceph-mon[200197]: pgmap v788: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 583 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 10966/28020 objects degraded (39.136%); 3254/28020 objects misplaced (11.613%); 2.0 MiB/s, 53 objects/s recovering 2023-10-09T10:56:54.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:54 smithi155 ceph-mon[184854]: pgmap v788: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 583 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 10966/28020 objects degraded (39.136%); 3254/28020 objects misplaced (11.613%); 2.0 MiB/s, 53 objects/s recovering 2023-10-09T10:56:54.985 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:54.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:54.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:55.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:55.131 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:55.154 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:55.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:55.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:55.224 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:55.258 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:56:55.288 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:55.308 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:56:55.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:55.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:56:55.414 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:55.482 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:55.578 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:55.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:55.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:55.607 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:56:55.607 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:56:55.685 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:55.716 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:55.754 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:55.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:55.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:55.859 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:56:55.907 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:55.936 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:56:55.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:55.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:56:56.045 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:56.078 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:56.105 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:56.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:56.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:56.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:56:56.247 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:56.277 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:56.278 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:56:56.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:56.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:56:56.384 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:56.433 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:56.509 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:56.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:56.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:56.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:56:56.614 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:56.717 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:56.761 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:56.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:56.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:56.789 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:56:56.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:56:56.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:56 smithi155 ceph-mon[184854]: pgmap v789: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 583 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 10966/28020 objects degraded (39.136%); 3254/28020 objects misplaced (11.613%); 2.0 MiB/s, 53 objects/s recovering 2023-10-09T10:56:56.990 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:56 smithi125 ceph-mon[195507]: pgmap v789: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 583 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 10966/28020 objects degraded (39.136%); 3254/28020 objects misplaced (11.613%); 2.0 MiB/s, 53 objects/s recovering 2023-10-09T10:56:56.990 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:56 smithi125 ceph-mon[200197]: pgmap v789: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 583 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 10966/28020 objects degraded (39.136%); 3254/28020 objects misplaced (11.613%); 2.0 MiB/s, 53 objects/s recovering 2023-10-09T10:56:57.000 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:57.669 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:57 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3352920454' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:56:57.670 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:57 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3352920454' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:56:57.703 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:56:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:57 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3352920454' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:56:58.616 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:56:58.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:58.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:56:58.723 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:56:58.762 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:58.785 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:56:58.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:58.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:56:58.889 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:58.957 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[200197]: pgmap v790: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 430 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 10545/20688 objects degraded (50.972%); 3044/20688 objects misplaced (14.714%); 2.0 MiB/s, 53 objects/s recovering 2023-10-09T10:56:58.957 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:56:58.957 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[200197]: failed to encode map e318 with expected crc 2023-10-09T10:56:58.957 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3352920454' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:56:58.957 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[200197]: osdmap e318: 8 total, 8 up, 8 in 2023-10-09T10:56:58.957 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[200197]: failed to encode map e318 with expected crc 2023-10-09T10:56:58.957 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[200197]: failed to encode map e318 with expected crc 2023-10-09T10:56:58.957 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 10545/20688 objects degraded (50.972%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:58.958 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[195507]: pgmap v790: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 430 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 10545/20688 objects degraded (50.972%); 3044/20688 objects misplaced (14.714%); 2.0 MiB/s, 53 objects/s recovering 2023-10-09T10:56:58.958 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:56:58.958 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[195507]: failed to encode map e318 with expected crc 2023-10-09T10:56:58.958 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3352920454' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:56:58.958 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[195507]: osdmap e318: 8 total, 8 up, 8 in 2023-10-09T10:56:58.959 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[195507]: failed to encode map e318 with expected crc 2023-10-09T10:56:58.959 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[195507]: failed to encode map e318 with expected crc 2023-10-09T10:56:58.959 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:56:58 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 10545/20688 objects degraded (50.972%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:58.959 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:58.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:58 smithi155 ceph-mon[184854]: pgmap v790: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 430 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 10545/20688 objects degraded (50.972%); 3044/20688 objects misplaced (14.714%); 2.0 MiB/s, 53 objects/s recovering 2023-10-09T10:56:58.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:58 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:56:58.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:58 smithi155 ceph-mon[184854]: failed to encode map e318 with expected crc 2023-10-09T10:56:58.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:58 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3352920454' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:56:58.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:58 smithi155 ceph-mon[184854]: osdmap e318: 8 total, 8 up, 8 in 2023-10-09T10:56:58.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:58 smithi155 ceph-mon[184854]: failed to encode map e318 with expected crc 2023-10-09T10:56:58.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:58 smithi155 ceph-mon[184854]: failed to encode map e318 with expected crc 2023-10-09T10:56:58.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:56:58 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 10545/20688 objects degraded (50.972%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T10:56:59.170 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:59.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:59.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:59.276 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:56:59.346 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:59.430 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:56:59.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:59.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:56:59.490 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:59.536 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:59.553 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:56:59.608 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:59.650 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:59.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:59.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:56:59.759 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:56:59.831 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:56:59.886 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:56:59.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:56:59.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:00.000 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:00.030 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:00.054 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:00.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:00.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:00.160 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:00.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T10:57:00.074+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e318 definitely_dead 0 2023-10-09T10:57:00.227 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:00.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[195507]: pgmap v792: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 360 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 10341/17304 objects degraded (59.761%); 3044/17304 objects misplaced (17.591%); 0 B/s, 40 objects/s recovering 2023-10-09T10:57:00.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:00.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:00.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:57:00.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/260846163' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:57:00.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:57:00.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[195507]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T10:57:00.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:57:00.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:57:00.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:00.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[200197]: pgmap v792: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 360 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 10341/17304 objects degraded (59.761%); 3044/17304 objects misplaced (17.591%); 0 B/s, 40 objects/s recovering 2023-10-09T10:57:00.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:00.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:00.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:57:00.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/260846163' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:57:00.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:57:00.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[200197]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T10:57:00.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:57:00.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:57:00.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:00.740 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.7. 2023-10-09T10:57:00.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:00 smithi155 ceph-mon[184854]: pgmap v792: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 360 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 10341/17304 objects degraded (59.761%); 3044/17304 objects misplaced (17.591%); 0 B/s, 40 objects/s recovering 2023-10-09T10:57:00.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:00.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:00.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:57:00.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:00 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/260846163' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:57:00.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:00 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T10:57:00.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:00 smithi155 ceph-mon[184854]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T10:57:00.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:57:00.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:57:00.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:01.120 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:57:01.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:57:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:57:01] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2023-10-09T10:57:01.923 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:01.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:01.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:01.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:01 smithi155 ceph-mon[184854]: failed to encode map e319 with expected crc 2023-10-09T10:57:01.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:01 smithi155 ceph-mon[184854]: failed to encode map e319 with expected crc 2023-10-09T10:57:01.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:01 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T10:57:01.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:01 smithi155 ceph-mon[184854]: osdmap e319: 8 total, 8 up, 7 in 2023-10-09T10:57:02.029 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:02.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:01 smithi125 ceph-mon[195507]: failed to encode map e319 with expected crc 2023-10-09T10:57:02.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:01 smithi125 ceph-mon[195507]: failed to encode map e319 with expected crc 2023-10-09T10:57:02.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:01 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T10:57:02.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:01 smithi125 ceph-mon[195507]: osdmap e319: 8 total, 8 up, 7 in 2023-10-09T10:57:02.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:01 smithi125 ceph-mon[200197]: failed to encode map e319 with expected crc 2023-10-09T10:57:02.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:01 smithi125 ceph-mon[200197]: failed to encode map e319 with expected crc 2023-10-09T10:57:02.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:01 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T10:57:02.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:01 smithi125 ceph-mon[200197]: osdmap e319: 8 total, 8 up, 7 in 2023-10-09T10:57:02.071 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:02.090 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:02.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:02.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:02.193 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:02.220 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:02.238 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:02.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:02.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:02.342 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:02.372 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:02.680 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:02.692 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:02.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:02.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:02.795 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:57:02.795 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:57:02.804 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:02.840 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:02.864 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:02.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:02.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:02.969 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:02 smithi155 ceph-mon[184854]: pgmap v794: 129 pgs: 1 remapped+peering, 12 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 230 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 706 op/s; 6787/11061 objects degraded (61.360%); 2827/11061 objects misplaced (25.558%); 0 B/s, 25 objects/s recovering 2023-10-09T10:57:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:02 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T10:57:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:02 smithi155 ceph-mon[184854]: failed to encode map e320 with expected crc 2023-10-09T10:57:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:02 smithi155 ceph-mon[184854]: failed to encode map e320 with expected crc 2023-10-09T10:57:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:02 smithi155 ceph-mon[184854]: failed to encode map e320 with expected crc 2023-10-09T10:57:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:02 smithi155 ceph-mon[184854]: osdmap e320: 8 total, 8 up, 7 in 2023-10-09T10:57:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:02 smithi155 ceph-mon[184854]: failed to encode map e320 with expected crc 2023-10-09T10:57:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:02 smithi155 ceph-mon[184854]: failed to encode map e320 with expected crc 2023-10-09T10:57:02.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:02 smithi155 ceph-mon[184854]: failed to encode map e320 with expected crc 2023-10-09T10:57:02.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:02 smithi155 ceph-mon[184854]: failed to encode map e320 with expected crc 2023-10-09T10:57:03.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[195507]: pgmap v794: 129 pgs: 1 remapped+peering, 12 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 230 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 706 op/s; 6787/11061 objects degraded (61.360%); 2827/11061 objects misplaced (25.558%); 0 B/s, 25 objects/s recovering 2023-10-09T10:57:03.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T10:57:03.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[195507]: failed to encode map e320 with expected crc 2023-10-09T10:57:03.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[195507]: failed to encode map e320 with expected crc 2023-10-09T10:57:03.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[195507]: failed to encode map e320 with expected crc 2023-10-09T10:57:03.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[195507]: osdmap e320: 8 total, 8 up, 7 in 2023-10-09T10:57:03.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[195507]: failed to encode map e320 with expected crc 2023-10-09T10:57:03.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[195507]: failed to encode map e320 with expected crc 2023-10-09T10:57:03.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[195507]: failed to encode map e320 with expected crc 2023-10-09T10:57:03.008 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[195507]: failed to encode map e320 with expected crc 2023-10-09T10:57:03.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[200197]: pgmap v794: 129 pgs: 1 remapped+peering, 12 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 230 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 706 op/s; 6787/11061 objects degraded (61.360%); 2827/11061 objects misplaced (25.558%); 0 B/s, 25 objects/s recovering 2023-10-09T10:57:03.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T10:57:03.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[200197]: failed to encode map e320 with expected crc 2023-10-09T10:57:03.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[200197]: failed to encode map e320 with expected crc 2023-10-09T10:57:03.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[200197]: failed to encode map e320 with expected crc 2023-10-09T10:57:03.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[200197]: osdmap e320: 8 total, 8 up, 7 in 2023-10-09T10:57:03.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[200197]: failed to encode map e320 with expected crc 2023-10-09T10:57:03.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[200197]: failed to encode map e320 with expected crc 2023-10-09T10:57:03.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[200197]: failed to encode map e320 with expected crc 2023-10-09T10:57:03.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:02 smithi125 ceph-mon[200197]: failed to encode map e320 with expected crc 2023-10-09T10:57:03.018 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:03.043 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:03.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:03.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:03.147 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:03.191 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:03.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:03.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:03.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:03.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:03.357 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:03.380 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:03.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:03.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:03.486 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:03.514 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:03.532 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:03.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:03.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:03.562 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:57:03.635 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:03.643 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:57:03.662 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:03.666 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:57:03.680 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:03.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:03.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:03.689 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:57:03.712 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:57:03.744 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:57:03.775 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:57:03.784 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:03.801 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:57:03.811 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:03.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: osdmap e321: 8 total, 8 up, 7 in 2023-10-09T10:57:03.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: 8.d starting backfill to osd.6 from (0'0,280'5145] MAX to 318'11791 2023-10-09T10:57:03.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: 8.16 starting backfill to osd.2 from (0'0,0'0] MAX to 318'9019 2023-10-09T10:57:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: 8.18 starting backfill to osd.0 from (0'0,0'0] MAX to 318'8945 2023-10-09T10:57:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: 8.6 starting backfill to osd.2 from (0'0,0'0] MAX to 318'8932 2023-10-09T10:57:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: 8.5 starting backfill to osd.0 from (0'0,0'0] MAX to 318'8894 2023-10-09T10:57:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: 8.c starting backfill to osd.5 from (0'0,0'0] MAX to 318'11991 2023-10-09T10:57:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: 8.8 starting backfill to osd.0 from (0'0,0'0] MAX to 318'8963 2023-10-09T10:57:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: 2.0 continuing backfill to osd.3 from (0'0,0'0] MIN to 0'0 2023-10-09T10:57:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: 8.15 starting backfill to osd.0 from (0'0,0'0] MAX to 318'8843 2023-10-09T10:57:03.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 6787/11061 objects degraded (61.360%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:57:03.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: osdmap e322: 8 total, 8 up, 7 in 2023-10-09T10:57:03.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: failed to encode map e322 with expected crc 2023-10-09T10:57:03.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: failed to encode map e322 with expected crc 2023-10-09T10:57:03.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: failed to encode map e322 with expected crc 2023-10-09T10:57:03.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/113157975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:03.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: osdmap e321: 8 total, 8 up, 7 in 2023-10-09T10:57:03.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: 8.d starting backfill to osd.6 from (0'0,280'5145] MAX to 318'11791 2023-10-09T10:57:03.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: 8.16 starting backfill to osd.2 from (0'0,0'0] MAX to 318'9019 2023-10-09T10:57:03.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: 8.18 starting backfill to osd.0 from (0'0,0'0] MAX to 318'8945 2023-10-09T10:57:03.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: 8.6 starting backfill to osd.2 from (0'0,0'0] MAX to 318'8932 2023-10-09T10:57:03.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: 8.5 starting backfill to osd.0 from (0'0,0'0] MAX to 318'8894 2023-10-09T10:57:03.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: 8.c starting backfill to osd.5 from (0'0,0'0] MAX to 318'11991 2023-10-09T10:57:03.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: 8.8 starting backfill to osd.0 from (0'0,0'0] MAX to 318'8963 2023-10-09T10:57:03.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: 2.0 continuing backfill to osd.3 from (0'0,0'0] MIN to 0'0 2023-10-09T10:57:03.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: 8.15 starting backfill to osd.0 from (0'0,0'0] MAX to 318'8843 2023-10-09T10:57:03.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 6787/11061 objects degraded (61.360%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:57:03.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: osdmap e322: 8 total, 8 up, 7 in 2023-10-09T10:57:03.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: failed to encode map e322 with expected crc 2023-10-09T10:57:03.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: failed to encode map e322 with expected crc 2023-10-09T10:57:03.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: failed to encode map e322 with expected crc 2023-10-09T10:57:03.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/113157975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:03.824 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:57:03.825 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:57:03.825 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:57:03.825 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:57:03.825 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:57:03.825 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:57:03.825 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:57:03.825 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:57:03.825 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:57:03.829 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:03.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:03.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:03.909 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:57:03.936 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:03.966 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: osdmap e321: 8 total, 8 up, 7 in 2023-10-09T10:57:03.966 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.966 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.967 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: 8.d starting backfill to osd.6 from (0'0,280'5145] MAX to 318'11791 2023-10-09T10:57:03.967 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.967 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: 8.16 starting backfill to osd.2 from (0'0,0'0] MAX to 318'9019 2023-10-09T10:57:03.967 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: 8.18 starting backfill to osd.0 from (0'0,0'0] MAX to 318'8945 2023-10-09T10:57:03.967 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.967 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: 8.6 starting backfill to osd.2 from (0'0,0'0] MAX to 318'8932 2023-10-09T10:57:03.967 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.967 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.967 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: failed to encode map e321 with expected crc 2023-10-09T10:57:03.967 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: 8.5 starting backfill to osd.0 from (0'0,0'0] MAX to 318'8894 2023-10-09T10:57:03.967 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: 8.c starting backfill to osd.5 from (0'0,0'0] MAX to 318'11991 2023-10-09T10:57:03.967 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: 8.8 starting backfill to osd.0 from (0'0,0'0] MAX to 318'8963 2023-10-09T10:57:03.968 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: 2.0 continuing backfill to osd.3 from (0'0,0'0] MIN to 0'0 2023-10-09T10:57:03.968 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: 8.15 starting backfill to osd.0 from (0'0,0'0] MAX to 318'8843 2023-10-09T10:57:03.968 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 6787/11061 objects degraded (61.360%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-10-09T10:57:03.968 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: osdmap e322: 8 total, 8 up, 7 in 2023-10-09T10:57:03.968 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: failed to encode map e322 with expected crc 2023-10-09T10:57:03.968 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: failed to encode map e322 with expected crc 2023-10-09T10:57:03.968 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: failed to encode map e322 with expected crc 2023-10-09T10:57:03.968 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/113157975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:03.968 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:04.102 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:04.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:04.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:04.209 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:04.245 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:04.627 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:04.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:04.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:04.717 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:04 smithi125 ceph-mon[195507]: pgmap v798: 129 pgs: 9 remapped+peering, 13 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 173 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 6130/8361 objects degraded (73.317%); 2827/8361 objects misplaced (33.812%); 1.2 MiB/s, 163 objects/s recovering 2023-10-09T10:57:04.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:04 smithi125 ceph-mon[195507]: osdmap e323: 8 total, 8 up, 7 in 2023-10-09T10:57:04.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:04 smithi125 ceph-mon[195507]: failed to encode map e323 with expected crc 2023-10-09T10:57:04.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:04 smithi125 ceph-mon[195507]: failed to encode map e323 with expected crc 2023-10-09T10:57:04.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:04 smithi125 ceph-mon[195507]: failed to encode map e323 with expected crc 2023-10-09T10:57:04.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:04 smithi125 ceph-mon[195507]: failed to encode map e323 with expected crc 2023-10-09T10:57:04.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:04 smithi125 ceph-mon[195507]: failed to encode map e323 with expected crc 2023-10-09T10:57:04.731 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:04.777 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:04.798 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:04.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:04.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:04.903 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:04.931 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:04.953 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:04.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:04.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:04 smithi155 ceph-mon[184854]: pgmap v798: 129 pgs: 9 remapped+peering, 13 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 173 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 6130/8361 objects degraded (73.317%); 2827/8361 objects misplaced (33.812%); 1.2 MiB/s, 163 objects/s recovering 2023-10-09T10:57:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:04 smithi155 ceph-mon[184854]: osdmap e323: 8 total, 8 up, 7 in 2023-10-09T10:57:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:04 smithi155 ceph-mon[184854]: failed to encode map e323 with expected crc 2023-10-09T10:57:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:04 smithi155 ceph-mon[184854]: failed to encode map e323 with expected crc 2023-10-09T10:57:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:04 smithi155 ceph-mon[184854]: failed to encode map e323 with expected crc 2023-10-09T10:57:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:04 smithi155 ceph-mon[184854]: failed to encode map e323 with expected crc 2023-10-09T10:57:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:04 smithi155 ceph-mon[184854]: failed to encode map e323 with expected crc 2023-10-09T10:57:04.986 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:57:04.987 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:57:04.987 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T10:57:04.987 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:57:04.995 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:57:05.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:04 smithi125 ceph-mon[200197]: pgmap v798: 129 pgs: 9 remapped+peering, 13 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 173 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 6130/8361 objects degraded (73.317%); 2827/8361 objects misplaced (33.812%); 1.2 MiB/s, 163 objects/s recovering 2023-10-09T10:57:05.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:04 smithi125 ceph-mon[200197]: osdmap e323: 8 total, 8 up, 7 in 2023-10-09T10:57:05.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:04 smithi125 ceph-mon[200197]: failed to encode map e323 with expected crc 2023-10-09T10:57:05.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:04 smithi125 ceph-mon[200197]: failed to encode map e323 with expected crc 2023-10-09T10:57:05.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:04 smithi125 ceph-mon[200197]: failed to encode map e323 with expected crc 2023-10-09T10:57:05.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:04 smithi125 ceph-mon[200197]: failed to encode map e323 with expected crc 2023-10-09T10:57:05.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:04 smithi125 ceph-mon[200197]: failed to encode map e323 with expected crc 2023-10-09T10:57:05.057 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:05.148 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:05.288 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:05.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:05.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:05.317 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:57:05.396 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:05.488 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:05.516 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:05.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:05.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:05.625 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:05.649 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:57:05.673 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:05.700 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:05.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:05.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:05.806 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:05.834 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:05.854 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:05.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:05.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:05.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:05.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:05 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/612148728' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:05.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:05 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:05.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:05 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:05.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:05 smithi155 ceph-mon[184854]: failed to encode map e324 with expected crc 2023-10-09T10:57:05.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:05 smithi155 ceph-mon[184854]: failed to encode map e324 with expected crc 2023-10-09T10:57:05.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:05 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:57:05.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:05 smithi155 ceph-mon[184854]: osdmap e324: 8 total, 8 up, 7 in 2023-10-09T10:57:05.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:05 smithi155 ceph-mon[184854]: failed to encode map e324 with expected crc 2023-10-09T10:57:05.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:05 smithi155 ceph-mon[184854]: failed to encode map e324 with expected crc 2023-10-09T10:57:05.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:05 smithi155 ceph-mon[184854]: failed to encode map e324 with expected crc 2023-10-09T10:57:05.990 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/612148728' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:05.990 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:05.990 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:05.990 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[195507]: failed to encode map e324 with expected crc 2023-10-09T10:57:05.991 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[195507]: failed to encode map e324 with expected crc 2023-10-09T10:57:05.991 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:57:05.991 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[195507]: osdmap e324: 8 total, 8 up, 7 in 2023-10-09T10:57:05.991 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[195507]: failed to encode map e324 with expected crc 2023-10-09T10:57:05.991 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[195507]: failed to encode map e324 with expected crc 2023-10-09T10:57:05.991 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[195507]: failed to encode map e324 with expected crc 2023-10-09T10:57:05.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/612148728' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:05.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:05.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:05.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[200197]: failed to encode map e324 with expected crc 2023-10-09T10:57:05.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[200197]: failed to encode map e324 with expected crc 2023-10-09T10:57:05.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:57:05.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[200197]: osdmap e324: 8 total, 8 up, 7 in 2023-10-09T10:57:05.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[200197]: failed to encode map e324 with expected crc 2023-10-09T10:57:05.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[200197]: failed to encode map e324 with expected crc 2023-10-09T10:57:05.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:05 smithi125 ceph-mon[200197]: failed to encode map e324 with expected crc 2023-10-09T10:57:05.993 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:06.021 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:06.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:06.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:06.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:06.178 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:06.215 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:06.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:06.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:06.324 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:06.371 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:06.376 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:57:06.507 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:06.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:06.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:06.612 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:06.672 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:06.699 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:06.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:06.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:06.721 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:57:06.722 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:57:06.722 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:57:06.722 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:57:06.722 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:57:06.811 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:06.862 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:06.892 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:06.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:06.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:06.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:07.024 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:07.029 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:06 smithi155 ceph-mon[184854]: pgmap v801: 129 pgs: 9 remapped+peering, 13 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 173 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 234 op/s; 6130/8361 objects degraded (73.317%); 2827/8361 objects misplaced (33.812%); 1.3 MiB/s, 172 objects/s recovering 2023-10-09T10:57:07.029 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:06 smithi155 ceph-mon[184854]: failed to encode map e325 with expected crc 2023-10-09T10:57:07.029 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:06 smithi155 ceph-mon[184854]: osdmap e325: 8 total, 8 up, 7 in 2023-10-09T10:57:07.029 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:06 smithi155 ceph-mon[184854]: failed to encode map e325 with expected crc 2023-10-09T10:57:07.030 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:06 smithi155 ceph-mon[184854]: failed to encode map e325 with expected crc 2023-10-09T10:57:07.030 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:06 smithi155 ceph-mon[184854]: failed to encode map e325 with expected crc 2023-10-09T10:57:07.044 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:06 smithi125 ceph-mon[195507]: pgmap v801: 129 pgs: 9 remapped+peering, 13 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 173 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 234 op/s; 6130/8361 objects degraded (73.317%); 2827/8361 objects misplaced (33.812%); 1.3 MiB/s, 172 objects/s recovering 2023-10-09T10:57:07.044 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:06 smithi125 ceph-mon[195507]: failed to encode map e325 with expected crc 2023-10-09T10:57:07.044 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:06 smithi125 ceph-mon[195507]: osdmap e325: 8 total, 8 up, 7 in 2023-10-09T10:57:07.044 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:06 smithi125 ceph-mon[195507]: failed to encode map e325 with expected crc 2023-10-09T10:57:07.044 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:06 smithi125 ceph-mon[195507]: failed to encode map e325 with expected crc 2023-10-09T10:57:07.044 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:06 smithi125 ceph-mon[195507]: failed to encode map e325 with expected crc 2023-10-09T10:57:07.044 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:06 smithi125 ceph-mon[200197]: pgmap v801: 129 pgs: 9 remapped+peering, 13 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 173 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 234 op/s; 6130/8361 objects degraded (73.317%); 2827/8361 objects misplaced (33.812%); 1.3 MiB/s, 172 objects/s recovering 2023-10-09T10:57:07.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:06 smithi125 ceph-mon[200197]: failed to encode map e325 with expected crc 2023-10-09T10:57:07.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:06 smithi125 ceph-mon[200197]: osdmap e325: 8 total, 8 up, 7 in 2023-10-09T10:57:07.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:06 smithi125 ceph-mon[200197]: failed to encode map e325 with expected crc 2023-10-09T10:57:07.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:06 smithi125 ceph-mon[200197]: failed to encode map e325 with expected crc 2023-10-09T10:57:07.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:06 smithi125 ceph-mon[200197]: failed to encode map e325 with expected crc 2023-10-09T10:57:07.069 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:07.093 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:07.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:07.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:07.112 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:57:07.197 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:07.289 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:07.333 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:07.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:07.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:07.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:57:07.440 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:07.481 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:07.503 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:57:07.510 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:07.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:07.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:07.586 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 2, 1] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:57:07.586 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:57:07.586 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T10:57:07.586 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.981158 2023-10-09T10:57:07.586 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0.9811576053054418 2023-10-09T10:57:07.614 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:07.747 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:07.839 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:07.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:07.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:07.899 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:07 smithi125 ceph-mon[200197]: Health check update: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:57:07.901 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:07 smithi125 ceph-mon[195507]: Health check update: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:57:07.947 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:07.985 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:08.136 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:07 smithi155 ceph-mon[184854]: Health check update: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T10:57:08.159 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:08.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:08.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:08.170 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:08.198 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:08.224 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:57:08.263 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:08.263 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Removed 96651 objects 2023-10-09T10:57:08.263 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Clean up completed and total clean up time :138.271 2023-10-09T10:57:08.266 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:57:08.292 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:57:08.297 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:08.321 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:57:08.348 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:57:08.371 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:57:08.401 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:57:08.423 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:57:08.427 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:57:08.427 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:57:08.427 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:57:08.427 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:57:08.427 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:57:08.427 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:57:08.427 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:57:08.427 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:57:08.512 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:57:08.512 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:57:08.620 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:08.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:08.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:08.726 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:08.757 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:08.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:08 smithi125 ceph-mon[195507]: pgmap v803: 129 pgs: 1 active+recovering+remapped, 5 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 125 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 168 op/s; 5487/6039 objects degraded (90.859%); 2849/6039 objects misplaced (47.177%); 2.9 MiB/s, 2 keys/s, 166 objects/s recovering 2023-10-09T10:57:08.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:08 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 5487/6039 objects degraded (90.859%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:57:08.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:08 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3576557910' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:08.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:08 smithi125 ceph-mon[200197]: pgmap v803: 129 pgs: 1 active+recovering+remapped, 5 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 125 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 168 op/s; 5487/6039 objects degraded (90.859%); 2849/6039 objects misplaced (47.177%); 2.9 MiB/s, 2 keys/s, 166 objects/s recovering 2023-10-09T10:57:08.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:08 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 5487/6039 objects degraded (90.859%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:57:08.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:08 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3576557910' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:08.912 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:57:08.980 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:08.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:08.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:09.084 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:09.109 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:08 smithi155 ceph-mon[184854]: pgmap v803: 129 pgs: 1 active+recovering+remapped, 5 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 125 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 168 op/s; 5487/6039 objects degraded (90.859%); 2849/6039 objects misplaced (47.177%); 2.9 MiB/s, 2 keys/s, 166 objects/s recovering 2023-10-09T10:57:09.109 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:08 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 5487/6039 objects degraded (90.859%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:57:09.109 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:08 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3576557910' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:09.112 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:09.132 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:09.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:09.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:09.177 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:57:09.235 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:09.265 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:09.438 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:09.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:09.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:09.544 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:09.572 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:09.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:09.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:09.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:09.697 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:09.723 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:09.793 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:09.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:09.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:09.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:09.933 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:09.957 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:09.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:09.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:09.980 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:09 smithi155 ceph-mon[184854]: pgmap v804: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 69 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 284 op/s; 5441/3348 objects degraded (162.515%); 2929/3348 objects misplaced (87.485%); 3.6 MiB/s, 1 keys/s, 157 objects/s recovering 2023-10-09T10:57:09.980 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:09 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T10:57:10.061 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:10.091 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:10.110 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:10.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:10.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:10.134 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:57:10.134 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:57:10.214 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:10.240 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:09 smithi125 ceph-mon[195507]: pgmap v804: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 69 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 284 op/s; 5441/3348 objects degraded (162.515%); 2929/3348 objects misplaced (87.485%); 3.6 MiB/s, 1 keys/s, 157 objects/s recovering 2023-10-09T10:57:10.240 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:09 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T10:57:10.241 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:09 smithi125 ceph-mon[200197]: pgmap v804: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 69 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 284 op/s; 5441/3348 objects degraded (162.515%); 2929/3348 objects misplaced (87.485%); 3.6 MiB/s, 1 keys/s, 157 objects/s recovering 2023-10-09T10:57:10.241 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:09 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T10:57:10.242 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:10.264 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:10.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:10.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:10.368 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:10.397 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:10.467 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:10.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:10.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:10.502 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:57:10.571 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:10.599 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-10-09T10:57:10.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-10-09T10:57:10.614 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:10.634 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:10.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:10.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:10.738 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:10.766 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:10.785 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:10.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:10.813 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:57:10.860 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:57:10.861 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.1 primary-affinity to 0.981158 (8643012) 2023-10-09T10:57:10.881 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:10 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1358615268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:57:10.881 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:10 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:57:10.881 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:10 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/93565560' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9811576053054418}]: dispatch 2023-10-09T10:57:10.881 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:10 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1358615268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:57:10.882 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:10 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:57:10.882 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:10 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/93565560' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9811576053054418}]: dispatch 2023-10-09T10:57:10.888 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:57:10.889 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:10.926 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:11.055 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:11.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:11.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:11.134 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:10 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1358615268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:57:11.134 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:10 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:57:11.135 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:10 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/93565560' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9811576053054418}]: dispatch 2023-10-09T10:57:11.159 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:11.187 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:11.561 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:11.563 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:57:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:57:11] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2023-10-09T10:57:11.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:11.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:11.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:11.694 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:11.871 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:11.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:11.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:11.883 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:57:11.883 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:57:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:11 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:11 smithi155 ceph-mon[184854]: failed to encode map e326 with expected crc 2023-10-09T10:57:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:11 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:57:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:11 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/93565560' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9811576053054418}]': finished 2023-10-09T10:57:11.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:11 smithi155 ceph-mon[184854]: osdmap e326: 8 total, 8 up, 7 in 2023-10-09T10:57:11.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:11 smithi155 ceph-mon[184854]: failed to encode map e326 with expected crc 2023-10-09T10:57:11.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:11 smithi155 ceph-mon[184854]: pgmap v806: 129 pgs: 1 peering, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 22 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 4440/1119 objects degraded (396.783%); 2767/1119 objects misplaced (247.274%); 3.3 MiB/s, 1 keys/s, 363 objects/s recovering 2023-10-09T10:57:11.977 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:12.003 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:12.277 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:57:12.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:11 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:12.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:11 smithi125 ceph-mon[200197]: failed to encode map e326 with expected crc 2023-10-09T10:57:12.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:11 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:57:12.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:11 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/93565560' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9811576053054418}]': finished 2023-10-09T10:57:12.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:11 smithi125 ceph-mon[200197]: osdmap e326: 8 total, 8 up, 7 in 2023-10-09T10:57:12.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:11 smithi125 ceph-mon[200197]: failed to encode map e326 with expected crc 2023-10-09T10:57:12.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:11 smithi125 ceph-mon[200197]: pgmap v806: 129 pgs: 1 peering, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 22 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 4440/1119 objects degraded (396.783%); 2767/1119 objects misplaced (247.274%); 3.3 MiB/s, 1 keys/s, 363 objects/s recovering 2023-10-09T10:57:12.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:11 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:12.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:11 smithi125 ceph-mon[195507]: failed to encode map e326 with expected crc 2023-10-09T10:57:12.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:11 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:57:12.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:11 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/93565560' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.9811576053054418}]': finished 2023-10-09T10:57:12.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:11 smithi125 ceph-mon[195507]: osdmap e326: 8 total, 8 up, 7 in 2023-10-09T10:57:12.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:11 smithi125 ceph-mon[195507]: failed to encode map e326 with expected crc 2023-10-09T10:57:12.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:11 smithi125 ceph-mon[195507]: pgmap v806: 129 pgs: 1 peering, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 22 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 4440/1119 objects degraded (396.783%); 2767/1119 objects misplaced (247.274%); 3.3 MiB/s, 1 keys/s, 363 objects/s recovering 2023-10-09T10:57:12.582 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:57:12.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:12 smithi155 ceph-mon[184854]: failed to encode map e326 with expected crc 2023-10-09T10:57:12.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:12 smithi155 ceph-mon[184854]: failed to encode map e326 with expected crc 2023-10-09T10:57:12.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:12 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:57:12.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:12 smithi155 ceph-mon[184854]: failed to encode map e327 with expected crc 2023-10-09T10:57:12.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:12 smithi155 ceph-mon[184854]: osdmap e327: 8 total, 8 up, 7 in 2023-10-09T10:57:12.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:12 smithi155 ceph-mon[184854]: failed to encode map e327 with expected crc 2023-10-09T10:57:12.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:12 smithi155 ceph-mon[184854]: failed to encode map e327 with expected crc 2023-10-09T10:57:12.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:12 smithi155 ceph-mon[184854]: failed to encode map e327 with expected crc 2023-10-09T10:57:13.146 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[195507]: failed to encode map e326 with expected crc 2023-10-09T10:57:13.146 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[195507]: failed to encode map e326 with expected crc 2023-10-09T10:57:13.146 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:57:13.147 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[195507]: failed to encode map e327 with expected crc 2023-10-09T10:57:13.147 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[195507]: osdmap e327: 8 total, 8 up, 7 in 2023-10-09T10:57:13.147 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[195507]: failed to encode map e327 with expected crc 2023-10-09T10:57:13.147 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[195507]: failed to encode map e327 with expected crc 2023-10-09T10:57:13.147 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[195507]: failed to encode map e327 with expected crc 2023-10-09T10:57:13.147 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[200197]: failed to encode map e326 with expected crc 2023-10-09T10:57:13.147 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[200197]: failed to encode map e326 with expected crc 2023-10-09T10:57:13.147 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:57:13.147 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[200197]: failed to encode map e327 with expected crc 2023-10-09T10:57:13.147 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[200197]: osdmap e327: 8 total, 8 up, 7 in 2023-10-09T10:57:13.147 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[200197]: failed to encode map e327 with expected crc 2023-10-09T10:57:13.148 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[200197]: failed to encode map e327 with expected crc 2023-10-09T10:57:13.148 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:12 smithi125 ceph-mon[200197]: failed to encode map e327 with expected crc 2023-10-09T10:57:13.583 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:57:13.583 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:57:13.953 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:57:14.181 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:14.219 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:57:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:13 smithi155 ceph-mon[184854]: osdmap e328: 8 total, 8 up, 7 in 2023-10-09T10:57:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:13 smithi155 ceph-mon[184854]: failed to encode map e328 with expected crc 2023-10-09T10:57:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:13 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 4440/1119 objects degraded (396.783%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:57:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:13 smithi155 ceph-mon[184854]: failed to encode map e328 with expected crc 2023-10-09T10:57:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:13 smithi155 ceph-mon[184854]: pgmap v809: 129 pgs: 1 peering, 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 13 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 331 op/s; 4440/690 objects degraded (643.478%); 2726/690 objects misplaced (395.072%); 1.5 MiB/s, 269 objects/s recovering 2023-10-09T10:57:14.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:13 smithi125 ceph-mon[200197]: osdmap e328: 8 total, 8 up, 7 in 2023-10-09T10:57:14.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:13 smithi125 ceph-mon[200197]: failed to encode map e328 with expected crc 2023-10-09T10:57:14.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:13 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 4440/1119 objects degraded (396.783%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:57:14.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:13 smithi125 ceph-mon[200197]: failed to encode map e328 with expected crc 2023-10-09T10:57:14.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:13 smithi125 ceph-mon[200197]: pgmap v809: 129 pgs: 1 peering, 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 13 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 331 op/s; 4440/690 objects degraded (643.478%); 2726/690 objects misplaced (395.072%); 1.5 MiB/s, 269 objects/s recovering 2023-10-09T10:57:14.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:14.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:14.264 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:14.264 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:13 smithi125 ceph-mon[195507]: osdmap e328: 8 total, 8 up, 7 in 2023-10-09T10:57:14.264 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:13 smithi125 ceph-mon[195507]: failed to encode map e328 with expected crc 2023-10-09T10:57:14.264 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:13 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 4440/1119 objects degraded (396.783%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:57:14.264 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:13 smithi125 ceph-mon[195507]: failed to encode map e328 with expected crc 2023-10-09T10:57:14.264 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:13 smithi125 ceph-mon[195507]: pgmap v809: 129 pgs: 1 peering, 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 13 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 331 op/s; 4440/690 objects degraded (643.478%); 2726/690 objects misplaced (395.072%); 1.5 MiB/s, 269 objects/s recovering 2023-10-09T10:57:14.364 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:14.393 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:14.454 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:14.462 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:14.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:14.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:14.566 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:14.598 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:14.818 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:14.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:14.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:14.922 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:14.954 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:15.024 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:15.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:15.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:15.128 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:15.153 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:14 smithi125 ceph-mon[195507]: failed to encode map e328 with expected crc 2023-10-09T10:57:15.153 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:14 smithi125 ceph-mon[200197]: failed to encode map e328 with expected crc 2023-10-09T10:57:15.156 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:15.175 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:15.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:15.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:15.192 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:57:15.193 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:57:15.279 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:15.310 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:15.414 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:14 smithi155 ceph-mon[184854]: failed to encode map e328 with expected crc 2023-10-09T10:57:15.429 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:15.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:15.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:15.534 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:15.563 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:15.583 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:15.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:15.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:15.596 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:57:15.687 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:15.721 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:15.748 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:15.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:15.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:15.833 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:57:15.852 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:15.854 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:57:15.874 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:57:15.881 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:15.895 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:57:15.904 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:15.904 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:57:15.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:15.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:15.913 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:57:15.932 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:57:15.951 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:57:15.971 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:57:15.972 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:57:15.972 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:57:15.972 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:57:15.972 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:57:15.973 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:57:15.973 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:57:15.973 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:57:15.973 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:57:16.008 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:16.039 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:16.066 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_2' removed 2023-10-09T10:57:16.217 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:16.217 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:57:16.217 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:16 smithi155 ceph-mon[184854]: pgmap v810: 129 pgs: 1 peering, 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 13 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 4440/690 objects degraded (643.478%); 2726/690 objects misplaced (395.072%); 11 KiB/s, 237 objects/s recovering 2023-10-09T10:57:16.217 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:16 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3219266961' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:16.217 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:16 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3836222588' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:57:16.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:16.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:57:16.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:16 smithi125 ceph-mon[195507]: pgmap v810: 129 pgs: 1 peering, 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 13 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 4440/690 objects degraded (643.478%); 2726/690 objects misplaced (395.072%); 11 KiB/s, 237 objects/s recovering 2023-10-09T10:57:16.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:16 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3219266961' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:16.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:16 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3836222588' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:57:16.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:16.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:57:16.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:16 smithi125 ceph-mon[200197]: pgmap v810: 129 pgs: 1 peering, 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 13 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 4440/690 objects degraded (643.478%); 2726/690 objects misplaced (395.072%); 11 KiB/s, 237 objects/s recovering 2023-10-09T10:57:16.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:16 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3219266961' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:16.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:16 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3836222588' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T10:57:16.372 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:16.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:16.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:16.460 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:57:16.477 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:16.505 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:16.524 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:16.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:16.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:16.626 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T10:57:16.627 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T10:57:16.627 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-10-09T10:57:16.627 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-10-09T10:57:16.628 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:16.666 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:16.787 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:16.787 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:16.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:16.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:16.890 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:16.920 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:16.940 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:16.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:16.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:16.997 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:57:16.998 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:57:16.998 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:57:16.998 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:57:16.998 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:57:17.043 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:17.072 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:17.176 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:17.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:17 smithi125 ceph-mon[200197]: failed to encode map e329 with expected crc 2023-10-09T10:57:17.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3836222588' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:57:17.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:17 smithi125 ceph-mon[200197]: osdmap e329: 8 total, 8 up, 7 in 2023-10-09T10:57:17.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:17 smithi125 ceph-mon[200197]: failed to encode map e329 with expected crc 2023-10-09T10:57:17.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:17 smithi125 ceph-mon[195507]: failed to encode map e329 with expected crc 2023-10-09T10:57:17.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3836222588' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:57:17.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:17 smithi125 ceph-mon[195507]: osdmap e329: 8 total, 8 up, 7 in 2023-10-09T10:57:17.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:17 smithi125 ceph-mon[195507]: failed to encode map e329 with expected crc 2023-10-09T10:57:17.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:17.297 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:17 smithi155 ceph-mon[184854]: failed to encode map e329 with expected crc 2023-10-09T10:57:17.297 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3836222588' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T10:57:17.297 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:17 smithi155 ceph-mon[184854]: osdmap e329: 8 total, 8 up, 7 in 2023-10-09T10:57:17.297 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:17 smithi155 ceph-mon[184854]: failed to encode map e329 with expected crc 2023-10-09T10:57:17.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:17.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:17.313 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:57:17.399 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:17.428 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:17.447 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:17.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:17.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:17.551 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:17.580 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:17.599 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:17.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:17.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:17.616 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:57:17.705 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:17.735 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:17.758 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:17.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:17.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:17.862 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:17.889 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:17.908 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:17.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:17.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:18.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:18.040 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:18.167 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 2, 1] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:57:18.167 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:57:18.167 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-09T10:57:18.167 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-09T10:57:18.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:18 smithi155 ceph-mon[184854]: failed to encode map e329 with expected crc 2023-10-09T10:57:18.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:18 smithi155 ceph-mon[184854]: pgmap v812: 103 pgs: 103 active+clean; 665 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T10:57:18.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:18 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 4440/690 objects degraded (643.478%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:57:18.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2284351424' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:18.211 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:18.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:18.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:18.316 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:18.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:18 smithi125 ceph-mon[195507]: failed to encode map e329 with expected crc 2023-10-09T10:57:18.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:18 smithi125 ceph-mon[195507]: pgmap v812: 103 pgs: 103 active+clean; 665 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T10:57:18.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:18 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 4440/690 objects degraded (643.478%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:57:18.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:18 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2284351424' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:18.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:18 smithi125 ceph-mon[200197]: failed to encode map e329 with expected crc 2023-10-09T10:57:18.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:18 smithi125 ceph-mon[200197]: pgmap v812: 103 pgs: 103 active+clean; 665 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T10:57:18.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:18 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 4440/690 objects degraded (643.478%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T10:57:18.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:18 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2284351424' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:18.345 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:18.370 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:18.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:18.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:18.387 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:57:18.453 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:18.474 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:18.503 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:18.553 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:57:18.553 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:57:18.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:57:18.553 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:57:18.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.554 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:95197 7.d 7:b9b00ad5:::benchmark_data_smithi125_295816_object4060:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-10-09T10:57:18.554 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:57.383916+0000", 2023-10-09T10:57:18.554 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 381.16646096599999, 2023-10-09T10:57:18.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.070851236999999997, 2023-10-09T10:57:18.554 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.554 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.554 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:57:18.555 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:57:18.555 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:57:18.555 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 95197 2023-10-09T10:57:18.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.555 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383916+0000", 2023-10-09T10:57:18.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383916+0000", 2023-10-09T10:57:18.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.557 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383915+0000", 2023-10-09T10:57:18.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:18.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.557 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383920+0000", 2023-10-09T10:57:18.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.84e-06 2023-10-09T10:57:18.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383931+0000", 2023-10-09T10:57:18.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0657e-05 2023-10-09T10:57:18.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383938+0000", 2023-10-09T10:57:18.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5139999999999998e-06 2023-10-09T10:57:18.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.411508+0000", 2023-10-09T10:57:18.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027570921000000002 2023-10-09T10:57:18.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:57:18.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.416286+0000", 2023-10-09T10:57:18.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0047777430000000001 2023-10-09T10:57:18.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.562 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.437950+0000", 2023-10-09T10:57:18.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021664251999999998 2023-10-09T10:57:18.562 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.562 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.562 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.438046+0000", 2023-10-09T10:57:18.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5459999999999997e-05 2023-10-09T10:57:18.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:57:18.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.438186+0000", 2023-10-09T10:57:18.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014000800000000001 2023-10-09T10:57:18.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:57:18.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.443989+0000", 2023-10-09T10:57:18.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0058031369999999999 2023-10-09T10:57:18.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.452036+0000", 2023-10-09T10:57:18.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0080469280000000001 2023-10-09T10:57:18.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.452814+0000", 2023-10-09T10:57:18.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00077831199999999997 2023-10-09T10:57:18.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.452840+0000", 2023-10-09T10:57:18.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5947000000000001e-05 2023-10-09T10:57:18.567 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.567 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.567 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.567 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.454767+0000", 2023-10-09T10:57:18.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019270769999999999 2023-10-09T10:57:18.567 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.568 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.568 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.568 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:95787 7.d 7:b6e713eb:::benchmark_data_smithi125_295816_object4650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:57:18.568 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:58.300479+0000", 2023-10-09T10:57:18.568 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 380.24989853300002, 2023-10-09T10:57:18.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072656459000000007, 2023-10-09T10:57:18.569 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.569 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.569 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:57:18.569 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:57:18.569 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:57:18.569 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 95787 2023-10-09T10:57:18.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.570 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300479+0000", 2023-10-09T10:57:18.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300479+0000", 2023-10-09T10:57:18.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300477+0000", 2023-10-09T10:57:18.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:57:18.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300487+0000", 2023-10-09T10:57:18.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0638999999999999e-05 2023-10-09T10:57:18.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300511+0000", 2023-10-09T10:57:18.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3059999999999999e-05 2023-10-09T10:57:18.574 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.574 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.574 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300518+0000", 2023-10-09T10:57:18.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0029999999999997e-06 2023-10-09T10:57:18.574 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.575 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.322068+0000", 2023-10-09T10:57:18.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021550625 2023-10-09T10:57:18.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:57:18.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.333963+0000", 2023-10-09T10:57:18.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011895039 2023-10-09T10:57:18.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.576 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.351448+0000", 2023-10-09T10:57:18.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017484831999999999 2023-10-09T10:57:18.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.352038+0000", 2023-10-09T10:57:18.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00059005600000000004 2023-10-09T10:57:18.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:57:18.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.352181+0000", 2023-10-09T10:57:18.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014277800000000001 2023-10-09T10:57:18.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.367667+0000", 2023-10-09T10:57:18.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015486592 2023-10-09T10:57:18.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.579 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:57:18.579 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.370532+0000", 2023-10-09T10:57:18.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028642279999999999 2023-10-09T10:57:18.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.580 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.371142+0000", 2023-10-09T10:57:18.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00060984699999999995 2023-10-09T10:57:18.580 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.580 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.581 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.371180+0000", 2023-10-09T10:57:18.581 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8529000000000003e-05 2023-10-09T10:57:18.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.581 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.581 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.581 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.373135+0000", 2023-10-09T10:57:18.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001954973 2023-10-09T10:57:18.582 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.582 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.583 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.583 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.584 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96817 7.d 7:b9f7ad08:::benchmark_data_smithi125_295816_object5680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:57:18.584 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:59.884613+0000", 2023-10-09T10:57:18.584 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 378.66576407999997, 2023-10-09T10:57:18.584 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14656745900000001, 2023-10-09T10:57:18.584 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.584 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.585 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:57:18.585 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:57:18.585 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:57:18.585 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96817 2023-10-09T10:57:18.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.586 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884613+0000", 2023-10-09T10:57:18.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884613+0000", 2023-10-09T10:57:18.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884612+0000", 2023-10-09T10:57:18.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:18.588 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.588 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.588 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884618+0000", 2023-10-09T10:57:18.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5880000000000003e-06 2023-10-09T10:57:18.588 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.589 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.589 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.589 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884633+0000", 2023-10-09T10:57:18.589 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5109999999999999e-05 2023-10-09T10:57:18.589 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884639+0000", 2023-10-09T10:57:18.590 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.164e-06 2023-10-09T10:57:18.590 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.887892+0000", 2023-10-09T10:57:18.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032536610000000001 2023-10-09T10:57:18.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.591 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:57:18.591 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.888551+0000", 2023-10-09T10:57:18.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00065867400000000004 2023-10-09T10:57:18.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.591 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.591 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910766+0000", 2023-10-09T10:57:18.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022214655 2023-10-09T10:57:18.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910852+0000", 2023-10-09T10:57:18.592 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6551999999999994e-05 2023-10-09T10:57:18.592 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.592 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:57:18.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910980+0000", 2023-10-09T10:57:18.592 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000127704 2023-10-09T10:57:18.592 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.592 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:57:18.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.914262+0000", 2023-10-09T10:57:18.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032824070000000002 2023-10-09T10:57:18.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.593 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.593 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.020644+0000", 2023-10-09T10:57:18.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.106381932 2023-10-09T10:57:18.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.593 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.593 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.029029+0000", 2023-10-09T10:57:18.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0083848010000000008 2023-10-09T10:57:18.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.594 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.594 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.594 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.029088+0000", 2023-10-09T10:57:18.594 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8641999999999999e-05 2023-10-09T10:57:18.594 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.594 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.594 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.594 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.031180+0000", 2023-10-09T10:57:18.594 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020928359999999998 2023-10-09T10:57:18.594 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.594 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.594 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.595 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.595 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.595 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96884 7.d 7:b183b169:::benchmark_data_smithi125_295816_object5747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:57:18.595 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.113486+0000", 2023-10-09T10:57:18.595 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 378.43689079000001, 2023-10-09T10:57:18.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.082588165000000005, 2023-10-09T10:57:18.595 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.595 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.595 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:57:18.595 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:57:18.595 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:57:18.595 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96884 2023-10-09T10:57:18.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.596 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.596 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.596 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.596 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113486+0000", 2023-10-09T10:57:18.596 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.596 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.596 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.596 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113486+0000", 2023-10-09T10:57:18.596 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113485+0000", 2023-10-09T10:57:18.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:57:18.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113491+0000", 2023-10-09T10:57:18.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3009999999999997e-06 2023-10-09T10:57:18.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113501+0000", 2023-10-09T10:57:18.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0342e-05 2023-10-09T10:57:18.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113505+0000", 2023-10-09T10:57:18.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4530000000000004e-06 2023-10-09T10:57:18.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.117310+0000", 2023-10-09T10:57:18.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0038049329999999999 2023-10-09T10:57:18.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.599 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.599 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:57:18.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.117924+0000", 2023-10-09T10:57:18.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00061396699999999998 2023-10-09T10:57:18.599 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.599 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.599 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.140859+0000", 2023-10-09T10:57:18.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022935140999999999 2023-10-09T10:57:18.599 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.599 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.599 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.140963+0000", 2023-10-09T10:57:18.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010341 2023-10-09T10:57:18.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.600 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:57:18.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.141120+0000", 2023-10-09T10:57:18.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000157405 2023-10-09T10:57:18.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.600 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:57:18.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.150528+0000", 2023-10-09T10:57:18.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0094081660000000008 2023-10-09T10:57:18.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.160789+0000", 2023-10-09T10:57:18.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010260429 2023-10-09T10:57:18.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.194166+0000", 2023-10-09T10:57:18.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.033377636000000002 2023-10-09T10:57:18.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.194226+0000", 2023-10-09T10:57:18.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9855999999999997e-05 2023-10-09T10:57:18.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196074+0000", 2023-10-09T10:57:18.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001848192 2023-10-09T10:57:18.602 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.602 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.602 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.603 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96892 7.d 7:bd59fbdb:::benchmark_data_smithi125_295816_object5755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:57:18.603 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.148429+0000", 2023-10-09T10:57:18.603 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 378.401948538, 2023-10-09T10:57:18.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.089082649999999999, 2023-10-09T10:57:18.603 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.603 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.603 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:57:18.603 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:57:18.603 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:57:18.603 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96892 2023-10-09T10:57:18.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.603 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148429+0000", 2023-10-09T10:57:18.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148429+0000", 2023-10-09T10:57:18.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148428+0000", 2023-10-09T10:57:18.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967296 2023-10-09T10:57:18.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148430+0000", 2023-10-09T10:57:18.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5570000000000001e-06 2023-10-09T10:57:18.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148433+0000", 2023-10-09T10:57:18.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.892e-06 2023-10-09T10:57:18.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148435+0000", 2023-10-09T10:57:18.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5339999999999998e-06 2023-10-09T10:57:18.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.152891+0000", 2023-10-09T10:57:18.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0044553409999999998 2023-10-09T10:57:18.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:57:18.607 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.153532+0000", 2023-10-09T10:57:18.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000641539 2023-10-09T10:57:18.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.607 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.182585+0000", 2023-10-09T10:57:18.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029053295 2023-10-09T10:57:18.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.608 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.182682+0000", 2023-10-09T10:57:18.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6623e-05 2023-10-09T10:57:18.608 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.608 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.608 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:57:18.608 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.182825+0000", 2023-10-09T10:57:18.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014288300000000001 2023-10-09T10:57:18.608 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.608 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.608 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:57:18.608 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.198214+0000", 2023-10-09T10:57:18.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015389162 2023-10-09T10:57:18.608 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.223087+0000", 2023-10-09T10:57:18.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024872423000000001 2023-10-09T10:57:18.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.235280+0000", 2023-10-09T10:57:18.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012193017 2023-10-09T10:57:18.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.235328+0000", 2023-10-09T10:57:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8211000000000003e-05 2023-10-09T10:57:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.237511+0000", 2023-10-09T10:57:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021833960000000002 2023-10-09T10:57:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.611 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96898 7.d 7:b4ab753b:::benchmark_data_smithi125_295816_object5761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:57:18.611 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.166362+0000", 2023-10-09T10:57:18.611 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 378.38401504400002, 2023-10-09T10:57:18.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072656032999999995, 2023-10-09T10:57:18.611 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.611 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.611 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:57:18.611 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:57:18.611 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:57:18.611 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96898 2023-10-09T10:57:18.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.611 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166362+0000", 2023-10-09T10:57:18.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166362+0000", 2023-10-09T10:57:18.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.613 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166360+0000", 2023-10-09T10:57:18.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:57:18.613 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.613 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.613 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.613 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166369+0000", 2023-10-09T10:57:18.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.166e-06 2023-10-09T10:57:18.613 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.613 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.613 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.613 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166385+0000", 2023-10-09T10:57:18.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5214e-05 2023-10-09T10:57:18.614 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.614 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.614 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.614 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166392+0000", 2023-10-09T10:57:18.614 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1659999999999997e-06 2023-10-09T10:57:18.614 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.614 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.614 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.614 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166446+0000", 2023-10-09T10:57:18.614 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3819000000000001e-05 2023-10-09T10:57:18.614 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.615 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.615 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:57:18.615 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.167178+0000", 2023-10-09T10:57:18.615 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00073199300000000001 2023-10-09T10:57:18.615 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.615 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.615 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.615 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.191480+0000", 2023-10-09T10:57:18.615 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024302082999999999 2023-10-09T10:57:18.615 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.616 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.191594+0000", 2023-10-09T10:57:18.616 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000114527 2023-10-09T10:57:18.616 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:57:18.616 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.191761+0000", 2023-10-09T10:57:18.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000166599 2023-10-09T10:57:18.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:57:18.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.198249+0000", 2023-10-09T10:57:18.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064882439999999998 2023-10-09T10:57:18.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.223116+0000", 2023-10-09T10:57:18.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024867330999999999 2023-10-09T10:57:18.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.237546+0000", 2023-10-09T10:57:18.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014429365 2023-10-09T10:57:18.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.619 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.237573+0000", 2023-10-09T10:57:18.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7534000000000001e-05 2023-10-09T10:57:18.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.620 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.620 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.620 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.239018+0000", 2023-10-09T10:57:18.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001444776 2023-10-09T10:57:18.620 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.620 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.620 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.620 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.621 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.621 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96901 7.d 7:bd396a29:::benchmark_data_smithi125_295816_object5764:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:57:18.621 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.177994+0000", 2023-10-09T10:57:18.621 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 378.37238322299999, 2023-10-09T10:57:18.621 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.079151244999999995, 2023-10-09T10:57:18.621 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.621 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.622 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:57:18.622 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:57:18.622 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:57:18.622 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96901 2023-10-09T10:57:18.622 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.622 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.622 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.622 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.623 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177994+0000", 2023-10-09T10:57:18.623 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.623 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.623 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.623 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.623 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177994+0000", 2023-10-09T10:57:18.623 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.624 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.624 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177993+0000", 2023-10-09T10:57:18.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:57:18.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.624 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177998+0000", 2023-10-09T10:57:18.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6699999999999999e-06 2023-10-09T10:57:18.625 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.178008+0000", 2023-10-09T10:57:18.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6520000000000001e-06 2023-10-09T10:57:18.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.626 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.178013+0000", 2023-10-09T10:57:18.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.028e-06 2023-10-09T10:57:18.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196130+0000", 2023-10-09T10:57:18.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018116621999999999 2023-10-09T10:57:18.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:57:18.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196670+0000", 2023-10-09T10:57:18.628 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00054042399999999996 2023-10-09T10:57:18.628 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.628 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.628 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.227020+0000", 2023-10-09T10:57:18.628 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030349832 2023-10-09T10:57:18.628 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.628 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.628 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.227095+0000", 2023-10-09T10:57:18.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4785999999999999e-05 2023-10-09T10:57:18.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.629 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.629 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:57:18.629 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.227204+0000", 2023-10-09T10:57:18.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000109079 2023-10-09T10:57:18.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.629 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.629 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:57:18.629 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.239698+0000", 2023-10-09T10:57:18.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012494081000000001 2023-10-09T10:57:18.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.629 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.630 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.630 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242208+0000", 2023-10-09T10:57:18.630 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00251001 2023-10-09T10:57:18.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.630 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.630 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.254821+0000", 2023-10-09T10:57:18.630 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012612978 2023-10-09T10:57:18.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.630 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.630 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.254858+0000", 2023-10-09T10:57:18.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6785000000000001e-05 2023-10-09T10:57:18.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.257145+0000", 2023-10-09T10:57:18.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002287374 2023-10-09T10:57:18.631 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.631 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.631 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.631 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96908 7.d 7:bd9652bb:::benchmark_data_smithi125_295816_object5771:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:57:18.632 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.184970+0000", 2023-10-09T10:57:18.632 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 378.36540683099997, 2023-10-09T10:57:18.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.107155712, 2023-10-09T10:57:18.632 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.632 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.632 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:57:18.632 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:57:18.632 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:57:18.632 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96908 2023-10-09T10:57:18.632 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.632 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184970+0000", 2023-10-09T10:57:18.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184970+0000", 2023-10-09T10:57:18.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184969+0000", 2023-10-09T10:57:18.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:18.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184976+0000", 2023-10-09T10:57:18.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0849999999999999e-06 2023-10-09T10:57:18.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184988+0000", 2023-10-09T10:57:18.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2438e-05 2023-10-09T10:57:18.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.635 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184994+0000", 2023-10-09T10:57:18.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.835e-06 2023-10-09T10:57:18.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.635 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196710+0000", 2023-10-09T10:57:18.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011715557 2023-10-09T10:57:18.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.635 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:57:18.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.197312+0000", 2023-10-09T10:57:18.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000602675 2023-10-09T10:57:18.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.232847+0000", 2023-10-09T10:57:18.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.035534143999999997 2023-10-09T10:57:18.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.232937+0000", 2023-10-09T10:57:18.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0303999999999995e-05 2023-10-09T10:57:18.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:57:18.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.233064+0000", 2023-10-09T10:57:18.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012689799999999999 2023-10-09T10:57:18.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:57:18.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242946+0000", 2023-10-09T10:57:18.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0098825000000000007 2023-10-09T10:57:18.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.269531+0000", 2023-10-09T10:57:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026584907000000001 2023-10-09T10:57:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.289926+0000", 2023-10-09T10:57:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020394791999999998 2023-10-09T10:57:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.289981+0000", 2023-10-09T10:57:18.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4540999999999998e-05 2023-10-09T10:57:18.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.292126+0000", 2023-10-09T10:57:18.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021454270000000001 2023-10-09T10:57:18.639 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.639 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.639 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.639 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96935 7.d 7:b8c4db12:::benchmark_data_smithi125_295816_object5798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:57:18.640 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.240816+0000", 2023-10-09T10:57:18.640 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 378.30956094999999, 2023-10-09T10:57:18.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.096789209000000001, 2023-10-09T10:57:18.640 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.640 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.640 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:57:18.640 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:57:18.640 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:57:18.640 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96935 2023-10-09T10:57:18.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.640 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240816+0000", 2023-10-09T10:57:18.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240816+0000", 2023-10-09T10:57:18.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240815+0000", 2023-10-09T10:57:18.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:18.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.642 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240821+0000", 2023-10-09T10:57:18.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1140000000000002e-06 2023-10-09T10:57:18.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.642 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240831+0000", 2023-10-09T10:57:18.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6309999999999998e-06 2023-10-09T10:57:18.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.643 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.643 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240845+0000", 2023-10-09T10:57:18.643 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4705000000000001e-05 2023-10-09T10:57:18.643 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.643 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.643 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.643 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242238+0000", 2023-10-09T10:57:18.643 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001392646 2023-10-09T10:57:18.643 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.643 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.643 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:57:18.643 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242937+0000", 2023-10-09T10:57:18.644 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00069863399999999995 2023-10-09T10:57:18.644 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.644 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.644 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.644 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.295227+0000", 2023-10-09T10:57:18.644 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.052290652999999999 2023-10-09T10:57:18.644 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.644 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.644 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.644 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.295329+0000", 2023-10-09T10:57:18.644 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010212700000000001 2023-10-09T10:57:18.644 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:57:18.645 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.295755+0000", 2023-10-09T10:57:18.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00042587400000000002 2023-10-09T10:57:18.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:57:18.645 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.308570+0000", 2023-10-09T10:57:18.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012814397 2023-10-09T10:57:18.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.329769+0000", 2023-10-09T10:57:18.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021199234000000001 2023-10-09T10:57:18.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.335181+0000", 2023-10-09T10:57:18.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0054126360000000002 2023-10-09T10:57:18.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.335231+0000", 2023-10-09T10:57:18.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9344000000000001e-05 2023-10-09T10:57:18.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.647 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.647 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.337605+0000", 2023-10-09T10:57:18.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002374555 2023-10-09T10:57:18.647 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.647 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.647 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.647 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96940 7.d 7:bfb000f4:::benchmark_data_smithi125_295816_object5803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:57:18.647 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.268461+0000", 2023-10-09T10:57:18.648 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 378.28191640400001, 2023-10-09T10:57:18.648 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.071660901999999999, 2023-10-09T10:57:18.648 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.648 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.648 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:57:18.648 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:57:18.648 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:57:18.648 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96940 2023-10-09T10:57:18.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.648 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268461+0000", 2023-10-09T10:57:18.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.649 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268461+0000", 2023-10-09T10:57:18.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.649 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268459+0000", 2023-10-09T10:57:18.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:18.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.650 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.650 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268477+0000", 2023-10-09T10:57:18.650 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7377999999999999e-05 2023-10-09T10:57:18.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.650 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.650 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268488+0000", 2023-10-09T10:57:18.650 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1579e-05 2023-10-09T10:57:18.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.651 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.651 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268495+0000", 2023-10-09T10:57:18.651 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5429999999999999e-06 2023-10-09T10:57:18.651 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.651 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.651 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.651 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.273724+0000", 2023-10-09T10:57:18.651 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0052292049999999998 2023-10-09T10:57:18.651 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.651 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.651 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:57:18.651 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.274443+0000", 2023-10-09T10:57:18.652 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00071879099999999998 2023-10-09T10:57:18.652 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.652 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.652 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.652 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.306460+0000", 2023-10-09T10:57:18.652 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.032016797999999999 2023-10-09T10:57:18.652 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.652 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.652 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.652 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.306545+0000", 2023-10-09T10:57:18.652 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5277000000000004e-05 2023-10-09T10:57:18.652 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.653 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:57:18.653 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.306674+0000", 2023-10-09T10:57:18.653 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012941600000000001 2023-10-09T10:57:18.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.653 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:57:18.653 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.317498+0000", 2023-10-09T10:57:18.653 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010823523999999999 2023-10-09T10:57:18.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.653 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.329799+0000", 2023-10-09T10:57:18.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012301559 2023-10-09T10:57:18.654 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.337661+0000", 2023-10-09T10:57:18.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007861303 2023-10-09T10:57:18.654 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.337718+0000", 2023-10-09T10:57:18.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6974e-05 2023-10-09T10:57:18.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.655 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.655 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.340122+0000", 2023-10-09T10:57:18.655 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024038660000000002 2023-10-09T10:57:18.655 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.655 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.655 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.655 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:98648 7.d 7:bbc4eb35:::benchmark_data_smithi125_295816_object7511:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:57:18.655 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:02.635379+0000", 2023-10-09T10:57:18.656 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 375.91499849399997, 2023-10-09T10:57:18.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.078747929999999994, 2023-10-09T10:57:18.656 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.656 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.656 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:57:18.656 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:57:18.656 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:57:18.656 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98648 2023-10-09T10:57:18.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.656 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635379+0000", 2023-10-09T10:57:18.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.657 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.657 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.657 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635379+0000", 2023-10-09T10:57:18.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.657 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.657 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.657 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.657 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635377+0000", 2023-10-09T10:57:18.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:18.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.658 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.658 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635382+0000", 2023-10-09T10:57:18.658 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3580000000000002e-06 2023-10-09T10:57:18.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.658 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.658 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635397+0000", 2023-10-09T10:57:18.658 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4265e-05 2023-10-09T10:57:18.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.659 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635403+0000", 2023-10-09T10:57:18.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4049999999999997e-06 2023-10-09T10:57:18.659 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.659 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.654397+0000", 2023-10-09T10:57:18.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018993432000000001 2023-10-09T10:57:18.659 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:57:18.659 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.661791+0000", 2023-10-09T10:57:18.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007394655 2023-10-09T10:57:18.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.660 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.671916+0000", 2023-10-09T10:57:18.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010125187000000001 2023-10-09T10:57:18.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.660 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.671975+0000", 2023-10-09T10:57:18.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.897e-05 2023-10-09T10:57:18.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.661 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:57:18.661 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.672065+0000", 2023-10-09T10:57:18.661 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9809999999999996e-05 2023-10-09T10:57:18.661 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.661 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:57:18.661 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.682405+0000", 2023-10-09T10:57:18.661 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010339924 2023-10-09T10:57:18.661 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.661 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.661 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.682443+0000", 2023-10-09T10:57:18.661 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7886000000000003e-05 2023-10-09T10:57:18.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.662 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.662 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.662 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.712060+0000", 2023-10-09T10:57:18.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029616844 2023-10-09T10:57:18.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.662 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.662 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.662 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.712116+0000", 2023-10-09T10:57:18.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6081000000000001e-05 2023-10-09T10:57:18.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.662 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.663 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.663 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.714127+0000", 2023-10-09T10:57:18.663 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002010572 2023-10-09T10:57:18.663 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.663 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.663 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.663 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.663 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:98673 7.d 7:b818009b:::benchmark_data_smithi125_295816_object7536:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:57:18.663 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:02.680579+0000", 2023-10-09T10:57:18.663 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 375.86979826300001, 2023-10-09T10:57:18.663 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.095416154000000003, 2023-10-09T10:57:18.663 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.664 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.664 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:57:18.664 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:57:18.664 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:57:18.664 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98673 2023-10-09T10:57:18.664 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.664 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.664 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.664 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.664 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680579+0000", 2023-10-09T10:57:18.664 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.664 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.665 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.665 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.665 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680579+0000", 2023-10-09T10:57:18.665 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.665 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.665 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.665 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.665 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680578+0000", 2023-10-09T10:57:18.665 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:57:18.665 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.665 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.665 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.666 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680582+0000", 2023-10-09T10:57:18.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8e-06 2023-10-09T10:57:18.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.666 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.666 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680588+0000", 2023-10-09T10:57:18.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4670000000000001e-06 2023-10-09T10:57:18.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.666 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.666 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680591+0000", 2023-10-09T10:57:18.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9519999999999999e-06 2023-10-09T10:57:18.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.727779+0000", 2023-10-09T10:57:18.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047187937999999999 2023-10-09T10:57:18.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:57:18.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.728358+0000", 2023-10-09T10:57:18.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00057898100000000001 2023-10-09T10:57:18.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.740190+0000", 2023-10-09T10:57:18.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01183184 2023-10-09T10:57:18.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.740303+0000", 2023-10-09T10:57:18.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112762 2023-10-09T10:57:18.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:57:18.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.740474+0000", 2023-10-09T10:57:18.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017171100000000001 2023-10-09T10:57:18.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:57:18.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.746764+0000", 2023-10-09T10:57:18.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062892499999999997 2023-10-09T10:57:18.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.756355+0000", 2023-10-09T10:57:18.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0095916060000000008 2023-10-09T10:57:18.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.774088+0000", 2023-10-09T10:57:18.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017732947999999998 2023-10-09T10:57:18.670 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.774137+0000", 2023-10-09T10:57:18.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8616999999999999e-05 2023-10-09T10:57:18.670 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.775995+0000", 2023-10-09T10:57:18.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018582340000000001 2023-10-09T10:57:18.671 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.671 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.671 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.671 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.671 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:98701 7.d 7:bd16628e:::benchmark_data_smithi125_295816_object7564:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:57:18.671 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:02.717662+0000", 2023-10-09T10:57:18.671 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 375.83271491199997, 2023-10-09T10:57:18.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.079670053000000005, 2023-10-09T10:57:18.671 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.671 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.671 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:57:18.672 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:57:18.672 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:57:18.672 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98701 2023-10-09T10:57:18.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.672 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.672 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.672 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717662+0000", 2023-10-09T10:57:18.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.672 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.672 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717662+0000", 2023-10-09T10:57:18.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717661+0000", 2023-10-09T10:57:18.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:57:18.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717666+0000", 2023-10-09T10:57:18.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.22e-06 2023-10-09T10:57:18.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717676+0000", 2023-10-09T10:57:18.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.944e-06 2023-10-09T10:57:18.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717681+0000", 2023-10-09T10:57:18.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3719999999999998e-06 2023-10-09T10:57:18.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.729691+0000", 2023-10-09T10:57:18.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012010140000000001 2023-10-09T10:57:18.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:57:18.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.732552+0000", 2023-10-09T10:57:18.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028607770000000001 2023-10-09T10:57:18.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.771785+0000", 2023-10-09T10:57:18.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039233280000000002 2023-10-09T10:57:18.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.771847+0000", 2023-10-09T10:57:18.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1909000000000006e-05 2023-10-09T10:57:18.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:57:18.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.771940+0000", 2023-10-09T10:57:18.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3058000000000002e-05 2023-10-09T10:57:18.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:57:18.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.778867+0000", 2023-10-09T10:57:18.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0069268180000000004 2023-10-09T10:57:18.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.783491+0000", 2023-10-09T10:57:18.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0046242490000000004 2023-10-09T10:57:18.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:18.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.795089+0000", 2023-10-09T10:57:18.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011598489999999999 2023-10-09T10:57:18.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.795150+0000", 2023-10-09T10:57:18.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0517999999999999e-05 2023-10-09T10:57:18.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.797332+0000", 2023-10-09T10:57:18.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021824079999999998 2023-10-09T10:57:18.678 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.678 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.679 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.679 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.679 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:91615 8.f e286/282 8:fb4b59c0:::benchmark_data_smithi125_329074_object91614:head v 286'5764, mlcod=0'0)", 2023-10-09T10:57:18.679 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:32.633558+0000", 2023-10-09T10:57:18.679 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 165.91681883800001, 2023-10-09T10:57:18.679 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028974272999999998, 2023-10-09T10:57:18.679 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.679 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.679 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.679 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.679 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633558+0000", 2023-10-09T10:57:18.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.680 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.680 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.680 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.680 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633558+0000", 2023-10-09T10:57:18.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.680 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.680 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.680 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.680 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633557+0000", 2023-10-09T10:57:18.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:18.680 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.681 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633600+0000", 2023-10-09T10:57:18.681 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3139000000000002e-05 2023-10-09T10:57:18.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.681 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633614+0000", 2023-10-09T10:57:18.681 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3512999999999999e-05 2023-10-09T10:57:18.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633621+0000", 2023-10-09T10:57:18.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0550000000000002e-06 2023-10-09T10:57:18.682 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.682 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.682 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633685+0000", 2023-10-09T10:57:18.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4276999999999995e-05 2023-10-09T10:57:18.682 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.682 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.682 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633716+0000", 2023-10-09T10:57:18.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1134999999999999e-05 2023-10-09T10:57:18.682 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.683 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.683 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.662492+0000", 2023-10-09T10:57:18.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028775618999999999 2023-10-09T10:57:18.683 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.683 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.683 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.662533+0000", 2023-10-09T10:57:18.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0768e-05 2023-10-09T10:57:18.683 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.683 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.683 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.684 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.684 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.684 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:91623 8.7 e286/282 8:e8cb112d:::benchmark_data_smithi125_329074_object91622:head v 286'5732, mlcod=0'0)", 2023-10-09T10:57:18.684 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:32.634471+0000", 2023-10-09T10:57:18.684 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 165.91590605900001, 2023-10-09T10:57:18.684 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028088733000000001, 2023-10-09T10:57:18.684 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.684 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.684 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.684 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.684 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.684 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634471+0000", 2023-10-09T10:57:18.685 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.685 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.685 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.685 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.685 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634471+0000", 2023-10-09T10:57:18.685 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.685 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.685 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.685 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.685 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634470+0000", 2023-10-09T10:57:18.685 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:57:18.685 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.686 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.686 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.686 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634540+0000", 2023-10-09T10:57:18.686 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9406000000000006e-05 2023-10-09T10:57:18.686 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.686 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.686 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.686 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634556+0000", 2023-10-09T10:57:18.686 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5801999999999999e-05 2023-10-09T10:57:18.686 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.686 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.686 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.687 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634562+0000", 2023-10-09T10:57:18.687 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9430000000000003e-06 2023-10-09T10:57:18.687 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.687 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.687 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.687 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.640689+0000", 2023-10-09T10:57:18.687 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061267650000000002 2023-10-09T10:57:18.687 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.687 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.687 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.687 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.640718+0000", 2023-10-09T10:57:18.687 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9187e-05 2023-10-09T10:57:18.688 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.688 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.688 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.688 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.662537+0000", 2023-10-09T10:57:18.688 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021818627 2023-10-09T10:57:18.688 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.688 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.688 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.688 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.662560+0000", 2023-10-09T10:57:18.688 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2708999999999999e-05 2023-10-09T10:57:18.688 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.688 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.688 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.689 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.689 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.689 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:91868 8.f e286/282 8:f4bd67fe:::benchmark_data_smithi125_329074_object91867:head v 286'5787, mlcod=0'0)", 2023-10-09T10:57:18.689 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:33.101662+0000", 2023-10-09T10:57:18.689 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 165.448714709, 2023-10-09T10:57:18.689 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026291749, 2023-10-09T10:57:18.689 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.689 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.689 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.689 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.689 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.689 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101662+0000", 2023-10-09T10:57:18.690 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.690 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.690 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.690 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.690 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101662+0000", 2023-10-09T10:57:18.690 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.690 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.690 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.690 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.690 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101661+0000", 2023-10-09T10:57:18.690 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:18.691 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.691 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.691 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.691 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101711+0000", 2023-10-09T10:57:18.691 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9974e-05 2023-10-09T10:57:18.691 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.691 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.691 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.691 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101726+0000", 2023-10-09T10:57:18.691 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4945e-05 2023-10-09T10:57:18.691 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.691 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.691 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.692 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101733+0000", 2023-10-09T10:57:18.692 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1350000000000003e-06 2023-10-09T10:57:18.692 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.692 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.692 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.692 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.105791+0000", 2023-10-09T10:57:18.692 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040581360000000004 2023-10-09T10:57:18.692 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.692 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.692 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.692 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.105821+0000", 2023-10-09T10:57:18.693 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9422999999999999e-05 2023-10-09T10:57:18.693 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.693 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.693 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.693 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.127912+0000", 2023-10-09T10:57:18.693 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022090789999999999 2023-10-09T10:57:18.693 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.693 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.693 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.693 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.127954+0000", 2023-10-09T10:57:18.693 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2577999999999998e-05 2023-10-09T10:57:18.693 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.693 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.694 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.694 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.694 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.694 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:92334 8.f e286/282 8:fe8e3200:::benchmark_data_smithi125_329074_object92333:head v 286'5819, mlcod=0'0)", 2023-10-09T10:57:18.694 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:34.002645+0000", 2023-10-09T10:57:18.694 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 164.547731755, 2023-10-09T10:57:18.694 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028096320000000001, 2023-10-09T10:57:18.694 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.694 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.694 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.694 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.695 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.695 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002645+0000", 2023-10-09T10:57:18.695 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.695 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.695 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.695 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.695 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002645+0000", 2023-10-09T10:57:18.695 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.695 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.695 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.695 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.696 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002644+0000", 2023-10-09T10:57:18.696 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:57:18.696 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.696 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.696 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.696 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002709+0000", 2023-10-09T10:57:18.696 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4896999999999999e-05 2023-10-09T10:57:18.696 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.696 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.696 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.696 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002724+0000", 2023-10-09T10:57:18.696 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4543e-05 2023-10-09T10:57:18.696 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.697 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.697 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.697 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002735+0000", 2023-10-09T10:57:18.697 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1148000000000001e-05 2023-10-09T10:57:18.697 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.697 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.697 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.697 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002795+0000", 2023-10-09T10:57:18.697 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9908000000000003e-05 2023-10-09T10:57:18.697 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.697 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.697 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.698 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002824+0000", 2023-10-09T10:57:18.698 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9065e-05 2023-10-09T10:57:18.698 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.698 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.698 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.698 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.030695+0000", 2023-10-09T10:57:18.698 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027871015999999998 2023-10-09T10:57:18.698 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.698 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.698 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.698 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.030742+0000", 2023-10-09T10:57:18.698 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6714e-05 2023-10-09T10:57:18.699 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.699 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.699 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.699 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.699 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.699 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:92495 8.9 e286/265 8:99c80bd2:::benchmark_data_smithi125_329074_object92494:head v 286'5773, mlcod=286'5771)", 2023-10-09T10:57:18.699 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:34.258707+0000", 2023-10-09T10:57:18.699 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 164.29166959700001, 2023-10-09T10:57:18.699 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025821361000000001, 2023-10-09T10:57:18.699 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.699 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.699 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.699 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258707+0000", 2023-10-09T10:57:18.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.700 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.700 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258707+0000", 2023-10-09T10:57:18.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.700 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.700 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258707+0000", 2023-10-09T10:57:18.701 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:57:18.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.701 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.701 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.701 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258744+0000", 2023-10-09T10:57:18.701 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6458000000000002e-05 2023-10-09T10:57:18.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.701 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.701 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.701 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258754+0000", 2023-10-09T10:57:18.701 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0482000000000001e-05 2023-10-09T10:57:18.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258758+0000", 2023-10-09T10:57:18.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2910000000000001e-06 2023-10-09T10:57:18.702 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.267039+0000", 2023-10-09T10:57:18.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0082809919999999992 2023-10-09T10:57:18.702 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.267062+0000", 2023-10-09T10:57:18.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2532999999999999e-05 2023-10-09T10:57:18.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.284491+0000", 2023-10-09T10:57:18.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017429417999999999 2023-10-09T10:57:18.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.284529+0000", 2023-10-09T10:57:18.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7570000000000001e-05 2023-10-09T10:57:18.704 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.704 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.704 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.704 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.704 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.704 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:94596 8.f e288/282 8:ffee9f18:::benchmark_data_smithi125_329074_object94595:head v 288'5955, mlcod=0'0)", 2023-10-09T10:57:18.704 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:37.928730+0000", 2023-10-09T10:57:18.704 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 160.62164745499999, 2023-10-09T10:57:18.704 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028718649999999998, 2023-10-09T10:57:18.704 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.704 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.704 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.704 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.705 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928730+0000", 2023-10-09T10:57:18.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.705 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.705 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.705 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928730+0000", 2023-10-09T10:57:18.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.705 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.705 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.705 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928729+0000", 2023-10-09T10:57:18.706 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:57:18.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.706 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.706 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.706 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928781+0000", 2023-10-09T10:57:18.706 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1758000000000001e-05 2023-10-09T10:57:18.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.706 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.706 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.706 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928787+0000", 2023-10-09T10:57:18.706 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3180000000000003e-06 2023-10-09T10:57:18.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.707 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.707 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.707 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928792+0000", 2023-10-09T10:57:18.707 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6199999999999998e-06 2023-10-09T10:57:18.707 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.707 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.707 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.707 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.941317+0000", 2023-10-09T10:57:18.707 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012525087000000001 2023-10-09T10:57:18.707 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.707 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.707 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.707 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.941341+0000", 2023-10-09T10:57:18.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3408999999999999e-05 2023-10-09T10:57:18.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.708 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.708 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.957410+0000", 2023-10-09T10:57:18.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016068948999999999 2023-10-09T10:57:18.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.708 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.708 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.957448+0000", 2023-10-09T10:57:18.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8769000000000001e-05 2023-10-09T10:57:18.708 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.709 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.709 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.709 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.709 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:94597 8.f e288/282 8:f1e41889:::benchmark_data_smithi125_329074_object94596:head v 288'5956, mlcod=0'0)", 2023-10-09T10:57:18.709 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:37.928795+0000", 2023-10-09T10:57:18.709 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 160.62158169, 2023-10-09T10:57:18.709 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028674681, 2023-10-09T10:57:18.709 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:57:18.709 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:18.709 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:57:18.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:57:18.710 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928795+0000", 2023-10-09T10:57:18.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.710 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.710 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.710 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:57:18.710 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928795+0000", 2023-10-09T10:57:18.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:57:18.710 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.710 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.710 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:57:18.710 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928795+0000", 2023-10-09T10:57:18.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967296 2023-10-09T10:57:18.711 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.711 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.711 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:57:18.711 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928916+0000", 2023-10-09T10:57:18.711 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000120494 2023-10-09T10:57:18.711 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.711 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.711 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:57:18.711 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928927+0000", 2023-10-09T10:57:18.711 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1634000000000001e-05 2023-10-09T10:57:18.711 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.711 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.711 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:57:18.712 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928931+0000", 2023-10-09T10:57:18.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0450000000000002e-06 2023-10-09T10:57:18.713 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.713 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.713 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:57:18.713 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.946895+0000", 2023-10-09T10:57:18.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017963923 2023-10-09T10:57:18.713 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.713 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.714 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:57:18.714 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.946918+0000", 2023-10-09T10:57:18.714 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2425999999999998e-05 2023-10-09T10:57:18.714 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.714 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.714 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:57:18.714 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.957452+0000", 2023-10-09T10:57:18.714 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010534484 2023-10-09T10:57:18.714 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:57:18.714 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:57:18.714 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:57:18.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.957470+0000", 2023-10-09T10:57:18.715 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7890999999999999e-05 2023-10-09T10:57:18.715 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.715 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.715 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.715 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:57:18.715 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:57:18.715 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:57:18.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:18.716 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:18.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:18.817 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:18.841 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:18.848 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:18.867 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:18.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:18.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:18.927 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:57:18.972 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:18.991 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:57:19.000 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_3' created 2023-10-09T10:57:19.015 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:19.084 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:19.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:19.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:19.195 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:19.196 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:57:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:19 smithi155 ceph-mon[184854]: 1.14 scrub starts 2023-10-09T10:57:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:19 smithi155 ceph-mon[184854]: 1.14 scrub ok 2023-10-09T10:57:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:19 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:57:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:19 smithi155 ceph-mon[184854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4440/690 objects degraded (643.478%), 7 pgs degraded, 8 pgs undersized) 2023-10-09T10:57:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:19 smithi155 ceph-mon[184854]: 1.16 scrub starts 2023-10-09T10:57:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:19 smithi155 ceph-mon[184854]: 1.16 scrub ok 2023-10-09T10:57:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:19 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1436438006' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T10:57:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:19 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T10:57:19.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:19 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:57:19.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:19 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T10:57:19.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:19 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2284351424' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:57:19.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:19 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T10:57:19.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:19 smithi155 ceph-mon[184854]: osdmap e330: 8 total, 8 up, 7 in 2023-10-09T10:57:19.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:19 smithi155 ceph-mon[184854]: failed to encode map e330 with expected crc 2023-10-09T10:57:19.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:19 smithi155 ceph-mon[184854]: failed to encode map e330 with expected crc 2023-10-09T10:57:19.224 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:19.293 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:19.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:19.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:19.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[195507]: 1.14 scrub starts 2023-10-09T10:57:19.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[195507]: 1.14 scrub ok 2023-10-09T10:57:19.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:57:19.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[195507]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4440/690 objects degraded (643.478%), 7 pgs degraded, 8 pgs undersized) 2023-10-09T10:57:19.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[195507]: 1.16 scrub starts 2023-10-09T10:57:19.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[195507]: 1.16 scrub ok 2023-10-09T10:57:19.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1436438006' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T10:57:19.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T10:57:19.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:57:19.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T10:57:19.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2284351424' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:57:19.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T10:57:19.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[195507]: osdmap e330: 8 total, 8 up, 7 in 2023-10-09T10:57:19.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[195507]: failed to encode map e330 with expected crc 2023-10-09T10:57:19.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[195507]: failed to encode map e330 with expected crc 2023-10-09T10:57:19.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[200197]: 1.14 scrub starts 2023-10-09T10:57:19.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[200197]: 1.14 scrub ok 2023-10-09T10:57:19.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T10:57:19.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[200197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4440/690 objects degraded (643.478%), 7 pgs degraded, 8 pgs undersized) 2023-10-09T10:57:19.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[200197]: 1.16 scrub starts 2023-10-09T10:57:19.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[200197]: 1.16 scrub ok 2023-10-09T10:57:19.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1436438006' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T10:57:19.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T10:57:19.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:57:19.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T10:57:19.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2284351424' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:57:19.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T10:57:19.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[200197]: osdmap e330: 8 total, 8 up, 7 in 2023-10-09T10:57:19.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[200197]: failed to encode map e330 with expected crc 2023-10-09T10:57:19.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:19 smithi125 ceph-mon[200197]: failed to encode map e330 with expected crc 2023-10-09T10:57:19.396 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:19.423 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:19.493 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:19.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:19.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:19.598 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:19.626 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:19.645 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:19.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:19.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:19.749 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:19.777 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:19.896 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:19.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:19.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:19.999 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:20.029 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:20.050 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:20.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:20.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:20.103 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:20 smithi125 ceph-mon[200197]: failed to encode map e330 with expected crc 2023-10-09T10:57:20.103 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:20 smithi125 ceph-mon[200197]: failed to encode map e330 with expected crc 2023-10-09T10:57:20.103 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:20 smithi125 ceph-mon[200197]: failed to encode map e330 with expected crc 2023-10-09T10:57:20.103 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:20 smithi125 ceph-mon[200197]: failed to encode map e330 with expected crc 2023-10-09T10:57:20.104 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:20 smithi125 ceph-mon[200197]: pgmap v814: 119 pgs: 16 unknown, 103 active+clean; 665 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T10:57:20.104 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:20 smithi125 ceph-mon[200197]: osdmap e331: 8 total, 8 up, 7 in 2023-10-09T10:57:20.104 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:20 smithi125 ceph-mon[195507]: failed to encode map e330 with expected crc 2023-10-09T10:57:20.104 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:20 smithi125 ceph-mon[195507]: failed to encode map e330 with expected crc 2023-10-09T10:57:20.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:20 smithi125 ceph-mon[195507]: failed to encode map e330 with expected crc 2023-10-09T10:57:20.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:20 smithi125 ceph-mon[195507]: failed to encode map e330 with expected crc 2023-10-09T10:57:20.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:20 smithi125 ceph-mon[195507]: pgmap v814: 119 pgs: 16 unknown, 103 active+clean; 665 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T10:57:20.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:20 smithi125 ceph-mon[195507]: osdmap e331: 8 total, 8 up, 7 in 2023-10-09T10:57:20.154 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:20.185 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:20.216 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:57:20.216 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:57:20.216 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T10:57:20.216 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:57:20.301 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-10-09T10:57:20.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:20.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:20.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:20.371 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:20 smithi155 ceph-mon[184854]: failed to encode map e330 with expected crc 2023-10-09T10:57:20.371 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:20 smithi155 ceph-mon[184854]: failed to encode map e330 with expected crc 2023-10-09T10:57:20.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:20 smithi155 ceph-mon[184854]: failed to encode map e330 with expected crc 2023-10-09T10:57:20.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:20 smithi155 ceph-mon[184854]: failed to encode map e330 with expected crc 2023-10-09T10:57:20.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:20 smithi155 ceph-mon[184854]: pgmap v814: 119 pgs: 16 unknown, 103 active+clean; 665 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-10-09T10:57:20.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:20 smithi155 ceph-mon[184854]: osdmap e331: 8 total, 8 up, 7 in 2023-10-09T10:57:20.460 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:20.490 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:20.542 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:57:20.559 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:20.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:20.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:20.562 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:57:20.576 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:57:20.581 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:57:20.602 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:57:20.621 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:57:20.623 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:20.640 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:57:20.660 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:57:20.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:20.680 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:57:20.681 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:57:20.681 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:57:20.681 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:57:20.681 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:57:20.681 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:57:20.681 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:57:20.682 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:57:20.682 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:57:20.691 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:20.731 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:20 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T10:57:20.469+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e331 definitely_dead 0 2023-10-09T10:57:20.881 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:57:20.997 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.7. 2023-10-09T10:57:21.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:21.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:21.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:21.115 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:21.146 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:21.170 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:21.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:21.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:21.273 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:21.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:21 smithi155 ceph-mon[184854]: failed to encode map e330 with expected crc 2023-10-09T10:57:21.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:21 smithi155 ceph-mon[184854]: failed to encode map e331 with expected crc 2023-10-09T10:57:21.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:21 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3915246686' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T10:57:21.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:21 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3054889189' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:21.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:21 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3915246686' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T10:57:21.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:21 smithi155 ceph-mon[184854]: osdmap e332: 8 total, 8 up, 8 in 2023-10-09T10:57:21.300 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:21 smithi155 ceph-mon[184854]: failed to encode map e332 with expected crc 2023-10-09T10:57:21.302 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:21.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:57:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:57:21] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.43.0" 2023-10-09T10:57:21.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:21 smithi125 ceph-mon[195507]: failed to encode map e330 with expected crc 2023-10-09T10:57:21.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:21 smithi125 ceph-mon[195507]: failed to encode map e331 with expected crc 2023-10-09T10:57:21.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:21 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3915246686' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T10:57:21.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:21 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3054889189' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:21.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:21 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3915246686' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T10:57:21.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:21 smithi125 ceph-mon[195507]: osdmap e332: 8 total, 8 up, 8 in 2023-10-09T10:57:21.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:21 smithi125 ceph-mon[195507]: failed to encode map e332 with expected crc 2023-10-09T10:57:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:21 smithi125 ceph-mon[200197]: failed to encode map e330 with expected crc 2023-10-09T10:57:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:21 smithi125 ceph-mon[200197]: failed to encode map e331 with expected crc 2023-10-09T10:57:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:21 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3915246686' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T10:57:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:21 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3054889189' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:21 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3915246686' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T10:57:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:21 smithi125 ceph-mon[200197]: osdmap e332: 8 total, 8 up, 8 in 2023-10-09T10:57:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:21 smithi125 ceph-mon[200197]: failed to encode map e332 with expected crc 2023-10-09T10:57:21.478 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-09T10:57:21.523 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:21.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:21.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:21.627 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:21.654 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:21.724 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:21.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:21.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:21.827 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:21.858 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:21.879 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:21.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:21.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:21.919 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:57:21.919 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:57:21.983 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:22.041 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:22.062 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:22.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:22.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:22.168 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:22.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:57:22.196 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: failed to encode map e331 with expected crc 2023-10-09T10:57:22.196 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.196 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.196 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.196 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: 1.1d scrub starts 2023-10-09T10:57:22.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: 1.1d scrub ok 2023-10-09T10:57:22.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: pgmap v817: 119 pgs: 21 peering, 7 unknown, 91 active+clean; 643 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-09T10:57:22.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: osdmap e333: 8 total, 8 up, 8 in 2023-10-09T10:57:22.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: failed to encode map e333 with expected crc 2023-10-09T10:57:22.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: failed to encode map e333 with expected crc 2023-10-09T10:57:22.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:22 smithi155 ceph-mon[184854]: failed to encode map e333 with expected crc 2023-10-09T10:57:22.198 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: failed to encode map e331 with expected crc 2023-10-09T10:57:22.198 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.199 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.199 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.199 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.199 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.199 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.199 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.199 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.200 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: 1.1d scrub starts 2023-10-09T10:57:22.200 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: 1.1d scrub ok 2023-10-09T10:57:22.200 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: pgmap v817: 119 pgs: 21 peering, 7 unknown, 91 active+clean; 643 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-09T10:57:22.200 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: osdmap e333: 8 total, 8 up, 8 in 2023-10-09T10:57:22.200 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: failed to encode map e333 with expected crc 2023-10-09T10:57:22.200 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: failed to encode map e333 with expected crc 2023-10-09T10:57:22.200 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[200197]: failed to encode map e333 with expected crc 2023-10-09T10:57:22.200 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: failed to encode map e331 with expected crc 2023-10-09T10:57:22.200 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.200 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.201 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.201 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.201 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.201 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.201 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.201 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: failed to encode map e332 with expected crc 2023-10-09T10:57:22.202 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: 1.1d scrub starts 2023-10-09T10:57:22.202 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: 1.1d scrub ok 2023-10-09T10:57:22.202 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: pgmap v817: 119 pgs: 21 peering, 7 unknown, 91 active+clean; 643 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-09T10:57:22.202 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: osdmap e333: 8 total, 8 up, 8 in 2023-10-09T10:57:22.202 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: failed to encode map e333 with expected crc 2023-10-09T10:57:22.202 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: failed to encode map e333 with expected crc 2023-10-09T10:57:22.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:22 smithi125 ceph-mon[195507]: failed to encode map e333 with expected crc 2023-10-09T10:57:22.203 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:22.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:57:22.318 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:22.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:22.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:22.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:22.439 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:22.448 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:22.467 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:22.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:22.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:22.571 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:22.586 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:57:22.600 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:22.619 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:22.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:22.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:22.723 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:22.751 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:22.770 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:22.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:22.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:22.874 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:22.904 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:22.989 INFO:teuthology.orchestra.run.smithi125.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-10-09T10:57:23.023 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:23.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:23.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:23.128 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:23.154 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[195507]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.155 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[195507]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.155 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[195507]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.155 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[195507]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.155 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[195507]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.155 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[195507]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.155 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[195507]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.155 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[195507]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.155 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3573852952' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:57:23.155 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3573852952' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:57:23.155 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[195507]: osdmap e334: 8 total, 8 up, 8 in 2023-10-09T10:57:23.156 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[195507]: failed to encode map e334 with expected crc 2023-10-09T10:57:23.156 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[195507]: failed to encode map e334 with expected crc 2023-10-09T10:57:23.156 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[200197]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.156 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[200197]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.156 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[200197]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.156 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[200197]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.156 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[200197]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.156 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[200197]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.156 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[200197]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.156 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[200197]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.157 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3573852952' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:57:23.157 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3573852952' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:57:23.157 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[200197]: osdmap e334: 8 total, 8 up, 8 in 2023-10-09T10:57:23.157 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[200197]: failed to encode map e334 with expected crc 2023-10-09T10:57:23.157 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:23 smithi125 ceph-mon[200197]: failed to encode map e334 with expected crc 2023-10-09T10:57:23.157 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:23.178 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:57:23.276 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:23.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:23.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:23.331 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:23 smithi155 ceph-mon[184854]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.331 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:23 smithi155 ceph-mon[184854]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.331 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:23 smithi155 ceph-mon[184854]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.331 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:23 smithi155 ceph-mon[184854]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.332 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:23 smithi155 ceph-mon[184854]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.332 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:23 smithi155 ceph-mon[184854]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.332 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:23 smithi155 ceph-mon[184854]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.332 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:23 smithi155 ceph-mon[184854]: failed to encode map e333 with expected crc 2023-10-09T10:57:23.332 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:23 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3573852952' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T10:57:23.332 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:23 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3573852952' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T10:57:23.332 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:23 smithi155 ceph-mon[184854]: osdmap e334: 8 total, 8 up, 8 in 2023-10-09T10:57:23.332 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:23 smithi155 ceph-mon[184854]: failed to encode map e334 with expected crc 2023-10-09T10:57:23.332 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:23 smithi155 ceph-mon[184854]: failed to encode map e334 with expected crc 2023-10-09T10:57:23.380 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:23.409 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:23.479 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:23.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:23.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:23.521 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:23.537 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:57:23.537 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:57:23.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:23.614 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:23.636 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:23.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:23.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:23.740 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:23.771 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:23.890 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:23.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:23.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:23.901 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:57:23.994 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:24.024 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:24.044 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:24.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:24.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:24.091 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:57:24.148 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:24.177 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:24.297 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:24.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:24.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:24.341 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[195507]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.341 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[195507]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.341 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[195507]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.341 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[195507]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.341 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[195507]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.341 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[195507]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.341 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[195507]: pgmap v820: 119 pgs: 2 remapped+peering, 24 peering, 93 active+clean; 629 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-09T10:57:24.341 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/96573767' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:57:24.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[200197]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[200197]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[200197]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[200197]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[200197]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[200197]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[200197]: pgmap v820: 119 pgs: 2 remapped+peering, 24 peering, 93 active+clean; 629 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-09T10:57:24.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:24 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/96573767' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:57:24.401 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:24.401 DEBUG:teuthology.orchestra.run.smithi125:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-10-09T10:57:24.404 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T10:57:24.404 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T10:57:24.427 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:24 smithi155 ceph-mon[184854]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.428 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:24 smithi155 ceph-mon[184854]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.428 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:24 smithi155 ceph-mon[184854]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.428 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:24 smithi155 ceph-mon[184854]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.428 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:24 smithi155 ceph-mon[184854]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.428 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:24 smithi155 ceph-mon[184854]: failed to encode map e334 with expected crc 2023-10-09T10:57:24.428 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:24 smithi155 ceph-mon[184854]: pgmap v820: 119 pgs: 2 remapped+peering, 24 peering, 93 active+clean; 629 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-09T10:57:24.428 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:24 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/96573767' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:57:24.430 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:24.450 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:24.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:24.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:24.553 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:24.581 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:24.611 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:hints = 1 2023-10-09T10:57:24.611 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T10:57:24.611 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object prefix: benchmark_data_smithi125_363789 2023-10-09T10:57:24.612 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:57:24.612 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 0 0 0 0 0 0 - 0 2023-10-09T10:57:24.710 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:24.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:24.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:24.717 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:24.815 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:24.838 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:57:24.854 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:24.868 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:57:24.882 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:24.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:24.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:24.911 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:57:24.948 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:57:24.982 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:57:24.987 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:25.024 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:57:25.045 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:25.059 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:57:25.089 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:57:25.091 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:57:25.091 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:57:25.091 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:57:25.091 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:57:25.091 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:57:25.092 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:57:25.092 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:57:25.092 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:57:25.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:25 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:25.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:25 smithi155 ceph-mon[184854]: failed to encode map e335 with expected crc 2023-10-09T10:57:25.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:25 smithi155 ceph-mon[184854]: failed to encode map e335 with expected crc 2023-10-09T10:57:25.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:25 smithi155 ceph-mon[184854]: failed to encode map e335 with expected crc 2023-10-09T10:57:25.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:25 smithi155 ceph-mon[184854]: failed to encode map e335 with expected crc 2023-10-09T10:57:25.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:25 smithi155 ceph-mon[184854]: failed to encode map e335 with expected crc 2023-10-09T10:57:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/96573767' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:57:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:25 smithi155 ceph-mon[184854]: osdmap e335: 8 total, 8 up, 8 in 2023-10-09T10:57:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/453744341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:25.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:25.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[195507]: failed to encode map e335 with expected crc 2023-10-09T10:57:25.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[195507]: failed to encode map e335 with expected crc 2023-10-09T10:57:25.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[195507]: failed to encode map e335 with expected crc 2023-10-09T10:57:25.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[195507]: failed to encode map e335 with expected crc 2023-10-09T10:57:25.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[195507]: failed to encode map e335 with expected crc 2023-10-09T10:57:25.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/96573767' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:57:25.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[195507]: osdmap e335: 8 total, 8 up, 8 in 2023-10-09T10:57:25.239 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/453744341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:25.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:25.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[200197]: failed to encode map e335 with expected crc 2023-10-09T10:57:25.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[200197]: failed to encode map e335 with expected crc 2023-10-09T10:57:25.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[200197]: failed to encode map e335 with expected crc 2023-10-09T10:57:25.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[200197]: failed to encode map e335 with expected crc 2023-10-09T10:57:25.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[200197]: failed to encode map e335 with expected crc 2023-10-09T10:57:25.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/96573767' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:57:25.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[200197]: osdmap e335: 8 total, 8 up, 8 in 2023-10-09T10:57:25.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/453744341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:25.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:25.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:25.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:25.431 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:25.480 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:25.612 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 1 16 2885 2869 179.23 179.312 0.00320793 0.00556612 2023-10-09T10:57:25.661 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:25.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:25.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:25.778 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:25.843 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:25.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:25.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:25.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:25.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:26.019 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:26.046 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:26.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:26.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:26.150 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:26.179 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:57:26.179 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:57:26.180 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 0 2023-10-09T10:57:26.180 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:57:26.200 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:26.230 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:26.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:26.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:26.336 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:26.382 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:26 smithi125 ceph-mon[195507]: osdmap e336: 8 total, 8 up, 8 in 2023-10-09T10:57:26.382 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:26 smithi125 ceph-mon[195507]: failed to encode map e336 with expected crc 2023-10-09T10:57:26.382 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:26 smithi125 ceph-mon[195507]: failed to encode map e336 with expected crc 2023-10-09T10:57:26.383 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:26 smithi125 ceph-mon[195507]: failed to encode map e336 with expected crc 2023-10-09T10:57:26.383 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:26 smithi125 ceph-mon[195507]: failed to encode map e336 with expected crc 2023-10-09T10:57:26.383 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:26 smithi125 ceph-mon[195507]: pgmap v823: 119 pgs: 2 remapped+peering, 24 peering, 93 active+clean; 629 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-09T10:57:26.383 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:26 smithi125 ceph-mon[200197]: osdmap e336: 8 total, 8 up, 8 in 2023-10-09T10:57:26.383 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:26 smithi125 ceph-mon[200197]: failed to encode map e336 with expected crc 2023-10-09T10:57:26.383 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:26 smithi125 ceph-mon[200197]: failed to encode map e336 with expected crc 2023-10-09T10:57:26.383 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:26 smithi125 ceph-mon[200197]: failed to encode map e336 with expected crc 2023-10-09T10:57:26.383 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:26 smithi125 ceph-mon[200197]: failed to encode map e336 with expected crc 2023-10-09T10:57:26.383 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:26 smithi125 ceph-mon[200197]: pgmap v823: 119 pgs: 2 remapped+peering, 24 peering, 93 active+clean; 629 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-09T10:57:26.386 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:26.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:26 smithi155 ceph-mon[184854]: osdmap e336: 8 total, 8 up, 8 in 2023-10-09T10:57:26.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:26 smithi155 ceph-mon[184854]: failed to encode map e336 with expected crc 2023-10-09T10:57:26.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:26 smithi155 ceph-mon[184854]: failed to encode map e336 with expected crc 2023-10-09T10:57:26.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:26 smithi155 ceph-mon[184854]: failed to encode map e336 with expected crc 2023-10-09T10:57:26.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:26 smithi155 ceph-mon[184854]: failed to encode map e336 with expected crc 2023-10-09T10:57:26.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:26 smithi155 ceph-mon[184854]: pgmap v823: 119 pgs: 2 remapped+peering, 24 peering, 93 active+clean; 629 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-09T10:57:26.479 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 2, 1, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:57:26.479 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:57:26.479 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T10:57:26.479 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:57:26.614 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 2 16 5691 5675 177.267 175.375 0.00612252 0.00563321 2023-10-09T10:57:26.621 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:57:27.613 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 3 16 8608 8592 178.943 182.312 0.00402893 0.00558085 2023-10-09T10:57:27.700 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:57:27.922 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:57:27.931 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:27.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:27.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:28.034 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:28.066 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:28.088 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:28.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:28.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:28.098 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:28.192 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:28.233 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:28.256 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:28.259 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:28.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:28.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:28.362 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:28.406 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:28.439 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:28.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:28.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:28.544 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:28.582 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:28.612 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:28.613 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 4 16 10983 10967 171.315 148.438 0.00753285 0.0058292 2023-10-09T10:57:28.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:28.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:28.718 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:28.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:28 smithi155 ceph-mon[184854]: pgmap v824: 119 pgs: 119 active+clean; 226 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 41 MiB/s wr, 659 op/s; 49 KiB/s, 5 keys/s, 5 objects/s recovering 2023-10-09T10:57:28.766 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:28.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:28 smithi125 ceph-mon[195507]: pgmap v824: 119 pgs: 119 active+clean; 226 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 41 MiB/s wr, 659 op/s; 49 KiB/s, 5 keys/s, 5 objects/s recovering 2023-10-09T10:57:28.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:28 smithi125 ceph-mon[200197]: pgmap v824: 119 pgs: 119 active+clean; 226 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 41 MiB/s wr, 659 op/s; 49 KiB/s, 5 keys/s, 5 objects/s recovering 2023-10-09T10:57:28.851 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:28.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:28.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:28.961 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:28.998 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:29.025 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:29.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:29.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:29.130 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:29.169 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:29.195 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:29.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:29.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:29.298 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:29.341 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:29.371 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:29.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:29.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:29.476 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:29.517 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:29.543 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:29.544 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:29.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:29.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:29.613 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 5 16 13376 13360 166.962 149.562 0.0048548 0.00598056 2023-10-09T10:57:29.648 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:29.689 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:29.713 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:29.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:29.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:29.817 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:29.865 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:30.043 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:30.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:30.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:30.158 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:30.205 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:30.614 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 6 16 15919 15903 165.603 158.938 0.00487779 0.00603109 2023-10-09T10:57:30.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:30 smithi155 ceph-mon[184854]: pgmap v825: 119 pgs: 119 active+clean; 325 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s; 41 KiB/s, 4 keys/s, 5 objects/s recovering 2023-10-09T10:57:30.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:30.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:57:31.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:30 smithi125 ceph-mon[195507]: pgmap v825: 119 pgs: 119 active+clean; 325 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s; 41 KiB/s, 4 keys/s, 5 objects/s recovering 2023-10-09T10:57:31.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:31.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:57:31.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:30 smithi125 ceph-mon[200197]: pgmap v825: 119 pgs: 119 active+clean; 325 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s; 41 KiB/s, 4 keys/s, 5 objects/s recovering 2023-10-09T10:57:31.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:31.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:57:31.108 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:31.112 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:31.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:31.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:31.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:31.279 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:31.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:57:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:57:31] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-10-09T10:57:31.614 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 7 16 18240 18224 162.651 145.062 0.00492818 0.00614197 2023-10-09T10:57:32.116 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:32.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:32.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:32.227 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:32.281 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:32.615 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 8 16 20336 20320 158.694 131 0.00279481 0.0062923 2023-10-09T10:57:32.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:32 smithi125 ceph-mon[195507]: pgmap v826: 119 pgs: 119 active+clean; 779 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s; 34 KiB/s, 3 keys/s, 4 objects/s recovering 2023-10-09T10:57:32.937 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:32 smithi125 ceph-mon[200197]: pgmap v826: 119 pgs: 119 active+clean; 779 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s; 34 KiB/s, 3 keys/s, 4 objects/s recovering 2023-10-09T10:57:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:32 smithi155 ceph-mon[184854]: pgmap v826: 119 pgs: 119 active+clean; 779 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s; 34 KiB/s, 3 keys/s, 4 objects/s recovering 2023-10-09T10:57:33.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:33.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:33.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:33.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:33.615 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 9 16 22257 22241 154.401 120.062 0.0124748 0.00646677 2023-10-09T10:57:33.618 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:57:33.652 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:33.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:33 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1156370329' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:33.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:33 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:33.750 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:33 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1156370329' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:33.750 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:33 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:33.941 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:33.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:33.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:33 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1156370329' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:33 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:34.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:34.106 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:34.183 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:57:34.186 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:57:34.195 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:34.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:34.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:34.304 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:34.349 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:34.401 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:34.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:34.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:34.546 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:34.608 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:34.615 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 10 16 24071 24055 150.291 113.375 0.014971 0.00664754 2023-10-09T10:57:34.630 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:34 smithi155 ceph-mon[184854]: pgmap v827: 119 pgs: 119 active+clean; 1.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 118 MiB/s wr, 1.90k op/s; 29 KiB/s, 3 keys/s, 3 objects/s recovering 2023-10-09T10:57:34.630 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:34 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:34.630 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:34 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:57:34.630 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:34 smithi155 ceph-mon[184854]: osdmap e337: 8 total, 8 up, 8 in 2023-10-09T10:57:34.630 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:34 smithi155 ceph-mon[184854]: failed to encode map e337 with expected crc 2023-10-09T10:57:34.630 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:34 smithi155 ceph-mon[184854]: failed to encode map e337 with expected crc 2023-10-09T10:57:34.630 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:34 smithi155 ceph-mon[184854]: failed to encode map e337 with expected crc 2023-10-09T10:57:34.630 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:34 smithi155 ceph-mon[184854]: failed to encode map e337 with expected crc 2023-10-09T10:57:34.698 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:34.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:34.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:34.723 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[195507]: pgmap v827: 119 pgs: 119 active+clean; 1.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 118 MiB/s wr, 1.90k op/s; 29 KiB/s, 3 keys/s, 3 objects/s recovering 2023-10-09T10:57:34.723 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:34.723 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:57:34.724 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[195507]: osdmap e337: 8 total, 8 up, 8 in 2023-10-09T10:57:34.724 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[195507]: failed to encode map e337 with expected crc 2023-10-09T10:57:34.724 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[195507]: failed to encode map e337 with expected crc 2023-10-09T10:57:34.724 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[195507]: failed to encode map e337 with expected crc 2023-10-09T10:57:34.724 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[195507]: failed to encode map e337 with expected crc 2023-10-09T10:57:34.724 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[200197]: pgmap v827: 119 pgs: 119 active+clean; 1.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 118 MiB/s wr, 1.90k op/s; 29 KiB/s, 3 keys/s, 3 objects/s recovering 2023-10-09T10:57:34.724 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:34.724 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:57:34.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[200197]: osdmap e337: 8 total, 8 up, 8 in 2023-10-09T10:57:34.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[200197]: failed to encode map e337 with expected crc 2023-10-09T10:57:34.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[200197]: failed to encode map e337 with expected crc 2023-10-09T10:57:34.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[200197]: failed to encode map e337 with expected crc 2023-10-09T10:57:34.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:34 smithi125 ceph-mon[200197]: failed to encode map e337 with expected crc 2023-10-09T10:57:34.805 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:34.875 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:34.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:34.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:34.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:34.954 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:57:34.954 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:57:34.954 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:57:34.954 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:57:34.954 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:57:35.017 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:57:35.026 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:35.047 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:57:35.064 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:35.090 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:57:35.093 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:35.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:35.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:35.124 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:57:35.155 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:57:35.187 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":828,"stamp":"2023-10-09T10:57:33.598578+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1170925894,"num_objects":17881,"num_object_clones":0,"num_object_copies":53643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17881,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":17940,"num_write_kb":1143097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":1392092,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":22903,"ondisk_log_size":22903,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":367,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9075072,"kb_used_data":4386688,"kb_used_omap":126,"kb_used_meta":4688193,"kb_avail":740886144,"statfs":{"total":767960285184,"available":758667411456,"internally_reserved":0,"allocated":4491968512,"data_stored":4480013972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":4800710267},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[10,2,1,0,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1170281302,"num_objects":17868,"num_object_clones":0,"num_object_copies":53604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17891,"num_write_kb":1142876,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":277516,"num_keys_recovered":30,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.512076"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":692,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.104150+0000","last_change":"2023-10-09T10:57:25.104150+0000","last_active":"2023-10-09T10:57:25.104150+0000","last_peered":"2023-10-09T10:57:25.104150+0000","last_clean":"2023-10-09T10:57:25.104150+0000","last_became_active":"2023-10-09T10:57:25.103762+0000","last_became_peered":"2023-10-09T10:57:25.103762+0000","last_unstale":"2023-10-09T10:57:25.104150+0000","last_undegraded":"2023-10-09T10:57:25.104150+0000","last_fullsized":"2023-10-09T10:57:25.104150+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:40.174975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:40.174975+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:35.416780+0000","scrub_duration":0.010654205,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":42,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.083582+0000","last_change":"2023-10-09T10:57:11.874010+0000","last_active":"2023-10-09T10:57:24.083582+0000","last_peered":"2023-10-09T10:57:24.083582+0000","last_clean":"2023-10-09T10:57:24.083582+0000","last_became_active":"2023-10-09T10:57:11.873713+0000","last_became_peered":"2023-10-09T10:57:11.873713+0000","last_unstale":"2023-10-09T10:57:24.083582+0000","last_undegraded":"2023-10-09T10:57:24.083582+0000","last_fullsized":"2023-10-09T10:57:24.083582+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:58.049973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":344,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.079101+0000","last_change":"2023-10-09T10:54:47.945782+0000","last_active":"2023-10-09T10:57:24.079101+0000","last_peered":"2023-10-09T10:57:24.079101+0000","last_clean":"2023-10-09T10:57:24.079101+0000","last_became_active":"2023-10-09T10:54:47.945460+0000","last_became_peered":"2023-10-09T10:54:47.945460+0000","last_unstale":"2023-10-09T10:57:24.079101+0000","last_undegraded":"2023-10-09T10:57:24.079101+0000","last_fullsized":"2023-10-09T10:57:24.079101+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:50.537469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":717,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085677+0000","last_change":"2023-10-09T10:53:34.537094+0000","last_active":"2023-10-09T10:57:24.085677+0000","last_peered":"2023-10-09T10:57:24.085677+0000","last_clean":"2023-10-09T10:57:24.085677+0000","last_became_active":"2023-10-09T10:52:55.414995+0000","last_became_peered":"2023-10-09T10:52:55.414995+0000","last_unstale":"2023-10-09T10:57:24.085677+0000","last_undegraded":"2023-10-09T10:57:24.085677+0000","last_fullsized":"2023-10-09T10:57:24.085677+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.536962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.536962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:30.197404+0000","scrub_duration":0.0053325960000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":53,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.083088+0000","last_change":"2023-10-09T10:54:47.944768+0000","last_active":"2023-10-09T10:57:24.083088+0000","last_peered":"2023-10-09T10:57:24.083088+0000","last_clean":"2023-10-09T10:57:24.083088+0000","last_became_active":"2023-10-09T10:54:47.943888+0000","last_became_peered":"2023-10-09T10:54:47.943888+0000","last_unstale":"2023-10-09T10:57:24.083088+0000","last_undegraded":"2023-10-09T10:57:24.083088+0000","last_fullsized":"2023-10-09T10:57:24.083088+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:11.046762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":263,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086922+0000","last_change":"2023-10-09T10:57:22.007179+0000","last_active":"2023-10-09T10:57:24.086922+0000","last_peered":"2023-10-09T10:57:24.086922+0000","last_clean":"2023-10-09T10:57:24.086922+0000","last_became_active":"2023-10-09T10:57:22.006760+0000","last_became_peered":"2023-10-09T10:57:22.006760+0000","last_unstale":"2023-10-09T10:57:24.086922+0000","last_undegraded":"2023-10-09T10:57:24.086922+0000","last_fullsized":"2023-10-09T10:57:24.086922+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:13.771012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":759,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.103697+0000","last_change":"2023-10-09T10:57:25.103697+0000","last_active":"2023-10-09T10:57:25.103697+0000","last_peered":"2023-10-09T10:57:25.103697+0000","last_clean":"2023-10-09T10:57:25.103697+0000","last_became_active":"2023-10-09T10:57:25.103306+0000","last_became_peered":"2023-10-09T10:57:25.103306+0000","last_unstale":"2023-10-09T10:57:25.103697+0000","last_undegraded":"2023-10-09T10:57:25.103697+0000","last_fullsized":"2023-10-09T10:57:25.103697+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:23:07.530489+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":389,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.078457+0000","last_change":"2023-10-09T10:54:48.038948+0000","last_active":"2023-10-09T10:57:24.078457+0000","last_peered":"2023-10-09T10:57:24.078457+0000","last_clean":"2023-10-09T10:57:24.078457+0000","last_became_active":"2023-10-09T10:54:48.001556+0000","last_became_peered":"2023-10-09T10:54:48.001556+0000","last_unstale":"2023-10-09T10:57:24.078457+0000","last_undegraded":"2023-10-09T10:57:24.078457+0000","last_fullsized":"2023-10-09T10:57:24.078457+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:23.415040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":53,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086011+0000","last_change":"2023-10-09T10:54:47.914742+0000","last_active":"2023-10-09T10:57:24.086011+0000","last_peered":"2023-10-09T10:57:24.086011+0000","last_clean":"2023-10-09T10:57:24.086011+0000","last_became_active":"2023-10-09T10:54:47.914551+0000","last_became_peered":"2023-10-09T10:54:47.914551+0000","last_unstale":"2023-10-09T10:57:24.086011+0000","last_undegraded":"2023-10-09T10:57:24.086011+0000","last_fullsized":"2023-10-09T10:57:24.086011+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:16.067994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":751,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.098434+0000","last_change":"2023-10-09T10:57:25.098434+0000","last_active":"2023-10-09T10:57:25.098434+0000","last_peered":"2023-10-09T10:57:25.098434+0000","last_clean":"2023-10-09T10:57:25.098434+0000","last_became_active":"2023-10-09T10:57:25.098064+0000","last_became_peered":"2023-10-09T10:57:25.098064+0000","last_unstale":"2023-10-09T10:57:25.098434+0000","last_undegraded":"2023-10-09T10:57:25.098434+0000","last_fullsized":"2023-10-09T10:57:25.098434+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.787782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.787782+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:03.016769+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":208,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085713+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:57:24.085713+0000","last_peered":"2023-10-09T10:57:24.085713+0000","last_clean":"2023-10-09T10:57:24.085713+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:57:24.085713+0000","last_undegraded":"2023-10-09T10:57:24.085713+0000","last_fullsized":"2023-10-09T10:57:24.085713+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":345,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.108261+0000","last_change":"2023-10-09T10:54:48.039275+0000","last_active":"2023-10-09T10:57:25.108261+0000","last_peered":"2023-10-09T10:57:25.108261+0000","last_clean":"2023-10-09T10:57:25.108261+0000","last_became_active":"2023-10-09T10:54:48.017645+0000","last_became_peered":"2023-10-09T10:54:48.017645+0000","last_unstale":"2023-10-09T10:57:25.108261+0000","last_undegraded":"2023-10-09T10:57:25.108261+0000","last_fullsized":"2023-10-09T10:57:25.108261+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:55.986791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":627,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.103739+0000","last_change":"2023-10-09T10:57:25.103739+0000","last_active":"2023-10-09T10:57:25.103739+0000","last_peered":"2023-10-09T10:57:25.103739+0000","last_clean":"2023-10-09T10:57:25.103739+0000","last_became_active":"2023-10-09T10:57:25.103378+0000","last_became_peered":"2023-10-09T10:57:25.103378+0000","last_unstale":"2023-10-09T10:57:25.103739+0000","last_undegraded":"2023-10-09T10:57:25.103739+0000","last_fullsized":"2023-10-09T10:57:25.103739+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:49:50.889945+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.087546+0000","last_change":"2023-10-09T10:57:22.014188+0000","last_active":"2023-10-09T10:57:24.087546+0000","last_peered":"2023-10-09T10:57:24.087546+0000","last_clean":"2023-10-09T10:57:24.087546+0000","last_became_active":"2023-10-09T10:57:22.013975+0000","last_became_peered":"2023-10-09T10:57:22.013975+0000","last_unstale":"2023-10-09T10:57:24.087546+0000","last_undegraded":"2023-10-09T10:57:24.087546+0000","last_fullsized":"2023-10-09T10:57:24.087546+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:58.620741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":301,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.081297+0000","last_change":"2023-10-09T10:57:22.010202+0000","last_active":"2023-10-09T10:57:24.081297+0000","last_peered":"2023-10-09T10:57:24.081297+0000","last_clean":"2023-10-09T10:57:24.081297+0000","last_became_active":"2023-10-09T10:57:22.006867+0000","last_became_peered":"2023-10-09T10:57:22.006867+0000","last_unstale":"2023-10-09T10:57:24.081297+0000","last_undegraded":"2023-10-09T10:57:24.081297+0000","last_fullsized":"2023-10-09T10:57:24.081297+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:54.212660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":724,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085313+0000","last_change":"2023-10-09T10:53:32.540768+0000","last_active":"2023-10-09T10:57:24.085313+0000","last_peered":"2023-10-09T10:57:24.085313+0000","last_clean":"2023-10-09T10:57:24.085313+0000","last_became_active":"2023-10-09T10:52:55.414818+0000","last_became_peered":"2023-10-09T10:52:55.414818+0000","last_unstale":"2023-10-09T10:57:24.085313+0000","last_undegraded":"2023-10-09T10:57:24.085313+0000","last_fullsized":"2023-10-09T10:57:24.085313+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:32.540610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:32.540610+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:47.374383+0000","scrub_duration":0.0064251300000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":19,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.090374+0000","last_change":"2023-10-09T10:57:22.032392+0000","last_active":"2023-10-09T10:57:24.090374+0000","last_peered":"2023-10-09T10:57:24.090374+0000","last_clean":"2023-10-09T10:57:24.090374+0000","last_became_active":"2023-10-09T10:57:22.012620+0000","last_became_peered":"2023-10-09T10:57:22.012620+0000","last_unstale":"2023-10-09T10:57:24.090374+0000","last_undegraded":"2023-10-09T10:57:24.090374+0000","last_fullsized":"2023-10-09T10:57:24.090374+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:06:34.979769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":53,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.084022+0000","last_change":"2023-10-09T10:54:47.925654+0000","last_active":"2023-10-09T10:57:24.084022+0000","last_peered":"2023-10-09T10:57:24.084022+0000","last_clean":"2023-10-09T10:57:24.084022+0000","last_became_active":"2023-10-09T10:54:47.924989+0000","last_became_peered":"2023-10-09T10:54:47.924989+0000","last_unstale":"2023-10-09T10:57:24.084022+0000","last_undegraded":"2023-10-09T10:57:24.084022+0000","last_fullsized":"2023-10-09T10:57:24.084022+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:02:49.755988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":722,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085776+0000","last_change":"2023-10-09T10:57:18.113935+0000","last_active":"2023-10-09T10:57:24.085776+0000","last_peered":"2023-10-09T10:57:24.085776+0000","last_clean":"2023-10-09T10:57:24.085776+0000","last_became_active":"2023-10-09T10:54:50.913330+0000","last_became_peered":"2023-10-09T10:54:50.913330+0000","last_unstale":"2023-10-09T10:57:24.085776+0000","last_undegraded":"2023-10-09T10:57:24.085776+0000","last_fullsized":"2023-10-09T10:57:24.085776+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:18.113848+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.113848+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:51:24.460807+0000","scrub_duration":0.001474106,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":23,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.089676+0000","last_change":"2023-10-09T10:57:22.039995+0000","last_active":"2023-10-09T10:57:24.089676+0000","last_peered":"2023-10-09T10:57:24.089676+0000","last_clean":"2023-10-09T10:57:24.089676+0000","last_became_active":"2023-10-09T10:57:22.016287+0000","last_became_peered":"2023-10-09T10:57:22.016287+0000","last_unstale":"2023-10-09T10:57:24.089676+0000","last_undegraded":"2023-10-09T10:57:24.089676+0000","last_fullsized":"2023-10-09T10:57:24.089676+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:52.167501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":340,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.082777+0000","last_change":"2023-10-09T10:54:47.908531+0000","last_active":"2023-10-09T10:57:24.082777+0000","last_peered":"2023-10-09T10:57:24.082777+0000","last_clean":"2023-10-09T10:57:24.082777+0000","last_became_active":"2023-10-09T10:54:47.908232+0000","last_became_peered":"2023-10-09T10:54:47.908232+0000","last_unstale":"2023-10-09T10:57:24.082777+0000","last_undegraded":"2023-10-09T10:57:24.082777+0000","last_fullsized":"2023-10-09T10:57:24.082777+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:20.617543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":702,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086540+0000","last_change":"2023-10-09T10:49:18.708370+0000","last_active":"2023-10-09T10:57:24.086540+0000","last_peered":"2023-10-09T10:57:24.086540+0000","last_clean":"2023-10-09T10:57:24.086540+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:57:24.086540+0000","last_undegraded":"2023-10-09T10:57:24.086540+0000","last_fullsized":"2023-10-09T10:57:24.086540+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:57.808410+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":19,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086737+0000","last_change":"2023-10-09T10:57:22.026809+0000","last_active":"2023-10-09T10:57:24.086737+0000","last_peered":"2023-10-09T10:57:24.086737+0000","last_clean":"2023-10-09T10:57:24.086737+0000","last_became_active":"2023-10-09T10:57:22.011600+0000","last_became_peered":"2023-10-09T10:57:22.011600+0000","last_unstale":"2023-10-09T10:57:24.086737+0000","last_undegraded":"2023-10-09T10:57:24.086737+0000","last_fullsized":"2023-10-09T10:57:24.086737+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:10.453811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":412,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.080571+0000","last_change":"2023-10-09T10:57:22.006151+0000","last_active":"2023-10-09T10:57:24.080571+0000","last_peered":"2023-10-09T10:57:24.080571+0000","last_clean":"2023-10-09T10:57:24.080571+0000","last_became_active":"2023-10-09T10:57:22.005782+0000","last_became_peered":"2023-10-09T10:57:22.005782+0000","last_unstale":"2023-10-09T10:57:24.080571+0000","last_undegraded":"2023-10-09T10:57:24.080571+0000","last_fullsized":"2023-10-09T10:57:24.080571+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:09.733515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":655,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.098905+0000","last_change":"2023-10-09T10:57:25.098905+0000","last_active":"2023-10-09T10:57:25.098905+0000","last_peered":"2023-10-09T10:57:25.098905+0000","last_clean":"2023-10-09T10:57:25.098905+0000","last_became_active":"2023-10-09T10:57:25.098562+0000","last_became_peered":"2023-10-09T10:57:25.098562+0000","last_unstale":"2023-10-09T10:57:25.098905+0000","last_undegraded":"2023-10-09T10:57:25.098905+0000","last_fullsized":"2023-10-09T10:57:25.098905+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:23.020232+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":262,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085686+0000","last_change":"2023-10-09T10:57:22.004708+0000","last_active":"2023-10-09T10:57:24.085686+0000","last_peered":"2023-10-09T10:57:24.085686+0000","last_clean":"2023-10-09T10:57:24.085686+0000","last_became_active":"2023-10-09T10:57:22.004413+0000","last_became_peered":"2023-10-09T10:57:22.004413+0000","last_unstale":"2023-10-09T10:57:24.085686+0000","last_undegraded":"2023-10-09T10:57:24.085686+0000","last_fullsized":"2023-10-09T10:57:24.085686+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:54.003556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.091452+0000","last_change":"2023-10-09T10:57:22.014923+0000","last_active":"2023-10-09T10:57:24.091452+0000","last_peered":"2023-10-09T10:57:24.091452+0000","last_clean":"2023-10-09T10:57:24.091452+0000","last_became_active":"2023-10-09T10:57:22.014055+0000","last_became_peered":"2023-10-09T10:57:22.014055+0000","last_unstale":"2023-10-09T10:57:24.091452+0000","last_undegraded":"2023-10-09T10:57:24.091452+0000","last_fullsized":"2023-10-09T10:57:24.091452+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:50.546435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":641,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085338+0000","last_change":"2023-10-09T10:54:50.916062+0000","last_active":"2023-10-09T10:57:24.085338+0000","last_peered":"2023-10-09T10:57:24.085338+0000","last_clean":"2023-10-09T10:57:24.085338+0000","last_became_active":"2023-10-09T10:54:50.915189+0000","last_became_peered":"2023-10-09T10:54:50.915189+0000","last_unstale":"2023-10-09T10:57:24.085338+0000","last_undegraded":"2023-10-09T10:57:24.085338+0000","last_fullsized":"2023-10-09T10:57:24.085338+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.077788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.077788+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.087083+0000","last_change":"2023-10-09T10:57:22.016705+0000","last_active":"2023-10-09T10:57:24.087083+0000","last_peered":"2023-10-09T10:57:24.087083+0000","last_clean":"2023-10-09T10:57:24.087083+0000","last_became_active":"2023-10-09T10:57:22.016452+0000","last_became_peered":"2023-10-09T10:57:22.016452+0000","last_unstale":"2023-10-09T10:57:24.087083+0000","last_undegraded":"2023-10-09T10:57:24.087083+0000","last_fullsized":"2023-10-09T10:57:24.087083+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:43:44.382821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":244,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086687+0000","last_change":"2023-10-09T10:57:22.007046+0000","last_active":"2023-10-09T10:57:24.086687+0000","last_peered":"2023-10-09T10:57:24.086687+0000","last_clean":"2023-10-09T10:57:24.086687+0000","last_became_active":"2023-10-09T10:57:22.006360+0000","last_became_peered":"2023-10-09T10:57:22.006360+0000","last_unstale":"2023-10-09T10:57:24.086687+0000","last_undegraded":"2023-10-09T10:57:24.086687+0000","last_fullsized":"2023-10-09T10:57:24.086687+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:23.023072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":668,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.084348+0000","last_change":"2023-10-09T10:54:50.910027+0000","last_active":"2023-10-09T10:57:24.084348+0000","last_peered":"2023-10-09T10:57:24.084348+0000","last_clean":"2023-10-09T10:57:24.084348+0000","last_became_active":"2023-10-09T10:54:50.909698+0000","last_became_peered":"2023-10-09T10:54:50.909698+0000","last_unstale":"2023-10-09T10:57:24.084348+0000","last_undegraded":"2023-10-09T10:57:24.084348+0000","last_fullsized":"2023-10-09T10:57:24.084348+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:01.637196+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":211,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085760+0000","last_change":"2023-10-09T10:51:12.366910+0000","last_active":"2023-10-09T10:57:24.085760+0000","last_peered":"2023-10-09T10:57:24.085760+0000","last_clean":"2023-10-09T10:57:24.085760+0000","last_became_active":"2023-10-09T10:51:12.366565+0000","last_became_peered":"2023-10-09T10:51:12.366565+0000","last_unstale":"2023-10-09T10:57:24.085760+0000","last_undegraded":"2023-10-09T10:57:24.085760+0000","last_fullsized":"2023-10-09T10:57:24.085760+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:53.464251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":389,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.079938+0000","last_change":"2023-10-09T10:57:22.008343+0000","last_active":"2023-10-09T10:57:24.079938+0000","last_peered":"2023-10-09T10:57:24.079938+0000","last_clean":"2023-10-09T10:57:24.079938+0000","last_became_active":"2023-10-09T10:57:22.007747+0000","last_became_peered":"2023-10-09T10:57:22.007747+0000","last_unstale":"2023-10-09T10:57:24.079938+0000","last_undegraded":"2023-10-09T10:57:24.079938+0000","last_fullsized":"2023-10-09T10:57:24.079938+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:45.774703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"336'1279","reported_seq":1303,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:32.604816+0000","last_change":"2023-10-09T10:57:22.013979+0000","last_active":"2023-10-09T10:57:32.604816+0000","last_peered":"2023-10-09T10:57:32.604816+0000","last_clean":"2023-10-09T10:57:32.604816+0000","last_became_active":"2023-10-09T10:57:22.013514+0000","last_became_peered":"2023-10-09T10:57:22.013514+0000","last_unstale":"2023-10-09T10:57:32.604816+0000","last_undegraded":"2023-10-09T10:57:32.604816+0000","last_fullsized":"2023-10-09T10:57:32.604816+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":1279,"log_dups_size":0,"ondisk_log_size":1279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:16:42.975508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83820544,"num_objects":1279,"num_object_clones":0,"num_object_copies":3837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1279,"num_write_kb":81856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":676,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.102011+0000","last_change":"2023-10-09T10:57:25.102011+0000","last_active":"2023-10-09T10:57:25.102011+0000","last_peered":"2023-10-09T10:57:25.102011+0000","last_clean":"2023-10-09T10:57:25.102011+0000","last_became_active":"2023-10-09T10:57:25.101702+0000","last_became_peered":"2023-10-09T10:57:25.101702+0000","last_unstale":"2023-10-09T10:57:25.102011+0000","last_undegraded":"2023-10-09T10:57:25.102011+0000","last_fullsized":"2023-10-09T10:57:25.102011+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:34.671859+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":17,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.087145+0000","last_change":"2023-10-09T10:57:22.021225+0000","last_active":"2023-10-09T10:57:24.087145+0000","last_peered":"2023-10-09T10:57:24.087145+0000","last_clean":"2023-10-09T10:57:24.087145+0000","last_became_active":"2023-10-09T10:57:22.007997+0000","last_became_peered":"2023-10-09T10:57:22.007997+0000","last_unstale":"2023-10-09T10:57:24.087145+0000","last_undegraded":"2023-10-09T10:57:24.087145+0000","last_fullsized":"2023-10-09T10:57:24.087145+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:52:19.871862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.090281+0000","last_change":"2023-10-09T10:57:22.016166+0000","last_active":"2023-10-09T10:57:24.090281+0000","last_peered":"2023-10-09T10:57:24.090281+0000","last_clean":"2023-10-09T10:57:24.090281+0000","last_became_active":"2023-10-09T10:57:22.015850+0000","last_became_peered":"2023-10-09T10:57:22.015850+0000","last_unstale":"2023-10-09T10:57:24.090281+0000","last_undegraded":"2023-10-09T10:57:24.090281+0000","last_fullsized":"2023-10-09T10:57:24.090281+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:37.827566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"336'1148","reported_seq":1163,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:31.803082+0000","last_change":"2023-10-09T10:57:19.986807+0000","last_active":"2023-10-09T10:57:31.803082+0000","last_peered":"2023-10-09T10:57:31.803082+0000","last_clean":"2023-10-09T10:57:31.803082+0000","last_became_active":"2023-10-09T10:57:19.986525+0000","last_became_peered":"2023-10-09T10:57:19.986525+0000","last_unstale":"2023-10-09T10:57:31.803082+0000","last_undegraded":"2023-10-09T10:57:31.803082+0000","last_fullsized":"2023-10-09T10:57:31.803082+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":1148,"log_dups_size":0,"ondisk_log_size":1148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:23:53.293903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75235328,"num_objects":1148,"num_object_clones":0,"num_object_copies":3444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1148,"num_write_kb":73472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":696,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.079783+0000","last_change":"2023-10-09T10:53:25.540494+0000","last_active":"2023-10-09T10:57:24.079783+0000","last_peered":"2023-10-09T10:57:24.079783+0000","last_clean":"2023-10-09T10:57:24.079783+0000","last_became_active":"2023-10-09T10:51:01.295474+0000","last_became_peered":"2023-10-09T10:51:01.295474+0000","last_unstale":"2023-10-09T10:57:24.079783+0000","last_undegraded":"2023-10-09T10:57:24.079783+0000","last_fullsized":"2023-10-09T10:57:24.079783+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:39.069143+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":17,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.087784+0000","last_change":"2023-10-09T10:57:22.023259+0000","last_active":"2023-10-09T10:57:24.087784+0000","last_peered":"2023-10-09T10:57:24.087784+0000","last_clean":"2023-10-09T10:57:24.087784+0000","last_became_active":"2023-10-09T10:57:22.009992+0000","last_became_peered":"2023-10-09T10:57:22.009992+0000","last_unstale":"2023-10-09T10:57:24.087784+0000","last_undegraded":"2023-10-09T10:57:24.087784+0000","last_fullsized":"2023-10-09T10:57:24.087784+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.289131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":53,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.083002+0000","last_change":"2023-10-09T10:54:47.944930+0000","last_active":"2023-10-09T10:57:24.083002+0000","last_peered":"2023-10-09T10:57:24.083002+0000","last_clean":"2023-10-09T10:57:24.083002+0000","last_became_active":"2023-10-09T10:54:47.944485+0000","last_became_peered":"2023-10-09T10:54:47.944485+0000","last_unstale":"2023-10-09T10:57:24.083002+0000","last_undegraded":"2023-10-09T10:57:24.083002+0000","last_fullsized":"2023-10-09T10:57:24.083002+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:11.213129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"336'1084","reported_seq":1097,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:31.170615+0000","last_change":"2023-10-09T10:57:22.024907+0000","last_active":"2023-10-09T10:57:31.170615+0000","last_peered":"2023-10-09T10:57:31.170615+0000","last_clean":"2023-10-09T10:57:31.170615+0000","last_became_active":"2023-10-09T10:57:22.024660+0000","last_became_peered":"2023-10-09T10:57:22.024660+0000","last_unstale":"2023-10-09T10:57:31.170615+0000","last_undegraded":"2023-10-09T10:57:31.170615+0000","last_fullsized":"2023-10-09T10:57:31.170615+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":1084,"log_dups_size":0,"ondisk_log_size":1084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:25:10.105154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71041024,"num_objects":1084,"num_object_clones":0,"num_object_copies":3252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1084,"num_write_kb":69376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":648,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.082455+0000","last_change":"2023-10-09T10:53:36.680150+0000","last_active":"2023-10-09T10:57:24.082455+0000","last_peered":"2023-10-09T10:57:24.082455+0000","last_clean":"2023-10-09T10:57:24.082455+0000","last_became_active":"2023-10-09T10:51:50.371032+0000","last_became_peered":"2023-10-09T10:51:50.371032+0000","last_unstale":"2023-10-09T10:57:24.082455+0000","last_undegraded":"2023-10-09T10:57:24.082455+0000","last_fullsized":"2023-10-09T10:57:24.082455+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.680090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.680090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:08:28.952827+0000","scrub_duration":0.005577172,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":17,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.090378+0000","last_change":"2023-10-09T10:57:22.034075+0000","last_active":"2023-10-09T10:57:24.090378+0000","last_peered":"2023-10-09T10:57:24.090378+0000","last_clean":"2023-10-09T10:57:24.090378+0000","last_became_active":"2023-10-09T10:57:22.015930+0000","last_became_peered":"2023-10-09T10:57:22.015930+0000","last_unstale":"2023-10-09T10:57:24.090378+0000","last_undegraded":"2023-10-09T10:57:24.090378+0000","last_fullsized":"2023-10-09T10:57:24.090378+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:16.494793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":71,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.084234+0000","last_change":"2023-10-09T10:57:22.005094+0000","last_active":"2023-10-09T10:57:24.084234+0000","last_peered":"2023-10-09T10:57:24.084234+0000","last_clean":"2023-10-09T10:57:24.084234+0000","last_became_active":"2023-10-09T10:57:22.004592+0000","last_became_peered":"2023-10-09T10:57:22.004592+0000","last_unstale":"2023-10-09T10:57:24.084234+0000","last_undegraded":"2023-10-09T10:57:24.084234+0000","last_fullsized":"2023-10-09T10:57:24.084234+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:35.526442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"336'1072","reported_seq":1087,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:30.961936+0000","last_change":"2023-10-09T10:57:20.305591+0000","last_active":"2023-10-09T10:57:30.961936+0000","last_peered":"2023-10-09T10:57:30.961936+0000","last_clean":"2023-10-09T10:57:30.961936+0000","last_became_active":"2023-10-09T10:57:20.305290+0000","last_became_peered":"2023-10-09T10:57:20.305290+0000","last_unstale":"2023-10-09T10:57:30.961936+0000","last_undegraded":"2023-10-09T10:57:30.961936+0000","last_fullsized":"2023-10-09T10:57:30.961936+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":1072,"log_dups_size":0,"ondisk_log_size":1072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:54.880555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70254592,"num_objects":1072,"num_object_clones":0,"num_object_copies":3216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1072,"num_write_kb":68608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":821,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.111238+0000","last_change":"2023-10-09T10:57:25.111238+0000","last_active":"2023-10-09T10:57:25.111238+0000","last_peered":"2023-10-09T10:57:25.111238+0000","last_clean":"2023-10-09T10:57:25.111238+0000","last_became_active":"2023-10-09T10:57:25.110991+0000","last_became_peered":"2023-10-09T10:57:25.110991+0000","last_unstale":"2023-10-09T10:57:25.111238+0000","last_undegraded":"2023-10-09T10:57:25.111238+0000","last_fullsized":"2023-10-09T10:57:25.111238+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:57.060423+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":162,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.080789+0000","last_change":"2023-10-09T10:51:50.374527+0000","last_active":"2023-10-09T10:57:24.080789+0000","last_peered":"2023-10-09T10:57:24.080789+0000","last_clean":"2023-10-09T10:57:24.080789+0000","last_became_active":"2023-10-09T10:51:50.374297+0000","last_became_peered":"2023-10-09T10:51:50.374297+0000","last_unstale":"2023-10-09T10:57:24.080789+0000","last_undegraded":"2023-10-09T10:57:24.080789+0000","last_fullsized":"2023-10-09T10:57:24.080789+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:31:49.358307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":245,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086169+0000","last_change":"2023-10-09T10:54:47.939771+0000","last_active":"2023-10-09T10:57:24.086169+0000","last_peered":"2023-10-09T10:57:24.086169+0000","last_clean":"2023-10-09T10:57:24.086169+0000","last_became_active":"2023-10-09T10:54:47.939440+0000","last_became_peered":"2023-10-09T10:54:47.939440+0000","last_unstale":"2023-10-09T10:57:24.086169+0000","last_undegraded":"2023-10-09T10:57:24.086169+0000","last_fullsized":"2023-10-09T10:57:24.086169+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:58.335536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"336'1009","reported_seq":1033,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:30.541238+0000","last_change":"2023-10-09T10:57:22.009934+0000","last_active":"2023-10-09T10:57:30.541238+0000","last_peered":"2023-10-09T10:57:30.541238+0000","last_clean":"2023-10-09T10:57:30.541238+0000","last_became_active":"2023-10-09T10:57:22.009682+0000","last_became_peered":"2023-10-09T10:57:22.009682+0000","last_unstale":"2023-10-09T10:57:30.541238+0000","last_undegraded":"2023-10-09T10:57:30.541238+0000","last_fullsized":"2023-10-09T10:57:30.541238+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":1009,"log_dups_size":0,"ondisk_log_size":1009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:59.899734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66125824,"num_objects":1009,"num_object_clones":0,"num_object_copies":3027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1009,"num_write_kb":64576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":691,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.106422+0000","last_change":"2023-10-09T10:57:25.106422+0000","last_active":"2023-10-09T10:57:25.106422+0000","last_peered":"2023-10-09T10:57:25.106422+0000","last_clean":"2023-10-09T10:57:25.106422+0000","last_became_active":"2023-10-09T10:57:25.105998+0000","last_became_peered":"2023-10-09T10:57:25.105998+0000","last_unstale":"2023-10-09T10:57:25.106422+0000","last_undegraded":"2023-10-09T10:57:25.106422+0000","last_fullsized":"2023-10-09T10:57:25.106422+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:10:53.164214+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":333,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.078849+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:57:24.078849+0000","last_peered":"2023-10-09T10:57:24.078849+0000","last_clean":"2023-10-09T10:57:24.078849+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:57:24.078849+0000","last_undegraded":"2023-10-09T10:57:24.078849+0000","last_fullsized":"2023-10-09T10:57:24.078849+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":178,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.110488+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T10:57:25.110488+0000","last_peered":"2023-10-09T10:57:25.110488+0000","last_clean":"2023-10-09T10:57:25.110488+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T10:57:25.110488+0000","last_undegraded":"2023-10-09T10:57:25.110488+0000","last_fullsized":"2023-10-09T10:57:25.110488+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"336'1150","reported_seq":1165,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:31.810513+0000","last_change":"2023-10-09T10:57:20.327412+0000","last_active":"2023-10-09T10:57:31.810513+0000","last_peered":"2023-10-09T10:57:31.810513+0000","last_clean":"2023-10-09T10:57:31.810513+0000","last_became_active":"2023-10-09T10:57:20.327192+0000","last_became_peered":"2023-10-09T10:57:20.327192+0000","last_unstale":"2023-10-09T10:57:31.810513+0000","last_undegraded":"2023-10-09T10:57:31.810513+0000","last_fullsized":"2023-10-09T10:57:31.810513+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":1150,"log_dups_size":0,"ondisk_log_size":1150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:17:53.257184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75366400,"num_objects":1150,"num_object_clones":0,"num_object_copies":3450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1150,"num_write_kb":73600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":640,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.082775+0000","last_change":"2023-10-09T10:54:21.173078+0000","last_active":"2023-10-09T10:57:24.082775+0000","last_peered":"2023-10-09T10:57:24.082775+0000","last_clean":"2023-10-09T10:57:24.082775+0000","last_became_active":"2023-10-09T10:54:21.172767+0000","last_became_peered":"2023-10-09T10:54:21.172767+0000","last_unstale":"2023-10-09T10:57:24.082775+0000","last_undegraded":"2023-10-09T10:57:24.082775+0000","last_fullsized":"2023-10-09T10:57:24.082775+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.532890+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.532890+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:32.335841+0000","scrub_duration":0.0045290130000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":315,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085546+0000","last_change":"2023-10-09T10:54:47.908506+0000","last_active":"2023-10-09T10:57:24.085546+0000","last_peered":"2023-10-09T10:57:24.085546+0000","last_clean":"2023-10-09T10:57:24.085546+0000","last_became_active":"2023-10-09T10:54:47.908110+0000","last_became_peered":"2023-10-09T10:54:47.908110+0000","last_unstale":"2023-10-09T10:57:24.085546+0000","last_undegraded":"2023-10-09T10:57:24.085546+0000","last_fullsized":"2023-10-09T10:57:24.085546+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:38.808943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.083647+0000","last_change":"2023-10-09T10:54:47.948732+0000","last_active":"2023-10-09T10:57:24.083647+0000","last_peered":"2023-10-09T10:57:24.083647+0000","last_clean":"2023-10-09T10:57:24.083647+0000","last_became_active":"2023-10-09T10:54:47.948414+0000","last_became_peered":"2023-10-09T10:54:47.948414+0000","last_unstale":"2023-10-09T10:57:24.083647+0000","last_undegraded":"2023-10-09T10:57:24.083647+0000","last_fullsized":"2023-10-09T10:57:24.083647+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:38.020807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"336'1070","reported_seq":1083,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:31.141069+0000","last_change":"2023-10-09T10:57:22.024943+0000","last_active":"2023-10-09T10:57:31.141069+0000","last_peered":"2023-10-09T10:57:31.141069+0000","last_clean":"2023-10-09T10:57:31.141069+0000","last_became_active":"2023-10-09T10:57:22.024623+0000","last_became_peered":"2023-10-09T10:57:22.024623+0000","last_unstale":"2023-10-09T10:57:31.141069+0000","last_undegraded":"2023-10-09T10:57:31.141069+0000","last_fullsized":"2023-10-09T10:57:31.141069+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":1070,"log_dups_size":0,"ondisk_log_size":1070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:13:55.093719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70123520,"num_objects":1070,"num_object_clones":0,"num_object_copies":3210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1070,"num_write_kb":68480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":711,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085580+0000","last_change":"2023-10-09T10:54:50.914888+0000","last_active":"2023-10-09T10:57:24.085580+0000","last_peered":"2023-10-09T10:57:24.085580+0000","last_clean":"2023-10-09T10:57:24.085580+0000","last_became_active":"2023-10-09T10:54:50.909639+0000","last_became_peered":"2023-10-09T10:54:50.909639+0000","last_unstale":"2023-10-09T10:57:24.085580+0000","last_undegraded":"2023-10-09T10:57:24.085580+0000","last_fullsized":"2023-10-09T10:57:24.085580+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.103750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.103750+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":391,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.082267+0000","last_change":"2023-10-09T10:57:22.006224+0000","last_active":"2023-10-09T10:57:24.082267+0000","last_peered":"2023-10-09T10:57:24.082267+0000","last_clean":"2023-10-09T10:57:24.082267+0000","last_became_active":"2023-10-09T10:57:22.005898+0000","last_became_peered":"2023-10-09T10:57:22.005898+0000","last_unstale":"2023-10-09T10:57:24.082267+0000","last_undegraded":"2023-10-09T10:57:24.082267+0000","last_fullsized":"2023-10-09T10:57:24.082267+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:00.623898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":53,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085857+0000","last_change":"2023-10-09T10:54:47.913826+0000","last_active":"2023-10-09T10:57:24.085857+0000","last_peered":"2023-10-09T10:57:24.085857+0000","last_clean":"2023-10-09T10:57:24.085857+0000","last_became_active":"2023-10-09T10:54:47.913524+0000","last_became_peered":"2023-10-09T10:54:47.913524+0000","last_unstale":"2023-10-09T10:57:24.085857+0000","last_undegraded":"2023-10-09T10:57:24.085857+0000","last_fullsized":"2023-10-09T10:57:24.085857+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:56:23.699552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.2","version":"336'972","reported_seq":987,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:30.554444+0000","last_change":"2023-10-09T10:57:19.986622+0000","last_active":"2023-10-09T10:57:30.554444+0000","last_peered":"2023-10-09T10:57:30.554444+0000","last_clean":"2023-10-09T10:57:30.554444+0000","last_became_active":"2023-10-09T10:57:19.986337+0000","last_became_peered":"2023-10-09T10:57:19.986337+0000","last_unstale":"2023-10-09T10:57:30.554444+0000","last_undegraded":"2023-10-09T10:57:30.554444+0000","last_fullsized":"2023-10-09T10:57:30.554444+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":972,"log_dups_size":0,"ondisk_log_size":972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:59.817044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63700992,"num_objects":972,"num_object_clones":0,"num_object_copies":2916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":972,"num_write_kb":62208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":651,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.083671+0000","last_change":"2023-10-09T10:54:21.152148+0000","last_active":"2023-10-09T10:57:24.083671+0000","last_peered":"2023-10-09T10:57:24.083671+0000","last_clean":"2023-10-09T10:57:24.083671+0000","last_became_active":"2023-10-09T10:54:21.151809+0000","last_became_peered":"2023-10-09T10:54:21.151809+0000","last_unstale":"2023-10-09T10:57:24.083671+0000","last_undegraded":"2023-10-09T10:57:24.083671+0000","last_fullsized":"2023-10-09T10:57:24.083671+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.701128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.701128+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:27.021145+0000","scrub_duration":0.0025071830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":382,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.078374+0000","last_change":"2023-10-09T10:57:22.006965+0000","last_active":"2023-10-09T10:57:24.078374+0000","last_peered":"2023-10-09T10:57:24.078374+0000","last_clean":"2023-10-09T10:57:24.078374+0000","last_became_active":"2023-10-09T10:57:22.006456+0000","last_became_peered":"2023-10-09T10:57:22.006456+0000","last_unstale":"2023-10-09T10:57:24.078374+0000","last_undegraded":"2023-10-09T10:57:24.078374+0000","last_fullsized":"2023-10-09T10:57:24.078374+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:16.889422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":53,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086173+0000","last_change":"2023-10-09T10:54:48.012411+0000","last_active":"2023-10-09T10:57:24.086173+0000","last_peered":"2023-10-09T10:57:24.086173+0000","last_clean":"2023-10-09T10:57:24.086173+0000","last_became_active":"2023-10-09T10:54:48.011894+0000","last_became_peered":"2023-10-09T10:54:48.011894+0000","last_unstale":"2023-10-09T10:57:24.086173+0000","last_undegraded":"2023-10-09T10:57:24.086173+0000","last_fullsized":"2023-10-09T10:57:24.086173+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:28.067337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"336'1209","reported_seq":1224,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:31.806901+0000","last_change":"2023-10-09T10:57:20.306271+0000","last_active":"2023-10-09T10:57:31.806901+0000","last_peered":"2023-10-09T10:57:31.806901+0000","last_clean":"2023-10-09T10:57:31.806901+0000","last_became_active":"2023-10-09T10:57:20.305994+0000","last_became_peered":"2023-10-09T10:57:20.305994+0000","last_unstale":"2023-10-09T10:57:31.806901+0000","last_undegraded":"2023-10-09T10:57:31.806901+0000","last_fullsized":"2023-10-09T10:57:31.806901+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":1209,"log_dups_size":0,"ondisk_log_size":1209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:46.777641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79233024,"num_objects":1209,"num_object_clones":0,"num_object_copies":3627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1209,"num_write_kb":77376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":767,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.110928+0000","last_change":"2023-10-09T10:57:25.110928+0000","last_active":"2023-10-09T10:57:25.110928+0000","last_peered":"2023-10-09T10:57:25.110928+0000","last_clean":"2023-10-09T10:57:25.110928+0000","last_became_active":"2023-10-09T10:57:25.110240+0000","last_became_peered":"2023-10-09T10:57:25.110240+0000","last_unstale":"2023-10-09T10:57:25.110928+0000","last_undegraded":"2023-10-09T10:57:25.110928+0000","last_fullsized":"2023-10-09T10:57:25.110928+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:34.828051+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":71,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.084842+0000","last_change":"2023-10-09T10:57:22.005377+0000","last_active":"2023-10-09T10:57:24.084842+0000","last_peered":"2023-10-09T10:57:24.084842+0000","last_clean":"2023-10-09T10:57:24.084842+0000","last_became_active":"2023-10-09T10:57:22.005016+0000","last_became_peered":"2023-10-09T10:57:22.005016+0000","last_unstale":"2023-10-09T10:57:24.084842+0000","last_undegraded":"2023-10-09T10:57:24.084842+0000","last_fullsized":"2023-10-09T10:57:24.084842+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:03.296528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":256,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.079231+0000","last_change":"2023-10-09T10:54:17.886921+0000","last_active":"2023-10-09T10:57:24.079231+0000","last_peered":"2023-10-09T10:57:24.079231+0000","last_clean":"2023-10-09T10:57:24.079231+0000","last_became_active":"2023-10-09T10:54:17.886584+0000","last_became_peered":"2023-10-09T10:54:17.886584+0000","last_unstale":"2023-10-09T10:57:24.079231+0000","last_undegraded":"2023-10-09T10:57:24.079231+0000","last_fullsized":"2023-10-09T10:57:24.079231+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:15.848951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":53,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.084084+0000","last_change":"2023-10-09T10:54:47.908294+0000","last_active":"2023-10-09T10:57:24.084084+0000","last_peered":"2023-10-09T10:57:24.084084+0000","last_clean":"2023-10-09T10:57:24.084084+0000","last_became_active":"2023-10-09T10:54:47.908019+0000","last_became_peered":"2023-10-09T10:54:47.908019+0000","last_unstale":"2023-10-09T10:57:24.084084+0000","last_undegraded":"2023-10-09T10:57:24.084084+0000","last_fullsized":"2023-10-09T10:57:24.084084+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:48.662764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"336'1269","reported_seq":1293,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:32.639821+0000","last_change":"2023-10-09T10:57:22.005714+0000","last_active":"2023-10-09T10:57:32.639821+0000","last_peered":"2023-10-09T10:57:32.639821+0000","last_clean":"2023-10-09T10:57:32.639821+0000","last_became_active":"2023-10-09T10:57:22.005431+0000","last_became_peered":"2023-10-09T10:57:22.005431+0000","last_unstale":"2023-10-09T10:57:32.639821+0000","last_undegraded":"2023-10-09T10:57:32.639821+0000","last_fullsized":"2023-10-09T10:57:32.639821+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":1269,"log_dups_size":0,"ondisk_log_size":1269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:51.650262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83165184,"num_objects":1269,"num_object_clones":0,"num_object_copies":3807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1269,"num_write_kb":81216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":673,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086446+0000","last_change":"2023-10-09T10:54:21.163072+0000","last_active":"2023-10-09T10:57:24.086446+0000","last_peered":"2023-10-09T10:57:24.086446+0000","last_clean":"2023-10-09T10:57:24.086446+0000","last_became_active":"2023-10-09T10:54:21.162710+0000","last_became_peered":"2023-10-09T10:54:21.162710+0000","last_unstale":"2023-10-09T10:57:24.086446+0000","last_undegraded":"2023-10-09T10:57:24.086446+0000","last_fullsized":"2023-10-09T10:57:24.086446+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:15.932806+0000","scrub_duration":0.0019146790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":53,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.084154+0000","last_change":"2023-10-09T10:54:47.899967+0000","last_active":"2023-10-09T10:57:24.084154+0000","last_peered":"2023-10-09T10:57:24.084154+0000","last_clean":"2023-10-09T10:57:24.084154+0000","last_became_active":"2023-10-09T10:54:47.899584+0000","last_became_peered":"2023-10-09T10:54:47.899584+0000","last_unstale":"2023-10-09T10:57:24.084154+0000","last_undegraded":"2023-10-09T10:57:24.084154+0000","last_fullsized":"2023-10-09T10:57:24.084154+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:43.973868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":246,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085882+0000","last_change":"2023-10-09T10:50:43.388615+0000","last_active":"2023-10-09T10:57:24.085882+0000","last_peered":"2023-10-09T10:57:24.085882+0000","last_clean":"2023-10-09T10:57:24.085882+0000","last_became_active":"2023-10-09T10:50:43.388176+0000","last_became_peered":"2023-10-09T10:50:43.388176+0000","last_unstale":"2023-10-09T10:57:24.085882+0000","last_undegraded":"2023-10-09T10:57:24.085882+0000","last_fullsized":"2023-10-09T10:57:24.085882+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:41:17.550111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":53,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.084762+0000","last_change":"2023-10-09T10:54:47.950462+0000","last_active":"2023-10-09T10:57:24.084762+0000","last_peered":"2023-10-09T10:57:24.084762+0000","last_clean":"2023-10-09T10:57:24.084762+0000","last_became_active":"2023-10-09T10:54:47.940606+0000","last_became_peered":"2023-10-09T10:54:47.940606+0000","last_unstale":"2023-10-09T10:57:24.084762+0000","last_undegraded":"2023-10-09T10:57:24.084762+0000","last_fullsized":"2023-10-09T10:57:24.084762+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:50.557850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"336'1188","reported_seq":1212,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:31.817477+0000","last_change":"2023-10-09T10:57:22.004913+0000","last_active":"2023-10-09T10:57:31.817477+0000","last_peered":"2023-10-09T10:57:31.817477+0000","last_clean":"2023-10-09T10:57:31.817477+0000","last_became_active":"2023-10-09T10:57:22.004172+0000","last_became_peered":"2023-10-09T10:57:22.004172+0000","last_unstale":"2023-10-09T10:57:31.817477+0000","last_undegraded":"2023-10-09T10:57:31.817477+0000","last_fullsized":"2023-10-09T10:57:31.817477+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":1188,"log_dups_size":0,"ondisk_log_size":1188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:01.045466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77856768,"num_objects":1188,"num_object_clones":0,"num_object_copies":3564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1188,"num_write_kb":76032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":750,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.101168+0000","last_change":"2023-10-09T10:57:25.101168+0000","last_active":"2023-10-09T10:57:25.101168+0000","last_peered":"2023-10-09T10:57:25.101168+0000","last_clean":"2023-10-09T10:57:25.101168+0000","last_became_active":"2023-10-09T10:57:25.100860+0000","last_became_peered":"2023-10-09T10:57:25.100860+0000","last_unstale":"2023-10-09T10:57:25.101168+0000","last_undegraded":"2023-10-09T10:57:25.101168+0000","last_fullsized":"2023-10-09T10:57:25.101168+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:37.594583+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":66,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086464+0000","last_change":"2023-10-09T10:54:17.885030+0000","last_active":"2023-10-09T10:57:24.086464+0000","last_peered":"2023-10-09T10:57:24.086464+0000","last_clean":"2023-10-09T10:57:24.086464+0000","last_became_active":"2023-10-09T10:54:17.884752+0000","last_became_peered":"2023-10-09T10:54:17.884752+0000","last_unstale":"2023-10-09T10:57:24.086464+0000","last_undegraded":"2023-10-09T10:57:24.086464+0000","last_fullsized":"2023-10-09T10:57:24.086464+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:38.548132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":53,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.088865+0000","last_change":"2023-10-09T10:54:47.927822+0000","last_active":"2023-10-09T10:57:24.088865+0000","last_peered":"2023-10-09T10:57:24.088865+0000","last_clean":"2023-10-09T10:57:24.088865+0000","last_became_active":"2023-10-09T10:54:47.927455+0000","last_became_peered":"2023-10-09T10:54:47.927455+0000","last_unstale":"2023-10-09T10:57:24.088865+0000","last_undegraded":"2023-10-09T10:57:24.088865+0000","last_fullsized":"2023-10-09T10:57:24.088865+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:11.818203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":53,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086091+0000","last_change":"2023-10-09T10:54:48.012629+0000","last_active":"2023-10-09T10:57:24.086091+0000","last_peered":"2023-10-09T10:57:24.086091+0000","last_clean":"2023-10-09T10:57:24.086091+0000","last_became_active":"2023-10-09T10:54:48.012148+0000","last_became_peered":"2023-10-09T10:54:48.012148+0000","last_unstale":"2023-10-09T10:57:24.086091+0000","last_undegraded":"2023-10-09T10:57:24.086091+0000","last_fullsized":"2023-10-09T10:57:24.086091+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:22.894111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"336'936","reported_seq":950,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:30.559856+0000","last_change":"2023-10-09T10:57:20.305367+0000","last_active":"2023-10-09T10:57:30.559856+0000","last_peered":"2023-10-09T10:57:30.559856+0000","last_clean":"2023-10-09T10:57:30.559856+0000","last_became_active":"2023-10-09T10:57:20.305050+0000","last_became_peered":"2023-10-09T10:57:20.305050+0000","last_unstale":"2023-10-09T10:57:30.559856+0000","last_undegraded":"2023-10-09T10:57:30.559856+0000","last_fullsized":"2023-10-09T10:57:30.559856+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":936,"log_dups_size":0,"ondisk_log_size":936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:49.570577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61341696,"num_objects":936,"num_object_clones":0,"num_object_copies":2808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":936,"num_write_kb":59904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":84,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.083376+0000","last_change":"2023-10-09T10:57:22.006186+0000","last_active":"2023-10-09T10:57:24.083376+0000","last_peered":"2023-10-09T10:57:24.083376+0000","last_clean":"2023-10-09T10:57:24.083376+0000","last_became_active":"2023-10-09T10:57:22.005900+0000","last_became_peered":"2023-10-09T10:57:22.005900+0000","last_unstale":"2023-10-09T10:57:24.083376+0000","last_undegraded":"2023-10-09T10:57:24.083376+0000","last_fullsized":"2023-10-09T10:57:24.083376+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:14.033785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":767,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.100325+0000","last_change":"2023-10-09T10:57:25.100325+0000","last_active":"2023-10-09T10:57:25.100325+0000","last_peered":"2023-10-09T10:57:25.100325+0000","last_clean":"2023-10-09T10:57:25.100325+0000","last_became_active":"2023-10-09T10:57:25.099940+0000","last_became_peered":"2023-10-09T10:57:25.099940+0000","last_unstale":"2023-10-09T10:57:25.100325+0000","last_undegraded":"2023-10-09T10:57:25.100325+0000","last_fullsized":"2023-10-09T10:57:25.100325+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:33.410507+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":268,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.078557+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:57:24.078557+0000","last_peered":"2023-10-09T10:57:24.078557+0000","last_clean":"2023-10-09T10:57:24.078557+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:57:24.078557+0000","last_undegraded":"2023-10-09T10:57:24.078557+0000","last_fullsized":"2023-10-09T10:57:24.078557+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":268,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.083137+0000","last_change":"2023-10-09T10:57:22.013585+0000","last_active":"2023-10-09T10:57:24.083137+0000","last_peered":"2023-10-09T10:57:24.083137+0000","last_clean":"2023-10-09T10:57:24.083137+0000","last_became_active":"2023-10-09T10:57:22.013255+0000","last_became_peered":"2023-10-09T10:57:22.013255+0000","last_unstale":"2023-10-09T10:57:24.083137+0000","last_undegraded":"2023-10-09T10:57:24.083137+0000","last_fullsized":"2023-10-09T10:57:24.083137+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:29.725918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"336'1248","reported_seq":1263,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:32.636666+0000","last_change":"2023-10-09T10:57:20.328496+0000","last_active":"2023-10-09T10:57:32.636666+0000","last_peered":"2023-10-09T10:57:32.636666+0000","last_clean":"2023-10-09T10:57:32.636666+0000","last_became_active":"2023-10-09T10:57:20.328194+0000","last_became_peered":"2023-10-09T10:57:20.328194+0000","last_unstale":"2023-10-09T10:57:32.636666+0000","last_undegraded":"2023-10-09T10:57:32.636666+0000","last_fullsized":"2023-10-09T10:57:32.636666+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":1248,"log_dups_size":0,"ondisk_log_size":1248,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:15:57.867795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81788928,"num_objects":1248,"num_object_clones":0,"num_object_copies":3744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1248,"num_write_kb":79872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":684,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085427+0000","last_change":"2023-10-09T10:54:50.915855+0000","last_active":"2023-10-09T10:57:24.085427+0000","last_peered":"2023-10-09T10:57:24.085427+0000","last_clean":"2023-10-09T10:57:24.085427+0000","last_became_active":"2023-10-09T10:54:50.914536+0000","last_became_peered":"2023-10-09T10:54:50.914536+0000","last_unstale":"2023-10-09T10:57:24.085427+0000","last_undegraded":"2023-10-09T10:57:24.085427+0000","last_fullsized":"2023-10-09T10:57:24.085427+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.070026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.070026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":13,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.090493+0000","last_change":"2023-10-09T10:57:22.008209+0000","last_active":"2023-10-09T10:57:24.090493+0000","last_peered":"2023-10-09T10:57:24.090493+0000","last_clean":"2023-10-09T10:57:24.090493+0000","last_became_active":"2023-10-09T10:57:22.007918+0000","last_became_peered":"2023-10-09T10:57:22.007918+0000","last_unstale":"2023-10-09T10:57:24.090493+0000","last_undegraded":"2023-10-09T10:57:24.090493+0000","last_fullsized":"2023-10-09T10:57:24.090493+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:55.764820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":218,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086156+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:57:24.086156+0000","last_peered":"2023-10-09T10:57:24.086156+0000","last_clean":"2023-10-09T10:57:24.086156+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:57:24.086156+0000","last_undegraded":"2023-10-09T10:57:24.086156+0000","last_fullsized":"2023-10-09T10:57:24.086156+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":281,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086763+0000","last_change":"2023-10-09T10:54:47.939756+0000","last_active":"2023-10-09T10:57:24.086763+0000","last_peered":"2023-10-09T10:57:24.086763+0000","last_clean":"2023-10-09T10:57:24.086763+0000","last_became_active":"2023-10-09T10:54:47.938928+0000","last_became_peered":"2023-10-09T10:54:47.938928+0000","last_unstale":"2023-10-09T10:57:24.086763+0000","last_undegraded":"2023-10-09T10:57:24.086763+0000","last_fullsized":"2023-10-09T10:57:24.086763+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:05.310929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"336'1141","reported_seq":1154,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:31.129729+0000","last_change":"2023-10-09T10:57:22.010174+0000","last_active":"2023-10-09T10:57:31.129729+0000","last_peered":"2023-10-09T10:57:31.129729+0000","last_clean":"2023-10-09T10:57:31.129729+0000","last_became_active":"2023-10-09T10:57:22.009987+0000","last_became_peered":"2023-10-09T10:57:22.009987+0000","last_unstale":"2023-10-09T10:57:31.129729+0000","last_undegraded":"2023-10-09T10:57:31.129729+0000","last_fullsized":"2023-10-09T10:57:31.129729+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":1141,"log_dups_size":0,"ondisk_log_size":1141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:45.169457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74776576,"num_objects":1141,"num_object_clones":0,"num_object_copies":3423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1141,"num_write_kb":73024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":709,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.087004+0000","last_change":"2023-10-09T10:53:27.625180+0000","last_active":"2023-10-09T10:57:24.087004+0000","last_peered":"2023-10-09T10:57:24.087004+0000","last_clean":"2023-10-09T10:57:24.087004+0000","last_became_active":"2023-10-09T10:51:12.366828+0000","last_became_peered":"2023-10-09T10:51:12.366828+0000","last_unstale":"2023-10-09T10:57:24.087004+0000","last_undegraded":"2023-10-09T10:57:24.087004+0000","last_fullsized":"2023-10-09T10:57:24.087004+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.625079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.625079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:59:41.253976+0000","scrub_duration":0.0041067969999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":13,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.090273+0000","last_change":"2023-10-09T10:57:22.012147+0000","last_active":"2023-10-09T10:57:24.090273+0000","last_peered":"2023-10-09T10:57:24.090273+0000","last_clean":"2023-10-09T10:57:24.090273+0000","last_became_active":"2023-10-09T10:57:22.011818+0000","last_became_peered":"2023-10-09T10:57:22.011818+0000","last_unstale":"2023-10-09T10:57:24.090273+0000","last_undegraded":"2023-10-09T10:57:24.090273+0000","last_fullsized":"2023-10-09T10:57:24.090273+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:10.838344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":53,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085247+0000","last_change":"2023-10-09T10:54:47.925711+0000","last_active":"2023-10-09T10:57:24.085247+0000","last_peered":"2023-10-09T10:57:24.085247+0000","last_clean":"2023-10-09T10:57:24.085247+0000","last_became_active":"2023-10-09T10:54:47.924286+0000","last_became_peered":"2023-10-09T10:54:47.924286+0000","last_unstale":"2023-10-09T10:57:24.085247+0000","last_undegraded":"2023-10-09T10:57:24.085247+0000","last_fullsized":"2023-10-09T10:57:24.085247+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:58.379080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":263,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086018+0000","last_change":"2023-10-09T10:57:22.004838+0000","last_active":"2023-10-09T10:57:24.086018+0000","last_peered":"2023-10-09T10:57:24.086018+0000","last_clean":"2023-10-09T10:57:24.086018+0000","last_became_active":"2023-10-09T10:57:22.004519+0000","last_became_peered":"2023-10-09T10:57:22.004519+0000","last_unstale":"2023-10-09T10:57:24.086018+0000","last_undegraded":"2023-10-09T10:57:24.086018+0000","last_fullsized":"2023-10-09T10:57:24.086018+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:53.585466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"336'1115","reported_seq":1128,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:31.129678+0000","last_change":"2023-10-09T10:57:22.015103+0000","last_active":"2023-10-09T10:57:31.129678+0000","last_peered":"2023-10-09T10:57:31.129678+0000","last_clean":"2023-10-09T10:57:31.129678+0000","last_became_active":"2023-10-09T10:57:22.014603+0000","last_became_peered":"2023-10-09T10:57:22.014603+0000","last_unstale":"2023-10-09T10:57:31.129678+0000","last_undegraded":"2023-10-09T10:57:31.129678+0000","last_fullsized":"2023-10-09T10:57:31.129678+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":1115,"log_dups_size":0,"ondisk_log_size":1115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:22:34.112316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73072640,"num_objects":1115,"num_object_clones":0,"num_object_copies":3345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1115,"num_write_kb":71360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":731,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.083080+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T10:57:24.083080+0000","last_peered":"2023-10-09T10:57:24.083080+0000","last_clean":"2023-10-09T10:57:24.083080+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T10:57:24.083080+0000","last_undegraded":"2023-10-09T10:57:24.083080+0000","last_fullsized":"2023-10-09T10:57:24.083080+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":385,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.079891+0000","last_change":"2023-10-09T10:57:22.030437+0000","last_active":"2023-10-09T10:57:24.079891+0000","last_peered":"2023-10-09T10:57:24.079891+0000","last_clean":"2023-10-09T10:57:24.079891+0000","last_became_active":"2023-10-09T10:57:22.008090+0000","last_became_peered":"2023-10-09T10:57:22.008090+0000","last_unstale":"2023-10-09T10:57:24.079891+0000","last_undegraded":"2023-10-09T10:57:24.079891+0000","last_fullsized":"2023-10-09T10:57:24.079891+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:52:54.221183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":53,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086002+0000","last_change":"2023-10-09T10:54:47.908305+0000","last_active":"2023-10-09T10:57:24.086002+0000","last_peered":"2023-10-09T10:57:24.086002+0000","last_clean":"2023-10-09T10:57:24.086002+0000","last_became_active":"2023-10-09T10:54:47.907999+0000","last_became_peered":"2023-10-09T10:54:47.907999+0000","last_unstale":"2023-10-09T10:57:24.086002+0000","last_undegraded":"2023-10-09T10:57:24.086002+0000","last_fullsized":"2023-10-09T10:57:24.086002+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:48.405124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":376,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.078368+0000","last_change":"2023-10-09T10:57:22.007248+0000","last_active":"2023-10-09T10:57:24.078368+0000","last_peered":"2023-10-09T10:57:24.078368+0000","last_clean":"2023-10-09T10:57:24.078368+0000","last_became_active":"2023-10-09T10:57:22.006899+0000","last_became_peered":"2023-10-09T10:57:22.006899+0000","last_unstale":"2023-10-09T10:57:24.078368+0000","last_undegraded":"2023-10-09T10:57:24.078368+0000","last_fullsized":"2023-10-09T10:57:24.078368+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:45.159091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"336'967","reported_seq":991,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:30.535041+0000","last_change":"2023-10-09T10:57:22.006548+0000","last_active":"2023-10-09T10:57:30.535041+0000","last_peered":"2023-10-09T10:57:30.535041+0000","last_clean":"2023-10-09T10:57:30.535041+0000","last_became_active":"2023-10-09T10:57:22.006030+0000","last_became_peered":"2023-10-09T10:57:22.006030+0000","last_unstale":"2023-10-09T10:57:30.535041+0000","last_undegraded":"2023-10-09T10:57:30.535041+0000","last_fullsized":"2023-10-09T10:57:30.535041+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":967,"log_dups_size":0,"ondisk_log_size":967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:47.539524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63373312,"num_objects":967,"num_object_clones":0,"num_object_copies":2901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":967,"num_write_kb":61888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":761,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.106977+0000","last_change":"2023-10-09T10:57:25.106977+0000","last_active":"2023-10-09T10:57:25.106977+0000","last_peered":"2023-10-09T10:57:25.106977+0000","last_clean":"2023-10-09T10:57:25.106977+0000","last_became_active":"2023-10-09T10:57:25.106632+0000","last_became_peered":"2023-10-09T10:57:25.106632+0000","last_unstale":"2023-10-09T10:57:25.106977+0000","last_undegraded":"2023-10-09T10:57:25.106977+0000","last_fullsized":"2023-10-09T10:57:25.106977+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:29:20.989344+0000","scrub_duration":0.0053707060000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":13,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.087144+0000","last_change":"2023-10-09T10:57:22.016009+0000","last_active":"2023-10-09T10:57:24.087144+0000","last_peered":"2023-10-09T10:57:24.087144+0000","last_clean":"2023-10-09T10:57:24.087144+0000","last_became_active":"2023-10-09T10:57:22.015633+0000","last_became_peered":"2023-10-09T10:57:22.015633+0000","last_unstale":"2023-10-09T10:57:24.087144+0000","last_undegraded":"2023-10-09T10:57:24.087144+0000","last_fullsized":"2023-10-09T10:57:24.087144+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.957416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":265,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086848+0000","last_change":"2023-10-09T10:54:48.016159+0000","last_active":"2023-10-09T10:57:24.086848+0000","last_peered":"2023-10-09T10:57:24.086848+0000","last_clean":"2023-10-09T10:57:24.086848+0000","last_became_active":"2023-10-09T10:54:47.939409+0000","last_became_peered":"2023-10-09T10:54:47.939409+0000","last_unstale":"2023-10-09T10:57:24.086848+0000","last_undegraded":"2023-10-09T10:57:24.086848+0000","last_fullsized":"2023-10-09T10:57:24.086848+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:11:15.175207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":277,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.082265+0000","last_change":"2023-10-09T10:51:50.371282+0000","last_active":"2023-10-09T10:57:24.082265+0000","last_peered":"2023-10-09T10:57:24.082265+0000","last_clean":"2023-10-09T10:57:24.082265+0000","last_became_active":"2023-10-09T10:51:50.370973+0000","last_became_peered":"2023-10-09T10:51:50.370973+0000","last_unstale":"2023-10-09T10:57:24.082265+0000","last_undegraded":"2023-10-09T10:57:24.082265+0000","last_fullsized":"2023-10-09T10:57:24.082265+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:15.603164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":727,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.080346+0000","last_change":"2023-10-09T10:53:29.568199+0000","last_active":"2023-10-09T10:57:24.080346+0000","last_peered":"2023-10-09T10:57:24.080346+0000","last_clean":"2023-10-09T10:57:24.080346+0000","last_became_active":"2023-10-09T10:52:55.416357+0000","last_became_peered":"2023-10-09T10:52:55.416357+0000","last_unstale":"2023-10-09T10:57:24.080346+0000","last_undegraded":"2023-10-09T10:57:24.080346+0000","last_fullsized":"2023-10-09T10:57:24.080346+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:44.364650+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":262,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.086100+0000","last_change":"2023-10-09T10:57:22.005039+0000","last_active":"2023-10-09T10:57:24.086100+0000","last_peered":"2023-10-09T10:57:24.086100+0000","last_clean":"2023-10-09T10:57:24.086100+0000","last_became_active":"2023-10-09T10:57:22.004497+0000","last_became_peered":"2023-10-09T10:57:22.004497+0000","last_unstale":"2023-10-09T10:57:24.086100+0000","last_undegraded":"2023-10-09T10:57:24.086100+0000","last_fullsized":"2023-10-09T10:57:24.086100+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:11.006259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":75,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085806+0000","last_change":"2023-10-09T10:54:47.944380+0000","last_active":"2023-10-09T10:57:24.085806+0000","last_peered":"2023-10-09T10:57:24.085806+0000","last_clean":"2023-10-09T10:57:24.085806+0000","last_became_active":"2023-10-09T10:54:47.943766+0000","last_became_peered":"2023-10-09T10:54:47.943766+0000","last_unstale":"2023-10-09T10:57:24.085806+0000","last_undegraded":"2023-10-09T10:57:24.085806+0000","last_fullsized":"2023-10-09T10:57:24.085806+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:16:11.869603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":693,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.111170+0000","last_change":"2023-10-09T10:57:25.111170+0000","last_active":"2023-10-09T10:57:25.111170+0000","last_peered":"2023-10-09T10:57:25.111170+0000","last_clean":"2023-10-09T10:57:25.111170+0000","last_became_active":"2023-10-09T10:57:25.110405+0000","last_became_peered":"2023-10-09T10:57:25.110405+0000","last_unstale":"2023-10-09T10:57:25.111170+0000","last_undegraded":"2023-10-09T10:57:25.111170+0000","last_fullsized":"2023-10-09T10:57:25.111170+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:00.112600+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":162,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.083776+0000","last_change":"2023-10-09T10:54:47.923386+0000","last_active":"2023-10-09T10:57:24.083776+0000","last_peered":"2023-10-09T10:57:24.083776+0000","last_clean":"2023-10-09T10:57:24.083776+0000","last_became_active":"2023-10-09T10:54:47.914257+0000","last_became_peered":"2023-10-09T10:54:47.914257+0000","last_unstale":"2023-10-09T10:57:24.083776+0000","last_undegraded":"2023-10-09T10:57:24.083776+0000","last_fullsized":"2023-10-09T10:57:24.083776+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:24:18.190715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":353,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.078920+0000","last_change":"2023-10-09T10:54:47.977518+0000","last_active":"2023-10-09T10:57:24.078920+0000","last_peered":"2023-10-09T10:57:24.078920+0000","last_clean":"2023-10-09T10:57:24.078920+0000","last_became_active":"2023-10-09T10:54:47.977332+0000","last_became_peered":"2023-10-09T10:54:47.977332+0000","last_unstale":"2023-10-09T10:57:24.078920+0000","last_undegraded":"2023-10-09T10:57:24.078920+0000","last_fullsized":"2023-10-09T10:57:24.078920+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:29.620839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":286,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.089182+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:57:24.089182+0000","last_peered":"2023-10-09T10:57:24.089182+0000","last_clean":"2023-10-09T10:57:24.089182+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:57:24.089182+0000","last_undegraded":"2023-10-09T10:57:24.089182+0000","last_fullsized":"2023-10-09T10:57:24.089182+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":280,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.078611+0000","last_change":"2023-10-09T10:50:43.385150+0000","last_active":"2023-10-09T10:57:24.078611+0000","last_peered":"2023-10-09T10:57:24.078611+0000","last_clean":"2023-10-09T10:57:24.078611+0000","last_became_active":"2023-10-09T10:50:43.384788+0000","last_became_peered":"2023-10-09T10:50:43.384788+0000","last_unstale":"2023-10-09T10:57:24.078611+0000","last_undegraded":"2023-10-09T10:57:24.078611+0000","last_fullsized":"2023-10-09T10:57:24.078611+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:36.085910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":617,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-10-09T10:57:25.105295+0000","last_change":"2023-10-09T10:57:25.105295+0000","last_active":"2023-10-09T10:57:25.105295+0000","last_peered":"2023-10-09T10:57:25.105295+0000","last_clean":"2023-10-09T10:57:25.105295+0000","last_became_active":"2023-10-09T10:57:25.104921+0000","last_became_peered":"2023-10-09T10:57:25.104921+0000","last_unstale":"2023-10-09T10:57:25.105295+0000","last_undegraded":"2023-10-09T10:57:25.105295+0000","last_fullsized":"2023-10-09T10:57:25.105295+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:46.305288+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:46.305288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":360,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.079671+0000","last_change":"2023-10-09T10:54:17.886158+0000","last_active":"2023-10-09T10:57:24.079671+0000","last_peered":"2023-10-09T10:57:24.079671+0000","last_clean":"2023-10-09T10:57:24.079671+0000","last_became_active":"2023-10-09T10:54:17.885816+0000","last_became_peered":"2023-10-09T10:54:17.885816+0000","last_unstale":"2023-10-09T10:57:24.079671+0000","last_undegraded":"2023-10-09T10:57:24.079671+0000","last_fullsized":"2023-10-09T10:57:24.079671+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:05.270825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":363,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.081226+0000","last_change":"2023-10-09T10:54:17.957784+0000","last_active":"2023-10-09T10:57:24.081226+0000","last_peered":"2023-10-09T10:57:24.081226+0000","last_clean":"2023-10-09T10:57:24.081226+0000","last_became_active":"2023-10-09T10:54:17.954788+0000","last_became_peered":"2023-10-09T10:54:17.954788+0000","last_unstale":"2023-10-09T10:57:24.081226+0000","last_undegraded":"2023-10-09T10:57:24.081226+0000","last_fullsized":"2023-10-09T10:57:24.081226+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:48.709390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":623,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085360+0000","last_change":"2023-10-09T10:57:21.022980+0000","last_active":"2023-10-09T10:57:24.085360+0000","last_peered":"2023-10-09T10:57:24.085360+0000","last_clean":"2023-10-09T10:57:24.085360+0000","last_became_active":"2023-10-09T10:54:50.914005+0000","last_became_peered":"2023-10-09T10:54:50.914005+0000","last_unstale":"2023-10-09T10:57:24.085360+0000","last_undegraded":"2023-10-09T10:57:24.085360+0000","last_fullsized":"2023-10-09T10:57:24.085360+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:21.022863+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:21.022863+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:49.534216+0000","scrub_duration":0.00136929,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":245,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.085613+0000","last_change":"2023-10-09T10:54:18.089053+0000","last_active":"2023-10-09T10:57:24.085613+0000","last_peered":"2023-10-09T10:57:24.085613+0000","last_clean":"2023-10-09T10:57:24.085613+0000","last_became_active":"2023-10-09T10:54:17.886070+0000","last_became_peered":"2023-10-09T10:54:17.886070+0000","last_unstale":"2023-10-09T10:57:24.085613+0000","last_undegraded":"2023-10-09T10:57:24.085613+0000","last_fullsized":"2023-10-09T10:57:24.085613+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:35.628407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":340,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-10-09T10:57:24.083663+0000","last_change":"2023-10-09T10:54:47.908059+0000","last_active":"2023-10-09T10:57:24.083663+0000","last_peered":"2023-10-09T10:57:24.083663+0000","last_clean":"2023-10-09T10:57:24.083663+0000","last_became_active":"2023-10-09T10:54:47.907780+0000","last_became_peered":"2023-10-09T10:54:47.907780+0000","last_unstale":"2023-10-09T10:57:24.083663+0000","last_undegraded":"2023-10-09T10:57:24.083663+0000","last_fullsized":"2023-10-09T10:57:24.083663+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:55:59.064617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":16490,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1375602,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1170276352,"num_objects":17857,"num_object_clones":0,"num_object_copies":53571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17857,"num_write_kb":1142848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3485532160,"data_stored":3485532160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":17857,"ondisk_log_size":17857,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332371,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1233288,"kb_used_data":936000,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92511864,"statfs":{"total":95995035648,"available":94732148736,"internally_reserved":0,"allocated":958464000,"data_stored":957261479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,0,0,0,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:56:59 2023","interfaces":[{"interface":"back","average":{"1min":2.024,"5min":1.9139999999999999,"15min":1.462},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":7.2830000000000004,"5min":9.6259999999999994,"15min":9.6379999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.4260000000000002,"5min":2.0920000000000001,"15min":1.669},"min":{"1min":0.26000000000000001,"5min":0.246,"15min":0.24299999999999999},"max":{"1min":8.7769999999999992,"5min":8.7769999999999992,"15min":16.731000000000002},"last":0.95099999999999996}]},{"osd":1,"last update":"Mon Oct 9 10:57:09 2023","interfaces":[{"interface":"back","average":{"1min":1.661,"5min":2.2480000000000002,"15min":1.78},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":12.252000000000001,"5min":34.756,"15min":34.756},"last":1.107},{"interface":"front","average":{"1min":2.3620000000000001,"5min":2.081,"15min":1.5660000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":13.647,"5min":35.658999999999999,"15min":35.658999999999999},"last":1.1870000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:56:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8610000000000002,"5min":1.415,"15min":1.4650000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.202,"5min":9.202,"15min":11.058},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.3250000000000002,"5min":1.6379999999999999,"15min":1.3520000000000001},"min":{"1min":0.29599999999999999,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":9.4280000000000008,"5min":11.579000000000001,"15min":11.579000000000001},"last":0.32000000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:56:59 2023","interfaces":[{"interface":"back","average":{"1min":2.6640000000000001,"5min":2.0760000000000001,"15min":1.8440000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":11.760999999999999,"5min":11.760999999999999,"15min":11.760999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.8200000000000001,"5min":1.6160000000000001,"15min":1.607},"min":{"1min":0.375,"5min":0.313,"15min":0.313},"max":{"1min":4.9459999999999997,"5min":8.6300000000000008,"15min":19.234999999999999},"last":0.83199999999999996}]},{"osd":4,"last update":"Mon Oct 9 10:56:36 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.94399999999999995,"15min":1.234},"min":{"1min":0.27200000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.569,"5min":8.2940000000000005,"15min":16.695},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.88800000000000001,"15min":1.1299999999999999},"min":{"1min":0.28699999999999998,"5min":0.28199999999999997,"15min":0.26200000000000001},"max":{"1min":4.1100000000000003,"5min":7.9320000000000004,"15min":11.109999999999999},"last":0.45400000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:56:36 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.84499999999999997,"15min":1.163},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":3.7330000000000001,"5min":4.7619999999999996,"15min":13.262},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.98899999999999999,"15min":1.3440000000000001},"min":{"1min":0.33500000000000002,"5min":0.318,"15min":0.29599999999999999},"max":{"1min":2.363,"5min":8.4339999999999993,"15min":11.741},"last":0.46600000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:56:36 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.2070000000000001,"15min":1.46},"min":{"1min":0.315,"5min":0.315,"15min":0.28199999999999997},"max":{"1min":3.5139999999999998,"5min":8.3610000000000007,"15min":14.913},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.123,"5min":1.1120000000000001,"15min":1.3100000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.3259999999999996,"5min":12.268000000000001,"15min":19.187999999999999},"last":0.58799999999999997}]}]},{"osd":6,"up_from":44,"seq":188978561302,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1458040,"kb_used_data":557872,"kb_used_omap":0,"kb_used_meta":900160,"kb_avail":92287112,"statfs":{"total":95995035648,"available":94502002688,"internally_reserved":0,"allocated":571260928,"data_stored":570074101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921763840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:56:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":2.1509999999999998,"15min":1.575},"min":{"1min":0.23499999999999999,"5min":0.222,"15min":0.216},"max":{"1min":8.6890000000000001,"5min":54.743000000000002,"15min":54.743000000000002},"last":0.33200000000000002},{"interface":"front","average":{"1min":2.238,"5min":2.0950000000000002,"15min":1.5940000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.22800000000000001},"max":{"1min":8.657,"5min":53.701000000000001,"15min":53.701000000000001},"last":1.2470000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:57:14 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.3129999999999999,"15min":1.3959999999999999},"min":{"1min":0.31900000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.8959999999999999,"5min":7.6820000000000004,"15min":16.747},"last":0.44},{"interface":"front","average":{"1min":1.49,"5min":1.363,"15min":1.446},"min":{"1min":0.27900000000000003,"5min":0.23599999999999999,"15min":0.23100000000000001},"max":{"1min":5.3079999999999998,"5min":7.5049999999999999,"15min":16.734000000000002},"last":0.43099999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:56:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1389999999999998,"5min":1.873,"15min":1.508},"min":{"1min":0.29799999999999999,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":6.2400000000000002,"5min":8.6389999999999993,"15min":13.208},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.6839999999999999,"5min":1.7,"15min":1.635},"min":{"1min":0.32500000000000001,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":5.6829999999999998,"5min":12.83,"15min":12.83},"last":0.41799999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:56:56 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":1.843,"15min":1.573},"min":{"1min":0.33900000000000002,"5min":0.32500000000000001,"15min":0.255},"max":{"1min":7.8680000000000003,"5min":7.8680000000000003,"15min":12.391},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.778,"5min":2.1669999999999998,"15min":1.8149999999999999},"min":{"1min":0.31900000000000001,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":8.782,"5min":8.782,"15min":13.179},"last":0.40000000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":1.157,"15min":1.1759999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.23200000000000001},"max":{"1min":2.0110000000000001,"5min":16.978999999999999,"15min":16.978999999999999},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.93799999999999994,"15min":1.145},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.867,"5min":5.2380000000000004,"15min":12.481999999999999},"last":0.50700000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":1.4890000000000001,"15min":1.3069999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.5880000000000001,"5min":12.923,"15min":12.923},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.73299999999999998,"5min":1.1299999999999999,"15min":1.206},"min":{"1min":0.312,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.7719999999999998,"5min":9.6769999999999996,"15min":12.456},"last":0.71799999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:56:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":1.234,"15min":1.3460000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":3.3540000000000001,"5min":12.882,"15min":23.358000000000001},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.76700000000000002,"5min":1,"15min":1.145},"min":{"1min":0.30299999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.1259999999999999,"5min":6.1029999999999998,"15min":12.422000000000001},"last":0.58999999999999997}]}]},{"osd":1,"up_from":85,"seq":365072220341,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1048328,"kb_used_data":611648,"kb_used_omap":9,"kb_used_meta":436662,"kb_avail":92696824,"statfs":{"total":95995035648,"available":94921547776,"internally_reserved":0,"allocated":626327552,"data_stored":624545840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":447142889},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:57:07 2023","interfaces":[{"interface":"back","average":{"1min":3.153,"5min":2.5310000000000001,"15min":1.9259999999999999},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":8.2240000000000002,"5min":19.992999999999999,"15min":19.992999999999999},"last":0.44},{"interface":"front","average":{"1min":3.5800000000000001,"5min":3.048,"15min":2.0539999999999998},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":10.901999999999999,"5min":18.303000000000001,"15min":18.303000000000001},"last":6.9790000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:56:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8149999999999999,"5min":1.9450000000000001,"15min":1.504},"min":{"1min":0.26100000000000001,"5min":0.222,"15min":0.19700000000000001},"max":{"1min":14.878,"5min":14.878,"15min":15.356},"last":5.569},{"interface":"front","average":{"1min":3.0430000000000001,"5min":1.986,"15min":1.6519999999999999},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":11.481999999999999,"5min":18.004000000000001,"15min":18.004000000000001},"last":0.47399999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:56:57 2023","interfaces":[{"interface":"back","average":{"1min":4.5860000000000003,"5min":2.7189999999999999,"15min":2.129},"min":{"1min":0.28499999999999998,"5min":0.22900000000000001,"15min":0.218},"max":{"1min":18.289999999999999,"5min":20.036000000000001,"15min":20.036000000000001},"last":5.4539999999999997},{"interface":"front","average":{"1min":2.964,"5min":3.1320000000000001,"15min":2.117},"min":{"1min":0.27200000000000002,"5min":0.219,"15min":0.219},"max":{"1min":14.303000000000001,"5min":19.640999999999998,"15min":19.640999999999998},"last":6.9340000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.7330000000000001,"15min":1.4630000000000001},"min":{"1min":0.28899999999999998,"5min":0.26500000000000001,"15min":0.255},"max":{"1min":7.3369999999999997,"5min":14.763,"15min":14.763},"last":0.39100000000000001},{"interface":"front","average":{"1min":2.508,"5min":1.7869999999999999,"15min":1.6659999999999999},"min":{"1min":0.317,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.3079999999999998,"5min":8.3079999999999998,"15min":14.228999999999999},"last":0.379}]},{"osd":5,"last update":"Mon Oct 9 10:57:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2080000000000002,"5min":2.0499999999999998,"15min":1.554},"min":{"1min":0.34000000000000002,"5min":0.32700000000000001,"15min":0.253},"max":{"1min":8.8040000000000003,"5min":9.6289999999999996,"15min":9.7720000000000002},"last":5.9989999999999997},{"interface":"front","average":{"1min":2.0350000000000001,"5min":2.105,"15min":1.827},"min":{"1min":0.32500000000000001,"5min":0.30299999999999999,"15min":0.28899999999999998},"max":{"1min":8.5020000000000007,"5min":14.833,"15min":17.085000000000001},"last":0.36099999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:57:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2229999999999999,"5min":2.0030000000000001,"15min":1.6399999999999999},"min":{"1min":0.33400000000000002,"5min":0.32600000000000001,"15min":0.27400000000000002},"max":{"1min":8.8230000000000004,"5min":9.4049999999999994,"15min":10.425000000000001},"last":5.9630000000000001},{"interface":"front","average":{"1min":2.1179999999999999,"5min":1.9279999999999999,"15min":1.6459999999999999},"min":{"1min":0.34799999999999998,"5min":0.312,"15min":0.312},"max":{"1min":8.5730000000000004,"5min":14.804,"15min":14.804},"last":0.41999999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:57:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6950000000000001,"5min":2.1819999999999999,"15min":1.708},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34000000000000002},"max":{"1min":7.4009999999999998,"5min":16.312000000000001,"15min":16.312000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":2.2839999999999998,"5min":1.8660000000000001,"15min":1.671},"min":{"1min":0.374,"5min":0.35499999999999998,"15min":0.32500000000000001},"max":{"1min":6.4930000000000003,"5min":8.8149999999999995,"15min":14.273999999999999},"last":0.41099999999999998}]}]},{"osd":0,"up_from":80,"seq":343597383875,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":830860,"kb_used_data":303684,"kb_used_omap":28,"kb_used_meta":527139,"kb_avail":92914292,"statfs":{"total":95995035648,"available":95144235008,"internally_reserved":0,"allocated":310972416,"data_stored":309123574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":539791184},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:57:02 2023","interfaces":[{"interface":"back","average":{"1min":4.4290000000000003,"5min":2.2440000000000002,"15min":1.905},"min":{"1min":0.22900000000000001,"5min":0.20399999999999999,"15min":0.191},"max":{"1min":12.782,"5min":13.263999999999999,"15min":25.475000000000001},"last":0.19500000000000001},{"interface":"front","average":{"1min":3.726,"5min":2.0899999999999999,"15min":1.837},"min":{"1min":0.32300000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":10.478999999999999,"5min":10.478999999999999,"15min":20.832999999999998},"last":0.29099999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:56:37 2023","interfaces":[{"interface":"back","average":{"1min":4.1479999999999997,"5min":2.4969999999999999,"15min":1.8109999999999999},"min":{"1min":0.42199999999999999,"5min":0.189,"15min":0.189},"max":{"1min":13.976000000000001,"5min":24.852,"15min":24.852},"last":1.1819999999999999},{"interface":"front","average":{"1min":3.6400000000000001,"5min":3.331,"15min":2.2200000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.256,"5min":77.256,"15min":77.256},"last":0.26200000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:57:02 2023","interfaces":[{"interface":"back","average":{"1min":2.7509999999999999,"5min":2.3069999999999999,"15min":2.1000000000000001},"min":{"1min":0.29699999999999999,"5min":0.22500000000000001,"15min":0.20200000000000001},"max":{"1min":7.0780000000000003,"5min":18.972000000000001,"15min":18.972000000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":4.7439999999999998,"5min":2.7010000000000001,"15min":2.1859999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":12.722,"5min":13.305,"15min":14.391999999999999},"last":0.46999999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:56:57 2023","interfaces":[{"interface":"back","average":{"1min":2.52,"5min":2.8980000000000001,"15min":2.0249999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.26200000000000001},"max":{"1min":13.836,"5min":77.177999999999997,"15min":77.177999999999997},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.242,"5min":1.829,"15min":1.819},"min":{"1min":0.36099999999999999,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":10.513999999999999,"5min":10.513999999999999,"15min":12.318},"last":0.375}]},{"osd":5,"last update":"Mon Oct 9 10:56:57 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":1.9450000000000001,"15min":1.8420000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.29799999999999999},"max":{"1min":10.544,"5min":10.544,"15min":12.157},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.7490000000000001,"5min":1.8859999999999999,"15min":1.7470000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29499999999999998},"max":{"1min":10.494999999999999,"5min":24.754999999999999,"15min":24.754999999999999},"last":0.50600000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:56:57 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":2.2269999999999999,"15min":1.9810000000000001},"min":{"1min":0.378,"5min":0.34200000000000003,"15min":0.30399999999999999},"max":{"1min":10.433999999999999,"5min":18.920000000000002,"15min":18.920000000000002},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.7589999999999999,"5min":2.0859999999999999,"15min":1.6599999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.307},"max":{"1min":10.566000000000001,"5min":24.800000000000001,"15min":24.800000000000001},"last":0.56399999999999995}]},{"osd":7,"last update":"Mon Oct 9 10:56:57 2023","interfaces":[{"interface":"back","average":{"1min":2.7919999999999998,"5min":2.052,"15min":1.8540000000000001},"min":{"1min":0.35399999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":10.541,"5min":13.225,"15min":14.48},"last":0.53400000000000003},{"interface":"front","average":{"1min":2.1840000000000002,"5min":2.891,"15min":2.1429999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":7.6699999999999999,"5min":77.228999999999999,"15min":77.228999999999999},"last":0.59499999999999997}]}]},{"osd":2,"up_from":90,"seq":386547056815,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":516632,"kb_used_data":203152,"kb_used_omap":81,"kb_used_meta":313390,"kb_avail":93228520,"statfs":{"total":95995035648,"available":95466004480,"internally_reserved":0,"allocated":208027648,"data_stored":206343721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":320911578},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:56:37 2023","interfaces":[{"interface":"back","average":{"1min":3.0920000000000001,"5min":2.0150000000000001,"15min":1.5880000000000001},"min":{"1min":0.188,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":11.737,"5min":17.574000000000002,"15min":18.512},"last":0.219},{"interface":"front","average":{"1min":3.1440000000000001,"5min":2.0870000000000002,"15min":1.7190000000000001},"min":{"1min":0.34100000000000003,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":10.18,"5min":15.595000000000001,"15min":17.975000000000001},"last":0.255}]},{"osd":1,"last update":"Mon Oct 9 10:56:37 2023","interfaces":[{"interface":"back","average":{"1min":2.6600000000000001,"5min":1.879,"15min":1.671},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":16.359000000000002,"5min":16.359000000000002,"15min":16.359000000000002},"last":0.69199999999999995},{"interface":"front","average":{"1min":2.7440000000000002,"5min":1.633,"15min":1.4550000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.20699999999999999},"max":{"1min":12.693,"5min":12.693,"15min":16.454999999999998},"last":2.3919999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:57:00 2023","interfaces":[{"interface":"back","average":{"1min":4.157,"5min":2.4420000000000002,"15min":1.8129999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":13.276,"5min":19.539000000000001,"15min":19.539000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":2.899,"5min":2.415,"15min":2.0299999999999998},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.20200000000000001},"max":{"1min":15.058999999999999,"5min":16.702999999999999,"15min":16.702999999999999},"last":0.46400000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:56:37 2023","interfaces":[{"interface":"back","average":{"1min":2.464,"5min":1.8859999999999999,"15min":1.7190000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.254},"max":{"1min":12.452999999999999,"5min":15.672000000000001,"15min":16.231999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.853,"15min":1.657},"min":{"1min":0.32100000000000001,"5min":0.307,"15min":0.26300000000000001},"max":{"1min":8.157,"5min":15.696999999999999,"15min":15.696999999999999},"last":0.39300000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:56:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":1.6579999999999999,"15min":1.6279999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.245},"max":{"1min":10.135999999999999,"5min":12.128,"15min":17.920000000000002},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.3330000000000002,"5min":1.8520000000000001,"15min":1.665},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":12.484,"5min":16.597000000000001,"15min":17.882999999999999},"last":0.42999999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:56:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1459999999999999,"5min":1.6739999999999999,"15min":1.4530000000000001},"min":{"1min":0.40699999999999997,"5min":0.33400000000000002,"15min":0.247},"max":{"1min":7.7190000000000003,"5min":8.8460000000000001,"15min":14.029999999999999},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.911,"5min":1.6779999999999999,"15min":1.5880000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.29199999999999998},"max":{"1min":10.111000000000001,"5min":10.111000000000001,"15min":12.862},"last":0.50700000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:56:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1070000000000002,"5min":1.6479999999999999,"15min":1.4710000000000001},"min":{"1min":0.42199999999999999,"5min":0.371,"15min":0.32500000000000001},"max":{"1min":7.7439999999999998,"5min":8.8879999999999999,"15min":13.999000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.161,"5min":1.8899999999999999,"15min":1.679},"min":{"1min":0.378,"5min":0.34000000000000002,"15min":0.307},"max":{"1min":10.157999999999999,"5min":10.157999999999999,"15min":12.826000000000001},"last":0.54600000000000004}]}]},{"osd":3,"up_from":95,"seq":408021893289,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1186652,"kb_used_data":469652,"kb_used_omap":7,"kb_used_meta":716984,"kb_avail":92558500,"statfs":{"total":95995035648,"available":94779904000,"internally_reserved":0,"allocated":480923648,"data_stored":479058020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":734192232},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:57:05 2023","interfaces":[{"interface":"back","average":{"1min":3.6850000000000001,"5min":3.2250000000000001,"15min":2.1589999999999998},"min":{"1min":0.23400000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":10.239000000000001,"5min":38.817999999999998,"15min":38.817999999999998},"last":0.26600000000000001},{"interface":"front","average":{"1min":5.3879999999999999,"5min":3.9329999999999998,"15min":2.4929999999999999},"min":{"1min":0.318,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":14.263999999999999,"5min":17.312000000000001,"15min":17.312000000000001},"last":0.20200000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:57:05 2023","interfaces":[{"interface":"back","average":{"1min":3.9430000000000001,"5min":2.7530000000000001,"15min":2.2509999999999999},"min":{"1min":0.218,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":22.469000000000001,"5min":22.469000000000001,"15min":22.469000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":5.165,"5min":3.2959999999999998,"15min":2.298},"min":{"1min":0.27000000000000002,"5min":0.22,"15min":0.193},"max":{"1min":12.247,"5min":18.526,"15min":18.526},"last":0.40999999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:57:05 2023","interfaces":[{"interface":"back","average":{"1min":3.9510000000000001,"5min":2.7410000000000001,"15min":2.121},"min":{"1min":0.30399999999999999,"5min":0.26300000000000001,"15min":0.23699999999999999},"max":{"1min":20.402000000000001,"5min":20.402000000000001,"15min":20.402000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":5.048,"5min":3.7240000000000002,"15min":2.4249999999999998},"min":{"1min":0.379,"5min":0.248,"15min":0.248},"max":{"1min":27.538,"5min":27.538,"15min":27.538},"last":0.60399999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:57:05 2023","interfaces":[{"interface":"back","average":{"1min":2.79,"5min":2.3769999999999998,"15min":1.917},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":10.17,"5min":11.863,"15min":20.388000000000002},"last":0.318},{"interface":"front","average":{"1min":2.5979999999999999,"5min":2.2959999999999998,"15min":1.883},"min":{"1min":0.39400000000000002,"5min":0.28399999999999997,"15min":0.24299999999999999},"max":{"1min":7.5810000000000004,"5min":11.680999999999999,"15min":19.085999999999999},"last":0.38100000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:57:05 2023","interfaces":[{"interface":"back","average":{"1min":2.847,"5min":2.4089999999999998,"15min":2.1619999999999999},"min":{"1min":0.35899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.1519999999999992,"5min":14.045,"15min":51.509999999999998},"last":0.436},{"interface":"front","average":{"1min":3.125,"5min":2.0409999999999999,"15min":2.1179999999999999},"min":{"1min":0.34100000000000003,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":17.311,"5min":17.311,"15min":55.338999999999999},"last":0.503}]},{"osd":6,"last update":"Mon Oct 9 10:57:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7629999999999999,"5min":2.4550000000000001,"15min":1.8859999999999999},"min":{"1min":0.311,"5min":0.29199999999999998,"15min":0.28000000000000003},"max":{"1min":7.609,"5min":11.723000000000001,"15min":12.361000000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":3.3100000000000001,"5min":2.2839999999999998,"15min":2.0350000000000001},"min":{"1min":0.36599999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":17.295000000000002,"5min":17.295000000000002,"15min":17.295000000000002},"last":0.39500000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:57:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1360000000000001,"5min":2.3180000000000001,"15min":2.262},"min":{"1min":0.35499999999999998,"5min":0.34999999999999998,"15min":0.31},"max":{"1min":17.263999999999999,"5min":18.722000000000001,"15min":51.744},"last":0.47499999999999998},{"interface":"front","average":{"1min":3.0950000000000002,"5min":2.532,"15min":2.2690000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.31900000000000001},"max":{"1min":10.209,"5min":13.337,"15min":53.399000000000001},"last":0.45800000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019165,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":944772,"kb_used_data":350908,"kb_used_omap":0,"kb_used_meta":593856,"kb_avail":92800380,"statfs":{"total":95995035648,"available":95027589120,"internally_reserved":0,"allocated":359329792,"data_stored":358131076,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608108544},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:56:54 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":4.1260000000000003,"15min":2.169},"min":{"1min":0.26400000000000001,"5min":0.217,"15min":0.20999999999999999},"max":{"1min":15.128,"5min":213.88999999999999,"15min":213.88999999999999},"last":0.97599999999999998},{"interface":"front","average":{"1min":2.7469999999999999,"5min":4.3949999999999996,"15min":2.476},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.217},"max":{"1min":14.359999999999999,"5min":214.09800000000001,"15min":214.09800000000001},"last":0.65500000000000003}]},{"osd":1,"last update":"Mon Oct 9 10:56:59 2023","interfaces":[{"interface":"back","average":{"1min":2.2120000000000002,"5min":3.9460000000000002,"15min":2.1800000000000002},"min":{"1min":0.27300000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":6.4320000000000004,"5min":206.50399999999999,"15min":206.50399999999999},"last":1.0189999999999999},{"interface":"front","average":{"1min":2.3010000000000002,"5min":4.1600000000000001,"15min":2.3940000000000001},"min":{"1min":0.29999999999999999,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":6.4199999999999999,"5min":206.529,"15min":206.529},"last":0.46800000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:56:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.9390000000000001,"15min":1.619},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.253},"max":{"1min":5.7089999999999996,"5min":15.446999999999999,"15min":15.446999999999999},"last":0.68400000000000005},{"interface":"front","average":{"1min":2.141,"5min":2.29,"15min":1.851},"min":{"1min":0.316,"5min":0.29599999999999999,"15min":0.26300000000000001},"max":{"1min":6.5510000000000002,"5min":27.818000000000001,"15min":27.818000000000001},"last":0.61599999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:56:54 2023","interfaces":[{"interface":"back","average":{"1min":2.1880000000000002,"5min":4.1920000000000002,"15min":2.4319999999999999},"min":{"1min":0.35299999999999998,"5min":0.29599999999999999,"15min":0.27400000000000002},"max":{"1min":11.372999999999999,"5min":209.03700000000001,"15min":209.03700000000001},"last":0.64300000000000002},{"interface":"front","average":{"1min":2.0550000000000002,"5min":4.3460000000000001,"15min":2.6419999999999999},"min":{"1min":0.36299999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.6349999999999998,"5min":208.80199999999999,"15min":208.80199999999999},"last":0.76100000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:56:59 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":1.581,"15min":1.581},"min":{"1min":0.254,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":4.8360000000000003,"5min":19.068999999999999,"15min":24.149999999999999},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.47999999999999998,"5min":1.2350000000000001,"15min":1.3859999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.399,"5min":15.535,"15min":20.422000000000001},"last":0.71099999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:56:32 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":1.2070000000000001,"15min":1.242},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":3.0550000000000002,"5min":15.739000000000001,"15min":15.739000000000001},"last":1},{"interface":"front","average":{"1min":0.68700000000000006,"5min":1.484,"15min":1.4710000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.311},"max":{"1min":3.375,"5min":14.093999999999999,"15min":14.272},"last":0.502}]},{"osd":7,"last update":"Mon Oct 9 10:56:39 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.5149999999999999,"15min":1.3979999999999999},"min":{"1min":0.33400000000000002,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":4.7800000000000002,"5min":14.458,"15min":14.458},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.534,"15min":1.298},"min":{"1min":0.34499999999999997,"5min":0.255,"15min":0.255},"max":{"1min":4.4290000000000003,"5min":15.592000000000001,"15min":15.592000000000001},"last":0.83999999999999997}]}]},{"osd":5,"up_from":37,"seq":158913790234,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1856500,"kb_used_data":953772,"kb_used_omap":0,"kb_used_meta":902720,"kb_avail":91888652,"statfs":{"total":95995035648,"available":94093979648,"internally_reserved":0,"allocated":976662528,"data_stored":975476161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924385280},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:57:00 2023","interfaces":[{"interface":"back","average":{"1min":3.355,"5min":2.198,"15min":1.623},"min":{"1min":0.22900000000000001,"5min":0.215,"15min":0.215},"max":{"1min":9.5359999999999996,"5min":16.449000000000002,"15min":16.449000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":3.3879999999999999,"5min":2.2469999999999999,"15min":1.6799999999999999},"min":{"1min":0.31,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.5760000000000005,"5min":16.489000000000001,"15min":16.489000000000001},"last":0.47199999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:57:13 2023","interfaces":[{"interface":"back","average":{"1min":2.407,"5min":2.1379999999999999,"15min":1.8600000000000001},"min":{"1min":0.27500000000000002,"5min":0.215,"15min":0.215},"max":{"1min":13.988,"5min":13.988,"15min":13.988},"last":1.45},{"interface":"front","average":{"1min":2.2850000000000001,"5min":2.0590000000000002,"15min":1.581},"min":{"1min":0.253,"5min":0.253,"15min":0.24299999999999999},"max":{"1min":6.1660000000000004,"5min":9.2699999999999996,"15min":12.109},"last":1.0640000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:56:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8380000000000001,"5min":2.105,"15min":1.7509999999999999},"min":{"1min":0.28299999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.7789999999999999,"5min":11.257999999999999,"15min":11.393000000000001},"last":3.867},{"interface":"front","average":{"1min":3.0259999999999998,"5min":2.306,"15min":1.569},"min":{"1min":0.28799999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.2569999999999997,"5min":15.087999999999999,"15min":15.087999999999999},"last":0.85999999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:57:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7080000000000002,"5min":2.1760000000000002,"15min":2.0939999999999999},"min":{"1min":0.32700000000000001,"5min":0.314,"15min":0.25700000000000001},"max":{"1min":8.7910000000000004,"5min":14.006,"15min":63.508000000000003},"last":0.39600000000000002},{"interface":"front","average":{"1min":3.6539999999999999,"5min":2.464,"15min":2.246},"min":{"1min":0.41599999999999998,"5min":0.32600000000000001,"15min":0.26300000000000001},"max":{"1min":13.805,"5min":13.805,"15min":62.418999999999997},"last":6.6550000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:57:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.93400000000000005,"15min":1.393},"min":{"1min":0.27000000000000002,"5min":0.26300000000000001,"15min":0.23200000000000001},"max":{"1min":3.3540000000000001,"5min":6.6120000000000001,"15min":123.241},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.98399999999999999,"15min":1.623},"min":{"1min":0.28199999999999997,"5min":0.28000000000000003,"15min":0.24199999999999999},"max":{"1min":4.8550000000000004,"5min":4.8550000000000004,"15min":123.318},"last":0.36499999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:57:24 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":1.093,"15min":1.3380000000000001},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.147,"5min":8.9049999999999994,"15min":11.098000000000001},"last":1.006},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.89700000000000002,"15min":1.2290000000000001},"min":{"1min":0.34999999999999998,"5min":0.318,"15min":0.30299999999999999},"max":{"1min":3.2909999999999999,"5min":6.7210000000000001,"15min":20.256},"last":0.41199999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:56:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.353,"15min":1.3169999999999999},"min":{"1min":0.33300000000000002,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":4.9950000000000001,"5min":8.468,"15min":10.739000000000001},"last":1.046},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.115,"15min":1.131},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.4350000000000001,"5min":10.858000000000001,"15min":14.154},"last":0.92800000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":328925184,"data_stored":328925184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":696713216,"data_stored":696713216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":204668928,"data_stored":204668928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":513409024,"data_stored":513409024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":216662016,"data_stored":216662016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":666107904,"data_stored":666107904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":215023616,"data_stored":215023616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":644022272,"data_stored":644022272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:57:35.195 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:57:35.195 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 71 2023-10-09T10:57:35.198 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:35.219 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:57:35.263 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:57:35.300 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:35.301 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:57:35.313 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:57:35.313 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:57:35.313 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:57:35.313 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:57:35.313 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:57:35.313 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:57:35.314 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:57:35.314 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:57:35.588 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:57:35.617 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 11 16 25675 25659 145.73 100.25 0.0118119 0.00685502 2023-10-09T10:57:35.733 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:35.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:35.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:35.801 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:35.839 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:35.882 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:35 smithi125 ceph-mon[200197]: from='client.55627 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:57:35.882 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:35 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1906958757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:35.883 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:35 smithi125 ceph-mon[195507]: from='client.55627 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:57:35.883 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:35 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1906958757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:35.911 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:35.957 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:35.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:35.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:35 smithi155 ceph-mon[184854]: from='client.55627 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:57:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:35 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1906958757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:36.065 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:36.104 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:36.136 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:36.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:36.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:36.241 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:36.287 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:36.292 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:36.492 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:57:36.568 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:36.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:36.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:36.617 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 12 16 27361 27345 142.361 105.375 0.00730568 0.00701903 2023-10-09T10:57:36.678 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:36.723 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:36.794 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:36 smithi125 ceph-mon[200197]: pgmap v829: 119 pgs: 119 active+clean; 1.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s; 27 KiB/s, 2 keys/s, 3 objects/s recovering 2023-10-09T10:57:36.794 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:36 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:36.795 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:36 smithi125 ceph-mon[195507]: pgmap v829: 119 pgs: 119 active+clean; 1.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s; 27 KiB/s, 2 keys/s, 3 objects/s recovering 2023-10-09T10:57:36.795 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:36 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:36.962 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:36.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:36.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:37.070 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:37.122 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:37.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:36 smithi155 ceph-mon[184854]: pgmap v829: 119 pgs: 119 active+clean; 1.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s; 27 KiB/s, 2 keys/s, 3 objects/s recovering 2023-10-09T10:57:37.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:36 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:57:37.350 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:37.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:37.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:37.454 INFO:teuthology.orchestra.run.smithi125.stderr:Error EINVAL: specified pg_num 71 < pg_num_max 32 2023-10-09T10:57:37.456 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:37.508 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:37.617 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 13 16 28919 28903 138.901 97.375 0.0065573 0.00719464 2023-10-09T10:57:37.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:37 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1180561550' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T10:57:37.885 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:57:38.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:37 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1180561550' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T10:57:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:37 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1180561550' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T10:57:38.617 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 14 16 30856 30840 137.625 121.062 0.00374503 0.00726064 2023-10-09T10:57:38.954 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:39.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:38 smithi125 ceph-mon[195507]: pgmap v830: 119 pgs: 119 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-10-09T10:57:39.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:38 smithi125 ceph-mon[200197]: pgmap v830: 119 pgs: 119 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-10-09T10:57:39.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:38 smithi155 ceph-mon[184854]: pgmap v830: 119 pgs: 119 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-10-09T10:57:39.617 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 15 16 32704 32688 136.15 115.5 0.00476029 0.00734031 2023-10-09T10:57:39.752 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T10:57:39.800 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:39.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:39.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:39.905 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:39.947 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:39.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:39.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:39.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:40.086 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:40.130 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:40.158 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:40.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:40.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:40.235 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:40.263 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:40.295 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:40.315 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:57:40.315 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:57:40.323 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:40.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:40.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:40.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:40.470 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:40.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:40.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:40.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:40.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:40.617 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 16 16 34369 34353 134.145 104.062 0.00593962 0.00745071 2023-10-09T10:57:40.644 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:40.672 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:40.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:40.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:40.775 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:40.813 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:40.840 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:40.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:40.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:40.945 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:40.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:40 smithi155 ceph-mon[184854]: pgmap v831: 119 pgs: 119 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-10-09T10:57:40.973 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:40 smithi125 ceph-mon[195507]: pgmap v831: 119 pgs: 119 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-10-09T10:57:40.973 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:40 smithi125 ceph-mon[200197]: pgmap v831: 119 pgs: 119 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-10-09T10:57:40.985 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:41.005 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:57:41.019 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:41.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:41.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:41.124 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:41.196 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:41.228 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:41.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:41.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:41.240 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:57:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:57:41] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-10-09T10:57:41.336 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:41.394 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:41.422 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:41.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:41.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:41.526 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:41.543 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:41.592 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:41.621 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 17 16 35768 35752 131.368 87.4375 0.00702842 0.00760673 2023-10-09T10:57:41.779 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:41.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:41.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:41.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:41.945 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:41.977 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:41.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:41.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:42.083 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:42.125 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:42.153 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:42.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:42.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:42.198 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:57:42.237 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:57:42.259 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:42.274 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:57:42.299 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:42.307 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:57:42.326 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:42.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:42.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:42.354 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:57:42.397 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:57:42.432 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:57:42.435 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:42.469 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:57:42.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:57:42.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:57:42.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:57:42.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:57:42.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:57:42.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:57:42.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:57:42.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:57:42.481 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:42.508 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:42.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:42.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:42.612 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:42.621 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 18 16 37236 37220 129.167 91.75 0.00472985 0.00773683 2023-10-09T10:57:42.691 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:42.717 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:42.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:42.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:42.803 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:57:42.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:42 smithi125 ceph-mon[195507]: pgmap v832: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2023-10-09T10:57:42.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:42 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2709374788' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:57:42.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:42 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:57:42.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:42 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/840229500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:42.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:42.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:42 smithi125 ceph-mon[200197]: pgmap v832: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2023-10-09T10:57:42.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:42 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2709374788' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:57:42.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:42 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:57:42.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:42 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/840229500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:42.858 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:57:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:57:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:37540 9.4 9:2b654e0c:::benchmark_data_smithi125_363789_object37539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-10-09T10:57:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:57:42.843357+0000", 2023-10-09T10:57:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.013732642, 2023-10-09T10:57:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013765588, 2023-10-09T10:57:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:57:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T10:57:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T10:57:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 37540 2023-10-09T10:57:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.843357+0000", 2023-10-09T10:57:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.843357+0000", 2023-10-09T10:57:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.843356+0000", 2023-10-09T10:57:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:57:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.843375+0000", 2023-10-09T10:57:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8714000000000001e-05 2023-10-09T10:57:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.843381+0000", 2023-10-09T10:57:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3060000000000003e-06 2023-10-09T10:57:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.843386+0000", 2023-10-09T10:57:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7770000000000002e-06 2023-10-09T10:57:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.845104+0000", 2023-10-09T10:57:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0017175409999999999 2023-10-09T10:57:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.845238+0000", 2023-10-09T10:57:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000133912 2023-10-09T10:57:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:57:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.845358+0000", 2023-10-09T10:57:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120893 2023-10-09T10:57:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.847486+0000", 2023-10-09T10:57:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021280489999999999 2023-10-09T10:57:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.850715+0000", 2023-10-09T10:57:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032286989999999998 2023-10-09T10:57:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:37542 9.4 9:2feb19f6:::benchmark_data_smithi125_363789_object37541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-10-09T10:57:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:57:42.848720+0000", 2023-10-09T10:57:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.008369853, 2023-10-09T10:57:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.008506886, 2023-10-09T10:57:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:57:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:42.865 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T10:57:42.865 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T10:57:42.865 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 37542 2023-10-09T10:57:42.865 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.865 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:42.865 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.865 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:42.865 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.848720+0000", 2023-10-09T10:57:42.865 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.865 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.865 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.865 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:42.865 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.848720+0000", 2023-10-09T10:57:42.866 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.866 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.866 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.866 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:42.866 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.848718+0000", 2023-10-09T10:57:42.866 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:42.866 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.866 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.866 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:42.866 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.848739+0000", 2023-10-09T10:57:42.866 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1369000000000002e-05 2023-10-09T10:57:42.867 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.867 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.867 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:42.867 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.848749+0000", 2023-10-09T10:57:42.867 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2110000000000004e-06 2023-10-09T10:57:42.867 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.867 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.867 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:42.867 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.848755+0000", 2023-10-09T10:57:42.867 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1079999999999998e-06 2023-10-09T10:57:42.867 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.867 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.868 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:42.868 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.848797+0000", 2023-10-09T10:57:42.868 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1819000000000001e-05 2023-10-09T10:57:42.868 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.868 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.868 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:42.868 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.848925+0000", 2023-10-09T10:57:42.868 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012874100000000001 2023-10-09T10:57:42.868 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.868 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.868 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T10:57:42.868 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.849064+0000", 2023-10-09T10:57:42.868 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000138166 2023-10-09T10:57:42.869 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.869 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.869 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:42.869 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.851066+0000", 2023-10-09T10:57:42.869 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020029499999999999 2023-10-09T10:57:42.869 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.869 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.869 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:42.869 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.852195+0000", 2023-10-09T10:57:42.869 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011286410000000001 2023-10-09T10:57:42.869 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.869 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:42.870 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.870 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.870 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.870 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:37545 9.f 9:f6cd6862:::benchmark_data_smithi125_363789_object37544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-10-09T10:57:42.870 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:57:42.851119+0000", 2023-10-09T10:57:42.870 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0059705670000000004, 2023-10-09T10:57:42.870 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062039180000000001, 2023-10-09T10:57:42.870 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:42.870 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:57:42.870 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:42.870 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T10:57:42.870 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T10:57:42.870 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 37545 2023-10-09T10:57:42.871 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.871 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:42.871 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.871 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:42.871 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.851119+0000", 2023-10-09T10:57:42.871 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.871 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.871 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.871 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:42.871 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.851119+0000", 2023-10-09T10:57:42.871 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.871 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.871 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.872 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:42.872 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.851117+0000", 2023-10-09T10:57:42.872 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:57:42.872 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.872 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.872 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:42.872 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.851142+0000", 2023-10-09T10:57:42.872 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4754999999999999e-05 2023-10-09T10:57:42.872 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.872 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.872 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:42.872 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.851152+0000", 2023-10-09T10:57:42.872 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0040999999999999e-05 2023-10-09T10:57:42.873 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.873 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.873 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:42.873 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.851159+0000", 2023-10-09T10:57:42.873 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0979999999999996e-06 2023-10-09T10:57:42.873 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.873 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.873 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:42.873 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.852719+0000", 2023-10-09T10:57:42.873 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001559922 2023-10-09T10:57:42.873 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.873 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.874 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:42.874 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.852843+0000", 2023-10-09T10:57:42.874 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012417600000000001 2023-10-09T10:57:42.874 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.874 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.874 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,5", 2023-10-09T10:57:42.874 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.852971+0000", 2023-10-09T10:57:42.874 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012844500000000001 2023-10-09T10:57:42.874 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.874 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.874 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:42.874 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.856810+0000", 2023-10-09T10:57:42.874 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038387130000000001 2023-10-09T10:57:42.875 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.875 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:42.875 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.875 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.875 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.875 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:37549 9.a 9:5d9be7fd:::benchmark_data_smithi125_363789_object37548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-10-09T10:57:42.875 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:57:42.853048+0000", 2023-10-09T10:57:42.875 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0040416080000000004, 2023-10-09T10:57:42.876 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043834479999999999, 2023-10-09T10:57:42.876 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:42.876 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:57:42.876 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:42.876 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T10:57:42.876 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T10:57:42.877 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 37549 2023-10-09T10:57:42.877 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.877 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:42.877 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.877 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:42.877 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.853048+0000", 2023-10-09T10:57:42.877 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.877 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.877 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.878 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:42.878 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.853048+0000", 2023-10-09T10:57:42.878 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.878 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.878 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.878 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:42.878 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.853047+0000", 2023-10-09T10:57:42.879 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:57:42.879 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.879 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.879 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:42.879 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.853069+0000", 2023-10-09T10:57:42.879 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2382999999999999e-05 2023-10-09T10:57:42.879 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.879 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.879 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:42.879 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.853078+0000", 2023-10-09T10:57:42.879 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0059999999999998e-06 2023-10-09T10:57:42.879 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.880 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.880 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:42.880 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.853084+0000", 2023-10-09T10:57:42.880 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2060000000000004e-06 2023-10-09T10:57:42.880 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.880 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.880 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:42.880 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.853120+0000", 2023-10-09T10:57:42.880 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5896000000000003e-05 2023-10-09T10:57:42.880 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.880 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.880 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:42.880 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.853258+0000", 2023-10-09T10:57:42.881 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013809799999999999 2023-10-09T10:57:42.881 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.881 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.881 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:57:42.881 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.853397+0000", 2023-10-09T10:57:42.881 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013850000000000001 2023-10-09T10:57:42.881 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.881 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.881 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:42.881 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.856055+0000", 2023-10-09T10:57:42.881 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026585570000000002 2023-10-09T10:57:42.881 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.882 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:42.882 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.882 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.882 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.882 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45683.0:37548 9.3 e338/332 9:c98476c7:::benchmark_data_smithi125_363789_object37547:head v 338'2268, mlcod=338'2266)", 2023-10-09T10:57:42.882 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:57:42.852564+0000", 2023-10-09T10:57:42.882 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0045252019999999999, 2023-10-09T10:57:42.882 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049382360000000004, 2023-10-09T10:57:42.882 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:42.882 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:57:42.882 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:42.882 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.883 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:42.883 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.852564+0000", 2023-10-09T10:57:42.883 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.883 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.883 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.883 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:42.883 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.852564+0000", 2023-10-09T10:57:42.883 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.883 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.883 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.883 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:42.883 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.852564+0000", 2023-10-09T10:57:42.884 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:57:42.884 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.884 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.884 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:42.884 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.854439+0000", 2023-10-09T10:57:42.884 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018758 2023-10-09T10:57:42.884 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.884 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.884 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:42.884 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.854448+0000", 2023-10-09T10:57:42.884 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7299999999999994e-06 2023-10-09T10:57:42.884 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.885 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.885 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:42.885 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.854455+0000", 2023-10-09T10:57:42.885 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7630000000000003e-06 2023-10-09T10:57:42.885 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.885 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.885 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:42.885 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.854512+0000", 2023-10-09T10:57:42.885 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7333999999999997e-05 2023-10-09T10:57:42.885 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.885 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.885 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:42.885 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.854536+0000", 2023-10-09T10:57:42.886 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3754e-05 2023-10-09T10:57:42.886 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.886 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:42.886 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.886 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.886 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.886 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45683.0:37552 9.6 e338/330 9:659fdd6f:::benchmark_data_smithi125_363789_object37551:head v 338'2308, mlcod=338'2305)", 2023-10-09T10:57:42.886 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:57:42.855770+0000", 2023-10-09T10:57:42.886 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0013190400000000001, 2023-10-09T10:57:42.886 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001767539, 2023-10-09T10:57:42.886 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:42.887 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:57:42.887 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:42.887 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.887 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:42.887 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.855770+0000", 2023-10-09T10:57:42.887 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.887 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.887 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.887 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:42.887 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.855770+0000", 2023-10-09T10:57:42.887 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.888 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.888 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.888 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:42.888 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.855770+0000", 2023-10-09T10:57:42.888 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:57:42.888 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.888 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.888 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:42.888 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.855794+0000", 2023-10-09T10:57:42.888 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4522999999999999e-05 2023-10-09T10:57:42.888 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.889 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.889 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:42.889 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.855807+0000", 2023-10-09T10:57:42.889 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2673e-05 2023-10-09T10:57:42.889 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.889 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.889 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:42.889 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.855813+0000", 2023-10-09T10:57:42.889 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4330000000000001e-06 2023-10-09T10:57:42.889 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.889 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.889 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:42.889 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.855855+0000", 2023-10-09T10:57:42.890 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1542000000000003e-05 2023-10-09T10:57:42.890 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.890 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.890 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:42.890 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.855871+0000", 2023-10-09T10:57:42.890 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6671e-05 2023-10-09T10:57:42.890 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.890 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:42.890 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.890 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.890 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.890 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45683.0:37553 9.d e338/332 9:b9716b3a:::benchmark_data_smithi125_363789_object37552:head v 338'2399, mlcod=338'2397)", 2023-10-09T10:57:42.891 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:57:42.856176+0000", 2023-10-09T10:57:42.891 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.00091295999999999999, 2023-10-09T10:57:42.891 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001398287, 2023-10-09T10:57:42.891 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:42.891 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:57:42.891 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:42.891 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.891 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:42.891 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.856176+0000", 2023-10-09T10:57:42.891 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.891 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.891 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.892 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:42.892 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.856176+0000", 2023-10-09T10:57:42.892 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.892 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.892 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.892 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:42.892 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.856176+0000", 2023-10-09T10:57:42.892 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:57:42.892 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.892 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.892 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:42.892 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.856228+0000", 2023-10-09T10:57:42.892 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2652999999999999e-05 2023-10-09T10:57:42.892 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.893 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.893 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:42.893 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.856237+0000", 2023-10-09T10:57:42.893 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4270000000000008e-06 2023-10-09T10:57:42.893 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.893 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.893 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:42.893 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.856243+0000", 2023-10-09T10:57:42.893 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.567e-06 2023-10-09T10:57:42.893 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.893 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.894 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:42.894 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.856332+0000", 2023-10-09T10:57:42.894 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8504999999999994e-05 2023-10-09T10:57:42.894 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.894 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.894 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:42.894 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.856356+0000", 2023-10-09T10:57:42.894 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4094999999999999e-05 2023-10-09T10:57:42.894 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.894 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:42.894 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.894 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.894 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.895 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:37550 9.f 9.652ba62f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-10-09T10:57:42.895 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:57:42.857417+0000", 2023-10-09T10:57:42.895 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4294967295.9996719, 2023-10-09T10:57:42.895 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019102400000000001, 2023-10-09T10:57:42.895 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:42.895 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:57:42.895 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:42.895 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T10:57:42.895 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T10:57:42.895 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 37550 2023-10-09T10:57:42.895 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.895 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:42.896 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.896 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:42.896 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.857417+0000", 2023-10-09T10:57:42.896 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.896 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.896 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.896 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:42.896 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.857417+0000", 2023-10-09T10:57:42.896 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:42.896 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.896 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.896 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:42.896 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.857416+0000", 2023-10-09T10:57:42.897 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:42.897 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.897 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.897 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:42.897 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.857444+0000", 2023-10-09T10:57:42.897 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7725999999999998e-05 2023-10-09T10:57:42.897 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.897 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.897 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:42.897 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.857454+0000", 2023-10-09T10:57:42.897 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0626e-05 2023-10-09T10:57:42.898 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:42.898 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:42.898 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:42.898 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:42.857462+0000", 2023-10-09T10:57:42.898 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5979999999999999e-06 2023-10-09T10:57:42.898 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.898 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:42.898 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.898 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:42.898 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:57:42.898 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 8 2023-10-09T10:57:42.898 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:57:42.900 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:43.188 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:43.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:43.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:42 smithi155 ceph-mon[184854]: pgmap v832: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2023-10-09T10:57:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:42 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2709374788' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:57:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:42 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:57:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:42 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/840229500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:43.299 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:43.342 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:43.371 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:43.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:43.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:43.477 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:43.521 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:43.554 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:43.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:43.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:43.607 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:57:43.622 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 19 16 38803 38787 127.516 97.9375 0.00908958 0.00783618 2023-10-09T10:57:43.659 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:43.721 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:43.752 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:43.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:43.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:43.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:43 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:43.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:43 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:57:43.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:43 smithi125 ceph-mon[195507]: osdmap e338: 8 total, 8 up, 8 in 2023-10-09T10:57:43.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:43 smithi125 ceph-mon[195507]: failed to encode map e338 with expected crc 2023-10-09T10:57:43.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:43 smithi125 ceph-mon[195507]: failed to encode map e338 with expected crc 2023-10-09T10:57:43.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:43 smithi125 ceph-mon[195507]: failed to encode map e338 with expected crc 2023-10-09T10:57:43.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:43 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:43.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:43 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:57:43.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:43 smithi125 ceph-mon[200197]: osdmap e338: 8 total, 8 up, 8 in 2023-10-09T10:57:43.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:43 smithi125 ceph-mon[200197]: failed to encode map e338 with expected crc 2023-10-09T10:57:43.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:43 smithi125 ceph-mon[200197]: failed to encode map e338 with expected crc 2023-10-09T10:57:43.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:43 smithi125 ceph-mon[200197]: failed to encode map e338 with expected crc 2023-10-09T10:57:43.859 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:43.912 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:43.947 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:43.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:43.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:44.051 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:44.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:44.101 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:44.187 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:44.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:44.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:44.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:43 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:44.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:43 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:57:44.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:43 smithi155 ceph-mon[184854]: osdmap e338: 8 total, 8 up, 8 in 2023-10-09T10:57:44.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:43 smithi155 ceph-mon[184854]: failed to encode map e338 with expected crc 2023-10-09T10:57:44.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:43 smithi155 ceph-mon[184854]: failed to encode map e338 with expected crc 2023-10-09T10:57:44.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:43 smithi155 ceph-mon[184854]: failed to encode map e338 with expected crc 2023-10-09T10:57:44.292 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:44.337 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:44.415 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:44.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:44.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:44.521 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:44.562 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:44.599 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:44.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:44.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:44.623 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:57:44.624055+0000 min lat: 0.00120203 max lat: 0.0957076 avg lat: 0.00794313 2023-10-09T10:57:44.623 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:57:44.623 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 20 16 40298 40282 125.809 93.4375 0.00560755 0.00794313 2023-10-09T10:57:44.705 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:44.743 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:44.753 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 2, 1, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:57:44.753 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T10:57:44.774 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:44.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:44.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:44.879 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:44.918 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:44 smithi155 ceph-mon[184854]: pgmap v834: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-10-09T10:57:44.918 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:44 smithi155 ceph-mon[184854]: 1.3 scrub starts 2023-10-09T10:57:44.919 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:44 smithi155 ceph-mon[184854]: 1.3 scrub ok 2023-10-09T10:57:44.921 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:44.942 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:44.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:44.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:45.038 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:44 smithi125 ceph-mon[195507]: pgmap v834: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-10-09T10:57:45.038 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:44 smithi125 ceph-mon[195507]: 1.3 scrub starts 2023-10-09T10:57:45.038 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:44 smithi125 ceph-mon[195507]: 1.3 scrub ok 2023-10-09T10:57:45.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:44 smithi125 ceph-mon[200197]: pgmap v834: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-10-09T10:57:45.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:44 smithi125 ceph-mon[200197]: 1.3 scrub starts 2023-10-09T10:57:45.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:44 smithi125 ceph-mon[200197]: 1.3 scrub ok 2023-10-09T10:57:45.042 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:57:45.047 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:45.076 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:45.100 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:45.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:45.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:45.203 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:45.265 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:45.302 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:45.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:45.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:45.352 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:57:45.352 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:57:45.353 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:57:45.353 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:57:45.353 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:57:45.353 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:45.409 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:45.423 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:57:45.456 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:45.485 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:45.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:45.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:45.590 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:45.623 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 21 16 41549 41533 123.542 78.1875 0.0063962 0.00808863 2023-10-09T10:57:45.644 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:45.776 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:45.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:45.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:45.821 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:57:45.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:45 smithi125 ceph-mon[200197]: 1.13 scrub starts 2023-10-09T10:57:45.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:45 smithi125 ceph-mon[200197]: 1.13 scrub ok 2023-10-09T10:57:45.873 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:57:45.873 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-10-09T10:57:45.873 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-09T10:57:45.873 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-10-09T10:57:45.873 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:45 smithi125 ceph-mon[195507]: 1.13 scrub starts 2023-10-09T10:57:45.873 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:45 smithi125 ceph-mon[195507]: 1.13 scrub ok 2023-10-09T10:57:45.873 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:57:45.873 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-10-09T10:57:45.873 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-09T10:57:45.874 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-10-09T10:57:45.882 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:45.914 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:45.939 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:45.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:45.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:46.043 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:46.115 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:46.194 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:46.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:46.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:46.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:45 smithi155 ceph-mon[184854]: 1.13 scrub starts 2023-10-09T10:57:46.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:45 smithi155 ceph-mon[184854]: 1.13 scrub ok 2023-10-09T10:57:46.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:57:46.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-10-09T10:57:46.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-09T10:57:46.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-10-09T10:57:46.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:46.323 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:46.324 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:46.349 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:46.368 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:46.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:46.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:46.387 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:46.472 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:46.552 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:46.623 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 22 16 42759 42743 121.365 75.625 0.00607035 0.0082322 2023-10-09T10:57:47.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[195507]: pgmap v835: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T10:57:47.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[195507]: failed to encode map e339 with expected crc 2023-10-09T10:57:47.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[195507]: failed to encode map e339 with expected crc 2023-10-09T10:57:47.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-10-09T10:57:47.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-10-09T10:57:47.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-10-09T10:57:47.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[195507]: osdmap e339: 8 total, 8 up, 8 in 2023-10-09T10:57:47.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[195507]: 1.b scrub starts 2023-10-09T10:57:47.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[195507]: 1.b scrub ok 2023-10-09T10:57:47.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[200197]: pgmap v835: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T10:57:47.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[200197]: failed to encode map e339 with expected crc 2023-10-09T10:57:47.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[200197]: failed to encode map e339 with expected crc 2023-10-09T10:57:47.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-10-09T10:57:47.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-10-09T10:57:47.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-10-09T10:57:47.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[200197]: osdmap e339: 8 total, 8 up, 8 in 2023-10-09T10:57:47.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[200197]: 1.b scrub starts 2023-10-09T10:57:47.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:46 smithi125 ceph-mon[200197]: 1.b scrub ok 2023-10-09T10:57:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:46 smithi155 ceph-mon[184854]: pgmap v835: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T10:57:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:46 smithi155 ceph-mon[184854]: failed to encode map e339 with expected crc 2023-10-09T10:57:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:46 smithi155 ceph-mon[184854]: failed to encode map e339 with expected crc 2023-10-09T10:57:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-10-09T10:57:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-10-09T10:57:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-10-09T10:57:47.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:46 smithi155 ceph-mon[184854]: osdmap e339: 8 total, 8 up, 8 in 2023-10-09T10:57:47.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:46 smithi155 ceph-mon[184854]: 1.b scrub starts 2023-10-09T10:57:47.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:46 smithi155 ceph-mon[184854]: 1.b scrub ok 2023-10-09T10:57:47.453 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:47.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:47.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:47.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:47.604 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:47.623 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 23 16 44104 44088 119.743 84.0625 0.0161278 0.00834634 2023-10-09T10:57:47.640 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:47.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:47.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:47.744 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:47.832 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:47.863 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:47.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:47.870 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T10:57:47.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:47.903 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[195507]: osdmap e340: 8 total, 8 up, 8 in 2023-10-09T10:57:47.903 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[195507]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.903 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[195507]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.903 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[195507]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.903 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[195507]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.903 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[195507]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.904 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[195507]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.904 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[195507]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.904 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[195507]: 1.1c scrub starts 2023-10-09T10:57:47.904 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[195507]: 1.1c scrub ok 2023-10-09T10:57:47.904 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/590711868' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:57:47.904 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:57:47.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[200197]: osdmap e340: 8 total, 8 up, 8 in 2023-10-09T10:57:47.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[200197]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[200197]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[200197]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[200197]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[200197]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[200197]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[200197]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[200197]: 1.1c scrub starts 2023-10-09T10:57:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[200197]: 1.1c scrub ok 2023-10-09T10:57:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/590711868' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:57:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:47 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:57:47.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:47 smithi155 ceph-mon[184854]: osdmap e340: 8 total, 8 up, 8 in 2023-10-09T10:57:47.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:47 smithi155 ceph-mon[184854]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:47 smithi155 ceph-mon[184854]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:47 smithi155 ceph-mon[184854]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:47 smithi155 ceph-mon[184854]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:47 smithi155 ceph-mon[184854]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:47 smithi155 ceph-mon[184854]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:47 smithi155 ceph-mon[184854]: failed to encode map e340 with expected crc 2023-10-09T10:57:47.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:47 smithi155 ceph-mon[184854]: 1.1c scrub starts 2023-10-09T10:57:47.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:47 smithi155 ceph-mon[184854]: 1.1c scrub ok 2023-10-09T10:57:47.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:47 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/590711868' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:57:47.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:47 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T10:57:47.972 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:48.025 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:48.364 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:57:48.411 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:57:48.452 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:57:48.485 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:57:48.526 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:57:48.556 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:57:48.592 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:57:48.623 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 24 15 45307 45292 117.89 75.25 0.0193168 0.00847483 2023-10-09T10:57:48.627 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:57:48.654 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:57:48.654 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:57:48.654 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:57:48.655 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:57:48.655 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:57:48.655 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:57:48.655 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:57:48.655 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:57:48.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T10:57:49.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[195507]: pgmap v838: 119 pgs: 2 peering, 117 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T10:57:49.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:57:49.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:57:49.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[195507]: osdmap e341: 8 total, 8 up, 8 in 2023-10-09T10:57:49.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[195507]: failed to encode map e341 with expected crc 2023-10-09T10:57:49.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[195507]: failed to encode map e341 with expected crc 2023-10-09T10:57:49.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[195507]: failed to encode map e341 with expected crc 2023-10-09T10:57:49.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3742151425' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:49.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[200197]: pgmap v838: 119 pgs: 2 peering, 117 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T10:57:49.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:57:49.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:57:49.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[200197]: osdmap e341: 8 total, 8 up, 8 in 2023-10-09T10:57:49.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[200197]: failed to encode map e341 with expected crc 2023-10-09T10:57:49.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[200197]: failed to encode map e341 with expected crc 2023-10-09T10:57:49.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[200197]: failed to encode map e341 with expected crc 2023-10-09T10:57:49.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:48 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3742151425' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:49.132 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:49.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:49.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:49.163 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:57:49.163 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:57:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:48 smithi155 ceph-mon[184854]: pgmap v838: 119 pgs: 2 peering, 117 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T10:57:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:48 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:57:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:48 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T10:57:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:48 smithi155 ceph-mon[184854]: osdmap e341: 8 total, 8 up, 8 in 2023-10-09T10:57:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:48 smithi155 ceph-mon[184854]: failed to encode map e341 with expected crc 2023-10-09T10:57:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:48 smithi155 ceph-mon[184854]: failed to encode map e341 with expected crc 2023-10-09T10:57:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:48 smithi155 ceph-mon[184854]: failed to encode map e341 with expected crc 2023-10-09T10:57:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:48 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3742151425' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:49.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:49.296 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:49.431 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:49.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:49.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:49.482 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:49.528 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:57:49.583 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:49.679 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 25 16 46530 46514 116.23 76.375 0.0141321 0.00859744 2023-10-09T10:57:49.724 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:49.750 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:49.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:49.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:49.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:49.886 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:57:49.927 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:50.004 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:50.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:50.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:50.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:49 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/314527718' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:50.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:49 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:50.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:49 smithi125 ceph-mon[195507]: pgmap v840: 119 pgs: 2 peering, 117 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T10:57:50.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:49 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/314527718' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:50.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:49 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:50.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:49 smithi125 ceph-mon[200197]: pgmap v840: 119 pgs: 2 peering, 117 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T10:57:50.108 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:50.168 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:50.203 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:50.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:50.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:50.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:49 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/314527718' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:50.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:49 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:57:50.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:49 smithi155 ceph-mon[184854]: pgmap v840: 119 pgs: 2 peering, 117 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T10:57:50.308 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:50.362 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:50.396 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:50.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:50.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:50.488 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:50.504 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:50.556 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:50.623 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 26 16 47850 47834 114.933 82.5 0.00915503 0.00869567 2023-10-09T10:57:50.738 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:50.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:50.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:50.845 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:50.891 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:50.926 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:50.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:50.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:50.947 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:57:50.947 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:57:50.947 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[195507]: osdmap e342: 8 total, 8 up, 8 in 2023-10-09T10:57:50.948 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[195507]: failed to encode map e342 with expected crc 2023-10-09T10:57:50.948 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[195507]: failed to encode map e342 with expected crc 2023-10-09T10:57:50.948 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[195507]: failed to encode map e342 with expected crc 2023-10-09T10:57:50.948 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[195507]: failed to encode map e342 with expected crc 2023-10-09T10:57:50.948 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[195507]: failed to encode map e342 with expected crc 2023-10-09T10:57:50.948 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:57:50.948 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:57:50.948 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[200197]: osdmap e342: 8 total, 8 up, 8 in 2023-10-09T10:57:50.948 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[200197]: failed to encode map e342 with expected crc 2023-10-09T10:57:50.949 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[200197]: failed to encode map e342 with expected crc 2023-10-09T10:57:50.949 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[200197]: failed to encode map e342 with expected crc 2023-10-09T10:57:50.949 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[200197]: failed to encode map e342 with expected crc 2023-10-09T10:57:50.949 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:50 smithi125 ceph-mon[200197]: failed to encode map e342 with expected crc 2023-10-09T10:57:51.032 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:51.102 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:51.217 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:57:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:57:51] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2023-10-09T10:57:51.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:50 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:57:51.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:50 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:57:51.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:50 smithi155 ceph-mon[184854]: osdmap e342: 8 total, 8 up, 8 in 2023-10-09T10:57:51.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:50 smithi155 ceph-mon[184854]: failed to encode map e342 with expected crc 2023-10-09T10:57:51.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:50 smithi155 ceph-mon[184854]: failed to encode map e342 with expected crc 2023-10-09T10:57:51.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:50 smithi155 ceph-mon[184854]: failed to encode map e342 with expected crc 2023-10-09T10:57:51.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:50 smithi155 ceph-mon[184854]: failed to encode map e342 with expected crc 2023-10-09T10:57:51.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:50 smithi155 ceph-mon[184854]: failed to encode map e342 with expected crc 2023-10-09T10:57:51.246 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:51.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:51.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:51.277 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:57:51.353 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:51.388 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:51.415 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:51.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:51.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:51.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:51.587 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:51.624 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 27 16 49058 49042 113.469 75.5 0.00634561 0.00880563 2023-10-09T10:57:51.721 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:51.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:51.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:51.802 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:51.826 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:51.835 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:57:51.836 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:57:51.873 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:51.899 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T10:57:51.909 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:51.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:51.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:52.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:52.057 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:52.089 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:52.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:52.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:52.169 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3586776018' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:57:52.169 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:51 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:57:52.169 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:51 smithi125 ceph-mon[195507]: pgmap v842: 119 pgs: 119 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T10:57:52.169 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3586776018' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:57:52.169 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:51 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:57:52.169 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:51 smithi125 ceph-mon[200197]: pgmap v842: 119 pgs: 119 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T10:57:52.194 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3586776018' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:57:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:51 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T10:57:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:51 smithi155 ceph-mon[184854]: pgmap v842: 119 pgs: 119 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T10:57:52.232 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:52.265 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:52.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:52.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:52.370 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:52.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T10:57:52.434 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:52.455 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:57:52.526 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:52.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:52.565 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T10:57:52.625 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 28 16 50305 50289 112.198 77.9375 0.0143378 0.008906 2023-10-09T10:57:52.630 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:52.709 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:52.902 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:52.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:52.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:53.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:53.037 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:53.045 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:53.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:52 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:57:53.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:52 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T10:57:53.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:52 smithi125 ceph-mon[195507]: failed to encode map e343 with expected crc 2023-10-09T10:57:53.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:52 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:57:53.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:52 smithi125 ceph-mon[195507]: osdmap e343: 8 total, 8 up, 8 in 2023-10-09T10:57:53.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:52 smithi125 ceph-mon[195507]: failed to encode map e343 with expected crc 2023-10-09T10:57:53.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:52 smithi125 ceph-mon[195507]: failed to encode map e343 with expected crc 2023-10-09T10:57:53.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:52 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:57:53.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:52 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T10:57:53.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:52 smithi125 ceph-mon[200197]: failed to encode map e343 with expected crc 2023-10-09T10:57:53.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:52 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:57:53.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:52 smithi125 ceph-mon[200197]: osdmap e343: 8 total, 8 up, 8 in 2023-10-09T10:57:53.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:52 smithi125 ceph-mon[200197]: failed to encode map e343 with expected crc 2023-10-09T10:57:53.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:52 smithi125 ceph-mon[200197]: failed to encode map e343 with expected crc 2023-10-09T10:57:53.072 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:53.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:53.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:53.152 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:53.176 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:53.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:52 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:57:53.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:52 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T10:57:53.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:52 smithi155 ceph-mon[184854]: failed to encode map e343 with expected crc 2023-10-09T10:57:53.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:52 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T10:57:53.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:52 smithi155 ceph-mon[184854]: osdmap e343: 8 total, 8 up, 8 in 2023-10-09T10:57:53.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:52 smithi155 ceph-mon[184854]: failed to encode map e343 with expected crc 2023-10-09T10:57:53.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:52 smithi155 ceph-mon[184854]: failed to encode map e343 with expected crc 2023-10-09T10:57:53.243 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:53.315 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:53.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:53.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:53.419 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:53.487 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:57:53.491 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:53.538 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:57:53.576 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:57:53.624 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:57:53.625 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 29 16 51358 51342 110.598 65.8125 0.0135874 0.00903542 2023-10-09T10:57:53.655 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:57:53.691 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:53.692 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:57:53.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:53.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:53.732 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:57:53.772 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:57:53.788 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:57:53.788 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:57:53.788 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:57:53.788 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:57:53.788 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:57:53.788 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:57:53.788 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:57:53.789 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:57:53.799 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:53.851 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:53.923 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:53.941 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:53.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:53.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:54.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:54 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3123423747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:54.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:54 smithi125 ceph-mon[195507]: pgmap v844: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T10:57:54.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:54 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3123423747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:54.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:54 smithi125 ceph-mon[200197]: pgmap v844: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T10:57:54.044 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:54.078 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:54.111 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:54.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:54.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:54.218 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:54.265 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:54.295 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:54.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:54.300 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:54 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3123423747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:54.300 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:54 smithi155 ceph-mon[184854]: pgmap v844: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T10:57:54.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:54.402 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:54.474 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:54.626 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 30 16 52603 52587 109.506 77.8125 0.0136372 0.00912728 2023-10-09T10:57:54.965 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:54.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:54.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:55.075 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:55.146 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:55.626 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 31 16 53929 53913 108.647 82.875 0.0187477 0.00919828 2023-10-09T10:57:55.787 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:55.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:55.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:55.893 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:55.928 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:55.953 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:55.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:55.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:56.058 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:57:56.070 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:57:56.070 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:57:56.070 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:57:56.070 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:57:56.070 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:57:56.138 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:56.223 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:57:56.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:56.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:57:56.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:56.395 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:56.425 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:56.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:56.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:56.435 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:57:56.463 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T10:57:56.529 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:57:56.570 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:56.598 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:57:56.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:56.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:57:56.615 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:57:56.633 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 32 16 55011 54995 107.343 67.625 0.00555716 0.00930796 2023-10-09T10:57:56.637 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T10:57:56.663 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:56 smithi125 ceph-mon[195507]: pgmap v845: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T10:57:56.663 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4120991319' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:57:56.663 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:56 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:57:56.663 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3122316792' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:57:56.663 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:56 smithi125 ceph-mon[200197]: pgmap v845: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T10:57:56.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4120991319' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:57:56.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:56 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:57:56.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3122316792' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:57:56.702 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:56 smithi155 ceph-mon[184854]: pgmap v845: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T10:57:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4120991319' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:57:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:56 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:57:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3122316792' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T10:57:56.773 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:57.160 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:57.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:57.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:57.211 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T10:57:57.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:57.319 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:57.366 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:57.387 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:57.409 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:57.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:57.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:57.518 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:57.559 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:57.588 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:57.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:57.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:57.632 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 33 16 56155 56139 106.257 71.5 0.0113066 0.00940561 2023-10-09T10:57:57.694 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:57.750 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:57.778 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:57.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:57.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:57.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:57.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[195507]: failed to encode map e344 with expected crc 2023-10-09T10:57:57.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:57:57.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3122316792' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:57:57.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[195507]: osdmap e344: 8 total, 8 up, 8 in 2023-10-09T10:57:57.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[195507]: failed to encode map e344 with expected crc 2023-10-09T10:57:57.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[195507]: failed to encode map e344 with expected crc 2023-10-09T10:57:57.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[195507]: failed to encode map e344 with expected crc 2023-10-09T10:57:57.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:57.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[200197]: failed to encode map e344 with expected crc 2023-10-09T10:57:57.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:57:57.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3122316792' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:57:57.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[200197]: osdmap e344: 8 total, 8 up, 8 in 2023-10-09T10:57:57.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[200197]: failed to encode map e344 with expected crc 2023-10-09T10:57:57.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[200197]: failed to encode map e344 with expected crc 2023-10-09T10:57:57.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:57 smithi125 ceph-mon[200197]: failed to encode map e344 with expected crc 2023-10-09T10:57:57.882 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:57.945 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:57 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:57:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:57 smithi155 ceph-mon[184854]: failed to encode map e344 with expected crc 2023-10-09T10:57:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:57 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:57:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:57 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3122316792' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T10:57:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:57 smithi155 ceph-mon[184854]: osdmap e344: 8 total, 8 up, 8 in 2023-10-09T10:57:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:57 smithi155 ceph-mon[184854]: failed to encode map e344 with expected crc 2023-10-09T10:57:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:57 smithi155 ceph-mon[184854]: failed to encode map e344 with expected crc 2023-10-09T10:57:57.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:57 smithi155 ceph-mon[184854]: failed to encode map e344 with expected crc 2023-10-09T10:57:58.226 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:58.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:58.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:58.333 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:57:58.404 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:58.503 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:57:58.632 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 34 16 57369 57353 105.364 75.875 0.0214166 0.00948505 2023-10-09T10:57:58.635 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:57:58.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:58.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:57:58.743 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:58.774 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:58 smithi125 ceph-mon[195507]: pgmap v847: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-09T10:57:58.774 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:58 smithi125 ceph-mon[200197]: pgmap v847: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-09T10:57:58.799 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:58.826 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:58.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:58.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:58.930 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:57:58.957 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:57:58.957 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:57:58.957 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:57:58.957 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:57:58.957 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.957 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95743 8.3 8:c55106e5:::benchmark_data_smithi125_329074_object95742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:57:58.957 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:45.760025+0000", 2023-10-09T10:57:58.957 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 193.19509203199999, 2023-10-09T10:57:58.957 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.35609172, 2023-10-09T10:57:58.958 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:58.958 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:58.958 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:58.958 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:58.958 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:58.958 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95743 2023-10-09T10:57:58.958 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.958 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:58.958 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.958 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:58.958 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.760025+0000", 2023-10-09T10:57:58.958 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:58.959 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.959 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.959 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:58.959 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.760025+0000", 2023-10-09T10:57:58.959 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:58.959 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.959 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.959 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:58.959 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.760023+0000", 2023-10-09T10:57:58.959 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:58.959 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.959 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.960 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:58.960 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.760052+0000", 2023-10-09T10:57:58.960 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8801000000000001e-05 2023-10-09T10:57:58.960 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.960 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.960 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:58.960 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.760066+0000", 2023-10-09T10:57:58.960 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3723999999999999e-05 2023-10-09T10:57:58.960 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.960 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.960 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:58.960 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.760075+0000", 2023-10-09T10:57:58.961 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.414e-06 2023-10-09T10:57:58.961 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.961 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.961 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:58.961 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.043017+0000", 2023-10-09T10:57:58.961 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.28294185300000002 2023-10-09T10:57:58.961 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.961 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.961 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:58.961 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.046876+0000", 2023-10-09T10:57:58.961 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038587830000000002 2023-10-09T10:57:58.961 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.962 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.962 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:57:58.962 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.052755+0000", 2023-10-09T10:57:58.962 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0058790969999999998 2023-10-09T10:57:58.962 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.962 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.962 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:58.962 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.060724+0000", 2023-10-09T10:57:58.962 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0079689440000000004 2023-10-09T10:57:58.962 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.962 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.962 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:58.963 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.061803+0000", 2023-10-09T10:57:58.963 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010790210000000001 2023-10-09T10:57:58.963 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.963 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.963 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:58.963 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.108203+0000", 2023-10-09T10:57:58.963 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.046400400000000001 2023-10-09T10:57:58.963 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.963 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.963 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:58.963 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.109669+0000", 2023-10-09T10:57:58.963 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014653159999999999 2023-10-09T10:57:58.964 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.964 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.964 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:58.964 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.116117+0000", 2023-10-09T10:57:58.964 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0064479610000000003 2023-10-09T10:57:58.964 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:58.964 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:58.964 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:58.964 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.964 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.964 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95750 8.13 8:cab0bd25:::benchmark_data_smithi125_329074_object95749:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:57:58.964 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:45.796219+0000", 2023-10-09T10:57:58.965 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 193.158897989, 2023-10-09T10:57:58.965 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.34625348099999997, 2023-10-09T10:57:58.965 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:58.965 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:58.965 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:58.965 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:58.965 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:58.965 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95750 2023-10-09T10:57:58.965 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.965 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:58.965 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.965 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:58.966 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.796219+0000", 2023-10-09T10:57:58.966 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:58.966 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.966 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.966 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:58.966 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.796219+0000", 2023-10-09T10:57:58.966 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:58.966 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.966 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.966 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:58.966 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.796217+0000", 2023-10-09T10:57:58.966 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:57:58.967 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.967 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.967 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:58.967 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.796248+0000", 2023-10-09T10:57:58.967 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0877999999999998e-05 2023-10-09T10:57:58.967 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.967 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.967 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:58.967 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.796263+0000", 2023-10-09T10:57:58.967 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5088e-05 2023-10-09T10:57:58.967 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.967 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.967 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:58.968 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.796271+0000", 2023-10-09T10:57:58.968 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2390000000000002e-06 2023-10-09T10:57:58.968 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.968 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.968 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:58.968 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.070667+0000", 2023-10-09T10:57:58.968 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.27439532799999999 2023-10-09T10:57:58.968 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.968 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.968 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:58.968 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.074852+0000", 2023-10-09T10:57:58.968 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041858870000000001 2023-10-09T10:57:58.969 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.969 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.969 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:57:58.969 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.081478+0000", 2023-10-09T10:57:58.969 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066251260000000003 2023-10-09T10:57:58.969 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.969 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.969 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:58.969 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.100468+0000", 2023-10-09T10:57:58.969 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018990145 2023-10-09T10:57:58.969 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.969 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.970 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:58.970 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.101474+0000", 2023-10-09T10:57:58.970 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001006018 2023-10-09T10:57:58.970 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.970 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.970 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:58.970 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.132307+0000", 2023-10-09T10:57:58.970 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.030833182000000001 2023-10-09T10:57:58.970 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.970 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.970 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:58.970 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.134406+0000", 2023-10-09T10:57:58.970 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020994569999999999 2023-10-09T10:57:58.971 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.971 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.971 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:58.971 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.142472+0000", 2023-10-09T10:57:58.971 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0080661329999999996 2023-10-09T10:57:58.971 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:58.971 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:58.971 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:58.971 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.971 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:58 smithi155 ceph-mon[184854]: pgmap v847: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-09T10:57:58.973 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPush(8.13 293/289 [PushOp(8:cbe2b721:::benchmark_data_smithi125_329074_object29:head, version: 270'1, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:cbe2b721:::benchmark_data_smithi125_329074_object29:head@270'1, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:57:58.973 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:45.877250+0000", 2023-10-09T10:57:58.973 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 193.07786700899999, 2023-10-09T10:57:58.973 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.363915395, 2023-10-09T10:57:58.973 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:58.973 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:57:58.974 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:58.974 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.974 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:58.974 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877250+0000", 2023-10-09T10:57:58.974 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:58.974 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.974 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.974 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:58.974 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877250+0000", 2023-10-09T10:57:58.974 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:58.974 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.975 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:58.975 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877249+0000", 2023-10-09T10:57:58.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:58.975 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.975 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:58.975 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877303+0000", 2023-10-09T10:57:58.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3928999999999998e-05 2023-10-09T10:57:58.975 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.975 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:58.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877323+0000", 2023-10-09T10:57:58.976 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0276999999999999e-05 2023-10-09T10:57:58.976 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.976 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:58.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877339+0000", 2023-10-09T10:57:58.976 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6362999999999999e-05 2023-10-09T10:57:58.976 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.976 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:58.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.190367+0000", 2023-10-09T10:57:58.976 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3130274530000001 2023-10-09T10:57:58.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.977 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:58.977 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.190793+0000", 2023-10-09T10:57:58.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042582999999999998 2023-10-09T10:57:58.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.977 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:58.977 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.241165+0000", 2023-10-09T10:57:58.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.050372739 2023-10-09T10:57:58.977 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:58.977 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:58.977 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:58.978 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.978 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.978 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95836 8.3 8:c6d59e02:::benchmark_data_smithi125_329074_object95835:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:57:58.978 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:45.974376+0000", 2023-10-09T10:57:58.978 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 192.98074125100001, 2023-10-09T10:57:58.978 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.29878043500000001, 2023-10-09T10:57:58.978 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:58.978 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:58.978 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:58.978 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:58.978 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:58.979 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95836 2023-10-09T10:57:58.979 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.979 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:58.979 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.979 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:58.979 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.974376+0000", 2023-10-09T10:57:58.979 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:58.979 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.979 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.979 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:58.979 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.974376+0000", 2023-10-09T10:57:58.979 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:58.979 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.980 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.980 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:58.980 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.974374+0000", 2023-10-09T10:57:58.980 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:57:58.980 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.980 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.980 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:58.980 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.974399+0000", 2023-10-09T10:57:58.980 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5358e-05 2023-10-09T10:57:58.980 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.980 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.980 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:58.981 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.974410+0000", 2023-10-09T10:57:58.981 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0625000000000001e-05 2023-10-09T10:57:58.981 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.981 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.981 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:58.981 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.974416+0000", 2023-10-09T10:57:58.981 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5660000000000003e-06 2023-10-09T10:57:58.981 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.981 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.981 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:58.981 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.235361+0000", 2023-10-09T10:57:58.981 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.26094443099999998 2023-10-09T10:57:58.981 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.982 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.982 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:58.982 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.240275+0000", 2023-10-09T10:57:58.982 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049143030000000001 2023-10-09T10:57:58.982 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.982 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.982 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:57:58.982 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.247125+0000", 2023-10-09T10:57:58.982 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.006849997 2023-10-09T10:57:58.982 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.982 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.982 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:58.983 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.260446+0000", 2023-10-09T10:57:58.983 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013321319 2023-10-09T10:57:58.983 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.983 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.983 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:58.983 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.261732+0000", 2023-10-09T10:57:58.983 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012852300000000001 2023-10-09T10:57:58.983 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.983 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.983 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:58.983 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.262817+0000", 2023-10-09T10:57:58.983 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001085309 2023-10-09T10:57:58.983 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.984 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.984 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:58.984 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.264862+0000", 2023-10-09T10:57:58.984 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020456110000000001 2023-10-09T10:57:58.984 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.984 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.984 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:58.984 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.273156+0000", 2023-10-09T10:57:58.984 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0082937140000000006 2023-10-09T10:57:58.984 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:58.984 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:58.985 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:58.985 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.985 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.985 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95842 8.3 8:c3562596:::benchmark_data_smithi125_329074_object95841:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:57:58.985 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:45.992483+0000", 2023-10-09T10:57:58.985 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 192.962633931, 2023-10-09T10:57:58.985 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.34110989600000002, 2023-10-09T10:57:58.985 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:58.985 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:58.985 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:58.985 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:58.985 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:58.986 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95842 2023-10-09T10:57:58.986 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.986 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:58.986 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.986 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:58.986 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.992483+0000", 2023-10-09T10:57:58.986 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:58.986 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.986 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.986 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:58.986 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.992483+0000", 2023-10-09T10:57:58.986 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:58.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.987 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.987 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:58.987 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.992481+0000", 2023-10-09T10:57:58.987 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:58.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.987 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.987 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:58.987 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.992510+0000", 2023-10-09T10:57:58.987 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8849000000000001e-05 2023-10-09T10:57:58.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.988 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:58.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.992525+0000", 2023-10-09T10:57:58.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4632e-05 2023-10-09T10:57:58.988 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.988 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:58.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.992534+0000", 2023-10-09T10:57:58.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5889999999999992e-06 2023-10-09T10:57:58.988 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.988 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:58.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.273661+0000", 2023-10-09T10:57:58.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.281126449 2023-10-09T10:57:58.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.989 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:58.989 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.278225+0000", 2023-10-09T10:57:58.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045641459999999998 2023-10-09T10:57:58.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.989 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:57:58.989 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.288539+0000", 2023-10-09T10:57:58.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010314014999999999 2023-10-09T10:57:58.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.990 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.990 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:58.990 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.296771+0000", 2023-10-09T10:57:58.990 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0082322690000000004 2023-10-09T10:57:58.990 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.990 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.990 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:58.990 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.322322+0000", 2023-10-09T10:57:58.990 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.025550585000000001 2023-10-09T10:57:58.990 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.990 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.990 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:58.991 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.322748+0000", 2023-10-09T10:57:58.991 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042560800000000001 2023-10-09T10:57:58.991 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.991 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.991 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:58.991 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.325248+0000", 2023-10-09T10:57:58.991 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002500539 2023-10-09T10:57:58.991 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.991 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.991 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:58.991 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.333593+0000", 2023-10-09T10:57:58.991 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083448670000000006 2023-10-09T10:57:58.991 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:58.992 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:58.992 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:58.992 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.992 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.992 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95845 8.13 8:c92b0cb5:::benchmark_data_smithi125_329074_object95844:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:57:58.992 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.006226+0000", 2023-10-09T10:57:58.992 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 192.94889111000001, 2023-10-09T10:57:58.992 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.34395316799999998, 2023-10-09T10:57:58.992 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:58.992 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:58.992 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:58.992 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:58.993 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:58.993 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95845 2023-10-09T10:57:58.993 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.993 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:58.993 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.993 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:58.993 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.006226+0000", 2023-10-09T10:57:58.993 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:58.993 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.993 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.993 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:58.993 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.006226+0000", 2023-10-09T10:57:58.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:58.994 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.994 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.994 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:58.994 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.006224+0000", 2023-10-09T10:57:58.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:57:58.994 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.994 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.994 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:58.994 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.006246+0000", 2023-10-09T10:57:58.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1963999999999998e-05 2023-10-09T10:57:58.994 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.995 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:58.995 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.006260+0000", 2023-10-09T10:57:58.995 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3601000000000001e-05 2023-10-09T10:57:58.995 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.995 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:58.995 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.006270+0000", 2023-10-09T10:57:58.995 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0496000000000001e-05 2023-10-09T10:57:58.995 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.995 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:58.995 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.274272+0000", 2023-10-09T10:57:58.996 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.26800151500000002 2023-10-09T10:57:58.996 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.996 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.996 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:58.996 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.279959+0000", 2023-10-09T10:57:58.996 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.005687531 2023-10-09T10:57:58.996 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.996 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.996 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:57:58.996 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.297659+0000", 2023-10-09T10:57:58.996 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.017699719999999999 2023-10-09T10:57:58.997 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.997 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:58.997 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.320528+0000", 2023-10-09T10:57:58.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.022869356 2023-10-09T10:57:58.997 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.997 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:58.997 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.322039+0000", 2023-10-09T10:57:58.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00151096 2023-10-09T10:57:58.997 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.998 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:58.998 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.334688+0000", 2023-10-09T10:57:58.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012648609 2023-10-09T10:57:58.998 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.998 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.998 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:58.998 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.337196+0000", 2023-10-09T10:57:58.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025082479999999998 2023-10-09T10:57:58.998 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.998 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.998 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:58.998 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.350179+0000", 2023-10-09T10:57:58.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012982852 2023-10-09T10:57:58.999 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:58.999 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:58.999 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:58.999 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:58.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:58.999 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95854 8.3 8:c64ea022:::benchmark_data_smithi125_329074_object95853:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:57:58.999 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.037001+0000", 2023-10-09T10:57:58.999 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 192.918115685, 2023-10-09T10:57:58.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.36041259799999997, 2023-10-09T10:57:58.999 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:59.000 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:59.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:59.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:59.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:59.000 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95854 2023-10-09T10:57:59.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.000 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:59.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.000 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:59.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.037001+0000", 2023-10-09T10:57:59.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:59.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.037001+0000", 2023-10-09T10:57:59.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:59.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.036999+0000", 2023-10-09T10:57:59.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:57:59.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:59.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.037026+0000", 2023-10-09T10:57:59.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7047e-05 2023-10-09T10:57:59.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.002 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:59.002 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.037041+0000", 2023-10-09T10:57:59.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4082e-05 2023-10-09T10:57:59.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.002 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:59.002 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.037050+0000", 2023-10-09T10:57:59.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6879999999999994e-06 2023-10-09T10:57:59.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:59.003 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.297294+0000", 2023-10-09T10:57:59.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.26024395700000003 2023-10-09T10:57:59.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.003 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:59.003 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.302697+0000", 2023-10-09T10:57:59.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0054025640000000003 2023-10-09T10:57:59.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.003 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:57:59.003 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.312000+0000", 2023-10-09T10:57:59.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0093036569999999999 2023-10-09T10:57:59.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.342728+0000", 2023-10-09T10:57:59.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.030727616999999999 2023-10-09T10:57:59.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.344397+0000", 2023-10-09T10:57:59.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001668957 2023-10-09T10:57:59.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:59.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.388186+0000", 2023-10-09T10:57:59.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.043788541 2023-10-09T10:57:59.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:59.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.390218+0000", 2023-10-09T10:57:59.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020322959999999998 2023-10-09T10:57:59.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.006 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.006 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:59.006 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.397414+0000", 2023-10-09T10:57:59.006 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0071960519999999997 2023-10-09T10:57:59.006 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.006 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:59.006 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.006 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.006 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.006 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95858 8.3 8:c3410bce:::benchmark_data_smithi125_329074_object95857:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:57:59.006 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.044670+0000", 2023-10-09T10:57:59.006 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 192.91044746200001, 2023-10-09T10:57:59.007 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.36581522900000002, 2023-10-09T10:57:59.007 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:59.007 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:59.007 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:59.007 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:59.007 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:59.007 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95858 2023-10-09T10:57:59.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.007 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:59.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.007 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:59.007 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.044670+0000", 2023-10-09T10:57:59.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:59.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.044670+0000", 2023-10-09T10:57:59.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:59.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.044668+0000", 2023-10-09T10:57:59.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:57:59.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:59.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.044692+0000", 2023-10-09T10:57:59.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3935999999999999e-05 2023-10-09T10:57:59.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:59.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.044705+0000", 2023-10-09T10:57:59.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2911000000000001e-05 2023-10-09T10:57:59.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:59.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.044715+0000", 2023-10-09T10:57:59.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0413e-05 2023-10-09T10:57:59.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.010 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.010 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:59.010 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.350771+0000", 2023-10-09T10:57:59.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.30605545499999998 2023-10-09T10:57:59.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.010 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.010 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:59.010 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.355300+0000", 2023-10-09T10:57:59.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045296310000000001 2023-10-09T10:57:59.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.010 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:57:59.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.366313+0000", 2023-10-09T10:57:59.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011012921 2023-10-09T10:57:59.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.398739+0000", 2023-10-09T10:57:59.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.032425871000000002 2023-10-09T10:57:59.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.400018+0000", 2023-10-09T10:57:59.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001279213 2023-10-09T10:57:59.012 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.012 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.012 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:59.012 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.400451+0000", 2023-10-09T10:57:59.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00043300000000000001 2023-10-09T10:57:59.012 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.012 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.012 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:59.012 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.402357+0000", 2023-10-09T10:57:59.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001905885 2023-10-09T10:57:59.012 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.013 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:59.013 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.410485+0000", 2023-10-09T10:57:59.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0081277120000000005 2023-10-09T10:57:59.013 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.013 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:59.013 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.013 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95871 8.3 8:c48d05f2:::benchmark_data_smithi125_329074_object95870:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:57:59.013 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.078356+0000", 2023-10-09T10:57:59.013 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 192.87676090299999, 2023-10-09T10:57:59.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.35194249700000002, 2023-10-09T10:57:59.014 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:59.014 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:59.014 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:59.014 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:59.014 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:59.014 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95871 2023-10-09T10:57:59.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.014 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:59.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:59.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.078356+0000", 2023-10-09T10:57:59.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:59.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.078356+0000", 2023-10-09T10:57:59.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:59.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.078354+0000", 2023-10-09T10:57:59.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:57:59.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:59.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.078379+0000", 2023-10-09T10:57:59.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4553e-05 2023-10-09T10:57:59.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:59.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.078394+0000", 2023-10-09T10:57:59.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5007999999999999e-05 2023-10-09T10:57:59.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:59.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.078401+0000", 2023-10-09T10:57:59.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7610000000000007e-06 2023-10-09T10:57:59.017 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.017 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:59.017 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.371028+0000", 2023-10-09T10:57:59.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.29262697300000001 2023-10-09T10:57:59.017 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.017 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:59.017 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.374954+0000", 2023-10-09T10:57:59.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039256650000000001 2023-10-09T10:57:59.017 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:57:59.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.382362+0000", 2023-10-09T10:57:59.018 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074084119999999996 2023-10-09T10:57:59.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.018 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:59.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.410967+0000", 2023-10-09T10:57:59.018 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.028604319999999999 2023-10-09T10:57:59.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.018 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.412353+0000", 2023-10-09T10:57:59.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013863219999999999 2023-10-09T10:57:59.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.019 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.416616+0000", 2023-10-09T10:57:59.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042625199999999997 2023-10-09T10:57:59.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:59.019 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.418988+0000", 2023-10-09T10:57:59.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023724190000000002 2023-10-09T10:57:59.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.020 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:59.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.430299+0000", 2023-10-09T10:57:59.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011310534000000001 2023-10-09T10:57:59.020 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.020 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:59.020 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.020 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.020 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95875 8.3 8:c45f4548:::benchmark_data_smithi125_329074_object95874:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:57:59.020 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.104335+0000", 2023-10-09T10:57:59.020 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 192.85078247499999, 2023-10-09T10:57:59.021 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.39137140100000001, 2023-10-09T10:57:59.021 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:59.021 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:59.021 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:59.021 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:59.021 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:59.021 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95875 2023-10-09T10:57:59.021 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.021 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:59.021 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.021 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:59.021 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.104335+0000", 2023-10-09T10:57:59.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.022 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:59.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.104335+0000", 2023-10-09T10:57:59.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.022 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:59.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.104333+0000", 2023-10-09T10:57:59.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:59.022 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:59.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.104355+0000", 2023-10-09T10:57:59.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1704000000000001e-05 2023-10-09T10:57:59.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:59.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.104365+0000", 2023-10-09T10:57:59.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0842000000000001e-05 2023-10-09T10:57:59.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:59.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.104372+0000", 2023-10-09T10:57:59.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.139e-06 2023-10-09T10:57:59.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:59.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.430995+0000", 2023-10-09T10:57:59.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32662346599999997 2023-10-09T10:57:59.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:59.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.435804+0000", 2023-10-09T10:57:59.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004808545 2023-10-09T10:57:59.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:57:59.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.443158+0000", 2023-10-09T10:57:59.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0073543310000000004 2023-10-09T10:57:59.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.475849+0000", 2023-10-09T10:57:59.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.032691027999999997 2023-10-09T10:57:59.026 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.026 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:59.026 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.476399+0000", 2023-10-09T10:57:59.026 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00054972300000000001 2023-10-09T10:57:59.026 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.026 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.026 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.477978+0000", 2023-10-09T10:57:59.026 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015797599999999999 2023-10-09T10:57:59.026 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.027 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:59.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.484184+0000", 2023-10-09T10:57:59.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062053780000000001 2023-10-09T10:57:59.027 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.027 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.027 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:59.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.495706+0000", 2023-10-09T10:57:59.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011522045 2023-10-09T10:57:59.027 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.027 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:59.027 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.028 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95895 8.3 8:c64bfacc:::benchmark_data_smithi125_329074_object95894:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:57:59.028 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.161613+0000", 2023-10-09T10:57:59.028 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 192.79350445399999, 2023-10-09T10:57:59.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.35546433399999999, 2023-10-09T10:57:59.028 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:59.028 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:59.028 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:59.028 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:59.028 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:59.028 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95895 2023-10-09T10:57:59.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.029 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:59.029 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.029 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:59.029 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.161613+0000", 2023-10-09T10:57:59.029 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.029 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.029 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.029 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:59.029 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.161613+0000", 2023-10-09T10:57:59.029 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.029 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:59.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.161610+0000", 2023-10-09T10:57:59.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T10:57:59.030 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:59.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.161639+0000", 2023-10-09T10:57:59.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8572999999999999e-05 2023-10-09T10:57:59.030 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:59.031 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.161655+0000", 2023-10-09T10:57:59.031 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6172999999999999e-05 2023-10-09T10:57:59.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.031 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.031 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:59.031 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.161665+0000", 2023-10-09T10:57:59.031 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0304e-05 2023-10-09T10:57:59.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.031 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.031 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:59.031 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.449268+0000", 2023-10-09T10:57:59.031 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.28760287600000001 2023-10-09T10:57:59.032 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.032 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.032 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:59.032 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.453531+0000", 2023-10-09T10:57:59.032 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042631939999999997 2023-10-09T10:57:59.032 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.032 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.032 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:57:59.032 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.461412+0000", 2023-10-09T10:57:59.032 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0078809580000000004 2023-10-09T10:57:59.032 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.032 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.032 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:59.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.476830+0000", 2023-10-09T10:57:59.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015417854999999999 2023-10-09T10:57:59.033 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.497838+0000", 2023-10-09T10:57:59.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.021007978 2023-10-09T10:57:59.033 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.505361+0000", 2023-10-09T10:57:59.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0075223690000000001 2023-10-09T10:57:59.034 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.034 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.034 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:59.034 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.507459+0000", 2023-10-09T10:57:59.034 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020982729999999999 2023-10-09T10:57:59.034 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.034 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.034 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:59.034 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.517077+0000", 2023-10-09T10:57:59.034 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0096180799999999993 2023-10-09T10:57:59.034 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.034 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:59.035 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.035 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.035 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.035 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95902 8.13 8:c9dc5fe7:::benchmark_data_smithi125_329074_object95901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:57:59.035 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.181255+0000", 2023-10-09T10:57:59.035 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 192.77386178099999, 2023-10-09T10:57:59.035 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.46745452100000001, 2023-10-09T10:57:59.035 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:59.035 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:59.035 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:59.035 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:59.035 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:59.035 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95902 2023-10-09T10:57:59.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.036 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:59.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.036 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:59.036 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181255+0000", 2023-10-09T10:57:59.036 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.036 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:59.036 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181255+0000", 2023-10-09T10:57:59.036 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.037 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.037 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:59.037 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181253+0000", 2023-10-09T10:57:59.037 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:57:59.037 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.037 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.037 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:59.037 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181286+0000", 2023-10-09T10:57:59.037 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2928000000000001e-05 2023-10-09T10:57:59.037 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.037 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.037 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:59.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181306+0000", 2023-10-09T10:57:59.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9514999999999999e-05 2023-10-09T10:57:59.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:59.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181317+0000", 2023-10-09T10:57:59.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1070000000000001e-05 2023-10-09T10:57:59.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:59.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.503475+0000", 2023-10-09T10:57:59.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32215859400000002 2023-10-09T10:57:59.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.039 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:59.039 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.513376+0000", 2023-10-09T10:57:59.039 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0099005069999999994 2023-10-09T10:57:59.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.039 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:57:59.039 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.528220+0000", 2023-10-09T10:57:59.039 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014844346 2023-10-09T10:57:59.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.040 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.543824+0000", 2023-10-09T10:57:59.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015603361 2023-10-09T10:57:59.040 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.040 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.040 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.547461+0000", 2023-10-09T10:57:59.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036375349999999999 2023-10-09T10:57:59.040 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.040 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.040 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:59.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.633195+0000", 2023-10-09T10:57:59.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.085734224999999997 2023-10-09T10:57:59.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:59.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.636727+0000", 2023-10-09T10:57:59.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035320870000000002 2023-10-09T10:57:59.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:59.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.648710+0000", 2023-10-09T10:57:59.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011982244 2023-10-09T10:57:59.041 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.042 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:59.042 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.042 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.042 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.042 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95936 8.3 8:c568270b:::benchmark_data_smithi125_329074_object95935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:57:59.042 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.239203+0000", 2023-10-09T10:57:59.042 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 192.71591414599999, 2023-10-09T10:57:59.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.38808216600000001, 2023-10-09T10:57:59.042 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:59.042 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:59.042 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:59.042 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:59.042 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:59.043 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95936 2023-10-09T10:57:59.043 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.043 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:59.043 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.043 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:59.043 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.239203+0000", 2023-10-09T10:57:59.043 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.043 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.043 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.043 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:59.043 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.239203+0000", 2023-10-09T10:57:59.043 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:59.044 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.239201+0000", 2023-10-09T10:57:59.044 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:59.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:59.044 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.239231+0000", 2023-10-09T10:57:59.044 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9762999999999999e-05 2023-10-09T10:57:59.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:59.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.239244+0000", 2023-10-09T10:57:59.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3473e-05 2023-10-09T10:57:59.045 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.045 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:59.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.239254+0000", 2023-10-09T10:57:59.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0829999999999993e-06 2023-10-09T10:57:59.045 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.045 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:59.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.517531+0000", 2023-10-09T10:57:59.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.27827761899999998 2023-10-09T10:57:59.046 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.046 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.046 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:59.046 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.522622+0000", 2023-10-09T10:57:59.046 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0050906229999999998 2023-10-09T10:57:59.046 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.046 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.046 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:57:59.046 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.530722+0000", 2023-10-09T10:57:59.046 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0081005079999999993 2023-10-09T10:57:59.046 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.046 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.047 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.047 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.544008+0000", 2023-10-09T10:57:59.047 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013285732999999999 2023-10-09T10:57:59.047 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.047 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.047 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.545127+0000", 2023-10-09T10:57:59.047 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011191319999999999 2023-10-09T10:57:59.047 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.047 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:59.047 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.615780+0000", 2023-10-09T10:57:59.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.070652551999999993 2023-10-09T10:57:59.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.048 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:59.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.618009+0000", 2023-10-09T10:57:59.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022288569999999999 2023-10-09T10:57:59.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.048 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:59.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.627285+0000", 2023-10-09T10:57:59.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0092764270000000003 2023-10-09T10:57:59.048 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.048 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:59.049 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.049 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95942 8.3 8:c1416b96:::benchmark_data_smithi125_329074_object95941:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:57:59.049 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.246107+0000", 2023-10-09T10:57:59.049 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 192.709009963, 2023-10-09T10:57:59.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.41670001299999998, 2023-10-09T10:57:59.049 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:59.049 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:59.049 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:59.049 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:59.049 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:59.050 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95942 2023-10-09T10:57:59.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.050 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:59.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:59.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.246107+0000", 2023-10-09T10:57:59.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:59.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.246107+0000", 2023-10-09T10:57:59.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:59.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.246106+0000", 2023-10-09T10:57:59.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:59.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:59.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.246143+0000", 2023-10-09T10:57:59.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7042000000000002e-05 2023-10-09T10:57:59.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:59.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.246158+0000", 2023-10-09T10:57:59.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5378000000000001e-05 2023-10-09T10:57:59.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:59.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.246168+0000", 2023-10-09T10:57:59.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5440000000000004e-06 2023-10-09T10:57:59.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.053 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:59.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.549808+0000", 2023-10-09T10:57:59.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.30364068399999999 2023-10-09T10:57:59.053 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.053 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:59.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.555779+0000", 2023-10-09T10:57:59.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0059703960000000002 2023-10-09T10:57:59.053 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.053 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:57:59.054 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.562896+0000", 2023-10-09T10:57:59.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0071178830000000002 2023-10-09T10:57:59.054 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.054 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.054 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.054 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.571242+0000", 2023-10-09T10:57:59.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083458920000000006 2023-10-09T10:57:59.054 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.054 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.054 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.054 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.572716+0000", 2023-10-09T10:57:59.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014732759999999999 2023-10-09T10:57:59.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:59.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.653417+0000", 2023-10-09T10:57:59.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.080700999999999995 2023-10-09T10:57:59.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:59.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.655103+0000", 2023-10-09T10:57:59.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001686173 2023-10-09T10:57:59.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:59.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.662807+0000", 2023-10-09T10:57:59.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0077042059999999999 2023-10-09T10:57:59.056 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.056 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:59.056 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.056 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95991 8.3 8:c7234827:::benchmark_data_smithi125_329074_object95990:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:57:59.056 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.358750+0000", 2023-10-09T10:57:59.056 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 192.596366545, 2023-10-09T10:57:59.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.31338819600000001, 2023-10-09T10:57:59.056 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:59.057 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:59.057 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:59.057 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:59.057 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:59.057 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95991 2023-10-09T10:57:59.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.057 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:59.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:59.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.358750+0000", 2023-10-09T10:57:59.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:59.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.358750+0000", 2023-10-09T10:57:59.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:59.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.358749+0000", 2023-10-09T10:57:59.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:59.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:59.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.358772+0000", 2023-10-09T10:57:59.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3393000000000001e-05 2023-10-09T10:57:59.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.059 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.059 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:59.059 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.358785+0000", 2023-10-09T10:57:59.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2336e-05 2023-10-09T10:57:59.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.059 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.059 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:59.059 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.358794+0000", 2023-10-09T10:57:59.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8219999999999996e-06 2023-10-09T10:57:59.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:59.060 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.573131+0000", 2023-10-09T10:57:59.060 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.214337478 2023-10-09T10:57:59.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:59.060 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.579816+0000", 2023-10-09T10:57:59.060 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066854109999999996 2023-10-09T10:57:59.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:57:59.060 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.588464+0000", 2023-10-09T10:57:59.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0086475609999999998 2023-10-09T10:57:59.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.628692+0000", 2023-10-09T10:57:59.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.040228208000000001 2023-10-09T10:57:59.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.629893+0000", 2023-10-09T10:57:59.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012005519999999999 2023-10-09T10:57:59.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:59.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.663115+0000", 2023-10-09T10:57:59.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.033222139999999997 2023-10-09T10:57:59.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:59.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.664681+0000", 2023-10-09T10:57:59.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015656750000000001 2023-10-09T10:57:59.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:59.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.672139+0000", 2023-10-09T10:57:59.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074580239999999997 2023-10-09T10:57:59.063 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.063 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:59.063 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.063 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:96005 8.3 8:c030217b:::benchmark_data_smithi125_329074_object96004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:57:59.063 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.386683+0000", 2023-10-09T10:57:59.063 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 192.568433734, 2023-10-09T10:57:59.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.29650773000000002, 2023-10-09T10:57:59.064 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:59.064 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:59.064 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:59.064 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:59.064 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:59.064 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 96005 2023-10-09T10:57:59.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.064 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:59.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.064 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:59.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.386683+0000", 2023-10-09T10:57:59.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:59.065 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.386683+0000", 2023-10-09T10:57:59.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:59.065 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.386682+0000", 2023-10-09T10:57:59.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:59.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:59.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.386704+0000", 2023-10-09T10:57:59.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2478000000000001e-05 2023-10-09T10:57:59.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:59.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.386713+0000", 2023-10-09T10:57:59.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0580000000000004e-06 2023-10-09T10:57:59.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:59.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.386719+0000", 2023-10-09T10:57:59.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.784e-06 2023-10-09T10:57:59.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.067 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:59.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.595142+0000", 2023-10-09T10:57:59.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20842272100000001 2023-10-09T10:57:59.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.067 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:59.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.602128+0000", 2023-10-09T10:57:59.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0069863640000000001 2023-10-09T10:57:59.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T10:57:59.068 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.609782+0000", 2023-10-09T10:57:59.068 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076536219999999997 2023-10-09T10:57:59.068 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.068 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.630874+0000", 2023-10-09T10:57:59.068 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.021091682 2023-10-09T10:57:59.068 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.632323+0000", 2023-10-09T10:57:59.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001449868 2023-10-09T10:57:59.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.069 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:59.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.672484+0000", 2023-10-09T10:57:59.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.040160985000000003 2023-10-09T10:57:59.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.069 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:59.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.674513+0000", 2023-10-09T10:57:59.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020288659999999998 2023-10-09T10:57:59.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.070 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:59.070 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.683191+0000", 2023-10-09T10:57:59.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0086776709999999996 2023-10-09T10:57:59.070 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.070 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:59.070 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.070 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.070 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:191044 8.d 8:b00586e5:::benchmark_data_smithi125_329074_object94390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-10-09T10:57:59.070 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:57:03.945754+0000", 2023-10-09T10:57:59.071 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 55.009362535999998, 2023-10-09T10:57:59.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18610838900000001, 2023-10-09T10:57:59.071 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:59.071 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:59.071 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:59.071 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:59.071 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:59.071 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 191044 2023-10-09T10:57:59.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.071 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:59.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.071 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:59.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:03.945754+0000", 2023-10-09T10:57:59.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:59.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:03.945754+0000", 2023-10-09T10:57:59.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:59.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:03.945753+0000", 2023-10-09T10:57:59.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:59.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:59.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:03.945760+0000", 2023-10-09T10:57:59.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7209999999999997e-06 2023-10-09T10:57:59.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:59.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:03.945771+0000", 2023-10-09T10:57:59.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1619e-05 2023-10-09T10:57:59.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:59.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:03.945779+0000", 2023-10-09T10:57:59.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2869999999999999e-06 2023-10-09T10:57:59.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:59.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.037751+0000", 2023-10-09T10:57:59.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.091972353000000007 2023-10-09T10:57:59.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:59.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.038663+0000", 2023-10-09T10:57:59.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00091186700000000002 2023-10-09T10:57:59.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6,7", 2023-10-09T10:57:59.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.039806+0000", 2023-10-09T10:57:59.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001142732 2023-10-09T10:57:59.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:59.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.052791+0000", 2023-10-09T10:57:59.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012985473000000001 2023-10-09T10:57:59.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.076 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.076 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.053169+0000", 2023-10-09T10:57:59.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00037760099999999998 2023-10-09T10:57:59.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.076 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.076 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.065915+0000", 2023-10-09T10:57:59.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012745851000000001 2023-10-09T10:57:59.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.076 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.130307+0000", 2023-10-09T10:57:59.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.064392400000000002 2023-10-09T10:57:59.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:59.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.130675+0000", 2023-10-09T10:57:59.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00036781900000000002 2023-10-09T10:57:59.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:59.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.131863+0000", 2023-10-09T10:57:59.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011879970000000001 2023-10-09T10:57:59.077 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.078 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:59.078 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.078 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:191061 8.d 8:b53d7ae1:::benchmark_data_smithi125_329074_object94407:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-10-09T10:57:59.078 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:57:03.982486+0000", 2023-10-09T10:57:59.078 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 54.972631053000001, 2023-10-09T10:57:59.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.163802435, 2023-10-09T10:57:59.078 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:59.078 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:59.078 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:59.078 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:57:59.079 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:57:59.079 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 191061 2023-10-09T10:57:59.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.079 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:59.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:59.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:03.982486+0000", 2023-10-09T10:57:59.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:59.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:03.982486+0000", 2023-10-09T10:57:59.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.080 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:59.080 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:03.982485+0000", 2023-10-09T10:57:59.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:57:59.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.080 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:59.080 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:03.982490+0000", 2023-10-09T10:57:59.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6300000000000003e-06 2023-10-09T10:57:59.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.081 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:59.081 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:03.982502+0000", 2023-10-09T10:57:59.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1232e-05 2023-10-09T10:57:59.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.081 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.081 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:59.081 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:03.982509+0000", 2023-10-09T10:57:59.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5039999999999996e-06 2023-10-09T10:57:59.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.081 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.081 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:59.081 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.044678+0000", 2023-10-09T10:57:59.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.062168662 2023-10-09T10:57:59.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:59.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.045633+0000", 2023-10-09T10:57:59.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000955027 2023-10-09T10:57:59.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6,7", 2023-10-09T10:57:59.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.046804+0000", 2023-10-09T10:57:59.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001170847 2023-10-09T10:57:59.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.083 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:59.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.052868+0000", 2023-10-09T10:57:59.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0060638819999999996 2023-10-09T10:57:59.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.083 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.066122+0000", 2023-10-09T10:57:59.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.01325485 2023-10-09T10:57:59.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.083 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.066332+0000", 2023-10-09T10:57:59.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000209982 2023-10-09T10:57:59.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.144707+0000", 2023-10-09T10:57:59.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.078374789 2023-10-09T10:57:59.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:59.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.145079+0000", 2023-10-09T10:57:59.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00037176 2023-10-09T10:57:59.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:59.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:04.146288+0000", 2023-10-09T10:57:59.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012093410000000001 2023-10-09T10:57:59.085 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.085 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:59.085 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.085 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:21170 9.6 9:63f0979b:::benchmark_data_smithi125_363789_object21169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T10:57:59.085 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:57:32.968083+0000", 2023-10-09T10:57:59.085 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 25.987033602, 2023-10-09T10:57:59.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.087785366000000004, 2023-10-09T10:57:59.086 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:59.086 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:59.086 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:59.086 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T10:57:59.086 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T10:57:59.086 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 21170 2023-10-09T10:57:59.086 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.086 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:59.086 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.086 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:59.086 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:32.968083+0000", 2023-10-09T10:57:59.086 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:59.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:32.968083+0000", 2023-10-09T10:57:59.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:59.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:32.968082+0000", 2023-10-09T10:57:59.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:57:59.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:59.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:32.968104+0000", 2023-10-09T10:57:59.088 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1469999999999999e-05 2023-10-09T10:57:59.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.088 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:59.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:32.968113+0000", 2023-10-09T10:57:59.088 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9250000000000007e-06 2023-10-09T10:57:59.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.088 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:59.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:32.968119+0000", 2023-10-09T10:57:59.088 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4489999999999998e-06 2023-10-09T10:57:59.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.089 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.089 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:59.089 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:32.968155+0000", 2023-10-09T10:57:59.089 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6223999999999997e-05 2023-10-09T10:57:59.089 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.089 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.089 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:59.089 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:32.968260+0000", 2023-10-09T10:57:59.089 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000104855 2023-10-09T10:57:59.089 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.089 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.089 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:57:59.089 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:32.968344+0000", 2023-10-09T10:57:59.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3842000000000001e-05 2023-10-09T10:57:59.090 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.090 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.090 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:59.090 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:32.969161+0000", 2023-10-09T10:57:59.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00081724499999999997 2023-10-09T10:57:59.090 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.090 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.090 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.090 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:32.969560+0000", 2023-10-09T10:57:59.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00039917499999999999 2023-10-09T10:57:59.090 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.090 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.091 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.091 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.055774+0000", 2023-10-09T10:57:59.091 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.086213669000000007 2023-10-09T10:57:59.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.091 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:59.091 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.055829+0000", 2023-10-09T10:57:59.091 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5009000000000001e-05 2023-10-09T10:57:59.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.091 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:59.091 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.055869+0000", 2023-10-09T10:57:59.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.981e-05 2023-10-09T10:57:59.092 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.092 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:59.092 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.092 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:21659 9.6 9:650ad26f:::benchmark_data_smithi125_363789_object21658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T10:57:59.092 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:57:33.263579+0000", 2023-10-09T10:57:59.092 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 25.691538402999999, 2023-10-09T10:57:59.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.085193910999999997, 2023-10-09T10:57:59.092 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:57:59.092 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:57:59.092 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:57:59.093 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T10:57:59.093 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T10:57:59.093 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 21659 2023-10-09T10:57:59.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.093 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:57:59.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:57:59.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.263579+0000", 2023-10-09T10:57:59.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:57:59.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.263579+0000", 2023-10-09T10:57:59.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:57:59.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:57:59.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.263577+0000", 2023-10-09T10:57:59.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:57:59.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:57:59.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.263608+0000", 2023-10-09T10:57:59.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1202999999999999e-05 2023-10-09T10:57:59.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:57:59.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.263623+0000", 2023-10-09T10:57:59.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5143000000000001e-05 2023-10-09T10:57:59.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:57:59.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.263631+0000", 2023-10-09T10:57:59.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2090000000000005e-06 2023-10-09T10:57:59.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:57:59.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.263669+0000", 2023-10-09T10:57:59.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8031000000000002e-05 2023-10-09T10:57:59.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:57:59.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.263762+0000", 2023-10-09T10:57:59.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2739999999999996e-05 2023-10-09T10:57:59.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T10:57:59.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.263863+0000", 2023-10-09T10:57:59.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000100861 2023-10-09T10:57:59.097 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.097 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.097 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:57:59.097 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.264788+0000", 2023-10-09T10:57:59.097 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00092536400000000003 2023-10-09T10:57:59.097 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.097 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.097 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.097 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.266184+0000", 2023-10-09T10:57:59.097 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013955020000000001 2023-10-09T10:57:59.097 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.097 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:57:59.098 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.348665+0000", 2023-10-09T10:57:59.098 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.082481537999999993 2023-10-09T10:57:59.098 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.098 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:57:59.098 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.348717+0000", 2023-10-09T10:57:59.098 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1765000000000003e-05 2023-10-09T10:57:59.098 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:57:59.098 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:57:59.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:57:59.098 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:57:33.348772+0000", 2023-10-09T10:57:59.098 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5312999999999997e-05 2023-10-09T10:57:59.099 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.099 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:59.099 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.099 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:57:59.099 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:57:59.099 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:57:59.099 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:59.144 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:57:59.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:59.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:57:59.184 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:57:59.193 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:57:59.223 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:57:59.252 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:57:59.259 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:57:59.286 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:59.297 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:57:59.312 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:57:59.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:59.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:57:59.330 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:57:59.376 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:57:59.407 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:57:59.418 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:57:59.457 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:57:59.461 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:59.471 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:57:59.471 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:57:59.471 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:57:59.471 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:57:59.472 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:57:59.472 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:57:59.472 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:57:59.472 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:57:59.474 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:57:59.501 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:57:59.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:59.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:57:59.610 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:57:59.634 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 35 16 58403 58387 104.199 64.625 0.0158106 0.00959182 2023-10-09T10:57:59.643 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:59.668 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:57:59.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:59.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:57:59.772 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:59.817 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:57:59.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:57:59 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1073749237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:59.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:57:59 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1073749237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:59.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:57:59.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:57:59.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:57:59.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:57:59.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:57:59 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1073749237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:57:59.999 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:00.027 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:00.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:00.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:00.073 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:00.133 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:00.166 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:00.177 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:00.199 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:00.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:00.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:00.304 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:00.348 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:00.378 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:00.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:00.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:00.484 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:00.554 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:00.633 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 36 16 59570 59554 103.331 72.9375 0.0243478 0.00967212 2023-10-09T10:58:00.651 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:00 smithi155 ceph-mon[184854]: pgmap v848: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T10:58:00.651 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:58:00.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:00 smithi125 ceph-mon[195507]: pgmap v848: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T10:58:00.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:58:00.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:58:00.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:00 smithi125 ceph-mon[200197]: pgmap v848: 119 pgs: 119 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T10:58:00.802 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:58:00.802 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:58:00.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:58:00.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 10:58:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=cleanup t=2023-10-09T10:58:00.810799866Z level=info msg="Completed cleanup jobs" duration=18.0734ms 2023-10-09T10:58:01.049 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:01.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:01.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:01.178 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:01.239 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:01.291 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:01.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:01.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:01.322 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:58:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:58:01] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-10-09T10:58:01.397 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:01.436 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:01.633 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 37 16 60584 60568 102.251 63.375 0.00712121 0.00977069 2023-10-09T10:58:01.796 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T10:58:01.947 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:01.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:01.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:02.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:02.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1422990594' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:58:02.037 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:01 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:58:02.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:02.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1422990594' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:58:02.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:01 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:58:02.051 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:02.095 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:58:02.110 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:02.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:02.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1422990594' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:58:02.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:01 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T10:58:02.260 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T10:58:02.344 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:02.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:02.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:02.454 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:02.524 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:02.608 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:02.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:02.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:02.635 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 38 16 61612 61596 101.247 64.25 0.0268744 0.00986967 2023-10-09T10:58:02.713 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:02.776 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:02 smithi125 ceph-mon[195507]: pgmap v849: 119 pgs: 119 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:58:02.776 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:02 smithi125 ceph-mon[195507]: failed to encode map e345 with expected crc 2023-10-09T10:58:02.776 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:02 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:58:02.776 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:02 smithi125 ceph-mon[195507]: osdmap e345: 8 total, 8 up, 8 in 2023-10-09T10:58:02.776 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:02 smithi125 ceph-mon[195507]: failed to encode map e345 with expected crc 2023-10-09T10:58:02.812 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:03.004 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:03.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:03.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:03.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:02 smithi125 ceph-mon[200197]: pgmap v849: 119 pgs: 119 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:58:03.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:02 smithi125 ceph-mon[200197]: failed to encode map e345 with expected crc 2023-10-09T10:58:03.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:02 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:58:03.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:02 smithi125 ceph-mon[200197]: osdmap e345: 8 total, 8 up, 8 in 2023-10-09T10:58:03.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:02 smithi125 ceph-mon[200197]: failed to encode map e345 with expected crc 2023-10-09T10:58:03.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:03.186 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:03.220 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:03.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:02 smithi155 ceph-mon[184854]: pgmap v849: 119 pgs: 119 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:58:03.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:02 smithi155 ceph-mon[184854]: failed to encode map e345 with expected crc 2023-10-09T10:58:03.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:02 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T10:58:03.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:02 smithi155 ceph-mon[184854]: osdmap e345: 8 total, 8 up, 8 in 2023-10-09T10:58:03.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:02 smithi155 ceph-mon[184854]: failed to encode map e345 with expected crc 2023-10-09T10:58:03.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:03.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:03.325 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:03.376 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:03.460 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:03.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:03.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:03.485 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:03.491 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:03.503 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:58:03.504 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:58:03.565 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:03.635 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 39 16 62533 62517 100.128 57.5625 0.00614062 0.00998189 2023-10-09T10:58:03.649 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:03.929 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:03.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:03.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:04.036 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:04.064 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:58:04.109 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:04.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:04 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1455275732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:04.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:04 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:04.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:04 smithi125 ceph-mon[195507]: pgmap v851: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T10:58:04.160 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:04 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1455275732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:04.160 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:04 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:04.160 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:04 smithi125 ceph-mon[200197]: pgmap v851: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T10:58:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:04 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1455275732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:04 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:04 smithi155 ceph-mon[184854]: pgmap v851: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T10:58:04.490 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:04.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:04.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:04.556 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:58:04.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:58:04.636 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:58:04.637103+0000 min lat: 0.00120203 max lat: 0.0957076 avg lat: 0.0100622 2023-10-09T10:58:04.636 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:58:04.636 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 40 16 63629 63613 99.3345 68.5 0.0264653 0.0100622 2023-10-09T10:58:04.646 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:04.733 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:58:04.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:04.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:58:04.841 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:04.883 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:04.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:04.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:04.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:05.024 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:05.063 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:05.095 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:05.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:05.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:05.147 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:05.147 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:58:05.147 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[195507]: osdmap e346: 8 total, 8 up, 8 in 2023-10-09T10:58:05.148 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[195507]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.148 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[195507]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.148 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[195507]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.148 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[195507]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.148 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[195507]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.148 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[195507]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.148 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[195507]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.148 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[195507]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.148 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:05.148 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:05.148 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:05.149 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:58:05.149 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[200197]: osdmap e346: 8 total, 8 up, 8 in 2023-10-09T10:58:05.149 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[200197]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.149 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[200197]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.149 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[200197]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.149 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[200197]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.149 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[200197]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.149 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[200197]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.149 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[200197]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.149 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[200197]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.149 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:05.149 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:05 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:05.201 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:05.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:05 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:05.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:05 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:58:05.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:05 smithi155 ceph-mon[184854]: osdmap e346: 8 total, 8 up, 8 in 2023-10-09T10:58:05.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:05 smithi155 ceph-mon[184854]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:05 smithi155 ceph-mon[184854]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:05 smithi155 ceph-mon[184854]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:05 smithi155 ceph-mon[184854]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:05 smithi155 ceph-mon[184854]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:05 smithi155 ceph-mon[184854]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:05 smithi155 ceph-mon[184854]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:05 smithi155 ceph-mon[184854]: failed to encode map e346 with expected crc 2023-10-09T10:58:05.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:05 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:05.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:05 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:05.249 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:05.586 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:05.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:05.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:05.636 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 41 16 64656 64640 98.4775 64.1875 0.023064 0.01015 2023-10-09T10:58:05.690 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:05.753 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:05.788 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:05.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:05.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:05.897 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:05.916 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:05.918 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:58:05.931 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:05.952 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:05.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:05.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:05.978 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:58:06.028 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:58:06.056 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:06.080 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:58:06.090 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:06.118 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:06.120 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:58:06.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:06.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:06.148 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:58:06.159 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T10:58:06.171 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:58:06.216 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:58:06.216 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:58:06.216 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:58:06.217 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:58:06.217 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:58:06.217 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:58:06.217 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:58:06.217 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:58:06.217 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:58:06.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:06 smithi155 ceph-mon[184854]: pgmap v853: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T10:58:06.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:06 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/152466969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:06.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:06 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1095778569' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:58:06.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:06 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:58:06.222 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:06.304 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:06.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:06 smithi125 ceph-mon[195507]: pgmap v853: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T10:58:06.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:06 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/152466969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:06.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:06 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1095778569' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:58:06.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:06 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:58:06.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:06 smithi125 ceph-mon[200197]: pgmap v853: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T10:58:06.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:06 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/152466969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:06.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:06 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1095778569' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:58:06.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:06 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T10:58:06.636 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 42 16 65634 65618 97.5884 61.125 0.0166832 0.010242 2023-10-09T10:58:07.110 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:07.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:07.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:07.216 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:07.273 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:07.359 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:07.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:07.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:07.396 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:58:07.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:07 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:58:07.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:07 smithi125 ceph-mon[195507]: osdmap e347: 8 total, 8 up, 8 in 2023-10-09T10:58:07.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:07 smithi125 ceph-mon[195507]: failed to encode map e347 with expected crc 2023-10-09T10:58:07.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:07 smithi125 ceph-mon[195507]: failed to encode map e347 with expected crc 2023-10-09T10:58:07.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:07 smithi125 ceph-mon[195507]: failed to encode map e347 with expected crc 2023-10-09T10:58:07.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:07 smithi125 ceph-mon[195507]: failed to encode map e347 with expected crc 2023-10-09T10:58:07.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:07 smithi125 ceph-mon[195507]: failed to encode map e347 with expected crc 2023-10-09T10:58:07.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:07 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:58:07.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:07 smithi125 ceph-mon[200197]: osdmap e347: 8 total, 8 up, 8 in 2023-10-09T10:58:07.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:07 smithi125 ceph-mon[200197]: failed to encode map e347 with expected crc 2023-10-09T10:58:07.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:07 smithi125 ceph-mon[200197]: failed to encode map e347 with expected crc 2023-10-09T10:58:07.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:07 smithi125 ceph-mon[200197]: failed to encode map e347 with expected crc 2023-10-09T10:58:07.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:07 smithi125 ceph-mon[200197]: failed to encode map e347 with expected crc 2023-10-09T10:58:07.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:07 smithi125 ceph-mon[200197]: failed to encode map e347 with expected crc 2023-10-09T10:58:07.463 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:07 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T10:58:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:07 smithi155 ceph-mon[184854]: osdmap e347: 8 total, 8 up, 8 in 2023-10-09T10:58:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:07 smithi155 ceph-mon[184854]: failed to encode map e347 with expected crc 2023-10-09T10:58:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:07 smithi155 ceph-mon[184854]: failed to encode map e347 with expected crc 2023-10-09T10:58:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:07 smithi155 ceph-mon[184854]: failed to encode map e347 with expected crc 2023-10-09T10:58:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:07 smithi155 ceph-mon[184854]: failed to encode map e347 with expected crc 2023-10-09T10:58:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:07 smithi155 ceph-mon[184854]: failed to encode map e347 with expected crc 2023-10-09T10:58:07.501 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:07.637 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 43 16 66809 66793 97.0257 73.4375 0.0110665 0.0103014 2023-10-09T10:58:08.136 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:08.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:08.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:08.247 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:08.294 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:08.331 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:08.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:08.436 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:08.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:08 smithi155 ceph-mon[184854]: pgmap v855: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:58:08.481 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:08.510 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:08.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:08.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:08 smithi125 ceph-mon[195507]: pgmap v855: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:58:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:08 smithi125 ceph-mon[200197]: pgmap v855: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:58:08.614 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:08.637 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 44 16 68016 68000 96.5346 75.4375 0.0108419 0.0103546 2023-10-09T10:58:08.663 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:08.694 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:08.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:08.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:08.802 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:08.851 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:08.990 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:08.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:08.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:09.095 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:58:09.151 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:09.240 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:58:09.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:09.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:58:09.439 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:09.504 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:09.637 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 45 16 69186 69170 96.0145 73.125 0.0106129 0.0104104 2023-10-09T10:58:10.603 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:10.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:10.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:10.639 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 46 16 70439 70423 95.6268 78.3125 0.0107816 0.0104529 2023-10-09T10:58:10.709 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:10.769 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:10.808 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:10.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:10.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:10.914 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:10.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:10 smithi125 ceph-mon[195507]: pgmap v856: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2023-10-09T10:58:10.971 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:10 smithi125 ceph-mon[200197]: pgmap v856: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2023-10-09T10:58:10.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:10 smithi155 ceph-mon[184854]: pgmap v856: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2023-10-09T10:58:10.973 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:11.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:58:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:58:11] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-10-09T10:58:11.517 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:11.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:11.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:11.622 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:11.639 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 47 15 71627 71612 95.1734 74.3125 0.0115838 0.0105021 2023-10-09T10:58:11.674 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:11.702 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:11.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:11.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:11.781 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T10:58:11.807 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:11.851 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:11.870 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:11.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:11.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:11.974 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:12.013 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:12.038 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:12.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:12.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:12.142 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:12.173 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:12.236 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:12.242 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:12.431 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:12.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:12.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:12.541 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:12.572 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:12.597 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:12.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:12.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:12.604 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:12 smithi125 ceph-mon[195507]: pgmap v857: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T10:58:12.604 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:12 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:12.604 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:12 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:12.642 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 48 16 72494 72478 94.3143 54.125 0.0306444 0.0105966 2023-10-09T10:58:12.702 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:12.740 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:58:12.741 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:58:12.741 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:58:12.741 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:58:12.741 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:58:12.746 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:12.768 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:12.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:12.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:12.864 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:12 smithi125 ceph-mon[200197]: pgmap v857: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T10:58:12.865 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:12 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:12.865 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:12 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:12.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:12.937 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:12.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:12 smithi155 ceph-mon[184854]: pgmap v857: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T10:58:12.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:12 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:12.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:12 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:12.976 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:12.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:12.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:13.081 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:13.132 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:13.162 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:13.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:13.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:13.267 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:13.315 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:13.347 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:13.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:13.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:13.454 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:13.495 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:13.534 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:13.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:13.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:13.638 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:13.641 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 49 16 73564 73548 93.7546 66.875 0.0135135 0.0106615 2023-10-09T10:58:13.686 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:13.732 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:13.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:13.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:13.840 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:13.904 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:13.935 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:13.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:13.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:13.989 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:58:14.041 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:14.107 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:14.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:58:14.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:58:14.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:14.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:14 smithi155 ceph-mon[184854]: pgmap v858: 119 pgs: 119 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T10:58:14.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:58:14.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:58:14.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:14.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:14 smithi125 ceph-mon[195507]: pgmap v858: 119 pgs: 119 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T10:58:14.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:58:14.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:58:14.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:14.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:14 smithi125 ceph-mon[200197]: pgmap v858: 119 pgs: 119 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T10:58:14.641 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 50 16 74625 74609 93.2059 66.3125 0.00907034 0.0107244 2023-10-09T10:58:14.972 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:14.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:14.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:15.087 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:58:15.135 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:15.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:58:15.331 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:58:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:58:15.641 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 51 16 75742 75726 92.7473 69.8125 0.00805479 0.0107771 2023-10-09T10:58:16.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:16 smithi125 ceph-mon[200197]: pgmap v859: 119 pgs: 119 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T10:58:16.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:16 smithi125 ceph-mon[195507]: pgmap v859: 119 pgs: 119 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T10:58:16.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:16 smithi155 ceph-mon[184854]: pgmap v859: 119 pgs: 119 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T10:58:16.642 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 52 16 76896 76880 92.3507 72.125 0.00391674 0.0108227 2023-10-09T10:58:17.642 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 53 16 78061 78045 91.9822 72.8125 0.00913941 0.010867 2023-10-09T10:58:17.716 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:58:18.013 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:58:18.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:18.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:58:18.117 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:18.118 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:18.153 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:18.179 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:18.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:18.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:18.285 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:58:18.345 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:18.489 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:58:18.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:18.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:58:18.594 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:18.629 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:18.643 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 54 16 78917 78901 91.2678 53.5 0.0112499 0.0109516 2023-10-09T10:58:18.656 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:18.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:18.733 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:18.764 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:18.798 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:18.823 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:18.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:18.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:18.930 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:18.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:18 smithi155 ceph-mon[184854]: pgmap v860: 119 pgs: 119 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T10:58:18.976 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:18.996 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:18 smithi125 ceph-mon[195507]: pgmap v860: 119 pgs: 119 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T10:58:18.996 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:18 smithi125 ceph-mon[200197]: pgmap v860: 119 pgs: 119 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T10:58:19.003 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:19.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:19.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:19.110 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:19.153 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:19.333 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:19.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:19.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:19.442 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:19.492 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:19.519 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:19.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:19.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:19.625 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:19.634 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T10:58:19.634 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:58:19.643 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 55 16 79852 79836 90.6711 58.4375 0.0132796 0.0110226 2023-10-09T10:58:19.659 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:19.681 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:19.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:19.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:19.785 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:19.820 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:19 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3616933371' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:58:19.820 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:19 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:58:19.820 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:19 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/206627955' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:58:19.820 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:19 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:58:19.829 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:19.851 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:19.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:19.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:19.955 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:19.998 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:20.027 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:20.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:20.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:20.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:19 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3616933371' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:58:20.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:19 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:58:20.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:19 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/206627955' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:58:20.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:19 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:58:20.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:19 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3616933371' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:58:20.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:19 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T10:58:20.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:19 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/206627955' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:58:20.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:19 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:58:20.133 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:20.209 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:20.623 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T10:58:20.645 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 56 16 80793 80777 90.0988 58.8125 0.01578 0.0110931 2023-10-09T10:58:20.688 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:20 smithi125 ceph-mon[195507]: pgmap v861: 119 pgs: 119 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T10:58:20.689 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:20 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:20.689 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:20 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:58:20.689 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:20 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:58:20.689 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:20 smithi125 ceph-mon[195507]: osdmap e348: 8 total, 8 up, 8 in 2023-10-09T10:58:20.689 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:20 smithi125 ceph-mon[195507]: failed to encode map e348 with expected crc 2023-10-09T10:58:20.689 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:20 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1308655339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:20.689 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:20 smithi125 ceph-mon[200197]: pgmap v861: 119 pgs: 119 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T10:58:20.689 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:20 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:20.689 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:20 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:58:20.689 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:20 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:58:20.690 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:20 smithi125 ceph-mon[200197]: osdmap e348: 8 total, 8 up, 8 in 2023-10-09T10:58:20.690 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:20 smithi125 ceph-mon[200197]: failed to encode map e348 with expected crc 2023-10-09T10:58:20.690 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:20 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1308655339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:20.763 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:20.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:20.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:20.790 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:58:20.790 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:58:20.790 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:58:20.790 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:58:20.791 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:58:20.791 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:58:20.791 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:58:20.791 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:58:20.791 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:58:20.791 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:58:20.791 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:58:20.791 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:58:20.791 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:58:20.791 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:58:20.792 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:58:20.792 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:58:20.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:20.922 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:20.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:20 smithi155 ceph-mon[184854]: pgmap v861: 119 pgs: 119 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T10:58:20.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:20 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:20.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:20 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T10:58:20.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:20 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:58:20.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:20 smithi155 ceph-mon[184854]: osdmap e348: 8 total, 8 up, 8 in 2023-10-09T10:58:20.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:20 smithi155 ceph-mon[184854]: failed to encode map e348 with expected crc 2023-10-09T10:58:20.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:20 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1308655339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:21.212 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:21.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:21.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:21.321 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:21.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:58:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:58:21] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2023-10-09T10:58:21.355 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:21.389 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:21.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:21.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:21.494 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:21.551 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:21.552 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:21.598 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:21.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:21.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:21.627 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:58:21.627 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:58:21.645 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 57 16 81720 81704 89.5347 57.9375 0.0199135 0.0111626 2023-10-09T10:58:21.704 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:21.769 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:21.792 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:21.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:21.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:21.899 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:21.938 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:21.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:21.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:21.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:22.067 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:58:22.088 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:22.170 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:22.212 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:22.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:22.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:22.318 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:22.361 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:22.443 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:22.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:22.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:22.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:58:22.517 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:58:22.547 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:22.611 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:22.645 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 58 16 82668 82652 89.0128 59.25 0.026894 0.011229 2023-10-09T10:58:22.697 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:22.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:22.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:22.802 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:22.853 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:22.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:22 smithi155 ceph-mon[184854]: pgmap v863: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:58:22.993 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:22.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:22.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:23.043 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:23.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:22 smithi125 ceph-mon[195507]: pgmap v863: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:58:23.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:22 smithi125 ceph-mon[200197]: pgmap v863: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T10:58:23.099 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:23.150 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:23.179 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:23.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:23.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:23.187 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:58:23.284 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:23.332 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:23.466 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:23.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:23.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:23.570 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:23.635 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:23.646 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 59 16 83520 83504 88.4068 53.25 0.0291949 0.0113064 2023-10-09T10:58:23.669 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:23.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:23.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:23.773 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:23.783 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:58:23.783 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:58:23.838 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:23.878 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:23.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:23.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:23.910 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:23.987 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:24.033 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:24.045 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:24 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2616263908' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:58:24.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:24 smithi125 ceph-mon[195507]: pgmap v864: 119 pgs: 119 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T10:58:24.100 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T10:58:24.118 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:24.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:24.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:24.159 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:58:24.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:24 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2616263908' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:58:24.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:24 smithi155 ceph-mon[184854]: pgmap v864: 119 pgs: 119 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T10:58:24.223 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:24.272 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:24.307 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:24.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:24.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:24.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:24 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2616263908' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T10:58:24.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:24 smithi125 ceph-mon[200197]: pgmap v864: 119 pgs: 119 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T10:58:24.411 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:24.454 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:24.518 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:24.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:24.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:24.624 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:24.645 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:58:24.646585+0000 min lat: 0.00120203 max lat: 0.103158 avg lat: 0.0113762 2023-10-09T10:58:24.646 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:58:24.646 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 60 16 84410 84394 87.8607 55.625 0.0149517 0.0113762 2023-10-09T10:58:24.668 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:24.671 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:58:24.701 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:24.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:24.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:24.805 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:58:24.836 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:24.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:58:24.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:24.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:58:25.022 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:25.074 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:25.092 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2616263908' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:58:25.093 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:25 smithi155 ceph-mon[184854]: osdmap e349: 8 total, 8 up, 8 in 2023-10-09T10:58:25.093 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:25 smithi155 ceph-mon[184854]: failed to encode map e349 with expected crc 2023-10-09T10:58:25.093 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:25 smithi155 ceph-mon[184854]: failed to encode map e349 with expected crc 2023-10-09T10:58:25.093 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3858797189' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:25.102 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:58:25.211 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:25.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:25.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:25.316 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:25.357 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:25.398 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:25.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:25.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:25.505 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:25.560 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2616263908' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:58:25.560 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:25 smithi125 ceph-mon[195507]: osdmap e349: 8 total, 8 up, 8 in 2023-10-09T10:58:25.560 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:25 smithi125 ceph-mon[195507]: failed to encode map e349 with expected crc 2023-10-09T10:58:25.560 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:25 smithi125 ceph-mon[195507]: failed to encode map e349 with expected crc 2023-10-09T10:58:25.560 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3858797189' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:25.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2616263908' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T10:58:25.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:25 smithi125 ceph-mon[200197]: osdmap e349: 8 total, 8 up, 8 in 2023-10-09T10:58:25.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:25 smithi125 ceph-mon[200197]: failed to encode map e349 with expected crc 2023-10-09T10:58:25.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:25 smithi125 ceph-mon[200197]: failed to encode map e349 with expected crc 2023-10-09T10:58:25.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3858797189' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:25.565 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:25.645 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 61 16 85340 85324 87.3733 58.125 0.01489 0.01144 2023-10-09T10:58:25.956 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:58:25.956 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:58:26.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:26 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:58:26.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:26 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T10:58:26.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:26 smithi155 ceph-mon[184854]: failed to encode map e350 with expected crc 2023-10-09T10:58:26.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:26 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3858797189' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:58:26.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:26 smithi155 ceph-mon[184854]: osdmap e350: 8 total, 8 up, 8 in 2023-10-09T10:58:26.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:26 smithi155 ceph-mon[184854]: failed to encode map e350 with expected crc 2023-10-09T10:58:26.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:26 smithi155 ceph-mon[184854]: failed to encode map e350 with expected crc 2023-10-09T10:58:26.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:26 smithi155 ceph-mon[184854]: failed to encode map e350 with expected crc 2023-10-09T10:58:26.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:26 smithi155 ceph-mon[184854]: pgmap v867: 119 pgs: 119 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T10:58:26.346 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:58:26.379 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:58:26.379 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T10:58:26.379 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[195507]: failed to encode map e350 with expected crc 2023-10-09T10:58:26.379 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3858797189' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:58:26.379 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[195507]: osdmap e350: 8 total, 8 up, 8 in 2023-10-09T10:58:26.379 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[195507]: failed to encode map e350 with expected crc 2023-10-09T10:58:26.379 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[195507]: failed to encode map e350 with expected crc 2023-10-09T10:58:26.379 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[195507]: failed to encode map e350 with expected crc 2023-10-09T10:58:26.379 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[195507]: pgmap v867: 119 pgs: 119 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T10:58:26.380 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:58:26.380 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T10:58:26.380 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[200197]: failed to encode map e350 with expected crc 2023-10-09T10:58:26.380 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3858797189' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:58:26.380 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[200197]: osdmap e350: 8 total, 8 up, 8 in 2023-10-09T10:58:26.380 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[200197]: failed to encode map e350 with expected crc 2023-10-09T10:58:26.380 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[200197]: failed to encode map e350 with expected crc 2023-10-09T10:58:26.380 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[200197]: failed to encode map e350 with expected crc 2023-10-09T10:58:26.380 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:26 smithi125 ceph-mon[200197]: pgmap v867: 119 pgs: 119 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-09T10:58:26.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T10:58:26.648 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 62 16 86355 86339 86.9844 63.4375 0.016247 0.0114916 2023-10-09T10:58:26.740 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:58:26.866 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:26.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:26.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:26.971 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:26.971 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:27.044 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:27.379 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:27.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:27.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:27.483 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:27.561 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:27.626 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:58:27.648 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 63 16 87234 87218 86.4759 54.9375 0.0157402 0.0115591 2023-10-09T10:58:27.660 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:58:27.692 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:58:27.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:27.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:27.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:27.729 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:58:27.767 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:58:27.807 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:58:27.809 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:27 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1144659254' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:27.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:27 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1144659254' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:27.839 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:58:27.879 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:58:27.892 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:58:27.892 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:58:27.892 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:58:27.892 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:58:27.892 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:58:27.892 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:58:27.892 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:58:27.892 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:58:27.903 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:27.918 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:58:27.919 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:58:27.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:58:27.919 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:58:27.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.919 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:152608 7.b 7:d0fd2adb:::benchmark_data_smithi125_295816_object61471:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e245)", 2023-10-09T10:58:27.919 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:52:10.363211+0000", 2023-10-09T10:58:27.919 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 377.55153667299999, 2023-10-09T10:58:27.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8185187439999999, 2023-10-09T10:58:27.919 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:27.919 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:27.919 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:27.919 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:58:27.920 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:58:27.920 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 152608 2023-10-09T10:58:27.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.920 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:27.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:27.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363211+0000", 2023-10-09T10:58:27.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:27.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363211+0000", 2023-10-09T10:58:27.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:27.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363209+0000", 2023-10-09T10:58:27.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:58:27.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:27.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363218+0000", 2023-10-09T10:58:27.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8470000000000002e-06 2023-10-09T10:58:27.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:27.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363236+0000", 2023-10-09T10:58:27.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8062000000000001e-05 2023-10-09T10:58:27.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:27.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363244+0000", 2023-10-09T10:58:27.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2560000000000002e-06 2023-10-09T10:58:27.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.365179+0000", 2023-10-09T10:58:27.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001934755 2023-10-09T10:58:27.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op must wait for map", 2023-10-09T10:58:27.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.365188+0000", 2023-10-09T10:58:27.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.132e-06 2023-10-09T10:58:27.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.380926+0000", 2023-10-09T10:58:27.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015737860999999999 2023-10-09T10:58:27.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:58:27.924 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.380936+0000", 2023-10-09T10:58:27.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.679e-06 2023-10-09T10:58:27.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.924 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.172400+0000", 2023-10-09T10:58:27.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7914640509999999 2023-10-09T10:58:27.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:27.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.172506+0000", 2023-10-09T10:58:27.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106677 2023-10-09T10:58:27.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.925 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T10:58:27.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.172597+0000", 2023-10-09T10:58:27.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0718999999999997e-05 2023-10-09T10:58:27.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.925 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.176419+0000", 2023-10-09T10:58:27.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0038220480000000002 2023-10-09T10:58:27.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.176443+0000", 2023-10-09T10:58:27.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3583000000000001e-05 2023-10-09T10:58:27.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.176459+0000", 2023-10-09T10:58:27.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5815999999999999e-05 2023-10-09T10:58:27.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:27.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.181617+0000", 2023-10-09T10:58:27.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051585210000000001 2023-10-09T10:58:27.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:27.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.181654+0000", 2023-10-09T10:58:27.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6573999999999999e-05 2023-10-09T10:58:27.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:27.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.181730+0000", 2023-10-09T10:58:27.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6070999999999997e-05 2023-10-09T10:58:27.928 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.928 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:27.928 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.928 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:39213 8.2 8:4e7be898:::benchmark_data_smithi125_329074_object39212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T10:58:27.928 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:53:35.895072+0000", 2023-10-09T10:58:27.928 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 292.01967607300003, 2023-10-09T10:58:27.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.128363015, 2023-10-09T10:58:27.928 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:27.928 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:27.928 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:27.929 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:58:27.929 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:58:27.929 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39213 2023-10-09T10:58:27.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.929 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:27.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:27.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895072+0000", 2023-10-09T10:58:27.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:27.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895072+0000", 2023-10-09T10:58:27.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:27.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895070+0000", 2023-10-09T10:58:27.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:58:27.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:27.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895204+0000", 2023-10-09T10:58:27.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000133561 2023-10-09T10:58:27.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:27.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895215+0000", 2023-10-09T10:58:27.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0925e-05 2023-10-09T10:58:27.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:27.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895220+0000", 2023-10-09T10:58:27.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4949999999999999e-06 2023-10-09T10:58:27.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.932 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.932 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895798+0000", 2023-10-09T10:58:27.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00057733700000000003 2023-10-09T10:58:27.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.932 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.932 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:27.932 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895926+0000", 2023-10-09T10:58:27.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012778600000000001 2023-10-09T10:58:27.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.932 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.932 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:58:27.932 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.896072+0000", 2023-10-09T10:58:27.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014615599999999999 2023-10-09T10:58:27.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:27.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.897973+0000", 2023-10-09T10:58:27.933 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019008510000000001 2023-10-09T10:58:27.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.899072+0000", 2023-10-09T10:58:27.933 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010995239999999999 2023-10-09T10:58:27.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023353+0000", 2023-10-09T10:58:27.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.124281323 2023-10-09T10:58:27.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:27.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023405+0000", 2023-10-09T10:58:27.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1158e-05 2023-10-09T10:58:27.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:27.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023435+0000", 2023-10-09T10:58:27.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.006e-05 2023-10-09T10:58:27.935 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.935 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:27.935 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.935 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83954 8.4 8:2689e239:::benchmark_data_smithi125_329074_object83953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T10:58:27.935 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.108148+0000", 2023-10-09T10:58:27.935 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 250.80659992099999, 2023-10-09T10:58:27.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1275014759999999, 2023-10-09T10:58:27.935 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:27.936 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:27.936 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:27.936 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:58:27.936 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:58:27.936 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83954 2023-10-09T10:58:27.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.936 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:27.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:27.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108148+0000", 2023-10-09T10:58:27.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:27.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108148+0000", 2023-10-09T10:58:27.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:27.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108146+0000", 2023-10-09T10:58:27.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:58:27.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:27.938 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108191+0000", 2023-10-09T10:58:27.938 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4549000000000003e-05 2023-10-09T10:58:27.938 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.938 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:27.938 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108215+0000", 2023-10-09T10:58:27.938 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3994000000000001e-05 2023-10-09T10:58:27.938 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.938 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:27.938 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108222+0000", 2023-10-09T10:58:27.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4499999999999998e-06 2023-10-09T10:58:27.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.939 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108279+0000", 2023-10-09T10:58:27.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6643e-05 2023-10-09T10:58:27.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:58:27.939 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108288+0000", 2023-10-09T10:58:27.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5109999999999992e-06 2023-10-09T10:58:27.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.872892+0000", 2023-10-09T10:58:27.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.76460362800000004 2023-10-09T10:58:27.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:58:27.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.872903+0000", 2023-10-09T10:58:27.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1158000000000001e-05 2023-10-09T10:58:27.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213528+0000", 2023-10-09T10:58:27.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.340624501 2023-10-09T10:58:27.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.941 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:27.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213679+0000", 2023-10-09T10:58:27.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151578 2023-10-09T10:58:27.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.941 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T10:58:27.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213861+0000", 2023-10-09T10:58:27.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018154 2023-10-09T10:58:27.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:27.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219288+0000", 2023-10-09T10:58:27.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00542677 2023-10-09T10:58:27.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219340+0000", 2023-10-09T10:58:27.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2657000000000001e-05 2023-10-09T10:58:27.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219644+0000", 2023-10-09T10:58:27.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030375299999999998 2023-10-09T10:58:27.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235530+0000", 2023-10-09T10:58:27.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01588606 2023-10-09T10:58:27.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:27.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235575+0000", 2023-10-09T10:58:27.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5012999999999997e-05 2023-10-09T10:58:27.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.944 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:27.944 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235649+0000", 2023-10-09T10:58:27.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4252000000000007e-05 2023-10-09T10:58:27.944 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.944 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:27.944 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.944 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83959 8.4 8:27b6f913:::benchmark_data_smithi125_329074_object83958:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T10:58:27.944 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.112252+0000", 2023-10-09T10:58:27.945 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 250.80249589100001, 2023-10-09T10:58:27.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1235403750000001, 2023-10-09T10:58:27.945 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:27.945 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:27.945 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:27.945 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:58:27.945 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:58:27.945 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83959 2023-10-09T10:58:27.945 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.945 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:27.945 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.945 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:27.945 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112252+0000", 2023-10-09T10:58:27.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:27.946 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112252+0000", 2023-10-09T10:58:27.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:27.946 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112251+0000", 2023-10-09T10:58:27.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:58:27.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:27.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112312+0000", 2023-10-09T10:58:27.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0888999999999999e-05 2023-10-09T10:58:27.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:27.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112324+0000", 2023-10-09T10:58:27.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2585999999999999e-05 2023-10-09T10:58:27.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:27.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112329+0000", 2023-10-09T10:58:27.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9880000000000004e-06 2023-10-09T10:58:27.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112378+0000", 2023-10-09T10:58:27.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8905e-05 2023-10-09T10:58:27.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:58:27.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112389+0000", 2023-10-09T10:58:27.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0665e-05 2023-10-09T10:58:27.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.873769+0000", 2023-10-09T10:58:27.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.76138013699999996 2023-10-09T10:58:27.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:58:27.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.873776+0000", 2023-10-09T10:58:27.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.17e-06 2023-10-09T10:58:27.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.950 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216109+0000", 2023-10-09T10:58:27.950 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.342332699 2023-10-09T10:58:27.950 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:27.950 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216239+0000", 2023-10-09T10:58:27.950 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012997 2023-10-09T10:58:27.950 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T10:58:27.950 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216354+0000", 2023-10-09T10:58:27.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115116 2023-10-09T10:58:27.951 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.951 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.951 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219398+0000", 2023-10-09T10:58:27.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030438679999999999 2023-10-09T10:58:27.951 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.951 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.951 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:27.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219407+0000", 2023-10-09T10:58:27.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1870000000000003e-06 2023-10-09T10:58:27.951 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.952 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.952 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.952 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219682+0000", 2023-10-09T10:58:27.952 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00027478600000000001 2023-10-09T10:58:27.952 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.952 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.952 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.952 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235686+0000", 2023-10-09T10:58:27.952 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016004852 2023-10-09T10:58:27.952 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.952 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.952 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:27.952 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235725+0000", 2023-10-09T10:58:27.953 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8392000000000001e-05 2023-10-09T10:58:27.953 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.953 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:27.953 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235792+0000", 2023-10-09T10:58:27.953 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7311000000000007e-05 2023-10-09T10:58:27.953 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.953 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:27.953 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.953 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.953 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:93330 8.2 8:4f5a2bcc:::benchmark_data_smithi125_329074_object93329:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T10:58:27.954 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:35.625993+0000", 2023-10-09T10:58:27.954 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 232.28875500999999, 2023-10-09T10:58:27.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.12993565000000001, 2023-10-09T10:58:27.954 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:27.954 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:27.954 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:27.954 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:58:27.954 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:58:27.954 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 93330 2023-10-09T10:58:27.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.954 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:27.954 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:27.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625993+0000", 2023-10-09T10:58:27.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:27.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625993+0000", 2023-10-09T10:58:27.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:27.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625992+0000", 2023-10-09T10:58:27.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:58:27.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:27.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626043+0000", 2023-10-09T10:58:27.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0500000000000001e-05 2023-10-09T10:58:27.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:27.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626051+0000", 2023-10-09T10:58:27.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0509999999999997e-06 2023-10-09T10:58:27.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:27.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626056+0000", 2023-10-09T10:58:27.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9570000000000001e-06 2023-10-09T10:58:27.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626113+0000", 2023-10-09T10:58:27.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.736e-05 2023-10-09T10:58:27.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:27.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626283+0000", 2023-10-09T10:58:27.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017031699999999999 2023-10-09T10:58:27.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:58:27.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626463+0000", 2023-10-09T10:58:27.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018018 2023-10-09T10:58:27.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:27.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.634095+0000", 2023-10-09T10:58:27.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0076312469999999999 2023-10-09T10:58:27.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.959 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.959 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.959 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.637111+0000", 2023-10-09T10:58:27.959 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003016725 2023-10-09T10:58:27.959 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.959 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.959 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.959 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755853+0000", 2023-10-09T10:58:27.959 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11874143299999999 2023-10-09T10:58:27.959 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.959 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.959 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:27.960 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755887+0000", 2023-10-09T10:58:27.960 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4017999999999999e-05 2023-10-09T10:58:27.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.960 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.960 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:27.960 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755928+0000", 2023-10-09T10:58:27.960 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1452999999999998e-05 2023-10-09T10:58:27.960 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.960 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:27.960 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.960 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.961 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:93340 8.2 8:4653dd45:::benchmark_data_smithi125_329074_object93339:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T10:58:27.961 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:35.641334+0000", 2023-10-09T10:58:27.961 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 232.27341407, 2023-10-09T10:58:27.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.125653775, 2023-10-09T10:58:27.961 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:27.961 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:27.961 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:27.961 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:58:27.961 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:58:27.961 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 93340 2023-10-09T10:58:27.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.961 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:27.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:27.962 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641334+0000", 2023-10-09T10:58:27.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:27.962 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641334+0000", 2023-10-09T10:58:27.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:27.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641333+0000", 2023-10-09T10:58:27.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:58:27.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:27.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641362+0000", 2023-10-09T10:58:27.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9034999999999999e-05 2023-10-09T10:58:27.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:27.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641372+0000", 2023-10-09T10:58:27.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0597e-05 2023-10-09T10:58:27.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:27.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641377+0000", 2023-10-09T10:58:27.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7060000000000003e-06 2023-10-09T10:58:27.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641429+0000", 2023-10-09T10:58:27.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1721000000000002e-05 2023-10-09T10:58:27.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:27.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641529+0000", 2023-10-09T10:58:27.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000100602 2023-10-09T10:58:27.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:58:27.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641632+0000", 2023-10-09T10:58:27.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103099 2023-10-09T10:58:27.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.649088+0000", 2023-10-09T10:58:27.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0074551569999999996 2023-10-09T10:58:27.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.966 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.966 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:27.966 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.649159+0000", 2023-10-09T10:58:27.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1760000000000004e-05 2023-10-09T10:58:27.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.966 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.966 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.966 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766886+0000", 2023-10-09T10:58:27.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11772616499999999 2023-10-09T10:58:27.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.967 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:27.967 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766941+0000", 2023-10-09T10:58:27.967 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5186000000000003e-05 2023-10-09T10:58:27.967 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.967 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:27.967 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766987+0000", 2023-10-09T10:58:27.967 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6692e-05 2023-10-09T10:58:27.967 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.967 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:27.967 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.968 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.968 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.968 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:158888 8.2 8:4103043a:::benchmark_data_smithi125_329074_object62234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2023-10-09T10:58:27.968 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:56:15.356379+0000", 2023-10-09T10:58:27.968 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 132.55836871700001, 2023-10-09T10:58:27.968 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.123441579, 2023-10-09T10:58:27.968 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:27.968 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:27.968 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:27.968 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:58:27.968 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:58:27.968 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 158888 2023-10-09T10:58:27.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.969 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:27.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.969 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:27.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T10:58:27.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.969 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:27.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T10:58:27.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:27.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T10:58:27.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:58:27.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:27.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356381+0000", 2023-10-09T10:58:27.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.734e-06 2023-10-09T10:58:27.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:27.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356388+0000", 2023-10-09T10:58:27.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4559999999999997e-06 2023-10-09T10:58:27.971 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:27.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356392+0000", 2023-10-09T10:58:27.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0589999999999996e-06 2023-10-09T10:58:27.971 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.359480+0000", 2023-10-09T10:58:27.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030878379999999999 2023-10-09T10:58:27.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.972 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:27.972 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.359628+0000", 2023-10-09T10:58:27.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014820299999999999 2023-10-09T10:58:27.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.972 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:58:27.972 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.360074+0000", 2023-10-09T10:58:27.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00044630599999999999 2023-10-09T10:58:27.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:27.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.362052+0000", 2023-10-09T10:58:27.973 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019782720000000001 2023-10-09T10:58:27.973 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.362671+0000", 2023-10-09T10:58:27.973 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00061827099999999999 2023-10-09T10:58:27.973 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:27.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479765+0000", 2023-10-09T10:58:27.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11709412199999999 2023-10-09T10:58:27.974 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.974 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.974 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:27.974 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479796+0000", 2023-10-09T10:58:27.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0895000000000002e-05 2023-10-09T10:58:27.974 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.974 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.974 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:27.974 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479821+0000", 2023-10-09T10:58:27.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4816000000000001e-05 2023-10-09T10:58:27.975 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.975 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:27.975 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.975 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.975 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.975 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:77605 9.e e347/332 9:78442b40:::benchmark_data_smithi125_363789_object77604:head v 347'4969, mlcod=347'4967)", 2023-10-09T10:58:27.975 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:17.249655+0000", 2023-10-09T10:58:27.975 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 10.665092372, 2023-10-09T10:58:27.975 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013000650000000001, 2023-10-09T10:58:27.975 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:27.975 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:27.975 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:27.975 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:27.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.249655+0000", 2023-10-09T10:58:27.976 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:27.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.249655+0000", 2023-10-09T10:58:27.976 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:27.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.249654+0000", 2023-10-09T10:58:27.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:58:27.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:27.977 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.249691+0000", 2023-10-09T10:58:27.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6820999999999999e-05 2023-10-09T10:58:27.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:27.977 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.249703+0000", 2023-10-09T10:58:27.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1780000000000001e-05 2023-10-09T10:58:27.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:27.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.249710+0000", 2023-10-09T10:58:27.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.126e-06 2023-10-09T10:58:27.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.253831+0000", 2023-10-09T10:58:27.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041213919999999998 2023-10-09T10:58:27.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:27.979 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.253853+0000", 2023-10-09T10:58:27.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1331999999999998e-05 2023-10-09T10:58:27.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.979 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:27.979 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.262615+0000", 2023-10-09T10:58:27.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0087621350000000008 2023-10-09T10:58:27.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.979 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:27.979 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.262656+0000", 2023-10-09T10:58:27.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1121999999999997e-05 2023-10-09T10:58:27.980 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.980 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:27.980 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.980 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.980 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.980 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:77610 9.e e347/332 9:7ba62de0:::benchmark_data_smithi125_363789_object77609:head v 347'4970, mlcod=347'4967)", 2023-10-09T10:58:27.980 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:17.249723+0000", 2023-10-09T10:58:27.980 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 10.665024320000001, 2023-10-09T10:58:27.980 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012960047000000001, 2023-10-09T10:58:27.980 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:27.980 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:27.980 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:27.981 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.981 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:27.981 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.249723+0000", 2023-10-09T10:58:27.981 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.981 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.981 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.981 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:27.981 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.249723+0000", 2023-10-09T10:58:27.981 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.981 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.981 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.981 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:27.982 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.249723+0000", 2023-10-09T10:58:27.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:58:27.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.982 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.982 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:27.982 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.249745+0000", 2023-10-09T10:58:27.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2099999999999998e-05 2023-10-09T10:58:27.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.982 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.982 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:27.982 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.249751+0000", 2023-10-09T10:58:27.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.806e-06 2023-10-09T10:58:27.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:27.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.249755+0000", 2023-10-09T10:58:27.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9219999999999999e-06 2023-10-09T10:58:27.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.256165+0000", 2023-10-09T10:58:27.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.006410153 2023-10-09T10:58:27.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:27.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.256186+0000", 2023-10-09T10:58:27.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1243e-05 2023-10-09T10:58:27.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:27.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.262662+0000", 2023-10-09T10:58:27.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064758719999999997 2023-10-09T10:58:27.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:27.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.262683+0000", 2023-10-09T10:58:27.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.141e-05 2023-10-09T10:58:27.985 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.985 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:27.985 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.985 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:77710 9.e e347/332 9:7cde6bdd:::benchmark_data_smithi125_363789_object77709:head v 347'4983, mlcod=347'4980)", 2023-10-09T10:58:27.985 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:17.333506+0000", 2023-10-09T10:58:27.985 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 10.581242019999999, 2023-10-09T10:58:27.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0098861160000000003, 2023-10-09T10:58:27.985 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:27.985 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:27.986 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:27.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:27.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.333506+0000", 2023-10-09T10:58:27.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:27.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.333506+0000", 2023-10-09T10:58:27.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:27.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.333505+0000", 2023-10-09T10:58:27.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:58:27.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:27.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.333530+0000", 2023-10-09T10:58:27.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5357000000000001e-05 2023-10-09T10:58:27.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:27.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.333543+0000", 2023-10-09T10:58:27.988 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2724e-05 2023-10-09T10:58:27.988 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.988 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:27.988 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.333548+0000", 2023-10-09T10:58:27.988 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.451e-06 2023-10-09T10:58:27.988 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.988 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.988 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.333589+0000", 2023-10-09T10:58:27.988 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0036000000000001e-05 2023-10-09T10:58:27.988 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:27.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.333608+0000", 2023-10-09T10:58:27.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9907000000000001e-05 2023-10-09T10:58:27.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:27.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.343331+0000", 2023-10-09T10:58:27.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0097224459999999992 2023-10-09T10:58:27.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:27.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.343392+0000", 2023-10-09T10:58:27.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0902000000000001e-05 2023-10-09T10:58:27.990 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.990 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:27.990 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.990 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:77778 9.4 e347/330 9:2b8e9643:::benchmark_data_smithi125_363789_object77777:head v 347'4895, mlcod=347'4892)", 2023-10-09T10:58:27.990 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:17.398701+0000", 2023-10-09T10:58:27.990 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 10.516047155000001, 2023-10-09T10:58:27.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0099126409999999998, 2023-10-09T10:58:27.990 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:27.990 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:27.991 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:27.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:27.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.398701+0000", 2023-10-09T10:58:27.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:27.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.398701+0000", 2023-10-09T10:58:27.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:27.992 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.398700+0000", 2023-10-09T10:58:27.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:58:27.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:27.992 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.398734+0000", 2023-10-09T10:58:27.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3519000000000002e-05 2023-10-09T10:58:27.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:27.992 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.398740+0000", 2023-10-09T10:58:27.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5640000000000002e-06 2023-10-09T10:58:27.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:27.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.398744+0000", 2023-10-09T10:58:27.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9999999999999998e-06 2023-10-09T10:58:27.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.401241+0000", 2023-10-09T10:58:27.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024965730000000002 2023-10-09T10:58:27.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:27.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.401260+0000", 2023-10-09T10:58:27.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9349999999999999e-05 2023-10-09T10:58:27.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:27.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.408582+0000", 2023-10-09T10:58:27.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0073220129999999996 2023-10-09T10:58:27.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:27.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.408613+0000", 2023-10-09T10:58:27.995 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1004999999999999e-05 2023-10-09T10:58:27.995 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.995 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:27.995 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:27.995 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.995 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.995 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:77796 9.c e347/332 9:3519f8af:::benchmark_data_smithi125_363789_object77795:head v 347'4857, mlcod=347'4851)", 2023-10-09T10:58:27.995 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:17.410686+0000", 2023-10-09T10:58:27.995 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 10.504062189000001, 2023-10-09T10:58:27.995 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010706234, 2023-10-09T10:58:27.995 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:27.995 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:27.996 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:27.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.996 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:27.996 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.410686+0000", 2023-10-09T10:58:27.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.996 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:27.996 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.410686+0000", 2023-10-09T10:58:27.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:27.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.996 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:27.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.410685+0000", 2023-10-09T10:58:27.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:58:27.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:27.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.410757+0000", 2023-10-09T10:58:27.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2010999999999999e-05 2023-10-09T10:58:27.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:27.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.410765+0000", 2023-10-09T10:58:27.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9969999999999998e-06 2023-10-09T10:58:27.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:27.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.410770+0000", 2023-10-09T10:58:27.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8520000000000003e-06 2023-10-09T10:58:27.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:27.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.413147+0000", 2023-10-09T10:58:27.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023771159999999999 2023-10-09T10:58:27.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:27.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.413170+0000", 2023-10-09T10:58:27.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3014e-05 2023-10-09T10:58:27.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:27.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.421366+0000", 2023-10-09T10:58:27.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0081957709999999993 2023-10-09T10:58:27.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:27.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:27.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:27.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.421392+0000", 2023-10-09T10:58:27.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5942999999999999e-05 2023-10-09T10:58:28.000 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.000 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:28.000 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.000 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:77797 9.4 e347/330 9:2febf15a:::benchmark_data_smithi125_363789_object77796:head v 347'4897, mlcod=347'4894)", 2023-10-09T10:58:28.000 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:17.410825+0000", 2023-10-09T10:58:28.000 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 10.50392248, 2023-10-09T10:58:28.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010536660999999999, 2023-10-09T10:58:28.000 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:28.000 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:28.000 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:28.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:28.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.410825+0000", 2023-10-09T10:58:28.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:28.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.410825+0000", 2023-10-09T10:58:28.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:28.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.410825+0000", 2023-10-09T10:58:28.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:58:28.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:28.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.410874+0000", 2023-10-09T10:58:28.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9539999999999997e-05 2023-10-09T10:58:28.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:28.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.410884+0000", 2023-10-09T10:58:28.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3610000000000007e-06 2023-10-09T10:58:28.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:28.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.410888+0000", 2023-10-09T10:58:28.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6e-06 2023-10-09T10:58:28.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:28.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.413126+0000", 2023-10-09T10:58:28.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002237728 2023-10-09T10:58:28.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:28.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.413146+0000", 2023-10-09T10:58:28.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9599999999999999e-05 2023-10-09T10:58:28.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:28.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.421313+0000", 2023-10-09T10:58:28.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0081675080000000004 2023-10-09T10:58:28.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:28.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:17.421362+0000", 2023-10-09T10:58:28.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8789000000000003e-05 2023-10-09T10:58:28.005 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.005 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:28.005 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.005 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:78737 9.e e347/332 9:7d1b5ed6:::benchmark_data_smithi125_363789_object78736:head v 347'5068, mlcod=347'5064)", 2023-10-09T10:58:28.005 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:18.447403+0000", 2023-10-09T10:58:28.005 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 9.4673445039999997, 2023-10-09T10:58:28.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.009594811, 2023-10-09T10:58:28.005 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:28.005 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:28.005 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:28.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:28.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.447403+0000", 2023-10-09T10:58:28.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:28.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.447403+0000", 2023-10-09T10:58:28.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:28.007 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.447402+0000", 2023-10-09T10:58:28.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:58:28.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.007 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:28.007 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.447427+0000", 2023-10-09T10:58:28.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4524000000000001e-05 2023-10-09T10:58:28.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.007 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:28.007 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.447442+0000", 2023-10-09T10:58:28.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4803999999999999e-05 2023-10-09T10:58:28.008 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:28.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.447448+0000", 2023-10-09T10:58:28.008 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7190000000000004e-06 2023-10-09T10:58:28.008 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:28.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.447503+0000", 2023-10-09T10:58:28.008 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5024e-05 2023-10-09T10:58:28.008 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.009 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:28.009 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.447528+0000", 2023-10-09T10:58:28.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4456000000000001e-05 2023-10-09T10:58:28.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.009 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.009 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:28.009 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.456950+0000", 2023-10-09T10:58:28.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0094223369999999994 2023-10-09T10:58:28.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.009 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.009 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:28.009 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.456998+0000", 2023-10-09T10:58:28.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7796000000000001e-05 2023-10-09T10:58:28.010 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.010 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:28.010 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.010 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:79030 9.4 e347/330 9:2dda2cd8:::benchmark_data_smithi125_363789_object79029:head v 347'4978, mlcod=347'4976)", 2023-10-09T10:58:28.010 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:18.778842+0000", 2023-10-09T10:58:28.010 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 9.1359056219999992, 2023-10-09T10:58:28.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012052594, 2023-10-09T10:58:28.010 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:28.010 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:28.010 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:28.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:28.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.778842+0000", 2023-10-09T10:58:28.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:28.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.778842+0000", 2023-10-09T10:58:28.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:28.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.778841+0000", 2023-10-09T10:58:28.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:58:28.012 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:28.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.778885+0000", 2023-10-09T10:58:28.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4274999999999999e-05 2023-10-09T10:58:28.012 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:28.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.778900+0000", 2023-10-09T10:58:28.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4783000000000001e-05 2023-10-09T10:58:28.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:28.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.778906+0000", 2023-10-09T10:58:28.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8499999999999999e-06 2023-10-09T10:58:28.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:28.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.778946+0000", 2023-10-09T10:58:28.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9947000000000002e-05 2023-10-09T10:58:28.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:28.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.778964+0000", 2023-10-09T10:58:28.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8133000000000001e-05 2023-10-09T10:58:28.014 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:28.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.790844+0000", 2023-10-09T10:58:28.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011880139 2023-10-09T10:58:28.014 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:28.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.790895+0000", 2023-10-09T10:58:28.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0575999999999998e-05 2023-10-09T10:58:28.015 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.015 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:28.015 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.015 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.015 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.015 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:79033 9.e e347/332 9:73b90a9d:::benchmark_data_smithi125_363789_object79032:head v 347'5088, mlcod=347'5086)", 2023-10-09T10:58:28.015 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:18.779646+0000", 2023-10-09T10:58:28.015 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 9.1351017260000003, 2023-10-09T10:58:28.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012805591, 2023-10-09T10:58:28.015 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:28.016 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:28.016 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:28.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.016 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:28.016 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.779646+0000", 2023-10-09T10:58:28.016 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.016 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:28.016 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.779646+0000", 2023-10-09T10:58:28.016 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:28.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.779645+0000", 2023-10-09T10:58:28.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:58:28.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:28.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.779672+0000", 2023-10-09T10:58:28.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6886999999999999e-05 2023-10-09T10:58:28.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:28.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.779692+0000", 2023-10-09T10:58:28.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9678999999999999e-05 2023-10-09T10:58:28.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.018 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:28.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.779698+0000", 2023-10-09T10:58:28.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3940000000000001e-06 2023-10-09T10:58:28.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.018 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:28.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.779748+0000", 2023-10-09T10:58:28.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0034999999999998e-05 2023-10-09T10:58:28.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:28.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.779766+0000", 2023-10-09T10:58:28.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.819e-05 2023-10-09T10:58:28.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:28.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.792405+0000", 2023-10-09T10:58:28.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012638996 2023-10-09T10:58:28.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:28.020 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:18.792452+0000", 2023-10-09T10:58:28.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6162000000000003e-05 2023-10-09T10:58:28.020 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.020 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:28.020 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.020 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:79929 9.c e348/332 9:3cd55558:::benchmark_data_smithi125_363789_object79928:head v 348'4971, mlcod=348'4969)", 2023-10-09T10:58:28.020 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:19.753895+0000", 2023-10-09T10:58:28.020 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 8.1608529430000001, 2023-10-09T10:58:28.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011302436000000001, 2023-10-09T10:58:28.020 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:28.021 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:28.021 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:28.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:28.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.753895+0000", 2023-10-09T10:58:28.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:28.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.753895+0000", 2023-10-09T10:58:28.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.022 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:28.022 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.753894+0000", 2023-10-09T10:58:28.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:58:28.022 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.022 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.022 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:28.022 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.753966+0000", 2023-10-09T10:58:28.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1960999999999998e-05 2023-10-09T10:58:28.022 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.022 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.022 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:28.022 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.753980+0000", 2023-10-09T10:58:28.023 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4267000000000001e-05 2023-10-09T10:58:28.023 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.023 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.023 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:28.023 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.753986+0000", 2023-10-09T10:58:28.023 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.677e-06 2023-10-09T10:58:28.023 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.023 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.023 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:28.023 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.755182+0000", 2023-10-09T10:58:28.023 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011962640000000001 2023-10-09T10:58:28.023 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.024 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:28.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.755204+0000", 2023-10-09T10:58:28.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2243999999999999e-05 2023-10-09T10:58:28.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.024 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:28.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.765145+0000", 2023-10-09T10:58:28.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0099406560000000008 2023-10-09T10:58:28.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.024 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:28.025 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.765197+0000", 2023-10-09T10:58:28.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2043999999999998e-05 2023-10-09T10:58:28.025 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.025 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:28.025 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.025 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:79939 9.c e348/332 9:389eb408:::benchmark_data_smithi125_363789_object79938:head v 348'4972, mlcod=348'4969)", 2023-10-09T10:58:28.025 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:19.756650+0000", 2023-10-09T10:58:28.025 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 8.1580972650000003, 2023-10-09T10:58:28.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010572467, 2023-10-09T10:58:28.025 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:28.026 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:28.026 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:28.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.026 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:28.026 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.756650+0000", 2023-10-09T10:58:28.026 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.026 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:28.026 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.756650+0000", 2023-10-09T10:58:28.026 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.027 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:28.027 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.756649+0000", 2023-10-09T10:58:28.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:58:28.027 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.027 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.027 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:28.027 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.756752+0000", 2023-10-09T10:58:28.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000102845 2023-10-09T10:58:28.027 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.027 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.027 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:28.027 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.756765+0000", 2023-10-09T10:58:28.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2636000000000001e-05 2023-10-09T10:58:28.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:28.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.756771+0000", 2023-10-09T10:58:28.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2249999999999997e-06 2023-10-09T10:58:28.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:28.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.762506+0000", 2023-10-09T10:58:28.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0057345620000000003 2023-10-09T10:58:28.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.029 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:28.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.762527+0000", 2023-10-09T10:58:28.029 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1033e-05 2023-10-09T10:58:28.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.029 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:28.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.767194+0000", 2023-10-09T10:58:28.029 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0046675299999999996 2023-10-09T10:58:28.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.029 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:28.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:19.767223+0000", 2023-10-09T10:58:28.030 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8602000000000002e-05 2023-10-09T10:58:28.030 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.030 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:28.030 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.030 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.030 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.030 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:80227 9.c e348/332 9:354d561a:::benchmark_data_smithi125_363789_object80226:head v 348'4996, mlcod=348'4993)", 2023-10-09T10:58:28.030 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:20.055864+0000", 2023-10-09T10:58:28.030 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 7.8588840080000004, 2023-10-09T10:58:28.030 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0098049710000000009, 2023-10-09T10:58:28.030 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:28.031 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:28.031 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:28.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:28.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.055864+0000", 2023-10-09T10:58:28.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:28.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.055864+0000", 2023-10-09T10:58:28.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:28.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.055863+0000", 2023-10-09T10:58:28.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:58:28.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:28.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.055896+0000", 2023-10-09T10:58:28.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2984999999999997e-05 2023-10-09T10:58:28.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:28.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.055910+0000", 2023-10-09T10:58:28.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.367e-05 2023-10-09T10:58:28.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:28.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.055916+0000", 2023-10-09T10:58:28.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.28e-06 2023-10-09T10:58:28.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:28.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.055975+0000", 2023-10-09T10:58:28.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9073000000000002e-05 2023-10-09T10:58:28.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:28.034 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.055995+0000", 2023-10-09T10:58:28.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9562e-05 2023-10-09T10:58:28.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:28.034 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.065613+0000", 2023-10-09T10:58:28.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0096179690000000005 2023-10-09T10:58:28.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:28.035 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.065669+0000", 2023-10-09T10:58:28.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6152e-05 2023-10-09T10:58:28.035 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.035 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:28.035 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.035 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.035 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.035 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:80744 9.e e348/332 9:7cf15301:::benchmark_data_smithi125_363789_object80743:head v 348'5184, mlcod=348'5181)", 2023-10-09T10:58:28.035 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:20.599808+0000", 2023-10-09T10:58:28.035 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 7.3149393920000003, 2023-10-09T10:58:28.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0092245220000000006, 2023-10-09T10:58:28.035 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:28.036 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:28.036 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:28.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:28.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.599808+0000", 2023-10-09T10:58:28.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:28.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.599808+0000", 2023-10-09T10:58:28.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:28.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.037 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:28.037 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.599807+0000", 2023-10-09T10:58:28.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:58:28.037 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.037 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:28.037 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.599835+0000", 2023-10-09T10:58:28.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7946999999999999e-05 2023-10-09T10:58:28.037 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:28.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.599848+0000", 2023-10-09T10:58:28.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2473000000000001e-05 2023-10-09T10:58:28.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:28.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.599854+0000", 2023-10-09T10:58:28.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5610000000000004e-06 2023-10-09T10:58:28.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:28.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.599894+0000", 2023-10-09T10:58:28.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9657000000000003e-05 2023-10-09T10:58:28.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:28.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.599910+0000", 2023-10-09T10:58:28.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5421999999999999e-05 2023-10-09T10:58:28.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:28.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.608985+0000", 2023-10-09T10:58:28.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0090751479999999999 2023-10-09T10:58:28.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:28.040 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:28.040 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:28.040 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:20.609033+0000", 2023-10-09T10:58:28.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8154e-05 2023-10-09T10:58:28.040 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.040 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:28.040 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.040 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:28.040 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:28.040 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:58:28.041 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:27 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1144659254' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:28.088 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:28.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:28.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:28.194 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:28.267 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:28.412 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:28.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:28.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:28.520 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:28.571 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:28.651 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 64 16 88129 88113 85.9978 55.9375 0.0151319 0.0116228 2023-10-09T10:58:28.703 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:58:28.852 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:28.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:28 smithi155 ceph-mon[184854]: pgmap v868: 119 pgs: 119 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T10:58:28.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:28.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:28.962 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:29.006 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:29.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:28 smithi125 ceph-mon[195507]: pgmap v868: 119 pgs: 119 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T10:58:29.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:28 smithi125 ceph-mon[200197]: pgmap v868: 119 pgs: 119 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T10:58:29.151 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:29.650 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 65 16 89083 89067 85.5905 59.625 0.0123094 0.0116789 2023-10-09T10:58:29.694 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T10:58:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:29 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/18849167' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:58:30.029 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:29 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/18849167' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:58:30.029 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:29 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/18849167' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T10:58:30.208 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T10:58:30.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:58:30.297 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:58:30.298 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:30.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:30.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:30.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:58:30.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:30.441 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:30.467 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:30.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:30.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:30.572 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:30.622 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:30.652 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 66 15 89981 89966 85.144 56.1875 0.0241388 0.0117392 2023-10-09T10:58:30.656 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:30.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:30.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:30.761 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:30.797 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:30.824 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:30.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:30.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:30.937 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:30 smithi155 ceph-mon[184854]: pgmap v869: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2023-10-09T10:58:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:30 smithi155 ceph-mon[184854]: failed to encode map e351 with expected crc 2023-10-09T10:58:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:30 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/18849167' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:58:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:30 smithi155 ceph-mon[184854]: osdmap e351: 8 total, 8 up, 8 in 2023-10-09T10:58:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:30 smithi155 ceph-mon[184854]: failed to encode map e351 with expected crc 2023-10-09T10:58:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:58:30.997 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:30 smithi125 ceph-mon[195507]: pgmap v869: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2023-10-09T10:58:30.997 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:30 smithi125 ceph-mon[195507]: failed to encode map e351 with expected crc 2023-10-09T10:58:30.997 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:30 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/18849167' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:58:30.997 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:30 smithi125 ceph-mon[195507]: osdmap e351: 8 total, 8 up, 8 in 2023-10-09T10:58:30.997 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:30 smithi125 ceph-mon[195507]: failed to encode map e351 with expected crc 2023-10-09T10:58:30.998 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:30.998 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:58:30.998 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:30 smithi125 ceph-mon[200197]: pgmap v869: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2023-10-09T10:58:30.998 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:30 smithi125 ceph-mon[200197]: failed to encode map e351 with expected crc 2023-10-09T10:58:30.998 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:30 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/18849167' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T10:58:30.998 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:30 smithi125 ceph-mon[200197]: osdmap e351: 8 total, 8 up, 8 in 2023-10-09T10:58:30.999 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:30 smithi125 ceph-mon[200197]: failed to encode map e351 with expected crc 2023-10-09T10:58:30.999 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:30.999 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:58:31.002 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:31.188 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:31.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:31.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:31.296 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:31.306 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:31.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:58:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:58:31] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2023-10-09T10:58:31.338 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:31.362 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:31.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:31.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:31.373 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:31.395 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:58:31.472 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:31.514 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:31.606 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:31.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:31.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:31.657 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 67 16 90830 90814 84.6594 53 0.0182989 0.0118064 2023-10-09T10:58:31.711 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:31.738 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:58:31.770 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:31.810 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1527599653' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:58:31.810 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:31 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:58:31.810 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1527599653' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:58:31.810 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:31 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:58:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1527599653' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:58:31.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:31 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:58:32.057 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:32.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:32.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:32.160 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:32.223 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:32.280 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:32.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:32.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:32.388 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:58:32.442 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:32.522 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:58:32.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:32.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:58:32.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:32.656 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 68 16 91751 91735 84.261 57.5625 0.0198017 0.0118627 2023-10-09T10:58:32.671 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:32.723 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:32.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:32.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:32.827 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:32.884 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:32.889 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:32 smithi155 ceph-mon[184854]: pgmap v871: 119 pgs: 119 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2023-10-09T10:58:32.889 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:32 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:32.889 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:32 smithi155 ceph-mon[184854]: failed to encode map e352 with expected crc 2023-10-09T10:58:32.889 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:32 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:58:32.889 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:32 smithi155 ceph-mon[184854]: osdmap e352: 8 total, 8 up, 8 in 2023-10-09T10:58:32.889 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:32 smithi155 ceph-mon[184854]: failed to encode map e352 with expected crc 2023-10-09T10:58:32.889 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:32 smithi155 ceph-mon[184854]: failed to encode map e352 with expected crc 2023-10-09T10:58:32.914 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:32.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:32.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:32.976 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:32 smithi125 ceph-mon[195507]: pgmap v871: 119 pgs: 119 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2023-10-09T10:58:32.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:32 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:32.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:32 smithi125 ceph-mon[195507]: failed to encode map e352 with expected crc 2023-10-09T10:58:32.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:32 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:58:32.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:32 smithi125 ceph-mon[195507]: osdmap e352: 8 total, 8 up, 8 in 2023-10-09T10:58:32.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:32 smithi125 ceph-mon[195507]: failed to encode map e352 with expected crc 2023-10-09T10:58:32.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:32 smithi125 ceph-mon[195507]: failed to encode map e352 with expected crc 2023-10-09T10:58:32.978 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:32 smithi125 ceph-mon[200197]: pgmap v871: 119 pgs: 119 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2023-10-09T10:58:32.978 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:32 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:32.978 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:32 smithi125 ceph-mon[200197]: failed to encode map e352 with expected crc 2023-10-09T10:58:32.979 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:32 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:58:32.979 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:32 smithi125 ceph-mon[200197]: osdmap e352: 8 total, 8 up, 8 in 2023-10-09T10:58:32.979 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:32 smithi125 ceph-mon[200197]: failed to encode map e352 with expected crc 2023-10-09T10:58:32.979 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:32 smithi125 ceph-mon[200197]: failed to encode map e352 with expected crc 2023-10-09T10:58:33.019 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:33.083 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:33.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:33.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:33.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:33.136 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:58:33.136 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:58:33.136 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T10:58:33.136 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:58:33.144 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:58:33.188 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:58:33.221 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:33.245 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:58:33.268 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:33.297 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:33.297 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:58:33.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:33.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:33.356 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:58:33.395 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:58:33.400 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:33.435 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:33.451 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:58:33.471 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:33.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:33.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:33.500 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:58:33.512 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:58:33.512 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:58:33.512 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:58:33.513 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:58:33.513 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:58:33.513 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:58:33.513 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:58:33.513 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:58:33.533 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:58:33.545 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:58:33.566 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:58:33.576 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:33.636 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:33.656 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 69 16 92555 92539 83.7683 50.25 0.0186116 0.0119328 2023-10-09T10:58:33.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:33.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:33.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:33.778 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:33.829 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:33.915 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:33.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:33.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:33.957 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:58:34.024 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:34.085 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:34.118 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:34.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:34.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:34.194 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":872,"stamp":"2023-10-09T10:58:31.709916+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5818095520,"num_objects":88792,"num_object_clones":0,"num_object_copies":266376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":88792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88829,"num_write_kb":5681181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":1067242,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93714,"ondisk_log_size":93714,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22798852,"kb_used_data":17596676,"kb_used_omap":126,"kb_used_meta":5201985,"kb_avail":727162364,"statfs":{"total":767960285184,"available":744614260736,"internally_reserved":0,"allocated":18018996224,"data_stored":18006893254,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":5326833275},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,4,4,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":470351872,"num_objects":7177,"num_object_clones":0,"num_object_copies":21531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7177,"num_write_kb":459328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.635644"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":708,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.090705+0000","last_change":"2023-10-09T10:57:25.104150+0000","last_active":"2023-10-09T10:58:25.090705+0000","last_peered":"2023-10-09T10:58:25.090705+0000","last_clean":"2023-10-09T10:58:25.090705+0000","last_became_active":"2023-10-09T10:57:25.103762+0000","last_became_peered":"2023-10-09T10:57:25.103762+0000","last_unstale":"2023-10-09T10:58:25.090705+0000","last_undegraded":"2023-10-09T10:58:25.090705+0000","last_fullsized":"2023-10-09T10:58:25.090705+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:40.174975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:40.174975+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:35.416780+0000","scrub_duration":0.010654205,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":57,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.089696+0000","last_change":"2023-10-09T10:57:11.874010+0000","last_active":"2023-10-09T10:58:25.089696+0000","last_peered":"2023-10-09T10:58:25.089696+0000","last_clean":"2023-10-09T10:58:25.089696+0000","last_became_active":"2023-10-09T10:57:11.873713+0000","last_became_peered":"2023-10-09T10:57:11.873713+0000","last_unstale":"2023-10-09T10:58:25.089696+0000","last_undegraded":"2023-10-09T10:58:25.089696+0000","last_fullsized":"2023-10-09T10:58:25.089696+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:58.049973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":359,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079930+0000","last_change":"2023-10-09T10:54:47.945782+0000","last_active":"2023-10-09T10:58:25.079930+0000","last_peered":"2023-10-09T10:58:25.079930+0000","last_clean":"2023-10-09T10:58:25.079930+0000","last_became_active":"2023-10-09T10:54:47.945460+0000","last_became_peered":"2023-10-09T10:54:47.945460+0000","last_unstale":"2023-10-09T10:58:25.079930+0000","last_undegraded":"2023-10-09T10:58:25.079930+0000","last_fullsized":"2023-10-09T10:58:25.079930+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:50.537469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":731,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.086220+0000","last_change":"2023-10-09T10:53:34.537094+0000","last_active":"2023-10-09T10:58:24.086220+0000","last_peered":"2023-10-09T10:58:24.086220+0000","last_clean":"2023-10-09T10:58:24.086220+0000","last_became_active":"2023-10-09T10:52:55.414995+0000","last_became_peered":"2023-10-09T10:52:55.414995+0000","last_unstale":"2023-10-09T10:58:24.086220+0000","last_undegraded":"2023-10-09T10:58:24.086220+0000","last_fullsized":"2023-10-09T10:58:24.086220+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.536962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.536962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:30.197404+0000","scrub_duration":0.0053325960000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":67,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.087769+0000","last_change":"2023-10-09T10:54:47.944768+0000","last_active":"2023-10-09T10:58:24.087769+0000","last_peered":"2023-10-09T10:58:24.087769+0000","last_clean":"2023-10-09T10:58:24.087769+0000","last_became_active":"2023-10-09T10:54:47.943888+0000","last_became_peered":"2023-10-09T10:54:47.943888+0000","last_unstale":"2023-10-09T10:58:24.087769+0000","last_undegraded":"2023-10-09T10:58:24.087769+0000","last_fullsized":"2023-10-09T10:58:24.087769+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:11.046762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":278,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.087698+0000","last_change":"2023-10-09T10:57:22.007179+0000","last_active":"2023-10-09T10:58:24.087698+0000","last_peered":"2023-10-09T10:58:24.087698+0000","last_clean":"2023-10-09T10:58:24.087698+0000","last_became_active":"2023-10-09T10:57:22.006760+0000","last_became_peered":"2023-10-09T10:57:22.006760+0000","last_unstale":"2023-10-09T10:58:24.087698+0000","last_undegraded":"2023-10-09T10:58:24.087698+0000","last_fullsized":"2023-10-09T10:58:24.087698+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:13.771012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":774,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.099427+0000","last_change":"2023-10-09T10:57:25.103697+0000","last_active":"2023-10-09T10:58:25.099427+0000","last_peered":"2023-10-09T10:58:25.099427+0000","last_clean":"2023-10-09T10:58:25.099427+0000","last_became_active":"2023-10-09T10:57:25.103306+0000","last_became_peered":"2023-10-09T10:57:25.103306+0000","last_unstale":"2023-10-09T10:58:25.099427+0000","last_undegraded":"2023-10-09T10:58:25.099427+0000","last_fullsized":"2023-10-09T10:58:25.099427+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:23:07.530489+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":405,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079322+0000","last_change":"2023-10-09T10:54:48.038948+0000","last_active":"2023-10-09T10:58:25.079322+0000","last_peered":"2023-10-09T10:58:25.079322+0000","last_clean":"2023-10-09T10:58:25.079322+0000","last_became_active":"2023-10-09T10:54:48.001556+0000","last_became_peered":"2023-10-09T10:54:48.001556+0000","last_unstale":"2023-10-09T10:58:25.079322+0000","last_undegraded":"2023-10-09T10:58:25.079322+0000","last_fullsized":"2023-10-09T10:58:25.079322+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:23.415040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":67,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.091794+0000","last_change":"2023-10-09T10:54:47.914742+0000","last_active":"2023-10-09T10:58:24.091794+0000","last_peered":"2023-10-09T10:58:24.091794+0000","last_clean":"2023-10-09T10:58:24.091794+0000","last_became_active":"2023-10-09T10:54:47.914551+0000","last_became_peered":"2023-10-09T10:54:47.914551+0000","last_unstale":"2023-10-09T10:58:24.091794+0000","last_undegraded":"2023-10-09T10:58:24.091794+0000","last_fullsized":"2023-10-09T10:58:24.091794+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:16.067994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":766,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092079+0000","last_change":"2023-10-09T10:57:25.098434+0000","last_active":"2023-10-09T10:58:25.092079+0000","last_peered":"2023-10-09T10:58:25.092079+0000","last_clean":"2023-10-09T10:58:25.092079+0000","last_became_active":"2023-10-09T10:57:25.098064+0000","last_became_peered":"2023-10-09T10:57:25.098064+0000","last_unstale":"2023-10-09T10:58:25.092079+0000","last_undegraded":"2023-10-09T10:58:25.092079+0000","last_fullsized":"2023-10-09T10:58:25.092079+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.787782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.787782+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:03.016769+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":223,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.088204+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:58:24.088204+0000","last_peered":"2023-10-09T10:58:24.088204+0000","last_clean":"2023-10-09T10:58:24.088204+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:58:24.088204+0000","last_undegraded":"2023-10-09T10:58:24.088204+0000","last_fullsized":"2023-10-09T10:58:24.088204+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":360,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079318+0000","last_change":"2023-10-09T10:54:48.039275+0000","last_active":"2023-10-09T10:58:25.079318+0000","last_peered":"2023-10-09T10:58:25.079318+0000","last_clean":"2023-10-09T10:58:25.079318+0000","last_became_active":"2023-10-09T10:54:48.017645+0000","last_became_peered":"2023-10-09T10:54:48.017645+0000","last_unstale":"2023-10-09T10:58:25.079318+0000","last_undegraded":"2023-10-09T10:58:25.079318+0000","last_fullsized":"2023-10-09T10:58:25.079318+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:55.986791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":642,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.090080+0000","last_change":"2023-10-09T10:57:25.103739+0000","last_active":"2023-10-09T10:58:25.090080+0000","last_peered":"2023-10-09T10:58:25.090080+0000","last_clean":"2023-10-09T10:58:25.090080+0000","last_became_active":"2023-10-09T10:57:25.103378+0000","last_became_peered":"2023-10-09T10:57:25.103378+0000","last_unstale":"2023-10-09T10:58:25.090080+0000","last_undegraded":"2023-10-09T10:58:25.090080+0000","last_fullsized":"2023-10-09T10:58:25.090080+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:49:50.889945+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":28,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092487+0000","last_change":"2023-10-09T10:57:22.014188+0000","last_active":"2023-10-09T10:58:25.092487+0000","last_peered":"2023-10-09T10:58:25.092487+0000","last_clean":"2023-10-09T10:58:25.092487+0000","last_became_active":"2023-10-09T10:57:22.013975+0000","last_became_peered":"2023-10-09T10:57:22.013975+0000","last_unstale":"2023-10-09T10:58:25.092487+0000","last_undegraded":"2023-10-09T10:58:25.092487+0000","last_fullsized":"2023-10-09T10:58:25.092487+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:58.620741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":317,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079517+0000","last_change":"2023-10-09T10:57:22.010202+0000","last_active":"2023-10-09T10:58:25.079517+0000","last_peered":"2023-10-09T10:58:25.079517+0000","last_clean":"2023-10-09T10:58:25.079517+0000","last_became_active":"2023-10-09T10:57:22.006867+0000","last_became_peered":"2023-10-09T10:57:22.006867+0000","last_unstale":"2023-10-09T10:58:25.079517+0000","last_undegraded":"2023-10-09T10:58:25.079517+0000","last_fullsized":"2023-10-09T10:58:25.079517+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:54.212660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":739,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.087881+0000","last_change":"2023-10-09T10:53:32.540768+0000","last_active":"2023-10-09T10:58:24.087881+0000","last_peered":"2023-10-09T10:58:24.087881+0000","last_clean":"2023-10-09T10:58:24.087881+0000","last_became_active":"2023-10-09T10:52:55.414818+0000","last_became_peered":"2023-10-09T10:52:55.414818+0000","last_unstale":"2023-10-09T10:58:24.087881+0000","last_undegraded":"2023-10-09T10:58:24.087881+0000","last_fullsized":"2023-10-09T10:58:24.087881+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:32.540610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:32.540610+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:47.374383+0000","scrub_duration":0.0064251300000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":34,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092384+0000","last_change":"2023-10-09T10:57:22.032392+0000","last_active":"2023-10-09T10:58:25.092384+0000","last_peered":"2023-10-09T10:58:25.092384+0000","last_clean":"2023-10-09T10:58:25.092384+0000","last_became_active":"2023-10-09T10:57:22.012620+0000","last_became_peered":"2023-10-09T10:57:22.012620+0000","last_unstale":"2023-10-09T10:58:25.092384+0000","last_undegraded":"2023-10-09T10:58:25.092384+0000","last_fullsized":"2023-10-09T10:58:25.092384+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:06:34.979769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":44,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.089855+0000","last_change":"2023-10-09T10:57:46.838152+0000","last_active":"2023-10-09T10:58:25.089855+0000","last_peered":"2023-10-09T10:58:25.089855+0000","last_clean":"2023-10-09T10:58:25.089855+0000","last_became_active":"2023-10-09T10:57:46.837933+0000","last_became_peered":"2023-10-09T10:57:46.837933+0000","last_unstale":"2023-10-09T10:58:25.089855+0000","last_undegraded":"2023-10-09T10:58:25.089855+0000","last_fullsized":"2023-10-09T10:58:25.089855+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:26:23.218941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":736,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.089823+0000","last_change":"2023-10-09T10:57:18.113935+0000","last_active":"2023-10-09T10:58:24.089823+0000","last_peered":"2023-10-09T10:58:24.089823+0000","last_clean":"2023-10-09T10:58:24.089823+0000","last_became_active":"2023-10-09T10:54:50.913330+0000","last_became_peered":"2023-10-09T10:54:50.913330+0000","last_unstale":"2023-10-09T10:58:24.089823+0000","last_undegraded":"2023-10-09T10:58:24.089823+0000","last_fullsized":"2023-10-09T10:58:24.089823+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:18.113848+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.113848+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:51:24.460807+0000","scrub_duration":0.001474106,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":38,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.093297+0000","last_change":"2023-10-09T10:57:22.039995+0000","last_active":"2023-10-09T10:58:25.093297+0000","last_peered":"2023-10-09T10:58:25.093297+0000","last_clean":"2023-10-09T10:58:25.093297+0000","last_became_active":"2023-10-09T10:57:22.016287+0000","last_became_peered":"2023-10-09T10:57:22.016287+0000","last_unstale":"2023-10-09T10:58:25.093297+0000","last_undegraded":"2023-10-09T10:58:25.093297+0000","last_fullsized":"2023-10-09T10:58:25.093297+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:52.167501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":365,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.099381+0000","last_change":"2023-10-09T10:57:46.845755+0000","last_active":"2023-10-09T10:58:25.099381+0000","last_peered":"2023-10-09T10:58:25.099381+0000","last_clean":"2023-10-09T10:58:25.099381+0000","last_became_active":"2023-10-09T10:57:46.845475+0000","last_became_peered":"2023-10-09T10:57:46.845475+0000","last_unstale":"2023-10-09T10:58:25.099381+0000","last_undegraded":"2023-10-09T10:58:25.099381+0000","last_fullsized":"2023-10-09T10:58:25.099381+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:21.673994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":717,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.086746+0000","last_change":"2023-10-09T10:49:18.708370+0000","last_active":"2023-10-09T10:58:24.086746+0000","last_peered":"2023-10-09T10:58:24.086746+0000","last_clean":"2023-10-09T10:58:24.086746+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:58:24.086746+0000","last_undegraded":"2023-10-09T10:58:24.086746+0000","last_fullsized":"2023-10-09T10:58:24.086746+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:57.808410+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":34,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.093916+0000","last_change":"2023-10-09T10:57:22.026809+0000","last_active":"2023-10-09T10:58:25.093916+0000","last_peered":"2023-10-09T10:58:25.093916+0000","last_clean":"2023-10-09T10:58:25.093916+0000","last_became_active":"2023-10-09T10:57:22.011600+0000","last_became_peered":"2023-10-09T10:57:22.011600+0000","last_unstale":"2023-10-09T10:58:25.093916+0000","last_undegraded":"2023-10-09T10:58:25.093916+0000","last_fullsized":"2023-10-09T10:58:25.093916+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:10.453811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":428,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.084709+0000","last_change":"2023-10-09T10:57:22.006151+0000","last_active":"2023-10-09T10:58:25.084709+0000","last_peered":"2023-10-09T10:58:25.084709+0000","last_clean":"2023-10-09T10:58:25.084709+0000","last_became_active":"2023-10-09T10:57:22.005782+0000","last_became_peered":"2023-10-09T10:57:22.005782+0000","last_unstale":"2023-10-09T10:58:25.084709+0000","last_undegraded":"2023-10-09T10:58:25.084709+0000","last_fullsized":"2023-10-09T10:58:25.084709+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:09.733515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":670,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.091906+0000","last_change":"2023-10-09T10:57:25.098905+0000","last_active":"2023-10-09T10:58:25.091906+0000","last_peered":"2023-10-09T10:58:25.091906+0000","last_clean":"2023-10-09T10:58:25.091906+0000","last_became_active":"2023-10-09T10:57:25.098562+0000","last_became_peered":"2023-10-09T10:57:25.098562+0000","last_unstale":"2023-10-09T10:58:25.091906+0000","last_undegraded":"2023-10-09T10:58:25.091906+0000","last_fullsized":"2023-10-09T10:58:25.091906+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:23.020232+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":277,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.086116+0000","last_change":"2023-10-09T10:57:22.004708+0000","last_active":"2023-10-09T10:58:24.086116+0000","last_peered":"2023-10-09T10:58:24.086116+0000","last_clean":"2023-10-09T10:58:24.086116+0000","last_became_active":"2023-10-09T10:57:22.004413+0000","last_became_peered":"2023-10-09T10:57:22.004413+0000","last_unstale":"2023-10-09T10:58:24.086116+0000","last_undegraded":"2023-10-09T10:58:24.086116+0000","last_fullsized":"2023-10-09T10:58:24.086116+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:54.003556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":28,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092561+0000","last_change":"2023-10-09T10:57:22.014923+0000","last_active":"2023-10-09T10:58:25.092561+0000","last_peered":"2023-10-09T10:58:25.092561+0000","last_clean":"2023-10-09T10:58:25.092561+0000","last_became_active":"2023-10-09T10:57:22.014055+0000","last_became_peered":"2023-10-09T10:57:22.014055+0000","last_unstale":"2023-10-09T10:58:25.092561+0000","last_undegraded":"2023-10-09T10:58:25.092561+0000","last_fullsized":"2023-10-09T10:58:25.092561+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:50.546435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":667,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.089102+0000","last_change":"2023-10-09T10:57:44.843644+0000","last_active":"2023-10-09T10:58:24.089102+0000","last_peered":"2023-10-09T10:58:24.089102+0000","last_clean":"2023-10-09T10:58:24.089102+0000","last_became_active":"2023-10-09T10:54:50.915189+0000","last_became_peered":"2023-10-09T10:54:50.915189+0000","last_unstale":"2023-10-09T10:58:24.089102+0000","last_undegraded":"2023-10-09T10:58:24.089102+0000","last_fullsized":"2023-10-09T10:58:24.089102+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:44.843520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:44.843520+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:06.673360+0000","scrub_duration":0.0067258329999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":28,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.093729+0000","last_change":"2023-10-09T10:57:22.016705+0000","last_active":"2023-10-09T10:58:25.093729+0000","last_peered":"2023-10-09T10:58:25.093729+0000","last_clean":"2023-10-09T10:58:25.093729+0000","last_became_active":"2023-10-09T10:57:22.016452+0000","last_became_peered":"2023-10-09T10:57:22.016452+0000","last_unstale":"2023-10-09T10:58:25.093729+0000","last_undegraded":"2023-10-09T10:58:25.093729+0000","last_fullsized":"2023-10-09T10:58:25.093729+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:43:44.382821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":259,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.087474+0000","last_change":"2023-10-09T10:57:22.007046+0000","last_active":"2023-10-09T10:58:24.087474+0000","last_peered":"2023-10-09T10:58:24.087474+0000","last_clean":"2023-10-09T10:58:24.087474+0000","last_became_active":"2023-10-09T10:57:22.006360+0000","last_became_peered":"2023-10-09T10:57:22.006360+0000","last_unstale":"2023-10-09T10:58:24.087474+0000","last_undegraded":"2023-10-09T10:58:24.087474+0000","last_fullsized":"2023-10-09T10:58:24.087474+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:23.023072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":682,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.089686+0000","last_change":"2023-10-09T10:54:50.910027+0000","last_active":"2023-10-09T10:58:24.089686+0000","last_peered":"2023-10-09T10:58:24.089686+0000","last_clean":"2023-10-09T10:58:24.089686+0000","last_became_active":"2023-10-09T10:54:50.909698+0000","last_became_peered":"2023-10-09T10:54:50.909698+0000","last_unstale":"2023-10-09T10:58:24.089686+0000","last_undegraded":"2023-10-09T10:58:24.089686+0000","last_fullsized":"2023-10-09T10:58:24.089686+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:01.637196+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":226,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.086237+0000","last_change":"2023-10-09T10:51:12.366910+0000","last_active":"2023-10-09T10:58:24.086237+0000","last_peered":"2023-10-09T10:58:24.086237+0000","last_clean":"2023-10-09T10:58:24.086237+0000","last_became_active":"2023-10-09T10:51:12.366565+0000","last_became_peered":"2023-10-09T10:51:12.366565+0000","last_unstale":"2023-10-09T10:58:24.086237+0000","last_undegraded":"2023-10-09T10:58:24.086237+0000","last_fullsized":"2023-10-09T10:58:24.086237+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:53.464251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":405,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.082230+0000","last_change":"2023-10-09T10:57:22.008343+0000","last_active":"2023-10-09T10:58:25.082230+0000","last_peered":"2023-10-09T10:58:25.082230+0000","last_clean":"2023-10-09T10:58:25.082230+0000","last_became_active":"2023-10-09T10:57:22.007747+0000","last_became_peered":"2023-10-09T10:57:22.007747+0000","last_unstale":"2023-10-09T10:58:25.082230+0000","last_undegraded":"2023-10-09T10:58:25.082230+0000","last_fullsized":"2023-10-09T10:58:25.082230+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:45.774703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"350'5510","reported_seq":5548,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:27.658646+0000","last_change":"2023-10-09T10:57:22.013979+0000","last_active":"2023-10-09T10:58:27.658646+0000","last_peered":"2023-10-09T10:58:27.658646+0000","last_clean":"2023-10-09T10:58:27.658646+0000","last_became_active":"2023-10-09T10:57:22.013514+0000","last_became_peered":"2023-10-09T10:57:22.013514+0000","last_unstale":"2023-10-09T10:58:27.658646+0000","last_undegraded":"2023-10-09T10:58:27.658646+0000","last_fullsized":"2023-10-09T10:58:27.658646+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5510,"log_dups_size":0,"ondisk_log_size":5510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:16:42.975508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361103360,"num_objects":5510,"num_object_clones":0,"num_object_copies":16530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5510,"num_write_kb":352640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":691,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092240+0000","last_change":"2023-10-09T10:57:25.102011+0000","last_active":"2023-10-09T10:58:25.092240+0000","last_peered":"2023-10-09T10:58:25.092240+0000","last_clean":"2023-10-09T10:58:25.092240+0000","last_became_active":"2023-10-09T10:57:25.101702+0000","last_became_peered":"2023-10-09T10:57:25.101702+0000","last_unstale":"2023-10-09T10:58:25.092240+0000","last_undegraded":"2023-10-09T10:58:25.092240+0000","last_fullsized":"2023-10-09T10:58:25.092240+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:34.671859+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":32,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.093236+0000","last_change":"2023-10-09T10:57:22.021225+0000","last_active":"2023-10-09T10:58:25.093236+0000","last_peered":"2023-10-09T10:58:25.093236+0000","last_clean":"2023-10-09T10:58:25.093236+0000","last_became_active":"2023-10-09T10:57:22.007997+0000","last_became_peered":"2023-10-09T10:57:22.007997+0000","last_unstale":"2023-10-09T10:58:25.093236+0000","last_undegraded":"2023-10-09T10:58:25.093236+0000","last_fullsized":"2023-10-09T10:58:25.093236+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:52:19.871862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":28,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.091820+0000","last_change":"2023-10-09T10:57:22.016166+0000","last_active":"2023-10-09T10:58:25.091820+0000","last_peered":"2023-10-09T10:58:25.091820+0000","last_clean":"2023-10-09T10:58:25.091820+0000","last_became_active":"2023-10-09T10:57:22.015850+0000","last_became_peered":"2023-10-09T10:57:22.015850+0000","last_unstale":"2023-10-09T10:58:25.091820+0000","last_undegraded":"2023-10-09T10:58:25.091820+0000","last_fullsized":"2023-10-09T10:58:25.091820+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:37.827566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"350'5444","reported_seq":5473,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:26.862355+0000","last_change":"2023-10-09T10:57:19.986807+0000","last_active":"2023-10-09T10:58:26.862355+0000","last_peered":"2023-10-09T10:58:26.862355+0000","last_clean":"2023-10-09T10:58:26.862355+0000","last_became_active":"2023-10-09T10:57:19.986525+0000","last_became_peered":"2023-10-09T10:57:19.986525+0000","last_unstale":"2023-10-09T10:58:26.862355+0000","last_undegraded":"2023-10-09T10:58:26.862355+0000","last_fullsized":"2023-10-09T10:58:26.862355+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5444,"log_dups_size":0,"ondisk_log_size":5444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:23:53.293903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356777984,"num_objects":5444,"num_object_clones":0,"num_object_copies":16332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5444,"num_write_kb":348416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":712,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.081519+0000","last_change":"2023-10-09T10:53:25.540494+0000","last_active":"2023-10-09T10:58:25.081519+0000","last_peered":"2023-10-09T10:58:25.081519+0000","last_clean":"2023-10-09T10:58:25.081519+0000","last_became_active":"2023-10-09T10:51:01.295474+0000","last_became_peered":"2023-10-09T10:51:01.295474+0000","last_unstale":"2023-10-09T10:58:25.081519+0000","last_undegraded":"2023-10-09T10:58:25.081519+0000","last_fullsized":"2023-10-09T10:58:25.081519+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:39.069143+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":32,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092720+0000","last_change":"2023-10-09T10:57:22.023259+0000","last_active":"2023-10-09T10:58:25.092720+0000","last_peered":"2023-10-09T10:58:25.092720+0000","last_clean":"2023-10-09T10:58:25.092720+0000","last_became_active":"2023-10-09T10:57:22.009992+0000","last_became_peered":"2023-10-09T10:57:22.009992+0000","last_unstale":"2023-10-09T10:58:25.092720+0000","last_undegraded":"2023-10-09T10:58:25.092720+0000","last_fullsized":"2023-10-09T10:58:25.092720+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.289131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":67,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.087650+0000","last_change":"2023-10-09T10:54:47.944930+0000","last_active":"2023-10-09T10:58:24.087650+0000","last_peered":"2023-10-09T10:58:24.087650+0000","last_clean":"2023-10-09T10:58:24.087650+0000","last_became_active":"2023-10-09T10:54:47.944485+0000","last_became_peered":"2023-10-09T10:54:47.944485+0000","last_unstale":"2023-10-09T10:58:24.087650+0000","last_undegraded":"2023-10-09T10:58:24.087650+0000","last_fullsized":"2023-10-09T10:58:24.087650+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:11.213129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"351'5670","reported_seq":5697,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.196492+0000","last_change":"2023-10-09T10:57:22.024907+0000","last_active":"2023-10-09T10:58:31.196492+0000","last_peered":"2023-10-09T10:58:31.196492+0000","last_clean":"2023-10-09T10:58:31.196492+0000","last_became_active":"2023-10-09T10:57:22.024660+0000","last_became_peered":"2023-10-09T10:57:22.024660+0000","last_unstale":"2023-10-09T10:58:31.196492+0000","last_undegraded":"2023-10-09T10:58:31.196492+0000","last_fullsized":"2023-10-09T10:58:31.196492+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5670,"log_dups_size":0,"ondisk_log_size":5670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:25:10.105154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371589120,"num_objects":5670,"num_object_clones":0,"num_object_copies":17010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5670,"num_write_kb":362880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":663,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.100242+0000","last_change":"2023-10-09T10:53:36.680150+0000","last_active":"2023-10-09T10:58:25.100242+0000","last_peered":"2023-10-09T10:58:25.100242+0000","last_clean":"2023-10-09T10:58:25.100242+0000","last_became_active":"2023-10-09T10:51:50.371032+0000","last_became_peered":"2023-10-09T10:51:50.371032+0000","last_unstale":"2023-10-09T10:58:25.100242+0000","last_undegraded":"2023-10-09T10:58:25.100242+0000","last_fullsized":"2023-10-09T10:58:25.100242+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.680090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.680090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:08:28.952827+0000","scrub_duration":0.005577172,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":32,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092480+0000","last_change":"2023-10-09T10:57:22.034075+0000","last_active":"2023-10-09T10:58:25.092480+0000","last_peered":"2023-10-09T10:58:25.092480+0000","last_clean":"2023-10-09T10:58:25.092480+0000","last_became_active":"2023-10-09T10:57:22.015930+0000","last_became_peered":"2023-10-09T10:57:22.015930+0000","last_unstale":"2023-10-09T10:58:25.092480+0000","last_undegraded":"2023-10-09T10:58:25.092480+0000","last_fullsized":"2023-10-09T10:58:25.092480+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:16.494793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":85,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.089596+0000","last_change":"2023-10-09T10:57:22.005094+0000","last_active":"2023-10-09T10:58:24.089596+0000","last_peered":"2023-10-09T10:58:24.089596+0000","last_clean":"2023-10-09T10:58:24.089596+0000","last_became_active":"2023-10-09T10:57:22.004592+0000","last_became_peered":"2023-10-09T10:57:22.004592+0000","last_unstale":"2023-10-09T10:58:24.089596+0000","last_undegraded":"2023-10-09T10:58:24.089596+0000","last_fullsized":"2023-10-09T10:58:24.089596+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:35.526442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"351'5616","reported_seq":5646,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:30.978585+0000","last_change":"2023-10-09T10:57:20.305591+0000","last_active":"2023-10-09T10:58:30.978585+0000","last_peered":"2023-10-09T10:58:30.978585+0000","last_clean":"2023-10-09T10:58:30.978585+0000","last_became_active":"2023-10-09T10:57:20.305290+0000","last_became_peered":"2023-10-09T10:57:20.305290+0000","last_unstale":"2023-10-09T10:58:30.978585+0000","last_undegraded":"2023-10-09T10:58:30.978585+0000","last_fullsized":"2023-10-09T10:58:30.978585+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5616,"log_dups_size":0,"ondisk_log_size":5616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:54.880555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368050176,"num_objects":5616,"num_object_clones":0,"num_object_copies":16848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5616,"num_write_kb":359424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":836,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.102355+0000","last_change":"2023-10-09T10:57:25.111238+0000","last_active":"2023-10-09T10:58:25.102355+0000","last_peered":"2023-10-09T10:58:25.102355+0000","last_clean":"2023-10-09T10:58:25.102355+0000","last_became_active":"2023-10-09T10:57:25.110991+0000","last_became_peered":"2023-10-09T10:57:25.110991+0000","last_unstale":"2023-10-09T10:58:25.102355+0000","last_undegraded":"2023-10-09T10:58:25.102355+0000","last_fullsized":"2023-10-09T10:58:25.102355+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:57.060423+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":177,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.084555+0000","last_change":"2023-10-09T10:51:50.374527+0000","last_active":"2023-10-09T10:58:25.084555+0000","last_peered":"2023-10-09T10:58:25.084555+0000","last_clean":"2023-10-09T10:58:25.084555+0000","last_became_active":"2023-10-09T10:51:50.374297+0000","last_became_peered":"2023-10-09T10:51:50.374297+0000","last_unstale":"2023-10-09T10:58:25.084555+0000","last_undegraded":"2023-10-09T10:58:25.084555+0000","last_fullsized":"2023-10-09T10:58:25.084555+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:31:49.358307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":259,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.086493+0000","last_change":"2023-10-09T10:54:47.939771+0000","last_active":"2023-10-09T10:58:24.086493+0000","last_peered":"2023-10-09T10:58:24.086493+0000","last_clean":"2023-10-09T10:58:24.086493+0000","last_became_active":"2023-10-09T10:54:47.939440+0000","last_became_peered":"2023-10-09T10:54:47.939440+0000","last_unstale":"2023-10-09T10:58:24.086493+0000","last_undegraded":"2023-10-09T10:58:24.086493+0000","last_fullsized":"2023-10-09T10:58:24.086493+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:58.335536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"351'5612","reported_seq":5651,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:30.624051+0000","last_change":"2023-10-09T10:57:22.009934+0000","last_active":"2023-10-09T10:58:30.624051+0000","last_peered":"2023-10-09T10:58:30.624051+0000","last_clean":"2023-10-09T10:58:30.624051+0000","last_became_active":"2023-10-09T10:57:22.009682+0000","last_became_peered":"2023-10-09T10:57:22.009682+0000","last_unstale":"2023-10-09T10:58:30.624051+0000","last_undegraded":"2023-10-09T10:58:30.624051+0000","last_fullsized":"2023-10-09T10:58:30.624051+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5612,"log_dups_size":0,"ondisk_log_size":5612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:59.899734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367788032,"num_objects":5612,"num_object_clones":0,"num_object_copies":16836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5612,"num_write_kb":359168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":706,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.093550+0000","last_change":"2023-10-09T10:57:25.106422+0000","last_active":"2023-10-09T10:58:25.093550+0000","last_peered":"2023-10-09T10:58:25.093550+0000","last_clean":"2023-10-09T10:58:25.093550+0000","last_became_active":"2023-10-09T10:57:25.105998+0000","last_became_peered":"2023-10-09T10:57:25.105998+0000","last_unstale":"2023-10-09T10:58:25.093550+0000","last_undegraded":"2023-10-09T10:58:25.093550+0000","last_fullsized":"2023-10-09T10:58:25.093550+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:10:53.164214+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":348,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079203+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:58:25.079203+0000","last_peered":"2023-10-09T10:58:25.079203+0000","last_clean":"2023-10-09T10:58:25.079203+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:58:25.079203+0000","last_undegraded":"2023-10-09T10:58:25.079203+0000","last_fullsized":"2023-10-09T10:58:25.079203+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":193,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.084474+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T10:58:25.084474+0000","last_peered":"2023-10-09T10:58:25.084474+0000","last_clean":"2023-10-09T10:58:25.084474+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T10:58:25.084474+0000","last_undegraded":"2023-10-09T10:58:25.084474+0000","last_fullsized":"2023-10-09T10:58:25.084474+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"350'5338","reported_seq":5367,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:26.859750+0000","last_change":"2023-10-09T10:57:20.327412+0000","last_active":"2023-10-09T10:58:26.859750+0000","last_peered":"2023-10-09T10:58:26.859750+0000","last_clean":"2023-10-09T10:58:26.859750+0000","last_became_active":"2023-10-09T10:57:20.327192+0000","last_became_peered":"2023-10-09T10:57:20.327192+0000","last_unstale":"2023-10-09T10:58:26.859750+0000","last_undegraded":"2023-10-09T10:58:26.859750+0000","last_fullsized":"2023-10-09T10:58:26.859750+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5338,"log_dups_size":0,"ondisk_log_size":5338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:17:53.257184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349831168,"num_objects":5338,"num_object_clones":0,"num_object_copies":16014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5338,"num_write_kb":341632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":655,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.090633+0000","last_change":"2023-10-09T10:54:21.173078+0000","last_active":"2023-10-09T10:58:25.090633+0000","last_peered":"2023-10-09T10:58:25.090633+0000","last_clean":"2023-10-09T10:58:25.090633+0000","last_became_active":"2023-10-09T10:54:21.172767+0000","last_became_peered":"2023-10-09T10:54:21.172767+0000","last_unstale":"2023-10-09T10:58:25.090633+0000","last_undegraded":"2023-10-09T10:58:25.090633+0000","last_fullsized":"2023-10-09T10:58:25.090633+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.532890+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.532890+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:32.335841+0000","scrub_duration":0.0045290130000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":330,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.099575+0000","last_change":"2023-10-09T10:54:47.908506+0000","last_active":"2023-10-09T10:58:25.099575+0000","last_peered":"2023-10-09T10:58:25.099575+0000","last_clean":"2023-10-09T10:58:25.099575+0000","last_became_active":"2023-10-09T10:54:47.908110+0000","last_became_peered":"2023-10-09T10:54:47.908110+0000","last_unstale":"2023-10-09T10:58:25.099575+0000","last_undegraded":"2023-10-09T10:58:25.099575+0000","last_fullsized":"2023-10-09T10:58:25.099575+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:38.808943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":90,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.090382+0000","last_change":"2023-10-09T10:54:47.948732+0000","last_active":"2023-10-09T10:58:25.090382+0000","last_peered":"2023-10-09T10:58:25.090382+0000","last_clean":"2023-10-09T10:58:25.090382+0000","last_became_active":"2023-10-09T10:54:47.948414+0000","last_became_peered":"2023-10-09T10:54:47.948414+0000","last_unstale":"2023-10-09T10:58:25.090382+0000","last_undegraded":"2023-10-09T10:58:25.090382+0000","last_fullsized":"2023-10-09T10:58:25.090382+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:38.020807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"351'5585","reported_seq":5612,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.224799+0000","last_change":"2023-10-09T10:57:22.024943+0000","last_active":"2023-10-09T10:58:31.224799+0000","last_peered":"2023-10-09T10:58:31.224799+0000","last_clean":"2023-10-09T10:58:31.224799+0000","last_became_active":"2023-10-09T10:57:22.024623+0000","last_became_peered":"2023-10-09T10:57:22.024623+0000","last_unstale":"2023-10-09T10:58:31.224799+0000","last_undegraded":"2023-10-09T10:58:31.224799+0000","last_fullsized":"2023-10-09T10:58:31.224799+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5585,"log_dups_size":0,"ondisk_log_size":5585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:13:55.093719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366018560,"num_objects":5585,"num_object_clones":0,"num_object_copies":16755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5585,"num_write_kb":357440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":737,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.089266+0000","last_change":"2023-10-09T10:57:45.845982+0000","last_active":"2023-10-09T10:58:24.089266+0000","last_peered":"2023-10-09T10:58:24.089266+0000","last_clean":"2023-10-09T10:58:24.089266+0000","last_became_active":"2023-10-09T10:54:50.909639+0000","last_became_peered":"2023-10-09T10:54:50.909639+0000","last_unstale":"2023-10-09T10:58:24.089266+0000","last_undegraded":"2023-10-09T10:58:24.089266+0000","last_fullsized":"2023-10-09T10:58:24.089266+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:45.845850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:45.845850+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:52:51.027029+0000","scrub_duration":0.011054232000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":406,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.099237+0000","last_change":"2023-10-09T10:57:22.006224+0000","last_active":"2023-10-09T10:58:25.099237+0000","last_peered":"2023-10-09T10:58:25.099237+0000","last_clean":"2023-10-09T10:58:25.099237+0000","last_became_active":"2023-10-09T10:57:22.005898+0000","last_became_peered":"2023-10-09T10:57:22.005898+0000","last_unstale":"2023-10-09T10:58:25.099237+0000","last_undegraded":"2023-10-09T10:58:25.099237+0000","last_fullsized":"2023-10-09T10:58:25.099237+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:00.623898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":67,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.089866+0000","last_change":"2023-10-09T10:54:47.913826+0000","last_active":"2023-10-09T10:58:24.089866+0000","last_peered":"2023-10-09T10:58:24.089866+0000","last_clean":"2023-10-09T10:58:24.089866+0000","last_became_active":"2023-10-09T10:54:47.913524+0000","last_became_peered":"2023-10-09T10:54:47.913524+0000","last_unstale":"2023-10-09T10:58:24.089866+0000","last_undegraded":"2023-10-09T10:58:24.089866+0000","last_fullsized":"2023-10-09T10:58:24.089866+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:56:23.699552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.2","version":"351'5598","reported_seq":5628,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:30.504524+0000","last_change":"2023-10-09T10:57:19.986622+0000","last_active":"2023-10-09T10:58:30.504524+0000","last_peered":"2023-10-09T10:58:30.504524+0000","last_clean":"2023-10-09T10:58:30.504524+0000","last_became_active":"2023-10-09T10:57:19.986337+0000","last_became_peered":"2023-10-09T10:57:19.986337+0000","last_unstale":"2023-10-09T10:58:30.504524+0000","last_undegraded":"2023-10-09T10:58:30.504524+0000","last_fullsized":"2023-10-09T10:58:30.504524+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5598,"log_dups_size":0,"ondisk_log_size":5598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:59.817044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366870528,"num_objects":5598,"num_object_clones":0,"num_object_copies":16794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5598,"num_write_kb":358272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":666,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.089782+0000","last_change":"2023-10-09T10:54:21.152148+0000","last_active":"2023-10-09T10:58:25.089782+0000","last_peered":"2023-10-09T10:58:25.089782+0000","last_clean":"2023-10-09T10:58:25.089782+0000","last_became_active":"2023-10-09T10:54:21.151809+0000","last_became_peered":"2023-10-09T10:54:21.151809+0000","last_unstale":"2023-10-09T10:58:25.089782+0000","last_undegraded":"2023-10-09T10:58:25.089782+0000","last_fullsized":"2023-10-09T10:58:25.089782+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.701128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.701128+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:27.021145+0000","scrub_duration":0.0025071830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":397,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079151+0000","last_change":"2023-10-09T10:57:22.006965+0000","last_active":"2023-10-09T10:58:25.079151+0000","last_peered":"2023-10-09T10:58:25.079151+0000","last_clean":"2023-10-09T10:58:25.079151+0000","last_became_active":"2023-10-09T10:57:22.006456+0000","last_became_peered":"2023-10-09T10:57:22.006456+0000","last_unstale":"2023-10-09T10:58:25.079151+0000","last_undegraded":"2023-10-09T10:58:25.079151+0000","last_fullsized":"2023-10-09T10:58:25.079151+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:16.889422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":67,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.088845+0000","last_change":"2023-10-09T10:54:48.012411+0000","last_active":"2023-10-09T10:58:24.088845+0000","last_peered":"2023-10-09T10:58:24.088845+0000","last_clean":"2023-10-09T10:58:24.088845+0000","last_became_active":"2023-10-09T10:54:48.011894+0000","last_became_peered":"2023-10-09T10:54:48.011894+0000","last_unstale":"2023-10-09T10:58:24.088845+0000","last_undegraded":"2023-10-09T10:58:24.088845+0000","last_fullsized":"2023-10-09T10:58:24.088845+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:28.067337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"350'5475","reported_seq":5504,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:26.857051+0000","last_change":"2023-10-09T10:57:20.306271+0000","last_active":"2023-10-09T10:58:26.857051+0000","last_peered":"2023-10-09T10:58:26.857051+0000","last_clean":"2023-10-09T10:58:26.857051+0000","last_became_active":"2023-10-09T10:57:20.305994+0000","last_became_peered":"2023-10-09T10:57:20.305994+0000","last_unstale":"2023-10-09T10:58:26.857051+0000","last_undegraded":"2023-10-09T10:58:26.857051+0000","last_fullsized":"2023-10-09T10:58:26.857051+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5475,"log_dups_size":0,"ondisk_log_size":5475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:46.777641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358809600,"num_objects":5475,"num_object_clones":0,"num_object_copies":16425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5475,"num_write_kb":350400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":783,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.081932+0000","last_change":"2023-10-09T10:57:25.110928+0000","last_active":"2023-10-09T10:58:25.081932+0000","last_peered":"2023-10-09T10:58:25.081932+0000","last_clean":"2023-10-09T10:58:25.081932+0000","last_became_active":"2023-10-09T10:57:25.110240+0000","last_became_peered":"2023-10-09T10:57:25.110240+0000","last_unstale":"2023-10-09T10:58:25.081932+0000","last_undegraded":"2023-10-09T10:58:25.081932+0000","last_fullsized":"2023-10-09T10:58:25.081932+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:34.828051+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":85,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.090791+0000","last_change":"2023-10-09T10:57:22.005377+0000","last_active":"2023-10-09T10:58:24.090791+0000","last_peered":"2023-10-09T10:58:24.090791+0000","last_clean":"2023-10-09T10:58:24.090791+0000","last_became_active":"2023-10-09T10:57:22.005016+0000","last_became_peered":"2023-10-09T10:57:22.005016+0000","last_unstale":"2023-10-09T10:58:24.090791+0000","last_undegraded":"2023-10-09T10:58:24.090791+0000","last_fullsized":"2023-10-09T10:58:24.090791+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:03.296528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":272,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.080285+0000","last_change":"2023-10-09T10:54:17.886921+0000","last_active":"2023-10-09T10:58:25.080285+0000","last_peered":"2023-10-09T10:58:25.080285+0000","last_clean":"2023-10-09T10:58:25.080285+0000","last_became_active":"2023-10-09T10:54:17.886584+0000","last_became_peered":"2023-10-09T10:54:17.886584+0000","last_unstale":"2023-10-09T10:58:25.080285+0000","last_undegraded":"2023-10-09T10:58:25.080285+0000","last_fullsized":"2023-10-09T10:58:25.080285+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:15.848951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":67,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.089345+0000","last_change":"2023-10-09T10:54:47.908294+0000","last_active":"2023-10-09T10:58:24.089345+0000","last_peered":"2023-10-09T10:58:24.089345+0000","last_clean":"2023-10-09T10:58:24.089345+0000","last_became_active":"2023-10-09T10:54:47.908019+0000","last_became_peered":"2023-10-09T10:54:47.908019+0000","last_unstale":"2023-10-09T10:58:24.089345+0000","last_undegraded":"2023-10-09T10:58:24.089345+0000","last_fullsized":"2023-10-09T10:58:24.089345+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:48.662764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"350'5507","reported_seq":5545,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:27.622918+0000","last_change":"2023-10-09T10:57:22.005714+0000","last_active":"2023-10-09T10:58:27.622918+0000","last_peered":"2023-10-09T10:58:27.622918+0000","last_clean":"2023-10-09T10:58:27.622918+0000","last_became_active":"2023-10-09T10:57:22.005431+0000","last_became_peered":"2023-10-09T10:57:22.005431+0000","last_unstale":"2023-10-09T10:58:27.622918+0000","last_undegraded":"2023-10-09T10:58:27.622918+0000","last_fullsized":"2023-10-09T10:58:27.622918+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5507,"log_dups_size":0,"ondisk_log_size":5507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:51.650262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360906752,"num_objects":5507,"num_object_clones":0,"num_object_copies":16521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5507,"num_write_kb":352448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":688,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.086671+0000","last_change":"2023-10-09T10:54:21.163072+0000","last_active":"2023-10-09T10:58:24.086671+0000","last_peered":"2023-10-09T10:58:24.086671+0000","last_clean":"2023-10-09T10:58:24.086671+0000","last_became_active":"2023-10-09T10:54:21.162710+0000","last_became_peered":"2023-10-09T10:54:21.162710+0000","last_unstale":"2023-10-09T10:58:24.086671+0000","last_undegraded":"2023-10-09T10:58:24.086671+0000","last_fullsized":"2023-10-09T10:58:24.086671+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:15.932806+0000","scrub_duration":0.0019146790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":67,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.089512+0000","last_change":"2023-10-09T10:54:47.899967+0000","last_active":"2023-10-09T10:58:24.089512+0000","last_peered":"2023-10-09T10:58:24.089512+0000","last_clean":"2023-10-09T10:58:24.089512+0000","last_became_active":"2023-10-09T10:54:47.899584+0000","last_became_peered":"2023-10-09T10:54:47.899584+0000","last_unstale":"2023-10-09T10:58:24.089512+0000","last_undegraded":"2023-10-09T10:58:24.089512+0000","last_fullsized":"2023-10-09T10:58:24.089512+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:43.973868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":260,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.086339+0000","last_change":"2023-10-09T10:50:43.388615+0000","last_active":"2023-10-09T10:58:24.086339+0000","last_peered":"2023-10-09T10:58:24.086339+0000","last_clean":"2023-10-09T10:58:24.086339+0000","last_became_active":"2023-10-09T10:50:43.388176+0000","last_became_peered":"2023-10-09T10:50:43.388176+0000","last_unstale":"2023-10-09T10:58:24.086339+0000","last_undegraded":"2023-10-09T10:58:24.086339+0000","last_fullsized":"2023-10-09T10:58:24.086339+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:41:17.550111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":67,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.090719+0000","last_change":"2023-10-09T10:54:47.950462+0000","last_active":"2023-10-09T10:58:24.090719+0000","last_peered":"2023-10-09T10:58:24.090719+0000","last_clean":"2023-10-09T10:58:24.090719+0000","last_became_active":"2023-10-09T10:54:47.940606+0000","last_became_peered":"2023-10-09T10:54:47.940606+0000","last_unstale":"2023-10-09T10:58:24.090719+0000","last_undegraded":"2023-10-09T10:58:24.090719+0000","last_fullsized":"2023-10-09T10:58:24.090719+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:50.557850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"350'5362","reported_seq":5400,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:26.835175+0000","last_change":"2023-10-09T10:57:22.004913+0000","last_active":"2023-10-09T10:58:26.835175+0000","last_peered":"2023-10-09T10:58:26.835175+0000","last_clean":"2023-10-09T10:58:26.835175+0000","last_became_active":"2023-10-09T10:57:22.004172+0000","last_became_peered":"2023-10-09T10:57:22.004172+0000","last_unstale":"2023-10-09T10:58:26.835175+0000","last_undegraded":"2023-10-09T10:58:26.835175+0000","last_fullsized":"2023-10-09T10:58:26.835175+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5362,"log_dups_size":0,"ondisk_log_size":5362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:01.045466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351404032,"num_objects":5362,"num_object_clones":0,"num_object_copies":16086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5362,"num_write_kb":343168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":765,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092854+0000","last_change":"2023-10-09T10:57:25.101168+0000","last_active":"2023-10-09T10:58:25.092854+0000","last_peered":"2023-10-09T10:58:25.092854+0000","last_clean":"2023-10-09T10:58:25.092854+0000","last_became_active":"2023-10-09T10:57:25.100860+0000","last_became_peered":"2023-10-09T10:57:25.100860+0000","last_unstale":"2023-10-09T10:58:25.092854+0000","last_undegraded":"2023-10-09T10:58:25.092854+0000","last_fullsized":"2023-10-09T10:58:25.092854+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:37.594583+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":82,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:26.001823+0000","last_change":"2023-10-09T10:54:17.885030+0000","last_active":"2023-10-09T10:58:26.001823+0000","last_peered":"2023-10-09T10:58:26.001823+0000","last_clean":"2023-10-09T10:58:26.001823+0000","last_became_active":"2023-10-09T10:54:17.884752+0000","last_became_peered":"2023-10-09T10:54:17.884752+0000","last_unstale":"2023-10-09T10:58:26.001823+0000","last_undegraded":"2023-10-09T10:58:26.001823+0000","last_fullsized":"2023-10-09T10:58:26.001823+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:38.548132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":67,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.087899+0000","last_change":"2023-10-09T10:54:47.927822+0000","last_active":"2023-10-09T10:58:24.087899+0000","last_peered":"2023-10-09T10:58:24.087899+0000","last_clean":"2023-10-09T10:58:24.087899+0000","last_became_active":"2023-10-09T10:54:47.927455+0000","last_became_peered":"2023-10-09T10:54:47.927455+0000","last_unstale":"2023-10-09T10:58:24.087899+0000","last_undegraded":"2023-10-09T10:58:24.087899+0000","last_fullsized":"2023-10-09T10:58:24.087899+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:11.818203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":67,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.091870+0000","last_change":"2023-10-09T10:54:48.012629+0000","last_active":"2023-10-09T10:58:24.091870+0000","last_peered":"2023-10-09T10:58:24.091870+0000","last_clean":"2023-10-09T10:58:24.091870+0000","last_became_active":"2023-10-09T10:54:48.012148+0000","last_became_peered":"2023-10-09T10:54:48.012148+0000","last_unstale":"2023-10-09T10:58:24.091870+0000","last_undegraded":"2023-10-09T10:58:24.091870+0000","last_fullsized":"2023-10-09T10:58:24.091870+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:22.894111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"351'5584","reported_seq":5614,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:30.567901+0000","last_change":"2023-10-09T10:57:20.305367+0000","last_active":"2023-10-09T10:58:30.567901+0000","last_peered":"2023-10-09T10:58:30.567901+0000","last_clean":"2023-10-09T10:58:30.567901+0000","last_became_active":"2023-10-09T10:57:20.305050+0000","last_became_peered":"2023-10-09T10:57:20.305050+0000","last_unstale":"2023-10-09T10:58:30.567901+0000","last_undegraded":"2023-10-09T10:58:30.567901+0000","last_fullsized":"2023-10-09T10:58:30.567901+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5584,"log_dups_size":0,"ondisk_log_size":5584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:49.570577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365953024,"num_objects":5584,"num_object_clones":0,"num_object_copies":16752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5584,"num_write_kb":357376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":100,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.669707+0000","last_change":"2023-10-09T10:57:22.006186+0000","last_active":"2023-10-09T10:58:29.669707+0000","last_peered":"2023-10-09T10:58:29.669707+0000","last_clean":"2023-10-09T10:58:29.669707+0000","last_became_active":"2023-10-09T10:57:22.005900+0000","last_became_peered":"2023-10-09T10:57:22.005900+0000","last_unstale":"2023-10-09T10:58:29.669707+0000","last_undegraded":"2023-10-09T10:58:29.669707+0000","last_fullsized":"2023-10-09T10:58:29.669707+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:14.033785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":782,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092148+0000","last_change":"2023-10-09T10:57:25.100325+0000","last_active":"2023-10-09T10:58:25.092148+0000","last_peered":"2023-10-09T10:58:25.092148+0000","last_clean":"2023-10-09T10:58:25.092148+0000","last_became_active":"2023-10-09T10:57:25.099940+0000","last_became_peered":"2023-10-09T10:57:25.099940+0000","last_unstale":"2023-10-09T10:58:25.092148+0000","last_undegraded":"2023-10-09T10:58:25.092148+0000","last_fullsized":"2023-10-09T10:58:25.092148+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:33.410507+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":284,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079910+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:58:25.079910+0000","last_peered":"2023-10-09T10:58:25.079910+0000","last_clean":"2023-10-09T10:58:25.079910+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:58:25.079910+0000","last_undegraded":"2023-10-09T10:58:25.079910+0000","last_fullsized":"2023-10-09T10:58:25.079910+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":283,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.099919+0000","last_change":"2023-10-09T10:57:22.013585+0000","last_active":"2023-10-09T10:58:25.099919+0000","last_peered":"2023-10-09T10:58:25.099919+0000","last_clean":"2023-10-09T10:58:25.099919+0000","last_became_active":"2023-10-09T10:57:22.013255+0000","last_became_peered":"2023-10-09T10:57:22.013255+0000","last_unstale":"2023-10-09T10:58:25.099919+0000","last_undegraded":"2023-10-09T10:58:25.099919+0000","last_fullsized":"2023-10-09T10:58:25.099919+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:29.725918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"350'5424","reported_seq":5453,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:27.655403+0000","last_change":"2023-10-09T10:57:20.328496+0000","last_active":"2023-10-09T10:58:27.655403+0000","last_peered":"2023-10-09T10:58:27.655403+0000","last_clean":"2023-10-09T10:58:27.655403+0000","last_became_active":"2023-10-09T10:57:20.328194+0000","last_became_peered":"2023-10-09T10:57:20.328194+0000","last_unstale":"2023-10-09T10:58:27.655403+0000","last_undegraded":"2023-10-09T10:58:27.655403+0000","last_fullsized":"2023-10-09T10:58:27.655403+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5424,"log_dups_size":0,"ondisk_log_size":5424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:15:57.867795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355467264,"num_objects":5424,"num_object_clones":0,"num_object_copies":16272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5424,"num_write_kb":347136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":710,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.089150+0000","last_change":"2023-10-09T10:57:43.819618+0000","last_active":"2023-10-09T10:58:24.089150+0000","last_peered":"2023-10-09T10:58:24.089150+0000","last_clean":"2023-10-09T10:58:24.089150+0000","last_became_active":"2023-10-09T10:54:50.914536+0000","last_became_peered":"2023-10-09T10:54:50.914536+0000","last_unstale":"2023-10-09T10:58:24.089150+0000","last_undegraded":"2023-10-09T10:58:24.089150+0000","last_fullsized":"2023-10-09T10:58:24.089150+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:43.819496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:43.819496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:09.320712+0000","scrub_duration":0.0071681970000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":28,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092048+0000","last_change":"2023-10-09T10:57:22.008209+0000","last_active":"2023-10-09T10:58:25.092048+0000","last_peered":"2023-10-09T10:58:25.092048+0000","last_clean":"2023-10-09T10:58:25.092048+0000","last_became_active":"2023-10-09T10:57:22.007918+0000","last_became_peered":"2023-10-09T10:57:22.007918+0000","last_unstale":"2023-10-09T10:58:25.092048+0000","last_undegraded":"2023-10-09T10:58:25.092048+0000","last_fullsized":"2023-10-09T10:58:25.092048+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:55.764820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":232,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.086489+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:58:24.086489+0000","last_peered":"2023-10-09T10:58:24.086489+0000","last_clean":"2023-10-09T10:58:24.086489+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:58:24.086489+0000","last_undegraded":"2023-10-09T10:58:24.086489+0000","last_fullsized":"2023-10-09T10:58:24.086489+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":295,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.087544+0000","last_change":"2023-10-09T10:54:47.939756+0000","last_active":"2023-10-09T10:58:24.087544+0000","last_peered":"2023-10-09T10:58:24.087544+0000","last_clean":"2023-10-09T10:58:24.087544+0000","last_became_active":"2023-10-09T10:54:47.938928+0000","last_became_peered":"2023-10-09T10:54:47.938928+0000","last_unstale":"2023-10-09T10:58:24.087544+0000","last_undegraded":"2023-10-09T10:58:24.087544+0000","last_fullsized":"2023-10-09T10:58:24.087544+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:05.310929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"351'5787","reported_seq":5815,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.197211+0000","last_change":"2023-10-09T10:57:22.010174+0000","last_active":"2023-10-09T10:58:31.197211+0000","last_peered":"2023-10-09T10:58:31.197211+0000","last_clean":"2023-10-09T10:58:31.197211+0000","last_became_active":"2023-10-09T10:57:22.009987+0000","last_became_peered":"2023-10-09T10:57:22.009987+0000","last_unstale":"2023-10-09T10:58:31.197211+0000","last_undegraded":"2023-10-09T10:58:31.197211+0000","last_fullsized":"2023-10-09T10:58:31.197211+0000","mapping_epoch":332,"log_start":"336'100","ondisk_log_start":"336'100","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5687,"log_dups_size":0,"ondisk_log_size":5687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:45.169457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379256832,"num_objects":5787,"num_object_clones":0,"num_object_copies":17361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5787,"num_write_kb":370368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":723,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.087780+0000","last_change":"2023-10-09T10:53:27.625180+0000","last_active":"2023-10-09T10:58:24.087780+0000","last_peered":"2023-10-09T10:58:24.087780+0000","last_clean":"2023-10-09T10:58:24.087780+0000","last_became_active":"2023-10-09T10:51:12.366828+0000","last_became_peered":"2023-10-09T10:51:12.366828+0000","last_unstale":"2023-10-09T10:58:24.087780+0000","last_undegraded":"2023-10-09T10:58:24.087780+0000","last_fullsized":"2023-10-09T10:58:24.087780+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.625079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.625079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:59:41.253976+0000","scrub_duration":0.0041067969999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092160+0000","last_change":"2023-10-09T10:57:22.012147+0000","last_active":"2023-10-09T10:58:25.092160+0000","last_peered":"2023-10-09T10:58:25.092160+0000","last_clean":"2023-10-09T10:58:25.092160+0000","last_became_active":"2023-10-09T10:57:22.011818+0000","last_became_peered":"2023-10-09T10:57:22.011818+0000","last_unstale":"2023-10-09T10:58:25.092160+0000","last_undegraded":"2023-10-09T10:58:25.092160+0000","last_fullsized":"2023-10-09T10:58:25.092160+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:10.838344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":67,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.090976+0000","last_change":"2023-10-09T10:54:47.925711+0000","last_active":"2023-10-09T10:58:24.090976+0000","last_peered":"2023-10-09T10:58:24.090976+0000","last_clean":"2023-10-09T10:58:24.090976+0000","last_became_active":"2023-10-09T10:54:47.924286+0000","last_became_peered":"2023-10-09T10:54:47.924286+0000","last_unstale":"2023-10-09T10:58:24.090976+0000","last_undegraded":"2023-10-09T10:58:24.090976+0000","last_fullsized":"2023-10-09T10:58:24.090976+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:58.379080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":277,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.086390+0000","last_change":"2023-10-09T10:57:22.004838+0000","last_active":"2023-10-09T10:58:24.086390+0000","last_peered":"2023-10-09T10:58:24.086390+0000","last_clean":"2023-10-09T10:58:24.086390+0000","last_became_active":"2023-10-09T10:57:22.004519+0000","last_became_peered":"2023-10-09T10:57:22.004519+0000","last_unstale":"2023-10-09T10:58:24.086390+0000","last_undegraded":"2023-10-09T10:58:24.086390+0000","last_fullsized":"2023-10-09T10:58:24.086390+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:53.585466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"351'5645","reported_seq":5673,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.216986+0000","last_change":"2023-10-09T10:57:22.015103+0000","last_active":"2023-10-09T10:58:31.216986+0000","last_peered":"2023-10-09T10:58:31.216986+0000","last_clean":"2023-10-09T10:58:31.216986+0000","last_became_active":"2023-10-09T10:57:22.014603+0000","last_became_peered":"2023-10-09T10:57:22.014603+0000","last_unstale":"2023-10-09T10:58:31.216986+0000","last_undegraded":"2023-10-09T10:58:31.216986+0000","last_fullsized":"2023-10-09T10:58:31.216986+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5645,"log_dups_size":0,"ondisk_log_size":5645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:22:34.112316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369950720,"num_objects":5645,"num_object_clones":0,"num_object_copies":16935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5645,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":746,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.099851+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T10:58:25.099851+0000","last_peered":"2023-10-09T10:58:25.099851+0000","last_clean":"2023-10-09T10:58:25.099851+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T10:58:25.099851+0000","last_undegraded":"2023-10-09T10:58:25.099851+0000","last_fullsized":"2023-10-09T10:58:25.099851+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":400,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.082134+0000","last_change":"2023-10-09T10:57:22.030437+0000","last_active":"2023-10-09T10:58:25.082134+0000","last_peered":"2023-10-09T10:58:25.082134+0000","last_clean":"2023-10-09T10:58:25.082134+0000","last_became_active":"2023-10-09T10:57:22.008090+0000","last_became_peered":"2023-10-09T10:57:22.008090+0000","last_unstale":"2023-10-09T10:58:25.082134+0000","last_undegraded":"2023-10-09T10:58:25.082134+0000","last_fullsized":"2023-10-09T10:58:25.082134+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:52:54.221183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":67,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.089935+0000","last_change":"2023-10-09T10:54:47.908305+0000","last_active":"2023-10-09T10:58:24.089935+0000","last_peered":"2023-10-09T10:58:24.089935+0000","last_clean":"2023-10-09T10:58:24.089935+0000","last_became_active":"2023-10-09T10:54:47.907999+0000","last_became_peered":"2023-10-09T10:54:47.907999+0000","last_unstale":"2023-10-09T10:58:24.089935+0000","last_undegraded":"2023-10-09T10:58:24.089935+0000","last_fullsized":"2023-10-09T10:58:24.089935+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:48.405124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":392,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079245+0000","last_change":"2023-10-09T10:57:22.007248+0000","last_active":"2023-10-09T10:58:25.079245+0000","last_peered":"2023-10-09T10:58:25.079245+0000","last_clean":"2023-10-09T10:58:25.079245+0000","last_became_active":"2023-10-09T10:57:22.006899+0000","last_became_peered":"2023-10-09T10:57:22.006899+0000","last_unstale":"2023-10-09T10:58:25.079245+0000","last_undegraded":"2023-10-09T10:58:25.079245+0000","last_fullsized":"2023-10-09T10:58:25.079245+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:45.159091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"351'5611","reported_seq":5649,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:30.585755+0000","last_change":"2023-10-09T10:57:22.006548+0000","last_active":"2023-10-09T10:58:30.585755+0000","last_peered":"2023-10-09T10:58:30.585755+0000","last_clean":"2023-10-09T10:58:30.585755+0000","last_became_active":"2023-10-09T10:57:22.006030+0000","last_became_peered":"2023-10-09T10:57:22.006030+0000","last_unstale":"2023-10-09T10:58:30.585755+0000","last_undegraded":"2023-10-09T10:58:30.585755+0000","last_fullsized":"2023-10-09T10:58:30.585755+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5611,"log_dups_size":0,"ondisk_log_size":5611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:47.539524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367722496,"num_objects":5611,"num_object_clones":0,"num_object_copies":16833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5611,"num_write_kb":359104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":777,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.084268+0000","last_change":"2023-10-09T10:57:25.106977+0000","last_active":"2023-10-09T10:58:25.084268+0000","last_peered":"2023-10-09T10:58:25.084268+0000","last_clean":"2023-10-09T10:58:25.084268+0000","last_became_active":"2023-10-09T10:57:25.106632+0000","last_became_peered":"2023-10-09T10:57:25.106632+0000","last_unstale":"2023-10-09T10:58:25.084268+0000","last_undegraded":"2023-10-09T10:58:25.084268+0000","last_fullsized":"2023-10-09T10:58:25.084268+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:29:20.989344+0000","scrub_duration":0.0053707060000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.093302+0000","last_change":"2023-10-09T10:57:22.016009+0000","last_active":"2023-10-09T10:58:25.093302+0000","last_peered":"2023-10-09T10:58:25.093302+0000","last_clean":"2023-10-09T10:58:25.093302+0000","last_became_active":"2023-10-09T10:57:22.015633+0000","last_became_peered":"2023-10-09T10:57:22.015633+0000","last_unstale":"2023-10-09T10:58:25.093302+0000","last_undegraded":"2023-10-09T10:58:25.093302+0000","last_fullsized":"2023-10-09T10:58:25.093302+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.957416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":279,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.087626+0000","last_change":"2023-10-09T10:54:48.016159+0000","last_active":"2023-10-09T10:58:24.087626+0000","last_peered":"2023-10-09T10:58:24.087626+0000","last_clean":"2023-10-09T10:58:24.087626+0000","last_became_active":"2023-10-09T10:54:47.939409+0000","last_became_peered":"2023-10-09T10:54:47.939409+0000","last_unstale":"2023-10-09T10:58:24.087626+0000","last_undegraded":"2023-10-09T10:58:24.087626+0000","last_fullsized":"2023-10-09T10:58:24.087626+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:11:15.175207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":292,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.099166+0000","last_change":"2023-10-09T10:51:50.371282+0000","last_active":"2023-10-09T10:58:25.099166+0000","last_peered":"2023-10-09T10:58:25.099166+0000","last_clean":"2023-10-09T10:58:25.099166+0000","last_became_active":"2023-10-09T10:51:50.370973+0000","last_became_peered":"2023-10-09T10:51:50.370973+0000","last_unstale":"2023-10-09T10:58:25.099166+0000","last_undegraded":"2023-10-09T10:58:25.099166+0000","last_fullsized":"2023-10-09T10:58:25.099166+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:15.603164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":743,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079133+0000","last_change":"2023-10-09T10:53:29.568199+0000","last_active":"2023-10-09T10:58:25.079133+0000","last_peered":"2023-10-09T10:58:25.079133+0000","last_clean":"2023-10-09T10:58:25.079133+0000","last_became_active":"2023-10-09T10:52:55.416357+0000","last_became_peered":"2023-10-09T10:52:55.416357+0000","last_unstale":"2023-10-09T10:58:25.079133+0000","last_undegraded":"2023-10-09T10:58:25.079133+0000","last_fullsized":"2023-10-09T10:58:25.079133+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:44.364650+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":276,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.086450+0000","last_change":"2023-10-09T10:57:22.005039+0000","last_active":"2023-10-09T10:58:24.086450+0000","last_peered":"2023-10-09T10:58:24.086450+0000","last_clean":"2023-10-09T10:58:24.086450+0000","last_became_active":"2023-10-09T10:57:22.004497+0000","last_became_peered":"2023-10-09T10:57:22.004497+0000","last_unstale":"2023-10-09T10:58:24.086450+0000","last_undegraded":"2023-10-09T10:58:24.086450+0000","last_fullsized":"2023-10-09T10:58:24.086450+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:11.006259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":90,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.090195+0000","last_change":"2023-10-09T10:54:47.944380+0000","last_active":"2023-10-09T10:58:25.090195+0000","last_peered":"2023-10-09T10:58:25.090195+0000","last_clean":"2023-10-09T10:58:25.090195+0000","last_became_active":"2023-10-09T10:54:47.943766+0000","last_became_peered":"2023-10-09T10:54:47.943766+0000","last_unstale":"2023-10-09T10:58:25.090195+0000","last_undegraded":"2023-10-09T10:58:25.090195+0000","last_fullsized":"2023-10-09T10:58:25.090195+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:16:11.869603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":708,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.080303+0000","last_change":"2023-10-09T10:57:25.111170+0000","last_active":"2023-10-09T10:58:25.080303+0000","last_peered":"2023-10-09T10:58:25.080303+0000","last_clean":"2023-10-09T10:58:25.080303+0000","last_became_active":"2023-10-09T10:57:25.110405+0000","last_became_peered":"2023-10-09T10:57:25.110405+0000","last_unstale":"2023-10-09T10:58:25.080303+0000","last_undegraded":"2023-10-09T10:58:25.080303+0000","last_fullsized":"2023-10-09T10:58:25.080303+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:00.112600+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":177,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.102100+0000","last_change":"2023-10-09T10:54:47.923386+0000","last_active":"2023-10-09T10:58:25.102100+0000","last_peered":"2023-10-09T10:58:25.102100+0000","last_clean":"2023-10-09T10:58:25.102100+0000","last_became_active":"2023-10-09T10:54:47.914257+0000","last_became_peered":"2023-10-09T10:54:47.914257+0000","last_unstale":"2023-10-09T10:58:25.102100+0000","last_undegraded":"2023-10-09T10:58:25.102100+0000","last_fullsized":"2023-10-09T10:58:25.102100+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:24:18.190715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":378,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079795+0000","last_change":"2023-10-09T10:57:46.844827+0000","last_active":"2023-10-09T10:58:25.079795+0000","last_peered":"2023-10-09T10:58:25.079795+0000","last_clean":"2023-10-09T10:58:25.079795+0000","last_became_active":"2023-10-09T10:57:46.844584+0000","last_became_peered":"2023-10-09T10:57:46.844584+0000","last_unstale":"2023-10-09T10:58:25.079795+0000","last_undegraded":"2023-10-09T10:58:25.079795+0000","last_fullsized":"2023-10-09T10:58:25.079795+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:54.499813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":301,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.088439+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:58:25.088439+0000","last_peered":"2023-10-09T10:58:25.088439+0000","last_clean":"2023-10-09T10:58:25.088439+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:58:25.088439+0000","last_undegraded":"2023-10-09T10:58:25.088439+0000","last_fullsized":"2023-10-09T10:58:25.088439+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":296,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079951+0000","last_change":"2023-10-09T10:50:43.385150+0000","last_active":"2023-10-09T10:58:25.079951+0000","last_peered":"2023-10-09T10:58:25.079951+0000","last_clean":"2023-10-09T10:58:25.079951+0000","last_became_active":"2023-10-09T10:50:43.384788+0000","last_became_peered":"2023-10-09T10:50:43.384788+0000","last_unstale":"2023-10-09T10:58:25.079951+0000","last_undegraded":"2023-10-09T10:58:25.079951+0000","last_fullsized":"2023-10-09T10:58:25.079951+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:36.085910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":642,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.093386+0000","last_change":"2023-10-09T10:57:47.307757+0000","last_active":"2023-10-09T10:58:25.093386+0000","last_peered":"2023-10-09T10:58:25.093386+0000","last_clean":"2023-10-09T10:58:25.093386+0000","last_became_active":"2023-10-09T10:57:25.104921+0000","last_became_peered":"2023-10-09T10:57:25.104921+0000","last_unstale":"2023-10-09T10:58:25.093386+0000","last_undegraded":"2023-10-09T10:58:25.093386+0000","last_fullsized":"2023-10-09T10:58:25.093386+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:47.307693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:47.307693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:40.935036+0000","scrub_duration":0.003001985,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":376,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.080471+0000","last_change":"2023-10-09T10:54:17.886158+0000","last_active":"2023-10-09T10:58:25.080471+0000","last_peered":"2023-10-09T10:58:25.080471+0000","last_clean":"2023-10-09T10:58:25.080471+0000","last_became_active":"2023-10-09T10:54:17.885816+0000","last_became_peered":"2023-10-09T10:54:17.885816+0000","last_unstale":"2023-10-09T10:58:25.080471+0000","last_undegraded":"2023-10-09T10:58:25.080471+0000","last_fullsized":"2023-10-09T10:58:25.080471+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:05.270825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":379,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079447+0000","last_change":"2023-10-09T10:54:17.957784+0000","last_active":"2023-10-09T10:58:25.079447+0000","last_peered":"2023-10-09T10:58:25.079447+0000","last_clean":"2023-10-09T10:58:25.079447+0000","last_became_active":"2023-10-09T10:54:17.954788+0000","last_became_peered":"2023-10-09T10:54:17.954788+0000","last_unstale":"2023-10-09T10:58:25.079447+0000","last_undegraded":"2023-10-09T10:58:25.079447+0000","last_fullsized":"2023-10-09T10:58:25.079447+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:48.709390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":637,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.094695+0000","last_change":"2023-10-09T10:57:21.022980+0000","last_active":"2023-10-09T10:58:24.094695+0000","last_peered":"2023-10-09T10:58:24.094695+0000","last_clean":"2023-10-09T10:58:24.094695+0000","last_became_active":"2023-10-09T10:54:50.914005+0000","last_became_peered":"2023-10-09T10:54:50.914005+0000","last_unstale":"2023-10-09T10:58:24.094695+0000","last_undegraded":"2023-10-09T10:58:24.094695+0000","last_fullsized":"2023-10-09T10:58:24.094695+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:21.022863+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:21.022863+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:49.534216+0000","scrub_duration":0.00136929,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":259,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T10:58:24.088079+0000","last_change":"2023-10-09T10:54:18.089053+0000","last_active":"2023-10-09T10:58:24.088079+0000","last_peered":"2023-10-09T10:58:24.088079+0000","last_clean":"2023-10-09T10:58:24.088079+0000","last_became_active":"2023-10-09T10:54:17.886070+0000","last_became_peered":"2023-10-09T10:54:17.886070+0000","last_unstale":"2023-10-09T10:58:24.088079+0000","last_undegraded":"2023-10-09T10:58:24.088079+0000","last_fullsized":"2023-10-09T10:58:24.088079+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:35.628407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":355,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.098955+0000","last_change":"2023-10-09T10:54:47.908059+0000","last_active":"2023-10-09T10:58:25.098955+0000","last_peered":"2023-10-09T10:58:25.098955+0000","last_clean":"2023-10-09T10:58:25.098955+0000","last_became_active":"2023-10-09T10:54:47.907780+0000","last_became_peered":"2023-10-09T10:54:47.907780+0000","last_unstale":"2023-10-09T10:58:25.098955+0000","last_undegraded":"2023-10-09T10:58:25.098955+0000","last_fullsized":"2023-10-09T10:58:25.098955+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:55:59.064617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":16490,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5817499648,"num_objects":88768,"num_object_clones":0,"num_object_copies":266304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":88768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88768,"num_write_kb":5681152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17585668096,"data_stored":17585668096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88668,"ondisk_log_size":88668,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332383,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3765888,"kb_used_data":3460408,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":89979264,"statfs":{"total":95995035648,"available":92138766336,"internally_reserved":0,"allocated":3543457792,"data_stored":3542254167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,3,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:03 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.821,"15min":1.4870000000000001},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":4.5190000000000001,"5min":9.6259999999999994,"15min":9.6379999999999999},"last":8.9730000000000008},{"interface":"front","average":{"1min":1.1499999999999999,"5min":2.004,"15min":1.7030000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.4470000000000001,"5min":8.7769999999999992,"15min":16.731000000000002},"last":6.2489999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:58:10 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":1.835,"15min":1.827},"min":{"1min":0.26500000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.2880000000000003,"5min":12.252000000000001,"15min":34.756},"last":3.3799999999999999},{"interface":"front","average":{"1min":1.177,"5min":1.744,"15min":1.609},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":2.4649999999999999,"5min":13.647,"15min":35.658999999999999},"last":2.6680000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:57:35 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":1.538,"15min":1.5469999999999999},"min":{"1min":0.35199999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.0999999999999996,"5min":9.202,"15min":11.058},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.8899999999999999,"5min":1.746,"15min":1.4430000000000001},"min":{"1min":0.28899999999999998,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":13.037000000000001,"5min":13.037000000000001,"15min":13.037000000000001},"last":8.9280000000000008}]},{"osd":3,"last update":"Mon Oct 9 10:58:03 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":2.1080000000000001,"15min":1.913},"min":{"1min":0.33200000000000002,"5min":0.31,"15min":0.31},"max":{"1min":7.1970000000000001,"5min":11.760999999999999,"15min":11.760999999999999},"last":5.6500000000000004},{"interface":"front","average":{"1min":0.89900000000000002,"5min":1.5489999999999999,"15min":1.627},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":2.2040000000000002,"5min":8.6300000000000008,"15min":19.234999999999999},"last":0.54600000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:57:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.92200000000000004,"15min":1.2589999999999999},"min":{"1min":0.27900000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.0170000000000003,"5min":8.2940000000000005,"15min":16.695},"last":3.2930000000000001},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.79900000000000004,"15min":1.137},"min":{"1min":0.30299999999999999,"5min":0.28199999999999997,"15min":0.26200000000000001},"max":{"1min":2.222,"5min":7.9320000000000004,"15min":11.109999999999999},"last":2.323}]},{"osd":5,"last update":"Mon Oct 9 10:57:41 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.77900000000000003,"15min":1.1779999999999999},"min":{"1min":0.307,"5min":0.247,"15min":0.247},"max":{"1min":2.181,"5min":4.5999999999999996,"15min":13.262},"last":2.6309999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.81899999999999995,"15min":1.343},"min":{"1min":0.32700000000000001,"5min":0.318,"15min":0.29599999999999999},"max":{"1min":1.173,"5min":8.4339999999999993,"15min":11.741},"last":3.4950000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:57:41 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.98099999999999998,"15min":1.462},"min":{"1min":0.34599999999999997,"5min":0.315,"15min":0.28199999999999997},"max":{"1min":1.6459999999999999,"5min":8.3610000000000007,"15min":14.913},"last":3.4100000000000001},{"interface":"front","average":{"1min":0.747,"5min":0.91600000000000004,"15min":1.319},"min":{"1min":0.38300000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.2549999999999999,"5min":7.9809999999999999,"15min":19.187999999999999},"last":0.57699999999999996}]}]},{"osd":6,"up_from":44,"seq":188978561314,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2166960,"kb_used_data":1262696,"kb_used_omap":0,"kb_used_meta":904256,"kb_avail":91578192,"statfs":{"total":95995035648,"available":93776068608,"internally_reserved":0,"allocated":1293000704,"data_stored":1291812773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925958144},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:00 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":2.1469999999999998,"15min":1.6120000000000001},"min":{"1min":0.27500000000000002,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":5.0019999999999998,"5min":54.743000000000002,"15min":54.743000000000002},"last":0.375},{"interface":"front","average":{"1min":1.2370000000000001,"5min":2.0110000000000001,"15min":1.6359999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":4.9119999999999999,"5min":53.701000000000001,"15min":53.701000000000001},"last":0.29499999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:58:14 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":1.474,"15min":1.4770000000000001},"min":{"1min":0.44,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.96,"5min":7.6820000000000004,"15min":16.747},"last":0.255},{"interface":"front","average":{"1min":1.869,"5min":1.5229999999999999,"15min":1.534},"min":{"1min":0.376,"5min":0.26400000000000001,"15min":0.23100000000000001},"max":{"1min":8.6419999999999995,"5min":8.6419999999999995,"15min":16.734000000000002},"last":2.9940000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":1.6459999999999999,"15min":1.5660000000000001},"min":{"1min":0.34699999999999998,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":12.003,"5min":12.003,"15min":13.208},"last":3.0369999999999999},{"interface":"front","average":{"1min":0.79300000000000004,"5min":1.546,"15min":1.651},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28399999999999997},"max":{"1min":5.0199999999999996,"5min":12.83,"15min":12.83},"last":1.8260000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:58:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3819999999999999,"5min":1.736,"15min":1.629},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.255},"max":{"1min":5.2969999999999997,"5min":7.8680000000000003,"15min":12.391},"last":3.3279999999999998},{"interface":"front","average":{"1min":1.738,"5min":1.984,"15min":1.895},"min":{"1min":0.40000000000000002,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":5.3949999999999996,"5min":8.782,"15min":13.179},"last":3.0640000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.80200000000000005,"15min":1.171},"min":{"1min":0.26000000000000001,"5min":0.24199999999999999,"15min":0.23200000000000001},"max":{"1min":0.66400000000000003,"5min":16.978999999999999,"15min":16.978999999999999},"last":0.62},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.81799999999999995,"15min":1.1459999999999999},"min":{"1min":0.249,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.0830000000000002,"5min":4.7930000000000001,"15min":12.481999999999999},"last":0.41199999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.212,"15min":1.331},"min":{"1min":0.307,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.7279999999999998,"5min":12.923,"15min":12.923},"last":0.434},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.92700000000000005,"15min":1.2330000000000001},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.0559999999999992,"5min":9.0559999999999992,"15min":12.456},"last":0.60499999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:57:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.0329999999999999,"15min":1.3480000000000001},"min":{"1min":0.35999999999999999,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":1.1930000000000001,"5min":12.882,"15min":23.358000000000001},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.92000000000000004,"15min":1.147},"min":{"1min":0.36099999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.141,"5min":6.1029999999999998,"15min":12.422000000000001},"last":0.69499999999999995}]}]},{"osd":1,"up_from":85,"seq":365072220352,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3606552,"kb_used_data":3107536,"kb_used_omap":9,"kb_used_meta":498998,"kb_avail":90138600,"statfs":{"total":95995035648,"available":92301926400,"internally_reserved":0,"allocated":3182116864,"data_stored":3180303856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":510974953},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":2.464,"15min":2.0099999999999998},"min":{"1min":0.224,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":4.3049999999999997,"5min":19.992999999999999,"15min":19.992999999999999},"last":3.6219999999999999},{"interface":"front","average":{"1min":2.4369999999999998,"5min":2.9550000000000001,"15min":2.1899999999999999},"min":{"1min":0.27000000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":6.9790000000000001,"5min":18.303000000000001,"15min":18.303000000000001},"last":3.891}]},{"osd":2,"last update":"Mon Oct 9 10:57:37 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":2.2320000000000002,"15min":1.641},"min":{"1min":0.23899999999999999,"5min":0.223,"15min":0.19700000000000001},"max":{"1min":12.587,"5min":14.878,"15min":15.356},"last":1.4490000000000001},{"interface":"front","average":{"1min":2.2240000000000002,"5min":2.109,"15min":1.7749999999999999},"min":{"1min":0.22600000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":8.343,"5min":18.004000000000001,"15min":18.004000000000001},"last":0.318}]},{"osd":3,"last update":"Mon Oct 9 10:58:00 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":2.605,"15min":2.23},"min":{"1min":0.23100000000000001,"5min":0.22900000000000001,"15min":0.218},"max":{"1min":8.2080000000000002,"5min":20.036000000000001,"15min":20.036000000000001},"last":0.26600000000000001},{"interface":"front","average":{"1min":2.827,"5min":2.9809999999999999,"15min":2.2829999999999999},"min":{"1min":0.24099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":8.8680000000000003,"5min":19.640999999999998,"15min":19.640999999999998},"last":2.2949999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":1.5009999999999999,"15min":1.468},"min":{"1min":0.30099999999999999,"5min":0.28899999999999998,"15min":0.255},"max":{"1min":1.71,"5min":14.763,"15min":14.763},"last":0.441},{"interface":"front","average":{"1min":1.4099999999999999,"5min":1.75,"15min":1.7190000000000001},"min":{"1min":0.379,"5min":0.30399999999999999,"15min":0.27200000000000002},"max":{"1min":4.0199999999999996,"5min":8.3079999999999998,"15min":14.228999999999999},"last":0.34699999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":1.9670000000000001,"15min":1.631},"min":{"1min":0.376,"5min":0.32700000000000001,"15min":0.253},"max":{"1min":5.9989999999999997,"5min":9.6289999999999996,"15min":9.7720000000000002},"last":3.8519999999999999},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.873,"15min":1.8380000000000001},"min":{"1min":0.313,"5min":0.30299999999999999,"15min":0.28899999999999998},"max":{"1min":2.1930000000000001,"5min":14.833,"15min":17.085000000000001},"last":0.93500000000000005}]},{"osd":6,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.8640000000000001,"15min":1.716},"min":{"1min":0.41799999999999998,"5min":0.32600000000000001,"15min":0.27400000000000002},"max":{"1min":5.9630000000000001,"5min":8.8230000000000004,"15min":10.425000000000001},"last":2.4079999999999999},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.653,"15min":1.6479999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.312},"max":{"1min":1.74,"5min":14.804,"15min":14.804},"last":0.497}]},{"osd":7,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.8180000000000001,"15min":1.7150000000000001},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.34000000000000002},"max":{"1min":2.5619999999999998,"5min":14.856,"15min":16.312000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.2829999999999999,"5min":1.6839999999999999,"15min":1.7130000000000001},"min":{"1min":0.41099999999999998,"5min":0.35499999999999998,"15min":0.32500000000000001},"max":{"1min":2.8940000000000001,"5min":8.8149999999999995,"15min":14.273999999999999},"last":0.63200000000000001}]}]},{"osd":0,"up_from":80,"seq":343597383886,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1926752,"kb_used_data":1393816,"kb_used_omap":28,"kb_used_meta":532899,"kb_avail":91818400,"statfs":{"total":95995035648,"available":94022041600,"internally_reserved":0,"allocated":1427267584,"data_stored":1425383768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":545689424},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:58:03 2023","interfaces":[{"interface":"back","average":{"1min":2.254,"5min":2.4079999999999999,"15min":2.0310000000000001},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":12.301,"5min":13.263999999999999,"15min":25.475000000000001},"last":1.129},{"interface":"front","average":{"1min":1.8240000000000001,"5min":2.0790000000000002,"15min":1.9339999999999999},"min":{"1min":0.20200000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":6.1680000000000001,"5min":10.478999999999999,"15min":20.832999999999998},"last":0.78000000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:57:37 2023","interfaces":[{"interface":"back","average":{"1min":2.6030000000000002,"5min":2.5409999999999999,"15min":1.9610000000000001},"min":{"1min":0.23599999999999999,"5min":0.189,"15min":0.189},"max":{"1min":7.6420000000000003,"5min":24.852,"15min":24.852},"last":4.6299999999999999},{"interface":"front","average":{"1min":2.8250000000000002,"5min":3.4740000000000002,"15min":2.3849999999999998},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":12.672000000000001,"5min":77.256,"15min":77.256},"last":1.6279999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:58:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":2.1709999999999998,"15min":2.1509999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20200000000000001},"max":{"1min":8.3510000000000009,"5min":18.972000000000001,"15min":18.972000000000001},"last":3.5459999999999998},{"interface":"front","average":{"1min":2.2160000000000002,"5min":2.8140000000000001,"15min":2.298},"min":{"1min":0.27600000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":12.353999999999999,"5min":13.305,"15min":14.391999999999999},"last":4.6870000000000003}]},{"osd":4,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":2.8839999999999999,"15min":2.1019999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.26200000000000001},"max":{"1min":6.133,"5min":77.177999999999997,"15min":77.177999999999997},"last":1.5680000000000001},{"interface":"front","average":{"1min":1.532,"5min":1.679,"15min":1.8799999999999999},"min":{"1min":0.32600000000000001,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":6.843,"5min":10.513999999999999,"15min":12.318},"last":0.38300000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":1.766,"15min":1.8999999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.29799999999999999},"max":{"1min":6.9100000000000001,"5min":10.544,"15min":12.157},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.3740000000000001,"5min":1.76,"15min":1.796},"min":{"1min":0.39400000000000002,"5min":0.29899999999999999,"15min":0.29499999999999998},"max":{"1min":6.7469999999999999,"5min":24.754999999999999,"15min":24.754999999999999},"last":4.6639999999999997}]},{"osd":6,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.96,"15min":2.0329999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.30399999999999999},"max":{"1min":6.8840000000000003,"5min":18.920000000000002,"15min":18.920000000000002},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.47,"5min":2,"15min":1.714},"min":{"1min":0.38,"5min":0.32900000000000001,"15min":0.307},"max":{"1min":6.7130000000000001,"5min":24.800000000000001,"15min":24.800000000000001},"last":0.441}]},{"osd":7,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.8839999999999999,"15min":1.909},"min":{"1min":0.44900000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.6609999999999996,"5min":13.225,"15min":14.48},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.7470000000000001,"5min":2.8479999999999999,"15min":2.214},"min":{"1min":0.38,"5min":0.314,"15min":0.314},"max":{"1min":6.101,"5min":77.228999999999999,"15min":77.228999999999999},"last":1.603}]}]},{"osd":2,"up_from":90,"seq":386547056827,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1480884,"kb_used_data":1077164,"kb_used_omap":81,"kb_used_meta":403630,"kb_avail":92264268,"statfs":{"total":95995035648,"available":94478610432,"internally_reserved":0,"allocated":1103015936,"data_stored":1101296894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":413317338},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":1.8939999999999999,"15min":1.659},"min":{"1min":0.21099999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":5.6280000000000001,"5min":17.574000000000002,"15min":18.512},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.02,"5min":1.927,"15min":1.762},"min":{"1min":0.255,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":3.754,"5min":15.595000000000001,"15min":17.975000000000001},"last":1.6279999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.6799999999999999,"15min":1.702},"min":{"1min":0.317,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":3.7250000000000001,"5min":16.359000000000002,"15min":16.359000000000002},"last":1.6000000000000001},{"interface":"front","average":{"1min":2.036,"5min":1.855,"15min":1.5669999999999999},"min":{"1min":0.246,"5min":0.24199999999999999,"15min":0.20699999999999999},"max":{"1min":6.3659999999999997,"5min":12.693,"15min":16.454999999999998},"last":3.0209999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:58:00 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":2.355,"15min":1.839},"min":{"1min":0.24399999999999999,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":3.6349999999999998,"5min":19.539000000000001,"15min":19.539000000000001},"last":2.6779999999999999},{"interface":"front","average":{"1min":2.2410000000000001,"5min":2.4169999999999998,"15min":2.1499999999999999},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.20200000000000001},"max":{"1min":8.8729999999999993,"5min":16.702999999999999,"15min":16.702999999999999},"last":1.8899999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.9610000000000001,"5min":2.0099999999999998,"15min":1.8109999999999999},"min":{"1min":0.372,"5min":0.28599999999999998,"15min":0.254},"max":{"1min":6.407,"5min":15.672000000000001,"15min":16.231999999999999},"last":3.0720000000000001},{"interface":"front","average":{"1min":1.425,"5min":1.7589999999999999,"15min":1.7130000000000001},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.26300000000000001},"max":{"1min":5.6680000000000001,"5min":15.696999999999999,"15min":15.696999999999999},"last":1.2969999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":1.46,"15min":1.6399999999999999},"min":{"1min":0.375,"5min":0.26800000000000002,"15min":0.245},"max":{"1min":2.7850000000000001,"5min":12.128,"15min":17.920000000000002},"last":1.615},{"interface":"front","average":{"1min":1.9790000000000001,"5min":2.0569999999999999,"15min":1.7569999999999999},"min":{"1min":0.33900000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.4320000000000004,"5min":16.597000000000001,"15min":17.882999999999999},"last":3.0960000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":1.4950000000000001,"15min":1.5009999999999999},"min":{"1min":0.39500000000000002,"5min":0.33400000000000002,"15min":0.247},"max":{"1min":5.6929999999999996,"5min":8.8460000000000001,"15min":14.029999999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.83099999999999996,"5min":1.4550000000000001,"15min":1.605},"min":{"1min":0.36799999999999999,"5min":0.34200000000000003,"15min":0.29199999999999998},"max":{"1min":2.4889999999999999,"5min":10.111000000000001,"15min":12.862},"last":1.651}]},{"osd":7,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.375,"5min":1.5149999999999999,"15min":1.5229999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.32500000000000001},"max":{"1min":6.0860000000000003,"5min":8.8879999999999999,"15min":13.999000000000001},"last":2.7109999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":1.5609999999999999,"15min":1.6819999999999999},"min":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.307},"max":{"1min":1.1839999999999999,"5min":10.157999999999999,"15min":12.826000000000001},"last":4.2839999999999998}]}]},{"osd":3,"up_from":95,"seq":408021893301,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3618048,"kb_used_data":2873144,"kb_used_omap":7,"kb_used_meta":744888,"kb_avail":90127104,"statfs":{"total":95995035648,"available":92290154496,"internally_reserved":0,"allocated":2942099456,"data_stored":2940191199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":762765928},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":3.399,"15min":2.2930000000000001},"min":{"1min":0.23499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":10.140000000000001,"5min":38.817999999999998,"15min":38.817999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":1.776,"5min":3.661,"15min":2.5840000000000001},"min":{"1min":0.16500000000000001,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":7.6580000000000004,"5min":17.312000000000001,"15min":17.312000000000001},"last":0.42999999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":2.5990000000000002,"15min":2.3530000000000002},"min":{"1min":0.28000000000000003,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":13.244999999999999,"5min":22.469000000000001,"15min":22.469000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.52,"5min":3.4020000000000001,"15min":2.4409999999999998},"min":{"1min":0.28999999999999998,"5min":0.22,"15min":0.193},"max":{"1min":13.359,"5min":18.526,"15min":18.526},"last":0.34200000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":2.7559999999999998,"15min":2.214},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.6260000000000003,"5min":20.402000000000001,"15min":20.402000000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.3680000000000001,"5min":3.637,"15min":2.4900000000000002},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":4.4320000000000004,"5min":27.538,"15min":27.538},"last":0.64100000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":2.4180000000000001,"15min":1.992},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.8089999999999993,"5min":11.863,"15min":20.388000000000002},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.22,"5min":2.2490000000000001,"15min":1.9239999999999999},"min":{"1min":0.34000000000000002,"5min":0.28399999999999997,"15min":0.24299999999999999},"max":{"1min":4.0529999999999999,"5min":11.680999999999999,"15min":19.085999999999999},"last":0.55300000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":2.3290000000000002,"15min":2.2029999999999998},"min":{"1min":0.39300000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.0600000000000001,"5min":14.045,"15min":51.509999999999998},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.8839999999999999,"15min":2.1680000000000001},"min":{"1min":0.41899999999999998,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":4.3780000000000001,"5min":17.311,"15min":55.338999999999999},"last":0.51900000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":2.335,"15min":1.929},"min":{"1min":0.35599999999999998,"5min":0.29199999999999998,"15min":0.28000000000000003},"max":{"1min":3.1120000000000001,"5min":11.723000000000001,"15min":12.361000000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.306,"5min":2.036,"15min":2.0819999999999999},"min":{"1min":0.39500000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.5229999999999997,"5min":17.295000000000002,"15min":17.295000000000002},"last":0.48799999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":2.2269999999999999,"15min":2.3140000000000001},"min":{"1min":0.439,"5min":0.34999999999999998,"15min":0.31},"max":{"1min":4.3330000000000002,"5min":18.722000000000001,"15min":51.744},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.5609999999999999,"5min":2.5259999999999998,"15min":2.335},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.8569999999999993,"5min":13.337,"15min":53.399000000000001},"last":0.374}]}]},{"osd":4,"up_from":30,"seq":128849019177,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1994300,"kb_used_data":1097460,"kb_used_omap":0,"kb_used_meta":896832,"kb_avail":91750852,"statfs":{"total":95995035648,"available":93952872448,"internally_reserved":0,"allocated":1123799040,"data_stored":1122599220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918355968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:57:55 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":4.0410000000000004,"15min":2.202},"min":{"1min":0.224,"5min":0.224,"15min":0.20999999999999999},"max":{"1min":4.2629999999999999,"5min":213.88999999999999,"15min":213.88999999999999},"last":4.9900000000000002},{"interface":"front","average":{"1min":0.91800000000000004,"5min":3.9580000000000002,"15min":2.4980000000000002},"min":{"1min":0.26100000000000001,"5min":0.23200000000000001,"15min":0.217},"max":{"1min":3.012,"5min":214.09800000000001,"15min":214.09800000000001},"last":1.944}]},{"osd":1,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":3.7559999999999998,"15min":2.2149999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":3.286,"5min":206.50399999999999,"15min":206.50399999999999},"last":5.101},{"interface":"front","average":{"1min":0.99099999999999999,"5min":3.7839999999999998,"15min":2.419},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.2959999999999998,"5min":206.529,"15min":206.529},"last":4.0359999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:57:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.7450000000000001,"15min":1.653},"min":{"1min":0.28000000000000003,"5min":0.25900000000000001,"15min":0.253},"max":{"1min":5.0759999999999996,"5min":15.446999999999999,"15min":15.446999999999999},"last":5.3929999999999998},{"interface":"front","average":{"1min":1.4570000000000001,"5min":2.2240000000000002,"15min":1.911},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.26300000000000001},"max":{"1min":5.9210000000000003,"5min":27.818000000000001,"15min":27.818000000000001},"last":5.3659999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:57:55 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":3.8559999999999999,"15min":2.4500000000000002},"min":{"1min":0.374,"5min":0.34599999999999997,"15min":0.27400000000000002},"max":{"1min":2.7789999999999999,"5min":209.03700000000001,"15min":209.03700000000001},"last":1.958},{"interface":"front","average":{"1min":1.3580000000000001,"5min":4.1760000000000002,"15min":2.6930000000000001},"min":{"1min":0.39600000000000002,"5min":0.30499999999999999,"15min":0.29999999999999999},"max":{"1min":5.1529999999999996,"5min":208.80199999999999,"15min":208.80199999999999},"last":5.4119999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":1.1120000000000001,"15min":1.577},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.224},"max":{"1min":0.77800000000000002,"5min":19.068999999999999,"15min":24.149999999999999},"last":1.3049999999999999},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.05,"15min":1.3979999999999999},"min":{"1min":0.28499999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.8360000000000001,"5min":15.535,"15min":20.422000000000001},"last":5.3319999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:57:35 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.93799999999999994,"15min":1.2370000000000001},"min":{"1min":0.31900000000000001,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":1,"5min":15.739000000000001,"15min":15.739000000000001},"last":4.944},{"interface":"front","average":{"1min":0.50900000000000001,"5min":1.1060000000000001,"15min":1.466},"min":{"1min":0.35299999999999998,"5min":0.32700000000000001,"15min":0.311},"max":{"1min":0.876,"5min":14.093999999999999,"15min":14.272},"last":3.9710000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:57:42 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":1.1719999999999999,"15min":1.393},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28299999999999997},"max":{"1min":1.355,"5min":14.458,"15min":14.458},"last":4.0030000000000001},{"interface":"front","average":{"1min":0.68500000000000005,"5min":1.3360000000000001,"15min":1.304},"min":{"1min":0.35799999999999998,"5min":0.33800000000000002,"15min":0.255},"max":{"1min":1.8899999999999999,"5min":15.592000000000001,"15min":15.592000000000001},"last":2.2549999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790246,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4239468,"kb_used_data":3324452,"kb_used_omap":0,"kb_used_meta":915008,"kb_avail":89505684,"statfs":{"total":95995035648,"available":91653820416,"internally_reserved":0,"allocated":3404238848,"data_stored":3403051377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":936968192},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.927,"15min":1.653},"min":{"1min":0.248,"5min":0.215,"15min":0.215},"max":{"1min":3.9590000000000001,"5min":16.449000000000002,"15min":16.449000000000002},"last":1.2270000000000001},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.9770000000000001,"15min":1.7090000000000001},"min":{"1min":0.30499999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.984,"5min":16.489000000000001,"15min":16.489000000000001},"last":1.26}]},{"osd":1,"last update":"Mon Oct 9 10:58:15 2023","interfaces":[{"interface":"back","average":{"1min":4.2359999999999998,"5min":2.5880000000000001,"15min":2.1019999999999999},"min":{"1min":0.313,"5min":0.215,"15min":0.215},"max":{"1min":29.818999999999999,"5min":29.818999999999999,"15min":29.818999999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":3.9409999999999998,"5min":2.3029999999999999,"15min":1.804},"min":{"1min":0.28199999999999997,"5min":0.253,"15min":0.24299999999999999},"max":{"1min":29.600000000000001,"5min":29.600000000000001,"15min":29.600000000000001},"last":4.407}]},{"osd":2,"last update":"Mon Oct 9 10:57:33 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":2.0059999999999998,"15min":1.829},"min":{"1min":0.33800000000000002,"5min":0.28000000000000003,"15min":0.27800000000000002},"max":{"1min":8.0399999999999991,"5min":11.257999999999999,"15min":11.393000000000001},"last":4.7050000000000001},{"interface":"front","average":{"1min":1.7,"5min":1.976,"15min":1.6439999999999999},"min":{"1min":0.28899999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.093,"5min":9.2569999999999997,"15min":15.087999999999999},"last":1.7490000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:58:02 2023","interfaces":[{"interface":"back","average":{"1min":2.3999999999999999,"5min":2.165,"15min":2.206},"min":{"1min":0.32100000000000001,"5min":0.314,"15min":0.25700000000000001},"max":{"1min":25.846,"5min":25.846,"15min":63.508000000000003},"last":1.613},{"interface":"front","average":{"1min":2.7050000000000001,"5min":2.5169999999999999,"15min":2.3860000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.26300000000000001},"max":{"1min":25.073,"5min":25.073,"15min":62.418999999999997},"last":0.32900000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:58:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.68600000000000005,"15min":1.395},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23200000000000001},"max":{"1min":2.726,"5min":4.4989999999999997,"15min":123.241},"last":1.2909999999999999},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.93200000000000005,"15min":1.6339999999999999},"min":{"1min":0.29199999999999998,"5min":0.28000000000000003,"15min":0.24199999999999999},"max":{"1min":4.8840000000000003,"5min":4.8840000000000003,"15min":123.318},"last":0.36699999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:58:25 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":1.0880000000000001,"15min":1.3520000000000001},"min":{"1min":0.28899999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.1509999999999998,"5min":8.9049999999999994,"15min":11.098000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.79700000000000004,"15min":1.2290000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.1469999999999998,"5min":6.7210000000000001,"15min":20.256},"last":3.6190000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:57:49 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":1.085,"15min":1.3169999999999999},"min":{"1min":0.33900000000000002,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":2.968,"5min":8.468,"15min":10.739000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.82899999999999996,"15min":1.139},"min":{"1min":0.315,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.327,"5min":8.734,"15min":14.154},"last":4.2599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1431961600,"data_stored":1431961600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3188588544,"data_stored":3188588544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1101660160,"data_stored":1101660160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2944532480,"data_stored":2944532480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1121845248,"data_stored":1121845248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3348627456,"data_stored":3348627456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1106116608,"data_stored":1106116608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3342336000,"data_stored":3342336000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:58:34.203 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:58:34.226 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:34.305 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:34.334 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:34.338 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:34 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3391056463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:34.338 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:34 smithi125 ceph-mon[195507]: from='client.35850 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:58:34.338 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:34 smithi125 ceph-mon[195507]: pgmap v873: 119 pgs: 119 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-09T10:58:34.339 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:34 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3391056463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:34.339 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:34 smithi125 ceph-mon[200197]: from='client.35850 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:58:34.339 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:34 smithi125 ceph-mon[200197]: pgmap v873: 119 pgs: 119 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-09T10:58:34.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:34.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:34.353 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:58:34.441 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:34.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:34 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3391056463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:34.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:34 smithi155 ceph-mon[184854]: from='client.35850 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:58:34.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:34 smithi155 ceph-mon[184854]: pgmap v873: 119 pgs: 119 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-09T10:58:34.489 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:34.647 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:34.658 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 70 16 93415 93399 83.337 53.75 0.0154784 0.0119945 2023-10-09T10:58:34.666 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:34.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:34.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:34.773 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:58:34.874 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:34.903 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:58:34.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:34.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:58:35.011 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:35.077 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:35.107 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:35.109 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:58:35.109 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:58:35.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:35.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:35.172 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:35.212 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:35.265 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:35.658 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 71 16 94311 94295 82.9521 56 0.0184576 0.0120503 2023-10-09T10:58:35.955 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:58:36.155 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:36.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:36.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:36.188 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:58:36.272 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:36.336 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:36.372 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:36.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:36.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:36.477 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:36.536 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:36.658 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 72 16 95157 95141 82.5345 52.875 0.02706 0.0121115 2023-10-09T10:58:36.691 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:58:36.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:36 smithi125 ceph-mon[195507]: pgmap v874: 119 pgs: 119 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s 2023-10-09T10:58:36.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:36 smithi125 ceph-mon[200197]: pgmap v874: 119 pgs: 119 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s 2023-10-09T10:58:36.887 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:36 smithi155 ceph-mon[184854]: pgmap v874: 119 pgs: 119 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s 2023-10-09T10:58:37.277 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:58:37.279 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:37.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:37.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:37.296 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:58:37.387 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:37.441 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:37.467 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:37.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:37.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:37.572 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:37.623 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:37.636 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:37 smithi155 ceph-mon[184854]: from='client.55732 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:58:37.658 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 73 16 96045 96029 82.1643 55.5 0.0135589 0.0121659 2023-10-09T10:58:37.924 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:58:37.924 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:58:38.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:37 smithi125 ceph-mon[200197]: from='client.55732 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:58:38.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:37 smithi125 ceph-mon[195507]: from='client.55732 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:58:38.109 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:38.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:38.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:38.219 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:38.267 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:38.304 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:38.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:38.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:38.335 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:58:38.405 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":874,"stamp":"2023-10-09T10:58:35.555154+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5948446624,"num_objects":90781,"num_object_clones":0,"num_object_copies":272343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":90818,"num_write_kb":5808477,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":1067242,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95703,"ondisk_log_size":95703,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23031648,"kb_used_data":17833568,"kb_used_omap":126,"kb_used_meta":5197889,"kb_avail":726929568,"statfs":{"total":767960285184,"available":744375877632,"internally_reserved":0,"allocated":18261573632,"data_stored":18249466711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":5322638971},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,12,4,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":360185856,"num_objects":5496,"num_object_clones":0,"num_object_copies":16488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5496,"num_write_kb":351744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.004689"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":708,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.090705+0000","last_change":"2023-10-09T10:57:25.104150+0000","last_active":"2023-10-09T10:58:25.090705+0000","last_peered":"2023-10-09T10:58:25.090705+0000","last_clean":"2023-10-09T10:58:25.090705+0000","last_became_active":"2023-10-09T10:57:25.103762+0000","last_became_peered":"2023-10-09T10:57:25.103762+0000","last_unstale":"2023-10-09T10:58:25.090705+0000","last_undegraded":"2023-10-09T10:58:25.090705+0000","last_fullsized":"2023-10-09T10:58:25.090705+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:40.174975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:40.174975+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:35.416780+0000","scrub_duration":0.010654205,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":57,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.089696+0000","last_change":"2023-10-09T10:57:11.874010+0000","last_active":"2023-10-09T10:58:25.089696+0000","last_peered":"2023-10-09T10:58:25.089696+0000","last_clean":"2023-10-09T10:58:25.089696+0000","last_became_active":"2023-10-09T10:57:11.873713+0000","last_became_peered":"2023-10-09T10:57:11.873713+0000","last_unstale":"2023-10-09T10:58:25.089696+0000","last_undegraded":"2023-10-09T10:58:25.089696+0000","last_fullsized":"2023-10-09T10:58:25.089696+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:58.049973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":359,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079930+0000","last_change":"2023-10-09T10:54:47.945782+0000","last_active":"2023-10-09T10:58:25.079930+0000","last_peered":"2023-10-09T10:58:25.079930+0000","last_clean":"2023-10-09T10:58:25.079930+0000","last_became_active":"2023-10-09T10:54:47.945460+0000","last_became_peered":"2023-10-09T10:54:47.945460+0000","last_unstale":"2023-10-09T10:58:25.079930+0000","last_undegraded":"2023-10-09T10:58:25.079930+0000","last_fullsized":"2023-10-09T10:58:25.079930+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:50.537469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":734,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.727917+0000","last_change":"2023-10-09T10:53:34.537094+0000","last_active":"2023-10-09T10:58:31.727917+0000","last_peered":"2023-10-09T10:58:31.727917+0000","last_clean":"2023-10-09T10:58:31.727917+0000","last_became_active":"2023-10-09T10:52:55.414995+0000","last_became_peered":"2023-10-09T10:52:55.414995+0000","last_unstale":"2023-10-09T10:58:31.727917+0000","last_undegraded":"2023-10-09T10:58:31.727917+0000","last_fullsized":"2023-10-09T10:58:31.727917+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.536962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.536962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:30.197404+0000","scrub_duration":0.0053325960000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":69,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.668783+0000","last_change":"2023-10-09T10:54:47.944768+0000","last_active":"2023-10-09T10:58:29.668783+0000","last_peered":"2023-10-09T10:58:29.668783+0000","last_clean":"2023-10-09T10:58:29.668783+0000","last_became_active":"2023-10-09T10:54:47.943888+0000","last_became_peered":"2023-10-09T10:54:47.943888+0000","last_unstale":"2023-10-09T10:58:29.668783+0000","last_undegraded":"2023-10-09T10:58:29.668783+0000","last_fullsized":"2023-10-09T10:58:29.668783+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:11.046762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":280,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.679895+0000","last_change":"2023-10-09T10:57:22.007179+0000","last_active":"2023-10-09T10:58:29.679895+0000","last_peered":"2023-10-09T10:58:29.679895+0000","last_clean":"2023-10-09T10:58:29.679895+0000","last_became_active":"2023-10-09T10:57:22.006760+0000","last_became_peered":"2023-10-09T10:57:22.006760+0000","last_unstale":"2023-10-09T10:58:29.679895+0000","last_undegraded":"2023-10-09T10:58:29.679895+0000","last_fullsized":"2023-10-09T10:58:29.679895+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:13.771012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":774,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.099427+0000","last_change":"2023-10-09T10:57:25.103697+0000","last_active":"2023-10-09T10:58:25.099427+0000","last_peered":"2023-10-09T10:58:25.099427+0000","last_clean":"2023-10-09T10:58:25.099427+0000","last_became_active":"2023-10-09T10:57:25.103306+0000","last_became_peered":"2023-10-09T10:57:25.103306+0000","last_unstale":"2023-10-09T10:58:25.099427+0000","last_undegraded":"2023-10-09T10:58:25.099427+0000","last_fullsized":"2023-10-09T10:58:25.099427+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:23:07.530489+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":405,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079322+0000","last_change":"2023-10-09T10:54:48.038948+0000","last_active":"2023-10-09T10:58:25.079322+0000","last_peered":"2023-10-09T10:58:25.079322+0000","last_clean":"2023-10-09T10:58:25.079322+0000","last_became_active":"2023-10-09T10:54:48.001556+0000","last_became_peered":"2023-10-09T10:54:48.001556+0000","last_unstale":"2023-10-09T10:58:25.079322+0000","last_undegraded":"2023-10-09T10:58:25.079322+0000","last_fullsized":"2023-10-09T10:58:25.079322+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:23.415040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.671820+0000","last_change":"2023-10-09T10:54:47.914742+0000","last_active":"2023-10-09T10:58:29.671820+0000","last_peered":"2023-10-09T10:58:29.671820+0000","last_clean":"2023-10-09T10:58:29.671820+0000","last_became_active":"2023-10-09T10:54:47.914551+0000","last_became_peered":"2023-10-09T10:54:47.914551+0000","last_unstale":"2023-10-09T10:58:29.671820+0000","last_undegraded":"2023-10-09T10:58:29.671820+0000","last_fullsized":"2023-10-09T10:58:29.671820+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:16.067994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":766,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092079+0000","last_change":"2023-10-09T10:57:25.098434+0000","last_active":"2023-10-09T10:58:25.092079+0000","last_peered":"2023-10-09T10:58:25.092079+0000","last_clean":"2023-10-09T10:58:25.092079+0000","last_became_active":"2023-10-09T10:57:25.098064+0000","last_became_peered":"2023-10-09T10:57:25.098064+0000","last_unstale":"2023-10-09T10:58:25.092079+0000","last_undegraded":"2023-10-09T10:58:25.092079+0000","last_fullsized":"2023-10-09T10:58:25.092079+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.787782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.787782+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:03.016769+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":225,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.680432+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:58:29.680432+0000","last_peered":"2023-10-09T10:58:29.680432+0000","last_clean":"2023-10-09T10:58:29.680432+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:58:29.680432+0000","last_undegraded":"2023-10-09T10:58:29.680432+0000","last_fullsized":"2023-10-09T10:58:29.680432+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":360,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079318+0000","last_change":"2023-10-09T10:54:48.039275+0000","last_active":"2023-10-09T10:58:25.079318+0000","last_peered":"2023-10-09T10:58:25.079318+0000","last_clean":"2023-10-09T10:58:25.079318+0000","last_became_active":"2023-10-09T10:54:48.017645+0000","last_became_peered":"2023-10-09T10:54:48.017645+0000","last_unstale":"2023-10-09T10:58:25.079318+0000","last_undegraded":"2023-10-09T10:58:25.079318+0000","last_fullsized":"2023-10-09T10:58:25.079318+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:55.986791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":642,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.090080+0000","last_change":"2023-10-09T10:57:25.103739+0000","last_active":"2023-10-09T10:58:25.090080+0000","last_peered":"2023-10-09T10:58:25.090080+0000","last_clean":"2023-10-09T10:58:25.090080+0000","last_became_active":"2023-10-09T10:57:25.103378+0000","last_became_peered":"2023-10-09T10:57:25.103378+0000","last_unstale":"2023-10-09T10:58:25.090080+0000","last_undegraded":"2023-10-09T10:58:25.090080+0000","last_fullsized":"2023-10-09T10:58:25.090080+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:49:50.889945+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":28,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092487+0000","last_change":"2023-10-09T10:57:22.014188+0000","last_active":"2023-10-09T10:58:25.092487+0000","last_peered":"2023-10-09T10:58:25.092487+0000","last_clean":"2023-10-09T10:58:25.092487+0000","last_became_active":"2023-10-09T10:57:22.013975+0000","last_became_peered":"2023-10-09T10:57:22.013975+0000","last_unstale":"2023-10-09T10:58:25.092487+0000","last_undegraded":"2023-10-09T10:58:25.092487+0000","last_fullsized":"2023-10-09T10:58:25.092487+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:58.620741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":317,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079517+0000","last_change":"2023-10-09T10:57:22.010202+0000","last_active":"2023-10-09T10:58:25.079517+0000","last_peered":"2023-10-09T10:58:25.079517+0000","last_clean":"2023-10-09T10:58:25.079517+0000","last_became_active":"2023-10-09T10:57:22.006867+0000","last_became_peered":"2023-10-09T10:57:22.006867+0000","last_unstale":"2023-10-09T10:58:25.079517+0000","last_undegraded":"2023-10-09T10:58:25.079517+0000","last_fullsized":"2023-10-09T10:58:25.079517+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:54.212660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":741,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.680063+0000","last_change":"2023-10-09T10:53:32.540768+0000","last_active":"2023-10-09T10:58:29.680063+0000","last_peered":"2023-10-09T10:58:29.680063+0000","last_clean":"2023-10-09T10:58:29.680063+0000","last_became_active":"2023-10-09T10:52:55.414818+0000","last_became_peered":"2023-10-09T10:52:55.414818+0000","last_unstale":"2023-10-09T10:58:29.680063+0000","last_undegraded":"2023-10-09T10:58:29.680063+0000","last_fullsized":"2023-10-09T10:58:29.680063+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:32.540610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:32.540610+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:47.374383+0000","scrub_duration":0.0064251300000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":34,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092384+0000","last_change":"2023-10-09T10:57:22.032392+0000","last_active":"2023-10-09T10:58:25.092384+0000","last_peered":"2023-10-09T10:58:25.092384+0000","last_clean":"2023-10-09T10:58:25.092384+0000","last_became_active":"2023-10-09T10:57:22.012620+0000","last_became_peered":"2023-10-09T10:57:22.012620+0000","last_unstale":"2023-10-09T10:58:25.092384+0000","last_undegraded":"2023-10-09T10:58:25.092384+0000","last_fullsized":"2023-10-09T10:58:25.092384+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:06:34.979769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":44,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.089855+0000","last_change":"2023-10-09T10:57:46.838152+0000","last_active":"2023-10-09T10:58:25.089855+0000","last_peered":"2023-10-09T10:58:25.089855+0000","last_clean":"2023-10-09T10:58:25.089855+0000","last_became_active":"2023-10-09T10:57:46.837933+0000","last_became_peered":"2023-10-09T10:57:46.837933+0000","last_unstale":"2023-10-09T10:58:25.089855+0000","last_undegraded":"2023-10-09T10:58:25.089855+0000","last_fullsized":"2023-10-09T10:58:25.089855+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:26:23.218941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":738,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.669082+0000","last_change":"2023-10-09T10:57:18.113935+0000","last_active":"2023-10-09T10:58:29.669082+0000","last_peered":"2023-10-09T10:58:29.669082+0000","last_clean":"2023-10-09T10:58:29.669082+0000","last_became_active":"2023-10-09T10:54:50.913330+0000","last_became_peered":"2023-10-09T10:54:50.913330+0000","last_unstale":"2023-10-09T10:58:29.669082+0000","last_undegraded":"2023-10-09T10:58:29.669082+0000","last_fullsized":"2023-10-09T10:58:29.669082+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:18.113848+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.113848+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:51:24.460807+0000","scrub_duration":0.001474106,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":38,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.093297+0000","last_change":"2023-10-09T10:57:22.039995+0000","last_active":"2023-10-09T10:58:25.093297+0000","last_peered":"2023-10-09T10:58:25.093297+0000","last_clean":"2023-10-09T10:58:25.093297+0000","last_became_active":"2023-10-09T10:57:22.016287+0000","last_became_peered":"2023-10-09T10:57:22.016287+0000","last_unstale":"2023-10-09T10:58:25.093297+0000","last_undegraded":"2023-10-09T10:58:25.093297+0000","last_fullsized":"2023-10-09T10:58:25.093297+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:52.167501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":365,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.099381+0000","last_change":"2023-10-09T10:57:46.845755+0000","last_active":"2023-10-09T10:58:25.099381+0000","last_peered":"2023-10-09T10:58:25.099381+0000","last_clean":"2023-10-09T10:58:25.099381+0000","last_became_active":"2023-10-09T10:57:46.845475+0000","last_became_peered":"2023-10-09T10:57:46.845475+0000","last_unstale":"2023-10-09T10:58:25.099381+0000","last_undegraded":"2023-10-09T10:58:25.099381+0000","last_fullsized":"2023-10-09T10:58:25.099381+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:21.673994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":719,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.677228+0000","last_change":"2023-10-09T10:49:18.708370+0000","last_active":"2023-10-09T10:58:29.677228+0000","last_peered":"2023-10-09T10:58:29.677228+0000","last_clean":"2023-10-09T10:58:29.677228+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:58:29.677228+0000","last_undegraded":"2023-10-09T10:58:29.677228+0000","last_fullsized":"2023-10-09T10:58:29.677228+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:57.808410+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":34,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.093916+0000","last_change":"2023-10-09T10:57:22.026809+0000","last_active":"2023-10-09T10:58:25.093916+0000","last_peered":"2023-10-09T10:58:25.093916+0000","last_clean":"2023-10-09T10:58:25.093916+0000","last_became_active":"2023-10-09T10:57:22.011600+0000","last_became_peered":"2023-10-09T10:57:22.011600+0000","last_unstale":"2023-10-09T10:58:25.093916+0000","last_undegraded":"2023-10-09T10:58:25.093916+0000","last_fullsized":"2023-10-09T10:58:25.093916+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:10.453811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":428,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.084709+0000","last_change":"2023-10-09T10:57:22.006151+0000","last_active":"2023-10-09T10:58:25.084709+0000","last_peered":"2023-10-09T10:58:25.084709+0000","last_clean":"2023-10-09T10:58:25.084709+0000","last_became_active":"2023-10-09T10:57:22.005782+0000","last_became_peered":"2023-10-09T10:57:22.005782+0000","last_unstale":"2023-10-09T10:58:25.084709+0000","last_undegraded":"2023-10-09T10:58:25.084709+0000","last_fullsized":"2023-10-09T10:58:25.084709+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:09.733515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":670,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.091906+0000","last_change":"2023-10-09T10:57:25.098905+0000","last_active":"2023-10-09T10:58:25.091906+0000","last_peered":"2023-10-09T10:58:25.091906+0000","last_clean":"2023-10-09T10:58:25.091906+0000","last_became_active":"2023-10-09T10:57:25.098562+0000","last_became_peered":"2023-10-09T10:57:25.098562+0000","last_unstale":"2023-10-09T10:58:25.091906+0000","last_undegraded":"2023-10-09T10:58:25.091906+0000","last_fullsized":"2023-10-09T10:58:25.091906+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:23.020232+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":279,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.675671+0000","last_change":"2023-10-09T10:57:22.004708+0000","last_active":"2023-10-09T10:58:29.675671+0000","last_peered":"2023-10-09T10:58:29.675671+0000","last_clean":"2023-10-09T10:58:29.675671+0000","last_became_active":"2023-10-09T10:57:22.004413+0000","last_became_peered":"2023-10-09T10:57:22.004413+0000","last_unstale":"2023-10-09T10:58:29.675671+0000","last_undegraded":"2023-10-09T10:58:29.675671+0000","last_fullsized":"2023-10-09T10:58:29.675671+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:54.003556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":28,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092561+0000","last_change":"2023-10-09T10:57:22.014923+0000","last_active":"2023-10-09T10:58:25.092561+0000","last_peered":"2023-10-09T10:58:25.092561+0000","last_clean":"2023-10-09T10:58:25.092561+0000","last_became_active":"2023-10-09T10:57:22.014055+0000","last_became_peered":"2023-10-09T10:57:22.014055+0000","last_unstale":"2023-10-09T10:58:25.092561+0000","last_undegraded":"2023-10-09T10:58:25.092561+0000","last_fullsized":"2023-10-09T10:58:25.092561+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:50.546435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":669,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.669563+0000","last_change":"2023-10-09T10:57:44.843644+0000","last_active":"2023-10-09T10:58:29.669563+0000","last_peered":"2023-10-09T10:58:29.669563+0000","last_clean":"2023-10-09T10:58:29.669563+0000","last_became_active":"2023-10-09T10:54:50.915189+0000","last_became_peered":"2023-10-09T10:54:50.915189+0000","last_unstale":"2023-10-09T10:58:29.669563+0000","last_undegraded":"2023-10-09T10:58:29.669563+0000","last_fullsized":"2023-10-09T10:58:29.669563+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:44.843520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:44.843520+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:06.673360+0000","scrub_duration":0.0067258329999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":28,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.093729+0000","last_change":"2023-10-09T10:57:22.016705+0000","last_active":"2023-10-09T10:58:25.093729+0000","last_peered":"2023-10-09T10:58:25.093729+0000","last_clean":"2023-10-09T10:58:25.093729+0000","last_became_active":"2023-10-09T10:57:22.016452+0000","last_became_peered":"2023-10-09T10:57:22.016452+0000","last_unstale":"2023-10-09T10:58:25.093729+0000","last_undegraded":"2023-10-09T10:58:25.093729+0000","last_fullsized":"2023-10-09T10:58:25.093729+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:43:44.382821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":261,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.679676+0000","last_change":"2023-10-09T10:57:22.007046+0000","last_active":"2023-10-09T10:58:29.679676+0000","last_peered":"2023-10-09T10:58:29.679676+0000","last_clean":"2023-10-09T10:58:29.679676+0000","last_became_active":"2023-10-09T10:57:22.006360+0000","last_became_peered":"2023-10-09T10:57:22.006360+0000","last_unstale":"2023-10-09T10:58:29.679676+0000","last_undegraded":"2023-10-09T10:58:29.679676+0000","last_fullsized":"2023-10-09T10:58:29.679676+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:23.023072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":684,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.669426+0000","last_change":"2023-10-09T10:54:50.910027+0000","last_active":"2023-10-09T10:58:29.669426+0000","last_peered":"2023-10-09T10:58:29.669426+0000","last_clean":"2023-10-09T10:58:29.669426+0000","last_became_active":"2023-10-09T10:54:50.909698+0000","last_became_peered":"2023-10-09T10:54:50.909698+0000","last_unstale":"2023-10-09T10:58:29.669426+0000","last_undegraded":"2023-10-09T10:58:29.669426+0000","last_fullsized":"2023-10-09T10:58:29.669426+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:01.637196+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":228,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.675964+0000","last_change":"2023-10-09T10:51:12.366910+0000","last_active":"2023-10-09T10:58:29.675964+0000","last_peered":"2023-10-09T10:58:29.675964+0000","last_clean":"2023-10-09T10:58:29.675964+0000","last_became_active":"2023-10-09T10:51:12.366565+0000","last_became_peered":"2023-10-09T10:51:12.366565+0000","last_unstale":"2023-10-09T10:58:29.675964+0000","last_undegraded":"2023-10-09T10:58:29.675964+0000","last_fullsized":"2023-10-09T10:58:29.675964+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:53.464251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":405,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.082230+0000","last_change":"2023-10-09T10:57:22.008343+0000","last_active":"2023-10-09T10:58:25.082230+0000","last_peered":"2023-10-09T10:58:25.082230+0000","last_clean":"2023-10-09T10:58:25.082230+0000","last_became_active":"2023-10-09T10:57:22.007747+0000","last_became_peered":"2023-10-09T10:57:22.007747+0000","last_unstale":"2023-10-09T10:58:25.082230+0000","last_undegraded":"2023-10-09T10:58:25.082230+0000","last_fullsized":"2023-10-09T10:58:25.082230+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:45.774703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"352'5793","reported_seq":5833,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:32.621894+0000","last_change":"2023-10-09T10:57:22.013979+0000","last_active":"2023-10-09T10:58:32.621894+0000","last_peered":"2023-10-09T10:58:32.621894+0000","last_clean":"2023-10-09T10:58:32.621894+0000","last_became_active":"2023-10-09T10:57:22.013514+0000","last_became_peered":"2023-10-09T10:57:22.013514+0000","last_unstale":"2023-10-09T10:58:32.621894+0000","last_undegraded":"2023-10-09T10:58:32.621894+0000","last_fullsized":"2023-10-09T10:58:32.621894+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5793,"log_dups_size":0,"ondisk_log_size":5793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:16:42.975508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379650048,"num_objects":5793,"num_object_clones":0,"num_object_copies":17379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5793,"num_write_kb":370752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":691,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092240+0000","last_change":"2023-10-09T10:57:25.102011+0000","last_active":"2023-10-09T10:58:25.092240+0000","last_peered":"2023-10-09T10:58:25.092240+0000","last_clean":"2023-10-09T10:58:25.092240+0000","last_became_active":"2023-10-09T10:57:25.101702+0000","last_became_peered":"2023-10-09T10:57:25.101702+0000","last_unstale":"2023-10-09T10:58:25.092240+0000","last_undegraded":"2023-10-09T10:58:25.092240+0000","last_fullsized":"2023-10-09T10:58:25.092240+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:34.671859+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":32,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.093236+0000","last_change":"2023-10-09T10:57:22.021225+0000","last_active":"2023-10-09T10:58:25.093236+0000","last_peered":"2023-10-09T10:58:25.093236+0000","last_clean":"2023-10-09T10:58:25.093236+0000","last_became_active":"2023-10-09T10:57:22.007997+0000","last_became_peered":"2023-10-09T10:57:22.007997+0000","last_unstale":"2023-10-09T10:58:25.093236+0000","last_undegraded":"2023-10-09T10:58:25.093236+0000","last_fullsized":"2023-10-09T10:58:25.093236+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:52:19.871862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":28,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.091820+0000","last_change":"2023-10-09T10:57:22.016166+0000","last_active":"2023-10-09T10:58:25.091820+0000","last_peered":"2023-10-09T10:58:25.091820+0000","last_clean":"2023-10-09T10:58:25.091820+0000","last_became_active":"2023-10-09T10:57:22.015850+0000","last_became_peered":"2023-10-09T10:57:22.015850+0000","last_unstale":"2023-10-09T10:58:25.091820+0000","last_undegraded":"2023-10-09T10:58:25.091820+0000","last_fullsized":"2023-10-09T10:58:25.091820+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:37.827566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"352'5707","reported_seq":5738,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.888553+0000","last_change":"2023-10-09T10:57:19.986807+0000","last_active":"2023-10-09T10:58:31.888553+0000","last_peered":"2023-10-09T10:58:31.888553+0000","last_clean":"2023-10-09T10:58:31.888553+0000","last_became_active":"2023-10-09T10:57:19.986525+0000","last_became_peered":"2023-10-09T10:57:19.986525+0000","last_unstale":"2023-10-09T10:58:31.888553+0000","last_undegraded":"2023-10-09T10:58:31.888553+0000","last_fullsized":"2023-10-09T10:58:31.888553+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5707,"log_dups_size":0,"ondisk_log_size":5707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:23:53.293903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374013952,"num_objects":5707,"num_object_clones":0,"num_object_copies":17121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5707,"num_write_kb":365248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":712,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.081519+0000","last_change":"2023-10-09T10:53:25.540494+0000","last_active":"2023-10-09T10:58:25.081519+0000","last_peered":"2023-10-09T10:58:25.081519+0000","last_clean":"2023-10-09T10:58:25.081519+0000","last_became_active":"2023-10-09T10:51:01.295474+0000","last_became_peered":"2023-10-09T10:51:01.295474+0000","last_unstale":"2023-10-09T10:58:25.081519+0000","last_undegraded":"2023-10-09T10:58:25.081519+0000","last_fullsized":"2023-10-09T10:58:25.081519+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:39.069143+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":32,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092720+0000","last_change":"2023-10-09T10:57:22.023259+0000","last_active":"2023-10-09T10:58:25.092720+0000","last_peered":"2023-10-09T10:58:25.092720+0000","last_clean":"2023-10-09T10:58:25.092720+0000","last_became_active":"2023-10-09T10:57:22.009992+0000","last_became_peered":"2023-10-09T10:57:22.009992+0000","last_unstale":"2023-10-09T10:58:25.092720+0000","last_undegraded":"2023-10-09T10:58:25.092720+0000","last_fullsized":"2023-10-09T10:58:25.092720+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.289131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":69,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.668695+0000","last_change":"2023-10-09T10:54:47.944930+0000","last_active":"2023-10-09T10:58:29.668695+0000","last_peered":"2023-10-09T10:58:29.668695+0000","last_clean":"2023-10-09T10:58:29.668695+0000","last_became_active":"2023-10-09T10:54:47.944485+0000","last_became_peered":"2023-10-09T10:54:47.944485+0000","last_unstale":"2023-10-09T10:58:29.668695+0000","last_undegraded":"2023-10-09T10:58:29.668695+0000","last_fullsized":"2023-10-09T10:58:29.668695+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:11.213129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"351'5670","reported_seq":5697,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.196492+0000","last_change":"2023-10-09T10:57:22.024907+0000","last_active":"2023-10-09T10:58:31.196492+0000","last_peered":"2023-10-09T10:58:31.196492+0000","last_clean":"2023-10-09T10:58:31.196492+0000","last_became_active":"2023-10-09T10:57:22.024660+0000","last_became_peered":"2023-10-09T10:57:22.024660+0000","last_unstale":"2023-10-09T10:58:31.196492+0000","last_undegraded":"2023-10-09T10:58:31.196492+0000","last_fullsized":"2023-10-09T10:58:31.196492+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5670,"log_dups_size":0,"ondisk_log_size":5670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:25:10.105154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371589120,"num_objects":5670,"num_object_clones":0,"num_object_copies":17010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5670,"num_write_kb":362880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":663,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.100242+0000","last_change":"2023-10-09T10:53:36.680150+0000","last_active":"2023-10-09T10:58:25.100242+0000","last_peered":"2023-10-09T10:58:25.100242+0000","last_clean":"2023-10-09T10:58:25.100242+0000","last_became_active":"2023-10-09T10:51:50.371032+0000","last_became_peered":"2023-10-09T10:51:50.371032+0000","last_unstale":"2023-10-09T10:58:25.100242+0000","last_undegraded":"2023-10-09T10:58:25.100242+0000","last_fullsized":"2023-10-09T10:58:25.100242+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.680090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.680090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:08:28.952827+0000","scrub_duration":0.005577172,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":32,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092480+0000","last_change":"2023-10-09T10:57:22.034075+0000","last_active":"2023-10-09T10:58:25.092480+0000","last_peered":"2023-10-09T10:58:25.092480+0000","last_clean":"2023-10-09T10:58:25.092480+0000","last_became_active":"2023-10-09T10:57:22.015930+0000","last_became_peered":"2023-10-09T10:57:22.015930+0000","last_unstale":"2023-10-09T10:58:25.092480+0000","last_undegraded":"2023-10-09T10:58:25.092480+0000","last_fullsized":"2023-10-09T10:58:25.092480+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:16.494793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":87,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.669344+0000","last_change":"2023-10-09T10:57:22.005094+0000","last_active":"2023-10-09T10:58:29.669344+0000","last_peered":"2023-10-09T10:58:29.669344+0000","last_clean":"2023-10-09T10:58:29.669344+0000","last_became_active":"2023-10-09T10:57:22.004592+0000","last_became_peered":"2023-10-09T10:57:22.004592+0000","last_unstale":"2023-10-09T10:58:29.669344+0000","last_undegraded":"2023-10-09T10:58:29.669344+0000","last_fullsized":"2023-10-09T10:58:29.669344+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:35.526442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"351'5616","reported_seq":5646,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:30.978585+0000","last_change":"2023-10-09T10:57:20.305591+0000","last_active":"2023-10-09T10:58:30.978585+0000","last_peered":"2023-10-09T10:58:30.978585+0000","last_clean":"2023-10-09T10:58:30.978585+0000","last_became_active":"2023-10-09T10:57:20.305290+0000","last_became_peered":"2023-10-09T10:57:20.305290+0000","last_unstale":"2023-10-09T10:58:30.978585+0000","last_undegraded":"2023-10-09T10:58:30.978585+0000","last_fullsized":"2023-10-09T10:58:30.978585+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5616,"log_dups_size":0,"ondisk_log_size":5616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:54.880555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368050176,"num_objects":5616,"num_object_clones":0,"num_object_copies":16848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5616,"num_write_kb":359424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":836,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.102355+0000","last_change":"2023-10-09T10:57:25.111238+0000","last_active":"2023-10-09T10:58:25.102355+0000","last_peered":"2023-10-09T10:58:25.102355+0000","last_clean":"2023-10-09T10:58:25.102355+0000","last_became_active":"2023-10-09T10:57:25.110991+0000","last_became_peered":"2023-10-09T10:57:25.110991+0000","last_unstale":"2023-10-09T10:58:25.102355+0000","last_undegraded":"2023-10-09T10:58:25.102355+0000","last_fullsized":"2023-10-09T10:58:25.102355+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:57.060423+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":177,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.084555+0000","last_change":"2023-10-09T10:51:50.374527+0000","last_active":"2023-10-09T10:58:25.084555+0000","last_peered":"2023-10-09T10:58:25.084555+0000","last_clean":"2023-10-09T10:58:25.084555+0000","last_became_active":"2023-10-09T10:51:50.374297+0000","last_became_peered":"2023-10-09T10:51:50.374297+0000","last_unstale":"2023-10-09T10:58:25.084555+0000","last_undegraded":"2023-10-09T10:58:25.084555+0000","last_fullsized":"2023-10-09T10:58:25.084555+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:31:49.358307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":261,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.679854+0000","last_change":"2023-10-09T10:54:47.939771+0000","last_active":"2023-10-09T10:58:29.679854+0000","last_peered":"2023-10-09T10:58:29.679854+0000","last_clean":"2023-10-09T10:58:29.679854+0000","last_became_active":"2023-10-09T10:54:47.939440+0000","last_became_peered":"2023-10-09T10:54:47.939440+0000","last_unstale":"2023-10-09T10:58:29.679854+0000","last_undegraded":"2023-10-09T10:58:29.679854+0000","last_fullsized":"2023-10-09T10:58:29.679854+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:58.335536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"351'5612","reported_seq":5651,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:30.624051+0000","last_change":"2023-10-09T10:57:22.009934+0000","last_active":"2023-10-09T10:58:30.624051+0000","last_peered":"2023-10-09T10:58:30.624051+0000","last_clean":"2023-10-09T10:58:30.624051+0000","last_became_active":"2023-10-09T10:57:22.009682+0000","last_became_peered":"2023-10-09T10:57:22.009682+0000","last_unstale":"2023-10-09T10:58:30.624051+0000","last_undegraded":"2023-10-09T10:58:30.624051+0000","last_fullsized":"2023-10-09T10:58:30.624051+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5612,"log_dups_size":0,"ondisk_log_size":5612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:59.899734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367788032,"num_objects":5612,"num_object_clones":0,"num_object_copies":16836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5612,"num_write_kb":359168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":706,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.093550+0000","last_change":"2023-10-09T10:57:25.106422+0000","last_active":"2023-10-09T10:58:25.093550+0000","last_peered":"2023-10-09T10:58:25.093550+0000","last_clean":"2023-10-09T10:58:25.093550+0000","last_became_active":"2023-10-09T10:57:25.105998+0000","last_became_peered":"2023-10-09T10:57:25.105998+0000","last_unstale":"2023-10-09T10:58:25.093550+0000","last_undegraded":"2023-10-09T10:58:25.093550+0000","last_fullsized":"2023-10-09T10:58:25.093550+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:10:53.164214+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":348,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079203+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:58:25.079203+0000","last_peered":"2023-10-09T10:58:25.079203+0000","last_clean":"2023-10-09T10:58:25.079203+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:58:25.079203+0000","last_undegraded":"2023-10-09T10:58:25.079203+0000","last_fullsized":"2023-10-09T10:58:25.079203+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":193,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.084474+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T10:58:25.084474+0000","last_peered":"2023-10-09T10:58:25.084474+0000","last_clean":"2023-10-09T10:58:25.084474+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T10:58:25.084474+0000","last_undegraded":"2023-10-09T10:58:25.084474+0000","last_fullsized":"2023-10-09T10:58:25.084474+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"352'5620","reported_seq":5651,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.854142+0000","last_change":"2023-10-09T10:57:20.327412+0000","last_active":"2023-10-09T10:58:31.854142+0000","last_peered":"2023-10-09T10:58:31.854142+0000","last_clean":"2023-10-09T10:58:31.854142+0000","last_became_active":"2023-10-09T10:57:20.327192+0000","last_became_peered":"2023-10-09T10:57:20.327192+0000","last_unstale":"2023-10-09T10:58:31.854142+0000","last_undegraded":"2023-10-09T10:58:31.854142+0000","last_fullsized":"2023-10-09T10:58:31.854142+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5620,"log_dups_size":0,"ondisk_log_size":5620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:17:53.257184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368312320,"num_objects":5620,"num_object_clones":0,"num_object_copies":16860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5620,"num_write_kb":359680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":655,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.090633+0000","last_change":"2023-10-09T10:54:21.173078+0000","last_active":"2023-10-09T10:58:25.090633+0000","last_peered":"2023-10-09T10:58:25.090633+0000","last_clean":"2023-10-09T10:58:25.090633+0000","last_became_active":"2023-10-09T10:54:21.172767+0000","last_became_peered":"2023-10-09T10:54:21.172767+0000","last_unstale":"2023-10-09T10:58:25.090633+0000","last_undegraded":"2023-10-09T10:58:25.090633+0000","last_fullsized":"2023-10-09T10:58:25.090633+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.532890+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.532890+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:32.335841+0000","scrub_duration":0.0045290130000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":330,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.099575+0000","last_change":"2023-10-09T10:54:47.908506+0000","last_active":"2023-10-09T10:58:25.099575+0000","last_peered":"2023-10-09T10:58:25.099575+0000","last_clean":"2023-10-09T10:58:25.099575+0000","last_became_active":"2023-10-09T10:54:47.908110+0000","last_became_peered":"2023-10-09T10:54:47.908110+0000","last_unstale":"2023-10-09T10:58:25.099575+0000","last_undegraded":"2023-10-09T10:58:25.099575+0000","last_fullsized":"2023-10-09T10:58:25.099575+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:38.808943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":90,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.090382+0000","last_change":"2023-10-09T10:54:47.948732+0000","last_active":"2023-10-09T10:58:25.090382+0000","last_peered":"2023-10-09T10:58:25.090382+0000","last_clean":"2023-10-09T10:58:25.090382+0000","last_became_active":"2023-10-09T10:54:47.948414+0000","last_became_peered":"2023-10-09T10:54:47.948414+0000","last_unstale":"2023-10-09T10:58:25.090382+0000","last_undegraded":"2023-10-09T10:58:25.090382+0000","last_fullsized":"2023-10-09T10:58:25.090382+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:38.020807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"351'5585","reported_seq":5612,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.224799+0000","last_change":"2023-10-09T10:57:22.024943+0000","last_active":"2023-10-09T10:58:31.224799+0000","last_peered":"2023-10-09T10:58:31.224799+0000","last_clean":"2023-10-09T10:58:31.224799+0000","last_became_active":"2023-10-09T10:57:22.024623+0000","last_became_peered":"2023-10-09T10:57:22.024623+0000","last_unstale":"2023-10-09T10:58:31.224799+0000","last_undegraded":"2023-10-09T10:58:31.224799+0000","last_fullsized":"2023-10-09T10:58:31.224799+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5585,"log_dups_size":0,"ondisk_log_size":5585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:13:55.093719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366018560,"num_objects":5585,"num_object_clones":0,"num_object_copies":16755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5585,"num_write_kb":357440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":739,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.669779+0000","last_change":"2023-10-09T10:57:45.845982+0000","last_active":"2023-10-09T10:58:29.669779+0000","last_peered":"2023-10-09T10:58:29.669779+0000","last_clean":"2023-10-09T10:58:29.669779+0000","last_became_active":"2023-10-09T10:54:50.909639+0000","last_became_peered":"2023-10-09T10:54:50.909639+0000","last_unstale":"2023-10-09T10:58:29.669779+0000","last_undegraded":"2023-10-09T10:58:29.669779+0000","last_fullsized":"2023-10-09T10:58:29.669779+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:45.845850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:45.845850+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:52:51.027029+0000","scrub_duration":0.011054232000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":406,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.099237+0000","last_change":"2023-10-09T10:57:22.006224+0000","last_active":"2023-10-09T10:58:25.099237+0000","last_peered":"2023-10-09T10:58:25.099237+0000","last_clean":"2023-10-09T10:58:25.099237+0000","last_became_active":"2023-10-09T10:57:22.005898+0000","last_became_peered":"2023-10-09T10:57:22.005898+0000","last_unstale":"2023-10-09T10:58:25.099237+0000","last_undegraded":"2023-10-09T10:58:25.099237+0000","last_fullsized":"2023-10-09T10:58:25.099237+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:00.623898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.669161+0000","last_change":"2023-10-09T10:54:47.913826+0000","last_active":"2023-10-09T10:58:29.669161+0000","last_peered":"2023-10-09T10:58:29.669161+0000","last_clean":"2023-10-09T10:58:29.669161+0000","last_became_active":"2023-10-09T10:54:47.913524+0000","last_became_peered":"2023-10-09T10:54:47.913524+0000","last_unstale":"2023-10-09T10:58:29.669161+0000","last_undegraded":"2023-10-09T10:58:29.669161+0000","last_fullsized":"2023-10-09T10:58:29.669161+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:56:23.699552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.2","version":"351'5598","reported_seq":5628,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:30.504524+0000","last_change":"2023-10-09T10:57:19.986622+0000","last_active":"2023-10-09T10:58:30.504524+0000","last_peered":"2023-10-09T10:58:30.504524+0000","last_clean":"2023-10-09T10:58:30.504524+0000","last_became_active":"2023-10-09T10:57:19.986337+0000","last_became_peered":"2023-10-09T10:57:19.986337+0000","last_unstale":"2023-10-09T10:58:30.504524+0000","last_undegraded":"2023-10-09T10:58:30.504524+0000","last_fullsized":"2023-10-09T10:58:30.504524+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5598,"log_dups_size":0,"ondisk_log_size":5598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:59.817044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366870528,"num_objects":5598,"num_object_clones":0,"num_object_copies":16794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5598,"num_write_kb":358272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":666,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.089782+0000","last_change":"2023-10-09T10:54:21.152148+0000","last_active":"2023-10-09T10:58:25.089782+0000","last_peered":"2023-10-09T10:58:25.089782+0000","last_clean":"2023-10-09T10:58:25.089782+0000","last_became_active":"2023-10-09T10:54:21.151809+0000","last_became_peered":"2023-10-09T10:54:21.151809+0000","last_unstale":"2023-10-09T10:58:25.089782+0000","last_undegraded":"2023-10-09T10:58:25.089782+0000","last_fullsized":"2023-10-09T10:58:25.089782+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.701128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.701128+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:27.021145+0000","scrub_duration":0.0025071830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":397,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079151+0000","last_change":"2023-10-09T10:57:22.006965+0000","last_active":"2023-10-09T10:58:25.079151+0000","last_peered":"2023-10-09T10:58:25.079151+0000","last_clean":"2023-10-09T10:58:25.079151+0000","last_became_active":"2023-10-09T10:57:22.006456+0000","last_became_peered":"2023-10-09T10:57:22.006456+0000","last_unstale":"2023-10-09T10:58:25.079151+0000","last_undegraded":"2023-10-09T10:58:25.079151+0000","last_fullsized":"2023-10-09T10:58:25.079151+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:16.889422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":69,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.671953+0000","last_change":"2023-10-09T10:54:48.012411+0000","last_active":"2023-10-09T10:58:29.671953+0000","last_peered":"2023-10-09T10:58:29.671953+0000","last_clean":"2023-10-09T10:58:29.671953+0000","last_became_active":"2023-10-09T10:54:48.011894+0000","last_became_peered":"2023-10-09T10:54:48.011894+0000","last_unstale":"2023-10-09T10:58:29.671953+0000","last_undegraded":"2023-10-09T10:58:29.671953+0000","last_fullsized":"2023-10-09T10:58:29.671953+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:28.067337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"352'5773","reported_seq":5804,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.875167+0000","last_change":"2023-10-09T10:57:20.306271+0000","last_active":"2023-10-09T10:58:31.875167+0000","last_peered":"2023-10-09T10:58:31.875167+0000","last_clean":"2023-10-09T10:58:31.875167+0000","last_became_active":"2023-10-09T10:57:20.305994+0000","last_became_peered":"2023-10-09T10:57:20.305994+0000","last_unstale":"2023-10-09T10:58:31.875167+0000","last_undegraded":"2023-10-09T10:58:31.875167+0000","last_fullsized":"2023-10-09T10:58:31.875167+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5773,"log_dups_size":0,"ondisk_log_size":5773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:46.777641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378339328,"num_objects":5773,"num_object_clones":0,"num_object_copies":17319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5773,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":783,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.081932+0000","last_change":"2023-10-09T10:57:25.110928+0000","last_active":"2023-10-09T10:58:25.081932+0000","last_peered":"2023-10-09T10:58:25.081932+0000","last_clean":"2023-10-09T10:58:25.081932+0000","last_became_active":"2023-10-09T10:57:25.110240+0000","last_became_peered":"2023-10-09T10:57:25.110240+0000","last_unstale":"2023-10-09T10:58:25.081932+0000","last_undegraded":"2023-10-09T10:58:25.081932+0000","last_fullsized":"2023-10-09T10:58:25.081932+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:34.828051+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":87,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.670759+0000","last_change":"2023-10-09T10:57:22.005377+0000","last_active":"2023-10-09T10:58:29.670759+0000","last_peered":"2023-10-09T10:58:29.670759+0000","last_clean":"2023-10-09T10:58:29.670759+0000","last_became_active":"2023-10-09T10:57:22.005016+0000","last_became_peered":"2023-10-09T10:57:22.005016+0000","last_unstale":"2023-10-09T10:58:29.670759+0000","last_undegraded":"2023-10-09T10:58:29.670759+0000","last_fullsized":"2023-10-09T10:58:29.670759+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:03.296528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":272,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.080285+0000","last_change":"2023-10-09T10:54:17.886921+0000","last_active":"2023-10-09T10:58:25.080285+0000","last_peered":"2023-10-09T10:58:25.080285+0000","last_clean":"2023-10-09T10:58:25.080285+0000","last_became_active":"2023-10-09T10:54:17.886584+0000","last_became_peered":"2023-10-09T10:54:17.886584+0000","last_unstale":"2023-10-09T10:58:25.080285+0000","last_undegraded":"2023-10-09T10:58:25.080285+0000","last_fullsized":"2023-10-09T10:58:25.080285+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:15.848951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":69,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.669189+0000","last_change":"2023-10-09T10:54:47.908294+0000","last_active":"2023-10-09T10:58:29.669189+0000","last_peered":"2023-10-09T10:58:29.669189+0000","last_clean":"2023-10-09T10:58:29.669189+0000","last_became_active":"2023-10-09T10:54:47.908019+0000","last_became_peered":"2023-10-09T10:54:47.908019+0000","last_unstale":"2023-10-09T10:58:29.669189+0000","last_undegraded":"2023-10-09T10:58:29.669189+0000","last_fullsized":"2023-10-09T10:58:29.669189+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:48.662764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"352'5811","reported_seq":5851,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:32.631791+0000","last_change":"2023-10-09T10:57:22.005714+0000","last_active":"2023-10-09T10:58:32.631791+0000","last_peered":"2023-10-09T10:58:32.631791+0000","last_clean":"2023-10-09T10:58:32.631791+0000","last_became_active":"2023-10-09T10:57:22.005431+0000","last_became_peered":"2023-10-09T10:57:22.005431+0000","last_unstale":"2023-10-09T10:58:32.631791+0000","last_undegraded":"2023-10-09T10:58:32.631791+0000","last_fullsized":"2023-10-09T10:58:32.631791+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5811,"log_dups_size":0,"ondisk_log_size":5811,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:51.650262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380829696,"num_objects":5811,"num_object_clones":0,"num_object_copies":17433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5811,"num_write_kb":371904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":690,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.677139+0000","last_change":"2023-10-09T10:54:21.163072+0000","last_active":"2023-10-09T10:58:29.677139+0000","last_peered":"2023-10-09T10:58:29.677139+0000","last_clean":"2023-10-09T10:58:29.677139+0000","last_became_active":"2023-10-09T10:54:21.162710+0000","last_became_peered":"2023-10-09T10:54:21.162710+0000","last_unstale":"2023-10-09T10:58:29.677139+0000","last_undegraded":"2023-10-09T10:58:29.677139+0000","last_fullsized":"2023-10-09T10:58:29.677139+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:15.932806+0000","scrub_duration":0.0019146790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":69,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.669273+0000","last_change":"2023-10-09T10:54:47.899967+0000","last_active":"2023-10-09T10:58:29.669273+0000","last_peered":"2023-10-09T10:58:29.669273+0000","last_clean":"2023-10-09T10:58:29.669273+0000","last_became_active":"2023-10-09T10:54:47.899584+0000","last_became_peered":"2023-10-09T10:54:47.899584+0000","last_unstale":"2023-10-09T10:58:29.669273+0000","last_undegraded":"2023-10-09T10:58:29.669273+0000","last_fullsized":"2023-10-09T10:58:29.669273+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:43.973868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":262,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.679549+0000","last_change":"2023-10-09T10:50:43.388615+0000","last_active":"2023-10-09T10:58:29.679549+0000","last_peered":"2023-10-09T10:58:29.679549+0000","last_clean":"2023-10-09T10:58:29.679549+0000","last_became_active":"2023-10-09T10:50:43.388176+0000","last_became_peered":"2023-10-09T10:50:43.388176+0000","last_unstale":"2023-10-09T10:58:29.679549+0000","last_undegraded":"2023-10-09T10:58:29.679549+0000","last_fullsized":"2023-10-09T10:58:29.679549+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:41:17.550111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":69,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.670694+0000","last_change":"2023-10-09T10:54:47.950462+0000","last_active":"2023-10-09T10:58:29.670694+0000","last_peered":"2023-10-09T10:58:29.670694+0000","last_clean":"2023-10-09T10:58:29.670694+0000","last_became_active":"2023-10-09T10:54:47.940606+0000","last_became_peered":"2023-10-09T10:54:47.940606+0000","last_unstale":"2023-10-09T10:58:29.670694+0000","last_undegraded":"2023-10-09T10:58:29.670694+0000","last_fullsized":"2023-10-09T10:58:29.670694+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:50.557850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"352'5633","reported_seq":5673,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.811877+0000","last_change":"2023-10-09T10:57:22.004913+0000","last_active":"2023-10-09T10:58:31.811877+0000","last_peered":"2023-10-09T10:58:31.811877+0000","last_clean":"2023-10-09T10:58:31.811877+0000","last_became_active":"2023-10-09T10:57:22.004172+0000","last_became_peered":"2023-10-09T10:57:22.004172+0000","last_unstale":"2023-10-09T10:58:31.811877+0000","last_undegraded":"2023-10-09T10:58:31.811877+0000","last_fullsized":"2023-10-09T10:58:31.811877+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5633,"log_dups_size":0,"ondisk_log_size":5633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:01.045466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369164288,"num_objects":5633,"num_object_clones":0,"num_object_copies":16899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5633,"num_write_kb":360512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":765,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092854+0000","last_change":"2023-10-09T10:57:25.101168+0000","last_active":"2023-10-09T10:58:25.092854+0000","last_peered":"2023-10-09T10:58:25.092854+0000","last_clean":"2023-10-09T10:58:25.092854+0000","last_became_active":"2023-10-09T10:57:25.100860+0000","last_became_peered":"2023-10-09T10:57:25.100860+0000","last_unstale":"2023-10-09T10:58:25.092854+0000","last_undegraded":"2023-10-09T10:58:25.092854+0000","last_fullsized":"2023-10-09T10:58:25.092854+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:37.594583+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":82,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:26.001823+0000","last_change":"2023-10-09T10:54:17.885030+0000","last_active":"2023-10-09T10:58:26.001823+0000","last_peered":"2023-10-09T10:58:26.001823+0000","last_clean":"2023-10-09T10:58:26.001823+0000","last_became_active":"2023-10-09T10:54:17.884752+0000","last_became_peered":"2023-10-09T10:54:17.884752+0000","last_unstale":"2023-10-09T10:58:26.001823+0000","last_undegraded":"2023-10-09T10:58:26.001823+0000","last_fullsized":"2023-10-09T10:58:26.001823+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:38.548132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":69,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.668610+0000","last_change":"2023-10-09T10:54:47.927822+0000","last_active":"2023-10-09T10:58:29.668610+0000","last_peered":"2023-10-09T10:58:29.668610+0000","last_clean":"2023-10-09T10:58:29.668610+0000","last_became_active":"2023-10-09T10:54:47.927455+0000","last_became_peered":"2023-10-09T10:54:47.927455+0000","last_unstale":"2023-10-09T10:58:29.668610+0000","last_undegraded":"2023-10-09T10:58:29.668610+0000","last_fullsized":"2023-10-09T10:58:29.668610+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:11.818203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":69,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.671890+0000","last_change":"2023-10-09T10:54:48.012629+0000","last_active":"2023-10-09T10:58:29.671890+0000","last_peered":"2023-10-09T10:58:29.671890+0000","last_clean":"2023-10-09T10:58:29.671890+0000","last_became_active":"2023-10-09T10:54:48.012148+0000","last_became_peered":"2023-10-09T10:54:48.012148+0000","last_unstale":"2023-10-09T10:58:29.671890+0000","last_undegraded":"2023-10-09T10:58:29.671890+0000","last_fullsized":"2023-10-09T10:58:29.671890+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:22.894111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"351'5584","reported_seq":5614,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:30.567901+0000","last_change":"2023-10-09T10:57:20.305367+0000","last_active":"2023-10-09T10:58:30.567901+0000","last_peered":"2023-10-09T10:58:30.567901+0000","last_clean":"2023-10-09T10:58:30.567901+0000","last_became_active":"2023-10-09T10:57:20.305050+0000","last_became_peered":"2023-10-09T10:57:20.305050+0000","last_unstale":"2023-10-09T10:58:30.567901+0000","last_undegraded":"2023-10-09T10:58:30.567901+0000","last_fullsized":"2023-10-09T10:58:30.567901+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5584,"log_dups_size":0,"ondisk_log_size":5584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:49.570577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365953024,"num_objects":5584,"num_object_clones":0,"num_object_copies":16752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5584,"num_write_kb":357376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":100,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.669707+0000","last_change":"2023-10-09T10:57:22.006186+0000","last_active":"2023-10-09T10:58:29.669707+0000","last_peered":"2023-10-09T10:58:29.669707+0000","last_clean":"2023-10-09T10:58:29.669707+0000","last_became_active":"2023-10-09T10:57:22.005900+0000","last_became_peered":"2023-10-09T10:57:22.005900+0000","last_unstale":"2023-10-09T10:58:29.669707+0000","last_undegraded":"2023-10-09T10:58:29.669707+0000","last_fullsized":"2023-10-09T10:58:29.669707+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:14.033785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":782,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092148+0000","last_change":"2023-10-09T10:57:25.100325+0000","last_active":"2023-10-09T10:58:25.092148+0000","last_peered":"2023-10-09T10:58:25.092148+0000","last_clean":"2023-10-09T10:58:25.092148+0000","last_became_active":"2023-10-09T10:57:25.099940+0000","last_became_peered":"2023-10-09T10:57:25.099940+0000","last_unstale":"2023-10-09T10:58:25.092148+0000","last_undegraded":"2023-10-09T10:58:25.092148+0000","last_fullsized":"2023-10-09T10:58:25.092148+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:33.410507+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":284,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079910+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:58:25.079910+0000","last_peered":"2023-10-09T10:58:25.079910+0000","last_clean":"2023-10-09T10:58:25.079910+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:58:25.079910+0000","last_undegraded":"2023-10-09T10:58:25.079910+0000","last_fullsized":"2023-10-09T10:58:25.079910+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":283,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.099919+0000","last_change":"2023-10-09T10:57:22.013585+0000","last_active":"2023-10-09T10:58:25.099919+0000","last_peered":"2023-10-09T10:58:25.099919+0000","last_clean":"2023-10-09T10:58:25.099919+0000","last_became_active":"2023-10-09T10:57:22.013255+0000","last_became_peered":"2023-10-09T10:57:22.013255+0000","last_unstale":"2023-10-09T10:58:25.099919+0000","last_undegraded":"2023-10-09T10:58:25.099919+0000","last_fullsized":"2023-10-09T10:58:25.099919+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:29.725918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"352'5712","reported_seq":5743,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:32.581179+0000","last_change":"2023-10-09T10:57:20.328496+0000","last_active":"2023-10-09T10:58:32.581179+0000","last_peered":"2023-10-09T10:58:32.581179+0000","last_clean":"2023-10-09T10:58:32.581179+0000","last_became_active":"2023-10-09T10:57:20.328194+0000","last_became_peered":"2023-10-09T10:57:20.328194+0000","last_unstale":"2023-10-09T10:58:32.581179+0000","last_undegraded":"2023-10-09T10:58:32.581179+0000","last_fullsized":"2023-10-09T10:58:32.581179+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5712,"log_dups_size":0,"ondisk_log_size":5712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:15:57.867795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374341632,"num_objects":5712,"num_object_clones":0,"num_object_copies":17136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5712,"num_write_kb":365568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":712,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.669643+0000","last_change":"2023-10-09T10:57:43.819618+0000","last_active":"2023-10-09T10:58:29.669643+0000","last_peered":"2023-10-09T10:58:29.669643+0000","last_clean":"2023-10-09T10:58:29.669643+0000","last_became_active":"2023-10-09T10:54:50.914536+0000","last_became_peered":"2023-10-09T10:54:50.914536+0000","last_unstale":"2023-10-09T10:58:29.669643+0000","last_undegraded":"2023-10-09T10:58:29.669643+0000","last_fullsized":"2023-10-09T10:58:29.669643+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:43.819496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:43.819496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:09.320712+0000","scrub_duration":0.0071681970000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":28,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092048+0000","last_change":"2023-10-09T10:57:22.008209+0000","last_active":"2023-10-09T10:58:25.092048+0000","last_peered":"2023-10-09T10:58:25.092048+0000","last_clean":"2023-10-09T10:58:25.092048+0000","last_became_active":"2023-10-09T10:57:22.007918+0000","last_became_peered":"2023-10-09T10:57:22.007918+0000","last_unstale":"2023-10-09T10:58:25.092048+0000","last_undegraded":"2023-10-09T10:58:25.092048+0000","last_fullsized":"2023-10-09T10:58:25.092048+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:55.764820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":235,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.726703+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:58:31.726703+0000","last_peered":"2023-10-09T10:58:31.726703+0000","last_clean":"2023-10-09T10:58:31.726703+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:58:31.726703+0000","last_undegraded":"2023-10-09T10:58:31.726703+0000","last_fullsized":"2023-10-09T10:58:31.726703+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":297,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.679745+0000","last_change":"2023-10-09T10:54:47.939756+0000","last_active":"2023-10-09T10:58:29.679745+0000","last_peered":"2023-10-09T10:58:29.679745+0000","last_clean":"2023-10-09T10:58:29.679745+0000","last_became_active":"2023-10-09T10:54:47.938928+0000","last_became_peered":"2023-10-09T10:54:47.938928+0000","last_unstale":"2023-10-09T10:58:29.679745+0000","last_undegraded":"2023-10-09T10:58:29.679745+0000","last_fullsized":"2023-10-09T10:58:29.679745+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:05.310929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"351'5787","reported_seq":5815,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.197211+0000","last_change":"2023-10-09T10:57:22.010174+0000","last_active":"2023-10-09T10:58:31.197211+0000","last_peered":"2023-10-09T10:58:31.197211+0000","last_clean":"2023-10-09T10:58:31.197211+0000","last_became_active":"2023-10-09T10:57:22.009987+0000","last_became_peered":"2023-10-09T10:57:22.009987+0000","last_unstale":"2023-10-09T10:58:31.197211+0000","last_undegraded":"2023-10-09T10:58:31.197211+0000","last_fullsized":"2023-10-09T10:58:31.197211+0000","mapping_epoch":332,"log_start":"336'100","ondisk_log_start":"336'100","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5687,"log_dups_size":0,"ondisk_log_size":5687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:45.169457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379256832,"num_objects":5787,"num_object_clones":0,"num_object_copies":17361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5787,"num_write_kb":370368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":725,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.679968+0000","last_change":"2023-10-09T10:53:27.625180+0000","last_active":"2023-10-09T10:58:29.679968+0000","last_peered":"2023-10-09T10:58:29.679968+0000","last_clean":"2023-10-09T10:58:29.679968+0000","last_became_active":"2023-10-09T10:51:12.366828+0000","last_became_peered":"2023-10-09T10:51:12.366828+0000","last_unstale":"2023-10-09T10:58:29.679968+0000","last_undegraded":"2023-10-09T10:58:29.679968+0000","last_fullsized":"2023-10-09T10:58:29.679968+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.625079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.625079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:59:41.253976+0000","scrub_duration":0.0041067969999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.092160+0000","last_change":"2023-10-09T10:57:22.012147+0000","last_active":"2023-10-09T10:58:25.092160+0000","last_peered":"2023-10-09T10:58:25.092160+0000","last_clean":"2023-10-09T10:58:25.092160+0000","last_became_active":"2023-10-09T10:57:22.011818+0000","last_became_peered":"2023-10-09T10:57:22.011818+0000","last_unstale":"2023-10-09T10:58:25.092160+0000","last_undegraded":"2023-10-09T10:58:25.092160+0000","last_fullsized":"2023-10-09T10:58:25.092160+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:10.838344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":69,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.670923+0000","last_change":"2023-10-09T10:54:47.925711+0000","last_active":"2023-10-09T10:58:29.670923+0000","last_peered":"2023-10-09T10:58:29.670923+0000","last_clean":"2023-10-09T10:58:29.670923+0000","last_became_active":"2023-10-09T10:54:47.924286+0000","last_became_peered":"2023-10-09T10:54:47.924286+0000","last_unstale":"2023-10-09T10:58:29.670923+0000","last_undegraded":"2023-10-09T10:58:29.670923+0000","last_fullsized":"2023-10-09T10:58:29.670923+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:58.379080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":280,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.728166+0000","last_change":"2023-10-09T10:57:22.004838+0000","last_active":"2023-10-09T10:58:31.728166+0000","last_peered":"2023-10-09T10:58:31.728166+0000","last_clean":"2023-10-09T10:58:31.728166+0000","last_became_active":"2023-10-09T10:57:22.004519+0000","last_became_peered":"2023-10-09T10:57:22.004519+0000","last_unstale":"2023-10-09T10:58:31.728166+0000","last_undegraded":"2023-10-09T10:58:31.728166+0000","last_fullsized":"2023-10-09T10:58:31.728166+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:53.585466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"351'5645","reported_seq":5673,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.216986+0000","last_change":"2023-10-09T10:57:22.015103+0000","last_active":"2023-10-09T10:58:31.216986+0000","last_peered":"2023-10-09T10:58:31.216986+0000","last_clean":"2023-10-09T10:58:31.216986+0000","last_became_active":"2023-10-09T10:57:22.014603+0000","last_became_peered":"2023-10-09T10:57:22.014603+0000","last_unstale":"2023-10-09T10:58:31.216986+0000","last_undegraded":"2023-10-09T10:58:31.216986+0000","last_fullsized":"2023-10-09T10:58:31.216986+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5645,"log_dups_size":0,"ondisk_log_size":5645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:22:34.112316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369950720,"num_objects":5645,"num_object_clones":0,"num_object_copies":16935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5645,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":746,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.099851+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T10:58:25.099851+0000","last_peered":"2023-10-09T10:58:25.099851+0000","last_clean":"2023-10-09T10:58:25.099851+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T10:58:25.099851+0000","last_undegraded":"2023-10-09T10:58:25.099851+0000","last_fullsized":"2023-10-09T10:58:25.099851+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":400,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.082134+0000","last_change":"2023-10-09T10:57:22.030437+0000","last_active":"2023-10-09T10:58:25.082134+0000","last_peered":"2023-10-09T10:58:25.082134+0000","last_clean":"2023-10-09T10:58:25.082134+0000","last_became_active":"2023-10-09T10:57:22.008090+0000","last_became_peered":"2023-10-09T10:57:22.008090+0000","last_unstale":"2023-10-09T10:58:25.082134+0000","last_undegraded":"2023-10-09T10:58:25.082134+0000","last_fullsized":"2023-10-09T10:58:25.082134+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:52:54.221183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":69,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.669304+0000","last_change":"2023-10-09T10:54:47.908305+0000","last_active":"2023-10-09T10:58:29.669304+0000","last_peered":"2023-10-09T10:58:29.669304+0000","last_clean":"2023-10-09T10:58:29.669304+0000","last_became_active":"2023-10-09T10:54:47.907999+0000","last_became_peered":"2023-10-09T10:54:47.907999+0000","last_unstale":"2023-10-09T10:58:29.669304+0000","last_undegraded":"2023-10-09T10:58:29.669304+0000","last_fullsized":"2023-10-09T10:58:29.669304+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:48.405124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":392,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079245+0000","last_change":"2023-10-09T10:57:22.007248+0000","last_active":"2023-10-09T10:58:25.079245+0000","last_peered":"2023-10-09T10:58:25.079245+0000","last_clean":"2023-10-09T10:58:25.079245+0000","last_became_active":"2023-10-09T10:57:22.006899+0000","last_became_peered":"2023-10-09T10:57:22.006899+0000","last_unstale":"2023-10-09T10:58:25.079245+0000","last_undegraded":"2023-10-09T10:58:25.079245+0000","last_fullsized":"2023-10-09T10:58:25.079245+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:45.159091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"351'5611","reported_seq":5649,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:30.585755+0000","last_change":"2023-10-09T10:57:22.006548+0000","last_active":"2023-10-09T10:58:30.585755+0000","last_peered":"2023-10-09T10:58:30.585755+0000","last_clean":"2023-10-09T10:58:30.585755+0000","last_became_active":"2023-10-09T10:57:22.006030+0000","last_became_peered":"2023-10-09T10:57:22.006030+0000","last_unstale":"2023-10-09T10:58:30.585755+0000","last_undegraded":"2023-10-09T10:58:30.585755+0000","last_fullsized":"2023-10-09T10:58:30.585755+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5611,"log_dups_size":0,"ondisk_log_size":5611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:47.539524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367722496,"num_objects":5611,"num_object_clones":0,"num_object_copies":16833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5611,"num_write_kb":359104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":777,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.084268+0000","last_change":"2023-10-09T10:57:25.106977+0000","last_active":"2023-10-09T10:58:25.084268+0000","last_peered":"2023-10-09T10:58:25.084268+0000","last_clean":"2023-10-09T10:58:25.084268+0000","last_became_active":"2023-10-09T10:57:25.106632+0000","last_became_peered":"2023-10-09T10:57:25.106632+0000","last_unstale":"2023-10-09T10:58:25.084268+0000","last_undegraded":"2023-10-09T10:58:25.084268+0000","last_fullsized":"2023-10-09T10:58:25.084268+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:29:20.989344+0000","scrub_duration":0.0053707060000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.093302+0000","last_change":"2023-10-09T10:57:22.016009+0000","last_active":"2023-10-09T10:58:25.093302+0000","last_peered":"2023-10-09T10:58:25.093302+0000","last_clean":"2023-10-09T10:58:25.093302+0000","last_became_active":"2023-10-09T10:57:22.015633+0000","last_became_peered":"2023-10-09T10:57:22.015633+0000","last_unstale":"2023-10-09T10:58:25.093302+0000","last_undegraded":"2023-10-09T10:58:25.093302+0000","last_fullsized":"2023-10-09T10:58:25.093302+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.957416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":281,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.679824+0000","last_change":"2023-10-09T10:54:48.016159+0000","last_active":"2023-10-09T10:58:29.679824+0000","last_peered":"2023-10-09T10:58:29.679824+0000","last_clean":"2023-10-09T10:58:29.679824+0000","last_became_active":"2023-10-09T10:54:47.939409+0000","last_became_peered":"2023-10-09T10:54:47.939409+0000","last_unstale":"2023-10-09T10:58:29.679824+0000","last_undegraded":"2023-10-09T10:58:29.679824+0000","last_fullsized":"2023-10-09T10:58:29.679824+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:11:15.175207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":292,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.099166+0000","last_change":"2023-10-09T10:51:50.371282+0000","last_active":"2023-10-09T10:58:25.099166+0000","last_peered":"2023-10-09T10:58:25.099166+0000","last_clean":"2023-10-09T10:58:25.099166+0000","last_became_active":"2023-10-09T10:51:50.370973+0000","last_became_peered":"2023-10-09T10:51:50.370973+0000","last_unstale":"2023-10-09T10:58:25.099166+0000","last_undegraded":"2023-10-09T10:58:25.099166+0000","last_fullsized":"2023-10-09T10:58:25.099166+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:15.603164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":743,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079133+0000","last_change":"2023-10-09T10:53:29.568199+0000","last_active":"2023-10-09T10:58:25.079133+0000","last_peered":"2023-10-09T10:58:25.079133+0000","last_clean":"2023-10-09T10:58:25.079133+0000","last_became_active":"2023-10-09T10:52:55.416357+0000","last_became_peered":"2023-10-09T10:52:55.416357+0000","last_unstale":"2023-10-09T10:58:25.079133+0000","last_undegraded":"2023-10-09T10:58:25.079133+0000","last_fullsized":"2023-10-09T10:58:25.079133+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:44.364650+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":279,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.728246+0000","last_change":"2023-10-09T10:57:22.005039+0000","last_active":"2023-10-09T10:58:31.728246+0000","last_peered":"2023-10-09T10:58:31.728246+0000","last_clean":"2023-10-09T10:58:31.728246+0000","last_became_active":"2023-10-09T10:57:22.004497+0000","last_became_peered":"2023-10-09T10:57:22.004497+0000","last_unstale":"2023-10-09T10:58:31.728246+0000","last_undegraded":"2023-10-09T10:58:31.728246+0000","last_fullsized":"2023-10-09T10:58:31.728246+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:11.006259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":90,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.090195+0000","last_change":"2023-10-09T10:54:47.944380+0000","last_active":"2023-10-09T10:58:25.090195+0000","last_peered":"2023-10-09T10:58:25.090195+0000","last_clean":"2023-10-09T10:58:25.090195+0000","last_became_active":"2023-10-09T10:54:47.943766+0000","last_became_peered":"2023-10-09T10:54:47.943766+0000","last_unstale":"2023-10-09T10:58:25.090195+0000","last_undegraded":"2023-10-09T10:58:25.090195+0000","last_fullsized":"2023-10-09T10:58:25.090195+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:16:11.869603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":708,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.080303+0000","last_change":"2023-10-09T10:57:25.111170+0000","last_active":"2023-10-09T10:58:25.080303+0000","last_peered":"2023-10-09T10:58:25.080303+0000","last_clean":"2023-10-09T10:58:25.080303+0000","last_became_active":"2023-10-09T10:57:25.110405+0000","last_became_peered":"2023-10-09T10:57:25.110405+0000","last_unstale":"2023-10-09T10:58:25.080303+0000","last_undegraded":"2023-10-09T10:58:25.080303+0000","last_fullsized":"2023-10-09T10:58:25.080303+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:00.112600+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":177,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.102100+0000","last_change":"2023-10-09T10:54:47.923386+0000","last_active":"2023-10-09T10:58:25.102100+0000","last_peered":"2023-10-09T10:58:25.102100+0000","last_clean":"2023-10-09T10:58:25.102100+0000","last_became_active":"2023-10-09T10:54:47.914257+0000","last_became_peered":"2023-10-09T10:54:47.914257+0000","last_unstale":"2023-10-09T10:58:25.102100+0000","last_undegraded":"2023-10-09T10:58:25.102100+0000","last_fullsized":"2023-10-09T10:58:25.102100+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:24:18.190715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":378,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079795+0000","last_change":"2023-10-09T10:57:46.844827+0000","last_active":"2023-10-09T10:58:25.079795+0000","last_peered":"2023-10-09T10:58:25.079795+0000","last_clean":"2023-10-09T10:58:25.079795+0000","last_became_active":"2023-10-09T10:57:46.844584+0000","last_became_peered":"2023-10-09T10:57:46.844584+0000","last_unstale":"2023-10-09T10:58:25.079795+0000","last_undegraded":"2023-10-09T10:58:25.079795+0000","last_fullsized":"2023-10-09T10:58:25.079795+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:54.499813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":301,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.088439+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:58:25.088439+0000","last_peered":"2023-10-09T10:58:25.088439+0000","last_clean":"2023-10-09T10:58:25.088439+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:58:25.088439+0000","last_undegraded":"2023-10-09T10:58:25.088439+0000","last_fullsized":"2023-10-09T10:58:25.088439+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":296,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079951+0000","last_change":"2023-10-09T10:50:43.385150+0000","last_active":"2023-10-09T10:58:25.079951+0000","last_peered":"2023-10-09T10:58:25.079951+0000","last_clean":"2023-10-09T10:58:25.079951+0000","last_became_active":"2023-10-09T10:50:43.384788+0000","last_became_peered":"2023-10-09T10:50:43.384788+0000","last_unstale":"2023-10-09T10:58:25.079951+0000","last_undegraded":"2023-10-09T10:58:25.079951+0000","last_fullsized":"2023-10-09T10:58:25.079951+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:36.085910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":642,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.093386+0000","last_change":"2023-10-09T10:57:47.307757+0000","last_active":"2023-10-09T10:58:25.093386+0000","last_peered":"2023-10-09T10:58:25.093386+0000","last_clean":"2023-10-09T10:58:25.093386+0000","last_became_active":"2023-10-09T10:57:25.104921+0000","last_became_peered":"2023-10-09T10:57:25.104921+0000","last_unstale":"2023-10-09T10:58:25.093386+0000","last_undegraded":"2023-10-09T10:58:25.093386+0000","last_fullsized":"2023-10-09T10:58:25.093386+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:47.307693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:47.307693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:40.935036+0000","scrub_duration":0.003001985,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":376,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.080471+0000","last_change":"2023-10-09T10:54:17.886158+0000","last_active":"2023-10-09T10:58:25.080471+0000","last_peered":"2023-10-09T10:58:25.080471+0000","last_clean":"2023-10-09T10:58:25.080471+0000","last_became_active":"2023-10-09T10:54:17.885816+0000","last_became_peered":"2023-10-09T10:54:17.885816+0000","last_unstale":"2023-10-09T10:58:25.080471+0000","last_undegraded":"2023-10-09T10:58:25.080471+0000","last_fullsized":"2023-10-09T10:58:25.080471+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:05.270825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":379,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.079447+0000","last_change":"2023-10-09T10:54:17.957784+0000","last_active":"2023-10-09T10:58:25.079447+0000","last_peered":"2023-10-09T10:58:25.079447+0000","last_clean":"2023-10-09T10:58:25.079447+0000","last_became_active":"2023-10-09T10:54:17.954788+0000","last_became_peered":"2023-10-09T10:54:17.954788+0000","last_unstale":"2023-10-09T10:58:25.079447+0000","last_undegraded":"2023-10-09T10:58:25.079447+0000","last_fullsized":"2023-10-09T10:58:25.079447+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:48.709390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":639,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T10:58:29.669407+0000","last_change":"2023-10-09T10:57:21.022980+0000","last_active":"2023-10-09T10:58:29.669407+0000","last_peered":"2023-10-09T10:58:29.669407+0000","last_clean":"2023-10-09T10:58:29.669407+0000","last_became_active":"2023-10-09T10:54:50.914005+0000","last_became_peered":"2023-10-09T10:54:50.914005+0000","last_unstale":"2023-10-09T10:58:29.669407+0000","last_undegraded":"2023-10-09T10:58:29.669407+0000","last_fullsized":"2023-10-09T10:58:29.669407+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:21.022863+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:21.022863+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:49.534216+0000","scrub_duration":0.00136929,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":262,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.727565+0000","last_change":"2023-10-09T10:54:18.089053+0000","last_active":"2023-10-09T10:58:31.727565+0000","last_peered":"2023-10-09T10:58:31.727565+0000","last_clean":"2023-10-09T10:58:31.727565+0000","last_became_active":"2023-10-09T10:54:17.886070+0000","last_became_peered":"2023-10-09T10:54:17.886070+0000","last_unstale":"2023-10-09T10:58:31.727565+0000","last_undegraded":"2023-10-09T10:58:31.727565+0000","last_fullsized":"2023-10-09T10:58:31.727565+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:35.628407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":355,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T10:58:25.098955+0000","last_change":"2023-10-09T10:54:47.908059+0000","last_active":"2023-10-09T10:58:25.098955+0000","last_peered":"2023-10-09T10:58:25.098955+0000","last_clean":"2023-10-09T10:58:25.098955+0000","last_became_active":"2023-10-09T10:54:47.907780+0000","last_became_peered":"2023-10-09T10:54:47.907780+0000","last_unstale":"2023-10-09T10:58:25.098955+0000","last_undegraded":"2023-10-09T10:58:25.098955+0000","last_fullsized":"2023-10-09T10:58:25.098955+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:55:59.064617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":16490,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5947850752,"num_objects":90757,"num_object_clones":0,"num_object_copies":272271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":90757,"num_write_kb":5808448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17826512896,"data_stored":17826512896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90657,"ondisk_log_size":90657,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332383,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3765888,"kb_used_data":3460408,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":89979264,"statfs":{"total":95995035648,"available":92138766336,"internally_reserved":0,"allocated":3543457792,"data_stored":3542254167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,3,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:03 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.821,"15min":1.4870000000000001},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":4.5190000000000001,"5min":9.6259999999999994,"15min":9.6379999999999999},"last":8.9730000000000008},{"interface":"front","average":{"1min":1.1499999999999999,"5min":2.004,"15min":1.7030000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.4470000000000001,"5min":8.7769999999999992,"15min":16.731000000000002},"last":6.2489999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:58:10 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":1.835,"15min":1.827},"min":{"1min":0.26500000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.2880000000000003,"5min":12.252000000000001,"15min":34.756},"last":3.3799999999999999},{"interface":"front","average":{"1min":1.177,"5min":1.744,"15min":1.609},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":2.4649999999999999,"5min":13.647,"15min":35.658999999999999},"last":2.6680000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:57:35 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":1.538,"15min":1.5469999999999999},"min":{"1min":0.35199999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.0999999999999996,"5min":9.202,"15min":11.058},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.8899999999999999,"5min":1.746,"15min":1.4430000000000001},"min":{"1min":0.28899999999999998,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":13.037000000000001,"5min":13.037000000000001,"15min":13.037000000000001},"last":8.9280000000000008}]},{"osd":3,"last update":"Mon Oct 9 10:58:03 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":2.1080000000000001,"15min":1.913},"min":{"1min":0.33200000000000002,"5min":0.31,"15min":0.31},"max":{"1min":7.1970000000000001,"5min":11.760999999999999,"15min":11.760999999999999},"last":5.6500000000000004},{"interface":"front","average":{"1min":0.89900000000000002,"5min":1.5489999999999999,"15min":1.627},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":2.2040000000000002,"5min":8.6300000000000008,"15min":19.234999999999999},"last":0.54600000000000004}]},{"osd":4,"last update":"Mon Oct 9 10:57:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.92200000000000004,"15min":1.2589999999999999},"min":{"1min":0.27900000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.0170000000000003,"5min":8.2940000000000005,"15min":16.695},"last":3.2930000000000001},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.79900000000000004,"15min":1.137},"min":{"1min":0.30299999999999999,"5min":0.28199999999999997,"15min":0.26200000000000001},"max":{"1min":2.222,"5min":7.9320000000000004,"15min":11.109999999999999},"last":2.323}]},{"osd":5,"last update":"Mon Oct 9 10:57:41 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.77900000000000003,"15min":1.1779999999999999},"min":{"1min":0.307,"5min":0.247,"15min":0.247},"max":{"1min":2.181,"5min":4.5999999999999996,"15min":13.262},"last":2.6309999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.81899999999999995,"15min":1.343},"min":{"1min":0.32700000000000001,"5min":0.318,"15min":0.29599999999999999},"max":{"1min":1.173,"5min":8.4339999999999993,"15min":11.741},"last":3.4950000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:57:41 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.98099999999999998,"15min":1.462},"min":{"1min":0.34599999999999997,"5min":0.315,"15min":0.28199999999999997},"max":{"1min":1.6459999999999999,"5min":8.3610000000000007,"15min":14.913},"last":3.4100000000000001},{"interface":"front","average":{"1min":0.747,"5min":0.91600000000000004,"15min":1.319},"min":{"1min":0.38300000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.2549999999999999,"5min":7.9809999999999999,"15min":19.187999999999999},"last":0.57699999999999996}]}]},{"osd":6,"up_from":44,"seq":188978561314,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2166960,"kb_used_data":1262696,"kb_used_omap":0,"kb_used_meta":904256,"kb_avail":91578192,"statfs":{"total":95995035648,"available":93776068608,"internally_reserved":0,"allocated":1293000704,"data_stored":1291812773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925958144},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:00 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":2.1469999999999998,"15min":1.6120000000000001},"min":{"1min":0.27500000000000002,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":5.0019999999999998,"5min":54.743000000000002,"15min":54.743000000000002},"last":0.375},{"interface":"front","average":{"1min":1.2370000000000001,"5min":2.0110000000000001,"15min":1.6359999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":4.9119999999999999,"5min":53.701000000000001,"15min":53.701000000000001},"last":0.29499999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:58:14 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":1.474,"15min":1.4770000000000001},"min":{"1min":0.44,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.96,"5min":7.6820000000000004,"15min":16.747},"last":0.255},{"interface":"front","average":{"1min":1.869,"5min":1.5229999999999999,"15min":1.534},"min":{"1min":0.376,"5min":0.26400000000000001,"15min":0.23100000000000001},"max":{"1min":8.6419999999999995,"5min":8.6419999999999995,"15min":16.734000000000002},"last":2.9940000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":1.6459999999999999,"15min":1.5660000000000001},"min":{"1min":0.34699999999999998,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":12.003,"5min":12.003,"15min":13.208},"last":3.0369999999999999},{"interface":"front","average":{"1min":0.79300000000000004,"5min":1.546,"15min":1.651},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28399999999999997},"max":{"1min":5.0199999999999996,"5min":12.83,"15min":12.83},"last":1.8260000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:58:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3819999999999999,"5min":1.736,"15min":1.629},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.255},"max":{"1min":5.2969999999999997,"5min":7.8680000000000003,"15min":12.391},"last":3.3279999999999998},{"interface":"front","average":{"1min":1.738,"5min":1.984,"15min":1.895},"min":{"1min":0.40000000000000002,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":5.3949999999999996,"5min":8.782,"15min":13.179},"last":3.0640000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.80200000000000005,"15min":1.171},"min":{"1min":0.26000000000000001,"5min":0.24199999999999999,"15min":0.23200000000000001},"max":{"1min":0.66400000000000003,"5min":16.978999999999999,"15min":16.978999999999999},"last":0.62},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.81799999999999995,"15min":1.1459999999999999},"min":{"1min":0.249,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.0830000000000002,"5min":4.7930000000000001,"15min":12.481999999999999},"last":0.41199999999999998}]},{"osd":5,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.212,"15min":1.331},"min":{"1min":0.307,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.7279999999999998,"5min":12.923,"15min":12.923},"last":0.434},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.92700000000000005,"15min":1.2330000000000001},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.0559999999999992,"5min":9.0559999999999992,"15min":12.456},"last":0.60499999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:57:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.0329999999999999,"15min":1.3480000000000001},"min":{"1min":0.35999999999999999,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":1.1930000000000001,"5min":12.882,"15min":23.358000000000001},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.92000000000000004,"15min":1.147},"min":{"1min":0.36099999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.141,"5min":6.1029999999999998,"15min":12.422000000000001},"last":0.69499999999999995}]}]},{"osd":1,"up_from":85,"seq":365072220353,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3760864,"kb_used_data":3270040,"kb_used_omap":9,"kb_used_meta":490806,"kb_avail":89984288,"statfs":{"total":95995035648,"available":92143910912,"internally_reserved":0,"allocated":3348520960,"data_stored":3346707890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":502586345},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,8,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":2.464,"15min":2.0099999999999998},"min":{"1min":0.224,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":4.3049999999999997,"5min":19.992999999999999,"15min":19.992999999999999},"last":0.876},{"interface":"front","average":{"1min":2.4369999999999998,"5min":2.9550000000000001,"15min":2.1899999999999999},"min":{"1min":0.27000000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":6.9790000000000001,"5min":18.303000000000001,"15min":18.303000000000001},"last":0.26200000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:57:37 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":2.2320000000000002,"15min":1.641},"min":{"1min":0.23899999999999999,"5min":0.223,"15min":0.19700000000000001},"max":{"1min":12.587,"5min":14.878,"15min":15.356},"last":5.2460000000000004},{"interface":"front","average":{"1min":2.2240000000000002,"5min":2.109,"15min":1.7749999999999999},"min":{"1min":0.22600000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":8.343,"5min":18.004000000000001,"15min":18.004000000000001},"last":1.4710000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:58:00 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":2.605,"15min":2.23},"min":{"1min":0.23100000000000001,"5min":0.22900000000000001,"15min":0.218},"max":{"1min":8.2080000000000002,"5min":20.036000000000001,"15min":20.036000000000001},"last":0.754},{"interface":"front","average":{"1min":2.827,"5min":2.9809999999999999,"15min":2.2829999999999999},"min":{"1min":0.24099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":8.8680000000000003,"5min":19.640999999999998,"15min":19.640999999999998},"last":3.5819999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":1.5009999999999999,"15min":1.468},"min":{"1min":0.30099999999999999,"5min":0.28899999999999998,"15min":0.255},"max":{"1min":1.71,"5min":14.763,"15min":14.763},"last":3.5550000000000002},{"interface":"front","average":{"1min":1.4099999999999999,"5min":1.75,"15min":1.7190000000000001},"min":{"1min":0.379,"5min":0.30399999999999999,"15min":0.27200000000000002},"max":{"1min":4.0199999999999996,"5min":8.3079999999999998,"15min":14.228999999999999},"last":0.78300000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":1.9670000000000001,"15min":1.631},"min":{"1min":0.376,"5min":0.32700000000000001,"15min":0.253},"max":{"1min":5.9989999999999997,"5min":9.6289999999999996,"15min":9.7720000000000002},"last":2.6720000000000002},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.873,"15min":1.8380000000000001},"min":{"1min":0.313,"5min":0.30299999999999999,"15min":0.28899999999999998},"max":{"1min":2.1930000000000001,"5min":14.833,"15min":17.085000000000001},"last":3.4910000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.8640000000000001,"15min":1.716},"min":{"1min":0.41799999999999998,"5min":0.32600000000000001,"15min":0.27400000000000002},"max":{"1min":5.9630000000000001,"5min":8.8230000000000004,"15min":10.425000000000001},"last":2.7000000000000002},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.653,"15min":1.6479999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.312},"max":{"1min":1.74,"5min":14.804,"15min":14.804},"last":3.5960000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.8180000000000001,"15min":1.7150000000000001},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.34000000000000002},"max":{"1min":2.5619999999999998,"5min":14.856,"15min":16.312000000000001},"last":3.5339999999999998},{"interface":"front","average":{"1min":1.2829999999999999,"5min":1.6839999999999999,"15min":1.7130000000000001},"min":{"1min":0.41099999999999998,"5min":0.35499999999999998,"15min":0.32500000000000001},"max":{"1min":2.8940000000000001,"5min":8.8149999999999995,"15min":14.273999999999999},"last":0.84999999999999998}]}]},{"osd":0,"up_from":80,"seq":343597383887,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2005236,"kb_used_data":1468204,"kb_used_omap":28,"kb_used_meta":536995,"kb_avail":91739916,"statfs":{"total":95995035648,"available":93941673984,"internally_reserved":0,"allocated":1503440896,"data_stored":1501553191,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":549883728},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:58:03 2023","interfaces":[{"interface":"back","average":{"1min":2.254,"5min":2.4079999999999999,"15min":2.0310000000000001},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":12.301,"5min":13.263999999999999,"15min":25.475000000000001},"last":2.8170000000000002},{"interface":"front","average":{"1min":1.8240000000000001,"5min":2.0790000000000002,"15min":1.9339999999999999},"min":{"1min":0.20200000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":6.1680000000000001,"5min":10.478999999999999,"15min":20.832999999999998},"last":0.19500000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:57:37 2023","interfaces":[{"interface":"back","average":{"1min":2.6030000000000002,"5min":2.5409999999999999,"15min":1.9610000000000001},"min":{"1min":0.23599999999999999,"5min":0.189,"15min":0.189},"max":{"1min":7.6420000000000003,"5min":24.852,"15min":24.852},"last":6.484},{"interface":"front","average":{"1min":2.8250000000000002,"5min":3.4740000000000002,"15min":2.3849999999999998},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":12.672000000000001,"5min":77.256,"15min":77.256},"last":7.7430000000000003}]},{"osd":3,"last update":"Mon Oct 9 10:58:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":2.1709999999999998,"15min":2.1509999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20200000000000001},"max":{"1min":8.3510000000000009,"5min":18.972000000000001,"15min":18.972000000000001},"last":8.8550000000000004},{"interface":"front","average":{"1min":2.2160000000000002,"5min":2.8140000000000001,"15min":2.298},"min":{"1min":0.27600000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":12.353999999999999,"5min":13.305,"15min":14.391999999999999},"last":11.385}]},{"osd":4,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":2.8839999999999999,"15min":2.1019999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.26200000000000001},"max":{"1min":6.133,"5min":77.177999999999997,"15min":77.177999999999997},"last":7.7210000000000001},{"interface":"front","average":{"1min":1.532,"5min":1.679,"15min":1.8799999999999999},"min":{"1min":0.32600000000000001,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":6.843,"5min":10.513999999999999,"15min":12.318},"last":0.34300000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":1.766,"15min":1.8999999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.29799999999999999},"max":{"1min":6.9100000000000001,"5min":10.544,"15min":12.157},"last":0.31900000000000001},{"interface":"front","average":{"1min":1.3740000000000001,"5min":1.76,"15min":1.796},"min":{"1min":0.39400000000000002,"5min":0.29899999999999999,"15min":0.29499999999999998},"max":{"1min":6.7469999999999999,"5min":24.754999999999999,"15min":24.754999999999999},"last":2.8559999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.96,"15min":2.0329999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.30399999999999999},"max":{"1min":6.8840000000000003,"5min":18.920000000000002,"15min":18.920000000000002},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.47,"5min":2,"15min":1.714},"min":{"1min":0.38,"5min":0.32900000000000001,"15min":0.307},"max":{"1min":6.7130000000000001,"5min":24.800000000000001,"15min":24.800000000000001},"last":2.8809999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.8839999999999999,"15min":1.909},"min":{"1min":0.44900000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.6609999999999996,"5min":13.225,"15min":14.48},"last":2.9049999999999998},{"interface":"front","average":{"1min":1.7470000000000001,"5min":2.8479999999999999,"15min":2.214},"min":{"1min":0.38,"5min":0.314,"15min":0.314},"max":{"1min":6.101,"5min":77.228999999999999,"15min":77.228999999999999},"last":7.6799999999999997}]}]},{"osd":2,"up_from":90,"seq":386547056827,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1480884,"kb_used_data":1077164,"kb_used_omap":81,"kb_used_meta":403630,"kb_avail":92264268,"statfs":{"total":95995035648,"available":94478610432,"internally_reserved":0,"allocated":1103015936,"data_stored":1101296894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":413317338},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":1.8939999999999999,"15min":1.659},"min":{"1min":0.21099999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":5.6280000000000001,"5min":17.574000000000002,"15min":18.512},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.02,"5min":1.927,"15min":1.762},"min":{"1min":0.255,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":3.754,"5min":15.595000000000001,"15min":17.975000000000001},"last":1.6279999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.6799999999999999,"15min":1.702},"min":{"1min":0.317,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":3.7250000000000001,"5min":16.359000000000002,"15min":16.359000000000002},"last":1.6000000000000001},{"interface":"front","average":{"1min":2.036,"5min":1.855,"15min":1.5669999999999999},"min":{"1min":0.246,"5min":0.24199999999999999,"15min":0.20699999999999999},"max":{"1min":6.3659999999999997,"5min":12.693,"15min":16.454999999999998},"last":3.0209999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:58:00 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":2.355,"15min":1.839},"min":{"1min":0.24399999999999999,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":3.6349999999999998,"5min":19.539000000000001,"15min":19.539000000000001},"last":2.6779999999999999},{"interface":"front","average":{"1min":2.2410000000000001,"5min":2.4169999999999998,"15min":2.1499999999999999},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.20200000000000001},"max":{"1min":8.8729999999999993,"5min":16.702999999999999,"15min":16.702999999999999},"last":1.8899999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.9610000000000001,"5min":2.0099999999999998,"15min":1.8109999999999999},"min":{"1min":0.372,"5min":0.28599999999999998,"15min":0.254},"max":{"1min":6.407,"5min":15.672000000000001,"15min":16.231999999999999},"last":3.0720000000000001},{"interface":"front","average":{"1min":1.425,"5min":1.7589999999999999,"15min":1.7130000000000001},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.26300000000000001},"max":{"1min":5.6680000000000001,"5min":15.696999999999999,"15min":15.696999999999999},"last":1.2969999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":1.46,"15min":1.6399999999999999},"min":{"1min":0.375,"5min":0.26800000000000002,"15min":0.245},"max":{"1min":2.7850000000000001,"5min":12.128,"15min":17.920000000000002},"last":1.615},{"interface":"front","average":{"1min":1.9790000000000001,"5min":2.0569999999999999,"15min":1.7569999999999999},"min":{"1min":0.33900000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.4320000000000004,"5min":16.597000000000001,"15min":17.882999999999999},"last":3.0960000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":1.4950000000000001,"15min":1.5009999999999999},"min":{"1min":0.39500000000000002,"5min":0.33400000000000002,"15min":0.247},"max":{"1min":5.6929999999999996,"5min":8.8460000000000001,"15min":14.029999999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.83099999999999996,"5min":1.4550000000000001,"15min":1.605},"min":{"1min":0.36799999999999999,"5min":0.34200000000000003,"15min":0.29199999999999998},"max":{"1min":2.4889999999999999,"5min":10.111000000000001,"15min":12.862},"last":1.651}]},{"osd":7,"last update":"Mon Oct 9 10:57:38 2023","interfaces":[{"interface":"back","average":{"1min":1.375,"5min":1.5149999999999999,"15min":1.5229999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.32500000000000001},"max":{"1min":6.0860000000000003,"5min":8.8879999999999999,"15min":13.999000000000001},"last":2.7109999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":1.5609999999999999,"15min":1.6819999999999999},"min":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.307},"max":{"1min":1.1839999999999999,"5min":10.157999999999999,"15min":12.826000000000001},"last":4.2839999999999998}]}]},{"osd":3,"up_from":95,"seq":408021893301,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3618048,"kb_used_data":2873144,"kb_used_omap":7,"kb_used_meta":744888,"kb_avail":90127104,"statfs":{"total":95995035648,"available":92290154496,"internally_reserved":0,"allocated":2942099456,"data_stored":2940191199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":762765928},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":3.399,"15min":2.2930000000000001},"min":{"1min":0.23499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":10.140000000000001,"5min":38.817999999999998,"15min":38.817999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":1.776,"5min":3.661,"15min":2.5840000000000001},"min":{"1min":0.16500000000000001,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":7.6580000000000004,"5min":17.312000000000001,"15min":17.312000000000001},"last":0.42999999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":2.5990000000000002,"15min":2.3530000000000002},"min":{"1min":0.28000000000000003,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":13.244999999999999,"5min":22.469000000000001,"15min":22.469000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.52,"5min":3.4020000000000001,"15min":2.4409999999999998},"min":{"1min":0.28999999999999998,"5min":0.22,"15min":0.193},"max":{"1min":13.359,"5min":18.526,"15min":18.526},"last":0.34200000000000003}]},{"osd":2,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":2.7559999999999998,"15min":2.214},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.6260000000000003,"5min":20.402000000000001,"15min":20.402000000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.3680000000000001,"5min":3.637,"15min":2.4900000000000002},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":4.4320000000000004,"5min":27.538,"15min":27.538},"last":0.64100000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":2.4180000000000001,"15min":1.992},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.8089999999999993,"5min":11.863,"15min":20.388000000000002},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.22,"5min":2.2490000000000001,"15min":1.9239999999999999},"min":{"1min":0.34000000000000002,"5min":0.28399999999999997,"15min":0.24299999999999999},"max":{"1min":4.0529999999999999,"5min":11.680999999999999,"15min":19.085999999999999},"last":0.55300000000000005}]},{"osd":5,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":2.3290000000000002,"15min":2.2029999999999998},"min":{"1min":0.39300000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.0600000000000001,"5min":14.045,"15min":51.509999999999998},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.8839999999999999,"15min":2.1680000000000001},"min":{"1min":0.41899999999999998,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":4.3780000000000001,"5min":17.311,"15min":55.338999999999999},"last":0.51900000000000002}]},{"osd":6,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":2.335,"15min":1.929},"min":{"1min":0.35599999999999998,"5min":0.29199999999999998,"15min":0.28000000000000003},"max":{"1min":3.1120000000000001,"5min":11.723000000000001,"15min":12.361000000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.306,"5min":2.036,"15min":2.0819999999999999},"min":{"1min":0.39500000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.5229999999999997,"5min":17.295000000000002,"15min":17.295000000000002},"last":0.48799999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":2.2269999999999999,"15min":2.3140000000000001},"min":{"1min":0.439,"5min":0.34999999999999998,"15min":0.31},"max":{"1min":4.3330000000000002,"5min":18.722000000000001,"15min":51.744},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.5609999999999999,"5min":2.5259999999999998,"15min":2.335},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.8569999999999993,"5min":13.337,"15min":53.399000000000001},"last":0.374}]}]},{"osd":4,"up_from":30,"seq":128849019177,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1994300,"kb_used_data":1097460,"kb_used_omap":0,"kb_used_meta":896832,"kb_avail":91750852,"statfs":{"total":95995035648,"available":93952872448,"internally_reserved":0,"allocated":1123799040,"data_stored":1122599220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918355968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:57:55 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":4.0410000000000004,"15min":2.202},"min":{"1min":0.224,"5min":0.224,"15min":0.20999999999999999},"max":{"1min":4.2629999999999999,"5min":213.88999999999999,"15min":213.88999999999999},"last":4.9900000000000002},{"interface":"front","average":{"1min":0.91800000000000004,"5min":3.9580000000000002,"15min":2.4980000000000002},"min":{"1min":0.26100000000000001,"5min":0.23200000000000001,"15min":0.217},"max":{"1min":3.012,"5min":214.09800000000001,"15min":214.09800000000001},"last":1.944}]},{"osd":1,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":3.7559999999999998,"15min":2.2149999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":3.286,"5min":206.50399999999999,"15min":206.50399999999999},"last":5.101},{"interface":"front","average":{"1min":0.99099999999999999,"5min":3.7839999999999998,"15min":2.419},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.2959999999999998,"5min":206.529,"15min":206.529},"last":4.0359999999999996}]},{"osd":2,"last update":"Mon Oct 9 10:57:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.7450000000000001,"15min":1.653},"min":{"1min":0.28000000000000003,"5min":0.25900000000000001,"15min":0.253},"max":{"1min":5.0759999999999996,"5min":15.446999999999999,"15min":15.446999999999999},"last":5.3929999999999998},{"interface":"front","average":{"1min":1.4570000000000001,"5min":2.2240000000000002,"15min":1.911},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.26300000000000001},"max":{"1min":5.9210000000000003,"5min":27.818000000000001,"15min":27.818000000000001},"last":5.3659999999999997}]},{"osd":3,"last update":"Mon Oct 9 10:57:55 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":3.8559999999999999,"15min":2.4500000000000002},"min":{"1min":0.374,"5min":0.34599999999999997,"15min":0.27400000000000002},"max":{"1min":2.7789999999999999,"5min":209.03700000000001,"15min":209.03700000000001},"last":1.958},{"interface":"front","average":{"1min":1.3580000000000001,"5min":4.1760000000000002,"15min":2.6930000000000001},"min":{"1min":0.39600000000000002,"5min":0.30499999999999999,"15min":0.29999999999999999},"max":{"1min":5.1529999999999996,"5min":208.80199999999999,"15min":208.80199999999999},"last":5.4119999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":1.1120000000000001,"15min":1.577},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.224},"max":{"1min":0.77800000000000002,"5min":19.068999999999999,"15min":24.149999999999999},"last":1.3049999999999999},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.05,"15min":1.3979999999999999},"min":{"1min":0.28499999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.8360000000000001,"5min":15.535,"15min":20.422000000000001},"last":5.3319999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:57:35 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.93799999999999994,"15min":1.2370000000000001},"min":{"1min":0.31900000000000001,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":1,"5min":15.739000000000001,"15min":15.739000000000001},"last":4.944},{"interface":"front","average":{"1min":0.50900000000000001,"5min":1.1060000000000001,"15min":1.466},"min":{"1min":0.35299999999999998,"5min":0.32700000000000001,"15min":0.311},"max":{"1min":0.876,"5min":14.093999999999999,"15min":14.272},"last":3.9710000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:57:42 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":1.1719999999999999,"15min":1.393},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28299999999999997},"max":{"1min":1.355,"5min":14.458,"15min":14.458},"last":4.0030000000000001},{"interface":"front","average":{"1min":0.68500000000000005,"5min":1.3360000000000001,"15min":1.304},"min":{"1min":0.35799999999999998,"5min":0.33800000000000002,"15min":0.255},"max":{"1min":1.8899999999999999,"5min":15.592000000000001,"15min":15.592000000000001},"last":2.2549999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790246,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4239468,"kb_used_data":3324452,"kb_used_omap":0,"kb_used_meta":915008,"kb_avail":89505684,"statfs":{"total":95995035648,"available":91653820416,"internally_reserved":0,"allocated":3404238848,"data_stored":3403051377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":936968192},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.927,"15min":1.653},"min":{"1min":0.248,"5min":0.215,"15min":0.215},"max":{"1min":3.9590000000000001,"5min":16.449000000000002,"15min":16.449000000000002},"last":1.2270000000000001},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.9770000000000001,"15min":1.7090000000000001},"min":{"1min":0.30499999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.984,"5min":16.489000000000001,"15min":16.489000000000001},"last":1.26}]},{"osd":1,"last update":"Mon Oct 9 10:58:15 2023","interfaces":[{"interface":"back","average":{"1min":4.2359999999999998,"5min":2.5880000000000001,"15min":2.1019999999999999},"min":{"1min":0.313,"5min":0.215,"15min":0.215},"max":{"1min":29.818999999999999,"5min":29.818999999999999,"15min":29.818999999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":3.9409999999999998,"5min":2.3029999999999999,"15min":1.804},"min":{"1min":0.28199999999999997,"5min":0.253,"15min":0.24299999999999999},"max":{"1min":29.600000000000001,"5min":29.600000000000001,"15min":29.600000000000001},"last":4.407}]},{"osd":2,"last update":"Mon Oct 9 10:57:33 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":2.0059999999999998,"15min":1.829},"min":{"1min":0.33800000000000002,"5min":0.28000000000000003,"15min":0.27800000000000002},"max":{"1min":8.0399999999999991,"5min":11.257999999999999,"15min":11.393000000000001},"last":4.7050000000000001},{"interface":"front","average":{"1min":1.7,"5min":1.976,"15min":1.6439999999999999},"min":{"1min":0.28899999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.093,"5min":9.2569999999999997,"15min":15.087999999999999},"last":1.7490000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:58:02 2023","interfaces":[{"interface":"back","average":{"1min":2.3999999999999999,"5min":2.165,"15min":2.206},"min":{"1min":0.32100000000000001,"5min":0.314,"15min":0.25700000000000001},"max":{"1min":25.846,"5min":25.846,"15min":63.508000000000003},"last":1.613},{"interface":"front","average":{"1min":2.7050000000000001,"5min":2.5169999999999999,"15min":2.3860000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.26300000000000001},"max":{"1min":25.073,"5min":25.073,"15min":62.418999999999997},"last":0.32900000000000001}]},{"osd":4,"last update":"Mon Oct 9 10:58:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.68600000000000005,"15min":1.395},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23200000000000001},"max":{"1min":2.726,"5min":4.4989999999999997,"15min":123.241},"last":1.2909999999999999},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.93200000000000005,"15min":1.6339999999999999},"min":{"1min":0.29199999999999998,"5min":0.28000000000000003,"15min":0.24199999999999999},"max":{"1min":4.8840000000000003,"5min":4.8840000000000003,"15min":123.318},"last":0.36699999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:58:25 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":1.0880000000000001,"15min":1.3520000000000001},"min":{"1min":0.28899999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.1509999999999998,"5min":8.9049999999999994,"15min":11.098000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.79700000000000004,"15min":1.2290000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.1469999999999998,"5min":6.7210000000000001,"15min":20.256},"last":3.6190000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:57:49 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":1.085,"15min":1.3169999999999999},"min":{"1min":0.33900000000000002,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":2.968,"5min":8.468,"15min":10.739000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.82899999999999996,"15min":1.139},"min":{"1min":0.315,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.327,"5min":8.734,"15min":14.154},"last":4.2599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1508311040,"data_stored":1508311040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3353083904,"data_stored":3353083904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1101660160,"data_stored":1101660160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2944532480,"data_stored":2944532480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1121845248,"data_stored":1121845248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3348627456,"data_stored":3348627456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1106116608,"data_stored":1106116608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3342336000,"data_stored":3342336000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:58:38.414 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T10:58:38.414 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:58:38.414 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T10:58:38.414 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:58:38.414 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:58:38.414 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T10:58:38.415 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:38.453 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:38.477 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:38.511 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:38.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:38.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:38.618 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:38.659 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:38.660 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 74 16 96925 96909 81.7959 55 0.0372759 0.0122204 2023-10-09T10:58:38.702 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:38.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:38.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:38.781 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:58:38.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:38.854 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:38.925 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:38 smithi155 ceph-mon[184854]: pgmap v875: 119 pgs: 119 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T10:58:38.926 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:38.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:38.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:39.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:39.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:38 smithi125 ceph-mon[195507]: pgmap v875: 119 pgs: 119 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T10:58:39.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:38 smithi125 ceph-mon[200197]: pgmap v875: 119 pgs: 119 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T10:58:39.091 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:39.174 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:39.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:39.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:39.214 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:39.280 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:39.331 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:39.660 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 75 16 97730 97714 81.3757 50.3125 0.0197409 0.0122841 2023-10-09T10:58:39.695 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:58:39.753 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:39.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:39.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:39.859 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:39.917 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:39 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2202209618' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:39.917 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:39 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:39.929 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:39.960 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:39.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:39.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:40.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:39 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2202209618' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:40.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:39 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:40.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:39 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2202209618' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:40.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:39 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:40.074 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:40.129 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:40.164 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:40.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:40.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:40.245 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:58:40.245 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:58:40.272 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:40.320 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:40.406 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:40.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:40.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:40.511 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:40.525 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:58:40.557 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:40.564 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:58:40.596 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:58:40.645 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:40.648 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:58:40.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:40.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:40.666 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:58:40.669 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 76 16 98589 98573 81.0019 53.6875 0.00970096 0.0123406 2023-10-09T10:58:40.688 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:58:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:40 smithi155 ceph-mon[184854]: pgmap v876: 119 pgs: 119 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-09T10:58:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:40 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:40 smithi155 ceph-mon[184854]: failed to encode map e353 with expected crc 2023-10-09T10:58:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:40 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:58:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:40 smithi155 ceph-mon[184854]: osdmap e353: 8 total, 8 up, 8 in 2023-10-09T10:58:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:40 smithi155 ceph-mon[184854]: failed to encode map e353 with expected crc 2023-10-09T10:58:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:40 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2757912049' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:40.725 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:58:40.753 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:40.754 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:58:40.786 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:58:40.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:58:40.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:58:40.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:58:40.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:58:40.791 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:58:40.791 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:58:40.791 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:58:40.791 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:58:40.802 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:40 smithi125 ceph-mon[195507]: pgmap v876: 119 pgs: 119 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-09T10:58:40.802 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:40 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:40.802 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:40 smithi125 ceph-mon[195507]: failed to encode map e353 with expected crc 2023-10-09T10:58:40.803 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:40 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:58:40.803 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:40 smithi125 ceph-mon[195507]: osdmap e353: 8 total, 8 up, 8 in 2023-10-09T10:58:40.803 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:40 smithi125 ceph-mon[195507]: failed to encode map e353 with expected crc 2023-10-09T10:58:40.803 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:40 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2757912049' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:40.803 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:40 smithi125 ceph-mon[200197]: pgmap v876: 119 pgs: 119 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-09T10:58:40.803 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:40 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:40.803 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:40 smithi125 ceph-mon[200197]: failed to encode map e353 with expected crc 2023-10-09T10:58:40.803 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:40 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:58:40.803 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:40 smithi125 ceph-mon[200197]: osdmap e353: 8 total, 8 up, 8 in 2023-10-09T10:58:40.803 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:40 smithi125 ceph-mon[200197]: failed to encode map e353 with expected crc 2023-10-09T10:58:40.804 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:40 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2757912049' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:40.812 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:40.915 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:40.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:40.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:41.020 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:58:41.079 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:41.140 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:58:41.255 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:58:41.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:41.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:58:41.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:58:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:58:41] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2023-10-09T10:58:41.367 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:41.414 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:41.557 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:58:41.557 INFO:teuthology.orchestra.run.smithi125.stdout: "success": "" 2023-10-09T10:58:41.557 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:58:41.618 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:41.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:41.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:41.670 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 77 16 99445 99429 80.6449 53.5 0.021939 0.0123957 2023-10-09T10:58:41.721 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:41.784 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:41.816 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:41.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:41.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:41.921 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:41.981 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:42.016 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:42.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:42.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:42.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:42.183 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:42.213 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:42.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:42.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:42.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:42.357 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:58:42.357 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:58:42.357 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:58:42.357 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:58:42.357 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:58:42.368 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:42.397 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:42.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:42.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:42.500 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:42.562 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:42.671 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 78 16 100371 100355 80.3517 57.875 0.0110463 0.012441 2023-10-09T10:58:42.712 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:58:42.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:42.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:42.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:42.786 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:58:42.870 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:42.912 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:42.956 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:42.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:42.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:42.962 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:42 smithi125 ceph-mon[195507]: pgmap v878: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s 2023-10-09T10:58:42.963 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:42 smithi125 ceph-mon[200197]: pgmap v878: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s 2023-10-09T10:58:42.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:42 smithi155 ceph-mon[184854]: pgmap v878: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s 2023-10-09T10:58:43.063 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:43.128 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:43.145 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:43.156 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:43.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:43.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:43.234 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:58:43.263 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:43.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:58:43.314 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:43.401 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:43.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:43.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:43.508 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:43.555 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:43.588 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:43.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:43.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:43.676 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 79 16 101209 101193 79.9948 52.375 0.016156 0.0124965 2023-10-09T10:58:43.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:43.740 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:43.877 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:43.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:43.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:43.989 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:58:44.051 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:44.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:44 smithi125 ceph-mon[195507]: pgmap v879: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2023-10-09T10:58:44.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:44 smithi125 ceph-mon[200197]: pgmap v879: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2023-10-09T10:58:44.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:44 smithi155 ceph-mon[184854]: pgmap v879: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2023-10-09T10:58:44.674 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T10:58:44.675105+0000 min lat: 0.00120203 max lat: 0.143356 avg lat: 0.0125456 2023-10-09T10:58:44.674 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T10:58:44.674 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 80 16 102088 102072 79.6815 54.9375 0.0154803 0.0125456 2023-10-09T10:58:44.976 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:58:44.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:44.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:58:45.085 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:45.090 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:45.160 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:45.187 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:45.190 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:58:45.190 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:58:45.191 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:58:45.191 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:58:45.191 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.191 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:95197 7.d 7:b9b00ad5:::benchmark_data_smithi125_295816_object4060:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-10-09T10:58:45.191 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:57.383916+0000", 2023-10-09T10:58:45.191 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 467.79344053099999, 2023-10-09T10:58:45.191 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.070851236999999997, 2023-10-09T10:58:45.191 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.191 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.191 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:45.191 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:58:45.191 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:58:45.192 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 95197 2023-10-09T10:58:45.192 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.192 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.192 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.192 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.192 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383916+0000", 2023-10-09T10:58:45.192 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.192 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.192 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.192 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.192 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383916+0000", 2023-10-09T10:58:45.192 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.193 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.193 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.193 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.193 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383915+0000", 2023-10-09T10:58:45.193 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:58:45.193 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.193 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.193 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.193 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383920+0000", 2023-10-09T10:58:45.193 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.84e-06 2023-10-09T10:58:45.193 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.193 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.194 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.194 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383931+0000", 2023-10-09T10:58:45.194 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0657e-05 2023-10-09T10:58:45.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.194 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.194 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383938+0000", 2023-10-09T10:58:45.194 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5139999999999998e-06 2023-10-09T10:58:45.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.195 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.411508+0000", 2023-10-09T10:58:45.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027570921000000002 2023-10-09T10:58:45.195 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:58:45.195 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.416286+0000", 2023-10-09T10:58:45.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0047777430000000001 2023-10-09T10:58:45.195 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.196 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.437950+0000", 2023-10-09T10:58:45.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021664251999999998 2023-10-09T10:58:45.196 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.196 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.196 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.196 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.438046+0000", 2023-10-09T10:58:45.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5459999999999997e-05 2023-10-09T10:58:45.196 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.196 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.196 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:58:45.196 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.438186+0000", 2023-10-09T10:58:45.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014000800000000001 2023-10-09T10:58:45.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.197 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:45.197 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.443989+0000", 2023-10-09T10:58:45.197 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0058031369999999999 2023-10-09T10:58:45.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.197 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.197 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.452036+0000", 2023-10-09T10:58:45.197 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0080469280000000001 2023-10-09T10:58:45.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.198 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.198 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.452814+0000", 2023-10-09T10:58:45.198 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00077831199999999997 2023-10-09T10:58:45.198 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.198 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.198 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.198 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.452840+0000", 2023-10-09T10:58:45.198 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5947000000000001e-05 2023-10-09T10:58:45.198 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.198 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.199 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.199 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.454767+0000", 2023-10-09T10:58:45.199 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019270769999999999 2023-10-09T10:58:45.199 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.199 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.199 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.199 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.199 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.199 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:95787 7.d 7:b6e713eb:::benchmark_data_smithi125_295816_object4650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:58:45.199 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:58.300479+0000", 2023-10-09T10:58:45.199 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 466.87687809800002, 2023-10-09T10:58:45.199 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072656459000000007, 2023-10-09T10:58:45.199 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.200 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.200 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:45.200 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:58:45.200 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:58:45.200 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 95787 2023-10-09T10:58:45.200 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.200 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.200 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.200 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.200 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300479+0000", 2023-10-09T10:58:45.200 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.200 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.201 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300479+0000", 2023-10-09T10:58:45.201 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.201 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.201 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300477+0000", 2023-10-09T10:58:45.201 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:58:45.201 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300487+0000", 2023-10-09T10:58:45.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0638999999999999e-05 2023-10-09T10:58:45.202 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.202 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.202 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300511+0000", 2023-10-09T10:58:45.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3059999999999999e-05 2023-10-09T10:58:45.202 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.202 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.202 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300518+0000", 2023-10-09T10:58:45.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0029999999999997e-06 2023-10-09T10:58:45.202 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.203 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.203 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.203 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.322068+0000", 2023-10-09T10:58:45.203 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021550625 2023-10-09T10:58:45.203 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.203 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.203 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:58:45.203 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.333963+0000", 2023-10-09T10:58:45.203 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011895039 2023-10-09T10:58:45.203 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.203 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.203 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.203 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.351448+0000", 2023-10-09T10:58:45.204 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017484831999999999 2023-10-09T10:58:45.204 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.204 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.204 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.204 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.352038+0000", 2023-10-09T10:58:45.204 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00059005600000000004 2023-10-09T10:58:45.204 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.204 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.204 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:58:45.204 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.352181+0000", 2023-10-09T10:58:45.204 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014277800000000001 2023-10-09T10:58:45.204 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.205 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.205 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.205 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.367667+0000", 2023-10-09T10:58:45.205 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015486592 2023-10-09T10:58:45.205 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.205 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.205 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:45.205 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.370532+0000", 2023-10-09T10:58:45.205 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028642279999999999 2023-10-09T10:58:45.205 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.205 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.205 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.206 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.371142+0000", 2023-10-09T10:58:45.206 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00060984699999999995 2023-10-09T10:58:45.206 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.206 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.206 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.206 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.371180+0000", 2023-10-09T10:58:45.206 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8529000000000003e-05 2023-10-09T10:58:45.206 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.206 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.206 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.206 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.373135+0000", 2023-10-09T10:58:45.207 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001954973 2023-10-09T10:58:45.207 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.207 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.207 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.207 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.207 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.207 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96817 7.d 7:b9f7ad08:::benchmark_data_smithi125_295816_object5680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:58:45.207 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:59.884613+0000", 2023-10-09T10:58:45.207 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 465.29274364499997, 2023-10-09T10:58:45.207 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14656745900000001, 2023-10-09T10:58:45.207 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.207 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.207 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:45.208 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:58:45.208 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:58:45.208 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96817 2023-10-09T10:58:45.208 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.208 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.208 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.208 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.208 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884613+0000", 2023-10-09T10:58:45.208 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.208 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.208 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.208 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.209 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884613+0000", 2023-10-09T10:58:45.209 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.209 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.209 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.209 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.209 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884612+0000", 2023-10-09T10:58:45.209 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:58:45.209 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.209 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.209 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.209 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884618+0000", 2023-10-09T10:58:45.209 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5880000000000003e-06 2023-10-09T10:58:45.209 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.210 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.210 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.210 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884633+0000", 2023-10-09T10:58:45.210 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5109999999999999e-05 2023-10-09T10:58:45.210 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.210 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.210 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.210 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884639+0000", 2023-10-09T10:58:45.210 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.164e-06 2023-10-09T10:58:45.210 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.210 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.211 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.211 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.887892+0000", 2023-10-09T10:58:45.211 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032536610000000001 2023-10-09T10:58:45.211 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.211 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.211 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:58:45.211 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.888551+0000", 2023-10-09T10:58:45.211 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00065867400000000004 2023-10-09T10:58:45.211 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.211 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.211 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.211 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910766+0000", 2023-10-09T10:58:45.212 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022214655 2023-10-09T10:58:45.212 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.212 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.212 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.212 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910852+0000", 2023-10-09T10:58:45.212 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6551999999999994e-05 2023-10-09T10:58:45.212 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.212 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.212 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:58:45.212 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910980+0000", 2023-10-09T10:58:45.212 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000127704 2023-10-09T10:58:45.212 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.914262+0000", 2023-10-09T10:58:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032824070000000002 2023-10-09T10:58:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.020644+0000", 2023-10-09T10:58:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.106381932 2023-10-09T10:58:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.029029+0000", 2023-10-09T10:58:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0083848010000000008 2023-10-09T10:58:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.029088+0000", 2023-10-09T10:58:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8641999999999999e-05 2023-10-09T10:58:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.031180+0000", 2023-10-09T10:58:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020928359999999998 2023-10-09T10:58:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96884 7.d 7:b183b169:::benchmark_data_smithi125_295816_object5747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:58:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.113486+0000", 2023-10-09T10:58:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 465.06387035500001, 2023-10-09T10:58:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.082588165000000005, 2023-10-09T10:58:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:58:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:58:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96884 2023-10-09T10:58:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113486+0000", 2023-10-09T10:58:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113486+0000", 2023-10-09T10:58:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113485+0000", 2023-10-09T10:58:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:58:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113491+0000", 2023-10-09T10:58:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3009999999999997e-06 2023-10-09T10:58:45.218 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.218 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.218 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.218 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113501+0000", 2023-10-09T10:58:45.218 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0342e-05 2023-10-09T10:58:45.218 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.218 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.218 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.218 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113505+0000", 2023-10-09T10:58:45.218 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4530000000000004e-06 2023-10-09T10:58:45.218 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.218 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.218 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.219 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.117310+0000", 2023-10-09T10:58:45.219 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0038049329999999999 2023-10-09T10:58:45.219 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.219 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.219 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:58:45.219 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.117924+0000", 2023-10-09T10:58:45.219 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00061396699999999998 2023-10-09T10:58:45.219 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.219 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.219 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.220 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.140859+0000", 2023-10-09T10:58:45.220 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022935140999999999 2023-10-09T10:58:45.220 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.220 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.220 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.220 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.140963+0000", 2023-10-09T10:58:45.220 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010341 2023-10-09T10:58:45.220 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.220 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.220 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:58:45.220 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.141120+0000", 2023-10-09T10:58:45.220 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000157405 2023-10-09T10:58:45.221 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.221 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.221 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:45.221 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.150528+0000", 2023-10-09T10:58:45.221 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0094081660000000008 2023-10-09T10:58:45.221 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.221 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.221 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.221 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.160789+0000", 2023-10-09T10:58:45.221 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010260429 2023-10-09T10:58:45.221 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.221 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.221 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.221 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.194166+0000", 2023-10-09T10:58:45.222 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.033377636000000002 2023-10-09T10:58:45.222 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.222 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.222 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.222 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.194226+0000", 2023-10-09T10:58:45.222 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9855999999999997e-05 2023-10-09T10:58:45.222 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.222 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.222 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.222 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196074+0000", 2023-10-09T10:58:45.222 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001848192 2023-10-09T10:58:45.223 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.223 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.223 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.223 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.223 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.223 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96892 7.d 7:bd59fbdb:::benchmark_data_smithi125_295816_object5755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:58:45.223 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.148429+0000", 2023-10-09T10:58:45.223 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 465.028928103, 2023-10-09T10:58:45.223 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.089082649999999999, 2023-10-09T10:58:45.223 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.223 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.223 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:45.224 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:58:45.224 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:58:45.224 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96892 2023-10-09T10:58:45.224 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.224 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.224 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.224 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.224 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148429+0000", 2023-10-09T10:58:45.224 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.224 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.224 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.224 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.225 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148429+0000", 2023-10-09T10:58:45.225 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.225 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.225 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.225 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.225 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148428+0000", 2023-10-09T10:58:45.225 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967296 2023-10-09T10:58:45.225 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.225 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.225 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.225 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148430+0000", 2023-10-09T10:58:45.225 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5570000000000001e-06 2023-10-09T10:58:45.226 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.226 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.226 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.226 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148433+0000", 2023-10-09T10:58:45.226 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.892e-06 2023-10-09T10:58:45.226 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.226 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.226 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.226 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148435+0000", 2023-10-09T10:58:45.226 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5339999999999998e-06 2023-10-09T10:58:45.226 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.226 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.227 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.227 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.152891+0000", 2023-10-09T10:58:45.227 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0044553409999999998 2023-10-09T10:58:45.227 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.227 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.227 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:58:45.227 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.153532+0000", 2023-10-09T10:58:45.227 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000641539 2023-10-09T10:58:45.227 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.227 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.227 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.227 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.182585+0000", 2023-10-09T10:58:45.228 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029053295 2023-10-09T10:58:45.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.228 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.228 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.228 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.182682+0000", 2023-10-09T10:58:45.228 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6623e-05 2023-10-09T10:58:45.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.228 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.228 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:58:45.228 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.182825+0000", 2023-10-09T10:58:45.228 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014288300000000001 2023-10-09T10:58:45.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:45.229 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.198214+0000", 2023-10-09T10:58:45.229 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015389162 2023-10-09T10:58:45.229 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.229 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.223087+0000", 2023-10-09T10:58:45.229 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024872423000000001 2023-10-09T10:58:45.229 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.230 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.235280+0000", 2023-10-09T10:58:45.230 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012193017 2023-10-09T10:58:45.230 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.230 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.230 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.230 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.235328+0000", 2023-10-09T10:58:45.230 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8211000000000003e-05 2023-10-09T10:58:45.230 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.230 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.230 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.230 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.237511+0000", 2023-10-09T10:58:45.230 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021833960000000002 2023-10-09T10:58:45.231 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.231 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.231 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.231 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.231 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.231 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96898 7.d 7:b4ab753b:::benchmark_data_smithi125_295816_object5761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:58:45.231 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.166362+0000", 2023-10-09T10:58:45.231 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 465.01099460900002, 2023-10-09T10:58:45.231 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072656032999999995, 2023-10-09T10:58:45.231 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.231 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.231 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:45.231 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:58:45.232 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:58:45.232 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96898 2023-10-09T10:58:45.232 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.232 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.232 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.232 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.232 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166362+0000", 2023-10-09T10:58:45.232 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.232 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.232 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.232 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.232 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166362+0000", 2023-10-09T10:58:45.232 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.233 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.233 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.233 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166360+0000", 2023-10-09T10:58:45.233 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:58:45.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.233 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.233 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.233 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166369+0000", 2023-10-09T10:58:45.233 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.166e-06 2023-10-09T10:58:45.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.234 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.234 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.234 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166385+0000", 2023-10-09T10:58:45.234 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5214e-05 2023-10-09T10:58:45.234 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.234 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.234 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.234 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166392+0000", 2023-10-09T10:58:45.234 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1659999999999997e-06 2023-10-09T10:58:45.234 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.234 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.234 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.235 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166446+0000", 2023-10-09T10:58:45.235 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3819000000000001e-05 2023-10-09T10:58:45.235 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.235 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.235 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:58:45.235 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.167178+0000", 2023-10-09T10:58:45.235 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00073199300000000001 2023-10-09T10:58:45.235 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.235 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.235 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.235 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.191480+0000", 2023-10-09T10:58:45.235 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024302082999999999 2023-10-09T10:58:45.236 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.236 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.236 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.236 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.191594+0000", 2023-10-09T10:58:45.236 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000114527 2023-10-09T10:58:45.236 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.236 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.236 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:58:45.236 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.191761+0000", 2023-10-09T10:58:45.236 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000166599 2023-10-09T10:58:45.236 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.236 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:45.237 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.198249+0000", 2023-10-09T10:58:45.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064882439999999998 2023-10-09T10:58:45.237 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.237 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.223116+0000", 2023-10-09T10:58:45.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024867330999999999 2023-10-09T10:58:45.237 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.237 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.237546+0000", 2023-10-09T10:58:45.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014429365 2023-10-09T10:58:45.238 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.238 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.238 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.237573+0000", 2023-10-09T10:58:45.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7534000000000001e-05 2023-10-09T10:58:45.238 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.238 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.238 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.239018+0000", 2023-10-09T10:58:45.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001444776 2023-10-09T10:58:45.239 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.239 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.239 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.239 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.239 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.239 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96901 7.d 7:bd396a29:::benchmark_data_smithi125_295816_object5764:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:58:45.239 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.177994+0000", 2023-10-09T10:58:45.239 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 464.99936278799998, 2023-10-09T10:58:45.239 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.079151244999999995, 2023-10-09T10:58:45.239 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.239 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.240 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:45.240 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:58:45.240 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:58:45.240 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96901 2023-10-09T10:58:45.240 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.240 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.240 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.240 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.240 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177994+0000", 2023-10-09T10:58:45.240 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.240 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.240 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.241 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.241 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177994+0000", 2023-10-09T10:58:45.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.241 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.241 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.241 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.241 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177993+0000", 2023-10-09T10:58:45.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:58:45.241 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.241 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.241 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.241 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177998+0000", 2023-10-09T10:58:45.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6699999999999999e-06 2023-10-09T10:58:45.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.242 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.242 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.242 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.178008+0000", 2023-10-09T10:58:45.242 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6520000000000001e-06 2023-10-09T10:58:45.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.242 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.242 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.242 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.178013+0000", 2023-10-09T10:58:45.242 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.028e-06 2023-10-09T10:58:45.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.242 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.243 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196130+0000", 2023-10-09T10:58:45.243 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018116621999999999 2023-10-09T10:58:45.243 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.243 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:58:45.243 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196670+0000", 2023-10-09T10:58:45.243 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00054042399999999996 2023-10-09T10:58:45.243 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.243 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.243 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.227020+0000", 2023-10-09T10:58:45.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030349832 2023-10-09T10:58:45.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.244 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.244 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.244 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.227095+0000", 2023-10-09T10:58:45.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4785999999999999e-05 2023-10-09T10:58:45.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.244 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.244 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:58:45.244 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.227204+0000", 2023-10-09T10:58:45.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000109079 2023-10-09T10:58:45.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.245 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.245 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:45.245 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.239698+0000", 2023-10-09T10:58:45.245 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012494081000000001 2023-10-09T10:58:45.245 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.245 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.245 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.245 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242208+0000", 2023-10-09T10:58:45.245 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00251001 2023-10-09T10:58:45.245 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.245 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.245 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.246 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.254821+0000", 2023-10-09T10:58:45.246 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012612978 2023-10-09T10:58:45.246 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.246 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.246 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.246 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.254858+0000", 2023-10-09T10:58:45.246 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6785000000000001e-05 2023-10-09T10:58:45.246 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.246 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.246 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.246 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.257145+0000", 2023-10-09T10:58:45.247 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002287374 2023-10-09T10:58:45.247 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.247 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.247 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.247 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.247 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.247 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96908 7.d 7:bd9652bb:::benchmark_data_smithi125_295816_object5771:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:58:45.247 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.184970+0000", 2023-10-09T10:58:45.247 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 464.99238639599997, 2023-10-09T10:58:45.247 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.107155712, 2023-10-09T10:58:45.247 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.247 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.247 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:45.249 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:58:45.249 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:58:45.249 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96908 2023-10-09T10:58:45.249 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.249 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.249 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.250 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.250 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184970+0000", 2023-10-09T10:58:45.250 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.250 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.250 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.250 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.250 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184970+0000", 2023-10-09T10:58:45.250 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.250 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.250 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.250 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.251 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184969+0000", 2023-10-09T10:58:45.251 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:58:45.251 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.251 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.251 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.251 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184976+0000", 2023-10-09T10:58:45.251 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0849999999999999e-06 2023-10-09T10:58:45.251 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.251 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.251 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.251 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184988+0000", 2023-10-09T10:58:45.251 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2438e-05 2023-10-09T10:58:45.251 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.252 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.252 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.252 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184994+0000", 2023-10-09T10:58:45.252 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.835e-06 2023-10-09T10:58:45.252 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.252 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.252 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.252 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196710+0000", 2023-10-09T10:58:45.252 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011715557 2023-10-09T10:58:45.252 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.252 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.252 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:58:45.252 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.197312+0000", 2023-10-09T10:58:45.253 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000602675 2023-10-09T10:58:45.253 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.253 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.253 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.253 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.232847+0000", 2023-10-09T10:58:45.253 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.035534143999999997 2023-10-09T10:58:45.253 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.253 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.253 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.253 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.232937+0000", 2023-10-09T10:58:45.253 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0303999999999995e-05 2023-10-09T10:58:45.253 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.253 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.254 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:58:45.254 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.233064+0000", 2023-10-09T10:58:45.254 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012689799999999999 2023-10-09T10:58:45.254 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.254 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.254 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:45.254 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242946+0000", 2023-10-09T10:58:45.254 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0098825000000000007 2023-10-09T10:58:45.254 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.254 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.255 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.255 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.269531+0000", 2023-10-09T10:58:45.255 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026584907000000001 2023-10-09T10:58:45.255 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.255 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.255 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.255 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.289926+0000", 2023-10-09T10:58:45.255 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020394791999999998 2023-10-09T10:58:45.255 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.255 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.256 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.256 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.289981+0000", 2023-10-09T10:58:45.256 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4540999999999998e-05 2023-10-09T10:58:45.256 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.256 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.256 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.256 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.292126+0000", 2023-10-09T10:58:45.256 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021454270000000001 2023-10-09T10:58:45.256 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.256 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.256 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.257 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.257 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.257 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96935 7.d 7:b8c4db12:::benchmark_data_smithi125_295816_object5798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:58:45.257 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.240816+0000", 2023-10-09T10:58:45.257 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 464.93654051499999, 2023-10-09T10:58:45.257 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.096789209000000001, 2023-10-09T10:58:45.257 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.257 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.257 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:45.257 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:58:45.257 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:58:45.257 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96935 2023-10-09T10:58:45.258 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.258 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.258 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.258 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.258 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240816+0000", 2023-10-09T10:58:45.258 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.258 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.258 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.258 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.258 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240816+0000", 2023-10-09T10:58:45.258 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.258 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.259 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.259 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.259 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240815+0000", 2023-10-09T10:58:45.259 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:58:45.259 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.259 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.259 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.259 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240821+0000", 2023-10-09T10:58:45.259 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1140000000000002e-06 2023-10-09T10:58:45.259 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.259 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.259 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.260 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240831+0000", 2023-10-09T10:58:45.260 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6309999999999998e-06 2023-10-09T10:58:45.260 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.260 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.260 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.260 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240845+0000", 2023-10-09T10:58:45.260 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4705000000000001e-05 2023-10-09T10:58:45.260 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.260 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.260 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.261 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242238+0000", 2023-10-09T10:58:45.261 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001392646 2023-10-09T10:58:45.261 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.261 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.261 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:58:45.261 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242937+0000", 2023-10-09T10:58:45.261 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00069863399999999995 2023-10-09T10:58:45.261 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.261 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.261 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.261 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.295227+0000", 2023-10-09T10:58:45.261 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.052290652999999999 2023-10-09T10:58:45.261 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.262 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.262 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.262 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.295329+0000", 2023-10-09T10:58:45.262 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010212700000000001 2023-10-09T10:58:45.262 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.262 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.262 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:58:45.262 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.295755+0000", 2023-10-09T10:58:45.262 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00042587400000000002 2023-10-09T10:58:45.262 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.262 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.262 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:45.263 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.308570+0000", 2023-10-09T10:58:45.263 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012814397 2023-10-09T10:58:45.263 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.263 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.263 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.263 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.329769+0000", 2023-10-09T10:58:45.263 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021199234000000001 2023-10-09T10:58:45.263 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.263 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.263 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.263 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.335181+0000", 2023-10-09T10:58:45.263 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0054126360000000002 2023-10-09T10:58:45.263 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.264 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.264 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.264 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.335231+0000", 2023-10-09T10:58:45.264 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9344000000000001e-05 2023-10-09T10:58:45.264 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.264 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.264 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.264 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.337605+0000", 2023-10-09T10:58:45.264 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002374555 2023-10-09T10:58:45.264 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.264 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.264 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.265 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.265 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.265 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96940 7.d 7:bfb000f4:::benchmark_data_smithi125_295816_object5803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:58:45.265 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.268461+0000", 2023-10-09T10:58:45.265 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 464.90889596900001, 2023-10-09T10:58:45.265 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.071660901999999999, 2023-10-09T10:58:45.265 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.265 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.265 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:45.265 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:58:45.265 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:58:45.265 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96940 2023-10-09T10:58:45.265 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.266 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.266 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.266 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.266 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268461+0000", 2023-10-09T10:58:45.266 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.266 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.266 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.266 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.266 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268461+0000", 2023-10-09T10:58:45.266 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.266 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.266 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.267 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.267 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268459+0000", 2023-10-09T10:58:45.267 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:58:45.267 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.267 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.267 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.267 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268477+0000", 2023-10-09T10:58:45.267 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7377999999999999e-05 2023-10-09T10:58:45.267 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.267 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.267 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.267 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268488+0000", 2023-10-09T10:58:45.268 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1579e-05 2023-10-09T10:58:45.268 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.268 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.268 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.268 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268495+0000", 2023-10-09T10:58:45.268 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5429999999999999e-06 2023-10-09T10:58:45.268 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.268 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.268 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.268 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.273724+0000", 2023-10-09T10:58:45.268 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0052292049999999998 2023-10-09T10:58:45.268 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.269 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:58:45.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.274443+0000", 2023-10-09T10:58:45.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00071879099999999998 2023-10-09T10:58:45.269 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.269 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.306460+0000", 2023-10-09T10:58:45.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.032016797999999999 2023-10-09T10:58:45.269 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.269 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.270 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.306545+0000", 2023-10-09T10:58:45.270 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5277000000000004e-05 2023-10-09T10:58:45.270 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.270 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.270 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:58:45.270 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.306674+0000", 2023-10-09T10:58:45.270 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012941600000000001 2023-10-09T10:58:45.270 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.270 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.270 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:45.270 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.317498+0000", 2023-10-09T10:58:45.271 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010823523999999999 2023-10-09T10:58:45.271 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.271 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.271 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.271 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.329799+0000", 2023-10-09T10:58:45.271 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012301559 2023-10-09T10:58:45.271 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.271 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.271 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.271 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.337661+0000", 2023-10-09T10:58:45.271 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007861303 2023-10-09T10:58:45.272 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.272 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.272 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.272 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.337718+0000", 2023-10-09T10:58:45.272 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6974e-05 2023-10-09T10:58:45.272 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.272 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.272 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.272 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.340122+0000", 2023-10-09T10:58:45.272 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024038660000000002 2023-10-09T10:58:45.272 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.272 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.273 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.273 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.273 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.273 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:98648 7.d 7:bbc4eb35:::benchmark_data_smithi125_295816_object7511:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:58:45.273 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:02.635379+0000", 2023-10-09T10:58:45.273 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 462.54197805899997, 2023-10-09T10:58:45.273 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.078747929999999994, 2023-10-09T10:58:45.273 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.273 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.273 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:45.273 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:58:45.273 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:58:45.274 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98648 2023-10-09T10:58:45.274 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.274 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.274 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.274 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.274 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635379+0000", 2023-10-09T10:58:45.274 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.274 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.274 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.274 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.274 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635379+0000", 2023-10-09T10:58:45.274 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.275 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.275 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.275 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.275 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635377+0000", 2023-10-09T10:58:45.275 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:58:45.275 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.275 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.275 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.275 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635382+0000", 2023-10-09T10:58:45.275 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3580000000000002e-06 2023-10-09T10:58:45.275 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.275 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.276 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.276 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635397+0000", 2023-10-09T10:58:45.276 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4265e-05 2023-10-09T10:58:45.276 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.276 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.276 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.276 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635403+0000", 2023-10-09T10:58:45.276 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4049999999999997e-06 2023-10-09T10:58:45.276 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.276 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.276 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.276 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.654397+0000", 2023-10-09T10:58:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018993432000000001 2023-10-09T10:58:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:58:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.661791+0000", 2023-10-09T10:58:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007394655 2023-10-09T10:58:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.671916+0000", 2023-10-09T10:58:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010125187000000001 2023-10-09T10:58:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.278 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.278 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.671975+0000", 2023-10-09T10:58:45.278 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.897e-05 2023-10-09T10:58:45.278 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.278 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.278 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:58:45.278 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.672065+0000", 2023-10-09T10:58:45.278 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9809999999999996e-05 2023-10-09T10:58:45.278 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.278 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.278 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:45.279 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.682405+0000", 2023-10-09T10:58:45.279 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010339924 2023-10-09T10:58:45.279 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.279 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.279 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.279 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.682443+0000", 2023-10-09T10:58:45.279 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7886000000000003e-05 2023-10-09T10:58:45.279 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.279 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.279 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.279 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.712060+0000", 2023-10-09T10:58:45.279 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029616844 2023-10-09T10:58:45.279 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.280 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.280 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.280 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.712116+0000", 2023-10-09T10:58:45.280 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6081000000000001e-05 2023-10-09T10:58:45.280 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.280 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.280 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.280 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.714127+0000", 2023-10-09T10:58:45.280 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002010572 2023-10-09T10:58:45.280 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.280 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.280 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.281 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.281 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.281 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:98673 7.d 7:b818009b:::benchmark_data_smithi125_295816_object7536:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:58:45.281 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:02.680579+0000", 2023-10-09T10:58:45.281 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 462.49677782800001, 2023-10-09T10:58:45.281 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.095416154000000003, 2023-10-09T10:58:45.281 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.281 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.281 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:45.281 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:58:45.281 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:58:45.282 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98673 2023-10-09T10:58:45.282 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.282 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.282 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.282 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.282 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680579+0000", 2023-10-09T10:58:45.282 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.282 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.282 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.282 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.282 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680579+0000", 2023-10-09T10:58:45.282 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.283 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680578+0000", 2023-10-09T10:58:45.283 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:58:45.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.283 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680582+0000", 2023-10-09T10:58:45.283 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8e-06 2023-10-09T10:58:45.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680588+0000", 2023-10-09T10:58:45.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4670000000000001e-06 2023-10-09T10:58:45.284 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.284 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680591+0000", 2023-10-09T10:58:45.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9519999999999999e-06 2023-10-09T10:58:45.284 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.284 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.727779+0000", 2023-10-09T10:58:45.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047187937999999999 2023-10-09T10:58:45.285 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.285 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:58:45.285 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.728358+0000", 2023-10-09T10:58:45.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00057898100000000001 2023-10-09T10:58:45.285 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.285 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.285 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.740190+0000", 2023-10-09T10:58:45.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01183184 2023-10-09T10:58:45.286 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.286 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.286 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.286 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.740303+0000", 2023-10-09T10:58:45.286 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112762 2023-10-09T10:58:45.286 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.286 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.286 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:58:45.286 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.740474+0000", 2023-10-09T10:58:45.286 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017171100000000001 2023-10-09T10:58:45.286 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.287 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.287 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:45.287 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.746764+0000", 2023-10-09T10:58:45.287 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062892499999999997 2023-10-09T10:58:45.287 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.287 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.287 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.287 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.756355+0000", 2023-10-09T10:58:45.287 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0095916060000000008 2023-10-09T10:58:45.287 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.287 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.288 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.288 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.774088+0000", 2023-10-09T10:58:45.288 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017732947999999998 2023-10-09T10:58:45.288 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.288 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.288 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.288 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.774137+0000", 2023-10-09T10:58:45.288 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8616999999999999e-05 2023-10-09T10:58:45.288 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.288 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.288 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.288 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.775995+0000", 2023-10-09T10:58:45.288 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018582340000000001 2023-10-09T10:58:45.289 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.289 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.289 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.289 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.289 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.289 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:98701 7.d 7:bd16628e:::benchmark_data_smithi125_295816_object7564:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:58:45.289 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:02.717662+0000", 2023-10-09T10:58:45.289 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 462.45969447700003, 2023-10-09T10:58:45.289 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.079670053000000005, 2023-10-09T10:58:45.289 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.289 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.289 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:45.289 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:58:45.289 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:58:45.290 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98701 2023-10-09T10:58:45.290 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.290 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.290 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.290 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.290 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717662+0000", 2023-10-09T10:58:45.290 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.290 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.290 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.290 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.290 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717662+0000", 2023-10-09T10:58:45.291 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.291 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.291 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.291 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.291 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717661+0000", 2023-10-09T10:58:45.291 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:58:45.291 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.291 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.291 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.291 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717666+0000", 2023-10-09T10:58:45.292 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.22e-06 2023-10-09T10:58:45.292 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.292 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.292 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.292 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717676+0000", 2023-10-09T10:58:45.292 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.944e-06 2023-10-09T10:58:45.292 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.292 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.292 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.292 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717681+0000", 2023-10-09T10:58:45.292 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3719999999999998e-06 2023-10-09T10:58:45.293 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.293 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.293 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.293 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.729691+0000", 2023-10-09T10:58:45.293 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012010140000000001 2023-10-09T10:58:45.293 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.293 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.293 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:58:45.293 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.732552+0000", 2023-10-09T10:58:45.293 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028607770000000001 2023-10-09T10:58:45.293 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.294 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.294 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.294 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.771785+0000", 2023-10-09T10:58:45.294 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039233280000000002 2023-10-09T10:58:45.294 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.294 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.294 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.294 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.771847+0000", 2023-10-09T10:58:45.294 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1909000000000006e-05 2023-10-09T10:58:45.294 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.294 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.295 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:58:45.295 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.771940+0000", 2023-10-09T10:58:45.295 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3058000000000002e-05 2023-10-09T10:58:45.295 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.295 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.295 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:45.295 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.778867+0000", 2023-10-09T10:58:45.295 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0069268180000000004 2023-10-09T10:58:45.295 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.295 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.295 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.295 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.783491+0000", 2023-10-09T10:58:45.295 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0046242490000000004 2023-10-09T10:58:45.295 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.296 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.296 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:45.296 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.795089+0000", 2023-10-09T10:58:45.296 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011598489999999999 2023-10-09T10:58:45.296 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.296 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.296 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.296 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.795150+0000", 2023-10-09T10:58:45.296 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0517999999999999e-05 2023-10-09T10:58:45.296 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.296 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.296 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.296 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.797332+0000", 2023-10-09T10:58:45.297 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021824079999999998 2023-10-09T10:58:45.297 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.297 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.297 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.297 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.297 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.297 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:91615 8.f e286/282 8:fb4b59c0:::benchmark_data_smithi125_329074_object91614:head v 286'5764, mlcod=0'0)", 2023-10-09T10:58:45.297 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:32.633558+0000", 2023-10-09T10:58:45.297 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 252.54379840300001, 2023-10-09T10:58:45.297 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028974272999999998, 2023-10-09T10:58:45.297 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.297 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.297 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.298 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.298 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.298 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633558+0000", 2023-10-09T10:58:45.298 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.298 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.298 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.298 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.298 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633558+0000", 2023-10-09T10:58:45.298 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.298 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.298 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.298 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.298 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633557+0000", 2023-10-09T10:58:45.299 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:58:45.299 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.299 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.299 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633600+0000", 2023-10-09T10:58:45.299 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3139000000000002e-05 2023-10-09T10:58:45.299 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.299 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.299 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633614+0000", 2023-10-09T10:58:45.299 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3512999999999999e-05 2023-10-09T10:58:45.299 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.300 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.300 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633621+0000", 2023-10-09T10:58:45.300 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0550000000000002e-06 2023-10-09T10:58:45.300 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.300 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.300 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.300 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633685+0000", 2023-10-09T10:58:45.300 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4276999999999995e-05 2023-10-09T10:58:45.300 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.300 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.300 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.301 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633716+0000", 2023-10-09T10:58:45.301 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1134999999999999e-05 2023-10-09T10:58:45.301 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.301 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.301 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.301 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.662492+0000", 2023-10-09T10:58:45.301 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028775618999999999 2023-10-09T10:58:45.301 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.301 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.301 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.301 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.662533+0000", 2023-10-09T10:58:45.301 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0768e-05 2023-10-09T10:58:45.302 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.302 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.302 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.302 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.302 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.302 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:91623 8.7 e286/282 8:e8cb112d:::benchmark_data_smithi125_329074_object91622:head v 286'5732, mlcod=0'0)", 2023-10-09T10:58:45.302 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:32.634471+0000", 2023-10-09T10:58:45.302 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 252.54288562400001, 2023-10-09T10:58:45.302 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028088733000000001, 2023-10-09T10:58:45.302 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.302 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.302 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.303 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.303 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.303 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634471+0000", 2023-10-09T10:58:45.303 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.303 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.303 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.303 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.303 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634471+0000", 2023-10-09T10:58:45.303 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.303 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.303 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.303 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.304 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634470+0000", 2023-10-09T10:58:45.304 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:58:45.304 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.304 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.304 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.304 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634540+0000", 2023-10-09T10:58:45.304 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9406000000000006e-05 2023-10-09T10:58:45.304 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.304 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.304 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.304 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634556+0000", 2023-10-09T10:58:45.305 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5801999999999999e-05 2023-10-09T10:58:45.305 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.305 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.305 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.305 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634562+0000", 2023-10-09T10:58:45.305 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9430000000000003e-06 2023-10-09T10:58:45.305 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.305 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.305 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.305 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.640689+0000", 2023-10-09T10:58:45.305 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061267650000000002 2023-10-09T10:58:45.305 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.306 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.306 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.306 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.640718+0000", 2023-10-09T10:58:45.306 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9187e-05 2023-10-09T10:58:45.306 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.306 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.306 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.306 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.662537+0000", 2023-10-09T10:58:45.306 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021818627 2023-10-09T10:58:45.306 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.306 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.307 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.307 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.662560+0000", 2023-10-09T10:58:45.307 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2708999999999999e-05 2023-10-09T10:58:45.307 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.307 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.307 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.307 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.307 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.307 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:91868 8.f e286/282 8:f4bd67fe:::benchmark_data_smithi125_329074_object91867:head v 286'5787, mlcod=0'0)", 2023-10-09T10:58:45.307 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:33.101662+0000", 2023-10-09T10:58:45.307 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 252.075694274, 2023-10-09T10:58:45.307 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026291749, 2023-10-09T10:58:45.308 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.308 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.308 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.308 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.308 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.308 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101662+0000", 2023-10-09T10:58:45.308 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.308 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.308 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.308 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.308 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101662+0000", 2023-10-09T10:58:45.308 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.308 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.309 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.309 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.309 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101661+0000", 2023-10-09T10:58:45.309 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:58:45.309 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.309 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.309 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.309 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101711+0000", 2023-10-09T10:58:45.309 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9974e-05 2023-10-09T10:58:45.309 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.309 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.309 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.309 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101726+0000", 2023-10-09T10:58:45.310 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4945e-05 2023-10-09T10:58:45.310 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.310 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.310 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.310 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101733+0000", 2023-10-09T10:58:45.310 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1350000000000003e-06 2023-10-09T10:58:45.310 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.310 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.310 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.310 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.105791+0000", 2023-10-09T10:58:45.310 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040581360000000004 2023-10-09T10:58:45.310 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.311 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.311 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.311 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.105821+0000", 2023-10-09T10:58:45.311 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9422999999999999e-05 2023-10-09T10:58:45.311 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.311 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.311 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.311 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.127912+0000", 2023-10-09T10:58:45.311 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022090789999999999 2023-10-09T10:58:45.311 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.311 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.311 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.312 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.127954+0000", 2023-10-09T10:58:45.312 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2577999999999998e-05 2023-10-09T10:58:45.312 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.312 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.312 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.312 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.312 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.312 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:92334 8.f e286/282 8:fe8e3200:::benchmark_data_smithi125_329074_object92333:head v 286'5819, mlcod=0'0)", 2023-10-09T10:58:45.312 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:34.002645+0000", 2023-10-09T10:58:45.312 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 251.17471132, 2023-10-09T10:58:45.312 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028096320000000001, 2023-10-09T10:58:45.312 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.312 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.313 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.313 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.313 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.313 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002645+0000", 2023-10-09T10:58:45.313 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.313 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.313 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.313 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.313 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002645+0000", 2023-10-09T10:58:45.313 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.313 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.313 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.314 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.314 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002644+0000", 2023-10-09T10:58:45.314 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:58:45.314 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.314 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.314 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.314 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002709+0000", 2023-10-09T10:58:45.314 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4896999999999999e-05 2023-10-09T10:58:45.314 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.314 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.314 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.315 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002724+0000", 2023-10-09T10:58:45.315 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4543e-05 2023-10-09T10:58:45.315 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.315 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.315 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.315 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002735+0000", 2023-10-09T10:58:45.315 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1148000000000001e-05 2023-10-09T10:58:45.315 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.315 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.315 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.315 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002795+0000", 2023-10-09T10:58:45.315 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9908000000000003e-05 2023-10-09T10:58:45.315 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.316 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.316 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.316 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002824+0000", 2023-10-09T10:58:45.316 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9065e-05 2023-10-09T10:58:45.316 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.316 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.316 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.316 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.030695+0000", 2023-10-09T10:58:45.316 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027871015999999998 2023-10-09T10:58:45.316 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.316 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.316 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.317 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.030742+0000", 2023-10-09T10:58:45.317 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6714e-05 2023-10-09T10:58:45.317 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.317 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.317 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.317 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.317 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.317 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:92495 8.9 e286/265 8:99c80bd2:::benchmark_data_smithi125_329074_object92494:head v 286'5773, mlcod=286'5771)", 2023-10-09T10:58:45.317 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:34.258707+0000", 2023-10-09T10:58:45.317 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 250.91864916200001, 2023-10-09T10:58:45.317 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025821361000000001, 2023-10-09T10:58:45.317 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.318 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.318 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.318 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.318 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.318 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258707+0000", 2023-10-09T10:58:45.318 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.318 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.318 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.318 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.318 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258707+0000", 2023-10-09T10:58:45.318 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.318 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.319 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.319 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.319 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258707+0000", 2023-10-09T10:58:45.319 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:58:45.319 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.319 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.319 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.319 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258744+0000", 2023-10-09T10:58:45.319 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6458000000000002e-05 2023-10-09T10:58:45.319 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.319 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.319 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.319 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258754+0000", 2023-10-09T10:58:45.320 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0482000000000001e-05 2023-10-09T10:58:45.320 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.320 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.320 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.320 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258758+0000", 2023-10-09T10:58:45.320 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2910000000000001e-06 2023-10-09T10:58:45.320 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.320 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.320 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.320 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.267039+0000", 2023-10-09T10:58:45.320 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0082809919999999992 2023-10-09T10:58:45.320 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.321 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.321 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.321 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.267062+0000", 2023-10-09T10:58:45.321 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2532999999999999e-05 2023-10-09T10:58:45.321 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.321 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.321 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.321 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.284491+0000", 2023-10-09T10:58:45.321 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017429417999999999 2023-10-09T10:58:45.321 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.321 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.322 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.322 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.284529+0000", 2023-10-09T10:58:45.322 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7570000000000001e-05 2023-10-09T10:58:45.322 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.322 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.322 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.322 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.322 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.322 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:94596 8.f e288/282 8:ffee9f18:::benchmark_data_smithi125_329074_object94595:head v 288'5955, mlcod=0'0)", 2023-10-09T10:58:45.322 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:37.928730+0000", 2023-10-09T10:58:45.322 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 247.24862701999999, 2023-10-09T10:58:45.322 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028718649999999998, 2023-10-09T10:58:45.323 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.323 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.323 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.323 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.323 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.323 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928730+0000", 2023-10-09T10:58:45.323 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.323 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.323 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.323 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.323 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928730+0000", 2023-10-09T10:58:45.324 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.324 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.324 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.324 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.324 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928729+0000", 2023-10-09T10:58:45.324 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:58:45.324 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.324 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.324 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.324 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928781+0000", 2023-10-09T10:58:45.324 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1758000000000001e-05 2023-10-09T10:58:45.324 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.325 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.325 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.325 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928787+0000", 2023-10-09T10:58:45.325 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3180000000000003e-06 2023-10-09T10:58:45.325 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.325 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.325 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.325 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928792+0000", 2023-10-09T10:58:45.325 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6199999999999998e-06 2023-10-09T10:58:45.325 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.325 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.326 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.326 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.941317+0000", 2023-10-09T10:58:45.326 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012525087000000001 2023-10-09T10:58:45.326 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.326 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.326 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.326 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.941341+0000", 2023-10-09T10:58:45.326 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3408999999999999e-05 2023-10-09T10:58:45.326 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.326 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.326 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.326 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.957410+0000", 2023-10-09T10:58:45.327 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016068948999999999 2023-10-09T10:58:45.327 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.327 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.327 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.327 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.957448+0000", 2023-10-09T10:58:45.327 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8769000000000001e-05 2023-10-09T10:58:45.327 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.327 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.327 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.327 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.327 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.327 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:94597 8.f e288/282 8:f1e41889:::benchmark_data_smithi125_329074_object94596:head v 288'5956, mlcod=0'0)", 2023-10-09T10:58:45.328 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:37.928795+0000", 2023-10-09T10:58:45.328 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 247.248561255, 2023-10-09T10:58:45.328 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028674681, 2023-10-09T10:58:45.328 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:45.328 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:58:45.328 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:45.328 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.328 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:45.328 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928795+0000", 2023-10-09T10:58:45.328 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.328 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.328 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.329 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:45.329 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928795+0000", 2023-10-09T10:58:45.329 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:45.329 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.329 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.329 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:45.329 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928795+0000", 2023-10-09T10:58:45.329 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967296 2023-10-09T10:58:45.329 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.329 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.329 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:45.329 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928916+0000", 2023-10-09T10:58:45.330 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000120494 2023-10-09T10:58:45.330 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.330 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.330 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:45.330 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928927+0000", 2023-10-09T10:58:45.330 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1634000000000001e-05 2023-10-09T10:58:45.330 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.330 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.330 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:45.330 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928931+0000", 2023-10-09T10:58:45.330 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0450000000000002e-06 2023-10-09T10:58:45.331 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.331 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.331 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:45.331 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.946895+0000", 2023-10-09T10:58:45.331 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017963923 2023-10-09T10:58:45.331 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.331 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.331 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:45.331 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.946918+0000", 2023-10-09T10:58:45.331 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2425999999999998e-05 2023-10-09T10:58:45.331 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.331 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.332 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:58:45.332 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.957452+0000", 2023-10-09T10:58:45.332 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010534484 2023-10-09T10:58:45.332 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:45.332 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:45.332 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:58:45.332 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.957470+0000", 2023-10-09T10:58:45.332 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7890999999999999e-05 2023-10-09T10:58:45.332 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.332 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.332 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.333 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:45.333 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:45.333 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:58:45.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:45.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:45.435 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:45.498 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:45.564 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:58:45.675 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 81 16 102889 102873 79.3155 50.0625 0.00843784 0.0126037 2023-10-09T10:58:45.783 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:45.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:45.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:45.890 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:45.943 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:45.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:45.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:45.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:46.007 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:58:46.064 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:46.064 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:58:46.064 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:46 smithi125 ceph-mon[195507]: pgmap v880: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2023-10-09T10:58:46.064 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:46 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/891232451' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:58:46.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:46.101 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:58:46.150 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:46.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:46.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:58:46.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:46 smithi155 ceph-mon[184854]: pgmap v880: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2023-10-09T10:58:46.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:46 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/891232451' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:58:46.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:46 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:58:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:46 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:58:46.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:58:46.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:58:46.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:46 smithi125 ceph-mon[200197]: pgmap v880: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2023-10-09T10:58:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:46 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/891232451' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:58:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:46 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:58:46.366 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:46.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:46.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:46.471 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:46.541 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:46.678 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 82 16 103760 103744 79.0111 54.4375 0.0152305 0.0126516 2023-10-09T10:58:46.678 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:46.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:46.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:46.785 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:58:46.854 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:46.888 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:58:46.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:46.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:58:46.955 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 2, 1, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:58:46.956 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:58:46.956 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T10:58:46.956 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:58:46.992 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:58:47.037 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:58:47.065 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:47.102 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:58:47.104 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:58:47.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:47.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:58:47.148 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:58:47.148 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:58:47.148 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.148 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:104120 9.a e354/332 9:56a69a77:::benchmark_data_smithi125_363789_object104119:head v 354'6452, mlcod=354'6450)", 2023-10-09T10:58:47.148 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:47.142867+0000", 2023-10-09T10:58:47.148 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0046175369999999997, 2023-10-09T10:58:47.148 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0046455790000000004, 2023-10-09T10:58:47.148 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:47.148 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:58:47.148 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:47.149 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.149 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:47.149 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.142867+0000", 2023-10-09T10:58:47.149 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:47.149 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.149 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.149 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:47.149 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.142867+0000", 2023-10-09T10:58:47.149 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:47.149 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.150 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.150 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:47.150 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.142867+0000", 2023-10-09T10:58:47.150 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:58:47.150 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.150 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.150 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:47.150 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.142900+0000", 2023-10-09T10:58:47.150 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3340000000000003e-05 2023-10-09T10:58:47.151 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.151 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.151 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:47.151 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.142911+0000", 2023-10-09T10:58:47.151 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0346000000000001e-05 2023-10-09T10:58:47.152 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.152 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.152 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:47.152 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.142915+0000", 2023-10-09T10:58:47.152 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1150000000000004e-06 2023-10-09T10:58:47.152 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.152 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.152 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:47.152 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.143015+0000", 2023-10-09T10:58:47.152 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9772999999999995e-05 2023-10-09T10:58:47.152 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.153 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.153 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:47.153 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.143033+0000", 2023-10-09T10:58:47.153 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8802999999999999e-05 2023-10-09T10:58:47.153 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:47.153 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:47.153 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:47.153 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.153 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.153 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:104127 9.a e354/332 9:50cec9c2:::benchmark_data_smithi125_363789_object104126:head v 354'6453, mlcod=354'6450)", 2023-10-09T10:58:47.153 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:47.145510+0000", 2023-10-09T10:58:47.154 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0019747710000000002, 2023-10-09T10:58:47.154 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020707049999999999, 2023-10-09T10:58:47.154 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:47.154 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:58:47.154 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:47.154 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.154 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:47.154 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.145510+0000", 2023-10-09T10:58:47.154 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:47.154 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.154 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.155 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:47.155 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.145510+0000", 2023-10-09T10:58:47.155 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:47.155 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.155 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.155 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:47.155 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.145509+0000", 2023-10-09T10:58:47.155 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:58:47.155 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.155 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.155 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:47.155 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.145539+0000", 2023-10-09T10:58:47.156 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9122e-05 2023-10-09T10:58:47.156 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.156 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.156 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:47.156 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.145553+0000", 2023-10-09T10:58:47.156 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4667999999999999e-05 2023-10-09T10:58:47.156 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.156 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.156 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:47.156 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.145559+0000", 2023-10-09T10:58:47.156 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8590000000000001e-06 2023-10-09T10:58:47.157 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.157 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.157 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:47.157 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.146034+0000", 2023-10-09T10:58:47.157 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000474549 2023-10-09T10:58:47.157 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.157 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.157 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:47.157 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.146054+0000", 2023-10-09T10:58:47.157 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0132999999999998e-05 2023-10-09T10:58:47.157 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:47.157 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:47.158 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:47.158 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.158 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.158 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:104125 9.3 9:cd758a7f:::benchmark_data_smithi125_363789_object104124:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-10-09T10:58:47.158 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:47.134553+0000", 2023-10-09T10:58:47.158 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.012932302, 2023-10-09T10:58:47.158 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013099, 2023-10-09T10:58:47.158 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:58:47.158 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:58:47.158 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:58:47.158 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T10:58:47.159 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T10:58:47.159 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 104125 2023-10-09T10:58:47.159 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.159 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:58:47.159 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.159 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:58:47.159 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.134553+0000", 2023-10-09T10:58:47.159 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:47.159 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.159 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.159 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:58:47.159 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.134553+0000", 2023-10-09T10:58:47.160 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:58:47.160 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.160 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.160 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:58:47.160 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.134552+0000", 2023-10-09T10:58:47.160 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:58:47.160 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.160 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.160 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:58:47.160 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.134585+0000", 2023-10-09T10:58:47.160 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3352000000000002e-05 2023-10-09T10:58:47.160 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.161 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.161 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:58:47.161 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.134594+0000", 2023-10-09T10:58:47.161 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.721e-06 2023-10-09T10:58:47.161 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.161 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.161 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:58:47.161 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.134600+0000", 2023-10-09T10:58:47.161 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4709999999999998e-06 2023-10-09T10:58:47.161 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.161 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.161 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:58:47.161 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.134645+0000", 2023-10-09T10:58:47.162 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5741999999999997e-05 2023-10-09T10:58:47.162 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.162 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.162 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:58:47.162 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.134756+0000", 2023-10-09T10:58:47.162 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011045800000000001 2023-10-09T10:58:47.162 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.162 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.162 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T10:58:47.162 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.134892+0000", 2023-10-09T10:58:47.162 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013577000000000001 2023-10-09T10:58:47.163 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.163 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.163 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:58:47.163 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.137851+0000", 2023-10-09T10:58:47.163 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029597899999999999 2023-10-09T10:58:47.163 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:58:47.163 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:58:47.163 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:47.163 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:47.142967+0000", 2023-10-09T10:58:47.163 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051161449999999999 2023-10-09T10:58:47.163 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:47.163 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:58:47.164 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:47.164 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:58:47.164 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T10:58:47.164 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 3 2023-10-09T10:58:47.164 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:58:47.164 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:58:47.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:47 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:47.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:47 smithi155 ceph-mon[184854]: failed to encode map e354 with expected crc 2023-10-09T10:58:47.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:47 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:58:47.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:47 smithi155 ceph-mon[184854]: osdmap e354: 8 total, 8 up, 8 in 2023-10-09T10:58:47.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:47 smithi155 ceph-mon[184854]: failed to encode map e354 with expected crc 2023-10-09T10:58:47.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:47 smithi155 ceph-mon[184854]: failed to encode map e354 with expected crc 2023-10-09T10:58:47.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:47 smithi155 ceph-mon[184854]: failed to encode map e354 with expected crc 2023-10-09T10:58:47.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:47 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3860991111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:47.195 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:58:47.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:47.237 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:58:47.258 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:47.292 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:58:47.295 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:47.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:47.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:47.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:47.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[195507]: failed to encode map e354 with expected crc 2023-10-09T10:58:47.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:58:47.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[195507]: osdmap e354: 8 total, 8 up, 8 in 2023-10-09T10:58:47.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[195507]: failed to encode map e354 with expected crc 2023-10-09T10:58:47.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[195507]: failed to encode map e354 with expected crc 2023-10-09T10:58:47.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[195507]: failed to encode map e354 with expected crc 2023-10-09T10:58:47.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3860991111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:47.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:47.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[200197]: failed to encode map e354 with expected crc 2023-10-09T10:58:47.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:58:47.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[200197]: osdmap e354: 8 total, 8 up, 8 in 2023-10-09T10:58:47.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[200197]: failed to encode map e354 with expected crc 2023-10-09T10:58:47.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[200197]: failed to encode map e354 with expected crc 2023-10-09T10:58:47.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[200197]: failed to encode map e354 with expected crc 2023-10-09T10:58:47.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3860991111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:47.328 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:58:47.375 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:58:47.377 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:58:47.377 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:58:47.377 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:58:47.377 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:58:47.377 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:58:47.378 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:58:47.378 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:58:47.378 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:58:47.394 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:47.404 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:47.467 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:47.556 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:47.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:47.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:47.665 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:47.676 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 83 16 104552 104536 78.6554 49.5 0.0080858 0.0127091 2023-10-09T10:58:47.709 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:48.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:48.142 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:58:48.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:48.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:48.247 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:48.309 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:48.336 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:48.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:48.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:48.444 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:48 smithi155 ceph-mon[184854]: failed to encode map e354 with expected crc 2023-10-09T10:58:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:48 smithi155 ceph-mon[184854]: pgmap v882: 119 pgs: 119 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-09T10:58:48.500 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:48 smithi125 ceph-mon[195507]: failed to encode map e354 with expected crc 2023-10-09T10:58:48.500 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:48 smithi125 ceph-mon[195507]: pgmap v882: 119 pgs: 119 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-09T10:58:48.501 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:48 smithi125 ceph-mon[200197]: failed to encode map e354 with expected crc 2023-10-09T10:58:48.501 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:48 smithi125 ceph-mon[200197]: pgmap v882: 119 pgs: 119 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-09T10:58:48.505 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:48.533 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:48.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:48.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:48.558 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:58:48.622 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:58:48.637 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:48.682 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 84 16 105427 105411 78.3695 54.6875 0.0188514 0.0127554 2023-10-09T10:58:48.738 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:48.779 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:48.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:48.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:48.886 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:48.942 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:48.970 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:48.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:48.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:49.078 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:49.103 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:58:49.106 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:49.118 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:58:49.162 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:49.290 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:49.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:49.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:49.395 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:49.453 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:49.584 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:49.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:49.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:49.677 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 85 16 106230 106214 78.0381 50.1875 0.0111209 0.0128102 2023-10-09T10:58:49.689 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:49.696 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:58:49.696 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:58:49.728 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:49.731 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":882,"stamp":"2023-10-09T10:58:47.561400+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6739072928,"num_objects":102845,"num_object_clones":0,"num_object_copies":308535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":102845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":102882,"num_write_kb":6580573,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":1067242,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100067,"ondisk_log_size":100067,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":25303812,"kb_used_data":20099972,"kb_used_omap":126,"kb_used_meta":5203649,"kb_avail":724657404,"statfs":{"total":767960285184,"available":742049181696,"internally_reserved":0,"allocated":20582371328,"data_stored":20570236998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":5328537211},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,4,2,5,4,6],"upper_bound":64},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":510394368,"num_objects":7788,"num_object_clones":0,"num_object_copies":23364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7788,"num_write_kb":498432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.005151"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":710,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.728051+0000","last_change":"2023-10-09T10:57:25.104150+0000","last_active":"2023-10-09T10:58:31.728051+0000","last_peered":"2023-10-09T10:58:31.728051+0000","last_clean":"2023-10-09T10:58:31.728051+0000","last_became_active":"2023-10-09T10:57:25.103762+0000","last_became_peered":"2023-10-09T10:57:25.103762+0000","last_unstale":"2023-10-09T10:58:31.728051+0000","last_undegraded":"2023-10-09T10:58:31.728051+0000","last_fullsized":"2023-10-09T10:58:31.728051+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:40.174975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:40.174975+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:35.416780+0000","scrub_duration":0.010654205,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":59,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.728383+0000","last_change":"2023-10-09T10:57:11.874010+0000","last_active":"2023-10-09T10:58:31.728383+0000","last_peered":"2023-10-09T10:58:31.728383+0000","last_clean":"2023-10-09T10:58:31.728383+0000","last_became_active":"2023-10-09T10:57:11.873713+0000","last_became_peered":"2023-10-09T10:57:11.873713+0000","last_unstale":"2023-10-09T10:58:31.728383+0000","last_undegraded":"2023-10-09T10:58:31.728383+0000","last_fullsized":"2023-10-09T10:58:31.728383+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:58.049973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":362,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:35.636250+0000","last_change":"2023-10-09T10:54:47.945782+0000","last_active":"2023-10-09T10:58:35.636250+0000","last_peered":"2023-10-09T10:58:35.636250+0000","last_clean":"2023-10-09T10:58:35.636250+0000","last_became_active":"2023-10-09T10:54:47.945460+0000","last_became_peered":"2023-10-09T10:54:47.945460+0000","last_unstale":"2023-10-09T10:58:35.636250+0000","last_undegraded":"2023-10-09T10:58:35.636250+0000","last_fullsized":"2023-10-09T10:58:35.636250+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:50.537469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":735,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:32.666095+0000","last_change":"2023-10-09T10:53:34.537094+0000","last_active":"2023-10-09T10:58:32.666095+0000","last_peered":"2023-10-09T10:58:32.666095+0000","last_clean":"2023-10-09T10:58:32.666095+0000","last_became_active":"2023-10-09T10:52:55.414995+0000","last_became_peered":"2023-10-09T10:52:55.414995+0000","last_unstale":"2023-10-09T10:58:32.666095+0000","last_undegraded":"2023-10-09T10:58:32.666095+0000","last_fullsized":"2023-10-09T10:58:32.666095+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.536962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.536962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:30.197404+0000","scrub_duration":0.0053325960000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":71,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.689551+0000","last_change":"2023-10-09T10:54:47.944768+0000","last_active":"2023-10-09T10:58:39.689551+0000","last_peered":"2023-10-09T10:58:39.689551+0000","last_clean":"2023-10-09T10:58:39.689551+0000","last_became_active":"2023-10-09T10:54:47.943888+0000","last_became_peered":"2023-10-09T10:54:47.943888+0000","last_unstale":"2023-10-09T10:58:39.689551+0000","last_undegraded":"2023-10-09T10:58:39.689551+0000","last_fullsized":"2023-10-09T10:58:39.689551+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:11.046762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":281,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.727233+0000","last_change":"2023-10-09T10:57:22.007179+0000","last_active":"2023-10-09T10:58:31.727233+0000","last_peered":"2023-10-09T10:58:31.727233+0000","last_clean":"2023-10-09T10:58:31.727233+0000","last_became_active":"2023-10-09T10:57:22.006760+0000","last_became_peered":"2023-10-09T10:57:22.006760+0000","last_unstale":"2023-10-09T10:58:31.727233+0000","last_undegraded":"2023-10-09T10:58:31.727233+0000","last_fullsized":"2023-10-09T10:58:31.727233+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:47:13.771012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":777,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.686498+0000","last_change":"2023-10-09T10:57:25.103697+0000","last_active":"2023-10-09T10:58:39.686498+0000","last_peered":"2023-10-09T10:58:39.686498+0000","last_clean":"2023-10-09T10:58:39.686498+0000","last_became_active":"2023-10-09T10:57:25.103306+0000","last_became_peered":"2023-10-09T10:57:25.103306+0000","last_unstale":"2023-10-09T10:58:39.686498+0000","last_undegraded":"2023-10-09T10:58:39.686498+0000","last_fullsized":"2023-10-09T10:58:39.686498+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:23:07.530489+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":407,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.719999+0000","last_change":"2023-10-09T10:54:48.038948+0000","last_active":"2023-10-09T10:58:31.719999+0000","last_peered":"2023-10-09T10:58:31.719999+0000","last_clean":"2023-10-09T10:58:31.719999+0000","last_became_active":"2023-10-09T10:54:48.001556+0000","last_became_peered":"2023-10-09T10:54:48.001556+0000","last_unstale":"2023-10-09T10:58:31.719999+0000","last_undegraded":"2023-10-09T10:58:31.719999+0000","last_fullsized":"2023-10-09T10:58:31.719999+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:23.415040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":71,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.690001+0000","last_change":"2023-10-09T10:54:47.914742+0000","last_active":"2023-10-09T10:58:39.690001+0000","last_peered":"2023-10-09T10:58:39.690001+0000","last_clean":"2023-10-09T10:58:39.690001+0000","last_became_active":"2023-10-09T10:54:47.914551+0000","last_became_peered":"2023-10-09T10:54:47.914551+0000","last_unstale":"2023-10-09T10:58:39.690001+0000","last_undegraded":"2023-10-09T10:58:39.690001+0000","last_fullsized":"2023-10-09T10:58:39.690001+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:16.067994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":769,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.683614+0000","last_change":"2023-10-09T10:57:25.098434+0000","last_active":"2023-10-09T10:58:39.683614+0000","last_peered":"2023-10-09T10:58:39.683614+0000","last_clean":"2023-10-09T10:58:39.683614+0000","last_became_active":"2023-10-09T10:57:25.098064+0000","last_became_peered":"2023-10-09T10:57:25.098064+0000","last_unstale":"2023-10-09T10:58:39.683614+0000","last_undegraded":"2023-10-09T10:58:39.683614+0000","last_fullsized":"2023-10-09T10:58:39.683614+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.787782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.787782+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:03.016769+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":226,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.727676+0000","last_change":"2023-10-09T10:46:02.642614+0000","last_active":"2023-10-09T10:58:31.727676+0000","last_peered":"2023-10-09T10:58:31.727676+0000","last_clean":"2023-10-09T10:58:31.727676+0000","last_became_active":"2023-10-09T10:46:02.642134+0000","last_became_peered":"2023-10-09T10:46:02.642134+0000","last_unstale":"2023-10-09T10:58:31.727676+0000","last_undegraded":"2023-10-09T10:58:31.727676+0000","last_fullsized":"2023-10-09T10:58:31.727676+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:55:06.250944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":362,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.720072+0000","last_change":"2023-10-09T10:54:48.039275+0000","last_active":"2023-10-09T10:58:31.720072+0000","last_peered":"2023-10-09T10:58:31.720072+0000","last_clean":"2023-10-09T10:58:31.720072+0000","last_became_active":"2023-10-09T10:54:48.017645+0000","last_became_peered":"2023-10-09T10:54:48.017645+0000","last_unstale":"2023-10-09T10:58:31.720072+0000","last_undegraded":"2023-10-09T10:58:31.720072+0000","last_fullsized":"2023-10-09T10:58:31.720072+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:55.986791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":644,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.729412+0000","last_change":"2023-10-09T10:57:25.103739+0000","last_active":"2023-10-09T10:58:31.729412+0000","last_peered":"2023-10-09T10:58:31.729412+0000","last_clean":"2023-10-09T10:58:31.729412+0000","last_became_active":"2023-10-09T10:57:25.103378+0000","last_became_peered":"2023-10-09T10:57:25.103378+0000","last_unstale":"2023-10-09T10:58:31.729412+0000","last_undegraded":"2023-10-09T10:58:31.729412+0000","last_fullsized":"2023-10-09T10:58:31.729412+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:49:50.889945+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":31,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.683413+0000","last_change":"2023-10-09T10:57:22.014188+0000","last_active":"2023-10-09T10:58:39.683413+0000","last_peered":"2023-10-09T10:58:39.683413+0000","last_clean":"2023-10-09T10:58:39.683413+0000","last_became_active":"2023-10-09T10:57:22.013975+0000","last_became_peered":"2023-10-09T10:57:22.013975+0000","last_unstale":"2023-10-09T10:58:39.683413+0000","last_undegraded":"2023-10-09T10:58:39.683413+0000","last_fullsized":"2023-10-09T10:58:39.683413+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:58.620741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":319,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.716377+0000","last_change":"2023-10-09T10:57:22.010202+0000","last_active":"2023-10-09T10:58:31.716377+0000","last_peered":"2023-10-09T10:58:31.716377+0000","last_clean":"2023-10-09T10:58:31.716377+0000","last_became_active":"2023-10-09T10:57:22.006867+0000","last_became_peered":"2023-10-09T10:57:22.006867+0000","last_unstale":"2023-10-09T10:58:31.716377+0000","last_undegraded":"2023-10-09T10:58:31.716377+0000","last_fullsized":"2023-10-09T10:58:31.716377+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:54.212660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":742,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.727391+0000","last_change":"2023-10-09T10:53:32.540768+0000","last_active":"2023-10-09T10:58:31.727391+0000","last_peered":"2023-10-09T10:58:31.727391+0000","last_clean":"2023-10-09T10:58:31.727391+0000","last_became_active":"2023-10-09T10:52:55.414818+0000","last_became_peered":"2023-10-09T10:52:55.414818+0000","last_unstale":"2023-10-09T10:58:31.727391+0000","last_undegraded":"2023-10-09T10:58:31.727391+0000","last_fullsized":"2023-10-09T10:58:31.727391+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:32.540610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:32.540610+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:47.374383+0000","scrub_duration":0.0064251300000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":37,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.683868+0000","last_change":"2023-10-09T10:57:22.032392+0000","last_active":"2023-10-09T10:58:39.683868+0000","last_peered":"2023-10-09T10:58:39.683868+0000","last_clean":"2023-10-09T10:58:39.683868+0000","last_became_active":"2023-10-09T10:57:22.012620+0000","last_became_peered":"2023-10-09T10:57:22.012620+0000","last_unstale":"2023-10-09T10:58:39.683868+0000","last_undegraded":"2023-10-09T10:58:39.683868+0000","last_fullsized":"2023-10-09T10:58:39.683868+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:06:34.979769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.728542+0000","last_change":"2023-10-09T10:57:46.838152+0000","last_active":"2023-10-09T10:58:31.728542+0000","last_peered":"2023-10-09T10:58:31.728542+0000","last_clean":"2023-10-09T10:58:31.728542+0000","last_became_active":"2023-10-09T10:57:46.837933+0000","last_became_peered":"2023-10-09T10:57:46.837933+0000","last_unstale":"2023-10-09T10:58:31.728542+0000","last_undegraded":"2023-10-09T10:58:31.728542+0000","last_fullsized":"2023-10-09T10:58:31.728542+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:26:23.218941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":740,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.690172+0000","last_change":"2023-10-09T10:57:18.113935+0000","last_active":"2023-10-09T10:58:39.690172+0000","last_peered":"2023-10-09T10:58:39.690172+0000","last_clean":"2023-10-09T10:58:39.690172+0000","last_became_active":"2023-10-09T10:54:50.913330+0000","last_became_peered":"2023-10-09T10:54:50.913330+0000","last_unstale":"2023-10-09T10:58:39.690172+0000","last_undegraded":"2023-10-09T10:58:39.690172+0000","last_fullsized":"2023-10-09T10:58:39.690172+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:18.113848+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.113848+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:51:24.460807+0000","scrub_duration":0.001474106,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":41,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.684080+0000","last_change":"2023-10-09T10:57:22.039995+0000","last_active":"2023-10-09T10:58:39.684080+0000","last_peered":"2023-10-09T10:58:39.684080+0000","last_clean":"2023-10-09T10:58:39.684080+0000","last_became_active":"2023-10-09T10:57:22.016287+0000","last_became_peered":"2023-10-09T10:57:22.016287+0000","last_unstale":"2023-10-09T10:58:39.684080+0000","last_undegraded":"2023-10-09T10:58:39.684080+0000","last_fullsized":"2023-10-09T10:58:39.684080+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:52.167501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":368,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.686850+0000","last_change":"2023-10-09T10:57:46.845755+0000","last_active":"2023-10-09T10:58:39.686850+0000","last_peered":"2023-10-09T10:58:39.686850+0000","last_clean":"2023-10-09T10:58:39.686850+0000","last_became_active":"2023-10-09T10:57:46.845475+0000","last_became_peered":"2023-10-09T10:57:46.845475+0000","last_unstale":"2023-10-09T10:58:39.686850+0000","last_undegraded":"2023-10-09T10:58:39.686850+0000","last_fullsized":"2023-10-09T10:58:39.686850+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:21.673994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":720,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.726932+0000","last_change":"2023-10-09T10:49:18.708370+0000","last_active":"2023-10-09T10:58:31.726932+0000","last_peered":"2023-10-09T10:58:31.726932+0000","last_clean":"2023-10-09T10:58:31.726932+0000","last_became_active":"2023-10-09T10:49:17.832412+0000","last_became_peered":"2023-10-09T10:49:17.832412+0000","last_unstale":"2023-10-09T10:58:31.726932+0000","last_undegraded":"2023-10-09T10:58:31.726932+0000","last_fullsized":"2023-10-09T10:58:31.726932+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:57.808410+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":37,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.683552+0000","last_change":"2023-10-09T10:57:22.026809+0000","last_active":"2023-10-09T10:58:39.683552+0000","last_peered":"2023-10-09T10:58:39.683552+0000","last_clean":"2023-10-09T10:58:39.683552+0000","last_became_active":"2023-10-09T10:57:22.011600+0000","last_became_peered":"2023-10-09T10:57:22.011600+0000","last_unstale":"2023-10-09T10:58:39.683552+0000","last_undegraded":"2023-10-09T10:58:39.683552+0000","last_fullsized":"2023-10-09T10:58:39.683552+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:10.453811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":430,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.720869+0000","last_change":"2023-10-09T10:57:22.006151+0000","last_active":"2023-10-09T10:58:31.720869+0000","last_peered":"2023-10-09T10:58:31.720869+0000","last_clean":"2023-10-09T10:58:31.720869+0000","last_became_active":"2023-10-09T10:57:22.005782+0000","last_became_peered":"2023-10-09T10:57:22.005782+0000","last_unstale":"2023-10-09T10:58:31.720869+0000","last_undegraded":"2023-10-09T10:58:31.720869+0000","last_fullsized":"2023-10-09T10:58:31.720869+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:09.733515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":673,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.683230+0000","last_change":"2023-10-09T10:57:25.098905+0000","last_active":"2023-10-09T10:58:39.683230+0000","last_peered":"2023-10-09T10:58:39.683230+0000","last_clean":"2023-10-09T10:58:39.683230+0000","last_became_active":"2023-10-09T10:57:25.098562+0000","last_became_peered":"2023-10-09T10:57:25.098562+0000","last_unstale":"2023-10-09T10:58:39.683230+0000","last_undegraded":"2023-10-09T10:58:39.683230+0000","last_fullsized":"2023-10-09T10:58:39.683230+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:23.020232+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":280,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.726816+0000","last_change":"2023-10-09T10:57:22.004708+0000","last_active":"2023-10-09T10:58:31.726816+0000","last_peered":"2023-10-09T10:58:31.726816+0000","last_clean":"2023-10-09T10:58:31.726816+0000","last_became_active":"2023-10-09T10:57:22.004413+0000","last_became_peered":"2023-10-09T10:57:22.004413+0000","last_unstale":"2023-10-09T10:58:31.726816+0000","last_undegraded":"2023-10-09T10:58:31.726816+0000","last_fullsized":"2023-10-09T10:58:31.726816+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:54.003556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":31,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.684017+0000","last_change":"2023-10-09T10:57:22.014923+0000","last_active":"2023-10-09T10:58:39.684017+0000","last_peered":"2023-10-09T10:58:39.684017+0000","last_clean":"2023-10-09T10:58:39.684017+0000","last_became_active":"2023-10-09T10:57:22.014055+0000","last_became_peered":"2023-10-09T10:57:22.014055+0000","last_unstale":"2023-10-09T10:58:39.684017+0000","last_undegraded":"2023-10-09T10:58:39.684017+0000","last_fullsized":"2023-10-09T10:58:39.684017+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:50.546435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":671,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.689511+0000","last_change":"2023-10-09T10:57:44.843644+0000","last_active":"2023-10-09T10:58:39.689511+0000","last_peered":"2023-10-09T10:58:39.689511+0000","last_clean":"2023-10-09T10:58:39.689511+0000","last_became_active":"2023-10-09T10:54:50.915189+0000","last_became_peered":"2023-10-09T10:54:50.915189+0000","last_unstale":"2023-10-09T10:58:39.689511+0000","last_undegraded":"2023-10-09T10:58:39.689511+0000","last_fullsized":"2023-10-09T10:58:39.689511+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:44.843520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:44.843520+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:06.673360+0000","scrub_duration":0.0067258329999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":31,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.684111+0000","last_change":"2023-10-09T10:57:22.016705+0000","last_active":"2023-10-09T10:58:39.684111+0000","last_peered":"2023-10-09T10:58:39.684111+0000","last_clean":"2023-10-09T10:58:39.684111+0000","last_became_active":"2023-10-09T10:57:22.016452+0000","last_became_peered":"2023-10-09T10:57:22.016452+0000","last_unstale":"2023-10-09T10:58:39.684111+0000","last_undegraded":"2023-10-09T10:58:39.684111+0000","last_fullsized":"2023-10-09T10:58:39.684111+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:43:44.382821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":262,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.727021+0000","last_change":"2023-10-09T10:57:22.007046+0000","last_active":"2023-10-09T10:58:31.727021+0000","last_peered":"2023-10-09T10:58:31.727021+0000","last_clean":"2023-10-09T10:58:31.727021+0000","last_became_active":"2023-10-09T10:57:22.006360+0000","last_became_peered":"2023-10-09T10:57:22.006360+0000","last_unstale":"2023-10-09T10:58:31.727021+0000","last_undegraded":"2023-10-09T10:58:31.727021+0000","last_fullsized":"2023-10-09T10:58:31.727021+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:23.023072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":686,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.690721+0000","last_change":"2023-10-09T10:54:50.910027+0000","last_active":"2023-10-09T10:58:39.690721+0000","last_peered":"2023-10-09T10:58:39.690721+0000","last_clean":"2023-10-09T10:58:39.690721+0000","last_became_active":"2023-10-09T10:54:50.909698+0000","last_became_peered":"2023-10-09T10:54:50.909698+0000","last_unstale":"2023-10-09T10:58:39.690721+0000","last_undegraded":"2023-10-09T10:58:39.690721+0000","last_fullsized":"2023-10-09T10:58:39.690721+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:12:01.637196+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":229,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.728808+0000","last_change":"2023-10-09T10:51:12.366910+0000","last_active":"2023-10-09T10:58:31.728808+0000","last_peered":"2023-10-09T10:58:31.728808+0000","last_clean":"2023-10-09T10:58:31.728808+0000","last_became_active":"2023-10-09T10:51:12.366565+0000","last_became_peered":"2023-10-09T10:51:12.366565+0000","last_unstale":"2023-10-09T10:58:31.728808+0000","last_undegraded":"2023-10-09T10:58:31.728808+0000","last_fullsized":"2023-10-09T10:58:31.728808+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:53.464251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":407,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.719948+0000","last_change":"2023-10-09T10:57:22.008343+0000","last_active":"2023-10-09T10:58:31.719948+0000","last_peered":"2023-10-09T10:58:31.719948+0000","last_clean":"2023-10-09T10:58:31.719948+0000","last_became_active":"2023-10-09T10:57:22.007747+0000","last_became_peered":"2023-10-09T10:57:22.007747+0000","last_unstale":"2023-10-09T10:58:31.719948+0000","last_undegraded":"2023-10-09T10:58:31.719948+0000","last_fullsized":"2023-10-09T10:58:31.719948+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:45.774703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"353'6347","reported_seq":6388,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:42.653337+0000","last_change":"2023-10-09T10:57:22.013979+0000","last_active":"2023-10-09T10:58:42.653337+0000","last_peered":"2023-10-09T10:58:42.653337+0000","last_clean":"2023-10-09T10:58:42.653337+0000","last_became_active":"2023-10-09T10:57:22.013514+0000","last_became_peered":"2023-10-09T10:57:22.013514+0000","last_unstale":"2023-10-09T10:58:42.653337+0000","last_undegraded":"2023-10-09T10:58:42.653337+0000","last_fullsized":"2023-10-09T10:58:42.653337+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":6347,"log_dups_size":0,"ondisk_log_size":6347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:16:42.975508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415956992,"num_objects":6347,"num_object_clones":0,"num_object_copies":19041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6347,"num_write_kb":406208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":694,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.683779+0000","last_change":"2023-10-09T10:57:25.102011+0000","last_active":"2023-10-09T10:58:39.683779+0000","last_peered":"2023-10-09T10:58:39.683779+0000","last_clean":"2023-10-09T10:58:39.683779+0000","last_became_active":"2023-10-09T10:57:25.101702+0000","last_became_peered":"2023-10-09T10:57:25.101702+0000","last_unstale":"2023-10-09T10:58:39.683779+0000","last_undegraded":"2023-10-09T10:58:39.683779+0000","last_fullsized":"2023-10-09T10:58:39.683779+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:34.671859+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":35,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.684422+0000","last_change":"2023-10-09T10:57:22.021225+0000","last_active":"2023-10-09T10:58:39.684422+0000","last_peered":"2023-10-09T10:58:39.684422+0000","last_clean":"2023-10-09T10:58:39.684422+0000","last_became_active":"2023-10-09T10:57:22.007997+0000","last_became_peered":"2023-10-09T10:57:22.007997+0000","last_unstale":"2023-10-09T10:58:39.684422+0000","last_undegraded":"2023-10-09T10:58:39.684422+0000","last_fullsized":"2023-10-09T10:58:39.684422+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:52:19.871862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":31,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.683141+0000","last_change":"2023-10-09T10:57:22.016166+0000","last_active":"2023-10-09T10:58:39.683141+0000","last_peered":"2023-10-09T10:58:39.683141+0000","last_clean":"2023-10-09T10:58:39.683141+0000","last_became_active":"2023-10-09T10:57:22.015850+0000","last_became_peered":"2023-10-09T10:57:22.015850+0000","last_unstale":"2023-10-09T10:58:39.683141+0000","last_undegraded":"2023-10-09T10:58:39.683141+0000","last_fullsized":"2023-10-09T10:58:39.683141+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:37.827566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"354'6516","reported_seq":6549,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T10:58:46.868864+0000","last_change":"2023-10-09T10:57:19.986807+0000","last_active":"2023-10-09T10:58:46.868864+0000","last_peered":"2023-10-09T10:58:46.868864+0000","last_clean":"2023-10-09T10:58:46.868864+0000","last_became_active":"2023-10-09T10:57:19.986525+0000","last_became_peered":"2023-10-09T10:57:19.986525+0000","last_unstale":"2023-10-09T10:58:46.868864+0000","last_undegraded":"2023-10-09T10:58:46.868864+0000","last_fullsized":"2023-10-09T10:58:46.868864+0000","mapping_epoch":330,"log_start":"336'600","ondisk_log_start":"336'600","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5916,"log_dups_size":0,"ondisk_log_size":5916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:23:53.293903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427032576,"num_objects":6516,"num_object_clones":0,"num_object_copies":19548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6516,"num_write_kb":417024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":714,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.720694+0000","last_change":"2023-10-09T10:53:25.540494+0000","last_active":"2023-10-09T10:58:31.720694+0000","last_peered":"2023-10-09T10:58:31.720694+0000","last_clean":"2023-10-09T10:58:31.720694+0000","last_became_active":"2023-10-09T10:51:01.295474+0000","last_became_peered":"2023-10-09T10:51:01.295474+0000","last_unstale":"2023-10-09T10:58:31.720694+0000","last_undegraded":"2023-10-09T10:58:31.720694+0000","last_fullsized":"2023-10-09T10:58:31.720694+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:07:39.069143+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":35,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.683664+0000","last_change":"2023-10-09T10:57:22.023259+0000","last_active":"2023-10-09T10:58:39.683664+0000","last_peered":"2023-10-09T10:58:39.683664+0000","last_clean":"2023-10-09T10:58:39.683664+0000","last_became_active":"2023-10-09T10:57:22.009992+0000","last_became_peered":"2023-10-09T10:57:22.009992+0000","last_unstale":"2023-10-09T10:58:39.683664+0000","last_undegraded":"2023-10-09T10:58:39.683664+0000","last_fullsized":"2023-10-09T10:58:39.683664+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.289131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":71,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.689462+0000","last_change":"2023-10-09T10:54:47.944930+0000","last_active":"2023-10-09T10:58:39.689462+0000","last_peered":"2023-10-09T10:58:39.689462+0000","last_clean":"2023-10-09T10:58:39.689462+0000","last_became_active":"2023-10-09T10:54:47.944485+0000","last_became_peered":"2023-10-09T10:54:47.944485+0000","last_unstale":"2023-10-09T10:58:39.689462+0000","last_undegraded":"2023-10-09T10:58:39.689462+0000","last_fullsized":"2023-10-09T10:58:39.689462+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:11.213129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"354'6474","reported_seq":6505,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T10:58:46.206530+0000","last_change":"2023-10-09T10:57:22.024907+0000","last_active":"2023-10-09T10:58:46.206530+0000","last_peered":"2023-10-09T10:58:46.206530+0000","last_clean":"2023-10-09T10:58:46.206530+0000","last_became_active":"2023-10-09T10:57:22.024660+0000","last_became_peered":"2023-10-09T10:57:22.024660+0000","last_unstale":"2023-10-09T10:58:46.206530+0000","last_undegraded":"2023-10-09T10:58:46.206530+0000","last_fullsized":"2023-10-09T10:58:46.206530+0000","mapping_epoch":332,"log_start":"336'800","ondisk_log_start":"336'800","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:25:10.105154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424280064,"num_objects":6474,"num_object_clones":0,"num_object_copies":19422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6474,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":666,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.687041+0000","last_change":"2023-10-09T10:53:36.680150+0000","last_active":"2023-10-09T10:58:39.687041+0000","last_peered":"2023-10-09T10:58:39.687041+0000","last_clean":"2023-10-09T10:58:39.687041+0000","last_became_active":"2023-10-09T10:51:50.371032+0000","last_became_peered":"2023-10-09T10:51:50.371032+0000","last_unstale":"2023-10-09T10:58:39.687041+0000","last_undegraded":"2023-10-09T10:58:39.687041+0000","last_fullsized":"2023-10-09T10:58:39.687041+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.680090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.680090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:08:28.952827+0000","scrub_duration":0.005577172,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":35,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.683947+0000","last_change":"2023-10-09T10:57:22.034075+0000","last_active":"2023-10-09T10:58:39.683947+0000","last_peered":"2023-10-09T10:58:39.683947+0000","last_clean":"2023-10-09T10:58:39.683947+0000","last_became_active":"2023-10-09T10:57:22.015930+0000","last_became_peered":"2023-10-09T10:57:22.015930+0000","last_unstale":"2023-10-09T10:58:39.683947+0000","last_undegraded":"2023-10-09T10:58:39.683947+0000","last_fullsized":"2023-10-09T10:58:39.683947+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:16.494793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":89,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.690651+0000","last_change":"2023-10-09T10:57:22.005094+0000","last_active":"2023-10-09T10:58:39.690651+0000","last_peered":"2023-10-09T10:58:39.690651+0000","last_clean":"2023-10-09T10:58:39.690651+0000","last_became_active":"2023-10-09T10:57:22.004592+0000","last_became_peered":"2023-10-09T10:57:22.004592+0000","last_unstale":"2023-10-09T10:58:39.690651+0000","last_undegraded":"2023-10-09T10:58:39.690651+0000","last_fullsized":"2023-10-09T10:58:39.690651+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:35.526442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"353'6361","reported_seq":6393,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:46.002860+0000","last_change":"2023-10-09T10:57:20.305591+0000","last_active":"2023-10-09T10:58:46.002860+0000","last_peered":"2023-10-09T10:58:46.002860+0000","last_clean":"2023-10-09T10:58:46.002860+0000","last_became_active":"2023-10-09T10:57:20.305290+0000","last_became_peered":"2023-10-09T10:57:20.305290+0000","last_unstale":"2023-10-09T10:58:46.002860+0000","last_undegraded":"2023-10-09T10:58:46.002860+0000","last_fullsized":"2023-10-09T10:58:46.002860+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":6361,"log_dups_size":0,"ondisk_log_size":6361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:54.880555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416874496,"num_objects":6361,"num_object_clones":0,"num_object_copies":19083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6361,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":839,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.688677+0000","last_change":"2023-10-09T10:57:25.111238+0000","last_active":"2023-10-09T10:58:39.688677+0000","last_peered":"2023-10-09T10:58:39.688677+0000","last_clean":"2023-10-09T10:58:39.688677+0000","last_became_active":"2023-10-09T10:57:25.110991+0000","last_became_peered":"2023-10-09T10:57:25.110991+0000","last_unstale":"2023-10-09T10:58:39.688677+0000","last_undegraded":"2023-10-09T10:58:39.688677+0000","last_fullsized":"2023-10-09T10:58:39.688677+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:57.060423+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":180,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:35.636228+0000","last_change":"2023-10-09T10:51:50.374527+0000","last_active":"2023-10-09T10:58:35.636228+0000","last_peered":"2023-10-09T10:58:35.636228+0000","last_clean":"2023-10-09T10:58:35.636228+0000","last_became_active":"2023-10-09T10:51:50.374297+0000","last_became_peered":"2023-10-09T10:51:50.374297+0000","last_unstale":"2023-10-09T10:58:35.636228+0000","last_undegraded":"2023-10-09T10:58:35.636228+0000","last_fullsized":"2023-10-09T10:58:35.636228+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:31:49.358307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":262,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.728311+0000","last_change":"2023-10-09T10:54:47.939771+0000","last_active":"2023-10-09T10:58:31.728311+0000","last_peered":"2023-10-09T10:58:31.728311+0000","last_clean":"2023-10-09T10:58:31.728311+0000","last_became_active":"2023-10-09T10:54:47.939440+0000","last_became_peered":"2023-10-09T10:54:47.939440+0000","last_unstale":"2023-10-09T10:58:31.728311+0000","last_undegraded":"2023-10-09T10:58:31.728311+0000","last_fullsized":"2023-10-09T10:58:31.728311+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:58.335536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"353'6450","reported_seq":6488,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:45.636110+0000","last_change":"2023-10-09T10:57:22.009934+0000","last_active":"2023-10-09T10:58:45.636110+0000","last_peered":"2023-10-09T10:58:45.636110+0000","last_clean":"2023-10-09T10:58:45.636110+0000","last_became_active":"2023-10-09T10:57:22.009682+0000","last_became_peered":"2023-10-09T10:57:22.009682+0000","last_unstale":"2023-10-09T10:58:45.636110+0000","last_undegraded":"2023-10-09T10:58:45.636110+0000","last_fullsized":"2023-10-09T10:58:45.636110+0000","mapping_epoch":332,"log_start":"336'600","ondisk_log_start":"336'600","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5850,"log_dups_size":0,"ondisk_log_size":5850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:59.899734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":422707200,"num_objects":6450,"num_object_clones":0,"num_object_copies":19350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6450,"num_write_kb":412800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":709,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.683904+0000","last_change":"2023-10-09T10:57:25.106422+0000","last_active":"2023-10-09T10:58:39.683904+0000","last_peered":"2023-10-09T10:58:39.683904+0000","last_clean":"2023-10-09T10:58:39.683904+0000","last_became_active":"2023-10-09T10:57:25.105998+0000","last_became_peered":"2023-10-09T10:57:25.105998+0000","last_unstale":"2023-10-09T10:58:39.683904+0000","last_undegraded":"2023-10-09T10:58:39.683904+0000","last_fullsized":"2023-10-09T10:58:39.683904+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:10:53.164214+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":350,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.716512+0000","last_change":"2023-10-09T10:46:02.678285+0000","last_active":"2023-10-09T10:58:31.716512+0000","last_peered":"2023-10-09T10:58:31.716512+0000","last_clean":"2023-10-09T10:58:31.716512+0000","last_became_active":"2023-10-09T10:46:02.634705+0000","last_became_peered":"2023-10-09T10:46:02.634705+0000","last_unstale":"2023-10-09T10:58:31.716512+0000","last_undegraded":"2023-10-09T10:58:31.716512+0000","last_fullsized":"2023-10-09T10:58:31.716512+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:18.461717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":195,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.720945+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T10:58:31.720945+0000","last_peered":"2023-10-09T10:58:31.720945+0000","last_clean":"2023-10-09T10:58:31.720945+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T10:58:31.720945+0000","last_undegraded":"2023-10-09T10:58:31.720945+0000","last_fullsized":"2023-10-09T10:58:31.720945+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"354'6442","reported_seq":6475,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T10:58:46.840210+0000","last_change":"2023-10-09T10:57:20.327412+0000","last_active":"2023-10-09T10:58:46.840210+0000","last_peered":"2023-10-09T10:58:46.840210+0000","last_clean":"2023-10-09T10:58:46.840210+0000","last_became_active":"2023-10-09T10:57:20.327192+0000","last_became_peered":"2023-10-09T10:57:20.327192+0000","last_unstale":"2023-10-09T10:58:46.840210+0000","last_undegraded":"2023-10-09T10:58:46.840210+0000","last_fullsized":"2023-10-09T10:58:46.840210+0000","mapping_epoch":330,"log_start":"336'500","ondisk_log_start":"336'500","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5942,"log_dups_size":0,"ondisk_log_size":5942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:17:53.257184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":422182912,"num_objects":6442,"num_object_clones":0,"num_object_copies":19326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6442,"num_write_kb":412288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":658,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.690499+0000","last_change":"2023-10-09T10:54:21.173078+0000","last_active":"2023-10-09T10:58:39.690499+0000","last_peered":"2023-10-09T10:58:39.690499+0000","last_clean":"2023-10-09T10:58:39.690499+0000","last_became_active":"2023-10-09T10:54:21.172767+0000","last_became_peered":"2023-10-09T10:54:21.172767+0000","last_unstale":"2023-10-09T10:58:39.690499+0000","last_undegraded":"2023-10-09T10:58:39.690499+0000","last_fullsized":"2023-10-09T10:58:39.690499+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.532890+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.532890+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:32.335841+0000","scrub_duration":0.0045290130000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":333,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.686592+0000","last_change":"2023-10-09T10:54:47.908506+0000","last_active":"2023-10-09T10:58:39.686592+0000","last_peered":"2023-10-09T10:58:39.686592+0000","last_clean":"2023-10-09T10:58:39.686592+0000","last_became_active":"2023-10-09T10:54:47.908110+0000","last_became_peered":"2023-10-09T10:54:47.908110+0000","last_unstale":"2023-10-09T10:58:39.686592+0000","last_undegraded":"2023-10-09T10:58:39.686592+0000","last_fullsized":"2023-10-09T10:58:39.686592+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:38.808943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":92,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.729699+0000","last_change":"2023-10-09T10:54:47.948732+0000","last_active":"2023-10-09T10:58:31.729699+0000","last_peered":"2023-10-09T10:58:31.729699+0000","last_clean":"2023-10-09T10:58:31.729699+0000","last_became_active":"2023-10-09T10:54:47.948414+0000","last_became_peered":"2023-10-09T10:54:47.948414+0000","last_unstale":"2023-10-09T10:58:31.729699+0000","last_undegraded":"2023-10-09T10:58:31.729699+0000","last_fullsized":"2023-10-09T10:58:31.729699+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:38.020807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"354'6370","reported_seq":6401,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T10:58:46.205978+0000","last_change":"2023-10-09T10:57:22.024943+0000","last_active":"2023-10-09T10:58:46.205978+0000","last_peered":"2023-10-09T10:58:46.205978+0000","last_clean":"2023-10-09T10:58:46.205978+0000","last_became_active":"2023-10-09T10:57:22.024623+0000","last_became_peered":"2023-10-09T10:57:22.024623+0000","last_unstale":"2023-10-09T10:58:46.205978+0000","last_undegraded":"2023-10-09T10:58:46.205978+0000","last_fullsized":"2023-10-09T10:58:46.205978+0000","mapping_epoch":332,"log_start":"336'700","ondisk_log_start":"336'700","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5670,"log_dups_size":0,"ondisk_log_size":5670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:13:55.093719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417464320,"num_objects":6370,"num_object_clones":0,"num_object_copies":19110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6370,"num_write_kb":407680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":741,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.689756+0000","last_change":"2023-10-09T10:57:45.845982+0000","last_active":"2023-10-09T10:58:39.689756+0000","last_peered":"2023-10-09T10:58:39.689756+0000","last_clean":"2023-10-09T10:58:39.689756+0000","last_became_active":"2023-10-09T10:54:50.909639+0000","last_became_peered":"2023-10-09T10:54:50.909639+0000","last_unstale":"2023-10-09T10:58:39.689756+0000","last_undegraded":"2023-10-09T10:58:39.689756+0000","last_fullsized":"2023-10-09T10:58:39.689756+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:45.845850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:45.845850+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:52:51.027029+0000","scrub_duration":0.011054232000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":409,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.686534+0000","last_change":"2023-10-09T10:57:22.006224+0000","last_active":"2023-10-09T10:58:39.686534+0000","last_peered":"2023-10-09T10:58:39.686534+0000","last_clean":"2023-10-09T10:58:39.686534+0000","last_became_active":"2023-10-09T10:57:22.005898+0000","last_became_peered":"2023-10-09T10:57:22.005898+0000","last_unstale":"2023-10-09T10:58:39.686534+0000","last_undegraded":"2023-10-09T10:58:39.686534+0000","last_fullsized":"2023-10-09T10:58:39.686534+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:00.623898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":71,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.690240+0000","last_change":"2023-10-09T10:54:47.913826+0000","last_active":"2023-10-09T10:58:39.690240+0000","last_peered":"2023-10-09T10:58:39.690240+0000","last_clean":"2023-10-09T10:58:39.690240+0000","last_became_active":"2023-10-09T10:54:47.913524+0000","last_became_peered":"2023-10-09T10:54:47.913524+0000","last_unstale":"2023-10-09T10:58:39.690240+0000","last_undegraded":"2023-10-09T10:58:39.690240+0000","last_fullsized":"2023-10-09T10:58:39.690240+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:56:23.699552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.2","version":"353'6402","reported_seq":6433,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:45.635758+0000","last_change":"2023-10-09T10:57:19.986622+0000","last_active":"2023-10-09T10:58:45.635758+0000","last_peered":"2023-10-09T10:58:45.635758+0000","last_clean":"2023-10-09T10:58:45.635758+0000","last_became_active":"2023-10-09T10:57:19.986337+0000","last_became_peered":"2023-10-09T10:57:19.986337+0000","last_unstale":"2023-10-09T10:58:45.635758+0000","last_undegraded":"2023-10-09T10:58:45.635758+0000","last_fullsized":"2023-10-09T10:58:45.635758+0000","mapping_epoch":330,"log_start":"336'600","ondisk_log_start":"336'600","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5802,"log_dups_size":0,"ondisk_log_size":5802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:12:59.817044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419561472,"num_objects":6402,"num_object_clones":0,"num_object_copies":19206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6402,"num_write_kb":409728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":669,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.692107+0000","last_change":"2023-10-09T10:54:21.152148+0000","last_active":"2023-10-09T10:58:39.692107+0000","last_peered":"2023-10-09T10:58:39.692107+0000","last_clean":"2023-10-09T10:58:39.692107+0000","last_became_active":"2023-10-09T10:54:21.151809+0000","last_became_peered":"2023-10-09T10:54:21.151809+0000","last_unstale":"2023-10-09T10:58:39.692107+0000","last_undegraded":"2023-10-09T10:58:39.692107+0000","last_fullsized":"2023-10-09T10:58:39.692107+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.701128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.701128+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:27.021145+0000","scrub_duration":0.0025071830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":400,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.678174+0000","last_change":"2023-10-09T10:57:22.006965+0000","last_active":"2023-10-09T10:58:39.678174+0000","last_peered":"2023-10-09T10:58:39.678174+0000","last_clean":"2023-10-09T10:58:39.678174+0000","last_became_active":"2023-10-09T10:57:22.006456+0000","last_became_peered":"2023-10-09T10:57:22.006456+0000","last_unstale":"2023-10-09T10:58:39.678174+0000","last_undegraded":"2023-10-09T10:58:39.678174+0000","last_fullsized":"2023-10-09T10:58:39.678174+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:16.889422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":71,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.690136+0000","last_change":"2023-10-09T10:54:48.012411+0000","last_active":"2023-10-09T10:58:39.690136+0000","last_peered":"2023-10-09T10:58:39.690136+0000","last_clean":"2023-10-09T10:58:39.690136+0000","last_became_active":"2023-10-09T10:54:48.011894+0000","last_became_peered":"2023-10-09T10:54:48.011894+0000","last_unstale":"2023-10-09T10:58:39.690136+0000","last_undegraded":"2023-10-09T10:58:39.690136+0000","last_fullsized":"2023-10-09T10:58:39.690136+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:47:28.067337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"354'6616","reported_seq":6649,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T10:58:46.868865+0000","last_change":"2023-10-09T10:57:20.306271+0000","last_active":"2023-10-09T10:58:46.868865+0000","last_peered":"2023-10-09T10:58:46.868865+0000","last_clean":"2023-10-09T10:58:46.868865+0000","last_became_active":"2023-10-09T10:57:20.305994+0000","last_became_peered":"2023-10-09T10:57:20.305994+0000","last_unstale":"2023-10-09T10:58:46.868865+0000","last_undegraded":"2023-10-09T10:58:46.868865+0000","last_fullsized":"2023-10-09T10:58:46.868865+0000","mapping_epoch":330,"log_start":"336'700","ondisk_log_start":"336'700","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5916,"log_dups_size":0,"ondisk_log_size":5916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:46.777641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":433586176,"num_objects":6616,"num_object_clones":0,"num_object_copies":19848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6616,"num_write_kb":423424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":785,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.719657+0000","last_change":"2023-10-09T10:57:25.110928+0000","last_active":"2023-10-09T10:58:31.719657+0000","last_peered":"2023-10-09T10:58:31.719657+0000","last_clean":"2023-10-09T10:58:31.719657+0000","last_became_active":"2023-10-09T10:57:25.110240+0000","last_became_peered":"2023-10-09T10:57:25.110240+0000","last_unstale":"2023-10-09T10:58:31.719657+0000","last_undegraded":"2023-10-09T10:58:31.719657+0000","last_fullsized":"2023-10-09T10:58:31.719657+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:34.828051+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":89,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.692849+0000","last_change":"2023-10-09T10:57:22.005377+0000","last_active":"2023-10-09T10:58:39.692849+0000","last_peered":"2023-10-09T10:58:39.692849+0000","last_clean":"2023-10-09T10:58:39.692849+0000","last_became_active":"2023-10-09T10:57:22.005016+0000","last_became_peered":"2023-10-09T10:57:22.005016+0000","last_unstale":"2023-10-09T10:58:39.692849+0000","last_undegraded":"2023-10-09T10:58:39.692849+0000","last_fullsized":"2023-10-09T10:58:39.692849+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:03.296528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":274,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.717054+0000","last_change":"2023-10-09T10:54:17.886921+0000","last_active":"2023-10-09T10:58:31.717054+0000","last_peered":"2023-10-09T10:58:31.717054+0000","last_clean":"2023-10-09T10:58:31.717054+0000","last_became_active":"2023-10-09T10:54:17.886584+0000","last_became_peered":"2023-10-09T10:54:17.886584+0000","last_unstale":"2023-10-09T10:58:31.717054+0000","last_undegraded":"2023-10-09T10:58:31.717054+0000","last_fullsized":"2023-10-09T10:58:31.717054+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:15.848951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":71,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.690524+0000","last_change":"2023-10-09T10:54:47.908294+0000","last_active":"2023-10-09T10:58:39.690524+0000","last_peered":"2023-10-09T10:58:39.690524+0000","last_clean":"2023-10-09T10:58:39.690524+0000","last_became_active":"2023-10-09T10:54:47.908019+0000","last_became_peered":"2023-10-09T10:54:47.908019+0000","last_unstale":"2023-10-09T10:58:39.690524+0000","last_undegraded":"2023-10-09T10:58:39.690524+0000","last_fullsized":"2023-10-09T10:58:39.690524+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:48.662764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"353'6381","reported_seq":6422,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:42.596439+0000","last_change":"2023-10-09T10:57:22.005714+0000","last_active":"2023-10-09T10:58:42.596439+0000","last_peered":"2023-10-09T10:58:42.596439+0000","last_clean":"2023-10-09T10:58:42.596439+0000","last_became_active":"2023-10-09T10:57:22.005431+0000","last_became_peered":"2023-10-09T10:57:22.005431+0000","last_unstale":"2023-10-09T10:58:42.596439+0000","last_undegraded":"2023-10-09T10:58:42.596439+0000","last_fullsized":"2023-10-09T10:58:42.596439+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":6381,"log_dups_size":0,"ondisk_log_size":6381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:51.650262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418185216,"num_objects":6381,"num_object_clones":0,"num_object_copies":19143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6381,"num_write_kb":408384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":691,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.726855+0000","last_change":"2023-10-09T10:54:21.163072+0000","last_active":"2023-10-09T10:58:31.726855+0000","last_peered":"2023-10-09T10:58:31.726855+0000","last_clean":"2023-10-09T10:58:31.726855+0000","last_became_active":"2023-10-09T10:54:21.162710+0000","last_became_peered":"2023-10-09T10:54:21.162710+0000","last_unstale":"2023-10-09T10:58:31.726855+0000","last_undegraded":"2023-10-09T10:58:31.726855+0000","last_fullsized":"2023-10-09T10:58:31.726855+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:15.932806+0000","scrub_duration":0.0019146790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":71,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.690588+0000","last_change":"2023-10-09T10:54:47.899967+0000","last_active":"2023-10-09T10:58:39.690588+0000","last_peered":"2023-10-09T10:58:39.690588+0000","last_clean":"2023-10-09T10:58:39.690588+0000","last_became_active":"2023-10-09T10:54:47.899584+0000","last_became_peered":"2023-10-09T10:54:47.899584+0000","last_unstale":"2023-10-09T10:58:39.690588+0000","last_undegraded":"2023-10-09T10:58:39.690588+0000","last_fullsized":"2023-10-09T10:58:39.690588+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:43.973868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":263,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.728041+0000","last_change":"2023-10-09T10:50:43.388615+0000","last_active":"2023-10-09T10:58:31.728041+0000","last_peered":"2023-10-09T10:58:31.728041+0000","last_clean":"2023-10-09T10:58:31.728041+0000","last_became_active":"2023-10-09T10:50:43.388176+0000","last_became_peered":"2023-10-09T10:50:43.388176+0000","last_unstale":"2023-10-09T10:58:31.728041+0000","last_undegraded":"2023-10-09T10:58:31.728041+0000","last_fullsized":"2023-10-09T10:58:31.728041+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:41:17.550111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":71,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.692774+0000","last_change":"2023-10-09T10:54:47.950462+0000","last_active":"2023-10-09T10:58:39.692774+0000","last_peered":"2023-10-09T10:58:39.692774+0000","last_clean":"2023-10-09T10:58:39.692774+0000","last_became_active":"2023-10-09T10:54:47.940606+0000","last_became_peered":"2023-10-09T10:54:47.940606+0000","last_unstale":"2023-10-09T10:58:39.692774+0000","last_undegraded":"2023-10-09T10:58:39.692774+0000","last_fullsized":"2023-10-09T10:58:39.692774+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:50.557850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"354'6440","reported_seq":6482,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T10:58:46.859023+0000","last_change":"2023-10-09T10:57:22.004913+0000","last_active":"2023-10-09T10:58:46.859023+0000","last_peered":"2023-10-09T10:58:46.859023+0000","last_clean":"2023-10-09T10:58:46.859023+0000","last_became_active":"2023-10-09T10:57:22.004172+0000","last_became_peered":"2023-10-09T10:57:22.004172+0000","last_unstale":"2023-10-09T10:58:46.859023+0000","last_undegraded":"2023-10-09T10:58:46.859023+0000","last_fullsized":"2023-10-09T10:58:46.859023+0000","mapping_epoch":332,"log_start":"336'500","ondisk_log_start":"336'500","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5940,"log_dups_size":0,"ondisk_log_size":5940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:01.045466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":422051840,"num_objects":6440,"num_object_clones":0,"num_object_copies":19320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6440,"num_write_kb":412160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":768,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.683724+0000","last_change":"2023-10-09T10:57:25.101168+0000","last_active":"2023-10-09T10:58:39.683724+0000","last_peered":"2023-10-09T10:58:39.683724+0000","last_clean":"2023-10-09T10:58:39.683724+0000","last_became_active":"2023-10-09T10:57:25.100860+0000","last_became_peered":"2023-10-09T10:57:25.100860+0000","last_unstale":"2023-10-09T10:58:39.683724+0000","last_undegraded":"2023-10-09T10:58:39.683724+0000","last_fullsized":"2023-10-09T10:58:39.683724+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:37.594583+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":84,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.729988+0000","last_change":"2023-10-09T10:54:17.885030+0000","last_active":"2023-10-09T10:58:31.729988+0000","last_peered":"2023-10-09T10:58:31.729988+0000","last_clean":"2023-10-09T10:58:31.729988+0000","last_became_active":"2023-10-09T10:54:17.884752+0000","last_became_peered":"2023-10-09T10:54:17.884752+0000","last_unstale":"2023-10-09T10:58:31.729988+0000","last_undegraded":"2023-10-09T10:58:31.729988+0000","last_fullsized":"2023-10-09T10:58:31.729988+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:38.548132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":71,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.689563+0000","last_change":"2023-10-09T10:54:47.927822+0000","last_active":"2023-10-09T10:58:39.689563+0000","last_peered":"2023-10-09T10:58:39.689563+0000","last_clean":"2023-10-09T10:58:39.689563+0000","last_became_active":"2023-10-09T10:54:47.927455+0000","last_became_peered":"2023-10-09T10:54:47.927455+0000","last_unstale":"2023-10-09T10:58:39.689563+0000","last_undegraded":"2023-10-09T10:58:39.689563+0000","last_fullsized":"2023-10-09T10:58:39.689563+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:11.818203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":71,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.690068+0000","last_change":"2023-10-09T10:54:48.012629+0000","last_active":"2023-10-09T10:58:39.690068+0000","last_peered":"2023-10-09T10:58:39.690068+0000","last_clean":"2023-10-09T10:58:39.690068+0000","last_became_active":"2023-10-09T10:54:48.012148+0000","last_became_peered":"2023-10-09T10:54:48.012148+0000","last_unstale":"2023-10-09T10:58:39.690068+0000","last_undegraded":"2023-10-09T10:58:39.690068+0000","last_fullsized":"2023-10-09T10:58:39.690068+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:35:22.894111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"353'6402","reported_seq":6434,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:45.607506+0000","last_change":"2023-10-09T10:57:20.305367+0000","last_active":"2023-10-09T10:58:45.607506+0000","last_peered":"2023-10-09T10:58:45.607506+0000","last_clean":"2023-10-09T10:58:45.607506+0000","last_became_active":"2023-10-09T10:57:20.305050+0000","last_became_peered":"2023-10-09T10:57:20.305050+0000","last_unstale":"2023-10-09T10:58:45.607506+0000","last_undegraded":"2023-10-09T10:58:45.607506+0000","last_fullsized":"2023-10-09T10:58:45.607506+0000","mapping_epoch":330,"log_start":"336'600","ondisk_log_start":"336'600","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5802,"log_dups_size":0,"ondisk_log_size":5802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:49.570577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419561472,"num_objects":6402,"num_object_clones":0,"num_object_copies":19206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6402,"num_write_kb":409728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":102,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.729214+0000","last_change":"2023-10-09T10:57:22.006186+0000","last_active":"2023-10-09T10:58:31.729214+0000","last_peered":"2023-10-09T10:58:31.729214+0000","last_clean":"2023-10-09T10:58:31.729214+0000","last_became_active":"2023-10-09T10:57:22.005900+0000","last_became_peered":"2023-10-09T10:57:22.005900+0000","last_unstale":"2023-10-09T10:58:31.729214+0000","last_undegraded":"2023-10-09T10:58:31.729214+0000","last_fullsized":"2023-10-09T10:58:31.729214+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:14.033785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":785,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.683444+0000","last_change":"2023-10-09T10:57:25.100325+0000","last_active":"2023-10-09T10:58:39.683444+0000","last_peered":"2023-10-09T10:58:39.683444+0000","last_clean":"2023-10-09T10:58:39.683444+0000","last_became_active":"2023-10-09T10:57:25.099940+0000","last_became_peered":"2023-10-09T10:57:25.099940+0000","last_unstale":"2023-10-09T10:58:39.683444+0000","last_undegraded":"2023-10-09T10:58:39.683444+0000","last_fullsized":"2023-10-09T10:58:39.683444+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:33.410507+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":286,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.716269+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:58:31.716269+0000","last_peered":"2023-10-09T10:58:31.716269+0000","last_clean":"2023-10-09T10:58:31.716269+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:58:31.716269+0000","last_undegraded":"2023-10-09T10:58:31.716269+0000","last_fullsized":"2023-10-09T10:58:31.716269+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":286,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.687192+0000","last_change":"2023-10-09T10:57:22.013585+0000","last_active":"2023-10-09T10:58:39.687192+0000","last_peered":"2023-10-09T10:58:39.687192+0000","last_clean":"2023-10-09T10:58:39.687192+0000","last_became_active":"2023-10-09T10:57:22.013255+0000","last_became_peered":"2023-10-09T10:57:22.013255+0000","last_unstale":"2023-10-09T10:58:39.687192+0000","last_undegraded":"2023-10-09T10:58:39.687192+0000","last_fullsized":"2023-10-09T10:58:39.687192+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:29.725918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"353'6255","reported_seq":6287,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:42.608514+0000","last_change":"2023-10-09T10:57:20.328496+0000","last_active":"2023-10-09T10:58:42.608514+0000","last_peered":"2023-10-09T10:58:42.608514+0000","last_clean":"2023-10-09T10:58:42.608514+0000","last_became_active":"2023-10-09T10:57:20.328194+0000","last_became_peered":"2023-10-09T10:57:20.328194+0000","last_unstale":"2023-10-09T10:58:42.608514+0000","last_undegraded":"2023-10-09T10:58:42.608514+0000","last_fullsized":"2023-10-09T10:58:42.608514+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":6255,"log_dups_size":0,"ondisk_log_size":6255,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:15:57.867795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409927680,"num_objects":6255,"num_object_clones":0,"num_object_copies":18765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6255,"num_write_kb":400320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":714,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.689617+0000","last_change":"2023-10-09T10:57:43.819618+0000","last_active":"2023-10-09T10:58:39.689617+0000","last_peered":"2023-10-09T10:58:39.689617+0000","last_clean":"2023-10-09T10:58:39.689617+0000","last_became_active":"2023-10-09T10:54:50.914536+0000","last_became_peered":"2023-10-09T10:54:50.914536+0000","last_unstale":"2023-10-09T10:58:39.689617+0000","last_undegraded":"2023-10-09T10:58:39.689617+0000","last_fullsized":"2023-10-09T10:58:39.689617+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:43.819496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:43.819496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:09.320712+0000","scrub_duration":0.0071681970000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":31,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.683362+0000","last_change":"2023-10-09T10:57:22.008209+0000","last_active":"2023-10-09T10:58:39.683362+0000","last_peered":"2023-10-09T10:58:39.683362+0000","last_clean":"2023-10-09T10:58:39.683362+0000","last_became_active":"2023-10-09T10:57:22.007918+0000","last_became_peered":"2023-10-09T10:57:22.007918+0000","last_unstale":"2023-10-09T10:58:39.683362+0000","last_undegraded":"2023-10-09T10:58:39.683362+0000","last_fullsized":"2023-10-09T10:58:39.683362+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:55.764820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":236,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:32.666051+0000","last_change":"2023-10-09T10:46:57.208162+0000","last_active":"2023-10-09T10:58:32.666051+0000","last_peered":"2023-10-09T10:58:32.666051+0000","last_clean":"2023-10-09T10:58:32.666051+0000","last_became_active":"2023-10-09T10:46:57.207937+0000","last_became_peered":"2023-10-09T10:46:57.207937+0000","last_unstale":"2023-10-09T10:58:32.666051+0000","last_undegraded":"2023-10-09T10:58:32.666051+0000","last_fullsized":"2023-10-09T10:58:32.666051+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:49.907736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":298,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.727089+0000","last_change":"2023-10-09T10:54:47.939756+0000","last_active":"2023-10-09T10:58:31.727089+0000","last_peered":"2023-10-09T10:58:31.727089+0000","last_clean":"2023-10-09T10:58:31.727089+0000","last_became_active":"2023-10-09T10:54:47.938928+0000","last_became_peered":"2023-10-09T10:54:47.938928+0000","last_unstale":"2023-10-09T10:58:31.727089+0000","last_undegraded":"2023-10-09T10:58:31.727089+0000","last_fullsized":"2023-10-09T10:58:31.727089+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:05.310929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"354'6570","reported_seq":6601,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T10:58:46.225586+0000","last_change":"2023-10-09T10:57:22.010174+0000","last_active":"2023-10-09T10:58:46.225586+0000","last_peered":"2023-10-09T10:58:46.225586+0000","last_clean":"2023-10-09T10:58:46.225586+0000","last_became_active":"2023-10-09T10:57:22.009987+0000","last_became_peered":"2023-10-09T10:57:22.009987+0000","last_unstale":"2023-10-09T10:58:46.225586+0000","last_undegraded":"2023-10-09T10:58:46.225586+0000","last_fullsized":"2023-10-09T10:58:46.225586+0000","mapping_epoch":332,"log_start":"336'900","ondisk_log_start":"336'900","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5670,"log_dups_size":0,"ondisk_log_size":5670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:45.169457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430571520,"num_objects":6570,"num_object_clones":0,"num_object_copies":19710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6570,"num_write_kb":420480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":726,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.727302+0000","last_change":"2023-10-09T10:53:27.625180+0000","last_active":"2023-10-09T10:58:31.727302+0000","last_peered":"2023-10-09T10:58:31.727302+0000","last_clean":"2023-10-09T10:58:31.727302+0000","last_became_active":"2023-10-09T10:51:12.366828+0000","last_became_peered":"2023-10-09T10:51:12.366828+0000","last_unstale":"2023-10-09T10:58:31.727302+0000","last_undegraded":"2023-10-09T10:58:31.727302+0000","last_fullsized":"2023-10-09T10:58:31.727302+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.625079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.625079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:59:41.253976+0000","scrub_duration":0.0041067969999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":31,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.683702+0000","last_change":"2023-10-09T10:57:22.012147+0000","last_active":"2023-10-09T10:58:39.683702+0000","last_peered":"2023-10-09T10:58:39.683702+0000","last_clean":"2023-10-09T10:58:39.683702+0000","last_became_active":"2023-10-09T10:57:22.011818+0000","last_became_peered":"2023-10-09T10:57:22.011818+0000","last_unstale":"2023-10-09T10:58:39.683702+0000","last_undegraded":"2023-10-09T10:58:39.683702+0000","last_fullsized":"2023-10-09T10:58:39.683702+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:10.838344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":71,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.693056+0000","last_change":"2023-10-09T10:54:47.925711+0000","last_active":"2023-10-09T10:58:39.693056+0000","last_peered":"2023-10-09T10:58:39.693056+0000","last_clean":"2023-10-09T10:58:39.693056+0000","last_became_active":"2023-10-09T10:54:47.924286+0000","last_became_peered":"2023-10-09T10:54:47.924286+0000","last_unstale":"2023-10-09T10:58:39.693056+0000","last_undegraded":"2023-10-09T10:58:39.693056+0000","last_fullsized":"2023-10-09T10:58:39.693056+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:17:58.379080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":281,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:32.666120+0000","last_change":"2023-10-09T10:57:22.004838+0000","last_active":"2023-10-09T10:58:32.666120+0000","last_peered":"2023-10-09T10:58:32.666120+0000","last_clean":"2023-10-09T10:58:32.666120+0000","last_became_active":"2023-10-09T10:57:22.004519+0000","last_became_peered":"2023-10-09T10:57:22.004519+0000","last_unstale":"2023-10-09T10:58:32.666120+0000","last_undegraded":"2023-10-09T10:58:32.666120+0000","last_fullsized":"2023-10-09T10:58:32.666120+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:53.585466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"354'6418","reported_seq":6449,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T10:58:46.226497+0000","last_change":"2023-10-09T10:57:22.015103+0000","last_active":"2023-10-09T10:58:46.226497+0000","last_peered":"2023-10-09T10:58:46.226497+0000","last_clean":"2023-10-09T10:58:46.226497+0000","last_became_active":"2023-10-09T10:57:22.014603+0000","last_became_peered":"2023-10-09T10:57:22.014603+0000","last_unstale":"2023-10-09T10:58:46.226497+0000","last_undegraded":"2023-10-09T10:58:46.226497+0000","last_fullsized":"2023-10-09T10:58:46.226497+0000","mapping_epoch":332,"log_start":"336'700","ondisk_log_start":"336'700","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5718,"log_dups_size":0,"ondisk_log_size":5718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:22:34.112316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":420610048,"num_objects":6418,"num_object_clones":0,"num_object_copies":19254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6418,"num_write_kb":410752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":749,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.687120+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T10:58:39.687120+0000","last_peered":"2023-10-09T10:58:39.687120+0000","last_clean":"2023-10-09T10:58:39.687120+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T10:58:39.687120+0000","last_undegraded":"2023-10-09T10:58:39.687120+0000","last_fullsized":"2023-10-09T10:58:39.687120+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":403,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.681645+0000","last_change":"2023-10-09T10:57:22.030437+0000","last_active":"2023-10-09T10:58:39.681645+0000","last_peered":"2023-10-09T10:58:39.681645+0000","last_clean":"2023-10-09T10:58:39.681645+0000","last_became_active":"2023-10-09T10:57:22.008090+0000","last_became_peered":"2023-10-09T10:57:22.008090+0000","last_unstale":"2023-10-09T10:58:39.681645+0000","last_undegraded":"2023-10-09T10:58:39.681645+0000","last_fullsized":"2023-10-09T10:58:39.681645+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:52:54.221183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":71,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.690315+0000","last_change":"2023-10-09T10:54:47.908305+0000","last_active":"2023-10-09T10:58:39.690315+0000","last_peered":"2023-10-09T10:58:39.690315+0000","last_clean":"2023-10-09T10:58:39.690315+0000","last_became_active":"2023-10-09T10:54:47.907999+0000","last_became_peered":"2023-10-09T10:54:47.907999+0000","last_unstale":"2023-10-09T10:58:39.690315+0000","last_undegraded":"2023-10-09T10:58:39.690315+0000","last_fullsized":"2023-10-09T10:58:39.690315+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:48.405124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":394,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.719918+0000","last_change":"2023-10-09T10:57:22.007248+0000","last_active":"2023-10-09T10:58:31.719918+0000","last_peered":"2023-10-09T10:58:31.719918+0000","last_clean":"2023-10-09T10:58:31.719918+0000","last_became_active":"2023-10-09T10:57:22.006899+0000","last_became_peered":"2023-10-09T10:57:22.006899+0000","last_unstale":"2023-10-09T10:58:31.719918+0000","last_undegraded":"2023-10-09T10:58:31.719918+0000","last_fullsized":"2023-10-09T10:58:31.719918+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:45.159091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"353'6377","reported_seq":6418,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:45.603877+0000","last_change":"2023-10-09T10:57:22.006548+0000","last_active":"2023-10-09T10:58:45.603877+0000","last_peered":"2023-10-09T10:58:45.603877+0000","last_clean":"2023-10-09T10:58:45.603877+0000","last_became_active":"2023-10-09T10:57:22.006030+0000","last_became_peered":"2023-10-09T10:57:22.006030+0000","last_unstale":"2023-10-09T10:58:45.603877+0000","last_undegraded":"2023-10-09T10:58:45.603877+0000","last_fullsized":"2023-10-09T10:58:45.603877+0000","mapping_epoch":332,"log_start":"336'600","ondisk_log_start":"336'600","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5777,"log_dups_size":0,"ondisk_log_size":5777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:47.539524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417923072,"num_objects":6377,"num_object_clones":0,"num_object_copies":19131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6377,"num_write_kb":408128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":779,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.720828+0000","last_change":"2023-10-09T10:57:25.106977+0000","last_active":"2023-10-09T10:58:31.720828+0000","last_peered":"2023-10-09T10:58:31.720828+0000","last_clean":"2023-10-09T10:58:31.720828+0000","last_became_active":"2023-10-09T10:57:25.106632+0000","last_became_peered":"2023-10-09T10:57:25.106632+0000","last_unstale":"2023-10-09T10:58:31.720828+0000","last_undegraded":"2023-10-09T10:58:31.720828+0000","last_fullsized":"2023-10-09T10:58:31.720828+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:29:20.989344+0000","scrub_duration":0.0053707060000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":31,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.684515+0000","last_change":"2023-10-09T10:57:22.016009+0000","last_active":"2023-10-09T10:58:39.684515+0000","last_peered":"2023-10-09T10:58:39.684515+0000","last_clean":"2023-10-09T10:58:39.684515+0000","last_became_active":"2023-10-09T10:57:22.015633+0000","last_became_peered":"2023-10-09T10:57:22.015633+0000","last_unstale":"2023-10-09T10:58:39.684515+0000","last_undegraded":"2023-10-09T10:58:39.684515+0000","last_fullsized":"2023-10-09T10:58:39.684515+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.957416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":282,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.727162+0000","last_change":"2023-10-09T10:54:48.016159+0000","last_active":"2023-10-09T10:58:31.727162+0000","last_peered":"2023-10-09T10:58:31.727162+0000","last_clean":"2023-10-09T10:58:31.727162+0000","last_became_active":"2023-10-09T10:54:47.939409+0000","last_became_peered":"2023-10-09T10:54:47.939409+0000","last_unstale":"2023-10-09T10:58:31.727162+0000","last_undegraded":"2023-10-09T10:58:31.727162+0000","last_fullsized":"2023-10-09T10:58:31.727162+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:11:15.175207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":295,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.686455+0000","last_change":"2023-10-09T10:51:50.371282+0000","last_active":"2023-10-09T10:58:39.686455+0000","last_peered":"2023-10-09T10:58:39.686455+0000","last_clean":"2023-10-09T10:58:39.686455+0000","last_became_active":"2023-10-09T10:51:50.370973+0000","last_became_peered":"2023-10-09T10:51:50.370973+0000","last_unstale":"2023-10-09T10:58:39.686455+0000","last_undegraded":"2023-10-09T10:58:39.686455+0000","last_fullsized":"2023-10-09T10:58:39.686455+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:15.603164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":745,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.715902+0000","last_change":"2023-10-09T10:53:29.568199+0000","last_active":"2023-10-09T10:58:31.715902+0000","last_peered":"2023-10-09T10:58:31.715902+0000","last_clean":"2023-10-09T10:58:31.715902+0000","last_became_active":"2023-10-09T10:52:55.416357+0000","last_became_peered":"2023-10-09T10:52:55.416357+0000","last_unstale":"2023-10-09T10:58:31.715902+0000","last_undegraded":"2023-10-09T10:58:31.715902+0000","last_fullsized":"2023-10-09T10:58:31.715902+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:44.364650+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":280,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:32.666137+0000","last_change":"2023-10-09T10:57:22.005039+0000","last_active":"2023-10-09T10:58:32.666137+0000","last_peered":"2023-10-09T10:58:32.666137+0000","last_clean":"2023-10-09T10:58:32.666137+0000","last_became_active":"2023-10-09T10:57:22.004497+0000","last_became_peered":"2023-10-09T10:57:22.004497+0000","last_unstale":"2023-10-09T10:58:32.666137+0000","last_undegraded":"2023-10-09T10:58:32.666137+0000","last_fullsized":"2023-10-09T10:58:32.666137+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:23:11.006259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.729525+0000","last_change":"2023-10-09T10:54:47.944380+0000","last_active":"2023-10-09T10:58:31.729525+0000","last_peered":"2023-10-09T10:58:31.729525+0000","last_clean":"2023-10-09T10:58:31.729525+0000","last_became_active":"2023-10-09T10:54:47.943766+0000","last_became_peered":"2023-10-09T10:54:47.943766+0000","last_unstale":"2023-10-09T10:58:31.729525+0000","last_undegraded":"2023-10-09T10:58:31.729525+0000","last_fullsized":"2023-10-09T10:58:31.729525+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:16:11.869603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":710,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.716490+0000","last_change":"2023-10-09T10:57:25.111170+0000","last_active":"2023-10-09T10:58:31.716490+0000","last_peered":"2023-10-09T10:58:31.716490+0000","last_clean":"2023-10-09T10:58:31.716490+0000","last_became_active":"2023-10-09T10:57:25.110405+0000","last_became_peered":"2023-10-09T10:57:25.110405+0000","last_unstale":"2023-10-09T10:58:31.716490+0000","last_undegraded":"2023-10-09T10:58:31.716490+0000","last_fullsized":"2023-10-09T10:58:31.716490+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:00.112600+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":180,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.687138+0000","last_change":"2023-10-09T10:54:47.923386+0000","last_active":"2023-10-09T10:58:39.687138+0000","last_peered":"2023-10-09T10:58:39.687138+0000","last_clean":"2023-10-09T10:58:39.687138+0000","last_became_active":"2023-10-09T10:54:47.914257+0000","last_became_peered":"2023-10-09T10:54:47.914257+0000","last_unstale":"2023-10-09T10:58:39.687138+0000","last_undegraded":"2023-10-09T10:58:39.687138+0000","last_fullsized":"2023-10-09T10:58:39.687138+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:24:18.190715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":380,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.716545+0000","last_change":"2023-10-09T10:57:46.844827+0000","last_active":"2023-10-09T10:58:31.716545+0000","last_peered":"2023-10-09T10:58:31.716545+0000","last_clean":"2023-10-09T10:58:31.716545+0000","last_became_active":"2023-10-09T10:57:46.844584+0000","last_became_peered":"2023-10-09T10:57:46.844584+0000","last_unstale":"2023-10-09T10:58:31.716545+0000","last_undegraded":"2023-10-09T10:58:31.716545+0000","last_fullsized":"2023-10-09T10:58:31.716545+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:54.499813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":304,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.684694+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:58:39.684694+0000","last_peered":"2023-10-09T10:58:39.684694+0000","last_clean":"2023-10-09T10:58:39.684694+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:58:39.684694+0000","last_undegraded":"2023-10-09T10:58:39.684694+0000","last_fullsized":"2023-10-09T10:58:39.684694+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":298,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.716342+0000","last_change":"2023-10-09T10:50:43.385150+0000","last_active":"2023-10-09T10:58:31.716342+0000","last_peered":"2023-10-09T10:58:31.716342+0000","last_clean":"2023-10-09T10:58:31.716342+0000","last_became_active":"2023-10-09T10:50:43.384788+0000","last_became_peered":"2023-10-09T10:50:43.384788+0000","last_unstale":"2023-10-09T10:58:31.716342+0000","last_undegraded":"2023-10-09T10:58:31.716342+0000","last_fullsized":"2023-10-09T10:58:31.716342+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:36.085910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":645,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.684166+0000","last_change":"2023-10-09T10:57:47.307757+0000","last_active":"2023-10-09T10:58:39.684166+0000","last_peered":"2023-10-09T10:58:39.684166+0000","last_clean":"2023-10-09T10:58:39.684166+0000","last_became_active":"2023-10-09T10:57:25.104921+0000","last_became_peered":"2023-10-09T10:57:25.104921+0000","last_unstale":"2023-10-09T10:58:39.684166+0000","last_undegraded":"2023-10-09T10:58:39.684166+0000","last_fullsized":"2023-10-09T10:58:39.684166+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:47.307693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:47.307693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:40.935036+0000","scrub_duration":0.003001985,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":378,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.716835+0000","last_change":"2023-10-09T10:54:17.886158+0000","last_active":"2023-10-09T10:58:31.716835+0000","last_peered":"2023-10-09T10:58:31.716835+0000","last_clean":"2023-10-09T10:58:31.716835+0000","last_became_active":"2023-10-09T10:54:17.885816+0000","last_became_peered":"2023-10-09T10:54:17.885816+0000","last_unstale":"2023-10-09T10:58:31.716835+0000","last_undegraded":"2023-10-09T10:58:31.716835+0000","last_fullsized":"2023-10-09T10:58:31.716835+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:05.270825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":381,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:31.716296+0000","last_change":"2023-10-09T10:54:17.957784+0000","last_active":"2023-10-09T10:58:31.716296+0000","last_peered":"2023-10-09T10:58:31.716296+0000","last_clean":"2023-10-09T10:58:31.716296+0000","last_became_active":"2023-10-09T10:54:17.954788+0000","last_became_peered":"2023-10-09T10:54:17.954788+0000","last_unstale":"2023-10-09T10:58:31.716296+0000","last_undegraded":"2023-10-09T10:58:31.716296+0000","last_fullsized":"2023-10-09T10:58:31.716296+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:48.709390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":641,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.690028+0000","last_change":"2023-10-09T10:57:21.022980+0000","last_active":"2023-10-09T10:58:39.690028+0000","last_peered":"2023-10-09T10:58:39.690028+0000","last_clean":"2023-10-09T10:58:39.690028+0000","last_became_active":"2023-10-09T10:54:50.914005+0000","last_became_peered":"2023-10-09T10:54:50.914005+0000","last_unstale":"2023-10-09T10:58:39.690028+0000","last_undegraded":"2023-10-09T10:58:39.690028+0000","last_fullsized":"2023-10-09T10:58:39.690028+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:21.022863+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:21.022863+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:01:49.534216+0000","scrub_duration":0.00136929,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":263,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T10:58:32.666229+0000","last_change":"2023-10-09T10:54:18.089053+0000","last_active":"2023-10-09T10:58:32.666229+0000","last_peered":"2023-10-09T10:58:32.666229+0000","last_clean":"2023-10-09T10:58:32.666229+0000","last_became_active":"2023-10-09T10:54:17.886070+0000","last_became_peered":"2023-10-09T10:54:17.886070+0000","last_unstale":"2023-10-09T10:58:32.666229+0000","last_undegraded":"2023-10-09T10:58:32.666229+0000","last_fullsized":"2023-10-09T10:58:32.666229+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:35.628407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":358,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T10:58:39.686372+0000","last_change":"2023-10-09T10:54:47.908059+0000","last_active":"2023-10-09T10:58:39.686372+0000","last_peered":"2023-10-09T10:58:39.686372+0000","last_clean":"2023-10-09T10:58:39.686372+0000","last_became_active":"2023-10-09T10:54:47.907780+0000","last_became_peered":"2023-10-09T10:54:47.907780+0000","last_unstale":"2023-10-09T10:58:39.686372+0000","last_undegraded":"2023-10-09T10:58:39.686372+0000","last_fullsized":"2023-10-09T10:58:39.686372+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:55:59.064617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":16490,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":6738477056,"num_objects":102821,"num_object_clones":0,"num_object_copies":308463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":102821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":102821,"num_write_kb":6580544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20289748992,"data_stored":20289748992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95021,"ondisk_log_size":95021,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332386,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4179792,"kb_used_data":3870216,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":89565360,"statfs":{"total":95995035648,"available":91714928640,"internally_reserved":0,"allocated":3963101184,"data_stored":3961897435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,1,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:03 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.821,"15min":1.4870000000000001},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":4.5190000000000001,"5min":9.6259999999999994,"15min":9.6379999999999999},"last":4.7759999999999998},{"interface":"front","average":{"1min":1.1499999999999999,"5min":2.004,"15min":1.7030000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.4470000000000001,"5min":8.7769999999999992,"15min":16.731000000000002},"last":3.266}]},{"osd":1,"last update":"Mon Oct 9 10:58:10 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":1.835,"15min":1.827},"min":{"1min":0.26500000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.2880000000000003,"5min":12.252000000000001,"15min":34.756},"last":3.4590000000000001},{"interface":"front","average":{"1min":1.177,"5min":1.744,"15min":1.609},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":2.4649999999999999,"5min":13.647,"15min":35.658999999999999},"last":4.806}]},{"osd":2,"last update":"Mon Oct 9 10:58:35 2023","interfaces":[{"interface":"back","average":{"1min":1.5940000000000001,"5min":1.7490000000000001,"15min":1.6200000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":5.7619999999999996,"5min":9.202,"15min":11.058},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.5,"5min":2.077,"15min":1.575},"min":{"1min":0.34000000000000002,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":8.9280000000000008,"5min":13.037000000000001,"15min":13.037000000000001},"last":4.7469999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:58:03 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":2.1080000000000001,"15min":1.913},"min":{"1min":0.33200000000000002,"5min":0.31,"15min":0.31},"max":{"1min":7.1970000000000001,"5min":11.760999999999999,"15min":11.760999999999999},"last":6.149},{"interface":"front","average":{"1min":0.89900000000000002,"5min":1.5489999999999999,"15min":1.627},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":2.2040000000000002,"5min":8.6300000000000008,"15min":19.234999999999999},"last":0.35099999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":1.0389999999999999,"15min":1.3320000000000001},"min":{"1min":0.29399999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.6429999999999998,"5min":8.2940000000000005,"15min":16.695},"last":4.5940000000000003},{"interface":"front","average":{"1min":0.997,"5min":0.86599999999999999,"15min":1.1699999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26200000000000001},"max":{"1min":3.6779999999999999,"5min":7.9320000000000004,"15min":11.109999999999999},"last":0.496}]},{"osd":5,"last update":"Mon Oct 9 10:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":1.0189999999999999,"15min":1.2709999999999999},"min":{"1min":0.29399999999999998,"5min":0.247,"15min":0.247},"max":{"1min":7.9690000000000003,"5min":7.9690000000000003,"15min":13.262},"last":4.8250000000000002},{"interface":"front","average":{"1min":1.6299999999999999,"5min":0.97899999999999998,"15min":1.4139999999999999},"min":{"1min":0.33600000000000002,"5min":0.318,"15min":0.29599999999999999},"max":{"1min":4.7409999999999997,"5min":8.4339999999999993,"15min":11.741},"last":3.4319999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":1.079,"15min":1.538},"min":{"1min":0.39500000000000002,"5min":0.315,"15min":0.28199999999999997},"max":{"1min":5.2430000000000003,"5min":8.3610000000000007,"15min":14.913},"last":6.1790000000000003},{"interface":"front","average":{"1min":1.4099999999999999,"5min":1.0389999999999999,"15min":1.3740000000000001},"min":{"1min":0.37,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.1139999999999999,"5min":7.9809999999999999,"15min":19.187999999999999},"last":0.41699999999999998}]}]},{"osd":6,"up_from":44,"seq":188978561317,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2293248,"kb_used_data":1393080,"kb_used_omap":0,"kb_used_meta":900160,"kb_avail":91451904,"statfs":{"total":95995035648,"available":93646749696,"internally_reserved":0,"allocated":1426513920,"data_stored":1425325865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921763840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:00 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":2.1469999999999998,"15min":1.6120000000000001},"min":{"1min":0.27500000000000002,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":5.0019999999999998,"5min":54.743000000000002,"15min":54.743000000000002},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.2370000000000001,"5min":2.0110000000000001,"15min":1.6359999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":4.9119999999999999,"5min":53.701000000000001,"15min":53.701000000000001},"last":0.34399999999999997}]},{"osd":1,"last update":"Mon Oct 9 10:58:14 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":1.474,"15min":1.4770000000000001},"min":{"1min":0.44,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.96,"5min":7.6820000000000004,"15min":16.747},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.869,"5min":1.5229999999999999,"15min":1.534},"min":{"1min":0.376,"5min":0.26400000000000001,"15min":0.23100000000000001},"max":{"1min":8.6419999999999995,"5min":8.6419999999999995,"15min":16.734000000000002},"last":1.133}]},{"osd":2,"last update":"Mon Oct 9 10:58:40 2023","interfaces":[{"interface":"back","average":{"1min":2.8239999999999998,"5min":2.0409999999999999,"15min":1.7190000000000001},"min":{"1min":0.34699999999999998,"5min":0.29199999999999998,"15min":0.26800000000000002},"max":{"1min":8.6869999999999994,"5min":12.003,"15min":13.208},"last":1.222},{"interface":"front","average":{"1min":2.6520000000000001,"5min":1.8380000000000001,"15min":1.79},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28399999999999997},"max":{"1min":15.432,"5min":15.432,"15min":15.432},"last":2.0179999999999998}]},{"osd":3,"last update":"Mon Oct 9 10:58:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3819999999999999,"5min":1.736,"15min":1.629},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.255},"max":{"1min":5.2969999999999997,"5min":7.8680000000000003,"15min":12.391},"last":0.83699999999999997},{"interface":"front","average":{"1min":1.738,"5min":1.984,"15min":1.895},"min":{"1min":0.40000000000000002,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":5.3949999999999996,"5min":8.782,"15min":13.179},"last":1.173}]},{"osd":4,"last update":"Mon Oct 9 10:58:40 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.83299999999999996,"15min":1.1950000000000001},"min":{"1min":0.27900000000000003,"5min":0.24199999999999999,"15min":0.23200000000000001},"max":{"1min":3.4620000000000002,"5min":16.978999999999999,"15min":16.978999999999999},"last":0.29899999999999999},{"interface":"front","average":{"1min":1.1140000000000001,"5min":0.92200000000000004,"15min":1.1839999999999999},"min":{"1min":0.29899999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.5129999999999999,"5min":4.7930000000000001,"15min":12.481999999999999},"last":1.2050000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:58:40 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":1.171,"15min":1.373},"min":{"1min":0.35499999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.3220000000000001,"5min":12.923,"15min":12.923},"last":1.5900000000000001},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.032,"15min":1.2709999999999999},"min":{"1min":0.28899999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.46,"5min":9.0559999999999992,"15min":12.456},"last":0.40799999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:57:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.0329999999999999,"15min":1.3480000000000001},"min":{"1min":0.35999999999999999,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":1.1930000000000001,"5min":12.882,"15min":23.358000000000001},"last":2.2669999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.92000000000000004,"15min":1.147},"min":{"1min":0.36099999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.141,"5min":6.1029999999999998,"15min":12.422000000000001},"last":4.2830000000000004}]}]},{"osd":1,"up_from":85,"seq":365072220356,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4244096,"kb_used_data":3731128,"kb_used_omap":9,"kb_used_meta":512950,"kb_avail":89501056,"statfs":{"total":95995035648,"available":91649081344,"internally_reserved":0,"allocated":3820675072,"data_stored":3818854286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":525261801},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":2.464,"15min":2.0099999999999998},"min":{"1min":0.224,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":4.3049999999999997,"5min":19.992999999999999,"15min":19.992999999999999},"last":1.649},{"interface":"front","average":{"1min":2.4369999999999998,"5min":2.9550000000000001,"15min":2.1899999999999999},"min":{"1min":0.27000000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":6.9790000000000001,"5min":18.303000000000001,"15min":18.303000000000001},"last":2.0870000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:58:39 2023","interfaces":[{"interface":"back","average":{"1min":1.865,"5min":2.302,"15min":1.7410000000000001},"min":{"1min":0.23100000000000001,"5min":0.223,"15min":0.19700000000000001},"max":{"1min":5.9980000000000002,"5min":14.878,"15min":15.356},"last":1.893},{"interface":"front","average":{"1min":1.2490000000000001,"5min":2.1800000000000002,"15min":1.8320000000000001},"min":{"1min":0.221,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":4.6059999999999999,"5min":18.004000000000001,"15min":18.004000000000001},"last":1.724}]},{"osd":3,"last update":"Mon Oct 9 10:58:00 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":2.605,"15min":2.23},"min":{"1min":0.23100000000000001,"5min":0.22900000000000001,"15min":0.218},"max":{"1min":8.2080000000000002,"5min":20.036000000000001,"15min":20.036000000000001},"last":1.7390000000000001},{"interface":"front","average":{"1min":2.827,"5min":2.9809999999999999,"15min":2.2829999999999999},"min":{"1min":0.24099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":8.8680000000000003,"5min":19.640999999999998,"15min":19.640999999999998},"last":2.1219999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":1.5009999999999999,"15min":1.468},"min":{"1min":0.30099999999999999,"5min":0.28899999999999998,"15min":0.255},"max":{"1min":1.71,"5min":14.763,"15min":14.763},"last":0.439},{"interface":"front","average":{"1min":1.4099999999999999,"5min":1.75,"15min":1.7190000000000001},"min":{"1min":0.379,"5min":0.30399999999999999,"15min":0.27200000000000002},"max":{"1min":4.0199999999999996,"5min":8.3079999999999998,"15min":14.228999999999999},"last":1.6759999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":1.9670000000000001,"15min":1.631},"min":{"1min":0.376,"5min":0.32700000000000001,"15min":0.253},"max":{"1min":5.9989999999999997,"5min":9.6289999999999996,"15min":9.7720000000000002},"last":2.1549999999999998},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.873,"15min":1.8380000000000001},"min":{"1min":0.313,"5min":0.30299999999999999,"15min":0.28899999999999998},"max":{"1min":2.1930000000000001,"5min":14.833,"15min":17.085000000000001},"last":0.40600000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.8640000000000001,"15min":1.716},"min":{"1min":0.41799999999999998,"5min":0.32600000000000001,"15min":0.27400000000000002},"max":{"1min":5.9630000000000001,"5min":8.8230000000000004,"15min":10.425000000000001},"last":2.1840000000000002},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.653,"15min":1.6479999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.312},"max":{"1min":1.74,"5min":14.804,"15min":14.804},"last":0.46500000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.8180000000000001,"15min":1.7150000000000001},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.34000000000000002},"max":{"1min":2.5619999999999998,"5min":14.856,"15min":16.312000000000001},"last":1.607},{"interface":"front","average":{"1min":1.2829999999999999,"5min":1.6839999999999999,"15min":1.7130000000000001},"min":{"1min":0.41099999999999998,"5min":0.35499999999999998,"15min":0.32500000000000001},"max":{"1min":2.8940000000000001,"5min":8.8149999999999995,"15min":14.273999999999999},"last":1.7070000000000001}]}]},{"osd":0,"up_from":80,"seq":343597383889,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2155200,"kb_used_data":1609976,"kb_used_omap":28,"kb_used_meta":545187,"kb_avail":91589952,"statfs":{"total":95995035648,"available":93788110848,"internally_reserved":0,"allocated":1648615424,"data_stored":1646723892,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":558272336},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 10:58:03 2023","interfaces":[{"interface":"back","average":{"1min":2.254,"5min":2.4079999999999999,"15min":2.0310000000000001},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":12.301,"5min":13.263999999999999,"15min":25.475000000000001},"last":3.2170000000000001},{"interface":"front","average":{"1min":1.8240000000000001,"5min":2.0790000000000002,"15min":1.9339999999999999},"min":{"1min":0.20200000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":6.1680000000000001,"5min":10.478999999999999,"15min":20.832999999999998},"last":3.6440000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:58:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0179999999999998,"5min":2.6989999999999998,"15min":2.073},"min":{"1min":0.20999999999999999,"5min":0.189,"15min":0.189},"max":{"1min":6.484,"5min":24.852,"15min":24.852},"last":2.367},{"interface":"front","average":{"1min":3.1000000000000001,"5min":3.73,"15min":2.569},"min":{"1min":0.33500000000000002,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":12.494999999999999,"5min":77.256,"15min":77.256},"last":1.294}]},{"osd":3,"last update":"Mon Oct 9 10:58:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":2.1709999999999998,"15min":2.1509999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20200000000000001},"max":{"1min":8.3510000000000009,"5min":18.972000000000001,"15min":18.972000000000001},"last":0.218},{"interface":"front","average":{"1min":2.2160000000000002,"5min":2.8140000000000001,"15min":2.298},"min":{"1min":0.27600000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":12.353999999999999,"5min":13.305,"15min":14.391999999999999},"last":0.34899999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":2.8839999999999999,"15min":2.1019999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.26200000000000001},"max":{"1min":6.133,"5min":77.177999999999997,"15min":77.177999999999997},"last":2.3239999999999998},{"interface":"front","average":{"1min":1.532,"5min":1.679,"15min":1.8799999999999999},"min":{"1min":0.32600000000000001,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":6.843,"5min":10.513999999999999,"15min":12.318},"last":1.3600000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":1.766,"15min":1.8999999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.29799999999999999},"max":{"1min":6.9100000000000001,"5min":10.544,"15min":12.157},"last":1.284},{"interface":"front","average":{"1min":1.3740000000000001,"5min":1.76,"15min":1.796},"min":{"1min":0.39400000000000002,"5min":0.29899999999999999,"15min":0.29499999999999998},"max":{"1min":6.7469999999999999,"5min":24.754999999999999,"15min":24.754999999999999},"last":3.7730000000000001}]},{"osd":6,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.96,"15min":2.0329999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.30399999999999999},"max":{"1min":6.8840000000000003,"5min":18.920000000000002,"15min":18.920000000000002},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.47,"5min":2,"15min":1.714},"min":{"1min":0.38,"5min":0.32900000000000001,"15min":0.307},"max":{"1min":6.7130000000000001,"5min":24.800000000000001,"15min":24.800000000000001},"last":4.7610000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.8839999999999999,"15min":1.909},"min":{"1min":0.44900000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.6609999999999996,"5min":13.225,"15min":14.48},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.7470000000000001,"5min":2.8479999999999999,"15min":2.214},"min":{"1min":0.38,"5min":0.314,"15min":0.314},"max":{"1min":6.101,"5min":77.228999999999999,"15min":77.228999999999999},"last":7.2050000000000001}]}]},{"osd":2,"up_from":90,"seq":386547056830,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1625804,"kb_used_data":1230276,"kb_used_omap":81,"kb_used_meta":395438,"kb_avail":92119348,"statfs":{"total":95995035648,"available":94330212352,"internally_reserved":0,"allocated":1259802624,"data_stored":1258075928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":404928730},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.012,"15min":1.776},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":6.702,"5min":17.574000000000002,"15min":18.512},"last":0.40100000000000002},{"interface":"front","average":{"1min":2.698,"5min":2.1070000000000002,"15min":1.917},"min":{"1min":0.27400000000000002,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":7.4699999999999998,"5min":11.167,"15min":17.975000000000001},"last":4.0949999999999998}]},{"osd":1,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":3.1709999999999998,"5min":2.117,"15min":1.8879999999999999},"min":{"1min":0.308,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":11.148,"5min":16.359000000000002,"15min":16.359000000000002},"last":8.2789999999999999},{"interface":"front","average":{"1min":2.7669999999999999,"5min":2.254,"15min":1.7270000000000001},"min":{"1min":0.27800000000000002,"5min":0.24199999999999999,"15min":0.20699999999999999},"max":{"1min":7.8970000000000002,"5min":12.693,"15min":16.454999999999998},"last":5.7830000000000004}]},{"osd":3,"last update":"Mon Oct 9 10:58:00 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":2.355,"15min":1.839},"min":{"1min":0.24399999999999999,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":3.6349999999999998,"5min":19.539000000000001,"15min":19.539000000000001},"last":4.4550000000000001},{"interface":"front","average":{"1min":2.2410000000000001,"5min":2.4169999999999998,"15min":2.1499999999999999},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.20200000000000001},"max":{"1min":8.8729999999999993,"5min":16.702999999999999,"15min":16.702999999999999},"last":16.324000000000002}]},{"osd":4,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4399999999999999,"5min":2.23,"15min":1.9339999999999999},"min":{"1min":0.33700000000000002,"5min":0.28599999999999998,"15min":0.254},"max":{"1min":7.4080000000000004,"5min":15.672000000000001,"15min":16.231999999999999},"last":0.80500000000000005},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.8680000000000001,"15min":1.7929999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.26300000000000001},"max":{"1min":6.4020000000000001,"5min":15.696999999999999,"15min":15.696999999999999},"last":0.86299999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":2.468,"5min":1.784,"15min":1.7649999999999999},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.245},"max":{"1min":6.7709999999999999,"5min":12.128,"15min":17.920000000000002},"last":4.2050000000000001},{"interface":"front","average":{"1min":2.3530000000000002,"5min":2.2650000000000001,"15min":1.8740000000000001},"min":{"1min":0.313,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.3719999999999999,"5min":16.597000000000001,"15min":17.882999999999999},"last":0.66100000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":1.607,"15min":1.569},"min":{"1min":0.34499999999999997,"5min":0.33400000000000002,"15min":0.247},"max":{"1min":6.3689999999999998,"5min":8.8460000000000001,"15min":14.029999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":2.5169999999999999,"5min":1.7390000000000001,"15min":1.7330000000000001},"min":{"1min":0.40500000000000003,"5min":0.34200000000000003,"15min":0.29199999999999998},"max":{"1min":5.9909999999999997,"5min":10.111000000000001,"15min":12.862},"last":4.1399999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":1.762,"15min":1.625},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.3300000000000001,"5min":8.8879999999999999,"15min":13.999000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.819,"5min":1.929,"15min":1.831},"min":{"1min":0.41999999999999998,"5min":0.34000000000000002,"15min":0.307},"max":{"1min":5.9459999999999997,"5min":10.157999999999999,"15min":12.826000000000001},"last":4.1699999999999999}]}]},{"osd":3,"up_from":95,"seq":408021893304,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4036184,"kb_used_data":3287184,"kb_used_omap":7,"kb_used_meta":748984,"kb_avail":89708968,"statfs":{"total":95995035648,"available":91861983232,"internally_reserved":0,"allocated":3366076416,"data_stored":3364160505,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":766960232},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":3.399,"15min":2.2930000000000001},"min":{"1min":0.23499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":10.140000000000001,"5min":38.817999999999998,"15min":38.817999999999998},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.776,"5min":3.661,"15min":2.5840000000000001},"min":{"1min":0.16500000000000001,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":7.6580000000000004,"5min":17.312000000000001,"15min":17.312000000000001},"last":0.214}]},{"osd":1,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":2.5990000000000002,"15min":2.3530000000000002},"min":{"1min":0.28000000000000003,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":13.244999999999999,"5min":22.469000000000001,"15min":22.469000000000001},"last":1.861},{"interface":"front","average":{"1min":2.52,"5min":3.4020000000000001,"15min":2.4409999999999998},"min":{"1min":0.28999999999999998,"5min":0.22,"15min":0.193},"max":{"1min":13.359,"5min":18.526,"15min":18.526},"last":3.8319999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":2.7559999999999998,"15min":2.214},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.6260000000000003,"5min":20.402000000000001,"15min":20.402000000000001},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.3680000000000001,"5min":3.637,"15min":2.4900000000000002},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":4.4320000000000004,"5min":27.538,"15min":27.538},"last":0.47099999999999997}]},{"osd":4,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":2.4180000000000001,"15min":1.992},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.8089999999999993,"5min":11.863,"15min":20.388000000000002},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.22,"5min":2.2490000000000001,"15min":1.9239999999999999},"min":{"1min":0.34000000000000002,"5min":0.28399999999999997,"15min":0.24299999999999999},"max":{"1min":4.0529999999999999,"5min":11.680999999999999,"15min":19.085999999999999},"last":0.40200000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":2.3290000000000002,"15min":2.2029999999999998},"min":{"1min":0.39300000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.0600000000000001,"5min":14.045,"15min":51.509999999999998},"last":0.37},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.8839999999999999,"15min":2.1680000000000001},"min":{"1min":0.41899999999999998,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":4.3780000000000001,"5min":17.311,"15min":55.338999999999999},"last":0.91200000000000003}]},{"osd":6,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":2.335,"15min":1.929},"min":{"1min":0.35599999999999998,"5min":0.29199999999999998,"15min":0.28000000000000003},"max":{"1min":3.1120000000000001,"5min":11.723000000000001,"15min":12.361000000000001},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.306,"5min":2.036,"15min":2.0819999999999999},"min":{"1min":0.39500000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.5229999999999997,"5min":17.295000000000002,"15min":17.295000000000002},"last":0.59399999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:58:06 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":2.2269999999999999,"15min":2.3140000000000001},"min":{"1min":0.439,"5min":0.34999999999999998,"15min":0.31},"max":{"1min":4.3330000000000002,"5min":18.722000000000001,"15min":51.744},"last":1.474},{"interface":"front","average":{"1min":1.5609999999999999,"5min":2.5259999999999998,"15min":2.335},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.8569999999999993,"5min":13.337,"15min":53.399000000000001},"last":0.82199999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019180,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2137300,"kb_used_data":1248652,"kb_used_omap":0,"kb_used_meta":888640,"kb_avail":91607852,"statfs":{"total":95995035648,"available":93806440448,"internally_reserved":0,"allocated":1278619648,"data_stored":1277419642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":909967360},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:57:55 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":4.0410000000000004,"15min":2.202},"min":{"1min":0.224,"5min":0.224,"15min":0.20999999999999999},"max":{"1min":4.2629999999999999,"5min":213.88999999999999,"15min":213.88999999999999},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.91800000000000004,"5min":3.9580000000000002,"15min":2.4980000000000002},"min":{"1min":0.26100000000000001,"5min":0.23200000000000001,"15min":0.217},"max":{"1min":3.012,"5min":214.09800000000001,"15min":214.09800000000001},"last":3.7330000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":3.7559999999999998,"15min":2.2149999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":3.286,"5min":206.50399999999999,"15min":206.50399999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.99099999999999999,"5min":3.7839999999999998,"15min":2.419},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.2959999999999998,"5min":206.529,"15min":206.529},"last":0.69799999999999995}]},{"osd":2,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":1.9399999999999999,"5min":1.9139999999999999,"15min":1.744},"min":{"1min":0.27100000000000002,"5min":0.25900000000000001,"15min":0.253},"max":{"1min":5.6870000000000003,"5min":15.446999999999999,"15min":15.446999999999999},"last":3.8330000000000002},{"interface":"front","average":{"1min":2.9969999999999999,"5min":2.605,"15min":2.073},"min":{"1min":0.30099999999999999,"5min":0.29699999999999999,"15min":0.26300000000000001},"max":{"1min":6.5019999999999998,"5min":27.818000000000001,"15min":27.818000000000001},"last":3.859}]},{"osd":3,"last update":"Mon Oct 9 10:57:55 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":3.8559999999999999,"15min":2.4500000000000002},"min":{"1min":0.374,"5min":0.34599999999999997,"15min":0.27400000000000002},"max":{"1min":2.7789999999999999,"5min":209.03700000000001,"15min":209.03700000000001},"last":12.798999999999999},{"interface":"front","average":{"1min":1.3580000000000001,"5min":4.1760000000000002,"15min":2.6930000000000001},"min":{"1min":0.39600000000000002,"5min":0.30499999999999999,"15min":0.29999999999999999},"max":{"1min":5.1529999999999996,"5min":208.80199999999999,"15min":208.80199999999999},"last":3.7789999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:57:59 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":1.1120000000000001,"15min":1.577},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.224},"max":{"1min":0.77800000000000002,"5min":19.068999999999999,"15min":24.149999999999999},"last":4.2640000000000002},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.05,"15min":1.3979999999999999},"min":{"1min":0.28499999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.8360000000000001,"5min":15.535,"15min":20.422000000000001},"last":3.6749999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:58:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":0.98099999999999998,"15min":1.2709999999999999},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":4.944,"5min":15.739000000000001,"15min":15.739000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.046,"5min":1.0940000000000001,"15min":1.494},"min":{"1min":0.32800000000000001,"5min":0.32700000000000001,"15min":0.311},"max":{"1min":4.4240000000000004,"5min":14.093999999999999,"15min":14.272},"last":4.2389999999999999}]},{"osd":7,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":1.367,"15min":1.4830000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":9.7620000000000005,"5min":14.458,"15min":14.458},"last":0.76600000000000001},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.532,"15min":1.3919999999999999},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.255},"max":{"1min":9.9789999999999992,"5min":15.592000000000001,"15min":15.592000000000001},"last":3.8039999999999998}]}]},{"osd":5,"up_from":37,"seq":158913790249,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4632188,"kb_used_data":3729460,"kb_used_omap":0,"kb_used_meta":902720,"kb_avail":89112964,"statfs":{"total":95995035648,"available":91251675136,"internally_reserved":0,"allocated":3818967040,"data_stored":3817779445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924385280},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.927,"15min":1.653},"min":{"1min":0.248,"5min":0.215,"15min":0.215},"max":{"1min":3.9590000000000001,"5min":16.449000000000002,"15min":16.449000000000002},"last":3.1819999999999999},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.9770000000000001,"15min":1.7090000000000001},"min":{"1min":0.30499999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.984,"5min":16.489000000000001,"15min":16.489000000000001},"last":3.25}]},{"osd":1,"last update":"Mon Oct 9 10:58:15 2023","interfaces":[{"interface":"back","average":{"1min":4.2359999999999998,"5min":2.5880000000000001,"15min":2.1019999999999999},"min":{"1min":0.313,"5min":0.215,"15min":0.215},"max":{"1min":29.818999999999999,"5min":29.818999999999999,"15min":29.818999999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":3.9409999999999998,"5min":2.3029999999999999,"15min":1.804},"min":{"1min":0.28199999999999997,"5min":0.253,"15min":0.24299999999999999},"max":{"1min":29.600000000000001,"5min":29.600000000000001,"15min":29.600000000000001},"last":1.2869999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:58:34 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":2.2170000000000001,"15min":1.9199999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":4.7050000000000001,"5min":11.257999999999999,"15min":11.393000000000001},"last":2.7850000000000001},{"interface":"front","average":{"1min":2.0499999999999998,"5min":2.2559999999999998,"15min":1.7430000000000001},"min":{"1min":0.33100000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.7889999999999997,"5min":9.2569999999999997,"15min":15.087999999999999},"last":3.302}]},{"osd":3,"last update":"Mon Oct 9 10:58:02 2023","interfaces":[{"interface":"back","average":{"1min":2.3999999999999999,"5min":2.165,"15min":2.206},"min":{"1min":0.32100000000000001,"5min":0.314,"15min":0.25700000000000001},"max":{"1min":25.846,"5min":25.846,"15min":63.508000000000003},"last":6.5780000000000003},{"interface":"front","average":{"1min":2.7050000000000001,"5min":2.5169999999999999,"15min":2.3860000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.26300000000000001},"max":{"1min":25.073,"5min":25.073,"15min":62.418999999999997},"last":0.74099999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:58:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.68600000000000005,"15min":1.395},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23200000000000001},"max":{"1min":2.726,"5min":4.4989999999999997,"15min":123.241},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.93200000000000005,"15min":1.6339999999999999},"min":{"1min":0.29199999999999998,"5min":0.28000000000000003,"15min":0.24199999999999999},"max":{"1min":4.8840000000000003,"5min":4.8840000000000003,"15min":123.318},"last":0.56999999999999995}]},{"osd":6,"last update":"Mon Oct 9 10:58:25 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":1.0880000000000001,"15min":1.3520000000000001},"min":{"1min":0.28899999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.1509999999999998,"5min":8.9049999999999994,"15min":11.098000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.79700000000000004,"15min":1.2290000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.1469999999999998,"5min":6.7210000000000001,"15min":20.256},"last":0.51000000000000001}]},{"osd":7,"last update":"Mon Oct 9 10:57:49 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":1.085,"15min":1.3169999999999999},"min":{"1min":0.33900000000000002,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":2.968,"5min":8.468,"15min":10.739000000000001},"last":2.75},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.82899999999999996,"15min":1.139},"min":{"1min":0.315,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.327,"5min":8.734,"15min":14.154},"last":0.67500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1652883456,"data_stored":1652883456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3825336320,"data_stored":3825336320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1256456192,"data_stored":1256456192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3369402368,"data_stored":3369402368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1276444672,"data_stored":1276444672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3824746496,"data_stored":3824746496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1268449280,"data_stored":1268449280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3816030208,"data_stored":3816030208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:58:49.739 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:58:49.739 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 81 2023-10-09T10:58:49.811 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:49.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:49.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:49.895 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:58:49.918 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:58:50.055 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:50.076 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:58:50.148 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:58:50.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:50.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:58:50.255 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:50.301 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:50.334 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:50.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:50.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:50.439 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:50.440 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:50.455 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:58:50.487 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:50.514 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:50.611 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:50.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:50.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:50.677 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 86 16 106940 106924 77.6469 44.375 0.0295066 0.0128739 2023-10-09T10:58:50.717 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:50.766 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:50.770 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:50 smithi125 ceph-mon[195507]: from='client.55747 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:58:50.770 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:50 smithi125 ceph-mon[195507]: pgmap v883: 119 pgs: 119 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-09T10:58:50.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:50 smithi125 ceph-mon[200197]: from='client.55747 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:58:50.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:50 smithi125 ceph-mon[200197]: pgmap v883: 119 pgs: 119 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-09T10:58:50.805 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:50.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:50.913 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:50.953 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:50 smithi155 ceph-mon[184854]: from='client.55747 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:58:50.953 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:50 smithi155 ceph-mon[184854]: pgmap v883: 119 pgs: 119 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-09T10:58:50.955 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:51.491 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:51.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:51.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:51.501 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:58:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:58:51] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-10-09T10:58:51.596 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:51.643 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:51.668 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:58:51.678 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 87 16 107807 107791 77.3768 54.1875 0.014219 0.0129188 2023-10-09T10:58:51.722 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:51.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:51.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:51.765 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1276616410' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:51.765 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1276616410' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:51.828 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:51.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1276616410' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:58:51.879 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:51.970 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:51.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:51.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:52.047 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:58:52.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:58:52.091 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:58:52.136 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:58:52.141 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:52.204 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:58:52.228 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:58:52.228 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:58:52.250 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:58:52.301 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:58:52.335 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:58:52.377 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:58:52.406 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:58:52.406 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:58:52.407 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:58:52.407 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:58:52.407 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:58:52.407 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:58:52.407 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:58:52.407 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:58:52.560 INFO:teuthology.orchestra.run.smithi125.stderr:Error EINVAL: specified pg_num 81 < pg_num_max 32 2023-10-09T10:58:52.589 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:58:52.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:52.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:58:52.609 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:58:52.678 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 88 16 108544 108528 77.0211 46.0625 0.010842 0.0129787 2023-10-09T10:58:52.706 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:52.717 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[195507]: pgmap v884: 119 pgs: 119 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T10:58:52.717 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:58:52.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T10:58:52.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1276616410' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:58:52.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[195507]: osdmap e355: 8 total, 8 up, 8 in 2023-10-09T10:58:52.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[195507]: failed to encode map e355 with expected crc 2023-10-09T10:58:52.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[195507]: failed to encode map e355 with expected crc 2023-10-09T10:58:52.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[195507]: failed to encode map e355 with expected crc 2023-10-09T10:58:52.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2361476809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:52.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2973471101' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T10:58:52.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T10:58:52.718 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[200197]: pgmap v884: 119 pgs: 119 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T10:58:52.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:58:52.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T10:58:52.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1276616410' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:58:52.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[200197]: osdmap e355: 8 total, 8 up, 8 in 2023-10-09T10:58:52.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[200197]: failed to encode map e355 with expected crc 2023-10-09T10:58:52.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[200197]: failed to encode map e355 with expected crc 2023-10-09T10:58:52.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[200197]: failed to encode map e355 with expected crc 2023-10-09T10:58:52.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2361476809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:52.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2973471101' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T10:58:52.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:52 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T10:58:52.776 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:52.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:52 smithi155 ceph-mon[184854]: pgmap v884: 119 pgs: 119 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2023-10-09T10:58:52.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:52 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:58:52.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:52 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T10:58:52.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:52 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1276616410' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:58:52.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:52 smithi155 ceph-mon[184854]: osdmap e355: 8 total, 8 up, 8 in 2023-10-09T10:58:52.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:52 smithi155 ceph-mon[184854]: failed to encode map e355 with expected crc 2023-10-09T10:58:52.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:52 smithi155 ceph-mon[184854]: failed to encode map e355 with expected crc 2023-10-09T10:58:52.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:52 smithi155 ceph-mon[184854]: failed to encode map e355 with expected crc 2023-10-09T10:58:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:52 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2361476809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:58:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:52 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2973471101' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T10:58:52.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:52 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-10-09T10:58:53.682 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 89 16 109464 109448 76.799 57.5 0.0144796 0.0130161 2023-10-09T10:58:53.719 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:53.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:53.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:53.823 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:53.879 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:54.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:54 smithi125 ceph-mon[200197]: pgmap v886: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T10:58:54.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:54 smithi125 ceph-mon[195507]: pgmap v886: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T10:58:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:54 smithi155 ceph-mon[184854]: pgmap v886: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T10:58:54.568 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:54.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:54.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:54.599 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T10:58:54.634 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:58:54.640 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:54.675 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:54.682 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total time run: 90.0092 2023-10-09T10:58:54.682 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total writes made: 110296 2023-10-09T10:58:54.682 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Write size: 65536 2023-10-09T10:58:54.682 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object size: 65536 2023-10-09T10:58:54.682 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Bandwidth (MB/sec): 76.5866 2023-10-09T10:58:54.682 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Bandwidth: 31.5021 2023-10-09T10:58:54.682 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max bandwidth (MB/sec): 182.312 2023-10-09T10:58:54.682 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min bandwidth (MB/sec): 44.375 2023-10-09T10:58:54.683 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average IOPS: 1225 2023-10-09T10:58:54.683 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev IOPS: 504.033 2023-10-09T10:58:54.683 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max IOPS: 2917 2023-10-09T10:58:54.683 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min IOPS: 710 2023-10-09T10:58:54.683 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average Latency(s): 0.0130534 2023-10-09T10:58:54.683 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Latency(s): 0.00777274 2023-10-09T10:58:54.683 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max latency(s): 0.143356 2023-10-09T10:58:54.683 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min latency(s): 0.00120203 2023-10-09T10:58:54.688 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T10:58:54.723 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:54.751 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:54.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:54.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:54.858 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:54.895 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:54.930 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:54.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:54.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:55.033 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:55.079 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:55.109 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:55.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:55.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:55.213 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:55.257 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:55.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:55.298 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:55.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:55.399 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:55.441 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:55.468 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:55.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:55.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:55.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:55.610 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:55.654 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:55.685 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:55.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:55.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:55.789 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:58:55.852 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:55.947 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:58:55.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:55.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:58:56.053 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:56.121 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:56.156 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:56.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:56.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:56.230 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:58:56.231 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:58:56.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.231 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:111698 9.9 9:958e8389:::benchmark_data_smithi125_363789_object1399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2023-10-09T10:58:56.231 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:58:56.216302+0000", 2023-10-09T10:58:56.231 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0069496949999999997, 2023-10-09T10:58:56.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0069724929999999997, 2023-10-09T10:58:56.231 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:58:56.231 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:58:56.231 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:58:56.231 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T10:58:56.232 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T10:58:56.232 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 111698 2023-10-09T10:58:56.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.232 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:58:56.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:58:56.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.216302+0000", 2023-10-09T10:58:56.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:58:56.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:58:56.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.216302+0000", 2023-10-09T10:58:56.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:58:56.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:58:56.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.216300+0000", 2023-10-09T10:58:56.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:58:56.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:58:56.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.216306+0000", 2023-10-09T10:58:56.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2360000000000001e-06 2023-10-09T10:58:56.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:58:56.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.216317+0000", 2023-10-09T10:58:56.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0978999999999999e-05 2023-10-09T10:58:56.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:58:56.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.216325+0000", 2023-10-09T10:58:56.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3980000000000002e-06 2023-10-09T10:58:56.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:58:56.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.216359+0000", 2023-10-09T10:58:56.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4755999999999997e-05 2023-10-09T10:58:56.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:58:56.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.216496+0000", 2023-10-09T10:58:56.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013650000000000001 2023-10-09T10:58:56.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:58:56.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.216614+0000", 2023-10-09T10:58:56.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011821599999999999 2023-10-09T10:58:56.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:58:56.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.220390+0000", 2023-10-09T10:58:56.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037762049999999999 2023-10-09T10:58:56.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:58:56.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.222803+0000", 2023-10-09T10:58:56.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024123410000000001 2023-10-09T10:58:56.236 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:58:56.236 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:58:56.236 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:58:56.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.237 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45683.0:111697 9.8 e355/330 9:1b460199:::benchmark_data_smithi125_363789_object1398:head v 355'6941, mlcod=355'6937)", 2023-10-09T10:58:56.237 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:58:56.219484+0000", 2023-10-09T10:58:56.237 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.003766836, 2023-10-09T10:58:56.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038442720000000001, 2023-10-09T10:58:56.237 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:58:56.237 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:58:56.237 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:58:56.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:58:56.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.219484+0000", 2023-10-09T10:58:56.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:58:56.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.238 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:58:56.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.219484+0000", 2023-10-09T10:58:56.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:58:56.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.238 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:58:56.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.219484+0000", 2023-10-09T10:58:56.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:58:56.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:58:56.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.219492+0000", 2023-10-09T10:58:56.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8860000000000004e-06 2023-10-09T10:58:56.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:58:56.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.219507+0000", 2023-10-09T10:58:56.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5557e-05 2023-10-09T10:58:56.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:58:56.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.219521+0000", 2023-10-09T10:58:56.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3456e-05 2023-10-09T10:58:56.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:58:56.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.219553+0000", 2023-10-09T10:58:56.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1993999999999999e-05 2023-10-09T10:58:56.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:58:56.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:58:56.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:58:56.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:58:56.219574+0000", 2023-10-09T10:58:56.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1013999999999999e-05 2023-10-09T10:58:56.240 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:58:56.240 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:58:56.241 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:58:56.241 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:58:56.241 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:58:56.241 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 2 2023-10-09T10:58:56.241 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:58:56.260 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:56.302 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:56.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:56.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:56.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:56.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:56.455 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:58:56.494 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:56.538 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:56.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:56.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:56.608 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:58:56.647 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:56.693 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:56.721 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:56.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:56.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:56.759 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:56 smithi125 ceph-mon[195507]: pgmap v887: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 46 MiB/s wr, 737 op/s 2023-10-09T10:58:56.760 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:56 smithi125 ceph-mon[200197]: pgmap v887: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 46 MiB/s wr, 737 op/s 2023-10-09T10:58:56.827 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:58:56.868 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:56 smithi155 ceph-mon[184854]: pgmap v887: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 46 MiB/s wr, 737 op/s 2023-10-09T10:58:56.871 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:56.899 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:58:56.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:56.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:58:56.986 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:57.006 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:58:57.099 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:57.201 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:58:57.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:57.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:58:57.307 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:57.361 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:57.408 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:57.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:57.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:57.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:57.581 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:57.652 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:58:57.654 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:57.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:57.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:57.670 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:58:57.766 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:57.815 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:57 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/326296946' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:58:57.815 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:57 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:58:57.816 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:57 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/326296946' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:58:57.816 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:57 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:58:57.875 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:57.905 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:57.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:57.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:57.910 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:57 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/326296946' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:58:57.910 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:57 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:58:58.009 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:58:58.055 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:58.082 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:58:58.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:58.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:58:58.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:58:58.244 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:58.799 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:58:58.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:58.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:58:58.907 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:58.960 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:58 smithi155 ceph-mon[184854]: pgmap v888: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 45 MiB/s wr, 830 op/s 2023-10-09T10:58:58.960 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:58 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:58.960 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:58 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:58:58.960 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:58 smithi155 ceph-mon[184854]: osdmap e356: 8 total, 8 up, 8 in 2023-10-09T10:58:58.960 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:58 smithi155 ceph-mon[184854]: failed to encode map e356 with expected crc 2023-10-09T10:58:58.960 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:58 smithi155 ceph-mon[184854]: failed to encode map e356 with expected crc 2023-10-09T10:58:58.961 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:58 smithi155 ceph-mon[184854]: failed to encode map e356 with expected crc 2023-10-09T10:58:58.962 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:59.005 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:59.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:59.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:58:59.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:58 smithi125 ceph-mon[200197]: pgmap v888: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 45 MiB/s wr, 830 op/s 2023-10-09T10:58:59.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:58 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:59.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:58 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:58:59.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:58 smithi125 ceph-mon[200197]: osdmap e356: 8 total, 8 up, 8 in 2023-10-09T10:58:59.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:58 smithi125 ceph-mon[200197]: failed to encode map e356 with expected crc 2023-10-09T10:58:59.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:58 smithi125 ceph-mon[200197]: failed to encode map e356 with expected crc 2023-10-09T10:58:59.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:58 smithi125 ceph-mon[200197]: failed to encode map e356 with expected crc 2023-10-09T10:58:59.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:58 smithi125 ceph-mon[195507]: pgmap v888: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 45 MiB/s wr, 830 op/s 2023-10-09T10:58:59.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:58 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:58:59.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:58 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:58:59.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:58 smithi125 ceph-mon[195507]: osdmap e356: 8 total, 8 up, 8 in 2023-10-09T10:58:59.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:58 smithi125 ceph-mon[195507]: failed to encode map e356 with expected crc 2023-10-09T10:58:59.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:58 smithi125 ceph-mon[195507]: failed to encode map e356 with expected crc 2023-10-09T10:58:59.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:58 smithi125 ceph-mon[195507]: failed to encode map e356 with expected crc 2023-10-09T10:58:59.113 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:58:59.168 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:59.600 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 2, 1, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:58:59.600 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:58:59.600 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T10:58:59.600 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:58:59.600 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:58:59.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T10:58:59.612 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:58:59.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:59.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:58:59.717 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:58:59.770 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:59.783 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:58:59 smithi125 ceph-mon[195507]: failed to encode map e356 with expected crc 2023-10-09T10:58:59.783 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:58:59 smithi125 ceph-mon[200197]: failed to encode map e356 with expected crc 2023-10-09T10:58:59.810 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:58:59.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:59.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:58:59.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:58:59.955 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:58:59 smithi155 ceph-mon[184854]: failed to encode map e356 with expected crc 2023-10-09T10:58:59.960 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:58:59.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:58:59.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:58:59.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:00.012 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:59:00.052 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:59:00.062 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:59:00.092 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:59:00.104 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:59:00.143 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:59:00.164 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:00.179 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:59:00.216 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:59:00.217 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:59:00.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:00.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:59:00.256 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:59:00.293 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:59:00.307 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:59:00.307 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:59:00.312 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:59:00.312 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T10:59:00.312 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:59:00.312 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:59:00.313 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:59:00.313 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:59:00.313 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:59:00.313 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:59:00.325 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:00.401 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:00.582 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:00.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:00.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:00.689 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:00.756 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:00.848 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:00.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:00 smithi125 ceph-mon[195507]: pgmap v890: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 35 MiB/s wr, 751 op/s 2023-10-09T10:59:00.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:00 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/383125003' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:00.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:59:00.851 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:00 smithi125 ceph-mon[200197]: pgmap v890: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 35 MiB/s wr, 751 op/s 2023-10-09T10:59:00.852 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:00 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/383125003' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:00.852 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:59:00.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:00.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:00.878 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:59:00.953 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:00.969 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:00 smithi155 ceph-mon[184854]: pgmap v890: 119 pgs: 119 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 35 MiB/s wr, 751 op/s 2023-10-09T10:59:00.969 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:00 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/383125003' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:00.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:59:01.018 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:01.047 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:01.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:01.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:01.152 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:01.215 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:01.243 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:01.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:01.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:01.276 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:59:01.277 INFO:teuthology.orchestra.run.smithi155.stdout: "success": "" 2023-10-09T10:59:01.277 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:59:01.353 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:01.364 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:01.416 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:01.522 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:59:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:59:01] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.43.0" 2023-10-09T10:59:01.552 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:01.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:01.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:01.660 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:59:01.718 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:01.785 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:59:01.858 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:59:01.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:01.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:59:01.970 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:59:02.027 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:02.103 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:59:02.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:02.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:59:02.209 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:02.269 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:02.298 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:02.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:02.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:02.410 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:02.471 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:02.508 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:02.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:02.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:02.615 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:02.663 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:02.689 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:02.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:02.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:02.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:02 smithi155 ceph-mon[184854]: pgmap v891: 119 pgs: 119 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 103 B/s rd, 28 MiB/s wr, 866 op/s 2023-10-09T10:59:02.749 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:59:02.797 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:02.835 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:02.859 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:02.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:02.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:02.895 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:02.928 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:59:02.928 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:59:02.957 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:02 smithi125 ceph-mon[195507]: pgmap v891: 119 pgs: 119 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 103 B/s rd, 28 MiB/s wr, 866 op/s 2023-10-09T10:59:02.957 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:02 smithi125 ceph-mon[200197]: pgmap v891: 119 pgs: 119 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 103 B/s rd, 28 MiB/s wr, 866 op/s 2023-10-09T10:59:02.967 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:59:03.013 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:03.362 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:59:03.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:03.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:59:03.474 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T10:59:03.557 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:03.683 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T10:59:03.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:03.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T10:59:03.787 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:03.831 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:03.870 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:03.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:03.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:03.973 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:04.033 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:04.071 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:04.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:04.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:04.181 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:04.222 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:04.258 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:04.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:04.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:04.293 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T10:59:04.367 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:04.407 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:04 smithi125 ceph-mon[195507]: pgmap v892: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 17 MiB/s wr, 874 op/s 2023-10-09T10:59:04.408 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:04 smithi125 ceph-mon[200197]: pgmap v892: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 17 MiB/s wr, 874 op/s 2023-10-09T10:59:04.446 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:04 smithi155 ceph-mon[184854]: pgmap v892: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 17 MiB/s wr, 874 op/s 2023-10-09T10:59:04.483 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:04.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:04.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:04.563 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:04.589 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:04.625 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:04.653 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:04.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:04.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:04.757 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:04.823 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:04.865 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:04.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:04.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:04.978 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:05.008 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:05.039 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:05.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:05.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:05.079 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:05.145 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:05.151 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:59:05.188 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:05.216 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:05.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:05.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:05.321 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:05.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:05 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2946523276' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:05.394 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:05 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2946523276' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:05.409 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:05.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:05 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2946523276' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:05.496 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:05.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:05.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:05.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:05.662 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:05.700 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:05.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:05.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:05.804 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:05.846 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:05.875 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:05.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:05.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:05.979 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:06.023 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:06.117 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:06.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:06.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:06.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:06.257 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:59:06.281 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:06.281 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2946523276' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:59:06.281 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[195507]: osdmap e357: 8 total, 8 up, 8 in 2023-10-09T10:59:06.282 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[195507]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.282 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[195507]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.282 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[195507]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.282 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[195507]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.282 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[195507]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.282 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[195507]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.282 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[195507]: pgmap v894: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 2.4 MiB/s wr, 655 op/s 2023-10-09T10:59:06.282 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:06.283 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2946523276' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:59:06.283 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[200197]: osdmap e357: 8 total, 8 up, 8 in 2023-10-09T10:59:06.283 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[200197]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.283 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[200197]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.283 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[200197]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.283 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[200197]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.283 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[200197]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.283 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[200197]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.283 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:06 smithi125 ceph-mon[200197]: pgmap v894: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 2.4 MiB/s wr, 655 op/s 2023-10-09T10:59:06.298 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T10:59:06.307 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:06.345 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:59:06.386 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:06.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:06.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:06.406 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:59:06.452 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:59:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:06 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:06 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2946523276' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:59:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:06 smithi155 ceph-mon[184854]: osdmap e357: 8 total, 8 up, 8 in 2023-10-09T10:59:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:06 smithi155 ceph-mon[184854]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:06 smithi155 ceph-mon[184854]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:06 smithi155 ceph-mon[184854]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:06 smithi155 ceph-mon[184854]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:06 smithi155 ceph-mon[184854]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:06 smithi155 ceph-mon[184854]: failed to encode map e357 with expected crc 2023-10-09T10:59:06.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:06 smithi155 ceph-mon[184854]: pgmap v894: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 2.4 MiB/s wr, 655 op/s 2023-10-09T10:59:06.490 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:06.504 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:59:06.542 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:59:06.556 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:06.582 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:59:06.589 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:06.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:06.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:06.602 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:59:06.602 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T10:59:06.602 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:59:06.603 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:59:06.603 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:59:06.603 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:59:06.603 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:59:06.603 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:59:06.622 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:59:06.622 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:59:06.623 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.623 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45683.0:120664 9.e e357/332 9:7940cee4:::benchmark_data_smithi125_363789_object10365:head v 357'7707, mlcod=357'7705)", 2023-10-09T10:59:06.623 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:06.609605+0000", 2023-10-09T10:59:06.623 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.011929875, 2023-10-09T10:59:06.623 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011959375, 2023-10-09T10:59:06.623 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:06.623 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:59:06.623 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:06.623 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.623 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:06.624 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609605+0000", 2023-10-09T10:59:06.624 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:06.624 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.624 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.624 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:06.624 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609605+0000", 2023-10-09T10:59:06.624 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:06.624 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.624 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.624 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:06.625 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609604+0000", 2023-10-09T10:59:06.625 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:06.625 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.625 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.625 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:06.625 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609611+0000", 2023-10-09T10:59:06.625 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1219999999999997e-06 2023-10-09T10:59:06.625 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.625 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.625 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:06.626 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609622+0000", 2023-10-09T10:59:06.626 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0122000000000001e-05 2023-10-09T10:59:06.626 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.626 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.626 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:06.626 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609628+0000", 2023-10-09T10:59:06.626 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.691e-06 2023-10-09T10:59:06.626 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.626 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.627 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:06.627 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.610938+0000", 2023-10-09T10:59:06.627 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013094059999999999 2023-10-09T10:59:06.627 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.627 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.627 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:06.627 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.610960+0000", 2023-10-09T10:59:06.627 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2376999999999999e-05 2023-10-09T10:59:06.627 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:06.627 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:06.627 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:06.628 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.628 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.628 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45683.0:120670 9.e e357/332 9:7d35ba20:::benchmark_data_smithi125_363789_object10371:head v 357'7708, mlcod=357'7705)", 2023-10-09T10:59:06.628 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:06.609655+0000", 2023-10-09T10:59:06.628 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.011880125, 2023-10-09T10:59:06.628 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011982543, 2023-10-09T10:59:06.628 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:06.628 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:59:06.628 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:06.628 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.629 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:06.629 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609655+0000", 2023-10-09T10:59:06.629 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:06.629 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.629 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.629 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:06.629 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609655+0000", 2023-10-09T10:59:06.629 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:06.629 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.629 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.629 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:06.630 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609654+0000", 2023-10-09T10:59:06.630 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:06.630 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.630 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.630 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:06.630 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609659+0000", 2023-10-09T10:59:06.630 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6639999999999997e-06 2023-10-09T10:59:06.630 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.630 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.630 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:06.630 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609668+0000", 2023-10-09T10:59:06.631 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4150000000000005e-06 2023-10-09T10:59:06.631 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.631 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.631 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:06.631 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609673+0000", 2023-10-09T10:59:06.631 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9509999999999997e-06 2023-10-09T10:59:06.631 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.631 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.631 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:06.631 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.613924+0000", 2023-10-09T10:59:06.631 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042508329999999999 2023-10-09T10:59:06.632 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.632 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.632 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:06.632 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.613944+0000", 2023-10-09T10:59:06.632 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9678e-05 2023-10-09T10:59:06.632 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:06.632 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:06.632 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:06.632 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.632 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.632 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45683.0:120661 9.6 e357/330 9:6f4f15f2:::benchmark_data_smithi125_363789_object10362:head v 357'7443, mlcod=357'7441)", 2023-10-09T10:59:06.633 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:06.609777+0000", 2023-10-09T10:59:06.633 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0117573, 2023-10-09T10:59:06.633 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.01192968, 2023-10-09T10:59:06.633 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:06.633 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:06.633 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:06.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.633 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:06.633 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609777+0000", 2023-10-09T10:59:06.633 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:06.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:06.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609777+0000", 2023-10-09T10:59:06.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:06.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:06.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609777+0000", 2023-10-09T10:59:06.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:06.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:06.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609782+0000", 2023-10-09T10:59:06.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8489999999999997e-06 2023-10-09T10:59:06.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:06.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609789+0000", 2023-10-09T10:59:06.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2479999999999999e-06 2023-10-09T10:59:06.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:06.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609794+0000", 2023-10-09T10:59:06.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3129999999999998e-06 2023-10-09T10:59:06.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:06.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.612141+0000", 2023-10-09T10:59:06.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023468529999999999 2023-10-09T10:59:06.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:06.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.612164+0000", 2023-10-09T10:59:06.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2300999999999999e-05 2023-10-09T10:59:06.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:59:06.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.621705+0000", 2023-10-09T10:59:06.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0095415870000000007 2023-10-09T10:59:06.637 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:06.637 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:06.638 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:06.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.638 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45683.0:120666 9.6 e357/330 9:654564d0:::benchmark_data_smithi125_363789_object10367:head v 357'7444, mlcod=357'7441)", 2023-10-09T10:59:06.638 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:06.609799+0000", 2023-10-09T10:59:06.638 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.011735269, 2023-10-09T10:59:06.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011999556999999999, 2023-10-09T10:59:06.638 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:06.638 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:59:06.638 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:06.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:06.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609799+0000", 2023-10-09T10:59:06.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:06.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:06.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609799+0000", 2023-10-09T10:59:06.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:06.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:06.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609799+0000", 2023-10-09T10:59:06.640 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967296 2023-10-09T10:59:06.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.640 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:06.640 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609802+0000", 2023-10-09T10:59:06.640 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0599999999999999e-06 2023-10-09T10:59:06.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.640 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:06.640 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609808+0000", 2023-10-09T10:59:06.640 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5639999999999996e-06 2023-10-09T10:59:06.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.641 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:06.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.609812+0000", 2023-10-09T10:59:06.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.036e-06 2023-10-09T10:59:06.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.641 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:06.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.615952+0000", 2023-10-09T10:59:06.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0061403819999999998 2023-10-09T10:59:06.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:06.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:06.642 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:06.642 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:06.615972+0000", 2023-10-09T10:59:06.642 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0068000000000002e-05 2023-10-09T10:59:06.642 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:06.642 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:06.642 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:06.642 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:06.642 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:59:06.642 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 4 2023-10-09T10:59:06.642 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:59:06.655 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 2, 1, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:59:06.655 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:59:06.655 INFO:tasks.thrashosds.thrasher:Killing osd 1, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T10:59:06.655 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-09T10:59:06.655 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T10:59:06.702 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:06.746 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:06.783 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:06.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:06.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:06.824 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:59:06 smithi125 systemd[1]: Stopping Ceph osd.1 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T10:59:06.893 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:06.941 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:06.985 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:06.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:06.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:07.093 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:07.138 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:07.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:07 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3113508481' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:07.178 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:07.181 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T10:59:07.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:07.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:07.285 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:07.334 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:07.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:07.382 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:07.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:07 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3113508481' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:07.483 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:07.526 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:07.556 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:07.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:07.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:07.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:07 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3113508481' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:07.573 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:59:07 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[216014]: 2023-10-09T10:59:07.328+0000 7fe067fa6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T10:59:07.573 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:59:07 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[216014]: 2023-10-09T10:59:07.328+0000 7fe067fa6700 -1 osd.1 357 *** Got signal Terminated *** 2023-10-09T10:59:07.573 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:59:07 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[216014]: 2023-10-09T10:59:07.328+0000 7fe067fa6700 -1 osd.1 357 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T10:59:07.611 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:07.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:07.720 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:07.749 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:07.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:07.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:07.862 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:07.940 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:08.074 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:08.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:08.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:08.178 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:08.211 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:08.280 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:08.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:08.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:08.383 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:08.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:59:08.409 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:08 smithi155 ceph-mon[184854]: osd.1 marked itself down and dead 2023-10-09T10:59:08.410 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:08 smithi155 ceph-mon[184854]: pgmap v895: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 2.0 MiB/s wr, 876 op/s 2023-10-09T10:59:08.411 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:08.430 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:08.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:08.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:08.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:08 smithi125 ceph-mon[195507]: osd.1 marked itself down and dead 2023-10-09T10:59:08.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:08 smithi125 ceph-mon[195507]: pgmap v895: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 2.0 MiB/s wr, 876 op/s 2023-10-09T10:59:08.434 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:08 smithi125 ceph-mon[200197]: osd.1 marked itself down and dead 2023-10-09T10:59:08.434 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:08 smithi125 ceph-mon[200197]: pgmap v895: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 2.0 MiB/s wr, 876 op/s 2023-10-09T10:59:08.534 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:08.560 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:08.579 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:08.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:08.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:08.647 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:59:08.647 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:59:08.648 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:59:08.648 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:59:08.648 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:59:08.682 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:08.694 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:08.709 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:08.727 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:08.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:08.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:08.831 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:08.858 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:08.877 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:08.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:08.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:08.980 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:09.012 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:09.049 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:59:09.062 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T10:59:09.132 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:09.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:09.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:09.251 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:09.312 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:09.320 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:59:09 smithi125 bash[380597]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T10:59:09.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:09 smithi125 ceph-mon[195507]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:59:09.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:09 smithi125 ceph-mon[195507]: failed to encode map e358 with expected crc 2023-10-09T10:59:09.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:09 smithi125 ceph-mon[195507]: failed to encode map e358 with expected crc 2023-10-09T10:59:09.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:09 smithi125 ceph-mon[195507]: osdmap e358: 8 total, 7 up, 8 in 2023-10-09T10:59:09.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:09 smithi125 ceph-mon[195507]: failed to encode map e358 with expected crc 2023-10-09T10:59:09.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:09 smithi125 ceph-mon[195507]: failed to encode map e358 with expected crc 2023-10-09T10:59:09.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:09 smithi125 ceph-mon[195507]: failed to encode map e358 with expected crc 2023-10-09T10:59:09.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:09 smithi125 ceph-mon[200197]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:59:09.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:09 smithi125 ceph-mon[200197]: failed to encode map e358 with expected crc 2023-10-09T10:59:09.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:09 smithi125 ceph-mon[200197]: failed to encode map e358 with expected crc 2023-10-09T10:59:09.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:09 smithi125 ceph-mon[200197]: osdmap e358: 8 total, 7 up, 8 in 2023-10-09T10:59:09.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:09 smithi125 ceph-mon[200197]: failed to encode map e358 with expected crc 2023-10-09T10:59:09.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:09 smithi125 ceph-mon[200197]: failed to encode map e358 with expected crc 2023-10-09T10:59:09.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:09 smithi125 ceph-mon[200197]: failed to encode map e358 with expected crc 2023-10-09T10:59:09.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:09 smithi155 ceph-mon[184854]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T10:59:09.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:09 smithi155 ceph-mon[184854]: failed to encode map e358 with expected crc 2023-10-09T10:59:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:09 smithi155 ceph-mon[184854]: failed to encode map e358 with expected crc 2023-10-09T10:59:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:09 smithi155 ceph-mon[184854]: osdmap e358: 8 total, 7 up, 8 in 2023-10-09T10:59:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:09 smithi155 ceph-mon[184854]: failed to encode map e358 with expected crc 2023-10-09T10:59:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:09 smithi155 ceph-mon[184854]: failed to encode map e358 with expected crc 2023-10-09T10:59:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:09 smithi155 ceph-mon[184854]: failed to encode map e358 with expected crc 2023-10-09T10:59:09.549 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:09.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:09.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:09.622 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:09.655 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:09.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:09.730 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:09.794 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:09.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:09.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:09.901 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:09.970 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:10.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:10.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:10.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:10.122 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:10.188 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:10.217 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:10.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:10.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:10.293 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[195507]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.293 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[195507]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.294 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[195507]: osdmap e359: 8 total, 7 up, 8 in 2023-10-09T10:59:10.294 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[195507]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.294 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[195507]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.294 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[195507]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.294 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[195507]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.294 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[195507]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.294 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[195507]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.294 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[195507]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.294 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[195507]: pgmap v898: 119 pgs: 24 stale+active+clean, 95 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-10-09T10:59:10.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[200197]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[200197]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[200197]: osdmap e359: 8 total, 7 up, 8 in 2023-10-09T10:59:10.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[200197]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[200197]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[200197]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[200197]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[200197]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[200197]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.296 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[200197]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.296 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:10 smithi125 ceph-mon[200197]: pgmap v898: 119 pgs: 24 stale+active+clean, 95 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-10-09T10:59:10.327 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:10.383 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:10.411 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:10.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:10.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:10.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:10 smithi155 ceph-mon[184854]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:10 smithi155 ceph-mon[184854]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:10 smithi155 ceph-mon[184854]: osdmap e359: 8 total, 7 up, 8 in 2023-10-09T10:59:10.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:10 smithi155 ceph-mon[184854]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:10 smithi155 ceph-mon[184854]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:10 smithi155 ceph-mon[184854]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:10 smithi155 ceph-mon[184854]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:10 smithi155 ceph-mon[184854]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:10 smithi155 ceph-mon[184854]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:10 smithi155 ceph-mon[184854]: failed to encode map e359 with expected crc 2023-10-09T10:59:10.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:10 smithi155 ceph-mon[184854]: pgmap v898: 119 pgs: 24 stale+active+clean, 95 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-10-09T10:59:10.516 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:10.556 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:10.601 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:10.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:10.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:10.712 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:10.759 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:10.942 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:10.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:10.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:11.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:11.131 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:11.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:11.236 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:11.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:11.236 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:59:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:59:11] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.43.0" 2023-10-09T10:59:11.320 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:59:11.337 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:11.399 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:11.418 INFO:teuthology.orchestra.run.smithi125.stderr:admin_socket: exception getting command descriptions: [Errno 111] Connection refused 2023-10-09T10:59:11.449 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:11.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:11.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:11.554 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:59:11.556 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:11.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:11 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1588034650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:59:11.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:11 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1588034650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:59:11.586 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:59:11.614 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:59:11.617 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:11 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1588034650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:59:11.623 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:11.647 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:59:11.668 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:11.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:11.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:11.692 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:59:11.731 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:59:11.758 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:59:11.758 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:59:11.759 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-10-09T10:59:11.759 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:59:11.759 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:59:11.759 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:59:11.759 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:59:11.759 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:59:11.759 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:59:11.772 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:11.823 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:11.852 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:11.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:11.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:11.863 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T10:59:11.864 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T10:59:11.959 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:12.022 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:12.045 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:12.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:12.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:12.150 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:12.186 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:12.312 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:12.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:12.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:12.329 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:59:12.330 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:59:12.420 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:12.466 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:12.510 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:12.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:12.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:12.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:12 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:12.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:12 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1588034650' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:59:12.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:12 smithi125 ceph-mon[195507]: osdmap e360: 8 total, 7 up, 8 in 2023-10-09T10:59:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:12 smithi125 ceph-mon[195507]: failed to encode map e360 with expected crc 2023-10-09T10:59:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:12 smithi125 ceph-mon[195507]: failed to encode map e360 with expected crc 2023-10-09T10:59:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:12 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3599258747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:12.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:12 smithi125 ceph-mon[195507]: pgmap v900: 119 pgs: 11 active+undersized+degraded, 33 active+undersized, 4 stale+active+clean, 71 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 13 KiB/s rd, 33 KiB/s wr, 1.18k op/s; 54309/291681 objects degraded (18.619%); 24 KiB/s, 0 objects/s recovering 2023-10-09T10:59:12.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:12 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:12.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:12 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1588034650' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:59:12.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:12 smithi125 ceph-mon[200197]: osdmap e360: 8 total, 7 up, 8 in 2023-10-09T10:59:12.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:12 smithi125 ceph-mon[200197]: failed to encode map e360 with expected crc 2023-10-09T10:59:12.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:12 smithi125 ceph-mon[200197]: failed to encode map e360 with expected crc 2023-10-09T10:59:12.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:12 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3599258747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:12.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:12 smithi125 ceph-mon[200197]: pgmap v900: 119 pgs: 11 active+undersized+degraded, 33 active+undersized, 4 stale+active+clean, 71 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 13 KiB/s rd, 33 KiB/s wr, 1.18k op/s; 54309/291681 objects degraded (18.619%); 24 KiB/s, 0 objects/s recovering 2023-10-09T10:59:12.575 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:12.616 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:12.657 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:12 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:12.657 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:12 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1588034650' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:59:12.657 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:12 smithi155 ceph-mon[184854]: osdmap e360: 8 total, 7 up, 8 in 2023-10-09T10:59:12.657 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:12 smithi155 ceph-mon[184854]: failed to encode map e360 with expected crc 2023-10-09T10:59:12.658 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:12 smithi155 ceph-mon[184854]: failed to encode map e360 with expected crc 2023-10-09T10:59:12.658 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:12 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3599258747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:12.658 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:12 smithi155 ceph-mon[184854]: pgmap v900: 119 pgs: 11 active+undersized+degraded, 33 active+undersized, 4 stale+active+clean, 71 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 13 KiB/s rd, 33 KiB/s wr, 1.18k op/s; 54309/291681 objects degraded (18.619%); 24 KiB/s, 0 objects/s recovering 2023-10-09T10:59:12.662 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:12.697 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:12.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:12.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:12.804 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:12.865 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:12.912 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:12.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:12.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:13.016 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:13.070 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:13.112 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:13.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:13.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:13.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:13.270 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:13.317 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-09T10:59:13.357 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:13 smithi125 ceph-mon[195507]: Health check failed: Degraded data redundancy: 54309/291681 objects degraded (18.619%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:13.357 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:13 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:59:13.357 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:13 smithi125 ceph-mon[200197]: Health check failed: Degraded data redundancy: 54309/291681 objects degraded (18.619%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:13.357 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:13 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:59:13.357 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:59:13 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.1.service: Succeeded. 2023-10-09T10:59:13.357 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 10:59:13 smithi125 systemd[1]: Stopped Ceph osd.1 for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T10:59:13.422 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T10:59:13.422 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-09T10:59:13.453 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:13.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:13.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:13.562 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:13.637 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:13.661 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:13.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:13.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:13.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:13 smithi155 ceph-mon[184854]: Health check failed: Degraded data redundancy: 54309/291681 objects degraded (18.619%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:13.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:13 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T10:59:13.765 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:13.812 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:13.841 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:13.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:13.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:13.881 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:59:13.881 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:59:13.947 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:14.004 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:14.045 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:14.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:14.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:14.149 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:14.204 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:14.262 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:14.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:14.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:14.331 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:59:14.369 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:14.433 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:14 smithi155 ceph-mon[184854]: pgmap v901: 119 pgs: 12 active+undersized+degraded, 39 active+undersized, 68 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 15 KiB/s rd, 40 KiB/s wr, 731 op/s; 54310/289410 objects degraded (18.766%); 28 KiB/s, 0 objects/s recovering 2023-10-09T10:59:14.436 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:14.454 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:14 smithi125 ceph-mon[200197]: pgmap v901: 119 pgs: 12 active+undersized+degraded, 39 active+undersized, 68 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 15 KiB/s rd, 40 KiB/s wr, 731 op/s; 54310/289410 objects degraded (18.766%); 28 KiB/s, 0 objects/s recovering 2023-10-09T10:59:14.455 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T10:59:14.456 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:14 smithi125 ceph-mon[195507]: pgmap v901: 119 pgs: 12 active+undersized+degraded, 39 active+undersized, 68 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 15 KiB/s rd, 40 KiB/s wr, 731 op/s; 54310/289410 objects degraded (18.766%); 28 KiB/s, 0 objects/s recovering 2023-10-09T10:59:14.476 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:14.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:14.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:14.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:59:14.584 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:14.652 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:14.700 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:14.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:14.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:14.799 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:14.804 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:14.877 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:15.007 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:15.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:15.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:15.112 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:15.176 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:15.215 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:15.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:15.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:15.322 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:15.373 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:15.464 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:15.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:15.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:15.574 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:15.637 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:15.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:15.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:15.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:15.819 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:15.896 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:15.953 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:16.032 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:16.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:16.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:16.058 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:16.058 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:59:16.058 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:16 smithi155 ceph-mon[184854]: pgmap v902: 119 pgs: 12 active+undersized+degraded, 39 active+undersized, 68 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 12 KiB/s rd, 32 KiB/s wr, 484 op/s; 54310/289410 objects degraded (18.766%); 23 KiB/s, 0 objects/s recovering 2023-10-09T10:59:16.139 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:16.166 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:59:16.183 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:16.190 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:59:16.234 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:59:16.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:16.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:16.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:16.276 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:59:16.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:16.293 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:59:16.293 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:16 smithi125 ceph-mon[195507]: pgmap v902: 119 pgs: 12 active+undersized+degraded, 39 active+undersized, 68 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 12 KiB/s rd, 32 KiB/s wr, 484 op/s; 54310/289410 objects degraded (18.766%); 23 KiB/s, 0 objects/s recovering 2023-10-09T10:59:16.293 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:16.293 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:59:16.293 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:16 smithi125 ceph-mon[200197]: pgmap v902: 119 pgs: 12 active+undersized+degraded, 39 active+undersized, 68 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 12 KiB/s rd, 32 KiB/s wr, 484 op/s; 54310/289410 objects degraded (18.766%); 23 KiB/s, 0 objects/s recovering 2023-10-09T10:59:16.307 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:59:16.330 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:16.345 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:59:16.366 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:16.386 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:59:16.386 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:59:16.387 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-10-09T10:59:16.387 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:59:16.387 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:59:16.387 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:59:16.387 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:59:16.387 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:59:16.387 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:59:16.412 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:16.759 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:16.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:16.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:16.864 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:16.910 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:16.939 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:16.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:16.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:17.048 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:17.092 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:17.102 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1377437058' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:17.102 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1377437058' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:17.233 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:59:17.233 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:59:17.273 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:17.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:17.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:17.380 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:17.416 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1377437058' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:17.417 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:17.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:17.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:17.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:17.561 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:17.620 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:17.709 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:17.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:17.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:17.815 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:17.899 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:17.925 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:17.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:17.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:17.959 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:59:17.959 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:59:17.960 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:59:17.960 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:59:17.960 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:59:18.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:18.100 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:18.182 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:18.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:18.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:18.287 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:18.348 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:18.381 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:18.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:18.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:18.423 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 0, 2, 1, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-10-09T10:59:18.423 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:59:18.423 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [3, 4, 5, 6, 0, 2, 1, 7] 2023-10-09T10:59:18.423 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd out 0 2023-10-09T10:59:18.462 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:59:18.492 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:18.582 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T10:59:18.584 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:18.628 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:18 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:18.628 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:18 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:18.629 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:18 smithi125 ceph-mon[195507]: pgmap v903: 119 pgs: 12 active+undersized+degraded, 39 active+undersized, 68 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 11 KiB/s rd, 28 KiB/s wr, 890 op/s; 51568/277683 objects degraded (18.571%); 20 KiB/s, 0 objects/s recovering 2023-10-09T10:59:18.629 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:18 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2665501471' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:18.629 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:18 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 54310/289410 objects degraded (18.766%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:18.629 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:18 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:18.629 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:18 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:18.629 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:18 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:18.629 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:18 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:18.629 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:18 smithi125 ceph-mon[200197]: pgmap v903: 119 pgs: 12 active+undersized+degraded, 39 active+undersized, 68 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 11 KiB/s rd, 28 KiB/s wr, 890 op/s; 51568/277683 objects degraded (18.571%); 20 KiB/s, 0 objects/s recovering 2023-10-09T10:59:18.629 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:18 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2665501471' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:18.630 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:18 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 54310/289410 objects degraded (18.766%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:18.630 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:18 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:18.630 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:18 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:18.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:18 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:18.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:18 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:18.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:18 smithi155 ceph-mon[184854]: pgmap v903: 119 pgs: 12 active+undersized+degraded, 39 active+undersized, 68 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 11 KiB/s rd, 28 KiB/s wr, 890 op/s; 51568/277683 objects degraded (18.571%); 20 KiB/s, 0 objects/s recovering 2023-10-09T10:59:18.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2665501471' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:18.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:18 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 54310/289410 objects degraded (18.766%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:18.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:18 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:18.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:18 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:18.808 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:18.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:18.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:18.915 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:18.943 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:18.969 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:19.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:19.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:19.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:19.223 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:19.234 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:59:19.273 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:19.289 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:19.316 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:19.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:19.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:19.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:19.489 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:19.514 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:19.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:19.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:59:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[195507]: failed to encode map e361 with expected crc 2023-10-09T10:59:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2665501471' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:59:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[195507]: osdmap e361: 8 total, 7 up, 8 in 2023-10-09T10:59:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[195507]: failed to encode map e361 with expected crc 2023-10-09T10:59:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[195507]: failed to encode map e361 with expected crc 2023-10-09T10:59:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:59:19.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:59:19.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:59:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[200197]: failed to encode map e361 with expected crc 2023-10-09T10:59:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2665501471' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:59:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[200197]: osdmap e361: 8 total, 7 up, 8 in 2023-10-09T10:59:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[200197]: failed to encode map e361 with expected crc 2023-10-09T10:59:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[200197]: failed to encode map e361 with expected crc 2023-10-09T10:59:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:59:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:59:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:19 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:19.621 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:19.668 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:19.704 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:19.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:19.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:19.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:19 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:59:19.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:19 smithi155 ceph-mon[184854]: failed to encode map e361 with expected crc 2023-10-09T10:59:19.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:19 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2665501471' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:59:19.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:19 smithi155 ceph-mon[184854]: osdmap e361: 8 total, 7 up, 8 in 2023-10-09T10:59:19.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:19 smithi155 ceph-mon[184854]: failed to encode map e361 with expected crc 2023-10-09T10:59:19.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:19 smithi155 ceph-mon[184854]: failed to encode map e361 with expected crc 2023-10-09T10:59:19.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:19 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T10:59:19.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:19 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T10:59:19.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:19 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T10:59:19.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:19.870 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:19.896 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:19.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:19.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:20.001 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:20.042 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:20.095 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:20.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:20.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:20.206 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:20.286 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:20.316 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:20.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:20.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:20.383 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:20.426 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:20.467 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.0. 2023-10-09T10:59:20.472 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:20.502 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:20.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:20.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:20.609 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:20.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:20 smithi125 ceph-mon[200197]: pgmap v905: 119 pgs: 12 active+undersized+degraded, 39 active+undersized, 68 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 4.9 KiB/s rd, 13 KiB/s wr, 675 op/s; 51568/274992 objects degraded (18.753%); 9.5 KiB/s, 0 objects/s recovering 2023-10-09T10:59:20.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:20 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3501793412' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T10:59:20.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:20 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T10:59:20.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:20 smithi125 ceph-mon[200197]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T10:59:20.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:20 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T10:59:20.362+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e361 definitely_dead 0 2023-10-09T10:59:20.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:20 smithi125 ceph-mon[195507]: pgmap v905: 119 pgs: 12 active+undersized+degraded, 39 active+undersized, 68 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 4.9 KiB/s rd, 13 KiB/s wr, 675 op/s; 51568/274992 objects degraded (18.753%); 9.5 KiB/s, 0 objects/s recovering 2023-10-09T10:59:20.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:20 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3501793412' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T10:59:20.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:20 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T10:59:20.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:20 smithi125 ceph-mon[195507]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T10:59:20.638 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:20 smithi155 ceph-mon[184854]: pgmap v905: 119 pgs: 12 active+undersized+degraded, 39 active+undersized, 68 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 4.9 KiB/s rd, 13 KiB/s wr, 675 op/s; 51568/274992 objects degraded (18.753%); 9.5 KiB/s, 0 objects/s recovering 2023-10-09T10:59:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:20 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3501793412' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T10:59:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:20 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T10:59:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:20 smithi155 ceph-mon[184854]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T10:59:20.860 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:20.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:20.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:20.876 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:59:20.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:20.993 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:21.165 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:21.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:21.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:21.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:21.272 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:59:21.272 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T10:59:21.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T10:59:21.272 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:59:21.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.273 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152592 7.2 7:4784bf95:::benchmark_data_smithi125_295816_object61455:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2023-10-09T10:59:21.273 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.354411+0000", 2023-10-09T10:59:21.273 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 431.914888378, 2023-10-09T10:59:21.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8049755350000001, 2023-10-09T10:59:21.273 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.273 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:59:21.273 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.273 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:59:21.273 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:21.273 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152592 2023-10-09T10:59:21.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.273 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354411+0000", 2023-10-09T10:59:21.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354411+0000", 2023-10-09T10:59:21.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354410+0000", 2023-10-09T10:59:21.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:21.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354418+0000", 2023-10-09T10:59:21.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0949999999999996e-06 2023-10-09T10:59:21.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354436+0000", 2023-10-09T10:59:21.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8036000000000001e-05 2023-10-09T10:59:21.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354446+0000", 2023-10-09T10:59:21.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7010000000000008e-06 2023-10-09T10:59:21.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354499+0000", 2023-10-09T10:59:21.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.287e-05 2023-10-09T10:59:21.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354601+0000", 2023-10-09T10:59:21.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000102686 2023-10-09T10:59:21.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T10:59:21.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.354675+0000", 2023-10-09T10:59:21.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3386000000000005e-05 2023-10-09T10:59:21.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.357099+0000", 2023-10-09T10:59:21.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024243289999999998 2023-10-09T10:59:21.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.358255+0000", 2023-10-09T10:59:21.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011560100000000001 2023-10-09T10:59:21.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.371426+0000", 2023-10-09T10:59:21.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013170737 2023-10-09T10:59:21.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.371438+0000", 2023-10-09T10:59:21.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2675e-05 2023-10-09T10:59:21.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.361977+0000", 2023-10-09T10:59:21.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99053876299999999 2023-10-09T10:59:21.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.361992+0000", 2023-10-09T10:59:21.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4707e-05 2023-10-09T10:59:21.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.159310+0000", 2023-10-09T10:59:21.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7973182649999999 2023-10-09T10:59:21.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.159387+0000", 2023-10-09T10:59:21.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6847000000000005e-05 2023-10-09T10:59:21.283 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.283 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.283 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.283 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152602 7.2 7:4739f8ed:::benchmark_data_smithi125_295816_object61465:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T10:59:21.283 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.384235+0000", 2023-10-09T10:59:21.284 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 431.885064521, 2023-10-09T10:59:21.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7844712920000001, 2023-10-09T10:59:21.284 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.284 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:21.284 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.284 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:59:21.284 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:21.284 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152602 2023-10-09T10:59:21.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.284 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.384235+0000", 2023-10-09T10:59:21.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.384235+0000", 2023-10-09T10:59:21.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.384234+0000", 2023-10-09T10:59:21.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:21.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.384241+0000", 2023-10-09T10:59:21.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3780000000000003e-06 2023-10-09T10:59:21.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.384252+0000", 2023-10-09T10:59:21.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0661e-05 2023-10-09T10:59:21.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.384263+0000", 2023-10-09T10:59:21.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0562e-05 2023-10-09T10:59:21.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.288 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.384532+0000", 2023-10-09T10:59:21.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00026952900000000001 2023-10-09T10:59:21.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.288 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.384542+0000", 2023-10-09T10:59:21.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0244999999999999e-05 2023-10-09T10:59:21.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.288 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.362019+0000", 2023-10-09T10:59:21.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97747686 2023-10-09T10:59:21.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.362027+0000", 2023-10-09T10:59:21.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7179999999999996e-06 2023-10-09T10:59:21.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.159409+0000", 2023-10-09T10:59:21.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7973824619999998 2023-10-09T10:59:21.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.159529+0000", 2023-10-09T10:59:21.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119432 2023-10-09T10:59:21.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T10:59:21.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.159666+0000", 2023-10-09T10:59:21.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013681500000000001 2023-10-09T10:59:21.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.166688+0000", 2023-10-09T10:59:21.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0070224670000000001 2023-10-09T10:59:21.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.168323+0000", 2023-10-09T10:59:21.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016351060000000001 2023-10-09T10:59:21.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.168421+0000", 2023-10-09T10:59:21.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7775000000000005e-05 2023-10-09T10:59:21.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.168491+0000", 2023-10-09T10:59:21.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9694000000000001e-05 2023-10-09T10:59:21.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:59:21.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.168541+0000", 2023-10-09T10:59:21.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0216999999999997e-05 2023-10-09T10:59:21.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.168707+0000", 2023-10-09T10:59:21.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016566399999999999 2023-10-09T10:59:21.294 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.294 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.294 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.295 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152603 7.2 7:479139f6:::benchmark_data_smithi125_295816_object61466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T10:59:21.295 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.385265+0000", 2023-10-09T10:59:21.295 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 431.88403429300001, 2023-10-09T10:59:21.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7944500539999999, 2023-10-09T10:59:21.295 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.295 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:21.295 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.295 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:59:21.296 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:21.296 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152603 2023-10-09T10:59:21.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.296 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.385265+0000", 2023-10-09T10:59:21.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.385265+0000", 2023-10-09T10:59:21.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.385264+0000", 2023-10-09T10:59:21.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:59:21.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.385270+0000", 2023-10-09T10:59:21.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9610000000000002e-06 2023-10-09T10:59:21.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.385282+0000", 2023-10-09T10:59:21.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1559000000000001e-05 2023-10-09T10:59:21.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.385290+0000", 2023-10-09T10:59:21.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1459999999999996e-06 2023-10-09T10:59:21.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.385916+0000", 2023-10-09T10:59:21.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00062581400000000004 2023-10-09T10:59:21.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.385925+0000", 2023-10-09T10:59:21.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4690000000000003e-06 2023-10-09T10:59:21.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.362048+0000", 2023-10-09T10:59:21.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97612259800000001 2023-10-09T10:59:21.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.362055+0000", 2023-10-09T10:59:21.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4930000000000001e-06 2023-10-09T10:59:21.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.162132+0000", 2023-10-09T10:59:21.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8000763870000001 2023-10-09T10:59:21.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.162267+0000", 2023-10-09T10:59:21.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013555199999999999 2023-10-09T10:59:21.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T10:59:21.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.162384+0000", 2023-10-09T10:59:21.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000116538 2023-10-09T10:59:21.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.167437+0000", 2023-10-09T10:59:21.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0050531150000000004 2023-10-09T10:59:21.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.175089+0000", 2023-10-09T10:59:21.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076518619999999997 2023-10-09T10:59:21.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.175987+0000", 2023-10-09T10:59:21.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00089854400000000004 2023-10-09T10:59:21.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.179609+0000", 2023-10-09T10:59:21.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036213370000000001 2023-10-09T10:59:21.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:59:21.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.179644+0000", 2023-10-09T10:59:21.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5652999999999999e-05 2023-10-09T10:59:21.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.179716+0000", 2023-10-09T10:59:21.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1092000000000003e-05 2023-10-09T10:59:21.306 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.306 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.306 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.307 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152614 7.2 7:47a30126:::benchmark_data_smithi125_295816_object61477:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T10:59:21.307 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:09.421293+0000", 2023-10-09T10:59:21.307 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 431.84800641300001, 2023-10-09T10:59:21.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.759414553, 2023-10-09T10:59:21.308 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.308 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:21.308 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.308 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:59:21.308 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:21.308 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152614 2023-10-09T10:59:21.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.308 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.421293+0000", 2023-10-09T10:59:21.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.421293+0000", 2023-10-09T10:59:21.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.421292+0000", 2023-10-09T10:59:21.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:59:21.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.421298+0000", 2023-10-09T10:59:21.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6790000000000002e-06 2023-10-09T10:59:21.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.421310+0000", 2023-10-09T10:59:21.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1651000000000001e-05 2023-10-09T10:59:21.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.421317+0000", 2023-10-09T10:59:21.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3390000000000004e-06 2023-10-09T10:59:21.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.421360+0000", 2023-10-09T10:59:21.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2691000000000002e-05 2023-10-09T10:59:21.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:09.421368+0000", 2023-10-09T10:59:21.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0369999999999995e-06 2023-10-09T10:59:21.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.362077+0000", 2023-10-09T10:59:21.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.940709087 2023-10-09T10:59:21.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.362084+0000", 2023-10-09T10:59:21.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5190000000000003e-06 2023-10-09T10:59:21.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.164710+0000", 2023-10-09T10:59:21.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8026256140000001 2023-10-09T10:59:21.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.164849+0000", 2023-10-09T10:59:21.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000138827 2023-10-09T10:59:21.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T10:59:21.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.164975+0000", 2023-10-09T10:59:21.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000126518 2023-10-09T10:59:21.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.167444+0000", 2023-10-09T10:59:21.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024686700000000001 2023-10-09T10:59:21.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.175123+0000", 2023-10-09T10:59:21.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.007678542 2023-10-09T10:59:21.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.176034+0000", 2023-10-09T10:59:21.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00091092899999999999 2023-10-09T10:59:21.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.180539+0000", 2023-10-09T10:59:21.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045050189999999999 2023-10-09T10:59:21.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:59:21.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.180592+0000", 2023-10-09T10:59:21.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3609000000000001e-05 2023-10-09T10:59:21.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.180708+0000", 2023-10-09T10:59:21.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000115701 2023-10-09T10:59:21.316 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.316 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.316 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.316 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35187.0:152609 7.0 7:065b7d37:::benchmark_data_smithi125_295816_object61472:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e245)", 2023-10-09T10:59:21.317 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:52:10.362639+0000", 2023-10-09T10:59:21.317 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 430.90666065099998, 2023-10-09T10:59:21.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7838590380000001, 2023-10-09T10:59:21.317 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.317 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:21.317 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.317 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35187", 2023-10-09T10:59:21.317 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:21.317 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 152609 2023-10-09T10:59:21.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.317 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.362639+0000", 2023-10-09T10:59:21.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.362639+0000", 2023-10-09T10:59:21.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.362638+0000", 2023-10-09T10:59:21.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:21.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.362644+0000", 2023-10-09T10:59:21.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9330000000000004e-06 2023-10-09T10:59:21.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.362657+0000", 2023-10-09T10:59:21.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2295e-05 2023-10-09T10:59:21.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.362665+0000", 2023-10-09T10:59:21.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.738e-06 2023-10-09T10:59:21.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.368006+0000", 2023-10-09T10:59:21.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0053408529999999996 2023-10-09T10:59:21.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:10.368017+0000", 2023-10-09T10:59:21.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0608000000000001e-05 2023-10-09T10:59:21.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.138896+0000", 2023-10-09T10:59:21.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7708793969999999 2023-10-09T10:59:21.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.139011+0000", 2023-10-09T10:59:21.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000114292 2023-10-09T10:59:21.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T10:59:21.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.139096+0000", 2023-10-09T10:59:21.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5481000000000004e-05 2023-10-09T10:59:21.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.142105+0000", 2023-10-09T10:59:21.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030090099999999999 2023-10-09T10:59:21.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.145882+0000", 2023-10-09T10:59:21.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037766589999999999 2023-10-09T10:59:21.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.146058+0000", 2023-10-09T10:59:21.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017574999999999999 2023-10-09T10:59:21.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.146092+0000", 2023-10-09T10:59:21.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4260000000000001e-05 2023-10-09T10:59:21.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.324 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:59:21.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.146151+0000", 2023-10-09T10:59:21.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9135999999999998e-05 2023-10-09T10:59:21.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:52:12.146498+0000", 2023-10-09T10:59:21.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00034724400000000002 2023-10-09T10:59:21.325 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.325 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.325 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.325 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:83933 8.a 8:51241b62:::benchmark_data_smithi125_329074_object83932:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2023-10-09T10:59:21.326 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:17.045153+0000", 2023-10-09T10:59:21.326 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 304.22414629999997, 2023-10-09T10:59:21.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.187316187, 2023-10-09T10:59:21.326 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.326 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:59:21.326 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.326 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:59:21.326 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:21.326 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 83933 2023-10-09T10:59:21.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.327 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.045153+0000", 2023-10-09T10:59:21.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.045153+0000", 2023-10-09T10:59:21.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.045153+0000", 2023-10-09T10:59:21.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:21.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.045199+0000", 2023-10-09T10:59:21.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6390000000000001e-05 2023-10-09T10:59:21.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.045224+0000", 2023-10-09T10:59:21.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.444e-05 2023-10-09T10:59:21.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.045230+0000", 2023-10-09T10:59:21.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0680000000000002e-06 2023-10-09T10:59:21.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.330 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.052249+0000", 2023-10-09T10:59:21.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0070188029999999997 2023-10-09T10:59:21.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.330 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.052461+0000", 2023-10-09T10:59:21.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00021247 2023-10-09T10:59:21.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,5", 2023-10-09T10:59:21.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.052683+0000", 2023-10-09T10:59:21.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000221814 2023-10-09T10:59:21.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.055337+0000", 2023-10-09T10:59:21.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026543959999999998 2023-10-09T10:59:21.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.068945+0000", 2023-10-09T10:59:21.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013607258000000001 2023-10-09T10:59:21.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.106782+0000", 2023-10-09T10:59:21.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.037837542000000002 2023-10-09T10:59:21.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.106798+0000", 2023-10-09T10:59:21.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5829e-05 2023-10-09T10:59:21.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.866661+0000", 2023-10-09T10:59:21.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.75986285899999995 2023-10-09T10:59:21.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.866681+0000", 2023-10-09T10:59:21.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0715000000000001e-05 2023-10-09T10:59:21.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.232372+0000", 2023-10-09T10:59:21.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3656903570000001 2023-10-09T10:59:21.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.232470+0000", 2023-10-09T10:59:21.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7830000000000004e-05 2023-10-09T10:59:21.334 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.334 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.334 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.334 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:83935 8.a 8:5009c3b5:::benchmark_data_smithi125_329074_object83934:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2023-10-09T10:59:21.335 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:17.057128+0000", 2023-10-09T10:59:21.335 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 304.21217149400002, 2023-10-09T10:59:21.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1784643140000002, 2023-10-09T10:59:21.335 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.335 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:59:21.335 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.335 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:59:21.335 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:21.335 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 83935 2023-10-09T10:59:21.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.335 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.057128+0000", 2023-10-09T10:59:21.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.057128+0000", 2023-10-09T10:59:21.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.057126+0000", 2023-10-09T10:59:21.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999971 2023-10-09T10:59:21.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.057159+0000", 2023-10-09T10:59:21.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3416e-05 2023-10-09T10:59:21.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.057179+0000", 2023-10-09T10:59:21.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9737000000000001e-05 2023-10-09T10:59:21.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.057190+0000", 2023-10-09T10:59:21.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1527e-05 2023-10-09T10:59:21.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.057268+0000", 2023-10-09T10:59:21.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8055999999999999e-05 2023-10-09T10:59:21.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.057461+0000", 2023-10-09T10:59:21.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019318800000000001 2023-10-09T10:59:21.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,5", 2023-10-09T10:59:21.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.057942+0000", 2023-10-09T10:59:21.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00048105199999999999 2023-10-09T10:59:21.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.060622+0000", 2023-10-09T10:59:21.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026799789999999999 2023-10-09T10:59:21.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.068983+0000", 2023-10-09T10:59:21.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083601360000000007 2023-10-09T10:59:21.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.107356+0000", 2023-10-09T10:59:21.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.038373177000000001 2023-10-09T10:59:21.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.107366+0000", 2023-10-09T10:59:21.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0655999999999999e-05 2023-10-09T10:59:21.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.866718+0000", 2023-10-09T10:59:21.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.75935186200000004 2023-10-09T10:59:21.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.866728+0000", 2023-10-09T10:59:21.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9850000000000001e-06 2023-10-09T10:59:21.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.235494+0000", 2023-10-09T10:59:21.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.368765376 2023-10-09T10:59:21.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.235593+0000", 2023-10-09T10:59:21.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8930000000000003e-05 2023-10-09T10:59:21.343 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.343 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.343 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.344 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:83941 8.7 8:e9152048:::benchmark_data_smithi125_329074_object83940:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2023-10-09T10:59:21.344 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:17.068788+0000", 2023-10-09T10:59:21.344 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 304.20051156599999, 2023-10-09T10:59:21.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3694775859999999, 2023-10-09T10:59:21.344 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.344 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:59:21.344 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.344 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:59:21.344 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:21.344 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 83941 2023-10-09T10:59:21.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.344 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.068788+0000", 2023-10-09T10:59:21.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.068788+0000", 2023-10-09T10:59:21.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.068786+0000", 2023-10-09T10:59:21.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:59:21.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.068835+0000", 2023-10-09T10:59:21.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8433999999999997e-05 2023-10-09T10:59:21.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.068850+0000", 2023-10-09T10:59:21.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5302e-05 2023-10-09T10:59:21.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.068860+0000", 2023-10-09T10:59:21.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0213e-05 2023-10-09T10:59:21.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.068914+0000", 2023-10-09T10:59:21.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3708999999999997e-05 2023-10-09T10:59:21.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.069106+0000", 2023-10-09T10:59:21.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019138299999999999 2023-10-09T10:59:21.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T10:59:21.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.069281+0000", 2023-10-09T10:59:21.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017515900000000001 2023-10-09T10:59:21.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.071853+0000", 2023-10-09T10:59:21.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025724490000000001 2023-10-09T10:59:21.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.117475+0000", 2023-10-09T10:59:21.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.045621672000000002 2023-10-09T10:59:21.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.117488+0000", 2023-10-09T10:59:21.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3207000000000001e-05 2023-10-09T10:59:21.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.864350+0000", 2023-10-09T10:59:21.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.74686178199999997 2023-10-09T10:59:21.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.864364+0000", 2023-10-09T10:59:21.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4334e-05 2023-10-09T10:59:21.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.438196+0000", 2023-10-09T10:59:21.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5738323570000001 2023-10-09T10:59:21.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.438266+0000", 2023-10-09T10:59:21.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9313000000000005e-05 2023-10-09T10:59:21.351 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.351 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.351 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.351 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:83949 8.a 8:51b7d356:::benchmark_data_smithi125_329074_object83948:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2023-10-09T10:59:21.351 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:17.089656+0000", 2023-10-09T10:59:21.351 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 304.17964400800003, 2023-10-09T10:59:21.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1717330760000002, 2023-10-09T10:59:21.352 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.352 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:21.352 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.352 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:59:21.352 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:21.352 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 83949 2023-10-09T10:59:21.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.352 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.352 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.352 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.089656+0000", 2023-10-09T10:59:21.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.353 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.353 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.089656+0000", 2023-10-09T10:59:21.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.353 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.353 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.089654+0000", 2023-10-09T10:59:21.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:59:21.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.353 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.089688+0000", 2023-10-09T10:59:21.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3646999999999998e-05 2023-10-09T10:59:21.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.089705+0000", 2023-10-09T10:59:21.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7244e-05 2023-10-09T10:59:21.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.089717+0000", 2023-10-09T10:59:21.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1934e-05 2023-10-09T10:59:21.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.107396+0000", 2023-10-09T10:59:21.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.017679525000000001 2023-10-09T10:59:21.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.107405+0000", 2023-10-09T10:59:21.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4169999999999992e-06 2023-10-09T10:59:21.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.866754+0000", 2023-10-09T10:59:21.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.75934893800000003 2023-10-09T10:59:21.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.866762+0000", 2023-10-09T10:59:21.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1100000000000003e-06 2023-10-09T10:59:21.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.235624+0000", 2023-10-09T10:59:21.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.368861621 2023-10-09T10:59:21.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.235779+0000", 2023-10-09T10:59:21.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000155599 2023-10-09T10:59:21.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:59:21.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.235940+0000", 2023-10-09T10:59:21.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016126000000000001 2023-10-09T10:59:21.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.239005+0000", 2023-10-09T10:59:21.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003064916 2023-10-09T10:59:21.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.246344+0000", 2023-10-09T10:59:21.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0073385780000000001 2023-10-09T10:59:21.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.256969+0000", 2023-10-09T10:59:21.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010625262999999999 2023-10-09T10:59:21.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.257007+0000", 2023-10-09T10:59:21.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8173999999999997e-05 2023-10-09T10:59:21.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:59:21.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.257063+0000", 2023-10-09T10:59:21.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5689000000000002e-05 2023-10-09T10:59:21.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.261389+0000", 2023-10-09T10:59:21.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043258599999999999 2023-10-09T10:59:21.360 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.360 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.360 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.360 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:83950 8.b 8:d527cf60:::benchmark_data_smithi125_329074_object83949:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e280)", 2023-10-09T10:59:21.360 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:17.089729+0000", 2023-10-09T10:59:21.360 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 304.179571092, 2023-10-09T10:59:21.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1564821419999998, 2023-10-09T10:59:21.360 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.360 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:21.361 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.361 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:59:21.361 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:21.361 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 83950 2023-10-09T10:59:21.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.361 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.089729+0000", 2023-10-09T10:59:21.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.089729+0000", 2023-10-09T10:59:21.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.362 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.089728+0000", 2023-10-09T10:59:21.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:21.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.362 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.089757+0000", 2023-10-09T10:59:21.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8731e-05 2023-10-09T10:59:21.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.089768+0000", 2023-10-09T10:59:21.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1189000000000001e-05 2023-10-09T10:59:21.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.089774+0000", 2023-10-09T10:59:21.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4219999999999997e-06 2023-10-09T10:59:21.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.113812+0000", 2023-10-09T10:59:21.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.024037177 2023-10-09T10:59:21.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.113822+0000", 2023-10-09T10:59:21.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0052e-05 2023-10-09T10:59:21.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.858782+0000", 2023-10-09T10:59:21.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.74496024100000002 2023-10-09T10:59:21.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.858790+0000", 2023-10-09T10:59:21.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0509999999999997e-06 2023-10-09T10:59:21.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.225729+0000", 2023-10-09T10:59:21.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3669394100000001 2023-10-09T10:59:21.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.225900+0000", 2023-10-09T10:59:21.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017024600000000001 2023-10-09T10:59:21.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,4,7", 2023-10-09T10:59:21.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.226018+0000", 2023-10-09T10:59:21.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000117928 2023-10-09T10:59:21.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.235559+0000", 2023-10-09T10:59:21.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0095412829999999994 2023-10-09T10:59:21.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.235570+0000", 2023-10-09T10:59:21.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.119e-05 2023-10-09T10:59:21.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.235612+0000", 2023-10-09T10:59:21.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1699000000000002e-05 2023-10-09T10:59:21.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.241913+0000", 2023-10-09T10:59:21.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0063010030000000003 2023-10-09T10:59:21.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:59:21.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.241972+0000", 2023-10-09T10:59:21.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9199e-05 2023-10-09T10:59:21.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.246211+0000", 2023-10-09T10:59:21.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042388599999999997 2023-10-09T10:59:21.369 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.369 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.369 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.369 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:83956 8.a 8:50dfc873:::benchmark_data_smithi125_329074_object83955:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T10:59:21.369 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:17.105472+0000", 2023-10-09T10:59:21.370 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 304.16382770000001, 2023-10-09T10:59:21.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1642708810000002, 2023-10-09T10:59:21.370 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.370 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:21.370 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.370 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:59:21.370 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:21.370 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 83956 2023-10-09T10:59:21.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.370 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.105472+0000", 2023-10-09T10:59:21.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.105472+0000", 2023-10-09T10:59:21.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.105470+0000", 2023-10-09T10:59:21.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T10:59:21.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.105496+0000", 2023-10-09T10:59:21.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5683000000000001e-05 2023-10-09T10:59:21.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.105509+0000", 2023-10-09T10:59:21.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2754e-05 2023-10-09T10:59:21.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.105519+0000", 2023-10-09T10:59:21.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8959999999999998e-06 2023-10-09T10:59:21.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.107428+0000", 2023-10-09T10:59:21.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019094820000000001 2023-10-09T10:59:21.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.107437+0000", 2023-10-09T10:59:21.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4530000000000002e-06 2023-10-09T10:59:21.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.866783+0000", 2023-10-09T10:59:21.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.75934668800000005 2023-10-09T10:59:21.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.866792+0000", 2023-10-09T10:59:21.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.439e-06 2023-10-09T10:59:21.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.242643+0000", 2023-10-09T10:59:21.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3758512279999999 2023-10-09T10:59:21.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.242797+0000", 2023-10-09T10:59:21.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015357399999999999 2023-10-09T10:59:21.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T10:59:21.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.242954+0000", 2023-10-09T10:59:21.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015773499999999999 2023-10-09T10:59:21.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.261471+0000", 2023-10-09T10:59:21.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018516982000000001 2023-10-09T10:59:21.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.261484+0000", 2023-10-09T10:59:21.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3169999999999999e-05 2023-10-09T10:59:21.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.377 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.377 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.261523+0000", 2023-10-09T10:59:21.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8843000000000001e-05 2023-10-09T10:59:21.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.377 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.377 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.266292+0000", 2023-10-09T10:59:21.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0047681920000000001 2023-10-09T10:59:21.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:59:21.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.266353+0000", 2023-10-09T10:59:21.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1556000000000005e-05 2023-10-09T10:59:21.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.269743+0000", 2023-10-09T10:59:21.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0033898980000000001 2023-10-09T10:59:21.378 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.378 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.378 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.379 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:83962 8.b 8:d4495a34:::benchmark_data_smithi125_329074_object83961:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T10:59:21.379 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:17.114683+0000", 2023-10-09T10:59:21.379 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 304.154617116, 2023-10-09T10:59:21.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1447367279999998, 2023-10-09T10:59:21.379 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.379 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:21.379 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.379 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:59:21.379 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:21.379 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 83962 2023-10-09T10:59:21.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.379 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.114683+0000", 2023-10-09T10:59:21.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.114683+0000", 2023-10-09T10:59:21.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.114681+0000", 2023-10-09T10:59:21.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:21.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.114704+0000", 2023-10-09T10:59:21.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2543000000000001e-05 2023-10-09T10:59:21.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.114718+0000", 2023-10-09T10:59:21.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4481000000000001e-05 2023-10-09T10:59:21.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.114727+0000", 2023-10-09T10:59:21.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0610000000000002e-06 2023-10-09T10:59:21.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.114777+0000", 2023-10-09T10:59:21.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0024000000000001e-05 2023-10-09T10:59:21.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.114786+0000", 2023-10-09T10:59:21.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3559999999999999e-06 2023-10-09T10:59:21.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.858804+0000", 2023-10-09T10:59:21.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.74401758299999998 2023-10-09T10:59:21.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.384 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.384 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.858808+0000", 2023-10-09T10:59:21.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3200000000000001e-06 2023-10-09T10:59:21.384 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.384 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.384 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.384 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.233013+0000", 2023-10-09T10:59:21.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3742045780000001 2023-10-09T10:59:21.384 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.384 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.384 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.384 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.233147+0000", 2023-10-09T10:59:21.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000133905 2023-10-09T10:59:21.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,4,7", 2023-10-09T10:59:21.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.233288+0000", 2023-10-09T10:59:21.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000141285 2023-10-09T10:59:21.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.241311+0000", 2023-10-09T10:59:21.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0080231490000000003 2023-10-09T10:59:21.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.241399+0000", 2023-10-09T10:59:21.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7357999999999999e-05 2023-10-09T10:59:21.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.246590+0000", 2023-10-09T10:59:21.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0051917939999999996 2023-10-09T10:59:21.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.255827+0000", 2023-10-09T10:59:21.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0092369960000000008 2023-10-09T10:59:21.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:59:21.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.255894+0000", 2023-10-09T10:59:21.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6586000000000002e-05 2023-10-09T10:59:21.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.259419+0000", 2023-10-09T10:59:21.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035253490000000001 2023-10-09T10:59:21.390 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.390 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.391 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.391 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:83963 8.7 8:e6c91b26:::benchmark_data_smithi125_329074_object83962:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T10:59:21.391 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:17.114739+0000", 2023-10-09T10:59:21.391 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 304.15456064199998, 2023-10-09T10:59:21.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3451434170000001, 2023-10-09T10:59:21.392 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.392 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:21.392 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.392 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:59:21.392 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:21.392 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 83963 2023-10-09T10:59:21.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.393 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.393 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.393 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.114739+0000", 2023-10-09T10:59:21.393 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.393 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.114739+0000", 2023-10-09T10:59:21.394 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.394 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.394 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.395 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.114739+0000", 2023-10-09T10:59:21.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:21.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.395 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.395 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.114758+0000", 2023-10-09T10:59:21.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9341e-05 2023-10-09T10:59:21.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.395 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.395 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.114764+0000", 2023-10-09T10:59:21.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1260000000000003e-06 2023-10-09T10:59:21.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.114770+0000", 2023-10-09T10:59:21.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6019999999999999e-06 2023-10-09T10:59:21.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.117517+0000", 2023-10-09T10:59:21.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027472730000000002 2023-10-09T10:59:21.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.397 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.397 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.117525+0000", 2023-10-09T10:59:21.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1450000000000008e-06 2023-10-09T10:59:21.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.397 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.397 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.864398+0000", 2023-10-09T10:59:21.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.746873075 2023-10-09T10:59:21.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.397 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.398 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.864407+0000", 2023-10-09T10:59:21.398 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4060000000000005e-06 2023-10-09T10:59:21.398 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.398 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.398 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.438280+0000", 2023-10-09T10:59:21.398 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.573873291 2023-10-09T10:59:21.398 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.399 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.438387+0000", 2023-10-09T10:59:21.399 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000107156 2023-10-09T10:59:21.399 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.399 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.399 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T10:59:21.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.438491+0000", 2023-10-09T10:59:21.400 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010448 2023-10-09T10:59:21.400 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.400 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.400 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.400 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.443903+0000", 2023-10-09T10:59:21.400 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0054118769999999998 2023-10-09T10:59:21.400 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.401 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.401 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.401 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.443915+0000", 2023-10-09T10:59:21.401 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1182999999999999e-05 2023-10-09T10:59:21.402 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.402 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.402 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.402 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.444031+0000", 2023-10-09T10:59:21.402 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000116561 2023-10-09T10:59:21.402 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.403 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.403 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.403 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.456725+0000", 2023-10-09T10:59:21.403 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012693658 2023-10-09T10:59:21.403 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.403 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.403 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:59:21.405 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.456785+0000", 2023-10-09T10:59:21.405 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0489000000000003e-05 2023-10-09T10:59:21.405 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.406 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.406 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.406 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.459883+0000", 2023-10-09T10:59:21.406 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003097294 2023-10-09T10:59:21.406 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.406 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.407 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.407 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.407 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.407 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:83965 8.f 8:fb9b0de7:::benchmark_data_smithi125_329074_object83964:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T10:59:21.407 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:17.123489+0000", 2023-10-09T10:59:21.407 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 304.14581038599999, 2023-10-09T10:59:21.407 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3331353630000002, 2023-10-09T10:59:21.408 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.408 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:21.408 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.408 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:59:21.408 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:21.408 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 83965 2023-10-09T10:59:21.408 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.409 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.409 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.409 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.409 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.123489+0000", 2023-10-09T10:59:21.410 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.410 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.410 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.410 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.410 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.123489+0000", 2023-10-09T10:59:21.410 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.410 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.411 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.411 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.411 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.123488+0000", 2023-10-09T10:59:21.411 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:21.411 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.411 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.411 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.412 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.123511+0000", 2023-10-09T10:59:21.412 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3249000000000001e-05 2023-10-09T10:59:21.412 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.412 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.412 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.412 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.123522+0000", 2023-10-09T10:59:21.412 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0214000000000001e-05 2023-10-09T10:59:21.413 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.413 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.413 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.413 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.123527+0000", 2023-10-09T10:59:21.413 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.507e-06 2023-10-09T10:59:21.413 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.413 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.413 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.414 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.123570+0000", 2023-10-09T10:59:21.414 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3275000000000002e-05 2023-10-09T10:59:21.414 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.414 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.414 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.414 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.123579+0000", 2023-10-09T10:59:21.414 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8200000000000003e-06 2023-10-09T10:59:21.414 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.414 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.414 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.414 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.861991+0000", 2023-10-09T10:59:21.414 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.73841215299999996 2023-10-09T10:59:21.415 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.415 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.862004+0000", 2023-10-09T10:59:21.415 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2335e-05 2023-10-09T10:59:21.415 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.415 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.433126+0000", 2023-10-09T10:59:21.415 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5711226439999999 2023-10-09T10:59:21.415 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.433263+0000", 2023-10-09T10:59:21.416 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013657299999999999 2023-10-09T10:59:21.416 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.416 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.416 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T10:59:21.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.433384+0000", 2023-10-09T10:59:21.416 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000121616 2023-10-09T10:59:21.416 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.416 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.416 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.438067+0000", 2023-10-09T10:59:21.416 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046826239999999998 2023-10-09T10:59:21.417 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.417 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.417 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.417 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.438122+0000", 2023-10-09T10:59:21.417 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4993e-05 2023-10-09T10:59:21.417 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.417 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.417 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.417 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.438132+0000", 2023-10-09T10:59:21.417 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6209999999999999e-06 2023-10-09T10:59:21.417 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.417 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.418 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.418 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.452406+0000", 2023-10-09T10:59:21.418 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014274126999999999 2023-10-09T10:59:21.418 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.418 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.418 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:59:21.418 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.452475+0000", 2023-10-09T10:59:21.418 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9377999999999999e-05 2023-10-09T10:59:21.418 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.418 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.418 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.418 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.456625+0000", 2023-10-09T10:59:21.418 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041495389999999998 2023-10-09T10:59:21.419 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.419 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.419 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.419 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.419 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.419 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:83968 8.7 8:ec8386b5:::benchmark_data_smithi125_329074_object83967:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T10:59:21.419 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:17.136079+0000", 2023-10-09T10:59:21.419 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 304.13322078200002, 2023-10-09T10:59:21.419 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3287112589999999, 2023-10-09T10:59:21.419 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.419 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:21.419 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.420 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:59:21.420 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:21.420 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 83968 2023-10-09T10:59:21.420 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.420 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.420 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.420 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.420 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.136079+0000", 2023-10-09T10:59:21.420 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.420 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.420 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.420 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.420 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.136079+0000", 2023-10-09T10:59:21.421 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.421 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.421 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.421 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.421 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.136078+0000", 2023-10-09T10:59:21.421 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:21.421 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.421 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.421 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.421 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.136103+0000", 2023-10-09T10:59:21.421 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5548999999999999e-05 2023-10-09T10:59:21.421 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.421 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.422 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.422 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.136116+0000", 2023-10-09T10:59:21.422 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2712e-05 2023-10-09T10:59:21.422 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.422 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.422 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.422 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.136124+0000", 2023-10-09T10:59:21.422 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6389999999999992e-06 2023-10-09T10:59:21.422 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.422 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.422 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.422 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.136177+0000", 2023-10-09T10:59:21.422 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3010000000000002e-05 2023-10-09T10:59:21.423 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.423 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.423 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.423 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.136186+0000", 2023-10-09T10:59:21.423 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7289999999999995e-06 2023-10-09T10:59:21.423 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.423 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.423 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.423 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.864564+0000", 2023-10-09T10:59:21.423 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.72837760900000004 2023-10-09T10:59:21.423 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.423 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.424 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.424 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.864574+0000", 2023-10-09T10:59:21.424 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0431999999999999e-05 2023-10-09T10:59:21.424 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.424 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.424 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.424 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.441149+0000", 2023-10-09T10:59:21.424 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5765743250000002 2023-10-09T10:59:21.424 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.424 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.424 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.424 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.441321+0000", 2023-10-09T10:59:21.424 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017254400000000001 2023-10-09T10:59:21.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T10:59:21.425 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.441501+0000", 2023-10-09T10:59:21.425 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000180001 2023-10-09T10:59:21.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.425 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.443975+0000", 2023-10-09T10:59:21.425 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024741810000000002 2023-10-09T10:59:21.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.452347+0000", 2023-10-09T10:59:21.426 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083717010000000005 2023-10-09T10:59:21.426 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.426 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.426 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.461106+0000", 2023-10-09T10:59:21.426 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0087593570000000006 2023-10-09T10:59:21.426 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.426 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.426 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.461223+0000", 2023-10-09T10:59:21.426 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000116279 2023-10-09T10:59:21.427 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.427 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.427 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:59:21.427 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.461282+0000", 2023-10-09T10:59:21.427 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9079000000000001e-05 2023-10-09T10:59:21.427 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.427 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.427 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.427 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.464790+0000", 2023-10-09T10:59:21.427 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035084309999999998 2023-10-09T10:59:21.428 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.428 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.428 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.428 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.428 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.428 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:83969 8.b 8:d181614c:::benchmark_data_smithi125_329074_object83968:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T10:59:21.428 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:17.139045+0000", 2023-10-09T10:59:21.428 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 304.13025516699997, 2023-10-09T10:59:21.428 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1243467869999999, 2023-10-09T10:59:21.428 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.428 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:21.428 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.429 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:59:21.429 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:21.429 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 83969 2023-10-09T10:59:21.429 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.429 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.429 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.429 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.429 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.139045+0000", 2023-10-09T10:59:21.429 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.429 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.429 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.429 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.429 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.139045+0000", 2023-10-09T10:59:21.429 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.430 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.430 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.430 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.139043+0000", 2023-10-09T10:59:21.430 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:21.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.430 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.430 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.430 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.139066+0000", 2023-10-09T10:59:21.430 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2347000000000001e-05 2023-10-09T10:59:21.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.430 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.139077+0000", 2023-10-09T10:59:21.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1630999999999999e-05 2023-10-09T10:59:21.431 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.139086+0000", 2023-10-09T10:59:21.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7579999999999998e-06 2023-10-09T10:59:21.431 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.139144+0000", 2023-10-09T10:59:21.432 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8028000000000001e-05 2023-10-09T10:59:21.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.432 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.432 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.139155+0000", 2023-10-09T10:59:21.432 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0684e-05 2023-10-09T10:59:21.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.432 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.432 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.858822+0000", 2023-10-09T10:59:21.432 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.71966675700000005 2023-10-09T10:59:21.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.858827+0000", 2023-10-09T10:59:21.433 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5600000000000001e-06 2023-10-09T10:59:21.433 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.235640+0000", 2023-10-09T10:59:21.433 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3768129359999999 2023-10-09T10:59:21.433 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.235766+0000", 2023-10-09T10:59:21.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012619000000000001 2023-10-09T10:59:21.434 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.434 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.434 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,4,7", 2023-10-09T10:59:21.434 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.235877+0000", 2023-10-09T10:59:21.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110797 2023-10-09T10:59:21.434 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.434 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.434 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.434 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.241349+0000", 2023-10-09T10:59:21.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0054718630000000004 2023-10-09T10:59:21.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.435 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.241404+0000", 2023-10-09T10:59:21.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4753999999999998e-05 2023-10-09T10:59:21.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.435 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.255752+0000", 2023-10-09T10:59:21.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014348085999999999 2023-10-09T10:59:21.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.259483+0000", 2023-10-09T10:59:21.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003731107 2023-10-09T10:59:21.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:59:21.437 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.259544+0000", 2023-10-09T10:59:21.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0949000000000002e-05 2023-10-09T10:59:21.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.437 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.437 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.263391+0000", 2023-10-09T10:59:21.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003847559 2023-10-09T10:59:21.437 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.437 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.438 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.438 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.438 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.438 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:83975 8.b 8:d38df2db:::benchmark_data_smithi125_329074_object83974:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T10:59:21.438 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:17.157992+0000", 2023-10-09T10:59:21.438 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 304.11130771000001, 2023-10-09T10:59:21.438 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1097873109999998, 2023-10-09T10:59:21.438 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.438 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:21.438 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.438 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:59:21.438 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:21.439 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 83975 2023-10-09T10:59:21.439 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.439 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.439 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.439 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.439 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.157992+0000", 2023-10-09T10:59:21.439 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.439 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.439 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.439 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.439 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.157992+0000", 2023-10-09T10:59:21.439 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.440 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.440 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.440 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.440 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.157991+0000", 2023-10-09T10:59:21.440 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:21.440 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.440 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.440 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.440 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.158014+0000", 2023-10-09T10:59:21.440 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3232999999999999e-05 2023-10-09T10:59:21.440 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.441 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.158026+0000", 2023-10-09T10:59:21.441 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1698e-05 2023-10-09T10:59:21.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.441 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.158032+0000", 2023-10-09T10:59:21.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6429999999999998e-06 2023-10-09T10:59:21.442 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.442 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.158076+0000", 2023-10-09T10:59:21.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4181000000000003e-05 2023-10-09T10:59:21.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.158085+0000", 2023-10-09T10:59:21.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8380000000000008e-06 2023-10-09T10:59:21.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.858849+0000", 2023-10-09T10:59:21.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.70076327000000005 2023-10-09T10:59:21.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.444 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:17.858856+0000", 2023-10-09T10:59:21.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7449999999999995e-06 2023-10-09T10:59:21.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.444 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.238541+0000", 2023-10-09T10:59:21.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.379684973 2023-10-09T10:59:21.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.444 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.238720+0000", 2023-10-09T10:59:21.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000178402 2023-10-09T10:59:21.445 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.445 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.445 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,4,7", 2023-10-09T10:59:21.445 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.238904+0000", 2023-10-09T10:59:21.445 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018430200000000001 2023-10-09T10:59:21.445 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.445 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.445 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.445 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.241409+0000", 2023-10-09T10:59:21.445 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025045639999999999 2023-10-09T10:59:21.445 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.445 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.445 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.446 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.246553+0000", 2023-10-09T10:59:21.446 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0051443590000000003 2023-10-09T10:59:21.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.446 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.446 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.255792+0000", 2023-10-09T10:59:21.446 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00923882 2023-10-09T10:59:21.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.446 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.446 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.263488+0000", 2023-10-09T10:59:21.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076967520000000003 2023-10-09T10:59:21.447 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.447 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.447 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:59:21.447 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.263556+0000", 2023-10-09T10:59:21.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7350999999999999e-05 2023-10-09T10:59:21.447 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.447 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.447 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.447 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:19.267779+0000", 2023-10-09T10:59:21.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042235720000000001 2023-10-09T10:59:21.447 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.448 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.448 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.448 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.448 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:96590 8.b 8:d33b59bd:::benchmark_data_smithi125_329074_object96589:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T10:59:21.448 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:47.508059+0000", 2023-10-09T10:59:21.448 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 273.76124050800001, 2023-10-09T10:59:21.448 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6742213769999998, 2023-10-09T10:59:21.448 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.448 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:59:21.448 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.448 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T10:59:21.449 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:21.449 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 96590 2023-10-09T10:59:21.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.449 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.449 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.449 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.508059+0000", 2023-10-09T10:59:21.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.449 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.449 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.508059+0000", 2023-10-09T10:59:21.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.508059+0000", 2023-10-09T10:59:21.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:21.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.508105+0000", 2023-10-09T10:59:21.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6474000000000003e-05 2023-10-09T10:59:21.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.508113+0000", 2023-10-09T10:59:21.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8299999999999996e-06 2023-10-09T10:59:21.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.508118+0000", 2023-10-09T10:59:21.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4589999999999997e-06 2023-10-09T10:59:21.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.511772+0000", 2023-10-09T10:59:21.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036537980000000002 2023-10-09T10:59:21.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.511986+0000", 2023-10-09T10:59:21.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00021419399999999999 2023-10-09T10:59:21.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,4,7", 2023-10-09T10:59:21.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.512166+0000", 2023-10-09T10:59:21.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017932199999999999 2023-10-09T10:59:21.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.515458+0000", 2023-10-09T10:59:21.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032925789999999999 2023-10-09T10:59:21.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.521000+0000", 2023-10-09T10:59:21.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0055414899999999996 2023-10-09T10:59:21.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.521329+0000", 2023-10-09T10:59:21.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00032869699999999997 2023-10-09T10:59:21.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.662323+0000", 2023-10-09T10:59:21.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.14099461399999999 2023-10-09T10:59:21.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.662338+0000", 2023-10-09T10:59:21.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.518e-05 2023-10-09T10:59:21.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.900755+0000", 2023-10-09T10:59:21.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.238416778 2023-10-09T10:59:21.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.900768+0000", 2023-10-09T10:59:21.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2850000000000001e-05 2023-10-09T10:59:21.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.182179+0000", 2023-10-09T10:59:21.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2814107770000001 2023-10-09T10:59:21.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:49.182281+0000", 2023-10-09T10:59:21.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000101939 2023-10-09T10:59:21.456 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.456 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.456 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.456 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122030 9.2 9:4f76a93a:::benchmark_data_smithi125_363789_object11731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T10:59:21.457 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.156631+0000", 2023-10-09T10:59:21.457 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 13.112668807, 2023-10-09T10:59:21.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.081708254, 2023-10-09T10:59:21.457 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.457 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T10:59:21.457 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.457 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T10:59:21.457 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T10:59:21.457 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122030 2023-10-09T10:59:21.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.457 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.156631+0000", 2023-10-09T10:59:21.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.156631+0000", 2023-10-09T10:59:21.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.156631+0000", 2023-10-09T10:59:21.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:21.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.156634+0000", 2023-10-09T10:59:21.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4319999999999999e-06 2023-10-09T10:59:21.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.156643+0000", 2023-10-09T10:59:21.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1379999999999996e-06 2023-10-09T10:59:21.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.156648+0000", 2023-10-09T10:59:21.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2310000000000004e-06 2023-10-09T10:59:21.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.158226+0000", 2023-10-09T10:59:21.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015773040000000001 2023-10-09T10:59:21.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.158347+0000", 2023-10-09T10:59:21.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012126899999999999 2023-10-09T10:59:21.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T10:59:21.461 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.158457+0000", 2023-10-09T10:59:21.461 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110157 2023-10-09T10:59:21.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.461 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.176493+0000", 2023-10-09T10:59:21.461 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018035882999999999 2023-10-09T10:59:21.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.176504+0000", 2023-10-09T10:59:21.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1355000000000001e-05 2023-10-09T10:59:21.462 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.462 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.462 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.209008+0000", 2023-10-09T10:59:21.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0325033349999999 2023-10-09T10:59:21.462 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.462 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.462 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for missing object", 2023-10-09T10:59:21.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.209122+0000", 2023-10-09T10:59:21.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011432100000000001 2023-10-09T10:59:21.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.463 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.238254+0000", 2023-10-09T10:59:21.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.029132216999999998 2023-10-09T10:59:21.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.463 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.238339+0000", 2023-10-09T10:59:21.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5076999999999999e-05 2023-10-09T10:59:21.463 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.463 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.463 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.464 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.464 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.464 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122042 9.2 9:45a866b9:::benchmark_data_smithi125_363789_object11743:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T10:59:21.464 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.193768+0000", 2023-10-09T10:59:21.464 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 13.075531811999999, 2023-10-09T10:59:21.464 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.043778103, 2023-10-09T10:59:21.464 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:21.464 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:21.464 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:21.464 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T10:59:21.464 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T10:59:21.464 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122042 2023-10-09T10:59:21.464 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.465 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:21.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.465 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:21.465 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.193768+0000", 2023-10-09T10:59:21.465 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.465 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.465 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:21.465 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.193768+0000", 2023-10-09T10:59:21.465 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:21.465 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.465 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:21.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.193768+0000", 2023-10-09T10:59:21.466 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:21.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.466 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:21.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.193770+0000", 2023-10-09T10:59:21.466 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5600000000000001e-06 2023-10-09T10:59:21.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.466 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:21.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.193784+0000", 2023-10-09T10:59:21.466 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3601000000000001e-05 2023-10-09T10:59:21.467 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.467 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:21.467 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.193788+0000", 2023-10-09T10:59:21.467 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6469999999999997e-06 2023-10-09T10:59:21.467 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.467 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.467 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.204343+0000", 2023-10-09T10:59:21.467 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010554976000000001 2023-10-09T10:59:21.467 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.468 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:21.468 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.204357+0000", 2023-10-09T10:59:21.468 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3569e-05 2023-10-09T10:59:21.468 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.468 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.468 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:21.468 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.209329+0000", 2023-10-09T10:59:21.468 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.004971973 2023-10-09T10:59:21.468 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.468 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.468 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:21.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.209472+0000", 2023-10-09T10:59:21.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014322599999999999 2023-10-09T10:59:21.469 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.469 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.469 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2", 2023-10-09T10:59:21.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.209608+0000", 2023-10-09T10:59:21.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013538100000000001 2023-10-09T10:59:21.469 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.469 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.469 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T10:59:21.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.220334+0000", 2023-10-09T10:59:21.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010726951 2023-10-09T10:59:21.469 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:21.470 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.237394+0000", 2023-10-09T10:59:21.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.017059260999999999 2023-10-09T10:59:21.470 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T10:59:21.470 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.237451+0000", 2023-10-09T10:59:21.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7368999999999999e-05 2023-10-09T10:59:21.470 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:21.471 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:21.471 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T10:59:21.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.237546+0000", 2023-10-09T10:59:21.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4975000000000005e-05 2023-10-09T10:59:21.471 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.471 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.471 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.471 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:21.471 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:21.471 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:59:21.472 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:59:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:59:21] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-10-09T10:59:21.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:21.473 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:21.473 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:21.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:21.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:21.603 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:21.672 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:21.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:21.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:21.694 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:21 smithi155 ceph-mon[184854]: failed to encode map e362 with expected crc 2023-10-09T10:59:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:21 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T10:59:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:21 smithi155 ceph-mon[184854]: osdmap e362: 8 total, 7 up, 7 in 2023-10-09T10:59:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:21 smithi155 ceph-mon[184854]: failed to encode map e362 with expected crc 2023-10-09T10:59:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:21 smithi155 ceph-mon[184854]: failed to encode map e362 with expected crc 2023-10-09T10:59:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:21 smithi155 ceph-mon[184854]: failed to encode map e362 with expected crc 2023-10-09T10:59:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:21 smithi155 ceph-mon[184854]: failed to encode map e362 with expected crc 2023-10-09T10:59:21.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:21.784 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:21 smithi125 ceph-mon[195507]: failed to encode map e362 with expected crc 2023-10-09T10:59:21.784 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:21 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T10:59:21.784 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:21 smithi125 ceph-mon[195507]: osdmap e362: 8 total, 7 up, 7 in 2023-10-09T10:59:21.784 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:21 smithi125 ceph-mon[195507]: failed to encode map e362 with expected crc 2023-10-09T10:59:21.784 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:21 smithi125 ceph-mon[195507]: failed to encode map e362 with expected crc 2023-10-09T10:59:21.784 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:21 smithi125 ceph-mon[195507]: failed to encode map e362 with expected crc 2023-10-09T10:59:21.784 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:21 smithi125 ceph-mon[195507]: failed to encode map e362 with expected crc 2023-10-09T10:59:21.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:21 smithi125 ceph-mon[200197]: failed to encode map e362 with expected crc 2023-10-09T10:59:21.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:21 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T10:59:21.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:21 smithi125 ceph-mon[200197]: osdmap e362: 8 total, 7 up, 7 in 2023-10-09T10:59:21.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:21 smithi125 ceph-mon[200197]: failed to encode map e362 with expected crc 2023-10-09T10:59:21.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:21 smithi125 ceph-mon[200197]: failed to encode map e362 with expected crc 2023-10-09T10:59:21.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:21 smithi125 ceph-mon[200197]: failed to encode map e362 with expected crc 2023-10-09T10:59:21.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:21 smithi125 ceph-mon[200197]: failed to encode map e362 with expected crc 2023-10-09T10:59:21.806 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:21.876 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:21.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:21.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:21.885 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T10:59:21.981 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:22.012 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:22.098 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:59:22.119 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:59:22.137 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:59:22.156 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:59:22.171 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:59:22.175 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:59:22.193 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:59:22.211 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:59:22.211 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:59:22.211 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-10-09T10:59:22.212 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:59:22.212 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:59:22.212 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:59:22.212 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:59:22.212 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:59:22.212 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:59:22.433 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:22.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:22.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:22.539 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:22.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:22 smithi155 ceph-mon[184854]: failed to encode map e363 with expected crc 2023-10-09T10:59:22.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:22 smithi155 ceph-mon[184854]: osdmap e363: 8 total, 7 up, 7 in 2023-10-09T10:59:22.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:22 smithi155 ceph-mon[184854]: failed to encode map e363 with expected crc 2023-10-09T10:59:22.571 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:22 smithi155 ceph-mon[184854]: failed to encode map e363 with expected crc 2023-10-09T10:59:22.571 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:22 smithi155 ceph-mon[184854]: failed to encode map e363 with expected crc 2023-10-09T10:59:22.571 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:22 smithi155 ceph-mon[184854]: failed to encode map e363 with expected crc 2023-10-09T10:59:22.571 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:22 smithi155 ceph-mon[184854]: pgmap v908: 119 pgs: 9 unknown, 29 peering, 11 active+undersized+degraded, 32 active+undersized, 38 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 11 KiB/s rd, 0 B/s wr, 121 op/s; 49135/246390 objects degraded (19.942%); 22 KiB/s, 0 objects/s recovering 2023-10-09T10:59:22.571 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:22 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1661248426' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:22.573 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:22.611 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:22.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:22.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:22.679 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[195507]: failed to encode map e363 with expected crc 2023-10-09T10:59:22.680 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[195507]: osdmap e363: 8 total, 7 up, 7 in 2023-10-09T10:59:22.680 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[195507]: failed to encode map e363 with expected crc 2023-10-09T10:59:22.680 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[195507]: failed to encode map e363 with expected crc 2023-10-09T10:59:22.680 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[195507]: failed to encode map e363 with expected crc 2023-10-09T10:59:22.680 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[195507]: failed to encode map e363 with expected crc 2023-10-09T10:59:22.680 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[195507]: pgmap v908: 119 pgs: 9 unknown, 29 peering, 11 active+undersized+degraded, 32 active+undersized, 38 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 11 KiB/s rd, 0 B/s wr, 121 op/s; 49135/246390 objects degraded (19.942%); 22 KiB/s, 0 objects/s recovering 2023-10-09T10:59:22.680 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1661248426' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:22.680 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[200197]: failed to encode map e363 with expected crc 2023-10-09T10:59:22.680 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[200197]: osdmap e363: 8 total, 7 up, 7 in 2023-10-09T10:59:22.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[200197]: failed to encode map e363 with expected crc 2023-10-09T10:59:22.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[200197]: failed to encode map e363 with expected crc 2023-10-09T10:59:22.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[200197]: failed to encode map e363 with expected crc 2023-10-09T10:59:22.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[200197]: failed to encode map e363 with expected crc 2023-10-09T10:59:22.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[200197]: pgmap v908: 119 pgs: 9 unknown, 29 peering, 11 active+undersized+degraded, 32 active+undersized, 38 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 11 KiB/s rd, 0 B/s wr, 121 op/s; 49135/246390 objects degraded (19.942%); 22 KiB/s, 0 objects/s recovering 2023-10-09T10:59:22.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:22 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1661248426' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:22.715 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:22.729 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:59:22.730 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:59:22.763 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:22.843 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:22.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:22.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:22.949 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:22.992 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:23.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:23.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:23.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:23.141 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:23.181 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:23.210 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:23.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:23.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:23.314 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:23.395 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:23.399 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:59:23.399 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:59:23.428 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:23.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:23.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:23.536 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:23.578 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:23.613 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:23.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:23.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:23.717 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:23.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:23 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 14 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2023-10-09T10:59:23.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:23 smithi155 ceph-mon[184854]: failed to encode map e364 with expected crc 2023-10-09T10:59:23.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:23 smithi155 ceph-mon[184854]: failed to encode map e364 with expected crc 2023-10-09T10:59:23.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:23 smithi155 ceph-mon[184854]: osdmap e364: 8 total, 7 up, 7 in 2023-10-09T10:59:23.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:23 smithi155 ceph-mon[184854]: failed to encode map e364 with expected crc 2023-10-09T10:59:23.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:23 smithi155 ceph-mon[184854]: failed to encode map e364 with expected crc 2023-10-09T10:59:23.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:23 smithi155 ceph-mon[184854]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 361'8214 2023-10-09T10:59:23.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:23 smithi155 ceph-mon[184854]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 361'8236 2023-10-09T10:59:23.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:23 smithi155 ceph-mon[184854]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 361'8437 2023-10-09T10:59:23.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:23 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 49135/246390 objects degraded (19.942%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:23.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:23 smithi155 ceph-mon[184854]: failed to encode map e365 with expected crc 2023-10-09T10:59:23.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:23 smithi155 ceph-mon[184854]: osdmap e365: 8 total, 7 up, 7 in 2023-10-09T10:59:23.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:23 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1537192268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:59:23.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:23 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:59:23.791 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:23.808 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T10:59:23.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 14 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2023-10-09T10:59:23.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[195507]: failed to encode map e364 with expected crc 2023-10-09T10:59:23.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[195507]: failed to encode map e364 with expected crc 2023-10-09T10:59:23.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[195507]: osdmap e364: 8 total, 7 up, 7 in 2023-10-09T10:59:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[195507]: failed to encode map e364 with expected crc 2023-10-09T10:59:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[195507]: failed to encode map e364 with expected crc 2023-10-09T10:59:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[195507]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 361'8214 2023-10-09T10:59:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[195507]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 361'8236 2023-10-09T10:59:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[195507]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 361'8437 2023-10-09T10:59:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 49135/246390 objects degraded (19.942%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[195507]: failed to encode map e365 with expected crc 2023-10-09T10:59:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[195507]: osdmap e365: 8 total, 7 up, 7 in 2023-10-09T10:59:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1537192268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:59:23.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:59:23.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 14 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2023-10-09T10:59:23.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[200197]: failed to encode map e364 with expected crc 2023-10-09T10:59:23.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[200197]: failed to encode map e364 with expected crc 2023-10-09T10:59:23.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[200197]: osdmap e364: 8 total, 7 up, 7 in 2023-10-09T10:59:23.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[200197]: failed to encode map e364 with expected crc 2023-10-09T10:59:23.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[200197]: failed to encode map e364 with expected crc 2023-10-09T10:59:23.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[200197]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 361'8214 2023-10-09T10:59:23.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[200197]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 361'8236 2023-10-09T10:59:23.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[200197]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 361'8437 2023-10-09T10:59:23.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 49135/246390 objects degraded (19.942%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:23.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[200197]: failed to encode map e365 with expected crc 2023-10-09T10:59:23.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[200197]: osdmap e365: 8 total, 7 up, 7 in 2023-10-09T10:59:23.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1537192268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:59:23.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:23 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:59:23.830 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:23.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:23.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:23.939 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:23.979 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:24.013 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:24.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:24.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:24.119 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:24.132 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:59:24.183 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:24.239 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:24.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:24.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:24.280 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:59:24.348 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:24.398 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:24.529 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:24.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:24.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:24.635 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:24.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[200197]: failed to encode map e365 with expected crc 2023-10-09T10:59:24.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[200197]: failed to encode map e365 with expected crc 2023-10-09T10:59:24.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[200197]: failed to encode map e365 with expected crc 2023-10-09T10:59:24.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[200197]: failed to encode map e365 with expected crc 2023-10-09T10:59:24.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[200197]: failed to encode map e365 with expected crc 2023-10-09T10:59:24.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[200197]: pgmap v911: 119 pgs: 1 active+remapped+backfilling, 6 unknown, 29 peering, 1 active+undersized+remapped, 11 active+undersized+degraded, 32 active+undersized, 2 active+remapped, 37 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 1.04k op/s; 49135/262656 objects degraded (18.707%); 5409/262656 objects misplaced (2.059%); 263 KiB/s, 1 objects/s recovering 2023-10-09T10:59:24.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:24.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:59:24.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[200197]: failed to encode map e366 with expected crc 2023-10-09T10:59:24.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[200197]: osdmap e366: 8 total, 7 up, 7 in 2023-10-09T10:59:24.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[195507]: failed to encode map e365 with expected crc 2023-10-09T10:59:24.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[195507]: failed to encode map e365 with expected crc 2023-10-09T10:59:24.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[195507]: failed to encode map e365 with expected crc 2023-10-09T10:59:24.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[195507]: failed to encode map e365 with expected crc 2023-10-09T10:59:24.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[195507]: failed to encode map e365 with expected crc 2023-10-09T10:59:24.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[195507]: pgmap v911: 119 pgs: 1 active+remapped+backfilling, 6 unknown, 29 peering, 1 active+undersized+remapped, 11 active+undersized+degraded, 32 active+undersized, 2 active+remapped, 37 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 1.04k op/s; 49135/262656 objects degraded (18.707%); 5409/262656 objects misplaced (2.059%); 263 KiB/s, 1 objects/s recovering 2023-10-09T10:59:24.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:24.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:59:24.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[195507]: failed to encode map e366 with expected crc 2023-10-09T10:59:24.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:24 smithi125 ceph-mon[195507]: osdmap e366: 8 total, 7 up, 7 in 2023-10-09T10:59:24.676 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:24 smithi155 ceph-mon[184854]: failed to encode map e365 with expected crc 2023-10-09T10:59:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:24 smithi155 ceph-mon[184854]: failed to encode map e365 with expected crc 2023-10-09T10:59:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:24 smithi155 ceph-mon[184854]: failed to encode map e365 with expected crc 2023-10-09T10:59:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:24 smithi155 ceph-mon[184854]: failed to encode map e365 with expected crc 2023-10-09T10:59:24.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:24 smithi155 ceph-mon[184854]: failed to encode map e365 with expected crc 2023-10-09T10:59:24.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:24 smithi155 ceph-mon[184854]: pgmap v911: 119 pgs: 1 active+remapped+backfilling, 6 unknown, 29 peering, 1 active+undersized+remapped, 11 active+undersized+degraded, 32 active+undersized, 2 active+remapped, 37 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 1.04k op/s; 49135/262656 objects degraded (18.707%); 5409/262656 objects misplaced (2.059%); 263 KiB/s, 1 objects/s recovering 2023-10-09T10:59:24.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:24 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:24.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:24 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:59:24.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:24 smithi155 ceph-mon[184854]: failed to encode map e366 with expected crc 2023-10-09T10:59:24.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:24 smithi155 ceph-mon[184854]: osdmap e366: 8 total, 7 up, 7 in 2023-10-09T10:59:24.731 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:59:24.755 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:24.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:24.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:24.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:24.923 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:25.064 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:25.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:25.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:25.127 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:25.170 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:25.217 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:25.317 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:25.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:25.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:25.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:25.513 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:25.553 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:25.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:25.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:25.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[195507]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[195507]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[195507]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[195507]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[195507]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[195507]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[195507]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[195507]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[200197]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[200197]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[200197]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[200197]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[200197]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[200197]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[200197]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:25 smithi125 ceph-mon[200197]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.595 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:59:25.595 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:59:25.629 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:25 smithi155 ceph-mon[184854]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.629 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:25 smithi155 ceph-mon[184854]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.629 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:25 smithi155 ceph-mon[184854]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.629 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:25 smithi155 ceph-mon[184854]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.629 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:25 smithi155 ceph-mon[184854]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.629 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:25 smithi155 ceph-mon[184854]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.629 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:25 smithi155 ceph-mon[184854]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.629 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:25 smithi155 ceph-mon[184854]: failed to encode map e366 with expected crc 2023-10-09T10:59:25.659 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:25.711 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:25.744 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:25.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:25.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:25.851 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:25.904 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:26.197 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 7] out_osds: [0] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-10-09T10:59:26.197 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:59:26.198 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2023-10-09T10:59:26.198 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T10:59:26.270 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:59:26.312 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:59:26.363 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:59:26.390 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:59:26.423 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:59:26.464 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:59:26.504 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:59:26.504 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:59:26.504 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-10-09T10:59:26.504 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:59:26.504 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:59:26.504 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:59:26.505 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:59:26.505 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:59:26.505 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:59:26.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:26 smithi125 ceph-mon[195507]: pgmap v913: 119 pgs: 1 active+remapped+backfilling, 6 unknown, 29 peering, 1 active+undersized+remapped, 11 active+undersized+degraded, 32 active+undersized, 2 active+remapped, 37 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 3.4 KiB/s rd, 103 MiB/s wr, 1.99k op/s; 49135/262656 objects degraded (18.707%); 5409/262656 objects misplaced (2.059%); 223 KiB/s, 3 objects/s recovering 2023-10-09T10:59:26.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:26 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1035326736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:26.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:26 smithi125 ceph-mon[200197]: pgmap v913: 119 pgs: 1 active+remapped+backfilling, 6 unknown, 29 peering, 1 active+undersized+remapped, 11 active+undersized+degraded, 32 active+undersized, 2 active+remapped, 37 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 3.4 KiB/s rd, 103 MiB/s wr, 1.99k op/s; 49135/262656 objects degraded (18.707%); 5409/262656 objects misplaced (2.059%); 223 KiB/s, 3 objects/s recovering 2023-10-09T10:59:26.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:26 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1035326736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:26.773 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:59:26.890 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:59:26.891 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:59:26.912 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:26.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:26 smithi155 ceph-mon[184854]: pgmap v913: 119 pgs: 1 active+remapped+backfilling, 6 unknown, 29 peering, 1 active+undersized+remapped, 11 active+undersized+degraded, 32 active+undersized, 2 active+remapped, 37 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 3.4 KiB/s rd, 103 MiB/s wr, 1.99k op/s; 49135/262656 objects degraded (18.707%); 5409/262656 objects misplaced (2.059%); 223 KiB/s, 3 objects/s recovering 2023-10-09T10:59:26.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:26 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1035326736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:27.167 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:27.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:27.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:27.285 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:27.327 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:27.357 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:27.358 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:27.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:27.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:27.463 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:27.501 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T10:59:27.544 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:27.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:27.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:27.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:27.688 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:27.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:27 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 14 pgs inactive, 11 pgs peering) 2023-10-09T10:59:27.742 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:27.790 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:27.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:27.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:27.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:27.959 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:27 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 14 pgs inactive, 11 pgs peering) 2023-10-09T10:59:27.959 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:27 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 14 pgs inactive, 11 pgs peering) 2023-10-09T10:59:27.962 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:27.984 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:59:28.094 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:28.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:28.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:28.216 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:28.282 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:28.316 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:28.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:28.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:28.420 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T10:59:28.422 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:28.434 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T10:59:28.480 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:28.567 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:28.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:28.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:28.670 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:28.718 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:28.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:28 smithi125 ceph-mon[195507]: pgmap v914: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 2.3 KiB/s rd, 71 MiB/s wr, 1.89k op/s; 47313/253332 objects degraded (18.676%); 20725/253332 objects misplaced (8.181%); 7.2 MiB/s, 116 objects/s recovering 2023-10-09T10:59:28.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:28 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 47313/253332 objects degraded (18.676%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:28.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:28 smithi125 ceph-mon[200197]: pgmap v914: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 2.3 KiB/s rd, 71 MiB/s wr, 1.89k op/s; 47313/253332 objects degraded (18.676%); 20725/253332 objects misplaced (8.181%); 7.2 MiB/s, 116 objects/s recovering 2023-10-09T10:59:28.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:28 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 47313/253332 objects degraded (18.676%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:28.893 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:59:28.953 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:28.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:28.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:28.969 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":914,"stamp":"2023-10-09T10:59:27.587280+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5533125757,"num_objects":84444,"num_object_clones":0,"num_object_copies":253332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47313,"num_objects_misplaced":20725,"num_objects_unfound":0,"num_objects_dirty":84444,"num_whiteouts":0,"num_read":94,"num_read_kb":174,"num_write":136282,"num_write_kb":7059432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":757,"num_bytes_recovered":46950199,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95962,"ondisk_log_size":95962,"up":293,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":20399820,"kb_used_data":15709204,"kb_used_omap":98,"kb_used_meta":4690461,"kb_avail":635816244,"statfs":{"total":671965249536,"available":651075833856,"internally_reserved":0,"allocated":16086224896,"data_stored":16075809386,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":100565,"internal_metadata":4803032875},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,9],"upper_bound":4096},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":151659249,"num_objects":2314,"num_object_clones":0,"num_object_copies":6942,"num_objects_missing_on_primary":0,"num_objects_missing":-4,"num_objects_degraded":-1822,"num_objects_misplaced":20725,"num_objects_unfound":0,"num_objects_dirty":2314,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":11356,"num_write_kb":437349,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":700,"num_bytes_recovered":45574909,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001803"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":723,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.111945+0000","last_change":"2023-10-09T10:57:25.104150+0000","last_active":"2023-10-09T10:59:23.111945+0000","last_peered":"2023-10-09T10:59:23.111945+0000","last_clean":"2023-10-09T10:59:23.111945+0000","last_became_active":"2023-10-09T10:57:25.103762+0000","last_became_peered":"2023-10-09T10:57:25.103762+0000","last_unstale":"2023-10-09T10:59:23.111945+0000","last_undegraded":"2023-10-09T10:59:23.111945+0000","last_fullsized":"2023-10-09T10:59:23.111945+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:40.174975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:40.174975+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:35.416780+0000","scrub_duration":0.010654205,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":78,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.112651+0000","last_change":"2023-10-09T10:59:09.170612+0000","last_active":"2023-10-09T10:59:23.112651+0000","last_peered":"2023-10-09T10:59:23.112651+0000","last_clean":"2023-10-09T10:59:05.160632+0000","last_became_active":"2023-10-09T10:59:09.170612+0000","last_became_peered":"2023-10-09T10:59:09.170612+0000","last_unstale":"2023-10-09T10:59:23.112651+0000","last_undegraded":"2023-10-09T10:59:23.112651+0000","last_fullsized":"2023-10-09T10:59:09.162993+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:31.740032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":381,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.114023+0000","last_change":"2023-10-09T10:59:09.171707+0000","last_active":"2023-10-09T10:59:23.114023+0000","last_peered":"2023-10-09T10:59:23.114023+0000","last_clean":"2023-10-09T10:59:05.150443+0000","last_became_active":"2023-10-09T10:59:09.171707+0000","last_became_peered":"2023-10-09T10:59:09.171707+0000","last_unstale":"2023-10-09T10:59:23.114023+0000","last_undegraded":"2023-10-09T10:59:23.114023+0000","last_fullsized":"2023-10-09T10:59:09.163026+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:23:07.265028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":756,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T10:59:24.137744+0000","last_change":"2023-10-09T10:59:24.137744+0000","last_active":"2023-10-09T10:59:24.137744+0000","last_peered":"2023-10-09T10:59:24.137744+0000","last_clean":"2023-10-09T10:59:24.137744+0000","last_became_active":"2023-10-09T10:59:24.137049+0000","last_became_peered":"2023-10-09T10:59:24.137049+0000","last_unstale":"2023-10-09T10:59:24.137744+0000","last_undegraded":"2023-10-09T10:59:24.137744+0000","last_fullsized":"2023-10-09T10:59:24.137744+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.536962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.536962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:33:02.807718+0000","scrub_duration":0.0053325960000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":126,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.116578+0000","last_change":"2023-10-09T10:59:09.176078+0000","last_active":"2023-10-09T10:59:23.116578+0000","last_peered":"2023-10-09T10:59:23.116578+0000","last_clean":"2023-10-09T10:54:45.106363+0000","last_became_active":"2023-10-09T10:59:09.176078+0000","last_became_peered":"2023-10-09T10:59:09.176078+0000","last_unstale":"2023-10-09T10:59:23.116578+0000","last_undegraded":"2023-10-09T10:59:23.116578+0000","last_fullsized":"2023-10-09T10:59:09.165452+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:45.447880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":269,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.148167+0000","last_change":"2023-10-09T10:59:22.469297+0000","last_active":"2023-10-09T10:59:23.148167+0000","last_peered":"2023-10-09T10:59:23.148167+0000","last_clean":"2023-10-09T10:59:23.148167+0000","last_became_active":"2023-10-09T10:59:22.468929+0000","last_became_peered":"2023-10-09T10:59:22.468929+0000","last_unstale":"2023-10-09T10:59:23.148167+0000","last_undegraded":"2023-10-09T10:59:23.148167+0000","last_fullsized":"2023-10-09T10:59:23.148167+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:41.481405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3,0],"avail_no_missing":["7","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":796,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.127710+0000","last_change":"2023-10-09T10:59:09.173477+0000","last_active":"2023-10-09T10:59:23.127710+0000","last_peered":"2023-10-09T10:59:23.127710+0000","last_clean":"2023-10-09T10:59:05.146175+0000","last_became_active":"2023-10-09T10:59:09.173477+0000","last_became_peered":"2023-10-09T10:59:09.173477+0000","last_unstale":"2023-10-09T10:59:23.127710+0000","last_undegraded":"2023-10-09T10:59:23.127710+0000","last_fullsized":"2023-10-09T10:59:09.162641+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:07:39.524173+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":436,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.114325+0000","last_change":"2023-10-09T10:59:22.474880+0000","last_active":"2023-10-09T10:59:23.114325+0000","last_peered":"2023-10-09T10:59:23.114325+0000","last_clean":"2023-10-09T10:59:23.114325+0000","last_became_active":"2023-10-09T10:59:22.474452+0000","last_became_peered":"2023-10-09T10:59:22.474452+0000","last_unstale":"2023-10-09T10:59:23.114325+0000","last_undegraded":"2023-10-09T10:59:23.114325+0000","last_fullsized":"2023-10-09T10:59:23.114325+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:12.058089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,0],"avail_no_missing":["3","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":190,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.122482+0000","last_change":"2023-10-09T10:59:09.179588+0000","last_active":"2023-10-09T10:59:23.122482+0000","last_peered":"2023-10-09T10:59:23.122482+0000","last_clean":"2023-10-09T10:54:43.542439+0000","last_became_active":"2023-10-09T10:59:09.179588+0000","last_became_peered":"2023-10-09T10:59:09.179588+0000","last_unstale":"2023-10-09T10:59:23.122482+0000","last_undegraded":"2023-10-09T10:59:23.122482+0000","last_fullsized":"2023-10-09T10:59:09.165239+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:42:08.656298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":788,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.148018+0000","last_change":"2023-10-09T10:59:09.179420+0000","last_active":"2023-10-09T10:59:23.148018+0000","last_peered":"2023-10-09T10:59:23.148018+0000","last_clean":"2023-10-09T10:59:05.156788+0000","last_became_active":"2023-10-09T10:59:09.179420+0000","last_became_peered":"2023-10-09T10:59:09.179420+0000","last_unstale":"2023-10-09T10:59:23.148018+0000","last_undegraded":"2023-10-09T10:59:23.148018+0000","last_fullsized":"2023-10-09T10:59:09.163549+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.787782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.787782+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:17.634131+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":62,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.115147+0000","last_change":"2023-10-09T10:59:21.466007+0000","last_active":"2023-10-09T10:59:23.115147+0000","last_peered":"2023-10-09T10:59:23.115147+0000","last_clean":"2023-10-09T10:59:23.115147+0000","last_became_active":"2023-10-09T10:59:21.465256+0000","last_became_peered":"2023-10-09T10:59:21.465256+0000","last_unstale":"2023-10-09T10:59:23.115147+0000","last_undegraded":"2023-10-09T10:59:23.115147+0000","last_fullsized":"2023-10-09T10:59:23.115147+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:50.590097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":381,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.114463+0000","last_change":"2023-10-09T10:59:09.182061+0000","last_active":"2023-10-09T10:59:23.114463+0000","last_peered":"2023-10-09T10:59:23.114463+0000","last_clean":"2023-10-09T10:59:05.150819+0000","last_became_active":"2023-10-09T10:59:09.182061+0000","last_became_peered":"2023-10-09T10:59:09.182061+0000","last_unstale":"2023-10-09T10:59:23.114463+0000","last_undegraded":"2023-10-09T10:59:23.114463+0000","last_fullsized":"2023-10-09T10:59:09.162791+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:21.026502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":658,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.112193+0000","last_change":"2023-10-09T10:57:25.103739+0000","last_active":"2023-10-09T10:59:23.112193+0000","last_peered":"2023-10-09T10:59:23.112193+0000","last_clean":"2023-10-09T10:59:23.112193+0000","last_became_active":"2023-10-09T10:57:25.103378+0000","last_became_peered":"2023-10-09T10:57:25.103378+0000","last_unstale":"2023-10-09T10:59:23.112193+0000","last_undegraded":"2023-10-09T10:59:23.112193+0000","last_fullsized":"2023-10-09T10:59:23.112193+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:49:50.889945+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":50,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.148482+0000","last_change":"2023-10-09T10:59:09.185142+0000","last_active":"2023-10-09T10:59:23.148482+0000","last_peered":"2023-10-09T10:59:23.148482+0000","last_clean":"2023-10-09T10:59:05.157338+0000","last_became_active":"2023-10-09T10:59:09.185142+0000","last_became_peered":"2023-10-09T10:59:09.185142+0000","last_unstale":"2023-10-09T10:59:23.148482+0000","last_undegraded":"2023-10-09T10:59:23.148482+0000","last_fullsized":"2023-10-09T10:59:09.163299+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:44.806417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":332,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.116478+0000","last_change":"2023-10-09T10:57:22.010202+0000","last_active":"2023-10-09T10:59:23.116478+0000","last_peered":"2023-10-09T10:59:23.116478+0000","last_clean":"2023-10-09T10:59:23.116478+0000","last_became_active":"2023-10-09T10:57:22.006867+0000","last_became_peered":"2023-10-09T10:57:22.006867+0000","last_unstale":"2023-10-09T10:59:23.116478+0000","last_undegraded":"2023-10-09T10:59:23.116478+0000","last_fullsized":"2023-10-09T10:59:23.116478+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:54.212660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":765,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T10:59:24.156439+0000","last_change":"2023-10-09T10:59:24.156439+0000","last_active":"2023-10-09T10:59:24.156439+0000","last_peered":"2023-10-09T10:59:24.156439+0000","last_clean":"2023-10-09T10:59:24.156439+0000","last_became_active":"2023-10-09T10:59:24.156108+0000","last_became_peered":"2023-10-09T10:59:24.156108+0000","last_unstale":"2023-10-09T10:59:24.156439+0000","last_undegraded":"2023-10-09T10:59:24.156439+0000","last_fullsized":"2023-10-09T10:59:24.156439+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:32.540610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:32.540610+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:37:25.995205+0000","scrub_duration":0.0064251300000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":49,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.148611+0000","last_change":"2023-10-09T10:57:22.032392+0000","last_active":"2023-10-09T10:59:23.148611+0000","last_peered":"2023-10-09T10:59:23.148611+0000","last_clean":"2023-10-09T10:59:23.148611+0000","last_became_active":"2023-10-09T10:57:22.012620+0000","last_became_peered":"2023-10-09T10:57:22.012620+0000","last_unstale":"2023-10-09T10:59:23.148611+0000","last_undegraded":"2023-10-09T10:59:23.148611+0000","last_fullsized":"2023-10-09T10:59:23.148611+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:06:34.979769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":59,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.112857+0000","last_change":"2023-10-09T10:57:46.838152+0000","last_active":"2023-10-09T10:59:23.112857+0000","last_peered":"2023-10-09T10:59:23.112857+0000","last_clean":"2023-10-09T10:59:23.112857+0000","last_became_active":"2023-10-09T10:57:46.837933+0000","last_became_peered":"2023-10-09T10:57:46.837933+0000","last_unstale":"2023-10-09T10:59:23.112857+0000","last_undegraded":"2023-10-09T10:59:23.112857+0000","last_fullsized":"2023-10-09T10:59:23.112857+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:26:23.218941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":555,"reported_epoch":366,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:24.131720+0000","last_change":"2023-10-09T10:59:24.131720+0000","last_active":"2023-10-09T10:59:24.131720+0000","last_peered":"2023-10-09T10:59:24.131720+0000","last_clean":"2023-10-09T10:59:24.131720+0000","last_became_active":"2023-10-09T10:59:24.131269+0000","last_became_peered":"2023-10-09T10:59:24.131269+0000","last_unstale":"2023-10-09T10:59:24.131720+0000","last_undegraded":"2023-10-09T10:59:24.131720+0000","last_fullsized":"2023-10-09T10:59:24.131720+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:18.113848+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.113848+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,0],"avail_no_missing":["3","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":53,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.151363+0000","last_change":"2023-10-09T10:57:22.039995+0000","last_active":"2023-10-09T10:59:23.151363+0000","last_peered":"2023-10-09T10:59:23.151363+0000","last_clean":"2023-10-09T10:59:23.151363+0000","last_became_active":"2023-10-09T10:57:22.016287+0000","last_became_peered":"2023-10-09T10:57:22.016287+0000","last_unstale":"2023-10-09T10:59:23.151363+0000","last_undegraded":"2023-10-09T10:59:23.151363+0000","last_fullsized":"2023-10-09T10:59:23.151363+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:52.167501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":380,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.129225+0000","last_change":"2023-10-09T10:57:46.845755+0000","last_active":"2023-10-09T10:59:23.129225+0000","last_peered":"2023-10-09T10:59:23.129225+0000","last_clean":"2023-10-09T10:59:23.129225+0000","last_became_active":"2023-10-09T10:57:46.845475+0000","last_became_peered":"2023-10-09T10:57:46.845475+0000","last_unstale":"2023-10-09T10:59:23.129225+0000","last_undegraded":"2023-10-09T10:59:23.129225+0000","last_fullsized":"2023-10-09T10:59:23.129225+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:21.673994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":547,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.115990+0000","last_change":"2023-10-09T10:59:22.467776+0000","last_active":"2023-10-09T10:59:23.115990+0000","last_peered":"2023-10-09T10:59:23.115990+0000","last_clean":"2023-10-09T10:59:23.115990+0000","last_became_active":"2023-10-09T10:59:22.467320+0000","last_became_peered":"2023-10-09T10:59:22.467320+0000","last_unstale":"2023-10-09T10:59:23.115990+0000","last_undegraded":"2023-10-09T10:59:23.115990+0000","last_fullsized":"2023-10-09T10:59:23.115990+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,0],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":49,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.150110+0000","last_change":"2023-10-09T10:57:22.026809+0000","last_active":"2023-10-09T10:59:23.150110+0000","last_peered":"2023-10-09T10:59:23.150110+0000","last_clean":"2023-10-09T10:59:23.150110+0000","last_became_active":"2023-10-09T10:57:22.011600+0000","last_became_peered":"2023-10-09T10:57:22.011600+0000","last_unstale":"2023-10-09T10:59:23.150110+0000","last_undegraded":"2023-10-09T10:59:23.150110+0000","last_fullsized":"2023-10-09T10:59:23.150110+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:10.453811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":449,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.116058+0000","last_change":"2023-10-09T10:59:09.174962+0000","last_active":"2023-10-09T10:59:23.116058+0000","last_peered":"2023-10-09T10:59:23.116058+0000","last_clean":"2023-10-09T10:59:05.155931+0000","last_became_active":"2023-10-09T10:59:09.174962+0000","last_became_peered":"2023-10-09T10:59:09.174962+0000","last_unstale":"2023-10-09T10:59:23.116058+0000","last_undegraded":"2023-10-09T10:59:23.116058+0000","last_fullsized":"2023-10-09T10:59:09.164355+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:13:57.668931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":685,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.147987+0000","last_change":"2023-10-09T10:57:25.098905+0000","last_active":"2023-10-09T10:59:23.147987+0000","last_peered":"2023-10-09T10:59:23.147987+0000","last_clean":"2023-10-09T10:59:23.147987+0000","last_became_active":"2023-10-09T10:57:25.098562+0000","last_became_peered":"2023-10-09T10:57:25.098562+0000","last_unstale":"2023-10-09T10:59:23.147987+0000","last_undegraded":"2023-10-09T10:59:23.147987+0000","last_fullsized":"2023-10-09T10:59:23.147987+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:23.020232+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":268,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.150791+0000","last_change":"2023-10-09T10:59:22.461078+0000","last_active":"2023-10-09T10:59:23.150791+0000","last_peered":"2023-10-09T10:59:23.150791+0000","last_clean":"2023-10-09T10:59:23.150791+0000","last_became_active":"2023-10-09T10:59:22.460724+0000","last_became_peered":"2023-10-09T10:59:22.460724+0000","last_unstale":"2023-10-09T10:59:23.150791+0000","last_undegraded":"2023-10-09T10:59:23.150791+0000","last_fullsized":"2023-10-09T10:59:23.150791+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:39.948055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3,0],"avail_no_missing":["7","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":43,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.151507+0000","last_change":"2023-10-09T10:57:22.014923+0000","last_active":"2023-10-09T10:59:23.151507+0000","last_peered":"2023-10-09T10:59:23.151507+0000","last_clean":"2023-10-09T10:59:23.151507+0000","last_became_active":"2023-10-09T10:57:22.014055+0000","last_became_peered":"2023-10-09T10:57:22.014055+0000","last_unstale":"2023-10-09T10:59:23.151507+0000","last_undegraded":"2023-10-09T10:59:23.151507+0000","last_fullsized":"2023-10-09T10:59:23.151507+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:50.546435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":633,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.118150+0000","last_change":"2023-10-09T10:59:09.173447+0000","last_active":"2023-10-09T10:59:23.118150+0000","last_peered":"2023-10-09T10:59:23.118150+0000","last_clean":"2023-10-09T10:54:45.111469+0000","last_became_active":"2023-10-09T10:59:09.173447+0000","last_became_peered":"2023-10-09T10:59:09.173447+0000","last_unstale":"2023-10-09T10:59:23.118150+0000","last_undegraded":"2023-10-09T10:59:23.118150+0000","last_fullsized":"2023-10-09T10:59:09.164621+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:44.843520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:44.843520+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":43,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.153570+0000","last_change":"2023-10-09T10:57:22.016705+0000","last_active":"2023-10-09T10:59:23.153570+0000","last_peered":"2023-10-09T10:59:23.153570+0000","last_clean":"2023-10-09T10:59:23.153570+0000","last_became_active":"2023-10-09T10:57:22.016452+0000","last_became_peered":"2023-10-09T10:57:22.016452+0000","last_unstale":"2023-10-09T10:59:23.153570+0000","last_undegraded":"2023-10-09T10:59:23.153570+0000","last_fullsized":"2023-10-09T10:59:23.153570+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:43:44.382821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":247,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.117670+0000","last_change":"2023-10-09T10:59:21.465109+0000","last_active":"2023-10-09T10:59:23.117670+0000","last_peered":"2023-10-09T10:59:23.117670+0000","last_clean":"2023-10-09T10:59:23.117670+0000","last_became_active":"2023-10-09T10:59:21.464721+0000","last_became_peered":"2023-10-09T10:59:21.464721+0000","last_unstale":"2023-10-09T10:59:23.117670+0000","last_undegraded":"2023-10-09T10:59:23.117670+0000","last_fullsized":"2023-10-09T10:59:23.117670+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:38.604601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":639,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.129268+0000","last_change":"2023-10-09T10:59:09.180054+0000","last_active":"2023-10-09T10:59:23.129268+0000","last_peered":"2023-10-09T10:59:23.129268+0000","last_clean":"2023-10-09T10:51:40.583631+0000","last_became_active":"2023-10-09T10:59:09.180054+0000","last_became_peered":"2023-10-09T10:59:09.180054+0000","last_unstale":"2023-10-09T10:59:23.129268+0000","last_undegraded":"2023-10-09T10:59:23.129268+0000","last_fullsized":"2023-10-09T10:59:09.176820+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:02.893884+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":24,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.113699+0000","last_change":"2023-10-09T10:59:21.473714+0000","last_active":"2023-10-09T10:59:23.113699+0000","last_peered":"2023-10-09T10:59:23.113699+0000","last_clean":"2023-10-09T10:59:23.113699+0000","last_became_active":"2023-10-09T10:59:21.464540+0000","last_became_peered":"2023-10-09T10:59:21.464540+0000","last_unstale":"2023-10-09T10:59:23.113699+0000","last_undegraded":"2023-10-09T10:59:23.113699+0000","last_fullsized":"2023-10-09T10:59:23.113699+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:45:34.753232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":426,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.115466+0000","last_change":"2023-10-09T10:59:09.173756+0000","last_active":"2023-10-09T10:59:23.115466+0000","last_peered":"2023-10-09T10:59:23.115466+0000","last_clean":"2023-10-09T10:59:05.151031+0000","last_became_active":"2023-10-09T10:59:09.173756+0000","last_became_peered":"2023-10-09T10:59:09.173756+0000","last_unstale":"2023-10-09T10:59:23.115466+0000","last_undegraded":"2023-10-09T10:59:23.115466+0000","last_fullsized":"2023-10-09T10:59:09.165417+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:45.293132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"366'8602","reported_seq":8667,"reported_epoch":366,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:26.227684+0000","last_change":"2023-10-09T10:59:22.503396+0000","last_active":"2023-10-09T10:59:26.227684+0000","last_peered":"2023-10-09T10:59:26.227684+0000","last_clean":"2023-10-09T10:59:26.227684+0000","last_became_active":"2023-10-09T10:59:22.464405+0000","last_became_peered":"2023-10-09T10:59:22.464405+0000","last_unstale":"2023-10-09T10:59:26.227684+0000","last_undegraded":"2023-10-09T10:59:26.227684+0000","last_fullsized":"2023-10-09T10:59:26.227684+0000","mapping_epoch":363,"log_start":"344'3484","ondisk_log_start":"344'3484","created":330,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5118,"log_dups_size":0,"ondisk_log_size":5118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:04:55.382154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345899008,"num_objects":5278,"num_object_clones":0,"num_object_copies":15834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5278,"num_objects_unfound":0,"num_objects_dirty":5278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8602,"num_write_kb":444160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5,0],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":5278}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":713,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.148307+0000","last_change":"2023-10-09T10:59:09.174881+0000","last_active":"2023-10-09T10:59:23.148307+0000","last_peered":"2023-10-09T10:59:23.148307+0000","last_clean":"2023-10-09T10:59:05.156938+0000","last_became_active":"2023-10-09T10:59:09.174881+0000","last_became_peered":"2023-10-09T10:59:09.174881+0000","last_unstale":"2023-10-09T10:59:23.148307+0000","last_undegraded":"2023-10-09T10:59:23.148307+0000","last_fullsized":"2023-10-09T10:59:09.163792+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:59:21.140904+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":59,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.149331+0000","last_change":"2023-10-09T10:59:22.470440+0000","last_active":"2023-10-09T10:59:23.149331+0000","last_peered":"2023-10-09T10:59:23.149331+0000","last_clean":"2023-10-09T10:59:23.149331+0000","last_became_active":"2023-10-09T10:59:22.469299+0000","last_became_peered":"2023-10-09T10:59:22.469299+0000","last_unstale":"2023-10-09T10:59:23.149331+0000","last_undegraded":"2023-10-09T10:59:23.149331+0000","last_fullsized":"2023-10-09T10:59:23.149331+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:34.323284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4,0],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":43,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.147898+0000","last_change":"2023-10-09T10:57:22.016166+0000","last_active":"2023-10-09T10:59:23.147898+0000","last_peered":"2023-10-09T10:59:23.147898+0000","last_clean":"2023-10-09T10:59:23.147898+0000","last_became_active":"2023-10-09T10:57:22.015850+0000","last_became_peered":"2023-10-09T10:57:22.015850+0000","last_unstale":"2023-10-09T10:59:23.147898+0000","last_undegraded":"2023-10-09T10:59:23.147898+0000","last_fullsized":"2023-10-09T10:59:23.147898+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:37.827566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"366'8467","reported_seq":8517,"reported_epoch":366,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:59:25.659699+0000","last_change":"2023-10-09T10:59:09.175908+0000","last_active":"2023-10-09T10:59:25.659699+0000","last_peered":"2023-10-09T10:59:25.659699+0000","last_clean":"2023-10-09T10:59:08.112685+0000","last_became_active":"2023-10-09T10:59:09.175908+0000","last_became_peered":"2023-10-09T10:59:09.175908+0000","last_unstale":"2023-10-09T10:59:25.659699+0000","last_undegraded":"2023-10-09T10:59:09.164019+0000","last_fullsized":"2023-10-09T10:59:09.163928+0000","mapping_epoch":358,"log_start":"342'3053","ondisk_log_start":"342'3053","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5414,"log_dups_size":0,"ondisk_log_size":5414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:34:34.431472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350027776,"num_objects":5341,"num_object_clones":0,"num_object_copies":16023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5341,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8467,"num_write_kb":441856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":5341}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":748,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T10:59:24.131740+0000","last_change":"2023-10-09T10:59:24.131740+0000","last_active":"2023-10-09T10:59:24.131740+0000","last_peered":"2023-10-09T10:59:24.131740+0000","last_clean":"2023-10-09T10:59:24.131740+0000","last_became_active":"2023-10-09T10:59:24.131363+0000","last_became_peered":"2023-10-09T10:59:24.131363+0000","last_unstale":"2023-10-09T10:59:24.131740+0000","last_undegraded":"2023-10-09T10:59:24.131740+0000","last_fullsized":"2023-10-09T10:59:24.131740+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:07:41.404230+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":54,"reported_epoch":365,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:59:23.148807+0000","last_change":"2023-10-09T10:59:09.174698+0000","last_active":"2023-10-09T10:59:23.148807+0000","last_peered":"2023-10-09T10:59:23.148807+0000","last_clean":"2023-10-09T10:59:05.157584+0000","last_became_active":"2023-10-09T10:59:09.174698+0000","last_became_peered":"2023-10-09T10:59:09.174698+0000","last_unstale":"2023-10-09T10:59:23.148807+0000","last_undegraded":"2023-10-09T10:59:09.164099+0000","last_fullsized":"2023-10-09T10:59:09.163998+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:07:09.736720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":284,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.116295+0000","last_change":"2023-10-09T10:59:09.175772+0000","last_active":"2023-10-09T10:59:23.116295+0000","last_peered":"2023-10-09T10:59:23.116295+0000","last_clean":"2023-10-09T10:54:45.106040+0000","last_became_active":"2023-10-09T10:59:09.175772+0000","last_became_peered":"2023-10-09T10:59:09.175772+0000","last_unstale":"2023-10-09T10:59:23.116295+0000","last_undegraded":"2023-10-09T10:59:23.116295+0000","last_fullsized":"2023-10-09T10:59:09.164899+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:56.498553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.7","version":"366'8564","reported_seq":8607,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T10:59:26.258128+0000","last_change":"2023-10-09T10:57:22.024907+0000","last_active":"2023-10-09T10:59:26.258128+0000","last_peered":"2023-10-09T10:59:26.258128+0000","last_clean":"2023-10-09T10:59:26.258128+0000","last_became_active":"2023-10-09T10:57:22.024660+0000","last_became_peered":"2023-10-09T10:57:22.024660+0000","last_unstale":"2023-10-09T10:59:26.258128+0000","last_undegraded":"2023-10-09T10:59:26.258128+0000","last_fullsized":"2023-10-09T10:59:26.258128+0000","mapping_epoch":332,"log_start":"343'3445","ondisk_log_start":"343'3445","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5119,"log_dups_size":0,"ondisk_log_size":5119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:25:10.105154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340787200,"num_objects":5200,"num_object_clones":0,"num_object_copies":15600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8564,"num_write_kb":440448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":678,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.128200+0000","last_change":"2023-10-09T10:53:36.680150+0000","last_active":"2023-10-09T10:59:23.128200+0000","last_peered":"2023-10-09T10:59:23.128200+0000","last_clean":"2023-10-09T10:59:23.128200+0000","last_became_active":"2023-10-09T10:51:50.371032+0000","last_became_peered":"2023-10-09T10:51:50.371032+0000","last_unstale":"2023-10-09T10:59:23.128200+0000","last_undegraded":"2023-10-09T10:59:23.128200+0000","last_fullsized":"2023-10-09T10:59:23.128200+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.680090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.680090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:08:28.952827+0000","scrub_duration":0.005577172,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":47,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.151412+0000","last_change":"2023-10-09T10:57:22.034075+0000","last_active":"2023-10-09T10:59:23.151412+0000","last_peered":"2023-10-09T10:59:23.151412+0000","last_clean":"2023-10-09T10:59:23.151412+0000","last_became_active":"2023-10-09T10:57:22.015930+0000","last_became_peered":"2023-10-09T10:57:22.015930+0000","last_unstale":"2023-10-09T10:59:23.151412+0000","last_undegraded":"2023-10-09T10:59:23.151412+0000","last_fullsized":"2023-10-09T10:59:23.151412+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:16.494793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":76,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.150332+0000","last_change":"2023-10-09T10:59:09.170691+0000","last_active":"2023-10-09T10:59:23.150332+0000","last_peered":"2023-10-09T10:59:23.150332+0000","last_clean":"2023-10-09T10:57:19.981714+0000","last_became_active":"2023-10-09T10:59:09.170691+0000","last_became_peered":"2023-10-09T10:59:09.170691+0000","last_unstale":"2023-10-09T10:59:23.150332+0000","last_undegraded":"2023-10-09T10:59:23.150332+0000","last_fullsized":"2023-10-09T10:59:09.162256+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:58.858927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.6","version":"366'8423","reported_seq":8471,"reported_epoch":366,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:59:25.957088+0000","last_change":"2023-10-09T10:59:09.185548+0000","last_active":"2023-10-09T10:59:25.957088+0000","last_peered":"2023-10-09T10:59:25.957088+0000","last_clean":"2023-10-09T10:59:08.153092+0000","last_became_active":"2023-10-09T10:59:09.185548+0000","last_became_peered":"2023-10-09T10:59:09.185548+0000","last_unstale":"2023-10-09T10:59:25.957088+0000","last_undegraded":"2023-10-09T10:59:09.162554+0000","last_fullsized":"2023-10-09T10:59:09.162438+0000","mapping_epoch":358,"log_start":"336'308","ondisk_log_start":"336'308","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":8115,"log_dups_size":0,"ondisk_log_size":8115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:00:05.594162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338493440,"num_objects":5165,"num_object_clones":0,"num_object_copies":15495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5165,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8423,"num_write_kb":434816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":5165}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":851,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.132880+0000","last_change":"2023-10-09T10:57:25.111238+0000","last_active":"2023-10-09T10:59:23.132880+0000","last_peered":"2023-10-09T10:59:23.132880+0000","last_clean":"2023-10-09T10:59:23.132880+0000","last_became_active":"2023-10-09T10:57:25.110991+0000","last_became_peered":"2023-10-09T10:57:25.110991+0000","last_unstale":"2023-10-09T10:59:23.132880+0000","last_undegraded":"2023-10-09T10:59:23.132880+0000","last_fullsized":"2023-10-09T10:59:23.132880+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:57.060423+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":193,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.116291+0000","last_change":"2023-10-09T10:51:50.374527+0000","last_active":"2023-10-09T10:59:23.116291+0000","last_peered":"2023-10-09T10:59:23.116291+0000","last_clean":"2023-10-09T10:59:23.116291+0000","last_became_active":"2023-10-09T10:51:50.374297+0000","last_became_peered":"2023-10-09T10:51:50.374297+0000","last_unstale":"2023-10-09T10:59:23.116291+0000","last_undegraded":"2023-10-09T10:59:23.116291+0000","last_fullsized":"2023-10-09T10:59:23.116291+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:31:49.358307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":15,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.117680+0000","last_change":"2023-10-09T10:59:22.479402+0000","last_active":"2023-10-09T10:59:23.117680+0000","last_peered":"2023-10-09T10:59:23.117680+0000","last_clean":"2023-10-09T10:59:23.117680+0000","last_became_active":"2023-10-09T10:59:22.478968+0000","last_became_peered":"2023-10-09T10:59:22.478968+0000","last_unstale":"2023-10-09T10:59:23.117680+0000","last_undegraded":"2023-10-09T10:59:23.117680+0000","last_fullsized":"2023-10-09T10:59:23.117680+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:48:34.837708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,0],"avail_no_missing":["3","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1","version":"366'8609","reported_seq":8662,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T10:59:25.665626+0000","last_change":"2023-10-09T10:57:22.009934+0000","last_active":"2023-10-09T10:59:25.665626+0000","last_peered":"2023-10-09T10:59:25.665626+0000","last_clean":"2023-10-09T10:59:25.665626+0000","last_became_active":"2023-10-09T10:57:22.009682+0000","last_became_peered":"2023-10-09T10:57:22.009682+0000","last_unstale":"2023-10-09T10:59:25.665626+0000","last_undegraded":"2023-10-09T10:59:25.665626+0000","last_fullsized":"2023-10-09T10:59:25.665626+0000","mapping_epoch":332,"log_start":"343'3156","ondisk_log_start":"343'3156","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5453,"log_dups_size":0,"ondisk_log_size":5453,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:59.899734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346619904,"num_objects":5289,"num_object_clones":0,"num_object_copies":15867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8609,"num_write_kb":444736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":721,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.148183+0000","last_change":"2023-10-09T10:57:25.106422+0000","last_active":"2023-10-09T10:59:23.148183+0000","last_peered":"2023-10-09T10:59:23.148183+0000","last_clean":"2023-10-09T10:59:23.148183+0000","last_became_active":"2023-10-09T10:57:25.105998+0000","last_became_peered":"2023-10-09T10:57:25.105998+0000","last_unstale":"2023-10-09T10:59:23.148183+0000","last_undegraded":"2023-10-09T10:59:23.148183+0000","last_fullsized":"2023-10-09T10:59:23.148183+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:10:53.164214+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":374,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.117905+0000","last_change":"2023-10-09T10:59:22.479451+0000","last_active":"2023-10-09T10:59:23.117905+0000","last_peered":"2023-10-09T10:59:23.117905+0000","last_clean":"2023-10-09T10:59:23.117905+0000","last_became_active":"2023-10-09T10:59:22.478963+0000","last_became_peered":"2023-10-09T10:59:22.478963+0000","last_unstale":"2023-10-09T10:59:23.117905+0000","last_undegraded":"2023-10-09T10:59:23.117905+0000","last_fullsized":"2023-10-09T10:59:23.117905+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:10.589899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,0],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3,4","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":208,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.116217+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T10:59:23.116217+0000","last_peered":"2023-10-09T10:59:23.116217+0000","last_clean":"2023-10-09T10:59:23.116217+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T10:59:23.116217+0000","last_undegraded":"2023-10-09T10:59:23.116217+0000","last_fullsized":"2023-10-09T10:59:23.116217+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"366'8418","reported_seq":8468,"reported_epoch":366,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:59:26.011416+0000","last_change":"2023-10-09T10:59:09.171437+0000","last_active":"2023-10-09T10:59:26.011416+0000","last_peered":"2023-10-09T10:59:26.011416+0000","last_clean":"2023-10-09T10:59:08.099725+0000","last_became_active":"2023-10-09T10:59:09.171437+0000","last_became_peered":"2023-10-09T10:59:09.171437+0000","last_unstale":"2023-10-09T10:59:26.011416+0000","last_undegraded":"2023-10-09T10:59:09.161264+0000","last_fullsized":"2023-10-09T10:59:09.161112+0000","mapping_epoch":358,"log_start":"337'1600","ondisk_log_start":"337'1600","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":6818,"log_dups_size":0,"ondisk_log_size":6818,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:06.555211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344850432,"num_objects":5262,"num_object_clones":0,"num_object_copies":15786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5262,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8418,"num_write_kb":437760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":5262}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":671,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.111853+0000","last_change":"2023-10-09T10:54:21.173078+0000","last_active":"2023-10-09T10:59:23.111853+0000","last_peered":"2023-10-09T10:59:23.111853+0000","last_clean":"2023-10-09T10:59:23.111853+0000","last_became_active":"2023-10-09T10:54:21.172767+0000","last_became_peered":"2023-10-09T10:54:21.172767+0000","last_unstale":"2023-10-09T10:59:23.111853+0000","last_undegraded":"2023-10-09T10:59:23.111853+0000","last_fullsized":"2023-10-09T10:59:23.111853+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.532890+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.532890+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:32.335841+0000","scrub_duration":0.0045290130000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":352,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.127990+0000","last_change":"2023-10-09T10:59:09.170989+0000","last_active":"2023-10-09T10:59:23.127990+0000","last_peered":"2023-10-09T10:59:23.127990+0000","last_clean":"2023-10-09T10:59:05.146442+0000","last_became_active":"2023-10-09T10:59:09.170989+0000","last_became_peered":"2023-10-09T10:59:09.170989+0000","last_unstale":"2023-10-09T10:59:23.127990+0000","last_undegraded":"2023-10-09T10:59:23.127990+0000","last_fullsized":"2023-10-09T10:59:09.163087+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:09:35.718922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":112,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.111950+0000","last_change":"2023-10-09T10:59:09.168363+0000","last_active":"2023-10-09T10:59:23.111950+0000","last_peered":"2023-10-09T10:59:23.111950+0000","last_clean":"2023-10-09T10:59:05.160728+0000","last_became_active":"2023-10-09T10:59:09.168363+0000","last_became_peered":"2023-10-09T10:59:09.168363+0000","last_unstale":"2023-10-09T10:59:23.111950+0000","last_undegraded":"2023-10-09T10:59:23.111950+0000","last_fullsized":"2023-10-09T10:59:09.161757+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:41:21.121188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"366'8426","reported_seq":8475,"reported_epoch":366,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:59:26.204353+0000","last_change":"2023-10-09T10:59:09.172773+0000","last_active":"2023-10-09T10:59:26.204353+0000","last_peered":"2023-10-09T10:59:26.204353+0000","last_clean":"2023-10-09T10:59:08.124428+0000","last_became_active":"2023-10-09T10:59:09.172773+0000","last_became_peered":"2023-10-09T10:59:09.172773+0000","last_unstale":"2023-10-09T10:59:26.204353+0000","last_undegraded":"2023-10-09T10:59:09.162204+0000","last_fullsized":"2023-10-09T10:59:09.162083+0000","mapping_epoch":358,"log_start":"344'3330","ondisk_log_start":"344'3330","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5096,"log_dups_size":0,"ondisk_log_size":5096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:45:48.391647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340656128,"num_objects":5198,"num_object_clones":0,"num_object_copies":15594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5198,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8426,"num_write_kb":435968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":5198}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":689,"reported_epoch":366,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:24.137485+0000","last_change":"2023-10-09T10:59:24.137485+0000","last_active":"2023-10-09T10:59:24.137485+0000","last_peered":"2023-10-09T10:59:24.137485+0000","last_clean":"2023-10-09T10:59:24.137485+0000","last_became_active":"2023-10-09T10:59:24.136618+0000","last_became_peered":"2023-10-09T10:59:24.136618+0000","last_unstale":"2023-10-09T10:59:24.137485+0000","last_undegraded":"2023-10-09T10:59:24.137485+0000","last_fullsized":"2023-10-09T10:59:24.137485+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:45.845850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:45.845850+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5,0],"avail_no_missing":["2","0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":421,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.128054+0000","last_change":"2023-10-09T10:57:22.006224+0000","last_active":"2023-10-09T10:59:23.128054+0000","last_peered":"2023-10-09T10:59:23.128054+0000","last_clean":"2023-10-09T10:59:23.128054+0000","last_became_active":"2023-10-09T10:57:22.005898+0000","last_became_peered":"2023-10-09T10:57:22.005898+0000","last_unstale":"2023-10-09T10:59:23.128054+0000","last_undegraded":"2023-10-09T10:59:23.128054+0000","last_fullsized":"2023-10-09T10:59:23.128054+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:36:00.623898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":212,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.133090+0000","last_change":"2023-10-09T10:59:09.171567+0000","last_active":"2023-10-09T10:59:23.133090+0000","last_peered":"2023-10-09T10:59:23.133090+0000","last_clean":"2023-10-09T10:54:45.113453+0000","last_became_active":"2023-10-09T10:59:09.171567+0000","last_became_peered":"2023-10-09T10:59:09.171567+0000","last_unstale":"2023-10-09T10:59:23.133090+0000","last_undegraded":"2023-10-09T10:59:23.133090+0000","last_fullsized":"2023-10-09T10:59:09.166246+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:11:52.734017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"366'8461","reported_seq":8517,"reported_epoch":366,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:59:25.674700+0000","last_change":"2023-10-09T10:59:09.238532+0000","last_active":"2023-10-09T10:59:25.674700+0000","last_peered":"2023-10-09T10:59:25.674700+0000","last_clean":"2023-10-09T10:59:08.142682+0000","last_became_active":"2023-10-09T10:59:09.199648+0000","last_became_peered":"2023-10-09T10:59:09.199648+0000","last_unstale":"2023-10-09T10:59:25.674700+0000","last_undegraded":"2023-10-09T10:59:09.182910+0000","last_fullsized":"2023-10-09T10:59:09.166007+0000","mapping_epoch":358,"log_start":"342'3021","ondisk_log_start":"342'3021","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5440,"log_dups_size":0,"ondisk_log_size":5440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:43.691608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349372416,"num_objects":5331,"num_object_clones":0,"num_object_copies":15993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8461,"num_write_kb":441344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":5331}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":682,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.112779+0000","last_change":"2023-10-09T10:54:21.152148+0000","last_active":"2023-10-09T10:59:23.112779+0000","last_peered":"2023-10-09T10:59:23.112779+0000","last_clean":"2023-10-09T10:59:23.112779+0000","last_became_active":"2023-10-09T10:54:21.151809+0000","last_became_peered":"2023-10-09T10:54:21.151809+0000","last_unstale":"2023-10-09T10:59:23.112779+0000","last_undegraded":"2023-10-09T10:59:23.112779+0000","last_fullsized":"2023-10-09T10:59:23.112779+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:26.701128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:26.701128+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:27.021145+0000","scrub_duration":0.0025071830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":419,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.114063+0000","last_change":"2023-10-09T10:59:09.172694+0000","last_active":"2023-10-09T10:59:23.114063+0000","last_peered":"2023-10-09T10:59:23.114063+0000","last_clean":"2023-10-09T10:59:05.150611+0000","last_became_active":"2023-10-09T10:59:09.172694+0000","last_became_peered":"2023-10-09T10:59:09.172694+0000","last_unstale":"2023-10-09T10:59:23.114063+0000","last_undegraded":"2023-10-09T10:59:23.114063+0000","last_fullsized":"2023-10-09T10:59:09.162222+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:12.696156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":126,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.115244+0000","last_change":"2023-10-09T10:59:09.173430+0000","last_active":"2023-10-09T10:59:23.115244+0000","last_peered":"2023-10-09T10:59:23.115244+0000","last_clean":"2023-10-09T10:54:45.117175+0000","last_became_active":"2023-10-09T10:59:09.173430+0000","last_became_peered":"2023-10-09T10:59:09.173430+0000","last_unstale":"2023-10-09T10:59:23.115244+0000","last_undegraded":"2023-10-09T10:59:23.115244+0000","last_fullsized":"2023-10-09T10:59:09.165264+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:55.854811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"366'8645","reported_seq":8695,"reported_epoch":366,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:59:25.652174+0000","last_change":"2023-10-09T10:59:09.182603+0000","last_active":"2023-10-09T10:59:25.652174+0000","last_peered":"2023-10-09T10:59:25.652174+0000","last_clean":"2023-10-09T10:59:08.136852+0000","last_became_active":"2023-10-09T10:59:09.182603+0000","last_became_peered":"2023-10-09T10:59:09.182603+0000","last_unstale":"2023-10-09T10:59:25.652174+0000","last_undegraded":"2023-10-09T10:59:09.162792+0000","last_fullsized":"2023-10-09T10:59:09.162688+0000","mapping_epoch":358,"log_start":"343'3212","ondisk_log_start":"343'3212","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5433,"log_dups_size":0,"ondisk_log_size":5433,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:30.450654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353566720,"num_objects":5395,"num_object_clones":0,"num_object_copies":16185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5395,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8645,"num_write_kb":449280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":5395}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":798,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.115059+0000","last_change":"2023-10-09T10:57:25.110928+0000","last_active":"2023-10-09T10:59:23.115059+0000","last_peered":"2023-10-09T10:59:23.115059+0000","last_clean":"2023-10-09T10:59:23.115059+0000","last_became_active":"2023-10-09T10:57:25.110240+0000","last_became_peered":"2023-10-09T10:57:25.110240+0000","last_unstale":"2023-10-09T10:59:23.115059+0000","last_undegraded":"2023-10-09T10:59:23.115059+0000","last_fullsized":"2023-10-09T10:59:23.115059+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:34.828051+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":301,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.151217+0000","last_change":"2023-10-09T10:59:22.460972+0000","last_active":"2023-10-09T10:59:23.151217+0000","last_peered":"2023-10-09T10:59:23.151217+0000","last_clean":"2023-10-09T10:59:23.151217+0000","last_became_active":"2023-10-09T10:59:22.460590+0000","last_became_peered":"2023-10-09T10:59:22.460590+0000","last_unstale":"2023-10-09T10:59:23.151217+0000","last_undegraded":"2023-10-09T10:59:23.151217+0000","last_fullsized":"2023-10-09T10:59:23.151217+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:22.113955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,0],"avail_no_missing":["7","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":287,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.114222+0000","last_change":"2023-10-09T10:54:17.886921+0000","last_active":"2023-10-09T10:59:23.114222+0000","last_peered":"2023-10-09T10:59:23.114222+0000","last_clean":"2023-10-09T10:59:23.114222+0000","last_became_active":"2023-10-09T10:54:17.886584+0000","last_became_peered":"2023-10-09T10:54:17.886584+0000","last_unstale":"2023-10-09T10:59:23.114222+0000","last_undegraded":"2023-10-09T10:59:23.114222+0000","last_fullsized":"2023-10-09T10:59:23.114222+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:15.848951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":77,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.128233+0000","last_change":"2023-10-09T10:59:09.182114+0000","last_active":"2023-10-09T10:59:23.128233+0000","last_peered":"2023-10-09T10:59:23.128233+0000","last_clean":"2023-10-09T10:35:56.801286+0000","last_became_active":"2023-10-09T10:59:09.182114+0000","last_became_peered":"2023-10-09T10:59:09.182114+0000","last_unstale":"2023-10-09T10:59:23.128233+0000","last_undegraded":"2023-10-09T10:59:23.128233+0000","last_fullsized":"2023-10-09T10:59:09.163380+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:07.576611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"366'8667","reported_seq":9495,"reported_epoch":366,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T10:59:26.260768+0000","last_change":"2023-10-09T10:59:22.670555+0000","last_active":"2023-10-09T10:59:26.260768+0000","last_peered":"2023-10-09T10:59:26.260768+0000","last_clean":"2023-10-09T10:59:20.411264+0000","last_became_active":"2023-10-09T10:59:22.584796+0000","last_became_peered":"2023-10-09T10:59:22.584796+0000","last_unstale":"2023-10-09T10:59:26.260768+0000","last_undegraded":"2023-10-09T10:59:26.260768+0000","last_fullsized":"2023-10-09T10:59:26.260768+0000","mapping_epoch":363,"log_start":"344'3553","ondisk_log_start":"344'3553","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5114,"log_dups_size":0,"ondisk_log_size":5114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:15.381620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352911360,"num_objects":5385,"num_object_clones":0,"num_object_copies":16155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5017,"num_objects_unfound":0,"num_objects_dirty":5385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8667,"num_write_kb":449664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":379,"num_bytes_recovered":24772608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,0],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":5385}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":626,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T10:59:24.136866+0000","last_change":"2023-10-09T10:59:24.136866+0000","last_active":"2023-10-09T10:59:24.136866+0000","last_peered":"2023-10-09T10:59:24.136866+0000","last_clean":"2023-10-09T10:59:24.136866+0000","last_became_active":"2023-10-09T10:59:24.136521+0000","last_became_peered":"2023-10-09T10:59:24.136521+0000","last_unstale":"2023-10-09T10:59:24.136866+0000","last_undegraded":"2023-10-09T10:59:24.136866+0000","last_fullsized":"2023-10-09T10:59:24.136866+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:05.811428+0000","scrub_duration":0.0019146790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":233,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.129106+0000","last_change":"2023-10-09T10:59:09.179737+0000","last_active":"2023-10-09T10:59:23.129106+0000","last_peered":"2023-10-09T10:59:23.129106+0000","last_clean":"2023-10-09T10:54:43.540910+0000","last_became_active":"2023-10-09T10:59:09.179737+0000","last_became_peered":"2023-10-09T10:59:09.179737+0000","last_unstale":"2023-10-09T10:59:23.129106+0000","last_undegraded":"2023-10-09T10:59:23.129106+0000","last_fullsized":"2023-10-09T10:59:09.177206+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:20:45.531486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":224,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.130786+0000","last_change":"2023-10-09T10:59:22.485972+0000","last_active":"2023-10-09T10:59:23.130786+0000","last_peered":"2023-10-09T10:59:23.130786+0000","last_clean":"2023-10-09T10:59:23.130786+0000","last_became_active":"2023-10-09T10:59:22.485321+0000","last_became_peered":"2023-10-09T10:59:22.485321+0000","last_unstale":"2023-10-09T10:59:23.130786+0000","last_undegraded":"2023-10-09T10:59:23.130786+0000","last_fullsized":"2023-10-09T10:59:23.130786+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:04:57.547692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,0],"avail_no_missing":["4","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":37,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.111614+0000","last_change":"2023-10-09T10:59:09.179459+0000","last_active":"2023-10-09T10:59:23.111614+0000","last_peered":"2023-10-09T10:59:23.111614+0000","last_clean":"2023-10-09T10:54:45.115170+0000","last_became_active":"2023-10-09T10:59:09.179459+0000","last_became_peered":"2023-10-09T10:59:09.179459+0000","last_unstale":"2023-10-09T10:59:23.111614+0000","last_undegraded":"2023-10-09T10:59:23.111614+0000","last_fullsized":"2023-10-09T10:59:09.161826+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:01.056262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"366'8539","reported_seq":8599,"reported_epoch":366,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:59:26.251318+0000","last_change":"2023-10-09T10:59:09.184607+0000","last_active":"2023-10-09T10:59:26.251318+0000","last_peered":"2023-10-09T10:59:26.251318+0000","last_clean":"2023-10-09T10:59:08.151874+0000","last_became_active":"2023-10-09T10:59:09.184607+0000","last_became_peered":"2023-10-09T10:59:09.184607+0000","last_unstale":"2023-10-09T10:59:26.251318+0000","last_undegraded":"2023-10-09T10:59:09.162818+0000","last_fullsized":"2023-10-09T10:59:09.162716+0000","mapping_epoch":358,"log_start":"344'3435","ondisk_log_start":"344'3435","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5104,"log_dups_size":0,"ondisk_log_size":5104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:20:42.854967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336658432,"num_objects":5137,"num_object_clones":0,"num_object_copies":15411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5137,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8539,"num_write_kb":437632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":5137}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":800,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T10:59:24.138317+0000","last_change":"2023-10-09T10:59:24.138317+0000","last_active":"2023-10-09T10:59:24.138317+0000","last_peered":"2023-10-09T10:59:24.138317+0000","last_clean":"2023-10-09T10:59:24.138317+0000","last_became_active":"2023-10-09T10:59:24.138037+0000","last_became_peered":"2023-10-09T10:59:24.138037+0000","last_unstale":"2023-10-09T10:59:24.138317+0000","last_undegraded":"2023-10-09T10:59:24.138317+0000","last_fullsized":"2023-10-09T10:59:24.138317+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:17:59.164704+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":107,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.112143+0000","last_change":"2023-10-09T10:59:22.470051+0000","last_active":"2023-10-09T10:59:23.112143+0000","last_peered":"2023-10-09T10:59:23.112143+0000","last_clean":"2023-10-09T10:59:23.112143+0000","last_became_active":"2023-10-09T10:59:22.469694+0000","last_became_peered":"2023-10-09T10:59:22.469694+0000","last_unstale":"2023-10-09T10:59:23.112143+0000","last_undegraded":"2023-10-09T10:59:23.112143+0000","last_fullsized":"2023-10-09T10:59:23.112143+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:47:23.957531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3,0],"avail_no_missing":["2","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":37,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.112389+0000","last_change":"2023-10-09T10:59:09.171069+0000","last_active":"2023-10-09T10:59:23.112389+0000","last_peered":"2023-10-09T10:59:23.112389+0000","last_clean":"2023-10-09T10:54:05.899213+0000","last_became_active":"2023-10-09T10:59:09.171069+0000","last_became_peered":"2023-10-09T10:59:09.171069+0000","last_unstale":"2023-10-09T10:59:23.112389+0000","last_undegraded":"2023-10-09T10:59:23.112389+0000","last_fullsized":"2023-10-09T10:59:09.161486+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:32.591380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":15,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.148911+0000","last_change":"2023-10-09T10:59:22.470146+0000","last_active":"2023-10-09T10:59:23.148911+0000","last_peered":"2023-10-09T10:59:23.148911+0000","last_clean":"2023-10-09T10:59:23.148911+0000","last_became_active":"2023-10-09T10:59:22.468883+0000","last_became_peered":"2023-10-09T10:59:22.468883+0000","last_unstale":"2023-10-09T10:59:23.148911+0000","last_undegraded":"2023-10-09T10:59:23.148911+0000","last_fullsized":"2023-10-09T10:59:23.148911+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:12.066337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5,0],"avail_no_missing":["7","0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.8","version":"366'8405","reported_seq":9057,"reported_epoch":366,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T10:59:25.681420+0000","last_change":"2023-10-09T10:59:22.574009+0000","last_active":"2023-10-09T10:59:25.681420+0000","last_peered":"2023-10-09T10:59:25.681420+0000","last_clean":"2023-10-09T10:59:20.368955+0000","last_became_active":"2023-10-09T10:59:22.540180+0000","last_became_peered":"2023-10-09T10:59:22.540180+0000","last_unstale":"2023-10-09T10:59:25.681420+0000","last_undegraded":"2023-10-09T10:59:25.681420+0000","last_fullsized":"2023-10-09T10:59:25.681420+0000","mapping_epoch":363,"log_start":"342'2957","ondisk_log_start":"342'2957","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5448,"log_dups_size":0,"ondisk_log_size":5448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:25.283167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348061696,"num_objects":5311,"num_object_clones":0,"num_object_copies":15933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5017,"num_objects_unfound":0,"num_objects_dirty":5311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8405,"num_write_kb":438912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":19529728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,0],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":5311}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":115,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.113014+0000","last_change":"2023-10-09T10:57:22.006186+0000","last_active":"2023-10-09T10:59:23.113014+0000","last_peered":"2023-10-09T10:59:23.113014+0000","last_clean":"2023-10-09T10:59:23.113014+0000","last_became_active":"2023-10-09T10:57:22.005900+0000","last_became_peered":"2023-10-09T10:57:22.005900+0000","last_unstale":"2023-10-09T10:59:23.113014+0000","last_undegraded":"2023-10-09T10:59:23.113014+0000","last_fullsized":"2023-10-09T10:59:23.113014+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:14.033785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":809,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.148320+0000","last_change":"2023-10-09T10:59:22.461811+0000","last_active":"2023-10-09T10:59:23.148320+0000","last_peered":"2023-10-09T10:59:23.148320+0000","last_clean":"2023-10-09T10:59:23.148320+0000","last_became_active":"2023-10-09T10:59:22.461400+0000","last_became_peered":"2023-10-09T10:59:22.461400+0000","last_unstale":"2023-10-09T10:59:23.148320+0000","last_undegraded":"2023-10-09T10:59:23.148320+0000","last_fullsized":"2023-10-09T10:59:23.148320+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:26:16.109884+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6,0],"avail_no_missing":["7","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":299,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.115613+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T10:59:23.115613+0000","last_peered":"2023-10-09T10:59:23.115613+0000","last_clean":"2023-10-09T10:59:23.115613+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T10:59:23.115613+0000","last_undegraded":"2023-10-09T10:59:23.115613+0000","last_fullsized":"2023-10-09T10:59:23.115613+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":298,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.128418+0000","last_change":"2023-10-09T10:57:22.013585+0000","last_active":"2023-10-09T10:59:23.128418+0000","last_peered":"2023-10-09T10:59:23.128418+0000","last_clean":"2023-10-09T10:59:23.128418+0000","last_became_active":"2023-10-09T10:57:22.013255+0000","last_became_peered":"2023-10-09T10:57:22.013255+0000","last_unstale":"2023-10-09T10:59:23.128418+0000","last_undegraded":"2023-10-09T10:59:23.128418+0000","last_fullsized":"2023-10-09T10:59:23.128418+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:29.725918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"364'8244","reported_seq":8323,"reported_epoch":364,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T10:59:22.690017+0000","last_change":"2023-10-09T10:59:22.613832+0000","last_active":"2023-10-09T10:59:22.690017+0000","last_peered":"2023-10-09T10:59:22.690017+0000","last_clean":"2023-10-09T10:59:20.436384+0000","last_became_active":"2023-10-09T10:59:22.581691+0000","last_became_peered":"2023-10-09T10:59:22.581691+0000","last_unstale":"2023-10-09T10:59:22.690017+0000","last_undegraded":"2023-10-09T10:59:22.690017+0000","last_fullsized":"2023-10-09T10:59:22.690017+0000","mapping_epoch":363,"log_start":"336'700","ondisk_log_start":"336'700","created":330,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":7544,"log_dups_size":0,"ondisk_log_size":7544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:05:41.441052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355336192,"num_objects":5422,"num_object_clones":0,"num_object_copies":16266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5409,"num_objects_unfound":0,"num_objects_dirty":5422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8244,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[0,6,5],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":5422}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":655,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.129050+0000","last_change":"2023-10-09T10:59:09.171056+0000","last_active":"2023-10-09T10:59:23.129050+0000","last_peered":"2023-10-09T10:59:23.129050+0000","last_clean":"2023-10-09T10:54:44.823610+0000","last_became_active":"2023-10-09T10:59:09.171056+0000","last_became_peered":"2023-10-09T10:59:09.171056+0000","last_unstale":"2023-10-09T10:59:23.129050+0000","last_undegraded":"2023-10-09T10:59:23.129050+0000","last_fullsized":"2023-10-09T10:59:09.164501+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:43.819496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:43.819496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":62,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.148135+0000","last_change":"2023-10-09T10:59:22.481684+0000","last_active":"2023-10-09T10:59:23.148135+0000","last_peered":"2023-10-09T10:59:23.148135+0000","last_clean":"2023-10-09T10:59:23.148135+0000","last_became_active":"2023-10-09T10:59:22.481273+0000","last_became_peered":"2023-10-09T10:59:22.481273+0000","last_unstale":"2023-10-09T10:59:23.148135+0000","last_undegraded":"2023-10-09T10:59:23.148135+0000","last_fullsized":"2023-10-09T10:59:23.148135+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:59.690338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3,0],"avail_no_missing":["7","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":113,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.118315+0000","last_change":"2023-10-09T10:59:21.466422+0000","last_active":"2023-10-09T10:59:23.118315+0000","last_peered":"2023-10-09T10:59:23.118315+0000","last_clean":"2023-10-09T10:59:23.118315+0000","last_became_active":"2023-10-09T10:59:21.465933+0000","last_became_peered":"2023-10-09T10:59:21.465933+0000","last_unstale":"2023-10-09T10:59:23.118315+0000","last_undegraded":"2023-10-09T10:59:23.118315+0000","last_fullsized":"2023-10-09T10:59:23.118315+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:09.013818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":321,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.113003+0000","last_change":"2023-10-09T10:59:22.459143+0000","last_active":"2023-10-09T10:59:23.113003+0000","last_peered":"2023-10-09T10:59:23.113003+0000","last_clean":"2023-10-09T10:59:23.113003+0000","last_became_active":"2023-10-09T10:59:22.458788+0000","last_became_peered":"2023-10-09T10:59:22.458788+0000","last_unstale":"2023-10-09T10:59:23.113003+0000","last_undegraded":"2023-10-09T10:59:23.113003+0000","last_fullsized":"2023-10-09T10:59:23.113003+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:17.150711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7,0],"avail_no_missing":["2","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.e","version":"366'8734","reported_seq":8784,"reported_epoch":366,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:59:26.236156+0000","last_change":"2023-10-09T10:59:09.173588+0000","last_active":"2023-10-09T10:59:26.236156+0000","last_peered":"2023-10-09T10:59:26.236156+0000","last_clean":"2023-10-09T10:59:08.136176+0000","last_became_active":"2023-10-09T10:59:09.173588+0000","last_became_peered":"2023-10-09T10:59:09.173588+0000","last_unstale":"2023-10-09T10:59:26.236156+0000","last_undegraded":"2023-10-09T10:59:09.163720+0000","last_fullsized":"2023-10-09T10:59:09.163611+0000","mapping_epoch":358,"log_start":"344'3602","ondisk_log_start":"344'3602","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5132,"log_dups_size":0,"ondisk_log_size":5132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:51.810055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346685440,"num_objects":5290,"num_object_clones":0,"num_object_copies":15870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5290,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8734,"num_write_kb":448768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":5290}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":555,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T10:59:24.131522+0000","last_change":"2023-10-09T10:59:24.131522+0000","last_active":"2023-10-09T10:59:24.131522+0000","last_peered":"2023-10-09T10:59:24.131522+0000","last_clean":"2023-10-09T10:59:24.131522+0000","last_became_active":"2023-10-09T10:59:24.131162+0000","last_became_peered":"2023-10-09T10:59:24.131162+0000","last_unstale":"2023-10-09T10:59:24.131522+0000","last_undegraded":"2023-10-09T10:59:24.131522+0000","last_fullsized":"2023-10-09T10:59:24.131522+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:27.625079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:27.625079+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":55,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.148161+0000","last_change":"2023-10-09T10:59:22.503760+0000","last_active":"2023-10-09T10:59:23.148161+0000","last_peered":"2023-10-09T10:59:23.148161+0000","last_clean":"2023-10-09T10:59:23.148161+0000","last_became_active":"2023-10-09T10:59:22.503338+0000","last_became_peered":"2023-10-09T10:59:22.503338+0000","last_unstale":"2023-10-09T10:59:23.148161+0000","last_undegraded":"2023-10-09T10:59:23.148161+0000","last_fullsized":"2023-10-09T10:59:23.148161+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:03.763568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4,0],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":37,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.112307+0000","last_change":"2023-10-09T10:59:09.179811+0000","last_active":"2023-10-09T10:59:23.112307+0000","last_peered":"2023-10-09T10:59:23.112307+0000","last_clean":"2023-10-09T10:54:05.900426+0000","last_became_active":"2023-10-09T10:59:09.179811+0000","last_became_peered":"2023-10-09T10:59:09.179811+0000","last_unstale":"2023-10-09T10:59:23.112307+0000","last_undegraded":"2023-10-09T10:59:23.112307+0000","last_fullsized":"2023-10-09T10:59:09.162077+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:50.684070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":269,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.150193+0000","last_change":"2023-10-09T10:59:22.479221+0000","last_active":"2023-10-09T10:59:23.150193+0000","last_peered":"2023-10-09T10:59:23.150193+0000","last_clean":"2023-10-09T10:59:23.150193+0000","last_became_active":"2023-10-09T10:59:22.478893+0000","last_became_peered":"2023-10-09T10:59:22.478893+0000","last_unstale":"2023-10-09T10:59:23.150193+0000","last_undegraded":"2023-10-09T10:59:23.150193+0000","last_fullsized":"2023-10-09T10:59:23.150193+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:01.703735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5,0],"avail_no_missing":["7","0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.d","version":"366'8544","reported_seq":8594,"reported_epoch":366,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:59:26.227534+0000","last_change":"2023-10-09T10:59:09.179357+0000","last_active":"2023-10-09T10:59:26.227534+0000","last_peered":"2023-10-09T10:59:26.227534+0000","last_clean":"2023-10-09T10:59:08.152416+0000","last_became_active":"2023-10-09T10:59:09.179357+0000","last_became_peered":"2023-10-09T10:59:09.179357+0000","last_unstale":"2023-10-09T10:59:26.227534+0000","last_undegraded":"2023-10-09T10:59:09.163333+0000","last_fullsized":"2023-10-09T10:59:09.163222+0000","mapping_epoch":358,"log_start":"343'3433","ondisk_log_start":"343'3433","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5111,"log_dups_size":0,"ondisk_log_size":5111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:12:18.615167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340197400,"num_objects":5192,"num_object_clones":0,"num_object_copies":15576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5192,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5192,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8544,"num_write_kb":439489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":5192}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":761,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.127810+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T10:59:23.127810+0000","last_peered":"2023-10-09T10:59:23.127810+0000","last_clean":"2023-10-09T10:59:23.127810+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T10:59:23.127810+0000","last_undegraded":"2023-10-09T10:59:23.127810+0000","last_fullsized":"2023-10-09T10:59:23.127810+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":429,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.115372+0000","last_change":"2023-10-09T10:59:21.475760+0000","last_active":"2023-10-09T10:59:23.115372+0000","last_peered":"2023-10-09T10:59:23.115372+0000","last_clean":"2023-10-09T10:59:23.115372+0000","last_became_active":"2023-10-09T10:59:21.465697+0000","last_became_peered":"2023-10-09T10:59:21.465697+0000","last_unstale":"2023-10-09T10:59:23.115372+0000","last_undegraded":"2023-10-09T10:59:23.115372+0000","last_fullsized":"2023-10-09T10:59:23.115372+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:16.491057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":99,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.132934+0000","last_change":"2023-10-09T10:59:09.171375+0000","last_active":"2023-10-09T10:59:23.132934+0000","last_peered":"2023-10-09T10:59:23.132934+0000","last_clean":"2023-10-09T10:35:19.857573+0000","last_became_active":"2023-10-09T10:59:09.171375+0000","last_became_peered":"2023-10-09T10:59:09.171375+0000","last_unstale":"2023-10-09T10:59:23.132934+0000","last_undegraded":"2023-10-09T10:59:23.132934+0000","last_fullsized":"2023-10-09T10:59:09.165914+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:48.797574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":417,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.114185+0000","last_change":"2023-10-09T10:59:22.540447+0000","last_active":"2023-10-09T10:59:23.114185+0000","last_peered":"2023-10-09T10:59:23.114185+0000","last_clean":"2023-10-09T10:59:23.114185+0000","last_became_active":"2023-10-09T10:59:22.540003+0000","last_became_peered":"2023-10-09T10:59:22.540003+0000","last_unstale":"2023-10-09T10:59:23.114185+0000","last_undegraded":"2023-10-09T10:59:23.114185+0000","last_fullsized":"2023-10-09T10:59:23.114185+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:54:26.566745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7,0],"avail_no_missing":["3","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"366'8427","reported_seq":8481,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T10:59:25.646359+0000","last_change":"2023-10-09T10:57:22.006548+0000","last_active":"2023-10-09T10:59:25.646359+0000","last_peered":"2023-10-09T10:59:25.646359+0000","last_clean":"2023-10-09T10:59:25.646359+0000","last_became_active":"2023-10-09T10:57:22.006030+0000","last_became_peered":"2023-10-09T10:57:22.006030+0000","last_unstale":"2023-10-09T10:59:25.646359+0000","last_undegraded":"2023-10-09T10:59:25.646359+0000","last_fullsized":"2023-10-09T10:59:25.646359+0000","mapping_epoch":332,"log_start":"342'2970","ondisk_log_start":"342'2970","created":330,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:57:18.972069+0000","last_clean_scrub_stamp":"2023-10-09T10:57:18.972069+0000","objects_scrubbed":0,"log_size":5457,"log_dups_size":0,"ondisk_log_size":5457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:47.539524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342294528,"num_objects":5223,"num_object_clones":0,"num_object_copies":15669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8427,"num_write_kb":436800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":792,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.115970+0000","last_change":"2023-10-09T10:57:25.106977+0000","last_active":"2023-10-09T10:59:23.115970+0000","last_peered":"2023-10-09T10:59:23.115970+0000","last_clean":"2023-10-09T10:59:23.115970+0000","last_became_active":"2023-10-09T10:57:25.106632+0000","last_became_peered":"2023-10-09T10:57:25.106632+0000","last_unstale":"2023-10-09T10:59:23.115970+0000","last_undegraded":"2023-10-09T10:59:23.115970+0000","last_fullsized":"2023-10-09T10:59:23.115970+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:29:20.989344+0000","scrub_duration":0.0053707060000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":43,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.149745+0000","last_change":"2023-10-09T10:57:22.016009+0000","last_active":"2023-10-09T10:59:23.149745+0000","last_peered":"2023-10-09T10:59:23.149745+0000","last_clean":"2023-10-09T10:59:23.149745+0000","last_became_active":"2023-10-09T10:57:22.015633+0000","last_became_peered":"2023-10-09T10:57:22.015633+0000","last_unstale":"2023-10-09T10:59:23.149745+0000","last_undegraded":"2023-10-09T10:59:23.149745+0000","last_fullsized":"2023-10-09T10:59:23.149745+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:59.957416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":33,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.133011+0000","last_change":"2023-10-09T10:59:22.486832+0000","last_active":"2023-10-09T10:59:23.133011+0000","last_peered":"2023-10-09T10:59:23.133011+0000","last_clean":"2023-10-09T10:59:23.133011+0000","last_became_active":"2023-10-09T10:59:22.467099+0000","last_became_peered":"2023-10-09T10:59:22.467099+0000","last_unstale":"2023-10-09T10:59:23.133011+0000","last_undegraded":"2023-10-09T10:59:23.133011+0000","last_fullsized":"2023-10-09T10:59:23.133011+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:30:02.536205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,0],"avail_no_missing":["4","0","7"],"object_location_counts":[{"shards":"0,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":307,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.127965+0000","last_change":"2023-10-09T10:51:50.371282+0000","last_active":"2023-10-09T10:59:23.127965+0000","last_peered":"2023-10-09T10:59:23.127965+0000","last_clean":"2023-10-09T10:59:23.127965+0000","last_became_active":"2023-10-09T10:51:50.370973+0000","last_became_peered":"2023-10-09T10:51:50.370973+0000","last_unstale":"2023-10-09T10:59:23.127965+0000","last_undegraded":"2023-10-09T10:59:23.127965+0000","last_fullsized":"2023-10-09T10:59:23.127965+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:15.603164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":775,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T10:59:24.139059+0000","last_change":"2023-10-09T10:59:24.139059+0000","last_active":"2023-10-09T10:59:24.139059+0000","last_peered":"2023-10-09T10:59:24.139059+0000","last_clean":"2023-10-09T10:59:24.139059+0000","last_became_active":"2023-10-09T10:59:24.138512+0000","last_became_peered":"2023-10-09T10:59:24.138512+0000","last_unstale":"2023-10-09T10:59:24.139059+0000","last_undegraded":"2023-10-09T10:59:24.139059+0000","last_fullsized":"2023-10-09T10:59:24.139059+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:35.839052+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":268,"reported_epoch":365,"state":"active+clean+remapped","last_fresh":"2023-10-09T10:59:23.150330+0000","last_change":"2023-10-09T10:59:22.479537+0000","last_active":"2023-10-09T10:59:23.150330+0000","last_peered":"2023-10-09T10:59:23.150330+0000","last_clean":"2023-10-09T10:59:23.150330+0000","last_became_active":"2023-10-09T10:59:22.479128+0000","last_became_peered":"2023-10-09T10:59:22.479128+0000","last_unstale":"2023-10-09T10:59:23.150330+0000","last_undegraded":"2023-10-09T10:59:23.150330+0000","last_fullsized":"2023-10-09T10:59:23.150330+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:04:33.841438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4,0],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":112,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.112502+0000","last_change":"2023-10-09T10:59:09.179685+0000","last_active":"2023-10-09T10:59:23.112502+0000","last_peered":"2023-10-09T10:59:23.112502+0000","last_clean":"2023-10-09T10:59:05.160996+0000","last_became_active":"2023-10-09T10:59:09.179685+0000","last_became_peered":"2023-10-09T10:59:09.179685+0000","last_unstale":"2023-10-09T10:59:23.112502+0000","last_undegraded":"2023-10-09T10:59:23.112502+0000","last_fullsized":"2023-10-09T10:59:09.162123+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:33.405185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":724,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.115822+0000","last_change":"2023-10-09T10:57:25.111170+0000","last_active":"2023-10-09T10:59:23.115822+0000","last_peered":"2023-10-09T10:59:23.115822+0000","last_clean":"2023-10-09T10:59:23.115822+0000","last_became_active":"2023-10-09T10:57:25.110405+0000","last_became_peered":"2023-10-09T10:57:25.110405+0000","last_unstale":"2023-10-09T10:59:23.115822+0000","last_undegraded":"2023-10-09T10:59:23.115822+0000","last_fullsized":"2023-10-09T10:59:23.115822+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:00.112600+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":199,"reported_epoch":365,"state":"active+undersized+degraded","last_fresh":"2023-10-09T10:59:23.133549+0000","last_change":"2023-10-09T10:59:09.182672+0000","last_active":"2023-10-09T10:59:23.133549+0000","last_peered":"2023-10-09T10:59:23.133549+0000","last_clean":"2023-10-09T10:59:05.147099+0000","last_became_active":"2023-10-09T10:59:09.182672+0000","last_became_peered":"2023-10-09T10:59:09.182672+0000","last_unstale":"2023-10-09T10:59:23.133549+0000","last_undegraded":"2023-10-09T10:59:09.164844+0000","last_fullsized":"2023-10-09T10:59:09.164730+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:09.662080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":393,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.118047+0000","last_change":"2023-10-09T10:57:46.844827+0000","last_active":"2023-10-09T10:59:23.118047+0000","last_peered":"2023-10-09T10:59:23.118047+0000","last_clean":"2023-10-09T10:59:23.118047+0000","last_became_active":"2023-10-09T10:57:46.844584+0000","last_became_peered":"2023-10-09T10:57:46.844584+0000","last_unstale":"2023-10-09T10:59:23.118047+0000","last_undegraded":"2023-10-09T10:59:23.118047+0000","last_fullsized":"2023-10-09T10:59:23.118047+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:54.499813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":316,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.129550+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T10:59:23.129550+0000","last_peered":"2023-10-09T10:59:23.129550+0000","last_clean":"2023-10-09T10:59:23.129550+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T10:59:23.129550+0000","last_undegraded":"2023-10-09T10:59:23.129550+0000","last_fullsized":"2023-10-09T10:59:23.129550+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":320,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.115699+0000","last_change":"2023-10-09T10:59:21.464840+0000","last_active":"2023-10-09T10:59:23.115699+0000","last_peered":"2023-10-09T10:59:23.115699+0000","last_clean":"2023-10-09T10:59:23.115699+0000","last_became_active":"2023-10-09T10:59:21.463054+0000","last_became_peered":"2023-10-09T10:59:21.463054+0000","last_unstale":"2023-10-09T10:59:23.115699+0000","last_undegraded":"2023-10-09T10:59:23.115699+0000","last_fullsized":"2023-10-09T10:59:23.115699+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:14.736964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":664,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.151458+0000","last_change":"2023-10-09T10:59:09.170522+0000","last_active":"2023-10-09T10:59:23.151458+0000","last_peered":"2023-10-09T10:59:23.151458+0000","last_clean":"2023-10-09T10:59:05.157045+0000","last_became_active":"2023-10-09T10:59:09.170522+0000","last_became_peered":"2023-10-09T10:59:09.170522+0000","last_unstale":"2023-10-09T10:59:23.151458+0000","last_undegraded":"2023-10-09T10:59:23.151458+0000","last_fullsized":"2023-10-09T10:59:09.162631+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:47.307693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:47.307693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:03:49.496920+0000","scrub_duration":0.003001985,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":391,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.117821+0000","last_change":"2023-10-09T10:54:17.886158+0000","last_active":"2023-10-09T10:59:23.117821+0000","last_peered":"2023-10-09T10:59:23.117821+0000","last_clean":"2023-10-09T10:59:23.117821+0000","last_became_active":"2023-10-09T10:54:17.885816+0000","last_became_peered":"2023-10-09T10:54:17.885816+0000","last_unstale":"2023-10-09T10:59:23.117821+0000","last_undegraded":"2023-10-09T10:59:23.117821+0000","last_fullsized":"2023-10-09T10:59:23.117821+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:05.270825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":403,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.116406+0000","last_change":"2023-10-09T10:59:21.466233+0000","last_active":"2023-10-09T10:59:23.116406+0000","last_peered":"2023-10-09T10:59:23.116406+0000","last_clean":"2023-10-09T10:59:23.116406+0000","last_became_active":"2023-10-09T10:59:21.465061+0000","last_became_peered":"2023-10-09T10:59:21.465061+0000","last_unstale":"2023-10-09T10:59:23.116406+0000","last_undegraded":"2023-10-09T10:59:23.116406+0000","last_fullsized":"2023-10-09T10:59:23.116406+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:03.747784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":598,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.113825+0000","last_change":"2023-10-09T10:59:09.168496+0000","last_active":"2023-10-09T10:59:23.113825+0000","last_peered":"2023-10-09T10:59:23.113825+0000","last_clean":"2023-10-09T10:54:45.106080+0000","last_became_active":"2023-10-09T10:59:09.168496+0000","last_became_peered":"2023-10-09T10:59:09.168496+0000","last_unstale":"2023-10-09T10:59:23.113825+0000","last_undegraded":"2023-10-09T10:59:23.113825+0000","last_fullsized":"2023-10-09T10:59:09.162630+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:21.022863+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:21.022863+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":195,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T10:59:23.111976+0000","last_change":"2023-10-09T10:59:21.469367+0000","last_active":"2023-10-09T10:59:23.111976+0000","last_peered":"2023-10-09T10:59:23.111976+0000","last_clean":"2023-10-09T10:59:23.111976+0000","last_became_active":"2023-10-09T10:59:21.463733+0000","last_became_peered":"2023-10-09T10:59:21.463733+0000","last_unstale":"2023-10-09T10:59:23.111976+0000","last_undegraded":"2023-10-09T10:59:23.111976+0000","last_fullsized":"2023-10-09T10:59:23.111976+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:02.567866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":377,"reported_epoch":365,"state":"active+undersized","last_fresh":"2023-10-09T10:59:23.127261+0000","last_change":"2023-10-09T10:59:09.170532+0000","last_active":"2023-10-09T10:59:23.127261+0000","last_peered":"2023-10-09T10:59:23.127261+0000","last_clean":"2023-10-09T10:59:05.145971+0000","last_became_active":"2023-10-09T10:59:09.170532+0000","last_became_peered":"2023-10-09T10:59:09.170532+0000","last_unstale":"2023-10-09T10:59:23.127261+0000","last_undegraded":"2023-10-09T10:59:23.127261+0000","last_fullsized":"2023-10-09T10:59:09.162637+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:11:24.535897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":76,"acting":82,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":3,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":16490,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":80,"acting":87,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":702181,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":93,"num_read_kb":173,"num_write":46,"num_write_kb":458,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":1713869,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":81,"acting":85,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5532418072,"num_objects":84419,"num_object_clones":0,"num_object_copies":253257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47311,"num_objects_misplaced":20721,"num_objects_unfound":0,"num_objects_dirty":84419,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":136175,"num_write_kb":7058945,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":692,"num_bytes_recovered":45219840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17244368896,"data_stored":17244356680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90916,"ondisk_log_size":90916,"up":38,"acting":39,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":18,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332394,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3442828,"kb_used_data":3137348,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":90302324,"statfs":{"total":95995035648,"available":92469579776,"internally_reserved":0,"allocated":3212644352,"data_stored":3211423047,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,9],"upper_bound":4096},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:59:07 2023","interfaces":[{"interface":"back","average":{"1min":2.9449999999999998,"5min":2.1880000000000002,"15min":1.649},"min":{"1min":0.23000000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":8.9730000000000008,"5min":9.6259999999999994,"15min":9.6379999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":2.2429999999999999,"5min":2.0979999999999999,"15min":1.8169999999999999},"min":{"1min":0.249,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.5609999999999999,"5min":8.7769999999999992,"15min":16.731000000000002},"last":0.36199999999999999}]},{"osd":1,"last update":"Mon Oct 9 10:58:10 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":1.835,"15min":1.827},"min":{"1min":0.26500000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.2880000000000003,"5min":12.252000000000001,"15min":34.756},"last":0.92400000000000004},{"interface":"front","average":{"1min":1.177,"5min":1.744,"15min":1.609},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":2.4649999999999999,"5min":13.647,"15min":35.658999999999999},"last":5.7850000000000001}]},{"osd":2,"last update":"Mon Oct 9 10:58:35 2023","interfaces":[{"interface":"back","average":{"1min":1.5940000000000001,"5min":1.7490000000000001,"15min":1.6200000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":5.7619999999999996,"5min":9.202,"15min":11.058},"last":0.75},{"interface":"front","average":{"1min":2.5,"5min":2.077,"15min":1.575},"min":{"1min":0.34000000000000002,"5min":0.26600000000000001,"15min":0.251},"max":{"1min":8.9280000000000008,"5min":13.037000000000001,"15min":13.037000000000001},"last":0.39200000000000002}]},{"osd":3,"last update":"Mon Oct 9 10:59:07 2023","interfaces":[{"interface":"back","average":{"1min":2.379,"5min":2.2269999999999999,"15min":2.0329999999999999},"min":{"1min":0.315,"5min":0.31,"15min":0.31},"max":{"1min":8.2829999999999995,"5min":11.760999999999999,"15min":11.760999999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.6059999999999999,"5min":1.8919999999999999,"15min":1.7609999999999999},"min":{"1min":0.35099999999999998,"5min":0.31,"15min":0.31},"max":{"1min":7.7210000000000001,"5min":8.6300000000000008,"15min":19.234999999999999},"last":0.497}]},{"osd":4,"last update":"Mon Oct 9 10:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":1.0389999999999999,"15min":1.3320000000000001},"min":{"1min":0.29399999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.6429999999999998,"5min":8.2940000000000005,"15min":16.695},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.997,"5min":0.86599999999999999,"15min":1.1699999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26200000000000001},"max":{"1min":3.6779999999999999,"5min":7.9320000000000004,"15min":11.109999999999999},"last":0.376}]},{"osd":5,"last update":"Mon Oct 9 10:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":1.0189999999999999,"15min":1.2709999999999999},"min":{"1min":0.29399999999999998,"5min":0.247,"15min":0.247},"max":{"1min":7.9690000000000003,"5min":7.9690000000000003,"15min":13.262},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.6299999999999999,"5min":0.97899999999999998,"15min":1.4139999999999999},"min":{"1min":0.33600000000000002,"5min":0.318,"15min":0.29599999999999999},"max":{"1min":4.7409999999999997,"5min":8.4339999999999993,"15min":11.741},"last":0.433}]},{"osd":6,"last update":"Mon Oct 9 10:58:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":1.079,"15min":1.538},"min":{"1min":0.39500000000000002,"5min":0.315,"15min":0.28199999999999997},"max":{"1min":5.2430000000000003,"5min":8.3610000000000007,"15min":14.913},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.4099999999999999,"5min":1.0389999999999999,"15min":1.3740000000000001},"min":{"1min":0.37,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.1139999999999999,"5min":7.9809999999999999,"15min":19.187999999999999},"last":0.41499999999999998}]}]},{"osd":6,"up_from":44,"seq":188978561325,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2000892,"kb_used_data":1096628,"kb_used_omap":0,"kb_used_meta":904256,"kb_avail":91744260,"statfs":{"total":95995035648,"available":93946122240,"internally_reserved":0,"allocated":1122947072,"data_stored":1121741991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925958144},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:59:02 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":2.1850000000000001,"15min":1.645},"min":{"1min":0.25800000000000001,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":2.8439999999999999,"5min":54.743000000000002,"15min":54.743000000000002},"last":4.0039999999999996},{"interface":"front","average":{"1min":2.3540000000000001,"5min":2.2759999999999998,"15min":1.7509999999999999},"min":{"1min":0.29499999999999998,"5min":0.223,"15min":0.223},"max":{"1min":12.768000000000001,"5min":53.701000000000001,"15min":53.701000000000001},"last":4.0890000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:58:14 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":1.474,"15min":1.4770000000000001},"min":{"1min":0.44,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.96,"5min":7.6820000000000004,"15min":16.747},"last":2.681},{"interface":"front","average":{"1min":1.869,"5min":1.5229999999999999,"15min":1.534},"min":{"1min":0.376,"5min":0.26400000000000001,"15min":0.23100000000000001},"max":{"1min":8.6419999999999995,"5min":8.6419999999999995,"15min":16.734000000000002},"last":2.6230000000000002}]},{"osd":2,"last update":"Mon Oct 9 10:58:40 2023","interfaces":[{"interface":"back","average":{"1min":2.8239999999999998,"5min":2.0409999999999999,"15min":1.7190000000000001},"min":{"1min":0.34699999999999998,"5min":0.29199999999999998,"15min":0.26800000000000002},"max":{"1min":8.6869999999999994,"5min":12.003,"15min":13.208},"last":3.9609999999999999},{"interface":"front","average":{"1min":2.6520000000000001,"5min":1.8380000000000001,"15min":1.79},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28399999999999997},"max":{"1min":15.432,"5min":15.432,"15min":15.432},"last":9.0350000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:59:02 2023","interfaces":[{"interface":"back","average":{"1min":2.9980000000000002,"5min":2.052,"15min":1.7929999999999999},"min":{"1min":0.375,"5min":0.32500000000000001,"15min":0.255},"max":{"1min":10.018000000000001,"5min":10.018000000000001,"15min":12.391},"last":4.0300000000000002},{"interface":"front","average":{"1min":2.8730000000000002,"5min":2.2970000000000002,"15min":2.0499999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.28100000000000003},"max":{"1min":11.161,"5min":11.161,"15min":13.179},"last":3.919}]},{"osd":4,"last update":"Mon Oct 9 10:58:40 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.83299999999999996,"15min":1.1950000000000001},"min":{"1min":0.27900000000000003,"5min":0.24199999999999999,"15min":0.23200000000000001},"max":{"1min":3.4620000000000002,"5min":16.978999999999999,"15min":16.978999999999999},"last":9.0060000000000002},{"interface":"front","average":{"1min":1.1140000000000001,"5min":0.92200000000000004,"15min":1.1839999999999999},"min":{"1min":0.29899999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.5129999999999999,"5min":4.7930000000000001,"15min":12.481999999999999},"last":4.7389999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:58:40 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":1.171,"15min":1.373},"min":{"1min":0.35499999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.3220000000000001,"5min":12.923,"15min":12.923},"last":8.9610000000000003},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.032,"15min":1.2709999999999999},"min":{"1min":0.28899999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.46,"5min":9.0559999999999992,"15min":12.456},"last":4.1180000000000003}]},{"osd":7,"last update":"Mon Oct 9 10:58:58 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":1.113,"15min":1.4019999999999999},"min":{"1min":0.35199999999999998,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":6.2800000000000002,"5min":12.882,"15min":23.358000000000001},"last":9.1120000000000001},{"interface":"front","average":{"1min":1.0369999999999999,"5min":0.96399999999999997,"15min":1.173},"min":{"1min":0.33300000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.2830000000000004,"5min":6.1029999999999998,"15min":12.422000000000001},"last":4.0620000000000003}]}]},{"osd":1,"up_from":85,"seq":365072220360,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4096416,"kb_used_data":3587544,"kb_used_omap":9,"kb_used_meta":508854,"kb_avail":89648736,"statfs":{"total":95995035648,"available":91800305664,"internally_reserved":0,"allocated":3673645056,"data_stored":3671816247,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":521067497},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":2.464,"15min":2.0099999999999998},"min":{"1min":0.224,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":4.3049999999999997,"5min":19.992999999999999,"15min":19.992999999999999},"last":1.6180000000000001},{"interface":"front","average":{"1min":2.4369999999999998,"5min":2.9550000000000001,"15min":2.1899999999999999},"min":{"1min":0.27000000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":6.9790000000000001,"5min":18.303000000000001,"15min":18.303000000000001},"last":1.0369999999999999}]},{"osd":2,"last update":"Mon Oct 9 10:58:39 2023","interfaces":[{"interface":"back","average":{"1min":1.865,"5min":2.302,"15min":1.7410000000000001},"min":{"1min":0.23100000000000001,"5min":0.223,"15min":0.19700000000000001},"max":{"1min":5.9980000000000002,"5min":14.878,"15min":15.356},"last":1.2789999999999999},{"interface":"front","average":{"1min":1.2490000000000001,"5min":2.1800000000000002,"15min":1.8320000000000001},"min":{"1min":0.221,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":4.6059999999999999,"5min":18.004000000000001,"15min":18.004000000000001},"last":1.653}]},{"osd":3,"last update":"Mon Oct 9 10:59:05 2023","interfaces":[{"interface":"back","average":{"1min":3.2869999999999999,"5min":3.129,"15min":2.4180000000000001},"min":{"1min":0.26600000000000001,"5min":0.22900000000000001,"15min":0.218},"max":{"1min":7.673,"5min":20.036000000000001,"15min":20.036000000000001},"last":1.7090000000000001},{"interface":"front","average":{"1min":2.9540000000000002,"5min":3.3639999999999999,"15min":2.4569999999999999},"min":{"1min":0.26000000000000001,"5min":0.219,"15min":0.219},"max":{"1min":10.647,"5min":19.640999999999998,"15min":19.640999999999998},"last":0.92800000000000005}]},{"osd":4,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":1.5009999999999999,"15min":1.468},"min":{"1min":0.30099999999999999,"5min":0.28899999999999998,"15min":0.255},"max":{"1min":1.71,"5min":14.763,"15min":14.763},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.4099999999999999,"5min":1.75,"15min":1.7190000000000001},"min":{"1min":0.379,"5min":0.30399999999999999,"15min":0.27200000000000002},"max":{"1min":4.0199999999999996,"5min":8.3079999999999998,"15min":14.228999999999999},"last":1.6819999999999999}]},{"osd":5,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":1.9670000000000001,"15min":1.631},"min":{"1min":0.376,"5min":0.32700000000000001,"15min":0.253},"max":{"1min":5.9989999999999997,"5min":9.6289999999999996,"15min":9.7720000000000002},"last":5.3659999999999997},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.873,"15min":1.8380000000000001},"min":{"1min":0.313,"5min":0.30299999999999999,"15min":0.28899999999999998},"max":{"1min":2.1930000000000001,"5min":14.833,"15min":17.085000000000001},"last":0.41099999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.8640000000000001,"15min":1.716},"min":{"1min":0.41799999999999998,"5min":0.32600000000000001,"15min":0.27400000000000002},"max":{"1min":5.9630000000000001,"5min":8.8230000000000004,"15min":10.425000000000001},"last":1.369},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.653,"15min":1.6479999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.312},"max":{"1min":1.74,"5min":14.804,"15min":14.804},"last":0.47099999999999997}]},{"osd":7,"last update":"Mon Oct 9 10:58:09 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.8180000000000001,"15min":1.7150000000000001},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.34000000000000002},"max":{"1min":2.5619999999999998,"5min":14.856,"15min":16.312000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.2829999999999999,"5min":1.6839999999999999,"15min":1.7130000000000001},"min":{"1min":0.41099999999999998,"5min":0.35499999999999998,"15min":0.32500000000000001},"max":{"1min":2.8940000000000001,"5min":8.8149999999999995,"15min":14.273999999999999},"last":4.1929999999999996}]}]},{"osd":0,"up_from":0,"seq":343597383897,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":90,"seq":386547056838,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1453032,"kb_used_data":1061600,"kb_used_omap":81,"kb_used_meta":391342,"kb_avail":92292120,"statfs":{"total":95995035648,"available":94507130880,"internally_reserved":0,"allocated":1087078400,"data_stored":1085307787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":400734426},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.012,"15min":1.776},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":6.702,"5min":17.574000000000002,"15min":18.512},"last":0.34699999999999998},{"interface":"front","average":{"1min":2.698,"5min":2.1070000000000002,"15min":1.917},"min":{"1min":0.27400000000000002,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":7.4699999999999998,"5min":11.167,"15min":17.975000000000001},"last":0.38800000000000001}]},{"osd":1,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":3.1709999999999998,"5min":2.117,"15min":1.8879999999999999},"min":{"1min":0.308,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":11.148,"5min":16.359000000000002,"15min":16.359000000000002},"last":0.27700000000000002},{"interface":"front","average":{"1min":2.7669999999999999,"5min":2.254,"15min":1.7270000000000001},"min":{"1min":0.27800000000000002,"5min":0.24199999999999999,"15min":0.20699999999999999},"max":{"1min":7.8970000000000002,"5min":12.693,"15min":16.454999999999998},"last":0.30099999999999999}]},{"osd":3,"last update":"Mon Oct 9 10:59:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4590000000000001,"5min":2.7109999999999999,"15min":1.9750000000000001},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":11.151,"5min":19.539000000000001,"15min":19.539000000000001},"last":0.314},{"interface":"front","average":{"1min":3.3050000000000002,"5min":2.835,"15min":2.3410000000000002},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.20200000000000001},"max":{"1min":16.324000000000002,"5min":16.702999999999999,"15min":16.702999999999999},"last":0.35699999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4399999999999999,"5min":2.23,"15min":1.9339999999999999},"min":{"1min":0.33700000000000002,"5min":0.28599999999999998,"15min":0.254},"max":{"1min":7.4080000000000004,"5min":15.672000000000001,"15min":16.231999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.8680000000000001,"15min":1.7929999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.26300000000000001},"max":{"1min":6.4020000000000001,"5min":15.696999999999999,"15min":15.696999999999999},"last":0.51800000000000002}]},{"osd":5,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":2.468,"5min":1.784,"15min":1.7649999999999999},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.245},"max":{"1min":6.7709999999999999,"5min":12.128,"15min":17.920000000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":2.3530000000000002,"5min":2.2650000000000001,"15min":1.8740000000000001},"min":{"1min":0.313,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.3719999999999999,"5min":16.597000000000001,"15min":17.882999999999999},"last":0.5}]},{"osd":6,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":1.607,"15min":1.569},"min":{"1min":0.34499999999999997,"5min":0.33400000000000002,"15min":0.247},"max":{"1min":6.3689999999999998,"5min":8.8460000000000001,"15min":14.029999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.5169999999999999,"5min":1.7390000000000001,"15min":1.7330000000000001},"min":{"1min":0.40500000000000003,"5min":0.34200000000000003,"15min":0.29199999999999998},"max":{"1min":5.9909999999999997,"5min":10.111000000000001,"15min":12.862},"last":0.41999999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":1.762,"15min":1.625},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.3300000000000001,"5min":8.8879999999999999,"15min":13.999000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.819,"5min":1.929,"15min":1.831},"min":{"1min":0.41999999999999998,"5min":0.34000000000000002,"15min":0.307},"max":{"1min":5.9459999999999997,"5min":10.157999999999999,"15min":12.826000000000001},"last":0.44700000000000001}]}]},{"osd":3,"up_from":95,"seq":408021893312,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3545344,"kb_used_data":2760248,"kb_used_omap":7,"kb_used_meta":785080,"kb_avail":90199808,"statfs":{"total":95995035648,"available":92364603392,"internally_reserved":0,"allocated":2826493952,"data_stored":2824522829,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":803922536},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:59:06 2023","interfaces":[{"interface":"back","average":{"1min":2.9049999999999998,"5min":3.7200000000000002,"15min":2.4649999999999999},"min":{"1min":0.25600000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":9.0939999999999994,"5min":38.817999999999998,"15min":38.817999999999998},"last":0.30099999999999999},{"interface":"front","average":{"1min":2.4340000000000002,"5min":3.649,"15min":2.7189999999999999},"min":{"1min":0.214,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":9.2170000000000005,"5min":14.263999999999999,"15min":17.312000000000001},"last":1.29}]},{"osd":1,"last update":"Mon Oct 9 10:59:06 2023","interfaces":[{"interface":"back","average":{"1min":2.6429999999999998,"5min":2.8849999999999998,"15min":2.5070000000000001},"min":{"1min":0.24399999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":9.3840000000000003,"5min":22.469000000000001,"15min":22.469000000000001},"last":2.923},{"interface":"front","average":{"1min":3.544,"5min":3.7650000000000001,"15min":2.6520000000000001},"min":{"1min":0.32900000000000001,"5min":0.22,"15min":0.193},"max":{"1min":9.6590000000000007,"5min":18.526,"15min":18.526},"last":5.702}]},{"osd":2,"last update":"Mon Oct 9 10:59:06 2023","interfaces":[{"interface":"back","average":{"1min":2.8500000000000001,"5min":3.0880000000000001,"15min":2.3799999999999999},"min":{"1min":0.26000000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":10.65,"5min":20.402000000000001,"15min":20.402000000000001},"last":1.411},{"interface":"front","average":{"1min":2.6869999999999998,"5min":3.6739999999999999,"15min":2.6419999999999999},"min":{"1min":0.47099999999999997,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":9.2390000000000008,"5min":27.538,"15min":27.538},"last":1.3799999999999999}]},{"osd":4,"last update":"Mon Oct 9 10:59:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":2.6040000000000001,"15min":2.113},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":8.5670000000000002,"5min":11.863,"15min":20.388000000000002},"last":0.32900000000000001},{"interface":"front","average":{"1min":2.2469999999999999,"5min":2.3100000000000001,"15min":2.0329999999999999},"min":{"1min":0.40200000000000002,"5min":0.28399999999999997,"15min":0.24299999999999999},"max":{"1min":9.1310000000000002,"5min":11.269,"15min":19.085999999999999},"last":1.3320000000000001}]},{"osd":5,"last update":"Mon Oct 9 10:59:06 2023","interfaces":[{"interface":"back","average":{"1min":2.7669999999999999,"5min":2.4969999999999999,"15min":2.3490000000000002},"min":{"1min":0.37,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.380000000000001,"5min":14.045,"15min":51.509999999999998},"last":4.6890000000000001},{"interface":"front","average":{"1min":1.984,"5min":1.968,"15min":2.2629999999999999},"min":{"1min":0.42499999999999999,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":5.6740000000000004,"5min":17.311,"15min":55.338999999999999},"last":2.972}]},{"osd":6,"last update":"Mon Oct 9 10:59:06 2023","interfaces":[{"interface":"back","average":{"1min":2.911,"5min":2.5059999999999998,"15min":2.0840000000000001},"min":{"1min":0.436,"5min":0.29199999999999998,"15min":0.28000000000000003},"max":{"1min":10.420999999999999,"5min":10.420999999999999,"15min":12.361000000000001},"last":3.2690000000000001},{"interface":"front","average":{"1min":2.2759999999999998,"5min":2.2080000000000002,"15min":2.1930000000000001},"min":{"1min":0.39500000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":6.8970000000000002,"5min":17.295000000000002,"15min":17.295000000000002},"last":2.7719999999999998}]},{"osd":7,"last update":"Mon Oct 9 10:59:06 2023","interfaces":[{"interface":"back","average":{"1min":1.887,"5min":2.262,"15min":2.3999999999999999},"min":{"1min":0.35899999999999999,"5min":0.34999999999999998,"15min":0.31},"max":{"1min":5.601,"5min":18.722000000000001,"15min":51.744},"last":3.3420000000000001},{"interface":"front","average":{"1min":2.5179999999999998,"5min":2.722,"15min":2.464},"min":{"1min":0.374,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.4980000000000002,"5min":13.337,"15min":53.399000000000001},"last":0.69399999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019188,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1913092,"kb_used_data":1016252,"kb_used_omap":0,"kb_used_meta":896832,"kb_avail":91832060,"statfs":{"total":95995035648,"available":94036029440,"internally_reserved":0,"allocated":1040642048,"data_stored":1039428620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918355968},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:58:56 2023","interfaces":[{"interface":"back","average":{"1min":1.923,"5min":4.2919999999999998,"15min":2.294},"min":{"1min":0.25900000000000001,"5min":0.224,"15min":0.20999999999999999},"max":{"1min":6.9379999999999997,"5min":213.88999999999999,"15min":213.88999999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.1219999999999999,"5min":4.2460000000000004,"15min":2.6030000000000002},"min":{"1min":0.28000000000000003,"5min":0.23200000000000001,"15min":0.217},"max":{"1min":5.7149999999999999,"5min":214.09800000000001,"15min":214.09800000000001},"last":0.33300000000000002}]},{"osd":1,"last update":"Mon Oct 9 10:59:04 2023","interfaces":[{"interface":"back","average":{"1min":2.5219999999999998,"5min":4.1459999999999999,"15min":2.347},"min":{"1min":0.24199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":11.231999999999999,"5min":206.50399999999999,"15min":206.50399999999999},"last":1.48},{"interface":"front","average":{"1min":3.0569999999999999,"5min":4.2549999999999999,"15min":2.581},"min":{"1min":0.28299999999999997,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.8109999999999999,"5min":206.529,"15min":206.529},"last":4.2460000000000004}]},{"osd":2,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":1.9399999999999999,"5min":1.9139999999999999,"15min":1.744},"min":{"1min":0.27100000000000002,"5min":0.25900000000000001,"15min":0.253},"max":{"1min":5.6870000000000003,"5min":15.446999999999999,"15min":15.446999999999999},"last":0.5},{"interface":"front","average":{"1min":2.9969999999999999,"5min":2.605,"15min":2.073},"min":{"1min":0.30099999999999999,"5min":0.29699999999999999,"15min":0.26300000000000001},"max":{"1min":6.5019999999999998,"5min":27.818000000000001,"15min":27.818000000000001},"last":0.44700000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:58:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0059999999999998,"5min":4.2949999999999999,"15min":2.6030000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.27400000000000002},"max":{"1min":12.798999999999999,"5min":209.03700000000001,"15min":209.03700000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.931,"5min":4.6230000000000002,"15min":2.8490000000000002},"min":{"1min":0.432,"5min":0.30499999999999999,"15min":0.29999999999999999},"max":{"1min":6.5369999999999999,"5min":208.80199999999999,"15min":208.80199999999999},"last":0.52900000000000003}]},{"osd":5,"last update":"Mon Oct 9 10:59:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2749999999999999,"5min":1.415,"15min":1.6950000000000001},"min":{"1min":0.28199999999999997,"5min":0.253,"15min":0.224},"max":{"1min":9.6530000000000005,"5min":19.068999999999999,"15min":24.149999999999999},"last":0.372},{"interface":"front","average":{"1min":2.734,"5min":1.4430000000000001,"15min":1.544},"min":{"1min":0.35699999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.459,"5min":15.535,"15min":20.422000000000001},"last":0.47299999999999998}]},{"osd":6,"last update":"Mon Oct 9 10:58:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":0.98099999999999998,"15min":1.2709999999999999},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.24099999999999999},"max":{"1min":4.944,"5min":15.739000000000001,"15min":15.739000000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.046,"5min":1.0940000000000001,"15min":1.494},"min":{"1min":0.32800000000000001,"5min":0.32700000000000001,"15min":0.311},"max":{"1min":4.4240000000000004,"5min":14.093999999999999,"15min":14.272},"last":0.45800000000000002}]},{"osd":7,"last update":"Mon Oct 9 10:58:42 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":1.367,"15min":1.4830000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":9.7620000000000005,"5min":14.458,"15min":14.458},"last":0.435},{"interface":"front","average":{"1min":1.9530000000000001,"5min":1.532,"15min":1.3919999999999999},"min":{"1min":0.38,"5min":0.34499999999999997,"15min":0.255},"max":{"1min":9.9789999999999992,"5min":15.592000000000001,"15min":15.592000000000001},"last":0.55600000000000005}]}]},{"osd":5,"up_from":37,"seq":158913790257,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3948216,"kb_used_data":3049584,"kb_used_omap":0,"kb_used_meta":898624,"kb_avail":89796936,"statfs":{"total":95995035648,"available":91952062464,"internally_reserved":0,"allocated":3122774016,"data_stored":3121568865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920190976},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 10:59:03 2023","interfaces":[{"interface":"back","average":{"1min":3.9060000000000001,"5min":2.4609999999999999,"15min":1.8740000000000001},"min":{"1min":0.38500000000000001,"5min":0.215,"15min":0.215},"max":{"1min":29.678999999999998,"5min":29.678999999999998,"15min":29.678999999999998},"last":0.51900000000000002},{"interface":"front","average":{"1min":3.9329999999999998,"5min":2.5070000000000001,"15min":1.929},"min":{"1min":0.35299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":29.759,"5min":29.759,"15min":29.759},"last":0.54300000000000004}]},{"osd":1,"last update":"Mon Oct 9 10:58:15 2023","interfaces":[{"interface":"back","average":{"1min":4.2359999999999998,"5min":2.5880000000000001,"15min":2.1019999999999999},"min":{"1min":0.313,"5min":0.215,"15min":0.215},"max":{"1min":29.818999999999999,"5min":29.818999999999999,"15min":29.818999999999999},"last":3.157},{"interface":"front","average":{"1min":3.9409999999999998,"5min":2.3029999999999999,"15min":1.804},"min":{"1min":0.28199999999999997,"5min":0.253,"15min":0.24299999999999999},"max":{"1min":29.600000000000001,"5min":29.600000000000001,"15min":29.600000000000001},"last":0.35099999999999998}]},{"osd":2,"last update":"Mon Oct 9 10:58:34 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":2.2170000000000001,"15min":1.9199999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":4.7050000000000001,"5min":11.257999999999999,"15min":11.393000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.0499999999999998,"5min":2.2559999999999998,"15min":1.7430000000000001},"min":{"1min":0.33100000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.7889999999999997,"5min":9.2569999999999997,"15min":15.087999999999999},"last":0.44900000000000001}]},{"osd":3,"last update":"Mon Oct 9 10:59:03 2023","interfaces":[{"interface":"back","average":{"1min":3.9489999999999998,"5min":2.7519999999999998,"15min":2.4209999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.25700000000000001},"max":{"1min":32.810000000000002,"5min":32.810000000000002,"15min":63.508000000000003},"last":3.883},{"interface":"front","average":{"1min":3.8980000000000001,"5min":3.1280000000000001,"15min":2.605},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.26300000000000001},"max":{"1min":32.659999999999997,"5min":32.659999999999997,"15min":62.418999999999997},"last":3.8199999999999998}]},{"osd":4,"last update":"Mon Oct 9 10:59:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":0.71099999999999997,"15min":1.4350000000000001},"min":{"1min":0.29099999999999998,"5min":0.26000000000000001,"15min":0.23200000000000001},"max":{"1min":4.8760000000000003,"5min":4.8760000000000003,"15min":123.241},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.2370000000000001,"5min":0.96499999999999997,"15min":1.6819999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.24199999999999999},"max":{"1min":5.7480000000000002,"5min":5.7480000000000002,"15min":123.318},"last":3.8519999999999999}]},{"osd":6,"last update":"Mon Oct 9 10:59:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":1.0580000000000001,"15min":1.4059999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.883,"5min":8.9049999999999994,"15min":11.098000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.327,"5min":0.81299999999999994,"15min":1.2789999999999999},"min":{"1min":0.39400000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.899,"5min":5.899,"15min":20.256},"last":0.57499999999999996}]},{"osd":7,"last update":"Mon Oct 9 10:58:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5369999999999999,"5min":1.1759999999999999,"15min":1.3819999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28699999999999998},"max":{"1min":5.5010000000000003,"5min":8.468,"15min":10.739000000000001},"last":0.432},{"interface":"front","average":{"1min":1.448,"5min":0.96999999999999997,"15min":1.194},"min":{"1min":0.374,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.1070000000000002,"5min":8.734,"15min":14.154},"last":3.254}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1443102720,"data_stored":1443102720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3667136512,"data_stored":3667132440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1084358656,"data_stored":1084358656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2795896832,"data_stored":2795896832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1032519680,"data_stored":1032519680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3082096640,"data_stored":3082092568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1028390912,"data_stored":1028390912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3110866944,"data_stored":3110862872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T10:59:28.976 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T10:59:28.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_3 pg_num 26 2023-10-09T10:59:28.977 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:28 smithi155 ceph-mon[184854]: pgmap v914: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 2.3 KiB/s rd, 71 MiB/s wr, 1.89k op/s; 47313/253332 objects degraded (18.676%); 20725/253332 objects misplaced (8.181%); 7.2 MiB/s, 116 objects/s recovering 2023-10-09T10:59:28.977 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:28 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 47313/253332 objects degraded (18.676%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:29.059 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:29.098 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:29.127 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:29.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:29.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:29.234 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:29.271 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:29.349 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:29.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:29.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:29.368 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:29.455 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:29.465 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:29.509 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:29.535 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:29.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:29.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:29.643 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:29.741 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:29.766 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:59:29.771 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:29 smithi125 ceph-mon[195507]: from='client.45893 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:59:29.771 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:29 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/594465754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:29.771 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:29 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:29.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:29 smithi125 ceph-mon[200197]: from='client.45893 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:59:29.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:29 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/594465754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:29.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:29 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:29.874 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:29.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:29 smithi155 ceph-mon[184854]: from='client.45893 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T10:59:29.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:29 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/594465754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:29.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:29 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:29.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:29.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:29.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:30.042 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:30.077 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:30.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:30.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:30.186 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:30.188 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:59:30.188 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:59:30.188 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:59:30.188 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:59:30.188 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.189 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:152608 7.b 7:d0fd2adb:::benchmark_data_smithi125_295816_object61471:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e245)", 2023-10-09T10:59:30.189 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:52:10.363211+0000", 2023-10-09T10:59:30.189 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 439.81642295099999, 2023-10-09T10:59:30.189 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8185187439999999, 2023-10-09T10:59:30.189 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.189 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.189 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:30.189 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:59:30.189 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:30.189 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 152608 2023-10-09T10:59:30.189 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.190 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.190 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.190 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.190 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363211+0000", 2023-10-09T10:59:30.190 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.190 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.190 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.190 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.190 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363211+0000", 2023-10-09T10:59:30.190 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.190 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.190 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.191 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.191 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363209+0000", 2023-10-09T10:59:30.191 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:59:30.191 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.191 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.191 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.191 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363218+0000", 2023-10-09T10:59:30.191 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8470000000000002e-06 2023-10-09T10:59:30.191 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.191 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.191 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.192 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363236+0000", 2023-10-09T10:59:30.192 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8062000000000001e-05 2023-10-09T10:59:30.192 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.192 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.192 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.192 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363244+0000", 2023-10-09T10:59:30.192 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2560000000000002e-06 2023-10-09T10:59:30.192 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.192 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.192 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.192 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.365179+0000", 2023-10-09T10:59:30.192 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001934755 2023-10-09T10:59:30.192 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.193 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.193 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op must wait for map", 2023-10-09T10:59:30.193 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.365188+0000", 2023-10-09T10:59:30.193 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.132e-06 2023-10-09T10:59:30.193 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.193 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.193 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.193 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.380926+0000", 2023-10-09T10:59:30.193 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015737860999999999 2023-10-09T10:59:30.193 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.193 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.193 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:59:30.194 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.380936+0000", 2023-10-09T10:59:30.194 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.679e-06 2023-10-09T10:59:30.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.194 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.194 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.172400+0000", 2023-10-09T10:59:30.194 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7914640509999999 2023-10-09T10:59:30.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.194 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.194 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.172506+0000", 2023-10-09T10:59:30.194 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106677 2023-10-09T10:59:30.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T10:59:30.195 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.172597+0000", 2023-10-09T10:59:30.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0718999999999997e-05 2023-10-09T10:59:30.195 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.195 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.176419+0000", 2023-10-09T10:59:30.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0038220480000000002 2023-10-09T10:59:30.195 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.196 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.176443+0000", 2023-10-09T10:59:30.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3583000000000001e-05 2023-10-09T10:59:30.196 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.196 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.196 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.196 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.176459+0000", 2023-10-09T10:59:30.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5815999999999999e-05 2023-10-09T10:59:30.196 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.196 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.196 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:30.196 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.181617+0000", 2023-10-09T10:59:30.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051585210000000001 2023-10-09T10:59:30.196 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.197 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.197 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.181654+0000", 2023-10-09T10:59:30.197 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6573999999999999e-05 2023-10-09T10:59:30.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.197 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.197 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.181730+0000", 2023-10-09T10:59:30.197 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6070999999999997e-05 2023-10-09T10:59:30.197 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.197 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.197 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.198 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.198 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:39213 8.2 8:4e7be898:::benchmark_data_smithi125_329074_object39212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T10:59:30.198 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:53:35.895072+0000", 2023-10-09T10:59:30.198 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 354.28456235099998, 2023-10-09T10:59:30.198 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.128363015, 2023-10-09T10:59:30.198 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.198 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.198 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:30.198 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:59:30.198 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:30.198 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39213 2023-10-09T10:59:30.198 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.199 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.199 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.199 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.199 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895072+0000", 2023-10-09T10:59:30.199 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.199 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.199 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.199 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.199 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895072+0000", 2023-10-09T10:59:30.199 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.199 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.199 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.199 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.200 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895070+0000", 2023-10-09T10:59:30.200 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:59:30.200 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.200 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.200 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.200 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895204+0000", 2023-10-09T10:59:30.200 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000133561 2023-10-09T10:59:30.200 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.200 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.200 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.200 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895215+0000", 2023-10-09T10:59:30.200 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0925e-05 2023-10-09T10:59:30.201 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.201 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895220+0000", 2023-10-09T10:59:30.201 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4949999999999999e-06 2023-10-09T10:59:30.201 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.201 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895798+0000", 2023-10-09T10:59:30.201 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00057733700000000003 2023-10-09T10:59:30.201 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895926+0000", 2023-10-09T10:59:30.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012778600000000001 2023-10-09T10:59:30.202 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.202 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.202 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:59:30.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.896072+0000", 2023-10-09T10:59:30.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014615599999999999 2023-10-09T10:59:30.202 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.202 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.202 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:30.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.897973+0000", 2023-10-09T10:59:30.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019008510000000001 2023-10-09T10:59:30.202 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.203 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.203 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.203 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.899072+0000", 2023-10-09T10:59:30.203 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010995239999999999 2023-10-09T10:59:30.203 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.203 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.203 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.203 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023353+0000", 2023-10-09T10:59:30.203 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.124281323 2023-10-09T10:59:30.203 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.203 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.204 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.204 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023405+0000", 2023-10-09T10:59:30.204 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1158e-05 2023-10-09T10:59:30.204 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.204 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.204 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.204 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023435+0000", 2023-10-09T10:59:30.204 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.006e-05 2023-10-09T10:59:30.204 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.204 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.204 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.204 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.204 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.205 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83954 8.4 8:2689e239:::benchmark_data_smithi125_329074_object83953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T10:59:30.205 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.108148+0000", 2023-10-09T10:59:30.205 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 313.07148619899999, 2023-10-09T10:59:30.205 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1275014759999999, 2023-10-09T10:59:30.205 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.205 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.205 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:30.205 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:59:30.205 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:30.205 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83954 2023-10-09T10:59:30.205 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.206 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.206 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.206 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.206 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108148+0000", 2023-10-09T10:59:30.206 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.206 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.206 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.206 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.206 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108148+0000", 2023-10-09T10:59:30.206 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.206 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.207 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.207 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.207 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108146+0000", 2023-10-09T10:59:30.207 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:30.207 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.207 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.207 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.207 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108191+0000", 2023-10-09T10:59:30.207 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4549000000000003e-05 2023-10-09T10:59:30.207 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.207 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.207 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.208 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108215+0000", 2023-10-09T10:59:30.208 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3994000000000001e-05 2023-10-09T10:59:30.208 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.208 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.208 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.208 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108222+0000", 2023-10-09T10:59:30.208 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4499999999999998e-06 2023-10-09T10:59:30.208 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.208 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.208 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.208 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108279+0000", 2023-10-09T10:59:30.209 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6643e-05 2023-10-09T10:59:30.209 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.209 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.209 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:59:30.209 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108288+0000", 2023-10-09T10:59:30.209 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5109999999999992e-06 2023-10-09T10:59:30.209 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.209 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.209 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.209 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.872892+0000", 2023-10-09T10:59:30.209 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.76460362800000004 2023-10-09T10:59:30.210 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.210 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.210 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:59:30.210 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.872903+0000", 2023-10-09T10:59:30.210 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1158000000000001e-05 2023-10-09T10:59:30.210 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.210 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.210 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.210 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213528+0000", 2023-10-09T10:59:30.210 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.340624501 2023-10-09T10:59:30.210 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.211 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.211 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.211 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213679+0000", 2023-10-09T10:59:30.211 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151578 2023-10-09T10:59:30.211 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.211 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.211 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T10:59:30.211 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213861+0000", 2023-10-09T10:59:30.211 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018154 2023-10-09T10:59:30.211 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.211 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.211 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:30.211 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219288+0000", 2023-10-09T10:59:30.212 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00542677 2023-10-09T10:59:30.212 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.212 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.212 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.212 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219340+0000", 2023-10-09T10:59:30.212 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2657000000000001e-05 2023-10-09T10:59:30.212 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.212 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.212 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.212 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219644+0000", 2023-10-09T10:59:30.212 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030375299999999998 2023-10-09T10:59:30.213 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.213 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.213 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.213 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235530+0000", 2023-10-09T10:59:30.213 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01588606 2023-10-09T10:59:30.213 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.213 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.213 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.213 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235575+0000", 2023-10-09T10:59:30.213 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5012999999999997e-05 2023-10-09T10:59:30.213 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.213 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.214 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.214 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235649+0000", 2023-10-09T10:59:30.214 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4252000000000007e-05 2023-10-09T10:59:30.214 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.214 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.214 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.214 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.214 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.214 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83959 8.4 8:27b6f913:::benchmark_data_smithi125_329074_object83958:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T10:59:30.214 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.112252+0000", 2023-10-09T10:59:30.214 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 313.06738216899998, 2023-10-09T10:59:30.214 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1235403750000001, 2023-10-09T10:59:30.215 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.215 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.215 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:30.215 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:59:30.215 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:30.215 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83959 2023-10-09T10:59:30.215 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.215 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.215 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.215 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.215 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112252+0000", 2023-10-09T10:59:30.215 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.216 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.216 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.216 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.216 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112252+0000", 2023-10-09T10:59:30.216 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.216 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.216 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.216 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.216 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112251+0000", 2023-10-09T10:59:30.216 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:59:30.216 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.216 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.217 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.217 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112312+0000", 2023-10-09T10:59:30.217 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0888999999999999e-05 2023-10-09T10:59:30.217 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.217 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.217 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.217 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112324+0000", 2023-10-09T10:59:30.217 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2585999999999999e-05 2023-10-09T10:59:30.217 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.217 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.217 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.217 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112329+0000", 2023-10-09T10:59:30.218 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9880000000000004e-06 2023-10-09T10:59:30.218 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.218 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.218 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.218 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112378+0000", 2023-10-09T10:59:30.218 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8905e-05 2023-10-09T10:59:30.218 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.218 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.218 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:59:30.218 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112389+0000", 2023-10-09T10:59:30.218 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0665e-05 2023-10-09T10:59:30.219 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.219 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.219 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.219 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.873769+0000", 2023-10-09T10:59:30.219 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.76138013699999996 2023-10-09T10:59:30.219 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.219 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.219 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T10:59:30.219 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.873776+0000", 2023-10-09T10:59:30.219 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.17e-06 2023-10-09T10:59:30.220 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.220 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.220 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.220 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216109+0000", 2023-10-09T10:59:30.220 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.342332699 2023-10-09T10:59:30.220 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.220 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.220 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.220 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216239+0000", 2023-10-09T10:59:30.220 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012997 2023-10-09T10:59:30.220 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.220 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.221 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T10:59:30.221 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216354+0000", 2023-10-09T10:59:30.221 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115116 2023-10-09T10:59:30.221 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.221 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.221 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.221 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219398+0000", 2023-10-09T10:59:30.221 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030438679999999999 2023-10-09T10:59:30.221 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.221 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.221 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:30.221 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219407+0000", 2023-10-09T10:59:30.222 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1870000000000003e-06 2023-10-09T10:59:30.222 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.222 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.222 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.222 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219682+0000", 2023-10-09T10:59:30.222 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00027478600000000001 2023-10-09T10:59:30.222 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.222 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.222 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.222 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235686+0000", 2023-10-09T10:59:30.223 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016004852 2023-10-09T10:59:30.223 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.223 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.223 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.223 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235725+0000", 2023-10-09T10:59:30.223 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8392000000000001e-05 2023-10-09T10:59:30.223 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.223 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.223 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.223 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235792+0000", 2023-10-09T10:59:30.223 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7311000000000007e-05 2023-10-09T10:59:30.223 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.223 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.224 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.224 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.224 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.224 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:93330 8.2 8:4f5a2bcc:::benchmark_data_smithi125_329074_object93329:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T10:59:30.224 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:35.625993+0000", 2023-10-09T10:59:30.224 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 294.55364128799999, 2023-10-09T10:59:30.224 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.12993565000000001, 2023-10-09T10:59:30.224 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.224 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.224 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:30.224 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:59:30.224 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:30.225 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 93330 2023-10-09T10:59:30.225 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.225 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.225 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.225 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.225 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625993+0000", 2023-10-09T10:59:30.225 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.225 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.225 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.225 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.225 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625993+0000", 2023-10-09T10:59:30.226 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.226 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.226 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.226 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.226 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625992+0000", 2023-10-09T10:59:30.226 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:30.226 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.226 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.226 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.226 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626043+0000", 2023-10-09T10:59:30.226 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0500000000000001e-05 2023-10-09T10:59:30.226 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.226 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.227 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.227 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626051+0000", 2023-10-09T10:59:30.227 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0509999999999997e-06 2023-10-09T10:59:30.227 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.227 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.227 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.227 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626056+0000", 2023-10-09T10:59:30.227 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9570000000000001e-06 2023-10-09T10:59:30.227 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.227 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.227 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.227 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626113+0000", 2023-10-09T10:59:30.228 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.736e-05 2023-10-09T10:59:30.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.228 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.228 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.228 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626283+0000", 2023-10-09T10:59:30.228 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017031699999999999 2023-10-09T10:59:30.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.228 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.228 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:59:30.228 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626463+0000", 2023-10-09T10:59:30.228 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018018 2023-10-09T10:59:30.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.228 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:30.229 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.634095+0000", 2023-10-09T10:59:30.229 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0076312469999999999 2023-10-09T10:59:30.229 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.229 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.637111+0000", 2023-10-09T10:59:30.229 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003016725 2023-10-09T10:59:30.229 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.230 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755853+0000", 2023-10-09T10:59:30.230 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11874143299999999 2023-10-09T10:59:30.230 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.230 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.230 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.230 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755887+0000", 2023-10-09T10:59:30.230 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4017999999999999e-05 2023-10-09T10:59:30.230 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.230 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.230 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.231 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755928+0000", 2023-10-09T10:59:30.231 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1452999999999998e-05 2023-10-09T10:59:30.231 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.231 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.231 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.231 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.231 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.231 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:93340 8.2 8:4653dd45:::benchmark_data_smithi125_329074_object93339:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T10:59:30.231 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:35.641334+0000", 2023-10-09T10:59:30.231 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 294.53830034800001, 2023-10-09T10:59:30.232 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.125653775, 2023-10-09T10:59:30.232 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.232 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.232 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:30.232 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:59:30.232 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:30.232 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 93340 2023-10-09T10:59:30.232 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.232 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.232 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.233 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.233 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641334+0000", 2023-10-09T10:59:30.233 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.233 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.233 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.233 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641334+0000", 2023-10-09T10:59:30.233 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.233 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.233 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.233 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641333+0000", 2023-10-09T10:59:30.234 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:59:30.234 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.234 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.234 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.234 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641362+0000", 2023-10-09T10:59:30.234 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9034999999999999e-05 2023-10-09T10:59:30.234 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.234 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.234 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.234 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641372+0000", 2023-10-09T10:59:30.234 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0597e-05 2023-10-09T10:59:30.235 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.235 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.235 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.235 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641377+0000", 2023-10-09T10:59:30.235 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7060000000000003e-06 2023-10-09T10:59:30.235 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.235 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.235 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.235 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641429+0000", 2023-10-09T10:59:30.235 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1721000000000002e-05 2023-10-09T10:59:30.235 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.235 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.236 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.236 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641529+0000", 2023-10-09T10:59:30.236 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000100602 2023-10-09T10:59:30.236 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.236 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.236 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:59:30.236 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641632+0000", 2023-10-09T10:59:30.236 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103099 2023-10-09T10:59:30.236 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.236 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.236 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.236 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.649088+0000", 2023-10-09T10:59:30.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0074551569999999996 2023-10-09T10:59:30.237 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:30.237 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.649159+0000", 2023-10-09T10:59:30.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1760000000000004e-05 2023-10-09T10:59:30.237 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.237 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766886+0000", 2023-10-09T10:59:30.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11772616499999999 2023-10-09T10:59:30.237 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.238 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766941+0000", 2023-10-09T10:59:30.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5186000000000003e-05 2023-10-09T10:59:30.238 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.238 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.238 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766987+0000", 2023-10-09T10:59:30.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6692e-05 2023-10-09T10:59:30.238 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.238 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.238 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.239 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.239 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.239 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:158888 8.2 8:4103043a:::benchmark_data_smithi125_329074_object62234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2023-10-09T10:59:30.239 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:56:15.356379+0000", 2023-10-09T10:59:30.239 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 194.82325499500001, 2023-10-09T10:59:30.239 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.123441579, 2023-10-09T10:59:30.239 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.239 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.239 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:30.239 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T10:59:30.239 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T10:59:30.239 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 158888 2023-10-09T10:59:30.240 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.240 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.240 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.240 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.240 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T10:59:30.240 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.240 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.240 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.240 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.240 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T10:59:30.240 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.241 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.241 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.241 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.241 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T10:59:30.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:30.241 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.241 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.241 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.241 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356381+0000", 2023-10-09T10:59:30.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.734e-06 2023-10-09T10:59:30.241 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.241 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.242 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.242 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356388+0000", 2023-10-09T10:59:30.242 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4559999999999997e-06 2023-10-09T10:59:30.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.242 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.242 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.242 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356392+0000", 2023-10-09T10:59:30.242 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0589999999999996e-06 2023-10-09T10:59:30.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.242 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.242 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.242 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.359480+0000", 2023-10-09T10:59:30.242 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030878379999999999 2023-10-09T10:59:30.243 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.243 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.243 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.359628+0000", 2023-10-09T10:59:30.243 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014820299999999999 2023-10-09T10:59:30.243 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.243 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T10:59:30.243 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.360074+0000", 2023-10-09T10:59:30.243 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00044630599999999999 2023-10-09T10:59:30.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.244 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.244 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:30.244 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.362052+0000", 2023-10-09T10:59:30.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019782720000000001 2023-10-09T10:59:30.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.244 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.244 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.244 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.362671+0000", 2023-10-09T10:59:30.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00061827099999999999 2023-10-09T10:59:30.245 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.245 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.245 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:30.245 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479765+0000", 2023-10-09T10:59:30.245 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11709412199999999 2023-10-09T10:59:30.245 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.245 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.245 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.245 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479796+0000", 2023-10-09T10:59:30.245 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0895000000000002e-05 2023-10-09T10:59:30.245 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.245 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.246 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.246 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479821+0000", 2023-10-09T10:59:30.246 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4816000000000001e-05 2023-10-09T10:59:30.246 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.246 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.246 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.246 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.246 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.246 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:89677 9.e e351/332 9:72edb1d1:::benchmark_data_smithi125_363789_object89676:head v 351'5733, mlcod=351'5730)", 2023-10-09T10:59:30.246 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:30.255231+0000", 2023-10-09T10:59:30.247 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 59.924403087000002, 2023-10-09T10:59:30.247 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022306585, 2023-10-09T10:59:30.247 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.247 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.247 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.247 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.247 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.247 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:30.255231+0000", 2023-10-09T10:59:30.247 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.247 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.247 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.247 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.247 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:30.255231+0000", 2023-10-09T10:59:30.248 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.248 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.248 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.248 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.248 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:30.255229+0000", 2023-10-09T10:59:30.248 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:30.248 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.248 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.248 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.248 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:30.255259+0000", 2023-10-09T10:59:30.248 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9196e-05 2023-10-09T10:59:30.248 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.249 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.249 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.249 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:30.255279+0000", 2023-10-09T10:59:30.249 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9953e-05 2023-10-09T10:59:30.249 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.249 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.249 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.249 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:30.255285+0000", 2023-10-09T10:59:30.249 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8170000000000001e-06 2023-10-09T10:59:30.249 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.249 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.249 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.249 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:30.255915+0000", 2023-10-09T10:59:30.250 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00062924300000000003 2023-10-09T10:59:30.250 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.250 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.250 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.250 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:30.255937+0000", 2023-10-09T10:59:30.250 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2370000000000001e-05 2023-10-09T10:59:30.250 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.250 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.250 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.250 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:30.277490+0000", 2023-10-09T10:59:30.250 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0215529 2023-10-09T10:59:30.250 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.251 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.251 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.251 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:30.277537+0000", 2023-10-09T10:59:30.251 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7503999999999998e-05 2023-10-09T10:59:30.251 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.251 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.251 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.251 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.251 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.251 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:91250 9.4 e352/330 9:2ffb6675:::benchmark_data_smithi125_363789_object91249:head v 352'5721, mlcod=352'5718)", 2023-10-09T10:59:30.251 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:32.125408+0000", 2023-10-09T10:59:30.251 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 58.054225539999997, 2023-10-09T10:59:30.251 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021426303000000001, 2023-10-09T10:59:30.252 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.252 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.252 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.252 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.252 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.252 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:32.125408+0000", 2023-10-09T10:59:30.252 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.252 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.252 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.252 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.252 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:32.125408+0000", 2023-10-09T10:59:30.252 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.253 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.253 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.253 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.253 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:32.125408+0000", 2023-10-09T10:59:30.253 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:30.253 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.253 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.253 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.253 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:32.125484+0000", 2023-10-09T10:59:30.253 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6035e-05 2023-10-09T10:59:30.253 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.253 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.253 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.254 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:32.125498+0000", 2023-10-09T10:59:30.254 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3769e-05 2023-10-09T10:59:30.254 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.254 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.254 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.254 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:32.125503+0000", 2023-10-09T10:59:30.254 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6899999999999997e-06 2023-10-09T10:59:30.254 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.254 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.254 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.254 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:32.129171+0000", 2023-10-09T10:59:30.255 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0036681399999999999 2023-10-09T10:59:30.255 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.255 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.255 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.255 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:32.129190+0000", 2023-10-09T10:59:30.255 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8229000000000001e-05 2023-10-09T10:59:30.255 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.255 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.255 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.255 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:32.146791+0000", 2023-10-09T10:59:30.255 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017601135 2023-10-09T10:59:30.255 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.256 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.256 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.256 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:32.146835+0000", 2023-10-09T10:59:30.256 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3968999999999999e-05 2023-10-09T10:59:30.256 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.256 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.256 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.256 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.256 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.256 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:99655 9.4 e353/330 9:2ab25dc7:::benchmark_data_smithi125_363789_object99654:head v 353'6243, mlcod=353'6240)", 2023-10-09T10:59:30.256 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:41.890549+0000", 2023-10-09T10:59:30.256 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 48.289085149000002, 2023-10-09T10:59:30.256 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0204247, 2023-10-09T10:59:30.257 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.257 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.257 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.257 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.257 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.257 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:41.890549+0000", 2023-10-09T10:59:30.257 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.257 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.257 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.257 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.257 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:41.890549+0000", 2023-10-09T10:59:30.257 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.257 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.258 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.258 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.258 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:41.890548+0000", 2023-10-09T10:59:30.258 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:59:30.258 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.258 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.258 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.258 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:41.890587+0000", 2023-10-09T10:59:30.258 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9728999999999999e-05 2023-10-09T10:59:30.258 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.258 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.258 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.259 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:41.890606+0000", 2023-10-09T10:59:30.259 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8057999999999999e-05 2023-10-09T10:59:30.259 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.259 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.259 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.259 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:41.890612+0000", 2023-10-09T10:59:30.259 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5220000000000004e-06 2023-10-09T10:59:30.259 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.259 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.259 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.259 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:41.894543+0000", 2023-10-09T10:59:30.259 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039313289999999999 2023-10-09T10:59:30.259 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.259 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.260 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.260 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:41.894583+0000", 2023-10-09T10:59:30.260 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9137999999999997e-05 2023-10-09T10:59:30.260 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.260 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.260 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.260 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:41.910928+0000", 2023-10-09T10:59:30.260 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016345284000000002 2023-10-09T10:59:30.260 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.260 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.260 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.260 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:41.910974+0000", 2023-10-09T10:59:30.261 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5692000000000002e-05 2023-10-09T10:59:30.261 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.261 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.261 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.261 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.261 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.261 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:100843 9.c e353/332 9:3d29ba4c:::benchmark_data_smithi125_363789_object100842:head v 353'6258, mlcod=353'6256)", 2023-10-09T10:59:30.261 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:43.227548+0000", 2023-10-09T10:59:30.261 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 46.952086102000003, 2023-10-09T10:59:30.261 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0253425, 2023-10-09T10:59:30.261 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.261 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.261 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.262 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.262 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.262 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227548+0000", 2023-10-09T10:59:30.262 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.262 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.262 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.262 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.262 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227548+0000", 2023-10-09T10:59:30.262 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.262 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.262 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.262 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.262 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227547+0000", 2023-10-09T10:59:30.263 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:59:30.263 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.263 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.263 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.263 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227594+0000", 2023-10-09T10:59:30.263 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7648000000000002e-05 2023-10-09T10:59:30.263 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.263 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.263 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.263 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227615+0000", 2023-10-09T10:59:30.263 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0435e-05 2023-10-09T10:59:30.263 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.264 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.264 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.264 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227622+0000", 2023-10-09T10:59:30.264 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4850000000000003e-06 2023-10-09T10:59:30.264 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.264 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.264 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.264 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.229957+0000", 2023-10-09T10:59:30.264 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002334381 2023-10-09T10:59:30.264 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.264 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.264 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.265 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.229976+0000", 2023-10-09T10:59:30.265 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9596999999999999e-05 2023-10-09T10:59:30.265 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.265 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.265 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.265 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.252847+0000", 2023-10-09T10:59:30.265 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022871209 2023-10-09T10:59:30.265 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.265 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.265 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.265 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.252890+0000", 2023-10-09T10:59:30.265 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2920000000000002e-05 2023-10-09T10:59:30.266 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.266 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.266 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.266 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.266 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.266 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:102485 9.c e353/332 9:3d3829d9:::benchmark_data_smithi125_363789_object102484:head v 353'6351, mlcod=353'6348)", 2023-10-09T10:59:30.266 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:45.158190+0000", 2023-10-09T10:59:30.266 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 45.021444137000003, 2023-10-09T10:59:30.266 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024704652000000001, 2023-10-09T10:59:30.266 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.266 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.266 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.266 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.267 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.267 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158190+0000", 2023-10-09T10:59:30.267 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.267 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.267 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.267 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.267 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158190+0000", 2023-10-09T10:59:30.267 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.267 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.267 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.267 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.267 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158189+0000", 2023-10-09T10:59:30.267 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:30.268 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.268 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.268 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.268 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158233+0000", 2023-10-09T10:59:30.268 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3201999999999997e-05 2023-10-09T10:59:30.268 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.268 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.268 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.268 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158242+0000", 2023-10-09T10:59:30.268 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9400000000000008e-06 2023-10-09T10:59:30.268 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.268 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158248+0000", 2023-10-09T10:59:30.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.844e-06 2023-10-09T10:59:30.269 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.269 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.161725+0000", 2023-10-09T10:59:30.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034765450000000002 2023-10-09T10:59:30.269 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.269 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.161750+0000", 2023-10-09T10:59:30.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4538000000000001e-05 2023-10-09T10:59:30.270 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.270 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.270 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.270 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.182847+0000", 2023-10-09T10:59:30.270 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.02109778 2023-10-09T10:59:30.270 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.270 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.270 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.270 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.182894+0000", 2023-10-09T10:59:30.270 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7141000000000001e-05 2023-10-09T10:59:30.270 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.271 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.271 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.271 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.271 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.271 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:102487 9.c e353/332 9:37d8c2ee:::benchmark_data_smithi125_363789_object102486:head v 353'6352, mlcod=353'6348)", 2023-10-09T10:59:30.271 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:45.158254+0000", 2023-10-09T10:59:30.271 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 45.021379818, 2023-10-09T10:59:30.271 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027965380000000001, 2023-10-09T10:59:30.271 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.271 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.271 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.271 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.271 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.272 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158254+0000", 2023-10-09T10:59:30.272 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.272 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.272 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.272 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.272 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158254+0000", 2023-10-09T10:59:30.272 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.272 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.272 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.272 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.273 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158254+0000", 2023-10-09T10:59:30.273 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:30.273 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.273 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.273 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.273 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158280+0000", 2023-10-09T10:59:30.273 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6463000000000001e-05 2023-10-09T10:59:30.273 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.273 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.273 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.273 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158296+0000", 2023-10-09T10:59:30.274 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6246999999999999e-05 2023-10-09T10:59:30.274 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.274 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.274 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.274 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158305+0000", 2023-10-09T10:59:30.274 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9639999999999992e-06 2023-10-09T10:59:30.274 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.274 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.274 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.274 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165187+0000", 2023-10-09T10:59:30.274 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0068813679999999997 2023-10-09T10:59:30.274 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.274 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.275 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.275 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165209+0000", 2023-10-09T10:59:30.275 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1926999999999999e-05 2023-10-09T10:59:30.275 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.275 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.275 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.275 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186167+0000", 2023-10-09T10:59:30.275 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020958641 2023-10-09T10:59:30.275 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.275 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.275 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.275 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186220+0000", 2023-10-09T10:59:30.276 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2243000000000001e-05 2023-10-09T10:59:30.276 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.276 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.276 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.276 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.276 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.276 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:102490 9.4 e353/330 9:207553f4:::benchmark_data_smithi125_363789_object102489:head v 353'6429, mlcod=353'6425)", 2023-10-09T10:59:30.276 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:45.159295+0000", 2023-10-09T10:59:30.276 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 45.020339001000004, 2023-10-09T10:59:30.276 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026994220999999999, 2023-10-09T10:59:30.276 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.277 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.277 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.277 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.277 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159295+0000", 2023-10-09T10:59:30.277 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.277 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.277 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.277 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159295+0000", 2023-10-09T10:59:30.277 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.277 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.277 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.278 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159294+0000", 2023-10-09T10:59:30.278 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:30.278 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.278 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.278 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.278 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159332+0000", 2023-10-09T10:59:30.278 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7502000000000001e-05 2023-10-09T10:59:30.278 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.278 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.278 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.278 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159344+0000", 2023-10-09T10:59:30.278 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2418999999999999e-05 2023-10-09T10:59:30.279 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.279 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.279 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.279 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159349+0000", 2023-10-09T10:59:30.279 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2619999999999998e-06 2023-10-09T10:59:30.279 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.279 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.279 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.279 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165240+0000", 2023-10-09T10:59:30.279 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0058900580000000001 2023-10-09T10:59:30.279 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.279 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.279 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.279 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165259+0000", 2023-10-09T10:59:30.280 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9369e-05 2023-10-09T10:59:30.280 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.280 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.280 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.280 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186223+0000", 2023-10-09T10:59:30.280 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020963682000000001 2023-10-09T10:59:30.280 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.280 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.280 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.280 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186289+0000", 2023-10-09T10:59:30.280 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6600000000000006e-05 2023-10-09T10:59:30.280 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.280 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.280 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.281 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.281 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.281 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:102494 9.4 e353/330 9:2c49d3a3:::benchmark_data_smithi125_363789_object102493:head v 353'6430, mlcod=353'6425)", 2023-10-09T10:59:30.281 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:45.162300+0000", 2023-10-09T10:59:30.281 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 45.017333682, 2023-10-09T10:59:30.281 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024081163999999999, 2023-10-09T10:59:30.281 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.281 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.281 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.281 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.281 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.281 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162300+0000", 2023-10-09T10:59:30.281 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.281 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.282 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.282 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.282 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162300+0000", 2023-10-09T10:59:30.282 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.282 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.282 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.282 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.282 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162299+0000", 2023-10-09T10:59:30.282 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:59:30.282 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.282 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.282 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.282 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162360+0000", 2023-10-09T10:59:30.283 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1148999999999993e-05 2023-10-09T10:59:30.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.283 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162375+0000", 2023-10-09T10:59:30.283 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.469e-05 2023-10-09T10:59:30.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.283 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162383+0000", 2023-10-09T10:59:30.283 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3289999999999996e-06 2023-10-09T10:59:30.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.284 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.168660+0000", 2023-10-09T10:59:30.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062775629999999999 2023-10-09T10:59:30.284 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.284 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.168682+0000", 2023-10-09T10:59:30.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1498999999999999e-05 2023-10-09T10:59:30.284 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.284 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186292+0000", 2023-10-09T10:59:30.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017610388000000001 2023-10-09T10:59:30.285 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.285 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.285 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186381+0000", 2023-10-09T10:59:30.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9489999999999999e-05 2023-10-09T10:59:30.285 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.285 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.285 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.285 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.285 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:102496 9.4 e353/330 9:23ba7b37:::benchmark_data_smithi125_363789_object102495:head v 353'6431, mlcod=353'6425)", 2023-10-09T10:59:30.286 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:45.163861+0000", 2023-10-09T10:59:30.286 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 45.015772634999998, 2023-10-09T10:59:30.286 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022571760999999999, 2023-10-09T10:59:30.286 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.286 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.286 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.286 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.286 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.286 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.163861+0000", 2023-10-09T10:59:30.286 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.286 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.286 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.287 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.287 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.163861+0000", 2023-10-09T10:59:30.287 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.287 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.287 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.287 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.287 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.163860+0000", 2023-10-09T10:59:30.287 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:30.287 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.287 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.287 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.287 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.163916+0000", 2023-10-09T10:59:30.288 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6226e-05 2023-10-09T10:59:30.288 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.288 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.288 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.288 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.163930+0000", 2023-10-09T10:59:30.288 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3712e-05 2023-10-09T10:59:30.288 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.288 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.288 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.288 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.163937+0000", 2023-10-09T10:59:30.288 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6490000000000002e-06 2023-10-09T10:59:30.288 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.288 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.289 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.289 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.179474+0000", 2023-10-09T10:59:30.289 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015537252 2023-10-09T10:59:30.289 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.289 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.289 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.289 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.179503+0000", 2023-10-09T10:59:30.289 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9193999999999999e-05 2023-10-09T10:59:30.289 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.289 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.289 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.289 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186386+0000", 2023-10-09T10:59:30.289 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0068830760000000001 2023-10-09T10:59:30.289 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.290 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.290 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.290 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186433+0000", 2023-10-09T10:59:30.290 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6907000000000003e-05 2023-10-09T10:59:30.290 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.290 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.290 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.290 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.290 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.290 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:110482 9.c e355/332 9:31924eda:::benchmark_data_smithi125_363789_object183:head v 355'6835, mlcod=355'6833)", 2023-10-09T10:59:30.290 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:54.892574+0000", 2023-10-09T10:59:30.291 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 35.287059704000001, 2023-10-09T10:59:30.291 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020061527999999999, 2023-10-09T10:59:30.291 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.291 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.291 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.291 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.291 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.291 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:54.892574+0000", 2023-10-09T10:59:30.291 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.291 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.291 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.292 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.292 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:54.892574+0000", 2023-10-09T10:59:30.292 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.292 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.292 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.292 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.292 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:54.892573+0000", 2023-10-09T10:59:30.292 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:30.292 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.292 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.292 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.292 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:54.892585+0000", 2023-10-09T10:59:30.292 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2194999999999999e-05 2023-10-09T10:59:30.292 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.293 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.293 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.293 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:54.892607+0000", 2023-10-09T10:59:30.293 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2376000000000001e-05 2023-10-09T10:59:30.293 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.293 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.293 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.293 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:54.892615+0000", 2023-10-09T10:59:30.293 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9570000000000002e-06 2023-10-09T10:59:30.293 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.293 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.293 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.294 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:54.892665+0000", 2023-10-09T10:59:30.294 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9549999999999998e-05 2023-10-09T10:59:30.294 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.294 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.294 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.294 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:54.892687+0000", 2023-10-09T10:59:30.294 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1914999999999999e-05 2023-10-09T10:59:30.294 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.294 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.294 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.294 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:54.912589+0000", 2023-10-09T10:59:30.295 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019902416999999999 2023-10-09T10:59:30.295 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.295 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.295 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.295 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:54.912636+0000", 2023-10-09T10:59:30.295 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6746999999999998e-05 2023-10-09T10:59:30.295 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.295 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.295 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.295 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.295 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.295 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:122920 9.4 e359/358 9:288551e9:::benchmark_data_smithi125_363789_object12621:head v 359'7671, mlcod=359'7667)", 2023-10-09T10:59:30.296 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:10.079747+0000", 2023-10-09T10:59:30.296 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 20.099887047999999, 2023-10-09T10:59:30.296 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020477229999999999, 2023-10-09T10:59:30.296 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.296 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.296 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.296 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.296 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.296 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:10.079747+0000", 2023-10-09T10:59:30.296 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.296 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.296 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.296 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.297 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:10.079747+0000", 2023-10-09T10:59:30.297 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.297 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.297 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.297 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.297 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:10.079746+0000", 2023-10-09T10:59:30.297 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:59:30.297 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.297 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.297 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.297 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:10.079757+0000", 2023-10-09T10:59:30.297 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1034e-05 2023-10-09T10:59:30.298 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.298 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.298 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.298 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:10.079773+0000", 2023-10-09T10:59:30.298 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5662e-05 2023-10-09T10:59:30.298 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.298 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.298 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.298 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:10.079779+0000", 2023-10-09T10:59:30.298 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4849999999999997e-06 2023-10-09T10:59:30.298 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.299 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.299 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:10.080154+0000", 2023-10-09T10:59:30.299 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00037506099999999999 2023-10-09T10:59:30.299 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.299 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.299 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:10.080175+0000", 2023-10-09T10:59:30.299 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0554999999999999e-05 2023-10-09T10:59:30.299 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.299 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.300 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:10.100177+0000", 2023-10-09T10:59:30.300 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020002087000000002 2023-10-09T10:59:30.300 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.300 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.300 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.300 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:10.100224+0000", 2023-10-09T10:59:30.300 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7231000000000002e-05 2023-10-09T10:59:30.300 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.300 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.300 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.300 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.300 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.300 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:129219 9.4 e360/358 9:200b90fd:::benchmark_data_smithi125_363789_object18920:head v 360'8063, mlcod=360'8061)", 2023-10-09T10:59:30.301 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:16.561667+0000", 2023-10-09T10:59:30.301 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 13.617967096999999, 2023-10-09T10:59:30.301 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021403095, 2023-10-09T10:59:30.301 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.301 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.301 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.301 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.301 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.301 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:16.561667+0000", 2023-10-09T10:59:30.301 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.302 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.302 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.302 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.302 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:16.561667+0000", 2023-10-09T10:59:30.302 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.302 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.302 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.302 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.302 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:16.561666+0000", 2023-10-09T10:59:30.302 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:30.302 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.302 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.303 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.303 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:16.561674+0000", 2023-10-09T10:59:30.303 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7539999999999995e-06 2023-10-09T10:59:30.303 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.303 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.303 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.303 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:16.561686+0000", 2023-10-09T10:59:30.303 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1249e-05 2023-10-09T10:59:30.303 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.303 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.303 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.303 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:16.561692+0000", 2023-10-09T10:59:30.304 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1990000000000003e-06 2023-10-09T10:59:30.304 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.304 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.304 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.304 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:16.561741+0000", 2023-10-09T10:59:30.304 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9539000000000001e-05 2023-10-09T10:59:30.304 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.304 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.304 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.304 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:16.561763+0000", 2023-10-09T10:59:30.304 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1226e-05 2023-10-09T10:59:30.304 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.305 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.305 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.305 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:16.583029+0000", 2023-10-09T10:59:30.305 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021265974999999999 2023-10-09T10:59:30.305 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.305 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.305 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.305 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:16.583070+0000", 2023-10-09T10:59:30.305 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1402000000000001e-05 2023-10-09T10:59:30.305 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.305 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.305 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.306 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.306 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.306 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:129834 9.4 e360/358 9:248af970:::benchmark_data_smithi125_363789_object19535:head v 360'8107, mlcod=360'8103)", 2023-10-09T10:59:30.306 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:17.160251+0000", 2023-10-09T10:59:30.306 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 13.019383461, 2023-10-09T10:59:30.306 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021269380000000001, 2023-10-09T10:59:30.306 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:30.306 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:30.306 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:30.306 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.306 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:30.306 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:17.160251+0000", 2023-10-09T10:59:30.306 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.307 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.307 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.307 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:30.307 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:17.160251+0000", 2023-10-09T10:59:30.307 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:30.307 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.307 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.307 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:30.307 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:17.160250+0000", 2023-10-09T10:59:30.307 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:30.307 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.307 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.307 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:30.308 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:17.160255+0000", 2023-10-09T10:59:30.308 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7389999999999999e-06 2023-10-09T10:59:30.308 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.308 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.308 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:30.308 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:17.160262+0000", 2023-10-09T10:59:30.308 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6190000000000002e-06 2023-10-09T10:59:30.308 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.308 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.308 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:30.308 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:17.160267+0000", 2023-10-09T10:59:30.308 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3229999999999999e-06 2023-10-09T10:59:30.308 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.309 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.309 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:30.309 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:17.163695+0000", 2023-10-09T10:59:30.309 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034286870000000001 2023-10-09T10:59:30.309 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.309 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.309 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:30.309 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:17.163719+0000", 2023-10-09T10:59:30.309 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3975e-05 2023-10-09T10:59:30.309 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.309 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.309 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:30.309 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:17.181491+0000", 2023-10-09T10:59:30.309 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017772000999999999 2023-10-09T10:59:30.310 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:30.310 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:30.310 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:30.310 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:17.181520+0000", 2023-10-09T10:59:30.310 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8503000000000001e-05 2023-10-09T10:59:30.310 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.310 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.310 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.310 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:30.310 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:30.310 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:59:30.311 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:30.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T10:59:30.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:30 smithi155 ceph-mon[184854]: pgmap v915: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 5.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 2.0 KiB/s rd, 60 MiB/s wr, 1.63k op/s; 47313/252414 objects degraded (18.744%); 19351/252414 objects misplaced (7.666%); 16 MiB/s, 252 objects/s recovering 2023-10-09T10:59:30.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:30 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:30.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:30 smithi155 ceph-mon[184854]: failed to encode map e367 with expected crc 2023-10-09T10:59:30.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:30 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:59:30.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:30 smithi155 ceph-mon[184854]: osdmap e367: 8 total, 7 up, 7 in 2023-10-09T10:59:30.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:30 smithi155 ceph-mon[184854]: failed to encode map e367 with expected crc 2023-10-09T10:59:30.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:30 smithi155 ceph-mon[184854]: failed to encode map e367 with expected crc 2023-10-09T10:59:30.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:59:31.028 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:59:31.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[195507]: pgmap v915: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 5.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 2.0 KiB/s rd, 60 MiB/s wr, 1.63k op/s; 47313/252414 objects degraded (18.744%); 19351/252414 objects misplaced (7.666%); 16 MiB/s, 252 objects/s recovering 2023-10-09T10:59:31.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:31.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[195507]: failed to encode map e367 with expected crc 2023-10-09T10:59:31.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:59:31.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[195507]: osdmap e367: 8 total, 7 up, 7 in 2023-10-09T10:59:31.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[195507]: failed to encode map e367 with expected crc 2023-10-09T10:59:31.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[195507]: failed to encode map e367 with expected crc 2023-10-09T10:59:31.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:59:31.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[200197]: pgmap v915: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 5.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 2.0 KiB/s rd, 60 MiB/s wr, 1.63k op/s; 47313/252414 objects degraded (18.744%); 19351/252414 objects misplaced (7.666%); 16 MiB/s, 252 objects/s recovering 2023-10-09T10:59:31.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:31.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[200197]: failed to encode map e367 with expected crc 2023-10-09T10:59:31.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:59:31.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[200197]: osdmap e367: 8 total, 7 up, 7 in 2023-10-09T10:59:31.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[200197]: failed to encode map e367 with expected crc 2023-10-09T10:59:31.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[200197]: failed to encode map e367 with expected crc 2023-10-09T10:59:31.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:59:31.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:59:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:59:31] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.43.0" 2023-10-09T10:59:32.279 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:59:32.281 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:59:32.281 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.281 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:142461 9.5 e367/363 9:ad9ff701:::benchmark_data_smithi125_363789_object32162:head v 367'8952, mlcod=367'8949)", 2023-10-09T10:59:32.281 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:32.261330+0000", 2023-10-09T10:59:32.281 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.017379543000000001, 2023-10-09T10:59:32.281 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017400622000000001, 2023-10-09T10:59:32.281 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:32.282 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:59:32.282 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:32.282 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.282 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:32.282 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.261330+0000", 2023-10-09T10:59:32.282 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.282 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.282 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:32.283 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.261330+0000", 2023-10-09T10:59:32.283 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:32.283 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.261330+0000", 2023-10-09T10:59:32.283 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967296 2023-10-09T10:59:32.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.284 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:32.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.261333+0000", 2023-10-09T10:59:32.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3780000000000001e-06 2023-10-09T10:59:32.284 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.284 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:32.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.261338+0000", 2023-10-09T10:59:32.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7280000000000003e-06 2023-10-09T10:59:32.285 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.285 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:32.285 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.261341+0000", 2023-10-09T10:59:32.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9919999999999999e-06 2023-10-09T10:59:32.285 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.286 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:32.286 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.266787+0000", 2023-10-09T10:59:32.286 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.005445783 2023-10-09T10:59:32.286 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.286 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.286 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:32.286 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.266805+0000", 2023-10-09T10:59:32.287 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8447000000000001e-05 2023-10-09T10:59:32.287 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.287 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:32.287 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.287 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.287 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.287 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:142473 9.6 e367/358 9:6e60def3:::benchmark_data_smithi125_363789_object32174:head v 367'8764, mlcod=367'8762)", 2023-10-09T10:59:32.288 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:32.271064+0000", 2023-10-09T10:59:32.288 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.007646099, 2023-10-09T10:59:32.288 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0077082039999999998, 2023-10-09T10:59:32.288 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:32.288 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:59:32.288 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:32.289 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.289 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:32.289 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271064+0000", 2023-10-09T10:59:32.289 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.289 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.289 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.289 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:32.289 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271064+0000", 2023-10-09T10:59:32.289 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.289 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.289 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.289 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:32.289 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271063+0000", 2023-10-09T10:59:32.290 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:59:32.290 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.290 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.290 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:32.290 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271074+0000", 2023-10-09T10:59:32.290 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0883999999999999e-05 2023-10-09T10:59:32.290 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.290 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.290 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:32.290 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271084+0000", 2023-10-09T10:59:32.290 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0929e-05 2023-10-09T10:59:32.291 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.291 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.291 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:32.291 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271089+0000", 2023-10-09T10:59:32.291 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4669999999999998e-06 2023-10-09T10:59:32.291 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.292 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.292 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:32.292 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271128+0000", 2023-10-09T10:59:32.292 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8788000000000002e-05 2023-10-09T10:59:32.292 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.292 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.292 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:32.292 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271147+0000", 2023-10-09T10:59:32.292 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9255000000000001e-05 2023-10-09T10:59:32.293 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.293 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:32.293 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.293 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.293 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.293 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:142467 9.f e367/358 9:fcfe0979:::benchmark_data_smithi125_363789_object32168:head v 367'9070, mlcod=367'9064)", 2023-10-09T10:59:32.293 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:32.271110+0000", 2023-10-09T10:59:32.293 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0076001640000000004, 2023-10-09T10:59:32.293 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0076989830000000004, 2023-10-09T10:59:32.293 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:32.293 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:59:32.293 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:32.294 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.294 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:32.294 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271110+0000", 2023-10-09T10:59:32.294 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.294 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.294 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.294 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:32.294 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271110+0000", 2023-10-09T10:59:32.294 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.294 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.294 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.294 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:32.294 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271109+0000", 2023-10-09T10:59:32.295 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:32.295 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.295 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.295 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:32.295 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271117+0000", 2023-10-09T10:59:32.295 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4610000000000002e-06 2023-10-09T10:59:32.295 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.295 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.295 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:32.295 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271123+0000", 2023-10-09T10:59:32.295 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4359999999999999e-06 2023-10-09T10:59:32.295 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.295 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.296 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:32.296 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271127+0000", 2023-10-09T10:59:32.296 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5540000000000002e-06 2023-10-09T10:59:32.296 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.296 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.296 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:32.296 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271707+0000", 2023-10-09T10:59:32.296 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00058060999999999996 2023-10-09T10:59:32.296 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.296 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.296 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:32.297 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271727+0000", 2023-10-09T10:59:32.297 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9958000000000001e-05 2023-10-09T10:59:32.297 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.297 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:32.297 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.297 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.297 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.297 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:142468 9.d e367/358 9:b630359a:::benchmark_data_smithi125_363789_object32169:head v 367'8900, mlcod=367'8897)", 2023-10-09T10:59:32.297 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:32.271672+0000", 2023-10-09T10:59:32.297 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0070376659999999997, 2023-10-09T10:59:32.297 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0071773490000000004, 2023-10-09T10:59:32.298 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:32.298 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:59:32.298 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:32.298 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.298 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:32.298 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271672+0000", 2023-10-09T10:59:32.298 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.298 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.298 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.298 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:32.298 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271672+0000", 2023-10-09T10:59:32.299 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.299 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.299 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:32.299 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271671+0000", 2023-10-09T10:59:32.299 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:59:32.299 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.299 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:32.299 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271688+0000", 2023-10-09T10:59:32.299 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7180000000000002e-05 2023-10-09T10:59:32.299 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.299 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:32.300 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271700+0000", 2023-10-09T10:59:32.300 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1164e-05 2023-10-09T10:59:32.300 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.300 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.300 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:32.300 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.271705+0000", 2023-10-09T10:59:32.300 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6690000000000003e-06 2023-10-09T10:59:32.300 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.300 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.300 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:32.300 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.273245+0000", 2023-10-09T10:59:32.300 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015397049999999999 2023-10-09T10:59:32.300 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.301 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.301 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:32.301 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.273267+0000", 2023-10-09T10:59:32.301 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2507e-05 2023-10-09T10:59:32.301 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.301 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:32.301 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.301 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.301 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.301 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:142472 9.b e367/363 9:d145b657:::benchmark_data_smithi125_363789_object32173:head v 367'8806, mlcod=367'8804)", 2023-10-09T10:59:32.301 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:32.272343+0000", 2023-10-09T10:59:32.301 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0063670050000000002, 2023-10-09T10:59:32.301 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0065436349999999999, 2023-10-09T10:59:32.302 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:32.302 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:59:32.302 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:32.302 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.302 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:32.302 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.272343+0000", 2023-10-09T10:59:32.302 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.302 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.302 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.302 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:32.302 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.272343+0000", 2023-10-09T10:59:32.303 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.303 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.303 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.303 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:32.303 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.272342+0000", 2023-10-09T10:59:32.303 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:32.303 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.303 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.303 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:32.303 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.272349+0000", 2023-10-09T10:59:32.303 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2309999999999999e-06 2023-10-09T10:59:32.303 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.303 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.304 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:32.304 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.272360+0000", 2023-10-09T10:59:32.304 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0934999999999999e-05 2023-10-09T10:59:32.304 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.304 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.304 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:32.304 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.272366+0000", 2023-10-09T10:59:32.304 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9050000000000002e-06 2023-10-09T10:59:32.304 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.304 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.304 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:32.304 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.272403+0000", 2023-10-09T10:59:32.304 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6794e-05 2023-10-09T10:59:32.304 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.305 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.305 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:32.305 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.272424+0000", 2023-10-09T10:59:32.305 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0979999999999999e-05 2023-10-09T10:59:32.305 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.305 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:32.305 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.305 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.305 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.305 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:142475 9.8 e367/363 9:13a52965:::benchmark_data_smithi125_363789_object32176:head v 367'8781, mlcod=367'8779)", 2023-10-09T10:59:32.305 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:32.275126+0000", 2023-10-09T10:59:32.305 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.003584088, 2023-10-09T10:59:32.306 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003794918, 2023-10-09T10:59:32.306 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:32.306 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:59:32.306 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:32.306 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.306 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:32.306 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.275126+0000", 2023-10-09T10:59:32.306 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.306 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.306 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.306 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:32.307 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.275126+0000", 2023-10-09T10:59:32.307 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.307 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.307 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.307 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:32.307 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.275125+0000", 2023-10-09T10:59:32.307 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:59:32.307 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.307 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.308 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:32.308 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.275132+0000", 2023-10-09T10:59:32.308 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.041e-06 2023-10-09T10:59:32.308 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.308 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.308 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:32.308 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.275140+0000", 2023-10-09T10:59:32.308 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8250000000000005e-06 2023-10-09T10:59:32.308 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.308 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.308 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:32.309 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.275145+0000", 2023-10-09T10:59:32.309 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9520000000000002e-06 2023-10-09T10:59:32.309 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.309 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.309 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:32.309 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.275195+0000", 2023-10-09T10:59:32.309 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0297000000000003e-05 2023-10-09T10:59:32.309 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.309 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.310 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:32.310 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.275221+0000", 2023-10-09T10:59:32.310 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.601e-05 2023-10-09T10:59:32.310 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.310 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:32.310 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.310 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.311 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.311 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:142476 9.f e367/358 9:f29601bf:::benchmark_data_smithi125_363789_object32177:head v 367'9071, mlcod=367'9066)", 2023-10-09T10:59:32.311 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:32.276504+0000", 2023-10-09T10:59:32.311 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0022059419999999998, 2023-10-09T10:59:32.311 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024504660000000001, 2023-10-09T10:59:32.311 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:32.311 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:59:32.311 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:32.312 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.312 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:32.312 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.276504+0000", 2023-10-09T10:59:32.312 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.312 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.312 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.312 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:32.312 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.276504+0000", 2023-10-09T10:59:32.313 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.313 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.313 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.313 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:32.313 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.276503+0000", 2023-10-09T10:59:32.313 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:32.313 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.313 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.314 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:32.314 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.276513+0000", 2023-10-09T10:59:32.314 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0034000000000001e-05 2023-10-09T10:59:32.314 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.314 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.314 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:32.314 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.276522+0000", 2023-10-09T10:59:32.314 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2479999999999996e-06 2023-10-09T10:59:32.314 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.314 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.314 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:32.314 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.276526+0000", 2023-10-09T10:59:32.315 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0899999999999998e-06 2023-10-09T10:59:32.315 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.315 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.315 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:32.315 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.276558+0000", 2023-10-09T10:59:32.315 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2094999999999997e-05 2023-10-09T10:59:32.315 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.315 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.315 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:32.315 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.276573+0000", 2023-10-09T10:59:32.315 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5444e-05 2023-10-09T10:59:32.315 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.315 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:32.316 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.316 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.316 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.316 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:142474 9.9 e367/363 9:90a4cd60:::benchmark_data_smithi125_363789_object32175:head v 367'9066, mlcod=367'9062)", 2023-10-09T10:59:32.316 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:32.276745+0000", 2023-10-09T10:59:32.316 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0019649089999999999, 2023-10-09T10:59:32.316 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022430919999999999, 2023-10-09T10:59:32.316 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:32.316 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T10:59:32.316 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:32.316 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.317 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:32.317 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.276745+0000", 2023-10-09T10:59:32.317 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.317 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.317 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.317 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:32.317 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.276745+0000", 2023-10-09T10:59:32.317 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:32.318 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.318 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.318 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:32.318 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.276744+0000", 2023-10-09T10:59:32.318 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:32.318 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.318 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.318 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:32.318 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.276750+0000", 2023-10-09T10:59:32.319 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4319999999999998e-06 2023-10-09T10:59:32.319 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.319 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.319 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:32.319 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.276759+0000", 2023-10-09T10:59:32.319 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2199999999999998e-06 2023-10-09T10:59:32.319 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.319 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.320 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:32.320 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.276764+0000", 2023-10-09T10:59:32.320 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2329999999999997e-06 2023-10-09T10:59:32.320 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.320 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.320 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:32.320 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.277270+0000", 2023-10-09T10:59:32.320 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00050569899999999997 2023-10-09T10:59:32.321 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:32.321 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:32.321 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:32.321 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.277303+0000", 2023-10-09T10:59:32.321 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2798e-05 2023-10-09T10:59:32.321 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.321 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:32.321 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.322 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:32.322 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T10:59:32.322 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 8 2023-10-09T10:59:32.322 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:59:32.679 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T10:59:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:32 smithi155 ceph-mon[184854]: pgmap v917: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 988 op/s; 44666/238920 objects degraded (18.695%); 17444/238920 objects misplaced (7.301%); 22 MiB/s, 355 objects/s recovering 2023-10-09T10:59:33.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:32 smithi125 ceph-mon[200197]: pgmap v917: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 988 op/s; 44666/238920 objects degraded (18.695%); 17444/238920 objects misplaced (7.301%); 22 MiB/s, 355 objects/s recovering 2023-10-09T10:59:33.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:32 smithi125 ceph-mon[195507]: pgmap v917: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 988 op/s; 44666/238920 objects degraded (18.695%); 17444/238920 objects misplaced (7.301%); 22 MiB/s, 355 objects/s recovering 2023-10-09T10:59:33.115 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:59:33.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:33 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 44666/238920 objects degraded (18.695%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:33.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:33 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 44666/238920 objects degraded (18.695%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:34.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:33 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 44666/238920 objects degraded (18.695%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:34.357 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:59:34.357 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:59:34.357 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:59:34.357 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:59:34.358 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:59:35.050 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:34 smithi155 ceph-mon[184854]: pgmap v918: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 863 op/s; 44666/238080 objects degraded (18.761%); 16142/238080 objects misplaced (6.780%); 26 MiB/s, 415 objects/s recovering 2023-10-09T10:59:35.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:34 smithi125 ceph-mon[195507]: pgmap v918: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 863 op/s; 44666/238080 objects degraded (18.761%); 16142/238080 objects misplaced (6.780%); 26 MiB/s, 415 objects/s recovering 2023-10-09T10:59:35.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:34 smithi125 ceph-mon[200197]: pgmap v918: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 863 op/s; 44666/238080 objects degraded (18.761%); 16142/238080 objects misplaced (6.780%); 26 MiB/s, 415 objects/s recovering 2023-10-09T10:59:35.490 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:35.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:35.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:35.594 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:35.638 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:35.671 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:35.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:35.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:35.783 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:35.833 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:36.619 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T10:59:36.945 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:36 smithi155 ceph-mon[184854]: pgmap v919: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 44666/238080 objects degraded (18.761%); 16142/238080 objects misplaced (6.780%); 25 MiB/s, 394 objects/s recovering 2023-10-09T10:59:37.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:36 smithi125 ceph-mon[195507]: pgmap v919: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 44666/238080 objects degraded (18.761%); 16142/238080 objects misplaced (6.780%); 25 MiB/s, 394 objects/s recovering 2023-10-09T10:59:37.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:36 smithi125 ceph-mon[200197]: pgmap v919: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 44666/238080 objects degraded (18.761%); 16142/238080 objects misplaced (6.780%); 25 MiB/s, 394 objects/s recovering 2023-10-09T10:59:37.126 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:37.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:37.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:37.235 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:37.263 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:59:37.299 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:37.331 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:37.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:37.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:37.362 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:59:37.440 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:37.493 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:37.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:37.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:37.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:37.691 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:37.739 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:38.017 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:38.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:38.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:38.121 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:38.177 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:38.218 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:38.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:38.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:38.304 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:59:38.312 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:38.324 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:38.347 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:59:38.372 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:38.403 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:59:38.455 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:59:38.500 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:59:38.538 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:59:38.574 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:59:38.574 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:59:38.575 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-10-09T10:59:38.575 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:59:38.575 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:59:38.575 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:59:38.575 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:59:38.575 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:59:38.575 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:59:38.758 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:38.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:38.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:38.863 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:38.924 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:38 smithi155 ceph-mon[184854]: pgmap v920: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1k op/s; 41735/223311 objects degraded (18.689%); 14225/223311 objects misplaced (6.370%); 27 MiB/s, 433 objects/s recovering 2023-10-09T10:59:38.924 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:38 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 41735/223311 objects degraded (18.689%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:38.925 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:38 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2723720054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:38.925 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:38 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3207508563' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:59:38.925 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:38 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:59:38.933 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:38.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:38.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:38.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:39.049 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:38 smithi125 ceph-mon[195507]: pgmap v920: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1k op/s; 41735/223311 objects degraded (18.689%); 14225/223311 objects misplaced (6.370%); 27 MiB/s, 433 objects/s recovering 2023-10-09T10:59:39.049 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:38 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 41735/223311 objects degraded (18.689%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:39.049 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:38 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2723720054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:39.049 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:38 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3207508563' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:59:39.049 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:38 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:59:39.049 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:38 smithi125 ceph-mon[200197]: pgmap v920: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1k op/s; 41735/223311 objects degraded (18.689%); 14225/223311 objects misplaced (6.370%); 27 MiB/s, 433 objects/s recovering 2023-10-09T10:59:39.049 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:38 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 41735/223311 objects degraded (18.689%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:39.050 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:38 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2723720054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:39.050 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:38 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3207508563' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:59:39.050 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:38 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T10:59:39.087 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:39.132 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 9 pg_num to 26 2023-10-09T10:59:39.164 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:39.200 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:39.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:39.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:39.286 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:59:39.286 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:59:39.314 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:39.361 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:39.404 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:39.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:39.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:39.512 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:39.550 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:39.622 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:59:39.624 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:59:39.693 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T10:59:39.776 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:39.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:39.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:39.889 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:39.946 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:39.974 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:39.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:39.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:40.081 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:40.129 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:40 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:59:40.129 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:40 smithi155 ceph-mon[184854]: osdmap e368: 8 total, 7 up, 7 in 2023-10-09T10:59:40.129 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:40 smithi155 ceph-mon[184854]: failed to encode map e368 with expected crc 2023-10-09T10:59:40.130 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:40 smithi155 ceph-mon[184854]: failed to encode map e368 with expected crc 2023-10-09T10:59:40.130 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:40 smithi155 ceph-mon[184854]: pgmap v922: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 41735/222291 objects degraded (18.775%); 12910/222291 objects misplaced (5.808%); 28 MiB/s, 448 objects/s recovering 2023-10-09T10:59:40.130 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:40 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:59:40.130 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:40 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1349907662' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:59:40.130 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:40 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:59:40.131 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:40.135 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:59:40.156 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T10:59:40.193 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:40.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:40.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:40.225 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:59:40.226 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[195507]: osdmap e368: 8 total, 7 up, 7 in 2023-10-09T10:59:40.226 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[195507]: failed to encode map e368 with expected crc 2023-10-09T10:59:40.226 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[195507]: failed to encode map e368 with expected crc 2023-10-09T10:59:40.226 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[195507]: pgmap v922: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 41735/222291 objects degraded (18.775%); 12910/222291 objects misplaced (5.808%); 28 MiB/s, 448 objects/s recovering 2023-10-09T10:59:40.226 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:59:40.226 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1349907662' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:59:40.226 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:59:40.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T10:59:40.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[200197]: osdmap e368: 8 total, 7 up, 7 in 2023-10-09T10:59:40.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[200197]: failed to encode map e368 with expected crc 2023-10-09T10:59:40.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[200197]: failed to encode map e368 with expected crc 2023-10-09T10:59:40.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[200197]: pgmap v922: 119 pgs: 22 active+clean+remapped, 3 active+remapped+backfilling, 11 active+undersized+degraded, 31 active+undersized, 52 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 41735/222291 objects degraded (18.775%); 12910/222291 objects misplaced (5.808%); 28 MiB/s, 448 objects/s recovering 2023-10-09T10:59:40.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T10:59:40.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1349907662' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:59:40.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:40 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T10:59:40.323 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:40.699 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:40.872 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:40.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:40.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:40.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:41.006 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:41.026 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:41.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:41.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:41.130 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:41.178 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:41.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:59:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:59:41] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.43.0" 2023-10-09T10:59:41.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:41 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:41.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:41 smithi125 ceph-mon[195507]: failed to encode map e369 with expected crc 2023-10-09T10:59:41.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:41 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:59:41.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:41 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:59:41.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:41 smithi125 ceph-mon[195507]: osdmap e369: 8 total, 7 up, 7 in 2023-10-09T10:59:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:41 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:41 smithi125 ceph-mon[200197]: failed to encode map e369 with expected crc 2023-10-09T10:59:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:41 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:59:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:41 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:59:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:41 smithi125 ceph-mon[200197]: osdmap e369: 8 total, 7 up, 7 in 2023-10-09T10:59:41.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:41 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:41.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:41 smithi155 ceph-mon[184854]: failed to encode map e369 with expected crc 2023-10-09T10:59:41.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:41 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T10:59:41.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:41 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T10:59:41.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:41 smithi155 ceph-mon[184854]: osdmap e369: 8 total, 7 up, 7 in 2023-10-09T10:59:41.375 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:41.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:41.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:41.479 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:41.510 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:41.531 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:41.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:41.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:41.625 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:59:41.636 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:41.699 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:41.711 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:59:41.711 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:59:41.721 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:41.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:41.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:41.825 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:41.857 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:41.879 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:41.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:41.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:41.935 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:41.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:42.014 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:42.084 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:42.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:42.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:42.091 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T10:59:42.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:42.218 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:42.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[200197]: 1.11 scrub starts 2023-10-09T10:59:42.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[200197]: failed to encode map e370 with expected crc 2023-10-09T10:59:42.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[200197]: osdmap e370: 8 total, 7 up, 7 in 2023-10-09T10:59:42.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[200197]: failed to encode map e370 with expected crc 2023-10-09T10:59:42.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[200197]: failed to encode map e370 with expected crc 2023-10-09T10:59:42.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[200197]: failed to encode map e370 with expected crc 2023-10-09T10:59:42.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[200197]: failed to encode map e370 with expected crc 2023-10-09T10:59:42.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[200197]: 9.8 starting backfill to osd.7 from (345'3857,368'9277] MIN to 368'9278 2023-10-09T10:59:42.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[200197]: 9.9 continuing backfill to osd.3 from (347'4453,368'9570] MIN to 368'9570 2023-10-09T10:59:42.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[200197]: pgmap v925: 129 pgs: 1 remapped+peering, 7 peering, 6 unknown, 21 active+clean+remapped, 2 active+remapped+backfilling, 8 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 31 active+undersized, 52 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.48k op/s; 21957/211482 objects degraded (10.382%); 5014/211482 objects misplaced (2.371%); 33 MiB/s, 532 objects/s recovering 2023-10-09T10:59:42.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[195507]: 1.11 scrub starts 2023-10-09T10:59:42.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[195507]: failed to encode map e370 with expected crc 2023-10-09T10:59:42.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[195507]: osdmap e370: 8 total, 7 up, 7 in 2023-10-09T10:59:42.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[195507]: failed to encode map e370 with expected crc 2023-10-09T10:59:42.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[195507]: failed to encode map e370 with expected crc 2023-10-09T10:59:42.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[195507]: failed to encode map e370 with expected crc 2023-10-09T10:59:42.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[195507]: failed to encode map e370 with expected crc 2023-10-09T10:59:42.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[195507]: 9.8 starting backfill to osd.7 from (345'3857,368'9277] MIN to 368'9278 2023-10-09T10:59:42.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[195507]: 9.9 continuing backfill to osd.3 from (347'4453,368'9570] MIN to 368'9570 2023-10-09T10:59:42.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:42 smithi125 ceph-mon[195507]: pgmap v925: 129 pgs: 1 remapped+peering, 7 peering, 6 unknown, 21 active+clean+remapped, 2 active+remapped+backfilling, 8 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 31 active+undersized, 52 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.48k op/s; 21957/211482 objects degraded (10.382%); 5014/211482 objects misplaced (2.371%); 33 MiB/s, 532 objects/s recovering 2023-10-09T10:59:42.342 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:42.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:42.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:42.406 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:59:42.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:42 smithi155 ceph-mon[184854]: 1.11 scrub starts 2023-10-09T10:59:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:42 smithi155 ceph-mon[184854]: failed to encode map e370 with expected crc 2023-10-09T10:59:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:42 smithi155 ceph-mon[184854]: osdmap e370: 8 total, 7 up, 7 in 2023-10-09T10:59:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:42 smithi155 ceph-mon[184854]: failed to encode map e370 with expected crc 2023-10-09T10:59:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:42 smithi155 ceph-mon[184854]: failed to encode map e370 with expected crc 2023-10-09T10:59:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:42 smithi155 ceph-mon[184854]: failed to encode map e370 with expected crc 2023-10-09T10:59:42.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:42 smithi155 ceph-mon[184854]: failed to encode map e370 with expected crc 2023-10-09T10:59:42.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:42 smithi155 ceph-mon[184854]: 9.8 starting backfill to osd.7 from (345'3857,368'9277] MIN to 368'9278 2023-10-09T10:59:42.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:42 smithi155 ceph-mon[184854]: 9.9 continuing backfill to osd.3 from (347'4453,368'9570] MIN to 368'9570 2023-10-09T10:59:42.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:42 smithi155 ceph-mon[184854]: pgmap v925: 129 pgs: 1 remapped+peering, 7 peering, 6 unknown, 21 active+clean+remapped, 2 active+remapped+backfilling, 8 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 31 active+undersized, 52 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.48k op/s; 21957/211482 objects degraded (10.382%); 5014/211482 objects misplaced (2.371%); 33 MiB/s, 532 objects/s recovering 2023-10-09T10:59:42.477 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:42.548 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:42.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:42.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:42.558 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T10:59:42.652 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:42.683 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:42.755 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:42.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:42.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:42.859 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:42.886 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:42.943 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:42.978 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:59:42.998 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:59:43.006 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:43.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:43.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:43.017 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:59:43.037 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:59:43.058 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:59:43.077 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:59:43.095 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:59:43.096 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:59:43.096 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-10-09T10:59:43.096 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:59:43.096 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:59:43.096 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:59:43.096 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:59:43.096 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:59:43.096 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:59:43.110 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:43.155 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:43.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[195507]: failed to encode map e371 with expected crc 2023-10-09T10:59:43.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[195507]: failed to encode map e371 with expected crc 2023-10-09T10:59:43.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[195507]: osdmap e371: 8 total, 7 up, 7 in 2023-10-09T10:59:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[195507]: failed to encode map e371 with expected crc 2023-10-09T10:59:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[195507]: failed to encode map e371 with expected crc 2023-10-09T10:59:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[195507]: 1.b scrub starts 2023-10-09T10:59:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[195507]: 1.b scrub ok 2023-10-09T10:59:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1088369256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 21957/211482 objects degraded (10.382%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[195507]: osdmap e372: 8 total, 7 up, 7 in 2023-10-09T10:59:43.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[200197]: failed to encode map e371 with expected crc 2023-10-09T10:59:43.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[200197]: failed to encode map e371 with expected crc 2023-10-09T10:59:43.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[200197]: osdmap e371: 8 total, 7 up, 7 in 2023-10-09T10:59:43.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[200197]: failed to encode map e371 with expected crc 2023-10-09T10:59:43.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[200197]: failed to encode map e371 with expected crc 2023-10-09T10:59:43.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[200197]: 1.b scrub starts 2023-10-09T10:59:43.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[200197]: 1.b scrub ok 2023-10-09T10:59:43.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1088369256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:43.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 21957/211482 objects degraded (10.382%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:43.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:43 smithi125 ceph-mon[200197]: osdmap e372: 8 total, 7 up, 7 in 2023-10-09T10:59:43.382 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:43.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:43.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:43.435 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:59:43.435 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T10:59:43.435 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T10:59:43.435 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T10:59:43.435 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:59:43.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:43 smithi155 ceph-mon[184854]: failed to encode map e371 with expected crc 2023-10-09T10:59:43.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:43 smithi155 ceph-mon[184854]: failed to encode map e371 with expected crc 2023-10-09T10:59:43.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:43 smithi155 ceph-mon[184854]: osdmap e371: 8 total, 7 up, 7 in 2023-10-09T10:59:43.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:43 smithi155 ceph-mon[184854]: failed to encode map e371 with expected crc 2023-10-09T10:59:43.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:43 smithi155 ceph-mon[184854]: failed to encode map e371 with expected crc 2023-10-09T10:59:43.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:43 smithi155 ceph-mon[184854]: 1.b scrub starts 2023-10-09T10:59:43.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:43 smithi155 ceph-mon[184854]: 1.b scrub ok 2023-10-09T10:59:43.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:43 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1088369256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:43.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:43 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 21957/211482 objects degraded (10.382%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:43.462 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:43 smithi155 ceph-mon[184854]: osdmap e372: 8 total, 7 up, 7 in 2023-10-09T10:59:43.488 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:43.523 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:43.545 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:43.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:43.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:43.583 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:59:43.584 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:59:43.650 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:43.680 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:43.753 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:43.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:43.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:43.818 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T10:59:43.858 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:43.890 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:43.912 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:43.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:43.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:44.016 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:44.048 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:44.070 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:44.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:44.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:44.137 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:59:44.175 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:44.256 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:44.332 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:44.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:44.335 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:44 smithi155 ceph-mon[184854]: failed to encode map e372 with expected crc 2023-10-09T10:59:44.336 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:44 smithi155 ceph-mon[184854]: failed to encode map e372 with expected crc 2023-10-09T10:59:44.336 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:44 smithi155 ceph-mon[184854]: failed to encode map e372 with expected crc 2023-10-09T10:59:44.336 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:44 smithi155 ceph-mon[184854]: failed to encode map e372 with expected crc 2023-10-09T10:59:44.336 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:44 smithi155 ceph-mon[184854]: 9.19 continuing backfill to osd.3 from (347'4453,368'9570] MIN to 368'9570 2023-10-09T10:59:44.336 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:44 smithi155 ceph-mon[184854]: 9.18 starting backfill to osd.7 from (345'3857,368'9277] MIN to 368'9278 2023-10-09T10:59:44.336 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:44 smithi155 ceph-mon[184854]: pgmap v928: 129 pgs: 1 remapped+peering, 7 peering, 6 unknown, 21 active+clean+remapped, 2 active+remapped+backfilling, 8 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 31 active+undersized, 52 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 900 op/s; 21957/211482 objects degraded (10.382%); 5014/211482 objects misplaced (2.371%); 15 MiB/s, 233 objects/s recovering 2023-10-09T10:59:44.336 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:44 smithi155 ceph-mon[184854]: osdmap e373: 8 total, 7 up, 7 in 2023-10-09T10:59:44.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:44.437 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:44.514 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:44.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[200197]: failed to encode map e372 with expected crc 2023-10-09T10:59:44.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[200197]: failed to encode map e372 with expected crc 2023-10-09T10:59:44.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[200197]: failed to encode map e372 with expected crc 2023-10-09T10:59:44.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[200197]: failed to encode map e372 with expected crc 2023-10-09T10:59:44.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[200197]: 9.19 continuing backfill to osd.3 from (347'4453,368'9570] MIN to 368'9570 2023-10-09T10:59:44.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[200197]: 9.18 starting backfill to osd.7 from (345'3857,368'9277] MIN to 368'9278 2023-10-09T10:59:44.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[200197]: pgmap v928: 129 pgs: 1 remapped+peering, 7 peering, 6 unknown, 21 active+clean+remapped, 2 active+remapped+backfilling, 8 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 31 active+undersized, 52 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 900 op/s; 21957/211482 objects degraded (10.382%); 5014/211482 objects misplaced (2.371%); 15 MiB/s, 233 objects/s recovering 2023-10-09T10:59:44.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[200197]: osdmap e373: 8 total, 7 up, 7 in 2023-10-09T10:59:44.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[195507]: failed to encode map e372 with expected crc 2023-10-09T10:59:44.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[195507]: failed to encode map e372 with expected crc 2023-10-09T10:59:44.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[195507]: failed to encode map e372 with expected crc 2023-10-09T10:59:44.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[195507]: failed to encode map e372 with expected crc 2023-10-09T10:59:44.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[195507]: 9.19 continuing backfill to osd.3 from (347'4453,368'9570] MIN to 368'9570 2023-10-09T10:59:44.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[195507]: 9.18 starting backfill to osd.7 from (345'3857,368'9277] MIN to 368'9278 2023-10-09T10:59:44.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[195507]: pgmap v928: 129 pgs: 1 remapped+peering, 7 peering, 6 unknown, 21 active+clean+remapped, 2 active+remapped+backfilling, 8 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 31 active+undersized, 52 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 900 op/s; 21957/211482 objects degraded (10.382%); 5014/211482 objects misplaced (2.371%); 15 MiB/s, 233 objects/s recovering 2023-10-09T10:59:44.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:44 smithi125 ceph-mon[195507]: osdmap e373: 8 total, 7 up, 7 in 2023-10-09T10:59:44.801 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 7] out_osds: [0] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-10-09T10:59:44.801 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:59:44.801 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-10-09T10:59:44.801 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd in 0 2023-10-09T10:59:44.907 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:44.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:44.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:45.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:45.065 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:45.149 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[195507]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.150 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[200197]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.150 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[200197]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.151 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[200197]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.151 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[200197]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.151 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[200197]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.151 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[200197]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.151 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1978364622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:45.151 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:59:45.168 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:45.171 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:45.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:45.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:45.176 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T10:59:45.273 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:45.309 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:45.333 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:45.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:45.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:45.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[195507]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[195507]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[195507]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[195507]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[195507]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1978364622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:45.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:59:45.437 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:59:45.438 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T10:59:45.438 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T10:59:45.438 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T10:59:45.438 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.438 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:95197 7.d 7:b9b00ad5:::benchmark_data_smithi125_295816_object4060:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-10-09T10:59:45.438 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:57.383916+0000", 2023-10-09T10:59:45.438 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 528.051033416, 2023-10-09T10:59:45.438 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.070851236999999997, 2023-10-09T10:59:45.439 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.439 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.439 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:45.439 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:59:45.439 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:45.439 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 95197 2023-10-09T10:59:45.439 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.439 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.440 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.440 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.440 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383916+0000", 2023-10-09T10:59:45.440 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.440 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.440 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.440 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.440 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383916+0000", 2023-10-09T10:59:45.441 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.441 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.441 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.441 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.441 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383915+0000", 2023-10-09T10:59:45.441 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:45.441 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.441 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383920+0000", 2023-10-09T10:59:45.442 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.84e-06 2023-10-09T10:59:45.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383931+0000", 2023-10-09T10:59:45.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0657e-05 2023-10-09T10:59:45.443 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.443 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.443 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.443 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383938+0000", 2023-10-09T10:59:45.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5139999999999998e-06 2023-10-09T10:59:45.443 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.443 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.444 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.444 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.411508+0000", 2023-10-09T10:59:45.444 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027570921000000002 2023-10-09T10:59:45.444 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.444 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.444 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:59:45.444 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.416286+0000", 2023-10-09T10:59:45.444 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0047777430000000001 2023-10-09T10:59:45.445 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.445 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.437950+0000", 2023-10-09T10:59:45.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021664251999999998 2023-10-09T10:59:45.446 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.446 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.446 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.446 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.438046+0000", 2023-10-09T10:59:45.446 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5459999999999997e-05 2023-10-09T10:59:45.446 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.447 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.447 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:59:45.447 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.438186+0000", 2023-10-09T10:59:45.447 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014000800000000001 2023-10-09T10:59:45.447 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.447 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.447 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:45.447 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.443989+0000", 2023-10-09T10:59:45.447 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0058031369999999999 2023-10-09T10:59:45.447 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.448 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.448 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.448 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.452036+0000", 2023-10-09T10:59:45.448 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0080469280000000001 2023-10-09T10:59:45.448 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.448 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.448 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.448 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.452814+0000", 2023-10-09T10:59:45.448 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00077831199999999997 2023-10-09T10:59:45.448 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.448 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.448 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.449 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.452840+0000", 2023-10-09T10:59:45.449 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5947000000000001e-05 2023-10-09T10:59:45.449 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.449 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.449 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.454767+0000", 2023-10-09T10:59:45.449 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019270769999999999 2023-10-09T10:59:45.449 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.449 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.449 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.449 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.449 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:95787 7.d 7:b6e713eb:::benchmark_data_smithi125_295816_object4650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:59:45.450 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:58.300479+0000", 2023-10-09T10:59:45.450 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 527.13447098300003, 2023-10-09T10:59:45.450 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072656459000000007, 2023-10-09T10:59:45.450 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.450 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.450 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:45.450 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:59:45.450 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:45.450 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 95787 2023-10-09T10:59:45.450 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.450 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.451 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.451 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300479+0000", 2023-10-09T10:59:45.451 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.451 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.451 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.451 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300479+0000", 2023-10-09T10:59:45.451 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.451 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.451 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.451 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300477+0000", 2023-10-09T10:59:45.452 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:59:45.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300487+0000", 2023-10-09T10:59:45.452 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0638999999999999e-05 2023-10-09T10:59:45.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300511+0000", 2023-10-09T10:59:45.452 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3059999999999999e-05 2023-10-09T10:59:45.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.453 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300518+0000", 2023-10-09T10:59:45.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0029999999999997e-06 2023-10-09T10:59:45.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.453 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.453 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.322068+0000", 2023-10-09T10:59:45.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021550625 2023-10-09T10:59:45.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.453 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.453 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:59:45.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.333963+0000", 2023-10-09T10:59:45.454 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011895039 2023-10-09T10:59:45.454 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.454 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.454 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.454 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.351448+0000", 2023-10-09T10:59:45.454 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017484831999999999 2023-10-09T10:59:45.454 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.454 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.454 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.454 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.352038+0000", 2023-10-09T10:59:45.454 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00059005600000000004 2023-10-09T10:59:45.454 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.455 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.455 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:59:45.455 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.352181+0000", 2023-10-09T10:59:45.455 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014277800000000001 2023-10-09T10:59:45.455 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.455 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.455 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.455 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.367667+0000", 2023-10-09T10:59:45.455 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015486592 2023-10-09T10:59:45.455 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.455 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.455 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:45.456 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.370532+0000", 2023-10-09T10:59:45.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028642279999999999 2023-10-09T10:59:45.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.456 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.456 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.456 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.371142+0000", 2023-10-09T10:59:45.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00060984699999999995 2023-10-09T10:59:45.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.456 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.456 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.456 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.371180+0000", 2023-10-09T10:59:45.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8529000000000003e-05 2023-10-09T10:59:45.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.457 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.457 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.373135+0000", 2023-10-09T10:59:45.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001954973 2023-10-09T10:59:45.457 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.457 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.457 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.457 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.457 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.457 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96817 7.d 7:b9f7ad08:::benchmark_data_smithi125_295816_object5680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:59:45.457 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:59.884613+0000", 2023-10-09T10:59:45.458 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 525.55033652999998, 2023-10-09T10:59:45.458 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14656745900000001, 2023-10-09T10:59:45.458 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.458 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.458 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:45.458 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:59:45.458 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:45.458 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96817 2023-10-09T10:59:45.458 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.458 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.458 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884613+0000", 2023-10-09T10:59:45.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.459 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884613+0000", 2023-10-09T10:59:45.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.459 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884612+0000", 2023-10-09T10:59:45.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:45.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884618+0000", 2023-10-09T10:59:45.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5880000000000003e-06 2023-10-09T10:59:45.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884633+0000", 2023-10-09T10:59:45.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5109999999999999e-05 2023-10-09T10:59:45.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884639+0000", 2023-10-09T10:59:45.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.164e-06 2023-10-09T10:59:45.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.887892+0000", 2023-10-09T10:59:45.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032536610000000001 2023-10-09T10:59:45.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:59:45.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.888551+0000", 2023-10-09T10:59:45.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00065867400000000004 2023-10-09T10:59:45.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910766+0000", 2023-10-09T10:59:45.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022214655 2023-10-09T10:59:45.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910852+0000", 2023-10-09T10:59:45.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6551999999999994e-05 2023-10-09T10:59:45.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.463 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:59:45.463 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910980+0000", 2023-10-09T10:59:45.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000127704 2023-10-09T10:59:45.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.463 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:45.463 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.914262+0000", 2023-10-09T10:59:45.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032824070000000002 2023-10-09T10:59:45.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.463 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.463 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.020644+0000", 2023-10-09T10:59:45.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.106381932 2023-10-09T10:59:45.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.029029+0000", 2023-10-09T10:59:45.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0083848010000000008 2023-10-09T10:59:45.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.029088+0000", 2023-10-09T10:59:45.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8641999999999999e-05 2023-10-09T10:59:45.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.465 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.031180+0000", 2023-10-09T10:59:45.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020928359999999998 2023-10-09T10:59:45.465 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.465 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.465 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.465 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96884 7.d 7:b183b169:::benchmark_data_smithi125_295816_object5747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:59:45.465 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.113486+0000", 2023-10-09T10:59:45.465 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 525.32146323999996, 2023-10-09T10:59:45.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.082588165000000005, 2023-10-09T10:59:45.466 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.466 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.466 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:45.466 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:59:45.466 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:45.466 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96884 2023-10-09T10:59:45.466 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.466 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.466 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.466 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113486+0000", 2023-10-09T10:59:45.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.467 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113486+0000", 2023-10-09T10:59:45.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.467 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113485+0000", 2023-10-09T10:59:45.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:59:45.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113491+0000", 2023-10-09T10:59:45.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3009999999999997e-06 2023-10-09T10:59:45.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113501+0000", 2023-10-09T10:59:45.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0342e-05 2023-10-09T10:59:45.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113505+0000", 2023-10-09T10:59:45.469 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4530000000000004e-06 2023-10-09T10:59:45.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.469 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.469 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.117310+0000", 2023-10-09T10:59:45.469 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0038049329999999999 2023-10-09T10:59:45.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.469 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:59:45.469 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.117924+0000", 2023-10-09T10:59:45.469 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00061396699999999998 2023-10-09T10:59:45.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.470 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.470 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.140859+0000", 2023-10-09T10:59:45.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022935140999999999 2023-10-09T10:59:45.470 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.470 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.470 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.470 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.140963+0000", 2023-10-09T10:59:45.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010341 2023-10-09T10:59:45.470 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.470 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.470 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:59:45.470 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.141120+0000", 2023-10-09T10:59:45.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000157405 2023-10-09T10:59:45.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.471 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:45.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.150528+0000", 2023-10-09T10:59:45.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0094081660000000008 2023-10-09T10:59:45.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.471 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.160789+0000", 2023-10-09T10:59:45.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010260429 2023-10-09T10:59:45.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.194166+0000", 2023-10-09T10:59:45.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.033377636000000002 2023-10-09T10:59:45.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.194226+0000", 2023-10-09T10:59:45.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9855999999999997e-05 2023-10-09T10:59:45.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196074+0000", 2023-10-09T10:59:45.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001848192 2023-10-09T10:59:45.474 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.474 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.474 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.474 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96892 7.d 7:bd59fbdb:::benchmark_data_smithi125_295816_object5755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:59:45.474 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.148429+0000", 2023-10-09T10:59:45.475 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 525.28652098800001, 2023-10-09T10:59:45.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.089082649999999999, 2023-10-09T10:59:45.475 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.475 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.475 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:45.475 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:59:45.475 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:45.475 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96892 2023-10-09T10:59:45.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.475 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.475 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.475 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148429+0000", 2023-10-09T10:59:45.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.476 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.476 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148429+0000", 2023-10-09T10:59:45.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.476 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.476 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148428+0000", 2023-10-09T10:59:45.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967296 2023-10-09T10:59:45.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148430+0000", 2023-10-09T10:59:45.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5570000000000001e-06 2023-10-09T10:59:45.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148433+0000", 2023-10-09T10:59:45.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.892e-06 2023-10-09T10:59:45.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148435+0000", 2023-10-09T10:59:45.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5339999999999998e-06 2023-10-09T10:59:45.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.152891+0000", 2023-10-09T10:59:45.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0044553409999999998 2023-10-09T10:59:45.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:59:45.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.153532+0000", 2023-10-09T10:59:45.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000641539 2023-10-09T10:59:45.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.182585+0000", 2023-10-09T10:59:45.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029053295 2023-10-09T10:59:45.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.182682+0000", 2023-10-09T10:59:45.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6623e-05 2023-10-09T10:59:45.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:59:45.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.182825+0000", 2023-10-09T10:59:45.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014288300000000001 2023-10-09T10:59:45.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:45.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.198214+0000", 2023-10-09T10:59:45.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015389162 2023-10-09T10:59:45.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.223087+0000", 2023-10-09T10:59:45.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024872423000000001 2023-10-09T10:59:45.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.235280+0000", 2023-10-09T10:59:45.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012193017 2023-10-09T10:59:45.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.235328+0000", 2023-10-09T10:59:45.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8211000000000003e-05 2023-10-09T10:59:45.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.237511+0000", 2023-10-09T10:59:45.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021833960000000002 2023-10-09T10:59:45.482 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.482 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.482 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.482 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96898 7.d 7:b4ab753b:::benchmark_data_smithi125_295816_object5761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:59:45.482 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.166362+0000", 2023-10-09T10:59:45.482 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 525.26858749400003, 2023-10-09T10:59:45.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072656032999999995, 2023-10-09T10:59:45.483 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.483 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.483 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:45.483 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:59:45.483 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:45.483 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96898 2023-10-09T10:59:45.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.483 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.483 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.483 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166362+0000", 2023-10-09T10:59:45.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166362+0000", 2023-10-09T10:59:45.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166360+0000", 2023-10-09T10:59:45.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T10:59:45.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166369+0000", 2023-10-09T10:59:45.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.166e-06 2023-10-09T10:59:45.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166385+0000", 2023-10-09T10:59:45.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5214e-05 2023-10-09T10:59:45.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166392+0000", 2023-10-09T10:59:45.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1659999999999997e-06 2023-10-09T10:59:45.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166446+0000", 2023-10-09T10:59:45.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3819000000000001e-05 2023-10-09T10:59:45.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:59:45.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.167178+0000", 2023-10-09T10:59:45.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00073199300000000001 2023-10-09T10:59:45.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.191480+0000", 2023-10-09T10:59:45.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024302082999999999 2023-10-09T10:59:45.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.191594+0000", 2023-10-09T10:59:45.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000114527 2023-10-09T10:59:45.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:59:45.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.191761+0000", 2023-10-09T10:59:45.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000166599 2023-10-09T10:59:45.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:45.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.198249+0000", 2023-10-09T10:59:45.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064882439999999998 2023-10-09T10:59:45.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.223116+0000", 2023-10-09T10:59:45.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024867330999999999 2023-10-09T10:59:45.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.237546+0000", 2023-10-09T10:59:45.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014429365 2023-10-09T10:59:45.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.237573+0000", 2023-10-09T10:59:45.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7534000000000001e-05 2023-10-09T10:59:45.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.239018+0000", 2023-10-09T10:59:45.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001444776 2023-10-09T10:59:45.490 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.490 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.490 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.490 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.490 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96901 7.d 7:bd396a29:::benchmark_data_smithi125_295816_object5764:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:59:45.490 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.177994+0000", 2023-10-09T10:59:45.490 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 525.25695567299999, 2023-10-09T10:59:45.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.079151244999999995, 2023-10-09T10:59:45.490 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.491 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.491 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:45.491 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:59:45.491 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:45.491 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96901 2023-10-09T10:59:45.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.491 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177994+0000", 2023-10-09T10:59:45.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177994+0000", 2023-10-09T10:59:45.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177993+0000", 2023-10-09T10:59:45.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:59:45.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177998+0000", 2023-10-09T10:59:45.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6699999999999999e-06 2023-10-09T10:59:45.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.178008+0000", 2023-10-09T10:59:45.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6520000000000001e-06 2023-10-09T10:59:45.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.178013+0000", 2023-10-09T10:59:45.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.028e-06 2023-10-09T10:59:45.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196130+0000", 2023-10-09T10:59:45.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018116621999999999 2023-10-09T10:59:45.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:59:45.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196670+0000", 2023-10-09T10:59:45.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00054042399999999996 2023-10-09T10:59:45.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.227020+0000", 2023-10-09T10:59:45.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030349832 2023-10-09T10:59:45.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.227095+0000", 2023-10-09T10:59:45.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4785999999999999e-05 2023-10-09T10:59:45.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:59:45.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.227204+0000", 2023-10-09T10:59:45.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000109079 2023-10-09T10:59:45.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:45.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.239698+0000", 2023-10-09T10:59:45.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012494081000000001 2023-10-09T10:59:45.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242208+0000", 2023-10-09T10:59:45.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00251001 2023-10-09T10:59:45.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.254821+0000", 2023-10-09T10:59:45.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012612978 2023-10-09T10:59:45.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.254858+0000", 2023-10-09T10:59:45.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6785000000000001e-05 2023-10-09T10:59:45.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.257145+0000", 2023-10-09T10:59:45.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002287374 2023-10-09T10:59:45.498 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.498 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.498 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.498 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96908 7.d 7:bd9652bb:::benchmark_data_smithi125_295816_object5771:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:59:45.498 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.184970+0000", 2023-10-09T10:59:45.498 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 525.24997928100004, 2023-10-09T10:59:45.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.107155712, 2023-10-09T10:59:45.498 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.498 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.499 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:45.499 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:59:45.499 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:45.499 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96908 2023-10-09T10:59:45.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.499 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184970+0000", 2023-10-09T10:59:45.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184970+0000", 2023-10-09T10:59:45.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184969+0000", 2023-10-09T10:59:45.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:45.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184976+0000", 2023-10-09T10:59:45.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0849999999999999e-06 2023-10-09T10:59:45.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184988+0000", 2023-10-09T10:59:45.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2438e-05 2023-10-09T10:59:45.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184994+0000", 2023-10-09T10:59:45.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.835e-06 2023-10-09T10:59:45.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196710+0000", 2023-10-09T10:59:45.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011715557 2023-10-09T10:59:45.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:59:45.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.197312+0000", 2023-10-09T10:59:45.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000602675 2023-10-09T10:59:45.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.232847+0000", 2023-10-09T10:59:45.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.035534143999999997 2023-10-09T10:59:45.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.232937+0000", 2023-10-09T10:59:45.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0303999999999995e-05 2023-10-09T10:59:45.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:59:45.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.233064+0000", 2023-10-09T10:59:45.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012689799999999999 2023-10-09T10:59:45.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:45.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242946+0000", 2023-10-09T10:59:45.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0098825000000000007 2023-10-09T10:59:45.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.269531+0000", 2023-10-09T10:59:45.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026584907000000001 2023-10-09T10:59:45.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.289926+0000", 2023-10-09T10:59:45.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020394791999999998 2023-10-09T10:59:45.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.505 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.505 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.289981+0000", 2023-10-09T10:59:45.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4540999999999998e-05 2023-10-09T10:59:45.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.505 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.505 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.292126+0000", 2023-10-09T10:59:45.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021454270000000001 2023-10-09T10:59:45.505 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.506 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.506 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.506 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.506 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.506 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96935 7.d 7:b8c4db12:::benchmark_data_smithi125_295816_object5798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:59:45.506 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.240816+0000", 2023-10-09T10:59:45.506 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 525.19413340000006, 2023-10-09T10:59:45.506 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.096789209000000001, 2023-10-09T10:59:45.506 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.506 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.506 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:45.506 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:59:45.507 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:45.507 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96935 2023-10-09T10:59:45.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.507 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.507 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.507 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240816+0000", 2023-10-09T10:59:45.507 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.507 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.508 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240816+0000", 2023-10-09T10:59:45.508 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.508 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.508 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.508 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.508 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240815+0000", 2023-10-09T10:59:45.508 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:45.508 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.508 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.508 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.508 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240821+0000", 2023-10-09T10:59:45.508 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1140000000000002e-06 2023-10-09T10:59:45.508 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.509 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240831+0000", 2023-10-09T10:59:45.509 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6309999999999998e-06 2023-10-09T10:59:45.509 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.509 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240845+0000", 2023-10-09T10:59:45.509 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4705000000000001e-05 2023-10-09T10:59:45.509 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.510 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242238+0000", 2023-10-09T10:59:45.510 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001392646 2023-10-09T10:59:45.510 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.510 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.510 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:59:45.510 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242937+0000", 2023-10-09T10:59:45.510 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00069863399999999995 2023-10-09T10:59:45.510 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.510 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.510 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.510 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.295227+0000", 2023-10-09T10:59:45.511 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.052290652999999999 2023-10-09T10:59:45.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.511 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.511 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.511 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.295329+0000", 2023-10-09T10:59:45.511 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010212700000000001 2023-10-09T10:59:45.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.511 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.511 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:59:45.511 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.295755+0000", 2023-10-09T10:59:45.511 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00042587400000000002 2023-10-09T10:59:45.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:45.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.308570+0000", 2023-10-09T10:59:45.512 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012814397 2023-10-09T10:59:45.512 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.329769+0000", 2023-10-09T10:59:45.513 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021199234000000001 2023-10-09T10:59:45.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.513 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.513 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.335181+0000", 2023-10-09T10:59:45.513 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0054126360000000002 2023-10-09T10:59:45.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.513 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.335231+0000", 2023-10-09T10:59:45.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9344000000000001e-05 2023-10-09T10:59:45.514 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.514 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.337605+0000", 2023-10-09T10:59:45.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002374555 2023-10-09T10:59:45.515 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.515 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.515 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.515 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96940 7.d 7:bfb000f4:::benchmark_data_smithi125_295816_object5803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T10:59:45.515 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.268461+0000", 2023-10-09T10:59:45.516 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 525.16648885400002, 2023-10-09T10:59:45.516 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.071660901999999999, 2023-10-09T10:59:45.516 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.516 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.516 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:45.516 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:59:45.516 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:45.516 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96940 2023-10-09T10:59:45.517 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.517 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.517 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.517 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268461+0000", 2023-10-09T10:59:45.517 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.517 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.518 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.518 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.518 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268461+0000", 2023-10-09T10:59:45.518 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.518 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.518 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.518 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.518 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268459+0000", 2023-10-09T10:59:45.519 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:45.519 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.519 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.519 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.519 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268477+0000", 2023-10-09T10:59:45.519 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7377999999999999e-05 2023-10-09T10:59:45.519 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.519 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.520 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.520 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268488+0000", 2023-10-09T10:59:45.520 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1579e-05 2023-10-09T10:59:45.520 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.520 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.520 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.520 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268495+0000", 2023-10-09T10:59:45.520 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5429999999999999e-06 2023-10-09T10:59:45.521 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.521 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.273724+0000", 2023-10-09T10:59:45.521 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0052292049999999998 2023-10-09T10:59:45.521 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:59:45.522 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.274443+0000", 2023-10-09T10:59:45.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00071879099999999998 2023-10-09T10:59:45.522 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.522 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.522 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.522 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.306460+0000", 2023-10-09T10:59:45.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.032016797999999999 2023-10-09T10:59:45.523 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.523 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.306545+0000", 2023-10-09T10:59:45.523 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5277000000000004e-05 2023-10-09T10:59:45.523 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:59:45.524 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.306674+0000", 2023-10-09T10:59:45.524 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012941600000000001 2023-10-09T10:59:45.524 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.524 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.524 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:45.524 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.317498+0000", 2023-10-09T10:59:45.524 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010823523999999999 2023-10-09T10:59:45.524 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.525 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.525 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.525 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.329799+0000", 2023-10-09T10:59:45.525 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012301559 2023-10-09T10:59:45.525 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.525 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.525 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.525 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.337661+0000", 2023-10-09T10:59:45.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007861303 2023-10-09T10:59:45.526 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.526 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.526 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.526 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.337718+0000", 2023-10-09T10:59:45.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6974e-05 2023-10-09T10:59:45.526 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.527 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.527 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.527 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.340122+0000", 2023-10-09T10:59:45.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024038660000000002 2023-10-09T10:59:45.527 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.527 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.527 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.528 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:98648 7.d 7:bbc4eb35:::benchmark_data_smithi125_295816_object7511:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:59:45.528 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:02.635379+0000", 2023-10-09T10:59:45.528 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 522.79957094400004, 2023-10-09T10:59:45.528 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.078747929999999994, 2023-10-09T10:59:45.528 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.528 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.528 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:45.528 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:59:45.529 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:45.529 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98648 2023-10-09T10:59:45.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.529 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.529 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635379+0000", 2023-10-09T10:59:45.529 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.530 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.530 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635379+0000", 2023-10-09T10:59:45.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.531 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.531 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635377+0000", 2023-10-09T10:59:45.531 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:45.531 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.531 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.531 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.531 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635382+0000", 2023-10-09T10:59:45.531 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3580000000000002e-06 2023-10-09T10:59:45.532 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.532 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.532 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.532 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635397+0000", 2023-10-09T10:59:45.532 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4265e-05 2023-10-09T10:59:45.532 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.532 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.532 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.533 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635403+0000", 2023-10-09T10:59:45.533 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4049999999999997e-06 2023-10-09T10:59:45.533 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.533 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.533 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.533 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.654397+0000", 2023-10-09T10:59:45.533 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018993432000000001 2023-10-09T10:59:45.533 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.534 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.534 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:59:45.534 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.661791+0000", 2023-10-09T10:59:45.534 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007394655 2023-10-09T10:59:45.534 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.534 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.534 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.534 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.671916+0000", 2023-10-09T10:59:45.535 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010125187000000001 2023-10-09T10:59:45.535 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.535 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.535 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.535 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.671975+0000", 2023-10-09T10:59:45.535 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.897e-05 2023-10-09T10:59:45.535 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.535 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.536 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:59:45.536 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.672065+0000", 2023-10-09T10:59:45.536 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9809999999999996e-05 2023-10-09T10:59:45.536 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.536 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.536 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:45.536 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.682405+0000", 2023-10-09T10:59:45.536 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010339924 2023-10-09T10:59:45.536 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.537 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.682443+0000", 2023-10-09T10:59:45.537 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7886000000000003e-05 2023-10-09T10:59:45.537 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.537 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.712060+0000", 2023-10-09T10:59:45.537 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029616844 2023-10-09T10:59:45.537 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.538 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.712116+0000", 2023-10-09T10:59:45.538 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6081000000000001e-05 2023-10-09T10:59:45.538 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.538 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.538 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.714127+0000", 2023-10-09T10:59:45.538 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002010572 2023-10-09T10:59:45.538 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.538 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.538 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.539 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.539 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.539 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:98673 7.d 7:b818009b:::benchmark_data_smithi125_295816_object7536:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:59:45.539 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:02.680579+0000", 2023-10-09T10:59:45.539 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 522.75437071299996, 2023-10-09T10:59:45.539 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.095416154000000003, 2023-10-09T10:59:45.539 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.539 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.539 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:45.539 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:59:45.539 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:45.539 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98673 2023-10-09T10:59:45.540 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.540 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.540 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.540 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680579+0000", 2023-10-09T10:59:45.540 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.540 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.540 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.540 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680579+0000", 2023-10-09T10:59:45.540 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.540 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.541 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680578+0000", 2023-10-09T10:59:45.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:59:45.541 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.541 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.541 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680582+0000", 2023-10-09T10:59:45.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8e-06 2023-10-09T10:59:45.541 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.541 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.541 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680588+0000", 2023-10-09T10:59:45.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4670000000000001e-06 2023-10-09T10:59:45.542 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.542 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.542 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.542 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680591+0000", 2023-10-09T10:59:45.542 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9519999999999999e-06 2023-10-09T10:59:45.542 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.542 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.542 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.542 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.727779+0000", 2023-10-09T10:59:45.543 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047187937999999999 2023-10-09T10:59:45.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:59:45.543 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.728358+0000", 2023-10-09T10:59:45.543 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00057898100000000001 2023-10-09T10:59:45.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.740190+0000", 2023-10-09T10:59:45.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01183184 2023-10-09T10:59:45.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.740303+0000", 2023-10-09T10:59:45.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112762 2023-10-09T10:59:45.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:59:45.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.740474+0000", 2023-10-09T10:59:45.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017171100000000001 2023-10-09T10:59:45.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.546 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:45.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.746764+0000", 2023-10-09T10:59:45.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062892499999999997 2023-10-09T10:59:45.546 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.546 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.546 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.756355+0000", 2023-10-09T10:59:45.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0095916060000000008 2023-10-09T10:59:45.547 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.547 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.547 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.774088+0000", 2023-10-09T10:59:45.547 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017732947999999998 2023-10-09T10:59:45.547 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.547 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.547 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.774137+0000", 2023-10-09T10:59:45.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8616999999999999e-05 2023-10-09T10:59:45.548 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.548 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.548 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.548 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.775995+0000", 2023-10-09T10:59:45.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018582340000000001 2023-10-09T10:59:45.548 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.548 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.548 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.548 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.548 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.548 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:98701 7.d 7:bd16628e:::benchmark_data_smithi125_295816_object7564:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T10:59:45.549 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:02.717662+0000", 2023-10-09T10:59:45.549 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 522.71728736199998, 2023-10-09T10:59:45.549 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.079670053000000005, 2023-10-09T10:59:45.549 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.549 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.549 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T10:59:45.549 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T10:59:45.549 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T10:59:45.549 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98701 2023-10-09T10:59:45.549 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.549 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.549 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.549 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.550 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717662+0000", 2023-10-09T10:59:45.550 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.550 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.550 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.550 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.550 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717662+0000", 2023-10-09T10:59:45.550 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.550 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.550 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.550 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.550 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717661+0000", 2023-10-09T10:59:45.550 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:59:45.551 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.551 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.551 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.551 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717666+0000", 2023-10-09T10:59:45.551 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.22e-06 2023-10-09T10:59:45.551 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.551 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.551 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.551 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717676+0000", 2023-10-09T10:59:45.551 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.944e-06 2023-10-09T10:59:45.551 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.552 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717681+0000", 2023-10-09T10:59:45.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3719999999999998e-06 2023-10-09T10:59:45.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.552 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.729691+0000", 2023-10-09T10:59:45.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012010140000000001 2023-10-09T10:59:45.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T10:59:45.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.732552+0000", 2023-10-09T10:59:45.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028607770000000001 2023-10-09T10:59:45.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.771785+0000", 2023-10-09T10:59:45.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039233280000000002 2023-10-09T10:59:45.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.771847+0000", 2023-10-09T10:59:45.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1909000000000006e-05 2023-10-09T10:59:45.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T10:59:45.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.771940+0000", 2023-10-09T10:59:45.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3058000000000002e-05 2023-10-09T10:59:45.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T10:59:45.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.778867+0000", 2023-10-09T10:59:45.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0069268180000000004 2023-10-09T10:59:45.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.783491+0000", 2023-10-09T10:59:45.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0046242490000000004 2023-10-09T10:59:45.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:45.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.795089+0000", 2023-10-09T10:59:45.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011598489999999999 2023-10-09T10:59:45.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.795150+0000", 2023-10-09T10:59:45.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0517999999999999e-05 2023-10-09T10:59:45.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.797332+0000", 2023-10-09T10:59:45.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021824079999999998 2023-10-09T10:59:45.556 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.556 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.556 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.556 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:91615 8.f e286/282 8:fb4b59c0:::benchmark_data_smithi125_329074_object91614:head v 286'5764, mlcod=0'0)", 2023-10-09T10:59:45.557 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:32.633558+0000", 2023-10-09T10:59:45.557 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 312.80139128799999, 2023-10-09T10:59:45.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028974272999999998, 2023-10-09T10:59:45.557 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.557 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.557 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.557 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633558+0000", 2023-10-09T10:59:45.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633558+0000", 2023-10-09T10:59:45.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633557+0000", 2023-10-09T10:59:45.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:45.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633600+0000", 2023-10-09T10:59:45.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3139000000000002e-05 2023-10-09T10:59:45.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633614+0000", 2023-10-09T10:59:45.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3512999999999999e-05 2023-10-09T10:59:45.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633621+0000", 2023-10-09T10:59:45.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0550000000000002e-06 2023-10-09T10:59:45.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633685+0000", 2023-10-09T10:59:45.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4276999999999995e-05 2023-10-09T10:59:45.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.633716+0000", 2023-10-09T10:59:45.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1134999999999999e-05 2023-10-09T10:59:45.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.662492+0000", 2023-10-09T10:59:45.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028775618999999999 2023-10-09T10:59:45.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.662533+0000", 2023-10-09T10:59:45.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0768e-05 2023-10-09T10:59:45.561 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.561 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.561 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.561 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:91623 8.7 e286/282 8:e8cb112d:::benchmark_data_smithi125_329074_object91622:head v 286'5732, mlcod=0'0)", 2023-10-09T10:59:45.562 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:32.634471+0000", 2023-10-09T10:59:45.562 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 312.80047850900002, 2023-10-09T10:59:45.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028088733000000001, 2023-10-09T10:59:45.562 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.562 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.562 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.562 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.562 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634471+0000", 2023-10-09T10:59:45.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.562 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634471+0000", 2023-10-09T10:59:45.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634470+0000", 2023-10-09T10:59:45.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:45.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634540+0000", 2023-10-09T10:59:45.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9406000000000006e-05 2023-10-09T10:59:45.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634556+0000", 2023-10-09T10:59:45.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5801999999999999e-05 2023-10-09T10:59:45.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.634562+0000", 2023-10-09T10:59:45.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9430000000000003e-06 2023-10-09T10:59:45.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.640689+0000", 2023-10-09T10:59:45.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061267650000000002 2023-10-09T10:59:45.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.640718+0000", 2023-10-09T10:59:45.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9187e-05 2023-10-09T10:59:45.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.662537+0000", 2023-10-09T10:59:45.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021818627 2023-10-09T10:59:45.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:32.662560+0000", 2023-10-09T10:59:45.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2708999999999999e-05 2023-10-09T10:59:45.566 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.566 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.566 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.567 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.567 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:91868 8.f e286/282 8:f4bd67fe:::benchmark_data_smithi125_329074_object91867:head v 286'5787, mlcod=0'0)", 2023-10-09T10:59:45.567 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:33.101662+0000", 2023-10-09T10:59:45.567 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 312.33328715900001, 2023-10-09T10:59:45.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026291749, 2023-10-09T10:59:45.567 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.567 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.567 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.567 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.567 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101662+0000", 2023-10-09T10:59:45.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101662+0000", 2023-10-09T10:59:45.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101661+0000", 2023-10-09T10:59:45.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:45.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101711+0000", 2023-10-09T10:59:45.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9974e-05 2023-10-09T10:59:45.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101726+0000", 2023-10-09T10:59:45.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4945e-05 2023-10-09T10:59:45.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.101733+0000", 2023-10-09T10:59:45.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1350000000000003e-06 2023-10-09T10:59:45.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.105791+0000", 2023-10-09T10:59:45.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040581360000000004 2023-10-09T10:59:45.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.105821+0000", 2023-10-09T10:59:45.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9422999999999999e-05 2023-10-09T10:59:45.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.127912+0000", 2023-10-09T10:59:45.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022090789999999999 2023-10-09T10:59:45.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:33.127954+0000", 2023-10-09T10:59:45.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2577999999999998e-05 2023-10-09T10:59:45.574 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.574 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.574 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.574 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.574 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:92334 8.f e286/282 8:fe8e3200:::benchmark_data_smithi125_329074_object92333:head v 286'5819, mlcod=0'0)", 2023-10-09T10:59:45.574 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:34.002645+0000", 2023-10-09T10:59:45.574 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 311.43230420499998, 2023-10-09T10:59:45.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028096320000000001, 2023-10-09T10:59:45.574 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.574 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.575 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.575 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002645+0000", 2023-10-09T10:59:45.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.575 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002645+0000", 2023-10-09T10:59:45.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002644+0000", 2023-10-09T10:59:45.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T10:59:45.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.576 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002709+0000", 2023-10-09T10:59:45.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4896999999999999e-05 2023-10-09T10:59:45.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.576 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002724+0000", 2023-10-09T10:59:45.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4543e-05 2023-10-09T10:59:45.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002735+0000", 2023-10-09T10:59:45.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1148000000000001e-05 2023-10-09T10:59:45.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002795+0000", 2023-10-09T10:59:45.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9908000000000003e-05 2023-10-09T10:59:45.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.002824+0000", 2023-10-09T10:59:45.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9065e-05 2023-10-09T10:59:45.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.030695+0000", 2023-10-09T10:59:45.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027871015999999998 2023-10-09T10:59:45.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.030742+0000", 2023-10-09T10:59:45.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6714e-05 2023-10-09T10:59:45.579 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.579 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.579 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.579 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:92495 8.9 e286/265 8:99c80bd2:::benchmark_data_smithi125_329074_object92494:head v 286'5773, mlcod=286'5771)", 2023-10-09T10:59:45.579 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:34.258707+0000", 2023-10-09T10:59:45.579 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 311.17624204700002, 2023-10-09T10:59:45.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025821361000000001, 2023-10-09T10:59:45.579 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.579 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.579 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258707+0000", 2023-10-09T10:59:45.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.580 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.580 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258707+0000", 2023-10-09T10:59:45.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.580 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.580 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258707+0000", 2023-10-09T10:59:45.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:45.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.581 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.581 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.581 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258744+0000", 2023-10-09T10:59:45.581 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6458000000000002e-05 2023-10-09T10:59:45.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.581 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.581 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.581 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258754+0000", 2023-10-09T10:59:45.581 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0482000000000001e-05 2023-10-09T10:59:45.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.581 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.581 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.582 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.258758+0000", 2023-10-09T10:59:45.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2910000000000001e-06 2023-10-09T10:59:45.582 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.582 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.582 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.582 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.267039+0000", 2023-10-09T10:59:45.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0082809919999999992 2023-10-09T10:59:45.582 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.582 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.582 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.582 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.267062+0000", 2023-10-09T10:59:45.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2532999999999999e-05 2023-10-09T10:59:45.583 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.583 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.583 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.284491+0000", 2023-10-09T10:59:45.583 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017429417999999999 2023-10-09T10:59:45.583 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.583 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.583 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:34.284529+0000", 2023-10-09T10:59:45.583 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7570000000000001e-05 2023-10-09T10:59:45.583 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.583 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.583 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.584 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.584 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.584 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:94596 8.f e288/282 8:ffee9f18:::benchmark_data_smithi125_329074_object94595:head v 288'5955, mlcod=0'0)", 2023-10-09T10:59:45.584 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:37.928730+0000", 2023-10-09T10:59:45.584 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 307.50621990500002, 2023-10-09T10:59:45.584 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028718649999999998, 2023-10-09T10:59:45.584 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.584 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.584 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.584 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.584 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.584 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928730+0000", 2023-10-09T10:59:45.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928730+0000", 2023-10-09T10:59:45.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928729+0000", 2023-10-09T10:59:45.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:45.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928781+0000", 2023-10-09T10:59:45.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1758000000000001e-05 2023-10-09T10:59:45.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928787+0000", 2023-10-09T10:59:45.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3180000000000003e-06 2023-10-09T10:59:45.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928792+0000", 2023-10-09T10:59:45.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6199999999999998e-06 2023-10-09T10:59:45.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.941317+0000", 2023-10-09T10:59:45.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012525087000000001 2023-10-09T10:59:45.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.941341+0000", 2023-10-09T10:59:45.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3408999999999999e-05 2023-10-09T10:59:45.588 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.588 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.588 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.957410+0000", 2023-10-09T10:59:45.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016068948999999999 2023-10-09T10:59:45.588 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.588 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.588 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.957448+0000", 2023-10-09T10:59:45.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8769000000000001e-05 2023-10-09T10:59:45.588 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.589 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.589 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.589 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.589 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.589 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.35433.0:94597 8.f e288/282 8:f1e41889:::benchmark_data_smithi125_329074_object94596:head v 288'5956, mlcod=0'0)", 2023-10-09T10:59:45.589 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:37.928795+0000", 2023-10-09T10:59:45.589 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 307.50615413999998, 2023-10-09T10:59:45.589 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.028674681, 2023-10-09T10:59:45.589 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T10:59:45.589 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T10:59:45.589 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T10:59:45.589 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T10:59:45.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928795+0000", 2023-10-09T10:59:45.590 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.590 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T10:59:45.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928795+0000", 2023-10-09T10:59:45.590 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T10:59:45.590 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T10:59:45.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928795+0000", 2023-10-09T10:59:45.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967296 2023-10-09T10:59:45.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.591 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T10:59:45.591 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928916+0000", 2023-10-09T10:59:45.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000120494 2023-10-09T10:59:45.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.591 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T10:59:45.591 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928927+0000", 2023-10-09T10:59:45.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1634000000000001e-05 2023-10-09T10:59:45.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T10:59:45.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.928931+0000", 2023-10-09T10:59:45.592 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0450000000000002e-06 2023-10-09T10:59:45.592 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.592 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T10:59:45.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.946895+0000", 2023-10-09T10:59:45.592 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017963923 2023-10-09T10:59:45.592 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.592 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T10:59:45.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.946918+0000", 2023-10-09T10:59:45.592 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2425999999999998e-05 2023-10-09T10:59:45.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.593 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T10:59:45.593 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.957452+0000", 2023-10-09T10:59:45.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010534484 2023-10-09T10:59:45.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T10:59:45.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T10:59:45.593 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T10:59:45.593 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:37.957470+0000", 2023-10-09T10:59:45.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7890999999999999e-05 2023-10-09T10:59:45.593 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.593 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.594 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.594 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T10:59:45.594 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T10:59:45.594 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:59:45.594 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:45.595 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:59:45.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:45 smithi155 ceph-mon[184854]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:45 smithi155 ceph-mon[184854]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:45 smithi155 ceph-mon[184854]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:45 smithi155 ceph-mon[184854]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.597 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:45 smithi155 ceph-mon[184854]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.597 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:45 smithi155 ceph-mon[184854]: failed to encode map e373 with expected crc 2023-10-09T10:59:45.597 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:45 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1978364622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:45.597 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T10:59:45.658 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:45.858 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:45.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:45.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:45.897 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T10:59:45.973 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:46.048 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:46.098 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:46.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:46.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:46.115 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:46.201 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:46.270 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:46.310 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:46.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:46.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:46.420 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:46.431 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:59:46.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:59:46.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1978364622' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:59:46.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[195507]: osdmap e374: 8 total, 7 up, 7 in 2023-10-09T10:59:46.469 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[195507]: failed to encode map e374 with expected crc 2023-10-09T10:59:46.469 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[195507]: failed to encode map e374 with expected crc 2023-10-09T10:59:46.469 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[195507]: failed to encode map e374 with expected crc 2023-10-09T10:59:46.469 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[195507]: failed to encode map e374 with expected crc 2023-10-09T10:59:46.469 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[195507]: failed to encode map e374 with expected crc 2023-10-09T10:59:46.469 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[195507]: pgmap v931: 129 pgs: 1 remapped+peering, 7 peering, 6 unknown, 21 active+clean+remapped, 2 active+remapped+backfilling, 8 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 31 active+undersized, 52 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 21957/211482 objects degraded (10.382%); 5014/211482 objects misplaced (2.371%) 2023-10-09T10:59:46.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:59:46.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1978364622' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:59:46.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[200197]: osdmap e374: 8 total, 7 up, 7 in 2023-10-09T10:59:46.470 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[200197]: failed to encode map e374 with expected crc 2023-10-09T10:59:46.470 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[200197]: failed to encode map e374 with expected crc 2023-10-09T10:59:46.470 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[200197]: failed to encode map e374 with expected crc 2023-10-09T10:59:46.470 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[200197]: failed to encode map e374 with expected crc 2023-10-09T10:59:46.470 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[200197]: failed to encode map e374 with expected crc 2023-10-09T10:59:46.470 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-mon[200197]: pgmap v931: 129 pgs: 1 remapped+peering, 7 peering, 6 unknown, 21 active+clean+remapped, 2 active+remapped+backfilling, 8 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 31 active+undersized, 52 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 21957/211482 objects degraded (10.382%); 5014/211482 objects misplaced (2.371%) 2023-10-09T10:59:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:46 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T10:59:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:46 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1978364622' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T10:59:46.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:46 smithi155 ceph-mon[184854]: osdmap e374: 8 total, 7 up, 7 in 2023-10-09T10:59:46.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:46 smithi155 ceph-mon[184854]: failed to encode map e374 with expected crc 2023-10-09T10:59:46.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:46 smithi155 ceph-mon[184854]: failed to encode map e374 with expected crc 2023-10-09T10:59:46.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:46 smithi155 ceph-mon[184854]: failed to encode map e374 with expected crc 2023-10-09T10:59:46.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:46 smithi155 ceph-mon[184854]: failed to encode map e374 with expected crc 2023-10-09T10:59:46.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:46 smithi155 ceph-mon[184854]: failed to encode map e374 with expected crc 2023-10-09T10:59:46.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:46 smithi155 ceph-mon[184854]: pgmap v931: 129 pgs: 1 remapped+peering, 7 peering, 6 unknown, 21 active+clean+remapped, 2 active+remapped+backfilling, 8 active+undersized+degraded, 1 active+clean+remapped+scrubbing, 31 active+undersized, 52 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 21957/211482 objects degraded (10.382%); 5014/211482 objects misplaced (2.371%) 2023-10-09T10:59:46.472 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:46.497 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:46.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:46.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:46.601 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:46.638 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:46.720 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:46.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:46.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:46.827 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:46.869 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:46.900 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:46.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:46.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:47.004 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:47.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:46 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T10:59:46.870+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e374 definitely_dead 0 2023-10-09T10:59:47.075 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:47.182 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:47.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:47.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:47.192 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.0. 2023-10-09T10:59:47.288 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:47.308 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:59:47.313 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:47 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2768717442' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T10:59:47.315 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:47.331 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:59:47.335 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:47.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:47.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:47.348 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2768717442' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T10:59:47.349 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2768717442' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T10:59:47.350 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:59:47.369 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:59:47.387 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:59:47.406 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:59:47.424 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:59:47.424 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:59:47.424 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-10-09T10:59:47.425 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:59:47.425 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:59:47.425 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:59:47.425 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:59:47.425 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:59:47.425 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:59:47.437 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:47.467 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:47.536 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:59:47.536 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:59:47.586 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:47.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:47.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:47.674 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-10-09T10:59:47.685 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T10:59:47.690 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:47.719 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:47.839 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:47.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:47.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:47.852 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:59:47.853 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:59:47.902 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T10:59:47.943 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:47.950 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:47.972 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:47.990 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:47.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:47.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:48.094 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:48.138 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:48.159 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:48.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:48.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:48.189 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:59:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:48 smithi155 ceph-mon[184854]: failed to encode map e375 with expected crc 2023-10-09T10:59:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:48 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2768717442' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T10:59:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:48 smithi155 ceph-mon[184854]: osdmap e375: 8 total, 7 up, 8 in 2023-10-09T10:59:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:48 smithi155 ceph-mon[184854]: failed to encode map e375 with expected crc 2023-10-09T10:59:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:48 smithi155 ceph-mon[184854]: failed to encode map e375 with expected crc 2023-10-09T10:59:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:48 smithi155 ceph-mon[184854]: failed to encode map e375 with expected crc 2023-10-09T10:59:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:48 smithi155 ceph-mon[184854]: failed to encode map e375 with expected crc 2023-10-09T10:59:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:48 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2528527365' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:48 smithi155 ceph-mon[184854]: pgmap v933: 129 pgs: 2 active+remapped+backfilling, 23 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 57 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 37909/202356 objects degraded (18.734%); 6211/202356 objects misplaced (3.069%); 55 MiB/s, 875 objects/s recovering 2023-10-09T10:59:48.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:48 smithi155 ceph-mon[184854]: osdmap e376: 8 total, 7 up, 8 in 2023-10-09T10:59:48.263 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:48.293 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:48.412 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:48.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[195507]: failed to encode map e375 with expected crc 2023-10-09T10:59:48.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2768717442' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T10:59:48.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[195507]: osdmap e375: 8 total, 7 up, 8 in 2023-10-09T10:59:48.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[195507]: failed to encode map e375 with expected crc 2023-10-09T10:59:48.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[195507]: failed to encode map e375 with expected crc 2023-10-09T10:59:48.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[195507]: failed to encode map e375 with expected crc 2023-10-09T10:59:48.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[195507]: failed to encode map e375 with expected crc 2023-10-09T10:59:48.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2528527365' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:48.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[195507]: pgmap v933: 129 pgs: 2 active+remapped+backfilling, 23 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 57 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 37909/202356 objects degraded (18.734%); 6211/202356 objects misplaced (3.069%); 55 MiB/s, 875 objects/s recovering 2023-10-09T10:59:48.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[195507]: osdmap e376: 8 total, 7 up, 8 in 2023-10-09T10:59:48.415 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[200197]: failed to encode map e375 with expected crc 2023-10-09T10:59:48.415 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2768717442' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T10:59:48.416 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[200197]: osdmap e375: 8 total, 7 up, 8 in 2023-10-09T10:59:48.416 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[200197]: failed to encode map e375 with expected crc 2023-10-09T10:59:48.416 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[200197]: failed to encode map e375 with expected crc 2023-10-09T10:59:48.416 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[200197]: failed to encode map e375 with expected crc 2023-10-09T10:59:48.416 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[200197]: failed to encode map e375 with expected crc 2023-10-09T10:59:48.416 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2528527365' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:48.416 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[200197]: pgmap v933: 129 pgs: 2 active+remapped+backfilling, 23 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 57 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 37909/202356 objects degraded (18.734%); 6211/202356 objects misplaced (3.069%); 55 MiB/s, 875 objects/s recovering 2023-10-09T10:59:48.416 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:48 smithi125 ceph-mon[200197]: osdmap e376: 8 total, 7 up, 8 in 2023-10-09T10:59:48.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:48.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:48.518 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:48.550 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:48.776 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:48.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:48.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:48.881 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:48.912 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:48.985 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:48.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:48.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:49.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:49.131 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:49.152 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:49.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:49.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:49.198 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:59:49.198 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:59:49.256 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:49.303 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:49.330 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:49.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:49.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:49.433 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:49.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:49 smithi155 ceph-mon[184854]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:49 smithi155 ceph-mon[184854]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:49 smithi155 ceph-mon[184854]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:49 smithi155 ceph-mon[184854]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:49 smithi155 ceph-mon[184854]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:49 smithi155 ceph-mon[184854]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:49 smithi155 ceph-mon[184854]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:49 smithi155 ceph-mon[184854]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:49 smithi155 ceph-mon[184854]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:49 smithi155 ceph-mon[184854]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:49 smithi155 ceph-mon[184854]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:49 smithi155 ceph-mon[184854]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:49 smithi155 ceph-mon[184854]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:49 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 37909/202356 objects degraded (18.734%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:49.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:49 smithi155 ceph-mon[184854]: osdmap e377: 8 total, 7 up, 8 in 2023-10-09T10:59:49.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[195507]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[195507]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[195507]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[195507]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[195507]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[195507]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[195507]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[195507]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[195507]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[195507]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[195507]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[195507]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[195507]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 37909/202356 objects degraded (18.734%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:49.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[195507]: osdmap e377: 8 total, 7 up, 8 in 2023-10-09T10:59:49.474 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[200197]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.474 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[200197]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[200197]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[200197]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[200197]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[200197]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[200197]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[200197]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[200197]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[200197]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[200197]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[200197]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[200197]: failed to encode map e376 with expected crc 2023-10-09T10:59:49.476 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 37909/202356 objects degraded (18.734%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:49.476 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:49 smithi125 ceph-mon[200197]: osdmap e377: 8 total, 7 up, 8 in 2023-10-09T10:59:49.476 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:49.505 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:49.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:49.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:49.580 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T10:59:49.608 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:49.652 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:49.684 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:49.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:49.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:49.788 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:49.831 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:49.854 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:59:49.859 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:49.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:49.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:49.961 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:49.990 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:59:50.009 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:50.047 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:50.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:50.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:50.130 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T10:59:50.157 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:50.192 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:50.266 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:50.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:50.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:50.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:50.402 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:50 smithi155 ceph-mon[184854]: 9.1 deep-scrub starts 2023-10-09T10:59:50.402 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:50 smithi155 ceph-mon[184854]: failed to encode map e377 with expected crc 2023-10-09T10:59:50.402 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:50 smithi155 ceph-mon[184854]: 9.b starting backfill to osd.0 from (336'700,370'9318] MIN to 374'9595 2023-10-09T10:59:50.402 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:50 smithi155 ceph-mon[184854]: failed to encode map e377 with expected crc 2023-10-09T10:59:50.403 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:50 smithi155 ceph-mon[184854]: failed to encode map e377 with expected crc 2023-10-09T10:59:50.403 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:50 smithi155 ceph-mon[184854]: failed to encode map e377 with expected crc 2023-10-09T10:59:50.403 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:50 smithi155 ceph-mon[184854]: failed to encode map e377 with expected crc 2023-10-09T10:59:50.403 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:50 smithi155 ceph-mon[184854]: 9.8 starting backfill to osd.0 from (345'3857,370'9284] MIN to 374'9423 2023-10-09T10:59:50.403 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:50 smithi155 ceph-mon[184854]: 9.9 starting backfill to osd.0 from (347'4453,370'9573] MIN to 374'9706 2023-10-09T10:59:50.403 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:50 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/104487162' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:59:50.403 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:50 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:59:50.403 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:50 smithi155 ceph-mon[184854]: pgmap v936: 129 pgs: 6 peering, 5 unknown, 2 active+remapped+backfilling, 23 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 46 active+clean; 4.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 37909/203169 objects degraded (18.659%); 6211/203169 objects misplaced (3.057%); 48 MiB/s, 761 objects/s recovering 2023-10-09T10:59:50.403 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:50 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:50.403 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:50 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:59:50.403 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:50 smithi155 ceph-mon[184854]: osdmap e378: 8 total, 7 up, 8 in 2023-10-09T10:59:50.413 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:50.440 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:50.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:50.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:50.521 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:50.525 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[195507]: 9.1 deep-scrub starts 2023-10-09T10:59:50.526 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[195507]: failed to encode map e377 with expected crc 2023-10-09T10:59:50.526 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[195507]: 9.b starting backfill to osd.0 from (336'700,370'9318] MIN to 374'9595 2023-10-09T10:59:50.526 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[195507]: failed to encode map e377 with expected crc 2023-10-09T10:59:50.526 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[195507]: failed to encode map e377 with expected crc 2023-10-09T10:59:50.526 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[195507]: failed to encode map e377 with expected crc 2023-10-09T10:59:50.526 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[195507]: failed to encode map e377 with expected crc 2023-10-09T10:59:50.526 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[195507]: 9.8 starting backfill to osd.0 from (345'3857,370'9284] MIN to 374'9423 2023-10-09T10:59:50.526 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[195507]: 9.9 starting backfill to osd.0 from (347'4453,370'9573] MIN to 374'9706 2023-10-09T10:59:50.526 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/104487162' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:59:50.526 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:59:50.527 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[195507]: pgmap v936: 129 pgs: 6 peering, 5 unknown, 2 active+remapped+backfilling, 23 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 46 active+clean; 4.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 37909/203169 objects degraded (18.659%); 6211/203169 objects misplaced (3.057%); 48 MiB/s, 761 objects/s recovering 2023-10-09T10:59:50.527 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:50.527 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:59:50.527 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[195507]: osdmap e378: 8 total, 7 up, 8 in 2023-10-09T10:59:50.527 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[200197]: 9.1 deep-scrub starts 2023-10-09T10:59:50.527 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[200197]: failed to encode map e377 with expected crc 2023-10-09T10:59:50.527 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[200197]: 9.b starting backfill to osd.0 from (336'700,370'9318] MIN to 374'9595 2023-10-09T10:59:50.527 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[200197]: failed to encode map e377 with expected crc 2023-10-09T10:59:50.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[200197]: failed to encode map e377 with expected crc 2023-10-09T10:59:50.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[200197]: failed to encode map e377 with expected crc 2023-10-09T10:59:50.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[200197]: failed to encode map e377 with expected crc 2023-10-09T10:59:50.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[200197]: 9.8 starting backfill to osd.0 from (345'3857,370'9284] MIN to 374'9423 2023-10-09T10:59:50.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[200197]: 9.9 starting backfill to osd.0 from (347'4453,370'9573] MIN to 374'9706 2023-10-09T10:59:50.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/104487162' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:59:50.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T10:59:50.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[200197]: pgmap v936: 129 pgs: 6 peering, 5 unknown, 2 active+remapped+backfilling, 23 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 46 active+clean; 4.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 37909/203169 objects degraded (18.659%); 6211/203169 objects misplaced (3.057%); 48 MiB/s, 761 objects/s recovering 2023-10-09T10:59:50.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:50.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T10:59:50.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:50 smithi125 ceph-mon[200197]: osdmap e378: 8 total, 7 up, 8 in 2023-10-09T10:59:50.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:50.584 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:50.659 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:50.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:50.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:50.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:50.812 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:50.943 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:50.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:50.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:51.047 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:51.104 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:51.186 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:51.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:51.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:51.263 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T10:59:51.263 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:59:51.286 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:51 smithi155 ceph-mon[184854]: failed to encode map e378 with expected crc 2023-10-09T10:59:51.286 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:51 smithi155 ceph-mon[184854]: failed to encode map e378 with expected crc 2023-10-09T10:59:51.286 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:51 smithi155 ceph-mon[184854]: failed to encode map e378 with expected crc 2023-10-09T10:59:51.286 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:51 smithi155 ceph-mon[184854]: failed to encode map e378 with expected crc 2023-10-09T10:59:51.286 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:51 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:59:51.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:51 smithi155 ceph-mon[184854]: osdmap e379: 8 total, 7 up, 8 in 2023-10-09T10:59:51.291 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:51.324 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:51.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:51 smithi125 ceph-mon[195507]: failed to encode map e378 with expected crc 2023-10-09T10:59:51.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:51 smithi125 ceph-mon[195507]: failed to encode map e378 with expected crc 2023-10-09T10:59:51.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:51 smithi125 ceph-mon[195507]: failed to encode map e378 with expected crc 2023-10-09T10:59:51.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:51 smithi125 ceph-mon[195507]: failed to encode map e378 with expected crc 2023-10-09T10:59:51.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:51 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:59:51.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:51 smithi125 ceph-mon[195507]: osdmap e379: 8 total, 7 up, 8 in 2023-10-09T10:59:51.395 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:51 smithi125 ceph-mon[200197]: failed to encode map e378 with expected crc 2023-10-09T10:59:51.395 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:51 smithi125 ceph-mon[200197]: failed to encode map e378 with expected crc 2023-10-09T10:59:51.395 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:51 smithi125 ceph-mon[200197]: failed to encode map e378 with expected crc 2023-10-09T10:59:51.396 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:51 smithi125 ceph-mon[200197]: failed to encode map e378 with expected crc 2023-10-09T10:59:51.396 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:51 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T10:59:51.396 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:51 smithi125 ceph-mon[200197]: osdmap e379: 8 total, 7 up, 8 in 2023-10-09T10:59:51.396 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 10:59:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:10:59:51] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.43.0" 2023-10-09T10:59:51.400 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:51.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:51.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:51.505 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:51.546 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:51.613 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:59:51.618 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:51.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:51.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:51.635 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:59:51.657 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:59:51.660 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T10:59:51.685 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:59:51.707 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:59:51.722 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:51.728 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:59:51.752 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:51.757 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:59:51.757 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T10:59:51.758 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-10-09T10:59:51.758 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T10:59:51.758 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T10:59:51.758 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T10:59:51.758 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T10:59:51.758 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T10:59:51.758 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T10:59:51.775 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:51.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:51.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:51.879 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:51.916 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:52.036 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:52.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:52.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:52.127 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:59:52.127 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:59:52.134 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:52.140 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:52.179 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:52.216 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:52.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:52.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:52.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[195507]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[195507]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[195507]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[195507]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[195507]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[195507]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[195507]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[195507]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[195507]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2170247532' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:52.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[195507]: pgmap v939: 129 pgs: 7 peering, 4 unknown, 2 active+remapped+backfilling, 18 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing+deep, 16 active+undersized+degraded, 31 active+undersized, 49 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 36205/194082 objects degraded (18.654%); 10689/194082 objects misplaced (5.507%); 5.5 MiB/s, 86 objects/s recovering 2023-10-09T10:59:52.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[195507]: osdmap e380: 8 total, 7 up, 8 in 2023-10-09T10:59:52.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[200197]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[200197]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[200197]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[200197]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[200197]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[200197]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[200197]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[200197]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[200197]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2170247532' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:52.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[200197]: pgmap v939: 129 pgs: 7 peering, 4 unknown, 2 active+remapped+backfilling, 18 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing+deep, 16 active+undersized+degraded, 31 active+undersized, 49 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 36205/194082 objects degraded (18.654%); 10689/194082 objects misplaced (5.507%); 5.5 MiB/s, 86 objects/s recovering 2023-10-09T10:59:52.288 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:52 smithi125 ceph-mon[200197]: osdmap e380: 8 total, 7 up, 8 in 2023-10-09T10:59:52.320 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:52.348 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:52.370 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:52.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:52.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:52 smithi155 ceph-mon[184854]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:52 smithi155 ceph-mon[184854]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:52 smithi155 ceph-mon[184854]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:52 smithi155 ceph-mon[184854]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:52 smithi155 ceph-mon[184854]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:52 smithi155 ceph-mon[184854]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:52 smithi155 ceph-mon[184854]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:52 smithi155 ceph-mon[184854]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:52 smithi155 ceph-mon[184854]: failed to encode map e379 with expected crc 2023-10-09T10:59:52.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:52 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2170247532' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:52.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:52 smithi155 ceph-mon[184854]: pgmap v939: 129 pgs: 7 peering, 4 unknown, 2 active+remapped+backfilling, 18 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing+deep, 16 active+undersized+degraded, 31 active+undersized, 49 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 36205/194082 objects degraded (18.654%); 10689/194082 objects misplaced (5.507%); 5.5 MiB/s, 86 objects/s recovering 2023-10-09T10:59:52.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:52 smithi155 ceph-mon[184854]: osdmap e380: 8 total, 7 up, 8 in 2023-10-09T10:59:52.474 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:52.512 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:52.535 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:52.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:52.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:52.547 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T10:59:52.638 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:52.672 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:52.675 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 7, 0] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-10-09T10:59:52.675 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T10:59:52.675 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T10:59:52.675 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T10:59:52.675 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T10:59:52.676 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T10:59:52.703 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:52.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:52.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:52.813 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:52.847 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:52.873 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:52.876 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:52.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:52.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:52.988 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:53.020 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T10:59:53.036 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:53.079 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:53.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:53.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:53.194 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:53.257 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:53.267 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:59:53.267 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T10:59:53.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.267 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157305 9.9 9.3681749 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T10:59:53.268 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:53.128382+0000", 2023-10-09T10:59:53.268 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.12757422800000001, 2023-10-09T10:59:53.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12759812700000001, 2023-10-09T10:59:53.268 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:53.268 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "delayed", 2023-10-09T10:59:53.268 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:53.268 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T10:59:53.268 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T10:59:53.268 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157305 2023-10-09T10:59:53.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.268 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:53.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:53.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128382+0000", 2023-10-09T10:59:53.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:53.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128382+0000", 2023-10-09T10:59:53.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:53.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128381+0000", 2023-10-09T10:59:53.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:53.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:53.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128386+0000", 2023-10-09T10:59:53.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1699999999999996e-06 2023-10-09T10:59:53.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:53.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128396+0000", 2023-10-09T10:59:53.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0259e-05 2023-10-09T10:59:53.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:53.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128404+0000", 2023-10-09T10:59:53.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.554e-06 2023-10-09T10:59:53.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:53.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.129279+0000", 2023-10-09T10:59:53.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00087499400000000003 2023-10-09T10:59:53.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:53.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.129287+0000", 2023-10-09T10:59:53.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4479999999999994e-06 2023-10-09T10:59:53.272 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.272 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:53.272 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.272 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "pg_backfill(progress 9.b e 381/381 lb 9:d079c756:::benchmark_data_smithi125_363789_object73878:head)", 2023-10-09T10:59:53.272 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:53.254540+0000", 2023-10-09T10:59:53.272 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0014169580000000001, 2023-10-09T10:59:53.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015143139999999999, 2023-10-09T10:59:53.272 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:53.272 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:59:53.272 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:53.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:53.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254540+0000", 2023-10-09T10:59:53.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:53.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254540+0000", 2023-10-09T10:59:53.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:53.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254539+0000", 2023-10-09T10:59:53.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:59:53.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:53.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254551+0000", 2023-10-09T10:59:53.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2169e-05 2023-10-09T10:59:53.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:53.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254564+0000", 2023-10-09T10:59:53.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3278999999999999e-05 2023-10-09T10:59:53.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:53.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254574+0000", 2023-10-09T10:59:53.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2820000000000002e-06 2023-10-09T10:59:53.275 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.275 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:53.275 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.275 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157332 9.9 9.cfa13929 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T10:59:53.275 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:53.128409+0000", 2023-10-09T10:59:53.275 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.127547255, 2023-10-09T10:59:53.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12767467399999999, 2023-10-09T10:59:53.275 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:53.275 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "delayed", 2023-10-09T10:59:53.275 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:53.276 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T10:59:53.276 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T10:59:53.276 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157332 2023-10-09T10:59:53.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.276 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:53.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:53.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128409+0000", 2023-10-09T10:59:53.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:53.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128409+0000", 2023-10-09T10:59:53.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:53.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128409+0000", 2023-10-09T10:59:53.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:53.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:53.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128411+0000", 2023-10-09T10:59:53.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0779999999999998e-06 2023-10-09T10:59:53.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:53.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128416+0000", 2023-10-09T10:59:53.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2399999999999998e-06 2023-10-09T10:59:53.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:53.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128421+0000", 2023-10-09T10:59:53.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4909999999999999e-06 2023-10-09T10:59:53.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:53.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.129306+0000", 2023-10-09T10:59:53.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00088470200000000001 2023-10-09T10:59:53.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:53.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.129311+0000", 2023-10-09T10:59:53.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3970000000000002e-06 2023-10-09T10:59:53.279 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.279 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:53.279 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.280 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157405 9.9 9.56887249 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T10:59:53.280 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:53.204403+0000", 2023-10-09T10:59:53.280 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.051554039000000003, 2023-10-09T10:59:53.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.051726034999999997, 2023-10-09T10:59:53.280 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:53.280 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "delayed", 2023-10-09T10:59:53.280 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:53.280 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T10:59:53.280 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T10:59:53.280 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157405 2023-10-09T10:59:53.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.280 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:53.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:53.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.204403+0000", 2023-10-09T10:59:53.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:53.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.204403+0000", 2023-10-09T10:59:53.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:53.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.204401+0000", 2023-10-09T10:59:53.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:53.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:53.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.204409+0000", 2023-10-09T10:59:53.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6029999999999999e-06 2023-10-09T10:59:53.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:53.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.204427+0000", 2023-10-09T10:59:53.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8692999999999999e-05 2023-10-09T10:59:53.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:53.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.204437+0000", 2023-10-09T10:59:53.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3109999999999995e-06 2023-10-09T10:59:53.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:53.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.204503+0000", 2023-10-09T10:59:53.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6019999999999995e-05 2023-10-09T10:59:53.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T10:59:53.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.204511+0000", 2023-10-09T10:59:53.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9950000000000005e-06 2023-10-09T10:59:53.284 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.284 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:53.284 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.284 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45683.0:157450 9.b e381/376 9:dafb8619:::benchmark_data_smithi125_363789_object47151:head v 381'9763, mlcod=381'9761)", 2023-10-09T10:59:53.284 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:53.254587+0000", 2023-10-09T10:59:53.284 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.001369182, 2023-10-09T10:59:53.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015899359999999999, 2023-10-09T10:59:53.284 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:53.284 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:59:53.284 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:53.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:53.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254587+0000", 2023-10-09T10:59:53.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:53.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254587+0000", 2023-10-09T10:59:53.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:53.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254587+0000", 2023-10-09T10:59:53.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:53.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:53.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254592+0000", 2023-10-09T10:59:53.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4399999999999996e-06 2023-10-09T10:59:53.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:53.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254599+0000", 2023-10-09T10:59:53.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9210000000000003e-06 2023-10-09T10:59:53.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:53.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254604+0000", 2023-10-09T10:59:53.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0320000000000003e-06 2023-10-09T10:59:53.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:53.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254772+0000", 2023-10-09T10:59:53.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000167706 2023-10-09T10:59:53.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:53.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254795+0000", 2023-10-09T10:59:53.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2918e-05 2023-10-09T10:59:53.288 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.288 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:53.288 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.288 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45683.0:157454 9.b e381/376)", 2023-10-09T10:59:53.288 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:53.254610+0000", 2023-10-09T10:59:53.288 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.001347121, 2023-10-09T10:59:53.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016331259999999999, 2023-10-09T10:59:53.288 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:53.288 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:59:53.288 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:53.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:53.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254610+0000", 2023-10-09T10:59:53.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:53.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254610+0000", 2023-10-09T10:59:53.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:53.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254609+0000", 2023-10-09T10:59:53.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:53.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:53.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254631+0000", 2023-10-09T10:59:53.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1721000000000001e-05 2023-10-09T10:59:53.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:53.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254637+0000", 2023-10-09T10:59:53.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1759999999999998e-06 2023-10-09T10:59:53.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:53.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254642+0000", 2023-10-09T10:59:53.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5549999999999996e-06 2023-10-09T10:59:53.291 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.291 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:53.291 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.291 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPush(9.b 381/376 [PushOp(9:d07ad4a7:::benchmark_data_smithi125_363789_object72748:head, version: 347'4487, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:d07ad4a7:::benchmark_data_smithi125_363789_object72748:head@347'4487, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T10:59:53.291 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:53.254647+0000", 2023-10-09T10:59:53.291 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0013097370000000001, 2023-10-09T10:59:53.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016700619999999999, 2023-10-09T10:59:53.291 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:53.291 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T10:59:53.291 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:53.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:53.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254647+0000", 2023-10-09T10:59:53.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:53.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254647+0000", 2023-10-09T10:59:53.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:53.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254647+0000", 2023-10-09T10:59:53.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T10:59:53.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:53.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254684+0000", 2023-10-09T10:59:53.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6952999999999997e-05 2023-10-09T10:59:53.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:53.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254698+0000", 2023-10-09T10:59:53.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4119e-05 2023-10-09T10:59:53.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:53.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.254711+0000", 2023-10-09T10:59:53.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3244999999999999e-05 2023-10-09T10:59:53.294 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.294 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:53.294 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.294 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45683.0:157448 9.18 e381/376 9:1f3820aa:::benchmark_data_smithi125_363789_object47149:head v 381'9524, mlcod=381'9522)", 2023-10-09T10:59:53.294 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:53.249235+0000", 2023-10-09T10:59:53.294 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0067214969999999999, 2023-10-09T10:59:53.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0071353639999999999, 2023-10-09T10:59:53.294 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:53.295 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T10:59:53.295 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:53.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:53.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.249235+0000", 2023-10-09T10:59:53.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:53.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.249235+0000", 2023-10-09T10:59:53.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:53.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.249234+0000", 2023-10-09T10:59:53.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T10:59:53.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:53.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.249243+0000", 2023-10-09T10:59:53.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1579999999999994e-06 2023-10-09T10:59:53.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:53.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.249260+0000", 2023-10-09T10:59:53.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7073999999999999e-05 2023-10-09T10:59:53.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:53.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.249269+0000", 2023-10-09T10:59:53.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1689999999999992e-06 2023-10-09T10:59:53.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:53.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.249313+0000", 2023-10-09T10:59:53.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4286999999999998e-05 2023-10-09T10:59:53.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:53.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.249334+0000", 2023-10-09T10:59:53.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1610000000000001e-05 2023-10-09T10:59:53.298 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.298 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:53.298 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.298 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157456 9.15 9:a9066651:::benchmark_data_smithi125_363789_object47157:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T10:59:53.298 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:53.252174+0000", 2023-10-09T10:59:53.298 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0037830020000000002, 2023-10-09T10:59:53.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042628140000000002, 2023-10-09T10:59:53.299 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T10:59:53.299 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T10:59:53.299 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T10:59:53.299 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T10:59:53.299 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T10:59:53.299 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157456 2023-10-09T10:59:53.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.299 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T10:59:53.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T10:59:53.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.252174+0000", 2023-10-09T10:59:53.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T10:59:53.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.252174+0000", 2023-10-09T10:59:53.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T10:59:53.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T10:59:53.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.252172+0000", 2023-10-09T10:59:53.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T10:59:53.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T10:59:53.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.252180+0000", 2023-10-09T10:59:53.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.011e-06 2023-10-09T10:59:53.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T10:59:53.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.252198+0000", 2023-10-09T10:59:53.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7499999999999998e-05 2023-10-09T10:59:53.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T10:59:53.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.252236+0000", 2023-10-09T10:59:53.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8206e-05 2023-10-09T10:59:53.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T10:59:53.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.252655+0000", 2023-10-09T10:59:53.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00041971 2023-10-09T10:59:53.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T10:59:53.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.252798+0000", 2023-10-09T10:59:53.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014209100000000001 2023-10-09T10:59:53.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,7", 2023-10-09T10:59:53.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.252896+0000", 2023-10-09T10:59:53.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8152999999999994e-05 2023-10-09T10:59:53.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:53.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.255029+0000", 2023-10-09T10:59:53.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021331290000000001 2023-10-09T10:59:53.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T10:59:53.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T10:59:53.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T10:59:53.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.256039+0000", 2023-10-09T10:59:53.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001010482 2023-10-09T10:59:53.303 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.303 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T10:59:53.303 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.303 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T10:59:53.304 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T10:59:53.304 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 9 2023-10-09T10:59:53.304 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:59:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:53 smithi155 ceph-mon[184854]: failed to encode map e380 with expected crc 2023-10-09T10:59:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:53 smithi155 ceph-mon[184854]: failed to encode map e380 with expected crc 2023-10-09T10:59:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:53 smithi155 ceph-mon[184854]: failed to encode map e380 with expected crc 2023-10-09T10:59:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:53 smithi155 ceph-mon[184854]: osdmap e381: 8 total, 7 up, 8 in 2023-10-09T10:59:53.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:53 smithi125 ceph-mon[195507]: failed to encode map e380 with expected crc 2023-10-09T10:59:53.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:53 smithi125 ceph-mon[195507]: failed to encode map e380 with expected crc 2023-10-09T10:59:53.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:53 smithi125 ceph-mon[195507]: failed to encode map e380 with expected crc 2023-10-09T10:59:53.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:53 smithi125 ceph-mon[195507]: osdmap e381: 8 total, 7 up, 8 in 2023-10-09T10:59:53.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:53 smithi125 ceph-mon[200197]: failed to encode map e380 with expected crc 2023-10-09T10:59:53.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:53 smithi125 ceph-mon[200197]: failed to encode map e380 with expected crc 2023-10-09T10:59:53.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:53 smithi125 ceph-mon[200197]: failed to encode map e380 with expected crc 2023-10-09T10:59:53.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:53 smithi125 ceph-mon[200197]: osdmap e381: 8 total, 7 up, 8 in 2023-10-09T10:59:53.938 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:53.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:53.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:54.043 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:54.076 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:54.128 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T10:59:54.223 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T10:59:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:54 smithi155 ceph-mon[184854]: failed to encode map e381 with expected crc 2023-10-09T10:59:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:54 smithi155 ceph-mon[184854]: failed to encode map e381 with expected crc 2023-10-09T10:59:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:54 smithi155 ceph-mon[184854]: failed to encode map e381 with expected crc 2023-10-09T10:59:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:54 smithi155 ceph-mon[184854]: pgmap v942: 129 pgs: 2 peering, 1 active+remapped+backfilling, 10 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing+deep, 16 active+undersized+degraded, 31 active+undersized, 67 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 36205/193092 objects degraded (18.750%); 5978/193092 objects misplaced (3.096%); 16 MiB/s, 256 objects/s recovering 2023-10-09T10:59:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:54 smithi155 ceph-mon[184854]: osdmap e382: 8 total, 7 up, 8 in 2023-10-09T10:59:54.555 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:54.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:54 smithi125 ceph-mon[195507]: failed to encode map e381 with expected crc 2023-10-09T10:59:54.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:54 smithi125 ceph-mon[195507]: failed to encode map e381 with expected crc 2023-10-09T10:59:54.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:54 smithi125 ceph-mon[195507]: failed to encode map e381 with expected crc 2023-10-09T10:59:54.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:54 smithi125 ceph-mon[195507]: pgmap v942: 129 pgs: 2 peering, 1 active+remapped+backfilling, 10 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing+deep, 16 active+undersized+degraded, 31 active+undersized, 67 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 36205/193092 objects degraded (18.750%); 5978/193092 objects misplaced (3.096%); 16 MiB/s, 256 objects/s recovering 2023-10-09T10:59:54.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:54 smithi125 ceph-mon[195507]: osdmap e382: 8 total, 7 up, 8 in 2023-10-09T10:59:54.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:54 smithi125 ceph-mon[200197]: failed to encode map e381 with expected crc 2023-10-09T10:59:54.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:54 smithi125 ceph-mon[200197]: failed to encode map e381 with expected crc 2023-10-09T10:59:54.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:54 smithi125 ceph-mon[200197]: failed to encode map e381 with expected crc 2023-10-09T10:59:54.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:54 smithi125 ceph-mon[200197]: pgmap v942: 129 pgs: 2 peering, 1 active+remapped+backfilling, 10 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing+deep, 16 active+undersized+degraded, 31 active+undersized, 67 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 36205/193092 objects degraded (18.750%); 5978/193092 objects misplaced (3.096%); 16 MiB/s, 256 objects/s recovering 2023-10-09T10:59:54.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:54 smithi125 ceph-mon[200197]: osdmap e382: 8 total, 7 up, 8 in 2023-10-09T10:59:54.636 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:54.753 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:54.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:54.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:54.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T10:59:54.899 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:55.024 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T10:59:55.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:55.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T10:59:55.128 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:55.155 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:55.268 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T10:59:55.294 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:55 smithi125 ceph-mon[195507]: failed to encode map e382 with expected crc 2023-10-09T10:59:55.295 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:55 smithi125 ceph-mon[195507]: failed to encode map e382 with expected crc 2023-10-09T10:59:55.295 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:55 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 36205/193092 objects degraded (18.750%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:55.295 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:55 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:59:55.295 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:55 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3898711432' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:55.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:55 smithi125 ceph-mon[200197]: failed to encode map e382 with expected crc 2023-10-09T10:59:55.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:55 smithi125 ceph-mon[200197]: failed to encode map e382 with expected crc 2023-10-09T10:59:55.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:55 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 36205/193092 objects degraded (18.750%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:55.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:55 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:59:55.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:55 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3898711432' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:55.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:55 smithi155 ceph-mon[184854]: failed to encode map e382 with expected crc 2023-10-09T10:59:55.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:55 smithi155 ceph-mon[184854]: failed to encode map e382 with expected crc 2023-10-09T10:59:55.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:55 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 36205/193092 objects degraded (18.750%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T10:59:55.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:55 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T10:59:55.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:55 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3898711432' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T10:59:55.683 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:55.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:55.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:55.788 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:55.831 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:55.863 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:55.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:55.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:55.971 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:56.012 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:56.044 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T10:59:56.044 INFO:teuthology.orchestra.run.smithi155.stdout: "success": "" 2023-10-09T10:59:56.044 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T10:59:56.135 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:56.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:56.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:56.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:56.257 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:56 smithi125 ceph-mon[195507]: 9.1 deep-scrub ok 2023-10-09T10:59:56.257 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:56 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:56.257 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3898711432' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:59:56.257 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:56 smithi125 ceph-mon[195507]: failed to encode map e383 with expected crc 2023-10-09T10:59:56.257 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:56 smithi125 ceph-mon[195507]: osdmap e383: 8 total, 7 up, 8 in 2023-10-09T10:59:56.257 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:56 smithi125 ceph-mon[195507]: failed to encode map e383 with expected crc 2023-10-09T10:59:56.257 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:56 smithi125 ceph-mon[195507]: pgmap v945: 129 pgs: 2 peering, 1 active+remapped+backfilling, 10 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing+deep, 16 active+undersized+degraded, 31 active+undersized, 67 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 80 op/s; 36205/193092 objects degraded (18.750%); 5978/193092 objects misplaced (3.096%); 1.8 MiB/s, 31 objects/s recovering 2023-10-09T10:59:56.257 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:56 smithi125 ceph-mon[200197]: 9.1 deep-scrub ok 2023-10-09T10:59:56.258 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:56 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:56.258 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3898711432' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:59:56.258 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:56 smithi125 ceph-mon[200197]: failed to encode map e383 with expected crc 2023-10-09T10:59:56.258 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:56 smithi125 ceph-mon[200197]: osdmap e383: 8 total, 7 up, 8 in 2023-10-09T10:59:56.258 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:56 smithi125 ceph-mon[200197]: failed to encode map e383 with expected crc 2023-10-09T10:59:56.258 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:56 smithi125 ceph-mon[200197]: pgmap v945: 129 pgs: 2 peering, 1 active+remapped+backfilling, 10 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing+deep, 16 active+undersized+degraded, 31 active+undersized, 67 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 80 op/s; 36205/193092 objects degraded (18.750%); 5978/193092 objects misplaced (3.096%); 1.8 MiB/s, 31 objects/s recovering 2023-10-09T10:59:56.266 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T10:59:56.266 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T10:59:56.267 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T10:59:56.267 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T10:59:56.267 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T10:59:56.305 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:56.357 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T10:59:56.380 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T10:59:56.387 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:56.388 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:56 smithi155 ceph-mon[184854]: 9.1 deep-scrub ok 2023-10-09T10:59:56.388 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:56 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T10:59:56.388 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3898711432' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T10:59:56.388 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:56 smithi155 ceph-mon[184854]: failed to encode map e383 with expected crc 2023-10-09T10:59:56.388 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:56 smithi155 ceph-mon[184854]: osdmap e383: 8 total, 7 up, 8 in 2023-10-09T10:59:56.388 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:56 smithi155 ceph-mon[184854]: failed to encode map e383 with expected crc 2023-10-09T10:59:56.388 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:56 smithi155 ceph-mon[184854]: pgmap v945: 129 pgs: 2 peering, 1 active+remapped+backfilling, 10 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing+deep, 16 active+undersized+degraded, 31 active+undersized, 67 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 80 op/s; 36205/193092 objects degraded (18.750%); 5978/193092 objects misplaced (3.096%); 1.8 MiB/s, 31 objects/s recovering 2023-10-09T10:59:56.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:56.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:56.407 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T10:59:56.433 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T10:59:56.456 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T10:59:56.479 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T10:59:56.492 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:56.500 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T10:59:56.500 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T10:59:56.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-10-09T10:59:56.500 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T10:59:56.500 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T10:59:56.500 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T10:59:56.500 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T10:59:56.500 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T10:59:56.500 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T10:59:56.522 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:56.543 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:56.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:56.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:56.648 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:56.676 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:56.694 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:56.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:56.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:56.717 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T10:59:56.797 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:56.824 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:56.842 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:56.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:56.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:56.901 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T10:59:56.901 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T10:59:56.946 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:56.972 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:56.991 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:56.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:56.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:57.024 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:57.094 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:57.121 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:57.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:57.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:57.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:57.243 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T10:59:57.270 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:57.290 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T10:59:57.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:57.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T10:59:57.394 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:57.448 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:57 smithi125 ceph-mon[195507]: failed to encode map e384 with expected crc 2023-10-09T10:59:57.448 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:57 smithi125 ceph-mon[195507]: osdmap e384: 8 total, 7 up, 8 in 2023-10-09T10:59:57.448 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:57 smithi125 ceph-mon[195507]: failed to encode map e384 with expected crc 2023-10-09T10:59:57.448 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:57 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1214623354' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:57.449 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:57 smithi125 ceph-mon[200197]: failed to encode map e384 with expected crc 2023-10-09T10:59:57.449 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:57 smithi125 ceph-mon[200197]: osdmap e384: 8 total, 7 up, 8 in 2023-10-09T10:59:57.449 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:57 smithi125 ceph-mon[200197]: failed to encode map e384 with expected crc 2023-10-09T10:59:57.449 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:57 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1214623354' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:57.454 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:57.487 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:57.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:57.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:57.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T10:59:57.671 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:57 smithi155 ceph-mon[184854]: failed to encode map e384 with expected crc 2023-10-09T10:59:57.671 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:57 smithi155 ceph-mon[184854]: osdmap e384: 8 total, 7 up, 8 in 2023-10-09T10:59:57.671 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:57 smithi155 ceph-mon[184854]: failed to encode map e384 with expected crc 2023-10-09T10:59:57.671 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:57 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1214623354' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T10:59:57.673 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:57.704 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T10:59:57.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:57.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T10:59:57.811 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:57.853 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:57.875 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T10:59:57.979 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T10:59:57.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:57.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T10:59:58.087 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:58.110 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T10:59:58.110 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T10:59:58.136 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:58.164 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:58.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:58.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:58.270 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T10:59:58.315 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:58.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:58 smithi125 ceph-mon[195507]: failed to encode map e385 with expected crc 2023-10-09T10:59:58.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:58 smithi125 ceph-mon[195507]: osdmap e385: 8 total, 7 up, 8 in 2023-10-09T10:59:58.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:58 smithi125 ceph-mon[195507]: failed to encode map e385 with expected crc 2023-10-09T10:59:58.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:58 smithi125 ceph-mon[195507]: failed to encode map e385 with expected crc 2023-10-09T10:59:58.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:58 smithi125 ceph-mon[195507]: failed to encode map e385 with expected crc 2023-10-09T10:59:58.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:58 smithi125 ceph-mon[195507]: pgmap v948: 129 pgs: 10 active+clean+remapped, 1 active+remapped+backfill_wait, 16 active+undersized+degraded, 31 active+undersized, 71 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 34383/184620 objects degraded (18.624%); 2060/184620 objects misplaced (1.116%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T10:59:58.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:58 smithi125 ceph-mon[200197]: failed to encode map e385 with expected crc 2023-10-09T10:59:58.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:58 smithi125 ceph-mon[200197]: osdmap e385: 8 total, 7 up, 8 in 2023-10-09T10:59:58.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:58 smithi125 ceph-mon[200197]: failed to encode map e385 with expected crc 2023-10-09T10:59:58.423 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:58 smithi125 ceph-mon[200197]: failed to encode map e385 with expected crc 2023-10-09T10:59:58.423 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:58 smithi125 ceph-mon[200197]: failed to encode map e385 with expected crc 2023-10-09T10:59:58.423 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:58 smithi125 ceph-mon[200197]: pgmap v948: 129 pgs: 10 active+clean+remapped, 1 active+remapped+backfill_wait, 16 active+undersized+degraded, 31 active+undersized, 71 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 34383/184620 objects degraded (18.624%); 2060/184620 objects misplaced (1.116%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T10:59:58.445 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T10:59:58.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:58.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T10:59:58.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:58 smithi155 ceph-mon[184854]: failed to encode map e385 with expected crc 2023-10-09T10:59:58.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:58 smithi155 ceph-mon[184854]: osdmap e385: 8 total, 7 up, 8 in 2023-10-09T10:59:58.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:58 smithi155 ceph-mon[184854]: failed to encode map e385 with expected crc 2023-10-09T10:59:58.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:58 smithi155 ceph-mon[184854]: failed to encode map e385 with expected crc 2023-10-09T10:59:58.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:58 smithi155 ceph-mon[184854]: failed to encode map e385 with expected crc 2023-10-09T10:59:58.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:58 smithi155 ceph-mon[184854]: pgmap v948: 129 pgs: 10 active+clean+remapped, 1 active+remapped+backfill_wait, 16 active+undersized+degraded, 31 active+undersized, 71 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 34383/184620 objects degraded (18.624%); 2060/184620 objects misplaced (1.116%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T10:59:58.477 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T10:59:58.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T10:59:58.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T10:59:58.606 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:58.634 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T10:59:58.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:58.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T10:59:58.738 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T10:59:58.780 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:58.820 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T10:59:58.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T10:59:58.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T10:59:58.902 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T10:59:58.927 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T10:59:58.987 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T10:59:59.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 10:59:59 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 34383/183012 objects degraded (18.787%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:00.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 10:59:59 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 34383/183012 objects degraded (18.787%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:00.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 10:59:59 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 34383/183012 objects degraded (18.787%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:00.193 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:00.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:00.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:00.245 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:00.305 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:00.370 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:00.381 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:00.408 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:00.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:00.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:00.517 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:00.573 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:00.608 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:00.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:00.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:00.716 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:00.759 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:00.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pgmap v949: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 3.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 34383/183012 objects degraded (18.787%); 1/183012 objects misplaced (0.001%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T11:00:00.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 34383/183012 objects degraded (18.787%), 16 pgs degraded 2023-10-09T11:00:00.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T11:00:00.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: [WRN] OSD_DOWN: 1 osds down 2023-10-09T11:00:00.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: osd.1 (root=default,host=smithi125) is down 2023-10-09T11:00:00.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: [WRN] PG_DEGRADED: Degraded data redundancy: 34383/183012 objects degraded (18.787%), 16 pgs degraded 2023-10-09T11:00:00.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 3.e is active+undersized+degraded, acting [7,4] 2023-10-09T11:00:00.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 3.1c is active+undersized+degraded, acting [4,5] 2023-10-09T11:00:00.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 9.0 is active+undersized+degraded, acting [2,6] 2023-10-09T11:00:00.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 9.2 is active+undersized+degraded, acting [3,2] 2023-10-09T11:00:00.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 9.3 is active+undersized+degraded, acting [7,3] 2023-10-09T11:00:00.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 9.4 is active+undersized+degraded, acting [3,4] 2023-10-09T11:00:00.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 9.6 is active+undersized+degraded, acting [2,5] 2023-10-09T11:00:00.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 9.a is active+undersized+degraded, acting [7,5] 2023-10-09T11:00:00.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 9.d is active+undersized+degraded, acting [7,5] 2023-10-09T11:00:00.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 9.e is active+undersized+degraded, acting [7,4] 2023-10-09T11:00:00.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 9.f is active+undersized+degraded, acting [3,5] 2023-10-09T11:00:00.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 9.10 is active+undersized+degraded, acting [2,6] 2023-10-09T11:00:00.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 9.12 is active+undersized+degraded, acting [3,2] 2023-10-09T11:00:00.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 9.13 is active+undersized+degraded, acting [7,3] 2023-10-09T11:00:00.975 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 9.14 is active+undersized+degraded, acting [3,4] 2023-10-09T11:00:00.975 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: pg 9.16 is active+undersized+degraded, acting [2,5] 2023-10-09T11:00:00.975 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:00:01.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pgmap v949: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 3.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 34383/183012 objects degraded (18.787%); 1/183012 objects misplaced (0.001%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T11:00:01.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 34383/183012 objects degraded (18.787%), 16 pgs degraded 2023-10-09T11:00:01.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T11:00:01.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: [WRN] OSD_DOWN: 1 osds down 2023-10-09T11:00:01.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: osd.1 (root=default,host=smithi125) is down 2023-10-09T11:00:01.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: [WRN] PG_DEGRADED: Degraded data redundancy: 34383/183012 objects degraded (18.787%), 16 pgs degraded 2023-10-09T11:00:01.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 3.e is active+undersized+degraded, acting [7,4] 2023-10-09T11:00:01.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 3.1c is active+undersized+degraded, acting [4,5] 2023-10-09T11:00:01.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 9.0 is active+undersized+degraded, acting [2,6] 2023-10-09T11:00:01.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 9.2 is active+undersized+degraded, acting [3,2] 2023-10-09T11:00:01.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 9.3 is active+undersized+degraded, acting [7,3] 2023-10-09T11:00:01.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 9.4 is active+undersized+degraded, acting [3,4] 2023-10-09T11:00:01.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 9.6 is active+undersized+degraded, acting [2,5] 2023-10-09T11:00:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 9.a is active+undersized+degraded, acting [7,5] 2023-10-09T11:00:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 9.d is active+undersized+degraded, acting [7,5] 2023-10-09T11:00:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 9.e is active+undersized+degraded, acting [7,4] 2023-10-09T11:00:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 9.f is active+undersized+degraded, acting [3,5] 2023-10-09T11:00:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 9.10 is active+undersized+degraded, acting [2,6] 2023-10-09T11:00:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 9.12 is active+undersized+degraded, acting [3,2] 2023-10-09T11:00:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 9.13 is active+undersized+degraded, acting [7,3] 2023-10-09T11:00:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 9.14 is active+undersized+degraded, acting [3,4] 2023-10-09T11:00:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: pg 9.16 is active+undersized+degraded, acting [2,5] 2023-10-09T11:00:01.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:00:01.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pgmap v949: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 3.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 34383/183012 objects degraded (18.787%); 1/183012 objects misplaced (0.001%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T11:00:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 34383/183012 objects degraded (18.787%), 16 pgs degraded 2023-10-09T11:00:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T11:00:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: [WRN] OSD_DOWN: 1 osds down 2023-10-09T11:00:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: osd.1 (root=default,host=smithi125) is down 2023-10-09T11:00:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: [WRN] PG_DEGRADED: Degraded data redundancy: 34383/183012 objects degraded (18.787%), 16 pgs degraded 2023-10-09T11:00:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 3.e is active+undersized+degraded, acting [7,4] 2023-10-09T11:00:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 3.1c is active+undersized+degraded, acting [4,5] 2023-10-09T11:00:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 9.0 is active+undersized+degraded, acting [2,6] 2023-10-09T11:00:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 9.2 is active+undersized+degraded, acting [3,2] 2023-10-09T11:00:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 9.3 is active+undersized+degraded, acting [7,3] 2023-10-09T11:00:01.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 9.4 is active+undersized+degraded, acting [3,4] 2023-10-09T11:00:01.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 9.6 is active+undersized+degraded, acting [2,5] 2023-10-09T11:00:01.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 9.a is active+undersized+degraded, acting [7,5] 2023-10-09T11:00:01.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 9.d is active+undersized+degraded, acting [7,5] 2023-10-09T11:00:01.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 9.e is active+undersized+degraded, acting [7,4] 2023-10-09T11:00:01.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 9.f is active+undersized+degraded, acting [3,5] 2023-10-09T11:00:01.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 9.10 is active+undersized+degraded, acting [2,6] 2023-10-09T11:00:01.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 9.12 is active+undersized+degraded, acting [3,2] 2023-10-09T11:00:01.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 9.13 is active+undersized+degraded, acting [7,3] 2023-10-09T11:00:01.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 9.14 is active+undersized+degraded, acting [3,4] 2023-10-09T11:00:01.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: pg 9.16 is active+undersized+degraded, acting [2,5] 2023-10-09T11:00:01.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:00:01.085 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:01.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:01.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:01.190 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:01.231 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:01.326 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:01.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:01.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:01.426 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:00:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:00:01] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.43.0" 2023-10-09T11:00:01.435 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:01.440 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 7, 0] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-10-09T11:00:01.440 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:00:01.440 INFO:tasks.thrashosds.thrasher:Reviving osd 1 2023-10-09T11:00:01.440 INFO:tasks.cephadm.osd.1:Restarting osd.1 (starting--it wasn't running)... 2023-10-09T11:00:01.440 DEBUG:teuthology.orchestra.run.smithi125:osd.1> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-09T11:00:01.477 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:01.482 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl start ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T11:00:01.511 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:01.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:01.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:01.538 INFO:journalctl@ceph.osd.1.smithi125.stdout:-- Logs begin at Mon 2023-10-09 10:18:49 UTC. -- 2023-10-09T11:00:01.617 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:01.675 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:00:01 smithi125 systemd[1]: Starting Ceph osd.1 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:00:01.677 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:02.018 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:02.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:02.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:02.134 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:02.188 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:02.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:02.239 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:02.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:02.340 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:02.393 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:02.467 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:02.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:02.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:02.571 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:02.614 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:02.639 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:02.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:02.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:02.699 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:00:02.734 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:00:02.744 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:00:02.745 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:02.775 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:00:02.775 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:00:02.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:02.775 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:168037 9.10 9:0fccd976:::benchmark_data_smithi125_363789_object57738:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-10-09T11:00:02.775 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:00:02.769836+0000", 2023-10-09T11:00:02.775 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0044424690000000001, 2023-10-09T11:00:02.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0044696229999999998, 2023-10-09T11:00:02.775 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:02.775 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:00:02.776 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:02.776 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:02.776 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:02.776 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 168037 2023-10-09T11:00:02.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:02.776 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:02.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:02.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:02.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:02.769836+0000", 2023-10-09T11:00:02.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:02.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:02.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:02.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:02.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:02.769836+0000", 2023-10-09T11:00:02.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:02.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:02.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:02.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:02.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:02.769833+0000", 2023-10-09T11:00:02.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T11:00:02.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:02.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:02.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:02.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:02.769844+0000", 2023-10-09T11:00:02.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0825000000000001e-05 2023-10-09T11:00:02.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:02.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:02.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:02.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:02.769867+0000", 2023-10-09T11:00:02.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3139999999999999e-05 2023-10-09T11:00:02.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:02.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:02.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:02.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:02.769877+0000", 2023-10-09T11:00:02.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0013000000000001e-05 2023-10-09T11:00:02.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:02.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:02.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:02.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:02.769930+0000", 2023-10-09T11:00:02.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2608999999999997e-05 2023-10-09T11:00:02.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:02.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:02.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:02.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:02.770059+0000", 2023-10-09T11:00:02.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012887799999999999 2023-10-09T11:00:02.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:02.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:02.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 6", 2023-10-09T11:00:02.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:02.770160+0000", 2023-10-09T11:00:02.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000101748 2023-10-09T11:00:02.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:02.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:02.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:02.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:02.772490+0000", 2023-10-09T11:00:02.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023292109999999999 2023-10-09T11:00:02.780 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:02.780 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:02.780 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:02.780 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:02.780 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T11:00:02.780 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 1 2023-10-09T11:00:02.780 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:00:02.782 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:00:02.823 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:00:02.849 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:00:02.857 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:02.881 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:00:02.899 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:02.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:02.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:02.920 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:00:02.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:00:02.921 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-10-09T11:00:02.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:00:02.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:00:02.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:00:02.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:00:02.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:00:02.921 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:00:02.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:02 smithi155 ceph-mon[184854]: pgmap v950: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 3.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 30855/167871 objects degraded (18.380%); 1/167871 objects misplaced (0.001%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T11:00:02.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:02 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2029352344' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:00:02.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:02 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:00:02.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:02 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/63356779' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:03.006 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:03.045 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:02 smithi125 ceph-mon[195507]: pgmap v950: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 3.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 30855/167871 objects degraded (18.380%); 1/167871 objects misplaced (0.001%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T11:00:03.045 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:02 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2029352344' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:00:03.045 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:02 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:00:03.045 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:02 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/63356779' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:03.046 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:02 smithi125 ceph-mon[200197]: pgmap v950: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 3.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 30855/167871 objects degraded (18.380%); 1/167871 objects misplaced (0.001%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T11:00:03.046 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:02 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2029352344' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:00:03.046 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:02 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:00:03.046 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:02 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/63356779' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:03.047 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:03.328 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:03.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:03.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:03.432 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:03.475 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:03.508 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:03.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:03.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:03.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T11:00:03.617 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:03.687 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:03.699 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T11:00:03.700 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T11:00:03.839 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:03.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:03.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:03.955 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:03.989 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:00:03 smithi125 bash[390781]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T11:00:03.989 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:00:03 smithi125 bash[390781]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T11:00:03.989 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:00:03 smithi125 bash[390781]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-09T11:00:03.989 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:00:03 smithi125 bash[390781]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-09T11:00:03.989 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:00:03 smithi125 bash[390781]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-09T11:00:03.989 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:00:03 smithi125 bash[390781]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T11:00:03.990 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:00:03 smithi125 bash[390781]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T11:00:03.990 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:00:03 smithi125 bash[390781]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-09T11:00:03.990 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:00:03.990 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:00:03.990 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[200197]: osdmap e386: 8 total, 7 up, 8 in 2023-10-09T11:00:03.990 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[200197]: failed to encode map e386 with expected crc 2023-10-09T11:00:03.990 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[200197]: failed to encode map e386 with expected crc 2023-10-09T11:00:03.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[200197]: failed to encode map e386 with expected crc 2023-10-09T11:00:03.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[200197]: failed to encode map e386 with expected crc 2023-10-09T11:00:03.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[200197]: failed to encode map e386 with expected crc 2023-10-09T11:00:03.991 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:00:03.991 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:00:03.991 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[195507]: osdmap e386: 8 total, 7 up, 8 in 2023-10-09T11:00:03.991 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[195507]: failed to encode map e386 with expected crc 2023-10-09T11:00:03.991 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[195507]: failed to encode map e386 with expected crc 2023-10-09T11:00:03.991 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[195507]: failed to encode map e386 with expected crc 2023-10-09T11:00:03.991 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[195507]: failed to encode map e386 with expected crc 2023-10-09T11:00:03.992 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:03 smithi125 ceph-mon[195507]: failed to encode map e386 with expected crc 2023-10-09T11:00:03.993 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:04.021 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:04.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:04.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:04.118 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:04.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:04.178 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:04.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:03 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:00:04.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:03 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:00:04.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:03 smithi155 ceph-mon[184854]: osdmap e386: 8 total, 7 up, 8 in 2023-10-09T11:00:04.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:03 smithi155 ceph-mon[184854]: failed to encode map e386 with expected crc 2023-10-09T11:00:04.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:03 smithi155 ceph-mon[184854]: failed to encode map e386 with expected crc 2023-10-09T11:00:04.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:03 smithi155 ceph-mon[184854]: failed to encode map e386 with expected crc 2023-10-09T11:00:04.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:03 smithi155 ceph-mon[184854]: failed to encode map e386 with expected crc 2023-10-09T11:00:04.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:03 smithi155 ceph-mon[184854]: failed to encode map e386 with expected crc 2023-10-09T11:00:04.304 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:04.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:04.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:04.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:04.460 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:04.490 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:04.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:04.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:04.590 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T11:00:04.597 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:04.655 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:04.683 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:04.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:04.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:04.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:04.836 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:04.874 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:04.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:04.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:04.880 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:04 smithi125 ceph-mon[200197]: pgmap v952: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 3.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 30855/163647 objects degraded (18.855%); 1/163647 objects misplaced (0.001%); 935 KiB/s, 14 objects/s recovering 2023-10-09T11:00:04.880 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:04 smithi125 ceph-mon[200197]: 1.b scrub starts 2023-10-09T11:00:04.881 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:04 smithi125 ceph-mon[200197]: 1.b scrub ok 2023-10-09T11:00:04.881 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:04 smithi125 ceph-mon[200197]: 9.7 scrub starts 2023-10-09T11:00:04.881 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:00:04 smithi125 ceph-osd[391470]: -- 172.21.15.125:0/2561690320 <== mon.2 v2:172.21.15.125:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55df793444e0 con 0x55df79314c00 2023-10-09T11:00:04.881 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:00:04 smithi125 bash[391334]: dd79a31493871f91c25026d91b2a56caf24303f7d4d6ca22a9cffcb8dc87fc5c 2023-10-09T11:00:04.881 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:00:04 smithi125 systemd[1]: Started Ceph osd.1 for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T11:00:04.881 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:04 smithi125 ceph-mon[195507]: pgmap v952: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 3.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 30855/163647 objects degraded (18.855%); 1/163647 objects misplaced (0.001%); 935 KiB/s, 14 objects/s recovering 2023-10-09T11:00:04.881 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:04 smithi125 ceph-mon[195507]: 1.b scrub starts 2023-10-09T11:00:04.881 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:04 smithi125 ceph-mon[195507]: 1.b scrub ok 2023-10-09T11:00:04.882 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:04 smithi125 ceph-mon[195507]: 9.7 scrub starts 2023-10-09T11:00:04.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:04 smithi155 ceph-mon[184854]: pgmap v952: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 3.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 30855/163647 objects degraded (18.855%); 1/163647 objects misplaced (0.001%); 935 KiB/s, 14 objects/s recovering 2023-10-09T11:00:04.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:04 smithi155 ceph-mon[184854]: 1.b scrub starts 2023-10-09T11:00:04.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:04 smithi155 ceph-mon[184854]: 1.b scrub ok 2023-10-09T11:00:04.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:04 smithi155 ceph-mon[184854]: 9.7 scrub starts 2023-10-09T11:00:04.981 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:04.994 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:05.024 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:05.132 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:00:05 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[391396]: 2023-10-09T11:00:05.113+0000 7f8d41570640 -1 Falling back to public interface 2023-10-09T11:00:05.268 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:00:05.461 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:05.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:05.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:05.546 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:00:05.546 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:00:05.547 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T11:00:05.547 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T11:00:05.547 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:00:05.566 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:05.617 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:05.701 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:00:05.704 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:05.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:05.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:05.810 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:00:05.852 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:05.888 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:00:05.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:05.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:00:05.993 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:06.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:05 smithi125 ceph-mon[195507]: 9.7 scrub ok 2023-10-09T11:00:06.050 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:05 smithi125 ceph-mon[200197]: 9.7 scrub ok 2023-10-09T11:00:06.055 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:06.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:06.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:06.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:06.195 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:06.200 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T11:00:06.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:05 smithi155 ceph-mon[184854]: 9.7 scrub ok 2023-10-09T11:00:06.251 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:06.286 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:06.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:06.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:06.391 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:06.441 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:06.551 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:06.611 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:06.621 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:06.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:06.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:06.650 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:06.680 INFO:teuthology.orchestra.run.smithi125.stderr:no valid command found; 10 closest matches: 2023-10-09T11:00:06.680 INFO:teuthology.orchestra.run.smithi125.stderr:0 2023-10-09T11:00:06.680 INFO:teuthology.orchestra.run.smithi125.stderr:1 2023-10-09T11:00:06.680 INFO:teuthology.orchestra.run.smithi125.stderr:2 2023-10-09T11:00:06.680 INFO:teuthology.orchestra.run.smithi125.stderr:abort 2023-10-09T11:00:06.680 INFO:teuthology.orchestra.run.smithi125.stderr:assert 2023-10-09T11:00:06.680 INFO:teuthology.orchestra.run.smithi125.stderr:config diff 2023-10-09T11:00:06.680 INFO:teuthology.orchestra.run.smithi125.stderr:config diff get 2023-10-09T11:00:06.680 INFO:teuthology.orchestra.run.smithi125.stderr:config get 2023-10-09T11:00:06.681 INFO:teuthology.orchestra.run.smithi125.stderr:config help [] 2023-10-09T11:00:06.681 INFO:teuthology.orchestra.run.smithi125.stderr:config set ... 2023-10-09T11:00:06.681 INFO:teuthology.orchestra.run.smithi125.stderr:admin_socket: invalid command 2023-10-09T11:00:06.729 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:00:06.777 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:06.811 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:00:06.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:06.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:00:06.916 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:00:06.972 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:07.003 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:00:07.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:07.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:00:07.006 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:06 smithi155 ceph-mon[184854]: pgmap v953: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 3.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 30855/163647 objects degraded (18.855%); 1/163647 objects misplaced (0.001%); 823 KiB/s, 12 objects/s recovering 2023-10-09T11:00:07.006 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:06 smithi155 ceph-mon[184854]: 9.11 scrub starts 2023-10-09T11:00:07.006 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:06 smithi155 ceph-mon[184854]: 9.11 scrub ok 2023-10-09T11:00:07.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:06 smithi125 ceph-mon[195507]: pgmap v953: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 3.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 30855/163647 objects degraded (18.855%); 1/163647 objects misplaced (0.001%); 823 KiB/s, 12 objects/s recovering 2023-10-09T11:00:07.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:06 smithi125 ceph-mon[195507]: 9.11 scrub starts 2023-10-09T11:00:07.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:06 smithi125 ceph-mon[195507]: 9.11 scrub ok 2023-10-09T11:00:07.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:06 smithi125 ceph-mon[200197]: pgmap v953: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 3.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 30855/163647 objects degraded (18.855%); 1/163647 objects misplaced (0.001%); 823 KiB/s, 12 objects/s recovering 2023-10-09T11:00:07.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:06 smithi125 ceph-mon[200197]: 9.11 scrub starts 2023-10-09T11:00:07.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:06 smithi125 ceph-mon[200197]: 9.11 scrub ok 2023-10-09T11:00:07.106 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:07.161 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:07.203 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:07.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:07.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:07.312 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:07.360 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:07.389 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:07.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:07.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:07.497 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:07.554 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:07.769 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:07 smithi125 ceph-mon[200197]: 9.18 scrub starts 2023-10-09T11:00:07.770 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:07 smithi125 ceph-mon[200197]: 9.18 scrub ok 2023-10-09T11:00:07.770 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:07 smithi125 ceph-mon[200197]: 9.5 scrub starts 2023-10-09T11:00:07.770 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:07 smithi125 ceph-mon[200197]: 9.5 scrub ok 2023-10-09T11:00:08.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:07 smithi125 ceph-mon[195507]: 9.18 scrub starts 2023-10-09T11:00:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:07 smithi125 ceph-mon[195507]: 9.18 scrub ok 2023-10-09T11:00:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:07 smithi125 ceph-mon[195507]: 9.5 scrub starts 2023-10-09T11:00:08.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:07 smithi125 ceph-mon[195507]: 9.5 scrub ok 2023-10-09T11:00:08.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:07 smithi155 ceph-mon[184854]: 9.18 scrub starts 2023-10-09T11:00:08.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:07 smithi155 ceph-mon[184854]: 9.18 scrub ok 2023-10-09T11:00:08.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:07 smithi155 ceph-mon[184854]: 9.5 scrub starts 2023-10-09T11:00:08.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:07 smithi155 ceph-mon[184854]: 9.5 scrub ok 2023-10-09T11:00:08.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:08 smithi125 ceph-mon[195507]: pgmap v954: 129 pgs: 1 active+clean+scrubbing, 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 72 active+clean; 3.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 27021/146775 objects degraded (18.410%); 1/146775 objects misplaced (0.001%); 685 KiB/s, 10 objects/s recovering 2023-10-09T11:00:08.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:08 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 27021/146775 objects degraded (18.410%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:08.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:08 smithi125 ceph-mon[200197]: pgmap v954: 129 pgs: 1 active+clean+scrubbing, 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 72 active+clean; 3.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 27021/146775 objects degraded (18.410%); 1/146775 objects misplaced (0.001%); 685 KiB/s, 10 objects/s recovering 2023-10-09T11:00:08.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:08 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 27021/146775 objects degraded (18.410%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:09.052 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:09.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:09.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:09.159 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:09.194 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:09.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:08 smithi155 ceph-mon[184854]: pgmap v954: 129 pgs: 1 active+clean+scrubbing, 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 72 active+clean; 3.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 27021/146775 objects degraded (18.410%); 1/146775 objects misplaced (0.001%); 685 KiB/s, 10 objects/s recovering 2023-10-09T11:00:09.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:08 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 27021/146775 objects degraded (18.410%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:09.220 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T11:00:09.221 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-1, ['dump_ops_in_flight'] 2023-10-09T11:00:11.078 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:10 smithi125 ceph-mon[200197]: pgmap v955: 129 pgs: 1 active+clean+scrubbing, 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 72 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 251 op/s; 27021/143130 objects degraded (18.879%); 1/143130 objects misplaced (0.001%) 2023-10-09T11:00:11.078 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:10 smithi125 ceph-mon[200197]: 9.8 scrub starts 2023-10-09T11:00:11.079 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:10 smithi125 ceph-mon[200197]: 9.8 scrub ok 2023-10-09T11:00:11.079 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:10 smithi125 ceph-mon[195507]: pgmap v955: 129 pgs: 1 active+clean+scrubbing, 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 72 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 251 op/s; 27021/143130 objects degraded (18.879%); 1/143130 objects misplaced (0.001%) 2023-10-09T11:00:11.079 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:10 smithi125 ceph-mon[195507]: 9.8 scrub starts 2023-10-09T11:00:11.079 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:10 smithi125 ceph-mon[195507]: 9.8 scrub ok 2023-10-09T11:00:11.194 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:11.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:11.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:11.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:10 smithi155 ceph-mon[184854]: pgmap v955: 129 pgs: 1 active+clean+scrubbing, 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 72 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 251 op/s; 27021/143130 objects degraded (18.879%); 1/143130 objects misplaced (0.001%) 2023-10-09T11:00:11.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:10 smithi155 ceph-mon[184854]: 9.8 scrub starts 2023-10-09T11:00:11.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:10 smithi155 ceph-mon[184854]: 9.8 scrub ok 2023-10-09T11:00:11.309 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:00:11.414 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:11.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:00:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:00:11] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.43.0" 2023-10-09T11:00:11.605 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:00:11.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:11.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:00:11.732 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:11.780 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:12.070 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:00:12 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[391396]: 2023-10-09T11:00:12.010+0000 7f8d41570640 -1 osd.1 357 log_to_monitors true 2023-10-09T11:00:12.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:11 smithi125 ceph-mon[195507]: 9.19 scrub starts 2023-10-09T11:00:12.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:11 smithi125 ceph-mon[195507]: 9.19 scrub ok 2023-10-09T11:00:12.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:11 smithi125 ceph-mon[200197]: 9.19 scrub starts 2023-10-09T11:00:12.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:11 smithi125 ceph-mon[200197]: 9.19 scrub ok 2023-10-09T11:00:12.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:11 smithi155 ceph-mon[184854]: 9.19 scrub starts 2023-10-09T11:00:12.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:11 smithi155 ceph-mon[184854]: 9.19 scrub ok 2023-10-09T11:00:13.020 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:13.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:12 smithi125 ceph-mon[195507]: pgmap v956: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23022/126213 objects degraded (18.241%); 1/126213 objects misplaced (0.001%) 2023-10-09T11:00:13.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:12 smithi125 ceph-mon[195507]: from='osd.1 [v2:172.21.15.125:6810/2306109326,v1:172.21.15.125:6811/2306109326]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T11:00:13.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:12 smithi125 ceph-mon[195507]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T11:00:13.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:12 smithi125 ceph-mon[200197]: pgmap v956: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23022/126213 objects degraded (18.241%); 1/126213 objects misplaced (0.001%) 2023-10-09T11:00:13.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:12 smithi125 ceph-mon[200197]: from='osd.1 [v2:172.21.15.125:6810/2306109326,v1:172.21.15.125:6811/2306109326]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T11:00:13.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:12 smithi125 ceph-mon[200197]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T11:00:13.076 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:00:12 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[391396]: 2023-10-09T11:00:12.850+0000 7f8d309d8700 -1 osd.1 357 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T11:00:13.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:13.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:13.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:12 smithi155 ceph-mon[184854]: pgmap v956: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23022/126213 objects degraded (18.241%); 1/126213 objects misplaced (0.001%) 2023-10-09T11:00:13.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:12 smithi155 ceph-mon[184854]: from='osd.1 [v2:172.21.15.125:6810/2306109326,v1:172.21.15.125:6811/2306109326]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T11:00:13.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:12 smithi155 ceph-mon[184854]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T11:00:13.273 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:13.305 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:13 smithi155 ceph-mon[184854]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T11:00:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:13 smithi155 ceph-mon[184854]: osdmap e387: 8 total, 7 up, 8 in 2023-10-09T11:00:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:13 smithi155 ceph-mon[184854]: failed to encode map e387 with expected crc 2023-10-09T11:00:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:13 smithi155 ceph-mon[184854]: from='osd.1 [v2:172.21.15.125:6810/2306109326,v1:172.21.15.125:6811/2306109326]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T11:00:14.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:13 smithi155 ceph-mon[184854]: failed to encode map e387 with expected crc 2023-10-09T11:00:14.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:13 smithi155 ceph-mon[184854]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T11:00:14.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:13 smithi155 ceph-mon[184854]: failed to encode map e387 with expected crc 2023-10-09T11:00:14.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:13 smithi155 ceph-mon[184854]: failed to encode map e387 with expected crc 2023-10-09T11:00:14.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:13 smithi155 ceph-mon[184854]: 9.17 scrub starts 2023-10-09T11:00:14.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:13 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 23022/126213 objects degraded (18.241%), 16 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-10-09T11:00:14.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:13 smithi155 ceph-mon[184854]: 9.17 scrub ok 2023-10-09T11:00:14.221 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T11:00:14.248 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[195507]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T11:00:14.248 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[195507]: osdmap e387: 8 total, 7 up, 8 in 2023-10-09T11:00:14.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[195507]: failed to encode map e387 with expected crc 2023-10-09T11:00:14.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[195507]: from='osd.1 [v2:172.21.15.125:6810/2306109326,v1:172.21.15.125:6811/2306109326]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T11:00:14.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[195507]: failed to encode map e387 with expected crc 2023-10-09T11:00:14.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[195507]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T11:00:14.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[195507]: failed to encode map e387 with expected crc 2023-10-09T11:00:14.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[195507]: failed to encode map e387 with expected crc 2023-10-09T11:00:14.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[195507]: 9.17 scrub starts 2023-10-09T11:00:14.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 23022/126213 objects degraded (18.241%), 16 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-10-09T11:00:14.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[195507]: 9.17 scrub ok 2023-10-09T11:00:14.250 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[200197]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T11:00:14.250 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[200197]: osdmap e387: 8 total, 7 up, 8 in 2023-10-09T11:00:14.250 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[200197]: failed to encode map e387 with expected crc 2023-10-09T11:00:14.250 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[200197]: from='osd.1 [v2:172.21.15.125:6810/2306109326,v1:172.21.15.125:6811/2306109326]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T11:00:14.250 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[200197]: failed to encode map e387 with expected crc 2023-10-09T11:00:14.250 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[200197]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T11:00:14.250 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[200197]: failed to encode map e387 with expected crc 2023-10-09T11:00:14.250 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[200197]: failed to encode map e387 with expected crc 2023-10-09T11:00:14.250 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[200197]: 9.17 scrub starts 2023-10-09T11:00:14.250 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 23022/126213 objects degraded (18.241%), 16 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-10-09T11:00:14.251 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:13 smithi125 ceph-mon[200197]: 9.17 scrub ok 2023-10-09T11:00:14.293 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:14.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:14.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:14.397 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:14.452 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:14.472 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:14.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:14.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:14.575 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:14.603 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:14.622 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:14.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:14.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:14.726 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:00:14.752 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:14.771 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:00:14.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:14.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:00:14.875 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:14.912 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:14.931 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:14.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:14.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:15.035 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:15.066 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:15.137 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:15.139 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:15.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:14 smithi125 ceph-mon[195507]: pgmap v958: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23022/122196 objects degraded (18.840%); 1/122196 objects misplaced (0.001%) 2023-10-09T11:00:15.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:14 smithi125 ceph-mon[195507]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T11:00:15.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:14 smithi125 ceph-mon[195507]: failed to encode map e388 with expected crc 2023-10-09T11:00:15.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:14 smithi125 ceph-mon[195507]: osd.1 [v2:172.21.15.125:6810/2306109326,v1:172.21.15.125:6811/2306109326] boot 2023-10-09T11:00:15.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:14 smithi125 ceph-mon[195507]: osdmap e388: 8 total, 8 up, 8 in 2023-10-09T11:00:15.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T11:00:15.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:14 smithi125 ceph-mon[195507]: failed to encode map e388 with expected crc 2023-10-09T11:00:15.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:14 smithi125 ceph-mon[200197]: pgmap v958: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23022/122196 objects degraded (18.840%); 1/122196 objects misplaced (0.001%) 2023-10-09T11:00:15.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:14 smithi125 ceph-mon[200197]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T11:00:15.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:14 smithi125 ceph-mon[200197]: failed to encode map e388 with expected crc 2023-10-09T11:00:15.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:14 smithi125 ceph-mon[200197]: osd.1 [v2:172.21.15.125:6810/2306109326,v1:172.21.15.125:6811/2306109326] boot 2023-10-09T11:00:15.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:14 smithi125 ceph-mon[200197]: osdmap e388: 8 total, 8 up, 8 in 2023-10-09T11:00:15.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T11:00:15.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:14 smithi125 ceph-mon[200197]: failed to encode map e388 with expected crc 2023-10-09T11:00:15.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:15.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:15.210 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:00:15.210 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T11:00:15.210 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T11:00:15.210 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:00:15.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.211 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPush(8.13 293/289 [PushOp(8:cbe2b721:::benchmark_data_smithi125_329074_object29:head, version: 270'1, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:cbe2b721:::benchmark_data_smithi125_329074_object29:head@270'1, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T11:00:15.211 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:45.877250+0000", 2023-10-09T11:00:15.211 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 329.33023626400001, 2023-10-09T11:00:15.211 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.363915395, 2023-10-09T11:00:15.211 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.211 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:00:15.211 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.211 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.211 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877250+0000", 2023-10-09T11:00:15.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.212 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.212 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.212 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.212 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877250+0000", 2023-10-09T11:00:15.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.212 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.212 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.212 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.212 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877249+0000", 2023-10-09T11:00:15.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:15.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877303+0000", 2023-10-09T11:00:15.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3928999999999998e-05 2023-10-09T11:00:15.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877323+0000", 2023-10-09T11:00:15.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0276999999999999e-05 2023-10-09T11:00:15.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.214 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.214 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877339+0000", 2023-10-09T11:00:15.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6362999999999999e-05 2023-10-09T11:00:15.214 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.214 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.214 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.190367+0000", 2023-10-09T11:00:15.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3130274530000001 2023-10-09T11:00:15.214 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.215 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.215 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.190793+0000", 2023-10-09T11:00:15.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042582999999999998 2023-10-09T11:00:15.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.215 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.215 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.241165+0000", 2023-10-09T11:00:15.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.050372739 2023-10-09T11:00:15.215 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.215 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.215 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.216 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95902 8.13 8:c9dc5fe7:::benchmark_data_smithi125_329074_object95901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T11:00:15.216 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.181255+0000", 2023-10-09T11:00:15.216 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 329.02623103600001, 2023-10-09T11:00:15.216 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.46745452100000001, 2023-10-09T11:00:15.216 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.216 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:15.216 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.216 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T11:00:15.216 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:00:15.216 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95902 2023-10-09T11:00:15.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.217 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181255+0000", 2023-10-09T11:00:15.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181255+0000", 2023-10-09T11:00:15.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.218 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181253+0000", 2023-10-09T11:00:15.218 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:00:15.218 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.218 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181286+0000", 2023-10-09T11:00:15.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2928000000000001e-05 2023-10-09T11:00:15.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.219 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181306+0000", 2023-10-09T11:00:15.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9514999999999999e-05 2023-10-09T11:00:15.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.219 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181317+0000", 2023-10-09T11:00:15.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1070000000000001e-05 2023-10-09T11:00:15.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.220 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.503475+0000", 2023-10-09T11:00:15.220 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32215859400000002 2023-10-09T11:00:15.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.220 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.513376+0000", 2023-10-09T11:00:15.220 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0099005069999999994 2023-10-09T11:00:15.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T11:00:15.220 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.528220+0000", 2023-10-09T11:00:15.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014844346 2023-10-09T11:00:15.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.221 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.543824+0000", 2023-10-09T11:00:15.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015603361 2023-10-09T11:00:15.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.221 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.547461+0000", 2023-10-09T11:00:15.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036375349999999999 2023-10-09T11:00:15.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.222 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:15.222 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.633195+0000", 2023-10-09T11:00:15.222 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.085734224999999997 2023-10-09T11:00:15.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.222 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:00:15.222 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.636727+0000", 2023-10-09T11:00:15.222 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035320870000000002 2023-10-09T11:00:15.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.222 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.222 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.648710+0000", 2023-10-09T11:00:15.223 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011982244 2023-10-09T11:00:15.223 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.223 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.223 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.223 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.223 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.223 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122010 9.6 9:6f14ab77:::benchmark_data_smithi125_363789_object11711:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:00:15.223 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.136716+0000", 2023-10-09T11:00:15.223 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 67.070770550999995, 2023-10-09T11:00:15.223 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0621660900000001, 2023-10-09T11:00:15.223 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.223 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:00:15.224 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.224 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.224 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.224 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122010 2023-10-09T11:00:15.224 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.224 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.224 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.224 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.224 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136716+0000", 2023-10-09T11:00:15.224 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.224 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.224 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136716+0000", 2023-10-09T11:00:15.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136715+0000", 2023-10-09T11:00:15.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:00:15.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136721+0000", 2023-10-09T11:00:15.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7660000000000004e-06 2023-10-09T11:00:15.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136731+0000", 2023-10-09T11:00:15.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0898e-05 2023-10-09T11:00:15.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136739+0000", 2023-10-09T11:00:15.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8450000000000003e-06 2023-10-09T11:00:15.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.140487+0000", 2023-10-09T11:00:15.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037476330000000002 2023-10-09T11:00:15.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.140607+0000", 2023-10-09T11:00:15.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012029299999999999 2023-10-09T11:00:15.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T11:00:15.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.140726+0000", 2023-10-09T11:00:15.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119057 2023-10-09T11:00:15.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.228 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:15.228 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.150456+0000", 2023-10-09T11:00:15.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.009729283 2023-10-09T11:00:15.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.228 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.228 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.153186+0000", 2023-10-09T11:00:15.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00273077 2023-10-09T11:00:15.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162778+0000", 2023-10-09T11:00:15.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0095913500000000002 2023-10-09T11:00:15.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:00:15.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162790+0000", 2023-10-09T11:00:15.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.221e-05 2023-10-09T11:00:15.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.198810+0000", 2023-10-09T11:00:15.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.036020156 2023-10-09T11:00:15.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.198882+0000", 2023-10-09T11:00:15.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1717999999999993e-05 2023-10-09T11:00:15.230 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.230 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.230 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.231 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122013 9.6 9:6b7eeaa4:::benchmark_data_smithi125_363789_object11714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:00:15.231 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.138283+0000", 2023-10-09T11:00:15.231 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 67.069203197999997, 2023-10-09T11:00:15.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.060936141, 2023-10-09T11:00:15.231 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.231 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:00:15.231 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.231 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.231 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.231 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122013 2023-10-09T11:00:15.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.232 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138283+0000", 2023-10-09T11:00:15.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138283+0000", 2023-10-09T11:00:15.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138282+0000", 2023-10-09T11:00:15.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:00:15.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138288+0000", 2023-10-09T11:00:15.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5029999999999997e-06 2023-10-09T11:00:15.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138300+0000", 2023-10-09T11:00:15.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2357e-05 2023-10-09T11:00:15.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138309+0000", 2023-10-09T11:00:15.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.499e-06 2023-10-09T11:00:15.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.144017+0000", 2023-10-09T11:00:15.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0057070339999999997 2023-10-09T11:00:15.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.144165+0000", 2023-10-09T11:00:15.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000148345 2023-10-09T11:00:15.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T11:00:15.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.144293+0000", 2023-10-09T11:00:15.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000128069 2023-10-09T11:00:15.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:15.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.152647+0000", 2023-10-09T11:00:15.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083540590000000005 2023-10-09T11:00:15.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.153226+0000", 2023-10-09T11:00:15.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00057917199999999998 2023-10-09T11:00:15.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162825+0000", 2023-10-09T11:00:15.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0095991010000000005 2023-10-09T11:00:15.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:00:15.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162833+0000", 2023-10-09T11:00:15.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3429999999999998e-06 2023-10-09T11:00:15.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199105+0000", 2023-10-09T11:00:15.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0362724169999999 2023-10-09T11:00:15.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.238 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199219+0000", 2023-10-09T11:00:15.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000114141 2023-10-09T11:00:15.238 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.238 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.238 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.239 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122028 9.6 9:6992c4ac:::benchmark_data_smithi125_363789_object11729:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:00:15.239 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.154935+0000", 2023-10-09T11:00:15.239 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 67.052551152999996, 2023-10-09T11:00:15.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.044381056, 2023-10-09T11:00:15.239 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.239 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:00:15.239 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.239 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.239 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.239 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122028 2023-10-09T11:00:15.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.240 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154935+0000", 2023-10-09T11:00:15.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154935+0000", 2023-10-09T11:00:15.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154934+0000", 2023-10-09T11:00:15.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:00:15.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154941+0000", 2023-10-09T11:00:15.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9600000000000003e-06 2023-10-09T11:00:15.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154956+0000", 2023-10-09T11:00:15.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4976e-05 2023-10-09T11:00:15.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154964+0000", 2023-10-09T11:00:15.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8639999999999996e-06 2023-10-09T11:00:15.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.155012+0000", 2023-10-09T11:00:15.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7960999999999997e-05 2023-10-09T11:00:15.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.155124+0000", 2023-10-09T11:00:15.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011204 2023-10-09T11:00:15.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T11:00:15.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.155237+0000", 2023-10-09T11:00:15.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011364799999999999 2023-10-09T11:00:15.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162936+0000", 2023-10-09T11:00:15.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.007698369 2023-10-09T11:00:15.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.244 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:00:15.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162945+0000", 2023-10-09T11:00:15.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6039999999999999e-06 2023-10-09T11:00:15.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.244 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.245 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199250+0000", 2023-10-09T11:00:15.245 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.036304248 2023-10-09T11:00:15.245 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.245 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.245 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.245 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199316+0000", 2023-10-09T11:00:15.245 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6549000000000003e-05 2023-10-09T11:00:15.245 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.245 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.245 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.245 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.246 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122037 9.6 9:68399f49:::benchmark_data_smithi125_363789_object11738:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:00:15.246 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.181454+0000", 2023-10-09T11:00:15.246 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 67.026031883000002, 2023-10-09T11:00:15.246 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.032324094, 2023-10-09T11:00:15.246 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.246 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:15.246 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.246 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.246 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.247 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122037 2023-10-09T11:00:15.247 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.247 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.247 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.247 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.247 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181454+0000", 2023-10-09T11:00:15.247 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.247 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.247 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.247 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.247 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181454+0000", 2023-10-09T11:00:15.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181453+0000", 2023-10-09T11:00:15.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:00:15.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181461+0000", 2023-10-09T11:00:15.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4800000000000001e-06 2023-10-09T11:00:15.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181478+0000", 2023-10-09T11:00:15.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7215e-05 2023-10-09T11:00:15.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181488+0000", 2023-10-09T11:00:15.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5240000000000006e-06 2023-10-09T11:00:15.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181572+0000", 2023-10-09T11:00:15.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3784999999999998e-05 2023-10-09T11:00:15.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:00:15.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181581+0000", 2023-10-09T11:00:15.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7249999999999992e-06 2023-10-09T11:00:15.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199341+0000", 2023-10-09T11:00:15.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.017760062 2023-10-09T11:00:15.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199463+0000", 2023-10-09T11:00:15.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012115799999999999 2023-10-09T11:00:15.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5", 2023-10-09T11:00:15.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199575+0000", 2023-10-09T11:00:15.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000112404 2023-10-09T11:00:15.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:15.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.207005+0000", 2023-10-09T11:00:15.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074305350000000003 2023-10-09T11:00:15.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.213644+0000", 2023-10-09T11:00:15.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066387779999999997 2023-10-09T11:00:15.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:00:15.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.213695+0000", 2023-10-09T11:00:15.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0748999999999998e-05 2023-10-09T11:00:15.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.253 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.253 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.253 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.213778+0000", 2023-10-09T11:00:15.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3434000000000001e-05 2023-10-09T11:00:15.253 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.253 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.253 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.253 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.253 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.253 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122039 9.6 9:679871c1:::benchmark_data_smithi125_363789_object11740:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:00:15.253 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:00:15.253 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 67.025970920000006, 2023-10-09T11:00:15.254 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0328282099999999, 2023-10-09T11:00:15.254 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.254 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:15.254 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.254 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.254 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.254 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122039 2023-10-09T11:00:15.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.254 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.254 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.254 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.254 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:00:15.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.255 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.255 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.255 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:00:15.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.255 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.255 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.255 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:00:15.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:00:15.255 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181518+0000", 2023-10-09T11:00:15.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4350000000000001e-06 2023-10-09T11:00:15.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181525+0000", 2023-10-09T11:00:15.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5359999999999998e-06 2023-10-09T11:00:15.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181530+0000", 2023-10-09T11:00:15.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9019999999999998e-06 2023-10-09T11:00:15.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181601+0000", 2023-10-09T11:00:15.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1168999999999996e-05 2023-10-09T11:00:15.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:00:15.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181606+0000", 2023-10-09T11:00:15.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1919999999999996e-06 2023-10-09T11:00:15.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.203647+0000", 2023-10-09T11:00:15.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.022041073 2023-10-09T11:00:15.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.203754+0000", 2023-10-09T11:00:15.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000106529 2023-10-09T11:00:15.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5", 2023-10-09T11:00:15.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.203828+0000", 2023-10-09T11:00:15.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4286e-05 2023-10-09T11:00:15.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:15.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.210714+0000", 2023-10-09T11:00:15.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0068858050000000001 2023-10-09T11:00:15.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.214203+0000", 2023-10-09T11:00:15.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003489242 2023-10-09T11:00:15.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:00:15.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.214268+0000", 2023-10-09T11:00:15.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5104000000000006e-05 2023-10-09T11:00:15.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.214344+0000", 2023-10-09T11:00:15.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5471999999999998e-05 2023-10-09T11:00:15.260 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.260 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.260 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.261 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:133037 9.b 9.da5dd31b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e362)", 2023-10-09T11:00:15.261 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:20.474469+0000", 2023-10-09T11:00:15.261 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 54.733017433000001, 2023-10-09T11:00:15.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97365184000000005, 2023-10-09T11:00:15.261 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.261 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:00:15.261 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.261 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.261 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.261 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133037 2023-10-09T11:00:15.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.261 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.262 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.262 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.262 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474469+0000", 2023-10-09T11:00:15.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.262 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.262 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.262 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.262 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474469+0000", 2023-10-09T11:00:15.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.263 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.263 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474467+0000", 2023-10-09T11:00:15.263 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:15.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.263 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.263 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474475+0000", 2023-10-09T11:00:15.263 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6009999999999997e-06 2023-10-09T11:00:15.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474493+0000", 2023-10-09T11:00:15.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8150000000000001e-05 2023-10-09T11:00:15.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474502+0000", 2023-10-09T11:00:15.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1400000000000006e-06 2023-10-09T11:00:15.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474569+0000", 2023-10-09T11:00:15.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6849999999999999e-05 2023-10-09T11:00:15.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:00:15.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474579+0000", 2023-10-09T11:00:15.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9369999999999999e-06 2023-10-09T11:00:15.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448111+0000", 2023-10-09T11:00:15.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97353240399999996 2023-10-09T11:00:15.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448121+0000", 2023-10-09T11:00:15.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1679999999999993e-06 2023-10-09T11:00:15.266 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.266 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.266 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.266 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:133044 9.b 9.5473ef4b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e362)", 2023-10-09T11:00:15.266 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:20.482185+0000", 2023-10-09T11:00:15.266 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 54.725301059000003, 2023-10-09T11:00:15.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96595614900000004, 2023-10-09T11:00:15.266 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.267 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:00:15.267 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.267 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.267 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.267 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133044 2023-10-09T11:00:15.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.267 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482185+0000", 2023-10-09T11:00:15.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482185+0000", 2023-10-09T11:00:15.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482184+0000", 2023-10-09T11:00:15.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:15.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482193+0000", 2023-10-09T11:00:15.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0310000000000004e-06 2023-10-09T11:00:15.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482217+0000", 2023-10-09T11:00:15.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4677000000000001e-05 2023-10-09T11:00:15.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482228+0000", 2023-10-09T11:00:15.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.043e-05 2023-10-09T11:00:15.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482326+0000", 2023-10-09T11:00:15.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8241999999999999e-05 2023-10-09T11:00:15.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:00:15.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482334+0000", 2023-10-09T11:00:15.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7910000000000005e-06 2023-10-09T11:00:15.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448135+0000", 2023-10-09T11:00:15.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96580099799999997 2023-10-09T11:00:15.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448141+0000", 2023-10-09T11:00:15.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5719999999999999e-06 2023-10-09T11:00:15.271 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.271 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.271 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.271 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:133073 9.b 9.b3395a9b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e362)", 2023-10-09T11:00:15.272 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:20.525035+0000", 2023-10-09T11:00:15.272 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 54.682451151999999, 2023-10-09T11:00:15.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.92312863000000001, 2023-10-09T11:00:15.272 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.272 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:00:15.272 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.272 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.272 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.272 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133073 2023-10-09T11:00:15.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.272 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525035+0000", 2023-10-09T11:00:15.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525035+0000", 2023-10-09T11:00:15.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525034+0000", 2023-10-09T11:00:15.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:00:15.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525039+0000", 2023-10-09T11:00:15.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9710000000000003e-06 2023-10-09T11:00:15.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525049+0000", 2023-10-09T11:00:15.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4450000000000002e-06 2023-10-09T11:00:15.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525054+0000", 2023-10-09T11:00:15.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.101e-06 2023-10-09T11:00:15.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525084+0000", 2023-10-09T11:00:15.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0049999999999999e-05 2023-10-09T11:00:15.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:00:15.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525089+0000", 2023-10-09T11:00:15.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1599999999999997e-06 2023-10-09T11:00:15.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448158+0000", 2023-10-09T11:00:15.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.92306902099999999 2023-10-09T11:00:15.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448164+0000", 2023-10-09T11:00:15.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7660000000000004e-06 2023-10-09T11:00:15.277 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.277 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.277 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.277 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157704 9.b 9:d7bfb393:::benchmark_data_smithi125_363789_object47405:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T11:00:15.277 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:53.921502+0000", 2023-10-09T11:00:15.277 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 21.285983911999999, 2023-10-09T11:00:15.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.46842198400000001, 2023-10-09T11:00:15.277 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.277 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:15.277 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.278 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.278 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.278 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157704 2023-10-09T11:00:15.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.278 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.921502+0000", 2023-10-09T11:00:15.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.921502+0000", 2023-10-09T11:00:15.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.921501+0000", 2023-10-09T11:00:15.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:15.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.921506+0000", 2023-10-09T11:00:15.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5609999999999998e-06 2023-10-09T11:00:15.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.921516+0000", 2023-10-09T11:00:15.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8779999999999993e-06 2023-10-09T11:00:15.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.921524+0000", 2023-10-09T11:00:15.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4229999999999999e-06 2023-10-09T11:00:15.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.117325+0000", 2023-10-09T11:00:15.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19580091799999999 2023-10-09T11:00:15.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.117403+0000", 2023-10-09T11:00:15.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8139999999999994e-05 2023-10-09T11:00:15.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:00:15.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.117503+0000", 2023-10-09T11:00:15.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010062500000000001 2023-10-09T11:00:15.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:15.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.389378+0000", 2023-10-09T11:00:15.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.271874587 2023-10-09T11:00:15.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.389418+0000", 2023-10-09T11:00:15.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0237000000000001e-05 2023-10-09T11:00:15.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.389765+0000", 2023-10-09T11:00:15.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000346375 2023-10-09T11:00:15.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.389788+0000", 2023-10-09T11:00:15.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2849000000000001e-05 2023-10-09T11:00:15.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:00:15.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.389822+0000", 2023-10-09T11:00:15.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4718999999999997e-05 2023-10-09T11:00:15.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.389924+0000", 2023-10-09T11:00:15.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000102058 2023-10-09T11:00:15.284 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.284 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.284 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.285 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157950 9.b 9:d2b9e9fa:::benchmark_data_smithi125_363789_object47651:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:00:15.285 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.472950+0000", 2023-10-09T11:00:15.285 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 20.734536101, 2023-10-09T11:00:15.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.54794583900000005, 2023-10-09T11:00:15.285 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.285 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:15.285 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.285 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.285 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.285 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157950 2023-10-09T11:00:15.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.286 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.472950+0000", 2023-10-09T11:00:15.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.472950+0000", 2023-10-09T11:00:15.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.472949+0000", 2023-10-09T11:00:15.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:15.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.472956+0000", 2023-10-09T11:00:15.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5429999999999996e-06 2023-10-09T11:00:15.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.472969+0000", 2023-10-09T11:00:15.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2547e-05 2023-10-09T11:00:15.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.288 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.472978+0000", 2023-10-09T11:00:15.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0909999999999999e-06 2023-10-09T11:00:15.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.288 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.672426+0000", 2023-10-09T11:00:15.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19944830099999999 2023-10-09T11:00:15.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.672563+0000", 2023-10-09T11:00:15.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013726 2023-10-09T11:00:15.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:00:15.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.672696+0000", 2023-10-09T11:00:15.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000132454 2023-10-09T11:00:15.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:15.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.680470+0000", 2023-10-09T11:00:15.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0077745660000000001 2023-10-09T11:00:15.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020539+0000", 2023-10-09T11:00:15.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.34006837899999998 2023-10-09T11:00:15.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020648+0000", 2023-10-09T11:00:15.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000109096 2023-10-09T11:00:15.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020701+0000", 2023-10-09T11:00:15.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2862000000000003e-05 2023-10-09T11:00:15.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:00:15.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020755+0000", 2023-10-09T11:00:15.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4200000000000003e-05 2023-10-09T11:00:15.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020896+0000", 2023-10-09T11:00:15.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000140934 2023-10-09T11:00:15.292 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.292 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.292 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.292 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157952 9.b 9:d7179396:::benchmark_data_smithi125_363789_object47653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:00:15.292 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.476086+0000", 2023-10-09T11:00:15.292 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 20.731399975999999, 2023-10-09T11:00:15.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.54503019500000005, 2023-10-09T11:00:15.292 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.293 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:15.293 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.293 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.293 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.293 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157952 2023-10-09T11:00:15.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.293 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.476086+0000", 2023-10-09T11:00:15.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.476086+0000", 2023-10-09T11:00:15.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.476085+0000", 2023-10-09T11:00:15.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:15.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.476092+0000", 2023-10-09T11:00:15.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4309999999999997e-06 2023-10-09T11:00:15.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.476106+0000", 2023-10-09T11:00:15.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3465999999999999e-05 2023-10-09T11:00:15.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.476113+0000", 2023-10-09T11:00:15.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4080000000000001e-06 2023-10-09T11:00:15.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.676444+0000", 2023-10-09T11:00:15.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20033067900000001 2023-10-09T11:00:15.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.676599+0000", 2023-10-09T11:00:15.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015494599999999999 2023-10-09T11:00:15.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:00:15.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.676743+0000", 2023-10-09T11:00:15.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014419199999999999 2023-10-09T11:00:15.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:15.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.687837+0000", 2023-10-09T11:00:15.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011093949000000001 2023-10-09T11:00:15.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020621+0000", 2023-10-09T11:00:15.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.332784258 2023-10-09T11:00:15.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020983+0000", 2023-10-09T11:00:15.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00036201099999999997 2023-10-09T11:00:15.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021014+0000", 2023-10-09T11:00:15.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0582e-05 2023-10-09T11:00:15.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:00:15.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021045+0000", 2023-10-09T11:00:15.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1804999999999998e-05 2023-10-09T11:00:15.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021116+0000", 2023-10-09T11:00:15.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0957999999999994e-05 2023-10-09T11:00:15.299 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.300 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.300 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.300 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157962 9.b 9:db800329:::benchmark_data_smithi125_363789_object47663:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:00:15.300 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.526479+0000", 2023-10-09T11:00:15.300 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 20.681007034, 2023-10-09T11:00:15.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.49495239499999999, 2023-10-09T11:00:15.300 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.300 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:15.300 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.300 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.300 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.301 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157962 2023-10-09T11:00:15.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.301 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.526479+0000", 2023-10-09T11:00:15.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.526479+0000", 2023-10-09T11:00:15.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.526477+0000", 2023-10-09T11:00:15.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:00:15.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.526486+0000", 2023-10-09T11:00:15.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9439999999999994e-06 2023-10-09T11:00:15.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.526506+0000", 2023-10-09T11:00:15.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9497999999999999e-05 2023-10-09T11:00:15.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.526517+0000", 2023-10-09T11:00:15.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1100999999999999e-05 2023-10-09T11:00:15.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.680558+0000", 2023-10-09T11:00:15.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.1540415 2023-10-09T11:00:15.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.680682+0000", 2023-10-09T11:00:15.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000123459 2023-10-09T11:00:15.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:00:15.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.680800+0000", 2023-10-09T11:00:15.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011798600000000001 2023-10-09T11:00:15.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:15.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020479+0000", 2023-10-09T11:00:15.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.339679283 2023-10-09T11:00:15.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020937+0000", 2023-10-09T11:00:15.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00045773999999999999 2023-10-09T11:00:15.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021212+0000", 2023-10-09T11:00:15.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027538200000000001 2023-10-09T11:00:15.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021260+0000", 2023-10-09T11:00:15.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7484999999999997e-05 2023-10-09T11:00:15.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:00:15.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021311+0000", 2023-10-09T11:00:15.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.109e-05 2023-10-09T11:00:15.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021432+0000", 2023-10-09T11:00:15.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120673 2023-10-09T11:00:15.308 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.309 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.309 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.309 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157985 9.b 9:d7678e95:::benchmark_data_smithi125_363789_object47686:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:00:15.309 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.693350+0000", 2023-10-09T11:00:15.309 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 20.514136220000001, 2023-10-09T11:00:15.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.49022632999999999, 2023-10-09T11:00:15.309 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.309 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:15.309 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.309 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.310 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.310 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157985 2023-10-09T11:00:15.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.310 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.693350+0000", 2023-10-09T11:00:15.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.693350+0000", 2023-10-09T11:00:15.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.693349+0000", 2023-10-09T11:00:15.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:15.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.693355+0000", 2023-10-09T11:00:15.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8109999999999999e-06 2023-10-09T11:00:15.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.693379+0000", 2023-10-09T11:00:15.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4541000000000001e-05 2023-10-09T11:00:15.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.693386+0000", 2023-10-09T11:00:15.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3900000000000004e-06 2023-10-09T11:00:15.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021982+0000", 2023-10-09T11:00:15.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32859580599999999 2023-10-09T11:00:15.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.022117+0000", 2023-10-09T11:00:15.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000134907 2023-10-09T11:00:15.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:00:15.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.022262+0000", 2023-10-09T11:00:15.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014447 2023-10-09T11:00:15.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:15.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.180849+0000", 2023-10-09T11:00:15.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.15858782900000001 2023-10-09T11:00:15.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183388+0000", 2023-10-09T11:00:15.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025387890000000001 2023-10-09T11:00:15.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183417+0000", 2023-10-09T11:00:15.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8996000000000002e-05 2023-10-09T11:00:15.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183444+0000", 2023-10-09T11:00:15.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6706000000000001e-05 2023-10-09T11:00:15.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:00:15.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183480+0000", 2023-10-09T11:00:15.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5753000000000001e-05 2023-10-09T11:00:15.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183576+0000", 2023-10-09T11:00:15.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6614999999999996e-05 2023-10-09T11:00:15.317 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.317 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.317 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.317 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157991 9.b 9:d8d7183f:::benchmark_data_smithi125_363789_object47692:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:00:15.317 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.702255+0000", 2023-10-09T11:00:15.317 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 20.505230739000002, 2023-10-09T11:00:15.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.48154451799999998, 2023-10-09T11:00:15.317 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.317 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:15.317 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.318 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.318 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.318 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157991 2023-10-09T11:00:15.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.318 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.702255+0000", 2023-10-09T11:00:15.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.702255+0000", 2023-10-09T11:00:15.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.702254+0000", 2023-10-09T11:00:15.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:00:15.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.702262+0000", 2023-10-09T11:00:15.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5890000000000003e-06 2023-10-09T11:00:15.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.702275+0000", 2023-10-09T11:00:15.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3117e-05 2023-10-09T11:00:15.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.702282+0000", 2023-10-09T11:00:15.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2239999999999998e-06 2023-10-09T11:00:15.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.024521+0000", 2023-10-09T11:00:15.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32223829100000001 2023-10-09T11:00:15.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.024658+0000", 2023-10-09T11:00:15.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013682200000000001 2023-10-09T11:00:15.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:00:15.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.024793+0000", 2023-10-09T11:00:15.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013507800000000001 2023-10-09T11:00:15.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:15.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183395+0000", 2023-10-09T11:00:15.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.15860237099999999 2023-10-09T11:00:15.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183624+0000", 2023-10-09T11:00:15.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00022910300000000001 2023-10-09T11:00:15.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183652+0000", 2023-10-09T11:00:15.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8217000000000001e-05 2023-10-09T11:00:15.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183694+0000", 2023-10-09T11:00:15.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1566999999999997e-05 2023-10-09T11:00:15.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:00:15.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183724+0000", 2023-10-09T11:00:15.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0412e-05 2023-10-09T11:00:15.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.324 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183800+0000", 2023-10-09T11:00:15.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5706999999999999e-05 2023-10-09T11:00:15.324 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.324 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.324 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.325 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:158019 9.b 9:df039cc9:::benchmark_data_smithi125_363789_object47720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:00:15.325 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.727586+0000", 2023-10-09T11:00:15.325 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 20.479899830000001, 2023-10-09T11:00:15.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.46238494899999999, 2023-10-09T11:00:15.325 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.325 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:15.325 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.325 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.325 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.325 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 158019 2023-10-09T11:00:15.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.326 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727586+0000", 2023-10-09T11:00:15.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727586+0000", 2023-10-09T11:00:15.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727585+0000", 2023-10-09T11:00:15.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:00:15.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727595+0000", 2023-10-09T11:00:15.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1929999999999999e-06 2023-10-09T11:00:15.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727606+0000", 2023-10-09T11:00:15.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1147e-05 2023-10-09T11:00:15.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727611+0000", 2023-10-09T11:00:15.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6520000000000003e-06 2023-10-09T11:00:15.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.180904+0000", 2023-10-09T11:00:15.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.45329216 2023-10-09T11:00:15.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.181027+0000", 2023-10-09T11:00:15.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000123736 2023-10-09T11:00:15.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:00:15.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.181123+0000", 2023-10-09T11:00:15.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5544999999999994e-05 2023-10-09T11:00:15.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:15.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183661+0000", 2023-10-09T11:00:15.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025378850000000001 2023-10-09T11:00:15.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.330 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.189737+0000", 2023-10-09T11:00:15.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0060765560000000003 2023-10-09T11:00:15.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.330 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.189830+0000", 2023-10-09T11:00:15.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2592999999999999e-05 2023-10-09T11:00:15.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.189861+0000", 2023-10-09T11:00:15.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1245999999999999e-05 2023-10-09T11:00:15.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.331 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:00:15.331 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.189897+0000", 2023-10-09T11:00:15.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5506000000000002e-05 2023-10-09T11:00:15.331 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.189971+0000", 2023-10-09T11:00:15.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4732000000000002e-05 2023-10-09T11:00:15.332 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.332 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.332 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.332 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:158020 9.b 9:d7bb8056:::benchmark_data_smithi125_363789_object47721:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:00:15.332 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.727616+0000", 2023-10-09T11:00:15.333 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 20.479870238, 2023-10-09T11:00:15.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.49230036799999999, 2023-10-09T11:00:15.333 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.333 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:15.333 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.333 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.333 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.333 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 158020 2023-10-09T11:00:15.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.333 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727616+0000", 2023-10-09T11:00:15.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727616+0000", 2023-10-09T11:00:15.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727616+0000", 2023-10-09T11:00:15.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:00:15.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.335 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727617+0000", 2023-10-09T11:00:15.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5889999999999999e-06 2023-10-09T11:00:15.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.335 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727622+0000", 2023-10-09T11:00:15.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.301e-06 2023-10-09T11:00:15.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727624+0000", 2023-10-09T11:00:15.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7810000000000001e-06 2023-10-09T11:00:15.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183941+0000", 2023-10-09T11:00:15.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.45631645599999998 2023-10-09T11:00:15.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.184064+0000", 2023-10-09T11:00:15.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012321399999999999 2023-10-09T11:00:15.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:00:15.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.184187+0000", 2023-10-09T11:00:15.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000122929 2023-10-09T11:00:15.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.189782+0000", 2023-10-09T11:00:15.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0055952470000000002 2023-10-09T11:00:15.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:15.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.189840+0000", 2023-10-09T11:00:15.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7769999999999997e-05 2023-10-09T11:00:15.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.196501+0000", 2023-10-09T11:00:15.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066608980000000002 2023-10-09T11:00:15.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.219724+0000", 2023-10-09T11:00:15.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.023222804 2023-10-09T11:00:15.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:00:15.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.219782+0000", 2023-10-09T11:00:15.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8514000000000003e-05 2023-10-09T11:00:15.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.219916+0000", 2023-10-09T11:00:15.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013412799999999999 2023-10-09T11:00:15.339 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.339 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.339 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.340 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:158030 9.b 9:dd6e0b4d:::benchmark_data_smithi125_363789_object47731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:00:15.340 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.738309+0000", 2023-10-09T11:00:15.340 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 20.469177035000001, 2023-10-09T11:00:15.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.48184437899999999, 2023-10-09T11:00:15.340 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.340 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:15.340 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.340 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.340 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.340 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 158030 2023-10-09T11:00:15.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.341 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.738309+0000", 2023-10-09T11:00:15.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.738309+0000", 2023-10-09T11:00:15.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.738307+0000", 2023-10-09T11:00:15.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:00:15.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.738317+0000", 2023-10-09T11:00:15.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3840000000000003e-06 2023-10-09T11:00:15.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.738334+0000", 2023-10-09T11:00:15.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7668000000000001e-05 2023-10-09T11:00:15.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.738345+0000", 2023-10-09T11:00:15.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0739e-05 2023-10-09T11:00:15.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.187098+0000", 2023-10-09T11:00:15.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.44875295999999998 2023-10-09T11:00:15.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.344 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.344 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.187203+0000", 2023-10-09T11:00:15.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000105021 2023-10-09T11:00:15.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.344 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:00:15.344 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.187295+0000", 2023-10-09T11:00:15.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1476000000000003e-05 2023-10-09T11:00:15.344 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.344 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.344 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:15.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.193554+0000", 2023-10-09T11:00:15.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062596179999999998 2023-10-09T11:00:15.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.196437+0000", 2023-10-09T11:00:15.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028830459999999998 2023-10-09T11:00:15.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.196573+0000", 2023-10-09T11:00:15.345 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000135556 2023-10-09T11:00:15.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.219968+0000", 2023-10-09T11:00:15.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.023394873 2023-10-09T11:00:15.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:00:15.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.220021+0000", 2023-10-09T11:00:15.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3173000000000001e-05 2023-10-09T11:00:15.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.220154+0000", 2023-10-09T11:00:15.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013273700000000001 2023-10-09T11:00:15.347 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.347 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.347 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.347 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:158054 9.b 9:d18fda64:::benchmark_data_smithi125_363789_object47755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:00:15.347 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.765080+0000", 2023-10-09T11:00:15.347 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 20.442406306999999, 2023-10-09T11:00:15.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.45531745400000001, 2023-10-09T11:00:15.348 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:00:15.348 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:15.348 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:00:15.348 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:00:15.348 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:15.348 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 158054 2023-10-09T11:00:15.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.348 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:00:15.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:00:15.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.765080+0000", 2023-10-09T11:00:15.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:00:15.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.765080+0000", 2023-10-09T11:00:15.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:00:15.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:00:15.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.765078+0000", 2023-10-09T11:00:15.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:00:15.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:00:15.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.765088+0000", 2023-10-09T11:00:15.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5389999999999996e-06 2023-10-09T11:00:15.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:00:15.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.765101+0000", 2023-10-09T11:00:15.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3061000000000001e-05 2023-10-09T11:00:15.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:00:15.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.765113+0000", 2023-10-09T11:00:15.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2103000000000001e-05 2023-10-09T11:00:15.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:00:15.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.190017+0000", 2023-10-09T11:00:15.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.42490388800000001 2023-10-09T11:00:15.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:00:15.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.190166+0000", 2023-10-09T11:00:15.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014956999999999999 2023-10-09T11:00:15.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.352 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:00:15.352 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.190308+0000", 2023-10-09T11:00:15.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000141698 2023-10-09T11:00:15.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.352 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:00:15.352 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.196514+0000", 2023-10-09T11:00:15.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062055110000000004 2023-10-09T11:00:15.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.353 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.353 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.202781+0000", 2023-10-09T11:00:15.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062676490000000001 2023-10-09T11:00:15.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.353 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.353 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.202829+0000", 2023-10-09T11:00:15.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7598000000000001e-05 2023-10-09T11:00:15.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:15.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.220230+0000", 2023-10-09T11:00:15.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.017401478000000001 2023-10-09T11:00:15.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:00:15.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.220286+0000", 2023-10-09T11:00:15.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.592e-05 2023-10-09T11:00:15.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:00:15.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:00:15.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:00:15.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.220397+0000", 2023-10-09T11:00:15.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111214 2023-10-09T11:00:15.355 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.355 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.355 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.355 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:00:15.355 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:00:15.355 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:00:15.356 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:00:15.356 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:00:15.356 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:00:15.356 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:00:15.356 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:00:15.356 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:00:15.357 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:00:15.357 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:00:15.357 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:00:15.357 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:00:15.357 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:00:15.357 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:00:15.357 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:00:15.357 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:00:15.357 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:00:15.358 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:00:15.358 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:00:15.358 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:14 smithi155 ceph-mon[184854]: pgmap v958: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23022/122196 objects degraded (18.840%); 1/122196 objects misplaced (0.001%) 2023-10-09T11:00:15.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:14 smithi155 ceph-mon[184854]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T11:00:15.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:14 smithi155 ceph-mon[184854]: failed to encode map e388 with expected crc 2023-10-09T11:00:15.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:14 smithi155 ceph-mon[184854]: osd.1 [v2:172.21.15.125:6810/2306109326,v1:172.21.15.125:6811/2306109326] boot 2023-10-09T11:00:15.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:14 smithi155 ceph-mon[184854]: osdmap e388: 8 total, 8 up, 8 in 2023-10-09T11:00:15.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T11:00:15.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:14 smithi155 ceph-mon[184854]: failed to encode map e388 with expected crc 2023-10-09T11:00:15.386 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:15.405 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:00:15.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:15.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:00:15.509 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:15.540 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:15.690 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T11:00:15.810 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:15.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:15.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:15.867 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:00:15.915 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:15.941 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:15.961 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:15.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:15.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:16.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[195507]: failed to encode map e389 with expected crc 2023-10-09T11:00:16.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[195507]: failed to encode map e389 with expected crc 2023-10-09T11:00:16.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[195507]: osdmap e389: 8 total, 8 up, 8 in 2023-10-09T11:00:16.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[195507]: failed to encode map e389 with expected crc 2023-10-09T11:00:16.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[195507]: failed to encode map e389 with expected crc 2023-10-09T11:00:16.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:16.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:00:16.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4242631549' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:16.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/153964461' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:16.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:16.004 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[200197]: failed to encode map e389 with expected crc 2023-10-09T11:00:16.004 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[200197]: failed to encode map e389 with expected crc 2023-10-09T11:00:16.004 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[200197]: osdmap e389: 8 total, 8 up, 8 in 2023-10-09T11:00:16.004 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[200197]: failed to encode map e389 with expected crc 2023-10-09T11:00:16.004 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[200197]: failed to encode map e389 with expected crc 2023-10-09T11:00:16.004 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:16.004 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:00:16.004 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4242631549' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:16.004 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/153964461' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:16.004 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:15 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:16.064 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:16.090 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:16.109 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:16.111 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:15 smithi155 ceph-mon[184854]: failed to encode map e389 with expected crc 2023-10-09T11:00:16.112 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:15 smithi155 ceph-mon[184854]: failed to encode map e389 with expected crc 2023-10-09T11:00:16.112 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:15 smithi155 ceph-mon[184854]: osdmap e389: 8 total, 8 up, 8 in 2023-10-09T11:00:16.112 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:15 smithi155 ceph-mon[184854]: failed to encode map e389 with expected crc 2023-10-09T11:00:16.112 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:15 smithi155 ceph-mon[184854]: failed to encode map e389 with expected crc 2023-10-09T11:00:16.112 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:16.112 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:00:16.112 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:15 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4242631549' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:16.112 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:15 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/153964461' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:16.112 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:15 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:16.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:16.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:16.214 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:16.341 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:17.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:17.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:17.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:17.142 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:17.177 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:16 smithi155 ceph-mon[184854]: pgmap v961: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23022/122196 objects degraded (18.840%); 1/122196 objects misplaced (0.001%) 2023-10-09T11:00:17.177 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:16 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:00:17.177 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:16 smithi155 ceph-mon[184854]: failed to encode map e390 with expected crc 2023-10-09T11:00:17.177 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:16 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:00:17.177 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:16 smithi155 ceph-mon[184854]: osdmap e390: 8 total, 8 up, 8 in 2023-10-09T11:00:17.178 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:16 smithi155 ceph-mon[184854]: failed to encode map e390 with expected crc 2023-10-09T11:00:17.178 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:16 smithi155 ceph-mon[184854]: failed to encode map e390 with expected crc 2023-10-09T11:00:17.178 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:16 smithi155 ceph-mon[184854]: failed to encode map e390 with expected crc 2023-10-09T11:00:17.178 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:16 smithi155 ceph-mon[184854]: failed to encode map e390 with expected crc 2023-10-09T11:00:17.179 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:17.214 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:17.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:17.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:17.318 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:00:17.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[195507]: pgmap v961: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23022/122196 objects degraded (18.840%); 1/122196 objects misplaced (0.001%) 2023-10-09T11:00:17.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:00:17.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[195507]: failed to encode map e390 with expected crc 2023-10-09T11:00:17.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:00:17.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[195507]: osdmap e390: 8 total, 8 up, 8 in 2023-10-09T11:00:17.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[195507]: failed to encode map e390 with expected crc 2023-10-09T11:00:17.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[195507]: failed to encode map e390 with expected crc 2023-10-09T11:00:17.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[195507]: failed to encode map e390 with expected crc 2023-10-09T11:00:17.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[195507]: failed to encode map e390 with expected crc 2023-10-09T11:00:17.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[200197]: pgmap v961: 129 pgs: 9 active+clean+remapped, 16 active+undersized+degraded, 31 active+undersized, 73 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23022/122196 objects degraded (18.840%); 1/122196 objects misplaced (0.001%) 2023-10-09T11:00:17.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:00:17.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[200197]: failed to encode map e390 with expected crc 2023-10-09T11:00:17.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:00:17.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[200197]: osdmap e390: 8 total, 8 up, 8 in 2023-10-09T11:00:17.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[200197]: failed to encode map e390 with expected crc 2023-10-09T11:00:17.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[200197]: failed to encode map e390 with expected crc 2023-10-09T11:00:17.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[200197]: failed to encode map e390 with expected crc 2023-10-09T11:00:17.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:16 smithi125 ceph-mon[200197]: failed to encode map e390 with expected crc 2023-10-09T11:00:17.362 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:17.393 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:00:17.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:17.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:00:17.498 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:17.543 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:17.580 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:17.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:17.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:17.685 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:17.718 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:17.744 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:17.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:17.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:17.848 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:17.914 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:17 smithi125 ceph-mon[195507]: 9.15 scrub starts 2023-10-09T11:00:17.914 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:17 smithi125 ceph-mon[200197]: 9.15 scrub starts 2023-10-09T11:00:17.919 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:18.154 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:18.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:18.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:18.207 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:18.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:17 smithi155 ceph-mon[184854]: 9.15 scrub starts 2023-10-09T11:00:18.261 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:18.301 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:18.327 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:18.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:18.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:18.436 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:00:18.481 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:00:18.482 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:18.514 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:18.550 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:18.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:18.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:18.655 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:18.714 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:18.906 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:18.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:18.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:18.969 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:18 smithi125 ceph-mon[195507]: pgmap v963: 129 pgs: 4 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 7 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 16 active+undersized, 91 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 20997/112326 objects degraded (18.693%); 1/112326 objects misplaced (0.001%); 0 B/s, 6 objects/s recovering 2023-10-09T11:00:18.970 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:18 smithi125 ceph-mon[195507]: 1.17 scrub starts 2023-10-09T11:00:18.970 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:18 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 20997/112326 objects degraded (18.693%), 8 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T11:00:18.970 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:18 smithi125 ceph-mon[200197]: pgmap v963: 129 pgs: 4 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 7 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 16 active+undersized, 91 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 20997/112326 objects degraded (18.693%); 1/112326 objects misplaced (0.001%); 0 B/s, 6 objects/s recovering 2023-10-09T11:00:18.970 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:18 smithi125 ceph-mon[200197]: 1.17 scrub starts 2023-10-09T11:00:18.970 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:18 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 20997/112326 objects degraded (18.693%), 8 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T11:00:19.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:19.048 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:19.066 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:00:19.073 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:19.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:19.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:19.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:19.209 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:18 smithi155 ceph-mon[184854]: pgmap v963: 129 pgs: 4 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 7 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 16 active+undersized, 91 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 20997/112326 objects degraded (18.693%); 1/112326 objects misplaced (0.001%); 0 B/s, 6 objects/s recovering 2023-10-09T11:00:19.210 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:18 smithi155 ceph-mon[184854]: 1.17 scrub starts 2023-10-09T11:00:19.210 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:18 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 20997/112326 objects degraded (18.693%), 8 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T11:00:19.211 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:19.239 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:19.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:19.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:19.344 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:19.420 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:19.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:19.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:19.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:19.717 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:19.724 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:00:19.725 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:00:19.726 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:19.824 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:19.880 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:19.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:19.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:19.983 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:20.051 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:19 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:00:20.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:19 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:00:20.058 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:00:20.059 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:20.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:19 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:00:20.266 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:20.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:20.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:20.371 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:20.448 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:20.645 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T11:00:20.751 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:20.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:20.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:20.857 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:20.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:20 smithi155 ceph-mon[184854]: pgmap v964: 129 pgs: 4 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 114 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 20997/110601 objects degraded (18.984%); 0 B/s, 4 objects/s recovering 2023-10-09T11:00:20.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:20 smithi125 ceph-mon[200197]: pgmap v964: 129 pgs: 4 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 114 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 20997/110601 objects degraded (18.984%); 0 B/s, 4 objects/s recovering 2023-10-09T11:00:20.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:20 smithi125 ceph-mon[195507]: pgmap v964: 129 pgs: 4 activating+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 114 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 20997/110601 objects degraded (18.984%); 0 B/s, 4 objects/s recovering 2023-10-09T11:00:21.008 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:21.098 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:21.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:21.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:21.202 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:00:21.258 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:00:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:00:21] "GET /metrics HTTP/1.1" 200 36329 "" "Prometheus/2.43.0" 2023-10-09T11:00:21.273 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:21.274 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:21.363 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:00:21.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:21.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:00:21.441 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:00:21.475 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:00:21.487 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:00:21.549 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:21.549 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:00:21.614 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:00:21.645 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:00:21.694 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:00:21.730 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:00:21.759 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:00:21.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:21.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:00:21.775 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:00:21.795 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:00:21.795 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:00:21.795 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:00:21.795 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:00:21.795 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:00:21.796 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:00:21.796 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:00:21.796 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:00:21.867 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:00:21.880 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:21.925 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:21.955 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:00:21.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:21.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:00:22.066 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:00:22.116 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:21 smithi125 ceph-mon[200197]: 1.6 scrub starts 2023-10-09T11:00:22.116 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:21 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1474174174' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:22.116 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:21 smithi125 ceph-mon[195507]: 1.6 scrub starts 2023-10-09T11:00:22.116 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:21 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1474174174' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:22.119 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:22.148 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:00:22.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:22.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:00:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:21 smithi155 ceph-mon[184854]: 1.6 scrub starts 2023-10-09T11:00:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:21 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1474174174' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:22.257 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:22.346 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:22.445 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:22.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:22.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:22.551 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:22.625 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:22.713 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:22.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:22.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:22.824 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:22.895 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:23.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:22 smithi125 ceph-mon[195507]: pgmap v965: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 162 op/s; 32538/101196 objects degraded (32.153%); 0 B/s, 43 objects/s recovering 2023-10-09T11:00:23.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:22 smithi125 ceph-mon[200197]: pgmap v965: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 162 op/s; 32538/101196 objects degraded (32.153%); 0 B/s, 43 objects/s recovering 2023-10-09T11:00:23.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:22 smithi155 ceph-mon[184854]: pgmap v965: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 162 op/s; 32538/101196 objects degraded (32.153%); 0 B/s, 43 objects/s recovering 2023-10-09T11:00:23.276 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:23.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:23.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:23.382 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:23.468 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:23.880 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T11:00:23.880 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:00:23.881 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T11:00:23.881 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T11:00:23.881 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T11:00:23.881 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T11:00:23.930 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:23 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 32538/101196 objects degraded (32.153%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:23.930 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:23.930 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:23.930 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:23 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1811877718' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:00:23.930 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:23 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:00:23.930 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:23 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 32538/101196 objects degraded (32.153%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:23.931 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:23.931 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:23.931 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:23 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1811877718' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:00:23.931 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:23 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:00:24.122 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:24.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:24.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:24.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:23 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 32538/101196 objects degraded (32.153%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:24.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:24.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:24.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:23 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1811877718' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:00:24.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:23 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:00:24.229 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:00:24.302 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:24.335 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:00:24.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:24.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:00:24.374 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:24.376 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:00:24.376 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:00:24.376 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T11:00:24.376 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T11:00:24.376 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:00:24.448 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:24.508 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:24.570 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:00:24.638 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:00:25.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:24 smithi155 ceph-mon[184854]: pgmap v966: 129 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 32538/99171 objects degraded (32.810%); 0 B/s, 38 objects/s recovering 2023-10-09T11:00:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:24 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:00:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:24 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:00:25.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:24 smithi155 ceph-mon[184854]: osdmap e391: 8 total, 8 up, 8 in 2023-10-09T11:00:25.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:24 smithi125 ceph-mon[195507]: pgmap v966: 129 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 32538/99171 objects degraded (32.810%); 0 B/s, 38 objects/s recovering 2023-10-09T11:00:25.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:24 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:00:25.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:24 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:00:25.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:24 smithi125 ceph-mon[195507]: osdmap e391: 8 total, 8 up, 8 in 2023-10-09T11:00:25.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:24 smithi125 ceph-mon[200197]: pgmap v966: 129 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 32538/99171 objects degraded (32.810%); 0 B/s, 38 objects/s recovering 2023-10-09T11:00:25.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:24 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:00:25.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:24 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:00:25.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:24 smithi125 ceph-mon[200197]: osdmap e391: 8 total, 8 up, 8 in 2023-10-09T11:00:26.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:25 smithi155 ceph-mon[184854]: failed to encode map e391 with expected crc 2023-10-09T11:00:26.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:25 smithi125 ceph-mon[195507]: failed to encode map e391 with expected crc 2023-10-09T11:00:26.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:25 smithi125 ceph-mon[200197]: failed to encode map e391 with expected crc 2023-10-09T11:00:26.950 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:27.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:27.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:27.107 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:27.179 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:27.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:26 smithi155 ceph-mon[184854]: pgmap v968: 129 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 30 op/s; 32538/99171 objects degraded (32.810%); 0 B/s, 35 objects/s recovering 2023-10-09T11:00:27.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:26 smithi125 ceph-mon[195507]: pgmap v968: 129 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 30 op/s; 32538/99171 objects degraded (32.810%); 0 B/s, 35 objects/s recovering 2023-10-09T11:00:27.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:26 smithi125 ceph-mon[200197]: pgmap v968: 129 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 30 op/s; 32538/99171 objects degraded (32.810%); 0 B/s, 35 objects/s recovering 2023-10-09T11:00:29.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:28 smithi125 ceph-mon[200197]: pgmap v969: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 13 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 32538/89823 objects degraded (36.225%); 0 B/s, 63 objects/s recovering 2023-10-09T11:00:29.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:28 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 32538/89823 objects degraded (36.225%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:29.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:28 smithi125 ceph-mon[195507]: pgmap v969: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 13 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 32538/89823 objects degraded (36.225%); 0 B/s, 63 objects/s recovering 2023-10-09T11:00:29.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:28 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 32538/89823 objects degraded (36.225%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:29.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:28 smithi155 ceph-mon[184854]: pgmap v969: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 13 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 32538/89823 objects degraded (36.225%); 0 B/s, 63 objects/s recovering 2023-10-09T11:00:29.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:28 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 32538/89823 objects degraded (36.225%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:29.652 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:29.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:29.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:29.912 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:29.953 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:29.980 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:29.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:29.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:30.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:29 smithi125 ceph-mon[195507]: pgmap v970: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 32538/87345 objects degraded (37.252%); 0 B/s, 63 objects/s recovering 2023-10-09T11:00:30.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:29 smithi125 ceph-mon[200197]: pgmap v970: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 32538/87345 objects degraded (37.252%); 0 B/s, 63 objects/s recovering 2023-10-09T11:00:30.085 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:30.133 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:30.176 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:30.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:30.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:30.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:29 smithi155 ceph-mon[184854]: pgmap v970: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 32538/87345 objects degraded (37.252%); 0 B/s, 63 objects/s recovering 2023-10-09T11:00:30.283 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:30.329 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:30.364 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:30.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:30.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:30.471 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:30.511 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:30.546 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:30.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:30.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:30.652 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:30.701 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:30.733 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:30.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:30.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:30.840 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:00:30.888 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:30.922 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:00:30.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:30.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:00:30.982 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T11:00:31.031 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:31.063 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:31.090 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:31.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:31.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:31.195 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:31.222 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:31 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:00:31.222 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:00:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:00:31] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-10-09T11:00:31.223 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:31 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:00:31.229 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:31.257 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:31.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:31.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:31.260 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:31 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:00:31.361 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:31.459 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:31.517 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:31.818 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:31.965 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:00:32.009 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:32.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:32.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:32.116 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:32.145 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:32.168 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:32.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:32.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:32.272 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:32.331 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:00:32.331 INFO:teuthology.orchestra.run.smithi125.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T11:00:32.331 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:00:32.373 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:32.509 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:32.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:32.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:32.622 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:32.688 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:33.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:32 smithi155 ceph-mon[184854]: pgmap v971: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 32538/76314 objects degraded (42.637%); 0 B/s, 72 objects/s recovering 2023-10-09T11:00:33.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:32 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:33.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:32 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:33.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:32 smithi125 ceph-mon[195507]: pgmap v971: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 32538/76314 objects degraded (42.637%); 0 B/s, 72 objects/s recovering 2023-10-09T11:00:33.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:32 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:33.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:32 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:33.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:32 smithi125 ceph-mon[200197]: pgmap v971: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 32538/76314 objects degraded (42.637%); 0 B/s, 72 objects/s recovering 2023-10-09T11:00:33.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:32 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:33.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:32 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:34.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:33 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 32538/76314 objects degraded (42.637%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:34.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:33 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 32538/76314 objects degraded (42.637%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:34.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:33 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 32538/76314 objects degraded (42.637%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:35.211 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:35.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:34 smithi155 ceph-mon[184854]: pgmap v972: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 32538/73842 objects degraded (44.064%); 0 B/s, 72 objects/s recovering 2023-10-09T11:00:35.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:35.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:35.246 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:34 smithi125 ceph-mon[200197]: pgmap v972: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 32538/73842 objects degraded (44.064%); 0 B/s, 72 objects/s recovering 2023-10-09T11:00:35.250 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:34 smithi125 ceph-mon[195507]: pgmap v972: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 32538/73842 objects degraded (44.064%); 0 B/s, 72 objects/s recovering 2023-10-09T11:00:35.342 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:35.431 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:35.490 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:35.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:35.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:35.682 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:35.747 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:36.045 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:36.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:36.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:36.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:36.201 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:00:36.202 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:00:36.202 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:35 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:36.207 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:36.229 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:36.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:36.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:36.267 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:00:36.268 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:00:36.268 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:35 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:36.268 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:00:36.268 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:00:36.268 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:35 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:00:36.338 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:36.424 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:36.475 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:00:36.506 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:36.508 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:00:36.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:36.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:36.547 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:36.567 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:00:36.601 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:00:36.614 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:36.634 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:00:36.651 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:36.680 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:36.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:36.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:36.692 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:00:36.730 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:00:36.782 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:00:36.785 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:36.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:00:36.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:00:36.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:00:36.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:00:36.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:00:36.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:00:36.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:00:36.791 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:00:36.900 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:37.019 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:00:37.020 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:00:37.193 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:36 smithi125 ceph-mon[195507]: pgmap v973: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 32538/73842 objects degraded (44.064%); 0 B/s, 65 objects/s recovering 2023-10-09T11:00:37.193 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:36 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1628273248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:37.197 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:36 smithi125 ceph-mon[200197]: pgmap v973: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 32538/73842 objects degraded (44.064%); 0 B/s, 65 objects/s recovering 2023-10-09T11:00:37.197 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:36 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1628273248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:37.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:36 smithi155 ceph-mon[184854]: pgmap v973: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 32538/73842 objects degraded (44.064%); 0 B/s, 65 objects/s recovering 2023-10-09T11:00:37.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:36 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1628273248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:37.607 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:37.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:37.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:37.711 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:00:37.792 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:37.849 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:00:37.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:37.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:00:37.860 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T11:00:37.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:38.018 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:38.077 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:38.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:38.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:38.182 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:38.253 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:38.282 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:38.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:38.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:38.327 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:00:38.396 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:38.437 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:38.469 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:38.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:38.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:38.578 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:38.623 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:38.675 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:38.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:38.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:38.782 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:38.881 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:38.926 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:38.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:38.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:38.949 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:38 smithi155 ceph-mon[184854]: pgmap v974: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 32538/63909 objects degraded (50.913%); 0 B/s, 93 objects/s recovering 2023-10-09T11:00:38.949 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:38 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 32538/63909 objects degraded (50.913%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:38.949 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:38 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/643912314' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:38.949 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:38 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:39.032 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:39.095 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:39.123 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:39.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:39.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:39.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:38 smithi125 ceph-mon[200197]: pgmap v974: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 32538/63909 objects degraded (50.913%); 0 B/s, 93 objects/s recovering 2023-10-09T11:00:39.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:38 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 32538/63909 objects degraded (50.913%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:39.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:38 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/643912314' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:39.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:38 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:39.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:38 smithi125 ceph-mon[195507]: pgmap v974: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 32538/63909 objects degraded (50.913%); 0 B/s, 93 objects/s recovering 2023-10-09T11:00:39.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:38 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 32538/63909 objects degraded (50.913%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:39.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:38 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/643912314' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:39.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:38 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:39.166 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:00:39.231 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:00:39.234 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:39.336 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:39.365 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:39.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:39.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:39.479 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:39.512 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:00:39.512 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:00:39.588 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:39.631 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:39.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:39.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:39.704 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:39.739 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:39.791 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:39.823 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:39.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:39.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:39.888 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T11:00:39.928 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:39.982 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:40.025 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:40.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:40.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:40.120 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:39 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:00:40.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:39 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:00:40.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:39 smithi125 ceph-mon[195507]: osdmap e392: 8 total, 8 up, 8 in 2023-10-09T11:00:40.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:39 smithi125 ceph-mon[195507]: failed to encode map e392 with expected crc 2023-10-09T11:00:40.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:39 smithi125 ceph-mon[195507]: failed to encode map e392 with expected crc 2023-10-09T11:00:40.121 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:39 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:00:40.121 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:39 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:00:40.121 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:39 smithi125 ceph-mon[200197]: osdmap e392: 8 total, 8 up, 8 in 2023-10-09T11:00:40.121 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:39 smithi125 ceph-mon[200197]: failed to encode map e392 with expected crc 2023-10-09T11:00:40.122 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:39 smithi125 ceph-mon[200197]: failed to encode map e392 with expected crc 2023-10-09T11:00:40.128 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:40.193 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:40.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:39 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:00:40.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:39 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:00:40.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:39 smithi155 ceph-mon[184854]: osdmap e392: 8 total, 8 up, 8 in 2023-10-09T11:00:40.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:39 smithi155 ceph-mon[184854]: failed to encode map e392 with expected crc 2023-10-09T11:00:40.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:39 smithi155 ceph-mon[184854]: failed to encode map e392 with expected crc 2023-10-09T11:00:40.279 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:40.280 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:00:40.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:40.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:40.384 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:00:40.429 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:40.463 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:00:40.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:40.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:00:40.567 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:40.605 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:40.743 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:40.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:40.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:40.851 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:40.893 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:40.967 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:40.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:40.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:41.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:40 smithi125 ceph-mon[195507]: failed to encode map e392 with expected crc 2023-10-09T11:00:41.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:40 smithi125 ceph-mon[195507]: pgmap v976: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 32538/61494 objects degraded (52.912%); 0 B/s, 80 objects/s recovering 2023-10-09T11:00:41.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:40 smithi125 ceph-mon[200197]: failed to encode map e392 with expected crc 2023-10-09T11:00:41.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:40 smithi125 ceph-mon[200197]: pgmap v976: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 32538/61494 objects degraded (52.912%); 0 B/s, 80 objects/s recovering 2023-10-09T11:00:41.075 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:41.112 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:00:41.114 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:40 smithi155 ceph-mon[184854]: failed to encode map e392 with expected crc 2023-10-09T11:00:41.114 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:40 smithi155 ceph-mon[184854]: pgmap v976: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 32538/61494 objects degraded (52.912%); 0 B/s, 80 objects/s recovering 2023-10-09T11:00:41.117 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:41.152 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:00:41.185 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:00:41.197 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:41.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:41.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:41.221 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:00:41.252 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:00:41.287 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:00:41.307 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:41.326 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:00:41.359 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:00:41.385 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:41.388 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:00:41.388 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:00:41.389 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:00:41.389 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:00:41.389 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:00:41.389 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:00:41.389 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:00:41.389 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:00:41.419 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T11:00:41.419 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:00:41.420 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T11:00:41.420 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.281555 2023-10-09T11:00:41.420 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 0.2815554078413448 2023-10-09T11:00:41.423 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:00:41.423 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:00:41.423 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T11:00:41.423 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T11:00:41.423 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:00:41.433 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:41.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:41.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:41.487 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:00:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:00:41] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-10-09T11:00:41.544 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:00:41.564 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:00:41.587 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:41.717 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:00:41.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:41.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:00:41.787 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T11:00:41.822 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:41.864 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:41.895 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:41.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:41.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:41.933 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:42.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:42.052 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:42.079 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:42.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:42.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:42.167 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:00:42.183 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:42.228 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:42.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:41 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/857805387' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:42.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:41 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/857805387' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:42.344 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:41 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/857805387' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:42.357 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:42.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:42.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:42.467 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:42.501 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:42.627 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:42.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:42.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:42.731 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:00:42.763 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:43.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:42 smithi125 ceph-mon[195507]: pgmap v977: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 32538/51777 objects degraded (62.843%); 0 B/s, 83 objects/s recovering 2023-10-09T11:00:43.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:42 smithi125 ceph-mon[195507]: failed to encode map e393 with expected crc 2023-10-09T11:00:43.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:42 smithi125 ceph-mon[195507]: osdmap e393: 8 total, 8 up, 8 in 2023-10-09T11:00:43.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:42 smithi125 ceph-mon[195507]: failed to encode map e393 with expected crc 2023-10-09T11:00:43.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:42 smithi125 ceph-mon[200197]: pgmap v977: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 32538/51777 objects degraded (62.843%); 0 B/s, 83 objects/s recovering 2023-10-09T11:00:43.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:42 smithi125 ceph-mon[200197]: failed to encode map e393 with expected crc 2023-10-09T11:00:43.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:42 smithi125 ceph-mon[200197]: osdmap e393: 8 total, 8 up, 8 in 2023-10-09T11:00:43.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:42 smithi125 ceph-mon[200197]: failed to encode map e393 with expected crc 2023-10-09T11:00:43.232 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:00:43.232 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T11:00:43.232 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T11:00:43.232 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:00:43.232 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.233 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:152608 7.b 7:d0fd2adb:::benchmark_data_smithi125_295816_object61471:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e245)", 2023-10-09T11:00:43.233 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:52:10.363211+0000", 2023-10-09T11:00:43.233 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 512.86572210500003, 2023-10-09T11:00:43.233 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8185187439999999, 2023-10-09T11:00:43.233 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.233 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.233 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:43.233 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T11:00:43.233 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T11:00:43.233 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 152608 2023-10-09T11:00:43.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.233 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.234 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.234 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.234 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363211+0000", 2023-10-09T11:00:43.234 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.234 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.234 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.234 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.234 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363211+0000", 2023-10-09T11:00:43.234 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.234 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.234 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.235 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.235 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363209+0000", 2023-10-09T11:00:43.235 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T11:00:43.235 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.235 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.235 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.235 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363218+0000", 2023-10-09T11:00:43.235 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8470000000000002e-06 2023-10-09T11:00:43.235 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.235 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.235 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.235 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363236+0000", 2023-10-09T11:00:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8062000000000001e-05 2023-10-09T11:00:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.363244+0000", 2023-10-09T11:00:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2560000000000002e-06 2023-10-09T11:00:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.365179+0000", 2023-10-09T11:00:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001934755 2023-10-09T11:00:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op must wait for map", 2023-10-09T11:00:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.365188+0000", 2023-10-09T11:00:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.132e-06 2023-10-09T11:00:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.380926+0000", 2023-10-09T11:00:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015737860999999999 2023-10-09T11:00:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:00:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:10.380936+0000", 2023-10-09T11:00:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.679e-06 2023-10-09T11:00:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.172400+0000", 2023-10-09T11:00:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7914640509999999 2023-10-09T11:00:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.172506+0000", 2023-10-09T11:00:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106677 2023-10-09T11:00:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T11:00:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.172597+0000", 2023-10-09T11:00:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0718999999999997e-05 2023-10-09T11:00:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.176419+0000", 2023-10-09T11:00:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0038220480000000002 2023-10-09T11:00:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.176443+0000", 2023-10-09T11:00:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3583000000000001e-05 2023-10-09T11:00:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.176459+0000", 2023-10-09T11:00:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5815999999999999e-05 2023-10-09T11:00:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.181617+0000", 2023-10-09T11:00:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051585210000000001 2023-10-09T11:00:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.181654+0000", 2023-10-09T11:00:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6573999999999999e-05 2023-10-09T11:00:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:52:12.181730+0000", 2023-10-09T11:00:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6070999999999997e-05 2023-10-09T11:00:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:39213 8.2 8:4e7be898:::benchmark_data_smithi125_329074_object39212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T11:00:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:53:35.895072+0000", 2023-10-09T11:00:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 427.33386150500002, 2023-10-09T11:00:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.128363015, 2023-10-09T11:00:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:00:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:00:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39213 2023-10-09T11:00:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895072+0000", 2023-10-09T11:00:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895072+0000", 2023-10-09T11:00:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895070+0000", 2023-10-09T11:00:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895204+0000", 2023-10-09T11:00:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000133561 2023-10-09T11:00:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.245 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.245 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.245 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895215+0000", 2023-10-09T11:00:43.245 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0925e-05 2023-10-09T11:00:43.245 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.245 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.245 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.245 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895220+0000", 2023-10-09T11:00:43.245 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4949999999999999e-06 2023-10-09T11:00:43.245 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.245 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.245 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.245 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895798+0000", 2023-10-09T11:00:43.246 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00057733700000000003 2023-10-09T11:00:43.246 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.246 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.246 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.246 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895926+0000", 2023-10-09T11:00:43.246 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012778600000000001 2023-10-09T11:00:43.246 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.246 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.246 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:00:43.246 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.896072+0000", 2023-10-09T11:00:43.246 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014615599999999999 2023-10-09T11:00:43.247 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.247 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.247 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:43.247 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.897973+0000", 2023-10-09T11:00:43.247 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019008510000000001 2023-10-09T11:00:43.247 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.247 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.247 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.247 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.899072+0000", 2023-10-09T11:00:43.247 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010995239999999999 2023-10-09T11:00:43.247 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.247 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.248 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.248 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023353+0000", 2023-10-09T11:00:43.248 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.124281323 2023-10-09T11:00:43.248 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.248 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.248 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.248 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023405+0000", 2023-10-09T11:00:43.248 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1158e-05 2023-10-09T11:00:43.248 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.248 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.248 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.248 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023435+0000", 2023-10-09T11:00:43.249 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.006e-05 2023-10-09T11:00:43.249 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.249 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.249 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.249 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.249 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.249 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83954 8.4 8:2689e239:::benchmark_data_smithi125_329074_object83953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T11:00:43.249 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.108148+0000", 2023-10-09T11:00:43.249 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 386.12078535299997, 2023-10-09T11:00:43.249 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1275014759999999, 2023-10-09T11:00:43.249 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.249 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.250 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:43.250 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:00:43.250 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:00:43.250 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83954 2023-10-09T11:00:43.250 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.250 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.250 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.250 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.250 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108148+0000", 2023-10-09T11:00:43.250 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.250 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.250 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.251 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.251 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108148+0000", 2023-10-09T11:00:43.251 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.251 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.251 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.251 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.251 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108146+0000", 2023-10-09T11:00:43.251 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:43.251 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.251 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.251 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.251 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108191+0000", 2023-10-09T11:00:43.252 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4549000000000003e-05 2023-10-09T11:00:43.252 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.252 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.252 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.252 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108215+0000", 2023-10-09T11:00:43.252 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3994000000000001e-05 2023-10-09T11:00:43.252 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.252 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.252 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.252 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108222+0000", 2023-10-09T11:00:43.252 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4499999999999998e-06 2023-10-09T11:00:43.252 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.253 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.253 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.253 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108279+0000", 2023-10-09T11:00:43.253 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6643e-05 2023-10-09T11:00:43.253 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.253 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.253 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:00:43.253 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108288+0000", 2023-10-09T11:00:43.253 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5109999999999992e-06 2023-10-09T11:00:43.253 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.253 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.253 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.254 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.872892+0000", 2023-10-09T11:00:43.254 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.76460362800000004 2023-10-09T11:00:43.254 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.254 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.254 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:00:43.254 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.872903+0000", 2023-10-09T11:00:43.254 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1158000000000001e-05 2023-10-09T11:00:43.254 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.254 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.254 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.254 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213528+0000", 2023-10-09T11:00:43.254 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.340624501 2023-10-09T11:00:43.254 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.255 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.255 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.255 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213679+0000", 2023-10-09T11:00:43.255 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151578 2023-10-09T11:00:43.255 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.255 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.255 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T11:00:43.255 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213861+0000", 2023-10-09T11:00:43.255 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018154 2023-10-09T11:00:43.255 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.255 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.255 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:43.256 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219288+0000", 2023-10-09T11:00:43.256 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00542677 2023-10-09T11:00:43.256 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.256 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.256 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.256 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219340+0000", 2023-10-09T11:00:43.256 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2657000000000001e-05 2023-10-09T11:00:43.256 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.256 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.256 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.256 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219644+0000", 2023-10-09T11:00:43.256 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030375299999999998 2023-10-09T11:00:43.257 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.257 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.257 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.257 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235530+0000", 2023-10-09T11:00:43.257 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01588606 2023-10-09T11:00:43.257 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.257 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.257 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.257 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235575+0000", 2023-10-09T11:00:43.257 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5012999999999997e-05 2023-10-09T11:00:43.257 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.257 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.257 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.258 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235649+0000", 2023-10-09T11:00:43.258 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4252000000000007e-05 2023-10-09T11:00:43.258 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.258 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.258 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.258 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.258 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.258 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83959 8.4 8:27b6f913:::benchmark_data_smithi125_329074_object83958:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T11:00:43.258 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.112252+0000", 2023-10-09T11:00:43.258 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 386.11668132300002, 2023-10-09T11:00:43.258 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1235403750000001, 2023-10-09T11:00:43.259 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.259 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.259 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:43.259 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:00:43.259 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:00:43.259 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83959 2023-10-09T11:00:43.259 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.259 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.259 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.259 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.259 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112252+0000", 2023-10-09T11:00:43.259 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.260 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.260 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.260 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.260 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112252+0000", 2023-10-09T11:00:43.260 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.260 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.260 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.260 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.260 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112251+0000", 2023-10-09T11:00:43.260 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:43.260 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.260 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.260 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.261 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112312+0000", 2023-10-09T11:00:43.261 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0888999999999999e-05 2023-10-09T11:00:43.261 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.261 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.261 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.261 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112324+0000", 2023-10-09T11:00:43.261 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2585999999999999e-05 2023-10-09T11:00:43.261 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.261 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.261 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.261 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112329+0000", 2023-10-09T11:00:43.261 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9880000000000004e-06 2023-10-09T11:00:43.262 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.262 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.262 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.262 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112378+0000", 2023-10-09T11:00:43.262 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8905e-05 2023-10-09T11:00:43.262 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.262 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.262 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:00:43.262 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112389+0000", 2023-10-09T11:00:43.262 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0665e-05 2023-10-09T11:00:43.262 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.262 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.263 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.263 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.873769+0000", 2023-10-09T11:00:43.263 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.76138013699999996 2023-10-09T11:00:43.263 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.263 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.263 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:00:43.263 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.873776+0000", 2023-10-09T11:00:43.263 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.17e-06 2023-10-09T11:00:43.263 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.263 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.263 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.263 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216109+0000", 2023-10-09T11:00:43.264 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.342332699 2023-10-09T11:00:43.264 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.264 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.264 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.264 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216239+0000", 2023-10-09T11:00:43.264 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012997 2023-10-09T11:00:43.264 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.264 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.264 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T11:00:43.264 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216354+0000", 2023-10-09T11:00:43.264 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115116 2023-10-09T11:00:43.264 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.265 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.265 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.265 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219398+0000", 2023-10-09T11:00:43.265 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030438679999999999 2023-10-09T11:00:43.265 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.265 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.265 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:43.265 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219407+0000", 2023-10-09T11:00:43.265 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1870000000000003e-06 2023-10-09T11:00:43.265 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.265 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.265 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.265 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219682+0000", 2023-10-09T11:00:43.266 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00027478600000000001 2023-10-09T11:00:43.266 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.266 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.266 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.266 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235686+0000", 2023-10-09T11:00:43.266 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016004852 2023-10-09T11:00:43.266 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.266 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.266 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.266 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235725+0000", 2023-10-09T11:00:43.266 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8392000000000001e-05 2023-10-09T11:00:43.267 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.267 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.267 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.267 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235792+0000", 2023-10-09T11:00:43.267 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7311000000000007e-05 2023-10-09T11:00:43.267 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.267 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.267 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.267 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.267 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.267 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:93330 8.2 8:4f5a2bcc:::benchmark_data_smithi125_329074_object93329:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T11:00:43.267 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:35.625993+0000", 2023-10-09T11:00:43.268 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 367.60294044199998, 2023-10-09T11:00:43.268 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.12993565000000001, 2023-10-09T11:00:43.268 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.268 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.268 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:43.268 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:00:43.268 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:00:43.268 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 93330 2023-10-09T11:00:43.268 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.268 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.268 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625993+0000", 2023-10-09T11:00:43.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.269 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.269 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625993+0000", 2023-10-09T11:00:43.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.269 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.269 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625992+0000", 2023-10-09T11:00:43.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:00:43.270 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.270 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.270 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.270 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626043+0000", 2023-10-09T11:00:43.270 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0500000000000001e-05 2023-10-09T11:00:43.270 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.270 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.270 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.270 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626051+0000", 2023-10-09T11:00:43.270 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0509999999999997e-06 2023-10-09T11:00:43.270 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.271 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.271 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.271 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626056+0000", 2023-10-09T11:00:43.271 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9570000000000001e-06 2023-10-09T11:00:43.271 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.271 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.271 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.271 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626113+0000", 2023-10-09T11:00:43.271 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.736e-05 2023-10-09T11:00:43.271 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.271 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.271 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.271 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626283+0000", 2023-10-09T11:00:43.272 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017031699999999999 2023-10-09T11:00:43.272 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.272 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.272 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:00:43.272 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626463+0000", 2023-10-09T11:00:43.272 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018018 2023-10-09T11:00:43.272 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.272 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.272 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:43.272 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.634095+0000", 2023-10-09T11:00:43.272 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0076312469999999999 2023-10-09T11:00:43.272 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.273 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.273 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.273 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.637111+0000", 2023-10-09T11:00:43.273 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003016725 2023-10-09T11:00:43.273 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.273 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.273 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.273 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755853+0000", 2023-10-09T11:00:43.273 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11874143299999999 2023-10-09T11:00:43.273 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.273 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.273 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.274 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755887+0000", 2023-10-09T11:00:43.274 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4017999999999999e-05 2023-10-09T11:00:43.274 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.274 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.274 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.274 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755928+0000", 2023-10-09T11:00:43.274 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1452999999999998e-05 2023-10-09T11:00:43.274 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.274 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.274 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.274 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.274 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.275 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:93340 8.2 8:4653dd45:::benchmark_data_smithi125_329074_object93339:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T11:00:43.275 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:35.641334+0000", 2023-10-09T11:00:43.275 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 367.58759950199999, 2023-10-09T11:00:43.275 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.125653775, 2023-10-09T11:00:43.275 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.275 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.275 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:43.275 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:00:43.275 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:00:43.275 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 93340 2023-10-09T11:00:43.275 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.275 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.276 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.276 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.276 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641334+0000", 2023-10-09T11:00:43.276 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.276 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.276 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.276 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.276 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641334+0000", 2023-10-09T11:00:43.276 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.276 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.276 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.276 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.277 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641333+0000", 2023-10-09T11:00:43.277 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:43.277 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.277 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.277 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641362+0000", 2023-10-09T11:00:43.277 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9034999999999999e-05 2023-10-09T11:00:43.277 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.277 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.277 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641372+0000", 2023-10-09T11:00:43.277 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0597e-05 2023-10-09T11:00:43.278 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.278 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.278 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.278 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641377+0000", 2023-10-09T11:00:43.278 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7060000000000003e-06 2023-10-09T11:00:43.278 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.278 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.278 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.278 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641429+0000", 2023-10-09T11:00:43.278 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1721000000000002e-05 2023-10-09T11:00:43.278 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.278 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.279 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.279 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641529+0000", 2023-10-09T11:00:43.279 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000100602 2023-10-09T11:00:43.279 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.279 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.279 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:00:43.279 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641632+0000", 2023-10-09T11:00:43.279 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103099 2023-10-09T11:00:43.279 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.279 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.279 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.279 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.649088+0000", 2023-10-09T11:00:43.280 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0074551569999999996 2023-10-09T11:00:43.280 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.280 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.280 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:43.280 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.649159+0000", 2023-10-09T11:00:43.280 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1760000000000004e-05 2023-10-09T11:00:43.280 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.280 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.280 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.280 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766886+0000", 2023-10-09T11:00:43.280 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11772616499999999 2023-10-09T11:00:43.280 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.281 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.281 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.281 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766941+0000", 2023-10-09T11:00:43.281 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5186000000000003e-05 2023-10-09T11:00:43.281 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.281 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.281 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.281 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766987+0000", 2023-10-09T11:00:43.281 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6692e-05 2023-10-09T11:00:43.281 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.281 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.281 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.281 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.282 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.282 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:158888 8.2 8:4103043a:::benchmark_data_smithi125_329074_object62234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2023-10-09T11:00:43.282 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:00:43.282 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 267.872554149, 2023-10-09T11:00:43.282 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.123441579, 2023-10-09T11:00:43.282 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.282 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.282 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:43.282 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:00:43.282 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:00:43.282 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 158888 2023-10-09T11:00:43.282 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.283 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.283 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:00:43.283 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.283 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:00:43.283 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:00:43.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:00:43.284 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.284 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356381+0000", 2023-10-09T11:00:43.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.734e-06 2023-10-09T11:00:43.284 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.284 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356388+0000", 2023-10-09T11:00:43.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4559999999999997e-06 2023-10-09T11:00:43.285 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.285 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.285 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356392+0000", 2023-10-09T11:00:43.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0589999999999996e-06 2023-10-09T11:00:43.285 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.285 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.285 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.359480+0000", 2023-10-09T11:00:43.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030878379999999999 2023-10-09T11:00:43.285 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.286 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.286 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.359628+0000", 2023-10-09T11:00:43.286 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014820299999999999 2023-10-09T11:00:43.286 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.286 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.286 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:00:43.286 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.360074+0000", 2023-10-09T11:00:43.286 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00044630599999999999 2023-10-09T11:00:43.286 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.286 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.286 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:43.286 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.362052+0000", 2023-10-09T11:00:43.287 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019782720000000001 2023-10-09T11:00:43.287 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.287 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.287 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.287 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.362671+0000", 2023-10-09T11:00:43.287 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00061827099999999999 2023-10-09T11:00:43.287 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.287 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.287 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:43.287 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479765+0000", 2023-10-09T11:00:43.287 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11709412199999999 2023-10-09T11:00:43.287 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.288 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.288 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.288 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479796+0000", 2023-10-09T11:00:43.288 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0895000000000002e-05 2023-10-09T11:00:43.288 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.288 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.288 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.288 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479821+0000", 2023-10-09T11:00:43.288 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4816000000000001e-05 2023-10-09T11:00:43.288 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.288 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.288 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.288 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.289 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.289 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:100843 9.c e353/332 9:3d29ba4c:::benchmark_data_smithi125_363789_object100842:head v 353'6258, mlcod=353'6256)", 2023-10-09T11:00:43.289 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:43.227548+0000", 2023-10-09T11:00:43.289 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 120.00138525600001, 2023-10-09T11:00:43.289 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0253425, 2023-10-09T11:00:43.289 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.289 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.289 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.289 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.289 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.289 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227548+0000", 2023-10-09T11:00:43.289 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.289 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.290 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.290 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.290 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227548+0000", 2023-10-09T11:00:43.290 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.290 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.290 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.290 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.290 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227547+0000", 2023-10-09T11:00:43.290 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:43.290 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.290 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.290 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.290 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227594+0000", 2023-10-09T11:00:43.291 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7648000000000002e-05 2023-10-09T11:00:43.291 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.291 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.291 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.291 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227615+0000", 2023-10-09T11:00:43.291 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0435e-05 2023-10-09T11:00:43.291 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.291 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.291 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.291 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227622+0000", 2023-10-09T11:00:43.291 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4850000000000003e-06 2023-10-09T11:00:43.291 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.292 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.292 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.292 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.229957+0000", 2023-10-09T11:00:43.292 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002334381 2023-10-09T11:00:43.292 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.292 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.292 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.292 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.229976+0000", 2023-10-09T11:00:43.292 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9596999999999999e-05 2023-10-09T11:00:43.292 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.292 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.292 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.293 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.252847+0000", 2023-10-09T11:00:43.293 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022871209 2023-10-09T11:00:43.293 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.293 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.293 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.293 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.252890+0000", 2023-10-09T11:00:43.293 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2920000000000002e-05 2023-10-09T11:00:43.293 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.293 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.293 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.293 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.293 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.294 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:102485 9.c e353/332 9:3d3829d9:::benchmark_data_smithi125_363789_object102484:head v 353'6351, mlcod=353'6348)", 2023-10-09T11:00:43.294 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:45.158190+0000", 2023-10-09T11:00:43.294 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 118.070743291, 2023-10-09T11:00:43.294 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024704652000000001, 2023-10-09T11:00:43.294 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.294 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.294 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.294 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.294 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.294 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158190+0000", 2023-10-09T11:00:43.294 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.294 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.295 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.295 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.295 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158190+0000", 2023-10-09T11:00:43.295 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.295 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.295 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.295 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.295 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158189+0000", 2023-10-09T11:00:43.295 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:00:43.295 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.295 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.295 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.296 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158233+0000", 2023-10-09T11:00:43.296 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3201999999999997e-05 2023-10-09T11:00:43.296 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.296 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.296 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.296 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158242+0000", 2023-10-09T11:00:43.296 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9400000000000008e-06 2023-10-09T11:00:43.296 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.296 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.296 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.296 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158248+0000", 2023-10-09T11:00:43.296 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.844e-06 2023-10-09T11:00:43.296 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.297 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.297 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.297 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.161725+0000", 2023-10-09T11:00:43.297 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034765450000000002 2023-10-09T11:00:43.297 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.297 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.297 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.297 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.161750+0000", 2023-10-09T11:00:43.297 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4538000000000001e-05 2023-10-09T11:00:43.297 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.297 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.298 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.298 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.182847+0000", 2023-10-09T11:00:43.298 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.02109778 2023-10-09T11:00:43.298 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.298 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.298 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.298 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.182894+0000", 2023-10-09T11:00:43.298 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7141000000000001e-05 2023-10-09T11:00:43.298 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.298 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.298 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.298 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.299 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:102487 9.c e353/332 9:37d8c2ee:::benchmark_data_smithi125_363789_object102486:head v 353'6352, mlcod=353'6348)", 2023-10-09T11:00:43.299 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:45.158254+0000", 2023-10-09T11:00:43.299 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 118.070678972, 2023-10-09T11:00:43.299 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027965380000000001, 2023-10-09T11:00:43.299 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.299 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.299 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.299 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.299 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158254+0000", 2023-10-09T11:00:43.299 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.300 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.300 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.300 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.300 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158254+0000", 2023-10-09T11:00:43.300 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.300 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.300 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.300 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.300 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158254+0000", 2023-10-09T11:00:43.300 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:00:43.300 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.300 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.301 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.301 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158280+0000", 2023-10-09T11:00:43.301 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6463000000000001e-05 2023-10-09T11:00:43.301 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.301 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.301 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.301 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158296+0000", 2023-10-09T11:00:43.301 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6246999999999999e-05 2023-10-09T11:00:43.301 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.301 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.301 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.301 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158305+0000", 2023-10-09T11:00:43.301 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9639999999999992e-06 2023-10-09T11:00:43.302 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.302 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.302 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.302 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165187+0000", 2023-10-09T11:00:43.302 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0068813679999999997 2023-10-09T11:00:43.302 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.302 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.302 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.302 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165209+0000", 2023-10-09T11:00:43.302 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1926999999999999e-05 2023-10-09T11:00:43.302 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.302 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.303 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.303 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186167+0000", 2023-10-09T11:00:43.303 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020958641 2023-10-09T11:00:43.303 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.303 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.303 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.303 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186220+0000", 2023-10-09T11:00:43.303 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2243000000000001e-05 2023-10-09T11:00:43.303 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.303 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.303 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.303 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.304 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.304 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:102490 9.4 e353/330 9:207553f4:::benchmark_data_smithi125_363789_object102489:head v 353'6429, mlcod=353'6425)", 2023-10-09T11:00:43.304 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:45.159295+0000", 2023-10-09T11:00:43.304 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 118.06963815500001, 2023-10-09T11:00:43.304 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026994220999999999, 2023-10-09T11:00:43.304 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.304 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.304 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.304 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.304 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.304 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159295+0000", 2023-10-09T11:00:43.304 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.305 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.305 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.305 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.305 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159295+0000", 2023-10-09T11:00:43.305 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.305 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.305 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.305 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.305 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159294+0000", 2023-10-09T11:00:43.305 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:00:43.305 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.305 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.306 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.306 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159332+0000", 2023-10-09T11:00:43.306 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7502000000000001e-05 2023-10-09T11:00:43.306 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.306 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.306 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.306 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159344+0000", 2023-10-09T11:00:43.306 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2418999999999999e-05 2023-10-09T11:00:43.306 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.306 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.306 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.306 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159349+0000", 2023-10-09T11:00:43.307 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2619999999999998e-06 2023-10-09T11:00:43.307 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.307 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.307 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.307 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165240+0000", 2023-10-09T11:00:43.307 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0058900580000000001 2023-10-09T11:00:43.307 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.307 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.307 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.307 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165259+0000", 2023-10-09T11:00:43.307 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9369e-05 2023-10-09T11:00:43.307 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.307 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.308 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.308 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186223+0000", 2023-10-09T11:00:43.308 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020963682000000001 2023-10-09T11:00:43.308 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.308 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.308 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.308 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186289+0000", 2023-10-09T11:00:43.308 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6600000000000006e-05 2023-10-09T11:00:43.308 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.308 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.308 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.308 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.309 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.309 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:102494 9.4 e353/330 9:2c49d3a3:::benchmark_data_smithi125_363789_object102493:head v 353'6430, mlcod=353'6425)", 2023-10-09T11:00:43.309 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:45.162300+0000", 2023-10-09T11:00:43.309 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 118.066632836, 2023-10-09T11:00:43.309 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024081163999999999, 2023-10-09T11:00:43.309 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.309 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.309 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.309 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.309 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.309 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162300+0000", 2023-10-09T11:00:43.309 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.309 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.310 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.310 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.310 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162300+0000", 2023-10-09T11:00:43.310 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.310 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.310 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.310 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.310 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162299+0000", 2023-10-09T11:00:43.310 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:43.310 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.310 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.310 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.311 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162360+0000", 2023-10-09T11:00:43.311 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1148999999999993e-05 2023-10-09T11:00:43.311 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.311 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.311 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.311 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162375+0000", 2023-10-09T11:00:43.311 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.469e-05 2023-10-09T11:00:43.311 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.311 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.311 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.311 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162383+0000", 2023-10-09T11:00:43.311 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3289999999999996e-06 2023-10-09T11:00:43.312 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.312 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.312 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.312 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.168660+0000", 2023-10-09T11:00:43.312 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062775629999999999 2023-10-09T11:00:43.312 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.312 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.312 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.312 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.168682+0000", 2023-10-09T11:00:43.312 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1498999999999999e-05 2023-10-09T11:00:43.312 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.313 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.313 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.313 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186292+0000", 2023-10-09T11:00:43.313 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017610388000000001 2023-10-09T11:00:43.313 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.313 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.313 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.313 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186381+0000", 2023-10-09T11:00:43.313 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9489999999999999e-05 2023-10-09T11:00:43.313 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.313 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.313 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.313 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.314 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.314 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:141363 9.e e367/358 9:7b79acb8:::benchmark_data_smithi125_363789_object31064:head v 367'9030, mlcod=367'9028)", 2023-10-09T11:00:43.314 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:31.211612+0000", 2023-10-09T11:00:43.314 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 72.017320816999998, 2023-10-09T11:00:43.314 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023916006, 2023-10-09T11:00:43.314 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.314 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.314 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.314 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.314 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.314 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211612+0000", 2023-10-09T11:00:43.315 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.315 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.315 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.315 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.315 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211612+0000", 2023-10-09T11:00:43.315 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.315 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.315 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.315 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.315 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211611+0000", 2023-10-09T11:00:43.315 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:43.315 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.315 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.316 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.316 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211622+0000", 2023-10-09T11:00:43.316 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0907000000000001e-05 2023-10-09T11:00:43.316 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.316 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.316 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.316 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211641+0000", 2023-10-09T11:00:43.316 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9372e-05 2023-10-09T11:00:43.316 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.316 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.316 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.316 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211648+0000", 2023-10-09T11:00:43.317 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0870000000000001e-06 2023-10-09T11:00:43.317 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.317 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.317 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.317 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.213421+0000", 2023-10-09T11:00:43.317 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0017721220000000001 2023-10-09T11:00:43.317 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.317 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.317 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.317 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.213445+0000", 2023-10-09T11:00:43.317 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4845e-05 2023-10-09T11:00:43.317 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.317 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.318 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.318 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.235498+0000", 2023-10-09T11:00:43.318 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022052275999999999 2023-10-09T11:00:43.318 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.318 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.318 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.318 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.235528+0000", 2023-10-09T11:00:43.318 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0688000000000002e-05 2023-10-09T11:00:43.318 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.318 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.318 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.318 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.319 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.319 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:141367 9.4 e367/358 9:295b4262:::benchmark_data_smithi125_363789_object31068:head v 367'8817, mlcod=367'8815)", 2023-10-09T11:00:43.319 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:31.211676+0000", 2023-10-09T11:00:43.319 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 72.017257047000001, 2023-10-09T11:00:43.319 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023662195, 2023-10-09T11:00:43.319 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.319 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.319 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.319 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.319 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.319 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211676+0000", 2023-10-09T11:00:43.319 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.319 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.320 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.320 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.320 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211676+0000", 2023-10-09T11:00:43.320 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.320 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.320 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.320 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.320 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211676+0000", 2023-10-09T11:00:43.320 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:00:43.320 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.320 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.320 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.320 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211681+0000", 2023-10-09T11:00:43.321 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.643e-06 2023-10-09T11:00:43.321 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.321 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.321 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.321 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211688+0000", 2023-10-09T11:00:43.321 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3849999999999999e-06 2023-10-09T11:00:43.321 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.321 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.321 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.321 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211692+0000", 2023-10-09T11:00:43.321 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1189999999999999e-06 2023-10-09T11:00:43.321 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.321 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.322 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.322 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.216075+0000", 2023-10-09T11:00:43.322 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043827110000000001 2023-10-09T11:00:43.322 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.322 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.322 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.322 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.216099+0000", 2023-10-09T11:00:43.322 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4402000000000001e-05 2023-10-09T11:00:43.322 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.322 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.322 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.323 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.235290+0000", 2023-10-09T11:00:43.323 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019190570000000001 2023-10-09T11:00:43.323 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.323 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.323 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.323 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.235338+0000", 2023-10-09T11:00:43.323 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8767999999999998e-05 2023-10-09T11:00:43.323 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.323 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.323 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.323 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.323 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.323 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:141369 9.c e367/332 9:350d8e58:::benchmark_data_smithi125_363789_object31070:head v 367'8783, mlcod=367'8780)", 2023-10-09T11:00:43.324 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:31.211701+0000", 2023-10-09T11:00:43.324 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 72.017231702999993, 2023-10-09T11:00:43.324 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023662124, 2023-10-09T11:00:43.324 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.324 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.324 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.324 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.324 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.324 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211701+0000", 2023-10-09T11:00:43.324 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.324 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.324 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.325 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.325 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211701+0000", 2023-10-09T11:00:43.325 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.325 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.325 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.325 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.325 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211701+0000", 2023-10-09T11:00:43.325 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967296 2023-10-09T11:00:43.325 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.325 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.325 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.325 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211706+0000", 2023-10-09T11:00:43.325 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4440000000000001e-06 2023-10-09T11:00:43.326 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.326 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.326 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.326 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211710+0000", 2023-10-09T11:00:43.326 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.656e-06 2023-10-09T11:00:43.326 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.326 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.326 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.326 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211713+0000", 2023-10-09T11:00:43.326 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0960000000000001e-06 2023-10-09T11:00:43.326 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.326 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.327 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.327 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.218937+0000", 2023-10-09T11:00:43.327 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0072231760000000004 2023-10-09T11:00:43.327 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.327 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.327 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.327 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.218951+0000", 2023-10-09T11:00:43.327 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4817000000000001e-05 2023-10-09T11:00:43.327 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.327 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.327 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.327 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.235342+0000", 2023-10-09T11:00:43.328 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016390442000000002 2023-10-09T11:00:43.328 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.328 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.328 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.328 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.235364+0000", 2023-10-09T11:00:43.328 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1656999999999999e-05 2023-10-09T11:00:43.328 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.328 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.328 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.328 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.328 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.328 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:142871 9.4 e367/358 9:2119d7bf:::benchmark_data_smithi125_363789_object32572:head v 367'8924, mlcod=367'8922)", 2023-10-09T11:00:43.328 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:32.654253+0000", 2023-10-09T11:00:43.329 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 70.574680196000003, 2023-10-09T11:00:43.329 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022687891000000002, 2023-10-09T11:00:43.329 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.329 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.329 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.329 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.329 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.329 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.654253+0000", 2023-10-09T11:00:43.329 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.329 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.329 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.329 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.330 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.654253+0000", 2023-10-09T11:00:43.330 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.330 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.330 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.330 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.330 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.654253+0000", 2023-10-09T11:00:43.330 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:00:43.330 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.330 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.330 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.330 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.654257+0000", 2023-10-09T11:00:43.330 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7060000000000003e-06 2023-10-09T11:00:43.331 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.331 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.331 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.331 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.654263+0000", 2023-10-09T11:00:43.331 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.677e-06 2023-10-09T11:00:43.331 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.331 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.331 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.331 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.654266+0000", 2023-10-09T11:00:43.331 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5159999999999999e-06 2023-10-09T11:00:43.331 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.331 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.332 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.332 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.657246+0000", 2023-10-09T11:00:43.332 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029793290000000002 2023-10-09T11:00:43.332 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.332 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.332 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.332 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.657269+0000", 2023-10-09T11:00:43.332 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3118000000000001e-05 2023-10-09T11:00:43.332 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.332 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.332 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.332 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.676900+0000", 2023-10-09T11:00:43.332 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019631546 2023-10-09T11:00:43.333 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.333 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.333 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.333 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:32.676941+0000", 2023-10-09T11:00:43.333 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0388e-05 2023-10-09T11:00:43.333 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.333 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.333 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.333 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.333 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.333 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:144286 9.c e367/332 9:365a49ad:::benchmark_data_smithi125_363789_object33987:head v 367'8962, mlcod=367'8958)", 2023-10-09T11:00:43.333 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:34.058408+0000", 2023-10-09T11:00:43.333 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 69.170525045999995, 2023-10-09T11:00:43.334 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023312234000000001, 2023-10-09T11:00:43.334 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.334 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.334 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.334 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.334 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.334 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:34.058408+0000", 2023-10-09T11:00:43.334 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.334 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.334 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.334 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.334 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:34.058408+0000", 2023-10-09T11:00:43.335 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.335 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.335 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.335 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.335 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:34.058407+0000", 2023-10-09T11:00:43.335 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:43.335 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.335 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.335 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.335 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:34.058415+0000", 2023-10-09T11:00:43.335 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8259999999999993e-06 2023-10-09T11:00:43.335 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.336 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.336 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.336 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:34.058430+0000", 2023-10-09T11:00:43.336 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4722999999999999e-05 2023-10-09T11:00:43.336 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.336 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.336 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.336 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:34.058436+0000", 2023-10-09T11:00:43.336 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6250000000000001e-06 2023-10-09T11:00:43.336 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.336 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.336 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.337 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:34.058484+0000", 2023-10-09T11:00:43.337 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7375e-05 2023-10-09T11:00:43.337 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.337 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.337 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.337 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:34.058502+0000", 2023-10-09T11:00:43.337 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8408999999999999e-05 2023-10-09T11:00:43.337 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.337 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.337 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.337 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:34.081689+0000", 2023-10-09T11:00:43.338 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023186594000000001 2023-10-09T11:00:43.338 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.338 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.338 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.338 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:34.081720+0000", 2023-10-09T11:00:43.338 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1646000000000002e-05 2023-10-09T11:00:43.338 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.338 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.338 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.338 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.338 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.338 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:147234 9.4 e367/358 9:2d6b73af:::benchmark_data_smithi125_363789_object36935:head v 367'9215, mlcod=367'9213)", 2023-10-09T11:00:43.338 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:36.828586+0000", 2023-10-09T11:00:43.339 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 66.400347109999998, 2023-10-09T11:00:43.339 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024492692999999999, 2023-10-09T11:00:43.339 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.339 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.339 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.339 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.339 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.339 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.828586+0000", 2023-10-09T11:00:43.339 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.339 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.339 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.339 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.340 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.828586+0000", 2023-10-09T11:00:43.340 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.340 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.340 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.340 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.340 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.828586+0000", 2023-10-09T11:00:43.340 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:00:43.340 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.340 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.340 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.340 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.828594+0000", 2023-10-09T11:00:43.340 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.867e-06 2023-10-09T11:00:43.340 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.341 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.341 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.341 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.828601+0000", 2023-10-09T11:00:43.341 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.759e-06 2023-10-09T11:00:43.341 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.341 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.341 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.341 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.828605+0000", 2023-10-09T11:00:43.341 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0010000000000003e-06 2023-10-09T11:00:43.341 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.341 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.341 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.342 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.832080+0000", 2023-10-09T11:00:43.342 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034751349999999999 2023-10-09T11:00:43.342 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.342 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.342 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.342 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.832098+0000", 2023-10-09T11:00:43.342 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7558999999999999e-05 2023-10-09T11:00:43.342 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.342 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.342 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.342 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.853051+0000", 2023-10-09T11:00:43.343 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020953171 2023-10-09T11:00:43.343 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.343 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.343 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.343 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.853079+0000", 2023-10-09T11:00:43.343 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7631999999999999e-05 2023-10-09T11:00:43.343 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.343 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.343 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.343 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.343 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.343 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:147236 9.e e367/358 9:7700734b:::benchmark_data_smithi125_363789_object36937:head v 367'9403, mlcod=367'9399)", 2023-10-09T11:00:43.343 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:36.829529+0000", 2023-10-09T11:00:43.344 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 66.399404579000006, 2023-10-09T11:00:43.344 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023554265000000001, 2023-10-09T11:00:43.344 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.344 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.344 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.344 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.344 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.344 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.829529+0000", 2023-10-09T11:00:43.344 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.344 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.344 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.345 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.345 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.829529+0000", 2023-10-09T11:00:43.345 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.345 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.345 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.345 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.345 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.829528+0000", 2023-10-09T11:00:43.345 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:43.345 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.345 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.345 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.345 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.829535+0000", 2023-10-09T11:00:43.345 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0700000000000001e-06 2023-10-09T11:00:43.346 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.346 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.346 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.346 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.829545+0000", 2023-10-09T11:00:43.346 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0081000000000001e-05 2023-10-09T11:00:43.346 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.346 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.346 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.346 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.829550+0000", 2023-10-09T11:00:43.346 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1599999999999997e-06 2023-10-09T11:00:43.346 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.346 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.347 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.347 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.831682+0000", 2023-10-09T11:00:43.347 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021318539999999999 2023-10-09T11:00:43.347 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.347 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.347 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.347 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.831705+0000", 2023-10-09T11:00:43.347 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2676000000000001e-05 2023-10-09T11:00:43.347 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.347 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.347 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.347 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.853016+0000", 2023-10-09T11:00:43.347 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021311538000000001 2023-10-09T11:00:43.348 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.348 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.348 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.348 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.853083+0000", 2023-10-09T11:00:43.348 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6624000000000005e-05 2023-10-09T11:00:43.348 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.348 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.348 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.348 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.348 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.348 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:153414 9.c e374/332 9:3768b944:::benchmark_data_smithi125_363789_object43115:head v 374'9510, mlcod=374'9508)", 2023-10-09T11:00:43.348 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:46.250266+0000", 2023-10-09T11:00:43.349 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 56.978666943, 2023-10-09T11:00:43.349 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025683302000000002, 2023-10-09T11:00:43.349 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:43.349 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:43.349 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:43.349 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.349 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:43.349 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250266+0000", 2023-10-09T11:00:43.349 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.349 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.349 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.349 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:43.350 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250266+0000", 2023-10-09T11:00:43.350 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:43.350 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.350 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.350 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:43.350 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250265+0000", 2023-10-09T11:00:43.350 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:43.350 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.350 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.350 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:43.350 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250275+0000", 2023-10-09T11:00:43.350 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7089999999999997e-06 2023-10-09T11:00:43.351 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.351 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.351 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:43.351 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250292+0000", 2023-10-09T11:00:43.351 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7082e-05 2023-10-09T11:00:43.351 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.351 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.351 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:43.351 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250300+0000", 2023-10-09T11:00:43.351 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4689999999999997e-06 2023-10-09T11:00:43.351 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.351 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.352 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:43.352 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250340+0000", 2023-10-09T11:00:43.352 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9758000000000001e-05 2023-10-09T11:00:43.352 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.352 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.352 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:43.352 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250362+0000", 2023-10-09T11:00:43.352 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2463000000000002e-05 2023-10-09T11:00:43.352 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.352 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.352 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:43.352 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.275910+0000", 2023-10-09T11:00:43.352 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025547325999999999 2023-10-09T11:00:43.353 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:43.353 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:43.353 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:43.353 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.275949+0000", 2023-10-09T11:00:43.353 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9786999999999997e-05 2023-10-09T11:00:43.353 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.353 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.353 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.353 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:43.353 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:43.353 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:00:43.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:42 smithi155 ceph-mon[184854]: pgmap v977: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 32538/51777 objects degraded (62.843%); 0 B/s, 83 objects/s recovering 2023-10-09T11:00:43.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:42 smithi155 ceph-mon[184854]: failed to encode map e393 with expected crc 2023-10-09T11:00:43.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:42 smithi155 ceph-mon[184854]: osdmap e393: 8 total, 8 up, 8 in 2023-10-09T11:00:43.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:42 smithi155 ceph-mon[184854]: failed to encode map e393 with expected crc 2023-10-09T11:00:43.634 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T11:00:43.818 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:00:44.041 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:00:44.143 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:44.147 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[200197]: osdmap e394: 8 total, 8 up, 8 in 2023-10-09T11:00:44.148 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[200197]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.148 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[200197]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.148 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[200197]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.148 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[200197]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.148 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[200197]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.148 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[200197]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.148 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 32538/51777 objects degraded (62.843%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:44.148 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[195507]: osdmap e394: 8 total, 8 up, 8 in 2023-10-09T11:00:44.149 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[195507]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.149 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[195507]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.149 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[195507]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.149 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[195507]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.149 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[195507]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.149 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[195507]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.149 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:43 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 32538/51777 objects degraded (62.843%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:44.163 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:00:44.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:44.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:00:44.271 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:44.317 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:43 smithi155 ceph-mon[184854]: osdmap e394: 8 total, 8 up, 8 in 2023-10-09T11:00:44.317 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:43 smithi155 ceph-mon[184854]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.317 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:43 smithi155 ceph-mon[184854]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.317 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:43 smithi155 ceph-mon[184854]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.317 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:43 smithi155 ceph-mon[184854]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.317 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:43 smithi155 ceph-mon[184854]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:43 smithi155 ceph-mon[184854]: failed to encode map e394 with expected crc 2023-10-09T11:00:44.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:43 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 32538/51777 objects degraded (62.843%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:44.320 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:44.501 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:44.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:44.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:44.605 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:44.682 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:45.212 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:00:45.212 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:00:45.212 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:206797 9.a 9:5ade28f2:::benchmark_data_smithi125_363789_object96498:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2023-10-09T11:00:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:45.209152+0000", 2023-10-09T11:00:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0028974059999999999, 2023-10-09T11:00:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029216799999999999, 2023-10-09T11:00:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:00:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:00:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:00:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 206797 2023-10-09T11:00:45.213 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:45.209152+0000", 2023-10-09T11:00:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:45.209152+0000", 2023-10-09T11:00:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:45.214 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:45.209151+0000", 2023-10-09T11:00:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:45.209158+0000", 2023-10-09T11:00:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0180000000000004e-06 2023-10-09T11:00:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:45.215 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:45.209174+0000", 2023-10-09T11:00:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5883999999999999e-05 2023-10-09T11:00:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:45.209181+0000", 2023-10-09T11:00:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.109e-06 2023-10-09T11:00:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:45.209235+0000", 2023-10-09T11:00:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4174e-05 2023-10-09T11:00:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:45.216 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:45.209385+0000", 2023-10-09T11:00:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015025600000000001 2023-10-09T11:00:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,7", 2023-10-09T11:00:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:45.209538+0000", 2023-10-09T11:00:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015307299999999999 2023-10-09T11:00:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:45.217 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:45.218 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T11:00:45.218 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 1 2023-10-09T11:00:45.218 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:00:45.219 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:44 smithi125 ceph-mon[195507]: pgmap v980: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 32538/49548 objects degraded (65.670%); 0 B/s, 73 objects/s recovering 2023-10-09T11:00:45.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:44 smithi155 ceph-mon[184854]: pgmap v980: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 32538/49548 objects degraded (65.670%); 0 B/s, 73 objects/s recovering 2023-10-09T11:00:45.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:44 smithi125 ceph-mon[200197]: pgmap v980: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 32538/49548 objects degraded (65.670%); 0 B/s, 73 objects/s recovering 2023-10-09T11:00:45.339 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:45.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:45.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:45.448 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:45.490 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:45.515 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:45.565 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:45.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:45.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:45.584 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T11:00:45.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:45.710 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:45.741 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:45.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:45.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:45.850 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:45.893 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:45.936 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:45.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:45.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:46.040 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:00:46.040 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:46.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:00:46.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:46 smithi125 ceph-mon[195507]: pgmap v981: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 32538/49548 objects degraded (65.670%); 0 B/s, 68 objects/s recovering 2023-10-09T11:00:46.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:00:46.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:46 smithi125 ceph-mon[200197]: pgmap v981: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 32538/49548 objects degraded (65.670%); 0 B/s, 68 objects/s recovering 2023-10-09T11:00:46.111 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:46.211 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:46.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:46.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:46.318 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:46.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:00:46.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:46 smithi155 ceph-mon[184854]: pgmap v981: 129 pgs: 1 active+recovering+undersized+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 32538/49548 objects degraded (65.670%); 0 B/s, 68 objects/s recovering 2023-10-09T11:00:46.384 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:46.538 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:46.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:46.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:46.644 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:46.684 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:46.713 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:46.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:46.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:46.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:46.895 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:46.928 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:46.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:46.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:46.996 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:00:47.037 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:47.046 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:00:47.071 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.6 primary-affinity to 0.281555 (8184522) 2023-10-09T11:00:47.091 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:47.101 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:00:47.120 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:47.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:47.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:47.162 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:00:47.162 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:00:47.162 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T11:00:47.162 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T11:00:47.162 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:00:47.173 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:00:47.215 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:00:47.225 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:00:47.269 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:47.272 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:00:47.298 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1129420261' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.2815554078413448}]: dispatch 2023-10-09T11:00:47.298 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:47 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.2815554078413448}]: dispatch 2023-10-09T11:00:47.298 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4238206239' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:47.298 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1129420261' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.2815554078413448}]: dispatch 2023-10-09T11:00:47.298 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:47 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.2815554078413448}]: dispatch 2023-10-09T11:00:47.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4238206239' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:47.299 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:00:47.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:47 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1129420261' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.2815554078413448}]: dispatch 2023-10-09T11:00:47.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:47 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.2815554078413448}]: dispatch 2023-10-09T11:00:47.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:47 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4238206239' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:47.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:47.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:00:47.308 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:00:47.353 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:00:47.373 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:00:47.373 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:00:47.373 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:00:47.373 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:00:47.373 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:00:47.373 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:00:47.373 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:00:47.373 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:00:47.404 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:00:47.453 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:47.544 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:00:47.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:47.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:00:47.570 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T11:00:47.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:47.704 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:47.785 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:47.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:47.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:47.891 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:47.949 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:47.959 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:00:47.992 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:47.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:47.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:48.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:48.111 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:00:48.148 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:48.237 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:48.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:48.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:48 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.2815554078413448}]': finished 2023-10-09T11:00:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:48 smithi125 ceph-mon[195507]: osdmap e395: 8 total, 8 up, 8 in 2023-10-09T11:00:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:48 smithi125 ceph-mon[195507]: failed to encode map e395 with expected crc 2023-10-09T11:00:48.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:48 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/385043687' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:00:48.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:48 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:00:48.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:48 smithi125 ceph-mon[195507]: pgmap v983: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 853 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 30581/40989 objects degraded (74.608%); 0 B/s, 54 objects/s recovering 2023-10-09T11:00:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:48 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.2815554078413448}]': finished 2023-10-09T11:00:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:48 smithi125 ceph-mon[200197]: osdmap e395: 8 total, 8 up, 8 in 2023-10-09T11:00:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:48 smithi125 ceph-mon[200197]: failed to encode map e395 with expected crc 2023-10-09T11:00:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:48 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/385043687' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:00:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:48 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:00:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:48 smithi125 ceph-mon[200197]: pgmap v983: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 853 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 30581/40989 objects degraded (74.608%); 0 B/s, 54 objects/s recovering 2023-10-09T11:00:48.344 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:48.390 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:48 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.2815554078413448}]': finished 2023-10-09T11:00:48.391 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:48 smithi155 ceph-mon[184854]: osdmap e395: 8 total, 8 up, 8 in 2023-10-09T11:00:48.391 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:48 smithi155 ceph-mon[184854]: failed to encode map e395 with expected crc 2023-10-09T11:00:48.391 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:48 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/385043687' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:00:48.391 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:48 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:00:48.391 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:48 smithi155 ceph-mon[184854]: pgmap v983: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 853 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 30581/40989 objects degraded (74.608%); 0 B/s, 54 objects/s recovering 2023-10-09T11:00:48.401 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:48.580 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:48.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:48.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:48.685 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:48.732 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:48.821 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:48.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:48.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:48.925 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:48.986 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:49.032 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:49.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:49.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:49.054 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:00:49.054 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:00:49.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:49.180 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:49.205 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:49.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:49.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:49.314 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:00:49.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:49 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:00:49.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:49 smithi125 ceph-mon[195507]: failed to encode map e396 with expected crc 2023-10-09T11:00:49.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:49 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:00:49.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:49 smithi125 ceph-mon[195507]: osdmap e396: 8 total, 8 up, 8 in 2023-10-09T11:00:49.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:49 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30581/40989 objects degraded (74.608%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:49.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:49 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:00:49.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:49 smithi125 ceph-mon[200197]: failed to encode map e396 with expected crc 2023-10-09T11:00:49.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:49 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:00:49.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:49 smithi125 ceph-mon[200197]: osdmap e396: 8 total, 8 up, 8 in 2023-10-09T11:00:49.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:49 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30581/40989 objects degraded (74.608%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:49.349 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:49 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:00:49.349 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:49 smithi155 ceph-mon[184854]: failed to encode map e396 with expected crc 2023-10-09T11:00:49.349 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:49 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:00:49.349 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:49 smithi155 ceph-mon[184854]: osdmap e396: 8 total, 8 up, 8 in 2023-10-09T11:00:49.349 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:49 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30581/40989 objects degraded (74.608%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:49.351 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:49.382 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:00:49.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:49.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:00:49.438 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T11:00:49.492 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:49.545 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:49.585 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:49.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:49.691 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:49.751 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:49.773 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:49.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:49.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:49.810 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:00:49.813 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:00:49.880 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:49.937 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:49.965 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:49.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:49.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:50.071 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:50.114 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:50.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:50 smithi155 ceph-mon[184854]: pgmap v985: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 808 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 30581/38838 objects degraded (78.740%); 0 B/s, 49 objects/s recovering 2023-10-09T11:00:50.237 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:50.291 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:50.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:50.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:50.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:50 smithi125 ceph-mon[195507]: pgmap v985: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 808 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 30581/38838 objects degraded (78.740%); 0 B/s, 49 objects/s recovering 2023-10-09T11:00:50.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:50 smithi125 ceph-mon[200197]: pgmap v985: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 808 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 30581/38838 objects degraded (78.740%); 0 B/s, 49 objects/s recovering 2023-10-09T11:00:50.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:50.439 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:50.494 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:50.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:50.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:50.523 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:00:50.604 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:50.691 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:50.835 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:50.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:50.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:50.943 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:00:50.983 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:00:50.984 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:00:51.002 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:51.076 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:00:51.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:51.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:00:51.173 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:51.185 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:51.231 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:51.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:51 smithi155 ceph-mon[184854]: 9.15 scrub starts 2023-10-09T11:00:51.311 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:51 smithi125 ceph-mon[195507]: 9.15 scrub starts 2023-10-09T11:00:51.312 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:00:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:00:51] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T11:00:51.312 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:51 smithi125 ceph-mon[200197]: 9.15 scrub starts 2023-10-09T11:00:51.316 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:51.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:51.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:51.420 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T11:00:51.420 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:51.467 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:51.504 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:51.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:51.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:51.612 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:51.662 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:00:51.664 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:51.693 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:51.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:51.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:51.711 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:00:51.759 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:00:51.797 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:00:51.804 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:00:51.807 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:00:51.833 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:00:51.888 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:00:51.888 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:51.926 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:00:51.973 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:00:51.984 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:00:51.985 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:00:51.985 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:00:51.985 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:00:51.985 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:00:51.985 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:00:51.985 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:00:51.985 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:00:52.085 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:00:52.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:52.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:00:52.190 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:52 smithi155 ceph-mon[184854]: 1.17 scrub starts 2023-10-09T11:00:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:52 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1421686485' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:52 smithi155 ceph-mon[184854]: pgmap v986: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 610 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 30581/29304 objects degraded (104.358%); 0 B/s, 73 objects/s recovering 2023-10-09T11:00:52.229 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:52.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:52 smithi125 ceph-mon[195507]: 1.17 scrub starts 2023-10-09T11:00:52.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:52 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1421686485' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:52.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:52 smithi125 ceph-mon[195507]: pgmap v986: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 610 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 30581/29304 objects degraded (104.358%); 0 B/s, 73 objects/s recovering 2023-10-09T11:00:52.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:52 smithi125 ceph-mon[200197]: 1.17 scrub starts 2023-10-09T11:00:52.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:52 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1421686485' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:52.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:52 smithi125 ceph-mon[200197]: pgmap v986: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 610 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 30581/29304 objects degraded (104.358%); 0 B/s, 73 objects/s recovering 2023-10-09T11:00:52.567 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T11:00:52.567 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:00:52.567 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T11:00:52.568 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T11:00:52.568 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T11:00:52.568 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T11:00:53.074 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:53.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:53.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:53.180 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:00:53.238 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:53.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:53 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30581/29304 objects degraded (104.358%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:53.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:53 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30581/29304 objects degraded (104.358%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:53.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:53 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30581/29304 objects degraded (104.358%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:54.212 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:00:54.212 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:00:54.212 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T11:00:54.212 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T11:00:54.212 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:00:54.442 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:54 smithi125 ceph-mon[195507]: pgmap v987: 129 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 563 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 30581/27072 objects degraded (112.962%); 0 B/s, 73 objects/s recovering 2023-10-09T11:00:54.442 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:54 smithi125 ceph-mon[200197]: pgmap v987: 129 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 563 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 30581/27072 objects degraded (112.962%); 0 B/s, 73 objects/s recovering 2023-10-09T11:00:54.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:54 smithi155 ceph-mon[184854]: pgmap v987: 129 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 563 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 30581/27072 objects degraded (112.962%); 0 B/s, 73 objects/s recovering 2023-10-09T11:00:54.609 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T11:00:55.024 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:00:55.107 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:55.132 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:00:55.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:55.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:00:55.242 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:55.280 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:55.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:55 smithi125 ceph-mon[195507]: 1.6 scrub starts 2023-10-09T11:00:55.299 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:55 smithi125 ceph-mon[200197]: 1.6 scrub starts 2023-10-09T11:00:55.365 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:55.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:55.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:55 smithi155 ceph-mon[184854]: 1.6 scrub starts 2023-10-09T11:00:55.473 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:55.565 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:55.649 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:00:55.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:55.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:00:55.754 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:00:55.852 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:56.108 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:00:56.108 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T11:00:56.109 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T11:00:56.109 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:00:56.109 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.109 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:95197 7.d 7:b9b00ad5:::benchmark_data_smithi125_295816_object4060:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e226)", 2023-10-09T11:00:56.109 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:57.383916+0000", 2023-10-09T11:00:56.109 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 598.72037625999997, 2023-10-09T11:00:56.109 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.070851236999999997, 2023-10-09T11:00:56.109 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.109 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.109 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:56.109 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T11:00:56.110 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T11:00:56.110 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 95197 2023-10-09T11:00:56.110 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.110 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.110 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.110 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.110 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383916+0000", 2023-10-09T11:00:56.110 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.110 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.110 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.110 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.110 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383916+0000", 2023-10-09T11:00:56.111 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.111 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.111 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.111 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.111 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383915+0000", 2023-10-09T11:00:56.111 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:56.111 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.111 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.111 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.111 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383920+0000", 2023-10-09T11:00:56.111 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.84e-06 2023-10-09T11:00:56.111 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.112 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.112 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.112 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383931+0000", 2023-10-09T11:00:56.112 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0657e-05 2023-10-09T11:00:56.112 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.112 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.112 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.112 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.383938+0000", 2023-10-09T11:00:56.112 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5139999999999998e-06 2023-10-09T11:00:56.112 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.112 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.112 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.113 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.411508+0000", 2023-10-09T11:00:56.113 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027570921000000002 2023-10-09T11:00:56.113 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.113 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.113 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T11:00:56.113 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.416286+0000", 2023-10-09T11:00:56.113 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0047777430000000001 2023-10-09T11:00:56.113 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.113 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.113 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.113 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.437950+0000", 2023-10-09T11:00:56.113 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021664251999999998 2023-10-09T11:00:56.114 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.114 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.114 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.114 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.438046+0000", 2023-10-09T11:00:56.114 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5459999999999997e-05 2023-10-09T11:00:56.114 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.114 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.114 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:00:56.114 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.438186+0000", 2023-10-09T11:00:56.114 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014000800000000001 2023-10-09T11:00:56.114 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.114 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.115 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:56.115 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.443989+0000", 2023-10-09T11:00:56.115 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0058031369999999999 2023-10-09T11:00:56.115 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.115 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.115 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.115 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.452036+0000", 2023-10-09T11:00:56.115 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0080469280000000001 2023-10-09T11:00:56.115 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.115 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.115 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.115 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.452814+0000", 2023-10-09T11:00:56.115 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00077831199999999997 2023-10-09T11:00:56.116 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.116 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.116 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.116 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.452840+0000", 2023-10-09T11:00:56.116 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5947000000000001e-05 2023-10-09T11:00:56.116 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.116 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.116 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.116 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:57.454767+0000", 2023-10-09T11:00:56.116 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019270769999999999 2023-10-09T11:00:56.116 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.116 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.117 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.117 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.117 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.117 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:95787 7.d 7:b6e713eb:::benchmark_data_smithi125_295816_object4650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T11:00:56.117 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:58.300479+0000", 2023-10-09T11:00:56.117 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 597.803813827, 2023-10-09T11:00:56.117 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072656459000000007, 2023-10-09T11:00:56.117 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.117 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.117 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:56.117 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T11:00:56.117 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T11:00:56.118 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 95787 2023-10-09T11:00:56.118 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.118 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.118 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.118 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.118 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300479+0000", 2023-10-09T11:00:56.118 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.118 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.118 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.118 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.118 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300479+0000", 2023-10-09T11:00:56.118 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.119 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.119 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.119 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.119 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300477+0000", 2023-10-09T11:00:56.119 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T11:00:56.119 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.119 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.119 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.119 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300487+0000", 2023-10-09T11:00:56.119 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0638999999999999e-05 2023-10-09T11:00:56.119 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.119 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.120 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.120 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300511+0000", 2023-10-09T11:00:56.120 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3059999999999999e-05 2023-10-09T11:00:56.120 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.120 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.120 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.120 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.300518+0000", 2023-10-09T11:00:56.120 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0029999999999997e-06 2023-10-09T11:00:56.120 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.120 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.120 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.120 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.322068+0000", 2023-10-09T11:00:56.120 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021550625 2023-10-09T11:00:56.121 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.121 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.121 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T11:00:56.121 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.333963+0000", 2023-10-09T11:00:56.121 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011895039 2023-10-09T11:00:56.121 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.121 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.121 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.121 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.351448+0000", 2023-10-09T11:00:56.121 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017484831999999999 2023-10-09T11:00:56.121 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.121 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.121 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.122 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.352038+0000", 2023-10-09T11:00:56.122 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00059005600000000004 2023-10-09T11:00:56.122 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.122 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.122 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:00:56.122 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.352181+0000", 2023-10-09T11:00:56.122 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014277800000000001 2023-10-09T11:00:56.122 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.122 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.122 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.122 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.367667+0000", 2023-10-09T11:00:56.122 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015486592 2023-10-09T11:00:56.123 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.123 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.123 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:56.123 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.370532+0000", 2023-10-09T11:00:56.123 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028642279999999999 2023-10-09T11:00:56.123 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.123 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.123 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.123 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.371142+0000", 2023-10-09T11:00:56.123 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00060984699999999995 2023-10-09T11:00:56.123 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.123 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.123 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.124 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.371180+0000", 2023-10-09T11:00:56.124 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8529000000000003e-05 2023-10-09T11:00:56.124 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.124 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.124 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.124 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:58.373135+0000", 2023-10-09T11:00:56.124 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001954973 2023-10-09T11:00:56.124 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.124 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.124 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.124 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.124 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.125 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96817 7.d 7:b9f7ad08:::benchmark_data_smithi125_295816_object5680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T11:00:56.125 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:50:59.884613+0000", 2023-10-09T11:00:56.125 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 596.21967937399995, 2023-10-09T11:00:56.125 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14656745900000001, 2023-10-09T11:00:56.125 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.125 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.125 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:56.125 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T11:00:56.125 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T11:00:56.125 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96817 2023-10-09T11:00:56.125 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.125 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.126 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.126 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.126 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884613+0000", 2023-10-09T11:00:56.126 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.126 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.126 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.126 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.126 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884613+0000", 2023-10-09T11:00:56.126 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.126 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.126 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.126 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.126 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884612+0000", 2023-10-09T11:00:56.127 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:56.127 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.127 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.127 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.127 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884618+0000", 2023-10-09T11:00:56.127 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5880000000000003e-06 2023-10-09T11:00:56.127 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.127 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.127 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.127 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884633+0000", 2023-10-09T11:00:56.127 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5109999999999999e-05 2023-10-09T11:00:56.127 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.128 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.128 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.128 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.884639+0000", 2023-10-09T11:00:56.128 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.164e-06 2023-10-09T11:00:56.128 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.128 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.128 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.128 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.887892+0000", 2023-10-09T11:00:56.128 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032536610000000001 2023-10-09T11:00:56.128 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.128 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.128 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T11:00:56.129 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.888551+0000", 2023-10-09T11:00:56.129 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00065867400000000004 2023-10-09T11:00:56.129 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.129 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.129 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.129 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910766+0000", 2023-10-09T11:00:56.129 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022214655 2023-10-09T11:00:56.129 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.129 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.129 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.129 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910852+0000", 2023-10-09T11:00:56.129 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6551999999999994e-05 2023-10-09T11:00:56.129 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.130 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.130 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:00:56.130 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.910980+0000", 2023-10-09T11:00:56.130 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000127704 2023-10-09T11:00:56.130 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.130 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.130 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:56.130 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:50:59.914262+0000", 2023-10-09T11:00:56.130 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032824070000000002 2023-10-09T11:00:56.130 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.130 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.130 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.131 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.020644+0000", 2023-10-09T11:00:56.131 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.106381932 2023-10-09T11:00:56.131 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.131 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.131 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.131 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.029029+0000", 2023-10-09T11:00:56.131 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0083848010000000008 2023-10-09T11:00:56.131 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.131 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.131 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.131 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.029088+0000", 2023-10-09T11:00:56.131 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8641999999999999e-05 2023-10-09T11:00:56.132 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.132 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.132 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.132 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.031180+0000", 2023-10-09T11:00:56.132 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020928359999999998 2023-10-09T11:00:56.132 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.132 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.132 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.132 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.132 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.132 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96884 7.d 7:b183b169:::benchmark_data_smithi125_295816_object5747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T11:00:56.132 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.113486+0000", 2023-10-09T11:00:56.132 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 595.99080608400004, 2023-10-09T11:00:56.133 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.082588165000000005, 2023-10-09T11:00:56.133 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.133 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.133 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:56.133 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T11:00:56.133 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T11:00:56.133 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96884 2023-10-09T11:00:56.133 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.133 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.133 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.133 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.133 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113486+0000", 2023-10-09T11:00:56.134 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.134 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.134 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.134 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.134 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113486+0000", 2023-10-09T11:00:56.134 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.134 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.134 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.134 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.134 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113485+0000", 2023-10-09T11:00:56.134 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:56.134 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.134 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.135 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.135 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113491+0000", 2023-10-09T11:00:56.135 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3009999999999997e-06 2023-10-09T11:00:56.135 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.135 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.135 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.135 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113501+0000", 2023-10-09T11:00:56.135 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0342e-05 2023-10-09T11:00:56.135 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.135 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.135 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.135 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.113505+0000", 2023-10-09T11:00:56.136 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4530000000000004e-06 2023-10-09T11:00:56.136 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.136 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.136 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.136 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.117310+0000", 2023-10-09T11:00:56.136 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0038049329999999999 2023-10-09T11:00:56.136 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.136 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.136 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T11:00:56.136 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.117924+0000", 2023-10-09T11:00:56.136 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00061396699999999998 2023-10-09T11:00:56.136 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.136 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.137 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.137 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.140859+0000", 2023-10-09T11:00:56.137 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022935140999999999 2023-10-09T11:00:56.137 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.137 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.137 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.137 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.140963+0000", 2023-10-09T11:00:56.137 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010341 2023-10-09T11:00:56.137 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.137 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.137 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:00:56.137 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.141120+0000", 2023-10-09T11:00:56.138 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000157405 2023-10-09T11:00:56.138 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.138 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.138 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:56.138 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.150528+0000", 2023-10-09T11:00:56.138 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0094081660000000008 2023-10-09T11:00:56.138 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.138 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.138 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.138 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.160789+0000", 2023-10-09T11:00:56.138 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010260429 2023-10-09T11:00:56.138 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.139 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.139 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.139 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.194166+0000", 2023-10-09T11:00:56.139 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.033377636000000002 2023-10-09T11:00:56.139 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.139 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.139 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.139 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.194226+0000", 2023-10-09T11:00:56.139 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9855999999999997e-05 2023-10-09T11:00:56.139 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.139 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.139 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.140 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196074+0000", 2023-10-09T11:00:56.140 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001848192 2023-10-09T11:00:56.140 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.140 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.140 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.140 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.140 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.140 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96892 7.d 7:bd59fbdb:::benchmark_data_smithi125_295816_object5755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T11:00:56.140 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.148429+0000", 2023-10-09T11:00:56.140 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 595.95586383199998, 2023-10-09T11:00:56.140 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.089082649999999999, 2023-10-09T11:00:56.140 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.141 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.141 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:56.141 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T11:00:56.141 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T11:00:56.141 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96892 2023-10-09T11:00:56.141 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.141 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.141 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.141 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.141 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148429+0000", 2023-10-09T11:00:56.141 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.141 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.141 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.142 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.142 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148429+0000", 2023-10-09T11:00:56.142 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.142 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.142 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.142 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.142 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148428+0000", 2023-10-09T11:00:56.142 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967296 2023-10-09T11:00:56.142 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.142 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.142 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.142 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148430+0000", 2023-10-09T11:00:56.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5570000000000001e-06 2023-10-09T11:00:56.143 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.143 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.143 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.143 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148433+0000", 2023-10-09T11:00:56.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.892e-06 2023-10-09T11:00:56.143 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.143 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.143 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.143 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.148435+0000", 2023-10-09T11:00:56.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5339999999999998e-06 2023-10-09T11:00:56.143 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.143 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.144 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.144 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.152891+0000", 2023-10-09T11:00:56.144 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0044553409999999998 2023-10-09T11:00:56.144 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.144 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.144 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T11:00:56.144 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.153532+0000", 2023-10-09T11:00:56.144 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000641539 2023-10-09T11:00:56.144 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.144 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.144 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.144 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.182585+0000", 2023-10-09T11:00:56.145 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029053295 2023-10-09T11:00:56.145 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.145 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.145 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.145 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.182682+0000", 2023-10-09T11:00:56.145 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6623e-05 2023-10-09T11:00:56.145 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.145 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.145 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:00:56.145 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.182825+0000", 2023-10-09T11:00:56.145 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014288300000000001 2023-10-09T11:00:56.145 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.145 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.146 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:56.146 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.198214+0000", 2023-10-09T11:00:56.146 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015389162 2023-10-09T11:00:56.146 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.146 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.146 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.146 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.223087+0000", 2023-10-09T11:00:56.146 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024872423000000001 2023-10-09T11:00:56.146 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.146 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.146 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.146 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.235280+0000", 2023-10-09T11:00:56.147 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012193017 2023-10-09T11:00:56.147 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.147 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.147 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.147 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.235328+0000", 2023-10-09T11:00:56.147 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8211000000000003e-05 2023-10-09T11:00:56.147 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.147 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.147 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.147 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.237511+0000", 2023-10-09T11:00:56.147 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021833960000000002 2023-10-09T11:00:56.147 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.148 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.148 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.148 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.148 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.148 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96898 7.d 7:b4ab753b:::benchmark_data_smithi125_295816_object5761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T11:00:56.148 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.166362+0000", 2023-10-09T11:00:56.148 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 595.937930338, 2023-10-09T11:00:56.148 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072656032999999995, 2023-10-09T11:00:56.148 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.148 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.148 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:56.148 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T11:00:56.149 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T11:00:56.149 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96898 2023-10-09T11:00:56.149 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.149 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.149 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.149 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.149 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166362+0000", 2023-10-09T11:00:56.149 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.149 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.149 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.149 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.149 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166362+0000", 2023-10-09T11:00:56.149 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.150 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.150 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.150 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.150 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166360+0000", 2023-10-09T11:00:56.150 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T11:00:56.150 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.150 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.150 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.150 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166369+0000", 2023-10-09T11:00:56.150 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.166e-06 2023-10-09T11:00:56.150 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.150 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.151 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.151 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166385+0000", 2023-10-09T11:00:56.151 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5214e-05 2023-10-09T11:00:56.151 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.151 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.151 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.151 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166392+0000", 2023-10-09T11:00:56.151 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1659999999999997e-06 2023-10-09T11:00:56.151 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.151 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.151 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.151 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.166446+0000", 2023-10-09T11:00:56.151 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3819000000000001e-05 2023-10-09T11:00:56.152 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.152 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.152 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T11:00:56.152 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.167178+0000", 2023-10-09T11:00:56.152 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00073199300000000001 2023-10-09T11:00:56.152 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.152 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.152 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.152 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.191480+0000", 2023-10-09T11:00:56.152 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024302082999999999 2023-10-09T11:00:56.152 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.152 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.153 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.153 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.191594+0000", 2023-10-09T11:00:56.153 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000114527 2023-10-09T11:00:56.153 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.153 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.153 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:00:56.153 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.191761+0000", 2023-10-09T11:00:56.153 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000166599 2023-10-09T11:00:56.153 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.153 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.153 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:56.153 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.198249+0000", 2023-10-09T11:00:56.153 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064882439999999998 2023-10-09T11:00:56.154 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.154 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.154 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.154 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.223116+0000", 2023-10-09T11:00:56.154 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024867330999999999 2023-10-09T11:00:56.154 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.154 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.154 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.154 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.237546+0000", 2023-10-09T11:00:56.154 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014429365 2023-10-09T11:00:56.154 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.155 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.155 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.155 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.237573+0000", 2023-10-09T11:00:56.155 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7534000000000001e-05 2023-10-09T11:00:56.155 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.155 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.155 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.155 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.239018+0000", 2023-10-09T11:00:56.155 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001444776 2023-10-09T11:00:56.155 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.155 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.155 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.156 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.156 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.156 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96901 7.d 7:bd396a29:::benchmark_data_smithi125_295816_object5764:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T11:00:56.156 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.177994+0000", 2023-10-09T11:00:56.156 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 595.92629851699996, 2023-10-09T11:00:56.156 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.079151244999999995, 2023-10-09T11:00:56.156 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.156 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.156 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:56.156 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T11:00:56.156 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T11:00:56.157 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96901 2023-10-09T11:00:56.157 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.157 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.157 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.157 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.157 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177994+0000", 2023-10-09T11:00:56.157 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.157 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.157 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.157 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.157 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177994+0000", 2023-10-09T11:00:56.157 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.157 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.158 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.158 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.158 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177993+0000", 2023-10-09T11:00:56.158 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:56.158 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.158 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.158 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.158 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.177998+0000", 2023-10-09T11:00:56.158 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6699999999999999e-06 2023-10-09T11:00:56.158 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.158 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.159 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.159 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.178008+0000", 2023-10-09T11:00:56.159 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6520000000000001e-06 2023-10-09T11:00:56.159 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.159 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.159 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.159 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.178013+0000", 2023-10-09T11:00:56.159 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.028e-06 2023-10-09T11:00:56.159 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.159 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.159 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.159 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196130+0000", 2023-10-09T11:00:56.160 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018116621999999999 2023-10-09T11:00:56.160 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.160 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.160 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T11:00:56.160 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196670+0000", 2023-10-09T11:00:56.160 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00054042399999999996 2023-10-09T11:00:56.160 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.160 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.160 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.160 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.227020+0000", 2023-10-09T11:00:56.160 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030349832 2023-10-09T11:00:56.160 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.160 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.161 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.161 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.227095+0000", 2023-10-09T11:00:56.161 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4785999999999999e-05 2023-10-09T11:00:56.161 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.161 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.161 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:00:56.161 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.227204+0000", 2023-10-09T11:00:56.161 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000109079 2023-10-09T11:00:56.161 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.161 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.161 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:56.161 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.239698+0000", 2023-10-09T11:00:56.162 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012494081000000001 2023-10-09T11:00:56.162 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.162 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.162 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.162 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242208+0000", 2023-10-09T11:00:56.162 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00251001 2023-10-09T11:00:56.162 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.162 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.162 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.162 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.254821+0000", 2023-10-09T11:00:56.162 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012612978 2023-10-09T11:00:56.162 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.163 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.163 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.163 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.254858+0000", 2023-10-09T11:00:56.163 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6785000000000001e-05 2023-10-09T11:00:56.163 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.163 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.163 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.163 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.257145+0000", 2023-10-09T11:00:56.163 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002287374 2023-10-09T11:00:56.163 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.163 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.163 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.164 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.164 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.164 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96908 7.d 7:bd9652bb:::benchmark_data_smithi125_295816_object5771:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T11:00:56.164 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.184970+0000", 2023-10-09T11:00:56.164 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 595.91932212500001, 2023-10-09T11:00:56.164 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.107155712, 2023-10-09T11:00:56.164 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.164 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.164 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:56.164 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T11:00:56.164 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T11:00:56.164 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96908 2023-10-09T11:00:56.165 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.165 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.165 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.165 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.165 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184970+0000", 2023-10-09T11:00:56.165 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.165 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.165 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.165 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.165 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184970+0000", 2023-10-09T11:00:56.165 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.165 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.165 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.166 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.166 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184969+0000", 2023-10-09T11:00:56.166 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:56.166 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.166 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.166 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.166 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184976+0000", 2023-10-09T11:00:56.166 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0849999999999999e-06 2023-10-09T11:00:56.166 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.166 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.166 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.167 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184988+0000", 2023-10-09T11:00:56.167 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2438e-05 2023-10-09T11:00:56.167 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.167 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.167 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.167 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.184994+0000", 2023-10-09T11:00:56.167 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.835e-06 2023-10-09T11:00:56.167 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.167 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.167 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.167 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.196710+0000", 2023-10-09T11:00:56.167 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011715557 2023-10-09T11:00:56.167 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.168 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.168 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T11:00:56.168 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.197312+0000", 2023-10-09T11:00:56.168 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000602675 2023-10-09T11:00:56.168 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.168 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.168 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.168 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.232847+0000", 2023-10-09T11:00:56.168 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.035534143999999997 2023-10-09T11:00:56.168 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.168 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.168 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.169 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.232937+0000", 2023-10-09T11:00:56.169 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0303999999999995e-05 2023-10-09T11:00:56.169 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.169 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.169 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:00:56.169 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.233064+0000", 2023-10-09T11:00:56.169 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012689799999999999 2023-10-09T11:00:56.169 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.169 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.169 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:56.169 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242946+0000", 2023-10-09T11:00:56.169 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0098825000000000007 2023-10-09T11:00:56.169 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.170 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.170 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.170 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.269531+0000", 2023-10-09T11:00:56.170 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026584907000000001 2023-10-09T11:00:56.170 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.170 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.170 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.170 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.289926+0000", 2023-10-09T11:00:56.170 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020394791999999998 2023-10-09T11:00:56.170 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.170 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.170 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.171 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.289981+0000", 2023-10-09T11:00:56.171 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4540999999999998e-05 2023-10-09T11:00:56.171 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.171 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.171 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.171 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.292126+0000", 2023-10-09T11:00:56.171 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021454270000000001 2023-10-09T11:00:56.171 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.171 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.171 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.171 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.171 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.171 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96935 7.d 7:b8c4db12:::benchmark_data_smithi125_295816_object5798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T11:00:56.172 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.240816+0000", 2023-10-09T11:00:56.172 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 595.86347624400003, 2023-10-09T11:00:56.172 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.096789209000000001, 2023-10-09T11:00:56.172 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.172 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.172 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:56.172 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T11:00:56.172 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T11:00:56.172 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96935 2023-10-09T11:00:56.172 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.172 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.172 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.173 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.173 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240816+0000", 2023-10-09T11:00:56.173 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.173 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.173 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.173 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.173 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240816+0000", 2023-10-09T11:00:56.173 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.173 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.173 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.173 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.173 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240815+0000", 2023-10-09T11:00:56.174 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:56.174 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.174 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.174 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.174 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240821+0000", 2023-10-09T11:00:56.174 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1140000000000002e-06 2023-10-09T11:00:56.174 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.174 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.174 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.174 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240831+0000", 2023-10-09T11:00:56.174 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6309999999999998e-06 2023-10-09T11:00:56.174 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.175 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.175 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.175 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.240845+0000", 2023-10-09T11:00:56.175 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4705000000000001e-05 2023-10-09T11:00:56.175 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.175 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.175 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.175 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242238+0000", 2023-10-09T11:00:56.175 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001392646 2023-10-09T11:00:56.175 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.175 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.175 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T11:00:56.176 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.242937+0000", 2023-10-09T11:00:56.176 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00069863399999999995 2023-10-09T11:00:56.176 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.176 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.176 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.176 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.295227+0000", 2023-10-09T11:00:56.176 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.052290652999999999 2023-10-09T11:00:56.176 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.176 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.176 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.176 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.295329+0000", 2023-10-09T11:00:56.176 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010212700000000001 2023-10-09T11:00:56.177 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.177 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.177 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:00:56.177 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.295755+0000", 2023-10-09T11:00:56.177 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00042587400000000002 2023-10-09T11:00:56.177 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.177 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.177 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:56.177 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.308570+0000", 2023-10-09T11:00:56.177 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012814397 2023-10-09T11:00:56.177 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.177 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.178 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.178 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.329769+0000", 2023-10-09T11:00:56.178 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021199234000000001 2023-10-09T11:00:56.178 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.178 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.178 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.178 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.335181+0000", 2023-10-09T11:00:56.178 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0054126360000000002 2023-10-09T11:00:56.178 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.178 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.178 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.178 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.335231+0000", 2023-10-09T11:00:56.179 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9344000000000001e-05 2023-10-09T11:00:56.179 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.179 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.179 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.179 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.337605+0000", 2023-10-09T11:00:56.179 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002374555 2023-10-09T11:00:56.179 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.179 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.179 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.179 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.179 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.179 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:96940 7.d 7:bfb000f4:::benchmark_data_smithi125_295816_object5803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T11:00:56.179 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:00.268461+0000", 2023-10-09T11:00:56.180 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 595.83583169799999, 2023-10-09T11:00:56.180 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.071660901999999999, 2023-10-09T11:00:56.180 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.180 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.180 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:56.180 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T11:00:56.180 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T11:00:56.180 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96940 2023-10-09T11:00:56.180 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.180 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.180 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.180 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.181 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268461+0000", 2023-10-09T11:00:56.181 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.181 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.181 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.181 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.181 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268461+0000", 2023-10-09T11:00:56.181 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.181 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.181 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.181 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.181 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268459+0000", 2023-10-09T11:00:56.181 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:56.182 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.182 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.182 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.182 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268477+0000", 2023-10-09T11:00:56.182 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7377999999999999e-05 2023-10-09T11:00:56.182 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.182 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.182 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.182 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268488+0000", 2023-10-09T11:00:56.182 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1579e-05 2023-10-09T11:00:56.182 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.182 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.183 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.183 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.268495+0000", 2023-10-09T11:00:56.183 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5429999999999999e-06 2023-10-09T11:00:56.183 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.183 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.183 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.183 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.273724+0000", 2023-10-09T11:00:56.183 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0052292049999999998 2023-10-09T11:00:56.183 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.183 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.183 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T11:00:56.183 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.274443+0000", 2023-10-09T11:00:56.184 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00071879099999999998 2023-10-09T11:00:56.184 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.184 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.184 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.184 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.306460+0000", 2023-10-09T11:00:56.184 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.032016797999999999 2023-10-09T11:00:56.184 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.184 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.184 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.184 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.306545+0000", 2023-10-09T11:00:56.184 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5277000000000004e-05 2023-10-09T11:00:56.184 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.185 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.185 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:00:56.185 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.306674+0000", 2023-10-09T11:00:56.185 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012941600000000001 2023-10-09T11:00:56.185 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.185 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.185 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:56.185 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.317498+0000", 2023-10-09T11:00:56.185 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010823523999999999 2023-10-09T11:00:56.185 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.185 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.185 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.186 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.329799+0000", 2023-10-09T11:00:56.186 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012301559 2023-10-09T11:00:56.186 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.186 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.186 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.186 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.337661+0000", 2023-10-09T11:00:56.186 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007861303 2023-10-09T11:00:56.186 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.186 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.186 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.186 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.337718+0000", 2023-10-09T11:00:56.186 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6974e-05 2023-10-09T11:00:56.187 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.187 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.187 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.187 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:00.340122+0000", 2023-10-09T11:00:56.187 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024038660000000002 2023-10-09T11:00:56.187 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.187 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.187 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.187 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.187 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.187 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:98648 7.d 7:bbc4eb35:::benchmark_data_smithi125_295816_object7511:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T11:00:56.187 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:02.635379+0000", 2023-10-09T11:00:56.188 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 593.46891378800001, 2023-10-09T11:00:56.188 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.078747929999999994, 2023-10-09T11:00:56.188 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.188 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.188 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:56.188 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T11:00:56.188 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T11:00:56.188 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98648 2023-10-09T11:00:56.188 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.188 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.188 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.188 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.189 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635379+0000", 2023-10-09T11:00:56.189 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.189 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.189 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.189 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.189 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635379+0000", 2023-10-09T11:00:56.189 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.189 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.189 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.189 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.189 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635377+0000", 2023-10-09T11:00:56.189 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:00:56.189 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.190 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.190 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.190 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635382+0000", 2023-10-09T11:00:56.190 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3580000000000002e-06 2023-10-09T11:00:56.190 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.190 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.190 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.190 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635397+0000", 2023-10-09T11:00:56.190 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4265e-05 2023-10-09T11:00:56.190 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.190 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.190 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.191 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.635403+0000", 2023-10-09T11:00:56.191 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4049999999999997e-06 2023-10-09T11:00:56.191 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.191 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.191 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.191 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.654397+0000", 2023-10-09T11:00:56.191 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018993432000000001 2023-10-09T11:00:56.191 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.191 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.191 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T11:00:56.191 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.661791+0000", 2023-10-09T11:00:56.191 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007394655 2023-10-09T11:00:56.192 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.192 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.192 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.192 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.671916+0000", 2023-10-09T11:00:56.192 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010125187000000001 2023-10-09T11:00:56.192 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.192 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.192 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.192 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.671975+0000", 2023-10-09T11:00:56.192 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.897e-05 2023-10-09T11:00:56.192 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.192 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.193 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:00:56.193 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.672065+0000", 2023-10-09T11:00:56.193 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9809999999999996e-05 2023-10-09T11:00:56.193 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.193 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.193 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:56.193 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.682405+0000", 2023-10-09T11:00:56.193 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010339924 2023-10-09T11:00:56.193 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.193 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.193 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.193 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.682443+0000", 2023-10-09T11:00:56.193 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7886000000000003e-05 2023-10-09T11:00:56.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.194 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.194 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.712060+0000", 2023-10-09T11:00:56.194 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.029616844 2023-10-09T11:00:56.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.194 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.194 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.712116+0000", 2023-10-09T11:00:56.194 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6081000000000001e-05 2023-10-09T11:00:56.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.195 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.714127+0000", 2023-10-09T11:00:56.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002010572 2023-10-09T11:00:56.195 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.195 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.195 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.195 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.195 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:98673 7.d 7:b818009b:::benchmark_data_smithi125_295816_object7536:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T11:00:56.195 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:02.680579+0000", 2023-10-09T11:00:56.195 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 593.42371355700004, 2023-10-09T11:00:56.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.095416154000000003, 2023-10-09T11:00:56.196 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.196 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.196 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:56.196 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T11:00:56.196 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T11:00:56.196 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98673 2023-10-09T11:00:56.196 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.196 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.196 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.196 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.196 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680579+0000", 2023-10-09T11:00:56.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.196 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.197 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.197 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680579+0000", 2023-10-09T11:00:56.197 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.197 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.197 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680578+0000", 2023-10-09T11:00:56.197 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:56.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.197 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.198 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680582+0000", 2023-10-09T11:00:56.198 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8e-06 2023-10-09T11:00:56.198 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.198 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.198 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.198 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680588+0000", 2023-10-09T11:00:56.198 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4670000000000001e-06 2023-10-09T11:00:56.198 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.198 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.198 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.198 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.680591+0000", 2023-10-09T11:00:56.198 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9519999999999999e-06 2023-10-09T11:00:56.198 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.199 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.199 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.199 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.727779+0000", 2023-10-09T11:00:56.199 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.047187937999999999 2023-10-09T11:00:56.199 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.199 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.199 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T11:00:56.199 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.728358+0000", 2023-10-09T11:00:56.199 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00057898100000000001 2023-10-09T11:00:56.199 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.199 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.199 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.200 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.740190+0000", 2023-10-09T11:00:56.200 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01183184 2023-10-09T11:00:56.200 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.200 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.200 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.200 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.740303+0000", 2023-10-09T11:00:56.200 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112762 2023-10-09T11:00:56.200 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.200 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.200 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:00:56.200 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.740474+0000", 2023-10-09T11:00:56.200 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017171100000000001 2023-10-09T11:00:56.200 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:56.201 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.746764+0000", 2023-10-09T11:00:56.201 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062892499999999997 2023-10-09T11:00:56.201 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.201 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.756355+0000", 2023-10-09T11:00:56.201 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0095916060000000008 2023-10-09T11:00:56.201 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.774088+0000", 2023-10-09T11:00:56.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017732947999999998 2023-10-09T11:00:56.202 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.202 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.202 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.774137+0000", 2023-10-09T11:00:56.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8616999999999999e-05 2023-10-09T11:00:56.202 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.202 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.202 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.775995+0000", 2023-10-09T11:00:56.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018582340000000001 2023-10-09T11:00:56.203 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.203 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.203 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.203 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.203 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.203 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35187.0:98701 7.d 7:bd16628e:::benchmark_data_smithi125_295816_object7564:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T11:00:56.203 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:51:02.717662+0000", 2023-10-09T11:00:56.203 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 593.38663020599995, 2023-10-09T11:00:56.203 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.079670053000000005, 2023-10-09T11:00:56.203 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.203 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.203 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:00:56.204 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35187", 2023-10-09T11:00:56.204 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4200889085", 2023-10-09T11:00:56.204 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98701 2023-10-09T11:00:56.204 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.204 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.204 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.204 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.204 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717662+0000", 2023-10-09T11:00:56.204 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.204 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.204 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.204 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.204 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717662+0000", 2023-10-09T11:00:56.205 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.205 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.205 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.205 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.205 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717661+0000", 2023-10-09T11:00:56.205 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:56.205 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.205 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.205 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.205 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717666+0000", 2023-10-09T11:00:56.205 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.22e-06 2023-10-09T11:00:56.205 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.206 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.206 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.206 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717676+0000", 2023-10-09T11:00:56.206 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.944e-06 2023-10-09T11:00:56.206 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.206 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.206 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.206 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.717681+0000", 2023-10-09T11:00:56.206 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3719999999999998e-06 2023-10-09T11:00:56.206 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.206 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.206 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.207 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.729691+0000", 2023-10-09T11:00:56.207 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012010140000000001 2023-10-09T11:00:56.207 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.207 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.207 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T11:00:56.207 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.732552+0000", 2023-10-09T11:00:56.207 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028607770000000001 2023-10-09T11:00:56.207 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.207 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.207 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.207 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.771785+0000", 2023-10-09T11:00:56.207 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039233280000000002 2023-10-09T11:00:56.207 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.208 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.208 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.208 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.771847+0000", 2023-10-09T11:00:56.208 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1909000000000006e-05 2023-10-09T11:00:56.208 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.208 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.208 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:00:56.208 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.771940+0000", 2023-10-09T11:00:56.208 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3058000000000002e-05 2023-10-09T11:00:56.208 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.208 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.208 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:00:56.208 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.778867+0000", 2023-10-09T11:00:56.209 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0069268180000000004 2023-10-09T11:00:56.209 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.209 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.209 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.209 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.783491+0000", 2023-10-09T11:00:56.209 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0046242490000000004 2023-10-09T11:00:56.209 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.209 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.209 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:00:56.209 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.795089+0000", 2023-10-09T11:00:56.209 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011598489999999999 2023-10-09T11:00:56.209 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.210 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.210 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.210 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.795150+0000", 2023-10-09T11:00:56.210 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0517999999999999e-05 2023-10-09T11:00:56.210 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.210 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.210 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.210 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:51:02.797332+0000", 2023-10-09T11:00:56.210 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021824079999999998 2023-10-09T11:00:56.210 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.210 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.210 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.210 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.211 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.211 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:156983 9.1 e380/369 9:87937723:::benchmark_data_smithi125_363789_object46684:head v 380'11075, mlcod=380'11072)", 2023-10-09T11:00:56.211 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:52.653549+0000", 2023-10-09T11:00:56.211 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 63.450743629000002, 2023-10-09T11:00:56.211 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.035008312999999999, 2023-10-09T11:00:56.211 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.211 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.211 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.211 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.211 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.211 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.653549+0000", 2023-10-09T11:00:56.211 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.212 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.212 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.212 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.212 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.653549+0000", 2023-10-09T11:00:56.212 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.212 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.212 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.212 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.212 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.653548+0000", 2023-10-09T11:00:56.212 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:56.212 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.212 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.212 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.213 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.653555+0000", 2023-10-09T11:00:56.213 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.537e-06 2023-10-09T11:00:56.213 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.213 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.213 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.213 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.653579+0000", 2023-10-09T11:00:56.213 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3583000000000001e-05 2023-10-09T11:00:56.213 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.213 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.213 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.213 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.653585+0000", 2023-10-09T11:00:56.213 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2090000000000002e-06 2023-10-09T11:00:56.214 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.214 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.214 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.214 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.653683+0000", 2023-10-09T11:00:56.214 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8177999999999994e-05 2023-10-09T11:00:56.214 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.214 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.214 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.214 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.653702+0000", 2023-10-09T11:00:56.214 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8586000000000001e-05 2023-10-09T11:00:56.214 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.214 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.215 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.215 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.688518+0000", 2023-10-09T11:00:56.215 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.034816056999999997 2023-10-09T11:00:56.215 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.215 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.215 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.215 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.688557+0000", 2023-10-09T11:00:56.215 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8980000000000003e-05 2023-10-09T11:00:56.215 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.215 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.215 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.215 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.216 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.216 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:156985 9.1 e380/369 9:86cbc819:::benchmark_data_smithi125_363789_object46686:head v 380'11076, mlcod=380'11072)", 2023-10-09T11:00:56.216 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:52.655651+0000", 2023-10-09T11:00:56.216 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 63.448641828, 2023-10-09T11:00:56.216 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.032932246999999998, 2023-10-09T11:00:56.216 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.216 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.216 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.216 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.216 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.216 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.655651+0000", 2023-10-09T11:00:56.216 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.216 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.217 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.217 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.217 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.655651+0000", 2023-10-09T11:00:56.217 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.217 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.217 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.217 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.217 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.655650+0000", 2023-10-09T11:00:56.217 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:56.217 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.217 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.217 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.218 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.655659+0000", 2023-10-09T11:00:56.218 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8279999999999992e-06 2023-10-09T11:00:56.218 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.218 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.218 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.218 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.655670+0000", 2023-10-09T11:00:56.218 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1579999999999999e-05 2023-10-09T11:00:56.218 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.218 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.218 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.218 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.655676+0000", 2023-10-09T11:00:56.219 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.643e-06 2023-10-09T11:00:56.219 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.219 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.219 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.219 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.655714+0000", 2023-10-09T11:00:56.219 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7922e-05 2023-10-09T11:00:56.219 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.219 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.219 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.219 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.655728+0000", 2023-10-09T11:00:56.219 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4473e-05 2023-10-09T11:00:56.219 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.219 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.220 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.220 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.688563+0000", 2023-10-09T11:00:56.220 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.032834853999999997 2023-10-09T11:00:56.220 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.220 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.220 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.220 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.688583+0000", 2023-10-09T11:00:56.220 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9677000000000002e-05 2023-10-09T11:00:56.220 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.220 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.220 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.220 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.221 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.221 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(osd.3.0:228274 9.1 e380/369 9:8583edf9:::benchmark_data_smithi125_363789_object103527:head v 380'11094, mlcod=380'11079)", 2023-10-09T11:00:56.221 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:52.935494+0000", 2023-10-09T11:00:56.221 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 63.168798215000002, 2023-10-09T11:00:56.221 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.033173022000000003, 2023-10-09T11:00:56.221 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.221 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.221 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.221 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.221 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.221 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.935494+0000", 2023-10-09T11:00:56.221 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.222 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.222 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.222 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.222 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.935494+0000", 2023-10-09T11:00:56.222 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.222 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.222 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.222 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.222 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.935493+0000", 2023-10-09T11:00:56.222 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:56.222 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.222 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.222 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.223 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.935501+0000", 2023-10-09T11:00:56.223 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3969999999999994e-06 2023-10-09T11:00:56.223 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.223 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.223 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.223 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.935511+0000", 2023-10-09T11:00:56.223 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9709999999999999e-06 2023-10-09T11:00:56.223 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.223 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.223 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.223 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.935517+0000", 2023-10-09T11:00:56.223 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3040000000000004e-06 2023-10-09T11:00:56.224 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.224 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.224 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.224 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.943296+0000", 2023-10-09T11:00:56.224 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007778943 2023-10-09T11:00:56.224 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.224 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.224 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.224 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.943313+0000", 2023-10-09T11:00:56.224 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6901e-05 2023-10-09T11:00:56.224 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.224 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.224 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.225 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.968643+0000", 2023-10-09T11:00:56.225 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025330022000000001 2023-10-09T11:00:56.225 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.225 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.225 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.225 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.968667+0000", 2023-10-09T11:00:56.225 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4615999999999999e-05 2023-10-09T11:00:56.225 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.225 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.225 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.225 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.225 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.226 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(osd.3.0:228276 9.1 e380/369 9:85844033:::benchmark_data_smithi125_363789_object59812:head v 380'11095, mlcod=380'11079)", 2023-10-09T11:00:56.226 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:52.935523+0000", 2023-10-09T11:00:56.226 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 63.168769494000003, 2023-10-09T11:00:56.226 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.03315833, 2023-10-09T11:00:56.226 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.226 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.226 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.226 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.226 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.226 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.935523+0000", 2023-10-09T11:00:56.226 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.226 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.227 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.227 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.227 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.935523+0000", 2023-10-09T11:00:56.227 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.227 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.227 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.227 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.227 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.935523+0000", 2023-10-09T11:00:56.227 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:00:56.227 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.227 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.227 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.227 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.935526+0000", 2023-10-09T11:00:56.228 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.76e-06 2023-10-09T11:00:56.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.228 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.228 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.228 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.935532+0000", 2023-10-09T11:00:56.228 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5770000000000001e-06 2023-10-09T11:00:56.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.228 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.228 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.228 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.935535+0000", 2023-10-09T11:00:56.228 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2380000000000002e-06 2023-10-09T11:00:56.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.229 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.945447+0000", 2023-10-09T11:00:56.229 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0099114489999999993 2023-10-09T11:00:56.229 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.229 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.945470+0000", 2023-10-09T11:00:56.229 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.304e-05 2023-10-09T11:00:56.229 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.229 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.968670+0000", 2023-10-09T11:00:56.230 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023200531 2023-10-09T11:00:56.230 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.230 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.230 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.230 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:52.968681+0000", 2023-10-09T11:00:56.230 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0995999999999999e-05 2023-10-09T11:00:56.230 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.230 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.230 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.230 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.230 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.230 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(osd.3.0:228337 9.1 e380/369 9:85979492:::benchmark_data_smithi125_363789_object102902:head v 380'11120, mlcod=380'11108)", 2023-10-09T11:00:56.231 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:53.046455+0000", 2023-10-09T11:00:56.231 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 63.057836989999998, 2023-10-09T11:00:56.231 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.032204440000000001, 2023-10-09T11:00:56.231 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.231 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.231 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.231 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.231 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.231 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.046455+0000", 2023-10-09T11:00:56.231 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.231 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.231 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.232 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.232 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.046455+0000", 2023-10-09T11:00:56.232 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.232 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.232 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.232 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.232 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.046455+0000", 2023-10-09T11:00:56.232 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:56.232 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.232 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.232 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.232 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.046462+0000", 2023-10-09T11:00:56.233 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4320000000000002e-06 2023-10-09T11:00:56.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.233 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.233 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.233 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.046488+0000", 2023-10-09T11:00:56.233 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6278000000000001e-05 2023-10-09T11:00:56.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.233 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.233 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.233 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.046494+0000", 2023-10-09T11:00:56.233 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8579999999999996e-06 2023-10-09T11:00:56.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.234 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.234 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.234 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.051508+0000", 2023-10-09T11:00:56.234 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0050134539999999997 2023-10-09T11:00:56.234 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.234 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.234 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.234 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.051528+0000", 2023-10-09T11:00:56.234 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0774000000000001e-05 2023-10-09T11:00:56.234 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.234 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.234 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.234 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.078624+0000", 2023-10-09T11:00:56.235 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027095332999999999 2023-10-09T11:00:56.235 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.235 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.235 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.235 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.078660+0000", 2023-10-09T11:00:56.235 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6108000000000001e-05 2023-10-09T11:00:56.235 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.235 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.235 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.235 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.235 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.235 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(osd.3.0:228338 9.1 e380/369 9:85983b52:::benchmark_data_smithi125_363789_object74230:head v 380'11121, mlcod=380'11108)", 2023-10-09T11:00:56.236 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:53.048315+0000", 2023-10-09T11:00:56.236 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 63.055977620999997, 2023-10-09T11:00:56.236 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030371321, 2023-10-09T11:00:56.236 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.236 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.236 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.236 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.236 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.236 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.048315+0000", 2023-10-09T11:00:56.236 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.236 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.236 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.236 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.237 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.048315+0000", 2023-10-09T11:00:56.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.237 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.237 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.048314+0000", 2023-10-09T11:00:56.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:56.237 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.237 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.048322+0000", 2023-10-09T11:00:56.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7389999999999999e-06 2023-10-09T11:00:56.238 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.238 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.238 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.048337+0000", 2023-10-09T11:00:56.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5328e-05 2023-10-09T11:00:56.238 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.238 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.238 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.048342+0000", 2023-10-09T11:00:56.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0089999999999998e-06 2023-10-09T11:00:56.238 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.238 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.239 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.239 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.053504+0000", 2023-10-09T11:00:56.239 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.005162374 2023-10-09T11:00:56.239 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.239 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.239 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.239 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.053522+0000", 2023-10-09T11:00:56.239 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8175000000000001e-05 2023-10-09T11:00:56.239 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.239 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.239 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.239 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.078663+0000", 2023-10-09T11:00:56.239 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025140092999999999 2023-10-09T11:00:56.240 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.240 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.240 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.240 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:53.078686+0000", 2023-10-09T11:00:56.240 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3515000000000001e-05 2023-10-09T11:00:56.240 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.240 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.240 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.240 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.240 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.240 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:158198 9.b e382/376 9:d74ff65a:::benchmark_data_smithi125_363789_object47899:head v 382'9809, mlcod=382'9804)", 2023-10-09T11:00:56.240 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:55.230160+0000", 2023-10-09T11:00:56.240 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 60.874131982000002, 2023-10-09T11:00:56.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030607599999999999, 2023-10-09T11:00:56.241 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:00:56.241 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:00:56.241 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:00:56.241 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.241 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:00:56.241 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:55.230160+0000", 2023-10-09T11:00:56.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.241 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.241 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.241 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:00:56.241 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:55.230160+0000", 2023-10-09T11:00:56.242 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:00:56.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.242 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.242 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:00:56.242 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:55.230160+0000", 2023-10-09T11:00:56.242 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:00:56.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.242 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.242 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:00:56.242 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:55.230167+0000", 2023-10-09T11:00:56.242 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5989999999999996e-06 2023-10-09T11:00:56.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.243 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:00:56.243 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:55.230178+0000", 2023-10-09T11:00:56.243 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1001e-05 2023-10-09T11:00:56.243 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.243 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:00:56.243 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:55.230185+0000", 2023-10-09T11:00:56.243 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3910000000000003e-06 2023-10-09T11:00:56.243 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.243 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:00:56.243 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:55.230228+0000", 2023-10-09T11:00:56.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3859000000000001e-05 2023-10-09T11:00:56.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.244 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.244 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:00:56.244 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:55.230251+0000", 2023-10-09T11:00:56.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3093999999999999e-05 2023-10-09T11:00:56.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.244 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.244 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:00:56.244 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:55.260734+0000", 2023-10-09T11:00:56.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030482065999999999 2023-10-09T11:00:56.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:00:56.245 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:00:56.245 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:00:56.245 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:55.260768+0000", 2023-10-09T11:00:56.245 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4483000000000002e-05 2023-10-09T11:00:56.245 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.245 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.245 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.245 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:00:56.245 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:00:56.245 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:00:56.246 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:00:56.437 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:56 smithi155 ceph-mon[184854]: pgmap v988: 129 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 563 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 30581/27072 objects degraded (112.962%); 0 B/s, 30 objects/s recovering 2023-10-09T11:00:56.437 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/552112883' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:56.437 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:56 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:56.531 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T11:00:56.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:56 smithi125 ceph-mon[195507]: pgmap v988: 129 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 563 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 30581/27072 objects degraded (112.962%); 0 B/s, 30 objects/s recovering 2023-10-09T11:00:56.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/552112883' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:56.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:56 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:56 smithi125 ceph-mon[200197]: pgmap v988: 129 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 563 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 30581/27072 objects degraded (112.962%); 0 B/s, 30 objects/s recovering 2023-10-09T11:00:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/552112883' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:56 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:00:56.927 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:00:57.036 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:00:57.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:57.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:00:57.083 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:00:57.143 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:00:57.214 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:57.291 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:00:57.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:57 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:00:57.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:57 smithi125 ceph-mon[195507]: failed to encode map e397 with expected crc 2023-10-09T11:00:57.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:57 smithi125 ceph-mon[195507]: failed to encode map e397 with expected crc 2023-10-09T11:00:57.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:57 smithi125 ceph-mon[195507]: failed to encode map e397 with expected crc 2023-10-09T11:00:57.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:57 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:00:57.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:57 smithi125 ceph-mon[195507]: osdmap e397: 8 total, 8 up, 8 in 2023-10-09T11:00:57.292 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:57 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:00:57.293 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:57 smithi125 ceph-mon[200197]: failed to encode map e397 with expected crc 2023-10-09T11:00:57.293 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:57 smithi125 ceph-mon[200197]: failed to encode map e397 with expected crc 2023-10-09T11:00:57.293 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:57 smithi125 ceph-mon[200197]: failed to encode map e397 with expected crc 2023-10-09T11:00:57.293 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:57 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:00:57.293 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:57 smithi125 ceph-mon[200197]: osdmap e397: 8 total, 8 up, 8 in 2023-10-09T11:00:57.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:57.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:00:57.399 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:00:57.433 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:57 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:00:57.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:57 smithi155 ceph-mon[184854]: failed to encode map e397 with expected crc 2023-10-09T11:00:57.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:57 smithi155 ceph-mon[184854]: failed to encode map e397 with expected crc 2023-10-09T11:00:57.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:57 smithi155 ceph-mon[184854]: failed to encode map e397 with expected crc 2023-10-09T11:00:57.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:57 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:00:57.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:57 smithi155 ceph-mon[184854]: osdmap e397: 8 total, 8 up, 8 in 2023-10-09T11:00:57.437 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:57.645 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:00:57.713 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:00:57.714 INFO:teuthology.orchestra.run.smithi125.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T11:00:57.714 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:00:58.420 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:58 smithi125 ceph-mon[200197]: 9.9 scrub starts 2023-10-09T11:00:58.420 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:58 smithi125 ceph-mon[200197]: pgmap v990: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 12 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 356 MiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 30581/17151 objects degraded (178.304%); 0 B/s, 53 objects/s recovering 2023-10-09T11:00:58.421 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:58 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30581/27072 objects degraded (112.962%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:58.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:58 smithi125 ceph-mon[195507]: 9.9 scrub starts 2023-10-09T11:00:58.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:58 smithi125 ceph-mon[195507]: pgmap v990: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 12 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 356 MiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 30581/17151 objects degraded (178.304%); 0 B/s, 53 objects/s recovering 2023-10-09T11:00:58.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:58 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30581/27072 objects degraded (112.962%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:58.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:00:58.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:58 smithi155 ceph-mon[184854]: 9.9 scrub starts 2023-10-09T11:00:58.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:58 smithi155 ceph-mon[184854]: pgmap v990: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 12 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 356 MiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 30581/17151 objects degraded (178.304%); 0 B/s, 53 objects/s recovering 2023-10-09T11:00:58.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:58 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30581/27072 objects degraded (112.962%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:00:58.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:58.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:00:58.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:58.741 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:59.001 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:00:59.038 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:00:59.072 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:00:59.121 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:00:59.147 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:00:59.180 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:00:59.213 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:00:59.244 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:00:59.253 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:00:59.253 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:00:59.253 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:00:59.254 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:00:59.254 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:00:59.254 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:00:59.254 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:00:59.254 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:00:59.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:00:59.370 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:59.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:59.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:59.474 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:00:59.525 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:00:59 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4163350939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:59.528 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:59.557 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:00:59.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:59.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:00:59.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:00:59 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4163350939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:59.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:00:59 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4163350939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:00:59.634 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:00:59.635 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:00:59.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:00:59.700 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:00:59.782 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:00:59.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:00:59.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:00:59.889 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:00:59.953 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:00 smithi155 ceph-mon[184854]: pgmap v991: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 309 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 30581/14901 objects degraded (205.228%); 0 B/s, 51 objects/s recovering 2023-10-09T11:01:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:01:00.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:00 smithi125 ceph-mon[195507]: pgmap v991: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 309 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 30581/14901 objects degraded (205.228%); 0 B/s, 51 objects/s recovering 2023-10-09T11:01:00.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:00.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:01:00.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:00 smithi125 ceph-mon[200197]: pgmap v991: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 309 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 30581/14901 objects degraded (205.228%); 0 B/s, 51 objects/s recovering 2023-10-09T11:01:00.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:00.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:01:00.661 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:00.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:00.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:00.731 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:00.775 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:00.838 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:00.877 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:00.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:00.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:00.983 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:01.029 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:01.108 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:01.110 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T11:01:01.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:01.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:01.212 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:01.222 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:01:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:01:01] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.43.0" 2023-10-09T11:01:01.281 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:01.418 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:01.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:01.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:01.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:01 smithi155 ceph-mon[184854]: 9.15 scrub starts 2023-10-09T11:01:01.504 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:01:01.527 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:01.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:01 smithi125 ceph-mon[195507]: 9.15 scrub starts 2023-10-09T11:01:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:01 smithi125 ceph-mon[200197]: 9.15 scrub starts 2023-10-09T11:01:01.590 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:02.037 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:02.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:02.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:02.143 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:02.219 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:02.259 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:02.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:02.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:02.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:02 smithi155 ceph-mon[184854]: pgmap v992: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 113 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 30581/5490 objects degraded (557.031%); 0 B/s, 57 objects/s recovering 2023-10-09T11:01:02.367 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:02.421 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:02.478 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:02.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:02.482 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:02 smithi125 ceph-mon[195507]: pgmap v992: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 113 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 30581/5490 objects degraded (557.031%); 0 B/s, 57 objects/s recovering 2023-10-09T11:01:02.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:02.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:02 smithi125 ceph-mon[200197]: pgmap v992: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 113 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 30581/5490 objects degraded (557.031%); 0 B/s, 57 objects/s recovering 2023-10-09T11:01:02.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:02.635 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:02.671 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:01:02.683 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:02.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:02.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:02.769 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Removed 110296 objects 2023-10-09T11:01:02.769 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Clean up completed and total clean up time :128.081 2023-10-09T11:01:02.789 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:02.819 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:02.840 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:02.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:02.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:02.943 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:02.973 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:03.042 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:03.191 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T11:01:03.191 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:01:03.191 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T11:01:03.191 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T11:01:03.191 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T11:01:03.192 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T11:01:03.213 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:03.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:03.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:03.238 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:01:03.238 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:01:03.293 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:01:03.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:03.347 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:03 smithi155 ceph-mon[184854]: 1.17 scrub starts 2023-10-09T11:01:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2975672259' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:01:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:03 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:01:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:03 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30581/5490 objects degraded (557.031%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:01:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:03 smithi125 ceph-mon[195507]: 1.17 scrub starts 2023-10-09T11:01:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2975672259' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:01:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:03 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:01:03.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:03 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30581/5490 objects degraded (557.031%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:01:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:03 smithi125 ceph-mon[200197]: 1.17 scrub starts 2023-10-09T11:01:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2975672259' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:01:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:03 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:01:03.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:03 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30581/5490 objects degraded (557.031%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:01:03.804 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T11:01:03.829 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:01:03.920 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:03.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:03.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:04.024 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:04.053 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:04.140 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:04.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:04.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:04.168 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-10-09T11:01:04.168 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-10-09T11:01:04.245 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:04.270 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:01:04.277 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:04.362 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:04.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:04.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:04.466 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:04 smithi155 ceph-mon[184854]: 1.6 scrub starts 2023-10-09T11:01:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:04 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:04 smithi155 ceph-mon[184854]: failed to encode map e398 with expected crc 2023-10-09T11:01:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:04 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:01:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:04 smithi155 ceph-mon[184854]: osdmap e398: 8 total, 8 up, 8 in 2023-10-09T11:01:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:04 smithi155 ceph-mon[184854]: failed to encode map e398 with expected crc 2023-10-09T11:01:04.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:04 smithi155 ceph-mon[184854]: pgmap v994: 129 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 71 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 986 op/s; 30581/3468 objects degraded (881.805%); 0 B/s, 71 objects/s recovering 2023-10-09T11:01:04.494 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:04.514 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:04 smithi125 ceph-mon[195507]: 1.6 scrub starts 2023-10-09T11:01:04.515 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:04 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:04.515 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:04 smithi125 ceph-mon[195507]: failed to encode map e398 with expected crc 2023-10-09T11:01:04.515 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:04 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:01:04.515 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:04 smithi125 ceph-mon[195507]: osdmap e398: 8 total, 8 up, 8 in 2023-10-09T11:01:04.515 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:04 smithi125 ceph-mon[195507]: failed to encode map e398 with expected crc 2023-10-09T11:01:04.515 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:04 smithi125 ceph-mon[195507]: pgmap v994: 129 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 71 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 986 op/s; 30581/3468 objects degraded (881.805%); 0 B/s, 71 objects/s recovering 2023-10-09T11:01:04.515 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:04 smithi125 ceph-mon[200197]: 1.6 scrub starts 2023-10-09T11:01:04.515 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:04 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:04.516 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:04 smithi125 ceph-mon[200197]: failed to encode map e398 with expected crc 2023-10-09T11:01:04.516 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:04 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:01:04.516 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:04 smithi125 ceph-mon[200197]: osdmap e398: 8 total, 8 up, 8 in 2023-10-09T11:01:04.516 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:04 smithi125 ceph-mon[200197]: failed to encode map e398 with expected crc 2023-10-09T11:01:04.516 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:04 smithi125 ceph-mon[200197]: pgmap v994: 129 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 71 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 986 op/s; 30581/3468 objects degraded (881.805%); 0 B/s, 71 objects/s recovering 2023-10-09T11:01:04.516 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:04.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:04.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:04.590 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:04.621 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:04.650 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:04.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:04.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:04.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:04.831 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:04.866 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:04.898 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:04.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:04.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:04.918 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:01:04.919 INFO:teuthology.orchestra.run.smithi155.stdout: "success": "" 2023-10-09T11:01:04.919 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:01:05.002 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:05.034 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:05.154 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:05.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:05.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:05.257 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:05.286 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:05.349 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:05.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:05.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:05.453 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:05.482 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:05.522 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:05 smithi155 ceph-mon[184854]: failed to encode map e398 with expected crc 2023-10-09T11:01:05.522 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:05 smithi155 ceph-mon[184854]: failed to encode map e398 with expected crc 2023-10-09T11:01:05.522 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:05 smithi155 ceph-mon[184854]: failed to encode map e398 with expected crc 2023-10-09T11:01:05.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:05 smithi125 ceph-mon[200197]: failed to encode map e398 with expected crc 2023-10-09T11:01:05.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:05 smithi125 ceph-mon[200197]: failed to encode map e398 with expected crc 2023-10-09T11:01:05.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:05 smithi125 ceph-mon[200197]: failed to encode map e398 with expected crc 2023-10-09T11:01:05.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:05 smithi125 ceph-mon[195507]: failed to encode map e398 with expected crc 2023-10-09T11:01:05.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:05 smithi125 ceph-mon[195507]: failed to encode map e398 with expected crc 2023-10-09T11:01:05.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:05 smithi125 ceph-mon[195507]: failed to encode map e398 with expected crc 2023-10-09T11:01:05.567 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:05.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:05.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:05.687 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:05.698 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:01:05.698 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T11:01:05.698 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T11:01:05.698 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:01:05.698 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.698 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122023 9.d 9:b63f6ab3:::benchmark_data_smithi125_363789_object11724:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:01:05.699 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.154825+0000", 2023-10-09T11:01:05.699 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 117.54088952399999, 2023-10-09T11:01:05.699 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0345569450000001, 2023-10-09T11:01:05.699 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.699 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:01:05.699 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.699 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.699 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.699 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122023 2023-10-09T11:01:05.699 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.699 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.699 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.699 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154825+0000", 2023-10-09T11:01:05.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.700 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.700 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154825+0000", 2023-10-09T11:01:05.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.700 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.700 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154823+0000", 2023-10-09T11:01:05.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:01:05.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.701 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.701 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.701 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154829+0000", 2023-10-09T11:01:05.701 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6269999999999997e-06 2023-10-09T11:01:05.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.701 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.701 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.701 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154842+0000", 2023-10-09T11:01:05.701 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3869999999999999e-05 2023-10-09T11:01:05.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.701 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154849+0000", 2023-10-09T11:01:05.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5599999999999999e-06 2023-10-09T11:01:05.702 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154882+0000", 2023-10-09T11:01:05.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2580999999999999e-05 2023-10-09T11:01:05.702 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154977+0000", 2023-10-09T11:01:05.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5298999999999997e-05 2023-10-09T11:01:05.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,5", 2023-10-09T11:01:05.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.155058+0000", 2023-10-09T11:01:05.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1518000000000001e-05 2023-10-09T11:01:05.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.161278+0000", 2023-10-09T11:01:05.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062194049999999999 2023-10-09T11:01:05.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.704 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.704 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.704 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.162677+0000", 2023-10-09T11:01:05.704 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0013996379999999999 2023-10-09T11:01:05.704 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.704 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.704 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.704 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.180908+0000", 2023-10-09T11:01:05.704 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018230321000000001 2023-10-09T11:01:05.704 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.704 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.704 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.704 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.180916+0000", 2023-10-09T11:01:05.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7170000000000008e-06 2023-10-09T11:01:05.705 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.705 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.705 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.189335+0000", 2023-10-09T11:01:05.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.00841958 2023-10-09T11:01:05.705 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.705 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.705 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.189381+0000", 2023-10-09T11:01:05.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6349e-05 2023-10-09T11:01:05.705 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.706 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.706 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.706 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.706 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122031 9.3 9:cb1920df:::benchmark_data_smithi125_363789_object11732:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:01:05.706 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.154885+0000", 2023-10-09T11:01:05.706 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 117.540828625, 2023-10-09T11:01:05.706 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0272100049999999, 2023-10-09T11:01:05.706 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.706 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:01:05.706 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.706 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.707 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.707 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122031 2023-10-09T11:01:05.707 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.707 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.707 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.707 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.707 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154885+0000", 2023-10-09T11:01:05.707 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.707 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.707 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.707 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.707 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154885+0000", 2023-10-09T11:01:05.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.708 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.708 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154885+0000", 2023-10-09T11:01:05.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:01:05.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.708 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.708 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154888+0000", 2023-10-09T11:01:05.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6879999999999999e-06 2023-10-09T11:01:05.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.709 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154894+0000", 2023-10-09T11:01:05.709 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.823e-06 2023-10-09T11:01:05.709 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.709 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154897+0000", 2023-10-09T11:01:05.709 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8659999999999999e-06 2023-10-09T11:01:05.709 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.709 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154941+0000", 2023-10-09T11:01:05.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3460000000000001e-05 2023-10-09T11:01:05.710 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.710 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.710 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.710 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.155049+0000", 2023-10-09T11:01:05.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000107932 2023-10-09T11:01:05.710 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.710 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.710 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:01:05.710 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.155156+0000", 2023-10-09T11:01:05.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010671300000000001 2023-10-09T11:01:05.710 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.711 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.711 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.711 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.161290+0000", 2023-10-09T11:01:05.711 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061342480000000001 2023-10-09T11:01:05.711 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.711 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.711 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.711 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178638+0000", 2023-10-09T11:01:05.711 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017348484000000001 2023-10-09T11:01:05.711 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.711 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.711 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.711 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178645+0000", 2023-10-09T11:01:05.712 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.613e-06 2023-10-09T11:01:05.712 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.712 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.712 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.182021+0000", 2023-10-09T11:01:05.712 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.003375959 2023-10-09T11:01:05.712 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.712 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.712 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.182095+0000", 2023-10-09T11:01:05.712 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4606999999999999e-05 2023-10-09T11:01:05.712 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.712 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.713 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.713 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.713 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.713 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122032 9.a 9:5ee579a3:::benchmark_data_smithi125_363789_object11733:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:01:05.713 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.163592+0000", 2023-10-09T11:01:05.713 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 117.532122034, 2023-10-09T11:01:05.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.03044172, 2023-10-09T11:01:05.713 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.713 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:01:05.713 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.713 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.713 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.714 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122032 2023-10-09T11:01:05.714 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.714 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.714 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.714 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.714 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163592+0000", 2023-10-09T11:01:05.714 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.714 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.714 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.714 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.714 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163592+0000", 2023-10-09T11:01:05.714 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.715 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163590+0000", 2023-10-09T11:01:05.715 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T11:01:05.715 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163601+0000", 2023-10-09T11:01:05.715 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1037e-05 2023-10-09T11:01:05.715 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163614+0000", 2023-10-09T11:01:05.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3001e-05 2023-10-09T11:01:05.716 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.716 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163621+0000", 2023-10-09T11:01:05.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7270000000000001e-06 2023-10-09T11:01:05.716 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.716 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.164646+0000", 2023-10-09T11:01:05.717 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010251909999999999 2023-10-09T11:01:05.717 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.717 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.717 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op must wait for map", 2023-10-09T11:01:05.717 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.164654+0000", 2023-10-09T11:01:05.717 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5059999999999998e-06 2023-10-09T11:01:05.717 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.717 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.717 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.717 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178523+0000", 2023-10-09T11:01:05.717 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013869365999999999 2023-10-09T11:01:05.717 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.717 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.718 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.718 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178531+0000", 2023-10-09T11:01:05.718 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7510000000000008e-06 2023-10-09T11:01:05.718 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.718 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.718 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.718 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.192721+0000", 2023-10-09T11:01:05.718 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.014189995 2023-10-09T11:01:05.718 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.718 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.718 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.718 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.194034+0000", 2023-10-09T11:01:05.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001312917 2023-10-09T11:01:05.719 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.719 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.719 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.719 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.719 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122035 9.3 9:cf84482a:::benchmark_data_smithi125_363789_object11736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:01:05.719 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.176778+0000", 2023-10-09T11:01:05.719 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 117.51893583099999, 2023-10-09T11:01:05.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0230526680000001, 2023-10-09T11:01:05.719 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.719 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.720 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.720 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.720 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.720 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122035 2023-10-09T11:01:05.720 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.720 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.720 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176778+0000", 2023-10-09T11:01:05.720 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.720 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.721 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.721 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176778+0000", 2023-10-09T11:01:05.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.721 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.721 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.721 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.721 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176777+0000", 2023-10-09T11:01:05.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:01:05.721 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.721 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.721 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.721 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176782+0000", 2023-10-09T11:01:05.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.374e-06 2023-10-09T11:01:05.722 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.722 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.722 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.722 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176790+0000", 2023-10-09T11:01:05.722 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2199999999999992e-06 2023-10-09T11:01:05.722 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.722 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.722 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.722 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176795+0000", 2023-10-09T11:01:05.722 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5099999999999998e-06 2023-10-09T11:01:05.722 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.723 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.723 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178896+0000", 2023-10-09T11:01:05.723 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021008910000000001 2023-10-09T11:01:05.723 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.723 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.723 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178902+0000", 2023-10-09T11:01:05.723 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0800000000000002e-06 2023-10-09T11:01:05.723 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.723 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.723 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.182122+0000", 2023-10-09T11:01:05.724 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.003219329 2023-10-09T11:01:05.724 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.724 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.724 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.724 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.182244+0000", 2023-10-09T11:01:05.724 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012187 2023-10-09T11:01:05.724 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.724 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.724 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3", 2023-10-09T11:01:05.724 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.182386+0000", 2023-10-09T11:01:05.724 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000142373 2023-10-09T11:01:05.724 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.725 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.725 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.725 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.190416+0000", 2023-10-09T11:01:05.725 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0080298230000000002 2023-10-09T11:01:05.725 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.725 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.725 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.725 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199736+0000", 2023-10-09T11:01:05.725 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0093200999999999996 2023-10-09T11:01:05.725 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.725 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.725 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.725 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199767+0000", 2023-10-09T11:01:05.726 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1440999999999999e-05 2023-10-09T11:01:05.726 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.726 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.726 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.726 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199831+0000", 2023-10-09T11:01:05.726 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3633999999999994e-05 2023-10-09T11:01:05.726 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.726 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.726 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.726 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.726 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.727 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122040 9.3 9:cc3a16b1:::benchmark_data_smithi125_363789_object11741:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:01:05.727 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.187372+0000", 2023-10-09T11:01:05.727 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 117.508341941, 2023-10-09T11:01:05.727 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.012574294, 2023-10-09T11:01:05.727 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.727 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.727 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.727 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.727 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.727 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122040 2023-10-09T11:01:05.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.727 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.727 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187372+0000", 2023-10-09T11:01:05.728 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.728 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.728 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187372+0000", 2023-10-09T11:01:05.728 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.728 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.728 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187371+0000", 2023-10-09T11:01:05.728 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:01:05.729 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.729 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.729 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.729 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187375+0000", 2023-10-09T11:01:05.729 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6299999999999997e-06 2023-10-09T11:01:05.729 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.729 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.729 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.729 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187385+0000", 2023-10-09T11:01:05.729 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9709999999999999e-06 2023-10-09T11:01:05.729 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.730 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.730 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187392+0000", 2023-10-09T11:01:05.730 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7299999999999999e-06 2023-10-09T11:01:05.730 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.730 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.730 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187440+0000", 2023-10-09T11:01:05.730 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7558000000000001e-05 2023-10-09T11:01:05.730 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.731 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.731 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187449+0000", 2023-10-09T11:01:05.731 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3740000000000004e-06 2023-10-09T11:01:05.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.731 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.731 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.731 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.185252+0000", 2023-10-09T11:01:05.731 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.99780342499999997 2023-10-09T11:01:05.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.731 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.731 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.731 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.185378+0000", 2023-10-09T11:01:05.732 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012527 2023-10-09T11:01:05.732 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3", 2023-10-09T11:01:05.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.185510+0000", 2023-10-09T11:01:05.732 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013254599999999999 2023-10-09T11:01:05.732 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.190604+0000", 2023-10-09T11:01:05.732 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0050936460000000003 2023-10-09T11:01:05.732 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.733 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.733 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.733 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199865+0000", 2023-10-09T11:01:05.733 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0092616620000000004 2023-10-09T11:01:05.733 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.733 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.733 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.733 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199897+0000", 2023-10-09T11:01:05.733 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1223000000000002e-05 2023-10-09T11:01:05.733 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.733 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.733 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.734 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199946+0000", 2023-10-09T11:01:05.734 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9728999999999998e-05 2023-10-09T11:01:05.734 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.734 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.734 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.734 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.734 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.734 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122046 9.3 9:cc593156:::benchmark_data_smithi125_363789_object11747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:01:05.734 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.201649+0000", 2023-10-09T11:01:05.734 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 117.49406549299999, 2023-10-09T11:01:05.734 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.010992307, 2023-10-09T11:01:05.735 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.735 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.735 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.735 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.735 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.735 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122046 2023-10-09T11:01:05.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.735 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.735 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.735 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201649+0000", 2023-10-09T11:01:05.735 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.736 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.736 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201649+0000", 2023-10-09T11:01:05.736 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.736 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.736 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.736 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201647+0000", 2023-10-09T11:01:05.736 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:01:05.736 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.736 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.737 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201653+0000", 2023-10-09T11:01:05.737 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1840000000000004e-06 2023-10-09T11:01:05.737 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.737 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.737 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.737 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201665+0000", 2023-10-09T11:01:05.737 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1396999999999999e-05 2023-10-09T11:01:05.737 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.737 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.737 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.737 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201671+0000", 2023-10-09T11:01:05.737 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3849999999999999e-06 2023-10-09T11:01:05.738 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.738 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.738 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.738 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201716+0000", 2023-10-09T11:01:05.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4969999999999998e-05 2023-10-09T11:01:05.738 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.738 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.738 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.738 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201725+0000", 2023-10-09T11:01:05.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3589999999999997e-06 2023-10-09T11:01:05.738 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.738 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.739 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.739 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.187839+0000", 2023-10-09T11:01:05.739 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98611354799999995 2023-10-09T11:01:05.739 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.739 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.739 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.739 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.187980+0000", 2023-10-09T11:01:05.739 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014071799999999999 2023-10-09T11:01:05.739 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.739 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.739 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3", 2023-10-09T11:01:05.739 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.188118+0000", 2023-10-09T11:01:05.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013808599999999999 2023-10-09T11:01:05.740 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.740 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.740 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.740 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.190608+0000", 2023-10-09T11:01:05.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00249037 2023-10-09T11:01:05.740 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.740 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.740 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.740 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.212484+0000", 2023-10-09T11:01:05.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021876442999999999 2023-10-09T11:01:05.740 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.740 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.741 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.741 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.212541+0000", 2023-10-09T11:01:05.741 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6255999999999998e-05 2023-10-09T11:01:05.741 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.741 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.741 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.741 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.212641+0000", 2023-10-09T11:01:05.741 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010011200000000001 2023-10-09T11:01:05.741 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.741 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.741 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.741 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.741 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.742 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150423 9.9 9:91f76946:::benchmark_data_smithi125_363789_object40124:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:01:05.742 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.193246+0000", 2023-10-09T11:01:05.742 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 82.502468034000003, 2023-10-09T11:01:05.742 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93784610000000002, 2023-10-09T11:01:05.742 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.742 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.742 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.742 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.742 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.742 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150423 2023-10-09T11:01:05.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.742 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.743 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.743 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.743 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193246+0000", 2023-10-09T11:01:05.743 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.743 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.743 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.743 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.743 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193246+0000", 2023-10-09T11:01:05.743 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.743 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.743 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.743 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.744 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193245+0000", 2023-10-09T11:01:05.744 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:01:05.744 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.744 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.744 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.744 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193249+0000", 2023-10-09T11:01:05.744 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2279999999999998e-06 2023-10-09T11:01:05.744 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.744 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.744 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.744 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193256+0000", 2023-10-09T11:01:05.744 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1010000000000003e-06 2023-10-09T11:01:05.744 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.745 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193260+0000", 2023-10-09T11:01:05.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3409999999999996e-06 2023-10-09T11:01:05.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.745 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193306+0000", 2023-10-09T11:01:05.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5636000000000001e-05 2023-10-09T11:01:05.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193313+0000", 2023-10-09T11:01:05.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0849999999999999e-06 2023-10-09T11:01:05.746 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.746 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.121393+0000", 2023-10-09T11:01:05.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92808017799999998 2023-10-09T11:01:05.746 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.746 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.121556+0000", 2023-10-09T11:01:05.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016344600000000001 2023-10-09T11:01:05.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.747 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:01:05.747 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.121862+0000", 2023-10-09T11:01:05.747 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030565099999999999 2023-10-09T11:01:05.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.747 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.747 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.126975+0000", 2023-10-09T11:01:05.747 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051129119999999998 2023-10-09T11:01:05.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.748 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.130983+0000", 2023-10-09T11:01:05.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040081470000000001 2023-10-09T11:01:05.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.748 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.748 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131014+0000", 2023-10-09T11:01:05.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0698000000000003e-05 2023-10-09T11:01:05.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.748 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.748 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131055+0000", 2023-10-09T11:01:05.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1390999999999997e-05 2023-10-09T11:01:05.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131092+0000", 2023-10-09T11:01:05.749 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6896e-05 2023-10-09T11:01:05.749 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.749 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.749 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.749 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150426 9.9 9:911979c4:::benchmark_data_smithi125_363789_object40127:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:01:05.749 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.193290+0000", 2023-10-09T11:01:05.749 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 82.502424523000002, 2023-10-09T11:01:05.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93976706099999996, 2023-10-09T11:01:05.750 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.750 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.750 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.750 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.750 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.750 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150426 2023-10-09T11:01:05.750 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.750 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.750 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.750 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.750 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193290+0000", 2023-10-09T11:01:05.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193290+0000", 2023-10-09T11:01:05.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193289+0000", 2023-10-09T11:01:05.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:01:05.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193291+0000", 2023-10-09T11:01:05.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2649999999999999e-06 2023-10-09T11:01:05.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193297+0000", 2023-10-09T11:01:05.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5210000000000002e-06 2023-10-09T11:01:05.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.754 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193301+0000", 2023-10-09T11:01:05.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7730000000000001e-06 2023-10-09T11:01:05.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.754 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193345+0000", 2023-10-09T11:01:05.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3959999999999999e-05 2023-10-09T11:01:05.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.755 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.755 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193350+0000", 2023-10-09T11:01:05.755 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9949999999999996e-06 2023-10-09T11:01:05.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.755 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.755 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.123175+0000", 2023-10-09T11:01:05.755 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92982503900000002 2023-10-09T11:01:05.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.755 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.123286+0000", 2023-10-09T11:01:05.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000111036 2023-10-09T11:01:05.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:01:05.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.123498+0000", 2023-10-09T11:01:05.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00021212800000000001 2023-10-09T11:01:05.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131127+0000", 2023-10-09T11:01:05.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0076286820000000003 2023-10-09T11:01:05.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.757 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.757 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131182+0000", 2023-10-09T11:01:05.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5912999999999998e-05 2023-10-09T11:01:05.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.757 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.757 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.132971+0000", 2023-10-09T11:01:05.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001788191 2023-10-09T11:01:05.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.757 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133015+0000", 2023-10-09T11:01:05.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4014000000000002e-05 2023-10-09T11:01:05.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133057+0000", 2023-10-09T11:01:05.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.193e-05 2023-10-09T11:01:05.758 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.758 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.758 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.759 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150439 9.9 9:92459488:::benchmark_data_smithi125_363789_object40140:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:01:05.759 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.207668+0000", 2023-10-09T11:01:05.759 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 82.488046413999996, 2023-10-09T11:01:05.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92585638599999998, 2023-10-09T11:01:05.759 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.759 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.759 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.759 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.759 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.759 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150439 2023-10-09T11:01:05.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.759 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.760 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.760 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.760 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207668+0000", 2023-10-09T11:01:05.760 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.760 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.760 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.760 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.760 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207668+0000", 2023-10-09T11:01:05.760 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.760 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.760 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.760 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.760 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207667+0000", 2023-10-09T11:01:05.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:01:05.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.761 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.761 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.761 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207671+0000", 2023-10-09T11:01:05.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2339999999999996e-06 2023-10-09T11:01:05.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.761 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.761 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.761 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207680+0000", 2023-10-09T11:01:05.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1670000000000005e-06 2023-10-09T11:01:05.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.761 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207686+0000", 2023-10-09T11:01:05.762 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3720000000000004e-06 2023-10-09T11:01:05.762 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.208495+0000", 2023-10-09T11:01:05.762 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000809142 2023-10-09T11:01:05.762 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.763 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.208502+0000", 2023-10-09T11:01:05.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9210000000000003e-06 2023-10-09T11:01:05.763 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.763 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.763 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.763 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.124568+0000", 2023-10-09T11:01:05.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.91606632200000004 2023-10-09T11:01:05.763 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.763 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.763 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.763 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.124637+0000", 2023-10-09T11:01:05.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8700000000000003e-05 2023-10-09T11:01:05.763 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.764 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.764 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:01:05.764 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.124777+0000", 2023-10-09T11:01:05.764 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013975399999999999 2023-10-09T11:01:05.764 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.764 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.764 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.764 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131155+0000", 2023-10-09T11:01:05.764 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0063785200000000004 2023-10-09T11:01:05.764 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.764 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.765 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.765 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133065+0000", 2023-10-09T11:01:05.765 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019096849999999999 2023-10-09T11:01:05.765 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.765 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.765 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.765 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133440+0000", 2023-10-09T11:01:05.765 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00037491400000000001 2023-10-09T11:01:05.765 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.765 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.765 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.765 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133486+0000", 2023-10-09T11:01:05.766 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6683e-05 2023-10-09T11:01:05.766 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.766 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.766 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.766 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133524+0000", 2023-10-09T11:01:05.766 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7685000000000003e-05 2023-10-09T11:01:05.766 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.766 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.766 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.766 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.766 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.766 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150470 9.9 9:9415b14b:::benchmark_data_smithi125_363789_object40171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:01:05.767 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.229566+0000", 2023-10-09T11:01:05.767 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 82.466148011000001, 2023-10-09T11:01:05.767 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.90407114499999996, 2023-10-09T11:01:05.767 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.767 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.767 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.767 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.767 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.767 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150470 2023-10-09T11:01:05.767 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.767 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.767 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.767 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229566+0000", 2023-10-09T11:01:05.768 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.768 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.768 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229566+0000", 2023-10-09T11:01:05.768 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.768 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.768 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229566+0000", 2023-10-09T11:01:05.768 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:01:05.768 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.769 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.769 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.769 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229568+0000", 2023-10-09T11:01:05.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.943e-06 2023-10-09T11:01:05.769 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.769 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.769 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.769 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229574+0000", 2023-10-09T11:01:05.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.003e-06 2023-10-09T11:01:05.769 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.769 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.769 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.769 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229579+0000", 2023-10-09T11:01:05.770 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0790000000000002e-06 2023-10-09T11:01:05.770 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.770 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.770 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.770 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229656+0000", 2023-10-09T11:01:05.770 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6940999999999994e-05 2023-10-09T11:01:05.770 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.770 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.770 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.770 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229662+0000", 2023-10-09T11:01:05.770 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7020000000000004e-06 2023-10-09T11:01:05.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.125755+0000", 2023-10-09T11:01:05.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.89609301900000005 2023-10-09T11:01:05.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.125827+0000", 2023-10-09T11:01:05.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1804999999999994e-05 2023-10-09T11:01:05.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:01:05.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.125995+0000", 2023-10-09T11:01:05.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016823900000000001 2023-10-09T11:01:05.772 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.772 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.772 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.772 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131210+0000", 2023-10-09T11:01:05.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0052145810000000003 2023-10-09T11:01:05.772 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.772 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.772 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.772 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133073+0000", 2023-10-09T11:01:05.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018628869999999999 2023-10-09T11:01:05.772 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.772 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133562+0000", 2023-10-09T11:01:05.773 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00048968299999999998 2023-10-09T11:01:05.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133602+0000", 2023-10-09T11:01:05.773 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0022999999999999e-05 2023-10-09T11:01:05.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133637+0000", 2023-10-09T11:01:05.773 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4715999999999998e-05 2023-10-09T11:01:05.774 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.774 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.774 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.774 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.774 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.774 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150495 9.9 9:90b0d4cb:::benchmark_data_smithi125_363789_object40196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:01:05.774 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.248473+0000", 2023-10-09T11:01:05.774 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 82.447240703999995, 2023-10-09T11:01:05.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.88740656399999995, 2023-10-09T11:01:05.774 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.774 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.774 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.774 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.775 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.775 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150495 2023-10-09T11:01:05.775 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.775 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.775 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248473+0000", 2023-10-09T11:01:05.775 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.775 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.775 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248473+0000", 2023-10-09T11:01:05.775 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.776 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.776 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.776 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.776 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248472+0000", 2023-10-09T11:01:05.776 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:01:05.776 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.776 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.776 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.776 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248483+0000", 2023-10-09T11:01:05.776 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0905e-05 2023-10-09T11:01:05.776 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.776 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.776 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.777 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248494+0000", 2023-10-09T11:01:05.777 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1153e-05 2023-10-09T11:01:05.777 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.777 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.777 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.777 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248501+0000", 2023-10-09T11:01:05.777 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2570000000000004e-06 2023-10-09T11:01:05.777 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.777 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.777 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.777 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248547+0000", 2023-10-09T11:01:05.777 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6202999999999998e-05 2023-10-09T11:01:05.778 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.778 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.778 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.778 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248555+0000", 2023-10-09T11:01:05.778 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4530000000000002e-06 2023-10-09T11:01:05.778 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.778 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.778 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.778 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.127018+0000", 2023-10-09T11:01:05.778 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.878463302 2023-10-09T11:01:05.778 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.778 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.779 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.779 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.127131+0000", 2023-10-09T11:01:05.779 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011225 2023-10-09T11:01:05.779 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.779 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.779 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:01:05.779 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.127305+0000", 2023-10-09T11:01:05.779 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017452 2023-10-09T11:01:05.779 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.779 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.779 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.779 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131237+0000", 2023-10-09T11:01:05.779 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039315369999999997 2023-10-09T11:01:05.780 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.780 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.780 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.780 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133688+0000", 2023-10-09T11:01:05.780 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002451614 2023-10-09T11:01:05.780 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.780 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.780 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.780 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.135791+0000", 2023-10-09T11:01:05.780 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002102746 2023-10-09T11:01:05.780 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.780 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.780 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.781 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.135837+0000", 2023-10-09T11:01:05.781 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5710000000000001e-05 2023-10-09T11:01:05.781 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.781 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.781 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.781 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.135880+0000", 2023-10-09T11:01:05.781 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3037999999999997e-05 2023-10-09T11:01:05.781 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.781 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.781 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.781 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.781 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.782 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150552 9.9 9:90f11fdc:::benchmark_data_smithi125_363789_object40253:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:01:05.782 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.294722+0000", 2023-10-09T11:01:05.782 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 82.400991970000007, 2023-10-09T11:01:05.782 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.84415905599999996, 2023-10-09T11:01:05.782 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.782 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.782 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.782 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.782 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.782 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150552 2023-10-09T11:01:05.782 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.782 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.782 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.783 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.783 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294722+0000", 2023-10-09T11:01:05.783 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.783 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.783 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.783 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.783 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294722+0000", 2023-10-09T11:01:05.783 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.783 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.783 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.783 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.783 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294721+0000", 2023-10-09T11:01:05.783 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:01:05.784 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.784 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.784 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.784 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294727+0000", 2023-10-09T11:01:05.784 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5539999999999997e-06 2023-10-09T11:01:05.784 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.784 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.784 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.784 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294734+0000", 2023-10-09T11:01:05.784 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2389999999999996e-06 2023-10-09T11:01:05.784 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.784 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.785 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.785 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294739+0000", 2023-10-09T11:01:05.785 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2279999999999998e-06 2023-10-09T11:01:05.785 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.785 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.785 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.785 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294780+0000", 2023-10-09T11:01:05.785 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1128000000000003e-05 2023-10-09T11:01:05.785 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.785 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.785 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.785 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294788+0000", 2023-10-09T11:01:05.785 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4499999999999998e-06 2023-10-09T11:01:05.786 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.786 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.786 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.786 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.128120+0000", 2023-10-09T11:01:05.786 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.83333184999999999 2023-10-09T11:01:05.786 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.786 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.786 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.786 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.128227+0000", 2023-10-09T11:01:05.786 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010778200000000001 2023-10-09T11:01:05.786 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.786 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.786 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:01:05.787 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.128411+0000", 2023-10-09T11:01:05.787 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000183117 2023-10-09T11:01:05.787 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.787 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.787 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.787 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.132962+0000", 2023-10-09T11:01:05.787 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045515520000000004 2023-10-09T11:01:05.787 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.787 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.787 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.787 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133727+0000", 2023-10-09T11:01:05.787 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00076511400000000001 2023-10-09T11:01:05.787 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.788 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.788 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.788 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138805+0000", 2023-10-09T11:01:05.788 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0050773340000000002 2023-10-09T11:01:05.788 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.788 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.788 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.788 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138840+0000", 2023-10-09T11:01:05.788 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5831000000000003e-05 2023-10-09T11:01:05.788 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.788 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.788 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.788 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138881+0000", 2023-10-09T11:01:05.789 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0710999999999997e-05 2023-10-09T11:01:05.789 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.789 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.789 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.789 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.789 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.789 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150590 9.9 9:909dc84e:::benchmark_data_smithi125_363789_object40291:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:01:05.789 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.333845+0000", 2023-10-09T11:01:05.789 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 82.361868572000006, 2023-10-09T11:01:05.789 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.80510336599999999, 2023-10-09T11:01:05.789 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.789 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.789 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.790 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.790 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.790 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150590 2023-10-09T11:01:05.790 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.790 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.790 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.790 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.790 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333845+0000", 2023-10-09T11:01:05.790 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.790 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.790 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.790 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333845+0000", 2023-10-09T11:01:05.791 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.791 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333845+0000", 2023-10-09T11:01:05.791 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:01:05.791 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333850+0000", 2023-10-09T11:01:05.791 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7879999999999997e-06 2023-10-09T11:01:05.791 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.792 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333858+0000", 2023-10-09T11:01:05.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7569999999999993e-06 2023-10-09T11:01:05.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.792 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333863+0000", 2023-10-09T11:01:05.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.724e-06 2023-10-09T11:01:05.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.792 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.335088+0000", 2023-10-09T11:01:05.793 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012250270000000001 2023-10-09T11:01:05.793 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.793 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.793 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.793 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.335095+0000", 2023-10-09T11:01:05.793 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7660000000000001e-06 2023-10-09T11:01:05.793 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.793 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.793 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.793 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.129225+0000", 2023-10-09T11:01:05.793 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.79412993399999998 2023-10-09T11:01:05.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.794 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.794 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.129291+0000", 2023-10-09T11:01:05.794 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6072e-05 2023-10-09T11:01:05.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.794 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:01:05.794 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.129474+0000", 2023-10-09T11:01:05.794 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018292800000000001 2023-10-09T11:01:05.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133175+0000", 2023-10-09T11:01:05.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0037009399999999998 2023-10-09T11:01:05.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.795 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.135784+0000", 2023-10-09T11:01:05.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026093900000000001 2023-10-09T11:01:05.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.795 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138888+0000", 2023-10-09T11:01:05.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0031041950000000001 2023-10-09T11:01:05.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.796 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.796 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138921+0000", 2023-10-09T11:01:05.796 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3025999999999999e-05 2023-10-09T11:01:05.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.796 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.796 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138949+0000", 2023-10-09T11:01:05.796 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7532999999999999e-05 2023-10-09T11:01:05.796 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.796 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.797 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.797 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.797 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.797 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204565 9.13 9:cfe54d63:::benchmark_data_smithi125_363789_object94266:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T11:01:05.797 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:41.994119+0000", 2023-10-09T11:01:05.797 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 23.701594757999999, 2023-10-09T11:01:05.797 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.018728254, 2023-10-09T11:01:05.797 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.797 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:01:05.797 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.797 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.797 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.797 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204565 2023-10-09T11:01:05.798 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.798 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.798 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.798 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.798 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994119+0000", 2023-10-09T11:01:05.798 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.798 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.798 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.798 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.798 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994119+0000", 2023-10-09T11:01:05.798 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.798 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.799 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.799 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.799 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994118+0000", 2023-10-09T11:01:05.799 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:01:05.799 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.799 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.799 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.799 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994124+0000", 2023-10-09T11:01:05.799 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6509999999999998e-06 2023-10-09T11:01:05.799 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.799 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.799 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.800 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994133+0000", 2023-10-09T11:01:05.800 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1109999999999997e-06 2023-10-09T11:01:05.800 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.800 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.800 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.800 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994137+0000", 2023-10-09T11:01:05.800 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5879999999999999e-06 2023-10-09T11:01:05.800 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.800 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.800 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.800 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994205+0000", 2023-10-09T11:01:05.800 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7852e-05 2023-10-09T11:01:05.800 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.801 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.801 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.801 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994355+0000", 2023-10-09T11:01:05.801 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015022700000000001 2023-10-09T11:01:05.801 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.801 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.801 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:01:05.801 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994806+0000", 2023-10-09T11:01:05.801 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00045023900000000001 2023-10-09T11:01:05.801 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.801 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.801 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.801 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.997059+0000", 2023-10-09T11:01:05.802 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022531949999999999 2023-10-09T11:01:05.802 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.802 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.802 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.802 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.003461+0000", 2023-10-09T11:01:05.802 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064024329999999999 2023-10-09T11:01:05.802 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.802 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.802 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.802 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.003480+0000", 2023-10-09T11:01:05.802 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8272000000000001e-05 2023-10-09T11:01:05.802 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.803 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.803 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.803 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.012782+0000", 2023-10-09T11:01:05.803 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0093026169999999 2023-10-09T11:01:05.803 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.803 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.803 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.803 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.012848+0000", 2023-10-09T11:01:05.803 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5415999999999998e-05 2023-10-09T11:01:05.803 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.803 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.803 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.804 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.804 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.804 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204572 9.13 9:cc352b3f:::benchmark_data_smithi125_363789_object94273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:01:05.804 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.008011+0000", 2023-10-09T11:01:05.804 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 23.687703321000001, 2023-10-09T11:01:05.804 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0262834300000001, 2023-10-09T11:01:05.804 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.804 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.804 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.804 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.804 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.804 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204572 2023-10-09T11:01:05.804 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.805 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.805 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.805 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008011+0000", 2023-10-09T11:01:05.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.805 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.805 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008011+0000", 2023-10-09T11:01:05.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.805 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.806 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008010+0000", 2023-10-09T11:01:05.806 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:01:05.806 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.806 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.806 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.806 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008014+0000", 2023-10-09T11:01:05.806 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9009999999999996e-06 2023-10-09T11:01:05.806 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.806 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.806 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.806 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008025+0000", 2023-10-09T11:01:05.806 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0730999999999999e-05 2023-10-09T11:01:05.807 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.807 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.807 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.807 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008032+0000", 2023-10-09T11:01:05.807 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9970000000000001e-06 2023-10-09T11:01:05.807 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.807 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.807 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.807 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008125+0000", 2023-10-09T11:01:05.807 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2991999999999993e-05 2023-10-09T11:01:05.807 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.807 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008136+0000", 2023-10-09T11:01:05.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1068e-05 2023-10-09T11:01:05.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.012867+0000", 2023-10-09T11:01:05.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.004731034 2023-10-09T11:01:05.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.012951+0000", 2023-10-09T11:01:05.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3851999999999996e-05 2023-10-09T11:01:05.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:01:05.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.013280+0000", 2023-10-09T11:01:05.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00032942300000000002 2023-10-09T11:01:05.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.021065+0000", 2023-10-09T11:01:05.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0077846759999999999 2023-10-09T11:01:05.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.810 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.810 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.810 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034119+0000", 2023-10-09T11:01:05.810 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013054412 2023-10-09T11:01:05.810 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.810 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.810 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.810 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034203+0000", 2023-10-09T11:01:05.810 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4079999999999995e-05 2023-10-09T11:01:05.810 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.810 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.810 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.811 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034252+0000", 2023-10-09T11:01:05.811 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8622999999999998e-05 2023-10-09T11:01:05.811 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.811 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.811 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.811 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034294+0000", 2023-10-09T11:01:05.811 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2256000000000003e-05 2023-10-09T11:01:05.811 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.811 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.811 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.811 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.811 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.812 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204591 9.13 9:cfb36959:::benchmark_data_smithi125_363789_object94292:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:01:05.812 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.043404+0000", 2023-10-09T11:01:05.812 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 23.652310271000001, 2023-10-09T11:01:05.812 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.990990861, 2023-10-09T11:01:05.812 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.812 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.812 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.812 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.812 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.812 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204591 2023-10-09T11:01:05.812 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.812 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.813 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.813 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.813 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043404+0000", 2023-10-09T11:01:05.813 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.813 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.813 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.813 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.813 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043404+0000", 2023-10-09T11:01:05.813 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.813 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.813 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.813 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.814 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043402+0000", 2023-10-09T11:01:05.814 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T11:01:05.814 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.814 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.814 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.814 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043412+0000", 2023-10-09T11:01:05.814 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0499000000000001e-05 2023-10-09T11:01:05.814 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.814 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.814 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.814 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043433+0000", 2023-10-09T11:01:05.814 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0477999999999999e-05 2023-10-09T11:01:05.815 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.815 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.815 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.815 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043442+0000", 2023-10-09T11:01:05.815 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9609999999999994e-06 2023-10-09T11:01:05.815 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.815 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.815 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.815 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043619+0000", 2023-10-09T11:01:05.815 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000177558 2023-10-09T11:01:05.815 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.815 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.815 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.816 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043630+0000", 2023-10-09T11:01:05.816 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0382e-05 2023-10-09T11:01:05.816 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.816 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.816 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.816 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.014803+0000", 2023-10-09T11:01:05.816 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97117335900000001 2023-10-09T11:01:05.816 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.816 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.816 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.816 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.014927+0000", 2023-10-09T11:01:05.816 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012421900000000001 2023-10-09T11:01:05.817 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.817 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.817 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:01:05.817 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.015207+0000", 2023-10-09T11:01:05.817 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00027950199999999999 2023-10-09T11:01:05.817 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.817 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.817 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.817 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.022320+0000", 2023-10-09T11:01:05.817 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0071137290000000001 2023-10-09T11:01:05.817 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.817 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.818 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.818 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034155+0000", 2023-10-09T11:01:05.818 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011834905 2023-10-09T11:01:05.818 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.818 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.818 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.818 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034329+0000", 2023-10-09T11:01:05.818 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017399200000000001 2023-10-09T11:01:05.818 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.818 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.818 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.818 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034361+0000", 2023-10-09T11:01:05.819 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1545999999999999e-05 2023-10-09T11:01:05.819 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.819 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.819 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.819 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034395+0000", 2023-10-09T11:01:05.819 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3772999999999997e-05 2023-10-09T11:01:05.819 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.819 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.819 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.819 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.819 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.819 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204618 9.13 9:c9e84990:::benchmark_data_smithi125_363789_object94319:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:01:05.819 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.073111+0000", 2023-10-09T11:01:05.820 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 23.622602599, 2023-10-09T11:01:05.820 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.96151479900000003, 2023-10-09T11:01:05.820 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.820 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.820 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.820 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.820 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.820 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204618 2023-10-09T11:01:05.820 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.820 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.820 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.820 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.820 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073111+0000", 2023-10-09T11:01:05.821 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.821 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.821 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.821 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.821 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073111+0000", 2023-10-09T11:01:05.821 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.821 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.821 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.821 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.821 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073110+0000", 2023-10-09T11:01:05.821 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:01:05.821 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.821 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.822 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.822 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073117+0000", 2023-10-09T11:01:05.822 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9820000000000002e-06 2023-10-09T11:01:05.822 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.822 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.822 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.822 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073129+0000", 2023-10-09T11:01:05.822 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1608e-05 2023-10-09T11:01:05.822 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.822 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.822 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.822 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073135+0000", 2023-10-09T11:01:05.823 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0249999999999999e-06 2023-10-09T11:01:05.823 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.823 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.823 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.823 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073191+0000", 2023-10-09T11:01:05.823 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5810999999999998e-05 2023-10-09T11:01:05.823 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.823 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.823 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.823 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073197+0000", 2023-10-09T11:01:05.823 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3579999999999999e-06 2023-10-09T11:01:05.823 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.823 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.824 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.824 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.016587+0000", 2023-10-09T11:01:05.824 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.94339012799999999 2023-10-09T11:01:05.824 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.824 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.824 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.824 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.016657+0000", 2023-10-09T11:01:05.824 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0301000000000003e-05 2023-10-09T11:01:05.824 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.824 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.824 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:01:05.824 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.016830+0000", 2023-10-09T11:01:05.825 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017296799999999999 2023-10-09T11:01:05.825 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.825 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.825 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.825 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.022324+0000", 2023-10-09T11:01:05.825 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0054936739999999996 2023-10-09T11:01:05.825 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.825 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.825 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.825 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034436+0000", 2023-10-09T11:01:05.825 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012112327000000001 2023-10-09T11:01:05.825 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.826 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.826 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.826 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034545+0000", 2023-10-09T11:01:05.826 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000108213 2023-10-09T11:01:05.826 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.826 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.826 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.826 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034587+0000", 2023-10-09T11:01:05.826 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2840000000000003e-05 2023-10-09T11:01:05.826 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.826 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.826 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.827 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034626+0000", 2023-10-09T11:01:05.827 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8868999999999997e-05 2023-10-09T11:01:05.827 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.827 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.827 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.827 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.827 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.827 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204624 9.13 9:cb2a155f:::benchmark_data_smithi125_363789_object94325:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:01:05.827 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.084343+0000", 2023-10-09T11:01:05.827 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 23.611370807, 2023-10-09T11:01:05.827 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95037137000000005, 2023-10-09T11:01:05.827 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.827 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.828 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.828 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.828 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.828 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204624 2023-10-09T11:01:05.828 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.828 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.828 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.828 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.828 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084343+0000", 2023-10-09T11:01:05.828 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.828 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.828 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.828 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.829 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084343+0000", 2023-10-09T11:01:05.829 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.829 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.829 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.829 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.829 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084340+0000", 2023-10-09T11:01:05.829 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999967 2023-10-09T11:01:05.829 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.829 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.829 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.829 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084352+0000", 2023-10-09T11:01:05.829 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1593999999999999e-05 2023-10-09T11:01:05.829 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.829 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.830 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.830 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084367+0000", 2023-10-09T11:01:05.830 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5668e-05 2023-10-09T11:01:05.830 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.830 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.830 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.830 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084373+0000", 2023-10-09T11:01:05.830 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5799999999999999e-06 2023-10-09T11:01:05.830 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.830 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.830 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.830 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084441+0000", 2023-10-09T11:01:05.831 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8324999999999994e-05 2023-10-09T11:01:05.831 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.831 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.831 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.831 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084453+0000", 2023-10-09T11:01:05.831 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1297999999999999e-05 2023-10-09T11:01:05.831 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.831 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.831 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.831 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.017927+0000", 2023-10-09T11:01:05.831 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93347489400000005 2023-10-09T11:01:05.831 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.831 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.832 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.832 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.018027+0000", 2023-10-09T11:01:05.832 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9511000000000003e-05 2023-10-09T11:01:05.832 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.832 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.832 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:01:05.832 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.018203+0000", 2023-10-09T11:01:05.832 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017629500000000001 2023-10-09T11:01:05.832 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.832 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.832 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.832 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.022327+0000", 2023-10-09T11:01:05.832 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041237510000000002 2023-10-09T11:01:05.832 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.833 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.833 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.833 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034475+0000", 2023-10-09T11:01:05.833 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012147857999999999 2023-10-09T11:01:05.833 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.833 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.833 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.833 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034657+0000", 2023-10-09T11:01:05.833 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000182504 2023-10-09T11:01:05.833 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.833 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.833 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.834 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034684+0000", 2023-10-09T11:01:05.834 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6936000000000001e-05 2023-10-09T11:01:05.834 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.834 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.834 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.834 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034715+0000", 2023-10-09T11:01:05.834 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0264e-05 2023-10-09T11:01:05.834 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.834 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.834 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.834 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.834 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.834 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204699 9.13 9:ca5a965b:::benchmark_data_smithi125_363789_object94400:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:01:05.835 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.182784+0000", 2023-10-09T11:01:05.835 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 23.512930036, 2023-10-09T11:01:05.835 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.85207209399999995, 2023-10-09T11:01:05.835 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.835 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.835 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.835 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.835 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.835 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204699 2023-10-09T11:01:05.835 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.835 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.835 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.836 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.836 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182784+0000", 2023-10-09T11:01:05.836 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.836 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.836 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.836 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.836 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182784+0000", 2023-10-09T11:01:05.836 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.836 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.836 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.836 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.836 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182783+0000", 2023-10-09T11:01:05.836 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:01:05.837 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.837 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.837 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.837 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182787+0000", 2023-10-09T11:01:05.837 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.377e-06 2023-10-09T11:01:05.837 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.837 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.837 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.837 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182797+0000", 2023-10-09T11:01:05.837 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9119999999999993e-06 2023-10-09T11:01:05.837 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.837 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.838 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.838 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182801+0000", 2023-10-09T11:01:05.838 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6480000000000002e-06 2023-10-09T11:01:05.838 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.838 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.838 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.838 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182860+0000", 2023-10-09T11:01:05.838 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8857000000000002e-05 2023-10-09T11:01:05.838 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.838 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.839 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.839 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182870+0000", 2023-10-09T11:01:05.839 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0223e-05 2023-10-09T11:01:05.839 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.839 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.839 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.839 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.019382+0000", 2023-10-09T11:01:05.839 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.83651138199999997 2023-10-09T11:01:05.839 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.839 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.839 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.839 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.019500+0000", 2023-10-09T11:01:05.839 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118187 2023-10-09T11:01:05.840 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.840 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.840 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:01:05.840 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.019722+0000", 2023-10-09T11:01:05.840 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022236099999999999 2023-10-09T11:01:05.840 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.840 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.840 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.840 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.023434+0000", 2023-10-09T11:01:05.840 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00371164 2023-10-09T11:01:05.840 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.840 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.841 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.841 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034764+0000", 2023-10-09T11:01:05.841 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011329824 2023-10-09T11:01:05.841 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.841 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.841 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.841 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034792+0000", 2023-10-09T11:01:05.841 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8123000000000002e-05 2023-10-09T11:01:05.841 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.841 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.841 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.841 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034817+0000", 2023-10-09T11:01:05.841 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5009999999999999e-05 2023-10-09T11:01:05.842 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.842 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.842 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.842 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034856+0000", 2023-10-09T11:01:05.842 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9178999999999999e-05 2023-10-09T11:01:05.842 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.842 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.842 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.842 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.842 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.842 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204804 9.13 9:cd6b9308:::benchmark_data_smithi125_363789_object94505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:01:05.842 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.320768+0000", 2023-10-09T11:01:05.843 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 23.374946300000001, 2023-10-09T11:01:05.843 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.71443022199999995, 2023-10-09T11:01:05.843 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:01:05.843 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:01:05.843 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:01:05.843 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:01:05.843 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:01:05.843 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204804 2023-10-09T11:01:05.843 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.843 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:01:05.843 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.843 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:01:05.844 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320768+0000", 2023-10-09T11:01:05.844 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.844 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.844 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.844 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:01:05.844 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320768+0000", 2023-10-09T11:01:05.844 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:01:05.844 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.844 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.844 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:01:05.844 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320767+0000", 2023-10-09T11:01:05.844 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:01:05.844 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.845 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.845 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:01:05.845 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320770+0000", 2023-10-09T11:01:05.845 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.768e-06 2023-10-09T11:01:05.845 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.845 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.845 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:01:05.845 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320777+0000", 2023-10-09T11:01:05.845 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.182e-06 2023-10-09T11:01:05.845 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.845 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.845 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:01:05.846 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320780+0000", 2023-10-09T11:01:05.846 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8890000000000002e-06 2023-10-09T11:01:05.846 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.846 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.846 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.846 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320856+0000", 2023-10-09T11:01:05.846 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5768999999999999e-05 2023-10-09T11:01:05.846 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.846 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.846 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:01:05.846 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320866+0000", 2023-10-09T11:01:05.846 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9410000000000002e-06 2023-10-09T11:01:05.847 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.847 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.847 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:01:05.847 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.021091+0000", 2023-10-09T11:01:05.847 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.70022501000000004 2023-10-09T11:01:05.847 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.847 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.847 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:01:05.847 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.021196+0000", 2023-10-09T11:01:05.847 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104948 2023-10-09T11:01:05.847 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.847 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.848 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:01:05.848 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.021416+0000", 2023-10-09T11:01:05.848 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022006799999999999 2023-10-09T11:01:05.848 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.848 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.848 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:01:05.848 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.024573+0000", 2023-10-09T11:01:05.848 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0031569430000000002 2023-10-09T11:01:05.848 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.848 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.848 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.848 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034901+0000", 2023-10-09T11:01:05.848 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010327935 2023-10-09T11:01:05.849 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.849 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.849 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:05.849 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.035121+0000", 2023-10-09T11:01:05.849 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00021983500000000001 2023-10-09T11:01:05.849 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.849 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.849 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:01:05.849 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.035163+0000", 2023-10-09T11:01:05.849 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2920000000000002e-05 2023-10-09T11:01:05.849 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:01:05.849 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:01:05.850 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:01:05.850 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.035198+0000", 2023-10-09T11:01:05.850 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4539999999999998e-05 2023-10-09T11:01:05.850 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.850 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.850 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.850 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:01:05.850 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:01:05.850 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:01:05.851 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:05.851 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:01:05.851 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:01:05.851 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:01:05.851 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:01:05.851 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:01:05.852 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:01:05.852 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:01:05.852 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:01:05.852 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:01:05.852 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:01:05.852 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:01:05.852 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:01:05.852 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:01:05.852 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:01:05.853 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:01:05.853 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:01:05.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:05.853 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:05.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:05.954 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:05.984 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:06.003 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:06.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:06.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:06.065 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T11:01:06.107 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:06.135 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:06.154 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:06.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:06.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:06.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:01:06.258 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:06.288 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:06.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:06 smithi125 ceph-mon[195507]: failed to encode map e399 with expected crc 2023-10-09T11:01:06.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:06 smithi125 ceph-mon[195507]: osdmap e399: 8 total, 8 up, 8 in 2023-10-09T11:01:06.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:06 smithi125 ceph-mon[195507]: failed to encode map e399 with expected crc 2023-10-09T11:01:06.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:06 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3955313946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:06.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:06 smithi125 ceph-mon[195507]: pgmap v996: 129 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 71 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 30581/3468 objects degraded (881.805%); 0 B/s, 39 objects/s recovering 2023-10-09T11:01:06.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:06 smithi125 ceph-mon[200197]: failed to encode map e399 with expected crc 2023-10-09T11:01:06.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:06 smithi125 ceph-mon[200197]: osdmap e399: 8 total, 8 up, 8 in 2023-10-09T11:01:06.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:06 smithi125 ceph-mon[200197]: failed to encode map e399 with expected crc 2023-10-09T11:01:06.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:06 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3955313946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:06.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:06 smithi125 ceph-mon[200197]: pgmap v996: 129 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 71 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 30581/3468 objects degraded (881.805%); 0 B/s, 39 objects/s recovering 2023-10-09T11:01:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:06 smithi155 ceph-mon[184854]: failed to encode map e399 with expected crc 2023-10-09T11:01:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:06 smithi155 ceph-mon[184854]: osdmap e399: 8 total, 8 up, 8 in 2023-10-09T11:01:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:06 smithi155 ceph-mon[184854]: failed to encode map e399 with expected crc 2023-10-09T11:01:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:06 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3955313946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:06 smithi155 ceph-mon[184854]: pgmap v996: 129 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 71 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 30581/3468 objects degraded (881.805%); 0 B/s, 39 objects/s recovering 2023-10-09T11:01:07.176 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:07.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:07.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:07.281 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:07.314 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:07.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:07 smithi125 ceph-mon[195507]: failed to encode map e399 with expected crc 2023-10-09T11:01:07.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:07 smithi125 ceph-mon[195507]: failed to encode map e399 with expected crc 2023-10-09T11:01:07.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:07 smithi125 ceph-mon[195507]: failed to encode map e400 with expected crc 2023-10-09T11:01:07.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:07 smithi125 ceph-mon[195507]: failed to encode map e400 with expected crc 2023-10-09T11:01:07.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:07 smithi125 ceph-mon[195507]: osdmap e400: 8 total, 8 up, 8 in 2023-10-09T11:01:07.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:07 smithi125 ceph-mon[195507]: failed to encode map e400 with expected crc 2023-10-09T11:01:07.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:07 smithi125 ceph-mon[200197]: failed to encode map e399 with expected crc 2023-10-09T11:01:07.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:07 smithi125 ceph-mon[200197]: failed to encode map e399 with expected crc 2023-10-09T11:01:07.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:07 smithi125 ceph-mon[200197]: failed to encode map e400 with expected crc 2023-10-09T11:01:07.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:07 smithi125 ceph-mon[200197]: failed to encode map e400 with expected crc 2023-10-09T11:01:07.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:07 smithi125 ceph-mon[200197]: osdmap e400: 8 total, 8 up, 8 in 2023-10-09T11:01:07.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:07 smithi125 ceph-mon[200197]: failed to encode map e400 with expected crc 2023-10-09T11:01:07.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:07 smithi155 ceph-mon[184854]: failed to encode map e399 with expected crc 2023-10-09T11:01:07.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:07 smithi155 ceph-mon[184854]: failed to encode map e399 with expected crc 2023-10-09T11:01:07.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:07 smithi155 ceph-mon[184854]: failed to encode map e400 with expected crc 2023-10-09T11:01:07.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:07 smithi155 ceph-mon[184854]: failed to encode map e400 with expected crc 2023-10-09T11:01:07.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:07 smithi155 ceph-mon[184854]: osdmap e400: 8 total, 8 up, 8 in 2023-10-09T11:01:07.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:07 smithi155 ceph-mon[184854]: failed to encode map e400 with expected crc 2023-10-09T11:01:08.197 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:08.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:08.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:08.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:08.339 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:08.360 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:08.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:08.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:08.464 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:08.494 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:08.514 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:08.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:08.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:08 smithi125 ceph-mon[195507]: failed to encode map e400 with expected crc 2023-10-09T11:01:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:08 smithi125 ceph-mon[195507]: pgmap v998: 129 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 13 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 289 op/s; 28631/672 objects degraded (4260.565%); 0 B/s, 58 objects/s recovering 2023-10-09T11:01:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:08 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30581/3468 objects degraded (881.805%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:01:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:08 smithi125 ceph-mon[200197]: failed to encode map e400 with expected crc 2023-10-09T11:01:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:08 smithi125 ceph-mon[200197]: pgmap v998: 129 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 13 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 289 op/s; 28631/672 objects degraded (4260.565%); 0 B/s, 58 objects/s recovering 2023-10-09T11:01:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:08 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30581/3468 objects degraded (881.805%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:01:08.572 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:08.618 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:08.651 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:08.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:08 smithi155 ceph-mon[184854]: failed to encode map e400 with expected crc 2023-10-09T11:01:08.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:08 smithi155 ceph-mon[184854]: pgmap v998: 129 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 13 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 289 op/s; 28631/672 objects degraded (4260.565%); 0 B/s, 58 objects/s recovering 2023-10-09T11:01:08.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:08 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30581/3468 objects degraded (881.805%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:01:08.724 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:08.726 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:08.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:08.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:08.834 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:08.867 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:08.888 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:08.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:08.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:08.991 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:09.022 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:09.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:09.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:09.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:09.146 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:09.174 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:09.193 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:09.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:09.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:09.297 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:09.324 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:09.346 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:09.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:09.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:09.350 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_3' removed 2023-10-09T11:01:09.451 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:09.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:09 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4170076708' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:01:09.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:09 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:01:09.478 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:09.496 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:09.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:09.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:09.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:09.626 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:09 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4170076708' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:01:09.626 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:09 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:01:09.626 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:09 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4170076708' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:01:09.626 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:09 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:01:09.629 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:09.900 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:09.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:09.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:10.004 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:10.031 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:10.050 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:10.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:10.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:10.153 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:10.180 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:10.198 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:10.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:10.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:10.221 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:01:10.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:10.329 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:10.348 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:10.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:10.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:10.401 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T11:01:10.402 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T11:01:10.402 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-10-09T11:01:10.402 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-10-09T11:01:10.451 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:10.478 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:10.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:10 smithi125 ceph-mon[195507]: failed to encode map e401 with expected crc 2023-10-09T11:01:10.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:10 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T11:01:10.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:10 smithi125 ceph-mon[195507]: osdmap e401: 8 total, 8 up, 8 in 2023-10-09T11:01:10.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:10 smithi125 ceph-mon[195507]: pgmap v1000: 103 pgs: 103 active+clean; 643 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:01:10.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:10 smithi125 ceph-mon[195507]: failed to encode map e401 with expected crc 2023-10-09T11:01:10.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:10 smithi125 ceph-mon[195507]: failed to encode map e401 with expected crc 2023-10-09T11:01:10.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:10 smithi125 ceph-mon[200197]: failed to encode map e401 with expected crc 2023-10-09T11:01:10.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:10 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T11:01:10.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:10 smithi125 ceph-mon[200197]: osdmap e401: 8 total, 8 up, 8 in 2023-10-09T11:01:10.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:10 smithi125 ceph-mon[200197]: pgmap v1000: 103 pgs: 103 active+clean; 643 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:01:10.494 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:10 smithi125 ceph-mon[200197]: failed to encode map e401 with expected crc 2023-10-09T11:01:10.494 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:10 smithi125 ceph-mon[200197]: failed to encode map e401 with expected crc 2023-10-09T11:01:10.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:10.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:10.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:10.600 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:10.622 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:10.625 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:10 smithi155 ceph-mon[184854]: failed to encode map e401 with expected crc 2023-10-09T11:01:10.625 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:10 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T11:01:10.625 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:10 smithi155 ceph-mon[184854]: osdmap e401: 8 total, 8 up, 8 in 2023-10-09T11:01:10.626 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:10 smithi155 ceph-mon[184854]: pgmap v1000: 103 pgs: 103 active+clean; 643 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:01:10.626 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:10 smithi155 ceph-mon[184854]: failed to encode map e401 with expected crc 2023-10-09T11:01:10.626 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:10 smithi155 ceph-mon[184854]: failed to encode map e401 with expected crc 2023-10-09T11:01:10.626 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T11:01:10.626 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:01:10.627 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T11:01:10.627 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 1.000000 2023-10-09T11:01:10.627 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T11:01:10.628 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:10.646 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:10.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:10.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:10.749 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:10.785 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:10.859 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:10.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:10.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:10.959 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:01:10.960 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:01:10.960 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 0 2023-10-09T11:01:10.960 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:01:10.964 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:10.991 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:11.009 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:11.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:11.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:11.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:11.140 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:11.159 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:11.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:11.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:11.263 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:11.291 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:11.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:01:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:01:11] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.43.0" 2023-10-09T11:01:11.364 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:01:11.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:11 smithi125 ceph-mon[195507]: failed to encode map e401 with expected crc 2023-10-09T11:01:11.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:11 smithi125 ceph-mon[195507]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28631/672 objects degraded (4260.565%), 11 pgs degraded) 2023-10-09T11:01:11.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:11 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2805599380' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:01:11.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:11 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:01:11.631 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:11 smithi125 ceph-mon[200197]: failed to encode map e401 with expected crc 2023-10-09T11:01:11.631 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:11 smithi125 ceph-mon[200197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28631/672 objects degraded (4260.565%), 11 pgs degraded) 2023-10-09T11:01:11.631 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:11 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2805599380' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:01:11.631 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:11 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:01:11.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:11 smithi155 ceph-mon[184854]: failed to encode map e401 with expected crc 2023-10-09T11:01:11.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:11 smithi155 ceph-mon[184854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28631/672 objects degraded (4260.565%), 11 pgs degraded) 2023-10-09T11:01:11.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:11 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2805599380' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:01:11.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:11 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:01:11.863 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:11.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:11.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:11.903 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T11:01:11.917 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:11.933 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:11.967 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:11.999 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:12.020 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:12.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:12.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:12.124 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:12.156 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:12.280 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:12.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:12.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:12.351 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:01:12.370 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:01:12.384 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:12.392 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:01:12.414 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:01:12.416 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:12.433 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:01:12.437 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:12.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:12.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:12.452 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:01:12.472 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:01:12.490 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:01:12.492 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:01:12.492 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:01:12.492 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:01:12.492 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:01:12.492 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:01:12.492 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:01:12.492 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:01:12.492 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:01:12.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:12.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[195507]: failed to encode map e402 with expected crc 2023-10-09T11:01:12.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:01:12.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[195507]: osdmap e402: 8 total, 8 up, 8 in 2023-10-09T11:01:12.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[195507]: pgmap v1002: 103 pgs: 103 active+clean; 643 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:01:12.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[195507]: failed to encode map e402 with expected crc 2023-10-09T11:01:12.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[195507]: failed to encode map e402 with expected crc 2023-10-09T11:01:12.512 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3502848898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:12.512 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:12.512 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[200197]: failed to encode map e402 with expected crc 2023-10-09T11:01:12.512 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:01:12.512 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[200197]: osdmap e402: 8 total, 8 up, 8 in 2023-10-09T11:01:12.512 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[200197]: pgmap v1002: 103 pgs: 103 active+clean; 643 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:01:12.512 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[200197]: failed to encode map e402 with expected crc 2023-10-09T11:01:12.512 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[200197]: failed to encode map e402 with expected crc 2023-10-09T11:01:12.512 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:12 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3502848898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:12.541 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:12.570 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:12.590 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:12.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:12.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:12.695 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:12 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:12 smithi155 ceph-mon[184854]: failed to encode map e402 with expected crc 2023-10-09T11:01:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:12 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:01:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:12 smithi155 ceph-mon[184854]: osdmap e402: 8 total, 8 up, 8 in 2023-10-09T11:01:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:12 smithi155 ceph-mon[184854]: pgmap v1002: 103 pgs: 103 active+clean; 643 KiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:01:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:12 smithi155 ceph-mon[184854]: failed to encode map e402 with expected crc 2023-10-09T11:01:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:12 smithi155 ceph-mon[184854]: failed to encode map e402 with expected crc 2023-10-09T11:01:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:12 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3502848898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:12.724 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:12.743 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:12.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:12.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:12.847 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:12.876 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:13.197 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:13.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:13.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:13.301 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:13.334 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:13.405 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:13.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:13.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:13.439 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:13 smithi125 ceph-mon[195507]: failed to encode map e402 with expected crc 2023-10-09T11:01:13.439 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:13 smithi125 ceph-mon[200197]: failed to encode map e402 with expected crc 2023-10-09T11:01:13.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:13.538 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:13.604 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:13.615 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:13.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:13.720 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:13.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:13 smithi155 ceph-mon[184854]: failed to encode map e402 with expected crc 2023-10-09T11:01:13.754 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:13.774 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:13.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:13.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:13.875 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:01:13.878 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:13.945 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:14.017 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:14.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:14.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:14.122 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:14.154 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:14.175 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:14.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:14.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:14.279 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:14.310 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:14.320 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:14.330 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:14.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:14.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:14.382 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T11:01:14.383 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_4' created 2023-10-09T11:01:14.434 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:14.465 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:14.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:14 smithi155 ceph-mon[184854]: pgmap v1003: 103 pgs: 103 active+clean; 643 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:01:14.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:14 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2277868988' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T11:01:14.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:14 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/447739981' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:01:14.477 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:14 smithi125 ceph-mon[195507]: pgmap v1003: 103 pgs: 103 active+clean; 643 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:01:14.477 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:14 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2277868988' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T11:01:14.477 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:14 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/447739981' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:01:14.478 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:14 smithi125 ceph-mon[200197]: pgmap v1003: 103 pgs: 103 active+clean; 643 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:01:14.478 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:14 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2277868988' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T11:01:14.478 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:14 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/447739981' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:01:14.537 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:14.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:14.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:14.642 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:14.670 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:14.801 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-10-09T11:01:15.142 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:15.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:15.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:15.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:01:15.233 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:01:15.234 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:01:15.247 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:15.281 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:15.301 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:15.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:15.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:15.405 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:15.434 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:15.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:15.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:15.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:15.557 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:15.584 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:15.612 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:15 smithi125 ceph-mon[195507]: failed to encode map e403 with expected crc 2023-10-09T11:01:15.613 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:15 smithi125 ceph-mon[195507]: failed to encode map e403 with expected crc 2023-10-09T11:01:15.613 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:15 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2277868988' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T11:01:15.613 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:15 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/447739981' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:01:15.613 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:15 smithi125 ceph-mon[195507]: osdmap e403: 8 total, 8 up, 8 in 2023-10-09T11:01:15.613 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:15.613 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:01:15.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:15 smithi125 ceph-mon[200197]: failed to encode map e403 with expected crc 2023-10-09T11:01:15.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:15 smithi125 ceph-mon[200197]: failed to encode map e403 with expected crc 2023-10-09T11:01:15.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:15 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2277868988' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T11:01:15.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:15 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/447739981' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:01:15.614 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:15 smithi125 ceph-mon[200197]: osdmap e403: 8 total, 8 up, 8 in 2023-10-09T11:01:15.614 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:15.614 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:01:15.642 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:15.655 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:15.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:15.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:15 smithi155 ceph-mon[184854]: failed to encode map e403 with expected crc 2023-10-09T11:01:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:15 smithi155 ceph-mon[184854]: failed to encode map e403 with expected crc 2023-10-09T11:01:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:15 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2277868988' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T11:01:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:15 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/447739981' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:01:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:15 smithi155 ceph-mon[184854]: osdmap e403: 8 total, 8 up, 8 in 2023-10-09T11:01:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:15.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:01:15.758 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:15.789 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T11:01:15.790 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:15.864 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:15.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:15.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:15.969 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:16.001 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:16.021 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:16.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:16.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:16.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:16.157 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:16.190 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:16.333 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:16.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:16.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:16.395 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:01:16.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[195507]: failed to encode map e404 with expected crc 2023-10-09T11:01:16.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[195507]: failed to encode map e404 with expected crc 2023-10-09T11:01:16.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[195507]: osdmap e404: 8 total, 8 up, 8 in 2023-10-09T11:01:16.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[195507]: failed to encode map e404 with expected crc 2023-10-09T11:01:16.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[195507]: failed to encode map e404 with expected crc 2023-10-09T11:01:16.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[195507]: pgmap v1006: 119 pgs: 16 unknown, 103 active+clean; 643 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:01:16.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:16.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3191794983' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:01:16.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[200197]: failed to encode map e404 with expected crc 2023-10-09T11:01:16.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[200197]: failed to encode map e404 with expected crc 2023-10-09T11:01:16.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[200197]: osdmap e404: 8 total, 8 up, 8 in 2023-10-09T11:01:16.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[200197]: failed to encode map e404 with expected crc 2023-10-09T11:01:16.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[200197]: failed to encode map e404 with expected crc 2023-10-09T11:01:16.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[200197]: pgmap v1006: 119 pgs: 16 unknown, 103 active+clean; 643 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:01:16.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:16.423 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:16 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3191794983' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:01:16.437 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:16.468 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:16.489 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:16.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:16.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:16.593 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:16.623 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:16.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:16 smithi155 ceph-mon[184854]: failed to encode map e404 with expected crc 2023-10-09T11:01:16.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:16 smithi155 ceph-mon[184854]: failed to encode map e404 with expected crc 2023-10-09T11:01:16.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:16 smithi155 ceph-mon[184854]: osdmap e404: 8 total, 8 up, 8 in 2023-10-09T11:01:16.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:16 smithi155 ceph-mon[184854]: failed to encode map e404 with expected crc 2023-10-09T11:01:16.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:16 smithi155 ceph-mon[184854]: failed to encode map e404 with expected crc 2023-10-09T11:01:16.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:16 smithi155 ceph-mon[184854]: pgmap v1006: 119 pgs: 16 unknown, 103 active+clean; 643 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:01:16.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:16.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:16 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3191794983' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:01:16.994 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:16.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:16.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:17.097 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:17.098 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:17.131 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:17.153 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:17.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:17.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:17.257 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:17.287 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:17.307 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:17.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:17.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:17.404 INFO:teuthology.orchestra.run.smithi125.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-10-09T11:01:17.410 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:17.439 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:17.458 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:17.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:17.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:17.562 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:17.590 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:17.608 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:17.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:17.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:17.654 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:01:17.655 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:01:17.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:17.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[195507]: failed to encode map e405 with expected crc 2023-10-09T11:01:17.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3191794983' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:01:17.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[195507]: osdmap e405: 8 total, 8 up, 8 in 2023-10-09T11:01:17.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[195507]: failed to encode map e405 with expected crc 2023-10-09T11:01:17.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[195507]: failed to encode map e405 with expected crc 2023-10-09T11:01:17.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[195507]: 1.17 scrub starts 2023-10-09T11:01:17.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[195507]: 1.17 scrub ok 2023-10-09T11:01:17.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1293998255' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:01:17.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:01:17.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:17.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[200197]: failed to encode map e405 with expected crc 2023-10-09T11:01:17.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3191794983' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:01:17.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[200197]: osdmap e405: 8 total, 8 up, 8 in 2023-10-09T11:01:17.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[200197]: failed to encode map e405 with expected crc 2023-10-09T11:01:17.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[200197]: failed to encode map e405 with expected crc 2023-10-09T11:01:17.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[200197]: 1.17 scrub starts 2023-10-09T11:01:17.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[200197]: 1.17 scrub ok 2023-10-09T11:01:17.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1293998255' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:01:17.667 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:17 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:01:17.712 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:17.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:17 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:17.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:17 smithi155 ceph-mon[184854]: failed to encode map e405 with expected crc 2023-10-09T11:01:17.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3191794983' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:01:17.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:17 smithi155 ceph-mon[184854]: osdmap e405: 8 total, 8 up, 8 in 2023-10-09T11:01:17.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:17 smithi155 ceph-mon[184854]: failed to encode map e405 with expected crc 2023-10-09T11:01:17.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:17 smithi155 ceph-mon[184854]: failed to encode map e405 with expected crc 2023-10-09T11:01:17.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:17 smithi155 ceph-mon[184854]: 1.17 scrub starts 2023-10-09T11:01:17.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:17 smithi155 ceph-mon[184854]: 1.17 scrub ok 2023-10-09T11:01:17.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1293998255' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:01:17.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:17 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:01:17.742 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:17.864 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:17.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:17.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:17.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:17.998 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:18.117 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:18.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:18.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:18.221 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:18.248 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:18.287 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T11:01:18.317 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:18.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:18.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:18.421 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:18.454 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:18.476 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:18.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:18.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:18.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[195507]: failed to encode map e405 with expected crc 2023-10-09T11:01:18.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[195507]: 1.6 scrub starts 2023-10-09T11:01:18.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[195507]: 1.6 scrub ok 2023-10-09T11:01:18.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[195507]: 1.3 scrub starts 2023-10-09T11:01:18.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[195507]: 1.3 scrub ok 2023-10-09T11:01:18.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T11:01:18.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[195507]: osdmap e406: 8 total, 8 up, 8 in 2023-10-09T11:01:18.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[195507]: pgmap v1009: 119 pgs: 5 unknown, 114 active+clean; 500 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 45 KiB/s rd, 119 KiB/s wr, 35 op/s 2023-10-09T11:01:18.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[195507]: failed to encode map e406 with expected crc 2023-10-09T11:01:18.518 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[195507]: failed to encode map e406 with expected crc 2023-10-09T11:01:18.518 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[195507]: failed to encode map e406 with expected crc 2023-10-09T11:01:18.518 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[200197]: failed to encode map e405 with expected crc 2023-10-09T11:01:18.518 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[200197]: 1.6 scrub starts 2023-10-09T11:01:18.518 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[200197]: 1.6 scrub ok 2023-10-09T11:01:18.518 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[200197]: 1.3 scrub starts 2023-10-09T11:01:18.518 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[200197]: 1.3 scrub ok 2023-10-09T11:01:18.518 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T11:01:18.518 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[200197]: osdmap e406: 8 total, 8 up, 8 in 2023-10-09T11:01:18.518 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[200197]: pgmap v1009: 119 pgs: 5 unknown, 114 active+clean; 500 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 45 KiB/s rd, 119 KiB/s wr, 35 op/s 2023-10-09T11:01:18.519 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[200197]: failed to encode map e406 with expected crc 2023-10-09T11:01:18.519 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[200197]: failed to encode map e406 with expected crc 2023-10-09T11:01:18.519 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:18 smithi125 ceph-mon[200197]: failed to encode map e406 with expected crc 2023-10-09T11:01:18.580 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:18.610 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:18.631 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:18.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:18.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:18.705 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:18.710 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:01:18.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:18 smithi155 ceph-mon[184854]: failed to encode map e405 with expected crc 2023-10-09T11:01:18.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:18 smithi155 ceph-mon[184854]: 1.6 scrub starts 2023-10-09T11:01:18.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:18 smithi155 ceph-mon[184854]: 1.6 scrub ok 2023-10-09T11:01:18.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:18 smithi155 ceph-mon[184854]: 1.3 scrub starts 2023-10-09T11:01:18.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:18 smithi155 ceph-mon[184854]: 1.3 scrub ok 2023-10-09T11:01:18.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:18 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T11:01:18.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:18 smithi155 ceph-mon[184854]: osdmap e406: 8 total, 8 up, 8 in 2023-10-09T11:01:18.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:18 smithi155 ceph-mon[184854]: pgmap v1009: 119 pgs: 5 unknown, 114 active+clean; 500 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 45 KiB/s rd, 119 KiB/s wr, 35 op/s 2023-10-09T11:01:18.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:18 smithi155 ceph-mon[184854]: failed to encode map e406 with expected crc 2023-10-09T11:01:18.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:18 smithi155 ceph-mon[184854]: failed to encode map e406 with expected crc 2023-10-09T11:01:18.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:18 smithi155 ceph-mon[184854]: failed to encode map e406 with expected crc 2023-10-09T11:01:18.729 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:01:18.735 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:18.748 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:01:18.763 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:18.767 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:01:18.783 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:18.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:18.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:18.787 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:01:18.805 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:01:18.822 DEBUG:teuthology.orchestra.run.smithi125:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-10-09T11:01:18.824 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T11:01:18.824 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T11:01:18.825 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:01:18.846 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:01:18.848 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:01:18.848 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:01:18.848 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:01:18.848 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:01:18.848 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:01:18.848 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:01:18.848 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:01:18.848 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:01:18.879 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:hints = 1 2023-10-09T11:01:18.879 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T11:01:18.879 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object prefix: benchmark_data_smithi125_402869 2023-10-09T11:01:18.879 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T11:01:18.880 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 0 0 0 0 0 0 - 0 2023-10-09T11:01:18.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:18.947 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:18.976 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:18.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:18.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:19.081 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:19.090 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:01:19.127 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:19.368 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:19.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:19.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:19.477 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:19.520 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:19.549 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:19.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:19.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:19.656 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:19.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:19 smithi125 ceph-mon[195507]: failed to encode map e406 with expected crc 2023-10-09T11:01:19.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:19 smithi125 ceph-mon[195507]: 1.18 scrub starts 2023-10-09T11:01:19.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:19 smithi125 ceph-mon[195507]: 1.18 scrub ok 2023-10-09T11:01:19.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:19 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2629800684' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:19.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:19 smithi125 ceph-mon[200197]: failed to encode map e406 with expected crc 2023-10-09T11:01:19.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:19 smithi125 ceph-mon[200197]: 1.18 scrub starts 2023-10-09T11:01:19.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:19 smithi125 ceph-mon[200197]: 1.18 scrub ok 2023-10-09T11:01:19.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:19 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2629800684' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:19.691 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:19 smithi155 ceph-mon[184854]: failed to encode map e406 with expected crc 2023-10-09T11:01:19.691 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:19 smithi155 ceph-mon[184854]: 1.18 scrub starts 2023-10-09T11:01:19.691 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:19 smithi155 ceph-mon[184854]: 1.18 scrub ok 2023-10-09T11:01:19.691 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:19 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2629800684' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:19.693 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:19.722 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:19.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:19.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:19.734 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:19.828 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:19.872 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:19.880 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 1 16 3195 3179 198.676 198.688 0.00784036 0.00501313 2023-10-09T11:01:19.900 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:19.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:19.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:20.005 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:20.061 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:20.106 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T11:01:20.106 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:01:20.106 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [3, 4, 5, 6, 2, 1, 7, 0] 2023-10-09T11:01:20.106 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-09T11:01:20.195 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:20.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:20.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:20.304 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:20.357 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:01:20.357 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:01:20.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.357 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:4467 10.4 10:23dd9a61:::benchmark_data_smithi125_402869_object4466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-10-09T11:01:20.357 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:20.345791+0000", 2023-10-09T11:01:20.357 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0035439579999999998, 2023-10-09T11:01:20.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035793040000000002, 2023-10-09T11:01:20.357 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:01:20.357 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:01:20.357 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:01:20.358 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:01:20.358 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:01:20.358 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 4467 2023-10-09T11:01:20.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.358 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:01:20.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:01:20.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.345791+0000", 2023-10-09T11:01:20.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:01:20.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.345791+0000", 2023-10-09T11:01:20.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:01:20.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.345790+0000", 2023-10-09T11:01:20.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:01:20.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:01:20.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.345813+0000", 2023-10-09T11:01:20.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3876999999999999e-05 2023-10-09T11:01:20.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:01:20.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.345827+0000", 2023-10-09T11:01:20.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3696999999999999e-05 2023-10-09T11:01:20.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:01:20.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.345836+0000", 2023-10-09T11:01:20.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9660000000000002e-06 2023-10-09T11:01:20.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:01:20.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.345936+0000", 2023-10-09T11:01:20.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9661999999999995e-05 2023-10-09T11:01:20.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:01:20.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.346081+0000", 2023-10-09T11:01:20.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000145663 2023-10-09T11:01:20.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T11:01:20.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.346266+0000", 2023-10-09T11:01:20.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000184172 2023-10-09T11:01:20.361 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.362 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:01:20.362 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.362 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:4468 10.4 10:2a6a1379:::benchmark_data_smithi125_402869_object4467:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-10-09T11:01:20.362 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:20.345855+0000", 2023-10-09T11:01:20.362 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0034801139999999999, 2023-10-09T11:01:20.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036155699999999998, 2023-10-09T11:01:20.362 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:01:20.362 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:01:20.362 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:01:20.363 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:01:20.363 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:01:20.363 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 4468 2023-10-09T11:01:20.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.363 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:01:20.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:01:20.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.345855+0000", 2023-10-09T11:01:20.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:01:20.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.345855+0000", 2023-10-09T11:01:20.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:01:20.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.345854+0000", 2023-10-09T11:01:20.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:01:20.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:01:20.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.345875+0000", 2023-10-09T11:01:20.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0347e-05 2023-10-09T11:01:20.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:01:20.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.345883+0000", 2023-10-09T11:01:20.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0630000000000006e-06 2023-10-09T11:01:20.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:01:20.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.345893+0000", 2023-10-09T11:01:20.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0236e-05 2023-10-09T11:01:20.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:01:20.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.346673+0000", 2023-10-09T11:01:20.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00078034399999999998 2023-10-09T11:01:20.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:01:20.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.346794+0000", 2023-10-09T11:01:20.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012072499999999999 2023-10-09T11:01:20.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T11:01:20.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.346925+0000", 2023-10-09T11:01:20.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013108799999999999 2023-10-09T11:01:20.366 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.366 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:01:20.366 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.367 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:4466 10.2 e406/403 10:424243fe:::benchmark_data_smithi125_402869_object4465:head v 406'260, mlcod=406'256)", 2023-10-09T11:01:20.367 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:20.346874+0000", 2023-10-09T11:01:20.367 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0024609039999999999, 2023-10-09T11:01:20.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026797320000000002, 2023-10-09T11:01:20.367 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:01:20.367 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:01:20.367 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:01:20.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:01:20.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.346874+0000", 2023-10-09T11:01:20.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:01:20.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.346874+0000", 2023-10-09T11:01:20.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:01:20.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.346873+0000", 2023-10-09T11:01:20.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:01:20.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:01:20.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.346910+0000", 2023-10-09T11:01:20.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7020999999999997e-05 2023-10-09T11:01:20.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:01:20.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.346921+0000", 2023-10-09T11:01:20.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1049e-05 2023-10-09T11:01:20.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:01:20.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.346929+0000", 2023-10-09T11:01:20.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5349999999999999e-06 2023-10-09T11:01:20.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:01:20.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347244+0000", 2023-10-09T11:01:20.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000314727 2023-10-09T11:01:20.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:01:20.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347263+0000", 2023-10-09T11:01:20.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9153000000000001e-05 2023-10-09T11:01:20.370 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.371 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:01:20.371 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.371 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:4460 10.6 e406/403 10:69b9a30a:::benchmark_data_smithi125_402869_object4459:head v 406'283, mlcod=406'281)", 2023-10-09T11:01:20.371 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:20.347229+0000", 2023-10-09T11:01:20.371 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0021056759999999999, 2023-10-09T11:01:20.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023914800000000001, 2023-10-09T11:01:20.371 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:01:20.371 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:01:20.371 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:01:20.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:01:20.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347229+0000", 2023-10-09T11:01:20.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:01:20.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347229+0000", 2023-10-09T11:01:20.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:01:20.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347229+0000", 2023-10-09T11:01:20.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:01:20.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:01:20.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347265+0000", 2023-10-09T11:01:20.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6486999999999998e-05 2023-10-09T11:01:20.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:01:20.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347274+0000", 2023-10-09T11:01:20.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5650000000000002e-06 2023-10-09T11:01:20.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:01:20.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347280+0000", 2023-10-09T11:01:20.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7570000000000001e-06 2023-10-09T11:01:20.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:01:20.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347320+0000", 2023-10-09T11:01:20.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0598e-05 2023-10-09T11:01:20.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:01:20.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347338+0000", 2023-10-09T11:01:20.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7832000000000001e-05 2023-10-09T11:01:20.374 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.375 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:01:20.375 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.375 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:4464 10.6 e406/403 10:61c8c6da:::benchmark_data_smithi125_402869_object4463:head v 406'284, mlcod=406'281)", 2023-10-09T11:01:20.375 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:20.347295+0000", 2023-10-09T11:01:20.375 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0020402910000000001, 2023-10-09T11:01:20.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002393634, 2023-10-09T11:01:20.375 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:01:20.375 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:01:20.375 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:01:20.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:01:20.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347295+0000", 2023-10-09T11:01:20.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:01:20.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347295+0000", 2023-10-09T11:01:20.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:01:20.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347295+0000", 2023-10-09T11:01:20.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:01:20.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.377 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:01:20.377 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347330+0000", 2023-10-09T11:01:20.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5782999999999999e-05 2023-10-09T11:01:20.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.377 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:01:20.377 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347337+0000", 2023-10-09T11:01:20.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8809999999999998e-06 2023-10-09T11:01:20.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:01:20.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347342+0000", 2023-10-09T11:01:20.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.028e-06 2023-10-09T11:01:20.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:01:20.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347752+0000", 2023-10-09T11:01:20.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00040933300000000001 2023-10-09T11:01:20.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:01:20.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.347771+0000", 2023-10-09T11:01:20.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9108000000000001e-05 2023-10-09T11:01:20.379 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.379 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:01:20.379 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.379 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:4470 10.d 10:b7d8ba1a:::benchmark_data_smithi125_402869_object4469:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-10-09T11:01:20.379 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:20.348298+0000", 2023-10-09T11:01:20.379 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.001037254, 2023-10-09T11:01:20.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014651589999999999, 2023-10-09T11:01:20.379 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:01:20.379 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:01:20.379 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:01:20.380 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:01:20.380 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:01:20.380 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 4470 2023-10-09T11:01:20.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.380 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:01:20.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:01:20.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.348298+0000", 2023-10-09T11:01:20.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:01:20.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.348298+0000", 2023-10-09T11:01:20.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:01:20.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.348297+0000", 2023-10-09T11:01:20.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:01:20.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:01:20.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.348318+0000", 2023-10-09T11:01:20.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1827999999999998e-05 2023-10-09T11:01:20.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:01:20.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.348329+0000", 2023-10-09T11:01:20.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0164e-05 2023-10-09T11:01:20.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:01:20.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.348334+0000", 2023-10-09T11:01:20.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7989999999999999e-06 2023-10-09T11:01:20.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:01:20.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.348383+0000", 2023-10-09T11:01:20.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8576999999999999e-05 2023-10-09T11:01:20.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:01:20.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.348546+0000", 2023-10-09T11:01:20.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000163077 2023-10-09T11:01:20.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:01:20.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.348697+0000", 2023-10-09T11:01:20.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015145999999999999 2023-10-09T11:01:20.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.384 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.384 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:01:20.384 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.349775+0000", 2023-10-09T11:01:20.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001077148 2023-10-09T11:01:20.384 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.384 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:01:20.384 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.384 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.384 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.384 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:4472 10.d 10:b00192c8:::benchmark_data_smithi125_402869_object4471:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-10-09T11:01:20.384 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:20.348346+0000", 2023-10-09T11:01:20.384 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.00098880999999999991, 2023-10-09T11:01:20.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001510342, 2023-10-09T11:01:20.385 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:01:20.385 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:01:20.385 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:01:20.385 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:01:20.385 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:01:20.385 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 4472 2023-10-09T11:01:20.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.385 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:01:20.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:01:20.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.348346+0000", 2023-10-09T11:01:20.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:01:20.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.348346+0000", 2023-10-09T11:01:20.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:01:20.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.348346+0000", 2023-10-09T11:01:20.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:01:20.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:01:20.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.348365+0000", 2023-10-09T11:01:20.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9298000000000001e-05 2023-10-09T11:01:20.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:01:20.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.348372+0000", 2023-10-09T11:01:20.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6630000000000004e-06 2023-10-09T11:01:20.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:01:20.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.348377+0000", 2023-10-09T11:01:20.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5799999999999999e-06 2023-10-09T11:01:20.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:01:20.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.349135+0000", 2023-10-09T11:01:20.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00075742500000000005 2023-10-09T11:01:20.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:01:20.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.349283+0000", 2023-10-09T11:01:20.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014788 2023-10-09T11:01:20.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:01:20.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.349423+0000", 2023-10-09T11:01:20.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000140715 2023-10-09T11:01:20.389 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.389 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:01:20.389 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.389 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:4475 10.4 10:2ba006cc:::benchmark_data_smithi125_402869_object4474:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-10-09T11:01:20.389 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:20.349547+0000", 2023-10-09T11:01:20.389 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4294967295.9997878, 2023-10-09T11:01:20.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00039543400000000001, 2023-10-09T11:01:20.390 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:01:20.390 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:01:20.390 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:01:20.390 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:01:20.390 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:01:20.390 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 4475 2023-10-09T11:01:20.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.390 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:01:20.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:01:20.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.349547+0000", 2023-10-09T11:01:20.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:01:20.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.349547+0000", 2023-10-09T11:01:20.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:20.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:01:20.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.349547+0000", 2023-10-09T11:01:20.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:01:20.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.392 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:01:20.392 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.349567+0000", 2023-10-09T11:01:20.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0163e-05 2023-10-09T11:01:20.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.392 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.392 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:01:20.392 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.349576+0000", 2023-10-09T11:01:20.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0580000000000004e-06 2023-10-09T11:01:20.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.392 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.392 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:01:20.392 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.349582+0000", 2023-10-09T11:01:20.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1630000000000001e-06 2023-10-09T11:01:20.393 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.393 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:01:20.393 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.349789+0000", 2023-10-09T11:01:20.393 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020711699999999999 2023-10-09T11:01:20.393 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:20.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:20.393 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:01:20.393 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:20.349914+0000", 2023-10-09T11:01:20.393 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012479400000000001 2023-10-09T11:01:20.393 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.393 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:01:20.393 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.394 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:20.394 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T11:01:20.394 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 8 2023-10-09T11:01:20.394 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:01:20.394 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:20.511 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:20.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:20.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:20.617 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:20.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:20 smithi125 ceph-mon[195507]: pgmap v1010: 119 pgs: 119 active+clean; 470 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 25 KiB/s rd, 66 KiB/s wr, 20 op/s 2023-10-09T11:01:20.656 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:20 smithi125 ceph-mon[200197]: pgmap v1010: 119 pgs: 119 active+clean; 470 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 25 KiB/s rd, 66 KiB/s wr, 20 op/s 2023-10-09T11:01:20.661 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:20 smithi155 ceph-mon[184854]: pgmap v1010: 119 pgs: 119 active+clean; 470 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 25 KiB/s rd, 66 KiB/s wr, 20 op/s 2023-10-09T11:01:20.739 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:20.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:20.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:20.782 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T11:01:20.844 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:20.879 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 2 16 5914 5898 184.299 169.938 0.00812994 0.00540221 2023-10-09T11:01:20.923 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:20.949 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:20.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:20.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:21.059 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:21.064 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:21.092 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:21.118 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:21.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:21.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:21.218 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:01:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:01:21] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2023-10-09T11:01:21.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:21.262 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:21.286 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:21.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:21.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:21.390 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:21.403 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:01:21.444 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:21.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:21 smithi125 ceph-mon[195507]: 1.8 scrub starts 2023-10-09T11:01:21.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:21 smithi125 ceph-mon[195507]: 1.8 scrub ok 2023-10-09T11:01:21.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:21 smithi125 ceph-mon[200197]: 1.8 scrub starts 2023-10-09T11:01:21.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:21 smithi125 ceph-mon[200197]: 1.8 scrub ok 2023-10-09T11:01:21.582 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:21.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:21.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:21.686 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:21 smithi155 ceph-mon[184854]: 1.8 scrub starts 2023-10-09T11:01:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:21 smithi155 ceph-mon[184854]: 1.8 scrub ok 2023-10-09T11:01:21.739 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:21.880 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 3 16 8342 8326 173.424 151.75 0.00590911 0.00575854 2023-10-09T11:01:22.471 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:01:22.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:22 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2961258104' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:01:22.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:22 smithi155 ceph-mon[184854]: pgmap v1011: 119 pgs: 119 active+clean; 302 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 8.0 KiB/s rd, 48 MiB/s wr, 777 op/s 2023-10-09T11:01:22.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:22 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2961258104' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:01:22.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:22 smithi125 ceph-mon[195507]: pgmap v1011: 119 pgs: 119 active+clean; 302 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 8.0 KiB/s rd, 48 MiB/s wr, 777 op/s 2023-10-09T11:01:22.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:22 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2961258104' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:01:22.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:22 smithi125 ceph-mon[200197]: pgmap v1011: 119 pgs: 119 active+clean; 302 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 8.0 KiB/s rd, 48 MiB/s wr, 777 op/s 2023-10-09T11:01:22.880 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 4 16 11278 11262 175.93 183.5 0.00432828 0.00567735 2023-10-09T11:01:23.289 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:23.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:23.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:23.395 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:23.438 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:23.491 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:23.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:23.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:23.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:01:23.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T11:01:23.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[195507]: failed to encode map e407 with expected crc 2023-10-09T11:01:23.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2961258104' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:01:23.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[195507]: osdmap e407: 8 total, 8 up, 8 in 2023-10-09T11:01:23.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[195507]: failed to encode map e407 with expected crc 2023-10-09T11:01:23.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[195507]: 1.f scrub starts 2023-10-09T11:01:23.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[195507]: 1.f scrub ok 2023-10-09T11:01:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:01:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T11:01:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[200197]: failed to encode map e407 with expected crc 2023-10-09T11:01:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2961258104' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:01:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[200197]: osdmap e407: 8 total, 8 up, 8 in 2023-10-09T11:01:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[200197]: failed to encode map e407 with expected crc 2023-10-09T11:01:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[200197]: 1.f scrub starts 2023-10-09T11:01:23.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:23 smithi125 ceph-mon[200197]: 1.f scrub ok 2023-10-09T11:01:23.586 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:23.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:23.639 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:23.667 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:23.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:23.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:23 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:01:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:23 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T11:01:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:23 smithi155 ceph-mon[184854]: failed to encode map e407 with expected crc 2023-10-09T11:01:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:23 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2961258104' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:01:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:23 smithi155 ceph-mon[184854]: osdmap e407: 8 total, 8 up, 8 in 2023-10-09T11:01:23.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:23 smithi155 ceph-mon[184854]: failed to encode map e407 with expected crc 2023-10-09T11:01:23.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:23 smithi155 ceph-mon[184854]: 1.f scrub starts 2023-10-09T11:01:23.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:23 smithi155 ceph-mon[184854]: 1.f scrub ok 2023-10-09T11:01:23.772 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:23.822 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:23.853 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:23.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:23.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:23.880 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 5 16 14059 14043 175.5 173.812 0.00461913 0.00569308 2023-10-09T11:01:23.958 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:24.004 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:24.038 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:24.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:24.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:24.143 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:24.188 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:24.223 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:24.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:24.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:24.329 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:24.385 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:24 smithi155 ceph-mon[184854]: pgmap v1013: 119 pgs: 119 active+clean; 522 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 6.9 KiB/s rd, 72 MiB/s wr, 1.15k op/s 2023-10-09T11:01:24.761 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:24.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:24.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:24.776 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:24 smithi125 ceph-mon[195507]: pgmap v1013: 119 pgs: 119 active+clean; 522 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 6.9 KiB/s rd, 72 MiB/s wr, 1.15k op/s 2023-10-09T11:01:24.776 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:24 smithi125 ceph-mon[200197]: pgmap v1013: 119 pgs: 119 active+clean; 522 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 6.9 KiB/s rd, 72 MiB/s wr, 1.15k op/s 2023-10-09T11:01:24.868 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:24.881 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 6 16 16421 16405 170.852 147.625 0.00788761 0.00584461 2023-10-09T11:01:24.904 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:24.932 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:24.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:24.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:25.037 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:25.080 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:25.095 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:25.108 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:25.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:25.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:25.212 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:25.266 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:25.646 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.7. 2023-10-09T11:01:25.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1733204812' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T11:01:25.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:25 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T11:01:25.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:25 smithi125 ceph-mon[200197]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T11:01:25.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:25 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T11:01:25.562+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e407 definitely_dead 0 2023-10-09T11:01:25.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1733204812' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T11:01:25.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:25 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T11:01:25.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:25 smithi125 ceph-mon[195507]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T11:01:25.881 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 7 16 17906 17890 159.703 92.8125 0.00143425 0.00607295 2023-10-09T11:01:25.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1733204812' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T11:01:25.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:25 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T11:01:25.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:25 smithi155 ceph-mon[184854]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T11:01:26.447 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:26.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:26.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:26.551 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:26.580 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:26.700 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:26.702 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[200197]: 1.13 scrub starts 2023-10-09T11:01:26.702 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[200197]: 1.13 scrub ok 2023-10-09T11:01:26.702 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[200197]: failed to encode map e408 with expected crc 2023-10-09T11:01:26.702 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T11:01:26.702 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[200197]: osdmap e408: 8 total, 8 up, 7 in 2023-10-09T11:01:26.702 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[200197]: failed to encode map e408 with expected crc 2023-10-09T11:01:26.702 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[200197]: failed to encode map e408 with expected crc 2023-10-09T11:01:26.702 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[200197]: failed to encode map e408 with expected crc 2023-10-09T11:01:26.703 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[200197]: pgmap v1015: 119 pgs: 119 active+clean; 522 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T11:01:26.703 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:26.703 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[200197]: osdmap e409: 8 total, 8 up, 7 in 2023-10-09T11:01:26.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[195507]: 1.13 scrub starts 2023-10-09T11:01:26.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[195507]: 1.13 scrub ok 2023-10-09T11:01:26.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[195507]: failed to encode map e408 with expected crc 2023-10-09T11:01:26.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T11:01:26.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[195507]: osdmap e408: 8 total, 8 up, 7 in 2023-10-09T11:01:26.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[195507]: failed to encode map e408 with expected crc 2023-10-09T11:01:26.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[195507]: failed to encode map e408 with expected crc 2023-10-09T11:01:26.704 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[195507]: failed to encode map e408 with expected crc 2023-10-09T11:01:26.704 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[195507]: pgmap v1015: 119 pgs: 119 active+clean; 522 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T11:01:26.704 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:26.704 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:26 smithi125 ceph-mon[195507]: osdmap e409: 8 total, 8 up, 7 in 2023-10-09T11:01:26.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:26.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:26.801 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:01:26.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:26.870 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:26.881 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 8 16 17906 17890 139.742 0 - 0.00607295 2023-10-09T11:01:26.943 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:26.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:26.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:26 smithi155 ceph-mon[184854]: 1.13 scrub starts 2023-10-09T11:01:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:26 smithi155 ceph-mon[184854]: 1.13 scrub ok 2023-10-09T11:01:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:26 smithi155 ceph-mon[184854]: failed to encode map e408 with expected crc 2023-10-09T11:01:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:26 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T11:01:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:26 smithi155 ceph-mon[184854]: osdmap e408: 8 total, 8 up, 7 in 2023-10-09T11:01:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:26 smithi155 ceph-mon[184854]: failed to encode map e408 with expected crc 2023-10-09T11:01:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:26 smithi155 ceph-mon[184854]: failed to encode map e408 with expected crc 2023-10-09T11:01:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:26 smithi155 ceph-mon[184854]: failed to encode map e408 with expected crc 2023-10-09T11:01:26.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:26 smithi155 ceph-mon[184854]: pgmap v1015: 119 pgs: 119 active+clean; 522 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T11:01:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:26 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:26.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:26 smithi155 ceph-mon[184854]: osdmap e409: 8 total, 8 up, 7 in 2023-10-09T11:01:27.047 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:27.076 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:27.095 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:27.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:27.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:27.202 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:27.229 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:27.248 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:27.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:27.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:27.351 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:27.378 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:27.396 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:27.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:27.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:27.500 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:27.527 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:27.545 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:27.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:27.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:27.551 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:27.566 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:01:27.566 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:01:27.566 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T11:01:27.566 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T11:01:27.566 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:01:27.650 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:27.655 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:01:27.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[195507]: failed to encode map e409 with expected crc 2023-10-09T11:01:27.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[195507]: failed to encode map e409 with expected crc 2023-10-09T11:01:27.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[195507]: failed to encode map e409 with expected crc 2023-10-09T11:01:27.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[195507]: failed to encode map e409 with expected crc 2023-10-09T11:01:27.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[195507]: failed to encode map e409 with expected crc 2023-10-09T11:01:27.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[195507]: failed to encode map e409 with expected crc 2023-10-09T11:01:27.709 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:27.730 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:01:27.759 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:01:27.778 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:01:27.796 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:01:27.815 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:01:27.833 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:01:27.869 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:01:27.873 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:01:27.873 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:01:27.873 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:01:27.873 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:01:27.874 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:01:27.874 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:01:27.874 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:01:27.874 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:01:27.881 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 9 16 18067 18051 125.334 5.03125 0.0104111 0.00797187 2023-10-09T11:01:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:27 smithi155 ceph-mon[184854]: failed to encode map e409 with expected crc 2023-10-09T11:01:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:27 smithi155 ceph-mon[184854]: failed to encode map e409 with expected crc 2023-10-09T11:01:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:27 smithi155 ceph-mon[184854]: failed to encode map e409 with expected crc 2023-10-09T11:01:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:27 smithi155 ceph-mon[184854]: failed to encode map e409 with expected crc 2023-10-09T11:01:27.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:27 smithi155 ceph-mon[184854]: failed to encode map e409 with expected crc 2023-10-09T11:01:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:27 smithi155 ceph-mon[184854]: failed to encode map e409 with expected crc 2023-10-09T11:01:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:27 smithi155 ceph-mon[184854]: failed to encode map e409 with expected crc 2023-10-09T11:01:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:27 smithi155 ceph-mon[184854]: failed to encode map e409 with expected crc 2023-10-09T11:01:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:27 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4293203450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:27.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:27 smithi155 ceph-mon[184854]: osdmap e410: 8 total, 8 up, 7 in 2023-10-09T11:01:28.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[195507]: failed to encode map e409 with expected crc 2023-10-09T11:01:28.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[195507]: failed to encode map e409 with expected crc 2023-10-09T11:01:28.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4293203450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:28.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[195507]: osdmap e410: 8 total, 8 up, 7 in 2023-10-09T11:01:28.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[200197]: failed to encode map e409 with expected crc 2023-10-09T11:01:28.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[200197]: failed to encode map e409 with expected crc 2023-10-09T11:01:28.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[200197]: failed to encode map e409 with expected crc 2023-10-09T11:01:28.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[200197]: failed to encode map e409 with expected crc 2023-10-09T11:01:28.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[200197]: failed to encode map e409 with expected crc 2023-10-09T11:01:28.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[200197]: failed to encode map e409 with expected crc 2023-10-09T11:01:28.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[200197]: failed to encode map e409 with expected crc 2023-10-09T11:01:28.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[200197]: failed to encode map e409 with expected crc 2023-10-09T11:01:28.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4293203450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:28.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:27 smithi125 ceph-mon[200197]: osdmap e410: 8 total, 8 up, 7 in 2023-10-09T11:01:28.105 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:28.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:28.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:28.210 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:28.266 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:28.446 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:28.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:28.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:28.497 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T11:01:28.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:28.586 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:28.616 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:28.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:28.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:28.720 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:28.759 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:28.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:28.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[195507]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[195507]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[195507]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[195507]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[195507]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[195507]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[195507]: pgmap v1018: 119 pgs: 2 unknown, 29 peering, 88 active+clean; 901 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T11:01:28.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[195507]: osdmap e411: 8 total, 8 up, 7 in 2023-10-09T11:01:28.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[195507]: failed to encode map e411 with expected crc 2023-10-09T11:01:28.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[195507]: failed to encode map e411 with expected crc 2023-10-09T11:01:28.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[195507]: failed to encode map e411 with expected crc 2023-10-09T11:01:28.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 3 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T11:01:28.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[200197]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[200197]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[200197]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[200197]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[200197]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[200197]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[200197]: pgmap v1018: 119 pgs: 2 unknown, 29 peering, 88 active+clean; 901 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T11:01:28.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[200197]: osdmap e411: 8 total, 8 up, 7 in 2023-10-09T11:01:28.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[200197]: failed to encode map e411 with expected crc 2023-10-09T11:01:28.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[200197]: failed to encode map e411 with expected crc 2023-10-09T11:01:28.789 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[200197]: failed to encode map e411 with expected crc 2023-10-09T11:01:28.789 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:28 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 3 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T11:01:28.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:28.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:28.881 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 10 16 19789 19773 123.562 107.625 0.0124406 0.00808459 2023-10-09T11:01:28.890 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:28.924 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:28 smithi155 ceph-mon[184854]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.924 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:28 smithi155 ceph-mon[184854]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.924 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:28 smithi155 ceph-mon[184854]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.924 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:28 smithi155 ceph-mon[184854]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.924 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:28 smithi155 ceph-mon[184854]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.925 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:28 smithi155 ceph-mon[184854]: failed to encode map e410 with expected crc 2023-10-09T11:01:28.925 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:28 smithi155 ceph-mon[184854]: pgmap v1018: 119 pgs: 2 unknown, 29 peering, 88 active+clean; 901 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-09T11:01:28.925 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:28 smithi155 ceph-mon[184854]: osdmap e411: 8 total, 8 up, 7 in 2023-10-09T11:01:28.925 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:28 smithi155 ceph-mon[184854]: failed to encode map e411 with expected crc 2023-10-09T11:01:28.925 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:28 smithi155 ceph-mon[184854]: failed to encode map e411 with expected crc 2023-10-09T11:01:28.925 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:28 smithi155 ceph-mon[184854]: failed to encode map e411 with expected crc 2023-10-09T11:01:28.925 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:28 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 3 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T11:01:28.927 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:28.958 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:28.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:28.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:29.064 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:29.109 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:29.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:29.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:29.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:29.191 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:29.243 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:29.281 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:29.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:29.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:29.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:29.414 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:29.474 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:29.505 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:29.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:29.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:29.612 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:29.655 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:29.684 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:29.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:29.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:29.788 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:29.791 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:29 smithi125 ceph-mon[200197]: failed to encode map e411 with expected crc 2023-10-09T11:01:29.791 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:29 smithi125 ceph-mon[200197]: failed to encode map e411 with expected crc 2023-10-09T11:01:29.791 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:29 smithi125 ceph-mon[200197]: 1.1d scrub starts 2023-10-09T11:01:29.791 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:29 smithi125 ceph-mon[200197]: failed to encode map e412 with expected crc 2023-10-09T11:01:29.791 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:29 smithi125 ceph-mon[200197]: osdmap e412: 8 total, 8 up, 7 in 2023-10-09T11:01:29.791 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:29 smithi125 ceph-mon[200197]: failed to encode map e412 with expected crc 2023-10-09T11:01:29.791 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:29 smithi125 ceph-mon[200197]: failed to encode map e412 with expected crc 2023-10-09T11:01:29.791 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:29 smithi125 ceph-mon[195507]: failed to encode map e411 with expected crc 2023-10-09T11:01:29.791 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:29 smithi125 ceph-mon[195507]: failed to encode map e411 with expected crc 2023-10-09T11:01:29.791 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:29 smithi125 ceph-mon[195507]: 1.1d scrub starts 2023-10-09T11:01:29.792 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:29 smithi125 ceph-mon[195507]: failed to encode map e412 with expected crc 2023-10-09T11:01:29.792 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:29 smithi125 ceph-mon[195507]: osdmap e412: 8 total, 8 up, 7 in 2023-10-09T11:01:29.792 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:29 smithi125 ceph-mon[195507]: failed to encode map e412 with expected crc 2023-10-09T11:01:29.792 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:29 smithi125 ceph-mon[195507]: failed to encode map e412 with expected crc 2023-10-09T11:01:29.845 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:29.881 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 11 16 21433 21417 121.669 102.75 0.0145882 0.00821275 2023-10-09T11:01:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:29 smithi155 ceph-mon[184854]: failed to encode map e411 with expected crc 2023-10-09T11:01:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:29 smithi155 ceph-mon[184854]: failed to encode map e411 with expected crc 2023-10-09T11:01:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:29 smithi155 ceph-mon[184854]: 1.1d scrub starts 2023-10-09T11:01:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:29 smithi155 ceph-mon[184854]: failed to encode map e412 with expected crc 2023-10-09T11:01:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:29 smithi155 ceph-mon[184854]: osdmap e412: 8 total, 8 up, 7 in 2023-10-09T11:01:29.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:29 smithi155 ceph-mon[184854]: failed to encode map e412 with expected crc 2023-10-09T11:01:29.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:29 smithi155 ceph-mon[184854]: failed to encode map e412 with expected crc 2023-10-09T11:01:30.705 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:01:30.866 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:01:30.881 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 12 16 23384 23368 121.691 121.938 0.00499947 0.00821044 2023-10-09T11:01:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:30 smithi155 ceph-mon[184854]: failed to encode map e412 with expected crc 2023-10-09T11:01:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:30 smithi155 ceph-mon[184854]: failed to encode map e412 with expected crc 2023-10-09T11:01:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:30 smithi155 ceph-mon[184854]: failed to encode map e412 with expected crc 2023-10-09T11:01:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:30 smithi155 ceph-mon[184854]: pgmap v1021: 119 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 unknown, 29 peering, 81 active+clean; 1.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 150 MiB/s wr, 2.40k op/s; 3396/53955 objects degraded (6.294%); 21 KiB/s, 0 objects/s recovering 2023-10-09T11:01:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:30 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3878228524' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:01:30.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:30 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:01:30.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:30.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:01:30.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:30 smithi155 ceph-mon[184854]: Health check failed: Degraded data redundancy: 3396/53955 objects degraded (6.294%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:01:31.008 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[195507]: failed to encode map e412 with expected crc 2023-10-09T11:01:31.009 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[195507]: failed to encode map e412 with expected crc 2023-10-09T11:01:31.009 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[195507]: failed to encode map e412 with expected crc 2023-10-09T11:01:31.009 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[195507]: pgmap v1021: 119 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 unknown, 29 peering, 81 active+clean; 1.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 150 MiB/s wr, 2.40k op/s; 3396/53955 objects degraded (6.294%); 21 KiB/s, 0 objects/s recovering 2023-10-09T11:01:31.009 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3878228524' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:01:31.009 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:01:31.009 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:31.009 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:01:31.009 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[195507]: Health check failed: Degraded data redundancy: 3396/53955 objects degraded (6.294%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:01:31.009 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[200197]: failed to encode map e412 with expected crc 2023-10-09T11:01:31.010 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[200197]: failed to encode map e412 with expected crc 2023-10-09T11:01:31.010 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[200197]: failed to encode map e412 with expected crc 2023-10-09T11:01:31.010 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[200197]: pgmap v1021: 119 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 unknown, 29 peering, 81 active+clean; 1.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 150 MiB/s wr, 2.40k op/s; 3396/53955 objects degraded (6.294%); 21 KiB/s, 0 objects/s recovering 2023-10-09T11:01:31.010 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3878228524' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:01:31.010 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:01:31.010 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:31.010 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:01:31.010 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:30 smithi125 ceph-mon[200197]: Health check failed: Degraded data redundancy: 3396/53955 objects degraded (6.294%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:01:31.307 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:31.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:31.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:31.317 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:01:31] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-10-09T11:01:31.417 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:31.452 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:31.478 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:31.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:31.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:31.582 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:31.621 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:31.650 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:31.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:31.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:31.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:31 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:31.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:31 smithi155 ceph-mon[184854]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:31 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:01:31.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:31 smithi155 ceph-mon[184854]: osdmap e413: 8 total, 8 up, 7 in 2023-10-09T11:01:31.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:31 smithi155 ceph-mon[184854]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:31 smithi155 ceph-mon[184854]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:31 smithi155 ceph-mon[184854]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:31 smithi155 ceph-mon[184854]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:31 smithi155 ceph-mon[184854]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:31 smithi155 ceph-mon[184854]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:31 smithi155 ceph-mon[184854]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.756 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:31.798 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:31.818 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:31.852 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:31.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:31.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:31.884 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 13 16 25001 24985 120.082 101.062 0.0127479 0.00832192 2023-10-09T11:01:31.907 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:01:31.907 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:01:31.923 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 0] out_osds: [7] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T11:01:31.923 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:01:31.923 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T11:01:31.923 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T11:01:31.923 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T11:01:31.923 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T11:01:31.939 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:31.940 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[195507]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.940 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:01:31.940 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[195507]: osdmap e413: 8 total, 8 up, 7 in 2023-10-09T11:01:31.940 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[195507]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.940 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[195507]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.940 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[195507]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.940 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[195507]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.940 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[195507]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.940 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[195507]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.940 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[195507]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.941 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:31.941 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[200197]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.941 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:01:31.941 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[200197]: osdmap e413: 8 total, 8 up, 7 in 2023-10-09T11:01:31.941 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[200197]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.941 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[200197]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.941 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[200197]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.941 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[200197]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.941 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[200197]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.941 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[200197]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.942 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:31 smithi125 ceph-mon[200197]: failed to encode map e413 with expected crc 2023-10-09T11:01:31.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:32.014 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:32.041 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:32.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:32.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:32.146 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:32.190 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:32.222 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:32.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:32.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:32.330 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:32.400 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:32.410 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T11:01:32.879 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:32.884 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 14 16 26531 26515 118.335 95.625 0.00332065 0.00844234 2023-10-09T11:01:32.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:32.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:32 smithi155 ceph-mon[184854]: pgmap v1023: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 111 active+clean; 1.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s; 3396/64389 objects degraded (5.274%); 400/64389 objects misplaced (0.621%); 12 MiB/s, 2 keys/s, 188 objects/s recovering 2023-10-09T11:01:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:32 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 8 pgs peering) 2023-10-09T11:01:32.981 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:33.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:33.040 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:33.067 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:33.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:32 smithi125 ceph-mon[200197]: pgmap v1023: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 111 active+clean; 1.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s; 3396/64389 objects degraded (5.274%); 400/64389 objects misplaced (0.621%); 12 MiB/s, 2 keys/s, 188 objects/s recovering 2023-10-09T11:01:33.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:32 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 8 pgs peering) 2023-10-09T11:01:33.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:32 smithi125 ceph-mon[195507]: pgmap v1023: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 111 active+clean; 1.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s; 3396/64389 objects degraded (5.274%); 400/64389 objects misplaced (0.621%); 12 MiB/s, 2 keys/s, 188 objects/s recovering 2023-10-09T11:01:33.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:32 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 8 pgs peering) 2023-10-09T11:01:33.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:33.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:33.136 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:33.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:33.226 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:33.689 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:01:33.884 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 15 16 27264 27248 113.501 45.8125 0.00213698 0.00844333 2023-10-09T11:01:33.903 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:33.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:33.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:34.008 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:34.043 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:01:34.044 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:34.067 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:01:34.087 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:01:34.106 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:01:34.125 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:01:34.145 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:01:34.171 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:01:34.210 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:01:34.219 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:34.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:34.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:34.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:01:34.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:01:34.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:01:34.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:01:34.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:01:34.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:01:34.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:01:34.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:01:34.326 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:34.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:34 smithi125 ceph-mon[195507]: osdmap e414: 8 total, 8 up, 7 in 2023-10-09T11:01:34.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:34 smithi125 ceph-mon[195507]: failed to encode map e414 with expected crc 2023-10-09T11:01:34.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:34 smithi125 ceph-mon[195507]: failed to encode map e414 with expected crc 2023-10-09T11:01:34.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:34 smithi125 ceph-mon[195507]: pgmap v1025: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 3396/75618 objects degraded (4.491%); 400/75618 objects misplaced (0.529%); 8.5 MiB/s, 1 keys/s, 137 objects/s recovering 2023-10-09T11:01:34.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:34 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/68975995' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:34.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:34 smithi125 ceph-mon[200197]: osdmap e414: 8 total, 8 up, 7 in 2023-10-09T11:01:34.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:34 smithi125 ceph-mon[200197]: failed to encode map e414 with expected crc 2023-10-09T11:01:34.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:34 smithi125 ceph-mon[200197]: failed to encode map e414 with expected crc 2023-10-09T11:01:34.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:34 smithi125 ceph-mon[200197]: pgmap v1025: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 3396/75618 objects degraded (4.491%); 400/75618 objects misplaced (0.529%); 8.5 MiB/s, 1 keys/s, 137 objects/s recovering 2023-10-09T11:01:34.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:34 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/68975995' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:34.389 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:34.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:34 smithi155 ceph-mon[184854]: osdmap e414: 8 total, 8 up, 7 in 2023-10-09T11:01:34.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:34 smithi155 ceph-mon[184854]: failed to encode map e414 with expected crc 2023-10-09T11:01:34.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:34 smithi155 ceph-mon[184854]: failed to encode map e414 with expected crc 2023-10-09T11:01:34.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:34 smithi155 ceph-mon[184854]: pgmap v1025: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 3396/75618 objects degraded (4.491%); 400/75618 objects misplaced (0.529%); 8.5 MiB/s, 1 keys/s, 137 objects/s recovering 2023-10-09T11:01:34.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:34 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/68975995' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:34.884 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 16 16 28211 28195 110.106 59.1875 0.00540637 0.00907546 2023-10-09T11:01:35.181 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:35.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:35.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:35.295 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:35.334 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:35.360 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:35.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:35.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:35.464 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:35 smithi125 ceph-mon[195507]: failed to encode map e415 with expected crc 2023-10-09T11:01:35.464 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:35 smithi125 ceph-mon[195507]: failed to encode map e415 with expected crc 2023-10-09T11:01:35.464 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:35 smithi125 ceph-mon[195507]: osdmap e415: 8 total, 8 up, 7 in 2023-10-09T11:01:35.464 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:35 smithi125 ceph-mon[195507]: failed to encode map e415 with expected crc 2023-10-09T11:01:35.465 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:35 smithi125 ceph-mon[195507]: failed to encode map e415 with expected crc 2023-10-09T11:01:35.465 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:35 smithi125 ceph-mon[200197]: failed to encode map e415 with expected crc 2023-10-09T11:01:35.465 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:35 smithi125 ceph-mon[200197]: failed to encode map e415 with expected crc 2023-10-09T11:01:35.465 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:35 smithi125 ceph-mon[200197]: osdmap e415: 8 total, 8 up, 7 in 2023-10-09T11:01:35.465 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:35 smithi125 ceph-mon[200197]: failed to encode map e415 with expected crc 2023-10-09T11:01:35.465 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:35 smithi125 ceph-mon[200197]: failed to encode map e415 with expected crc 2023-10-09T11:01:35.467 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:35.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:35 smithi155 ceph-mon[184854]: failed to encode map e415 with expected crc 2023-10-09T11:01:35.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:35 smithi155 ceph-mon[184854]: failed to encode map e415 with expected crc 2023-10-09T11:01:35.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:35 smithi155 ceph-mon[184854]: osdmap e415: 8 total, 8 up, 7 in 2023-10-09T11:01:35.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:35 smithi155 ceph-mon[184854]: failed to encode map e415 with expected crc 2023-10-09T11:01:35.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:35 smithi155 ceph-mon[184854]: failed to encode map e415 with expected crc 2023-10-09T11:01:35.547 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:35.745 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:35.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:35.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:35.796 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:35.855 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:35.884 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 17 16 29619 29603 108.806 88 0.0123733 0.00918297 2023-10-09T11:01:35.895 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:35.922 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:35.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:35.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:35.933 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:01:35.934 INFO:teuthology.orchestra.run.smithi125.stdout: "success": "" 2023-10-09T11:01:35.934 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:01:36.028 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:36.085 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:36.309 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:01:36.310 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:01:36.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:36.310 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:30058 10.8 e415/409 10:1497d000:::benchmark_data_smithi125_402869_object30057:head v 415'1873, mlcod=406'543)", 2023-10-09T11:01:36.310 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:36.305416+0000", 2023-10-09T11:01:36.310 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0034069479999999999, 2023-10-09T11:01:36.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003445198, 2023-10-09T11:01:36.310 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:01:36.310 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:01:36.310 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:01:36.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:36.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:01:36.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:36.305416+0000", 2023-10-09T11:01:36.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:36.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:36.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:36.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:01:36.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:36.305416+0000", 2023-10-09T11:01:36.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:36.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:36.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:36.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:01:36.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:36.305415+0000", 2023-10-09T11:01:36.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:01:36.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:36.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:36.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:01:36.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:36.305443+0000", 2023-10-09T11:01:36.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7733e-05 2023-10-09T11:01:36.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:36.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:36.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:01:36.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:36.305457+0000", 2023-10-09T11:01:36.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3944000000000001e-05 2023-10-09T11:01:36.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:36.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:36.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:01:36.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:36.305465+0000", 2023-10-09T11:01:36.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0439999999999996e-06 2023-10-09T11:01:36.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:36.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:36.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:01:36.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:36.305611+0000", 2023-10-09T11:01:36.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014594599999999999 2023-10-09T11:01:36.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:36.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:36.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:01:36.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:36.305634+0000", 2023-10-09T11:01:36.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3773000000000001e-05 2023-10-09T11:01:36.314 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:36.314 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:01:36.314 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:36.314 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:36.314 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T11:01:36.314 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 1 2023-10-09T11:01:36.314 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:01:36.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:36 smithi125 ceph-mon[195507]: pgmap v1027: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s; 3396/75618 objects degraded (4.491%); 400/75618 objects misplaced (0.529%); 7.8 MiB/s, 1 keys/s, 125 objects/s recovering 2023-10-09T11:01:36.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:36 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:01:36.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:36 smithi125 ceph-mon[200197]: pgmap v1027: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s; 3396/75618 objects degraded (4.491%); 400/75618 objects misplaced (0.529%); 7.8 MiB/s, 1 keys/s, 125 objects/s recovering 2023-10-09T11:01:36.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:36 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:01:36.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:36 smithi155 ceph-mon[184854]: pgmap v1027: 119 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s; 3396/75618 objects degraded (4.491%); 400/75618 objects misplaced (0.529%); 7.8 MiB/s, 1 keys/s, 125 objects/s recovering 2023-10-09T11:01:36.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:36 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:01:36.884 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 18 16 31041 31025 107.698 88.875 0.00502595 0.00927959 2023-10-09T11:01:37.500 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:01:37.884 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 19 16 32599 32583 107.155 97.375 0.00767808 0.00932708 2023-10-09T11:01:38.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:37 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 3396/85851 objects degraded (3.956%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:01:38.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:37 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 3396/85851 objects degraded (3.956%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:01:38.099 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:37 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 3396/85851 objects degraded (3.956%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:01:38.392 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:38.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:38.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:38.414 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T11:01:38.499 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:38.546 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:38.577 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:38.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:38.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:38.629 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:38.685 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:38.723 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:38.800 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:38.801 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:38 smithi155 ceph-mon[184854]: pgmap v1028: 119 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s; 3396/85851 objects degraded (3.956%); 10 MiB/s, 1 keys/s, 165 objects/s recovering 2023-10-09T11:01:38.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:38.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:38.885 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T11:01:38.885760+0000 min lat: 0.0011398 max lat: 2.07371 avg lat: 0.0094397 2023-10-09T11:01:38.885 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T11:01:38.885 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 20 16 33897 33881 105.851 81.125 0.00876421 0.0094397 2023-10-09T11:01:38.903 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:38.948 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:38.958 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:38 smithi125 ceph-mon[195507]: pgmap v1028: 119 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s; 3396/85851 objects degraded (3.956%); 10 MiB/s, 1 keys/s, 165 objects/s recovering 2023-10-09T11:01:38.959 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:38 smithi125 ceph-mon[200197]: pgmap v1028: 119 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s; 3396/85851 objects degraded (3.956%); 10 MiB/s, 1 keys/s, 165 objects/s recovering 2023-10-09T11:01:39.081 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:39.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:39.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:39.186 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:39.228 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:39.629 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:39.886 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 21 16 35069 35053 104.293 73.25 0.0163342 0.00958225 2023-10-09T11:01:40.184 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:01:40.221 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:01:40.253 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:01:40.299 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:01:40.332 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:01:40.370 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:01:40.410 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:01:40.455 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:01:40.473 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:01:40.473 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:01:40.473 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:01:40.473 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:01:40.473 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:01:40.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:01:40.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:01:40.474 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:01:40.483 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:40.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:40.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:40.587 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:40.647 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:40.678 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:40.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:40.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:40.783 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:40.788 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:01:40.837 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:40.886 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 22 16 36240 36224 102.879 73.1875 0.011453 0.00971534 2023-10-09T11:01:41.020 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:40 smithi125 ceph-mon[195507]: pgmap v1029: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 2270/93912 objects degraded (2.417%); 394/93912 objects misplaced (0.420%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T11:01:41.020 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:40 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/314581158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:01:41.020 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:40 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/320152962' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:41.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:40 smithi125 ceph-mon[200197]: pgmap v1029: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 2270/93912 objects degraded (2.417%); 394/93912 objects misplaced (0.420%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T11:01:41.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:40 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/314581158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:01:41.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:40 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/320152962' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:41.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:40 smithi155 ceph-mon[184854]: pgmap v1029: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 2270/93912 objects degraded (2.417%); 394/93912 objects misplaced (0.420%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T11:01:41.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:40 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/314581158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:01:41.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:40 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/320152962' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:41.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:01:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:01:41] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-10-09T11:01:41.886 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 23 16 37487 37471 101.794 77.9375 0.011774 0.00981762 2023-10-09T11:01:41.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:41 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:41.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:41 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/314581158' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:01:41.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:41 smithi155 ceph-mon[184854]: osdmap e416: 8 total, 8 up, 7 in 2023-10-09T11:01:41.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:41 smithi155 ceph-mon[184854]: failed to encode map e416 with expected crc 2023-10-09T11:01:41.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:41 smithi155 ceph-mon[184854]: failed to encode map e416 with expected crc 2023-10-09T11:01:41.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:41 smithi155 ceph-mon[184854]: failed to encode map e416 with expected crc 2023-10-09T11:01:41.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:41 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:42.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:41 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:42.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:41 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/314581158' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:01:42.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:41 smithi125 ceph-mon[195507]: osdmap e416: 8 total, 8 up, 7 in 2023-10-09T11:01:42.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:41 smithi125 ceph-mon[195507]: failed to encode map e416 with expected crc 2023-10-09T11:01:42.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:41 smithi125 ceph-mon[195507]: failed to encode map e416 with expected crc 2023-10-09T11:01:42.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:41 smithi125 ceph-mon[195507]: failed to encode map e416 with expected crc 2023-10-09T11:01:42.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:41 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:42.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:41 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:42.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:41 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/314581158' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:01:42.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:41 smithi125 ceph-mon[200197]: osdmap e416: 8 total, 8 up, 7 in 2023-10-09T11:01:42.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:41 smithi125 ceph-mon[200197]: failed to encode map e416 with expected crc 2023-10-09T11:01:42.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:41 smithi125 ceph-mon[200197]: failed to encode map e416 with expected crc 2023-10-09T11:01:42.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:41 smithi125 ceph-mon[200197]: failed to encode map e416 with expected crc 2023-10-09T11:01:42.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:41 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:42.886 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 24 16 37652 37636 97.9832 10.3125 0.00205707 0.00980419 2023-10-09T11:01:43.037 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:01:43.037 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:01:43.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[195507]: pgmap v1031: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 77 MiB/s wr, 1.22k op/s; 2270/105057 objects degraded (2.161%); 394/105057 objects misplaced (0.375%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T11:01:43.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[195507]: osdmap e417: 8 total, 8 up, 7 in 2023-10-09T11:01:43.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[195507]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[195507]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[195507]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[195507]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[195507]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[195507]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:43.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:43.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[200197]: pgmap v1031: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 77 MiB/s wr, 1.22k op/s; 2270/105057 objects degraded (2.161%); 394/105057 objects misplaced (0.375%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T11:01:43.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[200197]: osdmap e417: 8 total, 8 up, 7 in 2023-10-09T11:01:43.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[200197]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[200197]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[200197]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[200197]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[200197]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[200197]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:43.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:42 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:43.177 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:43.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:43.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:42 smithi155 ceph-mon[184854]: pgmap v1031: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 77 MiB/s wr, 1.22k op/s; 2270/105057 objects degraded (2.161%); 394/105057 objects misplaced (0.375%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T11:01:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:42 smithi155 ceph-mon[184854]: osdmap e417: 8 total, 8 up, 7 in 2023-10-09T11:01:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:42 smithi155 ceph-mon[184854]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:42 smithi155 ceph-mon[184854]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:42 smithi155 ceph-mon[184854]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:42 smithi155 ceph-mon[184854]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:42 smithi155 ceph-mon[184854]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:42 smithi155 ceph-mon[184854]: failed to encode map e417 with expected crc 2023-10-09T11:01:43.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:42 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:43.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:42 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:43.289 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:43.348 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:43.381 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 0] out_osds: [7] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T11:01:43.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T11:01:43.534 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:43.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:43.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:43.551 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T11:01:43.640 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:43.672 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:43.693 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:43.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:43.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:43.798 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:43.847 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:43.888 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 25 16 38813 38797 96.9603 72.5625 0.014162 0.0103052 2023-10-09T11:01:43.892 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:43.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:43.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:43 smithi155 ceph-mon[184854]: osdmap e418: 8 total, 8 up, 7 in 2023-10-09T11:01:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:43 smithi155 ceph-mon[184854]: failed to encode map e418 with expected crc 2023-10-09T11:01:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:43 smithi155 ceph-mon[184854]: failed to encode map e418 with expected crc 2023-10-09T11:01:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:43 smithi155 ceph-mon[184854]: failed to encode map e418 with expected crc 2023-10-09T11:01:43.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:43 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 2270/105057 objects degraded (2.161%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T11:01:44.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:44.035 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:44.060 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:44.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:44.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:44.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:43 smithi125 ceph-mon[195507]: osdmap e418: 8 total, 8 up, 7 in 2023-10-09T11:01:44.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:43 smithi125 ceph-mon[195507]: failed to encode map e418 with expected crc 2023-10-09T11:01:44.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:43 smithi125 ceph-mon[195507]: failed to encode map e418 with expected crc 2023-10-09T11:01:44.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:43 smithi125 ceph-mon[195507]: failed to encode map e418 with expected crc 2023-10-09T11:01:44.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:43 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 2270/105057 objects degraded (2.161%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T11:01:44.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:43 smithi125 ceph-mon[200197]: osdmap e418: 8 total, 8 up, 7 in 2023-10-09T11:01:44.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:43 smithi125 ceph-mon[200197]: failed to encode map e418 with expected crc 2023-10-09T11:01:44.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:43 smithi125 ceph-mon[200197]: failed to encode map e418 with expected crc 2023-10-09T11:01:44.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:43 smithi125 ceph-mon[200197]: failed to encode map e418 with expected crc 2023-10-09T11:01:44.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:43 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 2270/105057 objects degraded (2.161%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T11:01:44.163 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:44.164 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:44.193 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:44.215 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:44.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:44.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:44.321 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:44.375 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:44.406 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:44.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:44.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:44.518 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:44.576 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:44.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:44.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:44.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:44.672 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:44.713 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:44.771 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:44.888 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 26 16 40040 40024 96.1801 76.6875 0.00758706 0.0103923 2023-10-09T11:01:44.896 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:44 smithi125 ceph-mon[195507]: pgmap v1034: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 1135/111402 objects degraded (1.019%); 773/111402 objects misplaced (0.694%); 15 MiB/s, 247 objects/s recovering 2023-10-09T11:01:44.896 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:44 smithi125 ceph-mon[200197]: pgmap v1034: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 1135/111402 objects degraded (1.019%); 773/111402 objects misplaced (0.694%); 15 MiB/s, 247 objects/s recovering 2023-10-09T11:01:45.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:01:45.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:44 smithi155 ceph-mon[184854]: pgmap v1034: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 1135/111402 objects degraded (1.019%); 773/111402 objects misplaced (0.694%); 15 MiB/s, 247 objects/s recovering 2023-10-09T11:01:45.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:01:45.888 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 27 16 41468 41452 95.9232 89.25 0.0156233 0.0104189 2023-10-09T11:01:46.069 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:46.069 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:01:46.069 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:46.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:01:46.181 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:46.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:46.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:46.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:46.322 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:46.322 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:01:46.324 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:46.349 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:46.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:46.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:46.454 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:46.485 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:46.511 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:46.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:46.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:46.618 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:46.650 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:46.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:46.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:46.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:46.777 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:46.814 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:46.839 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:46.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:46.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:46.888 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 28 16 42632 42616 95.0956 72.75 0.00599685 0.0105105 2023-10-09T11:01:46.943 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:46.977 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:46.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:47.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:47.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:47.025 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:47.084 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:47.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:47.184 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:47.270 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:47 smithi125 ceph-mon[195507]: pgmap v1035: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 61 MiB/s wr, 971 op/s; 1135/111402 objects degraded (1.019%); 773/111402 objects misplaced (0.694%); 7.9 MiB/s, 125 objects/s recovering 2023-10-09T11:01:47.270 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:47 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:47.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:47 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:47.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:47 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:01:47.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:47 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:01:47.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:47 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:47.271 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:47 smithi125 ceph-mon[200197]: pgmap v1035: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 61 MiB/s wr, 971 op/s; 1135/111402 objects degraded (1.019%); 773/111402 objects misplaced (0.694%); 7.9 MiB/s, 125 objects/s recovering 2023-10-09T11:01:47.271 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:47 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:47.271 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:47 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:47.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:47 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:01:47.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:47 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:01:47.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:47 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:47.274 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:47.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:47.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:47.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:47.447 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:47 smithi155 ceph-mon[184854]: pgmap v1035: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 61 MiB/s wr, 971 op/s; 1135/111402 objects degraded (1.019%); 773/111402 objects misplaced (0.694%); 7.9 MiB/s, 125 objects/s recovering 2023-10-09T11:01:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:47 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:47 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:47 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:01:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:47 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:01:47.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:47 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:01:47.594 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:01:47.594 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:01:47.753 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:47.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:47.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:47.864 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:47.889 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 29 16 43714 43698 94.1479 67.625 0.00271026 0.0106151 2023-10-09T11:01:47.901 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:47.928 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:47.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:47.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:48.033 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:48.072 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:48.098 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:48.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:48.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:48.119 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T11:01:48.203 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:48.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:48 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2549000832' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:01:48.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:48 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:01:48.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:48 smithi155 ceph-mon[184854]: pgmap v1036: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 1135/120414 objects degraded (0.943%); 773/120414 objects misplaced (0.642%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T11:01:48.244 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:48.269 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:48.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:48.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:48.421 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:48.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:48 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2549000832' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:01:48.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:48 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:01:48.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:48 smithi125 ceph-mon[195507]: pgmap v1036: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 1135/120414 objects degraded (0.943%); 773/120414 objects misplaced (0.642%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T11:01:48.474 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:48 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2549000832' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:01:48.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:48 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:01:48.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:48 smithi125 ceph-mon[200197]: pgmap v1036: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 1135/120414 objects degraded (0.943%); 773/120414 objects misplaced (0.642%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T11:01:48.475 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:48.890 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 30 16 44996 44980 93.6767 80.125 0.0115169 0.0106693 2023-10-09T11:01:48.919 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:48.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:48.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:49.024 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:49.060 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:49.085 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:49.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:49.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:49.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:49.241 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T11:01:49.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[195507]: osdmap e419: 8 total, 8 up, 7 in 2023-10-09T11:01:49.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[195507]: failed to encode map e419 with expected crc 2023-10-09T11:01:49.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[195507]: failed to encode map e419 with expected crc 2023-10-09T11:01:49.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[195507]: failed to encode map e419 with expected crc 2023-10-09T11:01:49.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[195507]: failed to encode map e419 with expected crc 2023-10-09T11:01:49.243 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[195507]: failed to encode map e419 with expected crc 2023-10-09T11:01:49.243 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 1135/120414 objects degraded (0.943%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:01:49.243 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T11:01:49.243 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[200197]: osdmap e419: 8 total, 8 up, 7 in 2023-10-09T11:01:49.243 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[200197]: failed to encode map e419 with expected crc 2023-10-09T11:01:49.243 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[200197]: failed to encode map e419 with expected crc 2023-10-09T11:01:49.243 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[200197]: failed to encode map e419 with expected crc 2023-10-09T11:01:49.244 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[200197]: failed to encode map e419 with expected crc 2023-10-09T11:01:49.244 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[200197]: failed to encode map e419 with expected crc 2023-10-09T11:01:49.244 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:49 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 1135/120414 objects degraded (0.943%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:01:49.244 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:49.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:49.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:49.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:49.429 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:49.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:49 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T11:01:49.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:49 smithi155 ceph-mon[184854]: osdmap e419: 8 total, 8 up, 7 in 2023-10-09T11:01:49.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:49 smithi155 ceph-mon[184854]: failed to encode map e419 with expected crc 2023-10-09T11:01:49.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:49 smithi155 ceph-mon[184854]: failed to encode map e419 with expected crc 2023-10-09T11:01:49.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:49 smithi155 ceph-mon[184854]: failed to encode map e419 with expected crc 2023-10-09T11:01:49.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:49 smithi155 ceph-mon[184854]: failed to encode map e419 with expected crc 2023-10-09T11:01:49.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:49 smithi155 ceph-mon[184854]: failed to encode map e419 with expected crc 2023-10-09T11:01:49.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:49 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 1135/120414 objects degraded (0.943%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:01:49.503 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:49.639 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:49.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:49.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:49.756 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:49.794 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:49.822 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T11:01:49.823 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:49.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:49.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:49.860 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T11:01:49.891 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 31 16 46303 46287 93.2871 81.6875 0.0119002 0.0107146 2023-10-09T11:01:49.929 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:50.046 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:50.071 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:50.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:50.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:50.178 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:50.212 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:50.234 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:50.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:50.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:50.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:50 smithi125 ceph-mon[195507]: pgmap v1038: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1k op/s; 1135/128463 objects degraded (0.884%); 448/128463 objects misplaced (0.349%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T11:01:50.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:50 smithi125 ceph-mon[200197]: pgmap v1038: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1k op/s; 1135/128463 objects degraded (0.884%); 448/128463 objects misplaced (0.349%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T11:01:50.337 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:50.358 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:01:50.370 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:50.392 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:50.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:50.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:50.398 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:01:50.446 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:50.446 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:01:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:50 smithi155 ceph-mon[184854]: pgmap v1038: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1k op/s; 1135/128463 objects degraded (0.884%); 448/128463 objects misplaced (0.349%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T11:01:50.486 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:01:50.496 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:50.498 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:50.530 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:01:50.537 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:50.558 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:50.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:50.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:50.567 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:01:50.607 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:01:50.635 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:01:50.645 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:01:50.645 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:01:50.645 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:01:50.646 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:01:50.646 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:01:50.646 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:01:50.646 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:01:50.646 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:01:50.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:50.768 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:50.802 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:50.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:50.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:50.891 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 32 16 47203 47187 92.1298 56.25 0.00474317 0.0108488 2023-10-09T11:01:50.906 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:50.977 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:51.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:51.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:51.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:51.184 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:01:51.219 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:51.261 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:51.287 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:51.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:51.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:51.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2906166793' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:51.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1764507811' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:01:51.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:51 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:01:51.322 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:01:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:01:51] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.43.0" 2023-10-09T11:01:51.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2906166793' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:51.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1764507811' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:01:51.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:51 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:01:51.391 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:51.433 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:51.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2906166793' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:51.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1764507811' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:01:51.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:51 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:01:51.460 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:51.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:51.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:51.566 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:51.606 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:51.636 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:51.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:51.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:51.743 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:51.784 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:51.807 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:51.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:51.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:51.893 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 33 16 48479 48463 91.7493 79.75 0.00676001 0.0108935 2023-10-09T11:01:51.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:51.958 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:51.994 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:51.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:51.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:52.100 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:52.156 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:52.290 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:52.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:52.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:52.396 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:52.439 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:52 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:52.439 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:52 smithi155 ceph-mon[184854]: failed to encode map e420 with expected crc 2023-10-09T11:01:52.439 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:52 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:01:52.439 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:52 smithi155 ceph-mon[184854]: osdmap e420: 8 total, 8 up, 7 in 2023-10-09T11:01:52.439 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:52 smithi155 ceph-mon[184854]: failed to encode map e420 with expected crc 2023-10-09T11:01:52.439 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:52 smithi155 ceph-mon[184854]: failed to encode map e420 with expected crc 2023-10-09T11:01:52.439 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:52 smithi155 ceph-mon[184854]: pgmap v1040: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s; 1135/138126 objects degraded (0.822%); 448/138126 objects misplaced (0.324%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T11:01:52.441 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:52.474 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:52.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:52.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:52.553 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:52 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:52.553 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:52 smithi125 ceph-mon[195507]: failed to encode map e420 with expected crc 2023-10-09T11:01:52.553 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:52 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:01:52.553 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:52 smithi125 ceph-mon[195507]: osdmap e420: 8 total, 8 up, 7 in 2023-10-09T11:01:52.553 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:52 smithi125 ceph-mon[195507]: failed to encode map e420 with expected crc 2023-10-09T11:01:52.554 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:52 smithi125 ceph-mon[195507]: failed to encode map e420 with expected crc 2023-10-09T11:01:52.554 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:52 smithi125 ceph-mon[195507]: pgmap v1040: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s; 1135/138126 objects degraded (0.822%); 448/138126 objects misplaced (0.324%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T11:01:52.554 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:52 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:01:52.554 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:52 smithi125 ceph-mon[200197]: failed to encode map e420 with expected crc 2023-10-09T11:01:52.554 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:52 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:01:52.554 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:52 smithi125 ceph-mon[200197]: osdmap e420: 8 total, 8 up, 7 in 2023-10-09T11:01:52.554 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:52 smithi125 ceph-mon[200197]: failed to encode map e420 with expected crc 2023-10-09T11:01:52.555 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:52 smithi125 ceph-mon[200197]: failed to encode map e420 with expected crc 2023-10-09T11:01:52.555 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:52 smithi125 ceph-mon[200197]: pgmap v1040: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s; 1135/138126 objects degraded (0.822%); 448/138126 objects misplaced (0.324%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T11:01:52.580 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:52.631 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:52.893 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 34 16 49803 49787 91.4845 82.75 0.00317907 0.010924 2023-10-09T11:01:53.125 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:53.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:53.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:53.231 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:53.302 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:53.337 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:53.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:53.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:53.451 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:53 smithi155 ceph-mon[184854]: 1.a scrub starts 2023-10-09T11:01:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:53 smithi155 ceph-mon[184854]: 1.a scrub ok 2023-10-09T11:01:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:53 smithi155 ceph-mon[184854]: 1.1d scrub starts 2023-10-09T11:01:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:53 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 1135/138126 objects degraded (0.822%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:01:53.489 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:53.514 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:53.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:53.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:53.536 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:01:53.536 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:01:53.536 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.536 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:50497 10.f e420/403 10:f82dffe0:::benchmark_data_smithi125_402869_object50496:head v 420'3192, mlcod=420'3189)", 2023-10-09T11:01:53.536 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:53.520538+0000", 2023-10-09T11:01:53.536 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0043693619999999999, 2023-10-09T11:01:53.536 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0044031690000000002, 2023-10-09T11:01:53.537 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:01:53.537 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:01:53.537 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:01:53.537 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.537 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:01:53.537 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.520538+0000", 2023-10-09T11:01:53.537 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:53.537 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.537 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.537 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:01:53.537 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.520538+0000", 2023-10-09T11:01:53.538 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:53.538 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.538 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.538 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:01:53.538 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.520537+0000", 2023-10-09T11:01:53.538 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:01:53.538 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.538 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.538 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:01:53.539 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.520568+0000", 2023-10-09T11:01:53.539 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1004000000000003e-05 2023-10-09T11:01:53.539 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.539 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.539 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:01:53.539 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.520576+0000", 2023-10-09T11:01:53.539 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7649999999999993e-06 2023-10-09T11:01:53.539 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.539 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.539 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:01:53.539 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.520583+0000", 2023-10-09T11:01:53.539 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7649999999999996e-06 2023-10-09T11:01:53.540 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.540 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.540 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:01:53.540 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.520652+0000", 2023-10-09T11:01:53.540 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8757000000000006e-05 2023-10-09T11:01:53.540 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.540 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.540 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:01:53.540 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.520671+0000", 2023-10-09T11:01:53.541 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8958e-05 2023-10-09T11:01:53.541 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:53.541 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:01:53.541 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:53.541 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.541 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.541 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:50504 10.f e420/403 10:fb532f58:::benchmark_data_smithi125_402869_object50503:head v 420'3193, mlcod=420'3189)", 2023-10-09T11:01:53.541 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:53.522916+0000", 2023-10-09T11:01:53.541 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.001991451, 2023-10-09T11:01:53.541 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020927350000000001, 2023-10-09T11:01:53.541 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:01:53.541 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:01:53.542 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:01:53.542 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.542 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:01:53.542 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.522916+0000", 2023-10-09T11:01:53.542 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:53.542 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.542 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.542 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:01:53.542 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.522916+0000", 2023-10-09T11:01:53.542 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:53.542 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.542 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.543 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:01:53.543 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.522916+0000", 2023-10-09T11:01:53.543 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:01:53.543 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.543 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.543 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:01:53.543 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.522951+0000", 2023-10-09T11:01:53.543 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5886999999999997e-05 2023-10-09T11:01:53.543 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.543 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.543 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:01:53.544 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.522958+0000", 2023-10-09T11:01:53.544 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5989999999999998e-06 2023-10-09T11:01:53.544 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.544 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.544 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:01:53.544 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.522963+0000", 2023-10-09T11:01:53.544 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4199999999999998e-06 2023-10-09T11:01:53.544 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.544 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.544 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:01:53.544 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.523237+0000", 2023-10-09T11:01:53.544 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027398 2023-10-09T11:01:53.545 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.545 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.545 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:01:53.545 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.523255+0000", 2023-10-09T11:01:53.545 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7533e-05 2023-10-09T11:01:53.545 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:53.545 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:01:53.545 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:53.545 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.545 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.545 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:50499 10.7 10:e4f6b06f:::benchmark_data_smithi125_402869_object50498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T11:01:53.545 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:53.517296+0000", 2023-10-09T11:01:53.546 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0076117479999999998, 2023-10-09T11:01:53.546 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0077874110000000002, 2023-10-09T11:01:53.546 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:01:53.546 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:01:53.546 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:01:53.546 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:01:53.546 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:01:53.546 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 50499 2023-10-09T11:01:53.546 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.546 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:01:53.546 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.547 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:01:53.547 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.517296+0000", 2023-10-09T11:01:53.547 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:53.547 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.547 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.547 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:01:53.547 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.517296+0000", 2023-10-09T11:01:53.547 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:01:53.547 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.547 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.547 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:01:53.547 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.517294+0000", 2023-10-09T11:01:53.547 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:01:53.548 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.548 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.548 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:01:53.548 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.517315+0000", 2023-10-09T11:01:53.548 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1169e-05 2023-10-09T11:01:53.548 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.548 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.548 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:01:53.548 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.517324+0000", 2023-10-09T11:01:53.548 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9989999999999997e-06 2023-10-09T11:01:53.548 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.548 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.549 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:01:53.549 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.517330+0000", 2023-10-09T11:01:53.549 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7620000000000001e-06 2023-10-09T11:01:53.549 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.549 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.549 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:01:53.549 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.518356+0000", 2023-10-09T11:01:53.549 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001026342 2023-10-09T11:01:53.549 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.549 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.549 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:01:53.549 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.518507+0000", 2023-10-09T11:01:53.549 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015070599999999999 2023-10-09T11:01:53.550 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.550 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.550 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T11:01:53.550 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.518651+0000", 2023-10-09T11:01:53.550 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014370100000000001 2023-10-09T11:01:53.550 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:01:53.550 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:01:53.550 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:01:53.550 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:53.522511+0000", 2023-10-09T11:01:53.550 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038599469999999999 2023-10-09T11:01:53.550 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:53.550 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:01:53.551 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:53.551 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:01:53.551 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T11:01:53.551 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 3 2023-10-09T11:01:53.551 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:01:53.557 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:53 smithi125 ceph-mon[195507]: 1.a scrub starts 2023-10-09T11:01:53.557 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:53 smithi125 ceph-mon[195507]: 1.a scrub ok 2023-10-09T11:01:53.557 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:53 smithi125 ceph-mon[195507]: 1.1d scrub starts 2023-10-09T11:01:53.557 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:53 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 1135/138126 objects degraded (0.822%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:01:53.557 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:53 smithi125 ceph-mon[200197]: 1.a scrub starts 2023-10-09T11:01:53.557 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:53 smithi125 ceph-mon[200197]: 1.a scrub ok 2023-10-09T11:01:53.557 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:53 smithi125 ceph-mon[200197]: 1.1d scrub starts 2023-10-09T11:01:53.558 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:53 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 1135/138126 objects degraded (0.822%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:01:53.619 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:53.653 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:53.678 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:53.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:53.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:53.782 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:53.832 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:53.878 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:53.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:53.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:53.896 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 35 16 50911 50895 90.8406 69.25 0.0168501 0.0110031 2023-10-09T11:01:53.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T11:01:53.983 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:54.029 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:54.058 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:54.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:54.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:54.162 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:54.199 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:54.227 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:54.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:54.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:54.246 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T11:01:54.336 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:54.363 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:54 smithi155 ceph-mon[184854]: 1.1b scrub starts 2023-10-09T11:01:54.363 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:54 smithi155 ceph-mon[184854]: pgmap v1041: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 1135/146097 objects degraded (0.777%); 72/146097 objects misplaced (0.049%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T11:01:54.363 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:54 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/291726176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:01:54.366 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:54.393 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:54.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:54.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:54.498 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:54.525 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:54.543 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:54.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:54.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:54.555 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:54 smithi125 ceph-mon[195507]: 1.1b scrub starts 2023-10-09T11:01:54.555 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:54 smithi125 ceph-mon[195507]: pgmap v1041: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 1135/146097 objects degraded (0.777%); 72/146097 objects misplaced (0.049%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T11:01:54.555 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:54 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/291726176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:01:54.555 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:54 smithi125 ceph-mon[200197]: 1.1b scrub starts 2023-10-09T11:01:54.555 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:54 smithi125 ceph-mon[200197]: pgmap v1041: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 1135/146097 objects degraded (0.777%); 72/146097 objects misplaced (0.049%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T11:01:54.555 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:54 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/291726176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:01:54.623 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T11:01:54.647 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:01:54.666 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:54.702 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:54.823 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:01:54.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:54.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:01:54.866 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:01:54.896 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 36 16 51480 51464 89.3056 35.5625 0.00267061 0.011016 2023-10-09T11:01:54.928 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:54.955 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:54.973 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:54.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:54.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:55.026 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:01:55.077 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:55.103 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:55.122 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:55.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:55.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:55.131 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:55.226 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:55.277 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:55.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:55 smithi125 ceph-mon[195507]: failed to encode map e421 with expected crc 2023-10-09T11:01:55.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:55 smithi125 ceph-mon[195507]: failed to encode map e421 with expected crc 2023-10-09T11:01:55.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:55 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/291726176' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T11:01:55.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:55 smithi125 ceph-mon[195507]: osdmap e421: 8 total, 8 up, 7 in 2023-10-09T11:01:55.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:55 smithi125 ceph-mon[195507]: failed to encode map e421 with expected crc 2023-10-09T11:01:55.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:55 smithi125 ceph-mon[195507]: failed to encode map e421 with expected crc 2023-10-09T11:01:55.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:55 smithi125 ceph-mon[200197]: failed to encode map e421 with expected crc 2023-10-09T11:01:55.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:55 smithi125 ceph-mon[200197]: failed to encode map e421 with expected crc 2023-10-09T11:01:55.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:55 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/291726176' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T11:01:55.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:55 smithi125 ceph-mon[200197]: osdmap e421: 8 total, 8 up, 7 in 2023-10-09T11:01:55.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:55 smithi125 ceph-mon[200197]: failed to encode map e421 with expected crc 2023-10-09T11:01:55.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:55 smithi125 ceph-mon[200197]: failed to encode map e421 with expected crc 2023-10-09T11:01:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:55 smithi155 ceph-mon[184854]: failed to encode map e421 with expected crc 2023-10-09T11:01:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:55 smithi155 ceph-mon[184854]: failed to encode map e421 with expected crc 2023-10-09T11:01:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:55 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/291726176' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T11:01:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:55 smithi155 ceph-mon[184854]: osdmap e421: 8 total, 8 up, 7 in 2023-10-09T11:01:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:55 smithi155 ceph-mon[184854]: failed to encode map e421 with expected crc 2023-10-09T11:01:55.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:55 smithi155 ceph-mon[184854]: failed to encode map e421 with expected crc 2023-10-09T11:01:55.897 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 37 16 52293 52277 88.2652 50.8125 0.00739734 0.011324 2023-10-09T11:01:56.265 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:56.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:56.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:56.305 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:56.372 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:56.373 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:56 smithi125 ceph-mon[195507]: 1.15 scrub starts 2023-10-09T11:01:56.373 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:56 smithi125 ceph-mon[195507]: osdmap e422: 8 total, 8 up, 7 in 2023-10-09T11:01:56.373 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:56 smithi125 ceph-mon[195507]: failed to encode map e422 with expected crc 2023-10-09T11:01:56.373 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:56 smithi125 ceph-mon[195507]: failed to encode map e422 with expected crc 2023-10-09T11:01:56.373 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:56 smithi125 ceph-mon[195507]: failed to encode map e422 with expected crc 2023-10-09T11:01:56.374 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:56 smithi125 ceph-mon[195507]: pgmap v1044: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 977 op/s; 1135/146097 objects degraded (0.777%); 72/146097 objects misplaced (0.049%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T11:01:56.374 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:56 smithi125 ceph-mon[200197]: 1.15 scrub starts 2023-10-09T11:01:56.374 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:56 smithi125 ceph-mon[200197]: osdmap e422: 8 total, 8 up, 7 in 2023-10-09T11:01:56.374 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:56 smithi125 ceph-mon[200197]: failed to encode map e422 with expected crc 2023-10-09T11:01:56.374 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:56 smithi125 ceph-mon[200197]: failed to encode map e422 with expected crc 2023-10-09T11:01:56.374 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:56 smithi125 ceph-mon[200197]: failed to encode map e422 with expected crc 2023-10-09T11:01:56.374 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:56 smithi125 ceph-mon[200197]: pgmap v1044: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 977 op/s; 1135/146097 objects degraded (0.777%); 72/146097 objects misplaced (0.049%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T11:01:56.433 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:56.519 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:56.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:56.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:56.624 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:01:56.633 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:56 smithi155 ceph-mon[184854]: 1.15 scrub starts 2023-10-09T11:01:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:56 smithi155 ceph-mon[184854]: osdmap e422: 8 total, 8 up, 7 in 2023-10-09T11:01:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:56 smithi155 ceph-mon[184854]: failed to encode map e422 with expected crc 2023-10-09T11:01:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:56 smithi155 ceph-mon[184854]: failed to encode map e422 with expected crc 2023-10-09T11:01:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:56 smithi155 ceph-mon[184854]: failed to encode map e422 with expected crc 2023-10-09T11:01:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:56 smithi155 ceph-mon[184854]: pgmap v1044: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 977 op/s; 1135/146097 objects degraded (0.777%); 72/146097 objects misplaced (0.049%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T11:01:56.728 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:56.765 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:01:56.768 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:01:56.769 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:01:56.769 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T11:01:56.769 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T11:01:56.769 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:01:56.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:56.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:01:56.871 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:01:56.898 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 38 16 53382 53366 87.7336 68.0625 0.0163714 0.0113921 2023-10-09T11:01:56.905 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:56.936 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:01:56.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:56.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:01:57.043 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:57.082 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:57.106 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:57.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:57.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:57.212 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:57.251 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:57.281 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:57.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:57.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:57.362 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T11:01:57.386 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:57.417 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:57.427 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:57 smithi125 ceph-mon[200197]: 1.1b scrub starts 2023-10-09T11:01:57.428 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:57 smithi125 ceph-mon[195507]: 1.1b scrub starts 2023-10-09T11:01:57.445 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:57.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:57.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:57.549 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:57.644 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:57.694 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:01:57.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:57.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:01:57.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:57 smithi155 ceph-mon[184854]: 1.1b scrub starts 2023-10-09T11:01:57.800 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:01:57.836 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:57.858 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:01:57.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:57.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:01:57.897 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 39 16 54444 54428 87.1858 66.375 0.00812072 0.0114638 2023-10-09T11:01:57.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:01:58.075 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:58.154 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:01:58.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:58.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:01:58.258 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:58 smithi125 ceph-mon[195507]: pgmap v1045: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 1135/154668 objects degraded (0.734%); 72/154668 objects misplaced (0.047%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T11:01:58.258 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:58 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 1135/146097 objects degraded (0.777%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:01:58.264 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:58.301 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:01:58.301 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:58.317 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:01:58.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:58.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:58.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:58.350 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:01:58.382 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:01:58.424 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:01:58.436 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:01:58.463 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:01:58.473 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:58.496 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:01:58.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:58.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:01:58.506 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:01:58.549 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:01:58.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:58 smithi125 ceph-mon[200197]: pgmap v1045: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 1135/154668 objects degraded (0.734%); 72/154668 objects misplaced (0.047%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T11:01:58.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:58 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 1135/146097 objects degraded (0.777%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:01:58.583 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:01:58.600 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:58.615 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:01:58.615 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:01:58.615 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:01:58.616 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:01:58.616 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:01:58.616 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:01:58.616 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:01:58.616 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:01:58.637 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:58 smithi155 ceph-mon[184854]: pgmap v1045: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 1135/154668 objects degraded (0.734%); 72/154668 objects misplaced (0.047%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T11:01:58.637 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:58 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 1135/146097 objects degraded (0.777%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:01:58.641 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:58.674 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:58.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:58.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:58.778 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:01:58.816 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:58.848 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:01:58.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:01:58.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:01:58.897 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T11:01:58.897878+0000 min lat: 0.0011398 max lat: 2.07371 avg lat: 0.0115523 2023-10-09T11:01:58.897 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T11:01:58.897 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 40 16 55415 55399 86.5235 60.6875 0.0143501 0.0115523 2023-10-09T11:01:58.954 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:01:59.019 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:01:59.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:59 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1052278694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:59.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:01:59 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3397218326' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:01:59.290 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T11:01:59.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:59 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1052278694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:59.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:01:59 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3397218326' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:01:59.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:59 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1052278694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:01:59.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:01:59 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3397218326' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:01:59.897 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 41 16 56646 56630 86.289 76.9375 0.00601801 0.0115834 2023-10-09T11:02:00.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:00.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:00.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:00.536 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:00.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3397218326' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:02:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[195507]: osdmap e423: 8 total, 8 up, 7 in 2023-10-09T11:02:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[195507]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[195507]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[195507]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[195507]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[195507]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[195507]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[195507]: pgmap v1047: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 51 MiB/s wr, 809 op/s; 716/160683 objects misplaced (0.446%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T11:02:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:02:00.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3397218326' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:02:00.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[200197]: osdmap e423: 8 total, 8 up, 7 in 2023-10-09T11:02:00.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[200197]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[200197]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[200197]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[200197]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[200197]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[200197]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[200197]: pgmap v1047: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 51 MiB/s wr, 809 op/s; 716/160683 objects misplaced (0.446%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T11:02:00.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:02:00.595 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:00.672 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:00.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:00.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:00 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3397218326' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:02:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:00 smithi155 ceph-mon[184854]: osdmap e423: 8 total, 8 up, 7 in 2023-10-09T11:02:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:00 smithi155 ceph-mon[184854]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:00 smithi155 ceph-mon[184854]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:00 smithi155 ceph-mon[184854]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:00 smithi155 ceph-mon[184854]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:00 smithi155 ceph-mon[184854]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:00 smithi155 ceph-mon[184854]: failed to encode map e423 with expected crc 2023-10-09T11:02:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:00 smithi155 ceph-mon[184854]: pgmap v1047: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 51 MiB/s wr, 809 op/s; 716/160683 objects misplaced (0.446%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T11:02:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:02:00.778 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:00.832 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:00.897 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 42 16 57917 57901 86.1258 79.4375 0.0104245 0.0116054 2023-10-09T11:02:01.127 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:01.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:01.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:01.245 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:01.284 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:01.316 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:01.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:01.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:01.420 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:01.458 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:01.481 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:01.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:01.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:01.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:01 smithi125 ceph-mon[195507]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1135/154668 objects degraded (0.734%), 1 pg degraded) 2023-10-09T11:02:01.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:02:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:02:01] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2023-10-09T11:02:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:01 smithi125 ceph-mon[200197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1135/154668 objects degraded (0.734%), 1 pg degraded) 2023-10-09T11:02:01.585 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:01.630 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:01.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:01 smithi155 ceph-mon[184854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1135/154668 objects degraded (0.734%), 1 pg degraded) 2023-10-09T11:02:01.826 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T11:02:01.898 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 43 16 59020 59004 85.726 68.9375 0.0154074 0.0116599 2023-10-09T11:02:01.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:01.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:01.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:02.073 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:02.153 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:02.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:02.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:02.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:02.267 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:02:02.267 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:02:02.304 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:02.346 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:02.402 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:02:02.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:02 smithi125 ceph-mon[195507]: 1.15 scrub starts 2023-10-09T11:02:02.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:02 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3888955770' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:02.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:02 smithi125 ceph-mon[195507]: pgmap v1048: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 716/170058 objects misplaced (0.421%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T11:02:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:02 smithi125 ceph-mon[200197]: 1.15 scrub starts 2023-10-09T11:02:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:02 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3888955770' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:02 smithi125 ceph-mon[200197]: pgmap v1048: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 716/170058 objects misplaced (0.421%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T11:02:02.701 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:02:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:02 smithi155 ceph-mon[184854]: 1.15 scrub starts 2023-10-09T11:02:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:02 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3888955770' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:02 smithi155 ceph-mon[184854]: pgmap v1048: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 716/170058 objects misplaced (0.421%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T11:02:02.900 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 44 16 59975 59959 85.1342 59.6875 0.0103563 0.0117413 2023-10-09T11:02:02.909 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:02.934 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:02.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:02.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:03.041 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:03.075 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:03.112 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:03.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:03.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:03.225 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:03.266 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:03.289 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:03.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:03.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:03.393 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:03.435 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:03.469 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:03.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:03.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:03.576 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:03.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:03 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:02:03.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:03 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T11:02:03.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:03 smithi155 ceph-mon[184854]: failed to encode map e424 with expected crc 2023-10-09T11:02:03.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:03 smithi155 ceph-mon[184854]: failed to encode map e424 with expected crc 2023-10-09T11:02:03.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3888955770' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:02:03.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:03 smithi155 ceph-mon[184854]: osdmap e424: 8 total, 8 up, 7 in 2023-10-09T11:02:03.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:03 smithi155 ceph-mon[184854]: 1.1b scrub starts 2023-10-09T11:02:03.614 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:03.643 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:03.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:03.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:03.749 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:03.759 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:03 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:02:03.759 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:03 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T11:02:03.759 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:03 smithi125 ceph-mon[195507]: failed to encode map e424 with expected crc 2023-10-09T11:02:03.759 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:03 smithi125 ceph-mon[195507]: failed to encode map e424 with expected crc 2023-10-09T11:02:03.759 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3888955770' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:02:03.759 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:03 smithi125 ceph-mon[195507]: osdmap e424: 8 total, 8 up, 7 in 2023-10-09T11:02:03.759 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:03 smithi125 ceph-mon[195507]: 1.1b scrub starts 2023-10-09T11:02:03.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:03 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:02:03.760 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:03 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T11:02:03.760 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:03 smithi125 ceph-mon[200197]: failed to encode map e424 with expected crc 2023-10-09T11:02:03.760 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:03 smithi125 ceph-mon[200197]: failed to encode map e424 with expected crc 2023-10-09T11:02:03.760 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3888955770' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:02:03.760 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:03 smithi125 ceph-mon[200197]: osdmap e424: 8 total, 8 up, 7 in 2023-10-09T11:02:03.760 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:03 smithi125 ceph-mon[200197]: 1.1b scrub starts 2023-10-09T11:02:03.842 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:03.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T11:02:03.897 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 45 16 61015 60999 84.6868 65 0.00699758 0.0118013 2023-10-09T11:02:04.076 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:04.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:04.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:04.183 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:04.228 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:04.231 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:02:04.256 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:04.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:04.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:04.363 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:04.404 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:04.438 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:04.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:04.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:04.490 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:04.514 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:04 smithi155 ceph-mon[184854]: 1.15 deep-scrub starts 2023-10-09T11:02:04.514 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:04 smithi155 ceph-mon[184854]: pgmap v1050: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 320/177303 objects misplaced (0.180%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T11:02:04.542 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:04.606 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:04.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:04 smithi125 ceph-mon[195507]: 1.15 deep-scrub starts 2023-10-09T11:02:04.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:04 smithi125 ceph-mon[195507]: pgmap v1050: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 320/177303 objects misplaced (0.180%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T11:02:04.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:04 smithi125 ceph-mon[200197]: 1.15 deep-scrub starts 2023-10-09T11:02:04.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:04 smithi125 ceph-mon[200197]: pgmap v1050: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 320/177303 objects misplaced (0.180%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T11:02:04.898 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 46 16 62078 62062 84.29 66.4375 0.0136005 0.0118571 2023-10-09T11:02:04.941 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:04.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:04.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:05.049 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:05.099 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:05.226 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:05.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:05.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:05.332 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:05.380 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:05.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:05.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:05.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:05.491 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:02:05.492 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:02:05.515 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:05.516 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T11:02:05.571 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:05.619 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:05 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2105014568' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:02:05.619 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:05 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:02:05.619 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:05 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2105014568' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:02:05.619 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:05 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:02:05.657 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:05.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:05.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:05.765 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:05.823 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:05.827 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:05 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2105014568' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:02:05.827 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:05 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:02:05.851 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:05.854 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T11:02:05.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:05.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:05.898 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 47 16 63120 63104 83.8822 65.125 0.0181062 0.011916 2023-10-09T11:02:05.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T11:02:05.960 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:06.074 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:06.093 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:02:06.115 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:06.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:06.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:06.157 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:02:06.210 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:02:06.217 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:02:06.221 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:06.263 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:02:06.304 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:06.321 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:02:06.365 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:02:06.394 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:06.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:06.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:06.421 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:02:06.454 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:02:06.462 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:06.499 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:02:06.511 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:06.518 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:02:06.518 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:02:06.519 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:02:06.519 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:02:06.519 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:02:06.519 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:02:06.519 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:02:06.519 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:02:06.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:06 smithi125 ceph-mon[195507]: failed to encode map e425 with expected crc 2023-10-09T11:02:06.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:06 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T11:02:06.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:06 smithi125 ceph-mon[195507]: osdmap e425: 8 total, 8 up, 7 in 2023-10-09T11:02:06.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:06 smithi125 ceph-mon[195507]: failed to encode map e425 with expected crc 2023-10-09T11:02:06.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:06 smithi125 ceph-mon[195507]: pgmap v1052: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s; 320/177303 objects misplaced (0.180%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T11:02:06.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:06 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3886040986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:06.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:06 smithi125 ceph-mon[200197]: failed to encode map e425 with expected crc 2023-10-09T11:02:06.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:06 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T11:02:06.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:06 smithi125 ceph-mon[200197]: osdmap e425: 8 total, 8 up, 7 in 2023-10-09T11:02:06.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:06 smithi125 ceph-mon[200197]: failed to encode map e425 with expected crc 2023-10-09T11:02:06.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:06 smithi125 ceph-mon[200197]: pgmap v1052: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s; 320/177303 objects misplaced (0.180%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T11:02:06.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:06 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3886040986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:06.581 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:06.652 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:06 smithi155 ceph-mon[184854]: failed to encode map e425 with expected crc 2023-10-09T11:02:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:06 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T11:02:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:06 smithi155 ceph-mon[184854]: osdmap e425: 8 total, 8 up, 7 in 2023-10-09T11:02:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:06 smithi155 ceph-mon[184854]: failed to encode map e425 with expected crc 2023-10-09T11:02:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:06 smithi155 ceph-mon[184854]: pgmap v1052: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s; 320/177303 objects misplaced (0.180%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T11:02:06.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:06 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3886040986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:06.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:06.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:06.773 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:06.874 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:06.898 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 48 16 64027 64011 83.3157 56.6875 0.0130838 0.0119965 2023-10-09T11:02:06.935 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:06.984 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:06.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:06.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:07.089 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:07.137 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:07.344 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:02:07.344 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:02:07.344 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T11:02:07.345 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T11:02:07.345 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:02:07.898 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 49 16 64948 64932 82.7902 57.5625 0.00354524 0.012014 2023-10-09T11:02:08.075 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:08.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:08.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:08.179 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:08.211 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:08.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:08 smithi125 ceph-mon[200197]: failed to encode map e426 with expected crc 2023-10-09T11:02:08.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:08 smithi125 ceph-mon[200197]: osdmap e426: 8 total, 8 up, 7 in 2023-10-09T11:02:08.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:08 smithi125 ceph-mon[200197]: failed to encode map e426 with expected crc 2023-10-09T11:02:08.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:08 smithi125 ceph-mon[200197]: pgmap v1054: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 885 op/s; 320/186009 objects misplaced (0.172%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T11:02:08.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:08 smithi125 ceph-mon[195507]: failed to encode map e426 with expected crc 2023-10-09T11:02:08.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:08 smithi125 ceph-mon[195507]: osdmap e426: 8 total, 8 up, 7 in 2023-10-09T11:02:08.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:08 smithi125 ceph-mon[195507]: failed to encode map e426 with expected crc 2023-10-09T11:02:08.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:08 smithi125 ceph-mon[195507]: pgmap v1054: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 885 op/s; 320/186009 objects misplaced (0.172%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T11:02:08.898 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 50 16 65290 65274 81.5622 21.375 0.0165945 0.0122557 2023-10-09T11:02:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:08 smithi155 ceph-mon[184854]: failed to encode map e426 with expected crc 2023-10-09T11:02:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:08 smithi155 ceph-mon[184854]: osdmap e426: 8 total, 8 up, 7 in 2023-10-09T11:02:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:08 smithi155 ceph-mon[184854]: failed to encode map e426 with expected crc 2023-10-09T11:02:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:08 smithi155 ceph-mon[184854]: pgmap v1054: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 885 op/s; 320/186009 objects misplaced (0.172%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T11:02:09.560 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:09.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:09.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:09.580 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T11:02:09.668 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:09.724 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:09.750 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:09.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:09.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:09.857 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:09.898 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 51 16 66336 66320 81.2448 65.375 0.0167914 0.012303 2023-10-09T11:02:09.899 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:09 smithi125 ceph-mon[195507]: osdmap e427: 8 total, 8 up, 7 in 2023-10-09T11:02:09.899 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:09 smithi125 ceph-mon[195507]: failed to encode map e427 with expected crc 2023-10-09T11:02:09.899 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:09 smithi125 ceph-mon[195507]: failed to encode map e427 with expected crc 2023-10-09T11:02:09.899 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:09 smithi125 ceph-mon[195507]: failed to encode map e427 with expected crc 2023-10-09T11:02:09.899 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:09 smithi125 ceph-mon[195507]: failed to encode map e427 with expected crc 2023-10-09T11:02:09.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:09 smithi125 ceph-mon[200197]: osdmap e427: 8 total, 8 up, 7 in 2023-10-09T11:02:09.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:09 smithi125 ceph-mon[200197]: failed to encode map e427 with expected crc 2023-10-09T11:02:09.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:09 smithi125 ceph-mon[200197]: failed to encode map e427 with expected crc 2023-10-09T11:02:09.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:09 smithi125 ceph-mon[200197]: failed to encode map e427 with expected crc 2023-10-09T11:02:09.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:09 smithi125 ceph-mon[200197]: failed to encode map e427 with expected crc 2023-10-09T11:02:09.904 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:09.935 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:09.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:09.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:09.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:09 smithi155 ceph-mon[184854]: osdmap e427: 8 total, 8 up, 7 in 2023-10-09T11:02:09.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:09 smithi155 ceph-mon[184854]: failed to encode map e427 with expected crc 2023-10-09T11:02:09.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:09 smithi155 ceph-mon[184854]: failed to encode map e427 with expected crc 2023-10-09T11:02:09.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:09 smithi155 ceph-mon[184854]: failed to encode map e427 with expected crc 2023-10-09T11:02:09.973 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:09 smithi155 ceph-mon[184854]: failed to encode map e427 with expected crc 2023-10-09T11:02:10.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:02:10.027 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:02:10.040 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:10.090 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:10.182 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:10.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:10.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:10.286 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:10.340 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:10.377 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:10.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:10.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:10.454 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:10.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:10.556 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:10.590 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:10.599 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T11:02:10.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:10.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:10.605 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:02:10.701 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:10.764 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:10.857 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:10 smithi155 ceph-mon[184854]: failed to encode map e427 with expected crc 2023-10-09T11:02:10.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:10 smithi155 ceph-mon[184854]: 1.16 scrub starts 2023-10-09T11:02:10.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:10 smithi155 ceph-mon[184854]: 1.16 scrub ok 2023-10-09T11:02:10.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:10 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3601148958' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:02:10.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:10 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:02:10.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:10 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1873859118' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:02:10.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:10 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:02:10.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:10 smithi155 ceph-mon[184854]: pgmap v1056: 119 pgs: 1 peering, 118 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 52 MiB/s wr, 838 op/s; 3.3 MiB/s, 53 objects/s recovering 2023-10-09T11:02:10.898 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 52 16 67383 67367 80.9408 65.4375 0.0160271 0.0123494 2023-10-09T11:02:10.919 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[200197]: failed to encode map e427 with expected crc 2023-10-09T11:02:10.919 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[200197]: 1.16 scrub starts 2023-10-09T11:02:10.919 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[200197]: 1.16 scrub ok 2023-10-09T11:02:10.919 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3601148958' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:02:10.919 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:02:10.919 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1873859118' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:02:10.919 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:02:10.920 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[200197]: pgmap v1056: 119 pgs: 1 peering, 118 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 52 MiB/s wr, 838 op/s; 3.3 MiB/s, 53 objects/s recovering 2023-10-09T11:02:10.920 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[195507]: failed to encode map e427 with expected crc 2023-10-09T11:02:10.920 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[195507]: 1.16 scrub starts 2023-10-09T11:02:10.920 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[195507]: 1.16 scrub ok 2023-10-09T11:02:10.920 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3601148958' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:02:10.920 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:02:10.920 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1873859118' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:02:10.920 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:02:10.920 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:10 smithi125 ceph-mon[195507]: pgmap v1056: 119 pgs: 1 peering, 118 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 52 MiB/s wr, 838 op/s; 3.3 MiB/s, 53 objects/s recovering 2023-10-09T11:02:11.194 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T11:02:11.332 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:11.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:11.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:11.436 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:11.477 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:02:11] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2023-10-09T11:02:11.481 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:11.611 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:11.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:11.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:11.708 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:11 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:11.709 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:11 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T11:02:11.709 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:11 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:02:11.709 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:11 smithi155 ceph-mon[184854]: osdmap e428: 8 total, 8 up, 7 in 2023-10-09T11:02:11.709 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:11 smithi155 ceph-mon[184854]: failed to encode map e428 with expected crc 2023-10-09T11:02:11.709 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:11 smithi155 ceph-mon[184854]: failed to encode map e428 with expected crc 2023-10-09T11:02:11.709 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:11 smithi155 ceph-mon[184854]: failed to encode map e428 with expected crc 2023-10-09T11:02:11.709 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:11 smithi155 ceph-mon[184854]: failed to encode map e428 with expected crc 2023-10-09T11:02:11.709 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:11 smithi155 ceph-mon[184854]: failed to encode map e428 with expected crc 2023-10-09T11:02:11.716 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:11.724 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:11.754 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:11.791 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:11.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:11.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:11.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:11.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T11:02:11.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:02:11.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[200197]: osdmap e428: 8 total, 8 up, 7 in 2023-10-09T11:02:11.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[200197]: failed to encode map e428 with expected crc 2023-10-09T11:02:11.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[200197]: failed to encode map e428 with expected crc 2023-10-09T11:02:11.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[200197]: failed to encode map e428 with expected crc 2023-10-09T11:02:11.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[200197]: failed to encode map e428 with expected crc 2023-10-09T11:02:11.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[200197]: failed to encode map e428 with expected crc 2023-10-09T11:02:11.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:11.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T11:02:11.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:02:11.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[195507]: osdmap e428: 8 total, 8 up, 7 in 2023-10-09T11:02:11.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[195507]: failed to encode map e428 with expected crc 2023-10-09T11:02:11.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[195507]: failed to encode map e428 with expected crc 2023-10-09T11:02:11.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[195507]: failed to encode map e428 with expected crc 2023-10-09T11:02:11.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[195507]: failed to encode map e428 with expected crc 2023-10-09T11:02:11.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:11 smithi125 ceph-mon[195507]: failed to encode map e428 with expected crc 2023-10-09T11:02:11.898 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 53 16 68450 68434 80.6718 66.6875 0.0104379 0.0123908 2023-10-09T11:02:11.900 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:11.943 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:02:11.943 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T11:02:11.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T11:02:11.944 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:02:11.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.944 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:39213 8.2 8:4e7be898:::benchmark_data_smithi125_329074_object39212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T11:02:11.944 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:53:35.895072+0000", 2023-10-09T11:02:11.944 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 516.03280112000004, 2023-10-09T11:02:11.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.128363015, 2023-10-09T11:02:11.944 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:11.944 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:11.944 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:11.944 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:02:11.944 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:02:11.945 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39213 2023-10-09T11:02:11.945 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.945 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:11.945 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.945 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:11.945 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895072+0000", 2023-10-09T11:02:11.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.945 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.945 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.945 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:11.945 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895072+0000", 2023-10-09T11:02:11.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.945 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:11.946 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895070+0000", 2023-10-09T11:02:11.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:02:11.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:11.946 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895204+0000", 2023-10-09T11:02:11.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000133561 2023-10-09T11:02:11.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:11.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895215+0000", 2023-10-09T11:02:11.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0925e-05 2023-10-09T11:02:11.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:11.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895220+0000", 2023-10-09T11:02:11.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4949999999999999e-06 2023-10-09T11:02:11.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:11.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895798+0000", 2023-10-09T11:02:11.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00057733700000000003 2023-10-09T11:02:11.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:11.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895926+0000", 2023-10-09T11:02:11.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012778600000000001 2023-10-09T11:02:11.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:02:11.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.896072+0000", 2023-10-09T11:02:11.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014615599999999999 2023-10-09T11:02:11.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:11.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.897973+0000", 2023-10-09T11:02:11.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019008510000000001 2023-10-09T11:02:11.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:11.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.899072+0000", 2023-10-09T11:02:11.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010995239999999999 2023-10-09T11:02:11.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:11.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023353+0000", 2023-10-09T11:02:11.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.124281323 2023-10-09T11:02:11.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:11.950 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023405+0000", 2023-10-09T11:02:11.950 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1158e-05 2023-10-09T11:02:11.950 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:11.950 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023435+0000", 2023-10-09T11:02:11.950 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.006e-05 2023-10-09T11:02:11.950 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.950 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:11.950 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.950 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.950 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83954 8.4 8:2689e239:::benchmark_data_smithi125_329074_object83953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T11:02:11.951 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.108148+0000", 2023-10-09T11:02:11.951 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 474.819724968, 2023-10-09T11:02:11.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1275014759999999, 2023-10-09T11:02:11.951 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:11.951 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:11.951 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:11.951 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:02:11.951 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:02:11.951 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83954 2023-10-09T11:02:11.951 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.951 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:11.951 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.951 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:11.952 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108148+0000", 2023-10-09T11:02:11.952 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.952 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.952 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.952 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:11.952 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108148+0000", 2023-10-09T11:02:11.952 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.952 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.952 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.952 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:11.952 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108146+0000", 2023-10-09T11:02:11.952 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:02:11.952 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.953 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:11.953 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108191+0000", 2023-10-09T11:02:11.953 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4549000000000003e-05 2023-10-09T11:02:11.953 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.953 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:11.953 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108215+0000", 2023-10-09T11:02:11.953 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3994000000000001e-05 2023-10-09T11:02:11.953 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.953 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:11.953 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108222+0000", 2023-10-09T11:02:11.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4499999999999998e-06 2023-10-09T11:02:11.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.954 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.954 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:11.954 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108279+0000", 2023-10-09T11:02:11.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6643e-05 2023-10-09T11:02:11.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.954 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.954 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:11.954 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108288+0000", 2023-10-09T11:02:11.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5109999999999992e-06 2023-10-09T11:02:11.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:11.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.872892+0000", 2023-10-09T11:02:11.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.76460362800000004 2023-10-09T11:02:11.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:11.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.872903+0000", 2023-10-09T11:02:11.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1158000000000001e-05 2023-10-09T11:02:11.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:11.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213528+0000", 2023-10-09T11:02:11.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.340624501 2023-10-09T11:02:11.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:11.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213679+0000", 2023-10-09T11:02:11.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151578 2023-10-09T11:02:11.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T11:02:11.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213861+0000", 2023-10-09T11:02:11.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018154 2023-10-09T11:02:11.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:11.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219288+0000", 2023-10-09T11:02:11.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00542677 2023-10-09T11:02:11.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:11.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219340+0000", 2023-10-09T11:02:11.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2657000000000001e-05 2023-10-09T11:02:11.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:11.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219644+0000", 2023-10-09T11:02:11.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030375299999999998 2023-10-09T11:02:11.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:11.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235530+0000", 2023-10-09T11:02:11.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01588606 2023-10-09T11:02:11.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:11.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235575+0000", 2023-10-09T11:02:11.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5012999999999997e-05 2023-10-09T11:02:11.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.959 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.959 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:11.959 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235649+0000", 2023-10-09T11:02:11.959 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4252000000000007e-05 2023-10-09T11:02:11.959 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.959 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:11.959 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.959 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.959 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.959 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83959 8.4 8:27b6f913:::benchmark_data_smithi125_329074_object83958:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T11:02:11.959 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.112252+0000", 2023-10-09T11:02:11.959 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 474.815620938, 2023-10-09T11:02:11.959 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1235403750000001, 2023-10-09T11:02:11.960 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:11.960 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:11.960 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:11.960 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:02:11.960 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:02:11.960 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83959 2023-10-09T11:02:11.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.960 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:11.960 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.960 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:11.960 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112252+0000", 2023-10-09T11:02:11.960 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.961 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:11.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112252+0000", 2023-10-09T11:02:11.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.961 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:11.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112251+0000", 2023-10-09T11:02:11.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:02:11.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.961 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:11.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112312+0000", 2023-10-09T11:02:11.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0888999999999999e-05 2023-10-09T11:02:11.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:11.962 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112324+0000", 2023-10-09T11:02:11.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2585999999999999e-05 2023-10-09T11:02:11.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:11.962 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112329+0000", 2023-10-09T11:02:11.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9880000000000004e-06 2023-10-09T11:02:11.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:11.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112378+0000", 2023-10-09T11:02:11.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8905e-05 2023-10-09T11:02:11.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:11.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112389+0000", 2023-10-09T11:02:11.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0665e-05 2023-10-09T11:02:11.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:11.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.873769+0000", 2023-10-09T11:02:11.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.76138013699999996 2023-10-09T11:02:11.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:11.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.873776+0000", 2023-10-09T11:02:11.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.17e-06 2023-10-09T11:02:11.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:11.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216109+0000", 2023-10-09T11:02:11.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.342332699 2023-10-09T11:02:11.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:11.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216239+0000", 2023-10-09T11:02:11.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012997 2023-10-09T11:02:11.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T11:02:11.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216354+0000", 2023-10-09T11:02:11.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115116 2023-10-09T11:02:11.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:11.966 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219398+0000", 2023-10-09T11:02:11.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030438679999999999 2023-10-09T11:02:11.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.966 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.966 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:11.966 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219407+0000", 2023-10-09T11:02:11.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1870000000000003e-06 2023-10-09T11:02:11.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.966 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.966 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:11.966 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219682+0000", 2023-10-09T11:02:11.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00027478600000000001 2023-10-09T11:02:11.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.967 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:11.967 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235686+0000", 2023-10-09T11:02:11.967 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016004852 2023-10-09T11:02:11.967 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.967 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:11.967 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235725+0000", 2023-10-09T11:02:11.967 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8392000000000001e-05 2023-10-09T11:02:11.967 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.967 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:11.967 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235792+0000", 2023-10-09T11:02:11.968 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7311000000000007e-05 2023-10-09T11:02:11.968 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.968 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:11.968 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.968 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.968 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.968 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:93330 8.2 8:4f5a2bcc:::benchmark_data_smithi125_329074_object93329:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T11:02:11.968 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:35.625993+0000", 2023-10-09T11:02:11.968 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 456.30188005700001, 2023-10-09T11:02:11.968 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.12993565000000001, 2023-10-09T11:02:11.968 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:11.968 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:11.968 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:11.969 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:02:11.969 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:02:11.969 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 93330 2023-10-09T11:02:11.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.969 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:11.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.969 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:11.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625993+0000", 2023-10-09T11:02:11.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.969 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:11.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625993+0000", 2023-10-09T11:02:11.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:11.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625992+0000", 2023-10-09T11:02:11.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:11.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:11.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626043+0000", 2023-10-09T11:02:11.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0500000000000001e-05 2023-10-09T11:02:11.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:11.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626051+0000", 2023-10-09T11:02:11.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0509999999999997e-06 2023-10-09T11:02:11.971 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:11.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626056+0000", 2023-10-09T11:02:11.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9570000000000001e-06 2023-10-09T11:02:11.971 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:11.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626113+0000", 2023-10-09T11:02:11.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.736e-05 2023-10-09T11:02:11.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.972 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:11.972 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626283+0000", 2023-10-09T11:02:11.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017031699999999999 2023-10-09T11:02:11.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.972 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:02:11.972 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626463+0000", 2023-10-09T11:02:11.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018018 2023-10-09T11:02:11.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:11.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.634095+0000", 2023-10-09T11:02:11.973 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0076312469999999999 2023-10-09T11:02:11.973 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:11.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.637111+0000", 2023-10-09T11:02:11.973 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003016725 2023-10-09T11:02:11.973 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:11.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755853+0000", 2023-10-09T11:02:11.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11874143299999999 2023-10-09T11:02:11.974 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.974 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.974 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:11.974 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755887+0000", 2023-10-09T11:02:11.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4017999999999999e-05 2023-10-09T11:02:11.974 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.974 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.974 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:11.974 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755928+0000", 2023-10-09T11:02:11.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1452999999999998e-05 2023-10-09T11:02:11.974 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.974 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:11.975 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.975 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.975 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.975 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:93340 8.2 8:4653dd45:::benchmark_data_smithi125_329074_object93339:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T11:02:11.975 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:35.641334+0000", 2023-10-09T11:02:11.975 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 456.28653911700002, 2023-10-09T11:02:11.975 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.125653775, 2023-10-09T11:02:11.975 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:11.975 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:11.975 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:11.975 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:02:11.975 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:02:11.976 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 93340 2023-10-09T11:02:11.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.976 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:11.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:11.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641334+0000", 2023-10-09T11:02:11.976 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:11.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641334+0000", 2023-10-09T11:02:11.976 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:11.977 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641333+0000", 2023-10-09T11:02:11.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:02:11.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:11.977 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641362+0000", 2023-10-09T11:02:11.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9034999999999999e-05 2023-10-09T11:02:11.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:11.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641372+0000", 2023-10-09T11:02:11.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0597e-05 2023-10-09T11:02:11.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:11.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641377+0000", 2023-10-09T11:02:11.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7060000000000003e-06 2023-10-09T11:02:11.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:11.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641429+0000", 2023-10-09T11:02:11.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1721000000000002e-05 2023-10-09T11:02:11.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.979 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:11.979 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641529+0000", 2023-10-09T11:02:11.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000100602 2023-10-09T11:02:11.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.979 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:02:11.979 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641632+0000", 2023-10-09T11:02:11.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103099 2023-10-09T11:02:11.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.980 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:11.980 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.649088+0000", 2023-10-09T11:02:11.980 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0074551569999999996 2023-10-09T11:02:11.980 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.980 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.980 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:11.980 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.649159+0000", 2023-10-09T11:02:11.980 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1760000000000004e-05 2023-10-09T11:02:11.980 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.980 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.980 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:11.980 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766886+0000", 2023-10-09T11:02:11.980 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11772616499999999 2023-10-09T11:02:11.981 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.981 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.981 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:11.981 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766941+0000", 2023-10-09T11:02:11.981 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5186000000000003e-05 2023-10-09T11:02:11.981 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.981 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.981 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:11.981 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766987+0000", 2023-10-09T11:02:11.981 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6692e-05 2023-10-09T11:02:11.981 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.981 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:11.981 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.982 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.982 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:158888 8.2 8:4103043a:::benchmark_data_smithi125_329074_object62234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2023-10-09T11:02:11.982 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:02:11.982 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 356.57149376400002, 2023-10-09T11:02:11.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.123441579, 2023-10-09T11:02:11.982 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:11.982 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:11.982 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:11.982 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:02:11.982 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:02:11.982 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 158888 2023-10-09T11:02:11.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.983 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:11.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:11.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:02:11.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:11.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:02:11.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:11.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:02:11.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:11.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:11.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356381+0000", 2023-10-09T11:02:11.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.734e-06 2023-10-09T11:02:11.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:11.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356388+0000", 2023-10-09T11:02:11.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4559999999999997e-06 2023-10-09T11:02:11.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.985 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:11.985 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356392+0000", 2023-10-09T11:02:11.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0589999999999996e-06 2023-10-09T11:02:11.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.985 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:11.985 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.359480+0000", 2023-10-09T11:02:11.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030878379999999999 2023-10-09T11:02:11.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:11.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.359628+0000", 2023-10-09T11:02:11.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014820299999999999 2023-10-09T11:02:11.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:02:11.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.360074+0000", 2023-10-09T11:02:11.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00044630599999999999 2023-10-09T11:02:11.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:11.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.362052+0000", 2023-10-09T11:02:11.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019782720000000001 2023-10-09T11:02:11.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:11.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.362671+0000", 2023-10-09T11:02:11.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00061827099999999999 2023-10-09T11:02:11.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:11.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479765+0000", 2023-10-09T11:02:11.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11709412199999999 2023-10-09T11:02:11.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.988 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:11.988 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479796+0000", 2023-10-09T11:02:11.988 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0895000000000002e-05 2023-10-09T11:02:11.988 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.988 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:11.988 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479821+0000", 2023-10-09T11:02:11.988 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4816000000000001e-05 2023-10-09T11:02:11.988 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.988 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:11.988 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.988 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.989 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:100843 9.c e353/332 9:3d29ba4c:::benchmark_data_smithi125_363789_object100842:head v 353'6258, mlcod=353'6256)", 2023-10-09T11:02:11.989 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:43.227548+0000", 2023-10-09T11:02:11.989 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 208.70032487099999, 2023-10-09T11:02:11.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0253425, 2023-10-09T11:02:11.989 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:11.989 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:11.989 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:11.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:11.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227548+0000", 2023-10-09T11:02:11.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:11.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227548+0000", 2023-10-09T11:02:11.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:11.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227547+0000", 2023-10-09T11:02:11.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:02:11.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:11.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227594+0000", 2023-10-09T11:02:11.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7648000000000002e-05 2023-10-09T11:02:11.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:11.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227615+0000", 2023-10-09T11:02:11.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0435e-05 2023-10-09T11:02:11.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:11.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227622+0000", 2023-10-09T11:02:11.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4850000000000003e-06 2023-10-09T11:02:11.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:11.992 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.229957+0000", 2023-10-09T11:02:11.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002334381 2023-10-09T11:02:11.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:11.992 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.229976+0000", 2023-10-09T11:02:11.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9596999999999999e-05 2023-10-09T11:02:11.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:11.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.252847+0000", 2023-10-09T11:02:11.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022871209 2023-10-09T11:02:11.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:11.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.252890+0000", 2023-10-09T11:02:11.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2920000000000002e-05 2023-10-09T11:02:11.993 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.993 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:11.993 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.994 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:102485 9.c e353/332 9:3d3829d9:::benchmark_data_smithi125_363789_object102484:head v 353'6351, mlcod=353'6348)", 2023-10-09T11:02:11.994 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:45.158190+0000", 2023-10-09T11:02:11.994 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 206.76968290600001, 2023-10-09T11:02:11.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024704652000000001, 2023-10-09T11:02:11.994 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:11.994 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:11.994 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:11.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:11.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158190+0000", 2023-10-09T11:02:11.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.995 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.995 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:11.995 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158190+0000", 2023-10-09T11:02:11.995 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.995 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.995 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.995 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:11.995 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158189+0000", 2023-10-09T11:02:11.995 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:11.995 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.995 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.995 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:11.995 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158233+0000", 2023-10-09T11:02:11.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3201999999999997e-05 2023-10-09T11:02:11.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.996 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:11.996 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158242+0000", 2023-10-09T11:02:11.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9400000000000008e-06 2023-10-09T11:02:11.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.996 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:11.996 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158248+0000", 2023-10-09T11:02:11.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.844e-06 2023-10-09T11:02:11.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:11.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.161725+0000", 2023-10-09T11:02:11.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034765450000000002 2023-10-09T11:02:11.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:11.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.161750+0000", 2023-10-09T11:02:11.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4538000000000001e-05 2023-10-09T11:02:11.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:11.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.182847+0000", 2023-10-09T11:02:11.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.02109778 2023-10-09T11:02:11.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:11.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.182894+0000", 2023-10-09T11:02:11.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7141000000000001e-05 2023-10-09T11:02:11.998 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.998 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:11.998 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:11.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.998 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:102487 9.c e353/332 9:37d8c2ee:::benchmark_data_smithi125_363789_object102486:head v 353'6352, mlcod=353'6348)", 2023-10-09T11:02:11.998 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:45.158254+0000", 2023-10-09T11:02:11.999 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 206.769618587, 2023-10-09T11:02:11.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027965380000000001, 2023-10-09T11:02:11.999 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:11.999 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:11.999 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:11.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:11.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158254+0000", 2023-10-09T11:02:11.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:11.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:11.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:11.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:11.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158254+0000", 2023-10-09T11:02:11.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.000 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:12.000 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158254+0000", 2023-10-09T11:02:12.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:12.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.000 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:12.000 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158280+0000", 2023-10-09T11:02:12.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6463000000000001e-05 2023-10-09T11:02:12.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.000 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:12.000 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158296+0000", 2023-10-09T11:02:12.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6246999999999999e-05 2023-10-09T11:02:12.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:12.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158305+0000", 2023-10-09T11:02:12.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9639999999999992e-06 2023-10-09T11:02:12.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:12.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165187+0000", 2023-10-09T11:02:12.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0068813679999999997 2023-10-09T11:02:12.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:12.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165209+0000", 2023-10-09T11:02:12.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1926999999999999e-05 2023-10-09T11:02:12.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:12.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186167+0000", 2023-10-09T11:02:12.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020958641 2023-10-09T11:02:12.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:12.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186220+0000", 2023-10-09T11:02:12.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2243000000000001e-05 2023-10-09T11:02:12.003 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.003 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:12.003 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.003 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:102490 9.4 e353/330 9:207553f4:::benchmark_data_smithi125_363789_object102489:head v 353'6429, mlcod=353'6425)", 2023-10-09T11:02:12.003 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:45.159295+0000", 2023-10-09T11:02:12.003 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 206.76857777000001, 2023-10-09T11:02:12.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026994220999999999, 2023-10-09T11:02:12.003 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:12.003 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:12.003 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:12.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:12.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159295+0000", 2023-10-09T11:02:12.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:12.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159295+0000", 2023-10-09T11:02:12.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:12.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159294+0000", 2023-10-09T11:02:12.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:12.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:12.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159332+0000", 2023-10-09T11:02:12.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7502000000000001e-05 2023-10-09T11:02:12.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:12.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159344+0000", 2023-10-09T11:02:12.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2418999999999999e-05 2023-10-09T11:02:12.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:12.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159349+0000", 2023-10-09T11:02:12.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2619999999999998e-06 2023-10-09T11:02:12.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:12.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165240+0000", 2023-10-09T11:02:12.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0058900580000000001 2023-10-09T11:02:12.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:12.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165259+0000", 2023-10-09T11:02:12.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9369e-05 2023-10-09T11:02:12.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.007 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:12.007 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186223+0000", 2023-10-09T11:02:12.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020963682000000001 2023-10-09T11:02:12.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.007 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:12.007 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186289+0000", 2023-10-09T11:02:12.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6600000000000006e-05 2023-10-09T11:02:12.007 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.007 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:12.007 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.008 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.008 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:102494 9.4 e353/330 9:2c49d3a3:::benchmark_data_smithi125_363789_object102493:head v 353'6430, mlcod=353'6425)", 2023-10-09T11:02:12.008 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:45.162300+0000", 2023-10-09T11:02:12.008 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 206.765572451, 2023-10-09T11:02:12.008 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024081163999999999, 2023-10-09T11:02:12.008 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:12.008 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:12.008 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:12.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:12.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162300+0000", 2023-10-09T11:02:12.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.009 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.009 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:12.009 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162300+0000", 2023-10-09T11:02:12.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.009 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.009 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:12.009 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162299+0000", 2023-10-09T11:02:12.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:02:12.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.010 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:12.010 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162360+0000", 2023-10-09T11:02:12.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1148999999999993e-05 2023-10-09T11:02:12.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.010 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:12.010 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162375+0000", 2023-10-09T11:02:12.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.469e-05 2023-10-09T11:02:12.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.010 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:12.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.162383+0000", 2023-10-09T11:02:12.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3289999999999996e-06 2023-10-09T11:02:12.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:12.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.168660+0000", 2023-10-09T11:02:12.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062775629999999999 2023-10-09T11:02:12.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:12.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.168682+0000", 2023-10-09T11:02:12.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1498999999999999e-05 2023-10-09T11:02:12.012 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:12.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186292+0000", 2023-10-09T11:02:12.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017610388000000001 2023-10-09T11:02:12.012 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:12.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186381+0000", 2023-10-09T11:02:12.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9489999999999999e-05 2023-10-09T11:02:12.012 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.012 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:12.013 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.013 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:141363 9.e e367/358 9:7b79acb8:::benchmark_data_smithi125_363789_object31064:head v 367'9030, mlcod=367'9028)", 2023-10-09T11:02:12.013 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:31.211612+0000", 2023-10-09T11:02:12.013 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 160.71626043200001, 2023-10-09T11:02:12.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023916006, 2023-10-09T11:02:12.013 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:12.013 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:12.013 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:12.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:12.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211612+0000", 2023-10-09T11:02:12.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.014 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:12.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211612+0000", 2023-10-09T11:02:12.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.014 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:12.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211611+0000", 2023-10-09T11:02:12.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:02:12.015 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.015 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.015 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:12.015 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211622+0000", 2023-10-09T11:02:12.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0907000000000001e-05 2023-10-09T11:02:12.015 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.015 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.015 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:12.015 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211641+0000", 2023-10-09T11:02:12.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9372e-05 2023-10-09T11:02:12.015 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.015 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.015 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:12.015 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.211648+0000", 2023-10-09T11:02:12.016 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0870000000000001e-06 2023-10-09T11:02:12.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.016 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:12.016 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.213421+0000", 2023-10-09T11:02:12.016 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0017721220000000001 2023-10-09T11:02:12.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.016 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:12.016 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.213445+0000", 2023-10-09T11:02:12.016 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4845e-05 2023-10-09T11:02:12.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.016 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:12.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.235498+0000", 2023-10-09T11:02:12.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022052275999999999 2023-10-09T11:02:12.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:12.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:31.235528+0000", 2023-10-09T11:02:12.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0688000000000002e-05 2023-10-09T11:02:12.017 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.017 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:12.017 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.018 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.018 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:147234 9.4 e367/358 9:2d6b73af:::benchmark_data_smithi125_363789_object36935:head v 367'9215, mlcod=367'9213)", 2023-10-09T11:02:12.018 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:36.828586+0000", 2023-10-09T11:02:12.018 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 155.09928672500001, 2023-10-09T11:02:12.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024492692999999999, 2023-10-09T11:02:12.018 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:12.018 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:12.018 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:12.018 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:12.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.828586+0000", 2023-10-09T11:02:12.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:12.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.828586+0000", 2023-10-09T11:02:12.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:12.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.828586+0000", 2023-10-09T11:02:12.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:12.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:12.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.828594+0000", 2023-10-09T11:02:12.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.867e-06 2023-10-09T11:02:12.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.020 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:12.020 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.828601+0000", 2023-10-09T11:02:12.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.759e-06 2023-10-09T11:02:12.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.020 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:12.020 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.828605+0000", 2023-10-09T11:02:12.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0010000000000003e-06 2023-10-09T11:02:12.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:12.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.832080+0000", 2023-10-09T11:02:12.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034751349999999999 2023-10-09T11:02:12.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:12.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.832098+0000", 2023-10-09T11:02:12.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7558999999999999e-05 2023-10-09T11:02:12.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:12.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.853051+0000", 2023-10-09T11:02:12.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020953171 2023-10-09T11:02:12.022 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.022 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.022 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:12.022 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:36.853079+0000", 2023-10-09T11:02:12.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7631999999999999e-05 2023-10-09T11:02:12.022 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.022 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:12.022 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.022 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.022 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.022 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:153414 9.c e374/332 9:3768b944:::benchmark_data_smithi125_363789_object43115:head v 374'9510, mlcod=374'9508)", 2023-10-09T11:02:12.022 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:46.250266+0000", 2023-10-09T11:02:12.022 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 145.67760655800001, 2023-10-09T11:02:12.023 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025683302000000002, 2023-10-09T11:02:12.023 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:12.023 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:12.023 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:12.023 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.023 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:12.023 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250266+0000", 2023-10-09T11:02:12.023 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.023 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.023 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.023 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:12.023 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250266+0000", 2023-10-09T11:02:12.023 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.024 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:12.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250265+0000", 2023-10-09T11:02:12.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:02:12.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.024 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:12.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250275+0000", 2023-10-09T11:02:12.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7089999999999997e-06 2023-10-09T11:02:12.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.024 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:12.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250292+0000", 2023-10-09T11:02:12.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7082e-05 2023-10-09T11:02:12.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.025 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:12.025 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250300+0000", 2023-10-09T11:02:12.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4689999999999997e-06 2023-10-09T11:02:12.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.025 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:12.025 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250340+0000", 2023-10-09T11:02:12.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9758000000000001e-05 2023-10-09T11:02:12.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.026 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:12.026 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250362+0000", 2023-10-09T11:02:12.026 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2463000000000002e-05 2023-10-09T11:02:12.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.026 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:12.026 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.275910+0000", 2023-10-09T11:02:12.026 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025547325999999999 2023-10-09T11:02:12.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.026 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:12.026 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.275949+0000", 2023-10-09T11:02:12.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9786999999999997e-05 2023-10-09T11:02:12.027 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.027 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:12.027 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.027 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.027 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.027 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:9816 10.6 e406/403 10:610863d1:::benchmark_data_smithi125_402869_object9815:head v 406'620, mlcod=406'618)", 2023-10-09T11:02:12.027 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:22.339372+0000", 2023-10-09T11:02:12.027 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 49.588500371000002, 2023-10-09T11:02:12.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072690006000000001, 2023-10-09T11:02:12.027 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:12.027 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:12.028 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:12.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:12.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339372+0000", 2023-10-09T11:02:12.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:12.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339372+0000", 2023-10-09T11:02:12.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:12.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339372+0000", 2023-10-09T11:02:12.029 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:12.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.029 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:12.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339395+0000", 2023-10-09T11:02:12.029 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3051e-05 2023-10-09T11:02:12.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.029 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:12.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339404+0000", 2023-10-09T11:02:12.029 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.713e-06 2023-10-09T11:02:12.030 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.030 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.030 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:12.030 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339410+0000", 2023-10-09T11:02:12.030 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.181e-06 2023-10-09T11:02:12.030 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.030 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.030 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:12.030 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339487+0000", 2023-10-09T11:02:12.030 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6940999999999994e-05 2023-10-09T11:02:12.030 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.030 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:12.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339505+0000", 2023-10-09T11:02:12.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8700000000000001e-05 2023-10-09T11:02:12.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:12.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.412023+0000", 2023-10-09T11:02:12.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072518113999999995 2023-10-09T11:02:12.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:12.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.412062+0000", 2023-10-09T11:02:12.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8991e-05 2023-10-09T11:02:12.032 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.032 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:12.032 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.032 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:9820 10.d e406/403 10:b0a398bf:::benchmark_data_smithi125_402869_object9819:head v 406'628, mlcod=406'626)", 2023-10-09T11:02:12.032 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:22.339685+0000", 2023-10-09T11:02:12.032 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 49.588187945999998, 2023-10-09T11:02:12.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072206606000000007, 2023-10-09T11:02:12.032 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:12.032 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:12.032 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:12.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:12.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339685+0000", 2023-10-09T11:02:12.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:12.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339685+0000", 2023-10-09T11:02:12.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:12.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339684+0000", 2023-10-09T11:02:12.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:12.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:12.034 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339744+0000", 2023-10-09T11:02:12.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9889999999999997e-05 2023-10-09T11:02:12.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:12.034 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339754+0000", 2023-10-09T11:02:12.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0492999999999999e-05 2023-10-09T11:02:12.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.035 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:12.035 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339760+0000", 2023-10-09T11:02:12.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.699e-06 2023-10-09T11:02:12.035 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.035 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.035 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:12.035 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339792+0000", 2023-10-09T11:02:12.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2079999999999998e-05 2023-10-09T11:02:12.035 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.035 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.035 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:12.035 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339805+0000", 2023-10-09T11:02:12.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2809999999999999e-05 2023-10-09T11:02:12.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:12.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411854+0000", 2023-10-09T11:02:12.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072048880999999995 2023-10-09T11:02:12.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:12.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411891+0000", 2023-10-09T11:02:12.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7434000000000001e-05 2023-10-09T11:02:12.036 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.036 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:12.037 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.037 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.037 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:9825 10.2 e406/403 10:49e48d05:::benchmark_data_smithi125_402869_object9824:head v 406'599, mlcod=406'597)", 2023-10-09T11:02:12.037 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:22.340607+0000", 2023-10-09T11:02:12.037 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 49.587265324000001, 2023-10-09T11:02:12.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.071013035000000002, 2023-10-09T11:02:12.037 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:12.037 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:12.037 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:12.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.037 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:12.037 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340607+0000", 2023-10-09T11:02:12.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:12.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340607+0000", 2023-10-09T11:02:12.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:12.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340607+0000", 2023-10-09T11:02:12.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:12.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340667+0000", 2023-10-09T11:02:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0310999999999999e-05 2023-10-09T11:02:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340681+0000", 2023-10-09T11:02:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4199000000000001e-05 2023-10-09T11:02:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:12.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340688+0000", 2023-10-09T11:02:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9979999999999997e-06 2023-10-09T11:02:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410579+0000", 2023-10-09T11:02:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069890939999999999 2023-10-09T11:02:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410601+0000", 2023-10-09T11:02:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1953000000000001e-05 2023-10-09T11:02:12.040 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411576+0000", 2023-10-09T11:02:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00097502500000000002 2023-10-09T11:02:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411620+0000", 2023-10-09T11:02:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4305999999999999e-05 2023-10-09T11:02:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.041 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:9818 10.6 e406/403 10:67ea2a60:::benchmark_data_smithi125_402869_object9817:head v 406'621, mlcod=406'618)", 2023-10-09T11:02:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:22.340768+0000", 2023-10-09T11:02:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 49.587104773999997, 2023-10-09T11:02:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.071318931000000002, 2023-10-09T11:02:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340768+0000", 2023-10-09T11:02:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.042 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340768+0000", 2023-10-09T11:02:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340767+0000", 2023-10-09T11:02:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340793+0000", 2023-10-09T11:02:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5539000000000001e-05 2023-10-09T11:02:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410523+0000", 2023-10-09T11:02:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069730406999999994 2023-10-09T11:02:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410538+0000", 2023-10-09T11:02:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4747e-05 2023-10-09T11:02:12.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410587+0000", 2023-10-09T11:02:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8903000000000003e-05 2023-10-09T11:02:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410610+0000", 2023-10-09T11:02:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3288000000000001e-05 2023-10-09T11:02:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:12.045 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.412066+0000", 2023-10-09T11:02:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001456127 2023-10-09T11:02:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.412087+0000", 2023-10-09T11:02:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0557999999999999e-05 2023-10-09T11:02:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.046 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:9823 10.d e406/403 10:b90906b6:::benchmark_data_smithi125_402869_object9822:head v 406'629, mlcod=406'626)", 2023-10-09T11:02:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:22.341423+0000", 2023-10-09T11:02:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 49.586449369, 2023-10-09T11:02:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.07048981, 2023-10-09T11:02:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.341423+0000", 2023-10-09T11:02:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.047 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.341423+0000", 2023-10-09T11:02:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.341423+0000", 2023-10-09T11:02:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.341489+0000", 2023-10-09T11:02:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6185000000000004e-05 2023-10-09T11:02:12.048 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410523+0000", 2023-10-09T11:02:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069033960000000005 2023-10-09T11:02:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410538+0000", 2023-10-09T11:02:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5545000000000001e-05 2023-10-09T11:02:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.049 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410587+0000", 2023-10-09T11:02:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8813000000000002e-05 2023-10-09T11:02:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410610+0000", 2023-10-09T11:02:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3119999999999999e-05 2023-10-09T11:02:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411895+0000", 2023-10-09T11:02:12.050 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012849770000000001 2023-10-09T11:02:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411913+0000", 2023-10-09T11:02:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7864e-05 2023-10-09T11:02:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:67600 10.6 e428/403 10:64cebe01:::benchmark_data_smithi125_402869_object67599:head v 428'4349, mlcod=428'4347)", 2023-10-09T11:02:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:02:11.083670+0000", 2023-10-09T11:02:12.051 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.84420223999999999, 2023-10-09T11:02:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013979207, 2023-10-09T11:02:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:11.083670+0000", 2023-10-09T11:02:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:12.052 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:11.083670+0000", 2023-10-09T11:02:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:11.083670+0000", 2023-10-09T11:02:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:11.083687+0000", 2023-10-09T11:02:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7300999999999999e-05 2023-10-09T11:02:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:11.083692+0000", 2023-10-09T11:02:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3869999999999997e-06 2023-10-09T11:02:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:11.083695+0000", 2023-10-09T11:02:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.773e-06 2023-10-09T11:02:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:12.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:11.084108+0000", 2023-10-09T11:02:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00041356900000000002 2023-10-09T11:02:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:11.084127+0000", 2023-10-09T11:02:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9028000000000001e-05 2023-10-09T11:02:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:11.097615+0000", 2023-10-09T11:02:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013487888 2023-10-09T11:02:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:12.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:11.097650+0000", 2023-10-09T11:02:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4570000000000003e-05 2023-10-09T11:02:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:12.056 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:12.056 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:02:12.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:12.057 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:12.058 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:12.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:12.158 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:12.223 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:12.305 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:12.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:12.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:12.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:12.429 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:02:12.450 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:12.496 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:02:12.523 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:02:12.546 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:12.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:12.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:12.566 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:02:12.611 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:02:12.654 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:12.657 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:02:12.677 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:02:12.704 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:12.717 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:02:12.735 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:12.737 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:02:12.737 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:02:12.738 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:02:12.738 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:02:12.738 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:02:12.738 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:02:12.738 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:02:12.738 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:02:12.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:12.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:12.770 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:12 smithi125 ceph-mon[195507]: pgmap v1058: 119 pgs: 1 peering, 118 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 3.3 MiB/s, 53 objects/s recovering 2023-10-09T11:02:12.770 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:12 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/833570096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:12.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:12 smithi125 ceph-mon[200197]: pgmap v1058: 119 pgs: 1 peering, 118 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 3.3 MiB/s, 53 objects/s recovering 2023-10-09T11:02:12.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:12 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/833570096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:12.841 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:12.882 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:12 smithi155 ceph-mon[184854]: pgmap v1058: 119 pgs: 1 peering, 118 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 3.3 MiB/s, 53 objects/s recovering 2023-10-09T11:02:12.882 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:12 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/833570096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:12.886 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:12.898 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 54 16 69435 69419 80.3179 61.5625 0.0130401 0.0124442 2023-10-09T11:02:13.367 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:02:13.821 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:13.901 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 55 16 70493 70477 80.0567 66.125 0.00940524 0.0124858 2023-10-09T11:02:14.181 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T11:02:14.217 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:14 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2269268072' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:02:14.217 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:14 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:02:14.217 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:14 smithi125 ceph-mon[195507]: pgmap v1059: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s; 3.3 MiB/s, 52 objects/s recovering 2023-10-09T11:02:14.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:14 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2269268072' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:02:14.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:14 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:02:14.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:14 smithi125 ceph-mon[200197]: pgmap v1059: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s; 3.3 MiB/s, 52 objects/s recovering 2023-10-09T11:02:14.377 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:14.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:14.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:14.390 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T11:02:14.467 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:14 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2269268072' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:02:14.467 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:14 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:02:14.467 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:14 smithi155 ceph-mon[184854]: pgmap v1059: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s; 3.3 MiB/s, 52 objects/s recovering 2023-10-09T11:02:14.484 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:14.540 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:14.579 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:14.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:14.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:14.686 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:14.746 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:14.776 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:14.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:14.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:14.803 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T11:02:14.850 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:02:14.885 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:14.901 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 56 16 71540 71524 79.7957 65.4375 0.0296596 0.0125266 2023-10-09T11:02:14.920 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:15.217 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:02:15.259 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:15.461 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T11:02:15.462 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[195507]: osdmap e429: 8 total, 8 up, 7 in 2023-10-09T11:02:15.462 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[195507]: failed to encode map e429 with expected crc 2023-10-09T11:02:15.462 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[195507]: failed to encode map e429 with expected crc 2023-10-09T11:02:15.462 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[195507]: failed to encode map e429 with expected crc 2023-10-09T11:02:15.462 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[195507]: failed to encode map e429 with expected crc 2023-10-09T11:02:15.462 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[195507]: failed to encode map e429 with expected crc 2023-10-09T11:02:15.462 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:15.462 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:02:15.462 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T11:02:15.463 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[200197]: osdmap e429: 8 total, 8 up, 7 in 2023-10-09T11:02:15.463 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[200197]: failed to encode map e429 with expected crc 2023-10-09T11:02:15.463 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[200197]: failed to encode map e429 with expected crc 2023-10-09T11:02:15.463 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[200197]: failed to encode map e429 with expected crc 2023-10-09T11:02:15.463 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[200197]: failed to encode map e429 with expected crc 2023-10-09T11:02:15.463 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[200197]: failed to encode map e429 with expected crc 2023-10-09T11:02:15.463 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:15.463 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:02:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:15 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T11:02:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:15 smithi155 ceph-mon[184854]: osdmap e429: 8 total, 8 up, 7 in 2023-10-09T11:02:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:15 smithi155 ceph-mon[184854]: failed to encode map e429 with expected crc 2023-10-09T11:02:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:15 smithi155 ceph-mon[184854]: failed to encode map e429 with expected crc 2023-10-09T11:02:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:15 smithi155 ceph-mon[184854]: failed to encode map e429 with expected crc 2023-10-09T11:02:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:15 smithi155 ceph-mon[184854]: failed to encode map e429 with expected crc 2023-10-09T11:02:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:15 smithi155 ceph-mon[184854]: failed to encode map e429 with expected crc 2023-10-09T11:02:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:02:15.607 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:15.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:15.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:15.712 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:15.757 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:15.790 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:15.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:15.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:15.899 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:15.902 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 57 16 72427 72411 79.3684 55.4375 0.00591753 0.0125943 2023-10-09T11:02:15.951 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:16.038 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:16.040 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:16.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:16.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:16.144 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:16.186 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:16.263 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:16.265 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:02:16.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:16.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:16.368 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:16.419 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:16.429 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:16 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3521466597' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:16.429 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:16 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:16.429 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:16 smithi125 ceph-mon[200197]: pgmap v1061: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 39 MiB/s wr, 630 op/s 2023-10-09T11:02:16.429 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:16 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3521466597' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:16.429 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:16 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:16.429 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:16 smithi125 ceph-mon[195507]: pgmap v1061: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 39 MiB/s wr, 630 op/s 2023-10-09T11:02:16.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:16 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3521466597' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:16.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:16 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:16.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:16 smithi155 ceph-mon[184854]: pgmap v1061: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 39 MiB/s wr, 630 op/s 2023-10-09T11:02:16.502 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:16.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:16.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:16.608 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:16.662 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:16.750 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:02:16.750 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:02:16.901 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 58 16 73467 73451 79.1206 65 0.0132171 0.0126343 2023-10-09T11:02:17.139 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T11:02:17.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:17.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:02:17.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[200197]: osdmap e430: 8 total, 8 up, 7 in 2023-10-09T11:02:17.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[200197]: failed to encode map e430 with expected crc 2023-10-09T11:02:17.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[200197]: failed to encode map e430 with expected crc 2023-10-09T11:02:17.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[200197]: failed to encode map e430 with expected crc 2023-10-09T11:02:17.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[200197]: failed to encode map e430 with expected crc 2023-10-09T11:02:17.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[200197]: failed to encode map e430 with expected crc 2023-10-09T11:02:17.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:17.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:02:17.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[195507]: osdmap e430: 8 total, 8 up, 7 in 2023-10-09T11:02:17.223 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[195507]: failed to encode map e430 with expected crc 2023-10-09T11:02:17.223 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[195507]: failed to encode map e430 with expected crc 2023-10-09T11:02:17.223 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[195507]: failed to encode map e430 with expected crc 2023-10-09T11:02:17.223 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[195507]: failed to encode map e430 with expected crc 2023-10-09T11:02:17.223 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:17 smithi125 ceph-mon[195507]: failed to encode map e430 with expected crc 2023-10-09T11:02:17.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:17 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:17.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:17 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:02:17.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:17 smithi155 ceph-mon[184854]: osdmap e430: 8 total, 8 up, 7 in 2023-10-09T11:02:17.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:17 smithi155 ceph-mon[184854]: failed to encode map e430 with expected crc 2023-10-09T11:02:17.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:17 smithi155 ceph-mon[184854]: failed to encode map e430 with expected crc 2023-10-09T11:02:17.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:17 smithi155 ceph-mon[184854]: failed to encode map e430 with expected crc 2023-10-09T11:02:17.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:17 smithi155 ceph-mon[184854]: failed to encode map e430 with expected crc 2023-10-09T11:02:17.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:17 smithi155 ceph-mon[184854]: failed to encode map e430 with expected crc 2023-10-09T11:02:17.551 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:17.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:17.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:17.575 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:02:17.656 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:17.727 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:17.903 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 59 16 74553 74537 78.927 67.875 0.0129147 0.0126648 2023-10-09T11:02:18.070 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:18.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:18.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:18.176 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:18.225 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:18.271 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T11:02:18.333 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:18 smithi125 ceph-mon[195507]: pgmap v1063: 119 pgs: 119 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T11:02:18.333 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:18 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3007724640' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:02:18.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:18 smithi125 ceph-mon[200197]: pgmap v1063: 119 pgs: 119 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T11:02:18.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:18 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3007724640' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:02:18.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:18 smithi155 ceph-mon[184854]: pgmap v1063: 119 pgs: 119 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T11:02:18.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3007724640' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:02:18.758 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:02:18.758 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:02:18.903 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T11:02:18.904320+0000 min lat: 0.0011398 max lat: 2.07371 avg lat: 0.0127126 2023-10-09T11:02:18.903 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T11:02:18.904 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 60 16 75535 75519 78.6345 61.375 0.0185258 0.0127126 2023-10-09T11:02:19.162 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T11:02:19.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:19 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3007724640' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T11:02:19.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:19 smithi155 ceph-mon[184854]: osdmap e431: 8 total, 8 up, 7 in 2023-10-09T11:02:19.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:19 smithi155 ceph-mon[184854]: failed to encode map e431 with expected crc 2023-10-09T11:02:19.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:19 smithi155 ceph-mon[184854]: failed to encode map e431 with expected crc 2023-10-09T11:02:19.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:19 smithi155 ceph-mon[184854]: failed to encode map e431 with expected crc 2023-10-09T11:02:19.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:19 smithi155 ceph-mon[184854]: failed to encode map e431 with expected crc 2023-10-09T11:02:19.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:19 smithi155 ceph-mon[184854]: failed to encode map e431 with expected crc 2023-10-09T11:02:19.558 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:02:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:19 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3007724640' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T11:02:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:19 smithi125 ceph-mon[195507]: osdmap e431: 8 total, 8 up, 7 in 2023-10-09T11:02:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:19 smithi125 ceph-mon[195507]: failed to encode map e431 with expected crc 2023-10-09T11:02:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:19 smithi125 ceph-mon[195507]: failed to encode map e431 with expected crc 2023-10-09T11:02:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:19 smithi125 ceph-mon[195507]: failed to encode map e431 with expected crc 2023-10-09T11:02:19.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:19 smithi125 ceph-mon[195507]: failed to encode map e431 with expected crc 2023-10-09T11:02:19.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:19 smithi125 ceph-mon[195507]: failed to encode map e431 with expected crc 2023-10-09T11:02:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:19 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3007724640' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T11:02:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:19 smithi125 ceph-mon[200197]: osdmap e431: 8 total, 8 up, 7 in 2023-10-09T11:02:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:19 smithi125 ceph-mon[200197]: failed to encode map e431 with expected crc 2023-10-09T11:02:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:19 smithi125 ceph-mon[200197]: failed to encode map e431 with expected crc 2023-10-09T11:02:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:19 smithi125 ceph-mon[200197]: failed to encode map e431 with expected crc 2023-10-09T11:02:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:19 smithi125 ceph-mon[200197]: failed to encode map e431 with expected crc 2023-10-09T11:02:19.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:19 smithi125 ceph-mon[200197]: failed to encode map e431 with expected crc 2023-10-09T11:02:19.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:02:19.805 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:19.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:19.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:19.904 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 61 16 76551 76535 78.3859 63.5 0.00944694 0.0127516 2023-10-09T11:02:19.914 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:19.991 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:20.078 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T11:02:20.078 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:02:20.078 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:20.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:20.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:20.186 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:20.267 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:20.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:20 smithi125 ceph-mon[195507]: pgmap v1065: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 839 op/s 2023-10-09T11:02:20.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:20 smithi125 ceph-mon[200197]: pgmap v1065: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 839 op/s 2023-10-09T11:02:20.398 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:20.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:20.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:20.431 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:20 smithi155 ceph-mon[184854]: pgmap v1065: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 839 op/s 2023-10-09T11:02:20.469 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:20.505 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:20.548 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:20.567 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:02:20.575 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:20.586 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:20.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:20.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:20.603 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:02:20.654 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:02:20.686 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:02:20.686 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:02:20.691 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:20.699 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:02:20.758 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:20.790 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:02:20.802 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:02:20.828 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:02:20.845 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:20.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:20.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:20.853 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:02:20.864 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:02:20.864 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:02:20.864 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:02:20.864 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:02:20.864 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:02:20.864 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:02:20.864 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:02:20.864 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:02:20.904 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 62 16 77386 77370 77.9634 52.1875 0.0188494 0.0128211 2023-10-09T11:02:20.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:20.991 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:21.080 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:21.082 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T11:02:21.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:21.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:21.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:21.262 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:21.267 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:21 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4203893021' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:21.267 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:21 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:21.471 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:02:21.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:21 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4203893021' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:21.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:21 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:21.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:02:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:02:21] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.43.0" 2023-10-09T11:02:21.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:21 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4203893021' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:21.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:21 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:21.904 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 63 16 78338 78322 77.67 59.5 0.0141285 0.0128691 2023-10-09T11:02:22.226 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:22.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:22.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:22.331 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:22.389 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:22.491 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:22 smithi125 ceph-mon[195507]: pgmap v1066: 119 pgs: 119 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T11:02:22.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:22 smithi125 ceph-mon[200197]: pgmap v1066: 119 pgs: 119 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T11:02:22.621 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:22.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:22.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:22.664 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:02:22.664 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:02:22.665 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.665 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:79132 10.0 10:0a424425:::benchmark_data_smithi125_402869_object79131:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-10-09T11:02:22.665 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:02:22.662549+0000", 2023-10-09T11:02:22.665 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.001442846, 2023-10-09T11:02:22.665 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00146612, 2023-10-09T11:02:22.665 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:22.665 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:02:22.665 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:22.665 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:02:22.665 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:02:22.665 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 79132 2023-10-09T11:02:22.665 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.666 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:22.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.666 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:22.666 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662549+0000", 2023-10-09T11:02:22.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:22.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.666 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:22.666 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662549+0000", 2023-10-09T11:02:22.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:22.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:22.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662547+0000", 2023-10-09T11:02:22.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T11:02:22.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:22.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662589+0000", 2023-10-09T11:02:22.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1953000000000003e-05 2023-10-09T11:02:22.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:22.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662601+0000", 2023-10-09T11:02:22.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1358999999999999e-05 2023-10-09T11:02:22.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:22.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662607+0000", 2023-10-09T11:02:22.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2600000000000002e-06 2023-10-09T11:02:22.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:22.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662658+0000", 2023-10-09T11:02:22.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1165000000000002e-05 2023-10-09T11:02:22.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:22.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662804+0000", 2023-10-09T11:02:22.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000145282 2023-10-09T11:02:22.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:02:22.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662955+0000", 2023-10-09T11:02:22.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151729 2023-10-09T11:02:22.669 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:22.669 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:22.670 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:22.670 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.670 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:79136 10.a 10:58fd1029:::benchmark_data_smithi125_402869_object79135:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-10-09T11:02:22.670 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:02:22.662617+0000", 2023-10-09T11:02:22.670 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0013753330000000001, 2023-10-09T11:02:22.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001449959, 2023-10-09T11:02:22.670 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:22.670 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:02:22.670 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:22.670 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:02:22.670 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:02:22.671 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 79136 2023-10-09T11:02:22.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.671 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:22.671 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.671 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:22.671 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662617+0000", 2023-10-09T11:02:22.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:22.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.671 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.671 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:22.671 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662617+0000", 2023-10-09T11:02:22.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:22.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.672 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:22.672 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662617+0000", 2023-10-09T11:02:22.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:22.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.672 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:22.672 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662650+0000", 2023-10-09T11:02:22.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3840999999999997e-05 2023-10-09T11:02:22.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662655+0000", 2023-10-09T11:02:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1100000000000002e-06 2023-10-09T11:02:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662659+0000", 2023-10-09T11:02:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1149999999999998e-06 2023-10-09T11:02:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:22.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662695+0000", 2023-10-09T11:02:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6128999999999999e-05 2023-10-09T11:02:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662804+0000", 2023-10-09T11:02:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010962400000000001 2023-10-09T11:02:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:02:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.662955+0000", 2023-10-09T11:02:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000150975 2023-10-09T11:02:22.674 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:79137 10.7 e431/403 10:e97fc1e6:::benchmark_data_smithi125_402869_object79136:head v 431'4914, mlcod=431'4911)", 2023-10-09T11:02:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:02:22.663051+0000", 2023-10-09T11:02:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.00094079400000000001, 2023-10-09T11:02:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010576920000000001, 2023-10-09T11:02:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T11:02:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:22.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.663051+0000", 2023-10-09T11:02:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.663051+0000", 2023-10-09T11:02:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:22.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.663051+0000", 2023-10-09T11:02:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:02:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.663085+0000", 2023-10-09T11:02:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3862000000000002e-05 2023-10-09T11:02:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.663095+0000", 2023-10-09T11:02:22.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0197000000000001e-05 2023-10-09T11:02:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.663100+0000", 2023-10-09T11:02:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5339999999999999e-06 2023-10-09T11:02:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.663207+0000", 2023-10-09T11:02:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106488 2023-10-09T11:02:22.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:22.663235+0000", 2023-10-09T11:02:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8632e-05 2023-10-09T11:02:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T11:02:22.679 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 3 2023-10-09T11:02:22.679 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:02:22.688 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:22 smithi155 ceph-mon[184854]: pgmap v1066: 119 pgs: 119 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T11:02:22.725 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:22.792 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:22.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:22.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:22.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:22.905 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 64 16 79342 79326 77.4359 62.75 0.041446 0.0129082 2023-10-09T11:02:22.989 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:23.064 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:23.153 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:23.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:23.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:23.258 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:23.297 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:23.631 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:02:23.644 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:02:23.905 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 65 16 80237 80221 77.1052 55.9375 0.00614952 0.0129645 2023-10-09T11:02:24.050 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1066,"stamp":"2023-10-09T11:02:21.687266+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5005166166,"num_objects":76387,"num_object_clones":0,"num_object_copies":229161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76387,"num_whiteouts":0,"num_read":39,"num_read_kb":70,"num_write":76444,"num_write_kb":4887445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4604,"num_bytes_recovered":298761900,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":81409,"ondisk_log_size":81409,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":20300592,"kb_used_data":15180728,"kb_used_omap":130,"kb_used_meta":5119677,"kb_avail":635915472,"statfs":{"total":671965249536,"available":651177443328,"internally_reserved":0,"allocated":15545065472,"data_stored":15532012763,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":133799,"internal_metadata":5242549593},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":508690432,"num_objects":7762,"num_object_clones":0,"num_object_copies":23286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7762,"num_write_kb":496768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.517838"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":646,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.241998+0000","last_change":"2023-10-09T11:01:29.156745+0000","last_active":"2023-10-09T11:02:16.241998+0000","last_peered":"2023-10-09T11:02:16.241998+0000","last_clean":"2023-10-09T11:02:16.241998+0000","last_became_active":"2023-10-09T11:01:29.156243+0000","last_became_peered":"2023-10-09T11:01:29.156243+0000","last_unstale":"2023-10-09T11:02:16.241998+0000","last_undegraded":"2023-10-09T11:02:16.241998+0000","last_fullsized":"2023-10-09T11:02:16.241998+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:24.854753+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":154,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.229209+0000","last_change":"2023-10-09T11:00:14.910332+0000","last_active":"2023-10-09T11:02:16.229209+0000","last_peered":"2023-10-09T11:02:16.229209+0000","last_clean":"2023-10-09T11:02:16.229209+0000","last_became_active":"2023-10-09T11:00:14.910108+0000","last_became_peered":"2023-10-09T11:00:14.910108+0000","last_unstale":"2023-10-09T11:02:16.229209+0000","last_undegraded":"2023-10-09T11:02:16.229209+0000","last_fullsized":"2023-10-09T11:02:16.229209+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:30:41.668874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":456,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.236045+0000","last_change":"2023-10-09T11:00:14.870884+0000","last_active":"2023-10-09T11:02:16.236045+0000","last_peered":"2023-10-09T11:02:16.236045+0000","last_clean":"2023-10-09T11:02:16.236045+0000","last_became_active":"2023-10-09T11:00:14.870515+0000","last_became_peered":"2023-10-09T11:00:14.870515+0000","last_unstale":"2023-10-09T11:02:16.236045+0000","last_undegraded":"2023-10-09T11:02:16.236045+0000","last_fullsized":"2023-10-09T11:02:16.236045+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:02.499841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":578,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.243033+0000","last_change":"2023-10-09T11:02:02.395757+0000","last_active":"2023-10-09T11:02:16.243033+0000","last_peered":"2023-10-09T11:02:16.243033+0000","last_clean":"2023-10-09T11:02:16.243033+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:02:16.243033+0000","last_undegraded":"2023-10-09T11:02:16.243033+0000","last_fullsized":"2023-10-09T11:02:16.243033+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.536962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.536962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":108,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.203694+0000","last_change":"2023-10-09T11:00:14.876415+0000","last_active":"2023-10-09T11:02:14.203694+0000","last_peered":"2023-10-09T11:02:14.203694+0000","last_clean":"2023-10-09T11:02:14.203694+0000","last_became_active":"2023-10-09T11:00:14.864861+0000","last_became_peered":"2023-10-09T11:00:14.864861+0000","last_unstale":"2023-10-09T11:02:14.203694+0000","last_undegraded":"2023-10-09T11:02:14.203694+0000","last_fullsized":"2023-10-09T11:02:14.203694+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:59.494851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":363,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.189857+0000","last_change":"2023-10-09T11:01:26.641585+0000","last_active":"2023-10-09T11:02:14.189857+0000","last_peered":"2023-10-09T11:02:14.189857+0000","last_clean":"2023-10-09T11:02:14.189857+0000","last_became_active":"2023-10-09T11:01:26.641278+0000","last_became_peered":"2023-10-09T11:01:26.641278+0000","last_unstale":"2023-10-09T11:02:14.189857+0000","last_undegraded":"2023-10-09T11:02:14.189857+0000","last_fullsized":"2023-10-09T11:02:14.189857+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:44.889126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":891,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.251096+0000","last_change":"2023-10-09T11:01:29.158198+0000","last_active":"2023-10-09T11:02:16.251096+0000","last_peered":"2023-10-09T11:02:16.251096+0000","last_clean":"2023-10-09T11:02:16.251096+0000","last_became_active":"2023-10-09T11:01:29.157984+0000","last_became_peered":"2023-10-09T11:01:29.157984+0000","last_unstale":"2023-10-09T11:02:16.251096+0000","last_undegraded":"2023-10-09T11:02:16.251096+0000","last_fullsized":"2023-10-09T11:02:16.251096+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:39:23.112099+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":522,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232175+0000","last_change":"2023-10-09T11:00:15.889100+0000","last_active":"2023-10-09T11:02:16.232175+0000","last_peered":"2023-10-09T11:02:16.232175+0000","last_clean":"2023-10-09T11:02:16.232175+0000","last_became_active":"2023-10-09T11:00:15.888886+0000","last_became_peered":"2023-10-09T11:00:15.888886+0000","last_unstale":"2023-10-09T11:02:16.232175+0000","last_undegraded":"2023-10-09T11:02:16.232175+0000","last_fullsized":"2023-10-09T11:02:16.232175+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:40.695500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":220,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244624+0000","last_change":"2023-10-09T11:01:15.401167+0000","last_active":"2023-10-09T11:02:16.244624+0000","last_peered":"2023-10-09T11:02:16.244624+0000","last_clean":"2023-10-09T11:02:16.244624+0000","last_became_active":"2023-10-09T11:01:15.400876+0000","last_became_peered":"2023-10-09T11:01:15.400876+0000","last_unstale":"2023-10-09T11:02:16.244624+0000","last_undegraded":"2023-10-09T11:02:16.244624+0000","last_fullsized":"2023-10-09T11:02:16.244624+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:56.342226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":583,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242860+0000","last_change":"2023-10-09T11:02:03.383482+0000","last_active":"2023-10-09T11:02:16.242860+0000","last_peered":"2023-10-09T11:02:16.242860+0000","last_clean":"2023-10-09T11:02:16.242860+0000","last_became_active":"2023-10-09T11:01:29.161949+0000","last_became_peered":"2023-10-09T11:01:29.161949+0000","last_unstale":"2023-10-09T11:02:16.242860+0000","last_undegraded":"2023-10-09T11:02:16.242860+0000","last_fullsized":"2023-10-09T11:02:16.242860+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.787782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.787782+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":66,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.192662+0000","last_change":"2023-10-09T10:59:48.124873+0000","last_active":"2023-10-09T11:02:14.192662+0000","last_peered":"2023-10-09T11:02:14.192662+0000","last_clean":"2023-10-09T11:02:14.192662+0000","last_became_active":"2023-10-09T10:59:48.124600+0000","last_became_peered":"2023-10-09T10:59:48.124600+0000","last_unstale":"2023-10-09T11:02:14.192662+0000","last_undegraded":"2023-10-09T11:02:14.192662+0000","last_fullsized":"2023-10-09T11:02:14.192662+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:04.818987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":457,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232251+0000","last_change":"2023-10-09T11:00:14.869998+0000","last_active":"2023-10-09T11:02:16.232251+0000","last_peered":"2023-10-09T11:02:16.232251+0000","last_clean":"2023-10-09T11:02:16.232251+0000","last_became_active":"2023-10-09T11:00:14.869671+0000","last_became_peered":"2023-10-09T11:00:14.869671+0000","last_unstale":"2023-10-09T11:02:16.232251+0000","last_undegraded":"2023-10-09T11:02:16.232251+0000","last_fullsized":"2023-10-09T11:02:16.232251+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:19.564770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":741,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.228901+0000","last_change":"2023-10-09T11:01:29.157411+0000","last_active":"2023-10-09T11:02:16.228901+0000","last_peered":"2023-10-09T11:02:16.228901+0000","last_clean":"2023-10-09T11:02:16.228901+0000","last_became_active":"2023-10-09T11:01:29.156986+0000","last_became_peered":"2023-10-09T11:01:29.156986+0000","last_unstale":"2023-10-09T11:02:16.228901+0000","last_undegraded":"2023-10-09T11:02:16.228901+0000","last_fullsized":"2023-10-09T11:02:16.228901+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:13:32.638842+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":272,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244154+0000","last_change":"2023-10-09T11:01:26.637394+0000","last_active":"2023-10-09T11:02:16.244154+0000","last_peered":"2023-10-09T11:02:16.244154+0000","last_clean":"2023-10-09T11:02:16.244154+0000","last_became_active":"2023-10-09T11:01:26.637067+0000","last_became_peered":"2023-10-09T11:01:26.637067+0000","last_unstale":"2023-10-09T11:02:16.244154+0000","last_undegraded":"2023-10-09T11:02:16.244154+0000","last_fullsized":"2023-10-09T11:02:16.244154+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:37.815349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":408,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233805+0000","last_change":"2023-10-09T11:01:26.643175+0000","last_active":"2023-10-09T11:02:16.233805+0000","last_peered":"2023-10-09T11:02:16.233805+0000","last_clean":"2023-10-09T11:02:16.233805+0000","last_became_active":"2023-10-09T11:01:26.642855+0000","last_became_peered":"2023-10-09T11:01:26.642855+0000","last_unstale":"2023-10-09T11:02:16.233805+0000","last_undegraded":"2023-10-09T11:02:16.233805+0000","last_fullsized":"2023-10-09T11:02:16.233805+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:02.173276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":620,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244386+0000","last_change":"2023-10-09T11:01:16.673177+0000","last_active":"2023-10-09T11:02:16.244386+0000","last_peered":"2023-10-09T11:02:16.244386+0000","last_clean":"2023-10-09T11:02:16.244386+0000","last_became_active":"2023-10-09T11:01:15.401411+0000","last_became_peered":"2023-10-09T11:01:15.401411+0000","last_unstale":"2023-10-09T11:02:16.244386+0000","last_undegraded":"2023-10-09T11:02:16.244386+0000","last_fullsized":"2023-10-09T11:02:16.244386+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:46.989699+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":325,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.236077+0000","last_change":"2023-10-09T11:01:26.696000+0000","last_active":"2023-10-09T11:02:16.236077+0000","last_peered":"2023-10-09T11:02:16.236077+0000","last_clean":"2023-10-09T11:02:16.236077+0000","last_became_active":"2023-10-09T11:01:26.642917+0000","last_became_peered":"2023-10-09T11:01:26.642917+0000","last_unstale":"2023-10-09T11:02:16.236077+0000","last_undegraded":"2023-10-09T11:02:16.236077+0000","last_fullsized":"2023-10-09T11:02:16.236077+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:35.375183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":189,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242906+0000","last_change":"2023-10-09T11:01:15.401892+0000","last_active":"2023-10-09T11:02:16.242906+0000","last_peered":"2023-10-09T11:02:16.242906+0000","last_clean":"2023-10-09T11:02:16.242906+0000","last_became_active":"2023-10-09T11:01:15.401025+0000","last_became_peered":"2023-10-09T11:01:15.401025+0000","last_unstale":"2023-10-09T11:02:16.242906+0000","last_undegraded":"2023-10-09T11:02:16.242906+0000","last_fullsized":"2023-10-09T11:02:16.242906+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:48:03.531920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":779,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.204106+0000","last_change":"2023-10-09T11:02:09.381602+0000","last_active":"2023-10-09T11:02:14.204106+0000","last_peered":"2023-10-09T11:02:14.204106+0000","last_clean":"2023-10-09T11:02:14.204106+0000","last_became_active":"2023-10-09T11:00:16.068155+0000","last_became_peered":"2023-10-09T11:00:16.068155+0000","last_unstale":"2023-10-09T11:02:14.204106+0000","last_undegraded":"2023-10-09T11:02:14.204106+0000","last_fullsized":"2023-10-09T11:02:14.204106+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:56.972770+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":43,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.228732+0000","last_change":"2023-10-09T11:01:26.693517+0000","last_active":"2023-10-09T11:02:16.228732+0000","last_peered":"2023-10-09T11:02:16.228732+0000","last_clean":"2023-10-09T11:02:16.228732+0000","last_became_active":"2023-10-09T11:01:26.648334+0000","last_became_peered":"2023-10-09T11:01:26.648334+0000","last_unstale":"2023-10-09T11:02:16.228732+0000","last_undegraded":"2023-10-09T11:02:16.228732+0000","last_fullsized":"2023-10-09T11:02:16.228732+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:18.451728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":445,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.248377+0000","last_change":"2023-10-09T10:57:46.845755+0000","last_active":"2023-10-09T11:02:16.248377+0000","last_peered":"2023-10-09T11:02:16.248377+0000","last_clean":"2023-10-09T11:02:16.248377+0000","last_became_active":"2023-10-09T10:57:46.845475+0000","last_became_peered":"2023-10-09T10:57:46.845475+0000","last_unstale":"2023-10-09T11:02:16.248377+0000","last_undegraded":"2023-10-09T11:02:16.248377+0000","last_fullsized":"2023-10-09T11:02:16.248377+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:21.673994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":792,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.189708+0000","last_change":"2023-10-09T10:59:49.175255+0000","last_active":"2023-10-09T11:02:14.189708+0000","last_peered":"2023-10-09T11:02:14.189708+0000","last_clean":"2023-10-09T11:02:14.189708+0000","last_became_active":"2023-10-09T10:59:49.174666+0000","last_became_peered":"2023-10-09T10:59:49.174666+0000","last_unstale":"2023-10-09T11:02:14.189708+0000","last_undegraded":"2023-10-09T11:02:14.189708+0000","last_fullsized":"2023-10-09T11:02:14.189708+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:00:58.069275+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":100,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.229148+0000","last_change":"2023-10-09T11:01:26.673987+0000","last_active":"2023-10-09T11:02:16.229148+0000","last_peered":"2023-10-09T11:02:16.229148+0000","last_clean":"2023-10-09T11:02:16.229148+0000","last_became_active":"2023-10-09T11:01:26.644907+0000","last_became_peered":"2023-10-09T11:01:26.644907+0000","last_unstale":"2023-10-09T11:02:16.229148+0000","last_undegraded":"2023-10-09T11:02:16.229148+0000","last_fullsized":"2023-10-09T11:02:16.229148+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:13.828945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":534,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232977+0000","last_change":"2023-10-09T11:01:26.635356+0000","last_active":"2023-10-09T11:02:16.232977+0000","last_peered":"2023-10-09T11:02:16.232977+0000","last_clean":"2023-10-09T11:02:16.232977+0000","last_became_active":"2023-10-09T11:01:26.634709+0000","last_became_peered":"2023-10-09T11:01:26.634709+0000","last_unstale":"2023-10-09T11:02:16.232977+0000","last_undegraded":"2023-10-09T11:02:16.232977+0000","last_fullsized":"2023-10-09T11:02:16.232977+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:38.908158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":682,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242844+0000","last_change":"2023-10-09T11:01:29.161463+0000","last_active":"2023-10-09T11:02:16.242844+0000","last_peered":"2023-10-09T11:02:16.242844+0000","last_clean":"2023-10-09T11:02:16.242844+0000","last_became_active":"2023-10-09T11:01:29.161068+0000","last_became_peered":"2023-10-09T11:01:29.161068+0000","last_unstale":"2023-10-09T11:02:16.242844+0000","last_undegraded":"2023-10-09T11:02:16.242844+0000","last_fullsized":"2023-10-09T11:02:16.242844+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:55.586383+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":361,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.191325+0000","last_change":"2023-10-09T11:01:26.644626+0000","last_active":"2023-10-09T11:02:14.191325+0000","last_peered":"2023-10-09T11:02:14.191325+0000","last_clean":"2023-10-09T11:02:14.191325+0000","last_became_active":"2023-10-09T11:01:26.644324+0000","last_became_peered":"2023-10-09T11:01:26.644324+0000","last_unstale":"2023-10-09T11:02:14.191325+0000","last_undegraded":"2023-10-09T11:02:14.191325+0000","last_fullsized":"2023-10-09T11:02:14.191325+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:51:47.358468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":305,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242721+0000","last_change":"2023-10-09T11:01:26.653786+0000","last_active":"2023-10-09T11:02:16.242721+0000","last_peered":"2023-10-09T11:02:16.242721+0000","last_clean":"2023-10-09T11:02:16.242721+0000","last_became_active":"2023-10-09T11:01:26.652916+0000","last_became_peered":"2023-10-09T11:01:26.652916+0000","last_unstale":"2023-10-09T11:02:16.242721+0000","last_undegraded":"2023-10-09T11:02:16.242721+0000","last_fullsized":"2023-10-09T11:02:16.242721+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:09.061877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":721,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.204103+0000","last_change":"2023-10-09T11:01:25.620261+0000","last_active":"2023-10-09T11:02:14.204103+0000","last_peered":"2023-10-09T11:02:14.204103+0000","last_clean":"2023-10-09T11:02:14.204103+0000","last_became_active":"2023-10-09T11:00:14.871258+0000","last_became_peered":"2023-10-09T11:00:14.871258+0000","last_unstale":"2023-10-09T11:02:14.204103+0000","last_undegraded":"2023-10-09T11:02:14.204103+0000","last_fullsized":"2023-10-09T11:02:14.204103+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:13.861006+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":291,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.250931+0000","last_change":"2023-10-09T11:01:26.650498+0000","last_active":"2023-10-09T11:02:16.250931+0000","last_peered":"2023-10-09T11:02:16.250931+0000","last_clean":"2023-10-09T11:02:16.250931+0000","last_became_active":"2023-10-09T11:01:26.650153+0000","last_became_peered":"2023-10-09T11:01:26.650153+0000","last_unstale":"2023-10-09T11:02:16.250931+0000","last_undegraded":"2023-10-09T11:02:16.250931+0000","last_fullsized":"2023-10-09T11:02:16.250931+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:06.419869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":75,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.189571+0000","last_change":"2023-10-09T11:01:26.640215+0000","last_active":"2023-10-09T11:02:14.189571+0000","last_peered":"2023-10-09T11:02:14.189571+0000","last_clean":"2023-10-09T11:02:14.189571+0000","last_became_active":"2023-10-09T11:01:26.639896+0000","last_became_peered":"2023-10-09T11:01:26.639896+0000","last_unstale":"2023-10-09T11:02:14.189571+0000","last_undegraded":"2023-10-09T11:02:14.189571+0000","last_fullsized":"2023-10-09T11:02:14.189571+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:37.209505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":699,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244748+0000","last_change":"2023-10-09T11:01:15.406928+0000","last_active":"2023-10-09T11:02:16.244748+0000","last_peered":"2023-10-09T11:02:16.244748+0000","last_clean":"2023-10-09T11:02:16.244748+0000","last_became_active":"2023-10-09T11:01:15.406499+0000","last_became_peered":"2023-10-09T11:01:15.406499+0000","last_unstale":"2023-10-09T11:02:16.244748+0000","last_undegraded":"2023-10-09T11:02:16.244748+0000","last_fullsized":"2023-10-09T11:02:16.244748+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:33.050336+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":158,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.239020+0000","last_change":"2023-10-09T11:00:48.109643+0000","last_active":"2023-10-09T11:02:16.239020+0000","last_peered":"2023-10-09T11:02:16.239020+0000","last_clean":"2023-10-09T11:02:16.239020+0000","last_became_active":"2023-10-09T11:00:48.109299+0000","last_became_peered":"2023-10-09T11:00:48.109299+0000","last_unstale":"2023-10-09T11:02:16.239020+0000","last_undegraded":"2023-10-09T11:02:16.239020+0000","last_fullsized":"2023-10-09T11:02:16.239020+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:31.163237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":510,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233997+0000","last_change":"2023-10-09T11:01:26.637554+0000","last_active":"2023-10-09T11:02:16.233997+0000","last_peered":"2023-10-09T11:02:16.233997+0000","last_clean":"2023-10-09T11:02:16.233997+0000","last_became_active":"2023-10-09T11:01:26.637117+0000","last_became_peered":"2023-10-09T11:01:26.637117+0000","last_unstale":"2023-10-09T11:02:16.233997+0000","last_undegraded":"2023-10-09T11:02:16.233997+0000","last_fullsized":"2023-10-09T11:02:16.233997+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:02.104893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"431'5010","reported_seq":5046,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.576569+0000","last_change":"2023-10-09T11:01:15.402898+0000","last_active":"2023-10-09T11:02:21.576569+0000","last_peered":"2023-10-09T11:02:21.576569+0000","last_clean":"2023-10-09T11:02:21.576569+0000","last_became_active":"2023-10-09T11:01:15.402646+0000","last_became_peered":"2023-10-09T11:01:15.402646+0000","last_unstale":"2023-10-09T11:02:21.576569+0000","last_undegraded":"2023-10-09T11:02:21.576569+0000","last_fullsized":"2023-10-09T11:02:21.576569+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5010,"log_dups_size":0,"ondisk_log_size":5010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:42:07.397321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328335360,"num_objects":5010,"num_object_clones":0,"num_object_copies":15030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5010,"num_write_kb":320640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":709,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.206165+0000","last_change":"2023-10-09T11:01:29.163248+0000","last_active":"2023-10-09T11:02:14.206165+0000","last_peered":"2023-10-09T11:02:14.206165+0000","last_clean":"2023-10-09T11:02:14.206165+0000","last_became_active":"2023-10-09T11:01:29.162829+0000","last_became_peered":"2023-10-09T11:01:29.162829+0000","last_unstale":"2023-10-09T11:02:14.206165+0000","last_undegraded":"2023-10-09T11:02:14.206165+0000","last_fullsized":"2023-10-09T11:02:14.206165+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:43:25.947651+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":183,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.248297+0000","last_change":"2023-10-09T11:01:26.684538+0000","last_active":"2023-10-09T11:02:16.248297+0000","last_peered":"2023-10-09T11:02:16.248297+0000","last_clean":"2023-10-09T11:02:16.248297+0000","last_became_active":"2023-10-09T11:01:26.649495+0000","last_became_peered":"2023-10-09T11:01:26.649495+0000","last_unstale":"2023-10-09T11:02:16.248297+0000","last_undegraded":"2023-10-09T11:02:16.248297+0000","last_fullsized":"2023-10-09T11:02:16.248297+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:25.524018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":343,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242080+0000","last_change":"2023-10-09T11:01:26.653478+0000","last_active":"2023-10-09T11:02:16.242080+0000","last_peered":"2023-10-09T11:02:16.242080+0000","last_clean":"2023-10-09T11:02:16.242080+0000","last_became_active":"2023-10-09T11:01:26.653181+0000","last_became_peered":"2023-10-09T11:01:26.653181+0000","last_unstale":"2023-10-09T11:02:16.242080+0000","last_undegraded":"2023-10-09T11:02:16.242080+0000","last_fullsized":"2023-10-09T11:02:16.242080+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:28.010266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"431'4787","reported_seq":4824,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:20.833231+0000","last_change":"2023-10-09T11:01:15.394927+0000","last_active":"2023-10-09T11:02:20.833231+0000","last_peered":"2023-10-09T11:02:20.833231+0000","last_clean":"2023-10-09T11:02:20.833231+0000","last_became_active":"2023-10-09T11:01:15.394658+0000","last_became_peered":"2023-10-09T11:01:15.394658+0000","last_unstale":"2023-10-09T11:02:20.833231+0000","last_undegraded":"2023-10-09T11:02:20.833231+0000","last_fullsized":"2023-10-09T11:02:20.833231+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4787,"log_dups_size":0,"ondisk_log_size":4787,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:58.565353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313720832,"num_objects":4787,"num_object_clones":0,"num_object_copies":14361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4787,"num_write_kb":306368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":835,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232838+0000","last_change":"2023-10-09T10:59:51.140014+0000","last_active":"2023-10-09T11:02:16.232838+0000","last_peered":"2023-10-09T11:02:16.232838+0000","last_clean":"2023-10-09T11:02:16.232838+0000","last_became_active":"2023-10-09T10:59:51.139665+0000","last_became_peered":"2023-10-09T10:59:51.139665+0000","last_unstale":"2023-10-09T11:02:16.232838+0000","last_undegraded":"2023-10-09T11:02:16.232838+0000","last_fullsized":"2023-10-09T11:02:16.232838+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:29.505223+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":116,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.203886+0000","last_change":"2023-10-09T11:01:26.680401+0000","last_active":"2023-10-09T11:02:14.203886+0000","last_peered":"2023-10-09T11:02:14.203886+0000","last_clean":"2023-10-09T11:02:14.203886+0000","last_became_active":"2023-10-09T11:01:26.647295+0000","last_became_peered":"2023-10-09T11:01:26.647295+0000","last_unstale":"2023-10-09T11:02:14.203886+0000","last_undegraded":"2023-10-09T11:02:14.203886+0000","last_fullsized":"2023-10-09T11:02:14.203886+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:28.731546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":108,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.203597+0000","last_change":"2023-10-09T11:00:14.872673+0000","last_active":"2023-10-09T11:02:14.203597+0000","last_peered":"2023-10-09T11:02:14.203597+0000","last_clean":"2023-10-09T11:02:14.203597+0000","last_became_active":"2023-10-09T11:00:14.872345+0000","last_became_peered":"2023-10-09T11:00:14.872345+0000","last_unstale":"2023-10-09T11:02:14.203597+0000","last_undegraded":"2023-10-09T11:02:14.203597+0000","last_fullsized":"2023-10-09T11:02:14.203597+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:32:43.821085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"430'4620","reported_seq":4656,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:17.964456+0000","last_change":"2023-10-09T11:01:15.402608+0000","last_active":"2023-10-09T11:02:17.964456+0000","last_peered":"2023-10-09T11:02:17.964456+0000","last_clean":"2023-10-09T11:02:17.964456+0000","last_became_active":"2023-10-09T11:01:15.402329+0000","last_became_peered":"2023-10-09T11:01:15.402329+0000","last_unstale":"2023-10-09T11:02:17.964456+0000","last_undegraded":"2023-10-09T11:02:17.964456+0000","last_fullsized":"2023-10-09T11:02:17.964456+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4620,"log_dups_size":0,"ondisk_log_size":4620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:31:16.567673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302776320,"num_objects":4620,"num_object_clones":0,"num_object_copies":13860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4620,"num_write_kb":295680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":627,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244473+0000","last_change":"2023-10-09T11:01:22.599166+0000","last_active":"2023-10-09T11:02:16.244473+0000","last_peered":"2023-10-09T11:02:16.244473+0000","last_clean":"2023-10-09T11:02:16.244473+0000","last_became_active":"2023-10-09T11:01:15.402193+0000","last_became_peered":"2023-10-09T11:01:15.402193+0000","last_unstale":"2023-10-09T11:02:16.244473+0000","last_undegraded":"2023-10-09T11:02:16.244473+0000","last_fullsized":"2023-10-09T11:02:16.244473+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:37:16.611219+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":211,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242810+0000","last_change":"2023-10-09T11:01:26.684793+0000","last_active":"2023-10-09T11:02:16.242810+0000","last_peered":"2023-10-09T11:02:16.242810+0000","last_clean":"2023-10-09T11:02:16.242810+0000","last_became_active":"2023-10-09T11:01:26.653430+0000","last_became_peered":"2023-10-09T11:01:26.653430+0000","last_unstale":"2023-10-09T11:02:16.242810+0000","last_undegraded":"2023-10-09T11:02:16.242810+0000","last_fullsized":"2023-10-09T11:02:16.242810+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:46:56.385204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":270,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.239311+0000","last_change":"2023-10-09T11:01:26.658802+0000","last_active":"2023-10-09T11:02:16.239311+0000","last_peered":"2023-10-09T11:02:16.239311+0000","last_clean":"2023-10-09T11:02:16.239311+0000","last_became_active":"2023-10-09T11:01:26.658477+0000","last_became_peered":"2023-10-09T11:01:26.658477+0000","last_unstale":"2023-10-09T11:02:16.239311+0000","last_undegraded":"2023-10-09T11:02:16.239311+0000","last_fullsized":"2023-10-09T11:02:16.239311+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:52.018229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"431'4895","reported_seq":4932,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.040434+0000","last_change":"2023-10-09T11:01:15.393071+0000","last_active":"2023-10-09T11:02:21.040434+0000","last_peered":"2023-10-09T11:02:21.040434+0000","last_clean":"2023-10-09T11:02:21.040434+0000","last_became_active":"2023-10-09T11:01:15.392793+0000","last_became_peered":"2023-10-09T11:01:15.392793+0000","last_unstale":"2023-10-09T11:02:21.040434+0000","last_undegraded":"2023-10-09T11:02:21.040434+0000","last_fullsized":"2023-10-09T11:02:21.040434+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4895,"log_dups_size":0,"ondisk_log_size":4895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:27.128762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320798720,"num_objects":4895,"num_object_clones":0,"num_object_copies":14685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4895,"num_write_kb":313280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":936,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.251020+0000","last_change":"2023-10-09T11:01:29.173445+0000","last_active":"2023-10-09T11:02:16.251020+0000","last_peered":"2023-10-09T11:02:16.251020+0000","last_clean":"2023-10-09T11:02:16.251020+0000","last_became_active":"2023-10-09T11:01:29.173101+0000","last_became_peered":"2023-10-09T11:01:29.173101+0000","last_unstale":"2023-10-09T11:02:16.251020+0000","last_undegraded":"2023-10-09T11:02:16.251020+0000","last_fullsized":"2023-10-09T11:02:16.251020+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:50.742268+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":242,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242276+0000","last_change":"2023-10-09T11:01:15.400732+0000","last_active":"2023-10-09T11:02:16.242276+0000","last_peered":"2023-10-09T11:02:16.242276+0000","last_clean":"2023-10-09T11:02:16.242276+0000","last_became_active":"2023-10-09T11:01:15.399370+0000","last_became_peered":"2023-10-09T11:01:15.399370+0000","last_unstale":"2023-10-09T11:02:16.242276+0000","last_undegraded":"2023-10-09T11:02:16.242276+0000","last_fullsized":"2023-10-09T11:02:16.242276+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:53.948144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":351,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.189111+0000","last_change":"2023-10-09T11:00:15.865848+0000","last_active":"2023-10-09T11:02:14.189111+0000","last_peered":"2023-10-09T11:02:14.189111+0000","last_clean":"2023-10-09T11:02:14.189111+0000","last_became_active":"2023-10-09T11:00:15.865504+0000","last_became_peered":"2023-10-09T11:00:15.865504+0000","last_unstale":"2023-10-09T11:02:14.189111+0000","last_undegraded":"2023-10-09T11:02:14.189111+0000","last_fullsized":"2023-10-09T11:02:14.189111+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:32.352564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"430'4662","reported_seq":4696,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:18.019241+0000","last_change":"2023-10-09T11:01:15.401760+0000","last_active":"2023-10-09T11:02:18.019241+0000","last_peered":"2023-10-09T11:02:18.019241+0000","last_clean":"2023-10-09T11:02:18.019241+0000","last_became_active":"2023-10-09T11:01:15.401510+0000","last_became_peered":"2023-10-09T11:01:15.401510+0000","last_unstale":"2023-10-09T11:02:18.019241+0000","last_undegraded":"2023-10-09T11:02:18.019241+0000","last_fullsized":"2023-10-09T11:02:18.019241+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4662,"log_dups_size":0,"ondisk_log_size":4662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:21.262441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305528832,"num_objects":4662,"num_object_clones":0,"num_object_copies":13986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4662,"num_write_kb":298368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":783,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233526+0000","last_change":"2023-10-09T11:01:29.165873+0000","last_active":"2023-10-09T11:02:16.233526+0000","last_peered":"2023-10-09T11:02:16.233526+0000","last_clean":"2023-10-09T11:02:16.233526+0000","last_became_active":"2023-10-09T11:01:29.165597+0000","last_became_peered":"2023-10-09T11:01:29.165597+0000","last_unstale":"2023-10-09T11:02:16.233526+0000","last_undegraded":"2023-10-09T11:02:16.233526+0000","last_fullsized":"2023-10-09T11:02:16.233526+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:54.234048+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":450,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232382+0000","last_change":"2023-10-09T10:59:49.127924+0000","last_active":"2023-10-09T11:02:16.232382+0000","last_peered":"2023-10-09T11:02:16.232382+0000","last_clean":"2023-10-09T11:02:16.232382+0000","last_became_active":"2023-10-09T10:59:49.127549+0000","last_became_peered":"2023-10-09T10:59:49.127549+0000","last_unstale":"2023-10-09T11:02:16.232382+0000","last_undegraded":"2023-10-09T11:02:16.232382+0000","last_fullsized":"2023-10-09T11:02:16.232382+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:01.781131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":275,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233129+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T11:02:16.233129+0000","last_peered":"2023-10-09T11:02:16.233129+0000","last_clean":"2023-10-09T11:02:16.233129+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T11:02:16.233129+0000","last_undegraded":"2023-10-09T11:02:16.233129+0000","last_fullsized":"2023-10-09T11:02:16.233129+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"431'4805","reported_seq":4842,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.068353+0000","last_change":"2023-10-09T11:01:15.394098+0000","last_active":"2023-10-09T11:02:21.068353+0000","last_peered":"2023-10-09T11:02:21.068353+0000","last_clean":"2023-10-09T11:02:21.068353+0000","last_became_active":"2023-10-09T11:01:15.393709+0000","last_became_peered":"2023-10-09T11:01:15.393709+0000","last_unstale":"2023-10-09T11:02:21.068353+0000","last_undegraded":"2023-10-09T11:02:21.068353+0000","last_fullsized":"2023-10-09T11:02:21.068353+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4805,"log_dups_size":0,"ondisk_log_size":4805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:08.421072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314900480,"num_objects":4805,"num_object_clones":0,"num_object_copies":14415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4805,"num_write_kb":307520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":622,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242743+0000","last_change":"2023-10-09T11:01:20.667499+0000","last_active":"2023-10-09T11:02:16.242743+0000","last_peered":"2023-10-09T11:02:16.242743+0000","last_clean":"2023-10-09T11:02:16.242743+0000","last_became_active":"2023-10-09T11:01:15.399808+0000","last_became_peered":"2023-10-09T11:01:15.399808+0000","last_unstale":"2023-10-09T11:02:16.242743+0000","last_undegraded":"2023-10-09T11:02:16.242743+0000","last_fullsized":"2023-10-09T11:02:16.242743+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:53:29.298692+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":427,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.251241+0000","last_change":"2023-10-09T11:00:14.891176+0000","last_active":"2023-10-09T11:02:16.251241+0000","last_peered":"2023-10-09T11:02:16.251241+0000","last_clean":"2023-10-09T11:02:16.251241+0000","last_became_active":"2023-10-09T11:00:14.890890+0000","last_became_peered":"2023-10-09T11:00:14.890890+0000","last_unstale":"2023-10-09T11:02:16.251241+0000","last_undegraded":"2023-10-09T11:02:16.251241+0000","last_fullsized":"2023-10-09T11:02:16.251241+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:38.158759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":187,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232388+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:02:16.232388+0000","last_peered":"2023-10-09T11:02:16.232388+0000","last_clean":"2023-10-09T11:02:16.232388+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:02:16.232388+0000","last_undegraded":"2023-10-09T11:02:16.232388+0000","last_fullsized":"2023-10-09T11:02:16.232388+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"431'4798","reported_seq":4834,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.456754+0000","last_change":"2023-10-09T11:01:15.406742+0000","last_active":"2023-10-09T11:02:21.456754+0000","last_peered":"2023-10-09T11:02:21.456754+0000","last_clean":"2023-10-09T11:02:21.456754+0000","last_became_active":"2023-10-09T11:01:15.406429+0000","last_became_peered":"2023-10-09T11:01:15.406429+0000","last_unstale":"2023-10-09T11:02:21.456754+0000","last_undegraded":"2023-10-09T11:02:21.456754+0000","last_fullsized":"2023-10-09T11:02:21.456754+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4798,"log_dups_size":0,"ondisk_log_size":4798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:44:05.691874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314441728,"num_objects":4798,"num_object_clones":0,"num_object_copies":14394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4798,"num_write_kb":307072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":778,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.204182+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:02:14.204182+0000","last_peered":"2023-10-09T11:02:14.204182+0000","last_clean":"2023-10-09T11:02:14.204182+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:02:14.204182+0000","last_undegraded":"2023-10-09T11:02:14.204182+0000","last_fullsized":"2023-10-09T11:02:14.204182+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":496,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.251522+0000","last_change":"2023-10-09T11:01:26.641439+0000","last_active":"2023-10-09T11:02:16.251522+0000","last_peered":"2023-10-09T11:02:16.251522+0000","last_clean":"2023-10-09T11:02:16.251522+0000","last_became_active":"2023-10-09T11:01:26.641109+0000","last_became_peered":"2023-10-09T11:01:26.641109+0000","last_unstale":"2023-10-09T11:02:16.251522+0000","last_undegraded":"2023-10-09T11:02:16.251522+0000","last_fullsized":"2023-10-09T11:02:16.251522+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:59:59.718601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":108,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.203921+0000","last_change":"2023-10-09T11:00:14.900130+0000","last_active":"2023-10-09T11:02:14.203921+0000","last_peered":"2023-10-09T11:02:14.203921+0000","last_clean":"2023-10-09T11:02:14.203921+0000","last_became_active":"2023-10-09T11:00:14.899536+0000","last_became_peered":"2023-10-09T11:00:14.899536+0000","last_unstale":"2023-10-09T11:02:14.203921+0000","last_undegraded":"2023-10-09T11:02:14.203921+0000","last_fullsized":"2023-10-09T11:02:14.203921+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:07.151510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"430'4714","reported_seq":7035,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:17.951525+0000","last_change":"2023-10-09T11:02:08.575348+0000","last_active":"2023-10-09T11:02:17.951525+0000","last_peered":"2023-10-09T11:02:17.951525+0000","last_clean":"2023-10-09T11:02:17.951525+0000","last_became_active":"2023-10-09T11:02:08.572298+0000","last_became_peered":"2023-10-09T11:02:08.572298+0000","last_unstale":"2023-10-09T11:02:17.951525+0000","last_undegraded":"2023-10-09T11:02:17.951525+0000","last_fullsized":"2023-10-09T11:02:17.951525+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4714,"log_dups_size":0,"ondisk_log_size":4714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:02:03.819617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308936704,"num_objects":4714,"num_object_clones":0,"num_object_copies":14142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4714,"num_write_kb":301696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1135,"num_bytes_recovered":74383360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":633,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244616+0000","last_change":"2023-10-09T11:01:51.463338+0000","last_active":"2023-10-09T11:02:16.244616+0000","last_peered":"2023-10-09T11:02:16.244616+0000","last_clean":"2023-10-09T11:02:16.244616+0000","last_became_active":"2023-10-09T11:01:15.399431+0000","last_became_peered":"2023-10-09T11:01:15.399431+0000","last_unstale":"2023-10-09T11:02:16.244616+0000","last_undegraded":"2023-10-09T11:02:16.244616+0000","last_fullsized":"2023-10-09T11:02:16.244616+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:01:07.923413+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":503,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232075+0000","last_change":"2023-10-09T11:01:26.635487+0000","last_active":"2023-10-09T11:02:16.232075+0000","last_peered":"2023-10-09T11:02:16.232075+0000","last_clean":"2023-10-09T11:02:16.232075+0000","last_became_active":"2023-10-09T11:01:26.635092+0000","last_became_peered":"2023-10-09T11:01:26.635092+0000","last_unstale":"2023-10-09T11:02:16.232075+0000","last_undegraded":"2023-10-09T11:02:16.232075+0000","last_fullsized":"2023-10-09T11:02:16.232075+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:17.902346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":108,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.206375+0000","last_change":"2023-10-09T11:00:14.917869+0000","last_active":"2023-10-09T11:02:14.206375+0000","last_peered":"2023-10-09T11:02:14.206375+0000","last_clean":"2023-10-09T11:02:14.206375+0000","last_became_active":"2023-10-09T11:00:14.917535+0000","last_became_peered":"2023-10-09T11:00:14.917535+0000","last_unstale":"2023-10-09T11:02:14.206375+0000","last_undegraded":"2023-10-09T11:02:14.206375+0000","last_fullsized":"2023-10-09T11:02:14.206375+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:10.223866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"430'4666","reported_seq":6985,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:17.979035+0000","last_change":"2023-10-09T11:01:55.237850+0000","last_active":"2023-10-09T11:02:17.979035+0000","last_peered":"2023-10-09T11:02:17.979035+0000","last_clean":"2023-10-09T11:02:17.979035+0000","last_became_active":"2023-10-09T11:01:55.234587+0000","last_became_peered":"2023-10-09T11:01:55.234587+0000","last_unstale":"2023-10-09T11:02:17.979035+0000","last_undegraded":"2023-10-09T11:02:17.979035+0000","last_fullsized":"2023-10-09T11:02:17.979035+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4666,"log_dups_size":0,"ondisk_log_size":4666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:25.848462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305790976,"num_objects":4666,"num_object_clones":0,"num_object_copies":13998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4666,"num_write_kb":298624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1135,"num_bytes_recovered":74383360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":882,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233709+0000","last_change":"2023-10-09T11:01:29.171877+0000","last_active":"2023-10-09T11:02:16.233709+0000","last_peered":"2023-10-09T11:02:16.233709+0000","last_clean":"2023-10-09T11:02:16.233709+0000","last_became_active":"2023-10-09T11:01:29.171560+0000","last_became_peered":"2023-10-09T11:01:29.171560+0000","last_unstale":"2023-10-09T11:02:16.233709+0000","last_undegraded":"2023-10-09T11:02:16.233709+0000","last_fullsized":"2023-10-09T11:02:16.233709+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:00:08.554345+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":130,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.259395+0000","last_change":"2023-10-09T11:01:26.641036+0000","last_active":"2023-10-09T11:02:16.259395+0000","last_peered":"2023-10-09T11:02:16.259395+0000","last_clean":"2023-10-09T11:02:16.259395+0000","last_became_active":"2023-10-09T11:01:26.640698+0000","last_became_peered":"2023-10-09T11:01:26.640698+0000","last_unstale":"2023-10-09T11:02:16.259395+0000","last_undegraded":"2023-10-09T11:02:16.259395+0000","last_fullsized":"2023-10-09T11:02:16.259395+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:53.546628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":269,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242593+0000","last_change":"2023-10-09T11:01:15.399780+0000","last_active":"2023-10-09T11:02:16.242593+0000","last_peered":"2023-10-09T11:02:16.242593+0000","last_clean":"2023-10-09T11:02:16.242593+0000","last_became_active":"2023-10-09T11:01:15.399493+0000","last_became_peered":"2023-10-09T11:01:15.399493+0000","last_unstale":"2023-10-09T11:02:16.242593+0000","last_undegraded":"2023-10-09T11:02:16.242593+0000","last_fullsized":"2023-10-09T11:02:16.242593+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:41.639383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":109,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.259995+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:02:16.259995+0000","last_peered":"2023-10-09T11:02:16.259995+0000","last_clean":"2023-10-09T11:02:16.259995+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:02:16.259995+0000","last_undegraded":"2023-10-09T11:02:16.259995+0000","last_fullsized":"2023-10-09T11:02:16.259995+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"431'4841","reported_seq":4877,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.446358+0000","last_change":"2023-10-09T11:01:15.406192+0000","last_active":"2023-10-09T11:02:21.446358+0000","last_peered":"2023-10-09T11:02:21.446358+0000","last_clean":"2023-10-09T11:02:21.446358+0000","last_became_active":"2023-10-09T11:01:15.405811+0000","last_became_peered":"2023-10-09T11:01:15.405811+0000","last_unstale":"2023-10-09T11:02:21.446358+0000","last_undegraded":"2023-10-09T11:02:21.446358+0000","last_fullsized":"2023-10-09T11:02:21.446358+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4841,"log_dups_size":0,"ondisk_log_size":4841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:48:38.805496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317259776,"num_objects":4841,"num_object_clones":0,"num_object_copies":14523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4841,"num_write_kb":309824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":631,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244257+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:02:16.244257+0000","last_peered":"2023-10-09T11:02:16.244257+0000","last_clean":"2023-10-09T11:02:16.244257+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:02:16.244257+0000","last_undegraded":"2023-10-09T11:02:16.244257+0000","last_fullsized":"2023-10-09T11:02:16.244257+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":293,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244400+0000","last_change":"2023-10-09T11:01:15.401697+0000","last_active":"2023-10-09T11:02:16.244400+0000","last_peered":"2023-10-09T11:02:16.244400+0000","last_clean":"2023-10-09T11:02:16.244400+0000","last_became_active":"2023-10-09T11:01:15.401386+0000","last_became_peered":"2023-10-09T11:01:15.401386+0000","last_unstale":"2023-10-09T11:02:16.244400+0000","last_undegraded":"2023-10-09T11:02:16.244400+0000","last_fullsized":"2023-10-09T11:02:16.244400+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:40.572070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":336,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.188812+0000","last_change":"2023-10-09T10:59:49.126425+0000","last_active":"2023-10-09T11:02:14.188812+0000","last_peered":"2023-10-09T11:02:14.188812+0000","last_clean":"2023-10-09T11:02:14.188812+0000","last_became_active":"2023-10-09T10:59:49.125712+0000","last_became_peered":"2023-10-09T10:59:49.125712+0000","last_unstale":"2023-10-09T11:02:14.188812+0000","last_undegraded":"2023-10-09T11:02:14.188812+0000","last_fullsized":"2023-10-09T11:02:14.188812+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:40:53.898922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":109,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.259260+0000","last_change":"2023-10-09T11:00:14.878579+0000","last_active":"2023-10-09T11:02:16.259260+0000","last_peered":"2023-10-09T11:02:16.259260+0000","last_clean":"2023-10-09T11:02:16.259260+0000","last_became_active":"2023-10-09T11:00:14.867930+0000","last_became_peered":"2023-10-09T11:00:14.867930+0000","last_unstale":"2023-10-09T11:02:16.259260+0000","last_undegraded":"2023-10-09T11:02:16.259260+0000","last_fullsized":"2023-10-09T11:02:16.259260+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:34.925327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"431'4803","reported_seq":7148,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.599616+0000","last_change":"2023-10-09T11:01:34.181226+0000","last_active":"2023-10-09T11:02:21.599616+0000","last_peered":"2023-10-09T11:02:21.599616+0000","last_clean":"2023-10-09T11:02:21.599616+0000","last_became_active":"2023-10-09T11:01:34.166162+0000","last_became_peered":"2023-10-09T11:01:34.166162+0000","last_unstale":"2023-10-09T11:02:21.599616+0000","last_undegraded":"2023-10-09T11:02:21.599616+0000","last_fullsized":"2023-10-09T11:02:21.599616+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4803,"log_dups_size":0,"ondisk_log_size":4803,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:18.420715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314769408,"num_objects":4803,"num_object_clones":0,"num_object_copies":14409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4803,"num_write_kb":307392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1144,"num_bytes_recovered":74973184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":855,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.191100+0000","last_change":"2023-10-09T11:01:29.161986+0000","last_active":"2023-10-09T11:02:14.191100+0000","last_peered":"2023-10-09T11:02:14.191100+0000","last_clean":"2023-10-09T11:02:14.191100+0000","last_became_active":"2023-10-09T11:01:29.161669+0000","last_became_peered":"2023-10-09T11:01:29.161669+0000","last_unstale":"2023-10-09T11:02:14.191100+0000","last_undegraded":"2023-10-09T11:02:14.191100+0000","last_fullsized":"2023-10-09T11:02:14.191100+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:35.491541+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":182,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.228736+0000","last_change":"2023-10-09T10:59:48.122743+0000","last_active":"2023-10-09T11:02:16.228736+0000","last_peered":"2023-10-09T11:02:16.228736+0000","last_clean":"2023-10-09T11:02:16.228736+0000","last_became_active":"2023-10-09T10:59:48.122447+0000","last_became_peered":"2023-10-09T10:59:48.122447+0000","last_unstale":"2023-10-09T11:02:16.228736+0000","last_undegraded":"2023-10-09T11:02:16.228736+0000","last_fullsized":"2023-10-09T11:02:16.228736+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:28.368331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":109,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.203761+0000","last_change":"2023-10-09T11:00:14.897166+0000","last_active":"2023-10-09T11:02:14.203761+0000","last_peered":"2023-10-09T11:02:14.203761+0000","last_clean":"2023-10-09T11:02:14.203761+0000","last_became_active":"2023-10-09T11:00:14.896917+0000","last_became_peered":"2023-10-09T11:00:14.896917+0000","last_unstale":"2023-10-09T11:02:14.203761+0000","last_undegraded":"2023-10-09T11:02:14.203761+0000","last_fullsized":"2023-10-09T11:02:14.203761+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:09.954705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":106,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.206155+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:02:14.206155+0000","last_peered":"2023-10-09T11:02:14.206155+0000","last_clean":"2023-10-09T11:02:14.206155+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:02:14.206155+0000","last_undegraded":"2023-10-09T11:02:14.206155+0000","last_fullsized":"2023-10-09T11:02:14.206155+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"430'4682","reported_seq":4718,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:17.985327+0000","last_change":"2023-10-09T11:01:15.405163+0000","last_active":"2023-10-09T11:02:17.985327+0000","last_peered":"2023-10-09T11:02:17.985327+0000","last_clean":"2023-10-09T11:02:17.985327+0000","last_became_active":"2023-10-09T11:01:15.404873+0000","last_became_peered":"2023-10-09T11:01:15.404873+0000","last_unstale":"2023-10-09T11:02:17.985327+0000","last_undegraded":"2023-10-09T11:02:17.985327+0000","last_fullsized":"2023-10-09T11:02:17.985327+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4682,"log_dups_size":0,"ondisk_log_size":4682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:08.894661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306839552,"num_objects":4682,"num_object_clones":0,"num_object_copies":14046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4682,"num_write_kb":299648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":276,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.243105+0000","last_change":"2023-10-09T11:01:26.639395+0000","last_active":"2023-10-09T11:02:16.243105+0000","last_peered":"2023-10-09T11:02:16.243105+0000","last_clean":"2023-10-09T11:02:16.243105+0000","last_became_active":"2023-10-09T11:01:26.639105+0000","last_became_peered":"2023-10-09T11:01:26.639105+0000","last_unstale":"2023-10-09T11:02:16.243105+0000","last_undegraded":"2023-10-09T11:02:16.243105+0000","last_fullsized":"2023-10-09T11:02:16.243105+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:10.391398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":800,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.190635+0000","last_change":"2023-10-09T11:01:29.186101+0000","last_active":"2023-10-09T11:02:14.190635+0000","last_peered":"2023-10-09T11:02:14.190635+0000","last_clean":"2023-10-09T11:02:14.190635+0000","last_became_active":"2023-10-09T11:01:29.185773+0000","last_became_peered":"2023-10-09T11:01:29.185773+0000","last_unstale":"2023-10-09T11:02:14.190635+0000","last_undegraded":"2023-10-09T11:02:14.190635+0000","last_fullsized":"2023-10-09T11:02:14.190635+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:51.271260+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":366,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:18.245359+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T11:02:18.245359+0000","last_peered":"2023-10-09T11:02:18.245359+0000","last_clean":"2023-10-09T11:02:18.245359+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T11:02:18.245359+0000","last_undegraded":"2023-10-09T11:02:18.245359+0000","last_fullsized":"2023-10-09T11:02:18.245359+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":373,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.253359+0000","last_change":"2023-10-09T11:01:26.660425+0000","last_active":"2023-10-09T11:02:16.253359+0000","last_peered":"2023-10-09T11:02:16.253359+0000","last_clean":"2023-10-09T11:02:16.253359+0000","last_became_active":"2023-10-09T11:01:26.660095+0000","last_became_peered":"2023-10-09T11:01:26.660095+0000","last_unstale":"2023-10-09T11:02:16.253359+0000","last_undegraded":"2023-10-09T11:02:16.253359+0000","last_fullsized":"2023-10-09T11:02:16.253359+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:01.819696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"430'4681","reported_seq":6987,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:17.973917+0000","last_change":"2023-10-09T11:01:42.939222+0000","last_active":"2023-10-09T11:02:17.973917+0000","last_peered":"2023-10-09T11:02:17.973917+0000","last_clean":"2023-10-09T11:02:17.973917+0000","last_became_active":"2023-10-09T11:01:42.935626+0000","last_became_peered":"2023-10-09T11:01:42.935626+0000","last_unstale":"2023-10-09T11:02:17.973917+0000","last_undegraded":"2023-10-09T11:02:17.973917+0000","last_fullsized":"2023-10-09T11:02:17.973917+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4681,"log_dups_size":0,"ondisk_log_size":4681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:51:09.694587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306774016,"num_objects":4681,"num_object_clones":0,"num_object_copies":14043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4681,"num_write_kb":299584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1126,"num_bytes_recovered":73793536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":765,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.259111+0000","last_change":"2023-10-09T11:01:17.369788+0000","last_active":"2023-10-09T11:02:16.259111+0000","last_peered":"2023-10-09T11:02:16.259111+0000","last_clean":"2023-10-09T11:02:16.259111+0000","last_became_active":"2023-10-09T11:00:14.881494+0000","last_became_peered":"2023-10-09T11:00:14.881494+0000","last_unstale":"2023-10-09T11:02:16.259111+0000","last_undegraded":"2023-10-09T11:02:16.259111+0000","last_fullsized":"2023-10-09T11:02:16.259111+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:44.208863+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":316,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.203531+0000","last_change":"2023-10-09T11:01:26.641036+0000","last_active":"2023-10-09T11:02:14.203531+0000","last_peered":"2023-10-09T11:02:14.203531+0000","last_clean":"2023-10-09T11:02:14.203531+0000","last_became_active":"2023-10-09T11:01:26.640701+0000","last_became_peered":"2023-10-09T11:01:26.640701+0000","last_unstale":"2023-10-09T11:02:14.203531+0000","last_undegraded":"2023-10-09T11:02:14.203531+0000","last_fullsized":"2023-10-09T11:02:14.203531+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:05.374525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":66,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.189588+0000","last_change":"2023-10-09T10:59:48.135546+0000","last_active":"2023-10-09T11:02:14.189588+0000","last_peered":"2023-10-09T11:02:14.189588+0000","last_clean":"2023-10-09T11:02:14.189588+0000","last_became_active":"2023-10-09T10:59:48.135227+0000","last_became_peered":"2023-10-09T10:59:48.135227+0000","last_unstale":"2023-10-09T11:02:14.189588+0000","last_undegraded":"2023-10-09T11:02:14.189588+0000","last_fullsized":"2023-10-09T11:02:14.189588+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:50.561785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":387,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.189695+0000","last_change":"2023-10-09T11:00:15.892896+0000","last_active":"2023-10-09T11:02:14.189695+0000","last_peered":"2023-10-09T11:02:14.189695+0000","last_clean":"2023-10-09T11:02:14.189695+0000","last_became_active":"2023-10-09T11:00:15.892573+0000","last_became_peered":"2023-10-09T11:00:15.892573+0000","last_unstale":"2023-10-09T11:02:14.189695+0000","last_undegraded":"2023-10-09T11:02:14.189695+0000","last_fullsized":"2023-10-09T11:02:14.189695+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:13.280089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"430'4634","reported_seq":4670,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:17.971366+0000","last_change":"2023-10-09T11:01:15.402557+0000","last_active":"2023-10-09T11:02:17.971366+0000","last_peered":"2023-10-09T11:02:17.971366+0000","last_clean":"2023-10-09T11:02:17.971366+0000","last_became_active":"2023-10-09T11:01:15.402279+0000","last_became_peered":"2023-10-09T11:01:15.402279+0000","last_unstale":"2023-10-09T11:02:17.971366+0000","last_undegraded":"2023-10-09T11:02:17.971366+0000","last_fullsized":"2023-10-09T11:02:17.971366+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4634,"log_dups_size":0,"ondisk_log_size":4634,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:19.333267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303693824,"num_objects":4634,"num_object_clones":0,"num_object_copies":13902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4634,"num_write_kb":296576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":643,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.189496+0000","last_change":"2023-10-09T11:01:16.782558+0000","last_active":"2023-10-09T11:02:14.189496+0000","last_peered":"2023-10-09T11:02:14.189496+0000","last_clean":"2023-10-09T11:02:14.189496+0000","last_became_active":"2023-10-09T10:59:51.137287+0000","last_became_peered":"2023-10-09T10:59:51.137287+0000","last_unstale":"2023-10-09T11:02:14.189496+0000","last_undegraded":"2023-10-09T11:02:14.189496+0000","last_fullsized":"2023-10-09T11:02:14.189496+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:58:29.405479+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":269,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.189541+0000","last_change":"2023-10-09T11:01:26.649138+0000","last_active":"2023-10-09T11:02:14.189541+0000","last_peered":"2023-10-09T11:02:14.189541+0000","last_clean":"2023-10-09T11:02:14.189541+0000","last_became_active":"2023-10-09T11:01:26.648437+0000","last_became_peered":"2023-10-09T11:01:26.648437+0000","last_unstale":"2023-10-09T11:02:14.189541+0000","last_undegraded":"2023-10-09T11:02:14.189541+0000","last_fullsized":"2023-10-09T11:02:14.189541+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:29.065233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":109,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.259493+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:02:16.259493+0000","last_peered":"2023-10-09T11:02:16.259493+0000","last_clean":"2023-10-09T11:02:16.259493+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:02:16.259493+0000","last_undegraded":"2023-10-09T11:02:16.259493+0000","last_fullsized":"2023-10-09T11:02:16.259493+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":362,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.188950+0000","last_change":"2023-10-09T11:01:26.643724+0000","last_active":"2023-10-09T11:02:14.188950+0000","last_peered":"2023-10-09T11:02:14.188950+0000","last_clean":"2023-10-09T11:02:14.188950+0000","last_became_active":"2023-10-09T11:01:26.643301+0000","last_became_peered":"2023-10-09T11:01:26.643301+0000","last_unstale":"2023-10-09T11:02:14.188950+0000","last_undegraded":"2023-10-09T11:02:14.188950+0000","last_fullsized":"2023-10-09T11:02:14.188950+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:49:36.408009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"431'4834","reported_seq":4870,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.433060+0000","last_change":"2023-10-09T11:01:15.406858+0000","last_active":"2023-10-09T11:02:21.433060+0000","last_peered":"2023-10-09T11:02:21.433060+0000","last_clean":"2023-10-09T11:02:21.433060+0000","last_became_active":"2023-10-09T11:01:15.406513+0000","last_became_peered":"2023-10-09T11:01:15.406513+0000","last_unstale":"2023-10-09T11:02:21.433060+0000","last_undegraded":"2023-10-09T11:02:21.433060+0000","last_fullsized":"2023-10-09T11:02:21.433060+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4834,"log_dups_size":0,"ondisk_log_size":4834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:28:25.137720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316801024,"num_objects":4834,"num_object_clones":0,"num_object_copies":14502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4834,"num_write_kb":309376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":826,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.249509+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T11:02:16.249509+0000","last_peered":"2023-10-09T11:02:16.249509+0000","last_clean":"2023-10-09T11:02:16.249509+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T11:02:16.249509+0000","last_undegraded":"2023-10-09T11:02:16.249509+0000","last_fullsized":"2023-10-09T11:02:16.249509+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":521,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233923+0000","last_change":"2023-10-09T11:01:26.666078+0000","last_active":"2023-10-09T11:02:16.233923+0000","last_peered":"2023-10-09T11:02:16.233923+0000","last_clean":"2023-10-09T11:02:16.233923+0000","last_became_active":"2023-10-09T11:01:26.637507+0000","last_became_peered":"2023-10-09T11:01:26.637507+0000","last_unstale":"2023-10-09T11:02:16.233923+0000","last_undegraded":"2023-10-09T11:02:16.233923+0000","last_fullsized":"2023-10-09T11:02:16.233923+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:55.708083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":108,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.203978+0000","last_change":"2023-10-09T11:00:14.900305+0000","last_active":"2023-10-09T11:02:14.203978+0000","last_peered":"2023-10-09T11:02:14.203978+0000","last_clean":"2023-10-09T11:02:14.203978+0000","last_became_active":"2023-10-09T11:00:14.899724+0000","last_became_peered":"2023-10-09T11:00:14.899724+0000","last_unstale":"2023-10-09T11:02:14.203978+0000","last_undegraded":"2023-10-09T11:02:14.203978+0000","last_fullsized":"2023-10-09T11:02:14.203978+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:59.022161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":502,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232120+0000","last_change":"2023-10-09T11:01:26.637115+0000","last_active":"2023-10-09T11:02:16.232120+0000","last_peered":"2023-10-09T11:02:16.232120+0000","last_clean":"2023-10-09T11:02:16.232120+0000","last_became_active":"2023-10-09T11:01:26.636783+0000","last_became_peered":"2023-10-09T11:01:26.636783+0000","last_unstale":"2023-10-09T11:02:16.232120+0000","last_undegraded":"2023-10-09T11:02:16.232120+0000","last_fullsized":"2023-10-09T11:02:16.232120+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:50.446443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"431'4931","reported_seq":4967,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.464631+0000","last_change":"2023-10-09T11:01:15.397123+0000","last_active":"2023-10-09T11:02:21.464631+0000","last_peered":"2023-10-09T11:02:21.464631+0000","last_clean":"2023-10-09T11:02:21.464631+0000","last_became_active":"2023-10-09T11:01:15.396842+0000","last_became_peered":"2023-10-09T11:01:15.396842+0000","last_unstale":"2023-10-09T11:02:21.464631+0000","last_undegraded":"2023-10-09T11:02:21.464631+0000","last_fullsized":"2023-10-09T11:02:21.464631+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4931,"log_dups_size":0,"ondisk_log_size":4931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:54.895524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323158016,"num_objects":4931,"num_object_clones":0,"num_object_copies":14793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4931,"num_write_kb":315584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":876,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232910+0000","last_change":"2023-10-09T11:01:29.165146+0000","last_active":"2023-10-09T11:02:16.232910+0000","last_peered":"2023-10-09T11:02:16.232910+0000","last_clean":"2023-10-09T11:02:16.232910+0000","last_became_active":"2023-10-09T11:01:29.164818+0000","last_became_peered":"2023-10-09T11:01:29.164818+0000","last_unstale":"2023-10-09T11:02:16.232910+0000","last_undegraded":"2023-10-09T11:02:16.232910+0000","last_fullsized":"2023-10-09T11:02:16.232910+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:42.211670+0000","scrub_duration":0.0053707060000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":206,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.228583+0000","last_change":"2023-10-09T11:01:26.648453+0000","last_active":"2023-10-09T11:02:16.228583+0000","last_peered":"2023-10-09T11:02:16.228583+0000","last_clean":"2023-10-09T11:02:16.228583+0000","last_became_active":"2023-10-09T11:01:26.648115+0000","last_became_peered":"2023-10-09T11:01:26.648115+0000","last_unstale":"2023-10-09T11:02:16.228583+0000","last_undegraded":"2023-10-09T11:02:16.228583+0000","last_fullsized":"2023-10-09T11:02:16.228583+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:26.270236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":371,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.189782+0000","last_change":"2023-10-09T11:00:15.869009+0000","last_active":"2023-10-09T11:02:14.189782+0000","last_peered":"2023-10-09T11:02:14.189782+0000","last_clean":"2023-10-09T11:02:14.189782+0000","last_became_active":"2023-10-09T11:00:15.868637+0000","last_became_peered":"2023-10-09T11:00:15.868637+0000","last_unstale":"2023-10-09T11:02:14.189782+0000","last_undegraded":"2023-10-09T11:02:14.189782+0000","last_fullsized":"2023-10-09T11:02:14.189782+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:37.538741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":216,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244174+0000","last_change":"2023-10-09T11:01:15.402661+0000","last_active":"2023-10-09T11:02:16.244174+0000","last_peered":"2023-10-09T11:02:16.244174+0000","last_clean":"2023-10-09T11:02:16.244174+0000","last_became_active":"2023-10-09T11:01:15.402327+0000","last_became_peered":"2023-10-09T11:01:15.402327+0000","last_unstale":"2023-10-09T11:02:16.244174+0000","last_undegraded":"2023-10-09T11:02:16.244174+0000","last_fullsized":"2023-10-09T11:02:16.244174+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:32.513751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":858,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233385+0000","last_change":"2023-10-09T10:59:51.145348+0000","last_active":"2023-10-09T11:02:16.233385+0000","last_peered":"2023-10-09T11:02:16.233385+0000","last_clean":"2023-10-09T11:02:16.233385+0000","last_became_active":"2023-10-09T10:59:51.145100+0000","last_became_peered":"2023-10-09T10:59:51.145100+0000","last_unstale":"2023-10-09T11:02:16.233385+0000","last_undegraded":"2023-10-09T11:02:16.233385+0000","last_fullsized":"2023-10-09T11:02:16.233385+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:51.049260+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":361,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.189034+0000","last_change":"2023-10-09T11:01:26.649487+0000","last_active":"2023-10-09T11:02:14.189034+0000","last_peered":"2023-10-09T11:02:14.189034+0000","last_clean":"2023-10-09T11:02:14.189034+0000","last_became_active":"2023-10-09T11:01:26.649133+0000","last_became_peered":"2023-10-09T11:01:26.649133+0000","last_unstale":"2023-10-09T11:02:14.189034+0000","last_undegraded":"2023-10-09T11:02:14.189034+0000","last_fullsized":"2023-10-09T11:02:14.189034+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:40.585289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":187,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.229065+0000","last_change":"2023-10-09T11:00:14.886903+0000","last_active":"2023-10-09T11:02:16.229065+0000","last_peered":"2023-10-09T11:02:16.229065+0000","last_clean":"2023-10-09T11:02:16.229065+0000","last_became_active":"2023-10-09T11:00:14.886635+0000","last_became_peered":"2023-10-09T11:00:14.886635+0000","last_unstale":"2023-10-09T11:02:16.229065+0000","last_undegraded":"2023-10-09T11:02:16.229065+0000","last_fullsized":"2023-10-09T11:02:16.229065+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:04.982085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":807,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233198+0000","last_change":"2023-10-09T11:01:29.173501+0000","last_active":"2023-10-09T11:02:16.233198+0000","last_peered":"2023-10-09T11:02:16.233198+0000","last_clean":"2023-10-09T11:02:16.233198+0000","last_became_active":"2023-10-09T11:01:29.172950+0000","last_became_peered":"2023-10-09T11:01:29.172950+0000","last_unstale":"2023-10-09T11:02:16.233198+0000","last_undegraded":"2023-10-09T11:02:16.233198+0000","last_fullsized":"2023-10-09T11:02:16.233198+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:40:05.032667+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":258,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242204+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:02:16.242204+0000","last_peered":"2023-10-09T11:02:16.242204+0000","last_clean":"2023-10-09T11:02:16.242204+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:02:16.242204+0000","last_undegraded":"2023-10-09T11:02:16.242204+0000","last_fullsized":"2023-10-09T11:02:16.242204+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":459,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232468+0000","last_change":"2023-10-09T10:57:46.844827+0000","last_active":"2023-10-09T11:02:16.232468+0000","last_peered":"2023-10-09T11:02:16.232468+0000","last_clean":"2023-10-09T11:02:16.232468+0000","last_became_active":"2023-10-09T10:57:46.844584+0000","last_became_peered":"2023-10-09T10:57:46.844584+0000","last_unstale":"2023-10-09T11:02:16.232468+0000","last_undegraded":"2023-10-09T11:02:16.232468+0000","last_fullsized":"2023-10-09T11:02:16.232468+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:54.499813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":381,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242510+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T11:02:16.242510+0000","last_peered":"2023-10-09T11:02:16.242510+0000","last_clean":"2023-10-09T11:02:16.242510+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T11:02:16.242510+0000","last_undegraded":"2023-10-09T11:02:16.242510+0000","last_fullsized":"2023-10-09T11:02:16.242510+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":395,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232468+0000","last_change":"2023-10-09T10:59:48.136562+0000","last_active":"2023-10-09T11:02:16.232468+0000","last_peered":"2023-10-09T11:02:16.232468+0000","last_clean":"2023-10-09T11:02:16.232468+0000","last_became_active":"2023-10-09T10:59:48.136184+0000","last_became_peered":"2023-10-09T10:59:48.136184+0000","last_unstale":"2023-10-09T11:02:16.232468+0000","last_undegraded":"2023-10-09T11:02:16.232468+0000","last_fullsized":"2023-10-09T11:02:16.232468+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:16.944093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":650,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.205849+0000","last_change":"2023-10-09T11:01:29.163551+0000","last_active":"2023-10-09T11:02:14.205849+0000","last_peered":"2023-10-09T11:02:14.205849+0000","last_clean":"2023-10-09T11:02:14.205849+0000","last_became_active":"2023-10-09T11:01:29.163220+0000","last_became_peered":"2023-10-09T11:01:29.163220+0000","last_unstale":"2023-10-09T11:02:14.205849+0000","last_undegraded":"2023-10-09T11:02:14.205849+0000","last_fullsized":"2023-10-09T11:02:14.205849+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:47.307693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:47.307693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":458,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233518+0000","last_change":"2023-10-09T10:54:17.886158+0000","last_active":"2023-10-09T11:02:16.233518+0000","last_peered":"2023-10-09T11:02:16.233518+0000","last_clean":"2023-10-09T11:02:16.233518+0000","last_became_active":"2023-10-09T10:54:17.885816+0000","last_became_peered":"2023-10-09T10:54:17.885816+0000","last_unstale":"2023-10-09T11:02:16.233518+0000","last_undegraded":"2023-10-09T11:02:16.233518+0000","last_fullsized":"2023-10-09T11:02:16.233518+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:05.270825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":478,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233721+0000","last_change":"2023-10-09T10:59:48.125955+0000","last_active":"2023-10-09T11:02:16.233721+0000","last_peered":"2023-10-09T11:02:16.233721+0000","last_clean":"2023-10-09T11:02:16.233721+0000","last_became_active":"2023-10-09T10:59:48.125641+0000","last_became_peered":"2023-10-09T10:59:48.125641+0000","last_unstale":"2023-10-09T11:02:16.233721+0000","last_undegraded":"2023-10-09T11:02:16.233721+0000","last_fullsized":"2023-10-09T11:02:16.233721+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:38.041100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":674,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.206316+0000","last_change":"2023-10-09T11:01:54.238624+0000","last_active":"2023-10-09T11:02:14.206316+0000","last_peered":"2023-10-09T11:02:14.206316+0000","last_clean":"2023-10-09T11:02:14.206316+0000","last_became_active":"2023-10-09T11:00:14.867744+0000","last_became_peered":"2023-10-09T11:00:14.867744+0000","last_unstale":"2023-10-09T11:02:14.206316+0000","last_undegraded":"2023-10-09T11:02:14.206316+0000","last_fullsized":"2023-10-09T11:02:14.206316+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:21.022863+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:21.022863+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":72,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T11:02:14.192475+0000","last_change":"2023-10-09T10:59:48.143732+0000","last_active":"2023-10-09T11:02:14.192475+0000","last_peered":"2023-10-09T11:02:14.192475+0000","last_clean":"2023-10-09T11:02:14.192475+0000","last_became_active":"2023-10-09T10:59:48.129001+0000","last_became_peered":"2023-10-09T10:59:48.129001+0000","last_unstale":"2023-10-09T11:02:14.192475+0000","last_undegraded":"2023-10-09T11:02:14.192475+0000","last_fullsized":"2023-10-09T11:02:14.192475+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:22:36.946348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":452,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.248859+0000","last_change":"2023-10-09T11:00:14.869353+0000","last_active":"2023-10-09T11:02:16.248859+0000","last_peered":"2023-10-09T11:02:16.248859+0000","last_clean":"2023-10-09T11:02:16.248859+0000","last_became_active":"2023-10-09T11:00:14.869061+0000","last_became_peered":"2023-10-09T11:00:14.869061+0000","last_unstale":"2023-10-09T11:02:16.248859+0000","last_undegraded":"2023-10-09T11:02:16.248859+0000","last_fullsized":"2023-10-09T11:02:16.248859+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:30.318094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5004525568,"num_objects":76363,"num_object_clones":0,"num_object_copies":229089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":76363,"num_write_kb":4887232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4540,"num_bytes_recovered":297533440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15175712768,"data_stored":15175712768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76363,"ondisk_log_size":76363,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":12396,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8961,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635094,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":39,"num_read_kb":70,"num_write":20,"num_write_kb":184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1216064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":219043332429,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":44,"seq":188978561360,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3070512,"kb_used_data":2174440,"kb_used_omap":0,"kb_used_meta":896064,"kb_avail":90674640,"statfs":{"total":95995035648,"available":92850831360,"internally_reserved":0,"allocated":2226626560,"data_stored":2225259232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917569536},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:07 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":1.8009999999999999,"15min":1.877},"min":{"1min":0.224,"5min":0.224,"15min":0.216},"max":{"1min":3.9460000000000002,"5min":46.603999999999999,"15min":54.743000000000002},"last":4.9130000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":2.0760000000000001,"15min":1.9790000000000001},"min":{"1min":0.28999999999999998,"5min":0.223,"15min":0.223},"max":{"1min":1.9039999999999999,"5min":46.679000000000002,"15min":53.701000000000001},"last":4.7640000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:02:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0529999999999999,"5min":1.2130000000000001,"15min":0.90700000000000003},"min":{"1min":0.29399999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":41.241999999999997,"5min":41.241999999999997,"15min":41.241999999999997},"last":4.8460000000000001},{"interface":"front","average":{"1min":3.5070000000000001,"5min":1.792,"15min":1.506},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":41.476999999999997,"5min":41.476999999999997,"15min":41.476999999999997},"last":2.6720000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.794,"15min":1.843},"min":{"1min":0.34999999999999998,"5min":0.30599999999999999,"15min":0.28399999999999997},"max":{"1min":3.2759999999999998,"5min":12.003,"15min":13.208},"last":2.601},{"interface":"front","average":{"1min":1.7829999999999999,"5min":2.0150000000000001,"15min":2.0009999999999999},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.28599999999999998},"max":{"1min":7.1870000000000003,"5min":15.432,"15min":15.432},"last":1.766}]},{"osd":3,"last update":"Mon Oct 9 11:02:07 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.6719999999999999,"15min":1.8320000000000001},"min":{"1min":0.35499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":3.266,"5min":10.018000000000001,"15min":12.391},"last":4.9429999999999996},{"interface":"front","average":{"1min":1.101,"5min":1.885,"15min":2.0870000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":3.6389999999999998,"5min":11.161,"15min":13.179},"last":2.6429999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.90100000000000002,"15min":1.2949999999999999},"min":{"1min":0.25900000000000001,"5min":0.25800000000000001,"15min":0.23200000000000001},"max":{"1min":3.9449999999999998,"5min":9.0060000000000002,"15min":16.978999999999999},"last":1.6040000000000001},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.88300000000000001,"15min":1.2450000000000001},"min":{"1min":0.27800000000000002,"5min":0.249,"15min":0.24399999999999999},"max":{"1min":2.944,"5min":5.7869999999999999,"15min":12.481999999999999},"last":4.8049999999999997}]},{"osd":5,"last update":"Mon Oct 9 11:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.385,"15min":1.5600000000000001},"min":{"1min":0.30499999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.9260000000000002,"5min":19.786999999999999,"15min":19.786999999999999},"last":1.6479999999999999},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.083,"15min":1.339},"min":{"1min":0.35099999999999998,"5min":0.28899999999999998,"15min":0.26600000000000001},"max":{"1min":3.6850000000000001,"5min":9.0559999999999992,"15min":12.456},"last":4.8810000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:01:59 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.2989999999999999,"15min":1.5549999999999999},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.26600000000000001},"max":{"1min":1.885,"5min":9.1120000000000001,"15min":23.358000000000001},"last":1.6850000000000001},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.96699999999999997,"15min":1.216},"min":{"1min":0.32400000000000001,"5min":0.32300000000000001,"15min":0.27500000000000002},"max":{"1min":0.83199999999999996,"5min":6.1920000000000002,"15min":12.422000000000001},"last":4.9690000000000003}]}]},{"osd":1,"up_from":388,"seq":1666447310874,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1900836,"kb_used_data":1656604,"kb_used_omap":13,"kb_used_meta":244210,"kb_avail":91844316,"statfs":{"total":95995035648,"available":94048579584,"internally_reserved":0,"allocated":1696362496,"data_stored":1694130385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":250071751},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:01:16 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":2.258,"15min":2.258},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":10.609,"5min":10.609,"15min":10.609},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.859,"5min":6.859,"15min":6.859},"last":0.46600000000000003}]},{"osd":2,"last update":"Mon Oct 9 11:01:16 2023","interfaces":[{"interface":"back","average":{"1min":3.2189999999999999,"5min":3.2189999999999999,"15min":3.2189999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":12.009,"5min":12.009,"15min":12.009},"last":0.53400000000000003},{"interface":"front","average":{"1min":2.2330000000000001,"5min":2.2330000000000001,"15min":2.2330000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.2959999999999994,"5min":8.2959999999999994,"15min":8.2959999999999994},"last":0.63100000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:01:16 2023","interfaces":[{"interface":"back","average":{"1min":2.8279999999999998,"5min":2.8279999999999998,"15min":2.8279999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":12.523999999999999,"5min":12.523999999999999,"15min":12.523999999999999},"last":5.3579999999999997},{"interface":"front","average":{"1min":4.0640000000000001,"5min":4.0640000000000001,"15min":4.0640000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":18.079999999999998,"5min":18.079999999999998,"15min":18.079999999999998},"last":4.4699999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:01:16 2023","interfaces":[{"interface":"back","average":{"1min":2.863,"5min":2.863,"15min":2.863},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":10.481,"5min":10.481,"15min":10.481},"last":0.501},{"interface":"front","average":{"1min":1.385,"5min":1.385,"15min":1.385},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":4.4180000000000001,"5min":4.4180000000000001,"15min":4.4180000000000001},"last":0.65900000000000003}]},{"osd":5,"last update":"Mon Oct 9 11:01:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5419999999999998,"5min":2.5419999999999998,"15min":2.5419999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":18.053000000000001,"5min":18.053000000000001,"15min":18.053000000000001},"last":4.4199999999999999},{"interface":"front","average":{"1min":2.423,"5min":2.423,"15min":2.423},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":10.558,"5min":10.558,"15min":10.558},"last":2.3620000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:01:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5529999999999999,"5min":2.5529999999999999,"15min":2.5529999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":18.085999999999999,"5min":18.085999999999999,"15min":18.085999999999999},"last":4.3769999999999998},{"interface":"front","average":{"1min":2.4079999999999999,"5min":2.4079999999999999,"15min":2.4079999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":10.531000000000001,"5min":10.531000000000001,"15min":10.531000000000001},"last":2.5720000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:01:16 2023","interfaces":[{"interface":"back","average":{"1min":1.2869999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.8280000000000003,"5min":4.8280000000000003,"15min":4.8280000000000003},"last":0.59299999999999997},{"interface":"front","average":{"1min":2.613,"5min":2.613,"15min":2.613},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":18.071000000000002,"5min":18.071000000000002,"15min":18.071000000000002},"last":4.4470000000000001}]}]},{"osd":0,"up_from":80,"seq":343597383932,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3684620,"kb_used_data":2994820,"kb_used_omap":28,"kb_used_meta":689763,"kb_avail":90060532,"statfs":{"total":95995035648,"available":92221984768,"internally_reserved":0,"allocated":3066695680,"data_stored":3064419783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":706318160},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:02:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":3.0379999999999998,"15min":3.2040000000000002},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":9.6460000000000008,"5min":19.608000000000001,"15min":19.608000000000001},"last":0.217},{"interface":"front","average":{"1min":1.879,"5min":3.0710000000000002,"15min":3.2690000000000001},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.218},"max":{"1min":7.8730000000000002,"5min":19.992999999999999,"15min":19.992999999999999},"last":2.2269999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:01:45 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.958,"15min":2.1480000000000001},"min":{"1min":0.24199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":4.6639999999999997,"5min":12.638,"15min":24.852},"last":2.5179999999999998},{"interface":"front","average":{"1min":1.6299999999999999,"5min":2.4609999999999999,"15min":2.6219999999999999},"min":{"1min":0.20899999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":8.7140000000000004,"5min":12.672000000000001,"15min":77.256},"last":3.899}]},{"osd":3,"last update":"Mon Oct 9 11:02:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":2.1589999999999998,"15min":2.3090000000000002},"min":{"1min":0.26800000000000002,"5min":0.215,"15min":0.20200000000000001},"max":{"1min":8.8699999999999992,"5min":28.946000000000002,"15min":28.946000000000002},"last":0.20499999999999999},{"interface":"front","average":{"1min":1.46,"5min":2.4350000000000001,"15min":2.5790000000000002},"min":{"1min":0.309,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.9850000000000003,"5min":19.952999999999999,"15min":19.952999999999999},"last":0.29299999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":2.2679999999999998,"15min":2.4649999999999999},"min":{"1min":0.32900000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.9560000000000004,"5min":11.507,"15min":77.177999999999997},"last":0.76000000000000001},{"interface":"front","average":{"1min":1.405,"5min":1.6499999999999999,"15min":2.0190000000000001},"min":{"1min":0.38400000000000001,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":6.4450000000000003,"5min":9.9489999999999998,"15min":12.318},"last":0.308}]},{"osd":5,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.827,"15min":2.077},"min":{"1min":0.36599999999999999,"5min":0.308,"15min":0.29799999999999999},"max":{"1min":6.4880000000000004,"5min":9.8710000000000004,"15min":12.157},"last":4.891},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.9350000000000001,"15min":2.0750000000000002},"min":{"1min":0.436,"5min":0.29999999999999999,"15min":0.29899999999999999},"max":{"1min":2.859,"5min":7.0919999999999996,"15min":24.754999999999999},"last":0.32600000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5700000000000001,"5min":1.855,"15min":2.2349999999999999},"min":{"1min":0.38800000000000001,"5min":0.314,"15min":0.30399999999999999},"max":{"1min":6.5179999999999998,"5min":10.012,"15min":18.920000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.9490000000000001,"15min":1.9810000000000001},"min":{"1min":0.45700000000000002,"5min":0.31900000000000001,"15min":0.307},"max":{"1min":2.9209999999999998,"5min":8.8249999999999993,"15min":24.800000000000001},"last":2.5430000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.958,"15min":2.177},"min":{"1min":0.44900000000000001,"5min":0.34499999999999997,"15min":0.32700000000000001},"max":{"1min":2.8929999999999998,"5min":11.285,"15min":14.48},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.3740000000000001,"5min":2.3809999999999998,"15min":2.5910000000000002},"min":{"1min":0.41499999999999998,"5min":0.372,"15min":0.314},"max":{"1min":6.008,"5min":10.398,"15min":77.228999999999999},"last":0.54600000000000004}]}]},{"osd":2,"up_from":90,"seq":386547056873,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2370160,"kb_used_data":1858088,"kb_used_omap":81,"kb_used_meta":511982,"kb_avail":91374992,"statfs":{"total":95995035648,"available":93567991808,"internally_reserved":0,"allocated":1902682112,"data_stored":1900560975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":524269786},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":1.3819999999999999,"15min":1.7569999999999999},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":8.9990000000000006,"5min":8.9990000000000006,"15min":18.512},"last":2.3820000000000001},{"interface":"front","average":{"1min":1.5269999999999999,"5min":1.5609999999999999,"15min":1.9710000000000001},"min":{"1min":0.20699999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":7.7469999999999999,"5min":7.7469999999999999,"15min":17.975000000000001},"last":0.48199999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:02:18 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":2.403,"15min":2.4830000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":10.502000000000001,"5min":10.502000000000001,"15min":10.502000000000001},"last":1.9139999999999999},{"interface":"front","average":{"1min":1.6200000000000001,"5min":2.448,"15min":2.5859999999999999},"min":{"1min":0.27300000000000002,"5min":0.214,"15min":0.214},"max":{"1min":5.125,"5min":11.92,"15min":11.92},"last":5.125}]},{"osd":3,"last update":"Mon Oct 9 11:02:09 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":1.6279999999999999,"15min":1.863},"min":{"1min":0.24399999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.5979999999999999,"5min":11.151,"15min":19.539000000000001},"last":2.7450000000000001},{"interface":"front","average":{"1min":1.01,"5min":1.913,"15min":2.093},"min":{"1min":0.27200000000000002,"5min":0.219,"15min":0.219},"max":{"1min":3.524,"5min":16.324000000000002,"15min":16.702999999999999},"last":2.1040000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":2.016,"15min":1.984},"min":{"1min":0.35399999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.4859999999999998,"5min":9.3290000000000006,"15min":16.231999999999999},"last":1.948},{"interface":"front","average":{"1min":1.458,"5min":1.512,"15min":1.782},"min":{"1min":0.33400000000000002,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":4.8470000000000004,"5min":6.8490000000000002,"15min":15.696999999999999},"last":2.4209999999999998}]},{"osd":5,"last update":"Mon Oct 9 11:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.581,"15min":1.8080000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.6369999999999996,"5min":8.5380000000000003,"15min":17.920000000000002},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.6639999999999999,"5min":1.954,"15min":1.899},"min":{"1min":0.311,"5min":0.311,"15min":0.30299999999999999},"max":{"1min":7.5250000000000004,"5min":8.5869999999999997,"15min":17.882999999999999},"last":1.9730000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":1.341,"15min":1.5609999999999999},"min":{"1min":0.373,"5min":0.32100000000000001,"15min":0.30499999999999999},"max":{"1min":4.3239999999999998,"5min":7.4009999999999998,"15min":10.443},"last":2.452},{"interface":"front","average":{"1min":1.383,"5min":1.429,"15min":1.74},"min":{"1min":0.36099999999999999,"5min":0.313,"15min":0.313},"max":{"1min":6.6749999999999998,"5min":6.6749999999999998,"15min":12.862},"last":0.65300000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.679,"5min":1.6399999999999999,"15min":1.6699999999999999},"min":{"1min":0.39000000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.5609999999999999,"5min":6.3300000000000001,"15min":10.470000000000001},"last":2.7970000000000002},{"interface":"front","average":{"1min":1.393,"5min":1.607,"15min":1.883},"min":{"1min":0.34699999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.71,"5min":7.6550000000000002,"15min":12.826000000000001},"last":1.4750000000000001}]}]},{"osd":3,"up_from":95,"seq":408021893347,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2833472,"kb_used_data":1851256,"kb_used_omap":7,"kb_used_meta":982200,"kb_avail":90911680,"statfs":{"total":95995035648,"available":93093560320,"internally_reserved":0,"allocated":1895686144,"data_stored":1893367869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1005773416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":2.556,"15min":2.6800000000000002},"min":{"1min":0.222,"5min":0.20499999999999999,"15min":0.17499999999999999},"max":{"1min":8.1709999999999994,"5min":10.140000000000001,"15min":38.817999999999998},"last":5.9690000000000003},{"interface":"front","average":{"1min":2.4769999999999999,"5min":2.3530000000000002,"15min":2.9649999999999999},"min":{"1min":0.246,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":8.4239999999999995,"5min":17.042000000000002,"15min":17.312000000000001},"last":3.8719999999999999}]},{"osd":1,"last update":"Mon Oct 9 11:02:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8,"5min":2.9199999999999999,"15min":3.1059999999999999},"min":{"1min":0.25,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":8.8030000000000008,"5min":8.9100000000000001,"15min":8.9100000000000001},"last":5.4930000000000003},{"interface":"front","average":{"1min":1.4650000000000001,"5min":2.5840000000000001,"15min":2.77},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.2300000000000004,"5min":9.7449999999999992,"15min":9.7449999999999992},"last":0.66700000000000004}]},{"osd":2,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":2.3100000000000001,"5min":2.3069999999999999,"15min":2.448},"min":{"1min":0.21099999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":5.5350000000000001,"5min":10.65,"15min":20.402000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.0489999999999999,"5min":2.2839999999999998,"15min":2.8260000000000001},"min":{"1min":0.23899999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":8.2729999999999997,"5min":10.012,"15min":27.538},"last":0.46600000000000003}]},{"osd":4,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.355,"5min":2.101,"15min":2.2770000000000001},"min":{"1min":0.30299999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.3639999999999999,"5min":9.8089999999999993,"15min":20.388000000000002},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.3029999999999999,"5min":1.7929999999999999,"15min":2.157},"min":{"1min":0.34499999999999997,"5min":0.34000000000000002,"15min":0.24299999999999999},"max":{"1min":3.5910000000000002,"5min":9.1310000000000002,"15min":19.085999999999999},"last":0.50600000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.948,"15min":2.2890000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.26000000000000001},"max":{"1min":3.5099999999999998,"5min":10.380000000000001,"15min":14.045},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.4630000000000001,"5min":1.7470000000000001,"15min":2.0369999999999999},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2839999999999998,"5min":8.3949999999999996,"15min":17.311},"last":0.48999999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":2.1419999999999999,"15min":2.27},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28000000000000003},"max":{"1min":8.3119999999999994,"5min":10.420999999999999,"15min":12.361000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.835,"5min":1.9430000000000001,"15min":2.2090000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.8959999999999999,"5min":7.1230000000000002,"15min":17.295000000000002},"last":0.629}]},{"osd":7,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":1.9419999999999999,"15min":2.2440000000000002},"min":{"1min":0.46400000000000002,"5min":0.309,"15min":0.309},"max":{"1min":7.492,"5min":8.6649999999999991,"15min":18.722000000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.6180000000000001,"5min":2.194,"15min":2.4670000000000001},"min":{"1min":0.439,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.4020000000000001,"5min":9.8569999999999993,"15min":15.212999999999999},"last":1.0249999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019223,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3676152,"kb_used_data":2779312,"kb_used_omap":0,"kb_used_meta":896832,"kb_avail":90069000,"statfs":{"total":95995035648,"available":92230656000,"internally_reserved":0,"allocated":2846015488,"data_stored":2844640333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918355968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.411,"15min":2.3370000000000002},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":2.9870000000000001,"5min":8.6180000000000003,"15min":213.88999999999999},"last":0.26100000000000001},{"interface":"front","average":{"1min":1.393,"5min":1.359,"15min":2.6019999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.217},"max":{"1min":6.1269999999999998,"5min":6.1269999999999998,"15min":214.09800000000001},"last":2.4049999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:02:15 2023","interfaces":[{"interface":"back","average":{"1min":8.0950000000000006,"5min":3.2919999999999998,"15min":2.492},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":121.41,"5min":121.41,"15min":121.41},"last":0.32500000000000001},{"interface":"front","average":{"1min":7.8860000000000001,"5min":2.6699999999999999,"15min":1.8009999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":122.21899999999999,"5min":122.21899999999999,"15min":122.21899999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:01:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":1.4550000000000001,"15min":1.7809999999999999},"min":{"1min":0.27900000000000003,"5min":0.27100000000000002,"15min":0.253},"max":{"1min":3.9390000000000001,"5min":5.6870000000000003,"15min":15.446999999999999},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.492,"5min":1.9299999999999999,"15min":2.1739999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.9540000000000002,"5min":9.9480000000000004,"15min":27.818000000000001},"last":1.675}]},{"osd":3,"last update":"Mon Oct 9 11:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":1.7509999999999999,"15min":2.6469999999999998},"min":{"1min":0.46999999999999997,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":2.254,"5min":12.798999999999999,"15min":209.03700000000001},"last":2.6669999999999998},{"interface":"front","average":{"1min":0.878,"5min":1.7749999999999999,"15min":2.899},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29999999999999999},"max":{"1min":1.8480000000000001,"5min":8.6010000000000009,"15min":208.80199999999999},"last":2.7120000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":1.2909999999999999,"15min":1.7669999999999999},"min":{"1min":0.307,"5min":0.245,"15min":0.23799999999999999},"max":{"1min":2.024,"5min":9.7880000000000003,"15min":24.149999999999999},"last":0.314},{"interface":"front","average":{"1min":0.78800000000000003,"5min":1.2190000000000001,"15min":1.556},"min":{"1min":0.36899999999999999,"5min":0.25800000000000001,"15min":0.22600000000000001},"max":{"1min":3.1859999999999999,"5min":12.459,"15min":20.422000000000001},"last":0.96899999999999997}]},{"osd":6,"last update":"Mon Oct 9 11:01:43 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.99399999999999999,"15min":1.357},"min":{"1min":0.34999999999999998,"5min":0.27800000000000002,"15min":0.24099999999999999},"max":{"1min":7.2309999999999999,"5min":8.5739999999999998,"15min":15.739000000000001},"last":0.436},{"interface":"front","average":{"1min":1.052,"5min":1.454,"15min":1.7270000000000001},"min":{"1min":0.34100000000000003,"5min":0.32800000000000001,"15min":0.311},"max":{"1min":4.5220000000000002,"5min":9.0489999999999995,"15min":14.272},"last":0.502}]},{"osd":7,"last update":"Mon Oct 9 11:01:50 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":1.7170000000000001,"15min":1.72},"min":{"1min":0.33200000000000002,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":4.9429999999999996,"5min":19.923999999999999,"15min":19.923999999999999},"last":1.0009999999999999},{"interface":"front","average":{"1min":1.677,"5min":1.5489999999999999,"15min":1.585},"min":{"1min":0.34599999999999997,"5min":0.28799999999999998,"15min":0.255},"max":{"1min":8.6509999999999998,"5min":19.859999999999999,"15min":19.859999999999999},"last":1.0609999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790292,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2764840,"kb_used_data":1866208,"kb_used_omap":0,"kb_used_meta":898624,"kb_avail":90980312,"statfs":{"total":95995035648,"available":93163839488,"internally_reserved":0,"allocated":1910996992,"data_stored":1909634186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920190976},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:09 2023","interfaces":[{"interface":"back","average":{"1min":4.931,"5min":2.4180000000000001,"15min":2.1890000000000001},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.215},"max":{"1min":58.271999999999998,"5min":58.271999999999998,"15min":58.271999999999998},"last":0.51000000000000001},{"interface":"front","average":{"1min":5,"5min":2.4689999999999999,"15min":2.2429999999999999},"min":{"1min":0.32600000000000001,"5min":0.30499999999999999,"15min":0.28599999999999998},"max":{"1min":58.25,"5min":58.25,"15min":58.25},"last":0.54000000000000004}]},{"osd":1,"last update":"Mon Oct 9 11:01:17 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.9890000000000001,"15min":1.9890000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.4570000000000007,"5min":9.4570000000000007,"15min":9.4570000000000007},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.6120000000000001,"5min":1.6120000000000001,"15min":1.6120000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.4450000000000003,"5min":9.4450000000000003,"15min":9.4450000000000003},"last":0.60699999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:01:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.758,"15min":2.0379999999999998},"min":{"1min":0.25800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":6.851,"5min":9.3539999999999992,"15min":11.393000000000001},"last":2.9710000000000001},{"interface":"front","average":{"1min":2.04,"5min":1.8200000000000001,"15min":1.954},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":6.7130000000000001,"5min":14.621,"15min":15.087999999999999},"last":2.8879999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:02:09 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":2.5449999999999999,"15min":2.5209999999999999},"min":{"1min":0.29699999999999999,"5min":0.28000000000000003,"15min":0.25700000000000001},"max":{"1min":11.590999999999999,"5min":32.810000000000002,"15min":63.508000000000003},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.8069999999999999,"5min":2.5699999999999998,"15min":2.702},"min":{"1min":0.33900000000000002,"5min":0.30199999999999999,"15min":0.26300000000000001},"max":{"1min":11.292999999999999,"5min":32.659999999999997,"15min":62.418999999999997},"last":3.488}]},{"osd":4,"last update":"Mon Oct 9 11:01:17 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.76800000000000002,"15min":1.4690000000000001},"min":{"1min":0.311,"5min":0.216,"15min":0.216},"max":{"1min":1.5700000000000001,"5min":4.8760000000000003,"15min":123.241},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.2789999999999999,"15min":1.835},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":6.4950000000000001,"5min":9.1869999999999994,"15min":123.318},"last":0.45500000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:01:31 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":1.0249999999999999,"15min":1.4910000000000001},"min":{"1min":0.30299999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.4429999999999996,"5min":8.4260000000000002,"15min":11.098000000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.87,"15min":1.3180000000000001},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.5840000000000001,"5min":5.899,"15min":20.256},"last":0.78900000000000003}]},{"osd":7,"last update":"Mon Oct 9 11:01:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.323,"15min":1.534},"min":{"1min":0.35599999999999998,"5min":0.29299999999999998,"15min":0.28699999999999998},"max":{"1min":3.4870000000000001,"5min":7.6680000000000001,"15min":10.739000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":1.1759999999999999,"15min":1.2989999999999999},"min":{"1min":0.379,"5min":0.315,"15min":0.30099999999999999},"max":{"1min":1.7,"5min":8.1430000000000007,"15min":14.154},"last":0.64700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3068723200,"data_stored":3068723200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1223163904,"data_stored":1223163904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1905721344,"data_stored":1905721344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1900478464,"data_stored":1900478464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2865233920,"data_stored":2865233920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1914568704,"data_stored":1914568704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2225602560,"data_stored":2225602560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":72220672,"data_stored":72220672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:02:24.058 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:02:24.206 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:02:24.263 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:02:24.315 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:24 smithi125 ceph-mon[195507]: from='client.36225 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:02:24.315 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:24 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/191269597' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:02:24.315 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:24 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:02:24.315 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:24 smithi125 ceph-mon[195507]: pgmap v1067: 119 pgs: 119 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T11:02:24.316 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:24 smithi125 ceph-mon[200197]: from='client.36225 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:02:24.316 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:24 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/191269597' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:02:24.316 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:24 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:02:24.316 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:24 smithi125 ceph-mon[200197]: pgmap v1067: 119 pgs: 119 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T11:02:24.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:24 smithi155 ceph-mon[184854]: from='client.36225 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:02:24.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:24 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/191269597' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:02:24.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:24 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:02:24.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:24 smithi155 ceph-mon[184854]: pgmap v1067: 119 pgs: 119 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T11:02:24.588 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:24.593 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T11:02:24.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:24.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:24.695 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:24.755 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:24.791 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:24.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:24.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:24.898 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:24.909 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 66 16 81171 81155 76.8206 58.375 0.0126604 0.0130127 2023-10-09T11:02:24.909 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:24.973 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:25.032 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:02:25.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:25.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:25.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:25.074 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:25.155 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:25.203 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:25.250 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:25.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:25.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:25 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:25 smithi125 ceph-mon[195507]: failed to encode map e432 with expected crc 2023-10-09T11:02:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:25 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:02:25.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:25 smithi125 ceph-mon[195507]: osdmap e432: 8 total, 8 up, 7 in 2023-10-09T11:02:25.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:25 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:25.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:25 smithi125 ceph-mon[200197]: failed to encode map e432 with expected crc 2023-10-09T11:02:25.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:25 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:02:25.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:25 smithi125 ceph-mon[200197]: osdmap e432: 8 total, 8 up, 7 in 2023-10-09T11:02:25.352 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:25.392 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:25.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:25 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:25.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:25 smithi155 ceph-mon[184854]: failed to encode map e432 with expected crc 2023-10-09T11:02:25.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:25 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:02:25.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:25 smithi155 ceph-mon[184854]: osdmap e432: 8 total, 8 up, 7 in 2023-10-09T11:02:25.723 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:25.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:25.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:25.829 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:25.884 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:25.906 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 67 16 82100 82084 76.5407 58.0625 0.0177487 0.0130595 2023-10-09T11:02:25.968 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:25.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:25.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:26.076 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:26.153 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:26.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:26 smithi155 ceph-mon[184854]: 1.1d scrub starts 2023-10-09T11:02:26.199 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:26 smithi155 ceph-mon[184854]: 1.1d scrub ok 2023-10-09T11:02:26.199 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:26 smithi155 ceph-mon[184854]: pgmap v1069: 119 pgs: 119 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 55 MiB/s wr, 887 op/s 2023-10-09T11:02:26.232 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:02:26.232 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:02:26.232 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T11:02:26.232 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T11:02:26.232 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:02:26.279 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:26.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:26.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:26.376 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:26 smithi125 ceph-mon[195507]: 1.1d scrub starts 2023-10-09T11:02:26.376 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:26 smithi125 ceph-mon[195507]: 1.1d scrub ok 2023-10-09T11:02:26.376 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:26 smithi125 ceph-mon[195507]: pgmap v1069: 119 pgs: 119 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 55 MiB/s wr, 887 op/s 2023-10-09T11:02:26.376 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:26 smithi125 ceph-mon[200197]: 1.1d scrub starts 2023-10-09T11:02:26.376 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:26 smithi125 ceph-mon[200197]: 1.1d scrub ok 2023-10-09T11:02:26.376 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:26 smithi125 ceph-mon[200197]: pgmap v1069: 119 pgs: 119 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 55 MiB/s wr, 887 op/s 2023-10-09T11:02:26.389 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:26.424 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:26.683 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:02:26.732 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:02:26.734 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:02:26.749 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:02:26.771 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:02:26.819 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:02:26.831 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:02:26.876 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:02:26.908 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 68 16 82956 82940 76.2019 53.5 0.00445953 0.0131179 2023-10-09T11:02:26.932 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:02:26.966 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:02:26.989 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:26.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:26.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:27.002 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:02:27.002 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:02:27.003 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:02:27.003 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:02:27.003 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:02:27.003 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:02:27.003 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:02:27.003 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:02:27.003 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:02:27.007 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T11:02:27.094 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:27.163 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:27.204 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:27.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:27.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:27.310 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:27.358 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:27.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:27 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3831481019' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:27.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:27 smithi155 ceph-mon[184854]: from='client.46184 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:02:27.470 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:02:27.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:27 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3831481019' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:27.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:27 smithi125 ceph-mon[195507]: from='client.46184 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:02:27.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:27 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3831481019' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:27.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:27 smithi125 ceph-mon[200197]: from='client.46184 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:02:27.906 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 69 16 83939 83923 75.9878 61.4375 0.0115042 0.0131558 2023-10-09T11:02:28.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:28 smithi155 ceph-mon[184854]: 1.1b scrub starts 2023-10-09T11:02:28.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:28 smithi155 ceph-mon[184854]: 1.1b scrub ok 2023-10-09T11:02:28.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:28 smithi155 ceph-mon[184854]: pgmap v1070: 119 pgs: 119 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 49 MiB/s wr, 788 op/s 2023-10-09T11:02:28.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:28 smithi125 ceph-mon[195507]: 1.1b scrub starts 2023-10-09T11:02:28.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:28 smithi125 ceph-mon[195507]: 1.1b scrub ok 2023-10-09T11:02:28.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:28 smithi125 ceph-mon[195507]: pgmap v1070: 119 pgs: 119 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 49 MiB/s wr, 788 op/s 2023-10-09T11:02:28.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:28 smithi125 ceph-mon[200197]: 1.1b scrub starts 2023-10-09T11:02:28.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:28 smithi125 ceph-mon[200197]: 1.1b scrub ok 2023-10-09T11:02:28.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:28 smithi125 ceph-mon[200197]: pgmap v1070: 119 pgs: 119 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 49 MiB/s wr, 788 op/s 2023-10-09T11:02:28.626 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:02:28.626 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:02:28.706 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:28.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:28.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:28.744 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1069,"stamp":"2023-10-09T11:02:25.688520+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5145019990,"num_objects":78521,"num_object_clones":0,"num_object_copies":235563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":78521,"num_whiteouts":0,"num_read":39,"num_read_kb":70,"num_write":78578,"num_write_kb":5024021,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4604,"num_bytes_recovered":298761900,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83543,"ondisk_log_size":83543,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":20516484,"kb_used_data":15417100,"kb_used_omap":130,"kb_used_meta":5099197,"kb_avail":635699580,"statfs":{"total":671965249536,"available":650956369920,"internally_reserved":0,"allocated":15787110400,"data_stored":15774053810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":133799,"internal_metadata":5221578073},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,6],"upper_bound":8},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":465436672,"num_objects":7102,"num_object_clones":0,"num_object_copies":21306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7102,"num_write_kb":454528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002160"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":646,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.241998+0000","last_change":"2023-10-09T11:01:29.156745+0000","last_active":"2023-10-09T11:02:16.241998+0000","last_peered":"2023-10-09T11:02:16.241998+0000","last_clean":"2023-10-09T11:02:16.241998+0000","last_became_active":"2023-10-09T11:01:29.156243+0000","last_became_peered":"2023-10-09T11:01:29.156243+0000","last_unstale":"2023-10-09T11:02:16.241998+0000","last_undegraded":"2023-10-09T11:02:16.241998+0000","last_fullsized":"2023-10-09T11:02:16.241998+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:24.854753+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":154,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.229209+0000","last_change":"2023-10-09T11:00:14.910332+0000","last_active":"2023-10-09T11:02:16.229209+0000","last_peered":"2023-10-09T11:02:16.229209+0000","last_clean":"2023-10-09T11:02:16.229209+0000","last_became_active":"2023-10-09T11:00:14.910108+0000","last_became_peered":"2023-10-09T11:00:14.910108+0000","last_unstale":"2023-10-09T11:02:16.229209+0000","last_undegraded":"2023-10-09T11:02:16.229209+0000","last_fullsized":"2023-10-09T11:02:16.229209+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:30:41.668874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":456,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.236045+0000","last_change":"2023-10-09T11:00:14.870884+0000","last_active":"2023-10-09T11:02:16.236045+0000","last_peered":"2023-10-09T11:02:16.236045+0000","last_clean":"2023-10-09T11:02:16.236045+0000","last_became_active":"2023-10-09T11:00:14.870515+0000","last_became_peered":"2023-10-09T11:00:14.870515+0000","last_unstale":"2023-10-09T11:02:16.236045+0000","last_undegraded":"2023-10-09T11:02:16.236045+0000","last_fullsized":"2023-10-09T11:02:16.236045+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:02.499841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":578,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.243033+0000","last_change":"2023-10-09T11:02:02.395757+0000","last_active":"2023-10-09T11:02:16.243033+0000","last_peered":"2023-10-09T11:02:16.243033+0000","last_clean":"2023-10-09T11:02:16.243033+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:02:16.243033+0000","last_undegraded":"2023-10-09T11:02:16.243033+0000","last_fullsized":"2023-10-09T11:02:16.243033+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.536962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.536962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":109,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.255856+0000","last_change":"2023-10-09T11:00:14.876415+0000","last_active":"2023-10-09T11:02:16.255856+0000","last_peered":"2023-10-09T11:02:16.255856+0000","last_clean":"2023-10-09T11:02:16.255856+0000","last_became_active":"2023-10-09T11:00:14.864861+0000","last_became_peered":"2023-10-09T11:00:14.864861+0000","last_unstale":"2023-10-09T11:02:16.255856+0000","last_undegraded":"2023-10-09T11:02:16.255856+0000","last_fullsized":"2023-10-09T11:02:16.255856+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:59.494851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":364,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.235845+0000","last_change":"2023-10-09T11:01:26.641585+0000","last_active":"2023-10-09T11:02:16.235845+0000","last_peered":"2023-10-09T11:02:16.235845+0000","last_clean":"2023-10-09T11:02:16.235845+0000","last_became_active":"2023-10-09T11:01:26.641278+0000","last_became_peered":"2023-10-09T11:01:26.641278+0000","last_unstale":"2023-10-09T11:02:16.235845+0000","last_undegraded":"2023-10-09T11:02:16.235845+0000","last_fullsized":"2023-10-09T11:02:16.235845+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:44.889126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":891,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.251096+0000","last_change":"2023-10-09T11:01:29.158198+0000","last_active":"2023-10-09T11:02:16.251096+0000","last_peered":"2023-10-09T11:02:16.251096+0000","last_clean":"2023-10-09T11:02:16.251096+0000","last_became_active":"2023-10-09T11:01:29.157984+0000","last_became_peered":"2023-10-09T11:01:29.157984+0000","last_unstale":"2023-10-09T11:02:16.251096+0000","last_undegraded":"2023-10-09T11:02:16.251096+0000","last_fullsized":"2023-10-09T11:02:16.251096+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:39:23.112099+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":522,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232175+0000","last_change":"2023-10-09T11:00:15.889100+0000","last_active":"2023-10-09T11:02:16.232175+0000","last_peered":"2023-10-09T11:02:16.232175+0000","last_clean":"2023-10-09T11:02:16.232175+0000","last_became_active":"2023-10-09T11:00:15.888886+0000","last_became_peered":"2023-10-09T11:00:15.888886+0000","last_unstale":"2023-10-09T11:02:16.232175+0000","last_undegraded":"2023-10-09T11:02:16.232175+0000","last_fullsized":"2023-10-09T11:02:16.232175+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:40.695500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":220,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244624+0000","last_change":"2023-10-09T11:01:15.401167+0000","last_active":"2023-10-09T11:02:16.244624+0000","last_peered":"2023-10-09T11:02:16.244624+0000","last_clean":"2023-10-09T11:02:16.244624+0000","last_became_active":"2023-10-09T11:01:15.400876+0000","last_became_peered":"2023-10-09T11:01:15.400876+0000","last_unstale":"2023-10-09T11:02:16.244624+0000","last_undegraded":"2023-10-09T11:02:16.244624+0000","last_fullsized":"2023-10-09T11:02:16.244624+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:56.342226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":583,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242860+0000","last_change":"2023-10-09T11:02:03.383482+0000","last_active":"2023-10-09T11:02:16.242860+0000","last_peered":"2023-10-09T11:02:16.242860+0000","last_clean":"2023-10-09T11:02:16.242860+0000","last_became_active":"2023-10-09T11:01:29.161949+0000","last_became_peered":"2023-10-09T11:01:29.161949+0000","last_unstale":"2023-10-09T11:02:16.242860+0000","last_undegraded":"2023-10-09T11:02:16.242860+0000","last_fullsized":"2023-10-09T11:02:16.242860+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.787782+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.787782+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":67,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.236066+0000","last_change":"2023-10-09T10:59:48.124873+0000","last_active":"2023-10-09T11:02:16.236066+0000","last_peered":"2023-10-09T11:02:16.236066+0000","last_clean":"2023-10-09T11:02:16.236066+0000","last_became_active":"2023-10-09T10:59:48.124600+0000","last_became_peered":"2023-10-09T10:59:48.124600+0000","last_unstale":"2023-10-09T11:02:16.236066+0000","last_undegraded":"2023-10-09T11:02:16.236066+0000","last_fullsized":"2023-10-09T11:02:16.236066+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:04.818987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":457,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232251+0000","last_change":"2023-10-09T11:00:14.869998+0000","last_active":"2023-10-09T11:02:16.232251+0000","last_peered":"2023-10-09T11:02:16.232251+0000","last_clean":"2023-10-09T11:02:16.232251+0000","last_became_active":"2023-10-09T11:00:14.869671+0000","last_became_peered":"2023-10-09T11:00:14.869671+0000","last_unstale":"2023-10-09T11:02:16.232251+0000","last_undegraded":"2023-10-09T11:02:16.232251+0000","last_fullsized":"2023-10-09T11:02:16.232251+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:19.564770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":741,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.228901+0000","last_change":"2023-10-09T11:01:29.157411+0000","last_active":"2023-10-09T11:02:16.228901+0000","last_peered":"2023-10-09T11:02:16.228901+0000","last_clean":"2023-10-09T11:02:16.228901+0000","last_became_active":"2023-10-09T11:01:29.156986+0000","last_became_peered":"2023-10-09T11:01:29.156986+0000","last_unstale":"2023-10-09T11:02:16.228901+0000","last_undegraded":"2023-10-09T11:02:16.228901+0000","last_fullsized":"2023-10-09T11:02:16.228901+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:13:32.638842+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":272,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244154+0000","last_change":"2023-10-09T11:01:26.637394+0000","last_active":"2023-10-09T11:02:16.244154+0000","last_peered":"2023-10-09T11:02:16.244154+0000","last_clean":"2023-10-09T11:02:16.244154+0000","last_became_active":"2023-10-09T11:01:26.637067+0000","last_became_peered":"2023-10-09T11:01:26.637067+0000","last_unstale":"2023-10-09T11:02:16.244154+0000","last_undegraded":"2023-10-09T11:02:16.244154+0000","last_fullsized":"2023-10-09T11:02:16.244154+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:37.815349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":408,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233805+0000","last_change":"2023-10-09T11:01:26.643175+0000","last_active":"2023-10-09T11:02:16.233805+0000","last_peered":"2023-10-09T11:02:16.233805+0000","last_clean":"2023-10-09T11:02:16.233805+0000","last_became_active":"2023-10-09T11:01:26.642855+0000","last_became_peered":"2023-10-09T11:01:26.642855+0000","last_unstale":"2023-10-09T11:02:16.233805+0000","last_undegraded":"2023-10-09T11:02:16.233805+0000","last_fullsized":"2023-10-09T11:02:16.233805+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:02.173276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":620,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244386+0000","last_change":"2023-10-09T11:01:16.673177+0000","last_active":"2023-10-09T11:02:16.244386+0000","last_peered":"2023-10-09T11:02:16.244386+0000","last_clean":"2023-10-09T11:02:16.244386+0000","last_became_active":"2023-10-09T11:01:15.401411+0000","last_became_peered":"2023-10-09T11:01:15.401411+0000","last_unstale":"2023-10-09T11:02:16.244386+0000","last_undegraded":"2023-10-09T11:02:16.244386+0000","last_fullsized":"2023-10-09T11:02:16.244386+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:46.989699+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":325,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.236077+0000","last_change":"2023-10-09T11:01:26.696000+0000","last_active":"2023-10-09T11:02:16.236077+0000","last_peered":"2023-10-09T11:02:16.236077+0000","last_clean":"2023-10-09T11:02:16.236077+0000","last_became_active":"2023-10-09T11:01:26.642917+0000","last_became_peered":"2023-10-09T11:01:26.642917+0000","last_unstale":"2023-10-09T11:02:16.236077+0000","last_undegraded":"2023-10-09T11:02:16.236077+0000","last_fullsized":"2023-10-09T11:02:16.236077+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:35.375183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":189,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242906+0000","last_change":"2023-10-09T11:01:15.401892+0000","last_active":"2023-10-09T11:02:16.242906+0000","last_peered":"2023-10-09T11:02:16.242906+0000","last_clean":"2023-10-09T11:02:16.242906+0000","last_became_active":"2023-10-09T11:01:15.401025+0000","last_became_peered":"2023-10-09T11:01:15.401025+0000","last_unstale":"2023-10-09T11:02:16.242906+0000","last_undegraded":"2023-10-09T11:02:16.242906+0000","last_fullsized":"2023-10-09T11:02:16.242906+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:48:03.531920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":780,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.260332+0000","last_change":"2023-10-09T11:02:09.381602+0000","last_active":"2023-10-09T11:02:16.260332+0000","last_peered":"2023-10-09T11:02:16.260332+0000","last_clean":"2023-10-09T11:02:16.260332+0000","last_became_active":"2023-10-09T11:00:16.068155+0000","last_became_peered":"2023-10-09T11:00:16.068155+0000","last_unstale":"2023-10-09T11:02:16.260332+0000","last_undegraded":"2023-10-09T11:02:16.260332+0000","last_fullsized":"2023-10-09T11:02:16.260332+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:56.972770+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":43,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.228732+0000","last_change":"2023-10-09T11:01:26.693517+0000","last_active":"2023-10-09T11:02:16.228732+0000","last_peered":"2023-10-09T11:02:16.228732+0000","last_clean":"2023-10-09T11:02:16.228732+0000","last_became_active":"2023-10-09T11:01:26.648334+0000","last_became_peered":"2023-10-09T11:01:26.648334+0000","last_unstale":"2023-10-09T11:02:16.228732+0000","last_undegraded":"2023-10-09T11:02:16.228732+0000","last_fullsized":"2023-10-09T11:02:16.228732+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:18.451728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":445,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.248377+0000","last_change":"2023-10-09T10:57:46.845755+0000","last_active":"2023-10-09T11:02:16.248377+0000","last_peered":"2023-10-09T11:02:16.248377+0000","last_clean":"2023-10-09T11:02:16.248377+0000","last_became_active":"2023-10-09T10:57:46.845475+0000","last_became_peered":"2023-10-09T10:57:46.845475+0000","last_unstale":"2023-10-09T11:02:16.248377+0000","last_undegraded":"2023-10-09T11:02:16.248377+0000","last_fullsized":"2023-10-09T11:02:16.248377+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:21.673994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":794,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:18.270337+0000","last_change":"2023-10-09T10:59:49.175255+0000","last_active":"2023-10-09T11:02:18.270337+0000","last_peered":"2023-10-09T11:02:18.270337+0000","last_clean":"2023-10-09T11:02:18.270337+0000","last_became_active":"2023-10-09T10:59:49.174666+0000","last_became_peered":"2023-10-09T10:59:49.174666+0000","last_unstale":"2023-10-09T11:02:18.270337+0000","last_undegraded":"2023-10-09T11:02:18.270337+0000","last_fullsized":"2023-10-09T11:02:18.270337+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:00:58.069275+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":100,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.229148+0000","last_change":"2023-10-09T11:01:26.673987+0000","last_active":"2023-10-09T11:02:16.229148+0000","last_peered":"2023-10-09T11:02:16.229148+0000","last_clean":"2023-10-09T11:02:16.229148+0000","last_became_active":"2023-10-09T11:01:26.644907+0000","last_became_peered":"2023-10-09T11:01:26.644907+0000","last_unstale":"2023-10-09T11:02:16.229148+0000","last_undegraded":"2023-10-09T11:02:16.229148+0000","last_fullsized":"2023-10-09T11:02:16.229148+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:42:13.828945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":534,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232977+0000","last_change":"2023-10-09T11:01:26.635356+0000","last_active":"2023-10-09T11:02:16.232977+0000","last_peered":"2023-10-09T11:02:16.232977+0000","last_clean":"2023-10-09T11:02:16.232977+0000","last_became_active":"2023-10-09T11:01:26.634709+0000","last_became_peered":"2023-10-09T11:01:26.634709+0000","last_unstale":"2023-10-09T11:02:16.232977+0000","last_undegraded":"2023-10-09T11:02:16.232977+0000","last_fullsized":"2023-10-09T11:02:16.232977+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:38.908158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":682,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242844+0000","last_change":"2023-10-09T11:01:29.161463+0000","last_active":"2023-10-09T11:02:16.242844+0000","last_peered":"2023-10-09T11:02:16.242844+0000","last_clean":"2023-10-09T11:02:16.242844+0000","last_became_active":"2023-10-09T11:01:29.161068+0000","last_became_peered":"2023-10-09T11:01:29.161068+0000","last_unstale":"2023-10-09T11:02:16.242844+0000","last_undegraded":"2023-10-09T11:02:16.242844+0000","last_fullsized":"2023-10-09T11:02:16.242844+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:55.586383+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":363,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:18.264691+0000","last_change":"2023-10-09T11:01:26.644626+0000","last_active":"2023-10-09T11:02:18.264691+0000","last_peered":"2023-10-09T11:02:18.264691+0000","last_clean":"2023-10-09T11:02:18.264691+0000","last_became_active":"2023-10-09T11:01:26.644324+0000","last_became_peered":"2023-10-09T11:01:26.644324+0000","last_unstale":"2023-10-09T11:02:18.264691+0000","last_undegraded":"2023-10-09T11:02:18.264691+0000","last_fullsized":"2023-10-09T11:02:18.264691+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:51:47.358468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":305,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242721+0000","last_change":"2023-10-09T11:01:26.653786+0000","last_active":"2023-10-09T11:02:16.242721+0000","last_peered":"2023-10-09T11:02:16.242721+0000","last_clean":"2023-10-09T11:02:16.242721+0000","last_became_active":"2023-10-09T11:01:26.652916+0000","last_became_peered":"2023-10-09T11:01:26.652916+0000","last_unstale":"2023-10-09T11:02:16.242721+0000","last_undegraded":"2023-10-09T11:02:16.242721+0000","last_fullsized":"2023-10-09T11:02:16.242721+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:09.061877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":722,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.259352+0000","last_change":"2023-10-09T11:01:25.620261+0000","last_active":"2023-10-09T11:02:16.259352+0000","last_peered":"2023-10-09T11:02:16.259352+0000","last_clean":"2023-10-09T11:02:16.259352+0000","last_became_active":"2023-10-09T11:00:14.871258+0000","last_became_peered":"2023-10-09T11:00:14.871258+0000","last_unstale":"2023-10-09T11:02:16.259352+0000","last_undegraded":"2023-10-09T11:02:16.259352+0000","last_fullsized":"2023-10-09T11:02:16.259352+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:13.861006+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":291,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.250931+0000","last_change":"2023-10-09T11:01:26.650498+0000","last_active":"2023-10-09T11:02:16.250931+0000","last_peered":"2023-10-09T11:02:16.250931+0000","last_clean":"2023-10-09T11:02:16.250931+0000","last_became_active":"2023-10-09T11:01:26.650153+0000","last_became_peered":"2023-10-09T11:01:26.650153+0000","last_unstale":"2023-10-09T11:02:16.250931+0000","last_undegraded":"2023-10-09T11:02:16.250931+0000","last_fullsized":"2023-10-09T11:02:16.250931+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:06.419869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":76,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.235634+0000","last_change":"2023-10-09T11:01:26.640215+0000","last_active":"2023-10-09T11:02:16.235634+0000","last_peered":"2023-10-09T11:02:16.235634+0000","last_clean":"2023-10-09T11:02:16.235634+0000","last_became_active":"2023-10-09T11:01:26.639896+0000","last_became_peered":"2023-10-09T11:01:26.639896+0000","last_unstale":"2023-10-09T11:02:16.235634+0000","last_undegraded":"2023-10-09T11:02:16.235634+0000","last_fullsized":"2023-10-09T11:02:16.235634+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:37.209505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":699,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244748+0000","last_change":"2023-10-09T11:01:15.406928+0000","last_active":"2023-10-09T11:02:16.244748+0000","last_peered":"2023-10-09T11:02:16.244748+0000","last_clean":"2023-10-09T11:02:16.244748+0000","last_became_active":"2023-10-09T11:01:15.406499+0000","last_became_peered":"2023-10-09T11:01:15.406499+0000","last_unstale":"2023-10-09T11:02:16.244748+0000","last_undegraded":"2023-10-09T11:02:16.244748+0000","last_fullsized":"2023-10-09T11:02:16.244748+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:33.050336+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":158,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.239020+0000","last_change":"2023-10-09T11:00:48.109643+0000","last_active":"2023-10-09T11:02:16.239020+0000","last_peered":"2023-10-09T11:02:16.239020+0000","last_clean":"2023-10-09T11:02:16.239020+0000","last_became_active":"2023-10-09T11:00:48.109299+0000","last_became_peered":"2023-10-09T11:00:48.109299+0000","last_unstale":"2023-10-09T11:02:16.239020+0000","last_undegraded":"2023-10-09T11:02:16.239020+0000","last_fullsized":"2023-10-09T11:02:16.239020+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:31.163237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":510,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233997+0000","last_change":"2023-10-09T11:01:26.637554+0000","last_active":"2023-10-09T11:02:16.233997+0000","last_peered":"2023-10-09T11:02:16.233997+0000","last_clean":"2023-10-09T11:02:16.233997+0000","last_became_active":"2023-10-09T11:01:26.637117+0000","last_became_peered":"2023-10-09T11:01:26.637117+0000","last_unstale":"2023-10-09T11:02:16.233997+0000","last_undegraded":"2023-10-09T11:02:16.233997+0000","last_fullsized":"2023-10-09T11:02:16.233997+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:02.104893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"431'5010","reported_seq":5046,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.576569+0000","last_change":"2023-10-09T11:01:15.402898+0000","last_active":"2023-10-09T11:02:21.576569+0000","last_peered":"2023-10-09T11:02:21.576569+0000","last_clean":"2023-10-09T11:02:21.576569+0000","last_became_active":"2023-10-09T11:01:15.402646+0000","last_became_peered":"2023-10-09T11:01:15.402646+0000","last_unstale":"2023-10-09T11:02:21.576569+0000","last_undegraded":"2023-10-09T11:02:21.576569+0000","last_fullsized":"2023-10-09T11:02:21.576569+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5010,"log_dups_size":0,"ondisk_log_size":5010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:42:07.397321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328335360,"num_objects":5010,"num_object_clones":0,"num_object_copies":15030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5010,"num_write_kb":320640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":710,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.260047+0000","last_change":"2023-10-09T11:01:29.163248+0000","last_active":"2023-10-09T11:02:16.260047+0000","last_peered":"2023-10-09T11:02:16.260047+0000","last_clean":"2023-10-09T11:02:16.260047+0000","last_became_active":"2023-10-09T11:01:29.162829+0000","last_became_peered":"2023-10-09T11:01:29.162829+0000","last_unstale":"2023-10-09T11:02:16.260047+0000","last_undegraded":"2023-10-09T11:02:16.260047+0000","last_fullsized":"2023-10-09T11:02:16.260047+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:43:25.947651+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":183,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.248297+0000","last_change":"2023-10-09T11:01:26.684538+0000","last_active":"2023-10-09T11:02:16.248297+0000","last_peered":"2023-10-09T11:02:16.248297+0000","last_clean":"2023-10-09T11:02:16.248297+0000","last_became_active":"2023-10-09T11:01:26.649495+0000","last_became_peered":"2023-10-09T11:01:26.649495+0000","last_unstale":"2023-10-09T11:02:16.248297+0000","last_undegraded":"2023-10-09T11:02:16.248297+0000","last_fullsized":"2023-10-09T11:02:16.248297+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:25.524018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":343,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242080+0000","last_change":"2023-10-09T11:01:26.653478+0000","last_active":"2023-10-09T11:02:16.242080+0000","last_peered":"2023-10-09T11:02:16.242080+0000","last_clean":"2023-10-09T11:02:16.242080+0000","last_became_active":"2023-10-09T11:01:26.653181+0000","last_became_peered":"2023-10-09T11:01:26.653181+0000","last_unstale":"2023-10-09T11:02:16.242080+0000","last_undegraded":"2023-10-09T11:02:16.242080+0000","last_fullsized":"2023-10-09T11:02:16.242080+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:28.010266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"431'4787","reported_seq":4824,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:20.833231+0000","last_change":"2023-10-09T11:01:15.394927+0000","last_active":"2023-10-09T11:02:20.833231+0000","last_peered":"2023-10-09T11:02:20.833231+0000","last_clean":"2023-10-09T11:02:20.833231+0000","last_became_active":"2023-10-09T11:01:15.394658+0000","last_became_peered":"2023-10-09T11:01:15.394658+0000","last_unstale":"2023-10-09T11:02:20.833231+0000","last_undegraded":"2023-10-09T11:02:20.833231+0000","last_fullsized":"2023-10-09T11:02:20.833231+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4787,"log_dups_size":0,"ondisk_log_size":4787,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:58.565353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313720832,"num_objects":4787,"num_object_clones":0,"num_object_copies":14361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4787,"num_write_kb":306368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":835,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232838+0000","last_change":"2023-10-09T10:59:51.140014+0000","last_active":"2023-10-09T11:02:16.232838+0000","last_peered":"2023-10-09T11:02:16.232838+0000","last_clean":"2023-10-09T11:02:16.232838+0000","last_became_active":"2023-10-09T10:59:51.139665+0000","last_became_peered":"2023-10-09T10:59:51.139665+0000","last_unstale":"2023-10-09T11:02:16.232838+0000","last_undegraded":"2023-10-09T11:02:16.232838+0000","last_fullsized":"2023-10-09T11:02:16.232838+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:29.505223+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":117,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.259893+0000","last_change":"2023-10-09T11:01:26.680401+0000","last_active":"2023-10-09T11:02:16.259893+0000","last_peered":"2023-10-09T11:02:16.259893+0000","last_clean":"2023-10-09T11:02:16.259893+0000","last_became_active":"2023-10-09T11:01:26.647295+0000","last_became_peered":"2023-10-09T11:01:26.647295+0000","last_unstale":"2023-10-09T11:02:16.259893+0000","last_undegraded":"2023-10-09T11:02:16.259893+0000","last_fullsized":"2023-10-09T11:02:16.259893+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:28.731546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":110,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:18.277909+0000","last_change":"2023-10-09T11:00:14.872673+0000","last_active":"2023-10-09T11:02:18.277909+0000","last_peered":"2023-10-09T11:02:18.277909+0000","last_clean":"2023-10-09T11:02:18.277909+0000","last_became_active":"2023-10-09T11:00:14.872345+0000","last_became_peered":"2023-10-09T11:00:14.872345+0000","last_unstale":"2023-10-09T11:02:18.277909+0000","last_undegraded":"2023-10-09T11:02:18.277909+0000","last_fullsized":"2023-10-09T11:02:18.277909+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:32:43.821085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"431'4923","reported_seq":4958,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:22.978328+0000","last_change":"2023-10-09T11:01:15.402608+0000","last_active":"2023-10-09T11:02:22.978328+0000","last_peered":"2023-10-09T11:02:22.978328+0000","last_clean":"2023-10-09T11:02:22.978328+0000","last_became_active":"2023-10-09T11:01:15.402329+0000","last_became_peered":"2023-10-09T11:01:15.402329+0000","last_unstale":"2023-10-09T11:02:22.978328+0000","last_undegraded":"2023-10-09T11:02:22.978328+0000","last_fullsized":"2023-10-09T11:02:22.978328+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4923,"log_dups_size":0,"ondisk_log_size":4923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:31:16.567673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322633728,"num_objects":4923,"num_object_clones":0,"num_object_copies":14769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4923,"num_write_kb":315072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":627,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244473+0000","last_change":"2023-10-09T11:01:22.599166+0000","last_active":"2023-10-09T11:02:16.244473+0000","last_peered":"2023-10-09T11:02:16.244473+0000","last_clean":"2023-10-09T11:02:16.244473+0000","last_became_active":"2023-10-09T11:01:15.402193+0000","last_became_peered":"2023-10-09T11:01:15.402193+0000","last_unstale":"2023-10-09T11:02:16.244473+0000","last_undegraded":"2023-10-09T11:02:16.244473+0000","last_fullsized":"2023-10-09T11:02:16.244473+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:37:16.611219+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":211,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242810+0000","last_change":"2023-10-09T11:01:26.684793+0000","last_active":"2023-10-09T11:02:16.242810+0000","last_peered":"2023-10-09T11:02:16.242810+0000","last_clean":"2023-10-09T11:02:16.242810+0000","last_became_active":"2023-10-09T11:01:26.653430+0000","last_became_peered":"2023-10-09T11:01:26.653430+0000","last_unstale":"2023-10-09T11:02:16.242810+0000","last_undegraded":"2023-10-09T11:02:16.242810+0000","last_fullsized":"2023-10-09T11:02:16.242810+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:46:56.385204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":270,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.239311+0000","last_change":"2023-10-09T11:01:26.658802+0000","last_active":"2023-10-09T11:02:16.239311+0000","last_peered":"2023-10-09T11:02:16.239311+0000","last_clean":"2023-10-09T11:02:16.239311+0000","last_became_active":"2023-10-09T11:01:26.658477+0000","last_became_peered":"2023-10-09T11:01:26.658477+0000","last_unstale":"2023-10-09T11:02:16.239311+0000","last_undegraded":"2023-10-09T11:02:16.239311+0000","last_fullsized":"2023-10-09T11:02:16.239311+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:52.018229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"431'4895","reported_seq":4932,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.040434+0000","last_change":"2023-10-09T11:01:15.393071+0000","last_active":"2023-10-09T11:02:21.040434+0000","last_peered":"2023-10-09T11:02:21.040434+0000","last_clean":"2023-10-09T11:02:21.040434+0000","last_became_active":"2023-10-09T11:01:15.392793+0000","last_became_peered":"2023-10-09T11:01:15.392793+0000","last_unstale":"2023-10-09T11:02:21.040434+0000","last_undegraded":"2023-10-09T11:02:21.040434+0000","last_fullsized":"2023-10-09T11:02:21.040434+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4895,"log_dups_size":0,"ondisk_log_size":4895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:27.128762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320798720,"num_objects":4895,"num_object_clones":0,"num_object_copies":14685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4895,"num_write_kb":313280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":936,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.251020+0000","last_change":"2023-10-09T11:01:29.173445+0000","last_active":"2023-10-09T11:02:16.251020+0000","last_peered":"2023-10-09T11:02:16.251020+0000","last_clean":"2023-10-09T11:02:16.251020+0000","last_became_active":"2023-10-09T11:01:29.173101+0000","last_became_peered":"2023-10-09T11:01:29.173101+0000","last_unstale":"2023-10-09T11:02:16.251020+0000","last_undegraded":"2023-10-09T11:02:16.251020+0000","last_fullsized":"2023-10-09T11:02:16.251020+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:50.742268+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":242,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242276+0000","last_change":"2023-10-09T11:01:15.400732+0000","last_active":"2023-10-09T11:02:16.242276+0000","last_peered":"2023-10-09T11:02:16.242276+0000","last_clean":"2023-10-09T11:02:16.242276+0000","last_became_active":"2023-10-09T11:01:15.399370+0000","last_became_peered":"2023-10-09T11:01:15.399370+0000","last_unstale":"2023-10-09T11:02:16.242276+0000","last_undegraded":"2023-10-09T11:02:16.242276+0000","last_fullsized":"2023-10-09T11:02:16.242276+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:53.948144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":352,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.235028+0000","last_change":"2023-10-09T11:00:15.865848+0000","last_active":"2023-10-09T11:02:16.235028+0000","last_peered":"2023-10-09T11:02:16.235028+0000","last_clean":"2023-10-09T11:02:16.235028+0000","last_became_active":"2023-10-09T11:00:15.865504+0000","last_became_peered":"2023-10-09T11:00:15.865504+0000","last_unstale":"2023-10-09T11:02:16.235028+0000","last_undegraded":"2023-10-09T11:02:16.235028+0000","last_fullsized":"2023-10-09T11:02:16.235028+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:32.352564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"431'4961","reported_seq":4998,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:22.943503+0000","last_change":"2023-10-09T11:01:15.401760+0000","last_active":"2023-10-09T11:02:22.943503+0000","last_peered":"2023-10-09T11:02:22.943503+0000","last_clean":"2023-10-09T11:02:22.943503+0000","last_became_active":"2023-10-09T11:01:15.401510+0000","last_became_peered":"2023-10-09T11:01:15.401510+0000","last_unstale":"2023-10-09T11:02:22.943503+0000","last_undegraded":"2023-10-09T11:02:22.943503+0000","last_fullsized":"2023-10-09T11:02:22.943503+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4961,"log_dups_size":0,"ondisk_log_size":4961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:21.262441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325124096,"num_objects":4961,"num_object_clones":0,"num_object_copies":14883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4961,"num_write_kb":317504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":783,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233526+0000","last_change":"2023-10-09T11:01:29.165873+0000","last_active":"2023-10-09T11:02:16.233526+0000","last_peered":"2023-10-09T11:02:16.233526+0000","last_clean":"2023-10-09T11:02:16.233526+0000","last_became_active":"2023-10-09T11:01:29.165597+0000","last_became_peered":"2023-10-09T11:01:29.165597+0000","last_unstale":"2023-10-09T11:02:16.233526+0000","last_undegraded":"2023-10-09T11:02:16.233526+0000","last_fullsized":"2023-10-09T11:02:16.233526+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:54.234048+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":450,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232382+0000","last_change":"2023-10-09T10:59:49.127924+0000","last_active":"2023-10-09T11:02:16.232382+0000","last_peered":"2023-10-09T11:02:16.232382+0000","last_clean":"2023-10-09T11:02:16.232382+0000","last_became_active":"2023-10-09T10:59:49.127549+0000","last_became_peered":"2023-10-09T10:59:49.127549+0000","last_unstale":"2023-10-09T11:02:16.232382+0000","last_undegraded":"2023-10-09T11:02:16.232382+0000","last_fullsized":"2023-10-09T11:02:16.232382+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:01.781131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":275,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233129+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T11:02:16.233129+0000","last_peered":"2023-10-09T11:02:16.233129+0000","last_clean":"2023-10-09T11:02:16.233129+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T11:02:16.233129+0000","last_undegraded":"2023-10-09T11:02:16.233129+0000","last_fullsized":"2023-10-09T11:02:16.233129+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"431'4805","reported_seq":4842,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.068353+0000","last_change":"2023-10-09T11:01:15.394098+0000","last_active":"2023-10-09T11:02:21.068353+0000","last_peered":"2023-10-09T11:02:21.068353+0000","last_clean":"2023-10-09T11:02:21.068353+0000","last_became_active":"2023-10-09T11:01:15.393709+0000","last_became_peered":"2023-10-09T11:01:15.393709+0000","last_unstale":"2023-10-09T11:02:21.068353+0000","last_undegraded":"2023-10-09T11:02:21.068353+0000","last_fullsized":"2023-10-09T11:02:21.068353+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4805,"log_dups_size":0,"ondisk_log_size":4805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:08.421072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314900480,"num_objects":4805,"num_object_clones":0,"num_object_copies":14415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4805,"num_write_kb":307520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":622,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242743+0000","last_change":"2023-10-09T11:01:20.667499+0000","last_active":"2023-10-09T11:02:16.242743+0000","last_peered":"2023-10-09T11:02:16.242743+0000","last_clean":"2023-10-09T11:02:16.242743+0000","last_became_active":"2023-10-09T11:01:15.399808+0000","last_became_peered":"2023-10-09T11:01:15.399808+0000","last_unstale":"2023-10-09T11:02:16.242743+0000","last_undegraded":"2023-10-09T11:02:16.242743+0000","last_fullsized":"2023-10-09T11:02:16.242743+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:53:29.298692+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":427,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.251241+0000","last_change":"2023-10-09T11:00:14.891176+0000","last_active":"2023-10-09T11:02:16.251241+0000","last_peered":"2023-10-09T11:02:16.251241+0000","last_clean":"2023-10-09T11:02:16.251241+0000","last_became_active":"2023-10-09T11:00:14.890890+0000","last_became_peered":"2023-10-09T11:00:14.890890+0000","last_unstale":"2023-10-09T11:02:16.251241+0000","last_undegraded":"2023-10-09T11:02:16.251241+0000","last_fullsized":"2023-10-09T11:02:16.251241+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:38.158759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":187,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232388+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:02:16.232388+0000","last_peered":"2023-10-09T11:02:16.232388+0000","last_clean":"2023-10-09T11:02:16.232388+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:02:16.232388+0000","last_undegraded":"2023-10-09T11:02:16.232388+0000","last_fullsized":"2023-10-09T11:02:16.232388+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"431'4798","reported_seq":4834,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.456754+0000","last_change":"2023-10-09T11:01:15.406742+0000","last_active":"2023-10-09T11:02:21.456754+0000","last_peered":"2023-10-09T11:02:21.456754+0000","last_clean":"2023-10-09T11:02:21.456754+0000","last_became_active":"2023-10-09T11:01:15.406429+0000","last_became_peered":"2023-10-09T11:01:15.406429+0000","last_unstale":"2023-10-09T11:02:21.456754+0000","last_undegraded":"2023-10-09T11:02:21.456754+0000","last_fullsized":"2023-10-09T11:02:21.456754+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4798,"log_dups_size":0,"ondisk_log_size":4798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:44:05.691874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314441728,"num_objects":4798,"num_object_clones":0,"num_object_copies":14394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4798,"num_write_kb":307072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":779,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.259439+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:02:16.259439+0000","last_peered":"2023-10-09T11:02:16.259439+0000","last_clean":"2023-10-09T11:02:16.259439+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:02:16.259439+0000","last_undegraded":"2023-10-09T11:02:16.259439+0000","last_fullsized":"2023-10-09T11:02:16.259439+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":496,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.251522+0000","last_change":"2023-10-09T11:01:26.641439+0000","last_active":"2023-10-09T11:02:16.251522+0000","last_peered":"2023-10-09T11:02:16.251522+0000","last_clean":"2023-10-09T11:02:16.251522+0000","last_became_active":"2023-10-09T11:01:26.641109+0000","last_became_peered":"2023-10-09T11:01:26.641109+0000","last_unstale":"2023-10-09T11:02:16.251522+0000","last_undegraded":"2023-10-09T11:02:16.251522+0000","last_fullsized":"2023-10-09T11:02:16.251522+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:59:59.718601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":109,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.260024+0000","last_change":"2023-10-09T11:00:14.900130+0000","last_active":"2023-10-09T11:02:16.260024+0000","last_peered":"2023-10-09T11:02:16.260024+0000","last_clean":"2023-10-09T11:02:16.260024+0000","last_became_active":"2023-10-09T11:00:14.899536+0000","last_became_peered":"2023-10-09T11:00:14.899536+0000","last_unstale":"2023-10-09T11:02:16.260024+0000","last_undegraded":"2023-10-09T11:02:16.260024+0000","last_fullsized":"2023-10-09T11:02:16.260024+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:07.151510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"431'5016","reported_seq":7338,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:22.944721+0000","last_change":"2023-10-09T11:02:08.575348+0000","last_active":"2023-10-09T11:02:22.944721+0000","last_peered":"2023-10-09T11:02:22.944721+0000","last_clean":"2023-10-09T11:02:22.944721+0000","last_became_active":"2023-10-09T11:02:08.572298+0000","last_became_peered":"2023-10-09T11:02:08.572298+0000","last_unstale":"2023-10-09T11:02:22.944721+0000","last_undegraded":"2023-10-09T11:02:22.944721+0000","last_fullsized":"2023-10-09T11:02:22.944721+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5016,"log_dups_size":0,"ondisk_log_size":5016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:02:03.819617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328728576,"num_objects":5016,"num_object_clones":0,"num_object_copies":15048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5016,"num_write_kb":321024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1135,"num_bytes_recovered":74383360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":633,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244616+0000","last_change":"2023-10-09T11:01:51.463338+0000","last_active":"2023-10-09T11:02:16.244616+0000","last_peered":"2023-10-09T11:02:16.244616+0000","last_clean":"2023-10-09T11:02:16.244616+0000","last_became_active":"2023-10-09T11:01:15.399431+0000","last_became_peered":"2023-10-09T11:01:15.399431+0000","last_unstale":"2023-10-09T11:02:16.244616+0000","last_undegraded":"2023-10-09T11:02:16.244616+0000","last_fullsized":"2023-10-09T11:02:16.244616+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:01:07.923413+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":503,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232075+0000","last_change":"2023-10-09T11:01:26.635487+0000","last_active":"2023-10-09T11:02:16.232075+0000","last_peered":"2023-10-09T11:02:16.232075+0000","last_clean":"2023-10-09T11:02:16.232075+0000","last_became_active":"2023-10-09T11:01:26.635092+0000","last_became_peered":"2023-10-09T11:01:26.635092+0000","last_unstale":"2023-10-09T11:02:16.232075+0000","last_undegraded":"2023-10-09T11:02:16.232075+0000","last_fullsized":"2023-10-09T11:02:16.232075+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:17.902346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":109,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.260608+0000","last_change":"2023-10-09T11:00:14.917869+0000","last_active":"2023-10-09T11:02:16.260608+0000","last_peered":"2023-10-09T11:02:16.260608+0000","last_clean":"2023-10-09T11:02:16.260608+0000","last_became_active":"2023-10-09T11:00:14.917535+0000","last_became_peered":"2023-10-09T11:00:14.917535+0000","last_unstale":"2023-10-09T11:02:16.260608+0000","last_undegraded":"2023-10-09T11:02:16.260608+0000","last_fullsized":"2023-10-09T11:02:16.260608+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:10.223866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"431'4973","reported_seq":7294,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:22.978426+0000","last_change":"2023-10-09T11:01:55.237850+0000","last_active":"2023-10-09T11:02:22.978426+0000","last_peered":"2023-10-09T11:02:22.978426+0000","last_clean":"2023-10-09T11:02:22.978426+0000","last_became_active":"2023-10-09T11:01:55.234587+0000","last_became_peered":"2023-10-09T11:01:55.234587+0000","last_unstale":"2023-10-09T11:02:22.978426+0000","last_undegraded":"2023-10-09T11:02:22.978426+0000","last_fullsized":"2023-10-09T11:02:22.978426+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4973,"log_dups_size":0,"ondisk_log_size":4973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:25.848462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325910528,"num_objects":4973,"num_object_clones":0,"num_object_copies":14919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4973,"num_write_kb":318272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1135,"num_bytes_recovered":74383360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":882,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233709+0000","last_change":"2023-10-09T11:01:29.171877+0000","last_active":"2023-10-09T11:02:16.233709+0000","last_peered":"2023-10-09T11:02:16.233709+0000","last_clean":"2023-10-09T11:02:16.233709+0000","last_became_active":"2023-10-09T11:01:29.171560+0000","last_became_peered":"2023-10-09T11:01:29.171560+0000","last_unstale":"2023-10-09T11:02:16.233709+0000","last_undegraded":"2023-10-09T11:02:16.233709+0000","last_fullsized":"2023-10-09T11:02:16.233709+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:00:08.554345+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":131,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:18.019616+0000","last_change":"2023-10-09T11:01:26.641036+0000","last_active":"2023-10-09T11:02:18.019616+0000","last_peered":"2023-10-09T11:02:18.019616+0000","last_clean":"2023-10-09T11:02:18.019616+0000","last_became_active":"2023-10-09T11:01:26.640698+0000","last_became_peered":"2023-10-09T11:01:26.640698+0000","last_unstale":"2023-10-09T11:02:18.019616+0000","last_undegraded":"2023-10-09T11:02:18.019616+0000","last_fullsized":"2023-10-09T11:02:18.019616+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:53.546628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":269,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242593+0000","last_change":"2023-10-09T11:01:15.399780+0000","last_active":"2023-10-09T11:02:16.242593+0000","last_peered":"2023-10-09T11:02:16.242593+0000","last_clean":"2023-10-09T11:02:16.242593+0000","last_became_active":"2023-10-09T11:01:15.399493+0000","last_became_peered":"2023-10-09T11:01:15.399493+0000","last_unstale":"2023-10-09T11:02:16.242593+0000","last_undegraded":"2023-10-09T11:02:16.242593+0000","last_fullsized":"2023-10-09T11:02:16.242593+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:41.639383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":110,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:18.019522+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:02:18.019522+0000","last_peered":"2023-10-09T11:02:18.019522+0000","last_clean":"2023-10-09T11:02:18.019522+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:02:18.019522+0000","last_undegraded":"2023-10-09T11:02:18.019522+0000","last_fullsized":"2023-10-09T11:02:18.019522+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"431'4841","reported_seq":4877,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.446358+0000","last_change":"2023-10-09T11:01:15.406192+0000","last_active":"2023-10-09T11:02:21.446358+0000","last_peered":"2023-10-09T11:02:21.446358+0000","last_clean":"2023-10-09T11:02:21.446358+0000","last_became_active":"2023-10-09T11:01:15.405811+0000","last_became_peered":"2023-10-09T11:01:15.405811+0000","last_unstale":"2023-10-09T11:02:21.446358+0000","last_undegraded":"2023-10-09T11:02:21.446358+0000","last_fullsized":"2023-10-09T11:02:21.446358+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4841,"log_dups_size":0,"ondisk_log_size":4841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:48:38.805496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317259776,"num_objects":4841,"num_object_clones":0,"num_object_copies":14523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4841,"num_write_kb":309824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":631,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244257+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:02:16.244257+0000","last_peered":"2023-10-09T11:02:16.244257+0000","last_clean":"2023-10-09T11:02:16.244257+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:02:16.244257+0000","last_undegraded":"2023-10-09T11:02:16.244257+0000","last_fullsized":"2023-10-09T11:02:16.244257+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":293,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244400+0000","last_change":"2023-10-09T11:01:15.401697+0000","last_active":"2023-10-09T11:02:16.244400+0000","last_peered":"2023-10-09T11:02:16.244400+0000","last_clean":"2023-10-09T11:02:16.244400+0000","last_became_active":"2023-10-09T11:01:15.401386+0000","last_became_peered":"2023-10-09T11:01:15.401386+0000","last_unstale":"2023-10-09T11:02:16.244400+0000","last_undegraded":"2023-10-09T11:02:16.244400+0000","last_fullsized":"2023-10-09T11:02:16.244400+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:40.572070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":337,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.234760+0000","last_change":"2023-10-09T10:59:49.126425+0000","last_active":"2023-10-09T11:02:16.234760+0000","last_peered":"2023-10-09T11:02:16.234760+0000","last_clean":"2023-10-09T11:02:16.234760+0000","last_became_active":"2023-10-09T10:59:49.125712+0000","last_became_peered":"2023-10-09T10:59:49.125712+0000","last_unstale":"2023-10-09T11:02:16.234760+0000","last_undegraded":"2023-10-09T11:02:16.234760+0000","last_fullsized":"2023-10-09T11:02:16.234760+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:40:53.898922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":110,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:18.019594+0000","last_change":"2023-10-09T11:00:14.878579+0000","last_active":"2023-10-09T11:02:18.019594+0000","last_peered":"2023-10-09T11:02:18.019594+0000","last_clean":"2023-10-09T11:02:18.019594+0000","last_became_active":"2023-10-09T11:00:14.867930+0000","last_became_peered":"2023-10-09T11:00:14.867930+0000","last_unstale":"2023-10-09T11:02:18.019594+0000","last_undegraded":"2023-10-09T11:02:18.019594+0000","last_fullsized":"2023-10-09T11:02:18.019594+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:34.925327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"431'4803","reported_seq":7148,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.599616+0000","last_change":"2023-10-09T11:01:34.181226+0000","last_active":"2023-10-09T11:02:21.599616+0000","last_peered":"2023-10-09T11:02:21.599616+0000","last_clean":"2023-10-09T11:02:21.599616+0000","last_became_active":"2023-10-09T11:01:34.166162+0000","last_became_peered":"2023-10-09T11:01:34.166162+0000","last_unstale":"2023-10-09T11:02:21.599616+0000","last_undegraded":"2023-10-09T11:02:21.599616+0000","last_fullsized":"2023-10-09T11:02:21.599616+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4803,"log_dups_size":0,"ondisk_log_size":4803,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:18.420715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314769408,"num_objects":4803,"num_object_clones":0,"num_object_copies":14409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4803,"num_write_kb":307392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1144,"num_bytes_recovered":74973184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":856,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.238425+0000","last_change":"2023-10-09T11:01:29.161986+0000","last_active":"2023-10-09T11:02:16.238425+0000","last_peered":"2023-10-09T11:02:16.238425+0000","last_clean":"2023-10-09T11:02:16.238425+0000","last_became_active":"2023-10-09T11:01:29.161669+0000","last_became_peered":"2023-10-09T11:01:29.161669+0000","last_unstale":"2023-10-09T11:02:16.238425+0000","last_undegraded":"2023-10-09T11:02:16.238425+0000","last_fullsized":"2023-10-09T11:02:16.238425+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:35.491541+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":182,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.228736+0000","last_change":"2023-10-09T10:59:48.122743+0000","last_active":"2023-10-09T11:02:16.228736+0000","last_peered":"2023-10-09T11:02:16.228736+0000","last_clean":"2023-10-09T11:02:16.228736+0000","last_became_active":"2023-10-09T10:59:48.122447+0000","last_became_peered":"2023-10-09T10:59:48.122447+0000","last_unstale":"2023-10-09T11:02:16.228736+0000","last_undegraded":"2023-10-09T11:02:16.228736+0000","last_fullsized":"2023-10-09T11:02:16.228736+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:28.368331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":110,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.255992+0000","last_change":"2023-10-09T11:00:14.897166+0000","last_active":"2023-10-09T11:02:16.255992+0000","last_peered":"2023-10-09T11:02:16.255992+0000","last_clean":"2023-10-09T11:02:16.255992+0000","last_became_active":"2023-10-09T11:00:14.896917+0000","last_became_peered":"2023-10-09T11:00:14.896917+0000","last_unstale":"2023-10-09T11:02:16.255992+0000","last_undegraded":"2023-10-09T11:02:16.255992+0000","last_fullsized":"2023-10-09T11:02:16.255992+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:09.954705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":108,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:18.289743+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:02:18.289743+0000","last_peered":"2023-10-09T11:02:18.289743+0000","last_clean":"2023-10-09T11:02:18.289743+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:02:18.289743+0000","last_undegraded":"2023-10-09T11:02:18.289743+0000","last_fullsized":"2023-10-09T11:02:18.289743+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"431'4993","reported_seq":5030,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:22.975536+0000","last_change":"2023-10-09T11:01:15.405163+0000","last_active":"2023-10-09T11:02:22.975536+0000","last_peered":"2023-10-09T11:02:22.975536+0000","last_clean":"2023-10-09T11:02:22.975536+0000","last_became_active":"2023-10-09T11:01:15.404873+0000","last_became_peered":"2023-10-09T11:01:15.404873+0000","last_unstale":"2023-10-09T11:02:22.975536+0000","last_undegraded":"2023-10-09T11:02:22.975536+0000","last_fullsized":"2023-10-09T11:02:22.975536+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4993,"log_dups_size":0,"ondisk_log_size":4993,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:08.894661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327221248,"num_objects":4993,"num_object_clones":0,"num_object_copies":14979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4993,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4993,"num_write_kb":319552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":276,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.243105+0000","last_change":"2023-10-09T11:01:26.639395+0000","last_active":"2023-10-09T11:02:16.243105+0000","last_peered":"2023-10-09T11:02:16.243105+0000","last_clean":"2023-10-09T11:02:16.243105+0000","last_became_active":"2023-10-09T11:01:26.639105+0000","last_became_peered":"2023-10-09T11:01:26.639105+0000","last_unstale":"2023-10-09T11:02:16.243105+0000","last_undegraded":"2023-10-09T11:02:16.243105+0000","last_fullsized":"2023-10-09T11:02:16.243105+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:10.391398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":801,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.235379+0000","last_change":"2023-10-09T11:01:29.186101+0000","last_active":"2023-10-09T11:02:16.235379+0000","last_peered":"2023-10-09T11:02:16.235379+0000","last_clean":"2023-10-09T11:02:16.235379+0000","last_became_active":"2023-10-09T11:01:29.185773+0000","last_became_peered":"2023-10-09T11:01:29.185773+0000","last_unstale":"2023-10-09T11:02:16.235379+0000","last_undegraded":"2023-10-09T11:02:16.235379+0000","last_fullsized":"2023-10-09T11:02:16.235379+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:51.271260+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":366,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:18.245359+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T11:02:18.245359+0000","last_peered":"2023-10-09T11:02:18.245359+0000","last_clean":"2023-10-09T11:02:18.245359+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T11:02:18.245359+0000","last_undegraded":"2023-10-09T11:02:18.245359+0000","last_fullsized":"2023-10-09T11:02:18.245359+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":373,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.253359+0000","last_change":"2023-10-09T11:01:26.660425+0000","last_active":"2023-10-09T11:02:16.253359+0000","last_peered":"2023-10-09T11:02:16.253359+0000","last_clean":"2023-10-09T11:02:16.253359+0000","last_became_active":"2023-10-09T11:01:26.660095+0000","last_became_peered":"2023-10-09T11:01:26.660095+0000","last_unstale":"2023-10-09T11:02:16.253359+0000","last_undegraded":"2023-10-09T11:02:16.253359+0000","last_fullsized":"2023-10-09T11:02:16.253359+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:01.819696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"431'4993","reported_seq":7300,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:23.001124+0000","last_change":"2023-10-09T11:01:42.939222+0000","last_active":"2023-10-09T11:02:23.001124+0000","last_peered":"2023-10-09T11:02:23.001124+0000","last_clean":"2023-10-09T11:02:23.001124+0000","last_became_active":"2023-10-09T11:01:42.935626+0000","last_became_peered":"2023-10-09T11:01:42.935626+0000","last_unstale":"2023-10-09T11:02:23.001124+0000","last_undegraded":"2023-10-09T11:02:23.001124+0000","last_fullsized":"2023-10-09T11:02:23.001124+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4993,"log_dups_size":0,"ondisk_log_size":4993,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:51:09.694587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327221248,"num_objects":4993,"num_object_clones":0,"num_object_copies":14979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4993,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4993,"num_write_kb":319552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1126,"num_bytes_recovered":73793536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":766,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:18.019540+0000","last_change":"2023-10-09T11:01:17.369788+0000","last_active":"2023-10-09T11:02:18.019540+0000","last_peered":"2023-10-09T11:02:18.019540+0000","last_clean":"2023-10-09T11:02:18.019540+0000","last_became_active":"2023-10-09T11:00:14.881494+0000","last_became_peered":"2023-10-09T11:00:14.881494+0000","last_unstale":"2023-10-09T11:02:18.019540+0000","last_undegraded":"2023-10-09T11:02:18.019540+0000","last_fullsized":"2023-10-09T11:02:18.019540+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:44.208863+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":317,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.255694+0000","last_change":"2023-10-09T11:01:26.641036+0000","last_active":"2023-10-09T11:02:16.255694+0000","last_peered":"2023-10-09T11:02:16.255694+0000","last_clean":"2023-10-09T11:02:16.255694+0000","last_became_active":"2023-10-09T11:01:26.640701+0000","last_became_peered":"2023-10-09T11:01:26.640701+0000","last_unstale":"2023-10-09T11:02:16.255694+0000","last_undegraded":"2023-10-09T11:02:16.255694+0000","last_fullsized":"2023-10-09T11:02:16.255694+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:14:05.374525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":67,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.240042+0000","last_change":"2023-10-09T10:59:48.135546+0000","last_active":"2023-10-09T11:02:16.240042+0000","last_peered":"2023-10-09T11:02:16.240042+0000","last_clean":"2023-10-09T11:02:16.240042+0000","last_became_active":"2023-10-09T10:59:48.135227+0000","last_became_peered":"2023-10-09T10:59:48.135227+0000","last_unstale":"2023-10-09T11:02:16.240042+0000","last_undegraded":"2023-10-09T11:02:16.240042+0000","last_fullsized":"2023-10-09T11:02:16.240042+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:50.561785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":388,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.235751+0000","last_change":"2023-10-09T11:00:15.892896+0000","last_active":"2023-10-09T11:02:16.235751+0000","last_peered":"2023-10-09T11:02:16.235751+0000","last_clean":"2023-10-09T11:02:16.235751+0000","last_became_active":"2023-10-09T11:00:15.892573+0000","last_became_peered":"2023-10-09T11:00:15.892573+0000","last_unstale":"2023-10-09T11:02:16.235751+0000","last_undegraded":"2023-10-09T11:02:16.235751+0000","last_fullsized":"2023-10-09T11:02:16.235751+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:13.280089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"431'4934","reported_seq":4971,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:22.993462+0000","last_change":"2023-10-09T11:01:15.402557+0000","last_active":"2023-10-09T11:02:22.993462+0000","last_peered":"2023-10-09T11:02:22.993462+0000","last_clean":"2023-10-09T11:02:22.993462+0000","last_became_active":"2023-10-09T11:01:15.402279+0000","last_became_peered":"2023-10-09T11:01:15.402279+0000","last_unstale":"2023-10-09T11:02:22.993462+0000","last_undegraded":"2023-10-09T11:02:22.993462+0000","last_fullsized":"2023-10-09T11:02:22.993462+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4934,"log_dups_size":0,"ondisk_log_size":4934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:19.333267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323354624,"num_objects":4934,"num_object_clones":0,"num_object_copies":14802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4934,"num_write_kb":315776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":644,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.235557+0000","last_change":"2023-10-09T11:01:16.782558+0000","last_active":"2023-10-09T11:02:16.235557+0000","last_peered":"2023-10-09T11:02:16.235557+0000","last_clean":"2023-10-09T11:02:16.235557+0000","last_became_active":"2023-10-09T10:59:51.137287+0000","last_became_peered":"2023-10-09T10:59:51.137287+0000","last_unstale":"2023-10-09T11:02:16.235557+0000","last_undegraded":"2023-10-09T11:02:16.235557+0000","last_fullsized":"2023-10-09T11:02:16.235557+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:58:29.405479+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":270,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.235682+0000","last_change":"2023-10-09T11:01:26.649138+0000","last_active":"2023-10-09T11:02:16.235682+0000","last_peered":"2023-10-09T11:02:16.235682+0000","last_clean":"2023-10-09T11:02:16.235682+0000","last_became_active":"2023-10-09T11:01:26.648437+0000","last_became_peered":"2023-10-09T11:01:26.648437+0000","last_unstale":"2023-10-09T11:02:16.235682+0000","last_undegraded":"2023-10-09T11:02:16.235682+0000","last_fullsized":"2023-10-09T11:02:16.235682+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:29.065233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":110,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:18.019640+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:02:18.019640+0000","last_peered":"2023-10-09T11:02:18.019640+0000","last_clean":"2023-10-09T11:02:18.019640+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:02:18.019640+0000","last_undegraded":"2023-10-09T11:02:18.019640+0000","last_fullsized":"2023-10-09T11:02:18.019640+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":363,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.234875+0000","last_change":"2023-10-09T11:01:26.643724+0000","last_active":"2023-10-09T11:02:16.234875+0000","last_peered":"2023-10-09T11:02:16.234875+0000","last_clean":"2023-10-09T11:02:16.234875+0000","last_became_active":"2023-10-09T11:01:26.643301+0000","last_became_peered":"2023-10-09T11:01:26.643301+0000","last_unstale":"2023-10-09T11:02:16.234875+0000","last_undegraded":"2023-10-09T11:02:16.234875+0000","last_fullsized":"2023-10-09T11:02:16.234875+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:49:36.408009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"431'4834","reported_seq":4870,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.433060+0000","last_change":"2023-10-09T11:01:15.406858+0000","last_active":"2023-10-09T11:02:21.433060+0000","last_peered":"2023-10-09T11:02:21.433060+0000","last_clean":"2023-10-09T11:02:21.433060+0000","last_became_active":"2023-10-09T11:01:15.406513+0000","last_became_peered":"2023-10-09T11:01:15.406513+0000","last_unstale":"2023-10-09T11:02:21.433060+0000","last_undegraded":"2023-10-09T11:02:21.433060+0000","last_fullsized":"2023-10-09T11:02:21.433060+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4834,"log_dups_size":0,"ondisk_log_size":4834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:28:25.137720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316801024,"num_objects":4834,"num_object_clones":0,"num_object_copies":14502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4834,"num_write_kb":309376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":826,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.249509+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T11:02:16.249509+0000","last_peered":"2023-10-09T11:02:16.249509+0000","last_clean":"2023-10-09T11:02:16.249509+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T11:02:16.249509+0000","last_undegraded":"2023-10-09T11:02:16.249509+0000","last_fullsized":"2023-10-09T11:02:16.249509+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":521,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233923+0000","last_change":"2023-10-09T11:01:26.666078+0000","last_active":"2023-10-09T11:02:16.233923+0000","last_peered":"2023-10-09T11:02:16.233923+0000","last_clean":"2023-10-09T11:02:16.233923+0000","last_became_active":"2023-10-09T11:01:26.637507+0000","last_became_peered":"2023-10-09T11:01:26.637507+0000","last_unstale":"2023-10-09T11:02:16.233923+0000","last_undegraded":"2023-10-09T11:02:16.233923+0000","last_fullsized":"2023-10-09T11:02:16.233923+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:55.708083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":110,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:18.289743+0000","last_change":"2023-10-09T11:00:14.900305+0000","last_active":"2023-10-09T11:02:18.289743+0000","last_peered":"2023-10-09T11:02:18.289743+0000","last_clean":"2023-10-09T11:02:18.289743+0000","last_became_active":"2023-10-09T11:00:14.899724+0000","last_became_peered":"2023-10-09T11:00:14.899724+0000","last_unstale":"2023-10-09T11:02:18.289743+0000","last_undegraded":"2023-10-09T11:02:18.289743+0000","last_fullsized":"2023-10-09T11:02:18.289743+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:59.022161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":502,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232120+0000","last_change":"2023-10-09T11:01:26.637115+0000","last_active":"2023-10-09T11:02:16.232120+0000","last_peered":"2023-10-09T11:02:16.232120+0000","last_clean":"2023-10-09T11:02:16.232120+0000","last_became_active":"2023-10-09T11:01:26.636783+0000","last_became_peered":"2023-10-09T11:01:26.636783+0000","last_unstale":"2023-10-09T11:02:16.232120+0000","last_undegraded":"2023-10-09T11:02:16.232120+0000","last_fullsized":"2023-10-09T11:02:16.232120+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:22:50.446443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"431'4931","reported_seq":4967,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T11:02:21.464631+0000","last_change":"2023-10-09T11:01:15.397123+0000","last_active":"2023-10-09T11:02:21.464631+0000","last_peered":"2023-10-09T11:02:21.464631+0000","last_clean":"2023-10-09T11:02:21.464631+0000","last_became_active":"2023-10-09T11:01:15.396842+0000","last_became_peered":"2023-10-09T11:01:15.396842+0000","last_unstale":"2023-10-09T11:02:21.464631+0000","last_undegraded":"2023-10-09T11:02:21.464631+0000","last_fullsized":"2023-10-09T11:02:21.464631+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":4931,"log_dups_size":0,"ondisk_log_size":4931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:54.895524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323158016,"num_objects":4931,"num_object_clones":0,"num_object_copies":14793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4931,"num_write_kb":315584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":876,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232910+0000","last_change":"2023-10-09T11:01:29.165146+0000","last_active":"2023-10-09T11:02:16.232910+0000","last_peered":"2023-10-09T11:02:16.232910+0000","last_clean":"2023-10-09T11:02:16.232910+0000","last_became_active":"2023-10-09T11:01:29.164818+0000","last_became_peered":"2023-10-09T11:01:29.164818+0000","last_unstale":"2023-10-09T11:02:16.232910+0000","last_undegraded":"2023-10-09T11:02:16.232910+0000","last_fullsized":"2023-10-09T11:02:16.232910+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:42.211670+0000","scrub_duration":0.0053707060000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":206,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.228583+0000","last_change":"2023-10-09T11:01:26.648453+0000","last_active":"2023-10-09T11:02:16.228583+0000","last_peered":"2023-10-09T11:02:16.228583+0000","last_clean":"2023-10-09T11:02:16.228583+0000","last_became_active":"2023-10-09T11:01:26.648115+0000","last_became_peered":"2023-10-09T11:01:26.648115+0000","last_unstale":"2023-10-09T11:02:16.228583+0000","last_undegraded":"2023-10-09T11:02:16.228583+0000","last_fullsized":"2023-10-09T11:02:16.228583+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:26.270236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":372,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.235800+0000","last_change":"2023-10-09T11:00:15.869009+0000","last_active":"2023-10-09T11:02:16.235800+0000","last_peered":"2023-10-09T11:02:16.235800+0000","last_clean":"2023-10-09T11:02:16.235800+0000","last_became_active":"2023-10-09T11:00:15.868637+0000","last_became_peered":"2023-10-09T11:00:15.868637+0000","last_unstale":"2023-10-09T11:02:16.235800+0000","last_undegraded":"2023-10-09T11:02:16.235800+0000","last_fullsized":"2023-10-09T11:02:16.235800+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:37.538741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":216,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.244174+0000","last_change":"2023-10-09T11:01:15.402661+0000","last_active":"2023-10-09T11:02:16.244174+0000","last_peered":"2023-10-09T11:02:16.244174+0000","last_clean":"2023-10-09T11:02:16.244174+0000","last_became_active":"2023-10-09T11:01:15.402327+0000","last_became_peered":"2023-10-09T11:01:15.402327+0000","last_unstale":"2023-10-09T11:02:16.244174+0000","last_undegraded":"2023-10-09T11:02:16.244174+0000","last_fullsized":"2023-10-09T11:02:16.244174+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:32.513751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":858,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233385+0000","last_change":"2023-10-09T10:59:51.145348+0000","last_active":"2023-10-09T11:02:16.233385+0000","last_peered":"2023-10-09T11:02:16.233385+0000","last_clean":"2023-10-09T11:02:16.233385+0000","last_became_active":"2023-10-09T10:59:51.145100+0000","last_became_peered":"2023-10-09T10:59:51.145100+0000","last_unstale":"2023-10-09T11:02:16.233385+0000","last_undegraded":"2023-10-09T11:02:16.233385+0000","last_fullsized":"2023-10-09T11:02:16.233385+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:51.049260+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":362,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.234954+0000","last_change":"2023-10-09T11:01:26.649487+0000","last_active":"2023-10-09T11:02:16.234954+0000","last_peered":"2023-10-09T11:02:16.234954+0000","last_clean":"2023-10-09T11:02:16.234954+0000","last_became_active":"2023-10-09T11:01:26.649133+0000","last_became_peered":"2023-10-09T11:01:26.649133+0000","last_unstale":"2023-10-09T11:02:16.234954+0000","last_undegraded":"2023-10-09T11:02:16.234954+0000","last_fullsized":"2023-10-09T11:02:16.234954+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:40.585289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":187,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.229065+0000","last_change":"2023-10-09T11:00:14.886903+0000","last_active":"2023-10-09T11:02:16.229065+0000","last_peered":"2023-10-09T11:02:16.229065+0000","last_clean":"2023-10-09T11:02:16.229065+0000","last_became_active":"2023-10-09T11:00:14.886635+0000","last_became_peered":"2023-10-09T11:00:14.886635+0000","last_unstale":"2023-10-09T11:02:16.229065+0000","last_undegraded":"2023-10-09T11:02:16.229065+0000","last_fullsized":"2023-10-09T11:02:16.229065+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:04.982085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":807,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233198+0000","last_change":"2023-10-09T11:01:29.173501+0000","last_active":"2023-10-09T11:02:16.233198+0000","last_peered":"2023-10-09T11:02:16.233198+0000","last_clean":"2023-10-09T11:02:16.233198+0000","last_became_active":"2023-10-09T11:01:29.172950+0000","last_became_peered":"2023-10-09T11:01:29.172950+0000","last_unstale":"2023-10-09T11:02:16.233198+0000","last_undegraded":"2023-10-09T11:02:16.233198+0000","last_fullsized":"2023-10-09T11:02:16.233198+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:40:05.032667+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":258,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242204+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:02:16.242204+0000","last_peered":"2023-10-09T11:02:16.242204+0000","last_clean":"2023-10-09T11:02:16.242204+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:02:16.242204+0000","last_undegraded":"2023-10-09T11:02:16.242204+0000","last_fullsized":"2023-10-09T11:02:16.242204+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":459,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232468+0000","last_change":"2023-10-09T10:57:46.844827+0000","last_active":"2023-10-09T11:02:16.232468+0000","last_peered":"2023-10-09T11:02:16.232468+0000","last_clean":"2023-10-09T11:02:16.232468+0000","last_became_active":"2023-10-09T10:57:46.844584+0000","last_became_peered":"2023-10-09T10:57:46.844584+0000","last_unstale":"2023-10-09T11:02:16.232468+0000","last_undegraded":"2023-10-09T11:02:16.232468+0000","last_fullsized":"2023-10-09T11:02:16.232468+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:54.499813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":381,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.242510+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T11:02:16.242510+0000","last_peered":"2023-10-09T11:02:16.242510+0000","last_clean":"2023-10-09T11:02:16.242510+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T11:02:16.242510+0000","last_undegraded":"2023-10-09T11:02:16.242510+0000","last_fullsized":"2023-10-09T11:02:16.242510+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":395,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.232468+0000","last_change":"2023-10-09T10:59:48.136562+0000","last_active":"2023-10-09T11:02:16.232468+0000","last_peered":"2023-10-09T11:02:16.232468+0000","last_clean":"2023-10-09T11:02:16.232468+0000","last_became_active":"2023-10-09T10:59:48.136184+0000","last_became_peered":"2023-10-09T10:59:48.136184+0000","last_unstale":"2023-10-09T11:02:16.232468+0000","last_undegraded":"2023-10-09T11:02:16.232468+0000","last_fullsized":"2023-10-09T11:02:16.232468+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:16.944093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":651,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.259408+0000","last_change":"2023-10-09T11:01:29.163551+0000","last_active":"2023-10-09T11:02:16.259408+0000","last_peered":"2023-10-09T11:02:16.259408+0000","last_clean":"2023-10-09T11:02:16.259408+0000","last_became_active":"2023-10-09T11:01:29.163220+0000","last_became_peered":"2023-10-09T11:01:29.163220+0000","last_unstale":"2023-10-09T11:02:16.259408+0000","last_undegraded":"2023-10-09T11:02:16.259408+0000","last_fullsized":"2023-10-09T11:02:16.259408+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:47.307693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:47.307693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":458,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233518+0000","last_change":"2023-10-09T10:54:17.886158+0000","last_active":"2023-10-09T11:02:16.233518+0000","last_peered":"2023-10-09T11:02:16.233518+0000","last_clean":"2023-10-09T11:02:16.233518+0000","last_became_active":"2023-10-09T10:54:17.885816+0000","last_became_peered":"2023-10-09T10:54:17.885816+0000","last_unstale":"2023-10-09T11:02:16.233518+0000","last_undegraded":"2023-10-09T11:02:16.233518+0000","last_fullsized":"2023-10-09T11:02:16.233518+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:05.270825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":478,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.233721+0000","last_change":"2023-10-09T10:59:48.125955+0000","last_active":"2023-10-09T11:02:16.233721+0000","last_peered":"2023-10-09T11:02:16.233721+0000","last_clean":"2023-10-09T11:02:16.233721+0000","last_became_active":"2023-10-09T10:59:48.125641+0000","last_became_peered":"2023-10-09T10:59:48.125641+0000","last_unstale":"2023-10-09T11:02:16.233721+0000","last_undegraded":"2023-10-09T11:02:16.233721+0000","last_fullsized":"2023-10-09T11:02:16.233721+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:38.041100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":675,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.260190+0000","last_change":"2023-10-09T11:01:54.238624+0000","last_active":"2023-10-09T11:02:16.260190+0000","last_peered":"2023-10-09T11:02:16.260190+0000","last_clean":"2023-10-09T11:02:16.260190+0000","last_became_active":"2023-10-09T11:00:14.867744+0000","last_became_peered":"2023-10-09T11:00:14.867744+0000","last_unstale":"2023-10-09T11:02:16.260190+0000","last_undegraded":"2023-10-09T11:02:16.260190+0000","last_fullsized":"2023-10-09T11:02:16.260190+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:21.022863+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:21.022863+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":73,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.235944+0000","last_change":"2023-10-09T10:59:48.143732+0000","last_active":"2023-10-09T11:02:16.235944+0000","last_peered":"2023-10-09T11:02:16.235944+0000","last_clean":"2023-10-09T11:02:16.235944+0000","last_became_active":"2023-10-09T10:59:48.129001+0000","last_became_peered":"2023-10-09T10:59:48.129001+0000","last_unstale":"2023-10-09T11:02:16.235944+0000","last_undegraded":"2023-10-09T11:02:16.235944+0000","last_fullsized":"2023-10-09T11:02:16.235944+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:22:36.946348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":452,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T11:02:16.248859+0000","last_change":"2023-10-09T11:00:14.869353+0000","last_active":"2023-10-09T11:02:16.248859+0000","last_peered":"2023-10-09T11:02:16.248859+0000","last_clean":"2023-10-09T11:02:16.248859+0000","last_became_active":"2023-10-09T11:00:14.869061+0000","last_became_peered":"2023-10-09T11:00:14.869061+0000","last_unstale":"2023-10-09T11:02:16.248859+0000","last_undegraded":"2023-10-09T11:02:16.248859+0000","last_fullsized":"2023-10-09T11:02:16.248859+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:30.318094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5144379392,"num_objects":78497,"num_object_clones":0,"num_object_copies":235491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":78497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":78497,"num_write_kb":5023808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4540,"num_bytes_recovered":297533440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15454175232,"data_stored":15454175232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":78497,"ondisk_log_size":78497,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":12396,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8961,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635094,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":39,"num_read_kb":70,"num_write":20,"num_write_kb":184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1216064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":219043332429,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":44,"seq":188978561360,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3070512,"kb_used_data":2174440,"kb_used_omap":0,"kb_used_meta":896064,"kb_avail":90674640,"statfs":{"total":95995035648,"available":92850831360,"internally_reserved":0,"allocated":2226626560,"data_stored":2225259232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917569536},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:07 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":1.8009999999999999,"15min":1.877},"min":{"1min":0.224,"5min":0.224,"15min":0.216},"max":{"1min":3.9460000000000002,"5min":46.603999999999999,"15min":54.743000000000002},"last":4.9130000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":2.0760000000000001,"15min":1.9790000000000001},"min":{"1min":0.28999999999999998,"5min":0.223,"15min":0.223},"max":{"1min":1.9039999999999999,"5min":46.679000000000002,"15min":53.701000000000001},"last":4.7640000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:02:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0529999999999999,"5min":1.2130000000000001,"15min":0.90700000000000003},"min":{"1min":0.29399999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":41.241999999999997,"5min":41.241999999999997,"15min":41.241999999999997},"last":4.8460000000000001},{"interface":"front","average":{"1min":3.5070000000000001,"5min":1.792,"15min":1.506},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":41.476999999999997,"5min":41.476999999999997,"15min":41.476999999999997},"last":2.6720000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.794,"15min":1.843},"min":{"1min":0.34999999999999998,"5min":0.30599999999999999,"15min":0.28399999999999997},"max":{"1min":3.2759999999999998,"5min":12.003,"15min":13.208},"last":2.601},{"interface":"front","average":{"1min":1.7829999999999999,"5min":2.0150000000000001,"15min":2.0009999999999999},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.28599999999999998},"max":{"1min":7.1870000000000003,"5min":15.432,"15min":15.432},"last":1.766}]},{"osd":3,"last update":"Mon Oct 9 11:02:07 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.6719999999999999,"15min":1.8320000000000001},"min":{"1min":0.35499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":3.266,"5min":10.018000000000001,"15min":12.391},"last":4.9429999999999996},{"interface":"front","average":{"1min":1.101,"5min":1.885,"15min":2.0870000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":3.6389999999999998,"5min":11.161,"15min":13.179},"last":2.6429999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.90100000000000002,"15min":1.2949999999999999},"min":{"1min":0.25900000000000001,"5min":0.25800000000000001,"15min":0.23200000000000001},"max":{"1min":3.9449999999999998,"5min":9.0060000000000002,"15min":16.978999999999999},"last":1.6040000000000001},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.88300000000000001,"15min":1.2450000000000001},"min":{"1min":0.27800000000000002,"5min":0.249,"15min":0.24399999999999999},"max":{"1min":2.944,"5min":5.7869999999999999,"15min":12.481999999999999},"last":4.8049999999999997}]},{"osd":5,"last update":"Mon Oct 9 11:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.385,"15min":1.5600000000000001},"min":{"1min":0.30499999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.9260000000000002,"5min":19.786999999999999,"15min":19.786999999999999},"last":1.6479999999999999},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.083,"15min":1.339},"min":{"1min":0.35099999999999998,"5min":0.28899999999999998,"15min":0.26600000000000001},"max":{"1min":3.6850000000000001,"5min":9.0559999999999992,"15min":12.456},"last":4.8810000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:01:59 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.2989999999999999,"15min":1.5549999999999999},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.26600000000000001},"max":{"1min":1.885,"5min":9.1120000000000001,"15min":23.358000000000001},"last":1.6850000000000001},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.96699999999999997,"15min":1.216},"min":{"1min":0.32400000000000001,"5min":0.32300000000000001,"15min":0.27500000000000002},"max":{"1min":0.83199999999999996,"5min":6.1920000000000002,"15min":12.422000000000001},"last":4.9690000000000003}]}]},{"osd":1,"up_from":388,"seq":1666447310875,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1926828,"kb_used_data":1694884,"kb_used_omap":13,"kb_used_meta":231922,"kb_avail":91818324,"statfs":{"total":95995035648,"available":94021963776,"internally_reserved":0,"allocated":1735561216,"data_stored":1733329043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":237488839},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":3.2690000000000001,"5min":2.46,"15min":2.3250000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":13.334,"5min":13.334,"15min":13.334},"last":0.22500000000000001},{"interface":"front","average":{"1min":2.968,"5min":1.6419999999999999,"15min":1.421},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.6829999999999998,"5min":9.6829999999999998,"15min":9.6829999999999998},"last":0.25}]},{"osd":2,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":2.9790000000000001,"15min":3.1389999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.4670000000000005,"5min":12.009,"15min":12.009},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.5499999999999998,"5min":2.2959999999999998,"15min":2.254},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.8249999999999993,"5min":9.8249999999999993,"15min":9.8249999999999993},"last":0.28100000000000003}]},{"osd":3,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7879999999999998,"5min":2.8199999999999998,"15min":2.8250000000000002},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":9.6460000000000008,"5min":12.523999999999999,"15min":12.523999999999999},"last":30.75},{"interface":"front","average":{"1min":1.518,"5min":3.5539999999999998,"15min":3.8940000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.3799999999999999,"5min":18.079999999999998,"15min":18.079999999999998},"last":30.055}]},{"osd":4,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":2.536,"15min":2.754},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.2300000000000004,"5min":10.481,"15min":10.481},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.702,"5min":1.448,"15min":1.4059999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":9.8480000000000008,"5min":9.8480000000000008,"15min":9.8480000000000008},"last":0.46000000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":2.29,"15min":2.4580000000000002},"min":{"1min":0.35599999999999998,"5min":0.311,"15min":0.311},"max":{"1min":4.8799999999999999,"5min":18.053000000000001,"15min":18.053000000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.526,"5min":2.2429999999999999,"15min":2.363},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.2830000000000004,"5min":10.558,"15min":10.558},"last":0.46999999999999997}]},{"osd":6,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":2.2759999999999998,"15min":2.46},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.8520000000000003,"5min":18.085999999999999,"15min":18.085999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.3049999999999999,"5min":2.1869999999999998,"15min":2.3340000000000001},"min":{"1min":0.38600000000000001,"5min":0.373,"15min":0.373},"max":{"1min":5.3129999999999997,"5min":10.531000000000001,"15min":10.531000000000001},"last":0.501}]},{"osd":7,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.393,"15min":1.3220000000000001},"min":{"1min":0.41899999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.8000000000000007,"5min":9.8000000000000007,"15min":9.8000000000000007},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.1910000000000001,"5min":2.3279999999999998,"15min":2.5179999999999998},"min":{"1min":0.435,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.8019999999999996,"5min":18.071000000000002,"15min":18.071000000000002},"last":0.44900000000000001}]}]},{"osd":0,"up_from":80,"seq":343597383933,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3874520,"kb_used_data":3192912,"kb_used_omap":28,"kb_used_meta":681571,"kb_avail":89870632,"statfs":{"total":95995035648,"available":92027527168,"internally_reserved":0,"allocated":3269541888,"data_stored":3267262172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":697929552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,6],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:02:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":3.0379999999999998,"15min":3.2040000000000002},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":9.6460000000000008,"5min":19.608000000000001,"15min":19.608000000000001},"last":9.2799999999999994},{"interface":"front","average":{"1min":1.879,"5min":3.0710000000000002,"15min":3.2690000000000001},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.218},"max":{"1min":7.8730000000000002,"5min":19.992999999999999,"15min":19.992999999999999},"last":0.60099999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:01:45 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.958,"15min":2.1480000000000001},"min":{"1min":0.24199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":4.6639999999999997,"5min":12.638,"15min":24.852},"last":3.915},{"interface":"front","average":{"1min":1.6299999999999999,"5min":2.4609999999999999,"15min":2.6219999999999999},"min":{"1min":0.20899999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":8.7140000000000004,"5min":12.672000000000001,"15min":77.256},"last":1.071}]},{"osd":3,"last update":"Mon Oct 9 11:02:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":2.1589999999999998,"15min":2.3090000000000002},"min":{"1min":0.26800000000000002,"5min":0.215,"15min":0.20200000000000001},"max":{"1min":8.8699999999999992,"5min":28.946000000000002,"15min":28.946000000000002},"last":0.68400000000000005},{"interface":"front","average":{"1min":1.46,"5min":2.4350000000000001,"15min":2.5790000000000002},"min":{"1min":0.309,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.9850000000000003,"5min":19.952999999999999,"15min":19.952999999999999},"last":0.53200000000000003}]},{"osd":4,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":2.2679999999999998,"15min":2.4649999999999999},"min":{"1min":0.32900000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.9560000000000004,"5min":11.507,"15min":77.177999999999997},"last":1.0980000000000001},{"interface":"front","average":{"1min":1.405,"5min":1.6499999999999999,"15min":2.0190000000000001},"min":{"1min":0.38400000000000001,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":6.4450000000000003,"5min":9.9489999999999998,"15min":12.318},"last":0.63600000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.827,"15min":2.077},"min":{"1min":0.36599999999999999,"5min":0.308,"15min":0.29799999999999999},"max":{"1min":6.4880000000000004,"5min":9.8710000000000004,"15min":12.157},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.9350000000000001,"15min":2.0750000000000002},"min":{"1min":0.436,"5min":0.29999999999999999,"15min":0.29899999999999999},"max":{"1min":2.859,"5min":7.0919999999999996,"15min":24.754999999999999},"last":0.61699999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5700000000000001,"5min":1.855,"15min":2.2349999999999999},"min":{"1min":0.38800000000000001,"5min":0.314,"15min":0.30399999999999999},"max":{"1min":6.5179999999999998,"5min":10.012,"15min":18.920000000000002},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.9490000000000001,"15min":1.9810000000000001},"min":{"1min":0.45700000000000002,"5min":0.31900000000000001,"15min":0.307},"max":{"1min":2.9209999999999998,"5min":8.8249999999999993,"15min":24.800000000000001},"last":0.64900000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.958,"15min":2.177},"min":{"1min":0.44900000000000001,"5min":0.34499999999999997,"15min":0.32700000000000001},"max":{"1min":2.8929999999999998,"5min":11.285,"15min":14.48},"last":0.67100000000000004},{"interface":"front","average":{"1min":1.3740000000000001,"5min":2.3809999999999998,"15min":2.5910000000000002},"min":{"1min":0.41499999999999998,"5min":0.372,"15min":0.314},"max":{"1min":6.008,"5min":10.398,"15min":77.228999999999999},"last":1.127}]}]},{"osd":2,"up_from":90,"seq":386547056873,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2370160,"kb_used_data":1858088,"kb_used_omap":81,"kb_used_meta":511982,"kb_avail":91374992,"statfs":{"total":95995035648,"available":93567991808,"internally_reserved":0,"allocated":1902682112,"data_stored":1900560975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":524269786},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":1.3819999999999999,"15min":1.7569999999999999},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":8.9990000000000006,"5min":8.9990000000000006,"15min":18.512},"last":2.3820000000000001},{"interface":"front","average":{"1min":1.5269999999999999,"5min":1.5609999999999999,"15min":1.9710000000000001},"min":{"1min":0.20699999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":7.7469999999999999,"5min":7.7469999999999999,"15min":17.975000000000001},"last":0.48199999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:02:18 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":2.403,"15min":2.4830000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":10.502000000000001,"5min":10.502000000000001,"15min":10.502000000000001},"last":1.9139999999999999},{"interface":"front","average":{"1min":1.6200000000000001,"5min":2.448,"15min":2.5859999999999999},"min":{"1min":0.27300000000000002,"5min":0.214,"15min":0.214},"max":{"1min":5.125,"5min":11.92,"15min":11.92},"last":5.125}]},{"osd":3,"last update":"Mon Oct 9 11:02:09 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":1.6279999999999999,"15min":1.863},"min":{"1min":0.24399999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.5979999999999999,"5min":11.151,"15min":19.539000000000001},"last":2.7450000000000001},{"interface":"front","average":{"1min":1.01,"5min":1.913,"15min":2.093},"min":{"1min":0.27200000000000002,"5min":0.219,"15min":0.219},"max":{"1min":3.524,"5min":16.324000000000002,"15min":16.702999999999999},"last":2.1040000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":2.016,"15min":1.984},"min":{"1min":0.35399999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.4859999999999998,"5min":9.3290000000000006,"15min":16.231999999999999},"last":1.948},{"interface":"front","average":{"1min":1.458,"5min":1.512,"15min":1.782},"min":{"1min":0.33400000000000002,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":4.8470000000000004,"5min":6.8490000000000002,"15min":15.696999999999999},"last":2.4209999999999998}]},{"osd":5,"last update":"Mon Oct 9 11:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.581,"15min":1.8080000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.6369999999999996,"5min":8.5380000000000003,"15min":17.920000000000002},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.6639999999999999,"5min":1.954,"15min":1.899},"min":{"1min":0.311,"5min":0.311,"15min":0.30299999999999999},"max":{"1min":7.5250000000000004,"5min":8.5869999999999997,"15min":17.882999999999999},"last":1.9730000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":1.341,"15min":1.5609999999999999},"min":{"1min":0.373,"5min":0.32100000000000001,"15min":0.30499999999999999},"max":{"1min":4.3239999999999998,"5min":7.4009999999999998,"15min":10.443},"last":2.452},{"interface":"front","average":{"1min":1.383,"5min":1.429,"15min":1.74},"min":{"1min":0.36099999999999999,"5min":0.313,"15min":0.313},"max":{"1min":6.6749999999999998,"5min":6.6749999999999998,"15min":12.862},"last":0.65300000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.679,"5min":1.6399999999999999,"15min":1.6699999999999999},"min":{"1min":0.39000000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.5609999999999999,"5min":6.3300000000000001,"15min":10.470000000000001},"last":2.7970000000000002},{"interface":"front","average":{"1min":1.393,"5min":1.607,"15min":1.883},"min":{"1min":0.34699999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.71,"5min":7.6550000000000002,"15min":12.826000000000001},"last":1.4750000000000001}]}]},{"osd":3,"up_from":95,"seq":408021893347,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2833472,"kb_used_data":1851256,"kb_used_omap":7,"kb_used_meta":982200,"kb_avail":90911680,"statfs":{"total":95995035648,"available":93093560320,"internally_reserved":0,"allocated":1895686144,"data_stored":1893367869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1005773416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":2.556,"15min":2.6800000000000002},"min":{"1min":0.222,"5min":0.20499999999999999,"15min":0.17499999999999999},"max":{"1min":8.1709999999999994,"5min":10.140000000000001,"15min":38.817999999999998},"last":5.9690000000000003},{"interface":"front","average":{"1min":2.4769999999999999,"5min":2.3530000000000002,"15min":2.9649999999999999},"min":{"1min":0.246,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":8.4239999999999995,"5min":17.042000000000002,"15min":17.312000000000001},"last":3.8719999999999999}]},{"osd":1,"last update":"Mon Oct 9 11:02:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8,"5min":2.9199999999999999,"15min":3.1059999999999999},"min":{"1min":0.25,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":8.8030000000000008,"5min":8.9100000000000001,"15min":8.9100000000000001},"last":5.4930000000000003},{"interface":"front","average":{"1min":1.4650000000000001,"5min":2.5840000000000001,"15min":2.77},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.2300000000000004,"5min":9.7449999999999992,"15min":9.7449999999999992},"last":0.66700000000000004}]},{"osd":2,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":2.3100000000000001,"5min":2.3069999999999999,"15min":2.448},"min":{"1min":0.21099999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":5.5350000000000001,"5min":10.65,"15min":20.402000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.0489999999999999,"5min":2.2839999999999998,"15min":2.8260000000000001},"min":{"1min":0.23899999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":8.2729999999999997,"5min":10.012,"15min":27.538},"last":0.46600000000000003}]},{"osd":4,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.355,"5min":2.101,"15min":2.2770000000000001},"min":{"1min":0.30299999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.3639999999999999,"5min":9.8089999999999993,"15min":20.388000000000002},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.3029999999999999,"5min":1.7929999999999999,"15min":2.157},"min":{"1min":0.34499999999999997,"5min":0.34000000000000002,"15min":0.24299999999999999},"max":{"1min":3.5910000000000002,"5min":9.1310000000000002,"15min":19.085999999999999},"last":0.50600000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.948,"15min":2.2890000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.26000000000000001},"max":{"1min":3.5099999999999998,"5min":10.380000000000001,"15min":14.045},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.4630000000000001,"5min":1.7470000000000001,"15min":2.0369999999999999},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2839999999999998,"5min":8.3949999999999996,"15min":17.311},"last":0.48999999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":2.1419999999999999,"15min":2.27},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28000000000000003},"max":{"1min":8.3119999999999994,"5min":10.420999999999999,"15min":12.361000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.835,"5min":1.9430000000000001,"15min":2.2090000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.8959999999999999,"5min":7.1230000000000002,"15min":17.295000000000002},"last":0.629}]},{"osd":7,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":1.9419999999999999,"15min":2.2440000000000002},"min":{"1min":0.46400000000000002,"5min":0.309,"15min":0.309},"max":{"1min":7.492,"5min":8.6649999999999991,"15min":18.722000000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.6180000000000001,"5min":2.194,"15min":2.4670000000000001},"min":{"1min":0.439,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.4020000000000001,"5min":9.8569999999999993,"15min":15.212999999999999},"last":1.0249999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019223,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3676152,"kb_used_data":2779312,"kb_used_omap":0,"kb_used_meta":896832,"kb_avail":90069000,"statfs":{"total":95995035648,"available":92230656000,"internally_reserved":0,"allocated":2846015488,"data_stored":2844640333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918355968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.411,"15min":2.3370000000000002},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":2.9870000000000001,"5min":8.6180000000000003,"15min":213.88999999999999},"last":0.26100000000000001},{"interface":"front","average":{"1min":1.393,"5min":1.359,"15min":2.6019999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.217},"max":{"1min":6.1269999999999998,"5min":6.1269999999999998,"15min":214.09800000000001},"last":2.4049999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:02:15 2023","interfaces":[{"interface":"back","average":{"1min":8.0950000000000006,"5min":3.2919999999999998,"15min":2.492},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":121.41,"5min":121.41,"15min":121.41},"last":0.32500000000000001},{"interface":"front","average":{"1min":7.8860000000000001,"5min":2.6699999999999999,"15min":1.8009999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":122.21899999999999,"5min":122.21899999999999,"15min":122.21899999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:01:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":1.4550000000000001,"15min":1.7809999999999999},"min":{"1min":0.27900000000000003,"5min":0.27100000000000002,"15min":0.253},"max":{"1min":3.9390000000000001,"5min":5.6870000000000003,"15min":15.446999999999999},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.492,"5min":1.9299999999999999,"15min":2.1739999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.9540000000000002,"5min":9.9480000000000004,"15min":27.818000000000001},"last":1.675}]},{"osd":3,"last update":"Mon Oct 9 11:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":1.7509999999999999,"15min":2.6469999999999998},"min":{"1min":0.46999999999999997,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":2.254,"5min":12.798999999999999,"15min":209.03700000000001},"last":2.6669999999999998},{"interface":"front","average":{"1min":0.878,"5min":1.7749999999999999,"15min":2.899},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29999999999999999},"max":{"1min":1.8480000000000001,"5min":8.6010000000000009,"15min":208.80199999999999},"last":2.7120000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":1.2909999999999999,"15min":1.7669999999999999},"min":{"1min":0.307,"5min":0.245,"15min":0.23799999999999999},"max":{"1min":2.024,"5min":9.7880000000000003,"15min":24.149999999999999},"last":0.314},{"interface":"front","average":{"1min":0.78800000000000003,"5min":1.2190000000000001,"15min":1.556},"min":{"1min":0.36899999999999999,"5min":0.25800000000000001,"15min":0.22600000000000001},"max":{"1min":3.1859999999999999,"5min":12.459,"15min":20.422000000000001},"last":0.96899999999999997}]},{"osd":6,"last update":"Mon Oct 9 11:01:43 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.99399999999999999,"15min":1.357},"min":{"1min":0.34999999999999998,"5min":0.27800000000000002,"15min":0.24099999999999999},"max":{"1min":7.2309999999999999,"5min":8.5739999999999998,"15min":15.739000000000001},"last":0.436},{"interface":"front","average":{"1min":1.052,"5min":1.454,"15min":1.7270000000000001},"min":{"1min":0.34100000000000003,"5min":0.32800000000000001,"15min":0.311},"max":{"1min":4.5220000000000002,"5min":9.0489999999999995,"15min":14.272},"last":0.502}]},{"osd":7,"last update":"Mon Oct 9 11:01:50 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":1.7170000000000001,"15min":1.72},"min":{"1min":0.33200000000000002,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":4.9429999999999996,"5min":19.923999999999999,"15min":19.923999999999999},"last":1.0009999999999999},{"interface":"front","average":{"1min":1.677,"5min":1.5489999999999999,"15min":1.585},"min":{"1min":0.34599999999999997,"5min":0.28799999999999998,"15min":0.255},"max":{"1min":8.6509999999999998,"5min":19.859999999999999,"15min":19.859999999999999},"last":1.0609999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790292,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2764840,"kb_used_data":1866208,"kb_used_omap":0,"kb_used_meta":898624,"kb_avail":90980312,"statfs":{"total":95995035648,"available":93163839488,"internally_reserved":0,"allocated":1910996992,"data_stored":1909634186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920190976},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:09 2023","interfaces":[{"interface":"back","average":{"1min":4.931,"5min":2.4180000000000001,"15min":2.1890000000000001},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.215},"max":{"1min":58.271999999999998,"5min":58.271999999999998,"15min":58.271999999999998},"last":0.51000000000000001},{"interface":"front","average":{"1min":5,"5min":2.4689999999999999,"15min":2.2429999999999999},"min":{"1min":0.32600000000000001,"5min":0.30499999999999999,"15min":0.28599999999999998},"max":{"1min":58.25,"5min":58.25,"15min":58.25},"last":0.54000000000000004}]},{"osd":1,"last update":"Mon Oct 9 11:01:17 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.9890000000000001,"15min":1.9890000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.4570000000000007,"5min":9.4570000000000007,"15min":9.4570000000000007},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.6120000000000001,"5min":1.6120000000000001,"15min":1.6120000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.4450000000000003,"5min":9.4450000000000003,"15min":9.4450000000000003},"last":0.60699999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:01:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.758,"15min":2.0379999999999998},"min":{"1min":0.25800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":6.851,"5min":9.3539999999999992,"15min":11.393000000000001},"last":2.9710000000000001},{"interface":"front","average":{"1min":2.04,"5min":1.8200000000000001,"15min":1.954},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":6.7130000000000001,"5min":14.621,"15min":15.087999999999999},"last":2.8879999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:02:09 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":2.5449999999999999,"15min":2.5209999999999999},"min":{"1min":0.29699999999999999,"5min":0.28000000000000003,"15min":0.25700000000000001},"max":{"1min":11.590999999999999,"5min":32.810000000000002,"15min":63.508000000000003},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.8069999999999999,"5min":2.5699999999999998,"15min":2.702},"min":{"1min":0.33900000000000002,"5min":0.30199999999999999,"15min":0.26300000000000001},"max":{"1min":11.292999999999999,"5min":32.659999999999997,"15min":62.418999999999997},"last":3.488}]},{"osd":4,"last update":"Mon Oct 9 11:01:17 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.76800000000000002,"15min":1.4690000000000001},"min":{"1min":0.311,"5min":0.216,"15min":0.216},"max":{"1min":1.5700000000000001,"5min":4.8760000000000003,"15min":123.241},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.2789999999999999,"15min":1.835},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":6.4950000000000001,"5min":9.1869999999999994,"15min":123.318},"last":0.45500000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:01:31 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":1.0249999999999999,"15min":1.4910000000000001},"min":{"1min":0.30299999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.4429999999999996,"5min":8.4260000000000002,"15min":11.098000000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.87,"15min":1.3180000000000001},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.5840000000000001,"5min":5.899,"15min":20.256},"last":0.78900000000000003}]},{"osd":7,"last update":"Mon Oct 9 11:01:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.323,"15min":1.534},"min":{"1min":0.35599999999999998,"5min":0.29299999999999998,"15min":0.28699999999999998},"max":{"1min":3.4870000000000001,"5min":7.6680000000000001,"15min":10.739000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":1.1759999999999999,"15min":1.2989999999999999},"min":{"1min":0.379,"5min":0.315,"15min":0.30099999999999999},"max":{"1min":1.7,"5min":8.1430000000000007,"15min":14.154},"last":0.64700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3267952640,"data_stored":3267952640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1302396928,"data_stored":1302396928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1905721344,"data_stored":1905721344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1900478464,"data_stored":1900478464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2865233920,"data_stored":2865233920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1914568704,"data_stored":1914568704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2225602560,"data_stored":2225602560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":72220672,"data_stored":72220672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:02:28.752 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T11:02:28.752 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:02:28.752 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-09T11:02:28.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-09T11:02:28.813 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:28.895 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:28.907 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 70 16 84930 84914 75.7858 61.9375 0.0287181 0.0131888 2023-10-09T11:02:29.002 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:29.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:29.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:29.025 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T11:02:29.048 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:29.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:29.184 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:29.224 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:29.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:29.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:29.252 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:29 smithi125 ceph-mon[200197]: 1.15 scrub starts 2023-10-09T11:02:29.252 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:29 smithi125 ceph-mon[200197]: 1.15 scrub ok 2023-10-09T11:02:29.252 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:29 smithi125 ceph-mon[195507]: 1.15 scrub starts 2023-10-09T11:02:29.252 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:29 smithi125 ceph-mon[195507]: 1.15 scrub ok 2023-10-09T11:02:29.332 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:29.398 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:29.440 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:29.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:29.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:29.467 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:02:29.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:29.594 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:29 smithi155 ceph-mon[184854]: 1.15 scrub starts 2023-10-09T11:02:29.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:29 smithi155 ceph-mon[184854]: 1.15 scrub ok 2023-10-09T11:02:29.597 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:29.638 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:29.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:29.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:29.743 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:29.795 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:29.821 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:29.909 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 71 16 85843 85827 75.5221 57.0625 0.0306884 0.0132362 2023-10-09T11:02:29.934 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:29.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:29.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:30.039 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:30.120 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:30.197 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:30.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:30.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:30.302 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:30.355 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:30.403 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:30.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:30.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:30.506 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:30.558 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:30 smithi155 ceph-mon[184854]: pgmap v1071: 119 pgs: 119 active+clean; 5.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T11:02:30.558 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:02:30.568 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:30.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:30 smithi125 ceph-mon[195507]: pgmap v1071: 119 pgs: 119 active+clean; 5.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T11:02:30.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:02:30.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:30 smithi125 ceph-mon[200197]: pgmap v1071: 119 pgs: 119 active+clean; 5.1 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T11:02:30.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:02:30.609 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:02:30.609 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:02:30.615 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:30.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:30.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:30.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:30.805 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:30.833 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:30.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:30.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:30.908 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 72 16 86699 86683 75.2157 53.5 0.0110323 0.0132896 2023-10-09T11:02:30.948 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:30.987 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T11:02:31.010 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:31.022 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:02:31.085 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:31.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:31.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:31.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:31.218 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:02:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:02:31] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.43.0" 2023-10-09T11:02:31.266 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:31.301 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:31.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:31.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:31.316 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:02:31.354 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:02:31.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:31.448 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:31.494 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:31.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:31.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:31.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1180649303' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:31.558 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1180649303' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:31.558 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1180649303' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:31.600 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:31.654 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:31.742 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:31.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:31.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:31.770 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:31.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T11:02:31.601+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e433 definitely_dead 0 2023-10-09T11:02:31.846 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:31.912 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 73 16 87548 87532 74.9083 53.0625 0.00940349 0.0133449 2023-10-09T11:02:31.917 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:31.958 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:31.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:31.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:32.071 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:32.148 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:32.268 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:02:32.268 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T11:02:32.268 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[195507]: failed to encode map e433 with expected crc 2023-10-09T11:02:32.268 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1180649303' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:02:32.268 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[195507]: osdmap e433: 8 total, 8 up, 7 in 2023-10-09T11:02:32.268 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/540953321' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T11:02:32.269 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T11:02:32.269 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[195507]: pgmap v1073: 119 pgs: 119 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 57 MiB/s wr, 905 op/s 2023-10-09T11:02:32.297 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:32.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:32.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:32.315 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.7. 2023-10-09T11:02:32.402 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:32.434 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:32.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:32.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:32.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:32.483 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:02:32.483 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:02:32.523 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:02:32.523 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T11:02:32.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[200197]: failed to encode map e433 with expected crc 2023-10-09T11:02:32.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1180649303' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:02:32.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[200197]: osdmap e433: 8 total, 8 up, 7 in 2023-10-09T11:02:32.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/540953321' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T11:02:32.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T11:02:32.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:32 smithi125 ceph-mon[200197]: pgmap v1073: 119 pgs: 119 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 57 MiB/s wr, 905 op/s 2023-10-09T11:02:32.555 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:32.581 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:32 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:02:32.582 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:32 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T11:02:32.582 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:32 smithi155 ceph-mon[184854]: failed to encode map e433 with expected crc 2023-10-09T11:02:32.582 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:32 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1180649303' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:02:32.582 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:32 smithi155 ceph-mon[184854]: osdmap e433: 8 total, 8 up, 7 in 2023-10-09T11:02:32.582 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:32 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/540953321' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T11:02:32.582 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:32 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T11:02:32.582 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:32 smithi155 ceph-mon[184854]: pgmap v1073: 119 pgs: 119 active+clean; 5.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 57 MiB/s wr, 905 op/s 2023-10-09T11:02:32.584 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:32.653 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:32.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:32.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:32.757 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:32.786 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:32.857 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:32.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:32.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:32.862 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-09T11:02:32.886 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T11:02:32.912 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 74 16 87987 87971 74.267 27.4375 0.00326291 0.0133621 2023-10-09T11:02:32.961 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:32.989 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:33.008 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:33.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:33.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:33.062 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:02:33.080 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:02:33.098 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:02:33.111 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:33.117 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:02:33.135 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:02:33.139 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:33.147 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:02:33.154 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:02:33.161 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:33.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:33.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:33.182 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:02:33.232 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:02:33.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:02:33.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:02:33.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:02:33.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:02:33.234 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:02:33.234 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:02:33.234 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:02:33.234 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:02:33.264 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:33.297 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:33.318 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:33.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:33.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:33.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:33.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:33 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T11:02:33.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:33 smithi155 ceph-mon[184854]: osdmap e434: 8 total, 8 up, 8 in 2023-10-09T11:02:33.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:33 smithi155 ceph-mon[184854]: failed to encode map e434 with expected crc 2023-10-09T11:02:33.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:33 smithi155 ceph-mon[184854]: failed to encode map e434 with expected crc 2023-10-09T11:02:33.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:33 smithi155 ceph-mon[184854]: failed to encode map e434 with expected crc 2023-10-09T11:02:33.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:33 smithi155 ceph-mon[184854]: failed to encode map e434 with expected crc 2023-10-09T11:02:33.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:33 smithi155 ceph-mon[184854]: failed to encode map e434 with expected crc 2023-10-09T11:02:33.448 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:33 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1619752073' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:33.448 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:33 smithi155 ceph-mon[184854]: osdmap e435: 8 total, 8 up, 8 in 2023-10-09T11:02:33.449 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:33.569 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T11:02:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[195507]: osdmap e434: 8 total, 8 up, 8 in 2023-10-09T11:02:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[195507]: failed to encode map e434 with expected crc 2023-10-09T11:02:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[195507]: failed to encode map e434 with expected crc 2023-10-09T11:02:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[195507]: failed to encode map e434 with expected crc 2023-10-09T11:02:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[195507]: failed to encode map e434 with expected crc 2023-10-09T11:02:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[195507]: failed to encode map e434 with expected crc 2023-10-09T11:02:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1619752073' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:33.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[195507]: osdmap e435: 8 total, 8 up, 8 in 2023-10-09T11:02:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T11:02:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[200197]: osdmap e434: 8 total, 8 up, 8 in 2023-10-09T11:02:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[200197]: failed to encode map e434 with expected crc 2023-10-09T11:02:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[200197]: failed to encode map e434 with expected crc 2023-10-09T11:02:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[200197]: failed to encode map e434 with expected crc 2023-10-09T11:02:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[200197]: failed to encode map e434 with expected crc 2023-10-09T11:02:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[200197]: failed to encode map e434 with expected crc 2023-10-09T11:02:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1619752073' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:33 smithi125 ceph-mon[200197]: osdmap e435: 8 total, 8 up, 8 in 2023-10-09T11:02:33.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:33.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:33.673 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:33.700 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:33.756 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:02:33.769 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:33.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:33.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:33.873 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:33.901 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:33.912 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 75 16 87987 87971 73.2771 0 - 0.0133621 2023-10-09T11:02:33.920 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:33.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:33.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:34.026 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:34.027 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:34.055 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:34.089 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:34.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:34.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:34.136 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:02:34.136 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T11:02:34.136 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T11:02:34.136 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:02:34.136 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.137 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122023 9.d 9:b63f6ab3:::benchmark_data_smithi125_363789_object11724:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:02:34.137 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.154825+0000", 2023-10-09T11:02:34.137 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 205.97877467699999, 2023-10-09T11:02:34.137 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0345569450000001, 2023-10-09T11:02:34.137 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.137 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:02:34.137 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.137 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.137 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.137 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122023 2023-10-09T11:02:34.137 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.137 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.138 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.138 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.138 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154825+0000", 2023-10-09T11:02:34.138 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.138 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.138 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.138 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.138 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154825+0000", 2023-10-09T11:02:34.138 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.138 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.138 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.139 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.139 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154823+0000", 2023-10-09T11:02:34.139 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:02:34.139 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.139 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.139 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.139 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154829+0000", 2023-10-09T11:02:34.139 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6269999999999997e-06 2023-10-09T11:02:34.139 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.139 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.139 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.139 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154842+0000", 2023-10-09T11:02:34.139 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3869999999999999e-05 2023-10-09T11:02:34.140 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.140 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.140 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.140 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154849+0000", 2023-10-09T11:02:34.140 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5599999999999999e-06 2023-10-09T11:02:34.140 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.140 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.140 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.140 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154882+0000", 2023-10-09T11:02:34.140 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2580999999999999e-05 2023-10-09T11:02:34.140 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.140 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.141 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.141 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154977+0000", 2023-10-09T11:02:34.141 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5298999999999997e-05 2023-10-09T11:02:34.141 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.141 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.141 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,5", 2023-10-09T11:02:34.141 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.155058+0000", 2023-10-09T11:02:34.141 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1518000000000001e-05 2023-10-09T11:02:34.141 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.141 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.141 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.141 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.161278+0000", 2023-10-09T11:02:34.141 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062194049999999999 2023-10-09T11:02:34.142 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.142 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.142 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.142 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.162677+0000", 2023-10-09T11:02:34.142 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0013996379999999999 2023-10-09T11:02:34.142 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.142 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.142 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.142 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.180908+0000", 2023-10-09T11:02:34.142 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018230321000000001 2023-10-09T11:02:34.142 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.142 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.143 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.143 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.180916+0000", 2023-10-09T11:02:34.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7170000000000008e-06 2023-10-09T11:02:34.143 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.143 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.143 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.143 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.189335+0000", 2023-10-09T11:02:34.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.00841958 2023-10-09T11:02:34.143 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.143 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.143 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.143 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.189381+0000", 2023-10-09T11:02:34.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6349e-05 2023-10-09T11:02:34.144 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.144 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.144 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.144 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.144 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.144 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122031 9.3 9:cb1920df:::benchmark_data_smithi125_363789_object11732:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:02:34.144 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.154885+0000", 2023-10-09T11:02:34.144 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 205.97871377800001, 2023-10-09T11:02:34.144 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0272100049999999, 2023-10-09T11:02:34.144 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.144 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:02:34.144 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.144 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.145 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.145 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122031 2023-10-09T11:02:34.145 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.145 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.145 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.145 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.145 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154885+0000", 2023-10-09T11:02:34.145 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.145 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.145 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.145 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.145 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154885+0000", 2023-10-09T11:02:34.145 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.146 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.146 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.146 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.146 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154885+0000", 2023-10-09T11:02:34.146 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:34.146 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.146 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.146 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.146 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154888+0000", 2023-10-09T11:02:34.146 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6879999999999999e-06 2023-10-09T11:02:34.146 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.146 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.147 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.147 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154894+0000", 2023-10-09T11:02:34.147 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.823e-06 2023-10-09T11:02:34.147 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.147 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.147 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.147 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154897+0000", 2023-10-09T11:02:34.147 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8659999999999999e-06 2023-10-09T11:02:34.147 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.147 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.147 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.147 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154941+0000", 2023-10-09T11:02:34.147 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3460000000000001e-05 2023-10-09T11:02:34.148 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.148 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.148 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.148 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.155049+0000", 2023-10-09T11:02:34.148 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000107932 2023-10-09T11:02:34.148 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.148 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.148 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:02:34.148 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.155156+0000", 2023-10-09T11:02:34.148 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010671300000000001 2023-10-09T11:02:34.148 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.148 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.148 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.149 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.161290+0000", 2023-10-09T11:02:34.149 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061342480000000001 2023-10-09T11:02:34.149 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.149 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.149 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.149 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178638+0000", 2023-10-09T11:02:34.149 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017348484000000001 2023-10-09T11:02:34.149 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.149 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.149 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.149 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178645+0000", 2023-10-09T11:02:34.149 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.613e-06 2023-10-09T11:02:34.150 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.150 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.150 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.150 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.182021+0000", 2023-10-09T11:02:34.150 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.003375959 2023-10-09T11:02:34.150 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.150 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.151 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.151 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.182095+0000", 2023-10-09T11:02:34.151 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4606999999999999e-05 2023-10-09T11:02:34.151 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.151 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.151 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.151 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.151 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.152 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122032 9.a 9:5ee579a3:::benchmark_data_smithi125_363789_object11733:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:02:34.152 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.163592+0000", 2023-10-09T11:02:34.152 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 205.97000718699999, 2023-10-09T11:02:34.152 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.03044172, 2023-10-09T11:02:34.152 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.152 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:02:34.152 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.152 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.152 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.152 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122032 2023-10-09T11:02:34.152 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.153 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.153 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.153 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.153 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163592+0000", 2023-10-09T11:02:34.153 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.153 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.153 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.153 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.153 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163592+0000", 2023-10-09T11:02:34.153 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.153 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.153 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.153 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.154 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163590+0000", 2023-10-09T11:02:34.154 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T11:02:34.154 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.154 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.154 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.154 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163601+0000", 2023-10-09T11:02:34.154 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1037e-05 2023-10-09T11:02:34.154 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.154 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.154 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.154 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163614+0000", 2023-10-09T11:02:34.155 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3001e-05 2023-10-09T11:02:34.155 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.155 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.155 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.155 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163621+0000", 2023-10-09T11:02:34.155 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7270000000000001e-06 2023-10-09T11:02:34.155 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.155 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.155 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.155 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.164646+0000", 2023-10-09T11:02:34.155 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010251909999999999 2023-10-09T11:02:34.155 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.156 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.156 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op must wait for map", 2023-10-09T11:02:34.156 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.164654+0000", 2023-10-09T11:02:34.156 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5059999999999998e-06 2023-10-09T11:02:34.156 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.156 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.156 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.156 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178523+0000", 2023-10-09T11:02:34.156 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013869365999999999 2023-10-09T11:02:34.156 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.156 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.156 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.157 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178531+0000", 2023-10-09T11:02:34.157 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7510000000000008e-06 2023-10-09T11:02:34.157 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.157 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.157 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.157 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.192721+0000", 2023-10-09T11:02:34.157 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.014189995 2023-10-09T11:02:34.157 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.157 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.157 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.157 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.194034+0000", 2023-10-09T11:02:34.157 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001312917 2023-10-09T11:02:34.157 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.158 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.158 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.158 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.158 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.158 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122035 9.3 9:cf84482a:::benchmark_data_smithi125_363789_object11736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:02:34.158 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.176778+0000", 2023-10-09T11:02:34.158 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 205.95682098399999, 2023-10-09T11:02:34.158 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0230526680000001, 2023-10-09T11:02:34.158 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.158 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.158 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.159 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.159 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.159 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122035 2023-10-09T11:02:34.159 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.159 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.159 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.159 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.159 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176778+0000", 2023-10-09T11:02:34.159 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.159 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.159 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.159 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.160 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176778+0000", 2023-10-09T11:02:34.160 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.160 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.160 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.160 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.160 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176777+0000", 2023-10-09T11:02:34.160 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:02:34.160 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.160 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.160 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.160 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176782+0000", 2023-10-09T11:02:34.160 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.374e-06 2023-10-09T11:02:34.161 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.161 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.161 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.161 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176790+0000", 2023-10-09T11:02:34.161 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2199999999999992e-06 2023-10-09T11:02:34.161 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.161 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.161 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.161 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176795+0000", 2023-10-09T11:02:34.161 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5099999999999998e-06 2023-10-09T11:02:34.161 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.161 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.162 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.162 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178896+0000", 2023-10-09T11:02:34.162 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021008910000000001 2023-10-09T11:02:34.162 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.162 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.162 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.162 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178902+0000", 2023-10-09T11:02:34.162 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0800000000000002e-06 2023-10-09T11:02:34.162 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.162 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.162 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.162 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.182122+0000", 2023-10-09T11:02:34.163 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.003219329 2023-10-09T11:02:34.163 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.163 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.163 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.163 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.182244+0000", 2023-10-09T11:02:34.163 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012187 2023-10-09T11:02:34.163 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.163 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.163 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3", 2023-10-09T11:02:34.163 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.182386+0000", 2023-10-09T11:02:34.163 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000142373 2023-10-09T11:02:34.163 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.164 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.164 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.164 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.190416+0000", 2023-10-09T11:02:34.164 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0080298230000000002 2023-10-09T11:02:34.164 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.164 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.164 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.164 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199736+0000", 2023-10-09T11:02:34.164 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0093200999999999996 2023-10-09T11:02:34.164 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.164 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.164 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.164 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199767+0000", 2023-10-09T11:02:34.165 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1440999999999999e-05 2023-10-09T11:02:34.165 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.165 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.165 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.165 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199831+0000", 2023-10-09T11:02:34.165 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3633999999999994e-05 2023-10-09T11:02:34.165 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.165 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.165 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.165 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.165 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.165 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122040 9.3 9:cc3a16b1:::benchmark_data_smithi125_363789_object11741:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:02:34.166 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.187372+0000", 2023-10-09T11:02:34.166 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 205.94622709399999, 2023-10-09T11:02:34.166 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.012574294, 2023-10-09T11:02:34.166 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.166 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.166 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.166 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.166 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.166 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122040 2023-10-09T11:02:34.166 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.166 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.166 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.167 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.167 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187372+0000", 2023-10-09T11:02:34.167 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.167 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.167 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.167 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.167 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187372+0000", 2023-10-09T11:02:34.167 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.167 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.167 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.167 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.167 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187371+0000", 2023-10-09T11:02:34.167 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:02:34.168 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.168 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.168 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.168 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187375+0000", 2023-10-09T11:02:34.168 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6299999999999997e-06 2023-10-09T11:02:34.168 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.168 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.168 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.168 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187385+0000", 2023-10-09T11:02:34.168 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9709999999999999e-06 2023-10-09T11:02:34.168 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.168 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.169 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.169 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187392+0000", 2023-10-09T11:02:34.169 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7299999999999999e-06 2023-10-09T11:02:34.169 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.169 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.169 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.169 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187440+0000", 2023-10-09T11:02:34.169 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7558000000000001e-05 2023-10-09T11:02:34.169 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.169 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.169 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.169 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187449+0000", 2023-10-09T11:02:34.169 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3740000000000004e-06 2023-10-09T11:02:34.170 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.170 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.170 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.170 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.185252+0000", 2023-10-09T11:02:34.170 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.99780342499999997 2023-10-09T11:02:34.170 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.170 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.170 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.170 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.185378+0000", 2023-10-09T11:02:34.170 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012527 2023-10-09T11:02:34.170 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.170 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.171 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3", 2023-10-09T11:02:34.171 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.185510+0000", 2023-10-09T11:02:34.171 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013254599999999999 2023-10-09T11:02:34.171 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.171 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.171 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.171 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.190604+0000", 2023-10-09T11:02:34.171 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0050936460000000003 2023-10-09T11:02:34.171 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.171 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.171 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.171 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199865+0000", 2023-10-09T11:02:34.172 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0092616620000000004 2023-10-09T11:02:34.172 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.172 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.172 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.172 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199897+0000", 2023-10-09T11:02:34.172 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1223000000000002e-05 2023-10-09T11:02:34.172 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.172 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.172 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.172 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199946+0000", 2023-10-09T11:02:34.172 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9728999999999998e-05 2023-10-09T11:02:34.172 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.172 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.173 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.173 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.173 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.173 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122046 9.3 9:cc593156:::benchmark_data_smithi125_363789_object11747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:02:34.173 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.201649+0000", 2023-10-09T11:02:34.173 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 205.93195064599999, 2023-10-09T11:02:34.173 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.010992307, 2023-10-09T11:02:34.173 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.173 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.173 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.173 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.174 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.174 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122046 2023-10-09T11:02:34.174 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.174 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.174 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.174 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.174 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201649+0000", 2023-10-09T11:02:34.174 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.174 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.174 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.174 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.174 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201649+0000", 2023-10-09T11:02:34.175 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.175 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.175 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.175 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.175 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201647+0000", 2023-10-09T11:02:34.175 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:02:34.175 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.175 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.175 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.175 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201653+0000", 2023-10-09T11:02:34.175 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1840000000000004e-06 2023-10-09T11:02:34.175 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.175 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.176 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.176 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201665+0000", 2023-10-09T11:02:34.176 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1396999999999999e-05 2023-10-09T11:02:34.176 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.176 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.176 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.176 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201671+0000", 2023-10-09T11:02:34.176 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3849999999999999e-06 2023-10-09T11:02:34.176 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.176 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.176 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.176 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201716+0000", 2023-10-09T11:02:34.177 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4969999999999998e-05 2023-10-09T11:02:34.177 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.177 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.177 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.177 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201725+0000", 2023-10-09T11:02:34.177 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3589999999999997e-06 2023-10-09T11:02:34.177 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.177 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.177 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.177 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.187839+0000", 2023-10-09T11:02:34.177 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98611354799999995 2023-10-09T11:02:34.177 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.177 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.178 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.178 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.187980+0000", 2023-10-09T11:02:34.178 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014071799999999999 2023-10-09T11:02:34.178 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.178 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.178 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3", 2023-10-09T11:02:34.178 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.188118+0000", 2023-10-09T11:02:34.178 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013808599999999999 2023-10-09T11:02:34.178 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.178 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.178 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.178 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.190608+0000", 2023-10-09T11:02:34.179 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00249037 2023-10-09T11:02:34.179 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.179 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.179 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.179 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.212484+0000", 2023-10-09T11:02:34.179 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021876442999999999 2023-10-09T11:02:34.179 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.179 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.179 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.179 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.212541+0000", 2023-10-09T11:02:34.179 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6255999999999998e-05 2023-10-09T11:02:34.179 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.180 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.180 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.180 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.212641+0000", 2023-10-09T11:02:34.180 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010011200000000001 2023-10-09T11:02:34.180 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.180 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.180 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.180 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.180 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.180 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150423 9.9 9:91f76946:::benchmark_data_smithi125_363789_object40124:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:02:34.180 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.193246+0000", 2023-10-09T11:02:34.180 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 170.940353187, 2023-10-09T11:02:34.181 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93784610000000002, 2023-10-09T11:02:34.181 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.181 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.181 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.181 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.181 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.181 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150423 2023-10-09T11:02:34.181 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.181 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.181 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.181 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.181 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193246+0000", 2023-10-09T11:02:34.181 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.182 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.182 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.182 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.182 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193246+0000", 2023-10-09T11:02:34.182 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.182 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.182 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.182 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.182 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193245+0000", 2023-10-09T11:02:34.182 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:34.182 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.182 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.183 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.183 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193249+0000", 2023-10-09T11:02:34.183 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2279999999999998e-06 2023-10-09T11:02:34.183 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.183 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.183 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.183 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193256+0000", 2023-10-09T11:02:34.183 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1010000000000003e-06 2023-10-09T11:02:34.183 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.183 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.183 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.183 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193260+0000", 2023-10-09T11:02:34.183 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3409999999999996e-06 2023-10-09T11:02:34.184 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.184 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.184 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.184 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193306+0000", 2023-10-09T11:02:34.184 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5636000000000001e-05 2023-10-09T11:02:34.184 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.184 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.184 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.184 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193313+0000", 2023-10-09T11:02:34.184 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0849999999999999e-06 2023-10-09T11:02:34.184 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.184 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.185 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.185 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.121393+0000", 2023-10-09T11:02:34.185 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92808017799999998 2023-10-09T11:02:34.185 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.185 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.185 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.185 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.121556+0000", 2023-10-09T11:02:34.185 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016344600000000001 2023-10-09T11:02:34.185 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.185 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.185 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:02:34.185 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.121862+0000", 2023-10-09T11:02:34.186 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030565099999999999 2023-10-09T11:02:34.186 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.186 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.186 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.186 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.126975+0000", 2023-10-09T11:02:34.186 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051129119999999998 2023-10-09T11:02:34.186 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.186 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.186 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.186 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.130983+0000", 2023-10-09T11:02:34.186 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040081470000000001 2023-10-09T11:02:34.186 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.187 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.187 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.187 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131014+0000", 2023-10-09T11:02:34.187 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0698000000000003e-05 2023-10-09T11:02:34.187 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.187 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.187 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.187 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131055+0000", 2023-10-09T11:02:34.187 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1390999999999997e-05 2023-10-09T11:02:34.187 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.187 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.187 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.188 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131092+0000", 2023-10-09T11:02:34.188 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6896e-05 2023-10-09T11:02:34.188 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.188 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.188 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.188 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.188 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.188 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150426 9.9 9:911979c4:::benchmark_data_smithi125_363789_object40127:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:02:34.188 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.193290+0000", 2023-10-09T11:02:34.188 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 170.940309676, 2023-10-09T11:02:34.188 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93976706099999996, 2023-10-09T11:02:34.188 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.189 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.189 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.189 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.189 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.189 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150426 2023-10-09T11:02:34.189 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.189 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.189 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.189 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.189 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193290+0000", 2023-10-09T11:02:34.189 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.189 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.190 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.190 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.190 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193290+0000", 2023-10-09T11:02:34.190 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.190 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.190 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.190 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.190 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193289+0000", 2023-10-09T11:02:34.190 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:34.190 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.190 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.190 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.191 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193291+0000", 2023-10-09T11:02:34.191 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2649999999999999e-06 2023-10-09T11:02:34.191 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.191 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.191 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.191 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193297+0000", 2023-10-09T11:02:34.191 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5210000000000002e-06 2023-10-09T11:02:34.191 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.191 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.191 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.191 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193301+0000", 2023-10-09T11:02:34.191 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7730000000000001e-06 2023-10-09T11:02:34.191 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.192 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.192 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.192 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193345+0000", 2023-10-09T11:02:34.192 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3959999999999999e-05 2023-10-09T11:02:34.192 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.192 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.192 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.192 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193350+0000", 2023-10-09T11:02:34.192 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9949999999999996e-06 2023-10-09T11:02:34.192 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.192 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.192 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.193 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.123175+0000", 2023-10-09T11:02:34.193 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92982503900000002 2023-10-09T11:02:34.193 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.193 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.193 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.193 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.123286+0000", 2023-10-09T11:02:34.193 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000111036 2023-10-09T11:02:34.193 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.193 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.193 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:02:34.193 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.123498+0000", 2023-10-09T11:02:34.193 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00021212800000000001 2023-10-09T11:02:34.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.194 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.194 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131127+0000", 2023-10-09T11:02:34.194 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0076286820000000003 2023-10-09T11:02:34.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.194 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.194 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131182+0000", 2023-10-09T11:02:34.194 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5912999999999998e-05 2023-10-09T11:02:34.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.194 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.195 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.132971+0000", 2023-10-09T11:02:34.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001788191 2023-10-09T11:02:34.195 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.195 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133015+0000", 2023-10-09T11:02:34.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4014000000000002e-05 2023-10-09T11:02:34.195 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.195 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133057+0000", 2023-10-09T11:02:34.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.193e-05 2023-10-09T11:02:34.196 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.196 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.196 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.196 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.196 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.196 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150439 9.9 9:92459488:::benchmark_data_smithi125_363789_object40140:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:02:34.196 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.207668+0000", 2023-10-09T11:02:34.196 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 170.92593156699999, 2023-10-09T11:02:34.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92585638599999998, 2023-10-09T11:02:34.196 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.196 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.196 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.197 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.197 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.197 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150439 2023-10-09T11:02:34.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.197 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.197 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.197 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207668+0000", 2023-10-09T11:02:34.197 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.197 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.198 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207668+0000", 2023-10-09T11:02:34.198 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.198 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.198 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.198 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.198 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207667+0000", 2023-10-09T11:02:34.198 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:34.198 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.198 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.198 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.198 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207671+0000", 2023-10-09T11:02:34.198 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2339999999999996e-06 2023-10-09T11:02:34.199 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.199 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.199 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.199 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207680+0000", 2023-10-09T11:02:34.199 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1670000000000005e-06 2023-10-09T11:02:34.199 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.199 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.199 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.199 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207686+0000", 2023-10-09T11:02:34.199 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3720000000000004e-06 2023-10-09T11:02:34.199 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.200 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.200 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.200 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.208495+0000", 2023-10-09T11:02:34.200 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000809142 2023-10-09T11:02:34.200 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.200 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.200 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.200 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.208502+0000", 2023-10-09T11:02:34.200 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9210000000000003e-06 2023-10-09T11:02:34.200 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.200 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.200 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.201 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.124568+0000", 2023-10-09T11:02:34.201 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.91606632200000004 2023-10-09T11:02:34.201 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.201 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.124637+0000", 2023-10-09T11:02:34.201 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8700000000000003e-05 2023-10-09T11:02:34.201 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:02:34.201 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.124777+0000", 2023-10-09T11:02:34.201 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013975399999999999 2023-10-09T11:02:34.202 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.202 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.202 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131155+0000", 2023-10-09T11:02:34.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0063785200000000004 2023-10-09T11:02:34.202 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.202 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.202 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133065+0000", 2023-10-09T11:02:34.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019096849999999999 2023-10-09T11:02:34.202 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.203 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.203 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.203 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133440+0000", 2023-10-09T11:02:34.203 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00037491400000000001 2023-10-09T11:02:34.203 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.203 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.203 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.203 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133486+0000", 2023-10-09T11:02:34.203 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6683e-05 2023-10-09T11:02:34.203 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.203 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.203 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.204 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133524+0000", 2023-10-09T11:02:34.204 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7685000000000003e-05 2023-10-09T11:02:34.204 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.204 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.204 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.204 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.204 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.204 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150470 9.9 9:9415b14b:::benchmark_data_smithi125_363789_object40171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:02:34.204 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.229566+0000", 2023-10-09T11:02:34.204 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 170.904033164, 2023-10-09T11:02:34.204 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.90407114499999996, 2023-10-09T11:02:34.204 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.205 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.205 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.205 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.205 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.205 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150470 2023-10-09T11:02:34.205 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.205 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.205 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.205 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.205 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229566+0000", 2023-10-09T11:02:34.205 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.205 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.206 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.206 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.206 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229566+0000", 2023-10-09T11:02:34.206 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.206 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.206 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.206 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.206 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229566+0000", 2023-10-09T11:02:34.206 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:34.206 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.206 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.206 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.207 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229568+0000", 2023-10-09T11:02:34.207 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.943e-06 2023-10-09T11:02:34.207 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.207 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.207 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.207 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229574+0000", 2023-10-09T11:02:34.207 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.003e-06 2023-10-09T11:02:34.207 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.207 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.207 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.207 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229579+0000", 2023-10-09T11:02:34.207 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0790000000000002e-06 2023-10-09T11:02:34.207 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.208 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.208 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.208 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229656+0000", 2023-10-09T11:02:34.208 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6940999999999994e-05 2023-10-09T11:02:34.208 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.208 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.208 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.208 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229662+0000", 2023-10-09T11:02:34.208 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7020000000000004e-06 2023-10-09T11:02:34.208 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.208 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.208 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.209 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.125755+0000", 2023-10-09T11:02:34.209 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.89609301900000005 2023-10-09T11:02:34.209 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.209 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.209 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.209 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.125827+0000", 2023-10-09T11:02:34.209 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1804999999999994e-05 2023-10-09T11:02:34.209 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.209 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.209 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:02:34.209 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.125995+0000", 2023-10-09T11:02:34.209 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016823900000000001 2023-10-09T11:02:34.210 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.210 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.210 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.210 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131210+0000", 2023-10-09T11:02:34.210 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0052145810000000003 2023-10-09T11:02:34.210 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.210 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.210 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.210 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133073+0000", 2023-10-09T11:02:34.210 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018628869999999999 2023-10-09T11:02:34.210 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.210 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.211 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.211 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133562+0000", 2023-10-09T11:02:34.211 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00048968299999999998 2023-10-09T11:02:34.211 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.211 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.211 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.211 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133602+0000", 2023-10-09T11:02:34.211 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0022999999999999e-05 2023-10-09T11:02:34.211 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.211 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.211 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.211 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133637+0000", 2023-10-09T11:02:34.212 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4715999999999998e-05 2023-10-09T11:02:34.212 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.212 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.212 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.212 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.212 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.212 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150495 9.9 9:90b0d4cb:::benchmark_data_smithi125_363789_object40196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:02:34.212 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.248473+0000", 2023-10-09T11:02:34.212 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 170.88512585699999, 2023-10-09T11:02:34.212 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.88740656399999995, 2023-10-09T11:02:34.212 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.212 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.212 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.213 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.213 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.213 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150495 2023-10-09T11:02:34.213 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.213 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.213 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.213 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.213 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248473+0000", 2023-10-09T11:02:34.213 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.213 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.213 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.213 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.214 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248473+0000", 2023-10-09T11:02:34.214 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.214 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.214 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.214 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.214 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248472+0000", 2023-10-09T11:02:34.214 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:02:34.214 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.214 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.214 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.214 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248483+0000", 2023-10-09T11:02:34.214 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0905e-05 2023-10-09T11:02:34.215 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.215 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.215 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.215 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248494+0000", 2023-10-09T11:02:34.215 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1153e-05 2023-10-09T11:02:34.215 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.215 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.215 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.215 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248501+0000", 2023-10-09T11:02:34.215 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2570000000000004e-06 2023-10-09T11:02:34.215 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.215 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.216 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.216 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248547+0000", 2023-10-09T11:02:34.216 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6202999999999998e-05 2023-10-09T11:02:34.216 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.216 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.216 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.216 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248555+0000", 2023-10-09T11:02:34.216 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4530000000000002e-06 2023-10-09T11:02:34.216 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.216 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.216 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.216 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.127018+0000", 2023-10-09T11:02:34.217 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.878463302 2023-10-09T11:02:34.217 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.217 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.217 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.217 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.127131+0000", 2023-10-09T11:02:34.217 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011225 2023-10-09T11:02:34.217 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.217 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.217 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:02:34.217 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.127305+0000", 2023-10-09T11:02:34.217 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017452 2023-10-09T11:02:34.217 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.218 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.218 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.218 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131237+0000", 2023-10-09T11:02:34.218 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039315369999999997 2023-10-09T11:02:34.218 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.218 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.218 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.218 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133688+0000", 2023-10-09T11:02:34.218 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002451614 2023-10-09T11:02:34.218 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.218 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.218 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.218 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.135791+0000", 2023-10-09T11:02:34.219 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002102746 2023-10-09T11:02:34.219 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.219 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.219 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.219 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.135837+0000", 2023-10-09T11:02:34.219 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5710000000000001e-05 2023-10-09T11:02:34.219 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.219 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.219 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.219 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.135880+0000", 2023-10-09T11:02:34.219 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3037999999999997e-05 2023-10-09T11:02:34.219 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.220 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.220 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.220 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.220 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.220 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150552 9.9 9:90f11fdc:::benchmark_data_smithi125_363789_object40253:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:02:34.220 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.294722+0000", 2023-10-09T11:02:34.220 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 170.838877123, 2023-10-09T11:02:34.220 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.84415905599999996, 2023-10-09T11:02:34.220 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.220 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.220 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.220 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.221 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.221 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150552 2023-10-09T11:02:34.221 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.221 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.221 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.221 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.221 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294722+0000", 2023-10-09T11:02:34.221 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.221 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.221 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.221 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.221 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294722+0000", 2023-10-09T11:02:34.222 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.222 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.222 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.222 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.222 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294721+0000", 2023-10-09T11:02:34.222 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:02:34.222 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.222 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.222 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.222 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294727+0000", 2023-10-09T11:02:34.222 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5539999999999997e-06 2023-10-09T11:02:34.222 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.222 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.223 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.223 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294734+0000", 2023-10-09T11:02:34.223 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2389999999999996e-06 2023-10-09T11:02:34.223 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.223 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.223 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.223 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294739+0000", 2023-10-09T11:02:34.223 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2279999999999998e-06 2023-10-09T11:02:34.223 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.223 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.223 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.223 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294780+0000", 2023-10-09T11:02:34.224 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1128000000000003e-05 2023-10-09T11:02:34.224 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.224 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.224 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.224 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294788+0000", 2023-10-09T11:02:34.224 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4499999999999998e-06 2023-10-09T11:02:34.224 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.224 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.224 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.224 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.128120+0000", 2023-10-09T11:02:34.224 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.83333184999999999 2023-10-09T11:02:34.224 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.224 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.225 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.225 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.128227+0000", 2023-10-09T11:02:34.225 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010778200000000001 2023-10-09T11:02:34.225 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.225 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.225 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:02:34.225 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.128411+0000", 2023-10-09T11:02:34.225 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000183117 2023-10-09T11:02:34.225 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.225 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.225 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.225 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.132962+0000", 2023-10-09T11:02:34.226 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045515520000000004 2023-10-09T11:02:34.226 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.226 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.226 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.226 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133727+0000", 2023-10-09T11:02:34.226 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00076511400000000001 2023-10-09T11:02:34.226 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.226 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.226 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.226 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138805+0000", 2023-10-09T11:02:34.226 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0050773340000000002 2023-10-09T11:02:34.226 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.226 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.227 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.227 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138840+0000", 2023-10-09T11:02:34.227 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5831000000000003e-05 2023-10-09T11:02:34.227 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.227 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.227 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.227 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138881+0000", 2023-10-09T11:02:34.227 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0710999999999997e-05 2023-10-09T11:02:34.227 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.227 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.227 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.228 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.228 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150590 9.9 9:909dc84e:::benchmark_data_smithi125_363789_object40291:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:02:34.228 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.333845+0000", 2023-10-09T11:02:34.228 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 170.79975372499999, 2023-10-09T11:02:34.228 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.80510336599999999, 2023-10-09T11:02:34.228 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.228 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.228 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.228 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.228 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.228 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150590 2023-10-09T11:02:34.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.229 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.229 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333845+0000", 2023-10-09T11:02:34.229 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.229 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.229 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333845+0000", 2023-10-09T11:02:34.229 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.229 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.230 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.230 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333845+0000", 2023-10-09T11:02:34.230 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:34.230 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.230 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.230 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.230 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333850+0000", 2023-10-09T11:02:34.230 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7879999999999997e-06 2023-10-09T11:02:34.230 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.230 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.230 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.230 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333858+0000", 2023-10-09T11:02:34.231 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7569999999999993e-06 2023-10-09T11:02:34.231 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.231 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.231 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.231 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333863+0000", 2023-10-09T11:02:34.231 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.724e-06 2023-10-09T11:02:34.231 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.231 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.231 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.231 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.335088+0000", 2023-10-09T11:02:34.231 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012250270000000001 2023-10-09T11:02:34.231 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.231 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.232 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.232 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.335095+0000", 2023-10-09T11:02:34.232 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7660000000000001e-06 2023-10-09T11:02:34.232 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.232 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.232 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.232 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.129225+0000", 2023-10-09T11:02:34.232 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.79412993399999998 2023-10-09T11:02:34.232 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.232 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.232 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.232 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.129291+0000", 2023-10-09T11:02:34.232 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6072e-05 2023-10-09T11:02:34.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.233 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.233 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:02:34.233 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.129474+0000", 2023-10-09T11:02:34.233 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018292800000000001 2023-10-09T11:02:34.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.233 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.233 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.233 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133175+0000", 2023-10-09T11:02:34.233 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0037009399999999998 2023-10-09T11:02:34.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.233 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.234 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.234 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.135784+0000", 2023-10-09T11:02:34.234 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026093900000000001 2023-10-09T11:02:34.234 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.234 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.234 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.234 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138888+0000", 2023-10-09T11:02:34.234 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0031041950000000001 2023-10-09T11:02:34.234 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.234 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.234 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.234 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138921+0000", 2023-10-09T11:02:34.235 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3025999999999999e-05 2023-10-09T11:02:34.235 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.235 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.235 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.235 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138949+0000", 2023-10-09T11:02:34.235 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7532999999999999e-05 2023-10-09T11:02:34.235 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.235 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.235 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.235 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.235 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.235 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204565 9.13 9:cfe54d63:::benchmark_data_smithi125_363789_object94266:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T11:02:34.235 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:41.994119+0000", 2023-10-09T11:02:34.236 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 112.139479911, 2023-10-09T11:02:34.236 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.018728254, 2023-10-09T11:02:34.236 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.236 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:02:34.236 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.236 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.236 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.236 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204565 2023-10-09T11:02:34.236 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.236 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.236 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.236 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.237 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994119+0000", 2023-10-09T11:02:34.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.237 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.237 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994119+0000", 2023-10-09T11:02:34.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.237 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.237 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994118+0000", 2023-10-09T11:02:34.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:02:34.237 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.238 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.238 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994124+0000", 2023-10-09T11:02:34.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6509999999999998e-06 2023-10-09T11:02:34.238 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.238 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.238 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994133+0000", 2023-10-09T11:02:34.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1109999999999997e-06 2023-10-09T11:02:34.238 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.238 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.238 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.239 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994137+0000", 2023-10-09T11:02:34.239 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5879999999999999e-06 2023-10-09T11:02:34.239 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.239 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.239 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.239 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994205+0000", 2023-10-09T11:02:34.239 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7852e-05 2023-10-09T11:02:34.239 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.239 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.239 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.239 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994355+0000", 2023-10-09T11:02:34.239 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015022700000000001 2023-10-09T11:02:34.240 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.240 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.240 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:02:34.240 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994806+0000", 2023-10-09T11:02:34.240 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00045023900000000001 2023-10-09T11:02:34.240 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.240 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.240 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.240 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.997059+0000", 2023-10-09T11:02:34.240 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022531949999999999 2023-10-09T11:02:34.240 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.240 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.241 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.241 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.003461+0000", 2023-10-09T11:02:34.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064024329999999999 2023-10-09T11:02:34.241 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.241 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.241 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.241 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.003480+0000", 2023-10-09T11:02:34.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8272000000000001e-05 2023-10-09T11:02:34.241 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.241 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.241 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.241 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.012782+0000", 2023-10-09T11:02:34.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0093026169999999 2023-10-09T11:02:34.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.242 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.242 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.242 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.012848+0000", 2023-10-09T11:02:34.242 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5415999999999998e-05 2023-10-09T11:02:34.242 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.242 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.242 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.242 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.242 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204572 9.13 9:cc352b3f:::benchmark_data_smithi125_363789_object94273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:02:34.242 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.008011+0000", 2023-10-09T11:02:34.243 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 112.125588474, 2023-10-09T11:02:34.243 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0262834300000001, 2023-10-09T11:02:34.243 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.243 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.243 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.243 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.243 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.243 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204572 2023-10-09T11:02:34.243 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.243 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.243 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.244 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008011+0000", 2023-10-09T11:02:34.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.244 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.244 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.244 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008011+0000", 2023-10-09T11:02:34.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.244 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.245 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.245 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008010+0000", 2023-10-09T11:02:34.245 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:34.245 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.245 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.245 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.245 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008014+0000", 2023-10-09T11:02:34.245 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9009999999999996e-06 2023-10-09T11:02:34.245 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.245 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.245 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.245 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008025+0000", 2023-10-09T11:02:34.246 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0730999999999999e-05 2023-10-09T11:02:34.246 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.246 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.246 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.246 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008032+0000", 2023-10-09T11:02:34.246 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9970000000000001e-06 2023-10-09T11:02:34.246 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.246 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.246 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.246 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008125+0000", 2023-10-09T11:02:34.246 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2991999999999993e-05 2023-10-09T11:02:34.247 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.247 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.247 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.247 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008136+0000", 2023-10-09T11:02:34.247 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1068e-05 2023-10-09T11:02:34.247 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.247 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.247 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.247 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.012867+0000", 2023-10-09T11:02:34.247 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.004731034 2023-10-09T11:02:34.247 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.247 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.247 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.248 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.012951+0000", 2023-10-09T11:02:34.248 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3851999999999996e-05 2023-10-09T11:02:34.248 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.248 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.248 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:02:34.248 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.013280+0000", 2023-10-09T11:02:34.248 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00032942300000000002 2023-10-09T11:02:34.248 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.248 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.248 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.248 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.021065+0000", 2023-10-09T11:02:34.248 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0077846759999999999 2023-10-09T11:02:34.249 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.249 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.249 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.249 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034119+0000", 2023-10-09T11:02:34.249 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013054412 2023-10-09T11:02:34.249 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.249 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.249 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.249 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034203+0000", 2023-10-09T11:02:34.249 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4079999999999995e-05 2023-10-09T11:02:34.249 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.249 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.250 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.250 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034252+0000", 2023-10-09T11:02:34.250 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8622999999999998e-05 2023-10-09T11:02:34.250 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.250 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.250 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.250 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034294+0000", 2023-10-09T11:02:34.250 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2256000000000003e-05 2023-10-09T11:02:34.250 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.250 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.250 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.250 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.251 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.251 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204591 9.13 9:cfb36959:::benchmark_data_smithi125_363789_object94292:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:02:34.251 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.043404+0000", 2023-10-09T11:02:34.251 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 112.090195424, 2023-10-09T11:02:34.251 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.990990861, 2023-10-09T11:02:34.251 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.251 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.251 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.251 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.251 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.251 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204591 2023-10-09T11:02:34.251 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.252 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.252 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.252 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.252 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043404+0000", 2023-10-09T11:02:34.252 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.252 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.252 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.252 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.252 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043404+0000", 2023-10-09T11:02:34.252 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.252 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.252 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.253 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.253 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043402+0000", 2023-10-09T11:02:34.253 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T11:02:34.253 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.253 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.253 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.253 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043412+0000", 2023-10-09T11:02:34.253 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0499000000000001e-05 2023-10-09T11:02:34.253 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.253 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.253 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.253 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043433+0000", 2023-10-09T11:02:34.253 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0477999999999999e-05 2023-10-09T11:02:34.254 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.254 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.254 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.254 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043442+0000", 2023-10-09T11:02:34.254 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9609999999999994e-06 2023-10-09T11:02:34.254 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.254 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.254 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.254 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043619+0000", 2023-10-09T11:02:34.254 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000177558 2023-10-09T11:02:34.254 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.254 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.255 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.255 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043630+0000", 2023-10-09T11:02:34.255 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0382e-05 2023-10-09T11:02:34.255 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.255 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.255 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.255 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.014803+0000", 2023-10-09T11:02:34.255 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97117335900000001 2023-10-09T11:02:34.255 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.255 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.255 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.255 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.014927+0000", 2023-10-09T11:02:34.256 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012421900000000001 2023-10-09T11:02:34.256 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.256 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.256 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:02:34.256 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.015207+0000", 2023-10-09T11:02:34.256 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00027950199999999999 2023-10-09T11:02:34.256 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.256 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.256 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.256 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.022320+0000", 2023-10-09T11:02:34.256 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0071137290000000001 2023-10-09T11:02:34.256 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.257 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.257 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.257 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034155+0000", 2023-10-09T11:02:34.257 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011834905 2023-10-09T11:02:34.257 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.257 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.257 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.257 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034329+0000", 2023-10-09T11:02:34.257 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017399200000000001 2023-10-09T11:02:34.257 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.257 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.257 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.257 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034361+0000", 2023-10-09T11:02:34.258 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1545999999999999e-05 2023-10-09T11:02:34.258 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.258 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.258 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.258 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034395+0000", 2023-10-09T11:02:34.258 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3772999999999997e-05 2023-10-09T11:02:34.258 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.258 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.258 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.258 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.258 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.258 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204618 9.13 9:c9e84990:::benchmark_data_smithi125_363789_object94319:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:02:34.259 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.073111+0000", 2023-10-09T11:02:34.259 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 112.060487752, 2023-10-09T11:02:34.259 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.96151479900000003, 2023-10-09T11:02:34.259 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.259 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.259 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.259 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.259 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.259 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204618 2023-10-09T11:02:34.259 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.259 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.259 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.260 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.260 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073111+0000", 2023-10-09T11:02:34.260 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.260 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.260 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.260 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.260 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073111+0000", 2023-10-09T11:02:34.260 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.260 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.260 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.260 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.260 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073110+0000", 2023-10-09T11:02:34.261 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:02:34.261 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.261 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.261 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.261 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073117+0000", 2023-10-09T11:02:34.261 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9820000000000002e-06 2023-10-09T11:02:34.261 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.261 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.261 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.261 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073129+0000", 2023-10-09T11:02:34.261 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1608e-05 2023-10-09T11:02:34.261 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.261 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.262 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.262 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073135+0000", 2023-10-09T11:02:34.262 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0249999999999999e-06 2023-10-09T11:02:34.262 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.262 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.262 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.262 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073191+0000", 2023-10-09T11:02:34.262 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5810999999999998e-05 2023-10-09T11:02:34.262 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.262 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.262 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.262 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073197+0000", 2023-10-09T11:02:34.263 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3579999999999999e-06 2023-10-09T11:02:34.263 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.263 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.263 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.263 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.016587+0000", 2023-10-09T11:02:34.263 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.94339012799999999 2023-10-09T11:02:34.263 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.263 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.263 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.263 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.016657+0000", 2023-10-09T11:02:34.263 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0301000000000003e-05 2023-10-09T11:02:34.263 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.263 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.264 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:02:34.264 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.016830+0000", 2023-10-09T11:02:34.264 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017296799999999999 2023-10-09T11:02:34.264 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.264 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.264 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.264 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.022324+0000", 2023-10-09T11:02:34.264 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0054936739999999996 2023-10-09T11:02:34.264 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.264 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.264 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.264 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034436+0000", 2023-10-09T11:02:34.265 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012112327000000001 2023-10-09T11:02:34.265 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.265 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.265 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.265 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034545+0000", 2023-10-09T11:02:34.265 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000108213 2023-10-09T11:02:34.265 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.265 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.265 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.265 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034587+0000", 2023-10-09T11:02:34.265 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2840000000000003e-05 2023-10-09T11:02:34.265 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.265 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.266 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.266 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034626+0000", 2023-10-09T11:02:34.266 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8868999999999997e-05 2023-10-09T11:02:34.266 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.266 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.266 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.266 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.266 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.266 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204624 9.13 9:cb2a155f:::benchmark_data_smithi125_363789_object94325:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:02:34.266 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.084343+0000", 2023-10-09T11:02:34.266 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 112.04925596, 2023-10-09T11:02:34.266 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95037137000000005, 2023-10-09T11:02:34.267 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.267 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.267 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.267 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.267 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.267 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204624 2023-10-09T11:02:34.267 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.267 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.267 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.267 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.267 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084343+0000", 2023-10-09T11:02:34.267 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.267 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.268 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.268 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.268 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084343+0000", 2023-10-09T11:02:34.268 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.268 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.268 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.268 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.268 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084340+0000", 2023-10-09T11:02:34.268 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999967 2023-10-09T11:02:34.268 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.268 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.268 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084352+0000", 2023-10-09T11:02:34.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1593999999999999e-05 2023-10-09T11:02:34.269 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.269 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084367+0000", 2023-10-09T11:02:34.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5668e-05 2023-10-09T11:02:34.269 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.269 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084373+0000", 2023-10-09T11:02:34.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5799999999999999e-06 2023-10-09T11:02:34.269 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.270 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.270 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.270 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084441+0000", 2023-10-09T11:02:34.270 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8324999999999994e-05 2023-10-09T11:02:34.270 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.270 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.270 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.270 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084453+0000", 2023-10-09T11:02:34.270 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1297999999999999e-05 2023-10-09T11:02:34.270 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.270 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.270 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.271 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.017927+0000", 2023-10-09T11:02:34.271 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93347489400000005 2023-10-09T11:02:34.271 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.271 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.271 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.271 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.018027+0000", 2023-10-09T11:02:34.271 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9511000000000003e-05 2023-10-09T11:02:34.271 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.271 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.271 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:02:34.271 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.018203+0000", 2023-10-09T11:02:34.271 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017629500000000001 2023-10-09T11:02:34.272 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.272 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.272 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.272 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.022327+0000", 2023-10-09T11:02:34.272 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041237510000000002 2023-10-09T11:02:34.272 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.272 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.272 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.272 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034475+0000", 2023-10-09T11:02:34.272 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012147857999999999 2023-10-09T11:02:34.272 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.272 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.273 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.273 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034657+0000", 2023-10-09T11:02:34.273 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000182504 2023-10-09T11:02:34.273 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.273 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.273 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.273 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034684+0000", 2023-10-09T11:02:34.273 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6936000000000001e-05 2023-10-09T11:02:34.273 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.273 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.273 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.273 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034715+0000", 2023-10-09T11:02:34.273 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0264e-05 2023-10-09T11:02:34.274 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.274 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.274 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.274 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.274 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.274 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204699 9.13 9:ca5a965b:::benchmark_data_smithi125_363789_object94400:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:02:34.274 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.182784+0000", 2023-10-09T11:02:34.274 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 111.950815189, 2023-10-09T11:02:34.274 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.85207209399999995, 2023-10-09T11:02:34.274 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.274 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.274 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.275 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.275 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.275 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204699 2023-10-09T11:02:34.275 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.275 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.275 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.275 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.275 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182784+0000", 2023-10-09T11:02:34.275 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.275 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.275 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.275 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.275 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182784+0000", 2023-10-09T11:02:34.276 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.276 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.276 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.276 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.276 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182783+0000", 2023-10-09T11:02:34.276 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:34.276 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.276 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.276 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.276 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182787+0000", 2023-10-09T11:02:34.276 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.377e-06 2023-10-09T11:02:34.276 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.277 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.277 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182797+0000", 2023-10-09T11:02:34.277 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9119999999999993e-06 2023-10-09T11:02:34.277 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.277 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.277 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182801+0000", 2023-10-09T11:02:34.277 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6480000000000002e-06 2023-10-09T11:02:34.277 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.277 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.277 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182860+0000", 2023-10-09T11:02:34.278 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8857000000000002e-05 2023-10-09T11:02:34.278 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.278 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.278 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.278 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182870+0000", 2023-10-09T11:02:34.278 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0223e-05 2023-10-09T11:02:34.278 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.278 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.278 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.278 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.019382+0000", 2023-10-09T11:02:34.278 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.83651138199999997 2023-10-09T11:02:34.278 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.279 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.279 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.279 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.019500+0000", 2023-10-09T11:02:34.279 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118187 2023-10-09T11:02:34.279 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.279 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.279 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:02:34.279 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.019722+0000", 2023-10-09T11:02:34.279 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022236099999999999 2023-10-09T11:02:34.279 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.279 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.279 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.279 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.023434+0000", 2023-10-09T11:02:34.280 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00371164 2023-10-09T11:02:34.280 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.280 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.280 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.280 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034764+0000", 2023-10-09T11:02:34.280 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011329824 2023-10-09T11:02:34.280 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.280 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.280 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.280 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034792+0000", 2023-10-09T11:02:34.280 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8123000000000002e-05 2023-10-09T11:02:34.280 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.281 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.281 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.281 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034817+0000", 2023-10-09T11:02:34.281 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5009999999999999e-05 2023-10-09T11:02:34.281 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.281 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.281 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.281 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034856+0000", 2023-10-09T11:02:34.281 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9178999999999999e-05 2023-10-09T11:02:34.281 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.281 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.281 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.281 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.282 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.282 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204804 9.13 9:cd6b9308:::benchmark_data_smithi125_363789_object94505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:02:34.282 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.320768+0000", 2023-10-09T11:02:34.282 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 111.812831453, 2023-10-09T11:02:34.282 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.71443022199999995, 2023-10-09T11:02:34.282 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:02:34.282 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:02:34.282 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:02:34.282 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:02:34.282 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:02:34.282 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204804 2023-10-09T11:02:34.282 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.282 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:02:34.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:02:34.283 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320768+0000", 2023-10-09T11:02:34.283 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:02:34.283 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320768+0000", 2023-10-09T11:02:34.283 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:02:34.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:02:34.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320767+0000", 2023-10-09T11:02:34.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:34.284 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.284 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:02:34.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320770+0000", 2023-10-09T11:02:34.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.768e-06 2023-10-09T11:02:34.284 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.284 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:02:34.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320777+0000", 2023-10-09T11:02:34.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.182e-06 2023-10-09T11:02:34.284 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.285 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:02:34.285 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320780+0000", 2023-10-09T11:02:34.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8890000000000002e-06 2023-10-09T11:02:34.285 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.285 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.285 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320856+0000", 2023-10-09T11:02:34.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5768999999999999e-05 2023-10-09T11:02:34.285 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.285 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:02:34.285 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320866+0000", 2023-10-09T11:02:34.286 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9410000000000002e-06 2023-10-09T11:02:34.286 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.286 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.286 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:02:34.286 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.021091+0000", 2023-10-09T11:02:34.286 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.70022501000000004 2023-10-09T11:02:34.286 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.286 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.286 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:02:34.286 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.021196+0000", 2023-10-09T11:02:34.286 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104948 2023-10-09T11:02:34.286 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.286 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.287 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:02:34.287 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.021416+0000", 2023-10-09T11:02:34.287 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022006799999999999 2023-10-09T11:02:34.287 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.287 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.287 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:02:34.287 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.024573+0000", 2023-10-09T11:02:34.287 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0031569430000000002 2023-10-09T11:02:34.287 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.287 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.287 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.287 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034901+0000", 2023-10-09T11:02:34.288 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010327935 2023-10-09T11:02:34.288 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.288 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.288 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:02:34.288 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.035121+0000", 2023-10-09T11:02:34.288 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00021983500000000001 2023-10-09T11:02:34.288 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.288 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.288 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:02:34.288 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.035163+0000", 2023-10-09T11:02:34.288 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2920000000000002e-05 2023-10-09T11:02:34.288 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:02:34.288 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:02:34.289 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:02:34.289 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.035198+0000", 2023-10-09T11:02:34.289 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4539999999999998e-05 2023-10-09T11:02:34.289 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.289 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.289 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.289 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:02:34.289 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:02:34.289 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:02:34.289 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:34.323 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:34.353 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:34.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:34.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:34.458 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:34.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[195507]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[195507]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[195507]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[195507]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[195507]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[195507]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[195507]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[195507]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[195507]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[195507]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[195507]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[195507]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[195507]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[195507]: pgmap v1076: 119 pgs: 7 peering, 112 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-09T11:02:34.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[195507]: osdmap e436: 8 total, 8 up, 8 in 2023-10-09T11:02:34.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[200197]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[200197]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[200197]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[200197]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[200197]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[200197]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[200197]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[200197]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[200197]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[200197]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[200197]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[200197]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[200197]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.503 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[200197]: pgmap v1076: 119 pgs: 7 peering, 112 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-09T11:02:34.504 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:34 smithi125 ceph-mon[200197]: osdmap e436: 8 total, 8 up, 8 in 2023-10-09T11:02:34.505 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:34.518 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T11:02:34.684 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:34.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:34.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:34.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:34 smithi155 ceph-mon[184854]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:34 smithi155 ceph-mon[184854]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:34 smithi155 ceph-mon[184854]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:34 smithi155 ceph-mon[184854]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:34 smithi155 ceph-mon[184854]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:34 smithi155 ceph-mon[184854]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:34 smithi155 ceph-mon[184854]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:34 smithi155 ceph-mon[184854]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:34 smithi155 ceph-mon[184854]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:34 smithi155 ceph-mon[184854]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:34 smithi155 ceph-mon[184854]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:34 smithi155 ceph-mon[184854]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:34 smithi155 ceph-mon[184854]: failed to encode map e435 with expected crc 2023-10-09T11:02:34.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:34 smithi155 ceph-mon[184854]: pgmap v1076: 119 pgs: 7 peering, 112 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-09T11:02:34.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:34 smithi155 ceph-mon[184854]: osdmap e436: 8 total, 8 up, 8 in 2023-10-09T11:02:34.791 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:34.836 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:34.875 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:34.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:34.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:34.912 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 76 16 88395 88379 72.6486 12.75 0.0139615 0.0137599 2023-10-09T11:02:34.982 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:35.037 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:35.064 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:35.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:35.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:35.168 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:35.176 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:35.253 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:35.329 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:35.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:35.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:35.433 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:35.500 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[195507]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.500 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[195507]: 10.1 starting backfill to osd.7 from (0'0,407'1135] MIN to 433'5561 2023-10-09T11:02:35.500 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[195507]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[195507]: 10.c starting backfill to osd.7 from (0'0,407'1135] MIN to 433'5543 2023-10-09T11:02:35.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[195507]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[195507]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[195507]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[195507]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[195507]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[195507]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[195507]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[195507]: osdmap e437: 8 total, 8 up, 8 in 2023-10-09T11:02:35.501 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[200197]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.501 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[200197]: 10.1 starting backfill to osd.7 from (0'0,407'1135] MIN to 433'5561 2023-10-09T11:02:35.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[200197]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[200197]: 10.c starting backfill to osd.7 from (0'0,407'1135] MIN to 433'5543 2023-10-09T11:02:35.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[200197]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[200197]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[200197]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[200197]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[200197]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[200197]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[200197]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:35 smithi125 ceph-mon[200197]: osdmap e437: 8 total, 8 up, 8 in 2023-10-09T11:02:35.504 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:35.585 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:02:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:35 smithi155 ceph-mon[184854]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:35 smithi155 ceph-mon[184854]: 10.1 starting backfill to osd.7 from (0'0,407'1135] MIN to 433'5561 2023-10-09T11:02:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:35 smithi155 ceph-mon[184854]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:35 smithi155 ceph-mon[184854]: 10.c starting backfill to osd.7 from (0'0,407'1135] MIN to 433'5543 2023-10-09T11:02:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:35 smithi155 ceph-mon[184854]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:35 smithi155 ceph-mon[184854]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:35 smithi155 ceph-mon[184854]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:35 smithi155 ceph-mon[184854]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:35 smithi155 ceph-mon[184854]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:35 smithi155 ceph-mon[184854]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:35 smithi155 ceph-mon[184854]: failed to encode map e436 with expected crc 2023-10-09T11:02:35.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:35 smithi155 ceph-mon[184854]: osdmap e437: 8 total, 8 up, 8 in 2023-10-09T11:02:35.894 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:35.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:35.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:35.913 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 77 16 89137 89121 72.3068 46.375 0.0167663 0.0138246 2023-10-09T11:02:35.997 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:36.049 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:36.232 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:02:36.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[195507]: failed to encode map e437 with expected crc 2023-10-09T11:02:36.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[195507]: failed to encode map e437 with expected crc 2023-10-09T11:02:36.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/893090500' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:02:36.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:02:36.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[195507]: pgmap v1079: 119 pgs: 7 peering, 112 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 19 MiB/s wr, 296 op/s 2023-10-09T11:02:36.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:36.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:02:36.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[195507]: osdmap e438: 8 total, 8 up, 8 in 2023-10-09T11:02:36.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[200197]: failed to encode map e437 with expected crc 2023-10-09T11:02:36.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[200197]: failed to encode map e437 with expected crc 2023-10-09T11:02:36.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/893090500' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:02:36.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:02:36.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[200197]: pgmap v1079: 119 pgs: 7 peering, 112 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 19 MiB/s wr, 296 op/s 2023-10-09T11:02:36.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:36.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:02:36.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:36 smithi125 ceph-mon[200197]: osdmap e438: 8 total, 8 up, 8 in 2023-10-09T11:02:36.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:36 smithi155 ceph-mon[184854]: failed to encode map e437 with expected crc 2023-10-09T11:02:36.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:36 smithi155 ceph-mon[184854]: failed to encode map e437 with expected crc 2023-10-09T11:02:36.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:36 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/893090500' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:02:36.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:36 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:02:36.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:36 smithi155 ceph-mon[184854]: pgmap v1079: 119 pgs: 7 peering, 112 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 19 MiB/s wr, 296 op/s 2023-10-09T11:02:36.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:36 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:36.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:36 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:02:36.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:36 smithi155 ceph-mon[184854]: osdmap e438: 8 total, 8 up, 8 in 2023-10-09T11:02:36.916 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 78 16 89967 89951 72.0424 51.875 0.0201744 0.0138752 2023-10-09T11:02:37.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:37 smithi155 ceph-mon[184854]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:37 smithi155 ceph-mon[184854]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:37 smithi155 ceph-mon[184854]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:37 smithi155 ceph-mon[184854]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:37 smithi155 ceph-mon[184854]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:37 smithi155 ceph-mon[184854]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:37 smithi155 ceph-mon[184854]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.769 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:37 smithi125 ceph-mon[195507]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.769 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:37 smithi125 ceph-mon[195507]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.769 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:37 smithi125 ceph-mon[195507]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.770 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:37 smithi125 ceph-mon[195507]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.770 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:37 smithi125 ceph-mon[195507]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.770 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:37 smithi125 ceph-mon[195507]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.770 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:37 smithi125 ceph-mon[195507]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.770 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:37 smithi125 ceph-mon[200197]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.770 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:37 smithi125 ceph-mon[200197]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.770 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:37 smithi125 ceph-mon[200197]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.770 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:37 smithi125 ceph-mon[200197]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:37 smithi125 ceph-mon[200197]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:37 smithi125 ceph-mon[200197]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:37 smithi125 ceph-mon[200197]: failed to encode map e438 with expected crc 2023-10-09T11:02:37.793 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:37.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:37.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:37.862 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 0, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T11:02:37.863 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:02:37.863 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-10-09T11:02:37.863 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:02:37.902 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:37.916 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 79 16 90818 90802 71.8038 53.1875 0.0276046 0.013921 2023-10-09T11:02:37.965 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:38.008 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:38.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:38.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:38.077 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:38.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:38.196 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:38.385 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:02:38.386 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:02:38.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.386 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:91117 10.b 10:d72815de:::benchmark_data_smithi125_402869_object91116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T11:02:38.386 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:38.365398+0000", 2023-10-09T11:02:38.386 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0059849930000000001, 2023-10-09T11:02:38.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0060264280000000003, 2023-10-09T11:02:38.386 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:02:38.386 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:02:38.386 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:02:38.386 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:02:38.387 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:02:38.387 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 91117 2023-10-09T11:02:38.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.387 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:02:38.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:02:38.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365398+0000", 2023-10-09T11:02:38.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:02:38.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:02:38.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365398+0000", 2023-10-09T11:02:38.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:02:38.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:02:38.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365396+0000", 2023-10-09T11:02:38.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:02:38.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:02:38.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365423+0000", 2023-10-09T11:02:38.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7058000000000001e-05 2023-10-09T11:02:38.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:02:38.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365441+0000", 2023-10-09T11:02:38.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7907e-05 2023-10-09T11:02:38.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:02:38.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365451+0000", 2023-10-09T11:02:38.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0447e-05 2023-10-09T11:02:38.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:02:38.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365502+0000", 2023-10-09T11:02:38.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1005999999999999e-05 2023-10-09T11:02:38.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:02:38.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365630+0000", 2023-10-09T11:02:38.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012722100000000001 2023-10-09T11:02:38.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:02:38.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365778+0000", 2023-10-09T11:02:38.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000147916 2023-10-09T11:02:38.391 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:02:38.391 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:02:38.391 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:02:38.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.391 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:91114 10.0 e438/403 10:05422fea:::benchmark_data_smithi125_402869_object91113:head v 438'5592, mlcod=438'5590)", 2023-10-09T11:02:38.391 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:38.365492+0000", 2023-10-09T11:02:38.391 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.005890302, 2023-10-09T11:02:38.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0060143820000000004, 2023-10-09T11:02:38.391 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:02:38.392 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:02:38.392 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:02:38.392 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.392 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:02:38.392 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365492+0000", 2023-10-09T11:02:38.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:02:38.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.392 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.392 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:02:38.392 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365492+0000", 2023-10-09T11:02:38.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:02:38.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.393 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:02:38.393 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365492+0000", 2023-10-09T11:02:38.393 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:02:38.393 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.393 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:02:38.393 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365525+0000", 2023-10-09T11:02:38.393 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3609999999999998e-05 2023-10-09T11:02:38.393 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.393 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:02:38.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365535+0000", 2023-10-09T11:02:38.394 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5999999999999996e-06 2023-10-09T11:02:38.394 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.394 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:02:38.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365541+0000", 2023-10-09T11:02:38.394 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7370000000000003e-06 2023-10-09T11:02:38.394 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.394 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:02:38.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365719+0000", 2023-10-09T11:02:38.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0001784 2023-10-09T11:02:38.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.395 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:02:38.395 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.365744+0000", 2023-10-09T11:02:38.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5310999999999999e-05 2023-10-09T11:02:38.395 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:02:38.395 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:02:38.395 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:02:38.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.395 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:91120 10.0 e438/403 10:0650b9c8:::benchmark_data_smithi125_402869_object91119:head v 438'5593, mlcod=438'5590)", 2023-10-09T11:02:38.396 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:38.368631+0000", 2023-10-09T11:02:38.396 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.002752202, 2023-10-09T11:02:38.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029378410000000001, 2023-10-09T11:02:38.396 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:02:38.396 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:02:38.396 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:02:38.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:02:38.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.368631+0000", 2023-10-09T11:02:38.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:02:38.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.397 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:02:38.397 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.368631+0000", 2023-10-09T11:02:38.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:02:38.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.397 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:02:38.397 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.368630+0000", 2023-10-09T11:02:38.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:02:38.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.397 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:02:38.397 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.368667+0000", 2023-10-09T11:02:38.398 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7249999999999997e-05 2023-10-09T11:02:38.398 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.398 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:02:38.398 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.368678+0000", 2023-10-09T11:02:38.398 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1068e-05 2023-10-09T11:02:38.398 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.398 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:02:38.398 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.368685+0000", 2023-10-09T11:02:38.398 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0990000000000001e-06 2023-10-09T11:02:38.398 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.399 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.399 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:02:38.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.369184+0000", 2023-10-09T11:02:38.399 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00049892100000000004 2023-10-09T11:02:38.399 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.399 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.399 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:02:38.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.369214+0000", 2023-10-09T11:02:38.399 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9507999999999999e-05 2023-10-09T11:02:38.399 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:02:38.399 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:02:38.399 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:02:38.400 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.400 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.400 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:91113 10.2 10:42905498:::benchmark_data_smithi125_402869_object91112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T11:02:38.400 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:38.362890+0000", 2023-10-09T11:02:38.400 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0084930219999999994, 2023-10-09T11:02:38.400 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0087439890000000006, 2023-10-09T11:02:38.400 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:02:38.400 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:02:38.400 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:02:38.400 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:02:38.400 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:02:38.400 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 91113 2023-10-09T11:02:38.401 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.401 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:02:38.401 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.401 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:02:38.401 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.362890+0000", 2023-10-09T11:02:38.401 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:02:38.401 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.401 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.401 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:02:38.401 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.362890+0000", 2023-10-09T11:02:38.401 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:02:38.401 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.402 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.402 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:02:38.402 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.362888+0000", 2023-10-09T11:02:38.402 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:02:38.402 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.402 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.402 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:02:38.402 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.362918+0000", 2023-10-09T11:02:38.402 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0215000000000001e-05 2023-10-09T11:02:38.402 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.402 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.403 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:02:38.403 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.362942+0000", 2023-10-09T11:02:38.403 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4335999999999999e-05 2023-10-09T11:02:38.403 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.403 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.403 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:02:38.403 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.362954+0000", 2023-10-09T11:02:38.403 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1732e-05 2023-10-09T11:02:38.403 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.403 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.403 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:02:38.403 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.363007+0000", 2023-10-09T11:02:38.404 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3149999999999998e-05 2023-10-09T11:02:38.404 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.404 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.404 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:02:38.404 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.363166+0000", 2023-10-09T11:02:38.404 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015862800000000001 2023-10-09T11:02:38.404 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:38.404 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:38.404 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,4", 2023-10-09T11:02:38.404 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.363311+0000", 2023-10-09T11:02:38.404 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000145439 2023-10-09T11:02:38.404 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:02:38.405 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:02:38.405 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:02:38.405 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:02:38.405 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T11:02:38.405 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 4 2023-10-09T11:02:38.405 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:02:38.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:38 smithi155 ceph-mon[184854]: pgmap v1081: 119 pgs: 1 active+recovering+undersized+remapped, 11 peering, 107 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 20 MiB/s wr, 314 op/s; 5371/265185 objects misplaced (2.025%); 920 KiB/s, 2 keys/s, 14 objects/s recovering 2023-10-09T11:02:38.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:38 smithi125 ceph-mon[195507]: pgmap v1081: 119 pgs: 1 active+recovering+undersized+remapped, 11 peering, 107 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 20 MiB/s wr, 314 op/s; 5371/265185 objects misplaced (2.025%); 920 KiB/s, 2 keys/s, 14 objects/s recovering 2023-10-09T11:02:38.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:38 smithi125 ceph-mon[200197]: pgmap v1081: 119 pgs: 1 active+recovering+undersized+remapped, 11 peering, 107 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 20 MiB/s wr, 314 op/s; 5371/265185 objects misplaced (2.025%); 920 KiB/s, 2 keys/s, 14 objects/s recovering 2023-10-09T11:02:38.916 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T11:02:38.917226+0000 min lat: 0.0011398 max lat: 2.12362 avg lat: 0.0139816 2023-10-09T11:02:38.916 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T11:02:38.916 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 80 16 91569 91553 71.493 46.9375 0.0529126 0.0139816 2023-10-09T11:02:39.125 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:39.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:39.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:39.231 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:39.312 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:39.466 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:39.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:39.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:39.498 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T11:02:39.576 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:39.654 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:39.732 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:39.755 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:39.812 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:39 smithi125 ceph-mon[195507]: Health check failed: Degraded data redundancy: 5503/270384 objects degraded (2.035%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:02:39.812 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:39 smithi125 ceph-mon[200197]: Health check failed: Degraded data redundancy: 5503/270384 objects degraded (2.035%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:02:39.917 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 81 16 92367 92351 71.2258 49.875 0.0138131 0.0140351 2023-10-09T11:02:39.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:39 smithi155 ceph-mon[184854]: Health check failed: Degraded data redundancy: 5503/270384 objects degraded (2.035%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:02:40.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:40.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:40.389 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:40.461 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:40.476 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:02:40.513 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:40.513 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:02:40.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:40.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:40.564 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:02:40.574 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:40.615 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:02:40.617 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:40.658 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:02:40.663 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:40.686 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:40.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:40.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:40.704 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:02:40.734 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:02:40.769 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:02:40.783 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:02:40.783 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:02:40.784 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:02:40.784 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:02:40.784 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:02:40.784 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:02:40.784 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:02:40.784 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:02:40.791 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:40.844 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:40.875 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:40.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:40.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:40.917 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 82 16 93119 93103 70.9305 47 0.0211124 0.0140942 2023-10-09T11:02:40.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:40 smithi155 ceph-mon[184854]: pgmap v1082: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 4 peering, 111 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 525 op/s; 5503/270384 objects degraded (2.035%); 16830/270384 objects misplaced (6.224%); 705 KiB/s, 1 keys/s, 11 objects/s recovering 2023-10-09T11:02:40.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:40 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1625419057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:40.979 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:41.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:40 smithi125 ceph-mon[195507]: pgmap v1082: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 4 peering, 111 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 525 op/s; 5503/270384 objects degraded (2.035%); 16830/270384 objects misplaced (6.224%); 705 KiB/s, 1 keys/s, 11 objects/s recovering 2023-10-09T11:02:41.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:40 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1625419057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:41.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:40 smithi125 ceph-mon[200197]: pgmap v1082: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 4 peering, 111 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 525 op/s; 5503/270384 objects degraded (2.035%); 16830/270384 objects misplaced (6.224%); 705 KiB/s, 1 keys/s, 11 objects/s recovering 2023-10-09T11:02:41.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:40 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1625419057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:41.050 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:41.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:41.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:41.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:41.192 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:41.242 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:41.276 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:41.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:41.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:41.322 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:02:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:02:41] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.43.0" 2023-10-09T11:02:41.343 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:02:41.382 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:41.431 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:41.456 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:02:41.460 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:41.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:41.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:41.474 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:02:41.572 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:41.626 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:41.670 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:41.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:41.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:41.773 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:41.818 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:41.860 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:41.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:41.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:41.917 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 83 16 93856 93840 70.6309 46.0625 0.019277 0.0141529 2023-10-09T11:02:41.966 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:42.029 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:42.255 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:42.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:42.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:42.366 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:42.415 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:42.705 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:42.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:42.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:42.730 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1082,"stamp":"2023-10-09T11:02:39.694683+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5905660561,"num_objects":90128,"num_object_clones":0,"num_object_copies":270384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5503,"num_objects_misplaced":16830,"num_objects_unfound":0,"num_objects_dirty":90128,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":90169,"num_write_kb":5766722,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4669,"num_bytes_recovered":302985181,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94050,"ondisk_log_size":94050,"up":357,"acting":355,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":379,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23590888,"kb_used_data":17569960,"kb_used_omap":130,"kb_used_meta":6020733,"kb_avail":726370328,"statfs":{"total":767960285184,"available":743803215872,"internally_reserved":0,"allocated":17991639040,"data_stored":17977070321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":133799,"internal_metadata":6165230937},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,2,3,5,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":206831616,"num_objects":3156,"num_object_clones":0,"num_object_copies":9468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5503,"num_objects_misplaced":16830,"num_objects_unfound":0,"num_objects_dirty":3156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3156,"num_write_kb":201984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":4330622,"num_keys_recovered":10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002166"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":677,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.217074+0000","last_change":"2023-10-09T11:02:36.217074+0000","last_active":"2023-10-09T11:02:36.217074+0000","last_peered":"2023-10-09T11:02:36.217074+0000","last_clean":"2023-10-09T11:02:36.217074+0000","last_became_active":"2023-10-09T11:02:36.216494+0000","last_became_peered":"2023-10-09T11:02:36.216494+0000","last_unstale":"2023-10-09T11:02:36.217074+0000","last_undegraded":"2023-10-09T11:02:36.217074+0000","last_fullsized":"2023-10-09T11:02:36.217074+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:56.760915+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":160,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.169277+0000","last_change":"2023-10-09T11:00:14.910332+0000","last_active":"2023-10-09T11:02:34.169277+0000","last_peered":"2023-10-09T11:02:34.169277+0000","last_clean":"2023-10-09T11:02:34.169277+0000","last_became_active":"2023-10-09T11:00:14.910108+0000","last_became_peered":"2023-10-09T11:00:14.910108+0000","last_unstale":"2023-10-09T11:02:34.169277+0000","last_undegraded":"2023-10-09T11:02:34.169277+0000","last_fullsized":"2023-10-09T11:02:34.169277+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:30:41.668874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":462,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.173394+0000","last_change":"2023-10-09T11:00:14.870884+0000","last_active":"2023-10-09T11:02:34.173394+0000","last_peered":"2023-10-09T11:02:34.173394+0000","last_clean":"2023-10-09T11:02:34.173394+0000","last_became_active":"2023-10-09T11:00:14.870515+0000","last_became_peered":"2023-10-09T11:00:14.870515+0000","last_unstale":"2023-10-09T11:02:34.173394+0000","last_undegraded":"2023-10-09T11:02:34.173394+0000","last_fullsized":"2023-10-09T11:02:34.173394+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:02.499841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":595,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.184546+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:02:35.184546+0000","last_peered":"2023-10-09T11:02:35.184546+0000","last_clean":"2023-10-09T11:02:35.184546+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:02:35.184546+0000","last_undegraded":"2023-10-09T11:02:35.184546+0000","last_fullsized":"2023-10-09T11:02:35.184546+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":117,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.206095+0000","last_change":"2023-10-09T11:00:14.876415+0000","last_active":"2023-10-09T11:02:36.206095+0000","last_peered":"2023-10-09T11:02:36.206095+0000","last_clean":"2023-10-09T11:02:36.206095+0000","last_became_active":"2023-10-09T11:00:14.864861+0000","last_became_peered":"2023-10-09T11:00:14.864861+0000","last_unstale":"2023-10-09T11:02:36.206095+0000","last_undegraded":"2023-10-09T11:02:36.206095+0000","last_fullsized":"2023-10-09T11:02:36.206095+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:59.494851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":380,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.200622+0000","last_change":"2023-10-09T11:02:33.198683+0000","last_active":"2023-10-09T11:02:35.200622+0000","last_peered":"2023-10-09T11:02:35.200622+0000","last_clean":"2023-10-09T11:02:35.200622+0000","last_became_active":"2023-10-09T11:02:33.197952+0000","last_became_peered":"2023-10-09T11:02:33.197952+0000","last_unstale":"2023-10-09T11:02:35.200622+0000","last_undegraded":"2023-10-09T11:02:35.200622+0000","last_fullsized":"2023-10-09T11:02:35.200622+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:29:09.209024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":918,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.217484+0000","last_change":"2023-10-09T11:02:36.217484+0000","last_active":"2023-10-09T11:02:36.217484+0000","last_peered":"2023-10-09T11:02:36.217484+0000","last_clean":"2023-10-09T11:02:36.217484+0000","last_became_active":"2023-10-09T11:02:36.217091+0000","last_became_peered":"2023-10-09T11:02:36.217091+0000","last_unstale":"2023-10-09T11:02:36.217484+0000","last_undegraded":"2023-10-09T11:02:36.217484+0000","last_fullsized":"2023-10-09T11:02:36.217484+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:59.912586+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":528,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.172483+0000","last_change":"2023-10-09T11:00:15.889100+0000","last_active":"2023-10-09T11:02:34.172483+0000","last_peered":"2023-10-09T11:02:34.172483+0000","last_clean":"2023-10-09T11:02:34.172483+0000","last_became_active":"2023-10-09T11:00:15.888886+0000","last_became_peered":"2023-10-09T11:00:15.888886+0000","last_unstale":"2023-10-09T11:02:34.172483+0000","last_undegraded":"2023-10-09T11:02:34.172483+0000","last_fullsized":"2023-10-09T11:02:34.172483+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:40.695500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":227,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.185851+0000","last_change":"2023-10-09T11:01:15.401167+0000","last_active":"2023-10-09T11:02:35.185851+0000","last_peered":"2023-10-09T11:02:35.185851+0000","last_clean":"2023-10-09T11:02:35.185851+0000","last_became_active":"2023-10-09T11:01:15.400876+0000","last_became_peered":"2023-10-09T11:01:15.400876+0000","last_unstale":"2023-10-09T11:02:35.185851+0000","last_undegraded":"2023-10-09T11:02:35.185851+0000","last_fullsized":"2023-10-09T11:02:35.185851+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:56.342226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":759,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.216785+0000","last_change":"2023-10-09T11:02:36.216785+0000","last_active":"2023-10-09T11:02:36.216785+0000","last_peered":"2023-10-09T11:02:36.216785+0000","last_clean":"2023-10-09T11:02:36.216785+0000","last_became_active":"2023-10-09T11:02:36.215919+0000","last_became_peered":"2023-10-09T11:02:36.215919+0000","last_unstale":"2023-10-09T11:02:36.216785+0000","last_undegraded":"2023-10-09T11:02:36.216785+0000","last_fullsized":"2023-10-09T11:02:36.216785+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:08:40.893630+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":74,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.200776+0000","last_change":"2023-10-09T10:59:48.124873+0000","last_active":"2023-10-09T11:02:35.200776+0000","last_peered":"2023-10-09T11:02:35.200776+0000","last_clean":"2023-10-09T11:02:35.200776+0000","last_became_active":"2023-10-09T10:59:48.124600+0000","last_became_peered":"2023-10-09T10:59:48.124600+0000","last_unstale":"2023-10-09T11:02:35.200776+0000","last_undegraded":"2023-10-09T11:02:35.200776+0000","last_fullsized":"2023-10-09T11:02:35.200776+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:04.818987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":463,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.172564+0000","last_change":"2023-10-09T11:00:14.869998+0000","last_active":"2023-10-09T11:02:34.172564+0000","last_peered":"2023-10-09T11:02:34.172564+0000","last_clean":"2023-10-09T11:02:34.172564+0000","last_became_active":"2023-10-09T11:00:14.869671+0000","last_became_peered":"2023-10-09T11:00:14.869671+0000","last_unstale":"2023-10-09T11:02:34.172564+0000","last_undegraded":"2023-10-09T11:02:34.172564+0000","last_fullsized":"2023-10-09T11:02:34.172564+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:19.564770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":756,"reported_epoch":437,"state":"peering","last_fresh":"2023-10-09T11:02:35.182259+0000","last_change":"2023-10-09T11:02:35.182409+0000","last_active":"2023-10-09T11:02:35.182259+0000","last_peered":"2023-10-09T11:02:34.192474+0000","last_clean":"2023-10-09T11:02:31.300523+0000","last_became_active":"2023-10-09T11:02:34.192177+0000","last_became_peered":"2023-10-09T11:02:34.192177+0000","last_unstale":"2023-10-09T11:02:35.182259+0000","last_undegraded":"2023-10-09T11:02:35.182259+0000","last_fullsized":"2023-10-09T11:02:35.182259+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:48:22.287859+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4,7],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":289,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.185689+0000","last_change":"2023-10-09T11:02:33.198185+0000","last_active":"2023-10-09T11:02:35.185689+0000","last_peered":"2023-10-09T11:02:35.185689+0000","last_clean":"2023-10-09T11:02:35.185689+0000","last_became_active":"2023-10-09T11:02:33.197870+0000","last_became_peered":"2023-10-09T11:02:33.197870+0000","last_unstale":"2023-10-09T11:02:35.185689+0000","last_undegraded":"2023-10-09T11:02:35.185689+0000","last_fullsized":"2023-10-09T11:02:35.185689+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:21.200979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":423,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.172694+0000","last_change":"2023-10-09T11:02:33.198055+0000","last_active":"2023-10-09T11:02:34.172694+0000","last_peered":"2023-10-09T11:02:34.172694+0000","last_clean":"2023-10-09T11:02:34.172694+0000","last_became_active":"2023-10-09T11:02:33.197748+0000","last_became_peered":"2023-10-09T11:02:33.197748+0000","last_unstale":"2023-10-09T11:02:34.172694+0000","last_undegraded":"2023-10-09T11:02:34.172694+0000","last_fullsized":"2023-10-09T11:02:34.172694+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:14:15.601866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":627,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.185711+0000","last_change":"2023-10-09T11:01:16.673177+0000","last_active":"2023-10-09T11:02:35.185711+0000","last_peered":"2023-10-09T11:02:35.185711+0000","last_clean":"2023-10-09T11:02:35.185711+0000","last_became_active":"2023-10-09T11:01:15.401411+0000","last_became_peered":"2023-10-09T11:01:15.401411+0000","last_unstale":"2023-10-09T11:02:35.185711+0000","last_undegraded":"2023-10-09T11:02:35.185711+0000","last_fullsized":"2023-10-09T11:02:35.185711+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:46.989699+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":19,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.199045+0000","last_change":"2023-10-09T11:02:33.219972+0000","last_active":"2023-10-09T11:02:35.199045+0000","last_peered":"2023-10-09T11:02:35.199045+0000","last_clean":"2023-10-09T11:02:35.199045+0000","last_became_active":"2023-10-09T11:02:33.197428+0000","last_became_peered":"2023-10-09T11:02:33.197428+0000","last_unstale":"2023-10-09T11:02:35.199045+0000","last_undegraded":"2023-10-09T11:02:35.199045+0000","last_fullsized":"2023-10-09T11:02:35.199045+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:02.203640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":196,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.181154+0000","last_change":"2023-10-09T11:01:15.401892+0000","last_active":"2023-10-09T11:02:35.181154+0000","last_peered":"2023-10-09T11:02:35.181154+0000","last_clean":"2023-10-09T11:02:35.181154+0000","last_became_active":"2023-10-09T11:01:15.401025+0000","last_became_peered":"2023-10-09T11:01:15.401025+0000","last_unstale":"2023-10-09T11:02:35.181154+0000","last_undegraded":"2023-10-09T11:02:35.181154+0000","last_fullsized":"2023-10-09T11:02:35.181154+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:48:03.531920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":788,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.202218+0000","last_change":"2023-10-09T11:02:09.381602+0000","last_active":"2023-10-09T11:02:36.202218+0000","last_peered":"2023-10-09T11:02:36.202218+0000","last_clean":"2023-10-09T11:02:36.202218+0000","last_became_active":"2023-10-09T11:00:16.068155+0000","last_became_peered":"2023-10-09T11:00:16.068155+0000","last_unstale":"2023-10-09T11:02:36.202218+0000","last_undegraded":"2023-10-09T11:02:36.202218+0000","last_fullsized":"2023-10-09T11:02:36.202218+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:56.972770+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":23,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.199361+0000","last_change":"2023-10-09T11:02:33.213462+0000","last_active":"2023-10-09T11:02:35.199361+0000","last_peered":"2023-10-09T11:02:35.199361+0000","last_clean":"2023-10-09T11:02:35.199361+0000","last_became_active":"2023-10-09T11:02:33.197602+0000","last_became_peered":"2023-10-09T11:02:33.197602+0000","last_unstale":"2023-10-09T11:02:35.199361+0000","last_undegraded":"2023-10-09T11:02:35.199361+0000","last_fullsized":"2023-10-09T11:02:35.199361+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.773902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":452,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.190633+0000","last_change":"2023-10-09T10:57:46.845755+0000","last_active":"2023-10-09T11:02:35.190633+0000","last_peered":"2023-10-09T11:02:35.190633+0000","last_clean":"2023-10-09T11:02:35.190633+0000","last_became_active":"2023-10-09T10:57:46.845475+0000","last_became_peered":"2023-10-09T10:57:46.845475+0000","last_unstale":"2023-10-09T11:02:35.190633+0000","last_undegraded":"2023-10-09T11:02:35.190633+0000","last_fullsized":"2023-10-09T11:02:35.190633+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:21.673994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":801,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.207192+0000","last_change":"2023-10-09T10:59:49.175255+0000","last_active":"2023-10-09T11:02:35.207192+0000","last_peered":"2023-10-09T11:02:35.207192+0000","last_clean":"2023-10-09T11:02:35.207192+0000","last_became_active":"2023-10-09T10:59:49.174666+0000","last_became_peered":"2023-10-09T10:59:49.174666+0000","last_unstale":"2023-10-09T11:02:35.207192+0000","last_undegraded":"2023-10-09T11:02:35.207192+0000","last_fullsized":"2023-10-09T11:02:35.207192+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:00:58.069275+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":19,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.195089+0000","last_change":"2023-10-09T11:02:33.207032+0000","last_active":"2023-10-09T11:02:35.195089+0000","last_peered":"2023-10-09T11:02:35.195089+0000","last_clean":"2023-10-09T11:02:35.195089+0000","last_became_active":"2023-10-09T11:02:33.196379+0000","last_became_peered":"2023-10-09T11:02:33.196379+0000","last_unstale":"2023-10-09T11:02:35.195089+0000","last_undegraded":"2023-10-09T11:02:35.195089+0000","last_fullsized":"2023-10-09T11:02:35.195089+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:56.935603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":549,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.174123+0000","last_change":"2023-10-09T11:02:33.198289+0000","last_active":"2023-10-09T11:02:34.174123+0000","last_peered":"2023-10-09T11:02:34.174123+0000","last_clean":"2023-10-09T11:02:34.174123+0000","last_became_active":"2023-10-09T11:02:33.197962+0000","last_became_peered":"2023-10-09T11:02:33.197962+0000","last_unstale":"2023-10-09T11:02:34.174123+0000","last_undegraded":"2023-10-09T11:02:34.174123+0000","last_fullsized":"2023-10-09T11:02:34.174123+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:05:34.023093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":709,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.216877+0000","last_change":"2023-10-09T11:02:36.216877+0000","last_active":"2023-10-09T11:02:36.216877+0000","last_peered":"2023-10-09T11:02:36.216877+0000","last_clean":"2023-10-09T11:02:36.216877+0000","last_became_active":"2023-10-09T11:02:36.216072+0000","last_became_peered":"2023-10-09T11:02:36.216072+0000","last_unstale":"2023-10-09T11:02:36.216877+0000","last_undegraded":"2023-10-09T11:02:36.216877+0000","last_fullsized":"2023-10-09T11:02:36.216877+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:47.741775+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":379,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.206460+0000","last_change":"2023-10-09T11:02:33.198858+0000","last_active":"2023-10-09T11:02:35.206460+0000","last_peered":"2023-10-09T11:02:35.206460+0000","last_clean":"2023-10-09T11:02:35.206460+0000","last_became_active":"2023-10-09T11:02:33.198557+0000","last_became_peered":"2023-10-09T11:02:33.198557+0000","last_unstale":"2023-10-09T11:02:35.206460+0000","last_undegraded":"2023-10-09T11:02:35.206460+0000","last_fullsized":"2023-10-09T11:02:35.206460+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:32.140678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":322,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.184292+0000","last_change":"2023-10-09T11:02:33.203078+0000","last_active":"2023-10-09T11:02:35.184292+0000","last_peered":"2023-10-09T11:02:35.184292+0000","last_clean":"2023-10-09T11:02:35.184292+0000","last_became_active":"2023-10-09T11:02:33.202772+0000","last_became_peered":"2023-10-09T11:02:33.202772+0000","last_unstale":"2023-10-09T11:02:35.184292+0000","last_undegraded":"2023-10-09T11:02:35.184292+0000","last_fullsized":"2023-10-09T11:02:35.184292+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:28:14.080239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":730,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.194219+0000","last_change":"2023-10-09T11:01:25.620261+0000","last_active":"2023-10-09T11:02:35.194219+0000","last_peered":"2023-10-09T11:02:35.194219+0000","last_clean":"2023-10-09T11:02:35.194219+0000","last_became_active":"2023-10-09T11:00:14.871258+0000","last_became_peered":"2023-10-09T11:00:14.871258+0000","last_unstale":"2023-10-09T11:02:35.194219+0000","last_undegraded":"2023-10-09T11:02:35.194219+0000","last_fullsized":"2023-10-09T11:02:35.194219+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:13.861006+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.198702+0000","last_change":"2023-10-09T11:02:33.198572+0000","last_active":"2023-10-09T11:02:35.198702+0000","last_peered":"2023-10-09T11:02:35.198702+0000","last_clean":"2023-10-09T11:02:35.198702+0000","last_became_active":"2023-10-09T11:02:33.198218+0000","last_became_peered":"2023-10-09T11:02:33.198218+0000","last_unstale":"2023-10-09T11:02:35.198702+0000","last_undegraded":"2023-10-09T11:02:35.198702+0000","last_fullsized":"2023-10-09T11:02:35.198702+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:59.066866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":92,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.200441+0000","last_change":"2023-10-09T11:02:33.198882+0000","last_active":"2023-10-09T11:02:35.200441+0000","last_peered":"2023-10-09T11:02:35.200441+0000","last_clean":"2023-10-09T11:02:35.200441+0000","last_became_active":"2023-10-09T11:02:33.198380+0000","last_became_peered":"2023-10-09T11:02:33.198380+0000","last_unstale":"2023-10-09T11:02:35.200441+0000","last_undegraded":"2023-10-09T11:02:35.200441+0000","last_fullsized":"2023-10-09T11:02:35.200441+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:21:35.066055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":706,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.181616+0000","last_change":"2023-10-09T11:01:15.406928+0000","last_active":"2023-10-09T11:02:35.181616+0000","last_peered":"2023-10-09T11:02:35.181616+0000","last_clean":"2023-10-09T11:02:35.181616+0000","last_became_active":"2023-10-09T11:01:15.406499+0000","last_became_peered":"2023-10-09T11:01:15.406499+0000","last_unstale":"2023-10-09T11:02:35.181616+0000","last_undegraded":"2023-10-09T11:02:35.181616+0000","last_fullsized":"2023-10-09T11:02:35.181616+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:33.050336+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":165,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.197923+0000","last_change":"2023-10-09T11:00:48.109643+0000","last_active":"2023-10-09T11:02:35.197923+0000","last_peered":"2023-10-09T11:02:35.197923+0000","last_clean":"2023-10-09T11:02:35.197923+0000","last_became_active":"2023-10-09T11:00:48.109299+0000","last_became_peered":"2023-10-09T11:00:48.109299+0000","last_unstale":"2023-10-09T11:02:35.197923+0000","last_undegraded":"2023-10-09T11:02:35.197923+0000","last_fullsized":"2023-10-09T11:02:35.197923+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:31.163237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":525,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.175284+0000","last_change":"2023-10-09T11:02:33.200032+0000","last_active":"2023-10-09T11:02:34.175284+0000","last_peered":"2023-10-09T11:02:34.175284+0000","last_clean":"2023-10-09T11:02:34.175284+0000","last_became_active":"2023-10-09T11:02:33.199691+0000","last_became_peered":"2023-10-09T11:02:33.199691+0000","last_unstale":"2023-10-09T11:02:34.175284+0000","last_undegraded":"2023-10-09T11:02:34.175284+0000","last_fullsized":"2023-10-09T11:02:34.175284+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:15.777063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"438'5725","reported_seq":5768,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.616705+0000","last_change":"2023-10-09T11:01:15.402898+0000","last_active":"2023-10-09T11:02:36.616705+0000","last_peered":"2023-10-09T11:02:36.616705+0000","last_clean":"2023-10-09T11:02:36.616705+0000","last_became_active":"2023-10-09T11:01:15.402646+0000","last_became_peered":"2023-10-09T11:01:15.402646+0000","last_unstale":"2023-10-09T11:02:36.616705+0000","last_undegraded":"2023-10-09T11:02:36.616705+0000","last_fullsized":"2023-10-09T11:02:36.616705+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5725,"log_dups_size":0,"ondisk_log_size":5725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:42:07.397321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375193600,"num_objects":5725,"num_object_clones":0,"num_object_copies":17175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5725,"num_write_kb":366400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":727,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.217050+0000","last_change":"2023-10-09T11:02:36.217050+0000","last_active":"2023-10-09T11:02:36.217050+0000","last_peered":"2023-10-09T11:02:36.217050+0000","last_clean":"2023-10-09T11:02:36.217050+0000","last_became_active":"2023-10-09T11:02:36.216387+0000","last_became_peered":"2023-10-09T11:02:36.216387+0000","last_unstale":"2023-10-09T11:02:36.217050+0000","last_undegraded":"2023-10-09T11:02:36.217050+0000","last_fullsized":"2023-10-09T11:02:36.217050+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:19.627177+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":17,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.200453+0000","last_change":"2023-10-09T11:02:33.221990+0000","last_active":"2023-10-09T11:02:35.200453+0000","last_peered":"2023-10-09T11:02:35.200453+0000","last_clean":"2023-10-09T11:02:35.200453+0000","last_became_active":"2023-10-09T11:02:33.198155+0000","last_became_peered":"2023-10-09T11:02:33.198155+0000","last_unstale":"2023-10-09T11:02:35.200453+0000","last_undegraded":"2023-10-09T11:02:35.200453+0000","last_fullsized":"2023-10-09T11:02:35.200453+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:32.563344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":360,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.183358+0000","last_change":"2023-10-09T11:02:33.200229+0000","last_active":"2023-10-09T11:02:35.183358+0000","last_peered":"2023-10-09T11:02:35.183358+0000","last_clean":"2023-10-09T11:02:35.183358+0000","last_became_active":"2023-10-09T11:02:33.199936+0000","last_became_peered":"2023-10-09T11:02:33.199936+0000","last_unstale":"2023-10-09T11:02:35.183358+0000","last_undegraded":"2023-10-09T11:02:35.183358+0000","last_fullsized":"2023-10-09T11:02:35.183358+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:28.854719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"437'5544","reported_seq":5587,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.867583+0000","last_change":"2023-10-09T11:01:15.394927+0000","last_active":"2023-10-09T11:02:35.867583+0000","last_peered":"2023-10-09T11:02:35.867583+0000","last_clean":"2023-10-09T11:02:35.867583+0000","last_became_active":"2023-10-09T11:01:15.394658+0000","last_became_peered":"2023-10-09T11:01:15.394658+0000","last_unstale":"2023-10-09T11:02:35.867583+0000","last_undegraded":"2023-10-09T11:02:35.867583+0000","last_fullsized":"2023-10-09T11:02:35.867583+0000","mapping_epoch":403,"log_start":"406'100","ondisk_log_start":"406'100","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5444,"log_dups_size":0,"ondisk_log_size":5444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:58.565353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363331584,"num_objects":5544,"num_object_clones":0,"num_object_copies":16632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5544,"num_write_kb":354816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":841,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.173545+0000","last_change":"2023-10-09T10:59:51.140014+0000","last_active":"2023-10-09T11:02:34.173545+0000","last_peered":"2023-10-09T11:02:34.173545+0000","last_clean":"2023-10-09T11:02:34.173545+0000","last_became_active":"2023-10-09T10:59:51.139665+0000","last_became_peered":"2023-10-09T10:59:51.139665+0000","last_unstale":"2023-10-09T11:02:34.173545+0000","last_undegraded":"2023-10-09T11:02:34.173545+0000","last_fullsized":"2023-10-09T11:02:34.173545+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:29.505223+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":17,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.200001+0000","last_change":"2023-10-09T11:02:33.208705+0000","last_active":"2023-10-09T11:02:35.200001+0000","last_peered":"2023-10-09T11:02:35.200001+0000","last_clean":"2023-10-09T11:02:35.200001+0000","last_became_active":"2023-10-09T11:02:33.197454+0000","last_became_peered":"2023-10-09T11:02:33.197454+0000","last_unstale":"2023-10-09T11:02:35.200001+0000","last_undegraded":"2023-10-09T11:02:35.200001+0000","last_fullsized":"2023-10-09T11:02:35.200001+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:52.649912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":117,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.183929+0000","last_change":"2023-10-09T11:00:14.872673+0000","last_active":"2023-10-09T11:02:35.183929+0000","last_peered":"2023-10-09T11:02:35.183929+0000","last_clean":"2023-10-09T11:02:35.183929+0000","last_became_active":"2023-10-09T11:00:14.872345+0000","last_became_peered":"2023-10-09T11:00:14.872345+0000","last_unstale":"2023-10-09T11:02:35.183929+0000","last_undegraded":"2023-10-09T11:02:35.183929+0000","last_fullsized":"2023-10-09T11:02:35.183929+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:32:43.821085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"438'5613","reported_seq":5657,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:37.975820+0000","last_change":"2023-10-09T11:01:15.402608+0000","last_active":"2023-10-09T11:02:37.975820+0000","last_peered":"2023-10-09T11:02:37.975820+0000","last_clean":"2023-10-09T11:02:37.975820+0000","last_became_active":"2023-10-09T11:01:15.402329+0000","last_became_peered":"2023-10-09T11:01:15.402329+0000","last_unstale":"2023-10-09T11:02:37.975820+0000","last_undegraded":"2023-10-09T11:02:37.975820+0000","last_fullsized":"2023-10-09T11:02:37.975820+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5613,"log_dups_size":0,"ondisk_log_size":5613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:31:16.567673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367853568,"num_objects":5613,"num_object_clones":0,"num_object_copies":16839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5613,"num_write_kb":359232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":634,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.185766+0000","last_change":"2023-10-09T11:01:22.599166+0000","last_active":"2023-10-09T11:02:35.185766+0000","last_peered":"2023-10-09T11:02:35.185766+0000","last_clean":"2023-10-09T11:02:35.185766+0000","last_became_active":"2023-10-09T11:01:15.402193+0000","last_became_peered":"2023-10-09T11:01:15.402193+0000","last_unstale":"2023-10-09T11:02:35.185766+0000","last_undegraded":"2023-10-09T11:02:35.185766+0000","last_fullsized":"2023-10-09T11:02:35.185766+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:37:16.611219+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":17,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.199156+0000","last_change":"2023-10-09T11:02:33.223801+0000","last_active":"2023-10-09T11:02:35.199156+0000","last_peered":"2023-10-09T11:02:35.199156+0000","last_clean":"2023-10-09T11:02:35.199156+0000","last_became_active":"2023-10-09T11:02:33.198699+0000","last_became_peered":"2023-10-09T11:02:33.198699+0000","last_unstale":"2023-10-09T11:02:35.199156+0000","last_undegraded":"2023-10-09T11:02:35.199156+0000","last_fullsized":"2023-10-09T11:02:35.199156+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:13.818599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":287,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.196959+0000","last_change":"2023-10-09T11:02:33.199120+0000","last_active":"2023-10-09T11:02:35.196959+0000","last_peered":"2023-10-09T11:02:35.196959+0000","last_clean":"2023-10-09T11:02:35.196959+0000","last_became_active":"2023-10-09T11:02:33.198673+0000","last_became_peered":"2023-10-09T11:02:33.198673+0000","last_unstale":"2023-10-09T11:02:35.196959+0000","last_undegraded":"2023-10-09T11:02:35.196959+0000","last_fullsized":"2023-10-09T11:02:35.196959+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:58:06.417132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"437'5652","reported_seq":5695,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.085334+0000","last_change":"2023-10-09T11:01:15.393071+0000","last_active":"2023-10-09T11:02:36.085334+0000","last_peered":"2023-10-09T11:02:36.085334+0000","last_clean":"2023-10-09T11:02:36.085334+0000","last_became_active":"2023-10-09T11:01:15.392793+0000","last_became_peered":"2023-10-09T11:01:15.392793+0000","last_unstale":"2023-10-09T11:02:36.085334+0000","last_undegraded":"2023-10-09T11:02:36.085334+0000","last_fullsized":"2023-10-09T11:02:36.085334+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5652,"log_dups_size":0,"ondisk_log_size":5652,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:27.128762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370409472,"num_objects":5652,"num_object_clones":0,"num_object_copies":16956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5652,"num_write_kb":361728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":963,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.218864+0000","last_change":"2023-10-09T11:02:36.218864+0000","last_active":"2023-10-09T11:02:36.218864+0000","last_peered":"2023-10-09T11:02:36.218864+0000","last_clean":"2023-10-09T11:02:36.218864+0000","last_became_active":"2023-10-09T11:02:36.218557+0000","last_became_peered":"2023-10-09T11:02:36.218557+0000","last_unstale":"2023-10-09T11:02:36.218864+0000","last_undegraded":"2023-10-09T11:02:36.218864+0000","last_fullsized":"2023-10-09T11:02:36.218864+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:49.124970+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":249,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.182014+0000","last_change":"2023-10-09T11:01:15.400732+0000","last_active":"2023-10-09T11:02:35.182014+0000","last_peered":"2023-10-09T11:02:35.182014+0000","last_clean":"2023-10-09T11:02:35.182014+0000","last_became_active":"2023-10-09T11:01:15.399370+0000","last_became_peered":"2023-10-09T11:01:15.399370+0000","last_unstale":"2023-10-09T11:02:35.182014+0000","last_undegraded":"2023-10-09T11:02:35.182014+0000","last_fullsized":"2023-10-09T11:02:35.182014+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:53.948144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":359,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.200157+0000","last_change":"2023-10-09T11:00:15.865848+0000","last_active":"2023-10-09T11:02:35.200157+0000","last_peered":"2023-10-09T11:02:35.200157+0000","last_clean":"2023-10-09T11:02:35.200157+0000","last_became_active":"2023-10-09T11:00:15.865504+0000","last_became_peered":"2023-10-09T11:00:15.865504+0000","last_unstale":"2023-10-09T11:02:35.200157+0000","last_undegraded":"2023-10-09T11:02:35.200157+0000","last_fullsized":"2023-10-09T11:02:35.200157+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:32.352564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"438'5678","reported_seq":5722,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:38.023228+0000","last_change":"2023-10-09T11:01:15.401760+0000","last_active":"2023-10-09T11:02:38.023228+0000","last_peered":"2023-10-09T11:02:38.023228+0000","last_clean":"2023-10-09T11:02:38.023228+0000","last_became_active":"2023-10-09T11:01:15.401510+0000","last_became_peered":"2023-10-09T11:01:15.401510+0000","last_unstale":"2023-10-09T11:02:38.023228+0000","last_undegraded":"2023-10-09T11:02:38.023228+0000","last_fullsized":"2023-10-09T11:02:38.023228+0000","mapping_epoch":403,"log_start":"406'500","ondisk_log_start":"406'500","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5178,"log_dups_size":0,"ondisk_log_size":5178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:21.262441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372113408,"num_objects":5678,"num_object_clones":0,"num_object_copies":17034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5678,"num_write_kb":363392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":699,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.228163+0000","last_change":"2023-10-09T11:02:36.228163+0000","last_active":"2023-10-09T11:02:36.228163+0000","last_peered":"2023-10-09T11:02:36.228163+0000","last_clean":"2023-10-09T11:02:36.228163+0000","last_became_active":"2023-10-09T11:02:36.227871+0000","last_became_peered":"2023-10-09T11:02:36.227871+0000","last_unstale":"2023-10-09T11:02:36.228163+0000","last_undegraded":"2023-10-09T11:02:36.228163+0000","last_fullsized":"2023-10-09T11:02:36.228163+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:35.196080+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":456,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.173251+0000","last_change":"2023-10-09T10:59:49.127924+0000","last_active":"2023-10-09T11:02:34.173251+0000","last_peered":"2023-10-09T11:02:34.173251+0000","last_clean":"2023-10-09T11:02:34.173251+0000","last_became_active":"2023-10-09T10:59:49.127549+0000","last_became_peered":"2023-10-09T10:59:49.127549+0000","last_unstale":"2023-10-09T11:02:34.173251+0000","last_undegraded":"2023-10-09T11:02:34.173251+0000","last_fullsized":"2023-10-09T11:02:34.173251+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:01.781131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":281,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.174261+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T11:02:34.174261+0000","last_peered":"2023-10-09T11:02:34.174261+0000","last_clean":"2023-10-09T11:02:34.174261+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T11:02:34.174261+0000","last_undegraded":"2023-10-09T11:02:34.174261+0000","last_fullsized":"2023-10-09T11:02:34.174261+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"437'5539","reported_seq":5581,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.064883+0000","last_change":"2023-10-09T11:01:15.394098+0000","last_active":"2023-10-09T11:02:36.064883+0000","last_peered":"2023-10-09T11:02:36.064883+0000","last_clean":"2023-10-09T11:02:36.064883+0000","last_became_active":"2023-10-09T11:01:15.393709+0000","last_became_peered":"2023-10-09T11:01:15.393709+0000","last_unstale":"2023-10-09T11:02:36.064883+0000","last_undegraded":"2023-10-09T11:02:36.064883+0000","last_fullsized":"2023-10-09T11:02:36.064883+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5539,"log_dups_size":0,"ondisk_log_size":5539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:08.421072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363003904,"num_objects":5539,"num_object_clones":0,"num_object_copies":16617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5539,"num_write_kb":354496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":629,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.187322+0000","last_change":"2023-10-09T11:01:20.667499+0000","last_active":"2023-10-09T11:02:35.187322+0000","last_peered":"2023-10-09T11:02:35.187322+0000","last_clean":"2023-10-09T11:02:35.187322+0000","last_became_active":"2023-10-09T11:01:15.399808+0000","last_became_peered":"2023-10-09T11:01:15.399808+0000","last_unstale":"2023-10-09T11:02:35.187322+0000","last_undegraded":"2023-10-09T11:02:35.187322+0000","last_fullsized":"2023-10-09T11:02:35.187322+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:53:29.298692+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":434,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.191521+0000","last_change":"2023-10-09T11:00:14.891176+0000","last_active":"2023-10-09T11:02:35.191521+0000","last_peered":"2023-10-09T11:02:35.191521+0000","last_clean":"2023-10-09T11:02:35.191521+0000","last_became_active":"2023-10-09T11:00:14.890890+0000","last_became_peered":"2023-10-09T11:00:14.890890+0000","last_unstale":"2023-10-09T11:02:35.191521+0000","last_undegraded":"2023-10-09T11:02:35.191521+0000","last_fullsized":"2023-10-09T11:02:35.191521+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:38.158759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":193,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.169896+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:02:34.169896+0000","last_peered":"2023-10-09T11:02:34.169896+0000","last_clean":"2023-10-09T11:02:34.169896+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:02:34.169896+0000","last_undegraded":"2023-10-09T11:02:34.169896+0000","last_fullsized":"2023-10-09T11:02:34.169896+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"438'5503","reported_seq":5546,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.444802+0000","last_change":"2023-10-09T11:01:15.406742+0000","last_active":"2023-10-09T11:02:36.444802+0000","last_peered":"2023-10-09T11:02:36.444802+0000","last_clean":"2023-10-09T11:02:36.444802+0000","last_became_active":"2023-10-09T11:01:15.406429+0000","last_became_peered":"2023-10-09T11:01:15.406429+0000","last_unstale":"2023-10-09T11:02:36.444802+0000","last_undegraded":"2023-10-09T11:02:36.444802+0000","last_fullsized":"2023-10-09T11:02:36.444802+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5503,"log_dups_size":0,"ondisk_log_size":5503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:44:05.691874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360644608,"num_objects":5503,"num_object_clones":0,"num_object_copies":16509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5503,"num_write_kb":352192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":787,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.194298+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:02:35.194298+0000","last_peered":"2023-10-09T11:02:35.194298+0000","last_clean":"2023-10-09T11:02:35.194298+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:02:35.194298+0000","last_undegraded":"2023-10-09T11:02:35.194298+0000","last_fullsized":"2023-10-09T11:02:35.194298+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":513,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.190770+0000","last_change":"2023-10-09T11:02:33.200106+0000","last_active":"2023-10-09T11:02:35.190770+0000","last_peered":"2023-10-09T11:02:35.190770+0000","last_clean":"2023-10-09T11:02:35.190770+0000","last_became_active":"2023-10-09T11:02:33.199840+0000","last_became_peered":"2023-10-09T11:02:33.199840+0000","last_unstale":"2023-10-09T11:02:35.190770+0000","last_undegraded":"2023-10-09T11:02:35.190770+0000","last_fullsized":"2023-10-09T11:02:35.190770+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:13:17.557980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":117,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.195197+0000","last_change":"2023-10-09T11:00:14.900130+0000","last_active":"2023-10-09T11:02:35.195197+0000","last_peered":"2023-10-09T11:02:35.195197+0000","last_clean":"2023-10-09T11:02:35.195197+0000","last_became_active":"2023-10-09T11:00:14.899536+0000","last_became_peered":"2023-10-09T11:00:14.899536+0000","last_unstale":"2023-10-09T11:02:35.195197+0000","last_undegraded":"2023-10-09T11:02:35.195197+0000","last_fullsized":"2023-10-09T11:02:35.195197+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:07.151510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"438'5745","reported_seq":8080,"reported_epoch":438,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:02:37.993686+0000","last_change":"2023-10-09T11:02:34.381323+0000","last_active":"2023-10-09T11:02:37.993686+0000","last_peered":"2023-10-09T11:02:37.993686+0000","last_clean":"2023-10-09T11:02:32.284069+0000","last_became_active":"2023-10-09T11:02:34.373546+0000","last_became_peered":"2023-10-09T11:02:34.373546+0000","last_unstale":"2023-10-09T11:02:37.993686+0000","last_undegraded":"2023-10-09T11:02:37.993686+0000","last_fullsized":"2023-10-09T11:02:37.993686+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:22.107483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376504320,"num_objects":5745,"num_object_clones":0,"num_object_copies":17235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5745,"num_objects_unfound":0,"num_objects_dirty":5745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5745,"num_write_kb":367680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1135,"num_bytes_recovered":74383360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[0,4,1],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":5745}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":640,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.181515+0000","last_change":"2023-10-09T11:01:51.463338+0000","last_active":"2023-10-09T11:02:35.181515+0000","last_peered":"2023-10-09T11:02:35.181515+0000","last_clean":"2023-10-09T11:02:35.181515+0000","last_became_active":"2023-10-09T11:01:15.399431+0000","last_became_peered":"2023-10-09T11:01:15.399431+0000","last_unstale":"2023-10-09T11:02:35.181515+0000","last_undegraded":"2023-10-09T11:02:35.181515+0000","last_fullsized":"2023-10-09T11:02:35.181515+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:01:07.923413+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":518,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.172299+0000","last_change":"2023-10-09T11:02:33.198651+0000","last_active":"2023-10-09T11:02:34.172299+0000","last_peered":"2023-10-09T11:02:34.172299+0000","last_clean":"2023-10-09T11:02:34.172299+0000","last_became_active":"2023-10-09T11:02:33.198378+0000","last_became_peered":"2023-10-09T11:02:33.198378+0000","last_unstale":"2023-10-09T11:02:34.172299+0000","last_undegraded":"2023-10-09T11:02:34.172299+0000","last_fullsized":"2023-10-09T11:02:34.172299+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:27.837451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":117,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.201574+0000","last_change":"2023-10-09T11:00:14.917869+0000","last_active":"2023-10-09T11:02:35.201574+0000","last_peered":"2023-10-09T11:02:35.201574+0000","last_clean":"2023-10-09T11:02:35.201574+0000","last_became_active":"2023-10-09T11:00:14.917535+0000","last_became_peered":"2023-10-09T11:00:14.917535+0000","last_unstale":"2023-10-09T11:02:35.201574+0000","last_undegraded":"2023-10-09T11:02:35.201574+0000","last_fullsized":"2023-10-09T11:02:35.201574+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:10.223866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"438'5714","reported_seq":8052,"reported_epoch":438,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:02:37.975650+0000","last_change":"2023-10-09T11:02:34.408923+0000","last_active":"2023-10-09T11:02:37.975650+0000","last_peered":"2023-10-09T11:02:37.975650+0000","last_clean":"2023-10-09T11:02:32.310266+0000","last_became_active":"2023-10-09T11:02:34.300972+0000","last_became_peered":"2023-10-09T11:02:34.300972+0000","last_unstale":"2023-10-09T11:02:37.975650+0000","last_undegraded":"2023-10-09T11:02:37.975650+0000","last_fullsized":"2023-10-09T11:02:37.975650+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5714,"log_dups_size":0,"ondisk_log_size":5714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:40.292013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374472704,"num_objects":5714,"num_object_clones":0,"num_object_copies":17142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5714,"num_objects_unfound":0,"num_objects_dirty":5714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5714,"num_write_kb":365696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1135,"num_bytes_recovered":74383360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[0,4,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":5714}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":897,"reported_epoch":437,"state":"peering","last_fresh":"2023-10-09T11:02:35.207059+0000","last_change":"2023-10-09T11:02:35.207214+0000","last_active":"2023-10-09T11:02:35.207059+0000","last_peered":"2023-10-09T11:02:34.188979+0000","last_clean":"2023-10-09T11:02:31.327495+0000","last_became_active":"2023-10-09T11:02:34.188756+0000","last_became_peered":"2023-10-09T11:02:34.188756+0000","last_unstale":"2023-10-09T11:02:35.207059+0000","last_undegraded":"2023-10-09T11:02:35.207059+0000","last_fullsized":"2023-10-09T11:02:35.207059+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:15:02.440158+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5,6,7],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":147,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.194202+0000","last_change":"2023-10-09T11:02:33.199808+0000","last_active":"2023-10-09T11:02:35.194202+0000","last_peered":"2023-10-09T11:02:35.194202+0000","last_clean":"2023-10-09T11:02:35.194202+0000","last_became_active":"2023-10-09T11:02:33.199332+0000","last_became_peered":"2023-10-09T11:02:33.199332+0000","last_unstale":"2023-10-09T11:02:35.194202+0000","last_undegraded":"2023-10-09T11:02:35.194202+0000","last_fullsized":"2023-10-09T11:02:35.194202+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:23:19.999621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":276,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.183615+0000","last_change":"2023-10-09T11:01:15.399780+0000","last_active":"2023-10-09T11:02:35.183615+0000","last_peered":"2023-10-09T11:02:35.183615+0000","last_clean":"2023-10-09T11:02:35.183615+0000","last_became_active":"2023-10-09T11:01:15.399493+0000","last_became_peered":"2023-10-09T11:01:15.399493+0000","last_unstale":"2023-10-09T11:02:35.183615+0000","last_undegraded":"2023-10-09T11:02:35.183615+0000","last_fullsized":"2023-10-09T11:02:35.183615+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:41.639383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":117,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.197828+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:02:35.197828+0000","last_peered":"2023-10-09T11:02:35.197828+0000","last_clean":"2023-10-09T11:02:35.197828+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:02:35.197828+0000","last_undegraded":"2023-10-09T11:02:35.197828+0000","last_fullsized":"2023-10-09T11:02:35.197828+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"438'5575","reported_seq":5618,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.473633+0000","last_change":"2023-10-09T11:01:15.406192+0000","last_active":"2023-10-09T11:02:36.473633+0000","last_peered":"2023-10-09T11:02:36.473633+0000","last_clean":"2023-10-09T11:02:36.473633+0000","last_became_active":"2023-10-09T11:01:15.405811+0000","last_became_peered":"2023-10-09T11:01:15.405811+0000","last_unstale":"2023-10-09T11:02:36.473633+0000","last_undegraded":"2023-10-09T11:02:36.473633+0000","last_fullsized":"2023-10-09T11:02:36.473633+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5575,"log_dups_size":0,"ondisk_log_size":5575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:48:38.805496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365363200,"num_objects":5575,"num_object_clones":0,"num_object_copies":16725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5575,"num_write_kb":356800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":638,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.193262+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:02:35.193262+0000","last_peered":"2023-10-09T11:02:35.193262+0000","last_clean":"2023-10-09T11:02:35.193262+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:02:35.193262+0000","last_undegraded":"2023-10-09T11:02:35.193262+0000","last_fullsized":"2023-10-09T11:02:35.193262+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":300,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.181298+0000","last_change":"2023-10-09T11:01:15.401697+0000","last_active":"2023-10-09T11:02:35.181298+0000","last_peered":"2023-10-09T11:02:35.181298+0000","last_clean":"2023-10-09T11:02:35.181298+0000","last_became_active":"2023-10-09T11:01:15.401386+0000","last_became_peered":"2023-10-09T11:01:15.401386+0000","last_unstale":"2023-10-09T11:02:35.181298+0000","last_undegraded":"2023-10-09T11:02:35.181298+0000","last_fullsized":"2023-10-09T11:02:35.181298+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:40.572070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":344,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.199980+0000","last_change":"2023-10-09T10:59:49.126425+0000","last_active":"2023-10-09T11:02:35.199980+0000","last_peered":"2023-10-09T11:02:35.199980+0000","last_clean":"2023-10-09T11:02:35.199980+0000","last_became_active":"2023-10-09T10:59:49.125712+0000","last_became_peered":"2023-10-09T10:59:49.125712+0000","last_unstale":"2023-10-09T11:02:35.199980+0000","last_undegraded":"2023-10-09T11:02:35.199980+0000","last_fullsized":"2023-10-09T11:02:35.199980+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:40:53.898922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":117,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.194008+0000","last_change":"2023-10-09T11:00:14.878579+0000","last_active":"2023-10-09T11:02:35.194008+0000","last_peered":"2023-10-09T11:02:35.194008+0000","last_clean":"2023-10-09T11:02:35.194008+0000","last_became_active":"2023-10-09T11:00:14.867930+0000","last_became_peered":"2023-10-09T11:00:14.867930+0000","last_unstale":"2023-10-09T11:02:35.194008+0000","last_undegraded":"2023-10-09T11:02:35.194008+0000","last_fullsized":"2023-10-09T11:02:35.194008+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:34.925327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"438'5555","reported_seq":8040,"reported_epoch":438,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T11:02:36.622147+0000","last_change":"2023-10-09T11:02:34.415421+0000","last_active":"2023-10-09T11:02:36.622147+0000","last_peered":"2023-10-09T11:02:36.622147+0000","last_clean":"2023-10-09T11:02:32.272642+0000","last_became_active":"2023-10-09T11:02:34.380073+0000","last_became_peered":"2023-10-09T11:02:34.380073+0000","last_unstale":"2023-10-09T11:02:36.622147+0000","last_undegraded":"2023-10-09T11:02:36.622147+0000","last_fullsized":"2023-10-09T11:02:34.172972+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5555,"log_dups_size":0,"ondisk_log_size":5555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:02:17.935352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364052480,"num_objects":5555,"num_object_clones":0,"num_object_copies":16665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5371,"num_objects_unfound":0,"num_objects_dirty":5555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5555,"num_write_kb":355520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1205,"num_bytes_recovered":78970880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":5371},{"shards":"2,5","objects":184}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":873,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.216448+0000","last_change":"2023-10-09T11:02:36.216448+0000","last_active":"2023-10-09T11:02:36.216448+0000","last_peered":"2023-10-09T11:02:36.216448+0000","last_clean":"2023-10-09T11:02:36.216448+0000","last_became_active":"2023-10-09T11:02:36.216040+0000","last_became_peered":"2023-10-09T11:02:36.216040+0000","last_unstale":"2023-10-09T11:02:36.216448+0000","last_undegraded":"2023-10-09T11:02:36.216448+0000","last_fullsized":"2023-10-09T11:02:36.216448+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:13:40.104907+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":189,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.180916+0000","last_change":"2023-10-09T10:59:48.122743+0000","last_active":"2023-10-09T11:02:35.180916+0000","last_peered":"2023-10-09T11:02:35.180916+0000","last_clean":"2023-10-09T11:02:35.180916+0000","last_became_active":"2023-10-09T10:59:48.122447+0000","last_became_peered":"2023-10-09T10:59:48.122447+0000","last_unstale":"2023-10-09T11:02:35.180916+0000","last_undegraded":"2023-10-09T11:02:35.180916+0000","last_fullsized":"2023-10-09T11:02:35.180916+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:28.368331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":117,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.185457+0000","last_change":"2023-10-09T11:00:14.897166+0000","last_active":"2023-10-09T11:02:35.185457+0000","last_peered":"2023-10-09T11:02:35.185457+0000","last_clean":"2023-10-09T11:02:35.185457+0000","last_became_active":"2023-10-09T11:00:14.896917+0000","last_became_peered":"2023-10-09T11:00:14.896917+0000","last_unstale":"2023-10-09T11:02:35.185457+0000","last_undegraded":"2023-10-09T11:02:35.185457+0000","last_fullsized":"2023-10-09T11:02:35.185457+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:09.954705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":115,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.201342+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:02:35.201342+0000","last_peered":"2023-10-09T11:02:35.201342+0000","last_clean":"2023-10-09T11:02:35.201342+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:02:35.201342+0000","last_undegraded":"2023-10-09T11:02:35.201342+0000","last_fullsized":"2023-10-09T11:02:35.201342+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"438'5709","reported_seq":5753,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:38.023629+0000","last_change":"2023-10-09T11:01:15.405163+0000","last_active":"2023-10-09T11:02:38.023629+0000","last_peered":"2023-10-09T11:02:38.023629+0000","last_clean":"2023-10-09T11:02:38.023629+0000","last_became_active":"2023-10-09T11:01:15.404873+0000","last_became_peered":"2023-10-09T11:01:15.404873+0000","last_unstale":"2023-10-09T11:02:38.023629+0000","last_undegraded":"2023-10-09T11:02:38.023629+0000","last_fullsized":"2023-10-09T11:02:38.023629+0000","mapping_epoch":403,"log_start":"406'500","ondisk_log_start":"406'500","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5209,"log_dups_size":0,"ondisk_log_size":5209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:08.894661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374145024,"num_objects":5709,"num_object_clones":0,"num_object_copies":17127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5709,"num_write_kb":365376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":293,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.185018+0000","last_change":"2023-10-09T11:02:33.198446+0000","last_active":"2023-10-09T11:02:35.185018+0000","last_peered":"2023-10-09T11:02:35.185018+0000","last_clean":"2023-10-09T11:02:35.185018+0000","last_became_active":"2023-10-09T11:02:33.197858+0000","last_became_peered":"2023-10-09T11:02:33.197858+0000","last_unstale":"2023-10-09T11:02:35.185018+0000","last_undegraded":"2023-10-09T11:02:35.185018+0000","last_fullsized":"2023-10-09T11:02:35.185018+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:17.362664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":818,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.220226+0000","last_change":"2023-10-09T11:02:36.220226+0000","last_active":"2023-10-09T11:02:36.220226+0000","last_peered":"2023-10-09T11:02:36.220226+0000","last_clean":"2023-10-09T11:02:36.220226+0000","last_became_active":"2023-10-09T11:02:36.219889+0000","last_became_peered":"2023-10-09T11:02:36.219889+0000","last_unstale":"2023-10-09T11:02:36.220226+0000","last_undegraded":"2023-10-09T11:02:36.220226+0000","last_fullsized":"2023-10-09T11:02:36.220226+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:48:54.740905+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":372,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.172838+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T11:02:34.172838+0000","last_peered":"2023-10-09T11:02:34.172838+0000","last_clean":"2023-10-09T11:02:34.172838+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T11:02:34.172838+0000","last_undegraded":"2023-10-09T11:02:34.172838+0000","last_fullsized":"2023-10-09T11:02:34.172838+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":390,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.192514+0000","last_change":"2023-10-09T11:02:33.200249+0000","last_active":"2023-10-09T11:02:35.192514+0000","last_peered":"2023-10-09T11:02:35.192514+0000","last_clean":"2023-10-09T11:02:35.192514+0000","last_became_active":"2023-10-09T11:02:33.199802+0000","last_became_peered":"2023-10-09T11:02:33.199802+0000","last_unstale":"2023-10-09T11:02:35.192514+0000","last_undegraded":"2023-10-09T11:02:35.192514+0000","last_fullsized":"2023-10-09T11:02:35.192514+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:30.703377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"438'5697","reported_seq":8022,"reported_epoch":438,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:02:37.923012+0000","last_change":"2023-10-09T11:02:34.401455+0000","last_active":"2023-10-09T11:02:37.923012+0000","last_peered":"2023-10-09T11:02:37.923012+0000","last_clean":"2023-10-09T11:02:32.305733+0000","last_became_active":"2023-10-09T11:02:34.388542+0000","last_became_peered":"2023-10-09T11:02:34.388542+0000","last_unstale":"2023-10-09T11:02:37.923012+0000","last_undegraded":"2023-10-09T11:02:34.288741+0000","last_fullsized":"2023-10-09T11:02:34.172797+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5697,"log_dups_size":0,"ondisk_log_size":5697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:57:49.189665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373358592,"num_objects":5697,"num_object_clones":0,"num_object_copies":17091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5503,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5697,"num_write_kb":364608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1127,"num_bytes_recovered":73859072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":194},{"shards":"0,2,5","objects":5503}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":773,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.193987+0000","last_change":"2023-10-09T11:01:17.369788+0000","last_active":"2023-10-09T11:02:35.193987+0000","last_peered":"2023-10-09T11:02:35.193987+0000","last_clean":"2023-10-09T11:02:35.193987+0000","last_became_active":"2023-10-09T11:00:14.881494+0000","last_became_peered":"2023-10-09T11:00:14.881494+0000","last_unstale":"2023-10-09T11:02:35.193987+0000","last_undegraded":"2023-10-09T11:02:35.193987+0000","last_fullsized":"2023-10-09T11:02:35.193987+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:44.208863+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":13,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.199858+0000","last_change":"2023-10-09T11:02:33.198484+0000","last_active":"2023-10-09T11:02:35.199858+0000","last_peered":"2023-10-09T11:02:35.199858+0000","last_clean":"2023-10-09T11:02:35.199858+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:02:35.199858+0000","last_undegraded":"2023-10-09T11:02:35.199858+0000","last_fullsized":"2023-10-09T11:02:35.199858+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":74,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.206994+0000","last_change":"2023-10-09T10:59:48.135546+0000","last_active":"2023-10-09T11:02:35.206994+0000","last_peered":"2023-10-09T11:02:35.206994+0000","last_clean":"2023-10-09T11:02:35.206994+0000","last_became_active":"2023-10-09T10:59:48.135227+0000","last_became_peered":"2023-10-09T10:59:48.135227+0000","last_unstale":"2023-10-09T11:02:35.206994+0000","last_undegraded":"2023-10-09T11:02:35.206994+0000","last_fullsized":"2023-10-09T11:02:35.206994+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:50.561785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":395,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.200525+0000","last_change":"2023-10-09T11:00:15.892896+0000","last_active":"2023-10-09T11:02:35.200525+0000","last_peered":"2023-10-09T11:02:35.200525+0000","last_clean":"2023-10-09T11:02:35.200525+0000","last_became_active":"2023-10-09T11:00:15.892573+0000","last_became_peered":"2023-10-09T11:00:15.892573+0000","last_unstale":"2023-10-09T11:02:35.200525+0000","last_undegraded":"2023-10-09T11:02:35.200525+0000","last_fullsized":"2023-10-09T11:02:35.200525+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:13.280089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"438'5635","reported_seq":5679,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:37.989715+0000","last_change":"2023-10-09T11:01:15.402557+0000","last_active":"2023-10-09T11:02:37.989715+0000","last_peered":"2023-10-09T11:02:37.989715+0000","last_clean":"2023-10-09T11:02:37.989715+0000","last_became_active":"2023-10-09T11:01:15.402279+0000","last_became_peered":"2023-10-09T11:01:15.402279+0000","last_unstale":"2023-10-09T11:02:37.989715+0000","last_undegraded":"2023-10-09T11:02:37.989715+0000","last_fullsized":"2023-10-09T11:02:37.989715+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5635,"log_dups_size":0,"ondisk_log_size":5635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:19.333267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369295360,"num_objects":5635,"num_object_clones":0,"num_object_copies":16905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5635,"num_write_kb":360640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":651,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.199897+0000","last_change":"2023-10-09T11:01:16.782558+0000","last_active":"2023-10-09T11:02:35.199897+0000","last_peered":"2023-10-09T11:02:35.199897+0000","last_clean":"2023-10-09T11:02:35.199897+0000","last_became_active":"2023-10-09T10:59:51.137287+0000","last_became_peered":"2023-10-09T10:59:51.137287+0000","last_unstale":"2023-10-09T11:02:35.199897+0000","last_undegraded":"2023-10-09T11:02:35.199897+0000","last_fullsized":"2023-10-09T11:02:35.199897+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:58:29.405479+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":13,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.195352+0000","last_change":"2023-10-09T11:02:33.199031+0000","last_active":"2023-10-09T11:02:35.195352+0000","last_peered":"2023-10-09T11:02:35.195352+0000","last_clean":"2023-10-09T11:02:35.195352+0000","last_became_active":"2023-10-09T11:02:33.198251+0000","last_became_peered":"2023-10-09T11:02:33.198251+0000","last_unstale":"2023-10-09T11:02:35.195352+0000","last_undegraded":"2023-10-09T11:02:35.195352+0000","last_fullsized":"2023-10-09T11:02:35.195352+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:21.904005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":117,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.194705+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:02:35.194705+0000","last_peered":"2023-10-09T11:02:35.194705+0000","last_clean":"2023-10-09T11:02:35.194705+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:02:35.194705+0000","last_undegraded":"2023-10-09T11:02:35.194705+0000","last_fullsized":"2023-10-09T11:02:35.194705+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":379,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.200058+0000","last_change":"2023-10-09T11:02:33.198297+0000","last_active":"2023-10-09T11:02:35.200058+0000","last_peered":"2023-10-09T11:02:35.200058+0000","last_clean":"2023-10-09T11:02:35.200058+0000","last_became_active":"2023-10-09T11:02:33.197978+0000","last_became_peered":"2023-10-09T11:02:33.197978+0000","last_unstale":"2023-10-09T11:02:35.200058+0000","last_undegraded":"2023-10-09T11:02:35.200058+0000","last_fullsized":"2023-10-09T11:02:35.200058+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:41.689146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"438'5567","reported_seq":5610,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.434756+0000","last_change":"2023-10-09T11:01:15.406858+0000","last_active":"2023-10-09T11:02:36.434756+0000","last_peered":"2023-10-09T11:02:36.434756+0000","last_clean":"2023-10-09T11:02:36.434756+0000","last_became_active":"2023-10-09T11:01:15.406513+0000","last_became_peered":"2023-10-09T11:01:15.406513+0000","last_unstale":"2023-10-09T11:02:36.434756+0000","last_undegraded":"2023-10-09T11:02:36.434756+0000","last_fullsized":"2023-10-09T11:02:36.434756+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5567,"log_dups_size":0,"ondisk_log_size":5567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:28:25.137720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364838912,"num_objects":5567,"num_object_clones":0,"num_object_copies":16701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5567,"num_write_kb":356288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":833,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.192435+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T11:02:35.192435+0000","last_peered":"2023-10-09T11:02:35.192435+0000","last_clean":"2023-10-09T11:02:35.192435+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T11:02:35.192435+0000","last_undegraded":"2023-10-09T11:02:35.192435+0000","last_fullsized":"2023-10-09T11:02:35.192435+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":540,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.175193+0000","last_change":"2023-10-09T11:02:33.206628+0000","last_active":"2023-10-09T11:02:34.175193+0000","last_peered":"2023-10-09T11:02:34.175193+0000","last_clean":"2023-10-09T11:02:34.175193+0000","last_became_active":"2023-10-09T11:02:33.200124+0000","last_became_peered":"2023-10-09T11:02:33.200124+0000","last_unstale":"2023-10-09T11:02:34.175193+0000","last_undegraded":"2023-10-09T11:02:34.175193+0000","last_fullsized":"2023-10-09T11:02:34.175193+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:42:19.368427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":117,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.195282+0000","last_change":"2023-10-09T11:00:14.900305+0000","last_active":"2023-10-09T11:02:35.195282+0000","last_peered":"2023-10-09T11:02:35.195282+0000","last_clean":"2023-10-09T11:02:35.195282+0000","last_became_active":"2023-10-09T11:00:14.899724+0000","last_became_peered":"2023-10-09T11:00:14.899724+0000","last_unstale":"2023-10-09T11:02:35.195282+0000","last_undegraded":"2023-10-09T11:02:35.195282+0000","last_fullsized":"2023-10-09T11:02:35.195282+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:59.022161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":517,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.172402+0000","last_change":"2023-10-09T11:02:33.198248+0000","last_active":"2023-10-09T11:02:34.172402+0000","last_peered":"2023-10-09T11:02:34.172402+0000","last_clean":"2023-10-09T11:02:34.172402+0000","last_became_active":"2023-10-09T11:02:33.197876+0000","last_became_peered":"2023-10-09T11:02:33.197876+0000","last_unstale":"2023-10-09T11:02:34.172402+0000","last_undegraded":"2023-10-09T11:02:34.172402+0000","last_fullsized":"2023-10-09T11:02:34.172402+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:17.182284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"438'5653","reported_seq":5696,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.445119+0000","last_change":"2023-10-09T11:01:15.397123+0000","last_active":"2023-10-09T11:02:36.445119+0000","last_peered":"2023-10-09T11:02:36.445119+0000","last_clean":"2023-10-09T11:02:36.445119+0000","last_became_active":"2023-10-09T11:01:15.396842+0000","last_became_peered":"2023-10-09T11:01:15.396842+0000","last_unstale":"2023-10-09T11:02:36.445119+0000","last_undegraded":"2023-10-09T11:02:36.445119+0000","last_fullsized":"2023-10-09T11:02:36.445119+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5653,"log_dups_size":0,"ondisk_log_size":5653,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:54.895524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370475008,"num_objects":5653,"num_object_clones":0,"num_object_copies":16959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5653,"num_write_kb":361792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":891,"reported_epoch":437,"state":"peering","last_fresh":"2023-10-09T11:02:35.198306+0000","last_change":"2023-10-09T11:02:35.198467+0000","last_active":"2023-10-09T11:02:35.198306+0000","last_peered":"2023-10-09T11:02:34.191745+0000","last_clean":"2023-10-09T11:02:31.327799+0000","last_became_active":"2023-10-09T11:02:34.191532+0000","last_became_peered":"2023-10-09T11:02:34.191532+0000","last_unstale":"2023-10-09T11:02:35.198306+0000","last_undegraded":"2023-10-09T11:02:35.198306+0000","last_fullsized":"2023-10-09T11:02:35.198306+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:19.911311+0000","scrub_duration":0.0053707060000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5,6,7],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":13,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.200758+0000","last_change":"2023-10-09T11:02:33.198517+0000","last_active":"2023-10-09T11:02:35.200758+0000","last_peered":"2023-10-09T11:02:35.200758+0000","last_clean":"2023-10-09T11:02:35.200758+0000","last_became_active":"2023-10-09T11:02:33.197676+0000","last_became_peered":"2023-10-09T11:02:33.197676+0000","last_unstale":"2023-10-09T11:02:35.200758+0000","last_undegraded":"2023-10-09T11:02:35.200758+0000","last_fullsized":"2023-10-09T11:02:35.200758+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:40:41.647813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":379,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.200579+0000","last_change":"2023-10-09T11:00:15.869009+0000","last_active":"2023-10-09T11:02:35.200579+0000","last_peered":"2023-10-09T11:02:35.200579+0000","last_clean":"2023-10-09T11:02:35.200579+0000","last_became_active":"2023-10-09T11:00:15.868637+0000","last_became_peered":"2023-10-09T11:00:15.868637+0000","last_unstale":"2023-10-09T11:02:35.200579+0000","last_undegraded":"2023-10-09T11:02:35.200579+0000","last_fullsized":"2023-10-09T11:02:35.200579+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:37.538741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":223,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.193169+0000","last_change":"2023-10-09T11:01:15.402661+0000","last_active":"2023-10-09T11:02:35.193169+0000","last_peered":"2023-10-09T11:02:35.193169+0000","last_clean":"2023-10-09T11:02:35.193169+0000","last_became_active":"2023-10-09T11:01:15.402327+0000","last_became_peered":"2023-10-09T11:01:15.402327+0000","last_unstale":"2023-10-09T11:02:35.193169+0000","last_undegraded":"2023-10-09T11:02:35.193169+0000","last_fullsized":"2023-10-09T11:02:35.193169+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:32.513751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":864,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.172481+0000","last_change":"2023-10-09T10:59:51.145348+0000","last_active":"2023-10-09T11:02:34.172481+0000","last_peered":"2023-10-09T11:02:34.172481+0000","last_clean":"2023-10-09T11:02:34.172481+0000","last_became_active":"2023-10-09T10:59:51.145100+0000","last_became_peered":"2023-10-09T10:59:51.145100+0000","last_unstale":"2023-10-09T11:02:34.172481+0000","last_undegraded":"2023-10-09T11:02:34.172481+0000","last_fullsized":"2023-10-09T11:02:34.172481+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:51.049260+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":378,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.200111+0000","last_change":"2023-10-09T11:02:33.201950+0000","last_active":"2023-10-09T11:02:35.200111+0000","last_peered":"2023-10-09T11:02:35.200111+0000","last_clean":"2023-10-09T11:02:35.200111+0000","last_became_active":"2023-10-09T11:02:33.201559+0000","last_became_peered":"2023-10-09T11:02:33.201559+0000","last_unstale":"2023-10-09T11:02:35.200111+0000","last_undegraded":"2023-10-09T11:02:35.200111+0000","last_fullsized":"2023-10-09T11:02:35.200111+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:45.851351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":193,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.169984+0000","last_change":"2023-10-09T11:00:14.886903+0000","last_active":"2023-10-09T11:02:34.169984+0000","last_peered":"2023-10-09T11:02:34.169984+0000","last_clean":"2023-10-09T11:02:34.169984+0000","last_became_active":"2023-10-09T11:00:14.886635+0000","last_became_peered":"2023-10-09T11:00:14.886635+0000","last_unstale":"2023-10-09T11:02:34.169984+0000","last_undegraded":"2023-10-09T11:02:34.169984+0000","last_fullsized":"2023-10-09T11:02:34.169984+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:04.982085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":822,"reported_epoch":437,"state":"peering","last_fresh":"2023-10-09T11:02:35.199037+0000","last_change":"2023-10-09T11:02:35.199178+0000","last_active":"2023-10-09T11:02:35.199037+0000","last_peered":"2023-10-09T11:02:34.188956+0000","last_clean":"2023-10-09T11:02:31.326201+0000","last_became_active":"2023-10-09T11:02:34.188635+0000","last_became_peered":"2023-10-09T11:02:34.188635+0000","last_unstale":"2023-10-09T11:02:35.199037+0000","last_undegraded":"2023-10-09T11:02:35.199037+0000","last_fullsized":"2023-10-09T11:02:35.199037+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:55:55.577010+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,6,7],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":265,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.181923+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:02:35.181923+0000","last_peered":"2023-10-09T11:02:35.181923+0000","last_clean":"2023-10-09T11:02:35.181923+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:02:35.181923+0000","last_undegraded":"2023-10-09T11:02:35.181923+0000","last_fullsized":"2023-10-09T11:02:35.181923+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":465,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.173501+0000","last_change":"2023-10-09T10:57:46.844827+0000","last_active":"2023-10-09T11:02:34.173501+0000","last_peered":"2023-10-09T11:02:34.173501+0000","last_clean":"2023-10-09T11:02:34.173501+0000","last_became_active":"2023-10-09T10:57:46.844584+0000","last_became_peered":"2023-10-09T10:57:46.844584+0000","last_unstale":"2023-10-09T11:02:34.173501+0000","last_undegraded":"2023-10-09T11:02:34.173501+0000","last_fullsized":"2023-10-09T11:02:34.173501+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:54.499813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":388,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.183525+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T11:02:35.183525+0000","last_peered":"2023-10-09T11:02:35.183525+0000","last_clean":"2023-10-09T11:02:35.183525+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T11:02:35.183525+0000","last_undegraded":"2023-10-09T11:02:35.183525+0000","last_fullsized":"2023-10-09T11:02:35.183525+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":401,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.172912+0000","last_change":"2023-10-09T10:59:48.136562+0000","last_active":"2023-10-09T11:02:34.172912+0000","last_peered":"2023-10-09T11:02:34.172912+0000","last_clean":"2023-10-09T11:02:34.172912+0000","last_became_active":"2023-10-09T10:59:48.136184+0000","last_became_peered":"2023-10-09T10:59:48.136184+0000","last_unstale":"2023-10-09T11:02:34.172912+0000","last_undegraded":"2023-10-09T11:02:34.172912+0000","last_fullsized":"2023-10-09T11:02:34.172912+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:16.944093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":668,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-09T11:02:36.217717+0000","last_change":"2023-10-09T11:02:36.217717+0000","last_active":"2023-10-09T11:02:36.217717+0000","last_peered":"2023-10-09T11:02:36.217717+0000","last_clean":"2023-10-09T11:02:36.217717+0000","last_became_active":"2023-10-09T11:02:36.217373+0000","last_became_peered":"2023-10-09T11:02:36.217373+0000","last_unstale":"2023-10-09T11:02:36.217717+0000","last_undegraded":"2023-10-09T11:02:36.217717+0000","last_fullsized":"2023-10-09T11:02:36.217717+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:47.307693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:47.307693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":464,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.174129+0000","last_change":"2023-10-09T10:54:17.886158+0000","last_active":"2023-10-09T11:02:34.174129+0000","last_peered":"2023-10-09T11:02:34.174129+0000","last_clean":"2023-10-09T11:02:34.174129+0000","last_became_active":"2023-10-09T10:54:17.885816+0000","last_became_peered":"2023-10-09T10:54:17.885816+0000","last_unstale":"2023-10-09T11:02:34.174129+0000","last_undegraded":"2023-10-09T11:02:34.174129+0000","last_fullsized":"2023-10-09T11:02:34.174129+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:05.270825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":484,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T11:02:34.172671+0000","last_change":"2023-10-09T10:59:48.125955+0000","last_active":"2023-10-09T11:02:34.172671+0000","last_peered":"2023-10-09T11:02:34.172671+0000","last_clean":"2023-10-09T11:02:34.172671+0000","last_became_active":"2023-10-09T10:59:48.125641+0000","last_became_peered":"2023-10-09T10:59:48.125641+0000","last_unstale":"2023-10-09T11:02:34.172671+0000","last_undegraded":"2023-10-09T11:02:34.172671+0000","last_fullsized":"2023-10-09T11:02:34.172671+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:38.041100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":694,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.195610+0000","last_change":"2023-10-09T11:02:25.273280+0000","last_active":"2023-10-09T11:02:35.195610+0000","last_peered":"2023-10-09T11:02:35.195610+0000","last_clean":"2023-10-09T11:02:35.195610+0000","last_became_active":"2023-10-09T11:00:14.867744+0000","last_became_peered":"2023-10-09T11:00:14.867744+0000","last_unstale":"2023-10-09T11:02:35.195610+0000","last_undegraded":"2023-10-09T11:02:35.195610+0000","last_fullsized":"2023-10-09T11:02:35.195610+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:30:37.506583+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":80,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.200687+0000","last_change":"2023-10-09T10:59:48.143732+0000","last_active":"2023-10-09T11:02:35.200687+0000","last_peered":"2023-10-09T11:02:35.200687+0000","last_clean":"2023-10-09T11:02:35.200687+0000","last_became_active":"2023-10-09T10:59:48.129001+0000","last_became_peered":"2023-10-09T10:59:48.129001+0000","last_unstale":"2023-10-09T11:02:35.200687+0000","last_undegraded":"2023-10-09T11:02:35.200687+0000","last_fullsized":"2023-10-09T11:02:35.200687+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:22:36.946348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":459,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T11:02:35.194509+0000","last_change":"2023-10-09T11:00:14.869353+0000","last_active":"2023-10-09T11:02:35.194509+0000","last_peered":"2023-10-09T11:02:35.194509+0000","last_clean":"2023-10-09T11:02:35.194509+0000","last_became_active":"2023-10-09T11:00:14.869061+0000","last_became_peered":"2023-10-09T11:00:14.869061+0000","last_unstale":"2023-10-09T11:02:35.194509+0000","last_undegraded":"2023-10-09T11:02:35.194509+0000","last_fullsized":"2023-10-09T11:02:35.194509+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:30.318094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5905055744,"num_objects":90104,"num_object_clones":0,"num_object_copies":270312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5503,"num_objects_misplaced":16830,"num_objects_unfound":0,"num_objects_dirty":90104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":90104,"num_write_kb":5766656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4602,"num_bytes_recovered":301596672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17705140224,"data_stored":17705140224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89004,"ondisk_log_size":89004,"up":48,"acting":46,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":17098,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8961,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":599313,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1371411,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332432,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":951608,"kb_used_data":48112,"kb_used_omap":0,"kb_used_meta":903488,"kb_avail":92793544,"statfs":{"total":95995035648,"available":95020589056,"internally_reserved":0,"allocated":49266688,"data_stored":47876252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925171712},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:11 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":1.6359999999999999,"15min":1.7729999999999999},"min":{"1min":0.40999999999999998,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":2.8210000000000002,"5min":8.9730000000000008,"15min":9.6379999999999999},"last":4.0049999999999999},{"interface":"front","average":{"1min":0.49399999999999999,"5min":1.429,"15min":1.881},"min":{"1min":0.23499999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.95899999999999996,"5min":7.5609999999999999,"15min":16.731000000000002},"last":4.258}]},{"osd":1,"last update":"Mon Oct 9 11:02:15 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.5,"15min":1.609},"min":{"1min":0.26900000000000002,"5min":0.247,"15min":0.247},"max":{"1min":2.589,"5min":6.202,"15min":6.202},"last":2.9409999999999998},{"interface":"front","average":{"1min":0.65800000000000003,"5min":1.772,"15min":1.958},"min":{"1min":0.26600000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.0379999999999998,"5min":12.193,"15min":12.193},"last":3.5649999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:01:45 2023","interfaces":[{"interface":"back","average":{"1min":1.274,"5min":1.595,"15min":1.6699999999999999},"min":{"1min":0.30199999999999999,"5min":0.245,"15min":0.245},"max":{"1min":3.8610000000000002,"5min":6.0999999999999996,"15min":11.058},"last":2.605},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.946,"15min":1.7649999999999999},"min":{"1min":0.315,"5min":0.28699999999999998,"15min":0.26600000000000001},"max":{"1min":3.2480000000000002,"5min":13.037000000000001,"15min":13.037000000000001},"last":2.0059999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":1.6819999999999999,"15min":2.0470000000000002},"min":{"1min":0.378,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.698,"5min":13.464,"15min":13.464},"last":1.1970000000000001},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.734,"15min":1.8440000000000001},"min":{"1min":0.371,"5min":0.31,"15min":0.31},"max":{"1min":5.0919999999999996,"5min":9.0210000000000008,"15min":19.234999999999999},"last":0.74199999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":1.266,"15min":1.4370000000000001},"min":{"1min":0.33300000000000002,"5min":0.248,"15min":0.24099999999999999},"max":{"1min":0.67600000000000005,"5min":7.96,"15min":16.695},"last":2.7360000000000002},{"interface":"front","average":{"1min":0.752,"5min":1.2330000000000001,"15min":1.337},"min":{"1min":0.315,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.6110000000000002,"5min":7.0510000000000002,"15min":11.109999999999999},"last":0.51200000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":1.6819999999999999,"15min":1.5429999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":2.9260000000000002,"5min":13.407,"15min":13.407},"last":2.8969999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.1910000000000001,"15min":1.4970000000000001},"min":{"1min":0.30499999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.9510000000000001,"5min":7.9180000000000001,"15min":11.741},"last":1.2529999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.321,"15min":1.6439999999999999},"min":{"1min":0.38400000000000001,"5min":0.34599999999999997,"15min":0.311},"max":{"1min":2.27,"5min":10.388999999999999,"15min":14.913},"last":1.278},{"interface":"front","average":{"1min":0.73899999999999999,"5min":1.1990000000000001,"15min":1.4990000000000001},"min":{"1min":0.33900000000000002,"5min":0.32600000000000001,"15min":0.29399999999999998},"max":{"1min":2.5710000000000002,"5min":7.1139999999999999,"15min":19.187999999999999},"last":0.48199999999999998}]}]},{"osd":6,"up_from":44,"seq":188978561363,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3390504,"kb_used_data":2494432,"kb_used_omap":0,"kb_used_meta":896064,"kb_avail":90354648,"statfs":{"total":95995035648,"available":92523159552,"internally_reserved":0,"allocated":2554298368,"data_stored":2552923626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917569536},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:07 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":1.8009999999999999,"15min":1.877},"min":{"1min":0.224,"5min":0.224,"15min":0.216},"max":{"1min":3.9460000000000002,"5min":46.603999999999999,"15min":54.743000000000002},"last":29.379999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":2.0760000000000001,"15min":1.9790000000000001},"min":{"1min":0.28999999999999998,"5min":0.223,"15min":0.223},"max":{"1min":1.9039999999999999,"5min":46.679000000000002,"15min":53.701000000000001},"last":29.132999999999999}]},{"osd":1,"last update":"Mon Oct 9 11:02:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0529999999999999,"5min":1.2130000000000001,"15min":0.90700000000000003},"min":{"1min":0.29399999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":41.241999999999997,"5min":41.241999999999997,"15min":41.241999999999997},"last":0.39000000000000001},{"interface":"front","average":{"1min":3.5070000000000001,"5min":1.792,"15min":1.506},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":41.476999999999997,"5min":41.476999999999997,"15min":41.476999999999997},"last":0.26200000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.794,"15min":1.843},"min":{"1min":0.34999999999999998,"5min":0.30599999999999999,"15min":0.28399999999999997},"max":{"1min":3.2759999999999998,"5min":12.003,"15min":13.208},"last":0.314},{"interface":"front","average":{"1min":1.7829999999999999,"5min":2.0150000000000001,"15min":2.0009999999999999},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.28599999999999998},"max":{"1min":7.1870000000000003,"5min":15.432,"15min":15.432},"last":0.49199999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:02:07 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.6719999999999999,"15min":1.8320000000000001},"min":{"1min":0.35499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":3.266,"5min":10.018000000000001,"15min":12.391},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.101,"5min":1.885,"15min":2.0870000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":3.6389999999999998,"5min":11.161,"15min":13.179},"last":0.34399999999999997}]},{"osd":4,"last update":"Mon Oct 9 11:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.90100000000000002,"15min":1.2949999999999999},"min":{"1min":0.25900000000000001,"5min":0.25800000000000001,"15min":0.23200000000000001},"max":{"1min":3.9449999999999998,"5min":9.0060000000000002,"15min":16.978999999999999},"last":0.438},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.88300000000000001,"15min":1.2450000000000001},"min":{"1min":0.27800000000000002,"5min":0.249,"15min":0.24399999999999999},"max":{"1min":2.944,"5min":5.7869999999999999,"15min":12.481999999999999},"last":0.51000000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.385,"15min":1.5600000000000001},"min":{"1min":0.30499999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.9260000000000002,"5min":19.786999999999999,"15min":19.786999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.083,"15min":1.339},"min":{"1min":0.35099999999999998,"5min":0.28899999999999998,"15min":0.26600000000000001},"max":{"1min":3.6850000000000001,"5min":9.0559999999999992,"15min":12.456},"last":0.45300000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:01:59 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.2989999999999999,"15min":1.5549999999999999},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.26600000000000001},"max":{"1min":1.885,"5min":9.1120000000000001,"15min":23.358000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.96699999999999997,"15min":1.216},"min":{"1min":0.32400000000000001,"5min":0.32300000000000001,"15min":0.27500000000000002},"max":{"1min":0.83199999999999996,"5min":6.1920000000000002,"15min":12.422000000000001},"last":0.47399999999999998}]}]},{"osd":1,"up_from":388,"seq":1666447310878,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2003004,"kb_used_data":1762868,"kb_used_omap":13,"kb_used_meta":240114,"kb_avail":91742148,"statfs":{"total":95995035648,"available":93943959552,"internally_reserved":0,"allocated":1805176832,"data_stored":1802923744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":245877447},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":3.2690000000000001,"5min":2.46,"15min":2.3250000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":13.334,"5min":13.334,"15min":13.334},"last":1.4530000000000001},{"interface":"front","average":{"1min":2.968,"5min":1.6419999999999999,"15min":1.421},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.6829999999999998,"5min":9.6829999999999998,"15min":9.6829999999999998},"last":1.123}]},{"osd":2,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":2.9790000000000001,"15min":3.1389999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.4670000000000005,"5min":12.009,"15min":12.009},"last":1.407},{"interface":"front","average":{"1min":2.5499999999999998,"5min":2.2959999999999998,"15min":2.254},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.8249999999999993,"5min":9.8249999999999993,"15min":9.8249999999999993},"last":0.752}]},{"osd":3,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7879999999999998,"5min":2.8199999999999998,"15min":2.8250000000000002},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":9.6460000000000008,"5min":12.523999999999999,"15min":12.523999999999999},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.518,"5min":3.5539999999999998,"15min":3.8940000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.3799999999999999,"5min":18.079999999999998,"15min":18.079999999999998},"last":0.621}]},{"osd":4,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":2.536,"15min":2.754},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.2300000000000004,"5min":10.481,"15min":10.481},"last":1.379},{"interface":"front","average":{"1min":1.702,"5min":1.448,"15min":1.4059999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":9.8480000000000008,"5min":9.8480000000000008,"15min":9.8480000000000008},"last":0.66500000000000004}]},{"osd":5,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":2.29,"15min":2.4580000000000002},"min":{"1min":0.35599999999999998,"5min":0.311,"15min":0.311},"max":{"1min":4.8799999999999999,"5min":18.053000000000001,"15min":18.053000000000001},"last":4.4420000000000002},{"interface":"front","average":{"1min":1.526,"5min":2.2429999999999999,"15min":2.363},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.2830000000000004,"5min":10.558,"15min":10.558},"last":4.4299999999999997}]},{"osd":6,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":2.2759999999999998,"15min":2.46},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.8520000000000003,"5min":18.085999999999999,"15min":18.085999999999999},"last":0.76400000000000001},{"interface":"front","average":{"1min":1.3049999999999999,"5min":2.1869999999999998,"15min":2.3340000000000001},"min":{"1min":0.38600000000000001,"5min":0.373,"15min":0.373},"max":{"1min":5.3129999999999997,"5min":10.531000000000001,"15min":10.531000000000001},"last":1.3320000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.393,"15min":1.3220000000000001},"min":{"1min":0.41899999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.8000000000000007,"5min":9.8000000000000007,"15min":9.8000000000000007},"last":0.69799999999999995},{"interface":"front","average":{"1min":1.1910000000000001,"5min":2.3279999999999998,"15min":2.5179999999999998},"min":{"1min":0.435,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.8019999999999996,"5min":18.071000000000002,"15min":18.071000000000002},"last":0.96299999999999997}]}]},{"osd":0,"up_from":80,"seq":343597383936,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4349612,"kb_used_data":3649956,"kb_used_omap":28,"kb_used_meta":699619,"kb_avail":89395540,"statfs":{"total":95995035648,"available":91541032960,"internally_reserved":0,"allocated":3737554944,"data_stored":3735250391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":716410704},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:02:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":3.0379999999999998,"15min":3.2040000000000002},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":9.6460000000000008,"5min":19.608000000000001,"15min":19.608000000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.879,"5min":3.0710000000000002,"15min":3.2690000000000001},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.218},"max":{"1min":7.8730000000000002,"5min":19.992999999999999,"15min":19.992999999999999},"last":0.40699999999999997}]},{"osd":2,"last update":"Mon Oct 9 11:01:45 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.958,"15min":2.1480000000000001},"min":{"1min":0.24199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":4.6639999999999997,"5min":12.638,"15min":24.852},"last":0.627},{"interface":"front","average":{"1min":1.6299999999999999,"5min":2.4609999999999999,"15min":2.6219999999999999},"min":{"1min":0.20899999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":8.7140000000000004,"5min":12.672000000000001,"15min":77.256},"last":0.97799999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:02:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":2.1589999999999998,"15min":2.3090000000000002},"min":{"1min":0.26800000000000002,"5min":0.215,"15min":0.20200000000000001},"max":{"1min":8.8699999999999992,"5min":28.946000000000002,"15min":28.946000000000002},"last":0.44},{"interface":"front","average":{"1min":1.46,"5min":2.4350000000000001,"15min":2.5790000000000002},"min":{"1min":0.309,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.9850000000000003,"5min":19.952999999999999,"15min":19.952999999999999},"last":0.60999999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":2.2679999999999998,"15min":2.4649999999999999},"min":{"1min":0.32900000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.9560000000000004,"5min":11.507,"15min":77.177999999999997},"last":2.0920000000000001},{"interface":"front","average":{"1min":1.405,"5min":1.6499999999999999,"15min":2.0190000000000001},"min":{"1min":0.38400000000000001,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":6.4450000000000003,"5min":9.9489999999999998,"15min":12.318},"last":2.98}]},{"osd":5,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.827,"15min":2.077},"min":{"1min":0.36599999999999999,"5min":0.308,"15min":0.29799999999999999},"max":{"1min":6.4880000000000004,"5min":9.8710000000000004,"15min":12.157},"last":1.9890000000000001},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.9350000000000001,"15min":2.0750000000000002},"min":{"1min":0.436,"5min":0.29999999999999999,"15min":0.29899999999999999},"max":{"1min":2.859,"5min":7.0919999999999996,"15min":24.754999999999999},"last":2.206}]},{"osd":6,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5700000000000001,"5min":1.855,"15min":2.2349999999999999},"min":{"1min":0.38800000000000001,"5min":0.314,"15min":0.30399999999999999},"max":{"1min":6.5179999999999998,"5min":10.012,"15min":18.920000000000002},"last":1.4370000000000001},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.9490000000000001,"15min":1.9810000000000001},"min":{"1min":0.45700000000000002,"5min":0.31900000000000001,"15min":0.307},"max":{"1min":2.9209999999999998,"5min":8.8249999999999993,"15min":24.800000000000001},"last":3.8479999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.958,"15min":2.177},"min":{"1min":0.44900000000000001,"5min":0.34499999999999997,"15min":0.32700000000000001},"max":{"1min":2.8929999999999998,"5min":11.285,"15min":14.48},"last":3.7080000000000002},{"interface":"front","average":{"1min":1.3740000000000001,"5min":2.3809999999999998,"15min":2.5910000000000002},"min":{"1min":0.41499999999999998,"5min":0.372,"15min":0.314},"max":{"1min":6.008,"5min":10.398,"15min":77.228999999999999},"last":2.6579999999999999}]}]},{"osd":2,"up_from":90,"seq":386547056876,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2649652,"kb_used_data":2141676,"kb_used_omap":81,"kb_used_meta":507886,"kb_avail":91095500,"statfs":{"total":95995035648,"available":93281792000,"internally_reserved":0,"allocated":2193076224,"data_stored":2190934369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":520075482},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":1.3819999999999999,"15min":1.7569999999999999},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":8.9990000000000006,"5min":8.9990000000000006,"15min":18.512},"last":1.212},{"interface":"front","average":{"1min":1.5269999999999999,"5min":1.5609999999999999,"15min":1.9710000000000001},"min":{"1min":0.20699999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":7.7469999999999999,"5min":7.7469999999999999,"15min":17.975000000000001},"last":1.0900000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:02:18 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":2.403,"15min":2.4830000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":10.502000000000001,"5min":10.502000000000001,"15min":10.502000000000001},"last":0.20899999999999999},{"interface":"front","average":{"1min":1.6200000000000001,"5min":2.448,"15min":2.5859999999999999},"min":{"1min":0.27300000000000002,"5min":0.214,"15min":0.214},"max":{"1min":5.125,"5min":11.92,"15min":11.92},"last":0.83599999999999997}]},{"osd":3,"last update":"Mon Oct 9 11:02:09 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":1.6279999999999999,"15min":1.863},"min":{"1min":0.24399999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.5979999999999999,"5min":11.151,"15min":19.539000000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":1.01,"5min":1.913,"15min":2.093},"min":{"1min":0.27200000000000002,"5min":0.219,"15min":0.219},"max":{"1min":3.524,"5min":16.324000000000002,"15min":16.702999999999999},"last":0.48099999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":2.016,"15min":1.984},"min":{"1min":0.35399999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.4859999999999998,"5min":9.3290000000000006,"15min":16.231999999999999},"last":1.2270000000000001},{"interface":"front","average":{"1min":1.458,"5min":1.512,"15min":1.782},"min":{"1min":0.33400000000000002,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":4.8470000000000004,"5min":6.8490000000000002,"15min":15.696999999999999},"last":1.2450000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.581,"15min":1.8080000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.6369999999999996,"5min":8.5380000000000003,"15min":17.920000000000002},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.6639999999999999,"5min":1.954,"15min":1.899},"min":{"1min":0.311,"5min":0.311,"15min":0.30299999999999999},"max":{"1min":7.5250000000000004,"5min":8.5869999999999997,"15min":17.882999999999999},"last":0.56899999999999995}]},{"osd":6,"last update":"Mon Oct 9 11:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":1.341,"15min":1.5609999999999999},"min":{"1min":0.373,"5min":0.32100000000000001,"15min":0.30499999999999999},"max":{"1min":4.3239999999999998,"5min":7.4009999999999998,"15min":10.443},"last":6.4569999999999999},{"interface":"front","average":{"1min":1.383,"5min":1.429,"15min":1.74},"min":{"1min":0.36099999999999999,"5min":0.313,"15min":0.313},"max":{"1min":6.6749999999999998,"5min":6.6749999999999998,"15min":12.862},"last":0.51300000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.679,"5min":1.6399999999999999,"15min":1.6699999999999999},"min":{"1min":0.39000000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.5609999999999999,"5min":6.3300000000000001,"15min":10.470000000000001},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.393,"5min":1.607,"15min":1.883},"min":{"1min":0.34699999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.71,"5min":7.6550000000000002,"15min":12.826000000000001},"last":0.68100000000000005}]}]},{"osd":3,"up_from":95,"seq":408021893350,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3115272,"kb_used_data":2137152,"kb_used_omap":7,"kb_used_meta":978104,"kb_avail":90629880,"statfs":{"total":95995035648,"available":92804997120,"internally_reserved":0,"allocated":2188443648,"data_stored":2186104550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1001579112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":2.556,"15min":2.6800000000000002},"min":{"1min":0.222,"5min":0.20499999999999999,"15min":0.17499999999999999},"max":{"1min":8.1709999999999994,"5min":10.140000000000001,"15min":38.817999999999998},"last":0.23000000000000001},{"interface":"front","average":{"1min":2.4769999999999999,"5min":2.3530000000000002,"15min":2.9649999999999999},"min":{"1min":0.246,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":8.4239999999999995,"5min":17.042000000000002,"15min":17.312000000000001},"last":2.5030000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:02:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8,"5min":2.9199999999999999,"15min":3.1059999999999999},"min":{"1min":0.25,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":8.8030000000000008,"5min":8.9100000000000001,"15min":8.9100000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.4650000000000001,"5min":2.5840000000000001,"15min":2.77},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.2300000000000004,"5min":9.7449999999999992,"15min":9.7449999999999992},"last":0.47499999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":2.3100000000000001,"5min":2.3069999999999999,"15min":2.448},"min":{"1min":0.21099999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":5.5350000000000001,"5min":10.65,"15min":20.402000000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.0489999999999999,"5min":2.2839999999999998,"15min":2.8260000000000001},"min":{"1min":0.23899999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":8.2729999999999997,"5min":10.012,"15min":27.538},"last":2.536}]},{"osd":4,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.355,"5min":2.101,"15min":2.2770000000000001},"min":{"1min":0.30299999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.3639999999999999,"5min":9.8089999999999993,"15min":20.388000000000002},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.3029999999999999,"5min":1.7929999999999999,"15min":2.157},"min":{"1min":0.34499999999999997,"5min":0.34000000000000002,"15min":0.24299999999999999},"max":{"1min":3.5910000000000002,"5min":9.1310000000000002,"15min":19.085999999999999},"last":2.633}]},{"osd":5,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.948,"15min":2.2890000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.26000000000000001},"max":{"1min":3.5099999999999998,"5min":10.380000000000001,"15min":14.045},"last":4.3570000000000002},{"interface":"front","average":{"1min":1.4630000000000001,"5min":1.7470000000000001,"15min":2.0369999999999999},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2839999999999998,"5min":8.3949999999999996,"15min":17.311},"last":2.8479999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":2.1419999999999999,"15min":2.27},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28000000000000003},"max":{"1min":8.3119999999999994,"5min":10.420999999999999,"15min":12.361000000000001},"last":4.306},{"interface":"front","average":{"1min":1.835,"5min":1.9430000000000001,"15min":2.2090000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.8959999999999999,"5min":7.1230000000000002,"15min":17.295000000000002},"last":4.3360000000000003}]},{"osd":7,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":1.9419999999999999,"15min":2.2440000000000002},"min":{"1min":0.46400000000000002,"5min":0.309,"15min":0.309},"max":{"1min":7.492,"5min":8.6649999999999991,"15min":18.722000000000001},"last":0.69999999999999996},{"interface":"front","average":{"1min":1.6180000000000001,"5min":2.194,"15min":2.4670000000000001},"min":{"1min":0.439,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.4020000000000001,"5min":9.8569999999999993,"15min":15.212999999999999},"last":0.45600000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019226,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4078004,"kb_used_data":3193452,"kb_used_omap":0,"kb_used_meta":884544,"kb_avail":89667148,"statfs":{"total":95995035648,"available":91819159552,"internally_reserved":0,"allocated":3270094848,"data_stored":3268708217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":905773056},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.411,"15min":2.3370000000000002},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":2.9870000000000001,"5min":8.6180000000000003,"15min":213.88999999999999},"last":1.5980000000000001},{"interface":"front","average":{"1min":1.393,"5min":1.359,"15min":2.6019999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.217},"max":{"1min":6.1269999999999998,"5min":6.1269999999999998,"15min":214.09800000000001},"last":2.1419999999999999}]},{"osd":1,"last update":"Mon Oct 9 11:02:15 2023","interfaces":[{"interface":"back","average":{"1min":8.0950000000000006,"5min":3.2919999999999998,"15min":2.492},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":121.41,"5min":121.41,"15min":121.41},"last":6.5140000000000002},{"interface":"front","average":{"1min":7.8860000000000001,"5min":2.6699999999999999,"15min":1.8009999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":122.21899999999999,"5min":122.21899999999999,"15min":122.21899999999999},"last":1.52}]},{"osd":2,"last update":"Mon Oct 9 11:01:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":1.4550000000000001,"15min":1.7809999999999999},"min":{"1min":0.27900000000000003,"5min":0.27100000000000002,"15min":0.253},"max":{"1min":3.9390000000000001,"5min":5.6870000000000003,"15min":15.446999999999999},"last":2.5739999999999998},{"interface":"front","average":{"1min":1.492,"5min":1.9299999999999999,"15min":2.1739999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.9540000000000002,"5min":9.9480000000000004,"15min":27.818000000000001},"last":2.29}]},{"osd":3,"last update":"Mon Oct 9 11:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":1.7509999999999999,"15min":2.6469999999999998},"min":{"1min":0.46999999999999997,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":2.254,"5min":12.798999999999999,"15min":209.03700000000001},"last":5.1280000000000001},{"interface":"front","average":{"1min":0.878,"5min":1.7749999999999999,"15min":2.899},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29999999999999999},"max":{"1min":1.8480000000000001,"5min":8.6010000000000009,"15min":208.80199999999999},"last":8.2319999999999993}]},{"osd":5,"last update":"Mon Oct 9 11:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":1.2909999999999999,"15min":1.7669999999999999},"min":{"1min":0.307,"5min":0.245,"15min":0.23799999999999999},"max":{"1min":2.024,"5min":9.7880000000000003,"15min":24.149999999999999},"last":4.8810000000000002},{"interface":"front","average":{"1min":0.78800000000000003,"5min":1.2190000000000001,"15min":1.556},"min":{"1min":0.36899999999999999,"5min":0.25800000000000001,"15min":0.22600000000000001},"max":{"1min":3.1859999999999999,"5min":12.459,"15min":20.422000000000001},"last":3.2189999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:01:43 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.99399999999999999,"15min":1.357},"min":{"1min":0.34999999999999998,"5min":0.27800000000000002,"15min":0.24099999999999999},"max":{"1min":7.2309999999999999,"5min":8.5739999999999998,"15min":15.739000000000001},"last":1.5669999999999999},{"interface":"front","average":{"1min":1.052,"5min":1.454,"15min":1.7270000000000001},"min":{"1min":0.34100000000000003,"5min":0.32800000000000001,"15min":0.311},"max":{"1min":4.5220000000000002,"5min":9.0489999999999995,"15min":14.272},"last":4.9500000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:01:50 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":1.7170000000000001,"15min":1.72},"min":{"1min":0.33200000000000002,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":4.9429999999999996,"5min":19.923999999999999,"15min":19.923999999999999},"last":4.9210000000000003},{"interface":"front","average":{"1min":1.677,"5min":1.5489999999999999,"15min":1.585},"min":{"1min":0.34599999999999997,"5min":0.28799999999999998,"15min":0.255},"max":{"1min":8.6509999999999998,"5min":19.859999999999999,"15min":19.859999999999999},"last":1.29}]}]},{"osd":5,"up_from":37,"seq":158913790295,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3053232,"kb_used_data":2142312,"kb_used_omap":0,"kb_used_meta":910912,"kb_avail":90691920,"statfs":{"total":95995035648,"available":92868526080,"internally_reserved":0,"allocated":2193727488,"data_stored":2192349172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932773888},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:09 2023","interfaces":[{"interface":"back","average":{"1min":4.931,"5min":2.4180000000000001,"15min":2.1890000000000001},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.215},"max":{"1min":58.271999999999998,"5min":58.271999999999998,"15min":58.271999999999998},"last":0.29499999999999998},{"interface":"front","average":{"1min":5,"5min":2.4689999999999999,"15min":2.2429999999999999},"min":{"1min":0.32600000000000001,"5min":0.30499999999999999,"15min":0.28599999999999998},"max":{"1min":58.25,"5min":58.25,"15min":58.25},"last":0.377}]},{"osd":1,"last update":"Mon Oct 9 11:02:22 2023","interfaces":[{"interface":"back","average":{"1min":2.081,"5min":2.0070000000000001,"15min":1.9950000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":7.0419999999999998,"5min":9.4570000000000007,"15min":9.4570000000000007},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.966,"5min":1.6819999999999999,"15min":1.635},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.0599999999999996,"5min":9.4450000000000003,"15min":9.4450000000000003},"last":0.65800000000000003}]},{"osd":2,"last update":"Mon Oct 9 11:01:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.758,"15min":2.0379999999999998},"min":{"1min":0.25800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":6.851,"5min":9.3539999999999992,"15min":11.393000000000001},"last":1.1339999999999999},{"interface":"front","average":{"1min":2.04,"5min":1.8200000000000001,"15min":1.954},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":6.7130000000000001,"5min":14.621,"15min":15.087999999999999},"last":1.954}]},{"osd":3,"last update":"Mon Oct 9 11:02:09 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":2.5449999999999999,"15min":2.5209999999999999},"min":{"1min":0.29699999999999999,"5min":0.28000000000000003,"15min":0.25700000000000001},"max":{"1min":11.590999999999999,"5min":32.810000000000002,"15min":63.508000000000003},"last":5.016},{"interface":"front","average":{"1min":1.8069999999999999,"5min":2.5699999999999998,"15min":2.702},"min":{"1min":0.33900000000000002,"5min":0.30199999999999999,"15min":0.26300000000000001},"max":{"1min":11.292999999999999,"5min":32.659999999999997,"15min":62.418999999999997},"last":0.33400000000000002}]},{"osd":4,"last update":"Mon Oct 9 11:02:22 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.82599999999999996,"15min":1.4910000000000001},"min":{"1min":0.23799999999999999,"5min":0.216,"15min":0.216},"max":{"1min":7.8449999999999998,"5min":7.8449999999999998,"15min":123.241},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.298,"15min":1.8280000000000001},"min":{"1min":0.26400000000000001,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":3.0910000000000002,"5min":9.1869999999999994,"15min":123.318},"last":0.36499999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:02:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":1.212,"15min":1.52},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.1589999999999998,"5min":8.4260000000000002,"15min":11.098000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.004,"15min":1.3520000000000001},"min":{"1min":0.34999999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.0800000000000001,"5min":5.899,"15min":20.256},"last":0.59799999999999998}]},{"osd":7,"last update":"Mon Oct 9 11:01:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.323,"15min":1.534},"min":{"1min":0.35599999999999998,"5min":0.29299999999999998,"15min":0.28699999999999998},"max":{"1min":3.4870000000000001,"5min":7.6680000000000001,"15min":10.739000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.59199999999999997,"5min":1.1759999999999999,"15min":1.2989999999999999},"min":{"1min":0.379,"5min":0.315,"15min":0.30099999999999999},"max":{"1min":1.7,"5min":8.1430000000000007,"15min":14.154},"last":0.51300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3735879680,"data_stored":3735879680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1488257024,"data_stored":1488257024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2195456000,"data_stored":2195456000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2192769024,"data_stored":2192769024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3289120768,"data_stored":3289120768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2196439040,"data_stored":2196439040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2559770624,"data_stored":2559770624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":47448064,"data_stored":47448064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:02:42.741 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T11:02:42.741 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pg_num 26 2023-10-09T11:02:42.790 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:42.821 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:42.885 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:42.917 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 84 16 94641 94625 70.3741 49.0625 0.0308427 0.0142037 2023-10-09T11:02:42.922 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:42.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:42.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:42.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:42 smithi125 ceph-mon[195507]: from='client.46214 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:02:42.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:42 smithi125 ceph-mon[195507]: pgmap v1083: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 5503/276960 objects degraded (1.987%); 16682/276960 objects misplaced (6.023%); 1.8 MiB/s, 1 keys/s, 28 objects/s recovering 2023-10-09T11:02:42.937 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:42 smithi125 ceph-mon[200197]: from='client.46214 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:02:42.937 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:42 smithi125 ceph-mon[200197]: pgmap v1083: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 5503/276960 objects degraded (1.987%); 16682/276960 objects misplaced (6.023%); 1.8 MiB/s, 1 keys/s, 28 objects/s recovering 2023-10-09T11:02:43.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:43.089 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:43.136 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:43.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:43.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:43.216 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:02:43.216 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:02:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:42 smithi155 ceph-mon[184854]: from='client.46214 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:02:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:42 smithi155 ceph-mon[184854]: pgmap v1083: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 5503/276960 objects degraded (1.987%); 16682/276960 objects misplaced (6.023%); 1.8 MiB/s, 1 keys/s, 28 objects/s recovering 2023-10-09T11:02:43.243 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:43.299 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:43.405 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:43.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:43.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:43.510 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:43.567 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:43.632 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:43.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:43.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:43.739 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:43.808 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T11:02:43.815 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:43.858 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:43.862 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:43.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:43.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:43.918 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 85 16 95375 95359 70.085 45.875 0.0187865 0.0142622 2023-10-09T11:02:43.968 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:44.007 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:44.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:44.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:44.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:44.145 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:44.216 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:44.278 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:02:44.300 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:44.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:44.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:44.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:44 smithi125 ceph-mon[195507]: pgmap v1084: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s; 5503/282141 objects degraded (1.950%); 17161/282141 objects misplaced (6.082%); 1.6 MiB/s, 1 keys/s, 25 objects/s recovering 2023-10-09T11:02:44.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:44 smithi125 ceph-mon[200197]: pgmap v1084: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s; 5503/282141 objects degraded (1.950%); 17161/282141 objects misplaced (6.082%); 1.6 MiB/s, 1 keys/s, 25 objects/s recovering 2023-10-09T11:02:44.391 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:44.404 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:44.433 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:44 smithi155 ceph-mon[184854]: pgmap v1084: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s; 5503/282141 objects degraded (1.950%); 17161/282141 objects misplaced (6.082%); 1.6 MiB/s, 1 keys/s, 25 objects/s recovering 2023-10-09T11:02:44.435 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:44.458 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:44.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:44.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:44.563 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:44.639 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:44.918 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 86 16 96037 96021 69.7513 41.375 0.0188882 0.0143325 2023-10-09T11:02:45.245 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:02:45.527 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:45 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/522247863' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:45.527 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:45 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:45.527 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:45.527 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:02:45.528 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:45 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/522247863' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:45.528 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:45 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:45.528 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:45.528 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:02:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:45 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/522247863' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:45 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:02:45.744 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:45.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:45.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:45.850 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:45.916 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:45.918 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 87 16 96821 96805 69.5128 49 0.045823 0.0143801 2023-10-09T11:02:45.958 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:45.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:45.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:46.065 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:46.120 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:46.174 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:46.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:46.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:46.223 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:46.282 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:46.339 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:46.365 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:46 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:46.365 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:46 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:02:46.365 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:46 smithi125 ceph-mon[195507]: osdmap e439: 8 total, 8 up, 8 in 2023-10-09T11:02:46.365 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:46 smithi125 ceph-mon[195507]: failed to encode map e439 with expected crc 2023-10-09T11:02:46.366 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:46 smithi125 ceph-mon[195507]: failed to encode map e439 with expected crc 2023-10-09T11:02:46.366 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:46 smithi125 ceph-mon[195507]: pgmap v1086: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 693 op/s; 5503/282141 objects degraded (1.950%); 17161/282141 objects misplaced (6.082%); 1.4 MiB/s, 1 keys/s, 22 objects/s recovering 2023-10-09T11:02:46.366 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:46.366 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:46 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:46.366 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:46 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:02:46.366 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:46 smithi125 ceph-mon[200197]: osdmap e439: 8 total, 8 up, 8 in 2023-10-09T11:02:46.366 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:46 smithi125 ceph-mon[200197]: failed to encode map e439 with expected crc 2023-10-09T11:02:46.366 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:46 smithi125 ceph-mon[200197]: failed to encode map e439 with expected crc 2023-10-09T11:02:46.367 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:46 smithi125 ceph-mon[200197]: pgmap v1086: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 693 op/s; 5503/282141 objects degraded (1.950%); 17161/282141 objects misplaced (6.082%); 1.4 MiB/s, 1 keys/s, 22 objects/s recovering 2023-10-09T11:02:46.367 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:46.368 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:46.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:46.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:46.473 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:46.527 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:46.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:46.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:46.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:46.713 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:46 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:46 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:02:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:46 smithi155 ceph-mon[184854]: osdmap e439: 8 total, 8 up, 8 in 2023-10-09T11:02:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:46 smithi155 ceph-mon[184854]: failed to encode map e439 with expected crc 2023-10-09T11:02:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:46 smithi155 ceph-mon[184854]: failed to encode map e439 with expected crc 2023-10-09T11:02:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:46 smithi155 ceph-mon[184854]: pgmap v1086: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 693 op/s; 5503/282141 objects degraded (1.950%); 17161/282141 objects misplaced (6.082%); 1.4 MiB/s, 1 keys/s, 22 objects/s recovering 2023-10-09T11:02:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:46.753 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:46.790 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:46.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:46.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:46.895 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:46.918 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 88 15 97541 97526 69.235 45.0625 0.0168589 0.0144389 2023-10-09T11:02:46.986 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:47.143 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:02:47.143 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:02:47.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3010201162' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T11:02:47.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:47 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T11:02:47.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:47 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:02:47.309 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 10 pg_num to 26 2023-10-09T11:02:47.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:47.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:47.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:47.467 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:47.523 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:47.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3010201162' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T11:02:47.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:47 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T11:02:47.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:47 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:02:47.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:47 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3010201162' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T11:02:47.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:47 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T11:02:47.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:47 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:02:47.919 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 89 16 98251 98235 68.955 44.3125 0.0190546 0.0144975 2023-10-09T11:02:48.136 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:48.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:48.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:48.192 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T11:02:48.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:02:48.240 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:48.281 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:48.362 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:48.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:48.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:48.466 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:48.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:48 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T11:02:48.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:48 smithi155 ceph-mon[184854]: osdmap e440: 8 total, 8 up, 8 in 2023-10-09T11:02:48.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:48 smithi155 ceph-mon[184854]: failed to encode map e440 with expected crc 2023-10-09T11:02:48.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:48 smithi155 ceph-mon[184854]: failed to encode map e440 with expected crc 2023-10-09T11:02:48.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:48 smithi155 ceph-mon[184854]: pgmap v1088: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 5.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s; 5503/288471 objects degraded (1.908%); 17010/288471 objects misplaced (5.897%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T11:02:48.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:48 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T11:02:48.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:48 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 5503/282141 objects degraded (1.950%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:02:48.522 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:48 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T11:02:48.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:48 smithi125 ceph-mon[195507]: osdmap e440: 8 total, 8 up, 8 in 2023-10-09T11:02:48.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:48 smithi125 ceph-mon[195507]: failed to encode map e440 with expected crc 2023-10-09T11:02:48.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:48 smithi125 ceph-mon[195507]: failed to encode map e440 with expected crc 2023-10-09T11:02:48.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:48 smithi125 ceph-mon[195507]: pgmap v1088: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 5.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s; 5503/288471 objects degraded (1.908%); 17010/288471 objects misplaced (5.897%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T11:02:48.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:48 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T11:02:48.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:48 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 5503/282141 objects degraded (1.950%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:02:48.523 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:48 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T11:02:48.523 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:48 smithi125 ceph-mon[200197]: osdmap e440: 8 total, 8 up, 8 in 2023-10-09T11:02:48.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:48 smithi125 ceph-mon[200197]: failed to encode map e440 with expected crc 2023-10-09T11:02:48.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:48 smithi125 ceph-mon[200197]: failed to encode map e440 with expected crc 2023-10-09T11:02:48.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:48 smithi125 ceph-mon[200197]: pgmap v1088: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 5.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s; 5503/288471 objects degraded (1.908%); 17010/288471 objects misplaced (5.897%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T11:02:48.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:48 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T11:02:48.524 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:48 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 5503/282141 objects degraded (1.950%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:02:48.669 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:48.919 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 90 16 98566 98550 68.4078 19.6875 0.248319 0.0145425 2023-10-09T11:02:48.951 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:49.013 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:02:49.034 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:02:49.055 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:02:49.077 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:02:49.103 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:02:49.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:49.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:49.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:49.127 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:02:49.146 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:02:49.160 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:49.170 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:02:49.172 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:02:49.172 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:02:49.172 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:02:49.173 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:02:49.173 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:02:49.173 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:02:49.173 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:02:49.173 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:02:49.218 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:49.246 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:49.266 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:49.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:49.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:49.370 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:49.399 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:49.421 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:49.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:49.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:49.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:49 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T11:02:49.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:49 smithi125 ceph-mon[195507]: osdmap e441: 8 total, 8 up, 8 in 2023-10-09T11:02:49.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:49 smithi125 ceph-mon[195507]: failed to encode map e441 with expected crc 2023-10-09T11:02:49.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:49 smithi125 ceph-mon[195507]: failed to encode map e441 with expected crc 2023-10-09T11:02:49.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:49 smithi125 ceph-mon[195507]: failed to encode map e441 with expected crc 2023-10-09T11:02:49.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:49 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2909340420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:49.474 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:49 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T11:02:49.474 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:49 smithi125 ceph-mon[200197]: osdmap e441: 8 total, 8 up, 8 in 2023-10-09T11:02:49.474 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:49 smithi125 ceph-mon[200197]: failed to encode map e441 with expected crc 2023-10-09T11:02:49.474 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:49 smithi125 ceph-mon[200197]: failed to encode map e441 with expected crc 2023-10-09T11:02:49.474 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:49 smithi125 ceph-mon[200197]: failed to encode map e441 with expected crc 2023-10-09T11:02:49.474 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:49 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2909340420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:49.525 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:49.551 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:49 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T11:02:49.551 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:49 smithi155 ceph-mon[184854]: osdmap e441: 8 total, 8 up, 8 in 2023-10-09T11:02:49.551 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:49 smithi155 ceph-mon[184854]: failed to encode map e441 with expected crc 2023-10-09T11:02:49.551 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:49 smithi155 ceph-mon[184854]: failed to encode map e441 with expected crc 2023-10-09T11:02:49.551 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:49 smithi155 ceph-mon[184854]: failed to encode map e441 with expected crc 2023-10-09T11:02:49.551 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:49 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2909340420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:49.553 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:49.623 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:49.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:49.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:49.728 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:49.757 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:49.781 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:49.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:49.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:49.885 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:49.916 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:49.919 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 91 3 98566 98563 67.6652 0.8125 0.812968 0.0146591 2023-10-09T11:02:50.036 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:50.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:50.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:50.140 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:50.169 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:50.190 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:50.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:50.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:50.296 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:50.326 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:50 smithi155 ceph-mon[184854]: failed to encode map e442 with expected crc 2023-10-09T11:02:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:50 smithi155 ceph-mon[184854]: failed to encode map e442 with expected crc 2023-10-09T11:02:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:50 smithi155 ceph-mon[184854]: osdmap e442: 8 total, 8 up, 8 in 2023-10-09T11:02:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:50 smithi155 ceph-mon[184854]: failed to encode map e442 with expected crc 2023-10-09T11:02:50.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:50 smithi155 ceph-mon[184854]: failed to encode map e442 with expected crc 2023-10-09T11:02:50.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:50 smithi155 ceph-mon[184854]: 10.1 continuing backfill to osd.7 from (0'0,440'6234] MIN to 440'6234 2023-10-09T11:02:50.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:50 smithi155 ceph-mon[184854]: pgmap v1091: 129 pgs: 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s; 5503/293088 objects degraded (1.878%); 17427/293088 objects misplaced (5.946%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T11:02:50.587 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:50 smithi125 ceph-mon[195507]: failed to encode map e442 with expected crc 2023-10-09T11:02:50.725 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:50 smithi125 ceph-mon[195507]: failed to encode map e442 with expected crc 2023-10-09T11:02:50.725 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:50 smithi125 ceph-mon[195507]: osdmap e442: 8 total, 8 up, 8 in 2023-10-09T11:02:50.725 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:50 smithi125 ceph-mon[195507]: failed to encode map e442 with expected crc 2023-10-09T11:02:50.725 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:50 smithi125 ceph-mon[195507]: failed to encode map e442 with expected crc 2023-10-09T11:02:50.725 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:50 smithi125 ceph-mon[195507]: 10.1 continuing backfill to osd.7 from (0'0,440'6234] MIN to 440'6234 2023-10-09T11:02:50.726 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:50 smithi125 ceph-mon[195507]: pgmap v1091: 129 pgs: 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s; 5503/293088 objects degraded (1.878%); 17427/293088 objects misplaced (5.946%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T11:02:50.726 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:50 smithi125 ceph-mon[200197]: failed to encode map e442 with expected crc 2023-10-09T11:02:50.726 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:50 smithi125 ceph-mon[200197]: failed to encode map e442 with expected crc 2023-10-09T11:02:50.726 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:50 smithi125 ceph-mon[200197]: osdmap e442: 8 total, 8 up, 8 in 2023-10-09T11:02:50.727 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:50 smithi125 ceph-mon[200197]: failed to encode map e442 with expected crc 2023-10-09T11:02:50.727 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:50 smithi125 ceph-mon[200197]: failed to encode map e442 with expected crc 2023-10-09T11:02:50.727 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:50 smithi125 ceph-mon[200197]: 10.1 continuing backfill to osd.7 from (0'0,440'6234] MIN to 440'6234 2023-10-09T11:02:50.727 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:50 smithi125 ceph-mon[200197]: pgmap v1091: 129 pgs: 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s; 5503/293088 objects degraded (1.878%); 17427/293088 objects misplaced (5.946%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T11:02:50.805 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:50.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:50.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:50.831 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:02:50.831 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:02:50.831 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.832 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:98555 10.11 10.b1b5b8d1 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e443)", 2023-10-09T11:02:50.832 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:50.335541+0000", 2023-10-09T11:02:50.832 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.49628014500000001, 2023-10-09T11:02:50.832 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.49629906800000001, 2023-10-09T11:02:50.832 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:02:50.832 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "delayed", 2023-10-09T11:02:50.832 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:02:50.832 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:02:50.832 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:02:50.832 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 98555 2023-10-09T11:02:50.832 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.833 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:02:50.833 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.833 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:02:50.833 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335541+0000", 2023-10-09T11:02:50.833 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:02:50.833 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.833 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.833 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:02:50.834 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335541+0000", 2023-10-09T11:02:50.834 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:02:50.834 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.834 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.834 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:02:50.834 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335540+0000", 2023-10-09T11:02:50.835 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:02:50.835 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.835 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.835 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:02:50.835 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335565+0000", 2023-10-09T11:02:50.835 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4861000000000001e-05 2023-10-09T11:02:50.835 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.836 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:02:50.836 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335578+0000", 2023-10-09T11:02:50.836 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3101e-05 2023-10-09T11:02:50.836 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.837 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:02:50.837 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335588+0000", 2023-10-09T11:02:50.837 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0098000000000001e-05 2023-10-09T11:02:50.837 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.837 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.837 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:02:50.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336134+0000", 2023-10-09T11:02:50.838 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00054624999999999997 2023-10-09T11:02:50.838 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.838 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.838 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:02:50.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336146+0000", 2023-10-09T11:02:50.839 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1460999999999999e-05 2023-10-09T11:02:50.839 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:02:50.839 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:02:50.839 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:02:50.839 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.840 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.840 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:98558 10.11 10.32c67d91 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e443)", 2023-10-09T11:02:50.840 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:50.336104+0000", 2023-10-09T11:02:50.840 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.49571763200000002, 2023-10-09T11:02:50.840 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.49577962599999997, 2023-10-09T11:02:50.840 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:02:50.841 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "delayed", 2023-10-09T11:02:50.841 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:02:50.841 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:02:50.841 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:02:50.841 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 98558 2023-10-09T11:02:50.841 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.841 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:02:50.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.842 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:02:50.842 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336104+0000", 2023-10-09T11:02:50.842 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:02:50.842 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.843 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:02:50.843 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336104+0000", 2023-10-09T11:02:50.843 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:02:50.843 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.843 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.843 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:02:50.844 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336103+0000", 2023-10-09T11:02:50.844 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:02:50.844 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.844 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.844 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:02:50.844 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336129+0000", 2023-10-09T11:02:50.844 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5403000000000001e-05 2023-10-09T11:02:50.844 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.845 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.845 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:02:50.845 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336140+0000", 2023-10-09T11:02:50.845 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1060000000000001e-05 2023-10-09T11:02:50.845 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.845 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.846 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:02:50.846 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336149+0000", 2023-10-09T11:02:50.846 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8810000000000002e-06 2023-10-09T11:02:50.846 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.846 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.846 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:02:50.847 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336186+0000", 2023-10-09T11:02:50.847 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6980999999999997e-05 2023-10-09T11:02:50.847 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:02:50.847 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:02:50.847 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:02:50.847 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336192+0000", 2023-10-09T11:02:50.847 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7560000000000002e-06 2023-10-09T11:02:50.847 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:02:50.848 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:02:50.848 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:02:50.848 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:02:50.848 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T11:02:50.848 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 2 2023-10-09T11:02:50.848 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:02:50.910 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:50.919 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 92 2 98566 98564 66.9306 0.0625 1.98726 0.0146791 2023-10-09T11:02:50.941 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:50.962 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:50.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:50.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:51.066 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:51.097 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:51.371 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T11:02:51.373 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:51.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:51.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:51.399 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[195507]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[195507]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[195507]: osdmap e443: 8 total, 8 up, 8 in 2023-10-09T11:02:51.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[195507]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[195507]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[195507]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:51.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:51.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[195507]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.400 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:02:51] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-10-09T11:02:51.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[200197]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[200197]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[200197]: osdmap e443: 8 total, 8 up, 8 in 2023-10-09T11:02:51.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[200197]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[200197]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[200197]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:51.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:51.401 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:51 smithi125 ceph-mon[200197]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.476 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:51.504 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:51.525 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:51.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:51.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:51.630 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:51.659 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:51.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:51 smithi155 ceph-mon[184854]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:51 smithi155 ceph-mon[184854]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:51 smithi155 ceph-mon[184854]: osdmap e443: 8 total, 8 up, 8 in 2023-10-09T11:02:51.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:51 smithi155 ceph-mon[184854]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:51 smithi155 ceph-mon[184854]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:51 smithi155 ceph-mon[184854]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:51 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:51.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:51 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:51.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:51 smithi155 ceph-mon[184854]: failed to encode map e443 with expected crc 2023-10-09T11:02:51.747 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:02:51.779 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:51.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:51.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:51.884 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:51.912 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:51.919 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total time run: 92.4658 2023-10-09T11:02:51.919 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total writes made: 98566 2023-10-09T11:02:51.919 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Write size: 65536 2023-10-09T11:02:51.920 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object size: 65536 2023-10-09T11:02:51.920 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Bandwidth (MB/sec): 66.6233 2023-10-09T11:02:51.920 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Bandwidth: 36.5649 2023-10-09T11:02:51.920 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max bandwidth (MB/sec): 198.688 2023-10-09T11:02:51.920 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min bandwidth (MB/sec): 0 2023-10-09T11:02:51.920 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average IOPS: 1065 2023-10-09T11:02:51.920 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev IOPS: 585.047 2023-10-09T11:02:51.921 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max IOPS: 3179 2023-10-09T11:02:51.921 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min IOPS: 0 2023-10-09T11:02:51.921 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average Latency(s): 0.0147384 2023-10-09T11:02:51.921 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Latency(s): 0.0480887 2023-10-09T11:02:51.921 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max latency(s): 2.958 2023-10-09T11:02:51.921 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min latency(s): 0.0011398 2023-10-09T11:02:51.924 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T11:02:51.942 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:51.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:51.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:52.045 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:52.049 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:52.123 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:52.258 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:52.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:52.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:52.362 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:52.386 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:02:52.421 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[200197]: failed to encode map e444 with expected crc 2023-10-09T11:02:52.421 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[200197]: failed to encode map e444 with expected crc 2023-10-09T11:02:52.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[200197]: osdmap e444: 8 total, 8 up, 8 in 2023-10-09T11:02:52.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[200197]: failed to encode map e444 with expected crc 2023-10-09T11:02:52.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[200197]: 10.11 continuing backfill to osd.7 from (0'0,440'6234] MIN to 440'6234 2023-10-09T11:02:52.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[200197]: failed to encode map e444 with expected crc 2023-10-09T11:02:52.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1433110855' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:02:52.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:02:52.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[200197]: failed to encode map e444 with expected crc 2023-10-09T11:02:52.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[200197]: pgmap v1094: 129 pgs: 1 active+recovering+undersized+remapped, 4 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 120 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 8049/295455 objects degraded (2.724%); 14517/295455 objects misplaced (4.913%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T11:02:52.425 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[195507]: failed to encode map e444 with expected crc 2023-10-09T11:02:52.425 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[195507]: failed to encode map e444 with expected crc 2023-10-09T11:02:52.425 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[195507]: osdmap e444: 8 total, 8 up, 8 in 2023-10-09T11:02:52.425 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[195507]: failed to encode map e444 with expected crc 2023-10-09T11:02:52.425 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[195507]: 10.11 continuing backfill to osd.7 from (0'0,440'6234] MIN to 440'6234 2023-10-09T11:02:52.425 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[195507]: failed to encode map e444 with expected crc 2023-10-09T11:02:52.426 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1433110855' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:02:52.426 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:02:52.426 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[195507]: failed to encode map e444 with expected crc 2023-10-09T11:02:52.426 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:52 smithi125 ceph-mon[195507]: pgmap v1094: 129 pgs: 1 active+recovering+undersized+remapped, 4 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 120 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 8049/295455 objects degraded (2.724%); 14517/295455 objects misplaced (4.913%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T11:02:52.426 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:52.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:52 smithi155 ceph-mon[184854]: failed to encode map e444 with expected crc 2023-10-09T11:02:52.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:52 smithi155 ceph-mon[184854]: failed to encode map e444 with expected crc 2023-10-09T11:02:52.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:52 smithi155 ceph-mon[184854]: osdmap e444: 8 total, 8 up, 8 in 2023-10-09T11:02:52.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:52 smithi155 ceph-mon[184854]: failed to encode map e444 with expected crc 2023-10-09T11:02:52.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:52 smithi155 ceph-mon[184854]: 10.11 continuing backfill to osd.7 from (0'0,440'6234] MIN to 440'6234 2023-10-09T11:02:52.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:52 smithi155 ceph-mon[184854]: failed to encode map e444 with expected crc 2023-10-09T11:02:52.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:52 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1433110855' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:02:52.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:52 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:02:52.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:52 smithi155 ceph-mon[184854]: failed to encode map e444 with expected crc 2023-10-09T11:02:52.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:52 smithi155 ceph-mon[184854]: pgmap v1094: 129 pgs: 1 active+recovering+undersized+remapped, 4 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 120 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 8049/295455 objects degraded (2.724%); 14517/295455 objects misplaced (4.913%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T11:02:52.862 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:52.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:52.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:52.967 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:53.032 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:53.067 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:53.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:53.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:53.116 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:53.172 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:53.229 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:53.256 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:53.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:53.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:53.363 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:53.411 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:53.449 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:53.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:53.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:53.561 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:53.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:53 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:53.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:53 smithi125 ceph-mon[195507]: failed to encode map e445 with expected crc 2023-10-09T11:02:53.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:53 smithi125 ceph-mon[195507]: failed to encode map e445 with expected crc 2023-10-09T11:02:53.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:53 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:02:53.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:53 smithi125 ceph-mon[195507]: osdmap e445: 8 total, 8 up, 8 in 2023-10-09T11:02:53.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:53 smithi125 ceph-mon[195507]: 10.0 scrub starts 2023-10-09T11:02:53.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:53 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 8049/295455 objects degraded (2.724%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T11:02:53.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:53 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:53.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:53 smithi125 ceph-mon[200197]: failed to encode map e445 with expected crc 2023-10-09T11:02:53.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:53 smithi125 ceph-mon[200197]: failed to encode map e445 with expected crc 2023-10-09T11:02:53.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:53 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:02:53.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:53 smithi125 ceph-mon[200197]: osdmap e445: 8 total, 8 up, 8 in 2023-10-09T11:02:53.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:53 smithi125 ceph-mon[200197]: 10.0 scrub starts 2023-10-09T11:02:53.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:53 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 8049/295455 objects degraded (2.724%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T11:02:53.630 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:53 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:02:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:53 smithi155 ceph-mon[184854]: failed to encode map e445 with expected crc 2023-10-09T11:02:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:53 smithi155 ceph-mon[184854]: failed to encode map e445 with expected crc 2023-10-09T11:02:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:53 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:02:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:53 smithi155 ceph-mon[184854]: osdmap e445: 8 total, 8 up, 8 in 2023-10-09T11:02:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:53 smithi155 ceph-mon[184854]: 10.0 scrub starts 2023-10-09T11:02:53.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:53 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 8049/295455 objects degraded (2.724%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T11:02:53.845 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 0, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T11:02:53.846 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:02:53.846 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-10-09T11:02:53.846 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:02:53.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:53.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:53.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:53.914 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:02:53.915 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:02:53.915 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T11:02:53.915 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T11:02:53.915 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:02:53.980 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:54.037 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:54.121 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:54.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:54.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:54.230 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:54.279 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:54.319 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:54.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:54.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:54.432 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:54.498 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:54.530 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:54.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:54.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:54 smithi125 ceph-mon[195507]: pgmap v1096: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 233 B/s rd, 13 MiB/s wr, 314 op/s; 7976/294324 objects degraded (2.710%); 14294/294324 objects misplaced (4.857%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T11:02:54.540 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:54 smithi125 ceph-mon[195507]: 10.0 scrub ok 2023-10-09T11:02:54.540 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:54 smithi125 ceph-mon[195507]: 10.10 scrub starts 2023-10-09T11:02:54.540 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:54 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:54.540 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:54 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:54.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:54 smithi125 ceph-mon[200197]: pgmap v1096: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 233 B/s rd, 13 MiB/s wr, 314 op/s; 7976/294324 objects degraded (2.710%); 14294/294324 objects misplaced (4.857%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T11:02:54.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:54 smithi125 ceph-mon[200197]: 10.0 scrub ok 2023-10-09T11:02:54.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:54 smithi125 ceph-mon[200197]: 10.10 scrub starts 2023-10-09T11:02:54.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:54 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:54.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:54 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:54.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:54.642 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:54.678 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:54.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:54 smithi155 ceph-mon[184854]: pgmap v1096: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 233 B/s rd, 13 MiB/s wr, 314 op/s; 7976/294324 objects degraded (2.710%); 14294/294324 objects misplaced (4.857%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T11:02:54.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:54 smithi155 ceph-mon[184854]: 10.0 scrub ok 2023-10-09T11:02:54.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:54 smithi155 ceph-mon[184854]: 10.10 scrub starts 2023-10-09T11:02:54.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:54 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:54.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:54 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:54.717 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:54.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:54.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:54.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:02:54.841 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:54.842 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T11:02:54.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:02:54.932 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:55.036 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:02:55.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:55.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:02:55.141 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:55.175 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:55.197 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:55.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:55.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:55.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:55.359 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:55.376 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:55 smithi125 ceph-mon[195507]: 10.7 scrub starts 2023-10-09T11:02:55.376 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:55 smithi125 ceph-mon[195507]: 10.10 scrub ok 2023-10-09T11:02:55.376 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:55 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:02:55.376 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:55 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:02:55.376 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:55 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:55.391 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:55.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:55.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:55.399 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:02:55.441 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:02:55.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:55 smithi155 ceph-mon[184854]: 10.7 scrub starts 2023-10-09T11:02:55.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:55 smithi155 ceph-mon[184854]: 10.10 scrub ok 2023-10-09T11:02:55.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:55 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:02:55.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:55 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:02:55.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:55 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:55.493 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:02:55.498 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:55.535 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:02:55.547 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:55.568 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:02:55.587 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:55.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:55.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:55.605 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:02:55.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:55 smithi125 ceph-mon[200197]: 10.7 scrub starts 2023-10-09T11:02:55.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:55 smithi125 ceph-mon[200197]: 10.10 scrub ok 2023-10-09T11:02:55.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:55 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:02:55.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:55 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:02:55.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:55 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:02:55.646 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:02:55.693 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:02:55.696 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:55.705 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:02:55.705 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:02:55.705 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:02:55.705 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:02:55.706 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:02:55.706 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:02:55.706 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:02:55.706 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:02:55.715 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:55.733 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:55.759 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:55.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:55.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:55.864 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:55.906 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:55.938 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:55.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:55.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:56.045 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:02:56.103 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:56.148 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:02:56.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:56.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:02:56.252 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:56.298 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:56.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:56.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:56.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:56.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:56.471 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:56.479 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:56.500 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:56.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:56.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:56.534 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2784916616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:56.534 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:56 smithi125 ceph-mon[195507]: 10.2 scrub starts 2023-10-09T11:02:56.535 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:56 smithi125 ceph-mon[195507]: pgmap v1097: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 170 B/s rd, 9.3 MiB/s wr, 229 op/s; 7976/294324 objects degraded (2.710%); 14294/294324 objects misplaced (4.857%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T11:02:56.535 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2784916616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:56.535 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:56 smithi125 ceph-mon[200197]: 10.2 scrub starts 2023-10-09T11:02:56.535 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:56 smithi125 ceph-mon[200197]: pgmap v1097: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 170 B/s rd, 9.3 MiB/s wr, 229 op/s; 7976/294324 objects degraded (2.710%); 14294/294324 objects misplaced (4.857%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T11:02:56.608 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:02:56.653 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2784916616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:02:56.653 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:56 smithi155 ceph-mon[184854]: 10.2 scrub starts 2023-10-09T11:02:56.653 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:56 smithi155 ceph-mon[184854]: pgmap v1097: 129 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 170 B/s rd, 9.3 MiB/s wr, 229 op/s; 7976/294324 objects degraded (2.710%); 14294/294324 objects misplaced (4.857%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T11:02:56.654 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:56.704 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:02:56.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:56.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:02:56.811 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:56.884 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:56.914 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:56.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:56.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:56.942 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:02:56.967 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:02:57.019 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:02:57.062 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:57.102 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:02:57.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:57.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:02:57.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:57.271 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:57.313 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:57.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:57.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:57.418 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:57.478 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:57.550 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:57 smithi125 ceph-mon[195507]: 10.17 scrub starts 2023-10-09T11:02:57.550 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:57 smithi125 ceph-mon[195507]: from='client.46250 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:02:57.550 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:57 smithi125 ceph-mon[200197]: 10.17 scrub starts 2023-10-09T11:02:57.550 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:57 smithi125 ceph-mon[200197]: from='client.46250 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:02:57.564 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:57.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:57.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:57.672 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:57.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:57 smithi155 ceph-mon[184854]: 10.17 scrub starts 2023-10-09T11:02:57.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:57 smithi155 ceph-mon[184854]: from='client.46250 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:02:57.727 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:57.765 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:57.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:57.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:57.776 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1097,"stamp":"2023-10-09T11:02:55.701100+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6428572329,"num_objects":98108,"num_object_clones":0,"num_object_copies":294324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7976,"num_objects_misplaced":14294,"num_objects_unfound":0,"num_objects_dirty":98108,"num_whiteouts":0,"num_read":9,"num_read_kb":15,"num_write":99115,"num_write_kb":6308291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5511,"num_bytes_recovered":358166493,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":163255,"ondisk_log_size":163255,"up":387,"acting":385,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":388,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":25297572,"kb_used_data":19201956,"kb_used_omap":130,"kb_used_meta":6095357,"kb_avail":724663644,"statfs":{"total":767960285184,"available":742055571456,"internally_reserved":0,"allocated":19662802944,"data_stored":19647978972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":133799,"internal_metadata":6241645913},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":21,"apply_latency_ms":21,"commit_latency_ns":21000000,"apply_latency_ns":21000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":26935320,"num_objects":412,"num_object_clones":0,"num_object_copies":1236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2473,"num_objects_misplaced":-3133,"num_objects_unfound":0,"num_objects_dirty":412,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1378,"num_write_kb":57217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":543,"num_bytes_recovered":35586048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002746"},"pg_stats":[{"pgid":"10.13","version":"440'6119","reported_seq":6178,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.334595+0000","last_change":"2023-10-09T11:02:49.374584+0000","last_active":"2023-10-09T11:02:50.334595+0000","last_peered":"2023-10-09T11:02:50.334595+0000","last_clean":"2023-10-09T11:02:50.334595+0000","last_became_active":"2023-10-09T11:02:49.372737+0000","last_became_peered":"2023-10-09T11:02:49.372737+0000","last_unstale":"2023-10-09T11:02:50.334595+0000","last_undegraded":"2023-10-09T11:02:50.334595+0000","last_fullsized":"2023-10-09T11:02:50.334595+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6119,"log_dups_size":0,"ondisk_log_size":6119,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200507392,"num_objects":3060,"num_object_clones":0,"num_object_copies":9180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3060,"num_write_kb":195808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"76'161","reported_seq":683,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332037+0000","last_change":"2023-10-09T11:02:36.217074+0000","last_active":"2023-10-09T11:02:50.332037+0000","last_peered":"2023-10-09T11:02:50.332037+0000","last_clean":"2023-10-09T11:02:50.332037+0000","last_became_active":"2023-10-09T11:02:36.216494+0000","last_became_peered":"2023-10-09T11:02:36.216494+0000","last_unstale":"2023-10-09T11:02:50.332037+0000","last_undegraded":"2023-10-09T11:02:50.332037+0000","last_fullsized":"2023-10-09T11:02:50.332037+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:56.760915+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":166,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-10-09T11:02:49.317221+0000","last_change":"2023-10-09T11:00:14.910332+0000","last_active":"2023-10-09T11:02:49.317221+0000","last_peered":"2023-10-09T11:02:49.317221+0000","last_clean":"2023-10-09T11:02:49.317221+0000","last_became_active":"2023-10-09T11:00:14.910108+0000","last_became_peered":"2023-10-09T11:00:14.910108+0000","last_unstale":"2023-10-09T11:02:49.317221+0000","last_undegraded":"2023-10-09T11:02:49.317221+0000","last_fullsized":"2023-10-09T11:02:49.317221+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:30:41.668874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":467,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.346634+0000","last_change":"2023-10-09T11:00:14.870884+0000","last_active":"2023-10-09T11:02:48.346634+0000","last_peered":"2023-10-09T11:02:48.346634+0000","last_clean":"2023-10-09T11:02:48.346634+0000","last_became_active":"2023-10-09T11:00:14.870515+0000","last_became_peered":"2023-10-09T11:00:14.870515+0000","last_unstale":"2023-10-09T11:02:48.346634+0000","last_undegraded":"2023-10-09T11:02:48.346634+0000","last_fullsized":"2023-10-09T11:02:48.346634+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:02.499841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"442'6058","reported_seq":6117,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.990206+0000","last_change":"2023-10-09T11:02:49.358712+0000","last_active":"2023-10-09T11:02:50.990206+0000","last_peered":"2023-10-09T11:02:50.990206+0000","last_clean":"2023-10-09T11:02:50.990206+0000","last_became_active":"2023-10-09T11:02:49.354248+0000","last_became_peered":"2023-10-09T11:02:49.354248+0000","last_unstale":"2023-10-09T11:02:50.990206+0000","last_undegraded":"2023-10-09T11:02:50.990206+0000","last_fullsized":"2023-10-09T11:02:50.990206+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6058,"log_dups_size":0,"ondisk_log_size":6058,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198541312,"num_objects":3030,"num_object_clones":0,"num_object_copies":9090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3030,"num_write_kb":193888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":601,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332284+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:02:50.332284+0000","last_peered":"2023-10-09T11:02:50.332284+0000","last_clean":"2023-10-09T11:02:50.332284+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:02:50.332284+0000","last_undegraded":"2023-10-09T11:02:50.332284+0000","last_fullsized":"2023-10-09T11:02:50.332284+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":124,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.335325+0000","last_change":"2023-10-09T11:00:14.876415+0000","last_active":"2023-10-09T11:02:51.335325+0000","last_peered":"2023-10-09T11:02:51.335325+0000","last_clean":"2023-10-09T11:02:51.335325+0000","last_became_active":"2023-10-09T11:00:14.864861+0000","last_became_peered":"2023-10-09T11:00:14.864861+0000","last_unstale":"2023-10-09T11:02:51.335325+0000","last_undegraded":"2023-10-09T11:02:51.335325+0000","last_fullsized":"2023-10-09T11:02:51.335325+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:59.494851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":387,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.332701+0000","last_change":"2023-10-09T11:02:33.198683+0000","last_active":"2023-10-09T11:02:51.332701+0000","last_peered":"2023-10-09T11:02:51.332701+0000","last_clean":"2023-10-09T11:02:51.332701+0000","last_became_active":"2023-10-09T11:02:33.197952+0000","last_became_peered":"2023-10-09T11:02:33.197952+0000","last_unstale":"2023-10-09T11:02:51.332701+0000","last_undegraded":"2023-10-09T11:02:51.332701+0000","last_fullsized":"2023-10-09T11:02:51.332701+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:29:09.209024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"445'6271","reported_seq":8620,"reported_epoch":445,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:02:53.017528+0000","last_change":"2023-10-09T11:02:51.343343+0000","last_active":"2023-10-09T11:02:53.017528+0000","last_peered":"2023-10-09T11:02:53.017528+0000","last_clean":"2023-10-09T11:02:32.284069+0000","last_became_active":"2023-10-09T11:02:51.339919+0000","last_became_peered":"2023-10-09T11:02:51.339919+0000","last_unstale":"2023-10-09T11:02:53.017528+0000","last_undegraded":"2023-10-09T11:02:53.017528+0000","last_fullsized":"2023-10-09T11:02:53.017528+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6271,"log_dups_size":0,"ondisk_log_size":6271,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202113024,"num_objects":3084,"num_object_clones":0,"num_object_copies":9252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3084,"num_objects_unfound":0,"num_objects_dirty":3084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3154,"num_write_kb":199616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":568,"num_bytes_recovered":37191680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[0,4,1],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3084}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":924,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.329975+0000","last_change":"2023-10-09T11:02:36.217484+0000","last_active":"2023-10-09T11:02:50.329975+0000","last_peered":"2023-10-09T11:02:50.329975+0000","last_clean":"2023-10-09T11:02:50.329975+0000","last_became_active":"2023-10-09T11:02:36.217091+0000","last_became_peered":"2023-10-09T11:02:36.217091+0000","last_unstale":"2023-10-09T11:02:50.329975+0000","last_undegraded":"2023-10-09T11:02:50.329975+0000","last_fullsized":"2023-10-09T11:02:50.329975+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:59.912586+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":533,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.337502+0000","last_change":"2023-10-09T11:00:15.889100+0000","last_active":"2023-10-09T11:02:48.337502+0000","last_peered":"2023-10-09T11:02:48.337502+0000","last_clean":"2023-10-09T11:02:48.337502+0000","last_became_active":"2023-10-09T11:00:15.888886+0000","last_became_peered":"2023-10-09T11:00:15.888886+0000","last_unstale":"2023-10-09T11:02:48.337502+0000","last_undegraded":"2023-10-09T11:02:48.337502+0000","last_fullsized":"2023-10-09T11:02:48.337502+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:30:40.695500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":233,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332898+0000","last_change":"2023-10-09T11:01:15.401167+0000","last_active":"2023-10-09T11:02:50.332898+0000","last_peered":"2023-10-09T11:02:50.332898+0000","last_clean":"2023-10-09T11:02:50.332898+0000","last_became_active":"2023-10-09T11:01:15.400876+0000","last_became_peered":"2023-10-09T11:01:15.400876+0000","last_unstale":"2023-10-09T11:02:50.332898+0000","last_undegraded":"2023-10-09T11:02:50.332898+0000","last_fullsized":"2023-10-09T11:02:50.332898+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:56.342226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":765,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.334206+0000","last_change":"2023-10-09T11:02:36.216785+0000","last_active":"2023-10-09T11:02:50.334206+0000","last_peered":"2023-10-09T11:02:50.334206+0000","last_clean":"2023-10-09T11:02:50.334206+0000","last_became_active":"2023-10-09T11:02:36.215919+0000","last_became_peered":"2023-10-09T11:02:36.215919+0000","last_unstale":"2023-10-09T11:02:50.334206+0000","last_undegraded":"2023-10-09T11:02:50.334206+0000","last_fullsized":"2023-10-09T11:02:50.334206+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:08:40.893630+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":81,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.332950+0000","last_change":"2023-10-09T10:59:48.124873+0000","last_active":"2023-10-09T11:02:51.332950+0000","last_peered":"2023-10-09T11:02:51.332950+0000","last_clean":"2023-10-09T11:02:51.332950+0000","last_became_active":"2023-10-09T10:59:48.124600+0000","last_became_peered":"2023-10-09T10:59:48.124600+0000","last_unstale":"2023-10-09T11:02:51.332950+0000","last_undegraded":"2023-10-09T11:02:51.332950+0000","last_fullsized":"2023-10-09T11:02:51.332950+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:04.818987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":468,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.337569+0000","last_change":"2023-10-09T11:00:14.869998+0000","last_active":"2023-10-09T11:02:48.337569+0000","last_peered":"2023-10-09T11:02:48.337569+0000","last_clean":"2023-10-09T11:02:48.337569+0000","last_became_active":"2023-10-09T11:00:14.869671+0000","last_became_peered":"2023-10-09T11:00:14.869671+0000","last_unstale":"2023-10-09T11:02:48.337569+0000","last_undegraded":"2023-10-09T11:02:48.337569+0000","last_fullsized":"2023-10-09T11:02:48.337569+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:19.564770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":770,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-10-09T11:02:49.317531+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:02:49.317531+0000","last_peered":"2023-10-09T11:02:49.317531+0000","last_clean":"2023-10-09T11:02:49.317531+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:02:49.317531+0000","last_undegraded":"2023-10-09T11:02:49.317531+0000","last_fullsized":"2023-10-09T11:02:49.317531+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":295,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332869+0000","last_change":"2023-10-09T11:02:33.198185+0000","last_active":"2023-10-09T11:02:50.332869+0000","last_peered":"2023-10-09T11:02:50.332869+0000","last_clean":"2023-10-09T11:02:50.332869+0000","last_became_active":"2023-10-09T11:02:33.197870+0000","last_became_peered":"2023-10-09T11:02:33.197870+0000","last_unstale":"2023-10-09T11:02:50.332869+0000","last_undegraded":"2023-10-09T11:02:50.332869+0000","last_fullsized":"2023-10-09T11:02:50.332869+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:21.200979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":428,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.343657+0000","last_change":"2023-10-09T11:02:33.198055+0000","last_active":"2023-10-09T11:02:48.343657+0000","last_peered":"2023-10-09T11:02:48.343657+0000","last_clean":"2023-10-09T11:02:48.343657+0000","last_became_active":"2023-10-09T11:02:33.197748+0000","last_became_peered":"2023-10-09T11:02:33.197748+0000","last_unstale":"2023-10-09T11:02:48.343657+0000","last_undegraded":"2023-10-09T11:02:48.343657+0000","last_fullsized":"2023-10-09T11:02:48.343657+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:14:15.601866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":633,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332716+0000","last_change":"2023-10-09T11:01:16.673177+0000","last_active":"2023-10-09T11:02:50.332716+0000","last_peered":"2023-10-09T11:02:50.332716+0000","last_clean":"2023-10-09T11:02:50.332716+0000","last_became_active":"2023-10-09T11:01:15.401411+0000","last_became_peered":"2023-10-09T11:01:15.401411+0000","last_unstale":"2023-10-09T11:02:50.332716+0000","last_undegraded":"2023-10-09T11:02:50.332716+0000","last_fullsized":"2023-10-09T11:02:50.332716+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:46.989699+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":25,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.334019+0000","last_change":"2023-10-09T11:02:33.219972+0000","last_active":"2023-10-09T11:02:50.334019+0000","last_peered":"2023-10-09T11:02:50.334019+0000","last_clean":"2023-10-09T11:02:50.334019+0000","last_became_active":"2023-10-09T11:02:33.197428+0000","last_became_peered":"2023-10-09T11:02:33.197428+0000","last_unstale":"2023-10-09T11:02:50.334019+0000","last_undegraded":"2023-10-09T11:02:50.334019+0000","last_fullsized":"2023-10-09T11:02:50.334019+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:02.203640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":202,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332094+0000","last_change":"2023-10-09T11:01:15.401892+0000","last_active":"2023-10-09T11:02:50.332094+0000","last_peered":"2023-10-09T11:02:50.332094+0000","last_clean":"2023-10-09T11:02:50.332094+0000","last_became_active":"2023-10-09T11:01:15.401025+0000","last_became_peered":"2023-10-09T11:01:15.401025+0000","last_unstale":"2023-10-09T11:02:50.332094+0000","last_undegraded":"2023-10-09T11:02:50.332094+0000","last_fullsized":"2023-10-09T11:02:50.332094+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:48:03.531920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":795,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.336456+0000","last_change":"2023-10-09T11:02:09.381602+0000","last_active":"2023-10-09T11:02:51.336456+0000","last_peered":"2023-10-09T11:02:51.336456+0000","last_clean":"2023-10-09T11:02:51.336456+0000","last_became_active":"2023-10-09T11:00:16.068155+0000","last_became_peered":"2023-10-09T11:00:16.068155+0000","last_unstale":"2023-10-09T11:02:51.336456+0000","last_undegraded":"2023-10-09T11:02:51.336456+0000","last_fullsized":"2023-10-09T11:02:51.336456+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:56.972770+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":29,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.334130+0000","last_change":"2023-10-09T11:02:33.213462+0000","last_active":"2023-10-09T11:02:50.334130+0000","last_peered":"2023-10-09T11:02:50.334130+0000","last_clean":"2023-10-09T11:02:50.334130+0000","last_became_active":"2023-10-09T11:02:33.197602+0000","last_became_peered":"2023-10-09T11:02:33.197602+0000","last_unstale":"2023-10-09T11:02:50.334130+0000","last_undegraded":"2023-10-09T11:02:50.334130+0000","last_fullsized":"2023-10-09T11:02:50.334130+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.773902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":458,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.329775+0000","last_change":"2023-10-09T10:57:46.845755+0000","last_active":"2023-10-09T11:02:50.329775+0000","last_peered":"2023-10-09T11:02:50.329775+0000","last_clean":"2023-10-09T11:02:50.329775+0000","last_became_active":"2023-10-09T10:57:46.845475+0000","last_became_peered":"2023-10-09T10:57:46.845475+0000","last_unstale":"2023-10-09T11:02:50.329775+0000","last_undegraded":"2023-10-09T11:02:50.329775+0000","last_fullsized":"2023-10-09T11:02:50.329775+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:21.673994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":808,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.332105+0000","last_change":"2023-10-09T10:59:49.175255+0000","last_active":"2023-10-09T11:02:51.332105+0000","last_peered":"2023-10-09T11:02:51.332105+0000","last_clean":"2023-10-09T11:02:51.332105+0000","last_became_active":"2023-10-09T10:59:49.174666+0000","last_became_peered":"2023-10-09T10:59:49.174666+0000","last_unstale":"2023-10-09T11:02:51.332105+0000","last_undegraded":"2023-10-09T11:02:51.332105+0000","last_fullsized":"2023-10-09T11:02:51.332105+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:00:58.069275+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":25,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.333996+0000","last_change":"2023-10-09T11:02:33.207032+0000","last_active":"2023-10-09T11:02:50.333996+0000","last_peered":"2023-10-09T11:02:50.333996+0000","last_clean":"2023-10-09T11:02:50.333996+0000","last_became_active":"2023-10-09T11:02:33.196379+0000","last_became_peered":"2023-10-09T11:02:33.196379+0000","last_unstale":"2023-10-09T11:02:50.333996+0000","last_undegraded":"2023-10-09T11:02:50.333996+0000","last_fullsized":"2023-10-09T11:02:50.333996+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:56.935603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":554,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.361413+0000","last_change":"2023-10-09T11:02:33.198289+0000","last_active":"2023-10-09T11:02:48.361413+0000","last_peered":"2023-10-09T11:02:48.361413+0000","last_clean":"2023-10-09T11:02:48.361413+0000","last_became_active":"2023-10-09T11:02:33.197962+0000","last_became_peered":"2023-10-09T11:02:33.197962+0000","last_unstale":"2023-10-09T11:02:48.361413+0000","last_undegraded":"2023-10-09T11:02:48.361413+0000","last_fullsized":"2023-10-09T11:02:48.361413+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:05:34.023093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":715,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332300+0000","last_change":"2023-10-09T11:02:36.216877+0000","last_active":"2023-10-09T11:02:50.332300+0000","last_peered":"2023-10-09T11:02:50.332300+0000","last_clean":"2023-10-09T11:02:50.332300+0000","last_became_active":"2023-10-09T11:02:36.216072+0000","last_became_peered":"2023-10-09T11:02:36.216072+0000","last_unstale":"2023-10-09T11:02:50.332300+0000","last_undegraded":"2023-10-09T11:02:50.332300+0000","last_fullsized":"2023-10-09T11:02:50.332300+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:47.741775+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":386,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.331954+0000","last_change":"2023-10-09T11:02:33.198858+0000","last_active":"2023-10-09T11:02:51.331954+0000","last_peered":"2023-10-09T11:02:51.331954+0000","last_clean":"2023-10-09T11:02:51.331954+0000","last_became_active":"2023-10-09T11:02:33.198557+0000","last_became_peered":"2023-10-09T11:02:33.198557+0000","last_unstale":"2023-10-09T11:02:51.331954+0000","last_undegraded":"2023-10-09T11:02:51.331954+0000","last_fullsized":"2023-10-09T11:02:51.331954+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:32.140678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":328,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332636+0000","last_change":"2023-10-09T11:02:33.203078+0000","last_active":"2023-10-09T11:02:50.332636+0000","last_peered":"2023-10-09T11:02:50.332636+0000","last_clean":"2023-10-09T11:02:50.332636+0000","last_became_active":"2023-10-09T11:02:33.202772+0000","last_became_peered":"2023-10-09T11:02:33.202772+0000","last_unstale":"2023-10-09T11:02:50.332636+0000","last_undegraded":"2023-10-09T11:02:50.332636+0000","last_fullsized":"2023-10-09T11:02:50.332636+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:28:14.080239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.18","version":"445'6246","reported_seq":8648,"reported_epoch":445,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T11:02:53.043213+0000","last_change":"2023-10-09T11:02:49.399055+0000","last_active":"2023-10-09T11:02:53.043213+0000","last_peered":"2023-10-09T11:02:53.043213+0000","last_clean":"2023-10-09T11:02:32.305733+0000","last_became_active":"2023-10-09T11:02:49.395614+0000","last_became_peered":"2023-10-09T11:02:49.395614+0000","last_unstale":"2023-10-09T11:02:53.043213+0000","last_undegraded":"2023-10-09T11:02:49.392298+0000","last_fullsized":"2023-10-09T11:02:53.043213+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6246,"log_dups_size":0,"ondisk_log_size":6246,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201785344,"num_objects":3079,"num_object_clones":0,"num_object_copies":9237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2666,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3139,"num_write_kb":198976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":595,"num_bytes_recovered":38961152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":2666},{"shards":"0,2,7","objects":413}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":737,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.335551+0000","last_change":"2023-10-09T11:01:25.620261+0000","last_active":"2023-10-09T11:02:51.335551+0000","last_peered":"2023-10-09T11:02:51.335551+0000","last_clean":"2023-10-09T11:02:51.335551+0000","last_became_active":"2023-10-09T11:00:14.871258+0000","last_became_peered":"2023-10-09T11:00:14.871258+0000","last_unstale":"2023-10-09T11:02:51.335551+0000","last_undegraded":"2023-10-09T11:02:51.335551+0000","last_fullsized":"2023-10-09T11:02:51.335551+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:13.861006+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":19,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.334545+0000","last_change":"2023-10-09T11:02:33.198572+0000","last_active":"2023-10-09T11:02:50.334545+0000","last_peered":"2023-10-09T11:02:50.334545+0000","last_clean":"2023-10-09T11:02:50.334545+0000","last_became_active":"2023-10-09T11:02:33.198218+0000","last_became_peered":"2023-10-09T11:02:33.198218+0000","last_unstale":"2023-10-09T11:02:50.334545+0000","last_undegraded":"2023-10-09T11:02:50.334545+0000","last_fullsized":"2023-10-09T11:02:50.334545+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:59.066866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":99,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.332522+0000","last_change":"2023-10-09T11:02:33.198882+0000","last_active":"2023-10-09T11:02:51.332522+0000","last_peered":"2023-10-09T11:02:51.332522+0000","last_clean":"2023-10-09T11:02:51.332522+0000","last_became_active":"2023-10-09T11:02:33.198380+0000","last_became_peered":"2023-10-09T11:02:33.198380+0000","last_unstale":"2023-10-09T11:02:51.332522+0000","last_undegraded":"2023-10-09T11:02:51.332522+0000","last_fullsized":"2023-10-09T11:02:51.332522+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:21:35.066055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"440'6139","reported_seq":9338,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:02:51.349018+0000","last_change":"2023-10-09T11:02:51.366454+0000","last_active":"2023-10-09T11:02:51.349018+0000","last_peered":"2023-10-09T11:02:51.349018+0000","last_clean":"2023-10-09T11:02:32.272642+0000","last_became_active":"2023-10-09T11:02:51.345927+0000","last_became_peered":"2023-10-09T11:02:51.345927+0000","last_unstale":"2023-10-09T11:02:51.349018+0000","last_undegraded":"2023-10-09T11:02:51.338438+0000","last_fullsized":"2023-10-09T11:02:51.334387+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6139,"log_dups_size":0,"ondisk_log_size":6139,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201162752,"num_objects":3070,"num_object_clones":0,"num_object_copies":9210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2546,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3070,"num_write_kb":196448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":777,"num_bytes_recovered":50921472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":2546},{"shards":"2,5","objects":524}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":712,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332363+0000","last_change":"2023-10-09T11:01:15.406928+0000","last_active":"2023-10-09T11:02:50.332363+0000","last_peered":"2023-10-09T11:02:50.332363+0000","last_clean":"2023-10-09T11:02:50.332363+0000","last_became_active":"2023-10-09T11:01:15.406499+0000","last_became_peered":"2023-10-09T11:01:15.406499+0000","last_unstale":"2023-10-09T11:02:50.332363+0000","last_undegraded":"2023-10-09T11:02:50.332363+0000","last_fullsized":"2023-10-09T11:02:50.332363+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:33.050336+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":171,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.990313+0000","last_change":"2023-10-09T11:00:48.109643+0000","last_active":"2023-10-09T11:02:50.990313+0000","last_peered":"2023-10-09T11:02:50.990313+0000","last_clean":"2023-10-09T11:02:50.990313+0000","last_became_active":"2023-10-09T11:00:48.109299+0000","last_became_peered":"2023-10-09T11:00:48.109299+0000","last_unstale":"2023-10-09T11:02:50.990313+0000","last_undegraded":"2023-10-09T11:02:50.990313+0000","last_fullsized":"2023-10-09T11:02:50.990313+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:31.163237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":530,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.373885+0000","last_change":"2023-10-09T11:02:33.200032+0000","last_active":"2023-10-09T11:02:48.373885+0000","last_peered":"2023-10-09T11:02:48.373885+0000","last_clean":"2023-10-09T11:02:48.373885+0000","last_became_active":"2023-10-09T11:02:33.199691+0000","last_became_peered":"2023-10-09T11:02:33.199691+0000","last_unstale":"2023-10-09T11:02:48.373885+0000","last_undegraded":"2023-10-09T11:02:48.373885+0000","last_fullsized":"2023-10-09T11:02:48.373885+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:15.777063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"442'6265","reported_seq":6325,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332833+0000","last_change":"2023-10-09T11:02:49.333682+0000","last_active":"2023-10-09T11:02:50.332833+0000","last_peered":"2023-10-09T11:02:50.332833+0000","last_clean":"2023-10-09T11:02:50.332833+0000","last_became_active":"2023-10-09T11:02:49.327718+0000","last_became_peered":"2023-10-09T11:02:49.327718+0000","last_unstale":"2023-10-09T11:02:50.332833+0000","last_undegraded":"2023-10-09T11:02:50.332833+0000","last_fullsized":"2023-10-09T11:02:50.332833+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6265,"log_dups_size":0,"ondisk_log_size":6265,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205324288,"num_objects":3133,"num_object_clones":0,"num_object_copies":9399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3133,"num_write_kb":200512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":733,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.334115+0000","last_change":"2023-10-09T11:02:36.217050+0000","last_active":"2023-10-09T11:02:50.334115+0000","last_peered":"2023-10-09T11:02:50.334115+0000","last_clean":"2023-10-09T11:02:50.334115+0000","last_became_active":"2023-10-09T11:02:36.216387+0000","last_became_peered":"2023-10-09T11:02:36.216387+0000","last_unstale":"2023-10-09T11:02:50.334115+0000","last_undegraded":"2023-10-09T11:02:50.334115+0000","last_fullsized":"2023-10-09T11:02:50.334115+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:19.627177+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":23,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.334900+0000","last_change":"2023-10-09T11:02:33.221990+0000","last_active":"2023-10-09T11:02:50.334900+0000","last_peered":"2023-10-09T11:02:50.334900+0000","last_clean":"2023-10-09T11:02:50.334900+0000","last_became_active":"2023-10-09T11:02:33.198155+0000","last_became_peered":"2023-10-09T11:02:33.198155+0000","last_unstale":"2023-10-09T11:02:50.334900+0000","last_undegraded":"2023-10-09T11:02:50.334900+0000","last_fullsized":"2023-10-09T11:02:50.334900+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:32.563344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":366,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332089+0000","last_change":"2023-10-09T11:02:33.200229+0000","last_active":"2023-10-09T11:02:50.332089+0000","last_peered":"2023-10-09T11:02:50.332089+0000","last_clean":"2023-10-09T11:02:50.332089+0000","last_became_active":"2023-10-09T11:02:33.199936+0000","last_became_peered":"2023-10-09T11:02:33.199936+0000","last_unstale":"2023-10-09T11:02:50.332089+0000","last_undegraded":"2023-10-09T11:02:50.332089+0000","last_fullsized":"2023-10-09T11:02:50.332089+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:28.854719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"442'6110","reported_seq":6168,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-10-09T11:02:49.360324+0000","last_change":"2023-10-09T11:02:49.350396+0000","last_active":"2023-10-09T11:02:49.360324+0000","last_peered":"2023-10-09T11:02:49.360324+0000","last_clean":"2023-10-09T11:02:49.360324+0000","last_became_active":"2023-10-09T11:02:49.347031+0000","last_became_peered":"2023-10-09T11:02:49.347031+0000","last_unstale":"2023-10-09T11:02:49.360324+0000","last_undegraded":"2023-10-09T11:02:49.360324+0000","last_fullsized":"2023-10-09T11:02:49.360324+0000","mapping_epoch":403,"log_start":"406'100","ondisk_log_start":"406'100","created":403,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6010,"log_dups_size":0,"ondisk_log_size":6010,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200245248,"num_objects":3055,"num_object_clones":0,"num_object_copies":9165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3055,"num_write_kb":195552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":847,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.361247+0000","last_change":"2023-10-09T10:59:51.140014+0000","last_active":"2023-10-09T11:02:48.361247+0000","last_peered":"2023-10-09T11:02:48.361247+0000","last_clean":"2023-10-09T11:02:48.361247+0000","last_became_active":"2023-10-09T10:59:51.139665+0000","last_became_peered":"2023-10-09T10:59:51.139665+0000","last_unstale":"2023-10-09T11:02:48.361247+0000","last_undegraded":"2023-10-09T11:02:48.361247+0000","last_fullsized":"2023-10-09T11:02:48.361247+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:29.505223+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":23,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.334243+0000","last_change":"2023-10-09T11:02:33.208705+0000","last_active":"2023-10-09T11:02:50.334243+0000","last_peered":"2023-10-09T11:02:50.334243+0000","last_clean":"2023-10-09T11:02:50.334243+0000","last_became_active":"2023-10-09T11:02:33.197454+0000","last_became_peered":"2023-10-09T11:02:33.197454+0000","last_unstale":"2023-10-09T11:02:50.334243+0000","last_undegraded":"2023-10-09T11:02:50.334243+0000","last_fullsized":"2023-10-09T11:02:50.334243+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:52.649912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":124,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.335298+0000","last_change":"2023-10-09T11:00:14.872673+0000","last_active":"2023-10-09T11:02:51.335298+0000","last_peered":"2023-10-09T11:02:51.335298+0000","last_clean":"2023-10-09T11:02:51.335298+0000","last_became_active":"2023-10-09T11:00:14.872345+0000","last_became_peered":"2023-10-09T11:00:14.872345+0000","last_unstale":"2023-10-09T11:02:51.335298+0000","last_undegraded":"2023-10-09T11:02:51.335298+0000","last_fullsized":"2023-10-09T11:02:51.335298+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:32:43.821085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"445'6107","reported_seq":6168,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-10-09T11:02:52.759698+0000","last_change":"2023-10-09T11:02:49.365913+0000","last_active":"2023-10-09T11:02:52.759698+0000","last_peered":"2023-10-09T11:02:52.759698+0000","last_clean":"2023-10-09T11:02:52.759698+0000","last_became_active":"2023-10-09T11:02:49.361117+0000","last_became_peered":"2023-10-09T11:02:49.361117+0000","last_unstale":"2023-10-09T11:02:52.759698+0000","last_undegraded":"2023-10-09T11:02:52.759698+0000","last_fullsized":"2023-10-09T11:02:52.759698+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6107,"log_dups_size":0,"ondisk_log_size":6107,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197656576,"num_objects":3016,"num_object_clones":0,"num_object_copies":9048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3066,"num_write_kb":194624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":640,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332805+0000","last_change":"2023-10-09T11:01:22.599166+0000","last_active":"2023-10-09T11:02:50.332805+0000","last_peered":"2023-10-09T11:02:50.332805+0000","last_clean":"2023-10-09T11:02:50.332805+0000","last_became_active":"2023-10-09T11:01:15.402193+0000","last_became_peered":"2023-10-09T11:01:15.402193+0000","last_unstale":"2023-10-09T11:02:50.332805+0000","last_undegraded":"2023-10-09T11:02:50.332805+0000","last_fullsized":"2023-10-09T11:02:50.332805+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:37:16.611219+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":23,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.334598+0000","last_change":"2023-10-09T11:02:33.223801+0000","last_active":"2023-10-09T11:02:50.334598+0000","last_peered":"2023-10-09T11:02:50.334598+0000","last_clean":"2023-10-09T11:02:50.334598+0000","last_became_active":"2023-10-09T11:02:33.198699+0000","last_became_peered":"2023-10-09T11:02:33.198699+0000","last_unstale":"2023-10-09T11:02:50.334598+0000","last_undegraded":"2023-10-09T11:02:50.334598+0000","last_fullsized":"2023-10-09T11:02:50.334598+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:13.818599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":293,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.990907+0000","last_change":"2023-10-09T11:02:33.199120+0000","last_active":"2023-10-09T11:02:50.990907+0000","last_peered":"2023-10-09T11:02:50.990907+0000","last_clean":"2023-10-09T11:02:50.990907+0000","last_became_active":"2023-10-09T11:02:33.198673+0000","last_became_peered":"2023-10-09T11:02:33.198673+0000","last_unstale":"2023-10-09T11:02:50.990907+0000","last_undegraded":"2023-10-09T11:02:50.990907+0000","last_fullsized":"2023-10-09T11:02:50.990907+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:58:06.417132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"440'6235","reported_seq":6292,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-10-09T11:02:49.382082+0000","last_change":"2023-10-09T11:02:49.375401+0000","last_active":"2023-10-09T11:02:49.382082+0000","last_peered":"2023-10-09T11:02:49.382082+0000","last_clean":"2023-10-09T11:02:49.382082+0000","last_became_active":"2023-10-09T11:02:49.372519+0000","last_became_peered":"2023-10-09T11:02:49.372519+0000","last_unstale":"2023-10-09T11:02:49.382082+0000","last_undegraded":"2023-10-09T11:02:49.382082+0000","last_fullsized":"2023-10-09T11:02:49.382082+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6235,"log_dups_size":0,"ondisk_log_size":6235,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204308480,"num_objects":3117,"num_object_clones":0,"num_object_copies":9351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3117,"num_write_kb":199520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":969,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.330079+0000","last_change":"2023-10-09T11:02:36.218864+0000","last_active":"2023-10-09T11:02:50.330079+0000","last_peered":"2023-10-09T11:02:50.330079+0000","last_clean":"2023-10-09T11:02:50.330079+0000","last_became_active":"2023-10-09T11:02:36.218557+0000","last_became_peered":"2023-10-09T11:02:36.218557+0000","last_unstale":"2023-10-09T11:02:50.330079+0000","last_undegraded":"2023-10-09T11:02:50.330079+0000","last_fullsized":"2023-10-09T11:02:50.330079+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:49.124970+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":255,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332563+0000","last_change":"2023-10-09T11:01:15.400732+0000","last_active":"2023-10-09T11:02:50.332563+0000","last_peered":"2023-10-09T11:02:50.332563+0000","last_clean":"2023-10-09T11:02:50.332563+0000","last_became_active":"2023-10-09T11:01:15.399370+0000","last_became_peered":"2023-10-09T11:01:15.399370+0000","last_unstale":"2023-10-09T11:02:50.332563+0000","last_undegraded":"2023-10-09T11:02:50.332563+0000","last_fullsized":"2023-10-09T11:02:50.332563+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:53.948144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":366,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.332298+0000","last_change":"2023-10-09T11:00:15.865848+0000","last_active":"2023-10-09T11:02:51.332298+0000","last_peered":"2023-10-09T11:02:51.332298+0000","last_clean":"2023-10-09T11:02:51.332298+0000","last_became_active":"2023-10-09T11:00:15.865504+0000","last_became_peered":"2023-10-09T11:00:15.865504+0000","last_unstale":"2023-10-09T11:02:51.332298+0000","last_undegraded":"2023-10-09T11:02:51.332298+0000","last_fullsized":"2023-10-09T11:02:51.332298+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:32.352564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"445'6187","reported_seq":6247,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-10-09T11:02:52.925046+0000","last_change":"2023-10-09T11:02:49.346194+0000","last_active":"2023-10-09T11:02:52.925046+0000","last_peered":"2023-10-09T11:02:52.925046+0000","last_clean":"2023-10-09T11:02:52.925046+0000","last_became_active":"2023-10-09T11:02:49.340473+0000","last_became_peered":"2023-10-09T11:02:49.340473+0000","last_unstale":"2023-10-09T11:02:52.925046+0000","last_undegraded":"2023-10-09T11:02:52.925046+0000","last_fullsized":"2023-10-09T11:02:52.925046+0000","mapping_epoch":403,"log_start":"406'500","ondisk_log_start":"406'500","created":403,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5687,"log_dups_size":0,"ondisk_log_size":5687,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199688192,"num_objects":3047,"num_object_clones":0,"num_object_copies":9141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3109,"num_write_kb":196992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":705,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.334604+0000","last_change":"2023-10-09T11:02:36.228163+0000","last_active":"2023-10-09T11:02:50.334604+0000","last_peered":"2023-10-09T11:02:50.334604+0000","last_clean":"2023-10-09T11:02:50.334604+0000","last_became_active":"2023-10-09T11:02:36.227871+0000","last_became_peered":"2023-10-09T11:02:36.227871+0000","last_unstale":"2023-10-09T11:02:50.334604+0000","last_undegraded":"2023-10-09T11:02:50.334604+0000","last_fullsized":"2023-10-09T11:02:50.334604+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:35.196080+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":461,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.346153+0000","last_change":"2023-10-09T10:59:49.127924+0000","last_active":"2023-10-09T11:02:48.346153+0000","last_peered":"2023-10-09T11:02:48.346153+0000","last_clean":"2023-10-09T11:02:48.346153+0000","last_became_active":"2023-10-09T10:59:49.127549+0000","last_became_peered":"2023-10-09T10:59:49.127549+0000","last_unstale":"2023-10-09T11:02:48.346153+0000","last_undegraded":"2023-10-09T11:02:48.346153+0000","last_fullsized":"2023-10-09T11:02:48.346153+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:01.781131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":286,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.361527+0000","last_change":"2023-10-09T10:50:43.384976+0000","last_active":"2023-10-09T11:02:48.361527+0000","last_peered":"2023-10-09T11:02:48.361527+0000","last_clean":"2023-10-09T11:02:48.361527+0000","last_became_active":"2023-10-09T10:50:43.384615+0000","last_became_peered":"2023-10-09T10:50:43.384615+0000","last_unstale":"2023-10-09T11:02:48.361527+0000","last_undegraded":"2023-10-09T11:02:48.361527+0000","last_fullsized":"2023-10-09T11:02:48.361527+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:49:51.548940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"440'6119","reported_seq":6178,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-10-09T11:02:49.334899+0000","last_change":"2023-10-09T11:02:49.334428+0000","last_active":"2023-10-09T11:02:49.334899+0000","last_peered":"2023-10-09T11:02:49.334899+0000","last_clean":"2023-10-09T11:02:49.334899+0000","last_became_active":"2023-10-09T11:02:49.328926+0000","last_became_peered":"2023-10-09T11:02:49.328926+0000","last_unstale":"2023-10-09T11:02:49.334899+0000","last_undegraded":"2023-10-09T11:02:49.334899+0000","last_fullsized":"2023-10-09T11:02:49.334899+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6119,"log_dups_size":0,"ondisk_log_size":6119,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200507392,"num_objects":3059,"num_object_clones":0,"num_object_copies":9177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3059,"num_write_kb":195808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":635,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332266+0000","last_change":"2023-10-09T11:01:20.667499+0000","last_active":"2023-10-09T11:02:50.332266+0000","last_peered":"2023-10-09T11:02:50.332266+0000","last_clean":"2023-10-09T11:02:50.332266+0000","last_became_active":"2023-10-09T11:01:15.399808+0000","last_became_peered":"2023-10-09T11:01:15.399808+0000","last_unstale":"2023-10-09T11:02:50.332266+0000","last_undegraded":"2023-10-09T11:02:50.332266+0000","last_fullsized":"2023-10-09T11:02:50.332266+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:53:29.298692+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":440,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.330160+0000","last_change":"2023-10-09T11:00:14.891176+0000","last_active":"2023-10-09T11:02:50.330160+0000","last_peered":"2023-10-09T11:02:50.330160+0000","last_clean":"2023-10-09T11:02:50.330160+0000","last_became_active":"2023-10-09T11:00:14.890890+0000","last_became_peered":"2023-10-09T11:00:14.890890+0000","last_unstale":"2023-10-09T11:02:50.330160+0000","last_undegraded":"2023-10-09T11:02:50.330160+0000","last_fullsized":"2023-10-09T11:02:50.330160+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:38.158759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":199,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-10-09T11:02:49.322146+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:02:49.322146+0000","last_peered":"2023-10-09T11:02:49.322146+0000","last_clean":"2023-10-09T11:02:49.322146+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:02:49.322146+0000","last_undegraded":"2023-10-09T11:02:49.322146+0000","last_fullsized":"2023-10-09T11:02:49.322146+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"440'6057","reported_seq":6117,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.990206+0000","last_change":"2023-10-09T11:02:49.342305+0000","last_active":"2023-10-09T11:02:50.990206+0000","last_peered":"2023-10-09T11:02:50.990206+0000","last_clean":"2023-10-09T11:02:50.990206+0000","last_became_active":"2023-10-09T11:02:49.339201+0000","last_became_peered":"2023-10-09T11:02:49.339201+0000","last_unstale":"2023-10-09T11:02:50.990206+0000","last_undegraded":"2023-10-09T11:02:50.990206+0000","last_fullsized":"2023-10-09T11:02:50.990206+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6057,"log_dups_size":0,"ondisk_log_size":6057,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198475776,"num_objects":3028,"num_object_clones":0,"num_object_copies":9084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3028,"num_write_kb":193824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":794,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.335832+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:02:51.335832+0000","last_peered":"2023-10-09T11:02:51.335832+0000","last_clean":"2023-10-09T11:02:51.335832+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:02:51.335832+0000","last_undegraded":"2023-10-09T11:02:51.335832+0000","last_fullsized":"2023-10-09T11:02:51.335832+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":519,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.330399+0000","last_change":"2023-10-09T11:02:33.200106+0000","last_active":"2023-10-09T11:02:50.330399+0000","last_peered":"2023-10-09T11:02:50.330399+0000","last_clean":"2023-10-09T11:02:50.330399+0000","last_became_active":"2023-10-09T11:02:33.199840+0000","last_became_peered":"2023-10-09T11:02:33.199840+0000","last_unstale":"2023-10-09T11:02:50.330399+0000","last_undegraded":"2023-10-09T11:02:50.330399+0000","last_fullsized":"2023-10-09T11:02:50.330399+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:13:17.557980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":124,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.336202+0000","last_change":"2023-10-09T11:00:14.900130+0000","last_active":"2023-10-09T11:02:51.336202+0000","last_peered":"2023-10-09T11:02:51.336202+0000","last_clean":"2023-10-09T11:02:51.336202+0000","last_became_active":"2023-10-09T11:00:14.899536+0000","last_became_peered":"2023-10-09T11:00:14.899536+0000","last_unstale":"2023-10-09T11:02:51.336202+0000","last_undegraded":"2023-10-09T11:02:51.336202+0000","last_fullsized":"2023-10-09T11:02:51.336202+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:07.151510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"445'6267","reported_seq":8617,"reported_epoch":445,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:02:53.010483+0000","last_change":"2023-10-09T11:02:49.358320+0000","last_active":"2023-10-09T11:02:53.010483+0000","last_peered":"2023-10-09T11:02:53.010483+0000","last_clean":"2023-10-09T11:02:32.284069+0000","last_became_active":"2023-10-09T11:02:49.355202+0000","last_became_peered":"2023-10-09T11:02:49.355202+0000","last_unstale":"2023-10-09T11:02:53.010483+0000","last_undegraded":"2023-10-09T11:02:53.010483+0000","last_fullsized":"2023-10-09T11:02:53.010483+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6267,"log_dups_size":0,"ondisk_log_size":6267,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202244096,"num_objects":3086,"num_object_clones":0,"num_object_copies":9258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3086,"num_objects_unfound":0,"num_objects_dirty":3086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3150,"num_write_kb":199552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":567,"num_bytes_recovered":37191680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[0,4,1],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3086}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":646,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332273+0000","last_change":"2023-10-09T11:01:51.463338+0000","last_active":"2023-10-09T11:02:50.332273+0000","last_peered":"2023-10-09T11:02:50.332273+0000","last_clean":"2023-10-09T11:02:50.332273+0000","last_became_active":"2023-10-09T11:01:15.399431+0000","last_became_peered":"2023-10-09T11:01:15.399431+0000","last_unstale":"2023-10-09T11:02:50.332273+0000","last_undegraded":"2023-10-09T11:02:50.332273+0000","last_fullsized":"2023-10-09T11:02:50.332273+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:01:07.923413+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":523,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.337360+0000","last_change":"2023-10-09T11:02:33.198651+0000","last_active":"2023-10-09T11:02:48.337360+0000","last_peered":"2023-10-09T11:02:48.337360+0000","last_clean":"2023-10-09T11:02:48.337360+0000","last_became_active":"2023-10-09T11:02:33.198378+0000","last_became_peered":"2023-10-09T11:02:33.198378+0000","last_unstale":"2023-10-09T11:02:48.337360+0000","last_undegraded":"2023-10-09T11:02:48.337360+0000","last_fullsized":"2023-10-09T11:02:48.337360+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:27.837451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":124,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.336490+0000","last_change":"2023-10-09T11:00:14.917869+0000","last_active":"2023-10-09T11:02:51.336490+0000","last_peered":"2023-10-09T11:02:51.336490+0000","last_clean":"2023-10-09T11:02:51.336490+0000","last_became_active":"2023-10-09T11:00:14.917535+0000","last_became_peered":"2023-10-09T11:00:14.917535+0000","last_unstale":"2023-10-09T11:02:51.336490+0000","last_undegraded":"2023-10-09T11:02:51.336490+0000","last_fullsized":"2023-10-09T11:02:51.336490+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:10.223866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"445'6249","reported_seq":8803,"reported_epoch":445,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:02:53.023931+0000","last_change":"2023-10-09T11:02:49.384585+0000","last_active":"2023-10-09T11:02:53.023931+0000","last_peered":"2023-10-09T11:02:53.023931+0000","last_clean":"2023-10-09T11:02:32.310266+0000","last_became_active":"2023-10-09T11:02:34.300972+0000","last_became_peered":"2023-10-09T11:02:34.300972+0000","last_unstale":"2023-10-09T11:02:53.023931+0000","last_undegraded":"2023-10-09T11:02:53.023931+0000","last_fullsized":"2023-10-09T11:02:53.023931+0000","mapping_epoch":435,"log_start":"406'387","ondisk_log_start":"406'387","created":403,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5862,"log_dups_size":0,"ondisk_log_size":5862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:40.292013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397344768,"num_objects":6063,"num_object_clones":0,"num_object_copies":18189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5962,"num_objects_unfound":0,"num_objects_dirty":6063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6249,"num_write_kb":393984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1239,"num_bytes_recovered":81199104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[0,4,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":6063}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":910,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.373634+0000","last_change":"2023-10-09T11:02:36.222403+0000","last_active":"2023-10-09T11:02:48.373634+0000","last_peered":"2023-10-09T11:02:48.373634+0000","last_clean":"2023-10-09T11:02:48.373634+0000","last_became_active":"2023-10-09T11:02:36.220390+0000","last_became_peered":"2023-10-09T11:02:36.220390+0000","last_unstale":"2023-10-09T11:02:48.373634+0000","last_undegraded":"2023-10-09T11:02:48.373634+0000","last_fullsized":"2023-10-09T11:02:48.373634+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:57.480540+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":154,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.335947+0000","last_change":"2023-10-09T11:02:33.199808+0000","last_active":"2023-10-09T11:02:51.335947+0000","last_peered":"2023-10-09T11:02:51.335947+0000","last_clean":"2023-10-09T11:02:51.335947+0000","last_became_active":"2023-10-09T11:02:33.199332+0000","last_became_peered":"2023-10-09T11:02:33.199332+0000","last_unstale":"2023-10-09T11:02:51.335947+0000","last_undegraded":"2023-10-09T11:02:51.335947+0000","last_fullsized":"2023-10-09T11:02:51.335947+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:23:19.999621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":282,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332535+0000","last_change":"2023-10-09T11:01:15.399780+0000","last_active":"2023-10-09T11:02:50.332535+0000","last_peered":"2023-10-09T11:02:50.332535+0000","last_clean":"2023-10-09T11:02:50.332535+0000","last_became_active":"2023-10-09T11:01:15.399493+0000","last_became_peered":"2023-10-09T11:01:15.399493+0000","last_unstale":"2023-10-09T11:02:50.332535+0000","last_undegraded":"2023-10-09T11:02:50.332535+0000","last_fullsized":"2023-10-09T11:02:50.332535+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:41.639383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":124,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.335683+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:02:51.335683+0000","last_peered":"2023-10-09T11:02:51.335683+0000","last_clean":"2023-10-09T11:02:51.335683+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:02:51.335683+0000","last_undegraded":"2023-10-09T11:02:51.335683+0000","last_fullsized":"2023-10-09T11:02:51.335683+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"441'6131","reported_seq":6180,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.990728+0000","last_change":"2023-10-09T11:01:15.406192+0000","last_active":"2023-10-09T11:02:50.990728+0000","last_peered":"2023-10-09T11:02:50.990728+0000","last_clean":"2023-10-09T11:02:50.990728+0000","last_became_active":"2023-10-09T11:01:15.405811+0000","last_became_peered":"2023-10-09T11:01:15.405811+0000","last_unstale":"2023-10-09T11:02:50.990728+0000","last_undegraded":"2023-10-09T11:02:50.990728+0000","last_fullsized":"2023-10-09T11:02:50.990728+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6131,"log_dups_size":0,"ondisk_log_size":6131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:48:38.805496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401801216,"num_objects":6131,"num_object_clones":0,"num_object_copies":18393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6131,"num_write_kb":392384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":644,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.333108+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:02:50.333108+0000","last_peered":"2023-10-09T11:02:50.333108+0000","last_clean":"2023-10-09T11:02:50.333108+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:02:50.333108+0000","last_undegraded":"2023-10-09T11:02:50.333108+0000","last_fullsized":"2023-10-09T11:02:50.333108+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":306,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332185+0000","last_change":"2023-10-09T11:01:15.401697+0000","last_active":"2023-10-09T11:02:50.332185+0000","last_peered":"2023-10-09T11:02:50.332185+0000","last_clean":"2023-10-09T11:02:50.332185+0000","last_became_active":"2023-10-09T11:01:15.401386+0000","last_became_peered":"2023-10-09T11:01:15.401386+0000","last_unstale":"2023-10-09T11:02:50.332185+0000","last_undegraded":"2023-10-09T11:02:50.332185+0000","last_fullsized":"2023-10-09T11:02:50.332185+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:40.572070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":351,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.332061+0000","last_change":"2023-10-09T10:59:49.126425+0000","last_active":"2023-10-09T11:02:51.332061+0000","last_peered":"2023-10-09T11:02:51.332061+0000","last_clean":"2023-10-09T11:02:51.332061+0000","last_became_active":"2023-10-09T10:59:49.125712+0000","last_became_peered":"2023-10-09T10:59:49.125712+0000","last_unstale":"2023-10-09T11:02:51.332061+0000","last_undegraded":"2023-10-09T11:02:51.332061+0000","last_fullsized":"2023-10-09T11:02:51.332061+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:40:53.898922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":124,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.335862+0000","last_change":"2023-10-09T11:00:14.878579+0000","last_active":"2023-10-09T11:02:51.335862+0000","last_peered":"2023-10-09T11:02:51.335862+0000","last_clean":"2023-10-09T11:02:51.335862+0000","last_became_active":"2023-10-09T11:00:14.867930+0000","last_became_peered":"2023-10-09T11:00:14.867930+0000","last_unstale":"2023-10-09T11:02:51.335862+0000","last_undegraded":"2023-10-09T11:02:51.335862+0000","last_fullsized":"2023-10-09T11:02:51.335862+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:14:34.925327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"440'6139","reported_seq":9966,"reported_epoch":444,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T11:02:51.644782+0000","last_change":"2023-10-09T11:02:49.385813+0000","last_active":"2023-10-09T11:02:51.644782+0000","last_peered":"2023-10-09T11:02:51.644782+0000","last_clean":"2023-10-09T11:02:32.272642+0000","last_became_active":"2023-10-09T11:02:49.369688+0000","last_became_peered":"2023-10-09T11:02:49.369688+0000","last_unstale":"2023-10-09T11:02:51.644782+0000","last_undegraded":"2023-10-09T11:02:51.644782+0000","last_fullsized":"2023-10-09T11:02:49.318098+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6139,"log_dups_size":0,"ondisk_log_size":6139,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201162752,"num_objects":3069,"num_object_clones":0,"num_object_copies":9207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2162,"num_objects_unfound":0,"num_objects_dirty":3069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3069,"num_write_kb":196448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1091,"num_bytes_recovered":71499776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":2162},{"shards":"2,5","objects":907}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":879,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.334504+0000","last_change":"2023-10-09T11:02:36.216448+0000","last_active":"2023-10-09T11:02:50.334504+0000","last_peered":"2023-10-09T11:02:50.334504+0000","last_clean":"2023-10-09T11:02:50.334504+0000","last_became_active":"2023-10-09T11:02:36.216040+0000","last_became_peered":"2023-10-09T11:02:36.216040+0000","last_unstale":"2023-10-09T11:02:50.334504+0000","last_undegraded":"2023-10-09T11:02:50.334504+0000","last_fullsized":"2023-10-09T11:02:50.334504+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:13:40.104907+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":195,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-10-09T11:02:49.318280+0000","last_change":"2023-10-09T10:59:48.122743+0000","last_active":"2023-10-09T11:02:49.318280+0000","last_peered":"2023-10-09T11:02:49.318280+0000","last_clean":"2023-10-09T11:02:49.318280+0000","last_became_active":"2023-10-09T10:59:48.122447+0000","last_became_peered":"2023-10-09T10:59:48.122447+0000","last_unstale":"2023-10-09T11:02:49.318280+0000","last_undegraded":"2023-10-09T11:02:49.318280+0000","last_fullsized":"2023-10-09T11:02:49.318280+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:28.368331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":124,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.335111+0000","last_change":"2023-10-09T11:00:14.897166+0000","last_active":"2023-10-09T11:02:51.335111+0000","last_peered":"2023-10-09T11:02:51.335111+0000","last_clean":"2023-10-09T11:02:51.335111+0000","last_became_active":"2023-10-09T11:00:14.896917+0000","last_became_peered":"2023-10-09T11:00:14.896917+0000","last_unstale":"2023-10-09T11:02:51.335111+0000","last_undegraded":"2023-10-09T11:02:51.335111+0000","last_fullsized":"2023-10-09T11:02:51.335111+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:09.954705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":122,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.336281+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:02:51.336281+0000","last_peered":"2023-10-09T11:02:51.336281+0000","last_clean":"2023-10-09T11:02:51.336281+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:02:51.336281+0000","last_undegraded":"2023-10-09T11:02:51.336281+0000","last_fullsized":"2023-10-09T11:02:51.336281+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"445'6257","reported_seq":6308,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-10-09T11:02:52.975458+0000","last_change":"2023-10-09T11:01:15.405163+0000","last_active":"2023-10-09T11:02:52.975458+0000","last_peered":"2023-10-09T11:02:52.975458+0000","last_clean":"2023-10-09T11:02:52.975458+0000","last_became_active":"2023-10-09T11:01:15.404873+0000","last_became_peered":"2023-10-09T11:01:15.404873+0000","last_unstale":"2023-10-09T11:02:52.975458+0000","last_undegraded":"2023-10-09T11:02:52.975458+0000","last_fullsized":"2023-10-09T11:02:52.975458+0000","mapping_epoch":403,"log_start":"406'500","ondisk_log_start":"406'500","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5757,"log_dups_size":0,"ondisk_log_size":5757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:08.894661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399966208,"num_objects":6103,"num_object_clones":0,"num_object_copies":18309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6257,"num_write_kb":395520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":299,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332400+0000","last_change":"2023-10-09T11:02:33.198446+0000","last_active":"2023-10-09T11:02:50.332400+0000","last_peered":"2023-10-09T11:02:50.332400+0000","last_clean":"2023-10-09T11:02:50.332400+0000","last_became_active":"2023-10-09T11:02:33.197858+0000","last_became_peered":"2023-10-09T11:02:33.197858+0000","last_unstale":"2023-10-09T11:02:50.332400+0000","last_undegraded":"2023-10-09T11:02:50.332400+0000","last_fullsized":"2023-10-09T11:02:50.332400+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:17.362664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":824,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.333693+0000","last_change":"2023-10-09T11:02:36.220226+0000","last_active":"2023-10-09T11:02:50.333693+0000","last_peered":"2023-10-09T11:02:50.333693+0000","last_clean":"2023-10-09T11:02:50.333693+0000","last_became_active":"2023-10-09T11:02:36.219889+0000","last_became_peered":"2023-10-09T11:02:36.219889+0000","last_unstale":"2023-10-09T11:02:50.333693+0000","last_undegraded":"2023-10-09T11:02:50.333693+0000","last_fullsized":"2023-10-09T11:02:50.333693+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:48:54.740905+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":377,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.337656+0000","last_change":"2023-10-09T10:43:35.485969+0000","last_active":"2023-10-09T11:02:48.337656+0000","last_peered":"2023-10-09T11:02:48.337656+0000","last_clean":"2023-10-09T11:02:48.337656+0000","last_became_active":"2023-10-09T10:43:35.485027+0000","last_became_peered":"2023-10-09T10:43:35.485027+0000","last_unstale":"2023-10-09T11:02:48.337656+0000","last_undegraded":"2023-10-09T11:02:48.337656+0000","last_fullsized":"2023-10-09T11:02:48.337656+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:03:24.341027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":396,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.330514+0000","last_change":"2023-10-09T11:02:33.200249+0000","last_active":"2023-10-09T11:02:50.330514+0000","last_peered":"2023-10-09T11:02:50.330514+0000","last_clean":"2023-10-09T11:02:50.330514+0000","last_became_active":"2023-10-09T11:02:33.199802+0000","last_became_peered":"2023-10-09T11:02:33.199802+0000","last_unstale":"2023-10-09T11:02:50.330514+0000","last_undegraded":"2023-10-09T11:02:50.330514+0000","last_fullsized":"2023-10-09T11:02:50.330514+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:30.703377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"445'6256","reported_seq":8681,"reported_epoch":445,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T11:02:53.041663+0000","last_change":"2023-10-09T11:02:50.369325+0000","last_active":"2023-10-09T11:02:53.041663+0000","last_peered":"2023-10-09T11:02:53.041663+0000","last_clean":"2023-10-09T11:02:32.305733+0000","last_became_active":"2023-10-09T11:02:50.341571+0000","last_became_peered":"2023-10-09T11:02:50.341571+0000","last_unstale":"2023-10-09T11:02:53.041663+0000","last_undegraded":"2023-10-09T11:02:50.336279+0000","last_fullsized":"2023-10-09T11:02:53.041663+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6256,"log_dups_size":0,"ondisk_log_size":6256,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200998912,"num_objects":3067,"num_object_clones":0,"num_object_copies":9201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2764,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3149,"num_write_kb":198912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":605,"num_bytes_recovered":39682048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":2764},{"shards":"0,2,7","objects":303}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":780,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.335331+0000","last_change":"2023-10-09T11:01:17.369788+0000","last_active":"2023-10-09T11:02:51.335331+0000","last_peered":"2023-10-09T11:02:51.335331+0000","last_clean":"2023-10-09T11:02:51.335331+0000","last_became_active":"2023-10-09T11:00:14.881494+0000","last_became_peered":"2023-10-09T11:00:14.881494+0000","last_unstale":"2023-10-09T11:02:51.335331+0000","last_undegraded":"2023-10-09T11:02:51.335331+0000","last_fullsized":"2023-10-09T11:02:51.335331+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:44.208863+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.333981+0000","last_change":"2023-10-09T11:02:33.198484+0000","last_active":"2023-10-09T11:02:50.333981+0000","last_peered":"2023-10-09T11:02:50.333981+0000","last_clean":"2023-10-09T11:02:50.333981+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:02:50.333981+0000","last_undegraded":"2023-10-09T11:02:50.333981+0000","last_fullsized":"2023-10-09T11:02:50.333981+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":81,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.331934+0000","last_change":"2023-10-09T10:59:48.135546+0000","last_active":"2023-10-09T11:02:51.331934+0000","last_peered":"2023-10-09T11:02:51.331934+0000","last_clean":"2023-10-09T11:02:51.331934+0000","last_became_active":"2023-10-09T10:59:48.135227+0000","last_became_peered":"2023-10-09T10:59:48.135227+0000","last_unstale":"2023-10-09T11:02:51.331934+0000","last_undegraded":"2023-10-09T11:02:51.331934+0000","last_fullsized":"2023-10-09T11:02:51.331934+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:50.561785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":402,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.332592+0000","last_change":"2023-10-09T11:00:15.892896+0000","last_active":"2023-10-09T11:02:51.332592+0000","last_peered":"2023-10-09T11:02:51.332592+0000","last_clean":"2023-10-09T11:02:51.332592+0000","last_became_active":"2023-10-09T11:00:15.892573+0000","last_became_peered":"2023-10-09T11:00:15.892573+0000","last_unstale":"2023-10-09T11:02:51.332592+0000","last_undegraded":"2023-10-09T11:02:51.332592+0000","last_fullsized":"2023-10-09T11:02:51.332592+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:13.280089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"445'6196","reported_seq":6248,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-10-09T11:02:52.973454+0000","last_change":"2023-10-09T11:01:15.402557+0000","last_active":"2023-10-09T11:02:52.973454+0000","last_peered":"2023-10-09T11:02:52.973454+0000","last_clean":"2023-10-09T11:02:52.973454+0000","last_became_active":"2023-10-09T11:01:15.402279+0000","last_became_peered":"2023-10-09T11:01:15.402279+0000","last_unstale":"2023-10-09T11:02:52.973454+0000","last_undegraded":"2023-10-09T11:02:52.973454+0000","last_fullsized":"2023-10-09T11:02:52.973454+0000","mapping_epoch":403,"log_start":"406'363","ondisk_log_start":"406'363","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5833,"log_dups_size":0,"ondisk_log_size":5833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:19.333267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397869080,"num_objects":6072,"num_object_clones":0,"num_object_copies":18216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6072,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6196,"num_write_kb":392513,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":658,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.332477+0000","last_change":"2023-10-09T11:01:16.782558+0000","last_active":"2023-10-09T11:02:51.332477+0000","last_peered":"2023-10-09T11:02:51.332477+0000","last_clean":"2023-10-09T11:02:51.332477+0000","last_became_active":"2023-10-09T10:59:51.137287+0000","last_became_peered":"2023-10-09T10:59:51.137287+0000","last_unstale":"2023-10-09T11:02:51.332477+0000","last_undegraded":"2023-10-09T11:02:51.332477+0000","last_fullsized":"2023-10-09T11:02:51.332477+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:58:29.405479+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.334007+0000","last_change":"2023-10-09T11:02:33.199031+0000","last_active":"2023-10-09T11:02:50.334007+0000","last_peered":"2023-10-09T11:02:50.334007+0000","last_clean":"2023-10-09T11:02:50.334007+0000","last_became_active":"2023-10-09T11:02:33.198251+0000","last_became_peered":"2023-10-09T11:02:33.198251+0000","last_unstale":"2023-10-09T11:02:50.334007+0000","last_undegraded":"2023-10-09T11:02:50.334007+0000","last_fullsized":"2023-10-09T11:02:50.334007+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:21.904005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":124,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.336037+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:02:51.336037+0000","last_peered":"2023-10-09T11:02:51.336037+0000","last_clean":"2023-10-09T11:02:51.336037+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:02:51.336037+0000","last_undegraded":"2023-10-09T11:02:51.336037+0000","last_fullsized":"2023-10-09T11:02:51.336037+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":387,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.332184+0000","last_change":"2023-10-09T11:02:33.198297+0000","last_active":"2023-10-09T11:02:51.332184+0000","last_peered":"2023-10-09T11:02:51.332184+0000","last_clean":"2023-10-09T11:02:51.332184+0000","last_became_active":"2023-10-09T11:02:33.197978+0000","last_became_peered":"2023-10-09T11:02:33.197978+0000","last_unstale":"2023-10-09T11:02:51.332184+0000","last_undegraded":"2023-10-09T11:02:51.332184+0000","last_fullsized":"2023-10-09T11:02:51.332184+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:41.689146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"440'6143","reported_seq":6192,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.990985+0000","last_change":"2023-10-09T11:01:15.406858+0000","last_active":"2023-10-09T11:02:50.990985+0000","last_peered":"2023-10-09T11:02:50.990985+0000","last_clean":"2023-10-09T11:02:50.990985+0000","last_became_active":"2023-10-09T11:01:15.406513+0000","last_became_peered":"2023-10-09T11:01:15.406513+0000","last_unstale":"2023-10-09T11:02:50.990985+0000","last_undegraded":"2023-10-09T11:02:50.990985+0000","last_fullsized":"2023-10-09T11:02:50.990985+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6143,"log_dups_size":0,"ondisk_log_size":6143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:28:25.137720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402587648,"num_objects":6143,"num_object_clones":0,"num_object_copies":18429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6143,"num_write_kb":393152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":839,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.330425+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T11:02:50.330425+0000","last_peered":"2023-10-09T11:02:50.330425+0000","last_clean":"2023-10-09T11:02:50.330425+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T11:02:50.330425+0000","last_undegraded":"2023-10-09T11:02:50.330425+0000","last_fullsized":"2023-10-09T11:02:50.330425+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":545,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.373820+0000","last_change":"2023-10-09T11:02:33.206628+0000","last_active":"2023-10-09T11:02:48.373820+0000","last_peered":"2023-10-09T11:02:48.373820+0000","last_clean":"2023-10-09T11:02:48.373820+0000","last_became_active":"2023-10-09T11:02:33.200124+0000","last_became_peered":"2023-10-09T11:02:33.200124+0000","last_unstale":"2023-10-09T11:02:48.373820+0000","last_undegraded":"2023-10-09T11:02:48.373820+0000","last_fullsized":"2023-10-09T11:02:48.373820+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:42:19.368427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":124,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.336281+0000","last_change":"2023-10-09T11:00:14.900305+0000","last_active":"2023-10-09T11:02:51.336281+0000","last_peered":"2023-10-09T11:02:51.336281+0000","last_clean":"2023-10-09T11:02:51.336281+0000","last_became_active":"2023-10-09T11:00:14.899724+0000","last_became_peered":"2023-10-09T11:00:14.899724+0000","last_unstale":"2023-10-09T11:02:51.336281+0000","last_undegraded":"2023-10-09T11:02:51.336281+0000","last_fullsized":"2023-10-09T11:02:51.336281+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:59.022161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":522,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.337431+0000","last_change":"2023-10-09T11:02:33.198248+0000","last_active":"2023-10-09T11:02:48.337431+0000","last_peered":"2023-10-09T11:02:48.337431+0000","last_clean":"2023-10-09T11:02:48.337431+0000","last_became_active":"2023-10-09T11:02:33.197876+0000","last_became_peered":"2023-10-09T11:02:33.197876+0000","last_unstale":"2023-10-09T11:02:48.337431+0000","last_undegraded":"2023-10-09T11:02:48.337431+0000","last_fullsized":"2023-10-09T11:02:48.337431+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:17.182284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"441'6202","reported_seq":6251,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.991240+0000","last_change":"2023-10-09T11:01:15.397123+0000","last_active":"2023-10-09T11:02:50.991240+0000","last_peered":"2023-10-09T11:02:50.991240+0000","last_clean":"2023-10-09T11:02:50.991240+0000","last_became_active":"2023-10-09T11:01:15.396842+0000","last_became_peered":"2023-10-09T11:01:15.396842+0000","last_unstale":"2023-10-09T11:02:50.991240+0000","last_undegraded":"2023-10-09T11:02:50.991240+0000","last_fullsized":"2023-10-09T11:02:50.991240+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6202,"log_dups_size":0,"ondisk_log_size":6202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:54.895524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406454272,"num_objects":6202,"num_object_clones":0,"num_object_copies":18606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6202,"num_write_kb":396928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":904,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.361338+0000","last_change":"2023-10-09T11:02:36.227631+0000","last_active":"2023-10-09T11:02:48.361338+0000","last_peered":"2023-10-09T11:02:48.361338+0000","last_clean":"2023-10-09T11:02:48.361338+0000","last_became_active":"2023-10-09T11:02:36.219694+0000","last_became_peered":"2023-10-09T11:02:36.219694+0000","last_unstale":"2023-10-09T11:02:48.361338+0000","last_undegraded":"2023-10-09T11:02:48.361338+0000","last_fullsized":"2023-10-09T11:02:48.361338+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:50.682518+0000","scrub_duration":0.0053707060000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.335012+0000","last_change":"2023-10-09T11:02:33.198517+0000","last_active":"2023-10-09T11:02:50.335012+0000","last_peered":"2023-10-09T11:02:50.335012+0000","last_clean":"2023-10-09T11:02:50.335012+0000","last_became_active":"2023-10-09T11:02:33.197676+0000","last_became_peered":"2023-10-09T11:02:33.197676+0000","last_unstale":"2023-10-09T11:02:50.335012+0000","last_undegraded":"2023-10-09T11:02:50.335012+0000","last_fullsized":"2023-10-09T11:02:50.335012+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:40:41.647813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":386,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.332647+0000","last_change":"2023-10-09T11:00:15.869009+0000","last_active":"2023-10-09T11:02:51.332647+0000","last_peered":"2023-10-09T11:02:51.332647+0000","last_clean":"2023-10-09T11:02:51.332647+0000","last_became_active":"2023-10-09T11:00:15.868637+0000","last_became_peered":"2023-10-09T11:00:15.868637+0000","last_unstale":"2023-10-09T11:02:51.332647+0000","last_undegraded":"2023-10-09T11:02:51.332647+0000","last_fullsized":"2023-10-09T11:02:51.332647+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:37.538741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":229,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.333020+0000","last_change":"2023-10-09T11:01:15.402661+0000","last_active":"2023-10-09T11:02:50.333020+0000","last_peered":"2023-10-09T11:02:50.333020+0000","last_clean":"2023-10-09T11:02:50.333020+0000","last_became_active":"2023-10-09T11:01:15.402327+0000","last_became_peered":"2023-10-09T11:01:15.402327+0000","last_unstale":"2023-10-09T11:02:50.333020+0000","last_undegraded":"2023-10-09T11:02:50.333020+0000","last_fullsized":"2023-10-09T11:02:50.333020+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:32.513751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.12","version":"445'6180","reported_seq":6244,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-10-09T11:02:52.991416+0000","last_change":"2023-10-09T11:02:49.389995+0000","last_active":"2023-10-09T11:02:52.991416+0000","last_peered":"2023-10-09T11:02:52.991416+0000","last_clean":"2023-10-09T11:02:52.991416+0000","last_became_active":"2023-10-09T11:02:49.375825+0000","last_became_peered":"2023-10-09T11:02:49.375825+0000","last_unstale":"2023-10-09T11:02:52.991416+0000","last_undegraded":"2023-10-09T11:02:52.991416+0000","last_fullsized":"2023-10-09T11:02:52.991416+0000","mapping_epoch":441,"log_start":"406'500","ondisk_log_start":"406'500","created":441,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5680,"log_dups_size":0,"ondisk_log_size":5680,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200146944,"num_objects":3054,"num_object_clones":0,"num_object_copies":9162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3102,"num_write_kb":196992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":869,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.343343+0000","last_change":"2023-10-09T10:59:51.145348+0000","last_active":"2023-10-09T11:02:48.343343+0000","last_peered":"2023-10-09T11:02:48.343343+0000","last_clean":"2023-10-09T11:02:48.343343+0000","last_became_active":"2023-10-09T10:59:51.145100+0000","last_became_peered":"2023-10-09T10:59:51.145100+0000","last_unstale":"2023-10-09T11:02:48.343343+0000","last_undegraded":"2023-10-09T11:02:48.343343+0000","last_fullsized":"2023-10-09T11:02:48.343343+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:51.049260+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":385,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.332240+0000","last_change":"2023-10-09T11:02:33.201950+0000","last_active":"2023-10-09T11:02:51.332240+0000","last_peered":"2023-10-09T11:02:51.332240+0000","last_clean":"2023-10-09T11:02:51.332240+0000","last_became_active":"2023-10-09T11:02:33.201559+0000","last_became_peered":"2023-10-09T11:02:33.201559+0000","last_unstale":"2023-10-09T11:02:51.332240+0000","last_undegraded":"2023-10-09T11:02:51.332240+0000","last_fullsized":"2023-10-09T11:02:51.332240+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:45.851351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":199,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-10-09T11:02:49.317649+0000","last_change":"2023-10-09T11:00:14.886903+0000","last_active":"2023-10-09T11:02:49.317649+0000","last_peered":"2023-10-09T11:02:49.317649+0000","last_clean":"2023-10-09T11:02:49.317649+0000","last_became_active":"2023-10-09T11:00:14.886635+0000","last_became_peered":"2023-10-09T11:00:14.886635+0000","last_unstale":"2023-10-09T11:02:49.317649+0000","last_undegraded":"2023-10-09T11:02:49.317649+0000","last_fullsized":"2023-10-09T11:02:49.317649+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:04.982085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.15","version":"442'6236","reported_seq":6297,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.335664+0000","last_change":"2023-10-09T11:02:49.394261+0000","last_active":"2023-10-09T11:02:50.335664+0000","last_peered":"2023-10-09T11:02:50.335664+0000","last_clean":"2023-10-09T11:02:50.335664+0000","last_became_active":"2023-10-09T11:02:49.382243+0000","last_became_peered":"2023-10-09T11:02:49.382243+0000","last_unstale":"2023-10-09T11:02:50.335664+0000","last_undegraded":"2023-10-09T11:02:50.335664+0000","last_fullsized":"2023-10-09T11:02:50.335664+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6236,"log_dups_size":0,"ondisk_log_size":6236,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204374016,"num_objects":3119,"num_object_clones":0,"num_object_copies":9357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3119,"num_write_kb":199584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":835,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.337310+0000","last_change":"2023-10-09T11:02:36.219019+0000","last_active":"2023-10-09T11:02:48.337310+0000","last_peered":"2023-10-09T11:02:48.337310+0000","last_clean":"2023-10-09T11:02:48.337310+0000","last_became_active":"2023-10-09T11:02:36.218713+0000","last_became_peered":"2023-10-09T11:02:36.218713+0000","last_unstale":"2023-10-09T11:02:48.337310+0000","last_undegraded":"2023-10-09T11:02:48.337310+0000","last_fullsized":"2023-10-09T11:02:48.337310+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:49:33.500684+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":271,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332471+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:02:50.332471+0000","last_peered":"2023-10-09T11:02:50.332471+0000","last_clean":"2023-10-09T11:02:50.332471+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:02:50.332471+0000","last_undegraded":"2023-10-09T11:02:50.332471+0000","last_fullsized":"2023-10-09T11:02:50.332471+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":470,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.346256+0000","last_change":"2023-10-09T10:57:46.844827+0000","last_active":"2023-10-09T11:02:48.346256+0000","last_peered":"2023-10-09T11:02:48.346256+0000","last_clean":"2023-10-09T11:02:48.346256+0000","last_became_active":"2023-10-09T10:57:46.844584+0000","last_became_peered":"2023-10-09T10:57:46.844584+0000","last_unstale":"2023-10-09T11:02:48.346256+0000","last_undegraded":"2023-10-09T11:02:48.346256+0000","last_fullsized":"2023-10-09T11:02:48.346256+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:54.499813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.14","version":"445'6116","reported_seq":6177,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-10-09T11:02:53.005010+0000","last_change":"2023-10-09T11:02:49.391044+0000","last_active":"2023-10-09T11:02:53.005010+0000","last_peered":"2023-10-09T11:02:53.005010+0000","last_clean":"2023-10-09T11:02:53.005010+0000","last_became_active":"2023-10-09T11:02:49.384777+0000","last_became_peered":"2023-10-09T11:02:49.384777+0000","last_unstale":"2023-10-09T11:02:53.005010+0000","last_undegraded":"2023-10-09T11:02:53.005010+0000","last_fullsized":"2023-10-09T11:02:53.005010+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6116,"log_dups_size":0,"ondisk_log_size":6116,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197197824,"num_objects":3009,"num_object_clones":0,"num_object_copies":9027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3075,"num_write_kb":194688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":394,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332525+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T11:02:50.332525+0000","last_peered":"2023-10-09T11:02:50.332525+0000","last_clean":"2023-10-09T11:02:50.332525+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T11:02:50.332525+0000","last_undegraded":"2023-10-09T11:02:50.332525+0000","last_fullsized":"2023-10-09T11:02:50.332525+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":406,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.337726+0000","last_change":"2023-10-09T10:59:48.136562+0000","last_active":"2023-10-09T11:02:48.337726+0000","last_peered":"2023-10-09T11:02:48.337726+0000","last_clean":"2023-10-09T11:02:48.337726+0000","last_became_active":"2023-10-09T10:59:48.136184+0000","last_became_peered":"2023-10-09T10:59:48.136184+0000","last_unstale":"2023-10-09T11:02:48.337726+0000","last_undegraded":"2023-10-09T11:02:48.337726+0000","last_fullsized":"2023-10-09T11:02:48.337726+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:16.944093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.17","version":"442'6110","reported_seq":6168,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-10-09T11:02:49.379584+0000","last_change":"2023-10-09T11:02:49.358395+0000","last_active":"2023-10-09T11:02:49.379584+0000","last_peered":"2023-10-09T11:02:49.379584+0000","last_clean":"2023-10-09T11:02:49.379584+0000","last_became_active":"2023-10-09T11:02:49.355236+0000","last_became_peered":"2023-10-09T11:02:49.355236+0000","last_unstale":"2023-10-09T11:02:49.379584+0000","last_undegraded":"2023-10-09T11:02:49.379584+0000","last_fullsized":"2023-10-09T11:02:49.379584+0000","mapping_epoch":441,"log_start":"406'100","ondisk_log_start":"406'100","created":441,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6010,"log_dups_size":0,"ondisk_log_size":6010,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200245248,"num_objects":3056,"num_object_clones":0,"num_object_copies":9168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3056,"num_write_kb":195552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":674,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.333883+0000","last_change":"2023-10-09T11:02:36.217717+0000","last_active":"2023-10-09T11:02:50.333883+0000","last_peered":"2023-10-09T11:02:50.333883+0000","last_clean":"2023-10-09T11:02:50.333883+0000","last_became_active":"2023-10-09T11:02:36.217373+0000","last_became_peered":"2023-10-09T11:02:36.217373+0000","last_unstale":"2023-10-09T11:02:50.333883+0000","last_undegraded":"2023-10-09T11:02:50.333883+0000","last_fullsized":"2023-10-09T11:02:50.333883+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:47.307693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:47.307693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":469,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.337657+0000","last_change":"2023-10-09T10:54:17.886158+0000","last_active":"2023-10-09T11:02:48.337657+0000","last_peered":"2023-10-09T11:02:48.337657+0000","last_clean":"2023-10-09T11:02:48.337657+0000","last_became_active":"2023-10-09T10:54:17.885816+0000","last_became_peered":"2023-10-09T10:54:17.885816+0000","last_unstale":"2023-10-09T11:02:48.337657+0000","last_undegraded":"2023-10-09T11:02:48.337657+0000","last_fullsized":"2023-10-09T11:02:48.337657+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:05.270825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":489,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T11:02:48.343590+0000","last_change":"2023-10-09T10:59:48.125955+0000","last_active":"2023-10-09T11:02:48.343590+0000","last_peered":"2023-10-09T11:02:48.343590+0000","last_clean":"2023-10-09T11:02:48.343590+0000","last_became_active":"2023-10-09T10:59:48.125641+0000","last_became_peered":"2023-10-09T10:59:48.125641+0000","last_unstale":"2023-10-09T11:02:48.343590+0000","last_undegraded":"2023-10-09T11:02:48.343590+0000","last_fullsized":"2023-10-09T11:02:48.343590+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:38.041100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.16","version":"440'6264","reported_seq":6323,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.332693+0000","last_change":"2023-10-09T11:02:49.378964+0000","last_active":"2023-10-09T11:02:50.332693+0000","last_peered":"2023-10-09T11:02:50.332693+0000","last_clean":"2023-10-09T11:02:50.332693+0000","last_became_active":"2023-10-09T11:02:49.376770+0000","last_became_peered":"2023-10-09T11:02:49.376770+0000","last_unstale":"2023-10-09T11:02:50.332693+0000","last_undegraded":"2023-10-09T11:02:50.332693+0000","last_fullsized":"2023-10-09T11:02:50.332693+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6264,"log_dups_size":0,"ondisk_log_size":6264,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205258752,"num_objects":3132,"num_object_clones":0,"num_object_copies":9396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3132,"num_write_kb":200448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":702,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.336096+0000","last_change":"2023-10-09T11:02:25.273280+0000","last_active":"2023-10-09T11:02:51.336096+0000","last_peered":"2023-10-09T11:02:51.336096+0000","last_clean":"2023-10-09T11:02:51.336096+0000","last_became_active":"2023-10-09T11:00:14.867744+0000","last_became_peered":"2023-10-09T11:00:14.867744+0000","last_unstale":"2023-10-09T11:02:51.336096+0000","last_undegraded":"2023-10-09T11:02:51.336096+0000","last_fullsized":"2023-10-09T11:02:51.336096+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:30:37.506583+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":87,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-10-09T11:02:51.332797+0000","last_change":"2023-10-09T10:59:48.143732+0000","last_active":"2023-10-09T11:02:51.332797+0000","last_peered":"2023-10-09T11:02:51.332797+0000","last_clean":"2023-10-09T11:02:51.332797+0000","last_became_active":"2023-10-09T10:59:48.129001+0000","last_became_peered":"2023-10-09T10:59:48.129001+0000","last_unstale":"2023-10-09T11:02:51.332797+0000","last_undegraded":"2023-10-09T11:02:51.332797+0000","last_fullsized":"2023-10-09T11:02:51.332797+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:22:36.946348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":465,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-10-09T11:02:50.329439+0000","last_change":"2023-10-09T11:00:14.869353+0000","last_active":"2023-10-09T11:02:50.329439+0000","last_peered":"2023-10-09T11:02:50.329439+0000","last_clean":"2023-10-09T11:02:50.329439+0000","last_became_active":"2023-10-09T11:00:14.869061+0000","last_became_peered":"2023-10-09T11:00:14.869061+0000","last_unstale":"2023-10-09T11:02:50.329439+0000","last_undegraded":"2023-10-09T11:02:50.329439+0000","last_fullsized":"2023-10-09T11:02:50.329439+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:30.318094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":6427967512,"num_objects":98084,"num_object_clones":0,"num_object_copies":294252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7976,"num_objects_misplaced":14294,"num_objects_unfound":0,"num_objects_dirty":98084,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":99050,"num_write_kb":6308225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5444,"num_bytes_recovered":356777984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19395907584,"data_stored":19395903512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":158209,"ondisk_log_size":158209,"up":78,"acting":76,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":17098,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8961,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":599313,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1371411,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332435,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1072512,"kb_used_data":173112,"kb_used_omap":0,"kb_used_meta":899392,"kb_avail":92672640,"statfs":{"total":95995035648,"available":94896783360,"internally_reserved":0,"allocated":177266688,"data_stored":175852060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920977408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:11 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":1.6359999999999999,"15min":1.7729999999999999},"min":{"1min":0.40999999999999998,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":2.8210000000000002,"5min":8.9730000000000008,"15min":9.6379999999999999},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.49399999999999999,"5min":1.429,"15min":1.881},"min":{"1min":0.23499999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.95899999999999996,"5min":7.5609999999999999,"15min":16.731000000000002},"last":0.27700000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:02:15 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.5,"15min":1.609},"min":{"1min":0.26900000000000002,"5min":0.247,"15min":0.247},"max":{"1min":2.589,"5min":6.202,"15min":6.202},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.65800000000000003,"5min":1.772,"15min":1.958},"min":{"1min":0.26600000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.0379999999999998,"5min":12.193,"15min":12.193},"last":0.36699999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:02:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1899999999999999,"5min":1.6839999999999999,"15min":1.7250000000000001},"min":{"1min":0.30399999999999999,"5min":0.245,"15min":0.245},"max":{"1min":8.8970000000000002,"5min":8.8970000000000002,"15min":11.058},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.746,"5min":1.917,"15min":1.7969999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":4.9550000000000001,"5min":8.9280000000000008,"15min":13.037000000000001},"last":0.50900000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":1.6819999999999999,"15min":2.0470000000000002},"min":{"1min":0.378,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.698,"5min":13.464,"15min":13.464},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.734,"15min":1.8440000000000001},"min":{"1min":0.371,"5min":0.31,"15min":0.31},"max":{"1min":5.0919999999999996,"5min":9.0210000000000008,"15min":19.234999999999999},"last":0.47799999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":1.266,"15min":1.4370000000000001},"min":{"1min":0.33300000000000002,"5min":0.248,"15min":0.24099999999999999},"max":{"1min":0.67600000000000005,"5min":7.96,"15min":16.695},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.752,"5min":1.2330000000000001,"15min":1.337},"min":{"1min":0.315,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.6110000000000002,"5min":7.0510000000000002,"15min":11.109999999999999},"last":0.69499999999999995}]},{"osd":5,"last update":"Mon Oct 9 11:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":1.6819999999999999,"15min":1.5429999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":2.9260000000000002,"5min":13.407,"15min":13.407},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.1910000000000001,"15min":1.4970000000000001},"min":{"1min":0.30499999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.9510000000000001,"5min":7.9180000000000001,"15min":11.741},"last":0.435}]},{"osd":6,"last update":"Mon Oct 9 11:01:51 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.321,"15min":1.6439999999999999},"min":{"1min":0.38400000000000001,"5min":0.34599999999999997,"15min":0.311},"max":{"1min":2.27,"5min":10.388999999999999,"15min":14.913},"last":0.495},{"interface":"front","average":{"1min":0.73899999999999999,"5min":1.1990000000000001,"15min":1.4990000000000001},"min":{"1min":0.33900000000000002,"5min":0.32600000000000001,"15min":0.29399999999999998},"max":{"1min":2.5710000000000002,"5min":7.1139999999999999,"15min":19.187999999999999},"last":0.60099999999999998}]}]},{"osd":6,"up_from":44,"seq":188978561366,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3646320,"kb_used_data":2754344,"kb_used_omap":0,"kb_used_meta":891968,"kb_avail":90098832,"statfs":{"total":95995035648,"available":92261203968,"internally_reserved":0,"allocated":2820448256,"data_stored":2819049322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913375232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:07 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":1.8009999999999999,"15min":1.877},"min":{"1min":0.224,"5min":0.224,"15min":0.216},"max":{"1min":3.9460000000000002,"5min":46.603999999999999,"15min":54.743000000000002},"last":0.434},{"interface":"front","average":{"1min":0.60699999999999998,"5min":2.0760000000000001,"15min":1.9790000000000001},"min":{"1min":0.28999999999999998,"5min":0.223,"15min":0.223},"max":{"1min":1.9039999999999999,"5min":46.679000000000002,"15min":53.701000000000001},"last":0.501}]},{"osd":1,"last update":"Mon Oct 9 11:02:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0529999999999999,"5min":1.2130000000000001,"15min":0.90700000000000003},"min":{"1min":0.29399999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":41.241999999999997,"5min":41.241999999999997,"15min":41.241999999999997},"last":0.47199999999999998},{"interface":"front","average":{"1min":3.5070000000000001,"5min":1.792,"15min":1.506},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":41.476999999999997,"5min":41.476999999999997,"15min":41.476999999999997},"last":0.38200000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.738,"15min":1.8260000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28399999999999997},"max":{"1min":3.5470000000000002,"5min":8.6869999999999994,"15min":13.208},"last":0.52400000000000002},{"interface":"front","average":{"1min":2.6629999999999998,"5min":2.3889999999999998,"15min":2.0670000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.0839999999999996,"5min":15.432,"15min":15.432},"last":0.72399999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:02:07 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.6719999999999999,"15min":1.8320000000000001},"min":{"1min":0.35499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":3.266,"5min":10.018000000000001,"15min":12.391},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.101,"5min":1.885,"15min":2.0870000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":3.6389999999999998,"5min":11.161,"15min":13.179},"last":0.45200000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.218,"5min":1.0649999999999999,"15min":1.304},"min":{"1min":0.29799999999999999,"5min":0.25800000000000001,"15min":0.23200000000000001},"max":{"1min":3.9630000000000001,"5min":9.0060000000000002,"15min":16.978999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.92900000000000005,"15min":1.2310000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.24399999999999999},"max":{"1min":4.8049999999999997,"5min":5.7869999999999999,"15min":12.481999999999999},"last":0.54200000000000004}]},{"osd":5,"last update":"Mon Oct 9 11:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.458,"15min":1.5640000000000001},"min":{"1min":0.33700000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.2290000000000001,"5min":19.786999999999999,"15min":19.786999999999999},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.0820000000000001,"15min":1.337},"min":{"1min":0.32500000000000001,"5min":0.28899999999999998,"15min":0.26600000000000001},"max":{"1min":4.8810000000000002,"5min":8.6020000000000003,"15min":12.456},"last":0.56000000000000005}]},{"osd":7,"last update":"Mon Oct 9 11:01:59 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.2989999999999999,"15min":1.5549999999999999},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.26600000000000001},"max":{"1min":1.885,"5min":9.1120000000000001,"15min":23.358000000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.96699999999999997,"15min":1.216},"min":{"1min":0.32400000000000001,"5min":0.32300000000000001,"15min":0.27500000000000002},"max":{"1min":0.83199999999999996,"5min":6.1920000000000002,"15min":12.422000000000001},"last":0.58099999999999996}]}]},{"osd":1,"up_from":388,"seq":1666447310881,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2006872,"kb_used_data":1774928,"kb_used_omap":13,"kb_used_meta":231922,"kb_avail":91738280,"statfs":{"total":95995035648,"available":93939998720,"internally_reserved":0,"allocated":1817526272,"data_stored":1815231051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":237488839},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":3.2690000000000001,"5min":2.46,"15min":2.3250000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":13.334,"5min":13.334,"15min":13.334},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.968,"5min":1.6419999999999999,"15min":1.421},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.6829999999999998,"5min":9.6829999999999998,"15min":9.6829999999999998},"last":0.35999999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":2.9790000000000001,"15min":3.1389999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.4670000000000005,"5min":12.009,"15min":12.009},"last":0.5},{"interface":"front","average":{"1min":2.5499999999999998,"5min":2.2959999999999998,"15min":2.254},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.8249999999999993,"5min":9.8249999999999993,"15min":9.8249999999999993},"last":0.47499999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7879999999999998,"5min":2.8199999999999998,"15min":2.8250000000000002},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":9.6460000000000008,"5min":12.523999999999999,"15min":12.523999999999999},"last":0.317},{"interface":"front","average":{"1min":1.518,"5min":3.5539999999999998,"15min":3.8940000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.3799999999999999,"5min":18.079999999999998,"15min":18.079999999999998},"last":0.30099999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":2.536,"15min":2.754},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.2300000000000004,"5min":10.481,"15min":10.481},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.702,"5min":1.448,"15min":1.4059999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":9.8480000000000008,"5min":9.8480000000000008,"15min":9.8480000000000008},"last":0.41099999999999998}]},{"osd":5,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":2.29,"15min":2.4580000000000002},"min":{"1min":0.35599999999999998,"5min":0.311,"15min":0.311},"max":{"1min":4.8799999999999999,"5min":18.053000000000001,"15min":18.053000000000001},"last":0.372},{"interface":"front","average":{"1min":1.526,"5min":2.2429999999999999,"15min":2.363},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.2830000000000004,"5min":10.558,"15min":10.558},"last":0.39300000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":2.2759999999999998,"15min":2.46},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.8520000000000003,"5min":18.085999999999999,"15min":18.085999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.3049999999999999,"5min":2.1869999999999998,"15min":2.3340000000000001},"min":{"1min":0.38600000000000001,"5min":0.373,"15min":0.373},"max":{"1min":5.3129999999999997,"5min":10.531000000000001,"15min":10.531000000000001},"last":0.45900000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.393,"15min":1.3220000000000001},"min":{"1min":0.41899999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.8000000000000007,"5min":9.8000000000000007,"15min":9.8000000000000007},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.1910000000000001,"5min":2.3279999999999998,"15min":2.5179999999999998},"min":{"1min":0.435,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.8019999999999996,"5min":18.071000000000002,"15min":18.071000000000002},"last":0.48399999999999999}]}]},{"osd":0,"up_from":80,"seq":343597383939,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4642580,"kb_used_data":3924812,"kb_used_omap":28,"kb_used_meta":717667,"kb_avail":89102572,"statfs":{"total":95995035648,"available":91241033728,"internally_reserved":0,"allocated":4019007488,"data_stored":4016656154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":734891856},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:02:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":3.0379999999999998,"15min":3.2040000000000002},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":9.6460000000000008,"5min":19.608000000000001,"15min":19.608000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.879,"5min":3.0710000000000002,"15min":3.2690000000000001},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.218},"max":{"1min":7.8730000000000002,"5min":19.992999999999999,"15min":19.992999999999999},"last":0.32200000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:02:49 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":1.9910000000000001,"15min":2.2130000000000001},"min":{"1min":0.23899999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":12.035,"5min":12.638,"15min":24.852},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.3370000000000002,"5min":2.363,"15min":2.661},"min":{"1min":0.255,"5min":0.20899999999999999,"15min":0.19800000000000001},"max":{"1min":6.7030000000000003,"5min":12.494999999999999,"15min":77.256},"last":0.255}]},{"osd":3,"last update":"Mon Oct 9 11:02:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":2.1589999999999998,"15min":2.3090000000000002},"min":{"1min":0.26800000000000002,"5min":0.215,"15min":0.20200000000000001},"max":{"1min":8.8699999999999992,"5min":28.946000000000002,"15min":28.946000000000002},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.46,"5min":2.4350000000000001,"15min":2.5790000000000002},"min":{"1min":0.309,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.9850000000000003,"5min":19.952999999999999,"15min":19.952999999999999},"last":0.26800000000000002}]},{"osd":4,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":2.2679999999999998,"15min":2.4649999999999999},"min":{"1min":0.32900000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.9560000000000004,"5min":11.507,"15min":77.177999999999997},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.405,"5min":1.6499999999999999,"15min":2.0190000000000001},"min":{"1min":0.38400000000000001,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":6.4450000000000003,"5min":9.9489999999999998,"15min":12.318},"last":0.41399999999999998}]},{"osd":5,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.827,"15min":2.077},"min":{"1min":0.36599999999999999,"5min":0.308,"15min":0.29799999999999999},"max":{"1min":6.4880000000000004,"5min":9.8710000000000004,"15min":12.157},"last":0.378},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.9350000000000001,"15min":2.0750000000000002},"min":{"1min":0.436,"5min":0.29999999999999999,"15min":0.29899999999999999},"max":{"1min":2.859,"5min":7.0919999999999996,"15min":24.754999999999999},"last":0.36799999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5700000000000001,"5min":1.855,"15min":2.2349999999999999},"min":{"1min":0.38800000000000001,"5min":0.314,"15min":0.30399999999999999},"max":{"1min":6.5179999999999998,"5min":10.012,"15min":18.920000000000002},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.9490000000000001,"15min":1.9810000000000001},"min":{"1min":0.45700000000000002,"5min":0.31900000000000001,"15min":0.307},"max":{"1min":2.9209999999999998,"5min":8.8249999999999993,"15min":24.800000000000001},"last":1.355}]},{"osd":7,"last update":"Mon Oct 9 11:02:08 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.958,"15min":2.177},"min":{"1min":0.44900000000000001,"5min":0.34499999999999997,"15min":0.32700000000000001},"max":{"1min":2.8929999999999998,"5min":11.285,"15min":14.48},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.3740000000000001,"5min":2.3809999999999998,"15min":2.5910000000000002},"min":{"1min":0.41499999999999998,"5min":0.372,"15min":0.314},"max":{"1min":6.008,"5min":10.398,"15min":77.228999999999999},"last":1.369}]}]},{"osd":2,"up_from":90,"seq":386547056879,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2915724,"kb_used_data":2377412,"kb_used_omap":81,"kb_used_meta":538222,"kb_avail":90829428,"statfs":{"total":95995035648,"available":93009334272,"internally_reserved":0,"allocated":2434469888,"data_stored":2432289230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":551139546},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:49 2023","interfaces":[{"interface":"back","average":{"1min":4.9080000000000004,"5min":2.0880000000000001,"15min":1.917},"min":{"1min":0.19800000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":48.933999999999997,"5min":48.933999999999997,"15min":48.933999999999997},"last":0.37},{"interface":"front","average":{"1min":1.677,"5min":1.6919999999999999,"15min":1.8009999999999999},"min":{"1min":0.24399999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":14.515000000000001,"5min":14.515000000000001,"15min":15.952999999999999},"last":0.32900000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:02:18 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":2.403,"15min":2.4830000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":10.502000000000001,"5min":10.502000000000001,"15min":10.502000000000001},"last":0.26600000000000001},{"interface":"front","average":{"1min":1.6200000000000001,"5min":2.448,"15min":2.5859999999999999},"min":{"1min":0.27300000000000002,"5min":0.214,"15min":0.214},"max":{"1min":5.125,"5min":11.92,"15min":11.92},"last":0.42499999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:02:09 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":1.6279999999999999,"15min":1.863},"min":{"1min":0.24399999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.5979999999999999,"5min":11.151,"15min":19.539000000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":1.01,"5min":1.913,"15min":2.093},"min":{"1min":0.27200000000000002,"5min":0.219,"15min":0.219},"max":{"1min":3.524,"5min":16.324000000000002,"15min":16.702999999999999},"last":0.35699999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:02:49 2023","interfaces":[{"interface":"back","average":{"1min":1.292,"5min":1.8819999999999999,"15min":1.9710000000000001},"min":{"1min":0.313,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.4640000000000004,"5min":9.3290000000000006,"15min":16.231999999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":4.2069999999999999,"5min":2.0680000000000001,"15min":1.9710000000000001},"min":{"1min":0.35099999999999998,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":48.795000000000002,"5min":48.795000000000002,"15min":48.795000000000002},"last":0.505}]},{"osd":5,"last update":"Mon Oct 9 11:02:49 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.6299999999999999,"15min":1.663},"min":{"1min":0.32600000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.0679999999999996,"5min":8.5380000000000003,"15min":17.920000000000002},"last":0.41299999999999998},{"interface":"front","average":{"1min":3.504,"5min":2.2589999999999999,"15min":2.036},"min":{"1min":0.38200000000000001,"5min":0.311,"15min":0.30299999999999999},"max":{"1min":49.061,"5min":49.061,"15min":49.061},"last":0.38200000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:02:49 2023","interfaces":[{"interface":"back","average":{"1min":4.6379999999999999,"5min":2.0089999999999999,"15min":1.78},"min":{"1min":0.318,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":48.845999999999997,"5min":48.845999999999997,"15min":48.845999999999997},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.504,"15min":1.587},"min":{"1min":0.378,"5min":0.313,"15min":0.313},"max":{"1min":7.0570000000000004,"5min":7.0570000000000004,"15min":10.111000000000001},"last":0.443}]},{"osd":7,"last update":"Mon Oct 9 11:02:49 2023","interfaces":[{"interface":"back","average":{"1min":4.3250000000000002,"5min":2.23,"15min":1.8220000000000001},"min":{"1min":0.41199999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":48.966000000000001,"5min":48.966000000000001,"15min":48.966000000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.6379999999999999,"5min":1.8080000000000001,"15min":1.774},"min":{"1min":0.45300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":14.94,"5min":14.94,"15min":14.94},"last":0.48399999999999999}]}]},{"osd":3,"up_from":95,"seq":408021893353,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3376596,"kb_used_data":2368140,"kb_used_omap":7,"kb_used_meta":1008440,"kb_avail":90368556,"statfs":{"total":95995035648,"available":92537401344,"internally_reserved":0,"allocated":2424975360,"data_stored":2422601460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1032643176},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":2.556,"15min":2.6800000000000002},"min":{"1min":0.222,"5min":0.20499999999999999,"15min":0.17499999999999999},"max":{"1min":8.1709999999999994,"5min":10.140000000000001,"15min":38.817999999999998},"last":0.65800000000000003},{"interface":"front","average":{"1min":2.4769999999999999,"5min":2.3530000000000002,"15min":2.9649999999999999},"min":{"1min":0.246,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":8.4239999999999995,"5min":17.042000000000002,"15min":17.312000000000001},"last":1.0629999999999999}]},{"osd":1,"last update":"Mon Oct 9 11:02:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8,"5min":2.9199999999999999,"15min":3.1059999999999999},"min":{"1min":0.25,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":8.8030000000000008,"5min":8.9100000000000001,"15min":8.9100000000000001},"last":0.621},{"interface":"front","average":{"1min":1.4650000000000001,"5min":2.5840000000000001,"15min":2.77},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.2300000000000004,"5min":9.7449999999999992,"15min":9.7449999999999992},"last":1.0960000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":2.3100000000000001,"5min":2.3069999999999999,"15min":2.448},"min":{"1min":0.21099999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":5.5350000000000001,"5min":10.65,"15min":20.402000000000001},"last":4.5270000000000001},{"interface":"front","average":{"1min":2.0489999999999999,"5min":2.2839999999999998,"15min":2.8260000000000001},"min":{"1min":0.23899999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":8.2729999999999997,"5min":10.012,"15min":27.538},"last":3.984}]},{"osd":4,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.355,"5min":2.101,"15min":2.2770000000000001},"min":{"1min":0.30299999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.3639999999999999,"5min":9.8089999999999993,"15min":20.388000000000002},"last":0.71799999999999997},{"interface":"front","average":{"1min":1.3029999999999999,"5min":1.7929999999999999,"15min":2.157},"min":{"1min":0.34499999999999997,"5min":0.34000000000000002,"15min":0.24299999999999999},"max":{"1min":3.5910000000000002,"5min":9.1310000000000002,"15min":19.085999999999999},"last":1.1739999999999999}]},{"osd":5,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.948,"15min":2.2890000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.26000000000000001},"max":{"1min":3.5099999999999998,"5min":10.380000000000001,"15min":14.045},"last":1.3080000000000001},{"interface":"front","average":{"1min":1.4630000000000001,"5min":1.7470000000000001,"15min":2.0369999999999999},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2839999999999998,"5min":8.3949999999999996,"15min":17.311},"last":4.5060000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":2.1419999999999999,"15min":2.27},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28000000000000003},"max":{"1min":8.3119999999999994,"5min":10.420999999999999,"15min":12.361000000000001},"last":1.141},{"interface":"front","average":{"1min":1.835,"5min":1.9430000000000001,"15min":2.2090000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.8959999999999999,"5min":7.1230000000000002,"15min":17.295000000000002},"last":1.1930000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":1.9419999999999999,"15min":2.2440000000000002},"min":{"1min":0.46400000000000002,"5min":0.309,"15min":0.309},"max":{"1min":7.492,"5min":8.6649999999999991,"15min":18.722000000000001},"last":4.4740000000000002},{"interface":"front","average":{"1min":1.6180000000000001,"5min":2.194,"15min":2.4670000000000001},"min":{"1min":0.439,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.4020000000000001,"5min":9.8569999999999993,"15min":15.212999999999999},"last":0.69099999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019229,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4403964,"kb_used_data":3503028,"kb_used_omap":0,"kb_used_meta":900928,"kb_avail":89341188,"statfs":{"total":95995035648,"available":91485376512,"internally_reserved":0,"allocated":3587100672,"data_stored":3585689849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":922550272},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.411,"15min":2.3370000000000002},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":2.9870000000000001,"5min":8.6180000000000003,"15min":213.88999999999999},"last":7.9749999999999996},{"interface":"front","average":{"1min":1.393,"5min":1.359,"15min":2.6019999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.217},"max":{"1min":6.1269999999999998,"5min":6.1269999999999998,"15min":214.09800000000001},"last":1.198}]},{"osd":1,"last update":"Mon Oct 9 11:02:15 2023","interfaces":[{"interface":"back","average":{"1min":8.0950000000000006,"5min":3.2919999999999998,"15min":2.492},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":121.41,"5min":121.41,"15min":121.41},"last":0.315},{"interface":"front","average":{"1min":7.8860000000000001,"5min":2.6699999999999999,"15min":1.8009999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":122.21899999999999,"5min":122.21899999999999,"15min":122.21899999999999},"last":0.83499999999999996}]},{"osd":2,"last update":"Mon Oct 9 11:01:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":1.4550000000000001,"15min":1.7809999999999999},"min":{"1min":0.27900000000000003,"5min":0.27100000000000002,"15min":0.253},"max":{"1min":3.9390000000000001,"5min":5.6870000000000003,"15min":15.446999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.492,"5min":1.9299999999999999,"15min":2.1739999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.9540000000000002,"5min":9.9480000000000004,"15min":27.818000000000001},"last":0.46100000000000002}]},{"osd":3,"last update":"Mon Oct 9 11:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":1.7509999999999999,"15min":2.6469999999999998},"min":{"1min":0.46999999999999997,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":2.254,"5min":12.798999999999999,"15min":209.03700000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.878,"5min":1.7749999999999999,"15min":2.899},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29999999999999999},"max":{"1min":1.8480000000000001,"5min":8.6010000000000009,"15min":208.80199999999999},"last":0.58099999999999996}]},{"osd":5,"last update":"Mon Oct 9 11:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":1.2909999999999999,"15min":1.7669999999999999},"min":{"1min":0.307,"5min":0.245,"15min":0.23799999999999999},"max":{"1min":2.024,"5min":9.7880000000000003,"15min":24.149999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.78800000000000003,"5min":1.2190000000000001,"15min":1.556},"min":{"1min":0.36899999999999999,"5min":0.25800000000000001,"15min":0.22600000000000001},"max":{"1min":3.1859999999999999,"5min":12.459,"15min":20.422000000000001},"last":0.434}]},{"osd":6,"last update":"Mon Oct 9 11:02:45 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.1899999999999999,"15min":1.387},"min":{"1min":0.39900000000000002,"5min":0.27800000000000002,"15min":0.24099999999999999},"max":{"1min":7.1520000000000001,"5min":8.5739999999999998,"15min":15.739000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.1139999999999999,"5min":1.7749999999999999,"15min":1.819},"min":{"1min":0.32900000000000001,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":7.8490000000000002,"5min":9.0489999999999995,"15min":14.272},"last":0.47299999999999998}]},{"osd":7,"last update":"Mon Oct 9 11:01:50 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":1.7170000000000001,"15min":1.72},"min":{"1min":0.33200000000000002,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":4.9429999999999996,"5min":19.923999999999999,"15min":19.923999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.677,"5min":1.5489999999999999,"15min":1.585},"min":{"1min":0.34599999999999997,"5min":0.28799999999999998,"15min":0.255},"max":{"1min":8.6509999999999998,"5min":19.859999999999999,"15min":19.859999999999999},"last":0.49099999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790298,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3233004,"kb_used_data":2326180,"kb_used_omap":0,"kb_used_meta":906816,"kb_avail":90512148,"statfs":{"total":95995035648,"available":92684439552,"internally_reserved":0,"allocated":2382008320,"data_stored":2380609846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928579584},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:09 2023","interfaces":[{"interface":"back","average":{"1min":4.931,"5min":2.4180000000000001,"15min":2.1890000000000001},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.215},"max":{"1min":58.271999999999998,"5min":58.271999999999998,"15min":58.271999999999998},"last":0.47399999999999998},{"interface":"front","average":{"1min":5,"5min":2.4689999999999999,"15min":2.2429999999999999},"min":{"1min":0.32600000000000001,"5min":0.30499999999999999,"15min":0.28599999999999998},"max":{"1min":58.25,"5min":58.25,"15min":58.25},"last":0.42699999999999999}]},{"osd":1,"last update":"Mon Oct 9 11:02:22 2023","interfaces":[{"interface":"back","average":{"1min":2.081,"5min":2.0070000000000001,"15min":1.9950000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":7.0419999999999998,"5min":9.4570000000000007,"15min":9.4570000000000007},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.966,"5min":1.6819999999999999,"15min":1.635},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.0599999999999996,"5min":9.4450000000000003,"15min":9.4450000000000003},"last":0.52600000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:02:40 2023","interfaces":[{"interface":"back","average":{"1min":2.456,"5min":1.899,"15min":2.1230000000000002},"min":{"1min":0.24199999999999999,"5min":0.221,"15min":0.221},"max":{"1min":6.96,"5min":9.3539999999999992,"15min":11.393000000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.7130000000000001,"5min":1.8220000000000001,"15min":1.9510000000000001},"min":{"1min":0.28100000000000003,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":4.8419999999999996,"5min":14.621,"15min":15.087999999999999},"last":0.54500000000000004}]},{"osd":3,"last update":"Mon Oct 9 11:02:09 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":2.5449999999999999,"15min":2.5209999999999999},"min":{"1min":0.29699999999999999,"5min":0.28000000000000003,"15min":0.25700000000000001},"max":{"1min":11.590999999999999,"5min":32.810000000000002,"15min":63.508000000000003},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.8069999999999999,"5min":2.5699999999999998,"15min":2.702},"min":{"1min":0.33900000000000002,"5min":0.30199999999999999,"15min":0.26300000000000001},"max":{"1min":11.292999999999999,"5min":32.659999999999997,"15min":62.418999999999997},"last":0.48799999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:02:22 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.82599999999999996,"15min":1.4910000000000001},"min":{"1min":0.23799999999999999,"5min":0.216,"15min":0.216},"max":{"1min":7.8449999999999998,"5min":7.8449999999999998,"15min":123.241},"last":0.375},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.298,"15min":1.8280000000000001},"min":{"1min":0.26400000000000001,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":3.0910000000000002,"5min":9.1869999999999994,"15min":123.318},"last":0.44500000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:02:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":1.212,"15min":1.52},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.1589999999999998,"5min":8.4260000000000002,"15min":11.098000000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.004,"15min":1.3520000000000001},"min":{"1min":0.34999999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.0800000000000001,"5min":5.899,"15min":20.256},"last":0.50600000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:01:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.323,"15min":1.534},"min":{"1min":0.35599999999999998,"5min":0.29299999999999998,"15min":0.28699999999999998},"max":{"1min":3.4870000000000001,"5min":7.6680000000000001,"15min":10.739000000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":1.1759999999999999,"15min":1.2989999999999999},"min":{"1min":0.379,"5min":0.315,"15min":0.30099999999999999},"max":{"1min":1.7,"5min":8.1430000000000007,"15min":14.154},"last":0.56200000000000006}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4001894400,"data_stored":4001890328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1597440000,"data_stored":1597440000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2428502016,"data_stored":2428502016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2418737152,"data_stored":2418737152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3582394368,"data_stored":3582394368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2376925184,"data_stored":2376925184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2815295488,"data_stored":2815295488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":174718976,"data_stored":174718976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:02:57.784 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 26 2023-10-09T11:02:57.877 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:57.935 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:57.977 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:57.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:57.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:58.088 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:02:58.153 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:02:58.160 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:58.190 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:02:58.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:58.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:02:58.246 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:02:58.246 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:02:58.295 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:02:58.361 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:58.423 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:58 smithi125 ceph-mon[195507]: 10.12 scrub starts 2023-10-09T11:02:58.423 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:58 smithi125 ceph-mon[195507]: 10.7 scrub starts 2023-10-09T11:02:58.424 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:58 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1674749495' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:58.424 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:58 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:58.424 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:58 smithi125 ceph-mon[195507]: pgmap v1098: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 138 B/s rd, 0 op/s; 7976/287370 objects degraded (2.776%); 13850/287370 objects misplaced (4.820%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T11:02:58.424 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:58 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 7976/294324 objects degraded (2.710%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:02:58.438 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:02:58.636 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:58.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:58 smithi155 ceph-mon[184854]: 10.12 scrub starts 2023-10-09T11:02:58.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:58 smithi155 ceph-mon[184854]: 10.7 scrub starts 2023-10-09T11:02:58.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:58 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1674749495' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:58.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:58 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:58.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:58 smithi155 ceph-mon[184854]: pgmap v1098: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 138 B/s rd, 0 op/s; 7976/287370 objects degraded (2.776%); 13850/287370 objects misplaced (4.820%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T11:02:58.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:58 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 7976/294324 objects degraded (2.710%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:02:58.740 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:58 smithi125 ceph-mon[200197]: 10.12 scrub starts 2023-10-09T11:02:58.741 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:58 smithi125 ceph-mon[200197]: 10.7 scrub starts 2023-10-09T11:02:58.741 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:58 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1674749495' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:58.741 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:58 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:02:58.741 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:58 smithi125 ceph-mon[200197]: pgmap v1098: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 138 B/s rd, 0 op/s; 7976/287370 objects degraded (2.776%); 13850/287370 objects misplaced (4.820%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T11:02:58.741 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:58 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 7976/294324 objects degraded (2.710%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:02:58.864 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:02:58.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:58.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:02:58.873 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T11:02:58.970 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:02:59.087 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:59.222 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:02:59.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:59 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:02:59.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:59 smithi125 ceph-mon[195507]: failed to encode map e446 with expected crc 2023-10-09T11:02:59.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:59 smithi125 ceph-mon[195507]: failed to encode map e446 with expected crc 2023-10-09T11:02:59.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:59 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:02:59.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:02:59 smithi125 ceph-mon[195507]: osdmap e446: 8 total, 8 up, 8 in 2023-10-09T11:02:59.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:59 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:02:59.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:59 smithi125 ceph-mon[200197]: failed to encode map e446 with expected crc 2023-10-09T11:02:59.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:59 smithi125 ceph-mon[200197]: failed to encode map e446 with expected crc 2023-10-09T11:02:59.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:59 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:02:59.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:02:59 smithi125 ceph-mon[200197]: osdmap e446: 8 total, 8 up, 8 in 2023-10-09T11:02:59.592 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:02:59.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:59.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:02:59.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:59.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:59 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:02:59.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:59 smithi155 ceph-mon[184854]: failed to encode map e446 with expected crc 2023-10-09T11:02:59.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:59 smithi155 ceph-mon[184854]: failed to encode map e446 with expected crc 2023-10-09T11:02:59.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:59 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:02:59.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:02:59 smithi155 ceph-mon[184854]: osdmap e446: 8 total, 8 up, 8 in 2023-10-09T11:02:59.741 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:59.794 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:59.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:59.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:02:59.898 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:02:59.944 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:02:59.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:02:59.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:02:59.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:00.084 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:00.135 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:00.172 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:00.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:00.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:00.277 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:00.322 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:00.361 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:00.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:00.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:00.372 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:00.467 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:00.549 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:00.838 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:00.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:00.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:00.942 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:00.997 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:03:01.005 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:01.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:01.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:01.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:01.106 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:03:01.145 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:01.148 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:03:01.210 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:03:01.216 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:01.254 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:01.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:01.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:01.261 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:03:01.263 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:01 smithi125 ceph-mon[195507]: pgmap v1100: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 7685/281442 objects degraded (2.731%); 13296/281442 objects misplaced (4.724%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T11:03:01.264 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:01.264 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:03:01.264 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/620068533' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T11:03:01.264 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1906167465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:01.264 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:03:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:03:01] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-10-09T11:03:01.264 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:01 smithi125 ceph-mon[200197]: pgmap v1100: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 7685/281442 objects degraded (2.731%); 13296/281442 objects misplaced (4.724%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T11:03:01.264 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:01.265 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:03:01.265 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/620068533' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T11:03:01.265 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1906167465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:01.301 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:03:01.326 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:03:01.359 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:03:01.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:01.396 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:03:01.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:03:01.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:03:01.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:03:01.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:03:01.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:03:01.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:03:01.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:03:01.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:03:01.425 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:01.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:01 smithi155 ceph-mon[184854]: pgmap v1100: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 7685/281442 objects degraded (2.731%); 13296/281442 objects misplaced (4.724%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T11:03:01.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:01.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:03:01.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/620068533' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T11:03:01.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1906167465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:01.475 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 10 pgp_num to 26 2023-10-09T11:03:01.777 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:03:01.778 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:03:01.778 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 0 2023-10-09T11:03:01.778 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:03:01.803 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:01.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:01.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:01.908 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:01.958 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:02.137 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:02.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:02.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:02.245 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:02.295 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:02.327 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:02.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:02.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:02.337 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:02 smithi125 ceph-mon[195507]: 10.2 scrub starts 2023-10-09T11:03:02.337 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:02 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/620068533' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T11:03:02.337 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:02 smithi125 ceph-mon[195507]: osdmap e447: 8 total, 8 up, 8 in 2023-10-09T11:03:02.337 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:02 smithi125 ceph-mon[195507]: failed to encode map e447 with expected crc 2023-10-09T11:03:02.337 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:02 smithi125 ceph-mon[195507]: failed to encode map e447 with expected crc 2023-10-09T11:03:02.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:02 smithi125 ceph-mon[200197]: 10.2 scrub starts 2023-10-09T11:03:02.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:02 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/620068533' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T11:03:02.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:02 smithi125 ceph-mon[200197]: osdmap e447: 8 total, 8 up, 8 in 2023-10-09T11:03:02.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:02 smithi125 ceph-mon[200197]: failed to encode map e447 with expected crc 2023-10-09T11:03:02.343 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:02 smithi125 ceph-mon[200197]: failed to encode map e447 with expected crc 2023-10-09T11:03:02.431 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:02 smithi155 ceph-mon[184854]: 10.2 scrub starts 2023-10-09T11:03:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:02 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/620068533' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T11:03:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:02 smithi155 ceph-mon[184854]: osdmap e447: 8 total, 8 up, 8 in 2023-10-09T11:03:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:02 smithi155 ceph-mon[184854]: failed to encode map e447 with expected crc 2023-10-09T11:03:02.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:02 smithi155 ceph-mon[184854]: failed to encode map e447 with expected crc 2023-10-09T11:03:02.495 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:02.500 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:02.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T11:03:02.529 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:02.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:02.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:02.642 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:02.690 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:02.732 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:02.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:02.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:02.839 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:02.926 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:02.958 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:02.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:02.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:02.985 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:03.065 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:03.112 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:03.146 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:03.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:03.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:03.192 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:03 smithi125 ceph-mon[195507]: 10.12 scrub starts 2023-10-09T11:03:03.193 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:03 smithi125 ceph-mon[195507]: failed to encode map e447 with expected crc 2023-10-09T11:03:03.193 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:03 smithi125 ceph-mon[195507]: pgmap v1102: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 7685/273444 objects degraded (2.810%); 12780/273444 objects misplaced (4.674%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T11:03:03.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:03 smithi125 ceph-mon[200197]: 10.12 scrub starts 2023-10-09T11:03:03.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:03 smithi125 ceph-mon[200197]: failed to encode map e447 with expected crc 2023-10-09T11:03:03.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:03 smithi125 ceph-mon[200197]: pgmap v1102: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 7685/273444 objects degraded (2.810%); 12780/273444 objects misplaced (4.674%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T11:03:03.255 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:03.320 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:03.363 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:03.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:03.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:03 smithi155 ceph-mon[184854]: 10.12 scrub starts 2023-10-09T11:03:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:03 smithi155 ceph-mon[184854]: failed to encode map e447 with expected crc 2023-10-09T11:03:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:03 smithi155 ceph-mon[184854]: pgmap v1102: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 7685/273444 objects degraded (2.810%); 12780/273444 objects misplaced (4.674%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T11:03:03.473 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:03.543 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:03.620 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:03.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:03.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:03.727 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:03.788 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:03.817 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:03.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:03.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:03.920 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:03.993 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:04.026 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:04.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:04.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:04.110 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:04 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 7685/273444 objects degraded (2.810%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:04.136 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:03:04.205 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:04.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:04 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 7685/273444 objects degraded (2.810%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:04.233 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:04 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 7685/273444 objects degraded (2.810%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:04.244 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:03:04.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:04.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:03:04.349 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:04.366 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:03:04.414 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:04.445 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:04.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:04.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:04.512 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:03:04.512 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:03:04.512 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T11:03:04.512 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T11:03:04.512 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:03:04.554 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:04.597 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:04.636 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:04.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:04.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:04.740 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:04.793 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:04.820 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:04.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:04.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:04.912 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T11:03:04.928 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:04.967 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:04.995 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:04.995 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:05.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:05.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:05.106 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:05.124 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:05 smithi125 ceph-mon[195507]: 10.2 scrub starts 2023-10-09T11:03:05.125 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:05 smithi125 ceph-mon[195507]: pgmap v1103: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 366 op/s; 7377/267045 objects degraded (2.762%); 12203/267045 objects misplaced (4.570%); 12 MiB/s, 194 objects/s recovering 2023-10-09T11:03:05.125 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:05 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1314426875' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:03:05.125 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:05 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:03:05.196 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:05.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:05 smithi155 ceph-mon[184854]: 10.2 scrub starts 2023-10-09T11:03:05.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:05 smithi155 ceph-mon[184854]: pgmap v1103: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 366 op/s; 7377/267045 objects degraded (2.762%); 12203/267045 objects misplaced (4.570%); 12 MiB/s, 194 objects/s recovering 2023-10-09T11:03:05.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:05 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1314426875' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:03:05.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:05 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:03:05.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:05.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:05.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:05.240 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:03:05.337 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:05.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:05 smithi125 ceph-mon[200197]: 10.2 scrub starts 2023-10-09T11:03:05.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:05 smithi125 ceph-mon[200197]: pgmap v1103: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 366 op/s; 7377/267045 objects degraded (2.762%); 12203/267045 objects misplaced (4.570%); 12 MiB/s, 194 objects/s recovering 2023-10-09T11:03:05.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:05 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1314426875' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:03:05.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:05 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:03:05.388 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:05.620 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:05.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:05.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:05.723 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:03:05.775 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:05.905 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:03:05.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:05.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:03:05.948 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:06.008 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:06.056 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:06.086 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:06.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:06.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:06.195 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:06.267 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:06.299 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:06.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:06.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:06.312 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:06 smithi125 ceph-mon[195507]: 10.12 scrub starts 2023-10-09T11:03:06.312 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:06 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:06.312 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:06 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:03:06.312 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:06 smithi125 ceph-mon[195507]: osdmap e448: 8 total, 8 up, 8 in 2023-10-09T11:03:06.312 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:06 smithi125 ceph-mon[195507]: failed to encode map e448 with expected crc 2023-10-09T11:03:06.312 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:06 smithi125 ceph-mon[195507]: pgmap v1105: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 7377/267045 objects degraded (2.762%); 12203/267045 objects misplaced (4.570%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T11:03:06.312 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:06 smithi125 ceph-mon[200197]: 10.12 scrub starts 2023-10-09T11:03:06.313 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:06 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:06.313 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:06 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:03:06.313 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:06 smithi125 ceph-mon[200197]: osdmap e448: 8 total, 8 up, 8 in 2023-10-09T11:03:06.313 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:06 smithi125 ceph-mon[200197]: failed to encode map e448 with expected crc 2023-10-09T11:03:06.313 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:06 smithi125 ceph-mon[200197]: pgmap v1105: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 7377/267045 objects degraded (2.762%); 12203/267045 objects misplaced (4.570%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T11:03:06.405 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:06.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:06 smithi155 ceph-mon[184854]: 10.12 scrub starts 2023-10-09T11:03:06.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:06 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:06.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:06 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:03:06.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:06 smithi155 ceph-mon[184854]: osdmap e448: 8 total, 8 up, 8 in 2023-10-09T11:03:06.448 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:06 smithi155 ceph-mon[184854]: failed to encode map e448 with expected crc 2023-10-09T11:03:06.448 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:06 smithi155 ceph-mon[184854]: pgmap v1105: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 7377/267045 objects degraded (2.762%); 12203/267045 objects misplaced (4.570%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T11:03:06.449 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:06.495 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:06.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:06.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:06.602 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:03:06.650 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:06.684 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:03:06.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:06.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:03:06.790 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:06.836 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:06.837 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:03:06.869 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:03:06.877 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:06.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:06.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:06.905 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:03:06.959 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:03:06.986 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:06.989 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:03:07.023 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:03:07.038 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:07.067 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:03:07.068 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:07.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:07.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:07.099 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:03:07.103 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:03:07.103 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:03:07.103 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:03:07.103 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:03:07.103 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:03:07.103 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:03:07.103 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:03:07.104 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:03:07.176 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:07.178 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 0, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T11:03:07.178 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:03:07.178 INFO:tasks.thrashosds.thrasher:Killing osd 3, live_osds are [0, 2, 3, 4, 5, 6, 7, 1] 2023-10-09T11:03:07.178 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-09T11:03:07.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T11:03:07.238 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:07.266 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:07.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:07.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:07.370 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:07.417 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:07.458 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:07.460 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:03:07.461 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:03:07.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:07.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:07.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:07 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3844607293' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:07.494 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:07 smithi125 systemd[1]: Stopping Ceph osd.3 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:03:07.495 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:07 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3844607293' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:07.495 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:07 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3844607293' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:07.564 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:07.609 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:07.642 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:07.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:07.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:07.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:07.810 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:07.909 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:07.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:07.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:07.958 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T11:03:08.017 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:08.069 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:07 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[229483]: 2023-10-09T11:03:07.979+0000 7f8ecf14e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T11:03:08.069 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:07 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[229483]: 2023-10-09T11:03:07.979+0000 7f8ecf14e700 -1 osd.3 448 *** Got signal Terminated *** 2023-10-09T11:03:08.069 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:07 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[229483]: 2023-10-09T11:03:07.979+0000 7f8ecf14e700 -1 osd.3 448 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T11:03:08.073 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:08.110 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:08.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:08.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:08.215 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:08.237 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:03:08.278 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:08 smithi155 ceph-mon[184854]: pgmap v1106: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 7377/258879 objects degraded (2.850%); 11620/258879 objects misplaced (4.489%); 11 MiB/s, 175 objects/s recovering 2023-10-09T11:03:08.278 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:08 smithi155 ceph-mon[184854]: osd.3 marked itself down and dead 2023-10-09T11:03:08.278 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:08 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 7377/267045 objects degraded (2.762%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:08.285 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:08.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:08 smithi125 ceph-mon[195507]: pgmap v1106: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 7377/258879 objects degraded (2.850%); 11620/258879 objects misplaced (4.489%); 11 MiB/s, 175 objects/s recovering 2023-10-09T11:03:08.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:08 smithi125 ceph-mon[195507]: osd.3 marked itself down and dead 2023-10-09T11:03:08.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:08 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 7377/267045 objects degraded (2.762%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:08.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:08 smithi125 ceph-mon[200197]: pgmap v1106: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 7377/258879 objects degraded (2.850%); 11620/258879 objects misplaced (4.489%); 11 MiB/s, 175 objects/s recovering 2023-10-09T11:03:08.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:08 smithi125 ceph-mon[200197]: osd.3 marked itself down and dead 2023-10-09T11:03:08.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:08 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 7377/267045 objects degraded (2.762%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:08.321 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:08.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:08.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:08.369 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:03:08.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:08.458 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:08.528 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:08.604 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:08.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:08.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:08.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:08.739 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:08.759 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:08.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:08.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:08.862 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:08.893 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:09.163 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:09.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:09.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:09.241 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:09 smithi125 bash[421703]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T11:03:09.241 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:09 smithi125 ceph-mon[200197]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T11:03:09.241 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:09 smithi125 ceph-mon[200197]: osdmap e449: 8 total, 7 up, 8 in 2023-10-09T11:03:09.241 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:09 smithi125 ceph-mon[200197]: failed to encode map e449 with expected crc 2023-10-09T11:03:09.241 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:09 smithi125 ceph-mon[200197]: failed to encode map e449 with expected crc 2023-10-09T11:03:09.266 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:09.332 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:09.421 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:09.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:09.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:09.429 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:09 smithi155 ceph-mon[184854]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T11:03:09.429 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:09 smithi155 ceph-mon[184854]: osdmap e449: 8 total, 7 up, 8 in 2023-10-09T11:03:09.429 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:09 smithi155 ceph-mon[184854]: failed to encode map e449 with expected crc 2023-10-09T11:03:09.429 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:09 smithi155 ceph-mon[184854]: failed to encode map e449 with expected crc 2023-10-09T11:03:09.429 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:09 smithi155 ceph-mon[184854]: failed to encode map e449 with expected crc 2023-10-09T11:03:09.529 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:09.568 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:09 smithi125 ceph-mon[200197]: failed to encode map e449 with expected crc 2023-10-09T11:03:09.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:09 smithi125 ceph-mon[195507]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T11:03:09.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:09 smithi125 ceph-mon[195507]: osdmap e449: 8 total, 7 up, 8 in 2023-10-09T11:03:09.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:09 smithi125 ceph-mon[195507]: failed to encode map e449 with expected crc 2023-10-09T11:03:09.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:09 smithi125 ceph-mon[195507]: failed to encode map e449 with expected crc 2023-10-09T11:03:09.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:09 smithi125 ceph-mon[195507]: failed to encode map e449 with expected crc 2023-10-09T11:03:09.573 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:09.611 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:03:09.806 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:09.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:09.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:09.915 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:09.965 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:10.003 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:10.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:10.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:10.110 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:10.169 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:10.234 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:10.253 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:10.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:10.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:10.270 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[195507]: osdmap e450: 8 total, 7 up, 8 in 2023-10-09T11:03:10.270 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[195507]: failed to encode map e450 with expected crc 2023-10-09T11:03:10.270 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[195507]: failed to encode map e450 with expected crc 2023-10-09T11:03:10.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[195507]: failed to encode map e450 with expected crc 2023-10-09T11:03:10.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[195507]: failed to encode map e450 with expected crc 2023-10-09T11:03:10.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[195507]: failed to encode map e450 with expected crc 2023-10-09T11:03:10.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[195507]: 10.c continuing backfill to osd.7 from (412'1287,448'7126] 10:3045b39d:::benchmark_data_smithi125_402869_object35805:head to 448'7126 2023-10-09T11:03:10.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[195507]: pgmap v1109: 129 pgs: 23 stale+active+clean, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 99 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 7080/252396 objects degraded (2.805%); 10996/252396 objects misplaced (4.357%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T11:03:10.335 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:03:10.335 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:03:10.359 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:10.409 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:10 smithi155 ceph-mon[184854]: osdmap e450: 8 total, 7 up, 8 in 2023-10-09T11:03:10.409 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:10 smithi155 ceph-mon[184854]: failed to encode map e450 with expected crc 2023-10-09T11:03:10.409 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:10 smithi155 ceph-mon[184854]: failed to encode map e450 with expected crc 2023-10-09T11:03:10.409 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:10 smithi155 ceph-mon[184854]: failed to encode map e450 with expected crc 2023-10-09T11:03:10.409 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:10 smithi155 ceph-mon[184854]: failed to encode map e450 with expected crc 2023-10-09T11:03:10.409 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:10 smithi155 ceph-mon[184854]: failed to encode map e450 with expected crc 2023-10-09T11:03:10.409 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:10 smithi155 ceph-mon[184854]: 10.c continuing backfill to osd.7 from (412'1287,448'7126] 10:3045b39d:::benchmark_data_smithi125_402869_object35805:head to 448'7126 2023-10-09T11:03:10.409 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:10 smithi155 ceph-mon[184854]: pgmap v1109: 129 pgs: 23 stale+active+clean, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 99 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 7080/252396 objects degraded (2.805%); 10996/252396 objects misplaced (4.357%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T11:03:10.412 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:10.487 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:10.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:10.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:10.559 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[200197]: osdmap e450: 8 total, 7 up, 8 in 2023-10-09T11:03:10.559 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[200197]: failed to encode map e450 with expected crc 2023-10-09T11:03:10.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[200197]: failed to encode map e450 with expected crc 2023-10-09T11:03:10.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[200197]: failed to encode map e450 with expected crc 2023-10-09T11:03:10.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[200197]: failed to encode map e450 with expected crc 2023-10-09T11:03:10.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[200197]: failed to encode map e450 with expected crc 2023-10-09T11:03:10.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[200197]: 10.c continuing backfill to osd.7 from (412'1287,448'7126] 10:3045b39d:::benchmark_data_smithi125_402869_object35805:head to 448'7126 2023-10-09T11:03:10.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:10 smithi125 ceph-mon[200197]: pgmap v1109: 129 pgs: 23 stale+active+clean, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 99 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 7080/252396 objects degraded (2.805%); 10996/252396 objects misplaced (4.357%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T11:03:10.592 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:10.632 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:10.659 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:10.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:10.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:10.725 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T11:03:10.763 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:10.807 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:10.880 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:10.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:10.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:10.984 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:11.018 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:11.046 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:03:11.190 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:11.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:11.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:11.216 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:03:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:03:11] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-10-09T11:03:11.281 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:03:11.295 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:11.350 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:11.485 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:11.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:11.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:11.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:11 smithi125 ceph-mon[195507]: osdmap e451: 8 total, 7 up, 8 in 2023-10-09T11:03:11.506 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:11 smithi125 ceph-mon[195507]: failed to encode map e451 with expected crc 2023-10-09T11:03:11.506 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:11 smithi125 ceph-mon[195507]: failed to encode map e451 with expected crc 2023-10-09T11:03:11.506 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:11 smithi125 ceph-mon[195507]: failed to encode map e451 with expected crc 2023-10-09T11:03:11.506 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:11 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/475744861' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:03:11.506 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:11 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:03:11.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:11 smithi125 ceph-mon[200197]: osdmap e451: 8 total, 7 up, 8 in 2023-10-09T11:03:11.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:11 smithi125 ceph-mon[200197]: failed to encode map e451 with expected crc 2023-10-09T11:03:11.507 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:11 smithi125 ceph-mon[200197]: failed to encode map e451 with expected crc 2023-10-09T11:03:11.507 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:11 smithi125 ceph-mon[200197]: failed to encode map e451 with expected crc 2023-10-09T11:03:11.507 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:11 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/475744861' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:03:11.507 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:11 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:03:11.589 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:11.633 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:11 smithi155 ceph-mon[184854]: osdmap e451: 8 total, 7 up, 8 in 2023-10-09T11:03:11.633 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:11 smithi155 ceph-mon[184854]: failed to encode map e451 with expected crc 2023-10-09T11:03:11.633 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:11 smithi155 ceph-mon[184854]: failed to encode map e451 with expected crc 2023-10-09T11:03:11.633 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:11 smithi155 ceph-mon[184854]: failed to encode map e451 with expected crc 2023-10-09T11:03:11.633 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:11 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/475744861' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:03:11.634 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:11 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:03:11.636 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:11.670 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:11.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:11.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:11.740 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-09T11:03:11.762 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:03:11.762 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:03:11.762 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:03:11.762 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:03:11.762 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:03:11.763 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:03:11.777 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:11.789 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:03:11.789 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:03:11.789 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:03:11.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:03:11.790 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T11:03:11.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:03:11.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:03:11.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:03:11.790 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:03:11.808 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:11 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.3.service: Succeeded. 2023-10-09T11:03:11.808 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:11 smithi125 systemd[1]: Stopped Ceph osd.3 for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T11:03:11.824 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:11.828 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T11:03:11.828 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-09T11:03:11.961 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:11.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:11.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:12.071 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:12.129 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:12.162 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:12.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:12.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:12.242 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:03:12.242 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:03:12.242 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T11:03:12.242 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T11:03:12.242 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:03:12.268 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:12.283 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T11:03:12.284 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T11:03:12.317 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:12.344 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:12.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:12.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:12.451 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:12.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:12.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:03:12.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[200197]: osdmap e452: 8 total, 7 up, 8 in 2023-10-09T11:03:12.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[200197]: failed to encode map e452 with expected crc 2023-10-09T11:03:12.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[200197]: failed to encode map e452 with expected crc 2023-10-09T11:03:12.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[200197]: failed to encode map e452 with expected crc 2023-10-09T11:03:12.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1794959145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:12.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[200197]: pgmap v1112: 129 pgs: 9 stale+active+clean, 1 active+recovery_wait+undersized+degraded, 19 active+undersized, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+remapped+backfill_wait, 86 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 12 KiB/s rd, 32 KiB/s wr, 1.02k op/s; 27503/246681 objects degraded (11.149%); 10377/246681 objects misplaced (4.207%); 14 MiB/s, 229 objects/s recovering 2023-10-09T11:03:12.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:12.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:03:12.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[195507]: osdmap e452: 8 total, 7 up, 8 in 2023-10-09T11:03:12.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[195507]: failed to encode map e452 with expected crc 2023-10-09T11:03:12.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[195507]: failed to encode map e452 with expected crc 2023-10-09T11:03:12.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[195507]: failed to encode map e452 with expected crc 2023-10-09T11:03:12.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1794959145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:12.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:12 smithi125 ceph-mon[195507]: pgmap v1112: 129 pgs: 9 stale+active+clean, 1 active+recovery_wait+undersized+degraded, 19 active+undersized, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+remapped+backfill_wait, 86 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 12 KiB/s rd, 32 KiB/s wr, 1.02k op/s; 27503/246681 objects degraded (11.149%); 10377/246681 objects misplaced (4.207%); 14 MiB/s, 229 objects/s recovering 2023-10-09T11:03:12.493 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:12.527 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:12.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:12.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:12.631 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:12.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:12 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:12.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:12 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:03:12.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:12 smithi155 ceph-mon[184854]: osdmap e452: 8 total, 7 up, 8 in 2023-10-09T11:03:12.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:12 smithi155 ceph-mon[184854]: failed to encode map e452 with expected crc 2023-10-09T11:03:12.687 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:12 smithi155 ceph-mon[184854]: failed to encode map e452 with expected crc 2023-10-09T11:03:12.687 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:12 smithi155 ceph-mon[184854]: failed to encode map e452 with expected crc 2023-10-09T11:03:12.687 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:12 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1794959145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:12.687 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:12 smithi155 ceph-mon[184854]: pgmap v1112: 129 pgs: 9 stale+active+clean, 1 active+recovery_wait+undersized+degraded, 19 active+undersized, 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 3 active+remapped+backfill_wait, 86 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 12 KiB/s rd, 32 KiB/s wr, 1.02k op/s; 27503/246681 objects degraded (11.149%); 10377/246681 objects misplaced (4.207%); 14 MiB/s, 229 objects/s recovering 2023-10-09T11:03:12.692 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:12.726 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:12.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:12.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:12.831 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:12.876 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:13.122 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T11:03:13.130 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:13.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:13.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:13.237 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:13.279 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:13.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:13.340 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:13.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:13.441 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:13.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:13 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 27503/246681 objects degraded (11.149%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:13.480 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:13.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:13.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:13.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:13.562 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:03:13.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:13 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 27503/246681 objects degraded (11.149%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:13.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:13 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 27503/246681 objects degraded (11.149%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:13.605 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:13.660 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:13.895 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:13.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:13.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:13.935 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:03:14.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:14.043 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:14.285 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:03:14.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:14 smithi125 ceph-mon[195507]: pgmap v1113: 129 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8.8 KiB/s rd, 23 KiB/s wr, 658 op/s; 34936/241707 objects degraded (14.454%); 7001/241707 objects misplaced (2.896%); 12 MiB/s, 185 objects/s recovering 2023-10-09T11:03:14.346 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:14.388 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:14.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:14.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:14.492 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:14.567 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:14.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:14 smithi125 ceph-mon[200197]: pgmap v1113: 129 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8.8 KiB/s rd, 23 KiB/s wr, 658 op/s; 34936/241707 objects degraded (14.454%); 7001/241707 objects misplaced (2.896%); 12 MiB/s, 185 objects/s recovering 2023-10-09T11:03:14.616 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:14.618 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:14 smithi155 ceph-mon[184854]: pgmap v1113: 129 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8.8 KiB/s rd, 23 KiB/s wr, 658 op/s; 34936/241707 objects degraded (14.454%); 7001/241707 objects misplaced (2.896%); 12 MiB/s, 185 objects/s recovering 2023-10-09T11:03:14.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:14.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:14.721 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:14.793 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:14.827 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:14.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:14.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:14.919 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:14.935 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:14.939 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:03:14.939 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T11:03:14.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T11:03:14.939 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:03:14.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.939 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:39213 8.2 8:4e7be898:::benchmark_data_smithi125_329074_object39212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T11:03:14.939 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:53:35.895072+0000", 2023-10-09T11:03:14.939 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 579.03257141899996, 2023-10-09T11:03:14.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.128363015, 2023-10-09T11:03:14.939 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:14.940 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:14.940 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:14.940 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:03:14.940 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:03:14.940 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39213 2023-10-09T11:03:14.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.940 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:14.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:14.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895072+0000", 2023-10-09T11:03:14.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:14.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.941 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:14.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895072+0000", 2023-10-09T11:03:14.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:14.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.941 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:14.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895070+0000", 2023-10-09T11:03:14.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:03:14.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:14.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895204+0000", 2023-10-09T11:03:14.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000133561 2023-10-09T11:03:14.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:14.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895215+0000", 2023-10-09T11:03:14.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0925e-05 2023-10-09T11:03:14.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:14.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895220+0000", 2023-10-09T11:03:14.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4949999999999999e-06 2023-10-09T11:03:14.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:14.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895798+0000", 2023-10-09T11:03:14.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00057733700000000003 2023-10-09T11:03:14.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:14.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.895926+0000", 2023-10-09T11:03:14.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012778600000000001 2023-10-09T11:03:14.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.944 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:03:14.944 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.896072+0000", 2023-10-09T11:03:14.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014615599999999999 2023-10-09T11:03:14.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.944 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:14.944 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.897973+0000", 2023-10-09T11:03:14.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019008510000000001 2023-10-09T11:03:14.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.945 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.945 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:14.945 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:35.899072+0000", 2023-10-09T11:03:14.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010995239999999999 2023-10-09T11:03:14.945 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.945 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.945 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:14.945 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023353+0000", 2023-10-09T11:03:14.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.124281323 2023-10-09T11:03:14.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:14.946 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023405+0000", 2023-10-09T11:03:14.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1158e-05 2023-10-09T11:03:14.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:14.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:53:36.023435+0000", 2023-10-09T11:03:14.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.006e-05 2023-10-09T11:03:14.947 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:14.947 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:14.947 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:14.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.948 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83954 8.4 8:2689e239:::benchmark_data_smithi125_329074_object83953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T11:03:14.948 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.108148+0000", 2023-10-09T11:03:14.948 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 537.81949526699998, 2023-10-09T11:03:14.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1275014759999999, 2023-10-09T11:03:14.948 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:14.948 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:14.949 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:14.949 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:03:14.949 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:03:14.949 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83954 2023-10-09T11:03:14.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.949 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:14.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:14.950 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108148+0000", 2023-10-09T11:03:14.950 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:14.950 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:14.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108148+0000", 2023-10-09T11:03:14.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:14.951 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.951 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.951 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:14.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108146+0000", 2023-10-09T11:03:14.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:14.951 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.952 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.952 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:14.952 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108191+0000", 2023-10-09T11:03:14.952 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4549000000000003e-05 2023-10-09T11:03:14.952 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.952 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.953 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:14.953 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108215+0000", 2023-10-09T11:03:14.953 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3994000000000001e-05 2023-10-09T11:03:14.953 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.953 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:14.953 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108222+0000", 2023-10-09T11:03:14.953 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4499999999999998e-06 2023-10-09T11:03:14.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.954 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.954 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:14.954 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108279+0000", 2023-10-09T11:03:14.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6643e-05 2023-10-09T11:03:14.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:14.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.108288+0000", 2023-10-09T11:03:14.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5109999999999992e-06 2023-10-09T11:03:14.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:14.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.872892+0000", 2023-10-09T11:03:14.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.76460362800000004 2023-10-09T11:03:14.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:14.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.872903+0000", 2023-10-09T11:03:14.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1158000000000001e-05 2023-10-09T11:03:14.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:14.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213528+0000", 2023-10-09T11:03:14.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.340624501 2023-10-09T11:03:14.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:14.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213679+0000", 2023-10-09T11:03:14.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151578 2023-10-09T11:03:14.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T11:03:14.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.213861+0000", 2023-10-09T11:03:14.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018154 2023-10-09T11:03:14.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.959 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:14.959 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219288+0000", 2023-10-09T11:03:14.959 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00542677 2023-10-09T11:03:14.959 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.959 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.959 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:14.959 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219340+0000", 2023-10-09T11:03:14.959 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2657000000000001e-05 2023-10-09T11:03:14.959 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.959 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.959 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:14.959 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219644+0000", 2023-10-09T11:03:14.960 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030375299999999998 2023-10-09T11:03:14.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.960 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.960 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:14.960 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235530+0000", 2023-10-09T11:03:14.960 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01588606 2023-10-09T11:03:14.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.960 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.960 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:14.960 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235575+0000", 2023-10-09T11:03:14.960 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5012999999999997e-05 2023-10-09T11:03:14.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.961 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:14.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235649+0000", 2023-10-09T11:03:14.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4252000000000007e-05 2023-10-09T11:03:14.961 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:14.961 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:14.961 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:14.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.961 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:83959 8.4 8:27b6f913:::benchmark_data_smithi125_329074_object83958:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-10-09T11:03:14.961 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:17.112252+0000", 2023-10-09T11:03:14.962 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 537.81539123699997, 2023-10-09T11:03:14.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1235403750000001, 2023-10-09T11:03:14.962 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:14.962 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:14.962 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:14.962 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:03:14.962 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:03:14.962 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83959 2023-10-09T11:03:14.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.962 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:14.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:14.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112252+0000", 2023-10-09T11:03:14.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:14.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:14.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112252+0000", 2023-10-09T11:03:14.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:14.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:14.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112251+0000", 2023-10-09T11:03:14.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:03:14.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:14.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112312+0000", 2023-10-09T11:03:14.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0888999999999999e-05 2023-10-09T11:03:14.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:14.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112324+0000", 2023-10-09T11:03:14.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2585999999999999e-05 2023-10-09T11:03:14.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:14.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112329+0000", 2023-10-09T11:03:14.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9880000000000004e-06 2023-10-09T11:03:14.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:14.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112378+0000", 2023-10-09T11:03:14.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8905e-05 2023-10-09T11:03:14.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:14.966 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.112389+0000", 2023-10-09T11:03:14.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0665e-05 2023-10-09T11:03:14.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.966 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.966 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:14.966 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.873769+0000", 2023-10-09T11:03:14.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.76138013699999996 2023-10-09T11:03:14.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.966 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.967 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:14.967 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:17.873776+0000", 2023-10-09T11:03:14.967 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.17e-06 2023-10-09T11:03:14.967 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.967 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:14.967 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216109+0000", 2023-10-09T11:03:14.967 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.342332699 2023-10-09T11:03:14.967 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.967 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:14.967 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216239+0000", 2023-10-09T11:03:14.968 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012997 2023-10-09T11:03:14.968 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.968 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.968 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,5,6", 2023-10-09T11:03:14.968 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.216354+0000", 2023-10-09T11:03:14.968 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115116 2023-10-09T11:03:14.968 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.968 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.968 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:14.968 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219398+0000", 2023-10-09T11:03:14.968 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030438679999999999 2023-10-09T11:03:14.968 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.969 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:14.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219407+0000", 2023-10-09T11:03:14.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1870000000000003e-06 2023-10-09T11:03:14.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.969 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:14.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.219682+0000", 2023-10-09T11:03:14.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00027478600000000001 2023-10-09T11:03:14.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:14.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235686+0000", 2023-10-09T11:03:14.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016004852 2023-10-09T11:03:14.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:14.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235725+0000", 2023-10-09T11:03:14.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8392000000000001e-05 2023-10-09T11:03:14.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:14.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:19.235792+0000", 2023-10-09T11:03:14.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7311000000000007e-05 2023-10-09T11:03:14.971 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:14.971 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:14.971 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:14.971 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.971 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:93330 8.2 8:4f5a2bcc:::benchmark_data_smithi125_329074_object93329:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T11:03:14.971 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:35.625993+0000", 2023-10-09T11:03:14.972 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 519.30165035599998, 2023-10-09T11:03:14.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.12993565000000001, 2023-10-09T11:03:14.972 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:14.972 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:14.972 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:14.972 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:03:14.972 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:03:14.972 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 93330 2023-10-09T11:03:14.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.972 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:14.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.972 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:14.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625993+0000", 2023-10-09T11:03:14.973 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:14.973 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:14.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625993+0000", 2023-10-09T11:03:14.973 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:14.973 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:14.974 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.625992+0000", 2023-10-09T11:03:14.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:14.974 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.974 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.974 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:14.974 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626043+0000", 2023-10-09T11:03:14.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0500000000000001e-05 2023-10-09T11:03:14.974 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.974 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.974 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:14.974 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626051+0000", 2023-10-09T11:03:14.975 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0509999999999997e-06 2023-10-09T11:03:14.975 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.975 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.975 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:14.975 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626056+0000", 2023-10-09T11:03:14.975 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9570000000000001e-06 2023-10-09T11:03:14.975 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.975 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.975 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:14.975 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626113+0000", 2023-10-09T11:03:14.975 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.736e-05 2023-10-09T11:03:14.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:14.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626283+0000", 2023-10-09T11:03:14.976 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017031699999999999 2023-10-09T11:03:14.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:03:14.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.626463+0000", 2023-10-09T11:03:14.976 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018018 2023-10-09T11:03:14.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:14.977 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.634095+0000", 2023-10-09T11:03:14.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0076312469999999999 2023-10-09T11:03:14.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:14.977 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.637111+0000", 2023-10-09T11:03:14.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003016725 2023-10-09T11:03:14.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:14.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755853+0000", 2023-10-09T11:03:14.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11874143299999999 2023-10-09T11:03:14.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:14.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755887+0000", 2023-10-09T11:03:14.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4017999999999999e-05 2023-10-09T11:03:14.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:14.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.755928+0000", 2023-10-09T11:03:14.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1452999999999998e-05 2023-10-09T11:03:14.979 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:14.979 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:14.979 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:14.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.979 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:93340 8.2 8:4653dd45:::benchmark_data_smithi125_329074_object93339:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T11:03:14.979 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:54:35.641334+0000", 2023-10-09T11:03:14.979 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 519.28630941599999, 2023-10-09T11:03:14.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.125653775, 2023-10-09T11:03:14.979 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:14.979 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:14.980 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:14.980 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:03:14.980 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:03:14.980 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 93340 2023-10-09T11:03:14.980 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.980 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:14.980 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.980 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:14.980 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641334+0000", 2023-10-09T11:03:14.980 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:14.980 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.981 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.981 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:14.981 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641334+0000", 2023-10-09T11:03:14.981 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:14.981 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.981 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.981 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:14.981 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641333+0000", 2023-10-09T11:03:14.981 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:03:14.981 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.981 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.981 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:14.982 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641362+0000", 2023-10-09T11:03:14.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9034999999999999e-05 2023-10-09T11:03:14.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.982 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.982 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:14.982 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641372+0000", 2023-10-09T11:03:14.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0597e-05 2023-10-09T11:03:14.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:14.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641377+0000", 2023-10-09T11:03:14.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7060000000000003e-06 2023-10-09T11:03:14.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:14.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641429+0000", 2023-10-09T11:03:14.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1721000000000002e-05 2023-10-09T11:03:14.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:14.985 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641529+0000", 2023-10-09T11:03:14.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000100602 2023-10-09T11:03:14.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.985 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:03:14.985 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.641632+0000", 2023-10-09T11:03:14.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103099 2023-10-09T11:03:14.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.985 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:14.985 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.649088+0000", 2023-10-09T11:03:14.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0074551569999999996 2023-10-09T11:03:14.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:14.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.649159+0000", 2023-10-09T11:03:14.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1760000000000004e-05 2023-10-09T11:03:14.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:14.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766886+0000", 2023-10-09T11:03:14.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11772616499999999 2023-10-09T11:03:14.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:14.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766941+0000", 2023-10-09T11:03:14.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5186000000000003e-05 2023-10-09T11:03:14.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:14.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:54:35.766987+0000", 2023-10-09T11:03:14.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6692e-05 2023-10-09T11:03:14.987 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:14.987 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:14.988 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:14.988 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.988 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:158888 8.2 8:4103043a:::benchmark_data_smithi125_329074_object62234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2023-10-09T11:03:14.988 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:03:14.988 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 419.571264063, 2023-10-09T11:03:14.988 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.123441579, 2023-10-09T11:03:14.988 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:14.988 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:14.988 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:14.988 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:03:14.988 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:03:14.989 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 158888 2023-10-09T11:03:14.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.989 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:14.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:14.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:03:14.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:14.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:14.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:03:14.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:14.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:14.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:03:14.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:14.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:14.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356381+0000", 2023-10-09T11:03:14.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.734e-06 2023-10-09T11:03:14.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:14.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356388+0000", 2023-10-09T11:03:14.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4559999999999997e-06 2023-10-09T11:03:14.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:14.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356392+0000", 2023-10-09T11:03:14.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0589999999999996e-06 2023-10-09T11:03:14.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:14.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.359480+0000", 2023-10-09T11:03:14.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030878379999999999 2023-10-09T11:03:14.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:14.992 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.359628+0000", 2023-10-09T11:03:14.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014820299999999999 2023-10-09T11:03:14.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:03:14.992 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.360074+0000", 2023-10-09T11:03:14.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00044630599999999999 2023-10-09T11:03:14.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:14.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.362052+0000", 2023-10-09T11:03:14.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019782720000000001 2023-10-09T11:03:14.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:14.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.362671+0000", 2023-10-09T11:03:14.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00061827099999999999 2023-10-09T11:03:14.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:14.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479765+0000", 2023-10-09T11:03:14.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11709412199999999 2023-10-09T11:03:14.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:14.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479796+0000", 2023-10-09T11:03:14.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0895000000000002e-05 2023-10-09T11:03:14.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:14.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479821+0000", 2023-10-09T11:03:14.995 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4816000000000001e-05 2023-10-09T11:03:14.995 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:14.995 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:14.995 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:14.995 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.995 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.995 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:100843 9.c e353/332 9:3d29ba4c:::benchmark_data_smithi125_363789_object100842:head v 353'6258, mlcod=353'6256)", 2023-10-09T11:03:14.995 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:43.227548+0000", 2023-10-09T11:03:14.995 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 271.70009517, 2023-10-09T11:03:14.995 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0253425, 2023-10-09T11:03:14.995 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:14.995 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:14.996 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:14.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.996 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:14.996 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227548+0000", 2023-10-09T11:03:14.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:14.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:14.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227548+0000", 2023-10-09T11:03:14.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:14.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:14.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227547+0000", 2023-10-09T11:03:14.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:03:14.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:14.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227594+0000", 2023-10-09T11:03:14.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7648000000000002e-05 2023-10-09T11:03:14.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:14.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227615+0000", 2023-10-09T11:03:14.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0435e-05 2023-10-09T11:03:14.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:14.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.227622+0000", 2023-10-09T11:03:14.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4850000000000003e-06 2023-10-09T11:03:14.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:14.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:14.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:14.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.229957+0000", 2023-10-09T11:03:14.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002334381 2023-10-09T11:03:15.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.000 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:15.000 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.229976+0000", 2023-10-09T11:03:15.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9596999999999999e-05 2023-10-09T11:03:15.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.000 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:15.000 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.252847+0000", 2023-10-09T11:03:15.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022871209 2023-10-09T11:03:15.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:15.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:43.252890+0000", 2023-10-09T11:03:15.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2920000000000002e-05 2023-10-09T11:03:15.001 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.001 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:15.001 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.001 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:102487 9.c e353/332 9:37d8c2ee:::benchmark_data_smithi125_363789_object102486:head v 353'6352, mlcod=353'6348)", 2023-10-09T11:03:15.001 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:45.158254+0000", 2023-10-09T11:03:15.001 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 269.769388886, 2023-10-09T11:03:15.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027965380000000001, 2023-10-09T11:03:15.002 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:15.002 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:15.002 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:15.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:15.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158254+0000", 2023-10-09T11:03:15.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:15.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158254+0000", 2023-10-09T11:03:15.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:15.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158254+0000", 2023-10-09T11:03:15.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:15.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:15.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158280+0000", 2023-10-09T11:03:15.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6463000000000001e-05 2023-10-09T11:03:15.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:15.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158296+0000", 2023-10-09T11:03:15.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6246999999999999e-05 2023-10-09T11:03:15.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:15.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.158305+0000", 2023-10-09T11:03:15.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9639999999999992e-06 2023-10-09T11:03:15.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:15.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165187+0000", 2023-10-09T11:03:15.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0068813679999999997 2023-10-09T11:03:15.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:15.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165209+0000", 2023-10-09T11:03:15.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1926999999999999e-05 2023-10-09T11:03:15.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:15.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186167+0000", 2023-10-09T11:03:15.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020958641 2023-10-09T11:03:15.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:15.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186220+0000", 2023-10-09T11:03:15.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2243000000000001e-05 2023-10-09T11:03:15.006 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.006 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:15.006 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.006 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:102490 9.4 e353/330 9:207553f4:::benchmark_data_smithi125_363789_object102489:head v 353'6429, mlcod=353'6425)", 2023-10-09T11:03:15.007 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:58:45.159295+0000", 2023-10-09T11:03:15.007 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 269.76834806900001, 2023-10-09T11:03:15.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026994220999999999, 2023-10-09T11:03:15.007 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:15.007 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:15.007 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:15.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.007 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:15.007 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159295+0000", 2023-10-09T11:03:15.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:15.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159295+0000", 2023-10-09T11:03:15.008 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.008 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:15.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159294+0000", 2023-10-09T11:03:15.008 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:15.008 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:15.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159332+0000", 2023-10-09T11:03:15.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7502000000000001e-05 2023-10-09T11:03:15.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.009 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.009 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:15.009 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159344+0000", 2023-10-09T11:03:15.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2418999999999999e-05 2023-10-09T11:03:15.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.009 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.009 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:15.009 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.159349+0000", 2023-10-09T11:03:15.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2619999999999998e-06 2023-10-09T11:03:15.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.010 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:15.010 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165240+0000", 2023-10-09T11:03:15.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0058900580000000001 2023-10-09T11:03:15.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.010 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:15.010 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.165259+0000", 2023-10-09T11:03:15.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9369e-05 2023-10-09T11:03:15.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:15.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186223+0000", 2023-10-09T11:03:15.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020963682000000001 2023-10-09T11:03:15.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:15.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:58:45.186289+0000", 2023-10-09T11:03:15.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6600000000000006e-05 2023-10-09T11:03:15.011 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.011 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:15.011 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.012 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:153414 9.c e374/332 9:3768b944:::benchmark_data_smithi125_363789_object43115:head v 374'9510, mlcod=374'9508)", 2023-10-09T11:03:15.012 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:46.250266+0000", 2023-10-09T11:03:15.012 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 208.67737685700001, 2023-10-09T11:03:15.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025683302000000002, 2023-10-09T11:03:15.012 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:15.012 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:15.012 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:15.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:15.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250266+0000", 2023-10-09T11:03:15.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:15.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250266+0000", 2023-10-09T11:03:15.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:15.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250265+0000", 2023-10-09T11:03:15.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:15.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:15.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250275+0000", 2023-10-09T11:03:15.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7089999999999997e-06 2023-10-09T11:03:15.014 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:15.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250292+0000", 2023-10-09T11:03:15.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7082e-05 2023-10-09T11:03:15.014 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:15.015 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250300+0000", 2023-10-09T11:03:15.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4689999999999997e-06 2023-10-09T11:03:15.015 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.015 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.015 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:15.015 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250340+0000", 2023-10-09T11:03:15.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9758000000000001e-05 2023-10-09T11:03:15.015 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.015 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.015 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:15.015 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.250362+0000", 2023-10-09T11:03:15.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2463000000000002e-05 2023-10-09T11:03:15.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.016 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:15.016 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.275910+0000", 2023-10-09T11:03:15.016 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025547325999999999 2023-10-09T11:03:15.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.016 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:15.016 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:46.275949+0000", 2023-10-09T11:03:15.016 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9786999999999997e-05 2023-10-09T11:03:15.016 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.016 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:15.017 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.017 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:9816 10.6 e406/403 10:610863d1:::benchmark_data_smithi125_402869_object9815:head v 406'620, mlcod=406'618)", 2023-10-09T11:03:15.017 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:22.339372+0000", 2023-10-09T11:03:15.017 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 112.58827067, 2023-10-09T11:03:15.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072690006000000001, 2023-10-09T11:03:15.017 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:15.017 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:15.017 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:15.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:15.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339372+0000", 2023-10-09T11:03:15.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.018 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:15.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339372+0000", 2023-10-09T11:03:15.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.018 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:15.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339372+0000", 2023-10-09T11:03:15.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:15.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:15.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339395+0000", 2023-10-09T11:03:15.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3051e-05 2023-10-09T11:03:15.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:15.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339404+0000", 2023-10-09T11:03:15.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.713e-06 2023-10-09T11:03:15.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.020 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:15.020 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339410+0000", 2023-10-09T11:03:15.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.181e-06 2023-10-09T11:03:15.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.020 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:15.020 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339487+0000", 2023-10-09T11:03:15.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6940999999999994e-05 2023-10-09T11:03:15.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:15.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339505+0000", 2023-10-09T11:03:15.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8700000000000001e-05 2023-10-09T11:03:15.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:15.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.412023+0000", 2023-10-09T11:03:15.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072518113999999995 2023-10-09T11:03:15.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:15.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.412062+0000", 2023-10-09T11:03:15.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8991e-05 2023-10-09T11:03:15.022 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.022 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:15.022 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.022 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.022 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.022 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:9820 10.d e406/403 10:b0a398bf:::benchmark_data_smithi125_402869_object9819:head v 406'628, mlcod=406'626)", 2023-10-09T11:03:15.022 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:22.339685+0000", 2023-10-09T11:03:15.022 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 112.587958245, 2023-10-09T11:03:15.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072206606000000007, 2023-10-09T11:03:15.022 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:15.022 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:15.023 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:15.023 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.023 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:15.023 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339685+0000", 2023-10-09T11:03:15.023 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.023 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.023 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.023 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:15.023 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339685+0000", 2023-10-09T11:03:15.023 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.023 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.023 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:15.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339684+0000", 2023-10-09T11:03:15.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:15.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.024 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:15.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339744+0000", 2023-10-09T11:03:15.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9889999999999997e-05 2023-10-09T11:03:15.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.024 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:15.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339754+0000", 2023-10-09T11:03:15.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0492999999999999e-05 2023-10-09T11:03:15.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.025 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:15.025 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339760+0000", 2023-10-09T11:03:15.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.699e-06 2023-10-09T11:03:15.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.025 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:15.025 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339792+0000", 2023-10-09T11:03:15.026 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2079999999999998e-05 2023-10-09T11:03:15.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.026 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:15.026 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339805+0000", 2023-10-09T11:03:15.026 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2809999999999999e-05 2023-10-09T11:03:15.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.027 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.027 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:15.027 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411854+0000", 2023-10-09T11:03:15.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072048880999999995 2023-10-09T11:03:15.027 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.027 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:15.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411891+0000", 2023-10-09T11:03:15.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7434000000000001e-05 2023-10-09T11:03:15.028 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.028 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:15.029 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.029 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:9825 10.2 e406/403 10:49e48d05:::benchmark_data_smithi125_402869_object9824:head v 406'599, mlcod=406'597)", 2023-10-09T11:03:15.029 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:22.340607+0000", 2023-10-09T11:03:15.029 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 112.58703562300001, 2023-10-09T11:03:15.030 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.071013035000000002, 2023-10-09T11:03:15.030 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:15.030 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:15.030 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:15.030 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.030 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:15.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340607+0000", 2023-10-09T11:03:15.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:15.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340607+0000", 2023-10-09T11:03:15.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:15.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340607+0000", 2023-10-09T11:03:15.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:15.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:15.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340667+0000", 2023-10-09T11:03:15.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0310999999999999e-05 2023-10-09T11:03:15.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:15.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340681+0000", 2023-10-09T11:03:15.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4199000000000001e-05 2023-10-09T11:03:15.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:15.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340688+0000", 2023-10-09T11:03:15.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9979999999999997e-06 2023-10-09T11:03:15.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:15.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410579+0000", 2023-10-09T11:03:15.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069890939999999999 2023-10-09T11:03:15.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:15.034 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410601+0000", 2023-10-09T11:03:15.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1953000000000001e-05 2023-10-09T11:03:15.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:15.034 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411576+0000", 2023-10-09T11:03:15.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00097502500000000002 2023-10-09T11:03:15.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:15.035 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411620+0000", 2023-10-09T11:03:15.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4305999999999999e-05 2023-10-09T11:03:15.035 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.035 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:15.035 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.035 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.035 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.035 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:9818 10.6 e406/403 10:67ea2a60:::benchmark_data_smithi125_402869_object9817:head v 406'621, mlcod=406'618)", 2023-10-09T11:03:15.035 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:22.340768+0000", 2023-10-09T11:03:15.035 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 112.586875073, 2023-10-09T11:03:15.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.071318931000000002, 2023-10-09T11:03:15.035 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:15.035 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:15.036 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:15.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:15.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340768+0000", 2023-10-09T11:03:15.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:15.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340768+0000", 2023-10-09T11:03:15.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.037 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:15.037 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340767+0000", 2023-10-09T11:03:15.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:15.037 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.037 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:15.037 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340793+0000", 2023-10-09T11:03:15.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5539000000000001e-05 2023-10-09T11:03:15.037 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.037 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:15.037 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410523+0000", 2023-10-09T11:03:15.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069730406999999994 2023-10-09T11:03:15.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:15.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410538+0000", 2023-10-09T11:03:15.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4747e-05 2023-10-09T11:03:15.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:15.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410587+0000", 2023-10-09T11:03:15.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8903000000000003e-05 2023-10-09T11:03:15.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:15.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410610+0000", 2023-10-09T11:03:15.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3288000000000001e-05 2023-10-09T11:03:15.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:15.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.412066+0000", 2023-10-09T11:03:15.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001456127 2023-10-09T11:03:15.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.040 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:15.040 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.412087+0000", 2023-10-09T11:03:15.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0557999999999999e-05 2023-10-09T11:03:15.040 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.040 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:15.040 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.040 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.040 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.040 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:9823 10.d e406/403 10:b90906b6:::benchmark_data_smithi125_402869_object9822:head v 406'629, mlcod=406'626)", 2023-10-09T11:03:15.040 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:22.341423+0000", 2023-10-09T11:03:15.040 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 112.586219668, 2023-10-09T11:03:15.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.07048981, 2023-10-09T11:03:15.041 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:15.041 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:15.041 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:15.041 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.041 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:15.041 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.341423+0000", 2023-10-09T11:03:15.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.041 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.041 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.041 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:15.041 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.341423+0000", 2023-10-09T11:03:15.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.042 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:15.042 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.341423+0000", 2023-10-09T11:03:15.042 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:15.042 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:15.042 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.341489+0000", 2023-10-09T11:03:15.042 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6185000000000004e-05 2023-10-09T11:03:15.042 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:15.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410523+0000", 2023-10-09T11:03:15.043 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069033960000000005 2023-10-09T11:03:15.043 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:15.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410538+0000", 2023-10-09T11:03:15.043 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5545000000000001e-05 2023-10-09T11:03:15.043 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:15.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410587+0000", 2023-10-09T11:03:15.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8813000000000002e-05 2023-10-09T11:03:15.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.044 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:15.044 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410610+0000", 2023-10-09T11:03:15.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3119999999999999e-05 2023-10-09T11:03:15.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.044 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:15.044 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411895+0000", 2023-10-09T11:03:15.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012849770000000001 2023-10-09T11:03:15.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:15.045 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411913+0000", 2023-10-09T11:03:15.045 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7864e-05 2023-10-09T11:03:15.045 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.045 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:15.045 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.045 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.045 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.045 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:80415 10.0 e431/403 10:0b1ab251:::benchmark_data_smithi125_402869_object80414:head v 431'4945, mlcod=431'4942)", 2023-10-09T11:03:15.045 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:02:24.077730+0000", 2023-10-09T11:03:15.046 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 50.849913370000003, 2023-10-09T11:03:15.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025244194000000001, 2023-10-09T11:03:15.046 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:15.046 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:15.046 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:15.046 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.046 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:15.046 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.077730+0000", 2023-10-09T11:03:15.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.046 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.047 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.047 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:15.047 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.077730+0000", 2023-10-09T11:03:15.047 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.047 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.047 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.047 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:15.047 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.077729+0000", 2023-10-09T11:03:15.047 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:15.047 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.047 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.047 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:15.048 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.077751+0000", 2023-10-09T11:03:15.048 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1577999999999999e-05 2023-10-09T11:03:15.048 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.048 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.048 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:15.048 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.077761+0000", 2023-10-09T11:03:15.048 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0453e-05 2023-10-09T11:03:15.048 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.048 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.048 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:15.048 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.077767+0000", 2023-10-09T11:03:15.049 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1809999999999998e-06 2023-10-09T11:03:15.049 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.049 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.049 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:15.049 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.079368+0000", 2023-10-09T11:03:15.049 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0016010849999999999 2023-10-09T11:03:15.049 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.049 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.049 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:15.049 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.079388+0000", 2023-10-09T11:03:15.049 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9273e-05 2023-10-09T11:03:15.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:15.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.102936+0000", 2023-10-09T11:03:15.050 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023548716000000001 2023-10-09T11:03:15.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:15.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.102974+0000", 2023-10-09T11:03:15.050 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7552999999999998e-05 2023-10-09T11:03:15.050 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.050 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:15.051 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.051 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.051 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.051 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:80413 10.c e431/421 10:3f4c4151:::benchmark_data_smithi125_402869_object80412:head v 431'5045, mlcod=431'5042)", 2023-10-09T11:03:15.051 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:02:24.079168+0000", 2023-10-09T11:03:15.051 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 50.848475055999998, 2023-10-09T11:03:15.051 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026157119999999999, 2023-10-09T11:03:15.051 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:15.051 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:15.051 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:15.051 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.051 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:15.052 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.079168+0000", 2023-10-09T11:03:15.052 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.052 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.052 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.052 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:15.052 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.079168+0000", 2023-10-09T11:03:15.052 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.052 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.052 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.052 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:15.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.079167+0000", 2023-10-09T11:03:15.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:15.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.053 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:15.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.079224+0000", 2023-10-09T11:03:15.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6598999999999998e-05 2023-10-09T11:03:15.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.053 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:15.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.079232+0000", 2023-10-09T11:03:15.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1729999999999995e-06 2023-10-09T11:03:15.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:15.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.079236+0000", 2023-10-09T11:03:15.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2769999999999999e-06 2023-10-09T11:03:15.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:15.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.079277+0000", 2023-10-09T11:03:15.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0679999999999997e-05 2023-10-09T11:03:15.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.055 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:15.055 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.079297+0000", 2023-10-09T11:03:15.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9899e-05 2023-10-09T11:03:15.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.055 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:15.055 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.105281+0000", 2023-10-09T11:03:15.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025983692999999999 2023-10-09T11:03:15.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.055 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:15.055 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.105325+0000", 2023-10-09T11:03:15.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4487999999999998e-05 2023-10-09T11:03:15.056 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.056 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:15.056 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.056 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.056 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.056 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:80416 10.c e431/421 10:390944d0:::benchmark_data_smithi125_402869_object80415:head v 431'5046, mlcod=431'5042)", 2023-10-09T11:03:15.056 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:02:24.081650+0000", 2023-10-09T11:03:15.056 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 50.845993231000001, 2023-10-09T11:03:15.056 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026355475999999999, 2023-10-09T11:03:15.056 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:15.056 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:15.056 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:15.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.057 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:15.057 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.081650+0000", 2023-10-09T11:03:15.057 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.057 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.057 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:15.057 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.081650+0000", 2023-10-09T11:03:15.057 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.057 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:15.058 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.081649+0000", 2023-10-09T11:03:15.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:03:15.058 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.058 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:15.058 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.081696+0000", 2023-10-09T11:03:15.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6823000000000002e-05 2023-10-09T11:03:15.058 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.058 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:15.059 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.081710+0000", 2023-10-09T11:03:15.059 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3876000000000001e-05 2023-10-09T11:03:15.059 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.059 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.059 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:15.059 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.081715+0000", 2023-10-09T11:03:15.059 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3890000000000004e-06 2023-10-09T11:03:15.059 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.059 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.059 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:15.059 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.081807+0000", 2023-10-09T11:03:15.059 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2107000000000004e-05 2023-10-09T11:03:15.060 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.060 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.060 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:15.060 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.102912+0000", 2023-10-09T11:03:15.060 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021105071 2023-10-09T11:03:15.060 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.060 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.060 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:15.060 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.107963+0000", 2023-10-09T11:03:15.060 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.005050705 2023-10-09T11:03:15.060 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.060 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.061 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:15.061 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:24.108005+0000", 2023-10-09T11:03:15.061 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2265000000000003e-05 2023-10-09T11:03:15.061 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.061 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:15.061 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.061 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.061 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.061 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:90009 10.4 e438/403 10:23a6b7d6:::benchmark_data_smithi125_402869_object90008:head v 438'5569, mlcod=438'5564)", 2023-10-09T11:03:15.061 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:02:36.960525+0000", 2023-10-09T11:03:15.061 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 37.967117631000001, 2023-10-09T11:03:15.061 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.026612440000000001, 2023-10-09T11:03:15.062 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:15.062 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:15.062 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:15.062 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.062 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:15.062 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:36.960525+0000", 2023-10-09T11:03:15.062 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.062 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.062 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.062 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:15.062 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:36.960525+0000", 2023-10-09T11:03:15.062 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.063 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.063 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.063 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:15.063 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:36.960525+0000", 2023-10-09T11:03:15.063 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:15.063 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.063 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.063 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:15.063 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:36.960553+0000", 2023-10-09T11:03:15.063 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7368999999999998e-05 2023-10-09T11:03:15.063 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.063 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.063 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:15.064 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:36.960558+0000", 2023-10-09T11:03:15.064 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4020000000000001e-06 2023-10-09T11:03:15.064 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.064 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.064 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:15.064 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:36.960561+0000", 2023-10-09T11:03:15.064 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.247e-06 2023-10-09T11:03:15.064 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.064 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.064 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:15.064 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:36.966441+0000", 2023-10-09T11:03:15.064 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0058801299999999999 2023-10-09T11:03:15.065 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.065 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.065 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:15.065 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:36.966462+0000", 2023-10-09T11:03:15.065 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0429999999999999e-05 2023-10-09T11:03:15.065 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.065 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.065 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:15.065 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:36.987076+0000", 2023-10-09T11:03:15.065 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020614751000000001 2023-10-09T11:03:15.065 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.065 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.066 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:15.066 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:36.987138+0000", 2023-10-09T11:03:15.066 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1371999999999994e-05 2023-10-09T11:03:15.066 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.066 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:15.066 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.066 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.066 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.066 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:92781 10.c e438/435 10:35de6549:::benchmark_data_smithi125_402869_object92780:head v 438'5815, mlcod=438'5813)", 2023-10-09T11:03:15.066 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:02:40.467997+0000", 2023-10-09T11:03:15.066 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 34.459646343999999, 2023-10-09T11:03:15.067 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025730734000000002, 2023-10-09T11:03:15.067 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:15.067 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:15.067 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:15.067 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.067 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:15.067 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:40.467997+0000", 2023-10-09T11:03:15.067 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.067 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.067 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.067 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:15.067 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:40.467997+0000", 2023-10-09T11:03:15.068 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:15.068 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.068 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.068 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:15.068 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:40.467996+0000", 2023-10-09T11:03:15.068 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:15.068 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.068 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.068 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:15.068 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:40.468037+0000", 2023-10-09T11:03:15.068 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1139e-05 2023-10-09T11:03:15.068 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.069 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.069 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:15.069 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:40.468044+0000", 2023-10-09T11:03:15.069 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.083e-06 2023-10-09T11:03:15.069 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.069 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.069 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:15.069 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:40.468047+0000", 2023-10-09T11:03:15.069 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8399999999999997e-06 2023-10-09T11:03:15.069 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.069 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.069 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:15.070 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:40.471261+0000", 2023-10-09T11:03:15.070 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032136679999999998 2023-10-09T11:03:15.070 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.070 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.070 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:15.070 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:40.471287+0000", 2023-10-09T11:03:15.070 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5531999999999999e-05 2023-10-09T11:03:15.070 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.070 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.071 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:15.071 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:40.493673+0000", 2023-10-09T11:03:15.071 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022386834000000001 2023-10-09T11:03:15.071 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:15.071 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:15.071 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:15.071 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:40.493727+0000", 2023-10-09T11:03:15.071 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3998e-05 2023-10-09T11:03:15.071 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.071 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:15.071 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.071 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:15.072 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:15.072 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:03:15.072 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:15.109 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:15.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:15.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:15.215 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:15.277 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:15.339 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T11:03:15.352 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:15.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:15.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:15.462 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:15.514 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:15.544 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:15.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:15.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:15.649 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:15.743 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:15.779 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:15.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:15.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:15.850 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:03:15.894 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:15.936 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:16.102 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:16.102 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:03:16.102 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:16.103 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:03:16.316 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:16.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:16.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:16.421 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:16.439 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:16.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:03:16.462 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:16.597 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:16.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:16.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:16.713 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:16.771 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:16.829 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 0, 7] out_osds: [] dead_osds: [3] live_osds: [0, 2, 4, 5, 6, 7, 1] 2023-10-09T11:03:16.829 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:03:16.829 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T11:03:16.829 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.794008 2023-10-09T11:03:16.829 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0.7940077391632515 2023-10-09T11:03:16.854 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:16.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:16.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:16.960 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:17.009 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:17.072 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:17.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:17.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:17.093 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:17 smithi155 ceph-mon[184854]: pgmap v1114: 129 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 7.4 KiB/s rd, 20 KiB/s wr, 556 op/s; 34936/241707 objects degraded (14.454%); 7001/241707 objects misplaced (2.896%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T11:03:17.107 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:03:17.107 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:03:17.107 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T11:03:17.107 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:03:17.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:17.205 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:03:17.244 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:17.253 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:03:17.285 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:17.317 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:03:17.319 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:17.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:17.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:17.359 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:03:17.390 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:03:17.421 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:03:17.425 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:17.430 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:17 smithi125 ceph-mon[195507]: pgmap v1114: 129 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 7.4 KiB/s rd, 20 KiB/s wr, 556 op/s; 34936/241707 objects degraded (14.454%); 7001/241707 objects misplaced (2.896%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T11:03:17.433 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:17 smithi125 ceph-mon[200197]: pgmap v1114: 129 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 7.4 KiB/s rd, 20 KiB/s wr, 556 op/s; 34936/241707 objects degraded (14.454%); 7001/241707 objects misplaced (2.896%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T11:03:17.462 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:03:17.462 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:03:17.462 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:03:17.463 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:03:17.463 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T11:03:17.463 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:03:17.463 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:03:17.463 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:03:17.463 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:03:17.471 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:17.509 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:17.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:17.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:17.524 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T11:03:17.621 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:17.665 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:17.752 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:17.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:17.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:17.859 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:17.908 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:17.966 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:03:17.981 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:17.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:17.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:18.087 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:18.129 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:18.130 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:03:18.172 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:18.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:18.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:18.241 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T11:03:18.242 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T11:03:18.278 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:18.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:18 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3394918518' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:03:18.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:18 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:03:18.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:18 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2182336700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:18.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:18 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3394918518' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:03:18.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:18 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:03:18.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:18 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2182336700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:18.331 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:18.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3394918518' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:03:18.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:18 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:03:18.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2182336700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:18.532 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:18.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:18.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:18.640 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:18.700 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:18.752 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:18.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:18.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:18.861 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:18.914 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:18.949 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:18.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:18.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:19.055 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:19.149 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:19.176 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:19.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:19.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:19.180 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:19 smithi155 ceph-mon[184854]: pgmap v1115: 129 pgs: 1 active+recovery_wait+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 6.0 KiB/s rd, 16 KiB/s wr, 834 op/s; 33549/232482 objects degraded (14.431%); 7001/232482 objects misplaced (3.011%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T11:03:19.180 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:19 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:19.180 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:19 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:03:19.180 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:19 smithi155 ceph-mon[184854]: osdmap e453: 8 total, 7 up, 8 in 2023-10-09T11:03:19.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:19 smithi155 ceph-mon[184854]: failed to encode map e453 with expected crc 2023-10-09T11:03:19.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:19 smithi155 ceph-mon[184854]: failed to encode map e453 with expected crc 2023-10-09T11:03:19.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:19 smithi155 ceph-mon[184854]: failed to encode map e453 with expected crc 2023-10-09T11:03:19.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:19 smithi155 ceph-mon[184854]: failed to encode map e453 with expected crc 2023-10-09T11:03:19.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:19 smithi155 ceph-mon[184854]: failed to encode map e453 with expected crc 2023-10-09T11:03:19.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:19 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 33549/232482 objects degraded (14.431%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:19.198 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:03:19.198 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:03:19.281 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:19.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[195507]: pgmap v1115: 129 pgs: 1 active+recovery_wait+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 6.0 KiB/s rd, 16 KiB/s wr, 834 op/s; 33549/232482 objects degraded (14.431%); 7001/232482 objects misplaced (3.011%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T11:03:19.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:19.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:03:19.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[195507]: osdmap e453: 8 total, 7 up, 8 in 2023-10-09T11:03:19.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[195507]: failed to encode map e453 with expected crc 2023-10-09T11:03:19.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[195507]: failed to encode map e453 with expected crc 2023-10-09T11:03:19.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[195507]: failed to encode map e453 with expected crc 2023-10-09T11:03:19.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[195507]: failed to encode map e453 with expected crc 2023-10-09T11:03:19.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[195507]: failed to encode map e453 with expected crc 2023-10-09T11:03:19.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 33549/232482 objects degraded (14.431%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:19.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[200197]: pgmap v1115: 129 pgs: 1 active+recovery_wait+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 6.0 KiB/s rd, 16 KiB/s wr, 834 op/s; 33549/232482 objects degraded (14.431%); 7001/232482 objects misplaced (3.011%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T11:03:19.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:19.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:03:19.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[200197]: osdmap e453: 8 total, 7 up, 8 in 2023-10-09T11:03:19.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[200197]: failed to encode map e453 with expected crc 2023-10-09T11:03:19.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[200197]: failed to encode map e453 with expected crc 2023-10-09T11:03:19.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[200197]: failed to encode map e453 with expected crc 2023-10-09T11:03:19.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[200197]: failed to encode map e453 with expected crc 2023-10-09T11:03:19.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[200197]: failed to encode map e453 with expected crc 2023-10-09T11:03:19.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:19 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 33549/232482 objects degraded (14.431%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:19.360 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:19.563 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:19.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:19.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:19.595 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T11:03:19.672 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:19.754 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:19.807 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:19.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:19.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:19.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:19.990 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:20.051 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:20.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:20.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:20.089 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:03:20.149 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.2 primary-affinity to 0.794008 (8520362) 2023-10-09T11:03:20.162 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:20.200 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:20.242 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:03:20.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:20 smithi155 ceph-mon[184854]: 10.2 scrub starts 2023-10-09T11:03:20.246 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:20 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2108248761' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.7940077391632515}]: dispatch 2023-10-09T11:03:20.278 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:20.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:20.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:20.392 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:20.440 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:20 smithi125 ceph-mon[195507]: 10.2 scrub starts 2023-10-09T11:03:20.440 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:20 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2108248761' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.7940077391632515}]: dispatch 2023-10-09T11:03:20.440 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:20 smithi125 ceph-mon[200197]: 10.2 scrub starts 2023-10-09T11:03:20.440 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:20 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2108248761' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.7940077391632515}]: dispatch 2023-10-09T11:03:20.444 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:20.527 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:20.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:20.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:20.633 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:20.655 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:20.686 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:20.697 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:03:20.825 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:20.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:20.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:20.930 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:20.990 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:21.017 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:21.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:21.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:21.128 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:21.203 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:21.249 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:21.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:21.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:21.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:21 smithi155 ceph-mon[184854]: pgmap v1117: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 5.7 KiB/s rd, 15 KiB/s wr, 1.02k op/s; 32709/225450 objects degraded (14.508%); 6353/225450 objects misplaced (2.818%); 11 MiB/s, 175 objects/s recovering 2023-10-09T11:03:21.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:21 smithi155 ceph-mon[184854]: failed to encode map e454 with expected crc 2023-10-09T11:03:21.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:21 smithi155 ceph-mon[184854]: failed to encode map e454 with expected crc 2023-10-09T11:03:21.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:21 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2108248761' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.7940077391632515}]': finished 2023-10-09T11:03:21.255 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:21 smithi155 ceph-mon[184854]: osdmap e454: 8 total, 7 up, 8 in 2023-10-09T11:03:21.307 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:03:21.307 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:03:21.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:21 smithi125 ceph-mon[195507]: pgmap v1117: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 5.7 KiB/s rd, 15 KiB/s wr, 1.02k op/s; 32709/225450 objects degraded (14.508%); 6353/225450 objects misplaced (2.818%); 11 MiB/s, 175 objects/s recovering 2023-10-09T11:03:21.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:21 smithi125 ceph-mon[195507]: failed to encode map e454 with expected crc 2023-10-09T11:03:21.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:21 smithi125 ceph-mon[195507]: failed to encode map e454 with expected crc 2023-10-09T11:03:21.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:21 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2108248761' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.7940077391632515}]': finished 2023-10-09T11:03:21.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:21 smithi125 ceph-mon[195507]: osdmap e454: 8 total, 7 up, 8 in 2023-10-09T11:03:21.322 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:03:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:03:21] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-10-09T11:03:21.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:21 smithi125 ceph-mon[200197]: pgmap v1117: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 5.7 KiB/s rd, 15 KiB/s wr, 1.02k op/s; 32709/225450 objects degraded (14.508%); 6353/225450 objects misplaced (2.818%); 11 MiB/s, 175 objects/s recovering 2023-10-09T11:03:21.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:21 smithi125 ceph-mon[200197]: failed to encode map e454 with expected crc 2023-10-09T11:03:21.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:21 smithi125 ceph-mon[200197]: failed to encode map e454 with expected crc 2023-10-09T11:03:21.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:21 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2108248761' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.7940077391632515}]': finished 2023-10-09T11:03:21.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:21 smithi125 ceph-mon[200197]: osdmap e454: 8 total, 7 up, 8 in 2023-10-09T11:03:21.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:21.403 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:21.438 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:21.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:21.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:21.516 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:21.548 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:21.599 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:21.710 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T11:03:21.779 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:21.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:21.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:21.952 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:22.011 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:22.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:22.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:22.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:22.163 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:03:22.192 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:22.221 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:03:22.252 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:22.254 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:03:22.281 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:03:22.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:22 smithi125 ceph-mon[195507]: failed to encode map e455 with expected crc 2023-10-09T11:03:22.328 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:22 smithi125 ceph-mon[195507]: osdmap e455: 8 total, 7 up, 8 in 2023-10-09T11:03:22.328 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:22 smithi125 ceph-mon[195507]: failed to encode map e455 with expected crc 2023-10-09T11:03:22.328 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:22 smithi125 ceph-mon[195507]: failed to encode map e455 with expected crc 2023-10-09T11:03:22.328 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:22 smithi125 ceph-mon[195507]: pgmap v1120: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.39k op/s; 31387/216732 objects degraded (14.482%); 6353/216732 objects misplaced (2.931%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T11:03:22.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:22 smithi125 ceph-mon[200197]: failed to encode map e455 with expected crc 2023-10-09T11:03:22.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:22 smithi125 ceph-mon[200197]: osdmap e455: 8 total, 7 up, 8 in 2023-10-09T11:03:22.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:22 smithi125 ceph-mon[200197]: failed to encode map e455 with expected crc 2023-10-09T11:03:22.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:22 smithi125 ceph-mon[200197]: failed to encode map e455 with expected crc 2023-10-09T11:03:22.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:22 smithi125 ceph-mon[200197]: pgmap v1120: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.39k op/s; 31387/216732 objects degraded (14.482%); 6353/216732 objects misplaced (2.931%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T11:03:22.332 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:03:22.335 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:22.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:22.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:22.357 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:03:22.398 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:03:22.433 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:03:22.433 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:03:22.434 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:03:22.434 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:03:22.434 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T11:03:22.434 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:03:22.434 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:03:22.434 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:03:22.434 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:03:22.444 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:22.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:22 smithi155 ceph-mon[184854]: failed to encode map e455 with expected crc 2023-10-09T11:03:22.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:22 smithi155 ceph-mon[184854]: osdmap e455: 8 total, 7 up, 8 in 2023-10-09T11:03:22.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:22 smithi155 ceph-mon[184854]: failed to encode map e455 with expected crc 2023-10-09T11:03:22.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:22 smithi155 ceph-mon[184854]: failed to encode map e455 with expected crc 2023-10-09T11:03:22.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:22 smithi155 ceph-mon[184854]: pgmap v1120: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.39k op/s; 31387/216732 objects degraded (14.482%); 6353/216732 objects misplaced (2.931%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T11:03:22.515 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:22.601 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:22.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:22.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:22.708 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:22.787 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:22.941 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T11:03:22.941 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T11:03:22.980 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:22.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:22.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:23.086 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:23.138 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:23.174 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:23.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:23.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:23.279 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:23.344 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:23.368 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:23.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:23.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:23.461 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:03:23.461 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:03:23.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.461 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:128548 10.0 10:066389ad:::benchmark_data_smithi125_402869_object29979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T11:03:23.461 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:03:23.448193+0000", 2023-10-09T11:03:23.461 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0086911269999999999, 2023-10-09T11:03:23.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0087248789999999996, 2023-10-09T11:03:23.461 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:23.462 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:03:23.462 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:23.462 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:03:23.462 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:03:23.462 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 128548 2023-10-09T11:03:23.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.462 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:23.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:23.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.448193+0000", 2023-10-09T11:03:23.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:23.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.463 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:23.463 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.448193+0000", 2023-10-09T11:03:23.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:23.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.463 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:23.463 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.448192+0000", 2023-10-09T11:03:23.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:03:23.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:23.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.448198+0000", 2023-10-09T11:03:23.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5099999999999998e-06 2023-10-09T11:03:23.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:23.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.448210+0000", 2023-10-09T11:03:23.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2336e-05 2023-10-09T11:03:23.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:23.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.448216+0000", 2023-10-09T11:03:23.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8259999999999998e-06 2023-10-09T11:03:23.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:23.465 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.448277+0000", 2023-10-09T11:03:23.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1414999999999994e-05 2023-10-09T11:03:23.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:23.465 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.448397+0000", 2023-10-09T11:03:23.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000119787 2023-10-09T11:03:23.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.466 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:03:23.466 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.448783+0000", 2023-10-09T11:03:23.466 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00038563300000000003 2023-10-09T11:03:23.466 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.466 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:23.466 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.454160+0000", 2023-10-09T11:03:23.466 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0053774629999999999 2023-10-09T11:03:23.466 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:23.466 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:23.467 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:23.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.467 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:128550 10.f 10:ffc769ff:::benchmark_data_smithi125_402869_object29981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T11:03:23.467 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:03:23.448225+0000", 2023-10-09T11:03:23.467 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0086593390000000003, 2023-10-09T11:03:23.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0087914289999999999, 2023-10-09T11:03:23.467 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:23.467 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:03:23.467 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:23.467 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:03:23.467 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:03:23.468 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 128550 2023-10-09T11:03:23.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.468 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:23.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:23.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.448225+0000", 2023-10-09T11:03:23.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:23.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:23.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.448225+0000", 2023-10-09T11:03:23.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:23.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.469 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:23.469 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.448225+0000", 2023-10-09T11:03:23.469 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:23.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.469 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:23.469 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.448227+0000", 2023-10-09T11:03:23.469 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4119999999999999e-06 2023-10-09T11:03:23.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.469 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:23.470 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.448233+0000", 2023-10-09T11:03:23.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6380000000000001e-06 2023-10-09T11:03:23.470 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.470 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.470 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:23.470 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.448236+0000", 2023-10-09T11:03:23.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.495e-06 2023-10-09T11:03:23.470 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.470 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.470 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:23.470 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.450521+0000", 2023-10-09T11:03:23.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002284309 2023-10-09T11:03:23.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.471 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:23.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.450637+0000", 2023-10-09T11:03:23.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000116723 2023-10-09T11:03:23.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.471 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2", 2023-10-09T11:03:23.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.450757+0000", 2023-10-09T11:03:23.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000120136 2023-10-09T11:03:23.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:23.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.454188+0000", 2023-10-09T11:03:23.472 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003430921 2023-10-09T11:03:23.472 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:23.472 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:23.472 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:23.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.472 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop_reply(client.36123.0:128548 10.0 e455/441)", 2023-10-09T11:03:23.472 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:03:23.454951+0000", 2023-10-09T11:03:23.472 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.001933719, 2023-10-09T11:03:23.472 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021386959999999998, 2023-10-09T11:03:23.473 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:23.473 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "queued for pg", 2023-10-09T11:03:23.473 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:23.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:23.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.454951+0000", 2023-10-09T11:03:23.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:23.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:23.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.454951+0000", 2023-10-09T11:03:23.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:23.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:23.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.454950+0000", 2023-10-09T11:03:23.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:23.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:23.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.454955+0000", 2023-10-09T11:03:23.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2360000000000004e-06 2023-10-09T11:03:23.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.475 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:23.475 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.454967+0000", 2023-10-09T11:03:23.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1482e-05 2023-10-09T11:03:23.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.475 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:23.475 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.454972+0000", 2023-10-09T11:03:23.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7010000000000001e-06 2023-10-09T11:03:23.475 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:23.475 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:23.475 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:23.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.476 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:128536 10.a 10:566b7ac3:::benchmark_data_smithi125_402869_object29967:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T11:03:23.476 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:03:23.439382+0000", 2023-10-09T11:03:23.476 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.017501972000000001, 2023-10-09T11:03:23.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017751097, 2023-10-09T11:03:23.476 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:23.476 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:03:23.476 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:23.476 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:03:23.476 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:03:23.476 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 128536 2023-10-09T11:03:23.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.477 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:23.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:23.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.439382+0000", 2023-10-09T11:03:23.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:23.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:23.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.439382+0000", 2023-10-09T11:03:23.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:23.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:23.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.439381+0000", 2023-10-09T11:03:23.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:03:23.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:23.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.439387+0000", 2023-10-09T11:03:23.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5369999999999997e-06 2023-10-09T11:03:23.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:23.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.439395+0000", 2023-10-09T11:03:23.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6499999999999996e-06 2023-10-09T11:03:23.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:23.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.439399+0000", 2023-10-09T11:03:23.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.651e-06 2023-10-09T11:03:23.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:23.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.440288+0000", 2023-10-09T11:03:23.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00088886800000000001 2023-10-09T11:03:23.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:23.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.440394+0000", 2023-10-09T11:03:23.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000105912 2023-10-09T11:03:23.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:03:23.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.440511+0000", 2023-10-09T11:03:23.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000117154 2023-10-09T11:03:23.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:23.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.447741+0000", 2023-10-09T11:03:23.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0072295529999999997 2023-10-09T11:03:23.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:23.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.447748+0000", 2023-10-09T11:03:23.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5669999999999997e-06 2023-10-09T11:03:23.481 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:23.481 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:23.481 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:23.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.482 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:128538 10.a 10:5d075511:::benchmark_data_smithi125_402869_object29969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e455)", 2023-10-09T11:03:23.482 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:03:23.441378+0000", 2023-10-09T11:03:23.482 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.015506506, 2023-10-09T11:03:23.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015828020000000002, 2023-10-09T11:03:23.482 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:23.482 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:03:23.482 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:23.482 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:03:23.482 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:03:23.482 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 128538 2023-10-09T11:03:23.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.483 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:23.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.483 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:23.483 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.441378+0000", 2023-10-09T11:03:23.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:23.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.483 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:23.483 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.441378+0000", 2023-10-09T11:03:23.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:23.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:23.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.441377+0000", 2023-10-09T11:03:23.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:03:23.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:23.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.441382+0000", 2023-10-09T11:03:23.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8940000000000001e-06 2023-10-09T11:03:23.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:23.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.441391+0000", 2023-10-09T11:03:23.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3449999999999995e-06 2023-10-09T11:03:23.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:23.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.441396+0000", 2023-10-09T11:03:23.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9180000000000002e-06 2023-10-09T11:03:23.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:23.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.443655+0000", 2023-10-09T11:03:23.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022593000000000001 2023-10-09T11:03:23.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:23.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.443767+0000", 2023-10-09T11:03:23.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000111606 2023-10-09T11:03:23.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:03:23.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.443875+0000", 2023-10-09T11:03:23.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000107947 2023-10-09T11:03:23.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:23.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.453975+0000", 2023-10-09T11:03:23.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01009972 2023-10-09T11:03:23.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:23.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:23.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:23.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:23.454006+0000", 2023-10-09T11:03:23.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0793999999999997e-05 2023-10-09T11:03:23.487 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:23.487 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:23.487 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:23.487 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:23.488 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T11:03:23.488 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 5 2023-10-09T11:03:23.488 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:03:23.488 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:23.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:23 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/623601217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:23.554 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:23.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:23 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/623601217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:23.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:23 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/623601217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:23.602 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:23.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:23.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:23.708 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:23.761 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:23.885 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T11:03:23.894 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:23.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:23.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:24.005 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:24.049 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:24.081 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:24.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:24.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:24.182 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:24 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 31387/216732 objects degraded (14.482%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:24.182 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:24 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2017037440' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:03:24.183 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:24 smithi155 ceph-mon[184854]: pgmap v1121: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.25k op/s; 30602/209889 objects degraded (14.580%); 5717/209889 objects misplaced (2.724%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T11:03:24.186 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:24.222 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:03:24.269 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:24.308 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:24.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:24.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:24.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:24 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 31387/216732 objects degraded (14.482%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:24 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2017037440' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:03:24.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:24 smithi125 ceph-mon[195507]: pgmap v1121: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.25k op/s; 30602/209889 objects degraded (14.580%); 5717/209889 objects misplaced (2.724%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T11:03:24.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:24 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 31387/216732 objects degraded (14.482%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:24.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:24 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2017037440' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:03:24.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:24 smithi125 ceph-mon[200197]: pgmap v1121: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.25k op/s; 30602/209889 objects degraded (14.580%); 5717/209889 objects misplaced (2.724%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T11:03:24.361 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:03:24.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:24.466 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:24.595 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:24.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:24.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:24.701 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:24.760 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:24.791 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:24.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:24.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:24.899 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:24.934 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:24.942 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:03:24.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:24.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:24.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:25.070 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:25.138 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:25.189 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:25 smithi125 ceph-mon[200197]: 10.12 scrub starts 2023-10-09T11:03:25.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:25 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:03:25.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:25 smithi125 ceph-mon[200197]: failed to encode map e456 with expected crc 2023-10-09T11:03:25.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2017037440' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:03:25.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:25 smithi125 ceph-mon[200197]: osdmap e456: 8 total, 7 up, 8 in 2023-10-09T11:03:25.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:25 smithi125 ceph-mon[200197]: failed to encode map e456 with expected crc 2023-10-09T11:03:25.191 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:25 smithi125 ceph-mon[200197]: failed to encode map e456 with expected crc 2023-10-09T11:03:25.191 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:25.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:25.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:25.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:25.338 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:25.348 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:25.389 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:25.391 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:25 smithi155 ceph-mon[184854]: 10.12 scrub starts 2023-10-09T11:03:25.391 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:25 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:03:25.391 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:25 smithi155 ceph-mon[184854]: failed to encode map e456 with expected crc 2023-10-09T11:03:25.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2017037440' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:03:25.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:25 smithi155 ceph-mon[184854]: osdmap e456: 8 total, 7 up, 8 in 2023-10-09T11:03:25.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:25 smithi155 ceph-mon[184854]: failed to encode map e456 with expected crc 2023-10-09T11:03:25.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:25 smithi155 ceph-mon[184854]: failed to encode map e456 with expected crc 2023-10-09T11:03:25.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:25.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:25.471 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:25 smithi125 ceph-mon[195507]: 10.12 scrub starts 2023-10-09T11:03:25.471 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:25 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:03:25.471 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:25 smithi125 ceph-mon[195507]: failed to encode map e456 with expected crc 2023-10-09T11:03:25.471 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2017037440' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:03:25.471 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:25 smithi125 ceph-mon[195507]: osdmap e456: 8 total, 7 up, 8 in 2023-10-09T11:03:25.471 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:25 smithi125 ceph-mon[195507]: failed to encode map e456 with expected crc 2023-10-09T11:03:25.471 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:25 smithi125 ceph-mon[195507]: failed to encode map e456 with expected crc 2023-10-09T11:03:25.497 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:25.553 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 0, 7] out_osds: [] dead_osds: [3] live_osds: [0, 2, 4, 5, 6, 7, 1] 2023-10-09T11:03:25.553 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:03:25.553 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T11:03:25.553 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:03:25.554 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:25.585 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:25.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:25.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:25.678 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:03:25.678 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:03:25.678 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T11:03:25.678 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T11:03:25.679 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:03:25.691 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:25.742 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:26.099 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:26.105 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T11:03:26.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:26.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:26.200 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:26 smithi125 ceph-mon[195507]: 10.2 scrub starts 2023-10-09T11:03:26.200 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:26 smithi125 ceph-mon[195507]: pgmap v1123: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 30602/209889 objects degraded (14.580%); 5717/209889 objects misplaced (2.724%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T11:03:26.209 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:26.272 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:26.327 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:26.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:26.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:26.336 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:26 smithi155 ceph-mon[184854]: 10.2 scrub starts 2023-10-09T11:03:26.336 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:26 smithi155 ceph-mon[184854]: pgmap v1123: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 30602/209889 objects degraded (14.580%); 5717/209889 objects misplaced (2.724%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T11:03:26.433 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:26.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:26 smithi125 ceph-mon[200197]: 10.2 scrub starts 2023-10-09T11:03:26.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:26 smithi125 ceph-mon[200197]: pgmap v1123: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 30602/209889 objects degraded (14.580%); 5717/209889 objects misplaced (2.724%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T11:03:26.473 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:26.550 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:26.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:26.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:26.605 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:03:26.609 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:03:26.660 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:26.730 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:26.743 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:26.766 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:03:26.773 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:26.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:26.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:26.817 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:03:26.873 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:03:26.877 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:26.911 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:03:26.918 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:26.958 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:03:26.995 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:27.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:27.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:27.010 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:03:27.045 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:03:27.045 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:03:27.045 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:03:27.045 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:03:27.045 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T11:03:27.046 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:03:27.046 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:03:27.046 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:03:27.046 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:03:27.105 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:27.157 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:27.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:27 smithi155 ceph-mon[184854]: 10.12 deep-scrub starts 2023-10-09T11:03:27.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:27 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4062565662' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:27.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:27 smithi125 ceph-mon[195507]: 10.12 deep-scrub starts 2023-10-09T11:03:27.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:27 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4062565662' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:27.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:27 smithi125 ceph-mon[200197]: 10.12 deep-scrub starts 2023-10-09T11:03:27.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:27 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4062565662' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:27.728 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:27.736 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:27.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:27.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:27.841 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:27.866 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:03:27.866 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:03:27.883 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:27.921 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:27.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:27.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T11:03:27.977 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T11:03:28.032 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:28.090 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:28.227 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:28.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:28 smithi125 ceph-mon[195507]: 10.2 scrub starts 2023-10-09T11:03:28.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:28 smithi125 ceph-mon[195507]: pgmap v1124: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 29294/201045 objects degraded (14.571%); 5717/201045 objects misplaced (2.844%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T11:03:28.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:28 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 30602/209889 objects degraded (14.580%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:28.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:28.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:28.268 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T11:03:28.334 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:28.407 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:28.458 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:28.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:28.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:28.486 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:03:28.491 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:03:28.519 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:28 smithi125 ceph-mon[200197]: 10.2 scrub starts 2023-10-09T11:03:28.519 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:28 smithi125 ceph-mon[200197]: pgmap v1124: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 29294/201045 objects degraded (14.571%); 5717/201045 objects misplaced (2.844%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T11:03:28.519 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:28 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 30602/209889 objects degraded (14.580%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:28.562 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:28.615 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:28.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:28.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:28.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:28.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:28 smithi155 ceph-mon[184854]: 10.2 scrub starts 2023-10-09T11:03:28.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:28 smithi155 ceph-mon[184854]: pgmap v1124: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 29294/201045 objects degraded (14.571%); 5717/201045 objects misplaced (2.844%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T11:03:28.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:28 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 30602/209889 objects degraded (14.580%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:28.739 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:03:28.811 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:28.851 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:29.035 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:29.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:29.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:29.055 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1124,"stamp":"2023-10-09T11:03:27.722580+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4390892789,"num_objects":67015,"num_object_clones":0,"num_object_copies":201045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29294,"num_objects_misplaced":5717,"num_objects_unfound":0,"num_objects_dirty":67015,"num_whiteouts":0,"num_read":34,"num_read_kb":63,"num_write":124064,"num_write_kb":5920707,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9880,"num_bytes_recovered":644325056,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":183148,"ondisk_log_size":183148,"up":334,"acting":332,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":395,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19989516,"kb_used_data":13830412,"kb_used_omap":130,"kb_used_meta":6158525,"kb_avail":729971700,"statfs":{"total":767960285184,"available":747491020800,"internally_reserved":0,"allocated":14162341888,"data_stored":14147068981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":133799,"internal_metadata":6306329945},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,5,6,6,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-533135360,"num_objects":-8135,"num_object_clones":0,"num_object_copies":-24405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3415,"num_objects_misplaced":-636,"num_objects_unfound":0,"num_objects_dirty":-8135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8135,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":452,"num_bytes_recovered":29622272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.592535"},"pg_stats":[{"pgid":"10.13","version":"456'7105","reported_seq":7183,"reported_epoch":456,"state":"active+undersized+degraded","last_fresh":"2023-10-09T11:03:26.119994+0000","last_change":"2023-10-09T11:03:09.261452+0000","last_active":"2023-10-09T11:03:26.119994+0000","last_peered":"2023-10-09T11:03:26.119994+0000","last_clean":"2023-10-09T11:03:08.233384+0000","last_became_active":"2023-10-09T11:03:09.261452+0000","last_became_peered":"2023-10-09T11:03:09.261452+0000","last_unstale":"2023-10-09T11:03:26.119994+0000","last_undegraded":"2023-10-09T11:03:09.253057+0000","last_fullsized":"2023-10-09T11:03:09.252944+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":7105,"log_dups_size":0,"ondisk_log_size":7105,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135888896,"num_objects":2074,"num_object_clones":0,"num_object_copies":6222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2074,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4046,"num_write_kb":195808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":2074}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"76'161","reported_seq":695,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.165836+0000","last_change":"2023-10-09T11:02:36.217074+0000","last_active":"2023-10-09T11:03:21.165836+0000","last_peered":"2023-10-09T11:03:21.165836+0000","last_clean":"2023-10-09T11:03:21.165836+0000","last_became_active":"2023-10-09T11:02:36.216494+0000","last_became_peered":"2023-10-09T11:02:36.216494+0000","last_unstale":"2023-10-09T11:03:21.165836+0000","last_undegraded":"2023-10-09T11:03:21.165836+0000","last_fullsized":"2023-10-09T11:03:21.165836+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:56.760915+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":238,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.192458+0000","last_change":"2023-10-09T11:03:21.185383+0000","last_active":"2023-10-09T11:03:21.192458+0000","last_peered":"2023-10-09T11:03:21.192458+0000","last_clean":"2023-10-09T11:03:21.192458+0000","last_became_active":"2023-10-09T11:03:21.185022+0000","last_became_peered":"2023-10-09T11:03:21.185022+0000","last_unstale":"2023-10-09T11:03:21.192458+0000","last_undegraded":"2023-10-09T11:03:21.192458+0000","last_fullsized":"2023-10-09T11:03:21.192458+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:05.230434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":418,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.151117+0000","last_change":"2023-10-09T11:03:09.281513+0000","last_active":"2023-10-09T11:03:20.151117+0000","last_peered":"2023-10-09T11:03:20.151117+0000","last_clean":"2023-10-09T10:59:05.150443+0000","last_became_active":"2023-10-09T11:03:09.281513+0000","last_became_peered":"2023-10-09T11:03:09.281513+0000","last_unstale":"2023-10-09T11:03:20.151117+0000","last_undegraded":"2023-10-09T11:03:20.151117+0000","last_fullsized":"2023-10-09T11:03:09.255083+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:10.908317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.10","version":"456'7067","reported_seq":7269,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-09T11:03:26.367428+0000","last_change":"2023-10-09T11:02:54.959383+0000","last_active":"2023-10-09T11:03:26.367428+0000","last_peered":"2023-10-09T11:03:26.367428+0000","last_clean":"2023-10-09T11:03:26.367428+0000","last_became_active":"2023-10-09T11:02:49.354248+0000","last_became_peered":"2023-10-09T11:02:49.354248+0000","last_unstale":"2023-10-09T11:03:26.367428+0000","last_undegraded":"2023-10-09T11:03:26.367428+0000","last_fullsized":"2023-10-09T11:03:26.367428+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"445'6135","last_scrub_stamp":"2023-10-09T11:02:54.959296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:02:54.959296+0000","objects_scrubbed":3027,"log_size":7067,"log_dups_size":0,"ondisk_log_size":7067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:06.262764+0000","scrub_duration":0.90587347299999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136314880,"num_objects":2080,"num_object_clones":0,"num_object_copies":6240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":947,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":613,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.166910+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:03:21.166910+0000","last_peered":"2023-10-09T11:03:21.166910+0000","last_clean":"2023-10-09T11:03:21.166910+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:03:21.166910+0000","last_undegraded":"2023-10-09T11:03:21.166910+0000","last_fullsized":"2023-10-09T11:03:21.166910+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":140,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.150234+0000","last_change":"2023-10-09T11:03:09.281956+0000","last_active":"2023-10-09T11:03:20.150234+0000","last_peered":"2023-10-09T11:03:20.150234+0000","last_clean":"2023-10-09T11:03:05.204978+0000","last_became_active":"2023-10-09T11:03:09.281956+0000","last_became_peered":"2023-10-09T11:03:09.281956+0000","last_unstale":"2023-10-09T11:03:20.150234+0000","last_undegraded":"2023-10-09T11:03:20.150234+0000","last_fullsized":"2023-10-09T11:03:09.257044+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:22:18.518064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":403,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.147904+0000","last_change":"2023-10-09T11:03:09.267136+0000","last_active":"2023-10-09T11:03:20.147904+0000","last_peered":"2023-10-09T11:03:20.147904+0000","last_clean":"2023-10-09T11:03:05.238597+0000","last_became_active":"2023-10-09T11:03:09.267136+0000","last_became_peered":"2023-10-09T11:03:09.267136+0000","last_unstale":"2023-10-09T11:03:20.147904+0000","last_undegraded":"2023-10-09T11:03:20.147904+0000","last_fullsized":"2023-10-09T11:03:09.259492+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:29:28.233045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"455'7175","reported_seq":9534,"reported_epoch":455,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:03:23.070510+0000","last_change":"2023-10-09T11:02:51.343343+0000","last_active":"2023-10-09T11:03:23.070510+0000","last_peered":"2023-10-09T11:03:23.070510+0000","last_clean":"2023-10-09T11:02:32.284069+0000","last_became_active":"2023-10-09T11:02:51.339919+0000","last_became_peered":"2023-10-09T11:02:51.339919+0000","last_unstale":"2023-10-09T11:03:23.070510+0000","last_undegraded":"2023-10-09T11:03:23.070510+0000","last_fullsized":"2023-10-09T11:03:23.070510+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":7175,"log_dups_size":0,"ondisk_log_size":7175,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142868480,"num_objects":2180,"num_object_clones":0,"num_object_copies":6540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2180,"num_objects_unfound":0,"num_objects_dirty":2180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4058,"num_write_kb":199616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":568,"num_bytes_recovered":37191680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[0,4,1],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2180}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":936,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.177390+0000","last_change":"2023-10-09T11:02:36.217484+0000","last_active":"2023-10-09T11:03:21.177390+0000","last_peered":"2023-10-09T11:03:21.177390+0000","last_clean":"2023-10-09T11:03:21.177390+0000","last_became_active":"2023-10-09T11:02:36.217091+0000","last_became_peered":"2023-10-09T11:02:36.217091+0000","last_unstale":"2023-10-09T11:03:21.177390+0000","last_undegraded":"2023-10-09T11:03:21.177390+0000","last_fullsized":"2023-10-09T11:03:21.177390+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:59.912586+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":485,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.144341+0000","last_change":"2023-10-09T11:03:09.265139+0000","last_active":"2023-10-09T11:03:20.144341+0000","last_peered":"2023-10-09T11:03:20.144341+0000","last_clean":"2023-10-09T11:00:12.824548+0000","last_became_active":"2023-10-09T11:03:09.265139+0000","last_became_peered":"2023-10-09T11:03:09.265139+0000","last_unstale":"2023-10-09T11:03:20.144341+0000","last_undegraded":"2023-10-09T11:03:20.144341+0000","last_fullsized":"2023-10-09T11:03:09.258614+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:21.491250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":245,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.170268+0000","last_change":"2023-10-09T11:01:15.401167+0000","last_active":"2023-10-09T11:03:21.170268+0000","last_peered":"2023-10-09T11:03:21.170268+0000","last_clean":"2023-10-09T11:03:21.170268+0000","last_became_active":"2023-10-09T11:01:15.400876+0000","last_became_peered":"2023-10-09T11:01:15.400876+0000","last_unstale":"2023-10-09T11:03:21.170268+0000","last_undegraded":"2023-10-09T11:03:21.170268+0000","last_fullsized":"2023-10-09T11:03:21.170268+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:56.342226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":777,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.189815+0000","last_change":"2023-10-09T11:02:36.216785+0000","last_active":"2023-10-09T11:03:21.189815+0000","last_peered":"2023-10-09T11:03:21.189815+0000","last_clean":"2023-10-09T11:03:21.189815+0000","last_became_active":"2023-10-09T11:02:36.215919+0000","last_became_peered":"2023-10-09T11:02:36.215919+0000","last_unstale":"2023-10-09T11:03:21.189815+0000","last_undegraded":"2023-10-09T11:03:21.189815+0000","last_fullsized":"2023-10-09T11:03:21.189815+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:08:40.893630+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":97,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.148078+0000","last_change":"2023-10-09T11:03:09.280968+0000","last_active":"2023-10-09T11:03:20.148078+0000","last_peered":"2023-10-09T11:03:20.148078+0000","last_clean":"2023-10-09T11:03:05.236727+0000","last_became_active":"2023-10-09T11:03:09.280968+0000","last_became_peered":"2023-10-09T11:03:09.280968+0000","last_unstale":"2023-10-09T11:03:20.148078+0000","last_undegraded":"2023-10-09T11:03:20.148078+0000","last_fullsized":"2023-10-09T11:03:09.259246+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:09.293086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":419,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.149936+0000","last_change":"2023-10-09T11:03:09.264554+0000","last_active":"2023-10-09T11:03:20.149936+0000","last_peered":"2023-10-09T11:03:20.149936+0000","last_clean":"2023-10-09T10:59:05.150819+0000","last_became_active":"2023-10-09T11:03:09.264554+0000","last_became_peered":"2023-10-09T11:03:09.264554+0000","last_unstale":"2023-10-09T11:03:20.149936+0000","last_undegraded":"2023-10-09T11:03:20.149936+0000","last_fullsized":"2023-10-09T11:03:09.255554+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:05.069371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":783,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.179329+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:03:21.179329+0000","last_peered":"2023-10-09T11:03:21.179329+0000","last_clean":"2023-10-09T11:03:21.179329+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:03:21.179329+0000","last_undegraded":"2023-10-09T11:03:21.179329+0000","last_fullsized":"2023-10-09T11:03:21.179329+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":307,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.166059+0000","last_change":"2023-10-09T11:02:33.198185+0000","last_active":"2023-10-09T11:03:21.166059+0000","last_peered":"2023-10-09T11:03:21.166059+0000","last_clean":"2023-10-09T11:03:21.166059+0000","last_became_active":"2023-10-09T11:02:33.197870+0000","last_became_peered":"2023-10-09T11:02:33.197870+0000","last_unstale":"2023-10-09T11:03:21.166059+0000","last_undegraded":"2023-10-09T11:03:21.166059+0000","last_fullsized":"2023-10-09T11:03:21.166059+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:21.200979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":428,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.191011+0000","last_change":"2023-10-09T11:03:09.267647+0000","last_active":"2023-10-09T11:03:21.191011+0000","last_peered":"2023-10-09T11:03:21.191011+0000","last_clean":"2023-10-09T11:02:31.325660+0000","last_became_active":"2023-10-09T11:03:09.267647+0000","last_became_peered":"2023-10-09T11:03:09.267647+0000","last_unstale":"2023-10-09T11:03:21.191011+0000","last_undegraded":"2023-10-09T11:03:21.191011+0000","last_fullsized":"2023-10-09T11:03:09.259787+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:07.990293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":645,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.170036+0000","last_change":"2023-10-09T11:01:16.673177+0000","last_active":"2023-10-09T11:03:21.170036+0000","last_peered":"2023-10-09T11:03:21.170036+0000","last_clean":"2023-10-09T11:03:21.170036+0000","last_became_active":"2023-10-09T11:01:15.401411+0000","last_became_peered":"2023-10-09T11:01:15.401411+0000","last_unstale":"2023-10-09T11:03:21.170036+0000","last_undegraded":"2023-10-09T11:03:21.170036+0000","last_fullsized":"2023-10-09T11:03:21.170036+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:46.989699+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":44,"reported_epoch":455,"state":"active+undersized+degraded","last_fresh":"2023-10-09T11:03:21.189518+0000","last_change":"2023-10-09T11:03:09.279848+0000","last_active":"2023-10-09T11:03:21.189518+0000","last_peered":"2023-10-09T11:03:21.189518+0000","last_clean":"2023-10-09T11:03:05.218580+0000","last_became_active":"2023-10-09T11:03:09.279848+0000","last_became_peered":"2023-10-09T11:03:09.279848+0000","last_unstale":"2023-10-09T11:03:21.189518+0000","last_undegraded":"2023-10-09T11:03:09.254601+0000","last_fullsized":"2023-10-09T11:03:09.254495+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:58:37.746107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":214,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.166617+0000","last_change":"2023-10-09T11:01:15.401892+0000","last_active":"2023-10-09T11:03:21.166617+0000","last_peered":"2023-10-09T11:03:21.166617+0000","last_clean":"2023-10-09T11:03:21.166617+0000","last_became_active":"2023-10-09T11:01:15.401025+0000","last_became_peered":"2023-10-09T11:01:15.401025+0000","last_unstale":"2023-10-09T11:03:21.166617+0000","last_undegraded":"2023-10-09T11:03:21.166617+0000","last_fullsized":"2023-10-09T11:03:21.166617+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:48:03.531920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":811,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.151873+0000","last_change":"2023-10-09T11:03:09.262200+0000","last_active":"2023-10-09T11:03:20.151873+0000","last_peered":"2023-10-09T11:03:20.151873+0000","last_clean":"2023-10-09T11:03:05.208718+0000","last_became_active":"2023-10-09T11:03:09.262200+0000","last_became_peered":"2023-10-09T11:03:09.262200+0000","last_unstale":"2023-10-09T11:03:20.151873+0000","last_undegraded":"2023-10-09T11:03:20.151873+0000","last_fullsized":"2023-10-09T11:03:09.255114+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:26.167661+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":41,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.190786+0000","last_change":"2023-10-09T11:02:33.213462+0000","last_active":"2023-10-09T11:03:21.190786+0000","last_peered":"2023-10-09T11:03:21.190786+0000","last_clean":"2023-10-09T11:03:21.190786+0000","last_became_active":"2023-10-09T11:02:33.197602+0000","last_became_peered":"2023-10-09T11:02:33.197602+0000","last_unstale":"2023-10-09T11:03:21.190786+0000","last_undegraded":"2023-10-09T11:03:21.190786+0000","last_fullsized":"2023-10-09T11:03:21.190786+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:08.773902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":470,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.176461+0000","last_change":"2023-10-09T10:57:46.845755+0000","last_active":"2023-10-09T11:03:21.176461+0000","last_peered":"2023-10-09T11:03:21.176461+0000","last_clean":"2023-10-09T11:03:21.176461+0000","last_became_active":"2023-10-09T10:57:46.845475+0000","last_became_peered":"2023-10-09T10:57:46.845475+0000","last_unstale":"2023-10-09T11:03:21.176461+0000","last_undegraded":"2023-10-09T11:03:21.176461+0000","last_fullsized":"2023-10-09T11:03:21.176461+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:21.673994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":824,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.146133+0000","last_change":"2023-10-09T11:03:09.280392+0000","last_active":"2023-10-09T11:03:20.146133+0000","last_peered":"2023-10-09T11:03:20.146133+0000","last_clean":"2023-10-09T11:03:05.237354+0000","last_became_active":"2023-10-09T11:03:09.280392+0000","last_became_peered":"2023-10-09T11:03:09.280392+0000","last_unstale":"2023-10-09T11:03:20.146133+0000","last_undegraded":"2023-10-09T11:03:20.146133+0000","last_fullsized":"2023-10-09T11:03:09.258613+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:05.550653+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":37,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.189223+0000","last_change":"2023-10-09T11:02:33.207032+0000","last_active":"2023-10-09T11:03:21.189223+0000","last_peered":"2023-10-09T11:03:21.189223+0000","last_clean":"2023-10-09T11:03:21.189223+0000","last_became_active":"2023-10-09T11:02:33.196379+0000","last_became_peered":"2023-10-09T11:02:33.196379+0000","last_unstale":"2023-10-09T11:03:21.189223+0000","last_undegraded":"2023-10-09T11:03:21.189223+0000","last_fullsized":"2023-10-09T11:03:21.189223+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:56.935603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":552,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.150655+0000","last_change":"2023-10-09T11:03:09.266080+0000","last_active":"2023-10-09T11:03:20.150655+0000","last_peered":"2023-10-09T11:03:20.150655+0000","last_clean":"2023-10-09T11:02:31.327868+0000","last_became_active":"2023-10-09T11:03:09.266080+0000","last_became_peered":"2023-10-09T11:03:09.266080+0000","last_unstale":"2023-10-09T11:03:20.150655+0000","last_undegraded":"2023-10-09T11:03:20.150655+0000","last_fullsized":"2023-10-09T11:03:09.256067+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:57:50.493750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":727,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.166266+0000","last_change":"2023-10-09T11:02:36.216877+0000","last_active":"2023-10-09T11:03:21.166266+0000","last_peered":"2023-10-09T11:03:21.166266+0000","last_clean":"2023-10-09T11:03:21.166266+0000","last_became_active":"2023-10-09T11:02:36.216072+0000","last_became_peered":"2023-10-09T11:02:36.216072+0000","last_unstale":"2023-10-09T11:03:21.166266+0000","last_undegraded":"2023-10-09T11:03:21.166266+0000","last_fullsized":"2023-10-09T11:03:21.166266+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:47.741775+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":402,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.144964+0000","last_change":"2023-10-09T11:03:09.266847+0000","last_active":"2023-10-09T11:03:20.144964+0000","last_peered":"2023-10-09T11:03:20.144964+0000","last_clean":"2023-10-09T11:03:05.229848+0000","last_became_active":"2023-10-09T11:03:09.266847+0000","last_became_peered":"2023-10-09T11:03:09.266847+0000","last_unstale":"2023-10-09T11:03:20.144964+0000","last_undegraded":"2023-10-09T11:03:20.144964+0000","last_fullsized":"2023-10-09T11:03:09.258630+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:22.203107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":347,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.169342+0000","last_change":"2023-10-09T11:03:09.264870+0000","last_active":"2023-10-09T11:03:21.169342+0000","last_peered":"2023-10-09T11:03:21.169342+0000","last_clean":"2023-10-09T11:03:05.215416+0000","last_became_active":"2023-10-09T11:03:09.264870+0000","last_became_peered":"2023-10-09T11:03:09.264870+0000","last_unstale":"2023-10-09T11:03:21.169342+0000","last_undegraded":"2023-10-09T11:03:21.169342+0000","last_fullsized":"2023-10-09T11:03:09.255410+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:57:21.229073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.18","version":"455'7097","reported_seq":12088,"reported_epoch":455,"state":"active+recovering","last_fresh":"2023-10-09T11:03:23.103446+0000","last_change":"2023-10-09T11:03:09.673531+0000","last_active":"2023-10-09T11:03:23.103446+0000","last_peered":"2023-10-09T11:03:23.103446+0000","last_clean":"2023-10-09T11:02:32.305733+0000","last_became_active":"2023-10-09T11:02:49.395614+0000","last_became_peered":"2023-10-09T11:02:49.395614+0000","last_unstale":"2023-10-09T11:03:23.103446+0000","last_undegraded":"2023-10-09T11:03:23.103446+0000","last_fullsized":"2023-10-09T11:03:23.103446+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":7097,"log_dups_size":0,"ondisk_log_size":7097,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146014208,"num_objects":2228,"num_object_clones":0,"num_object_copies":6684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1377,"num_objects_unfound":0,"num_objects_dirty":2228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3990,"num_write_kb":198976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1884,"num_bytes_recovered":123437056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":1377},{"shards":"0,2,7","objects":851}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":753,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.151190+0000","last_change":"2023-10-09T11:03:09.275353+0000","last_active":"2023-10-09T11:03:20.151190+0000","last_peered":"2023-10-09T11:03:20.151190+0000","last_clean":"2023-10-09T11:03:05.205190+0000","last_became_active":"2023-10-09T11:03:09.275353+0000","last_became_peered":"2023-10-09T11:03:09.275353+0000","last_unstale":"2023-10-09T11:03:20.151190+0000","last_undegraded":"2023-10-09T11:03:20.151190+0000","last_fullsized":"2023-10-09T11:03:09.254654+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:12:34.157173+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":31,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.190921+0000","last_change":"2023-10-09T11:02:33.198572+0000","last_active":"2023-10-09T11:03:21.190921+0000","last_peered":"2023-10-09T11:03:21.190921+0000","last_clean":"2023-10-09T11:03:21.190921+0000","last_became_active":"2023-10-09T11:02:33.198218+0000","last_became_peered":"2023-10-09T11:02:33.198218+0000","last_unstale":"2023-10-09T11:03:21.190921+0000","last_undegraded":"2023-10-09T11:03:21.190921+0000","last_fullsized":"2023-10-09T11:03:21.190921+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:59.066866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":115,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.147697+0000","last_change":"2023-10-09T11:03:09.267441+0000","last_active":"2023-10-09T11:03:20.147697+0000","last_peered":"2023-10-09T11:03:20.147697+0000","last_clean":"2023-10-09T11:03:05.230770+0000","last_became_active":"2023-10-09T11:03:09.267441+0000","last_became_peered":"2023-10-09T11:03:09.267441+0000","last_unstale":"2023-10-09T11:03:20.147697+0000","last_undegraded":"2023-10-09T11:03:20.147697+0000","last_fullsized":"2023-10-09T11:03:09.259745+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:22:04.051853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"456'7172","reported_seq":10383,"reported_epoch":456,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:03:26.568437+0000","last_change":"2023-10-09T11:02:51.366454+0000","last_active":"2023-10-09T11:03:26.568437+0000","last_peered":"2023-10-09T11:03:26.568437+0000","last_clean":"2023-10-09T11:02:32.272642+0000","last_became_active":"2023-10-09T11:02:51.345927+0000","last_became_peered":"2023-10-09T11:02:51.345927+0000","last_unstale":"2023-10-09T11:03:26.568437+0000","last_undegraded":"2023-10-09T11:02:51.338438+0000","last_fullsized":"2023-10-09T11:02:51.334387+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":7172,"log_dups_size":0,"ondisk_log_size":7172,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133464064,"num_objects":2037,"num_object_clones":0,"num_object_copies":6111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2546,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4103,"num_write_kb":196448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":777,"num_bytes_recovered":50921472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":1712},{"shards":"2,5","objects":-509}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":724,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.167182+0000","last_change":"2023-10-09T11:01:15.406928+0000","last_active":"2023-10-09T11:03:21.167182+0000","last_peered":"2023-10-09T11:03:21.167182+0000","last_clean":"2023-10-09T11:03:21.167182+0000","last_became_active":"2023-10-09T11:01:15.406499+0000","last_became_peered":"2023-10-09T11:01:15.406499+0000","last_unstale":"2023-10-09T11:03:21.167182+0000","last_undegraded":"2023-10-09T11:03:21.167182+0000","last_fullsized":"2023-10-09T11:03:21.167182+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:33.050336+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":183,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.181870+0000","last_change":"2023-10-09T11:00:48.109643+0000","last_active":"2023-10-09T11:03:21.181870+0000","last_peered":"2023-10-09T11:03:21.181870+0000","last_clean":"2023-10-09T11:03:21.181870+0000","last_became_active":"2023-10-09T11:00:48.109299+0000","last_became_peered":"2023-10-09T11:00:48.109299+0000","last_unstale":"2023-10-09T11:03:21.181870+0000","last_undegraded":"2023-10-09T11:03:21.181870+0000","last_fullsized":"2023-10-09T11:03:21.181870+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:31.163237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":530,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.190599+0000","last_change":"2023-10-09T11:03:09.261887+0000","last_active":"2023-10-09T11:03:21.190599+0000","last_peered":"2023-10-09T11:03:21.190599+0000","last_clean":"2023-10-09T11:02:31.327787+0000","last_became_active":"2023-10-09T11:03:09.261887+0000","last_became_peered":"2023-10-09T11:03:09.261887+0000","last_unstale":"2023-10-09T11:03:21.190599+0000","last_undegraded":"2023-10-09T11:03:21.190599+0000","last_fullsized":"2023-10-09T11:03:09.257365+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:18.509878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"456'7241","reported_seq":7313,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-09T11:03:26.580397+0000","last_change":"2023-10-09T11:02:49.333682+0000","last_active":"2023-10-09T11:03:26.580397+0000","last_peered":"2023-10-09T11:03:26.580397+0000","last_clean":"2023-10-09T11:03:26.580397+0000","last_became_active":"2023-10-09T11:02:49.327718+0000","last_became_peered":"2023-10-09T11:02:49.327718+0000","last_unstale":"2023-10-09T11:03:26.580397+0000","last_undegraded":"2023-10-09T11:03:26.580397+0000","last_fullsized":"2023-10-09T11:03:26.580397+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":7241,"log_dups_size":0,"ondisk_log_size":7241,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141361152,"num_objects":2157,"num_object_clones":0,"num_object_copies":6471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4109,"num_write_kb":200512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":745,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.189732+0000","last_change":"2023-10-09T11:02:36.217050+0000","last_active":"2023-10-09T11:03:21.189732+0000","last_peered":"2023-10-09T11:03:21.189732+0000","last_clean":"2023-10-09T11:03:21.189732+0000","last_became_active":"2023-10-09T11:02:36.216387+0000","last_became_peered":"2023-10-09T11:02:36.216387+0000","last_unstale":"2023-10-09T11:03:21.189732+0000","last_undegraded":"2023-10-09T11:03:21.189732+0000","last_fullsized":"2023-10-09T11:03:21.189732+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:19.627177+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":35,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.190329+0000","last_change":"2023-10-09T11:02:33.221990+0000","last_active":"2023-10-09T11:03:21.190329+0000","last_peered":"2023-10-09T11:03:21.190329+0000","last_clean":"2023-10-09T11:03:21.190329+0000","last_became_active":"2023-10-09T11:02:33.198155+0000","last_became_peered":"2023-10-09T11:02:33.198155+0000","last_unstale":"2023-10-09T11:03:21.190329+0000","last_undegraded":"2023-10-09T11:03:21.190329+0000","last_fullsized":"2023-10-09T11:03:21.190329+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:32.563344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":378,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.165916+0000","last_change":"2023-10-09T11:02:33.200229+0000","last_active":"2023-10-09T11:03:21.165916+0000","last_peered":"2023-10-09T11:03:21.165916+0000","last_clean":"2023-10-09T11:03:21.165916+0000","last_became_active":"2023-10-09T11:02:33.199936+0000","last_became_peered":"2023-10-09T11:02:33.199936+0000","last_unstale":"2023-10-09T11:03:21.165916+0000","last_undegraded":"2023-10-09T11:03:21.165916+0000","last_fullsized":"2023-10-09T11:03:21.165916+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:28.854719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"456'7080","reported_seq":7159,"reported_epoch":456,"state":"active+undersized+degraded","last_fresh":"2023-10-09T11:03:26.648419+0000","last_change":"2023-10-09T11:03:09.263747+0000","last_active":"2023-10-09T11:03:26.648419+0000","last_peered":"2023-10-09T11:03:26.648419+0000","last_clean":"2023-10-09T11:03:08.215039+0000","last_became_active":"2023-10-09T11:03:09.263747+0000","last_became_peered":"2023-10-09T11:03:09.263747+0000","last_unstale":"2023-10-09T11:03:26.648419+0000","last_undegraded":"2023-10-09T11:03:09.257262+0000","last_fullsized":"2023-10-09T11:03:09.257156+0000","mapping_epoch":449,"log_start":"406'100","ondisk_log_start":"406'100","created":403,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6980,"log_dups_size":0,"ondisk_log_size":6980,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136675328,"num_objects":2085,"num_object_clones":0,"num_object_copies":6255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2085,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4025,"num_write_kb":195552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":2085}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":774,"reported_epoch":454,"state":"active+undersized+degraded","last_fresh":"2023-10-09T11:03:20.147359+0000","last_change":"2023-10-09T11:03:09.270315+0000","last_active":"2023-10-09T11:03:20.147359+0000","last_peered":"2023-10-09T11:03:20.147359+0000","last_clean":"2023-10-09T10:59:45.168592+0000","last_became_active":"2023-10-09T11:03:09.270315+0000","last_became_peered":"2023-10-09T11:03:09.270315+0000","last_unstale":"2023-10-09T11:03:20.147359+0000","last_undegraded":"2023-10-09T11:03:09.260434+0000","last_fullsized":"2023-10-09T11:03:09.260335+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:00.106259+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":35,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.190052+0000","last_change":"2023-10-09T11:02:33.208705+0000","last_active":"2023-10-09T11:03:21.190052+0000","last_peered":"2023-10-09T11:03:21.190052+0000","last_clean":"2023-10-09T11:03:21.190052+0000","last_became_active":"2023-10-09T11:02:33.197454+0000","last_became_peered":"2023-10-09T11:02:33.197454+0000","last_unstale":"2023-10-09T11:03:21.190052+0000","last_undegraded":"2023-10-09T11:03:21.190052+0000","last_fullsized":"2023-10-09T11:03:21.190052+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:52.649912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":140,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.150086+0000","last_change":"2023-10-09T11:03:09.281623+0000","last_active":"2023-10-09T11:03:20.150086+0000","last_peered":"2023-10-09T11:03:20.150086+0000","last_clean":"2023-10-09T11:03:05.204929+0000","last_became_active":"2023-10-09T11:03:09.281623+0000","last_became_peered":"2023-10-09T11:03:09.281623+0000","last_unstale":"2023-10-09T11:03:20.150086+0000","last_undegraded":"2023-10-09T11:03:20.150086+0000","last_fullsized":"2023-10-09T11:03:09.256767+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:22.540679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"455'6972","reported_seq":7048,"reported_epoch":455,"state":"active+undersized+degraded","last_fresh":"2023-10-09T11:03:23.052703+0000","last_change":"2023-10-09T11:03:09.275909+0000","last_active":"2023-10-09T11:03:23.052703+0000","last_peered":"2023-10-09T11:03:23.052703+0000","last_clean":"2023-10-09T11:03:08.252655+0000","last_became_active":"2023-10-09T11:03:09.275909+0000","last_became_peered":"2023-10-09T11:03:09.275909+0000","last_unstale":"2023-10-09T11:03:23.052703+0000","last_undegraded":"2023-10-09T11:03:09.259588+0000","last_fullsized":"2023-10-09T11:03:09.259477+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6972,"log_dups_size":0,"ondisk_log_size":6972,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140967936,"num_objects":2151,"num_object_clones":0,"num_object_copies":6453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2151,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2151,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3931,"num_write_kb":194624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":2151}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":652,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.170119+0000","last_change":"2023-10-09T11:01:22.599166+0000","last_active":"2023-10-09T11:03:21.170119+0000","last_peered":"2023-10-09T11:03:21.170119+0000","last_clean":"2023-10-09T11:03:21.170119+0000","last_became_active":"2023-10-09T11:01:15.402193+0000","last_became_peered":"2023-10-09T11:01:15.402193+0000","last_unstale":"2023-10-09T11:03:21.170119+0000","last_undegraded":"2023-10-09T11:03:21.170119+0000","last_fullsized":"2023-10-09T11:03:21.170119+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:37:16.611219+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":35,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.189652+0000","last_change":"2023-10-09T11:02:33.223801+0000","last_active":"2023-10-09T11:03:21.189652+0000","last_peered":"2023-10-09T11:03:21.189652+0000","last_clean":"2023-10-09T11:03:21.189652+0000","last_became_active":"2023-10-09T11:02:33.198699+0000","last_became_peered":"2023-10-09T11:02:33.198699+0000","last_unstale":"2023-10-09T11:03:21.189652+0000","last_undegraded":"2023-10-09T11:03:21.189652+0000","last_fullsized":"2023-10-09T11:03:21.189652+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:13.818599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":305,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.182781+0000","last_change":"2023-10-09T11:02:33.199120+0000","last_active":"2023-10-09T11:03:21.182781+0000","last_peered":"2023-10-09T11:03:21.182781+0000","last_clean":"2023-10-09T11:03:21.182781+0000","last_became_active":"2023-10-09T11:02:33.198673+0000","last_became_peered":"2023-10-09T11:02:33.198673+0000","last_unstale":"2023-10-09T11:03:21.182781+0000","last_undegraded":"2023-10-09T11:03:21.182781+0000","last_fullsized":"2023-10-09T11:03:21.182781+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:58:06.417132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"456'7273","reported_seq":7349,"reported_epoch":456,"state":"active+undersized+degraded","last_fresh":"2023-10-09T11:03:26.110570+0000","last_change":"2023-10-09T11:03:09.266774+0000","last_active":"2023-10-09T11:03:26.110570+0000","last_peered":"2023-10-09T11:03:26.110570+0000","last_clean":"2023-10-09T11:03:08.250774+0000","last_became_active":"2023-10-09T11:03:09.266774+0000","last_became_peered":"2023-10-09T11:03:09.266774+0000","last_unstale":"2023-10-09T11:03:26.110570+0000","last_undegraded":"2023-10-09T11:03:09.254668+0000","last_fullsized":"2023-10-09T11:03:09.254565+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":7273,"log_dups_size":0,"ondisk_log_size":7273,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136282112,"num_objects":2079,"num_object_clones":0,"num_object_copies":6237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2079,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4155,"num_write_kb":199520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":2079}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":988,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.180040+0000","last_change":"2023-10-09T11:03:09.282175+0000","last_active":"2023-10-09T11:03:21.180040+0000","last_peered":"2023-10-09T11:03:21.180040+0000","last_clean":"2023-10-09T11:03:05.229650+0000","last_became_active":"2023-10-09T11:03:09.282175+0000","last_became_peered":"2023-10-09T11:03:09.282175+0000","last_unstale":"2023-10-09T11:03:21.180040+0000","last_undegraded":"2023-10-09T11:03:21.180040+0000","last_fullsized":"2023-10-09T11:03:09.276936+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:20:29.873097+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":274,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.166589+0000","last_change":"2023-10-09T11:03:09.263724+0000","last_active":"2023-10-09T11:03:21.166589+0000","last_peered":"2023-10-09T11:03:21.166589+0000","last_clean":"2023-10-09T11:03:05.215223+0000","last_became_active":"2023-10-09T11:03:09.263724+0000","last_became_peered":"2023-10-09T11:03:09.263724+0000","last_unstale":"2023-10-09T11:03:21.166589+0000","last_undegraded":"2023-10-09T11:03:21.166589+0000","last_fullsized":"2023-10-09T11:03:09.255424+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:39:36.377531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":376,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T11:03:20.144837+0000","last_change":"2023-10-09T11:00:15.865848+0000","last_active":"2023-10-09T11:03:20.144837+0000","last_peered":"2023-10-09T11:03:20.144837+0000","last_clean":"2023-10-09T11:03:20.144837+0000","last_became_active":"2023-10-09T11:00:15.865504+0000","last_became_peered":"2023-10-09T11:00:15.865504+0000","last_unstale":"2023-10-09T11:03:20.144837+0000","last_undegraded":"2023-10-09T11:03:20.144837+0000","last_fullsized":"2023-10-09T11:03:20.144837+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:32.352564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"455'7079","reported_seq":7154,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:23.032032+0000","last_change":"2023-10-09T11:03:18.676117+0000","last_active":"2023-10-09T11:03:23.032032+0000","last_peered":"2023-10-09T11:03:23.032032+0000","last_clean":"2023-10-09T11:03:23.032032+0000","last_became_active":"2023-10-09T11:02:49.340473+0000","last_became_peered":"2023-10-09T11:02:49.340473+0000","last_unstale":"2023-10-09T11:03:23.032032+0000","last_undegraded":"2023-10-09T11:03:23.032032+0000","last_fullsized":"2023-10-09T11:03:23.032032+0000","mapping_epoch":403,"log_start":"406'500","ondisk_log_start":"406'500","created":403,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6579,"log_dups_size":0,"ondisk_log_size":6579,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141230080,"num_objects":2155,"num_object_clones":0,"num_object_copies":6465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4001,"num_write_kb":196992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":724,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.191092+0000","last_change":"2023-10-09T11:03:09.266657+0000","last_active":"2023-10-09T11:03:21.191092+0000","last_peered":"2023-10-09T11:03:21.191092+0000","last_clean":"2023-10-09T11:03:05.219409+0000","last_became_active":"2023-10-09T11:03:09.266657+0000","last_became_peered":"2023-10-09T11:03:09.266657+0000","last_unstale":"2023-10-09T11:03:21.191092+0000","last_undegraded":"2023-10-09T11:03:21.191092+0000","last_fullsized":"2023-10-09T11:03:09.260124+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:54.562764+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":136,"reported_epoch":454,"state":"active+undersized+degraded","last_fresh":"2023-10-09T11:03:20.144466+0000","last_change":"2023-10-09T11:03:09.275814+0000","last_active":"2023-10-09T11:03:20.144466+0000","last_peered":"2023-10-09T11:03:20.144466+0000","last_clean":"2023-10-09T10:45:59.235058+0000","last_became_active":"2023-10-09T11:03:09.275814+0000","last_became_peered":"2023-10-09T11:03:09.275814+0000","last_unstale":"2023-10-09T11:03:20.144466+0000","last_undegraded":"2023-10-09T11:03:09.259518+0000","last_fullsized":"2023-10-09T11:03:09.259413+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:07.711992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":201,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.180708+0000","last_change":"2023-10-09T11:03:09.278997+0000","last_active":"2023-10-09T11:03:21.180708+0000","last_peered":"2023-10-09T11:03:21.180708+0000","last_clean":"2023-10-09T10:43:06.671321+0000","last_became_active":"2023-10-09T11:03:09.278997+0000","last_became_peered":"2023-10-09T11:03:09.278997+0000","last_unstale":"2023-10-09T11:03:21.180708+0000","last_undegraded":"2023-10-09T11:03:21.180708+0000","last_fullsized":"2023-10-09T11:03:09.272546+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:35.685491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"456'7181","reported_seq":7260,"reported_epoch":456,"state":"active+undersized+degraded","last_fresh":"2023-10-09T11:03:26.089781+0000","last_change":"2023-10-09T11:03:09.261479+0000","last_active":"2023-10-09T11:03:26.089781+0000","last_peered":"2023-10-09T11:03:26.089781+0000","last_clean":"2023-10-09T11:03:08.233386+0000","last_became_active":"2023-10-09T11:03:09.261479+0000","last_became_peered":"2023-10-09T11:03:09.261479+0000","last_unstale":"2023-10-09T11:03:26.089781+0000","last_undegraded":"2023-10-09T11:03:09.253126+0000","last_fullsized":"2023-10-09T11:03:09.253011+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":7181,"log_dups_size":0,"ondisk_log_size":7181,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130908160,"num_objects":1997,"num_object_clones":0,"num_object_copies":5991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1997,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4121,"num_write_kb":195808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":1997}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":654,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.166140+0000","last_change":"2023-10-09T11:03:09.258127+0000","last_active":"2023-10-09T11:03:21.166140+0000","last_peered":"2023-10-09T11:03:21.166140+0000","last_clean":"2023-10-09T11:03:05.216315+0000","last_became_active":"2023-10-09T11:03:09.258127+0000","last_became_peered":"2023-10-09T11:03:09.258127+0000","last_unstale":"2023-10-09T11:03:21.166140+0000","last_undegraded":"2023-10-09T11:03:21.166140+0000","last_fullsized":"2023-10-09T11:03:09.254330+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:17.048230+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":452,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.177536+0000","last_change":"2023-10-09T11:00:14.891176+0000","last_active":"2023-10-09T11:03:21.177536+0000","last_peered":"2023-10-09T11:03:21.177536+0000","last_clean":"2023-10-09T11:03:21.177536+0000","last_became_active":"2023-10-09T11:00:14.890890+0000","last_became_peered":"2023-10-09T11:00:14.890890+0000","last_unstale":"2023-10-09T11:03:21.177536+0000","last_undegraded":"2023-10-09T11:03:21.177536+0000","last_fullsized":"2023-10-09T11:03:21.177536+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:38.158759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":213,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.179590+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:03:21.179590+0000","last_peered":"2023-10-09T11:03:21.179590+0000","last_clean":"2023-10-09T11:03:21.179590+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:03:21.179590+0000","last_undegraded":"2023-10-09T11:03:21.179590+0000","last_fullsized":"2023-10-09T11:03:21.179590+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"456'7102","reported_seq":7301,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-09T11:03:26.470781+0000","last_change":"2023-10-09T11:02:53.856062+0000","last_active":"2023-10-09T11:03:26.470781+0000","last_peered":"2023-10-09T11:03:26.470781+0000","last_clean":"2023-10-09T11:03:26.470781+0000","last_became_active":"2023-10-09T11:02:49.339201+0000","last_became_peered":"2023-10-09T11:02:49.339201+0000","last_unstale":"2023-10-09T11:03:26.470781+0000","last_undegraded":"2023-10-09T11:03:26.470781+0000","last_fullsized":"2023-10-09T11:03:26.470781+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"445'6103","last_scrub_stamp":"2023-10-09T11:02:53.855963+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:02:53.855963+0000","objects_scrubbed":2932,"log_size":7102,"log_dups_size":0,"ondisk_log_size":7102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:43.650276+0000","scrub_duration":0.83763310899999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126091264,"num_objects":1924,"num_object_clones":0,"num_object_copies":5772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1008,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":804,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T11:03:20.151327+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:03:20.151327+0000","last_peered":"2023-10-09T11:03:20.151327+0000","last_clean":"2023-10-09T11:03:20.151327+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:03:20.151327+0000","last_undegraded":"2023-10-09T11:03:20.151327+0000","last_fullsized":"2023-10-09T11:03:20.151327+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":531,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.177692+0000","last_change":"2023-10-09T11:02:33.200106+0000","last_active":"2023-10-09T11:03:21.177692+0000","last_peered":"2023-10-09T11:03:21.177692+0000","last_clean":"2023-10-09T11:03:21.177692+0000","last_became_active":"2023-10-09T11:02:33.199840+0000","last_became_peered":"2023-10-09T11:02:33.199840+0000","last_unstale":"2023-10-09T11:03:21.177692+0000","last_undegraded":"2023-10-09T11:03:21.177692+0000","last_fullsized":"2023-10-09T11:03:21.177692+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:13:17.557980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":134,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T11:03:20.151632+0000","last_change":"2023-10-09T11:00:14.900130+0000","last_active":"2023-10-09T11:03:20.151632+0000","last_peered":"2023-10-09T11:03:20.151632+0000","last_clean":"2023-10-09T11:03:20.151632+0000","last_became_active":"2023-10-09T11:00:14.899536+0000","last_became_peered":"2023-10-09T11:00:14.899536+0000","last_unstale":"2023-10-09T11:03:20.151632+0000","last_undegraded":"2023-10-09T11:03:20.151632+0000","last_fullsized":"2023-10-09T11:03:20.151632+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:07.151510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"455'7193","reported_seq":9556,"reported_epoch":455,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:03:23.074369+0000","last_change":"2023-10-09T11:03:09.727597+0000","last_active":"2023-10-09T11:03:23.074369+0000","last_peered":"2023-10-09T11:03:23.074369+0000","last_clean":"2023-10-09T11:02:32.284069+0000","last_became_active":"2023-10-09T11:02:49.355202+0000","last_became_peered":"2023-10-09T11:02:49.355202+0000","last_unstale":"2023-10-09T11:03:23.074369+0000","last_undegraded":"2023-10-09T11:03:23.074369+0000","last_fullsized":"2023-10-09T11:03:23.074369+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":7193,"log_dups_size":0,"ondisk_log_size":7193,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141557760,"num_objects":2160,"num_object_clones":0,"num_object_copies":6480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2160,"num_objects_unfound":0,"num_objects_dirty":2160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4076,"num_write_kb":199552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":567,"num_bytes_recovered":37191680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[0,4,1],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2160}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":658,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.167107+0000","last_change":"2023-10-09T11:01:51.463338+0000","last_active":"2023-10-09T11:03:21.167107+0000","last_peered":"2023-10-09T11:03:21.167107+0000","last_clean":"2023-10-09T11:03:21.167107+0000","last_became_active":"2023-10-09T11:01:15.399431+0000","last_became_peered":"2023-10-09T11:01:15.399431+0000","last_unstale":"2023-10-09T11:03:21.167107+0000","last_undegraded":"2023-10-09T11:03:21.167107+0000","last_fullsized":"2023-10-09T11:03:21.167107+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:01:07.923413+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":521,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.152055+0000","last_change":"2023-10-09T11:03:09.265754+0000","last_active":"2023-10-09T11:03:20.152055+0000","last_peered":"2023-10-09T11:03:20.152055+0000","last_clean":"2023-10-09T11:02:31.326713+0000","last_became_active":"2023-10-09T11:03:09.265754+0000","last_became_peered":"2023-10-09T11:03:09.265754+0000","last_unstale":"2023-10-09T11:03:20.152055+0000","last_undegraded":"2023-10-09T11:03:20.152055+0000","last_fullsized":"2023-10-09T11:03:09.255954+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:13.302768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":140,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.152204+0000","last_change":"2023-10-09T11:03:09.275217+0000","last_active":"2023-10-09T11:03:20.152204+0000","last_peered":"2023-10-09T11:03:20.152204+0000","last_clean":"2023-10-09T11:03:05.208923+0000","last_became_active":"2023-10-09T11:03:09.275217+0000","last_became_peered":"2023-10-09T11:03:09.275217+0000","last_unstale":"2023-10-09T11:03:20.152204+0000","last_undegraded":"2023-10-09T11:03:20.152204+0000","last_fullsized":"2023-10-09T11:03:09.256615+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:23.586519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"455'8043","reported_seq":10614,"reported_epoch":455,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-10-09T11:03:23.086396+0000","last_change":"2023-10-09T11:03:09.288691+0000","last_active":"2023-10-09T11:03:23.086396+0000","last_peered":"2023-10-09T11:03:23.086396+0000","last_clean":"2023-10-09T11:02:32.310266+0000","last_became_active":"2023-10-09T11:03:09.276966+0000","last_became_peered":"2023-10-09T11:03:09.276966+0000","last_unstale":"2023-10-09T11:03:23.086396+0000","last_undegraded":"2023-10-09T11:03:09.260798+0000","last_fullsized":"2023-10-09T11:03:09.260697+0000","mapping_epoch":449,"log_start":"415'2187","ondisk_log_start":"415'2187","created":403,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5856,"log_dups_size":0,"ondisk_log_size":5856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:51.962207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279773184,"num_objects":4269,"num_object_clones":0,"num_object_copies":12807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4198,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8043,"num_write_kb":393984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1239,"num_bytes_recovered":81199104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":4269}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":904,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.190451+0000","last_change":"2023-10-09T11:03:09.263121+0000","last_active":"2023-10-09T11:03:21.190451+0000","last_peered":"2023-10-09T11:03:21.190451+0000","last_clean":"2023-10-09T11:02:31.327495+0000","last_became_active":"2023-10-09T11:03:09.263121+0000","last_became_peered":"2023-10-09T11:03:09.263121+0000","last_unstale":"2023-10-09T11:03:21.190451+0000","last_undegraded":"2023-10-09T11:03:21.190451+0000","last_fullsized":"2023-10-09T11:03:09.257052+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:23.277748+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":164,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T11:03:20.150809+0000","last_change":"2023-10-09T11:02:33.199808+0000","last_active":"2023-10-09T11:03:20.150809+0000","last_peered":"2023-10-09T11:03:20.150809+0000","last_clean":"2023-10-09T11:03:20.150809+0000","last_became_active":"2023-10-09T11:02:33.199332+0000","last_became_peered":"2023-10-09T11:02:33.199332+0000","last_unstale":"2023-10-09T11:03:20.150809+0000","last_undegraded":"2023-10-09T11:03:20.150809+0000","last_fullsized":"2023-10-09T11:03:20.150809+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:23:19.999621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":301,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.166907+0000","last_change":"2023-10-09T11:03:09.258867+0000","last_active":"2023-10-09T11:03:21.166907+0000","last_peered":"2023-10-09T11:03:21.166907+0000","last_clean":"2023-10-09T11:03:05.215341+0000","last_became_active":"2023-10-09T11:03:09.258867+0000","last_became_peered":"2023-10-09T11:03:09.258867+0000","last_unstale":"2023-10-09T11:03:21.166907+0000","last_undegraded":"2023-10-09T11:03:21.166907+0000","last_fullsized":"2023-10-09T11:03:09.254844+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:10.115269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":134,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T11:03:20.153448+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:03:20.153448+0000","last_peered":"2023-10-09T11:03:20.153448+0000","last_clean":"2023-10-09T11:03:20.153448+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:03:20.153448+0000","last_undegraded":"2023-10-09T11:03:20.153448+0000","last_fullsized":"2023-10-09T11:03:20.153448+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"456'8191","reported_seq":8252,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-09T11:03:26.444568+0000","last_change":"2023-10-09T11:01:15.406192+0000","last_active":"2023-10-09T11:03:26.444568+0000","last_peered":"2023-10-09T11:03:26.444568+0000","last_clean":"2023-10-09T11:03:26.444568+0000","last_became_active":"2023-10-09T11:01:15.405811+0000","last_became_peered":"2023-10-09T11:01:15.405811+0000","last_unstale":"2023-10-09T11:03:26.444568+0000","last_undegraded":"2023-10-09T11:03:26.444568+0000","last_fullsized":"2023-10-09T11:03:26.444568+0000","mapping_epoch":403,"log_start":"415'2000","ondisk_log_start":"415'2000","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6191,"log_dups_size":0,"ondisk_log_size":6191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:48:38.805496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266797056,"num_objects":4071,"num_object_clones":0,"num_object_copies":12213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8191,"num_write_kb":392384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":656,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.171366+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:03:21.171366+0000","last_peered":"2023-10-09T11:03:21.171366+0000","last_clean":"2023-10-09T11:03:21.171366+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:03:21.171366+0000","last_undegraded":"2023-10-09T11:03:21.171366+0000","last_fullsized":"2023-10-09T11:03:21.171366+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":318,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.166748+0000","last_change":"2023-10-09T11:01:15.401697+0000","last_active":"2023-10-09T11:03:21.166748+0000","last_peered":"2023-10-09T11:03:21.166748+0000","last_clean":"2023-10-09T11:03:21.166748+0000","last_became_active":"2023-10-09T11:01:15.401386+0000","last_became_peered":"2023-10-09T11:01:15.401386+0000","last_unstale":"2023-10-09T11:03:21.166748+0000","last_undegraded":"2023-10-09T11:03:21.166748+0000","last_fullsized":"2023-10-09T11:03:21.166748+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:40.572070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":362,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T11:03:20.144582+0000","last_change":"2023-10-09T10:59:49.126425+0000","last_active":"2023-10-09T11:03:20.144582+0000","last_peered":"2023-10-09T11:03:20.144582+0000","last_clean":"2023-10-09T11:03:20.144582+0000","last_became_active":"2023-10-09T10:59:49.125712+0000","last_became_peered":"2023-10-09T10:59:49.125712+0000","last_unstale":"2023-10-09T11:03:20.144582+0000","last_undegraded":"2023-10-09T11:03:20.144582+0000","last_fullsized":"2023-10-09T11:03:20.144582+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:40:53.898922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":140,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.150554+0000","last_change":"2023-10-09T11:03:09.257327+0000","last_active":"2023-10-09T11:03:20.150554+0000","last_peered":"2023-10-09T11:03:20.150554+0000","last_clean":"2023-10-09T11:03:05.207636+0000","last_became_active":"2023-10-09T11:03:09.257327+0000","last_became_peered":"2023-10-09T11:03:09.257327+0000","last_unstale":"2023-10-09T11:03:20.150554+0000","last_undegraded":"2023-10-09T11:03:20.150554+0000","last_fullsized":"2023-10-09T11:03:09.255491+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:12.404201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"456'7156","reported_seq":15328,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-09T11:03:26.539668+0000","last_change":"2023-10-09T11:03:11.311056+0000","last_active":"2023-10-09T11:03:26.539668+0000","last_peered":"2023-10-09T11:03:26.539668+0000","last_clean":"2023-10-09T11:03:26.539668+0000","last_became_active":"2023-10-09T11:03:11.282004+0000","last_became_peered":"2023-10-09T11:03:11.282004+0000","last_unstale":"2023-10-09T11:03:26.539668+0000","last_undegraded":"2023-10-09T11:03:26.539668+0000","last_fullsized":"2023-10-09T11:03:26.539668+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":7156,"log_dups_size":0,"ondisk_log_size":7156,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134512640,"num_objects":2052,"num_object_clones":0,"num_object_copies":6156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4086,"num_write_kb":196448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3253,"num_bytes_recovered":213188608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":891,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.188969+0000","last_change":"2023-10-09T11:02:36.216448+0000","last_active":"2023-10-09T11:03:21.188969+0000","last_peered":"2023-10-09T11:03:21.188969+0000","last_clean":"2023-10-09T11:03:21.188969+0000","last_became_active":"2023-10-09T11:02:36.216040+0000","last_became_peered":"2023-10-09T11:02:36.216040+0000","last_unstale":"2023-10-09T11:03:21.188969+0000","last_undegraded":"2023-10-09T11:03:21.188969+0000","last_fullsized":"2023-10-09T11:03:21.188969+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:13:40.104907+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":214,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.177818+0000","last_change":"2023-10-09T11:03:09.263117+0000","last_active":"2023-10-09T11:03:21.177818+0000","last_peered":"2023-10-09T11:03:21.177818+0000","last_clean":"2023-10-09T11:03:05.236503+0000","last_became_active":"2023-10-09T11:03:09.263117+0000","last_became_peered":"2023-10-09T11:03:09.263117+0000","last_unstale":"2023-10-09T11:03:21.177818+0000","last_undegraded":"2023-10-09T11:03:21.177818+0000","last_fullsized":"2023-10-09T11:03:09.253283+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:58:43.579325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":134,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T11:03:20.150058+0000","last_change":"2023-10-09T11:00:14.897166+0000","last_active":"2023-10-09T11:03:20.150058+0000","last_peered":"2023-10-09T11:03:20.150058+0000","last_clean":"2023-10-09T11:03:20.150058+0000","last_became_active":"2023-10-09T11:00:14.896917+0000","last_became_peered":"2023-10-09T11:00:14.896917+0000","last_unstale":"2023-10-09T11:03:20.150058+0000","last_undegraded":"2023-10-09T11:03:20.150058+0000","last_fullsized":"2023-10-09T11:03:20.150058+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:09.954705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":132,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T11:03:20.152012+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:03:20.152012+0000","last_peered":"2023-10-09T11:03:20.152012+0000","last_clean":"2023-10-09T11:03:20.152012+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:03:20.152012+0000","last_undegraded":"2023-10-09T11:03:20.152012+0000","last_fullsized":"2023-10-09T11:03:20.152012+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"455'8049","reported_seq":8110,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:23.032522+0000","last_change":"2023-10-09T11:01:15.405163+0000","last_active":"2023-10-09T11:03:23.032522+0000","last_peered":"2023-10-09T11:03:23.032522+0000","last_clean":"2023-10-09T11:03:23.032522+0000","last_became_active":"2023-10-09T11:01:15.404873+0000","last_became_peered":"2023-10-09T11:01:15.404873+0000","last_unstale":"2023-10-09T11:03:23.032522+0000","last_undegraded":"2023-10-09T11:03:23.032522+0000","last_fullsized":"2023-10-09T11:03:23.032522+0000","mapping_epoch":403,"log_start":"415'2000","ondisk_log_start":"415'2000","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6049,"log_dups_size":0,"ondisk_log_size":6049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:08.894661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282525696,"num_objects":4311,"num_object_clones":0,"num_object_copies":12933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8049,"num_write_kb":395520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":311,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.172798+0000","last_change":"2023-10-09T11:02:33.198446+0000","last_active":"2023-10-09T11:03:21.172798+0000","last_peered":"2023-10-09T11:03:21.172798+0000","last_clean":"2023-10-09T11:03:21.172798+0000","last_became_active":"2023-10-09T11:02:33.197858+0000","last_became_peered":"2023-10-09T11:02:33.197858+0000","last_unstale":"2023-10-09T11:03:21.172798+0000","last_undegraded":"2023-10-09T11:03:21.172798+0000","last_fullsized":"2023-10-09T11:03:21.172798+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:17.362664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":836,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.188504+0000","last_change":"2023-10-09T11:02:36.220226+0000","last_active":"2023-10-09T11:03:21.188504+0000","last_peered":"2023-10-09T11:03:21.188504+0000","last_clean":"2023-10-09T11:03:21.188504+0000","last_became_active":"2023-10-09T11:02:36.219889+0000","last_became_peered":"2023-10-09T11:02:36.219889+0000","last_unstale":"2023-10-09T11:03:21.188504+0000","last_undegraded":"2023-10-09T11:03:21.188504+0000","last_fullsized":"2023-10-09T11:03:21.188504+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:48:54.740905+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":29,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.166881+0000","last_change":"2023-10-09T11:03:09.263685+0000","last_active":"2023-10-09T11:03:21.166881+0000","last_peered":"2023-10-09T11:03:21.166881+0000","last_clean":"2023-10-09T10:35:55.799457+0000","last_became_active":"2023-10-09T11:03:09.263685+0000","last_became_peered":"2023-10-09T11:03:09.263685+0000","last_unstale":"2023-10-09T11:03:21.166881+0000","last_undegraded":"2023-10-09T11:03:21.166881+0000","last_fullsized":"2023-10-09T11:03:09.255016+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:24:15.145465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":408,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.180134+0000","last_change":"2023-10-09T11:02:33.200249+0000","last_active":"2023-10-09T11:03:21.180134+0000","last_peered":"2023-10-09T11:03:21.180134+0000","last_clean":"2023-10-09T11:03:21.180134+0000","last_became_active":"2023-10-09T11:02:33.199802+0000","last_became_peered":"2023-10-09T11:02:33.199802+0000","last_unstale":"2023-10-09T11:03:21.180134+0000","last_undegraded":"2023-10-09T11:03:21.180134+0000","last_fullsized":"2023-10-09T11:03:21.180134+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:30.703377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"455'7176","reported_seq":11449,"reported_epoch":455,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T11:03:22.992482+0000","last_change":"2023-10-09T11:02:50.369325+0000","last_active":"2023-10-09T11:03:22.992482+0000","last_peered":"2023-10-09T11:03:22.992482+0000","last_clean":"2023-10-09T11:02:32.305733+0000","last_became_active":"2023-10-09T11:02:50.341571+0000","last_became_peered":"2023-10-09T11:02:50.341571+0000","last_unstale":"2023-10-09T11:03:22.992482+0000","last_undegraded":"2023-10-09T11:02:50.336279+0000","last_fullsized":"2023-10-09T11:03:22.992482+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":403,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":7176,"log_dups_size":0,"ondisk_log_size":7176,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140705792,"num_objects":2147,"num_object_clones":0,"num_object_copies":6441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1845,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4069,"num_write_kb":198912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1524,"num_bytes_recovered":99909632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":1845},{"shards":"0,2,7","objects":302}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":790,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T11:03:20.150958+0000","last_change":"2023-10-09T11:01:17.369788+0000","last_active":"2023-10-09T11:03:20.150958+0000","last_peered":"2023-10-09T11:03:20.150958+0000","last_clean":"2023-10-09T11:03:20.150958+0000","last_became_active":"2023-10-09T11:00:14.881494+0000","last_became_peered":"2023-10-09T11:00:14.881494+0000","last_unstale":"2023-10-09T11:03:20.150958+0000","last_undegraded":"2023-10-09T11:03:20.150958+0000","last_fullsized":"2023-10-09T11:03:20.150958+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:44.208863+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":31,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.188855+0000","last_change":"2023-10-09T11:02:33.198484+0000","last_active":"2023-10-09T11:03:21.188855+0000","last_peered":"2023-10-09T11:03:21.188855+0000","last_clean":"2023-10-09T11:03:21.188855+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:03:21.188855+0000","last_undegraded":"2023-10-09T11:03:21.188855+0000","last_fullsized":"2023-10-09T11:03:21.188855+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":97,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.145920+0000","last_change":"2023-10-09T11:03:09.278851+0000","last_active":"2023-10-09T11:03:20.145920+0000","last_peered":"2023-10-09T11:03:20.145920+0000","last_clean":"2023-10-09T11:03:05.237152+0000","last_became_active":"2023-10-09T11:03:09.278851+0000","last_became_peered":"2023-10-09T11:03:09.278851+0000","last_unstale":"2023-10-09T11:03:20.145920+0000","last_undegraded":"2023-10-09T11:03:20.145920+0000","last_fullsized":"2023-10-09T11:03:09.259535+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:36.068381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":413,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.183337+0000","last_change":"2023-10-09T11:00:15.892896+0000","last_active":"2023-10-09T11:03:21.183337+0000","last_peered":"2023-10-09T11:03:21.183337+0000","last_clean":"2023-10-09T11:03:21.183337+0000","last_became_active":"2023-10-09T11:00:15.892573+0000","last_became_peered":"2023-10-09T11:00:15.892573+0000","last_unstale":"2023-10-09T11:03:21.183337+0000","last_undegraded":"2023-10-09T11:03:21.183337+0000","last_fullsized":"2023-10-09T11:03:21.183337+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:13.280089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"455'8023","reported_seq":8084,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:23.091524+0000","last_change":"2023-10-09T11:01:15.402557+0000","last_active":"2023-10-09T11:03:23.091524+0000","last_peered":"2023-10-09T11:03:23.091524+0000","last_clean":"2023-10-09T11:03:23.091524+0000","last_became_active":"2023-10-09T11:01:15.402279+0000","last_became_peered":"2023-10-09T11:01:15.402279+0000","last_unstale":"2023-10-09T11:03:23.091524+0000","last_undegraded":"2023-10-09T11:03:23.091524+0000","last_fullsized":"2023-10-09T11:03:23.091524+0000","mapping_epoch":403,"log_start":"415'2163","ondisk_log_start":"415'2163","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":5860,"log_dups_size":0,"ondisk_log_size":5860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:19.333267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278134808,"num_objects":4245,"num_object_clones":0,"num_object_copies":12735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4245,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8023,"num_write_kb":392513,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":674,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.147497+0000","last_change":"2023-10-09T11:03:09.270323+0000","last_active":"2023-10-09T11:03:20.147497+0000","last_peered":"2023-10-09T11:03:20.147497+0000","last_clean":"2023-10-09T11:03:05.230684+0000","last_became_active":"2023-10-09T11:03:09.270323+0000","last_became_peered":"2023-10-09T11:03:09.270323+0000","last_unstale":"2023-10-09T11:03:20.147497+0000","last_undegraded":"2023-10-09T11:03:20.147497+0000","last_fullsized":"2023-10-09T11:03:09.259940+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:34.720582+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":31,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.189416+0000","last_change":"2023-10-09T11:02:33.199031+0000","last_active":"2023-10-09T11:03:21.189416+0000","last_peered":"2023-10-09T11:03:21.189416+0000","last_clean":"2023-10-09T11:03:21.189416+0000","last_became_active":"2023-10-09T11:02:33.198251+0000","last_became_peered":"2023-10-09T11:02:33.198251+0000","last_unstale":"2023-10-09T11:03:21.189416+0000","last_undegraded":"2023-10-09T11:03:21.189416+0000","last_fullsized":"2023-10-09T11:03:21.189416+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:21.904005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":134,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T11:03:20.150951+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:03:20.150951+0000","last_peered":"2023-10-09T11:03:20.150951+0000","last_clean":"2023-10-09T11:03:20.150951+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:03:20.150951+0000","last_undegraded":"2023-10-09T11:03:20.150951+0000","last_fullsized":"2023-10-09T11:03:20.150951+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":397,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T11:03:20.144699+0000","last_change":"2023-10-09T11:02:33.198297+0000","last_active":"2023-10-09T11:03:20.144699+0000","last_peered":"2023-10-09T11:03:20.144699+0000","last_clean":"2023-10-09T11:03:20.144699+0000","last_became_active":"2023-10-09T11:02:33.197978+0000","last_became_peered":"2023-10-09T11:02:33.197978+0000","last_unstale":"2023-10-09T11:03:20.144699+0000","last_undegraded":"2023-10-09T11:03:20.144699+0000","last_fullsized":"2023-10-09T11:03:20.144699+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:41.689146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"456'8167","reported_seq":8228,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-09T11:03:26.502676+0000","last_change":"2023-10-09T11:01:15.406858+0000","last_active":"2023-10-09T11:03:26.502676+0000","last_peered":"2023-10-09T11:03:26.502676+0000","last_clean":"2023-10-09T11:03:26.502676+0000","last_became_active":"2023-10-09T11:01:15.406513+0000","last_became_peered":"2023-10-09T11:01:15.406513+0000","last_unstale":"2023-10-09T11:03:26.502676+0000","last_undegraded":"2023-10-09T11:03:26.502676+0000","last_fullsized":"2023-10-09T11:03:26.502676+0000","mapping_epoch":403,"log_start":"415'2000","ondisk_log_start":"415'2000","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6167,"log_dups_size":0,"ondisk_log_size":6167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:28:25.137720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269942784,"num_objects":4119,"num_object_clones":0,"num_object_copies":12357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8167,"num_write_kb":393152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":851,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.180054+0000","last_change":"2023-10-09T10:54:21.167745+0000","last_active":"2023-10-09T11:03:21.180054+0000","last_peered":"2023-10-09T11:03:21.180054+0000","last_clean":"2023-10-09T11:03:21.180054+0000","last_became_active":"2023-10-09T10:54:21.166916+0000","last_became_peered":"2023-10-09T10:54:21.166916+0000","last_unstale":"2023-10-09T11:03:21.180054+0000","last_undegraded":"2023-10-09T11:03:21.180054+0000","last_fullsized":"2023-10-09T11:03:21.180054+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:24.127596+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":541,"reported_epoch":455,"state":"active+undersized+degraded","last_fresh":"2023-10-09T11:03:21.190875+0000","last_change":"2023-10-09T11:03:09.261415+0000","last_active":"2023-10-09T11:03:21.190875+0000","last_peered":"2023-10-09T11:03:21.190875+0000","last_clean":"2023-10-09T11:02:31.327702+0000","last_became_active":"2023-10-09T11:03:09.261415+0000","last_became_peered":"2023-10-09T11:03:09.261415+0000","last_unstale":"2023-10-09T11:03:21.190875+0000","last_undegraded":"2023-10-09T11:03:09.256835+0000","last_fullsized":"2023-10-09T11:03:09.256705+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:37:25.660269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":134,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T11:03:20.151703+0000","last_change":"2023-10-09T11:00:14.900305+0000","last_active":"2023-10-09T11:03:20.151703+0000","last_peered":"2023-10-09T11:03:20.151703+0000","last_clean":"2023-10-09T11:03:20.151703+0000","last_became_active":"2023-10-09T11:00:14.899724+0000","last_became_peered":"2023-10-09T11:00:14.899724+0000","last_unstale":"2023-10-09T11:03:20.151703+0000","last_undegraded":"2023-10-09T11:03:20.151703+0000","last_fullsized":"2023-10-09T11:03:20.151703+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:59.022161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":522,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.188603+0000","last_change":"2023-10-09T11:03:09.270472+0000","last_active":"2023-10-09T11:03:21.188603+0000","last_peered":"2023-10-09T11:03:21.188603+0000","last_clean":"2023-10-09T11:02:31.326784+0000","last_became_active":"2023-10-09T11:03:09.270472+0000","last_became_peered":"2023-10-09T11:03:09.270472+0000","last_unstale":"2023-10-09T11:03:21.188603+0000","last_undegraded":"2023-10-09T11:03:21.188603+0000","last_fullsized":"2023-10-09T11:03:09.253896+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:39.158118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"456'8305","reported_seq":8373,"reported_epoch":456,"state":"active+undersized+degraded","last_fresh":"2023-10-09T11:03:26.460952+0000","last_change":"2023-10-09T11:03:09.267451+0000","last_active":"2023-10-09T11:03:26.460952+0000","last_peered":"2023-10-09T11:03:26.460952+0000","last_clean":"2023-10-09T11:03:08.251967+0000","last_became_active":"2023-10-09T11:03:09.267451+0000","last_became_peered":"2023-10-09T11:03:09.267451+0000","last_unstale":"2023-10-09T11:03:26.460952+0000","last_undegraded":"2023-10-09T11:03:09.259253+0000","last_fullsized":"2023-10-09T11:03:09.259147+0000","mapping_epoch":449,"log_start":"415'2100","ondisk_log_start":"415'2100","created":403,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6205,"log_dups_size":0,"ondisk_log_size":6205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:26:19.306429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268632064,"num_objects":4099,"num_object_clones":0,"num_object_copies":12297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4099,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4099,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8305,"num_write_kb":396928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":4099}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":544,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.166391+0000","last_change":"2023-10-09T11:03:09.267278+0000","last_active":"2023-10-09T11:03:21.166391+0000","last_peered":"2023-10-09T11:03:21.166391+0000","last_clean":"2023-10-09T10:49:11.425484+0000","last_became_active":"2023-10-09T11:03:09.267278+0000","last_became_peered":"2023-10-09T11:03:09.267278+0000","last_unstale":"2023-10-09T11:03:21.166391+0000","last_undegraded":"2023-10-09T11:03:21.166391+0000","last_fullsized":"2023-10-09T11:03:09.255035+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.567039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.567039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":31,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.190719+0000","last_change":"2023-10-09T11:02:33.198517+0000","last_active":"2023-10-09T11:03:21.190719+0000","last_peered":"2023-10-09T11:03:21.190719+0000","last_clean":"2023-10-09T11:03:21.190719+0000","last_became_active":"2023-10-09T11:02:33.197676+0000","last_became_peered":"2023-10-09T11:02:33.197676+0000","last_unstale":"2023-10-09T11:03:21.190719+0000","last_undegraded":"2023-10-09T11:03:21.190719+0000","last_fullsized":"2023-10-09T11:03:21.190719+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:40:41.647813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":397,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.183400+0000","last_change":"2023-10-09T11:00:15.869009+0000","last_active":"2023-10-09T11:03:21.183400+0000","last_peered":"2023-10-09T11:03:21.183400+0000","last_clean":"2023-10-09T11:03:21.183400+0000","last_became_active":"2023-10-09T11:00:15.868637+0000","last_became_peered":"2023-10-09T11:00:15.868637+0000","last_unstale":"2023-10-09T11:03:21.183400+0000","last_undegraded":"2023-10-09T11:03:21.183400+0000","last_fullsized":"2023-10-09T11:03:21.183400+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:37.538741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":241,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.171280+0000","last_change":"2023-10-09T11:01:15.402661+0000","last_active":"2023-10-09T11:03:21.171280+0000","last_peered":"2023-10-09T11:03:21.171280+0000","last_clean":"2023-10-09T11:03:21.171280+0000","last_became_active":"2023-10-09T11:01:15.402327+0000","last_became_peered":"2023-10-09T11:01:15.402327+0000","last_unstale":"2023-10-09T11:03:21.171280+0000","last_undegraded":"2023-10-09T11:03:21.171280+0000","last_fullsized":"2023-10-09T11:03:21.171280+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:32.513751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.12","version":"455'7099","reported_seq":7176,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:22.978650+0000","last_change":"2023-10-09T11:03:04.442682+0000","last_active":"2023-10-09T11:03:22.978650+0000","last_peered":"2023-10-09T11:03:22.978650+0000","last_clean":"2023-10-09T11:03:22.978650+0000","last_became_active":"2023-10-09T11:02:49.375825+0000","last_became_peered":"2023-10-09T11:02:49.375825+0000","last_unstale":"2023-10-09T11:03:22.978650+0000","last_undegraded":"2023-10-09T11:03:22.978650+0000","last_fullsized":"2023-10-09T11:03:22.978650+0000","mapping_epoch":441,"log_start":"406'500","ondisk_log_start":"406'500","created":441,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6599,"log_dups_size":0,"ondisk_log_size":6599,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139919360,"num_objects":2135,"num_object_clones":0,"num_object_copies":6405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4021,"num_write_kb":196992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":801,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.145746+0000","last_change":"2023-10-09T11:03:09.270050+0000","last_active":"2023-10-09T11:03:20.145746+0000","last_peered":"2023-10-09T11:03:20.145746+0000","last_clean":"2023-10-09T10:59:45.166306+0000","last_became_active":"2023-10-09T11:03:09.270050+0000","last_became_peered":"2023-10-09T11:03:09.270050+0000","last_unstale":"2023-10-09T11:03:20.145746+0000","last_undegraded":"2023-10-09T11:03:20.145746+0000","last_fullsized":"2023-10-09T11:03:09.259487+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:10:47.799551+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":395,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T11:03:20.144771+0000","last_change":"2023-10-09T11:02:33.201950+0000","last_active":"2023-10-09T11:03:20.144771+0000","last_peered":"2023-10-09T11:03:20.144771+0000","last_clean":"2023-10-09T11:03:20.144771+0000","last_became_active":"2023-10-09T11:02:33.201559+0000","last_became_peered":"2023-10-09T11:02:33.201559+0000","last_unstale":"2023-10-09T11:03:20.144771+0000","last_undegraded":"2023-10-09T11:03:20.144771+0000","last_fullsized":"2023-10-09T11:03:20.144771+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:45.851351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":218,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.179437+0000","last_change":"2023-10-09T11:03:09.257609+0000","last_active":"2023-10-09T11:03:21.179437+0000","last_peered":"2023-10-09T11:03:21.179437+0000","last_clean":"2023-10-09T11:03:05.236652+0000","last_became_active":"2023-10-09T11:03:09.257609+0000","last_became_peered":"2023-10-09T11:03:09.257609+0000","last_unstale":"2023-10-09T11:03:21.179437+0000","last_undegraded":"2023-10-09T11:03:21.179437+0000","last_fullsized":"2023-10-09T11:03:09.254175+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:48:52.369245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.15","version":"456'7280","reported_seq":7360,"reported_epoch":456,"state":"active+undersized+degraded","last_fresh":"2023-10-09T11:03:26.057839+0000","last_change":"2023-10-09T11:03:09.271790+0000","last_active":"2023-10-09T11:03:26.057839+0000","last_peered":"2023-10-09T11:03:26.057839+0000","last_clean":"2023-10-09T11:03:08.211061+0000","last_became_active":"2023-10-09T11:03:09.271790+0000","last_became_peered":"2023-10-09T11:03:09.271790+0000","last_unstale":"2023-10-09T11:03:26.057839+0000","last_undegraded":"2023-10-09T11:03:09.254381+0000","last_fullsized":"2023-10-09T11:03:09.254274+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":7280,"log_dups_size":0,"ondisk_log_size":7280,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135954432,"num_objects":2075,"num_object_clones":0,"num_object_copies":6225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2075,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4163,"num_write_kb":199584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":2075}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":549,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.182797+0000","last_change":"2023-10-09T11:03:09.266911+0000","last_active":"2023-10-09T11:03:21.182797+0000","last_peered":"2023-10-09T11:03:21.182797+0000","last_clean":"2023-10-09T10:49:11.426788+0000","last_became_active":"2023-10-09T11:03:09.266911+0000","last_became_peered":"2023-10-09T11:03:09.266911+0000","last_unstale":"2023-10-09T11:03:21.182797+0000","last_undegraded":"2023-10-09T11:03:21.182797+0000","last_fullsized":"2023-10-09T11:03:09.262722+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":283,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.166509+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:03:21.166509+0000","last_peered":"2023-10-09T11:03:21.166509+0000","last_clean":"2023-10-09T11:03:21.166509+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:03:21.166509+0000","last_undegraded":"2023-10-09T11:03:21.166509+0000","last_fullsized":"2023-10-09T11:03:21.166509+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":372,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.178051+0000","last_change":"2023-10-09T11:03:09.275939+0000","last_active":"2023-10-09T11:03:21.178051+0000","last_peered":"2023-10-09T11:03:21.178051+0000","last_clean":"2023-10-09T10:57:42.797964+0000","last_became_active":"2023-10-09T11:03:09.275939+0000","last_became_peered":"2023-10-09T11:03:09.275939+0000","last_unstale":"2023-10-09T11:03:21.178051+0000","last_undegraded":"2023-10-09T11:03:21.178051+0000","last_fullsized":"2023-10-09T11:03:09.253325+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:59.949393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.14","version":"455'6979","reported_seq":7056,"reported_epoch":455,"state":"active+undersized+degraded","last_fresh":"2023-10-09T11:03:23.046520+0000","last_change":"2023-10-09T11:03:09.280940+0000","last_active":"2023-10-09T11:03:23.046520+0000","last_peered":"2023-10-09T11:03:23.046520+0000","last_clean":"2023-10-09T11:03:08.237197+0000","last_became_active":"2023-10-09T11:03:09.280940+0000","last_became_peered":"2023-10-09T11:03:09.280940+0000","last_unstale":"2023-10-09T11:03:23.046520+0000","last_undegraded":"2023-10-09T11:03:09.260144+0000","last_fullsized":"2023-10-09T11:03:09.260038+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":6979,"log_dups_size":0,"ondisk_log_size":6979,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140640256,"num_objects":2146,"num_object_clones":0,"num_object_copies":6438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2146,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3938,"num_write_kb":194688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":2146}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":406,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.166814+0000","last_change":"2023-10-09T10:35:56.820218+0000","last_active":"2023-10-09T11:03:21.166814+0000","last_peered":"2023-10-09T11:03:21.166814+0000","last_clean":"2023-10-09T11:03:21.166814+0000","last_became_active":"2023-10-09T10:35:56.819915+0000","last_became_peered":"2023-10-09T10:35:56.819915+0000","last_unstale":"2023-10-09T11:03:21.166814+0000","last_undegraded":"2023-10-09T11:03:21.166814+0000","last_fullsized":"2023-10-09T11:03:21.166814+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:00.978047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":344,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.144782+0000","last_change":"2023-10-09T11:03:09.275872+0000","last_active":"2023-10-09T11:03:20.144782+0000","last_peered":"2023-10-09T11:03:20.144782+0000","last_clean":"2023-10-09T10:59:45.168219+0000","last_became_active":"2023-10-09T11:03:09.275872+0000","last_became_peered":"2023-10-09T11:03:09.275872+0000","last_unstale":"2023-10-09T11:03:20.144782+0000","last_undegraded":"2023-10-09T11:03:20.144782+0000","last_fullsized":"2023-10-09T11:03:09.259010+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:54:42.622413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.17","version":"456'7172","reported_seq":7252,"reported_epoch":456,"state":"active+recovery_wait+undersized+degraded","last_fresh":"2023-10-09T11:03:26.630647+0000","last_change":"2023-10-09T11:03:09.280070+0000","last_active":"2023-10-09T11:03:26.630647+0000","last_peered":"2023-10-09T11:03:26.630647+0000","last_clean":"2023-10-09T11:03:08.220443+0000","last_became_active":"2023-10-09T11:03:09.263632+0000","last_became_peered":"2023-10-09T11:03:09.263632+0000","last_unstale":"2023-10-09T11:03:26.630647+0000","last_undegraded":"2023-10-09T11:03:09.256909+0000","last_fullsized":"2023-10-09T11:03:09.256411+0000","mapping_epoch":449,"log_start":"406'100","ondisk_log_start":"406'100","created":441,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":7072,"log_dups_size":0,"ondisk_log_size":7072,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130646016,"num_objects":1994,"num_object_clones":0,"num_object_copies":5982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1994,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4118,"num_write_kb":195552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1994}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":686,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.190530+0000","last_change":"2023-10-09T11:02:36.217717+0000","last_active":"2023-10-09T11:03:21.190530+0000","last_peered":"2023-10-09T11:03:21.190530+0000","last_clean":"2023-10-09T11:03:21.190530+0000","last_became_active":"2023-10-09T11:02:36.217373+0000","last_became_peered":"2023-10-09T11:02:36.217373+0000","last_unstale":"2023-10-09T11:03:21.190530+0000","last_undegraded":"2023-10-09T11:03:21.190530+0000","last_fullsized":"2023-10-09T11:03:21.190530+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:47.307693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:47.307693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":311,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.180766+0000","last_change":"2023-10-09T11:03:09.263444+0000","last_active":"2023-10-09T11:03:21.180766+0000","last_peered":"2023-10-09T11:03:21.180766+0000","last_clean":"2023-10-09T10:54:05.905773+0000","last_became_active":"2023-10-09T11:03:09.263444+0000","last_became_peered":"2023-10-09T11:03:09.263444+0000","last_unstale":"2023-10-09T11:03:21.180766+0000","last_undegraded":"2023-10-09T11:03:21.180766+0000","last_fullsized":"2023-10-09T11:03:09.255061+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:54.422426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":428,"reported_epoch":455,"state":"active+undersized","last_fresh":"2023-10-09T11:03:21.177707+0000","last_change":"2023-10-09T11:03:09.263471+0000","last_active":"2023-10-09T11:03:21.177707+0000","last_peered":"2023-10-09T11:03:21.177707+0000","last_clean":"2023-10-09T10:59:45.166627+0000","last_became_active":"2023-10-09T11:03:09.263471+0000","last_became_peered":"2023-10-09T11:03:09.263471+0000","last_unstale":"2023-10-09T11:03:21.177707+0000","last_undegraded":"2023-10-09T11:03:21.177707+0000","last_fullsized":"2023-10-09T11:03:09.252941+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:51:24.265015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.16","version":"456'7375","reported_seq":7446,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-09T11:03:26.591029+0000","last_change":"2023-10-09T11:02:49.378964+0000","last_active":"2023-10-09T11:03:26.591029+0000","last_peered":"2023-10-09T11:03:26.591029+0000","last_clean":"2023-10-09T11:03:26.591029+0000","last_became_active":"2023-10-09T11:02:49.376770+0000","last_became_peered":"2023-10-09T11:02:49.376770+0000","last_unstale":"2023-10-09T11:03:26.591029+0000","last_undegraded":"2023-10-09T11:03:26.591029+0000","last_fullsized":"2023-10-09T11:03:26.591029+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":441,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:01:14.357879+0000","last_clean_scrub_stamp":"2023-10-09T11:01:14.357879+0000","objects_scrubbed":0,"log_size":7375,"log_dups_size":0,"ondisk_log_size":7375,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132448256,"num_objects":2021,"num_object_clones":0,"num_object_copies":6063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4243,"num_write_kb":200448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":718,"reported_epoch":454,"state":"active+undersized","last_fresh":"2023-10-09T11:03:20.151338+0000","last_change":"2023-10-09T11:03:09.281867+0000","last_active":"2023-10-09T11:03:20.151338+0000","last_peered":"2023-10-09T11:03:20.151338+0000","last_clean":"2023-10-09T11:03:05.205612+0000","last_became_active":"2023-10-09T11:03:09.281867+0000","last_became_peered":"2023-10-09T11:03:09.281867+0000","last_unstale":"2023-10-09T11:03:20.151338+0000","last_undegraded":"2023-10-09T11:03:20.151338+0000","last_fullsized":"2023-10-09T11:03:09.255287+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:51.247225+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":97,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-09T11:03:20.147860+0000","last_change":"2023-10-09T10:59:48.143732+0000","last_active":"2023-10-09T11:03:20.147860+0000","last_peered":"2023-10-09T11:03:20.147860+0000","last_clean":"2023-10-09T11:03:20.147860+0000","last_became_active":"2023-10-09T10:59:48.129001+0000","last_became_peered":"2023-10-09T10:59:48.129001+0000","last_unstale":"2023-10-09T11:03:20.147860+0000","last_undegraded":"2023-10-09T11:03:20.147860+0000","last_fullsized":"2023-10-09T11:03:20.147860+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:22:36.946348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":477,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-09T11:03:21.177942+0000","last_change":"2023-10-09T11:00:14.869353+0000","last_active":"2023-10-09T11:03:21.177942+0000","last_peered":"2023-10-09T11:03:21.177942+0000","last_clean":"2023-10-09T11:03:21.177942+0000","last_became_active":"2023-10-09T11:00:14.869061+0000","last_became_peered":"2023-10-09T11:00:14.869061+0000","last_unstale":"2023-10-09T11:03:21.177942+0000","last_undegraded":"2023-10-09T11:03:21.177942+0000","last_fullsized":"2023-10-09T11:03:21.177942+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:03:30.318094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":79,"acting":79,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":4390256664,"num_objects":66991,"num_object_clones":0,"num_object_copies":200973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29289,"num_objects_misplaced":5717,"num_objects_unfound":0,"num_objects_dirty":66991,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":123986,"num_write_kb":5920513,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9815,"num_bytes_recovered":643170304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13980413952,"data_stored":13980401736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":178102,"ondisk_log_size":178102,"up":69,"acting":67,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":17098,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8961,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":83,"acting":83,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630621,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":33,"num_read_kb":62,"num_write":17,"num_write_kb":165,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1137654,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":80,"acting":80,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":23,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332442,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1178012,"kb_used_data":270420,"kb_used_omap":0,"kb_used_meta":907584,"kb_avail":92567140,"statfs":{"total":95995035648,"available":94788751360,"internally_reserved":0,"allocated":276910080,"data_stored":275443265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929366016},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:03:13 2023","interfaces":[{"interface":"back","average":{"1min":2.964,"5min":2.0379999999999998,"15min":1.9019999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22500000000000001},"max":{"1min":9.8040000000000003,"5min":9.8040000000000003,"15min":9.8040000000000003},"last":0.56100000000000005},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.6439999999999999,"15min":1.927},"min":{"1min":0.26500000000000001,"5min":0.216,"15min":0.216},"max":{"1min":9.6999999999999993,"5min":9.6999999999999993,"15min":16.731000000000002},"last":0.58499999999999996}]},{"osd":1,"last update":"Mon Oct 9 11:03:17 2023","interfaces":[{"interface":"back","average":{"1min":1.54,"5min":1.476,"15min":1.601},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.867,"5min":6.202,"15min":6.202},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.9870000000000001,"5min":1.7589999999999999,"15min":1.9530000000000001},"min":{"1min":0.29299999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.6620000000000008,"5min":12.193,"15min":12.193},"last":0.54500000000000004}]},{"osd":2,"last update":"Mon Oct 9 11:02:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1899999999999999,"5min":1.6839999999999999,"15min":1.7250000000000001},"min":{"1min":0.30399999999999999,"5min":0.245,"15min":0.245},"max":{"1min":8.8970000000000002,"5min":8.8970000000000002,"15min":11.058},"last":2.5760000000000001},{"interface":"front","average":{"1min":1.746,"5min":1.917,"15min":1.7969999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":4.9550000000000001,"5min":8.9280000000000008,"15min":13.037000000000001},"last":0.44700000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":1.6819999999999999,"15min":2.0470000000000002},"min":{"1min":0.378,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.698,"5min":13.464,"15min":13.464},"last":1.252},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.734,"15min":1.8440000000000001},"min":{"1min":0.371,"5min":0.31,"15min":0.31},"max":{"1min":5.0919999999999996,"5min":9.0210000000000008,"15min":19.234999999999999},"last":1.7569999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:02:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":1.444,"15min":1.496},"min":{"1min":0.27100000000000002,"5min":0.248,"15min":0.24099999999999999},"max":{"1min":7.6589999999999998,"5min":7.96,"15min":16.695},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.8690000000000002,"5min":1.677,"15min":1.4790000000000001},"min":{"1min":0.31,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":16.481000000000002,"5min":16.481000000000002,"15min":16.481000000000002},"last":2.5299999999999998}]},{"osd":5,"last update":"Mon Oct 9 11:02:55 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":1.913,"15min":1.6279999999999999},"min":{"1min":0.29499999999999998,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":9.7780000000000005,"5min":13.407,"15min":13.407},"last":0.379},{"interface":"front","average":{"1min":1.794,"5min":1.431,"15min":1.5489999999999999},"min":{"1min":0.317,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.8439999999999994,"5min":9.8439999999999994,"15min":11.741},"last":0.39300000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:02:55 2023","interfaces":[{"interface":"back","average":{"1min":1.9359999999999999,"5min":1.573,"15min":1.7090000000000001},"min":{"1min":0.375,"5min":0.34999999999999998,"15min":0.315},"max":{"1min":10.068,"5min":10.388999999999999,"15min":14.913},"last":0.433},{"interface":"front","average":{"1min":3.1400000000000001,"5min":1.6779999999999999,"15min":1.6579999999999999},"min":{"1min":0.34499999999999997,"5min":0.32600000000000001,"15min":0.29399999999999998},"max":{"1min":16.538,"5min":16.538,"15min":19.187999999999999},"last":5.1020000000000003}]}]},{"osd":6,"up_from":44,"seq":188978561373,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2747088,"kb_used_data":1842696,"kb_used_omap":0,"kb_used_meta":904256,"kb_avail":90998064,"statfs":{"total":95995035648,"available":93182017536,"internally_reserved":0,"allocated":1886920704,"data_stored":1885465511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925958144},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,3,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:03:12 2023","interfaces":[{"interface":"back","average":{"1min":3.0800000000000001,"5min":2.1850000000000001,"15min":2.0289999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.216},"max":{"1min":29.379999999999999,"5min":46.603999999999999,"15min":54.743000000000002},"last":0.74299999999999999},{"interface":"front","average":{"1min":3.2789999999999999,"5min":2.484,"15min":2.1299999999999999},"min":{"1min":0.34000000000000002,"5min":0.24199999999999999,"15min":0.223},"max":{"1min":29.132999999999999,"5min":46.679000000000002,"15min":53.701000000000001},"last":0.41899999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:03:17 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":1.304,"15min":0.93700000000000006},"min":{"1min":0.26000000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.6890000000000001,"5min":41.241999999999997,"15min":41.241999999999997},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.4060000000000001,"5min":2.0009999999999999,"15min":1.5760000000000001},"min":{"1min":0.26200000000000001,"5min":0.248,"15min":0.248},"max":{"1min":9.6690000000000005,"5min":41.476999999999997,"15min":41.476999999999997},"last":2.9780000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.738,"15min":1.8260000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28399999999999997},"max":{"1min":3.5470000000000002,"5min":8.6869999999999994,"15min":13.208},"last":2.9390000000000001},{"interface":"front","average":{"1min":2.6629999999999998,"5min":2.3889999999999998,"15min":2.0670000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.0839999999999996,"5min":15.432,"15min":15.432},"last":1.8360000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:02:07 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.6719999999999999,"15min":1.8320000000000001},"min":{"1min":0.35499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":3.266,"5min":10.018000000000001,"15min":12.391},"last":1.0760000000000001},{"interface":"front","average":{"1min":1.101,"5min":1.885,"15min":2.0870000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":3.6389999999999998,"5min":11.161,"15min":13.179},"last":1.2230000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.218,"5min":1.0649999999999999,"15min":1.304},"min":{"1min":0.29799999999999999,"5min":0.25800000000000001,"15min":0.23200000000000001},"max":{"1min":3.9630000000000001,"5min":9.0060000000000002,"15min":16.978999999999999},"last":1.96},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.92900000000000005,"15min":1.2310000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.24399999999999999},"max":{"1min":4.8049999999999997,"5min":5.7869999999999999,"15min":12.481999999999999},"last":0.59699999999999998}]},{"osd":5,"last update":"Mon Oct 9 11:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.458,"15min":1.5640000000000001},"min":{"1min":0.33700000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.2290000000000001,"5min":19.786999999999999,"15min":19.786999999999999},"last":1.891},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.0820000000000001,"15min":1.337},"min":{"1min":0.32500000000000001,"5min":0.28899999999999998,"15min":0.26600000000000001},"max":{"1min":4.8810000000000002,"5min":8.6020000000000003,"15min":12.456},"last":0.38500000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:03:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5430000000000001,"5min":1.6890000000000001,"15min":1.6779999999999999},"min":{"1min":0.33500000000000002,"5min":0.28100000000000003,"15min":0.26600000000000001},"max":{"1min":11.550000000000001,"5min":11.550000000000001,"15min":23.358000000000001},"last":5.9790000000000001},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.0429999999999999,"15min":1.214},"min":{"1min":0.374,"5min":0.32300000000000001,"15min":0.27500000000000002},"max":{"1min":4.9690000000000003,"5min":6.1920000000000002,"15min":12.422000000000001},"last":0.47099999999999997}]}]},{"osd":1,"up_from":388,"seq":1666447310887,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1384372,"kb_used_data":1111340,"kb_used_omap":13,"kb_used_meta":272946,"kb_avail":92360780,"statfs":{"total":95995035648,"available":94577438720,"internally_reserved":0,"allocated":1138012160,"data_stored":1135651490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":279497415},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":3.2690000000000001,"5min":2.46,"15min":2.3250000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":13.334,"5min":13.334,"15min":13.334},"last":0.193},{"interface":"front","average":{"1min":2.968,"5min":1.6419999999999999,"15min":1.421},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":9.6829999999999998,"5min":9.6829999999999998,"15min":9.6829999999999998},"last":0.26500000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":2.9790000000000001,"15min":3.1389999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.4670000000000005,"5min":12.009,"15min":12.009},"last":0.24399999999999999},{"interface":"front","average":{"1min":2.5499999999999998,"5min":2.2959999999999998,"15min":2.254},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.8249999999999993,"5min":9.8249999999999993,"15min":9.8249999999999993},"last":0.30499999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7879999999999998,"5min":2.8199999999999998,"15min":2.8250000000000002},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":9.6460000000000008,"5min":12.523999999999999,"15min":12.523999999999999},"last":3.3929999999999998},{"interface":"front","average":{"1min":1.518,"5min":3.5539999999999998,"15min":3.8940000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.3799999999999999,"5min":18.079999999999998,"15min":18.079999999999998},"last":0.44600000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":2.536,"15min":2.754},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.2300000000000004,"5min":10.481,"15min":10.481},"last":2.9740000000000002},{"interface":"front","average":{"1min":1.702,"5min":1.448,"15min":1.4059999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":9.8480000000000008,"5min":9.8480000000000008,"15min":9.8480000000000008},"last":1.655}]},{"osd":5,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":2.29,"15min":2.4580000000000002},"min":{"1min":0.35599999999999998,"5min":0.311,"15min":0.311},"max":{"1min":4.8799999999999999,"5min":18.053000000000001,"15min":18.053000000000001},"last":5.1200000000000001},{"interface":"front","average":{"1min":1.526,"5min":2.2429999999999999,"15min":2.363},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.2830000000000004,"5min":10.558,"15min":10.558},"last":5.1109999999999998}]},{"osd":6,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":2.2759999999999998,"15min":2.46},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.8520000000000003,"5min":18.085999999999999,"15min":18.085999999999999},"last":1.3049999999999999},{"interface":"front","average":{"1min":1.3049999999999999,"5min":2.1869999999999998,"15min":2.3340000000000001},"min":{"1min":0.38600000000000001,"5min":0.373,"15min":0.373},"max":{"1min":5.3129999999999997,"5min":10.531000000000001,"15min":10.531000000000001},"last":3.5899999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.393,"15min":1.3220000000000001},"min":{"1min":0.41899999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.8000000000000007,"5min":9.8000000000000007,"15min":9.8000000000000007},"last":4.9930000000000003},{"interface":"front","average":{"1min":1.1910000000000001,"5min":2.3279999999999998,"15min":2.5179999999999998},"min":{"1min":0.435,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.8019999999999996,"5min":18.071000000000002,"15min":18.071000000000002},"last":0.42899999999999999}]}]},{"osd":0,"up_from":80,"seq":343597383945,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3537564,"kb_used_data":2815700,"kb_used_omap":28,"kb_used_meta":721763,"kb_avail":90207588,"statfs":{"total":95995035648,"available":92372570112,"internally_reserved":0,"allocated":2883276800,"data_stored":2880870620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":739086160},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:03:18 2023","interfaces":[{"interface":"back","average":{"1min":2.6589999999999998,"5min":2.9119999999999999,"15min":3.1619999999999999},"min":{"1min":0.21299999999999999,"5min":0.186,"15min":0.186},"max":{"1min":9.2799999999999994,"5min":19.608000000000001,"15min":19.608000000000001},"last":0.23699999999999999},{"interface":"front","average":{"1min":2.4199999999999999,"5min":2.8809999999999998,"15min":3.206},"min":{"1min":0.26300000000000001,"5min":0.218,"15min":0.218},"max":{"1min":8.6300000000000008,"5min":19.992999999999999,"15min":19.992999999999999},"last":0.26500000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:02:49 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":1.9910000000000001,"15min":2.2130000000000001},"min":{"1min":0.23899999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":12.035,"5min":12.638,"15min":24.852},"last":0.21299999999999999},{"interface":"front","average":{"1min":2.3370000000000002,"5min":2.363,"15min":2.661},"min":{"1min":0.255,"5min":0.20899999999999999,"15min":0.19800000000000001},"max":{"1min":6.7030000000000003,"5min":12.494999999999999,"15min":77.256},"last":0.20300000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:02:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":2.1589999999999998,"15min":2.3090000000000002},"min":{"1min":0.26800000000000002,"5min":0.215,"15min":0.20200000000000001},"max":{"1min":8.8699999999999992,"5min":28.946000000000002,"15min":28.946000000000002},"last":2.3300000000000001},{"interface":"front","average":{"1min":1.46,"5min":2.4350000000000001,"15min":2.5790000000000002},"min":{"1min":0.309,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.9850000000000003,"5min":19.952999999999999,"15min":19.952999999999999},"last":0.56499999999999995}]},{"osd":4,"last update":"Mon Oct 9 11:03:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":2.2749999999999999,"15min":2.452},"min":{"1min":0.33100000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.3810000000000002,"5min":11.507,"15min":77.177999999999997},"last":0.25700000000000001},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.571,"15min":1.9359999999999999},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":3.5529999999999999,"5min":9.9489999999999998,"15min":12.318},"last":0.443}]},{"osd":5,"last update":"Mon Oct 9 11:03:10 2023","interfaces":[{"interface":"back","average":{"1min":2.129,"5min":1.9530000000000001,"15min":2.0720000000000001},"min":{"1min":0.378,"5min":0.309,"15min":0.29799999999999999},"max":{"1min":7.9039999999999999,"5min":9.8710000000000004,"15min":12.157},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.321,"5min":1.9239999999999999,"15min":2.048},"min":{"1min":0.308,"5min":0.29999999999999999,"15min":0.29899999999999999},"max":{"1min":5.9610000000000003,"5min":7.0919999999999996,"15min":24.754999999999999},"last":1.4490000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:03:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5940000000000001,"5min":1.8899999999999999,"15min":2.2000000000000002},"min":{"1min":0.39300000000000002,"5min":0.314,"15min":0.30399999999999999},"max":{"1min":5.9950000000000001,"5min":10.012,"15min":18.920000000000002},"last":2.2160000000000002},{"interface":"front","average":{"1min":1.7,"5min":1.9950000000000001,"15min":2.02},"min":{"1min":0.34999999999999998,"5min":0.31900000000000001,"15min":0.307},"max":{"1min":5.9160000000000004,"5min":8.8249999999999993,"15min":24.800000000000001},"last":0.434}]},{"osd":7,"last update":"Mon Oct 9 11:03:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":2.04,"15min":2.2050000000000001},"min":{"1min":0.443,"5min":0.34499999999999997,"15min":0.32700000000000001},"max":{"1min":5.8789999999999996,"5min":11.285,"15min":14.48},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.8879999999999999,"5min":2.4089999999999998,"15min":2.5419999999999998},"min":{"1min":0.42499999999999999,"5min":0.372,"15min":0.314},"max":{"1min":5.7240000000000002,"5min":10.398,"15min":77.228999999999999},"last":0.99099999999999999}]}]},{"osd":2,"up_from":90,"seq":386547056886,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2147908,"kb_used_data":1599740,"kb_used_omap":81,"kb_used_meta":548078,"kb_avail":91597244,"statfs":{"total":95995035648,"available":93795577856,"internally_reserved":0,"allocated":1638133760,"data_stored":1635876213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":561232090},"hb_peers":[0,1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:49 2023","interfaces":[{"interface":"back","average":{"1min":4.9080000000000004,"5min":2.0880000000000001,"15min":1.917},"min":{"1min":0.19800000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":48.933999999999997,"5min":48.933999999999997,"15min":48.933999999999997},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.677,"5min":1.6919999999999999,"15min":1.8009999999999999},"min":{"1min":0.24399999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":14.515000000000001,"5min":14.515000000000001,"15min":15.952999999999999},"last":0.35599999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:03:20 2023","interfaces":[{"interface":"back","average":{"1min":4.016,"5min":2.702,"15min":2.5819999999999999},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.19},"max":{"1min":49.033999999999999,"5min":49.033999999999999,"15min":49.033999999999999},"last":6.0750000000000002},{"interface":"front","average":{"1min":5.1150000000000002,"5min":2.9399999999999999,"15min":2.75},"min":{"1min":0.30199999999999999,"5min":0.214,"15min":0.214},"max":{"1min":48.874000000000002,"5min":48.874000000000002,"15min":48.874000000000002},"last":0.21299999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:02:09 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":1.6279999999999999,"15min":1.863},"min":{"1min":0.24399999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.5979999999999999,"5min":11.151,"15min":19.539000000000001},"last":0.24099999999999999},{"interface":"front","average":{"1min":1.01,"5min":1.913,"15min":2.093},"min":{"1min":0.27200000000000002,"5min":0.219,"15min":0.219},"max":{"1min":3.524,"5min":16.324000000000002,"15min":16.702999999999999},"last":1.5860000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:02:49 2023","interfaces":[{"interface":"back","average":{"1min":1.292,"5min":1.8819999999999999,"15min":1.9710000000000001},"min":{"1min":0.313,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.4640000000000004,"5min":9.3290000000000006,"15min":16.231999999999999},"last":6.1210000000000004},{"interface":"front","average":{"1min":4.2069999999999999,"5min":2.0680000000000001,"15min":1.9710000000000001},"min":{"1min":0.35099999999999998,"5min":0.29999999999999999,"15min":0.28999999999999998},"max":{"1min":48.795000000000002,"5min":48.795000000000002,"15min":48.795000000000002},"last":0.28499999999999998}]},{"osd":5,"last update":"Mon Oct 9 11:02:49 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.6299999999999999,"15min":1.663},"min":{"1min":0.32600000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.0679999999999996,"5min":8.5380000000000003,"15min":17.920000000000002},"last":0.40699999999999997},{"interface":"front","average":{"1min":3.504,"5min":2.2589999999999999,"15min":2.036},"min":{"1min":0.38200000000000001,"5min":0.311,"15min":0.30299999999999999},"max":{"1min":49.061,"5min":49.061,"15min":49.061},"last":6.1470000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:02:49 2023","interfaces":[{"interface":"back","average":{"1min":4.6379999999999999,"5min":2.0089999999999999,"15min":1.78},"min":{"1min":0.318,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":48.845999999999997,"5min":48.845999999999997,"15min":48.845999999999997},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.504,"15min":1.587},"min":{"1min":0.378,"5min":0.313,"15min":0.313},"max":{"1min":7.0570000000000004,"5min":7.0570000000000004,"15min":10.111000000000001},"last":3.04}]},{"osd":7,"last update":"Mon Oct 9 11:02:49 2023","interfaces":[{"interface":"back","average":{"1min":4.3250000000000002,"5min":2.23,"15min":1.8220000000000001},"min":{"1min":0.41199999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":48.966000000000001,"5min":48.966000000000001,"15min":48.966000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.6379999999999999,"5min":1.8080000000000001,"15min":1.774},"min":{"1min":0.45300000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":14.94,"5min":14.94,"15min":14.94},"last":0.58899999999999997}]}]},{"osd":3,"up_from":95,"seq":408021893356,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3075752,"kb_used_data":2067296,"kb_used_omap":7,"kb_used_meta":1008440,"kb_avail":90669400,"statfs":{"total":95995035648,"available":92845465600,"internally_reserved":0,"allocated":2116911104,"data_stored":2114502345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1032643176},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":2.556,"15min":2.6800000000000002},"min":{"1min":0.222,"5min":0.20499999999999999,"15min":0.17499999999999999},"max":{"1min":8.1709999999999994,"5min":10.140000000000001,"15min":38.817999999999998},"last":3.1040000000000001},{"interface":"front","average":{"1min":2.4769999999999999,"5min":2.3530000000000002,"15min":2.9649999999999999},"min":{"1min":0.246,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":8.4239999999999995,"5min":17.042000000000002,"15min":17.312000000000001},"last":6.7240000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:02:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8,"5min":2.9199999999999999,"15min":3.1059999999999999},"min":{"1min":0.25,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":8.8030000000000008,"5min":8.9100000000000001,"15min":8.9100000000000001},"last":3.2770000000000001},{"interface":"front","average":{"1min":1.4650000000000001,"5min":2.5840000000000001,"15min":2.77},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.2300000000000004,"5min":9.7449999999999992,"15min":9.7449999999999992},"last":6.1509999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":2.3100000000000001,"5min":2.3069999999999999,"15min":2.448},"min":{"1min":0.21099999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":5.5350000000000001,"5min":10.65,"15min":20.402000000000001},"last":0.29599999999999999},{"interface":"front","average":{"1min":2.0489999999999999,"5min":2.2839999999999998,"15min":2.8260000000000001},"min":{"1min":0.23899999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":8.2729999999999997,"5min":10.012,"15min":27.538},"last":0.26800000000000002}]},{"osd":4,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.355,"5min":2.101,"15min":2.2770000000000001},"min":{"1min":0.30299999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.3639999999999999,"5min":9.8089999999999993,"15min":20.388000000000002},"last":3.0699999999999998},{"interface":"front","average":{"1min":1.3029999999999999,"5min":1.7929999999999999,"15min":2.157},"min":{"1min":0.34499999999999997,"5min":0.34000000000000002,"15min":0.24299999999999999},"max":{"1min":3.5910000000000002,"5min":9.1310000000000002,"15min":19.085999999999999},"last":0.34699999999999998}]},{"osd":5,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.948,"15min":2.2890000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.26000000000000001},"max":{"1min":3.5099999999999998,"5min":10.380000000000001,"15min":14.045},"last":0.317},{"interface":"front","average":{"1min":1.4630000000000001,"5min":1.7470000000000001,"15min":2.0369999999999999},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2839999999999998,"5min":8.3949999999999996,"15min":17.311},"last":0.33800000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":2.1419999999999999,"15min":2.27},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28000000000000003},"max":{"1min":8.3119999999999994,"5min":10.420999999999999,"15min":12.361000000000001},"last":0.38},{"interface":"front","average":{"1min":1.835,"5min":1.9430000000000001,"15min":2.2090000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.8959999999999999,"5min":7.1230000000000002,"15min":17.295000000000002},"last":0.36599999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:02:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":1.9419999999999999,"15min":2.2440000000000002},"min":{"1min":0.46400000000000002,"5min":0.309,"15min":0.309},"max":{"1min":7.492,"5min":8.6649999999999991,"15min":18.722000000000001},"last":1.0109999999999999},{"interface":"front","average":{"1min":1.6180000000000001,"5min":2.194,"15min":2.4670000000000001},"min":{"1min":0.439,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.4020000000000001,"5min":9.8569999999999993,"15min":15.212999999999999},"last":3.7469999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019236,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3330844,"kb_used_data":2446164,"kb_used_omap":0,"kb_used_meta":884544,"kb_avail":90414308,"statfs":{"total":95995035648,"available":92584251392,"internally_reserved":0,"allocated":2504871936,"data_stored":2503404854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":905773056},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3],"upper_bound":2},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:03:08 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":1.585,"15min":2.3919999999999999},"min":{"1min":0.23499999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.9749999999999996,"5min":8.6180000000000003,"15min":213.88999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":3.149,"5min":1.8049999999999999,"15min":2.706},"min":{"1min":0.33600000000000002,"5min":0.251,"15min":0.217},"max":{"1min":11.128,"5min":11.128,"15min":214.09800000000001},"last":0.27200000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:03:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":3.2360000000000002,"15min":2.4729999999999999},"min":{"1min":0.26300000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.1230000000000002,"5min":121.41,"15min":121.41},"last":1.8500000000000001},{"interface":"front","average":{"1min":1.8460000000000001,"5min":2.766,"15min":1.833},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":9.4580000000000002,"5min":122.21899999999999,"15min":122.21899999999999},"last":0.432}]},{"osd":2,"last update":"Mon Oct 9 11:02:52 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.5589999999999999,"15min":1.806},"min":{"1min":0.29199999999999998,"5min":0.27100000000000002,"15min":0.25900000000000001},"max":{"1min":3.8690000000000002,"5min":5.6870000000000003,"15min":15.446999999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":2.149,"5min":2.0680000000000001,"15min":2.2120000000000002},"min":{"1min":0.38400000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.0119999999999996,"5min":9.9480000000000004,"15min":27.818000000000001},"last":0.56799999999999995}]},{"osd":3,"last update":"Mon Oct 9 11:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":1.7509999999999999,"15min":2.6469999999999998},"min":{"1min":0.46999999999999997,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":2.254,"5min":12.798999999999999,"15min":209.03700000000001},"last":2.7469999999999999},{"interface":"front","average":{"1min":0.878,"5min":1.7749999999999999,"15min":2.899},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29999999999999999},"max":{"1min":1.8480000000000001,"5min":8.6010000000000009,"15min":208.80199999999999},"last":7.3620000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:03:14 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":1.617,"15min":1.8169999999999999},"min":{"1min":0.28599999999999998,"5min":0.245,"15min":0.23799999999999999},"max":{"1min":7.8970000000000002,"5min":9.7880000000000003,"15min":24.149999999999999},"last":3.0819999999999999},{"interface":"front","average":{"1min":1.486,"5min":1.3819999999999999,"15min":1.544},"min":{"1min":0.32400000000000001,"5min":0.25800000000000001,"15min":0.22600000000000001},"max":{"1min":5.3230000000000004,"5min":12.459,"15min":20.422000000000001},"last":0.45400000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:02:45 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.1899999999999999,"15min":1.387},"min":{"1min":0.39900000000000002,"5min":0.27800000000000002,"15min":0.24099999999999999},"max":{"1min":7.1520000000000001,"5min":8.5739999999999998,"15min":15.739000000000001},"last":2.0630000000000002},{"interface":"front","average":{"1min":2.1139999999999999,"5min":1.7749999999999999,"15min":1.819},"min":{"1min":0.32900000000000001,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":7.8490000000000002,"5min":9.0489999999999995,"15min":14.272},"last":1.986}]},{"osd":7,"last update":"Mon Oct 9 11:02:52 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":1.9279999999999999,"15min":1.766},"min":{"1min":0.35199999999999998,"5min":0.30199999999999999,"15min":0.29899999999999999},"max":{"1min":5.8179999999999996,"5min":19.923999999999999,"15min":19.923999999999999},"last":1.867},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.6679999999999999,"15min":1.6100000000000001},"min":{"1min":0.37,"5min":0.28799999999999998,"15min":0.255},"max":{"1min":3.3380000000000001,"5min":19.859999999999999,"15min":19.859999999999999},"last":0.29799999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790305,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2587976,"kb_used_data":1677056,"kb_used_omap":0,"kb_used_meta":910912,"kb_avail":91157176,"statfs":{"total":95995035648,"available":93344948224,"internally_reserved":0,"allocated":1717305344,"data_stored":1715854683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932773888},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:03:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0830000000000002,"5min":2.625,"15min":2.27},"min":{"1min":0.218,"5min":0.218,"15min":0.215},"max":{"1min":6.2919999999999998,"5min":58.271999999999998,"15min":58.271999999999998},"last":0.21299999999999999},{"interface":"front","average":{"1min":2.1179999999999999,"5min":2.6699999999999999,"15min":2.3159999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28599999999999998},"max":{"1min":6.3390000000000004,"5min":58.25,"15min":58.25},"last":0.28399999999999997}]},{"osd":1,"last update":"Mon Oct 9 11:03:24 2023","interfaces":[{"interface":"back","average":{"1min":3.5289999999999999,"5min":2.3149999999999999,"15min":2.097},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":32.491999999999997,"5min":32.491999999999997,"15min":32.491999999999997},"last":2.4769999999999999},{"interface":"front","average":{"1min":3.8719999999999999,"5min":2.1339999999999999,"15min":1.786},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":32.537999999999997,"5min":32.537999999999997,"15min":32.537999999999997},"last":0.247}]},{"osd":2,"last update":"Mon Oct 9 11:02:40 2023","interfaces":[{"interface":"back","average":{"1min":2.456,"5min":1.899,"15min":2.1230000000000002},"min":{"1min":0.24199999999999999,"5min":0.221,"15min":0.221},"max":{"1min":6.96,"5min":9.3539999999999992,"15min":11.393000000000001},"last":2.5139999999999998},{"interface":"front","average":{"1min":1.7130000000000001,"5min":1.8220000000000001,"15min":1.9510000000000001},"min":{"1min":0.28100000000000003,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":4.8419999999999996,"5min":14.621,"15min":15.087999999999999},"last":0.33300000000000002}]},{"osd":3,"last update":"Mon Oct 9 11:02:09 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":2.5449999999999999,"15min":2.5209999999999999},"min":{"1min":0.29699999999999999,"5min":0.28000000000000003,"15min":0.25700000000000001},"max":{"1min":11.590999999999999,"5min":32.810000000000002,"15min":63.508000000000003},"last":1.7390000000000001},{"interface":"front","average":{"1min":1.8069999999999999,"5min":2.5699999999999998,"15min":2.702},"min":{"1min":0.33900000000000002,"5min":0.30199999999999999,"15min":0.26300000000000001},"max":{"1min":11.292999999999999,"5min":32.659999999999997,"15min":62.418999999999997},"last":0.374}]},{"osd":4,"last update":"Mon Oct 9 11:03:24 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.1200000000000001,"15min":1.4890000000000001},"min":{"1min":0.30099999999999999,"5min":0.216,"15min":0.216},"max":{"1min":6.6769999999999996,"5min":7.8449999999999998,"15min":123.241},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.909,"5min":1.5369999999999999,"15min":1.823},"min":{"1min":0.314,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":8.3599999999999994,"5min":9.1869999999999994,"15min":123.318},"last":4.3200000000000003}]},{"osd":6,"last update":"Mon Oct 9 11:02:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":1.212,"15min":1.52},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.1589999999999998,"5min":8.4260000000000002,"15min":11.098000000000001},"last":2.5409999999999999},{"interface":"front","average":{"1min":1.3300000000000001,"5min":1.004,"15min":1.3520000000000001},"min":{"1min":0.34999999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.0800000000000001,"5min":5.899,"15min":20.256},"last":0.72499999999999998}]},{"osd":7,"last update":"Mon Oct 9 11:02:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.506,"15min":1.595},"min":{"1min":0.36799999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.3539999999999992,"5min":9.3539999999999992,"15min":10.739000000000001},"last":5.2300000000000004},{"interface":"front","average":{"1min":1.994,"5min":1.423,"15min":1.377},"min":{"1min":0.436,"5min":0.318,"15min":0.30099999999999999},"max":{"1min":10.211,"5min":10.211,"15min":14.154},"last":0.312}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2831290368,"data_stored":2831286296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1125318656,"data_stored":1125318656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1622671360,"data_stored":1622671360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2096627712,"data_stored":2096627712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2467434496,"data_stored":2467430424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1699086336,"data_stored":1699086336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1866731520,"data_stored":1866727448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":271253504,"data_stored":271253504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:03:29.064 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 81 2023-10-09T11:03:29.143 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:29.206 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:29.440 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:29.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:29.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:29.454 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:29 smithi155 ceph-mon[184854]: from='client.36345 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:03:29.541 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:29.545 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:29.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:29 smithi125 ceph-mon[195507]: from='client.36345 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:03:29.580 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:29 smithi125 ceph-mon[200197]: from='client.36345 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:03:29.648 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:29.689 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:29.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:29.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:29.793 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:29.855 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:29.894 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:29.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:29.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:29.979 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:03:30.001 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:30.020 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:03:30.020 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:03:30.080 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:30.214 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:30.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:30.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:30.283 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:03:30.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:30 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3906112304' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:03:30.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:30 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:03:30.318 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:30 smithi125 ceph-mon[195507]: pgmap v1125: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 28477/194274 objects degraded (14.658%); 5103/194274 objects misplaced (2.627%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T11:03:30.318 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:30.318 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:03:30.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:30 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3906112304' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:03:30.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:30 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:03:30.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:30 smithi125 ceph-mon[200197]: pgmap v1125: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 28477/194274 objects degraded (14.658%); 5103/194274 objects misplaced (2.627%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T11:03:30.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:30.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:03:30.319 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:30.369 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:30 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3906112304' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:03:30.370 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:30 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:03:30.370 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:30 smithi155 ceph-mon[184854]: pgmap v1125: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 28477/194274 objects degraded (14.658%); 5103/194274 objects misplaced (2.627%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T11:03:30.370 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:30.370 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:03:30.376 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:30.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T11:03:30.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:30.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:30.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:30.543 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:30.620 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:30.666 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:30.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:30.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:30.768 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:30.777 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:30.836 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:30.876 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:30.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:30.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:30.908 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:03:30.983 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:31.037 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:31.174 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:31.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:31.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:31.279 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:31.338 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:31.449 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:31 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:31.450 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:31 smithi125 ceph-mon[200197]: failed to encode map e457 with expected crc 2023-10-09T11:03:31.450 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:31 smithi125 ceph-mon[200197]: failed to encode map e457 with expected crc 2023-10-09T11:03:31.450 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:31 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:03:31.450 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:31 smithi125 ceph-mon[200197]: osdmap e457: 8 total, 7 up, 8 in 2023-10-09T11:03:31.450 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:03:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:03:31] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.43.0" 2023-10-09T11:03:31.450 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:31 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:31.451 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:31 smithi125 ceph-mon[195507]: failed to encode map e457 with expected crc 2023-10-09T11:03:31.451 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:31 smithi125 ceph-mon[195507]: failed to encode map e457 with expected crc 2023-10-09T11:03:31.451 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:31 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:03:31.451 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:31 smithi125 ceph-mon[195507]: osdmap e457: 8 total, 7 up, 8 in 2023-10-09T11:03:31.464 INFO:teuthology.orchestra.run.smithi125.stderr:Error EINVAL: specified pgp_num 81 > pg_num 31 2023-10-09T11:03:31.470 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:31.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:31.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:31.567 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:31 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:31.567 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:31 smithi155 ceph-mon[184854]: failed to encode map e457 with expected crc 2023-10-09T11:03:31.567 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:31 smithi155 ceph-mon[184854]: failed to encode map e457 with expected crc 2023-10-09T11:03:31.567 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:31 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:03:31.567 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:31 smithi155 ceph-mon[184854]: osdmap e457: 8 total, 7 up, 8 in 2023-10-09T11:03:31.573 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:31.628 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:31.682 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:31.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:31.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:31.791 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:31.852 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:31.895 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T11:03:31.903 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:31.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:31.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:32.008 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:32.066 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:32.173 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:03:32.173 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:03:32.204 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:03:32.204 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:03:32.222 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:03:32.239 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:32.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:32.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:32.253 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:03:32.297 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:03:32.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:32 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3747772036' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-10-09T11:03:32.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:32 smithi125 ceph-mon[195507]: pgmap v1127: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 27169/185562 objects degraded (14.641%); 5103/185562 objects misplaced (2.750%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T11:03:32.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:32 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3314220980' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:32.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:32 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3747772036' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-10-09T11:03:32.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:32 smithi125 ceph-mon[200197]: pgmap v1127: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 27169/185562 objects degraded (14.641%); 5103/185562 objects misplaced (2.750%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T11:03:32.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:32 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3314220980' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:32.328 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:03:32.344 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:32.364 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:03:32.364 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:03:32.364 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:03:32.364 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:03:32.364 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T11:03:32.364 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:03:32.364 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:03:32.365 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:03:32.365 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:03:32.389 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:32 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3747772036' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-10-09T11:03:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:32 smithi155 ceph-mon[184854]: pgmap v1127: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 27169/185562 objects degraded (14.641%); 5103/185562 objects misplaced (2.750%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T11:03:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:32 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3314220980' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:32.751 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:03:33.399 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:33.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:33 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 27169/185562 objects degraded (14.641%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:33.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:33 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 27169/185562 objects degraded (14.641%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:33.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:33 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 27169/185562 objects degraded (14.641%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:33.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:33.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:33.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:33.885 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T11:03:33.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:34.006 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:34.391 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:03:34.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:34 smithi125 ceph-mon[200197]: pgmap v1128: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 26326/178818 objects degraded (14.722%); 4428/178818 objects misplaced (2.476%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T11:03:34.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:34 smithi125 ceph-mon[195507]: pgmap v1128: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 26326/178818 objects degraded (14.722%); 4428/178818 objects misplaced (2.476%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T11:03:34.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:34 smithi155 ceph-mon[184854]: pgmap v1128: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 26326/178818 objects degraded (14.722%); 4428/178818 objects misplaced (2.476%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T11:03:35.129 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T11:03:35.129 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T11:03:35.138 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:35.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:35.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:35.242 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:35.291 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:35.375 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:35.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:35.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:35.482 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:35.545 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:35.586 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:35.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:35.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:35.692 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:35.760 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:35.868 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:35.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:35.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:35.975 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:36.027 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:36.049 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:36.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:36.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:36.162 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:36.245 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:36.279 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:36.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:36.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:36.382 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:36.435 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:36.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:36.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:36.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:36.587 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:36.599 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:03:36.600 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T11:03:36.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T11:03:36.600 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:03:36.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.600 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122023 9.d 9:b63f6ab3:::benchmark_data_smithi125_363789_object11724:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:03:36.600 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.154825+0000", 2023-10-09T11:03:36.600 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 268.43856958700002, 2023-10-09T11:03:36.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0345569450000001, 2023-10-09T11:03:36.600 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.600 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:03:36.601 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.601 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.601 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.601 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122023 2023-10-09T11:03:36.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.601 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154825+0000", 2023-10-09T11:03:36.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154825+0000", 2023-10-09T11:03:36.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154823+0000", 2023-10-09T11:03:36.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:36.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.603 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154829+0000", 2023-10-09T11:03:36.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6269999999999997e-06 2023-10-09T11:03:36.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.603 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154842+0000", 2023-10-09T11:03:36.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3869999999999999e-05 2023-10-09T11:03:36.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154849+0000", 2023-10-09T11:03:36.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5599999999999999e-06 2023-10-09T11:03:36.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154882+0000", 2023-10-09T11:03:36.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2580999999999999e-05 2023-10-09T11:03:36.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154977+0000", 2023-10-09T11:03:36.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5298999999999997e-05 2023-10-09T11:03:36.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,5", 2023-10-09T11:03:36.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.155058+0000", 2023-10-09T11:03:36.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1518000000000001e-05 2023-10-09T11:03:36.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.161278+0000", 2023-10-09T11:03:36.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062194049999999999 2023-10-09T11:03:36.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.162677+0000", 2023-10-09T11:03:36.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0013996379999999999 2023-10-09T11:03:36.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.180908+0000", 2023-10-09T11:03:36.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018230321000000001 2023-10-09T11:03:36.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.607 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.180916+0000", 2023-10-09T11:03:36.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7170000000000008e-06 2023-10-09T11:03:36.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.607 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.189335+0000", 2023-10-09T11:03:36.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.00841958 2023-10-09T11:03:36.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.608 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.608 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.189381+0000", 2023-10-09T11:03:36.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6349e-05 2023-10-09T11:03:36.608 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.608 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.608 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.608 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.608 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.608 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122031 9.3 9:cb1920df:::benchmark_data_smithi125_363789_object11732:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:03:36.608 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.154885+0000", 2023-10-09T11:03:36.608 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 268.43850868800001, 2023-10-09T11:03:36.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0272100049999999, 2023-10-09T11:03:36.609 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.609 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:03:36.609 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.609 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.609 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.609 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122031 2023-10-09T11:03:36.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.609 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154885+0000", 2023-10-09T11:03:36.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154885+0000", 2023-10-09T11:03:36.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154885+0000", 2023-10-09T11:03:36.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:36.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154888+0000", 2023-10-09T11:03:36.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6879999999999999e-06 2023-10-09T11:03:36.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154894+0000", 2023-10-09T11:03:36.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.823e-06 2023-10-09T11:03:36.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154897+0000", 2023-10-09T11:03:36.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8659999999999999e-06 2023-10-09T11:03:36.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.154941+0000", 2023-10-09T11:03:36.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3460000000000001e-05 2023-10-09T11:03:36.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.155049+0000", 2023-10-09T11:03:36.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000107932 2023-10-09T11:03:36.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.613 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.613 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:03:36.613 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.155156+0000", 2023-10-09T11:03:36.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010671300000000001 2023-10-09T11:03:36.613 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.613 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.613 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.613 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.161290+0000", 2023-10-09T11:03:36.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061342480000000001 2023-10-09T11:03:36.613 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.613 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.613 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.614 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178638+0000", 2023-10-09T11:03:36.614 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017348484000000001 2023-10-09T11:03:36.614 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.614 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.614 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.614 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178645+0000", 2023-10-09T11:03:36.614 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.613e-06 2023-10-09T11:03:36.614 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.614 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.614 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.614 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.182021+0000", 2023-10-09T11:03:36.614 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.003375959 2023-10-09T11:03:36.615 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.615 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.615 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.615 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.182095+0000", 2023-10-09T11:03:36.615 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4606999999999999e-05 2023-10-09T11:03:36.615 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.615 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.615 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.616 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.616 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122032 9.a 9:5ee579a3:::benchmark_data_smithi125_363789_object11733:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:03:36.616 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.163592+0000", 2023-10-09T11:03:36.616 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 268.42980209699999, 2023-10-09T11:03:36.616 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.03044172, 2023-10-09T11:03:36.616 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.616 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:03:36.616 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.616 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.616 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.617 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122032 2023-10-09T11:03:36.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.617 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163592+0000", 2023-10-09T11:03:36.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163592+0000", 2023-10-09T11:03:36.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163590+0000", 2023-10-09T11:03:36.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T11:03:36.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.619 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163601+0000", 2023-10-09T11:03:36.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1037e-05 2023-10-09T11:03:36.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.619 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163614+0000", 2023-10-09T11:03:36.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3001e-05 2023-10-09T11:03:36.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.620 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.620 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.163621+0000", 2023-10-09T11:03:36.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7270000000000001e-06 2023-10-09T11:03:36.620 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.620 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.620 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.620 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.164646+0000", 2023-10-09T11:03:36.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010251909999999999 2023-10-09T11:03:36.620 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.621 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.621 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op must wait for map", 2023-10-09T11:03:36.621 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.164654+0000", 2023-10-09T11:03:36.621 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5059999999999998e-06 2023-10-09T11:03:36.621 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.621 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.621 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.621 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178523+0000", 2023-10-09T11:03:36.621 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013869365999999999 2023-10-09T11:03:36.621 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.621 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.621 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.622 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178531+0000", 2023-10-09T11:03:36.622 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7510000000000008e-06 2023-10-09T11:03:36.622 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.622 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.622 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.622 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.192721+0000", 2023-10-09T11:03:36.622 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.014189995 2023-10-09T11:03:36.622 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.622 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.622 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.622 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.194034+0000", 2023-10-09T11:03:36.623 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001312917 2023-10-09T11:03:36.623 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.623 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.623 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.623 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.623 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.623 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122035 9.3 9:cf84482a:::benchmark_data_smithi125_363789_object11736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:03:36.623 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.176778+0000", 2023-10-09T11:03:36.623 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 268.41661589400002, 2023-10-09T11:03:36.623 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0230526680000001, 2023-10-09T11:03:36.623 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.624 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.624 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.624 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.624 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.624 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122035 2023-10-09T11:03:36.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.624 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176778+0000", 2023-10-09T11:03:36.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.625 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176778+0000", 2023-10-09T11:03:36.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.626 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176777+0000", 2023-10-09T11:03:36.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:03:36.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.626 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176782+0000", 2023-10-09T11:03:36.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.374e-06 2023-10-09T11:03:36.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176790+0000", 2023-10-09T11:03:36.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2199999999999992e-06 2023-10-09T11:03:36.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.176795+0000", 2023-10-09T11:03:36.628 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5099999999999998e-06 2023-10-09T11:03:36.628 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.628 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.628 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178896+0000", 2023-10-09T11:03:36.628 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021008910000000001 2023-10-09T11:03:36.628 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.629 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.629 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.178902+0000", 2023-10-09T11:03:36.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0800000000000002e-06 2023-10-09T11:03:36.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.629 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.629 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.629 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.182122+0000", 2023-10-09T11:03:36.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.003219329 2023-10-09T11:03:36.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.629 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.630 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.630 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.182244+0000", 2023-10-09T11:03:36.630 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012187 2023-10-09T11:03:36.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.630 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3", 2023-10-09T11:03:36.630 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.182386+0000", 2023-10-09T11:03:36.630 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000142373 2023-10-09T11:03:36.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.630 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.190416+0000", 2023-10-09T11:03:36.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0080298230000000002 2023-10-09T11:03:36.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199736+0000", 2023-10-09T11:03:36.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0093200999999999996 2023-10-09T11:03:36.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199767+0000", 2023-10-09T11:03:36.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1440999999999999e-05 2023-10-09T11:03:36.632 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.632 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.632 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199831+0000", 2023-10-09T11:03:36.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3633999999999994e-05 2023-10-09T11:03:36.632 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.632 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.632 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.632 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.632 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122040 9.3 9:cc3a16b1:::benchmark_data_smithi125_363789_object11741:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:03:36.632 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.187372+0000", 2023-10-09T11:03:36.633 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 268.40602200400002, 2023-10-09T11:03:36.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.012574294, 2023-10-09T11:03:36.633 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.633 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.633 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.633 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.633 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.633 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122040 2023-10-09T11:03:36.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.633 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187372+0000", 2023-10-09T11:03:36.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187372+0000", 2023-10-09T11:03:36.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187371+0000", 2023-10-09T11:03:36.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:36.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.635 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187375+0000", 2023-10-09T11:03:36.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6299999999999997e-06 2023-10-09T11:03:36.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.635 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187385+0000", 2023-10-09T11:03:36.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9709999999999999e-06 2023-10-09T11:03:36.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187392+0000", 2023-10-09T11:03:36.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7299999999999999e-06 2023-10-09T11:03:36.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187440+0000", 2023-10-09T11:03:36.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7558000000000001e-05 2023-10-09T11:03:36.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.187449+0000", 2023-10-09T11:03:36.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3740000000000004e-06 2023-10-09T11:03:36.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.185252+0000", 2023-10-09T11:03:36.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.99780342499999997 2023-10-09T11:03:36.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.185378+0000", 2023-10-09T11:03:36.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012527 2023-10-09T11:03:36.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3", 2023-10-09T11:03:36.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.185510+0000", 2023-10-09T11:03:36.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013254599999999999 2023-10-09T11:03:36.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.190604+0000", 2023-10-09T11:03:36.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0050936460000000003 2023-10-09T11:03:36.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199865+0000", 2023-10-09T11:03:36.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0092616620000000004 2023-10-09T11:03:36.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199897+0000", 2023-10-09T11:03:36.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1223000000000002e-05 2023-10-09T11:03:36.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.199946+0000", 2023-10-09T11:03:36.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9728999999999998e-05 2023-10-09T11:03:36.640 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.640 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.640 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.640 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:122046 9.3 9:cc593156:::benchmark_data_smithi125_363789_object11747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:03:36.640 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:08.201649+0000", 2023-10-09T11:03:36.640 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 268.39174555599999, 2023-10-09T11:03:36.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.010992307, 2023-10-09T11:03:36.640 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.640 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.640 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.641 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.641 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.641 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122046 2023-10-09T11:03:36.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.641 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201649+0000", 2023-10-09T11:03:36.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201649+0000", 2023-10-09T11:03:36.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.642 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201647+0000", 2023-10-09T11:03:36.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:36.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.642 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201653+0000", 2023-10-09T11:03:36.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1840000000000004e-06 2023-10-09T11:03:36.643 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.643 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.643 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.643 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201665+0000", 2023-10-09T11:03:36.643 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1396999999999999e-05 2023-10-09T11:03:36.643 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.643 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.643 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.643 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201671+0000", 2023-10-09T11:03:36.643 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3849999999999999e-06 2023-10-09T11:03:36.644 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.644 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.644 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.644 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201716+0000", 2023-10-09T11:03:36.644 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4969999999999998e-05 2023-10-09T11:03:36.644 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.644 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.644 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.644 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:08.201725+0000", 2023-10-09T11:03:36.644 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3589999999999997e-06 2023-10-09T11:03:36.644 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.645 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.187839+0000", 2023-10-09T11:03:36.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98611354799999995 2023-10-09T11:03:36.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.645 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.187980+0000", 2023-10-09T11:03:36.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014071799999999999 2023-10-09T11:03:36.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3", 2023-10-09T11:03:36.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.188118+0000", 2023-10-09T11:03:36.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013808599999999999 2023-10-09T11:03:36.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.190608+0000", 2023-10-09T11:03:36.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00249037 2023-10-09T11:03:36.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.212484+0000", 2023-10-09T11:03:36.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021876442999999999 2023-10-09T11:03:36.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.647 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.647 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.212541+0000", 2023-10-09T11:03:36.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6255999999999998e-05 2023-10-09T11:03:36.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.647 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.647 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:09.212641+0000", 2023-10-09T11:03:36.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010011200000000001 2023-10-09T11:03:36.648 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.648 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.648 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.648 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150423 9.9 9:91f76946:::benchmark_data_smithi125_363789_object40124:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:03:36.648 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.193246+0000", 2023-10-09T11:03:36.648 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 233.400148097, 2023-10-09T11:03:36.648 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93784610000000002, 2023-10-09T11:03:36.648 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.648 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.649 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.649 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.649 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.649 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150423 2023-10-09T11:03:36.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.649 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.649 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193246+0000", 2023-10-09T11:03:36.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.650 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.650 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193246+0000", 2023-10-09T11:03:36.650 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.650 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.650 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193245+0000", 2023-10-09T11:03:36.650 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:36.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.650 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.651 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193249+0000", 2023-10-09T11:03:36.651 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2279999999999998e-06 2023-10-09T11:03:36.651 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.651 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.651 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.651 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193256+0000", 2023-10-09T11:03:36.651 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1010000000000003e-06 2023-10-09T11:03:36.651 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.651 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.651 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.651 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193260+0000", 2023-10-09T11:03:36.651 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3409999999999996e-06 2023-10-09T11:03:36.652 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.652 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.652 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.652 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193306+0000", 2023-10-09T11:03:36.652 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5636000000000001e-05 2023-10-09T11:03:36.652 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.652 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.652 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.652 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193313+0000", 2023-10-09T11:03:36.652 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0849999999999999e-06 2023-10-09T11:03:36.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.653 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.653 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.121393+0000", 2023-10-09T11:03:36.653 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92808017799999998 2023-10-09T11:03:36.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.653 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.653 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.121556+0000", 2023-10-09T11:03:36.653 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016344600000000001 2023-10-09T11:03:36.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:03:36.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.121862+0000", 2023-10-09T11:03:36.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030565099999999999 2023-10-09T11:03:36.654 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.126975+0000", 2023-10-09T11:03:36.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051129119999999998 2023-10-09T11:03:36.654 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.655 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.130983+0000", 2023-10-09T11:03:36.655 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040081470000000001 2023-10-09T11:03:36.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.655 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.655 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131014+0000", 2023-10-09T11:03:36.655 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0698000000000003e-05 2023-10-09T11:03:36.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.655 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.655 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131055+0000", 2023-10-09T11:03:36.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1390999999999997e-05 2023-10-09T11:03:36.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.656 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131092+0000", 2023-10-09T11:03:36.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6896e-05 2023-10-09T11:03:36.656 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.656 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.656 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.657 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150426 9.9 9:911979c4:::benchmark_data_smithi125_363789_object40127:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:03:36.657 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.193290+0000", 2023-10-09T11:03:36.657 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 233.400104586, 2023-10-09T11:03:36.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93976706099999996, 2023-10-09T11:03:36.657 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.657 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.657 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.657 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.657 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.657 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150426 2023-10-09T11:03:36.657 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.657 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.658 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.658 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193290+0000", 2023-10-09T11:03:36.658 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.658 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.658 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193290+0000", 2023-10-09T11:03:36.658 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.659 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193289+0000", 2023-10-09T11:03:36.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:36.659 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.659 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193291+0000", 2023-10-09T11:03:36.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2649999999999999e-06 2023-10-09T11:03:36.659 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193297+0000", 2023-10-09T11:03:36.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5210000000000002e-06 2023-10-09T11:03:36.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.660 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193301+0000", 2023-10-09T11:03:36.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7730000000000001e-06 2023-10-09T11:03:36.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.660 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193345+0000", 2023-10-09T11:03:36.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3959999999999999e-05 2023-10-09T11:03:36.661 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.661 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.661 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.193350+0000", 2023-10-09T11:03:36.661 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9949999999999996e-06 2023-10-09T11:03:36.661 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.661 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.661 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.123175+0000", 2023-10-09T11:03:36.661 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92982503900000002 2023-10-09T11:03:36.661 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.662 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.662 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.123286+0000", 2023-10-09T11:03:36.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000111036 2023-10-09T11:03:36.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.662 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.662 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:03:36.662 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.123498+0000", 2023-10-09T11:03:36.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00021212800000000001 2023-10-09T11:03:36.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.662 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.662 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.663 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131127+0000", 2023-10-09T11:03:36.663 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0076286820000000003 2023-10-09T11:03:36.663 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.663 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.663 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131182+0000", 2023-10-09T11:03:36.663 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5912999999999998e-05 2023-10-09T11:03:36.663 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.663 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.663 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.132971+0000", 2023-10-09T11:03:36.663 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001788191 2023-10-09T11:03:36.664 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.664 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.664 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.664 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133015+0000", 2023-10-09T11:03:36.664 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4014000000000002e-05 2023-10-09T11:03:36.664 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.664 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.664 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.664 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133057+0000", 2023-10-09T11:03:36.664 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.193e-05 2023-10-09T11:03:36.664 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.664 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.665 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.665 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.665 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.665 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150439 9.9 9:92459488:::benchmark_data_smithi125_363789_object40140:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:03:36.665 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.207668+0000", 2023-10-09T11:03:36.665 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 233.38572647699999, 2023-10-09T11:03:36.665 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92585638599999998, 2023-10-09T11:03:36.665 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.665 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.665 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.665 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.666 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.666 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150439 2023-10-09T11:03:36.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.666 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.666 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.666 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207668+0000", 2023-10-09T11:03:36.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207668+0000", 2023-10-09T11:03:36.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207667+0000", 2023-10-09T11:03:36.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:36.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207671+0000", 2023-10-09T11:03:36.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2339999999999996e-06 2023-10-09T11:03:36.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207680+0000", 2023-10-09T11:03:36.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1670000000000005e-06 2023-10-09T11:03:36.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.207686+0000", 2023-10-09T11:03:36.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3720000000000004e-06 2023-10-09T11:03:36.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.208495+0000", 2023-10-09T11:03:36.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000809142 2023-10-09T11:03:36.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.208502+0000", 2023-10-09T11:03:36.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9210000000000003e-06 2023-10-09T11:03:36.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.124568+0000", 2023-10-09T11:03:36.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.91606632200000004 2023-10-09T11:03:36.670 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.124637+0000", 2023-10-09T11:03:36.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8700000000000003e-05 2023-10-09T11:03:36.670 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:03:36.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.124777+0000", 2023-10-09T11:03:36.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013975399999999999 2023-10-09T11:03:36.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.671 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.671 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.671 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131155+0000", 2023-10-09T11:03:36.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0063785200000000004 2023-10-09T11:03:36.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.671 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.671 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.671 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133065+0000", 2023-10-09T11:03:36.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019096849999999999 2023-10-09T11:03:36.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.672 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.672 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133440+0000", 2023-10-09T11:03:36.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00037491400000000001 2023-10-09T11:03:36.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.672 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.672 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133486+0000", 2023-10-09T11:03:36.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6683e-05 2023-10-09T11:03:36.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133524+0000", 2023-10-09T11:03:36.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7685000000000003e-05 2023-10-09T11:03:36.673 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.673 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.673 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.673 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150470 9.9 9:9415b14b:::benchmark_data_smithi125_363789_object40171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:03:36.673 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.229566+0000", 2023-10-09T11:03:36.674 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 233.363828074, 2023-10-09T11:03:36.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.90407114499999996, 2023-10-09T11:03:36.674 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.674 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.674 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.674 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.674 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.674 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150470 2023-10-09T11:03:36.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.674 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229566+0000", 2023-10-09T11:03:36.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229566+0000", 2023-10-09T11:03:36.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229566+0000", 2023-10-09T11:03:36.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:36.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229568+0000", 2023-10-09T11:03:36.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.943e-06 2023-10-09T11:03:36.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229574+0000", 2023-10-09T11:03:36.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.003e-06 2023-10-09T11:03:36.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229579+0000", 2023-10-09T11:03:36.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0790000000000002e-06 2023-10-09T11:03:36.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229656+0000", 2023-10-09T11:03:36.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6940999999999994e-05 2023-10-09T11:03:36.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.229662+0000", 2023-10-09T11:03:36.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7020000000000004e-06 2023-10-09T11:03:36.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.125755+0000", 2023-10-09T11:03:36.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.89609301900000005 2023-10-09T11:03:36.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.679 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.679 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.125827+0000", 2023-10-09T11:03:36.679 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1804999999999994e-05 2023-10-09T11:03:36.679 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.679 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:03:36.679 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.125995+0000", 2023-10-09T11:03:36.679 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016823900000000001 2023-10-09T11:03:36.679 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.679 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.679 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131210+0000", 2023-10-09T11:03:36.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0052145810000000003 2023-10-09T11:03:36.680 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.680 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.680 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.680 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133073+0000", 2023-10-09T11:03:36.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018628869999999999 2023-10-09T11:03:36.680 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.680 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.680 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.680 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133562+0000", 2023-10-09T11:03:36.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00048968299999999998 2023-10-09T11:03:36.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.681 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133602+0000", 2023-10-09T11:03:36.681 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0022999999999999e-05 2023-10-09T11:03:36.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.681 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133637+0000", 2023-10-09T11:03:36.681 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4715999999999998e-05 2023-10-09T11:03:36.681 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.682 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.682 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.682 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.682 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.682 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150495 9.9 9:90b0d4cb:::benchmark_data_smithi125_363789_object40196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:03:36.682 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.248473+0000", 2023-10-09T11:03:36.682 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 233.34492076699999, 2023-10-09T11:03:36.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.88740656399999995, 2023-10-09T11:03:36.682 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.682 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.682 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.682 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.683 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.683 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150495 2023-10-09T11:03:36.683 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.683 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.683 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.683 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248473+0000", 2023-10-09T11:03:36.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.683 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.683 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.683 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248473+0000", 2023-10-09T11:03:36.684 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.684 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.684 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.684 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.684 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248472+0000", 2023-10-09T11:03:36.684 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:03:36.684 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.684 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.684 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.684 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248483+0000", 2023-10-09T11:03:36.684 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0905e-05 2023-10-09T11:03:36.684 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.685 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.685 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.685 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248494+0000", 2023-10-09T11:03:36.685 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1153e-05 2023-10-09T11:03:36.685 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.685 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.685 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.685 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248501+0000", 2023-10-09T11:03:36.685 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2570000000000004e-06 2023-10-09T11:03:36.685 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.685 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.686 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.686 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248547+0000", 2023-10-09T11:03:36.686 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6202999999999998e-05 2023-10-09T11:03:36.686 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.686 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.686 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.686 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.248555+0000", 2023-10-09T11:03:36.686 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4530000000000002e-06 2023-10-09T11:03:36.686 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.686 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.686 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.687 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.127018+0000", 2023-10-09T11:03:36.687 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.878463302 2023-10-09T11:03:36.687 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.687 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.687 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.687 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.127131+0000", 2023-10-09T11:03:36.687 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011225 2023-10-09T11:03:36.687 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.687 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.687 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:03:36.687 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.127305+0000", 2023-10-09T11:03:36.687 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017452 2023-10-09T11:03:36.688 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.688 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.688 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.688 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.131237+0000", 2023-10-09T11:03:36.688 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039315369999999997 2023-10-09T11:03:36.688 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.688 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.688 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.688 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133688+0000", 2023-10-09T11:03:36.688 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002451614 2023-10-09T11:03:36.688 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.688 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.689 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.689 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.135791+0000", 2023-10-09T11:03:36.689 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002102746 2023-10-09T11:03:36.689 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.689 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.689 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.689 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.135837+0000", 2023-10-09T11:03:36.689 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5710000000000001e-05 2023-10-09T11:03:36.689 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.689 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.689 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.689 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.135880+0000", 2023-10-09T11:03:36.690 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3037999999999997e-05 2023-10-09T11:03:36.690 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.690 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.690 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.690 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.690 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.690 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150552 9.9 9:90f11fdc:::benchmark_data_smithi125_363789_object40253:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:03:36.690 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.294722+0000", 2023-10-09T11:03:36.690 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 233.298672033, 2023-10-09T11:03:36.690 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.84415905599999996, 2023-10-09T11:03:36.690 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.691 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.691 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.691 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.691 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.691 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150552 2023-10-09T11:03:36.691 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.691 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.691 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.691 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.691 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294722+0000", 2023-10-09T11:03:36.691 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.692 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.692 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.692 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.692 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294722+0000", 2023-10-09T11:03:36.692 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.692 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.692 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.692 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.692 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294721+0000", 2023-10-09T11:03:36.692 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:03:36.692 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.693 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.693 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.693 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294727+0000", 2023-10-09T11:03:36.693 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5539999999999997e-06 2023-10-09T11:03:36.693 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.693 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.693 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.693 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294734+0000", 2023-10-09T11:03:36.693 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2389999999999996e-06 2023-10-09T11:03:36.693 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.693 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.693 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.694 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294739+0000", 2023-10-09T11:03:36.694 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2279999999999998e-06 2023-10-09T11:03:36.694 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.694 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.694 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.694 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294780+0000", 2023-10-09T11:03:36.694 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1128000000000003e-05 2023-10-09T11:03:36.694 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.694 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.694 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.694 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.294788+0000", 2023-10-09T11:03:36.694 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4499999999999998e-06 2023-10-09T11:03:36.695 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.695 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.695 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.695 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.128120+0000", 2023-10-09T11:03:36.695 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.83333184999999999 2023-10-09T11:03:36.695 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.695 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.695 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.695 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.128227+0000", 2023-10-09T11:03:36.695 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010778200000000001 2023-10-09T11:03:36.695 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.695 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.696 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:03:36.696 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.128411+0000", 2023-10-09T11:03:36.696 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000183117 2023-10-09T11:03:36.696 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.696 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.696 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.696 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.132962+0000", 2023-10-09T11:03:36.696 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045515520000000004 2023-10-09T11:03:36.696 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.696 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.696 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.696 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133727+0000", 2023-10-09T11:03:36.697 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00076511400000000001 2023-10-09T11:03:36.697 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.697 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.697 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.697 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138805+0000", 2023-10-09T11:03:36.697 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0050773340000000002 2023-10-09T11:03:36.697 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.697 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.697 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.697 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138840+0000", 2023-10-09T11:03:36.697 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5831000000000003e-05 2023-10-09T11:03:36.697 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.698 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.698 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.698 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138881+0000", 2023-10-09T11:03:36.698 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0710999999999997e-05 2023-10-09T11:03:36.698 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.698 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.698 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.698 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.698 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.698 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:150590 9.9 9:909dc84e:::benchmark_data_smithi125_363789_object40291:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-10-09T11:03:36.698 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:43.333845+0000", 2023-10-09T11:03:36.698 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 233.25954863499999, 2023-10-09T11:03:36.699 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.80510336599999999, 2023-10-09T11:03:36.699 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.699 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.699 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.699 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.699 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.699 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 150590 2023-10-09T11:03:36.699 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.699 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.699 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.699 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.699 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333845+0000", 2023-10-09T11:03:36.699 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.700 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.700 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333845+0000", 2023-10-09T11:03:36.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.700 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.700 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333845+0000", 2023-10-09T11:03:36.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:36.700 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.700 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.701 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.701 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333850+0000", 2023-10-09T11:03:36.701 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7879999999999997e-06 2023-10-09T11:03:36.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.701 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.701 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.701 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333858+0000", 2023-10-09T11:03:36.701 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7569999999999993e-06 2023-10-09T11:03:36.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.701 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.701 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.333863+0000", 2023-10-09T11:03:36.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.724e-06 2023-10-09T11:03:36.702 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.335088+0000", 2023-10-09T11:03:36.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012250270000000001 2023-10-09T11:03:36.702 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:43.335095+0000", 2023-10-09T11:03:36.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7660000000000001e-06 2023-10-09T11:03:36.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.129225+0000", 2023-10-09T11:03:36.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.79412993399999998 2023-10-09T11:03:36.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.129291+0000", 2023-10-09T11:03:36.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6072e-05 2023-10-09T11:03:36.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.704 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,5", 2023-10-09T11:03:36.704 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.129474+0000", 2023-10-09T11:03:36.704 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018292800000000001 2023-10-09T11:03:36.704 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.704 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.704 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.704 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.133175+0000", 2023-10-09T11:03:36.704 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0037009399999999998 2023-10-09T11:03:36.704 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.704 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.704 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.135784+0000", 2023-10-09T11:03:36.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026093900000000001 2023-10-09T11:03:36.705 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.705 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.705 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138888+0000", 2023-10-09T11:03:36.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0031041950000000001 2023-10-09T11:03:36.705 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.705 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.705 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138921+0000", 2023-10-09T11:03:36.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3025999999999999e-05 2023-10-09T11:03:36.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.706 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.706 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.706 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:44.138949+0000", 2023-10-09T11:03:36.706 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7532999999999999e-05 2023-10-09T11:03:36.706 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.706 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.706 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.706 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.706 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204565 9.13 9:cfe54d63:::benchmark_data_smithi125_363789_object94266:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T11:03:36.706 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:41.994119+0000", 2023-10-09T11:03:36.707 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 174.59927482099999, 2023-10-09T11:03:36.707 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.018728254, 2023-10-09T11:03:36.707 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.707 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:03:36.707 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.707 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.707 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.707 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204565 2023-10-09T11:03:36.707 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.707 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.707 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.707 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994119+0000", 2023-10-09T11:03:36.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.708 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.708 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994119+0000", 2023-10-09T11:03:36.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.708 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.708 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994118+0000", 2023-10-09T11:03:36.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:36.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.709 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994124+0000", 2023-10-09T11:03:36.709 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6509999999999998e-06 2023-10-09T11:03:36.709 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.709 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994133+0000", 2023-10-09T11:03:36.709 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1109999999999997e-06 2023-10-09T11:03:36.709 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.710 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.710 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994137+0000", 2023-10-09T11:03:36.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5879999999999999e-06 2023-10-09T11:03:36.710 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.710 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.710 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.710 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994205+0000", 2023-10-09T11:03:36.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7852e-05 2023-10-09T11:03:36.710 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.710 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.710 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.710 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994355+0000", 2023-10-09T11:03:36.711 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015022700000000001 2023-10-09T11:03:36.711 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.711 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.711 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:03:36.711 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.994806+0000", 2023-10-09T11:03:36.711 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00045023900000000001 2023-10-09T11:03:36.711 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.711 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.711 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.711 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:41.997059+0000", 2023-10-09T11:03:36.711 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022531949999999999 2023-10-09T11:03:36.712 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.712 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.712 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.003461+0000", 2023-10-09T11:03:36.712 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064024329999999999 2023-10-09T11:03:36.712 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.712 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.712 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.003480+0000", 2023-10-09T11:03:36.712 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8272000000000001e-05 2023-10-09T11:03:36.712 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.712 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.713 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.012782+0000", 2023-10-09T11:03:36.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0093026169999999 2023-10-09T11:03:36.713 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.713 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.713 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.713 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.012848+0000", 2023-10-09T11:03:36.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5415999999999998e-05 2023-10-09T11:03:36.713 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.713 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.713 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.713 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.714 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.714 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204572 9.13 9:cc352b3f:::benchmark_data_smithi125_363789_object94273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:03:36.714 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.008011+0000", 2023-10-09T11:03:36.714 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 174.58538338400001, 2023-10-09T11:03:36.714 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0262834300000001, 2023-10-09T11:03:36.714 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.714 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.714 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.714 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.714 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.714 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204572 2023-10-09T11:03:36.714 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.715 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008011+0000", 2023-10-09T11:03:36.715 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.715 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008011+0000", 2023-10-09T11:03:36.715 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.715 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.716 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008010+0000", 2023-10-09T11:03:36.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:36.716 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.716 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008014+0000", 2023-10-09T11:03:36.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9009999999999996e-06 2023-10-09T11:03:36.716 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.716 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.717 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008025+0000", 2023-10-09T11:03:36.717 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0730999999999999e-05 2023-10-09T11:03:36.717 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.717 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.717 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.717 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008032+0000", 2023-10-09T11:03:36.717 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9970000000000001e-06 2023-10-09T11:03:36.717 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.717 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.717 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.717 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008125+0000", 2023-10-09T11:03:36.718 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2991999999999993e-05 2023-10-09T11:03:36.718 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.718 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.718 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.718 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.008136+0000", 2023-10-09T11:03:36.718 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1068e-05 2023-10-09T11:03:36.718 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.718 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.718 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.718 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.012867+0000", 2023-10-09T11:03:36.718 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.004731034 2023-10-09T11:03:36.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.719 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.719 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.012951+0000", 2023-10-09T11:03:36.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3851999999999996e-05 2023-10-09T11:03:36.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.719 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.719 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:03:36.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.013280+0000", 2023-10-09T11:03:36.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00032942300000000002 2023-10-09T11:03:36.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.719 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.720 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.021065+0000", 2023-10-09T11:03:36.720 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0077846759999999999 2023-10-09T11:03:36.720 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.720 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034119+0000", 2023-10-09T11:03:36.720 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013054412 2023-10-09T11:03:36.720 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.721 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034203+0000", 2023-10-09T11:03:36.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4079999999999995e-05 2023-10-09T11:03:36.721 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.721 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.721 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.721 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034252+0000", 2023-10-09T11:03:36.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8622999999999998e-05 2023-10-09T11:03:36.721 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.721 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.721 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.721 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034294+0000", 2023-10-09T11:03:36.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2256000000000003e-05 2023-10-09T11:03:36.722 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.722 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.722 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.722 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.722 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.722 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204591 9.13 9:cfb36959:::benchmark_data_smithi125_363789_object94292:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:03:36.722 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.043404+0000", 2023-10-09T11:03:36.722 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 174.549990334, 2023-10-09T11:03:36.722 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.990990861, 2023-10-09T11:03:36.722 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.722 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.722 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.723 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.723 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.723 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204591 2023-10-09T11:03:36.723 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.723 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.723 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.723 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043404+0000", 2023-10-09T11:03:36.723 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.723 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.723 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.723 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.724 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043404+0000", 2023-10-09T11:03:36.724 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.724 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.724 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.724 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.724 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043402+0000", 2023-10-09T11:03:36.724 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T11:03:36.724 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.724 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.724 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.724 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043412+0000", 2023-10-09T11:03:36.725 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0499000000000001e-05 2023-10-09T11:03:36.725 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.725 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.725 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.725 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043433+0000", 2023-10-09T11:03:36.725 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0477999999999999e-05 2023-10-09T11:03:36.725 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.725 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.725 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.725 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043442+0000", 2023-10-09T11:03:36.725 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9609999999999994e-06 2023-10-09T11:03:36.725 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.726 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.726 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.726 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043619+0000", 2023-10-09T11:03:36.726 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000177558 2023-10-09T11:03:36.726 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.726 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.726 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.726 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.043630+0000", 2023-10-09T11:03:36.726 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0382e-05 2023-10-09T11:03:36.726 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.726 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.726 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.727 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.014803+0000", 2023-10-09T11:03:36.727 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97117335900000001 2023-10-09T11:03:36.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.727 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.727 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.014927+0000", 2023-10-09T11:03:36.727 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012421900000000001 2023-10-09T11:03:36.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.727 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:03:36.727 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.015207+0000", 2023-10-09T11:03:36.727 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00027950199999999999 2023-10-09T11:03:36.728 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.728 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.022320+0000", 2023-10-09T11:03:36.728 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0071137290000000001 2023-10-09T11:03:36.728 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.728 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034155+0000", 2023-10-09T11:03:36.728 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011834905 2023-10-09T11:03:36.728 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.728 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.729 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.729 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034329+0000", 2023-10-09T11:03:36.729 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017399200000000001 2023-10-09T11:03:36.729 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.729 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.729 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.729 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034361+0000", 2023-10-09T11:03:36.729 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1545999999999999e-05 2023-10-09T11:03:36.729 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.729 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.729 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.730 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034395+0000", 2023-10-09T11:03:36.730 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3772999999999997e-05 2023-10-09T11:03:36.730 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.730 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.730 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.730 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.730 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204618 9.13 9:c9e84990:::benchmark_data_smithi125_363789_object94319:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:03:36.730 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.073111+0000", 2023-10-09T11:03:36.730 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 174.520282662, 2023-10-09T11:03:36.730 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.96151479900000003, 2023-10-09T11:03:36.730 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.730 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.731 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.731 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.731 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.731 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204618 2023-10-09T11:03:36.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.731 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.731 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.731 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.731 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073111+0000", 2023-10-09T11:03:36.731 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073111+0000", 2023-10-09T11:03:36.732 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.732 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073110+0000", 2023-10-09T11:03:36.732 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:36.732 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073117+0000", 2023-10-09T11:03:36.733 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9820000000000002e-06 2023-10-09T11:03:36.733 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.733 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.733 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.733 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073129+0000", 2023-10-09T11:03:36.733 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1608e-05 2023-10-09T11:03:36.733 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.733 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.733 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.733 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073135+0000", 2023-10-09T11:03:36.734 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0249999999999999e-06 2023-10-09T11:03:36.734 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.734 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.734 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.734 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073191+0000", 2023-10-09T11:03:36.734 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5810999999999998e-05 2023-10-09T11:03:36.734 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.734 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.734 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.734 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.073197+0000", 2023-10-09T11:03:36.735 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3579999999999999e-06 2023-10-09T11:03:36.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.735 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.735 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.016587+0000", 2023-10-09T11:03:36.735 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.94339012799999999 2023-10-09T11:03:36.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.735 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.735 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.016657+0000", 2023-10-09T11:03:36.735 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0301000000000003e-05 2023-10-09T11:03:36.736 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.736 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:03:36.736 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.016830+0000", 2023-10-09T11:03:36.736 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017296799999999999 2023-10-09T11:03:36.736 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.736 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.736 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.022324+0000", 2023-10-09T11:03:36.736 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0054936739999999996 2023-10-09T11:03:36.736 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.737 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.737 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034436+0000", 2023-10-09T11:03:36.737 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012112327000000001 2023-10-09T11:03:36.737 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.737 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.737 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.737 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034545+0000", 2023-10-09T11:03:36.737 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000108213 2023-10-09T11:03:36.737 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.737 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.737 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.738 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034587+0000", 2023-10-09T11:03:36.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2840000000000003e-05 2023-10-09T11:03:36.738 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.738 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.738 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.738 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034626+0000", 2023-10-09T11:03:36.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8868999999999997e-05 2023-10-09T11:03:36.738 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.738 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.738 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.738 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.738 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.739 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204624 9.13 9:cb2a155f:::benchmark_data_smithi125_363789_object94325:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:03:36.739 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.084343+0000", 2023-10-09T11:03:36.739 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 174.50905087000001, 2023-10-09T11:03:36.739 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95037137000000005, 2023-10-09T11:03:36.739 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.739 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.739 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.739 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.739 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.739 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204624 2023-10-09T11:03:36.739 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.739 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.740 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.740 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.740 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084343+0000", 2023-10-09T11:03:36.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.740 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.740 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.740 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.740 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084343+0000", 2023-10-09T11:03:36.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.740 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.740 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.740 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.740 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084340+0000", 2023-10-09T11:03:36.741 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999967 2023-10-09T11:03:36.741 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.741 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.741 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.741 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084352+0000", 2023-10-09T11:03:36.741 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1593999999999999e-05 2023-10-09T11:03:36.741 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.741 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.741 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.741 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084367+0000", 2023-10-09T11:03:36.741 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5668e-05 2023-10-09T11:03:36.741 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.742 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.742 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.742 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084373+0000", 2023-10-09T11:03:36.742 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5799999999999999e-06 2023-10-09T11:03:36.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.742 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.742 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.742 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084441+0000", 2023-10-09T11:03:36.742 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8324999999999994e-05 2023-10-09T11:03:36.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.742 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.742 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.742 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.084453+0000", 2023-10-09T11:03:36.743 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1297999999999999e-05 2023-10-09T11:03:36.743 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.743 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.743 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.743 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.017927+0000", 2023-10-09T11:03:36.743 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93347489400000005 2023-10-09T11:03:36.743 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.743 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.743 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.743 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.018027+0000", 2023-10-09T11:03:36.743 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9511000000000003e-05 2023-10-09T11:03:36.744 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.744 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.744 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:03:36.744 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.018203+0000", 2023-10-09T11:03:36.744 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017629500000000001 2023-10-09T11:03:36.744 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.744 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.744 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.744 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.022327+0000", 2023-10-09T11:03:36.744 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041237510000000002 2023-10-09T11:03:36.744 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.744 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.744 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.745 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034475+0000", 2023-10-09T11:03:36.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012147857999999999 2023-10-09T11:03:36.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.745 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034657+0000", 2023-10-09T11:03:36.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000182504 2023-10-09T11:03:36.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034684+0000", 2023-10-09T11:03:36.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6936000000000001e-05 2023-10-09T11:03:36.746 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.746 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034715+0000", 2023-10-09T11:03:36.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0264e-05 2023-10-09T11:03:36.746 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.746 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.746 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.746 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.747 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204699 9.13 9:ca5a965b:::benchmark_data_smithi125_363789_object94400:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:03:36.747 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.182784+0000", 2023-10-09T11:03:36.747 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 174.410610099, 2023-10-09T11:03:36.747 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.85207209399999995, 2023-10-09T11:03:36.747 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.747 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.747 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.747 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.747 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.747 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204699 2023-10-09T11:03:36.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.748 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.748 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.748 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182784+0000", 2023-10-09T11:03:36.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.748 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.748 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182784+0000", 2023-10-09T11:03:36.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.748 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182783+0000", 2023-10-09T11:03:36.749 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:36.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182787+0000", 2023-10-09T11:03:36.749 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.377e-06 2023-10-09T11:03:36.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.750 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182797+0000", 2023-10-09T11:03:36.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9119999999999993e-06 2023-10-09T11:03:36.750 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.750 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.750 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.750 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182801+0000", 2023-10-09T11:03:36.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6480000000000002e-06 2023-10-09T11:03:36.750 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.750 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.750 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.750 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182860+0000", 2023-10-09T11:03:36.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8857000000000002e-05 2023-10-09T11:03:36.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.182870+0000", 2023-10-09T11:03:36.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0223e-05 2023-10-09T11:03:36.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.019382+0000", 2023-10-09T11:03:36.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.83651138199999997 2023-10-09T11:03:36.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.019500+0000", 2023-10-09T11:03:36.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118187 2023-10-09T11:03:36.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:03:36.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.019722+0000", 2023-10-09T11:03:36.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022236099999999999 2023-10-09T11:03:36.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.023434+0000", 2023-10-09T11:03:36.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00371164 2023-10-09T11:03:36.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034764+0000", 2023-10-09T11:03:36.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011329824 2023-10-09T11:03:36.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034792+0000", 2023-10-09T11:03:36.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8123000000000002e-05 2023-10-09T11:03:36.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.754 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034817+0000", 2023-10-09T11:03:36.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5009999999999999e-05 2023-10-09T11:03:36.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.754 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034856+0000", 2023-10-09T11:03:36.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9178999999999999e-05 2023-10-09T11:03:36.754 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.755 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.755 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.755 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:204804 9.13 9:cd6b9308:::benchmark_data_smithi125_363789_object94505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2023-10-09T11:03:36.755 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:42.320768+0000", 2023-10-09T11:03:36.755 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 174.272626363, 2023-10-09T11:03:36.755 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.71443022199999995, 2023-10-09T11:03:36.755 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:03:36.755 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:36.755 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:03:36.755 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:03:36.756 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:36.756 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 204804 2023-10-09T11:03:36.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.756 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:03:36.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:03:36.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320768+0000", 2023-10-09T11:03:36.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:03:36.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320768+0000", 2023-10-09T11:03:36.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:03:36.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.757 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.757 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:03:36.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320767+0000", 2023-10-09T11:03:36.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:36.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.757 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.757 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:03:36.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320770+0000", 2023-10-09T11:03:36.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.768e-06 2023-10-09T11:03:36.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:03:36.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320777+0000", 2023-10-09T11:03:36.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.182e-06 2023-10-09T11:03:36.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:03:36.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320780+0000", 2023-10-09T11:03:36.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8890000000000002e-06 2023-10-09T11:03:36.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320856+0000", 2023-10-09T11:03:36.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5768999999999999e-05 2023-10-09T11:03:36.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:03:36.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:42.320866+0000", 2023-10-09T11:03:36.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9410000000000002e-06 2023-10-09T11:03:36.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.760 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:03:36.760 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.021091+0000", 2023-10-09T11:03:36.760 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.70022501000000004 2023-10-09T11:03:36.760 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.760 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.760 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:03:36.760 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.021196+0000", 2023-10-09T11:03:36.760 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104948 2023-10-09T11:03:36.760 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.760 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.760 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,3", 2023-10-09T11:03:36.760 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.021416+0000", 2023-10-09T11:03:36.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022006799999999999 2023-10-09T11:03:36.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.761 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.761 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:03:36.761 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.024573+0000", 2023-10-09T11:03:36.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0031569430000000002 2023-10-09T11:03:36.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.761 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.761 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.761 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.034901+0000", 2023-10-09T11:03:36.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010327935 2023-10-09T11:03:36.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:36.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.035121+0000", 2023-10-09T11:03:36.762 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00021983500000000001 2023-10-09T11:03:36.762 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:03:36.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.035163+0000", 2023-10-09T11:03:36.762 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2920000000000002e-05 2023-10-09T11:03:36.762 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:03:36.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:03:36.763 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:03:36.763 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:43.035198+0000", 2023-10-09T11:03:36.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4539999999999998e-05 2023-10-09T11:03:36.763 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.763 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.763 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.763 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:03:36.763 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:03:36.763 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:03:36.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:36.765 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:36.765 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:36.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:36.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:36.895 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 2, 1, 0, 7] out_osds: [] dead_osds: [3] live_osds: [0, 2, 4, 5, 6, 7, 1] 2023-10-09T11:03:36.896 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:03:36.896 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [3, 4, 5, 6, 2, 1, 0, 7] 2023-10-09T11:03:36.896 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd out 3 2023-10-09T11:03:36.921 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:36 smithi155 ceph-mon[184854]: pgmap v1129: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 26326/178818 objects degraded (14.722%); 4428/178818 objects misplaced (2.476%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T11:03:36.926 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:36.933 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:36 smithi125 ceph-mon[195507]: pgmap v1129: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 26326/178818 objects degraded (14.722%); 4428/178818 objects misplaced (2.476%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T11:03:36.934 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:36 smithi125 ceph-mon[200197]: pgmap v1129: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 26326/178818 objects degraded (14.722%); 4428/178818 objects misplaced (2.476%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T11:03:36.956 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:36.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:36.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:36.999 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T11:03:37.064 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:37.114 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:37.130 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:03:37.147 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:37.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:37.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:37.250 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:37.334 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:37.344 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:37.383 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:37.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:37.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:37.488 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:37.493 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:37.534 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:37.581 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:37.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:37.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:37.693 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:37.766 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:37.837 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:03:37.876 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:37 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/774727709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:03:37.876 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:37 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:03:37.879 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:37 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/774727709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:03:37.879 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:37 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:03:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:37 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/774727709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:03:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:37 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:03:38.307 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:38.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:38.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:38.420 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:38.478 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:38.511 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:38.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:38.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:38.620 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:38.636 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:38.673 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:38.702 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:38.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:38.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:38.813 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:38.831 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:38 smithi155 ceph-mon[184854]: pgmap v1130: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 24922/169398 objects degraded (14.712%); 4428/169398 objects misplaced (2.614%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T11:03:38.831 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:38 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:38.831 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:38 smithi155 ceph-mon[184854]: failed to encode map e458 with expected crc 2023-10-09T11:03:38.831 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:38 smithi155 ceph-mon[184854]: failed to encode map e458 with expected crc 2023-10-09T11:03:38.831 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:38 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:03:38.831 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:38 smithi155 ceph-mon[184854]: osdmap e458: 8 total, 7 up, 8 in 2023-10-09T11:03:38.831 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:38 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 24922/169398 objects degraded (14.712%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:38.862 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:38.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:38 smithi125 ceph-mon[200197]: pgmap v1130: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 24922/169398 objects degraded (14.712%); 4428/169398 objects misplaced (2.614%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T11:03:38.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:38 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:38.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:38 smithi125 ceph-mon[200197]: failed to encode map e458 with expected crc 2023-10-09T11:03:38.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:38 smithi125 ceph-mon[200197]: failed to encode map e458 with expected crc 2023-10-09T11:03:38.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:38 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:03:38.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:38 smithi125 ceph-mon[200197]: osdmap e458: 8 total, 7 up, 8 in 2023-10-09T11:03:38.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:38 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 24922/169398 objects degraded (14.712%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:38.911 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:38 smithi125 ceph-mon[195507]: pgmap v1130: 129 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 40 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 24922/169398 objects degraded (14.712%); 4428/169398 objects misplaced (2.614%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T11:03:38.911 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:38 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:38.911 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:38 smithi125 ceph-mon[195507]: failed to encode map e458 with expected crc 2023-10-09T11:03:38.911 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:38 smithi125 ceph-mon[195507]: failed to encode map e458 with expected crc 2023-10-09T11:03:38.911 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:38 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:03:38.911 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:38 smithi125 ceph-mon[195507]: osdmap e458: 8 total, 7 up, 8 in 2023-10-09T11:03:38.911 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:38 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 24922/169398 objects degraded (14.712%), 16 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:38.912 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:38.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:38.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:39.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:39.062 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:39.091 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:39.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:39.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:39.196 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:39.261 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:39.497 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:39.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:39.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:39.603 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:39.607 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:03:39.607 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:03:39.607 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.607 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:145460 10.12 10:4e5c71a5:::benchmark_data_smithi125_402869_object46891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e458)", 2023-10-09T11:03:39.607 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:03:39.575435+0000", 2023-10-09T11:03:39.607 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.026424209000000001, 2023-10-09T11:03:39.607 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.026458306000000001, 2023-10-09T11:03:39.607 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:39.607 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:03:39.608 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:39.608 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:03:39.608 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:03:39.608 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 145460 2023-10-09T11:03:39.608 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.608 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:39.608 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.608 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:39.608 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.575435+0000", 2023-10-09T11:03:39.608 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:39.608 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.608 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.609 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:39.609 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.575435+0000", 2023-10-09T11:03:39.609 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:39.609 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.609 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.609 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:39.609 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.575434+0000", 2023-10-09T11:03:39.609 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:03:39.609 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.609 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.609 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:39.609 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.575439+0000", 2023-10-09T11:03:39.610 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2979999999999999e-06 2023-10-09T11:03:39.610 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.610 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.610 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:39.610 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.575453+0000", 2023-10-09T11:03:39.610 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3593e-05 2023-10-09T11:03:39.610 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.610 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.610 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:39.610 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.575461+0000", 2023-10-09T11:03:39.610 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2239999999999995e-06 2023-10-09T11:03:39.610 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.611 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.611 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:39.611 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.584122+0000", 2023-10-09T11:03:39.611 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0086605810000000005 2023-10-09T11:03:39.611 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.611 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.611 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:03:39.611 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.584270+0000", 2023-10-09T11:03:39.611 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014847499999999999 2023-10-09T11:03:39.611 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.611 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.611 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,4", 2023-10-09T11:03:39.612 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.585240+0000", 2023-10-09T11:03:39.612 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00096969799999999996 2023-10-09T11:03:39.612 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.612 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.612 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:39.612 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.589639+0000", 2023-10-09T11:03:39.612 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043989980000000003 2023-10-09T11:03:39.612 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.612 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.612 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:03:39.612 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.591417+0000", 2023-10-09T11:03:39.612 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001778265 2023-10-09T11:03:39.613 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:39.613 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:39.613 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:39.613 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.613 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.613 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:145471 10.2 10:45b54bf9:::benchmark_data_smithi125_402869_object46902:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e458)", 2023-10-09T11:03:39.613 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:03:39.586940+0000", 2023-10-09T11:03:39.613 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.014919813000000001, 2023-10-09T11:03:39.613 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015067785, 2023-10-09T11:03:39.613 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:39.613 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:03:39.613 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:39.614 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:03:39.614 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:03:39.614 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 145471 2023-10-09T11:03:39.614 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.614 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:39.614 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.614 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:39.614 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.586940+0000", 2023-10-09T11:03:39.614 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:39.614 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.614 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.615 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:39.615 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.586940+0000", 2023-10-09T11:03:39.615 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:39.615 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.615 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.615 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:39.615 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.586938+0000", 2023-10-09T11:03:39.615 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:39.615 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.615 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.615 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:39.615 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.586946+0000", 2023-10-09T11:03:39.616 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4170000000000003e-06 2023-10-09T11:03:39.616 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.616 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.616 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:39.616 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.586964+0000", 2023-10-09T11:03:39.616 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.863e-05 2023-10-09T11:03:39.616 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.616 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.616 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:39.616 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.586974+0000", 2023-10-09T11:03:39.616 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0028e-05 2023-10-09T11:03:39.616 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.617 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.617 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:39.617 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.587049+0000", 2023-10-09T11:03:39.617 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4618999999999998e-05 2023-10-09T11:03:39.617 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.617 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.617 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:03:39.617 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.587175+0000", 2023-10-09T11:03:39.617 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000125628 2023-10-09T11:03:39.617 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.617 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.617 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,4", 2023-10-09T11:03:39.618 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.587562+0000", 2023-10-09T11:03:39.618 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00038766199999999999 2023-10-09T11:03:39.618 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.618 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.618 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:39.618 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.591407+0000", 2023-10-09T11:03:39.618 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003844765 2023-10-09T11:03:39.618 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.618 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.618 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:03:39.618 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.592050+0000", 2023-10-09T11:03:39.619 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00064319799999999999 2023-10-09T11:03:39.619 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:39.619 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:39.619 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:39.619 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.619 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.619 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:145481 10.2 10:45949ba1:::benchmark_data_smithi125_402869_object46912:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e458)", 2023-10-09T11:03:39.619 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:03:39.596148+0000", 2023-10-09T11:03:39.619 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0057109689999999998, 2023-10-09T11:03:39.619 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0059618850000000001, 2023-10-09T11:03:39.619 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:39.620 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:03:39.620 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:39.620 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:03:39.620 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:03:39.620 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 145481 2023-10-09T11:03:39.620 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.620 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:39.620 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.620 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:39.620 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.596148+0000", 2023-10-09T11:03:39.620 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:39.620 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.621 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.621 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:39.621 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.596148+0000", 2023-10-09T11:03:39.621 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:39.621 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.621 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.621 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:39.621 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.596147+0000", 2023-10-09T11:03:39.621 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:39.621 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.621 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.621 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:39.622 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.596155+0000", 2023-10-09T11:03:39.622 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5420000000000005e-06 2023-10-09T11:03:39.622 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.622 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.622 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:39.622 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.596174+0000", 2023-10-09T11:03:39.622 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8918000000000001e-05 2023-10-09T11:03:39.622 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.622 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.622 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:39.622 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.596184+0000", 2023-10-09T11:03:39.623 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0059e-05 2023-10-09T11:03:39.623 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.623 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.623 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:39.623 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.596245+0000", 2023-10-09T11:03:39.623 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0748000000000002e-05 2023-10-09T11:03:39.623 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.623 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.623 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:03:39.623 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.596390+0000", 2023-10-09T11:03:39.623 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014457599999999999 2023-10-09T11:03:39.624 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.624 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.624 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,4", 2023-10-09T11:03:39.624 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.596793+0000", 2023-10-09T11:03:39.624 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00040297599999999999 2023-10-09T11:03:39.624 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.624 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.624 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:03:39.624 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.600595+0000", 2023-10-09T11:03:39.624 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038020369999999999 2023-10-09T11:03:39.624 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.625 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.625 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:39.625 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.601240+0000", 2023-10-09T11:03:39.625 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00064543499999999998 2023-10-09T11:03:39.625 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:39.625 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:39.625 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:39.625 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.625 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.625 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:145483 10.11 e458/443 10:88ed8679:::benchmark_data_smithi125_402869_object46914:head v 458'7729, mlcod=458'7727)", 2023-10-09T11:03:39.625 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:03:39.599620+0000", 2023-10-09T11:03:39.625 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0022391059999999998, 2023-10-09T11:03:39.626 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002597458, 2023-10-09T11:03:39.626 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:39.626 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:03:39.626 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:39.626 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.626 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:39.626 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.599620+0000", 2023-10-09T11:03:39.626 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:39.626 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.626 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.626 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:39.626 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.599620+0000", 2023-10-09T11:03:39.627 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:39.627 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.627 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.627 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:39.627 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.599619+0000", 2023-10-09T11:03:39.627 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:03:39.627 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.627 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.627 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:39.627 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.599628+0000", 2023-10-09T11:03:39.627 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.585e-06 2023-10-09T11:03:39.627 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.628 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.628 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:39.628 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.599644+0000", 2023-10-09T11:03:39.628 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6520000000000001e-05 2023-10-09T11:03:39.628 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.628 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.628 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:39.628 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.599654+0000", 2023-10-09T11:03:39.628 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2830000000000007e-06 2023-10-09T11:03:39.628 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.628 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.628 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:39.629 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.600599+0000", 2023-10-09T11:03:39.629 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00094544400000000004 2023-10-09T11:03:39.629 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:39.629 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:39.629 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:03:39.629 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:03:39.600628+0000", 2023-10-09T11:03:39.629 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8793999999999999e-05 2023-10-09T11:03:39.629 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:39.629 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:39.629 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:39.629 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:39.629 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T11:03:39.630 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 4 2023-10-09T11:03:39.630 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:03:39.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:39 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T11:03:39.614+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e458 definitely_dead 0 2023-10-09T11:03:39.654 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:39.881 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.3. 2023-10-09T11:03:39.902 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:39 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1131435374' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T11:03:39.902 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:39 smithi125 ceph-mon[195507]: Client client.admin marked osd.3 out, after it was down for 31 seconds 2023-10-09T11:03:39.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:39 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1131435374' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T11:03:39.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:39 smithi125 ceph-mon[200197]: Client client.admin marked osd.3 out, after it was down for 31 seconds 2023-10-09T11:03:40.081 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T11:03:40.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:40.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:40.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:40.190 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:40.215 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:39 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1131435374' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T11:03:40.215 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:39 smithi155 ceph-mon[184854]: Client client.admin marked osd.3 out, after it was down for 31 seconds 2023-10-09T11:03:40.218 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:40.237 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:40.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:40.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:40.341 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:40.369 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:40.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:40.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:40.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:40.544 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:40.573 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:40.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:40.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:40.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:40.601 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:40.665 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:03:40.669 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:03:40.686 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:03:40.696 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:40.709 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:03:40.724 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:40.729 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:03:40.746 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:40.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:40.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:40.754 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:03:40.774 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:03:40.793 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:03:40.794 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:03:40.794 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:03:40.794 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:03:40.794 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T11:03:40.794 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:03:40.794 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:03:40.794 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:03:40.794 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:03:40.850 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:40.894 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:40.914 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:40.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:40.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:41.018 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:41.045 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:41.065 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:41.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:41.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:41.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[195507]: pgmap v1132: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 24073/162372 objects degraded (14.826%); 3676/162372 objects misplaced (2.264%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T11:03:41.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[195507]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T11:03:41.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1131435374' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T11:03:41.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[195507]: osdmap e459: 8 total, 7 up, 7 in 2023-10-09T11:03:41.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[195507]: failed to encode map e459 with expected crc 2023-10-09T11:03:41.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[195507]: failed to encode map e459 with expected crc 2023-10-09T11:03:41.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[195507]: failed to encode map e459 with expected crc 2023-10-09T11:03:41.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2571450280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[200197]: pgmap v1132: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 24073/162372 objects degraded (14.826%); 3676/162372 objects misplaced (2.264%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T11:03:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[200197]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T11:03:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1131435374' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T11:03:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[200197]: osdmap e459: 8 total, 7 up, 7 in 2023-10-09T11:03:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[200197]: failed to encode map e459 with expected crc 2023-10-09T11:03:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[200197]: failed to encode map e459 with expected crc 2023-10-09T11:03:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[200197]: failed to encode map e459 with expected crc 2023-10-09T11:03:41.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:40 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2571450280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:41.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:41.195 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:40 smithi155 ceph-mon[184854]: pgmap v1132: 129 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 40 active+undersized, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+remapped+backfill_wait, 70 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 24073/162372 objects degraded (14.826%); 3676/162372 objects misplaced (2.264%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T11:03:41.195 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:40 smithi155 ceph-mon[184854]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T11:03:41.195 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:40 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1131435374' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T11:03:41.195 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:40 smithi155 ceph-mon[184854]: osdmap e459: 8 total, 7 up, 7 in 2023-10-09T11:03:41.195 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:40 smithi155 ceph-mon[184854]: failed to encode map e459 with expected crc 2023-10-09T11:03:41.195 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:40 smithi155 ceph-mon[184854]: failed to encode map e459 with expected crc 2023-10-09T11:03:41.195 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:40 smithi155 ceph-mon[184854]: failed to encode map e459 with expected crc 2023-10-09T11:03:41.195 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:40 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2571450280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:41.197 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:41.218 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:41.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:41.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:41.244 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T11:03:41.245 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T11:03:41.322 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:41.350 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:41.370 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:41.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:41.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:41.403 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:03:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:03:41] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.43.0" 2023-10-09T11:03:41.437 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:41.475 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:41.502 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:41.571 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:41.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:41.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:41.675 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:41.702 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:41.722 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:41.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:41.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:41.825 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:41.870 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:41.911 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:41.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:41.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:41 smithi125 ceph-mon[200197]: failed to encode map e459 with expected crc 2023-10-09T11:03:41.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:41 smithi125 ceph-mon[200197]: failed to encode map e460 with expected crc 2023-10-09T11:03:41.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:41 smithi125 ceph-mon[200197]: osdmap e460: 8 total, 7 up, 7 in 2023-10-09T11:03:41.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:41 smithi125 ceph-mon[200197]: failed to encode map e460 with expected crc 2023-10-09T11:03:41.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:41 smithi125 ceph-mon[200197]: failed to encode map e460 with expected crc 2023-10-09T11:03:41.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:41 smithi125 ceph-mon[200197]: failed to encode map e460 with expected crc 2023-10-09T11:03:41.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:41.919 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:41 smithi125 ceph-mon[195507]: failed to encode map e459 with expected crc 2023-10-09T11:03:41.919 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:41 smithi125 ceph-mon[195507]: failed to encode map e460 with expected crc 2023-10-09T11:03:41.919 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:41 smithi125 ceph-mon[195507]: osdmap e460: 8 total, 7 up, 7 in 2023-10-09T11:03:41.919 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:41 smithi125 ceph-mon[195507]: failed to encode map e460 with expected crc 2023-10-09T11:03:41.919 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:41 smithi125 ceph-mon[195507]: failed to encode map e460 with expected crc 2023-10-09T11:03:41.919 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:41 smithi125 ceph-mon[195507]: failed to encode map e460 with expected crc 2023-10-09T11:03:41.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:41 smithi155 ceph-mon[184854]: failed to encode map e459 with expected crc 2023-10-09T11:03:41.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:41 smithi155 ceph-mon[184854]: failed to encode map e460 with expected crc 2023-10-09T11:03:41.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:41 smithi155 ceph-mon[184854]: osdmap e460: 8 total, 7 up, 7 in 2023-10-09T11:03:41.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:41 smithi155 ceph-mon[184854]: failed to encode map e460 with expected crc 2023-10-09T11:03:41.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:41 smithi155 ceph-mon[184854]: failed to encode map e460 with expected crc 2023-10-09T11:03:41.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:41 smithi155 ceph-mon[184854]: failed to encode map e460 with expected crc 2023-10-09T11:03:42.019 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:42.025 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:03:42.025 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:03:42.105 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:42.153 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:42.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:42.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:42.263 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:42.291 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:42.334 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:42.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:42.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:42.442 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:42.493 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:42 smithi155 ceph-mon[184854]: failed to encode map e460 with expected crc 2023-10-09T11:03:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:42 smithi155 ceph-mon[184854]: pgmap v1135: 129 pgs: 13 remapped+peering, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+undersized+degraded, 2 active+remapped+backfill_wait, 83 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 9079/156582 objects degraded (5.798%); 4757/156582 objects misplaced (3.038%); 8.2 MiB/s, 132 objects/s recovering 2023-10-09T11:03:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:42 smithi155 ceph-mon[184854]: failed to encode map e461 with expected crc 2023-10-09T11:03:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:42 smithi155 ceph-mon[184854]: osdmap e461: 8 total, 7 up, 7 in 2023-10-09T11:03:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:42 smithi155 ceph-mon[184854]: failed to encode map e461 with expected crc 2023-10-09T11:03:43.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:42 smithi155 ceph-mon[184854]: failed to encode map e461 with expected crc 2023-10-09T11:03:43.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:42 smithi155 ceph-mon[184854]: failed to encode map e461 with expected crc 2023-10-09T11:03:43.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:42 smithi155 ceph-mon[184854]: 10.17 starting backfill to osd.2 from (0'0,0'0] MAX to 458'7615 2023-10-09T11:03:43.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:42 smithi155 ceph-mon[184854]: 10.7 starting backfill to osd.2 from (0'0,0'0] MAX to 458'7527 2023-10-09T11:03:43.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:42 smithi155 ceph-mon[184854]: 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 458'9193 2023-10-09T11:03:43.246 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:03:43.284 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[195507]: failed to encode map e460 with expected crc 2023-10-09T11:03:43.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[195507]: pgmap v1135: 129 pgs: 13 remapped+peering, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+undersized+degraded, 2 active+remapped+backfill_wait, 83 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 9079/156582 objects degraded (5.798%); 4757/156582 objects misplaced (3.038%); 8.2 MiB/s, 132 objects/s recovering 2023-10-09T11:03:43.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[195507]: failed to encode map e461 with expected crc 2023-10-09T11:03:43.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[195507]: osdmap e461: 8 total, 7 up, 7 in 2023-10-09T11:03:43.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[195507]: failed to encode map e461 with expected crc 2023-10-09T11:03:43.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[195507]: failed to encode map e461 with expected crc 2023-10-09T11:03:43.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[195507]: failed to encode map e461 with expected crc 2023-10-09T11:03:43.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[195507]: 10.17 starting backfill to osd.2 from (0'0,0'0] MAX to 458'7615 2023-10-09T11:03:43.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[195507]: 10.7 starting backfill to osd.2 from (0'0,0'0] MAX to 458'7527 2023-10-09T11:03:43.285 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[195507]: 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 458'9193 2023-10-09T11:03:43.286 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[200197]: failed to encode map e460 with expected crc 2023-10-09T11:03:43.286 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[200197]: pgmap v1135: 129 pgs: 13 remapped+peering, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+undersized+degraded, 2 active+remapped+backfill_wait, 83 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 9079/156582 objects degraded (5.798%); 4757/156582 objects misplaced (3.038%); 8.2 MiB/s, 132 objects/s recovering 2023-10-09T11:03:43.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[200197]: failed to encode map e461 with expected crc 2023-10-09T11:03:43.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[200197]: osdmap e461: 8 total, 7 up, 7 in 2023-10-09T11:03:43.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[200197]: failed to encode map e461 with expected crc 2023-10-09T11:03:43.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[200197]: failed to encode map e461 with expected crc 2023-10-09T11:03:43.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[200197]: failed to encode map e461 with expected crc 2023-10-09T11:03:43.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[200197]: 10.17 starting backfill to osd.2 from (0'0,0'0] MAX to 458'7615 2023-10-09T11:03:43.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[200197]: 10.7 starting backfill to osd.2 from (0'0,0'0] MAX to 458'7527 2023-10-09T11:03:43.288 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:42 smithi125 ceph-mon[200197]: 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 458'9193 2023-10-09T11:03:43.881 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:43.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:43.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:43.990 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:44.038 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:44.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:43 smithi155 ceph-mon[184854]: failed to encode map e461 with expected crc 2023-10-09T11:03:44.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:43 smithi155 ceph-mon[184854]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:43 smithi155 ceph-mon[184854]: osdmap e462: 8 total, 7 up, 7 in 2023-10-09T11:03:44.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:43 smithi155 ceph-mon[184854]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:43 smithi155 ceph-mon[184854]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:43 smithi155 ceph-mon[184854]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:43 smithi155 ceph-mon[184854]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:43 smithi155 ceph-mon[184854]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.226 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:43 smithi155 ceph-mon[184854]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.226 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:43 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 9079/156582 objects degraded (5.798%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:44.226 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[200197]: failed to encode map e461 with expected crc 2023-10-09T11:03:44.226 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[200197]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.226 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[200197]: osdmap e462: 8 total, 7 up, 7 in 2023-10-09T11:03:44.226 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[200197]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[200197]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[200197]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[200197]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[200197]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[200197]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 9079/156582 objects degraded (5.798%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:44.227 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[195507]: failed to encode map e461 with expected crc 2023-10-09T11:03:44.227 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[195507]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[195507]: osdmap e462: 8 total, 7 up, 7 in 2023-10-09T11:03:44.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[195507]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[195507]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[195507]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[195507]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[195507]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[195507]: failed to encode map e462 with expected crc 2023-10-09T11:03:44.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:43 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 9079/156582 objects degraded (5.798%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:45.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:44 smithi155 ceph-mon[184854]: pgmap v1138: 129 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 7 peering, 13 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 102 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 794 op/s; 10107/152829 objects degraded (6.613%); 4272/152829 objects misplaced (2.795%); 11 MiB/s, 173 objects/s recovering 2023-10-09T11:03:45.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:44 smithi155 ceph-mon[184854]: failed to encode map e463 with expected crc 2023-10-09T11:03:45.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:44 smithi155 ceph-mon[184854]: osdmap e463: 8 total, 7 up, 7 in 2023-10-09T11:03:45.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:44 smithi155 ceph-mon[184854]: failed to encode map e463 with expected crc 2023-10-09T11:03:45.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:44 smithi155 ceph-mon[184854]: failed to encode map e463 with expected crc 2023-10-09T11:03:45.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:44 smithi155 ceph-mon[184854]: failed to encode map e463 with expected crc 2023-10-09T11:03:45.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:44 smithi125 ceph-mon[195507]: pgmap v1138: 129 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 7 peering, 13 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 102 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 794 op/s; 10107/152829 objects degraded (6.613%); 4272/152829 objects misplaced (2.795%); 11 MiB/s, 173 objects/s recovering 2023-10-09T11:03:45.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:44 smithi125 ceph-mon[195507]: failed to encode map e463 with expected crc 2023-10-09T11:03:45.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:44 smithi125 ceph-mon[195507]: osdmap e463: 8 total, 7 up, 7 in 2023-10-09T11:03:45.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:44 smithi125 ceph-mon[195507]: failed to encode map e463 with expected crc 2023-10-09T11:03:45.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:44 smithi125 ceph-mon[195507]: failed to encode map e463 with expected crc 2023-10-09T11:03:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:44 smithi125 ceph-mon[195507]: failed to encode map e463 with expected crc 2023-10-09T11:03:45.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:44 smithi125 ceph-mon[200197]: pgmap v1138: 129 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 7 peering, 13 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 102 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 794 op/s; 10107/152829 objects degraded (6.613%); 4272/152829 objects misplaced (2.795%); 11 MiB/s, 173 objects/s recovering 2023-10-09T11:03:45.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:44 smithi125 ceph-mon[200197]: failed to encode map e463 with expected crc 2023-10-09T11:03:45.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:44 smithi125 ceph-mon[200197]: osdmap e463: 8 total, 7 up, 7 in 2023-10-09T11:03:45.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:44 smithi125 ceph-mon[200197]: failed to encode map e463 with expected crc 2023-10-09T11:03:45.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:44 smithi125 ceph-mon[200197]: failed to encode map e463 with expected crc 2023-10-09T11:03:45.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:44 smithi125 ceph-mon[200197]: failed to encode map e463 with expected crc 2023-10-09T11:03:45.481 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 6, 2, 1, 0, 7] out_osds: [3] dead_osds: [3] live_osds: [0, 2, 4, 5, 6, 7, 1] 2023-10-09T11:03:45.482 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:03:45.482 INFO:tasks.thrashosds.thrasher:Reviving osd 3 2023-10-09T11:03:45.482 INFO:tasks.cephadm.osd.3:Restarting osd.3 (starting--it wasn't running)... 2023-10-09T11:03:45.482 DEBUG:teuthology.orchestra.run.smithi125:osd.3> sudo journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-09T11:03:45.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl start ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T11:03:45.540 INFO:journalctl@ceph.osd.3.smithi125.stdout:-- Logs begin at Mon 2023-10-09 10:18:49 UTC. -- 2023-10-09T11:03:45.727 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T11:03:45.822 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:45 smithi125 systemd[1]: Starting Ceph osd.3 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:03:46.065 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:46.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:46.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:46.172 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:46.244 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:46.276 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:46.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:46.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:46.294 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:46.357 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:03:46.380 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:46.426 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:46.426 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:03:46.429 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:46.476 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:46.476 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:03:46.476 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:46.476 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:46.476 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:03:46.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:46.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:46.582 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:46.630 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:46.657 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:46.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:46.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:46.769 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:46.827 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:46.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:46.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:46.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:46.967 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:47.023 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:47.125 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:03:47.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:47 smithi125 ceph-mon[195507]: pgmap v1140: 129 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 7 peering, 13 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 102 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 10107/152829 objects degraded (6.613%); 4272/152829 objects misplaced (2.795%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T11:03:47.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/608210702' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:03:47.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:47 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:03:47.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:47 smithi125 ceph-mon[200197]: pgmap v1140: 129 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 7 peering, 13 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 102 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 10107/152829 objects degraded (6.613%); 4272/152829 objects misplaced (2.795%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T11:03:47.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/608210702' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:03:47.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:47 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:03:47.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:47 smithi155 ceph-mon[184854]: pgmap v1140: 129 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 7 peering, 13 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 102 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 10107/152829 objects degraded (6.613%); 4272/152829 objects misplaced (2.795%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T11:03:47.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:47 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/608210702' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:03:47.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:47 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:03:47.965 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:47.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:47.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:48.071 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:03:48.133 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:48.167 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:03:48.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:48.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:03:48.276 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:03:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:48 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:48 smithi125 ceph-mon[195507]: failed to encode map e464 with expected crc 2023-10-09T11:03:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:48 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:03:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:48 smithi125 ceph-mon[195507]: osdmap e464: 8 total, 7 up, 7 in 2023-10-09T11:03:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:48 smithi125 ceph-mon[195507]: failed to encode map e464 with expected crc 2023-10-09T11:03:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:48 smithi125 ceph-mon[195507]: failed to encode map e464 with expected crc 2023-10-09T11:03:48.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:48 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:48.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:48 smithi125 ceph-mon[200197]: failed to encode map e464 with expected crc 2023-10-09T11:03:48.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:48 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:03:48.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:48 smithi125 ceph-mon[200197]: osdmap e464: 8 total, 7 up, 7 in 2023-10-09T11:03:48.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:48 smithi125 ceph-mon[200197]: failed to encode map e464 with expected crc 2023-10-09T11:03:48.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:48 smithi125 ceph-mon[200197]: failed to encode map e464 with expected crc 2023-10-09T11:03:48.331 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:48.362 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:03:48.367 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:48 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:03:48.367 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:48 smithi155 ceph-mon[184854]: failed to encode map e464 with expected crc 2023-10-09T11:03:48.367 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:48 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:03:48.367 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:48 smithi155 ceph-mon[184854]: osdmap e464: 8 total, 7 up, 7 in 2023-10-09T11:03:48.367 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:48 smithi155 ceph-mon[184854]: failed to encode map e464 with expected crc 2023-10-09T11:03:48.367 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:48 smithi155 ceph-mon[184854]: failed to encode map e464 with expected crc 2023-10-09T11:03:48.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:48.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:03:48.469 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:03:48.525 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:48.559 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:03:48.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:48.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:03:48.665 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:48.734 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:48.775 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:48.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:48.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:48.879 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:48.945 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:49.131 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:49 smithi155 ceph-mon[184854]: pgmap v1142: 129 pgs: 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 7 peering, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 108 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 28335/145839 objects degraded (19.429%); 3905/145839 objects misplaced (2.678%); 9.0 MiB/s, 143 objects/s recovering 2023-10-09T11:03:49.131 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:49 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 28335/145839 objects degraded (19.429%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:49.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:49 smithi125 ceph-mon[195507]: pgmap v1142: 129 pgs: 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 7 peering, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 108 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 28335/145839 objects degraded (19.429%); 3905/145839 objects misplaced (2.678%); 9.0 MiB/s, 143 objects/s recovering 2023-10-09T11:03:49.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:49 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 28335/145839 objects degraded (19.429%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:49.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:49 smithi125 ceph-mon[200197]: pgmap v1142: 129 pgs: 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 7 peering, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 108 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 28335/145839 objects degraded (19.429%); 3905/145839 objects misplaced (2.678%); 9.0 MiB/s, 143 objects/s recovering 2023-10-09T11:03:49.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:49 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 28335/145839 objects degraded (19.429%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:03:49.940 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:49.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:49.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:50.048 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:50.108 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:51.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:51 smithi155 ceph-mon[184854]: pgmap v1143: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 806 op/s; 27596/139974 objects degraded (19.715%); 3617/139974 objects misplaced (2.584%); 12 MiB/s, 187 objects/s recovering 2023-10-09T11:03:51.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:51 smithi125 ceph-mon[195507]: pgmap v1143: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 806 op/s; 27596/139974 objects degraded (19.715%); 3617/139974 objects misplaced (2.584%); 12 MiB/s, 187 objects/s recovering 2023-10-09T11:03:51.570 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:51 smithi125 bash[427540]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T11:03:51.570 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:51 smithi125 bash[427540]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T11:03:51.570 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:51 smithi125 bash[427540]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-09T11:03:51.570 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:51 smithi125 bash[427540]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-09T11:03:51.570 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:51 smithi125 bash[427540]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-09T11:03:51.571 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:51 smithi125 bash[427540]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T11:03:51.571 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:51 smithi125 bash[427540]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T11:03:51.571 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:03:51 smithi125 bash[427540]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-09T11:03:51.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:03:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:03:51] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.43.0" 2023-10-09T11:03:51.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:51 smithi125 ceph-mon[200197]: pgmap v1143: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 806 op/s; 27596/139974 objects degraded (19.715%); 3617/139974 objects misplaced (2.584%); 12 MiB/s, 187 objects/s recovering 2023-10-09T11:03:51.994 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:52.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:52.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:52.101 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:52.153 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:53.446 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:53 smithi125 ceph-mon[195507]: pgmap v1144: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 26960/131988 objects degraded (20.426%); 3260/131988 objects misplaced (2.470%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T11:03:53.450 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:53 smithi125 ceph-mon[200197]: pgmap v1144: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 26960/131988 objects degraded (20.426%); 3260/131988 objects misplaced (2.470%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T11:03:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:53 smithi155 ceph-mon[184854]: pgmap v1144: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 26960/131988 objects degraded (20.426%); 3260/131988 objects misplaced (2.470%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T11:03:54.006 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:54.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:03:54.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:54.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:54.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:54 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 26960/131988 objects degraded (20.426%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:03:54.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:54 smithi125 ceph-mon[195507]: pgmap v1145: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 930 op/s; 26128/125295 objects degraded (20.853%); 2959/125295 objects misplaced (2.362%); 11 MiB/s, 173 objects/s recovering 2023-10-09T11:03:54.161 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:03:54.213 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:54 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 26960/131988 objects degraded (20.426%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:03:54.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:54 smithi155 ceph-mon[184854]: pgmap v1145: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 930 op/s; 26128/125295 objects degraded (20.853%); 2959/125295 objects misplaced (2.362%); 11 MiB/s, 173 objects/s recovering 2023-10-09T11:03:54.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:54 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 26960/131988 objects degraded (20.426%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:03:54.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:54 smithi125 ceph-mon[200197]: pgmap v1145: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 930 op/s; 26128/125295 objects degraded (20.853%); 2959/125295 objects misplaced (2.362%); 11 MiB/s, 173 objects/s recovering 2023-10-09T11:03:55.472 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:03:55.503 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:03:55.555 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:03:55.613 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:03:55.674 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:03:55.722 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:03:55.724 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:55 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:03:55.751 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:03:55.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:55 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:03:55.754 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:55 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:03:55.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:55.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:03:55.771 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:03:55.772 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:03:55.772 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:03:55.772 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:03:55.772 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T11:03:55.772 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:03:55.772 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:03:55.772 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:03:55.772 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:03:55.856 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:55.884 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:55.905 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:55.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:55.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:56.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:03:56.047 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:03:56.047 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T11:03:56.047 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T11:03:56.047 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:03:56.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.048 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:183389 9.4 9.de1a0e64 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T11:03:56.048 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:00:13.855442+0000", 2023-10-09T11:03:56.048 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 222.17098552100001, 2023-10-09T11:03:56.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0015768009999999, 2023-10-09T11:03:56.048 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.048 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:03:56.048 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.048 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:03:56.048 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:56.048 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183389 2023-10-09T11:03:56.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.049 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855442+0000", 2023-10-09T11:03:56.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855442+0000", 2023-10-09T11:03:56.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855441+0000", 2023-10-09T11:03:56.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:56.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855447+0000", 2023-10-09T11:03:56.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3509999999999998e-06 2023-10-09T11:03:56.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855462+0000", 2023-10-09T11:03:56.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4882000000000001e-05 2023-10-09T11:03:56.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855471+0000", 2023-10-09T11:03:56.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0880000000000001e-06 2023-10-09T11:03:56.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.861521+0000", 2023-10-09T11:03:56.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0060497880000000004 2023-10-09T11:03:56.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:03:56.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.861532+0000", 2023-10-09T11:03:56.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0728e-05 2023-10-09T11:03:56.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.857009+0000", 2023-10-09T11:03:56.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99547772800000001 2023-10-09T11:03:56.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.857019+0000", 2023-10-09T11:03:56.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8749999999999995e-06 2023-10-09T11:03:56.053 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.053 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.053 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.053 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.053 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:183391 9.f 9.7097019f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T11:03:56.053 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:00:13.855480+0000", 2023-10-09T11:03:56.053 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 222.17094757999999, 2023-10-09T11:03:56.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0089207659999999, 2023-10-09T11:03:56.054 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.054 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:03:56.054 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.054 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:03:56.054 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:56.054 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183391 2023-10-09T11:03:56.054 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.054 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.054 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.054 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.054 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855480+0000", 2023-10-09T11:03:56.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855480+0000", 2023-10-09T11:03:56.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855480+0000", 2023-10-09T11:03:56.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:56.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855483+0000", 2023-10-09T11:03:56.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.236e-06 2023-10-09T11:03:56.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.056 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855489+0000", 2023-10-09T11:03:56.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0959999999999997e-06 2023-10-09T11:03:56.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.056 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855495+0000", 2023-10-09T11:03:56.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8459999999999996e-06 2023-10-09T11:03:56.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855545+0000", 2023-10-09T11:03:56.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0142000000000002e-05 2023-10-09T11:03:56.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:03:56.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855553+0000", 2023-10-09T11:03:56.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2179999999999999e-06 2023-10-09T11:03:56.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.864389+0000", 2023-10-09T11:03:56.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.008835787 2023-10-09T11:03:56.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.864401+0000", 2023-10-09T11:03:56.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1950000000000001e-05 2023-10-09T11:03:56.058 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.058 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.058 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.059 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.059 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:183394 9.14 9.70b172b4 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T11:03:56.059 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:00:13.855507+0000", 2023-10-09T11:03:56.059 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 222.17092107799999, 2023-10-09T11:03:56.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0014076110000001, 2023-10-09T11:03:56.059 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.059 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:03:56.059 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.059 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:03:56.059 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:56.059 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183394 2023-10-09T11:03:56.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.060 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.060 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855507+0000", 2023-10-09T11:03:56.060 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.060 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855507+0000", 2023-10-09T11:03:56.060 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855506+0000", 2023-10-09T11:03:56.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:56.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855517+0000", 2023-10-09T11:03:56.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0178e-05 2023-10-09T11:03:56.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855523+0000", 2023-10-09T11:03:56.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3559999999999998e-06 2023-10-09T11:03:56.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855528+0000", 2023-10-09T11:03:56.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4190000000000001e-06 2023-10-09T11:03:56.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.861474+0000", 2023-10-09T11:03:56.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0059453459999999998 2023-10-09T11:03:56.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:03:56.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.861480+0000", 2023-10-09T11:03:56.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5590000000000002e-06 2023-10-09T11:03:56.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.856892+0000", 2023-10-09T11:03:56.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99541219000000003 2023-10-09T11:03:56.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.856914+0000", 2023-10-09T11:03:56.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2005999999999999e-05 2023-10-09T11:03:56.064 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.064 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.064 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.064 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:183397 9.10 9.8acb0fd0 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T11:03:56.064 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:00:13.855533+0000", 2023-10-09T11:03:56.064 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 222.17089443500001, 2023-10-09T11:03:56.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99257001600000005, 2023-10-09T11:03:56.064 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.065 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:03:56.065 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.065 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:03:56.065 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:56.065 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183397 2023-10-09T11:03:56.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.065 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.065 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855533+0000", 2023-10-09T11:03:56.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855533+0000", 2023-10-09T11:03:56.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855533+0000", 2023-10-09T11:03:56.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:56.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855535+0000", 2023-10-09T11:03:56.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1909999999999999e-06 2023-10-09T11:03:56.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.067 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855540+0000", 2023-10-09T11:03:56.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0080000000000002e-06 2023-10-09T11:03:56.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.067 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855546+0000", 2023-10-09T11:03:56.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6729999999999997e-06 2023-10-09T11:03:56.068 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.068 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855951+0000", 2023-10-09T11:03:56.068 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00040547899999999998 2023-10-09T11:03:56.068 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:03:56.068 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855961+0000", 2023-10-09T11:03:56.068 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5210000000000008e-06 2023-10-09T11:03:56.068 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.848094+0000", 2023-10-09T11:03:56.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99213351699999996 2023-10-09T11:03:56.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.069 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.848104+0000", 2023-10-09T11:03:56.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0240000000000003e-06 2023-10-09T11:03:56.069 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.069 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.069 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.070 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:183400 9.f 9.50a0c2ef (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T11:03:56.070 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:00:13.855550+0000", 2023-10-09T11:03:56.070 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 222.17087788000001, 2023-10-09T11:03:56.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0088860150000001, 2023-10-09T11:03:56.070 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.070 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:03:56.070 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.070 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:03:56.070 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:56.070 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183400 2023-10-09T11:03:56.070 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.071 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.071 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855550+0000", 2023-10-09T11:03:56.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.071 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855550+0000", 2023-10-09T11:03:56.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855550+0000", 2023-10-09T11:03:56.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:56.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855552+0000", 2023-10-09T11:03:56.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0490000000000002e-06 2023-10-09T11:03:56.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855557+0000", 2023-10-09T11:03:56.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4879999999999998e-06 2023-10-09T11:03:56.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855562+0000", 2023-10-09T11:03:56.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4819999999999996e-06 2023-10-09T11:03:56.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855593+0000", 2023-10-09T11:03:56.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1699000000000003e-05 2023-10-09T11:03:56.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:03:56.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855602+0000", 2023-10-09T11:03:56.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4330000000000004e-06 2023-10-09T11:03:56.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.864426+0000", 2023-10-09T11:03:56.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.00882435 2023-10-09T11:03:56.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.864436+0000", 2023-10-09T11:03:56.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8349999999999999e-06 2023-10-09T11:03:56.075 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.075 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.075 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.075 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:183403 9.0 9.d7454e60 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T11:03:56.075 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:00:13.855568+0000", 2023-10-09T11:03:56.075 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 222.170860033, 2023-10-09T11:03:56.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99250176000000001, 2023-10-09T11:03:56.075 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.075 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:03:56.076 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.076 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:03:56.076 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:56.076 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183403 2023-10-09T11:03:56.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.076 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.076 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.076 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855568+0000", 2023-10-09T11:03:56.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855568+0000", 2023-10-09T11:03:56.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855568+0000", 2023-10-09T11:03:56.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:56.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855570+0000", 2023-10-09T11:03:56.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3460000000000001e-06 2023-10-09T11:03:56.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.078 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.078 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855575+0000", 2023-10-09T11:03:56.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2889999999999997e-06 2023-10-09T11:03:56.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.078 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.078 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.855580+0000", 2023-10-09T11:03:56.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8799999999999999e-06 2023-10-09T11:03:56.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.859308+0000", 2023-10-09T11:03:56.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037279370000000002 2023-10-09T11:03:56.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:03:56.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.859321+0000", 2023-10-09T11:03:56.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2916e-05 2023-10-09T11:03:56.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.848057+0000", 2023-10-09T11:03:56.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98873621300000003 2023-10-09T11:03:56.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.080 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.080 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.848070+0000", 2023-10-09T11:03:56.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2534000000000001e-05 2023-10-09T11:03:56.080 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.080 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.080 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.081 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:183404 9.14 9.73af7294 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T11:03:56.081 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:00:13.856067+0000", 2023-10-09T11:03:56.081 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 222.17036080099999, 2023-10-09T11:03:56.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0009173440000001, 2023-10-09T11:03:56.081 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.081 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:03:56.081 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.081 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:03:56.081 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:56.081 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183404 2023-10-09T11:03:56.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.081 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.856067+0000", 2023-10-09T11:03:56.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.856067+0000", 2023-10-09T11:03:56.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.856066+0000", 2023-10-09T11:03:56.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:03:56.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.083 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.856071+0000", 2023-10-09T11:03:56.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6319999999999999e-06 2023-10-09T11:03:56.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.083 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.858276+0000", 2023-10-09T11:03:56.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022046190000000001 2023-10-09T11:03:56.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.858287+0000", 2023-10-09T11:03:56.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.182e-05 2023-10-09T11:03:56.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.861493+0000", 2023-10-09T11:03:56.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003205196 2023-10-09T11:03:56.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:03:56.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.861498+0000", 2023-10-09T11:03:56.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3260000000000003e-06 2023-10-09T11:03:56.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.856974+0000", 2023-10-09T11:03:56.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99547601299999999 2023-10-09T11:03:56.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.856984+0000", 2023-10-09T11:03:56.086 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.057e-05 2023-10-09T11:03:56.086 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.086 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.086 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.086 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.086 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.086 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:183408 9.14 9.c3f5acf4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T11:03:56.086 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:00:13.858301+0000", 2023-10-09T11:03:56.086 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 222.16812673699999, 2023-10-09T11:03:56.086 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0018803789999999, 2023-10-09T11:03:56.086 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.087 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:03:56.087 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.087 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:03:56.087 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:56.087 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183408 2023-10-09T11:03:56.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.087 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.858301+0000", 2023-10-09T11:03:56.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.088 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.858301+0000", 2023-10-09T11:03:56.088 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.088 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.858300+0000", 2023-10-09T11:03:56.088 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:03:56.089 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.089 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.089 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.089 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.858305+0000", 2023-10-09T11:03:56.089 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2580000000000003e-06 2023-10-09T11:03:56.089 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.089 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.089 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.089 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.858314+0000", 2023-10-09T11:03:56.089 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4449999999999996e-06 2023-10-09T11:03:56.089 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.089 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.089 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.090 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.858328+0000", 2023-10-09T11:03:56.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4545999999999999e-05 2023-10-09T11:03:56.090 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.090 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.090 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.090 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.861530+0000", 2023-10-09T11:03:56.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003201289 2023-10-09T11:03:56.090 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.090 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.090 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:03:56.091 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.861538+0000", 2023-10-09T11:03:56.091 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1149999999999994e-06 2023-10-09T11:03:56.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.091 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.091 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.859841+0000", 2023-10-09T11:03:56.091 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99830354799999999 2023-10-09T11:03:56.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.091 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.091 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.860182+0000", 2023-10-09T11:03:56.091 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00034021600000000001 2023-10-09T11:03:56.091 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.092 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.092 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.092 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:183415 9.a 9.b40f9dca (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T11:03:56.092 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:00:13.869801+0000", 2023-10-09T11:03:56.092 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 222.15662688800001, 2023-10-09T11:03:56.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98737902, 2023-10-09T11:03:56.092 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.092 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:03:56.092 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.092 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:03:56.093 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:56.093 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183415 2023-10-09T11:03:56.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.093 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.869801+0000", 2023-10-09T11:03:56.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.869801+0000", 2023-10-09T11:03:56.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.869800+0000", 2023-10-09T11:03:56.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:56.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.869808+0000", 2023-10-09T11:03:56.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1200000000000002e-06 2023-10-09T11:03:56.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.869825+0000", 2023-10-09T11:03:56.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7314999999999999e-05 2023-10-09T11:03:56.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.869833+0000", 2023-10-09T11:03:56.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7849999999999992e-06 2023-10-09T11:03:56.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.869889+0000", 2023-10-09T11:03:56.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6137000000000001e-05 2023-10-09T11:03:56.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:03:56.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.869902+0000", 2023-10-09T11:03:56.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2542e-05 2023-10-09T11:03:56.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.857170+0000", 2023-10-09T11:03:56.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.98726844800000002 2023-10-09T11:03:56.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.097 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.097 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.857180+0000", 2023-10-09T11:03:56.097 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0045e-05 2023-10-09T11:03:56.097 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.097 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.097 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.097 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.097 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.097 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:183417 9.f 9.94f1dc0f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T11:03:56.097 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:00:13.872665+0000", 2023-10-09T11:03:56.097 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 222.15376337800001, 2023-10-09T11:03:56.097 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99180991900000004, 2023-10-09T11:03:56.098 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.098 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:03:56.098 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.098 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:03:56.098 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:03:56.098 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183417 2023-10-09T11:03:56.098 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.098 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.098 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.098 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.872665+0000", 2023-10-09T11:03:56.098 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.099 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.099 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.099 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.099 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.872665+0000", 2023-10-09T11:03:56.099 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.099 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.099 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.099 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.099 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.872664+0000", 2023-10-09T11:03:56.100 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:03:56.100 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.100 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.100 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.872671+0000", 2023-10-09T11:03:56.100 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7530000000000004e-06 2023-10-09T11:03:56.100 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.872680+0000", 2023-10-09T11:03:56.101 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5659999999999996e-06 2023-10-09T11:03:56.101 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.101 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.872685+0000", 2023-10-09T11:03:56.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2809999999999999e-06 2023-10-09T11:03:56.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.102 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.102 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.102 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.872732+0000", 2023-10-09T11:03:56.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6808999999999999e-05 2023-10-09T11:03:56.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.102 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:03:56.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:13.872739+0000", 2023-10-09T11:03:56.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2679999999999997e-06 2023-10-09T11:03:56.103 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.864465+0000", 2023-10-09T11:03:56.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99172533600000001 2023-10-09T11:03:56.104 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.104 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.104 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.104 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:00:14.864474+0000", 2023-10-09T11:03:56.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.645e-06 2023-10-09T11:03:56.104 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.104 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.105 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.105 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.105 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.105 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:15608 10.2 10:45d3f29f:::benchmark_data_smithi125_402869_object15607:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T11:03:56.105 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:24.487757+0000", 2023-10-09T11:03:56.105 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 151.538670798, 2023-10-09T11:03:56.105 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.10149135200000001, 2023-10-09T11:03:56.105 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.106 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:56.106 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.106 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:03:56.106 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:03:56.106 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15608 2023-10-09T11:03:56.106 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.106 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.107 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.487757+0000", 2023-10-09T11:03:56.107 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.107 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.487757+0000", 2023-10-09T11:03:56.108 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.108 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.108 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.108 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.108 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.487756+0000", 2023-10-09T11:03:56.108 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:56.108 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.108 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.109 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.487780+0000", 2023-10-09T11:03:56.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4012999999999999e-05 2023-10-09T11:03:56.109 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.109 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.109 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.487791+0000", 2023-10-09T11:03:56.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0973e-05 2023-10-09T11:03:56.109 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.110 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.110 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.487799+0000", 2023-10-09T11:03:56.110 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6680000000000001e-06 2023-10-09T11:03:56.110 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.110 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.111 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.487862+0000", 2023-10-09T11:03:56.111 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3190999999999999e-05 2023-10-09T11:03:56.111 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.111 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.111 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:03:56.111 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.487986+0000", 2023-10-09T11:03:56.111 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000124611 2023-10-09T11:03:56.111 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.111 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.112 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,4", 2023-10-09T11:03:56.112 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.488197+0000", 2023-10-09T11:03:56.112 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000210173 2023-10-09T11:03:56.112 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.112 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.112 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:03:56.112 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.492077+0000", 2023-10-09T11:03:56.112 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038803499999999999 2023-10-09T11:03:56.113 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.113 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.113 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:56.113 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.492163+0000", 2023-10-09T11:03:56.113 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6347999999999994e-05 2023-10-09T11:03:56.113 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.113 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.113 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:56.114 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.589139+0000", 2023-10-09T11:03:56.114 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.096975304999999998 2023-10-09T11:03:56.114 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.114 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.114 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:03:56.114 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.589200+0000", 2023-10-09T11:03:56.114 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1032000000000001e-05 2023-10-09T11:03:56.114 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.115 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.115 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.115 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.589248+0000", 2023-10-09T11:03:56.115 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8911e-05 2023-10-09T11:03:56.115 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.115 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.115 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.116 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.116 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.116 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:15618 10.2 10:4b000449:::benchmark_data_smithi125_402869_object15617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T11:03:56.116 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:24.488315+0000", 2023-10-09T11:03:56.116 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 151.538112453, 2023-10-09T11:03:56.116 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.101478549, 2023-10-09T11:03:56.116 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.116 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:56.117 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.117 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:03:56.117 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:03:56.117 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15618 2023-10-09T11:03:56.117 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.117 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.117 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.117 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.117 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.488315+0000", 2023-10-09T11:03:56.117 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.117 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.117 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.118 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.118 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.488315+0000", 2023-10-09T11:03:56.118 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.118 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.118 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.118 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.118 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.488315+0000", 2023-10-09T11:03:56.118 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:56.118 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.118 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.118 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.119 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.488335+0000", 2023-10-09T11:03:56.119 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0384999999999999e-05 2023-10-09T11:03:56.119 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.119 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.119 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.119 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.488342+0000", 2023-10-09T11:03:56.119 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7549999999999997e-06 2023-10-09T11:03:56.119 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.119 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.119 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.119 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.488357+0000", 2023-10-09T11:03:56.119 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5051999999999999e-05 2023-10-09T11:03:56.120 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.120 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.120 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.120 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.488980+0000", 2023-10-09T11:03:56.120 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00062332100000000003 2023-10-09T11:03:56.120 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.120 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.120 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:03:56.120 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.489128+0000", 2023-10-09T11:03:56.120 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000147706 2023-10-09T11:03:56.120 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.120 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.121 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,4", 2023-10-09T11:03:56.121 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.489290+0000", 2023-10-09T11:03:56.121 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016229600000000001 2023-10-09T11:03:56.121 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.121 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.121 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:03:56.121 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.492081+0000", 2023-10-09T11:03:56.121 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002790992 2023-10-09T11:03:56.121 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.121 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.121 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:56.121 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.492720+0000", 2023-10-09T11:03:56.121 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00063882800000000003 2023-10-09T11:03:56.122 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.122 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.122 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:56.122 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.589708+0000", 2023-10-09T11:03:56.122 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.096987456 2023-10-09T11:03:56.122 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.122 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.122 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:03:56.122 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.589757+0000", 2023-10-09T11:03:56.122 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9088999999999997e-05 2023-10-09T11:03:56.122 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.122 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.123 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.123 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.589794+0000", 2023-10-09T11:03:56.123 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7183e-05 2023-10-09T11:03:56.123 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.123 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.123 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.123 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.123 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.123 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:15619 10.2 10:46627ba4:::benchmark_data_smithi125_402869_object15618:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T11:03:56.123 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:24.489482+0000", 2023-10-09T11:03:56.123 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 151.536945855, 2023-10-09T11:03:56.123 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.100425288, 2023-10-09T11:03:56.124 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.124 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:56.124 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.124 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:03:56.124 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:03:56.124 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15619 2023-10-09T11:03:56.124 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.124 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.124 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.124 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.124 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.489482+0000", 2023-10-09T11:03:56.124 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.124 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.124 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.125 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.125 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.489482+0000", 2023-10-09T11:03:56.125 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.125 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.125 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.125 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.125 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.489481+0000", 2023-10-09T11:03:56.125 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:56.125 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.125 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.125 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.125 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.489502+0000", 2023-10-09T11:03:56.126 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1603e-05 2023-10-09T11:03:56.126 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.126 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.126 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.126 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.489520+0000", 2023-10-09T11:03:56.126 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7141000000000001e-05 2023-10-09T11:03:56.126 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.126 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.126 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.126 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.489527+0000", 2023-10-09T11:03:56.126 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.542e-06 2023-10-09T11:03:56.127 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.127 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.127 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.127 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.490389+0000", 2023-10-09T11:03:56.127 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00086219599999999997 2023-10-09T11:03:56.127 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.127 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.127 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:03:56.127 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.490536+0000", 2023-10-09T11:03:56.127 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000147066 2023-10-09T11:03:56.127 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.127 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.128 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,4", 2023-10-09T11:03:56.128 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.490664+0000", 2023-10-09T11:03:56.128 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012790999999999999 2023-10-09T11:03:56.128 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.128 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.128 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:03:56.128 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.492086+0000", 2023-10-09T11:03:56.128 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014212509999999999 2023-10-09T11:03:56.128 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.128 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.128 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:56.128 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.492760+0000", 2023-10-09T11:03:56.129 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00067440100000000004 2023-10-09T11:03:56.129 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.129 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.129 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:56.129 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.589836+0000", 2023-10-09T11:03:56.129 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.097075681999999996 2023-10-09T11:03:56.129 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.129 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.129 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:03:56.129 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.589875+0000", 2023-10-09T11:03:56.129 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9385999999999999e-05 2023-10-09T11:03:56.129 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.130 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.130 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.130 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.589907+0000", 2023-10-09T11:03:56.130 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2353e-05 2023-10-09T11:03:56.130 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.130 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.130 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.130 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.130 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.130 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:15622 10.2 10:4dfde5e3:::benchmark_data_smithi125_402869_object15621:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T11:03:56.130 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:24.490268+0000", 2023-10-09T11:03:56.130 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 151.53615946400001, 2023-10-09T11:03:56.131 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.099973982000000003, 2023-10-09T11:03:56.131 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.131 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:56.131 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.131 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:03:56.131 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:03:56.131 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15622 2023-10-09T11:03:56.131 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.131 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.131 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.131 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.131 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.490268+0000", 2023-10-09T11:03:56.132 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.132 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.132 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.132 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.132 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.490268+0000", 2023-10-09T11:03:56.132 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.132 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.132 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.132 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.132 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.490267+0000", 2023-10-09T11:03:56.132 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:56.132 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.133 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.133 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.133 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.490291+0000", 2023-10-09T11:03:56.133 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4053000000000002e-05 2023-10-09T11:03:56.133 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.133 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.133 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.133 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.490306+0000", 2023-10-09T11:03:56.134 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4610000000000001e-05 2023-10-09T11:03:56.134 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.134 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.134 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.134 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.490315+0000", 2023-10-09T11:03:56.134 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7360000000000007e-06 2023-10-09T11:03:56.135 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.135 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.135 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.135 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.491343+0000", 2023-10-09T11:03:56.135 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010284859999999999 2023-10-09T11:03:56.135 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.135 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.135 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:03:56.136 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.491451+0000", 2023-10-09T11:03:56.136 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000107922 2023-10-09T11:03:56.136 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.136 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.136 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,4", 2023-10-09T11:03:56.136 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.491581+0000", 2023-10-09T11:03:56.136 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000130347 2023-10-09T11:03:56.136 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.137 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.137 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:03:56.137 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.492617+0000", 2023-10-09T11:03:56.137 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010351690000000001 2023-10-09T11:03:56.137 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.137 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.137 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:56.137 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.493580+0000", 2023-10-09T11:03:56.138 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00096365600000000002 2023-10-09T11:03:56.138 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.138 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.138 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:56.138 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.590155+0000", 2023-10-09T11:03:56.138 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.096574544999999998 2023-10-09T11:03:56.138 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.138 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.139 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:03:56.139 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.590200+0000", 2023-10-09T11:03:56.139 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5630000000000002e-05 2023-10-09T11:03:56.139 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.139 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.139 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.139 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:24.590242+0000", 2023-10-09T11:03:56.140 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.206e-05 2023-10-09T11:03:56.140 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.140 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.140 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.140 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.140 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.140 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:27677 10.1 e415/409 10:80a50fd8:::benchmark_data_smithi125_402869_object27676:head v 415'1775, mlcod=0'0)", 2023-10-09T11:03:56.140 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:34.468621+0000", 2023-10-09T11:03:56.141 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 141.55780740399999, 2023-10-09T11:03:56.141 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.15019200399999999, 2023-10-09T11:03:56.141 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.141 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:56.141 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.141 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.141 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.141 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.468621+0000", 2023-10-09T11:03:56.141 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.141 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.141 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.142 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.142 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.468621+0000", 2023-10-09T11:03:56.142 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.142 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.142 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.142 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.142 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.468620+0000", 2023-10-09T11:03:56.142 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:03:56.142 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.142 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.142 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.143 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.468643+0000", 2023-10-09T11:03:56.143 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3031999999999999e-05 2023-10-09T11:03:56.143 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.143 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.143 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.143 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.468652+0000", 2023-10-09T11:03:56.143 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2739999999999996e-06 2023-10-09T11:03:56.143 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.143 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.143 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.143 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.468657+0000", 2023-10-09T11:03:56.144 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9439999999999996e-06 2023-10-09T11:03:56.144 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.144 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.144 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.144 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.468701+0000", 2023-10-09T11:03:56.144 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4359e-05 2023-10-09T11:03:56.144 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.144 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.145 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:03:56.145 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.468727+0000", 2023-10-09T11:03:56.145 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5148999999999999e-05 2023-10-09T11:03:56.145 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.145 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.145 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:03:56.145 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.618781+0000", 2023-10-09T11:03:56.145 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.150054929 2023-10-09T11:03:56.146 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.146 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.146 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.146 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.618813+0000", 2023-10-09T11:03:56.146 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1049e-05 2023-10-09T11:03:56.146 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.147 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.147 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.147 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.147 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.147 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:27690 10.b 10:dbdd3c1f:::benchmark_data_smithi125_402869_object27689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:03:56.147 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:34.474307+0000", 2023-10-09T11:03:56.147 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 141.552120847, 2023-10-09T11:03:56.147 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.143884289, 2023-10-09T11:03:56.147 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.147 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:56.147 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.148 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:03:56.148 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:03:56.148 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 27690 2023-10-09T11:03:56.148 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.148 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.148 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.148 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.148 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.474307+0000", 2023-10-09T11:03:56.148 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.148 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.148 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.148 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.149 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.474307+0000", 2023-10-09T11:03:56.149 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.149 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.149 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.149 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.149 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.474305+0000", 2023-10-09T11:03:56.149 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:56.149 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.149 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.149 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.149 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.474328+0000", 2023-10-09T11:03:56.149 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2257999999999999e-05 2023-10-09T11:03:56.149 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.150 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.150 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.150 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.612854+0000", 2023-10-09T11:03:56.150 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.13852658100000001 2023-10-09T11:03:56.150 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.150 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.150 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.150 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.612870+0000", 2023-10-09T11:03:56.150 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6079999999999999e-05 2023-10-09T11:03:56.150 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.150 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.150 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.151 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.613867+0000", 2023-10-09T11:03:56.151 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00099646099999999992 2023-10-09T11:03:56.151 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.151 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.151 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:03:56.151 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.613974+0000", 2023-10-09T11:03:56.151 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000106838 2023-10-09T11:03:56.151 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.151 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.151 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:03:56.151 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.614117+0000", 2023-10-09T11:03:56.151 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014338600000000001 2023-10-09T11:03:56.152 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.152 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.152 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:56.152 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.618058+0000", 2023-10-09T11:03:56.152 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039409700000000002 2023-10-09T11:03:56.152 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.152 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.152 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:03:56.152 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.618066+0000", 2023-10-09T11:03:56.153 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6450000000000005e-06 2023-10-09T11:03:56.153 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.153 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.153 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:56.153 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.618118+0000", 2023-10-09T11:03:56.153 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2062999999999999e-05 2023-10-09T11:03:56.153 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.153 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.153 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:03:56.153 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.618157+0000", 2023-10-09T11:03:56.153 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8729999999999997e-05 2023-10-09T11:03:56.153 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.154 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.154 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.154 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.618191+0000", 2023-10-09T11:03:56.154 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4860999999999997e-05 2023-10-09T11:03:56.154 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.154 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.154 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.154 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.154 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.154 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:27693 10.0 e415/403 10:017bad4c:::benchmark_data_smithi125_402869_object27692:head v 415'1710, mlcod=415'1708)", 2023-10-09T11:03:56.154 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:34.474783+0000", 2023-10-09T11:03:56.154 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 141.55164472000001, 2023-10-09T11:03:56.155 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.14176056400000001, 2023-10-09T11:03:56.155 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.155 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:56.155 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.155 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.155 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.155 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.474783+0000", 2023-10-09T11:03:56.155 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.155 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.155 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.155 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.155 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.474783+0000", 2023-10-09T11:03:56.156 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.156 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.156 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.156 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.156 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.474782+0000", 2023-10-09T11:03:56.156 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:03:56.156 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.156 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.156 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.156 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.474834+0000", 2023-10-09T11:03:56.156 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1910999999999998e-05 2023-10-09T11:03:56.156 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.157 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.157 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.157 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.612937+0000", 2023-10-09T11:03:56.157 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.138103217 2023-10-09T11:03:56.157 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.157 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.157 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.157 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.612946+0000", 2023-10-09T11:03:56.157 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4919999999999993e-06 2023-10-09T11:03:56.157 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.157 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.157 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.158 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.613014+0000", 2023-10-09T11:03:56.158 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7809000000000001e-05 2023-10-09T11:03:56.158 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.158 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.158 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:03:56.158 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.613036+0000", 2023-10-09T11:03:56.158 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2008999999999999e-05 2023-10-09T11:03:56.159 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.159 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.159 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:03:56.159 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.616505+0000", 2023-10-09T11:03:56.159 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0034688700000000002 2023-10-09T11:03:56.159 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.159 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.160 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.160 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:34.616544+0000", 2023-10-09T11:03:56.160 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9116999999999998e-05 2023-10-09T11:03:56.160 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.160 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.160 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.161 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.161 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.161 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:91367 10.2 10:48de9735:::benchmark_data_smithi125_402869_object91366:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T11:03:56.161 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:38.667194+0000", 2023-10-09T11:03:56.161 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 77.359233936999999, 2023-10-09T11:03:56.161 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.103057039, 2023-10-09T11:03:56.161 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.162 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:56.162 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.162 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:03:56.162 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:03:56.162 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 91367 2023-10-09T11:03:56.162 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.162 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.162 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.162 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.163 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.667194+0000", 2023-10-09T11:03:56.163 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.163 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.163 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.163 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.163 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.667194+0000", 2023-10-09T11:03:56.163 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.163 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.163 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.163 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.163 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.667192+0000", 2023-10-09T11:03:56.163 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:03:56.164 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.164 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.164 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.164 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.667234+0000", 2023-10-09T11:03:56.164 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1958e-05 2023-10-09T11:03:56.164 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.164 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.164 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.164 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.667248+0000", 2023-10-09T11:03:56.164 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4256999999999999e-05 2023-10-09T11:03:56.164 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.164 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.165 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.165 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.667258+0000", 2023-10-09T11:03:56.165 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1989999999999995e-06 2023-10-09T11:03:56.165 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.165 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.165 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.165 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.667352+0000", 2023-10-09T11:03:56.165 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4953999999999999e-05 2023-10-09T11:03:56.165 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.165 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.165 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:03:56.166 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.667479+0000", 2023-10-09T11:03:56.166 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012682899999999999 2023-10-09T11:03:56.166 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.166 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.166 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,4", 2023-10-09T11:03:56.166 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.667631+0000", 2023-10-09T11:03:56.166 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015180099999999999 2023-10-09T11:03:56.166 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.166 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.167 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:03:56.167 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.674988+0000", 2023-10-09T11:03:56.167 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0073569459999999996 2023-10-09T11:03:56.167 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.167 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.167 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:56.167 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.764491+0000", 2023-10-09T11:03:56.168 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.089503009999999994 2023-10-09T11:03:56.168 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.168 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.168 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:56.168 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.770155+0000", 2023-10-09T11:03:56.168 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0056634779999999996 2023-10-09T11:03:56.168 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.169 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.169 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:03:56.169 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.770210+0000", 2023-10-09T11:03:56.169 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5143999999999999e-05 2023-10-09T11:03:56.169 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.169 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.169 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.169 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.770251+0000", 2023-10-09T11:03:56.170 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1343e-05 2023-10-09T11:03:56.170 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.170 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.170 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.170 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.170 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.170 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:91375 10.2 10:49858f16:::benchmark_data_smithi125_402869_object91374:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T11:03:56.171 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:38.671189+0000", 2023-10-09T11:03:56.171 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 77.355239240000003, 2023-10-09T11:03:56.171 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.09916925, 2023-10-09T11:03:56.171 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.171 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:56.171 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:03:56.171 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:03:56.171 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:03:56.172 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 91375 2023-10-09T11:03:56.172 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.172 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.172 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.172 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.172 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.671189+0000", 2023-10-09T11:03:56.172 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.172 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.173 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.173 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.173 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.671189+0000", 2023-10-09T11:03:56.173 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.173 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.173 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.173 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.173 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.671187+0000", 2023-10-09T11:03:56.173 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:03:56.173 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.174 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.174 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.174 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.671225+0000", 2023-10-09T11:03:56.174 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7759000000000002e-05 2023-10-09T11:03:56.174 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.174 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.174 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.174 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.671239+0000", 2023-10-09T11:03:56.174 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3733e-05 2023-10-09T11:03:56.174 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.174 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.174 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.174 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.671248+0000", 2023-10-09T11:03:56.175 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7189999999999996e-06 2023-10-09T11:03:56.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.175 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.175 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.671994+0000", 2023-10-09T11:03:56.175 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00074551200000000004 2023-10-09T11:03:56.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.175 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:03:56.175 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.672115+0000", 2023-10-09T11:03:56.175 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120782 2023-10-09T11:03:56.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.176 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,4", 2023-10-09T11:03:56.176 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.672259+0000", 2023-10-09T11:03:56.176 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000144497 2023-10-09T11:03:56.176 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.176 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.176 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:03:56.176 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.764506+0000", 2023-10-09T11:03:56.176 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.092247209999999996 2023-10-09T11:03:56.176 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.176 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.176 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:56.176 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.764584+0000", 2023-10-09T11:03:56.176 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8090000000000006e-05 2023-10-09T11:03:56.177 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.177 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.177 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:03:56.177 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.770300+0000", 2023-10-09T11:03:56.177 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0057153799999999999 2023-10-09T11:03:56.177 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.177 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.177 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:03:56.177 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.770331+0000", 2023-10-09T11:03:56.177 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1318000000000001e-05 2023-10-09T11:03:56.177 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.177 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.178 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.178 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.770358+0000", 2023-10-09T11:03:56.178 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6903e-05 2023-10-09T11:03:56.178 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.178 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.178 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.178 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.178 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.178 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:91376 10.1 e438/435 10:846227e1:::benchmark_data_smithi125_402869_object91375:head v 438'5770, mlcod=438'5768)", 2023-10-09T11:03:56.178 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:38.675962+0000", 2023-10-09T11:03:56.178 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 77.350466115000003, 2023-10-09T11:03:56.178 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.097561013000000002, 2023-10-09T11:03:56.179 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:03:56.179 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:03:56.179 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:03:56.179 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.179 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:03:56.179 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.675962+0000", 2023-10-09T11:03:56.179 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.179 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.179 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.179 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:03:56.180 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.675962+0000", 2023-10-09T11:03:56.180 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:03:56.180 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.180 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.180 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:03:56.180 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.675961+0000", 2023-10-09T11:03:56.180 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:03:56.180 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.180 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.180 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:03:56.181 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.675982+0000", 2023-10-09T11:03:56.181 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1348e-05 2023-10-09T11:03:56.181 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.181 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.181 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:03:56.181 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.762128+0000", 2023-10-09T11:03:56.181 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.086145639999999996 2023-10-09T11:03:56.181 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.181 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.181 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:03:56.181 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.762135+0000", 2023-10-09T11:03:56.182 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6789999999999999e-06 2023-10-09T11:03:56.182 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.182 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.182 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:03:56.182 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.762183+0000", 2023-10-09T11:03:56.182 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8393000000000002e-05 2023-10-09T11:03:56.182 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.182 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.182 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:03:56.182 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.762223+0000", 2023-10-09T11:03:56.182 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9406000000000002e-05 2023-10-09T11:03:56.183 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.183 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.183 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:03:56.183 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.773484+0000", 2023-10-09T11:03:56.183 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.01126164 2023-10-09T11:03:56.183 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:03:56.183 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:03:56.183 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:03:56.183 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:38.773523+0000", 2023-10-09T11:03:56.183 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8605000000000001e-05 2023-10-09T11:03:56.183 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.183 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.184 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.184 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:03:56.184 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:03:56.184 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:03:56.184 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:56.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1498983506' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:56.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:56 smithi155 ceph-mon[184854]: pgmap v1146: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 917 op/s; 26128/125295 objects degraded (20.853%); 2959/125295 objects misplaced (2.362%); 11 MiB/s, 171 objects/s recovering 2023-10-09T11:03:56.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:56 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:56.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:56 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:56.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:56 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:56.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:56 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:56.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1498983506' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:56.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:56 smithi125 ceph-mon[195507]: pgmap v1146: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 917 op/s; 26128/125295 objects degraded (20.853%); 2959/125295 objects misplaced (2.362%); 11 MiB/s, 171 objects/s recovering 2023-10-09T11:03:56.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:56.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:56.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:56.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:56.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1498983506' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:03:56.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:56 smithi125 ceph-mon[200197]: pgmap v1146: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 917 op/s; 26128/125295 objects degraded (20.853%); 2959/125295 objects misplaced (2.362%); 11 MiB/s, 171 objects/s recovering 2023-10-09T11:03:56.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:56 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:56.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:56 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:56.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:56 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:56.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:56 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:03:57.080 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:03:57.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:57.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:03:57.187 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:57.215 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:58.299 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:03:58.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:58.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:03:58.404 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:58.462 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:58.496 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:58.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:58.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:58.600 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:03:58.654 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:58.744 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:03:58.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:03:58.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:03:58.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:03:58.897 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:58 smithi125 ceph-mon[195507]: pgmap v1147: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 891 op/s; 25536/117435 objects degraded (21.745%); 2602/117435 objects misplaced (2.216%); 10 MiB/s, 162 objects/s recovering 2023-10-09T11:03:58.898 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:03:58 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 25536/117435 objects degraded (21.745%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:03:58.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:58 smithi125 ceph-mon[200197]: pgmap v1147: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 891 op/s; 25536/117435 objects degraded (21.745%); 2602/117435 objects misplaced (2.216%); 10 MiB/s, 162 objects/s recovering 2023-10-09T11:03:58.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:03:58 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 25536/117435 objects degraded (21.745%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:03:58.900 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:03:58.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:58 smithi155 ceph-mon[184854]: pgmap v1147: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 891 op/s; 25536/117435 objects degraded (21.745%); 2602/117435 objects misplaced (2.216%); 10 MiB/s, 162 objects/s recovering 2023-10-09T11:03:58.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:03:58 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 25536/117435 objects degraded (21.745%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:04:00.039 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T11:04:00.040 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T11:04:00.046 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:00.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:00.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:00.083 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T11:04:00.108 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:00.152 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:00.198 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:00.233 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:00.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:00.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:00.341 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:00.395 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:00.436 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:00.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:00.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:00.544 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:00.594 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:00.637 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:00.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:00.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:00.747 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:00.803 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:00.843 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:00.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:00.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:00.950 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:01.001 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:01.024 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:01.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:01.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:01.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:00 smithi125 ceph-mon[195507]: pgmap v1148: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 944 op/s; 24786/111804 objects degraded (22.169%); 2330/111804 objects misplaced (2.084%); 11 MiB/s, 184 objects/s recovering 2023-10-09T11:04:01.122 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:01.122 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:01.122 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:04:01.122 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:00 smithi125 ceph-mon[200197]: pgmap v1148: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 944 op/s; 24786/111804 objects degraded (22.169%); 2330/111804 objects misplaced (2.084%); 11 MiB/s, 184 objects/s recovering 2023-10-09T11:04:01.122 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:01.122 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:01.122 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:04:01.128 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:01.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:00 smithi155 ceph-mon[184854]: pgmap v1148: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 944 op/s; 24786/111804 objects degraded (22.169%); 2330/111804 objects misplaced (2.084%); 11 MiB/s, 184 objects/s recovering 2023-10-09T11:04:01.189 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:01.189 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:01.189 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:04:01.191 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:01.205 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T11:04:01.245 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:01.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:01.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:01.352 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:01.407 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:04:01 smithi125 bash[428608]: 9a7c208464b67bf56f1f8f6ab371ebd99be9141097c24480ae996829690ab1ca 2023-10-09T11:04:01.407 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:04:01 smithi125 systemd[1]: Started Ceph osd.3 for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T11:04:01.407 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:04:01 smithi125 ceph-osd[428733]: -- 172.21.15.125:0/62631516 <== mon.0 v2:172.21.15.125:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x564bc511b600 con 0x564bc50a3000 2023-10-09T11:04:01.408 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:04:01 smithi125 ceph-osd[428733]: -- 172.21.15.125:0/62631516 <== mon.0 v2:172.21.15.125:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564bc50d24e0 con 0x564bc50a3000 2023-10-09T11:04:01.408 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:04:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:04:01] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.43.0" 2023-10-09T11:04:01.413 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:02.042 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:04:02.056 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:02.116 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:04:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[428685]: 2023-10-09T11:04:01.832+0000 7f4e5aa1b640 -1 Falling back to public interface 2023-10-09T11:04:02.695 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:02.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:02.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:02.924 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:03.006 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:03.037 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:03.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:03.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:03.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:02 smithi125 ceph-mon[195507]: pgmap v1149: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 987 op/s; 23859/104403 objects degraded (22.853%); 2330/104403 objects misplaced (2.232%); 9.1 MiB/s, 178 objects/s recovering 2023-10-09T11:04:03.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:02 smithi125 ceph-mon[200197]: pgmap v1149: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 987 op/s; 23859/104403 objects degraded (22.853%); 2330/104403 objects misplaced (2.232%); 9.1 MiB/s, 178 objects/s recovering 2023-10-09T11:04:03.144 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:03.152 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:03.203 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:02 smithi155 ceph-mon[184854]: pgmap v1149: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 987 op/s; 23859/104403 objects degraded (22.853%); 2330/104403 objects misplaced (2.232%); 9.1 MiB/s, 178 objects/s recovering 2023-10-09T11:04:03.211 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:03.253 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:03.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:03.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:03.359 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:03.419 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:03.471 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:03.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:03.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:03.577 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:03.638 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:03.677 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:03.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:03.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:03.784 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:03.844 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:03.874 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:03.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:03.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:03.983 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:04.047 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:04.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:03 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 23859/104403 objects degraded (22.853%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:04:04.074 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3505002260' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:04.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:03 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 23859/104403 objects degraded (22.853%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:04:04.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3505002260' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:04.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:03 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 23859/104403 objects degraded (22.853%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:04:04.224 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3505002260' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:04.271 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:04:05.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:04 smithi155 ceph-mon[184854]: pgmap v1150: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 928 op/s; 23018/98550 objects degraded (23.357%); 2039/98550 objects misplaced (2.069%); 10 MiB/s, 195 objects/s recovering 2023-10-09T11:04:05.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:04 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:04:05.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:04 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3505002260' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:04:05.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:04 smithi155 ceph-mon[184854]: osdmap e465: 8 total, 7 up, 7 in 2023-10-09T11:04:05.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:04 smithi155 ceph-mon[184854]: failed to encode map e465 with expected crc 2023-10-09T11:04:05.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:04 smithi155 ceph-mon[184854]: failed to encode map e465 with expected crc 2023-10-09T11:04:05.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:04 smithi155 ceph-mon[184854]: failed to encode map e465 with expected crc 2023-10-09T11:04:05.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:04 smithi125 ceph-mon[195507]: pgmap v1150: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 928 op/s; 23018/98550 objects degraded (23.357%); 2039/98550 objects misplaced (2.069%); 10 MiB/s, 195 objects/s recovering 2023-10-09T11:04:05.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:04 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:04:05.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:04 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3505002260' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:04:05.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:04 smithi125 ceph-mon[195507]: osdmap e465: 8 total, 7 up, 7 in 2023-10-09T11:04:05.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:04 smithi125 ceph-mon[195507]: failed to encode map e465 with expected crc 2023-10-09T11:04:05.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:04 smithi125 ceph-mon[195507]: failed to encode map e465 with expected crc 2023-10-09T11:04:05.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:04 smithi125 ceph-mon[195507]: failed to encode map e465 with expected crc 2023-10-09T11:04:05.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:04 smithi125 ceph-mon[200197]: pgmap v1150: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 928 op/s; 23018/98550 objects degraded (23.357%); 2039/98550 objects misplaced (2.069%); 10 MiB/s, 195 objects/s recovering 2023-10-09T11:04:05.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:04 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:04:05.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:04 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3505002260' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:04:05.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:04 smithi125 ceph-mon[200197]: osdmap e465: 8 total, 7 up, 7 in 2023-10-09T11:04:05.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:04 smithi125 ceph-mon[200197]: failed to encode map e465 with expected crc 2023-10-09T11:04:05.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:04 smithi125 ceph-mon[200197]: failed to encode map e465 with expected crc 2023-10-09T11:04:05.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:04 smithi125 ceph-mon[200197]: failed to encode map e465 with expected crc 2023-10-09T11:04:05.388 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:05.503 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:05.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:05.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:05.614 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:05.684 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:05.764 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:05.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:05.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:05.871 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:05.944 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:05 smithi125 ceph-mon[195507]: failed to encode map e465 with expected crc 2023-10-09T11:04:05.944 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:05 smithi125 ceph-mon[200197]: failed to encode map e465 with expected crc 2023-10-09T11:04:05.945 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:05.981 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:05.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:05.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:06.095 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:06.155 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:06.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:05 smithi155 ceph-mon[184854]: failed to encode map e465 with expected crc 2023-10-09T11:04:06.194 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:06.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:06.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:06.300 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:06.344 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:06.356 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:04:06.356 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:04:06.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.356 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:167902 10.c 10:3c2c2e41:::benchmark_data_smithi125_402869_object69333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-10-09T11:04:06.357 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:04:06.338667+0000", 2023-10-09T11:04:06.357 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.011471739999999999, 2023-10-09T11:04:06.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011511225999999999, 2023-10-09T11:04:06.357 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:06.357 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:04:06.357 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:06.357 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:04:06.357 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:06.357 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 167902 2023-10-09T11:04:06.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.357 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:06.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:06.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.338667+0000", 2023-10-09T11:04:06.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:06.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.338667+0000", 2023-10-09T11:04:06.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:06.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.338666+0000", 2023-10-09T11:04:06.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:04:06.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:06.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.338675+0000", 2023-10-09T11:04:06.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5910000000000002e-06 2023-10-09T11:04:06.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:06.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.338689+0000", 2023-10-09T11:04:06.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3888e-05 2023-10-09T11:04:06.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:06.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.338699+0000", 2023-10-09T11:04:06.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4630000000000007e-06 2023-10-09T11:04:06.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:06.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341176+0000", 2023-10-09T11:04:06.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024775700000000001 2023-10-09T11:04:06.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:06.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341311+0000", 2023-10-09T11:04:06.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000134575 2023-10-09T11:04:06.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,7", 2023-10-09T11:04:06.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341455+0000", 2023-10-09T11:04:06.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014393199999999999 2023-10-09T11:04:06.361 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.361 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:06.361 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.362 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:167910 10.1 10:80acfc5b:::benchmark_data_smithi125_402869_object69341:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-10-09T11:04:06.362 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:04:06.340749+0000", 2023-10-09T11:04:06.362 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0093902880000000001, 2023-10-09T11:04:06.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0095253530000000003, 2023-10-09T11:04:06.362 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:06.362 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:04:06.362 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:06.362 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:04:06.362 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:06.362 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 167910 2023-10-09T11:04:06.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.363 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:06.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:06.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.340749+0000", 2023-10-09T11:04:06.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:06.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.340749+0000", 2023-10-09T11:04:06.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:06.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.340746+0000", 2023-10-09T11:04:06.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T11:04:06.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:06.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.340757+0000", 2023-10-09T11:04:06.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1273e-05 2023-10-09T11:04:06.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:06.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.340777+0000", 2023-10-09T11:04:06.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9049e-05 2023-10-09T11:04:06.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:06.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.340789+0000", 2023-10-09T11:04:06.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2238999999999999e-05 2023-10-09T11:04:06.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:06.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.347462+0000", 2023-10-09T11:04:06.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066736390000000003 2023-10-09T11:04:06.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:06.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.347606+0000", 2023-10-09T11:04:06.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000143158 2023-10-09T11:04:06.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T11:04:06.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.348097+0000", 2023-10-09T11:04:06.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00049117799999999999 2023-10-09T11:04:06.366 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.366 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:06.366 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.367 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:167903 10.16 e465/441 10:6c0d8763:::benchmark_data_smithi125_402869_object69334:head v 465'8546, mlcod=465'8543)", 2023-10-09T11:04:06.367 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:04:06.341291+0000", 2023-10-09T11:04:06.367 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0088483169999999996, 2023-10-09T11:04:06.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0090577709999999992, 2023-10-09T11:04:06.367 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:06.367 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:04:06.367 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:06.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.367 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:06.367 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341291+0000", 2023-10-09T11:04:06.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:06.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341291+0000", 2023-10-09T11:04:06.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:06.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341290+0000", 2023-10-09T11:04:06.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:04:06.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:06.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341303+0000", 2023-10-09T11:04:06.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3608000000000001e-05 2023-10-09T11:04:06.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:06.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341317+0000", 2023-10-09T11:04:06.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4132e-05 2023-10-09T11:04:06.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:06.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341325+0000", 2023-10-09T11:04:06.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8709999999999997e-06 2023-10-09T11:04:06.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:06.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.343866+0000", 2023-10-09T11:04:06.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025405979999999998 2023-10-09T11:04:06.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:06.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.343887+0000", 2023-10-09T11:04:06.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0801999999999998e-05 2023-10-09T11:04:06.371 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.371 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:06.371 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.371 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:167904 10.6 e465/441 10:63a050a7:::benchmark_data_smithi125_402869_object69335:head v 465'8439, mlcod=465'8437)", 2023-10-09T11:04:06.371 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:04:06.341333+0000", 2023-10-09T11:04:06.371 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0088059109999999996, 2023-10-09T11:04:06.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0090736440000000005, 2023-10-09T11:04:06.371 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:06.371 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:04:06.371 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:06.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:06.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341333+0000", 2023-10-09T11:04:06.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:06.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341333+0000", 2023-10-09T11:04:06.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:06.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341333+0000", 2023-10-09T11:04:06.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:06.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:06.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341337+0000", 2023-10-09T11:04:06.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2660000000000003e-06 2023-10-09T11:04:06.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:06.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341343+0000", 2023-10-09T11:04:06.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7649999999999999e-06 2023-10-09T11:04:06.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:06.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341347+0000", 2023-10-09T11:04:06.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7419999999999999e-06 2023-10-09T11:04:06.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:06.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.345594+0000", 2023-10-09T11:04:06.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042473520000000002 2023-10-09T11:04:06.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:06.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.345617+0000", 2023-10-09T11:04:06.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3473e-05 2023-10-09T11:04:06.375 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.375 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:06.375 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.375 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:167894 10.2 e465/441 10:42a13da1:::benchmark_data_smithi125_402869_object69325:head v 465'8278, mlcod=465'8276)", 2023-10-09T11:04:06.375 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:04:06.341397+0000", 2023-10-09T11:04:06.375 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0087418870000000003, 2023-10-09T11:04:06.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.009069489, 2023-10-09T11:04:06.376 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:06.376 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:04:06.376 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:06.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:06.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341397+0000", 2023-10-09T11:04:06.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:06.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341397+0000", 2023-10-09T11:04:06.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.377 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:06.377 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341397+0000", 2023-10-09T11:04:06.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:06.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.377 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:06.377 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341402+0000", 2023-10-09T11:04:06.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8450000000000002e-06 2023-10-09T11:04:06.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:06.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341408+0000", 2023-10-09T11:04:06.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5209999999999999e-06 2023-10-09T11:04:06.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:06.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341413+0000", 2023-10-09T11:04:06.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3750000000000002e-06 2023-10-09T11:04:06.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:06.379 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.348828+0000", 2023-10-09T11:04:06.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074140889999999996 2023-10-09T11:04:06.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.379 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:06.379 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.348847+0000", 2023-10-09T11:04:06.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9111999999999999e-05 2023-10-09T11:04:06.379 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.379 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:06.379 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.380 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:167895 10.5 e465/460 10:a2901166:::benchmark_data_smithi125_402869_object69326:head v 465'8448, mlcod=419'2967)", 2023-10-09T11:04:06.380 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:04:06.341421+0000", 2023-10-09T11:04:06.380 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0087180880000000006, 2023-10-09T11:04:06.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0091037740000000002, 2023-10-09T11:04:06.380 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:06.380 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:04:06.380 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:06.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:06.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341421+0000", 2023-10-09T11:04:06.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:06.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341421+0000", 2023-10-09T11:04:06.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:06.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341421+0000", 2023-10-09T11:04:06.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:06.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:06.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341426+0000", 2023-10-09T11:04:06.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0529999999999997e-06 2023-10-09T11:04:06.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:06.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341433+0000", 2023-10-09T11:04:06.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.917e-06 2023-10-09T11:04:06.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:06.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.341437+0000", 2023-10-09T11:04:06.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8239999999999999e-06 2023-10-09T11:04:06.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:06.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.342216+0000", 2023-10-09T11:04:06.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00077850800000000002 2023-10-09T11:04:06.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:06.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.342240+0000", 2023-10-09T11:04:06.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3753000000000001e-05 2023-10-09T11:04:06.383 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.383 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:06.384 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.384 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.384 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.384 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:167900 10.2 e465/441)", 2023-10-09T11:04:06.384 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:04:06.344608+0000", 2023-10-09T11:04:06.384 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0055309269999999997, 2023-10-09T11:04:06.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.005972621, 2023-10-09T11:04:06.384 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:06.384 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T11:04:06.384 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:06.384 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.384 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:06.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.344608+0000", 2023-10-09T11:04:06.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:06.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.344608+0000", 2023-10-09T11:04:06.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:06.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.344607+0000", 2023-10-09T11:04:06.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:04:06.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:06.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.344616+0000", 2023-10-09T11:04:06.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4290000000000001e-06 2023-10-09T11:04:06.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:06.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.344628+0000", 2023-10-09T11:04:06.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1973999999999999e-05 2023-10-09T11:04:06.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:06.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.344635+0000", 2023-10-09T11:04:06.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2019999999999998e-06 2023-10-09T11:04:06.387 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.387 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:06.387 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.387 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:167908 10.16 e465/441 10:6f3bfd53:::benchmark_data_smithi125_402869_object69339:head v 465'8547, mlcod=465'8543)", 2023-10-09T11:04:06.387 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:04:06.345636+0000", 2023-10-09T11:04:06.387 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0045026800000000002, 2023-10-09T11:04:06.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004993394, 2023-10-09T11:04:06.388 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:06.388 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:04:06.388 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:06.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:06.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.345636+0000", 2023-10-09T11:04:06.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:06.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.345636+0000", 2023-10-09T11:04:06.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:06.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.345636+0000", 2023-10-09T11:04:06.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:04:06.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:06.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.345646+0000", 2023-10-09T11:04:06.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9550000000000004e-06 2023-10-09T11:04:06.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:06.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.345656+0000", 2023-10-09T11:04:06.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0849999999999999e-05 2023-10-09T11:04:06.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:06.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.345663+0000", 2023-10-09T11:04:06.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.348e-06 2023-10-09T11:04:06.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:06.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.346166+0000", 2023-10-09T11:04:06.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00050300900000000001 2023-10-09T11:04:06.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:06.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.346178+0000", 2023-10-09T11:04:06.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2374999999999999e-05 2023-10-09T11:04:06.392 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.392 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:06.392 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.392 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.392 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:167911 10.14 10:2b9f924a:::benchmark_data_smithi125_402869_object69342:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-10-09T11:04:06.392 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:04:06.348283+0000", 2023-10-09T11:04:06.392 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0018563200000000001, 2023-10-09T11:04:06.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024093140000000001, 2023-10-09T11:04:06.392 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:06.392 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:04:06.392 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:06.393 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:04:06.393 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:06.393 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 167911 2023-10-09T11:04:06.393 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.393 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:06.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.393 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:06.393 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.348283+0000", 2023-10-09T11:04:06.393 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.393 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.393 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:06.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.348283+0000", 2023-10-09T11:04:06.394 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:06.394 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.394 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:06.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.348280+0000", 2023-10-09T11:04:06.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T11:04:06.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.395 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:06.395 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.348291+0000", 2023-10-09T11:04:06.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0158e-05 2023-10-09T11:04:06.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:06.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.348304+0000", 2023-10-09T11:04:06.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3132000000000001e-05 2023-10-09T11:04:06.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:06.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.348314+0000", 2023-10-09T11:04:06.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0479000000000001e-05 2023-10-09T11:04:06.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.397 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:06.397 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.348370+0000", 2023-10-09T11:04:06.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5627999999999997e-05 2023-10-09T11:04:06.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.397 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:06.397 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.348538+0000", 2023-10-09T11:04:06.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016847699999999999 2023-10-09T11:04:06.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:06.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:06.397 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:04:06.398 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:06.349041+0000", 2023-10-09T11:04:06.398 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00050221100000000002 2023-10-09T11:04:06.398 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.398 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:06.398 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.398 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:06.398 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T11:04:06.398 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 9 2023-10-09T11:04:06.398 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:04:06.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:06.399 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:06.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:06.500 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:06.603 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:07.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:06 smithi155 ceph-mon[184854]: pgmap v1152: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 890 op/s; 23018/98550 objects degraded (23.357%); 2039/98550 objects misplaced (2.069%); 8.9 MiB/s, 180 objects/s recovering 2023-10-09T11:04:07.269 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:06 smithi125 ceph-mon[195507]: pgmap v1152: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 890 op/s; 23018/98550 objects degraded (23.357%); 2039/98550 objects misplaced (2.069%); 8.9 MiB/s, 180 objects/s recovering 2023-10-09T11:04:07.269 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:06 smithi125 ceph-mon[200197]: pgmap v1152: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 890 op/s; 23018/98550 objects degraded (23.357%); 2039/98550 objects misplaced (2.069%); 8.9 MiB/s, 180 objects/s recovering 2023-10-09T11:04:07.355 INFO:teuthology.orchestra.run.smithi125.stderr:no valid command found; 10 closest matches: 2023-10-09T11:04:07.355 INFO:teuthology.orchestra.run.smithi125.stderr:0 2023-10-09T11:04:07.355 INFO:teuthology.orchestra.run.smithi125.stderr:1 2023-10-09T11:04:07.356 INFO:teuthology.orchestra.run.smithi125.stderr:2 2023-10-09T11:04:07.356 INFO:teuthology.orchestra.run.smithi125.stderr:abort 2023-10-09T11:04:07.356 INFO:teuthology.orchestra.run.smithi125.stderr:assert 2023-10-09T11:04:07.356 INFO:teuthology.orchestra.run.smithi125.stderr:bluefs debug_inject_read_zeros 2023-10-09T11:04:07.356 INFO:teuthology.orchestra.run.smithi125.stderr:bluefs files list 2023-10-09T11:04:07.356 INFO:teuthology.orchestra.run.smithi125.stderr:bluefs stats 2023-10-09T11:04:07.356 INFO:teuthology.orchestra.run.smithi125.stderr:bluestore bluefs device info [] 2023-10-09T11:04:07.356 INFO:teuthology.orchestra.run.smithi125.stderr:config diff 2023-10-09T11:04:07.356 INFO:teuthology.orchestra.run.smithi125.stderr:admin_socket: invalid command 2023-10-09T11:04:07.788 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:04:07.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:07.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:07.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:08.022 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:08.065 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:08.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:08.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:08.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:08.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:08.284 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:08.436 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:08.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:08.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:08.464 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T11:04:08.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:08.586 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:08.614 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:08.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:08.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:08.719 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:08.734 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T11:04:08.735 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-3, ['dump_ops_in_flight'] 2023-10-09T11:04:08.755 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:08.785 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:08.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:08.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:08.889 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:08.947 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:08 smithi125 ceph-mon[195507]: pgmap v1153: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 875 op/s; 21943/91140 objects degraded (24.076%); 2039/91140 objects misplaced (2.237%); 6.6 MiB/s, 197 objects/s recovering 2023-10-09T11:04:08.947 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:08 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 21943/91140 objects degraded (24.076%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:04:08.947 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:08 smithi125 ceph-mon[200197]: pgmap v1153: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 875 op/s; 21943/91140 objects degraded (24.076%); 2039/91140 objects misplaced (2.237%); 6.6 MiB/s, 197 objects/s recovering 2023-10-09T11:04:08.948 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:08 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 21943/91140 objects degraded (24.076%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:04:08.959 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:09.006 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:09.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:09.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:09.112 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:09.126 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:09.143 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:09.165 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:09.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:09.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:09.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:08 smithi155 ceph-mon[184854]: pgmap v1153: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 875 op/s; 21943/91140 objects degraded (24.076%); 2039/91140 objects misplaced (2.237%); 6.6 MiB/s, 197 objects/s recovering 2023-10-09T11:04:09.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:08 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 21943/91140 objects degraded (24.076%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:04:09.273 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:09.282 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:04:09.290 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:09.310 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:09.333 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:04:09.335 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:09.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:09.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:09.368 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:04:09.404 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:04:09.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:09.447 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:04:09.470 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:09.496 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:04:09.498 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:09.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:09.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:09.544 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:04:09.545 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:04:09.545 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:04:09.545 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:04:09.545 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-10-09T11:04:09.545 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:04:09.545 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:04:09.545 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:04:09.545 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:04:09.605 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:09.638 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:09.662 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:09.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:09.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:09.767 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:09.856 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:10.041 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:09 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1603527787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:10.041 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:09 smithi125 ceph-mon[195507]: osdmap e466: 8 total, 7 up, 7 in 2023-10-09T11:04:10.041 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:09 smithi125 ceph-mon[195507]: failed to encode map e466 with expected crc 2023-10-09T11:04:10.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:09 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1603527787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:10.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:09 smithi125 ceph-mon[200197]: osdmap e466: 8 total, 7 up, 7 in 2023-10-09T11:04:10.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:09 smithi125 ceph-mon[200197]: failed to encode map e466 with expected crc 2023-10-09T11:04:10.142 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:10.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:10.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:10.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:09 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1603527787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:10.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:09 smithi155 ceph-mon[184854]: osdmap e466: 8 total, 7 up, 7 in 2023-10-09T11:04:10.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:09 smithi155 ceph-mon[184854]: failed to encode map e466 with expected crc 2023-10-09T11:04:10.249 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:10.285 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:10.491 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T11:04:10.492 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T11:04:10.610 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:10.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:10.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:10.717 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:10.756 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:10.788 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:10.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:10.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:10.896 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:10.934 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:10.963 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:10.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:10.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:11.069 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:11.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:10 smithi155 ceph-mon[184854]: failed to encode map e466 with expected crc 2023-10-09T11:04:11.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:10 smithi155 ceph-mon[184854]: pgmap v1155: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 783 op/s; 21496/85575 objects degraded (25.119%); 1769/85575 objects misplaced (2.067%); 5.9 MiB/s, 196 objects/s recovering 2023-10-09T11:04:11.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:10 smithi155 ceph-mon[184854]: failed to encode map e467 with expected crc 2023-10-09T11:04:11.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:10 smithi155 ceph-mon[184854]: osdmap e467: 8 total, 7 up, 7 in 2023-10-09T11:04:11.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:10 smithi155 ceph-mon[184854]: failed to encode map e467 with expected crc 2023-10-09T11:04:11.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:10 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:11.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:10 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:11.104 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:11.134 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:11.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:11.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:11.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:10 smithi125 ceph-mon[195507]: failed to encode map e466 with expected crc 2023-10-09T11:04:11.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:10 smithi125 ceph-mon[195507]: pgmap v1155: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 783 op/s; 21496/85575 objects degraded (25.119%); 1769/85575 objects misplaced (2.067%); 5.9 MiB/s, 196 objects/s recovering 2023-10-09T11:04:11.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:10 smithi125 ceph-mon[195507]: failed to encode map e467 with expected crc 2023-10-09T11:04:11.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:10 smithi125 ceph-mon[195507]: osdmap e467: 8 total, 7 up, 7 in 2023-10-09T11:04:11.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:10 smithi125 ceph-mon[195507]: failed to encode map e467 with expected crc 2023-10-09T11:04:11.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:10 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:11.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:10 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:11.219 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:04:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:04:11] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.43.0" 2023-10-09T11:04:11.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:10 smithi125 ceph-mon[200197]: failed to encode map e466 with expected crc 2023-10-09T11:04:11.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:10 smithi125 ceph-mon[200197]: pgmap v1155: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 783 op/s; 21496/85575 objects degraded (25.119%); 1769/85575 objects misplaced (2.067%); 5.9 MiB/s, 196 objects/s recovering 2023-10-09T11:04:11.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:10 smithi125 ceph-mon[200197]: failed to encode map e467 with expected crc 2023-10-09T11:04:11.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:10 smithi125 ceph-mon[200197]: osdmap e467: 8 total, 7 up, 7 in 2023-10-09T11:04:11.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:10 smithi125 ceph-mon[200197]: failed to encode map e467 with expected crc 2023-10-09T11:04:11.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:10 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:11.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:10 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:11.243 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:11.284 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:11.311 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:11.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:11.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:11.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:11.516 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:11.610 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:11.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:11.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:11.724 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:11.770 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:11.799 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:11.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:11.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:11.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:11.977 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:12.137 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:12.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:12.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:12.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:11 smithi155 ceph-mon[184854]: failed to encode map e467 with expected crc 2023-10-09T11:04:12.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:11 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:04:12.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:11 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:04:12.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:11 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:12.241 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:12.318 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:04:12.319 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:04:12.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:11 smithi125 ceph-mon[195507]: failed to encode map e467 with expected crc 2023-10-09T11:04:12.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:11 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:04:12.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:11 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:04:12.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:11 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:12.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:11 smithi125 ceph-mon[200197]: failed to encode map e467 with expected crc 2023-10-09T11:04:12.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:11 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:04:12.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:11 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:04:12.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:11 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:12.346 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:12.384 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:12.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:12.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:12.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:04:12.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:12.579 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:12.968 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:04:12 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[428685]: 2023-10-09T11:04:12.666+0000 7f4e5aa1b640 -1 osd.3 448 log_to_monitors true 2023-10-09T11:04:13.022 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:04:13.271 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:12 smithi125 ceph-mon[200197]: pgmap v1157: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 844 op/s; 19108/79554 objects degraded (24.019%); 1769/79554 objects misplaced (2.224%); 2.9 MiB/s, 196 objects/s recovering 2023-10-09T11:04:13.271 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:12 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2158845246' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:04:13.271 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:12 smithi125 ceph-mon[200197]: from='osd.3 [v2:172.21.15.125:6826/152767826,v1:172.21.15.125:6827/152767826]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T11:04:13.271 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:12 smithi125 ceph-mon[200197]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T11:04:13.271 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[428685]: 2023-10-09T11:04:13.053+0000 7f4e48e81700 -1 osd.3 448 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T11:04:13.272 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:12 smithi125 ceph-mon[195507]: pgmap v1157: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 844 op/s; 19108/79554 objects degraded (24.019%); 1769/79554 objects misplaced (2.224%); 2.9 MiB/s, 196 objects/s recovering 2023-10-09T11:04:13.272 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:12 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2158845246' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:04:13.272 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:12 smithi125 ceph-mon[195507]: from='osd.3 [v2:172.21.15.125:6826/152767826,v1:172.21.15.125:6827/152767826]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T11:04:13.273 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:12 smithi125 ceph-mon[195507]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T11:04:13.282 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:13.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:13.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:13.305 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T11:04:13.382 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:13.392 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:13.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:12 smithi155 ceph-mon[184854]: pgmap v1157: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 844 op/s; 19108/79554 objects degraded (24.019%); 1769/79554 objects misplaced (2.224%); 2.9 MiB/s, 196 objects/s recovering 2023-10-09T11:04:13.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:12 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2158845246' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:04:13.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:12 smithi155 ceph-mon[184854]: from='osd.3 [v2:172.21.15.125:6826/152767826,v1:172.21.15.125:6827/152767826]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T11:04:13.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:12 smithi155 ceph-mon[184854]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T11:04:13.487 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:13.735 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T11:04:13.931 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:13.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:13.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:14.076 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:04:14.076 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[195507]: failed to encode map e468 with expected crc 2023-10-09T11:04:14.076 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2158845246' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:04:14.076 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[195507]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T11:04:14.076 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[195507]: osdmap e468: 8 total, 7 up, 7 in 2023-10-09T11:04:14.076 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[195507]: failed to encode map e468 with expected crc 2023-10-09T11:04:14.076 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[195507]: from='osd.3 [v2:172.21.15.125:6826/152767826,v1:172.21.15.125:6827/152767826]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T11:04:14.076 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[195507]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T11:04:14.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 19108/79554 objects degraded (24.019%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:04:14.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:04:14.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[200197]: failed to encode map e468 with expected crc 2023-10-09T11:04:14.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2158845246' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:04:14.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[200197]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T11:04:14.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[200197]: osdmap e468: 8 total, 7 up, 7 in 2023-10-09T11:04:14.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[200197]: failed to encode map e468 with expected crc 2023-10-09T11:04:14.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[200197]: from='osd.3 [v2:172.21.15.125:6826/152767826,v1:172.21.15.125:6827/152767826]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T11:04:14.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[200197]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T11:04:14.078 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:13 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 19108/79554 objects degraded (24.019%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:04:14.084 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:14.147 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:14.170 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:14.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:14.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:14.279 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:14.385 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:14.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:13 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:04:14.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:13 smithi155 ceph-mon[184854]: failed to encode map e468 with expected crc 2023-10-09T11:04:14.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:13 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2158845246' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:04:14.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:13 smithi155 ceph-mon[184854]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T11:04:14.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:13 smithi155 ceph-mon[184854]: osdmap e468: 8 total, 7 up, 7 in 2023-10-09T11:04:14.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:13 smithi155 ceph-mon[184854]: failed to encode map e468 with expected crc 2023-10-09T11:04:14.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:13 smithi155 ceph-mon[184854]: from='osd.3 [v2:172.21.15.125:6826/152767826,v1:172.21.15.125:6827/152767826]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T11:04:14.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:13 smithi155 ceph-mon[184854]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T11:04:14.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:13 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 19108/79554 objects degraded (24.019%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:04:15.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:14 smithi125 ceph-mon[195507]: pgmap v1159: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 891 op/s; 18803/75084 objects degraded (25.043%); 1540/75084 objects misplaced (2.051%); 4.7 MiB/s, 225 objects/s recovering 2023-10-09T11:04:15.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:14 smithi125 ceph-mon[195507]: osd.3 [v2:172.21.15.125:6826/152767826,v1:172.21.15.125:6827/152767826] boot 2023-10-09T11:04:15.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:14 smithi125 ceph-mon[195507]: osdmap e469: 8 total, 8 up, 7 in 2023-10-09T11:04:15.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:14 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T11:04:15.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:14 smithi125 ceph-mon[195507]: failed to encode map e469 with expected crc 2023-10-09T11:04:15.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:14 smithi125 ceph-mon[195507]: failed to encode map e469 with expected crc 2023-10-09T11:04:15.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:14 smithi125 ceph-mon[200197]: pgmap v1159: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 891 op/s; 18803/75084 objects degraded (25.043%); 1540/75084 objects misplaced (2.051%); 4.7 MiB/s, 225 objects/s recovering 2023-10-09T11:04:15.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:14 smithi125 ceph-mon[200197]: osd.3 [v2:172.21.15.125:6826/152767826,v1:172.21.15.125:6827/152767826] boot 2023-10-09T11:04:15.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:14 smithi125 ceph-mon[200197]: osdmap e469: 8 total, 8 up, 7 in 2023-10-09T11:04:15.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:14 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T11:04:15.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:14 smithi125 ceph-mon[200197]: failed to encode map e469 with expected crc 2023-10-09T11:04:15.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:14 smithi125 ceph-mon[200197]: failed to encode map e469 with expected crc 2023-10-09T11:04:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:14 smithi155 ceph-mon[184854]: pgmap v1159: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 891 op/s; 18803/75084 objects degraded (25.043%); 1540/75084 objects misplaced (2.051%); 4.7 MiB/s, 225 objects/s recovering 2023-10-09T11:04:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:14 smithi155 ceph-mon[184854]: osd.3 [v2:172.21.15.125:6826/152767826,v1:172.21.15.125:6827/152767826] boot 2023-10-09T11:04:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:14 smithi155 ceph-mon[184854]: osdmap e469: 8 total, 8 up, 7 in 2023-10-09T11:04:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:14 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T11:04:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:14 smithi155 ceph-mon[184854]: failed to encode map e469 with expected crc 2023-10-09T11:04:15.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:14 smithi155 ceph-mon[184854]: failed to encode map e469 with expected crc 2023-10-09T11:04:15.474 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:15.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:15.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:15.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:15.605 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:15.624 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:15.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:15.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:15.728 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:15.754 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:15.773 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:15.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:15.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:15.781 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:15.876 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:15.877 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:15.912 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:15.985 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:15.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:15.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:16.010 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:15 smithi125 ceph-mon[195507]: failed to encode map e469 with expected crc 2023-10-09T11:04:16.010 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:04:16.010 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:15 smithi125 ceph-mon[195507]: osdmap e470: 8 total, 8 up, 7 in 2023-10-09T11:04:16.010 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:15 smithi125 ceph-mon[195507]: failed to encode map e470 with expected crc 2023-10-09T11:04:16.010 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:15 smithi125 ceph-mon[200197]: failed to encode map e469 with expected crc 2023-10-09T11:04:16.011 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:04:16.011 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:15 smithi125 ceph-mon[200197]: osdmap e470: 8 total, 8 up, 7 in 2023-10-09T11:04:16.011 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:15 smithi125 ceph-mon[200197]: failed to encode map e470 with expected crc 2023-10-09T11:04:16.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:16.116 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:16.134 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:16.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:16.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:16.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:16.275 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:16.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:15 smithi155 ceph-mon[184854]: failed to encode map e469 with expected crc 2023-10-09T11:04:16.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:04:16.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:15 smithi155 ceph-mon[184854]: osdmap e470: 8 total, 8 up, 7 in 2023-10-09T11:04:16.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:15 smithi155 ceph-mon[184854]: failed to encode map e470 with expected crc 2023-10-09T11:04:17.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:17 smithi125 ceph-mon[200197]: failed to encode map e470 with expected crc 2023-10-09T11:04:17.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:17 smithi125 ceph-mon[200197]: pgmap v1162: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 18803/75084 objects degraded (25.043%); 1540/75084 objects misplaced (2.051%); 2.9 MiB/s, 163 objects/s recovering 2023-10-09T11:04:17.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:17 smithi125 ceph-mon[200197]: osdmap e471: 8 total, 8 up, 7 in 2023-10-09T11:04:17.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:17 smithi125 ceph-mon[200197]: failed to encode map e471 with expected crc 2023-10-09T11:04:17.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:17 smithi125 ceph-mon[200197]: failed to encode map e471 with expected crc 2023-10-09T11:04:17.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:17 smithi125 ceph-mon[200197]: failed to encode map e471 with expected crc 2023-10-09T11:04:17.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:17 smithi125 ceph-mon[200197]: failed to encode map e471 with expected crc 2023-10-09T11:04:17.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:17 smithi125 ceph-mon[200197]: 10.7 continuing backfill to osd.2 from (406'100,469'8451] MIN to 469'8451 2023-10-09T11:04:17.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:17 smithi125 ceph-mon[200197]: failed to encode map e471 with expected crc 2023-10-09T11:04:17.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:17 smithi125 ceph-mon[200197]: 10.c continuing backfill to osd.7 from (432'5394,469'10913] 10:3045b39d:::benchmark_data_smithi125_402869_object35805:head to 469'10913 2023-10-09T11:04:17.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:17 smithi125 ceph-mon[200197]: 10.17 continuing backfill to osd.2 from (406'100,469'8492] MIN to 469'8492 2023-10-09T11:04:17.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:16 smithi125 ceph-mon[195507]: failed to encode map e470 with expected crc 2023-10-09T11:04:17.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:16 smithi125 ceph-mon[195507]: pgmap v1162: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 18803/75084 objects degraded (25.043%); 1540/75084 objects misplaced (2.051%); 2.9 MiB/s, 163 objects/s recovering 2023-10-09T11:04:17.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:16 smithi125 ceph-mon[195507]: osdmap e471: 8 total, 8 up, 7 in 2023-10-09T11:04:17.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:16 smithi125 ceph-mon[195507]: failed to encode map e471 with expected crc 2023-10-09T11:04:17.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:16 smithi125 ceph-mon[195507]: failed to encode map e471 with expected crc 2023-10-09T11:04:17.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:16 smithi125 ceph-mon[195507]: failed to encode map e471 with expected crc 2023-10-09T11:04:17.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:16 smithi125 ceph-mon[195507]: failed to encode map e471 with expected crc 2023-10-09T11:04:17.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:16 smithi125 ceph-mon[195507]: 10.7 continuing backfill to osd.2 from (406'100,469'8451] MIN to 469'8451 2023-10-09T11:04:17.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:16 smithi125 ceph-mon[195507]: failed to encode map e471 with expected crc 2023-10-09T11:04:17.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:16 smithi125 ceph-mon[195507]: 10.c continuing backfill to osd.7 from (432'5394,469'10913] 10:3045b39d:::benchmark_data_smithi125_402869_object35805:head to 469'10913 2023-10-09T11:04:17.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:16 smithi125 ceph-mon[195507]: 10.17 continuing backfill to osd.2 from (406'100,469'8492] MIN to 469'8492 2023-10-09T11:04:17.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:16 smithi155 ceph-mon[184854]: failed to encode map e470 with expected crc 2023-10-09T11:04:17.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:17 smithi155 ceph-mon[184854]: pgmap v1162: 129 pgs: 1 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 18803/75084 objects degraded (25.043%); 1540/75084 objects misplaced (2.051%); 2.9 MiB/s, 163 objects/s recovering 2023-10-09T11:04:17.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:17 smithi155 ceph-mon[184854]: osdmap e471: 8 total, 8 up, 7 in 2023-10-09T11:04:17.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:17 smithi155 ceph-mon[184854]: failed to encode map e471 with expected crc 2023-10-09T11:04:17.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:17 smithi155 ceph-mon[184854]: failed to encode map e471 with expected crc 2023-10-09T11:04:17.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:17 smithi155 ceph-mon[184854]: failed to encode map e471 with expected crc 2023-10-09T11:04:17.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:17 smithi155 ceph-mon[184854]: failed to encode map e471 with expected crc 2023-10-09T11:04:17.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:17 smithi155 ceph-mon[184854]: 10.7 continuing backfill to osd.2 from (406'100,469'8451] MIN to 469'8451 2023-10-09T11:04:17.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:17 smithi155 ceph-mon[184854]: failed to encode map e471 with expected crc 2023-10-09T11:04:17.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:17 smithi155 ceph-mon[184854]: 10.c continuing backfill to osd.7 from (432'5394,469'10913] 10:3045b39d:::benchmark_data_smithi125_402869_object35805:head to 469'10913 2023-10-09T11:04:17.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:17 smithi155 ceph-mon[184854]: 10.17 continuing backfill to osd.2 from (406'100,469'8492] MIN to 469'8492 2023-10-09T11:04:17.629 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:04:18.018 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:18 smithi125 ceph-mon[195507]: failed to encode map e471 with expected crc 2023-10-09T11:04:18.018 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:18 smithi125 ceph-mon[195507]: failed to encode map e471 with expected crc 2023-10-09T11:04:18.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:18 smithi125 ceph-mon[195507]: 10.f continuing backfill to osd.4 from (438'5821,469'11011] MIN to 469'11011 2023-10-09T11:04:18.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:18 smithi125 ceph-mon[200197]: failed to encode map e471 with expected crc 2023-10-09T11:04:18.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:18 smithi125 ceph-mon[200197]: failed to encode map e471 with expected crc 2023-10-09T11:04:18.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:18 smithi125 ceph-mon[200197]: 10.f continuing backfill to osd.4 from (438'5821,469'11011] MIN to 469'11011 2023-10-09T11:04:18.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:18.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:18.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:18.457 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:18 smithi155 ceph-mon[184854]: failed to encode map e471 with expected crc 2023-10-09T11:04:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:18 smithi155 ceph-mon[184854]: failed to encode map e471 with expected crc 2023-10-09T11:04:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:18 smithi155 ceph-mon[184854]: 10.f continuing backfill to osd.4 from (438'5821,469'11011] MIN to 469'11011 2023-10-09T11:04:18.499 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:18.533 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:18.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:18.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:18.639 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:18.689 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:18.727 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:18.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:18.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:18.834 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:18.879 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:18.914 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:18.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:18.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:19.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:19.078 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:19.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:19.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:19.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:19.131 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:19 smithi125 ceph-mon[200197]: pgmap v1164: 129 pgs: 1 active+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 116 active+clean; 1.4 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 17612/69309 objects degraded (25.411%); 1874/69309 objects misplaced (2.704%); 6.0 MiB/s, 166 objects/s recovering 2023-10-09T11:04:19.131 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:19 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 17612/69309 objects degraded (25.411%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:19.133 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:19 smithi125 ceph-mon[195507]: pgmap v1164: 129 pgs: 1 active+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 116 active+clean; 1.4 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 17612/69309 objects degraded (25.411%); 1874/69309 objects misplaced (2.704%); 6.0 MiB/s, 166 objects/s recovering 2023-10-09T11:04:19.133 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:19 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 17612/69309 objects degraded (25.411%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:19.209 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:19.219 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:19 smithi155 ceph-mon[184854]: pgmap v1164: 129 pgs: 1 active+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 116 active+clean; 1.4 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 17612/69309 objects degraded (25.411%); 1874/69309 objects misplaced (2.704%); 6.0 MiB/s, 166 objects/s recovering 2023-10-09T11:04:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:19 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 17612/69309 objects degraded (25.411%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:19.266 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:19.408 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:19.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:19.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:19.525 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:19.566 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:19.587 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:04:19.587 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:04:19.592 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:19.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:19.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:19.622 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:04:19.679 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:04:19.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:19.721 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:04:19.741 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:19.757 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:04:19.769 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:19.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:19.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:19.801 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:04:19.850 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:04:19.875 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:19.906 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:04:19.943 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:04:19.957 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:19.958 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:04:19.958 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:04:19.958 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:04:19.958 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:04:19.958 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:04:19.958 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:04:19.958 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:04:19.958 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:04:19.960 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:19.974 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:04:19.974 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:04:19.974 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:19.974 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:177484 10.c e471/470 10:35a6f9d1:::benchmark_data_smithi125_402869_object78915:head v 471'11099, mlcod=448'7125)", 2023-10-09T11:04:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:04:19.968214+0000", 2023-10-09T11:04:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0063207879999999999, 2023-10-09T11:04:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0063579919999999998, 2023-10-09T11:04:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:04:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:19.968214+0000", 2023-10-09T11:04:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:19.968214+0000", 2023-10-09T11:04:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:19.968212+0000", 2023-10-09T11:04:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:04:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:19.968226+0000", 2023-10-09T11:04:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3161e-05 2023-10-09T11:04:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:19.968244+0000", 2023-10-09T11:04:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8856999999999999e-05 2023-10-09T11:04:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:19.968252+0000", 2023-10-09T11:04:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5209999999999997e-06 2023-10-09T11:04:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:19.972123+0000", 2023-10-09T11:04:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038708789999999998 2023-10-09T11:04:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:19.972151+0000", 2023-10-09T11:04:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8600000000000001e-05 2023-10-09T11:04:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:19.979 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:19.979 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T11:04:19.979 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 1 2023-10-09T11:04:19.979 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:04:20.014 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:20.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:20.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:20.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:20.193 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:20.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:20 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1097074724' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:20.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:20 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1097074724' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:20.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:20 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1097074724' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:20.531 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:20.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:20.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:20.646 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:20.703 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:21.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:21 smithi125 ceph-mon[195507]: pgmap v1165: 129 pgs: 1 active+recovery_wait+degraded, 1 active+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 116 active+clean; 1.3 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 17514/65562 objects degraded (26.714%); 1685/65562 objects misplaced (2.570%); 4.7 MiB/s, 181 objects/s recovering 2023-10-09T11:04:21.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:21 smithi125 ceph-mon[200197]: pgmap v1165: 129 pgs: 1 active+recovery_wait+degraded, 1 active+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 116 active+clean; 1.3 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 17514/65562 objects degraded (26.714%); 1685/65562 objects misplaced (2.570%); 4.7 MiB/s, 181 objects/s recovering 2023-10-09T11:04:21.322 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:04:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:04:21] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.43.0" 2023-10-09T11:04:21.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:21 smithi155 ceph-mon[184854]: pgmap v1165: 129 pgs: 1 active+recovery_wait+degraded, 1 active+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 116 active+clean; 1.3 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 17514/65562 objects degraded (26.714%); 1685/65562 objects misplaced (2.570%); 4.7 MiB/s, 181 objects/s recovering 2023-10-09T11:04:21.655 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:21.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:21.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:21.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T11:04:21.765 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:21.807 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:21.841 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:21.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:21.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:21.948 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:22.038 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:23.259 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:23.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:23 smithi125 ceph-mon[195507]: pgmap v1166: 129 pgs: 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 116 active+clean; 1.2 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 16735/58161 objects degraded (28.774%); 1361/58161 objects misplaced (2.340%); 6.5 MiB/s, 188 objects/s recovering 2023-10-09T11:04:23.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:23 smithi125 ceph-mon[195507]: osdmap e472: 8 total, 8 up, 7 in 2023-10-09T11:04:23.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:23 smithi125 ceph-mon[195507]: failed to encode map e472 with expected crc 2023-10-09T11:04:23.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:23 smithi125 ceph-mon[195507]: failed to encode map e472 with expected crc 2023-10-09T11:04:23.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:23 smithi125 ceph-mon[195507]: failed to encode map e472 with expected crc 2023-10-09T11:04:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:23 smithi125 ceph-mon[200197]: pgmap v1166: 129 pgs: 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 116 active+clean; 1.2 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 16735/58161 objects degraded (28.774%); 1361/58161 objects misplaced (2.340%); 6.5 MiB/s, 188 objects/s recovering 2023-10-09T11:04:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:23 smithi125 ceph-mon[200197]: osdmap e472: 8 total, 8 up, 7 in 2023-10-09T11:04:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:23 smithi125 ceph-mon[200197]: failed to encode map e472 with expected crc 2023-10-09T11:04:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:23 smithi125 ceph-mon[200197]: failed to encode map e472 with expected crc 2023-10-09T11:04:23.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:23 smithi125 ceph-mon[200197]: failed to encode map e472 with expected crc 2023-10-09T11:04:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:23 smithi155 ceph-mon[184854]: pgmap v1166: 129 pgs: 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 116 active+clean; 1.2 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 16735/58161 objects degraded (28.774%); 1361/58161 objects misplaced (2.340%); 6.5 MiB/s, 188 objects/s recovering 2023-10-09T11:04:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:23 smithi155 ceph-mon[184854]: osdmap e472: 8 total, 8 up, 7 in 2023-10-09T11:04:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:23 smithi155 ceph-mon[184854]: failed to encode map e472 with expected crc 2023-10-09T11:04:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:23 smithi155 ceph-mon[184854]: failed to encode map e472 with expected crc 2023-10-09T11:04:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:23 smithi155 ceph-mon[184854]: failed to encode map e472 with expected crc 2023-10-09T11:04:23.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:23.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:23.733 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:23.798 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:24.183 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:24.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:24.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:24.328 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:24.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:24 smithi125 ceph-mon[195507]: failed to encode map e473 with expected crc 2023-10-09T11:04:24.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:24 smithi125 ceph-mon[195507]: osdmap e473: 8 total, 8 up, 7 in 2023-10-09T11:04:24.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:24 smithi125 ceph-mon[195507]: failed to encode map e473 with expected crc 2023-10-09T11:04:24.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:24 smithi125 ceph-mon[195507]: failed to encode map e473 with expected crc 2023-10-09T11:04:24.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:24 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 16735/58161 objects degraded (28.774%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:24.396 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:24 smithi125 ceph-mon[200197]: failed to encode map e473 with expected crc 2023-10-09T11:04:24.396 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:24 smithi125 ceph-mon[200197]: osdmap e473: 8 total, 8 up, 7 in 2023-10-09T11:04:24.396 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:24 smithi125 ceph-mon[200197]: failed to encode map e473 with expected crc 2023-10-09T11:04:24.397 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:24 smithi125 ceph-mon[200197]: failed to encode map e473 with expected crc 2023-10-09T11:04:24.397 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:24 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 16735/58161 objects degraded (28.774%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:24.397 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:24 smithi155 ceph-mon[184854]: failed to encode map e473 with expected crc 2023-10-09T11:04:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:24 smithi155 ceph-mon[184854]: osdmap e473: 8 total, 8 up, 7 in 2023-10-09T11:04:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:24 smithi155 ceph-mon[184854]: failed to encode map e473 with expected crc 2023-10-09T11:04:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:24 smithi155 ceph-mon[184854]: failed to encode map e473 with expected crc 2023-10-09T11:04:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:24 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 16735/58161 objects degraded (28.774%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:24.930 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:24.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:24.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:25.036 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:25.053 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:25.074 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:25.109 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:25.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:25.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:25.215 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:25 smithi125 ceph-mon[195507]: pgmap v1169: 129 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 1.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 879 op/s; 14628/54075 objects degraded (27.051%); 1517/54075 objects misplaced (2.805%); 5.6 MiB/s, 205 objects/s recovering 2023-10-09T11:04:25.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:25 smithi125 ceph-mon[200197]: pgmap v1169: 129 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 1.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 879 op/s; 14628/54075 objects degraded (27.051%); 1517/54075 objects misplaced (2.805%); 5.6 MiB/s, 205 objects/s recovering 2023-10-09T11:04:25.216 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:25.249 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:25.278 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:25.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:25.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:25.384 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:25.449 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:25 smithi155 ceph-mon[184854]: pgmap v1169: 129 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 1.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 879 op/s; 14628/54075 objects degraded (27.051%); 1517/54075 objects misplaced (2.805%); 5.6 MiB/s, 205 objects/s recovering 2023-10-09T11:04:25.593 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:25.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:25.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:25.701 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:25.754 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:25.788 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:25.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:25.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:25.895 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:25.942 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:25.971 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:25.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:25.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:26.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:26.161 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:26.185 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:26 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2912787824' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:26.188 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:04:26.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:26 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2912787824' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:26.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:26 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2912787824' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:26.942 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:04:27.054 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:27.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:27.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:27.166 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:27.223 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:27.428 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:27 smithi125 ceph-mon[195507]: pgmap v1170: 129 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 1.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 14628/54075 objects degraded (27.051%); 1517/54075 objects misplaced (2.805%); 5.2 MiB/s, 192 objects/s recovering 2023-10-09T11:04:27.428 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:27 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:04:27.428 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:27 smithi125 ceph-mon[195507]: failed to encode map e474 with expected crc 2023-10-09T11:04:27.428 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:27 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2912787824' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:04:27.429 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:27 smithi125 ceph-mon[195507]: osdmap e474: 8 total, 8 up, 7 in 2023-10-09T11:04:27.429 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:27 smithi125 ceph-mon[195507]: failed to encode map e474 with expected crc 2023-10-09T11:04:27.429 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:27 smithi125 ceph-mon[200197]: pgmap v1170: 129 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 1.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 14628/54075 objects degraded (27.051%); 1517/54075 objects misplaced (2.805%); 5.2 MiB/s, 192 objects/s recovering 2023-10-09T11:04:27.429 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:27 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:04:27.429 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:27 smithi125 ceph-mon[200197]: failed to encode map e474 with expected crc 2023-10-09T11:04:27.429 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:27 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2912787824' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:04:27.429 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:27 smithi125 ceph-mon[200197]: osdmap e474: 8 total, 8 up, 7 in 2023-10-09T11:04:27.429 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:27 smithi125 ceph-mon[200197]: failed to encode map e474 with expected crc 2023-10-09T11:04:27.430 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:27.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:27.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:27.448 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:04:27.448 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:04:27.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.448 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:183015 10.3 10:c3655cd3:::benchmark_data_smithi125_402869_object84446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:04:27.448 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:04:27.427771+0000", 2023-10-09T11:04:27.449 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.011832171000000001, 2023-10-09T11:04:27.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011859936999999999, 2023-10-09T11:04:27.449 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:27.449 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:04:27.449 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:27.449 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:04:27.449 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:27.449 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 183015 2023-10-09T11:04:27.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.449 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:27.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:27.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.427771+0000", 2023-10-09T11:04:27.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:27.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:27.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.427771+0000", 2023-10-09T11:04:27.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:27.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:27.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.427769+0000", 2023-10-09T11:04:27.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:04:27.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:27.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.427778+0000", 2023-10-09T11:04:27.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6729999999999998e-06 2023-10-09T11:04:27.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:27.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.427791+0000", 2023-10-09T11:04:27.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2860999999999999e-05 2023-10-09T11:04:27.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:27.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.427804+0000", 2023-10-09T11:04:27.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3271000000000001e-05 2023-10-09T11:04:27.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:27.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.427853+0000", 2023-10-09T11:04:27.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8829000000000003e-05 2023-10-09T11:04:27.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:27.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.428039+0000", 2023-10-09T11:04:27.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018546900000000001 2023-10-09T11:04:27.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,5", 2023-10-09T11:04:27.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.428670+0000", 2023-10-09T11:04:27.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00063166600000000002 2023-10-09T11:04:27.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:27.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.434787+0000", 2023-10-09T11:04:27.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0061167260000000003 2023-10-09T11:04:27.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:27.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.435898+0000", 2023-10-09T11:04:27.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001110946 2023-10-09T11:04:27.454 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:27.454 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:27.454 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:27.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.454 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:183024 10.19 e474/466)", 2023-10-09T11:04:27.454 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:04:27.437125+0000", 2023-10-09T11:04:27.454 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0024789439999999999, 2023-10-09T11:04:27.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002591524, 2023-10-09T11:04:27.455 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:27.455 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T11:04:27.455 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:27.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:27.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.437125+0000", 2023-10-09T11:04:27.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:27.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:27.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.437125+0000", 2023-10-09T11:04:27.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:27.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:27.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.437124+0000", 2023-10-09T11:04:27.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:27.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:27.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.437133+0000", 2023-10-09T11:04:27.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2930000000000006e-06 2023-10-09T11:04:27.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:27.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.437149+0000", 2023-10-09T11:04:27.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5651e-05 2023-10-09T11:04:27.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:27.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.437156+0000", 2023-10-09T11:04:27.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3320000000000003e-06 2023-10-09T11:04:27.457 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:27.457 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:27.457 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:27.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.458 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36123.0:183027 10.9 e474/451)", 2023-10-09T11:04:27.458 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:04:27.437173+0000", 2023-10-09T11:04:27.458 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0024303459999999999, 2023-10-09T11:04:27.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002598094, 2023-10-09T11:04:27.458 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:27.458 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T11:04:27.458 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:27.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:27.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.437173+0000", 2023-10-09T11:04:27.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:27.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:27.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.437173+0000", 2023-10-09T11:04:27.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:27.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:27.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.437173+0000", 2023-10-09T11:04:27.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:27.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:27.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.437177+0000", 2023-10-09T11:04:27.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1949999999999996e-06 2023-10-09T11:04:27.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:27.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.437186+0000", 2023-10-09T11:04:27.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2329999999999995e-06 2023-10-09T11:04:27.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:27.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:27.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:27.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:27.437191+0000", 2023-10-09T11:04:27.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2209999999999999e-06 2023-10-09T11:04:27.460 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:27.460 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:27.460 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:27.461 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:27.461 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T11:04:27.461 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 3 2023-10-09T11:04:27.461 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:04:27.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:27 smithi155 ceph-mon[184854]: pgmap v1170: 129 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 1.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 14628/54075 objects degraded (27.051%); 1517/54075 objects misplaced (2.805%); 5.2 MiB/s, 192 objects/s recovering 2023-10-09T11:04:27.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:27 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:04:27.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:27 smithi155 ceph-mon[184854]: failed to encode map e474 with expected crc 2023-10-09T11:04:27.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:27 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2912787824' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:04:27.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:27 smithi155 ceph-mon[184854]: osdmap e474: 8 total, 8 up, 7 in 2023-10-09T11:04:27.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:27 smithi155 ceph-mon[184854]: failed to encode map e474 with expected crc 2023-10-09T11:04:27.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:27.572 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:27.657 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:27.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:27.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:27.761 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:27.814 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:27.870 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:27.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:27.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:27.884 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:27.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T11:04:27.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:28.036 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:28.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:28.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:28.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:28.199 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:28.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:28 smithi125 ceph-mon[195507]: pgmap v1172: 129 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 991 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 13964/47613 objects degraded (29.328%); 1507/47613 objects misplaced (3.165%); 2.7 MiB/s, 150 objects/s recovering 2023-10-09T11:04:28.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:28 smithi125 ceph-mon[200197]: pgmap v1172: 129 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 991 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 13964/47613 objects degraded (29.328%); 1507/47613 objects misplaced (3.165%); 2.7 MiB/s, 150 objects/s recovering 2023-10-09T11:04:28.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:28 smithi155 ceph-mon[184854]: pgmap v1172: 129 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 991 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 13964/47613 objects degraded (29.328%); 1507/47613 objects misplaced (3.165%); 2.7 MiB/s, 150 objects/s recovering 2023-10-09T11:04:28.238 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:28.272 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:28.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:28.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:28.380 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:28.459 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:28.648 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:28.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:28.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:28.674 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 6, 2, 1, 0, 7] out_osds: [3] dead_osds: [] live_osds: [0, 2, 4, 5, 6, 7, 1, 3] 2023-10-09T11:04:28.674 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:04:28.674 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [4, 5, 6, 2, 1, 0, 7] 2023-10-09T11:04:28.674 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-09T11:04:28.761 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:28.802 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:28.829 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:28.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:28.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:28.934 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:28.977 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:29.004 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:29.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:29.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:29.108 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:29.159 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:29.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:29 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 13964/47613 objects degraded (29.328%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:29.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:29 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 13964/47613 objects degraded (29.328%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:29.449 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:04:29.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:29 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 13964/47613 objects degraded (29.328%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:29.898 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:29.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:29.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:29.935 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:30.006 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:30.058 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:30.088 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:30.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:30.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:30.200 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:30.248 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:30.271 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:30.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:30.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:30.376 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:30.413 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:30 smithi155 ceph-mon[184854]: pgmap v1173: 129 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 900 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 13846/43245 objects degraded (32.018%); 1007/43245 objects misplaced (2.329%); 5.1 MiB/s, 165 objects/s recovering 2023-10-09T11:04:30.413 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:04:30.419 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:30.461 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:30.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:30.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:30.572 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:30.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:30 smithi125 ceph-mon[195507]: pgmap v1173: 129 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 900 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 13846/43245 objects degraded (32.018%); 1007/43245 objects misplaced (2.329%); 5.1 MiB/s, 165 objects/s recovering 2023-10-09T11:04:30.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:04:30.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:30 smithi125 ceph-mon[200197]: pgmap v1173: 129 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 900 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 13846/43245 objects degraded (32.018%); 1007/43245 objects misplaced (2.329%); 5.1 MiB/s, 165 objects/s recovering 2023-10-09T11:04:30.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:04:30.661 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:30.761 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:30.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:30.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:30.860 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:04:30.920 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:30.931 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:04:30.957 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:04:30.961 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:30.998 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:04:31.000 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:31.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:31.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:31.042 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:04:31.078 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:04:31.107 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:31.112 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:04:31.154 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:31.156 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:04:31.168 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:04:31.168 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:04:31.169 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:04:31.169 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:04:31.169 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:04:31.169 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:04:31.169 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:04:31.169 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:04:31.181 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:31.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:31.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:31.284 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/45644716' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:31.284 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:04:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:04:31] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-10-09T11:04:31.285 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:31.329 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:31.354 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:31.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:31.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:31.460 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:31.504 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:31.534 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:31.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:31.543 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/45644716' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:31.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:31.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/45644716' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:31.644 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:31.694 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:32.018 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:32.019 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:32.040 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:32.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:32.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:32.145 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:32.179 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:32.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:32.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:32.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:32.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:32.374 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:32.401 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:32.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:32.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:32 smithi155 ceph-mon[184854]: pgmap v1174: 129 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 750 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 846 op/s; 13128/36069 objects degraded (36.397%); 1007/36069 objects misplaced (2.792%); 4.5 MiB/s, 191 objects/s recovering 2023-10-09T11:04:32.506 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:32.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:32 smithi125 ceph-mon[195507]: pgmap v1174: 129 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 750 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 846 op/s; 13128/36069 objects degraded (36.397%); 1007/36069 objects misplaced (2.792%); 4.5 MiB/s, 191 objects/s recovering 2023-10-09T11:04:32.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:32 smithi125 ceph-mon[200197]: pgmap v1174: 129 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 750 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 846 op/s; 13128/36069 objects degraded (36.397%); 1007/36069 objects misplaced (2.792%); 4.5 MiB/s, 191 objects/s recovering 2023-10-09T11:04:32.594 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:32.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:32.706 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:32.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:32.720 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:04:32.721 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:04:32.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:32.873 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:33.327 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:33.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:33.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:33.433 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:33.518 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:33 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 13128/36069 objects degraded (36.397%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:33.520 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:33 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 13128/36069 objects degraded (36.397%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:33.526 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:33 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 13128/36069 objects degraded (36.397%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:34.177 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:34.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:34.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:34.287 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:34.315 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:34.333 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:34.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:34.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:34.437 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:34.485 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:34.491 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:04:34.518 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:34.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:34.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:34.543 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:34 smithi125 ceph-mon[195507]: pgmap v1175: 129 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 664 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 13034/31941 objects degraded (40.806%); 626/31941 objects misplaced (1.960%); 3.9 MiB/s, 151 objects/s recovering 2023-10-09T11:04:34.543 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:34 smithi125 ceph-mon[200197]: pgmap v1175: 129 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 664 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 13034/31941 objects degraded (40.806%); 626/31941 objects misplaced (1.960%); 3.9 MiB/s, 151 objects/s recovering 2023-10-09T11:04:34.623 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:34.659 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:34 smithi155 ceph-mon[184854]: pgmap v1175: 129 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 664 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 13034/31941 objects degraded (40.806%); 626/31941 objects misplaced (1.960%); 3.9 MiB/s, 151 objects/s recovering 2023-10-09T11:04:34.662 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:34.690 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:34.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:34.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:34.795 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:34.911 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:35.049 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:35.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:35.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:35.076 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T11:04:35.156 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:35.219 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:35.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:35.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:35.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:35.463 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:35.523 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:35.552 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:35.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:35.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:35.593 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:35.659 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:35.760 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:35.815 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:35.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:35.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:35.930 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:36.022 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:36.218 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:36.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:36.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:36.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:36 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T11:04:36.053+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e474 definitely_dead 0 2023-10-09T11:04:36.327 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:36.380 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:36.678 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:36.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:36.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:36.797 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:36.853 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:36.890 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:36.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:36.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:36.908 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.6. 2023-10-09T11:04:36.912 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:04:36.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:36 smithi125 ceph-mon[195507]: pgmap v1176: 129 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 664 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 13034/31941 objects degraded (40.806%); 626/31941 objects misplaced (1.960%); 3.9 MiB/s, 151 objects/s recovering 2023-10-09T11:04:36.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:36 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1533668242' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T11:04:36.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:36 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T11:04:36.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:36 smithi125 ceph-mon[195507]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T11:04:36.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:36 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1892550146' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:04:36.937 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:36 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:04:36.937 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:36 smithi125 ceph-mon[200197]: pgmap v1176: 129 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 664 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 13034/31941 objects degraded (40.806%); 626/31941 objects misplaced (1.960%); 3.9 MiB/s, 151 objects/s recovering 2023-10-09T11:04:36.938 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:36 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1533668242' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T11:04:36.938 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:36 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T11:04:36.938 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:36 smithi125 ceph-mon[200197]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T11:04:36.938 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:36 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1892550146' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:04:36.938 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:36 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:04:36.994 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:37.022 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:37.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:37.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:37.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:37.147 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:37.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:36 smithi155 ceph-mon[184854]: pgmap v1176: 129 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 664 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 13034/31941 objects degraded (40.806%); 626/31941 objects misplaced (1.960%); 3.9 MiB/s, 151 objects/s recovering 2023-10-09T11:04:37.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:36 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1533668242' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T11:04:37.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:36 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T11:04:37.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:36 smithi155 ceph-mon[184854]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T11:04:37.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:36 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1892550146' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:04:37.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:36 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:04:37.174 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:37.193 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:37.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:37.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:37.296 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:37.328 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:37.954 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:37.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:37.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:38.059 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:38.088 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:37 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:04:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:37 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T11:04:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:37 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:04:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:37 smithi155 ceph-mon[184854]: osdmap e475: 8 total, 8 up, 6 in 2023-10-09T11:04:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:37 smithi155 ceph-mon[184854]: failed to encode map e475 with expected crc 2023-10-09T11:04:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:37 smithi155 ceph-mon[184854]: failed to encode map e475 with expected crc 2023-10-09T11:04:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:37 smithi155 ceph-mon[184854]: failed to encode map e475 with expected crc 2023-10-09T11:04:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:37 smithi155 ceph-mon[184854]: failed to encode map e475 with expected crc 2023-10-09T11:04:38.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:37 smithi155 ceph-mon[184854]: 10.6 scrub starts 2023-10-09T11:04:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:04:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T11:04:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:04:38.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[195507]: osdmap e475: 8 total, 8 up, 6 in 2023-10-09T11:04:38.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[195507]: failed to encode map e475 with expected crc 2023-10-09T11:04:38.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[195507]: failed to encode map e475 with expected crc 2023-10-09T11:04:38.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[195507]: failed to encode map e475 with expected crc 2023-10-09T11:04:38.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[195507]: failed to encode map e475 with expected crc 2023-10-09T11:04:38.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[195507]: 10.6 scrub starts 2023-10-09T11:04:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:04:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T11:04:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:04:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[200197]: osdmap e475: 8 total, 8 up, 6 in 2023-10-09T11:04:38.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[200197]: failed to encode map e475 with expected crc 2023-10-09T11:04:38.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[200197]: failed to encode map e475 with expected crc 2023-10-09T11:04:38.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[200197]: failed to encode map e475 with expected crc 2023-10-09T11:04:38.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[200197]: failed to encode map e475 with expected crc 2023-10-09T11:04:38.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:37 smithi125 ceph-mon[200197]: 10.6 scrub starts 2023-10-09T11:04:39.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: failed to encode map e475 with expected crc 2023-10-09T11:04:39.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: failed to encode map e475 with expected crc 2023-10-09T11:04:39.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: pgmap v1178: 129 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 510 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 768 op/s; 12321/24531 objects degraded (50.226%); 626/24531 objects misplaced (2.552%); 3.9 MiB/s, 147 objects/s recovering 2023-10-09T11:04:39.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: osdmap e476: 8 total, 8 up, 6 in 2023-10-09T11:04:39.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: 10.7 starting backfill to osd.2 from (406'100,474'8964] MIN to 474'8965 2023-10-09T11:04:39.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: 10.17 continuing backfill to osd.2 from (406'100,474'8957] MIN to 474'8957 2023-10-09T11:04:39.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: 10.17 starting backfill to osd.4 from (0'0,0'0] MAX to 474'8957 2023-10-09T11:04:39.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: 10.f starting backfill to osd.1 from (0'0,0'0] MAX to 474'11984 2023-10-09T11:04:39.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: 10.f continuing backfill to osd.4 from (446'6721,474'11984] MIN to 474'11984 2023-10-09T11:04:39.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: 10.7 starting backfill to osd.4 from (0'0,0'0] MAX to 474'8965 2023-10-09T11:04:39.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 12321/24531 objects degraded (50.226%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:39.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: failed to encode map e475 with expected crc 2023-10-09T11:04:39.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: failed to encode map e475 with expected crc 2023-10-09T11:04:39.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: pgmap v1178: 129 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 510 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 768 op/s; 12321/24531 objects degraded (50.226%); 626/24531 objects misplaced (2.552%); 3.9 MiB/s, 147 objects/s recovering 2023-10-09T11:04:39.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: osdmap e476: 8 total, 8 up, 6 in 2023-10-09T11:04:39.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: 10.7 starting backfill to osd.2 from (406'100,474'8964] MIN to 474'8965 2023-10-09T11:04:39.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: 10.17 continuing backfill to osd.2 from (406'100,474'8957] MIN to 474'8957 2023-10-09T11:04:39.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: 10.17 starting backfill to osd.4 from (0'0,0'0] MAX to 474'8957 2023-10-09T11:04:39.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: 10.f starting backfill to osd.1 from (0'0,0'0] MAX to 474'11984 2023-10-09T11:04:39.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: 10.f continuing backfill to osd.4 from (446'6721,474'11984] MIN to 474'11984 2023-10-09T11:04:39.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: 10.7 starting backfill to osd.4 from (0'0,0'0] MAX to 474'8965 2023-10-09T11:04:39.074 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:38 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 12321/24531 objects degraded (50.226%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: failed to encode map e475 with expected crc 2023-10-09T11:04:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: failed to encode map e475 with expected crc 2023-10-09T11:04:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: pgmap v1178: 129 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 117 active+clean; 510 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 768 op/s; 12321/24531 objects degraded (50.226%); 626/24531 objects misplaced (2.552%); 3.9 MiB/s, 147 objects/s recovering 2023-10-09T11:04:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: osdmap e476: 8 total, 8 up, 6 in 2023-10-09T11:04:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: 10.7 starting backfill to osd.2 from (406'100,474'8964] MIN to 474'8965 2023-10-09T11:04:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: 10.17 continuing backfill to osd.2 from (406'100,474'8957] MIN to 474'8957 2023-10-09T11:04:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: 10.17 starting backfill to osd.4 from (0'0,0'0] MAX to 474'8957 2023-10-09T11:04:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: 10.f starting backfill to osd.1 from (0'0,0'0] MAX to 474'11984 2023-10-09T11:04:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: 10.f continuing backfill to osd.4 from (446'6721,474'11984] MIN to 474'11984 2023-10-09T11:04:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: 10.7 starting backfill to osd.4 from (0'0,0'0] MAX to 474'8965 2023-10-09T11:04:39.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:38 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 12321/24531 objects degraded (50.226%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:39.265 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:39.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:39.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:39.274 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:39.374 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:39.410 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:39.484 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:39.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:39.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:39.593 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:39.659 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:39.690 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:39.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:39.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:39.795 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:39.827 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:04:39.845 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:39.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:39.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:39.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:39.882 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:04:39.933 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:04:39.968 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:04:39.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:39 smithi155 ceph-mon[184854]: failed to encode map e476 with expected crc 2023-10-09T11:04:39.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:39 smithi155 ceph-mon[184854]: 10.10 starting backfill to osd.7 from (0'0,0'0] MAX to 474'8914 2023-10-09T11:04:39.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:39 smithi155 ceph-mon[184854]: osdmap e477: 8 total, 8 up, 6 in 2023-10-09T11:04:39.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:39 smithi155 ceph-mon[184854]: failed to encode map e477 with expected crc 2023-10-09T11:04:39.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:39 smithi155 ceph-mon[184854]: 10.b starting backfill to osd.4 from (0'0,0'0] MAX to 474'11937 2023-10-09T11:04:39.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:39 smithi155 ceph-mon[184854]: failed to encode map e477 with expected crc 2023-10-09T11:04:39.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:39 smithi155 ceph-mon[184854]: failed to encode map e477 with expected crc 2023-10-09T11:04:39.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:39 smithi155 ceph-mon[184854]: failed to encode map e477 with expected crc 2023-10-09T11:04:39.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:39 smithi155 ceph-mon[184854]: 10.0 starting backfill to osd.7 from (0'0,0'0] MAX to 474'8824 2023-10-09T11:04:39.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:39 smithi155 ceph-mon[184854]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 474'11859 2023-10-09T11:04:39.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:39 smithi155 ceph-mon[184854]: failed to encode map e477 with expected crc 2023-10-09T11:04:39.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:39 smithi155 ceph-mon[184854]: 10.d starting backfill to osd.7 from (0'0,0'0] MAX to 474'11815 2023-10-09T11:04:39.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:39 smithi155 ceph-mon[184854]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 474'11852 2023-10-09T11:04:39.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:39 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/373489251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:39.980 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:40.006 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:04:40.044 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:04:40.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[195507]: failed to encode map e476 with expected crc 2023-10-09T11:04:40.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[195507]: 10.10 starting backfill to osd.7 from (0'0,0'0] MAX to 474'8914 2023-10-09T11:04:40.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[195507]: osdmap e477: 8 total, 8 up, 6 in 2023-10-09T11:04:40.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[195507]: failed to encode map e477 with expected crc 2023-10-09T11:04:40.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[195507]: 10.b starting backfill to osd.4 from (0'0,0'0] MAX to 474'11937 2023-10-09T11:04:40.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[195507]: failed to encode map e477 with expected crc 2023-10-09T11:04:40.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[195507]: failed to encode map e477 with expected crc 2023-10-09T11:04:40.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[195507]: failed to encode map e477 with expected crc 2023-10-09T11:04:40.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[195507]: 10.0 starting backfill to osd.7 from (0'0,0'0] MAX to 474'8824 2023-10-09T11:04:40.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[195507]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 474'11859 2023-10-09T11:04:40.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[195507]: failed to encode map e477 with expected crc 2023-10-09T11:04:40.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[195507]: 10.d starting backfill to osd.7 from (0'0,0'0] MAX to 474'11815 2023-10-09T11:04:40.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[195507]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 474'11852 2023-10-09T11:04:40.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/373489251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:40.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[200197]: failed to encode map e476 with expected crc 2023-10-09T11:04:40.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[200197]: 10.10 starting backfill to osd.7 from (0'0,0'0] MAX to 474'8914 2023-10-09T11:04:40.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[200197]: osdmap e477: 8 total, 8 up, 6 in 2023-10-09T11:04:40.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[200197]: failed to encode map e477 with expected crc 2023-10-09T11:04:40.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[200197]: 10.b starting backfill to osd.4 from (0'0,0'0] MAX to 474'11937 2023-10-09T11:04:40.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[200197]: failed to encode map e477 with expected crc 2023-10-09T11:04:40.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[200197]: failed to encode map e477 with expected crc 2023-10-09T11:04:40.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[200197]: failed to encode map e477 with expected crc 2023-10-09T11:04:40.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[200197]: 10.0 starting backfill to osd.7 from (0'0,0'0] MAX to 474'8824 2023-10-09T11:04:40.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[200197]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 474'11859 2023-10-09T11:04:40.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[200197]: failed to encode map e477 with expected crc 2023-10-09T11:04:40.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[200197]: 10.d starting backfill to osd.7 from (0'0,0'0] MAX to 474'11815 2023-10-09T11:04:40.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[200197]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 474'11852 2023-10-09T11:04:40.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:39 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/373489251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:40.051 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:40.093 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:04:40.139 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:04:40.167 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:04:40.167 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:04:40.167 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:04:40.167 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:04:40.167 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:04:40.167 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:04:40.167 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:04:40.167 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:04:40.182 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:40.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:40.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:40.291 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:40.377 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:40.665 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:40.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:40.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:40.773 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:40.835 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:40.886 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:40.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:40.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:40.997 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:04:40.997 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T11:04:40.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T11:04:40.997 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:04:40.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:40.997 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPush(8.13 293/289 [PushOp(8:cbe2b721:::benchmark_data_smithi125_329074_object29:head, version: 270'1, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:cbe2b721:::benchmark_data_smithi125_329074_object29:head@270'1, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T11:04:40.998 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:45.877250+0000", 2023-10-09T11:04:40.998 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 595.116888475, 2023-10-09T11:04:40.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.363915395, 2023-10-09T11:04:40.998 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:40.998 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:04:40.998 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:40.998 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:40.998 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:40.998 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877250+0000", 2023-10-09T11:04:40.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:40.998 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:40.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:40.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:40.999 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877250+0000", 2023-10-09T11:04:40.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:40.999 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:40.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:40.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:40.999 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877249+0000", 2023-10-09T11:04:40.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:04:40.999 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:40.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:40.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877303+0000", 2023-10-09T11:04:41.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3928999999999998e-05 2023-10-09T11:04:41.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.000 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877323+0000", 2023-10-09T11:04:41.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0276999999999999e-05 2023-10-09T11:04:41.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.000 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:45.877339+0000", 2023-10-09T11:04:41.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6362999999999999e-05 2023-10-09T11:04:41.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.190367+0000", 2023-10-09T11:04:41.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3130274530000001 2023-10-09T11:04:41.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.190793+0000", 2023-10-09T11:04:41.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042582999999999998 2023-10-09T11:04:41.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.002 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.002 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:47.241165+0000", 2023-10-09T11:04:41.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.050372739 2023-10-09T11:04:41.002 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.002 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.002 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.002 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.35433.0:95902 8.13 8:c9dc5fe7:::benchmark_data_smithi125_329074_object95901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T11:04:41.002 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:54:46.181255+0000", 2023-10-09T11:04:41.002 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 594.81288324699995, 2023-10-09T11:04:41.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.46745452100000001, 2023-10-09T11:04:41.003 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.003 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:41.003 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.003 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.35433", 2023-10-09T11:04:41.003 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:04:41.003 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 95902 2023-10-09T11:04:41.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.003 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.003 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.003 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181255+0000", 2023-10-09T11:04:41.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181255+0000", 2023-10-09T11:04:41.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181253+0000", 2023-10-09T11:04:41.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:04:41.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181286+0000", 2023-10-09T11:04:41.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2928000000000001e-05 2023-10-09T11:04:41.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181306+0000", 2023-10-09T11:04:41.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9514999999999999e-05 2023-10-09T11:04:41.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.181317+0000", 2023-10-09T11:04:41.006 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1070000000000001e-05 2023-10-09T11:04:41.006 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.006 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.006 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.006 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.503475+0000", 2023-10-09T11:04:41.006 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32215859400000002 2023-10-09T11:04:41.006 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.006 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.006 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.006 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.513376+0000", 2023-10-09T11:04:41.006 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0099005069999999994 2023-10-09T11:04:41.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.007 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T11:04:41.007 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.528220+0000", 2023-10-09T11:04:41.007 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014844346 2023-10-09T11:04:41.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.007 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.007 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.543824+0000", 2023-10-09T11:04:41.007 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015603361 2023-10-09T11:04:41.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.547461+0000", 2023-10-09T11:04:41.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036375349999999999 2023-10-09T11:04:41.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:41.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.633195+0000", 2023-10-09T11:04:41.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.085734224999999997 2023-10-09T11:04:41.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:04:41.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.636727+0000", 2023-10-09T11:04:41.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035320870000000002 2023-10-09T11:04:41.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:54:46.648710+0000", 2023-10-09T11:04:41.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011982244 2023-10-09T11:04:41.009 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.009 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.009 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.009 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122010 9.6 9:6f14ab77:::benchmark_data_smithi125_363789_object11711:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:04:41.010 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.136716+0000", 2023-10-09T11:04:41.010 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 332.857422762, 2023-10-09T11:04:41.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0621660900000001, 2023-10-09T11:04:41.010 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.010 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:04:41.010 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.010 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.010 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.010 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122010 2023-10-09T11:04:41.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.010 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136716+0000", 2023-10-09T11:04:41.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136716+0000", 2023-10-09T11:04:41.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.012 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136715+0000", 2023-10-09T11:04:41.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:04:41.012 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.012 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.012 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.012 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136721+0000", 2023-10-09T11:04:41.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7660000000000004e-06 2023-10-09T11:04:41.012 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.012 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.012 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.012 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136731+0000", 2023-10-09T11:04:41.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0898e-05 2023-10-09T11:04:41.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.013 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.013 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136739+0000", 2023-10-09T11:04:41.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8450000000000003e-06 2023-10-09T11:04:41.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.013 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.013 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.140487+0000", 2023-10-09T11:04:41.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037476330000000002 2023-10-09T11:04:41.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.140607+0000", 2023-10-09T11:04:41.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012029299999999999 2023-10-09T11:04:41.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T11:04:41.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.140726+0000", 2023-10-09T11:04:41.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119057 2023-10-09T11:04:41.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:41.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.150456+0000", 2023-10-09T11:04:41.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.009729283 2023-10-09T11:04:41.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.153186+0000", 2023-10-09T11:04:41.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00273077 2023-10-09T11:04:41.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162778+0000", 2023-10-09T11:04:41.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0095913500000000002 2023-10-09T11:04:41.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:04:41.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162790+0000", 2023-10-09T11:04:41.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.221e-05 2023-10-09T11:04:41.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.198810+0000", 2023-10-09T11:04:41.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.036020156 2023-10-09T11:04:41.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.017 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.198882+0000", 2023-10-09T11:04:41.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1717999999999993e-05 2023-10-09T11:04:41.017 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.017 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.017 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.017 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.017 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122013 9.6 9:6b7eeaa4:::benchmark_data_smithi125_363789_object11714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:04:41.017 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.138283+0000", 2023-10-09T11:04:41.017 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 332.85585540900001, 2023-10-09T11:04:41.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.060936141, 2023-10-09T11:04:41.017 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.018 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:04:41.018 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.018 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.018 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.018 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122013 2023-10-09T11:04:41.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.018 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.018 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138283+0000", 2023-10-09T11:04:41.018 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.019 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138283+0000", 2023-10-09T11:04:41.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.019 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138282+0000", 2023-10-09T11:04:41.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:04:41.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138288+0000", 2023-10-09T11:04:41.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5029999999999997e-06 2023-10-09T11:04:41.020 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.020 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138300+0000", 2023-10-09T11:04:41.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2357e-05 2023-10-09T11:04:41.020 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.020 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138309+0000", 2023-10-09T11:04:41.021 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.499e-06 2023-10-09T11:04:41.021 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.021 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.021 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.021 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.144017+0000", 2023-10-09T11:04:41.021 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0057070339999999997 2023-10-09T11:04:41.021 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.021 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.021 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.021 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.144165+0000", 2023-10-09T11:04:41.021 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000148345 2023-10-09T11:04:41.021 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.021 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T11:04:41.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.144293+0000", 2023-10-09T11:04:41.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000128069 2023-10-09T11:04:41.022 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:41.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.152647+0000", 2023-10-09T11:04:41.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083540590000000005 2023-10-09T11:04:41.022 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.153226+0000", 2023-10-09T11:04:41.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00057917199999999998 2023-10-09T11:04:41.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162825+0000", 2023-10-09T11:04:41.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0095991010000000005 2023-10-09T11:04:41.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:04:41.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162833+0000", 2023-10-09T11:04:41.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3429999999999998e-06 2023-10-09T11:04:41.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199105+0000", 2023-10-09T11:04:41.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0362724169999999 2023-10-09T11:04:41.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199219+0000", 2023-10-09T11:04:41.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000114141 2023-10-09T11:04:41.024 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.024 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.024 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.025 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122028 9.6 9:6992c4ac:::benchmark_data_smithi125_363789_object11729:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:04:41.025 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.154935+0000", 2023-10-09T11:04:41.025 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 332.83920336400001, 2023-10-09T11:04:41.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.044381056, 2023-10-09T11:04:41.025 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.025 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:04:41.025 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.025 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.025 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.026 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122028 2023-10-09T11:04:41.026 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.026 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.026 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.026 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154935+0000", 2023-10-09T11:04:41.026 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.026 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.026 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.026 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154935+0000", 2023-10-09T11:04:41.026 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.027 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.027 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.027 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154934+0000", 2023-10-09T11:04:41.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:04:41.027 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.027 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.027 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154941+0000", 2023-10-09T11:04:41.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9600000000000003e-06 2023-10-09T11:04:41.027 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.027 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.027 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.028 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154956+0000", 2023-10-09T11:04:41.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4976e-05 2023-10-09T11:04:41.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.028 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.028 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154964+0000", 2023-10-09T11:04:41.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8639999999999996e-06 2023-10-09T11:04:41.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.028 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.028 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.155012+0000", 2023-10-09T11:04:41.029 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7960999999999997e-05 2023-10-09T11:04:41.029 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.029 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.029 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.029 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.155124+0000", 2023-10-09T11:04:41.029 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011204 2023-10-09T11:04:41.029 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.029 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.029 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T11:04:41.029 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.155237+0000", 2023-10-09T11:04:41.029 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011364799999999999 2023-10-09T11:04:41.029 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162936+0000", 2023-10-09T11:04:41.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.007698369 2023-10-09T11:04:41.030 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:04:41.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162945+0000", 2023-10-09T11:04:41.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6039999999999999e-06 2023-10-09T11:04:41.030 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.031 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199250+0000", 2023-10-09T11:04:41.031 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.036304248 2023-10-09T11:04:41.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.031 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.031 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.031 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199316+0000", 2023-10-09T11:04:41.031 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6549000000000003e-05 2023-10-09T11:04:41.031 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.031 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.031 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.032 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.032 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122037 9.6 9:68399f49:::benchmark_data_smithi125_363789_object11738:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:04:41.032 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.181454+0000", 2023-10-09T11:04:41.032 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 332.81268409400002, 2023-10-09T11:04:41.032 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.032324094, 2023-10-09T11:04:41.032 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.032 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:41.032 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.032 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.032 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.032 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122037 2023-10-09T11:04:41.032 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.033 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181454+0000", 2023-10-09T11:04:41.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.033 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181454+0000", 2023-10-09T11:04:41.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.033 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.034 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.034 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181453+0000", 2023-10-09T11:04:41.034 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:04:41.034 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.034 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.034 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.034 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181461+0000", 2023-10-09T11:04:41.034 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4800000000000001e-06 2023-10-09T11:04:41.034 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.034 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.034 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.035 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181478+0000", 2023-10-09T11:04:41.035 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7215e-05 2023-10-09T11:04:41.035 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.035 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.035 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.035 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181488+0000", 2023-10-09T11:04:41.035 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5240000000000006e-06 2023-10-09T11:04:41.035 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.035 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.035 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.035 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181572+0000", 2023-10-09T11:04:41.035 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3784999999999998e-05 2023-10-09T11:04:41.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.036 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:04:41.036 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181581+0000", 2023-10-09T11:04:41.036 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7249999999999992e-06 2023-10-09T11:04:41.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.036 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.036 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199341+0000", 2023-10-09T11:04:41.036 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.017760062 2023-10-09T11:04:41.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.037 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.037 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199463+0000", 2023-10-09T11:04:41.037 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012115799999999999 2023-10-09T11:04:41.037 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.037 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.037 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5", 2023-10-09T11:04:41.037 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199575+0000", 2023-10-09T11:04:41.037 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000112404 2023-10-09T11:04:41.037 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.037 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.037 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:41.037 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.207005+0000", 2023-10-09T11:04:41.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074305350000000003 2023-10-09T11:04:41.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.213644+0000", 2023-10-09T11:04:41.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066387779999999997 2023-10-09T11:04:41.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:04:41.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.213695+0000", 2023-10-09T11:04:41.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0748999999999998e-05 2023-10-09T11:04:41.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.039 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.039 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.213778+0000", 2023-10-09T11:04:41.039 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3434000000000001e-05 2023-10-09T11:04:41.039 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.039 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.039 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.039 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122039 9.6 9:679871c1:::benchmark_data_smithi125_363789_object11740:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:04:41.039 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:04:41.040 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 332.81262313100001, 2023-10-09T11:04:41.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0328282099999999, 2023-10-09T11:04:41.040 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.040 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:41.040 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.040 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.040 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.040 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122039 2023-10-09T11:04:41.040 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.040 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.040 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:04:41.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:04:41.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.042 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:04:41.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:41.042 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.042 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.042 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.042 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181518+0000", 2023-10-09T11:04:41.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4350000000000001e-06 2023-10-09T11:04:41.042 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.042 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.042 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.042 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181525+0000", 2023-10-09T11:04:41.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5359999999999998e-06 2023-10-09T11:04:41.043 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.043 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.043 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.043 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181530+0000", 2023-10-09T11:04:41.043 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9019999999999998e-06 2023-10-09T11:04:41.043 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.043 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.043 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.043 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181601+0000", 2023-10-09T11:04:41.043 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1168999999999996e-05 2023-10-09T11:04:41.043 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:04:41.044 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181606+0000", 2023-10-09T11:04:41.044 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1919999999999996e-06 2023-10-09T11:04:41.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.044 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.203647+0000", 2023-10-09T11:04:41.044 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.022041073 2023-10-09T11:04:41.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.203754+0000", 2023-10-09T11:04:41.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000106529 2023-10-09T11:04:41.045 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.045 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5", 2023-10-09T11:04:41.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.203828+0000", 2023-10-09T11:04:41.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4286e-05 2023-10-09T11:04:41.045 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.045 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:41.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.210714+0000", 2023-10-09T11:04:41.046 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0068858050000000001 2023-10-09T11:04:41.046 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.046 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.046 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.046 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.214203+0000", 2023-10-09T11:04:41.046 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003489242 2023-10-09T11:04:41.046 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.046 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.046 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:04:41.046 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.214268+0000", 2023-10-09T11:04:41.046 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5104000000000006e-05 2023-10-09T11:04:41.047 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.047 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.047 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.214344+0000", 2023-10-09T11:04:41.047 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5471999999999998e-05 2023-10-09T11:04:41.047 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.047 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.047 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.047 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.047 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:133037 9.b 9.da5dd31b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e362)", 2023-10-09T11:04:41.048 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:20.474469+0000", 2023-10-09T11:04:41.048 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 320.51966964399998, 2023-10-09T11:04:41.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97365184000000005, 2023-10-09T11:04:41.048 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.048 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:04:41.048 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.048 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.048 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.048 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133037 2023-10-09T11:04:41.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.048 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474469+0000", 2023-10-09T11:04:41.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474469+0000", 2023-10-09T11:04:41.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474467+0000", 2023-10-09T11:04:41.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:04:41.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474475+0000", 2023-10-09T11:04:41.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6009999999999997e-06 2023-10-09T11:04:41.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474493+0000", 2023-10-09T11:04:41.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8150000000000001e-05 2023-10-09T11:04:41.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474502+0000", 2023-10-09T11:04:41.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1400000000000006e-06 2023-10-09T11:04:41.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474569+0000", 2023-10-09T11:04:41.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6849999999999999e-05 2023-10-09T11:04:41.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:04:41.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474579+0000", 2023-10-09T11:04:41.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9369999999999999e-06 2023-10-09T11:04:41.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448111+0000", 2023-10-09T11:04:41.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97353240399999996 2023-10-09T11:04:41.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448121+0000", 2023-10-09T11:04:41.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1679999999999993e-06 2023-10-09T11:04:41.053 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.053 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.053 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.053 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.053 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:133044 9.b 9.5473ef4b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e362)", 2023-10-09T11:04:41.053 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:20.482185+0000", 2023-10-09T11:04:41.053 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 320.51195326999999, 2023-10-09T11:04:41.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96595614900000004, 2023-10-09T11:04:41.054 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.054 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:04:41.054 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.054 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.054 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.054 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133044 2023-10-09T11:04:41.054 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.054 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.054 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.054 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482185+0000", 2023-10-09T11:04:41.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482185+0000", 2023-10-09T11:04:41.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482184+0000", 2023-10-09T11:04:41.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:04:41.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.056 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482193+0000", 2023-10-09T11:04:41.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0310000000000004e-06 2023-10-09T11:04:41.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.056 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482217+0000", 2023-10-09T11:04:41.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4677000000000001e-05 2023-10-09T11:04:41.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482228+0000", 2023-10-09T11:04:41.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.043e-05 2023-10-09T11:04:41.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482326+0000", 2023-10-09T11:04:41.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8241999999999999e-05 2023-10-09T11:04:41.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:04:41.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482334+0000", 2023-10-09T11:04:41.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7910000000000005e-06 2023-10-09T11:04:41.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448135+0000", 2023-10-09T11:04:41.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96580099799999997 2023-10-09T11:04:41.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448141+0000", 2023-10-09T11:04:41.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5719999999999999e-06 2023-10-09T11:04:41.059 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.059 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.059 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.059 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.059 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:133073 9.b 9.b3395a9b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e362)", 2023-10-09T11:04:41.059 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:20.525035+0000", 2023-10-09T11:04:41.059 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 320.46910336299999, 2023-10-09T11:04:41.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.92312863000000001, 2023-10-09T11:04:41.059 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.059 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:04:41.060 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.060 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.060 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.060 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133073 2023-10-09T11:04:41.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.060 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.060 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525035+0000", 2023-10-09T11:04:41.060 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525035+0000", 2023-10-09T11:04:41.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525034+0000", 2023-10-09T11:04:41.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:04:41.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525039+0000", 2023-10-09T11:04:41.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9710000000000003e-06 2023-10-09T11:04:41.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525049+0000", 2023-10-09T11:04:41.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4450000000000002e-06 2023-10-09T11:04:41.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525054+0000", 2023-10-09T11:04:41.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.101e-06 2023-10-09T11:04:41.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525084+0000", 2023-10-09T11:04:41.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0049999999999999e-05 2023-10-09T11:04:41.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:04:41.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.525089+0000", 2023-10-09T11:04:41.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1599999999999997e-06 2023-10-09T11:04:41.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448158+0000", 2023-10-09T11:04:41.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.92306902099999999 2023-10-09T11:04:41.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.064 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448164+0000", 2023-10-09T11:04:41.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7660000000000004e-06 2023-10-09T11:04:41.064 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.064 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.064 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.065 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157704 9.b 9:d7bfb393:::benchmark_data_smithi125_363789_object47405:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T11:04:41.065 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:53.921502+0000", 2023-10-09T11:04:41.065 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 287.072636123, 2023-10-09T11:04:41.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.46842198400000001, 2023-10-09T11:04:41.065 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.065 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:41.065 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.065 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.065 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.065 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157704 2023-10-09T11:04:41.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.066 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.921502+0000", 2023-10-09T11:04:41.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.921502+0000", 2023-10-09T11:04:41.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.067 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.921501+0000", 2023-10-09T11:04:41.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:04:41.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.067 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.067 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.921506+0000", 2023-10-09T11:04:41.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5609999999999998e-06 2023-10-09T11:04:41.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.067 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.068 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.921516+0000", 2023-10-09T11:04:41.068 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8779999999999993e-06 2023-10-09T11:04:41.068 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.068 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.921524+0000", 2023-10-09T11:04:41.068 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4229999999999999e-06 2023-10-09T11:04:41.068 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.068 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.117325+0000", 2023-10-09T11:04:41.068 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19580091799999999 2023-10-09T11:04:41.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.069 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.117403+0000", 2023-10-09T11:04:41.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8139999999999994e-05 2023-10-09T11:04:41.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.069 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:04:41.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.117503+0000", 2023-10-09T11:04:41.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010062500000000001 2023-10-09T11:04:41.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.070 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:41.070 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.389378+0000", 2023-10-09T11:04:41.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.271874587 2023-10-09T11:04:41.070 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.070 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.070 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.389418+0000", 2023-10-09T11:04:41.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0237000000000001e-05 2023-10-09T11:04:41.070 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.070 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.389765+0000", 2023-10-09T11:04:41.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000346375 2023-10-09T11:04:41.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.071 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.389788+0000", 2023-10-09T11:04:41.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2849000000000001e-05 2023-10-09T11:04:41.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.071 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:04:41.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.389822+0000", 2023-10-09T11:04:41.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4718999999999997e-05 2023-10-09T11:04:41.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.389924+0000", 2023-10-09T11:04:41.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000102058 2023-10-09T11:04:41.072 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.072 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.072 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.073 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157950 9.b 9:d2b9e9fa:::benchmark_data_smithi125_363789_object47651:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:04:41.073 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.472950+0000", 2023-10-09T11:04:41.073 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 286.52118831199999, 2023-10-09T11:04:41.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.54794583900000005, 2023-10-09T11:04:41.073 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.073 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:41.073 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.073 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.073 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.073 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157950 2023-10-09T11:04:41.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.074 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.472950+0000", 2023-10-09T11:04:41.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.472950+0000", 2023-10-09T11:04:41.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.472949+0000", 2023-10-09T11:04:41.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:04:41.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.472956+0000", 2023-10-09T11:04:41.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5429999999999996e-06 2023-10-09T11:04:41.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.472969+0000", 2023-10-09T11:04:41.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2547e-05 2023-10-09T11:04:41.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.076 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.076 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.472978+0000", 2023-10-09T11:04:41.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0909999999999999e-06 2023-10-09T11:04:41.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.076 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.076 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.672426+0000", 2023-10-09T11:04:41.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19944830099999999 2023-10-09T11:04:41.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.672563+0000", 2023-10-09T11:04:41.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013726 2023-10-09T11:04:41.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:04:41.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.672696+0000", 2023-10-09T11:04:41.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000132454 2023-10-09T11:04:41.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:41.078 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.680470+0000", 2023-10-09T11:04:41.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0077745660000000001 2023-10-09T11:04:41.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.078 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.078 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020539+0000", 2023-10-09T11:04:41.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.34006837899999998 2023-10-09T11:04:41.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.078 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.078 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020648+0000", 2023-10-09T11:04:41.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000109096 2023-10-09T11:04:41.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020701+0000", 2023-10-09T11:04:41.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2862000000000003e-05 2023-10-09T11:04:41.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:04:41.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020755+0000", 2023-10-09T11:04:41.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4200000000000003e-05 2023-10-09T11:04:41.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.080 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.080 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020896+0000", 2023-10-09T11:04:41.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000140934 2023-10-09T11:04:41.080 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.080 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.080 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.080 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157952 9.b 9:d7179396:::benchmark_data_smithi125_363789_object47653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:04:41.080 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.476086+0000", 2023-10-09T11:04:41.080 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 286.51805218700002, 2023-10-09T11:04:41.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.54503019500000005, 2023-10-09T11:04:41.081 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.081 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:41.081 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.081 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.081 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.081 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157952 2023-10-09T11:04:41.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.081 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.081 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.081 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.081 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.476086+0000", 2023-10-09T11:04:41.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.476086+0000", 2023-10-09T11:04:41.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.476085+0000", 2023-10-09T11:04:41.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:04:41.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.083 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.476092+0000", 2023-10-09T11:04:41.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4309999999999997e-06 2023-10-09T11:04:41.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.083 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.083 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.476106+0000", 2023-10-09T11:04:41.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3465999999999999e-05 2023-10-09T11:04:41.083 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.476113+0000", 2023-10-09T11:04:41.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4080000000000001e-06 2023-10-09T11:04:41.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.676444+0000", 2023-10-09T11:04:41.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20033067900000001 2023-10-09T11:04:41.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.676599+0000", 2023-10-09T11:04:41.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015494599999999999 2023-10-09T11:04:41.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:04:41.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.676743+0000", 2023-10-09T11:04:41.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014419199999999999 2023-10-09T11:04:41.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:41.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.687837+0000", 2023-10-09T11:04:41.086 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011093949000000001 2023-10-09T11:04:41.086 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.086 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.086 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020621+0000", 2023-10-09T11:04:41.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.332784258 2023-10-09T11:04:41.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020983+0000", 2023-10-09T11:04:41.088 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00036201099999999997 2023-10-09T11:04:41.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.088 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021014+0000", 2023-10-09T11:04:41.088 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0582e-05 2023-10-09T11:04:41.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.088 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:04:41.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021045+0000", 2023-10-09T11:04:41.089 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1804999999999998e-05 2023-10-09T11:04:41.089 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.089 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.089 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.089 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021116+0000", 2023-10-09T11:04:41.089 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0957999999999994e-05 2023-10-09T11:04:41.089 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.089 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.089 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.089 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.089 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.089 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157962 9.b 9:db800329:::benchmark_data_smithi125_363789_object47663:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:04:41.090 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.526479+0000", 2023-10-09T11:04:41.090 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 286.46765924499999, 2023-10-09T11:04:41.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.49495239499999999, 2023-10-09T11:04:41.090 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.090 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:41.090 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.090 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.090 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.090 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157962 2023-10-09T11:04:41.090 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.090 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.090 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.091 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.091 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.526479+0000", 2023-10-09T11:04:41.091 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.091 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.091 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.526479+0000", 2023-10-09T11:04:41.091 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.091 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.091 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.526477+0000", 2023-10-09T11:04:41.091 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:04:41.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.092 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.092 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.526486+0000", 2023-10-09T11:04:41.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9439999999999994e-06 2023-10-09T11:04:41.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.092 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.092 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.526506+0000", 2023-10-09T11:04:41.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9497999999999999e-05 2023-10-09T11:04:41.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.526517+0000", 2023-10-09T11:04:41.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1100999999999999e-05 2023-10-09T11:04:41.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.680558+0000", 2023-10-09T11:04:41.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.1540415 2023-10-09T11:04:41.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.680682+0000", 2023-10-09T11:04:41.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000123459 2023-10-09T11:04:41.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:04:41.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.680800+0000", 2023-10-09T11:04:41.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011798600000000001 2023-10-09T11:04:41.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:41.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020479+0000", 2023-10-09T11:04:41.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.339679283 2023-10-09T11:04:41.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.020937+0000", 2023-10-09T11:04:41.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00045773999999999999 2023-10-09T11:04:41.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021212+0000", 2023-10-09T11:04:41.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027538200000000001 2023-10-09T11:04:41.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021260+0000", 2023-10-09T11:04:41.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7484999999999997e-05 2023-10-09T11:04:41.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:04:41.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021311+0000", 2023-10-09T11:04:41.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.109e-05 2023-10-09T11:04:41.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021432+0000", 2023-10-09T11:04:41.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120673 2023-10-09T11:04:41.096 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.097 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.097 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.097 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.097 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.097 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157985 9.b 9:d7678e95:::benchmark_data_smithi125_363789_object47686:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:04:41.097 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.693350+0000", 2023-10-09T11:04:41.097 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 286.300788431, 2023-10-09T11:04:41.097 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.49022632999999999, 2023-10-09T11:04:41.097 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.097 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:41.097 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.098 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.098 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.098 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157985 2023-10-09T11:04:41.098 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.098 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.098 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.098 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.693350+0000", 2023-10-09T11:04:41.098 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.098 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.098 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.099 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.693350+0000", 2023-10-09T11:04:41.099 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.099 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.099 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.099 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.099 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.693349+0000", 2023-10-09T11:04:41.099 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:04:41.099 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.099 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.099 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.099 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.693355+0000", 2023-10-09T11:04:41.099 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8109999999999999e-06 2023-10-09T11:04:41.099 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.100 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.100 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.693379+0000", 2023-10-09T11:04:41.100 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4541000000000001e-05 2023-10-09T11:04:41.100 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.100 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.100 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.693386+0000", 2023-10-09T11:04:41.100 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3900000000000004e-06 2023-10-09T11:04:41.100 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.021982+0000", 2023-10-09T11:04:41.101 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32859580599999999 2023-10-09T11:04:41.101 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.101 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.022117+0000", 2023-10-09T11:04:41.101 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000134907 2023-10-09T11:04:41.101 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.101 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:04:41.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.022262+0000", 2023-10-09T11:04:41.101 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014447 2023-10-09T11:04:41.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.102 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.102 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:41.102 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.180849+0000", 2023-10-09T11:04:41.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.15858782900000001 2023-10-09T11:04:41.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.102 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.102 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.102 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183388+0000", 2023-10-09T11:04:41.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025387890000000001 2023-10-09T11:04:41.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.102 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183417+0000", 2023-10-09T11:04:41.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8996000000000002e-05 2023-10-09T11:04:41.103 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183444+0000", 2023-10-09T11:04:41.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6706000000000001e-05 2023-10-09T11:04:41.103 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:04:41.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183480+0000", 2023-10-09T11:04:41.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5753000000000001e-05 2023-10-09T11:04:41.104 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.104 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.104 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.104 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183576+0000", 2023-10-09T11:04:41.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6614999999999996e-05 2023-10-09T11:04:41.104 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.104 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.104 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.104 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.104 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.104 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157991 9.b 9:d8d7183f:::benchmark_data_smithi125_363789_object47692:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:04:41.104 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.702255+0000", 2023-10-09T11:04:41.105 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 286.29188295, 2023-10-09T11:04:41.105 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.48154451799999998, 2023-10-09T11:04:41.105 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.105 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:41.105 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.105 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.105 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.105 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157991 2023-10-09T11:04:41.105 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.105 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.105 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.105 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.105 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.702255+0000", 2023-10-09T11:04:41.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.106 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.106 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.106 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.106 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.702255+0000", 2023-10-09T11:04:41.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.106 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.106 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.106 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.106 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.702254+0000", 2023-10-09T11:04:41.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:04:41.106 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.107 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.702262+0000", 2023-10-09T11:04:41.107 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5890000000000003e-06 2023-10-09T11:04:41.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.107 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.702275+0000", 2023-10-09T11:04:41.107 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3117e-05 2023-10-09T11:04:41.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.107 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.702282+0000", 2023-10-09T11:04:41.108 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2239999999999998e-06 2023-10-09T11:04:41.108 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.108 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.108 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.108 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.024521+0000", 2023-10-09T11:04:41.108 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32223829100000001 2023-10-09T11:04:41.108 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.108 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.108 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.108 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.024658+0000", 2023-10-09T11:04:41.108 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013682200000000001 2023-10-09T11:04:41.108 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.109 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.109 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:04:41.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.024793+0000", 2023-10-09T11:04:41.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013507800000000001 2023-10-09T11:04:41.109 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.109 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.109 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:41.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183395+0000", 2023-10-09T11:04:41.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.15860237099999999 2023-10-09T11:04:41.109 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.109 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.109 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183624+0000", 2023-10-09T11:04:41.110 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00022910300000000001 2023-10-09T11:04:41.110 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.110 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.110 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183652+0000", 2023-10-09T11:04:41.110 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8217000000000001e-05 2023-10-09T11:04:41.110 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.110 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.110 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183694+0000", 2023-10-09T11:04:41.110 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1566999999999997e-05 2023-10-09T11:04:41.110 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.111 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.111 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:04:41.111 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183724+0000", 2023-10-09T11:04:41.111 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0412e-05 2023-10-09T11:04:41.111 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.111 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.111 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.111 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183800+0000", 2023-10-09T11:04:41.111 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5706999999999999e-05 2023-10-09T11:04:41.111 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.111 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.111 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.112 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.112 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.112 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:158019 9.b 9:df039cc9:::benchmark_data_smithi125_363789_object47720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:04:41.112 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.727586+0000", 2023-10-09T11:04:41.112 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 286.26655204100001, 2023-10-09T11:04:41.112 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.46238494899999999, 2023-10-09T11:04:41.112 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.112 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:41.112 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.112 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.112 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.112 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 158019 2023-10-09T11:04:41.112 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.113 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.113 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.113 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.113 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727586+0000", 2023-10-09T11:04:41.113 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.113 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.113 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.113 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.113 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727586+0000", 2023-10-09T11:04:41.113 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.113 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.113 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.114 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.114 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727585+0000", 2023-10-09T11:04:41.114 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:04:41.114 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.114 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.114 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.114 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727595+0000", 2023-10-09T11:04:41.114 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1929999999999999e-06 2023-10-09T11:04:41.114 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.114 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.114 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.114 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727606+0000", 2023-10-09T11:04:41.115 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1147e-05 2023-10-09T11:04:41.115 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.115 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.115 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.115 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727611+0000", 2023-10-09T11:04:41.115 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6520000000000003e-06 2023-10-09T11:04:41.115 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.115 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.115 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.115 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.180904+0000", 2023-10-09T11:04:41.115 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.45329216 2023-10-09T11:04:41.115 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.115 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.116 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.116 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.181027+0000", 2023-10-09T11:04:41.116 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000123736 2023-10-09T11:04:41.116 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.116 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.116 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:04:41.116 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.181123+0000", 2023-10-09T11:04:41.116 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5544999999999994e-05 2023-10-09T11:04:41.116 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.116 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.116 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:41.116 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183661+0000", 2023-10-09T11:04:41.116 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025378850000000001 2023-10-09T11:04:41.117 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.117 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.117 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.117 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.189737+0000", 2023-10-09T11:04:41.117 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0060765560000000003 2023-10-09T11:04:41.117 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.117 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.117 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.117 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.189830+0000", 2023-10-09T11:04:41.117 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2592999999999999e-05 2023-10-09T11:04:41.117 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.117 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.118 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.118 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.189861+0000", 2023-10-09T11:04:41.118 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1245999999999999e-05 2023-10-09T11:04:41.118 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.118 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.118 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:04:41.118 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.189897+0000", 2023-10-09T11:04:41.118 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5506000000000002e-05 2023-10-09T11:04:41.118 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.118 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.118 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.118 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.189971+0000", 2023-10-09T11:04:41.119 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4732000000000002e-05 2023-10-09T11:04:41.119 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.119 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.119 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.119 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.119 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.119 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:158020 9.b 9:d7bb8056:::benchmark_data_smithi125_363789_object47721:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:04:41.119 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.727616+0000", 2023-10-09T11:04:41.119 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 286.26652244899998, 2023-10-09T11:04:41.119 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.49230036799999999, 2023-10-09T11:04:41.119 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.119 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:41.120 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.120 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.120 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.120 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 158020 2023-10-09T11:04:41.120 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.120 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.120 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.120 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.120 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727616+0000", 2023-10-09T11:04:41.120 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.120 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.120 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.121 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.121 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727616+0000", 2023-10-09T11:04:41.121 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.121 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.121 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.121 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.121 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727616+0000", 2023-10-09T11:04:41.121 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:41.121 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.121 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.121 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.121 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727617+0000", 2023-10-09T11:04:41.121 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5889999999999999e-06 2023-10-09T11:04:41.122 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.122 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.122 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.122 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727622+0000", 2023-10-09T11:04:41.122 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.301e-06 2023-10-09T11:04:41.122 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.122 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.122 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.122 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.727624+0000", 2023-10-09T11:04:41.122 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7810000000000001e-06 2023-10-09T11:04:41.122 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.122 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.123 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.123 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.183941+0000", 2023-10-09T11:04:41.123 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.45631645599999998 2023-10-09T11:04:41.123 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.123 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.123 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.123 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.184064+0000", 2023-10-09T11:04:41.123 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012321399999999999 2023-10-09T11:04:41.123 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.123 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.123 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:04:41.123 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.184187+0000", 2023-10-09T11:04:41.123 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000122929 2023-10-09T11:04:41.124 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.124 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.124 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.124 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.189782+0000", 2023-10-09T11:04:41.124 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0055952470000000002 2023-10-09T11:04:41.124 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.124 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.124 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:41.124 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.189840+0000", 2023-10-09T11:04:41.124 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7769999999999997e-05 2023-10-09T11:04:41.124 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.124 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.125 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.125 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.196501+0000", 2023-10-09T11:04:41.125 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066608980000000002 2023-10-09T11:04:41.125 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.125 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.125 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.125 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.219724+0000", 2023-10-09T11:04:41.125 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.023222804 2023-10-09T11:04:41.125 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.125 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.125 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:04:41.125 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.219782+0000", 2023-10-09T11:04:41.125 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8514000000000003e-05 2023-10-09T11:04:41.125 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.126 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.126 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.126 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.219916+0000", 2023-10-09T11:04:41.126 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013412799999999999 2023-10-09T11:04:41.126 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.126 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.126 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.126 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.126 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.126 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:158030 9.b 9:dd6e0b4d:::benchmark_data_smithi125_363789_object47731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:04:41.126 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.738309+0000", 2023-10-09T11:04:41.127 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 286.25582924600002, 2023-10-09T11:04:41.127 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.48184437899999999, 2023-10-09T11:04:41.127 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.127 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:41.127 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.127 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.127 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.127 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 158030 2023-10-09T11:04:41.127 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.127 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.127 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.127 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.127 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.738309+0000", 2023-10-09T11:04:41.128 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.128 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.128 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.128 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.128 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.738309+0000", 2023-10-09T11:04:41.128 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.128 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.128 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.128 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.128 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.738307+0000", 2023-10-09T11:04:41.128 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:04:41.128 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.129 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.129 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.129 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.738317+0000", 2023-10-09T11:04:41.129 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3840000000000003e-06 2023-10-09T11:04:41.129 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.129 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.129 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.129 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.738334+0000", 2023-10-09T11:04:41.129 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7668000000000001e-05 2023-10-09T11:04:41.129 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.129 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.129 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.129 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.738345+0000", 2023-10-09T11:04:41.130 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0739e-05 2023-10-09T11:04:41.130 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.130 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.130 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.130 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.187098+0000", 2023-10-09T11:04:41.130 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.44875295999999998 2023-10-09T11:04:41.130 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.130 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.130 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.130 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.187203+0000", 2023-10-09T11:04:41.130 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000105021 2023-10-09T11:04:41.130 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.131 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.131 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:04:41.131 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.187295+0000", 2023-10-09T11:04:41.131 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1476000000000003e-05 2023-10-09T11:04:41.131 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.131 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.131 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:41.131 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.193554+0000", 2023-10-09T11:04:41.131 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062596179999999998 2023-10-09T11:04:41.131 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.131 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.131 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.132 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.196437+0000", 2023-10-09T11:04:41.132 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028830459999999998 2023-10-09T11:04:41.132 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.132 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.132 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.132 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.196573+0000", 2023-10-09T11:04:41.132 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000135556 2023-10-09T11:04:41.132 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.132 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.132 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.132 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.219968+0000", 2023-10-09T11:04:41.132 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.023394873 2023-10-09T11:04:41.132 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.133 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.133 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:04:41.133 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.220021+0000", 2023-10-09T11:04:41.133 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3173000000000001e-05 2023-10-09T11:04:41.133 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.133 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.133 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.133 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.220154+0000", 2023-10-09T11:04:41.133 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013273700000000001 2023-10-09T11:04:41.133 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.133 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.133 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.134 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.134 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.134 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:158054 9.b 9:d18fda64:::benchmark_data_smithi125_363789_object47755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T11:04:41.134 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:54.765080+0000", 2023-10-09T11:04:41.134 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 286.22905851799999, 2023-10-09T11:04:41.134 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.45531745400000001, 2023-10-09T11:04:41.134 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:04:41.134 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:41.134 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:04:41.134 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:04:41.134 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:41.134 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 158054 2023-10-09T11:04:41.135 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.135 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:04:41.135 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.135 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:04:41.135 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.765080+0000", 2023-10-09T11:04:41.135 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.135 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.135 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.135 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:04:41.135 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.765080+0000", 2023-10-09T11:04:41.135 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:04:41.135 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.136 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.136 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:04:41.136 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.765078+0000", 2023-10-09T11:04:41.136 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:04:41.136 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.136 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.136 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:04:41.136 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.765088+0000", 2023-10-09T11:04:41.136 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5389999999999996e-06 2023-10-09T11:04:41.136 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.136 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.136 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:04:41.136 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.765101+0000", 2023-10-09T11:04:41.137 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3061000000000001e-05 2023-10-09T11:04:41.137 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.137 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.137 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:04:41.137 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.765113+0000", 2023-10-09T11:04:41.137 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2103000000000001e-05 2023-10-09T11:04:41.137 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.137 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.137 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:04:41.137 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.190017+0000", 2023-10-09T11:04:41.137 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.42490388800000001 2023-10-09T11:04:41.137 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.138 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.138 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:04:41.138 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.190166+0000", 2023-10-09T11:04:41.138 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014956999999999999 2023-10-09T11:04:41.138 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.138 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.138 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T11:04:41.138 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.190308+0000", 2023-10-09T11:04:41.138 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000141698 2023-10-09T11:04:41.138 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.138 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.138 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:04:41.139 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.196514+0000", 2023-10-09T11:04:41.139 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062055110000000004 2023-10-09T11:04:41.139 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.139 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.139 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.139 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.202781+0000", 2023-10-09T11:04:41.139 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062676490000000001 2023-10-09T11:04:41.139 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.139 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.139 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.139 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.202829+0000", 2023-10-09T11:04:41.139 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7598000000000001e-05 2023-10-09T11:04:41.139 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.140 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.140 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:41.140 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.220230+0000", 2023-10-09T11:04:41.140 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.017401478000000001 2023-10-09T11:04:41.140 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.140 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.140 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:04:41.140 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.220286+0000", 2023-10-09T11:04:41.140 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.592e-05 2023-10-09T11:04:41.140 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:04:41.140 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:04:41.140 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:04:41.141 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:55.220397+0000", 2023-10-09T11:04:41.141 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111214 2023-10-09T11:04:41.141 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.141 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.141 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.141 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:04:41.141 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:04:41.141 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:04:41.141 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:41.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:40 smithi125 ceph-mon[195507]: failed to encode map e477 with expected crc 2023-10-09T11:04:41.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:40 smithi125 ceph-mon[195507]: failed to encode map e477 with expected crc 2023-10-09T11:04:41.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:40 smithi125 ceph-mon[195507]: pgmap v1181: 129 pgs: 9 remapped+peering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 438 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 12321/21066 objects degraded (58.488%); 444/21066 objects misplaced (2.108%); 0 B/s, 160 objects/s recovering 2023-10-09T11:04:41.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:40 smithi125 ceph-mon[195507]: osdmap e478: 8 total, 8 up, 6 in 2023-10-09T11:04:41.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:40 smithi125 ceph-mon[195507]: failed to encode map e478 with expected crc 2023-10-09T11:04:41.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:40 smithi125 ceph-mon[195507]: failed to encode map e478 with expected crc 2023-10-09T11:04:41.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:40 smithi125 ceph-mon[195507]: failed to encode map e478 with expected crc 2023-10-09T11:04:41.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:40 smithi125 ceph-mon[200197]: failed to encode map e477 with expected crc 2023-10-09T11:04:41.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:40 smithi125 ceph-mon[200197]: failed to encode map e477 with expected crc 2023-10-09T11:04:41.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:40 smithi125 ceph-mon[200197]: pgmap v1181: 129 pgs: 9 remapped+peering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 438 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 12321/21066 objects degraded (58.488%); 444/21066 objects misplaced (2.108%); 0 B/s, 160 objects/s recovering 2023-10-09T11:04:41.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:40 smithi125 ceph-mon[200197]: osdmap e478: 8 total, 8 up, 6 in 2023-10-09T11:04:41.144 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:40 smithi125 ceph-mon[200197]: failed to encode map e478 with expected crc 2023-10-09T11:04:41.144 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:40 smithi125 ceph-mon[200197]: failed to encode map e478 with expected crc 2023-10-09T11:04:41.144 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:40 smithi125 ceph-mon[200197]: failed to encode map e478 with expected crc 2023-10-09T11:04:41.190 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:40 smithi155 ceph-mon[184854]: failed to encode map e477 with expected crc 2023-10-09T11:04:41.190 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:40 smithi155 ceph-mon[184854]: failed to encode map e477 with expected crc 2023-10-09T11:04:41.190 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:40 smithi155 ceph-mon[184854]: pgmap v1181: 129 pgs: 9 remapped+peering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 108 active+clean; 438 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 12321/21066 objects degraded (58.488%); 444/21066 objects misplaced (2.108%); 0 B/s, 160 objects/s recovering 2023-10-09T11:04:41.190 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:40 smithi155 ceph-mon[184854]: osdmap e478: 8 total, 8 up, 6 in 2023-10-09T11:04:41.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:40 smithi155 ceph-mon[184854]: failed to encode map e478 with expected crc 2023-10-09T11:04:41.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:40 smithi155 ceph-mon[184854]: failed to encode map e478 with expected crc 2023-10-09T11:04:41.191 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:40 smithi155 ceph-mon[184854]: failed to encode map e478 with expected crc 2023-10-09T11:04:41.198 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:41.226 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:41.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:41.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:41.304 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:04:41.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:04:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:04:41] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-10-09T11:04:41.333 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:41.398 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:41.417 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T11:04:41.441 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:41.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:41.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:41.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:41.615 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:41.648 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:41.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:41.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:41.746 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:41.754 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:41.804 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:41.828 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:41.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:41.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:41.878 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:41 smithi125 ceph-mon[200197]: failed to encode map e478 with expected crc 2023-10-09T11:04:41.878 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:41 smithi125 ceph-mon[200197]: failed to encode map e478 with expected crc 2023-10-09T11:04:41.879 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:41 smithi125 ceph-mon[200197]: osdmap e479: 8 total, 8 up, 6 in 2023-10-09T11:04:41.879 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:41 smithi125 ceph-mon[200197]: failed to encode map e479 with expected crc 2023-10-09T11:04:41.879 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:41 smithi125 ceph-mon[200197]: failed to encode map e479 with expected crc 2023-10-09T11:04:41.912 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:04:41.934 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:41.978 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:42.014 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:42.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:42.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:42.061 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:41 smithi155 ceph-mon[184854]: failed to encode map e478 with expected crc 2023-10-09T11:04:42.061 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:41 smithi155 ceph-mon[184854]: failed to encode map e478 with expected crc 2023-10-09T11:04:42.061 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:41 smithi155 ceph-mon[184854]: osdmap e479: 8 total, 8 up, 6 in 2023-10-09T11:04:42.062 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:41 smithi155 ceph-mon[184854]: failed to encode map e479 with expected crc 2023-10-09T11:04:42.062 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:41 smithi155 ceph-mon[184854]: failed to encode map e479 with expected crc 2023-10-09T11:04:42.062 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:41 smithi155 ceph-mon[184854]: failed to encode map e479 with expected crc 2023-10-09T11:04:42.062 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:41 smithi155 ceph-mon[184854]: failed to encode map e479 with expected crc 2023-10-09T11:04:42.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:42.165 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:41 smithi125 ceph-mon[200197]: failed to encode map e479 with expected crc 2023-10-09T11:04:42.165 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:41 smithi125 ceph-mon[200197]: failed to encode map e479 with expected crc 2023-10-09T11:04:42.166 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:41 smithi125 ceph-mon[195507]: failed to encode map e478 with expected crc 2023-10-09T11:04:42.166 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:41 smithi125 ceph-mon[195507]: failed to encode map e478 with expected crc 2023-10-09T11:04:42.166 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:41 smithi125 ceph-mon[195507]: osdmap e479: 8 total, 8 up, 6 in 2023-10-09T11:04:42.166 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:41 smithi125 ceph-mon[195507]: failed to encode map e479 with expected crc 2023-10-09T11:04:42.166 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:41 smithi125 ceph-mon[195507]: failed to encode map e479 with expected crc 2023-10-09T11:04:42.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:41 smithi125 ceph-mon[195507]: failed to encode map e479 with expected crc 2023-10-09T11:04:42.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:41 smithi125 ceph-mon[195507]: failed to encode map e479 with expected crc 2023-10-09T11:04:42.169 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:42.197 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:42.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:42.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:42.301 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:42.351 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:42.429 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:42.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:42.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:42.535 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:42.569 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:04:42.593 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:42.674 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:42.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:42.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:42.783 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:42.876 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:42.920 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:42.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:42.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:42.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:42 smithi155 ceph-mon[184854]: failed to encode map e479 with expected crc 2023-10-09T11:04:42.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:42 smithi155 ceph-mon[184854]: pgmap v1184: 129 pgs: 1 active+recovering+remapped, 9 remapped+peering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 107 active+clean; 373 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 547 op/s; 11281/17940 objects degraded (62.882%); 1206/17940 objects misplaced (6.722%); 0 B/s, 455 objects/s recovering 2023-10-09T11:04:43.027 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:43.076 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:42 smithi125 ceph-mon[195507]: failed to encode map e479 with expected crc 2023-10-09T11:04:43.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:42 smithi125 ceph-mon[195507]: pgmap v1184: 129 pgs: 1 active+recovering+remapped, 9 remapped+peering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 107 active+clean; 373 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 547 op/s; 11281/17940 objects degraded (62.882%); 1206/17940 objects misplaced (6.722%); 0 B/s, 455 objects/s recovering 2023-10-09T11:04:43.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:42 smithi125 ceph-mon[200197]: failed to encode map e479 with expected crc 2023-10-09T11:04:43.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:42 smithi125 ceph-mon[200197]: pgmap v1184: 129 pgs: 1 active+recovering+remapped, 9 remapped+peering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 107 active+clean; 373 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 547 op/s; 11281/17940 objects degraded (62.882%); 1206/17940 objects misplaced (6.722%); 0 B/s, 455 objects/s recovering 2023-10-09T11:04:43.093 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:43.103 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:43.138 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:43.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:43.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:43.235 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:04:43.235 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:04:43.235 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:193471 10.c 10:39ab10b3:::benchmark_data_smithi125_402869_object94902:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e479)", 2023-10-09T11:04:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:43.225523+0000", 2023-10-09T11:04:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0093050689999999991, 2023-10-09T11:04:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0093356010000000007, 2023-10-09T11:04:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:04:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:04:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:04:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 193471 2023-10-09T11:04:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.236 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.225523+0000", 2023-10-09T11:04:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.225523+0000", 2023-10-09T11:04:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.225522+0000", 2023-10-09T11:04:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:04:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.225529+0000", 2023-10-09T11:04:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5399999999999998e-06 2023-10-09T11:04:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.225547+0000", 2023-10-09T11:04:43.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7604e-05 2023-10-09T11:04:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.225554+0000", 2023-10-09T11:04:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6329999999999996e-06 2023-10-09T11:04:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.225628+0000", 2023-10-09T11:04:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4002000000000001e-05 2023-10-09T11:04:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.239 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.225784+0000", 2023-10-09T11:04:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015586199999999999 2023-10-09T11:04:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T11:04:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.225930+0000", 2023-10-09T11:04:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000145647 2023-10-09T11:04:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:04:43.240 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.234484+0000", 2023-10-09T11:04:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0085539599999999993 2023-10-09T11:04:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:193476 10.6 e479/441)", 2023-10-09T11:04:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:43.234499+0000", 2023-10-09T11:04:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.00032899399999999999, 2023-10-09T11:04:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00040951300000000002, 2023-10-09T11:04:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:43.241 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "queued for pg", 2023-10-09T11:04:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.234499+0000", 2023-10-09T11:04:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.234499+0000", 2023-10-09T11:04:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.242 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.234498+0000", 2023-10-09T11:04:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:04:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.234507+0000", 2023-10-09T11:04:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3059999999999998e-06 2023-10-09T11:04:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:43.243 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.234519+0000", 2023-10-09T11:04:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2061e-05 2023-10-09T11:04:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:43.234524+0000", 2023-10-09T11:04:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7640000000000002e-06 2023-10-09T11:04:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T11:04:43.244 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 2 2023-10-09T11:04:43.244 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:04:43.245 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:43.307 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:43.404 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:43.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:43.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:43.509 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:43.573 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:43.605 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:43.613 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T11:04:43.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:43.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:43.716 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:43.786 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:43.825 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:43.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:43.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:43.888 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:43 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 11281/17940 objects degraded (62.882%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:43.888 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:43 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2550468096' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:43.888 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:43 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:43.931 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:43.986 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:44.043 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:04:44.063 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:44.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:44.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:44.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:43 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 11281/17940 objects degraded (62.882%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:44.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:43 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2550468096' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:44.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:43 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:44.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:43 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 11281/17940 objects degraded (62.882%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:44.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:43 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2550468096' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:44.077 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:43 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:44.168 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:44.240 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:44.258 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 7] out_osds: [3, 6] dead_osds: [] live_osds: [0, 2, 4, 5, 6, 7, 1, 3] 2023-10-09T11:04:44.258 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:04:44.258 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-10-09T11:04:44.258 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd in 3 2023-10-09T11:04:44.296 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:44.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:44.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:44.317 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:04:44.375 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:04:44.405 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:44.423 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:04:44.460 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:44.463 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:04:44.497 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:04:44.500 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:44.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:44.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:44.536 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:04:44.580 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:04:44.607 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:44.617 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:04:44.646 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:04:44.653 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:04:44.653 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:04:44.653 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:04:44.653 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:04:44.653 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:04:44.654 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:04:44.654 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:04:44.654 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:04:44.665 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:44.805 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:44.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:44.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:44.909 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:44.952 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:44 smithi125 ceph-mon[195507]: pgmap v1185: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 272 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 12091/13122 objects degraded (92.143%); 1938/13122 objects misplaced (14.769%); 0 B/s, 309 objects/s recovering 2023-10-09T11:04:44.952 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:44 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:04:44.952 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:44 smithi125 ceph-mon[195507]: failed to encode map e480 with expected crc 2023-10-09T11:04:44.952 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:44 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:04:44.952 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:44 smithi125 ceph-mon[195507]: osdmap e480: 8 total, 8 up, 6 in 2023-10-09T11:04:44.952 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:44 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1439942869' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:44.952 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:44 smithi125 ceph-mon[200197]: pgmap v1185: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 272 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 12091/13122 objects degraded (92.143%); 1938/13122 objects misplaced (14.769%); 0 B/s, 309 objects/s recovering 2023-10-09T11:04:44.952 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:44 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:04:44.953 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:44 smithi125 ceph-mon[200197]: failed to encode map e480 with expected crc 2023-10-09T11:04:44.953 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:44 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:04:44.953 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:44 smithi125 ceph-mon[200197]: osdmap e480: 8 total, 8 up, 6 in 2023-10-09T11:04:44.953 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:44 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1439942869' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:44.963 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:45.121 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:44 smithi155 ceph-mon[184854]: pgmap v1185: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 272 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 12091/13122 objects degraded (92.143%); 1938/13122 objects misplaced (14.769%); 0 B/s, 309 objects/s recovering 2023-10-09T11:04:45.121 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:44 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:04:45.121 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:44 smithi155 ceph-mon[184854]: failed to encode map e480 with expected crc 2023-10-09T11:04:45.121 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:44 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:04:45.121 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:44 smithi155 ceph-mon[184854]: osdmap e480: 8 total, 8 up, 6 in 2023-10-09T11:04:45.121 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:44 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1439942869' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:45.744 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:04:45.744 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:04:46.075 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:45 smithi155 ceph-mon[184854]: failed to encode map e480 with expected crc 2023-10-09T11:04:46.075 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:46.075 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:04:46.075 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:45 smithi155 ceph-mon[184854]: osdmap e481: 8 total, 8 up, 6 in 2023-10-09T11:04:46.075 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:45 smithi155 ceph-mon[184854]: failed to encode map e481 with expected crc 2023-10-09T11:04:46.075 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:45 smithi155 ceph-mon[184854]: failed to encode map e481 with expected crc 2023-10-09T11:04:46.075 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:45 smithi155 ceph-mon[184854]: failed to encode map e481 with expected crc 2023-10-09T11:04:46.075 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:45 smithi155 ceph-mon[184854]: failed to encode map e481 with expected crc 2023-10-09T11:04:46.132 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T11:04:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[195507]: failed to encode map e480 with expected crc 2023-10-09T11:04:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:04:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[195507]: osdmap e481: 8 total, 8 up, 6 in 2023-10-09T11:04:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[195507]: failed to encode map e481 with expected crc 2023-10-09T11:04:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[195507]: failed to encode map e481 with expected crc 2023-10-09T11:04:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[195507]: failed to encode map e481 with expected crc 2023-10-09T11:04:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[195507]: failed to encode map e481 with expected crc 2023-10-09T11:04:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[200197]: failed to encode map e480 with expected crc 2023-10-09T11:04:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:04:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:04:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[200197]: osdmap e481: 8 total, 8 up, 6 in 2023-10-09T11:04:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[200197]: failed to encode map e481 with expected crc 2023-10-09T11:04:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[200197]: failed to encode map e481 with expected crc 2023-10-09T11:04:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[200197]: failed to encode map e481 with expected crc 2023-10-09T11:04:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:45 smithi125 ceph-mon[200197]: failed to encode map e481 with expected crc 2023-10-09T11:04:46.492 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:04:46.905 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:46.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:46.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:47.009 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:47.038 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:47.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[195507]: failed to encode map e481 with expected crc 2023-10-09T11:04:47.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[195507]: failed to encode map e481 with expected crc 2023-10-09T11:04:47.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[195507]: failed to encode map e481 with expected crc 2023-10-09T11:04:47.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[195507]: failed to encode map e481 with expected crc 2023-10-09T11:04:47.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[195507]: failed to encode map e481 with expected crc 2023-10-09T11:04:47.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[195507]: pgmap v1188: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 272 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 450 op/s; 12091/13122 objects degraded (92.143%); 1938/13122 objects misplaced (14.769%); 0 B/s, 212 objects/s recovering 2023-10-09T11:04:47.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[195507]: osdmap e482: 8 total, 8 up, 6 in 2023-10-09T11:04:47.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[195507]: failed to encode map e482 with expected crc 2023-10-09T11:04:47.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[195507]: failed to encode map e482 with expected crc 2023-10-09T11:04:47.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[195507]: failed to encode map e482 with expected crc 2023-10-09T11:04:47.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[200197]: failed to encode map e481 with expected crc 2023-10-09T11:04:47.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[200197]: failed to encode map e481 with expected crc 2023-10-09T11:04:47.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[200197]: failed to encode map e481 with expected crc 2023-10-09T11:04:47.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[200197]: failed to encode map e481 with expected crc 2023-10-09T11:04:47.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[200197]: failed to encode map e481 with expected crc 2023-10-09T11:04:47.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[200197]: pgmap v1188: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 272 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 450 op/s; 12091/13122 objects degraded (92.143%); 1938/13122 objects misplaced (14.769%); 0 B/s, 212 objects/s recovering 2023-10-09T11:04:47.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[200197]: osdmap e482: 8 total, 8 up, 6 in 2023-10-09T11:04:47.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[200197]: failed to encode map e482 with expected crc 2023-10-09T11:04:47.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[200197]: failed to encode map e482 with expected crc 2023-10-09T11:04:47.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:46 smithi125 ceph-mon[200197]: failed to encode map e482 with expected crc 2023-10-09T11:04:47.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:46 smithi155 ceph-mon[184854]: failed to encode map e481 with expected crc 2023-10-09T11:04:47.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:46 smithi155 ceph-mon[184854]: failed to encode map e481 with expected crc 2023-10-09T11:04:47.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:46 smithi155 ceph-mon[184854]: failed to encode map e481 with expected crc 2023-10-09T11:04:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:46 smithi155 ceph-mon[184854]: failed to encode map e481 with expected crc 2023-10-09T11:04:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:46 smithi155 ceph-mon[184854]: failed to encode map e481 with expected crc 2023-10-09T11:04:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:46 smithi155 ceph-mon[184854]: pgmap v1188: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 272 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 450 op/s; 12091/13122 objects degraded (92.143%); 1938/13122 objects misplaced (14.769%); 0 B/s, 212 objects/s recovering 2023-10-09T11:04:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:46 smithi155 ceph-mon[184854]: osdmap e482: 8 total, 8 up, 6 in 2023-10-09T11:04:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:46 smithi155 ceph-mon[184854]: failed to encode map e482 with expected crc 2023-10-09T11:04:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:46 smithi155 ceph-mon[184854]: failed to encode map e482 with expected crc 2023-10-09T11:04:47.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:46 smithi155 ceph-mon[184854]: failed to encode map e482 with expected crc 2023-10-09T11:04:47.295 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:47.308 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:47.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:47.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:47.413 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:47.482 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:47.513 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:47.517 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:04:47.517 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:04:47.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:47.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:47.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:47.682 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:47.718 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:47.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:47.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:47.823 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:47.890 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:47.979 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T11:04:47.982 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:47.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:47.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:48.044 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:47 smithi125 ceph-mon[195507]: failed to encode map e482 with expected crc 2023-10-09T11:04:48.044 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:47 smithi125 ceph-mon[195507]: failed to encode map e483 with expected crc 2023-10-09T11:04:48.044 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:47 smithi125 ceph-mon[195507]: osdmap e483: 8 total, 8 up, 6 in 2023-10-09T11:04:48.044 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:47 smithi125 ceph-mon[195507]: failed to encode map e483 with expected crc 2023-10-09T11:04:48.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:47 smithi125 ceph-mon[200197]: failed to encode map e482 with expected crc 2023-10-09T11:04:48.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:47 smithi125 ceph-mon[200197]: failed to encode map e483 with expected crc 2023-10-09T11:04:48.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:47 smithi125 ceph-mon[200197]: osdmap e483: 8 total, 8 up, 6 in 2023-10-09T11:04:48.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:47 smithi125 ceph-mon[200197]: failed to encode map e483 with expected crc 2023-10-09T11:04:48.087 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:48.150 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:48.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:47 smithi155 ceph-mon[184854]: failed to encode map e482 with expected crc 2023-10-09T11:04:48.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:47 smithi155 ceph-mon[184854]: failed to encode map e483 with expected crc 2023-10-09T11:04:48.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:47 smithi155 ceph-mon[184854]: osdmap e483: 8 total, 8 up, 6 in 2023-10-09T11:04:48.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:47 smithi155 ceph-mon[184854]: failed to encode map e483 with expected crc 2023-10-09T11:04:48.289 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:48.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:48.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:48.394 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:48.417 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:04:48.435 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:48.461 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:48.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:48.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:48.566 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:48.597 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:48.620 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:48.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:48.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:48.723 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:48.752 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:48.771 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:48.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:48.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:48.875 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:48.878 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:04:48.901 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T11:04:48.804+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e484 definitely_dead 0 2023-10-09T11:04:48.906 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:49.027 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:49.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:49.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:49.130 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:04:49.155 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:49.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:48 smithi155 ceph-mon[184854]: failed to encode map e483 with expected crc 2023-10-09T11:04:49.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:48 smithi155 ceph-mon[184854]: pgmap v1191: 129 pgs: 1 active+recovering+remapped, 3 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 159 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 452 op/s; 11328/7692 objects degraded (147.270%); 1171/7692 objects misplaced (15.224%); 2.2 MiB/s, 222 objects/s recovering 2023-10-09T11:04:49.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:48 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 12091/13122 objects degraded (92.143%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T11:04:49.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:48 smithi155 ceph-mon[184854]: osdmap e484: 8 total, 8 up, 6 in 2023-10-09T11:04:49.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:48 smithi155 ceph-mon[184854]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:48 smithi155 ceph-mon[184854]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:48 smithi155 ceph-mon[184854]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:48 smithi155 ceph-mon[184854]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:48 smithi155 ceph-mon[184854]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:48 smithi155 ceph-mon[184854]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:48 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2671777681' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T11:04:49.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:48 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T11:04:49.159 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:49.230 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:04:49.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:49.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:04:49.270 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[195507]: failed to encode map e483 with expected crc 2023-10-09T11:04:49.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[195507]: pgmap v1191: 129 pgs: 1 active+recovering+remapped, 3 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 159 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 452 op/s; 11328/7692 objects degraded (147.270%); 1171/7692 objects misplaced (15.224%); 2.2 MiB/s, 222 objects/s recovering 2023-10-09T11:04:49.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 12091/13122 objects degraded (92.143%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T11:04:49.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[195507]: osdmap e484: 8 total, 8 up, 6 in 2023-10-09T11:04:49.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[195507]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[195507]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[195507]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[195507]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[195507]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[195507]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.272 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2671777681' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T11:04:49.272 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T11:04:49.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[200197]: failed to encode map e483 with expected crc 2023-10-09T11:04:49.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[200197]: pgmap v1191: 129 pgs: 1 active+recovering+remapped, 3 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 110 active+clean; 159 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 452 op/s; 11328/7692 objects degraded (147.270%); 1171/7692 objects misplaced (15.224%); 2.2 MiB/s, 222 objects/s recovering 2023-10-09T11:04:49.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 12091/13122 objects degraded (92.143%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T11:04:49.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[200197]: osdmap e484: 8 total, 8 up, 6 in 2023-10-09T11:04:49.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[200197]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[200197]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[200197]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[200197]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[200197]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[200197]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2671777681' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T11:04:49.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:48 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T11:04:49.307 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.3. 2023-10-09T11:04:49.322 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:04:49.334 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:49.429 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:49.480 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:04:49.480 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:04:49.502 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:49.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:49.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:49.608 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:49.635 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:49.654 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:49.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:49.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:49.758 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:49.789 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:49.808 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:49.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:49.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:49.833 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T11:04:49.844 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-10-09T11:04:49.903 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[195507]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.903 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[195507]: 1.1c scrub starts 2023-10-09T11:04:49.903 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T11:04:49.903 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[195507]: osdmap e485: 8 total, 8 up, 7 in 2023-10-09T11:04:49.904 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[195507]: failed to encode map e485 with expected crc 2023-10-09T11:04:49.904 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[195507]: failed to encode map e485 with expected crc 2023-10-09T11:04:49.904 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[195507]: failed to encode map e485 with expected crc 2023-10-09T11:04:49.904 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[195507]: failed to encode map e485 with expected crc 2023-10-09T11:04:49.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[200197]: failed to encode map e484 with expected crc 2023-10-09T11:04:49.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[200197]: 1.1c scrub starts 2023-10-09T11:04:49.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T11:04:49.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[200197]: osdmap e485: 8 total, 8 up, 7 in 2023-10-09T11:04:49.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[200197]: failed to encode map e485 with expected crc 2023-10-09T11:04:49.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[200197]: failed to encode map e485 with expected crc 2023-10-09T11:04:49.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[200197]: failed to encode map e485 with expected crc 2023-10-09T11:04:49.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:49 smithi125 ceph-mon[200197]: failed to encode map e485 with expected crc 2023-10-09T11:04:49.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:04:49.941 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:50.009 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:04:50.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:50.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:04:50.096 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:50.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:50.140 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:49 smithi155 ceph-mon[184854]: failed to encode map e484 with expected crc 2023-10-09T11:04:50.141 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:49 smithi155 ceph-mon[184854]: 1.1c scrub starts 2023-10-09T11:04:50.141 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:49 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T11:04:50.141 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:49 smithi155 ceph-mon[184854]: osdmap e485: 8 total, 8 up, 7 in 2023-10-09T11:04:50.141 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:49 smithi155 ceph-mon[184854]: failed to encode map e485 with expected crc 2023-10-09T11:04:50.141 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:49 smithi155 ceph-mon[184854]: failed to encode map e485 with expected crc 2023-10-09T11:04:50.141 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:49 smithi155 ceph-mon[184854]: failed to encode map e485 with expected crc 2023-10-09T11:04:50.141 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:49 smithi155 ceph-mon[184854]: failed to encode map e485 with expected crc 2023-10-09T11:04:50.142 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:50.154 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:04:50.163 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:50.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:50.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:50.268 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:50.316 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:50.344 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:50.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:50.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:50.450 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:50.512 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:51.046 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:50 smithi155 ceph-mon[184854]: failed to encode map e485 with expected crc 2023-10-09T11:04:51.046 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:50 smithi155 ceph-mon[184854]: pgmap v1194: 129 pgs: 2 active+clean+remapped, 1 active+recovering+remapped, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 111 active+clean; 84 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 766 op/s; 11328/4068 objects degraded (278.466%); 366/4068 objects misplaced (8.997%); 3.4 MiB/s, 241 objects/s recovering 2023-10-09T11:04:51.047 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:50 smithi155 ceph-mon[184854]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.047 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:50 smithi155 ceph-mon[184854]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.047 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:50 smithi155 ceph-mon[184854]: osdmap e486: 8 total, 8 up, 7 in 2023-10-09T11:04:51.047 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:50 smithi155 ceph-mon[184854]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.047 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:50 smithi155 ceph-mon[184854]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.047 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:50 smithi155 ceph-mon[184854]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.047 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:50 smithi155 ceph-mon[184854]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.215 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[195507]: failed to encode map e485 with expected crc 2023-10-09T11:04:51.215 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[195507]: pgmap v1194: 129 pgs: 2 active+clean+remapped, 1 active+recovering+remapped, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 111 active+clean; 84 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 766 op/s; 11328/4068 objects degraded (278.466%); 366/4068 objects misplaced (8.997%); 3.4 MiB/s, 241 objects/s recovering 2023-10-09T11:04:51.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[195507]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[195507]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[195507]: osdmap e486: 8 total, 8 up, 7 in 2023-10-09T11:04:51.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[195507]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[195507]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[195507]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[195507]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[200197]: failed to encode map e485 with expected crc 2023-10-09T11:04:51.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[200197]: pgmap v1194: 129 pgs: 2 active+clean+remapped, 1 active+recovering+remapped, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 111 active+clean; 84 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 766 op/s; 11328/4068 objects degraded (278.466%); 366/4068 objects misplaced (8.997%); 3.4 MiB/s, 241 objects/s recovering 2023-10-09T11:04:51.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[200197]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[200197]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[200197]: osdmap e486: 8 total, 8 up, 7 in 2023-10-09T11:04:51.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[200197]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[200197]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[200197]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:50 smithi125 ceph-mon[200197]: failed to encode map e486 with expected crc 2023-10-09T11:04:51.229 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:04:51.230 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:04:51.323 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:04:51.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:04:51] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2023-10-09T11:04:51.607 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T11:04:51.912 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:04:51.961 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:51.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:51.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:52.065 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:52.095 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:52.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:52.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:52.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:52.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[195507]: failed to encode map e486 with expected crc 2023-10-09T11:04:52.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[195507]: 10.d starting backfill to osd.3 from (0'0,0'0] MAX to 484'12229 2023-10-09T11:04:52.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3061163574' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:04:52.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:04:52.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[195507]: failed to encode map e487 with expected crc 2023-10-09T11:04:52.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3061163574' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:04:52.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[195507]: osdmap e487: 8 total, 8 up, 7 in 2023-10-09T11:04:52.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[195507]: failed to encode map e487 with expected crc 2023-10-09T11:04:52.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[195507]: failed to encode map e487 with expected crc 2023-10-09T11:04:52.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[195507]: failed to encode map e487 with expected crc 2023-10-09T11:04:52.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[195507]: failed to encode map e487 with expected crc 2023-10-09T11:04:52.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[195507]: 10.f continuing backfill to osd.4 from (446'6721,484'12370] MIN to 484'12370 2023-10-09T11:04:52.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[195507]: 10.17 continuing backfill to osd.2 from (418'2375,484'9139] MIN to 484'9139 2023-10-09T11:04:52.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[195507]: 10.7 continuing backfill to osd.2 from (418'2640,484'9158] MIN to 484'9158 2023-10-09T11:04:52.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[200197]: failed to encode map e486 with expected crc 2023-10-09T11:04:52.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[200197]: 10.d starting backfill to osd.3 from (0'0,0'0] MAX to 484'12229 2023-10-09T11:04:52.145 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3061163574' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:04:52.145 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:04:52.145 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[200197]: failed to encode map e487 with expected crc 2023-10-09T11:04:52.145 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3061163574' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:04:52.145 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[200197]: osdmap e487: 8 total, 8 up, 7 in 2023-10-09T11:04:52.146 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[200197]: failed to encode map e487 with expected crc 2023-10-09T11:04:52.146 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[200197]: failed to encode map e487 with expected crc 2023-10-09T11:04:52.146 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[200197]: failed to encode map e487 with expected crc 2023-10-09T11:04:52.146 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[200197]: failed to encode map e487 with expected crc 2023-10-09T11:04:52.146 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[200197]: 10.f continuing backfill to osd.4 from (446'6721,484'12370] MIN to 484'12370 2023-10-09T11:04:52.146 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[200197]: 10.17 continuing backfill to osd.2 from (418'2375,484'9139] MIN to 484'9139 2023-10-09T11:04:52.146 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:51 smithi125 ceph-mon[200197]: 10.7 continuing backfill to osd.2 from (418'2640,484'9158] MIN to 484'9158 2023-10-09T11:04:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:51 smithi155 ceph-mon[184854]: failed to encode map e486 with expected crc 2023-10-09T11:04:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:51 smithi155 ceph-mon[184854]: 10.d starting backfill to osd.3 from (0'0,0'0] MAX to 484'12229 2023-10-09T11:04:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3061163574' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:04:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:51 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:04:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:51 smithi155 ceph-mon[184854]: failed to encode map e487 with expected crc 2023-10-09T11:04:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3061163574' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:04:52.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:51 smithi155 ceph-mon[184854]: osdmap e487: 8 total, 8 up, 7 in 2023-10-09T11:04:52.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:51 smithi155 ceph-mon[184854]: failed to encode map e487 with expected crc 2023-10-09T11:04:52.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:51 smithi155 ceph-mon[184854]: failed to encode map e487 with expected crc 2023-10-09T11:04:52.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:51 smithi155 ceph-mon[184854]: failed to encode map e487 with expected crc 2023-10-09T11:04:52.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:51 smithi155 ceph-mon[184854]: failed to encode map e487 with expected crc 2023-10-09T11:04:52.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:51 smithi155 ceph-mon[184854]: 10.f continuing backfill to osd.4 from (446'6721,484'12370] MIN to 484'12370 2023-10-09T11:04:52.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:51 smithi155 ceph-mon[184854]: 10.17 continuing backfill to osd.2 from (418'2375,484'9139] MIN to 484'9139 2023-10-09T11:04:52.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:51 smithi155 ceph-mon[184854]: 10.7 continuing backfill to osd.2 from (418'2640,484'9158] MIN to 484'9158 2023-10-09T11:04:52.223 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:52.253 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:52.273 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:52.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:52.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:52.377 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:52.406 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:52.525 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:04:52.547 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:04:52.579 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:04:52.599 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:04:52.620 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:04:52.639 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:04:52.660 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:04:52.681 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:04:52.682 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:04:52.682 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:04:52.682 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:04:52.683 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:04:52.683 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:04:52.683 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:04:52.683 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:04:52.683 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:04:52.730 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:52.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:52.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:52.834 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:52.873 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:53.049 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[195507]: failed to encode map e487 with expected crc 2023-10-09T11:04:53.049 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[195507]: 10.4 starting backfill to osd.3 from (0'0,448'6539] MIN to 484'9117 2023-10-09T11:04:53.049 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[195507]: 10.5 starting backfill to osd.3 from (0'0,448'6709] MIN to 484'9324 2023-10-09T11:04:53.049 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[195507]: pgmap v1197: 129 pgs: 1 active+recovering+remapped, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 remapped+peering, 2 active+clean+remapped, 1 peering, 3 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 107 active+clean; 52 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 462 op/s; 9747/2526 objects degraded (385.867%); 599/2526 objects misplaced (23.713%); 1.6 MiB/s, 240 objects/s recovering 2023-10-09T11:04:53.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[195507]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[195507]: osdmap e488: 8 total, 8 up, 7 in 2023-10-09T11:04:53.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[195507]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[195507]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[195507]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[195507]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[195507]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[195507]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[195507]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1003881716' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:53.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[200197]: failed to encode map e487 with expected crc 2023-10-09T11:04:53.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[200197]: 10.4 starting backfill to osd.3 from (0'0,448'6539] MIN to 484'9117 2023-10-09T11:04:53.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[200197]: 10.5 starting backfill to osd.3 from (0'0,448'6709] MIN to 484'9324 2023-10-09T11:04:53.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[200197]: pgmap v1197: 129 pgs: 1 active+recovering+remapped, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 remapped+peering, 2 active+clean+remapped, 1 peering, 3 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 107 active+clean; 52 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 462 op/s; 9747/2526 objects degraded (385.867%); 599/2526 objects misplaced (23.713%); 1.6 MiB/s, 240 objects/s recovering 2023-10-09T11:04:53.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[200197]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[200197]: osdmap e488: 8 total, 8 up, 7 in 2023-10-09T11:04:53.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[200197]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[200197]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[200197]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[200197]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.052 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[200197]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.052 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[200197]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.052 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[200197]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.052 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:52 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1003881716' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:53.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:52 smithi155 ceph-mon[184854]: failed to encode map e487 with expected crc 2023-10-09T11:04:53.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:52 smithi155 ceph-mon[184854]: 10.4 starting backfill to osd.3 from (0'0,448'6539] MIN to 484'9117 2023-10-09T11:04:53.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:52 smithi155 ceph-mon[184854]: 10.5 starting backfill to osd.3 from (0'0,448'6709] MIN to 484'9324 2023-10-09T11:04:53.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:52 smithi155 ceph-mon[184854]: pgmap v1197: 129 pgs: 1 active+recovering+remapped, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 remapped+peering, 2 active+clean+remapped, 1 peering, 3 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 107 active+clean; 52 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 462 op/s; 9747/2526 objects degraded (385.867%); 599/2526 objects misplaced (23.713%); 1.6 MiB/s, 240 objects/s recovering 2023-10-09T11:04:53.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:52 smithi155 ceph-mon[184854]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:52 smithi155 ceph-mon[184854]: osdmap e488: 8 total, 8 up, 7 in 2023-10-09T11:04:53.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:52 smithi155 ceph-mon[184854]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:52 smithi155 ceph-mon[184854]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:52 smithi155 ceph-mon[184854]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:52 smithi155 ceph-mon[184854]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:52 smithi155 ceph-mon[184854]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:52 smithi155 ceph-mon[184854]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:52 smithi155 ceph-mon[184854]: failed to encode map e488 with expected crc 2023-10-09T11:04:53.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:52 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1003881716' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:53.234 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Removed 98566 objects 2023-10-09T11:04:53.234 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Clean up completed and total clean up time :121.31 2023-10-09T11:04:53.529 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:53.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:53.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:53.633 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:53.661 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:54.204 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:04:54.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:53 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 9747/2526 objects degraded (385.867%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:04:54.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:53 smithi155 ceph-mon[184854]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:53 smithi155 ceph-mon[184854]: osdmap e489: 8 total, 8 up, 7 in 2023-10-09T11:04:54.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:53 smithi155 ceph-mon[184854]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:53 smithi155 ceph-mon[184854]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:53 smithi155 ceph-mon[184854]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:53 smithi155 ceph-mon[184854]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:53 smithi155 ceph-mon[184854]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.232 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 9747/2526 objects degraded (385.867%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:04:54.232 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[195507]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.232 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[195507]: osdmap e489: 8 total, 8 up, 7 in 2023-10-09T11:04:54.232 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[195507]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.232 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[195507]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.232 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[195507]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.233 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[195507]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.233 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[195507]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 9747/2526 objects degraded (385.867%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:04:54.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[200197]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[200197]: osdmap e489: 8 total, 8 up, 7 in 2023-10-09T11:04:54.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[200197]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[200197]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[200197]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[200197]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.234 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:53 smithi125 ceph-mon[200197]: failed to encode map e489 with expected crc 2023-10-09T11:04:54.495 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:54.845 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 7, 3] out_osds: [6] dead_osds: [] live_osds: [0, 2, 4, 5, 6, 7, 1, 3] 2023-10-09T11:04:54.845 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:04:54.845 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T11:04:54.845 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.000000 2023-10-09T11:04:54.846 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 0 2023-10-09T11:04:55.082 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:04:55.125 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[195507]: failed to encode map e489 with expected crc 2023-10-09T11:04:55.125 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[195507]: pgmap v1200: 129 pgs: 1 active+recovering+remapped, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 remapped+peering, 9 peering, 3 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 104 active+clean; 27 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 260 op/s; 10334/1347 objects degraded (767.186%); 6053/1347 objects misplaced (449.369%); 1.3 MiB/s, 236 objects/s recovering 2023-10-09T11:04:55.125 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[195507]: failed to encode map e490 with expected crc 2023-10-09T11:04:55.125 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[195507]: osdmap e490: 8 total, 8 up, 7 in 2023-10-09T11:04:55.125 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[195507]: failed to encode map e490 with expected crc 2023-10-09T11:04:55.126 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[195507]: failed to encode map e490 with expected crc 2023-10-09T11:04:55.126 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[195507]: failed to encode map e490 with expected crc 2023-10-09T11:04:55.126 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[195507]: failed to encode map e490 with expected crc 2023-10-09T11:04:55.126 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[200197]: failed to encode map e489 with expected crc 2023-10-09T11:04:55.126 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[200197]: pgmap v1200: 129 pgs: 1 active+recovering+remapped, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 remapped+peering, 9 peering, 3 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 104 active+clean; 27 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 260 op/s; 10334/1347 objects degraded (767.186%); 6053/1347 objects misplaced (449.369%); 1.3 MiB/s, 236 objects/s recovering 2023-10-09T11:04:55.126 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[200197]: failed to encode map e490 with expected crc 2023-10-09T11:04:55.126 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[200197]: osdmap e490: 8 total, 8 up, 7 in 2023-10-09T11:04:55.126 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[200197]: failed to encode map e490 with expected crc 2023-10-09T11:04:55.127 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[200197]: failed to encode map e490 with expected crc 2023-10-09T11:04:55.127 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[200197]: failed to encode map e490 with expected crc 2023-10-09T11:04:55.127 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:54 smithi125 ceph-mon[200197]: failed to encode map e490 with expected crc 2023-10-09T11:04:55.189 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:54 smithi155 ceph-mon[184854]: failed to encode map e489 with expected crc 2023-10-09T11:04:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:54 smithi155 ceph-mon[184854]: pgmap v1200: 129 pgs: 1 active+recovering+remapped, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 remapped+peering, 9 peering, 3 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 104 active+clean; 27 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 260 op/s; 10334/1347 objects degraded (767.186%); 6053/1347 objects misplaced (449.369%); 1.3 MiB/s, 236 objects/s recovering 2023-10-09T11:04:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:54 smithi155 ceph-mon[184854]: failed to encode map e490 with expected crc 2023-10-09T11:04:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:54 smithi155 ceph-mon[184854]: osdmap e490: 8 total, 8 up, 7 in 2023-10-09T11:04:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:54 smithi155 ceph-mon[184854]: failed to encode map e490 with expected crc 2023-10-09T11:04:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:54 smithi155 ceph-mon[184854]: failed to encode map e490 with expected crc 2023-10-09T11:04:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:54 smithi155 ceph-mon[184854]: failed to encode map e490 with expected crc 2023-10-09T11:04:55.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:54 smithi155 ceph-mon[184854]: failed to encode map e490 with expected crc 2023-10-09T11:04:55.279 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:55.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:55.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:55.383 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:55.411 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:55.440 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:55.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:55.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:55.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:55.576 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:55.700 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:55.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:55.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:55.804 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:04:55.832 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:55.925 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:56.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:55 smithi125 ceph-mon[195507]: failed to encode map e490 with expected crc 2023-10-09T11:04:56.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:55 smithi125 ceph-mon[200197]: failed to encode map e490 with expected crc 2023-10-09T11:04:56.058 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:04:56.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:56.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:04:56.162 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:56.185 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:55 smithi155 ceph-mon[184854]: failed to encode map e490 with expected crc 2023-10-09T11:04:56.194 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:56.416 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:56.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:56.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:56.470 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:04:56.470 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T11:04:56.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T11:04:56.470 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:04:56.470 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.470 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:148255 9.9 e367/363 9:9ff88bd4:::benchmark_data_smithi125_363789_object37956:head v 367'9433, mlcod=367'9431)", 2023-10-09T11:04:56.470 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:37.909582+0000", 2023-10-09T11:04:56.470 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 318.55791367400002, 2023-10-09T11:04:56.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.137790361, 2023-10-09T11:04:56.470 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.471 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.471 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.471 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.909582+0000", 2023-10-09T11:04:56.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.471 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.909582+0000", 2023-10-09T11:04:56.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.909582+0000", 2023-10-09T11:04:56.472 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967296 2023-10-09T11:04:56.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.909589+0000", 2023-10-09T11:04:56.472 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1130000000000003e-06 2023-10-09T11:04:56.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.909594+0000", 2023-10-09T11:04:56.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2499999999999997e-06 2023-10-09T11:04:56.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.909598+0000", 2023-10-09T11:04:56.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4060000000000001e-06 2023-10-09T11:04:56.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.910087+0000", 2023-10-09T11:04:56.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00048942300000000001 2023-10-09T11:04:56.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.910109+0000", 2023-10-09T11:04:56.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1722999999999999e-05 2023-10-09T11:04:56.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.916949+0000", 2023-10-09T11:04:56.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0068402970000000004 2023-10-09T11:04:56.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.475 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.475 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:38.047373+0000", 2023-10-09T11:04:56.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.130423332 2023-10-09T11:04:56.475 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.475 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.475 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.475 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:148262 9.8 e367/363 9:1d1af834:::benchmark_data_smithi125_363789_object37963:head v 367'9137, mlcod=367'9134)", 2023-10-09T11:04:56.475 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:59:37.910505+0000", 2023-10-09T11:04:56.475 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 318.55699073, 2023-10-09T11:04:56.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.136837452, 2023-10-09T11:04:56.476 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.476 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.476 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.476 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.476 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.910505+0000", 2023-10-09T11:04:56.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.476 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.476 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.910505+0000", 2023-10-09T11:04:56.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.910505+0000", 2023-10-09T11:04:56.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:04:56.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.910513+0000", 2023-10-09T11:04:56.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3799999999999994e-06 2023-10-09T11:04:56.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.910524+0000", 2023-10-09T11:04:56.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1241e-05 2023-10-09T11:04:56.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.910530+0000", 2023-10-09T11:04:56.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5849999999999999e-06 2023-10-09T11:04:56.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.910570+0000", 2023-10-09T11:04:56.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0225999999999997e-05 2023-10-09T11:04:56.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.910586+0000", 2023-10-09T11:04:56.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6143000000000001e-05 2023-10-09T11:04:56.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:37.916938+0000", 2023-10-09T11:04:56.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0063522179999999998 2023-10-09T11:04:56.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:59:38.047343+0000", 2023-10-09T11:04:56.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13040438500000001 2023-10-09T11:04:56.480 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.480 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.480 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.480 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45683.0:217562 9.a 9:5156b7da:::benchmark_data_smithi125_363789_object107263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-10-09T11:04:56.480 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:58.813901+0000", 2023-10-09T11:04:56.480 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 237.65359515399999, 2023-10-09T11:04:56.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18691018400000001, 2023-10-09T11:04:56.481 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.481 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.481 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:04:56.481 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45683", 2023-10-09T11:04:56.481 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:04:56.481 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 217562 2023-10-09T11:04:56.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.481 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.813901+0000", 2023-10-09T11:04:56.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.813901+0000", 2023-10-09T11:04:56.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.813899+0000", 2023-10-09T11:04:56.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:04:56.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.483 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.483 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.813905+0000", 2023-10-09T11:04:56.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4029999999999998e-06 2023-10-09T11:04:56.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.483 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.483 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.813915+0000", 2023-10-09T11:04:56.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0311e-05 2023-10-09T11:04:56.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.813920+0000", 2023-10-09T11:04:56.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9069999999999998e-06 2023-10-09T11:04:56.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.813963+0000", 2023-10-09T11:04:56.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3010999999999999e-05 2023-10-09T11:04:56.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.814056+0000", 2023-10-09T11:04:56.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3445999999999999e-05 2023-10-09T11:04:56.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,7", 2023-10-09T11:04:56.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.814131+0000", 2023-10-09T11:04:56.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4302999999999997e-05 2023-10-09T11:04:56.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:04:56.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.816968+0000", 2023-10-09T11:04:56.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002836884 2023-10-09T11:04:56.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.000189+0000", 2023-10-09T11:04:56.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18322112900000001 2023-10-09T11:04:56.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.000412+0000", 2023-10-09T11:04:56.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022303500000000001 2023-10-09T11:04:56.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.000471+0000", 2023-10-09T11:04:56.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8848999999999998e-05 2023-10-09T11:04:56.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.000811+0000", 2023-10-09T11:04:56.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00034039200000000002 2023-10-09T11:04:56.487 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.487 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.487 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.488 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop_reply(client.45683.0:217562 9.a e397/389 ondisk, result = 0)", 2023-10-09T11:04:56.488 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:58.816991+0000", 2023-10-09T11:04:56.488 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 237.65050532199999, 2023-10-09T11:04:56.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18321337500000001, 2023-10-09T11:04:56.488 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.488 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T11:04:56.488 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.816991+0000", 2023-10-09T11:04:56.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.816991+0000", 2023-10-09T11:04:56.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.816990+0000", 2023-10-09T11:04:56.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:04:56.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.816996+0000", 2023-10-09T11:04:56.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8930000000000002e-06 2023-10-09T11:04:56.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.490 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.490 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.490 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.817008+0000", 2023-10-09T11:04:56.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2366e-05 2023-10-09T11:04:56.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.490 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.490 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.490 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.817014+0000", 2023-10-09T11:04:56.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8549999999999998e-06 2023-10-09T11:04:56.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.490 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.817046+0000", 2023-10-09T11:04:56.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1766e-05 2023-10-09T11:04:56.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.817051+0000", 2023-10-09T11:04:56.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6890000000000001e-06 2023-10-09T11:04:56.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.000204+0000", 2023-10-09T11:04:56.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18315264000000001 2023-10-09T11:04:56.492 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.492 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.492 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.492 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:217560 9.5 e397/376 9:a4fceb2d:::benchmark_data_smithi125_363789_object107261:head v 397'11594, mlcod=397'11591)", 2023-10-09T11:04:56.492 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:58.819245+0000", 2023-10-09T11:04:56.492 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 237.64825059500001, 2023-10-09T11:04:56.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18562777, 2023-10-09T11:04:56.492 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.492 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.493 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.819245+0000", 2023-10-09T11:04:56.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.819245+0000", 2023-10-09T11:04:56.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.819245+0000", 2023-10-09T11:04:56.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:56.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.819251+0000", 2023-10-09T11:04:56.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6479999999999997e-06 2023-10-09T11:04:56.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.000192+0000", 2023-10-09T11:04:56.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.180940398 2023-10-09T11:04:56.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.000201+0000", 2023-10-09T11:04:56.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5750000000000007e-06 2023-10-09T11:04:56.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.000258+0000", 2023-10-09T11:04:56.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6314999999999999e-05 2023-10-09T11:04:56.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.000279+0000", 2023-10-09T11:04:56.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1203999999999999e-05 2023-10-09T11:04:56.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.004833+0000", 2023-10-09T11:04:56.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045543800000000002 2023-10-09T11:04:56.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.004873+0000", 2023-10-09T11:04:56.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9928999999999997e-05 2023-10-09T11:04:56.497 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.497 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.497 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.497 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:217556 9.f e397/389 9:f503d4b3:::benchmark_data_smithi125_363789_object107257:head v 397'13856, mlcod=357'7774)", 2023-10-09T11:04:56.497 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:58.820112+0000", 2023-10-09T11:04:56.497 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 237.647384045, 2023-10-09T11:04:56.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18576288699999999, 2023-10-09T11:04:56.498 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.498 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.498 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.498 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.498 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.820112+0000", 2023-10-09T11:04:56.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.498 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.498 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.820112+0000", 2023-10-09T11:04:56.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.820111+0000", 2023-10-09T11:04:56.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:56.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.820119+0000", 2023-10-09T11:04:56.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5259999999999996e-06 2023-10-09T11:04:56.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.000188+0000", 2023-10-09T11:04:56.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.180069336 2023-10-09T11:04:56.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.000200+0000", 2023-10-09T11:04:56.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1457000000000001e-05 2023-10-09T11:04:56.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.000242+0000", 2023-10-09T11:04:56.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2289000000000001e-05 2023-10-09T11:04:56.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.000272+0000", 2023-10-09T11:04:56.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9848999999999998e-05 2023-10-09T11:04:56.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.005839+0000", 2023-10-09T11:04:56.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.005566952 2023-10-09T11:04:56.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.005875+0000", 2023-10-09T11:04:56.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6158999999999997e-05 2023-10-09T11:04:56.502 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.502 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.502 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.502 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45683.0:217564 9.7 e397/369 9:e0d5129e:::benchmark_data_smithi125_363789_object107265:head v 397'11517, mlcod=397'11515)", 2023-10-09T11:04:56.502 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:00:58.821627+0000", 2023-10-09T11:04:56.503 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 237.64586923300001, 2023-10-09T11:04:56.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.184274351, 2023-10-09T11:04:56.503 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.503 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.503 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.821627+0000", 2023-10-09T11:04:56.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.821627+0000", 2023-10-09T11:04:56.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.821626+0000", 2023-10-09T11:04:56.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:56.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:58.821633+0000", 2023-10-09T11:04:56.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4060000000000002e-06 2023-10-09T11:04:56.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.505 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.505 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.000199+0000", 2023-10-09T11:04:56.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17856682600000001 2023-10-09T11:04:56.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.505 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.505 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.000208+0000", 2023-10-09T11:04:56.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6289999999999999e-06 2023-10-09T11:04:56.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.506 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.506 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.002863+0000", 2023-10-09T11:04:56.506 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026546220000000001 2023-10-09T11:04:56.506 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.506 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.506 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.506 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.002883+0000", 2023-10-09T11:04:56.506 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0387999999999999e-05 2023-10-09T11:04:56.506 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.506 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.506 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.507 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.005879+0000", 2023-10-09T11:04:56.507 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029960360000000001 2023-10-09T11:04:56.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.507 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.507 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:00:59.005901+0000", 2023-10-09T11:04:56.507 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2067999999999999e-05 2023-10-09T11:04:56.507 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.507 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.507 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.508 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:17842 10.9 10:9e537702:::benchmark_data_smithi125_402869_object17841:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T11:04:56.508 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:25.625511+0000", 2023-10-09T11:04:56.508 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 210.84198447599999, 2023-10-09T11:04:56.508 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0429926150000002, 2023-10-09T11:04:56.508 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.508 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:04:56.508 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:04:56.508 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:04:56.508 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:56.508 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 17842 2023-10-09T11:04:56.508 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.509 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.509 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625511+0000", 2023-10-09T11:04:56.509 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.509 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.509 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625511+0000", 2023-10-09T11:04:56.509 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.509 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.510 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.510 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625510+0000", 2023-10-09T11:04:56.510 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:04:56.510 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.510 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.510 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.510 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625561+0000", 2023-10-09T11:04:56.510 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1332999999999998e-05 2023-10-09T11:04:56.510 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.510 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.510 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.510 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625575+0000", 2023-10-09T11:04:56.510 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3494e-05 2023-10-09T11:04:56.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.511 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.511 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.511 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625581+0000", 2023-10-09T11:04:56.511 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1689999999999997e-06 2023-10-09T11:04:56.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.511 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.511 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.511 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625627+0000", 2023-10-09T11:04:56.511 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5561e-05 2023-10-09T11:04:56.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.511 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625730+0000", 2023-10-09T11:04:56.512 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103268 2023-10-09T11:04:56.512 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,7", 2023-10-09T11:04:56.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625821+0000", 2023-10-09T11:04:56.512 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1324999999999997e-05 2023-10-09T11:04:56.512 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:04:56.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.626893+0000", 2023-10-09T11:04:56.512 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001071734 2023-10-09T11:04:56.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.513 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.513 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.627060+0000", 2023-10-09T11:04:56.513 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000166935 2023-10-09T11:04:56.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.513 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.513 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.630997+0000", 2023-10-09T11:04:56.513 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039370630000000002 2023-10-09T11:04:56.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:04:56.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.631007+0000", 2023-10-09T11:04:56.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0301e-05 2023-10-09T11:04:56.514 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.514 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:26.629171+0000", 2023-10-09T11:04:56.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.99816343500000004 2023-10-09T11:04:56.514 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.514 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:04:56.515 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:26.629184+0000", 2023-10-09T11:04:56.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3192e-05 2023-10-09T11:04:56.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.515 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.515 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.668416+0000", 2023-10-09T11:04:56.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.039231926 2023-10-09T11:04:56.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.515 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.515 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.668504+0000", 2023-10-09T11:04:56.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8197999999999998e-05 2023-10-09T11:04:56.516 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.516 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.516 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.516 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.516 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.516 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:17873 10.9 10:95cd1191:::benchmark_data_smithi125_402869_object17872:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T11:04:56.516 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:25.660643+0000", 2023-10-09T11:04:56.516 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 210.806852781, 2023-10-09T11:04:56.516 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0183232979999999, 2023-10-09T11:04:56.516 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.516 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.516 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:04:56.517 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:04:56.517 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:56.517 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 17873 2023-10-09T11:04:56.517 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.517 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.517 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.517 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660643+0000", 2023-10-09T11:04:56.517 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.517 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.517 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.518 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660643+0000", 2023-10-09T11:04:56.518 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.518 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.518 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.518 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.518 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660643+0000", 2023-10-09T11:04:56.518 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:56.518 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.518 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.518 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.518 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660712+0000", 2023-10-09T11:04:56.519 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9728999999999996e-05 2023-10-09T11:04:56.519 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.519 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.519 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.519 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660721+0000", 2023-10-09T11:04:56.519 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8689999999999993e-06 2023-10-09T11:04:56.519 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.519 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.519 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.519 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660727+0000", 2023-10-09T11:04:56.519 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4430000000000002e-06 2023-10-09T11:04:56.519 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.520 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.520 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.520 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660785+0000", 2023-10-09T11:04:56.520 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7958999999999999e-05 2023-10-09T11:04:56.520 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.520 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.520 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:04:56.520 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660793+0000", 2023-10-09T11:04:56.520 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1980000000000001e-06 2023-10-09T11:04:56.520 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.520 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.520 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.521 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:26.629206+0000", 2023-10-09T11:04:56.521 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.96841274899999996 2023-10-09T11:04:56.521 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:04:56.521 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:26.629212+0000", 2023-10-09T11:04:56.521 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.81e-06 2023-10-09T11:04:56.521 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.521 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.668525+0000", 2023-10-09T11:04:56.521 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0393124840000001 2023-10-09T11:04:56.522 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.522 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.522 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.522 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.668619+0000", 2023-10-09T11:04:56.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3868000000000003e-05 2023-10-09T11:04:56.522 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.522 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.522 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,4", 2023-10-09T11:04:56.522 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.668736+0000", 2023-10-09T11:04:56.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000117102 2023-10-09T11:04:56.522 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:04:56.523 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.670900+0000", 2023-10-09T11:04:56.523 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021641379999999999 2023-10-09T11:04:56.523 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.523 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.671389+0000", 2023-10-09T11:04:56.523 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00048903200000000003 2023-10-09T11:04:56.523 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.524 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.678582+0000", 2023-10-09T11:04:56.524 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0071925959999999999 2023-10-09T11:04:56.524 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.524 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.524 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.524 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.678617+0000", 2023-10-09T11:04:56.524 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5842e-05 2023-10-09T11:04:56.524 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.524 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.524 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.524 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.678966+0000", 2023-10-09T11:04:56.524 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00034906200000000003 2023-10-09T11:04:56.525 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.525 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.525 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.525 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.525 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.525 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:17893 10.9 10:964b1fe0:::benchmark_data_smithi125_402869_object17892:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T11:04:56.525 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:25.752542+0000", 2023-10-09T11:04:56.525 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 210.71495383199999, 2023-10-09T11:04:56.525 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9269636349999999, 2023-10-09T11:04:56.525 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.525 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.525 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:04:56.526 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:04:56.526 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:56.526 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 17893 2023-10-09T11:04:56.526 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.526 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.526 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.526 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.526 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752542+0000", 2023-10-09T11:04:56.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.526 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.526 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.527 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.527 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752542+0000", 2023-10-09T11:04:56.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.527 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.527 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.527 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752541+0000", 2023-10-09T11:04:56.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:04:56.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.527 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.527 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.528 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752612+0000", 2023-10-09T11:04:56.528 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0902000000000007e-05 2023-10-09T11:04:56.528 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.528 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.528 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752626+0000", 2023-10-09T11:04:56.528 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3498e-05 2023-10-09T11:04:56.528 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.528 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.528 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752634+0000", 2023-10-09T11:04:56.528 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2910000000000007e-06 2023-10-09T11:04:56.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.529 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752687+0000", 2023-10-09T11:04:56.529 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2960000000000001e-05 2023-10-09T11:04:56.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.529 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:04:56.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752695+0000", 2023-10-09T11:04:56.529 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.072e-06 2023-10-09T11:04:56.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.530 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.530 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:26.629232+0000", 2023-10-09T11:04:56.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.87653732900000003 2023-10-09T11:04:56.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.530 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:04:56.530 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:26.629239+0000", 2023-10-09T11:04:56.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4590000000000003e-06 2023-10-09T11:04:56.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.530 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.531 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.669565+0000", 2023-10-09T11:04:56.531 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.040326184 2023-10-09T11:04:56.531 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.531 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.531 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.531 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.669680+0000", 2023-10-09T11:04:56.531 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000114832 2023-10-09T11:04:56.531 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.531 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.531 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,4", 2023-10-09T11:04:56.531 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.669797+0000", 2023-10-09T11:04:56.532 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000116857 2023-10-09T11:04:56.532 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.532 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.532 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:04:56.532 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.670905+0000", 2023-10-09T11:04:56.532 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011081909999999999 2023-10-09T11:04:56.532 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.532 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.532 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.532 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.671446+0000", 2023-10-09T11:04:56.532 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00054133399999999998 2023-10-09T11:04:56.532 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.533 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.533 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.533 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.679019+0000", 2023-10-09T11:04:56.533 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0075722669999999997 2023-10-09T11:04:56.533 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.533 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.533 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.533 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.679076+0000", 2023-10-09T11:04:56.533 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7707999999999997e-05 2023-10-09T11:04:56.533 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.533 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.533 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.534 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.679506+0000", 2023-10-09T11:04:56.534 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00042955000000000001 2023-10-09T11:04:56.534 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.534 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.534 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.534 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.534 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.534 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:90162 10.9 10:95d91b83:::benchmark_data_smithi125_402869_object90161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T11:04:56.534 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:02:37.122718+0000", 2023-10-09T11:04:56.534 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 139.344777806, 2023-10-09T11:04:56.534 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072702900000000001, 2023-10-09T11:04:56.535 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.535 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.535 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:04:56.535 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:04:56.535 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:56.535 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 90162 2023-10-09T11:04:56.535 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.535 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.535 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.535 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.535 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:37.122718+0000", 2023-10-09T11:04:56.536 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.536 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.536 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.536 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.536 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:37.122718+0000", 2023-10-09T11:04:56.536 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.536 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.536 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.536 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.536 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:37.122718+0000", 2023-10-09T11:04:56.536 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:56.536 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.537 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:37.122772+0000", 2023-10-09T11:04:56.537 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4245e-05 2023-10-09T11:04:56.537 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.537 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:37.122780+0000", 2023-10-09T11:04:56.537 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2490000000000001e-06 2023-10-09T11:04:56.537 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:37.122785+0000", 2023-10-09T11:04:56.538 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3630000000000004e-06 2023-10-09T11:04:56.538 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.538 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.538 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:37.131410+0000", 2023-10-09T11:04:56.538 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0086252849999999999 2023-10-09T11:04:56.538 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.538 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.538 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.539 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:37.131565+0000", 2023-10-09T11:04:56.539 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015522999999999999 2023-10-09T11:04:56.539 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.539 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.539 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,7", 2023-10-09T11:04:56.539 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:37.131748+0000", 2023-10-09T11:04:56.539 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018342599999999999 2023-10-09T11:04:56.539 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.539 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.539 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:04:56.539 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:37.137892+0000", 2023-10-09T11:04:56.539 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061432769999999999 2023-10-09T11:04:56.540 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.540 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.540 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:37.141681+0000", 2023-10-09T11:04:56.540 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0037891840000000001 2023-10-09T11:04:56.540 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.540 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.540 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:37.192066+0000", 2023-10-09T11:04:56.540 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.050384777999999998 2023-10-09T11:04:56.540 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.541 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:37.192130+0000", 2023-10-09T11:04:56.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4071000000000004e-05 2023-10-09T11:04:56.541 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.541 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.541 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:37.195421+0000", 2023-10-09T11:04:56.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003291296 2023-10-09T11:04:56.541 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.541 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.541 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.541 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.542 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.542 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:158833 10.19 10:9f817bea:::benchmark_data_smithi125_402869_object60264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2023-10-09T11:04:56.542 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:03:55.842319+0000", 2023-10-09T11:04:56.542 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 60.625176732, 2023-10-09T11:04:56.542 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.091711497000000003, 2023-10-09T11:04:56.542 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.542 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.542 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:04:56.542 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:04:56.542 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:56.542 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 158833 2023-10-09T11:04:56.542 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.543 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.543 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.842319+0000", 2023-10-09T11:04:56.543 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.543 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.842319+0000", 2023-10-09T11:04:56.543 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.842319+0000", 2023-10-09T11:04:56.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:56.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.842321+0000", 2023-10-09T11:04:56.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3760000000000002e-06 2023-10-09T11:04:56.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.842327+0000", 2023-10-09T11:04:56.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7119999999999997e-06 2023-10-09T11:04:56.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.842331+0000", 2023-10-09T11:04:56.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5760000000000002e-06 2023-10-09T11:04:56.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.846320+0000", 2023-10-09T11:04:56.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039897090000000001 2023-10-09T11:04:56.546 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.546 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.546 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.846457+0000", 2023-10-09T11:04:56.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013700999999999999 2023-10-09T11:04:56.546 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.546 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.546 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,7", 2023-10-09T11:04:56.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.846898+0000", 2023-10-09T11:04:56.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00044055899999999998 2023-10-09T11:04:56.547 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.547 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:04:56.547 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.849371+0000", 2023-10-09T11:04:56.547 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024732959999999998 2023-10-09T11:04:56.547 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.547 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.547 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.851459+0000", 2023-10-09T11:04:56.547 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020875630000000002 2023-10-09T11:04:56.547 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.548 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.548 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.933429+0000", 2023-10-09T11:04:56.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.081970304999999993 2023-10-09T11:04:56.548 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.548 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.548 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.548 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.933477+0000", 2023-10-09T11:04:56.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8130000000000002e-05 2023-10-09T11:04:56.548 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.548 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.548 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.548 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.934031+0000", 2023-10-09T11:04:56.549 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00055364200000000002 2023-10-09T11:04:56.549 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.549 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.549 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.549 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.549 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.549 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:180371 10.9 10:93cc9f44:::benchmark_data_smithi125_402869_object81802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T11:04:56.549 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:23.993531+0000", 2023-10-09T11:04:56.549 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 32.473964555999999, 2023-10-09T11:04:56.549 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.075856251, 2023-10-09T11:04:56.549 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.549 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.550 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:04:56.550 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:04:56.550 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:56.550 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 180371 2023-10-09T11:04:56.550 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.550 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.550 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.550 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.550 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:23.993531+0000", 2023-10-09T11:04:56.550 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.550 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.551 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.551 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.551 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:23.993531+0000", 2023-10-09T11:04:56.551 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.551 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.551 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.551 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.551 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:23.993531+0000", 2023-10-09T11:04:56.551 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:56.551 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.551 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.552 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:23.993535+0000", 2023-10-09T11:04:56.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5889999999999999e-06 2023-10-09T11:04:56.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.552 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:23.993543+0000", 2023-10-09T11:04:56.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9780000000000005e-06 2023-10-09T11:04:56.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.552 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:23.993546+0000", 2023-10-09T11:04:56.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.472e-06 2023-10-09T11:04:56.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:23.993609+0000", 2023-10-09T11:04:56.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2765e-05 2023-10-09T11:04:56.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:23.993702+0000", 2023-10-09T11:04:56.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2823000000000002e-05 2023-10-09T11:04:56.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,7", 2023-10-09T11:04:56.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:23.994180+0000", 2023-10-09T11:04:56.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00047835800000000002 2023-10-09T11:04:56.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:04:56.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:23.998534+0000", 2023-10-09T11:04:56.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043540949999999997 2023-10-09T11:04:56.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:23.999636+0000", 2023-10-09T11:04:56.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00110198 2023-10-09T11:04:56.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.069303+0000", 2023-10-09T11:04:56.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069666860999999997 2023-10-09T11:04:56.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.069349+0000", 2023-10-09T11:04:56.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5671999999999999e-05 2023-10-09T11:04:56.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.069388+0000", 2023-10-09T11:04:56.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9113000000000003e-05 2023-10-09T11:04:56.556 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.556 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.556 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.557 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:180889 10.7 10:e4b82f07:::benchmark_data_smithi125_402869_object82320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T11:04:56.557 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:24.612336+0000", 2023-10-09T11:04:56.557 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 31.855160273999999, 2023-10-09T11:04:56.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.111581625, 2023-10-09T11:04:56.557 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.557 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.557 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:04:56.557 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:04:56.557 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:56.557 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 180889 2023-10-09T11:04:56.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.558 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612336+0000", 2023-10-09T11:04:56.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612336+0000", 2023-10-09T11:04:56.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612335+0000", 2023-10-09T11:04:56.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:56.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612338+0000", 2023-10-09T11:04:56.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2339999999999999e-06 2023-10-09T11:04:56.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612352+0000", 2023-10-09T11:04:56.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3502e-05 2023-10-09T11:04:56.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612359+0000", 2023-10-09T11:04:56.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4159999999999998e-06 2023-10-09T11:04:56.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.616086+0000", 2023-10-09T11:04:56.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003726605 2023-10-09T11:04:56.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.616238+0000", 2023-10-09T11:04:56.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151903 2023-10-09T11:04:56.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T11:04:56.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.616775+0000", 2023-10-09T11:04:56.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00053741399999999995 2023-10-09T11:04:56.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.562 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.562 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.627396+0000", 2023-10-09T11:04:56.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010621116999999999 2023-10-09T11:04:56.562 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.562 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:04:56.562 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.627717+0000", 2023-10-09T11:04:56.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00032082500000000001 2023-10-09T11:04:56.562 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.634799+0000", 2023-10-09T11:04:56.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0070819309999999996 2023-10-09T11:04:56.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.723596+0000", 2023-10-09T11:04:56.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.088797290000000001 2023-10-09T11:04:56.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.723649+0000", 2023-10-09T11:04:56.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2398000000000002e-05 2023-10-09T11:04:56.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.723917+0000", 2023-10-09T11:04:56.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00026844599999999998 2023-10-09T11:04:56.564 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.564 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.564 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.565 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:180891 10.7 10:e7e2a040:::benchmark_data_smithi125_402869_object82322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T11:04:56.565 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:24.612393+0000", 2023-10-09T11:04:56.565 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 31.855102628000001, 2023-10-09T11:04:56.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.111745528, 2023-10-09T11:04:56.565 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.565 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.565 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:04:56.565 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:04:56.565 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:56.565 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 180891 2023-10-09T11:04:56.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.566 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612393+0000", 2023-10-09T11:04:56.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612393+0000", 2023-10-09T11:04:56.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.567 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.567 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.567 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612392+0000", 2023-10-09T11:04:56.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:04:56.567 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.567 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.567 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.567 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612400+0000", 2023-10-09T11:04:56.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7340000000000008e-06 2023-10-09T11:04:56.567 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.567 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612412+0000", 2023-10-09T11:04:56.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1547e-05 2023-10-09T11:04:56.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612419+0000", 2023-10-09T11:04:56.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8220000000000001e-06 2023-10-09T11:04:56.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.620506+0000", 2023-10-09T11:04:56.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0080873119999999993 2023-10-09T11:04:56.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.620627+0000", 2023-10-09T11:04:56.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000121055 2023-10-09T11:04:56.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T11:04:56.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.621101+0000", 2023-10-09T11:04:56.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00047359799999999999 2023-10-09T11:04:56.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.627743+0000", 2023-10-09T11:04:56.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0066426549999999999 2023-10-09T11:04:56.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:04:56.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.627758+0000", 2023-10-09T11:04:56.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4343e-05 2023-10-09T11:04:56.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.634858+0000", 2023-10-09T11:04:56.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0071002749999999996 2023-10-09T11:04:56.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.723962+0000", 2023-10-09T11:04:56.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.089104143999999996 2023-10-09T11:04:56.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.724005+0000", 2023-10-09T11:04:56.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.35e-05 2023-10-09T11:04:56.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.724139+0000", 2023-10-09T11:04:56.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000133406 2023-10-09T11:04:56.572 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.572 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.572 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.572 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:180893 10.17 10:eb2d38a4:::benchmark_data_smithi125_402869_object82324:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T11:04:56.573 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:24.612425+0000", 2023-10-09T11:04:56.573 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 31.855071317, 2023-10-09T11:04:56.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.111492518, 2023-10-09T11:04:56.573 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.573 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.573 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:04:56.573 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:04:56.573 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:56.573 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 180893 2023-10-09T11:04:56.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.573 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.574 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.574 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612425+0000", 2023-10-09T11:04:56.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.574 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.574 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.574 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612425+0000", 2023-10-09T11:04:56.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.574 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.574 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.575 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612424+0000", 2023-10-09T11:04:56.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:56.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.575 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612427+0000", 2023-10-09T11:04:56.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9359999999999999e-06 2023-10-09T11:04:56.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.575 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612434+0000", 2023-10-09T11:04:56.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4629999999999998e-06 2023-10-09T11:04:56.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.576 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612445+0000", 2023-10-09T11:04:56.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1797000000000001e-05 2023-10-09T11:04:56.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.576 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.619793+0000", 2023-10-09T11:04:56.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007347307 2023-10-09T11:04:56.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.619927+0000", 2023-10-09T11:04:56.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000134585 2023-10-09T11:04:56.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T11:04:56.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.620373+0000", 2023-10-09T11:04:56.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00044606099999999998 2023-10-09T11:04:56.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.627555+0000", 2023-10-09T11:04:56.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007181514 2023-10-09T11:04:56.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:04:56.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.627725+0000", 2023-10-09T11:04:56.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017040800000000001 2023-10-09T11:04:56.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.634831+0000", 2023-10-09T11:04:56.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0071059369999999997 2023-10-09T11:04:56.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.579 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.579 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.723625+0000", 2023-10-09T11:04:56.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.088793326000000006 2023-10-09T11:04:56.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.579 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.579 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.723658+0000", 2023-10-09T11:04:56.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3318000000000002e-05 2023-10-09T11:04:56.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.723917+0000", 2023-10-09T11:04:56.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025933599999999998 2023-10-09T11:04:56.580 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.580 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.580 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.580 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.580 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.580 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:180903 10.19 10:9b927c5a:::benchmark_data_smithi125_402869_object82334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T11:04:56.580 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:24.626809+0000", 2023-10-09T11:04:56.580 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 31.840686994999999, 2023-10-09T11:04:56.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.099327519000000003, 2023-10-09T11:04:56.581 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.581 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.581 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:04:56.581 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:04:56.581 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:56.581 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 180903 2023-10-09T11:04:56.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.581 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.581 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.581 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.581 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626809+0000", 2023-10-09T11:04:56.581 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.582 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.582 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.582 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626809+0000", 2023-10-09T11:04:56.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.582 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.582 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.582 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.582 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626808+0000", 2023-10-09T11:04:56.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:04:56.582 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.583 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.583 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626812+0000", 2023-10-09T11:04:56.583 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1370000000000004e-06 2023-10-09T11:04:56.583 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.583 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.583 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626825+0000", 2023-10-09T11:04:56.583 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.259e-05 2023-10-09T11:04:56.583 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.583 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.584 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626830+0000", 2023-10-09T11:04:56.584 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3399999999999997e-06 2023-10-09T11:04:56.584 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.584 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.584 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.584 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626876+0000", 2023-10-09T11:04:56.584 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5989999999999998e-05 2023-10-09T11:04:56.584 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.584 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.584 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.584 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626985+0000", 2023-10-09T11:04:56.584 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000109002 2023-10-09T11:04:56.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,7", 2023-10-09T11:04:56.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.627483+0000", 2023-10-09T11:04:56.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00049784199999999997 2023-10-09T11:04:56.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:04:56.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.630285+0000", 2023-10-09T11:04:56.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028015280000000002 2023-10-09T11:04:56.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.632992+0000", 2023-10-09T11:04:56.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0027076890000000001 2023-10-09T11:04:56.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.726053+0000", 2023-10-09T11:04:56.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.093060308999999994 2023-10-09T11:04:56.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.726100+0000", 2023-10-09T11:04:56.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7082000000000001e-05 2023-10-09T11:04:56.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.726136+0000", 2023-10-09T11:04:56.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6681999999999999e-05 2023-10-09T11:04:56.587 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.587 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.587 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.588 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.588 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:180912 10.19 10:9ed534bf:::benchmark_data_smithi125_402869_object82343:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T11:04:56.588 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:24.637080+0000", 2023-10-09T11:04:56.588 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 31.830416396, 2023-10-09T11:04:56.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.093240589999999998, 2023-10-09T11:04:56.588 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.588 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.588 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:04:56.588 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:04:56.588 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:56.588 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 180912 2023-10-09T11:04:56.588 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.589 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.589 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.589 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.589 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.637080+0000", 2023-10-09T11:04:56.589 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.589 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.589 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.589 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.589 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.637080+0000", 2023-10-09T11:04:56.589 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.590 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.637079+0000", 2023-10-09T11:04:56.590 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:04:56.590 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.637083+0000", 2023-10-09T11:04:56.590 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9080000000000003e-06 2023-10-09T11:04:56.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.591 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.591 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.637094+0000", 2023-10-09T11:04:56.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0482999999999999e-05 2023-10-09T11:04:56.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.591 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.591 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.637098+0000", 2023-10-09T11:04:56.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4270000000000001e-06 2023-10-09T11:04:56.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.637136+0000", 2023-10-09T11:04:56.592 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7738999999999999e-05 2023-10-09T11:04:56.592 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.592 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.637220+0000", 2023-10-09T11:04:56.592 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4092999999999996e-05 2023-10-09T11:04:56.592 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.592 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,7", 2023-10-09T11:04:56.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.637654+0000", 2023-10-09T11:04:56.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00043394000000000001 2023-10-09T11:04:56.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.593 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:04:56.593 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.639956+0000", 2023-10-09T11:04:56.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023018140000000001 2023-10-09T11:04:56.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.593 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.593 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.640193+0000", 2023-10-09T11:04:56.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00023712599999999999 2023-10-09T11:04:56.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.594 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.594 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.594 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.730244+0000", 2023-10-09T11:04:56.594 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.090051284999999995 2023-10-09T11:04:56.594 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.594 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.594 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.594 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.730283+0000", 2023-10-09T11:04:56.594 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8498999999999999e-05 2023-10-09T11:04:56.594 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.595 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.595 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.595 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.730320+0000", 2023-10-09T11:04:56.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7317000000000002e-05 2023-10-09T11:04:56.595 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.595 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.595 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.595 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.595 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.595 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:180914 10.9 10:97844566:::benchmark_data_smithi125_402869_object82345:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T11:04:56.595 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:24.641597+0000", 2023-10-09T11:04:56.595 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 31.825899020000001, 2023-10-09T11:04:56.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.088611157999999995, 2023-10-09T11:04:56.596 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.596 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.596 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:04:56.596 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:04:56.596 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:56.596 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 180914 2023-10-09T11:04:56.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.596 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.596 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.596 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.596 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.641597+0000", 2023-10-09T11:04:56.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.641597+0000", 2023-10-09T11:04:56.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.641595+0000", 2023-10-09T11:04:56.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:04:56.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.641604+0000", 2023-10-09T11:04:56.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9609999999999994e-06 2023-10-09T11:04:56.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.641616+0000", 2023-10-09T11:04:56.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1389000000000001e-05 2023-10-09T11:04:56.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.599 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.641620+0000", 2023-10-09T11:04:56.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4989999999999996e-06 2023-10-09T11:04:56.599 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.599 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.599 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.641661+0000", 2023-10-09T11:04:56.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0193e-05 2023-10-09T11:04:56.599 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.599 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.599 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.641743+0000", 2023-10-09T11:04:56.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2565000000000007e-05 2023-10-09T11:04:56.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.600 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,7", 2023-10-09T11:04:56.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.642141+0000", 2023-10-09T11:04:56.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00039770199999999999 2023-10-09T11:04:56.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.600 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:04:56.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.644480+0000", 2023-10-09T11:04:56.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023389299999999999 2023-10-09T11:04:56.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.645222+0000", 2023-10-09T11:04:56.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00074236300000000001 2023-10-09T11:04:56.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.730122+0000", 2023-10-09T11:04:56.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.084900130000000004 2023-10-09T11:04:56.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.730168+0000", 2023-10-09T11:04:56.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5847000000000003e-05 2023-10-09T11:04:56.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.730208+0000", 2023-10-09T11:04:56.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0030999999999997e-05 2023-10-09T11:04:56.602 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.602 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.602 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.603 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:180918 10.17 10:ef4afdba:::benchmark_data_smithi125_402869_object82349:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T11:04:56.603 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:24.646304+0000", 2023-10-09T11:04:56.603 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 31.821192008000001, 2023-10-09T11:04:56.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.082820221999999999, 2023-10-09T11:04:56.603 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:04:56.603 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:04:56.603 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:04:56.603 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:04:56.603 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:04:56.603 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 180918 2023-10-09T11:04:56.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.603 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:04:56.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:04:56.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.646304+0000", 2023-10-09T11:04:56.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:04:56.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.646304+0000", 2023-10-09T11:04:56.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:04:56.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:04:56.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.646303+0000", 2023-10-09T11:04:56.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:04:56.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:04:56.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.646308+0000", 2023-10-09T11:04:56.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9880000000000004e-06 2023-10-09T11:04:56.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:04:56.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.646320+0000", 2023-10-09T11:04:56.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2480000000000001e-05 2023-10-09T11:04:56.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:04:56.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.646326+0000", 2023-10-09T11:04:56.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.772e-06 2023-10-09T11:04:56.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:04:56.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.646365+0000", 2023-10-09T11:04:56.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8350999999999999e-05 2023-10-09T11:04:56.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:04:56.607 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.646473+0000", 2023-10-09T11:04:56.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010851799999999999 2023-10-09T11:04:56.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3,6", 2023-10-09T11:04:56.607 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.646881+0000", 2023-10-09T11:04:56.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000408269 2023-10-09T11:04:56.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:04:56.608 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.648927+0000", 2023-10-09T11:04:56.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002045757 2023-10-09T11:04:56.608 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.608 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.608 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.608 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.649397+0000", 2023-10-09T11:04:56.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00046976100000000002 2023-10-09T11:04:56.608 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.608 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.608 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.608 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.649667+0000", 2023-10-09T11:04:56.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00027054599999999998 2023-10-09T11:04:56.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:04:56.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.728847+0000", 2023-10-09T11:04:56.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.079179342999999999 2023-10-09T11:04:56.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:04:56.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.728890+0000", 2023-10-09T11:04:56.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3087000000000003e-05 2023-10-09T11:04:56.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:04:56.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:04:56.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:04:56.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.729124+0000", 2023-10-09T11:04:56.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00023436299999999999 2023-10-09T11:04:56.610 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.610 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.610 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.610 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:04:56.610 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:04:56.611 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:04:56.611 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:56.643 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:56.713 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:56.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:56.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:56.819 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:56.840 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T11:04:56.849 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:56.870 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:56.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:56.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:56.909 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:04:56.929 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:04:56.950 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.4 primary-affinity to 0 (802) 2023-10-09T11:04:56.952 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:04:56.957 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:04:56.976 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:56 smithi125 ceph-mon[200197]: pgmap v1202: 129 pgs: 1 active+recovering+remapped, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 remapped+peering, 9 peering, 3 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 104 active+clean; 27 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 175 op/s; 10334/1347 objects degraded (767.186%); 6053/1347 objects misplaced (449.369%); 840 KiB/s, 204 objects/s recovering 2023-10-09T11:04:56.976 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4193962440' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:56.976 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:56 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:56.976 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/305376352' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T11:04:56.977 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3778767093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:56.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:56 smithi125 ceph-mon[195507]: pgmap v1202: 129 pgs: 1 active+recovering+remapped, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 remapped+peering, 9 peering, 3 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 104 active+clean; 27 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 175 op/s; 10334/1347 objects degraded (767.186%); 6053/1347 objects misplaced (449.369%); 840 KiB/s, 204 objects/s recovering 2023-10-09T11:04:56.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4193962440' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:56.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:56 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:56.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/305376352' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T11:04:56.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3778767093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:56.978 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:04:56.979 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:04:56.997 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:04:57.007 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:57.017 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:04:57.027 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:04:57.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:57.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:04:57.037 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:04:57.056 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:04:57.057 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:04:57.058 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:04:57.058 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:04:57.058 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:04:57.058 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:04:57.058 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:04:57.058 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:04:57.058 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:04:57.131 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:57.134 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:04:57.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:56 smithi155 ceph-mon[184854]: pgmap v1202: 129 pgs: 1 active+recovering+remapped, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 remapped+peering, 9 peering, 3 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 104 active+clean; 27 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 175 op/s; 10334/1347 objects degraded (767.186%); 6053/1347 objects misplaced (449.369%); 840 KiB/s, 204 objects/s recovering 2023-10-09T11:04:57.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4193962440' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:57.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:56 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:04:57.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/305376352' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-09T11:04:57.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3778767093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:04:57.160 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:57.281 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:04:57.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:57.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:04:57.385 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:57.413 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:57.884 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:57.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:57.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:57 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:04:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:57 smithi155 ceph-mon[184854]: failed to encode map e491 with expected crc 2023-10-09T11:04:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:57 smithi155 ceph-mon[184854]: failed to encode map e491 with expected crc 2023-10-09T11:04:57.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:57 smithi155 ceph-mon[184854]: failed to encode map e491 with expected crc 2023-10-09T11:04:57.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:57 smithi155 ceph-mon[184854]: failed to encode map e491 with expected crc 2023-10-09T11:04:57.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:57 smithi155 ceph-mon[184854]: failed to encode map e491 with expected crc 2023-10-09T11:04:57.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:57 smithi155 ceph-mon[184854]: failed to encode map e491 with expected crc 2023-10-09T11:04:57.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:57 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:04:57.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:57 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/305376352' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-09T11:04:57.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:57 smithi155 ceph-mon[184854]: osdmap e491: 8 total, 8 up, 7 in 2023-10-09T11:04:57.987 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:58.012 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:04:58.012 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[195507]: failed to encode map e491 with expected crc 2023-10-09T11:04:58.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[195507]: failed to encode map e491 with expected crc 2023-10-09T11:04:58.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[195507]: failed to encode map e491 with expected crc 2023-10-09T11:04:58.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[195507]: failed to encode map e491 with expected crc 2023-10-09T11:04:58.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[195507]: failed to encode map e491 with expected crc 2023-10-09T11:04:58.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[195507]: failed to encode map e491 with expected crc 2023-10-09T11:04:58.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:04:58.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/305376352' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-09T11:04:58.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[195507]: osdmap e491: 8 total, 8 up, 7 in 2023-10-09T11:04:58.013 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:04:58.013 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[200197]: failed to encode map e491 with expected crc 2023-10-09T11:04:58.014 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[200197]: failed to encode map e491 with expected crc 2023-10-09T11:04:58.014 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[200197]: failed to encode map e491 with expected crc 2023-10-09T11:04:58.014 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[200197]: failed to encode map e491 with expected crc 2023-10-09T11:04:58.014 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[200197]: failed to encode map e491 with expected crc 2023-10-09T11:04:58.014 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[200197]: failed to encode map e491 with expected crc 2023-10-09T11:04:58.014 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:04:58.014 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/305376352' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-09T11:04:58.014 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:57 smithi125 ceph-mon[200197]: osdmap e491: 8 total, 8 up, 7 in 2023-10-09T11:04:58.015 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:58.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:58.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:58.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:58.086 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:04:58.086 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:04:58.138 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:58.164 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:58.183 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:58.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:58.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:58.287 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:04:58.314 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:58.335 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:04:58.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:58.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:04:58.439 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:58.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T11:04:58.465 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:58.475 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-10-09T11:04:58.475 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-10-09T11:04:58.485 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:58.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:58.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:58.589 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:04:58.619 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:58.638 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:04:58.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:58.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:04:58.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:04:58.750 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:04:58.762 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:04:58.771 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:58.798 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:04:58.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:04:58.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:04:58.918 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:04:58.952 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:04:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:58 smithi155 ceph-mon[184854]: pgmap v1204: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+remapped, 1 remapped+peering, 8 peering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 109 active+clean; 18 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 10327/885 objects degraded (1166.893%); 6061/885 objects misplaced (684.859%); 688 KiB/s, 210 objects/s recovering 2023-10-09T11:04:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:58 smithi155 ceph-mon[184854]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:58 smithi155 ceph-mon[184854]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:58 smithi155 ceph-mon[184854]: osdmap e492: 8 total, 8 up, 7 in 2023-10-09T11:04:59.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:58 smithi155 ceph-mon[184854]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:58 smithi155 ceph-mon[184854]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:58 smithi155 ceph-mon[184854]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:58 smithi155 ceph-mon[184854]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:58 smithi155 ceph-mon[184854]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:04:58 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 10327/885 objects degraded (1166.893%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:59.236 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[195507]: pgmap v1204: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+remapped, 1 remapped+peering, 8 peering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 109 active+clean; 18 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 10327/885 objects degraded (1166.893%); 6061/885 objects misplaced (684.859%); 688 KiB/s, 210 objects/s recovering 2023-10-09T11:04:59.236 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[195507]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[195507]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[195507]: osdmap e492: 8 total, 8 up, 7 in 2023-10-09T11:04:59.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[195507]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[195507]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[195507]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[195507]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[195507]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 10327/885 objects degraded (1166.893%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:59.237 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[200197]: pgmap v1204: 129 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+remapped, 1 remapped+peering, 8 peering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 109 active+clean; 18 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 10327/885 objects degraded (1166.893%); 6061/885 objects misplaced (684.859%); 688 KiB/s, 210 objects/s recovering 2023-10-09T11:04:59.237 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[200197]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.238 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[200197]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.238 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[200197]: osdmap e492: 8 total, 8 up, 7 in 2023-10-09T11:04:59.238 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[200197]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.238 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[200197]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.238 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[200197]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.238 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[200197]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.238 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[200197]: failed to encode map e492 with expected crc 2023-10-09T11:04:59.238 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:04:58 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 10327/885 objects degraded (1166.893%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T11:04:59.710 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:04:59.860 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:05:00.052 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:00.279 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:00.412 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:00.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:00.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:00.544 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:00.576 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:00.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:00.657 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:00.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:00.758 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:00.788 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:00.807 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:00.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:00.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:00.911 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:00.939 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:01.072 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_4' removed 2023-10-09T11:05:01.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:01 smithi155 ceph-mon[184854]: pgmap v1206: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 118 active+clean; 16 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 39 op/s; 10316/813 objects degraded (1268.881%); 6061/813 objects misplaced (745.510%); 0 B/s, 183 objects/s recovering 2023-10-09T11:05:01.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:01.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:05:01.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3074663393' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:05:01.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:01 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:05:01.302 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:05:01.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:01 smithi125 ceph-mon[195507]: pgmap v1206: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 118 active+clean; 16 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 39 op/s; 10316/813 objects degraded (1268.881%); 6061/813 objects misplaced (745.510%); 0 B/s, 183 objects/s recovering 2023-10-09T11:05:01.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:01.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:05:01.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3074663393' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:05:01.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:01 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:05:01.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:01 smithi125 ceph-mon[200197]: pgmap v1206: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 118 active+clean; 16 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 39 op/s; 10316/813 objects degraded (1268.881%); 6061/813 objects misplaced (745.510%); 0 B/s, 183 objects/s recovering 2023-10-09T11:05:01.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:01.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:05:01.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3074663393' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:05:01.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:01 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:05:01.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:05:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:05:01] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.43.0" 2023-10-09T11:05:01.330 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:05:01.355 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:05:01.387 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:05:01.410 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:05:01.430 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:05:01.450 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:05:01.466 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:01.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:01.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:01.490 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:05:01.496 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:05:01.496 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:05:01.496 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:05:01.496 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:05:01.497 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:05:01.497 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:05:01.497 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:05:01.497 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:05:01.573 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:01.588 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T11:05:01.589 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T11:05:01.589 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-10-09T11:05:01.589 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-10-09T11:05:01.613 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:01.736 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:05:01.736 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:05:01.736 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T11:05:01.736 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T11:05:01.736 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:05:01.990 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:01.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:01.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:02.061 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:02.086 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:05:02.094 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:02.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T11:05:02.121 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:02.209 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:02 smithi125 ceph-mon[195507]: failed to encode map e493 with expected crc 2023-10-09T11:05:02.209 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:02 smithi125 ceph-mon[195507]: failed to encode map e493 with expected crc 2023-10-09T11:05:02.209 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:02 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T11:05:02.210 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:02 smithi125 ceph-mon[195507]: osdmap e493: 8 total, 8 up, 7 in 2023-10-09T11:05:02.210 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:02 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/338239910' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:02.210 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:02 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4020437466' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:05:02.210 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:02 smithi125 ceph-mon[200197]: failed to encode map e493 with expected crc 2023-10-09T11:05:02.210 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:02 smithi125 ceph-mon[200197]: failed to encode map e493 with expected crc 2023-10-09T11:05:02.210 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:02 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T11:05:02.210 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:02 smithi125 ceph-mon[200197]: osdmap e493: 8 total, 8 up, 7 in 2023-10-09T11:05:02.210 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:02 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/338239910' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:02.210 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:02 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4020437466' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:05:02.291 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:02.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:02.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:02.379 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:05:02.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:02.422 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:02.467 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:02 smithi155 ceph-mon[184854]: failed to encode map e493 with expected crc 2023-10-09T11:05:02.467 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:02 smithi155 ceph-mon[184854]: failed to encode map e493 with expected crc 2023-10-09T11:05:02.467 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:02 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T11:05:02.467 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:02 smithi155 ceph-mon[184854]: osdmap e493: 8 total, 8 up, 7 in 2023-10-09T11:05:02.467 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:02 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/338239910' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:02.467 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:02 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4020437466' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:05:02.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:02.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:02.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:02.568 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 7, 3] out_osds: [6] dead_osds: [] live_osds: [0, 2, 4, 5, 6, 7, 1, 3] 2023-10-09T11:05:02.568 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:05:02.569 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T11:05:02.569 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T11:05:02.569 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T11:05:02.569 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T11:05:02.645 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:02.672 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:02.695 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:02.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:02.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:02.799 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:02.826 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:02.947 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:02.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:02.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:03.008 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:05:03.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:03.082 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:03.153 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:03.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:03.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:03.257 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:03.284 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:03.303 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:03.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:03.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:03.353 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:05:03.353 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T11:05:03.353 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T11:05:03.353 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:05:03.353 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.353 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:15613 10.a 10:5bf56187:::benchmark_data_smithi125_402869_object15612:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T11:05:03.353 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:24.488022+0000", 2023-10-09T11:05:03.354 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 218.86269477100001, 2023-10-09T11:05:03.354 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10172471299999999, 2023-10-09T11:05:03.354 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.354 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.354 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.354 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.354 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.354 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 15613 2023-10-09T11:05:03.354 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.354 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.354 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.355 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.355 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488022+0000", 2023-10-09T11:05:03.355 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.355 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.355 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.355 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.355 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488022+0000", 2023-10-09T11:05:03.355 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.355 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.355 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.355 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.356 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488022+0000", 2023-10-09T11:05:03.356 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:03.356 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.356 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.356 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.356 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488055+0000", 2023-10-09T11:05:03.356 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3102999999999998e-05 2023-10-09T11:05:03.356 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.356 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.356 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.356 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488065+0000", 2023-10-09T11:05:03.356 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0101e-05 2023-10-09T11:05:03.357 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.357 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.357 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.357 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488070+0000", 2023-10-09T11:05:03.357 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2909999999999998e-06 2023-10-09T11:05:03.357 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.357 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.357 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.357 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488112+0000", 2023-10-09T11:05:03.357 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1662000000000002e-05 2023-10-09T11:05:03.357 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.358 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.358 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.358 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488280+0000", 2023-10-09T11:05:03.358 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016848699999999999 2023-10-09T11:05:03.358 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.358 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.358 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:05:03.358 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488418+0000", 2023-10-09T11:05:03.358 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013803700000000001 2023-10-09T11:05:03.358 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.358 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.359 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.359 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.491088+0000", 2023-10-09T11:05:03.359 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026694919999999999 2023-10-09T11:05:03.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.359 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.359 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.359 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.491804+0000", 2023-10-09T11:05:03.359 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00071589899999999998 2023-10-09T11:05:03.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.359 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.359 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.360 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.589653+0000", 2023-10-09T11:05:03.360 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.097848911999999996 2023-10-09T11:05:03.360 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.360 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.360 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.360 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.589707+0000", 2023-10-09T11:05:03.360 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4593999999999999e-05 2023-10-09T11:05:03.360 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.360 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.360 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.360 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.589747+0000", 2023-10-09T11:05:03.360 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9787999999999999e-05 2023-10-09T11:05:03.361 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.361 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.361 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.361 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.361 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.361 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:27693 10.0 10:017bad4c:::benchmark_data_smithi125_402869_object27692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:05:03.361 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:34.474346+0000", 2023-10-09T11:05:03.361 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 208.876371307, 2023-10-09T11:05:03.361 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14285995000000001, 2023-10-09T11:05:03.361 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.361 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.361 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.362 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.362 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.362 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 27693 2023-10-09T11:05:03.362 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.362 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.362 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.362 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.362 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474346+0000", 2023-10-09T11:05:03.362 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.362 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.362 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.362 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.363 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474346+0000", 2023-10-09T11:05:03.363 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.363 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.363 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.363 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.363 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474345+0000", 2023-10-09T11:05:03.363 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:05:03.363 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.363 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.363 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.363 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474386+0000", 2023-10-09T11:05:03.363 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1455000000000002e-05 2023-10-09T11:05:03.364 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.364 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.364 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.364 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474401+0000", 2023-10-09T11:05:03.364 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5238e-05 2023-10-09T11:05:03.364 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.364 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.364 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.364 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474408+0000", 2023-10-09T11:05:03.364 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4489999999999996e-06 2023-10-09T11:05:03.364 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.364 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.365 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.365 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474455+0000", 2023-10-09T11:05:03.365 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7719999999999997e-05 2023-10-09T11:05:03.365 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.365 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.365 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.365 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474565+0000", 2023-10-09T11:05:03.365 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010948200000000001 2023-10-09T11:05:03.365 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.365 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.365 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:05:03.365 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474666+0000", 2023-10-09T11:05:03.366 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010145399999999999 2023-10-09T11:05:03.366 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.366 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.366 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.366 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.476207+0000", 2023-10-09T11:05:03.366 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015401950000000001 2023-10-09T11:05:03.366 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.366 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.366 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.366 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.476567+0000", 2023-10-09T11:05:03.366 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00036023899999999999 2023-10-09T11:05:03.367 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.367 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.367 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.367 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617122+0000", 2023-10-09T11:05:03.367 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14055489700000001 2023-10-09T11:05:03.367 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.367 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.367 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.367 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617168+0000", 2023-10-09T11:05:03.367 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6669000000000003e-05 2023-10-09T11:05:03.367 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.367 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.368 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.368 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617206+0000", 2023-10-09T11:05:03.368 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7505000000000001e-05 2023-10-09T11:05:03.368 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.368 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.368 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.368 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.368 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.368 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:27705 10.0 10:03463076:::benchmark_data_smithi125_402869_object27704:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:05:03.368 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:34.483244+0000", 2023-10-09T11:05:03.368 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 208.867473652, 2023-10-09T11:05:03.368 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13409575700000001, 2023-10-09T11:05:03.369 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.369 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.369 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.369 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.369 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.369 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 27705 2023-10-09T11:05:03.369 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.369 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.369 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.369 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.369 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483244+0000", 2023-10-09T11:05:03.369 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.370 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.370 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.370 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.370 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483244+0000", 2023-10-09T11:05:03.370 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.370 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.370 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.370 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.370 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483242+0000", 2023-10-09T11:05:03.370 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:05:03.370 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.370 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.371 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.371 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483306+0000", 2023-10-09T11:05:03.371 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3712000000000003e-05 2023-10-09T11:05:03.371 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.371 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.371 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.371 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483319+0000", 2023-10-09T11:05:03.371 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2860999999999999e-05 2023-10-09T11:05:03.371 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.371 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.371 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.371 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483325+0000", 2023-10-09T11:05:03.371 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0889999999999996e-06 2023-10-09T11:05:03.372 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.372 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.372 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.372 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483360+0000", 2023-10-09T11:05:03.372 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5259999999999998e-05 2023-10-09T11:05:03.372 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.372 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.372 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.372 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483505+0000", 2023-10-09T11:05:03.372 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014444300000000001 2023-10-09T11:05:03.372 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.373 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.373 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:05:03.373 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483657+0000", 2023-10-09T11:05:03.373 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151726 2023-10-09T11:05:03.373 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.373 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.373 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.373 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.484799+0000", 2023-10-09T11:05:03.373 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001142503 2023-10-09T11:05:03.373 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.373 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.373 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.373 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.485118+0000", 2023-10-09T11:05:03.374 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000318676 2023-10-09T11:05:03.374 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.374 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.374 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.374 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617247+0000", 2023-10-09T11:05:03.374 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.132128987 2023-10-09T11:05:03.374 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.374 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.374 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.374 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617299+0000", 2023-10-09T11:05:03.374 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1867000000000003e-05 2023-10-09T11:05:03.375 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.375 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.375 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.375 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617339+0000", 2023-10-09T11:05:03.375 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0726999999999998e-05 2023-10-09T11:05:03.375 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.375 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.375 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.375 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.375 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.375 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:27738 10.0 10:0a0f1cf2:::benchmark_data_smithi125_402869_object27737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:05:03.375 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:34.503650+0000", 2023-10-09T11:05:03.375 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 208.84706695, 2023-10-09T11:05:03.376 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.115650268, 2023-10-09T11:05:03.376 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.376 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.376 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.376 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.376 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.376 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 27738 2023-10-09T11:05:03.376 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.376 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.376 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.376 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.376 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503650+0000", 2023-10-09T11:05:03.377 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.377 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.377 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.377 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.377 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503650+0000", 2023-10-09T11:05:03.377 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.377 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.377 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.377 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.377 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503649+0000", 2023-10-09T11:05:03.377 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:05:03.378 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.378 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.378 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.378 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503716+0000", 2023-10-09T11:05:03.378 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7002999999999995e-05 2023-10-09T11:05:03.378 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.378 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.378 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.378 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503726+0000", 2023-10-09T11:05:03.378 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9170000000000001e-06 2023-10-09T11:05:03.378 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.378 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.379 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.379 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503732+0000", 2023-10-09T11:05:03.379 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2009999999999996e-06 2023-10-09T11:05:03.379 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.379 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.379 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.379 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503776+0000", 2023-10-09T11:05:03.379 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4100000000000001e-05 2023-10-09T11:05:03.379 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.379 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.379 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.379 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503892+0000", 2023-10-09T11:05:03.380 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115704 2023-10-09T11:05:03.380 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.380 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.380 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:05:03.380 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.504031+0000", 2023-10-09T11:05:03.380 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013921300000000001 2023-10-09T11:05:03.380 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.380 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.380 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.380 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.505195+0000", 2023-10-09T11:05:03.380 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001163765 2023-10-09T11:05:03.380 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.380 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.381 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.381 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.505623+0000", 2023-10-09T11:05:03.381 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00042792200000000001 2023-10-09T11:05:03.381 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.381 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.381 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.381 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.619243+0000", 2023-10-09T11:05:03.381 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11362040900000001 2023-10-09T11:05:03.381 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.381 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.381 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.381 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.619272+0000", 2023-10-09T11:05:03.382 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8118000000000001e-05 2023-10-09T11:05:03.382 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.382 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.382 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.382 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.619301+0000", 2023-10-09T11:05:03.382 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8958e-05 2023-10-09T11:05:03.382 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.382 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.382 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.382 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.382 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.382 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:34496 10.e 10:7e8e0fe7:::benchmark_data_smithi125_402869_object34495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:05:03.383 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:39.324998+0000", 2023-10-09T11:05:03.383 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 204.025718856, 2023-10-09T11:05:03.383 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.098875269000000002, 2023-10-09T11:05:03.383 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.383 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.383 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.383 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.383 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.383 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 34496 2023-10-09T11:05:03.383 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.383 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.383 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.384 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.384 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.324998+0000", 2023-10-09T11:05:03.384 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.384 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.384 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.384 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.384 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.324998+0000", 2023-10-09T11:05:03.384 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.384 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.384 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.384 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.384 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.324998+0000", 2023-10-09T11:05:03.385 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:03.385 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.385 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.385 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.385 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325045+0000", 2023-10-09T11:05:03.385 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6817000000000003e-05 2023-10-09T11:05:03.385 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.385 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.385 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.385 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325053+0000", 2023-10-09T11:05:03.385 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8709999999999997e-06 2023-10-09T11:05:03.385 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.386 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.386 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.386 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325057+0000", 2023-10-09T11:05:03.386 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6120000000000001e-06 2023-10-09T11:05:03.386 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.386 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.386 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.386 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325108+0000", 2023-10-09T11:05:03.386 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1187000000000003e-05 2023-10-09T11:05:03.386 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.386 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.386 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.387 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325227+0000", 2023-10-09T11:05:03.387 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118689 2023-10-09T11:05:03.387 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.387 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.387 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,4", 2023-10-09T11:05:03.387 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325363+0000", 2023-10-09T11:05:03.387 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000135735 2023-10-09T11:05:03.387 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.387 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.387 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.387 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327640+0000", 2023-10-09T11:05:03.387 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022776680000000001 2023-10-09T11:05:03.388 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.388 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.388 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.388 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327910+0000", 2023-10-09T11:05:03.388 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00026939599999999998 2023-10-09T11:05:03.388 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.388 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.388 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.388 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.423777+0000", 2023-10-09T11:05:03.388 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.095867252999999999 2023-10-09T11:05:03.388 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.388 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.389 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.389 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.423833+0000", 2023-10-09T11:05:03.389 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5897999999999999e-05 2023-10-09T11:05:03.389 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.389 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.389 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.389 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.423874+0000", 2023-10-09T11:05:03.389 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0713000000000001e-05 2023-10-09T11:05:03.389 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.389 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.389 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.389 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.390 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.390 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:34497 10.a 10:5b894bd9:::benchmark_data_smithi125_402869_object34496:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:05:03.390 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:39.325075+0000", 2023-10-09T11:05:03.390 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 204.02564222000001, 2023-10-09T11:05:03.390 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.099410517000000004, 2023-10-09T11:05:03.390 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.390 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.390 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.390 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.390 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.390 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 34497 2023-10-09T11:05:03.390 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.390 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.391 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.391 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.391 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325075+0000", 2023-10-09T11:05:03.391 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.391 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.391 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.391 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.391 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325075+0000", 2023-10-09T11:05:03.391 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.391 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.391 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.391 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.392 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325074+0000", 2023-10-09T11:05:03.392 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:03.392 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.392 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.392 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.392 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325125+0000", 2023-10-09T11:05:03.392 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0782999999999998e-05 2023-10-09T11:05:03.392 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.392 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.392 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.392 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325144+0000", 2023-10-09T11:05:03.392 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9225e-05 2023-10-09T11:05:03.393 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.393 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.393 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.393 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325149+0000", 2023-10-09T11:05:03.393 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4429999999999996e-06 2023-10-09T11:05:03.393 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.393 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.393 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.393 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325342+0000", 2023-10-09T11:05:03.393 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00019310200000000001 2023-10-09T11:05:03.393 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.393 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.394 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.394 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325474+0000", 2023-10-09T11:05:03.394 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013155999999999999 2023-10-09T11:05:03.394 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.394 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.394 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:05:03.394 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325610+0000", 2023-10-09T11:05:03.394 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000136954 2023-10-09T11:05:03.394 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.394 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.394 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.394 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327774+0000", 2023-10-09T11:05:03.394 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021631739999999999 2023-10-09T11:05:03.395 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.395 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.395 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.395 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327960+0000", 2023-10-09T11:05:03.395 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018653499999999999 2023-10-09T11:05:03.395 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.395 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.395 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.395 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424397+0000", 2023-10-09T11:05:03.395 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.096436500999999994 2023-10-09T11:05:03.395 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.396 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.396 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.396 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424439+0000", 2023-10-09T11:05:03.396 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2216999999999999e-05 2023-10-09T11:05:03.396 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.396 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.396 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.396 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424486+0000", 2023-10-09T11:05:03.396 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6650000000000002e-05 2023-10-09T11:05:03.396 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.396 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.396 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.396 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.397 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.397 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:34498 10.0 10:05649a4c:::benchmark_data_smithi125_402869_object34497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:05:03.397 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:39.325163+0000", 2023-10-09T11:05:03.397 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 204.02555448199999, 2023-10-09T11:05:03.397 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.100486519, 2023-10-09T11:05:03.397 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.397 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.397 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.397 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.397 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.397 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 34498 2023-10-09T11:05:03.398 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.398 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.398 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.398 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.398 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325163+0000", 2023-10-09T11:05:03.398 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.398 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.398 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.398 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.398 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325163+0000", 2023-10-09T11:05:03.398 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.398 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.399 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.399 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.399 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325162+0000", 2023-10-09T11:05:03.399 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:03.399 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.399 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.399 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.399 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325219+0000", 2023-10-09T11:05:03.399 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6532e-05 2023-10-09T11:05:03.399 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.399 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.399 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.400 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325227+0000", 2023-10-09T11:05:03.400 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8269999999999998e-06 2023-10-09T11:05:03.400 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.400 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.400 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.400 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325233+0000", 2023-10-09T11:05:03.400 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1909999999999997e-06 2023-10-09T11:05:03.400 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.400 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.400 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.400 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.326614+0000", 2023-10-09T11:05:03.401 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00138123 2023-10-09T11:05:03.401 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.401 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.401 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.401 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.326735+0000", 2023-10-09T11:05:03.401 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012105800000000001 2023-10-09T11:05:03.401 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.401 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.401 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:05:03.401 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.326862+0000", 2023-10-09T11:05:03.401 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000126699 2023-10-09T11:05:03.401 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.401 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.402 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.402 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.328648+0000", 2023-10-09T11:05:03.402 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0017860949999999999 2023-10-09T11:05:03.402 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.402 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.402 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.402 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.329261+0000", 2023-10-09T11:05:03.402 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00061273000000000002 2023-10-09T11:05:03.402 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.402 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.402 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.402 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.330863+0000", 2023-10-09T11:05:03.403 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001602612 2023-10-09T11:05:03.403 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.403 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.403 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.403 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.330908+0000", 2023-10-09T11:05:03.403 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4719000000000003e-05 2023-10-09T11:05:03.403 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.403 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.403 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.403 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.425649+0000", 2023-10-09T11:05:03.403 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.094741362999999995 2023-10-09T11:05:03.403 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.404 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.404 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.404 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.404 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.404 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:34499 10.a 10:5b46ec36:::benchmark_data_smithi125_402869_object34498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:05:03.404 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:39.325241+0000", 2023-10-09T11:05:03.404 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 204.025476529, 2023-10-09T11:05:03.404 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.099333877000000001, 2023-10-09T11:05:03.404 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.404 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.404 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.404 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.405 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.405 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 34499 2023-10-09T11:05:03.405 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.405 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.405 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.405 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.405 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325241+0000", 2023-10-09T11:05:03.405 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.405 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.405 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.405 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.405 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325241+0000", 2023-10-09T11:05:03.405 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.406 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.406 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.406 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.406 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325240+0000", 2023-10-09T11:05:03.406 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:03.406 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.406 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.406 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.406 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325273+0000", 2023-10-09T11:05:03.406 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3108000000000002e-05 2023-10-09T11:05:03.406 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.406 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.407 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.407 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325281+0000", 2023-10-09T11:05:03.407 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2400000000000001e-06 2023-10-09T11:05:03.407 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.407 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.407 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.407 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325285+0000", 2023-10-09T11:05:03.407 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.408e-06 2023-10-09T11:05:03.407 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.407 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.407 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.407 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.326873+0000", 2023-10-09T11:05:03.408 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015876180000000001 2023-10-09T11:05:03.408 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.408 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.408 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.408 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327003+0000", 2023-10-09T11:05:03.408 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013045099999999999 2023-10-09T11:05:03.408 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.408 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.408 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:05:03.408 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327094+0000", 2023-10-09T11:05:03.408 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1046999999999998e-05 2023-10-09T11:05:03.408 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.409 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.409 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.409 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.329164+0000", 2023-10-09T11:05:03.409 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020697559999999999 2023-10-09T11:05:03.409 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.409 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.409 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.409 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424370+0000", 2023-10-09T11:05:03.409 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.095206683 2023-10-09T11:05:03.409 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.409 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.409 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.410 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424514+0000", 2023-10-09T11:05:03.410 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000143547 2023-10-09T11:05:03.410 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.410 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.410 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.410 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424547+0000", 2023-10-09T11:05:03.410 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3194000000000001e-05 2023-10-09T11:05:03.410 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.410 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.410 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.410 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424575+0000", 2023-10-09T11:05:03.410 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7362999999999999e-05 2023-10-09T11:05:03.411 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.411 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.411 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.411 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.411 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.411 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:77753 10.f 10:f688b427:::benchmark_data_smithi125_402869_object77752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-10-09T11:05:03.411 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:02:21.288500+0000", 2023-10-09T11:05:03.411 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 162.06221693000001, 2023-10-09T11:05:03.411 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.111923807, 2023-10-09T11:05:03.411 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.411 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.411 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.412 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.412 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.412 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 77753 2023-10-09T11:05:03.412 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.412 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.412 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.412 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.412 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288500+0000", 2023-10-09T11:05:03.412 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.412 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.412 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.412 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.413 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288500+0000", 2023-10-09T11:05:03.413 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.413 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.413 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.413 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.413 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288500+0000", 2023-10-09T11:05:03.413 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:03.413 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.413 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.413 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.413 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288535+0000", 2023-10-09T11:05:03.413 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5206999999999997e-05 2023-10-09T11:05:03.413 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.414 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.414 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.414 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288544+0000", 2023-10-09T11:05:03.414 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5019999999999998e-06 2023-10-09T11:05:03.414 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.414 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.414 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.414 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288550+0000", 2023-10-09T11:05:03.414 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4079999999999997e-06 2023-10-09T11:05:03.414 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.414 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.414 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.415 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288589+0000", 2023-10-09T11:05:03.415 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9366000000000002e-05 2023-10-09T11:05:03.415 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.415 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.415 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.415 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288708+0000", 2023-10-09T11:05:03.415 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118998 2023-10-09T11:05:03.415 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.415 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.415 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T11:05:03.415 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288884+0000", 2023-10-09T11:05:03.415 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017539000000000001 2023-10-09T11:05:03.416 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.416 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.416 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.416 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.292679+0000", 2023-10-09T11:05:03.416 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003795138 2023-10-09T11:05:03.416 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.416 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.416 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.416 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.296941+0000", 2023-10-09T11:05:03.416 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042627710000000003 2023-10-09T11:05:03.416 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.416 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.417 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.417 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.400350+0000", 2023-10-09T11:05:03.417 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.103408823 2023-10-09T11:05:03.417 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.417 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.417 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.417 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.400392+0000", 2023-10-09T11:05:03.417 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1792000000000002e-05 2023-10-09T11:05:03.417 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.417 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.417 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.417 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.400424+0000", 2023-10-09T11:05:03.418 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2053e-05 2023-10-09T11:05:03.418 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.418 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.418 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.418 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.418 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.418 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:158832 10.a 10:5433f58d:::benchmark_data_smithi125_402869_object60263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2023-10-09T11:05:03.418 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:03:55.841518+0000", 2023-10-09T11:05:03.418 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 67.509198968999996, 2023-10-09T11:05:03.418 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.116037265, 2023-10-09T11:05:03.418 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.418 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.419 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.419 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.419 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.419 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 158832 2023-10-09T11:05:03.419 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.419 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.419 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.419 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.419 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841518+0000", 2023-10-09T11:05:03.419 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.419 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.419 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.419 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.420 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841518+0000", 2023-10-09T11:05:03.420 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.420 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.420 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.420 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841518+0000", 2023-10-09T11:05:03.420 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:03.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.420 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.420 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.420 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841521+0000", 2023-10-09T11:05:03.420 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7020000000000001e-06 2023-10-09T11:05:03.421 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.421 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.421 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841527+0000", 2023-10-09T11:05:03.421 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1310000000000003e-06 2023-10-09T11:05:03.421 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.421 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.421 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841531+0000", 2023-10-09T11:05:03.421 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1049999999999997e-06 2023-10-09T11:05:03.421 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.421 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.422 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841568+0000", 2023-10-09T11:05:03.422 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7651000000000003e-05 2023-10-09T11:05:03.422 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.422 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.422 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.422 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841681+0000", 2023-10-09T11:05:03.422 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011254100000000001 2023-10-09T11:05:03.422 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.422 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.422 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:05:03.422 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841795+0000", 2023-10-09T11:05:03.422 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000114193 2023-10-09T11:05:03.423 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.423 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.423 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.423 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.845131+0000", 2023-10-09T11:05:03.423 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003335569 2023-10-09T11:05:03.423 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.423 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.423 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.423 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.845137+0000", 2023-10-09T11:05:03.423 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9900000000000002e-06 2023-10-09T11:05:03.423 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.423 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.424 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.424 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.957443+0000", 2023-10-09T11:05:03.424 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.112306507 2023-10-09T11:05:03.424 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.424 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.424 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.424 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.957511+0000", 2023-10-09T11:05:03.424 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7582000000000004e-05 2023-10-09T11:05:03.424 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.424 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.424 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.424 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.957556+0000", 2023-10-09T11:05:03.425 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4696e-05 2023-10-09T11:05:03.425 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.425 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.425 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.425 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.425 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.425 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:180897 10.f 10:f1fc3a12:::benchmark_data_smithi125_402869_object82328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T11:05:03.425 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:24.612340+0000", 2023-10-09T11:05:03.425 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 38.738377305999997, 2023-10-09T11:05:03.425 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.107943447, 2023-10-09T11:05:03.425 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.425 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.426 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.426 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.426 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.426 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 180897 2023-10-09T11:05:03.426 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.426 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.426 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.426 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.426 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612340+0000", 2023-10-09T11:05:03.426 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.426 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.426 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.427 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.427 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612340+0000", 2023-10-09T11:05:03.427 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.427 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.427 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.427 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.427 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612339+0000", 2023-10-09T11:05:03.427 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:03.427 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.427 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.427 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.428 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612343+0000", 2023-10-09T11:05:03.428 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3069999999999998e-06 2023-10-09T11:05:03.428 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.428 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.428 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.428 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612353+0000", 2023-10-09T11:05:03.428 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9769999999999995e-06 2023-10-09T11:05:03.428 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.428 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.428 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.428 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612357+0000", 2023-10-09T11:05:03.428 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5109999999999997e-06 2023-10-09T11:05:03.429 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.429 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.429 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.429 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612420+0000", 2023-10-09T11:05:03.429 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2391999999999995e-05 2023-10-09T11:05:03.429 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.429 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.429 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.429 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612537+0000", 2023-10-09T11:05:03.429 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000117409 2023-10-09T11:05:03.429 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.429 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.430 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T11:05:03.430 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612654+0000", 2023-10-09T11:05:03.430 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000117248 2023-10-09T11:05:03.430 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.430 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.430 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.430 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.621223+0000", 2023-10-09T11:05:03.430 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0085686159999999994 2023-10-09T11:05:03.430 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.430 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.430 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.430 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626399+0000", 2023-10-09T11:05:03.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051757670000000004 2023-10-09T11:05:03.431 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.431 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.431 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.431 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626899+0000", 2023-10-09T11:05:03.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00049993800000000001 2023-10-09T11:05:03.431 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.431 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.431 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.431 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.719838+0000", 2023-10-09T11:05:03.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.092939021999999996 2023-10-09T11:05:03.431 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.432 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.432 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.432 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.719924+0000", 2023-10-09T11:05:03.432 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6328000000000004e-05 2023-10-09T11:05:03.432 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.432 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.432 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.432 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.720283+0000", 2023-10-09T11:05:03.432 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00035945900000000002 2023-10-09T11:05:03.432 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.432 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.432 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.433 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.433 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.433 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:180902 10.f 10:fa4f508c:::benchmark_data_smithi125_402869_object82333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T11:05:03.433 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:24.626845+0000", 2023-10-09T11:05:03.433 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 38.723872229000001, 2023-10-09T11:05:03.433 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10240413199999999, 2023-10-09T11:05:03.433 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.433 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.433 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.433 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.433 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.433 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 180902 2023-10-09T11:05:03.434 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.434 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.434 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.434 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.434 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626845+0000", 2023-10-09T11:05:03.434 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.434 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.434 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.434 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.434 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626845+0000", 2023-10-09T11:05:03.434 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.435 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.435 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626844+0000", 2023-10-09T11:05:03.435 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:05:03.435 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.435 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626848+0000", 2023-10-09T11:05:03.435 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.498e-06 2023-10-09T11:05:03.435 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.436 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.436 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626858+0000", 2023-10-09T11:05:03.436 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2920000000000001e-06 2023-10-09T11:05:03.436 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.436 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.436 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.436 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626861+0000", 2023-10-09T11:05:03.436 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.782e-06 2023-10-09T11:05:03.436 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.436 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.436 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.436 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626915+0000", 2023-10-09T11:05:03.437 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3905e-05 2023-10-09T11:05:03.437 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.437 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.437 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.437 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.627044+0000", 2023-10-09T11:05:03.437 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012853699999999999 2023-10-09T11:05:03.437 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.437 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.437 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T11:05:03.437 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.627159+0000", 2023-10-09T11:05:03.437 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000114894 2023-10-09T11:05:03.437 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.437 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.438 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.438 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.630494+0000", 2023-10-09T11:05:03.438 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003335176 2023-10-09T11:05:03.438 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.438 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.438 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.438 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.632097+0000", 2023-10-09T11:05:03.438 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001602989 2023-10-09T11:05:03.438 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.438 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.438 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.438 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.635044+0000", 2023-10-09T11:05:03.439 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002947098 2023-10-09T11:05:03.439 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.439 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.439 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.439 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.728793+0000", 2023-10-09T11:05:03.439 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.093748897999999997 2023-10-09T11:05:03.439 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.439 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.439 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.439 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.728842+0000", 2023-10-09T11:05:03.439 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9327999999999999e-05 2023-10-09T11:05:03.439 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.439 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.440 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.440 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.729249+0000", 2023-10-09T11:05:03.440 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00040702300000000001 2023-10-09T11:05:03.440 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.440 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.440 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.440 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.440 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.440 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183807 10.0 10:04af7371:::benchmark_data_smithi125_402869_object85238:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:05:03.440 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.483591+0000", 2023-10-09T11:05:03.440 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 34.867126485, 2023-10-09T11:05:03.440 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.114706303, 2023-10-09T11:05:03.441 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.441 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.441 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.441 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.441 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.441 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183807 2023-10-09T11:05:03.441 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.441 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.441 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.441 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.441 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483591+0000", 2023-10-09T11:05:03.441 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.441 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483591+0000", 2023-10-09T11:05:03.442 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483590+0000", 2023-10-09T11:05:03.442 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:05:03.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.443 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.443 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483595+0000", 2023-10-09T11:05:03.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8319999999999996e-06 2023-10-09T11:05:03.443 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.443 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.443 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.443 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483605+0000", 2023-10-09T11:05:03.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0108999999999999e-05 2023-10-09T11:05:03.443 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.443 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.443 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.443 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483610+0000", 2023-10-09T11:05:03.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7469999999999996e-06 2023-10-09T11:05:03.444 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.444 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.444 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.444 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483651+0000", 2023-10-09T11:05:03.444 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1591000000000002e-05 2023-10-09T11:05:03.444 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.444 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.444 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.444 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483743+0000", 2023-10-09T11:05:03.444 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1634999999999999e-05 2023-10-09T11:05:03.444 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.444 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:05:03.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.484167+0000", 2023-10-09T11:05:03.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000424702 2023-10-09T11:05:03.445 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.445 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.493854+0000", 2023-10-09T11:05:03.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0096867949999999998 2023-10-09T11:05:03.445 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.445 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495932+0000", 2023-10-09T11:05:03.446 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020782270000000002 2023-10-09T11:05:03.446 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.446 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.446 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.446 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598222+0000", 2023-10-09T11:05:03.446 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.102289846 2023-10-09T11:05:03.446 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.446 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.446 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.446 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598267+0000", 2023-10-09T11:05:03.446 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4545999999999997e-05 2023-10-09T11:05:03.446 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.447 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.447 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.447 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598297+0000", 2023-10-09T11:05:03.447 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0187000000000001e-05 2023-10-09T11:05:03.447 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.447 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.447 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.447 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.447 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.447 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183811 10.10 10:0ca62c6a:::benchmark_data_smithi125_402869_object85242:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:05:03.447 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.483618+0000", 2023-10-09T11:05:03.447 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 34.867099365999998, 2023-10-09T11:05:03.448 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11462522, 2023-10-09T11:05:03.448 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.448 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.448 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.448 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.448 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.448 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183811 2023-10-09T11:05:03.448 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.448 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.448 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.448 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.448 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483618+0000", 2023-10-09T11:05:03.448 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.449 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.449 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.449 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483618+0000", 2023-10-09T11:05:03.449 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.449 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.449 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.449 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483617+0000", 2023-10-09T11:05:03.449 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:03.449 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.450 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.450 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483620+0000", 2023-10-09T11:05:03.450 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6240000000000002e-06 2023-10-09T11:05:03.450 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.450 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.450 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.450 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483632+0000", 2023-10-09T11:05:03.450 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1652e-05 2023-10-09T11:05:03.450 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.450 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.450 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.450 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483636+0000", 2023-10-09T11:05:03.451 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1330000000000001e-06 2023-10-09T11:05:03.451 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.451 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.451 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483686+0000", 2023-10-09T11:05:03.451 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0438000000000001e-05 2023-10-09T11:05:03.451 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.451 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.451 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483793+0000", 2023-10-09T11:05:03.451 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106499 2023-10-09T11:05:03.451 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:05:03.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.484240+0000", 2023-10-09T11:05:03.452 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00044728299999999998 2023-10-09T11:05:03.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.493846+0000", 2023-10-09T11:05:03.452 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0096058700000000007 2023-10-09T11:05:03.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495948+0000", 2023-10-09T11:05:03.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021024860000000002 2023-10-09T11:05:03.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.453 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.453 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598183+0000", 2023-10-09T11:05:03.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.102234346 2023-10-09T11:05:03.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.453 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.453 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598216+0000", 2023-10-09T11:05:03.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2889999999999999e-05 2023-10-09T11:05:03.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.454 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.454 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.454 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598243+0000", 2023-10-09T11:05:03.454 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7449000000000001e-05 2023-10-09T11:05:03.454 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.454 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.454 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.454 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.454 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.454 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183812 10.10 10:0e10b5d1:::benchmark_data_smithi125_402869_object85243:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:05:03.454 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.483656+0000", 2023-10-09T11:05:03.454 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 34.867061137, 2023-10-09T11:05:03.455 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.114659012, 2023-10-09T11:05:03.455 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.455 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.455 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.455 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.455 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.455 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183812 2023-10-09T11:05:03.455 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.455 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.455 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.455 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.455 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483656+0000", 2023-10-09T11:05:03.455 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.456 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.456 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.456 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483656+0000", 2023-10-09T11:05:03.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.456 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.456 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.456 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483656+0000", 2023-10-09T11:05:03.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:03.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.456 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.457 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483659+0000", 2023-10-09T11:05:03.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0730000000000001e-06 2023-10-09T11:05:03.457 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.457 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.457 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483666+0000", 2023-10-09T11:05:03.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0539999999999997e-06 2023-10-09T11:05:03.457 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.457 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.457 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483670+0000", 2023-10-09T11:05:03.458 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8670000000000004e-06 2023-10-09T11:05:03.458 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.458 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.458 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486323+0000", 2023-10-09T11:05:03.458 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026528950000000002 2023-10-09T11:05:03.458 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.458 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.458 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486433+0000", 2023-10-09T11:05:03.458 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000110812 2023-10-09T11:05:03.458 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.459 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:05:03.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486801+0000", 2023-10-09T11:05:03.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00036751500000000001 2023-10-09T11:05:03.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.459 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495912+0000", 2023-10-09T11:05:03.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0091110519999999997 2023-10-09T11:05:03.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.459 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495984+0000", 2023-10-09T11:05:03.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1724999999999995e-05 2023-10-09T11:05:03.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598264+0000", 2023-10-09T11:05:03.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.102280443 2023-10-09T11:05:03.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598287+0000", 2023-10-09T11:05:03.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3285000000000002e-05 2023-10-09T11:05:03.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598315+0000", 2023-10-09T11:05:03.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.777e-05 2023-10-09T11:05:03.461 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.461 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.461 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.461 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183813 10.10 10:0c717176:::benchmark_data_smithi125_402869_object85244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:05:03.461 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.483680+0000", 2023-10-09T11:05:03.461 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 34.867037017000001, 2023-10-09T11:05:03.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.114707768, 2023-10-09T11:05:03.462 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.462 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.462 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.462 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.462 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.462 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183813 2023-10-09T11:05:03.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.462 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483680+0000", 2023-10-09T11:05:03.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.463 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.463 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483680+0000", 2023-10-09T11:05:03.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.463 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.463 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483680+0000", 2023-10-09T11:05:03.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:03.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483682+0000", 2023-10-09T11:05:03.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6599999999999999e-06 2023-10-09T11:05:03.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483688+0000", 2023-10-09T11:05:03.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6540000000000004e-06 2023-10-09T11:05:03.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.465 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483692+0000", 2023-10-09T11:05:03.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.67e-06 2023-10-09T11:05:03.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.465 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.489061+0000", 2023-10-09T11:05:03.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0053695840000000002 2023-10-09T11:05:03.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.465 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.489172+0000", 2023-10-09T11:05:03.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011078000000000001 2023-10-09T11:05:03.466 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.466 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:05:03.466 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.489480+0000", 2023-10-09T11:05:03.466 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000307855 2023-10-09T11:05:03.466 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.466 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.466 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495919+0000", 2023-10-09T11:05:03.466 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064387530000000002 2023-10-09T11:05:03.466 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.467 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.496010+0000", 2023-10-09T11:05:03.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1748000000000003e-05 2023-10-09T11:05:03.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.467 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598341+0000", 2023-10-09T11:05:03.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.102330948 2023-10-09T11:05:03.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598368+0000", 2023-10-09T11:05:03.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6183e-05 2023-10-09T11:05:03.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598388+0000", 2023-10-09T11:05:03.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0367999999999999e-05 2023-10-09T11:05:03.468 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.468 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.468 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.468 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183815 10.10 10:0dad7a83:::benchmark_data_smithi125_402869_object85246:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:05:03.469 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.486435+0000", 2023-10-09T11:05:03.469 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 34.864282551000002, 2023-10-09T11:05:03.469 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11203769, 2023-10-09T11:05:03.469 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.469 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.469 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.469 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.469 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.469 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183815 2023-10-09T11:05:03.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.469 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.470 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.470 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486435+0000", 2023-10-09T11:05:03.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.470 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.470 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.470 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.470 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486435+0000", 2023-10-09T11:05:03.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.470 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.470 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.470 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.470 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486433+0000", 2023-10-09T11:05:03.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:05:03.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.471 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486450+0000", 2023-10-09T11:05:03.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6589999999999999e-05 2023-10-09T11:05:03.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.471 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486463+0000", 2023-10-09T11:05:03.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3261000000000001e-05 2023-10-09T11:05:03.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486475+0000", 2023-10-09T11:05:03.472 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2328e-05 2023-10-09T11:05:03.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.491357+0000", 2023-10-09T11:05:03.472 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.004881364 2023-10-09T11:05:03.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.491487+0000", 2023-10-09T11:05:03.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013008299999999999 2023-10-09T11:05:03.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:05:03.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.491763+0000", 2023-10-09T11:05:03.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00027596599999999999 2023-10-09T11:05:03.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495954+0000", 2023-10-09T11:05:03.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041909010000000003 2023-10-09T11:05:03.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.497564+0000", 2023-10-09T11:05:03.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001610081 2023-10-09T11:05:03.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598414+0000", 2023-10-09T11:05:03.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10085058500000001 2023-10-09T11:05:03.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.475 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598445+0000", 2023-10-09T11:05:03.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.044e-05 2023-10-09T11:05:03.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.475 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.475 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598472+0000", 2023-10-09T11:05:03.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7593000000000002e-05 2023-10-09T11:05:03.475 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.475 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.475 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.476 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183821 10.10 10:0e04176b:::benchmark_data_smithi125_402869_object85252:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:05:03.476 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.492183+0000", 2023-10-09T11:05:03.476 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 34.858533907999998, 2023-10-09T11:05:03.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.106356412, 2023-10-09T11:05:03.476 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.476 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.476 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.476 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.476 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.476 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183821 2023-10-09T11:05:03.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.476 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492183+0000", 2023-10-09T11:05:03.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492183+0000", 2023-10-09T11:05:03.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492182+0000", 2023-10-09T11:05:03.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:05:03.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492189+0000", 2023-10-09T11:05:03.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6080000000000001e-06 2023-10-09T11:05:03.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492203+0000", 2023-10-09T11:05:03.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3824e-05 2023-10-09T11:05:03.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492209+0000", 2023-10-09T11:05:03.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6769999999999998e-06 2023-10-09T11:05:03.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.493870+0000", 2023-10-09T11:05:03.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0016607099999999999 2023-10-09T11:05:03.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.493983+0000", 2023-10-09T11:05:03.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112582 2023-10-09T11:05:03.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:05:03.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.494235+0000", 2023-10-09T11:05:03.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025229000000000001 2023-10-09T11:05:03.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.496090+0000", 2023-10-09T11:05:03.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018548270000000001 2023-10-09T11:05:03.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.497689+0000", 2023-10-09T11:05:03.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015987530000000001 2023-10-09T11:05:03.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598498+0000", 2023-10-09T11:05:03.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10080939899999999 2023-10-09T11:05:03.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598522+0000", 2023-10-09T11:05:03.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3609999999999999e-05 2023-10-09T11:05:03.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598540+0000", 2023-10-09T11:05:03.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8224e-05 2023-10-09T11:05:03.482 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.482 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.482 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.483 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183823 10.10 10:0c686624:::benchmark_data_smithi125_402869_object85254:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:05:03.483 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.500372+0000", 2023-10-09T11:05:03.483 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 34.850345249999997, 2023-10-09T11:05:03.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.098236665000000001, 2023-10-09T11:05:03.483 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.483 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.483 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.483 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.483 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.483 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183823 2023-10-09T11:05:03.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.484 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500372+0000", 2023-10-09T11:05:03.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500372+0000", 2023-10-09T11:05:03.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500371+0000", 2023-10-09T11:05:03.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:05:03.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500377+0000", 2023-10-09T11:05:03.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6710000000000004e-06 2023-10-09T11:05:03.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500387+0000", 2023-10-09T11:05:03.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9329999999999996e-06 2023-10-09T11:05:03.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500391+0000", 2023-10-09T11:05:03.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.887e-06 2023-10-09T11:05:03.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500428+0000", 2023-10-09T11:05:03.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6810999999999997e-05 2023-10-09T11:05:03.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500522+0000", 2023-10-09T11:05:03.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3919000000000007e-05 2023-10-09T11:05:03.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:05:03.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500870+0000", 2023-10-09T11:05:03.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000347762 2023-10-09T11:05:03.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.502778+0000", 2023-10-09T11:05:03.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019076640000000001 2023-10-09T11:05:03.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.502985+0000", 2023-10-09T11:05:03.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00020777600000000001 2023-10-09T11:05:03.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598562+0000", 2023-10-09T11:05:03.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.095576784999999997 2023-10-09T11:05:03.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598590+0000", 2023-10-09T11:05:03.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7881999999999998e-05 2023-10-09T11:05:03.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598609+0000", 2023-10-09T11:05:03.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8587999999999999e-05 2023-10-09T11:05:03.489 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.489 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.489 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.490 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.490 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:184330 10.a 10:57ae38ca:::benchmark_data_smithi125_402869_object85761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:05:03.490 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:29.197170+0000", 2023-10-09T11:05:03.490 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 34.153547484999997, 2023-10-09T11:05:03.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.120668026, 2023-10-09T11:05:03.490 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:05:03.490 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:03.490 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:05:03.490 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:05:03.490 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:05:03.490 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 184330 2023-10-09T11:05:03.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.491 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:05:03.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:05:03.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197170+0000", 2023-10-09T11:05:03.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:05:03.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197170+0000", 2023-10-09T11:05:03.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:05:03.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:05:03.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197169+0000", 2023-10-09T11:05:03.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:03.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:05:03.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197173+0000", 2023-10-09T11:05:03.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.106e-06 2023-10-09T11:05:03.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:05:03.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197180+0000", 2023-10-09T11:05:03.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1760000000000004e-06 2023-10-09T11:05:03.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:05:03.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197184+0000", 2023-10-09T11:05:03.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9639999999999997e-06 2023-10-09T11:05:03.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:05:03.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197223+0000", 2023-10-09T11:05:03.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9072999999999997e-05 2023-10-09T11:05:03.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:05:03.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197334+0000", 2023-10-09T11:05:03.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000111279 2023-10-09T11:05:03.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:05:03.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197448+0000", 2023-10-09T11:05:03.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000113848 2023-10-09T11:05:03.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:05:03.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.200444+0000", 2023-10-09T11:05:03.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029966039999999999 2023-10-09T11:05:03.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.206587+0000", 2023-10-09T11:05:03.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061423409999999999 2023-10-09T11:05:03.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:03.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.317756+0000", 2023-10-09T11:05:03.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.111169485 2023-10-09T11:05:03.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:05:03.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.317798+0000", 2023-10-09T11:05:03.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1838000000000002e-05 2023-10-09T11:05:03.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:05:03.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:05:03.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:05:03.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.317838+0000", 2023-10-09T11:05:03.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9628000000000001e-05 2023-10-09T11:05:03.496 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.496 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.496 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.497 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:05:03.497 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:05:03.497 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:05:03.497 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:05:03.498 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:03 smithi155 ceph-mon[184854]: pgmap v1208: 103 pgs: 103 active+clean; 608 KiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-10-09T11:05:03.498 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:03 smithi155 ceph-mon[184854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10316/813 objects degraded (1268.881%), 5 pgs degraded) 2023-10-09T11:05:03.498 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:03 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:05:03.498 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4020437466' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:05:03.498 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:03 smithi155 ceph-mon[184854]: osdmap e494: 8 total, 8 up, 7 in 2023-10-09T11:05:03.498 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:03 smithi155 ceph-mon[184854]: failed to encode map e494 with expected crc 2023-10-09T11:05:03.527 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:03 smithi125 ceph-mon[195507]: pgmap v1208: 103 pgs: 103 active+clean; 608 KiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-10-09T11:05:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:03 smithi125 ceph-mon[195507]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10316/813 objects degraded (1268.881%), 5 pgs degraded) 2023-10-09T11:05:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:03 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:05:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4020437466' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:05:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:03 smithi125 ceph-mon[195507]: osdmap e494: 8 total, 8 up, 7 in 2023-10-09T11:05:03.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:03 smithi125 ceph-mon[195507]: failed to encode map e494 with expected crc 2023-10-09T11:05:03.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:03 smithi125 ceph-mon[200197]: pgmap v1208: 103 pgs: 103 active+clean; 608 KiB data, 6.0 GiB used, 620 GiB / 626 GiB avail 2023-10-09T11:05:03.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:03 smithi125 ceph-mon[200197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10316/813 objects degraded (1268.881%), 5 pgs degraded) 2023-10-09T11:05:03.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:03 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:05:03.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4020437466' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:05:03.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:03 smithi125 ceph-mon[200197]: osdmap e494: 8 total, 8 up, 7 in 2023-10-09T11:05:03.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:03 smithi125 ceph-mon[200197]: failed to encode map e494 with expected crc 2023-10-09T11:05:03.880 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T11:05:03.898 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:05:03.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:03.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:05:03.975 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:05:04.003 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:04.044 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:04.063 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:04.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:04.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:04.166 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:04.196 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:04.216 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:04.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:04.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:04.235 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:05:04.246 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:04.263 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:05:04.263 INFO:teuthology.orchestra.run.smithi155.stdout: "success": "" 2023-10-09T11:05:04.263 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:05:04.281 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_5' created 2023-10-09T11:05:04.288 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:04 smithi155 ceph-mon[184854]: failed to encode map e494 with expected crc 2023-10-09T11:05:04.288 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:04 smithi155 ceph-mon[184854]: failed to encode map e494 with expected crc 2023-10-09T11:05:04.288 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:04 smithi155 ceph-mon[184854]: failed to encode map e494 with expected crc 2023-10-09T11:05:04.289 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:04 smithi155 ceph-mon[184854]: failed to encode map e494 with expected crc 2023-10-09T11:05:04.289 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:04 smithi155 ceph-mon[184854]: failed to encode map e494 with expected crc 2023-10-09T11:05:04.289 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:04 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1644359851' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T11:05:04.289 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:04 smithi155 ceph-mon[184854]: 1.4 scrub starts 2023-10-09T11:05:04.289 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:04 smithi155 ceph-mon[184854]: 1.4 scrub ok 2023-10-09T11:05:04.308 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[195507]: failed to encode map e494 with expected crc 2023-10-09T11:05:04.308 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[195507]: failed to encode map e494 with expected crc 2023-10-09T11:05:04.308 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[195507]: failed to encode map e494 with expected crc 2023-10-09T11:05:04.308 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[195507]: failed to encode map e494 with expected crc 2023-10-09T11:05:04.308 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[195507]: failed to encode map e494 with expected crc 2023-10-09T11:05:04.308 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1644359851' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T11:05:04.308 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[195507]: 1.4 scrub starts 2023-10-09T11:05:04.308 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[195507]: 1.4 scrub ok 2023-10-09T11:05:04.309 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[200197]: failed to encode map e494 with expected crc 2023-10-09T11:05:04.309 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[200197]: failed to encode map e494 with expected crc 2023-10-09T11:05:04.309 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[200197]: failed to encode map e494 with expected crc 2023-10-09T11:05:04.309 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[200197]: failed to encode map e494 with expected crc 2023-10-09T11:05:04.309 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[200197]: failed to encode map e494 with expected crc 2023-10-09T11:05:04.309 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1644359851' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T11:05:04.309 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[200197]: 1.4 scrub starts 2023-10-09T11:05:04.309 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:04 smithi125 ceph-mon[200197]: 1.4 scrub ok 2023-10-09T11:05:04.321 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:04.347 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:04.468 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:04.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:04.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:04.571 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:04.602 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:04.621 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:04.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:04.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:04.714 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-10-09T11:05:04.724 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:04.786 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:04.793 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:05:04.861 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:04.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:04.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:04.965 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:04.995 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:05.065 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:05.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:05.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:05.154 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:05.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:05:05.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:05 smithi155 ceph-mon[184854]: pgmap v1210: 103 pgs: 103 active+clean; 608 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T11:05:05.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:05 smithi155 ceph-mon[184854]: failed to encode map e495 with expected crc 2023-10-09T11:05:05.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:05 smithi155 ceph-mon[184854]: failed to encode map e495 with expected crc 2023-10-09T11:05:05.197 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:05 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1644359851' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T11:05:05.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:05 smithi155 ceph-mon[184854]: osdmap e495: 8 total, 8 up, 7 in 2023-10-09T11:05:05.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:05 smithi155 ceph-mon[184854]: failed to encode map e495 with expected crc 2023-10-09T11:05:05.200 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:05.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:05:05.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:05.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:05:05.285 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:05.299 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:05:05.299 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:05:05.299 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T11:05:05.299 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:05:05.302 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:05 smithi125 ceph-mon[195507]: pgmap v1210: 103 pgs: 103 active+clean; 608 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T11:05:05.302 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:05 smithi125 ceph-mon[195507]: failed to encode map e495 with expected crc 2023-10-09T11:05:05.302 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:05 smithi125 ceph-mon[195507]: failed to encode map e495 with expected crc 2023-10-09T11:05:05.302 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:05 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1644359851' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T11:05:05.302 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:05 smithi125 ceph-mon[195507]: osdmap e495: 8 total, 8 up, 7 in 2023-10-09T11:05:05.302 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:05 smithi125 ceph-mon[195507]: failed to encode map e495 with expected crc 2023-10-09T11:05:05.302 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:05 smithi125 ceph-mon[200197]: pgmap v1210: 103 pgs: 103 active+clean; 608 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T11:05:05.302 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:05 smithi125 ceph-mon[200197]: failed to encode map e495 with expected crc 2023-10-09T11:05:05.302 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:05 smithi125 ceph-mon[200197]: failed to encode map e495 with expected crc 2023-10-09T11:05:05.302 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:05 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1644359851' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T11:05:05.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:05 smithi125 ceph-mon[200197]: osdmap e495: 8 total, 8 up, 7 in 2023-10-09T11:05:05.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:05 smithi125 ceph-mon[200197]: failed to encode map e495 with expected crc 2023-10-09T11:05:05.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:05.353 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:05.424 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:05.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:05.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:05.528 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:05:05.528 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:05.548 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:05:05.554 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:05.571 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:05:05.591 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:05:05.609 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:05:05.624 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:05.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:05.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:05.629 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:05:05.650 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:05:05.662 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T11:05:05.669 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:05:05.670 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:05:05.670 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:05:05.670 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:05:05.670 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:05:05.670 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:05:05.670 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:05:05.671 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:05:05.671 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:05:05.728 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:05.758 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:05.781 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:05.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:05.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:05.885 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:05.913 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:05.969 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:05:05.986 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:05.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:05.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:06.090 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:06.119 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:06.490 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:06.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:06.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:06.503 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:06 smithi155 ceph-mon[184854]: 1.1c scrub starts 2023-10-09T11:05:06.503 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:06 smithi155 ceph-mon[184854]: 1.1c scrub ok 2023-10-09T11:05:06.504 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:06 smithi155 ceph-mon[184854]: osdmap e496: 8 total, 8 up, 7 in 2023-10-09T11:05:06.504 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:06 smithi155 ceph-mon[184854]: failed to encode map e496 with expected crc 2023-10-09T11:05:06.504 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:06 smithi155 ceph-mon[184854]: failed to encode map e496 with expected crc 2023-10-09T11:05:06.504 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:06 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3614662420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:06.504 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:06 smithi155 ceph-mon[184854]: pgmap v1213: 119 pgs: 16 unknown, 103 active+clean; 608 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T11:05:06.504 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:06 smithi155 ceph-mon[184854]: failed to encode map e496 with expected crc 2023-10-09T11:05:06.504 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:06 smithi155 ceph-mon[184854]: failed to encode map e496 with expected crc 2023-10-09T11:05:06.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[195507]: 1.1c scrub starts 2023-10-09T11:05:06.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[195507]: 1.1c scrub ok 2023-10-09T11:05:06.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[195507]: osdmap e496: 8 total, 8 up, 7 in 2023-10-09T11:05:06.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[195507]: failed to encode map e496 with expected crc 2023-10-09T11:05:06.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[195507]: failed to encode map e496 with expected crc 2023-10-09T11:05:06.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3614662420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:06.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[195507]: pgmap v1213: 119 pgs: 16 unknown, 103 active+clean; 608 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T11:05:06.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[195507]: failed to encode map e496 with expected crc 2023-10-09T11:05:06.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[195507]: failed to encode map e496 with expected crc 2023-10-09T11:05:06.534 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[200197]: 1.1c scrub starts 2023-10-09T11:05:06.534 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[200197]: 1.1c scrub ok 2023-10-09T11:05:06.534 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[200197]: osdmap e496: 8 total, 8 up, 7 in 2023-10-09T11:05:06.534 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[200197]: failed to encode map e496 with expected crc 2023-10-09T11:05:06.534 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[200197]: failed to encode map e496 with expected crc 2023-10-09T11:05:06.534 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3614662420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:06.534 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[200197]: pgmap v1213: 119 pgs: 16 unknown, 103 active+clean; 608 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T11:05:06.534 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[200197]: failed to encode map e496 with expected crc 2023-10-09T11:05:06.535 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:06 smithi125 ceph-mon[200197]: failed to encode map e496 with expected crc 2023-10-09T11:05:06.595 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:06.624 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:06.649 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:06.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:06.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:06.755 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:06.783 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:06.805 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:06.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:06.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:06.909 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:06.938 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:06.965 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:06.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:06.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:06.971 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:05:06.971 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:05:07.069 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:07.101 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:07.273 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:07.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:07.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:07.289 INFO:teuthology.orchestra.run.smithi125.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-10-09T11:05:07.289 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:05:07.328 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T11:05:07.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:07 smithi155 ceph-mon[184854]: failed to encode map e496 with expected crc 2023-10-09T11:05:07.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:07 smithi155 ceph-mon[184854]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:07 smithi155 ceph-mon[184854]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:07 smithi155 ceph-mon[184854]: osdmap e497: 8 total, 8 up, 7 in 2023-10-09T11:05:07.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:07 smithi155 ceph-mon[184854]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:07 smithi155 ceph-mon[184854]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:07 smithi155 ceph-mon[184854]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:07 smithi155 ceph-mon[184854]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.355 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:07 smithi155 ceph-mon[184854]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.355 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:07 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/633825203' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:05:07.355 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:07 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:05:07.355 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:07 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4076299399' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:05:07.377 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:07.404 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:07.424 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:07.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:07.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:07.528 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:07.556 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:07.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[195507]: failed to encode map e496 with expected crc 2023-10-09T11:05:07.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[195507]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[195507]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[195507]: osdmap e497: 8 total, 8 up, 7 in 2023-10-09T11:05:07.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[195507]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[195507]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[195507]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[195507]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[195507]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/633825203' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:05:07.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:05:07.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4076299399' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:05:07.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[200197]: failed to encode map e496 with expected crc 2023-10-09T11:05:07.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[200197]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[200197]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[200197]: osdmap e497: 8 total, 8 up, 7 in 2023-10-09T11:05:07.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[200197]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[200197]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[200197]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[200197]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[200197]: failed to encode map e497 with expected crc 2023-10-09T11:05:07.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/633825203' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:05:07.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:05:07.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:07 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4076299399' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:05:07.577 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:07.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:07.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:07.604 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:05:07.681 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:05:07.708 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:07.729 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:05:07.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:07.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:05:07.834 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:07.862 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:07.880 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:07.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:07.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:07.987 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:08.014 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:08.033 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:08.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:08.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:08.136 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:08.163 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:08.182 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:08.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:08.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:08.285 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:08.312 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:08 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:05:08.312 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:08 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T11:05:08.312 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:08 smithi155 ceph-mon[184854]: failed to encode map e498 with expected crc 2023-10-09T11:05:08.312 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:08 smithi155 ceph-mon[184854]: failed to encode map e498 with expected crc 2023-10-09T11:05:08.312 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:08 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T11:05:08.312 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:08 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4076299399' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:05:08.312 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:08 smithi155 ceph-mon[184854]: osdmap e498: 8 total, 8 up, 7 in 2023-10-09T11:05:08.312 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:08 smithi155 ceph-mon[184854]: pgmap v1216: 119 pgs: 10 unknown, 109 active+clean; 599 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T11:05:08.315 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:08.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:08.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:08.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:08.536 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:05:08.540 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:08.562 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:05:08.562 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T11:05:08.563 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[195507]: failed to encode map e498 with expected crc 2023-10-09T11:05:08.563 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[195507]: failed to encode map e498 with expected crc 2023-10-09T11:05:08.563 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T11:05:08.563 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4076299399' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:05:08.563 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[195507]: osdmap e498: 8 total, 8 up, 7 in 2023-10-09T11:05:08.563 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[195507]: pgmap v1216: 119 pgs: 10 unknown, 109 active+clean; 599 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T11:05:08.563 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:05:08.563 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T11:05:08.564 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[200197]: failed to encode map e498 with expected crc 2023-10-09T11:05:08.564 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[200197]: failed to encode map e498 with expected crc 2023-10-09T11:05:08.564 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T11:05:08.564 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4076299399' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:05:08.564 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[200197]: osdmap e498: 8 total, 8 up, 7 in 2023-10-09T11:05:08.564 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:08 smithi125 ceph-mon[200197]: pgmap v1216: 119 pgs: 10 unknown, 109 active+clean; 599 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-09T11:05:08.587 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:05:08.587 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:05:08.591 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:08.610 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:08.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:08.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:08.714 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:08.741 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:08.760 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:08.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:08.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:08.805 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:08.866 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:08.893 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:08.903 DEBUG:teuthology.orchestra.run.smithi125:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-10-09T11:05:08.905 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T11:05:08.906 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T11:05:08.915 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:08.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:08.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:08.945 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T11:05:08.957 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:hints = 1 2023-10-09T11:05:08.957 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T11:05:08.957 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object prefix: benchmark_data_smithi125_439543 2023-10-09T11:05:08.958 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T11:05:08.958 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 0 0 0 0 0 0 - 0 2023-10-09T11:05:09.019 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:05:09.054 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:09.078 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:05:09.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:09.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:05:09.186 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:09.218 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:09.242 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:09.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:09.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:09.346 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:09.390 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:09.519 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:09.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:09.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:09.588 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:09.637 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:09.682 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:09.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:09.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:09.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:09.772 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 7, 3] out_osds: [6] dead_osds: [] live_osds: [0, 2, 4, 5, 6, 7, 1, 3] 2023-10-09T11:05:09.772 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:05:09.772 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-09T11:05:09.772 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-09T11:05:09.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:05:09.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:09.926 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:09.948 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:09.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:09.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:09.969 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 1 16 2470 2454 151.718 153.375 0.0143387 0.0065245 2023-10-09T11:05:10.052 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:10.081 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:10.102 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:10.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:10.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:10.166 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:05:10.201 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:05:10.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:10.248 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:05:10.263 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:10.283 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:05:10.313 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:05:10.342 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:05:10.380 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:05:10.416 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:05:10.425 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:05:10.425 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:05:10.425 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:05:10.426 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:05:10.426 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:05:10.426 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:05:10.426 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:05:10.426 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:05:10.969 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 2 15 4631 4616 143.46 135.125 0.00715559 0.00695977 2023-10-09T11:05:11.217 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:10 smithi125 ceph-mon[195507]: pgmap v1217: 119 pgs: 119 active+clean; 582 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-09T11:05:11.217 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:10 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2463603002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:11.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:10 smithi125 ceph-mon[200197]: pgmap v1217: 119 pgs: 119 active+clean; 582 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-09T11:05:11.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:10 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2463603002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:11.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:10 smithi155 ceph-mon[184854]: pgmap v1217: 119 pgs: 119 active+clean; 582 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-09T11:05:11.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:10 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2463603002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:11.499 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:05:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:05:11] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.43.0" 2023-10-09T11:05:11.941 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:11.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:11.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:11.968 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 3 16 7237 7221 149.872 162.812 0.00633401 0.00666025 2023-10-09T11:05:12.058 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:12.096 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:12.124 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:12.127 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:11 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:05:12.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:12.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:12.229 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:12.266 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:12.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:11 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:05:12.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:11 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:05:12.397 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:12.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:12.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:12.502 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:05:12.536 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:12.613 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:05:12.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:12.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:05:12.719 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:12.766 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:12.969 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 4 16 9641 9625 149.963 150.25 0.00657409 0.00665734 2023-10-09T11:05:13.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:12 smithi125 ceph-mon[195507]: pgmap v1218: 119 pgs: 119 active+clean; 146 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 22 MiB/s wr, 355 op/s 2023-10-09T11:05:13.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:12 smithi125 ceph-mon[200197]: pgmap v1218: 119 pgs: 119 active+clean; 146 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 22 MiB/s wr, 355 op/s 2023-10-09T11:05:13.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:12 smithi155 ceph-mon[184854]: pgmap v1218: 119 pgs: 119 active+clean; 146 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 22 MiB/s wr, 355 op/s 2023-10-09T11:05:13.614 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:13.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:13.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:13.714 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:13.754 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:13.810 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:13.851 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:13.969 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 5 16 11836 11820 147.411 137.188 0.00721274 0.00677669 2023-10-09T11:05:14.135 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:14.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:14.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:14.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:14.275 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:14.382 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:14 smithi125 ceph-mon[200197]: pgmap v1219: 119 pgs: 119 active+clean; 537 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T11:05:14.382 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:14 smithi125 ceph-mon[195507]: pgmap v1219: 119 pgs: 119 active+clean; 537 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T11:05:14.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:14 smithi155 ceph-mon[184854]: pgmap v1219: 119 pgs: 119 active+clean; 537 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T11:05:14.626 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:05:14.626 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:05:14.626 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.626 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:13041 11.a e498/495 11:511254b4:::benchmark_data_smithi125_439543_object13040:head v 498'844, mlcod=498'840)", 2023-10-09T11:05:14.626 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:14.619709+0000", 2023-10-09T11:05:14.626 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0039771900000000002, 2023-10-09T11:05:14.626 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003983955, 2023-10-09T11:05:14.626 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:14.626 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:14.627 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:14.627 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.627 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:14.627 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.619709+0000", 2023-10-09T11:05:14.627 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:14.627 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.627 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.627 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:14.627 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.619709+0000", 2023-10-09T11:05:14.627 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:14.627 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.628 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.628 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:14.628 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.619709+0000", 2023-10-09T11:05:14.628 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:14.628 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.628 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.628 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:14.628 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.619730+0000", 2023-10-09T11:05:14.628 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0628e-05 2023-10-09T11:05:14.628 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.628 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.628 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:14.629 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.619735+0000", 2023-10-09T11:05:14.629 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.631e-06 2023-10-09T11:05:14.629 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.629 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.629 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:14.629 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.619739+0000", 2023-10-09T11:05:14.629 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1219999999999997e-06 2023-10-09T11:05:14.629 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.629 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.629 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:14.630 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.619803+0000", 2023-10-09T11:05:14.630 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4098999999999997e-05 2023-10-09T11:05:14.630 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.630 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.630 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:14.630 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.619820+0000", 2023-10-09T11:05:14.630 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6818000000000001e-05 2023-10-09T11:05:14.630 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.630 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.630 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:05:14.630 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623659+0000", 2023-10-09T11:05:14.631 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038384130000000002 2023-10-09T11:05:14.631 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.631 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.631 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:05:14.631 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623693+0000", 2023-10-09T11:05:14.631 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4483999999999997e-05 2023-10-09T11:05:14.631 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:14.631 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:14.631 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:14.631 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.631 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.631 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:13042 11.a e498/495 11:5392b078:::benchmark_data_smithi125_439543_object13041:head v 498'845, mlcod=498'840)", 2023-10-09T11:05:14.632 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:14.619749+0000", 2023-10-09T11:05:14.632 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0039372019999999999, 2023-10-09T11:05:14.632 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004053521, 2023-10-09T11:05:14.632 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:14.632 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:14.632 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:14.632 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.632 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:14.632 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.619749+0000", 2023-10-09T11:05:14.632 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:14.632 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.633 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:14.633 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.619749+0000", 2023-10-09T11:05:14.633 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:14.633 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.633 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:14.633 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.619749+0000", 2023-10-09T11:05:14.633 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967296 2023-10-09T11:05:14.633 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.633 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:14.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.619769+0000", 2023-10-09T11:05:14.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0169e-05 2023-10-09T11:05:14.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:14.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.619775+0000", 2023-10-09T11:05:14.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.451e-06 2023-10-09T11:05:14.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:14.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.619779+0000", 2023-10-09T11:05:14.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1139999999999999e-06 2023-10-09T11:05:14.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:14.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.620591+0000", 2023-10-09T11:05:14.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000812571 2023-10-09T11:05:14.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:14.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.620609+0000", 2023-10-09T11:05:14.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8096000000000001e-05 2023-10-09T11:05:14.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:05:14.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623790+0000", 2023-10-09T11:05:14.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00318039 2023-10-09T11:05:14.636 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:14.636 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:14.636 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:14.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.636 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:13046 11.a e498/495 11:52072463:::benchmark_data_smithi125_439543_object13045:head v 498'846, mlcod=498'841)", 2023-10-09T11:05:14.637 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:14.620533+0000", 2023-10-09T11:05:14.637 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.00315359, 2023-10-09T11:05:14.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0033399049999999998, 2023-10-09T11:05:14.637 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:14.637 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:14.637 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:14.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:14.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.620533+0000", 2023-10-09T11:05:14.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:14.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.638 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:14.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.620533+0000", 2023-10-09T11:05:14.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:14.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.638 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:14.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.620532+0000", 2023-10-09T11:05:14.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:14.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:14.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.620557+0000", 2023-10-09T11:05:14.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4564000000000001e-05 2023-10-09T11:05:14.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:14.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.620571+0000", 2023-10-09T11:05:14.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4303e-05 2023-10-09T11:05:14.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:14.640 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.620578+0000", 2023-10-09T11:05:14.640 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.17e-06 2023-10-09T11:05:14.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.640 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:14.640 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.620984+0000", 2023-10-09T11:05:14.640 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00040547700000000001 2023-10-09T11:05:14.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.640 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:14.640 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.620997+0000", 2023-10-09T11:05:14.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3655000000000001e-05 2023-10-09T11:05:14.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.641 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:05:14.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623867+0000", 2023-10-09T11:05:14.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028699939999999998 2023-10-09T11:05:14.641 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:14.641 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:14.641 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:14.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.641 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:13048 11.b e498/495 11:d9248886:::benchmark_data_smithi125_439543_object13047:head v 498'865, mlcod=498'861)", 2023-10-09T11:05:14.642 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:14.622634+0000", 2023-10-09T11:05:14.642 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0010525980000000001, 2023-10-09T11:05:14.642 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001830482, 2023-10-09T11:05:14.642 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:14.642 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:05:14.642 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:14.642 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.642 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:14.642 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.622634+0000", 2023-10-09T11:05:14.642 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:14.642 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.643 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.643 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:14.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.622634+0000", 2023-10-09T11:05:14.643 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:14.643 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.643 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.643 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:14.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.622633+0000", 2023-10-09T11:05:14.643 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:14.643 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.643 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:14.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.622652+0000", 2023-10-09T11:05:14.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8694000000000001e-05 2023-10-09T11:05:14.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:14.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.622660+0000", 2023-10-09T11:05:14.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5040000000000002e-06 2023-10-09T11:05:14.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:14.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.622665+0000", 2023-10-09T11:05:14.645 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.101e-06 2023-10-09T11:05:14.645 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.645 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.645 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:14.645 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.622704+0000", 2023-10-09T11:05:14.645 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8834000000000001e-05 2023-10-09T11:05:14.645 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.645 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.645 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:14.645 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.622720+0000", 2023-10-09T11:05:14.645 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6086000000000002e-05 2023-10-09T11:05:14.646 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:14.646 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:14.646 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:14.646 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.646 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.646 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:13054 11.e 11:77806398:::benchmark_data_smithi125_439543_object13053:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e498)", 2023-10-09T11:05:14.646 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:14.622875+0000", 2023-10-09T11:05:14.646 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.00081088799999999995, 2023-10-09T11:05:14.646 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016640520000000001, 2023-10-09T11:05:14.646 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:14.646 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:05:14.647 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:05:14.647 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:05:14.647 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:05:14.647 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 13054 2023-10-09T11:05:14.647 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.647 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:14.647 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.647 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:14.647 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.622875+0000", 2023-10-09T11:05:14.647 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:14.647 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.648 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.648 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:14.648 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.622875+0000", 2023-10-09T11:05:14.648 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:14.648 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.648 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.648 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:14.648 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.622874+0000", 2023-10-09T11:05:14.648 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:05:14.648 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.648 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:14.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.622897+0000", 2023-10-09T11:05:14.649 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2599000000000002e-05 2023-10-09T11:05:14.649 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:14.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.622908+0000", 2023-10-09T11:05:14.649 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1121000000000001e-05 2023-10-09T11:05:14.649 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:14.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.622915+0000", 2023-10-09T11:05:14.650 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6790000000000005e-06 2023-10-09T11:05:14.650 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.650 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.650 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:14.650 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623365+0000", 2023-10-09T11:05:14.650 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00044923000000000001 2023-10-09T11:05:14.650 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.650 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.650 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:14.650 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623481+0000", 2023-10-09T11:05:14.650 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011592200000000001 2023-10-09T11:05:14.651 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.651 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.651 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,7", 2023-10-09T11:05:14.651 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623621+0000", 2023-10-09T11:05:14.651 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000140149 2023-10-09T11:05:14.651 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:14.651 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:14.651 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:14.651 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.651 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.651 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:13052 11.3 e498/495 11:cce54d4e:::benchmark_data_smithi125_439543_object13051:head v 498'829, mlcod=498'827)", 2023-10-09T11:05:14.652 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:14.623513+0000", 2023-10-09T11:05:14.652 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.000173616, 2023-10-09T11:05:14.652 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001112039, 2023-10-09T11:05:14.652 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:14.652 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:05:14.652 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:14.652 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.652 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:14.652 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623513+0000", 2023-10-09T11:05:14.652 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:14.652 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.653 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.653 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:14.653 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623513+0000", 2023-10-09T11:05:14.653 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:14.653 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.653 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.653 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:14.653 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623512+0000", 2023-10-09T11:05:14.653 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:14.653 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.653 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.654 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:14.654 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623545+0000", 2023-10-09T11:05:14.654 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2625e-05 2023-10-09T11:05:14.654 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.654 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.654 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:14.654 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623554+0000", 2023-10-09T11:05:14.654 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7589999999999992e-06 2023-10-09T11:05:14.654 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.654 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.654 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:14.655 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623561+0000", 2023-10-09T11:05:14.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9369999999999998e-06 2023-10-09T11:05:14.655 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.655 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:14.655 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.624539+0000", 2023-10-09T11:05:14.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00097762400000000007 2023-10-09T11:05:14.655 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.655 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:14.655 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.624560+0000", 2023-10-09T11:05:14.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0562000000000001e-05 2023-10-09T11:05:14.656 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:14.656 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:14.656 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:14.656 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.656 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.656 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:13053 11.7 e498/495 11:ecb4164d:::benchmark_data_smithi125_439543_object13052:head v 498'805, mlcod=498'802)", 2023-10-09T11:05:14.656 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:14.623816+0000", 2023-10-09T11:05:14.656 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4294967295.9998703, 2023-10-09T11:05:14.656 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00086777399999999998, 2023-10-09T11:05:14.656 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:14.656 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:05:14.657 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:14.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.657 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:14.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623816+0000", 2023-10-09T11:05:14.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:14.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.657 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:14.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623816+0000", 2023-10-09T11:05:14.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:14.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.658 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:14.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623816+0000", 2023-10-09T11:05:14.658 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:14.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.658 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:14.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623845+0000", 2023-10-09T11:05:14.658 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9379000000000001e-05 2023-10-09T11:05:14.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.658 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:14.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623853+0000", 2023-10-09T11:05:14.659 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1489999999999994e-06 2023-10-09T11:05:14.659 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.659 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.659 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:14.659 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623858+0000", 2023-10-09T11:05:14.659 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9069999999999998e-06 2023-10-09T11:05:14.659 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.659 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.659 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:14.659 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623889+0000", 2023-10-09T11:05:14.659 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0862999999999999e-05 2023-10-09T11:05:14.660 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:14.660 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:14.660 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:14.660 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:14.623906+0000", 2023-10-09T11:05:14.660 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7464000000000001e-05 2023-10-09T11:05:14.660 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:14.660 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:14.660 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:14.660 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:14.660 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T11:05:14.660 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 7 2023-10-09T11:05:14.660 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:05:14.970 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 6 16 13760 13744 142.858 120.25 0.00241557 0.00699236 2023-10-09T11:05:15.351 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:15.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:15.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:15.455 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:05:15.490 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:15.530 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:05:15.915 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:05:15.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:15.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:05:15.970 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 7 16 15854 15838 141.147 130.875 0.0557966 0.00707785 2023-10-09T11:05:16.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:16.054 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:16.083 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:16.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:16.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:16.187 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:16.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:16.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:05:16.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:16 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:16.237 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:16.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:16.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:05:16.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:16 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:16.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:16.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:05:16.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:16 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:16.972 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 8 16 17908 17892 139.532 128.375 0.0110628 0.00715732 2023-10-09T11:05:17.131 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:17.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:17.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:17.176 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:17 smithi125 ceph-mon[195507]: pgmap v1220: 119 pgs: 119 active+clean; 537 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T11:05:17.176 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:17 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:17.176 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:17 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:17.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:17 smithi125 ceph-mon[200197]: pgmap v1220: 119 pgs: 119 active+clean; 537 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T11:05:17.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:17 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:17.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:17 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:17.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T11:05:17.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:17.350 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:17.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:17 smithi155 ceph-mon[184854]: pgmap v1220: 119 pgs: 119 active+clean; 537 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T11:05:17.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:17 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:17.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:17 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:17.503 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:17.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:17.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:17.610 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:17.660 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:17.742 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:17.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:17.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:17.809 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:17.847 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:17.877 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:17.896 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:17.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:17.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:17.974 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 9 16 19385 19369 134.261 92.3125 0.020405 0.00743691 2023-10-09T11:05:18.000 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:18.029 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:18.049 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:18.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:18.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:18.153 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:18.228 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:18.381 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:18.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:18.387 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:18.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:18.488 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:18.523 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:18.548 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:18.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:18.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:18.652 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:18.683 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:18 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T11:05:18.433+0000 7fbf30f1d700 -1 mon.a@0(leader).osd e498 definitely_dead 0 2023-10-09T11:05:18.710 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:18.795 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:18.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:18.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:18.900 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:18.931 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:18.959 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:18.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:18.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:18.974 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 10 16 20812 20796 129.759 89.1875 0.0108765 0.00769682 2023-10-09T11:05:19.064 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:19.141 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:19 smithi155 ceph-mon[184854]: pgmap v1221: 119 pgs: 119 active+clean; 790 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T11:05:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:19 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1181517351' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:05:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:19 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:05:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:19 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/562177824' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T11:05:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:19 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T11:05:19.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:19 smithi125 ceph-mon[195507]: pgmap v1221: 119 pgs: 119 active+clean; 790 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T11:05:19.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:19 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1181517351' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:05:19.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:19 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:05:19.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:19 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/562177824' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T11:05:19.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:19 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T11:05:19.306 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:19 smithi125 ceph-mon[200197]: pgmap v1221: 119 pgs: 119 active+clean; 790 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T11:05:19.306 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:19 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1181517351' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:05:19.306 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:19 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:05:19.306 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:19 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/562177824' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T11:05:19.306 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:19 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T11:05:19.306 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.6. 2023-10-09T11:05:19.315 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:05:19.490 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:05:19.508 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:05:19.526 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:05:19.545 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:05:19.564 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:05:19.584 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:05:19.603 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:05:19.622 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:05:19.623 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:05:19.623 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:05:19.623 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:05:19.623 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:05:19.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:05:19.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:05:19.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:05:19.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:05:19.974 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 11 16 21363 21347 121.105 34.4375 0.00188707 0.00774413 2023-10-09T11:05:20.309 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:05:20.309 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[195507]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.309 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:05:20.309 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T11:05:20.309 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[195507]: osdmap e499: 8 total, 8 up, 8 in 2023-10-09T11:05:20.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[195507]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[195507]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[195507]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[195507]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[195507]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[195507]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/5352165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:20.310 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:05:20.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[200197]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:05:20.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T11:05:20.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[200197]: osdmap e499: 8 total, 8 up, 8 in 2023-10-09T11:05:20.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[200197]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[200197]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[200197]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[200197]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[200197]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[200197]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:20 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/5352165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:20 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:05:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:20 smithi155 ceph-mon[184854]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:20 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:05:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:20 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T11:05:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:20 smithi155 ceph-mon[184854]: osdmap e499: 8 total, 8 up, 8 in 2023-10-09T11:05:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:20 smithi155 ceph-mon[184854]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:20 smithi155 ceph-mon[184854]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:20 smithi155 ceph-mon[184854]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:20 smithi155 ceph-mon[184854]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:20 smithi155 ceph-mon[184854]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:20 smithi155 ceph-mon[184854]: failed to encode map e499 with expected crc 2023-10-09T11:05:20.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:20 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/5352165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:20.482 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:20.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:20.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:20.544 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-09T11:05:20.586 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:20.614 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:20.632 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:20.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:20.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:20.736 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:20.763 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:20.781 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:20.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:20.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:20.821 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:05:20.821 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:05:20.885 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:20.912 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:20.930 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:20.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:20.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:20.974 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 12 16 21363 21347 111.026 0 - 0.00774413 2023-10-09T11:05:21.035 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:21.068 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:21 smithi155 ceph-mon[184854]: failed to encode map e499 with expected crc 2023-10-09T11:05:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:21 smithi155 ceph-mon[184854]: failed to encode map e499 with expected crc 2023-10-09T11:05:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:21 smithi155 ceph-mon[184854]: failed to encode map e499 with expected crc 2023-10-09T11:05:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:21 smithi155 ceph-mon[184854]: pgmap v1223: 119 pgs: 119 active+clean; 1.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T11:05:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:21 smithi155 ceph-mon[184854]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:21 smithi155 ceph-mon[184854]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:21 smithi155 ceph-mon[184854]: osdmap e500: 8 total, 8 up, 8 in 2023-10-09T11:05:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:21 smithi155 ceph-mon[184854]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:21 smithi155 ceph-mon[184854]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:21 smithi155 ceph-mon[184854]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:21 smithi155 ceph-mon[184854]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:21 smithi155 ceph-mon[184854]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:21 smithi155 ceph-mon[184854]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:21 smithi155 ceph-mon[184854]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:21 smithi155 ceph-mon[184854]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T11:05:21.239 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:21.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:21.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:21.343 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:21.415 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:21.451 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:21.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:21.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:21.558 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:21.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[195507]: failed to encode map e499 with expected crc 2023-10-09T11:05:21.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[195507]: failed to encode map e499 with expected crc 2023-10-09T11:05:21.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[195507]: failed to encode map e499 with expected crc 2023-10-09T11:05:21.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[195507]: pgmap v1223: 119 pgs: 119 active+clean; 1.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T11:05:21.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[195507]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[195507]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[195507]: osdmap e500: 8 total, 8 up, 8 in 2023-10-09T11:05:21.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[195507]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[195507]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[195507]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[195507]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[195507]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[195507]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[195507]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[195507]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.574 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:05:21] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.43.0" 2023-10-09T11:05:21.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[200197]: failed to encode map e499 with expected crc 2023-10-09T11:05:21.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[200197]: failed to encode map e499 with expected crc 2023-10-09T11:05:21.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[200197]: failed to encode map e499 with expected crc 2023-10-09T11:05:21.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[200197]: pgmap v1223: 119 pgs: 119 active+clean; 1.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T11:05:21.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[200197]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[200197]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[200197]: osdmap e500: 8 total, 8 up, 8 in 2023-10-09T11:05:21.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[200197]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[200197]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[200197]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[200197]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[200197]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[200197]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.577 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[200197]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.577 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:21 smithi125 ceph-mon[200197]: failed to encode map e500 with expected crc 2023-10-09T11:05:21.594 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:21.622 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:21.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:21.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:21.726 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:21.772 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:21.795 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:21.800 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:21.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:21.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:21.908 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:21.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:05:21.973 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:21.977 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 13 16 22143 22127 106.218 24.375 0.0214081 0.009405 2023-10-09T11:05:22.010 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:22.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:22.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:22.100 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:22 smithi155 ceph-mon[184854]: failed to encode map e500 with expected crc 2023-10-09T11:05:22.100 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:22 smithi155 ceph-mon[184854]: osdmap e501: 8 total, 8 up, 8 in 2023-10-09T11:05:22.100 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:22 smithi155 ceph-mon[184854]: failed to encode map e501 with expected crc 2023-10-09T11:05:22.100 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:22 smithi155 ceph-mon[184854]: failed to encode map e501 with expected crc 2023-10-09T11:05:22.100 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:22 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:22.100 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:22 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:22.117 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:22.179 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:22.303 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:22 smithi125 ceph-mon[195507]: failed to encode map e500 with expected crc 2023-10-09T11:05:22.303 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:22 smithi125 ceph-mon[195507]: osdmap e501: 8 total, 8 up, 8 in 2023-10-09T11:05:22.303 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:22 smithi125 ceph-mon[195507]: failed to encode map e501 with expected crc 2023-10-09T11:05:22.303 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:22 smithi125 ceph-mon[195507]: failed to encode map e501 with expected crc 2023-10-09T11:05:22.303 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:22 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:22.303 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:22 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:22.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:22 smithi125 ceph-mon[200197]: failed to encode map e500 with expected crc 2023-10-09T11:05:22.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:22 smithi125 ceph-mon[200197]: osdmap e501: 8 total, 8 up, 8 in 2023-10-09T11:05:22.304 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:22 smithi125 ceph-mon[200197]: failed to encode map e501 with expected crc 2023-10-09T11:05:22.304 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:22 smithi125 ceph-mon[200197]: failed to encode map e501 with expected crc 2023-10-09T11:05:22.304 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:22 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:22.304 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:22 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:22.491 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:22.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:22.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:22.545 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:05:22.597 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:22.700 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:22.739 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:22.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:22.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:22.784 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:22.845 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:22.912 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:22.942 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:22.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:22.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:22.977 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 14 16 23294 23278 103.772 71.9375 0.00591598 0.00962818 2023-10-09T11:05:23.046 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:23.085 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:23.108 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:23.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:23.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:23.188 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:05:23.188 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:05:23.213 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:23.219 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[195507]: failed to encode map e501 with expected crc 2023-10-09T11:05:23.219 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[195507]: pgmap v1226: 119 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 2 remapped+peering, 107 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s; 3981/61017 objects degraded (6.524%); 800 B/s, 1 objects/s recovering 2023-10-09T11:05:23.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[195507]: Health check failed: Degraded data redundancy: 3981/61017 objects degraded (6.524%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:23.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[195507]: failed to encode map e502 with expected crc 2023-10-09T11:05:23.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[195507]: failed to encode map e502 with expected crc 2023-10-09T11:05:23.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[195507]: osdmap e502: 8 total, 8 up, 8 in 2023-10-09T11:05:23.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[195507]: failed to encode map e502 with expected crc 2023-10-09T11:05:23.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[195507]: failed to encode map e502 with expected crc 2023-10-09T11:05:23.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:05:23.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:05:23.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:23.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[200197]: failed to encode map e501 with expected crc 2023-10-09T11:05:23.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[200197]: pgmap v1226: 119 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 2 remapped+peering, 107 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s; 3981/61017 objects degraded (6.524%); 800 B/s, 1 objects/s recovering 2023-10-09T11:05:23.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[200197]: Health check failed: Degraded data redundancy: 3981/61017 objects degraded (6.524%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:23.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[200197]: failed to encode map e502 with expected crc 2023-10-09T11:05:23.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[200197]: failed to encode map e502 with expected crc 2023-10-09T11:05:23.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[200197]: osdmap e502: 8 total, 8 up, 8 in 2023-10-09T11:05:23.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[200197]: failed to encode map e502 with expected crc 2023-10-09T11:05:23.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[200197]: failed to encode map e502 with expected crc 2023-10-09T11:05:23.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:05:23.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:05:23.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:23.272 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:23 smithi155 ceph-mon[184854]: failed to encode map e501 with expected crc 2023-10-09T11:05:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:23 smithi155 ceph-mon[184854]: pgmap v1226: 119 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 2 remapped+peering, 107 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s; 3981/61017 objects degraded (6.524%); 800 B/s, 1 objects/s recovering 2023-10-09T11:05:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:23 smithi155 ceph-mon[184854]: Health check failed: Degraded data redundancy: 3981/61017 objects degraded (6.524%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:23 smithi155 ceph-mon[184854]: failed to encode map e502 with expected crc 2023-10-09T11:05:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:23 smithi155 ceph-mon[184854]: failed to encode map e502 with expected crc 2023-10-09T11:05:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:23 smithi155 ceph-mon[184854]: osdmap e502: 8 total, 8 up, 8 in 2023-10-09T11:05:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:23 smithi155 ceph-mon[184854]: failed to encode map e502 with expected crc 2023-10-09T11:05:23.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:23 smithi155 ceph-mon[184854]: failed to encode map e502 with expected crc 2023-10-09T11:05:23.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:05:23.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:05:23.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:23.760 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:23.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:23.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:23.807 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:23.816 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T11:05:23.870 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:23.963 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:23.977 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 15 16 24743 24727 102.891 90.5625 0.0308902 0.00971018 2023-10-09T11:05:24.111 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:24.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:24.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:24.227 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:24.295 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:24.524 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:24.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:24.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:24.545 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:24 smithi125 ceph-mon[195507]: failed to encode map e503 with expected crc 2023-10-09T11:05:24.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:24 smithi125 ceph-mon[195507]: osdmap e503: 8 total, 8 up, 8 in 2023-10-09T11:05:24.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:24 smithi125 ceph-mon[195507]: failed to encode map e503 with expected crc 2023-10-09T11:05:24.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:24 smithi125 ceph-mon[195507]: failed to encode map e503 with expected crc 2023-10-09T11:05:24.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:24 smithi125 ceph-mon[195507]: failed to encode map e503 with expected crc 2023-10-09T11:05:24.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:24 smithi125 ceph-mon[195507]: pgmap v1229: 119 pgs: 6 peering, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 2 remapped+peering, 97 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 9325/68553 objects degraded (13.603%); 81 KiB/s, 2 objects/s recovering 2023-10-09T11:05:24.546 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:24 smithi125 ceph-mon[200197]: failed to encode map e503 with expected crc 2023-10-09T11:05:24.546 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:24 smithi125 ceph-mon[200197]: osdmap e503: 8 total, 8 up, 8 in 2023-10-09T11:05:24.546 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:24 smithi125 ceph-mon[200197]: failed to encode map e503 with expected crc 2023-10-09T11:05:24.546 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:24 smithi125 ceph-mon[200197]: failed to encode map e503 with expected crc 2023-10-09T11:05:24.547 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:24 smithi125 ceph-mon[200197]: failed to encode map e503 with expected crc 2023-10-09T11:05:24.547 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:24 smithi125 ceph-mon[200197]: pgmap v1229: 119 pgs: 6 peering, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 2 remapped+peering, 97 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 9325/68553 objects degraded (13.603%); 81 KiB/s, 2 objects/s recovering 2023-10-09T11:05:24.645 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:05:24.700 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:24 smithi155 ceph-mon[184854]: failed to encode map e503 with expected crc 2023-10-09T11:05:24.700 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:24 smithi155 ceph-mon[184854]: osdmap e503: 8 total, 8 up, 8 in 2023-10-09T11:05:24.701 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:24 smithi155 ceph-mon[184854]: failed to encode map e503 with expected crc 2023-10-09T11:05:24.701 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:24 smithi155 ceph-mon[184854]: failed to encode map e503 with expected crc 2023-10-09T11:05:24.701 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:24 smithi155 ceph-mon[184854]: failed to encode map e503 with expected crc 2023-10-09T11:05:24.701 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:24 smithi155 ceph-mon[184854]: pgmap v1229: 119 pgs: 6 peering, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 2 remapped+peering, 97 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 9325/68553 objects degraded (13.603%); 81 KiB/s, 2 objects/s recovering 2023-10-09T11:05:24.707 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:24.740 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:05:24.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:24.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:05:24.847 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:24.885 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:24.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:24.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:24.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:24.977 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 16 16 26159 26143 101.992 88.5 0.00699251 0.00979775 2023-10-09T11:05:25.024 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:25.067 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:25.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:25.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:25.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:25.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:05:25.250 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:25.281 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:05:25.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:25.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:05:25.386 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:25.441 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:25.545 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 7, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 5, 6, 7, 1, 3] 2023-10-09T11:05:25.545 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:05:25.545 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2023-10-09T11:05:25.546 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:05:25.647 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:05:25.672 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:25.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:25.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:25.681 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1710839337' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:05:25.681 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:25 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:05:25.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1710839337' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:05:25.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:25 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:05:25.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1710839337' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:05:25.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:25 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:05:25.781 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:25.845 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:25.978 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 17 16 27553 27537 101.115 87.125 0.00716627 0.00988431 2023-10-09T11:05:26.032 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:26.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:26.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:26.142 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:26.187 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:26.699 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:26 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:05:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:26 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:05:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:26 smithi155 ceph-mon[184854]: osdmap e504: 8 total, 8 up, 8 in 2023-10-09T11:05:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:26 smithi155 ceph-mon[184854]: failed to encode map e504 with expected crc 2023-10-09T11:05:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:26 smithi155 ceph-mon[184854]: failed to encode map e504 with expected crc 2023-10-09T11:05:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:26 smithi155 ceph-mon[184854]: pgmap v1231: 119 pgs: 6 peering, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 2 remapped+peering, 97 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 9325/68553 objects degraded (13.603%); 72 KiB/s, 2 objects/s recovering 2023-10-09T11:05:26.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:26 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:05:26.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:26 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:05:26.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:26 smithi125 ceph-mon[195507]: osdmap e504: 8 total, 8 up, 8 in 2023-10-09T11:05:26.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:26 smithi125 ceph-mon[195507]: failed to encode map e504 with expected crc 2023-10-09T11:05:26.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:26 smithi125 ceph-mon[195507]: failed to encode map e504 with expected crc 2023-10-09T11:05:26.818 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:26 smithi125 ceph-mon[195507]: pgmap v1231: 119 pgs: 6 peering, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 2 remapped+peering, 97 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 9325/68553 objects degraded (13.603%); 72 KiB/s, 2 objects/s recovering 2023-10-09T11:05:26.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:26 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:05:26.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:26 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:05:26.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:26 smithi125 ceph-mon[200197]: osdmap e504: 8 total, 8 up, 8 in 2023-10-09T11:05:26.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:26 smithi125 ceph-mon[200197]: failed to encode map e504 with expected crc 2023-10-09T11:05:26.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:26 smithi125 ceph-mon[200197]: failed to encode map e504 with expected crc 2023-10-09T11:05:26.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:26 smithi125 ceph-mon[200197]: pgmap v1231: 119 pgs: 6 peering, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+remapped, 2 remapped+peering, 97 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 9325/68553 objects degraded (13.603%); 72 KiB/s, 2 objects/s recovering 2023-10-09T11:05:26.978 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 18 16 29135 29119 100.99 98.875 0.00646941 0.00989472 2023-10-09T11:05:27.865 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:27.978 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 19 16 30776 30760 101.072 102.562 0.0122463 0.00988599 2023-10-09T11:05:28.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:28.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:28.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:28.159 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:27 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 9325/76068 objects degraded (12.259%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:28.159 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:27 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 9325/76068 objects degraded (12.259%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:28.162 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:28.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:27 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 9325/76068 objects degraded (12.259%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:28.883 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:28.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:28.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:28.978 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T11:05:28.979490+0000 min lat: 0.00101021 max lat: 2.16829 avg lat: 0.00989253 2023-10-09T11:05:28.978 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T11:05:28.979 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 20 16 32381 32365 101.034 100.312 0.00964954 0.00989253 2023-10-09T11:05:28.990 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:29.054 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:29.156 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:28 smithi125 ceph-mon[195507]: pgmap v1232: 119 pgs: 1 active+recovering+undersized+remapped, 6 peering, 7 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 9325/76068 objects degraded (12.259%); 941/76068 objects misplaced (1.237%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T11:05:29.159 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:28 smithi125 ceph-mon[200197]: pgmap v1232: 119 pgs: 1 active+recovering+undersized+remapped, 6 peering, 7 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 9325/76068 objects degraded (12.259%); 941/76068 objects misplaced (1.237%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T11:05:29.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:28 smithi155 ceph-mon[184854]: pgmap v1232: 119 pgs: 1 active+recovering+undersized+remapped, 6 peering, 7 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 9325/76068 objects degraded (12.259%); 941/76068 objects misplaced (1.237%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T11:05:29.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:29.500 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:29.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:29.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:29.609 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:29.631 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:05:29.651 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:29.658 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:05:29.678 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:29.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:29.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:29.693 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:05:29.733 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:05:29.779 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:05:29.787 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:29.818 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:05:29.840 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:29.860 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:05:29.879 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:05:29.884 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:05:29.884 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:05:29.884 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:05:29.884 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:05:29.884 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:05:29.884 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:05:29.884 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:05:29.884 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:05:29.924 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:29 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2087998632' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:29.979 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 21 16 33753 33737 100.306 85.75 0.00438689 0.00996313 2023-10-09T11:05:30.279 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:29 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2087998632' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:30.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:29 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2087998632' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:30.324 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:05:30.533 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:30.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:30.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:30.646 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:30.706 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:30.979 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 22 16 35034 35018 99.3847 80.0625 0.00743496 0.0100555 2023-10-09T11:05:30.989 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:30.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:30.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:31.096 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:31.102 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:31 smithi125 ceph-mon[195507]: pgmap v1233: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 9325/90207 objects degraded (10.337%); 941/90207 objects misplaced (1.043%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T11:05:31.102 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:31 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:31.102 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:31 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:05:31.103 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:31 smithi125 ceph-mon[200197]: pgmap v1233: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 9325/90207 objects degraded (10.337%); 941/90207 objects misplaced (1.043%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T11:05:31.103 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:31 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:31.103 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:31 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:05:31.146 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:31.220 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:31.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:31.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:31.329 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:31.359 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:05:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:05:31] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2023-10-09T11:05:31.403 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:31 smithi155 ceph-mon[184854]: pgmap v1233: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 9325/90207 objects degraded (10.337%); 941/90207 objects misplaced (1.043%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T11:05:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:31 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:31 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:05:31.483 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:31.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:31.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:31.590 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:31.661 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:31.691 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:31.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:31.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:31.796 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:31.831 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:31.852 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:05:31.852 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:05:31.852 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.852 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:36052 11.1 11:8e8e8b7b:::benchmark_data_smithi125_439543_object36051:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-10-09T11:05:31.852 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:31.836785+0000", 2023-10-09T11:05:31.852 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.012438704, 2023-10-09T11:05:31.852 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012475725, 2023-10-09T11:05:31.852 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:31.853 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:05:31.853 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:05:31.853 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:05:31.853 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:05:31.853 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 36052 2023-10-09T11:05:31.853 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.853 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:31.853 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.853 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:31.853 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836785+0000", 2023-10-09T11:05:31.853 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:31.854 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.854 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.854 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:31.854 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836785+0000", 2023-10-09T11:05:31.854 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:31.854 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.854 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.854 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:31.854 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836784+0000", 2023-10-09T11:05:31.854 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:05:31.855 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.855 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.855 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:31.855 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836823+0000", 2023-10-09T11:05:31.855 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8300000000000003e-05 2023-10-09T11:05:31.855 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.855 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.855 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:31.855 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836835+0000", 2023-10-09T11:05:31.855 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2116000000000001e-05 2023-10-09T11:05:31.855 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.856 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.856 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:31.856 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836843+0000", 2023-10-09T11:05:31.856 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7370000000000006e-06 2023-10-09T11:05:31.856 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.856 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.856 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:31.856 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.838633+0000", 2023-10-09T11:05:31.856 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0017905449999999999 2023-10-09T11:05:31.856 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.856 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.857 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:31.857 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.838770+0000", 2023-10-09T11:05:31.857 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000137243 2023-10-09T11:05:31.857 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.857 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.857 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:05:31.857 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.838932+0000", 2023-10-09T11:05:31.857 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016121899999999999 2023-10-09T11:05:31.857 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.857 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.857 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:05:31.858 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.842620+0000", 2023-10-09T11:05:31.858 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036881420000000002 2023-10-09T11:05:31.858 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.858 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.858 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:31.858 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.842649+0000", 2023-10-09T11:05:31.858 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8904999999999999e-05 2023-10-09T11:05:31.858 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.858 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.858 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:31.858 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.848896+0000", 2023-10-09T11:05:31.858 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062472669999999999 2023-10-09T11:05:31.859 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.859 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.859 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:05:31.859 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.848944+0000", 2023-10-09T11:05:31.859 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8470999999999997e-05 2023-10-09T11:05:31.859 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:31.859 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:31.859 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:31.859 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.859 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.859 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:36056 11.2 11:48fe4829:::benchmark_data_smithi125_439543_object36055:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-10-09T11:05:31.860 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:31.836849+0000", 2023-10-09T11:05:31.860 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.012375156999999999, 2023-10-09T11:05:31.860 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012535235, 2023-10-09T11:05:31.860 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:31.860 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:05:31.860 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:05:31.860 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:05:31.860 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:05:31.860 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 36056 2023-10-09T11:05:31.860 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.860 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:31.860 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.861 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:31.861 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836849+0000", 2023-10-09T11:05:31.861 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:31.861 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.861 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.861 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:31.861 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836849+0000", 2023-10-09T11:05:31.861 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:31.861 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.861 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.861 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:31.862 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836848+0000", 2023-10-09T11:05:31.862 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:31.862 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.862 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.862 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:31.862 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836867+0000", 2023-10-09T11:05:31.862 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8797999999999999e-05 2023-10-09T11:05:31.862 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.862 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.862 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:31.862 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836873+0000", 2023-10-09T11:05:31.862 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2820000000000002e-06 2023-10-09T11:05:31.863 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.863 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.863 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:31.863 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836878+0000", 2023-10-09T11:05:31.863 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9180000000000002e-06 2023-10-09T11:05:31.863 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.863 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.863 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:31.863 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.843529+0000", 2023-10-09T11:05:31.863 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066506990000000004 2023-10-09T11:05:31.863 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.863 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.864 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:31.864 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.843615+0000", 2023-10-09T11:05:31.864 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6268999999999997e-05 2023-10-09T11:05:31.864 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.864 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.864 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,2", 2023-10-09T11:05:31.864 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.843714+0000", 2023-10-09T11:05:31.864 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8948000000000002e-05 2023-10-09T11:05:31.864 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.864 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.864 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:05:31.864 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.846008+0000", 2023-10-09T11:05:31.865 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022934560000000001 2023-10-09T11:05:31.865 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:31.865 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:31.865 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:31.865 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.865 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.865 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:36057 11.a 11:52c23dc2:::benchmark_data_smithi125_439543_object36056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-10-09T11:05:31.865 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:31.836892+0000", 2023-10-09T11:05:31.865 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.012332097, 2023-10-09T11:05:31.865 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012587094, 2023-10-09T11:05:31.865 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:31.865 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:05:31.866 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:05:31.866 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:05:31.866 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:05:31.866 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 36057 2023-10-09T11:05:31.866 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.866 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:31.866 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.866 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:31.866 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836892+0000", 2023-10-09T11:05:31.866 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:31.866 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.866 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.867 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:31.867 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836892+0000", 2023-10-09T11:05:31.867 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:31.867 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.867 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.867 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:31.867 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836892+0000", 2023-10-09T11:05:31.867 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:31.867 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.867 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.867 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:31.867 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836913+0000", 2023-10-09T11:05:31.867 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1667000000000001e-05 2023-10-09T11:05:31.868 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.868 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.868 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:31.868 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836920+0000", 2023-10-09T11:05:31.868 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4649999999999999e-06 2023-10-09T11:05:31.868 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.868 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.868 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:31.868 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.836925+0000", 2023-10-09T11:05:31.868 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8269999999999997e-06 2023-10-09T11:05:31.868 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.868 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.869 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:31.869 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.844266+0000", 2023-10-09T11:05:31.869 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0073416760000000001 2023-10-09T11:05:31.869 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.869 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.869 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:31.869 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.844411+0000", 2023-10-09T11:05:31.869 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014514899999999999 2023-10-09T11:05:31.869 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.869 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.869 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T11:05:31.869 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.844588+0000", 2023-10-09T11:05:31.870 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017666099999999999 2023-10-09T11:05:31.870 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.870 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.870 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:05:31.870 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.847865+0000", 2023-10-09T11:05:31.870 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032766470000000002 2023-10-09T11:05:31.870 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:31.870 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:31.870 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:31.870 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.870 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.870 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:36063 11.1 11:888ad119:::benchmark_data_smithi125_439543_object36062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-10-09T11:05:31.871 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:31.844989+0000", 2023-10-09T11:05:31.871 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0042347349999999999, 2023-10-09T11:05:31.871 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045843790000000004, 2023-10-09T11:05:31.871 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:31.871 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:05:31.871 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:05:31.871 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:05:31.871 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:05:31.871 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 36063 2023-10-09T11:05:31.871 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.871 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:31.871 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.872 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:31.872 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.844989+0000", 2023-10-09T11:05:31.872 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:31.872 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.872 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.872 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:31.872 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.844989+0000", 2023-10-09T11:05:31.872 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:31.872 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.872 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.872 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:31.872 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.844988+0000", 2023-10-09T11:05:31.873 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:05:31.873 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.873 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.873 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:31.873 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.845444+0000", 2023-10-09T11:05:31.873 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00045624499999999998 2023-10-09T11:05:31.873 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.873 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.873 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:31.873 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.845460+0000", 2023-10-09T11:05:31.873 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5563e-05 2023-10-09T11:05:31.873 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.874 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.874 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:31.874 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.845468+0000", 2023-10-09T11:05:31.874 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2470000000000008e-06 2023-10-09T11:05:31.874 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.874 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.874 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:31.874 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.845611+0000", 2023-10-09T11:05:31.874 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014278899999999999 2023-10-09T11:05:31.874 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.874 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.875 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:31.875 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.845746+0000", 2023-10-09T11:05:31.875 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013498500000000001 2023-10-09T11:05:31.875 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.875 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.875 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:05:31.875 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.845896+0000", 2023-10-09T11:05:31.875 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015052800000000001 2023-10-09T11:05:31.875 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:31.875 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:31.875 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:31.875 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.875 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.876 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:36067 11.9 11.c18a08e9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-10-09T11:05:31.876 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:31.845482+0000", 2023-10-09T11:05:31.876 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.003741678, 2023-10-09T11:05:31.876 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0041728119999999997, 2023-10-09T11:05:31.876 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:31.876 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T11:05:31.876 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:05:31.876 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:05:31.876 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:05:31.876 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 36067 2023-10-09T11:05:31.876 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.876 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:31.877 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.877 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:31.877 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.845482+0000", 2023-10-09T11:05:31.877 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:31.877 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.877 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.877 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:31.877 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.845482+0000", 2023-10-09T11:05:31.877 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:31.877 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.877 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.877 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:31.878 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.845482+0000", 2023-10-09T11:05:31.878 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:31.878 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.878 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.878 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:31.878 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.845502+0000", 2023-10-09T11:05:31.878 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0279999999999999e-05 2023-10-09T11:05:31.878 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.878 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.878 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:31.878 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.845510+0000", 2023-10-09T11:05:31.878 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4730000000000003e-06 2023-10-09T11:05:31.879 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.879 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.879 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:31.879 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.845516+0000", 2023-10-09T11:05:31.879 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3589999999999996e-06 2023-10-09T11:05:31.879 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:31.879 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:31.879 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:31.879 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.879 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.879 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:36061 11.0 e504/495 11:037056f9:::benchmark_data_smithi125_439543_object36060:head v 504'2267, mlcod=504'2265)", 2023-10-09T11:05:31.879 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:31.846467+0000", 2023-10-09T11:05:31.880 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0027568060000000001, 2023-10-09T11:05:31.880 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003251351, 2023-10-09T11:05:31.880 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:31.880 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:05:31.880 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:31.880 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.880 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:31.880 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.846467+0000", 2023-10-09T11:05:31.880 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:31.880 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.880 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.880 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:31.880 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.846467+0000", 2023-10-09T11:05:31.881 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:31.881 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.881 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.881 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:31.881 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.846467+0000", 2023-10-09T11:05:31.881 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:05:31.881 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.881 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.881 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:31.881 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.846493+0000", 2023-10-09T11:05:31.881 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6126e-05 2023-10-09T11:05:31.881 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.881 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.882 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:31.882 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.846504+0000", 2023-10-09T11:05:31.882 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1221e-05 2023-10-09T11:05:31.882 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.882 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.882 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:31.882 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.846511+0000", 2023-10-09T11:05:31.882 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4030000000000001e-06 2023-10-09T11:05:31.882 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.882 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.882 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:31.882 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.846722+0000", 2023-10-09T11:05:31.883 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00021090799999999999 2023-10-09T11:05:31.883 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.883 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.883 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:31.883 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.846755+0000", 2023-10-09T11:05:31.883 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3207000000000002e-05 2023-10-09T11:05:31.883 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:31.883 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:31.883 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:31.883 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.883 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.883 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.36510.0:36056 11.2 e504/495)", 2023-10-09T11:05:31.883 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:31.847930+0000", 2023-10-09T11:05:31.884 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0012938959999999999, 2023-10-09T11:05:31.884 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018515070000000001, 2023-10-09T11:05:31.884 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:31.884 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T11:05:31.884 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:31.884 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.884 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:31.884 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.847930+0000", 2023-10-09T11:05:31.884 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:31.884 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.884 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.885 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:31.885 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.847930+0000", 2023-10-09T11:05:31.885 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:31.885 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.885 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.885 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:31.885 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.847930+0000", 2023-10-09T11:05:31.885 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:31.885 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.885 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.885 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:31.885 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.847934+0000", 2023-10-09T11:05:31.886 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5079999999999999e-06 2023-10-09T11:05:31.886 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.886 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.886 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:31.886 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.847941+0000", 2023-10-09T11:05:31.886 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2339999999999997e-06 2023-10-09T11:05:31.886 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:31.886 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:31.886 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:31.886 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:31.847946+0000", 2023-10-09T11:05:31.886 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.245e-06 2023-10-09T11:05:31.887 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:31.887 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:31.887 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:31.887 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:31.887 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T11:05:31.887 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 7 2023-10-09T11:05:31.887 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:05:31.888 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:31.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:31.888 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:31.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:31.979 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 23 16 36209 36193 98.2572 73.4375 0.00571256 0.0101702 2023-10-09T11:05:31.990 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:32.052 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:32.327 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:05:32.342 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:05:32.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:32.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:32.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:32.655 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:32.709 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:32.979 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 24 16 37539 37523 97.627 83.125 0.0120214 0.0102369 2023-10-09T11:05:33.098 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:05:33.195 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:33.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:33.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:33.301 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:33.352 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:33 smithi125 ceph-mon[195507]: pgmap v1234: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 9325/98457 objects degraded (9.471%); 347/98457 objects misplaced (0.352%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T11:05:33.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:33 smithi125 ceph-mon[200197]: pgmap v1234: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 9325/98457 objects degraded (9.471%); 347/98457 objects misplaced (0.352%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T11:05:33.355 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:33.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:33 smithi155 ceph-mon[184854]: pgmap v1234: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 9325/98457 objects degraded (9.471%); 347/98457 objects misplaced (0.352%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T11:05:33.980 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 25 16 38919 38903 97.1692 86.25 0.0134617 0.0102848 2023-10-09T11:05:34.465 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:34 smithi125 ceph-mon[195507]: from='client.36534 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:05:34.465 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:34 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 9325/98457 objects degraded (9.471%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:34.465 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:34 smithi125 ceph-mon[200197]: from='client.36534 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:05:34.465 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:34 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 9325/98457 objects degraded (9.471%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:34.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:34 smithi155 ceph-mon[184854]: from='client.36534 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:05:34.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:34 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 9325/98457 objects degraded (9.471%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:34.484 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T11:05:34.980 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 26 16 39524 39508 94.8879 37.8125 0.00262968 0.0102868 2023-10-09T11:05:35.403 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:35.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:35 smithi125 ceph-mon[195507]: pgmap v1235: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 9325/110985 objects degraded (8.402%); 347/110985 objects misplaced (0.313%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T11:05:35.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:35 smithi125 ceph-mon[195507]: osdmap e505: 8 total, 8 up, 8 in 2023-10-09T11:05:35.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:35 smithi125 ceph-mon[195507]: failed to encode map e505 with expected crc 2023-10-09T11:05:35.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:35 smithi125 ceph-mon[195507]: failed to encode map e505 with expected crc 2023-10-09T11:05:35.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:35.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:35.417 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:35 smithi125 ceph-mon[200197]: pgmap v1235: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 9325/110985 objects degraded (8.402%); 347/110985 objects misplaced (0.313%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T11:05:35.417 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:35 smithi125 ceph-mon[200197]: osdmap e505: 8 total, 8 up, 8 in 2023-10-09T11:05:35.417 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:35 smithi125 ceph-mon[200197]: failed to encode map e505 with expected crc 2023-10-09T11:05:35.417 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:35 smithi125 ceph-mon[200197]: failed to encode map e505 with expected crc 2023-10-09T11:05:35.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:35 smithi155 ceph-mon[184854]: pgmap v1235: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 9325/110985 objects degraded (8.402%); 347/110985 objects misplaced (0.313%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T11:05:35.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:35 smithi155 ceph-mon[184854]: osdmap e505: 8 total, 8 up, 8 in 2023-10-09T11:05:35.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:35 smithi155 ceph-mon[184854]: failed to encode map e505 with expected crc 2023-10-09T11:05:35.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:35 smithi155 ceph-mon[184854]: failed to encode map e505 with expected crc 2023-10-09T11:05:35.499 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1234,"stamp":"2023-10-09T11:05:31.801255+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2149848992,"num_objects":32819,"num_object_clones":0,"num_object_copies":98457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9325,"num_objects_misplaced":347,"num_objects_unfound":0,"num_objects_dirty":32819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":32856,"num_write_kb":2098909,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1169,"num_bytes_recovered":71396748,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":37841,"ondisk_log_size":37841,"up":357,"acting":349,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":381,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12988624,"kb_used_data":7456144,"kb_used_omap":134,"kb_used_meta":5532281,"kb_avail":736972592,"statfs":{"total":767960285184,"available":754659934208,"internally_reserved":0,"allocated":7635091456,"data_stored":7617949086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":137532,"internal_metadata":5665056452},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":653262848,"num_objects":9968,"num_object_clones":0,"num_object_copies":29904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":347,"num_objects_unfound":0,"num_objects_dirty":9968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9968,"num_write_kb":637952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1061,"num_bytes_recovered":69533696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.495872"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":743,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.314543+0000","last_change":"2023-10-09T11:02:36.217074+0000","last_active":"2023-10-09T11:05:23.314543+0000","last_peered":"2023-10-09T11:05:23.314543+0000","last_clean":"2023-10-09T11:05:23.314543+0000","last_became_active":"2023-10-09T11:02:36.216494+0000","last_became_peered":"2023-10-09T11:02:36.216494+0000","last_unstale":"2023-10-09T11:05:23.314543+0000","last_undegraded":"2023-10-09T11:05:23.314543+0000","last_fullsized":"2023-10-09T11:05:23.314543+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:56.760915+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":286,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.318305+0000","last_change":"2023-10-09T11:03:21.185383+0000","last_active":"2023-10-09T11:05:23.318305+0000","last_peered":"2023-10-09T11:05:23.318305+0000","last_clean":"2023-10-09T11:05:23.318305+0000","last_became_active":"2023-10-09T11:03:21.185022+0000","last_became_peered":"2023-10-09T11:03:21.185022+0000","last_unstale":"2023-10-09T11:05:23.318305+0000","last_undegraded":"2023-10-09T11:05:23.318305+0000","last_fullsized":"2023-10-09T11:05:23.318305+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:05.230434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":29,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.310722+0000","last_change":"2023-10-09T11:04:50.319477+0000","last_active":"2023-10-09T11:05:23.310722+0000","last_peered":"2023-10-09T11:05:23.310722+0000","last_clean":"2023-10-09T11:05:23.310722+0000","last_became_active":"2023-10-09T11:04:50.319148+0000","last_became_peered":"2023-10-09T11:04:50.319148+0000","last_unstale":"2023-10-09T11:05:23.310722+0000","last_undegraded":"2023-10-09T11:05:23.310722+0000","last_fullsized":"2023-10-09T11:05:23.310722+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:48.725566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":661,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.314817+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:05:23.314817+0000","last_peered":"2023-10-09T11:05:23.314817+0000","last_clean":"2023-10-09T11:05:23.314817+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:05:23.314817+0000","last_undegraded":"2023-10-09T11:05:23.314817+0000","last_fullsized":"2023-10-09T11:05:23.314817+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":217,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.320869+0000","last_change":"2023-10-09T11:04:50.333711+0000","last_active":"2023-10-09T11:05:23.320869+0000","last_peered":"2023-10-09T11:05:23.320869+0000","last_clean":"2023-10-09T11:05:23.320869+0000","last_became_active":"2023-10-09T11:04:50.333262+0000","last_became_peered":"2023-10-09T11:04:50.333262+0000","last_unstale":"2023-10-09T11:05:23.320869+0000","last_undegraded":"2023-10-09T11:05:23.320869+0000","last_fullsized":"2023-10-09T11:05:23.320869+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.843765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":471,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.316096+0000","last_change":"2023-10-09T11:04:50.326431+0000","last_active":"2023-10-09T11:05:23.316096+0000","last_peered":"2023-10-09T11:05:23.316096+0000","last_clean":"2023-10-09T11:05:23.316096+0000","last_became_active":"2023-10-09T11:04:50.325407+0000","last_became_peered":"2023-10-09T11:04:50.325407+0000","last_unstale":"2023-10-09T11:05:23.316096+0000","last_undegraded":"2023-10-09T11:05:23.316096+0000","last_fullsized":"2023-10-09T11:05:23.316096+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:24.792171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":924,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.308109+0000","last_change":"2023-10-09T11:04:57.954692+0000","last_active":"2023-10-09T11:05:23.308109+0000","last_peered":"2023-10-09T11:05:23.308109+0000","last_clean":"2023-10-09T11:05:23.308109+0000","last_became_active":"2023-10-09T11:04:57.953958+0000","last_became_peered":"2023-10-09T11:04:57.953958+0000","last_unstale":"2023-10-09T11:05:23.308109+0000","last_undegraded":"2023-10-09T11:05:23.308109+0000","last_fullsized":"2023-10-09T11:05:23.308109+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:06:35.079099+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":29,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.311696+0000","last_change":"2023-10-09T11:04:50.310477+0000","last_active":"2023-10-09T11:05:23.311696+0000","last_peered":"2023-10-09T11:05:23.311696+0000","last_clean":"2023-10-09T11:05:23.311696+0000","last_became_active":"2023-10-09T11:04:50.310067+0000","last_became_peered":"2023-10-09T11:04:50.310067+0000","last_unstale":"2023-10-09T11:05:23.311696+0000","last_undegraded":"2023-10-09T11:05:23.311696+0000","last_fullsized":"2023-10-09T11:05:23.311696+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:52.884716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":313,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.318689+0000","last_change":"2023-10-09T11:05:20.310581+0000","last_active":"2023-10-09T11:05:23.318689+0000","last_peered":"2023-10-09T11:05:23.318689+0000","last_clean":"2023-10-09T11:05:23.318689+0000","last_became_active":"2023-10-09T11:05:20.309830+0000","last_became_peered":"2023-10-09T11:05:20.309830+0000","last_unstale":"2023-10-09T11:05:23.318689+0000","last_undegraded":"2023-10-09T11:05:23.318689+0000","last_fullsized":"2023-10-09T11:05:23.318689+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:24.508211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":825,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.311850+0000","last_change":"2023-10-09T11:02:36.216785+0000","last_active":"2023-10-09T11:05:23.311850+0000","last_peered":"2023-10-09T11:05:23.311850+0000","last_clean":"2023-10-09T11:05:23.311850+0000","last_became_active":"2023-10-09T11:02:36.215919+0000","last_became_peered":"2023-10-09T11:02:36.215919+0000","last_unstale":"2023-10-09T11:05:23.311850+0000","last_undegraded":"2023-10-09T11:05:23.311850+0000","last_fullsized":"2023-10-09T11:05:23.311850+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:08:40.893630+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":174,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.311361+0000","last_change":"2023-10-09T11:04:50.337680+0000","last_active":"2023-10-09T11:05:23.311361+0000","last_peered":"2023-10-09T11:05:23.311361+0000","last_clean":"2023-10-09T11:05:23.311361+0000","last_became_active":"2023-10-09T11:04:50.337343+0000","last_became_peered":"2023-10-09T11:04:50.337343+0000","last_unstale":"2023-10-09T11:05:23.311361+0000","last_undegraded":"2023-10-09T11:05:23.311361+0000","last_fullsized":"2023-10-09T11:05:23.311361+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:29.936199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":38,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.312976+0000","last_change":"2023-10-09T11:05:20.299453+0000","last_active":"2023-10-09T11:05:23.312976+0000","last_peered":"2023-10-09T11:05:23.312976+0000","last_clean":"2023-10-09T11:05:23.312976+0000","last_became_active":"2023-10-09T11:05:20.299117+0000","last_became_peered":"2023-10-09T11:05:20.299117+0000","last_unstale":"2023-10-09T11:05:23.312976+0000","last_undegraded":"2023-10-09T11:05:23.312976+0000","last_fullsized":"2023-10-09T11:05:23.312976+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:30.072813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":832,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.311162+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:05:23.311162+0000","last_peered":"2023-10-09T11:05:23.311162+0000","last_clean":"2023-10-09T11:05:23.311162+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:05:23.311162+0000","last_undegraded":"2023-10-09T11:05:23.311162+0000","last_fullsized":"2023-10-09T11:05:23.311162+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":355,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.316516+0000","last_change":"2023-10-09T11:02:33.198185+0000","last_active":"2023-10-09T11:05:23.316516+0000","last_peered":"2023-10-09T11:05:23.316516+0000","last_clean":"2023-10-09T11:05:23.316516+0000","last_became_active":"2023-10-09T11:02:33.197870+0000","last_became_peered":"2023-10-09T11:02:33.197870+0000","last_unstale":"2023-10-09T11:05:23.316516+0000","last_undegraded":"2023-10-09T11:05:23.316516+0000","last_fullsized":"2023-10-09T11:05:23.316516+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:09:21.200979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":38,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.321144+0000","last_change":"2023-10-09T11:05:20.299219+0000","last_active":"2023-10-09T11:05:23.321144+0000","last_peered":"2023-10-09T11:05:23.321144+0000","last_clean":"2023-10-09T11:05:23.321144+0000","last_became_active":"2023-10-09T11:05:20.298965+0000","last_became_peered":"2023-10-09T11:05:20.298965+0000","last_unstale":"2023-10-09T11:05:23.321144+0000","last_undegraded":"2023-10-09T11:05:23.321144+0000","last_fullsized":"2023-10-09T11:05:23.321144+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:07:51.716239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":733,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.329664+0000","last_change":"2023-10-09T11:05:23.328981+0000","last_active":"2023-10-09T11:05:23.329664+0000","last_peered":"2023-10-09T11:05:23.329664+0000","last_clean":"2023-10-09T11:05:23.329664+0000","last_became_active":"2023-10-09T11:05:23.328616+0000","last_became_peered":"2023-10-09T11:05:23.328616+0000","last_unstale":"2023-10-09T11:05:23.329664+0000","last_undegraded":"2023-10-09T11:05:23.329664+0000","last_fullsized":"2023-10-09T11:05:23.329664+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:15.315819+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":124,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.313364+0000","last_change":"2023-10-09T11:04:50.569564+0000","last_active":"2023-10-09T11:05:23.313364+0000","last_peered":"2023-10-09T11:05:23.313364+0000","last_clean":"2023-10-09T11:05:23.313364+0000","last_became_active":"2023-10-09T11:04:50.338122+0000","last_became_peered":"2023-10-09T11:04:50.338122+0000","last_unstale":"2023-10-09T11:05:23.313364+0000","last_undegraded":"2023-10-09T11:05:23.313364+0000","last_fullsized":"2023-10-09T11:05:23.313364+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:19.475443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":282,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.318845+0000","last_change":"2023-10-09T11:05:20.316265+0000","last_active":"2023-10-09T11:05:23.318845+0000","last_peered":"2023-10-09T11:05:23.318845+0000","last_clean":"2023-10-09T11:05:23.318845+0000","last_became_active":"2023-10-09T11:05:20.315853+0000","last_became_peered":"2023-10-09T11:05:20.315853+0000","last_unstale":"2023-10-09T11:05:23.318845+0000","last_undegraded":"2023-10-09T11:05:23.318845+0000","last_fullsized":"2023-10-09T11:05:23.318845+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:35.093981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":895,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.322637+0000","last_change":"2023-10-09T11:04:53.274921+0000","last_active":"2023-10-09T11:05:23.322637+0000","last_peered":"2023-10-09T11:05:23.322637+0000","last_clean":"2023-10-09T11:05:23.322637+0000","last_became_active":"2023-10-09T11:04:53.274518+0000","last_became_peered":"2023-10-09T11:04:53.274518+0000","last_unstale":"2023-10-09T11:05:23.322637+0000","last_undegraded":"2023-10-09T11:05:23.322637+0000","last_fullsized":"2023-10-09T11:05:23.322637+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:14.315301+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":129,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.309048+0000","last_change":"2023-10-09T11:05:20.325237+0000","last_active":"2023-10-09T11:05:23.309048+0000","last_peered":"2023-10-09T11:05:23.309048+0000","last_clean":"2023-10-09T11:05:23.309048+0000","last_became_active":"2023-10-09T11:05:20.314929+0000","last_became_peered":"2023-10-09T11:05:20.314929+0000","last_unstale":"2023-10-09T11:05:23.309048+0000","last_undegraded":"2023-10-09T11:05:23.309048+0000","last_fullsized":"2023-10-09T11:05:23.309048+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:58.898626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":524,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.308491+0000","last_change":"2023-10-09T11:05:20.311424+0000","last_active":"2023-10-09T11:05:23.308491+0000","last_peered":"2023-10-09T11:05:23.308491+0000","last_clean":"2023-10-09T11:05:23.308491+0000","last_became_active":"2023-10-09T11:05:20.311097+0000","last_became_peered":"2023-10-09T11:05:20.311097+0000","last_unstale":"2023-10-09T11:05:23.308491+0000","last_undegraded":"2023-10-09T11:05:23.308491+0000","last_fullsized":"2023-10-09T11:05:23.308491+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:19.274582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":908,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.315190+0000","last_change":"2023-10-09T11:04:53.275016+0000","last_active":"2023-10-09T11:05:23.315190+0000","last_peered":"2023-10-09T11:05:23.315190+0000","last_clean":"2023-10-09T11:05:23.315190+0000","last_became_active":"2023-10-09T11:04:53.274681+0000","last_became_peered":"2023-10-09T11:04:53.274681+0000","last_unstale":"2023-10-09T11:05:23.315190+0000","last_undegraded":"2023-10-09T11:05:23.315190+0000","last_fullsized":"2023-10-09T11:05:23.315190+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:22.631058+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":85,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.309852+0000","last_change":"2023-10-09T11:02:33.207032+0000","last_active":"2023-10-09T11:05:23.309852+0000","last_peered":"2023-10-09T11:05:23.309852+0000","last_clean":"2023-10-09T11:05:23.309852+0000","last_became_active":"2023-10-09T11:02:33.196379+0000","last_became_peered":"2023-10-09T11:02:33.196379+0000","last_unstale":"2023-10-09T11:05:23.309852+0000","last_undegraded":"2023-10-09T11:05:23.309852+0000","last_fullsized":"2023-10-09T11:05:23.309852+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:56.935603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":29,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.310447+0000","last_change":"2023-10-09T11:04:50.323062+0000","last_active":"2023-10-09T11:05:23.310447+0000","last_peered":"2023-10-09T11:05:23.310447+0000","last_clean":"2023-10-09T11:05:23.310447+0000","last_became_active":"2023-10-09T11:04:50.322749+0000","last_became_peered":"2023-10-09T11:04:50.322749+0000","last_unstale":"2023-10-09T11:05:23.310447+0000","last_undegraded":"2023-10-09T11:05:23.310447+0000","last_fullsized":"2023-10-09T11:05:23.310447+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:44.886893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":815,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.327842+0000","last_change":"2023-10-09T11:05:23.323856+0000","last_active":"2023-10-09T11:05:23.327842+0000","last_peered":"2023-10-09T11:05:23.327842+0000","last_clean":"2023-10-09T11:05:23.327842+0000","last_became_active":"2023-10-09T11:05:23.322763+0000","last_became_peered":"2023-10-09T11:05:23.322763+0000","last_unstale":"2023-10-09T11:05:23.327842+0000","last_undegraded":"2023-10-09T11:05:23.327842+0000","last_fullsized":"2023-10-09T11:05:23.327842+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.760820+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":470,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.312224+0000","last_change":"2023-10-09T11:04:50.394768+0000","last_active":"2023-10-09T11:05:23.312224+0000","last_peered":"2023-10-09T11:05:23.312224+0000","last_clean":"2023-10-09T11:05:23.312224+0000","last_became_active":"2023-10-09T11:04:50.394320+0000","last_became_peered":"2023-10-09T11:04:50.394320+0000","last_unstale":"2023-10-09T11:05:23.312224+0000","last_undegraded":"2023-10-09T11:05:23.312224+0000","last_fullsized":"2023-10-09T11:05:23.312224+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:07.269747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":425,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.316818+0000","last_change":"2023-10-09T11:04:50.338281+0000","last_active":"2023-10-09T11:05:23.316818+0000","last_peered":"2023-10-09T11:05:23.316818+0000","last_clean":"2023-10-09T11:05:23.316818+0000","last_became_active":"2023-10-09T11:04:50.337943+0000","last_became_peered":"2023-10-09T11:04:50.337943+0000","last_unstale":"2023-10-09T11:05:23.316818+0000","last_undegraded":"2023-10-09T11:05:23.316818+0000","last_fullsized":"2023-10-09T11:05:23.316818+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:46:24.599678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":837,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.321381+0000","last_change":"2023-10-09T11:04:53.274236+0000","last_active":"2023-10-09T11:05:23.321381+0000","last_peered":"2023-10-09T11:05:23.321381+0000","last_clean":"2023-10-09T11:05:23.321381+0000","last_became_active":"2023-10-09T11:04:53.273817+0000","last_became_peered":"2023-10-09T11:04:53.273817+0000","last_unstale":"2023-10-09T11:05:23.321381+0000","last_undegraded":"2023-10-09T11:05:23.321381+0000","last_fullsized":"2023-10-09T11:05:23.321381+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:05.096905+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":99,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.308249+0000","last_change":"2023-10-09T11:05:20.312129+0000","last_active":"2023-10-09T11:05:23.308249+0000","last_peered":"2023-10-09T11:05:23.308249+0000","last_clean":"2023-10-09T11:05:23.308249+0000","last_became_active":"2023-10-09T11:05:20.311810+0000","last_became_peered":"2023-10-09T11:05:20.311810+0000","last_unstale":"2023-10-09T11:05:23.308249+0000","last_undegraded":"2023-10-09T11:05:23.308249+0000","last_fullsized":"2023-10-09T11:05:23.308249+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:48.813391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":183,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.315815+0000","last_change":"2023-10-09T11:04:50.326593+0000","last_active":"2023-10-09T11:05:23.315815+0000","last_peered":"2023-10-09T11:05:23.315815+0000","last_clean":"2023-10-09T11:05:23.315815+0000","last_became_active":"2023-10-09T11:04:50.325752+0000","last_became_peered":"2023-10-09T11:04:50.325752+0000","last_unstale":"2023-10-09T11:05:23.315815+0000","last_undegraded":"2023-10-09T11:05:23.315815+0000","last_fullsized":"2023-10-09T11:05:23.315815+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:21:55.890142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":812,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.328709+0000","last_change":"2023-10-09T11:05:23.324537+0000","last_active":"2023-10-09T11:05:23.328709+0000","last_peered":"2023-10-09T11:05:23.328709+0000","last_clean":"2023-10-09T11:05:23.328709+0000","last_became_active":"2023-10-09T11:05:23.324187+0000","last_became_peered":"2023-10-09T11:05:23.324187+0000","last_unstale":"2023-10-09T11:05:23.328709+0000","last_undegraded":"2023-10-09T11:05:23.328709+0000","last_fullsized":"2023-10-09T11:05:23.328709+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:25.286106+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":24,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.315225+0000","last_change":"2023-10-09T11:05:20.323565+0000","last_active":"2023-10-09T11:05:23.315225+0000","last_peered":"2023-10-09T11:05:23.315225+0000","last_clean":"2023-10-09T11:05:23.315225+0000","last_became_active":"2023-10-09T11:05:20.309672+0000","last_became_peered":"2023-10-09T11:05:20.309672+0000","last_unstale":"2023-10-09T11:05:23.315225+0000","last_undegraded":"2023-10-09T11:05:23.315225+0000","last_fullsized":"2023-10-09T11:05:23.315225+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:50.211465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":29,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.311181+0000","last_change":"2023-10-09T11:04:50.336681+0000","last_active":"2023-10-09T11:05:23.311181+0000","last_peered":"2023-10-09T11:05:23.311181+0000","last_clean":"2023-10-09T11:05:23.311181+0000","last_became_active":"2023-10-09T11:04:50.335659+0000","last_became_peered":"2023-10-09T11:04:50.335659+0000","last_unstale":"2023-10-09T11:05:23.311181+0000","last_undegraded":"2023-10-09T11:05:23.311181+0000","last_fullsized":"2023-10-09T11:05:23.311181+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:26:49.879883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"504'2254","reported_seq":2271,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T11:05:31.649115+0000","last_change":"2023-10-09T11:05:05.922326+0000","last_active":"2023-10-09T11:05:31.649115+0000","last_peered":"2023-10-09T11:05:31.649115+0000","last_clean":"2023-10-09T11:05:31.649115+0000","last_became_active":"2023-10-09T11:05:05.922035+0000","last_became_peered":"2023-10-09T11:05:05.922035+0000","last_unstale":"2023-10-09T11:05:31.649115+0000","last_undegraded":"2023-10-09T11:05:31.649115+0000","last_fullsized":"2023-10-09T11:05:31.649115+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":2254,"log_dups_size":0,"ondisk_log_size":2254,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:07.719517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147718144,"num_objects":2254,"num_object_clones":0,"num_object_copies":6762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2254,"num_write_kb":144256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":793,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.311766+0000","last_change":"2023-10-09T11:02:36.217050+0000","last_active":"2023-10-09T11:05:23.311766+0000","last_peered":"2023-10-09T11:05:23.311766+0000","last_clean":"2023-10-09T11:05:23.311766+0000","last_became_active":"2023-10-09T11:02:36.216387+0000","last_became_peered":"2023-10-09T11:02:36.216387+0000","last_unstale":"2023-10-09T11:05:23.311766+0000","last_undegraded":"2023-10-09T11:05:23.311766+0000","last_fullsized":"2023-10-09T11:05:23.311766+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:19.627177+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":83,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.308858+0000","last_change":"2023-10-09T11:02:33.221990+0000","last_active":"2023-10-09T11:05:23.308858+0000","last_peered":"2023-10-09T11:05:23.308858+0000","last_clean":"2023-10-09T11:05:23.308858+0000","last_became_active":"2023-10-09T11:02:33.198155+0000","last_became_peered":"2023-10-09T11:02:33.198155+0000","last_unstale":"2023-10-09T11:05:23.308858+0000","last_undegraded":"2023-10-09T11:05:23.308858+0000","last_fullsized":"2023-10-09T11:05:23.308858+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:32.563344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":446,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.314628+0000","last_change":"2023-10-09T11:05:20.309547+0000","last_active":"2023-10-09T11:05:23.314628+0000","last_peered":"2023-10-09T11:05:23.314628+0000","last_clean":"2023-10-09T11:05:23.314628+0000","last_became_active":"2023-10-09T11:05:20.309071+0000","last_became_peered":"2023-10-09T11:05:20.309071+0000","last_unstale":"2023-10-09T11:05:23.314628+0000","last_undegraded":"2023-10-09T11:05:23.314628+0000","last_fullsized":"2023-10-09T11:05:23.314628+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:01:11.094806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"504'1938","reported_seq":1956,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T11:05:28.250896+0000","last_change":"2023-10-09T11:05:05.291441+0000","last_active":"2023-10-09T11:05:28.250896+0000","last_peered":"2023-10-09T11:05:28.250896+0000","last_clean":"2023-10-09T11:05:28.250896+0000","last_became_active":"2023-10-09T11:05:05.291154+0000","last_became_peered":"2023-10-09T11:05:05.291154+0000","last_unstale":"2023-10-09T11:05:28.250896+0000","last_undegraded":"2023-10-09T11:05:28.250896+0000","last_fullsized":"2023-10-09T11:05:28.250896+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":1938,"log_dups_size":0,"ondisk_log_size":1938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:06:57.038484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127008768,"num_objects":1938,"num_object_clones":0,"num_object_copies":5814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1938,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1938,"num_write_kb":124032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":884,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.316562+0000","last_change":"2023-10-09T11:05:23.316129+0000","last_active":"2023-10-09T11:05:23.316562+0000","last_peered":"2023-10-09T11:05:23.316562+0000","last_clean":"2023-10-09T11:05:23.316562+0000","last_became_active":"2023-10-09T11:05:23.315792+0000","last_became_peered":"2023-10-09T11:05:23.315792+0000","last_unstale":"2023-10-09T11:05:23.316562+0000","last_undegraded":"2023-10-09T11:05:23.316562+0000","last_fullsized":"2023-10-09T11:05:23.316562+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:51.770547+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":83,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.309382+0000","last_change":"2023-10-09T11:02:33.208705+0000","last_active":"2023-10-09T11:05:23.309382+0000","last_peered":"2023-10-09T11:05:23.309382+0000","last_clean":"2023-10-09T11:05:23.309382+0000","last_became_active":"2023-10-09T11:02:33.197454+0000","last_became_peered":"2023-10-09T11:02:33.197454+0000","last_unstale":"2023-10-09T11:05:23.309382+0000","last_undegraded":"2023-10-09T11:05:23.309382+0000","last_fullsized":"2023-10-09T11:05:23.309382+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:52.649912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":217,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.320825+0000","last_change":"2023-10-09T11:04:50.333538+0000","last_active":"2023-10-09T11:05:23.320825+0000","last_peered":"2023-10-09T11:05:23.320825+0000","last_clean":"2023-10-09T11:05:23.320825+0000","last_became_active":"2023-10-09T11:04:50.332514+0000","last_became_peered":"2023-10-09T11:04:50.332514+0000","last_unstale":"2023-10-09T11:05:23.320825+0000","last_undegraded":"2023-10-09T11:05:23.320825+0000","last_fullsized":"2023-10-09T11:05:23.320825+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:22.933839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"504'2134","reported_seq":2161,"reported_epoch":504,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:05:31.545720+0000","last_change":"2023-10-09T11:05:21.390308+0000","last_active":"2023-10-09T11:05:31.545720+0000","last_peered":"2023-10-09T11:05:31.545720+0000","last_clean":"2023-10-09T11:05:19.279287+0000","last_became_active":"2023-10-09T11:05:21.387588+0000","last_became_peered":"2023-10-09T11:05:21.387588+0000","last_unstale":"2023-10-09T11:05:31.545720+0000","last_undegraded":"2023-10-09T11:05:21.299397+0000","last_fullsized":"2023-10-09T11:05:21.290839+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":2134,"log_dups_size":0,"ondisk_log_size":2134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:47.400751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139853824,"num_objects":2134,"num_object_clones":0,"num_object_copies":6402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1279,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2134,"num_write_kb":136576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"3,5,7","objects":1279},{"shards":"5,7","objects":855}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":760,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.323671+0000","last_change":"2023-10-09T11:05:23.322251+0000","last_active":"2023-10-09T11:05:23.323671+0000","last_peered":"2023-10-09T11:05:23.323671+0000","last_clean":"2023-10-09T11:05:23.323671+0000","last_became_active":"2023-10-09T11:05:23.321826+0000","last_became_peered":"2023-10-09T11:05:23.321826+0000","last_unstale":"2023-10-09T11:05:23.323671+0000","last_undegraded":"2023-10-09T11:05:23.323671+0000","last_fullsized":"2023-10-09T11:05:23.323671+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.858013+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":125,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.311679+0000","last_change":"2023-10-09T11:05:20.328410+0000","last_active":"2023-10-09T11:05:23.311679+0000","last_peered":"2023-10-09T11:05:23.311679+0000","last_clean":"2023-10-09T11:05:23.311679+0000","last_became_active":"2023-10-09T11:05:20.316182+0000","last_became_peered":"2023-10-09T11:05:20.316182+0000","last_unstale":"2023-10-09T11:05:23.311679+0000","last_undegraded":"2023-10-09T11:05:23.311679+0000","last_fullsized":"2023-10-09T11:05:23.311679+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:04:40.679946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.313628+0000","last_change":"2023-10-09T11:05:20.304182+0000","last_active":"2023-10-09T11:05:23.313628+0000","last_peered":"2023-10-09T11:05:23.313628+0000","last_clean":"2023-10-09T11:05:23.313628+0000","last_became_active":"2023-10-09T11:05:20.303865+0000","last_became_peered":"2023-10-09T11:05:20.303865+0000","last_unstale":"2023-10-09T11:05:23.313628+0000","last_undegraded":"2023-10-09T11:05:23.313628+0000","last_fullsized":"2023-10-09T11:05:23.313628+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:27.835910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"504'2205","reported_seq":2230,"reported_epoch":504,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:05:31.543814+0000","last_change":"2023-10-09T11:05:21.392237+0000","last_active":"2023-10-09T11:05:31.543814+0000","last_peered":"2023-10-09T11:05:31.543814+0000","last_clean":"2023-10-09T11:05:19.282678+0000","last_became_active":"2023-10-09T11:05:21.387435+0000","last_became_peered":"2023-10-09T11:05:21.387435+0000","last_unstale":"2023-10-09T11:05:31.543814+0000","last_undegraded":"2023-10-09T11:05:21.302776+0000","last_fullsized":"2023-10-09T11:05:21.290428+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":2205,"log_dups_size":0,"ondisk_log_size":2205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:43:03.987625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144506880,"num_objects":2205,"num_object_clones":0,"num_object_copies":6615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1344,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2205,"num_write_kb":141120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,2],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"1,2,7","objects":1344},{"shards":"2,7","objects":861}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":969,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.308921+0000","last_change":"2023-10-09T11:04:57.954381+0000","last_active":"2023-10-09T11:05:23.308921+0000","last_peered":"2023-10-09T11:05:23.308921+0000","last_clean":"2023-10-09T11:05:23.308921+0000","last_became_active":"2023-10-09T11:04:57.954012+0000","last_became_peered":"2023-10-09T11:04:57.954012+0000","last_unstale":"2023-10-09T11:05:23.308921+0000","last_undegraded":"2023-10-09T11:05:23.308921+0000","last_fullsized":"2023-10-09T11:05:23.308921+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:27:59.627037+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":362,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.318109+0000","last_change":"2023-10-09T11:05:20.309385+0000","last_active":"2023-10-09T11:05:23.318109+0000","last_peered":"2023-10-09T11:05:23.318109+0000","last_clean":"2023-10-09T11:05:23.318109+0000","last_became_active":"2023-10-09T11:05:20.309034+0000","last_became_peered":"2023-10-09T11:05:20.309034+0000","last_unstale":"2023-10-09T11:05:23.318109+0000","last_undegraded":"2023-10-09T11:05:23.318109+0000","last_fullsized":"2023-10-09T11:05:23.318109+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:31.857485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":426,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.312664+0000","last_change":"2023-10-09T11:00:15.865848+0000","last_active":"2023-10-09T11:05:23.312664+0000","last_peered":"2023-10-09T11:05:23.312664+0000","last_clean":"2023-10-09T11:05:23.312664+0000","last_became_active":"2023-10-09T11:00:15.865504+0000","last_became_peered":"2023-10-09T11:00:15.865504+0000","last_unstale":"2023-10-09T11:05:23.312664+0000","last_undegraded":"2023-10-09T11:05:23.312664+0000","last_fullsized":"2023-10-09T11:05:23.312664+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:32.352564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"504'2268","reported_seq":2295,"reported_epoch":504,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:05:31.516760+0000","last_change":"2023-10-09T11:05:21.415762+0000","last_active":"2023-10-09T11:05:31.516760+0000","last_peered":"2023-10-09T11:05:31.516760+0000","last_clean":"2023-10-09T11:05:19.265787+0000","last_became_active":"2023-10-09T11:05:21.411850+0000","last_became_peered":"2023-10-09T11:05:21.411850+0000","last_unstale":"2023-10-09T11:05:31.516760+0000","last_undegraded":"2023-10-09T11:05:21.301366+0000","last_fullsized":"2023-10-09T11:05:21.290364+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":2268,"log_dups_size":0,"ondisk_log_size":2268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:02:32.548317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148635648,"num_objects":2268,"num_object_clones":0,"num_object_copies":6804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1358,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2268,"num_write_kb":145152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,1],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":1358},{"shards":"1,7","objects":910}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":812,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.308118+0000","last_change":"2023-10-09T11:04:53.274645+0000","last_active":"2023-10-09T11:05:23.308118+0000","last_peered":"2023-10-09T11:05:23.308118+0000","last_clean":"2023-10-09T11:05:23.308118+0000","last_became_active":"2023-10-09T11:04:53.274240+0000","last_became_peered":"2023-10-09T11:04:53.274240+0000","last_unstale":"2023-10-09T11:05:23.308118+0000","last_undegraded":"2023-10-09T11:05:23.308118+0000","last_fullsized":"2023-10-09T11:05:23.308118+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:52.600638+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":33,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.310178+0000","last_change":"2023-10-09T11:04:50.529513+0000","last_active":"2023-10-09T11:05:23.310178+0000","last_peered":"2023-10-09T11:05:23.310178+0000","last_clean":"2023-10-09T11:05:23.310178+0000","last_became_active":"2023-10-09T11:04:50.339671+0000","last_became_peered":"2023-10-09T11:04:50.339671+0000","last_unstale":"2023-10-09T11:05:23.310178+0000","last_undegraded":"2023-10-09T11:05:23.310178+0000","last_fullsized":"2023-10-09T11:05:23.310178+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:24.473000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":38,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.310418+0000","last_change":"2023-10-09T11:05:20.313855+0000","last_active":"2023-10-09T11:05:23.310418+0000","last_peered":"2023-10-09T11:05:23.310418+0000","last_clean":"2023-10-09T11:05:23.310418+0000","last_became_active":"2023-10-09T11:05:20.313535+0000","last_became_peered":"2023-10-09T11:05:20.313535+0000","last_unstale":"2023-10-09T11:05:23.310418+0000","last_undegraded":"2023-10-09T11:05:23.310418+0000","last_fullsized":"2023-10-09T11:05:23.310418+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:25.266434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"504'1893","reported_seq":1911,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T11:05:28.100266+0000","last_change":"2023-10-09T11:05:05.291130+0000","last_active":"2023-10-09T11:05:28.100266+0000","last_peered":"2023-10-09T11:05:28.100266+0000","last_clean":"2023-10-09T11:05:28.100266+0000","last_became_active":"2023-10-09T11:05:05.290758+0000","last_became_peered":"2023-10-09T11:05:05.290758+0000","last_unstale":"2023-10-09T11:05:28.100266+0000","last_undegraded":"2023-10-09T11:05:28.100266+0000","last_fullsized":"2023-10-09T11:05:28.100266+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":1893,"log_dups_size":0,"ondisk_log_size":1893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:32:37.806212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124059648,"num_objects":1893,"num_object_clones":0,"num_object_copies":5679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1893,"num_write_kb":121152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":762,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.314865+0000","last_change":"2023-10-09T11:04:53.272323+0000","last_active":"2023-10-09T11:05:23.314865+0000","last_peered":"2023-10-09T11:05:23.314865+0000","last_clean":"2023-10-09T11:05:23.314865+0000","last_became_active":"2023-10-09T11:04:53.272010+0000","last_became_peered":"2023-10-09T11:04:53.272010+0000","last_unstale":"2023-10-09T11:05:23.314865+0000","last_undegraded":"2023-10-09T11:05:23.314865+0000","last_fullsized":"2023-10-09T11:05:23.314865+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:23.913605+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":526,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.321353+0000","last_change":"2023-10-09T11:05:20.307499+0000","last_active":"2023-10-09T11:05:23.321353+0000","last_peered":"2023-10-09T11:05:23.321353+0000","last_clean":"2023-10-09T11:05:23.321353+0000","last_became_active":"2023-10-09T11:05:20.307198+0000","last_became_peered":"2023-10-09T11:05:20.307198+0000","last_unstale":"2023-10-09T11:05:23.321353+0000","last_undegraded":"2023-10-09T11:05:23.321353+0000","last_fullsized":"2023-10-09T11:05:23.321353+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:53.726844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":261,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.313824+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:05:23.313824+0000","last_peered":"2023-10-09T11:05:23.313824+0000","last_clean":"2023-10-09T11:05:23.313824+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:05:23.313824+0000","last_undegraded":"2023-10-09T11:05:23.313824+0000","last_fullsized":"2023-10-09T11:05:23.313824+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"504'1981","reported_seq":2007,"reported_epoch":504,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:05:28.109690+0000","last_change":"2023-10-09T11:05:21.415743+0000","last_active":"2023-10-09T11:05:28.109690+0000","last_peered":"2023-10-09T11:05:28.109690+0000","last_clean":"2023-10-09T11:05:19.266392+0000","last_became_active":"2023-10-09T11:05:21.412478+0000","last_became_peered":"2023-10-09T11:05:21.412478+0000","last_unstale":"2023-10-09T11:05:28.109690+0000","last_undegraded":"2023-10-09T11:05:21.305534+0000","last_fullsized":"2023-10-09T11:05:21.286304+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":1981,"log_dups_size":0,"ondisk_log_size":1981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:23:12.234627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129826816,"num_objects":1981,"num_object_clones":0,"num_object_copies":5943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1330,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1981,"num_write_kb":126784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":1330},{"shards":"0,3","objects":651}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":854,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.321472+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:05:23.321472+0000","last_peered":"2023-10-09T11:05:23.321472+0000","last_clean":"2023-10-09T11:05:23.321472+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:05:23.321472+0000","last_undegraded":"2023-10-09T11:05:23.321472+0000","last_fullsized":"2023-10-09T11:05:23.321472+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":525,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.308360+0000","last_change":"2023-10-09T11:04:57.953307+0000","last_active":"2023-10-09T11:05:23.308360+0000","last_peered":"2023-10-09T11:05:23.308360+0000","last_clean":"2023-10-09T11:05:23.308360+0000","last_became_active":"2023-10-09T11:04:57.953095+0000","last_became_peered":"2023-10-09T11:04:57.953095+0000","last_unstale":"2023-10-09T11:05:23.308360+0000","last_undegraded":"2023-10-09T11:05:23.308360+0000","last_fullsized":"2023-10-09T11:05:23.308360+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:05.562643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":211,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.322526+0000","last_change":"2023-10-09T11:05:20.310255+0000","last_active":"2023-10-09T11:05:23.322526+0000","last_peered":"2023-10-09T11:05:23.322526+0000","last_clean":"2023-10-09T11:05:23.322526+0000","last_became_active":"2023-10-09T11:05:20.309109+0000","last_became_peered":"2023-10-09T11:05:20.309109+0000","last_unstale":"2023-10-09T11:05:23.322526+0000","last_undegraded":"2023-10-09T11:05:23.322526+0000","last_fullsized":"2023-10-09T11:05:23.322526+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:07.429039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.0","version":"504'1955","reported_seq":1972,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T11:05:28.108980+0000","last_change":"2023-10-09T11:05:05.298362+0000","last_active":"2023-10-09T11:05:28.108980+0000","last_peered":"2023-10-09T11:05:28.108980+0000","last_clean":"2023-10-09T11:05:28.108980+0000","last_became_active":"2023-10-09T11:05:05.298040+0000","last_became_peered":"2023-10-09T11:05:05.298040+0000","last_unstale":"2023-10-09T11:05:28.108980+0000","last_undegraded":"2023-10-09T11:05:28.108980+0000","last_fullsized":"2023-10-09T11:05:28.108980+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":1955,"log_dups_size":0,"ondisk_log_size":1955,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:53:45.054620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128122880,"num_objects":1955,"num_object_clones":0,"num_object_copies":5865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1955,"num_write_kb":125120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":746,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.329664+0000","last_change":"2023-10-09T11:05:23.329329+0000","last_active":"2023-10-09T11:05:23.329664+0000","last_peered":"2023-10-09T11:05:23.329664+0000","last_clean":"2023-10-09T11:05:23.329664+0000","last_became_active":"2023-10-09T11:05:23.328976+0000","last_became_peered":"2023-10-09T11:05:23.328976+0000","last_unstale":"2023-10-09T11:05:23.329664+0000","last_undegraded":"2023-10-09T11:05:23.329664+0000","last_fullsized":"2023-10-09T11:05:23.329664+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:41.848184+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":29,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.313131+0000","last_change":"2023-10-09T11:04:50.335299+0000","last_active":"2023-10-09T11:05:23.313131+0000","last_peered":"2023-10-09T11:05:23.313131+0000","last_clean":"2023-10-09T11:05:23.313131+0000","last_became_active":"2023-10-09T11:04:50.320432+0000","last_became_peered":"2023-10-09T11:04:50.320432+0000","last_unstale":"2023-10-09T11:05:23.313131+0000","last_undegraded":"2023-10-09T11:05:23.313131+0000","last_fullsized":"2023-10-09T11:05:23.313131+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:11.129084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":208,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.321289+0000","last_change":"2023-10-09T11:04:50.332978+0000","last_active":"2023-10-09T11:05:23.321289+0000","last_peered":"2023-10-09T11:05:23.321289+0000","last_clean":"2023-10-09T11:05:23.321289+0000","last_became_active":"2023-10-09T11:04:50.332735+0000","last_became_peered":"2023-10-09T11:04:50.332735+0000","last_unstale":"2023-10-09T11:05:23.321289+0000","last_undegraded":"2023-10-09T11:05:23.321289+0000","last_fullsized":"2023-10-09T11:05:23.321289+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:39:44.659751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"504'2288","reported_seq":4437,"reported_epoch":504,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T11:05:31.204708+0000","last_change":"2023-10-09T11:05:21.423884+0000","last_active":"2023-10-09T11:05:31.204708+0000","last_peered":"2023-10-09T11:05:31.204708+0000","last_clean":"2023-10-09T11:05:19.301589+0000","last_became_active":"2023-10-09T11:05:21.386360+0000","last_became_peered":"2023-10-09T11:05:21.386360+0000","last_unstale":"2023-10-09T11:05:31.204708+0000","last_undegraded":"2023-10-09T11:05:31.204708+0000","last_fullsized":"2023-10-09T11:05:21.286760+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":2288,"log_dups_size":0,"ondisk_log_size":2288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:00.628023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149946368,"num_objects":2288,"num_object_clones":0,"num_object_copies":6864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":347,"num_objects_unfound":0,"num_objects_dirty":2288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2288,"num_write_kb":146432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1061,"num_bytes_recovered":69533696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4],"avail_no_missing":["2","1","4"],"object_location_counts":[{"shards":"1,2,4","objects":347},{"shards":"2,4","objects":1941}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":995,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.314375+0000","last_change":"2023-10-09T11:05:23.314107+0000","last_active":"2023-10-09T11:05:23.314375+0000","last_peered":"2023-10-09T11:05:23.314375+0000","last_clean":"2023-10-09T11:05:23.314375+0000","last_became_active":"2023-10-09T11:05:23.313728+0000","last_became_peered":"2023-10-09T11:05:23.313728+0000","last_unstale":"2023-10-09T11:05:23.314375+0000","last_undegraded":"2023-10-09T11:05:23.314375+0000","last_fullsized":"2023-10-09T11:05:23.314375+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:04.899889+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":214,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.321948+0000","last_change":"2023-10-09T11:02:33.199808+0000","last_active":"2023-10-09T11:05:23.321948+0000","last_peered":"2023-10-09T11:05:23.321948+0000","last_clean":"2023-10-09T11:05:23.321948+0000","last_became_active":"2023-10-09T11:02:33.199332+0000","last_became_peered":"2023-10-09T11:02:33.199332+0000","last_unstale":"2023-10-09T11:05:23.321948+0000","last_undegraded":"2023-10-09T11:05:23.321948+0000","last_fullsized":"2023-10-09T11:05:23.321948+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:23:19.999621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":379,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.316716+0000","last_change":"2023-10-09T11:04:50.341353+0000","last_active":"2023-10-09T11:05:23.316716+0000","last_peered":"2023-10-09T11:05:23.316716+0000","last_clean":"2023-10-09T11:05:23.316716+0000","last_became_active":"2023-10-09T11:04:50.340801+0000","last_became_peered":"2023-10-09T11:04:50.340801+0000","last_unstale":"2023-10-09T11:05:23.316716+0000","last_undegraded":"2023-10-09T11:05:23.316716+0000","last_fullsized":"2023-10-09T11:05:23.316716+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:00.699069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":184,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.321473+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:05:23.321473+0000","last_peered":"2023-10-09T11:05:23.321473+0000","last_clean":"2023-10-09T11:05:23.321473+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:05:23.321473+0000","last_undegraded":"2023-10-09T11:05:23.321473+0000","last_fullsized":"2023-10-09T11:05:23.321473+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"504'1978","reported_seq":1996,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T11:05:28.111490+0000","last_change":"2023-10-09T11:05:05.295729+0000","last_active":"2023-10-09T11:05:28.111490+0000","last_peered":"2023-10-09T11:05:28.111490+0000","last_clean":"2023-10-09T11:05:28.111490+0000","last_became_active":"2023-10-09T11:05:05.295425+0000","last_became_peered":"2023-10-09T11:05:05.295425+0000","last_unstale":"2023-10-09T11:05:28.111490+0000","last_undegraded":"2023-10-09T11:05:28.111490+0000","last_fullsized":"2023-10-09T11:05:28.111490+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":1978,"log_dups_size":0,"ondisk_log_size":1978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:43:37.313230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129630208,"num_objects":1978,"num_object_clones":0,"num_object_copies":5934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1978,"num_write_kb":126592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":704,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.314543+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:05:23.314543+0000","last_peered":"2023-10-09T11:05:23.314543+0000","last_clean":"2023-10-09T11:05:23.314543+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:05:23.314543+0000","last_undegraded":"2023-10-09T11:05:23.314543+0000","last_fullsized":"2023-10-09T11:05:23.314543+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":386,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.316902+0000","last_change":"2023-10-09T11:05:20.316093+0000","last_active":"2023-10-09T11:05:23.316902+0000","last_peered":"2023-10-09T11:05:23.316902+0000","last_clean":"2023-10-09T11:05:23.316902+0000","last_became_active":"2023-10-09T11:05:20.315471+0000","last_became_peered":"2023-10-09T11:05:20.315471+0000","last_unstale":"2023-10-09T11:05:23.316902+0000","last_undegraded":"2023-10-09T11:05:23.316902+0000","last_fullsized":"2023-10-09T11:05:23.316902+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:47.548763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":430,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.312421+0000","last_change":"2023-10-09T11:05:20.309011+0000","last_active":"2023-10-09T11:05:23.312421+0000","last_peered":"2023-10-09T11:05:23.312421+0000","last_clean":"2023-10-09T11:05:23.312421+0000","last_became_active":"2023-10-09T11:05:20.308701+0000","last_became_peered":"2023-10-09T11:05:20.308701+0000","last_unstale":"2023-10-09T11:05:23.312421+0000","last_undegraded":"2023-10-09T11:05:23.312421+0000","last_fullsized":"2023-10-09T11:05:23.312421+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:05:10.042080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":217,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.321754+0000","last_change":"2023-10-09T11:04:50.311579+0000","last_active":"2023-10-09T11:05:23.321754+0000","last_peered":"2023-10-09T11:05:23.321754+0000","last_clean":"2023-10-09T11:05:23.321754+0000","last_became_active":"2023-10-09T11:04:50.311270+0000","last_became_peered":"2023-10-09T11:04:50.311270+0000","last_unstale":"2023-10-09T11:05:23.321754+0000","last_undegraded":"2023-10-09T11:05:23.321754+0000","last_fullsized":"2023-10-09T11:05:23.321754+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:49.850302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"504'1905","reported_seq":1930,"reported_epoch":504,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:05:28.083810+0000","last_change":"2023-10-09T11:05:21.389502+0000","last_active":"2023-10-09T11:05:28.083810+0000","last_peered":"2023-10-09T11:05:28.083810+0000","last_clean":"2023-10-09T11:05:19.275735+0000","last_became_active":"2023-10-09T11:05:21.387222+0000","last_became_peered":"2023-10-09T11:05:21.387222+0000","last_unstale":"2023-10-09T11:05:28.083810+0000","last_undegraded":"2023-10-09T11:05:21.307692+0000","last_fullsized":"2023-10-09T11:05:21.287344+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":1905,"log_dups_size":0,"ondisk_log_size":1905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:13.055600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124846080,"num_objects":1905,"num_object_clones":0,"num_object_copies":5715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1299,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1905,"num_write_kb":121920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1299},{"shards":"3,5","objects":606}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":939,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.308369+0000","last_change":"2023-10-09T11:02:36.216448+0000","last_active":"2023-10-09T11:05:23.308369+0000","last_peered":"2023-10-09T11:05:23.308369+0000","last_clean":"2023-10-09T11:05:23.308369+0000","last_became_active":"2023-10-09T11:02:36.216040+0000","last_became_peered":"2023-10-09T11:02:36.216040+0000","last_unstale":"2023-10-09T11:05:23.308369+0000","last_undegraded":"2023-10-09T11:05:23.308369+0000","last_fullsized":"2023-10-09T11:05:23.308369+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:13:40.104907+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":281,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.308637+0000","last_change":"2023-10-09T11:04:50.381744+0000","last_active":"2023-10-09T11:05:23.308637+0000","last_peered":"2023-10-09T11:05:23.308637+0000","last_clean":"2023-10-09T11:05:23.308637+0000","last_became_active":"2023-10-09T11:04:50.381253+0000","last_became_peered":"2023-10-09T11:04:50.381253+0000","last_unstale":"2023-10-09T11:05:23.308637+0000","last_undegraded":"2023-10-09T11:05:23.308637+0000","last_fullsized":"2023-10-09T11:05:23.308637+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:24.030274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":202,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.320642+0000","last_change":"2023-10-09T11:05:20.298542+0000","last_active":"2023-10-09T11:05:23.320642+0000","last_peered":"2023-10-09T11:05:23.320642+0000","last_clean":"2023-10-09T11:05:23.320642+0000","last_became_active":"2023-10-09T11:05:20.298119+0000","last_became_peered":"2023-10-09T11:05:20.298119+0000","last_unstale":"2023-10-09T11:05:23.320642+0000","last_undegraded":"2023-10-09T11:05:23.320642+0000","last_fullsized":"2023-10-09T11:05:23.320642+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:00.827214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":182,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.321484+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:05:23.321484+0000","last_peered":"2023-10-09T11:05:23.321484+0000","last_clean":"2023-10-09T11:05:23.321484+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:05:23.321484+0000","last_undegraded":"2023-10-09T11:05:23.321484+0000","last_fullsized":"2023-10-09T11:05:23.321484+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"504'1996","reported_seq":2021,"reported_epoch":504,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:05:28.090998+0000","last_change":"2023-10-09T11:05:21.393803+0000","last_active":"2023-10-09T11:05:28.090998+0000","last_peered":"2023-10-09T11:05:28.090998+0000","last_clean":"2023-10-09T11:05:19.261028+0000","last_became_active":"2023-10-09T11:05:21.391918+0000","last_became_peered":"2023-10-09T11:05:21.391918+0000","last_unstale":"2023-10-09T11:05:28.090998+0000","last_undegraded":"2023-10-09T11:05:21.304992+0000","last_fullsized":"2023-10-09T11:05:21.289427+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":1996,"log_dups_size":0,"ondisk_log_size":1996,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:17:12.771293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130809856,"num_objects":1996,"num_object_clones":0,"num_object_copies":5988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1996,"num_write_kb":127744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,5","objects":1382},{"shards":"3,5","objects":614}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":359,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.316587+0000","last_change":"2023-10-09T11:02:33.198446+0000","last_active":"2023-10-09T11:05:23.316587+0000","last_peered":"2023-10-09T11:05:23.316587+0000","last_clean":"2023-10-09T11:05:23.316587+0000","last_became_active":"2023-10-09T11:02:33.197858+0000","last_became_peered":"2023-10-09T11:02:33.197858+0000","last_unstale":"2023-10-09T11:05:23.316587+0000","last_undegraded":"2023-10-09T11:05:23.316587+0000","last_fullsized":"2023-10-09T11:05:23.316587+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:17.362664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":924,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.321594+0000","last_change":"2023-10-09T11:05:23.320842+0000","last_active":"2023-10-09T11:05:23.321594+0000","last_peered":"2023-10-09T11:05:23.321594+0000","last_clean":"2023-10-09T11:05:23.321594+0000","last_became_active":"2023-10-09T11:05:23.320461+0000","last_became_peered":"2023-10-09T11:05:23.320461+0000","last_unstale":"2023-10-09T11:05:23.321594+0000","last_undegraded":"2023-10-09T11:05:23.321594+0000","last_fullsized":"2023-10-09T11:05:23.321594+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:38.505611+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":38,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.311183+0000","last_change":"2023-10-09T11:05:20.310666+0000","last_active":"2023-10-09T11:05:23.311183+0000","last_peered":"2023-10-09T11:05:23.311183+0000","last_clean":"2023-10-09T11:05:23.311183+0000","last_became_active":"2023-10-09T11:05:20.310413+0000","last_became_peered":"2023-10-09T11:05:20.310413+0000","last_unstale":"2023-10-09T11:05:23.311183+0000","last_undegraded":"2023-10-09T11:05:23.311183+0000","last_fullsized":"2023-10-09T11:05:23.311183+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:34.169905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":464,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.308697+0000","last_change":"2023-10-09T11:05:20.316523+0000","last_active":"2023-10-09T11:05:23.308697+0000","last_peered":"2023-10-09T11:05:23.308697+0000","last_clean":"2023-10-09T11:05:23.308697+0000","last_became_active":"2023-10-09T11:05:20.314493+0000","last_became_peered":"2023-10-09T11:05:20.314493+0000","last_unstale":"2023-10-09T11:05:23.308697+0000","last_undegraded":"2023-10-09T11:05:23.308697+0000","last_fullsized":"2023-10-09T11:05:23.308697+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:41.618701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"504'1928","reported_seq":1952,"reported_epoch":504,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:05:28.051510+0000","last_change":"2023-10-09T11:05:21.438976+0000","last_active":"2023-10-09T11:05:28.051510+0000","last_peered":"2023-10-09T11:05:28.051510+0000","last_clean":"2023-10-09T11:05:19.275351+0000","last_became_active":"2023-10-09T11:05:21.437004+0000","last_became_peered":"2023-10-09T11:05:21.437004+0000","last_unstale":"2023-10-09T11:05:28.051510+0000","last_undegraded":"2023-10-09T11:05:21.311036+0000","last_fullsized":"2023-10-09T11:05:21.287793+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":1928,"log_dups_size":0,"ondisk_log_size":1928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:51.972919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126353408,"num_objects":1928,"num_object_clones":0,"num_object_copies":5784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1333,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1928,"num_write_kb":123392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,0],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":1333},{"shards":"0,3","objects":595}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":874,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.322706+0000","last_change":"2023-10-09T11:05:23.322074+0000","last_active":"2023-10-09T11:05:23.322706+0000","last_peered":"2023-10-09T11:05:23.322706+0000","last_clean":"2023-10-09T11:05:23.322706+0000","last_became_active":"2023-10-09T11:05:23.321749+0000","last_became_peered":"2023-10-09T11:05:23.321749+0000","last_unstale":"2023-10-09T11:05:23.322706+0000","last_undegraded":"2023-10-09T11:05:23.322706+0000","last_fullsized":"2023-10-09T11:05:23.322706+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:07.010932+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":79,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.308237+0000","last_change":"2023-10-09T11:02:33.198484+0000","last_active":"2023-10-09T11:05:23.308237+0000","last_peered":"2023-10-09T11:05:23.308237+0000","last_clean":"2023-10-09T11:05:23.308237+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:05:23.308237+0000","last_undegraded":"2023-10-09T11:05:23.308237+0000","last_fullsized":"2023-10-09T11:05:23.308237+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":165,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.315002+0000","last_change":"2023-10-09T11:04:50.322813+0000","last_active":"2023-10-09T11:05:23.315002+0000","last_peered":"2023-10-09T11:05:23.315002+0000","last_clean":"2023-10-09T11:05:23.315002+0000","last_became_active":"2023-10-09T11:04:50.322482+0000","last_became_peered":"2023-10-09T11:04:50.322482+0000","last_unstale":"2023-10-09T11:05:23.315002+0000","last_undegraded":"2023-10-09T11:05:23.315002+0000","last_fullsized":"2023-10-09T11:05:23.315002+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:18.873195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":462,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.315942+0000","last_change":"2023-10-09T11:00:15.892896+0000","last_active":"2023-10-09T11:05:23.315942+0000","last_peered":"2023-10-09T11:05:23.315942+0000","last_clean":"2023-10-09T11:05:23.315942+0000","last_became_active":"2023-10-09T11:00:15.892573+0000","last_became_peered":"2023-10-09T11:00:15.892573+0000","last_unstale":"2023-10-09T11:05:23.315942+0000","last_undegraded":"2023-10-09T11:05:23.315942+0000","last_fullsized":"2023-10-09T11:05:23.315942+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:37:13.280089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"504'1921","reported_seq":1939,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T11:05:28.108864+0000","last_change":"2023-10-09T11:05:05.921655+0000","last_active":"2023-10-09T11:05:28.108864+0000","last_peered":"2023-10-09T11:05:28.108864+0000","last_clean":"2023-10-09T11:05:28.108864+0000","last_became_active":"2023-10-09T11:05:05.921204+0000","last_became_peered":"2023-10-09T11:05:05.921204+0000","last_unstale":"2023-10-09T11:05:28.108864+0000","last_undegraded":"2023-10-09T11:05:28.108864+0000","last_fullsized":"2023-10-09T11:05:28.108864+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":1921,"log_dups_size":0,"ondisk_log_size":1921,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:16.714889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125894656,"num_objects":1921,"num_object_clones":0,"num_object_copies":5763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1921,"num_write_kb":122944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":792,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.327843+0000","last_change":"2023-10-09T11:05:23.327007+0000","last_active":"2023-10-09T11:05:23.327843+0000","last_peered":"2023-10-09T11:05:23.327843+0000","last_clean":"2023-10-09T11:05:23.327843+0000","last_became_active":"2023-10-09T11:05:23.326625+0000","last_became_peered":"2023-10-09T11:05:23.326625+0000","last_unstale":"2023-10-09T11:05:23.327843+0000","last_undegraded":"2023-10-09T11:05:23.327843+0000","last_fullsized":"2023-10-09T11:05:23.327843+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:53.785082+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":79,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.308774+0000","last_change":"2023-10-09T11:02:33.199031+0000","last_active":"2023-10-09T11:05:23.308774+0000","last_peered":"2023-10-09T11:05:23.308774+0000","last_clean":"2023-10-09T11:05:23.308774+0000","last_became_active":"2023-10-09T11:02:33.198251+0000","last_became_peered":"2023-10-09T11:02:33.198251+0000","last_unstale":"2023-10-09T11:05:23.308774+0000","last_undegraded":"2023-10-09T11:05:23.308774+0000","last_fullsized":"2023-10-09T11:05:23.308774+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:21.904005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":184,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.322139+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:05:23.322139+0000","last_peered":"2023-10-09T11:05:23.322139+0000","last_clean":"2023-10-09T11:05:23.322139+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:05:23.322139+0000","last_undegraded":"2023-10-09T11:05:23.322139+0000","last_fullsized":"2023-10-09T11:05:23.322139+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":447,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.312533+0000","last_change":"2023-10-09T11:02:33.198297+0000","last_active":"2023-10-09T11:05:23.312533+0000","last_peered":"2023-10-09T11:05:23.312533+0000","last_clean":"2023-10-09T11:05:23.312533+0000","last_became_active":"2023-10-09T11:02:33.197978+0000","last_became_peered":"2023-10-09T11:02:33.197978+0000","last_unstale":"2023-10-09T11:05:23.312533+0000","last_undegraded":"2023-10-09T11:05:23.312533+0000","last_fullsized":"2023-10-09T11:05:23.312533+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:41.689146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"504'2205","reported_seq":2222,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T11:05:31.562342+0000","last_change":"2023-10-09T11:05:05.295611+0000","last_active":"2023-10-09T11:05:31.562342+0000","last_peered":"2023-10-09T11:05:31.562342+0000","last_clean":"2023-10-09T11:05:31.562342+0000","last_became_active":"2023-10-09T11:05:05.295369+0000","last_became_peered":"2023-10-09T11:05:05.295369+0000","last_unstale":"2023-10-09T11:05:31.562342+0000","last_undegraded":"2023-10-09T11:05:31.562342+0000","last_fullsized":"2023-10-09T11:05:31.562342+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":2205,"log_dups_size":0,"ondisk_log_size":2205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:38:49.314348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144506880,"num_objects":2205,"num_object_clones":0,"num_object_copies":6615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2205,"num_write_kb":141120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":702,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.328044+0000","last_change":"2023-10-09T11:05:23.323368+0000","last_active":"2023-10-09T11:05:23.328044+0000","last_peered":"2023-10-09T11:05:23.328044+0000","last_clean":"2023-10-09T11:05:23.328044+0000","last_became_active":"2023-10-09T11:05:23.323032+0000","last_became_peered":"2023-10-09T11:05:23.323032+0000","last_unstale":"2023-10-09T11:05:23.328044+0000","last_undegraded":"2023-10-09T11:05:23.328044+0000","last_fullsized":"2023-10-09T11:05:23.328044+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:09.393584+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":33,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.311136+0000","last_change":"2023-10-09T11:04:50.552366+0000","last_active":"2023-10-09T11:05:23.311136+0000","last_peered":"2023-10-09T11:05:23.311136+0000","last_clean":"2023-10-09T11:05:23.311136+0000","last_became_active":"2023-10-09T11:04:50.339694+0000","last_became_peered":"2023-10-09T11:04:50.339694+0000","last_unstale":"2023-10-09T11:05:23.311136+0000","last_undegraded":"2023-10-09T11:05:23.311136+0000","last_fullsized":"2023-10-09T11:05:23.311136+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:39.577557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":211,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.322564+0000","last_change":"2023-10-09T11:05:20.310417+0000","last_active":"2023-10-09T11:05:23.322564+0000","last_peered":"2023-10-09T11:05:23.322564+0000","last_clean":"2023-10-09T11:05:23.322564+0000","last_became_active":"2023-10-09T11:05:20.309827+0000","last_became_peered":"2023-10-09T11:05:20.309827+0000","last_unstale":"2023-10-09T11:05:23.322564+0000","last_undegraded":"2023-10-09T11:05:23.322564+0000","last_fullsized":"2023-10-09T11:05:23.322564+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:23.934383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":29,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.313035+0000","last_change":"2023-10-09T11:04:50.336886+0000","last_active":"2023-10-09T11:05:23.313035+0000","last_peered":"2023-10-09T11:05:23.313035+0000","last_clean":"2023-10-09T11:05:23.313035+0000","last_became_active":"2023-10-09T11:04:50.336552+0000","last_became_peered":"2023-10-09T11:04:50.336552+0000","last_unstale":"2023-10-09T11:05:23.313035+0000","last_undegraded":"2023-10-09T11:05:23.313035+0000","last_fullsized":"2023-10-09T11:05:23.313035+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:04.030169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"504'1946","reported_seq":1964,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-09T11:05:28.091876+0000","last_change":"2023-10-09T11:05:05.921539+0000","last_active":"2023-10-09T11:05:28.091876+0000","last_peered":"2023-10-09T11:05:28.091876+0000","last_clean":"2023-10-09T11:05:28.091876+0000","last_became_active":"2023-10-09T11:05:05.921243+0000","last_became_peered":"2023-10-09T11:05:05.921243+0000","last_unstale":"2023-10-09T11:05:28.091876+0000","last_undegraded":"2023-10-09T11:05:28.091876+0000","last_fullsized":"2023-10-09T11:05:28.091876+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":1946,"log_dups_size":0,"ondisk_log_size":1946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:48.841133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127533056,"num_objects":1946,"num_object_clones":0,"num_object_copies":5838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1946,"num_write_kb":124544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":632,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.310366+0000","last_change":"2023-10-09T11:05:03.653459+0000","last_active":"2023-10-09T11:05:23.310366+0000","last_peered":"2023-10-09T11:05:23.310366+0000","last_clean":"2023-10-09T11:05:23.310366+0000","last_became_active":"2023-10-09T11:04:53.272218+0000","last_became_peered":"2023-10-09T11:04:53.272218+0000","last_unstale":"2023-10-09T11:05:23.310366+0000","last_undegraded":"2023-10-09T11:05:23.310366+0000","last_fullsized":"2023-10-09T11:05:23.310366+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:03.653371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:03.653371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:16:41.854113+0000","scrub_duration":0.001576771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":109,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.309315+0000","last_change":"2023-10-09T11:05:20.302072+0000","last_active":"2023-10-09T11:05:23.309315+0000","last_peered":"2023-10-09T11:05:23.309315+0000","last_clean":"2023-10-09T11:05:23.309315+0000","last_became_active":"2023-10-09T11:05:20.301735+0000","last_became_peered":"2023-10-09T11:05:20.301735+0000","last_unstale":"2023-10-09T11:05:23.309315+0000","last_undegraded":"2023-10-09T11:05:23.309315+0000","last_fullsized":"2023-10-09T11:05:23.309315+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:05.160338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":446,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.316026+0000","last_change":"2023-10-09T11:00:15.869009+0000","last_active":"2023-10-09T11:05:23.316026+0000","last_peered":"2023-10-09T11:05:23.316026+0000","last_clean":"2023-10-09T11:05:23.316026+0000","last_became_active":"2023-10-09T11:00:15.868637+0000","last_became_peered":"2023-10-09T11:00:15.868637+0000","last_unstale":"2023-10-09T11:05:23.316026+0000","last_undegraded":"2023-10-09T11:05:23.316026+0000","last_fullsized":"2023-10-09T11:05:23.316026+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:37.538741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":309,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.314402+0000","last_change":"2023-10-09T11:05:20.313560+0000","last_active":"2023-10-09T11:05:23.314402+0000","last_peered":"2023-10-09T11:05:23.314402+0000","last_clean":"2023-10-09T11:05:23.314402+0000","last_became_active":"2023-10-09T11:05:20.313214+0000","last_became_peered":"2023-10-09T11:05:20.313214+0000","last_unstale":"2023-10-09T11:05:23.314402+0000","last_undegraded":"2023-10-09T11:05:23.314402+0000","last_fullsized":"2023-10-09T11:05:23.314402+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:32.527205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":910,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.334894+0000","last_change":"2023-10-09T11:05:23.329886+0000","last_active":"2023-10-09T11:05:23.334894+0000","last_peered":"2023-10-09T11:05:23.334894+0000","last_clean":"2023-10-09T11:05:23.334894+0000","last_became_active":"2023-10-09T11:05:23.329561+0000","last_became_peered":"2023-10-09T11:05:23.329561+0000","last_unstale":"2023-10-09T11:05:23.334894+0000","last_undegraded":"2023-10-09T11:05:23.334894+0000","last_fullsized":"2023-10-09T11:05:23.334894+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:46.933799+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":445,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.312603+0000","last_change":"2023-10-09T11:02:33.201950+0000","last_active":"2023-10-09T11:05:23.312603+0000","last_peered":"2023-10-09T11:05:23.312603+0000","last_clean":"2023-10-09T11:05:23.312603+0000","last_became_active":"2023-10-09T11:02:33.201559+0000","last_became_peered":"2023-10-09T11:02:33.201559+0000","last_unstale":"2023-10-09T11:05:23.312603+0000","last_undegraded":"2023-10-09T11:05:23.312603+0000","last_fullsized":"2023-10-09T11:05:23.312603+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:45:45.851351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":285,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.311395+0000","last_change":"2023-10-09T11:04:50.510688+0000","last_active":"2023-10-09T11:05:23.311395+0000","last_peered":"2023-10-09T11:05:23.311395+0000","last_clean":"2023-10-09T11:05:23.311395+0000","last_became_active":"2023-10-09T11:04:50.507404+0000","last_became_peered":"2023-10-09T11:04:50.507404+0000","last_unstale":"2023-10-09T11:05:23.311395+0000","last_undegraded":"2023-10-09T11:05:23.311395+0000","last_fullsized":"2023-10-09T11:05:23.311395+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:13.663394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":867,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.322294+0000","last_change":"2023-10-09T11:05:23.321040+0000","last_active":"2023-10-09T11:05:23.322294+0000","last_peered":"2023-10-09T11:05:23.322294+0000","last_clean":"2023-10-09T11:05:23.322294+0000","last_became_active":"2023-10-09T11:05:23.320674+0000","last_became_peered":"2023-10-09T11:05:23.320674+0000","last_unstale":"2023-10-09T11:05:23.322294+0000","last_undegraded":"2023-10-09T11:05:23.322294+0000","last_fullsized":"2023-10-09T11:05:23.322294+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.527660+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":331,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.318020+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:05:23.318020+0000","last_peered":"2023-10-09T11:05:23.318020+0000","last_clean":"2023-10-09T11:05:23.318020+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:05:23.318020+0000","last_undegraded":"2023-10-09T11:05:23.318020+0000","last_fullsized":"2023-10-09T11:05:23.318020+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":29,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.310166+0000","last_change":"2023-10-09T11:04:50.511984+0000","last_active":"2023-10-09T11:05:23.310166+0000","last_peered":"2023-10-09T11:05:23.310166+0000","last_clean":"2023-10-09T11:05:23.310166+0000","last_became_active":"2023-10-09T11:04:50.510820+0000","last_became_peered":"2023-10-09T11:04:50.510820+0000","last_unstale":"2023-10-09T11:05:23.310166+0000","last_undegraded":"2023-10-09T11:05:23.310166+0000","last_fullsized":"2023-10-09T11:05:23.310166+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:38.662189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":482,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.316720+0000","last_change":"2023-10-09T11:05:20.327032+0000","last_active":"2023-10-09T11:05:23.316720+0000","last_peered":"2023-10-09T11:05:23.316720+0000","last_clean":"2023-10-09T11:05:23.316720+0000","last_became_active":"2023-10-09T11:05:20.320193+0000","last_became_peered":"2023-10-09T11:05:20.320193+0000","last_unstale":"2023-10-09T11:05:23.316720+0000","last_undegraded":"2023-10-09T11:05:23.316720+0000","last_fullsized":"2023-10-09T11:05:23.316720+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:44.862890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":29,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.311320+0000","last_change":"2023-10-09T11:04:50.384219+0000","last_active":"2023-10-09T11:05:23.311320+0000","last_peered":"2023-10-09T11:05:23.311320+0000","last_clean":"2023-10-09T11:05:23.311320+0000","last_became_active":"2023-10-09T11:04:50.384028+0000","last_became_peered":"2023-10-09T11:04:50.384028+0000","last_unstale":"2023-10-09T11:05:23.311320+0000","last_undegraded":"2023-10-09T11:05:23.311320+0000","last_fullsized":"2023-10-09T11:05:23.311320+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:04.721869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":787,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.333283+0000","last_change":"2023-10-09T11:05:23.329250+0000","last_active":"2023-10-09T11:05:23.333283+0000","last_peered":"2023-10-09T11:05:23.333283+0000","last_clean":"2023-10-09T11:05:23.333283+0000","last_became_active":"2023-10-09T11:05:23.328873+0000","last_became_peered":"2023-10-09T11:05:23.328873+0000","last_unstale":"2023-10-09T11:05:23.333283+0000","last_undegraded":"2023-10-09T11:05:23.333283+0000","last_fullsized":"2023-10-09T11:05:23.333283+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:04.790935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.790935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:26.753170+0000","scrub_duration":0.0013825300000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":38,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.312369+0000","last_change":"2023-10-09T11:05:20.299321+0000","last_active":"2023-10-09T11:05:23.312369+0000","last_peered":"2023-10-09T11:05:23.312369+0000","last_clean":"2023-10-09T11:05:23.312369+0000","last_became_active":"2023-10-09T11:05:20.299019+0000","last_became_peered":"2023-10-09T11:05:20.299019+0000","last_unstale":"2023-10-09T11:05:23.312369+0000","last_undegraded":"2023-10-09T11:05:23.312369+0000","last_fullsized":"2023-10-09T11:05:23.312369+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:08.645909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":29,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.321237+0000","last_change":"2023-10-09T11:04:50.366890+0000","last_active":"2023-10-09T11:05:23.321237+0000","last_peered":"2023-10-09T11:05:23.321237+0000","last_clean":"2023-10-09T11:05:23.321237+0000","last_became_active":"2023-10-09T11:04:50.366558+0000","last_became_peered":"2023-10-09T11:04:50.366558+0000","last_unstale":"2023-10-09T11:05:23.321237+0000","last_undegraded":"2023-10-09T11:05:23.321237+0000","last_fullsized":"2023-10-09T11:05:23.321237+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:50.993153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":802,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.321199+0000","last_change":"2023-10-09T11:04:53.274332+0000","last_active":"2023-10-09T11:05:23.321199+0000","last_peered":"2023-10-09T11:05:23.321199+0000","last_clean":"2023-10-09T11:05:23.321199+0000","last_became_active":"2023-10-09T11:04:53.273798+0000","last_became_peered":"2023-10-09T11:04:53.273798+0000","last_unstale":"2023-10-09T11:05:23.321199+0000","last_undegraded":"2023-10-09T11:05:23.321199+0000","last_fullsized":"2023-10-09T11:05:23.321199+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:04.220970+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":147,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.311058+0000","last_change":"2023-10-09T10:59:48.143732+0000","last_active":"2023-10-09T11:05:23.311058+0000","last_peered":"2023-10-09T11:05:23.311058+0000","last_clean":"2023-10-09T11:05:23.311058+0000","last_became_active":"2023-10-09T10:59:48.129001+0000","last_became_peered":"2023-10-09T10:59:48.129001+0000","last_unstale":"2023-10-09T11:05:23.311058+0000","last_undegraded":"2023-10-09T11:05:23.311058+0000","last_fullsized":"2023-10-09T11:05:23.311058+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:22:36.946348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":551,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-09T11:05:23.320731+0000","last_change":"2023-10-09T11:05:20.307347+0000","last_active":"2023-10-09T11:05:23.320731+0000","last_peered":"2023-10-09T11:05:23.320731+0000","last_clean":"2023-10-09T11:05:23.320731+0000","last_became_active":"2023-10-09T11:05:20.307126+0000","last_became_peered":"2023-10-09T11:05:20.307126+0000","last_unstale":"2023-10-09T11:05:23.320731+0000","last_undegraded":"2023-10-09T11:05:23.320731+0000","last_fullsized":"2023-10-09T11:05:23.320731+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:34.408515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":2149253120,"num_objects":32795,"num_object_clones":0,"num_object_copies":98385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9325,"num_objects_misplaced":347,"num_objects_unfound":0,"num_objects_dirty":32795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":32795,"num_write_kb":2098880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1061,"num_bytes_recovered":69533696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6712852480,"data_stored":6712852480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32795,"ondisk_log_size":32795,"up":48,"acting":40,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":26956,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12694,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1836096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332467,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1737412,"kb_used_data":829820,"kb_used_omap":0,"kb_used_meta":907584,"kb_avail":92007740,"statfs":{"total":95995035648,"available":94215925760,"internally_reserved":0,"allocated":849735680,"data_stored":848111768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929366016},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:05:14 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.5960000000000001,"15min":1.8380000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.3359999999999999,"5min":9.8040000000000003,"15min":9.8040000000000003},"last":0.499},{"interface":"front","average":{"1min":1.288,"5min":1.5649999999999999,"15min":1.831},"min":{"1min":0.24299999999999999,"5min":0.216,"15min":0.216},"max":{"1min":8.0180000000000007,"5min":9.6999999999999993,"15min":16.731000000000002},"last":0.34100000000000003}]},{"osd":1,"last update":"Mon Oct 9 11:05:20 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":1.3819999999999999,"15min":1.5700000000000001},"min":{"1min":0.26000000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.0209999999999999,"5min":6.9660000000000002,"15min":6.9660000000000002},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.623,"5min":1.5920000000000001,"15min":1.8979999999999999},"min":{"1min":0.32700000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.774,"5min":12.193,"15min":12.193},"last":0.307}]},{"osd":2,"last update":"Mon Oct 9 11:04:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6320000000000001,"5min":2.1629999999999998,"15min":1.845},"min":{"1min":0.309,"5min":0.27600000000000002,"15min":0.245},"max":{"1min":12.226000000000001,"5min":16.605,"15min":16.605},"last":1.869},{"interface":"front","average":{"1min":2.2269999999999999,"5min":1.714,"15min":1.792},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26600000000000001},"max":{"1min":6.8780000000000001,"5min":9.7469999999999999,"15min":13.037000000000001},"last":1.097}]},{"osd":3,"last update":"Mon Oct 9 11:05:14 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":1.399,"15min":1.399},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.8199999999999998,"5min":3.8199999999999998,"15min":3.8199999999999998},"last":0.371},{"interface":"front","average":{"1min":1.792,"5min":1.792,"15min":1.792},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":8.2330000000000005,"5min":8.2330000000000005,"15min":8.2330000000000005},"last":2.012}]},{"osd":4,"last update":"Mon Oct 9 11:05:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":1.218,"15min":1.3460000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":3.46,"5min":7.6589999999999998,"15min":16.695},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.071,"5min":1.7230000000000001,"15min":1.4390000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.734,"5min":16.481000000000002,"15min":16.481000000000002},"last":0.42399999999999999}]},{"osd":5,"last update":"Mon Oct 9 11:05:00 2023","interfaces":[{"interface":"back","average":{"1min":1.4119999999999999,"5min":1.554,"15min":1.5109999999999999},"min":{"1min":0.31900000000000001,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":6.9379999999999997,"5min":15.414999999999999,"15min":15.414999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.024,"5min":1.4490000000000001,"15min":1.393},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.504,"5min":15.369,"15min":15.369},"last":0.629}]},{"osd":6,"last update":"Mon Oct 9 11:05:00 2023","interfaces":[{"interface":"back","average":{"1min":1.4550000000000001,"5min":1.516,"15min":1.575},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":5.1120000000000001,"5min":14.159000000000001,"15min":14.913},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.952,"15min":1.615},"min":{"1min":0.373,"5min":0.32600000000000001,"15min":0.29399999999999998},"max":{"1min":4.3659999999999997,"5min":16.538,"15min":16.538},"last":0.77900000000000003}]}]},{"osd":6,"up_from":44,"seq":188978561398,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1399796,"kb_used_data":503724,"kb_used_omap":0,"kb_used_meta":896064,"kb_avail":92345356,"statfs":{"total":95995035648,"available":94561644544,"internally_reserved":0,"allocated":515813376,"data_stored":514205158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917569536},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:05:17 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":2.1699999999999999,"15min":1.9610000000000001},"min":{"1min":0.223,"5min":0.217,"15min":0.216},"max":{"1min":8.4250000000000007,"5min":46.603999999999999,"15min":54.743000000000002},"last":1.4610000000000001},{"interface":"front","average":{"1min":1.0569999999999999,"5min":2.1240000000000001,"15min":2.0539999999999998},"min":{"1min":0.28000000000000003,"5min":0.214,"15min":0.214},"max":{"1min":8.4789999999999992,"5min":46.679000000000002,"15min":53.701000000000001},"last":3.6360000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:05:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7310000000000001,"5min":1.7250000000000001,"15min":1.077},"min":{"1min":0.28299999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":16.225999999999999,"5min":41.241999999999997,"15min":41.241999999999997},"last":4.3179999999999996},{"interface":"front","average":{"1min":1.794,"5min":2.2450000000000001,"15min":1.657},"min":{"1min":0.31900000000000001,"5min":0.248,"15min":0.248},"max":{"1min":11.715,"5min":41.476999999999997,"15min":41.476999999999997},"last":0.79300000000000004}]},{"osd":2,"last update":"Mon Oct 9 11:04:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6299999999999999,"5min":1.9670000000000001,"15min":1.925},"min":{"1min":0.307,"5min":0.28699999999999998,"15min":0.28399999999999997},"max":{"1min":7.7489999999999997,"5min":10.481999999999999,"15min":12.003},"last":1.302},{"interface":"front","average":{"1min":2.331,"5min":2.2970000000000002,"15min":2.1219999999999999},"min":{"1min":0.27300000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":12.693,"5min":12.693,"15min":15.432},"last":0.30099999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:05:17 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.45,"15min":1.45},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":8.0099999999999998,"5min":8.0099999999999998,"15min":8.0099999999999998},"last":0.84199999999999997},{"interface":"front","average":{"1min":2.4390000000000001,"5min":2.4390000000000001,"15min":2.4390000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":23.815999999999999,"5min":23.815999999999999,"15min":23.815999999999999},"last":1.3680000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:04:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4379999999999999,"5min":1.474,"15min":1.325},"min":{"1min":0.255,"5min":0.255,"15min":0.24199999999999999},"max":{"1min":6.6609999999999996,"5min":17.216999999999999,"15min":17.216999999999999},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.95099999999999996,"15min":1.091},"min":{"1min":0.30599999999999999,"5min":0.252,"15min":0.24399999999999999},"max":{"1min":2.3610000000000002,"5min":10.025,"15min":12.061999999999999},"last":0.377}]},{"osd":5,"last update":"Mon Oct 9 11:04:48 2023","interfaces":[{"interface":"back","average":{"1min":1.415,"5min":1.409,"15min":1.5349999999999999},"min":{"1min":0.35699999999999998,"5min":0.29499999999999998,"15min":0.25600000000000001},"max":{"1min":3.7389999999999999,"5min":10.209,"15min":19.786999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.03,"5min":1.0980000000000001,"15min":1.23},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":4.085,"5min":9.9469999999999992,"15min":12.103999999999999},"last":0.42399999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":1.6379999999999999,"15min":1.4810000000000001},"min":{"1min":0.39000000000000001,"5min":0.32400000000000001,"15min":0.28100000000000003},"max":{"1min":2.8439999999999999,"5min":11.550000000000001,"15min":12.882},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.89700000000000002,"15min":1.034},"min":{"1min":0.33200000000000002,"5min":0.31,"15min":0.27500000000000002},"max":{"1min":1.472,"5min":11.557,"15min":11.557},"last":0.40500000000000003}]}]},{"osd":1,"up_from":388,"seq":1666447310912,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1649528,"kb_used_data":1308464,"kb_used_omap":13,"kb_used_meta":341042,"kb_avail":92095624,"statfs":{"total":95995035648,"available":94305918976,"internally_reserved":0,"allocated":1339867136,"data_stored":1337219864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":349227719},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.633,"5min":2.246,"15min":2.254},"min":{"1min":0.23499999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":6.056,"5min":13.334,"15min":13.334},"last":6.056},{"interface":"front","average":{"1min":1.9490000000000001,"5min":2.242,"15min":1.621},"min":{"1min":0.28699999999999998,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":9.0220000000000002,"5min":11.706,"15min":11.706},"last":4.2240000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":2.5800000000000001,"15min":3.0059999999999998},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":12.436999999999999,"5min":47.731000000000002,"15min":47.731000000000002},"last":5.8230000000000004},{"interface":"front","average":{"1min":1.994,"5min":2.6240000000000001,"15min":2.363},"min":{"1min":0.308,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.6929999999999996,"5min":47.786999999999999,"15min":47.786999999999999},"last":4.258}]},{"osd":3,"last update":"Mon Oct 9 11:05:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":1.4890000000000001,"15min":1.4890000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.5839999999999996,"5min":5.5839999999999996,"15min":5.5839999999999996},"last":1.147},{"interface":"front","average":{"1min":1.4830000000000001,"5min":1.4830000000000001,"15min":1.4830000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.1449999999999996,"5min":7.1449999999999996,"15min":7.1449999999999996},"last":9.0359999999999996}]},{"osd":4,"last update":"Mon Oct 9 11:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.7549999999999999,"15min":2.4929999999999999},"min":{"1min":0.314,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.0119999999999996,"5min":10.481,"15min":10.481},"last":6.0119999999999996},{"interface":"front","average":{"1min":1.5029999999999999,"5min":1.754,"15min":1.508},"min":{"1min":0.35799999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.6719999999999997,"5min":11.44,"15min":11.44},"last":4.1289999999999996}]},{"osd":5,"last update":"Mon Oct 9 11:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.9339999999999999,"15min":2.339},"min":{"1min":0.35199999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.2220000000000004,"5min":18.053000000000001,"15min":18.053000000000001},"last":1.2669999999999999},{"interface":"front","average":{"1min":1.496,"5min":2.0110000000000001,"15min":2.2850000000000001},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.31},"max":{"1min":6.0830000000000002,"5min":10.558,"15min":10.558},"last":6.0830000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":2.141,"15min":2.415},"min":{"1min":0.37,"5min":0.309,"15min":0.309},"max":{"1min":4.1719999999999997,"5min":18.085999999999999,"15min":18.085999999999999},"last":1.2370000000000001},{"interface":"front","average":{"1min":1.474,"5min":1.796,"15min":2.2040000000000002},"min":{"1min":0.40500000000000003,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.9790000000000001,"5min":10.531000000000001,"15min":10.531000000000001},"last":5.9790000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.8759999999999999,"15min":1.4830000000000001},"min":{"1min":0.40699999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.6399999999999997,"5min":11.484,"15min":11.484},"last":4.1719999999999997},{"interface":"front","average":{"1min":1.2809999999999999,"5min":1.833,"15min":2.3530000000000002},"min":{"1min":0.39200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.1280000000000001,"5min":18.071000000000002,"15min":18.071000000000002},"last":1.2090000000000001}]}]},{"osd":0,"up_from":80,"seq":343597383970,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1487896,"kb_used_data":690960,"kb_used_omap":28,"kb_used_meta":796899,"kb_avail":92257256,"statfs":{"total":95995035648,"available":94471430144,"internally_reserved":0,"allocated":707543040,"data_stored":704856308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":816025424},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:05:23 2023","interfaces":[{"interface":"back","average":{"1min":2.2050000000000001,"5min":2.2730000000000001,"15min":2.9489999999999998},"min":{"1min":0.19,"5min":0.186,"15min":0.186},"max":{"1min":14.644,"5min":19.608000000000001,"15min":19.608000000000001},"last":4.165},{"interface":"front","average":{"1min":1.292,"5min":2.0960000000000001,"15min":2.944},"min":{"1min":0.23400000000000001,"5min":0.218,"15min":0.218},"max":{"1min":8.9060000000000006,"5min":19.992999999999999,"15min":19.992999999999999},"last":4.5369999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:04:51 2023","interfaces":[{"interface":"back","average":{"1min":2.6059999999999999,"5min":2.274,"15min":2.2810000000000001},"min":{"1min":0.26300000000000001,"5min":0.20300000000000001,"15min":0.17199999999999999},"max":{"1min":12.459,"5min":12.638,"15min":24.852},"last":0.54900000000000004},{"interface":"front","average":{"1min":3.0219999999999998,"5min":2.3210000000000002,"15min":2.7290000000000001},"min":{"1min":0.20399999999999999,"5min":0.20300000000000001,"15min":0.19800000000000001},"max":{"1min":10.616,"5min":10.616,"15min":77.256},"last":5.2400000000000002}]},{"osd":3,"last update":"Mon Oct 9 11:05:15 2023","interfaces":[{"interface":"back","average":{"1min":3.532,"5min":3.532,"15min":3.532},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":25.050000000000001,"5min":25.050000000000001,"15min":25.050000000000001},"last":0.77500000000000002},{"interface":"front","average":{"1min":1.899,"5min":1.899,"15min":1.899},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":14.032999999999999,"5min":14.032999999999999,"15min":14.032999999999999},"last":0.44}]},{"osd":4,"last update":"Mon Oct 9 11:05:15 2023","interfaces":[{"interface":"back","average":{"1min":2.8159999999999998,"5min":2.0899999999999999,"15min":2.4849999999999999},"min":{"1min":0.39500000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":14.507,"5min":14.507,"15min":77.177999999999997},"last":0.84899999999999998},{"interface":"front","average":{"1min":1.2909999999999999,"5min":1.532,"15min":1.8049999999999999},"min":{"1min":0.33900000000000002,"5min":0.30199999999999999,"15min":0.28499999999999998},"max":{"1min":6.8550000000000004,"5min":9.9489999999999998,"15min":10.513999999999999},"last":0.39900000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:05:15 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.7949999999999999,"15min":1.9810000000000001},"min":{"1min":0.32200000000000001,"5min":0.30399999999999999,"15min":0.29799999999999999},"max":{"1min":4.1189999999999998,"5min":9.8710000000000004,"15min":10.544},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.837,"5min":1.5509999999999999,"15min":1.946},"min":{"1min":0.35799999999999998,"5min":0.308,"15min":0.29899999999999999},"max":{"1min":11.212999999999999,"5min":11.212999999999999,"15min":24.754999999999999},"last":0.629}]},{"osd":6,"last update":"Mon Oct 9 11:05:15 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.7949999999999999,"15min":2.0379999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":4.2599999999999998,"5min":10.012,"15min":18.920000000000002},"last":4.5039999999999996},{"interface":"front","average":{"1min":1.869,"5min":1.611,"15min":2.012},"min":{"1min":0.378,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":11.257999999999999,"5min":11.257999999999999,"15min":24.800000000000001},"last":0.60099999999999998}]},{"osd":7,"last update":"Mon Oct 9 11:05:15 2023","interfaces":[{"interface":"back","average":{"1min":2.2709999999999999,"5min":1.802,"15min":2.097},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.32700000000000001},"max":{"1min":17.672000000000001,"5min":17.672000000000001,"15min":17.672000000000001},"last":4.1200000000000001},{"interface":"front","average":{"1min":2.899,"5min":2.4039999999999999,"15min":2.5979999999999999},"min":{"1min":0.42099999999999999,"5min":0.371,"15min":0.314},"max":{"1min":17.539000000000001,"5min":17.539000000000001,"15min":77.228999999999999},"last":0.80800000000000005}]}]},{"osd":2,"up_from":90,"seq":386547056911,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1540040,"kb_used_data":932352,"kb_used_omap":81,"kb_used_meta":607598,"kb_avail":92205112,"statfs":{"total":95995035648,"available":94418034688,"internally_reserved":0,"allocated":954728448,"data_stored":952176485,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":622180570},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:04:56 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":2.3479999999999999,"15min":1.9970000000000001},"min":{"1min":0.251,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":8.2810000000000006,"5min":48.933999999999997,"15min":48.933999999999997},"last":1.3049999999999999},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.482,"15min":1.8149999999999999},"min":{"1min":0.28000000000000003,"5min":0.20499999999999999,"15min":0.19600000000000001},"max":{"1min":7.7130000000000001,"5min":14.515000000000001,"15min":15.952999999999999},"last":1.929}]},{"osd":1,"last update":"Mon Oct 9 11:05:23 2023","interfaces":[{"interface":"back","average":{"1min":1.7589999999999999,"5min":2.6099999999999999,"15min":2.552},"min":{"1min":0.23200000000000001,"5min":0.19,"15min":0.19},"max":{"1min":9.7599999999999998,"5min":49.033999999999999,"15min":49.033999999999999},"last":1.7969999999999999},{"interface":"front","average":{"1min":1.3500000000000001,"5min":2.6389999999999998,"15min":2.6499999999999999},"min":{"1min":0.247,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.8890000000000002,"5min":48.874000000000002,"15min":48.874000000000002},"last":0.70099999999999996}]},{"osd":3,"last update":"Mon Oct 9 11:05:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":1.7649999999999999,"15min":1.7649999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":8.1110000000000007,"5min":8.1110000000000007,"15min":8.1110000000000007},"last":1.8480000000000001},{"interface":"front","average":{"1min":2.145,"5min":2.145,"15min":2.145},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":8.1600000000000001,"5min":8.1600000000000001,"15min":8.1600000000000001},"last":0.53300000000000003}]},{"osd":4,"last update":"Mon Oct 9 11:04:56 2023","interfaces":[{"interface":"back","average":{"1min":2.726,"5min":1.978,"15min":1.9390000000000001},"min":{"1min":0.314,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.7989999999999995,"5min":9.7989999999999995,"15min":15.672000000000001},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.8120000000000001,"5min":2.2810000000000001,"15min":1.9379999999999999},"min":{"1min":0.32900000000000001,"5min":0.249,"15min":0.249},"max":{"1min":8.3949999999999996,"5min":48.795000000000002,"15min":48.795000000000002},"last":0.625}]},{"osd":5,"last update":"Mon Oct 9 11:04:56 2023","interfaces":[{"interface":"back","average":{"1min":1.272,"5min":1.204,"15min":1.5309999999999999},"min":{"1min":0.32100000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.9060000000000001,"5min":8.5380000000000003,"15min":12.128},"last":2.1499999999999999},{"interface":"front","average":{"1min":2.3370000000000002,"5min":2.3159999999999998,"15min":1.962},"min":{"1min":0.374,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":9.8260000000000005,"5min":49.061,"15min":49.061},"last":0.42799999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:04:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8560000000000001,"5min":2.3530000000000002,"15min":1.8560000000000001},"min":{"1min":0.35299999999999998,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":8.1950000000000003,"5min":48.845999999999997,"15min":48.845999999999997},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.429,"5min":1.405,"15min":1.5509999999999999},"min":{"1min":0.437,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.0350000000000001,"5min":7.0570000000000004,"15min":10.111000000000001},"last":2.2080000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:04:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.496,"15min":1.9199999999999999},"min":{"1min":0.41599999999999998,"5min":0.35599999999999998,"15min":0.31900000000000001},"max":{"1min":7.4009999999999998,"5min":48.966000000000001,"15min":48.966000000000001},"last":0.66000000000000003},{"interface":"front","average":{"1min":1.5900000000000001,"5min":1.532,"15min":1.6899999999999999},"min":{"1min":0.42499999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.9260000000000002,"5min":14.94,"15min":14.94},"last":2.1829999999999998}]}]},{"osd":3,"up_from":469,"seq":2014339661840,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2019344,"kb_used_data":1827592,"kb_used_omap":11,"kb_used_meta":191732,"kb_avail":91725808,"statfs":{"total":95995035648,"available":93927227392,"internally_reserved":0,"allocated":1871454208,"data_stored":1868657800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11309,"internal_metadata":196334547},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:05:15 2023","interfaces":[{"interface":"back","average":{"1min":1.7210000000000001,"5min":1.7210000000000001,"15min":1.7210000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":14.882,"5min":14.882,"15min":14.882},"last":1.8720000000000001},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.5230000000000001,"5min":2.5230000000000001,"15min":2.5230000000000001},"last":0.25700000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:05:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":10.140000000000001,"5min":10.140000000000001,"15min":10.140000000000001},"last":1.2430000000000001},{"interface":"front","average":{"1min":1.585,"5min":1.585,"15min":1.585},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.18,"5min":10.18,"15min":10.18},"last":2.0859999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:05:15 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.087,"15min":1.087},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.444,"5min":6.444,"15min":6.444},"last":0.78300000000000003},{"interface":"front","average":{"1min":1.0800000000000001,"5min":1.0800000000000001,"15min":1.0800000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.5389999999999997,"5min":5.5389999999999997,"15min":5.5389999999999997},"last":0.77000000000000002}]},{"osd":4,"last update":"Mon Oct 9 11:05:15 2023","interfaces":[{"interface":"back","average":{"1min":1.573,"5min":1.573,"15min":1.573},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":14.907999999999999,"5min":14.907999999999999,"15min":14.907999999999999},"last":5.1660000000000004},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":2.069,"5min":2.069,"15min":2.069},"last":4.1390000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:05:15 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":1.413,"15min":1.413},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.3300000000000001,"5min":9.3300000000000001,"15min":9.3300000000000001},"last":1.901},{"interface":"front","average":{"1min":1.4410000000000001,"5min":1.4410000000000001,"15min":1.4410000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":9.3130000000000006,"5min":9.3130000000000006,"15min":9.3130000000000006},"last":1.972}]},{"osd":6,"last update":"Mon Oct 9 11:05:15 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":1.7829999999999999,"15min":1.7829999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":15.455,"5min":15.455,"15min":15.455},"last":2.2490000000000001},{"interface":"front","average":{"1min":1.643,"5min":1.643,"15min":1.643},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":14.837,"5min":14.837,"15min":14.837},"last":2.2330000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:05:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":2.9279999999999999,"5min":2.9279999999999999,"15min":2.9279999999999999},"last":4.1790000000000003},{"interface":"front","average":{"1min":1.157,"5min":1.157,"15min":1.157},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":5.5620000000000003,"5min":5.5620000000000003,"15min":5.5620000000000003},"last":5.1210000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019261,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1473984,"kb_used_data":589432,"kb_used_omap":0,"kb_used_meta":884544,"kb_avail":92271168,"statfs":{"total":95995035648,"available":94485676032,"internally_reserved":0,"allocated":603578368,"data_stored":601958261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":905773056},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:05:13 2023","interfaces":[{"interface":"back","average":{"1min":2.7629999999999999,"5min":1.643,"15min":2.4609999999999999},"min":{"1min":0.223,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":13.162000000000001,"5min":13.162000000000001,"15min":213.88999999999999},"last":2.5099999999999998},{"interface":"front","average":{"1min":1.901,"5min":2.1190000000000002,"15min":2.7469999999999999},"min":{"1min":0.28799999999999998,"5min":0.24399999999999999,"15min":0.23200000000000001},"max":{"1min":9.0670000000000002,"5min":11.128,"15min":214.09800000000001},"last":1.0940000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:05:19 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":3.0579999999999998,"15min":2.4140000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":7.8630000000000004,"5min":121.41,"15min":121.41},"last":1.127},{"interface":"front","average":{"1min":1.522,"5min":2.9260000000000002,"15min":1.8859999999999999},"min":{"1min":0.26300000000000001,"5min":0.251,"15min":0.251},"max":{"1min":7.8849999999999998,"5min":122.21899999999999,"15min":122.21899999999999},"last":1.1910000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:04:56 2023","interfaces":[{"interface":"back","average":{"1min":2.7109999999999999,"5min":1.863,"15min":1.8129999999999999},"min":{"1min":0.29099999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":10.699999999999999,"5min":10.699999999999999,"15min":15.446999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.891,"5min":1.974,"15min":2.1269999999999998},"min":{"1min":0.27000000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.4250000000000007,"5min":8.4250000000000007,"15min":27.818000000000001},"last":1.0529999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:05:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.7509999999999999,"15min":1.7509999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.8369999999999997,"5min":7.8369999999999997,"15min":7.8369999999999997},"last":2.0939999999999999},{"interface":"front","average":{"1min":1.169,"5min":1.169,"15min":1.169},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":3.7309999999999999,"5min":3.7309999999999999,"15min":3.7309999999999999},"last":2.5259999999999998}]},{"osd":5,"last update":"Mon Oct 9 11:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.498,"15min":1.6479999999999999},"min":{"1min":0.27900000000000003,"5min":0.224,"15min":0.224},"max":{"1min":3.802,"5min":11.59,"15min":19.068999999999999},"last":2.6560000000000001},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.3999999999999999,"15min":1.409},"min":{"1min":0.27700000000000002,"5min":0.25800000000000001,"15min":0.22600000000000001},"max":{"1min":7.5229999999999997,"5min":9.8610000000000007,"15min":15.535},"last":0.47399999999999998}]},{"osd":6,"last update":"Mon Oct 9 11:04:52 2023","interfaces":[{"interface":"back","average":{"1min":1.929,"5min":1.302,"15min":1.2829999999999999},"min":{"1min":0.29999999999999999,"5min":0.29499999999999998,"15min":0.24099999999999999},"max":{"1min":7.7969999999999997,"5min":7.7969999999999997,"15min":15.739000000000001},"last":2.4750000000000001},{"interface":"front","average":{"1min":2.0190000000000001,"5min":1.6990000000000001,"15min":1.7290000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":12.795999999999999,"5min":12.795999999999999,"15min":14.272},"last":2.625}]},{"osd":7,"last update":"Mon Oct 9 11:04:56 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.823,"15min":1.7110000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.7719999999999998,"5min":19.923999999999999,"15min":19.923999999999999},"last":2.585},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.7509999999999999,"15min":1.571},"min":{"1min":0.34399999999999997,"5min":0.29799999999999999,"15min":0.255},"max":{"1min":4.0439999999999996,"5min":19.859999999999999,"15min":19.859999999999999},"last":0.54600000000000004}]}]},{"osd":5,"up_from":37,"seq":158913790330,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1680624,"kb_used_data":773800,"kb_used_omap":0,"kb_used_meta":906816,"kb_avail":92064528,"statfs":{"total":95995035648,"available":94274076672,"internally_reserved":0,"allocated":792371200,"data_stored":790763442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928579584},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:05:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":3.0710000000000002,"15min":2.4910000000000001},"min":{"1min":0.26800000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.4829999999999997,"5min":58.271999999999998,"15min":58.271999999999998},"last":1.462},{"interface":"front","average":{"1min":1.6100000000000001,"5min":3.117,"15min":2.5379999999999998},"min":{"1min":0.38900000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.5140000000000002,"5min":58.25,"15min":58.25},"last":1.4970000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:05:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":2.2109999999999999,"15min":2.0630000000000002},"min":{"1min":0.25,"5min":0.247,"15min":0.247},"max":{"1min":5.3140000000000001,"5min":32.491999999999997,"15min":32.491999999999997},"last":3.0779999999999998},{"interface":"front","average":{"1min":2.0230000000000001,"5min":2.331,"15min":1.851},"min":{"1min":0.35299999999999998,"5min":0.247,"15min":0.247},"max":{"1min":10.512,"5min":32.537999999999997,"15min":32.537999999999997},"last":3.0499999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:04:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8240000000000001,"5min":1.839,"15min":2.0779999999999998},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.221},"max":{"1min":6.3929999999999998,"5min":9.3539999999999992,"15min":11.368},"last":0.73999999999999999},{"interface":"front","average":{"1min":2.9430000000000001,"5min":2.1909999999999998,"15min":2.0350000000000001},"min":{"1min":0.29299999999999998,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":11.673,"5min":14.621,"15min":15.087999999999999},"last":0.45100000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:05:15 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":1.49,"15min":1.49},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.9649999999999999,"5min":6.9649999999999999,"15min":6.9649999999999999},"last":3.8500000000000001},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.8089999999999999,"15min":1.8089999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.0789999999999997,"5min":5.0789999999999997,"15min":5.0789999999999997},"last":3.7810000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:05:30 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":1.21,"15min":1.1160000000000001},"min":{"1min":0.28699999999999998,"5min":0.23799999999999999,"15min":0.216},"max":{"1min":7.5640000000000001,"5min":12.042,"15min":12.042},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.2410000000000001,"15min":1.329},"min":{"1min":0.28000000000000003,"5min":0.26400000000000001,"15min":0.24199999999999999},"max":{"1min":3.3260000000000001,"5min":9.2490000000000006,"15min":11.111000000000001},"last":2.056}]},{"osd":6,"last update":"Mon Oct 9 11:04:35 2023","interfaces":[{"interface":"back","average":{"1min":1.9139999999999999,"5min":1.5589999999999999,"15min":1.4470000000000001},"min":{"1min":0.34300000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.1020000000000003,"5min":9.1969999999999992,"15min":11.098000000000001},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.8009999999999999,"5min":1.304,"15min":1.2769999999999999},"min":{"1min":0.35699999999999998,"5min":0.32400000000000001,"15min":0.29099999999999998},"max":{"1min":11.983000000000001,"5min":11.983000000000001,"15min":15.16},"last":0.48799999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:05:01 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.591,"15min":1.5600000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.8330000000000002,"5min":9.3539999999999992,"15min":10.739000000000001},"last":0.84999999999999998},{"interface":"front","average":{"1min":1.095,"5min":1.52,"15min":1.327},"min":{"1min":0.379,"5min":0.312,"15min":0.30099999999999999},"max":{"1min":8.0009999999999994,"5min":11.726000000000001,"15min":11.726000000000001},"last":0.39000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3733,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":724893696,"data_stored":724893696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1376518144,"data_stored":1376518144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":948961280,"data_stored":948961280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":977731584,"data_stored":977731584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":589299712,"data_stored":589299712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":675282944,"data_stored":675282944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":547946496,"data_stored":547946496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":872218624,"data_stored":872218624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:05:35.506 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T11:05:35.507 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_5 pg_num 26 2023-10-09T11:05:35.517 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:05:35.558 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:35.591 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:05:35.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:35.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:05:35.697 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:35.743 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:35.784 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:35.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:35.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:35.820 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:35.890 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:35.941 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:35.969 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:35.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:35.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:35.972 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:35.980 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 27 16 40300 40284 93.171 48.5 0.0130072 0.0107261 2023-10-09T11:05:36.073 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:36.103 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:36.128 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:36.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:36.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:36.232 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:36.270 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:36 smithi125 ceph-mon[195507]: failed to encode map e505 with expected crc 2023-10-09T11:05:36.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:36 smithi125 ceph-mon[195507]: osdmap e506: 8 total, 8 up, 8 in 2023-10-09T11:05:36.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:36 smithi125 ceph-mon[195507]: failed to encode map e506 with expected crc 2023-10-09T11:05:36.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:36 smithi125 ceph-mon[195507]: failed to encode map e506 with expected crc 2023-10-09T11:05:36.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:36 smithi125 ceph-mon[195507]: failed to encode map e506 with expected crc 2023-10-09T11:05:36.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:36 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:36.271 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:36 smithi125 ceph-mon[200197]: failed to encode map e505 with expected crc 2023-10-09T11:05:36.271 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:36 smithi125 ceph-mon[200197]: osdmap e506: 8 total, 8 up, 8 in 2023-10-09T11:05:36.271 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:36 smithi125 ceph-mon[200197]: failed to encode map e506 with expected crc 2023-10-09T11:05:36.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:36 smithi125 ceph-mon[200197]: failed to encode map e506 with expected crc 2023-10-09T11:05:36.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:36 smithi125 ceph-mon[200197]: failed to encode map e506 with expected crc 2023-10-09T11:05:36.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:36 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:36.309 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:36.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:36 smithi155 ceph-mon[184854]: failed to encode map e505 with expected crc 2023-10-09T11:05:36.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:36 smithi155 ceph-mon[184854]: osdmap e506: 8 total, 8 up, 8 in 2023-10-09T11:05:36.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:36 smithi155 ceph-mon[184854]: failed to encode map e506 with expected crc 2023-10-09T11:05:36.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:36 smithi155 ceph-mon[184854]: failed to encode map e506 with expected crc 2023-10-09T11:05:36.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:36 smithi155 ceph-mon[184854]: failed to encode map e506 with expected crc 2023-10-09T11:05:36.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:36 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:36.684 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:36.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:36.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:36.789 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:36.832 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:36.863 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:36.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:36.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:36.969 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:36.980 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 28 16 41497 41481 92.5156 74.8125 0.0124115 0.0108034 2023-10-09T11:05:37.012 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:37.041 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:37.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:37.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:37.144 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:37.190 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:37.329 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:05:37.360 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:37 smithi125 ceph-mon[195507]: failed to encode map e506 with expected crc 2023-10-09T11:05:37.360 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:37 smithi125 ceph-mon[195507]: failed to encode map e506 with expected crc 2023-10-09T11:05:37.360 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:37 smithi125 ceph-mon[195507]: pgmap v1238: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s; 9325/110985 objects degraded (8.402%); 347/110985 objects misplaced (0.313%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T11:05:37.360 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:37 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2653696720' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:05:37.360 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:37 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:05:37.361 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:37 smithi125 ceph-mon[200197]: failed to encode map e506 with expected crc 2023-10-09T11:05:37.361 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:37 smithi125 ceph-mon[200197]: failed to encode map e506 with expected crc 2023-10-09T11:05:37.361 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:37 smithi125 ceph-mon[200197]: pgmap v1238: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s; 9325/110985 objects degraded (8.402%); 347/110985 objects misplaced (0.313%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T11:05:37.361 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:37 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2653696720' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:05:37.361 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:37 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:05:37.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:37 smithi155 ceph-mon[184854]: failed to encode map e506 with expected crc 2023-10-09T11:05:37.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:37 smithi155 ceph-mon[184854]: failed to encode map e506 with expected crc 2023-10-09T11:05:37.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:37 smithi155 ceph-mon[184854]: pgmap v1238: 119 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s; 9325/110985 objects degraded (8.402%); 347/110985 objects misplaced (0.313%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T11:05:37.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:37 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2653696720' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:05:37.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:37 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:05:37.981 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 29 16 42752 42736 92.026 78.4375 0.0100061 0.0108612 2023-10-09T11:05:38.033 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:38.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:38.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:38.139 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:38.192 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:38.229 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:38.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:38.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:38.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:38 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:05:38.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:38 smithi125 ceph-mon[195507]: failed to encode map e507 with expected crc 2023-10-09T11:05:38.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:38 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:05:38.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:38 smithi125 ceph-mon[195507]: osdmap e507: 8 total, 8 up, 8 in 2023-10-09T11:05:38.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:38 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:05:38.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:38 smithi125 ceph-mon[200197]: failed to encode map e507 with expected crc 2023-10-09T11:05:38.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:38 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:05:38.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:38 smithi125 ceph-mon[200197]: osdmap e507: 8 total, 8 up, 8 in 2023-10-09T11:05:38.335 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:38.388 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:38.459 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:38.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:38.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:38.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:38 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:05:38.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:38 smithi155 ceph-mon[184854]: failed to encode map e507 with expected crc 2023-10-09T11:05:38.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:38 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:05:38.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:38 smithi155 ceph-mon[184854]: osdmap e507: 8 total, 8 up, 8 in 2023-10-09T11:05:38.539 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:38.566 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:38.619 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:38.752 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:38.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:38.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:38.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:38.920 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:38.982 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 30 16 43993 43977 91.5414 77.5625 0.00732602 0.0109186 2023-10-09T11:05:39.015 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:39.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:39.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:39.120 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:39.208 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:39 smithi125 ceph-mon[195507]: pgmap v1240: 119 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 9325/116784 objects degraded (7.985%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T11:05:39.208 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:39 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 9325/110985 objects degraded (8.402%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:39.209 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:39 smithi125 ceph-mon[200197]: pgmap v1240: 119 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 9325/116784 objects degraded (7.985%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T11:05:39.209 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:39 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 9325/110985 objects degraded (8.402%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:39.214 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:39.261 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:05:39.306 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:39.312 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:05:39.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:39.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:39.351 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:05:39.388 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:05:39.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:39.425 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:05:39.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:39 smithi155 ceph-mon[184854]: pgmap v1240: 119 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 9325/116784 objects degraded (7.985%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T11:05:39.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:39 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 9325/110985 objects degraded (8.402%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:39.474 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:05:39.491 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:39.508 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:05:39.551 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:05:39.555 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:05:39.555 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:05:39.555 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:05:39.555 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:05:39.555 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:05:39.555 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:05:39.555 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:05:39.556 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:05:39.696 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:05:39.697 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:05:39.730 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:39.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:39.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:39.834 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:39.886 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:39.983 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 31 16 45113 45097 90.8468 70 0.0100554 0.0110023 2023-10-09T11:05:40.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:40 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3001620027' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:40.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:40 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3001620027' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:40.338 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:05:40.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:40 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3001620027' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:40.983 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 32 16 46427 46411 90.5742 82.125 0.0116735 0.0110343 2023-10-09T11:05:41.039 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:41.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:41.144 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:41.188 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:41.217 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:41 smithi125 ceph-mon[195507]: pgmap v1241: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s; 8026/126537 objects degraded (6.343%); 722/126537 objects misplaced (0.571%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T11:05:41.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:41 smithi125 ceph-mon[200197]: pgmap v1241: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s; 8026/126537 objects degraded (6.343%); 722/126537 objects misplaced (0.571%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T11:05:41.218 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:05:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:05:41] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2023-10-09T11:05:41.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:41.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:41.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:41.327 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:41.376 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:41.386 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T11:05:41.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:41 smithi155 ceph-mon[184854]: pgmap v1241: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s; 8026/126537 objects degraded (6.343%); 722/126537 objects misplaced (0.571%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T11:05:41.512 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:41.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:41.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:41.620 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:41.656 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:41.684 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:41.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:41.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:41.750 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:41.790 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:41.832 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:41.862 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:41.868 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:41.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:41.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:41.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:41.985 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 33 16 47517 47501 89.8942 68.125 0.0171886 0.0111175 2023-10-09T11:05:42.093 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:42.478 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:42.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:42.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:42.583 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:42.635 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:42.682 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:42.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:42.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:42.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:42.854 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:42.983 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 34 16 48701 48685 89.4251 74 0.011113 0.0111769 2023-10-09T11:05:43.045 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:43.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:43.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:43.152 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:43.167 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 11 pg_num to 26 2023-10-09T11:05:43.201 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:43.230 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:43.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:43.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:43.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:43 smithi125 ceph-mon[195507]: pgmap v1242: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 8026/133467 objects degraded (6.013%); 722/133467 objects misplaced (0.541%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T11:05:43.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:43 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2666152329' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T11:05:43.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:43 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T11:05:43.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:43 smithi125 ceph-mon[200197]: pgmap v1242: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 8026/133467 objects degraded (6.013%); 722/133467 objects misplaced (0.541%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T11:05:43.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:43 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2666152329' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T11:05:43.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:43 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T11:05:43.337 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:43.392 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:43.424 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:05:43.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:43 smithi155 ceph-mon[184854]: pgmap v1242: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 8026/133467 objects degraded (6.013%); 722/133467 objects misplaced (0.541%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T11:05:43.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:43 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2666152329' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T11:05:43.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:43 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T11:05:43.734 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:43.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:43.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:43.776 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:05:43.776 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:05:43.850 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:43.896 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:43.933 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:43.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:43.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:43.984 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 35 16 49732 49716 88.7115 64.4375 0.0107323 0.0112668 2023-10-09T11:05:44.037 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:44.089 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:44.131 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:44.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:44.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:44.183 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T11:05:44.197 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:44.237 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:44.315 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:44.333 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:05:44.378 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[195507]: failed to encode map e508 with expected crc 2023-10-09T11:05:44.378 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-10-09T11:05:44.378 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[195507]: osdmap e508: 8 total, 8 up, 8 in 2023-10-09T11:05:44.378 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[195507]: failed to encode map e508 with expected crc 2023-10-09T11:05:44.378 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[195507]: failed to encode map e508 with expected crc 2023-10-09T11:05:44.378 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[195507]: failed to encode map e508 with expected crc 2023-10-09T11:05:44.379 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 8026/133467 objects degraded (6.013%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:44.379 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T11:05:44.379 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3650657988' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:05:44.379 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[200197]: failed to encode map e508 with expected crc 2023-10-09T11:05:44.379 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-10-09T11:05:44.379 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[200197]: osdmap e508: 8 total, 8 up, 8 in 2023-10-09T11:05:44.379 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[200197]: failed to encode map e508 with expected crc 2023-10-09T11:05:44.379 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[200197]: failed to encode map e508 with expected crc 2023-10-09T11:05:44.379 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[200197]: failed to encode map e508 with expected crc 2023-10-09T11:05:44.379 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 8026/133467 objects degraded (6.013%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:44.380 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T11:05:44.380 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:44 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3650657988' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:05:44.393 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:44.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:44.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:44.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:44 smithi155 ceph-mon[184854]: failed to encode map e508 with expected crc 2023-10-09T11:05:44.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:44 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-10-09T11:05:44.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:44 smithi155 ceph-mon[184854]: osdmap e508: 8 total, 8 up, 8 in 2023-10-09T11:05:44.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:44 smithi155 ceph-mon[184854]: failed to encode map e508 with expected crc 2023-10-09T11:05:44.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:44 smithi155 ceph-mon[184854]: failed to encode map e508 with expected crc 2023-10-09T11:05:44.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:44 smithi155 ceph-mon[184854]: failed to encode map e508 with expected crc 2023-10-09T11:05:44.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:44 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 8026/133467 objects degraded (6.013%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:44.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:44 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T11:05:44.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:44 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3650657988' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:05:44.503 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:05:44.531 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:44.550 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:05:44.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:44.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:05:44.654 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:44.684 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:44.705 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:44.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:44.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:44.809 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:44.837 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:44.984 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 36 16 50115 50099 86.9133 23.9375 0.135932 0.0113029 2023-10-09T11:05:45.007 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:45.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:45.116 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:45.143 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:45.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[195507]: pgmap v1244: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s; 8026/144576 objects degraded (5.551%); 252/144576 objects misplaced (0.174%); 11 MiB/s, 174 objects/s recovering 2023-10-09T11:05:45.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:05:45.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T11:05:45.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3650657988' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:05:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[195507]: osdmap e509: 8 total, 8 up, 8 in 2023-10-09T11:05:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[195507]: failed to encode map e509 with expected crc 2023-10-09T11:05:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[195507]: failed to encode map e509 with expected crc 2023-10-09T11:05:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:05:45.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[200197]: pgmap v1244: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s; 8026/144576 objects degraded (5.551%); 252/144576 objects misplaced (0.174%); 11 MiB/s, 174 objects/s recovering 2023-10-09T11:05:45.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:05:45.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T11:05:45.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3650657988' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:05:45.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[200197]: osdmap e509: 8 total, 8 up, 8 in 2023-10-09T11:05:45.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[200197]: failed to encode map e509 with expected crc 2023-10-09T11:05:45.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[200197]: failed to encode map e509 with expected crc 2023-10-09T11:05:45.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:45.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:05:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:45 smithi155 ceph-mon[184854]: pgmap v1244: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s; 8026/144576 objects degraded (5.551%); 252/144576 objects misplaced (0.174%); 11 MiB/s, 174 objects/s recovering 2023-10-09T11:05:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:45 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:05:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T11:05:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:45 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3650657988' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:05:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:45 smithi155 ceph-mon[184854]: osdmap e509: 8 total, 8 up, 8 in 2023-10-09T11:05:45.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:45 smithi155 ceph-mon[184854]: failed to encode map e509 with expected crc 2023-10-09T11:05:45.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:45 smithi155 ceph-mon[184854]: failed to encode map e509 with expected crc 2023-10-09T11:05:45.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:45.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:05:45.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:45.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:45.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:45.984 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 37 16 50162 50146 84.6452 2.9375 0.00162642 0.0114773 2023-10-09T11:05:46.067 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:46.094 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:46.113 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:46.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:46.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:46.217 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:46.245 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:46.264 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:46.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:46.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[195507]: failed to encode map e509 with expected crc 2023-10-09T11:05:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[195507]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[195507]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[195507]: osdmap e510: 8 total, 8 up, 8 in 2023-10-09T11:05:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[195507]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[195507]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[195507]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[195507]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[200197]: failed to encode map e509 with expected crc 2023-10-09T11:05:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[200197]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[200197]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[200197]: osdmap e510: 8 total, 8 up, 8 in 2023-10-09T11:05:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[200197]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[200197]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[200197]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[200197]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:46.367 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:46.409 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:46.426 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:46 smithi155 ceph-mon[184854]: failed to encode map e509 with expected crc 2023-10-09T11:05:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:46 smithi155 ceph-mon[184854]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:46 smithi155 ceph-mon[184854]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:46 smithi155 ceph-mon[184854]: osdmap e510: 8 total, 8 up, 8 in 2023-10-09T11:05:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:46 smithi155 ceph-mon[184854]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:46 smithi155 ceph-mon[184854]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:46 smithi155 ceph-mon[184854]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:46 smithi155 ceph-mon[184854]: failed to encode map e510 with expected crc 2023-10-09T11:05:46.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:05:46.892 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:46.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:46.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:46.984 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 38 16 51152 51136 84.0462 61.875 0.00486301 0.0118929 2023-10-09T11:05:46.997 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:47.052 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:47.166 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:47 smithi125 ceph-mon[200197]: failed to encode map e510 with expected crc 2023-10-09T11:05:47.166 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:47 smithi125 ceph-mon[200197]: pgmap v1247: 129 pgs: 10 unknown, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 8026/144576 objects degraded (5.551%); 252/144576 objects misplaced (0.174%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T11:05:47.166 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:47 smithi125 ceph-mon[200197]: failed to encode map e511 with expected crc 2023-10-09T11:05:47.166 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:47 smithi125 ceph-mon[200197]: osdmap e511: 8 total, 8 up, 8 in 2023-10-09T11:05:47.167 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:47 smithi125 ceph-mon[200197]: failed to encode map e511 with expected crc 2023-10-09T11:05:47.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:47 smithi125 ceph-mon[195507]: failed to encode map e510 with expected crc 2023-10-09T11:05:47.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:47 smithi125 ceph-mon[195507]: pgmap v1247: 129 pgs: 10 unknown, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 8026/144576 objects degraded (5.551%); 252/144576 objects misplaced (0.174%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T11:05:47.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:47 smithi125 ceph-mon[195507]: failed to encode map e511 with expected crc 2023-10-09T11:05:47.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:47 smithi125 ceph-mon[195507]: osdmap e511: 8 total, 8 up, 8 in 2023-10-09T11:05:47.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:47 smithi125 ceph-mon[195507]: failed to encode map e511 with expected crc 2023-10-09T11:05:47.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:47 smithi125 ceph-mon[195507]: failed to encode map e511 with expected crc 2023-10-09T11:05:47.235 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:47.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:47.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:47.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:47.423 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:47 smithi125 ceph-mon[200197]: failed to encode map e511 with expected crc 2023-10-09T11:05:47.429 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:47.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:47 smithi155 ceph-mon[184854]: failed to encode map e510 with expected crc 2023-10-09T11:05:47.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:47 smithi155 ceph-mon[184854]: pgmap v1247: 129 pgs: 10 unknown, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 8026/144576 objects degraded (5.551%); 252/144576 objects misplaced (0.174%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T11:05:47.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:47 smithi155 ceph-mon[184854]: failed to encode map e511 with expected crc 2023-10-09T11:05:47.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:47 smithi155 ceph-mon[184854]: osdmap e511: 8 total, 8 up, 8 in 2023-10-09T11:05:47.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:47 smithi155 ceph-mon[184854]: failed to encode map e511 with expected crc 2023-10-09T11:05:47.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:47 smithi155 ceph-mon[184854]: failed to encode map e511 with expected crc 2023-10-09T11:05:47.576 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:47.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:47.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:47.684 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:47.742 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:47.892 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:47.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:47.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:47.984 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 39 16 52571 52555 84.1649 88.6875 0.00605659 0.0118761 2023-10-09T11:05:47.997 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:48.031 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:48.058 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:48.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:48.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:48.162 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:48.170 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:05:48.238 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:48.280 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:48.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:48.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:48.301 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:05:48.341 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:05:48.378 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:05:48.386 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:48.421 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:48 smithi155 ceph-mon[184854]: failed to encode map e511 with expected crc 2023-10-09T11:05:48.423 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:48.430 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:05:48.449 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:48.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:48.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:48.481 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:05:48.520 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:05:48.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:05:48.560 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:05:48.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:48 smithi125 ceph-mon[195507]: failed to encode map e511 with expected crc 2023-10-09T11:05:48.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:48 smithi125 ceph-mon[200197]: failed to encode map e511 with expected crc 2023-10-09T11:05:48.585 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:48.592 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:05:48.592 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:05:48.592 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.593 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:53274 11.6 11:665d6a46:::benchmark_data_smithi125_439543_object53273:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-10-09T11:05:48.593 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:48.589247+0000", 2023-10-09T11:05:48.593 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0023302879999999998, 2023-10-09T11:05:48.593 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002366, 2023-10-09T11:05:48.593 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:48.593 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:05:48.593 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:05:48.593 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:05:48.593 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:05:48.593 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 53274 2023-10-09T11:05:48.593 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.594 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:48.594 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.594 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:48.594 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589247+0000", 2023-10-09T11:05:48.594 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:48.594 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.594 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.594 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:48.594 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589247+0000", 2023-10-09T11:05:48.594 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:48.594 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.595 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.595 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:48.595 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589245+0000", 2023-10-09T11:05:48.595 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:05:48.595 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.595 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.595 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:48.595 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589308+0000", 2023-10-09T11:05:48.595 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2327000000000001e-05 2023-10-09T11:05:48.595 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.595 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.596 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:48.596 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589331+0000", 2023-10-09T11:05:48.596 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3006e-05 2023-10-09T11:05:48.596 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.596 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.596 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:48.596 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589342+0000", 2023-10-09T11:05:48.596 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.116e-05 2023-10-09T11:05:48.596 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.596 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.596 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:48.597 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589393+0000", 2023-10-09T11:05:48.597 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1647000000000002e-05 2023-10-09T11:05:48.597 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.597 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.597 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:48.597 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589555+0000", 2023-10-09T11:05:48.597 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000161633 2023-10-09T11:05:48.597 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.597 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.597 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T11:05:48.597 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589703+0000", 2023-10-09T11:05:48.598 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000148085 2023-10-09T11:05:48.598 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:48.598 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:48.598 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:48.598 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.598 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.598 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:53276 11.6 11:644f957b:::benchmark_data_smithi125_439543_object53275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-10-09T11:05:48.598 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:48.589359+0000", 2023-10-09T11:05:48.598 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0022184700000000002, 2023-10-09T11:05:48.598 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023412540000000001, 2023-10-09T11:05:48.599 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:48.599 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:05:48.599 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:05:48.599 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:05:48.599 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:05:48.599 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 53276 2023-10-09T11:05:48.599 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.599 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:48.599 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.599 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:48.599 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589359+0000", 2023-10-09T11:05:48.600 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:48.600 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.600 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.600 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:48.600 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589359+0000", 2023-10-09T11:05:48.600 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:48.600 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.600 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.600 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:48.600 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589359+0000", 2023-10-09T11:05:48.600 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:48.601 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.601 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.601 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:48.601 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589412+0000", 2023-10-09T11:05:48.601 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3545999999999999e-05 2023-10-09T11:05:48.601 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.601 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.601 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:48.601 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589422+0000", 2023-10-09T11:05:48.601 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4120000000000007e-06 2023-10-09T11:05:48.601 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.601 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.602 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:48.602 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589429+0000", 2023-10-09T11:05:48.602 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2389999999999996e-06 2023-10-09T11:05:48.602 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.602 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.602 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:48.602 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.590787+0000", 2023-10-09T11:05:48.602 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013578220000000001 2023-10-09T11:05:48.602 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.602 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.603 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:48.603 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.590934+0000", 2023-10-09T11:05:48.603 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014728300000000001 2023-10-09T11:05:48.603 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.603 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.603 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T11:05:48.603 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.591073+0000", 2023-10-09T11:05:48.603 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013903200000000001 2023-10-09T11:05:48.603 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:48.603 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:48.603 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:48.604 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.604 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.604 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:53263 11.1 e511/510 11:8558ad91:::benchmark_data_smithi125_439543_object53262:head v 511'3345, mlcod=0'0)", 2023-10-09T11:05:48.604 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:48.589457+0000", 2023-10-09T11:05:48.604 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0021202920000000002, 2023-10-09T11:05:48.604 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023191570000000001, 2023-10-09T11:05:48.604 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:48.604 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:05:48.604 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:48.604 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.605 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:48.605 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589457+0000", 2023-10-09T11:05:48.605 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:48.605 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.605 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.605 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:48.605 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589457+0000", 2023-10-09T11:05:48.605 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:48.605 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.605 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.605 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:48.606 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589457+0000", 2023-10-09T11:05:48.606 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:05:48.606 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.606 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.606 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:48.606 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589485+0000", 2023-10-09T11:05:48.606 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.809e-05 2023-10-09T11:05:48.606 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.606 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.606 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:48.607 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589494+0000", 2023-10-09T11:05:48.607 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3910000000000004e-06 2023-10-09T11:05:48.607 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.607 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.607 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:48.607 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589500+0000", 2023-10-09T11:05:48.607 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9320000000000001e-06 2023-10-09T11:05:48.607 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.607 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.607 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:48.607 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589798+0000", 2023-10-09T11:05:48.607 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000298473 2023-10-09T11:05:48.608 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.608 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.608 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:48.608 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589827+0000", 2023-10-09T11:05:48.608 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8286999999999999e-05 2023-10-09T11:05:48.608 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:48.608 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:48.608 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:48.608 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.608 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.608 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:53271 11.2 e511/509)", 2023-10-09T11:05:48.608 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:48.589519+0000", 2023-10-09T11:05:48.609 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0020588920000000001, 2023-10-09T11:05:48.609 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023158639999999999, 2023-10-09T11:05:48.609 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:48.609 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T11:05:48.609 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:48.609 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.609 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:48.609 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589519+0000", 2023-10-09T11:05:48.609 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:48.609 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.609 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.609 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:48.609 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589519+0000", 2023-10-09T11:05:48.609 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:48.610 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.610 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.610 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:48.610 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589518+0000", 2023-10-09T11:05:48.610 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:05:48.610 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.610 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.610 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:48.610 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589545+0000", 2023-10-09T11:05:48.610 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6268999999999999e-05 2023-10-09T11:05:48.610 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.611 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.611 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:48.611 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589552+0000", 2023-10-09T11:05:48.611 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2309999999999999e-06 2023-10-09T11:05:48.611 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.611 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.611 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:48.611 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589557+0000", 2023-10-09T11:05:48.611 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0189999999999997e-06 2023-10-09T11:05:48.611 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:48.611 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:48.611 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:48.611 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.612 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.612 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:53267 11.11 e511/509 11:8c41d2bc:::benchmark_data_smithi125_439543_object53266:head v 511'3357, mlcod=0'0)", 2023-10-09T11:05:48.612 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:48.589569+0000", 2023-10-09T11:05:48.612 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0020089489999999999, 2023-10-09T11:05:48.612 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023191179999999998, 2023-10-09T11:05:48.612 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:48.612 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:05:48.612 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:48.612 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.612 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:48.612 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589569+0000", 2023-10-09T11:05:48.612 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:48.613 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.613 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.613 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:48.613 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589569+0000", 2023-10-09T11:05:48.613 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:48.613 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.613 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.613 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:48.613 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589568+0000", 2023-10-09T11:05:48.613 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967296 2023-10-09T11:05:48.613 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.613 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.614 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:48.614 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589593+0000", 2023-10-09T11:05:48.614 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4428000000000001e-05 2023-10-09T11:05:48.614 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.614 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.614 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:48.614 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589599+0000", 2023-10-09T11:05:48.614 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5769999999999999e-06 2023-10-09T11:05:48.614 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.614 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.614 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:48.615 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589604+0000", 2023-10-09T11:05:48.615 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.685e-06 2023-10-09T11:05:48.615 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.615 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.615 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:05:48.615 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.590965+0000", 2023-10-09T11:05:48.615 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001360637 2023-10-09T11:05:48.615 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.615 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.615 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:05:48.615 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.590990+0000", 2023-10-09T11:05:48.615 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5221000000000001e-05 2023-10-09T11:05:48.616 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:48.616 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:48.616 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:48.616 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.616 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.616 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:53272 11.1 e511/510)", 2023-10-09T11:05:48.616 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:48.589608+0000", 2023-10-09T11:05:48.616 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.00196911, 2023-10-09T11:05:48.616 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002345209, 2023-10-09T11:05:48.616 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:48.616 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T11:05:48.616 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:48.617 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.617 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:48.617 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589608+0000", 2023-10-09T11:05:48.617 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:48.617 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.617 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.617 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:48.617 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589608+0000", 2023-10-09T11:05:48.617 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:48.617 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.617 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.617 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:48.618 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589608+0000", 2023-10-09T11:05:48.618 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967296 2023-10-09T11:05:48.618 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.618 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.618 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:48.618 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589633+0000", 2023-10-09T11:05:48.618 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4399000000000001e-05 2023-10-09T11:05:48.618 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.618 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.618 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:48.618 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589639+0000", 2023-10-09T11:05:48.619 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1920000000000002e-06 2023-10-09T11:05:48.619 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.619 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.619 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:48.619 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589644+0000", 2023-10-09T11:05:48.619 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7430000000000002e-06 2023-10-09T11:05:48.619 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:48.619 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:48.619 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:48.619 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.619 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.619 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:53273 11.1 e511/510)", 2023-10-09T11:05:48.619 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:48.589648+0000", 2023-10-09T11:05:48.620 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.001929362, 2023-10-09T11:05:48.620 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023367800000000001, 2023-10-09T11:05:48.620 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:05:48.620 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T11:05:48.620 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:05:48.620 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.620 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:05:48.620 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589648+0000", 2023-10-09T11:05:48.620 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:48.620 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.620 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.620 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:05:48.620 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589648+0000", 2023-10-09T11:05:48.621 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:05:48.621 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.621 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.621 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:05:48.621 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589648+0000", 2023-10-09T11:05:48.621 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967296 2023-10-09T11:05:48.621 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.621 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.621 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:05:48.621 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589674+0000", 2023-10-09T11:05:48.621 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6143e-05 2023-10-09T11:05:48.621 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.621 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.622 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:05:48.622 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589681+0000", 2023-10-09T11:05:48.622 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5200000000000003e-06 2023-10-09T11:05:48.622 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:05:48.622 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:05:48.622 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:05:48.622 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:48.589685+0000", 2023-10-09T11:05:48.622 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6720000000000003e-06 2023-10-09T11:05:48.622 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:48.622 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:05:48.622 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:48.622 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:05:48.623 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T11:05:48.623 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 7 2023-10-09T11:05:48.623 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:05:48.623 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:05:48.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:05:48.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:05:48.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:05:48.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:05:48.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:05:48.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:05:48.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:05:48.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:05:48.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:48.625 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:05:48.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:05:48.685 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:48.725 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:48.769 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:48.770 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 7, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 5, 6, 7, 1, 3] 2023-10-09T11:05:48.770 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:05:48.770 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T11:05:48.771 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T11:05:48.771 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T11:05:48.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T11:05:48.802 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:48.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:48.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:48.908 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:48.986 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T11:05:48.985857+0000 min lat: 0.00101021 max lat: 2.16829 avg lat: 0.0119031 2023-10-09T11:05:48.986 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T11:05:48.986 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 40 16 53799 53783 83.9781 76.75 0.0068409 0.0119031 2023-10-09T11:05:49.030 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:49.071 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:49.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:49.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:49.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:49 smithi155 ceph-mon[184854]: pgmap v1249: 129 pgs: 6 active+recovery_wait+degraded, 6 unknown, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 22 MiB/s wr, 354 op/s; 8026/148503 objects degraded (5.405%); 252/148503 objects misplaced (0.170%) 2023-10-09T11:05:49.199 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:49 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2565120987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:49.199 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:49 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 8026/148503 objects degraded (5.405%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:49.199 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:49 smithi155 ceph-mon[184854]: 11.6 scrub starts 2023-10-09T11:05:49.204 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:49.208 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:05:49.255 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:49.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:49 smithi125 ceph-mon[195507]: pgmap v1249: 129 pgs: 6 active+recovery_wait+degraded, 6 unknown, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 22 MiB/s wr, 354 op/s; 8026/148503 objects degraded (5.405%); 252/148503 objects misplaced (0.170%) 2023-10-09T11:05:49.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:49 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2565120987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:49.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:49 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 8026/148503 objects degraded (5.405%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:49.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:49 smithi125 ceph-mon[195507]: 11.6 scrub starts 2023-10-09T11:05:49.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:49 smithi125 ceph-mon[200197]: pgmap v1249: 129 pgs: 6 active+recovery_wait+degraded, 6 unknown, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 22 MiB/s wr, 354 op/s; 8026/148503 objects degraded (5.405%); 252/148503 objects misplaced (0.170%) 2023-10-09T11:05:49.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:49 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2565120987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:49.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:49 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 8026/148503 objects degraded (5.405%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:49.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:49 smithi125 ceph-mon[200197]: 11.6 scrub starts 2023-10-09T11:05:49.985 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 41 16 55328 55312 84.2601 95.5625 0.0140384 0.0118633 2023-10-09T11:05:49.996 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:50.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:50.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:50.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:50.161 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:50.195 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:50.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:50.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:50.318 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:05:50.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:50 smithi125 ceph-mon[195507]: 11.16 scrub starts 2023-10-09T11:05:50.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:50 smithi125 ceph-mon[200197]: 11.16 scrub starts 2023-10-09T11:05:50.383 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:50.408 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:05:50.408 INFO:teuthology.orchestra.run.smithi155.stdout: "success": "" 2023-10-09T11:05:50.408 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:05:50.414 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T11:05:50.418 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:05:50.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:50.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:05:50.422 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:50 smithi155 ceph-mon[184854]: 11.16 scrub starts 2023-10-09T11:05:50.523 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:50.588 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:50.985 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 42 16 56665 56649 84.2433 83.5625 0.00632804 0.0118649 2023-10-09T11:05:51.214 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:51 smithi155 ceph-mon[184854]: pgmap v1250: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+degraded, 116 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s; 6769/155889 objects degraded (4.342%); 795/155889 objects misplaced (0.510%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T11:05:51.537 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:51 smithi125 ceph-mon[195507]: pgmap v1250: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+degraded, 116 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s; 6769/155889 objects degraded (4.342%); 795/155889 objects misplaced (0.510%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T11:05:51.538 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:51 smithi125 ceph-mon[200197]: pgmap v1250: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+degraded, 116 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s; 6769/155889 objects degraded (4.342%); 795/155889 objects misplaced (0.510%); 7.5 MiB/s, 119 objects/s recovering 2023-10-09T11:05:51.538 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:05:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:05:51] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.43.0" 2023-10-09T11:05:51.947 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:51.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:51.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:51.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 43 16 58157 58141 84.449 93.25 0.0119752 0.011835 2023-10-09T11:05:52.013 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:52.060 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:52.105 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:52.135 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:52.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:52.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:52.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:52.292 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:52.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:52.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:52.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:52.582 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:52.681 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:52.885 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:52.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:52.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:52.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 44 15 59407 59392 84.3066 78.1875 0.013817 0.0118566 2023-10-09T11:05:52.991 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:53.027 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:53.058 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:53.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:53.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:53.165 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:53.200 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:05:53.206 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:53.229 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:05:53.237 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:53.240 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:53 smithi125 ceph-mon[195507]: pgmap v1251: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+degraded, 116 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s; 6769/163743 objects degraded (4.134%); 795/163743 objects misplaced (0.486%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T11:05:53.240 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:53 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2239614632' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:05:53.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:53 smithi125 ceph-mon[200197]: pgmap v1251: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+degraded, 116 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s; 6769/163743 objects degraded (4.134%); 795/163743 objects misplaced (0.486%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T11:05:53.240 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:53 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2239614632' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:05:53.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:53.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:53.342 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:53.367 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:53 smithi155 ceph-mon[184854]: pgmap v1251: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+degraded, 116 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s; 6769/163743 objects degraded (4.134%); 795/163743 objects misplaced (0.486%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T11:05:53.367 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:53 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2239614632' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:05:53.369 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:53.387 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:53.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:53.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:53.492 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:53.540 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:53.565 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:53.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:53.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:53.671 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:53.716 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:53.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 45 16 60460 60444 83.8943 65.75 0.0103175 0.0119145 2023-10-09T11:05:54.306 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:54.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:54.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:54.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:54 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:05:54.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:54 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2239614632' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:05:54.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:54 smithi125 ceph-mon[195507]: osdmap e512: 8 total, 8 up, 8 in 2023-10-09T11:05:54.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:54 smithi125 ceph-mon[195507]: failed to encode map e512 with expected crc 2023-10-09T11:05:54.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:54 smithi125 ceph-mon[195507]: failed to encode map e512 with expected crc 2023-10-09T11:05:54.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:54 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 6769/163743 objects degraded (4.134%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:54.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:54 smithi125 ceph-mon[195507]: pgmap v1253: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+degraded, 116 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 6769/176493 objects degraded (3.835%); 351/176493 objects misplaced (0.199%); 9.1 MiB/s, 144 objects/s recovering 2023-10-09T11:05:54.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:54 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:05:54.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:54 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2239614632' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:05:54.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:54 smithi125 ceph-mon[200197]: osdmap e512: 8 total, 8 up, 8 in 2023-10-09T11:05:54.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:54 smithi125 ceph-mon[200197]: failed to encode map e512 with expected crc 2023-10-09T11:05:54.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:54 smithi125 ceph-mon[200197]: failed to encode map e512 with expected crc 2023-10-09T11:05:54.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:54 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 6769/163743 objects degraded (4.134%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:54.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:54 smithi125 ceph-mon[200197]: pgmap v1253: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+degraded, 116 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 6769/176493 objects degraded (3.835%); 351/176493 objects misplaced (0.199%); 9.1 MiB/s, 144 objects/s recovering 2023-10-09T11:05:54.339 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:05:54.339 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:05:54.340 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T11:05:54.340 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T11:05:54.340 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:05:54.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:54.526 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:54.687 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:54.706 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:54.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:54.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:54.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:54 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:05:54.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:54 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2239614632' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:05:54.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:54 smithi155 ceph-mon[184854]: osdmap e512: 8 total, 8 up, 8 in 2023-10-09T11:05:54.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:54 smithi155 ceph-mon[184854]: failed to encode map e512 with expected crc 2023-10-09T11:05:54.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:54 smithi155 ceph-mon[184854]: failed to encode map e512 with expected crc 2023-10-09T11:05:54.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:54 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 6769/163743 objects degraded (4.134%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:54.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:54 smithi155 ceph-mon[184854]: pgmap v1253: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+degraded, 116 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 6769/176493 objects degraded (3.835%); 351/176493 objects misplaced (0.199%); 9.1 MiB/s, 144 objects/s recovering 2023-10-09T11:05:54.814 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:54.867 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T11:05:54.879 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:54.955 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:54.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:54.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:54.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 46 16 61794 61778 83.8829 83.375 0.0147294 0.0119161 2023-10-09T11:05:55.064 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:55.113 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:55.452 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:55.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:55.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:55.479 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:55.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:05:55.596 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:55.666 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:05:55.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:55.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:05:55.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:55.816 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:55.840 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 7, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 5, 6, 7, 1, 3] 2023-10-09T11:05:55.840 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:05:55.840 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T11:05:55.840 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T11:05:55.841 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T11:05:55.841 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T11:05:55.848 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:55.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:55.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:55.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:05:55.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 47 16 63058 63042 83.7789 79 0.0140364 0.0119304 2023-10-09T11:05:56.010 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:56.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:05:56.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:56.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:05:56.147 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:56.165 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:05:56.195 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:56.207 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:05:56.243 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:05:56.262 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:05:56.285 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:05:56.319 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:05:56.360 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:05:56.404 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:05:56.413 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:05:56.434 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:05:56.449 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:05:56.449 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:05:56.449 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:05:56.450 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:05:56.450 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:05:56.450 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:05:56.450 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:05:56.450 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:05:56.611 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:56.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:56.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:56.717 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:05:56.765 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:56.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 48 16 64229 64213 83.5583 73.1875 0.00857108 0.0119595 2023-10-09T11:05:57.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:56 smithi125 ceph-mon[195507]: pgmap v1254: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+degraded, 116 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 6769/176493 objects degraded (3.835%); 351/176493 objects misplaced (0.199%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T11:05:57.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4259473603' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:57.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:56 smithi125 ceph-mon[200197]: pgmap v1254: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+degraded, 116 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 6769/176493 objects degraded (3.835%); 351/176493 objects misplaced (0.199%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T11:05:57.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4259473603' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:57.127 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:56 smithi155 ceph-mon[184854]: pgmap v1254: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+degraded, 116 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 6769/176493 objects degraded (3.835%); 351/176493 objects misplaced (0.199%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T11:05:57.127 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4259473603' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:05:57.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:57.308 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:05:57.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:05:57.327 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:57.410 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:57.439 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:57.759 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:57.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:57.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:57.865 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:57.899 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:57.924 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:57 smithi125 ceph-mon[200197]: osdmap e513: 8 total, 8 up, 8 in 2023-10-09T11:05:57.924 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:57 smithi125 ceph-mon[200197]: failed to encode map e513 with expected crc 2023-10-09T11:05:57.924 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:57 smithi125 ceph-mon[200197]: failed to encode map e513 with expected crc 2023-10-09T11:05:57.924 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:57 smithi125 ceph-mon[200197]: failed to encode map e513 with expected crc 2023-10-09T11:05:57.924 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:57 smithi125 ceph-mon[195507]: osdmap e513: 8 total, 8 up, 8 in 2023-10-09T11:05:57.924 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:57 smithi125 ceph-mon[195507]: failed to encode map e513 with expected crc 2023-10-09T11:05:57.925 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:57 smithi125 ceph-mon[195507]: failed to encode map e513 with expected crc 2023-10-09T11:05:57.925 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:57 smithi125 ceph-mon[195507]: failed to encode map e513 with expected crc 2023-10-09T11:05:57.947 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:57.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:57.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:57.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:57 smithi155 ceph-mon[184854]: osdmap e513: 8 total, 8 up, 8 in 2023-10-09T11:05:57.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:57 smithi155 ceph-mon[184854]: failed to encode map e513 with expected crc 2023-10-09T11:05:57.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:57 smithi155 ceph-mon[184854]: failed to encode map e513 with expected crc 2023-10-09T11:05:57.972 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:57 smithi155 ceph-mon[184854]: failed to encode map e513 with expected crc 2023-10-09T11:05:57.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 49 16 64503 64487 82.2032 17.125 0.0146316 0.0121596 2023-10-09T11:05:58.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:58.103 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:58.158 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:05:58.158 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:05:58.181 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:58.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:58.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:58.250 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:05:58.250 INFO:teuthology.orchestra.run.smithi155.stdout: "success": "" 2023-10-09T11:05:58.250 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:05:58.286 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:58.336 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:58.569 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:05:58.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:58.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:05:58.674 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:05:58.726 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:58.766 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:05:58.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:58.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:05:58.779 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T11:05:58.876 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:58.931 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:58.987 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 50 16 65772 65756 82.1452 79.3125 0.0111537 0.0121698 2023-10-09T11:05:59.015 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:59.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:59.031 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[195507]: failed to encode map e513 with expected crc 2023-10-09T11:05:59.031 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[195507]: pgmap v1256: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+degraded, 116 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 6769/183645 objects degraded (3.686%); 351/183645 objects misplaced (0.191%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T11:05:59.031 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[195507]: failed to encode map e514 with expected crc 2023-10-09T11:05:59.032 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[195507]: failed to encode map e514 with expected crc 2023-10-09T11:05:59.032 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[195507]: failed to encode map e514 with expected crc 2023-10-09T11:05:59.032 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[195507]: failed to encode map e514 with expected crc 2023-10-09T11:05:59.032 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[195507]: osdmap e514: 8 total, 8 up, 8 in 2023-10-09T11:05:59.032 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 6769/183645 objects degraded (3.686%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:59.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[200197]: failed to encode map e513 with expected crc 2023-10-09T11:05:59.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[200197]: pgmap v1256: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+degraded, 116 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 6769/183645 objects degraded (3.686%); 351/183645 objects misplaced (0.191%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T11:05:59.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[200197]: failed to encode map e514 with expected crc 2023-10-09T11:05:59.032 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[200197]: failed to encode map e514 with expected crc 2023-10-09T11:05:59.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[200197]: failed to encode map e514 with expected crc 2023-10-09T11:05:59.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[200197]: failed to encode map e514 with expected crc 2023-10-09T11:05:59.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[200197]: osdmap e514: 8 total, 8 up, 8 in 2023-10-09T11:05:59.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:58 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 6769/183645 objects degraded (3.686%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:59.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:59.134 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:05:59.200 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:58 smithi155 ceph-mon[184854]: failed to encode map e513 with expected crc 2023-10-09T11:05:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:58 smithi155 ceph-mon[184854]: pgmap v1256: 129 pgs: 1 active+recovering+undersized+remapped, 12 active+recovery_wait+degraded, 116 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 6769/183645 objects degraded (3.686%); 351/183645 objects misplaced (0.191%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T11:05:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:58 smithi155 ceph-mon[184854]: failed to encode map e514 with expected crc 2023-10-09T11:05:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:58 smithi155 ceph-mon[184854]: failed to encode map e514 with expected crc 2023-10-09T11:05:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:58 smithi155 ceph-mon[184854]: failed to encode map e514 with expected crc 2023-10-09T11:05:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:58 smithi155 ceph-mon[184854]: failed to encode map e514 with expected crc 2023-10-09T11:05:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:58 smithi155 ceph-mon[184854]: osdmap e514: 8 total, 8 up, 8 in 2023-10-09T11:05:59.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:58 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 6769/183645 objects degraded (3.686%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T11:05:59.228 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:05:59.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:59.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:05:59.250 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:05:59.337 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:05:59.388 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:59.417 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:05:59.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:05:59.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:05:59.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:05:59.587 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:05:59.690 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:05:59.988 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 51 16 66937 66921 81.961 72.8125 0.0201598 0.0121963 2023-10-09T11:06:00.031 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:06:00.032 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:00.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:00.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:59 smithi125 ceph-mon[200197]: failed to encode map e514 with expected crc 2023-10-09T11:06:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:59 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3577882622' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:00.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:05:59 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:00.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:59 smithi125 ceph-mon[195507]: failed to encode map e514 with expected crc 2023-10-09T11:06:00.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:59 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3577882622' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:00.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:05:59 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:00.138 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:00.180 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:00.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:59 smithi155 ceph-mon[184854]: failed to encode map e514 with expected crc 2023-10-09T11:06:00.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:59 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3577882622' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:00.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:05:59 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:00.923 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[195507]: pgmap v1258: 129 pgs: 12 active+recovery_wait+degraded, 117 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 6769/192984 objects degraded (3.508%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T11:06:00.923 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:00.923 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[195507]: failed to encode map e515 with expected crc 2023-10-09T11:06:00.924 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[195507]: failed to encode map e515 with expected crc 2023-10-09T11:06:00.924 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:06:00.924 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[195507]: osdmap e515: 8 total, 8 up, 8 in 2023-10-09T11:06:00.924 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[195507]: failed to encode map e515 with expected crc 2023-10-09T11:06:00.924 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:00.924 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:06:00.988 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 52 16 68318 68302 82.0445 86.3125 0.00220376 0.0121815 2023-10-09T11:06:01.217 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:06:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:06:01] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2023-10-09T11:06:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[200197]: pgmap v1258: 129 pgs: 12 active+recovery_wait+degraded, 117 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 6769/192984 objects degraded (3.508%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T11:06:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[200197]: failed to encode map e515 with expected crc 2023-10-09T11:06:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[200197]: failed to encode map e515 with expected crc 2023-10-09T11:06:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:06:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[200197]: osdmap e515: 8 total, 8 up, 8 in 2023-10-09T11:06:01.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[200197]: failed to encode map e515 with expected crc 2023-10-09T11:06:01.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:01.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:06:01.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:00 smithi155 ceph-mon[184854]: pgmap v1258: 129 pgs: 12 active+recovery_wait+degraded, 117 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 6769/192984 objects degraded (3.508%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T11:06:01.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:00 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:01.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:00 smithi155 ceph-mon[184854]: failed to encode map e515 with expected crc 2023-10-09T11:06:01.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:00 smithi155 ceph-mon[184854]: failed to encode map e515 with expected crc 2023-10-09T11:06:01.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:00 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:06:01.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:00 smithi155 ceph-mon[184854]: osdmap e515: 8 total, 8 up, 8 in 2023-10-09T11:06:01.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:00 smithi155 ceph-mon[184854]: failed to encode map e515 with expected crc 2023-10-09T11:06:01.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:01.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:06:01.245 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:01.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:01.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:01.351 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:01.416 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:01.531 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:01.548 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:01.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:01.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:01.654 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:01.714 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:01.792 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:01.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:01.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:01.899 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:01.954 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:01.988 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 53 16 69560 69544 81.961 77.625 0.00391236 0.0121964 2023-10-09T11:06:02.039 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:02.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:02.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:02.119 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:06:02.119 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:06:02.147 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:02.201 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:02.533 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:02.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:02.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:02.647 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:02.695 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:02.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T11:06:02.822 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:02.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:02.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:02.926 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:02.960 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:02.989 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 54 16 70766 70750 81.839 75.375 0.0138305 0.0122141 2023-10-09T11:06:03.006 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:03.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:03.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:03.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:02 smithi125 ceph-mon[195507]: pgmap v1260: 129 pgs: 11 active+recovery_wait+degraded, 118 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 6107/199014 objects degraded (3.069%); 11 MiB/s, 168 objects/s recovering 2023-10-09T11:06:03.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:02 smithi125 ceph-mon[200197]: pgmap v1260: 129 pgs: 11 active+recovery_wait+degraded, 118 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 6107/199014 objects degraded (3.069%); 11 MiB/s, 168 objects/s recovering 2023-10-09T11:06:03.110 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:03.151 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:03.173 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:03.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:03.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:03.185 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:06:03.193 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:03.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:02 smithi155 ceph-mon[184854]: pgmap v1260: 129 pgs: 11 active+recovery_wait+degraded, 118 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 6107/199014 objects degraded (3.069%); 11 MiB/s, 168 objects/s recovering 2023-10-09T11:06:03.225 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:06:03.263 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:06:03.278 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:03.298 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:06:03.328 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:03.358 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:03.359 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:06:03.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:03.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:03.385 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:06:03.413 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:06:03.447 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:06:03.451 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:06:03.451 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:06:03.451 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:06:03.452 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:06:03.452 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:06:03.452 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:06:03.452 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:06:03.452 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:06:03.467 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:03.512 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:03.650 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 7, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 5, 6, 7, 1, 3] 2023-10-09T11:06:03.650 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:06:03.650 INFO:tasks.thrashosds.thrasher:Reweighting osd 7 to 0.2437673178265536 2023-10-09T11:06:03.650 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 0.2437673178265536 2023-10-09T11:06:03.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:06:03.707 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:03.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:03.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:03.818 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:03.850 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:03.877 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:03.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:03.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:03.983 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:03.989 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 55 16 71851 71835 81.5833 67.8125 0.0151344 0.0122514 2023-10-09T11:06:04.065 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:04.151 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:04.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:04.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:04.183 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:04.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2935767916' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:04.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:03 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 6107/199014 objects degraded (3.069%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:04.255 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:04.299 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:04.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2935767916' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:04.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:03 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 6107/199014 objects degraded (3.069%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:04.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2935767916' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:04.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:03 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 6107/199014 objects degraded (3.069%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:04.327 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:04.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:04.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:04.436 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:04.480 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:04.513 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:04.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:04.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:04.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:04.666 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:04.713 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:06:04.713 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:06:04.713 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T11:06:04.713 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T11:06:04.713 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:06:04.732 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:04.769 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:04.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:04.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:04.878 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:04.921 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:04.951 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:04.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:04.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:04.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:04 smithi125 ceph-mon[195507]: pgmap v1261: 129 pgs: 1 active+recovering, 10 active+recovery_wait+degraded, 118 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 5424/210630 objects degraded (2.575%); 234/210630 objects misplaced (0.111%); 13 MiB/s, 210 objects/s recovering 2023-10-09T11:06:04.978 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:04 smithi125 ceph-mon[200197]: pgmap v1261: 129 pgs: 1 active+recovering, 10 active+recovery_wait+degraded, 118 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 5424/210630 objects degraded (2.575%); 234/210630 objects misplaced (0.111%); 13 MiB/s, 210 objects/s recovering 2023-10-09T11:06:04.989 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 56 16 72902 72886 81.2995 65.6875 0.0261907 0.0122953 2023-10-09T11:06:05.060 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:05.099 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:05.131 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:05.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:05.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:05.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:04 smithi155 ceph-mon[184854]: pgmap v1261: 129 pgs: 1 active+recovering, 10 active+recovery_wait+degraded, 118 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 5424/210630 objects degraded (2.575%); 234/210630 objects misplaced (0.111%); 13 MiB/s, 210 objects/s recovering 2023-10-09T11:06:05.241 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:05.295 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:05.326 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:05.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:05.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:05.431 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:05.483 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:05.626 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T11:06:05.717 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:05.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:05.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:05.823 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:05.863 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:05.894 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:05.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:05.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:05.990 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 57 16 74042 74026 81.1222 71.25 0.0196751 0.0123212 2023-10-09T11:06:05.998 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:05.999 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:06:06.036 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:06.068 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:06.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:06.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:06.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:06.270 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:06.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:06.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:06.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:06.397 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:06.431 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:06.461 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:06.500 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:06.534 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:06.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:06.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:06.639 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:06.681 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:06.990 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 58 16 75095 75079 80.8583 65.8125 0.0170447 0.0123619 2023-10-09T11:06:07.012 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:07 smithi125 ceph-mon[195507]: pgmap v1262: 129 pgs: 1 active+recovering, 10 active+recovery_wait+degraded, 118 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 5424/210630 objects degraded (2.575%); 234/210630 objects misplaced (0.111%); 11 MiB/s, 182 objects/s recovering 2023-10-09T11:06:07.012 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:07 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3433617294' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.2437673178265536}]: dispatch 2023-10-09T11:06:07.012 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:07 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3021513001' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:06:07.012 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:07 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:06:07.019 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:06:07.024 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.7 to 0.243767 (3e67) 2023-10-09T11:06:07.181 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:07.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:07.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:07.284 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:07.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:07 smithi125 ceph-mon[200197]: pgmap v1262: 129 pgs: 1 active+recovering, 10 active+recovery_wait+degraded, 118 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 5424/210630 objects degraded (2.575%); 234/210630 objects misplaced (0.111%); 11 MiB/s, 182 objects/s recovering 2023-10-09T11:06:07.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:07 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3433617294' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.2437673178265536}]: dispatch 2023-10-09T11:06:07.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:07 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3021513001' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:06:07.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:07 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:06:07.313 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:07 smithi155 ceph-mon[184854]: pgmap v1262: 129 pgs: 1 active+recovering, 10 active+recovery_wait+degraded, 118 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 5424/210630 objects degraded (2.575%); 234/210630 objects misplaced (0.111%); 11 MiB/s, 182 objects/s recovering 2023-10-09T11:06:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:07 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3433617294' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.2437673178265536}]: dispatch 2023-10-09T11:06:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:07 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3021513001' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:06:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:07 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:06:07.990 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 59 16 75168 75152 79.5658 4.5625 0.00306364 0.0123639 2023-10-09T11:06:08.036 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:08.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:08.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:08.141 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:08.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:08.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3433617294' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.2437673178265536}]': finished 2023-10-09T11:06:08.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:06:08.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[195507]: osdmap e516: 8 total, 8 up, 8 in 2023-10-09T11:06:08.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[195507]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[195507]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.167 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[195507]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.168 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[195507]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.168 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[195507]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.168 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[195507]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.168 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[195507]: osdmap e517: 8 total, 8 up, 8 in 2023-10-09T11:06:08.168 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:08.168 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3433617294' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.2437673178265536}]': finished 2023-10-09T11:06:08.168 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:06:08.168 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[200197]: osdmap e516: 8 total, 8 up, 8 in 2023-10-09T11:06:08.168 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[200197]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.169 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[200197]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.169 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[200197]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.169 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[200197]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.169 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[200197]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.169 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[200197]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.169 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:08 smithi125 ceph-mon[200197]: osdmap e517: 8 total, 8 up, 8 in 2023-10-09T11:06:08.169 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:08.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:08.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:08.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:08 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:08 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3433617294' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.2437673178265536}]': finished 2023-10-09T11:06:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:08 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:06:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:08 smithi155 ceph-mon[184854]: osdmap e516: 8 total, 8 up, 8 in 2023-10-09T11:06:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:08 smithi155 ceph-mon[184854]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:08 smithi155 ceph-mon[184854]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:08 smithi155 ceph-mon[184854]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:08 smithi155 ceph-mon[184854]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:08 smithi155 ceph-mon[184854]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:08 smithi155 ceph-mon[184854]: failed to encode map e516 with expected crc 2023-10-09T11:06:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:08 smithi155 ceph-mon[184854]: osdmap e517: 8 total, 8 up, 8 in 2023-10-09T11:06:08.545 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:08.575 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:08.652 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:06:08.653 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T11:06:08.653 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T11:06:08.653 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:06:08.653 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.653 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122010 9.6 9:6f14ab77:::benchmark_data_smithi125_363789_object11711:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:06:08.653 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.136716+0000", 2023-10-09T11:06:08.653 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 420.51347544700002, 2023-10-09T11:06:08.653 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0621660900000001, 2023-10-09T11:06:08.653 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.653 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:06:08.653 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.654 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:06:08.654 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:06:08.654 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122010 2023-10-09T11:06:08.654 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.654 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.654 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.654 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.654 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136716+0000", 2023-10-09T11:06:08.654 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.654 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.654 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.654 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.655 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136716+0000", 2023-10-09T11:06:08.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.655 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.655 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.655 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136715+0000", 2023-10-09T11:06:08.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:06:08.655 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.655 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.655 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136721+0000", 2023-10-09T11:06:08.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7660000000000004e-06 2023-10-09T11:06:08.655 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.656 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.656 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.656 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136731+0000", 2023-10-09T11:06:08.656 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0898e-05 2023-10-09T11:06:08.656 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.656 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.656 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.656 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136739+0000", 2023-10-09T11:06:08.656 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8450000000000003e-06 2023-10-09T11:06:08.656 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.656 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.656 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.140487+0000", 2023-10-09T11:06:08.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037476330000000002 2023-10-09T11:06:08.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.657 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.140607+0000", 2023-10-09T11:06:08.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012029299999999999 2023-10-09T11:06:08.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.657 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T11:06:08.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.140726+0000", 2023-10-09T11:06:08.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119057 2023-10-09T11:06:08.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.658 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.150456+0000", 2023-10-09T11:06:08.658 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.009729283 2023-10-09T11:06:08.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.658 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.153186+0000", 2023-10-09T11:06:08.658 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00273077 2023-10-09T11:06:08.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.659 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.659 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162778+0000", 2023-10-09T11:06:08.659 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0095913500000000002 2023-10-09T11:06:08.659 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.659 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.659 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.659 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162790+0000", 2023-10-09T11:06:08.659 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.221e-05 2023-10-09T11:06:08.659 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.659 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.659 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.659 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.198810+0000", 2023-10-09T11:06:08.659 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.036020156 2023-10-09T11:06:08.660 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.660 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.660 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.660 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.198882+0000", 2023-10-09T11:06:08.660 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1717999999999993e-05 2023-10-09T11:06:08.660 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.660 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.660 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.660 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.660 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.660 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122013 9.6 9:6b7eeaa4:::benchmark_data_smithi125_363789_object11714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:06:08.660 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.138283+0000", 2023-10-09T11:06:08.661 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 420.51190809399998, 2023-10-09T11:06:08.661 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.060936141, 2023-10-09T11:06:08.661 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.661 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:06:08.661 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.661 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:06:08.661 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:06:08.661 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122013 2023-10-09T11:06:08.661 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.661 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.661 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.661 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.661 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138283+0000", 2023-10-09T11:06:08.662 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.662 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.662 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.662 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.662 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138283+0000", 2023-10-09T11:06:08.662 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.662 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.662 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.662 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.662 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138282+0000", 2023-10-09T11:06:08.662 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:06:08.663 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.663 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.663 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.663 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138288+0000", 2023-10-09T11:06:08.663 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5029999999999997e-06 2023-10-09T11:06:08.663 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.663 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.663 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.663 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138300+0000", 2023-10-09T11:06:08.663 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2357e-05 2023-10-09T11:06:08.663 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.663 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.664 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.664 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138309+0000", 2023-10-09T11:06:08.664 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.499e-06 2023-10-09T11:06:08.664 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.664 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.664 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.664 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.144017+0000", 2023-10-09T11:06:08.664 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0057070339999999997 2023-10-09T11:06:08.664 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.664 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.664 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.664 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.144165+0000", 2023-10-09T11:06:08.665 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000148345 2023-10-09T11:06:08.665 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.665 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.665 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T11:06:08.665 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.144293+0000", 2023-10-09T11:06:08.665 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000128069 2023-10-09T11:06:08.665 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.665 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.665 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.665 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.152647+0000", 2023-10-09T11:06:08.665 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083540590000000005 2023-10-09T11:06:08.665 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.666 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.666 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.666 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.153226+0000", 2023-10-09T11:06:08.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00057917199999999998 2023-10-09T11:06:08.666 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.666 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.666 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.666 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162825+0000", 2023-10-09T11:06:08.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0095991010000000005 2023-10-09T11:06:08.666 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.666 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.666 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.666 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162833+0000", 2023-10-09T11:06:08.667 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3429999999999998e-06 2023-10-09T11:06:08.667 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.667 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.667 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.667 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199105+0000", 2023-10-09T11:06:08.667 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0362724169999999 2023-10-09T11:06:08.667 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.667 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.667 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.667 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199219+0000", 2023-10-09T11:06:08.667 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000114141 2023-10-09T11:06:08.667 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.668 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.668 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.668 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.668 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.668 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122028 9.6 9:6992c4ac:::benchmark_data_smithi125_363789_object11729:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:06:08.668 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.154935+0000", 2023-10-09T11:06:08.668 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 420.49525604899998, 2023-10-09T11:06:08.668 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.044381056, 2023-10-09T11:06:08.668 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.668 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:06:08.668 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.668 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:06:08.668 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:06:08.669 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122028 2023-10-09T11:06:08.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.669 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.669 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.669 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154935+0000", 2023-10-09T11:06:08.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.669 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.669 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154935+0000", 2023-10-09T11:06:08.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.670 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.670 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154934+0000", 2023-10-09T11:06:08.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:06:08.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.670 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.670 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154941+0000", 2023-10-09T11:06:08.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9600000000000003e-06 2023-10-09T11:06:08.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.670 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.670 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154956+0000", 2023-10-09T11:06:08.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4976e-05 2023-10-09T11:06:08.671 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154964+0000", 2023-10-09T11:06:08.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8639999999999996e-06 2023-10-09T11:06:08.671 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.155012+0000", 2023-10-09T11:06:08.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7960999999999997e-05 2023-10-09T11:06:08.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.155124+0000", 2023-10-09T11:06:08.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011204 2023-10-09T11:06:08.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T11:06:08.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.155237+0000", 2023-10-09T11:06:08.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011364799999999999 2023-10-09T11:06:08.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162936+0000", 2023-10-09T11:06:08.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.007698369 2023-10-09T11:06:08.673 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.673 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.673 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162945+0000", 2023-10-09T11:06:08.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6039999999999999e-06 2023-10-09T11:06:08.673 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.673 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.673 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199250+0000", 2023-10-09T11:06:08.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.036304248 2023-10-09T11:06:08.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.674 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.674 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199316+0000", 2023-10-09T11:06:08.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6549000000000003e-05 2023-10-09T11:06:08.674 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.674 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.674 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.674 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122037 9.6 9:68399f49:::benchmark_data_smithi125_363789_object11738:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:06:08.675 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.181454+0000", 2023-10-09T11:06:08.675 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 420.46873677899998, 2023-10-09T11:06:08.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.032324094, 2023-10-09T11:06:08.675 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.675 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:08.675 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.675 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:06:08.675 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:06:08.675 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122037 2023-10-09T11:06:08.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.675 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.675 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.676 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.676 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181454+0000", 2023-10-09T11:06:08.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.676 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.676 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181454+0000", 2023-10-09T11:06:08.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.676 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181453+0000", 2023-10-09T11:06:08.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:06:08.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.677 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181461+0000", 2023-10-09T11:06:08.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4800000000000001e-06 2023-10-09T11:06:08.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.677 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181478+0000", 2023-10-09T11:06:08.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7215e-05 2023-10-09T11:06:08.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.678 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.678 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181488+0000", 2023-10-09T11:06:08.678 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5240000000000006e-06 2023-10-09T11:06:08.678 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.678 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.678 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181572+0000", 2023-10-09T11:06:08.678 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3784999999999998e-05 2023-10-09T11:06:08.678 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.678 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181581+0000", 2023-10-09T11:06:08.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7249999999999992e-06 2023-10-09T11:06:08.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199341+0000", 2023-10-09T11:06:08.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.017760062 2023-10-09T11:06:08.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199463+0000", 2023-10-09T11:06:08.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012115799999999999 2023-10-09T11:06:08.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5", 2023-10-09T11:06:08.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199575+0000", 2023-10-09T11:06:08.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000112404 2023-10-09T11:06:08.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.207005+0000", 2023-10-09T11:06:08.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074305350000000003 2023-10-09T11:06:08.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.213644+0000", 2023-10-09T11:06:08.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066387779999999997 2023-10-09T11:06:08.681 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.681 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:06:08.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.213695+0000", 2023-10-09T11:06:08.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0748999999999998e-05 2023-10-09T11:06:08.681 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.681 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.213778+0000", 2023-10-09T11:06:08.682 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3434000000000001e-05 2023-10-09T11:06:08.682 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.682 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.682 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.682 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122039 9.6 9:679871c1:::benchmark_data_smithi125_363789_object11740:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:06:08.682 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:06:08.682 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 420.46867581599997, 2023-10-09T11:06:08.682 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0328282099999999, 2023-10-09T11:06:08.682 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.682 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:08.683 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.683 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:06:08.683 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:06:08.683 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122039 2023-10-09T11:06:08.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.683 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:06:08.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.684 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:06:08.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.684 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.684 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.684 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.684 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:06:08.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:08.684 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.684 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.684 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.684 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181518+0000", 2023-10-09T11:06:08.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4350000000000001e-06 2023-10-09T11:06:08.685 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.685 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.685 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181525+0000", 2023-10-09T11:06:08.685 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5359999999999998e-06 2023-10-09T11:06:08.685 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.685 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.685 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181530+0000", 2023-10-09T11:06:08.685 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9019999999999998e-06 2023-10-09T11:06:08.685 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181601+0000", 2023-10-09T11:06:08.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1168999999999996e-05 2023-10-09T11:06:08.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181606+0000", 2023-10-09T11:06:08.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1919999999999996e-06 2023-10-09T11:06:08.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.203647+0000", 2023-10-09T11:06:08.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.022041073 2023-10-09T11:06:08.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.203754+0000", 2023-10-09T11:06:08.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000106529 2023-10-09T11:06:08.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5", 2023-10-09T11:06:08.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.203828+0000", 2023-10-09T11:06:08.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4286e-05 2023-10-09T11:06:08.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.688 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.210714+0000", 2023-10-09T11:06:08.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0068858050000000001 2023-10-09T11:06:08.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.688 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.214203+0000", 2023-10-09T11:06:08.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003489242 2023-10-09T11:06:08.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.688 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:06:08.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.214268+0000", 2023-10-09T11:06:08.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5104000000000006e-05 2023-10-09T11:06:08.689 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.689 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.689 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.214344+0000", 2023-10-09T11:06:08.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5471999999999998e-05 2023-10-09T11:06:08.689 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.689 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.689 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.689 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.689 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:133037 9.b 9.da5dd31b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e362)", 2023-10-09T11:06:08.690 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:20.474469+0000", 2023-10-09T11:06:08.690 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 408.175722329, 2023-10-09T11:06:08.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97365184000000005, 2023-10-09T11:06:08.690 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.690 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:06:08.690 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.690 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:06:08.690 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:06:08.690 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133037 2023-10-09T11:06:08.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.690 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474469+0000", 2023-10-09T11:06:08.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474469+0000", 2023-10-09T11:06:08.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474467+0000", 2023-10-09T11:06:08.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:08.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474475+0000", 2023-10-09T11:06:08.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6009999999999997e-06 2023-10-09T11:06:08.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474493+0000", 2023-10-09T11:06:08.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8150000000000001e-05 2023-10-09T11:06:08.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474502+0000", 2023-10-09T11:06:08.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1400000000000006e-06 2023-10-09T11:06:08.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474569+0000", 2023-10-09T11:06:08.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6849999999999999e-05 2023-10-09T11:06:08.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474579+0000", 2023-10-09T11:06:08.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9369999999999999e-06 2023-10-09T11:06:08.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448111+0000", 2023-10-09T11:06:08.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97353240399999996 2023-10-09T11:06:08.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448121+0000", 2023-10-09T11:06:08.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1679999999999993e-06 2023-10-09T11:06:08.694 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.694 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.695 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.695 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.697 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.697 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:133044 9.b 9.5473ef4b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e362)", 2023-10-09T11:06:08.697 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:20.482185+0000", 2023-10-09T11:06:08.697 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 408.16800595500001, 2023-10-09T11:06:08.697 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96595614900000004, 2023-10-09T11:06:08.698 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.698 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:06:08.698 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.698 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:06:08.698 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:06:08.698 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133044 2023-10-09T11:06:08.698 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.698 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.698 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.698 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.698 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482185+0000", 2023-10-09T11:06:08.699 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.699 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.699 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.699 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.699 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482185+0000", 2023-10-09T11:06:08.699 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.699 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.699 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.699 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.699 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482184+0000", 2023-10-09T11:06:08.699 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:08.699 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482193+0000", 2023-10-09T11:06:08.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0310000000000004e-06 2023-10-09T11:06:08.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482217+0000", 2023-10-09T11:06:08.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4677000000000001e-05 2023-10-09T11:06:08.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.701 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482228+0000", 2023-10-09T11:06:08.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.043e-05 2023-10-09T11:06:08.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.701 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.701 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482326+0000", 2023-10-09T11:06:08.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8241999999999999e-05 2023-10-09T11:06:08.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.701 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.701 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482334+0000", 2023-10-09T11:06:08.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7910000000000005e-06 2023-10-09T11:06:08.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.702 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448135+0000", 2023-10-09T11:06:08.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96580099799999997 2023-10-09T11:06:08.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.702 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448141+0000", 2023-10-09T11:06:08.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5719999999999999e-06 2023-10-09T11:06:08.702 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.702 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.703 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.703 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.703 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39381 11.d 11:b5913833:::benchmark_data_smithi125_439543_object39380:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:06:08.703 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.310294+0000", 2023-10-09T11:06:08.703 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 34.339896983999999, 2023-10-09T11:06:08.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0048304729999999, 2023-10-09T11:06:08.703 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.703 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:08.703 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.703 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:06:08.703 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:08.704 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39381 2023-10-09T11:06:08.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.704 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310294+0000", 2023-10-09T11:06:08.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310294+0000", 2023-10-09T11:06:08.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310292+0000", 2023-10-09T11:06:08.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:06:08.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310321+0000", 2023-10-09T11:06:08.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8965000000000001e-05 2023-10-09T11:06:08.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.706 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310335+0000", 2023-10-09T11:06:08.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3407999999999999e-05 2023-10-09T11:06:08.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.706 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310343+0000", 2023-10-09T11:06:08.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1030000000000002e-06 2023-10-09T11:06:08.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310413+0000", 2023-10-09T11:06:08.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9822999999999999e-05 2023-10-09T11:06:08.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.707 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.707 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310428+0000", 2023-10-09T11:06:08.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5021999999999999e-05 2023-10-09T11:06:08.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.707 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.707 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.311384+0000", 2023-10-09T11:06:08.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0009558540000001 2023-10-09T11:06:08.708 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.708 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.311492+0000", 2023-10-09T11:06:08.708 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108618 2023-10-09T11:06:08.708 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:06:08.708 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.311595+0000", 2023-10-09T11:06:08.708 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000102431 2023-10-09T11:06:08.708 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.709 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315040+0000", 2023-10-09T11:06:08.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0034454049999999999 2023-10-09T11:06:08.709 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.709 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.709 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.709 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315049+0000", 2023-10-09T11:06:08.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7749999999999998e-06 2023-10-09T11:06:08.709 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.709 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.709 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.709 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315067+0000", 2023-10-09T11:06:08.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8595999999999999e-05 2023-10-09T11:06:08.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.710 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:06:08.710 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315098+0000", 2023-10-09T11:06:08.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0185999999999999e-05 2023-10-09T11:06:08.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.710 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.710 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315125+0000", 2023-10-09T11:06:08.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6982999999999999e-05 2023-10-09T11:06:08.710 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.710 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.711 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.711 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39392 11.d 11:b103d758:::benchmark_data_smithi125_439543_object39391:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:06:08.711 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.311770+0000", 2023-10-09T11:06:08.711 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 34.338420982999999, 2023-10-09T11:06:08.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0081084490000001, 2023-10-09T11:06:08.711 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.711 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:08.711 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.711 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:06:08.711 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:08.712 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39392 2023-10-09T11:06:08.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.712 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311770+0000", 2023-10-09T11:06:08.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311770+0000", 2023-10-09T11:06:08.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311769+0000", 2023-10-09T11:06:08.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:08.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311790+0000", 2023-10-09T11:06:08.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1759999999999998e-05 2023-10-09T11:06:08.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311800+0000", 2023-10-09T11:06:08.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6430000000000007e-06 2023-10-09T11:06:08.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311806+0000", 2023-10-09T11:06:08.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0619999999999997e-06 2023-10-09T11:06:08.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311962+0000", 2023-10-09T11:06:08.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000156312 2023-10-09T11:06:08.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311972+0000", 2023-10-09T11:06:08.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0098000000000001e-05 2023-10-09T11:06:08.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.312904+0000", 2023-10-09T11:06:08.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0009318810000001 2023-10-09T11:06:08.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.312992+0000", 2023-10-09T11:06:08.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7459000000000004e-05 2023-10-09T11:06:08.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:06:08.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.313085+0000", 2023-10-09T11:06:08.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2730000000000002e-05 2023-10-09T11:06:08.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315270+0000", 2023-10-09T11:06:08.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021854069999999999 2023-10-09T11:06:08.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318204+0000", 2023-10-09T11:06:08.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002933719 2023-10-09T11:06:08.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319781+0000", 2023-10-09T11:06:08.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001577427 2023-10-09T11:06:08.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:06:08.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319838+0000", 2023-10-09T11:06:08.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6882000000000002e-05 2023-10-09T11:06:08.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.718 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.718 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319879+0000", 2023-10-09T11:06:08.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0627999999999998e-05 2023-10-09T11:06:08.718 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.718 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.718 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.719 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39394 11.d 11:ba71174e:::benchmark_data_smithi125_439543_object39393:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:06:08.719 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.316112+0000", 2023-10-09T11:06:08.719 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 34.334079135000003, 2023-10-09T11:06:08.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.003833969, 2023-10-09T11:06:08.719 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.719 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:08.719 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.719 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:06:08.719 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:08.719 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39394 2023-10-09T11:06:08.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.719 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316112+0000", 2023-10-09T11:06:08.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316112+0000", 2023-10-09T11:06:08.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316110+0000", 2023-10-09T11:06:08.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:06:08.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316139+0000", 2023-10-09T11:06:08.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9023e-05 2023-10-09T11:06:08.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316160+0000", 2023-10-09T11:06:08.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0852999999999998e-05 2023-10-09T11:06:08.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316170+0000", 2023-10-09T11:06:08.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8970000000000003e-06 2023-10-09T11:06:08.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316244+0000", 2023-10-09T11:06:08.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4184000000000007e-05 2023-10-09T11:06:08.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316256+0000", 2023-10-09T11:06:08.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1447000000000001e-05 2023-10-09T11:06:08.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.313863+0000", 2023-10-09T11:06:08.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99760760800000003 2023-10-09T11:06:08.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.313943+0000", 2023-10-09T11:06:08.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9276999999999993e-05 2023-10-09T11:06:08.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:06:08.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.314050+0000", 2023-10-09T11:06:08.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000107831 2023-10-09T11:06:08.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315326+0000", 2023-10-09T11:06:08.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001276103 2023-10-09T11:06:08.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318251+0000", 2023-10-09T11:06:08.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002924683 2023-10-09T11:06:08.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319894+0000", 2023-10-09T11:06:08.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016428599999999999 2023-10-09T11:06:08.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:06:08.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319928+0000", 2023-10-09T11:06:08.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3674999999999999e-05 2023-10-09T11:06:08.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319946+0000", 2023-10-09T11:06:08.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8297000000000001e-05 2023-10-09T11:06:08.727 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.727 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.727 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.727 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.727 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.727 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39401 11.d 11:b5dcd9bc:::benchmark_data_smithi125_439543_object39400:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:06:08.727 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.322614+0000", 2023-10-09T11:06:08.727 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 34.327577296999998, 2023-10-09T11:06:08.727 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.003544776, 2023-10-09T11:06:08.727 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.727 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:08.728 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.728 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:06:08.728 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:08.728 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39401 2023-10-09T11:06:08.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.728 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322614+0000", 2023-10-09T11:06:08.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322614+0000", 2023-10-09T11:06:08.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322612+0000", 2023-10-09T11:06:08.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:08.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322636+0000", 2023-10-09T11:06:08.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4015e-05 2023-10-09T11:06:08.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322650+0000", 2023-10-09T11:06:08.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.345e-05 2023-10-09T11:06:08.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322657+0000", 2023-10-09T11:06:08.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3490000000000003e-06 2023-10-09T11:06:08.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322718+0000", 2023-10-09T11:06:08.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0520999999999999e-05 2023-10-09T11:06:08.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322728+0000", 2023-10-09T11:06:08.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0216e-05 2023-10-09T11:06:08.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315358+0000", 2023-10-09T11:06:08.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.992629808 2023-10-09T11:06:08.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315493+0000", 2023-10-09T11:06:08.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013509899999999999 2023-10-09T11:06:08.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:06:08.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315627+0000", 2023-10-09T11:06:08.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000134077 2023-10-09T11:06:08.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318286+0000", 2023-10-09T11:06:08.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00265898 2023-10-09T11:06:08.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319980+0000", 2023-10-09T11:06:08.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016941 2023-10-09T11:06:08.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326073+0000", 2023-10-09T11:06:08.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0060933239999999998 2023-10-09T11:06:08.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:06:08.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326124+0000", 2023-10-09T11:06:08.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1088000000000003e-05 2023-10-09T11:06:08.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326159+0000", 2023-10-09T11:06:08.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4353000000000001e-05 2023-10-09T11:06:08.735 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.735 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.735 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.736 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39439 11.d 11:b59e136a:::benchmark_data_smithi125_439543_object39438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:06:08.736 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.352200+0000", 2023-10-09T11:06:08.736 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 34.297990785000003, 2023-10-09T11:06:08.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97404727300000005, 2023-10-09T11:06:08.736 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.736 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:08.736 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.736 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:06:08.736 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:08.736 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39439 2023-10-09T11:06:08.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.737 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352200+0000", 2023-10-09T11:06:08.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352200+0000", 2023-10-09T11:06:08.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352199+0000", 2023-10-09T11:06:08.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:08.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352248+0000", 2023-10-09T11:06:08.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9008000000000002e-05 2023-10-09T11:06:08.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352264+0000", 2023-10-09T11:06:08.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5722e-05 2023-10-09T11:06:08.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352272+0000", 2023-10-09T11:06:08.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2199999999999992e-06 2023-10-09T11:06:08.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352340+0000", 2023-10-09T11:06:08.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8590000000000006e-05 2023-10-09T11:06:08.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352353+0000", 2023-10-09T11:06:08.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2648e-05 2023-10-09T11:06:08.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.316802+0000", 2023-10-09T11:06:08.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96444901500000002 2023-10-09T11:06:08.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.316917+0000", 2023-10-09T11:06:08.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011529800000000001 2023-10-09T11:06:08.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:06:08.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.317050+0000", 2023-10-09T11:06:08.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000132442 2023-10-09T11:06:08.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319767+0000", 2023-10-09T11:06:08.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027173449999999999 2023-10-09T11:06:08.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326058+0000", 2023-10-09T11:06:08.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062911210000000002 2023-10-09T11:06:08.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326171+0000", 2023-10-09T11:06:08.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000112565 2023-10-09T11:06:08.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:06:08.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326218+0000", 2023-10-09T11:06:08.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7377999999999999e-05 2023-10-09T11:06:08.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326248+0000", 2023-10-09T11:06:08.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9482999999999999e-05 2023-10-09T11:06:08.743 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.743 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.743 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.744 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39440 11.d 11:bb23716e:::benchmark_data_smithi125_439543_object39439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:06:08.744 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.352285+0000", 2023-10-09T11:06:08.744 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 34.297906568000002, 2023-10-09T11:06:08.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97412860000000001, 2023-10-09T11:06:08.744 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.744 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:08.744 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.744 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:06:08.744 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:08.745 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39440 2023-10-09T11:06:08.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.745 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352285+0000", 2023-10-09T11:06:08.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352285+0000", 2023-10-09T11:06:08.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352284+0000", 2023-10-09T11:06:08.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:08.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352305+0000", 2023-10-09T11:06:08.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1382e-05 2023-10-09T11:06:08.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352314+0000", 2023-10-09T11:06:08.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5220000000000007e-06 2023-10-09T11:06:08.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352320+0000", 2023-10-09T11:06:08.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0850000000000002e-06 2023-10-09T11:06:08.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352375+0000", 2023-10-09T11:06:08.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5411999999999997e-05 2023-10-09T11:06:08.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352382+0000", 2023-10-09T11:06:08.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.849e-06 2023-10-09T11:06:08.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318309+0000", 2023-10-09T11:06:08.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96592686800000005 2023-10-09T11:06:08.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318427+0000", 2023-10-09T11:06:08.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118267 2023-10-09T11:06:08.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:06:08.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318567+0000", 2023-10-09T11:06:08.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013931800000000001 2023-10-09T11:06:08.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.320026+0000", 2023-10-09T11:06:08.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014593410000000001 2023-10-09T11:06:08.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326259+0000", 2023-10-09T11:06:08.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062325130000000003 2023-10-09T11:06:08.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326327+0000", 2023-10-09T11:06:08.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8077999999999994e-05 2023-10-09T11:06:08.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:06:08.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326376+0000", 2023-10-09T11:06:08.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8949000000000002e-05 2023-10-09T11:06:08.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326413+0000", 2023-10-09T11:06:08.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7545000000000001e-05 2023-10-09T11:06:08.752 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.752 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.752 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.752 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39464 11.d 11:b03a9b94:::benchmark_data_smithi125_439543_object39463:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:06:08.752 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.367860+0000", 2023-10-09T11:06:08.752 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 34.282330690000002, 2023-10-09T11:06:08.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96193788300000005, 2023-10-09T11:06:08.752 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.752 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:08.752 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.753 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:06:08.753 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:08.753 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39464 2023-10-09T11:06:08.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.753 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367860+0000", 2023-10-09T11:06:08.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367860+0000", 2023-10-09T11:06:08.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367859+0000", 2023-10-09T11:06:08.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:06:08.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367885+0000", 2023-10-09T11:06:08.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5820999999999999e-05 2023-10-09T11:06:08.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367899+0000", 2023-10-09T11:06:08.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4803e-05 2023-10-09T11:06:08.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367907+0000", 2023-10-09T11:06:08.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2010000000000002e-06 2023-10-09T11:06:08.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367967+0000", 2023-10-09T11:06:08.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0299e-05 2023-10-09T11:06:08.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367978+0000", 2023-10-09T11:06:08.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1616e-05 2023-10-09T11:06:08.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.320095+0000", 2023-10-09T11:06:08.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95211647399999999 2023-10-09T11:06:08.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.320237+0000", 2023-10-09T11:06:08.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014207400000000001 2023-10-09T11:06:08.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:06:08.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.320378+0000", 2023-10-09T11:06:08.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014121199999999999 2023-10-09T11:06:08.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326264+0000", 2023-10-09T11:06:08.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0058858579999999999 2023-10-09T11:06:08.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.329650+0000", 2023-10-09T11:06:08.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0033856849999999998 2023-10-09T11:06:08.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.329705+0000", 2023-10-09T11:06:08.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5633000000000001e-05 2023-10-09T11:06:08.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:06:08.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.329758+0000", 2023-10-09T11:06:08.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2395999999999998e-05 2023-10-09T11:06:08.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.329798+0000", 2023-10-09T11:06:08.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0491000000000002e-05 2023-10-09T11:06:08.760 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.760 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.760 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.760 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39465 11.d 11:baf45870:::benchmark_data_smithi125_439543_object39464:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:06:08.760 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.368960+0000", 2023-10-09T11:06:08.760 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 34.281231407, 2023-10-09T11:06:08.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96470808799999996, 2023-10-09T11:06:08.760 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.760 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:08.760 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.761 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:06:08.761 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:08.761 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39465 2023-10-09T11:06:08.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.761 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368960+0000", 2023-10-09T11:06:08.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368960+0000", 2023-10-09T11:06:08.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368959+0000", 2023-10-09T11:06:08.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:06:08.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368977+0000", 2023-10-09T11:06:08.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8229000000000001e-05 2023-10-09T11:06:08.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368987+0000", 2023-10-09T11:06:08.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0139999999999999e-05 2023-10-09T11:06:08.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368993+0000", 2023-10-09T11:06:08.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1700000000000002e-06 2023-10-09T11:06:08.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.369036+0000", 2023-10-09T11:06:08.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2355999999999999e-05 2023-10-09T11:06:08.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.369042+0000", 2023-10-09T11:06:08.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8070000000000002e-06 2023-10-09T11:06:08.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.321501+0000", 2023-10-09T11:06:08.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95245864800000002 2023-10-09T11:06:08.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.321617+0000", 2023-10-09T11:06:08.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011572 2023-10-09T11:06:08.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:06:08.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.321752+0000", 2023-10-09T11:06:08.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013548 2023-10-09T11:06:08.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326270+0000", 2023-10-09T11:06:08.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045178120000000004 2023-10-09T11:06:08.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.329989+0000", 2023-10-09T11:06:08.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037189179999999999 2023-10-09T11:06:08.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333591+0000", 2023-10-09T11:06:08.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036021080000000001 2023-10-09T11:06:08.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:06:08.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333626+0000", 2023-10-09T11:06:08.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5120000000000003e-05 2023-10-09T11:06:08.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333668+0000", 2023-10-09T11:06:08.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1616000000000003e-05 2023-10-09T11:06:08.768 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.768 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.768 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.768 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39467 11.d 11:b2ddd126:::benchmark_data_smithi125_439543_object39466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:06:08.768 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.370134+0000", 2023-10-09T11:06:08.768 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 34.280056653000003, 2023-10-09T11:06:08.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96363200699999996, 2023-10-09T11:06:08.769 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.769 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:08.769 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.769 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:06:08.769 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:08.769 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39467 2023-10-09T11:06:08.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.769 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370134+0000", 2023-10-09T11:06:08.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370134+0000", 2023-10-09T11:06:08.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370133+0000", 2023-10-09T11:06:08.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:06:08.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370183+0000", 2023-10-09T11:06:08.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9186e-05 2023-10-09T11:06:08.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370200+0000", 2023-10-09T11:06:08.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6937999999999999e-05 2023-10-09T11:06:08.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370213+0000", 2023-10-09T11:06:08.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3599e-05 2023-10-09T11:06:08.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370255+0000", 2023-10-09T11:06:08.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1913999999999999e-05 2023-10-09T11:06:08.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370262+0000", 2023-10-09T11:06:08.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6089999999999997e-06 2023-10-09T11:06:08.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.323037+0000", 2023-10-09T11:06:08.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95277573000000004 2023-10-09T11:06:08.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.323131+0000", 2023-10-09T11:06:08.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3330000000000003e-05 2023-10-09T11:06:08.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:06:08.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.323244+0000", 2023-10-09T11:06:08.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000113033 2023-10-09T11:06:08.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.328073+0000", 2023-10-09T11:06:08.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0048294849999999997 2023-10-09T11:06:08.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330038+0000", 2023-10-09T11:06:08.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019642829999999998 2023-10-09T11:06:08.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333706+0000", 2023-10-09T11:06:08.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003667942 2023-10-09T11:06:08.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:06:08.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333732+0000", 2023-10-09T11:06:08.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6774000000000001e-05 2023-10-09T11:06:08.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333766+0000", 2023-10-09T11:06:08.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4178999999999999e-05 2023-10-09T11:06:08.776 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.776 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.777 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.777 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39473 11.d 11:b96957ab:::benchmark_data_smithi125_439543_object39472:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:06:08.777 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.372932+0000", 2023-10-09T11:06:08.777 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 34.277258731000003, 2023-10-09T11:06:08.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96091150700000005, 2023-10-09T11:06:08.777 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.777 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:08.777 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.777 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:06:08.778 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:08.778 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39473 2023-10-09T11:06:08.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.778 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.372932+0000", 2023-10-09T11:06:08.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.372932+0000", 2023-10-09T11:06:08.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.372931+0000", 2023-10-09T11:06:08.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:08.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.372988+0000", 2023-10-09T11:06:08.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6953999999999997e-05 2023-10-09T11:06:08.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.373007+0000", 2023-10-09T11:06:08.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8711000000000001e-05 2023-10-09T11:06:08.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.373017+0000", 2023-10-09T11:06:08.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0400999999999999e-05 2023-10-09T11:06:08.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.373082+0000", 2023-10-09T11:06:08.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4999000000000006e-05 2023-10-09T11:06:08.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.373094+0000", 2023-10-09T11:06:08.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1766000000000001e-05 2023-10-09T11:06:08.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.324554+0000", 2023-10-09T11:06:08.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95145981199999996 2023-10-09T11:06:08.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.782 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.324645+0000", 2023-10-09T11:06:08.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1250999999999998e-05 2023-10-09T11:06:08.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:06:08.782 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.324763+0000", 2023-10-09T11:06:08.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011804899999999999 2023-10-09T11:06:08.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.328078+0000", 2023-10-09T11:06:08.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0033156180000000002 2023-10-09T11:06:08.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330081+0000", 2023-10-09T11:06:08.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020020369999999999 2023-10-09T11:06:08.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333788+0000", 2023-10-09T11:06:08.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037078150000000002 2023-10-09T11:06:08.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:06:08.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333813+0000", 2023-10-09T11:06:08.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4239e-05 2023-10-09T11:06:08.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333844+0000", 2023-10-09T11:06:08.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1325999999999998e-05 2023-10-09T11:06:08.785 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.785 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.785 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.785 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39476 11.d 11:bbb64df7:::benchmark_data_smithi125_439543_object39475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:06:08.785 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.377052+0000", 2023-10-09T11:06:08.785 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 34.273138961000001, 2023-10-09T11:06:08.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96144097100000003, 2023-10-09T11:06:08.785 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.785 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:08.785 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.786 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:06:08.786 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:08.786 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39476 2023-10-09T11:06:08.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.786 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377052+0000", 2023-10-09T11:06:08.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377052+0000", 2023-10-09T11:06:08.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377050+0000", 2023-10-09T11:06:08.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:06:08.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377075+0000", 2023-10-09T11:06:08.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4941000000000001e-05 2023-10-09T11:06:08.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377090+0000", 2023-10-09T11:06:08.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.428e-05 2023-10-09T11:06:08.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377098+0000", 2023-10-09T11:06:08.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7110000000000001e-06 2023-10-09T11:06:08.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377155+0000", 2023-10-09T11:06:08.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6977e-05 2023-10-09T11:06:08.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377164+0000", 2023-10-09T11:06:08.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1920000000000005e-06 2023-10-09T11:06:08.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326460+0000", 2023-10-09T11:06:08.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.94929656600000001 2023-10-09T11:06:08.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326573+0000", 2023-10-09T11:06:08.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011275100000000001 2023-10-09T11:06:08.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:06:08.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326705+0000", 2023-10-09T11:06:08.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013251199999999999 2023-10-09T11:06:08.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330000+0000", 2023-10-09T11:06:08.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032948890000000001 2023-10-09T11:06:08.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333869+0000", 2023-10-09T11:06:08.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038684290000000001 2023-10-09T11:06:08.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.792 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.792 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338415+0000", 2023-10-09T11:06:08.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045463700000000001 2023-10-09T11:06:08.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.792 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:06:08.792 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338458+0000", 2023-10-09T11:06:08.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2852999999999998e-05 2023-10-09T11:06:08.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338493+0000", 2023-10-09T11:06:08.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5231000000000002e-05 2023-10-09T11:06:08.793 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.793 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.793 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.793 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39490 11.d 11:b79c85dd:::benchmark_data_smithi125_439543_object39489:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:06:08.793 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.388748+0000", 2023-10-09T11:06:08.793 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 34.261443139000001, 2023-10-09T11:06:08.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.94984969600000002, 2023-10-09T11:06:08.794 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.794 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:08.794 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.794 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:06:08.794 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:08.794 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39490 2023-10-09T11:06:08.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.794 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388748+0000", 2023-10-09T11:06:08.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388748+0000", 2023-10-09T11:06:08.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388746+0000", 2023-10-09T11:06:08.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:06:08.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388771+0000", 2023-10-09T11:06:08.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4868e-05 2023-10-09T11:06:08.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388789+0000", 2023-10-09T11:06:08.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8077e-05 2023-10-09T11:06:08.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388801+0000", 2023-10-09T11:06:08.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1698999999999999e-05 2023-10-09T11:06:08.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388915+0000", 2023-10-09T11:06:08.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000114003 2023-10-09T11:06:08.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.798 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388926+0000", 2023-10-09T11:06:08.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0729e-05 2023-10-09T11:06:08.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.798 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.798 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.798 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.328103+0000", 2023-10-09T11:06:08.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.93917761600000005 2023-10-09T11:06:08.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.798 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.798 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.798 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.328228+0000", 2023-10-09T11:06:08.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012434699999999999 2023-10-09T11:06:08.799 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:06:08.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.328355+0000", 2023-10-09T11:06:08.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012757299999999999 2023-10-09T11:06:08.799 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330045+0000", 2023-10-09T11:06:08.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001689968 2023-10-09T11:06:08.799 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.337060+0000", 2023-10-09T11:06:08.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0070151830000000004 2023-10-09T11:06:08.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338523+0000", 2023-10-09T11:06:08.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001462514 2023-10-09T11:06:08.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:06:08.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338557+0000", 2023-10-09T11:06:08.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3948000000000001e-05 2023-10-09T11:06:08.801 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.801 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338598+0000", 2023-10-09T11:06:08.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0923000000000001e-05 2023-10-09T11:06:08.801 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.801 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.801 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.802 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.802 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.802 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39501 11.d 11:b409f08d:::benchmark_data_smithi125_439543_object39500:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:06:08.802 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.397575+0000", 2023-10-09T11:06:08.802 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 34.252616502999999, 2023-10-09T11:06:08.802 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.94120543800000001, 2023-10-09T11:06:08.802 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:06:08.802 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:08.802 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:06:08.802 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:06:08.802 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:08.802 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39501 2023-10-09T11:06:08.803 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.803 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:06:08.803 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.803 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:06:08.803 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397575+0000", 2023-10-09T11:06:08.803 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.803 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.803 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.803 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:06:08.803 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397575+0000", 2023-10-09T11:06:08.803 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:06:08.803 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.804 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:06:08.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397573+0000", 2023-10-09T11:06:08.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:06:08.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.804 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:06:08.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397613+0000", 2023-10-09T11:06:08.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0145000000000003e-05 2023-10-09T11:06:08.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:06:08.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397626+0000", 2023-10-09T11:06:08.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3421000000000001e-05 2023-10-09T11:06:08.805 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.805 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:06:08.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397634+0000", 2023-10-09T11:06:08.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6210000000000004e-06 2023-10-09T11:06:08.805 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.805 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.806 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397700+0000", 2023-10-09T11:06:08.806 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5512000000000006e-05 2023-10-09T11:06:08.806 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.806 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.806 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:06:08.806 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397709+0000", 2023-10-09T11:06:08.806 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3360000000000001e-06 2023-10-09T11:06:08.806 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.806 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.806 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:06:08.806 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330105+0000", 2023-10-09T11:06:08.806 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.93239575399999997 2023-10-09T11:06:08.807 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.807 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.807 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:06:08.807 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330264+0000", 2023-10-09T11:06:08.807 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015971200000000001 2023-10-09T11:06:08.807 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.807 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.807 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:06:08.807 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330365+0000", 2023-10-09T11:06:08.807 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000100578 2023-10-09T11:06:08.807 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.807 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.808 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:06:08.808 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333554+0000", 2023-10-09T11:06:08.808 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003188897 2023-10-09T11:06:08.808 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.808 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.808 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.808 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.337092+0000", 2023-10-09T11:06:08.808 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003537756 2023-10-09T11:06:08.808 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.808 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.808 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:08.808 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338698+0000", 2023-10-09T11:06:08.809 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001606648 2023-10-09T11:06:08.809 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.809 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.809 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:06:08.809 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338750+0000", 2023-10-09T11:06:08.809 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1326999999999998e-05 2023-10-09T11:06:08.809 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:06:08.809 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:06:08.809 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:06:08.809 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338780+0000", 2023-10-09T11:06:08.809 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0420999999999999e-05 2023-10-09T11:06:08.809 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.810 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.810 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.810 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:06:08.810 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:06:08.810 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:06:08.895 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:08.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:08.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:08.990 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T11:06:08.991312+0000 min lat: 0.00101021 max lat: 2.16829 avg lat: 0.0123639 2023-10-09T11:06:08.990 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T11:06:08.990 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 60 16 75168 75152 78.2403 0 - 0.0123639 2023-10-09T11:06:09.003 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:09.045 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[195507]: pgmap v1264: 129 pgs: 2 active+recovering, 9 active+recovery_wait+degraded, 118 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 4762/216978 objects degraded (2.195%); 606/216978 objects misplaced (0.279%); 11 MiB/s, 174 objects/s recovering 2023-10-09T11:06:09.045 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[195507]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.045 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[195507]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.045 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[195507]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[195507]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[195507]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[195507]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[195507]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[195507]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[195507]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[195507]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 4762/216978 objects degraded (2.195%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:09.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[195507]: osdmap e518: 8 total, 8 up, 8 in 2023-10-09T11:06:09.075 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:09.107 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:09.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:09.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:09.215 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:09.248 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:09.270 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:09.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:09.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:09.310 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[200197]: pgmap v1264: 129 pgs: 2 active+recovering, 9 active+recovery_wait+degraded, 118 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 4762/216978 objects degraded (2.195%); 606/216978 objects misplaced (0.279%); 11 MiB/s, 174 objects/s recovering 2023-10-09T11:06:09.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[200197]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[200197]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[200197]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[200197]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[200197]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[200197]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[200197]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[200197]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[200197]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[200197]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.312 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 4762/216978 objects degraded (2.195%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:09.312 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:09 smithi125 ceph-mon[200197]: osdmap e518: 8 total, 8 up, 8 in 2023-10-09T11:06:09.312 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T11:06:09.340 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:09 smithi155 ceph-mon[184854]: pgmap v1264: 129 pgs: 2 active+recovering, 9 active+recovery_wait+degraded, 118 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 4762/216978 objects degraded (2.195%); 606/216978 objects misplaced (0.279%); 11 MiB/s, 174 objects/s recovering 2023-10-09T11:06:09.340 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:09 smithi155 ceph-mon[184854]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.340 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:09 smithi155 ceph-mon[184854]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.340 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:09 smithi155 ceph-mon[184854]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.340 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:09 smithi155 ceph-mon[184854]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.340 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:09 smithi155 ceph-mon[184854]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.340 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:09 smithi155 ceph-mon[184854]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.340 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:09 smithi155 ceph-mon[184854]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.341 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:09 smithi155 ceph-mon[184854]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.341 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:09 smithi155 ceph-mon[184854]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.341 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:09 smithi155 ceph-mon[184854]: failed to encode map e517 with expected crc 2023-10-09T11:06:09.341 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:09 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 4762/216978 objects degraded (2.195%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:09.341 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:09 smithi155 ceph-mon[184854]: osdmap e518: 8 total, 8 up, 8 in 2023-10-09T11:06:09.373 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:09.394 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:06:09.404 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:09.415 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:06:09.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:09.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:09.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:09.439 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:06:09.459 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:06:09.479 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:06:09.498 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:06:09.517 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:06:09.531 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:09.536 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:06:09.537 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:06:09.537 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:06:09.538 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:06:09.538 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:06:09.538 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:06:09.538 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:06:09.538 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:06:09.538 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:06:09.561 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:09.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:09.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:09.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:09.635 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:06:09.688 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:09.716 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:09.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:09.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:09.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:09.890 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:09.918 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:09.930 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:06:09.938 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:09.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:09.990 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 61 16 75266 75250 77.0586 3.0625 0.00239649 0.012637 2023-10-09T11:06:10.042 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:10.073 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:10.144 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:10.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:10.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:10.248 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:10.253 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:10.276 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[195507]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.276 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[195507]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[195507]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[195507]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[195507]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[195507]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[195507]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[195507]: 11.13 starting backfill to osd.5 from (0'0,498'1356] MIN to 515'3965 2023-10-09T11:06:10.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[195507]: 11.6 scrub starts 2023-10-09T11:06:10.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1305337091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:10.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[195507]: osdmap e519: 8 total, 8 up, 8 in 2023-10-09T11:06:10.277 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[200197]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.278 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[200197]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.278 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[200197]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.278 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[200197]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.278 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[200197]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.278 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[200197]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.278 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[200197]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.278 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[200197]: 11.13 starting backfill to osd.5 from (0'0,498'1356] MIN to 515'3965 2023-10-09T11:06:10.278 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[200197]: 11.6 scrub starts 2023-10-09T11:06:10.278 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1305337091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:10.278 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:10 smithi125 ceph-mon[200197]: osdmap e519: 8 total, 8 up, 8 in 2023-10-09T11:06:10.279 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:10.401 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:10.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:10.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:10.427 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:10 smithi155 ceph-mon[184854]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.427 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:10 smithi155 ceph-mon[184854]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.427 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:10 smithi155 ceph-mon[184854]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.427 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:10 smithi155 ceph-mon[184854]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.427 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:10 smithi155 ceph-mon[184854]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.427 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:10 smithi155 ceph-mon[184854]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.427 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:10 smithi155 ceph-mon[184854]: failed to encode map e518 with expected crc 2023-10-09T11:06:10.427 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:10 smithi155 ceph-mon[184854]: 11.13 starting backfill to osd.5 from (0'0,498'1356] MIN to 515'3965 2023-10-09T11:06:10.428 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:10 smithi155 ceph-mon[184854]: 11.6 scrub starts 2023-10-09T11:06:10.428 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:10 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1305337091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:10.428 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:10 smithi155 ceph-mon[184854]: osdmap e519: 8 total, 8 up, 8 in 2023-10-09T11:06:10.505 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:10.533 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:10.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:10.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:10.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:10.807 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:10.835 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:10.891 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:06:10.891 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:06:10.905 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:10.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:10.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:10.990 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 62 16 75266 75250 75.8162 0 - 0.012637 2023-10-09T11:06:11.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:11.037 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:11.058 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:11.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:11.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:11.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[195507]: pgmap v1267: 129 pgs: 14 remapped+peering, 1 active+recovering, 7 active+recovery_wait+degraded, 107 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 2.3 KiB/s rd, 50 MiB/s wr, 801 op/s; 4007/225036 objects degraded (1.781%); 372/225036 objects misplaced (0.165%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T11:06:11.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[195507]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[195507]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[195507]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[195507]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[195507]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[195507]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[195507]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 3 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-10-09T11:06:11.106 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[195507]: osdmap e520: 8 total, 8 up, 8 in 2023-10-09T11:06:11.106 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[200197]: pgmap v1267: 129 pgs: 14 remapped+peering, 1 active+recovering, 7 active+recovery_wait+degraded, 107 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 2.3 KiB/s rd, 50 MiB/s wr, 801 op/s; 4007/225036 objects degraded (1.781%); 372/225036 objects misplaced (0.165%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T11:06:11.106 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[200197]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.106 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[200197]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.106 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[200197]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.106 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[200197]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.106 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[200197]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.106 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[200197]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.106 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[200197]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.107 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 3 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-10-09T11:06:11.107 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-mon[200197]: osdmap e520: 8 total, 8 up, 8 in 2023-10-09T11:06:11.163 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:11.177 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:11 smithi155 ceph-mon[184854]: pgmap v1267: 129 pgs: 14 remapped+peering, 1 active+recovering, 7 active+recovery_wait+degraded, 107 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 2.3 KiB/s rd, 50 MiB/s wr, 801 op/s; 4007/225036 objects degraded (1.781%); 372/225036 objects misplaced (0.165%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T11:06:11.177 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:11 smithi155 ceph-mon[184854]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.177 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:11 smithi155 ceph-mon[184854]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.177 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:11 smithi155 ceph-mon[184854]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.178 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:11 smithi155 ceph-mon[184854]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.178 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:11 smithi155 ceph-mon[184854]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.178 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:11 smithi155 ceph-mon[184854]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.178 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:11 smithi155 ceph-mon[184854]: failed to encode map e519 with expected crc 2023-10-09T11:06:11.178 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:11 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 3 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-10-09T11:06:11.178 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:11 smithi155 ceph-mon[184854]: osdmap e520: 8 total, 8 up, 8 in 2023-10-09T11:06:11.192 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:11.211 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:11.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:11.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:11.273 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T11:06:11.315 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:11.343 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:11.362 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:11.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:11.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:11.466 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:11.495 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:11.515 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:11.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:11.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:11.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:06:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:06:11] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2023-10-09T11:06:11.574 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:06:11.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:11.646 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:11.665 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:11.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:11.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:11.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:11.821 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:11.846 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:11.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:11.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:11.925 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:06:11.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:11.991 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 63 16 75562 75546 74.9061 9.25 0.0156635 0.0133445 2023-10-09T11:06:11.995 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:12.108 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:06:12.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:12 smithi155 ceph-mon[184854]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:12 smithi155 ceph-mon[184854]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:12 smithi155 ceph-mon[184854]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:12 smithi155 ceph-mon[184854]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:12 smithi155 ceph-mon[184854]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:12 smithi155 ceph-mon[184854]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:12 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1152990900' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:12.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:12 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:12.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:12 smithi155 ceph-mon[184854]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.177 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:12.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:12.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:12.279 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:12.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[195507]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[195507]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[195507]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[195507]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[195507]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[195507]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1152990900' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:12.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:12.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[195507]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[200197]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[200197]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[200197]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[200197]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[200197]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[200197]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1152990900' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:12.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:12.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:12 smithi125 ceph-mon[200197]: failed to encode map e520 with expected crc 2023-10-09T11:06:12.322 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:12.351 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:12.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:12.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:12.457 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:12.516 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:12.561 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:12.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:12.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:12.614 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:12.635 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:06:12.635 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:06:12.672 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:12.728 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:12.753 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:12.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:12.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:12.858 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:12.908 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:12.994 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 64 16 76566 76550 74.7162 62.75 0.00714303 0.013379 2023-10-09T11:06:12.996 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:13.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:13.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:13.038 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T11:06:13.040 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 7, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 5, 6, 7, 1, 3] 2023-10-09T11:06:13.041 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:06:13.041 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T11:06:13.041 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T11:06:13.041 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T11:06:13.041 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T11:06:13.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:13.186 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:13.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:13 smithi155 ceph-mon[184854]: pgmap v1270: 129 pgs: 1 active+recovering+undersized+remapped, 14 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 107 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 3.5 KiB/s rd, 45 MiB/s wr, 722 op/s; 4605/225612 objects degraded (2.041%); 1581/225612 objects misplaced (0.701%); 27 MiB/s, 429 objects/s recovering 2023-10-09T11:06:13.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:13 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:06:13.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:13 smithi155 ceph-mon[184854]: failed to encode map e521 with expected crc 2023-10-09T11:06:13.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:13 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:06:13.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:13 smithi155 ceph-mon[184854]: osdmap e521: 8 total, 8 up, 8 in 2023-10-09T11:06:13.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:13 smithi155 ceph-mon[184854]: failed to encode map e521 with expected crc 2023-10-09T11:06:13.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:13 smithi155 ceph-mon[184854]: failed to encode map e521 with expected crc 2023-10-09T11:06:13.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:13 smithi155 ceph-mon[184854]: failed to encode map e521 with expected crc 2023-10-09T11:06:13.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:13 smithi155 ceph-mon[184854]: failed to encode map e521 with expected crc 2023-10-09T11:06:13.325 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:13.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:13.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:13.432 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:13.437 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:06:13.468 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:13.473 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:13.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[195507]: pgmap v1270: 129 pgs: 1 active+recovering+undersized+remapped, 14 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 107 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 3.5 KiB/s rd, 45 MiB/s wr, 722 op/s; 4605/225612 objects degraded (2.041%); 1581/225612 objects misplaced (0.701%); 27 MiB/s, 429 objects/s recovering 2023-10-09T11:06:13.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:06:13.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[195507]: failed to encode map e521 with expected crc 2023-10-09T11:06:13.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:06:13.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[195507]: osdmap e521: 8 total, 8 up, 8 in 2023-10-09T11:06:13.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[195507]: failed to encode map e521 with expected crc 2023-10-09T11:06:13.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[195507]: failed to encode map e521 with expected crc 2023-10-09T11:06:13.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[195507]: failed to encode map e521 with expected crc 2023-10-09T11:06:13.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[195507]: failed to encode map e521 with expected crc 2023-10-09T11:06:13.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[200197]: pgmap v1270: 129 pgs: 1 active+recovering+undersized+remapped, 14 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 107 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 3.5 KiB/s rd, 45 MiB/s wr, 722 op/s; 4605/225612 objects degraded (2.041%); 1581/225612 objects misplaced (0.701%); 27 MiB/s, 429 objects/s recovering 2023-10-09T11:06:13.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:06:13.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[200197]: failed to encode map e521 with expected crc 2023-10-09T11:06:13.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:06:13.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[200197]: osdmap e521: 8 total, 8 up, 8 in 2023-10-09T11:06:13.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[200197]: failed to encode map e521 with expected crc 2023-10-09T11:06:13.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[200197]: failed to encode map e521 with expected crc 2023-10-09T11:06:13.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[200197]: failed to encode map e521 with expected crc 2023-10-09T11:06:13.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:13 smithi125 ceph-mon[200197]: failed to encode map e521 with expected crc 2023-10-09T11:06:13.605 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:13.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:13.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:13.707 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:13.743 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:13.917 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:13.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:13.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:13.968 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:06:13.991 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 65 16 77522 77506 74.4859 59.75 0.00928547 0.0134201 2023-10-09T11:06:14.004 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:06:14.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:14.051 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:06:14.059 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:14.096 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:06:14.139 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:14.145 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:06:14.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:14.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:14.181 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:06:14.218 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:06:14.246 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:14.267 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:06:14.276 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:06:14.276 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:06:14.276 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:06:14.276 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:06:14.277 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:06:14.277 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:06:14.277 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:06:14.277 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:06:14.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:14 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 4605/225612 objects degraded (2.041%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:14.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:14 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1369245572' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:14.300 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:14 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 4605/225612 objects degraded (2.041%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:14.301 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:14 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1369245572' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:14.334 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:14 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 4605/225612 objects degraded (2.041%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:14.335 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:14 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1369245572' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:14.343 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:14.437 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:14.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:14.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:14.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:14.568 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:06:14.611 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:14.672 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:06:14.672 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T11:06:14.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T11:06:14.672 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:06:14.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.672 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.35433.0:158888 8.2 8:4103043a:::benchmark_data_smithi125_329074_object62234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2023-10-09T11:06:14.672 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:06:14.672 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 599.31358890900003, 2023-10-09T11:06:14.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.123441579, 2023-10-09T11:06:14.672 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.673 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.673 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:14.673 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.35433", 2023-10-09T11:06:14.673 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3631187177", 2023-10-09T11:06:14.673 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 158888 2023-10-09T11:06:14.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.673 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:06:14.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:06:14.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356379+0000", 2023-10-09T11:06:14.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:14.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356381+0000", 2023-10-09T11:06:14.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.734e-06 2023-10-09T11:06:14.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356388+0000", 2023-10-09T11:06:14.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4559999999999997e-06 2023-10-09T11:06:14.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.356392+0000", 2023-10-09T11:06:14.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0589999999999996e-06 2023-10-09T11:06:14.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.359480+0000", 2023-10-09T11:06:14.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030878379999999999 2023-10-09T11:06:14.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.359628+0000", 2023-10-09T11:06:14.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014820299999999999 2023-10-09T11:06:14.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:06:14.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.360074+0000", 2023-10-09T11:06:14.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00044630599999999999 2023-10-09T11:06:14.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:14.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.362052+0000", 2023-10-09T11:06:14.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019782720000000001 2023-10-09T11:06:14.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.362671+0000", 2023-10-09T11:06:14.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00061827099999999999 2023-10-09T11:06:14.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479765+0000", 2023-10-09T11:06:14.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11709412199999999 2023-10-09T11:06:14.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479796+0000", 2023-10-09T11:06:14.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0895000000000002e-05 2023-10-09T11:06:14.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T10:56:15.479821+0000", 2023-10-09T11:06:14.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4816000000000001e-05 2023-10-09T11:06:14.679 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.679 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.679 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.679 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.679 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:9816 10.6 e406/403 10:610863d1:::benchmark_data_smithi125_402869_object9815:head v 406'620, mlcod=406'618)", 2023-10-09T11:06:14.679 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:22.339372+0000", 2023-10-09T11:06:14.679 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 292.33059551600002, 2023-10-09T11:06:14.679 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072690006000000001, 2023-10-09T11:06:14.679 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.679 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.679 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.680 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.680 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339372+0000", 2023-10-09T11:06:14.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.680 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.680 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.680 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.680 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339372+0000", 2023-10-09T11:06:14.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.680 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.680 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.680 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.680 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339372+0000", 2023-10-09T11:06:14.680 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:14.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.681 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339395+0000", 2023-10-09T11:06:14.681 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3051e-05 2023-10-09T11:06:14.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.681 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.681 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339404+0000", 2023-10-09T11:06:14.681 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.713e-06 2023-10-09T11:06:14.681 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.681 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.682 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339410+0000", 2023-10-09T11:06:14.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.181e-06 2023-10-09T11:06:14.682 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.682 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.682 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339487+0000", 2023-10-09T11:06:14.682 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6940999999999994e-05 2023-10-09T11:06:14.682 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.682 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.682 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.682 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339505+0000", 2023-10-09T11:06:14.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8700000000000001e-05 2023-10-09T11:06:14.683 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.683 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.683 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.412023+0000", 2023-10-09T11:06:14.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072518113999999995 2023-10-09T11:06:14.683 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.683 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.683 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.683 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.412062+0000", 2023-10-09T11:06:14.683 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8991e-05 2023-10-09T11:06:14.683 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.683 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.684 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.684 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.684 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.684 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:9820 10.d e406/403 10:b0a398bf:::benchmark_data_smithi125_402869_object9819:head v 406'628, mlcod=406'626)", 2023-10-09T11:06:14.684 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:22.339685+0000", 2023-10-09T11:06:14.684 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 292.33028309100001, 2023-10-09T11:06:14.684 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072206606000000007, 2023-10-09T11:06:14.684 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.684 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.684 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.684 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.684 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.684 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339685+0000", 2023-10-09T11:06:14.685 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.685 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.685 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.685 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.685 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339685+0000", 2023-10-09T11:06:14.685 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.685 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.685 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.685 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.685 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339684+0000", 2023-10-09T11:06:14.685 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:14.685 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.686 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.686 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.686 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339744+0000", 2023-10-09T11:06:14.686 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9889999999999997e-05 2023-10-09T11:06:14.686 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.686 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.686 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.686 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339754+0000", 2023-10-09T11:06:14.686 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0492999999999999e-05 2023-10-09T11:06:14.686 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.686 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.686 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.687 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339760+0000", 2023-10-09T11:06:14.687 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.699e-06 2023-10-09T11:06:14.687 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.687 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.687 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.687 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339792+0000", 2023-10-09T11:06:14.687 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2079999999999998e-05 2023-10-09T11:06:14.687 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.687 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.687 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.687 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.339805+0000", 2023-10-09T11:06:14.687 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2809999999999999e-05 2023-10-09T11:06:14.687 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.688 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.688 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.688 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411854+0000", 2023-10-09T11:06:14.688 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072048880999999995 2023-10-09T11:06:14.688 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.688 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.688 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.688 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411891+0000", 2023-10-09T11:06:14.688 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7434000000000001e-05 2023-10-09T11:06:14.688 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.688 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.689 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.689 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.689 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.689 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:9825 10.2 e406/403 10:49e48d05:::benchmark_data_smithi125_402869_object9824:head v 406'599, mlcod=406'597)", 2023-10-09T11:06:14.689 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:22.340607+0000", 2023-10-09T11:06:14.689 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 292.32936046899999, 2023-10-09T11:06:14.689 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.071013035000000002, 2023-10-09T11:06:14.689 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.689 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.689 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.689 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.689 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.690 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340607+0000", 2023-10-09T11:06:14.690 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.690 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.690 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.690 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.690 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340607+0000", 2023-10-09T11:06:14.690 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.690 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.690 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.690 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.690 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340607+0000", 2023-10-09T11:06:14.690 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:14.691 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.691 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.691 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.691 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340667+0000", 2023-10-09T11:06:14.691 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0310999999999999e-05 2023-10-09T11:06:14.691 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.691 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.691 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.691 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340681+0000", 2023-10-09T11:06:14.691 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4199000000000001e-05 2023-10-09T11:06:14.691 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.692 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.692 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.692 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340688+0000", 2023-10-09T11:06:14.692 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9979999999999997e-06 2023-10-09T11:06:14.692 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.692 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.692 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.692 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410579+0000", 2023-10-09T11:06:14.692 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069890939999999999 2023-10-09T11:06:14.692 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.692 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.693 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.693 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410601+0000", 2023-10-09T11:06:14.693 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1953000000000001e-05 2023-10-09T11:06:14.693 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.693 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.693 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.693 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411576+0000", 2023-10-09T11:06:14.693 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00097502500000000002 2023-10-09T11:06:14.693 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.693 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.693 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.693 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411620+0000", 2023-10-09T11:06:14.694 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4305999999999999e-05 2023-10-09T11:06:14.694 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.694 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.694 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.694 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.694 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.694 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:9818 10.6 e406/403 10:67ea2a60:::benchmark_data_smithi125_402869_object9817:head v 406'621, mlcod=406'618)", 2023-10-09T11:06:14.694 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:22.340768+0000", 2023-10-09T11:06:14.694 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 292.32919991900002, 2023-10-09T11:06:14.694 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.071318931000000002, 2023-10-09T11:06:14.694 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.695 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.695 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.695 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.695 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.695 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340768+0000", 2023-10-09T11:06:14.695 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.695 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.695 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.695 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.695 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340768+0000", 2023-10-09T11:06:14.695 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.695 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.696 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.696 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.696 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340767+0000", 2023-10-09T11:06:14.696 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:14.696 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.696 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.696 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.696 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.340793+0000", 2023-10-09T11:06:14.696 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5539000000000001e-05 2023-10-09T11:06:14.696 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.696 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.696 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.697 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410523+0000", 2023-10-09T11:06:14.697 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069730406999999994 2023-10-09T11:06:14.697 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.697 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.697 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.697 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410538+0000", 2023-10-09T11:06:14.697 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4747e-05 2023-10-09T11:06:14.697 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.697 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.697 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.697 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410587+0000", 2023-10-09T11:06:14.697 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8903000000000003e-05 2023-10-09T11:06:14.698 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.698 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.698 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.698 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410610+0000", 2023-10-09T11:06:14.698 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3288000000000001e-05 2023-10-09T11:06:14.698 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.698 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.698 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.698 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.412066+0000", 2023-10-09T11:06:14.698 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001456127 2023-10-09T11:06:14.698 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.698 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.699 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.699 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.412087+0000", 2023-10-09T11:06:14.699 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0557999999999999e-05 2023-10-09T11:06:14.699 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.699 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.699 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.699 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.699 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.699 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:9823 10.d e406/403 10:b90906b6:::benchmark_data_smithi125_402869_object9822:head v 406'629, mlcod=406'626)", 2023-10-09T11:06:14.699 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:22.341423+0000", 2023-10-09T11:06:14.699 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 292.32854451399999, 2023-10-09T11:06:14.699 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.07048981, 2023-10-09T11:06:14.700 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.700 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.700 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.700 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.341423+0000", 2023-10-09T11:06:14.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.700 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.700 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.700 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.700 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.341423+0000", 2023-10-09T11:06:14.700 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.701 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.701 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.701 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.341423+0000", 2023-10-09T11:06:14.701 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:14.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.701 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.701 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.701 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.341489+0000", 2023-10-09T11:06:14.701 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6185000000000004e-05 2023-10-09T11:06:14.701 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410523+0000", 2023-10-09T11:06:14.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069033960000000005 2023-10-09T11:06:14.702 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.702 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.702 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410538+0000", 2023-10-09T11:06:14.702 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5545000000000001e-05 2023-10-09T11:06:14.702 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.702 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410587+0000", 2023-10-09T11:06:14.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8813000000000002e-05 2023-10-09T11:06:14.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.410610+0000", 2023-10-09T11:06:14.703 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3119999999999999e-05 2023-10-09T11:06:14.703 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.703 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.703 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.703 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411895+0000", 2023-10-09T11:06:14.704 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012849770000000001 2023-10-09T11:06:14.704 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.704 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.704 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.704 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:22.411913+0000", 2023-10-09T11:06:14.704 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7864e-05 2023-10-09T11:06:14.704 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.704 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.704 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.704 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.705 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.705 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36123.0:161225 10.3 e464/460 10:c188d8d9:::benchmark_data_smithi125_402869_object62656:head v 464'8115, mlcod=419'2977)", 2023-10-09T11:06:14.705 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:03:58.731186+0000", 2023-10-09T11:06:14.705 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 135.93878227299999, 2023-10-09T11:06:14.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.039959936000000001, 2023-10-09T11:06:14.705 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.705 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.705 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.705 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.705 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.705 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:58.731186+0000", 2023-10-09T11:06:14.705 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.706 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.706 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.706 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:58.731186+0000", 2023-10-09T11:06:14.706 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.706 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.706 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.706 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:58.731185+0000", 2023-10-09T11:06:14.706 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:06:14.706 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.707 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.707 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.707 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:58.731191+0000", 2023-10-09T11:06:14.707 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1630000000000001e-06 2023-10-09T11:06:14.707 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.707 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.707 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.707 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:58.731207+0000", 2023-10-09T11:06:14.707 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6084999999999999e-05 2023-10-09T11:06:14.707 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.707 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.707 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:58.731214+0000", 2023-10-09T11:06:14.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0509999999999999e-06 2023-10-09T11:06:14.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.708 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.708 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:58.734422+0000", 2023-10-09T11:06:14.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032077659999999999 2023-10-09T11:06:14.708 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.708 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.708 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.708 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:58.734448+0000", 2023-10-09T11:06:14.708 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5743000000000001e-05 2023-10-09T11:06:14.709 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.709 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:58.771110+0000", 2023-10-09T11:06:14.709 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.036662298000000003 2023-10-09T11:06:14.709 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.709 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.709 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.709 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:58.771146+0000", 2023-10-09T11:06:14.709 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5743e-05 2023-10-09T11:06:14.709 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.709 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.710 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.710 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.710 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.710 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:182191 10.c 10:38b3d627:::benchmark_data_smithi125_402869_object83622:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:14.710 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:26.393583+0000", 2023-10-09T11:06:14.710 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 108.27638466099999, 2023-10-09T11:06:14.710 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.057033352000000002, 2023-10-09T11:06:14.710 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.710 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.710 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:14.710 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:14.711 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:14.711 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 182191 2023-10-09T11:06:14.711 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.711 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.711 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.711 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.711 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:26.393583+0000", 2023-10-09T11:06:14.711 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.711 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.711 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.711 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.711 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:26.393583+0000", 2023-10-09T11:06:14.712 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.712 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.712 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.712 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:26.393583+0000", 2023-10-09T11:06:14.712 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:14.712 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.712 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.712 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:26.393585+0000", 2023-10-09T11:06:14.712 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1749999999999999e-06 2023-10-09T11:06:14.712 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.713 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.713 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:26.393589+0000", 2023-10-09T11:06:14.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0890000000000002e-06 2023-10-09T11:06:14.713 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.713 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.713 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.713 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:26.393592+0000", 2023-10-09T11:06:14.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5349999999999999e-06 2023-10-09T11:06:14.713 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.713 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.713 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.713 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:26.398175+0000", 2023-10-09T11:06:14.714 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045830649999999999 2023-10-09T11:06:14.714 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.714 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.714 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.714 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:26.398302+0000", 2023-10-09T11:06:14.714 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012735200000000001 2023-10-09T11:06:14.714 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.714 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.714 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T11:06:14.714 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:26.398433+0000", 2023-10-09T11:06:14.714 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000130448 2023-10-09T11:06:14.714 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:26.403542+0000", 2023-10-09T11:06:14.715 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051097390000000003 2023-10-09T11:06:14.715 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:14.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:26.404784+0000", 2023-10-09T11:06:14.715 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001241713 2023-10-09T11:06:14.715 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:26.407822+0000", 2023-10-09T11:06:14.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030376470000000001 2023-10-09T11:06:14.716 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.716 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:26.450227+0000", 2023-10-09T11:06:14.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.042405598000000003 2023-10-09T11:06:14.716 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.716 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:26.450268+0000", 2023-10-09T11:06:14.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0782999999999999e-05 2023-10-09T11:06:14.716 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.717 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.717 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.717 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:26.450617+0000", 2023-10-09T11:06:14.717 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00034851199999999999 2023-10-09T11:06:14.717 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.717 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.717 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.717 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.717 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.717 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183147 10.c 10:36c9277a:::benchmark_data_smithi125_402869_object84578:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:14.717 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:27.601262+0000", 2023-10-09T11:06:14.718 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 107.068705498, 2023-10-09T11:06:14.718 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.045764109999999997, 2023-10-09T11:06:14.718 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.718 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.718 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:14.718 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:14.718 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:14.718 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183147 2023-10-09T11:06:14.718 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.718 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.718 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.718 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:27.601262+0000", 2023-10-09T11:06:14.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.719 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.719 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:27.601262+0000", 2023-10-09T11:06:14.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.719 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.719 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:27.601261+0000", 2023-10-09T11:06:14.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:14.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.720 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:27.601268+0000", 2023-10-09T11:06:14.720 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7610000000000002e-06 2023-10-09T11:06:14.720 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.720 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:27.601281+0000", 2023-10-09T11:06:14.720 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3722e-05 2023-10-09T11:06:14.720 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.721 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:27.601288+0000", 2023-10-09T11:06:14.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0839999999999997e-06 2023-10-09T11:06:14.721 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.721 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.721 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.721 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:27.601343+0000", 2023-10-09T11:06:14.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5952999999999998e-05 2023-10-09T11:06:14.721 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.721 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.721 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.721 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:27.601484+0000", 2023-10-09T11:06:14.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014077499999999999 2023-10-09T11:06:14.722 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.722 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.722 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T11:06:14.722 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:27.601616+0000", 2023-10-09T11:06:14.722 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013134600000000001 2023-10-09T11:06:14.722 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.722 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.722 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:14.722 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:27.604228+0000", 2023-10-09T11:06:14.722 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026126140000000001 2023-10-09T11:06:14.722 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.722 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.722 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:27.605853+0000", 2023-10-09T11:06:14.723 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0016251169999999999 2023-10-09T11:06:14.723 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.723 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.723 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:27.612602+0000", 2023-10-09T11:06:14.723 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.006748909 2023-10-09T11:06:14.723 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.723 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.723 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:27.646663+0000", 2023-10-09T11:06:14.723 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.034061084999999998 2023-10-09T11:06:14.724 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.724 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.724 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.724 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:27.646706+0000", 2023-10-09T11:06:14.724 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2858999999999997e-05 2023-10-09T11:06:14.724 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.724 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.724 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.724 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:27.647026+0000", 2023-10-09T11:06:14.724 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00032030300000000003 2023-10-09T11:06:14.724 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.724 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.725 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.725 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.725 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.725 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:184334 10.c 10:34793f68:::benchmark_data_smithi125_402869_object85765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:14.725 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:29.200061+0000", 2023-10-09T11:06:14.725 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 105.46990733299999, 2023-10-09T11:06:14.725 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.112198613, 2023-10-09T11:06:14.725 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.725 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.725 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:14.725 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:14.725 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:14.726 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 184334 2023-10-09T11:06:14.726 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.726 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.726 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.726 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.726 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.200061+0000", 2023-10-09T11:06:14.726 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.726 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.726 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.726 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.726 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.200061+0000", 2023-10-09T11:06:14.726 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.727 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.727 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.200059+0000", 2023-10-09T11:06:14.727 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:06:14.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.727 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.727 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.200066+0000", 2023-10-09T11:06:14.727 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5599999999999999e-06 2023-10-09T11:06:14.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.728 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.200082+0000", 2023-10-09T11:06:14.728 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5809999999999999e-05 2023-10-09T11:06:14.728 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.728 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.200089+0000", 2023-10-09T11:06:14.728 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0090000000000001e-06 2023-10-09T11:06:14.728 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.728 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.729 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.200144+0000", 2023-10-09T11:06:14.729 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5390999999999999e-05 2023-10-09T11:06:14.729 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.729 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.729 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.729 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.200268+0000", 2023-10-09T11:06:14.729 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012413699999999999 2023-10-09T11:06:14.729 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.729 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.729 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T11:06:14.729 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.200402+0000", 2023-10-09T11:06:14.729 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013374599999999999 2023-10-09T11:06:14.729 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.730 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:14.730 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.204506+0000", 2023-10-09T11:06:14.730 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041039570000000001 2023-10-09T11:06:14.730 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.730 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.730 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.206060+0000", 2023-10-09T11:06:14.730 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015544230000000001 2023-10-09T11:06:14.730 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.730 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.731 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.209462+0000", 2023-10-09T11:06:14.731 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034016519999999998 2023-10-09T11:06:14.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.731 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.731 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.731 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.311813+0000", 2023-10-09T11:06:14.731 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.102351043 2023-10-09T11:06:14.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.731 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.731 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.731 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.311862+0000", 2023-10-09T11:06:14.732 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9249999999999998e-05 2023-10-09T11:06:14.732 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.312259+0000", 2023-10-09T11:06:14.732 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00039677899999999999 2023-10-09T11:06:14.732 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.732 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.732 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.732 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.732 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:184407 10.c 10:39ed25e0:::benchmark_data_smithi125_402869_object85838:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:14.733 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:29.370383+0000", 2023-10-09T11:06:14.733 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 105.299584666, 2023-10-09T11:06:14.733 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.068598155999999993, 2023-10-09T11:06:14.733 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.733 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.733 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:14.733 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:14.733 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:14.733 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 184407 2023-10-09T11:06:14.733 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.733 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.733 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.733 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.734 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.370383+0000", 2023-10-09T11:06:14.734 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.734 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.734 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.734 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.734 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.370383+0000", 2023-10-09T11:06:14.734 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.734 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.734 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.734 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.734 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.370382+0000", 2023-10-09T11:06:14.734 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:06:14.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.735 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.735 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.370388+0000", 2023-10-09T11:06:14.735 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0009999999999998e-06 2023-10-09T11:06:14.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.735 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.735 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.370396+0000", 2023-10-09T11:06:14.735 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2120000000000003e-06 2023-10-09T11:06:14.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.736 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.736 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.370401+0000", 2023-10-09T11:06:14.736 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2339999999999996e-06 2023-10-09T11:06:14.736 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.736 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.736 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.373051+0000", 2023-10-09T11:06:14.736 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026508730000000002 2023-10-09T11:06:14.736 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.736 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.737 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.373196+0000", 2023-10-09T11:06:14.737 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014440000000000001 2023-10-09T11:06:14.737 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.737 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.737 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T11:06:14.737 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.373347+0000", 2023-10-09T11:06:14.737 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000150944 2023-10-09T11:06:14.737 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.737 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.737 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:14.737 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.378391+0000", 2023-10-09T11:06:14.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0050439589999999998 2023-10-09T11:06:14.738 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.738 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.738 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.738 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.381809+0000", 2023-10-09T11:06:14.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034185560000000001 2023-10-09T11:06:14.738 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.738 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.738 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.738 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.382306+0000", 2023-10-09T11:06:14.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00049662800000000004 2023-10-09T11:06:14.738 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.739 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.739 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.739 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.438530+0000", 2023-10-09T11:06:14.739 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.056223853999999997 2023-10-09T11:06:14.739 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.739 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.739 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.739 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.438587+0000", 2023-10-09T11:06:14.739 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7425e-05 2023-10-09T11:06:14.739 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.739 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.739 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.740 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.438981+0000", 2023-10-09T11:06:14.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00039416599999999999 2023-10-09T11:06:14.740 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.740 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.740 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.740 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.740 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.740 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:184410 10.c 10:33127fe9:::benchmark_data_smithi125_402869_object85841:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:14.740 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:29.371204+0000", 2023-10-09T11:06:14.740 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 105.29876348499999, 2023-10-09T11:06:14.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.068072879000000003, 2023-10-09T11:06:14.740 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.740 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.741 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:14.741 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:14.741 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:14.741 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 184410 2023-10-09T11:06:14.741 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.741 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.741 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.741 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.741 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.371204+0000", 2023-10-09T11:06:14.741 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.741 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.741 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.742 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.742 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.371204+0000", 2023-10-09T11:06:14.742 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.742 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.742 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.742 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.371203+0000", 2023-10-09T11:06:14.742 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:06:14.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.742 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.742 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.742 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.371209+0000", 2023-10-09T11:06:14.742 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8710000000000002e-06 2023-10-09T11:06:14.743 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.743 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.743 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.743 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.371221+0000", 2023-10-09T11:06:14.743 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1844e-05 2023-10-09T11:06:14.743 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.743 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.743 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.743 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.371227+0000", 2023-10-09T11:06:14.743 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1890000000000004e-06 2023-10-09T11:06:14.743 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.743 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.744 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.744 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.375535+0000", 2023-10-09T11:06:14.744 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043073269999999997 2023-10-09T11:06:14.744 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.744 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.744 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.744 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.375628+0000", 2023-10-09T11:06:14.744 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3418999999999994e-05 2023-10-09T11:06:14.744 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.744 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.744 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T11:06:14.744 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.375728+0000", 2023-10-09T11:06:14.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9560000000000002e-05 2023-10-09T11:06:14.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:14.745 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.378399+0000", 2023-10-09T11:06:14.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026711759999999999 2023-10-09T11:06:14.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.745 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.382256+0000", 2023-10-09T11:06:14.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003857227 2023-10-09T11:06:14.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.746 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.382334+0000", 2023-10-09T11:06:14.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7600000000000002e-05 2023-10-09T11:06:14.746 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.746 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.439020+0000", 2023-10-09T11:06:14.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.056686255999999997 2023-10-09T11:06:14.746 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.746 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.747 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.439061+0000", 2023-10-09T11:06:14.747 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0797999999999998e-05 2023-10-09T11:06:14.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.747 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.747 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.439277+0000", 2023-10-09T11:06:14.747 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00021659700000000001 2023-10-09T11:06:14.747 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.747 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.747 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.748 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:184420 10.c 10:31e35a07:::benchmark_data_smithi125_402869_object85851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:14.748 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:29.384860+0000", 2023-10-09T11:06:14.748 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 105.285108083, 2023-10-09T11:06:14.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.054652669000000001, 2023-10-09T11:06:14.748 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.748 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.748 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:14.748 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:14.748 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:14.748 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 184420 2023-10-09T11:06:14.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.748 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.384860+0000", 2023-10-09T11:06:14.749 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.384860+0000", 2023-10-09T11:06:14.749 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.384858+0000", 2023-10-09T11:06:14.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:14.750 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.750 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.750 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.750 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.384866+0000", 2023-10-09T11:06:14.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9410000000000007e-06 2023-10-09T11:06:14.750 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.750 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.750 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.750 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.384882+0000", 2023-10-09T11:06:14.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5778e-05 2023-10-09T11:06:14.750 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.384889+0000", 2023-10-09T11:06:14.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.323e-06 2023-10-09T11:06:14.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.384947+0000", 2023-10-09T11:06:14.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7590000000000003e-05 2023-10-09T11:06:14.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.385090+0000", 2023-10-09T11:06:14.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014359400000000001 2023-10-09T11:06:14.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T11:06:14.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.385235+0000", 2023-10-09T11:06:14.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000144566 2023-10-09T11:06:14.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.387793+0000", 2023-10-09T11:06:14.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0025580970000000001 2023-10-09T11:06:14.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:14.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.387879+0000", 2023-10-09T11:06:14.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6160000000000002e-05 2023-10-09T11:06:14.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.388777+0000", 2023-10-09T11:06:14.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00089828 2023-10-09T11:06:14.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.754 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.439310+0000", 2023-10-09T11:06:14.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.050532170000000001 2023-10-09T11:06:14.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.754 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.439347+0000", 2023-10-09T11:06:14.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7175000000000002e-05 2023-10-09T11:06:14.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.754 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.439512+0000", 2023-10-09T11:06:14.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000165637 2023-10-09T11:06:14.755 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.755 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.755 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.755 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:184989 10.c 10:3680422b:::benchmark_data_smithi125_402869_object86420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:14.755 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:30.180589+0000", 2023-10-09T11:06:14.755 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 104.489379056, 2023-10-09T11:06:14.755 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.04092929, 2023-10-09T11:06:14.755 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.755 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.755 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:14.756 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:14.756 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:14.756 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 184989 2023-10-09T11:06:14.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.756 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.180589+0000", 2023-10-09T11:06:14.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.180589+0000", 2023-10-09T11:06:14.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.757 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.757 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.180588+0000", 2023-10-09T11:06:14.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:14.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.757 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.757 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.180594+0000", 2023-10-09T11:06:14.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3849999999999999e-06 2023-10-09T11:06:14.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.180610+0000", 2023-10-09T11:06:14.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6169999999999999e-05 2023-10-09T11:06:14.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.180617+0000", 2023-10-09T11:06:14.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9929999999999998e-06 2023-10-09T11:06:14.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.187355+0000", 2023-10-09T11:06:14.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0067382919999999999 2023-10-09T11:06:14.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.187488+0000", 2023-10-09T11:06:14.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013230499999999999 2023-10-09T11:06:14.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.760 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T11:06:14.760 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.187918+0000", 2023-10-09T11:06:14.760 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00043073599999999998 2023-10-09T11:06:14.760 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.760 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.760 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:14.760 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.195758+0000", 2023-10-09T11:06:14.760 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0078399999999999997 2023-10-09T11:06:14.760 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.760 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.760 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.760 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.195799+0000", 2023-10-09T11:06:14.760 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0503000000000002e-05 2023-10-09T11:06:14.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.761 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.761 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.761 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.199677+0000", 2023-10-09T11:06:14.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003878355 2023-10-09T11:06:14.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.761 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.761 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.761 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.221189+0000", 2023-10-09T11:06:14.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021511539999999999 2023-10-09T11:06:14.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.221224+0000", 2023-10-09T11:06:14.762 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4830999999999999e-05 2023-10-09T11:06:14.762 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.221518+0000", 2023-10-09T11:06:14.762 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00029439400000000001 2023-10-09T11:06:14.762 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.762 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.762 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.763 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.763 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.763 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:184994 10.c 10:391d7e49:::benchmark_data_smithi125_402869_object86425:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:14.763 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:30.180634+0000", 2023-10-09T11:06:14.763 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 104.489333527, 2023-10-09T11:06:14.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.041132227, 2023-10-09T11:06:14.763 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.763 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.763 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:14.763 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:14.763 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:14.763 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 184994 2023-10-09T11:06:14.764 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.764 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.764 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.764 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.764 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.180634+0000", 2023-10-09T11:06:14.764 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.764 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.764 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.764 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.764 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.180634+0000", 2023-10-09T11:06:14.764 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.764 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.765 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.765 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.765 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.180634+0000", 2023-10-09T11:06:14.765 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:14.765 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.765 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.765 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.765 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.180638+0000", 2023-10-09T11:06:14.765 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0489999999999997e-06 2023-10-09T11:06:14.765 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.765 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.765 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.765 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.180645+0000", 2023-10-09T11:06:14.766 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1339999999999998e-06 2023-10-09T11:06:14.766 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.766 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.766 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.766 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.180650+0000", 2023-10-09T11:06:14.766 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6160000000000004e-06 2023-10-09T11:06:14.766 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.766 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.766 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.766 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.190694+0000", 2023-10-09T11:06:14.766 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010044875999999999 2023-10-09T11:06:14.767 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.767 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.767 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.767 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.190789+0000", 2023-10-09T11:06:14.767 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4918000000000002e-05 2023-10-09T11:06:14.767 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.767 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.767 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T11:06:14.767 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.190885+0000", 2023-10-09T11:06:14.767 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6028999999999998e-05 2023-10-09T11:06:14.767 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.767 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.195829+0000", 2023-10-09T11:06:14.768 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0049436870000000004 2023-10-09T11:06:14.768 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.768 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:14.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.195876+0000", 2023-10-09T11:06:14.768 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7225000000000003e-05 2023-10-09T11:06:14.768 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.768 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.199923+0000", 2023-10-09T11:06:14.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040465459999999998 2023-10-09T11:06:14.769 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.769 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.769 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.769 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.221562+0000", 2023-10-09T11:06:14.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021638898 2023-10-09T11:06:14.769 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.769 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.769 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.769 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.221593+0000", 2023-10-09T11:06:14.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1461999999999998e-05 2023-10-09T11:06:14.769 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.770 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.770 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.770 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.221767+0000", 2023-10-09T11:06:14.770 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017333600000000001 2023-10-09T11:06:14.770 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.770 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.770 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.770 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.770 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.770 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:184999 10.c 10:382525ad:::benchmark_data_smithi125_402869_object86430:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:14.770 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:30.187140+0000", 2023-10-09T11:06:14.770 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 104.482828055, 2023-10-09T11:06:14.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.041790858, 2023-10-09T11:06:14.771 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.771 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.771 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:14.771 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:14.771 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:14.771 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 184999 2023-10-09T11:06:14.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.771 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.187140+0000", 2023-10-09T11:06:14.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.772 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.772 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.772 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.772 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.187140+0000", 2023-10-09T11:06:14.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.772 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.772 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.772 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.772 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.187138+0000", 2023-10-09T11:06:14.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:14.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.187145+0000", 2023-10-09T11:06:14.773 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6719999999999998e-06 2023-10-09T11:06:14.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.187158+0000", 2023-10-09T11:06:14.773 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3174e-05 2023-10-09T11:06:14.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.774 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.774 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.187165+0000", 2023-10-09T11:06:14.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8029999999999999e-06 2023-10-09T11:06:14.774 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.774 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.774 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.774 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.193122+0000", 2023-10-09T11:06:14.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059565529999999998 2023-10-09T11:06:14.774 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.774 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.774 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.775 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.193255+0000", 2023-10-09T11:06:14.775 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000133497 2023-10-09T11:06:14.775 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T11:06:14.775 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.193380+0000", 2023-10-09T11:06:14.775 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012543799999999999 2023-10-09T11:06:14.775 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:14.775 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.195984+0000", 2023-10-09T11:06:14.775 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002603485 2023-10-09T11:06:14.775 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.776 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.776 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.776 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.200444+0000", 2023-10-09T11:06:14.776 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0044596469999999997 2023-10-09T11:06:14.776 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.776 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.776 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.776 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.202652+0000", 2023-10-09T11:06:14.776 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022087389999999999 2023-10-09T11:06:14.776 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.776 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.777 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:14.777 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.228486+0000", 2023-10-09T11:06:14.777 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025833319 2023-10-09T11:06:14.777 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.777 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.777 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.777 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.228543+0000", 2023-10-09T11:06:14.777 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6922000000000002e-05 2023-10-09T11:06:14.777 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.777 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.777 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.777 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:30.228931+0000", 2023-10-09T11:06:14.778 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00038811 2023-10-09T11:06:14.778 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.778 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.778 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.778 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.778 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.778 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36510.0:71924 11.e e515/495 11:78fcd855:::benchmark_data_smithi125_439543_object71923:head v 515'4444, mlcod=515'4442)", 2023-10-09T11:06:14.778 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:04.063769+0000", 2023-10-09T11:06:14.778 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 10.606198915, 2023-10-09T11:06:14.778 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.068036932999999994, 2023-10-09T11:06:14.778 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.778 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.779 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.779 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.779 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.779 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.063769+0000", 2023-10-09T11:06:14.779 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.779 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.779 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.779 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.779 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.063769+0000", 2023-10-09T11:06:14.779 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.779 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.779 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.779 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.780 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.063768+0000", 2023-10-09T11:06:14.780 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:14.780 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.780 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.780 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.780 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.063825+0000", 2023-10-09T11:06:14.780 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6993000000000001e-05 2023-10-09T11:06:14.780 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.780 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.780 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.780 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.063834+0000", 2023-10-09T11:06:14.780 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7949999999999996e-06 2023-10-09T11:06:14.781 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.781 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.781 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.781 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.063838+0000", 2023-10-09T11:06:14.781 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0550000000000001e-06 2023-10-09T11:06:14.781 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.781 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.781 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.781 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.063877+0000", 2023-10-09T11:06:14.781 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8594999999999999e-05 2023-10-09T11:06:14.781 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.781 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.781 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.782 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.063895+0000", 2023-10-09T11:06:14.782 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8170000000000001e-05 2023-10-09T11:06:14.782 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.782 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.782 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.782 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.131770+0000", 2023-10-09T11:06:14.782 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.067875116999999999 2023-10-09T11:06:14.782 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.782 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.782 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.782 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.131806+0000", 2023-10-09T11:06:14.782 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5768e-05 2023-10-09T11:06:14.783 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.783 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.783 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.783 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.783 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.783 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36510.0:71923 11.d e515/505 11:bb4decfe:::benchmark_data_smithi125_439543_object71922:head v 515'4615, mlcod=515'4613)", 2023-10-09T11:06:14.783 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:04.068612+0000", 2023-10-09T11:06:14.783 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 10.601355853999999, 2023-10-09T11:06:14.783 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069457541999999997, 2023-10-09T11:06:14.783 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.783 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.783 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.783 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.784 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.784 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.068612+0000", 2023-10-09T11:06:14.784 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.784 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.784 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.784 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.784 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.068612+0000", 2023-10-09T11:06:14.784 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.784 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.784 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.784 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.784 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.068611+0000", 2023-10-09T11:06:14.785 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:14.785 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.785 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.785 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.785 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.068646+0000", 2023-10-09T11:06:14.785 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5102999999999999e-05 2023-10-09T11:06:14.785 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.785 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.785 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.785 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.131758+0000", 2023-10-09T11:06:14.785 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.063112001000000001 2023-10-09T11:06:14.785 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.785 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.786 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.786 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.131773+0000", 2023-10-09T11:06:14.786 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5396999999999998e-05 2023-10-09T11:06:14.786 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.786 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.786 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.786 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.132015+0000", 2023-10-09T11:06:14.786 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00024144999999999999 2023-10-09T11:06:14.786 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.786 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.786 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.786 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.132032+0000", 2023-10-09T11:06:14.787 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7706e-05 2023-10-09T11:06:14.787 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.787 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.787 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.787 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.138024+0000", 2023-10-09T11:06:14.787 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059912009999999998 2023-10-09T11:06:14.787 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.787 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.787 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.787 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.138070+0000", 2023-10-09T11:06:14.787 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5935e-05 2023-10-09T11:06:14.787 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.788 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.788 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.788 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.788 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.788 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36510.0:71931 11.7 e515/509 11:e391f739:::benchmark_data_smithi125_439543_object71930:head v 515'3807, mlcod=515'3804)", 2023-10-09T11:06:14.788 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:04.074085+0000", 2023-10-09T11:06:14.788 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 10.595882822, 2023-10-09T11:06:14.788 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.061744663999999998, 2023-10-09T11:06:14.788 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.788 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.788 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.788 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.788 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.789 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.074085+0000", 2023-10-09T11:06:14.789 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.789 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.789 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.789 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.789 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.074085+0000", 2023-10-09T11:06:14.789 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.789 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.789 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.789 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.789 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.074084+0000", 2023-10-09T11:06:14.789 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:06:14.790 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.790 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.790 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.790 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.074117+0000", 2023-10-09T11:06:14.790 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3232999999999998e-05 2023-10-09T11:06:14.790 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.790 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.790 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.790 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.131761+0000", 2023-10-09T11:06:14.790 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.057643956000000003 2023-10-09T11:06:14.790 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.131772+0000", 2023-10-09T11:06:14.791 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1066999999999999e-05 2023-10-09T11:06:14.791 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.131829+0000", 2023-10-09T11:06:14.791 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6487000000000003e-05 2023-10-09T11:06:14.791 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.131853+0000", 2023-10-09T11:06:14.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4028000000000001e-05 2023-10-09T11:06:14.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.792 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.135790+0000", 2023-10-09T11:06:14.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039371409999999999 2023-10-09T11:06:14.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.792 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.135830+0000", 2023-10-09T11:06:14.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9727000000000001e-05 2023-10-09T11:06:14.793 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.793 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.793 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.793 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.793 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.793 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36510.0:71932 11.e e515/495 11:7d80c29d:::benchmark_data_smithi125_439543_object71931:head v 515'4445, mlcod=515'4442)", 2023-10-09T11:06:14.793 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:04.081819+0000", 2023-10-09T11:06:14.793 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 10.588148752, 2023-10-09T11:06:14.793 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.057051560000000001, 2023-10-09T11:06:14.793 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:14.793 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:14.793 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:14.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.794 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:14.794 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.081819+0000", 2023-10-09T11:06:14.794 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.794 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:14.794 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.081819+0000", 2023-10-09T11:06:14.794 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:14.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.794 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:14.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.081818+0000", 2023-10-09T11:06:14.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:06:14.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.795 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:14.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.081875+0000", 2023-10-09T11:06:14.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7185000000000002e-05 2023-10-09T11:06:14.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.795 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:14.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.131757+0000", 2023-10-09T11:06:14.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.049882152999999999 2023-10-09T11:06:14.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.796 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:14.796 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.131773+0000", 2023-10-09T11:06:14.796 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5291e-05 2023-10-09T11:06:14.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.796 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:14.796 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.131836+0000", 2023-10-09T11:06:14.796 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3337999999999996e-05 2023-10-09T11:06:14.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.797 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:14.797 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.131857+0000", 2023-10-09T11:06:14.797 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0907000000000002e-05 2023-10-09T11:06:14.797 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.797 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.797 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:14.797 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.138837+0000", 2023-10-09T11:06:14.797 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0069802570000000001 2023-10-09T11:06:14.797 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:14.797 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:14.797 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:14.797 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:04.138871+0000", 2023-10-09T11:06:14.798 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3433000000000003e-05 2023-10-09T11:06:14.798 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.798 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.798 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.798 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:14.798 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:14.798 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:06:14.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:14.800 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:14.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:14.901 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:14.947 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:14.974 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:14.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:14.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:14.994 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 66 16 78456 78440 74.2404 58.375 0.0192884 0.0134623 2023-10-09T11:06:15.078 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:15.080 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T11:06:15.080 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:06:15.081 INFO:teuthology.orchestra.run.smithi125.stdout: "success": "" 2023-10-09T11:06:15.081 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:06:15.100 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:15.157 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:15.239 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:15.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:15.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:15.345 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:15.389 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:15 smithi155 ceph-mon[184854]: pgmap v1272: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 18 MiB/s wr, 288 op/s; 16373/229215 objects degraded (7.143%); 4014/229215 objects misplaced (1.751%); 19 MiB/s, 308 objects/s recovering 2023-10-09T11:06:15.389 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:15 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 7 pgs peering) 2023-10-09T11:06:15.389 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:15.389 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:06:15.394 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:15.425 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:15.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:15.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:15.445 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:15 smithi125 ceph-mon[195507]: pgmap v1272: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 18 MiB/s wr, 288 op/s; 16373/229215 objects degraded (7.143%); 4014/229215 objects misplaced (1.751%); 19 MiB/s, 308 objects/s recovering 2023-10-09T11:06:15.445 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:15 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 7 pgs peering) 2023-10-09T11:06:15.445 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:15.445 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:06:15.448 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:15 smithi125 ceph-mon[200197]: pgmap v1272: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 18 MiB/s wr, 288 op/s; 16373/229215 objects degraded (7.143%); 4014/229215 objects misplaced (1.751%); 19 MiB/s, 308 objects/s recovering 2023-10-09T11:06:15.448 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:15 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 7 pgs peering) 2023-10-09T11:06:15.448 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:15.448 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:06:15.468 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:06:15.530 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:15.579 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:15.601 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:15.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:15.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:15.708 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:15.758 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:15.836 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:15.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:15.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:15.945 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:15.994 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 67 16 79480 79464 74.0866 64 0.0106053 0.0134917 2023-10-09T11:06:16.015 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:16.092 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:16.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:16.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:16.198 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:16.246 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:16.321 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:16.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:16.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:16.427 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:16.467 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:16.491 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:16.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:16.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:16.597 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:16.632 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:06:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:06:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:80060 11.17 11:ead297d1:::benchmark_data_smithi125_439543_object80059:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T11:06:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:16.612358+0000", 2023-10-09T11:06:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.019044266000000001, 2023-10-09T11:06:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019072229999999999, 2023-10-09T11:06:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:16.632 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:06:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:06:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 80060 2023-10-09T11:06:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.612358+0000", 2023-10-09T11:06:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.612358+0000", 2023-10-09T11:06:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.612357+0000", 2023-10-09T11:06:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.612394+0000", 2023-10-09T11:06:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7598999999999997e-05 2023-10-09T11:06:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.612409+0000", 2023-10-09T11:06:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4270999999999999e-05 2023-10-09T11:06:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.635 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.612415+0000", 2023-10-09T11:06:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7390000000000002e-06 2023-10-09T11:06:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.612479+0000", 2023-10-09T11:06:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3408e-05 2023-10-09T11:06:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.612636+0000", 2023-10-09T11:06:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015751 2023-10-09T11:06:16.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:06:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.612803+0000", 2023-10-09T11:06:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016673400000000001 2023-10-09T11:06:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.614848+0000", 2023-10-09T11:06:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002044536 2023-10-09T11:06:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.616577+0000", 2023-10-09T11:06:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0017292570000000001 2023-10-09T11:06:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:80072 11.17 11:eb381be4:::benchmark_data_smithi125_439543_object80071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e521)", 2023-10-09T11:06:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:16.629792+0000", 2023-10-09T11:06:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0016106359999999999, 2023-10-09T11:06:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001742476, 2023-10-09T11:06:16.638 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:06:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:06:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 80072 2023-10-09T11:06:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.629792+0000", 2023-10-09T11:06:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.629792+0000", 2023-10-09T11:06:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.629791+0000", 2023-10-09T11:06:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:06:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:16.640 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.629845+0000", 2023-10-09T11:06:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4469000000000003e-05 2023-10-09T11:06:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.629861+0000", 2023-10-09T11:06:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6186000000000001e-05 2023-10-09T11:06:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.629867+0000", 2023-10-09T11:06:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7130000000000002e-06 2023-10-09T11:06:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.630003+0000", 2023-10-09T11:06:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000136203 2023-10-09T11:06:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.630166+0000", 2023-10-09T11:06:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016235800000000001 2023-10-09T11:06:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:06:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:16.630343+0000", 2023-10-09T11:06:16.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017772099999999999 2023-10-09T11:06:16.643 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:16.643 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:16.643 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:16.643 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:16.643 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T11:06:16.643 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 2 2023-10-09T11:06:16.643 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:06:16.643 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:16.663 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:16.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:16.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:16.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:16.799 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:06:16.815 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:16.840 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:16.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:16.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:16.946 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:16.994 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 68 16 80390 80374 73.8327 56.875 0.0178648 0.0135394 2023-10-09T11:06:17.025 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:17.043 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T11:06:17.067 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:17.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:17.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:17.166 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:06:17.172 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:17.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:17 smithi155 ceph-mon[184854]: pgmap v1273: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 14 MiB/s wr, 231 op/s; 16373/229215 objects degraded (7.143%); 4014/229215 objects misplaced (1.751%); 16 MiB/s, 248 objects/s recovering 2023-10-09T11:06:17.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/690508893' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:06:17.241 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:17.267 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:17.272 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:17.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:17.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:17.376 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:17.389 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:06:17.432 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:17.565 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:17.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:17.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:17.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:17 smithi125 ceph-mon[195507]: pgmap v1273: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 14 MiB/s wr, 231 op/s; 16373/229215 objects degraded (7.143%); 4014/229215 objects misplaced (1.751%); 16 MiB/s, 248 objects/s recovering 2023-10-09T11:06:17.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/690508893' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:06:17.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:17 smithi125 ceph-mon[200197]: pgmap v1273: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 14 MiB/s wr, 231 op/s; 16373/229215 objects degraded (7.143%); 4014/229215 objects misplaced (1.751%); 16 MiB/s, 248 objects/s recovering 2023-10-09T11:06:17.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/690508893' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:06:17.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:17.713 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:17.999 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 69 16 81360 81344 73.6372 60.625 0.0427292 0.013575 2023-10-09T11:06:18.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:18.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:18.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:18.145 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:18.216 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:18.401 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:18.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:18.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:18 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/690508893' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:06:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:18 smithi155 ceph-mon[184854]: osdmap e522: 8 total, 8 up, 8 in 2023-10-09T11:06:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:18 smithi155 ceph-mon[184854]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:18 smithi155 ceph-mon[184854]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:18 smithi155 ceph-mon[184854]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:18 smithi155 ceph-mon[184854]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:18 smithi155 ceph-mon[184854]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:18 smithi155 ceph-mon[184854]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:18 smithi155 ceph-mon[184854]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.507 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:18.542 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:18.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:18.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/690508893' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:06:18.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[195507]: osdmap e522: 8 total, 8 up, 8 in 2023-10-09T11:06:18.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[195507]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[195507]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[195507]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[195507]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[195507]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[195507]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[195507]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:18.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/690508893' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:06:18.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[200197]: osdmap e522: 8 total, 8 up, 8 in 2023-10-09T11:06:18.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[200197]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[200197]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[200197]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[200197]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[200197]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[200197]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:18 smithi125 ceph-mon[200197]: failed to encode map e522 with expected crc 2023-10-09T11:06:18.999 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 70 16 82498 82482 73.6013 71.125 0.0146221 0.013582 2023-10-09T11:06:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:19 smithi155 ceph-mon[184854]: pgmap v1275: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 23 MiB/s wr, 371 op/s; 16373/232656 objects degraded (7.037%); 3638/232656 objects misplaced (1.564%); 17 MiB/s, 273 objects/s recovering 2023-10-09T11:06:19.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:19 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 16373/232656 objects degraded (7.037%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:19.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:19 smithi125 ceph-mon[195507]: pgmap v1275: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 23 MiB/s wr, 371 op/s; 16373/232656 objects degraded (7.037%); 3638/232656 objects misplaced (1.564%); 17 MiB/s, 273 objects/s recovering 2023-10-09T11:06:19.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:19 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 16373/232656 objects degraded (7.037%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:19.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:19 smithi125 ceph-mon[200197]: pgmap v1275: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 23 MiB/s wr, 371 op/s; 16373/232656 objects degraded (7.037%); 3638/232656 objects misplaced (1.564%); 17 MiB/s, 273 objects/s recovering 2023-10-09T11:06:19.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:19 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 16373/232656 objects degraded (7.037%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:19.999 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 71 16 83655 83639 73.5822 72.3125 0.00830046 0.0135845 2023-10-09T11:06:20.084 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:20.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:20.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:20.189 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:20.237 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:20.303 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:20.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:20.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:20.410 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:20.467 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:20.503 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 7, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 5, 6, 7, 1, 3] 2023-10-09T11:06:20.503 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:06:20.503 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T11:06:20.503 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.000000 2023-10-09T11:06:20.504 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 0 2023-10-09T11:06:20.753 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:20.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:20.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:20.863 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:20.899 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:20.923 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:20.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:20.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:21.000 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 72 16 84585 84569 73.3671 58.125 0.0113496 0.0136241 2023-10-09T11:06:21.030 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:21.064 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:21.064 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:21.092 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:21.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:21.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:21.197 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:21.243 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:21.277 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:21.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:21.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:21.288 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:06:21.288 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:06:21.288 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T11:06:21.288 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T11:06:21.288 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:06:21.315 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:21 smithi155 ceph-mon[184854]: pgmap v1276: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s; 16373/243561 objects degraded (6.722%); 3804/243561 objects misplaced (1.562%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T11:06:21.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:21 smithi125 ceph-mon[200197]: pgmap v1276: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s; 16373/243561 objects degraded (6.722%); 3804/243561 objects misplaced (1.562%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T11:06:21.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:21 smithi125 ceph-mon[195507]: pgmap v1276: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s; 16373/243561 objects degraded (6.722%); 3804/243561 objects misplaced (1.562%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T11:06:21.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:06:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:06:21] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.43.0" 2023-10-09T11:06:21.382 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:21.437 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:21.529 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:21.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:21.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:21.632 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T11:06:21.637 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:21.699 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:21.838 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:21.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:21.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:21.946 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:22.002 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 73 16 85531 85515 73.1703 59.125 0.0108269 0.0136616 2023-10-09T11:06:22.018 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:22.029 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:06:22.169 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:06:22.210 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:22.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:22.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:22.224 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:06:22.261 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:06:22.306 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:06:22.319 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:22.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:22 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4278638459' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:22.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:22 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4278638459' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:22.341 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:06:22.363 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:22.376 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:06:22.388 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:22.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:22.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:22.411 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:22 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4278638459' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:22.412 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:06:22.456 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:06:22.476 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:06:22.476 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:06:22.476 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:06:22.477 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:06:22.477 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:06:22.477 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:06:22.477 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:06:22.477 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:06:22.492 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:22.560 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:22.595 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:22.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:22.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:22.700 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:22.749 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:22.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:06:22.939 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:22.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:22.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:23.002 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 74 16 86364 86348 72.8852 52.0625 0.00691977 0.0137145 2023-10-09T11:06:23.046 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:23.078 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:23.237 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:23.291 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.0 primary-affinity to 0 (802) 2023-10-09T11:06:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:23 smithi155 ceph-mon[184854]: pgmap v1277: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 16373/247299 objects degraded (6.621%); 3391/247299 objects misplaced (1.371%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T11:06:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:23 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:06:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:23 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3280981817' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-10-09T11:06:23.557 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:23 smithi125 ceph-mon[195507]: pgmap v1277: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 16373/247299 objects degraded (6.621%); 3391/247299 objects misplaced (1.371%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T11:06:23.558 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:23 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:06:23.558 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:23 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3280981817' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-10-09T11:06:23.558 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:23 smithi125 ceph-mon[200197]: pgmap v1277: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 16373/247299 objects degraded (6.621%); 3391/247299 objects misplaced (1.371%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T11:06:23.558 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:23 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:06:23.558 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:23 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3280981817' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-10-09T11:06:23.782 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:23.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:23.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:23.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:23.915 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:24.002 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 75 16 86813 86797 72.2878 28.0625 0.00257984 0.0137249 2023-10-09T11:06:24.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:24.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:24.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:24.189 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:24.220 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:24.247 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:24 smithi125 ceph-mon[195507]: failed to encode map e523 with expected crc 2023-10-09T11:06:24.247 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:24 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3280981817' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-10-09T11:06:24.247 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:24 smithi125 ceph-mon[195507]: osdmap e523: 8 total, 8 up, 8 in 2023-10-09T11:06:24.247 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:24 smithi125 ceph-mon[195507]: failed to encode map e523 with expected crc 2023-10-09T11:06:24.247 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:24 smithi125 ceph-mon[195507]: failed to encode map e523 with expected crc 2023-10-09T11:06:24.247 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:24 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 16373/247299 objects degraded (6.621%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:24.247 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:24 smithi125 ceph-mon[195507]: osdmap e524: 8 total, 8 up, 8 in 2023-10-09T11:06:24.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:24 smithi155 ceph-mon[184854]: failed to encode map e523 with expected crc 2023-10-09T11:06:24.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:24 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3280981817' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-10-09T11:06:24.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:24 smithi155 ceph-mon[184854]: osdmap e523: 8 total, 8 up, 8 in 2023-10-09T11:06:24.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:24 smithi155 ceph-mon[184854]: failed to encode map e523 with expected crc 2023-10-09T11:06:24.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:24 smithi155 ceph-mon[184854]: failed to encode map e523 with expected crc 2023-10-09T11:06:24.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:24 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 16373/247299 objects degraded (6.621%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:24.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:24 smithi155 ceph-mon[184854]: osdmap e524: 8 total, 8 up, 8 in 2023-10-09T11:06:24.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:24 smithi125 ceph-mon[200197]: failed to encode map e523 with expected crc 2023-10-09T11:06:24.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:24 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3280981817' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-10-09T11:06:24.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:24 smithi125 ceph-mon[200197]: osdmap e523: 8 total, 8 up, 8 in 2023-10-09T11:06:24.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:24 smithi125 ceph-mon[200197]: failed to encode map e523 with expected crc 2023-10-09T11:06:24.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:24 smithi125 ceph-mon[200197]: failed to encode map e523 with expected crc 2023-10-09T11:06:24.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:24 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 16373/247299 objects degraded (6.621%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:24.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:24 smithi125 ceph-mon[200197]: osdmap e524: 8 total, 8 up, 8 in 2023-10-09T11:06:24.599 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:24.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:24.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:24.704 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:24.751 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:24.777 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:24.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:24.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:24.882 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:06:24.882 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:06:24.882 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:24.938 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:24.987 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:06:25.004 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 76 16 87486 87470 71.8891 42.0625 0.014181 0.0139055 2023-10-09T11:06:25.356 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:06:25.417 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:25.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:25.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:25.521 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:25.568 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:25 smithi155 ceph-mon[184854]: failed to encode map e523 with expected crc 2023-10-09T11:06:25.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:25 smithi155 ceph-mon[184854]: pgmap v1279: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 5.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 16373/257598 objects degraded (6.356%); 3557/257598 objects misplaced (1.381%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T11:06:25.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:25 smithi155 ceph-mon[184854]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:25 smithi155 ceph-mon[184854]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:25 smithi155 ceph-mon[184854]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:25 smithi155 ceph-mon[184854]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:25 smithi155 ceph-mon[184854]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:25 smithi155 ceph-mon[184854]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:25 smithi155 ceph-mon[184854]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:25 smithi155 ceph-mon[184854]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:25 smithi155 ceph-mon[184854]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2755682356' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:25.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:25 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:25.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[200197]: failed to encode map e523 with expected crc 2023-10-09T11:06:25.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[200197]: pgmap v1279: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 5.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 16373/257598 objects degraded (6.356%); 3557/257598 objects misplaced (1.381%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T11:06:25.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[200197]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[200197]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[200197]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[200197]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[200197]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[200197]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[200197]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[200197]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[200197]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2755682356' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:25.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[195507]: failed to encode map e523 with expected crc 2023-10-09T11:06:25.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[195507]: pgmap v1279: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 5.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 16373/257598 objects degraded (6.356%); 3557/257598 objects misplaced (1.381%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T11:06:25.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[195507]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[195507]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[195507]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[195507]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[195507]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[195507]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[195507]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[195507]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[195507]: failed to encode map e524 with expected crc 2023-10-09T11:06:25.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2755682356' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:25.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:25 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:25.573 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:25.612 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:25.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:25.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:25.717 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:25.750 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:25.786 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T11:06:25.833 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:25.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:25.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:25.942 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:25.992 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:26.004 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 77 16 88480 88464 71.7621 62.125 0.0250105 0.0139305 2023-10-09T11:06:26.030 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:26.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:26.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:26.136 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:26.195 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:26.310 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:06:26.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:26 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:26.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:26 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:06:26.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:26 smithi155 ceph-mon[184854]: osdmap e525: 8 total, 8 up, 8 in 2023-10-09T11:06:26.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:26 smithi155 ceph-mon[184854]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:26 smithi155 ceph-mon[184854]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:26 smithi155 ceph-mon[184854]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:26 smithi155 ceph-mon[184854]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:26 smithi155 ceph-mon[184854]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:26 smithi155 ceph-mon[184854]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.494 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:26.494 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:06:26.495 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[195507]: osdmap e525: 8 total, 8 up, 8 in 2023-10-09T11:06:26.495 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[195507]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.495 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[195507]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.495 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[195507]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.495 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[195507]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.495 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[195507]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.495 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[195507]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.495 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:26.495 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:06:26.496 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[200197]: osdmap e525: 8 total, 8 up, 8 in 2023-10-09T11:06:26.496 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[200197]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.496 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[200197]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.496 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[200197]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.496 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[200197]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.496 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[200197]: failed to encode map e525 with expected crc 2023-10-09T11:06:26.496 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:26 smithi125 ceph-mon[200197]: failed to encode map e525 with expected crc 2023-10-09T11:06:27.004 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 78 16 89492 89476 71.653 63.25 0.0197806 0.0139519 2023-10-09T11:06:27.224 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:27.290 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:27.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:27.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:27.403 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:27.469 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:27.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:27 smithi125 ceph-mon[195507]: pgmap v1282: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 5.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s; 16373/257598 objects degraded (6.356%); 3557/257598 objects misplaced (1.381%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T11:06:27.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:27 smithi125 ceph-mon[200197]: pgmap v1282: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 5.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s; 16373/257598 objects degraded (6.356%); 3557/257598 objects misplaced (1.381%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T11:06:27.556 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:27 smithi155 ceph-mon[184854]: pgmap v1282: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 117 active+clean; 5.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s; 16373/257598 objects degraded (6.356%); 3557/257598 objects misplaced (1.381%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T11:06:27.594 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:27.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:27.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:27.699 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:27.749 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:27.795 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:06:27.796 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:06:28.005 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 79 16 90420 90404 71.479 58 0.0427996 0.0139847 2023-10-09T11:06:28.082 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:28.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:28.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:28.186 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:28.253 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:28.308 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:28 smithi125 ceph-mon[195507]: pgmap v1283: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 116 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 7.0 KiB/s rd, 50 MiB/s wr, 808 op/s; 16373/261765 objects degraded (6.255%); 2932/261765 objects misplaced (1.120%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T11:06:28.437 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:28.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:28 smithi155 ceph-mon[184854]: pgmap v1283: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 116 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 7.0 KiB/s rd, 50 MiB/s wr, 808 op/s; 16373/261765 objects degraded (6.255%); 2932/261765 objects misplaced (1.120%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T11:06:28.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:28.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:28.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:28 smithi125 ceph-mon[200197]: pgmap v1283: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 116 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 7.0 KiB/s rd, 50 MiB/s wr, 808 op/s; 16373/261765 objects degraded (6.255%); 2932/261765 objects misplaced (1.120%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T11:06:28.578 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:28.631 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:28.663 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:28.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:28.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:28.755 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 7, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 5, 6, 7, 1, 3] 2023-10-09T11:06:28.755 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:06:28.756 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2023-10-09T11:06:28.757 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:06:28.768 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:28.805 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:28.841 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:06:28.889 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:06:28.924 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:06:28.936 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T11:06:28.985 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:06:29.006 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T11:06:29.007194+0000 min lat: 0.00101021 max lat: 4.7606 avg lat: 0.0140317 2023-10-09T11:06:29.007 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T11:06:29.007 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 80 16 91253 91237 71.2354 52.0625 0.0180155 0.0140317 2023-10-09T11:06:29.032 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:06:29.082 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:29.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:29.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:29.093 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:06:29.129 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:06:29.168 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:06:29.182 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:06:29.182 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:06:29.182 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:06:29.182 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:06:29.183 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:06:29.183 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:06:29.183 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:06:29.183 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:06:29.189 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:29.246 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:29.321 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:06:29.339 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:29.342 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:29 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 16373/261765 objects degraded (6.255%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:29.342 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:29 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/668925041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:29.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:29.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:29.444 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:29.481 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:29.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:29 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 16373/261765 objects degraded (6.255%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:29.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:29 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/668925041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:29.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:29 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 16373/261765 objects degraded (6.255%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:29.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:29 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/668925041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:29.615 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:29.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:29.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:29.644 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:06:29.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:29.748 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:30.006 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 81 16 91876 91860 70.8368 38.9375 0.00402833 0.0140622 2023-10-09T11:06:30.069 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:30.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:30.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:30.174 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:30.205 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:30.276 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:30.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:30.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:30.380 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:30.429 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:30.507 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:30.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:30.515 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:30 smithi155 ceph-mon[184854]: failed to encode map e526 with expected crc 2023-10-09T11:06:30.515 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:30 smithi155 ceph-mon[184854]: osdmap e526: 8 total, 8 up, 8 in 2023-10-09T11:06:30.515 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:30 smithi155 ceph-mon[184854]: failed to encode map e526 with expected crc 2023-10-09T11:06:30.515 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:30 smithi155 ceph-mon[184854]: failed to encode map e526 with expected crc 2023-10-09T11:06:30.515 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:30.515 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:30.515 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:30 smithi155 ceph-mon[184854]: pgmap v1285: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 116 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 7.0 KiB/s rd, 44 MiB/s wr, 708 op/s; 16373/270264 objects degraded (6.058%); 3065/270264 objects misplaced (1.134%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T11:06:30.515 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:30.515 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:06:30.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:30.605 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[195507]: failed to encode map e526 with expected crc 2023-10-09T11:06:30.605 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[195507]: osdmap e526: 8 total, 8 up, 8 in 2023-10-09T11:06:30.605 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[195507]: failed to encode map e526 with expected crc 2023-10-09T11:06:30.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[195507]: failed to encode map e526 with expected crc 2023-10-09T11:06:30.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:30.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:30.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[195507]: pgmap v1285: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 116 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 7.0 KiB/s rd, 44 MiB/s wr, 708 op/s; 16373/270264 objects degraded (6.058%); 3065/270264 objects misplaced (1.134%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T11:06:30.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:30.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:06:30.608 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[200197]: failed to encode map e526 with expected crc 2023-10-09T11:06:30.609 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[200197]: osdmap e526: 8 total, 8 up, 8 in 2023-10-09T11:06:30.609 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[200197]: failed to encode map e526 with expected crc 2023-10-09T11:06:30.609 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[200197]: failed to encode map e526 with expected crc 2023-10-09T11:06:30.609 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:30.609 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:30.609 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[200197]: pgmap v1285: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 116 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 7.0 KiB/s rd, 44 MiB/s wr, 708 op/s; 16373/270264 objects degraded (6.058%); 3065/270264 objects misplaced (1.134%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T11:06:30.609 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:30.609 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:06:30.617 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:30.658 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:30.679 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:30.703 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:30.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:30.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:30.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:30.817 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:30.820 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:06:30.820 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:06:30.820 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T11:06:30.820 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T11:06:30.820 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:06:30.905 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:30.938 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:30.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:30.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:31.006 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 82 16 92524 92508 70.467 40.5 0.00875654 0.0141742 2023-10-09T11:06:31.043 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:31.072 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:31.201 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:31.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:31.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:31.215 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T11:06:31.220 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:06:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:06:31] "GET /metrics HTTP/1.1" 200 36586 "" "Prometheus/2.43.0" 2023-10-09T11:06:31.305 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:31.344 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:31.376 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:31.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:31 smithi155 ceph-mon[184854]: failed to encode map e527 with expected crc 2023-10-09T11:06:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:31 smithi155 ceph-mon[184854]: osdmap e527: 8 total, 8 up, 8 in 2023-10-09T11:06:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:31 smithi155 ceph-mon[184854]: failed to encode map e527 with expected crc 2023-10-09T11:06:31.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:31 smithi155 ceph-mon[184854]: failed to encode map e527 with expected crc 2023-10-09T11:06:31.479 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:31.510 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:31 smithi125 ceph-mon[195507]: failed to encode map e527 with expected crc 2023-10-09T11:06:31.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:31 smithi125 ceph-mon[195507]: osdmap e527: 8 total, 8 up, 8 in 2023-10-09T11:06:31.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:31 smithi125 ceph-mon[195507]: failed to encode map e527 with expected crc 2023-10-09T11:06:31.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:31 smithi125 ceph-mon[195507]: failed to encode map e527 with expected crc 2023-10-09T11:06:31.511 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:31 smithi125 ceph-mon[200197]: failed to encode map e527 with expected crc 2023-10-09T11:06:31.511 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:31 smithi125 ceph-mon[200197]: osdmap e527: 8 total, 8 up, 8 in 2023-10-09T11:06:31.511 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:31 smithi125 ceph-mon[200197]: failed to encode map e527 with expected crc 2023-10-09T11:06:31.511 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:31 smithi125 ceph-mon[200197]: failed to encode map e527 with expected crc 2023-10-09T11:06:31.542 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:31.604 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:06:32.006 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 83 16 93431 93415 70.301 56.6875 0.0188281 0.014219 2023-10-09T11:06:32.391 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:32.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:32.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:32 smithi155 ceph-mon[184854]: failed to encode map e528 with expected crc 2023-10-09T11:06:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:32 smithi155 ceph-mon[184854]: osdmap e528: 8 total, 8 up, 8 in 2023-10-09T11:06:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:32 smithi155 ceph-mon[184854]: failed to encode map e528 with expected crc 2023-10-09T11:06:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:32 smithi155 ceph-mon[184854]: failed to encode map e528 with expected crc 2023-10-09T11:06:32.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:32 smithi155 ceph-mon[184854]: pgmap v1288: 129 pgs: 2 active+recovery_wait, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 118 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 7.0 KiB/s rd, 56 MiB/s wr, 907 op/s; 16059/273840 objects degraded (5.864%); 2898/273840 objects misplaced (1.058%); 12 MiB/s, 183 objects/s recovering 2023-10-09T11:06:32.501 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:32.549 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:32.573 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:32.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:32.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:32.637 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:06:32.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:32 smithi125 ceph-mon[195507]: failed to encode map e528 with expected crc 2023-10-09T11:06:32.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:32 smithi125 ceph-mon[195507]: osdmap e528: 8 total, 8 up, 8 in 2023-10-09T11:06:32.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:32 smithi125 ceph-mon[195507]: failed to encode map e528 with expected crc 2023-10-09T11:06:32.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:32 smithi125 ceph-mon[195507]: failed to encode map e528 with expected crc 2023-10-09T11:06:32.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:32 smithi125 ceph-mon[195507]: pgmap v1288: 129 pgs: 2 active+recovery_wait, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 118 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 7.0 KiB/s rd, 56 MiB/s wr, 907 op/s; 16059/273840 objects degraded (5.864%); 2898/273840 objects misplaced (1.058%); 12 MiB/s, 183 objects/s recovering 2023-10-09T11:06:32.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:32 smithi125 ceph-mon[200197]: failed to encode map e528 with expected crc 2023-10-09T11:06:32.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:32 smithi125 ceph-mon[200197]: osdmap e528: 8 total, 8 up, 8 in 2023-10-09T11:06:32.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:32 smithi125 ceph-mon[200197]: failed to encode map e528 with expected crc 2023-10-09T11:06:32.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:32 smithi125 ceph-mon[200197]: failed to encode map e528 with expected crc 2023-10-09T11:06:32.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:32 smithi125 ceph-mon[200197]: pgmap v1288: 129 pgs: 2 active+recovery_wait, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 118 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 7.0 KiB/s rd, 56 MiB/s wr, 907 op/s; 16059/273840 objects degraded (5.864%); 2898/273840 objects misplaced (1.058%); 12 MiB/s, 183 objects/s recovering 2023-10-09T11:06:32.676 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:32.768 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:06:32.768 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T11:06:32.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T11:06:32.769 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:06:32.769 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.769 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:15613 10.a 10:5bf56187:::benchmark_data_smithi125_402869_object15612:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T11:06:32.769 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:24.488022+0000", 2023-10-09T11:06:32.769 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 308.27785094000001, 2023-10-09T11:06:32.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10172471299999999, 2023-10-09T11:06:32.769 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.769 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.769 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.769 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.769 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.770 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 15613 2023-10-09T11:06:32.770 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.770 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.770 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.770 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.770 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488022+0000", 2023-10-09T11:06:32.770 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.770 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.770 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.770 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.770 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488022+0000", 2023-10-09T11:06:32.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488022+0000", 2023-10-09T11:06:32.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:32.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488055+0000", 2023-10-09T11:06:32.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3102999999999998e-05 2023-10-09T11:06:32.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.772 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.772 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488065+0000", 2023-10-09T11:06:32.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0101e-05 2023-10-09T11:06:32.772 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.772 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.772 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.772 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488070+0000", 2023-10-09T11:06:32.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2909999999999998e-06 2023-10-09T11:06:32.772 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.772 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.772 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.772 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488112+0000", 2023-10-09T11:06:32.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1662000000000002e-05 2023-10-09T11:06:32.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488280+0000", 2023-10-09T11:06:32.773 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016848699999999999 2023-10-09T11:06:32.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:06:32.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488418+0000", 2023-10-09T11:06:32.773 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013803700000000001 2023-10-09T11:06:32.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.774 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.491088+0000", 2023-10-09T11:06:32.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026694919999999999 2023-10-09T11:06:32.774 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.774 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.774 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.774 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.491804+0000", 2023-10-09T11:06:32.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00071589899999999998 2023-10-09T11:06:32.774 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.774 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.774 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.774 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.589653+0000", 2023-10-09T11:06:32.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.097848911999999996 2023-10-09T11:06:32.775 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.775 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.589707+0000", 2023-10-09T11:06:32.775 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4593999999999999e-05 2023-10-09T11:06:32.775 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.775 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.589747+0000", 2023-10-09T11:06:32.775 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9787999999999999e-05 2023-10-09T11:06:32.775 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.775 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.775 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.776 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.776 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.776 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:27693 10.0 10:017bad4c:::benchmark_data_smithi125_402869_object27692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:06:32.776 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:34.474346+0000", 2023-10-09T11:06:32.776 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 298.291527476, 2023-10-09T11:06:32.776 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14285995000000001, 2023-10-09T11:06:32.776 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.776 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.776 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.776 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.776 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.776 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 27693 2023-10-09T11:06:32.776 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.777 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.777 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.777 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.777 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474346+0000", 2023-10-09T11:06:32.777 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.777 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.777 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.777 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.777 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474346+0000", 2023-10-09T11:06:32.777 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.777 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.778 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.778 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.778 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474345+0000", 2023-10-09T11:06:32.778 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:32.778 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.778 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.778 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.778 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474386+0000", 2023-10-09T11:06:32.778 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1455000000000002e-05 2023-10-09T11:06:32.778 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.778 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.778 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.779 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474401+0000", 2023-10-09T11:06:32.779 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5238e-05 2023-10-09T11:06:32.779 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.779 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.779 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.779 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474408+0000", 2023-10-09T11:06:32.779 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4489999999999996e-06 2023-10-09T11:06:32.779 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.779 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.779 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.779 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474455+0000", 2023-10-09T11:06:32.779 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7719999999999997e-05 2023-10-09T11:06:32.779 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.780 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.780 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.780 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474565+0000", 2023-10-09T11:06:32.780 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010948200000000001 2023-10-09T11:06:32.780 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.780 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.780 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:06:32.780 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474666+0000", 2023-10-09T11:06:32.780 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010145399999999999 2023-10-09T11:06:32.780 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.780 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.780 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.781 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.476207+0000", 2023-10-09T11:06:32.781 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015401950000000001 2023-10-09T11:06:32.781 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.781 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.781 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.781 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.476567+0000", 2023-10-09T11:06:32.781 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00036023899999999999 2023-10-09T11:06:32.781 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.781 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.781 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.781 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617122+0000", 2023-10-09T11:06:32.781 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14055489700000001 2023-10-09T11:06:32.781 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.782 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.782 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.782 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617168+0000", 2023-10-09T11:06:32.782 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6669000000000003e-05 2023-10-09T11:06:32.782 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.782 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.782 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.782 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617206+0000", 2023-10-09T11:06:32.782 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7505000000000001e-05 2023-10-09T11:06:32.782 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.782 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.782 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.782 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.783 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.783 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:27705 10.0 10:03463076:::benchmark_data_smithi125_402869_object27704:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:06:32.783 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:34.483244+0000", 2023-10-09T11:06:32.783 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 298.282629821, 2023-10-09T11:06:32.783 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13409575700000001, 2023-10-09T11:06:32.783 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.783 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.783 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.783 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.783 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.783 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 27705 2023-10-09T11:06:32.783 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.783 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.784 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.784 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.784 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483244+0000", 2023-10-09T11:06:32.784 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.784 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.784 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.784 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.784 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483244+0000", 2023-10-09T11:06:32.784 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.784 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.784 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.784 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.784 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483242+0000", 2023-10-09T11:06:32.785 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:06:32.785 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.785 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.785 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.785 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483306+0000", 2023-10-09T11:06:32.785 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3712000000000003e-05 2023-10-09T11:06:32.785 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.785 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.785 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.785 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483319+0000", 2023-10-09T11:06:32.785 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2860999999999999e-05 2023-10-09T11:06:32.785 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.786 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.786 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.786 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483325+0000", 2023-10-09T11:06:32.786 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0889999999999996e-06 2023-10-09T11:06:32.786 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.786 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.786 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.786 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483360+0000", 2023-10-09T11:06:32.786 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5259999999999998e-05 2023-10-09T11:06:32.786 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.786 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.786 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.787 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483505+0000", 2023-10-09T11:06:32.787 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014444300000000001 2023-10-09T11:06:32.787 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.787 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.787 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:06:32.787 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483657+0000", 2023-10-09T11:06:32.787 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151726 2023-10-09T11:06:32.787 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.787 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.787 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.787 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.484799+0000", 2023-10-09T11:06:32.787 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001142503 2023-10-09T11:06:32.788 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.788 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.788 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.788 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.485118+0000", 2023-10-09T11:06:32.788 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000318676 2023-10-09T11:06:32.788 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.788 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.788 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.788 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617247+0000", 2023-10-09T11:06:32.788 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.132128987 2023-10-09T11:06:32.788 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.788 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.788 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.789 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617299+0000", 2023-10-09T11:06:32.789 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1867000000000003e-05 2023-10-09T11:06:32.789 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.789 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.789 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.789 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617339+0000", 2023-10-09T11:06:32.789 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0726999999999998e-05 2023-10-09T11:06:32.789 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.789 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.789 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.789 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.789 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.789 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:27738 10.0 10:0a0f1cf2:::benchmark_data_smithi125_402869_object27737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:06:32.790 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:34.503650+0000", 2023-10-09T11:06:32.790 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 298.262223119, 2023-10-09T11:06:32.790 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.115650268, 2023-10-09T11:06:32.790 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.790 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.790 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.790 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.790 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.790 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 27738 2023-10-09T11:06:32.790 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.790 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.790 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.790 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503650+0000", 2023-10-09T11:06:32.791 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.791 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503650+0000", 2023-10-09T11:06:32.791 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.791 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503649+0000", 2023-10-09T11:06:32.791 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:06:32.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.792 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503716+0000", 2023-10-09T11:06:32.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7002999999999995e-05 2023-10-09T11:06:32.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.792 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503726+0000", 2023-10-09T11:06:32.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9170000000000001e-06 2023-10-09T11:06:32.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.792 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.793 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503732+0000", 2023-10-09T11:06:32.793 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2009999999999996e-06 2023-10-09T11:06:32.793 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.793 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.793 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.793 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503776+0000", 2023-10-09T11:06:32.793 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4100000000000001e-05 2023-10-09T11:06:32.793 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.793 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.793 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.793 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503892+0000", 2023-10-09T11:06:32.793 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115704 2023-10-09T11:06:32.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.794 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:06:32.794 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.504031+0000", 2023-10-09T11:06:32.794 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013921300000000001 2023-10-09T11:06:32.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.794 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.794 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.505195+0000", 2023-10-09T11:06:32.794 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001163765 2023-10-09T11:06:32.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.505623+0000", 2023-10-09T11:06:32.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00042792200000000001 2023-10-09T11:06:32.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.795 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.619243+0000", 2023-10-09T11:06:32.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11362040900000001 2023-10-09T11:06:32.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.795 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.619272+0000", 2023-10-09T11:06:32.796 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8118000000000001e-05 2023-10-09T11:06:32.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.796 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.796 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.619301+0000", 2023-10-09T11:06:32.796 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8958e-05 2023-10-09T11:06:32.796 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.796 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.796 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.797 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:34496 10.e 10:7e8e0fe7:::benchmark_data_smithi125_402869_object34495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:06:32.797 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:39.324998+0000", 2023-10-09T11:06:32.797 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 293.44087502500003, 2023-10-09T11:06:32.797 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.098875269000000002, 2023-10-09T11:06:32.797 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.797 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.797 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.797 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.797 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.797 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 34496 2023-10-09T11:06:32.797 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.797 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.798 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.798 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.798 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.324998+0000", 2023-10-09T11:06:32.798 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.798 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.798 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.798 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.798 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.324998+0000", 2023-10-09T11:06:32.798 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.798 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.798 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.798 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.799 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.324998+0000", 2023-10-09T11:06:32.801 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:32.801 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.801 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.801 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.801 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325045+0000", 2023-10-09T11:06:32.801 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6817000000000003e-05 2023-10-09T11:06:32.802 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.802 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.802 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.802 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325053+0000", 2023-10-09T11:06:32.802 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8709999999999997e-06 2023-10-09T11:06:32.802 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.802 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.802 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.802 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325057+0000", 2023-10-09T11:06:32.802 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6120000000000001e-06 2023-10-09T11:06:32.802 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.802 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.803 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.803 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325108+0000", 2023-10-09T11:06:32.803 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1187000000000003e-05 2023-10-09T11:06:32.803 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.803 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.803 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.803 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325227+0000", 2023-10-09T11:06:32.803 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118689 2023-10-09T11:06:32.803 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.803 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.803 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,4", 2023-10-09T11:06:32.803 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325363+0000", 2023-10-09T11:06:32.804 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000135735 2023-10-09T11:06:32.804 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.804 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.804 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.804 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327640+0000", 2023-10-09T11:06:32.804 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022776680000000001 2023-10-09T11:06:32.804 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.804 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.804 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.804 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327910+0000", 2023-10-09T11:06:32.804 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00026939599999999998 2023-10-09T11:06:32.804 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.805 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.805 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.423777+0000", 2023-10-09T11:06:32.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.095867252999999999 2023-10-09T11:06:32.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.805 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.805 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.423833+0000", 2023-10-09T11:06:32.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5897999999999999e-05 2023-10-09T11:06:32.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.805 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.805 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.423874+0000", 2023-10-09T11:06:32.806 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0713000000000001e-05 2023-10-09T11:06:32.806 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.806 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.806 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.806 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.806 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.806 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:34497 10.a 10:5b894bd9:::benchmark_data_smithi125_402869_object34496:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:06:32.806 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:39.325075+0000", 2023-10-09T11:06:32.806 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 293.44079838900001, 2023-10-09T11:06:32.806 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.099410517000000004, 2023-10-09T11:06:32.806 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.806 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.807 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.807 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.807 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.807 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 34497 2023-10-09T11:06:32.807 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.807 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.807 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.807 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.807 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325075+0000", 2023-10-09T11:06:32.807 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.807 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.807 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325075+0000", 2023-10-09T11:06:32.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325074+0000", 2023-10-09T11:06:32.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:32.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325125+0000", 2023-10-09T11:06:32.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0782999999999998e-05 2023-10-09T11:06:32.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325144+0000", 2023-10-09T11:06:32.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9225e-05 2023-10-09T11:06:32.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325149+0000", 2023-10-09T11:06:32.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4429999999999996e-06 2023-10-09T11:06:32.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.810 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.810 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325342+0000", 2023-10-09T11:06:32.810 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00019310200000000001 2023-10-09T11:06:32.810 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.810 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.810 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.810 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325474+0000", 2023-10-09T11:06:32.810 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013155999999999999 2023-10-09T11:06:32.810 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.810 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.810 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:06:32.810 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325610+0000", 2023-10-09T11:06:32.811 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000136954 2023-10-09T11:06:32.811 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.811 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.811 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.811 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327774+0000", 2023-10-09T11:06:32.811 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021631739999999999 2023-10-09T11:06:32.811 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.811 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.811 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.811 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327960+0000", 2023-10-09T11:06:32.811 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018653499999999999 2023-10-09T11:06:32.811 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.812 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.812 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.812 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424397+0000", 2023-10-09T11:06:32.812 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.096436500999999994 2023-10-09T11:06:32.812 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.812 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.812 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.812 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424439+0000", 2023-10-09T11:06:32.812 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2216999999999999e-05 2023-10-09T11:06:32.812 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.812 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.812 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.813 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424486+0000", 2023-10-09T11:06:32.813 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6650000000000002e-05 2023-10-09T11:06:32.813 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.813 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.813 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.813 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.813 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.813 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:34498 10.0 10:05649a4c:::benchmark_data_smithi125_402869_object34497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:06:32.813 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:39.325163+0000", 2023-10-09T11:06:32.813 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 293.44071065100002, 2023-10-09T11:06:32.813 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.100486519, 2023-10-09T11:06:32.813 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.814 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.814 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.814 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.814 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.814 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 34498 2023-10-09T11:06:32.814 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.814 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.814 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.814 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.814 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325163+0000", 2023-10-09T11:06:32.814 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.814 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.815 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.815 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.815 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325163+0000", 2023-10-09T11:06:32.815 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.815 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.815 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.815 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.815 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325162+0000", 2023-10-09T11:06:32.815 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:32.815 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.815 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.815 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.815 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325219+0000", 2023-10-09T11:06:32.816 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6532e-05 2023-10-09T11:06:32.816 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.816 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.816 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.816 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325227+0000", 2023-10-09T11:06:32.816 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8269999999999998e-06 2023-10-09T11:06:32.816 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.816 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.816 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.816 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325233+0000", 2023-10-09T11:06:32.816 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1909999999999997e-06 2023-10-09T11:06:32.816 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.817 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.817 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.817 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.326614+0000", 2023-10-09T11:06:32.817 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00138123 2023-10-09T11:06:32.817 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.817 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.817 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.817 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.326735+0000", 2023-10-09T11:06:32.817 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012105800000000001 2023-10-09T11:06:32.817 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.817 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.817 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:06:32.818 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.326862+0000", 2023-10-09T11:06:32.818 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000126699 2023-10-09T11:06:32.818 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.818 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.818 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.818 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.328648+0000", 2023-10-09T11:06:32.818 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0017860949999999999 2023-10-09T11:06:32.818 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.818 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.818 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.818 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.329261+0000", 2023-10-09T11:06:32.818 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00061273000000000002 2023-10-09T11:06:32.819 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.819 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.819 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.819 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.330863+0000", 2023-10-09T11:06:32.819 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001602612 2023-10-09T11:06:32.819 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.819 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.819 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.819 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.330908+0000", 2023-10-09T11:06:32.819 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4719000000000003e-05 2023-10-09T11:06:32.819 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.819 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.820 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.820 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.425649+0000", 2023-10-09T11:06:32.820 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.094741362999999995 2023-10-09T11:06:32.820 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.820 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.820 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.820 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.820 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.820 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:34499 10.a 10:5b46ec36:::benchmark_data_smithi125_402869_object34498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:06:32.820 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:39.325241+0000", 2023-10-09T11:06:32.820 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 293.440632698, 2023-10-09T11:06:32.820 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.099333877000000001, 2023-10-09T11:06:32.821 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.821 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.821 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.821 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.821 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.821 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 34499 2023-10-09T11:06:32.821 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.821 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.821 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.821 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.821 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325241+0000", 2023-10-09T11:06:32.821 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.821 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.822 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.822 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.822 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325241+0000", 2023-10-09T11:06:32.822 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.822 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.822 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.822 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.822 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325240+0000", 2023-10-09T11:06:32.822 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:32.822 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.822 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.823 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.823 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325273+0000", 2023-10-09T11:06:32.823 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3108000000000002e-05 2023-10-09T11:06:32.823 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.823 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.823 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.823 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325281+0000", 2023-10-09T11:06:32.823 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2400000000000001e-06 2023-10-09T11:06:32.823 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.823 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.823 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.823 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325285+0000", 2023-10-09T11:06:32.824 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.408e-06 2023-10-09T11:06:32.824 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.824 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.824 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.824 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.326873+0000", 2023-10-09T11:06:32.824 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015876180000000001 2023-10-09T11:06:32.824 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.824 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.824 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.824 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327003+0000", 2023-10-09T11:06:32.824 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013045099999999999 2023-10-09T11:06:32.824 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.824 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.825 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:06:32.825 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327094+0000", 2023-10-09T11:06:32.825 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1046999999999998e-05 2023-10-09T11:06:32.825 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.825 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.825 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.825 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.329164+0000", 2023-10-09T11:06:32.825 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020697559999999999 2023-10-09T11:06:32.825 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.825 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.825 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.825 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424370+0000", 2023-10-09T11:06:32.826 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.095206683 2023-10-09T11:06:32.826 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.826 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.826 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.826 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424514+0000", 2023-10-09T11:06:32.826 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000143547 2023-10-09T11:06:32.826 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.826 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.826 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.826 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424547+0000", 2023-10-09T11:06:32.826 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3194000000000001e-05 2023-10-09T11:06:32.826 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.827 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.827 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.827 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424575+0000", 2023-10-09T11:06:32.827 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7362999999999999e-05 2023-10-09T11:06:32.827 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.827 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.827 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.827 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.827 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.827 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:77753 10.f 10:f688b427:::benchmark_data_smithi125_402869_object77752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-10-09T11:06:32.827 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:02:21.288500+0000", 2023-10-09T11:06:32.827 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 251.477373099, 2023-10-09T11:06:32.828 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.111923807, 2023-10-09T11:06:32.828 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.828 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.828 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.828 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.828 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.828 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 77753 2023-10-09T11:06:32.828 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.828 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.828 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.828 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.828 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288500+0000", 2023-10-09T11:06:32.829 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.829 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.829 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.829 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.829 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288500+0000", 2023-10-09T11:06:32.829 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.829 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.829 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.829 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.829 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288500+0000", 2023-10-09T11:06:32.829 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:32.829 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.829 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.830 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.830 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288535+0000", 2023-10-09T11:06:32.830 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5206999999999997e-05 2023-10-09T11:06:32.830 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.830 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.830 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.830 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288544+0000", 2023-10-09T11:06:32.830 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5019999999999998e-06 2023-10-09T11:06:32.830 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.830 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.830 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.831 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288550+0000", 2023-10-09T11:06:32.831 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4079999999999997e-06 2023-10-09T11:06:32.831 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.831 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.831 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.831 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288589+0000", 2023-10-09T11:06:32.831 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9366000000000002e-05 2023-10-09T11:06:32.831 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.831 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.831 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.831 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288708+0000", 2023-10-09T11:06:32.831 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118998 2023-10-09T11:06:32.832 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.832 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.832 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T11:06:32.832 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288884+0000", 2023-10-09T11:06:32.832 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017539000000000001 2023-10-09T11:06:32.832 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.832 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.832 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.832 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.292679+0000", 2023-10-09T11:06:32.832 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003795138 2023-10-09T11:06:32.832 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.832 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.833 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.833 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.296941+0000", 2023-10-09T11:06:32.833 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042627710000000003 2023-10-09T11:06:32.833 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.833 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.833 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.833 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.400350+0000", 2023-10-09T11:06:32.833 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.103408823 2023-10-09T11:06:32.833 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.833 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.833 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.833 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.400392+0000", 2023-10-09T11:06:32.834 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1792000000000002e-05 2023-10-09T11:06:32.834 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.834 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.834 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.834 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.400424+0000", 2023-10-09T11:06:32.834 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2053e-05 2023-10-09T11:06:32.834 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.834 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.834 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.834 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.834 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.834 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:158832 10.a 10:5433f58d:::benchmark_data_smithi125_402869_object60263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2023-10-09T11:06:32.835 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:03:55.841518+0000", 2023-10-09T11:06:32.835 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 156.92435513800001, 2023-10-09T11:06:32.835 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.116037265, 2023-10-09T11:06:32.835 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.835 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.835 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.835 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.835 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.835 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 158832 2023-10-09T11:06:32.835 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.835 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.835 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.836 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.836 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841518+0000", 2023-10-09T11:06:32.836 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.836 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.836 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.836 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.836 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841518+0000", 2023-10-09T11:06:32.836 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.836 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.836 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.836 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.836 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841518+0000", 2023-10-09T11:06:32.837 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:32.837 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.837 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.837 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.837 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841521+0000", 2023-10-09T11:06:32.837 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7020000000000001e-06 2023-10-09T11:06:32.837 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.837 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.837 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.837 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841527+0000", 2023-10-09T11:06:32.837 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1310000000000003e-06 2023-10-09T11:06:32.837 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.838 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.838 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.838 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841531+0000", 2023-10-09T11:06:32.838 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1049999999999997e-06 2023-10-09T11:06:32.838 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.838 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.838 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.838 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841568+0000", 2023-10-09T11:06:32.838 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7651000000000003e-05 2023-10-09T11:06:32.838 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.838 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.838 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.839 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841681+0000", 2023-10-09T11:06:32.839 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011254100000000001 2023-10-09T11:06:32.839 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.839 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.839 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:06:32.839 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841795+0000", 2023-10-09T11:06:32.839 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000114193 2023-10-09T11:06:32.839 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.839 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.839 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.839 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.845131+0000", 2023-10-09T11:06:32.839 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003335569 2023-10-09T11:06:32.840 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.840 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.840 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.840 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.845137+0000", 2023-10-09T11:06:32.840 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9900000000000002e-06 2023-10-09T11:06:32.840 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.840 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.840 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.840 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.957443+0000", 2023-10-09T11:06:32.840 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.112306507 2023-10-09T11:06:32.840 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.840 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.841 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.841 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.957511+0000", 2023-10-09T11:06:32.841 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7582000000000004e-05 2023-10-09T11:06:32.841 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.841 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.841 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.841 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.957556+0000", 2023-10-09T11:06:32.841 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4696e-05 2023-10-09T11:06:32.841 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.841 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.841 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.841 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.842 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.842 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:180897 10.f 10:f1fc3a12:::benchmark_data_smithi125_402869_object82328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T11:06:32.842 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:24.612340+0000", 2023-10-09T11:06:32.842 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 128.15353347499999, 2023-10-09T11:06:32.842 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.107943447, 2023-10-09T11:06:32.842 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.842 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.842 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.842 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.842 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.842 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 180897 2023-10-09T11:06:32.843 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.843 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.843 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.843 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.843 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612340+0000", 2023-10-09T11:06:32.843 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.843 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.843 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.843 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.843 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612340+0000", 2023-10-09T11:06:32.843 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.843 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.844 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.844 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.844 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612339+0000", 2023-10-09T11:06:32.844 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:32.844 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.844 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.844 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.844 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612343+0000", 2023-10-09T11:06:32.844 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3069999999999998e-06 2023-10-09T11:06:32.844 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.844 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.844 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.844 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612353+0000", 2023-10-09T11:06:32.845 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9769999999999995e-06 2023-10-09T11:06:32.845 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.845 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.845 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.845 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612357+0000", 2023-10-09T11:06:32.845 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5109999999999997e-06 2023-10-09T11:06:32.845 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.845 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.845 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.845 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612420+0000", 2023-10-09T11:06:32.845 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2391999999999995e-05 2023-10-09T11:06:32.845 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.846 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.846 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.846 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612537+0000", 2023-10-09T11:06:32.846 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000117409 2023-10-09T11:06:32.846 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.846 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.846 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T11:06:32.846 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612654+0000", 2023-10-09T11:06:32.846 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000117248 2023-10-09T11:06:32.846 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.846 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.846 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.847 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.621223+0000", 2023-10-09T11:06:32.847 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0085686159999999994 2023-10-09T11:06:32.847 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.847 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.847 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.847 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626399+0000", 2023-10-09T11:06:32.847 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051757670000000004 2023-10-09T11:06:32.847 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.847 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.847 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.847 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626899+0000", 2023-10-09T11:06:32.847 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00049993800000000001 2023-10-09T11:06:32.848 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.848 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.848 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.848 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.719838+0000", 2023-10-09T11:06:32.848 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.092939021999999996 2023-10-09T11:06:32.848 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.848 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.848 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.848 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.719924+0000", 2023-10-09T11:06:32.848 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6328000000000004e-05 2023-10-09T11:06:32.848 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.849 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.849 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.849 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.720283+0000", 2023-10-09T11:06:32.849 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00035945900000000002 2023-10-09T11:06:32.849 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.849 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.849 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.849 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.849 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.849 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:180902 10.f 10:fa4f508c:::benchmark_data_smithi125_402869_object82333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T11:06:32.849 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:24.626845+0000", 2023-10-09T11:06:32.849 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 128.13902839799999, 2023-10-09T11:06:32.850 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10240413199999999, 2023-10-09T11:06:32.850 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.850 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.850 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.850 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.850 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.850 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 180902 2023-10-09T11:06:32.850 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.850 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.850 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.850 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.850 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626845+0000", 2023-10-09T11:06:32.851 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.851 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.851 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.851 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.851 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626845+0000", 2023-10-09T11:06:32.851 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.851 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.851 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.851 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.851 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626844+0000", 2023-10-09T11:06:32.851 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:32.851 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.852 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.852 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.852 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626848+0000", 2023-10-09T11:06:32.852 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.498e-06 2023-10-09T11:06:32.852 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.852 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.852 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.852 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626858+0000", 2023-10-09T11:06:32.852 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2920000000000001e-06 2023-10-09T11:06:32.852 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.852 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.852 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.852 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626861+0000", 2023-10-09T11:06:32.853 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.782e-06 2023-10-09T11:06:32.853 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.853 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.853 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.853 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626915+0000", 2023-10-09T11:06:32.853 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3905e-05 2023-10-09T11:06:32.853 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.853 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.853 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.853 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.627044+0000", 2023-10-09T11:06:32.853 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012853699999999999 2023-10-09T11:06:32.853 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.854 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.854 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T11:06:32.854 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.627159+0000", 2023-10-09T11:06:32.854 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000114894 2023-10-09T11:06:32.854 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.854 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.854 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.854 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.630494+0000", 2023-10-09T11:06:32.854 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003335176 2023-10-09T11:06:32.854 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.854 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.854 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.855 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.632097+0000", 2023-10-09T11:06:32.855 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001602989 2023-10-09T11:06:32.855 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.855 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.855 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.855 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.635044+0000", 2023-10-09T11:06:32.855 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002947098 2023-10-09T11:06:32.855 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.855 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.855 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.855 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.728793+0000", 2023-10-09T11:06:32.855 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.093748897999999997 2023-10-09T11:06:32.856 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.856 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.856 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.856 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.728842+0000", 2023-10-09T11:06:32.856 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9327999999999999e-05 2023-10-09T11:06:32.856 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.856 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.856 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.856 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.729249+0000", 2023-10-09T11:06:32.856 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00040702300000000001 2023-10-09T11:06:32.856 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.856 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.857 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.857 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.857 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.857 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183807 10.0 10:04af7371:::benchmark_data_smithi125_402869_object85238:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:32.857 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.483591+0000", 2023-10-09T11:06:32.857 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 124.282282654, 2023-10-09T11:06:32.857 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.114706303, 2023-10-09T11:06:32.857 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.857 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.857 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.857 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.857 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.858 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183807 2023-10-09T11:06:32.858 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.858 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.858 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.858 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.858 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483591+0000", 2023-10-09T11:06:32.858 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.858 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.858 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.858 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.858 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483591+0000", 2023-10-09T11:06:32.859 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.859 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.859 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.859 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.859 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483590+0000", 2023-10-09T11:06:32.859 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:06:32.859 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.859 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.859 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.859 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483595+0000", 2023-10-09T11:06:32.859 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8319999999999996e-06 2023-10-09T11:06:32.859 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.859 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.860 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.860 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483605+0000", 2023-10-09T11:06:32.860 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0108999999999999e-05 2023-10-09T11:06:32.860 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.860 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.860 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.860 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483610+0000", 2023-10-09T11:06:32.860 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7469999999999996e-06 2023-10-09T11:06:32.860 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.860 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.860 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.860 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483651+0000", 2023-10-09T11:06:32.861 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1591000000000002e-05 2023-10-09T11:06:32.861 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.861 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.861 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.861 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483743+0000", 2023-10-09T11:06:32.861 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1634999999999999e-05 2023-10-09T11:06:32.861 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.861 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.861 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:06:32.861 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.484167+0000", 2023-10-09T11:06:32.861 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000424702 2023-10-09T11:06:32.861 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.862 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.862 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.862 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.493854+0000", 2023-10-09T11:06:32.862 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0096867949999999998 2023-10-09T11:06:32.862 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.862 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.862 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.862 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495932+0000", 2023-10-09T11:06:32.862 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020782270000000002 2023-10-09T11:06:32.862 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.862 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.862 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.863 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598222+0000", 2023-10-09T11:06:32.863 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.102289846 2023-10-09T11:06:32.863 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.863 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.863 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.863 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598267+0000", 2023-10-09T11:06:32.863 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4545999999999997e-05 2023-10-09T11:06:32.863 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.863 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.863 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.863 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598297+0000", 2023-10-09T11:06:32.863 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0187000000000001e-05 2023-10-09T11:06:32.864 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.864 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.864 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.864 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.864 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.864 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183811 10.10 10:0ca62c6a:::benchmark_data_smithi125_402869_object85242:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:32.864 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.483618+0000", 2023-10-09T11:06:32.864 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 124.282255535, 2023-10-09T11:06:32.864 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11462522, 2023-10-09T11:06:32.864 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.864 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.864 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.865 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.865 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.865 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183811 2023-10-09T11:06:32.865 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.865 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.865 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.865 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.865 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483618+0000", 2023-10-09T11:06:32.865 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.865 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.865 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.865 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.866 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483618+0000", 2023-10-09T11:06:32.866 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.866 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.866 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.866 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.866 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483617+0000", 2023-10-09T11:06:32.866 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:32.866 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.866 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.866 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.866 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483620+0000", 2023-10-09T11:06:32.866 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6240000000000002e-06 2023-10-09T11:06:32.867 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.867 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.867 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.867 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483632+0000", 2023-10-09T11:06:32.867 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1652e-05 2023-10-09T11:06:32.867 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.867 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.867 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.867 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483636+0000", 2023-10-09T11:06:32.867 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1330000000000001e-06 2023-10-09T11:06:32.867 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.867 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.868 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.868 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483686+0000", 2023-10-09T11:06:32.868 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0438000000000001e-05 2023-10-09T11:06:32.868 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.868 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.868 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.868 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483793+0000", 2023-10-09T11:06:32.868 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106499 2023-10-09T11:06:32.868 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.868 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.868 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:06:32.868 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.484240+0000", 2023-10-09T11:06:32.869 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00044728299999999998 2023-10-09T11:06:32.869 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.869 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.869 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.869 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.493846+0000", 2023-10-09T11:06:32.869 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0096058700000000007 2023-10-09T11:06:32.869 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.869 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.869 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.869 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495948+0000", 2023-10-09T11:06:32.869 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021024860000000002 2023-10-09T11:06:32.869 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.869 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.870 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.870 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598183+0000", 2023-10-09T11:06:32.870 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.102234346 2023-10-09T11:06:32.870 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.870 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.870 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.870 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598216+0000", 2023-10-09T11:06:32.870 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2889999999999999e-05 2023-10-09T11:06:32.870 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.870 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.870 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.870 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598243+0000", 2023-10-09T11:06:32.871 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7449000000000001e-05 2023-10-09T11:06:32.871 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.871 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.871 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.871 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.871 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.871 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183812 10.10 10:0e10b5d1:::benchmark_data_smithi125_402869_object85243:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:32.871 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.483656+0000", 2023-10-09T11:06:32.871 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 124.28221730600001, 2023-10-09T11:06:32.871 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.114659012, 2023-10-09T11:06:32.871 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.871 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.872 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.872 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.872 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.872 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183812 2023-10-09T11:06:32.872 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.872 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.872 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.872 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.872 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483656+0000", 2023-10-09T11:06:32.872 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.872 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.872 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.872 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.873 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483656+0000", 2023-10-09T11:06:32.873 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.873 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.873 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.873 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.873 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483656+0000", 2023-10-09T11:06:32.873 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:32.873 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.873 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.873 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.873 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483659+0000", 2023-10-09T11:06:32.873 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0730000000000001e-06 2023-10-09T11:06:32.874 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.874 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.874 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.874 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483666+0000", 2023-10-09T11:06:32.874 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0539999999999997e-06 2023-10-09T11:06:32.874 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.874 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.874 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.874 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483670+0000", 2023-10-09T11:06:32.874 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8670000000000004e-06 2023-10-09T11:06:32.874 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.874 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.875 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.875 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486323+0000", 2023-10-09T11:06:32.875 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026528950000000002 2023-10-09T11:06:32.875 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.875 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.875 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.875 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486433+0000", 2023-10-09T11:06:32.875 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000110812 2023-10-09T11:06:32.875 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.875 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.875 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:06:32.875 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486801+0000", 2023-10-09T11:06:32.876 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00036751500000000001 2023-10-09T11:06:32.876 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.876 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.876 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.876 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495912+0000", 2023-10-09T11:06:32.876 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0091110519999999997 2023-10-09T11:06:32.876 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.876 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.876 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.876 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495984+0000", 2023-10-09T11:06:32.876 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1724999999999995e-05 2023-10-09T11:06:32.876 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.877 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.877 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.877 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598264+0000", 2023-10-09T11:06:32.877 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.102280443 2023-10-09T11:06:32.877 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.877 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.877 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.877 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598287+0000", 2023-10-09T11:06:32.877 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3285000000000002e-05 2023-10-09T11:06:32.877 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.877 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.877 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.877 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598315+0000", 2023-10-09T11:06:32.878 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.777e-05 2023-10-09T11:06:32.878 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.878 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.878 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.878 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.878 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.878 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183813 10.10 10:0c717176:::benchmark_data_smithi125_402869_object85244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:32.878 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.483680+0000", 2023-10-09T11:06:32.878 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 124.282193186, 2023-10-09T11:06:32.878 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.114707768, 2023-10-09T11:06:32.878 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.879 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.879 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.879 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.879 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.879 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183813 2023-10-09T11:06:32.879 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.879 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.879 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.879 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.879 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483680+0000", 2023-10-09T11:06:32.879 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.879 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.880 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.880 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.880 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483680+0000", 2023-10-09T11:06:32.880 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.880 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.880 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.880 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.880 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483680+0000", 2023-10-09T11:06:32.880 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:32.880 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.880 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.880 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.881 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483682+0000", 2023-10-09T11:06:32.881 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6599999999999999e-06 2023-10-09T11:06:32.881 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.881 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.881 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.881 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483688+0000", 2023-10-09T11:06:32.881 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6540000000000004e-06 2023-10-09T11:06:32.881 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.881 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.881 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.881 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483692+0000", 2023-10-09T11:06:32.881 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.67e-06 2023-10-09T11:06:32.882 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.882 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.882 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.882 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.489061+0000", 2023-10-09T11:06:32.882 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0053695840000000002 2023-10-09T11:06:32.882 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.882 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.882 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.882 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.489172+0000", 2023-10-09T11:06:32.882 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011078000000000001 2023-10-09T11:06:32.882 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.882 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.883 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:06:32.883 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.489480+0000", 2023-10-09T11:06:32.883 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000307855 2023-10-09T11:06:32.883 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.883 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.883 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.883 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495919+0000", 2023-10-09T11:06:32.883 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064387530000000002 2023-10-09T11:06:32.883 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.883 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.883 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.883 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.496010+0000", 2023-10-09T11:06:32.883 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1748000000000003e-05 2023-10-09T11:06:32.884 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.884 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.884 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.884 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598341+0000", 2023-10-09T11:06:32.884 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.102330948 2023-10-09T11:06:32.884 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.884 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.884 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.884 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598368+0000", 2023-10-09T11:06:32.884 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6183e-05 2023-10-09T11:06:32.884 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.884 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.885 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.885 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598388+0000", 2023-10-09T11:06:32.885 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0367999999999999e-05 2023-10-09T11:06:32.885 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.885 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.885 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.885 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.885 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.885 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183815 10.10 10:0dad7a83:::benchmark_data_smithi125_402869_object85246:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:32.885 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.486435+0000", 2023-10-09T11:06:32.885 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 124.27943872, 2023-10-09T11:06:32.885 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11203769, 2023-10-09T11:06:32.886 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.886 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.886 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.886 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.886 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.886 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183815 2023-10-09T11:06:32.886 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.886 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.886 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.886 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.886 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486435+0000", 2023-10-09T11:06:32.886 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.887 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.887 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.887 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.887 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486435+0000", 2023-10-09T11:06:32.887 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.887 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.887 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.887 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.887 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486433+0000", 2023-10-09T11:06:32.887 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:32.887 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.887 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.888 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.888 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486450+0000", 2023-10-09T11:06:32.888 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6589999999999999e-05 2023-10-09T11:06:32.888 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.888 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.888 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.888 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486463+0000", 2023-10-09T11:06:32.888 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3261000000000001e-05 2023-10-09T11:06:32.888 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.888 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.888 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.888 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486475+0000", 2023-10-09T11:06:32.889 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2328e-05 2023-10-09T11:06:32.889 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.889 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.889 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.889 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.491357+0000", 2023-10-09T11:06:32.889 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.004881364 2023-10-09T11:06:32.889 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.889 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.889 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.889 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.491487+0000", 2023-10-09T11:06:32.889 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013008299999999999 2023-10-09T11:06:32.889 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.889 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.890 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:06:32.890 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.491763+0000", 2023-10-09T11:06:32.890 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00027596599999999999 2023-10-09T11:06:32.890 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.890 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.890 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.890 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495954+0000", 2023-10-09T11:06:32.890 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041909010000000003 2023-10-09T11:06:32.890 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.890 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.890 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.891 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.497564+0000", 2023-10-09T11:06:32.891 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001610081 2023-10-09T11:06:32.891 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.891 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.891 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.891 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598414+0000", 2023-10-09T11:06:32.891 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10085058500000001 2023-10-09T11:06:32.891 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.891 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.891 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.891 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598445+0000", 2023-10-09T11:06:32.891 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.044e-05 2023-10-09T11:06:32.892 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.892 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.892 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.892 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598472+0000", 2023-10-09T11:06:32.892 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7593000000000002e-05 2023-10-09T11:06:32.892 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.892 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.892 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.892 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.892 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.892 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183821 10.10 10:0e04176b:::benchmark_data_smithi125_402869_object85252:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:32.892 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.492183+0000", 2023-10-09T11:06:32.893 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 124.273690077, 2023-10-09T11:06:32.893 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.106356412, 2023-10-09T11:06:32.893 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.893 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.893 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.893 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.893 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.893 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183821 2023-10-09T11:06:32.893 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.893 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.893 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.893 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.893 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492183+0000", 2023-10-09T11:06:32.894 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.894 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.894 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.894 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492183+0000", 2023-10-09T11:06:32.894 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.894 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.894 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.894 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492182+0000", 2023-10-09T11:06:32.894 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:06:32.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.895 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.895 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.895 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492189+0000", 2023-10-09T11:06:32.895 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6080000000000001e-06 2023-10-09T11:06:32.895 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.895 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.895 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.895 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492203+0000", 2023-10-09T11:06:32.895 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3824e-05 2023-10-09T11:06:32.895 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.895 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.895 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.896 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492209+0000", 2023-10-09T11:06:32.896 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6769999999999998e-06 2023-10-09T11:06:32.896 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.896 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.896 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.896 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.493870+0000", 2023-10-09T11:06:32.896 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0016607099999999999 2023-10-09T11:06:32.896 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.896 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.896 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.896 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.493983+0000", 2023-10-09T11:06:32.896 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112582 2023-10-09T11:06:32.897 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.897 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.897 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:06:32.897 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.494235+0000", 2023-10-09T11:06:32.897 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025229000000000001 2023-10-09T11:06:32.897 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.897 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.897 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.897 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.496090+0000", 2023-10-09T11:06:32.897 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018548270000000001 2023-10-09T11:06:32.897 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.897 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.497689+0000", 2023-10-09T11:06:32.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015987530000000001 2023-10-09T11:06:32.898 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.898 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598498+0000", 2023-10-09T11:06:32.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10080939899999999 2023-10-09T11:06:32.898 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.898 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.899 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598522+0000", 2023-10-09T11:06:32.899 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3609999999999999e-05 2023-10-09T11:06:32.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.899 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.899 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.899 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598540+0000", 2023-10-09T11:06:32.899 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8224e-05 2023-10-09T11:06:32.899 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.899 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.899 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.899 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.900 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183823 10.10 10:0c686624:::benchmark_data_smithi125_402869_object85254:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:32.900 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.500372+0000", 2023-10-09T11:06:32.900 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 124.265501419, 2023-10-09T11:06:32.900 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.098236665000000001, 2023-10-09T11:06:32.900 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.900 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.900 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.900 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.900 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.900 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183823 2023-10-09T11:06:32.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.901 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.901 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.901 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.901 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500372+0000", 2023-10-09T11:06:32.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.901 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.901 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.901 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500372+0000", 2023-10-09T11:06:32.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.901 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.902 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500371+0000", 2023-10-09T11:06:32.902 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:06:32.902 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.902 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500377+0000", 2023-10-09T11:06:32.902 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6710000000000004e-06 2023-10-09T11:06:32.902 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.903 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500387+0000", 2023-10-09T11:06:32.903 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9329999999999996e-06 2023-10-09T11:06:32.903 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.903 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.903 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.903 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500391+0000", 2023-10-09T11:06:32.903 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.887e-06 2023-10-09T11:06:32.903 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.903 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.903 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.903 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500428+0000", 2023-10-09T11:06:32.903 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6810999999999997e-05 2023-10-09T11:06:32.904 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.904 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.904 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.904 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500522+0000", 2023-10-09T11:06:32.904 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3919000000000007e-05 2023-10-09T11:06:32.904 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.904 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.904 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:06:32.904 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500870+0000", 2023-10-09T11:06:32.904 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000347762 2023-10-09T11:06:32.904 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.904 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.905 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.502778+0000", 2023-10-09T11:06:32.905 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019076640000000001 2023-10-09T11:06:32.905 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.905 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.905 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.502985+0000", 2023-10-09T11:06:32.905 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00020777600000000001 2023-10-09T11:06:32.905 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.905 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.905 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598562+0000", 2023-10-09T11:06:32.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.095576784999999997 2023-10-09T11:06:32.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.906 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.906 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.906 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598590+0000", 2023-10-09T11:06:32.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7881999999999998e-05 2023-10-09T11:06:32.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.906 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.906 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.906 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598609+0000", 2023-10-09T11:06:32.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8587999999999999e-05 2023-10-09T11:06:32.906 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.907 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.907 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.907 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.907 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:184330 10.a 10:57ae38ca:::benchmark_data_smithi125_402869_object85761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:06:32.907 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:29.197170+0000", 2023-10-09T11:06:32.907 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 123.568703654, 2023-10-09T11:06:32.907 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.120668026, 2023-10-09T11:06:32.907 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:32.907 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:06:32.907 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:32.907 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:06:32.908 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:06:32.908 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 184330 2023-10-09T11:06:32.908 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.908 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:32.908 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.908 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:32.908 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197170+0000", 2023-10-09T11:06:32.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.908 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.908 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.908 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:32.908 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197170+0000", 2023-10-09T11:06:32.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:32.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.909 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:32.909 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197169+0000", 2023-10-09T11:06:32.909 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:32.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.909 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:32.909 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197173+0000", 2023-10-09T11:06:32.909 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.106e-06 2023-10-09T11:06:32.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.910 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:32.910 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197180+0000", 2023-10-09T11:06:32.910 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1760000000000004e-06 2023-10-09T11:06:32.910 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.910 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:32.910 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197184+0000", 2023-10-09T11:06:32.910 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9639999999999997e-06 2023-10-09T11:06:32.910 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.910 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:32.910 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197223+0000", 2023-10-09T11:06:32.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9072999999999997e-05 2023-10-09T11:06:32.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.911 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.911 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:32.911 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197334+0000", 2023-10-09T11:06:32.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000111279 2023-10-09T11:06:32.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.911 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.911 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:06:32.911 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197448+0000", 2023-10-09T11:06:32.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000113848 2023-10-09T11:06:32.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:32.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.200444+0000", 2023-10-09T11:06:32.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029966039999999999 2023-10-09T11:06:32.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.206587+0000", 2023-10-09T11:06:32.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061423409999999999 2023-10-09T11:06:32.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:32.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.317756+0000", 2023-10-09T11:06:32.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.111169485 2023-10-09T11:06:32.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:06:32.913 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.317798+0000", 2023-10-09T11:06:32.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1838000000000002e-05 2023-10-09T11:06:32.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:32.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:32.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:06:32.913 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.317838+0000", 2023-10-09T11:06:32.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9628000000000001e-05 2023-10-09T11:06:32.913 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.914 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.914 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.914 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:32.914 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:32.914 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:06:32.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:32.915 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:32.915 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:32.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:33.009 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 84 16 94294 94278 70.104 53.9375 0.0126337 0.0142589 2023-10-09T11:06:33.016 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:33.057 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:33.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:33.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T11:06:33.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:33.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:33.260 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:33.295 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:33.322 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:33.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:33.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:33.342 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:06:33.365 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:06:33.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:33.471 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:33.505 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:33.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:33.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:33.509 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:06:33.596 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:33.609 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:33.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:33 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:33.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:33 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:33.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:33 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:06:33.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:33 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:06:33.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:33 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:33.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:33 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 16059/273840 objects degraded (5.864%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:33.644 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:33.674 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:33 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:33.674 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:33 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:33.674 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:33 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:06:33.674 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:33 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:06:33.674 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:33 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:33.674 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:33 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 16059/273840 objects degraded (5.864%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:33.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:33 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:33.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:33 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:33.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:33 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:06:33.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:33 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:06:33.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:33 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:33.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:33 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 16059/273840 objects degraded (5.864%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:33.675 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:33.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:33.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:33.783 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:33.849 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:33.879 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1288,"stamp":"2023-10-09T11:06:31.838587+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5981119332,"num_objects":91280,"num_object_clones":0,"num_object_copies":273840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16059,"num_objects_misplaced":2898,"num_objects_unfound":0,"num_objects_dirty":91280,"num_whiteouts":0,"num_read":31,"num_read_kb":56,"num_write":91331,"num_write_kb":5840496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9154,"num_bytes_recovered":596333662,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121323,"ondisk_log_size":121323,"up":387,"acting":382,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":411,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23765740,"kb_used_data":17972396,"kb_used_omap":134,"kb_used_meta":5793145,"kb_avail":726195476,"statfs":{"total":767960285184,"available":743624167424,"internally_reserved":0,"allocated":18403733504,"data_stored":18385854770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":137532,"internal_metadata":5932181188},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":16,"apply_latency_ms":16,"commit_latency_ns":16000000,"apply_latency_ns":16000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":354773203,"num_objects":5414,"num_object_clones":0,"num_object_copies":16242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-314,"num_objects_misplaced":-659,"num_objects_unfound":0,"num_objects_dirty":5414,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":5424,"num_write_kb":346542,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1102,"num_bytes_recovered":72955397,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002153"},"pg_stats":[{"pgid":"11.12","version":"525'4349","reported_seq":4397,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:28.125249+0000","last_change":"2023-10-09T11:05:45.369342+0000","last_active":"2023-10-09T11:06:28.125249+0000","last_peered":"2023-10-09T11:06:28.125249+0000","last_clean":"2023-10-09T11:06:28.125249+0000","last_became_active":"2023-10-09T11:05:45.367721+0000","last_became_peered":"2023-10-09T11:05:45.367721+0000","last_unstale":"2023-10-09T11:06:28.125249+0000","last_undegraded":"2023-10-09T11:06:28.125249+0000","last_fullsized":"2023-10-09T11:06:28.125249+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4349,"log_dups_size":0,"ondisk_log_size":4349,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184221696,"num_objects":2811,"num_object_clones":0,"num_object_copies":8433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2811,"num_write_kb":179904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"76'161","reported_seq":791,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T11:06:31.398348+0000","last_change":"2023-10-09T11:06:31.398348+0000","last_active":"2023-10-09T11:06:31.398348+0000","last_peered":"2023-10-09T11:06:31.398348+0000","last_clean":"2023-10-09T11:06:31.398348+0000","last_became_active":"2023-10-09T11:06:31.397943+0000","last_became_peered":"2023-10-09T11:06:31.397943+0000","last_unstale":"2023-10-09T11:06:31.398348+0000","last_undegraded":"2023-10-09T11:06:31.398348+0000","last_fullsized":"2023-10-09T11:06:31.398348+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:41.638809+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":310,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.374080+0000","last_change":"2023-10-09T11:03:21.185383+0000","last_active":"2023-10-09T11:06:30.374080+0000","last_peered":"2023-10-09T11:06:30.374080+0000","last_clean":"2023-10-09T11:06:30.374080+0000","last_became_active":"2023-10-09T11:03:21.185022+0000","last_became_peered":"2023-10-09T11:03:21.185022+0000","last_unstale":"2023-10-09T11:06:30.374080+0000","last_undegraded":"2023-10-09T11:06:30.374080+0000","last_fullsized":"2023-10-09T11:06:30.374080+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:05.230434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":50,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.229674+0000","last_change":"2023-10-09T11:04:50.319477+0000","last_active":"2023-10-09T11:06:24.229674+0000","last_peered":"2023-10-09T11:06:24.229674+0000","last_clean":"2023-10-09T11:06:24.229674+0000","last_became_active":"2023-10-09T11:04:50.319148+0000","last_became_peered":"2023-10-09T11:04:50.319148+0000","last_unstale":"2023-10-09T11:06:24.229674+0000","last_undegraded":"2023-10-09T11:06:24.229674+0000","last_fullsized":"2023-10-09T11:06:24.229674+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:48.725566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"525'4545","reported_seq":4604,"reported_epoch":525,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T11:06:28.046161+0000","last_change":"2023-10-09T11:06:08.038257+0000","last_active":"2023-10-09T11:06:28.046161+0000","last_peered":"2023-10-09T11:06:28.046161+0000","last_clean":"2023-10-09T11:05:19.266392+0000","last_became_active":"2023-10-09T11:05:45.395271+0000","last_became_peered":"2023-10-09T11:05:45.395271+0000","last_unstale":"2023-10-09T11:06:28.046161+0000","last_undegraded":"2023-10-09T11:06:08.037501+0000","last_fullsized":"2023-10-09T11:06:28.046161+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4545,"log_dups_size":0,"ondisk_log_size":4545,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191791104,"num_objects":2927,"num_object_clones":0,"num_object_copies":8781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":658,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2927,"num_write_kb":187296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":658},{"shards":"0,3,6","objects":2269}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":685,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347638+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:06:30.347638+0000","last_peered":"2023-10-09T11:06:30.347638+0000","last_clean":"2023-10-09T11:06:30.347638+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:06:30.347638+0000","last_undegraded":"2023-10-09T11:06:30.347638+0000","last_fullsized":"2023-10-09T11:06:30.347638+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":238,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.234306+0000","last_change":"2023-10-09T11:04:50.333711+0000","last_active":"2023-10-09T11:06:24.234306+0000","last_peered":"2023-10-09T11:06:24.234306+0000","last_clean":"2023-10-09T11:06:24.234306+0000","last_became_active":"2023-10-09T11:04:50.333262+0000","last_became_peered":"2023-10-09T11:04:50.333262+0000","last_unstale":"2023-10-09T11:06:24.234306+0000","last_undegraded":"2023-10-09T11:06:24.234306+0000","last_fullsized":"2023-10-09T11:06:24.234306+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.843765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":497,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.333493+0000","last_change":"2023-10-09T11:06:24.246177+0000","last_active":"2023-10-09T11:06:25.333493+0000","last_peered":"2023-10-09T11:06:25.333493+0000","last_clean":"2023-10-09T11:06:25.333493+0000","last_became_active":"2023-10-09T11:06:24.245873+0000","last_became_peered":"2023-10-09T11:06:24.245873+0000","last_unstale":"2023-10-09T11:06:25.333493+0000","last_undegraded":"2023-10-09T11:06:25.333493+0000","last_fullsized":"2023-10-09T11:06:25.333493+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:58:19.530502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.10","version":"525'4401","reported_seq":4458,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:06:28.102541+0000","last_change":"2023-10-09T11:06:09.120525+0000","last_active":"2023-10-09T11:06:28.102541+0000","last_peered":"2023-10-09T11:06:28.102541+0000","last_clean":"2023-10-09T11:06:06.955693+0000","last_became_active":"2023-10-09T11:06:09.114640+0000","last_became_peered":"2023-10-09T11:06:09.114640+0000","last_unstale":"2023-10-09T11:06:28.102541+0000","last_undegraded":"2023-10-09T11:06:09.015936+0000","last_fullsized":"2023-10-09T11:06:08.998831+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4401,"log_dups_size":0,"ondisk_log_size":4401,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186023936,"num_objects":2839,"num_object_clones":0,"num_object_copies":8517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2362,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2839,"num_write_kb":181664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3],"avail_no_missing":["1","3","7"],"object_location_counts":[{"shards":"1,3","objects":477},{"shards":"1,3,7","objects":2362}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":690,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.237918+0000","last_change":"2023-10-09T11:06:11.011267+0000","last_active":"2023-10-09T11:06:24.237918+0000","last_peered":"2023-10-09T11:06:24.237918+0000","last_clean":"2023-10-09T11:06:24.237918+0000","last_became_active":"2023-10-09T11:06:11.010848+0000","last_became_peered":"2023-10-09T11:06:11.010848+0000","last_unstale":"2023-10-09T11:06:24.237918+0000","last_undegraded":"2023-10-09T11:06:24.237918+0000","last_fullsized":"2023-10-09T11:06:24.237918+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:21.624300+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":50,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.227955+0000","last_change":"2023-10-09T11:04:50.310477+0000","last_active":"2023-10-09T11:06:24.227955+0000","last_peered":"2023-10-09T11:06:24.227955+0000","last_clean":"2023-10-09T11:06:24.227955+0000","last_became_active":"2023-10-09T11:04:50.310067+0000","last_became_peered":"2023-10-09T11:04:50.310067+0000","last_unstale":"2023-10-09T11:06:24.227955+0000","last_undegraded":"2023-10-09T11:06:24.227955+0000","last_fullsized":"2023-10-09T11:06:24.227955+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:52.884716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":337,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.348603+0000","last_change":"2023-10-09T11:05:20.310581+0000","last_active":"2023-10-09T11:06:30.348603+0000","last_peered":"2023-10-09T11:06:30.348603+0000","last_clean":"2023-10-09T11:06:30.348603+0000","last_became_active":"2023-10-09T11:05:20.309830+0000","last_became_peered":"2023-10-09T11:05:20.309830+0000","last_unstale":"2023-10-09T11:06:30.348603+0000","last_undegraded":"2023-10-09T11:06:30.348603+0000","last_fullsized":"2023-10-09T11:06:30.348603+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:24.508211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":628,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.348469+0000","last_change":"2023-10-09T11:06:11.012022+0000","last_active":"2023-10-09T11:06:30.348469+0000","last_peered":"2023-10-09T11:06:30.348469+0000","last_clean":"2023-10-09T11:06:30.348469+0000","last_became_active":"2023-10-09T11:06:11.011684+0000","last_became_peered":"2023-10-09T11:06:11.011684+0000","last_unstale":"2023-10-09T11:06:30.348469+0000","last_undegraded":"2023-10-09T11:06:30.348469+0000","last_fullsized":"2023-10-09T11:06:30.348469+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:50:08.258595+0000","scrub_duration":0.012082125000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":164,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.348658+0000","last_change":"2023-10-09T11:06:24.245488+0000","last_active":"2023-10-09T11:06:30.348658+0000","last_peered":"2023-10-09T11:06:30.348658+0000","last_clean":"2023-10-09T11:06:30.348658+0000","last_became_active":"2023-10-09T11:06:24.245167+0000","last_became_peered":"2023-10-09T11:06:24.245167+0000","last_unstale":"2023-10-09T11:06:30.348658+0000","last_undegraded":"2023-10-09T11:06:30.348658+0000","last_fullsized":"2023-10-09T11:06:30.348658+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:31.702568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":59,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.228037+0000","last_change":"2023-10-09T11:05:20.299453+0000","last_active":"2023-10-09T11:06:24.228037+0000","last_peered":"2023-10-09T11:06:24.228037+0000","last_clean":"2023-10-09T11:06:24.228037+0000","last_became_active":"2023-10-09T11:05:20.299117+0000","last_became_peered":"2023-10-09T11:05:20.299117+0000","last_unstale":"2023-10-09T11:06:24.228037+0000","last_undegraded":"2023-10-09T11:06:24.228037+0000","last_fullsized":"2023-10-09T11:06:24.228037+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:30.072813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":856,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-09T11:06:29.363409+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:06:29.363409+0000","last_peered":"2023-10-09T11:06:29.363409+0000","last_clean":"2023-10-09T11:06:29.363409+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:06:29.363409+0000","last_undegraded":"2023-10-09T11:06:29.363409+0000","last_fullsized":"2023-10-09T11:06:29.363409+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":389,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347804+0000","last_change":"2023-10-09T11:06:08.015527+0000","last_active":"2023-10-09T11:06:30.347804+0000","last_peered":"2023-10-09T11:06:30.347804+0000","last_clean":"2023-10-09T11:06:30.347804+0000","last_became_active":"2023-10-09T11:06:08.015184+0000","last_became_peered":"2023-10-09T11:06:08.015184+0000","last_unstale":"2023-10-09T11:06:30.347804+0000","last_undegraded":"2023-10-09T11:06:30.347804+0000","last_fullsized":"2023-10-09T11:06:30.347804+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:35.040523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":68,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.229316+0000","last_change":"2023-10-09T11:06:08.014803+0000","last_active":"2023-10-09T11:06:24.229316+0000","last_peered":"2023-10-09T11:06:24.229316+0000","last_clean":"2023-10-09T11:06:24.229316+0000","last_became_active":"2023-10-09T11:06:08.014472+0000","last_became_peered":"2023-10-09T11:06:08.014472+0000","last_unstale":"2023-10-09T11:06:24.229316+0000","last_undegraded":"2023-10-09T11:06:24.229316+0000","last_fullsized":"2023-10-09T11:06:24.229316+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:43.195742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":757,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.348038+0000","last_change":"2023-10-09T11:05:23.328981+0000","last_active":"2023-10-09T11:06:30.348038+0000","last_peered":"2023-10-09T11:06:30.348038+0000","last_clean":"2023-10-09T11:06:30.348038+0000","last_became_active":"2023-10-09T11:05:23.328616+0000","last_became_peered":"2023-10-09T11:05:23.328616+0000","last_unstale":"2023-10-09T11:06:30.348038+0000","last_undegraded":"2023-10-09T11:06:30.348038+0000","last_fullsized":"2023-10-09T11:06:30.348038+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:15.315819+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":117,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.229675+0000","last_change":"2023-10-09T11:06:08.044481+0000","last_active":"2023-10-09T11:06:24.229675+0000","last_peered":"2023-10-09T11:06:24.229675+0000","last_clean":"2023-10-09T11:06:24.229675+0000","last_became_active":"2023-10-09T11:06:08.015811+0000","last_became_peered":"2023-10-09T11:06:08.015811+0000","last_unstale":"2023-10-09T11:06:24.229675+0000","last_undegraded":"2023-10-09T11:06:24.229675+0000","last_fullsized":"2023-10-09T11:06:24.229675+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:11.825243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":306,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347429+0000","last_change":"2023-10-09T11:05:20.316265+0000","last_active":"2023-10-09T11:06:30.347429+0000","last_peered":"2023-10-09T11:06:30.347429+0000","last_clean":"2023-10-09T11:06:30.347429+0000","last_became_active":"2023-10-09T11:05:20.315853+0000","last_became_peered":"2023-10-09T11:05:20.315853+0000","last_unstale":"2023-10-09T11:06:30.347429+0000","last_undegraded":"2023-10-09T11:06:30.347429+0000","last_fullsized":"2023-10-09T11:06:30.347429+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:35.093981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":916,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.238631+0000","last_change":"2023-10-09T11:04:53.274921+0000","last_active":"2023-10-09T11:06:24.238631+0000","last_peered":"2023-10-09T11:06:24.238631+0000","last_clean":"2023-10-09T11:06:24.238631+0000","last_became_active":"2023-10-09T11:04:53.274518+0000","last_became_peered":"2023-10-09T11:04:53.274518+0000","last_unstale":"2023-10-09T11:06:24.238631+0000","last_undegraded":"2023-10-09T11:06:24.238631+0000","last_fullsized":"2023-10-09T11:06:24.238631+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:14.315301+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":31,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-09T11:06:29.363265+0000","last_change":"2023-10-09T11:06:08.037084+0000","last_active":"2023-10-09T11:06:29.363265+0000","last_peered":"2023-10-09T11:06:29.363265+0000","last_clean":"2023-10-09T11:06:29.363265+0000","last_became_active":"2023-10-09T11:06:08.011446+0000","last_became_peered":"2023-10-09T11:06:08.011446+0000","last_unstale":"2023-10-09T11:06:29.363265+0000","last_undegraded":"2023-10-09T11:06:29.363265+0000","last_fullsized":"2023-10-09T11:06:29.363265+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:52.385259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":547,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-09T11:06:29.362689+0000","last_change":"2023-10-09T11:05:20.311424+0000","last_active":"2023-10-09T11:06:29.362689+0000","last_peered":"2023-10-09T11:06:29.362689+0000","last_clean":"2023-10-09T11:06:29.362689+0000","last_became_active":"2023-10-09T11:05:20.311097+0000","last_became_peered":"2023-10-09T11:05:20.311097+0000","last_unstale":"2023-10-09T11:06:29.362689+0000","last_undegraded":"2023-10-09T11:06:29.362689+0000","last_fullsized":"2023-10-09T11:06:29.362689+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:19.274582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":548,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347866+0000","last_change":"2023-10-09T11:06:24.246500+0000","last_active":"2023-10-09T11:06:30.347866+0000","last_peered":"2023-10-09T11:06:30.347866+0000","last_clean":"2023-10-09T11:06:30.347866+0000","last_became_active":"2023-10-09T11:06:24.245076+0000","last_became_peered":"2023-10-09T11:06:24.245076+0000","last_unstale":"2023-10-09T11:06:30.347866+0000","last_undegraded":"2023-10-09T11:06:30.347866+0000","last_fullsized":"2023-10-09T11:06:30.347866+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":129,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-09T11:06:29.363165+0000","last_change":"2023-10-09T11:06:08.027881+0000","last_active":"2023-10-09T11:06:29.363165+0000","last_peered":"2023-10-09T11:06:29.363165+0000","last_clean":"2023-10-09T11:06:29.363165+0000","last_became_active":"2023-10-09T11:06:08.011384+0000","last_became_peered":"2023-10-09T11:06:08.011384+0000","last_unstale":"2023-10-09T11:06:29.363165+0000","last_undegraded":"2023-10-09T11:06:29.363165+0000","last_fullsized":"2023-10-09T11:06:29.363165+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:29.036833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":59,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.228538+0000","last_change":"2023-10-09T11:06:08.003915+0000","last_active":"2023-10-09T11:06:24.228538+0000","last_peered":"2023-10-09T11:06:24.228538+0000","last_clean":"2023-10-09T11:06:24.228538+0000","last_became_active":"2023-10-09T11:06:08.003609+0000","last_became_peered":"2023-10-09T11:06:08.003609+0000","last_unstale":"2023-10-09T11:06:24.228538+0000","last_undegraded":"2023-10-09T11:06:24.228538+0000","last_fullsized":"2023-10-09T11:06:24.228538+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:33.706643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":839,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347594+0000","last_change":"2023-10-09T11:05:23.323856+0000","last_active":"2023-10-09T11:06:30.347594+0000","last_peered":"2023-10-09T11:06:30.347594+0000","last_clean":"2023-10-09T11:06:30.347594+0000","last_became_active":"2023-10-09T11:05:23.322763+0000","last_became_peered":"2023-10-09T11:05:23.322763+0000","last_unstale":"2023-10-09T11:06:30.347594+0000","last_undegraded":"2023-10-09T11:06:30.347594+0000","last_fullsized":"2023-10-09T11:06:30.347594+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.760820+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":496,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.327162+0000","last_change":"2023-10-09T11:06:24.246814+0000","last_active":"2023-10-09T11:06:25.327162+0000","last_peered":"2023-10-09T11:06:25.327162+0000","last_clean":"2023-10-09T11:06:25.327162+0000","last_became_active":"2023-10-09T11:06:24.246522+0000","last_became_peered":"2023-10-09T11:06:24.246522+0000","last_unstale":"2023-10-09T11:06:25.327162+0000","last_undegraded":"2023-10-09T11:06:25.327162+0000","last_fullsized":"2023-10-09T11:06:25.327162+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:00.683195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":459,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.348551+0000","last_change":"2023-10-09T11:06:08.024756+0000","last_active":"2023-10-09T11:06:30.348551+0000","last_peered":"2023-10-09T11:06:30.348551+0000","last_clean":"2023-10-09T11:06:30.348551+0000","last_became_active":"2023-10-09T11:06:08.022730+0000","last_became_peered":"2023-10-09T11:06:08.022730+0000","last_unstale":"2023-10-09T11:06:30.348551+0000","last_undegraded":"2023-10-09T11:06:30.348551+0000","last_fullsized":"2023-10-09T11:06:30.348551+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:56.148870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"525'4411","reported_seq":5834,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:28.160674+0000","last_change":"2023-10-09T11:06:05.166134+0000","last_active":"2023-10-09T11:06:28.160674+0000","last_peered":"2023-10-09T11:06:28.160674+0000","last_clean":"2023-10-09T11:06:28.160674+0000","last_became_active":"2023-10-09T11:05:45.388425+0000","last_became_peered":"2023-10-09T11:05:45.388425+0000","last_unstale":"2023-10-09T11:06:28.160674+0000","last_undegraded":"2023-10-09T11:06:28.160674+0000","last_fullsized":"2023-10-09T11:06:28.160674+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4411,"log_dups_size":0,"ondisk_log_size":4411,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186613760,"num_objects":2848,"num_object_clones":0,"num_object_copies":8544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2848,"num_write_kb":182240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":684,"num_bytes_recovered":44826624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":857,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.236570+0000","last_change":"2023-10-09T11:04:53.274236+0000","last_active":"2023-10-09T11:06:24.236570+0000","last_peered":"2023-10-09T11:06:24.236570+0000","last_clean":"2023-10-09T11:06:24.236570+0000","last_became_active":"2023-10-09T11:04:53.273817+0000","last_became_peered":"2023-10-09T11:04:53.273817+0000","last_unstale":"2023-10-09T11:06:24.236570+0000","last_undegraded":"2023-10-09T11:06:24.236570+0000","last_fullsized":"2023-10-09T11:06:24.236570+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:05.096905+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":123,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.349969+0000","last_change":"2023-10-09T11:05:20.312129+0000","last_active":"2023-10-09T11:06:30.349969+0000","last_peered":"2023-10-09T11:06:30.349969+0000","last_clean":"2023-10-09T11:06:30.349969+0000","last_became_active":"2023-10-09T11:05:20.311810+0000","last_became_peered":"2023-10-09T11:05:20.311810+0000","last_unstale":"2023-10-09T11:06:30.349969+0000","last_undegraded":"2023-10-09T11:06:30.349969+0000","last_fullsized":"2023-10-09T11:06:30.349969+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:48.813391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":209,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.333422+0000","last_change":"2023-10-09T11:06:24.247534+0000","last_active":"2023-10-09T11:06:25.333422+0000","last_peered":"2023-10-09T11:06:25.333422+0000","last_clean":"2023-10-09T11:06:25.333422+0000","last_became_active":"2023-10-09T11:06:24.247311+0000","last_became_peered":"2023-10-09T11:06:24.247311+0000","last_unstale":"2023-10-09T11:06:25.333422+0000","last_undegraded":"2023-10-09T11:06:25.333422+0000","last_fullsized":"2023-10-09T11:06:25.333422+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:23.265393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"525'4341","reported_seq":6861,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:28.150347+0000","last_change":"2023-10-09T11:06:07.154110+0000","last_active":"2023-10-09T11:06:28.150347+0000","last_peered":"2023-10-09T11:06:28.150347+0000","last_clean":"2023-10-09T11:06:28.150347+0000","last_became_active":"2023-10-09T11:05:45.388306+0000","last_became_peered":"2023-10-09T11:05:45.388306+0000","last_unstale":"2023-10-09T11:06:28.150347+0000","last_undegraded":"2023-10-09T11:06:28.150347+0000","last_fullsized":"2023-10-09T11:06:28.150347+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4341,"log_dups_size":0,"ondisk_log_size":4341,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183894016,"num_objects":2806,"num_object_clones":0,"num_object_copies":8418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2806,"num_write_kb":179584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":645,"num_bytes_recovered":42237952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":836,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347896+0000","last_change":"2023-10-09T11:05:23.324537+0000","last_active":"2023-10-09T11:06:30.347896+0000","last_peered":"2023-10-09T11:06:30.347896+0000","last_clean":"2023-10-09T11:06:30.347896+0000","last_became_active":"2023-10-09T11:05:23.324187+0000","last_became_peered":"2023-10-09T11:05:23.324187+0000","last_unstale":"2023-10-09T11:06:30.347896+0000","last_undegraded":"2023-10-09T11:06:30.347896+0000","last_fullsized":"2023-10-09T11:06:30.347896+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:25.286106+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":48,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.350198+0000","last_change":"2023-10-09T11:05:20.323565+0000","last_active":"2023-10-09T11:06:30.350198+0000","last_peered":"2023-10-09T11:06:30.350198+0000","last_clean":"2023-10-09T11:06:30.350198+0000","last_became_active":"2023-10-09T11:05:20.309672+0000","last_became_peered":"2023-10-09T11:05:20.309672+0000","last_unstale":"2023-10-09T11:06:30.350198+0000","last_undegraded":"2023-10-09T11:06:30.350198+0000","last_fullsized":"2023-10-09T11:06:30.350198+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:50.211465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":59,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.229167+0000","last_change":"2023-10-09T11:06:08.003280+0000","last_active":"2023-10-09T11:06:24.229167+0000","last_peered":"2023-10-09T11:06:24.229167+0000","last_clean":"2023-10-09T11:06:24.229167+0000","last_became_active":"2023-10-09T11:06:08.002995+0000","last_became_peered":"2023-10-09T11:06:08.002995+0000","last_unstale":"2023-10-09T11:06:24.229167+0000","last_undegraded":"2023-10-09T11:06:24.229167+0000","last_fullsized":"2023-10-09T11:06:24.229167+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:40.565017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"528'4434","reported_seq":4484,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T11:06:31.634431+0000","last_change":"2023-10-09T11:05:45.340741+0000","last_active":"2023-10-09T11:06:31.634431+0000","last_peered":"2023-10-09T11:06:31.634431+0000","last_clean":"2023-10-09T11:06:31.634431+0000","last_became_active":"2023-10-09T11:05:45.338519+0000","last_became_peered":"2023-10-09T11:05:45.338519+0000","last_unstale":"2023-10-09T11:06:31.634431+0000","last_undegraded":"2023-10-09T11:06:31.634431+0000","last_fullsized":"2023-10-09T11:06:31.634431+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4434,"log_dups_size":0,"ondisk_log_size":4434,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187727872,"num_objects":2864,"num_object_clones":0,"num_object_copies":8592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2864,"num_write_kb":183328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":747,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.235534+0000","last_change":"2023-10-09T11:06:11.011472+0000","last_active":"2023-10-09T11:06:24.235534+0000","last_peered":"2023-10-09T11:06:24.235534+0000","last_clean":"2023-10-09T11:06:24.235534+0000","last_became_active":"2023-10-09T11:06:11.011078+0000","last_became_peered":"2023-10-09T11:06:11.011078+0000","last_unstale":"2023-10-09T11:06:24.235534+0000","last_undegraded":"2023-10-09T11:06:24.235534+0000","last_fullsized":"2023-10-09T11:06:24.235534+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:08:24.783524+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":138,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.351360+0000","last_change":"2023-10-09T11:06:24.245768+0000","last_active":"2023-10-09T11:06:25.351360+0000","last_peered":"2023-10-09T11:06:25.351360+0000","last_clean":"2023-10-09T11:06:25.351360+0000","last_became_active":"2023-10-09T11:06:24.245420+0000","last_became_peered":"2023-10-09T11:06:24.245420+0000","last_unstale":"2023-10-09T11:06:25.351360+0000","last_undegraded":"2023-10-09T11:06:25.351360+0000","last_fullsized":"2023-10-09T11:06:25.351360+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:29.482241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":480,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347278+0000","last_change":"2023-10-09T11:06:08.020692+0000","last_active":"2023-10-09T11:06:30.347278+0000","last_peered":"2023-10-09T11:06:30.347278+0000","last_clean":"2023-10-09T11:06:30.347278+0000","last_became_active":"2023-10-09T11:06:08.020337+0000","last_became_peered":"2023-10-09T11:06:08.020337+0000","last_unstale":"2023-10-09T11:06:30.347278+0000","last_undegraded":"2023-10-09T11:06:30.347278+0000","last_fullsized":"2023-10-09T11:06:30.347278+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:17.752221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"527'4393","reported_seq":4455,"reported_epoch":527,"state":"active+recovery_wait","last_fresh":"2023-10-09T11:06:31.187122+0000","last_change":"2023-10-09T11:06:24.280945+0000","last_active":"2023-10-09T11:06:31.187122+0000","last_peered":"2023-10-09T11:06:31.187122+0000","last_clean":"2023-10-09T11:06:23.209416+0000","last_became_active":"2023-10-09T11:06:24.254388+0000","last_became_peered":"2023-10-09T11:06:24.254388+0000","last_unstale":"2023-10-09T11:06:31.187122+0000","last_undegraded":"2023-10-09T11:06:31.187122+0000","last_fullsized":"2023-10-09T11:06:31.187122+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4393,"log_dups_size":0,"ondisk_log_size":4393,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187891712,"num_objects":2867,"num_object_clones":0,"num_object_copies":8601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2867,"num_write_kb":183488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":2867}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":905,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.228617+0000","last_change":"2023-10-09T11:05:23.316129+0000","last_active":"2023-10-09T11:06:24.228617+0000","last_peered":"2023-10-09T11:06:24.228617+0000","last_clean":"2023-10-09T11:06:24.228617+0000","last_became_active":"2023-10-09T11:05:23.315792+0000","last_became_peered":"2023-10-09T11:05:23.315792+0000","last_unstale":"2023-10-09T11:06:24.228617+0000","last_undegraded":"2023-10-09T11:06:24.228617+0000","last_fullsized":"2023-10-09T11:06:24.228617+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:51.770547+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":142,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.238124+0000","last_change":"2023-10-09T11:06:08.040581+0000","last_active":"2023-10-09T11:06:24.238124+0000","last_peered":"2023-10-09T11:06:24.238124+0000","last_clean":"2023-10-09T11:06:24.238124+0000","last_became_active":"2023-10-09T11:06:08.021880+0000","last_became_peered":"2023-10-09T11:06:08.021880+0000","last_unstale":"2023-10-09T11:06:24.238124+0000","last_undegraded":"2023-10-09T11:06:24.238124+0000","last_fullsized":"2023-10-09T11:06:24.238124+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:19.059813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":238,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.234256+0000","last_change":"2023-10-09T11:04:50.333538+0000","last_active":"2023-10-09T11:06:24.234256+0000","last_peered":"2023-10-09T11:06:24.234256+0000","last_clean":"2023-10-09T11:06:24.234256+0000","last_became_active":"2023-10-09T11:04:50.332514+0000","last_became_peered":"2023-10-09T11:04:50.332514+0000","last_unstale":"2023-10-09T11:06:24.234256+0000","last_undegraded":"2023-10-09T11:06:24.234256+0000","last_fullsized":"2023-10-09T11:06:24.234256+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:22.933839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"528'4369","reported_seq":4441,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:06:31.623605+0000","last_change":"2023-10-09T11:06:09.149007+0000","last_active":"2023-10-09T11:06:31.623605+0000","last_peered":"2023-10-09T11:06:31.623605+0000","last_clean":"2023-10-09T11:05:19.279287+0000","last_became_active":"2023-10-09T11:06:09.122854+0000","last_became_peered":"2023-10-09T11:06:09.122854+0000","last_unstale":"2023-10-09T11:06:31.623605+0000","last_undegraded":"2023-10-09T11:06:09.077674+0000","last_fullsized":"2023-10-09T11:06:09.002798+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4369,"log_dups_size":0,"ondisk_log_size":4369,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186875904,"num_objects":2851,"num_object_clones":0,"num_object_copies":8553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2245,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2851,"num_write_kb":182496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"3,5,7","objects":617},{"shards":"5,6","objects":606},{"shards":"5,6,7","objects":1628}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":784,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.348053+0000","last_change":"2023-10-09T11:05:23.322251+0000","last_active":"2023-10-09T11:06:30.348053+0000","last_peered":"2023-10-09T11:06:30.348053+0000","last_clean":"2023-10-09T11:06:30.348053+0000","last_became_active":"2023-10-09T11:05:23.321826+0000","last_became_peered":"2023-10-09T11:05:23.321826+0000","last_unstale":"2023-10-09T11:06:30.348053+0000","last_undegraded":"2023-10-09T11:06:30.348053+0000","last_fullsized":"2023-10-09T11:06:30.348053+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.858013+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":240,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.352614+0000","last_change":"2023-10-09T11:06:08.038975+0000","last_active":"2023-10-09T11:06:30.352614+0000","last_peered":"2023-10-09T11:06:30.352614+0000","last_clean":"2023-10-09T11:06:30.352614+0000","last_became_active":"2023-10-09T11:06:08.023183+0000","last_became_peered":"2023-10-09T11:06:08.023183+0000","last_unstale":"2023-10-09T11:06:30.352614+0000","last_undegraded":"2023-10-09T11:06:30.352614+0000","last_fullsized":"2023-10-09T11:06:30.352614+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:21.011520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":48,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.350243+0000","last_change":"2023-10-09T11:06:08.023207+0000","last_active":"2023-10-09T11:06:30.350243+0000","last_peered":"2023-10-09T11:06:30.350243+0000","last_clean":"2023-10-09T11:06:30.350243+0000","last_became_active":"2023-10-09T11:06:08.022858+0000","last_became_peered":"2023-10-09T11:06:08.022858+0000","last_unstale":"2023-10-09T11:06:30.350243+0000","last_undegraded":"2023-10-09T11:06:30.350243+0000","last_fullsized":"2023-10-09T11:06:30.350243+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:18.687911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"528'4440","reported_seq":5858,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T11:06:31.504033+0000","last_change":"2023-10-09T11:06:09.806789+0000","last_active":"2023-10-09T11:06:31.504033+0000","last_peered":"2023-10-09T11:06:31.504033+0000","last_clean":"2023-10-09T11:06:31.504033+0000","last_became_active":"2023-10-09T11:05:46.357677+0000","last_became_peered":"2023-10-09T11:05:46.357677+0000","last_unstale":"2023-10-09T11:06:31.504033+0000","last_undegraded":"2023-10-09T11:06:31.504033+0000","last_fullsized":"2023-10-09T11:06:31.504033+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4440,"log_dups_size":0,"ondisk_log_size":4440,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189202432,"num_objects":2887,"num_object_clones":0,"num_object_copies":8661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2887,"num_write_kb":184768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":677,"num_bytes_recovered":44367872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":1070,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.228889+0000","last_change":"2023-10-09T11:06:11.007625+0000","last_active":"2023-10-09T11:06:24.228889+0000","last_peered":"2023-10-09T11:06:24.228889+0000","last_clean":"2023-10-09T11:06:24.228889+0000","last_became_active":"2023-10-09T11:06:11.007167+0000","last_became_peered":"2023-10-09T11:06:11.007167+0000","last_unstale":"2023-10-09T11:06:24.228889+0000","last_undegraded":"2023-10-09T11:06:24.228889+0000","last_fullsized":"2023-10-09T11:06:24.228889+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:31:22.925012+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":386,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347689+0000","last_change":"2023-10-09T11:05:20.309385+0000","last_active":"2023-10-09T11:06:30.347689+0000","last_peered":"2023-10-09T11:06:30.347689+0000","last_clean":"2023-10-09T11:06:30.347689+0000","last_became_active":"2023-10-09T11:05:20.309034+0000","last_became_peered":"2023-10-09T11:05:20.309034+0000","last_unstale":"2023-10-09T11:06:30.347689+0000","last_undegraded":"2023-10-09T11:06:30.347689+0000","last_fullsized":"2023-10-09T11:06:30.347689+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:31.857485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":314,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.344086+0000","last_change":"2023-10-09T11:06:24.245966+0000","last_active":"2023-10-09T11:06:25.344086+0000","last_peered":"2023-10-09T11:06:25.344086+0000","last_clean":"2023-10-09T11:06:25.344086+0000","last_became_active":"2023-10-09T11:06:24.245310+0000","last_became_peered":"2023-10-09T11:06:24.245310+0000","last_unstale":"2023-10-09T11:06:25.344086+0000","last_undegraded":"2023-10-09T11:06:25.344086+0000","last_fullsized":"2023-10-09T11:06:25.344086+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:23.966200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.3","version":"525'4495","reported_seq":4565,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:06:28.160032+0000","last_change":"2023-10-09T11:06:11.717691+0000","last_active":"2023-10-09T11:06:28.160032+0000","last_peered":"2023-10-09T11:06:28.160032+0000","last_clean":"2023-10-09T11:05:19.265787+0000","last_became_active":"2023-10-09T11:06:11.700109+0000","last_became_peered":"2023-10-09T11:06:11.700109+0000","last_unstale":"2023-10-09T11:06:28.160032+0000","last_undegraded":"2023-10-09T11:06:11.664343+0000","last_fullsized":"2023-10-09T11:06:11.661338+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4495,"log_dups_size":0,"ondisk_log_size":4495,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190283776,"num_objects":2903,"num_object_clones":0,"num_object_copies":8709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2435,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2903,"num_write_kb":185824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,5,7","objects":696},{"shards":"1,6","objects":468},{"shards":"1,6,7","objects":1739}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":806,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.227991+0000","last_change":"2023-10-09T11:06:11.008538+0000","last_active":"2023-10-09T11:06:24.227991+0000","last_peered":"2023-10-09T11:06:24.227991+0000","last_clean":"2023-10-09T11:06:24.227991+0000","last_became_active":"2023-10-09T11:06:11.008181+0000","last_became_peered":"2023-10-09T11:06:11.008181+0000","last_unstale":"2023-10-09T11:06:24.227991+0000","last_undegraded":"2023-10-09T11:06:24.227991+0000","last_fullsized":"2023-10-09T11:06:24.227991+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:35:20.041306+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":54,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.228437+0000","last_change":"2023-10-09T11:04:50.529513+0000","last_active":"2023-10-09T11:06:24.228437+0000","last_peered":"2023-10-09T11:06:24.228437+0000","last_clean":"2023-10-09T11:06:24.228437+0000","last_became_active":"2023-10-09T11:04:50.339671+0000","last_became_peered":"2023-10-09T11:04:50.339671+0000","last_unstale":"2023-10-09T11:06:24.228437+0000","last_undegraded":"2023-10-09T11:06:24.228437+0000","last_fullsized":"2023-10-09T11:06:24.228437+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:24.473000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":59,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.228466+0000","last_change":"2023-10-09T11:05:20.313855+0000","last_active":"2023-10-09T11:06:24.228466+0000","last_peered":"2023-10-09T11:06:24.228466+0000","last_clean":"2023-10-09T11:06:24.228466+0000","last_became_active":"2023-10-09T11:05:20.313535+0000","last_became_peered":"2023-10-09T11:05:20.313535+0000","last_unstale":"2023-10-09T11:06:24.228466+0000","last_undegraded":"2023-10-09T11:06:24.228466+0000","last_fullsized":"2023-10-09T11:06:24.228466+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:25.266434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"525'4304","reported_seq":4352,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:28.034476+0000","last_change":"2023-10-09T11:05:45.345473+0000","last_active":"2023-10-09T11:06:28.034476+0000","last_peered":"2023-10-09T11:06:28.034476+0000","last_clean":"2023-10-09T11:06:28.034476+0000","last_became_active":"2023-10-09T11:05:45.343752+0000","last_became_peered":"2023-10-09T11:05:45.343752+0000","last_unstale":"2023-10-09T11:06:28.034476+0000","last_undegraded":"2023-10-09T11:06:28.034476+0000","last_fullsized":"2023-10-09T11:06:28.034476+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4304,"log_dups_size":0,"ondisk_log_size":4304,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181272576,"num_objects":2766,"num_object_clones":0,"num_object_copies":8298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2766,"num_write_kb":177024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":786,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347501+0000","last_change":"2023-10-09T11:04:53.272323+0000","last_active":"2023-10-09T11:06:30.347501+0000","last_peered":"2023-10-09T11:06:30.347501+0000","last_clean":"2023-10-09T11:06:30.347501+0000","last_became_active":"2023-10-09T11:04:53.272010+0000","last_became_peered":"2023-10-09T11:04:53.272010+0000","last_unstale":"2023-10-09T11:06:30.347501+0000","last_undegraded":"2023-10-09T11:06:30.347501+0000","last_fullsized":"2023-10-09T11:06:30.347501+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:23.913605+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":547,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.237869+0000","last_change":"2023-10-09T11:05:20.307499+0000","last_active":"2023-10-09T11:06:24.237869+0000","last_peered":"2023-10-09T11:06:24.237869+0000","last_clean":"2023-10-09T11:06:24.237869+0000","last_became_active":"2023-10-09T11:05:20.307198+0000","last_became_peered":"2023-10-09T11:05:20.307198+0000","last_unstale":"2023-10-09T11:06:24.237869+0000","last_undegraded":"2023-10-09T11:06:24.237869+0000","last_fullsized":"2023-10-09T11:06:24.237869+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:53.726844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":285,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-09T11:06:29.366228+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:06:29.366228+0000","last_peered":"2023-10-09T11:06:29.366228+0000","last_clean":"2023-10-09T11:06:29.366228+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:06:29.366228+0000","last_undegraded":"2023-10-09T11:06:29.366228+0000","last_fullsized":"2023-10-09T11:06:29.366228+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"525'4509","reported_seq":4566,"reported_epoch":525,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T11:06:28.139714+0000","last_change":"2023-10-09T11:05:46.371093+0000","last_active":"2023-10-09T11:06:28.139714+0000","last_peered":"2023-10-09T11:06:28.139714+0000","last_clean":"2023-10-09T11:05:19.266392+0000","last_became_active":"2023-10-09T11:05:46.367557+0000","last_became_peered":"2023-10-09T11:05:46.367557+0000","last_unstale":"2023-10-09T11:06:28.139714+0000","last_undegraded":"2023-10-09T11:05:46.344345+0000","last_fullsized":"2023-10-09T11:06:28.139714+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4509,"log_dups_size":0,"ondisk_log_size":4509,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189431808,"num_objects":2890,"num_object_clones":0,"num_object_copies":8670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":671,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2890,"num_write_kb":184992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":671},{"shards":"0,3,6","objects":2219}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":874,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.236653+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:06:24.236653+0000","last_peered":"2023-10-09T11:06:24.236653+0000","last_clean":"2023-10-09T11:06:24.236653+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:06:24.236653+0000","last_undegraded":"2023-10-09T11:06:24.236653+0000","last_fullsized":"2023-10-09T11:06:24.236653+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":548,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-09T11:06:29.362792+0000","last_change":"2023-10-09T11:04:57.953307+0000","last_active":"2023-10-09T11:06:29.362792+0000","last_peered":"2023-10-09T11:06:29.362792+0000","last_clean":"2023-10-09T11:06:29.362792+0000","last_became_active":"2023-10-09T11:04:57.953095+0000","last_became_peered":"2023-10-09T11:04:57.953095+0000","last_unstale":"2023-10-09T11:06:29.362792+0000","last_undegraded":"2023-10-09T11:06:29.362792+0000","last_fullsized":"2023-10-09T11:06:29.362792+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:05.562643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":232,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.238362+0000","last_change":"2023-10-09T11:05:20.310255+0000","last_active":"2023-10-09T11:06:24.238362+0000","last_peered":"2023-10-09T11:06:24.238362+0000","last_clean":"2023-10-09T11:06:24.238362+0000","last_became_active":"2023-10-09T11:05:20.309109+0000","last_became_peered":"2023-10-09T11:05:20.309109+0000","last_unstale":"2023-10-09T11:06:24.238362+0000","last_undegraded":"2023-10-09T11:06:24.238362+0000","last_fullsized":"2023-10-09T11:06:24.238362+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:07.429039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.0","version":"525'4414","reported_seq":4470,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:06:28.030470+0000","last_change":"2023-10-09T11:06:09.123915+0000","last_active":"2023-10-09T11:06:28.030470+0000","last_peered":"2023-10-09T11:06:28.030470+0000","last_clean":"2023-10-09T11:06:06.975697+0000","last_became_active":"2023-10-09T11:06:09.120815+0000","last_became_peered":"2023-10-09T11:06:09.120815+0000","last_unstale":"2023-10-09T11:06:28.030470+0000","last_undegraded":"2023-10-09T11:06:09.036294+0000","last_fullsized":"2023-10-09T11:06:09.017945+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4414,"log_dups_size":0,"ondisk_log_size":4414,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186875904,"num_objects":2851,"num_object_clones":0,"num_object_copies":8553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2296,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2851,"num_write_kb":182496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3],"avail_no_missing":["1","3","7"],"object_location_counts":[{"shards":"1,3","objects":555},{"shards":"1,3,7","objects":2296}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":770,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347803+0000","last_change":"2023-10-09T11:05:23.329329+0000","last_active":"2023-10-09T11:06:30.347803+0000","last_peered":"2023-10-09T11:06:30.347803+0000","last_clean":"2023-10-09T11:06:30.347803+0000","last_became_active":"2023-10-09T11:05:23.328976+0000","last_became_peered":"2023-10-09T11:05:23.328976+0000","last_unstale":"2023-10-09T11:06:30.347803+0000","last_undegraded":"2023-10-09T11:06:30.347803+0000","last_fullsized":"2023-10-09T11:06:30.347803+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:41.848184+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":59,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.228298+0000","last_change":"2023-10-09T11:06:08.004080+0000","last_active":"2023-10-09T11:06:24.228298+0000","last_peered":"2023-10-09T11:06:24.228298+0000","last_clean":"2023-10-09T11:06:24.228298+0000","last_became_active":"2023-10-09T11:06:08.003814+0000","last_became_peered":"2023-10-09T11:06:08.003814+0000","last_unstale":"2023-10-09T11:06:24.228298+0000","last_undegraded":"2023-10-09T11:06:24.228298+0000","last_fullsized":"2023-10-09T11:06:24.228298+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:45:19.255544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":229,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.237319+0000","last_change":"2023-10-09T11:04:50.332978+0000","last_active":"2023-10-09T11:06:24.237319+0000","last_peered":"2023-10-09T11:06:24.237319+0000","last_clean":"2023-10-09T11:06:24.237319+0000","last_became_active":"2023-10-09T11:04:50.332735+0000","last_became_peered":"2023-10-09T11:04:50.332735+0000","last_unstale":"2023-10-09T11:06:24.237319+0000","last_undegraded":"2023-10-09T11:06:24.237319+0000","last_fullsized":"2023-10-09T11:06:24.237319+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:39:44.659751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"527'5848","reported_seq":8724,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:31.186920+0000","last_change":"2023-10-09T11:05:35.312841+0000","last_active":"2023-10-09T11:06:31.186920+0000","last_peered":"2023-10-09T11:06:31.186920+0000","last_clean":"2023-10-09T11:06:31.186920+0000","last_became_active":"2023-10-09T11:05:35.309412+0000","last_became_peered":"2023-10-09T11:05:35.309412+0000","last_unstale":"2023-10-09T11:06:31.186920+0000","last_undegraded":"2023-10-09T11:06:31.186920+0000","last_fullsized":"2023-10-09T11:06:31.186920+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5848,"log_dups_size":0,"ondisk_log_size":5848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:12.759777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383254528,"num_objects":5848,"num_object_clones":0,"num_object_copies":17544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5848,"num_write_kb":374272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1408,"num_bytes_recovered":92274688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":1033,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.229321+0000","last_change":"2023-10-09T11:06:11.008298+0000","last_active":"2023-10-09T11:06:24.229321+0000","last_peered":"2023-10-09T11:06:24.229321+0000","last_clean":"2023-10-09T11:06:24.229321+0000","last_became_active":"2023-10-09T11:06:11.007898+0000","last_became_peered":"2023-10-09T11:06:11.007898+0000","last_unstale":"2023-10-09T11:06:24.229321+0000","last_undegraded":"2023-10-09T11:06:24.229321+0000","last_fullsized":"2023-10-09T11:06:24.229321+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:02.771904+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":245,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.341437+0000","last_change":"2023-10-09T11:06:08.022923+0000","last_active":"2023-10-09T11:06:25.341437+0000","last_peered":"2023-10-09T11:06:25.341437+0000","last_clean":"2023-10-09T11:06:25.341437+0000","last_became_active":"2023-10-09T11:06:08.022611+0000","last_became_peered":"2023-10-09T11:06:08.022611+0000","last_unstale":"2023-10-09T11:06:25.341437+0000","last_undegraded":"2023-10-09T11:06:25.341437+0000","last_fullsized":"2023-10-09T11:06:25.341437+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:27:25.911404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":403,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.348098+0000","last_change":"2023-10-09T11:04:50.341353+0000","last_active":"2023-10-09T11:06:30.348098+0000","last_peered":"2023-10-09T11:06:30.348098+0000","last_clean":"2023-10-09T11:06:30.348098+0000","last_became_active":"2023-10-09T11:04:50.340801+0000","last_became_peered":"2023-10-09T11:04:50.340801+0000","last_unstale":"2023-10-09T11:06:30.348098+0000","last_undegraded":"2023-10-09T11:06:30.348098+0000","last_fullsized":"2023-10-09T11:06:30.348098+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:00.699069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":206,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.336300+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:06:25.336300+0000","last_peered":"2023-10-09T11:06:25.336300+0000","last_clean":"2023-10-09T11:06:25.336300+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:06:25.336300+0000","last_undegraded":"2023-10-09T11:06:25.336300+0000","last_fullsized":"2023-10-09T11:06:25.336300+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"525'5676","reported_seq":5715,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:28.128864+0000","last_change":"2023-10-09T11:05:05.295729+0000","last_active":"2023-10-09T11:06:28.128864+0000","last_peered":"2023-10-09T11:06:28.128864+0000","last_clean":"2023-10-09T11:06:28.128864+0000","last_became_active":"2023-10-09T11:05:05.295425+0000","last_became_peered":"2023-10-09T11:05:05.295425+0000","last_unstale":"2023-10-09T11:06:28.128864+0000","last_undegraded":"2023-10-09T11:06:28.128864+0000","last_fullsized":"2023-10-09T11:06:28.128864+0000","mapping_epoch":495,"log_start":"498'1287","ondisk_log_start":"498'1287","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4389,"log_dups_size":0,"ondisk_log_size":4389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:43:37.313230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371982336,"num_objects":5676,"num_object_clones":0,"num_object_copies":17028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5676,"num_write_kb":363264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":728,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.348031+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:06:30.348031+0000","last_peered":"2023-10-09T11:06:30.348031+0000","last_clean":"2023-10-09T11:06:30.348031+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:06:30.348031+0000","last_undegraded":"2023-10-09T11:06:30.348031+0000","last_fullsized":"2023-10-09T11:06:30.348031+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":410,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347515+0000","last_change":"2023-10-09T11:05:20.316093+0000","last_active":"2023-10-09T11:06:30.347515+0000","last_peered":"2023-10-09T11:06:30.347515+0000","last_clean":"2023-10-09T11:06:30.347515+0000","last_became_active":"2023-10-09T11:05:20.315471+0000","last_became_peered":"2023-10-09T11:05:20.315471+0000","last_unstale":"2023-10-09T11:06:30.347515+0000","last_undegraded":"2023-10-09T11:06:30.347515+0000","last_fullsized":"2023-10-09T11:06:30.347515+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:47.548763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":434,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.350725+0000","last_change":"2023-10-09T11:06:24.245282+0000","last_active":"2023-10-09T11:06:30.350725+0000","last_peered":"2023-10-09T11:06:30.350725+0000","last_clean":"2023-10-09T11:06:30.350725+0000","last_became_active":"2023-10-09T11:06:24.244981+0000","last_became_peered":"2023-10-09T11:06:24.244981+0000","last_unstale":"2023-10-09T11:06:30.350725+0000","last_undegraded":"2023-10-09T11:06:30.350725+0000","last_fullsized":"2023-10-09T11:06:30.350725+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:15.327655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":239,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.336367+0000","last_change":"2023-10-09T11:04:50.311579+0000","last_active":"2023-10-09T11:06:25.336367+0000","last_peered":"2023-10-09T11:06:25.336367+0000","last_clean":"2023-10-09T11:06:25.336367+0000","last_became_active":"2023-10-09T11:04:50.311270+0000","last_became_peered":"2023-10-09T11:04:50.311270+0000","last_unstale":"2023-10-09T11:06:25.336367+0000","last_undegraded":"2023-10-09T11:06:25.336367+0000","last_fullsized":"2023-10-09T11:06:25.336367+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:49.850302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"525'4298","reported_seq":6839,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:28.104774+0000","last_change":"2023-10-09T11:06:09.940480+0000","last_active":"2023-10-09T11:06:28.104774+0000","last_peered":"2023-10-09T11:06:28.104774+0000","last_clean":"2023-10-09T11:06:28.104774+0000","last_became_active":"2023-10-09T11:05:46.348515+0000","last_became_peered":"2023-10-09T11:05:46.348515+0000","last_unstale":"2023-10-09T11:06:28.104774+0000","last_undegraded":"2023-10-09T11:06:28.104774+0000","last_fullsized":"2023-10-09T11:06:28.104774+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4298,"log_dups_size":0,"ondisk_log_size":4298,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181075968,"num_objects":2763,"num_object_clones":0,"num_object_copies":8289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2763,"num_write_kb":176832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":654,"num_bytes_recovered":42893312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":885,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-09T11:06:29.365417+0000","last_change":"2023-10-09T11:06:24.249529+0000","last_active":"2023-10-09T11:06:29.365417+0000","last_peered":"2023-10-09T11:06:29.365417+0000","last_clean":"2023-10-09T11:06:29.365417+0000","last_became_active":"2023-10-09T11:06:24.249172+0000","last_became_peered":"2023-10-09T11:06:24.249172+0000","last_unstale":"2023-10-09T11:06:29.365417+0000","last_undegraded":"2023-10-09T11:06:29.365417+0000","last_fullsized":"2023-10-09T11:06:29.365417+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:22.651032+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":304,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-09T11:06:29.363018+0000","last_change":"2023-10-09T11:04:50.381744+0000","last_active":"2023-10-09T11:06:29.363018+0000","last_peered":"2023-10-09T11:06:29.363018+0000","last_clean":"2023-10-09T11:06:29.363018+0000","last_became_active":"2023-10-09T11:04:50.381253+0000","last_became_peered":"2023-10-09T11:04:50.381253+0000","last_unstale":"2023-10-09T11:06:29.363018+0000","last_undegraded":"2023-10-09T11:06:29.363018+0000","last_fullsized":"2023-10-09T11:06:29.363018+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:24.030274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":224,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.237369+0000","last_change":"2023-10-09T11:05:20.298542+0000","last_active":"2023-10-09T11:06:24.237369+0000","last_peered":"2023-10-09T11:06:24.237369+0000","last_clean":"2023-10-09T11:06:24.237369+0000","last_became_active":"2023-10-09T11:05:20.298119+0000","last_became_peered":"2023-10-09T11:05:20.298119+0000","last_unstale":"2023-10-09T11:06:24.237369+0000","last_undegraded":"2023-10-09T11:06:24.237369+0000","last_fullsized":"2023-10-09T11:06:24.237369+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:00.827214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":203,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.237568+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:06:24.237568+0000","last_peered":"2023-10-09T11:06:24.237568+0000","last_clean":"2023-10-09T11:06:24.237568+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:06:24.237568+0000","last_undegraded":"2023-10-09T11:06:24.237568+0000","last_fullsized":"2023-10-09T11:06:24.237568+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"525'5692","reported_seq":8513,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:28.089927+0000","last_change":"2023-10-09T11:05:57.904886+0000","last_active":"2023-10-09T11:06:28.089927+0000","last_peered":"2023-10-09T11:06:28.089927+0000","last_clean":"2023-10-09T11:06:28.089927+0000","last_became_active":"2023-10-09T11:05:57.899684+0000","last_became_peered":"2023-10-09T11:05:57.899684+0000","last_unstale":"2023-10-09T11:06:28.089927+0000","last_undegraded":"2023-10-09T11:06:28.089927+0000","last_fullsized":"2023-10-09T11:06:28.089927+0000","mapping_epoch":513,"log_start":"498'1224","ondisk_log_start":"498'1224","created":495,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4468,"log_dups_size":0,"ondisk_log_size":4468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:53.134195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373030912,"num_objects":5692,"num_object_clones":0,"num_object_copies":17076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5692,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5692,"num_write_kb":364288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1382,"num_bytes_recovered":90570752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":393,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347861+0000","last_change":"2023-10-09T11:06:08.015704+0000","last_active":"2023-10-09T11:06:30.347861+0000","last_peered":"2023-10-09T11:06:30.347861+0000","last_clean":"2023-10-09T11:06:30.347861+0000","last_became_active":"2023-10-09T11:06:08.015389+0000","last_became_peered":"2023-10-09T11:06:08.015389+0000","last_unstale":"2023-10-09T11:06:30.347861+0000","last_undegraded":"2023-10-09T11:06:30.347861+0000","last_fullsized":"2023-10-09T11:06:30.347861+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:38.784193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":828,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.335924+0000","last_change":"2023-10-09T11:06:24.243865+0000","last_active":"2023-10-09T11:06:25.335924+0000","last_peered":"2023-10-09T11:06:25.335924+0000","last_clean":"2023-10-09T11:06:25.335924+0000","last_became_active":"2023-10-09T11:06:24.243505+0000","last_became_peered":"2023-10-09T11:06:24.243505+0000","last_unstale":"2023-10-09T11:06:25.335924+0000","last_undegraded":"2023-10-09T11:06:25.335924+0000","last_fullsized":"2023-10-09T11:06:25.335924+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:25.070973+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":59,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.229569+0000","last_change":"2023-10-09T11:05:20.310666+0000","last_active":"2023-10-09T11:06:24.229569+0000","last_peered":"2023-10-09T11:06:24.229569+0000","last_clean":"2023-10-09T11:06:24.229569+0000","last_became_active":"2023-10-09T11:05:20.310413+0000","last_became_peered":"2023-10-09T11:05:20.310413+0000","last_unstale":"2023-10-09T11:06:24.229569+0000","last_undegraded":"2023-10-09T11:06:24.229569+0000","last_fullsized":"2023-10-09T11:06:24.229569+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:34.169905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":488,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.351753+0000","last_change":"2023-10-09T11:05:20.316523+0000","last_active":"2023-10-09T11:06:30.351753+0000","last_peered":"2023-10-09T11:06:30.351753+0000","last_clean":"2023-10-09T11:06:30.351753+0000","last_became_active":"2023-10-09T11:05:20.314493+0000","last_became_peered":"2023-10-09T11:05:20.314493+0000","last_unstale":"2023-10-09T11:06:30.351753+0000","last_undegraded":"2023-10-09T11:06:30.351753+0000","last_fullsized":"2023-10-09T11:06:30.351753+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:41.618701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"525'4392","reported_seq":4444,"reported_epoch":525,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T11:06:28.130854+0000","last_change":"2023-10-09T11:05:46.362352+0000","last_active":"2023-10-09T11:06:28.130854+0000","last_peered":"2023-10-09T11:06:28.130854+0000","last_clean":"2023-10-09T11:05:19.275351+0000","last_became_active":"2023-10-09T11:05:46.356286+0000","last_became_peered":"2023-10-09T11:05:46.356286+0000","last_unstale":"2023-10-09T11:06:28.130854+0000","last_undegraded":"2023-10-09T11:05:46.343844+0000","last_fullsized":"2023-10-09T11:06:28.130854+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4392,"log_dups_size":0,"ondisk_log_size":4392,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185368576,"num_objects":2828,"num_object_clones":0,"num_object_copies":8484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":650,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2828,"num_write_kb":181024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":["3","0","1"],"object_location_counts":[{"shards":"0,1,3","objects":650},{"shards":"0,3,6","objects":2178}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":895,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.341165+0000","last_change":"2023-10-09T11:05:23.322074+0000","last_active":"2023-10-09T11:06:25.341165+0000","last_peered":"2023-10-09T11:06:25.341165+0000","last_clean":"2023-10-09T11:06:25.341165+0000","last_became_active":"2023-10-09T11:05:23.321749+0000","last_became_peered":"2023-10-09T11:05:23.321749+0000","last_unstale":"2023-10-09T11:06:25.341165+0000","last_undegraded":"2023-10-09T11:06:25.341165+0000","last_fullsized":"2023-10-09T11:06:25.341165+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:07.010932+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":103,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.352863+0000","last_change":"2023-10-09T11:02:33.198484+0000","last_active":"2023-10-09T11:06:30.352863+0000","last_peered":"2023-10-09T11:06:30.352863+0000","last_clean":"2023-10-09T11:06:30.352863+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:06:30.352863+0000","last_undegraded":"2023-10-09T11:06:30.352863+0000","last_fullsized":"2023-10-09T11:06:30.352863+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":151,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.328402+0000","last_change":"2023-10-09T11:06:24.248362+0000","last_active":"2023-10-09T11:06:25.328402+0000","last_peered":"2023-10-09T11:06:25.328402+0000","last_clean":"2023-10-09T11:06:25.328402+0000","last_became_active":"2023-10-09T11:06:24.247348+0000","last_became_peered":"2023-10-09T11:06:24.247348+0000","last_unstale":"2023-10-09T11:06:25.328402+0000","last_undegraded":"2023-10-09T11:06:25.328402+0000","last_fullsized":"2023-10-09T11:06:25.328402+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:26.728754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":350,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.338559+0000","last_change":"2023-10-09T11:06:24.253511+0000","last_active":"2023-10-09T11:06:25.338559+0000","last_peered":"2023-10-09T11:06:25.338559+0000","last_clean":"2023-10-09T11:06:25.338559+0000","last_became_active":"2023-10-09T11:06:24.252739+0000","last_became_peered":"2023-10-09T11:06:24.252739+0000","last_unstale":"2023-10-09T11:06:25.338559+0000","last_undegraded":"2023-10-09T11:06:25.338559+0000","last_fullsized":"2023-10-09T11:06:25.338559+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:50.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"525'5654","reported_seq":5693,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:28.153867+0000","last_change":"2023-10-09T11:05:05.921655+0000","last_active":"2023-10-09T11:06:28.153867+0000","last_peered":"2023-10-09T11:06:28.153867+0000","last_clean":"2023-10-09T11:06:28.153867+0000","last_became_active":"2023-10-09T11:05:05.921204+0000","last_became_peered":"2023-10-09T11:05:05.921204+0000","last_unstale":"2023-10-09T11:06:28.153867+0000","last_undegraded":"2023-10-09T11:06:28.153867+0000","last_fullsized":"2023-10-09T11:06:28.153867+0000","mapping_epoch":495,"log_start":"498'1219","ondisk_log_start":"498'1219","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4435,"log_dups_size":0,"ondisk_log_size":4435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:16.714889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370540544,"num_objects":5654,"num_object_clones":0,"num_object_copies":16962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5654,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":755,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.333194+0000","last_change":"2023-10-09T11:06:24.239676+0000","last_active":"2023-10-09T11:06:25.333194+0000","last_peered":"2023-10-09T11:06:25.333194+0000","last_clean":"2023-10-09T11:06:25.333194+0000","last_became_active":"2023-10-09T11:06:24.239375+0000","last_became_peered":"2023-10-09T11:06:24.239375+0000","last_unstale":"2023-10-09T11:06:25.333194+0000","last_undegraded":"2023-10-09T11:06:25.333194+0000","last_fullsized":"2023-10-09T11:06:25.333194+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:27.953053+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":286,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.336235+0000","last_change":"2023-10-09T11:06:24.245532+0000","last_active":"2023-10-09T11:06:25.336235+0000","last_peered":"2023-10-09T11:06:25.336235+0000","last_clean":"2023-10-09T11:06:25.336235+0000","last_became_active":"2023-10-09T11:06:24.245225+0000","last_became_peered":"2023-10-09T11:06:24.245225+0000","last_unstale":"2023-10-09T11:06:25.336235+0000","last_undegraded":"2023-10-09T11:06:25.336235+0000","last_fullsized":"2023-10-09T11:06:25.336235+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:56:37.826895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":206,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.341691+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:06:25.341691+0000","last_peered":"2023-10-09T11:06:25.341691+0000","last_clean":"2023-10-09T11:06:25.341691+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:06:25.341691+0000","last_undegraded":"2023-10-09T11:06:25.341691+0000","last_fullsized":"2023-10-09T11:06:25.341691+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":477,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347792+0000","last_change":"2023-10-09T11:06:24.245532+0000","last_active":"2023-10-09T11:06:30.347792+0000","last_peered":"2023-10-09T11:06:30.347792+0000","last_clean":"2023-10-09T11:06:30.347792+0000","last_became_active":"2023-10-09T11:06:24.245185+0000","last_became_peered":"2023-10-09T11:06:24.245185+0000","last_unstale":"2023-10-09T11:06:30.347792+0000","last_undegraded":"2023-10-09T11:06:30.347792+0000","last_fullsized":"2023-10-09T11:06:30.347792+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:53.922569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"525'5715","reported_seq":5759,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:06:28.153316+0000","last_change":"2023-10-09T11:06:09.186902+0000","last_active":"2023-10-09T11:06:28.153316+0000","last_peered":"2023-10-09T11:06:28.153316+0000","last_clean":"2023-10-09T11:06:06.963741+0000","last_became_active":"2023-10-09T11:06:09.174460+0000","last_became_peered":"2023-10-09T11:06:09.174460+0000","last_unstale":"2023-10-09T11:06:28.153316+0000","last_undegraded":"2023-10-09T11:06:09.038779+0000","last_fullsized":"2023-10-09T11:06:08.998852+0000","mapping_epoch":517,"log_start":"498'1295","ondisk_log_start":"498'1295","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4420,"log_dups_size":0,"ondisk_log_size":4420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:06.687436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374538240,"num_objects":5715,"num_object_clones":0,"num_object_copies":17145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4742,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5715,"num_write_kb":365760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2","objects":973},{"shards":"1,2,7","objects":4742}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":725,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-09T11:06:29.366017+0000","last_change":"2023-10-09T11:05:23.323368+0000","last_active":"2023-10-09T11:06:29.366017+0000","last_peered":"2023-10-09T11:06:29.366017+0000","last_clean":"2023-10-09T11:06:29.366017+0000","last_became_active":"2023-10-09T11:05:23.323032+0000","last_became_peered":"2023-10-09T11:05:23.323032+0000","last_unstale":"2023-10-09T11:06:29.366017+0000","last_undegraded":"2023-10-09T11:06:29.366017+0000","last_fullsized":"2023-10-09T11:06:29.366017+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:09.393584+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":67,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.229083+0000","last_change":"2023-10-09T11:06:08.027812+0000","last_active":"2023-10-09T11:06:24.229083+0000","last_peered":"2023-10-09T11:06:24.229083+0000","last_clean":"2023-10-09T11:06:24.229083+0000","last_became_active":"2023-10-09T11:06:08.014997+0000","last_became_peered":"2023-10-09T11:06:08.014997+0000","last_unstale":"2023-10-09T11:06:24.229083+0000","last_undegraded":"2023-10-09T11:06:24.229083+0000","last_fullsized":"2023-10-09T11:06:24.229083+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:51:00.180123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":232,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.238440+0000","last_change":"2023-10-09T11:05:20.310417+0000","last_active":"2023-10-09T11:06:24.238440+0000","last_peered":"2023-10-09T11:06:24.238440+0000","last_clean":"2023-10-09T11:06:24.238440+0000","last_became_active":"2023-10-09T11:05:20.309827+0000","last_became_peered":"2023-10-09T11:05:20.309827+0000","last_unstale":"2023-10-09T11:06:24.238440+0000","last_undegraded":"2023-10-09T11:06:24.238440+0000","last_fullsized":"2023-10-09T11:06:24.238440+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:23.934383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":59,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.228079+0000","last_change":"2023-10-09T11:06:08.015340+0000","last_active":"2023-10-09T11:06:24.228079+0000","last_peered":"2023-10-09T11:06:24.228079+0000","last_clean":"2023-10-09T11:06:24.228079+0000","last_became_active":"2023-10-09T11:06:08.015029+0000","last_became_peered":"2023-10-09T11:06:08.015029+0000","last_unstale":"2023-10-09T11:06:24.228079+0000","last_undegraded":"2023-10-09T11:06:24.228079+0000","last_fullsized":"2023-10-09T11:06:24.228079+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:19:32.873111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"525'5581","reported_seq":5620,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:28.143341+0000","last_change":"2023-10-09T11:05:05.921539+0000","last_active":"2023-10-09T11:06:28.143341+0000","last_peered":"2023-10-09T11:06:28.143341+0000","last_clean":"2023-10-09T11:06:28.143341+0000","last_became_active":"2023-10-09T11:05:05.921243+0000","last_became_peered":"2023-10-09T11:05:05.921243+0000","last_unstale":"2023-10-09T11:06:28.143341+0000","last_undegraded":"2023-10-09T11:06:28.143341+0000","last_fullsized":"2023-10-09T11:06:28.143341+0000","mapping_epoch":495,"log_start":"498'1103","ondisk_log_start":"498'1103","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4478,"log_dups_size":0,"ondisk_log_size":4478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:53:48.841133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365756416,"num_objects":5581,"num_object_clones":0,"num_object_copies":16743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5581,"num_write_kb":357184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":670,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.228387+0000","last_change":"2023-10-09T11:06:11.009130+0000","last_active":"2023-10-09T11:06:24.228387+0000","last_peered":"2023-10-09T11:06:24.228387+0000","last_clean":"2023-10-09T11:06:24.228387+0000","last_became_active":"2023-10-09T11:06:11.008862+0000","last_became_peered":"2023-10-09T11:06:11.008862+0000","last_unstale":"2023-10-09T11:06:24.228387+0000","last_undegraded":"2023-10-09T11:06:24.228387+0000","last_fullsized":"2023-10-09T11:06:24.228387+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:03.653371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:03.653371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T23:02:04.020348+0000","scrub_duration":0.001576771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":229,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-09T11:06:29.364091+0000","last_change":"2023-10-09T11:06:08.013083+0000","last_active":"2023-10-09T11:06:29.364091+0000","last_peered":"2023-10-09T11:06:29.364091+0000","last_clean":"2023-10-09T11:06:29.364091+0000","last_became_active":"2023-10-09T11:06:08.012786+0000","last_became_peered":"2023-10-09T11:06:08.012786+0000","last_unstale":"2023-10-09T11:06:29.364091+0000","last_undegraded":"2023-10-09T11:06:29.364091+0000","last_fullsized":"2023-10-09T11:06:29.364091+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:49.739552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":223,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.338880+0000","last_change":"2023-10-09T11:06:24.253714+0000","last_active":"2023-10-09T11:06:25.338880+0000","last_peered":"2023-10-09T11:06:25.338880+0000","last_clean":"2023-10-09T11:06:25.338880+0000","last_became_active":"2023-10-09T11:06:24.253143+0000","last_became_peered":"2023-10-09T11:06:24.253143+0000","last_unstale":"2023-10-09T11:06:25.338880+0000","last_undegraded":"2023-10-09T11:06:25.338880+0000","last_fullsized":"2023-10-09T11:06:25.338880+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:05.682288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":333,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347588+0000","last_change":"2023-10-09T11:05:20.313560+0000","last_active":"2023-10-09T11:06:30.347588+0000","last_peered":"2023-10-09T11:06:30.347588+0000","last_clean":"2023-10-09T11:06:30.347588+0000","last_became_active":"2023-10-09T11:05:20.313214+0000","last_became_peered":"2023-10-09T11:05:20.313214+0000","last_unstale":"2023-10-09T11:06:30.347588+0000","last_undegraded":"2023-10-09T11:06:30.347588+0000","last_fullsized":"2023-10-09T11:06:30.347588+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:32.527205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"525'4489","reported_seq":5879,"reported_epoch":525,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:06:28.051817+0000","last_change":"2023-10-09T11:06:11.695063+0000","last_active":"2023-10-09T11:06:28.051817+0000","last_peered":"2023-10-09T11:06:28.051817+0000","last_clean":"2023-10-09T11:06:06.903330+0000","last_became_active":"2023-10-09T11:06:11.693504+0000","last_became_peered":"2023-10-09T11:06:11.693504+0000","last_unstale":"2023-10-09T11:06:28.051817+0000","last_undegraded":"2023-10-09T11:06:28.051817+0000","last_fullsized":"2023-10-09T11:06:28.051817+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4489,"log_dups_size":0,"ondisk_log_size":4489,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189890560,"num_objects":2898,"num_object_clones":0,"num_object_copies":8694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2898,"num_objects_unfound":0,"num_objects_dirty":2898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2898,"num_write_kb":185440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":662,"num_bytes_recovered":43384832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,7],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6,7","objects":2898}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":931,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.229403+0000","last_change":"2023-10-09T11:05:23.329886+0000","last_active":"2023-10-09T11:06:24.229403+0000","last_peered":"2023-10-09T11:06:24.229403+0000","last_clean":"2023-10-09T11:06:24.229403+0000","last_became_active":"2023-10-09T11:05:23.329561+0000","last_became_peered":"2023-10-09T11:05:23.329561+0000","last_unstale":"2023-10-09T11:06:24.229403+0000","last_undegraded":"2023-10-09T11:06:24.229403+0000","last_fullsized":"2023-10-09T11:06:24.229403+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:46.933799+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":472,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-09T11:06:25.340764+0000","last_change":"2023-10-09T11:06:24.246144+0000","last_active":"2023-10-09T11:06:25.340764+0000","last_peered":"2023-10-09T11:06:25.340764+0000","last_clean":"2023-10-09T11:06:25.340764+0000","last_became_active":"2023-10-09T11:06:24.245676+0000","last_became_peered":"2023-10-09T11:06:24.245676+0000","last_unstale":"2023-10-09T11:06:25.340764+0000","last_undegraded":"2023-10-09T11:06:25.340764+0000","last_fullsized":"2023-10-09T11:06:25.340764+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:35:22.720746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":308,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-09T11:06:29.363611+0000","last_change":"2023-10-09T11:04:50.510688+0000","last_active":"2023-10-09T11:06:29.363611+0000","last_peered":"2023-10-09T11:06:29.363611+0000","last_clean":"2023-10-09T11:06:29.363611+0000","last_became_active":"2023-10-09T11:04:50.507404+0000","last_became_peered":"2023-10-09T11:04:50.507404+0000","last_unstale":"2023-10-09T11:06:29.363611+0000","last_undegraded":"2023-10-09T11:06:29.363611+0000","last_fullsized":"2023-10-09T11:06:29.363611+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:13.663394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"528'4410","reported_seq":5808,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T11:06:31.433276+0000","last_change":"2023-10-09T11:06:29.552103+0000","last_active":"2023-10-09T11:06:31.433276+0000","last_peered":"2023-10-09T11:06:31.433276+0000","last_clean":"2023-10-09T11:06:31.433276+0000","last_became_active":"2023-10-09T11:05:45.387324+0000","last_became_peered":"2023-10-09T11:05:45.387324+0000","last_unstale":"2023-10-09T11:06:31.433276+0000","last_undegraded":"2023-10-09T11:06:31.433276+0000","last_fullsized":"2023-10-09T11:06:31.433276+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4410,"log_dups_size":0,"ondisk_log_size":4410,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187236352,"num_objects":2857,"num_object_clones":0,"num_object_copies":8571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2857,"num_write_kb":182848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":667,"num_bytes_recovered":43712512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":888,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.229421+0000","last_change":"2023-10-09T11:05:23.321040+0000","last_active":"2023-10-09T11:06:24.229421+0000","last_peered":"2023-10-09T11:06:24.229421+0000","last_clean":"2023-10-09T11:06:24.229421+0000","last_became_active":"2023-10-09T11:05:23.320674+0000","last_became_peered":"2023-10-09T11:05:23.320674+0000","last_unstale":"2023-10-09T11:06:24.229421+0000","last_undegraded":"2023-10-09T11:06:24.229421+0000","last_fullsized":"2023-10-09T11:06:24.229421+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.527660+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":355,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347564+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:06:30.347564+0000","last_peered":"2023-10-09T11:06:30.347564+0000","last_clean":"2023-10-09T11:06:30.347564+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:06:30.347564+0000","last_undegraded":"2023-10-09T11:06:30.347564+0000","last_fullsized":"2023-10-09T11:06:30.347564+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":50,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.228314+0000","last_change":"2023-10-09T11:04:50.511984+0000","last_active":"2023-10-09T11:06:24.228314+0000","last_peered":"2023-10-09T11:06:24.228314+0000","last_clean":"2023-10-09T11:06:24.228314+0000","last_became_active":"2023-10-09T11:04:50.510820+0000","last_became_peered":"2023-10-09T11:04:50.510820+0000","last_unstale":"2023-10-09T11:06:24.228314+0000","last_undegraded":"2023-10-09T11:06:24.228314+0000","last_fullsized":"2023-10-09T11:06:24.228314+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:38.662189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.15","version":"528'4363","reported_seq":9002,"reported_epoch":528,"state":"active+recovery_wait","last_fresh":"2023-10-09T11:06:31.698567+0000","last_change":"2023-10-09T11:06:30.390445+0000","last_active":"2023-10-09T11:06:31.698567+0000","last_peered":"2023-10-09T11:06:31.698567+0000","last_clean":"2023-10-09T11:05:19.279287+0000","last_became_active":"2023-10-09T11:06:30.358814+0000","last_became_peered":"2023-10-09T11:06:30.358814+0000","last_unstale":"2023-10-09T11:06:31.698567+0000","last_undegraded":"2023-10-09T11:06:31.698567+0000","last_fullsized":"2023-10-09T11:06:31.698567+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4363,"log_dups_size":0,"ondisk_log_size":4363,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186482688,"num_objects":2846,"num_object_clones":0,"num_object_copies":8538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2846,"num_write_kb":182112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2280,"num_bytes_recovered":149422080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":["5","3","6","7"],"object_location_counts":[{"shards":"3,5,6","objects":2846}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":506,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.347957+0000","last_change":"2023-10-09T11:05:20.327032+0000","last_active":"2023-10-09T11:06:30.347957+0000","last_peered":"2023-10-09T11:06:30.347957+0000","last_clean":"2023-10-09T11:06:30.347957+0000","last_became_active":"2023-10-09T11:05:20.320193+0000","last_became_peered":"2023-10-09T11:05:20.320193+0000","last_unstale":"2023-10-09T11:06:30.347957+0000","last_undegraded":"2023-10-09T11:06:30.347957+0000","last_fullsized":"2023-10-09T11:06:30.347957+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:44.862890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":50,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.229305+0000","last_change":"2023-10-09T11:04:50.384219+0000","last_active":"2023-10-09T11:06:24.229305+0000","last_peered":"2023-10-09T11:06:24.229305+0000","last_clean":"2023-10-09T11:06:24.229305+0000","last_became_active":"2023-10-09T11:04:50.384028+0000","last_became_peered":"2023-10-09T11:04:50.384028+0000","last_unstale":"2023-10-09T11:06:24.229305+0000","last_undegraded":"2023-10-09T11:06:24.229305+0000","last_fullsized":"2023-10-09T11:06:24.229305+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:04.721869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.16","version":"527'4423","reported_seq":4483,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:31.159452+0000","last_change":"2023-10-09T11:06:24.256923+0000","last_active":"2023-10-09T11:06:31.159452+0000","last_peered":"2023-10-09T11:06:31.159452+0000","last_clean":"2023-10-09T11:06:31.159452+0000","last_became_active":"2023-10-09T11:06:24.254366+0000","last_became_peered":"2023-10-09T11:06:24.254366+0000","last_unstale":"2023-10-09T11:06:31.159452+0000","last_undegraded":"2023-10-09T11:06:31.159452+0000","last_fullsized":"2023-10-09T11:06:31.159452+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4423,"log_dups_size":0,"ondisk_log_size":4423,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189857792,"num_objects":2897,"num_object_clones":0,"num_object_copies":8691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2897,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2897,"num_write_kb":185408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":688,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.235376+0000","last_change":"2023-10-09T11:06:11.011275+0000","last_active":"2023-10-09T11:06:24.235376+0000","last_peered":"2023-10-09T11:06:24.235376+0000","last_clean":"2023-10-09T11:06:24.235376+0000","last_became_active":"2023-10-09T11:06:11.010947+0000","last_became_peered":"2023-10-09T11:06:11.010947+0000","last_unstale":"2023-10-09T11:06:24.235376+0000","last_undegraded":"2023-10-09T11:06:24.235376+0000","last_fullsized":"2023-10-09T11:06:24.235376+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:04.790935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.790935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":59,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.229492+0000","last_change":"2023-10-09T11:05:20.299321+0000","last_active":"2023-10-09T11:06:24.229492+0000","last_peered":"2023-10-09T11:06:24.229492+0000","last_clean":"2023-10-09T11:06:24.229492+0000","last_became_active":"2023-10-09T11:05:20.299019+0000","last_became_peered":"2023-10-09T11:05:20.299019+0000","last_unstale":"2023-10-09T11:06:24.229492+0000","last_undegraded":"2023-10-09T11:06:24.229492+0000","last_fullsized":"2023-10-09T11:06:24.229492+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:08.645909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":50,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.229750+0000","last_change":"2023-10-09T11:04:50.366890+0000","last_active":"2023-10-09T11:06:24.229750+0000","last_peered":"2023-10-09T11:06:24.229750+0000","last_clean":"2023-10-09T11:06:24.229750+0000","last_became_active":"2023-10-09T11:04:50.366558+0000","last_became_peered":"2023-10-09T11:04:50.366558+0000","last_unstale":"2023-10-09T11:06:24.229750+0000","last_undegraded":"2023-10-09T11:06:24.229750+0000","last_fullsized":"2023-10-09T11:06:24.229750+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:50.993153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"528'4459","reported_seq":4508,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-09T11:06:31.682833+0000","last_change":"2023-10-09T11:05:45.360245+0000","last_active":"2023-10-09T11:06:31.682833+0000","last_peered":"2023-10-09T11:06:31.682833+0000","last_clean":"2023-10-09T11:06:31.682833+0000","last_became_active":"2023-10-09T11:05:45.356981+0000","last_became_peered":"2023-10-09T11:05:45.356981+0000","last_unstale":"2023-10-09T11:06:31.682833+0000","last_undegraded":"2023-10-09T11:06:31.682833+0000","last_fullsized":"2023-10-09T11:06:31.682833+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":4459,"log_dups_size":0,"ondisk_log_size":4459,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189366272,"num_objects":2890,"num_object_clones":0,"num_object_copies":8670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2890,"num_write_kb":184928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":823,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.235676+0000","last_change":"2023-10-09T11:04:53.274332+0000","last_active":"2023-10-09T11:06:24.235676+0000","last_peered":"2023-10-09T11:06:24.235676+0000","last_clean":"2023-10-09T11:06:24.235676+0000","last_became_active":"2023-10-09T11:04:53.273798+0000","last_became_peered":"2023-10-09T11:04:53.273798+0000","last_unstale":"2023-10-09T11:06:24.235676+0000","last_undegraded":"2023-10-09T11:06:24.235676+0000","last_fullsized":"2023-10-09T11:06:24.235676+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:04.220970+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":75,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-09T11:06:30.348478+0000","last_change":"2023-10-09T11:06:24.249877+0000","last_active":"2023-10-09T11:06:30.348478+0000","last_peered":"2023-10-09T11:06:30.348478+0000","last_clean":"2023-10-09T11:06:30.348478+0000","last_became_active":"2023-10-09T11:06:24.249573+0000","last_became_peered":"2023-10-09T11:06:24.249573+0000","last_unstale":"2023-10-09T11:06:30.348478+0000","last_undegraded":"2023-10-09T11:06:30.348478+0000","last_fullsized":"2023-10-09T11:06:30.348478+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:11.798192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":572,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T11:06:24.240419+0000","last_change":"2023-10-09T11:05:20.307347+0000","last_active":"2023-10-09T11:06:24.240419+0000","last_peered":"2023-10-09T11:06:24.240419+0000","last_clean":"2023-10-09T11:06:24.240419+0000","last_became_active":"2023-10-09T11:05:20.307126+0000","last_became_peered":"2023-10-09T11:05:20.307126+0000","last_unstale":"2023-10-09T11:06:24.240419+0000","last_undegraded":"2023-10-09T11:06:24.240419+0000","last_fullsized":"2023-10-09T11:06:24.240419+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:34.408515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":5980487680,"num_objects":91255,"num_object_clones":0,"num_object_copies":273765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16059,"num_objects_misplaced":2898,"num_objects_unfound":0,"num_objects_dirty":91255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":91255,"num_write_kb":5840320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9063,"num_bytes_recovered":593952768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18380554240,"data_stored":18380554240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116277,"ondisk_log_size":116277,"up":78,"acting":73,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":10930,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12694,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":626148,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":31,"num_read_kb":56,"num_write":15,"num_write_kb":147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2369964,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1515520,"data_stored":1508416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332479,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2874144,"kb_used_data":1970648,"kb_used_omap":0,"kb_used_meta":903488,"kb_avail":90871008,"statfs":{"total":95995035648,"available":93051912192,"internally_reserved":0,"allocated":2017943552,"data_stored":2016280599,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925171712},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":1.4850000000000001,"15min":1.742},"min":{"1min":0.20599999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.214,"5min":9.8040000000000003,"15min":9.8040000000000003},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.456,"15min":1.718},"min":{"1min":0.27400000000000002,"5min":0.23499999999999999,"15min":0.216},"max":{"1min":7.5330000000000004,"5min":9.6999999999999993,"15min":9.6999999999999993},"last":0.76700000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:06:22 2023","interfaces":[{"interface":"back","average":{"1min":1.3799999999999999,"5min":1.325,"15min":1.5509999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.619999999999999,"5min":11.619999999999999,"15min":11.619999999999999},"last":0.26100000000000001},{"interface":"front","average":{"1min":1.9950000000000001,"5min":1.581,"15min":1.8939999999999999},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.26000000000000001},"max":{"1min":8.7629999999999999,"5min":9.6620000000000008,"15min":12.193},"last":0.435}]},{"osd":2,"last update":"Mon Oct 9 11:05:50 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":2.1419999999999999,"15min":1.8009999999999999},"min":{"1min":0.33000000000000002,"5min":0.30199999999999999,"15min":0.245},"max":{"1min":8.5340000000000007,"5min":16.605,"15min":16.605},"last":0.876},{"interface":"front","average":{"1min":1.006,"5min":1.4770000000000001,"15min":1.754},"min":{"1min":0.34300000000000003,"5min":0.27600000000000002,"15min":0.26600000000000001},"max":{"1min":3.8050000000000002,"5min":9.7469999999999999,"15min":13.037000000000001},"last":0.36199999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:06:17 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.401,"15min":1.399},"min":{"1min":0.371,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.7869999999999999,"5min":5.7869999999999999,"15min":5.7869999999999999},"last":3.5600000000000001},{"interface":"front","average":{"1min":1.958,"5min":1.825,"15min":1.8029999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.3920000000000003,"5min":8.2330000000000005,"15min":8.2330000000000005},"last":4.5679999999999996}]},{"osd":4,"last update":"Mon Oct 9 11:06:01 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":1.028,"15min":1.2430000000000001},"min":{"1min":0.247,"5min":0.218,"15min":0.218},"max":{"1min":2.5289999999999999,"5min":7.6589999999999998,"15min":16.695},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.62,"5min":1.5720000000000001,"15min":1.3500000000000001},"min":{"1min":0.28599999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.5990000000000002,"5min":16.481000000000002,"15min":16.481000000000002},"last":0.91700000000000004}]},{"osd":5,"last update":"Mon Oct 9 11:06:01 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.3129999999999999,"15min":1.3959999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":2.1749999999999998,"5min":15.414999999999999,"15min":15.414999999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.65100000000000002,"5min":1.3149999999999999,"15min":1.2989999999999999},"min":{"1min":0.33800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.5590000000000002,"5min":15.369,"15min":15.369},"last":0.47499999999999998}]},{"osd":6,"last update":"Mon Oct 9 11:06:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":1.3859999999999999,"15min":1.478},"min":{"1min":0.36099999999999999,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":1.494,"5min":14.159000000000001,"15min":14.913},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.8520000000000001,"15min":1.5189999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":2.6219999999999999,"5min":16.538,"15min":16.538},"last":1.0549999999999999}]}]},{"osd":6,"up_from":44,"seq":188978561410,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3748208,"kb_used_data":2856232,"kb_used_omap":0,"kb_used_meta":891968,"kb_avail":89996944,"statfs":{"total":95995035648,"available":92156870656,"internally_reserved":0,"allocated":2924781568,"data_stored":2923106525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913375232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:06:20 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":1.6279999999999999,"15min":1.875},"min":{"1min":0.223,"5min":0.217,"15min":0.216},"max":{"1min":3.3159999999999998,"5min":29.379999999999999,"15min":54.743000000000002},"last":1.3899999999999999},{"interface":"front","average":{"1min":0.73599999999999999,"5min":1.4790000000000001,"15min":1.95},"min":{"1min":0.25800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":3.6360000000000001,"5min":29.132999999999999,"15min":53.701000000000001},"last":1.4239999999999999}]},{"osd":1,"last update":"Mon Oct 9 11:06:24 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":1.8460000000000001,"15min":1.1180000000000001},"min":{"1min":0.31,"5min":0.26000000000000001,"15min":0.23499999999999999},"max":{"1min":4.3179999999999996,"5min":41.241999999999997,"15min":41.241999999999997},"last":1.446},{"interface":"front","average":{"1min":0.91300000000000003,"5min":2.1539999999999999,"15min":1.627},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.248},"max":{"1min":2.4689999999999999,"5min":41.476999999999997,"15min":41.476999999999997},"last":0.46999999999999997}]},{"osd":2,"last update":"Mon Oct 9 11:05:50 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.7769999999999999,"15min":1.9059999999999999},"min":{"1min":0.29899999999999999,"5min":0.28699999999999998,"15min":0.28399999999999997},"max":{"1min":1.756,"5min":10.481999999999999,"15min":12.003},"last":2.9660000000000002},{"interface":"front","average":{"1min":0.73299999999999998,"5min":2.145,"15min":2.0899999999999999},"min":{"1min":0.29799999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.508,"5min":12.693,"15min":15.432},"last":0.56200000000000006}]},{"osd":3,"last update":"Mon Oct 9 11:06:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3420000000000001,"5min":1.4279999999999999,"15min":1.4419999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.1900000000000004,"5min":8.0099999999999998,"15min":8.0099999999999998},"last":3.3809999999999998},{"interface":"front","average":{"1min":2.008,"5min":2.3519999999999999,"15min":2.4100000000000001},"min":{"1min":0.33000000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.7089999999999996,"5min":23.815999999999999,"15min":23.815999999999999},"last":3.423}]},{"osd":4,"last update":"Mon Oct 9 11:05:50 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":1.425,"15min":1.3069999999999999},"min":{"1min":0.26900000000000002,"5min":0.255,"15min":0.24199999999999999},"max":{"1min":1.109,"5min":17.216999999999999,"15min":17.216999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.88500000000000001,"15min":1.038},"min":{"1min":0.32200000000000001,"5min":0.252,"15min":0.24399999999999999},"max":{"1min":0.88800000000000001,"5min":10.025,"15min":12.061999999999999},"last":1.5089999999999999}]},{"osd":5,"last update":"Mon Oct 9 11:05:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":1.357,"15min":1.5149999999999999},"min":{"1min":0.34300000000000003,"5min":0.30499999999999999,"15min":0.25600000000000001},"max":{"1min":1.0640000000000001,"5min":10.209,"15min":19.786999999999999},"last":1.478},{"interface":"front","average":{"1min":0.58299999999999996,"5min":1.0469999999999999,"15min":1.1990000000000001},"min":{"1min":0.34100000000000003,"5min":0.248,"15min":0.248},"max":{"1min":1.415,"5min":9.9469999999999992,"15min":12.103999999999999},"last":2.5270000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:06:08 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":1.4339999999999999,"15min":1.4670000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.1040000000000001,"5min":11.550000000000001,"15min":12.882},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.85199999999999998,"15min":1.0049999999999999},"min":{"1min":0.34100000000000003,"5min":0.31,"15min":0.27500000000000002},"max":{"1min":3.2269999999999999,"5min":11.557,"15min":11.557},"last":1.643}]}]},{"osd":1,"up_from":388,"seq":1666447310924,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3424024,"kb_used_data":3042768,"kb_used_omap":13,"kb_used_meta":381234,"kb_avail":90321128,"statfs":{"total":95995035648,"available":92488835072,"internally_reserved":0,"allocated":3115794432,"data_stored":3113033287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":390384327},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.633,"5min":2.246,"15min":2.254},"min":{"1min":0.23499999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":6.056,"5min":13.334,"15min":13.334},"last":6.452},{"interface":"front","average":{"1min":1.9490000000000001,"5min":2.242,"15min":1.621},"min":{"1min":0.28699999999999998,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":9.0220000000000002,"5min":11.706,"15min":11.706},"last":6.2640000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":2.5800000000000001,"15min":3.0059999999999998},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":12.436999999999999,"5min":47.731000000000002,"15min":47.731000000000002},"last":6.4290000000000003},{"interface":"front","average":{"1min":1.994,"5min":2.6240000000000001,"15min":2.363},"min":{"1min":0.308,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.6929999999999996,"5min":47.786999999999999,"15min":47.786999999999999},"last":2.0609999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:06:20 2023","interfaces":[{"interface":"back","average":{"1min":7.2389999999999999,"5min":2.6389999999999998,"15min":1.8720000000000001},"min":{"1min":0.317,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":92.447999999999993,"5min":92.447999999999993,"15min":92.447999999999993},"last":1.8400000000000001},{"interface":"front","average":{"1min":8.0250000000000004,"5min":2.7909999999999999,"15min":1.919},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":92.066999999999993,"5min":92.066999999999993,"15min":92.066999999999993},"last":2.9689999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.7549999999999999,"15min":2.4929999999999999},"min":{"1min":0.314,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.0119999999999996,"5min":10.481,"15min":10.481},"last":3.028},{"interface":"front","average":{"1min":1.5029999999999999,"5min":1.754,"15min":1.508},"min":{"1min":0.35799999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.6719999999999997,"5min":11.44,"15min":11.44},"last":2.036}]},{"osd":5,"last update":"Mon Oct 9 11:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.9339999999999999,"15min":2.339},"min":{"1min":0.35199999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.2220000000000004,"5min":18.053000000000001,"15min":18.053000000000001},"last":2.214},{"interface":"front","average":{"1min":1.496,"5min":2.0110000000000001,"15min":2.2850000000000001},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.31},"max":{"1min":6.0830000000000002,"5min":10.558,"15min":10.558},"last":3.0019999999999998}]},{"osd":6,"last update":"Mon Oct 9 11:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":2.141,"15min":2.415},"min":{"1min":0.37,"5min":0.309,"15min":0.309},"max":{"1min":4.1719999999999997,"5min":18.085999999999999,"15min":18.085999999999999},"last":1.9199999999999999},{"interface":"front","average":{"1min":1.474,"5min":1.796,"15min":2.2040000000000002},"min":{"1min":0.40500000000000003,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.9790000000000001,"5min":10.531000000000001,"15min":10.531000000000001},"last":6.3929999999999998}]},{"osd":7,"last update":"Mon Oct 9 11:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.8759999999999999,"15min":1.4830000000000001},"min":{"1min":0.40699999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.6399999999999997,"5min":11.484,"15min":11.484},"last":1.9990000000000001},{"interface":"front","average":{"1min":1.2809999999999999,"5min":1.833,"15min":2.3530000000000002},"min":{"1min":0.39200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.1280000000000001,"5min":18.071000000000002,"15min":18.071000000000002},"last":1.95}]}]},{"osd":0,"up_from":80,"seq":343597383982,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2721544,"kb_used_data":1876224,"kb_used_omap":28,"kb_used_meta":845283,"kb_avail":91023608,"statfs":{"total":95995035648,"available":93208174592,"internally_reserved":0,"allocated":1921253376,"data_stored":1918439038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":865570640},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:06:25 2023","interfaces":[{"interface":"back","average":{"1min":1.877,"5min":1.9910000000000001,"15min":2.855},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":4.5670000000000002,"5min":14.644,"15min":19.608000000000001},"last":1.1299999999999999},{"interface":"front","average":{"1min":2.3650000000000002,"5min":1.895,"15min":2.8769999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":6.7469999999999999,"5min":8.9060000000000006,"15min":19.992999999999999},"last":4.29}]},{"osd":2,"last update":"Mon Oct 9 11:05:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":2.145,"15min":2.2229999999999999},"min":{"1min":0.23200000000000001,"5min":0.21299999999999999,"15min":0.17199999999999999},"max":{"1min":8.798,"5min":12.459,"15min":24.852},"last":1.149},{"interface":"front","average":{"1min":1.732,"5min":2.1920000000000002,"15min":2.7029999999999998},"min":{"1min":0.27700000000000002,"5min":0.20300000000000001,"15min":0.19800000000000001},"max":{"1min":5.625,"5min":10.616,"15min":77.256},"last":4.234}]},{"osd":3,"last update":"Mon Oct 9 11:06:19 2023","interfaces":[{"interface":"back","average":{"1min":2.968,"5min":3.419,"15min":3.4940000000000002},"min":{"1min":0.32900000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.157,"5min":25.050000000000001,"15min":25.050000000000001},"last":4.266},{"interface":"front","average":{"1min":2.7090000000000001,"5min":2.0609999999999999,"15min":1.9530000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":10.775,"5min":14.032999999999999,"15min":14.032999999999999},"last":3.0510000000000002}]},{"osd":4,"last update":"Mon Oct 9 11:06:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":1.962,"15min":2.4359999999999999},"min":{"1min":0.28199999999999997,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.194,"5min":14.507,"15min":77.177999999999997},"last":4.1449999999999996},{"interface":"front","average":{"1min":1.544,"5min":1.5069999999999999,"15min":1.7110000000000001},"min":{"1min":0.35599999999999998,"5min":0.30199999999999999,"15min":0.28899999999999998},"max":{"1min":6.843,"5min":6.9749999999999996,"15min":10.513999999999999},"last":3.0910000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:06:19 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":1.7450000000000001,"15min":1.883},"min":{"1min":0.33200000000000002,"5min":0.30399999999999999,"15min":0.29799999999999999},"max":{"1min":6.7919999999999998,"5min":7.9039999999999999,"15min":10.544},"last":3.1179999999999999},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.4039999999999999,"15min":1.8280000000000001},"min":{"1min":0.33200000000000002,"5min":0.308,"15min":0.29899999999999999},"max":{"1min":4.4930000000000003,"5min":11.212999999999999,"15min":24.754999999999999},"last":1.1080000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:06:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.669,"15min":1.9490000000000001},"min":{"1min":0.38700000000000001,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":6.819,"5min":7.9119999999999999,"15min":18.920000000000002},"last":3.153},{"interface":"front","average":{"1min":1.4039999999999999,"5min":1.5629999999999999,"15min":1.923},"min":{"1min":0.378,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":4.4489999999999998,"5min":11.257999999999999,"15min":24.800000000000001},"last":1.085}]},{"osd":7,"last update":"Mon Oct 9 11:06:19 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.7829999999999999,"15min":2.004},"min":{"1min":0.41299999999999998,"5min":0.35099999999999998,"15min":0.32700000000000001},"max":{"1min":8.7409999999999997,"5min":17.672000000000001,"15min":17.672000000000001},"last":1.0509999999999999},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.1709999999999998,"15min":2.5859999999999999},"min":{"1min":0.40500000000000003,"5min":0.371,"15min":0.314},"max":{"1min":4.2290000000000001,"5min":17.539000000000001,"15min":77.228999999999999},"last":4.1959999999999997}]}]},{"osd":2,"up_from":90,"seq":386547056923,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3023756,"kb_used_data":2303684,"kb_used_omap":81,"kb_used_meta":719982,"kb_avail":90721396,"statfs":{"total":95995035648,"available":92898709504,"internally_reserved":0,"allocated":2358972416,"data_stored":2356292323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":737261786},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:05:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":2.4260000000000002,"15min":1.9470000000000001},"min":{"1min":0.188,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":3.9900000000000002,"5min":48.933999999999997,"15min":48.933999999999997},"last":1.8939999999999999},{"interface":"front","average":{"1min":1.837,"5min":1.669,"15min":1.798},"min":{"1min":0.248,"5min":0.20699999999999999,"15min":0.19600000000000001},"max":{"1min":7.0640000000000001,"5min":14.515000000000001,"15min":15.595000000000001},"last":0.45100000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:06:28 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":2.4889999999999999,"15min":2.5110000000000001},"min":{"1min":0.17100000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":6.149,"5min":49.033999999999999,"15min":49.033999999999999},"last":3.117},{"interface":"front","average":{"1min":1.627,"5min":2.4329999999999998,"15min":2.581},"min":{"1min":0.222,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":8.8190000000000008,"5min":48.874000000000002,"15min":48.874000000000002},"last":2.5139999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:06:17 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":1.8460000000000001,"15min":1.792},"min":{"1min":0.22700000000000001,"5min":0.214,"15min":0.214},"max":{"1min":10.499000000000001,"5min":10.499000000000001,"15min":10.499000000000001},"last":2.552},{"interface":"front","average":{"1min":2.198,"5min":2.1549999999999998,"15min":2.1480000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.782,"5min":8.782,"15min":8.782},"last":2.5739999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:05:58 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.7849999999999999,"15min":1.9279999999999999},"min":{"1min":0.311,"5min":0.28100000000000003,"15min":0.27000000000000002},"max":{"1min":3.605,"5min":9.7989999999999995,"15min":15.672000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.2030000000000001,"5min":2.2280000000000002,"15min":1.9330000000000001},"min":{"1min":0.376,"5min":0.249,"15min":0.249},"max":{"1min":4.024,"5min":48.795000000000002,"15min":48.795000000000002},"last":2.6179999999999999}]},{"osd":5,"last update":"Mon Oct 9 11:05:58 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":1.272,"15min":1.502},"min":{"1min":0.34899999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.9139999999999997,"5min":6.9139999999999997,"15min":12.128},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.389,"5min":2.1829999999999998,"15min":1.9850000000000001},"min":{"1min":0.35099999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.637,"5min":49.061,"15min":49.061},"last":0.57299999999999995}]},{"osd":6,"last update":"Mon Oct 9 11:05:58 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":2.3479999999999999,"15min":1.8340000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":4.0430000000000001,"5min":48.845999999999997,"15min":48.845999999999997},"last":2.6389999999999998},{"interface":"front","average":{"1min":2.036,"5min":1.601,"15min":1.573},"min":{"1min":0.39700000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.9939999999999998,"5min":7.0570000000000004,"15min":10.111000000000001},"last":0.49299999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:05:58 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":2.5019999999999998,"15min":1.9430000000000001},"min":{"1min":0.39800000000000002,"5min":0.39000000000000001,"15min":0.31900000000000001},"max":{"1min":4.6479999999999997,"5min":48.966000000000001,"15min":48.966000000000001},"last":2.597},{"interface":"front","average":{"1min":1.9330000000000001,"5min":1.619,"15min":1.679},"min":{"1min":0.41599999999999998,"5min":0.34499999999999997,"15min":0.30599999999999999},"max":{"1min":6.9630000000000001,"5min":14.94,"15min":14.94},"last":0.51500000000000001}]}]},{"osd":3,"up_from":469,"seq":2014339661852,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3124228,"kb_used_data":2860284,"kb_used_omap":11,"kb_used_meta":263924,"kb_avail":90620924,"statfs":{"total":95995035648,"available":92795826176,"internally_reserved":0,"allocated":2928930816,"data_stored":2926003025,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11309,"internal_metadata":270259155},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:06:15 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.597,"15min":1.679},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":4.6820000000000004,"5min":14.882,"15min":14.882},"last":7.3470000000000004},{"interface":"front","average":{"1min":1.1519999999999999,"5min":0.92400000000000004,"15min":0.88600000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":4.0069999999999997,"5min":4.0069999999999997,"15min":4.0069999999999997},"last":0.53400000000000003}]},{"osd":1,"last update":"Mon Oct 9 11:06:15 2023","interfaces":[{"interface":"back","average":{"1min":2.6909999999999998,"5min":1.659,"15min":1.4870000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":17.93,"5min":17.93,"15min":17.93},"last":11.757},{"interface":"front","average":{"1min":1.746,"5min":1.617,"15min":1.595},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":7.7670000000000003,"5min":10.18,"15min":10.18},"last":3.6949999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:06:15 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":1.1299999999999999,"15min":1.101},"min":{"1min":0.26000000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.9790000000000001,"5min":6.444,"15min":6.444},"last":7.375},{"interface":"front","average":{"1min":2.488,"5min":1.361,"15min":1.173},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":18.238,"5min":18.238,"15min":18.238},"last":7.3339999999999996}]},{"osd":4,"last update":"Mon Oct 9 11:06:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":1.5009999999999999,"15min":1.5489999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.1660000000000004,"5min":14.907999999999999,"15min":14.907999999999999},"last":3.6059999999999999},{"interface":"front","average":{"1min":1.0940000000000001,"5min":0.77500000000000002,"15min":0.72199999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.1390000000000002,"5min":4.1390000000000002,"15min":4.1390000000000002},"last":0.56100000000000005}]},{"osd":5,"last update":"Mon Oct 9 11:06:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.383,"15min":1.403},"min":{"1min":0.34699999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.653,"5min":9.3300000000000001,"15min":9.3300000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.127,"5min":1.3779999999999999,"15min":1.4199999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.6469999999999998,"5min":9.3130000000000006,"15min":9.3130000000000006},"last":3.641}]},{"osd":6,"last update":"Mon Oct 9 11:06:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4830000000000001,"5min":1.7230000000000001,"15min":1.7629999999999999},"min":{"1min":0.40500000000000003,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.8479999999999999,"5min":15.455,"15min":15.455},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.127,"5min":1.5389999999999999,"15min":1.6080000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.1600000000000001,"5min":14.837,"15min":14.837},"last":3.6669999999999998}]},{"osd":7,"last update":"Mon Oct 9 11:06:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4850000000000001,"5min":1.097,"15min":1.0329999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.1790000000000003,"5min":4.1790000000000003,"15min":4.1790000000000003},"last":0.59499999999999997},{"interface":"front","average":{"1min":2.0550000000000002,"5min":1.3360000000000001,"15min":1.216},"min":{"1min":0.51000000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":6.9119999999999999,"5min":6.9119999999999999,"15min":6.9119999999999999},"last":0.60599999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019273,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2368376,"kb_used_data":1479728,"kb_used_omap":0,"kb_used_meta":888640,"kb_avail":91376776,"statfs":{"total":95995035648,"available":93569818624,"internally_reserved":0,"allocated":1515241472,"data_stored":1513558602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":909967360},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:06:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.6100000000000001,"15min":2.391},"min":{"1min":0.224,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.3460000000000001,"5min":13.162000000000001,"15min":213.88999999999999},"last":1.1639999999999999},{"interface":"front","average":{"1min":1.468,"5min":2.1259999999999999,"15min":2.698},"min":{"1min":0.28799999999999998,"5min":0.24399999999999999,"15min":0.23200000000000001},"max":{"1min":3.9889999999999999,"5min":11.128,"15min":214.09800000000001},"last":0.84599999999999997}]},{"osd":1,"last update":"Mon Oct 9 11:06:20 2023","interfaces":[{"interface":"back","average":{"1min":4.8940000000000001,"5min":3.6190000000000002,"15min":2.601},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":57.966999999999999,"5min":121.41,"15min":121.41},"last":0.73899999999999999},{"interface":"front","average":{"1min":4.1289999999999996,"5min":3.4780000000000002,"15min":2.0699999999999998},"min":{"1min":0.315,"5min":0.251,"15min":0.251},"max":{"1min":57.832000000000001,"5min":122.21899999999999,"15min":122.21899999999999},"last":3.0489999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:05:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.7569999999999999,"15min":1.8},"min":{"1min":0.246,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.0330000000000004,"5min":10.699999999999999,"15min":15.446999999999999},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.361,"5min":1.984,"15min":2.109},"min":{"1min":0.35099999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.8419999999999996,"5min":8.4250000000000007,"15min":27.818000000000001},"last":0.42399999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:06:20 2023","interfaces":[{"interface":"back","average":{"1min":5.1420000000000003,"5min":2.4289999999999998,"15min":1.9770000000000001},"min":{"1min":0.39900000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":60.753,"5min":60.753,"15min":60.753},"last":0.40899999999999997},{"interface":"front","average":{"1min":4.5449999999999999,"5min":1.8440000000000001,"15min":1.3939999999999999},"min":{"1min":0.39100000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":61.473999999999997,"5min":61.473999999999997,"15min":61.473999999999997},"last":3.1230000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:06:20 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":1.5029999999999999,"15min":1.522},"min":{"1min":0.27600000000000002,"5min":0.224,"15min":0.224},"max":{"1min":4.9409999999999998,"5min":11.59,"15min":19.068999999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.391,"15min":1.343},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":8.3230000000000004,"5min":9.8610000000000007,"15min":15.535},"last":3.8999999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.379,"15min":1.288},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.24099999999999999},"max":{"1min":3.8479999999999999,"5min":7.7969999999999997,"15min":15.739000000000001},"last":1.024},{"interface":"front","average":{"1min":1.274,"5min":1.6319999999999999,"15min":1.6950000000000001},"min":{"1min":0.34799999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.1349999999999998,"5min":12.795999999999999,"15min":14.272},"last":1.0349999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:05:56 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.627,"15min":1.6890000000000001},"min":{"1min":0.31900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.5739999999999998,"5min":12.753,"15min":19.923999999999999},"last":3.536},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.589,"15min":1.573},"min":{"1min":0.35999999999999999,"5min":0.29799999999999999,"15min":0.255},"max":{"1min":4.6390000000000002,"5min":9.423,"15min":19.859999999999999},"last":3.5099999999999998}]}]},{"osd":5,"up_from":37,"seq":158913790342,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2481460,"kb_used_data":1582828,"kb_used_omap":0,"kb_used_meta":898624,"kb_avail":91263692,"statfs":{"total":95995035648,"available":93454020608,"internally_reserved":0,"allocated":1620815872,"data_stored":1619141371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920190976},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:06:24 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":3.056,"15min":2.367},"min":{"1min":0.219,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.9569999999999999,"5min":58.271999999999998,"15min":58.271999999999998},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.29,"5min":3.1059999999999999,"15min":2.4159999999999999},"min":{"1min":0.32200000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.9829999999999997,"5min":58.25,"15min":58.25},"last":0.54700000000000004}]},{"osd":1,"last update":"Mon Oct 9 11:06:30 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":2.1509999999999998,"15min":2.0430000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":16.962,"5min":32.491999999999997,"15min":32.491999999999997},"last":0.94699999999999995},{"interface":"front","average":{"1min":1.6990000000000001,"5min":2.3490000000000002,"15min":1.857},"min":{"1min":0.30499999999999999,"5min":0.247,"15min":0.247},"max":{"1min":17.033999999999999,"5min":32.537999999999997,"15min":32.537999999999997},"last":0.48699999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:05:47 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.6830000000000001,"15min":2.0209999999999999},"min":{"1min":0.32200000000000001,"5min":0.24199999999999999,"15min":0.221},"max":{"1min":3.75,"5min":6.96,"15min":11.368},"last":2.0059999999999998},{"interface":"front","average":{"1min":1.109,"5min":1.9650000000000001,"15min":2.056},"min":{"1min":0.35599999999999998,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":7.5110000000000001,"5min":11.673,"15min":15.087999999999999},"last":0.57599999999999996}]},{"osd":3,"last update":"Mon Oct 9 11:06:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":1.4530000000000001,"15min":1.4770000000000001},"min":{"1min":0.33000000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.8500000000000001,"5min":6.9649999999999999,"15min":6.9649999999999999},"last":1.3109999999999999},{"interface":"front","average":{"1min":1.5069999999999999,"5min":1.748,"15min":1.788},"min":{"1min":0.41399999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.931,"5min":5.0789999999999997,"15min":5.0789999999999997},"last":1.3720000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":1.2569999999999999,"15min":1.0349999999999999},"min":{"1min":0.26000000000000001,"5min":0.23799999999999999,"15min":0.216},"max":{"1min":3.0139999999999998,"5min":12.042,"15min":12.042},"last":0.79500000000000004},{"interface":"front","average":{"1min":1.119,"5min":1.2609999999999999,"15min":1.224},"min":{"1min":0.30599999999999999,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":3.48,"5min":9.2490000000000006,"15min":9.2490000000000006},"last":1.343}]},{"osd":6,"last update":"Mon Oct 9 11:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":1.4350000000000001,"15min":1.446},"min":{"1min":0.33900000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.7919999999999998,"5min":9.1969999999999992,"15min":11.098000000000001},"last":1.133},{"interface":"front","average":{"1min":1.038,"5min":1.286,"15min":1.3109999999999999},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.29099999999999998},"max":{"1min":7.5389999999999997,"5min":11.983000000000001,"15min":15.16},"last":0.76300000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:06:06 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.3919999999999999,"15min":1.456},"min":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.0979999999999999,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":2.327},{"interface":"front","average":{"1min":0.86699999999999999,"5min":1.387,"15min":1.3180000000000001},"min":{"1min":0.38,"5min":0.312,"15min":0.30099999999999999},"max":{"1min":5.6920000000000002,"5min":11.726000000000001,"15min":11.726000000000001},"last":0.96499999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3733,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1933508608,"data_stored":1933508608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3132686336,"data_stored":3132686336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2362638336,"data_stored":2362638336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2857762816,"data_stored":2857762816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1516699648,"data_stored":1516699648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1620443136,"data_stored":1620443136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2937192448,"data_stored":2937192448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2019622912,"data_stored":2019622912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:06:33.887 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T11:06:33.887 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_5 pg_num 36 2023-10-09T11:06:33.984 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:33.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:33.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:34.011 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 85 16 95060 95044 69.8407 47.875 0.0243206 0.014312 2023-10-09T11:06:34.091 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:34.126 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:34.203 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:34.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:34.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:34.307 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:34.340 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:34.507 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:34 smithi125 ceph-mon[195507]: from='client.56440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:06:34.508 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:34 smithi125 ceph-mon[195507]: pgmap v1289: 129 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 118 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 15409/281820 objects degraded (5.468%); 3148/281820 objects misplaced (1.117%); 10 MiB/s, 166 objects/s recovering 2023-10-09T11:06:34.527 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:06:34.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:34 smithi155 ceph-mon[184854]: from='client.56440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:06:34.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:34 smithi155 ceph-mon[184854]: pgmap v1289: 129 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 118 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 15409/281820 objects degraded (5.468%); 3148/281820 objects misplaced (1.117%); 10 MiB/s, 166 objects/s recovering 2023-10-09T11:06:34.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:34 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2997315070' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:06:34.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:34 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:06:34.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:34 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2997315070' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:06:34.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:34 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:06:34.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:34 smithi125 ceph-mon[200197]: from='client.56440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:06:34.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:34 smithi125 ceph-mon[200197]: pgmap v1289: 129 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 118 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 15409/281820 objects degraded (5.468%); 3148/281820 objects misplaced (1.117%); 10 MiB/s, 166 objects/s recovering 2023-10-09T11:06:34.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:34 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2997315070' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:06:34.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:34 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:06:34.931 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:34.969 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:34.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:34.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:35.011 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 86 16 95947 95931 69.6733 55.4375 0.00969539 0.0143478 2023-10-09T11:06:35.073 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:35.110 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:35.136 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:35.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:35.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:35.240 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:35.276 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:35.306 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:35.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:35.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:35.322 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:06:35.322 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:06:35.322 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.322 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:96196 11.14 11:29f04c79:::benchmark_data_smithi125_439543_object96195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e529)", 2023-10-09T11:06:35.322 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:35.313632+0000", 2023-10-09T11:06:35.322 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0057587200000000002, 2023-10-09T11:06:35.322 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0057956580000000004, 2023-10-09T11:06:35.323 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:35.323 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:06:35.323 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:35.323 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:06:35.323 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:35.323 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96196 2023-10-09T11:06:35.323 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.323 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:35.323 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.323 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:35.323 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.313632+0000", 2023-10-09T11:06:35.324 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:35.324 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.324 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.324 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:35.324 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.313632+0000", 2023-10-09T11:06:35.324 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:35.324 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.324 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.324 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:35.324 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.313631+0000", 2023-10-09T11:06:35.324 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:35.324 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.324 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.325 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:35.325 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.313700+0000", 2023-10-09T11:06:35.325 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9507999999999999e-05 2023-10-09T11:06:35.325 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.325 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.325 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:35.325 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.313719+0000", 2023-10-09T11:06:35.325 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8680000000000001e-05 2023-10-09T11:06:35.325 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.325 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.325 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:35.325 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.313728+0000", 2023-10-09T11:06:35.325 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8459999999999997e-06 2023-10-09T11:06:35.326 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.326 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.326 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:35.326 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.313784+0000", 2023-10-09T11:06:35.326 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6397999999999998e-05 2023-10-09T11:06:35.326 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.326 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.326 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:35.326 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.313943+0000", 2023-10-09T11:06:35.326 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015874699999999999 2023-10-09T11:06:35.326 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.326 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.327 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T11:06:35.327 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.314054+0000", 2023-10-09T11:06:35.327 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011116999999999999 2023-10-09T11:06:35.327 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.327 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.327 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:35.327 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.315793+0000", 2023-10-09T11:06:35.327 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001738545 2023-10-09T11:06:35.327 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.327 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.327 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:06:35.327 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.319185+0000", 2023-10-09T11:06:35.327 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0033919309999999999 2023-10-09T11:06:35.328 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:35.328 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:35.328 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:35.328 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.328 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.328 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:96208 11.14 11:2b750176:::benchmark_data_smithi125_439543_object96207:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e529)", 2023-10-09T11:06:35.328 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:35.316339+0000", 2023-10-09T11:06:35.328 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.003052393, 2023-10-09T11:06:35.328 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003192121, 2023-10-09T11:06:35.328 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:35.328 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:06:35.328 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:06:35.328 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:06:35.329 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:06:35.329 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 96208 2023-10-09T11:06:35.329 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.329 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:35.329 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.329 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:35.329 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316339+0000", 2023-10-09T11:06:35.329 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:35.329 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.329 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.329 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:35.329 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316339+0000", 2023-10-09T11:06:35.329 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:35.330 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.330 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.330 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:35.330 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316337+0000", 2023-10-09T11:06:35.330 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:35.330 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.330 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.330 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:35.330 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316407+0000", 2023-10-09T11:06:35.330 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0154000000000006e-05 2023-10-09T11:06:35.330 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.330 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.331 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:35.331 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316422+0000", 2023-10-09T11:06:35.331 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4594e-05 2023-10-09T11:06:35.331 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.331 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.331 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:35.331 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316429+0000", 2023-10-09T11:06:35.331 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0180000000000004e-06 2023-10-09T11:06:35.331 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.331 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.331 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:35.331 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316483+0000", 2023-10-09T11:06:35.332 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4622e-05 2023-10-09T11:06:35.332 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.332 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.332 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:35.332 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316628+0000", 2023-10-09T11:06:35.332 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000144513 2023-10-09T11:06:35.332 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.332 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.332 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T11:06:35.332 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316783+0000", 2023-10-09T11:06:35.332 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000155119 2023-10-09T11:06:35.333 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.333 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.333 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:06:35.333 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.318471+0000", 2023-10-09T11:06:35.333 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0016878710000000001 2023-10-09T11:06:35.333 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:35.333 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:35.333 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:35.333 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.333 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.333 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36510.0:96200 11.e e529/495 11:7c437113:::benchmark_data_smithi125_439543_object96199:head v 529'5959, mlcod=529'5957)", 2023-10-09T11:06:35.333 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:35.316653+0000", 2023-10-09T11:06:35.333 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.002737588, 2023-10-09T11:06:35.334 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002955269, 2023-10-09T11:06:35.334 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:35.334 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T11:06:35.334 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:35.334 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.334 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:35.334 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316653+0000", 2023-10-09T11:06:35.334 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:35.334 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.334 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.334 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:35.334 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316653+0000", 2023-10-09T11:06:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316652+0000", 2023-10-09T11:06:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:06:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316726+0000", 2023-10-09T11:06:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4213000000000002e-05 2023-10-09T11:06:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.335 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.336 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:35.336 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316738+0000", 2023-10-09T11:06:35.336 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1796e-05 2023-10-09T11:06:35.336 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.336 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.336 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:35.336 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316744+0000", 2023-10-09T11:06:35.336 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3409999999999999e-06 2023-10-09T11:06:35.336 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.336 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.336 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:35.336 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316792+0000", 2023-10-09T11:06:35.336 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7756000000000002e-05 2023-10-09T11:06:35.337 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.337 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.337 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:35.337 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316812+0000", 2023-10-09T11:06:35.337 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0339999999999998e-05 2023-10-09T11:06:35.337 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:35.337 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:35.337 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:35.337 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.337 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.337 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36510.0:96201 11.e e529/495 11:71ed820d:::benchmark_data_smithi125_439543_object96200:head v 529'5960, mlcod=529'5957)", 2023-10-09T11:06:35.337 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:35.316759+0000", 2023-10-09T11:06:35.337 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0026320039999999999, 2023-10-09T11:06:35.337 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029091439999999998, 2023-10-09T11:06:35.338 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:06:35.338 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T11:06:35.338 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:06:35.338 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.338 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:06:35.338 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316759+0000", 2023-10-09T11:06:35.338 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:35.338 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.338 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.338 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:06:35.338 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316759+0000", 2023-10-09T11:06:35.338 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:06:35.338 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.339 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.339 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:06:35.339 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316759+0000", 2023-10-09T11:06:35.339 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:06:35.339 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.339 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.339 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:06:35.339 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316790+0000", 2023-10-09T11:06:35.339 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1059000000000002e-05 2023-10-09T11:06:35.339 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.339 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.339 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:06:35.339 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316796+0000", 2023-10-09T11:06:35.340 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.669e-06 2023-10-09T11:06:35.340 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.340 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.340 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:06:35.340 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.316800+0000", 2023-10-09T11:06:35.340 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1330000000000001e-06 2023-10-09T11:06:35.340 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.340 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.340 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:06:35.340 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.318987+0000", 2023-10-09T11:06:35.340 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002186988 2023-10-09T11:06:35.340 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:06:35.340 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:06:35.341 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:06:35.341 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:35.319008+0000", 2023-10-09T11:06:35.341 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0818e-05 2023-10-09T11:06:35.341 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:35.341 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:06:35.341 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:35.341 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:06:35.341 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T11:06:35.341 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 4 2023-10-09T11:06:35.341 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:06:35.412 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:35.494 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:35.522 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:06:35.525 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:35.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:35.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:35.565 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:06:35.598 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:06:35.634 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:35.640 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:06:35.668 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:35.689 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:06:35.708 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:35.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:35.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:35.723 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T11:06:35.724 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:06:35.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:35.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:06:35.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[195507]: osdmap e529: 8 total, 8 up, 8 in 2023-10-09T11:06:35.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[195507]: failed to encode map e529 with expected crc 2023-10-09T11:06:35.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[195507]: failed to encode map e529 with expected crc 2023-10-09T11:06:35.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[195507]: failed to encode map e529 with expected crc 2023-10-09T11:06:35.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[195507]: failed to encode map e529 with expected crc 2023-10-09T11:06:35.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[195507]: failed to encode map e529 with expected crc 2023-10-09T11:06:35.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/56757075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:35.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:35.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:06:35.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[200197]: osdmap e529: 8 total, 8 up, 8 in 2023-10-09T11:06:35.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[200197]: failed to encode map e529 with expected crc 2023-10-09T11:06:35.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[200197]: failed to encode map e529 with expected crc 2023-10-09T11:06:35.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[200197]: failed to encode map e529 with expected crc 2023-10-09T11:06:35.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[200197]: failed to encode map e529 with expected crc 2023-10-09T11:06:35.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[200197]: failed to encode map e529 with expected crc 2023-10-09T11:06:35.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:35 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/56757075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:35.766 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:06:35.800 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:06:35.804 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:35 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:35.804 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:35 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:06:35.804 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:35 smithi155 ceph-mon[184854]: osdmap e529: 8 total, 8 up, 8 in 2023-10-09T11:06:35.804 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:35 smithi155 ceph-mon[184854]: failed to encode map e529 with expected crc 2023-10-09T11:06:35.805 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:35 smithi155 ceph-mon[184854]: failed to encode map e529 with expected crc 2023-10-09T11:06:35.805 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:35 smithi155 ceph-mon[184854]: failed to encode map e529 with expected crc 2023-10-09T11:06:35.805 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:35 smithi155 ceph-mon[184854]: failed to encode map e529 with expected crc 2023-10-09T11:06:35.805 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:35 smithi155 ceph-mon[184854]: failed to encode map e529 with expected crc 2023-10-09T11:06:35.805 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:35 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/56757075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:35.813 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:35.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:06:35.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:06:35.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:06:35.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:06:35.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:06:35.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:06:35.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:06:35.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:06:35.871 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:36.012 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 87 16 96770 96754 69.4637 51.4375 0.00633114 0.0143857 2023-10-09T11:06:36.100 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:06:36.781 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:36 smithi155 ceph-mon[184854]: 11.4 scrub starts 2023-10-09T11:06:36.781 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:36 smithi155 ceph-mon[184854]: pgmap v1291: 129 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 118 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 40 MiB/s wr, 641 op/s; 15409/281820 objects degraded (5.468%); 3148/281820 objects misplaced (1.117%); 10 MiB/s, 166 objects/s recovering 2023-10-09T11:06:36.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:36 smithi125 ceph-mon[195507]: 11.4 scrub starts 2023-10-09T11:06:36.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:36 smithi125 ceph-mon[195507]: pgmap v1291: 129 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 118 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 40 MiB/s wr, 641 op/s; 15409/281820 objects degraded (5.468%); 3148/281820 objects misplaced (1.117%); 10 MiB/s, 166 objects/s recovering 2023-10-09T11:06:36.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:36 smithi125 ceph-mon[200197]: 11.4 scrub starts 2023-10-09T11:06:36.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:36 smithi125 ceph-mon[200197]: pgmap v1291: 129 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 118 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 40 MiB/s wr, 641 op/s; 15409/281820 objects degraded (5.468%); 3148/281820 objects misplaced (1.117%); 10 MiB/s, 166 objects/s recovering 2023-10-09T11:06:36.928 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:06:37.019 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 88 16 97799 97783 69.4038 64.3125 0.0112469 0.0144029 2023-10-09T11:06:37.266 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:06:37.267 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:06:37.323 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:37.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:37.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:37.431 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:37.491 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:37.659 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T11:06:37.669 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:37.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:37.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:37.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:37.845 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:38.014 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 89 16 98710 98694 69.2633 56.9375 0.0142139 0.014433 2023-10-09T11:06:38.067 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:06:38.076 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:38.116 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:38.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:38.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:38.220 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:38.276 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:38.317 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:38.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:38.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:38.422 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:38.462 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:38.594 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:38.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:38.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:38.706 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:38.750 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:38.784 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:38.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:38.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:38.893 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:38.947 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:39.014 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total time run: 90.0103 2023-10-09T11:06:39.014 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total writes made: 99496 2023-10-09T11:06:39.014 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Write size: 65536 2023-10-09T11:06:39.015 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object size: 65536 2023-10-09T11:06:39.015 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Bandwidth (MB/sec): 69.0866 2023-10-09T11:06:39.015 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Bandwidth: 33.3855 2023-10-09T11:06:39.015 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max bandwidth (MB/sec): 162.812 2023-10-09T11:06:39.015 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min bandwidth (MB/sec): 0 2023-10-09T11:06:39.015 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average IOPS: 1105 2023-10-09T11:06:39.015 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev IOPS: 534.168 2023-10-09T11:06:39.015 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max IOPS: 2605 2023-10-09T11:06:39.015 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min IOPS: 0 2023-10-09T11:06:39.015 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average Latency(s): 0.0144707 2023-10-09T11:06:39.015 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Latency(s): 0.064171 2023-10-09T11:06:39.016 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max latency(s): 4.7606 2023-10-09T11:06:39.016 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min latency(s): 0.00101021 2023-10-09T11:06:39.019 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T11:06:39.073 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:39.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:39.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:39.162 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:06:39.162 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:06:39.177 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:39.180 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:38 smithi155 ceph-mon[184854]: pgmap v1292: 129 pgs: 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 119 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 695 op/s; 15409/285915 objects degraded (5.389%); 3148/285915 objects misplaced (1.101%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T11:06:39.180 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:38 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 15409/285915 objects degraded (5.389%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:39.214 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:39.291 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:39.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:39.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:39.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:38 smithi125 ceph-mon[195507]: pgmap v1292: 129 pgs: 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 119 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 695 op/s; 15409/285915 objects degraded (5.389%); 3148/285915 objects misplaced (1.101%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T11:06:39.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:38 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 15409/285915 objects degraded (5.389%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:39.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:38 smithi125 ceph-mon[200197]: pgmap v1292: 129 pgs: 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 119 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 695 op/s; 15409/285915 objects degraded (5.389%); 3148/285915 objects misplaced (1.101%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T11:06:39.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:38 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 15409/285915 objects degraded (5.389%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:39.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:39.463 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:39.570 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T11:06:39.987 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 11 pg_num to 36 2023-10-09T11:06:40.013 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:40.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:40.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:40.096 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:06:40.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:40.155 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:40.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:39 smithi155 ceph-mon[184854]: 11.16 scrub starts 2023-10-09T11:06:40.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:39 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1263078343' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T11:06:40.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:39 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T11:06:40.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:39 smithi155 ceph-mon[184854]: 11.14 scrub starts 2023-10-09T11:06:40.227 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:39 smithi125 ceph-mon[195507]: 11.16 scrub starts 2023-10-09T11:06:40.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:39 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1263078343' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T11:06:40.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:39 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T11:06:40.228 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:39 smithi125 ceph-mon[195507]: 11.14 scrub starts 2023-10-09T11:06:40.229 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:39 smithi125 ceph-mon[200197]: 11.16 scrub starts 2023-10-09T11:06:40.229 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:39 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1263078343' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T11:06:40.229 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:39 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T11:06:40.229 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:39 smithi125 ceph-mon[200197]: 11.14 scrub starts 2023-10-09T11:06:40.237 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:40.276 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:40.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:40.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:40.391 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:40.425 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:40.448 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:40.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:40.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:40.561 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:40.632 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:40.771 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:40.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:40.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:40.877 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:40.911 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:40.931 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:40.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:40.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:41.036 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[200197]: 11.6 scrub starts 2023-10-09T11:06:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[200197]: pgmap v1293: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 120 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 12974/295140 objects degraded (4.396%); 5371/295140 objects misplaced (1.820%); 10 MiB/s, 160 objects/s recovering 2023-10-09T11:06:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]': finished 2023-10-09T11:06:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[200197]: osdmap e530: 8 total, 8 up, 8 in 2023-10-09T11:06:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[200197]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[200197]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[200197]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[200197]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[200197]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[200197]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[195507]: 11.6 scrub starts 2023-10-09T11:06:41.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[195507]: pgmap v1293: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 120 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 12974/295140 objects degraded (4.396%); 5371/295140 objects misplaced (1.820%); 10 MiB/s, 160 objects/s recovering 2023-10-09T11:06:41.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]': finished 2023-10-09T11:06:41.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[195507]: osdmap e530: 8 total, 8 up, 8 in 2023-10-09T11:06:41.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[195507]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[195507]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[195507]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[195507]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[195507]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.073 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:40 smithi125 ceph-mon[195507]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.075 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:41.097 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:41.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:41.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:41.200 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:40 smithi155 ceph-mon[184854]: 11.6 scrub starts 2023-10-09T11:06:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:40 smithi155 ceph-mon[184854]: pgmap v1293: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 120 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 12974/295140 objects degraded (4.396%); 5371/295140 objects misplaced (1.820%); 10 MiB/s, 160 objects/s recovering 2023-10-09T11:06:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:40 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]': finished 2023-10-09T11:06:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:40 smithi155 ceph-mon[184854]: osdmap e530: 8 total, 8 up, 8 in 2023-10-09T11:06:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:40 smithi155 ceph-mon[184854]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:40 smithi155 ceph-mon[184854]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:40 smithi155 ceph-mon[184854]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:40 smithi155 ceph-mon[184854]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:40 smithi155 ceph-mon[184854]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:40 smithi155 ceph-mon[184854]: failed to encode map e530 with expected crc 2023-10-09T11:06:41.233 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:41.258 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:41.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:41.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:41.328 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:41.363 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:41.393 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:41.416 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:41.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:41.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:41.522 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:06:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:06:41] "GET /metrics HTTP/1.1" 200 36586 "" "Prometheus/2.43.0" 2023-10-09T11:06:41.522 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:41.550 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:41.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:41.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:41.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:41.688 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:41.728 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:41.757 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:41.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:41.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:41.862 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:41.932 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:41.961 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:06:42.009 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:42.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:42.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:42.117 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:06:42.117 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:06:42.140 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:42.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:41 smithi155 ceph-mon[184854]: 11.16 scrub starts 2023-10-09T11:06:42.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:41 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4235215170' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:42.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:41 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:42.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:41 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T11:06:42.246 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:41 smithi125 ceph-mon[195507]: 11.16 scrub starts 2023-10-09T11:06:42.246 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:41 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4235215170' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:42.246 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:41 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:42.246 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:41 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T11:06:42.246 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:41 smithi125 ceph-mon[200197]: 11.16 scrub starts 2023-10-09T11:06:42.247 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:41 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4235215170' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:42.247 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:41 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:42.247 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:41 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T11:06:42.250 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:42 smithi155 ceph-mon[184854]: pgmap v1295: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 120 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 40 MiB/s wr, 722 op/s; 12906/295209 objects degraded (4.372%); 5371/295209 objects misplaced (1.819%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T11:06:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:42 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:06:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:42 smithi155 ceph-mon[184854]: failed to encode map e531 with expected crc 2023-10-09T11:06:43.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:42 smithi155 ceph-mon[184854]: failed to encode map e531 with expected crc 2023-10-09T11:06:43.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:42 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:06:43.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:42 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T11:06:43.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:42 smithi155 ceph-mon[184854]: osdmap e531: 8 total, 8 up, 8 in 2023-10-09T11:06:43.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:42 smithi155 ceph-mon[184854]: 11.4 scrub starts 2023-10-09T11:06:43.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[200197]: pgmap v1295: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 120 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 40 MiB/s wr, 722 op/s; 12906/295209 objects degraded (4.372%); 5371/295209 objects misplaced (1.819%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T11:06:43.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:06:43.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[200197]: failed to encode map e531 with expected crc 2023-10-09T11:06:43.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[200197]: failed to encode map e531 with expected crc 2023-10-09T11:06:43.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:06:43.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T11:06:43.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[200197]: osdmap e531: 8 total, 8 up, 8 in 2023-10-09T11:06:43.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[200197]: 11.4 scrub starts 2023-10-09T11:06:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[195507]: pgmap v1295: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 120 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 40 MiB/s wr, 722 op/s; 12906/295209 objects degraded (4.372%); 5371/295209 objects misplaced (1.819%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T11:06:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:06:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[195507]: failed to encode map e531 with expected crc 2023-10-09T11:06:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[195507]: failed to encode map e531 with expected crc 2023-10-09T11:06:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:06:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T11:06:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[195507]: osdmap e531: 8 total, 8 up, 8 in 2023-10-09T11:06:43.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:42 smithi125 ceph-mon[195507]: 11.4 scrub starts 2023-10-09T11:06:44.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[195507]: failed to encode map e532 with expected crc 2023-10-09T11:06:44.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[195507]: failed to encode map e532 with expected crc 2023-10-09T11:06:44.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[195507]: osdmap e532: 8 total, 8 up, 8 in 2023-10-09T11:06:44.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[195507]: failed to encode map e532 with expected crc 2023-10-09T11:06:44.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[195507]: failed to encode map e532 with expected crc 2023-10-09T11:06:44.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 12906/295209 objects degraded (4.372%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:44.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[195507]: failed to encode map e533 with expected crc 2023-10-09T11:06:44.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[195507]: osdmap e533: 8 total, 8 up, 8 in 2023-10-09T11:06:44.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[195507]: failed to encode map e533 with expected crc 2023-10-09T11:06:44.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[195507]: failed to encode map e533 with expected crc 2023-10-09T11:06:44.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[200197]: failed to encode map e532 with expected crc 2023-10-09T11:06:44.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[200197]: failed to encode map e532 with expected crc 2023-10-09T11:06:44.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[200197]: osdmap e532: 8 total, 8 up, 8 in 2023-10-09T11:06:44.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[200197]: failed to encode map e532 with expected crc 2023-10-09T11:06:44.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[200197]: failed to encode map e532 with expected crc 2023-10-09T11:06:44.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 12906/295209 objects degraded (4.372%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:44.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[200197]: failed to encode map e533 with expected crc 2023-10-09T11:06:44.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[200197]: osdmap e533: 8 total, 8 up, 8 in 2023-10-09T11:06:44.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[200197]: failed to encode map e533 with expected crc 2023-10-09T11:06:44.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:43 smithi125 ceph-mon[200197]: failed to encode map e533 with expected crc 2023-10-09T11:06:44.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:43 smithi155 ceph-mon[184854]: failed to encode map e532 with expected crc 2023-10-09T11:06:44.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:43 smithi155 ceph-mon[184854]: failed to encode map e532 with expected crc 2023-10-09T11:06:44.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:43 smithi155 ceph-mon[184854]: osdmap e532: 8 total, 8 up, 8 in 2023-10-09T11:06:44.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:43 smithi155 ceph-mon[184854]: failed to encode map e532 with expected crc 2023-10-09T11:06:44.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:43 smithi155 ceph-mon[184854]: failed to encode map e532 with expected crc 2023-10-09T11:06:44.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:43 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 12906/295209 objects degraded (4.372%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:44.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:43 smithi155 ceph-mon[184854]: failed to encode map e533 with expected crc 2023-10-09T11:06:44.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:43 smithi155 ceph-mon[184854]: osdmap e533: 8 total, 8 up, 8 in 2023-10-09T11:06:44.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:43 smithi155 ceph-mon[184854]: failed to encode map e533 with expected crc 2023-10-09T11:06:44.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:43 smithi155 ceph-mon[184854]: failed to encode map e533 with expected crc 2023-10-09T11:06:45.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:44 smithi155 ceph-mon[184854]: failed to encode map e533 with expected crc 2023-10-09T11:06:45.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:44 smithi155 ceph-mon[184854]: failed to encode map e533 with expected crc 2023-10-09T11:06:45.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:44 smithi155 ceph-mon[184854]: pgmap v1299: 139 pgs: 1 remapped+peering, 2 peering, 2 unknown, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 124 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 255 B/s rd, 18 MiB/s wr, 921 op/s; 6962/290931 objects degraded (2.393%); 5060/290931 objects misplaced (1.739%); 12 MiB/s, 197 objects/s recovering 2023-10-09T11:06:45.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:44 smithi155 ceph-mon[184854]: osdmap e534: 8 total, 8 up, 8 in 2023-10-09T11:06:45.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:44 smithi155 ceph-mon[184854]: failed to encode map e534 with expected crc 2023-10-09T11:06:45.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:44 smithi155 ceph-mon[184854]: failed to encode map e534 with expected crc 2023-10-09T11:06:45.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:44 smithi155 ceph-mon[184854]: failed to encode map e534 with expected crc 2023-10-09T11:06:45.235 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:44 smithi125 ceph-mon[195507]: failed to encode map e533 with expected crc 2023-10-09T11:06:45.235 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:44 smithi125 ceph-mon[195507]: failed to encode map e533 with expected crc 2023-10-09T11:06:45.236 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:44 smithi125 ceph-mon[195507]: pgmap v1299: 139 pgs: 1 remapped+peering, 2 peering, 2 unknown, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 124 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 255 B/s rd, 18 MiB/s wr, 921 op/s; 6962/290931 objects degraded (2.393%); 5060/290931 objects misplaced (1.739%); 12 MiB/s, 197 objects/s recovering 2023-10-09T11:06:45.236 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:44 smithi125 ceph-mon[195507]: osdmap e534: 8 total, 8 up, 8 in 2023-10-09T11:06:45.236 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:44 smithi125 ceph-mon[195507]: failed to encode map e534 with expected crc 2023-10-09T11:06:45.236 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:44 smithi125 ceph-mon[195507]: failed to encode map e534 with expected crc 2023-10-09T11:06:45.236 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:44 smithi125 ceph-mon[195507]: failed to encode map e534 with expected crc 2023-10-09T11:06:45.236 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:44 smithi125 ceph-mon[200197]: failed to encode map e533 with expected crc 2023-10-09T11:06:45.236 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:44 smithi125 ceph-mon[200197]: failed to encode map e533 with expected crc 2023-10-09T11:06:45.236 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:44 smithi125 ceph-mon[200197]: pgmap v1299: 139 pgs: 1 remapped+peering, 2 peering, 2 unknown, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 124 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 255 B/s rd, 18 MiB/s wr, 921 op/s; 6962/290931 objects degraded (2.393%); 5060/290931 objects misplaced (1.739%); 12 MiB/s, 197 objects/s recovering 2023-10-09T11:06:45.237 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:44 smithi125 ceph-mon[200197]: osdmap e534: 8 total, 8 up, 8 in 2023-10-09T11:06:45.237 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:44 smithi125 ceph-mon[200197]: failed to encode map e534 with expected crc 2023-10-09T11:06:45.237 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:44 smithi125 ceph-mon[200197]: failed to encode map e534 with expected crc 2023-10-09T11:06:45.237 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:44 smithi125 ceph-mon[200197]: failed to encode map e534 with expected crc 2023-10-09T11:06:45.384 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:45.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:45.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:45.467 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T11:06:45.495 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:45.504 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 7, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 5, 6, 7, 1, 3] 2023-10-09T11:06:45.504 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:06:45.504 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T11:06:45.504 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T11:06:45.504 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T11:06:45.504 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T11:06:45.599 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:45.671 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:45.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:45.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:45.736 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:06:45.777 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:45.779 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:06:45.824 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:45.828 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:06:45.872 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:45.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:45.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:45.878 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:06:45.919 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:06:45.944 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:06:45.978 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:06:45.979 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:46.022 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:46.025 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:06:46.032 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:06:46.032 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:06:46.032 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:06:46.032 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:06:46.033 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:06:46.033 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:06:46.033 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:06:46.033 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:06:46.037 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:46.070 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:46.071 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:46.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:46.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:46.182 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:46.218 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:46.258 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:46.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:46.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:46.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:46 smithi125 ceph-mon[200197]: failed to encode map e534 with expected crc 2023-10-09T11:06:46.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:46.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:46 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:06:46.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:46 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2742992889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:46.331 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:46 smithi125 ceph-mon[195507]: failed to encode map e534 with expected crc 2023-10-09T11:06:46.331 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:46.331 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:46 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:06:46.331 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:46 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2742992889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:46.368 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:46.399 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:46 smithi155 ceph-mon[184854]: failed to encode map e534 with expected crc 2023-10-09T11:06:46.399 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:06:46.399 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:46 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:06:46.400 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:46 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2742992889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:46.402 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:46.427 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:46.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:46.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:46.531 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:46.609 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:46.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:46.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:46.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:46.793 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:46.828 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:46.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:46.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:46.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:46.962 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:47.002 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:47.029 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:47.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:47.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:47.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:06:47.135 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:47.164 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:47.186 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:47.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:47.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:47.290 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:47.319 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:47.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:47 smithi125 ceph-mon[195507]: pgmap v1301: 139 pgs: 1 remapped+peering, 2 peering, 2 unknown, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 124 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 7.1 MiB/s wr, 583 op/s; 6962/290931 objects degraded (2.393%); 5060/290931 objects misplaced (1.739%); 11 MiB/s, 180 objects/s recovering 2023-10-09T11:06:47.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:47 smithi125 ceph-mon[200197]: pgmap v1301: 139 pgs: 1 remapped+peering, 2 peering, 2 unknown, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 124 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 7.1 MiB/s wr, 583 op/s; 6962/290931 objects degraded (2.393%); 5060/290931 objects misplaced (1.739%); 11 MiB/s, 180 objects/s recovering 2023-10-09T11:06:47.340 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:47.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:47.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:47.443 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:47.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:47 smithi155 ceph-mon[184854]: pgmap v1301: 139 pgs: 1 remapped+peering, 2 peering, 2 unknown, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 124 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 7.1 MiB/s wr, 583 op/s; 6962/290931 objects degraded (2.393%); 5060/290931 objects misplaced (1.739%); 11 MiB/s, 180 objects/s recovering 2023-10-09T11:06:47.487 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:47.588 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:47.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:47.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:47.693 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:47.745 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:47.777 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:47.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:47.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:47.882 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:47.909 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:47.930 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:47.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:47.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:48.022 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:06:48.022 INFO:teuthology.orchestra.run.smithi125.stdout: "success": "" 2023-10-09T11:06:48.022 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:06:48.037 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:48.037 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:48.068 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:48.095 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:48.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:48.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:48.129 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:06:48.129 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:06:48.129 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T11:06:48.129 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T11:06:48.129 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:06:48.201 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:48.219 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:48 smithi155 ceph-mon[184854]: 11.14 scrub starts 2023-10-09T11:06:48.259 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:48.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:48 smithi125 ceph-mon[195507]: 11.14 scrub starts 2023-10-09T11:06:48.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:48 smithi125 ceph-mon[200197]: 11.14 scrub starts 2023-10-09T11:06:48.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:06:49.106 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:49.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:49.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:49.215 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:49.257 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:49.290 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:49.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:49.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:49.359 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T11:06:49.359 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:49 smithi125 ceph-mon[195507]: pgmap v1302: 139 pgs: 1 remapped+peering, 2 peering, 1 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 5 active+recovery_wait+degraded, 124 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 4.8 MiB/s wr, 583 op/s; 6866/287616 objects degraded (2.387%); 5060/287616 objects misplaced (1.759%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T11:06:49.359 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:49 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 6866/287616 objects degraded (2.387%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:49.360 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:49 smithi125 ceph-mon[200197]: pgmap v1302: 139 pgs: 1 remapped+peering, 2 peering, 1 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 5 active+recovery_wait+degraded, 124 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 4.8 MiB/s wr, 583 op/s; 6866/287616 objects degraded (2.387%); 5060/287616 objects misplaced (1.759%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T11:06:49.360 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:49 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 6866/287616 objects degraded (2.387%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:49.394 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:49.427 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:49 smithi155 ceph-mon[184854]: pgmap v1302: 139 pgs: 1 remapped+peering, 2 peering, 1 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 5 active+recovery_wait+degraded, 124 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 4.8 MiB/s wr, 583 op/s; 6866/287616 objects degraded (2.387%); 5060/287616 objects misplaced (1.759%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T11:06:49.427 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:49 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 6866/287616 objects degraded (2.387%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:49.430 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:49.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:49.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:49.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:49.557 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:49.587 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:49.609 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:49.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:49.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:49.713 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:49.749 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:49.773 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:49.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:49.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:49.873 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:49.877 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:49.956 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:50.044 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:50.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:50.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:50.152 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:50.214 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:50.346 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:50.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:50.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:50.455 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:50.505 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:51.044 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:51.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:51.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:51.149 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:51.193 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:06:51.210 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:51.318 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:51 smithi125 ceph-mon[195507]: pgmap v1303: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 6 active+recovery_wait+degraded, 124 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 4.1 MiB/s wr, 973 op/s; 12650/277788 objects degraded (4.554%); 3250/277788 objects misplaced (1.170%); 19 MiB/s, 308 objects/s recovering 2023-10-09T11:06:51.318 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2568012818' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:06:51.318 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:51 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:06:51.318 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:06:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:06:51] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.43.0" 2023-10-09T11:06:51.319 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:51 smithi125 ceph-mon[200197]: pgmap v1303: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 6 active+recovery_wait+degraded, 124 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 4.1 MiB/s wr, 973 op/s; 12650/277788 objects degraded (4.554%); 3250/277788 objects misplaced (1.170%); 19 MiB/s, 308 objects/s recovering 2023-10-09T11:06:51.319 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2568012818' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:06:51.319 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:51 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:06:51.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:51 smithi155 ceph-mon[184854]: pgmap v1303: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 6 active+recovery_wait+degraded, 124 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 4.1 MiB/s wr, 973 op/s; 12650/277788 objects degraded (4.554%); 3250/277788 objects misplaced (1.170%); 19 MiB/s, 308 objects/s recovering 2023-10-09T11:06:51.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2568012818' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:06:51.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:51 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:06:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:52 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:52 smithi155 ceph-mon[184854]: failed to encode map e535 with expected crc 2023-10-09T11:06:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:52 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:06:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:52 smithi155 ceph-mon[184854]: osdmap e535: 8 total, 8 up, 8 in 2023-10-09T11:06:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:52 smithi155 ceph-mon[184854]: failed to encode map e535 with expected crc 2023-10-09T11:06:52.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:52 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:52.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:52 smithi125 ceph-mon[195507]: failed to encode map e535 with expected crc 2023-10-09T11:06:52.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:52 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:06:52.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:52 smithi125 ceph-mon[195507]: osdmap e535: 8 total, 8 up, 8 in 2023-10-09T11:06:52.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:52 smithi125 ceph-mon[195507]: failed to encode map e535 with expected crc 2023-10-09T11:06:52.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:52 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:06:52.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:52 smithi125 ceph-mon[200197]: failed to encode map e535 with expected crc 2023-10-09T11:06:52.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:52 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:06:52.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:52 smithi125 ceph-mon[200197]: osdmap e535: 8 total, 8 up, 8 in 2023-10-09T11:06:52.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:52 smithi125 ceph-mon[200197]: failed to encode map e535 with expected crc 2023-10-09T11:06:52.835 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:53.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:53.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:53.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:53.259 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:53.290 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:53 smithi155 ceph-mon[184854]: pgmap v1305: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 6 active+recovery_wait+degraded, 124 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 12507/273033 objects degraded (4.581%); 3250/273033 objects misplaced (1.190%); 12 MiB/s, 193 objects/s recovering 2023-10-09T11:06:53.514 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:53 smithi125 ceph-mon[195507]: pgmap v1305: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 6 active+recovery_wait+degraded, 124 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 12507/273033 objects degraded (4.581%); 3250/273033 objects misplaced (1.190%); 12 MiB/s, 193 objects/s recovering 2023-10-09T11:06:53.514 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:53 smithi125 ceph-mon[200197]: pgmap v1305: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 6 active+recovery_wait+degraded, 124 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 12507/273033 objects degraded (4.581%); 3250/273033 objects misplaced (1.190%); 12 MiB/s, 193 objects/s recovering 2023-10-09T11:06:54.144 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 7, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 5, 6, 7, 1, 3] 2023-10-09T11:06:54.144 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T11:06:54.275 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:54.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:54.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:54.318 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:54 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 12507/273033 objects degraded (4.581%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:54.319 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:54 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 12507/273033 objects degraded (4.581%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:54.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:54.455 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:54.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:54 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 12507/273033 objects degraded (4.581%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:54.528 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:54.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:54.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:54.589 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:06:54.635 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:06:54.635 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:54.689 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:54.691 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:06:54.720 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:54.727 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:06:54.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:54.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:54.759 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:06:54.798 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:06:54.823 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:06:54.839 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:54.854 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:06:54.867 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:06:54.867 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:06:54.867 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:06:54.867 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:06:54.867 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:06:54.867 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:06:54.867 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:06:54.867 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:06:54.881 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:54.911 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:54.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:54.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:55.016 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:55.021 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:55.047 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:55.071 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:55.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:55.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:55.176 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:55.217 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:55.243 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:55.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:55.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:55.329 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:55 smithi125 ceph-mon[195507]: pgmap v1306: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 5 active+recovery_wait+degraded, 125 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 11447/261486 objects degraded (4.378%); 2794/261486 objects misplaced (1.069%); 19 MiB/s, 310 objects/s recovering 2023-10-09T11:06:55.329 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:55 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2059308904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:55.330 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:55 smithi125 ceph-mon[200197]: pgmap v1306: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 5 active+recovery_wait+degraded, 125 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 11447/261486 objects degraded (4.378%); 2794/261486 objects misplaced (1.069%); 19 MiB/s, 310 objects/s recovering 2023-10-09T11:06:55.331 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:55 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2059308904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:55.348 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:55.381 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:55.404 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:55.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:55.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:55 smithi155 ceph-mon[184854]: pgmap v1306: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 5 active+recovery_wait+degraded, 125 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 11447/261486 objects degraded (4.378%); 2794/261486 objects misplaced (1.069%); 19 MiB/s, 310 objects/s recovering 2023-10-09T11:06:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:55 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2059308904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:06:55.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:55.575 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:55.636 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:06:55.637 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:06:55.658 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:55.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:55.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:55.764 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:55.811 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:55.843 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:55.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:55.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:55.947 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:55.986 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:56.009 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:56.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:56.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:56.116 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:56.190 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:56.194 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T11:06:56.235 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:06:56.319 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:56.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:56.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:06:56.441 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:56.542 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:56.579 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:56.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:56.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:56.669 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:56.685 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:56.728 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:56.744 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:56.999 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:57.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:57.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:57.104 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:06:57.167 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:57.194 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:06:57.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:57.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:06:57.245 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T11:06:57.299 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:06:57.330 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:57.352 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:06:57.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:57.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:06:57.427 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:57 smithi125 ceph-mon[195507]: pgmap v1307: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 5 active+recovery_wait+degraded, 125 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 980 op/s; 11447/261486 objects degraded (4.378%); 2794/261486 objects misplaced (1.069%); 18 MiB/s, 294 objects/s recovering 2023-10-09T11:06:57.427 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:57 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2834899624' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:06:57.427 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:57 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:06:57.430 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:57 smithi125 ceph-mon[200197]: pgmap v1307: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 5 active+recovery_wait+degraded, 125 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 980 op/s; 11447/261486 objects degraded (4.378%); 2794/261486 objects misplaced (1.069%); 18 MiB/s, 294 objects/s recovering 2023-10-09T11:06:57.430 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:57 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2834899624' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:06:57.430 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:57 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:06:57.456 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:57 smithi155 ceph-mon[184854]: pgmap v1307: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 5 active+recovery_wait+degraded, 125 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 980 op/s; 11447/261486 objects degraded (4.378%); 2794/261486 objects misplaced (1.069%); 18 MiB/s, 294 objects/s recovering 2023-10-09T11:06:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:57 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2834899624' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:06:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:57 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:06:57.504 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:57.534 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:06:57.543 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:57.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:57.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:57.649 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:06:57.737 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:57.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T11:06:58.137 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:06:58.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:58.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:06:58.242 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:58.276 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:58.296 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:58.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:58.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:58.316 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:58 smithi125 ceph-mon[195507]: failed to encode map e536 with expected crc 2023-10-09T11:06:58.316 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:58 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T11:06:58.316 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:58 smithi125 ceph-mon[195507]: osdmap e536: 8 total, 8 up, 8 in 2023-10-09T11:06:58.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:58 smithi125 ceph-mon[195507]: failed to encode map e536 with expected crc 2023-10-09T11:06:58.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:58 smithi125 ceph-mon[200197]: failed to encode map e536 with expected crc 2023-10-09T11:06:58.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:58 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T11:06:58.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:58 smithi125 ceph-mon[200197]: osdmap e536: 8 total, 8 up, 8 in 2023-10-09T11:06:58.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:58 smithi125 ceph-mon[200197]: failed to encode map e536 with expected crc 2023-10-09T11:06:58.399 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:58.431 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:58.456 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:58.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:58.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:58.563 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:58.590 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:58 smithi155 ceph-mon[184854]: failed to encode map e536 with expected crc 2023-10-09T11:06:58.590 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:58 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T11:06:58.590 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:58 smithi155 ceph-mon[184854]: osdmap e536: 8 total, 8 up, 8 in 2023-10-09T11:06:58.590 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:58 smithi155 ceph-mon[184854]: failed to encode map e536 with expected crc 2023-10-09T11:06:58.591 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:58.597 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:58.613 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:58.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:58.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:58.649 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:06:58.694 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:06:58.694 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:06:58.719 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:06:58.756 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:58.783 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:06:58.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:58.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:06:58.888 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:06:58.924 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:58.947 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:06:58.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:58.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:06:59.053 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:59.087 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:59.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:59.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:59.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:59.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:06:59.267 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:59.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:59 smithi125 ceph-mon[195507]: pgmap v1309: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 5 active+recovery_wait+degraded, 126 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 9509/256263 objects degraded (3.711%); 4401/256263 objects misplaced (1.717%); 14 MiB/s, 234 objects/s recovering 2023-10-09T11:06:59.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:59 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 9509/256263 objects degraded (3.711%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:59.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:06:59 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3082015141' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:59.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:59 smithi125 ceph-mon[200197]: pgmap v1309: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 5 active+recovery_wait+degraded, 126 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 9509/256263 objects degraded (3.711%); 4401/256263 objects misplaced (1.717%); 14 MiB/s, 234 objects/s recovering 2023-10-09T11:06:59.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:59 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 9509/256263 objects degraded (3.711%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:59.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:06:59 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3082015141' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:59.367 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:06:59.401 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:06:59.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:59.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:06:59.521 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:06:59.540 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T11:06:59.556 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:59 smithi155 ceph-mon[184854]: pgmap v1309: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 5 active+recovery_wait+degraded, 126 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 9509/256263 objects degraded (3.711%); 4401/256263 objects misplaced (1.717%); 14 MiB/s, 234 objects/s recovering 2023-10-09T11:06:59.556 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:59 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 9509/256263 objects degraded (3.711%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T11:06:59.556 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:06:59 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3082015141' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:06:59.558 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:59.591 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:06:59.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:59.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:06:59.696 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:06:59.781 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:06:59.925 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:06:59.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:06:59.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:00.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:00.100 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:00.251 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:00.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:00.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:00.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:00 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:07:00.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:00 smithi125 ceph-mon[195507]: failed to encode map e537 with expected crc 2023-10-09T11:07:00.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:00 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3082015141' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:07:00.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:00 smithi125 ceph-mon[195507]: osdmap e537: 8 total, 8 up, 8 in 2023-10-09T11:07:00.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:00 smithi125 ceph-mon[195507]: failed to encode map e537 with expected crc 2023-10-09T11:07:00.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:00.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:07:00.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:00 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:07:00.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:00 smithi125 ceph-mon[200197]: failed to encode map e537 with expected crc 2023-10-09T11:07:00.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:00 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3082015141' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:07:00.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:00 smithi125 ceph-mon[200197]: osdmap e537: 8 total, 8 up, 8 in 2023-10-09T11:07:00.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:00 smithi125 ceph-mon[200197]: failed to encode map e537 with expected crc 2023-10-09T11:07:00.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:00.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:07:00.356 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:07:00.393 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:00.408 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:07:00.419 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:07:00.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:00.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:07:00.441 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:07:00.469 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:07:00.513 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:07:00.523 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:00.553 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:00.554 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:07:00.558 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:00 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:07:00.558 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:00 smithi155 ceph-mon[184854]: failed to encode map e537 with expected crc 2023-10-09T11:07:00.558 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:00 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3082015141' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:07:00.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:00 smithi155 ceph-mon[184854]: osdmap e537: 8 total, 8 up, 8 in 2023-10-09T11:07:00.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:00 smithi155 ceph-mon[184854]: failed to encode map e537 with expected crc 2023-10-09T11:07:00.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:00.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:07:00.577 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:00.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:00.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:00.591 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:07:00.625 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:07:00.669 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:07:00.683 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:00.685 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:07:00.685 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:07:00.686 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:07:00.686 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:07:00.686 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:07:00.686 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:07:00.686 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:07:00.686 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:07:00.715 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:00.740 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:00.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:00.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:00.785 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:00.846 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:00.912 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:00.938 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:00.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:00.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:01.044 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:01.123 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:01.211 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:01.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:01.217 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:07:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:07:01] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.43.0" 2023-10-09T11:07:01.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:01.318 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:01.352 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:01.375 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:01.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:01.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:01.392 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T11:07:01.480 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:01.516 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:01 smithi155 ceph-mon[184854]: failed to encode map e537 with expected crc 2023-10-09T11:07:01.517 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:01 smithi155 ceph-mon[184854]: pgmap v1311: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+degraded, 128 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 8983/245931 objects degraded (3.653%); 4080/245931 objects misplaced (1.659%); 19 MiB/s, 303 objects/s recovering 2023-10-09T11:07:01.517 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/620179032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:01.517 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2426247914' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:07:01.517 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:01 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:07:01.519 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:01.547 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:01.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:01.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:01.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:01 smithi125 ceph-mon[195507]: failed to encode map e537 with expected crc 2023-10-09T11:07:01.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:01 smithi125 ceph-mon[195507]: pgmap v1311: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+degraded, 128 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 8983/245931 objects degraded (3.653%); 4080/245931 objects misplaced (1.659%); 19 MiB/s, 303 objects/s recovering 2023-10-09T11:07:01.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/620179032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:01.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2426247914' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:07:01.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:01 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:07:01.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:01 smithi125 ceph-mon[200197]: failed to encode map e537 with expected crc 2023-10-09T11:07:01.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:01 smithi125 ceph-mon[200197]: pgmap v1311: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+degraded, 128 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 8983/245931 objects degraded (3.653%); 4080/245931 objects misplaced (1.659%); 19 MiB/s, 303 objects/s recovering 2023-10-09T11:07:01.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/620179032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:01.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2426247914' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:07:01.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:01 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:07:01.652 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:01.689 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:01.713 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:01.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:01.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:01.822 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:01.859 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:01.890 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:01.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:01.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:01.995 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:02.039 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:02.064 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:02.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:02.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:02.099 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T11:07:02.168 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:02.280 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:02.314 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:02.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:02.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:02.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:02.453 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:02.479 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:02.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:02.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:02.518 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:02 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:07:02.519 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:02 smithi125 ceph-mon[195507]: osdmap e538: 8 total, 8 up, 8 in 2023-10-09T11:07:02.519 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:02 smithi125 ceph-mon[195507]: failed to encode map e538 with expected crc 2023-10-09T11:07:02.519 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:02 smithi125 ceph-mon[195507]: failed to encode map e538 with expected crc 2023-10-09T11:07:02.519 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:02 smithi125 ceph-mon[195507]: failed to encode map e538 with expected crc 2023-10-09T11:07:02.519 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:02 smithi125 ceph-mon[195507]: pgmap v1313: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 128 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 8983/241401 objects degraded (3.721%); 3712/241401 objects misplaced (1.538%); 15 MiB/s, 232 objects/s recovering 2023-10-09T11:07:02.519 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:02 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:07:02.519 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:02 smithi125 ceph-mon[200197]: osdmap e538: 8 total, 8 up, 8 in 2023-10-09T11:07:02.519 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:02 smithi125 ceph-mon[200197]: failed to encode map e538 with expected crc 2023-10-09T11:07:02.519 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:02 smithi125 ceph-mon[200197]: failed to encode map e538 with expected crc 2023-10-09T11:07:02.519 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:02 smithi125 ceph-mon[200197]: failed to encode map e538 with expected crc 2023-10-09T11:07:02.520 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:02 smithi125 ceph-mon[200197]: pgmap v1313: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 128 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 8983/241401 objects degraded (3.721%); 3712/241401 objects misplaced (1.538%); 15 MiB/s, 232 objects/s recovering 2023-10-09T11:07:02.525 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:07:02.584 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:02.614 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:02.637 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:02.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:02 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:07:02.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:02 smithi155 ceph-mon[184854]: osdmap e538: 8 total, 8 up, 8 in 2023-10-09T11:07:02.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:02 smithi155 ceph-mon[184854]: failed to encode map e538 with expected crc 2023-10-09T11:07:02.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:02 smithi155 ceph-mon[184854]: failed to encode map e538 with expected crc 2023-10-09T11:07:02.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:02 smithi155 ceph-mon[184854]: failed to encode map e538 with expected crc 2023-10-09T11:07:02.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:02 smithi155 ceph-mon[184854]: pgmap v1313: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 128 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 8983/241401 objects degraded (3.721%); 3712/241401 objects misplaced (1.538%); 15 MiB/s, 232 objects/s recovering 2023-10-09T11:07:02.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:02.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:02.651 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:07:02.651 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:07:02.651 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T11:07:02.651 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T11:07:02.652 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:07:02.682 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:02.743 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:02.772 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:02.796 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:02.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:02.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:02.900 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:02.960 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:03.108 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:03.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:03.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:03.141 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T11:07:03.214 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:03.266 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:07:03.287 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:03.318 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:03.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:03.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:03.421 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:03.510 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:03.594 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:03.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:03.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:03.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:03.719 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:03 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 8983/241401 objects degraded (3.721%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:03.738 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:03.739 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:03 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 8983/241401 objects degraded (3.721%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:03.739 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:03 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 8983/241401 objects degraded (3.721%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:03.814 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:03.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:03.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:03.925 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:03.969 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:04.000 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:04.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:04.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:04.102 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:04.106 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:04.107 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:04.111 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:04.139 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:04.159 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:04.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:04.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:04.263 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:04.293 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:04.315 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:04.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:04.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:04.418 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:04.513 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:04.530 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:04 smithi125 ceph-mon[200197]: pgmap v1314: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.25k op/s; 8461/231318 objects degraded (3.658%); 3584/231318 objects misplaced (1.549%); 15 MiB/s, 237 objects/s recovering 2023-10-09T11:07:04.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:04 smithi125 ceph-mon[195507]: pgmap v1314: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.25k op/s; 8461/231318 objects degraded (3.658%); 3584/231318 objects misplaced (1.549%); 15 MiB/s, 237 objects/s recovering 2023-10-09T11:07:04.707 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:04.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:04.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:04.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:04 smithi155 ceph-mon[184854]: pgmap v1314: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.25k op/s; 8461/231318 objects degraded (3.658%); 3584/231318 objects misplaced (1.549%); 15 MiB/s, 237 objects/s recovering 2023-10-09T11:07:04.814 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:04.878 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:04.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:04.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:04.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:05.074 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:05.136 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:05.167 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:05.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:05.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:05.277 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:05.327 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:05.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:05.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:05.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:05.467 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:05.473 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T11:07:05.524 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:05.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:05.693 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:05 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3866940570' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:07:05.693 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:05 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:07:05.694 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:05 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3866940570' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:07:05.694 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:05 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:07:05.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:05.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:05.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:05 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3866940570' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:07:05.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:05 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:07:05.795 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:05.855 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:05.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T11:07:06.140 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:06.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:06.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:06.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:06.355 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:06.384 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:06.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:06.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:06.421 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:07:06.421 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T11:07:06.422 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T11:07:06.422 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:07:06.422 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.422 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:3548 11.1 11:8c8e4577:::benchmark_data_smithi125_439543_object3547:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e498)", 2023-10-09T11:07:06.422 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:10.492309+0000", 2023-10-09T11:07:06.422 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 115.923755602, 2023-10-09T11:07:06.422 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.127281062, 2023-10-09T11:07:06.422 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.422 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.423 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:06.423 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:06.423 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:06.423 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 3548 2023-10-09T11:07:06.423 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.423 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.423 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.423 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.423 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492309+0000", 2023-10-09T11:07:06.423 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.423 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.424 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.424 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.424 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492309+0000", 2023-10-09T11:07:06.424 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.424 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.424 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.424 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.424 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492308+0000", 2023-10-09T11:07:06.424 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:06.424 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.425 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492331+0000", 2023-10-09T11:07:06.425 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2861e-05 2023-10-09T11:07:06.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.425 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492340+0000", 2023-10-09T11:07:06.425 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4779999999999997e-06 2023-10-09T11:07:06.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.426 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.426 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492352+0000", 2023-10-09T11:07:06.426 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2148e-05 2023-10-09T11:07:06.426 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.426 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.426 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492405+0000", 2023-10-09T11:07:06.426 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2435999999999997e-05 2023-10-09T11:07:06.426 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.426 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.427 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.427 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492536+0000", 2023-10-09T11:07:06.427 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013124299999999999 2023-10-09T11:07:06.427 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.427 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.427 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,2", 2023-10-09T11:07:06.427 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492663+0000", 2023-10-09T11:07:06.427 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000126938 2023-10-09T11:07:06.427 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.427 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.427 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:06.428 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.494589+0000", 2023-10-09T11:07:06.428 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001926292 2023-10-09T11:07:06.428 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.428 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.428 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.428 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.494715+0000", 2023-10-09T11:07:06.428 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000126098 2023-10-09T11:07:06.428 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.428 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.428 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.428 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.497672+0000", 2023-10-09T11:07:06.428 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029564629999999999 2023-10-09T11:07:06.428 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.429 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.429 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.429 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.619543+0000", 2023-10-09T11:07:06.429 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12187139 2023-10-09T11:07:06.429 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.429 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.429 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.429 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.619590+0000", 2023-10-09T11:07:06.429 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6424999999999997e-05 2023-10-09T11:07:06.429 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.429 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.429 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.429 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.430 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.430 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:59720 11.19 11:9a472153:::benchmark_data_smithi125_439543_object59719:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-10-09T11:07:06.430 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.203684+0000", 2023-10-09T11:07:06.430 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 73.212380026000005, 2023-10-09T11:07:06.430 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18449432900000001, 2023-10-09T11:07:06.430 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.430 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.430 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:06.430 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:06.430 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:06.431 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 59720 2023-10-09T11:07:06.431 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.431 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.203684+0000", 2023-10-09T11:07:06.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.431 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.203684+0000", 2023-10-09T11:07:06.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.432 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.432 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.203684+0000", 2023-10-09T11:07:06.432 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:06.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.432 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.432 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.203712+0000", 2023-10-09T11:07:06.432 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8048999999999999e-05 2023-10-09T11:07:06.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.432 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.203723+0000", 2023-10-09T11:07:06.433 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1450999999999999e-05 2023-10-09T11:07:06.433 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.203730+0000", 2023-10-09T11:07:06.433 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5300000000000002e-06 2023-10-09T11:07:06.433 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.205127+0000", 2023-10-09T11:07:06.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013968769999999999 2023-10-09T11:07:06.434 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.434 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.434 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.434 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.205251+0000", 2023-10-09T11:07:06.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012439100000000001 2023-10-09T11:07:06.434 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.434 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.434 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:07:06.434 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.205387+0000", 2023-10-09T11:07:06.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013618899999999999 2023-10-09T11:07:06.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:06.435 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.207083+0000", 2023-10-09T11:07:06.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001695682 2023-10-09T11:07:06.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.435 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210270+0000", 2023-10-09T11:07:06.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0031871040000000001 2023-10-09T11:07:06.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210989+0000", 2023-10-09T11:07:06.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00071919799999999999 2023-10-09T11:07:06.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.211040+0000", 2023-10-09T11:07:06.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0840000000000001e-05 2023-10-09T11:07:06.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388178+0000", 2023-10-09T11:07:06.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17713855100000001 2023-10-09T11:07:06.437 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.437 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.437 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.437 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:59727 11.a 11:58601085:::benchmark_data_smithi125_439543_object59726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-10-09T11:07:06.437 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.208530+0000", 2023-10-09T11:07:06.437 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 73.207533744000003, 2023-10-09T11:07:06.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19139910500000001, 2023-10-09T11:07:06.437 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.437 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.438 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:06.438 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:06.438 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:06.438 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 59727 2023-10-09T11:07:06.438 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.438 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.438 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.438 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.438 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208530+0000", 2023-10-09T11:07:06.438 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.438 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.439 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.439 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.439 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208530+0000", 2023-10-09T11:07:06.439 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.439 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.439 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.439 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.439 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208529+0000", 2023-10-09T11:07:06.439 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:07:06.439 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.439 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.439 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.440 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208567+0000", 2023-10-09T11:07:06.440 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.786e-05 2023-10-09T11:07:06.440 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.440 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.440 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.440 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208579+0000", 2023-10-09T11:07:06.440 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2223e-05 2023-10-09T11:07:06.440 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.440 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.440 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.440 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208587+0000", 2023-10-09T11:07:06.440 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2109999999999998e-06 2023-10-09T11:07:06.440 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.441 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210748+0000", 2023-10-09T11:07:06.441 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002160861 2023-10-09T11:07:06.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.441 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210882+0000", 2023-10-09T11:07:06.441 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013406099999999999 2023-10-09T11:07:06.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T11:07:06.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.211049+0000", 2023-10-09T11:07:06.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016722899999999999 2023-10-09T11:07:06.442 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.442 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.392637+0000", 2023-10-09T11:07:06.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18158798400000001 2023-10-09T11:07:06.442 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.442 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.392674+0000", 2023-10-09T11:07:06.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6511999999999999e-05 2023-10-09T11:07:06.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:06.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.399528+0000", 2023-10-09T11:07:06.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0068543329999999998 2023-10-09T11:07:06.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.399590+0000", 2023-10-09T11:07:06.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1787999999999999e-05 2023-10-09T11:07:06.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.399930+0000", 2023-10-09T11:07:06.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00033950300000000001 2023-10-09T11:07:06.444 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.444 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.444 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.444 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:59730 11.b 11:d344def5:::benchmark_data_smithi125_439543_object59729:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-10-09T11:07:06.444 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.208596+0000", 2023-10-09T11:07:06.444 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 73.207468594000005, 2023-10-09T11:07:06.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18725814099999999, 2023-10-09T11:07:06.445 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.445 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.445 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:06.445 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:06.445 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:06.445 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 59730 2023-10-09T11:07:06.445 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.445 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.445 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.445 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.445 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208596+0000", 2023-10-09T11:07:06.446 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.446 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.446 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208596+0000", 2023-10-09T11:07:06.446 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.446 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.446 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208595+0000", 2023-10-09T11:07:06.446 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:06.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.447 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.447 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.447 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208617+0000", 2023-10-09T11:07:06.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1835e-05 2023-10-09T11:07:06.447 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.447 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.447 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.447 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208624+0000", 2023-10-09T11:07:06.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0700000000000001e-06 2023-10-09T11:07:06.447 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.447 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208629+0000", 2023-10-09T11:07:06.448 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3539999999999999e-06 2023-10-09T11:07:06.448 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.387822+0000", 2023-10-09T11:07:06.448 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.179192617 2023-10-09T11:07:06.448 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.387993+0000", 2023-10-09T11:07:06.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000171397 2023-10-09T11:07:06.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.449 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,1", 2023-10-09T11:07:06.449 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388182+0000", 2023-10-09T11:07:06.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018824799999999999 2023-10-09T11:07:06.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.449 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.449 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.394988+0000", 2023-10-09T11:07:06.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.006806449 2023-10-09T11:07:06.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:06.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.394995+0000", 2023-10-09T11:07:06.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1430000000000001e-06 2023-10-09T11:07:06.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.395753+0000", 2023-10-09T11:07:06.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00075767900000000003 2023-10-09T11:07:06.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.395797+0000", 2023-10-09T11:07:06.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4195e-05 2023-10-09T11:07:06.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.395854+0000", 2023-10-09T11:07:06.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6684999999999997e-05 2023-10-09T11:07:06.451 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.451 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.451 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.452 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:59725 11.0 e511/509 11:06360e61:::benchmark_data_smithi125_439543_object59724:head v 511'3409, mlcod=511'3406)", 2023-10-09T11:07:06.452 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.209430+0000", 2023-10-09T11:07:06.452 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 73.206634648999994, 2023-10-09T11:07:06.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17842855799999999, 2023-10-09T11:07:06.452 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.452 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.452 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209430+0000", 2023-10-09T11:07:06.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209430+0000", 2023-10-09T11:07:06.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209429+0000", 2023-10-09T11:07:06.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:06.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209460+0000", 2023-10-09T11:07:06.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1449999999999999e-05 2023-10-09T11:07:06.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209469+0000", 2023-10-09T11:07:06.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6710000000000005e-06 2023-10-09T11:07:06.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209475+0000", 2023-10-09T11:07:06.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4019999999999999e-06 2023-10-09T11:07:06.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209562+0000", 2023-10-09T11:07:06.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6658999999999998e-05 2023-10-09T11:07:06.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209575+0000", 2023-10-09T11:07:06.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2787e-05 2023-10-09T11:07:06.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.387811+0000", 2023-10-09T11:07:06.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17823647000000001 2023-10-09T11:07:06.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.387858+0000", 2023-10-09T11:07:06.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6767000000000001e-05 2023-10-09T11:07:06.457 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.457 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.457 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.457 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.36510.0:59720 11.19 e511/509 ondisk, result = 0)", 2023-10-09T11:07:06.457 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.210916+0000", 2023-10-09T11:07:06.457 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 73.205147847000006, 2023-10-09T11:07:06.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17726972299999999, 2023-10-09T11:07:06.457 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.457 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:07:06.457 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210916+0000", 2023-10-09T11:07:06.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210916+0000", 2023-10-09T11:07:06.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210916+0000", 2023-10-09T11:07:06.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:06.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210920+0000", 2023-10-09T11:07:06.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5900000000000001e-06 2023-10-09T11:07:06.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210927+0000", 2023-10-09T11:07:06.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6669999999999999e-06 2023-10-09T11:07:06.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210932+0000", 2023-10-09T11:07:06.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1590000000000001e-06 2023-10-09T11:07:06.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210975+0000", 2023-10-09T11:07:06.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2398999999999998e-05 2023-10-09T11:07:06.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.461 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210983+0000", 2023-10-09T11:07:06.461 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4430000000000003e-06 2023-10-09T11:07:06.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.461 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388186+0000", 2023-10-09T11:07:06.461 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.177203103 2023-10-09T11:07:06.461 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.461 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.461 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.462 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.462 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:59736 11.12 11:49022463:::benchmark_data_smithi125_439543_object59735:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-10-09T11:07:06.462 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.211151+0000", 2023-10-09T11:07:06.462 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 73.204913632, 2023-10-09T11:07:06.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19015921999999999, 2023-10-09T11:07:06.462 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.462 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.462 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:06.462 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:06.462 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:06.462 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 59736 2023-10-09T11:07:06.462 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.463 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.463 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.211151+0000", 2023-10-09T11:07:06.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.463 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.211151+0000", 2023-10-09T11:07:06.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.464 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.211149+0000", 2023-10-09T11:07:06.464 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:07:06.464 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.464 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.464 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.464 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.211171+0000", 2023-10-09T11:07:06.464 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1407e-05 2023-10-09T11:07:06.464 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.464 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.464 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.464 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.387780+0000", 2023-10-09T11:07:06.465 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17660874300000001 2023-10-09T11:07:06.465 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.465 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.465 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.387795+0000", 2023-10-09T11:07:06.465 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5179e-05 2023-10-09T11:07:06.465 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.465 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.465 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.387987+0000", 2023-10-09T11:07:06.465 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000192127 2023-10-09T11:07:06.465 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388106+0000", 2023-10-09T11:07:06.466 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119504 2023-10-09T11:07:06.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.466 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,2", 2023-10-09T11:07:06.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388384+0000", 2023-10-09T11:07:06.466 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027804799999999998 2023-10-09T11:07:06.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.466 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.391619+0000", 2023-10-09T11:07:06.467 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032347769999999999 2023-10-09T11:07:06.467 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.467 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:06.467 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.399516+0000", 2023-10-09T11:07:06.467 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0078969420000000005 2023-10-09T11:07:06.467 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.467 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.467 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.401227+0000", 2023-10-09T11:07:06.467 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001710528 2023-10-09T11:07:06.467 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.468 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.468 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.401272+0000", 2023-10-09T11:07:06.468 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5578999999999998e-05 2023-10-09T11:07:06.468 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.468 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.468 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.468 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.401310+0000", 2023-10-09T11:07:06.468 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7546000000000003e-05 2023-10-09T11:07:06.468 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.468 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.468 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.468 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.469 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.469 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:59728 11.0 e511/509 11:02f73ca9:::benchmark_data_smithi125_439543_object59727:head v 511'3410, mlcod=511'3406)", 2023-10-09T11:07:06.469 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.214456+0000", 2023-10-09T11:07:06.469 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 73.201608323000002, 2023-10-09T11:07:06.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18448844, 2023-10-09T11:07:06.469 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.469 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.469 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.469 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.469 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.214456+0000", 2023-10-09T11:07:06.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.469 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.470 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.214456+0000", 2023-10-09T11:07:06.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.470 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.470 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.214455+0000", 2023-10-09T11:07:06.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:07:06.470 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.214482+0000", 2023-10-09T11:07:06.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7591999999999999e-05 2023-10-09T11:07:06.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.471 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.471 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388348+0000", 2023-10-09T11:07:06.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.173865874 2023-10-09T11:07:06.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.471 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.471 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388356+0000", 2023-10-09T11:07:06.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0649999999999999e-06 2023-10-09T11:07:06.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.472 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.472 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.472 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.392408+0000", 2023-10-09T11:07:06.472 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040511490000000004 2023-10-09T11:07:06.472 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.472 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.472 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.472 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.392428+0000", 2023-10-09T11:07:06.472 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0920999999999998e-05 2023-10-09T11:07:06.472 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.472 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.472 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.472 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.398906+0000", 2023-10-09T11:07:06.473 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0064776460000000001 2023-10-09T11:07:06.473 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.473 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.473 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.473 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.398944+0000", 2023-10-09T11:07:06.473 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8172e-05 2023-10-09T11:07:06.473 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.473 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.473 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.473 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.473 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.473 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPushReply(11.a 511/500 [PushReplyOp(11:546b761d:::benchmark_data_smithi125_439543_object15921:head)])", 2023-10-09T11:07:06.473 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.220273+0000", 2023-10-09T11:07:06.474 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 73.195791082, 2023-10-09T11:07:06.474 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18383527099999999, 2023-10-09T11:07:06.474 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.474 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:07:06.474 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.474 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.474 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.474 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.220273+0000", 2023-10-09T11:07:06.474 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.474 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.474 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.475 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.475 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.220273+0000", 2023-10-09T11:07:06.475 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.475 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.475 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.475 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.475 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.220272+0000", 2023-10-09T11:07:06.475 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:07:06.475 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.475 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.475 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.475 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.220282+0000", 2023-10-09T11:07:06.476 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0423e-05 2023-10-09T11:07:06.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.476 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.476 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.476 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388222+0000", 2023-10-09T11:07:06.476 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.167939901 2023-10-09T11:07:06.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.476 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.476 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.476 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388230+0000", 2023-10-09T11:07:06.476 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5320000000000006e-06 2023-10-09T11:07:06.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.477 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.477 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.477 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.403669+0000", 2023-10-09T11:07:06.477 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015438881999999999 2023-10-09T11:07:06.477 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.477 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.477 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.477 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.404108+0000", 2023-10-09T11:07:06.477 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00043913399999999999 2023-10-09T11:07:06.477 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.477 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.477 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.478 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.478 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.478 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:60183 11.19 11:9e2099d9:::benchmark_data_smithi125_439543_object60182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T11:07:06.478 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.701324+0000", 2023-10-09T11:07:06.478 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.714740312000004, 2023-10-09T11:07:06.478 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.13270354000000001, 2023-10-09T11:07:06.478 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.478 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.478 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:06.478 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:06.478 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:06.478 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 60183 2023-10-09T11:07:06.479 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.479 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.479 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.479 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.479 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.701324+0000", 2023-10-09T11:07:06.479 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.479 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.479 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.479 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.479 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.701324+0000", 2023-10-09T11:07:06.479 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.479 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.480 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.480 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.480 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.701323+0000", 2023-10-09T11:07:06.480 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:06.480 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.480 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.480 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.480 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.701345+0000", 2023-10-09T11:07:06.480 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1982999999999999e-05 2023-10-09T11:07:06.480 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.480 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.480 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.701352+0000", 2023-10-09T11:07:06.481 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7719999999999997e-06 2023-10-09T11:07:06.481 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.481 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.481 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.701366+0000", 2023-10-09T11:07:06.481 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3806e-05 2023-10-09T11:07:06.481 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.481 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.481 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.702792+0000", 2023-10-09T11:07:06.481 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014258389999999999 2023-10-09T11:07:06.482 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.482 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.482 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.482 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.702935+0000", 2023-10-09T11:07:06.482 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014319800000000001 2023-10-09T11:07:06.482 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.482 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.482 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:07:06.482 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.703081+0000", 2023-10-09T11:07:06.482 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014630900000000001 2023-10-09T11:07:06.482 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.482 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.483 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.483 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830385+0000", 2023-10-09T11:07:06.483 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12730330400000001 2023-10-09T11:07:06.483 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.483 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.483 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.483 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830422+0000", 2023-10-09T11:07:06.483 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.752e-05 2023-10-09T11:07:06.483 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.483 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.483 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:06.483 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.833365+0000", 2023-10-09T11:07:06.484 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029424059999999998 2023-10-09T11:07:06.484 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.484 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.484 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.484 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.833415+0000", 2023-10-09T11:07:06.484 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0049999999999997e-05 2023-10-09T11:07:06.484 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.484 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.484 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.484 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.834027+0000", 2023-10-09T11:07:06.484 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00061278999999999999 2023-10-09T11:07:06.484 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.485 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.485 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.485 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.485 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.485 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:60185 11.a 11:5f9a8cdf:::benchmark_data_smithi125_439543_object60184:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T11:07:06.485 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.705137+0000", 2023-10-09T11:07:06.485 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.710927321, 2023-10-09T11:07:06.485 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.13415111399999999, 2023-10-09T11:07:06.485 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.485 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.485 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:06.485 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:06.486 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:06.486 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 60185 2023-10-09T11:07:06.486 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.486 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.486 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.486 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.486 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705137+0000", 2023-10-09T11:07:06.486 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.486 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.486 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.486 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.486 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705137+0000", 2023-10-09T11:07:06.487 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.487 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.487 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.487 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.487 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705135+0000", 2023-10-09T11:07:06.487 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:07:06.487 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.487 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.487 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.488 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705170+0000", 2023-10-09T11:07:06.489 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4335999999999998e-05 2023-10-09T11:07:06.489 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.489 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.489 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.489 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705178+0000", 2023-10-09T11:07:06.489 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4999999999999999e-06 2023-10-09T11:07:06.489 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.489 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.489 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.489 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705184+0000", 2023-10-09T11:07:06.489 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1819999999999997e-06 2023-10-09T11:07:06.490 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.490 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.490 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.490 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705293+0000", 2023-10-09T11:07:06.490 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000109036 2023-10-09T11:07:06.490 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.490 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.490 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.490 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705435+0000", 2023-10-09T11:07:06.490 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000142564 2023-10-09T11:07:06.490 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.490 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.491 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T11:07:06.491 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705599+0000", 2023-10-09T11:07:06.491 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016396499999999999 2023-10-09T11:07:06.491 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.491 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.491 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.491 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.833012+0000", 2023-10-09T11:07:06.491 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12741256100000001 2023-10-09T11:07:06.491 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.491 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.491 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.491 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.833049+0000", 2023-10-09T11:07:06.491 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7626999999999997e-05 2023-10-09T11:07:06.492 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.492 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.492 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:06.492 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.838831+0000", 2023-10-09T11:07:06.492 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0057820240000000002 2023-10-09T11:07:06.492 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.492 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.492 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.492 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.838880+0000", 2023-10-09T11:07:06.492 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8893000000000001e-05 2023-10-09T11:07:06.492 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.492 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.492 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.493 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.839288+0000", 2023-10-09T11:07:06.493 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00040779899999999998 2023-10-09T11:07:06.493 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.493 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.493 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.493 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.493 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.493 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:60186 11.b 11:d8ac8572:::benchmark_data_smithi125_439543_object60185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T11:07:06.493 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.705197+0000", 2023-10-09T11:07:06.493 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.710867308000005, 2023-10-09T11:07:06.493 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.13208626500000001, 2023-10-09T11:07:06.493 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.494 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.494 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:06.494 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:06.494 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:06.494 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 60186 2023-10-09T11:07:06.494 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.494 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.494 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.494 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.494 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705197+0000", 2023-10-09T11:07:06.494 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.494 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.495 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.495 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.495 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705197+0000", 2023-10-09T11:07:06.495 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.495 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.495 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.495 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.495 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705196+0000", 2023-10-09T11:07:06.495 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:06.495 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.495 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.495 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.496 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705237+0000", 2023-10-09T11:07:06.496 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0639000000000002e-05 2023-10-09T11:07:06.496 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.496 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.496 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.496 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705243+0000", 2023-10-09T11:07:06.496 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4809999999999997e-06 2023-10-09T11:07:06.496 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.496 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.496 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.496 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705246+0000", 2023-10-09T11:07:06.496 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3390000000000001e-06 2023-10-09T11:07:06.497 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.497 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.497 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.497 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705307+0000", 2023-10-09T11:07:06.497 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1023999999999997e-05 2023-10-09T11:07:06.497 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.497 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.497 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.497 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705419+0000", 2023-10-09T11:07:06.497 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011229499999999999 2023-10-09T11:07:06.497 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.497 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.497 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,1", 2023-10-09T11:07:06.498 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705572+0000", 2023-10-09T11:07:06.498 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000152349 2023-10-09T11:07:06.498 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.498 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.498 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.498 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.832363+0000", 2023-10-09T11:07:06.498 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12679153100000001 2023-10-09T11:07:06.498 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.498 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.498 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.498 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.832401+0000", 2023-10-09T11:07:06.499 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7738999999999999e-05 2023-10-09T11:07:06.499 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.499 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.499 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:06.499 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.837189+0000", 2023-10-09T11:07:06.499 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0047882649999999999 2023-10-09T11:07:06.499 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.499 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.499 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.499 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.837244+0000", 2023-10-09T11:07:06.499 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4719999999999998e-05 2023-10-09T11:07:06.499 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.499 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.500 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.500 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.837283+0000", 2023-10-09T11:07:06.500 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9283000000000003e-05 2023-10-09T11:07:06.500 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.500 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.500 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.500 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.500 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.500 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:60187 11.11 11:8970ccab:::benchmark_data_smithi125_439543_object60186:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T11:07:06.500 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.705256+0000", 2023-10-09T11:07:06.500 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.710807743999993, 2023-10-09T11:07:06.501 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.13120647499999999, 2023-10-09T11:07:06.501 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.501 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.501 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:06.501 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:06.501 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:06.501 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 60187 2023-10-09T11:07:06.501 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.501 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.501 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.501 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.501 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705256+0000", 2023-10-09T11:07:06.501 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.501 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.502 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.502 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.502 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705256+0000", 2023-10-09T11:07:06.502 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.502 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.502 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.502 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.502 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705256+0000", 2023-10-09T11:07:06.502 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:06.502 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.502 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.503 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.503 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705273+0000", 2023-10-09T11:07:06.503 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6568000000000001e-05 2023-10-09T11:07:06.503 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.503 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.503 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.503 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705277+0000", 2023-10-09T11:07:06.503 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9439999999999998e-06 2023-10-09T11:07:06.503 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.503 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.503 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.504 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705281+0000", 2023-10-09T11:07:06.504 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2069999999999997e-06 2023-10-09T11:07:06.504 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.504 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.504 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.504 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830449+0000", 2023-10-09T11:07:06.504 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12516856500000001 2023-10-09T11:07:06.504 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.504 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.504 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.504 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830612+0000", 2023-10-09T11:07:06.505 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016255299999999999 2023-10-09T11:07:06.505 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.505 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.505 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:07:06.505 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830791+0000", 2023-10-09T11:07:06.505 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017918900000000001 2023-10-09T11:07:06.505 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.505 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.505 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.505 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.833018+0000", 2023-10-09T11:07:06.505 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022270580000000001 2023-10-09T11:07:06.505 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.506 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.506 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:06.506 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.835358+0000", 2023-10-09T11:07:06.506 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023397209999999999 2023-10-09T11:07:06.506 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.506 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.506 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.506 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.835764+0000", 2023-10-09T11:07:06.506 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00040589199999999998 2023-10-09T11:07:06.506 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.506 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.506 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.507 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.835811+0000", 2023-10-09T11:07:06.507 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7602000000000003e-05 2023-10-09T11:07:06.507 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.507 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.507 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.507 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.836463+0000", 2023-10-09T11:07:06.507 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00065151399999999996 2023-10-09T11:07:06.507 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.507 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.507 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.507 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.507 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.507 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPushReply(11.a 512/500 [PushReplyOp(11:52aba3e0:::benchmark_data_smithi125_439543_object16408:head)])", 2023-10-09T11:07:06.507 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.705514+0000", 2023-10-09T11:07:06.508 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.710550206999997, 2023-10-09T11:07:06.508 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12745868099999999, 2023-10-09T11:07:06.508 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.508 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:07:06.508 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.508 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.508 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.508 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705514+0000", 2023-10-09T11:07:06.508 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.508 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.508 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.508 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.508 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705514+0000", 2023-10-09T11:07:06.509 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.509 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.509 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.509 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.509 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705514+0000", 2023-10-09T11:07:06.509 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:06.509 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.509 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.509 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.509 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705518+0000", 2023-10-09T11:07:06.509 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8509999999999998e-06 2023-10-09T11:07:06.510 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.510 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.510 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.510 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705530+0000", 2023-10-09T11:07:06.510 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1951e-05 2023-10-09T11:07:06.510 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.510 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.510 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.510 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705538+0000", 2023-10-09T11:07:06.510 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4900000000000003e-06 2023-10-09T11:07:06.510 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.510 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.510 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.511 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.832353+0000", 2023-10-09T11:07:06.511 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.126815452 2023-10-09T11:07:06.511 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.511 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.511 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.511 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.832973+0000", 2023-10-09T11:07:06.511 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00061930299999999998 2023-10-09T11:07:06.511 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.511 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.511 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.511 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.511 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.511 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:60188 11.1 11:80f1a15d:::benchmark_data_smithi125_439543_object60187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T11:07:06.511 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.705811+0000", 2023-10-09T11:07:06.512 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.710253473999998, 2023-10-09T11:07:06.512 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.13158576999999999, 2023-10-09T11:07:06.512 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.512 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.512 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:06.512 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:06.512 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:06.512 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 60188 2023-10-09T11:07:06.512 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.512 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.512 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.512 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.512 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705811+0000", 2023-10-09T11:07:06.513 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.513 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.513 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.513 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.513 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705811+0000", 2023-10-09T11:07:06.513 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.513 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.513 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.513 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.513 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705810+0000", 2023-10-09T11:07:06.513 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:06.513 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.513 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.514 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.514 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705830+0000", 2023-10-09T11:07:06.514 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9949000000000002e-05 2023-10-09T11:07:06.514 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.514 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.514 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.514 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830332+0000", 2023-10-09T11:07:06.514 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.124501739 2023-10-09T11:07:06.514 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.514 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.514 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.514 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830341+0000", 2023-10-09T11:07:06.515 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5899999999999997e-06 2023-10-09T11:07:06.515 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.515 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.515 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.515 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830700+0000", 2023-10-09T11:07:06.515 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000359062 2023-10-09T11:07:06.515 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.515 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.515 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.515 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830837+0000", 2023-10-09T11:07:06.515 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000136861 2023-10-09T11:07:06.515 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.515 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.516 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:07:06.516 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830983+0000", 2023-10-09T11:07:06.516 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000145616 2023-10-09T11:07:06.516 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.516 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.516 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.516 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.833055+0000", 2023-10-09T11:07:06.516 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002071603 2023-10-09T11:07:06.516 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.516 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.516 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.516 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.835734+0000", 2023-10-09T11:07:06.516 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026798730000000002 2023-10-09T11:07:06.516 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.517 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.517 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:06.517 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.836720+0000", 2023-10-09T11:07:06.517 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0009858200000000001 2023-10-09T11:07:06.517 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.517 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.517 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.517 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.836765+0000", 2023-10-09T11:07:06.517 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4527000000000002e-05 2023-10-09T11:07:06.517 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.517 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.517 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.518 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.837396+0000", 2023-10-09T11:07:06.518 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00063176300000000003 2023-10-09T11:07:06.518 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.518 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.518 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.518 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.518 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.518 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:84218 11.2 11:44986b72:::benchmark_data_smithi125_439543_object84217:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T11:07:06.518 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:06:20.502344+0000", 2023-10-09T11:07:06.518 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 45.913719880999999, 2023-10-09T11:07:06.518 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12896063699999999, 2023-10-09T11:07:06.519 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.519 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.519 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:06.519 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:06.519 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:06.519 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 84218 2023-10-09T11:07:06.519 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.519 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.519 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.519 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.519 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502344+0000", 2023-10-09T11:07:06.519 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.520 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.520 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.520 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.520 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502344+0000", 2023-10-09T11:07:06.520 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.520 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.520 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.520 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.520 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502343+0000", 2023-10-09T11:07:06.520 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:07:06.520 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.520 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.521 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.521 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502367+0000", 2023-10-09T11:07:06.521 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3883000000000002e-05 2023-10-09T11:07:06.521 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.521 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.521 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.521 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502377+0000", 2023-10-09T11:07:06.521 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8770000000000005e-06 2023-10-09T11:07:06.521 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.521 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.521 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.521 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502383+0000", 2023-10-09T11:07:06.522 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6320000000000002e-06 2023-10-09T11:07:06.522 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.522 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.522 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.522 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502491+0000", 2023-10-09T11:07:06.522 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108058 2023-10-09T11:07:06.522 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.522 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.522 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.522 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502611+0000", 2023-10-09T11:07:06.522 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119146 2023-10-09T11:07:06.522 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.523 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.523 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,2", 2023-10-09T11:07:06.523 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502746+0000", 2023-10-09T11:07:06.523 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000135814 2023-10-09T11:07:06.523 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.523 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.523 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.523 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.504710+0000", 2023-10-09T11:07:06.523 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019633710000000002 2023-10-09T11:07:06.523 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.523 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.523 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:06.524 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505713+0000", 2023-10-09T11:07:06.524 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00100332 2023-10-09T11:07:06.524 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.524 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.524 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.524 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.631217+0000", 2023-10-09T11:07:06.524 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12550412499999999 2023-10-09T11:07:06.524 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.524 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.524 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.524 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.631267+0000", 2023-10-09T11:07:06.524 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9614999999999998e-05 2023-10-09T11:07:06.525 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.525 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.525 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.525 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.631305+0000", 2023-10-09T11:07:06.525 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8152999999999999e-05 2023-10-09T11:07:06.525 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.525 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.525 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.525 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.525 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.525 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:84219 11.b 11:d750bcf9:::benchmark_data_smithi125_439543_object84218:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T11:07:06.525 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:06:20.504974+0000", 2023-10-09T11:07:06.526 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 45.911090315999999, 2023-10-09T11:07:06.526 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12751848499999999, 2023-10-09T11:07:06.526 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.526 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.526 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:06.526 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:06.526 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:06.526 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 84219 2023-10-09T11:07:06.526 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.526 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.526 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.526 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.527 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.504974+0000", 2023-10-09T11:07:06.527 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.527 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.527 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.527 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.527 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.504974+0000", 2023-10-09T11:07:06.527 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.527 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.527 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.527 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.527 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.504972+0000", 2023-10-09T11:07:06.528 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:07:06.528 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.528 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.528 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.528 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.504994+0000", 2023-10-09T11:07:06.528 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2175999999999999e-05 2023-10-09T11:07:06.528 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.528 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.528 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.528 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505004+0000", 2023-10-09T11:07:06.528 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0035e-05 2023-10-09T11:07:06.529 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.529 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.529 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.529 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505010+0000", 2023-10-09T11:07:06.529 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0580000000000003e-06 2023-10-09T11:07:06.529 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.529 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.529 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.529 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.506161+0000", 2023-10-09T11:07:06.530 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011510800000000001 2023-10-09T11:07:06.530 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.530 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.530 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.530 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.506300+0000", 2023-10-09T11:07:06.530 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013864699999999999 2023-10-09T11:07:06.530 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.530 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.530 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,1", 2023-10-09T11:07:06.530 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.506447+0000", 2023-10-09T11:07:06.530 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014706699999999999 2023-10-09T11:07:06.531 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.531 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.531 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:06.531 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.630287+0000", 2023-10-09T11:07:06.531 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.123840197 2023-10-09T11:07:06.531 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.531 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.531 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.531 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.631531+0000", 2023-10-09T11:07:06.531 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012432350000000001 2023-10-09T11:07:06.531 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.531 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.532 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.532 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632417+0000", 2023-10-09T11:07:06.532 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00088668199999999996 2023-10-09T11:07:06.532 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.532 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.532 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.532 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632460+0000", 2023-10-09T11:07:06.532 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3158999999999998e-05 2023-10-09T11:07:06.532 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.532 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.532 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.532 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632492+0000", 2023-10-09T11:07:06.533 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1985e-05 2023-10-09T11:07:06.533 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.533 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.533 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.533 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.533 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.533 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:84223 11.11 11:89bbe302:::benchmark_data_smithi125_439543_object84222:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T11:07:06.533 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:06:20.505039+0000", 2023-10-09T11:07:06.533 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 45.911024861999998, 2023-10-09T11:07:06.533 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12827670299999999, 2023-10-09T11:07:06.533 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.533 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.533 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:06.534 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:06.534 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:06.534 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 84223 2023-10-09T11:07:06.534 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.534 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.534 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.534 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.534 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505039+0000", 2023-10-09T11:07:06.534 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.534 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.534 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.535 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.535 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505039+0000", 2023-10-09T11:07:06.535 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.535 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.535 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.535 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.535 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505039+0000", 2023-10-09T11:07:06.535 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:06.535 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.535 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.535 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.535 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505065+0000", 2023-10-09T11:07:06.536 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6179000000000001e-05 2023-10-09T11:07:06.536 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.536 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.536 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.536 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505071+0000", 2023-10-09T11:07:06.536 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9349999999999999e-06 2023-10-09T11:07:06.536 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.536 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.536 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.536 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505075+0000", 2023-10-09T11:07:06.536 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9639999999999997e-06 2023-10-09T11:07:06.536 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.536 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.537 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.537 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505119+0000", 2023-10-09T11:07:06.537 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4150999999999998e-05 2023-10-09T11:07:06.537 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.537 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.537 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.537 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505291+0000", 2023-10-09T11:07:06.537 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017220500000000001 2023-10-09T11:07:06.537 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.537 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.537 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:07:06.537 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505402+0000", 2023-10-09T11:07:06.537 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011069399999999999 2023-10-09T11:07:06.538 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.538 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.538 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:06.538 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632096+0000", 2023-10-09T11:07:06.538 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.126693951 2023-10-09T11:07:06.538 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.538 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.538 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.538 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632163+0000", 2023-10-09T11:07:06.538 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7118000000000004e-05 2023-10-09T11:07:06.538 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.538 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.539 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.539 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632252+0000", 2023-10-09T11:07:06.539 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9019999999999998e-05 2023-10-09T11:07:06.539 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.539 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.539 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.539 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632614+0000", 2023-10-09T11:07:06.539 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00036246999999999999 2023-10-09T11:07:06.539 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.539 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.539 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.539 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.633316+0000", 2023-10-09T11:07:06.539 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00070155499999999997 2023-10-09T11:07:06.540 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.540 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.540 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.540 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.540 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.540 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:84226 11.11 11:8d2cd1ab:::benchmark_data_smithi125_439543_object84225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T11:07:06.540 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:06:20.505181+0000", 2023-10-09T11:07:06.540 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 45.910882860000001, 2023-10-09T11:07:06.540 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.129217903, 2023-10-09T11:07:06.540 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.540 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.541 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:06.541 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:06.541 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:06.541 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 84226 2023-10-09T11:07:06.541 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.541 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.541 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.541 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.541 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505181+0000", 2023-10-09T11:07:06.541 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.541 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.542 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.542 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.542 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505181+0000", 2023-10-09T11:07:06.542 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.542 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.542 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.542 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.542 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505181+0000", 2023-10-09T11:07:06.542 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:06.542 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.543 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.543 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.543 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505200+0000", 2023-10-09T11:07:06.543 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9017000000000001e-05 2023-10-09T11:07:06.543 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.543 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.543 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.543 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505220+0000", 2023-10-09T11:07:06.543 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9729e-05 2023-10-09T11:07:06.543 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.543 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.543 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.544 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505225+0000", 2023-10-09T11:07:06.544 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7969999999999997e-06 2023-10-09T11:07:06.544 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.544 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.544 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.544 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.506648+0000", 2023-10-09T11:07:06.544 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014222219999999999 2023-10-09T11:07:06.544 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.544 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.544 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.545 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.506725+0000", 2023-10-09T11:07:06.545 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7541999999999997e-05 2023-10-09T11:07:06.545 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.545 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.545 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:07:06.545 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.506815+0000", 2023-10-09T11:07:06.545 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0074999999999994e-05 2023-10-09T11:07:06.545 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.545 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.545 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:06.545 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632116+0000", 2023-10-09T11:07:06.545 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.125300404 2023-10-09T11:07:06.546 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.546 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.546 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.546 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632204+0000", 2023-10-09T11:07:06.546 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7865e-05 2023-10-09T11:07:06.546 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.546 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.546 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.546 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.633770+0000", 2023-10-09T11:07:06.546 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015663960000000001 2023-10-09T11:07:06.546 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.546 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.547 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.547 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.633826+0000", 2023-10-09T11:07:06.547 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5597000000000003e-05 2023-10-09T11:07:06.547 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.547 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.547 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.547 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.634399+0000", 2023-10-09T11:07:06.547 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00057372100000000002 2023-10-09T11:07:06.547 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.547 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.547 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.547 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.547 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.548 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:84230 11.b 11:d191ae1b:::benchmark_data_smithi125_439543_object84229:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T11:07:06.548 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:06:20.505234+0000", 2023-10-09T11:07:06.548 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 45.910829708999998, 2023-10-09T11:07:06.548 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.13026402200000001, 2023-10-09T11:07:06.548 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:06.548 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:06.548 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:06.548 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:06.548 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:06.548 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 84230 2023-10-09T11:07:06.548 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.548 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:06.549 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.549 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:06.549 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505234+0000", 2023-10-09T11:07:06.549 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.549 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.549 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.549 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:06.549 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505234+0000", 2023-10-09T11:07:06.549 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:06.549 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.549 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.549 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:06.550 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505234+0000", 2023-10-09T11:07:06.550 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:06.550 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.550 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.550 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:06.550 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505256+0000", 2023-10-09T11:07:06.550 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2177000000000001e-05 2023-10-09T11:07:06.550 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.550 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.550 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:06.550 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505263+0000", 2023-10-09T11:07:06.550 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7530000000000004e-06 2023-10-09T11:07:06.551 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.551 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.551 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:06.551 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505269+0000", 2023-10-09T11:07:06.551 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7359999999999998e-06 2023-10-09T11:07:06.551 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.551 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.551 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:06.551 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.508763+0000", 2023-10-09T11:07:06.551 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0034948259999999999 2023-10-09T11:07:06.551 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.551 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.552 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:06.552 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.508857+0000", 2023-10-09T11:07:06.552 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3800000000000003e-05 2023-10-09T11:07:06.552 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.552 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.552 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,1", 2023-10-09T11:07:06.552 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.508957+0000", 2023-10-09T11:07:06.552 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9436000000000001e-05 2023-10-09T11:07:06.552 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.552 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.552 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.552 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.631561+0000", 2023-10-09T11:07:06.552 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.122603957 2023-10-09T11:07:06.553 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.553 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.553 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:06.553 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632510+0000", 2023-10-09T11:07:06.553 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00094982900000000002 2023-10-09T11:07:06.553 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.553 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.553 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:06.553 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.635417+0000", 2023-10-09T11:07:06.553 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029067490000000001 2023-10-09T11:07:06.553 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.553 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.554 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:06.554 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.635461+0000", 2023-10-09T11:07:06.554 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3571e-05 2023-10-09T11:07:06.554 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:06.554 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:06.554 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:06.554 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.635499+0000", 2023-10-09T11:07:06.554 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7709000000000001e-05 2023-10-09T11:07:06.554 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.554 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.554 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.554 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:06.555 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:06.555 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:07:06.555 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:06.612 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T11:07:06.612 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[195507]: osdmap e539: 8 total, 8 up, 8 in 2023-10-09T11:07:06.612 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[195507]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.612 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[195507]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.612 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[195507]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.612 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[195507]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.612 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[195507]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.613 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[195507]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.613 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[195507]: pgmap v1316: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 8461/231318 objects degraded (3.658%); 3584/231318 objects misplaced (1.549%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T11:07:06.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T11:07:06.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[200197]: osdmap e539: 8 total, 8 up, 8 in 2023-10-09T11:07:06.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[200197]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[200197]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[200197]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[200197]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[200197]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.614 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[200197]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.614 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:06 smithi125 ceph-mon[200197]: pgmap v1316: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 8461/231318 objects degraded (3.658%); 3584/231318 objects misplaced (1.549%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T11:07:06.617 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:06.655 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:07:06.701 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:07:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:06 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T11:07:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:06 smithi155 ceph-mon[184854]: osdmap e539: 8 total, 8 up, 8 in 2023-10-09T11:07:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:06 smithi155 ceph-mon[184854]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:06 smithi155 ceph-mon[184854]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:06 smithi155 ceph-mon[184854]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:06 smithi155 ceph-mon[184854]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:06 smithi155 ceph-mon[184854]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:06 smithi155 ceph-mon[184854]: failed to encode map e539 with expected crc 2023-10-09T11:07:06.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:06 smithi155 ceph-mon[184854]: pgmap v1316: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 8461/231318 objects degraded (3.658%); 3584/231318 objects misplaced (1.549%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T11:07:06.734 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:07:06.764 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:07:06.798 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:07:06.850 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:07:06.888 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:07:06.916 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:07:06.944 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:07:06.944 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:07:06.944 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:07:06.944 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:07:06.945 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:07:06.945 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:07:06.945 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:07:06.945 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:07:07.021 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:07.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:07.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:07.129 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:07:07.160 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:07.181 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:07:07.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:07.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:07:07.285 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:07.337 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:07.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:07 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4199747033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:07.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:07 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1452001995' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:07:07.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:07 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:07:07.530 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:07:07.559 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:07 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4199747033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:07.559 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:07 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1452001995' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:07:07.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:07 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:07:07.560 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:07 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4199747033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:07.560 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:07 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1452001995' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:07:07.560 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:07 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:07:08.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:07:08.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:07:08.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[195507]: osdmap e540: 8 total, 8 up, 8 in 2023-10-09T11:07:08.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[195507]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[195507]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[195507]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[195507]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[195507]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[195507]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[195507]: 11.16 scrub starts 2023-10-09T11:07:08.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[195507]: pgmap v1318: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 8461/226974 objects degraded (3.728%); 3214/226974 objects misplaced (1.416%); 12 MiB/s, 193 objects/s recovering 2023-10-09T11:07:08.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 8461/231318 objects degraded (3.658%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:08.756 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:07:08.756 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:07:08.756 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[200197]: osdmap e540: 8 total, 8 up, 8 in 2023-10-09T11:07:08.756 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[200197]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.756 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[200197]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.756 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[200197]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.756 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[200197]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.756 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[200197]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.757 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[200197]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.757 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[200197]: 11.16 scrub starts 2023-10-09T11:07:08.757 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[200197]: pgmap v1318: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 8461/226974 objects degraded (3.728%); 3214/226974 objects misplaced (1.416%); 12 MiB/s, 193 objects/s recovering 2023-10-09T11:07:08.757 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:08 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 8461/231318 objects degraded (3.658%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:08.799 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T11:07:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:08 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:07:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:08 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:07:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:08 smithi155 ceph-mon[184854]: osdmap e540: 8 total, 8 up, 8 in 2023-10-09T11:07:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:08 smithi155 ceph-mon[184854]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:08 smithi155 ceph-mon[184854]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:08 smithi155 ceph-mon[184854]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:08 smithi155 ceph-mon[184854]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:08 smithi155 ceph-mon[184854]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:08 smithi155 ceph-mon[184854]: failed to encode map e540 with expected crc 2023-10-09T11:07:08.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:08 smithi155 ceph-mon[184854]: 11.16 scrub starts 2023-10-09T11:07:08.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:08 smithi155 ceph-mon[184854]: pgmap v1318: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 8461/226974 objects degraded (3.728%); 3214/226974 objects misplaced (1.416%); 12 MiB/s, 193 objects/s recovering 2023-10-09T11:07:08.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:08 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 8461/231318 objects degraded (3.658%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:08.993 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:09.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:09.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:09.106 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:09.141 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:09.170 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:09.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:09.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:09.276 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:09.333 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:09.623 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:09.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:09.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:09.729 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:09.764 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:09.791 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:09.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:09.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:09.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:09.947 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:11.219 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:10 smithi125 ceph-mon[195507]: pgmap v1319: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1k op/s; 7927/217365 objects degraded (3.647%); 3073/217365 objects misplaced (1.414%); 11 MiB/s, 178 objects/s recovering 2023-10-09T11:07:11.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:10 smithi125 ceph-mon[200197]: pgmap v1319: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1k op/s; 7927/217365 objects degraded (3.647%); 3073/217365 objects misplaced (1.414%); 11 MiB/s, 178 objects/s recovering 2023-10-09T11:07:11.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:10 smithi155 ceph-mon[184854]: pgmap v1319: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1k op/s; 7927/217365 objects degraded (3.647%); 3073/217365 objects misplaced (1.414%); 11 MiB/s, 178 objects/s recovering 2023-10-09T11:07:11.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:07:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:07:11] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.43.0" 2023-10-09T11:07:12.210 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:12.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:11 smithi155 ceph-mon[184854]: 11.6 scrub starts 2023-10-09T11:07:12.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:11 smithi155 ceph-mon[184854]: 11.4 scrub starts 2023-10-09T11:07:12.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:12.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:12.223 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:11 smithi125 ceph-mon[195507]: 11.6 scrub starts 2023-10-09T11:07:12.223 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:11 smithi125 ceph-mon[195507]: 11.4 scrub starts 2023-10-09T11:07:12.223 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:11 smithi125 ceph-mon[200197]: 11.6 scrub starts 2023-10-09T11:07:12.224 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:11 smithi125 ceph-mon[200197]: 11.4 scrub starts 2023-10-09T11:07:12.323 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:12.370 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:12.415 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:12.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:12.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:12.457 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:12.460 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:12.523 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:12.591 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:12.955 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:12 smithi125 ceph-mon[195507]: 11.d scrub starts 2023-10-09T11:07:12.955 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:12 smithi125 ceph-mon[195507]: pgmap v1320: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 7927/212292 objects degraded (3.734%); 2594/212292 objects misplaced (1.222%); 11 MiB/s, 172 objects/s recovering 2023-10-09T11:07:13.184 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:13.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:13.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:13.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:12 smithi155 ceph-mon[184854]: 11.d scrub starts 2023-10-09T11:07:13.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:12 smithi155 ceph-mon[184854]: pgmap v1320: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 7927/212292 objects degraded (3.734%); 2594/212292 objects misplaced (1.222%); 11 MiB/s, 172 objects/s recovering 2023-10-09T11:07:13.275 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:12 smithi125 ceph-mon[200197]: 11.d scrub starts 2023-10-09T11:07:13.275 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:12 smithi125 ceph-mon[200197]: pgmap v1320: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 7927/212292 objects degraded (3.734%); 2594/212292 objects misplaced (1.222%); 11 MiB/s, 172 objects/s recovering 2023-10-09T11:07:13.289 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:07:13.321 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:13.353 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:07:13.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:13.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:07:13.459 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:13.496 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:13.524 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:13.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:13.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:13.627 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:13.665 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:13.692 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:13.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:13.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:13.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:13.835 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:13.871 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:13.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:13.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:13.955 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:13 smithi155 ceph-mon[184854]: 11.14 scrub starts 2023-10-09T11:07:13.956 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:13 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 7927/212292 objects degraded (3.734%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T11:07:13.959 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:13 smithi125 ceph-mon[195507]: 11.14 scrub starts 2023-10-09T11:07:13.959 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:13 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 7927/212292 objects degraded (3.734%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T11:07:13.959 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:13 smithi125 ceph-mon[200197]: 11.14 scrub starts 2023-10-09T11:07:13.959 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:13 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 7927/212292 objects degraded (3.734%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T11:07:13.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:13.978 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:07:14.043 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:14.141 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:14.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:14.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:14.227 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:07:14.250 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:14.279 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:14.301 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:14.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:14.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:14.406 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:14.503 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:14.568 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:14.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:14.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:14.665 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:14.677 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:14.714 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:07:14.714 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:07:14.792 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:14.924 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:14.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:14.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:15.029 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:15.100 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:15.221 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:14 smithi155 ceph-mon[184854]: pgmap v1321: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s; 7298/200424 objects degraded (3.641%); 2427/200424 objects misplaced (1.211%); 15 MiB/s, 238 objects/s recovering 2023-10-09T11:07:15.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:14 smithi125 ceph-mon[195507]: pgmap v1321: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s; 7298/200424 objects degraded (3.641%); 2427/200424 objects misplaced (1.211%); 15 MiB/s, 238 objects/s recovering 2023-10-09T11:07:15.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:14 smithi125 ceph-mon[200197]: pgmap v1321: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s; 7298/200424 objects degraded (3.641%); 2427/200424 objects misplaced (1.211%); 15 MiB/s, 238 objects/s recovering 2023-10-09T11:07:15.360 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:15.392 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:15.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:15.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:15.443 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T11:07:15.502 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:15.538 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:15.571 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:15.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:15.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:15.678 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:15.760 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:15.853 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:15.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:15.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:15.959 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:07:15.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:15 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2778439299' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:07:15.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:07:15.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:15 smithi125 ceph-mon[195507]: 11.4 scrub starts 2023-10-09T11:07:15.995 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:15 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2778439299' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:07:15.995 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:07:15.995 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:15 smithi125 ceph-mon[200197]: 11.4 scrub starts 2023-10-09T11:07:16.000 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:16.022 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:07:16.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:16.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:07:16.029 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T11:07:16.127 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:16.158 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:16.181 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:16.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:16.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:16.285 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:16.298 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:07:16.311 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:15 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2778439299' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:07:16.312 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:07:16.312 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:15 smithi155 ceph-mon[184854]: 11.4 scrub starts 2023-10-09T11:07:16.314 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:16.337 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:16.339 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:07:16.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:16.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:16.366 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:07:16.413 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:07:16.439 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:07:16.440 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:16.467 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:07:16.470 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:16.495 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:16.497 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:07:16.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:16.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:16.520 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:07:16.525 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:07:16.525 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:07:16.525 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:07:16.525 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:07:16.525 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:07:16.525 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:07:16.525 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:07:16.525 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:07:16.592 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T11:07:16.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:16.670 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:16.747 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:16.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:16.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:16.852 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:07:16.879 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:16.897 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:07:16.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:16.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:07:17.001 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:17.035 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:17.074 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:17.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:17.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:17.098 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:17.169 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[195507]: pgmap v1322: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 7298/200424 objects degraded (3.641%); 2427/200424 objects misplaced (1.211%); 13 MiB/s, 201 objects/s recovering 2023-10-09T11:07:17.169 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[195507]: failed to encode map e541 with expected crc 2023-10-09T11:07:17.169 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[195507]: failed to encode map e541 with expected crc 2023-10-09T11:07:17.169 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2778439299' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T11:07:17.169 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[195507]: osdmap e541: 8 total, 8 up, 8 in 2023-10-09T11:07:17.169 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[195507]: failed to encode map e541 with expected crc 2023-10-09T11:07:17.169 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[195507]: failed to encode map e541 with expected crc 2023-10-09T11:07:17.169 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3846013834' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:17.170 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[200197]: pgmap v1322: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 7298/200424 objects degraded (3.641%); 2427/200424 objects misplaced (1.211%); 13 MiB/s, 201 objects/s recovering 2023-10-09T11:07:17.170 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[200197]: failed to encode map e541 with expected crc 2023-10-09T11:07:17.170 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[200197]: failed to encode map e541 with expected crc 2023-10-09T11:07:17.170 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2778439299' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T11:07:17.170 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[200197]: osdmap e541: 8 total, 8 up, 8 in 2023-10-09T11:07:17.170 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[200197]: failed to encode map e541 with expected crc 2023-10-09T11:07:17.170 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[200197]: failed to encode map e541 with expected crc 2023-10-09T11:07:17.170 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:16 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3846013834' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:17.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:17.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:16 smithi155 ceph-mon[184854]: pgmap v1322: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 7298/200424 objects degraded (3.641%); 2427/200424 objects misplaced (1.211%); 13 MiB/s, 201 objects/s recovering 2023-10-09T11:07:17.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:16 smithi155 ceph-mon[184854]: failed to encode map e541 with expected crc 2023-10-09T11:07:17.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:16 smithi155 ceph-mon[184854]: failed to encode map e541 with expected crc 2023-10-09T11:07:17.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:16 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2778439299' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T11:07:17.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:16 smithi155 ceph-mon[184854]: osdmap e541: 8 total, 8 up, 8 in 2023-10-09T11:07:17.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:16 smithi155 ceph-mon[184854]: failed to encode map e541 with expected crc 2023-10-09T11:07:17.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:16 smithi155 ceph-mon[184854]: failed to encode map e541 with expected crc 2023-10-09T11:07:17.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:16 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3846013834' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:17.227 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:17.322 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:17.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:17.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:17.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:17.508 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:17.566 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:17.598 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:17.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:17.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:17.705 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:07:17.738 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:17.766 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:07:17.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:17.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:07:17.871 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:17.957 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:18.037 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:07:18.039 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:18.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:18.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:18.151 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:18.188 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:18.201 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:18 smithi125 ceph-mon[200197]: 11.1d scrub starts 2023-10-09T11:07:18.201 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:18 smithi125 ceph-mon[200197]: failed to encode map e542 with expected crc 2023-10-09T11:07:18.201 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:18 smithi125 ceph-mon[200197]: failed to encode map e542 with expected crc 2023-10-09T11:07:18.201 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:18 smithi125 ceph-mon[200197]: osdmap e542: 8 total, 8 up, 8 in 2023-10-09T11:07:18.201 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:18 smithi125 ceph-mon[200197]: failed to encode map e542 with expected crc 2023-10-09T11:07:18.201 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:18 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3595161907' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:07:18.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:18 smithi125 ceph-mon[195507]: 11.1d scrub starts 2023-10-09T11:07:18.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:18 smithi125 ceph-mon[195507]: failed to encode map e542 with expected crc 2023-10-09T11:07:18.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:18 smithi125 ceph-mon[195507]: failed to encode map e542 with expected crc 2023-10-09T11:07:18.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:18 smithi125 ceph-mon[195507]: osdmap e542: 8 total, 8 up, 8 in 2023-10-09T11:07:18.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:18 smithi125 ceph-mon[195507]: failed to encode map e542 with expected crc 2023-10-09T11:07:18.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:18 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3595161907' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:07:18.216 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:18.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:18.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:18.321 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:07:18.358 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:18 smithi155 ceph-mon[184854]: 11.1d scrub starts 2023-10-09T11:07:18.358 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:18 smithi155 ceph-mon[184854]: failed to encode map e542 with expected crc 2023-10-09T11:07:18.358 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:18 smithi155 ceph-mon[184854]: failed to encode map e542 with expected crc 2023-10-09T11:07:18.358 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:18 smithi155 ceph-mon[184854]: osdmap e542: 8 total, 8 up, 8 in 2023-10-09T11:07:18.358 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:18 smithi155 ceph-mon[184854]: failed to encode map e542 with expected crc 2023-10-09T11:07:18.358 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3595161907' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:07:18.360 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:18.385 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:07:18.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:18.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:07:18.490 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:18.578 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:18.653 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:07:18.654 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:07:18.654 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T11:07:18.654 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T11:07:18.654 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:07:19.168 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:07:19.229 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[195507]: pgmap v1325: 139 pgs: 1 peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 7298/196047 objects degraded (3.723%); 2037/196047 objects misplaced (1.039%); 12 MiB/s, 187 objects/s recovering 2023-10-09T11:07:19.229 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:07:19.229 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[195507]: failed to encode map e543 with expected crc 2023-10-09T11:07:19.230 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[195507]: failed to encode map e543 with expected crc 2023-10-09T11:07:19.230 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3595161907' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:07:19.230 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[195507]: osdmap e543: 8 total, 8 up, 8 in 2023-10-09T11:07:19.230 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[195507]: 11.14 scrub starts 2023-10-09T11:07:19.230 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 7298/196047 objects degraded (3.723%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:07:19.230 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[200197]: pgmap v1325: 139 pgs: 1 peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 7298/196047 objects degraded (3.723%); 2037/196047 objects misplaced (1.039%); 12 MiB/s, 187 objects/s recovering 2023-10-09T11:07:19.230 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:07:19.230 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[200197]: failed to encode map e543 with expected crc 2023-10-09T11:07:19.230 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[200197]: failed to encode map e543 with expected crc 2023-10-09T11:07:19.231 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3595161907' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:07:19.231 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[200197]: osdmap e543: 8 total, 8 up, 8 in 2023-10-09T11:07:19.231 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[200197]: 11.14 scrub starts 2023-10-09T11:07:19.231 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:19 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 7298/196047 objects degraded (3.723%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:07:19.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:19 smithi155 ceph-mon[184854]: pgmap v1325: 139 pgs: 1 peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 7298/196047 objects degraded (3.723%); 2037/196047 objects misplaced (1.039%); 12 MiB/s, 187 objects/s recovering 2023-10-09T11:07:19.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:19 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:07:19.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:19 smithi155 ceph-mon[184854]: failed to encode map e543 with expected crc 2023-10-09T11:07:19.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:19 smithi155 ceph-mon[184854]: failed to encode map e543 with expected crc 2023-10-09T11:07:19.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:19 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3595161907' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:07:19.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:19 smithi155 ceph-mon[184854]: osdmap e543: 8 total, 8 up, 8 in 2023-10-09T11:07:19.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:19 smithi155 ceph-mon[184854]: 11.14 scrub starts 2023-10-09T11:07:19.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:19 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 7298/196047 objects degraded (3.723%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:07:20.173 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:20.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:20.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:20.282 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:20.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:20 smithi125 ceph-mon[195507]: 11.16 deep-scrub starts 2023-10-09T11:07:20.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:20 smithi125 ceph-mon[200197]: 11.16 deep-scrub starts 2023-10-09T11:07:20.338 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:20.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:20 smithi155 ceph-mon[184854]: 11.16 deep-scrub starts 2023-10-09T11:07:20.767 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:20.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:20.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:20.836 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:07:20.913 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:07:20.957 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:20.978 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:07:20.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:20.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:07:21.056 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:21 smithi125 ceph-mon[195507]: 11.6 scrub starts 2023-10-09T11:07:21.056 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:21 smithi125 ceph-mon[195507]: pgmap v1327: 139 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 6257/187563 objects degraded (3.336%); 2198/187563 objects misplaced (1.172%); 12 MiB/s, 192 objects/s recovering 2023-10-09T11:07:21.083 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:21.149 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:21 smithi155 ceph-mon[184854]: 11.6 scrub starts 2023-10-09T11:07:21.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:21 smithi155 ceph-mon[184854]: pgmap v1327: 139 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 6257/187563 objects degraded (3.336%); 2198/187563 objects misplaced (1.172%); 12 MiB/s, 192 objects/s recovering 2023-10-09T11:07:21.324 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:07:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:07:21] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.43.0" 2023-10-09T11:07:21.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:21 smithi125 ceph-mon[200197]: 11.6 scrub starts 2023-10-09T11:07:21.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:21 smithi125 ceph-mon[200197]: pgmap v1327: 139 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 128 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 6257/187563 objects degraded (3.336%); 2198/187563 objects misplaced (1.172%); 12 MiB/s, 192 objects/s recovering 2023-10-09T11:07:21.834 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:21.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:21.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:21.942 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:21.971 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:21.995 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:21.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:21.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:22.099 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:22.135 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:22.158 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:22.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:22.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:22 smithi155 ceph-mon[184854]: osdmap e544: 8 total, 8 up, 8 in 2023-10-09T11:07:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:22 smithi155 ceph-mon[184854]: failed to encode map e544 with expected crc 2023-10-09T11:07:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:22 smithi155 ceph-mon[184854]: failed to encode map e544 with expected crc 2023-10-09T11:07:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:22 smithi155 ceph-mon[184854]: failed to encode map e544 with expected crc 2023-10-09T11:07:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:22 smithi155 ceph-mon[184854]: failed to encode map e544 with expected crc 2023-10-09T11:07:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:22 smithi155 ceph-mon[184854]: failed to encode map e544 with expected crc 2023-10-09T11:07:22.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:22 smithi155 ceph-mon[184854]: 11.4 scrub starts 2023-10-09T11:07:22.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:22 smithi125 ceph-mon[195507]: osdmap e544: 8 total, 8 up, 8 in 2023-10-09T11:07:22.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:22 smithi125 ceph-mon[195507]: failed to encode map e544 with expected crc 2023-10-09T11:07:22.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:22 smithi125 ceph-mon[195507]: failed to encode map e544 with expected crc 2023-10-09T11:07:22.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:22 smithi125 ceph-mon[195507]: failed to encode map e544 with expected crc 2023-10-09T11:07:22.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:22 smithi125 ceph-mon[195507]: failed to encode map e544 with expected crc 2023-10-09T11:07:22.260 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:22 smithi125 ceph-mon[195507]: failed to encode map e544 with expected crc 2023-10-09T11:07:22.260 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:22 smithi125 ceph-mon[195507]: 11.4 scrub starts 2023-10-09T11:07:22.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:22 smithi125 ceph-mon[200197]: osdmap e544: 8 total, 8 up, 8 in 2023-10-09T11:07:22.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:22 smithi125 ceph-mon[200197]: failed to encode map e544 with expected crc 2023-10-09T11:07:22.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:22 smithi125 ceph-mon[200197]: failed to encode map e544 with expected crc 2023-10-09T11:07:22.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:22 smithi125 ceph-mon[200197]: failed to encode map e544 with expected crc 2023-10-09T11:07:22.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:22 smithi125 ceph-mon[200197]: failed to encode map e544 with expected crc 2023-10-09T11:07:22.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:22 smithi125 ceph-mon[200197]: failed to encode map e544 with expected crc 2023-10-09T11:07:22.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:22 smithi125 ceph-mon[200197]: 11.4 scrub starts 2023-10-09T11:07:22.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:22.284 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T11:07:22.300 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:22.328 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:22.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:22.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:22.435 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:22.493 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:22.585 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:22.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:22.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:22.692 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:22.722 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:22.746 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:22.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:22.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:22.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:22.884 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:22.902 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:22.906 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:22.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:22.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:22.950 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:23.012 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:23.045 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:23.050 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:23.068 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:23.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:23.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:23.172 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:23.223 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:23.371 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[195507]: 11.d scrub starts 2023-10-09T11:07:23.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[195507]: pgmap v1329: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 129 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 981 op/s; 6257/183069 objects degraded (3.418%); 2198/183069 objects misplaced (1.201%); 12 MiB/s, 195 objects/s recovering 2023-10-09T11:07:23.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[195507]: failed to encode map e545 with expected crc 2023-10-09T11:07:23.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[195507]: failed to encode map e545 with expected crc 2023-10-09T11:07:23.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[195507]: osdmap e545: 8 total, 8 up, 8 in 2023-10-09T11:07:23.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[195507]: failed to encode map e545 with expected crc 2023-10-09T11:07:23.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[195507]: failed to encode map e545 with expected crc 2023-10-09T11:07:23.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[195507]: 11.14 deep-scrub starts 2023-10-09T11:07:23.372 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[200197]: 11.d scrub starts 2023-10-09T11:07:23.373 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[200197]: pgmap v1329: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 129 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 981 op/s; 6257/183069 objects degraded (3.418%); 2198/183069 objects misplaced (1.201%); 12 MiB/s, 195 objects/s recovering 2023-10-09T11:07:23.373 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[200197]: failed to encode map e545 with expected crc 2023-10-09T11:07:23.373 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[200197]: failed to encode map e545 with expected crc 2023-10-09T11:07:23.373 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[200197]: osdmap e545: 8 total, 8 up, 8 in 2023-10-09T11:07:23.373 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[200197]: failed to encode map e545 with expected crc 2023-10-09T11:07:23.373 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[200197]: failed to encode map e545 with expected crc 2023-10-09T11:07:23.373 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:23 smithi125 ceph-mon[200197]: 11.14 deep-scrub starts 2023-10-09T11:07:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:23 smithi155 ceph-mon[184854]: 11.d scrub starts 2023-10-09T11:07:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:23 smithi155 ceph-mon[184854]: pgmap v1329: 139 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 129 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 981 op/s; 6257/183069 objects degraded (3.418%); 2198/183069 objects misplaced (1.201%); 12 MiB/s, 195 objects/s recovering 2023-10-09T11:07:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:23 smithi155 ceph-mon[184854]: failed to encode map e545 with expected crc 2023-10-09T11:07:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:23 smithi155 ceph-mon[184854]: failed to encode map e545 with expected crc 2023-10-09T11:07:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:23 smithi155 ceph-mon[184854]: osdmap e545: 8 total, 8 up, 8 in 2023-10-09T11:07:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:23 smithi155 ceph-mon[184854]: failed to encode map e545 with expected crc 2023-10-09T11:07:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:23 smithi155 ceph-mon[184854]: failed to encode map e545 with expected crc 2023-10-09T11:07:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:23 smithi155 ceph-mon[184854]: 11.14 deep-scrub starts 2023-10-09T11:07:23.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:23.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:23.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:23.583 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:23.617 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:23.646 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:23.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:23.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:23.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:23.815 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:24.124 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T11:07:24.150 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:24.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:24.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:24.256 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:24.316 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:24.356 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:24 smithi125 ceph-mon[195507]: 11.4 scrub starts 2023-10-09T11:07:24.356 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:24 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2651917408' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:07:24.356 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:24 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:07:24.356 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:24 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 6257/183069 objects degraded (3.418%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:07:24.356 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:24 smithi125 ceph-mon[195507]: 11.1d deep-scrub starts 2023-10-09T11:07:24.356 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:24 smithi125 ceph-mon[200197]: 11.4 scrub starts 2023-10-09T11:07:24.356 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:24 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2651917408' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:07:24.356 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:24 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:07:24.357 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:24 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 6257/183069 objects degraded (3.418%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:07:24.357 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:24 smithi125 ceph-mon[200197]: 11.1d deep-scrub starts 2023-10-09T11:07:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:24 smithi155 ceph-mon[184854]: 11.4 scrub starts 2023-10-09T11:07:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:24 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2651917408' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:07:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:24 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:07:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:24 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 6257/183069 objects degraded (3.418%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:07:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:24 smithi155 ceph-mon[184854]: 11.1d deep-scrub starts 2023-10-09T11:07:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:25 smithi155 ceph-mon[184854]: pgmap v1331: 139 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 131 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.27k op/s; 5171/173082 objects degraded (2.988%); 1875/173082 objects misplaced (1.083%); 20 MiB/s, 331 objects/s recovering 2023-10-09T11:07:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:25 smithi155 ceph-mon[184854]: failed to encode map e546 with expected crc 2023-10-09T11:07:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:25 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T11:07:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:25 smithi155 ceph-mon[184854]: osdmap e546: 8 total, 8 up, 8 in 2023-10-09T11:07:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:25 smithi155 ceph-mon[184854]: failed to encode map e546 with expected crc 2023-10-09T11:07:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:25 smithi155 ceph-mon[184854]: 11.16 scrub starts 2023-10-09T11:07:25.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:25 smithi125 ceph-mon[195507]: pgmap v1331: 139 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 131 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.27k op/s; 5171/173082 objects degraded (2.988%); 1875/173082 objects misplaced (1.083%); 20 MiB/s, 331 objects/s recovering 2023-10-09T11:07:25.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:25 smithi125 ceph-mon[195507]: failed to encode map e546 with expected crc 2023-10-09T11:07:25.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:25 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T11:07:25.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:25 smithi125 ceph-mon[195507]: osdmap e546: 8 total, 8 up, 8 in 2023-10-09T11:07:25.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:25 smithi125 ceph-mon[195507]: failed to encode map e546 with expected crc 2023-10-09T11:07:25.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:25 smithi125 ceph-mon[195507]: 11.16 scrub starts 2023-10-09T11:07:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:25 smithi125 ceph-mon[200197]: pgmap v1331: 139 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 131 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.27k op/s; 5171/173082 objects degraded (2.988%); 1875/173082 objects misplaced (1.083%); 20 MiB/s, 331 objects/s recovering 2023-10-09T11:07:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:25 smithi125 ceph-mon[200197]: failed to encode map e546 with expected crc 2023-10-09T11:07:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:25 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T11:07:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:25 smithi125 ceph-mon[200197]: osdmap e546: 8 total, 8 up, 8 in 2023-10-09T11:07:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:25 smithi125 ceph-mon[200197]: failed to encode map e546 with expected crc 2023-10-09T11:07:25.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:25 smithi125 ceph-mon[200197]: 11.16 scrub starts 2023-10-09T11:07:26.030 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:26.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:26.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:26.135 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:26.177 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:26.202 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:26.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:26.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:26.306 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:07:26.345 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:26.368 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:07:26.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:26.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:07:26.472 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:26.507 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:26.532 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:26.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:26.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:26.638 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:26.675 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:26.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:26.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:26.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:26.804 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:07:26.846 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:26.869 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:07:26.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:26.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:07:26.934 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T11:07:26.973 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:27.001 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:27.020 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:27.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:27.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:27.124 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:27.164 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:27 smithi125 ceph-mon[195507]: pgmap v1333: 139 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 131 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 5171/173082 objects degraded (2.988%); 1875/173082 objects misplaced (1.083%); 12 MiB/s, 204 objects/s recovering 2023-10-09T11:07:27.164 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:27 smithi125 ceph-mon[195507]: 11.14 scrub starts 2023-10-09T11:07:27.164 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:27 smithi125 ceph-mon[200197]: pgmap v1333: 139 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 131 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 5171/173082 objects degraded (2.988%); 1875/173082 objects misplaced (1.083%); 12 MiB/s, 204 objects/s recovering 2023-10-09T11:07:27.164 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:27 smithi125 ceph-mon[200197]: 11.14 scrub starts 2023-10-09T11:07:27.170 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:07:27.170 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T11:07:27.170 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T11:07:27.170 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:07:27.171 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.171 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157305 9.9 9:92e816c0:::benchmark_data_smithi125_363789_object47006:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T11:07:27.171 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:53.128382+0000", 2023-10-09T11:07:27.171 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 454.03650635600002, 2023-10-09T11:07:27.171 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0278687440000001, 2023-10-09T11:07:27.171 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.171 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:07:27.171 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.171 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:07:27.171 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:07:27.171 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157305 2023-10-09T11:07:27.171 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.172 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.172 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.172 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.172 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128382+0000", 2023-10-09T11:07:27.172 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.172 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.172 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.172 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.172 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128382+0000", 2023-10-09T11:07:27.172 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.172 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.172 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.173 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.173 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128381+0000", 2023-10-09T11:07:27.173 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:07:27.173 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.173 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.173 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.173 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128386+0000", 2023-10-09T11:07:27.173 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1699999999999996e-06 2023-10-09T11:07:27.173 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.173 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.173 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.173 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128396+0000", 2023-10-09T11:07:27.174 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0259e-05 2023-10-09T11:07:27.174 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.174 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.174 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.174 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128404+0000", 2023-10-09T11:07:27.174 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.554e-06 2023-10-09T11:07:27.174 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.174 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.174 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.174 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.129279+0000", 2023-10-09T11:07:27.174 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00087499400000000003 2023-10-09T11:07:27.174 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.175 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.175 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.129287+0000", 2023-10-09T11:07:27.175 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4479999999999994e-06 2023-10-09T11:07:27.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.175 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.175 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.156164+0000", 2023-10-09T11:07:27.175 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0268760690000001 2023-10-09T11:07:27.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.176 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.176 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.156251+0000", 2023-10-09T11:07:27.176 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7682000000000005e-05 2023-10-09T11:07:27.176 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.176 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.176 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.176 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.176 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.176 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157332 9.9 9:949c85f3:::benchmark_data_smithi125_363789_object47033:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T11:07:27.176 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:53.128409+0000", 2023-10-09T11:07:27.176 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 454.03647938300003, 2023-10-09T11:07:27.176 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.041208887, 2023-10-09T11:07:27.177 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.177 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.177 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.177 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:07:27.177 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:07:27.177 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157332 2023-10-09T11:07:27.177 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.177 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.177 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.177 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.177 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128409+0000", 2023-10-09T11:07:27.177 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.178 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.178 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.178 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.178 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128409+0000", 2023-10-09T11:07:27.178 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.178 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.178 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.178 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.178 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128409+0000", 2023-10-09T11:07:27.178 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:27.178 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.178 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.179 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.179 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128411+0000", 2023-10-09T11:07:27.179 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0779999999999998e-06 2023-10-09T11:07:27.179 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.179 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.179 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.179 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128416+0000", 2023-10-09T11:07:27.179 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2399999999999998e-06 2023-10-09T11:07:27.179 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.179 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.179 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.179 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128421+0000", 2023-10-09T11:07:27.180 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4909999999999999e-06 2023-10-09T11:07:27.180 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.180 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.180 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.180 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.129306+0000", 2023-10-09T11:07:27.180 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00088470200000000001 2023-10-09T11:07:27.180 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.180 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.180 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.180 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.129311+0000", 2023-10-09T11:07:27.180 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3970000000000002e-06 2023-10-09T11:07:27.180 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.181 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.181 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.181 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.156606+0000", 2023-10-09T11:07:27.181 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.027294844 2023-10-09T11:07:27.181 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.181 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.181 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.181 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.156717+0000", 2023-10-09T11:07:27.181 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111334 2023-10-09T11:07:27.181 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.181 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.181 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,7", 2023-10-09T11:07:27.182 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.156823+0000", 2023-10-09T11:07:27.182 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000105351 2023-10-09T11:07:27.182 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.182 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.182 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.182 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.164715+0000", 2023-10-09T11:07:27.182 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0078925379999999993 2023-10-09T11:07:27.182 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.182 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.182 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.182 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.166304+0000", 2023-10-09T11:07:27.182 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015885669999999999 2023-10-09T11:07:27.183 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.183 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.183 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.183 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.169523+0000", 2023-10-09T11:07:27.183 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032196880000000001 2023-10-09T11:07:27.183 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.183 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.183 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.183 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.169578+0000", 2023-10-09T11:07:27.183 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4339999999999998e-05 2023-10-09T11:07:27.183 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.183 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.184 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.184 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.169618+0000", 2023-10-09T11:07:27.184 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0618999999999999e-05 2023-10-09T11:07:27.184 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.184 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.184 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.184 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.184 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.184 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:51295 10.c 10:349d53bf:::benchmark_data_smithi125_402869_object51294:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T11:07:27.184 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:54.224908+0000", 2023-10-09T11:07:27.184 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 332.939980687, 2023-10-09T11:07:27.184 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0131283310000001, 2023-10-09T11:07:27.185 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.185 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:07:27.185 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.185 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.185 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.185 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 51295 2023-10-09T11:07:27.185 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.185 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.185 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.185 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.185 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.224908+0000", 2023-10-09T11:07:27.185 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.185 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.186 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.186 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.186 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.224908+0000", 2023-10-09T11:07:27.186 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.186 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.186 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.186 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.186 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.224907+0000", 2023-10-09T11:07:27.186 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:07:27.186 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.186 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.187 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.187 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.224943+0000", 2023-10-09T11:07:27.187 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6658e-05 2023-10-09T11:07:27.187 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.187 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.187 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.187 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.224954+0000", 2023-10-09T11:07:27.187 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0533000000000001e-05 2023-10-09T11:07:27.187 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.187 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.187 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.187 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.224962+0000", 2023-10-09T11:07:27.188 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2069999999999995e-06 2023-10-09T11:07:27.188 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.188 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.188 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.188 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.225016+0000", 2023-10-09T11:07:27.188 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4055999999999999e-05 2023-10-09T11:07:27.188 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.188 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.188 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.188 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.225130+0000", 2023-10-09T11:07:27.188 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000113702 2023-10-09T11:07:27.188 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.189 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.189 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T11:07:27.189 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.225260+0000", 2023-10-09T11:07:27.189 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000129834 2023-10-09T11:07:27.189 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.189 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.189 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.189 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.229508+0000", 2023-10-09T11:07:27.189 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042478300000000002 2023-10-09T11:07:27.189 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.189 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.189 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.190 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.239287+0000", 2023-10-09T11:07:27.190 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0097793959999999992 2023-10-09T11:07:27.190 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.190 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.190 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.190 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.239301+0000", 2023-10-09T11:07:27.190 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3771000000000001e-05 2023-10-09T11:07:27.190 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.190 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.190 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.190 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.237967+0000", 2023-10-09T11:07:27.190 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99866647399999997 2023-10-09T11:07:27.191 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.191 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.191 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.191 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.238036+0000", 2023-10-09T11:07:27.191 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9280000000000001e-05 2023-10-09T11:07:27.191 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.191 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.191 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.191 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.191 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.191 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:51299 10.c 10:39b4c66a:::benchmark_data_smithi125_402869_object51298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T11:07:27.192 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:54.232859+0000", 2023-10-09T11:07:27.192 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 332.93203006300001, 2023-10-09T11:07:27.192 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0181773119999999, 2023-10-09T11:07:27.192 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.192 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.192 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.192 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.192 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.192 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 51299 2023-10-09T11:07:27.192 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.192 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.192 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.192 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.193 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.232859+0000", 2023-10-09T11:07:27.193 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.193 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.193 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.193 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.193 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.232859+0000", 2023-10-09T11:07:27.193 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.193 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.193 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.193 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.193 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.232858+0000", 2023-10-09T11:07:27.193 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:27.194 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.194 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.194 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.194 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.232880+0000", 2023-10-09T11:07:27.194 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1630000000000001e-05 2023-10-09T11:07:27.194 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.194 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.194 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.194 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.232888+0000", 2023-10-09T11:07:27.194 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9459999999999998e-06 2023-10-09T11:07:27.194 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.195 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.195 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.232894+0000", 2023-10-09T11:07:27.195 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0240000000000003e-06 2023-10-09T11:07:27.195 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.195 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.195 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.239349+0000", 2023-10-09T11:07:27.195 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0064550709999999997 2023-10-09T11:07:27.195 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.196 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.239360+0000", 2023-10-09T11:07:27.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0786e-05 2023-10-09T11:07:27.196 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.196 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.196 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.238058+0000", 2023-10-09T11:07:27.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99869864600000002 2023-10-09T11:07:27.196 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.196 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.196 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.238183+0000", 2023-10-09T11:07:27.197 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000124444 2023-10-09T11:07:27.197 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.197 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.197 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T11:07:27.197 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.238332+0000", 2023-10-09T11:07:27.197 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000148734 2023-10-09T11:07:27.197 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.197 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.197 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.197 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.242271+0000", 2023-10-09T11:07:27.197 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039395869999999996 2023-10-09T11:07:27.198 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.198 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.198 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.198 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.247098+0000", 2023-10-09T11:07:27.198 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0048267379999999997 2023-10-09T11:07:27.198 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.198 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.198 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.198 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.250972+0000", 2023-10-09T11:07:27.198 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003874204 2023-10-09T11:07:27.198 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.198 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.251011+0000", 2023-10-09T11:07:27.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9406000000000002e-05 2023-10-09T11:07:27.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.199 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.251036+0000", 2023-10-09T11:07:27.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4553999999999999e-05 2023-10-09T11:07:27.199 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.199 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.199 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.200 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.200 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:51302 10.c 10:3c52ba0a:::benchmark_data_smithi125_402869_object51301:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T11:07:27.200 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:54.233473+0000", 2023-10-09T11:07:27.200 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 332.93141593000001, 2023-10-09T11:07:27.200 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0177018390000001, 2023-10-09T11:07:27.200 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.200 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.200 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.200 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.200 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.200 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 51302 2023-10-09T11:07:27.200 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.201 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.201 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.201 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.201 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.233473+0000", 2023-10-09T11:07:27.201 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.201 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.201 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.201 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.201 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.233473+0000", 2023-10-09T11:07:27.201 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.201 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.201 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.201 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.202 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.233472+0000", 2023-10-09T11:07:27.202 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:27.202 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.202 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.202 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.202 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.233495+0000", 2023-10-09T11:07:27.202 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2628000000000001e-05 2023-10-09T11:07:27.202 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.202 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.202 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.202 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.233502+0000", 2023-10-09T11:07:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2690000000000002e-06 2023-10-09T11:07:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.233509+0000", 2023-10-09T11:07:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2049999999999996e-06 2023-10-09T11:07:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.239417+0000", 2023-10-09T11:07:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.005907537 2023-10-09T11:07:27.203 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.239425+0000", 2023-10-09T11:07:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4460000000000001e-06 2023-10-09T11:07:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.240118+0000", 2023-10-09T11:07:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0006926060000001 2023-10-09T11:07:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.204 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.240257+0000", 2023-10-09T11:07:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013881399999999999 2023-10-09T11:07:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T11:07:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.240390+0000", 2023-10-09T11:07:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013303999999999999 2023-10-09T11:07:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.242304+0000", 2023-10-09T11:07:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019137710000000001 2023-10-09T11:07:27.205 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.206 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.206 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.206 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.251044+0000", 2023-10-09T11:07:27.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0087401330000000006 2023-10-09T11:07:27.206 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.206 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.206 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.206 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.251095+0000", 2023-10-09T11:07:27.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0915000000000003e-05 2023-10-09T11:07:27.206 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.206 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.206 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.207 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.251137+0000", 2023-10-09T11:07:27.207 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2345000000000002e-05 2023-10-09T11:07:27.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.207 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.251175+0000", 2023-10-09T11:07:27.207 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7639999999999999e-05 2023-10-09T11:07:27.207 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.207 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.207 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.208 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:64757 10.1 10:823ba87e:::benchmark_data_smithi125_402869_object64756:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-10-09T11:07:27.208 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:07.554401+0000", 2023-10-09T11:07:27.208 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 319.61048774599999, 2023-10-09T11:07:27.208 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.020742526, 2023-10-09T11:07:27.208 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.208 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:07:27.208 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.208 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.208 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.208 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 64757 2023-10-09T11:07:27.208 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.208 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.208 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.209 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.209 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554401+0000", 2023-10-09T11:07:27.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.209 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.209 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.209 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554401+0000", 2023-10-09T11:07:27.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.209 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.209 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.209 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554400+0000", 2023-10-09T11:07:27.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:07:27.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.210 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554441+0000", 2023-10-09T11:07:27.210 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0886999999999997e-05 2023-10-09T11:07:27.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.210 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554452+0000", 2023-10-09T11:07:27.210 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1292e-05 2023-10-09T11:07:27.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.211 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.211 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554459+0000", 2023-10-09T11:07:27.211 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9199999999999998e-06 2023-10-09T11:07:27.211 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.211 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.211 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.558760+0000", 2023-10-09T11:07:27.211 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043012119999999996 2023-10-09T11:07:27.211 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.211 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.211 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.558901+0000", 2023-10-09T11:07:27.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014095199999999999 2023-10-09T11:07:27.212 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.212 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.212 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:07:27.212 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.559060+0000", 2023-10-09T11:07:27.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015870399999999999 2023-10-09T11:07:27.212 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.212 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.212 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.212 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.564296+0000", 2023-10-09T11:07:27.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0052360820000000004 2023-10-09T11:07:27.212 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.564325+0000", 2023-10-09T11:07:27.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9091999999999999e-05 2023-10-09T11:07:27.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.568712+0000", 2023-10-09T11:07:27.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043873230000000003 2023-10-09T11:07:27.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.568726+0000", 2023-10-09T11:07:27.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4224e-05 2023-10-09T11:07:27.214 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.214 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.214 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:08.575077+0000", 2023-10-09T11:07:27.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.006350866 2023-10-09T11:07:27.214 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.214 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.214 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:08.575144+0000", 2023-10-09T11:07:27.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6302000000000003e-05 2023-10-09T11:07:27.215 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.215 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.215 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.215 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:64759 10.1 10:8e3eeb45:::benchmark_data_smithi125_402869_object64758:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-10-09T11:07:27.215 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:07.554465+0000", 2023-10-09T11:07:27.215 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 319.610423809, 2023-10-09T11:07:27.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0210216519999999, 2023-10-09T11:07:27.215 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.215 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:07:27.215 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.215 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.216 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.216 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 64759 2023-10-09T11:07:27.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.216 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554465+0000", 2023-10-09T11:07:27.216 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554465+0000", 2023-10-09T11:07:27.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554465+0000", 2023-10-09T11:07:27.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:27.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554520+0000", 2023-10-09T11:07:27.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5247000000000001e-05 2023-10-09T11:07:27.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.218 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554528+0000", 2023-10-09T11:07:27.218 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8480000000000001e-06 2023-10-09T11:07:27.218 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.218 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554534+0000", 2023-10-09T11:07:27.218 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9399999999999999e-06 2023-10-09T11:07:27.218 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.561310+0000", 2023-10-09T11:07:27.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0067761460000000003 2023-10-09T11:07:27.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.219 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.561472+0000", 2023-10-09T11:07:27.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016197600000000001 2023-10-09T11:07:27.219 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.219 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:07:27.219 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.561638+0000", 2023-10-09T11:07:27.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000166341 2023-10-09T11:07:27.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.220 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.564331+0000", 2023-10-09T11:07:27.220 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026929190000000002 2023-10-09T11:07:27.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.220 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.569105+0000", 2023-10-09T11:07:27.220 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0047744650000000003 2023-10-09T11:07:27.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.569117+0000", 2023-10-09T11:07:27.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1493e-05 2023-10-09T11:07:27.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.221 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:08.575439+0000", 2023-10-09T11:07:27.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0063218190000001 2023-10-09T11:07:27.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.221 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:08.575487+0000", 2023-10-09T11:07:27.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7911999999999998e-05 2023-10-09T11:07:27.221 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.222 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.222 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.222 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87922 10.c 10:3a18da39:::benchmark_data_smithi125_402869_object87921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:07:27.222 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178226+0000", 2023-10-09T11:07:27.222 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 293.98666261599999, 2023-10-09T11:07:27.222 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1423559400000001, 2023-10-09T11:07:27.222 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.222 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.222 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.222 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.223 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.223 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87922 2023-10-09T11:07:27.223 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.223 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.223 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.223 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.223 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178226+0000", 2023-10-09T11:07:27.223 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.223 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.223 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.223 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.224 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178226+0000", 2023-10-09T11:07:27.224 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.224 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.224 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.224 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.224 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178225+0000", 2023-10-09T11:07:27.224 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:07:27.224 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.224 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.224 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.224 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178265+0000", 2023-10-09T11:07:27.224 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0617000000000001e-05 2023-10-09T11:07:27.224 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178278+0000", 2023-10-09T11:07:27.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2721999999999999e-05 2023-10-09T11:07:27.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178289+0000", 2023-10-09T11:07:27.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0787e-05 2023-10-09T11:07:27.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178908+0000", 2023-10-09T11:07:27.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00061957899999999998 2023-10-09T11:07:27.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178919+0000", 2023-10-09T11:07:27.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0808999999999999e-05 2023-10-09T11:07:27.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.306310+0000", 2023-10-09T11:07:27.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.12739133 2023-10-09T11:07:27.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.306501+0000", 2023-10-09T11:07:27.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000190779 2023-10-09T11:07:27.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,7", 2023-10-09T11:07:27.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.306691+0000", 2023-10-09T11:07:27.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000189328 2023-10-09T11:07:27.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316145+0000", 2023-10-09T11:07:27.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0094546479999999995 2023-10-09T11:07:27.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.228 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.228 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316181+0000", 2023-10-09T11:07:27.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5404000000000002e-05 2023-10-09T11:07:27.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.228 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.228 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316319+0000", 2023-10-09T11:07:27.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013836499999999999 2023-10-09T11:07:27.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320404+0000", 2023-10-09T11:07:27.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040849550000000004 2023-10-09T11:07:27.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320459+0000", 2023-10-09T11:07:27.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5013000000000003e-05 2023-10-09T11:07:27.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.229 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.229 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.229 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320582+0000", 2023-10-09T11:07:27.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012316000000000001 2023-10-09T11:07:27.230 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.230 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.230 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.230 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87923 10.1 10:84c65cae:::benchmark_data_smithi125_402869_object87922:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:07:27.230 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178299+0000", 2023-10-09T11:07:27.230 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 293.98658945800003, 2023-10-09T11:07:27.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.208577113, 2023-10-09T11:07:27.230 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.230 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.230 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.231 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.231 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.231 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87923 2023-10-09T11:07:27.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.231 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178299+0000", 2023-10-09T11:07:27.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178299+0000", 2023-10-09T11:07:27.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178299+0000", 2023-10-09T11:07:27.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:27.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178336+0000", 2023-10-09T11:07:27.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7317999999999997e-05 2023-10-09T11:07:27.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178344+0000", 2023-10-09T11:07:27.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3289999999999994e-06 2023-10-09T11:07:27.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178351+0000", 2023-10-09T11:07:27.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4570000000000002e-06 2023-10-09T11:07:27.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178912+0000", 2023-10-09T11:07:27.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00056114399999999997 2023-10-09T11:07:27.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178923+0000", 2023-10-09T11:07:27.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0679000000000001e-05 2023-10-09T11:07:27.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.378720+0000", 2023-10-09T11:07:27.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1997972379999999 2023-10-09T11:07:27.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.378826+0000", 2023-10-09T11:07:27.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010621200000000001 2023-10-09T11:07:27.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T11:07:27.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.378928+0000", 2023-10-09T11:07:27.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000101439 2023-10-09T11:07:27.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386610+0000", 2023-10-09T11:07:27.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076823810000000003 2023-10-09T11:07:27.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386650+0000", 2023-10-09T11:07:27.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0089999999999997e-05 2023-10-09T11:07:27.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.236 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386697+0000", 2023-10-09T11:07:27.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7126999999999998e-05 2023-10-09T11:07:27.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386761+0000", 2023-10-09T11:07:27.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3379000000000004e-05 2023-10-09T11:07:27.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386796+0000", 2023-10-09T11:07:27.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5688000000000001e-05 2023-10-09T11:07:27.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386876+0000", 2023-10-09T11:07:27.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0235999999999995e-05 2023-10-09T11:07:27.238 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.238 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.238 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.238 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87926 10.1 10:895d4d32:::benchmark_data_smithi125_402869_object87925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:07:27.238 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178357+0000", 2023-10-09T11:07:27.238 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 293.98653154599998, 2023-10-09T11:07:27.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.20874682, 2023-10-09T11:07:27.238 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.239 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.239 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.239 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.239 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.239 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87926 2023-10-09T11:07:27.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.239 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178357+0000", 2023-10-09T11:07:27.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178357+0000", 2023-10-09T11:07:27.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178357+0000", 2023-10-09T11:07:27.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:27.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178411+0000", 2023-10-09T11:07:27.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4347e-05 2023-10-09T11:07:27.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178419+0000", 2023-10-09T11:07:27.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4209999999999995e-06 2023-10-09T11:07:27.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178426+0000", 2023-10-09T11:07:27.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7150000000000001e-06 2023-10-09T11:07:27.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178946+0000", 2023-10-09T11:07:27.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000519769 2023-10-09T11:07:27.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178954+0000", 2023-10-09T11:07:27.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1110000000000008e-06 2023-10-09T11:07:27.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.381425+0000", 2023-10-09T11:07:27.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.20247094 2023-10-09T11:07:27.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.381541+0000", 2023-10-09T11:07:27.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000116335 2023-10-09T11:07:27.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.243 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T11:07:27.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.381643+0000", 2023-10-09T11:07:27.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010179900000000001 2023-10-09T11:07:27.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.244 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386707+0000", 2023-10-09T11:07:27.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0050636739999999998 2023-10-09T11:07:27.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.244 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.244 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386726+0000", 2023-10-09T11:07:27.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9184000000000001e-05 2023-10-09T11:07:27.244 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.245 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.245 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.245 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386917+0000", 2023-10-09T11:07:27.245 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019101099999999999 2023-10-09T11:07:27.245 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.245 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.245 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.245 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386955+0000", 2023-10-09T11:07:27.245 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8059000000000003e-05 2023-10-09T11:07:27.245 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.245 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.245 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.246 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387002+0000", 2023-10-09T11:07:27.246 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7280999999999997e-05 2023-10-09T11:07:27.246 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.246 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.246 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387104+0000", 2023-10-09T11:07:27.246 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000101885 2023-10-09T11:07:27.246 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.246 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.246 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.246 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.247 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87932 10.c 10:32331c82:::benchmark_data_smithi125_402869_object87931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:07:27.247 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178432+0000", 2023-10-09T11:07:27.247 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 293.98645723700002, 2023-10-09T11:07:27.247 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1426129899999999, 2023-10-09T11:07:27.247 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.247 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.247 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.247 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.247 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.247 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87932 2023-10-09T11:07:27.247 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.247 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178432+0000", 2023-10-09T11:07:27.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178432+0000", 2023-10-09T11:07:27.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178431+0000", 2023-10-09T11:07:27.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:27.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178498+0000", 2023-10-09T11:07:27.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6989000000000005e-05 2023-10-09T11:07:27.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178507+0000", 2023-10-09T11:07:27.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6000000000000007e-06 2023-10-09T11:07:27.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178514+0000", 2023-10-09T11:07:27.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9029999999999998e-06 2023-10-09T11:07:27.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178941+0000", 2023-10-09T11:07:27.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000427873 2023-10-09T11:07:27.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178949+0000", 2023-10-09T11:07:27.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8080000000000005e-06 2023-10-09T11:07:27.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.309831+0000", 2023-10-09T11:07:27.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1308817790000001 2023-10-09T11:07:27.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.310021+0000", 2023-10-09T11:07:27.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019044099999999999 2023-10-09T11:07:27.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,7", 2023-10-09T11:07:27.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.310197+0000", 2023-10-09T11:07:27.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000175231 2023-10-09T11:07:27.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316241+0000", 2023-10-09T11:07:27.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.006044715 2023-10-09T11:07:27.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.253 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.253 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316249+0000", 2023-10-09T11:07:27.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7030000000000006e-06 2023-10-09T11:07:27.253 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.253 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.253 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.253 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320823+0000", 2023-10-09T11:07:27.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004573433 2023-10-09T11:07:27.253 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.253 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.253 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.253 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320877+0000", 2023-10-09T11:07:27.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4082000000000002e-05 2023-10-09T11:07:27.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.254 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.254 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.254 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320939+0000", 2023-10-09T11:07:27.254 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2507000000000003e-05 2023-10-09T11:07:27.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.254 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.254 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.254 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.321045+0000", 2023-10-09T11:07:27.254 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010545 2023-10-09T11:07:27.254 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.254 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.255 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.255 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.255 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87933 10.1 10:8333223a:::benchmark_data_smithi125_402869_object87932:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:07:27.255 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178522+0000", 2023-10-09T11:07:27.255 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 293.98636705799998, 2023-10-09T11:07:27.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2145040599999999, 2023-10-09T11:07:27.255 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.255 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.255 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.255 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.255 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.256 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87933 2023-10-09T11:07:27.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.256 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178522+0000", 2023-10-09T11:07:27.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178522+0000", 2023-10-09T11:07:27.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178521+0000", 2023-10-09T11:07:27.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:27.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178569+0000", 2023-10-09T11:07:27.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7472999999999998e-05 2023-10-09T11:07:27.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178577+0000", 2023-10-09T11:07:27.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1839999999999999e-06 2023-10-09T11:07:27.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178583+0000", 2023-10-09T11:07:27.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1e-06 2023-10-09T11:07:27.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178976+0000", 2023-10-09T11:07:27.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000393216 2023-10-09T11:07:27.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178984+0000", 2023-10-09T11:07:27.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0199999999999994e-06 2023-10-09T11:07:27.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.383925+0000", 2023-10-09T11:07:27.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2049411139999999 2023-10-09T11:07:27.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.384095+0000", 2023-10-09T11:07:27.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016973699999999999 2023-10-09T11:07:27.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T11:07:27.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.384262+0000", 2023-10-09T11:07:27.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000166964 2023-10-09T11:07:27.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387118+0000", 2023-10-09T11:07:27.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028561709999999998 2023-10-09T11:07:27.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.261 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387158+0000", 2023-10-09T11:07:27.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9368e-05 2023-10-09T11:07:27.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.261 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387200+0000", 2023-10-09T11:07:27.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2500999999999998e-05 2023-10-09T11:07:27.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.261 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.262 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.392880+0000", 2023-10-09T11:07:27.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0056800849999999996 2023-10-09T11:07:27.262 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.262 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.262 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.262 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.392916+0000", 2023-10-09T11:07:27.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6239999999999999e-05 2023-10-09T11:07:27.262 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.262 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.262 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.262 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393026+0000", 2023-10-09T11:07:27.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010939500000000001 2023-10-09T11:07:27.263 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.263 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.263 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.263 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87943 10.1 10:802b3a31:::benchmark_data_smithi125_402869_object87942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:07:27.263 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178589+0000", 2023-10-09T11:07:27.263 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 293.98630030200002, 2023-10-09T11:07:27.263 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.214764771, 2023-10-09T11:07:27.263 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.263 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.263 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.264 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.264 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.264 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87943 2023-10-09T11:07:27.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.264 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178589+0000", 2023-10-09T11:07:27.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178589+0000", 2023-10-09T11:07:27.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178588+0000", 2023-10-09T11:07:27.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:27.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178633+0000", 2023-10-09T11:07:27.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4960000000000003e-05 2023-10-09T11:07:27.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178641+0000", 2023-10-09T11:07:27.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8359999999999992e-06 2023-10-09T11:07:27.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178648+0000", 2023-10-09T11:07:27.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6750000000000002e-06 2023-10-09T11:07:27.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.179006+0000", 2023-10-09T11:07:27.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035727099999999999 2023-10-09T11:07:27.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.179014+0000", 2023-10-09T11:07:27.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9580000000000007e-06 2023-10-09T11:07:27.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387221+0000", 2023-10-09T11:07:27.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2082074030000001 2023-10-09T11:07:27.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387349+0000", 2023-10-09T11:07:27.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012751599999999999 2023-10-09T11:07:27.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T11:07:27.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387477+0000", 2023-10-09T11:07:27.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012794799999999999 2023-10-09T11:07:27.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393068+0000", 2023-10-09T11:07:27.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0055912039999999998 2023-10-09T11:07:27.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393078+0000", 2023-10-09T11:07:27.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0084e-05 2023-10-09T11:07:27.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393133+0000", 2023-10-09T11:07:27.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4762999999999997e-05 2023-10-09T11:07:27.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393179+0000", 2023-10-09T11:07:27.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6622999999999997e-05 2023-10-09T11:07:27.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393248+0000", 2023-10-09T11:07:27.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9108000000000003e-05 2023-10-09T11:07:27.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.271 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.271 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393353+0000", 2023-10-09T11:07:27.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000104959 2023-10-09T11:07:27.271 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.271 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.271 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.271 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87970 10.c 10:397744e6:::benchmark_data_smithi125_402869_object87969:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:07:27.272 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178654+0000", 2023-10-09T11:07:27.272 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 293.98623469299997, 2023-10-09T11:07:27.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2299234779999999, 2023-10-09T11:07:27.272 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.272 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.272 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.272 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.272 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.272 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87970 2023-10-09T11:07:27.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.272 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.272 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178654+0000", 2023-10-09T11:07:27.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178654+0000", 2023-10-09T11:07:27.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178654+0000", 2023-10-09T11:07:27.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:27.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178698+0000", 2023-10-09T11:07:27.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4047e-05 2023-10-09T11:07:27.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178706+0000", 2023-10-09T11:07:27.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8619999999999992e-06 2023-10-09T11:07:27.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178713+0000", 2023-10-09T11:07:27.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.46e-06 2023-10-09T11:07:27.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178969+0000", 2023-10-09T11:07:27.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000256562 2023-10-09T11:07:27.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178977+0000", 2023-10-09T11:07:27.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.204e-06 2023-10-09T11:07:27.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.313409+0000", 2023-10-09T11:07:27.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1344324960000001 2023-10-09T11:07:27.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.313530+0000", 2023-10-09T11:07:27.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000121099 2023-10-09T11:07:27.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,7", 2023-10-09T11:07:27.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.313651+0000", 2023-10-09T11:07:27.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120791 2023-10-09T11:07:27.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320352+0000", 2023-10-09T11:07:27.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0067007380000000004 2023-10-09T11:07:27.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320631+0000", 2023-10-09T11:07:27.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027918300000000001 2023-10-09T11:07:27.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.321145+0000", 2023-10-09T11:07:27.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051379300000000002 2023-10-09T11:07:27.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.408399+0000", 2023-10-09T11:07:27.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.087254527999999998 2023-10-09T11:07:27.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.408459+0000", 2023-10-09T11:07:27.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9611000000000002e-05 2023-10-09T11:07:27.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.408578+0000", 2023-10-09T11:07:27.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118651 2023-10-09T11:07:27.279 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.279 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.279 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.280 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87980 10.c 10:372347b9:::benchmark_data_smithi125_402869_object87979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:07:27.280 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178720+0000", 2023-10-09T11:07:27.280 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 293.98616904099998, 2023-10-09T11:07:27.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2300607610000001, 2023-10-09T11:07:27.280 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.280 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.280 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.280 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.280 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.280 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87980 2023-10-09T11:07:27.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.281 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178720+0000", 2023-10-09T11:07:27.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.281 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.281 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178720+0000", 2023-10-09T11:07:27.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178719+0000", 2023-10-09T11:07:27.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:27.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178763+0000", 2023-10-09T11:07:27.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3375999999999999e-05 2023-10-09T11:07:27.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178771+0000", 2023-10-09T11:07:27.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0530000000000007e-06 2023-10-09T11:07:27.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178777+0000", 2023-10-09T11:07:27.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3620000000000002e-06 2023-10-09T11:07:27.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178996+0000", 2023-10-09T11:07:27.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00021923799999999999 2023-10-09T11:07:27.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.179003+0000", 2023-10-09T11:07:27.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9890000000000003e-06 2023-10-09T11:07:27.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316530+0000", 2023-10-09T11:07:27.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.137527073 2023-10-09T11:07:27.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316665+0000", 2023-10-09T11:07:27.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013503899999999999 2023-10-09T11:07:27.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,7", 2023-10-09T11:07:27.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316850+0000", 2023-10-09T11:07:27.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000185062 2023-10-09T11:07:27.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.321062+0000", 2023-10-09T11:07:27.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042115169999999997 2023-10-09T11:07:27.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.321100+0000", 2023-10-09T11:07:27.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7601000000000001e-05 2023-10-09T11:07:27.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.321193+0000", 2023-10-09T11:07:27.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3624999999999999e-05 2023-10-09T11:07:27.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.408637+0000", 2023-10-09T11:07:27.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.087443641000000003 2023-10-09T11:07:27.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.408684+0000", 2023-10-09T11:07:27.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7571999999999998e-05 2023-10-09T11:07:27.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.408781+0000", 2023-10-09T11:07:27.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6157999999999997e-05 2023-10-09T11:07:27.288 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.288 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.288 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.288 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87987 10.1 10:834eb049:::benchmark_data_smithi125_402869_object87986:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:07:27.288 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178783+0000", 2023-10-09T11:07:27.288 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 293.98610596200001, 2023-10-09T11:07:27.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2178343309999999, 2023-10-09T11:07:27.288 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.288 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.288 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.289 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.289 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.289 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87987 2023-10-09T11:07:27.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.289 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178783+0000", 2023-10-09T11:07:27.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178783+0000", 2023-10-09T11:07:27.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178782+0000", 2023-10-09T11:07:27.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:27.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178824+0000", 2023-10-09T11:07:27.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2045999999999997e-05 2023-10-09T11:07:27.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178832+0000", 2023-10-09T11:07:27.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6930000000000007e-06 2023-10-09T11:07:27.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.291 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.291 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178838+0000", 2023-10-09T11:07:27.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0929999999999999e-06 2023-10-09T11:07:27.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.291 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.179035+0000", 2023-10-09T11:07:27.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000197152 2023-10-09T11:07:27.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.179043+0000", 2023-10-09T11:07:27.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9300000000000003e-06 2023-10-09T11:07:27.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.390120+0000", 2023-10-09T11:07:27.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.211076751 2023-10-09T11:07:27.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.390256+0000", 2023-10-09T11:07:27.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013596300000000001 2023-10-09T11:07:27.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T11:07:27.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.390361+0000", 2023-10-09T11:07:27.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000104608 2023-10-09T11:07:27.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393371+0000", 2023-10-09T11:07:27.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030100359999999998 2023-10-09T11:07:27.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393424+0000", 2023-10-09T11:07:27.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3146000000000002e-05 2023-10-09T11:07:27.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.396331+0000", 2023-10-09T11:07:27.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029073200000000001 2023-10-09T11:07:27.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.396445+0000", 2023-10-09T11:07:27.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011435 2023-10-09T11:07:27.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.396501+0000", 2023-10-09T11:07:27.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5992000000000002e-05 2023-10-09T11:07:27.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.396617+0000", 2023-10-09T11:07:27.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000115742 2023-10-09T11:07:27.296 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.296 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.297 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.297 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:98555 10.11 10:8b1dad8d:::benchmark_data_smithi125_402869_object98554:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e443)", 2023-10-09T11:07:27.297 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:50.335541+0000", 2023-10-09T11:07:27.297 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 276.82934736200002, 2023-10-09T11:07:27.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0107502129999999, 2023-10-09T11:07:27.297 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.297 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.297 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.297 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.298 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.298 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 98555 2023-10-09T11:07:27.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.298 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335541+0000", 2023-10-09T11:07:27.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335541+0000", 2023-10-09T11:07:27.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335540+0000", 2023-10-09T11:07:27.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:07:27.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335565+0000", 2023-10-09T11:07:27.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4861000000000001e-05 2023-10-09T11:07:27.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335578+0000", 2023-10-09T11:07:27.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3101e-05 2023-10-09T11:07:27.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.300 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.300 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335588+0000", 2023-10-09T11:07:27.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0098000000000001e-05 2023-10-09T11:07:27.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336134+0000", 2023-10-09T11:07:27.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00054624999999999997 2023-10-09T11:07:27.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336146+0000", 2023-10-09T11:07:27.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1460999999999999e-05 2023-10-09T11:07:27.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.341862+0000", 2023-10-09T11:07:27.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.005716211 2023-10-09T11:07:27.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.341988+0000", 2023-10-09T11:07:27.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012611400000000001 2023-10-09T11:07:27.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T11:07:27.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.342191+0000", 2023-10-09T11:07:27.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020272700000000001 2023-10-09T11:07:27.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.344401+0000", 2023-10-09T11:07:27.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022105839999999998 2023-10-09T11:07:27.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.344426+0000", 2023-10-09T11:07:27.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5215000000000002e-05 2023-10-09T11:07:27.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.345867+0000", 2023-10-09T11:07:27.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001441035 2023-10-09T11:07:27.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.346178+0000", 2023-10-09T11:07:27.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000310475 2023-10-09T11:07:27.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.346219+0000", 2023-10-09T11:07:27.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1458000000000002e-05 2023-10-09T11:07:27.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.346292+0000", 2023-10-09T11:07:27.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2249000000000006e-05 2023-10-09T11:07:27.305 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.305 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.305 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.305 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:98558 10.11 10:89be634c:::benchmark_data_smithi125_402869_object98557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e443)", 2023-10-09T11:07:27.305 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:50.336104+0000", 2023-10-09T11:07:27.305 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 276.82878484899999, 2023-10-09T11:07:27.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0103175630000001, 2023-10-09T11:07:27.306 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.306 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.306 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.306 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.306 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.306 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 98558 2023-10-09T11:07:27.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.306 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336104+0000", 2023-10-09T11:07:27.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336104+0000", 2023-10-09T11:07:27.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336103+0000", 2023-10-09T11:07:27.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:27.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336129+0000", 2023-10-09T11:07:27.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5403000000000001e-05 2023-10-09T11:07:27.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336140+0000", 2023-10-09T11:07:27.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1060000000000001e-05 2023-10-09T11:07:27.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336149+0000", 2023-10-09T11:07:27.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8810000000000002e-06 2023-10-09T11:07:27.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336186+0000", 2023-10-09T11:07:27.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6980999999999997e-05 2023-10-09T11:07:27.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336192+0000", 2023-10-09T11:07:27.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7560000000000002e-06 2023-10-09T11:07:27.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.343404+0000", 2023-10-09T11:07:27.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0072118370000001 2023-10-09T11:07:27.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.310 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.310 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.343498+0000", 2023-10-09T11:07:27.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3412000000000006e-05 2023-10-09T11:07:27.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T11:07:27.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.343638+0000", 2023-10-09T11:07:27.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014086200000000001 2023-10-09T11:07:27.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.344550+0000", 2023-10-09T11:07:27.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00091130400000000004 2023-10-09T11:07:27.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.344753+0000", 2023-10-09T11:07:27.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000203329 2023-10-09T11:07:27.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.346008+0000", 2023-10-09T11:07:27.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012553180000000001 2023-10-09T11:07:27.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.346322+0000", 2023-10-09T11:07:27.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00031327699999999999 2023-10-09T11:07:27.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.346356+0000", 2023-10-09T11:07:27.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3915999999999999e-05 2023-10-09T11:07:27.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.346422+0000", 2023-10-09T11:07:27.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5908999999999995e-05 2023-10-09T11:07:27.313 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.313 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.313 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.314 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:179298 10.4 10:22bc55cd:::benchmark_data_smithi125_402869_object80729:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-10-09T11:07:27.314 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:04:22.139061+0000", 2023-10-09T11:07:27.314 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 185.025827652, 2023-10-09T11:07:27.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96191459000000001, 2023-10-09T11:07:27.314 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.314 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.314 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:27.314 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:07:27.314 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:07:27.314 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 179298 2023-10-09T11:07:27.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.315 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139061+0000", 2023-10-09T11:07:27.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139061+0000", 2023-10-09T11:07:27.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139061+0000", 2023-10-09T11:07:27.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:27.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139064+0000", 2023-10-09T11:07:27.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5810000000000001e-06 2023-10-09T11:07:27.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139072+0000", 2023-10-09T11:07:27.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0299999999999994e-06 2023-10-09T11:07:27.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139079+0000", 2023-10-09T11:07:27.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3550000000000001e-06 2023-10-09T11:07:27.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139322+0000", 2023-10-09T11:07:27.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00024333699999999999 2023-10-09T11:07:27.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:27.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139330+0000", 2023-10-09T11:07:27.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1629999999999996e-06 2023-10-09T11:07:27.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.094275+0000", 2023-10-09T11:07:27.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95494484800000001 2023-10-09T11:07:27.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.094367+0000", 2023-10-09T11:07:27.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1500000000000001e-05 2023-10-09T11:07:27.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:07:27.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.094649+0000", 2023-10-09T11:07:27.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00028209399999999999 2023-10-09T11:07:27.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:27.319 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.100786+0000", 2023-10-09T11:07:27.319 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0061369880000000003 2023-10-09T11:07:27.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.100840+0000", 2023-10-09T11:07:27.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4830000000000002e-05 2023-10-09T11:07:27.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:27.320 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.100888+0000", 2023-10-09T11:07:27.320 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7370999999999998e-05 2023-10-09T11:07:27.320 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.320 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.320 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.100936+0000", 2023-10-09T11:07:27.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8653999999999999e-05 2023-10-09T11:07:27.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.321 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.321 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.100976+0000", 2023-10-09T11:07:27.321 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9283000000000003e-05 2023-10-09T11:07:27.321 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.321 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.321 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.321 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.321 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.322 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:142420 11.1f e545/531 11:fb0ad690:::benchmark_data_smithi125_439543_object42921:head v 545'7745, mlcod=507'2640)", 2023-10-09T11:07:27.322 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:07:23.911276+0000", 2023-10-09T11:07:27.322 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 3.2536124480000002, 2023-10-09T11:07:27.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.110437917, 2023-10-09T11:07:27.322 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:27.322 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:27.322 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:27.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:27.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:23.911276+0000", 2023-10-09T11:07:27.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:27.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:23.911276+0000", 2023-10-09T11:07:27.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:27.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:27.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:23.911276+0000", 2023-10-09T11:07:27.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:27.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:27.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:23.911281+0000", 2023-10-09T11:07:27.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5419999999999996e-06 2023-10-09T11:07:27.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.324 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:27.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:24.015537+0000", 2023-10-09T11:07:27.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.104255523 2023-10-09T11:07:27.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.324 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:27.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:24.015549+0000", 2023-10-09T11:07:27.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2184000000000001e-05 2023-10-09T11:07:27.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:27.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:24.018417+0000", 2023-10-09T11:07:27.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028677540000000001 2023-10-09T11:07:27.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:27.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:24.018437+0000", 2023-10-09T11:07:27.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9831999999999999e-05 2023-10-09T11:07:27.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:27.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:24.021682+0000", 2023-10-09T11:07:27.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032458330000000001 2023-10-09T11:07:27.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:27.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:27.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:27.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:24.021714+0000", 2023-10-09T11:07:27.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1776000000000002e-05 2023-10-09T11:07:27.326 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.326 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.326 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.326 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:27.327 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:27.327 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:07:27.327 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:27.327 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:07:27.327 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:07:27.369 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:07:27.416 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:07:27.457 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:07:27.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:27 smithi155 ceph-mon[184854]: pgmap v1333: 139 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 131 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 5171/173082 objects degraded (2.988%); 1875/173082 objects misplaced (1.083%); 12 MiB/s, 204 objects/s recovering 2023-10-09T11:07:27.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:27 smithi155 ceph-mon[184854]: 11.14 scrub starts 2023-10-09T11:07:27.509 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:07:27.543 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:07:27.585 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:07:27.607 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:07:27.607 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:07:27.607 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:07:27.607 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:07:27.607 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:07:27.607 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:07:27.607 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:07:27.608 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:07:28.218 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:28.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:28.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:28.331 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:28.381 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:28.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:28 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2776915511' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:28.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:28 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2776915511' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:28.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:28 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2776915511' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:29.081 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:29.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:29.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:29.090 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T11:07:29.187 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:29.271 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:29.304 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:29.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:29 smithi125 ceph-mon[195507]: pgmap v1334: 139 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 131 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 5171/168603 objects degraded (3.067%); 1875/168603 objects misplaced (1.112%); 10 MiB/s, 179 objects/s recovering 2023-10-09T11:07:29.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:29 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 5171/168603 objects degraded (3.067%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:07:29.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:29 smithi125 ceph-mon[195507]: 11.6 scrub starts 2023-10-09T11:07:29.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:29 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3416002587' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:07:29.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:29 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:07:29.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:29 smithi125 ceph-mon[200197]: pgmap v1334: 139 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 131 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 5171/168603 objects degraded (3.067%); 1875/168603 objects misplaced (1.112%); 10 MiB/s, 179 objects/s recovering 2023-10-09T11:07:29.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:29 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 5171/168603 objects degraded (3.067%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:07:29.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:29 smithi125 ceph-mon[200197]: 11.6 scrub starts 2023-10-09T11:07:29.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:29 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3416002587' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:07:29.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:29 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:07:29.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:29.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:29.393 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:29.412 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:29.442 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:07:29.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:29 smithi155 ceph-mon[184854]: pgmap v1334: 139 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 131 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 5171/168603 objects degraded (3.067%); 1875/168603 objects misplaced (1.112%); 10 MiB/s, 179 objects/s recovering 2023-10-09T11:07:29.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:29 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 5171/168603 objects degraded (3.067%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:07:29.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:29 smithi155 ceph-mon[184854]: 11.6 scrub starts 2023-10-09T11:07:29.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:29 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3416002587' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:07:29.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:29 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:07:29.502 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:29.696 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:29.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:29.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:29.802 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:29.835 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:29.860 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:29.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:29.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:29.963 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:07:29.997 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:30.021 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:07:30.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:30.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:07:30.126 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:07:30.158 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:30.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:30 smithi125 ceph-mon[195507]: 11.4 scrub starts 2023-10-09T11:07:30.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:30 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:07:30.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:30 smithi125 ceph-mon[195507]: failed to encode map e547 with expected crc 2023-10-09T11:07:30.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:30 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:07:30.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:30 smithi125 ceph-mon[195507]: osdmap e547: 8 total, 8 up, 8 in 2023-10-09T11:07:30.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:30 smithi125 ceph-mon[195507]: failed to encode map e547 with expected crc 2023-10-09T11:07:30.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:07:30.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:30 smithi125 ceph-mon[200197]: 11.4 scrub starts 2023-10-09T11:07:30.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:30 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:07:30.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:30 smithi125 ceph-mon[200197]: failed to encode map e547 with expected crc 2023-10-09T11:07:30.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:30 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:07:30.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:30 smithi125 ceph-mon[200197]: osdmap e547: 8 total, 8 up, 8 in 2023-10-09T11:07:30.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:30 smithi125 ceph-mon[200197]: failed to encode map e547 with expected crc 2023-10-09T11:07:30.176 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:07:30.183 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:07:30.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:30.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:07:30.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:30.300 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:30.344 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:30 smithi155 ceph-mon[184854]: 11.4 scrub starts 2023-10-09T11:07:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:30 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:07:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:30 smithi155 ceph-mon[184854]: failed to encode map e547 with expected crc 2023-10-09T11:07:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:30 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:07:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:30 smithi155 ceph-mon[184854]: osdmap e547: 8 total, 8 up, 8 in 2023-10-09T11:07:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:30 smithi155 ceph-mon[184854]: failed to encode map e547 with expected crc 2023-10-09T11:07:30.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:07:30.471 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:30.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:30.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:30.581 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:07:30.621 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:30.645 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:07:30.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:30.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:07:30.749 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:30.785 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:30.810 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:30.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:30.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:30.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:30.954 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:30.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:30.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:30.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:31.084 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:31.196 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:31.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:31 smithi155 ceph-mon[184854]: pgmap v1336: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 133 active+clean; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 4285/158208 objects degraded (2.708%); 1604/158208 objects misplaced (1.014%); 17 MiB/s, 288 objects/s recovering 2023-10-09T11:07:31.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:31 smithi155 ceph-mon[184854]: failed to encode map e548 with expected crc 2023-10-09T11:07:31.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:31 smithi155 ceph-mon[184854]: failed to encode map e548 with expected crc 2023-10-09T11:07:31.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:31 smithi155 ceph-mon[184854]: osdmap e548: 8 total, 8 up, 8 in 2023-10-09T11:07:31.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:31 smithi155 ceph-mon[184854]: failed to encode map e548 with expected crc 2023-10-09T11:07:31.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:31 smithi125 ceph-mon[195507]: pgmap v1336: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 133 active+clean; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 4285/158208 objects degraded (2.708%); 1604/158208 objects misplaced (1.014%); 17 MiB/s, 288 objects/s recovering 2023-10-09T11:07:31.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:31 smithi125 ceph-mon[195507]: failed to encode map e548 with expected crc 2023-10-09T11:07:31.221 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:31 smithi125 ceph-mon[195507]: failed to encode map e548 with expected crc 2023-10-09T11:07:31.221 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:31 smithi125 ceph-mon[195507]: osdmap e548: 8 total, 8 up, 8 in 2023-10-09T11:07:31.221 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:31 smithi125 ceph-mon[195507]: failed to encode map e548 with expected crc 2023-10-09T11:07:31.221 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:07:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:07:31] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2023-10-09T11:07:31.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:31 smithi125 ceph-mon[200197]: pgmap v1336: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 133 active+clean; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 4285/158208 objects degraded (2.708%); 1604/158208 objects misplaced (1.014%); 17 MiB/s, 288 objects/s recovering 2023-10-09T11:07:31.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:31 smithi125 ceph-mon[200197]: failed to encode map e548 with expected crc 2023-10-09T11:07:31.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:31 smithi125 ceph-mon[200197]: failed to encode map e548 with expected crc 2023-10-09T11:07:31.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:31 smithi125 ceph-mon[200197]: osdmap e548: 8 total, 8 up, 8 in 2023-10-09T11:07:31.222 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:31 smithi125 ceph-mon[200197]: failed to encode map e548 with expected crc 2023-10-09T11:07:31.238 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:31.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:31.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:31.343 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:07:31.345 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:31.433 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:31.518 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:31.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:31.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:31.622 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:31.705 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:31.749 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:07:31.749 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:07:31.762 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:31.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:31.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:31.862 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:31.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:31.915 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:32.014 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:07:32.210 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T11:07:32.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:32 smithi125 ceph-mon[195507]: failed to encode map e548 with expected crc 2023-10-09T11:07:32.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:32 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3898150356' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:07:32.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:32 smithi125 ceph-mon[200197]: failed to encode map e548 with expected crc 2023-10-09T11:07:32.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:32 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3898150356' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:07:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:32 smithi155 ceph-mon[184854]: failed to encode map e548 with expected crc 2023-10-09T11:07:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:32 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3898150356' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:07:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:33 smithi155 ceph-mon[184854]: pgmap v1338: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 134 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 4285/154074 objects degraded (2.781%); 1604/154074 objects misplaced (1.041%); 7.8 MiB/s, 131 objects/s recovering 2023-10-09T11:07:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:33 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3898150356' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T11:07:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:33 smithi155 ceph-mon[184854]: osdmap e549: 8 total, 8 up, 8 in 2023-10-09T11:07:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:33 smithi155 ceph-mon[184854]: failed to encode map e549 with expected crc 2023-10-09T11:07:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:33 smithi155 ceph-mon[184854]: failed to encode map e549 with expected crc 2023-10-09T11:07:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:33 smithi155 ceph-mon[184854]: failed to encode map e549 with expected crc 2023-10-09T11:07:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:33 smithi155 ceph-mon[184854]: failed to encode map e549 with expected crc 2023-10-09T11:07:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:33 smithi155 ceph-mon[184854]: failed to encode map e549 with expected crc 2023-10-09T11:07:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[195507]: pgmap v1338: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 134 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 4285/154074 objects degraded (2.781%); 1604/154074 objects misplaced (1.041%); 7.8 MiB/s, 131 objects/s recovering 2023-10-09T11:07:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3898150356' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T11:07:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[195507]: osdmap e549: 8 total, 8 up, 8 in 2023-10-09T11:07:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[195507]: failed to encode map e549 with expected crc 2023-10-09T11:07:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[195507]: failed to encode map e549 with expected crc 2023-10-09T11:07:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[195507]: failed to encode map e549 with expected crc 2023-10-09T11:07:33.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[195507]: failed to encode map e549 with expected crc 2023-10-09T11:07:33.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[195507]: failed to encode map e549 with expected crc 2023-10-09T11:07:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[200197]: pgmap v1338: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 134 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 4285/154074 objects degraded (2.781%); 1604/154074 objects misplaced (1.041%); 7.8 MiB/s, 131 objects/s recovering 2023-10-09T11:07:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3898150356' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T11:07:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[200197]: osdmap e549: 8 total, 8 up, 8 in 2023-10-09T11:07:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[200197]: failed to encode map e549 with expected crc 2023-10-09T11:07:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[200197]: failed to encode map e549 with expected crc 2023-10-09T11:07:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[200197]: failed to encode map e549 with expected crc 2023-10-09T11:07:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[200197]: failed to encode map e549 with expected crc 2023-10-09T11:07:33.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:33 smithi125 ceph-mon[200197]: failed to encode map e549 with expected crc 2023-10-09T11:07:33.716 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T11:07:34.042 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:34.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:34.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:34.157 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:34.185 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[195507]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[195507]: osdmap e550: 8 total, 8 up, 8 in 2023-10-09T11:07:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[195507]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[195507]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[195507]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[195507]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[195507]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[195507]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[195507]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[195507]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 4285/154074 objects degraded (2.781%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:07:34.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:07:34.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[200197]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[200197]: osdmap e550: 8 total, 8 up, 8 in 2023-10-09T11:07:34.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[200197]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[200197]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[200197]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[200197]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[200197]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[200197]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[200197]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[200197]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 4285/154074 objects degraded (2.781%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:07:34.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:34 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:07:34.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:34 smithi155 ceph-mon[184854]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:34 smithi155 ceph-mon[184854]: osdmap e550: 8 total, 8 up, 8 in 2023-10-09T11:07:34.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:34 smithi155 ceph-mon[184854]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:34 smithi155 ceph-mon[184854]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:34 smithi155 ceph-mon[184854]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:34 smithi155 ceph-mon[184854]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:34 smithi155 ceph-mon[184854]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:34 smithi155 ceph-mon[184854]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:34 smithi155 ceph-mon[184854]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:34 smithi155 ceph-mon[184854]: failed to encode map e550 with expected crc 2023-10-09T11:07:34.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:34 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 4285/154074 objects degraded (2.781%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:07:34.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:34 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:07:34.487 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T11:07:34.513 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:34.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:34.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:34.621 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:34.657 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:34.938 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:34.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:34.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:35.043 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:35.094 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:35.129 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:35.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:35.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:35.214 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: pgmap v1341: 139 pgs: 1 activating+undersized+degraded+remapped, 14 remapped+peering, 1 active+remapped+backfill_wait, 123 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 2407/146940 objects degraded (1.638%); 1604/146940 objects misplaced (1.092%); 5.2 MiB/s, 103 objects/s recovering 2023-10-09T11:07:35.214 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.214 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.214 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: osdmap e551: 8 total, 8 up, 8 in 2023-10-09T11:07:35.214 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.215 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: pgmap v1341: 139 pgs: 1 activating+undersized+degraded+remapped, 14 remapped+peering, 1 active+remapped+backfill_wait, 123 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 2407/146940 objects degraded (1.638%); 1604/146940 objects misplaced (1.092%); 5.2 MiB/s, 103 objects/s recovering 2023-10-09T11:07:35.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: osdmap e551: 8 total, 8 up, 8 in 2023-10-09T11:07:35.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:35.291 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:35.307 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:35.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:35.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:35.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:35.358 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:35.448 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:35.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: pgmap v1341: 139 pgs: 1 activating+undersized+degraded+remapped, 14 remapped+peering, 1 active+remapped+backfill_wait, 123 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 2407/146940 objects degraded (1.638%); 1604/146940 objects misplaced (1.092%); 5.2 MiB/s, 103 objects/s recovering 2023-10-09T11:07:35.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: osdmap e551: 8 total, 8 up, 8 in 2023-10-09T11:07:35.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: 11.20 starting backfill to osd.7 from (0'0,515'3890] MIN to 548'5515 2023-10-09T11:07:35.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: 11.15 starting backfill to osd.7 from (0'0,515'3778] MIN to 548'6109 2023-10-09T11:07:35.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: 11.23 starting backfill to osd.7 from (0'0,515'4019] MIN to 548'5584 2023-10-09T11:07:35.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: 11.3 starting backfill to osd.7 from (0'0,515'4020] MIN to 548'5641 2023-10-09T11:07:35.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: 11.5 starting backfill to osd.7 from (0'0,515'3786] MIN to 548'6087 2023-10-09T11:07:35.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:35 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T11:07:35.507 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.507 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.507 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.508 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: 11.20 starting backfill to osd.7 from (0'0,515'3890] MIN to 548'5515 2023-10-09T11:07:35.508 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.508 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: 11.15 starting backfill to osd.7 from (0'0,515'3778] MIN to 548'6109 2023-10-09T11:07:35.508 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: 11.23 starting backfill to osd.7 from (0'0,515'4019] MIN to 548'5584 2023-10-09T11:07:35.508 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: 11.3 starting backfill to osd.7 from (0'0,515'4020] MIN to 548'5641 2023-10-09T11:07:35.508 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: 11.5 starting backfill to osd.7 from (0'0,515'3786] MIN to 548'6087 2023-10-09T11:07:35.508 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T11:07:35.508 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.508 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.508 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: 11.20 starting backfill to osd.7 from (0'0,515'3890] MIN to 548'5515 2023-10-09T11:07:35.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: failed to encode map e551 with expected crc 2023-10-09T11:07:35.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: 11.15 starting backfill to osd.7 from (0'0,515'3778] MIN to 548'6109 2023-10-09T11:07:35.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: 11.23 starting backfill to osd.7 from (0'0,515'4019] MIN to 548'5584 2023-10-09T11:07:35.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: 11.3 starting backfill to osd.7 from (0'0,515'4020] MIN to 548'5641 2023-10-09T11:07:35.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: 11.5 starting backfill to osd.7 from (0'0,515'3786] MIN to 548'6087 2023-10-09T11:07:35.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:35 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T11:07:35.511 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:36.205 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:36.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:36.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:36.316 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:36.327 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:36.377 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:36 smithi125 ceph-mon[195507]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.377 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:36 smithi125 ceph-mon[195507]: osdmap e552: 8 total, 8 up, 8 in 2023-10-09T11:07:36.377 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:36 smithi125 ceph-mon[195507]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.377 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:36 smithi125 ceph-mon[195507]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.377 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:36 smithi125 ceph-mon[195507]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.377 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:36 smithi125 ceph-mon[195507]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.377 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:36 smithi125 ceph-mon[195507]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.377 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:36 smithi125 ceph-mon[200197]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.378 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:36 smithi125 ceph-mon[200197]: osdmap e552: 8 total, 8 up, 8 in 2023-10-09T11:07:36.378 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:36 smithi125 ceph-mon[200197]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.378 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:36 smithi125 ceph-mon[200197]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.378 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:36 smithi125 ceph-mon[200197]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.378 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:36 smithi125 ceph-mon[200197]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.378 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:36 smithi125 ceph-mon[200197]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.414 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:36.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:36 smithi155 ceph-mon[184854]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:36 smithi155 ceph-mon[184854]: osdmap e552: 8 total, 8 up, 8 in 2023-10-09T11:07:36.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:36 smithi155 ceph-mon[184854]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:36 smithi155 ceph-mon[184854]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:36 smithi155 ceph-mon[184854]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:36 smithi155 ceph-mon[184854]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:36 smithi155 ceph-mon[184854]: failed to encode map e552 with expected crc 2023-10-09T11:07:36.523 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:36.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:36.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:36.632 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:36.674 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:36.907 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:36.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:36.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:37.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:37.075 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:37.247 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:37 smithi125 ceph-mon[200197]: pgmap v1344: 139 pgs: 1 activating+undersized+degraded+remapped, 14 remapped+peering, 1 active+remapped+backfill_wait, 123 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 2407/146940 objects degraded (1.638%); 1604/146940 objects misplaced (1.092%); 5.8 MiB/s, 116 objects/s recovering 2023-10-09T11:07:37.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:37 smithi125 ceph-mon[200197]: failed to encode map e553 with expected crc 2023-10-09T11:07:37.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:37 smithi125 ceph-mon[200197]: osdmap e553: 8 total, 8 up, 8 in 2023-10-09T11:07:37.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:37 smithi125 ceph-mon[200197]: failed to encode map e553 with expected crc 2023-10-09T11:07:37.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:37 smithi125 ceph-mon[200197]: failed to encode map e553 with expected crc 2023-10-09T11:07:37.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:37 smithi125 ceph-mon[200197]: failed to encode map e553 with expected crc 2023-10-09T11:07:37.250 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:37 smithi125 ceph-mon[195507]: pgmap v1344: 139 pgs: 1 activating+undersized+degraded+remapped, 14 remapped+peering, 1 active+remapped+backfill_wait, 123 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 2407/146940 objects degraded (1.638%); 1604/146940 objects misplaced (1.092%); 5.8 MiB/s, 116 objects/s recovering 2023-10-09T11:07:37.250 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:37 smithi125 ceph-mon[195507]: failed to encode map e553 with expected crc 2023-10-09T11:07:37.250 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:37 smithi125 ceph-mon[195507]: osdmap e553: 8 total, 8 up, 8 in 2023-10-09T11:07:37.250 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:37 smithi125 ceph-mon[195507]: failed to encode map e553 with expected crc 2023-10-09T11:07:37.251 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:37 smithi125 ceph-mon[195507]: failed to encode map e553 with expected crc 2023-10-09T11:07:37.251 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:37 smithi125 ceph-mon[195507]: failed to encode map e553 with expected crc 2023-10-09T11:07:37.711 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:37.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:37.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:37.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:37 smithi155 ceph-mon[184854]: pgmap v1344: 139 pgs: 1 activating+undersized+degraded+remapped, 14 remapped+peering, 1 active+remapped+backfill_wait, 123 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 2407/146940 objects degraded (1.638%); 1604/146940 objects misplaced (1.092%); 5.8 MiB/s, 116 objects/s recovering 2023-10-09T11:07:37.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:37 smithi155 ceph-mon[184854]: failed to encode map e553 with expected crc 2023-10-09T11:07:37.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:37 smithi155 ceph-mon[184854]: osdmap e553: 8 total, 8 up, 8 in 2023-10-09T11:07:37.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:37 smithi155 ceph-mon[184854]: failed to encode map e553 with expected crc 2023-10-09T11:07:37.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:37 smithi155 ceph-mon[184854]: failed to encode map e553 with expected crc 2023-10-09T11:07:37.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:37 smithi155 ceph-mon[184854]: failed to encode map e553 with expected crc 2023-10-09T11:07:37.816 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:37.856 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:37.934 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:37.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:37.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:38.039 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:38.090 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:38.137 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:38.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:38.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:38.243 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:38.344 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:38.418 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:38 smithi125 ceph-mon[195507]: failed to encode map e553 with expected crc 2023-10-09T11:07:38.418 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:38 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:38.418 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:38 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:38.419 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:38 smithi125 ceph-mon[200197]: failed to encode map e553 with expected crc 2023-10-09T11:07:38.419 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:38 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:38.419 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:38 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:38.432 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:38.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:38.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:38 smithi155 ceph-mon[184854]: failed to encode map e553 with expected crc 2023-10-09T11:07:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:38 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:38.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:38 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:38.534 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:07:38.551 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:38.591 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:07:38.619 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:38.633 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:07:38.654 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:38.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:38.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:38.686 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:07:38.730 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:07:38.764 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:38.774 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:07:38.813 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:38.826 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:07:38.838 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:38.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:38.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:38.869 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:07:38.891 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:07:38.891 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:07:38.891 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:07:38.891 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:07:38.891 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:07:38.891 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:07:38.891 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:07:38.892 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:07:38.941 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:39.011 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:39.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:39 smithi125 ceph-mon[195507]: pgmap v1346: 139 pgs: 1 activating+undersized+degraded+remapped, 10 remapped+peering, 128 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 2407/144372 objects degraded (1.667%); 5.0 MiB/s, 2 keys/s, 101 objects/s recovering 2023-10-09T11:07:39.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:39 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T11:07:39.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:39 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 2407/144372 objects degraded (1.667%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:07:39.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:39 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2019360794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:39.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:39 smithi125 ceph-mon[200197]: pgmap v1346: 139 pgs: 1 activating+undersized+degraded+remapped, 10 remapped+peering, 128 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 2407/144372 objects degraded (1.667%); 5.0 MiB/s, 2 keys/s, 101 objects/s recovering 2023-10-09T11:07:39.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:39 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T11:07:39.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:39 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 2407/144372 objects degraded (1.667%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:07:39.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:39 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2019360794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:39.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:39 smithi155 ceph-mon[184854]: pgmap v1346: 139 pgs: 1 activating+undersized+degraded+remapped, 10 remapped+peering, 128 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 2407/144372 objects degraded (1.667%); 5.0 MiB/s, 2 keys/s, 101 objects/s recovering 2023-10-09T11:07:39.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:39 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T11:07:39.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:39 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 2407/144372 objects degraded (1.667%), 1 pg degraded (PG_DEGRADED) 2023-10-09T11:07:39.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:39 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2019360794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:40.152 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:40.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:40.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:40.257 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:40.334 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:40.938 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:40.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:40.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:41.043 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:41.088 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:41.123 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:41.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:41.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:41.229 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:41.293 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:41.331 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T11:07:41.345 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:07:41.356 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:41 smithi125 ceph-mon[195507]: pgmap v1347: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 6040/135906 objects degraded (4.444%); 5471/135906 objects misplaced (4.026%); 3.7 MiB/s, 1 keys/s, 61 objects/s recovering 2023-10-09T11:07:41.357 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:41 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3215441271' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:07:41.357 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:41 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:07:41.357 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:41 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2180288733' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:07:41.357 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:41 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:07:41.357 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:07:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:07:41] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2023-10-09T11:07:41.357 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:41 smithi125 ceph-mon[200197]: pgmap v1347: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 6040/135906 objects degraded (4.444%); 5471/135906 objects misplaced (4.026%); 3.7 MiB/s, 1 keys/s, 61 objects/s recovering 2023-10-09T11:07:41.357 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:41 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3215441271' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:07:41.358 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:41 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:07:41.358 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:41 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2180288733' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:07:41.358 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:41 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:07:41.380 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:41.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:41.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:41.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:41 smithi155 ceph-mon[184854]: pgmap v1347: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 6040/135906 objects degraded (4.444%); 5471/135906 objects misplaced (4.026%); 3.7 MiB/s, 1 keys/s, 61 objects/s recovering 2023-10-09T11:07:41.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:41 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3215441271' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:07:41.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:41 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:07:41.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:41 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2180288733' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:07:41.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:41 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:07:41.486 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:07:41.524 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:41.547 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:07:41.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:41.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:07:41.616 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:07:41.616 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:07:41.616 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T11:07:41.616 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T11:07:41.616 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:07:41.653 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:41.691 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:41.717 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:41.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:41.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:41.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:41.879 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:42.264 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T11:07:42.466 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:42.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:42.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:42.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:07:42.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T11:07:42.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:07:42.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[195507]: osdmap e554: 8 total, 8 up, 8 in 2023-10-09T11:07:42.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[195507]: failed to encode map e554 with expected crc 2023-10-09T11:07:42.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[195507]: failed to encode map e554 with expected crc 2023-10-09T11:07:42.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[195507]: failed to encode map e554 with expected crc 2023-10-09T11:07:42.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[195507]: failed to encode map e554 with expected crc 2023-10-09T11:07:42.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[195507]: failed to encode map e554 with expected crc 2023-10-09T11:07:42.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:07:42.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T11:07:42.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:07:42.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[200197]: osdmap e554: 8 total, 8 up, 8 in 2023-10-09T11:07:42.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[200197]: failed to encode map e554 with expected crc 2023-10-09T11:07:42.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[200197]: failed to encode map e554 with expected crc 2023-10-09T11:07:42.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[200197]: failed to encode map e554 with expected crc 2023-10-09T11:07:42.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[200197]: failed to encode map e554 with expected crc 2023-10-09T11:07:42.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:42 smithi125 ceph-mon[200197]: failed to encode map e554 with expected crc 2023-10-09T11:07:42.582 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:42.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:42 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:07:42.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:42 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T11:07:42.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:42 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:07:42.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:42 smithi155 ceph-mon[184854]: osdmap e554: 8 total, 8 up, 8 in 2023-10-09T11:07:42.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:42 smithi155 ceph-mon[184854]: failed to encode map e554 with expected crc 2023-10-09T11:07:42.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:42 smithi155 ceph-mon[184854]: failed to encode map e554 with expected crc 2023-10-09T11:07:42.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:42 smithi155 ceph-mon[184854]: failed to encode map e554 with expected crc 2023-10-09T11:07:42.613 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:42 smithi155 ceph-mon[184854]: failed to encode map e554 with expected crc 2023-10-09T11:07:42.613 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:42 smithi155 ceph-mon[184854]: failed to encode map e554 with expected crc 2023-10-09T11:07:42.616 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:42.641 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:42.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:42.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:42.748 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:42.789 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:42.834 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T11:07:42.879 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:42.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:42.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:42.968 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:42.988 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:43.097 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:43.184 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:43.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:43.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:43.289 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:43.352 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:43.436 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:43.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:43.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:43.454 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:43 smithi125 ceph-mon[200197]: pgmap v1349: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 6040/131844 objects degraded (4.581%); 5471/131844 objects misplaced (4.150%); 3.4 MiB/s, 1 keys/s, 55 objects/s recovering 2023-10-09T11:07:43.455 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:43 smithi125 ceph-mon[195507]: pgmap v1349: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 6040/131844 objects degraded (4.581%); 5471/131844 objects misplaced (4.150%); 3.4 MiB/s, 1 keys/s, 55 objects/s recovering 2023-10-09T11:07:43.552 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:43.579 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:43.598 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:43.618 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:43.623 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:43 smithi155 ceph-mon[184854]: pgmap v1349: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 6040/131844 objects degraded (4.581%); 5471/131844 objects misplaced (4.150%); 3.4 MiB/s, 1 keys/s, 55 objects/s recovering 2023-10-09T11:07:43.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:43.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:43.725 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:43.785 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:44.075 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:44.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:44.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:44.184 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:44.260 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:44.310 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:44.321 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:07:44.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:44.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:44.359 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:44 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 6040/131844 objects degraded (4.581%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:44.359 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:44 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:44.359 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:44 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:44.362 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:44 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 6040/131844 objects degraded (4.581%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:44.362 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:44 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:44.362 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:44 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:44.423 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:44.517 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:44.546 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:07:44.547 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T11:07:44.547 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T11:07:44.547 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:07:44.547 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.547 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122010 9.6 9:6f14ab77:::benchmark_data_smithi125_363789_object11711:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:07:44.547 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.136716+0000", 2023-10-09T11:07:44.547 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 516.40366577700001, 2023-10-09T11:07:44.547 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0621660900000001, 2023-10-09T11:07:44.547 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.547 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:07:44.547 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.548 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:07:44.548 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:07:44.548 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122010 2023-10-09T11:07:44.548 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.548 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.548 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.548 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.548 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136716+0000", 2023-10-09T11:07:44.548 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.548 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.548 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.548 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.548 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136716+0000", 2023-10-09T11:07:44.549 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.549 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.549 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.549 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.549 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136715+0000", 2023-10-09T11:07:44.549 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:07:44.549 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.549 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.549 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.549 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136721+0000", 2023-10-09T11:07:44.549 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7660000000000004e-06 2023-10-09T11:07:44.549 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.549 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.550 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.550 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136731+0000", 2023-10-09T11:07:44.550 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0898e-05 2023-10-09T11:07:44.550 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.550 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.550 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.550 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.136739+0000", 2023-10-09T11:07:44.550 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8450000000000003e-06 2023-10-09T11:07:44.550 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.550 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.550 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.550 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.140487+0000", 2023-10-09T11:07:44.551 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037476330000000002 2023-10-09T11:07:44.551 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.551 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.551 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.551 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.140607+0000", 2023-10-09T11:07:44.551 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012029299999999999 2023-10-09T11:07:44.551 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.551 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.551 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T11:07:44.551 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.140726+0000", 2023-10-09T11:07:44.551 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119057 2023-10-09T11:07:44.551 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.552 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.552 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.552 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.150456+0000", 2023-10-09T11:07:44.552 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.009729283 2023-10-09T11:07:44.552 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.552 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.552 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.552 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.153186+0000", 2023-10-09T11:07:44.552 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00273077 2023-10-09T11:07:44.552 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.552 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.552 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.553 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162778+0000", 2023-10-09T11:07:44.553 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0095913500000000002 2023-10-09T11:07:44.553 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.553 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.553 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.553 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162790+0000", 2023-10-09T11:07:44.553 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.221e-05 2023-10-09T11:07:44.553 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.553 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.553 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.553 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.198810+0000", 2023-10-09T11:07:44.553 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.036020156 2023-10-09T11:07:44.553 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.554 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.554 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.554 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.198882+0000", 2023-10-09T11:07:44.554 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1717999999999993e-05 2023-10-09T11:07:44.554 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.554 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.554 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.554 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.554 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.554 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122013 9.6 9:6b7eeaa4:::benchmark_data_smithi125_363789_object11714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:07:44.554 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.138283+0000", 2023-10-09T11:07:44.554 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 516.40209842399997, 2023-10-09T11:07:44.555 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.060936141, 2023-10-09T11:07:44.555 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.555 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:07:44.555 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.555 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:07:44.555 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:07:44.555 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122013 2023-10-09T11:07:44.555 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.555 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.555 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.555 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.555 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138283+0000", 2023-10-09T11:07:44.555 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.556 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.556 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.556 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.556 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138283+0000", 2023-10-09T11:07:44.556 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.556 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.556 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.556 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.556 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138282+0000", 2023-10-09T11:07:44.556 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:07:44.556 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.556 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.557 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.557 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138288+0000", 2023-10-09T11:07:44.557 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5029999999999997e-06 2023-10-09T11:07:44.557 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.557 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.557 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.557 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138300+0000", 2023-10-09T11:07:44.557 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2357e-05 2023-10-09T11:07:44.557 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.557 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.557 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.557 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.138309+0000", 2023-10-09T11:07:44.557 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.499e-06 2023-10-09T11:07:44.558 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.558 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.558 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.558 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.144017+0000", 2023-10-09T11:07:44.558 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0057070339999999997 2023-10-09T11:07:44.558 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.558 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.558 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.558 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.144165+0000", 2023-10-09T11:07:44.558 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000148345 2023-10-09T11:07:44.558 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.558 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.559 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T11:07:44.559 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.144293+0000", 2023-10-09T11:07:44.559 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000128069 2023-10-09T11:07:44.559 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.559 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.559 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.559 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.152647+0000", 2023-10-09T11:07:44.559 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083540590000000005 2023-10-09T11:07:44.559 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.559 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.559 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.560 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.153226+0000", 2023-10-09T11:07:44.560 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00057917199999999998 2023-10-09T11:07:44.560 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.560 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.560 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.560 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162825+0000", 2023-10-09T11:07:44.560 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0095991010000000005 2023-10-09T11:07:44.560 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.560 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.560 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.560 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162833+0000", 2023-10-09T11:07:44.560 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3429999999999998e-06 2023-10-09T11:07:44.561 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.561 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.561 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.561 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199105+0000", 2023-10-09T11:07:44.561 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0362724169999999 2023-10-09T11:07:44.561 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.561 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.561 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.561 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199219+0000", 2023-10-09T11:07:44.561 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000114141 2023-10-09T11:07:44.561 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.561 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.561 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.562 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.562 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.562 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122028 9.6 9:6992c4ac:::benchmark_data_smithi125_363789_object11729:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-10-09T11:07:44.562 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.154935+0000", 2023-10-09T11:07:44.562 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 516.38544637899997, 2023-10-09T11:07:44.562 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.044381056, 2023-10-09T11:07:44.562 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.562 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:07:44.562 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.562 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:07:44.562 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:07:44.562 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122028 2023-10-09T11:07:44.563 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.563 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.563 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.563 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.563 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154935+0000", 2023-10-09T11:07:44.563 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.563 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.563 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.563 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.563 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154935+0000", 2023-10-09T11:07:44.563 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.563 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.564 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.564 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.564 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154934+0000", 2023-10-09T11:07:44.564 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:07:44.564 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.564 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.564 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.564 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154941+0000", 2023-10-09T11:07:44.564 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9600000000000003e-06 2023-10-09T11:07:44.564 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.564 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.564 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.564 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154956+0000", 2023-10-09T11:07:44.565 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4976e-05 2023-10-09T11:07:44.565 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.565 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.565 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.565 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.154964+0000", 2023-10-09T11:07:44.565 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8639999999999996e-06 2023-10-09T11:07:44.565 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.565 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.565 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.565 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.155012+0000", 2023-10-09T11:07:44.565 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7960999999999997e-05 2023-10-09T11:07:44.565 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.566 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.566 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.566 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.155124+0000", 2023-10-09T11:07:44.566 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011204 2023-10-09T11:07:44.566 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.566 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.566 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T11:07:44.566 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.155237+0000", 2023-10-09T11:07:44.566 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011364799999999999 2023-10-09T11:07:44.566 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.566 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.566 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.566 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162936+0000", 2023-10-09T11:07:44.567 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.007698369 2023-10-09T11:07:44.567 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.567 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.567 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.567 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.162945+0000", 2023-10-09T11:07:44.567 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6039999999999999e-06 2023-10-09T11:07:44.567 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.567 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.567 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.567 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199250+0000", 2023-10-09T11:07:44.567 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.036304248 2023-10-09T11:07:44.567 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.568 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.568 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.568 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199316+0000", 2023-10-09T11:07:44.568 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6549000000000003e-05 2023-10-09T11:07:44.568 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.568 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.568 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.568 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.568 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.568 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122037 9.6 9:68399f49:::benchmark_data_smithi125_363789_object11738:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:07:44.568 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.181454+0000", 2023-10-09T11:07:44.568 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 516.35892710899998, 2023-10-09T11:07:44.569 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.032324094, 2023-10-09T11:07:44.569 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.569 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:44.569 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.569 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:07:44.569 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:07:44.569 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122037 2023-10-09T11:07:44.569 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.569 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.569 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.569 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.569 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181454+0000", 2023-10-09T11:07:44.569 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.570 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.570 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.570 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.570 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181454+0000", 2023-10-09T11:07:44.570 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.570 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.570 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.570 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.570 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181453+0000", 2023-10-09T11:07:44.570 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:07:44.570 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.570 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.571 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.571 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181461+0000", 2023-10-09T11:07:44.571 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4800000000000001e-06 2023-10-09T11:07:44.571 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.571 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.571 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.571 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181478+0000", 2023-10-09T11:07:44.571 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7215e-05 2023-10-09T11:07:44.571 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.571 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.571 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.571 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181488+0000", 2023-10-09T11:07:44.571 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5240000000000006e-06 2023-10-09T11:07:44.572 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.572 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.572 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.572 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181572+0000", 2023-10-09T11:07:44.572 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3784999999999998e-05 2023-10-09T11:07:44.572 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.572 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.572 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.572 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181581+0000", 2023-10-09T11:07:44.572 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7249999999999992e-06 2023-10-09T11:07:44.572 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.572 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.573 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.573 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199341+0000", 2023-10-09T11:07:44.573 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.017760062 2023-10-09T11:07:44.573 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.573 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.573 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.573 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199463+0000", 2023-10-09T11:07:44.573 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012115799999999999 2023-10-09T11:07:44.573 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.573 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.573 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5", 2023-10-09T11:07:44.573 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.199575+0000", 2023-10-09T11:07:44.573 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000112404 2023-10-09T11:07:44.574 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.574 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.574 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.574 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.207005+0000", 2023-10-09T11:07:44.574 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074305350000000003 2023-10-09T11:07:44.574 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.574 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.574 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.574 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.213644+0000", 2023-10-09T11:07:44.574 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0066387779999999997 2023-10-09T11:07:44.574 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.574 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.575 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:44.575 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.213695+0000", 2023-10-09T11:07:44.575 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0748999999999998e-05 2023-10-09T11:07:44.575 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.575 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.575 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.575 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.213778+0000", 2023-10-09T11:07:44.575 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3434000000000001e-05 2023-10-09T11:07:44.575 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.575 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.575 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.575 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.575 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.576 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:122039 9.6 9:679871c1:::benchmark_data_smithi125_363789_object11740:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-10-09T11:07:44.576 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:07:44.576 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 516.35886614599997, 2023-10-09T11:07:44.576 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0328282099999999, 2023-10-09T11:07:44.576 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.576 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:44.576 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.576 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:07:44.576 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:07:44.576 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 122039 2023-10-09T11:07:44.576 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.576 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.576 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.577 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.577 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:07:44.577 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.577 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.577 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.577 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.577 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:07:44.577 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.577 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.577 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.577 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.577 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181515+0000", 2023-10-09T11:07:44.577 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:44.578 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.578 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.578 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.578 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181518+0000", 2023-10-09T11:07:44.578 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4350000000000001e-06 2023-10-09T11:07:44.578 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.578 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.578 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.578 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181525+0000", 2023-10-09T11:07:44.578 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5359999999999998e-06 2023-10-09T11:07:44.578 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.578 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.579 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.579 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181530+0000", 2023-10-09T11:07:44.579 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9019999999999998e-06 2023-10-09T11:07:44.579 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.579 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.579 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.579 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181601+0000", 2023-10-09T11:07:44.579 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1168999999999996e-05 2023-10-09T11:07:44.579 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.579 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.579 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.579 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:08.181606+0000", 2023-10-09T11:07:44.579 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1919999999999996e-06 2023-10-09T11:07:44.580 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.580 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.580 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.580 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.203647+0000", 2023-10-09T11:07:44.580 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.022041073 2023-10-09T11:07:44.580 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.580 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.580 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.580 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.203754+0000", 2023-10-09T11:07:44.580 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000106529 2023-10-09T11:07:44.580 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.580 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.581 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5", 2023-10-09T11:07:44.581 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.203828+0000", 2023-10-09T11:07:44.581 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4286e-05 2023-10-09T11:07:44.581 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.581 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.581 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.581 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.210714+0000", 2023-10-09T11:07:44.581 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0068858050000000001 2023-10-09T11:07:44.581 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.581 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.581 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.581 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.214203+0000", 2023-10-09T11:07:44.582 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003489242 2023-10-09T11:07:44.582 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.582 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.582 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:44.582 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.214268+0000", 2023-10-09T11:07:44.582 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5104000000000006e-05 2023-10-09T11:07:44.582 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.582 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.582 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.582 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:09.214344+0000", 2023-10-09T11:07:44.582 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5471999999999998e-05 2023-10-09T11:07:44.582 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.582 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.583 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.583 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.583 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.583 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:133037 9.b 9.da5dd31b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e362)", 2023-10-09T11:07:44.583 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:20.474469+0000", 2023-10-09T11:07:44.583 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 504.06591265899999, 2023-10-09T11:07:44.583 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97365184000000005, 2023-10-09T11:07:44.583 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.583 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:07:44.583 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.583 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:07:44.583 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:07:44.584 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133037 2023-10-09T11:07:44.584 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.584 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.584 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.584 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.584 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474469+0000", 2023-10-09T11:07:44.584 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.584 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.584 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.584 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.584 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474469+0000", 2023-10-09T11:07:44.584 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.584 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.585 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.585 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.585 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474467+0000", 2023-10-09T11:07:44.585 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:07:44.585 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.585 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.585 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.585 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474475+0000", 2023-10-09T11:07:44.585 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6009999999999997e-06 2023-10-09T11:07:44.585 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.585 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.585 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.585 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474493+0000", 2023-10-09T11:07:44.586 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8150000000000001e-05 2023-10-09T11:07:44.586 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.586 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.586 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.586 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474502+0000", 2023-10-09T11:07:44.586 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1400000000000006e-06 2023-10-09T11:07:44.586 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.586 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.586 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.586 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474569+0000", 2023-10-09T11:07:44.586 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6849999999999999e-05 2023-10-09T11:07:44.586 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.586 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.587 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.587 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474579+0000", 2023-10-09T11:07:44.587 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9369999999999999e-06 2023-10-09T11:07:44.587 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.587 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.587 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.587 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448111+0000", 2023-10-09T11:07:44.587 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97353240399999996 2023-10-09T11:07:44.587 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.587 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.587 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.587 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448121+0000", 2023-10-09T11:07:44.588 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1679999999999993e-06 2023-10-09T11:07:44.588 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.588 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.588 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.588 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.588 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.588 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:133044 9.b 9.5473ef4b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e362)", 2023-10-09T11:07:44.588 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:20.482185+0000", 2023-10-09T11:07:44.588 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 504.05819628500001, 2023-10-09T11:07:44.588 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96595614900000004, 2023-10-09T11:07:44.588 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.588 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:07:44.588 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.589 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:07:44.589 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:07:44.589 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133044 2023-10-09T11:07:44.589 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.589 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.589 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.589 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.589 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482185+0000", 2023-10-09T11:07:44.589 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.589 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.589 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.589 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.589 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482185+0000", 2023-10-09T11:07:44.590 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.590 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.590 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.590 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.590 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482184+0000", 2023-10-09T11:07:44.590 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:07:44.590 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.590 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.590 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.590 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482193+0000", 2023-10-09T11:07:44.590 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0310000000000004e-06 2023-10-09T11:07:44.590 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.591 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.591 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.591 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482217+0000", 2023-10-09T11:07:44.591 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4677000000000001e-05 2023-10-09T11:07:44.591 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.591 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.591 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.591 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482228+0000", 2023-10-09T11:07:44.591 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.043e-05 2023-10-09T11:07:44.591 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.591 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.591 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.591 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482326+0000", 2023-10-09T11:07:44.592 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8241999999999999e-05 2023-10-09T11:07:44.592 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.592 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.592 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.592 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482334+0000", 2023-10-09T11:07:44.592 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7910000000000005e-06 2023-10-09T11:07:44.592 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.592 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.592 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.592 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448135+0000", 2023-10-09T11:07:44.592 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96580099799999997 2023-10-09T11:07:44.592 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.592 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.593 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.593 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448141+0000", 2023-10-09T11:07:44.593 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5719999999999999e-06 2023-10-09T11:07:44.593 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.593 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.593 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.593 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.593 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.593 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39381 11.d 11:b5913833:::benchmark_data_smithi125_439543_object39380:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:07:44.593 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.310294+0000", 2023-10-09T11:07:44.593 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.230087314, 2023-10-09T11:07:44.593 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0048304729999999, 2023-10-09T11:07:44.594 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.594 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:44.594 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.594 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:44.594 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:44.594 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39381 2023-10-09T11:07:44.594 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.594 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.594 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.594 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.594 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310294+0000", 2023-10-09T11:07:44.594 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.595 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.595 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.595 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.595 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310294+0000", 2023-10-09T11:07:44.595 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.595 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.595 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.595 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.595 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310292+0000", 2023-10-09T11:07:44.595 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:07:44.595 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.595 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.595 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.596 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310321+0000", 2023-10-09T11:07:44.596 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8965000000000001e-05 2023-10-09T11:07:44.596 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.596 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.596 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.596 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310335+0000", 2023-10-09T11:07:44.596 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3407999999999999e-05 2023-10-09T11:07:44.596 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.596 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.596 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.596 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310343+0000", 2023-10-09T11:07:44.596 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1030000000000002e-06 2023-10-09T11:07:44.596 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.597 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.597 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.597 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310413+0000", 2023-10-09T11:07:44.597 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9822999999999999e-05 2023-10-09T11:07:44.597 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.597 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.597 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.597 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310428+0000", 2023-10-09T11:07:44.597 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5021999999999999e-05 2023-10-09T11:07:44.597 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.597 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.597 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.598 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.311384+0000", 2023-10-09T11:07:44.598 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0009558540000001 2023-10-09T11:07:44.598 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.598 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.598 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.598 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.311492+0000", 2023-10-09T11:07:44.598 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108618 2023-10-09T11:07:44.598 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.598 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.598 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:07:44.598 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.311595+0000", 2023-10-09T11:07:44.598 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000102431 2023-10-09T11:07:44.599 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.599 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.599 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.599 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315040+0000", 2023-10-09T11:07:44.599 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0034454049999999999 2023-10-09T11:07:44.599 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.599 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.599 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.599 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315049+0000", 2023-10-09T11:07:44.599 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7749999999999998e-06 2023-10-09T11:07:44.599 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.599 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.599 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.600 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315067+0000", 2023-10-09T11:07:44.600 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8595999999999999e-05 2023-10-09T11:07:44.600 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.600 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.600 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:44.600 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315098+0000", 2023-10-09T11:07:44.600 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0185999999999999e-05 2023-10-09T11:07:44.600 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.600 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.600 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.600 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315125+0000", 2023-10-09T11:07:44.600 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6982999999999999e-05 2023-10-09T11:07:44.601 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.601 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.601 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.601 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.601 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.601 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39392 11.d 11:b103d758:::benchmark_data_smithi125_439543_object39391:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:07:44.601 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.311770+0000", 2023-10-09T11:07:44.601 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.22861131299999, 2023-10-09T11:07:44.601 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0081084490000001, 2023-10-09T11:07:44.601 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.601 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:44.601 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.601 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:44.602 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:44.602 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39392 2023-10-09T11:07:44.602 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.602 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.602 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.602 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.602 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311770+0000", 2023-10-09T11:07:44.602 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.602 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.602 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.602 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.602 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311770+0000", 2023-10-09T11:07:44.603 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.603 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.603 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.603 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.603 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311769+0000", 2023-10-09T11:07:44.603 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:07:44.603 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.603 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.603 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.603 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311790+0000", 2023-10-09T11:07:44.603 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1759999999999998e-05 2023-10-09T11:07:44.603 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.604 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.604 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.604 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311800+0000", 2023-10-09T11:07:44.604 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6430000000000007e-06 2023-10-09T11:07:44.604 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.604 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.604 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.604 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311806+0000", 2023-10-09T11:07:44.604 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0619999999999997e-06 2023-10-09T11:07:44.604 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.604 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.604 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.605 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311962+0000", 2023-10-09T11:07:44.605 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000156312 2023-10-09T11:07:44.605 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.605 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.605 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.605 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311972+0000", 2023-10-09T11:07:44.605 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0098000000000001e-05 2023-10-09T11:07:44.605 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.605 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.605 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.605 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.312904+0000", 2023-10-09T11:07:44.605 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0009318810000001 2023-10-09T11:07:44.605 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.606 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.606 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.606 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.312992+0000", 2023-10-09T11:07:44.606 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7459000000000004e-05 2023-10-09T11:07:44.606 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.606 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.606 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:07:44.606 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.313085+0000", 2023-10-09T11:07:44.606 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2730000000000002e-05 2023-10-09T11:07:44.606 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.606 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.606 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.607 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315270+0000", 2023-10-09T11:07:44.607 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021854069999999999 2023-10-09T11:07:44.607 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.607 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.607 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.607 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318204+0000", 2023-10-09T11:07:44.607 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002933719 2023-10-09T11:07:44.607 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.607 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.607 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.607 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319781+0000", 2023-10-09T11:07:44.607 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001577427 2023-10-09T11:07:44.607 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.608 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.608 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:44.608 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319838+0000", 2023-10-09T11:07:44.608 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6882000000000002e-05 2023-10-09T11:07:44.608 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.608 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.608 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.608 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319879+0000", 2023-10-09T11:07:44.608 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0627999999999998e-05 2023-10-09T11:07:44.608 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.608 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.608 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.608 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.609 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.609 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39394 11.d 11:ba71174e:::benchmark_data_smithi125_439543_object39393:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:07:44.609 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.316112+0000", 2023-10-09T11:07:44.609 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.22426946499999, 2023-10-09T11:07:44.609 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.003833969, 2023-10-09T11:07:44.609 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.609 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:44.609 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.609 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:44.609 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:44.609 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39394 2023-10-09T11:07:44.609 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.610 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.610 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.610 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.610 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316112+0000", 2023-10-09T11:07:44.610 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.610 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.610 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.610 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.610 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316112+0000", 2023-10-09T11:07:44.610 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.610 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.610 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.610 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.611 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316110+0000", 2023-10-09T11:07:44.611 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:07:44.611 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.611 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.611 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.611 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316139+0000", 2023-10-09T11:07:44.611 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9023e-05 2023-10-09T11:07:44.611 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.611 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.611 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.611 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316160+0000", 2023-10-09T11:07:44.611 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0852999999999998e-05 2023-10-09T11:07:44.612 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.612 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.612 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.612 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316170+0000", 2023-10-09T11:07:44.612 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8970000000000003e-06 2023-10-09T11:07:44.612 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.612 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.612 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.612 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316244+0000", 2023-10-09T11:07:44.612 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4184000000000007e-05 2023-10-09T11:07:44.612 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.612 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.612 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.613 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316256+0000", 2023-10-09T11:07:44.613 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1447000000000001e-05 2023-10-09T11:07:44.613 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.613 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.613 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.613 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.313863+0000", 2023-10-09T11:07:44.613 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99760760800000003 2023-10-09T11:07:44.613 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.613 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.613 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.613 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.313943+0000", 2023-10-09T11:07:44.613 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9276999999999993e-05 2023-10-09T11:07:44.614 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.614 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.614 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:07:44.614 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.314050+0000", 2023-10-09T11:07:44.614 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000107831 2023-10-09T11:07:44.614 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.614 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.614 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.614 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315326+0000", 2023-10-09T11:07:44.614 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001276103 2023-10-09T11:07:44.614 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.614 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.614 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.615 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318251+0000", 2023-10-09T11:07:44.615 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002924683 2023-10-09T11:07:44.615 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.615 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.615 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.615 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319894+0000", 2023-10-09T11:07:44.615 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016428599999999999 2023-10-09T11:07:44.615 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.615 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.615 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:44.615 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319928+0000", 2023-10-09T11:07:44.615 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3674999999999999e-05 2023-10-09T11:07:44.615 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.616 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.616 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.616 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319946+0000", 2023-10-09T11:07:44.616 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8297000000000001e-05 2023-10-09T11:07:44.616 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.616 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.616 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.616 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.616 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.616 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39401 11.d 11:b5dcd9bc:::benchmark_data_smithi125_439543_object39400:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:07:44.616 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.322614+0000", 2023-10-09T11:07:44.616 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.217767627, 2023-10-09T11:07:44.617 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.003544776, 2023-10-09T11:07:44.617 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.617 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:44.617 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.617 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:44.617 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:44.617 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39401 2023-10-09T11:07:44.617 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.617 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.617 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.617 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.617 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322614+0000", 2023-10-09T11:07:44.617 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.618 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.618 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.618 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.618 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322614+0000", 2023-10-09T11:07:44.618 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.618 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.618 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.618 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.618 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322612+0000", 2023-10-09T11:07:44.618 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:07:44.618 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.618 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.619 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.619 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322636+0000", 2023-10-09T11:07:44.619 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4015e-05 2023-10-09T11:07:44.619 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.619 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.619 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.619 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322650+0000", 2023-10-09T11:07:44.619 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.345e-05 2023-10-09T11:07:44.619 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.619 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.619 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.619 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322657+0000", 2023-10-09T11:07:44.620 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3490000000000003e-06 2023-10-09T11:07:44.620 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.620 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.620 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.620 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322718+0000", 2023-10-09T11:07:44.620 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0520999999999999e-05 2023-10-09T11:07:44.620 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.620 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.620 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.620 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322728+0000", 2023-10-09T11:07:44.620 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0216e-05 2023-10-09T11:07:44.620 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.620 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.621 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.621 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315358+0000", 2023-10-09T11:07:44.621 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.992629808 2023-10-09T11:07:44.621 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.621 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.621 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.621 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315493+0000", 2023-10-09T11:07:44.621 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013509899999999999 2023-10-09T11:07:44.621 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.621 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.621 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:07:44.621 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315627+0000", 2023-10-09T11:07:44.621 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000134077 2023-10-09T11:07:44.622 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.622 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.622 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.622 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318286+0000", 2023-10-09T11:07:44.622 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00265898 2023-10-09T11:07:44.622 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.622 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.622 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.622 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319980+0000", 2023-10-09T11:07:44.622 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016941 2023-10-09T11:07:44.622 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.622 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.623 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.623 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326073+0000", 2023-10-09T11:07:44.623 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0060933239999999998 2023-10-09T11:07:44.623 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.623 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.623 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:44.623 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326124+0000", 2023-10-09T11:07:44.623 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1088000000000003e-05 2023-10-09T11:07:44.623 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.623 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.623 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.623 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326159+0000", 2023-10-09T11:07:44.624 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4353000000000001e-05 2023-10-09T11:07:44.624 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.624 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.624 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.624 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.624 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.624 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39439 11.d 11:b59e136a:::benchmark_data_smithi125_439543_object39438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:07:44.624 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.352200+0000", 2023-10-09T11:07:44.624 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.18818111499999, 2023-10-09T11:07:44.624 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97404727300000005, 2023-10-09T11:07:44.624 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.625 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:44.625 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.625 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:44.625 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:44.625 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39439 2023-10-09T11:07:44.625 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.625 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.625 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.625 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.625 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352200+0000", 2023-10-09T11:07:44.625 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.625 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.626 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.626 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.626 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352200+0000", 2023-10-09T11:07:44.626 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.626 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.626 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.626 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.626 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352199+0000", 2023-10-09T11:07:44.626 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:07:44.626 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.626 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.626 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.627 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352248+0000", 2023-10-09T11:07:44.627 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9008000000000002e-05 2023-10-09T11:07:44.627 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.627 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.627 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.627 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352264+0000", 2023-10-09T11:07:44.627 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5722e-05 2023-10-09T11:07:44.627 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.627 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.627 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.627 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352272+0000", 2023-10-09T11:07:44.627 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2199999999999992e-06 2023-10-09T11:07:44.628 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.628 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.628 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.628 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352340+0000", 2023-10-09T11:07:44.628 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8590000000000006e-05 2023-10-09T11:07:44.628 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.628 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.628 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.628 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352353+0000", 2023-10-09T11:07:44.628 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2648e-05 2023-10-09T11:07:44.628 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.628 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.628 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.629 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.316802+0000", 2023-10-09T11:07:44.629 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96444901500000002 2023-10-09T11:07:44.629 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.629 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.629 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.629 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.316917+0000", 2023-10-09T11:07:44.629 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011529800000000001 2023-10-09T11:07:44.629 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.629 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.629 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:07:44.629 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.317050+0000", 2023-10-09T11:07:44.629 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000132442 2023-10-09T11:07:44.630 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.630 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.630 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.630 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319767+0000", 2023-10-09T11:07:44.630 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027173449999999999 2023-10-09T11:07:44.630 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.630 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.630 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.630 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326058+0000", 2023-10-09T11:07:44.630 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062911210000000002 2023-10-09T11:07:44.630 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.630 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.631 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.631 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326171+0000", 2023-10-09T11:07:44.631 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000112565 2023-10-09T11:07:44.631 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.631 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.631 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:44.631 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326218+0000", 2023-10-09T11:07:44.631 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7377999999999999e-05 2023-10-09T11:07:44.631 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.631 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.631 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.631 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326248+0000", 2023-10-09T11:07:44.631 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9482999999999999e-05 2023-10-09T11:07:44.632 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.632 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.632 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.632 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.632 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.632 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39440 11.d 11:bb23716e:::benchmark_data_smithi125_439543_object39439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:07:44.632 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.352285+0000", 2023-10-09T11:07:44.632 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.188096898, 2023-10-09T11:07:44.632 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97412860000000001, 2023-10-09T11:07:44.632 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.632 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:44.633 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.633 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:44.633 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:44.633 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39440 2023-10-09T11:07:44.633 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.633 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.633 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.633 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352285+0000", 2023-10-09T11:07:44.633 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.633 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352285+0000", 2023-10-09T11:07:44.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352284+0000", 2023-10-09T11:07:44.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:07:44.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352305+0000", 2023-10-09T11:07:44.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1382e-05 2023-10-09T11:07:44.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352314+0000", 2023-10-09T11:07:44.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5220000000000007e-06 2023-10-09T11:07:44.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352320+0000", 2023-10-09T11:07:44.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0850000000000002e-06 2023-10-09T11:07:44.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352375+0000", 2023-10-09T11:07:44.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5411999999999997e-05 2023-10-09T11:07:44.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352382+0000", 2023-10-09T11:07:44.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.849e-06 2023-10-09T11:07:44.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318309+0000", 2023-10-09T11:07:44.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96592686800000005 2023-10-09T11:07:44.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318427+0000", 2023-10-09T11:07:44.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118267 2023-10-09T11:07:44.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:07:44.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318567+0000", 2023-10-09T11:07:44.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013931800000000001 2023-10-09T11:07:44.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.320026+0000", 2023-10-09T11:07:44.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014593410000000001 2023-10-09T11:07:44.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.638 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326259+0000", 2023-10-09T11:07:44.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062325130000000003 2023-10-09T11:07:44.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.638 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326327+0000", 2023-10-09T11:07:44.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8077999999999994e-05 2023-10-09T11:07:44.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:44.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326376+0000", 2023-10-09T11:07:44.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8949000000000002e-05 2023-10-09T11:07:44.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326413+0000", 2023-10-09T11:07:44.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7545000000000001e-05 2023-10-09T11:07:44.639 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.639 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.640 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.640 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39464 11.d 11:b03a9b94:::benchmark_data_smithi125_439543_object39463:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:07:44.640 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.367860+0000", 2023-10-09T11:07:44.640 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.17252102, 2023-10-09T11:07:44.640 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96193788300000005, 2023-10-09T11:07:44.640 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.640 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:44.640 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.640 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:44.640 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:44.641 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39464 2023-10-09T11:07:44.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.641 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.641 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367860+0000", 2023-10-09T11:07:44.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.641 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367860+0000", 2023-10-09T11:07:44.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.642 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.642 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.642 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.642 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367859+0000", 2023-10-09T11:07:44.642 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:07:44.642 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.642 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.642 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.642 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367885+0000", 2023-10-09T11:07:44.642 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5820999999999999e-05 2023-10-09T11:07:44.642 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.642 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.643 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367899+0000", 2023-10-09T11:07:44.643 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4803e-05 2023-10-09T11:07:44.643 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.643 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.643 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367907+0000", 2023-10-09T11:07:44.643 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2010000000000002e-06 2023-10-09T11:07:44.643 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.643 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.643 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367967+0000", 2023-10-09T11:07:44.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0299e-05 2023-10-09T11:07:44.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367978+0000", 2023-10-09T11:07:44.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1616e-05 2023-10-09T11:07:44.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.320095+0000", 2023-10-09T11:07:44.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95211647399999999 2023-10-09T11:07:44.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.645 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.645 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.320237+0000", 2023-10-09T11:07:44.645 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014207400000000001 2023-10-09T11:07:44.645 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.645 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.645 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:07:44.645 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.320378+0000", 2023-10-09T11:07:44.645 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014121199999999999 2023-10-09T11:07:44.645 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.645 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.645 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.645 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326264+0000", 2023-10-09T11:07:44.646 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0058858579999999999 2023-10-09T11:07:44.646 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.646 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.646 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.646 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.329650+0000", 2023-10-09T11:07:44.646 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0033856849999999998 2023-10-09T11:07:44.646 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.646 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.646 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.646 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.329705+0000", 2023-10-09T11:07:44.646 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5633000000000001e-05 2023-10-09T11:07:44.646 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.647 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.647 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:44.647 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.329758+0000", 2023-10-09T11:07:44.647 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2395999999999998e-05 2023-10-09T11:07:44.647 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.647 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.647 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.647 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.329798+0000", 2023-10-09T11:07:44.647 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0491000000000002e-05 2023-10-09T11:07:44.647 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.647 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.647 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.647 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.648 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.648 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39465 11.d 11:baf45870:::benchmark_data_smithi125_439543_object39464:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:07:44.648 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.368960+0000", 2023-10-09T11:07:44.648 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.171421737, 2023-10-09T11:07:44.648 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96470808799999996, 2023-10-09T11:07:44.648 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.648 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:44.648 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.648 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:44.648 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:44.648 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39465 2023-10-09T11:07:44.648 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.649 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368960+0000", 2023-10-09T11:07:44.649 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.649 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368960+0000", 2023-10-09T11:07:44.649 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.649 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.650 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368959+0000", 2023-10-09T11:07:44.650 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:07:44.650 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.650 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.650 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.650 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368977+0000", 2023-10-09T11:07:44.650 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8229000000000001e-05 2023-10-09T11:07:44.650 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.650 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.650 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.650 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368987+0000", 2023-10-09T11:07:44.650 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0139999999999999e-05 2023-10-09T11:07:44.650 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.651 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.651 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.651 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368993+0000", 2023-10-09T11:07:44.651 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1700000000000002e-06 2023-10-09T11:07:44.651 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.651 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.651 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.651 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.369036+0000", 2023-10-09T11:07:44.651 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2355999999999999e-05 2023-10-09T11:07:44.651 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.651 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.651 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.652 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.369042+0000", 2023-10-09T11:07:44.652 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8070000000000002e-06 2023-10-09T11:07:44.652 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.652 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.652 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.652 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.321501+0000", 2023-10-09T11:07:44.652 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95245864800000002 2023-10-09T11:07:44.652 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.652 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.652 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.652 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.321617+0000", 2023-10-09T11:07:44.652 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011572 2023-10-09T11:07:44.652 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.653 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.653 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:07:44.653 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.321752+0000", 2023-10-09T11:07:44.653 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013548 2023-10-09T11:07:44.653 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.653 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.653 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.653 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326270+0000", 2023-10-09T11:07:44.653 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045178120000000004 2023-10-09T11:07:44.653 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.653 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.653 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.653 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.329989+0000", 2023-10-09T11:07:44.654 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037189179999999999 2023-10-09T11:07:44.654 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.654 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.654 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.654 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333591+0000", 2023-10-09T11:07:44.654 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036021080000000001 2023-10-09T11:07:44.654 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.654 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.654 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:44.654 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333626+0000", 2023-10-09T11:07:44.654 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5120000000000003e-05 2023-10-09T11:07:44.654 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.655 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.655 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333668+0000", 2023-10-09T11:07:44.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1616000000000003e-05 2023-10-09T11:07:44.655 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.655 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.655 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.655 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.655 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39467 11.d 11:b2ddd126:::benchmark_data_smithi125_439543_object39466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:07:44.655 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.370134+0000", 2023-10-09T11:07:44.656 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.170246983, 2023-10-09T11:07:44.656 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96363200699999996, 2023-10-09T11:07:44.656 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.656 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:44.656 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.656 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:44.656 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:44.656 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39467 2023-10-09T11:07:44.656 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.656 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.656 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.656 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370134+0000", 2023-10-09T11:07:44.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.657 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370134+0000", 2023-10-09T11:07:44.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.657 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370133+0000", 2023-10-09T11:07:44.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:07:44.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.658 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370183+0000", 2023-10-09T11:07:44.658 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9186e-05 2023-10-09T11:07:44.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.658 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370200+0000", 2023-10-09T11:07:44.658 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6937999999999999e-05 2023-10-09T11:07:44.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.659 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.659 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370213+0000", 2023-10-09T11:07:44.661 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3599e-05 2023-10-09T11:07:44.661 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.661 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.661 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.661 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370255+0000", 2023-10-09T11:07:44.661 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1913999999999999e-05 2023-10-09T11:07:44.661 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.661 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.662 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.662 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370262+0000", 2023-10-09T11:07:44.662 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6089999999999997e-06 2023-10-09T11:07:44.662 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.662 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.662 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.662 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.323037+0000", 2023-10-09T11:07:44.662 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95277573000000004 2023-10-09T11:07:44.662 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.662 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.662 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.662 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.323131+0000", 2023-10-09T11:07:44.663 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3330000000000003e-05 2023-10-09T11:07:44.663 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.663 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.663 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:07:44.663 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.323244+0000", 2023-10-09T11:07:44.663 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000113033 2023-10-09T11:07:44.663 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.663 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.663 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.663 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.328073+0000", 2023-10-09T11:07:44.663 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0048294849999999997 2023-10-09T11:07:44.663 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.663 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.664 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.664 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330038+0000", 2023-10-09T11:07:44.664 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019642829999999998 2023-10-09T11:07:44.664 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.664 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.664 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.664 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333706+0000", 2023-10-09T11:07:44.664 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003667942 2023-10-09T11:07:44.664 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.664 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.664 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:44.664 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333732+0000", 2023-10-09T11:07:44.665 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6774000000000001e-05 2023-10-09T11:07:44.665 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.665 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.665 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.665 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333766+0000", 2023-10-09T11:07:44.665 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4178999999999999e-05 2023-10-09T11:07:44.665 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.665 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.665 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.665 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.665 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.665 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39473 11.d 11:b96957ab:::benchmark_data_smithi125_439543_object39472:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:07:44.665 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.372932+0000", 2023-10-09T11:07:44.666 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.16744906100001, 2023-10-09T11:07:44.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96091150700000005, 2023-10-09T11:07:44.666 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.666 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:44.666 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.666 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:44.666 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:44.666 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39473 2023-10-09T11:07:44.666 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.666 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.666 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.666 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.666 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.372932+0000", 2023-10-09T11:07:44.667 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.667 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.667 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.667 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.667 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.372932+0000", 2023-10-09T11:07:44.667 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.667 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.667 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.667 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.667 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.372931+0000", 2023-10-09T11:07:44.667 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:07:44.667 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.668 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.668 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.668 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.372988+0000", 2023-10-09T11:07:44.668 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6953999999999997e-05 2023-10-09T11:07:44.668 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.668 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.668 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.668 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.373007+0000", 2023-10-09T11:07:44.668 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8711000000000001e-05 2023-10-09T11:07:44.668 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.668 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.668 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.668 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.373017+0000", 2023-10-09T11:07:44.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0400999999999999e-05 2023-10-09T11:07:44.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.669 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.669 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.373082+0000", 2023-10-09T11:07:44.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4999000000000006e-05 2023-10-09T11:07:44.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.669 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.669 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.373094+0000", 2023-10-09T11:07:44.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1766000000000001e-05 2023-10-09T11:07:44.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.670 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.324554+0000", 2023-10-09T11:07:44.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95145981199999996 2023-10-09T11:07:44.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.670 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.670 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.324645+0000", 2023-10-09T11:07:44.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1250999999999998e-05 2023-10-09T11:07:44.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.670 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.670 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:07:44.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.324763+0000", 2023-10-09T11:07:44.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011804899999999999 2023-10-09T11:07:44.671 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.328078+0000", 2023-10-09T11:07:44.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0033156180000000002 2023-10-09T11:07:44.671 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330081+0000", 2023-10-09T11:07:44.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020020369999999999 2023-10-09T11:07:44.671 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333788+0000", 2023-10-09T11:07:44.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037078150000000002 2023-10-09T11:07:44.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:44.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333813+0000", 2023-10-09T11:07:44.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4239e-05 2023-10-09T11:07:44.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333844+0000", 2023-10-09T11:07:44.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1325999999999998e-05 2023-10-09T11:07:44.673 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.673 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.673 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.673 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.673 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.673 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39476 11.d 11:bbb64df7:::benchmark_data_smithi125_439543_object39475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:07:44.673 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.377052+0000", 2023-10-09T11:07:44.673 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.163329291, 2023-10-09T11:07:44.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96144097100000003, 2023-10-09T11:07:44.673 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.673 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:44.674 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.674 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:44.674 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:44.674 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39476 2023-10-09T11:07:44.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.674 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.674 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.674 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377052+0000", 2023-10-09T11:07:44.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.674 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377052+0000", 2023-10-09T11:07:44.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.675 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.675 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377050+0000", 2023-10-09T11:07:44.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:07:44.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.675 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.675 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377075+0000", 2023-10-09T11:07:44.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4941000000000001e-05 2023-10-09T11:07:44.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.676 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.676 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377090+0000", 2023-10-09T11:07:44.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.428e-05 2023-10-09T11:07:44.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.676 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.676 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377098+0000", 2023-10-09T11:07:44.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7110000000000001e-06 2023-10-09T11:07:44.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377155+0000", 2023-10-09T11:07:44.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6977e-05 2023-10-09T11:07:44.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.677 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377164+0000", 2023-10-09T11:07:44.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1920000000000005e-06 2023-10-09T11:07:44.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.677 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326460+0000", 2023-10-09T11:07:44.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.94929656600000001 2023-10-09T11:07:44.678 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.678 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.678 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326573+0000", 2023-10-09T11:07:44.678 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011275100000000001 2023-10-09T11:07:44.678 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.678 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:07:44.678 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326705+0000", 2023-10-09T11:07:44.678 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013251199999999999 2023-10-09T11:07:44.678 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330000+0000", 2023-10-09T11:07:44.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032948890000000001 2023-10-09T11:07:44.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333869+0000", 2023-10-09T11:07:44.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038684290000000001 2023-10-09T11:07:44.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338415+0000", 2023-10-09T11:07:44.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045463700000000001 2023-10-09T11:07:44.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:44.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338458+0000", 2023-10-09T11:07:44.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2852999999999998e-05 2023-10-09T11:07:44.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338493+0000", 2023-10-09T11:07:44.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5231000000000002e-05 2023-10-09T11:07:44.680 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.680 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.681 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.681 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.681 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39490 11.d 11:b79c85dd:::benchmark_data_smithi125_439543_object39489:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:07:44.681 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.388748+0000", 2023-10-09T11:07:44.681 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.15163346899999, 2023-10-09T11:07:44.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.94984969600000002, 2023-10-09T11:07:44.681 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.681 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:44.681 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.681 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:44.681 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:44.681 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39490 2023-10-09T11:07:44.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.682 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.682 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.682 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388748+0000", 2023-10-09T11:07:44.682 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.682 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.682 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388748+0000", 2023-10-09T11:07:44.682 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388746+0000", 2023-10-09T11:07:44.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:07:44.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388771+0000", 2023-10-09T11:07:44.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4868e-05 2023-10-09T11:07:44.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388789+0000", 2023-10-09T11:07:44.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8077e-05 2023-10-09T11:07:44.684 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.684 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.684 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.684 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388801+0000", 2023-10-09T11:07:44.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1698999999999999e-05 2023-10-09T11:07:44.684 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.684 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.684 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.684 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388915+0000", 2023-10-09T11:07:44.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000114003 2023-10-09T11:07:44.684 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.685 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.685 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388926+0000", 2023-10-09T11:07:44.685 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0729e-05 2023-10-09T11:07:44.685 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.685 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.685 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.328103+0000", 2023-10-09T11:07:44.685 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.93917761600000005 2023-10-09T11:07:44.685 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.685 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.685 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.328228+0000", 2023-10-09T11:07:44.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012434699999999999 2023-10-09T11:07:44.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:07:44.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.328355+0000", 2023-10-09T11:07:44.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012757299999999999 2023-10-09T11:07:44.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330045+0000", 2023-10-09T11:07:44.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001689968 2023-10-09T11:07:44.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.337060+0000", 2023-10-09T11:07:44.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0070151830000000004 2023-10-09T11:07:44.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338523+0000", 2023-10-09T11:07:44.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001462514 2023-10-09T11:07:44.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:44.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338557+0000", 2023-10-09T11:07:44.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3948000000000001e-05 2023-10-09T11:07:44.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.688 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338598+0000", 2023-10-09T11:07:44.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0923000000000001e-05 2023-10-09T11:07:44.688 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.688 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.688 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.688 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39501 11.d 11:b409f08d:::benchmark_data_smithi125_439543_object39500:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:07:44.689 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.397575+0000", 2023-10-09T11:07:44.689 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 130.14280683300001, 2023-10-09T11:07:44.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.94120543800000001, 2023-10-09T11:07:44.689 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:07:44.689 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:07:44.689 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:07:44.689 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:07:44.689 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:07:44.689 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39501 2023-10-09T11:07:44.689 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.689 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:07:44.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:07:44.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397575+0000", 2023-10-09T11:07:44.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:07:44.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397575+0000", 2023-10-09T11:07:44.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:07:44.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:07:44.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397573+0000", 2023-10-09T11:07:44.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:07:44.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:07:44.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397613+0000", 2023-10-09T11:07:44.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0145000000000003e-05 2023-10-09T11:07:44.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:07:44.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397626+0000", 2023-10-09T11:07:44.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3421000000000001e-05 2023-10-09T11:07:44.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:07:44.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397634+0000", 2023-10-09T11:07:44.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6210000000000004e-06 2023-10-09T11:07:44.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397700+0000", 2023-10-09T11:07:44.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5512000000000006e-05 2023-10-09T11:07:44.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:07:44.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397709+0000", 2023-10-09T11:07:44.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3360000000000001e-06 2023-10-09T11:07:44.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:07:44.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330105+0000", 2023-10-09T11:07:44.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.93239575399999997 2023-10-09T11:07:44.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:07:44.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330264+0000", 2023-10-09T11:07:44.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015971200000000001 2023-10-09T11:07:44.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:07:44.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330365+0000", 2023-10-09T11:07:44.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000100578 2023-10-09T11:07:44.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:07:44.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333554+0000", 2023-10-09T11:07:44.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003188897 2023-10-09T11:07:44.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.337092+0000", 2023-10-09T11:07:44.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003537756 2023-10-09T11:07:44.695 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:07:44.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338698+0000", 2023-10-09T11:07:44.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001606648 2023-10-09T11:07:44.695 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:07:44.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338750+0000", 2023-10-09T11:07:44.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1326999999999998e-05 2023-10-09T11:07:44.696 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:07:44.696 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:07:44.696 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:07:44.696 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338780+0000", 2023-10-09T11:07:44.696 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0420999999999999e-05 2023-10-09T11:07:44.696 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.696 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.696 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.696 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:07:44.696 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:07:44.696 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:07:44.697 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:07:44.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:44 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 6040/131844 objects degraded (4.581%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:44.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:44 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:44.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:44 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:44.961 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:44.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:44.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:45.069 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:45.110 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:45.143 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:45.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:45.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:45.249 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:45.301 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:45.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:45 smithi125 ceph-mon[195507]: pgmap v1350: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 6040/122124 objects degraded (4.946%); 4708/122124 objects misplaced (3.855%); 5.2 MiB/s, 1 keys/s, 84 objects/s recovering 2023-10-09T11:07:45.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:07:45.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:07:45.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:45.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:07:45.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:45 smithi125 ceph-mon[200197]: pgmap v1350: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 6040/122124 objects degraded (4.946%); 4708/122124 objects misplaced (3.855%); 5.2 MiB/s, 1 keys/s, 84 objects/s recovering 2023-10-09T11:07:45.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:07:45.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:07:45.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:45.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:07:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:45 smithi155 ceph-mon[184854]: pgmap v1350: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 6040/122124 objects degraded (4.946%); 4708/122124 objects misplaced (3.855%); 5.2 MiB/s, 1 keys/s, 84 objects/s recovering 2023-10-09T11:07:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:07:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:07:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:07:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:07:46.602 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:46.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:46.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:46.607 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:46.708 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:46.719 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:46 smithi155 ceph-mon[184854]: pgmap v1351: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 6040/122124 objects degraded (4.946%); 4708/122124 objects misplaced (3.855%); 4.3 MiB/s, 1 keys/s, 70 objects/s recovering 2023-10-09T11:07:46.750 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:46 smithi125 ceph-mon[195507]: pgmap v1351: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 6040/122124 objects degraded (4.946%); 4708/122124 objects misplaced (3.855%); 4.3 MiB/s, 1 keys/s, 70 objects/s recovering 2023-10-09T11:07:46.751 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:46 smithi125 ceph-mon[200197]: pgmap v1351: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 6040/122124 objects degraded (4.946%); 4708/122124 objects misplaced (3.855%); 4.3 MiB/s, 1 keys/s, 70 objects/s recovering 2023-10-09T11:07:46.759 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:46.834 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:46.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:46.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:46.859 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T11:07:46.940 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:46.994 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:47.243 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:07:47.435 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:47.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:47.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:47.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:47.598 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:47.637 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:47.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:47.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:47.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:47.784 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:47.836 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:48.496 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:07:48.496 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:07:48.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:07:48.496 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36510.0:163788 11.d e554/531)", 2023-10-09T11:07:48.497 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:07:48.494823+0000", 2023-10-09T11:07:48.497 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.00011903700000000001, 2023-10-09T11:07:48.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000149877, 2023-10-09T11:07:48.497 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:07:48.497 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "queued for pg", 2023-10-09T11:07:48.497 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:07:48.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:07:48.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:07:48.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:48.494823+0000", 2023-10-09T11:07:48.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:07:48.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:07:48.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:07:48.498 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:07:48.498 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:48.494823+0000", 2023-10-09T11:07:48.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:07:48.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:07:48.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:07:48.498 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:07:48.498 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:48.494821+0000", 2023-10-09T11:07:48.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:07:48.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:07:48.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:07:48.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:07:48.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:48.494831+0000", 2023-10-09T11:07:48.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.302e-06 2023-10-09T11:07:48.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:07:48.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:07:48.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:07:48.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:48.494844+0000", 2023-10-09T11:07:48.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3560999999999999e-05 2023-10-09T11:07:48.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:07:48.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:07:48.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:07:48.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:48.494850+0000", 2023-10-09T11:07:48.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.122e-06 2023-10-09T11:07:48.500 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:07:48.500 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:07:48.500 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:07:48.500 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:07:48.500 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T11:07:48.500 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 1 2023-10-09T11:07:48.500 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:07:48.711 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:48.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:48.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:48.816 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:48.862 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:48.899 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T11:07:48.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:48 smithi125 ceph-mon[195507]: pgmap v1352: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 6040/118047 objects degraded (5.117%); 4708/118047 objects misplaced (3.988%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T11:07:48.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:48 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3807925233' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:07:48.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:48 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:07:48.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:48 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 6040/118047 objects degraded (5.117%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:48.971 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:48 smithi125 ceph-mon[200197]: pgmap v1352: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 6040/118047 objects degraded (5.117%); 4708/118047 objects misplaced (3.988%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T11:07:48.971 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:48 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3807925233' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:07:48.971 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:48 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:07:48.971 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:48 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 6040/118047 objects degraded (5.117%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:48.981 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T11:07:49.097 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:49.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:49.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:49.205 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:48 smithi155 ceph-mon[184854]: pgmap v1352: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 6040/118047 objects degraded (5.117%); 4708/118047 objects misplaced (3.988%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T11:07:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:48 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3807925233' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:07:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:48 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:07:49.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:48 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 6040/118047 objects degraded (5.117%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:49.270 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:49.285 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:07:49.350 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:49.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:49.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:49.468 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:49.519 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:49.608 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:49.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:49.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:49.716 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:49.781 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:50.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:50 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T11:07:50.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:50 smithi125 ceph-mon[200197]: osdmap e555: 8 total, 8 up, 8 in 2023-10-09T11:07:50.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:50 smithi125 ceph-mon[200197]: failed to encode map e555 with expected crc 2023-10-09T11:07:50.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:50 smithi125 ceph-mon[200197]: failed to encode map e555 with expected crc 2023-10-09T11:07:50.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:50 smithi125 ceph-mon[200197]: failed to encode map e555 with expected crc 2023-10-09T11:07:50.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:50 smithi125 ceph-mon[200197]: failed to encode map e555 with expected crc 2023-10-09T11:07:50.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:50 smithi125 ceph-mon[200197]: failed to encode map e555 with expected crc 2023-10-09T11:07:50.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:49 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T11:07:50.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:49 smithi125 ceph-mon[195507]: osdmap e555: 8 total, 8 up, 8 in 2023-10-09T11:07:50.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:49 smithi125 ceph-mon[195507]: failed to encode map e555 with expected crc 2023-10-09T11:07:50.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:49 smithi125 ceph-mon[195507]: failed to encode map e555 with expected crc 2023-10-09T11:07:50.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:49 smithi125 ceph-mon[195507]: failed to encode map e555 with expected crc 2023-10-09T11:07:50.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:49 smithi125 ceph-mon[195507]: failed to encode map e555 with expected crc 2023-10-09T11:07:50.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:49 smithi125 ceph-mon[195507]: failed to encode map e555 with expected crc 2023-10-09T11:07:50.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:49 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T11:07:50.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:49 smithi155 ceph-mon[184854]: osdmap e555: 8 total, 8 up, 8 in 2023-10-09T11:07:50.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:49 smithi155 ceph-mon[184854]: failed to encode map e555 with expected crc 2023-10-09T11:07:50.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:49 smithi155 ceph-mon[184854]: failed to encode map e555 with expected crc 2023-10-09T11:07:50.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:49 smithi155 ceph-mon[184854]: failed to encode map e555 with expected crc 2023-10-09T11:07:50.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:49 smithi155 ceph-mon[184854]: failed to encode map e555 with expected crc 2023-10-09T11:07:50.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:49 smithi155 ceph-mon[184854]: failed to encode map e555 with expected crc 2023-10-09T11:07:50.319 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T11:07:50.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:07:50.384 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:50.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:50.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:50.407 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:07:50.408 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:07:50.408 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T11:07:50.408 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T11:07:50.408 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:07:50.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:50.560 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:50.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:50.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:50.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:50.719 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:50.797 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T11:07:50.802 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:50.869 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:50.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:50.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:50.949 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:50.975 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:51.023 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:07:51.027 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:07:51.055 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:51.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:51 smithi125 ceph-mon[195507]: pgmap v1354: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 6040/107838 objects degraded (5.601%); 3898/107838 objects misplaced (3.615%); 4.3 MiB/s, 74 objects/s recovering 2023-10-09T11:07:51.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/556633497' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:07:51.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:51 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:07:51.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/226721656' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:51.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:51 smithi125 ceph-mon[200197]: pgmap v1354: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 6040/107838 objects degraded (5.601%); 3898/107838 objects misplaced (3.615%); 4.3 MiB/s, 74 objects/s recovering 2023-10-09T11:07:51.072 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/556633497' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:07:51.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:51 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:07:51.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/226721656' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:51.078 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:07:51.131 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:07:51.142 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:07:51.164 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:07:51.176 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:51.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:51.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:51.198 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:07:51.228 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:07:51.258 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:07:51.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:07:51.288 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:51 smithi155 ceph-mon[184854]: pgmap v1354: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 6040/107838 objects degraded (5.601%); 3898/107838 objects misplaced (3.615%); 4.3 MiB/s, 74 objects/s recovering 2023-10-09T11:07:51.289 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/556633497' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:07:51.289 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:51 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:07:51.289 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/226721656' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:07:51.292 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:07:51.321 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:07:51.321 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:07:51.322 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:07:51.322 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:07:51.322 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:07:51.322 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:07:51.322 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:07:51.322 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:07:51.323 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:07:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:07:51] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2023-10-09T11:07:51.328 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:51.709 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:07:51.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:51.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:07:51.815 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:51.886 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:52.028 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:52.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:52.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:52.134 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:52.192 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:07:52.192 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:07:52.192 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[195507]: osdmap e556: 8 total, 8 up, 8 in 2023-10-09T11:07:52.192 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[195507]: failed to encode map e556 with expected crc 2023-10-09T11:07:52.192 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[195507]: failed to encode map e556 with expected crc 2023-10-09T11:07:52.192 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[195507]: failed to encode map e556 with expected crc 2023-10-09T11:07:52.193 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[195507]: failed to encode map e556 with expected crc 2023-10-09T11:07:52.193 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[195507]: failed to encode map e556 with expected crc 2023-10-09T11:07:52.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:07:52.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:07:52.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[200197]: osdmap e556: 8 total, 8 up, 8 in 2023-10-09T11:07:52.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[200197]: failed to encode map e556 with expected crc 2023-10-09T11:07:52.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[200197]: failed to encode map e556 with expected crc 2023-10-09T11:07:52.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[200197]: failed to encode map e556 with expected crc 2023-10-09T11:07:52.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[200197]: failed to encode map e556 with expected crc 2023-10-09T11:07:52.194 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:52 smithi125 ceph-mon[200197]: failed to encode map e556 with expected crc 2023-10-09T11:07:52.194 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:52.217 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:52 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:07:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:52 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:07:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:52 smithi155 ceph-mon[184854]: osdmap e556: 8 total, 8 up, 8 in 2023-10-09T11:07:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:52 smithi155 ceph-mon[184854]: failed to encode map e556 with expected crc 2023-10-09T11:07:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:52 smithi155 ceph-mon[184854]: failed to encode map e556 with expected crc 2023-10-09T11:07:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:52 smithi155 ceph-mon[184854]: failed to encode map e556 with expected crc 2023-10-09T11:07:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:52 smithi155 ceph-mon[184854]: failed to encode map e556 with expected crc 2023-10-09T11:07:52.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:52 smithi155 ceph-mon[184854]: failed to encode map e556 with expected crc 2023-10-09T11:07:52.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:52.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:52.285 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:07:52.285 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:07:52.325 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:52.369 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:52.400 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:52.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:52.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:52.508 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:07:52.580 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:52.613 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:07:52.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:52.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:07:52.722 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:52.764 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:52.950 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:52.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:52.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:52.976 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:07:52.998 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:07:53.060 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:53.103 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:53.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:53 smithi125 ceph-mon[195507]: pgmap v1356: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 6040/103887 objects degraded (5.814%); 3898/103887 objects misplaced (3.752%); 2.2 MiB/s, 41 objects/s recovering 2023-10-09T11:07:53.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:53 smithi125 ceph-mon[200197]: pgmap v1356: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 6040/103887 objects degraded (5.814%); 3898/103887 objects misplaced (3.752%); 2.2 MiB/s, 41 objects/s recovering 2023-10-09T11:07:53.407 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1356,"stamp":"2023-10-09T11:07:51.897098+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2268373884,"num_objects":34629,"num_object_clones":0,"num_object_copies":103887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6040,"num_objects_misplaced":3898,"num_objects_unfound":0,"num_objects_dirty":34629,"num_whiteouts":0,"num_read":32,"num_read_kb":57,"num_write":164466,"num_write_kb":6367921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22584,"num_bytes_recovered":1451977034,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":233232,"ondisk_log_size":233232,"up":417,"acting":413,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":429,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14105276,"kb_used_data":7434876,"kb_used_omap":134,"kb_used_meta":6669625,"kb_avail":735855940,"statfs":{"total":767960285184,"available":753516482560,"internally_reserved":0,"allocated":7613313024,"data_stored":7594274503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":137532,"internal_metadata":6829696708},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,5,6,3,6,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-398393344,"num_objects":-6079,"num_object_clones":0,"num_object_copies":-18237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-810,"num_objects_unfound":0,"num_objects_dirty":-6079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6079,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":334,"num_bytes_recovered":18677760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002583"},"pg_stats":[{"pgid":"11.23","version":"554'5824","reported_seq":8842,"reported_epoch":554,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:07:48.194933+0000","last_change":"2023-10-09T11:07:34.314839+0000","last_active":"2023-10-09T11:07:48.194933+0000","last_peered":"2023-10-09T11:07:48.194933+0000","last_clean":"2023-10-09T11:07:32.177661+0000","last_became_active":"2023-10-09T11:07:34.308281+0000","last_became_peered":"2023-10-09T11:07:34.308281+0000","last_unstale":"2023-10-09T11:07:48.194933+0000","last_undegraded":"2023-10-09T11:07:48.194933+0000","last_fullsized":"2023-10-09T11:07:48.194933+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36847616,"num_objects":562,"num_object_clones":0,"num_object_copies":1686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":562,"num_objects_unfound":0,"num_objects_dirty":562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2608,"num_write_kb":101456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1213,"num_bytes_recovered":72024064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":562}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.22","version":"554'5610","reported_seq":5697,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.261770+0000","last_change":"2023-10-09T11:06:43.010089+0000","last_active":"2023-10-09T11:07:48.261770+0000","last_peered":"2023-10-09T11:07:48.261770+0000","last_clean":"2023-10-09T11:07:48.261770+0000","last_became_active":"2023-10-09T11:06:43.008608+0000","last_became_peered":"2023-10-09T11:06:43.008608+0000","last_unstale":"2023-10-09T11:07:48.261770+0000","last_undegraded":"2023-10-09T11:07:48.261770+0000","last_fullsized":"2023-10-09T11:07:48.261770+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5610,"log_dups_size":0,"ondisk_log_size":5610,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35913728,"num_objects":548,"num_object_clones":0,"num_object_copies":1644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2507,"num_write_kb":97760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.21","version":"554'5901","reported_seq":6767,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.137961+0000","last_change":"2023-10-09T11:07:23.970291+0000","last_active":"2023-10-09T11:07:48.137961+0000","last_peered":"2023-10-09T11:07:48.137961+0000","last_clean":"2023-10-09T11:07:48.137961+0000","last_became_active":"2023-10-09T11:06:43.015074+0000","last_became_peered":"2023-10-09T11:06:43.015074+0000","last_unstale":"2023-10-09T11:07:48.137961+0000","last_undegraded":"2023-10-09T11:07:48.137961+0000","last_fullsized":"2023-10-09T11:07:48.137961+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5901,"log_dups_size":0,"ondisk_log_size":5901,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31440896,"num_objects":480,"num_object_clones":0,"num_object_copies":1440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2668,"num_write_kb":100720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":344,"num_bytes_recovered":22511616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.20","version":"554'5701","reported_seq":8106,"reported_epoch":554,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:07:48.010155+0000","last_change":"2023-10-09T11:07:34.316666+0000","last_active":"2023-10-09T11:07:48.010155+0000","last_peered":"2023-10-09T11:07:48.010155+0000","last_clean":"2023-10-09T11:07:32.195530+0000","last_became_active":"2023-10-09T11:07:34.309531+0000","last_became_peered":"2023-10-09T11:07:34.309531+0000","last_unstale":"2023-10-09T11:07:48.010155+0000","last_undegraded":"2023-10-09T11:07:48.010155+0000","last_fullsized":"2023-10-09T11:07:48.010155+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38617088,"num_objects":589,"num_object_clones":0,"num_object_copies":1767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":589,"num_objects_unfound":0,"num_objects_dirty":589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2535,"num_write_kb":99984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1150,"num_bytes_recovered":72941568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,6],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,3,6","objects":589}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.12","version":"554'6632","reported_seq":6709,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.272994+0000","last_change":"2023-10-09T11:05:45.369342+0000","last_active":"2023-10-09T11:07:48.272994+0000","last_peered":"2023-10-09T11:07:48.272994+0000","last_clean":"2023-10-09T11:07:48.272994+0000","last_became_active":"2023-10-09T11:05:45.367721+0000","last_became_peered":"2023-10-09T11:05:45.367721+0000","last_unstale":"2023-10-09T11:07:48.272994+0000","last_undegraded":"2023-10-09T11:07:48.272994+0000","last_fullsized":"2023-10-09T11:07:48.272994+0000","mapping_epoch":509,"log_start":"498'703","ondisk_log_start":"498'703","created":509,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5929,"log_dups_size":0,"ondisk_log_size":5929,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70254592,"num_objects":1072,"num_object_clones":0,"num_object_copies":3216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5094,"num_write_kb":197312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"76'161","reported_seq":843,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.974725+0000","last_change":"2023-10-09T11:07:36.287455+0000","last_active":"2023-10-09T11:07:48.974725+0000","last_peered":"2023-10-09T11:07:48.974725+0000","last_clean":"2023-10-09T11:07:48.974725+0000","last_became_active":"2023-10-09T11:07:36.287052+0000","last_became_peered":"2023-10-09T11:07:36.287052+0000","last_unstale":"2023-10-09T11:07:48.974725+0000","last_undegraded":"2023-10-09T11:07:48.974725+0000","last_fullsized":"2023-10-09T11:07:48.974725+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:18.864873+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":338,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.985114+0000","last_change":"2023-10-09T11:03:21.185383+0000","last_active":"2023-10-09T11:07:48.985114+0000","last_peered":"2023-10-09T11:07:48.985114+0000","last_clean":"2023-10-09T11:07:48.985114+0000","last_became_active":"2023-10-09T11:03:21.185022+0000","last_became_peered":"2023-10-09T11:03:21.185022+0000","last_unstale":"2023-10-09T11:07:48.985114+0000","last_undegraded":"2023-10-09T11:07:48.985114+0000","last_fullsized":"2023-10-09T11:07:48.985114+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:05.230434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":80,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.268313+0000","last_change":"2023-10-09T11:04:50.319477+0000","last_active":"2023-10-09T11:07:36.268313+0000","last_peered":"2023-10-09T11:07:36.268313+0000","last_clean":"2023-10-09T11:07:36.268313+0000","last_became_active":"2023-10-09T11:04:50.319148+0000","last_became_peered":"2023-10-09T11:04:50.319148+0000","last_unstale":"2023-10-09T11:07:36.268313+0000","last_undegraded":"2023-10-09T11:07:36.268313+0000","last_fullsized":"2023-10-09T11:07:36.268313+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:48.725566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"554'6834","reported_seq":8242,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.273321+0000","last_change":"2023-10-09T11:06:54.752222+0000","last_active":"2023-10-09T11:07:48.273321+0000","last_peered":"2023-10-09T11:07:48.273321+0000","last_clean":"2023-10-09T11:07:48.273321+0000","last_became_active":"2023-10-09T11:05:45.395271+0000","last_became_peered":"2023-10-09T11:05:45.395271+0000","last_unstale":"2023-10-09T11:07:48.273321+0000","last_undegraded":"2023-10-09T11:07:48.273321+0000","last_fullsized":"2023-10-09T11:07:48.273321+0000","mapping_epoch":509,"log_start":"498'808","ondisk_log_start":"498'808","created":509,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6026,"log_dups_size":0,"ondisk_log_size":6026,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79003648,"num_objects":1206,"num_object_clones":0,"num_object_copies":3618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5216,"num_write_kb":205472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":659,"num_bytes_recovered":43188224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":713,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975112+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:07:48.975112+0000","last_peered":"2023-10-09T11:07:48.975112+0000","last_clean":"2023-10-09T11:07:48.975112+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:07:48.975112+0000","last_undegraded":"2023-10-09T11:07:48.975112+0000","last_fullsized":"2023-10-09T11:07:48.975112+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":268,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.275349+0000","last_change":"2023-10-09T11:04:50.333711+0000","last_active":"2023-10-09T11:07:36.275349+0000","last_peered":"2023-10-09T11:07:36.275349+0000","last_clean":"2023-10-09T11:07:36.275349+0000","last_became_active":"2023-10-09T11:04:50.333262+0000","last_became_peered":"2023-10-09T11:04:50.333262+0000","last_unstale":"2023-10-09T11:07:36.275349+0000","last_undegraded":"2023-10-09T11:07:36.275349+0000","last_fullsized":"2023-10-09T11:07:36.275349+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.843765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":16,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976642+0000","last_change":"2023-10-09T11:07:33.237903+0000","last_active":"2023-10-09T11:07:48.976642+0000","last_peered":"2023-10-09T11:07:48.976642+0000","last_clean":"2023-10-09T11:07:48.976642+0000","last_became_active":"2023-10-09T11:07:33.237618+0000","last_became_peered":"2023-10-09T11:07:33.237618+0000","last_unstale":"2023-10-09T11:07:48.976642+0000","last_undegraded":"2023-10-09T11:07:48.976642+0000","last_fullsized":"2023-10-09T11:07:48.976642+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:25.600818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.10","version":"554'6691","reported_seq":7375,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:07:48.308343+0000","last_change":"2023-10-09T11:07:34.347309+0000","last_active":"2023-10-09T11:07:48.308343+0000","last_peered":"2023-10-09T11:07:48.308343+0000","last_clean":"2023-10-09T11:06:06.955693+0000","last_became_active":"2023-10-09T11:07:33.305104+0000","last_became_peered":"2023-10-09T11:07:33.305104+0000","last_unstale":"2023-10-09T11:07:48.308343+0000","last_undegraded":"2023-10-09T11:07:34.345913+0000","last_fullsized":"2023-10-09T11:07:33.208203+0000","mapping_epoch":549,"log_start":"498'929","ondisk_log_start":"498'929","created":509,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5762,"log_dups_size":0,"ondisk_log_size":5762,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73170944,"num_objects":1117,"num_object_clones":0,"num_object_copies":3351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2269,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5129,"num_write_kb":199840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":292,"num_bytes_recovered":19136512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":461},{"shards":"1,3,6","objects":656}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":731,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976012+0000","last_change":"2023-10-09T11:07:36.284534+0000","last_active":"2023-10-09T11:07:48.976012+0000","last_peered":"2023-10-09T11:07:48.976012+0000","last_clean":"2023-10-09T11:07:48.976012+0000","last_became_active":"2023-10-09T11:07:36.284254+0000","last_became_peered":"2023-10-09T11:07:36.284254+0000","last_unstale":"2023-10-09T11:07:48.976012+0000","last_undegraded":"2023-10-09T11:07:48.976012+0000","last_fullsized":"2023-10-09T11:07:48.976012+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:19.159630+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":80,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.260167+0000","last_change":"2023-10-09T11:04:50.310477+0000","last_active":"2023-10-09T11:07:36.260167+0000","last_peered":"2023-10-09T11:07:36.260167+0000","last_clean":"2023-10-09T11:07:36.260167+0000","last_became_active":"2023-10-09T11:04:50.310067+0000","last_became_peered":"2023-10-09T11:04:50.310067+0000","last_unstale":"2023-10-09T11:07:36.260167+0000","last_undegraded":"2023-10-09T11:07:36.260167+0000","last_fullsized":"2023-10-09T11:07:36.260167+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:52.884716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":365,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977865+0000","last_change":"2023-10-09T11:05:20.310581+0000","last_active":"2023-10-09T11:07:48.977865+0000","last_peered":"2023-10-09T11:07:48.977865+0000","last_clean":"2023-10-09T11:07:48.977865+0000","last_became_active":"2023-10-09T11:05:20.309830+0000","last_became_peered":"2023-10-09T11:05:20.309830+0000","last_unstale":"2023-10-09T11:07:48.977865+0000","last_undegraded":"2023-10-09T11:07:48.977865+0000","last_fullsized":"2023-10-09T11:07:48.977865+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:24.508211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1f","version":"554'8417","reported_seq":13623,"reported_epoch":554,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T11:07:48.304846+0000","last_change":"2023-10-09T11:07:34.394394+0000","last_active":"2023-10-09T11:07:48.304846+0000","last_peered":"2023-10-09T11:07:48.304846+0000","last_clean":"2023-10-09T11:06:06.963741+0000","last_became_active":"2023-10-09T11:07:34.331654+0000","last_became_peered":"2023-10-09T11:07:34.331654+0000","last_unstale":"2023-10-09T11:07:48.304846+0000","last_undegraded":"2023-10-09T11:07:48.304846+0000","last_fullsized":"2023-10-09T11:07:34.216155+0000","mapping_epoch":550,"log_start":"504'1895","ondisk_log_start":"504'1895","created":531,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6522,"log_dups_size":0,"ondisk_log_size":6522,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72941568,"num_objects":1113,"num_object_clones":0,"num_object_copies":3339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5187,"num_write_kb":201600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2559,"num_bytes_recovered":164298752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":1113}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":771,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976427+0000","last_change":"2023-10-09T11:07:36.282502+0000","last_active":"2023-10-09T11:07:48.976427+0000","last_peered":"2023-10-09T11:07:48.976427+0000","last_clean":"2023-10-09T11:07:48.976427+0000","last_became_active":"2023-10-09T11:07:36.282133+0000","last_became_peered":"2023-10-09T11:07:36.282133+0000","last_unstale":"2023-10-09T11:07:48.976427+0000","last_undegraded":"2023-10-09T11:07:48.976427+0000","last_fullsized":"2023-10-09T11:07:48.976427+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:03.762383+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":192,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977943+0000","last_change":"2023-10-09T11:06:24.245488+0000","last_active":"2023-10-09T11:07:48.977943+0000","last_peered":"2023-10-09T11:07:48.977943+0000","last_clean":"2023-10-09T11:07:48.977943+0000","last_became_active":"2023-10-09T11:06:24.245167+0000","last_became_peered":"2023-10-09T11:06:24.245167+0000","last_unstale":"2023-10-09T11:07:48.977943+0000","last_undegraded":"2023-10-09T11:07:48.977943+0000","last_fullsized":"2023-10-09T11:07:48.977943+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:31.702568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":89,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.260255+0000","last_change":"2023-10-09T11:05:20.299453+0000","last_active":"2023-10-09T11:07:36.260255+0000","last_peered":"2023-10-09T11:07:36.260255+0000","last_clean":"2023-10-09T11:07:36.260255+0000","last_became_active":"2023-10-09T11:05:20.299117+0000","last_became_peered":"2023-10-09T11:05:20.299117+0000","last_unstale":"2023-10-09T11:07:36.260255+0000","last_undegraded":"2023-10-09T11:07:36.260255+0000","last_fullsized":"2023-10-09T11:07:36.260255+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:30.072813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1e","version":"554'8220","reported_seq":8297,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.176769+0000","last_change":"2023-10-09T11:06:43.029410+0000","last_active":"2023-10-09T11:07:48.176769+0000","last_peered":"2023-10-09T11:07:48.176769+0000","last_clean":"2023-10-09T11:07:48.176769+0000","last_became_active":"2023-10-09T11:06:43.026551+0000","last_became_peered":"2023-10-09T11:06:43.026551+0000","last_unstale":"2023-10-09T11:07:48.176769+0000","last_undegraded":"2023-10-09T11:07:48.176769+0000","last_fullsized":"2023-10-09T11:07:48.176769+0000","mapping_epoch":531,"log_start":"504'1803","ondisk_log_start":"504'1803","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6417,"log_dups_size":0,"ondisk_log_size":6417,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73990144,"num_objects":1129,"num_object_clones":0,"num_object_copies":3387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5045,"num_write_kb":197568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":885,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.979362+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:07:48.979362+0000","last_peered":"2023-10-09T11:07:48.979362+0000","last_clean":"2023-10-09T11:07:48.979362+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:07:48.979362+0000","last_undegraded":"2023-10-09T11:07:48.979362+0000","last_fullsized":"2023-10-09T11:07:48.979362+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":427,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975786+0000","last_change":"2023-10-09T11:07:33.242665+0000","last_active":"2023-10-09T11:07:48.975786+0000","last_peered":"2023-10-09T11:07:48.975786+0000","last_clean":"2023-10-09T11:07:48.975786+0000","last_became_active":"2023-10-09T11:07:33.242313+0000","last_became_peered":"2023-10-09T11:07:33.242313+0000","last_unstale":"2023-10-09T11:07:48.975786+0000","last_undegraded":"2023-10-09T11:07:48.975786+0000","last_fullsized":"2023-10-09T11:07:48.975786+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:20.565135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":107,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.268766+0000","last_change":"2023-10-09T11:07:33.225683+0000","last_active":"2023-10-09T11:07:36.268766+0000","last_peered":"2023-10-09T11:07:36.268766+0000","last_clean":"2023-10-09T11:07:36.268766+0000","last_became_active":"2023-10-09T11:07:33.225275+0000","last_became_peered":"2023-10-09T11:07:33.225275+0000","last_unstale":"2023-10-09T11:07:36.268766+0000","last_undegraded":"2023-10-09T11:07:36.268766+0000","last_fullsized":"2023-10-09T11:07:36.268766+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:10.131060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1d","version":"556'8498","reported_seq":11415,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.185371+0000","last_change":"2023-10-09T11:07:23.727748+0000","last_active":"2023-10-09T11:07:51.185371+0000","last_peered":"2023-10-09T11:07:51.185371+0000","last_clean":"2023-10-09T11:07:51.185371+0000","last_became_active":"2023-10-09T11:06:43.012614+0000","last_became_peered":"2023-10-09T11:06:43.012614+0000","last_unstale":"2023-10-09T11:07:51.185371+0000","last_undegraded":"2023-10-09T11:07:51.185371+0000","last_fullsized":"2023-10-09T11:07:51.185371+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":8498,"log_dups_size":0,"ondisk_log_size":8498,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66093068,"num_objects":1009,"num_object_clones":0,"num_object_copies":3027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1009,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5277,"num_write_kb":201121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":704,"num_bytes_recovered":46137344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":785,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977687+0000","last_change":"2023-10-09T11:05:23.328981+0000","last_active":"2023-10-09T11:07:48.977687+0000","last_peered":"2023-10-09T11:07:48.977687+0000","last_clean":"2023-10-09T11:07:48.977687+0000","last_became_active":"2023-10-09T11:05:23.328616+0000","last_became_peered":"2023-10-09T11:05:23.328616+0000","last_unstale":"2023-10-09T11:07:48.977687+0000","last_undegraded":"2023-10-09T11:07:48.977687+0000","last_fullsized":"2023-10-09T11:07:48.977687+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:15.315819+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":22,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976339+0000","last_change":"2023-10-09T11:07:33.265842+0000","last_active":"2023-10-09T11:07:48.976339+0000","last_peered":"2023-10-09T11:07:48.976339+0000","last_clean":"2023-10-09T11:07:48.976339+0000","last_became_active":"2023-10-09T11:07:33.232311+0000","last_became_peered":"2023-10-09T11:07:33.232311+0000","last_unstale":"2023-10-09T11:07:48.976339+0000","last_undegraded":"2023-10-09T11:07:48.976339+0000","last_fullsized":"2023-10-09T11:07:48.976339+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:23.402582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":334,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975945+0000","last_change":"2023-10-09T11:05:20.316265+0000","last_active":"2023-10-09T11:07:48.975945+0000","last_peered":"2023-10-09T11:07:48.975945+0000","last_clean":"2023-10-09T11:07:48.975945+0000","last_became_active":"2023-10-09T11:05:20.315853+0000","last_became_peered":"2023-10-09T11:05:20.315853+0000","last_unstale":"2023-10-09T11:07:48.975945+0000","last_undegraded":"2023-10-09T11:07:48.975945+0000","last_fullsized":"2023-10-09T11:07:48.975945+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:35.093981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1c","version":"554'8288","reported_seq":8365,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.307311+0000","last_change":"2023-10-09T11:06:43.025845+0000","last_active":"2023-10-09T11:07:48.307311+0000","last_peered":"2023-10-09T11:07:48.307311+0000","last_clean":"2023-10-09T11:07:48.307311+0000","last_became_active":"2023-10-09T11:06:43.022156+0000","last_became_peered":"2023-10-09T11:06:43.022156+0000","last_unstale":"2023-10-09T11:07:48.307311+0000","last_undegraded":"2023-10-09T11:07:48.307311+0000","last_fullsized":"2023-10-09T11:07:48.307311+0000","mapping_epoch":531,"log_start":"504'1919","ondisk_log_start":"504'1919","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6369,"log_dups_size":0,"ondisk_log_size":6369,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76251136,"num_objects":1164,"num_object_clones":0,"num_object_copies":3492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5080,"num_write_kb":199776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":946,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.278858+0000","last_change":"2023-10-09T11:04:53.274921+0000","last_active":"2023-10-09T11:07:36.278858+0000","last_peered":"2023-10-09T11:07:36.278858+0000","last_clean":"2023-10-09T11:07:36.278858+0000","last_became_active":"2023-10-09T11:04:53.274518+0000","last_became_peered":"2023-10-09T11:04:53.274518+0000","last_unstale":"2023-10-09T11:07:36.278858+0000","last_undegraded":"2023-10-09T11:07:36.278858+0000","last_fullsized":"2023-10-09T11:07:36.278858+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:14.315301+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":26,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976807+0000","last_change":"2023-10-09T11:07:33.262559+0000","last_active":"2023-10-09T11:07:48.976807+0000","last_peered":"2023-10-09T11:07:48.976807+0000","last_clean":"2023-10-09T11:07:48.976807+0000","last_became_active":"2023-10-09T11:07:33.231695+0000","last_became_peered":"2023-10-09T11:07:33.231695+0000","last_unstale":"2023-10-09T11:07:48.976807+0000","last_undegraded":"2023-10-09T11:07:48.976807+0000","last_fullsized":"2023-10-09T11:07:48.976807+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:59.006940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":577,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977079+0000","last_change":"2023-10-09T11:05:20.311424+0000","last_active":"2023-10-09T11:07:48.977079+0000","last_peered":"2023-10-09T11:07:48.977079+0000","last_clean":"2023-10-09T11:07:48.977079+0000","last_became_active":"2023-10-09T11:05:20.311097+0000","last_became_peered":"2023-10-09T11:05:20.311097+0000","last_unstale":"2023-10-09T11:07:48.977079+0000","last_undegraded":"2023-10-09T11:07:48.977079+0000","last_fullsized":"2023-10-09T11:07:48.977079+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:19.274582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1b","version":"554'8350","reported_seq":8428,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.236814+0000","last_change":"2023-10-09T11:06:43.035498+0000","last_active":"2023-10-09T11:07:48.236814+0000","last_peered":"2023-10-09T11:07:48.236814+0000","last_clean":"2023-10-09T11:07:48.236814+0000","last_became_active":"2023-10-09T11:06:43.031985+0000","last_became_peered":"2023-10-09T11:06:43.031985+0000","last_unstale":"2023-10-09T11:07:48.236814+0000","last_undegraded":"2023-10-09T11:07:48.236814+0000","last_fullsized":"2023-10-09T11:07:48.236814+0000","mapping_epoch":531,"log_start":"503'1687","ondisk_log_start":"503'1687","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6663,"log_dups_size":0,"ondisk_log_size":6663,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65732608,"num_objects":1003,"num_object_clones":0,"num_object_copies":3009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5180,"num_write_kb":197856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":576,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.974887+0000","last_change":"2023-10-09T11:06:24.246500+0000","last_active":"2023-10-09T11:07:48.974887+0000","last_peered":"2023-10-09T11:07:48.974887+0000","last_clean":"2023-10-09T11:07:48.974887+0000","last_became_active":"2023-10-09T11:06:24.245076+0000","last_became_peered":"2023-10-09T11:06:24.245076+0000","last_unstale":"2023-10-09T11:07:48.974887+0000","last_undegraded":"2023-10-09T11:07:48.974887+0000","last_fullsized":"2023-10-09T11:07:48.974887+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":22,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975872+0000","last_change":"2023-10-09T11:07:33.269200+0000","last_active":"2023-10-09T11:07:48.975872+0000","last_peered":"2023-10-09T11:07:48.975872+0000","last_clean":"2023-10-09T11:07:48.975872+0000","last_became_active":"2023-10-09T11:07:33.236205+0000","last_became_peered":"2023-10-09T11:07:33.236205+0000","last_unstale":"2023-10-09T11:07:48.975872+0000","last_undegraded":"2023-10-09T11:07:48.975872+0000","last_fullsized":"2023-10-09T11:07:48.975872+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:24.574734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":98,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.267768+0000","last_change":"2023-10-09T11:07:33.227803+0000","last_active":"2023-10-09T11:07:36.267768+0000","last_peered":"2023-10-09T11:07:36.267768+0000","last_clean":"2023-10-09T11:07:36.267768+0000","last_became_active":"2023-10-09T11:07:33.227486+0000","last_became_peered":"2023-10-09T11:07:33.227486+0000","last_unstale":"2023-10-09T11:07:36.267768+0000","last_undegraded":"2023-10-09T11:07:36.267768+0000","last_fullsized":"2023-10-09T11:07:36.267768+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:39.274875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1a","version":"554'8379","reported_seq":11237,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.271254+0000","last_change":"2023-10-09T11:06:43.001534+0000","last_active":"2023-10-09T11:07:48.271254+0000","last_peered":"2023-10-09T11:07:48.271254+0000","last_clean":"2023-10-09T11:07:48.271254+0000","last_became_active":"2023-10-09T11:06:42.998860+0000","last_became_peered":"2023-10-09T11:06:42.998860+0000","last_unstale":"2023-10-09T11:07:48.271254+0000","last_undegraded":"2023-10-09T11:07:48.271254+0000","last_fullsized":"2023-10-09T11:07:48.271254+0000","mapping_epoch":531,"log_start":"503'1724","ondisk_log_start":"503'1724","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6655,"log_dups_size":0,"ondisk_log_size":6655,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71761920,"num_objects":1095,"num_object_clones":0,"num_object_copies":3285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5169,"num_write_kb":200448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":691,"num_bytes_recovered":45285376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":867,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975102+0000","last_change":"2023-10-09T11:05:23.323856+0000","last_active":"2023-10-09T11:07:48.975102+0000","last_peered":"2023-10-09T11:07:48.975102+0000","last_clean":"2023-10-09T11:07:48.975102+0000","last_became_active":"2023-10-09T11:05:23.322763+0000","last_became_peered":"2023-10-09T11:05:23.322763+0000","last_unstale":"2023-10-09T11:07:48.975102+0000","last_undegraded":"2023-10-09T11:07:48.975102+0000","last_fullsized":"2023-10-09T11:07:48.975102+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.760820+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":16,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975929+0000","last_change":"2023-10-09T11:07:33.233510+0000","last_active":"2023-10-09T11:07:48.975929+0000","last_peered":"2023-10-09T11:07:48.975929+0000","last_clean":"2023-10-09T11:07:48.975929+0000","last_became_active":"2023-10-09T11:07:33.232339+0000","last_became_peered":"2023-10-09T11:07:33.232339+0000","last_unstale":"2023-10-09T11:07:48.975929+0000","last_undegraded":"2023-10-09T11:07:48.975929+0000","last_fullsized":"2023-10-09T11:07:48.975929+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:55.324912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":497,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975081+0000","last_change":"2023-10-09T11:07:33.226594+0000","last_active":"2023-10-09T11:07:48.975081+0000","last_peered":"2023-10-09T11:07:48.975081+0000","last_clean":"2023-10-09T11:07:48.975081+0000","last_became_active":"2023-10-09T11:07:33.226275+0000","last_became_peered":"2023-10-09T11:07:33.226275+0000","last_unstale":"2023-10-09T11:07:48.975081+0000","last_undegraded":"2023-10-09T11:07:48.975081+0000","last_fullsized":"2023-10-09T11:07:48.975081+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:41.180088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"554'6735","reported_seq":8187,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.270994+0000","last_change":"2023-10-09T11:06:05.166134+0000","last_active":"2023-10-09T11:07:48.270994+0000","last_peered":"2023-10-09T11:07:48.270994+0000","last_clean":"2023-10-09T11:07:48.270994+0000","last_became_active":"2023-10-09T11:05:45.388425+0000","last_became_peered":"2023-10-09T11:05:45.388425+0000","last_unstale":"2023-10-09T11:07:48.270994+0000","last_undegraded":"2023-10-09T11:07:48.270994+0000","last_fullsized":"2023-10-09T11:07:48.270994+0000","mapping_epoch":509,"log_start":"498'848","ondisk_log_start":"498'848","created":509,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5887,"log_dups_size":0,"ondisk_log_size":5887,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72712192,"num_objects":1110,"num_object_clones":0,"num_object_copies":3330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5172,"num_write_kb":200992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":684,"num_bytes_recovered":44826624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":887,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.280822+0000","last_change":"2023-10-09T11:04:53.274236+0000","last_active":"2023-10-09T11:07:36.280822+0000","last_peered":"2023-10-09T11:07:36.280822+0000","last_clean":"2023-10-09T11:07:36.280822+0000","last_became_active":"2023-10-09T11:04:53.273817+0000","last_became_peered":"2023-10-09T11:04:53.273817+0000","last_unstale":"2023-10-09T11:07:36.280822+0000","last_undegraded":"2023-10-09T11:07:36.280822+0000","last_fullsized":"2023-10-09T11:07:36.280822+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:05.096905+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":151,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975775+0000","last_change":"2023-10-09T11:05:20.312129+0000","last_active":"2023-10-09T11:07:48.975775+0000","last_peered":"2023-10-09T11:07:48.975775+0000","last_clean":"2023-10-09T11:07:48.975775+0000","last_became_active":"2023-10-09T11:05:20.311810+0000","last_became_peered":"2023-10-09T11:05:20.311810+0000","last_unstale":"2023-10-09T11:07:48.975775+0000","last_undegraded":"2023-10-09T11:07:48.975775+0000","last_fullsized":"2023-10-09T11:07:48.975775+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:48.813391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":247,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.265682+0000","last_change":"2023-10-09T11:07:33.218244+0000","last_active":"2023-10-09T11:07:36.265682+0000","last_peered":"2023-10-09T11:07:36.265682+0000","last_clean":"2023-10-09T11:07:36.265682+0000","last_became_active":"2023-10-09T11:07:33.217918+0000","last_became_peered":"2023-10-09T11:07:33.217918+0000","last_unstale":"2023-10-09T11:07:36.265682+0000","last_undegraded":"2023-10-09T11:07:36.265682+0000","last_fullsized":"2023-10-09T11:07:36.265682+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:16.150105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"554'6615","reported_seq":9164,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.242363+0000","last_change":"2023-10-09T11:06:07.154110+0000","last_active":"2023-10-09T11:07:48.242363+0000","last_peered":"2023-10-09T11:07:48.242363+0000","last_clean":"2023-10-09T11:07:48.242363+0000","last_became_active":"2023-10-09T11:05:45.388306+0000","last_became_peered":"2023-10-09T11:05:45.388306+0000","last_unstale":"2023-10-09T11:07:48.242363+0000","last_undegraded":"2023-10-09T11:07:48.242363+0000","last_fullsized":"2023-10-09T11:07:48.242363+0000","mapping_epoch":509,"log_start":"498'669","ondisk_log_start":"498'669","created":509,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5946,"log_dups_size":0,"ondisk_log_size":5946,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72876032,"num_objects":1112,"num_object_clones":0,"num_object_copies":3336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5080,"num_write_kb":198144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":645,"num_bytes_recovered":42237952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":864,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976352+0000","last_change":"2023-10-09T11:05:23.324537+0000","last_active":"2023-10-09T11:07:48.976352+0000","last_peered":"2023-10-09T11:07:48.976352+0000","last_clean":"2023-10-09T11:07:48.976352+0000","last_became_active":"2023-10-09T11:05:23.324187+0000","last_became_peered":"2023-10-09T11:05:23.324187+0000","last_unstale":"2023-10-09T11:07:48.976352+0000","last_undegraded":"2023-10-09T11:07:48.976352+0000","last_fullsized":"2023-10-09T11:07:48.976352+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:25.286106+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":76,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.979079+0000","last_change":"2023-10-09T11:05:20.323565+0000","last_active":"2023-10-09T11:07:48.979079+0000","last_peered":"2023-10-09T11:07:48.979079+0000","last_clean":"2023-10-09T11:07:48.979079+0000","last_became_active":"2023-10-09T11:05:20.309672+0000","last_became_peered":"2023-10-09T11:05:20.309672+0000","last_unstale":"2023-10-09T11:07:48.979079+0000","last_undegraded":"2023-10-09T11:07:48.979079+0000","last_fullsized":"2023-10-09T11:07:48.979079+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:50.211465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":98,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.267445+0000","last_change":"2023-10-09T11:07:33.228001+0000","last_active":"2023-10-09T11:07:36.267445+0000","last_peered":"2023-10-09T11:07:36.267445+0000","last_clean":"2023-10-09T11:07:36.267445+0000","last_became_active":"2023-10-09T11:07:33.227691+0000","last_became_peered":"2023-10-09T11:07:33.227691+0000","last_unstale":"2023-10-09T11:07:36.267445+0000","last_undegraded":"2023-10-09T11:07:36.267445+0000","last_fullsized":"2023-10-09T11:07:36.267445+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:24:16.098138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"556'6720","reported_seq":6798,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.721001+0000","last_change":"2023-10-09T11:05:45.340741+0000","last_active":"2023-10-09T11:07:51.721001+0000","last_peered":"2023-10-09T11:07:51.721001+0000","last_clean":"2023-10-09T11:07:51.721001+0000","last_became_active":"2023-10-09T11:05:45.338519+0000","last_became_peered":"2023-10-09T11:05:45.338519+0000","last_unstale":"2023-10-09T11:07:51.721001+0000","last_undegraded":"2023-10-09T11:07:51.721001+0000","last_fullsized":"2023-10-09T11:07:51.721001+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6720,"log_dups_size":0,"ondisk_log_size":6720,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62554112,"num_objects":954,"num_object_clones":0,"num_object_copies":2862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5150,"num_write_kb":195360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":788,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976513+0000","last_change":"2023-10-09T11:07:36.284927+0000","last_active":"2023-10-09T11:07:48.976513+0000","last_peered":"2023-10-09T11:07:48.976513+0000","last_clean":"2023-10-09T11:07:48.976513+0000","last_became_active":"2023-10-09T11:07:36.284631+0000","last_became_peered":"2023-10-09T11:07:36.284631+0000","last_unstale":"2023-10-09T11:07:48.976513+0000","last_undegraded":"2023-10-09T11:07:48.976513+0000","last_fullsized":"2023-10-09T11:07:48.976513+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:37.220243+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":20,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.978057+0000","last_change":"2023-10-09T11:07:33.273503+0000","last_active":"2023-10-09T11:07:48.978057+0000","last_peered":"2023-10-09T11:07:48.978057+0000","last_clean":"2023-10-09T11:07:48.978057+0000","last_became_active":"2023-10-09T11:07:33.242980+0000","last_became_peered":"2023-10-09T11:07:33.242980+0000","last_unstale":"2023-10-09T11:07:48.978057+0000","last_undegraded":"2023-10-09T11:07:48.978057+0000","last_fullsized":"2023-10-09T11:07:48.978057+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:18.934972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":518,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.974807+0000","last_change":"2023-10-09T11:07:33.304363+0000","last_active":"2023-10-09T11:07:48.974807+0000","last_peered":"2023-10-09T11:07:48.974807+0000","last_clean":"2023-10-09T11:07:48.974807+0000","last_became_active":"2023-10-09T11:07:33.304024+0000","last_became_peered":"2023-10-09T11:07:33.304024+0000","last_unstale":"2023-10-09T11:07:48.974807+0000","last_undegraded":"2023-10-09T11:07:48.974807+0000","last_fullsized":"2023-10-09T11:07:48.974807+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:21:51.522688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"556'6672","reported_seq":6771,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.224661+0000","last_change":"2023-10-09T11:07:28.705631+0000","last_active":"2023-10-09T11:07:51.224661+0000","last_peered":"2023-10-09T11:07:51.224661+0000","last_clean":"2023-10-09T11:07:51.224661+0000","last_became_active":"2023-10-09T11:06:24.254388+0000","last_became_peered":"2023-10-09T11:06:24.254388+0000","last_unstale":"2023-10-09T11:07:51.224661+0000","last_undegraded":"2023-10-09T11:07:51.224661+0000","last_fullsized":"2023-10-09T11:07:51.224661+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6672,"log_dups_size":0,"ondisk_log_size":6672,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65798144,"num_objects":1004,"num_object_clones":0,"num_object_copies":3012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5146,"num_write_kb":196800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":935,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.267816+0000","last_change":"2023-10-09T11:05:23.316129+0000","last_active":"2023-10-09T11:07:36.267816+0000","last_peered":"2023-10-09T11:07:36.267816+0000","last_clean":"2023-10-09T11:07:36.267816+0000","last_became_active":"2023-10-09T11:05:23.315792+0000","last_became_peered":"2023-10-09T11:05:23.315792+0000","last_unstale":"2023-10-09T11:07:36.267816+0000","last_undegraded":"2023-10-09T11:07:36.267816+0000","last_fullsized":"2023-10-09T11:07:36.267816+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:51.770547+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":20,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976724+0000","last_change":"2023-10-09T11:07:33.256691+0000","last_active":"2023-10-09T11:07:48.976724+0000","last_peered":"2023-10-09T11:07:48.976724+0000","last_clean":"2023-10-09T11:07:48.976724+0000","last_became_active":"2023-10-09T11:07:33.229554+0000","last_became_peered":"2023-10-09T11:07:33.229554+0000","last_unstale":"2023-10-09T11:07:48.976724+0000","last_undegraded":"2023-10-09T11:07:48.976724+0000","last_fullsized":"2023-10-09T11:07:48.976724+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:57.414795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":268,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.275273+0000","last_change":"2023-10-09T11:04:50.333538+0000","last_active":"2023-10-09T11:07:36.275273+0000","last_peered":"2023-10-09T11:07:36.275273+0000","last_clean":"2023-10-09T11:07:36.275273+0000","last_became_active":"2023-10-09T11:04:50.332514+0000","last_became_peered":"2023-10-09T11:04:50.332514+0000","last_unstale":"2023-10-09T11:07:36.275273+0000","last_undegraded":"2023-10-09T11:07:36.275273+0000","last_fullsized":"2023-10-09T11:07:36.275273+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:22.933839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"554'6483","reported_seq":11090,"reported_epoch":554,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:07:48.289379+0000","last_change":"2023-10-09T11:07:34.338632+0000","last_active":"2023-10-09T11:07:48.289379+0000","last_peered":"2023-10-09T11:07:48.289379+0000","last_clean":"2023-10-09T11:07:32.132824+0000","last_became_active":"2023-10-09T11:07:34.322525+0000","last_became_peered":"2023-10-09T11:07:34.322525+0000","last_unstale":"2023-10-09T11:07:48.289379+0000","last_undegraded":"2023-10-09T11:07:48.289379+0000","last_fullsized":"2023-10-09T11:07:48.289379+0000","mapping_epoch":550,"log_start":"498'404","ondisk_log_start":"498'404","created":495,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6079,"log_dups_size":0,"ondisk_log_size":6079,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74678272,"num_objects":1139,"num_object_clones":0,"num_object_copies":3417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1139,"num_objects_unfound":0,"num_objects_dirty":1139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4965,"num_write_kb":195360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2245,"num_bytes_recovered":147128320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":1139}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":812,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977685+0000","last_change":"2023-10-09T11:05:23.322251+0000","last_active":"2023-10-09T11:07:48.977685+0000","last_peered":"2023-10-09T11:07:48.977685+0000","last_clean":"2023-10-09T11:07:48.977685+0000","last_became_active":"2023-10-09T11:05:23.321826+0000","last_became_peered":"2023-10-09T11:05:23.321826+0000","last_unstale":"2023-10-09T11:07:48.977685+0000","last_undegraded":"2023-10-09T11:07:48.977685+0000","last_fullsized":"2023-10-09T11:07:48.977685+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.858013+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":20,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976732+0000","last_change":"2023-10-09T11:07:33.271238+0000","last_active":"2023-10-09T11:07:48.976732+0000","last_peered":"2023-10-09T11:07:48.976732+0000","last_clean":"2023-10-09T11:07:48.976732+0000","last_became_active":"2023-10-09T11:07:33.241886+0000","last_became_peered":"2023-10-09T11:07:33.241886+0000","last_unstale":"2023-10-09T11:07:48.976732+0000","last_undegraded":"2023-10-09T11:07:48.976732+0000","last_fullsized":"2023-10-09T11:07:48.976732+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:48.882589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":86,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.979048+0000","last_change":"2023-10-09T11:07:33.244131+0000","last_active":"2023-10-09T11:07:48.979048+0000","last_peered":"2023-10-09T11:07:48.979048+0000","last_clean":"2023-10-09T11:07:48.979048+0000","last_became_active":"2023-10-09T11:07:33.243797+0000","last_became_peered":"2023-10-09T11:07:33.243797+0000","last_unstale":"2023-10-09T11:07:48.979048+0000","last_undegraded":"2023-10-09T11:07:48.979048+0000","last_fullsized":"2023-10-09T11:07:48.979048+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:44.851043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"556'6735","reported_seq":8202,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.566814+0000","last_change":"2023-10-09T11:07:29.087418+0000","last_active":"2023-10-09T11:07:51.566814+0000","last_peered":"2023-10-09T11:07:51.566814+0000","last_clean":"2023-10-09T11:07:51.566814+0000","last_became_active":"2023-10-09T11:05:46.357677+0000","last_became_peered":"2023-10-09T11:05:46.357677+0000","last_unstale":"2023-10-09T11:07:51.566814+0000","last_undegraded":"2023-10-09T11:07:51.566814+0000","last_fullsized":"2023-10-09T11:07:51.566814+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6735,"log_dups_size":0,"ondisk_log_size":6735,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64880640,"num_objects":990,"num_object_clones":0,"num_object_copies":2970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5182,"num_write_kb":197504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":677,"num_bytes_recovered":44367872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":1066,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976855+0000","last_change":"2023-10-09T11:07:36.286417+0000","last_active":"2023-10-09T11:07:48.976855+0000","last_peered":"2023-10-09T11:07:48.976855+0000","last_clean":"2023-10-09T11:07:48.976855+0000","last_became_active":"2023-10-09T11:07:36.286061+0000","last_became_peered":"2023-10-09T11:07:36.286061+0000","last_unstale":"2023-10-09T11:07:48.976855+0000","last_undegraded":"2023-10-09T11:07:48.976855+0000","last_fullsized":"2023-10-09T11:07:48.976855+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:00.289500+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":414,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975662+0000","last_change":"2023-10-09T11:05:20.309385+0000","last_active":"2023-10-09T11:07:48.975662+0000","last_peered":"2023-10-09T11:07:48.975662+0000","last_clean":"2023-10-09T11:07:48.975662+0000","last_became_active":"2023-10-09T11:05:20.309034+0000","last_became_peered":"2023-10-09T11:05:20.309034+0000","last_unstale":"2023-10-09T11:07:48.975662+0000","last_undegraded":"2023-10-09T11:07:48.975662+0000","last_fullsized":"2023-10-09T11:07:48.975662+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:31.857485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":343,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.280138+0000","last_change":"2023-10-09T11:06:24.245966+0000","last_active":"2023-10-09T11:07:36.280138+0000","last_peered":"2023-10-09T11:07:36.280138+0000","last_clean":"2023-10-09T11:07:36.280138+0000","last_became_active":"2023-10-09T11:06:24.245310+0000","last_became_peered":"2023-10-09T11:06:24.245310+0000","last_unstale":"2023-10-09T11:07:36.280138+0000","last_undegraded":"2023-10-09T11:07:36.280138+0000","last_fullsized":"2023-10-09T11:07:36.280138+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:23.966200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.3","version":"554'5848","reported_seq":8885,"reported_epoch":554,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:07:48.125023+0000","last_change":"2023-10-09T11:07:34.314819+0000","last_active":"2023-10-09T11:07:48.125023+0000","last_peered":"2023-10-09T11:07:48.125023+0000","last_clean":"2023-10-09T11:07:32.185795+0000","last_became_active":"2023-10-09T11:07:34.308357+0000","last_became_peered":"2023-10-09T11:07:34.308357+0000","last_unstale":"2023-10-09T11:07:48.125023+0000","last_undegraded":"2023-10-09T11:07:48.125023+0000","last_fullsized":"2023-10-09T11:07:48.125023+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5848,"log_dups_size":0,"ondisk_log_size":5848,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35274752,"num_objects":538,"num_object_clones":0,"num_object_copies":1614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":538,"num_objects_unfound":0,"num_objects_dirty":538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2632,"num_write_kb":101456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1223,"num_bytes_recovered":72613888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":538}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":710,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975631+0000","last_change":"2023-10-09T11:07:36.283174+0000","last_active":"2023-10-09T11:07:48.975631+0000","last_peered":"2023-10-09T11:07:48.975631+0000","last_clean":"2023-10-09T11:07:48.975631+0000","last_became_active":"2023-10-09T11:07:36.278007+0000","last_became_peered":"2023-10-09T11:07:36.278007+0000","last_unstale":"2023-10-09T11:07:48.975631+0000","last_undegraded":"2023-10-09T11:07:48.975631+0000","last_fullsized":"2023-10-09T11:07:48.975631+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:51.693969+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":84,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.267109+0000","last_change":"2023-10-09T11:04:50.529513+0000","last_active":"2023-10-09T11:07:36.267109+0000","last_peered":"2023-10-09T11:07:36.267109+0000","last_clean":"2023-10-09T11:07:36.267109+0000","last_became_active":"2023-10-09T11:04:50.339671+0000","last_became_peered":"2023-10-09T11:04:50.339671+0000","last_unstale":"2023-10-09T11:07:36.267109+0000","last_undegraded":"2023-10-09T11:07:36.267109+0000","last_fullsized":"2023-10-09T11:07:36.267109+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:24.473000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":89,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.267685+0000","last_change":"2023-10-09T11:05:20.313855+0000","last_active":"2023-10-09T11:07:36.267685+0000","last_peered":"2023-10-09T11:07:36.267685+0000","last_clean":"2023-10-09T11:07:36.267685+0000","last_became_active":"2023-10-09T11:05:20.313535+0000","last_became_peered":"2023-10-09T11:05:20.313535+0000","last_unstale":"2023-10-09T11:07:36.267685+0000","last_undegraded":"2023-10-09T11:07:36.267685+0000","last_fullsized":"2023-10-09T11:07:36.267685+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:25.266434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"554'5614","reported_seq":5702,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.089495+0000","last_change":"2023-10-09T11:06:42.991743+0000","last_active":"2023-10-09T11:07:48.089495+0000","last_peered":"2023-10-09T11:07:48.089495+0000","last_clean":"2023-10-09T11:07:48.089495+0000","last_became_active":"2023-10-09T11:06:42.989452+0000","last_became_peered":"2023-10-09T11:06:42.989452+0000","last_unstale":"2023-10-09T11:07:48.089495+0000","last_undegraded":"2023-10-09T11:07:48.089495+0000","last_fullsized":"2023-10-09T11:07:48.089495+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5614,"log_dups_size":0,"ondisk_log_size":5614,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35651584,"num_objects":544,"num_object_clones":0,"num_object_copies":1632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2511,"num_write_kb":97760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":814,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975017+0000","last_change":"2023-10-09T11:04:53.272323+0000","last_active":"2023-10-09T11:07:48.975017+0000","last_peered":"2023-10-09T11:07:48.975017+0000","last_clean":"2023-10-09T11:07:48.975017+0000","last_became_active":"2023-10-09T11:04:53.272010+0000","last_became_peered":"2023-10-09T11:04:53.272010+0000","last_unstale":"2023-10-09T11:07:48.975017+0000","last_undegraded":"2023-10-09T11:07:48.975017+0000","last_fullsized":"2023-10-09T11:07:48.975017+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:23.913605+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":577,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.275576+0000","last_change":"2023-10-09T11:05:20.307499+0000","last_active":"2023-10-09T11:07:36.275576+0000","last_peered":"2023-10-09T11:07:36.275576+0000","last_clean":"2023-10-09T11:07:36.275576+0000","last_became_active":"2023-10-09T11:05:20.307198+0000","last_became_peered":"2023-10-09T11:05:20.307198+0000","last_unstale":"2023-10-09T11:07:36.275576+0000","last_undegraded":"2023-10-09T11:07:36.275576+0000","last_fullsized":"2023-10-09T11:07:36.275576+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:53.726844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":315,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.055200+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:07:51.055200+0000","last_peered":"2023-10-09T11:07:51.055200+0000","last_clean":"2023-10-09T11:07:51.055200+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:07:51.055200+0000","last_undegraded":"2023-10-09T11:07:51.055200+0000","last_fullsized":"2023-10-09T11:07:51.055200+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"554'5838","reported_seq":6675,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.054906+0000","last_change":"2023-10-09T11:07:20.806034+0000","last_active":"2023-10-09T11:07:48.054906+0000","last_peered":"2023-10-09T11:07:48.054906+0000","last_clean":"2023-10-09T11:07:48.054906+0000","last_became_active":"2023-10-09T11:06:42.992656+0000","last_became_peered":"2023-10-09T11:06:42.992656+0000","last_unstale":"2023-10-09T11:07:48.054906+0000","last_undegraded":"2023-10-09T11:07:48.054906+0000","last_fullsized":"2023-10-09T11:07:48.054906+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5838,"log_dups_size":0,"ondisk_log_size":5838,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35569664,"num_objects":542,"num_object_clones":0,"num_object_copies":1626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2604,"num_write_kb":100720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":328,"num_bytes_recovered":21528576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":904,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.280900+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:07:36.280900+0000","last_peered":"2023-10-09T11:07:36.280900+0000","last_clean":"2023-10-09T11:07:36.280900+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:07:36.280900+0000","last_undegraded":"2023-10-09T11:07:36.280900+0000","last_fullsized":"2023-10-09T11:07:36.280900+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":578,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977268+0000","last_change":"2023-10-09T11:04:57.953307+0000","last_active":"2023-10-09T11:07:48.977268+0000","last_peered":"2023-10-09T11:07:48.977268+0000","last_clean":"2023-10-09T11:07:48.977268+0000","last_became_active":"2023-10-09T11:04:57.953095+0000","last_became_peered":"2023-10-09T11:04:57.953095+0000","last_unstale":"2023-10-09T11:07:48.977268+0000","last_undegraded":"2023-10-09T11:07:48.977268+0000","last_fullsized":"2023-10-09T11:07:48.977268+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:05.562643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":262,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.278598+0000","last_change":"2023-10-09T11:05:20.310255+0000","last_active":"2023-10-09T11:07:36.278598+0000","last_peered":"2023-10-09T11:07:36.278598+0000","last_clean":"2023-10-09T11:07:36.278598+0000","last_became_active":"2023-10-09T11:05:20.309109+0000","last_became_peered":"2023-10-09T11:05:20.309109+0000","last_unstale":"2023-10-09T11:07:36.278598+0000","last_undegraded":"2023-10-09T11:07:36.278598+0000","last_fullsized":"2023-10-09T11:07:36.278598+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:07.429039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.0","version":"554'5698","reported_seq":8098,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:07:48.237420+0000","last_change":"2023-10-09T11:07:34.329311+0000","last_active":"2023-10-09T11:07:48.237420+0000","last_peered":"2023-10-09T11:07:48.237420+0000","last_clean":"2023-10-09T11:07:32.155021+0000","last_became_active":"2023-10-09T11:07:34.321010+0000","last_became_peered":"2023-10-09T11:07:34.321010+0000","last_unstale":"2023-10-09T11:07:48.237420+0000","last_undegraded":"2023-10-09T11:07:34.238729+0000","last_fullsized":"2023-10-09T11:07:34.219306+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5698,"log_dups_size":0,"ondisk_log_size":5698,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38813696,"num_objects":592,"num_object_clones":0,"num_object_copies":1776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1515,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2532,"num_write_kb":99984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1147,"num_bytes_recovered":72155136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,3","objects":-153},{"shards":"1,3,6","objects":542}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":798,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976280+0000","last_change":"2023-10-09T11:05:23.329329+0000","last_active":"2023-10-09T11:07:48.976280+0000","last_peered":"2023-10-09T11:07:48.976280+0000","last_clean":"2023-10-09T11:07:48.976280+0000","last_became_active":"2023-10-09T11:05:23.328976+0000","last_became_peered":"2023-10-09T11:05:23.328976+0000","last_unstale":"2023-10-09T11:07:48.976280+0000","last_undegraded":"2023-10-09T11:07:48.976280+0000","last_fullsized":"2023-10-09T11:07:48.976280+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:41.848184+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":98,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.260408+0000","last_change":"2023-10-09T11:07:33.236133+0000","last_active":"2023-10-09T11:07:36.260408+0000","last_peered":"2023-10-09T11:07:36.260408+0000","last_clean":"2023-10-09T11:07:36.260408+0000","last_became_active":"2023-10-09T11:07:33.235818+0000","last_became_peered":"2023-10-09T11:07:33.235818+0000","last_unstale":"2023-10-09T11:07:36.260408+0000","last_undegraded":"2023-10-09T11:07:36.260408+0000","last_fullsized":"2023-10-09T11:07:36.260408+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:20.627846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":259,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.286232+0000","last_change":"2023-10-09T11:04:50.332978+0000","last_active":"2023-10-09T11:07:36.286232+0000","last_peered":"2023-10-09T11:07:36.286232+0000","last_clean":"2023-10-09T11:07:36.286232+0000","last_became_active":"2023-10-09T11:04:50.332735+0000","last_became_peered":"2023-10-09T11:04:50.332735+0000","last_unstale":"2023-10-09T11:07:36.286232+0000","last_undegraded":"2023-10-09T11:07:36.286232+0000","last_fullsized":"2023-10-09T11:07:36.286232+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:39:44.659751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"556'8501","reported_seq":11423,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.239536+0000","last_change":"2023-10-09T11:07:21.753744+0000","last_active":"2023-10-09T11:07:51.239536+0000","last_peered":"2023-10-09T11:07:51.239536+0000","last_clean":"2023-10-09T11:07:51.239536+0000","last_became_active":"2023-10-09T11:06:42.979233+0000","last_became_peered":"2023-10-09T11:06:42.979233+0000","last_unstale":"2023-10-09T11:07:51.239536+0000","last_undegraded":"2023-10-09T11:07:51.239536+0000","last_fullsized":"2023-10-09T11:07:51.239536+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":8501,"log_dups_size":0,"ondisk_log_size":8501,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65896460,"num_objects":1006,"num_object_clones":0,"num_object_copies":3018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5280,"num_write_kb":201120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":705,"num_bytes_recovered":46137344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":1080,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.292501+0000","last_change":"2023-10-09T11:07:36.291716+0000","last_active":"2023-10-09T11:07:36.292501+0000","last_peered":"2023-10-09T11:07:36.292501+0000","last_clean":"2023-10-09T11:07:36.292501+0000","last_became_active":"2023-10-09T11:07:36.291424+0000","last_became_peered":"2023-10-09T11:07:36.291424+0000","last_unstale":"2023-10-09T11:07:36.292501+0000","last_undegraded":"2023-10-09T11:07:36.292501+0000","last_fullsized":"2023-10-09T11:07:36.292501+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:22.316387+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":283,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.276120+0000","last_change":"2023-10-09T11:07:33.226547+0000","last_active":"2023-10-09T11:07:36.276120+0000","last_peered":"2023-10-09T11:07:36.276120+0000","last_clean":"2023-10-09T11:07:36.276120+0000","last_became_active":"2023-10-09T11:07:33.226176+0000","last_became_peered":"2023-10-09T11:07:33.226176+0000","last_unstale":"2023-10-09T11:07:36.276120+0000","last_undegraded":"2023-10-09T11:07:36.276120+0000","last_fullsized":"2023-10-09T11:07:36.276120+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:16.888732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":431,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.974950+0000","last_change":"2023-10-09T11:04:50.341353+0000","last_active":"2023-10-09T11:07:48.974950+0000","last_peered":"2023-10-09T11:07:48.974950+0000","last_clean":"2023-10-09T11:07:48.974950+0000","last_became_active":"2023-10-09T11:04:50.340801+0000","last_became_peered":"2023-10-09T11:04:50.340801+0000","last_unstale":"2023-10-09T11:07:48.974950+0000","last_undegraded":"2023-10-09T11:07:48.974950+0000","last_fullsized":"2023-10-09T11:07:48.974950+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:00.699069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":235,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.275809+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:07:36.275809+0000","last_peered":"2023-10-09T11:07:36.275809+0000","last_clean":"2023-10-09T11:07:36.275809+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:07:36.275809+0000","last_undegraded":"2023-10-09T11:07:36.275809+0000","last_fullsized":"2023-10-09T11:07:36.275809+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"554'8254","reported_seq":8333,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.236408+0000","last_change":"2023-10-09T11:06:43.027139+0000","last_active":"2023-10-09T11:07:48.236408+0000","last_peered":"2023-10-09T11:07:48.236408+0000","last_clean":"2023-10-09T11:07:48.236408+0000","last_became_active":"2023-10-09T11:06:43.024164+0000","last_became_peered":"2023-10-09T11:06:43.024164+0000","last_unstale":"2023-10-09T11:07:48.236408+0000","last_undegraded":"2023-10-09T11:07:48.236408+0000","last_fullsized":"2023-10-09T11:07:48.236408+0000","mapping_epoch":495,"log_start":"503'1687","ondisk_log_start":"503'1687","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6567,"log_dups_size":0,"ondisk_log_size":6567,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72024064,"num_objects":1099,"num_object_clones":0,"num_object_copies":3297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1099,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5084,"num_write_kb":197856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":756,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.974808+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:07:48.974808+0000","last_peered":"2023-10-09T11:07:48.974808+0000","last_clean":"2023-10-09T11:07:48.974808+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:07:48.974808+0000","last_undegraded":"2023-10-09T11:07:48.974808+0000","last_fullsized":"2023-10-09T11:07:48.974808+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":438,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976067+0000","last_change":"2023-10-09T11:05:20.316093+0000","last_active":"2023-10-09T11:07:48.976067+0000","last_peered":"2023-10-09T11:07:48.976067+0000","last_clean":"2023-10-09T11:07:48.976067+0000","last_became_active":"2023-10-09T11:05:20.315471+0000","last_became_peered":"2023-10-09T11:05:20.315471+0000","last_unstale":"2023-10-09T11:07:48.976067+0000","last_undegraded":"2023-10-09T11:07:48.976067+0000","last_fullsized":"2023-10-09T11:07:48.976067+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:47.548763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":462,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.978532+0000","last_change":"2023-10-09T11:06:24.245282+0000","last_active":"2023-10-09T11:07:48.978532+0000","last_peered":"2023-10-09T11:07:48.978532+0000","last_clean":"2023-10-09T11:07:48.978532+0000","last_became_active":"2023-10-09T11:06:24.244981+0000","last_became_peered":"2023-10-09T11:06:24.244981+0000","last_unstale":"2023-10-09T11:07:48.978532+0000","last_undegraded":"2023-10-09T11:07:48.978532+0000","last_fullsized":"2023-10-09T11:07:48.978532+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:15.327655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":268,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.275407+0000","last_change":"2023-10-09T11:04:50.311579+0000","last_active":"2023-10-09T11:07:36.275407+0000","last_peered":"2023-10-09T11:07:36.275407+0000","last_clean":"2023-10-09T11:07:36.275407+0000","last_became_active":"2023-10-09T11:04:50.311270+0000","last_became_peered":"2023-10-09T11:04:50.311270+0000","last_unstale":"2023-10-09T11:07:36.275407+0000","last_undegraded":"2023-10-09T11:07:36.275407+0000","last_fullsized":"2023-10-09T11:07:36.275407+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:49.850302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"554'6542","reported_seq":9113,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.209199+0000","last_change":"2023-10-09T11:06:09.940480+0000","last_active":"2023-10-09T11:07:48.209199+0000","last_peered":"2023-10-09T11:07:48.209199+0000","last_clean":"2023-10-09T11:07:48.209199+0000","last_became_active":"2023-10-09T11:05:46.348515+0000","last_became_peered":"2023-10-09T11:05:46.348515+0000","last_unstale":"2023-10-09T11:07:48.209199+0000","last_undegraded":"2023-10-09T11:07:48.209199+0000","last_fullsized":"2023-10-09T11:07:48.209199+0000","mapping_epoch":510,"log_start":"498'483","ondisk_log_start":"498'483","created":495,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6059,"log_dups_size":0,"ondisk_log_size":6059,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72548352,"num_objects":1107,"num_object_clones":0,"num_object_copies":3321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5007,"num_write_kb":195648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":654,"num_bytes_recovered":42893312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":908,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976097+0000","last_change":"2023-10-09T11:07:36.284280+0000","last_active":"2023-10-09T11:07:48.976097+0000","last_peered":"2023-10-09T11:07:48.976097+0000","last_clean":"2023-10-09T11:07:48.976097+0000","last_became_active":"2023-10-09T11:07:36.283966+0000","last_became_peered":"2023-10-09T11:07:36.283966+0000","last_unstale":"2023-10-09T11:07:48.976097+0000","last_undegraded":"2023-10-09T11:07:48.976097+0000","last_fullsized":"2023-10-09T11:07:48.976097+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:54.128331+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":334,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977478+0000","last_change":"2023-10-09T11:04:50.381744+0000","last_active":"2023-10-09T11:07:48.977478+0000","last_peered":"2023-10-09T11:07:48.977478+0000","last_clean":"2023-10-09T11:07:48.977478+0000","last_became_active":"2023-10-09T11:04:50.381253+0000","last_became_peered":"2023-10-09T11:04:50.381253+0000","last_unstale":"2023-10-09T11:07:48.977478+0000","last_undegraded":"2023-10-09T11:07:48.977478+0000","last_fullsized":"2023-10-09T11:07:48.977478+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:24.030274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":253,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.277201+0000","last_change":"2023-10-09T11:05:20.298542+0000","last_active":"2023-10-09T11:07:36.277201+0000","last_peered":"2023-10-09T11:07:36.277201+0000","last_clean":"2023-10-09T11:07:36.277201+0000","last_became_active":"2023-10-09T11:05:20.298119+0000","last_became_peered":"2023-10-09T11:05:20.298119+0000","last_unstale":"2023-10-09T11:07:36.277201+0000","last_undegraded":"2023-10-09T11:07:36.277201+0000","last_fullsized":"2023-10-09T11:07:36.277201+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:00.827214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":233,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.287227+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:07:36.287227+0000","last_peered":"2023-10-09T11:07:36.287227+0000","last_clean":"2023-10-09T11:07:36.287227+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:07:36.287227+0000","last_undegraded":"2023-10-09T11:07:36.287227+0000","last_fullsized":"2023-10-09T11:07:36.287227+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"554'8346","reported_seq":11213,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.255065+0000","last_change":"2023-10-09T11:06:54.755841+0000","last_active":"2023-10-09T11:07:48.255065+0000","last_peered":"2023-10-09T11:07:48.255065+0000","last_clean":"2023-10-09T11:07:48.255065+0000","last_became_active":"2023-10-09T11:06:42.985753+0000","last_became_peered":"2023-10-09T11:06:42.985753+0000","last_unstale":"2023-10-09T11:07:48.255065+0000","last_undegraded":"2023-10-09T11:07:48.255065+0000","last_fullsized":"2023-10-09T11:07:48.255065+0000","mapping_epoch":513,"log_start":"503'1724","ondisk_log_start":"503'1724","created":495,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6622,"log_dups_size":0,"ondisk_log_size":6622,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73924608,"num_objects":1128,"num_object_clones":0,"num_object_copies":3384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5136,"num_write_kb":200448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":693,"num_bytes_recovered":45285376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":431,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975354+0000","last_change":"2023-10-09T11:07:33.227052+0000","last_active":"2023-10-09T11:07:48.975354+0000","last_peered":"2023-10-09T11:07:48.975354+0000","last_clean":"2023-10-09T11:07:48.975354+0000","last_became_active":"2023-10-09T11:07:33.226289+0000","last_became_peered":"2023-10-09T11:07:33.226289+0000","last_unstale":"2023-10-09T11:07:48.975354+0000","last_undegraded":"2023-10-09T11:07:48.975354+0000","last_fullsized":"2023-10-09T11:07:48.975354+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:15.605946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":853,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975380+0000","last_change":"2023-10-09T11:07:36.290700+0000","last_active":"2023-10-09T11:07:48.975380+0000","last_peered":"2023-10-09T11:07:48.975380+0000","last_clean":"2023-10-09T11:07:48.975380+0000","last_became_active":"2023-10-09T11:07:36.290357+0000","last_became_peered":"2023-10-09T11:07:36.290357+0000","last_unstale":"2023-10-09T11:07:48.975380+0000","last_undegraded":"2023-10-09T11:07:48.975380+0000","last_fullsized":"2023-10-09T11:07:48.975380+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:34.494777+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":89,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.260732+0000","last_change":"2023-10-09T11:05:20.310666+0000","last_active":"2023-10-09T11:07:36.260732+0000","last_peered":"2023-10-09T11:07:36.260732+0000","last_clean":"2023-10-09T11:07:36.260732+0000","last_became_active":"2023-10-09T11:05:20.310413+0000","last_became_peered":"2023-10-09T11:05:20.310413+0000","last_unstale":"2023-10-09T11:07:36.260732+0000","last_undegraded":"2023-10-09T11:07:36.260732+0000","last_fullsized":"2023-10-09T11:07:36.260732+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:34.169905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":516,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976655+0000","last_change":"2023-10-09T11:05:20.316523+0000","last_active":"2023-10-09T11:07:48.976655+0000","last_peered":"2023-10-09T11:07:48.976655+0000","last_clean":"2023-10-09T11:07:48.976655+0000","last_became_active":"2023-10-09T11:05:20.314493+0000","last_became_peered":"2023-10-09T11:05:20.314493+0000","last_unstale":"2023-10-09T11:07:48.976655+0000","last_undegraded":"2023-10-09T11:07:48.976655+0000","last_fullsized":"2023-10-09T11:07:48.976655+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:41.618701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"554'6609","reported_seq":7995,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.212174+0000","last_change":"2023-10-09T11:06:34.311475+0000","last_active":"2023-10-09T11:07:48.212174+0000","last_peered":"2023-10-09T11:07:48.212174+0000","last_clean":"2023-10-09T11:07:48.212174+0000","last_became_active":"2023-10-09T11:05:46.356286+0000","last_became_peered":"2023-10-09T11:05:46.356286+0000","last_unstale":"2023-10-09T11:07:48.212174+0000","last_undegraded":"2023-10-09T11:07:48.212174+0000","last_fullsized":"2023-10-09T11:07:48.212174+0000","mapping_epoch":510,"log_start":"498'539","ondisk_log_start":"498'539","created":495,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6070,"log_dups_size":0,"ondisk_log_size":6070,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73629696,"num_objects":1123,"num_object_clones":0,"num_object_copies":3369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5045,"num_write_kb":197408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":650,"num_bytes_recovered":42598400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":924,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.280733+0000","last_change":"2023-10-09T11:05:23.322074+0000","last_active":"2023-10-09T11:07:36.280733+0000","last_peered":"2023-10-09T11:07:36.280733+0000","last_clean":"2023-10-09T11:07:36.280733+0000","last_became_active":"2023-10-09T11:05:23.321749+0000","last_became_peered":"2023-10-09T11:05:23.321749+0000","last_unstale":"2023-10-09T11:07:36.280733+0000","last_undegraded":"2023-10-09T11:07:36.280733+0000","last_fullsized":"2023-10-09T11:07:36.280733+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:07.010932+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":131,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975719+0000","last_change":"2023-10-09T11:02:33.198484+0000","last_active":"2023-10-09T11:07:48.975719+0000","last_peered":"2023-10-09T11:07:48.975719+0000","last_clean":"2023-10-09T11:07:48.975719+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:07:48.975719+0000","last_undegraded":"2023-10-09T11:07:48.975719+0000","last_fullsized":"2023-10-09T11:07:48.975719+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":180,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.268576+0000","last_change":"2023-10-09T11:06:24.248362+0000","last_active":"2023-10-09T11:07:36.268576+0000","last_peered":"2023-10-09T11:07:36.268576+0000","last_clean":"2023-10-09T11:07:36.268576+0000","last_became_active":"2023-10-09T11:06:24.247348+0000","last_became_peered":"2023-10-09T11:06:24.247348+0000","last_unstale":"2023-10-09T11:07:36.268576+0000","last_undegraded":"2023-10-09T11:07:36.268576+0000","last_fullsized":"2023-10-09T11:07:36.268576+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:26.728754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":379,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.275458+0000","last_change":"2023-10-09T11:06:24.253511+0000","last_active":"2023-10-09T11:07:36.275458+0000","last_peered":"2023-10-09T11:07:36.275458+0000","last_clean":"2023-10-09T11:07:36.275458+0000","last_became_active":"2023-10-09T11:06:24.252739+0000","last_became_peered":"2023-10-09T11:06:24.252739+0000","last_unstale":"2023-10-09T11:07:36.275458+0000","last_undegraded":"2023-10-09T11:07:36.275458+0000","last_fullsized":"2023-10-09T11:07:36.275458+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:50.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"554'8346","reported_seq":8425,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.283437+0000","last_change":"2023-10-09T11:06:42.983758+0000","last_active":"2023-10-09T11:07:48.283437+0000","last_peered":"2023-10-09T11:07:48.283437+0000","last_clean":"2023-10-09T11:07:48.283437+0000","last_became_active":"2023-10-09T11:06:42.981460+0000","last_became_peered":"2023-10-09T11:06:42.981460+0000","last_unstale":"2023-10-09T11:07:48.283437+0000","last_undegraded":"2023-10-09T11:07:48.283437+0000","last_fullsized":"2023-10-09T11:07:48.283437+0000","mapping_epoch":495,"log_start":"504'1919","ondisk_log_start":"504'1919","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6427,"log_dups_size":0,"ondisk_log_size":6427,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72450048,"num_objects":1105,"num_object_clones":0,"num_object_copies":3315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5137,"num_write_kb":199776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":784,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.265459+0000","last_change":"2023-10-09T11:06:24.239676+0000","last_active":"2023-10-09T11:07:36.265459+0000","last_peered":"2023-10-09T11:07:36.265459+0000","last_clean":"2023-10-09T11:07:36.265459+0000","last_became_active":"2023-10-09T11:06:24.239375+0000","last_became_peered":"2023-10-09T11:06:24.239375+0000","last_unstale":"2023-10-09T11:07:36.265459+0000","last_undegraded":"2023-10-09T11:07:36.265459+0000","last_fullsized":"2023-10-09T11:07:36.265459+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:27.953053+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976253+0000","last_change":"2023-10-09T11:07:33.229415+0000","last_active":"2023-10-09T11:07:48.976253+0000","last_peered":"2023-10-09T11:07:48.976253+0000","last_clean":"2023-10-09T11:07:48.976253+0000","last_became_active":"2023-10-09T11:07:33.229061+0000","last_became_peered":"2023-10-09T11:07:33.229061+0000","last_unstale":"2023-10-09T11:07:48.976253+0000","last_undegraded":"2023-10-09T11:07:48.976253+0000","last_fullsized":"2023-10-09T11:07:48.976253+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:30.978293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":235,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.276284+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:07:36.276284+0000","last_peered":"2023-10-09T11:07:36.276284+0000","last_clean":"2023-10-09T11:07:36.276284+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:07:36.276284+0000","last_undegraded":"2023-10-09T11:07:36.276284+0000","last_fullsized":"2023-10-09T11:07:36.276284+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":515,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975281+0000","last_change":"2023-10-09T11:07:33.227271+0000","last_active":"2023-10-09T11:07:48.975281+0000","last_peered":"2023-10-09T11:07:48.975281+0000","last_clean":"2023-10-09T11:07:48.975281+0000","last_became_active":"2023-10-09T11:07:33.226760+0000","last_became_peered":"2023-10-09T11:07:33.226760+0000","last_unstale":"2023-10-09T11:07:48.975281+0000","last_undegraded":"2023-10-09T11:07:48.975281+0000","last_fullsized":"2023-10-09T11:07:48.975281+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:12.268123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"554'8352","reported_seq":11307,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:07:48.268613+0000","last_change":"2023-10-09T11:07:34.375236+0000","last_active":"2023-10-09T11:07:48.268613+0000","last_peered":"2023-10-09T11:07:48.268613+0000","last_clean":"2023-10-09T11:06:06.963741+0000","last_became_active":"2023-10-09T11:07:34.357802+0000","last_became_peered":"2023-10-09T11:07:34.357802+0000","last_unstale":"2023-10-09T11:07:48.268613+0000","last_undegraded":"2023-10-09T11:07:34.289955+0000","last_fullsized":"2023-10-09T11:07:34.281689+0000","mapping_epoch":550,"log_start":"504'1895","ondisk_log_start":"504'1895","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6457,"log_dups_size":0,"ondisk_log_size":6457,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77201408,"num_objects":1178,"num_object_clones":0,"num_object_copies":3534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2256,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5122,"num_write_kb":201600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1434,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1,2","objects":745},{"shards":"1,2","objects":433}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":755,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.978087+0000","last_change":"2023-10-09T11:05:23.323368+0000","last_active":"2023-10-09T11:07:48.978087+0000","last_peered":"2023-10-09T11:07:48.978087+0000","last_clean":"2023-10-09T11:07:48.978087+0000","last_became_active":"2023-10-09T11:05:23.323032+0000","last_became_peered":"2023-10-09T11:05:23.323032+0000","last_unstale":"2023-10-09T11:07:48.978087+0000","last_undegraded":"2023-10-09T11:07:48.978087+0000","last_fullsized":"2023-10-09T11:07:48.978087+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:09.393584+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":110,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.267385+0000","last_change":"2023-10-09T11:07:33.246378+0000","last_active":"2023-10-09T11:07:36.267385+0000","last_peered":"2023-10-09T11:07:36.267385+0000","last_clean":"2023-10-09T11:07:36.267385+0000","last_became_active":"2023-10-09T11:07:33.228019+0000","last_became_peered":"2023-10-09T11:07:33.228019+0000","last_unstale":"2023-10-09T11:07:36.267385+0000","last_undegraded":"2023-10-09T11:07:36.267385+0000","last_fullsized":"2023-10-09T11:07:36.267385+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:18.511517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":262,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.278737+0000","last_change":"2023-10-09T11:05:20.310417+0000","last_active":"2023-10-09T11:07:36.278737+0000","last_peered":"2023-10-09T11:07:36.278737+0000","last_clean":"2023-10-09T11:07:36.278737+0000","last_became_active":"2023-10-09T11:05:20.309827+0000","last_became_peered":"2023-10-09T11:05:20.309827+0000","last_unstale":"2023-10-09T11:07:36.278737+0000","last_undegraded":"2023-10-09T11:07:36.278737+0000","last_fullsized":"2023-10-09T11:07:36.278737+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:23.934383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":98,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.260330+0000","last_change":"2023-10-09T11:07:33.217595+0000","last_active":"2023-10-09T11:07:36.260330+0000","last_peered":"2023-10-09T11:07:36.260330+0000","last_clean":"2023-10-09T11:07:36.260330+0000","last_became_active":"2023-10-09T11:07:33.217155+0000","last_became_peered":"2023-10-09T11:07:33.217155+0000","last_unstale":"2023-10-09T11:07:36.260330+0000","last_undegraded":"2023-10-09T11:07:36.260330+0000","last_fullsized":"2023-10-09T11:07:36.260330+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:55.127329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"554'8272","reported_seq":8350,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.250972+0000","last_change":"2023-10-09T11:06:42.990792+0000","last_active":"2023-10-09T11:07:48.250972+0000","last_peered":"2023-10-09T11:07:48.250972+0000","last_clean":"2023-10-09T11:07:48.250972+0000","last_became_active":"2023-10-09T11:06:42.988050+0000","last_became_peered":"2023-10-09T11:06:42.988050+0000","last_unstale":"2023-10-09T11:07:48.250972+0000","last_undegraded":"2023-10-09T11:07:48.250972+0000","last_fullsized":"2023-10-09T11:07:48.250972+0000","mapping_epoch":495,"log_start":"504'1803","ondisk_log_start":"504'1803","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6469,"log_dups_size":0,"ondisk_log_size":6469,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70582272,"num_objects":1077,"num_object_clones":0,"num_object_copies":3231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5097,"num_write_kb":197568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":717,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.287926+0000","last_change":"2023-10-09T11:07:36.284491+0000","last_active":"2023-10-09T11:07:36.287926+0000","last_peered":"2023-10-09T11:07:36.287926+0000","last_clean":"2023-10-09T11:07:36.287926+0000","last_became_active":"2023-10-09T11:07:36.284249+0000","last_became_peered":"2023-10-09T11:07:36.284249+0000","last_unstale":"2023-10-09T11:07:36.287926+0000","last_undegraded":"2023-10-09T11:07:36.287926+0000","last_fullsized":"2023-10-09T11:07:36.287926+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:03.653371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:03.653371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:09.629417+0000","scrub_duration":0.001576771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.978134+0000","last_change":"2023-10-09T11:07:33.241346+0000","last_active":"2023-10-09T11:07:48.978134+0000","last_peered":"2023-10-09T11:07:48.978134+0000","last_clean":"2023-10-09T11:07:48.978134+0000","last_became_active":"2023-10-09T11:07:33.241013+0000","last_became_peered":"2023-10-09T11:07:33.241013+0000","last_unstale":"2023-10-09T11:07:48.978134+0000","last_undegraded":"2023-10-09T11:07:48.978134+0000","last_fullsized":"2023-10-09T11:07:48.978134+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:57.164502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":252,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.275360+0000","last_change":"2023-10-09T11:06:24.253714+0000","last_active":"2023-10-09T11:07:36.275360+0000","last_peered":"2023-10-09T11:07:36.275360+0000","last_clean":"2023-10-09T11:07:36.275360+0000","last_became_active":"2023-10-09T11:06:24.253143+0000","last_became_peered":"2023-10-09T11:06:24.253143+0000","last_unstale":"2023-10-09T11:07:36.275360+0000","last_undegraded":"2023-10-09T11:07:36.275360+0000","last_fullsized":"2023-10-09T11:07:36.275360+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:05.682288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":361,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.974720+0000","last_change":"2023-10-09T11:05:20.313560+0000","last_active":"2023-10-09T11:07:48.974720+0000","last_peered":"2023-10-09T11:07:48.974720+0000","last_clean":"2023-10-09T11:07:48.974720+0000","last_became_active":"2023-10-09T11:05:20.313214+0000","last_became_peered":"2023-10-09T11:05:20.313214+0000","last_unstale":"2023-10-09T11:07:48.974720+0000","last_undegraded":"2023-10-09T11:07:48.974720+0000","last_fullsized":"2023-10-09T11:07:48.974720+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:32.527205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"556'6899","reported_seq":8688,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.518346+0000","last_change":"2023-10-09T11:07:34.333312+0000","last_active":"2023-10-09T11:07:51.518346+0000","last_peered":"2023-10-09T11:07:51.518346+0000","last_clean":"2023-10-09T11:07:51.518346+0000","last_became_active":"2023-10-09T11:07:34.285534+0000","last_became_peered":"2023-10-09T11:07:34.285534+0000","last_unstale":"2023-10-09T11:07:51.518346+0000","last_undegraded":"2023-10-09T11:07:51.518346+0000","last_fullsized":"2023-10-09T11:07:51.518346+0000","mapping_epoch":550,"log_start":"498'1002","ondisk_log_start":"498'1002","created":509,"last_epoch_clean":551,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5897,"log_dups_size":0,"ondisk_log_size":5897,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69697536,"num_objects":1064,"num_object_clones":0,"num_object_copies":3192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5308,"num_write_kb":203872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":838,"num_bytes_recovered":54853632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":961,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.268808+0000","last_change":"2023-10-09T11:05:23.329886+0000","last_active":"2023-10-09T11:07:36.268808+0000","last_peered":"2023-10-09T11:07:36.268808+0000","last_clean":"2023-10-09T11:07:36.268808+0000","last_became_active":"2023-10-09T11:05:23.329561+0000","last_became_peered":"2023-10-09T11:05:23.329561+0000","last_unstale":"2023-10-09T11:07:36.268808+0000","last_undegraded":"2023-10-09T11:07:36.268808+0000","last_fullsized":"2023-10-09T11:07:36.268808+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:46.933799+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":16,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976148+0000","last_change":"2023-10-09T11:07:33.229864+0000","last_active":"2023-10-09T11:07:48.976148+0000","last_peered":"2023-10-09T11:07:48.976148+0000","last_clean":"2023-10-09T11:07:48.976148+0000","last_became_active":"2023-10-09T11:07:33.229547+0000","last_became_peered":"2023-10-09T11:07:33.229547+0000","last_unstale":"2023-10-09T11:07:48.976148+0000","last_undegraded":"2023-10-09T11:07:48.976148+0000","last_fullsized":"2023-10-09T11:07:48.976148+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:34.990439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":338,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.981611+0000","last_change":"2023-10-09T11:04:50.510688+0000","last_active":"2023-10-09T11:07:48.981611+0000","last_peered":"2023-10-09T11:07:48.981611+0000","last_clean":"2023-10-09T11:07:48.981611+0000","last_became_active":"2023-10-09T11:04:50.507404+0000","last_became_peered":"2023-10-09T11:04:50.507404+0000","last_unstale":"2023-10-09T11:07:48.981611+0000","last_undegraded":"2023-10-09T11:07:48.981611+0000","last_fullsized":"2023-10-09T11:07:48.981611+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:13.663394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"556'6721","reported_seq":8165,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.488584+0000","last_change":"2023-10-09T11:07:26.133461+0000","last_active":"2023-10-09T11:07:51.488584+0000","last_peered":"2023-10-09T11:07:51.488584+0000","last_clean":"2023-10-09T11:07:51.488584+0000","last_became_active":"2023-10-09T11:05:45.387324+0000","last_became_peered":"2023-10-09T11:05:45.387324+0000","last_unstale":"2023-10-09T11:07:51.488584+0000","last_undegraded":"2023-10-09T11:07:51.488584+0000","last_fullsized":"2023-10-09T11:07:51.488584+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6721,"log_dups_size":0,"ondisk_log_size":6721,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65536000,"num_objects":1000,"num_object_clones":0,"num_object_copies":3000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5168,"num_write_kb":197376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":667,"num_bytes_recovered":43712512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":918,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.265798+0000","last_change":"2023-10-09T11:05:23.321040+0000","last_active":"2023-10-09T11:07:36.265798+0000","last_peered":"2023-10-09T11:07:36.265798+0000","last_clean":"2023-10-09T11:07:36.265798+0000","last_became_active":"2023-10-09T11:05:23.320674+0000","last_became_peered":"2023-10-09T11:05:23.320674+0000","last_unstale":"2023-10-09T11:07:36.265798+0000","last_undegraded":"2023-10-09T11:07:36.265798+0000","last_fullsized":"2023-10-09T11:07:36.265798+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.527660+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":383,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975590+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:07:48.975590+0000","last_peered":"2023-10-09T11:07:48.975590+0000","last_clean":"2023-10-09T11:07:48.975590+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:07:48.975590+0000","last_undegraded":"2023-10-09T11:07:48.975590+0000","last_fullsized":"2023-10-09T11:07:48.975590+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":80,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.260706+0000","last_change":"2023-10-09T11:04:50.511984+0000","last_active":"2023-10-09T11:07:36.260706+0000","last_peered":"2023-10-09T11:07:36.260706+0000","last_clean":"2023-10-09T11:07:36.260706+0000","last_became_active":"2023-10-09T11:04:50.510820+0000","last_became_peered":"2023-10-09T11:04:50.510820+0000","last_unstale":"2023-10-09T11:07:36.260706+0000","last_undegraded":"2023-10-09T11:07:36.260706+0000","last_fullsized":"2023-10-09T11:07:36.260706+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:38.662189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.15","version":"554'6513","reported_seq":11189,"reported_epoch":554,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:07:48.126675+0000","last_change":"2023-10-09T11:07:34.356257+0000","last_active":"2023-10-09T11:07:48.126675+0000","last_peered":"2023-10-09T11:07:48.126675+0000","last_clean":"2023-10-09T11:07:32.098315+0000","last_became_active":"2023-10-09T11:07:34.338915+0000","last_became_peered":"2023-10-09T11:07:34.338915+0000","last_unstale":"2023-10-09T11:07:48.126675+0000","last_undegraded":"2023-10-09T11:07:48.126675+0000","last_fullsized":"2023-10-09T11:07:48.126675+0000","mapping_epoch":550,"log_start":"498'467","ondisk_log_start":"498'467","created":509,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6046,"log_dups_size":0,"ondisk_log_size":6046,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70090752,"num_objects":1070,"num_object_clones":0,"num_object_copies":3210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1070,"num_objects_unfound":0,"num_objects_dirty":1070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4996,"num_write_kb":194080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2280,"num_bytes_recovered":149422080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":1070}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":534,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.974872+0000","last_change":"2023-10-09T11:05:20.327032+0000","last_active":"2023-10-09T11:07:48.974872+0000","last_peered":"2023-10-09T11:07:48.974872+0000","last_clean":"2023-10-09T11:07:48.974872+0000","last_became_active":"2023-10-09T11:05:20.320193+0000","last_became_peered":"2023-10-09T11:05:20.320193+0000","last_unstale":"2023-10-09T11:07:48.974872+0000","last_undegraded":"2023-10-09T11:07:48.974872+0000","last_fullsized":"2023-10-09T11:07:48.974872+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:44.862890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":80,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.266253+0000","last_change":"2023-10-09T11:04:50.384219+0000","last_active":"2023-10-09T11:07:36.266253+0000","last_peered":"2023-10-09T11:07:36.266253+0000","last_clean":"2023-10-09T11:07:36.266253+0000","last_became_active":"2023-10-09T11:04:50.384028+0000","last_became_peered":"2023-10-09T11:04:50.384028+0000","last_unstale":"2023-10-09T11:07:36.266253+0000","last_undegraded":"2023-10-09T11:07:36.266253+0000","last_fullsized":"2023-10-09T11:07:36.266253+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:04.721869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.16","version":"556'6694","reported_seq":6788,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.129414+0000","last_change":"2023-10-09T11:07:24.733266+0000","last_active":"2023-10-09T11:07:51.129414+0000","last_peered":"2023-10-09T11:07:51.129414+0000","last_clean":"2023-10-09T11:07:51.129414+0000","last_became_active":"2023-10-09T11:06:24.254366+0000","last_became_peered":"2023-10-09T11:06:24.254366+0000","last_unstale":"2023-10-09T11:07:51.129414+0000","last_undegraded":"2023-10-09T11:07:51.129414+0000","last_fullsized":"2023-10-09T11:07:51.129414+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6694,"log_dups_size":0,"ondisk_log_size":6694,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68550656,"num_objects":1046,"num_object_clones":0,"num_object_copies":3138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5168,"num_write_kb":198848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":729,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976550+0000","last_change":"2023-10-09T11:07:36.291401+0000","last_active":"2023-10-09T11:07:48.976550+0000","last_peered":"2023-10-09T11:07:48.976550+0000","last_clean":"2023-10-09T11:07:48.976550+0000","last_became_active":"2023-10-09T11:07:36.291077+0000","last_became_peered":"2023-10-09T11:07:36.291077+0000","last_unstale":"2023-10-09T11:07:48.976550+0000","last_undegraded":"2023-10-09T11:07:48.976550+0000","last_fullsized":"2023-10-09T11:07:48.976550+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:04.790935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.790935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":89,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.265883+0000","last_change":"2023-10-09T11:05:20.299321+0000","last_active":"2023-10-09T11:07:36.265883+0000","last_peered":"2023-10-09T11:07:36.265883+0000","last_clean":"2023-10-09T11:07:36.265883+0000","last_became_active":"2023-10-09T11:05:20.299019+0000","last_became_peered":"2023-10-09T11:05:20.299019+0000","last_unstale":"2023-10-09T11:07:36.265883+0000","last_undegraded":"2023-10-09T11:07:36.265883+0000","last_fullsized":"2023-10-09T11:07:36.265883+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:08.645909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":80,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.268847+0000","last_change":"2023-10-09T11:04:50.366890+0000","last_active":"2023-10-09T11:07:36.268847+0000","last_peered":"2023-10-09T11:07:36.268847+0000","last_clean":"2023-10-09T11:07:36.268847+0000","last_became_active":"2023-10-09T11:04:50.366558+0000","last_became_peered":"2023-10-09T11:04:50.366558+0000","last_unstale":"2023-10-09T11:07:36.268847+0000","last_undegraded":"2023-10-09T11:07:36.268847+0000","last_fullsized":"2023-10-09T11:07:36.268847+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:50.993153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"556'6742","reported_seq":6819,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.713473+0000","last_change":"2023-10-09T11:05:45.360245+0000","last_active":"2023-10-09T11:07:51.713473+0000","last_peered":"2023-10-09T11:07:51.713473+0000","last_clean":"2023-10-09T11:07:51.713473+0000","last_became_active":"2023-10-09T11:05:45.356981+0000","last_became_peered":"2023-10-09T11:05:45.356981+0000","last_unstale":"2023-10-09T11:07:51.713473+0000","last_undegraded":"2023-10-09T11:07:51.713473+0000","last_fullsized":"2023-10-09T11:07:51.713473+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6742,"log_dups_size":0,"ondisk_log_size":6742,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64782336,"num_objects":989,"num_object_clones":0,"num_object_copies":2967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5173,"num_write_kb":197152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":853,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-09T11:07:41.331644+0000","last_change":"2023-10-09T11:04:53.274332+0000","last_active":"2023-10-09T11:07:41.331644+0000","last_peered":"2023-10-09T11:07:41.331644+0000","last_clean":"2023-10-09T11:07:41.331644+0000","last_became_active":"2023-10-09T11:04:53.273798+0000","last_became_peered":"2023-10-09T11:04:53.273798+0000","last_unstale":"2023-10-09T11:07:41.331644+0000","last_undegraded":"2023-10-09T11:07:41.331644+0000","last_fullsized":"2023-10-09T11:07:41.331644+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:04.220970+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":103,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977789+0000","last_change":"2023-10-09T11:06:24.249877+0000","last_active":"2023-10-09T11:07:48.977789+0000","last_peered":"2023-10-09T11:07:48.977789+0000","last_clean":"2023-10-09T11:07:48.977789+0000","last_became_active":"2023-10-09T11:06:24.249573+0000","last_became_peered":"2023-10-09T11:06:24.249573+0000","last_unstale":"2023-10-09T11:07:48.977789+0000","last_undegraded":"2023-10-09T11:07:48.977789+0000","last_fullsized":"2023-10-09T11:07:48.977789+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:11.798192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":602,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-09T11:07:36.278498+0000","last_change":"2023-10-09T11:05:20.307347+0000","last_active":"2023-10-09T11:07:36.278498+0000","last_peered":"2023-10-09T11:07:36.278498+0000","last_clean":"2023-10-09T11:07:36.278498+0000","last_became_active":"2023-10-09T11:05:20.307126+0000","last_became_peered":"2023-10-09T11:05:20.307126+0000","last_unstale":"2023-10-09T11:07:36.278498+0000","last_undegraded":"2023-10-09T11:07:36.278498+0000","last_fullsized":"2023-10-09T11:07:36.278498+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:34.408515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":36,"stat_sum":{"num_bytes":2267742232,"num_objects":34604,"num_object_clones":0,"num_object_copies":103812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6040,"num_objects_misplaced":3898,"num_objects_unfound":0,"num_objects_dirty":34604,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":164390,"num_write_kb":6367745,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22483,"num_bytes_recovered":1449328640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7456829440,"data_stored":7456817224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":228186,"ondisk_log_size":228186,"up":108,"acting":104,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":91,"num_bytes_recovered":15742,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12694,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":626148,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":31,"num_read_kb":56,"num_write":15,"num_write_kb":147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2632652,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332495,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2074956,"kb_used_data":1171396,"kb_used_omap":0,"kb_used_meta":903488,"kb_avail":91670196,"statfs":{"total":95995035648,"available":93870280704,"internally_reserved":0,"allocated":1199509504,"data_stored":1197719999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925171712},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":1.5009999999999999,"15min":1.6839999999999999},"min":{"1min":0.27300000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":5.2539999999999996,"5min":9.8040000000000003,"15min":9.8040000000000003},"last":6.7000000000000002},{"interface":"front","average":{"1min":1.7989999999999999,"5min":1.7170000000000001,"15min":1.746},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.216},"max":{"1min":15.853,"5min":15.853,"15min":15.853},"last":0.77500000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:07:23 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":1.3959999999999999,"15min":1.52},"min":{"1min":0.23799999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.9130000000000003,"5min":11.619999999999999,"15min":11.619999999999999},"last":0.499},{"interface":"front","average":{"1min":2.0019999999999998,"5min":1.8500000000000001,"15min":1.891},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":11.038,"5min":11.038,"15min":12.193},"last":0.374}]},{"osd":2,"last update":"Mon Oct 9 11:06:50 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":2.1099999999999999,"15min":1.7070000000000001},"min":{"1min":0.32200000000000001,"5min":0.30399999999999999,"15min":0.245},"max":{"1min":4.6269999999999998,"5min":16.605,"15min":16.605},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.135,"5min":1.5129999999999999,"15min":1.6990000000000001},"min":{"1min":0.33400000000000002,"5min":0.27600000000000002,"15min":0.26600000000000001},"max":{"1min":2.8130000000000002,"5min":9.7469999999999999,"15min":13.037000000000001},"last":0.95899999999999996}]},{"osd":3,"last update":"Mon Oct 9 11:07:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.524,"15min":1.4399999999999999},"min":{"1min":0.30499999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.4770000000000003,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.1789999999999998,"5min":1.9019999999999999,"15min":1.8280000000000001},"min":{"1min":0.38200000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.968,"5min":8.2330000000000005,"15min":8.2330000000000005},"last":0.47699999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:07:02 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.0389999999999999,"15min":1.083},"min":{"1min":0.25700000000000001,"5min":0.218,"15min":0.218},"max":{"1min":3.927,"5min":7.6589999999999998,"15min":8.2940000000000005},"last":0.80800000000000005},{"interface":"front","average":{"1min":1.008,"5min":1.623,"15min":1.248},"min":{"1min":0.28299999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":9.2759999999999998,"5min":16.481000000000002,"15min":16.481000000000002},"last":0.38800000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:07:02 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.2589999999999999,"15min":1.262},"min":{"1min":0.28499999999999998,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":2.1829999999999998,"5min":15.414999999999999,"15min":15.414999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.73599999999999999,"5min":1.3460000000000001,"15min":1.175},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.9780000000000002,"5min":15.369,"15min":15.369},"last":0.83799999999999997}]},{"osd":6,"last update":"Mon Oct 9 11:07:02 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.367,"15min":1.298},"min":{"1min":0.35999999999999999,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":4.0259999999999998,"5min":14.159000000000001,"15min":14.159000000000001},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.8759999999999999,"15min":1.3959999999999999},"min":{"1min":0.35499999999999998,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":3.9630000000000001,"5min":16.538,"15min":16.538},"last":1.2270000000000001}]}]},{"osd":6,"up_from":44,"seq":188978561426,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2703220,"kb_used_data":1198380,"kb_used_omap":0,"kb_used_meta":1504832,"kb_avail":91041932,"statfs":{"total":95995035648,"available":93226938368,"internally_reserved":0,"allocated":1227141120,"data_stored":1225367114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1540947968},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.6719999999999999,"15min":1.875},"min":{"1min":0.23899999999999999,"5min":0.217,"15min":0.217},"max":{"1min":3.7389999999999999,"5min":29.379999999999999,"15min":54.743000000000002},"last":3.4700000000000002},{"interface":"front","average":{"1min":1.3120000000000001,"5min":1.6200000000000001,"15min":1.9299999999999999},"min":{"1min":0.247,"5min":0.214,"15min":0.214},"max":{"1min":7.1890000000000001,"5min":29.132999999999999,"15min":53.701000000000001},"last":0.94099999999999995}]},{"osd":1,"last update":"Mon Oct 9 11:07:26 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":1.5369999999999999,"15min":1.1679999999999999},"min":{"1min":0.318,"5min":0.26000000000000001,"15min":0.23499999999999999},"max":{"1min":5.0110000000000001,"5min":16.225999999999999,"15min":41.241999999999997},"last":0.92600000000000005},{"interface":"front","average":{"1min":2.98,"5min":2.0489999999999999,"15min":1.7350000000000001},"min":{"1min":0.31,"5min":0.26200000000000001,"15min":0.248},"max":{"1min":9.173,"5min":11.715,"15min":41.476999999999997},"last":5.1959999999999997}]},{"osd":2,"last update":"Mon Oct 9 11:06:53 2023","interfaces":[{"interface":"back","average":{"1min":2.3069999999999999,"5min":2.0009999999999999,"15min":1.889},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":10.382999999999999,"5min":10.481999999999999,"15min":12.003},"last":1.444},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.956,"15min":1.891},"min":{"1min":0.313,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.0230000000000001,"5min":12.693,"15min":15.432},"last":0.40899999999999997}]},{"osd":3,"last update":"Mon Oct 9 11:07:23 2023","interfaces":[{"interface":"back","average":{"1min":3.125,"5min":1.7629999999999999,"15min":1.554},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":28.948,"5min":28.948,"15min":28.948},"last":10.250999999999999},{"interface":"front","average":{"1min":5.3739999999999997,"5min":2.9390000000000001,"15min":2.605},"min":{"1min":0.496,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":28.893000000000001,"5min":28.893000000000001,"15min":28.893000000000001},"last":2.8849999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:06:53 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":1.321,"15min":1.1259999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.6519999999999999,"5min":17.216999999999999,"15min":17.216999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.82699999999999996,"15min":0.88200000000000001},"min":{"1min":0.28100000000000003,"5min":0.252,"15min":0.24399999999999999},"max":{"1min":1.5089999999999999,"5min":10.025,"15min":10.025},"last":0.86599999999999999}]},{"osd":5,"last update":"Mon Oct 9 11:06:53 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":1.2769999999999999,"15min":1.3839999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.25600000000000001},"max":{"1min":1.6870000000000001,"5min":10.209,"15min":19.786999999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.97399999999999998,"15min":1.0620000000000001},"min":{"1min":0.30499999999999999,"5min":0.248,"15min":0.248},"max":{"1min":3.488,"5min":9.9469999999999992,"15min":9.9469999999999992},"last":0.90500000000000003}]},{"osd":7,"last update":"Mon Oct 9 11:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":1.419,"15min":1.3169999999999999},"min":{"1min":0.34200000000000003,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.7210000000000001,"5min":11.550000000000001,"15min":12.882},"last":1.079},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.85599999999999998,"15min":0.94099999999999995},"min":{"1min":0.32800000000000001,"5min":0.31,"15min":0.27500000000000002},"max":{"1min":1.643,"5min":11.557,"15min":11.557},"last":2.4009999999999998}]}]},{"osd":1,"up_from":388,"seq":1666447310940,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1633896,"kb_used_data":1172512,"kb_used_omap":13,"kb_used_meta":461362,"kb_avail":92111256,"statfs":{"total":95995035648,"available":94321926144,"internally_reserved":0,"allocated":1200652288,"data_stored":1197730612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":472435399},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":3.895,"5min":2.593,"15min":2.4369999999999998},"min":{"1min":0.26700000000000002,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":11.960000000000001,"5min":18.192,"15min":18.192},"last":0.66500000000000004},{"interface":"front","average":{"1min":3.9390000000000001,"5min":2.8260000000000001,"15min":1.9259999999999999},"min":{"1min":0.27000000000000002,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":10.099,"5min":17.957999999999998,"15min":17.957999999999998},"last":0.68600000000000005}]},{"osd":2,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8439999999999999,"5min":2.5920000000000001,"15min":2.9300000000000002},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":9.5630000000000006,"5min":47.731000000000002,"15min":47.731000000000002},"last":1.1559999999999999},{"interface":"front","average":{"1min":3.8199999999999998,"5min":2.903,"15min":2.4769999999999999},"min":{"1min":0.29399999999999998,"5min":0.216,"15min":0.216},"max":{"1min":10.138,"5min":47.786999999999999,"15min":47.786999999999999},"last":8.9930000000000003}]},{"osd":3,"last update":"Mon Oct 9 11:07:25 2023","interfaces":[{"interface":"back","average":{"1min":3.7330000000000001,"5min":3.0870000000000002,"15min":2.0209999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":20.791,"5min":92.447999999999993,"15min":92.447999999999993},"last":6.9589999999999996},{"interface":"front","average":{"1min":3.1160000000000001,"5min":3.1179999999999999,"15min":2.028},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.217},"max":{"1min":7.3630000000000004,"5min":92.066999999999993,"15min":92.066999999999993},"last":3.3700000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.7869999999999999,"15min":2.395},"min":{"1min":0.30499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.5199999999999996,"5min":11.701000000000001,"15min":11.701000000000001},"last":1.2210000000000001},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.04,"15min":1.6240000000000001},"min":{"1min":0.314,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":4.6079999999999997,"5min":11.589,"15min":11.589},"last":0.754}]},{"osd":5,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":2.0830000000000002,"15min":2.3050000000000002},"min":{"1min":0.28999999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.0720000000000001,"5min":20.689,"15min":20.689},"last":3.3540000000000001},{"interface":"front","average":{"1min":1.764,"5min":2.0590000000000002,"15min":2.2410000000000001},"min":{"1min":0.33800000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.4900000000000002,"5min":11.65,"15min":11.65},"last":1.26}]},{"osd":6,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.833,"5min":2.4049999999999998,"15min":2.411},"min":{"1min":0.312,"5min":0.309,"15min":0.309},"max":{"1min":6.0330000000000004,"5min":20.646999999999998,"15min":20.646999999999998},"last":3.4510000000000001},{"interface":"front","average":{"1min":1.784,"5min":1.9219999999999999,"15min":2.1720000000000002},"min":{"1min":0.35599999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":5.5380000000000003,"5min":11.75,"15min":11.75},"last":1.286}]},{"osd":7,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.869,"5min":2.1640000000000001,"15min":1.615},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.32900000000000001},"max":{"1min":4.5780000000000003,"5min":11.555999999999999,"15min":11.555999999999999},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.7789999999999999,"5min":2.0219999999999998,"15min":2.3210000000000002},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.1020000000000003,"5min":20.716000000000001,"15min":20.716000000000001},"last":3.3919999999999999}]}]},{"osd":0,"up_from":80,"seq":343597383998,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1773440,"kb_used_data":823800,"kb_used_omap":28,"kb_used_meta":949475,"kb_avail":91971712,"statfs":{"total":95995035648,"available":94179033088,"internally_reserved":0,"allocated":843571200,"data_stored":840558222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":972263248},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:07:25 2023","interfaces":[{"interface":"back","average":{"1min":2.923,"5min":2.1680000000000001,"15min":2.831},"min":{"1min":0.22700000000000001,"5min":0.186,"15min":0.186},"max":{"1min":10.271000000000001,"5min":14.644,"15min":19.608000000000001},"last":0.70199999999999996},{"interface":"front","average":{"1min":3.879,"5min":2.2949999999999999,"15min":2.911},"min":{"1min":0.29499999999999998,"5min":0.214,"15min":0.214},"max":{"1min":15.834,"5min":15.834,"15min":19.992999999999999},"last":0.23799999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:06:56 2023","interfaces":[{"interface":"back","average":{"1min":2.573,"5min":2.48,"15min":2.3119999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.17199999999999999},"max":{"1min":9.0890000000000004,"5min":12.459,"15min":24.852},"last":0.42399999999999999},{"interface":"front","average":{"1min":2.5049999999999999,"5min":2.367,"15min":2.7200000000000002},"min":{"1min":0.26800000000000002,"5min":0.20300000000000001,"15min":0.19800000000000001},"max":{"1min":11.68,"5min":11.68,"15min":77.256},"last":4.5650000000000004}]},{"osd":3,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":3.2730000000000001,"15min":3.4449999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":13.189,"5min":25.050000000000001,"15min":25.050000000000001},"last":4.5330000000000004},{"interface":"front","average":{"1min":2.98,"5min":2.2770000000000001,"15min":2.0249999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":14.125999999999999,"5min":14.125999999999999,"15min":14.125999999999999},"last":0.30499999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.3799999999999999,"5min":2.1749999999999998,"15min":2.4470000000000001},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":13.106,"5min":14.507,"15min":77.177999999999997},"last":4.4409999999999998},{"interface":"front","average":{"1min":1.6919999999999999,"5min":1.5640000000000001,"15min":1.681},"min":{"1min":0.39500000000000002,"5min":0.30199999999999999,"15min":0.28899999999999998},"max":{"1min":7.9249999999999998,"5min":7.9249999999999998,"15min":10.513999999999999},"last":1.6950000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.7490000000000001,"15min":1.8400000000000001},"min":{"1min":0.39300000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.9550000000000001,"5min":7.9550000000000001,"15min":10.544},"last":0.65400000000000003},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.6499999999999999,"15min":1.8240000000000001},"min":{"1min":0.38100000000000001,"5min":0.308,"15min":0.29899999999999999},"max":{"1min":8.8439999999999994,"5min":11.212999999999999,"15min":24.754999999999999},"last":0.96399999999999997}]},{"osd":6,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":1.736,"15min":1.9390000000000001},"min":{"1min":0.38,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":7.9829999999999997,"5min":7.9829999999999997,"15min":18.920000000000002},"last":1.248},{"interface":"front","average":{"1min":2.3839999999999999,"5min":1.841,"15min":1.9590000000000001},"min":{"1min":0.42499999999999999,"5min":0.34999999999999998,"15min":0.31900000000000001},"max":{"1min":8.8810000000000002,"5min":11.257999999999999,"15min":24.800000000000001},"last":0.45700000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":2.0790000000000002,"15min":2.0289999999999999},"min":{"1min":0.496,"5min":0.35099999999999998,"15min":0.32700000000000001},"max":{"1min":8.9139999999999997,"5min":17.672000000000001,"15min":17.672000000000001},"last":9.8710000000000004},{"interface":"front","average":{"1min":2.4369999999999998,"5min":2.3839999999999999,"15min":2.552},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":13.154,"5min":17.539000000000001,"15min":77.228999999999999},"last":4.4000000000000004}]}]},{"osd":2,"up_from":90,"seq":386547056939,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1552920,"kb_used_data":804688,"kb_used_omap":81,"kb_used_meta":747886,"kb_avail":92192232,"statfs":{"total":95995035648,"available":94404845568,"internally_reserved":0,"allocated":824000512,"data_stored":821152403,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":765835482},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7250000000000001,"5min":2.6549999999999998,"15min":2.0169999999999999},"min":{"1min":0.182,"5min":0.182,"15min":0.17499999999999999},"max":{"1min":15.609,"5min":48.933999999999997,"15min":48.933999999999997},"last":4.0899999999999999},{"interface":"front","average":{"1min":2.5990000000000002,"5min":1.883,"15min":1.8440000000000001},"min":{"1min":0.26900000000000002,"5min":0.23899999999999999,"15min":0.19600000000000001},"max":{"1min":9.8010000000000002,"5min":14.515000000000001,"15min":15.595000000000001},"last":3.4820000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:07:30 2023","interfaces":[{"interface":"back","average":{"1min":4.2800000000000002,"5min":2.96,"15min":2.629},"min":{"1min":0.20200000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":24.902000000000001,"5min":49.033999999999999,"15min":49.033999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":4.7300000000000004,"5min":3.0550000000000002,"15min":2.7200000000000002},"min":{"1min":0.27500000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":25.850999999999999,"5min":48.874000000000002,"15min":48.874000000000002},"last":2.2759999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:07:17 2023","interfaces":[{"interface":"back","average":{"1min":4.1989999999999998,"5min":2.3330000000000002,"15min":1.954},"min":{"1min":0.751,"5min":0.214,"15min":0.214},"max":{"1min":15.701000000000001,"5min":15.701000000000001,"15min":15.701000000000001},"last":2.363},{"interface":"front","average":{"1min":3.5350000000000001,"5min":2.4329999999999998,"15min":2.2410000000000001},"min":{"1min":0.313,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":15.73,"5min":15.73,"15min":15.73},"last":3.242}]},{"osd":4,"last update":"Mon Oct 9 11:07:00 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":1.7130000000000001,"15min":1.871},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.722,"5min":9.7989999999999995,"15min":15.672000000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":2.0110000000000001,"5min":2.3380000000000001,"15min":1.901},"min":{"1min":0.34399999999999997,"5min":0.249,"15min":0.249},"max":{"1min":15.571,"5min":48.795000000000002,"15min":48.795000000000002},"last":2.383}]},{"osd":5,"last update":"Mon Oct 9 11:07:00 2023","interfaces":[{"interface":"back","average":{"1min":1.98,"5min":1.405,"15min":1.548},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.25900000000000001},"max":{"1min":8.5310000000000006,"5min":8.5310000000000006,"15min":12.128},"last":2.1659999999999999},{"interface":"front","average":{"1min":1.329,"5min":2.1160000000000001,"15min":1.974},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.6859999999999999,"5min":49.061,"15min":49.061},"last":0.38}]},{"osd":6,"last update":"Mon Oct 9 11:07:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0259999999999998,"5min":2.5059999999999998,"15min":1.8400000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":15.536,"5min":48.845999999999997,"15min":48.845999999999997},"last":2.3029999999999999},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.7230000000000001,"15min":1.6100000000000001},"min":{"1min":0.36799999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.4939999999999998,"5min":8.4939999999999998,"15min":10.111000000000001},"last":2.2240000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:07:00 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":2.593,"15min":1.96},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":15.59,"5min":48.966000000000001,"15min":48.966000000000001},"last":2.3220000000000001},{"interface":"front","average":{"1min":2.0270000000000001,"5min":1.746,"15min":1.7470000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":8.5549999999999997,"5min":14.94,"15min":14.94},"last":2.2679999999999998}]}]},{"osd":3,"up_from":469,"seq":2014339661868,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1443460,"kb_used_data":1148476,"kb_used_omap":11,"kb_used_meta":294836,"kb_avail":92301692,"statfs":{"total":95995035648,"available":94516932608,"internally_reserved":0,"allocated":1176039424,"data_stored":1172906423,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11309,"internal_metadata":301913043},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":3.3700000000000001,"5min":1.927,"15min":1.7889999999999999},"min":{"1min":0.27400000000000002,"5min":0.183,"15min":0.183},"max":{"1min":9.9809999999999999,"5min":14.882,"15min":14.882},"last":2.274},{"interface":"front","average":{"1min":1.7050000000000001,"5min":1.0920000000000001,"15min":0.94199999999999995},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":5.6980000000000004,"5min":5.6980000000000004,"15min":5.6980000000000004},"last":2.2080000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":3.536,"5min":2.0859999999999999,"15min":1.6299999999999999},"min":{"1min":0.17100000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":11.757,"5min":17.93,"15min":17.93},"last":1.788},{"interface":"front","average":{"1min":3.3050000000000002,"5min":1.9610000000000001,"15min":1.71},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":19.163,"5min":19.163,"15min":19.163},"last":2.3079999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":4.46,"5min":1.804,"15min":1.3260000000000001},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":38.840000000000003,"5min":38.840000000000003,"15min":38.840000000000003},"last":1.306},{"interface":"front","average":{"1min":4.7619999999999996,"5min":2.0979999999999999,"15min":1.419},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":37.991999999999997,"5min":37.991999999999997,"15min":37.991999999999997},"last":2.4359999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":2.274,"5min":1.641,"15min":1.595},"min":{"1min":0.30599999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":10.058999999999999,"5min":14.907999999999999,"15min":14.907999999999999},"last":1.0880000000000001},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.873,"15min":0.755},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.294,"5min":4.1390000000000002,"15min":4.1390000000000002},"last":1.1719999999999999}]},{"osd":5,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":1.5,"15min":1.4419999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.8550000000000004,"5min":9.3300000000000001,"15min":9.3300000000000001},"last":1.9239999999999999},{"interface":"front","average":{"1min":2.2650000000000001,"5min":1.5429999999999999,"15min":1.4750000000000001},"min":{"1min":0.32800000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.029,"5min":10.029,"15min":10.029},"last":1.145}]},{"osd":6,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.76,"15min":1.7749999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.5289999999999999,"5min":15.455,"15min":15.455},"last":1.8879999999999999},{"interface":"front","average":{"1min":2.3100000000000001,"5min":1.673,"15min":1.653},"min":{"1min":0.374,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.099,"5min":14.837,"15min":14.837},"last":1.121}]},{"osd":7,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":1.208,"15min":1.0700000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.907,"5min":6.907,"15min":6.907},"last":5.665},{"interface":"front","average":{"1min":2.0590000000000002,"5min":1.5169999999999999,"15min":1.2769999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.9320000000000004,"5min":6.9119999999999999,"15min":6.9119999999999999},"last":5.7720000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019289,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1426304,"kb_used_data":529464,"kb_used_omap":0,"kb_used_meta":896832,"kb_avail":92318848,"statfs":{"total":95995035648,"available":94534500352,"internally_reserved":0,"allocated":542171136,"data_stored":540385241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918355968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:18 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.659,"15min":2.399},"min":{"1min":0.23200000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":3.6150000000000002,"5min":13.162000000000001,"15min":213.88999999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.569,"5min":2.161,"15min":2.6379999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":4.6699999999999999,"5min":11.128,"15min":214.09800000000001},"last":0.87}]},{"osd":1,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.54,"5min":2.508,"15min":2.6299999999999999},"min":{"1min":0.34599999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":8.1229999999999993,"5min":57.966999999999999,"15min":121.41},"last":0.505},{"interface":"front","average":{"1min":2.117,"5min":2.3250000000000002,"15min":2.1200000000000001},"min":{"1min":0.26300000000000001,"5min":0.251,"15min":0.251},"max":{"1min":8.0960000000000001,"5min":57.832000000000001,"15min":122.21899999999999},"last":0.58699999999999997}]},{"osd":2,"last update":"Mon Oct 9 11:06:58 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.8580000000000001,"15min":1.7509999999999999},"min":{"1min":0.29499999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":18.407,"5min":18.407,"15min":18.407},"last":0.93200000000000005},{"interface":"front","average":{"1min":1.476,"5min":1.9810000000000001,"15min":2.0670000000000002},"min":{"1min":0.33200000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.2119999999999997,"5min":8.4250000000000007,"15min":27.818000000000001},"last":1.296}]},{"osd":3,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.837,"5min":2.4460000000000002,"15min":1.982},"min":{"1min":0.40899999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.0529999999999999,"5min":60.753,"15min":60.753},"last":0.56299999999999994},{"interface":"front","average":{"1min":2.6469999999999998,"5min":2.1389999999999998,"15min":1.492},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.0220000000000002,"5min":61.473999999999997,"15min":61.473999999999997},"last":1.109}]},{"osd":5,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":1.5049999999999999,"15min":1.4590000000000001},"min":{"1min":0.27400000000000002,"5min":0.224,"15min":0.224},"max":{"1min":3.4860000000000002,"5min":11.59,"15min":19.068999999999999},"last":1.077},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.3620000000000001,"15min":1.272},"min":{"1min":0.27300000000000002,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":3.8999999999999999,"5min":9.8610000000000007,"15min":15.535},"last":1.667}]},{"osd":6,"last update":"Mon Oct 9 11:06:58 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.3879999999999999,"15min":1.196},"min":{"1min":0.29899999999999999,"5min":0.28999999999999998,"15min":0.26300000000000001},"max":{"1min":8.3930000000000007,"5min":8.3930000000000007,"15min":15.739000000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.619,"15min":1.5189999999999999},"min":{"1min":0.35699999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.0109999999999992,"5min":12.795999999999999,"15min":14.093999999999999},"last":1.0329999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:06:58 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.6870000000000001,"15min":1.6399999999999999},"min":{"1min":0.318,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":10.15,"5min":12.753,"15min":19.923999999999999},"last":1.135},{"interface":"front","average":{"1min":1.288,"5min":1.5109999999999999,"15min":1.5309999999999999},"min":{"1min":0.316,"5min":0.29799999999999999,"15min":0.255},"max":{"1min":10.853,"5min":10.853,"15min":19.859999999999999},"last":0.90700000000000003}]}]},{"osd":5,"up_from":37,"seq":158913790358,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1497080,"kb_used_data":586160,"kb_used_omap":0,"kb_used_meta":910912,"kb_avail":92248072,"statfs":{"total":95995035648,"available":94462025728,"internally_reserved":0,"allocated":600227840,"data_stored":598454489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932773888},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:25 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":2.3809999999999998,"15min":2.3330000000000002},"min":{"1min":0.22900000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":6.2249999999999996,"5min":49.665999999999997,"15min":58.271999999999998},"last":1.4670000000000001},{"interface":"front","average":{"1min":1.605,"5min":2.427,"15min":2.3809999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.2649999999999997,"5min":49.701999999999998,"15min":58.25},"last":1.4870000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:07:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":2.101,"15min":2.032},"min":{"1min":0.26600000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.2199999999999998,"5min":32.491999999999997,"15min":32.491999999999997},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.7709999999999999,"5min":2.3100000000000001,"15min":1.8680000000000001},"min":{"1min":0.39300000000000002,"5min":0.247,"15min":0.247},"max":{"1min":5.2130000000000001,"5min":32.537999999999997,"15min":32.537999999999997},"last":0.46100000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:07:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.454,"15min":1.786},"min":{"1min":0.307,"5min":0.254,"15min":0.221},"max":{"1min":5.1269999999999998,"5min":6.3929999999999998,"15min":11.257999999999999},"last":0.67100000000000004},{"interface":"front","average":{"1min":2.1869999999999998,"5min":2.0129999999999999,"15min":1.9370000000000001},"min":{"1min":0.29099999999999998,"5min":0.253,"15min":0.253},"max":{"1min":8.0359999999999996,"5min":11.673,"15min":14.621},"last":0.49299999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:07:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4449999999999998,"5min":1.6439999999999999,"15min":1.5409999999999999},"min":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":15.535,"5min":15.535,"15min":15.535},"last":0.47299999999999998},{"interface":"front","average":{"1min":2.3460000000000001,"5min":1.8560000000000001,"15min":1.8240000000000001},"min":{"1min":0.38700000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":8.8940000000000001,"5min":8.8940000000000001,"15min":8.8940000000000001},"last":7.8620000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:07:30 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":1.206,"15min":0.98899999999999999},"min":{"1min":0.28000000000000003,"5min":0.25700000000000001,"15min":0.216},"max":{"1min":2.0099999999999998,"5min":12.042,"15min":12.042},"last":0.97099999999999997},{"interface":"front","average":{"1min":0.70699999999999996,"5min":1.2150000000000001,"15min":1.165},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.5910000000000002,"5min":9.2490000000000006,"15min":9.2490000000000006},"last":1.887}]},{"osd":6,"last update":"Mon Oct 9 11:07:42 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.3300000000000001,"15min":1.2110000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27100000000000002},"max":{"1min":1.5669999999999999,"5min":9.1969999999999992,"15min":9.1969999999999992},"last":2.3780000000000001},{"interface":"front","average":{"1min":0.65300000000000002,"5min":1.288,"15min":1.0629999999999999},"min":{"1min":0.33700000000000002,"5min":0.32800000000000001,"15min":0.29099999999999998},"max":{"1min":2.6059999999999999,"5min":11.983000000000001,"15min":11.983000000000001},"last":0.51800000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:07:06 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.421,"15min":1.3660000000000001},"min":{"1min":0.307,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.5070000000000001,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":0.73599999999999999},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.4710000000000001,"15min":1.254},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30099999999999999},"max":{"1min":3.0409999999999999,"5min":11.726000000000001,"15min":11.726000000000001},"last":0.54800000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3733,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":828506112,"data_stored":828506112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1164705792,"data_stored":1164705792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":802099200,"data_stored":802095128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1157955584,"data_stored":1157955584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":526258176,"data_stored":526254104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":592773120,"data_stored":592773120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1200820224,"data_stored":1200816152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1183711232,"data_stored":1183711232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:07:53.417 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:07:53.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:53 smithi155 ceph-mon[184854]: pgmap v1356: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 6040/103887 objects degraded (5.814%); 3898/103887 objects misplaced (3.752%); 2.2 MiB/s, 41 objects/s recovering 2023-10-09T11:07:53.492 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:07:53.939 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:53.961 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:07:54.054 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:54.068 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:54.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:54.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:54.077 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T11:07:54.175 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:07:54.244 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:54.312 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:54 smithi125 ceph-mon[195507]: 11.7 scrub starts 2023-10-09T11:07:54.312 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:54 smithi125 ceph-mon[195507]: from='client.36726 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:07:54.312 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:54 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 6040/103887 objects degraded (5.814%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:54.312 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:54 smithi125 ceph-mon[200197]: 11.7 scrub starts 2023-10-09T11:07:54.312 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:54 smithi125 ceph-mon[200197]: from='client.36726 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:07:54.313 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:54 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 6040/103887 objects degraded (5.814%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:54.332 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:07:54.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:54.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:07:54.436 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:54 smithi155 ceph-mon[184854]: 11.7 scrub starts 2023-10-09T11:07:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:54 smithi155 ceph-mon[184854]: from='client.36726 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:07:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:54 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 6040/103887 objects degraded (5.814%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:54.490 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:07:54.496 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:54.591 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:54.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:54.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:54.697 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:54.730 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:55.067 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:55 smithi125 ceph-mon[195507]: 11.17 scrub starts 2023-10-09T11:07:55.067 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:55 smithi125 ceph-mon[195507]: pgmap v1357: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 6040/93708 objects degraded (6.446%); 3400/93708 objects misplaced (3.628%); 2.2 MiB/s, 89 objects/s recovering 2023-10-09T11:07:55.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:55 smithi125 ceph-mon[200197]: 11.17 scrub starts 2023-10-09T11:07:55.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:55 smithi125 ceph-mon[200197]: pgmap v1357: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 6040/93708 objects degraded (6.446%); 3400/93708 objects misplaced (3.628%); 2.2 MiB/s, 89 objects/s recovering 2023-10-09T11:07:55.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:55 smithi155 ceph-mon[184854]: 11.17 scrub starts 2023-10-09T11:07:55.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:55 smithi155 ceph-mon[184854]: pgmap v1357: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 6040/93708 objects degraded (6.446%); 3400/93708 objects misplaced (3.628%); 2.2 MiB/s, 89 objects/s recovering 2023-10-09T11:07:55.573 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:55.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:55.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:55.681 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:55.730 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:56.217 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:56 smithi155 ceph-mon[184854]: 1.11 scrub starts 2023-10-09T11:07:56.356 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:56 smithi125 ceph-mon[195507]: 1.11 scrub starts 2023-10-09T11:07:56.356 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:56 smithi125 ceph-mon[200197]: 1.11 scrub starts 2023-10-09T11:07:56.905 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:07:56.906 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:07:57.211 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:57 smithi155 ceph-mon[184854]: pgmap v1358: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 6040/93708 objects degraded (6.446%); 3400/93708 objects misplaced (3.628%); 2.2 MiB/s, 89 objects/s recovering 2023-10-09T11:07:57.283 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T11:07:57.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:57 smithi125 ceph-mon[200197]: pgmap v1358: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 6040/93708 objects degraded (6.446%); 3400/93708 objects misplaced (3.628%); 2.2 MiB/s, 89 objects/s recovering 2023-10-09T11:07:57.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:57 smithi125 ceph-mon[195507]: pgmap v1358: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 6040/93708 objects degraded (6.446%); 3400/93708 objects misplaced (3.628%); 2.2 MiB/s, 89 objects/s recovering 2023-10-09T11:07:57.674 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:07:57.680 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:57.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:57.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:57.703 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:07:57.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:57.844 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:57.896 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:57.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:57.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:58.001 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:58.057 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:58.887 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:07:58.888 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:07:59.115 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:07:59.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:59.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:07:59.180 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:59 smithi125 ceph-mon[200197]: 11.7 scrub starts 2023-10-09T11:07:59.180 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:59 smithi125 ceph-mon[200197]: pgmap v1359: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 6040/89475 objects degraded (6.750%); 3400/89475 objects misplaced (3.800%); 0 B/s, 42 objects/s recovering 2023-10-09T11:07:59.180 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:07:59 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 6040/89475 objects degraded (6.750%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:59.180 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:59 smithi125 ceph-mon[195507]: 11.7 scrub starts 2023-10-09T11:07:59.180 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:59 smithi125 ceph-mon[195507]: pgmap v1359: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 6040/89475 objects degraded (6.750%); 3400/89475 objects misplaced (3.800%); 0 B/s, 42 objects/s recovering 2023-10-09T11:07:59.180 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:07:59 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 6040/89475 objects degraded (6.750%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:59.193 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:59 smithi155 ceph-mon[184854]: 11.7 scrub starts 2023-10-09T11:07:59.193 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:59 smithi155 ceph-mon[184854]: pgmap v1359: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 6040/89475 objects degraded (6.750%); 3400/89475 objects misplaced (3.800%); 0 B/s, 42 objects/s recovering 2023-10-09T11:07:59.193 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:07:59 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 6040/89475 objects degraded (6.750%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:07:59.223 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:07:59.270 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:59.283 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T11:07:59.305 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:07:59.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:59.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:07:59.419 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:07:59.460 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:59.494 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:07:59.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:59.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:07:59.598 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:07:59.638 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:07:59.666 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:07:59.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:07:59.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:07:59.680 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:07:59.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:07:59.853 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:00.017 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:08:00.031 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:08:00.108 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:00.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:00 smithi125 ceph-mon[195507]: 11.17 scrub starts 2023-10-09T11:08:00.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:00 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1063371976' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:00.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:00 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:00.108 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:08:00.120 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:08:00.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:00.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:00.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:00.299 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:00 smithi155 ceph-mon[184854]: 11.17 scrub starts 2023-10-09T11:08:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:00 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1063371976' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:00 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:00.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:08:00.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:00 smithi125 ceph-mon[200197]: 11.17 scrub starts 2023-10-09T11:08:00.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:00 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1063371976' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:00.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:00 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:00.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:08:00.971 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 11:08:00 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a[169145]: logger=cleanup t=2023-10-09T11:08:00.843825105Z level=info msg="Completed cleanup jobs" duration=51.259267ms 2023-10-09T11:08:01.421 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:01 smithi155 ceph-mon[184854]: pgmap v1360: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 6040/78549 objects degraded (7.689%); 2851/78549 objects misplaced (3.630%); 0 B/s, 79 objects/s recovering 2023-10-09T11:08:01.422 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:01 smithi155 ceph-mon[184854]: from='client.46697 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:01.422 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:01 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:08:01.422 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:01 smithi155 ceph-mon[184854]: failed to encode map e557 with expected crc 2023-10-09T11:08:01.422 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:01 smithi155 ceph-mon[184854]: failed to encode map e557 with expected crc 2023-10-09T11:08:01.422 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:01 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:08:01.422 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:01 smithi155 ceph-mon[184854]: osdmap e557: 8 total, 8 up, 8 in 2023-10-09T11:08:01.422 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:01 smithi155 ceph-mon[184854]: failed to encode map e557 with expected crc 2023-10-09T11:08:01.499 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:01.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:01.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:01.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[195507]: pgmap v1360: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 6040/78549 objects degraded (7.689%); 2851/78549 objects misplaced (3.630%); 0 B/s, 79 objects/s recovering 2023-10-09T11:08:01.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[195507]: from='client.46697 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:01.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:08:01.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[195507]: failed to encode map e557 with expected crc 2023-10-09T11:08:01.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[195507]: failed to encode map e557 with expected crc 2023-10-09T11:08:01.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:08:01.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[195507]: osdmap e557: 8 total, 8 up, 8 in 2023-10-09T11:08:01.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[195507]: failed to encode map e557 with expected crc 2023-10-09T11:08:01.506 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:08:01] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.43.0" 2023-10-09T11:08:01.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[200197]: pgmap v1360: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 6040/78549 objects degraded (7.689%); 2851/78549 objects misplaced (3.630%); 0 B/s, 79 objects/s recovering 2023-10-09T11:08:01.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[200197]: from='client.46697 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:01.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:08:01.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[200197]: failed to encode map e557 with expected crc 2023-10-09T11:08:01.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[200197]: failed to encode map e557 with expected crc 2023-10-09T11:08:01.507 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:08:01.507 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[200197]: osdmap e557: 8 total, 8 up, 8 in 2023-10-09T11:08:01.507 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:01 smithi125 ceph-mon[200197]: failed to encode map e557 with expected crc 2023-10-09T11:08:01.606 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:01.673 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:01.805 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1360,"stamp":"2023-10-09T11:07:59.908017+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1714856828,"num_objects":26183,"num_object_clones":0,"num_object_copies":78549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6040,"num_objects_misplaced":2851,"num_objects_unfound":0,"num_objects_dirty":26183,"num_whiteouts":0,"num_read":32,"num_read_kb":57,"num_write":172912,"num_write_kb":6367921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23383,"num_bytes_recovered":1451977034,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":236357,"ondisk_log_size":236357,"up":417,"acting":413,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":428,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12737732,"kb_used_data":6080388,"kb_used_omap":134,"kb_used_meta":6656569,"kb_avail":737223484,"statfs":{"total":767960285184,"available":754916847616,"internally_reserved":0,"allocated":6226317312,"data_stored":6207217567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":137532,"internal_metadata":6816327364},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-639827968,"num_objects":-9763,"num_object_clones":0,"num_object_copies":-29289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1047,"num_objects_unfound":0,"num_objects_dirty":-9763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9763,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":799,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.011265"},"pg_stats":[{"pgid":"11.23","version":"556'5987","reported_seq":9007,"reported_epoch":556,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:07:58.057924+0000","last_change":"2023-10-09T11:07:34.314839+0000","last_active":"2023-10-09T11:07:58.057924+0000","last_peered":"2023-10-09T11:07:58.057924+0000","last_clean":"2023-10-09T11:07:32.177661+0000","last_became_active":"2023-10-09T11:07:34.308281+0000","last_became_peered":"2023-10-09T11:07:34.308281+0000","last_unstale":"2023-10-09T11:07:58.057924+0000","last_undegraded":"2023-10-09T11:07:58.057924+0000","last_fullsized":"2023-10-09T11:07:58.057924+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5987,"log_dups_size":0,"ondisk_log_size":5987,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26165248,"num_objects":399,"num_object_clones":0,"num_object_copies":1197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":399,"num_objects_unfound":0,"num_objects_dirty":399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2771,"num_write_kb":101456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1213,"num_bytes_recovered":72024064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":399}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.22","version":"556'5746","reported_seq":5835,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.062553+0000","last_change":"2023-10-09T11:06:43.010089+0000","last_active":"2023-10-09T11:07:58.062553+0000","last_peered":"2023-10-09T11:07:58.062553+0000","last_clean":"2023-10-09T11:07:58.062553+0000","last_became_active":"2023-10-09T11:06:43.008608+0000","last_became_peered":"2023-10-09T11:06:43.008608+0000","last_unstale":"2023-10-09T11:07:58.062553+0000","last_undegraded":"2023-10-09T11:07:58.062553+0000","last_fullsized":"2023-10-09T11:07:58.062553+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5746,"log_dups_size":0,"ondisk_log_size":5746,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27000832,"num_objects":412,"num_object_clones":0,"num_object_copies":1236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2643,"num_write_kb":97760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.21","version":"556'6054","reported_seq":6922,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.270001+0000","last_change":"2023-10-09T11:07:23.970291+0000","last_active":"2023-10-09T11:07:58.270001+0000","last_peered":"2023-10-09T11:07:58.270001+0000","last_clean":"2023-10-09T11:07:58.270001+0000","last_became_active":"2023-10-09T11:06:43.015074+0000","last_became_peered":"2023-10-09T11:06:43.015074+0000","last_unstale":"2023-10-09T11:07:58.270001+0000","last_undegraded":"2023-10-09T11:07:58.270001+0000","last_fullsized":"2023-10-09T11:07:58.270001+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6054,"log_dups_size":0,"ondisk_log_size":6054,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21413888,"num_objects":327,"num_object_clones":0,"num_object_copies":981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2821,"num_write_kb":100720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":344,"num_bytes_recovered":22511616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.20","version":"556'5859","reported_seq":8266,"reported_epoch":556,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:07:58.244425+0000","last_change":"2023-10-09T11:07:34.316666+0000","last_active":"2023-10-09T11:07:58.244425+0000","last_peered":"2023-10-09T11:07:58.244425+0000","last_clean":"2023-10-09T11:07:32.195530+0000","last_became_active":"2023-10-09T11:07:34.309531+0000","last_became_peered":"2023-10-09T11:07:34.309531+0000","last_unstale":"2023-10-09T11:07:58.244425+0000","last_undegraded":"2023-10-09T11:07:58.244425+0000","last_fullsized":"2023-10-09T11:07:58.244425+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5859,"log_dups_size":0,"ondisk_log_size":5859,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28262400,"num_objects":431,"num_object_clones":0,"num_object_copies":1293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":431,"num_objects_unfound":0,"num_objects_dirty":431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2693,"num_write_kb":99984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1150,"num_bytes_recovered":72941568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,6],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,3,6","objects":431}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.12","version":"556'6918","reported_seq":6997,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.267637+0000","last_change":"2023-10-09T11:05:45.369342+0000","last_active":"2023-10-09T11:07:58.267637+0000","last_peered":"2023-10-09T11:07:58.267637+0000","last_clean":"2023-10-09T11:07:58.267637+0000","last_became_active":"2023-10-09T11:05:45.367721+0000","last_became_peered":"2023-10-09T11:05:45.367721+0000","last_unstale":"2023-10-09T11:07:58.267637+0000","last_undegraded":"2023-10-09T11:07:58.267637+0000","last_fullsized":"2023-10-09T11:07:58.267637+0000","mapping_epoch":509,"log_start":"498'1287","ondisk_log_start":"498'1287","created":509,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5631,"log_dups_size":0,"ondisk_log_size":5631,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51511296,"num_objects":786,"num_object_clones":0,"num_object_copies":2358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5380,"num_write_kb":197312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"76'161","reported_seq":843,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.974725+0000","last_change":"2023-10-09T11:07:36.287455+0000","last_active":"2023-10-09T11:07:48.974725+0000","last_peered":"2023-10-09T11:07:48.974725+0000","last_clean":"2023-10-09T11:07:48.974725+0000","last_became_active":"2023-10-09T11:07:36.287052+0000","last_became_peered":"2023-10-09T11:07:36.287052+0000","last_unstale":"2023-10-09T11:07:48.974725+0000","last_undegraded":"2023-10-09T11:07:48.974725+0000","last_fullsized":"2023-10-09T11:07:48.974725+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:18.864873+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":338,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.985114+0000","last_change":"2023-10-09T11:03:21.185383+0000","last_active":"2023-10-09T11:07:48.985114+0000","last_peered":"2023-10-09T11:07:48.985114+0000","last_clean":"2023-10-09T11:07:48.985114+0000","last_became_active":"2023-10-09T11:03:21.185022+0000","last_became_peered":"2023-10-09T11:03:21.185022+0000","last_unstale":"2023-10-09T11:07:48.985114+0000","last_undegraded":"2023-10-09T11:07:48.985114+0000","last_fullsized":"2023-10-09T11:07:48.985114+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:05.230434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":82,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.966156+0000","last_change":"2023-10-09T11:04:50.319477+0000","last_active":"2023-10-09T11:07:48.966156+0000","last_peered":"2023-10-09T11:07:48.966156+0000","last_clean":"2023-10-09T11:07:48.966156+0000","last_became_active":"2023-10-09T11:04:50.319148+0000","last_became_peered":"2023-10-09T11:04:50.319148+0000","last_unstale":"2023-10-09T11:07:48.966156+0000","last_undegraded":"2023-10-09T11:07:48.966156+0000","last_fullsized":"2023-10-09T11:07:48.966156+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:48.725566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"556'7135","reported_seq":8545,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.278630+0000","last_change":"2023-10-09T11:06:54.752222+0000","last_active":"2023-10-09T11:07:58.278630+0000","last_peered":"2023-10-09T11:07:58.278630+0000","last_clean":"2023-10-09T11:07:58.278630+0000","last_became_active":"2023-10-09T11:05:45.395271+0000","last_became_peered":"2023-10-09T11:05:45.395271+0000","last_unstale":"2023-10-09T11:07:58.278630+0000","last_undegraded":"2023-10-09T11:07:58.278630+0000","last_fullsized":"2023-10-09T11:07:58.278630+0000","mapping_epoch":509,"log_start":"502'1416","ondisk_log_start":"502'1416","created":509,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5719,"log_dups_size":0,"ondisk_log_size":5719,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59277312,"num_objects":905,"num_object_clones":0,"num_object_copies":2715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5517,"num_write_kb":205472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":659,"num_bytes_recovered":43188224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":713,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975112+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:07:48.975112+0000","last_peered":"2023-10-09T11:07:48.975112+0000","last_clean":"2023-10-09T11:07:48.975112+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:07:48.975112+0000","last_undegraded":"2023-10-09T11:07:48.975112+0000","last_fullsized":"2023-10-09T11:07:48.975112+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":270,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.986071+0000","last_change":"2023-10-09T11:04:50.333711+0000","last_active":"2023-10-09T11:07:48.986071+0000","last_peered":"2023-10-09T11:07:48.986071+0000","last_clean":"2023-10-09T11:07:48.986071+0000","last_became_active":"2023-10-09T11:04:50.333262+0000","last_became_peered":"2023-10-09T11:04:50.333262+0000","last_unstale":"2023-10-09T11:07:48.986071+0000","last_undegraded":"2023-10-09T11:07:48.986071+0000","last_fullsized":"2023-10-09T11:07:48.986071+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.843765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":16,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976642+0000","last_change":"2023-10-09T11:07:33.237903+0000","last_active":"2023-10-09T11:07:48.976642+0000","last_peered":"2023-10-09T11:07:48.976642+0000","last_clean":"2023-10-09T11:07:48.976642+0000","last_became_active":"2023-10-09T11:07:33.237618+0000","last_became_peered":"2023-10-09T11:07:33.237618+0000","last_unstale":"2023-10-09T11:07:48.976642+0000","last_undegraded":"2023-10-09T11:07:48.976642+0000","last_fullsized":"2023-10-09T11:07:48.976642+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:25.600818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.10","version":"556'7003","reported_seq":7689,"reported_epoch":556,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:07:58.318551+0000","last_change":"2023-10-09T11:07:34.347309+0000","last_active":"2023-10-09T11:07:58.318551+0000","last_peered":"2023-10-09T11:07:58.318551+0000","last_clean":"2023-10-09T11:06:06.955693+0000","last_became_active":"2023-10-09T11:07:33.305104+0000","last_became_peered":"2023-10-09T11:07:33.305104+0000","last_unstale":"2023-10-09T11:07:58.318551+0000","last_undegraded":"2023-10-09T11:07:34.345913+0000","last_fullsized":"2023-10-09T11:07:33.208203+0000","mapping_epoch":549,"log_start":"498'1216","ondisk_log_start":"498'1216","created":509,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5787,"log_dups_size":0,"ondisk_log_size":5787,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52723712,"num_objects":805,"num_object_clones":0,"num_object_copies":2415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2269,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5441,"num_write_kb":199840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":292,"num_bytes_recovered":19136512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":149},{"shards":"1,3,6","objects":656}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":731,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976012+0000","last_change":"2023-10-09T11:07:36.284534+0000","last_active":"2023-10-09T11:07:48.976012+0000","last_peered":"2023-10-09T11:07:48.976012+0000","last_clean":"2023-10-09T11:07:48.976012+0000","last_became_active":"2023-10-09T11:07:36.284254+0000","last_became_peered":"2023-10-09T11:07:36.284254+0000","last_unstale":"2023-10-09T11:07:48.976012+0000","last_undegraded":"2023-10-09T11:07:48.976012+0000","last_fullsized":"2023-10-09T11:07:48.976012+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:19.159630+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":82,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.959664+0000","last_change":"2023-10-09T11:04:50.310477+0000","last_active":"2023-10-09T11:07:48.959664+0000","last_peered":"2023-10-09T11:07:48.959664+0000","last_clean":"2023-10-09T11:07:48.959664+0000","last_became_active":"2023-10-09T11:04:50.310067+0000","last_became_peered":"2023-10-09T11:04:50.310067+0000","last_unstale":"2023-10-09T11:07:48.959664+0000","last_undegraded":"2023-10-09T11:07:48.959664+0000","last_fullsized":"2023-10-09T11:07:48.959664+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:52.884716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":365,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977865+0000","last_change":"2023-10-09T11:05:20.310581+0000","last_active":"2023-10-09T11:07:48.977865+0000","last_peered":"2023-10-09T11:07:48.977865+0000","last_clean":"2023-10-09T11:07:48.977865+0000","last_became_active":"2023-10-09T11:05:20.309830+0000","last_became_peered":"2023-10-09T11:05:20.309830+0000","last_unstale":"2023-10-09T11:07:48.977865+0000","last_undegraded":"2023-10-09T11:07:48.977865+0000","last_fullsized":"2023-10-09T11:07:48.977865+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:24.508211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1f","version":"556'8739","reported_seq":15545,"reported_epoch":556,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T11:07:58.337191+0000","last_change":"2023-10-09T11:07:34.394394+0000","last_active":"2023-10-09T11:07:58.337191+0000","last_peered":"2023-10-09T11:07:58.337191+0000","last_clean":"2023-10-09T11:06:06.963741+0000","last_became_active":"2023-10-09T11:07:34.331654+0000","last_became_peered":"2023-10-09T11:07:34.331654+0000","last_unstale":"2023-10-09T11:07:58.337191+0000","last_undegraded":"2023-10-09T11:07:58.337191+0000","last_fullsized":"2023-10-09T11:07:34.216155+0000","mapping_epoch":550,"log_start":"504'1895","ondisk_log_start":"504'1895","created":531,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6844,"log_dups_size":0,"ondisk_log_size":6844,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51838976,"num_objects":791,"num_object_clones":0,"num_object_copies":2373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5509,"num_write_kb":201600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3358,"num_bytes_recovered":164298752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":791}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":771,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976427+0000","last_change":"2023-10-09T11:07:36.282502+0000","last_active":"2023-10-09T11:07:48.976427+0000","last_peered":"2023-10-09T11:07:48.976427+0000","last_clean":"2023-10-09T11:07:48.976427+0000","last_became_active":"2023-10-09T11:07:36.282133+0000","last_became_peered":"2023-10-09T11:07:36.282133+0000","last_unstale":"2023-10-09T11:07:48.976427+0000","last_undegraded":"2023-10-09T11:07:48.976427+0000","last_fullsized":"2023-10-09T11:07:48.976427+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:03.762383+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":192,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977943+0000","last_change":"2023-10-09T11:06:24.245488+0000","last_active":"2023-10-09T11:07:48.977943+0000","last_peered":"2023-10-09T11:07:48.977943+0000","last_clean":"2023-10-09T11:07:48.977943+0000","last_became_active":"2023-10-09T11:06:24.245167+0000","last_became_peered":"2023-10-09T11:06:24.245167+0000","last_unstale":"2023-10-09T11:07:48.977943+0000","last_undegraded":"2023-10-09T11:07:48.977943+0000","last_fullsized":"2023-10-09T11:07:48.977943+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:31.702568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":91,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.959827+0000","last_change":"2023-10-09T11:05:20.299453+0000","last_active":"2023-10-09T11:07:48.959827+0000","last_peered":"2023-10-09T11:07:48.959827+0000","last_clean":"2023-10-09T11:07:48.959827+0000","last_became_active":"2023-10-09T11:05:20.299117+0000","last_became_peered":"2023-10-09T11:05:20.299117+0000","last_unstale":"2023-10-09T11:07:48.959827+0000","last_undegraded":"2023-10-09T11:07:48.959827+0000","last_fullsized":"2023-10-09T11:07:48.959827+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:30.072813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1e","version":"556'8534","reported_seq":8613,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.304548+0000","last_change":"2023-10-09T11:06:43.029410+0000","last_active":"2023-10-09T11:07:58.304548+0000","last_peered":"2023-10-09T11:07:58.304548+0000","last_clean":"2023-10-09T11:07:58.304548+0000","last_became_active":"2023-10-09T11:06:43.026551+0000","last_became_peered":"2023-10-09T11:06:43.026551+0000","last_unstale":"2023-10-09T11:07:58.304548+0000","last_undegraded":"2023-10-09T11:07:58.304548+0000","last_fullsized":"2023-10-09T11:07:58.304548+0000","mapping_epoch":531,"log_start":"504'1803","ondisk_log_start":"504'1803","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6731,"log_dups_size":0,"ondisk_log_size":6731,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53411840,"num_objects":815,"num_object_clones":0,"num_object_copies":2445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5359,"num_write_kb":197568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":885,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.979362+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:07:48.979362+0000","last_peered":"2023-10-09T11:07:48.979362+0000","last_clean":"2023-10-09T11:07:48.979362+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:07:48.979362+0000","last_undegraded":"2023-10-09T11:07:48.979362+0000","last_fullsized":"2023-10-09T11:07:48.979362+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":427,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975786+0000","last_change":"2023-10-09T11:07:33.242665+0000","last_active":"2023-10-09T11:07:48.975786+0000","last_peered":"2023-10-09T11:07:48.975786+0000","last_clean":"2023-10-09T11:07:48.975786+0000","last_became_active":"2023-10-09T11:07:33.242313+0000","last_became_peered":"2023-10-09T11:07:33.242313+0000","last_unstale":"2023-10-09T11:07:48.975786+0000","last_undegraded":"2023-10-09T11:07:48.975786+0000","last_fullsized":"2023-10-09T11:07:48.975786+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:20.565135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":109,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.960341+0000","last_change":"2023-10-09T11:07:33.225683+0000","last_active":"2023-10-09T11:07:48.960341+0000","last_peered":"2023-10-09T11:07:48.960341+0000","last_clean":"2023-10-09T11:07:48.960341+0000","last_became_active":"2023-10-09T11:07:33.225275+0000","last_became_peered":"2023-10-09T11:07:33.225275+0000","last_unstale":"2023-10-09T11:07:48.960341+0000","last_undegraded":"2023-10-09T11:07:48.960341+0000","last_fullsized":"2023-10-09T11:07:48.960341+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:10.131060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1d","version":"556'8677","reported_seq":11594,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:56.243910+0000","last_change":"2023-10-09T11:07:23.727748+0000","last_active":"2023-10-09T11:07:56.243910+0000","last_peered":"2023-10-09T11:07:56.243910+0000","last_clean":"2023-10-09T11:07:56.243910+0000","last_became_active":"2023-10-09T11:06:43.012614+0000","last_became_peered":"2023-10-09T11:06:43.012614+0000","last_unstale":"2023-10-09T11:07:56.243910+0000","last_undegraded":"2023-10-09T11:07:56.243910+0000","last_fullsized":"2023-10-09T11:07:56.243910+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":8677,"log_dups_size":0,"ondisk_log_size":8677,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54362124,"num_objects":830,"num_object_clones":0,"num_object_copies":2490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":830,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5456,"num_write_kb":201121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":704,"num_bytes_recovered":46137344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":785,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977687+0000","last_change":"2023-10-09T11:05:23.328981+0000","last_active":"2023-10-09T11:07:48.977687+0000","last_peered":"2023-10-09T11:07:48.977687+0000","last_clean":"2023-10-09T11:07:48.977687+0000","last_became_active":"2023-10-09T11:05:23.328616+0000","last_became_peered":"2023-10-09T11:05:23.328616+0000","last_unstale":"2023-10-09T11:07:48.977687+0000","last_undegraded":"2023-10-09T11:07:48.977687+0000","last_fullsized":"2023-10-09T11:07:48.977687+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:15.315819+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":22,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976339+0000","last_change":"2023-10-09T11:07:33.265842+0000","last_active":"2023-10-09T11:07:48.976339+0000","last_peered":"2023-10-09T11:07:48.976339+0000","last_clean":"2023-10-09T11:07:48.976339+0000","last_became_active":"2023-10-09T11:07:33.232311+0000","last_became_peered":"2023-10-09T11:07:33.232311+0000","last_unstale":"2023-10-09T11:07:48.976339+0000","last_undegraded":"2023-10-09T11:07:48.976339+0000","last_fullsized":"2023-10-09T11:07:48.976339+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:23.402582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":334,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975945+0000","last_change":"2023-10-09T11:05:20.316265+0000","last_active":"2023-10-09T11:07:48.975945+0000","last_peered":"2023-10-09T11:07:48.975945+0000","last_clean":"2023-10-09T11:07:48.975945+0000","last_became_active":"2023-10-09T11:05:20.315853+0000","last_became_peered":"2023-10-09T11:05:20.315853+0000","last_unstale":"2023-10-09T11:07:48.975945+0000","last_undegraded":"2023-10-09T11:07:48.975945+0000","last_fullsized":"2023-10-09T11:07:48.975945+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:35.093981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1c","version":"556'8610","reported_seq":8690,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.298576+0000","last_change":"2023-10-09T11:06:43.025845+0000","last_active":"2023-10-09T11:07:58.298576+0000","last_peered":"2023-10-09T11:07:58.298576+0000","last_clean":"2023-10-09T11:07:58.298576+0000","last_became_active":"2023-10-09T11:06:43.022156+0000","last_became_peered":"2023-10-09T11:06:43.022156+0000","last_unstale":"2023-10-09T11:07:58.298576+0000","last_undegraded":"2023-10-09T11:07:58.298576+0000","last_fullsized":"2023-10-09T11:07:58.298576+0000","mapping_epoch":531,"log_start":"504'1919","ondisk_log_start":"504'1919","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6691,"log_dups_size":0,"ondisk_log_size":6691,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55148544,"num_objects":842,"num_object_clones":0,"num_object_copies":2526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5402,"num_write_kb":199776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":948,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.987078+0000","last_change":"2023-10-09T11:04:53.274921+0000","last_active":"2023-10-09T11:07:48.987078+0000","last_peered":"2023-10-09T11:07:48.987078+0000","last_clean":"2023-10-09T11:07:48.987078+0000","last_became_active":"2023-10-09T11:04:53.274518+0000","last_became_peered":"2023-10-09T11:04:53.274518+0000","last_unstale":"2023-10-09T11:07:48.987078+0000","last_undegraded":"2023-10-09T11:07:48.987078+0000","last_fullsized":"2023-10-09T11:07:48.987078+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:14.315301+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":26,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976807+0000","last_change":"2023-10-09T11:07:33.262559+0000","last_active":"2023-10-09T11:07:48.976807+0000","last_peered":"2023-10-09T11:07:48.976807+0000","last_clean":"2023-10-09T11:07:48.976807+0000","last_became_active":"2023-10-09T11:07:33.231695+0000","last_became_peered":"2023-10-09T11:07:33.231695+0000","last_unstale":"2023-10-09T11:07:48.976807+0000","last_undegraded":"2023-10-09T11:07:48.976807+0000","last_fullsized":"2023-10-09T11:07:48.976807+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:59.006940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":577,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977079+0000","last_change":"2023-10-09T11:05:20.311424+0000","last_active":"2023-10-09T11:07:48.977079+0000","last_peered":"2023-10-09T11:07:48.977079+0000","last_clean":"2023-10-09T11:07:48.977079+0000","last_became_active":"2023-10-09T11:05:20.311097+0000","last_became_peered":"2023-10-09T11:05:20.311097+0000","last_unstale":"2023-10-09T11:07:48.977079+0000","last_undegraded":"2023-10-09T11:07:48.977079+0000","last_fullsized":"2023-10-09T11:07:48.977079+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:19.274582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1b","version":"556'8650","reported_seq":8730,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.245553+0000","last_change":"2023-10-09T11:06:43.035498+0000","last_active":"2023-10-09T11:07:58.245553+0000","last_peered":"2023-10-09T11:07:58.245553+0000","last_clean":"2023-10-09T11:07:58.245553+0000","last_became_active":"2023-10-09T11:06:43.031985+0000","last_became_peered":"2023-10-09T11:06:43.031985+0000","last_unstale":"2023-10-09T11:07:58.245553+0000","last_undegraded":"2023-10-09T11:07:58.245553+0000","last_fullsized":"2023-10-09T11:07:58.245553+0000","mapping_epoch":531,"log_start":"503'1687","ondisk_log_start":"503'1687","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6963,"log_dups_size":0,"ondisk_log_size":6963,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46071808,"num_objects":703,"num_object_clones":0,"num_object_copies":2109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5480,"num_write_kb":197856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":579,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:54.711008+0000","last_change":"2023-10-09T11:07:54.711008+0000","last_active":"2023-10-09T11:07:54.711008+0000","last_peered":"2023-10-09T11:07:54.711008+0000","last_clean":"2023-10-09T11:07:54.711008+0000","last_became_active":"2023-10-09T11:06:24.245076+0000","last_became_peered":"2023-10-09T11:06:24.245076+0000","last_unstale":"2023-10-09T11:07:54.711008+0000","last_undegraded":"2023-10-09T11:07:54.711008+0000","last_fullsized":"2023-10-09T11:07:54.711008+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":22,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975872+0000","last_change":"2023-10-09T11:07:33.269200+0000","last_active":"2023-10-09T11:07:48.975872+0000","last_peered":"2023-10-09T11:07:48.975872+0000","last_clean":"2023-10-09T11:07:48.975872+0000","last_became_active":"2023-10-09T11:07:33.236205+0000","last_became_peered":"2023-10-09T11:07:33.236205+0000","last_unstale":"2023-10-09T11:07:48.975872+0000","last_undegraded":"2023-10-09T11:07:48.975872+0000","last_fullsized":"2023-10-09T11:07:48.975872+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:24.574734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":100,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.963749+0000","last_change":"2023-10-09T11:07:33.227803+0000","last_active":"2023-10-09T11:07:48.963749+0000","last_peered":"2023-10-09T11:07:48.963749+0000","last_clean":"2023-10-09T11:07:48.963749+0000","last_became_active":"2023-10-09T11:07:33.227486+0000","last_became_peered":"2023-10-09T11:07:33.227486+0000","last_unstale":"2023-10-09T11:07:48.963749+0000","last_undegraded":"2023-10-09T11:07:48.963749+0000","last_fullsized":"2023-10-09T11:07:48.963749+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:39.274875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1a","version":"556'8691","reported_seq":11551,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.264868+0000","last_change":"2023-10-09T11:06:43.001534+0000","last_active":"2023-10-09T11:07:58.264868+0000","last_peered":"2023-10-09T11:07:58.264868+0000","last_clean":"2023-10-09T11:07:58.264868+0000","last_became_active":"2023-10-09T11:06:42.998860+0000","last_became_peered":"2023-10-09T11:06:42.998860+0000","last_unstale":"2023-10-09T11:07:58.264868+0000","last_undegraded":"2023-10-09T11:07:58.264868+0000","last_fullsized":"2023-10-09T11:07:58.264868+0000","mapping_epoch":531,"log_start":"503'1724","ondisk_log_start":"503'1724","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6967,"log_dups_size":0,"ondisk_log_size":6967,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51314688,"num_objects":783,"num_object_clones":0,"num_object_copies":2349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5481,"num_write_kb":200448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":691,"num_bytes_recovered":45285376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":867,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975102+0000","last_change":"2023-10-09T11:05:23.323856+0000","last_active":"2023-10-09T11:07:48.975102+0000","last_peered":"2023-10-09T11:07:48.975102+0000","last_clean":"2023-10-09T11:07:48.975102+0000","last_became_active":"2023-10-09T11:05:23.322763+0000","last_became_peered":"2023-10-09T11:05:23.322763+0000","last_unstale":"2023-10-09T11:07:48.975102+0000","last_undegraded":"2023-10-09T11:07:48.975102+0000","last_fullsized":"2023-10-09T11:07:48.975102+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.760820+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":16,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975929+0000","last_change":"2023-10-09T11:07:33.233510+0000","last_active":"2023-10-09T11:07:48.975929+0000","last_peered":"2023-10-09T11:07:48.975929+0000","last_clean":"2023-10-09T11:07:48.975929+0000","last_became_active":"2023-10-09T11:07:33.232339+0000","last_became_peered":"2023-10-09T11:07:33.232339+0000","last_unstale":"2023-10-09T11:07:48.975929+0000","last_undegraded":"2023-10-09T11:07:48.975929+0000","last_fullsized":"2023-10-09T11:07:48.975929+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:55.324912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":497,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975081+0000","last_change":"2023-10-09T11:07:33.226594+0000","last_active":"2023-10-09T11:07:48.975081+0000","last_peered":"2023-10-09T11:07:48.975081+0000","last_clean":"2023-10-09T11:07:48.975081+0000","last_became_active":"2023-10-09T11:07:33.226275+0000","last_became_peered":"2023-10-09T11:07:33.226275+0000","last_unstale":"2023-10-09T11:07:48.975081+0000","last_undegraded":"2023-10-09T11:07:48.975081+0000","last_fullsized":"2023-10-09T11:07:48.975081+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:41.180088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"556'7045","reported_seq":8499,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.270167+0000","last_change":"2023-10-09T11:06:05.166134+0000","last_active":"2023-10-09T11:07:58.270167+0000","last_peered":"2023-10-09T11:07:58.270167+0000","last_clean":"2023-10-09T11:07:58.270167+0000","last_became_active":"2023-10-09T11:05:45.388425+0000","last_became_peered":"2023-10-09T11:05:45.388425+0000","last_unstale":"2023-10-09T11:07:58.270167+0000","last_undegraded":"2023-10-09T11:07:58.270167+0000","last_fullsized":"2023-10-09T11:07:58.270167+0000","mapping_epoch":509,"log_start":"502'1474","ondisk_log_start":"502'1474","created":509,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5571,"log_dups_size":0,"ondisk_log_size":5571,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52396032,"num_objects":800,"num_object_clones":0,"num_object_copies":2400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5482,"num_write_kb":200992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":684,"num_bytes_recovered":44826624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":889,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.984746+0000","last_change":"2023-10-09T11:04:53.274236+0000","last_active":"2023-10-09T11:07:48.984746+0000","last_peered":"2023-10-09T11:07:48.984746+0000","last_clean":"2023-10-09T11:07:48.984746+0000","last_became_active":"2023-10-09T11:04:53.273817+0000","last_became_peered":"2023-10-09T11:04:53.273817+0000","last_unstale":"2023-10-09T11:07:48.984746+0000","last_undegraded":"2023-10-09T11:07:48.984746+0000","last_fullsized":"2023-10-09T11:07:48.984746+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:05.096905+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":151,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975775+0000","last_change":"2023-10-09T11:05:20.312129+0000","last_active":"2023-10-09T11:07:48.975775+0000","last_peered":"2023-10-09T11:07:48.975775+0000","last_clean":"2023-10-09T11:07:48.975775+0000","last_became_active":"2023-10-09T11:05:20.311810+0000","last_became_peered":"2023-10-09T11:05:20.311810+0000","last_unstale":"2023-10-09T11:07:48.975775+0000","last_undegraded":"2023-10-09T11:07:48.975775+0000","last_fullsized":"2023-10-09T11:07:48.975775+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:48.813391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":249,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.963399+0000","last_change":"2023-10-09T11:07:33.218244+0000","last_active":"2023-10-09T11:07:48.963399+0000","last_peered":"2023-10-09T11:07:48.963399+0000","last_clean":"2023-10-09T11:07:48.963399+0000","last_became_active":"2023-10-09T11:07:33.217918+0000","last_became_peered":"2023-10-09T11:07:33.217918+0000","last_unstale":"2023-10-09T11:07:48.963399+0000","last_undegraded":"2023-10-09T11:07:48.963399+0000","last_fullsized":"2023-10-09T11:07:48.963399+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:16.150105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"556'6934","reported_seq":9485,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.282930+0000","last_change":"2023-10-09T11:06:07.154110+0000","last_active":"2023-10-09T11:07:58.282930+0000","last_peered":"2023-10-09T11:07:58.282930+0000","last_clean":"2023-10-09T11:07:58.282930+0000","last_became_active":"2023-10-09T11:05:45.388306+0000","last_became_peered":"2023-10-09T11:05:45.388306+0000","last_unstale":"2023-10-09T11:07:58.282930+0000","last_undegraded":"2023-10-09T11:07:58.282930+0000","last_fullsized":"2023-10-09T11:07:58.282930+0000","mapping_epoch":509,"log_start":"501'1306","ondisk_log_start":"501'1306","created":509,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5628,"log_dups_size":0,"ondisk_log_size":5628,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51970048,"num_objects":793,"num_object_clones":0,"num_object_copies":2379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5399,"num_write_kb":198144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":645,"num_bytes_recovered":42237952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":864,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976352+0000","last_change":"2023-10-09T11:05:23.324537+0000","last_active":"2023-10-09T11:07:48.976352+0000","last_peered":"2023-10-09T11:07:48.976352+0000","last_clean":"2023-10-09T11:07:48.976352+0000","last_became_active":"2023-10-09T11:05:23.324187+0000","last_became_peered":"2023-10-09T11:05:23.324187+0000","last_unstale":"2023-10-09T11:07:48.976352+0000","last_undegraded":"2023-10-09T11:07:48.976352+0000","last_fullsized":"2023-10-09T11:07:48.976352+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:25.286106+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":76,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.979079+0000","last_change":"2023-10-09T11:05:20.323565+0000","last_active":"2023-10-09T11:07:48.979079+0000","last_peered":"2023-10-09T11:07:48.979079+0000","last_clean":"2023-10-09T11:07:48.979079+0000","last_became_active":"2023-10-09T11:05:20.309672+0000","last_became_peered":"2023-10-09T11:05:20.309672+0000","last_unstale":"2023-10-09T11:07:48.979079+0000","last_undegraded":"2023-10-09T11:07:48.979079+0000","last_fullsized":"2023-10-09T11:07:48.979079+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:50.211465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":100,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.963340+0000","last_change":"2023-10-09T11:07:33.228001+0000","last_active":"2023-10-09T11:07:48.963340+0000","last_peered":"2023-10-09T11:07:48.963340+0000","last_clean":"2023-10-09T11:07:48.963340+0000","last_became_active":"2023-10-09T11:07:33.227691+0000","last_became_peered":"2023-10-09T11:07:33.227691+0000","last_unstale":"2023-10-09T11:07:48.963340+0000","last_undegraded":"2023-10-09T11:07:48.963340+0000","last_fullsized":"2023-10-09T11:07:48.963340+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:24:16.098138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"556'6884","reported_seq":6965,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:56.699343+0000","last_change":"2023-10-09T11:07:52.704312+0000","last_active":"2023-10-09T11:07:56.699343+0000","last_peered":"2023-10-09T11:07:56.699343+0000","last_clean":"2023-10-09T11:07:56.699343+0000","last_became_active":"2023-10-09T11:05:45.338519+0000","last_became_peered":"2023-10-09T11:05:45.338519+0000","last_unstale":"2023-10-09T11:07:56.699343+0000","last_undegraded":"2023-10-09T11:07:56.699343+0000","last_fullsized":"2023-10-09T11:07:56.699343+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6884,"log_dups_size":0,"ondisk_log_size":6884,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51806208,"num_objects":790,"num_object_clones":0,"num_object_copies":2370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5314,"num_write_kb":195360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":788,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976513+0000","last_change":"2023-10-09T11:07:36.284927+0000","last_active":"2023-10-09T11:07:48.976513+0000","last_peered":"2023-10-09T11:07:48.976513+0000","last_clean":"2023-10-09T11:07:48.976513+0000","last_became_active":"2023-10-09T11:07:36.284631+0000","last_became_peered":"2023-10-09T11:07:36.284631+0000","last_unstale":"2023-10-09T11:07:48.976513+0000","last_undegraded":"2023-10-09T11:07:48.976513+0000","last_fullsized":"2023-10-09T11:07:48.976513+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:37.220243+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":20,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.978057+0000","last_change":"2023-10-09T11:07:33.273503+0000","last_active":"2023-10-09T11:07:48.978057+0000","last_peered":"2023-10-09T11:07:48.978057+0000","last_clean":"2023-10-09T11:07:48.978057+0000","last_became_active":"2023-10-09T11:07:33.242980+0000","last_became_peered":"2023-10-09T11:07:33.242980+0000","last_unstale":"2023-10-09T11:07:48.978057+0000","last_undegraded":"2023-10-09T11:07:48.978057+0000","last_fullsized":"2023-10-09T11:07:48.978057+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:18.934972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":518,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.974807+0000","last_change":"2023-10-09T11:07:33.304363+0000","last_active":"2023-10-09T11:07:48.974807+0000","last_peered":"2023-10-09T11:07:48.974807+0000","last_clean":"2023-10-09T11:07:48.974807+0000","last_became_active":"2023-10-09T11:07:33.304024+0000","last_became_peered":"2023-10-09T11:07:33.304024+0000","last_unstale":"2023-10-09T11:07:48.974807+0000","last_undegraded":"2023-10-09T11:07:48.974807+0000","last_fullsized":"2023-10-09T11:07:48.974807+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:21:51.522688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"556'6819","reported_seq":6917,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:56.243626+0000","last_change":"2023-10-09T11:07:28.705631+0000","last_active":"2023-10-09T11:07:56.243626+0000","last_peered":"2023-10-09T11:07:56.243626+0000","last_clean":"2023-10-09T11:07:56.243626+0000","last_became_active":"2023-10-09T11:06:24.254388+0000","last_became_peered":"2023-10-09T11:06:24.254388+0000","last_unstale":"2023-10-09T11:07:56.243626+0000","last_undegraded":"2023-10-09T11:07:56.243626+0000","last_fullsized":"2023-10-09T11:07:56.243626+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6819,"log_dups_size":0,"ondisk_log_size":6819,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56164352,"num_objects":857,"num_object_clones":0,"num_object_copies":2571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5293,"num_write_kb":196800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":937,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.963827+0000","last_change":"2023-10-09T11:05:23.316129+0000","last_active":"2023-10-09T11:07:48.963827+0000","last_peered":"2023-10-09T11:07:48.963827+0000","last_clean":"2023-10-09T11:07:48.963827+0000","last_became_active":"2023-10-09T11:05:23.315792+0000","last_became_peered":"2023-10-09T11:05:23.315792+0000","last_unstale":"2023-10-09T11:07:48.963827+0000","last_undegraded":"2023-10-09T11:07:48.963827+0000","last_fullsized":"2023-10-09T11:07:48.963827+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:51.770547+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":20,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976724+0000","last_change":"2023-10-09T11:07:33.256691+0000","last_active":"2023-10-09T11:07:48.976724+0000","last_peered":"2023-10-09T11:07:48.976724+0000","last_clean":"2023-10-09T11:07:48.976724+0000","last_became_active":"2023-10-09T11:07:33.229554+0000","last_became_peered":"2023-10-09T11:07:33.229554+0000","last_unstale":"2023-10-09T11:07:48.976724+0000","last_undegraded":"2023-10-09T11:07:48.976724+0000","last_fullsized":"2023-10-09T11:07:48.976724+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:57.414795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":270,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.986029+0000","last_change":"2023-10-09T11:04:50.333538+0000","last_active":"2023-10-09T11:07:48.986029+0000","last_peered":"2023-10-09T11:07:48.986029+0000","last_clean":"2023-10-09T11:07:48.986029+0000","last_became_active":"2023-10-09T11:04:50.332514+0000","last_became_peered":"2023-10-09T11:04:50.332514+0000","last_unstale":"2023-10-09T11:07:48.986029+0000","last_undegraded":"2023-10-09T11:07:48.986029+0000","last_fullsized":"2023-10-09T11:07:48.986029+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:22.933839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"556'6768","reported_seq":11377,"reported_epoch":556,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:07:58.254543+0000","last_change":"2023-10-09T11:07:34.338632+0000","last_active":"2023-10-09T11:07:58.254543+0000","last_peered":"2023-10-09T11:07:58.254543+0000","last_clean":"2023-10-09T11:07:32.132824+0000","last_became_active":"2023-10-09T11:07:34.322525+0000","last_became_peered":"2023-10-09T11:07:34.322525+0000","last_unstale":"2023-10-09T11:07:58.254543+0000","last_undegraded":"2023-10-09T11:07:58.254543+0000","last_fullsized":"2023-10-09T11:07:58.254543+0000","mapping_epoch":550,"log_start":"498'949","ondisk_log_start":"498'949","created":495,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5819,"log_dups_size":0,"ondisk_log_size":5819,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56000512,"num_objects":854,"num_object_clones":0,"num_object_copies":2562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":854,"num_objects_unfound":0,"num_objects_dirty":854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5250,"num_write_kb":195360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2245,"num_bytes_recovered":147128320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":854}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":812,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977685+0000","last_change":"2023-10-09T11:05:23.322251+0000","last_active":"2023-10-09T11:07:48.977685+0000","last_peered":"2023-10-09T11:07:48.977685+0000","last_clean":"2023-10-09T11:07:48.977685+0000","last_became_active":"2023-10-09T11:05:23.321826+0000","last_became_peered":"2023-10-09T11:05:23.321826+0000","last_unstale":"2023-10-09T11:07:48.977685+0000","last_undegraded":"2023-10-09T11:07:48.977685+0000","last_fullsized":"2023-10-09T11:07:48.977685+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.858013+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":20,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976732+0000","last_change":"2023-10-09T11:07:33.271238+0000","last_active":"2023-10-09T11:07:48.976732+0000","last_peered":"2023-10-09T11:07:48.976732+0000","last_clean":"2023-10-09T11:07:48.976732+0000","last_became_active":"2023-10-09T11:07:33.241886+0000","last_became_peered":"2023-10-09T11:07:33.241886+0000","last_unstale":"2023-10-09T11:07:48.976732+0000","last_undegraded":"2023-10-09T11:07:48.976732+0000","last_fullsized":"2023-10-09T11:07:48.976732+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:48.882589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":86,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.979048+0000","last_change":"2023-10-09T11:07:33.244131+0000","last_active":"2023-10-09T11:07:48.979048+0000","last_peered":"2023-10-09T11:07:48.979048+0000","last_clean":"2023-10-09T11:07:48.979048+0000","last_became_active":"2023-10-09T11:07:33.243797+0000","last_became_peered":"2023-10-09T11:07:33.243797+0000","last_unstale":"2023-10-09T11:07:48.979048+0000","last_undegraded":"2023-10-09T11:07:48.979048+0000","last_fullsized":"2023-10-09T11:07:48.979048+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:44.851043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"556'6878","reported_seq":8345,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:56.530492+0000","last_change":"2023-10-09T11:07:29.087418+0000","last_active":"2023-10-09T11:07:56.530492+0000","last_peered":"2023-10-09T11:07:56.530492+0000","last_clean":"2023-10-09T11:07:56.530492+0000","last_became_active":"2023-10-09T11:05:46.357677+0000","last_became_peered":"2023-10-09T11:05:46.357677+0000","last_unstale":"2023-10-09T11:07:56.530492+0000","last_undegraded":"2023-10-09T11:07:56.530492+0000","last_fullsized":"2023-10-09T11:07:56.530492+0000","mapping_epoch":510,"log_start":"498'131","ondisk_log_start":"498'131","created":495,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6747,"log_dups_size":0,"ondisk_log_size":6747,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55508992,"num_objects":847,"num_object_clones":0,"num_object_copies":2541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5325,"num_write_kb":197504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":677,"num_bytes_recovered":44367872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":1066,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976855+0000","last_change":"2023-10-09T11:07:36.286417+0000","last_active":"2023-10-09T11:07:48.976855+0000","last_peered":"2023-10-09T11:07:48.976855+0000","last_clean":"2023-10-09T11:07:48.976855+0000","last_became_active":"2023-10-09T11:07:36.286061+0000","last_became_peered":"2023-10-09T11:07:36.286061+0000","last_unstale":"2023-10-09T11:07:48.976855+0000","last_undegraded":"2023-10-09T11:07:48.976855+0000","last_fullsized":"2023-10-09T11:07:48.976855+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:00.289500+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":414,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975662+0000","last_change":"2023-10-09T11:05:20.309385+0000","last_active":"2023-10-09T11:07:48.975662+0000","last_peered":"2023-10-09T11:07:48.975662+0000","last_clean":"2023-10-09T11:07:48.975662+0000","last_became_active":"2023-10-09T11:05:20.309034+0000","last_became_peered":"2023-10-09T11:05:20.309034+0000","last_unstale":"2023-10-09T11:07:48.975662+0000","last_undegraded":"2023-10-09T11:07:48.975662+0000","last_fullsized":"2023-10-09T11:07:48.975662+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:31.857485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":345,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.984546+0000","last_change":"2023-10-09T11:06:24.245966+0000","last_active":"2023-10-09T11:07:48.984546+0000","last_peered":"2023-10-09T11:07:48.984546+0000","last_clean":"2023-10-09T11:07:48.984546+0000","last_became_active":"2023-10-09T11:06:24.245310+0000","last_became_peered":"2023-10-09T11:06:24.245310+0000","last_unstale":"2023-10-09T11:07:48.984546+0000","last_undegraded":"2023-10-09T11:07:48.984546+0000","last_fullsized":"2023-10-09T11:07:48.984546+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:23.966200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.3","version":"556'6006","reported_seq":9045,"reported_epoch":556,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:07:58.288820+0000","last_change":"2023-10-09T11:07:34.314819+0000","last_active":"2023-10-09T11:07:58.288820+0000","last_peered":"2023-10-09T11:07:58.288820+0000","last_clean":"2023-10-09T11:07:32.185795+0000","last_became_active":"2023-10-09T11:07:34.308357+0000","last_became_peered":"2023-10-09T11:07:34.308357+0000","last_unstale":"2023-10-09T11:07:58.288820+0000","last_undegraded":"2023-10-09T11:07:58.288820+0000","last_fullsized":"2023-10-09T11:07:58.288820+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6006,"log_dups_size":0,"ondisk_log_size":6006,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24920064,"num_objects":380,"num_object_clones":0,"num_object_copies":1140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":380,"num_objects_unfound":0,"num_objects_dirty":380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2790,"num_write_kb":101456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1223,"num_bytes_recovered":72613888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":380}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":710,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975631+0000","last_change":"2023-10-09T11:07:36.283174+0000","last_active":"2023-10-09T11:07:48.975631+0000","last_peered":"2023-10-09T11:07:48.975631+0000","last_clean":"2023-10-09T11:07:48.975631+0000","last_became_active":"2023-10-09T11:07:36.278007+0000","last_became_peered":"2023-10-09T11:07:36.278007+0000","last_unstale":"2023-10-09T11:07:48.975631+0000","last_undegraded":"2023-10-09T11:07:48.975631+0000","last_fullsized":"2023-10-09T11:07:48.975631+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:51.693969+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":86,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.963771+0000","last_change":"2023-10-09T11:04:50.529513+0000","last_active":"2023-10-09T11:07:48.963771+0000","last_peered":"2023-10-09T11:07:48.963771+0000","last_clean":"2023-10-09T11:07:48.963771+0000","last_became_active":"2023-10-09T11:04:50.339671+0000","last_became_peered":"2023-10-09T11:04:50.339671+0000","last_unstale":"2023-10-09T11:07:48.963771+0000","last_undegraded":"2023-10-09T11:07:48.963771+0000","last_fullsized":"2023-10-09T11:07:48.963771+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:24.473000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":91,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.963684+0000","last_change":"2023-10-09T11:05:20.313855+0000","last_active":"2023-10-09T11:07:48.963684+0000","last_peered":"2023-10-09T11:07:48.963684+0000","last_clean":"2023-10-09T11:07:48.963684+0000","last_became_active":"2023-10-09T11:05:20.313535+0000","last_became_peered":"2023-10-09T11:05:20.313535+0000","last_unstale":"2023-10-09T11:07:48.963684+0000","last_undegraded":"2023-10-09T11:07:48.963684+0000","last_fullsized":"2023-10-09T11:07:48.963684+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:25.266434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"556'5783","reported_seq":5873,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.210296+0000","last_change":"2023-10-09T11:06:42.991743+0000","last_active":"2023-10-09T11:07:58.210296+0000","last_peered":"2023-10-09T11:07:58.210296+0000","last_clean":"2023-10-09T11:07:58.210296+0000","last_became_active":"2023-10-09T11:06:42.989452+0000","last_became_peered":"2023-10-09T11:06:42.989452+0000","last_unstale":"2023-10-09T11:07:58.210296+0000","last_undegraded":"2023-10-09T11:07:58.210296+0000","last_fullsized":"2023-10-09T11:07:58.210296+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5783,"log_dups_size":0,"ondisk_log_size":5783,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24576000,"num_objects":375,"num_object_clones":0,"num_object_copies":1125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2680,"num_write_kb":97760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":814,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975017+0000","last_change":"2023-10-09T11:04:53.272323+0000","last_active":"2023-10-09T11:07:48.975017+0000","last_peered":"2023-10-09T11:07:48.975017+0000","last_clean":"2023-10-09T11:07:48.975017+0000","last_became_active":"2023-10-09T11:04:53.272010+0000","last_became_peered":"2023-10-09T11:04:53.272010+0000","last_unstale":"2023-10-09T11:07:48.975017+0000","last_undegraded":"2023-10-09T11:07:48.975017+0000","last_fullsized":"2023-10-09T11:07:48.975017+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:23.913605+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":579,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.985867+0000","last_change":"2023-10-09T11:05:20.307499+0000","last_active":"2023-10-09T11:07:48.985867+0000","last_peered":"2023-10-09T11:07:48.985867+0000","last_clean":"2023-10-09T11:07:48.985867+0000","last_became_active":"2023-10-09T11:05:20.307198+0000","last_became_peered":"2023-10-09T11:05:20.307198+0000","last_unstale":"2023-10-09T11:07:48.985867+0000","last_undegraded":"2023-10-09T11:07:48.985867+0000","last_fullsized":"2023-10-09T11:07:48.985867+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:53.726844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":315,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.055200+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:07:51.055200+0000","last_peered":"2023-10-09T11:07:51.055200+0000","last_clean":"2023-10-09T11:07:51.055200+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:07:51.055200+0000","last_undegraded":"2023-10-09T11:07:51.055200+0000","last_fullsized":"2023-10-09T11:07:51.055200+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"556'5993","reported_seq":6832,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.179992+0000","last_change":"2023-10-09T11:07:20.806034+0000","last_active":"2023-10-09T11:07:58.179992+0000","last_peered":"2023-10-09T11:07:58.179992+0000","last_clean":"2023-10-09T11:07:58.179992+0000","last_became_active":"2023-10-09T11:06:42.992656+0000","last_became_peered":"2023-10-09T11:06:42.992656+0000","last_unstale":"2023-10-09T11:07:58.179992+0000","last_undegraded":"2023-10-09T11:07:58.179992+0000","last_fullsized":"2023-10-09T11:07:58.179992+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5993,"log_dups_size":0,"ondisk_log_size":5993,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25411584,"num_objects":387,"num_object_clones":0,"num_object_copies":1161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2759,"num_write_kb":100720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":328,"num_bytes_recovered":21528576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":906,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.984631+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:07:48.984631+0000","last_peered":"2023-10-09T11:07:48.984631+0000","last_clean":"2023-10-09T11:07:48.984631+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:07:48.984631+0000","last_undegraded":"2023-10-09T11:07:48.984631+0000","last_fullsized":"2023-10-09T11:07:48.984631+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":578,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977268+0000","last_change":"2023-10-09T11:04:57.953307+0000","last_active":"2023-10-09T11:07:48.977268+0000","last_peered":"2023-10-09T11:07:48.977268+0000","last_clean":"2023-10-09T11:07:48.977268+0000","last_became_active":"2023-10-09T11:04:57.953095+0000","last_became_peered":"2023-10-09T11:04:57.953095+0000","last_unstale":"2023-10-09T11:07:48.977268+0000","last_undegraded":"2023-10-09T11:07:48.977268+0000","last_fullsized":"2023-10-09T11:07:48.977268+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:05.562643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":264,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.986816+0000","last_change":"2023-10-09T11:05:20.310255+0000","last_active":"2023-10-09T11:07:48.986816+0000","last_peered":"2023-10-09T11:07:48.986816+0000","last_clean":"2023-10-09T11:07:48.986816+0000","last_became_active":"2023-10-09T11:05:20.309109+0000","last_became_peered":"2023-10-09T11:05:20.309109+0000","last_unstale":"2023-10-09T11:07:48.986816+0000","last_undegraded":"2023-10-09T11:07:48.986816+0000","last_fullsized":"2023-10-09T11:07:48.986816+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:07.429039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.0","version":"556'5852","reported_seq":8254,"reported_epoch":556,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:07:58.218697+0000","last_change":"2023-10-09T11:07:34.329311+0000","last_active":"2023-10-09T11:07:58.218697+0000","last_peered":"2023-10-09T11:07:58.218697+0000","last_clean":"2023-10-09T11:07:32.155021+0000","last_became_active":"2023-10-09T11:07:34.321010+0000","last_became_peered":"2023-10-09T11:07:34.321010+0000","last_unstale":"2023-10-09T11:07:58.218697+0000","last_undegraded":"2023-10-09T11:07:34.238729+0000","last_fullsized":"2023-10-09T11:07:34.219306+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5852,"log_dups_size":0,"ondisk_log_size":5852,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28721152,"num_objects":438,"num_object_clones":0,"num_object_copies":1314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1515,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2686,"num_write_kb":99984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1147,"num_bytes_recovered":72155136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,3","objects":-307},{"shards":"1,3,6","objects":388}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":798,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976280+0000","last_change":"2023-10-09T11:05:23.329329+0000","last_active":"2023-10-09T11:07:48.976280+0000","last_peered":"2023-10-09T11:07:48.976280+0000","last_clean":"2023-10-09T11:07:48.976280+0000","last_became_active":"2023-10-09T11:05:23.328976+0000","last_became_peered":"2023-10-09T11:05:23.328976+0000","last_unstale":"2023-10-09T11:07:48.976280+0000","last_undegraded":"2023-10-09T11:07:48.976280+0000","last_fullsized":"2023-10-09T11:07:48.976280+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:41.848184+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":100,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.960048+0000","last_change":"2023-10-09T11:07:33.236133+0000","last_active":"2023-10-09T11:07:48.960048+0000","last_peered":"2023-10-09T11:07:48.960048+0000","last_clean":"2023-10-09T11:07:48.960048+0000","last_became_active":"2023-10-09T11:07:33.235818+0000","last_became_peered":"2023-10-09T11:07:33.235818+0000","last_unstale":"2023-10-09T11:07:48.960048+0000","last_undegraded":"2023-10-09T11:07:48.960048+0000","last_fullsized":"2023-10-09T11:07:48.960048+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:20.627846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":261,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.987919+0000","last_change":"2023-10-09T11:04:50.332978+0000","last_active":"2023-10-09T11:07:48.987919+0000","last_peered":"2023-10-09T11:07:48.987919+0000","last_clean":"2023-10-09T11:07:48.987919+0000","last_became_active":"2023-10-09T11:04:50.332735+0000","last_became_peered":"2023-10-09T11:04:50.332735+0000","last_unstale":"2023-10-09T11:07:48.987919+0000","last_undegraded":"2023-10-09T11:07:48.987919+0000","last_fullsized":"2023-10-09T11:07:48.987919+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:39:44.659751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"556'8655","reported_seq":11577,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:56.230874+0000","last_change":"2023-10-09T11:07:21.753744+0000","last_active":"2023-10-09T11:07:56.230874+0000","last_peered":"2023-10-09T11:07:56.230874+0000","last_clean":"2023-10-09T11:07:56.230874+0000","last_became_active":"2023-10-09T11:06:42.979233+0000","last_became_peered":"2023-10-09T11:06:42.979233+0000","last_unstale":"2023-10-09T11:07:56.230874+0000","last_undegraded":"2023-10-09T11:07:56.230874+0000","last_fullsized":"2023-10-09T11:07:56.230874+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":8655,"log_dups_size":0,"ondisk_log_size":8655,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55803916,"num_objects":852,"num_object_clones":0,"num_object_copies":2556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5434,"num_write_kb":201120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":705,"num_bytes_recovered":46137344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":1082,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.963431+0000","last_change":"2023-10-09T11:07:36.291716+0000","last_active":"2023-10-09T11:07:48.963431+0000","last_peered":"2023-10-09T11:07:48.963431+0000","last_clean":"2023-10-09T11:07:48.963431+0000","last_became_active":"2023-10-09T11:07:36.291424+0000","last_became_peered":"2023-10-09T11:07:36.291424+0000","last_unstale":"2023-10-09T11:07:48.963431+0000","last_undegraded":"2023-10-09T11:07:48.963431+0000","last_fullsized":"2023-10-09T11:07:48.963431+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:22.316387+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":285,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.987572+0000","last_change":"2023-10-09T11:07:33.226547+0000","last_active":"2023-10-09T11:07:48.987572+0000","last_peered":"2023-10-09T11:07:48.987572+0000","last_clean":"2023-10-09T11:07:48.987572+0000","last_became_active":"2023-10-09T11:07:33.226176+0000","last_became_peered":"2023-10-09T11:07:33.226176+0000","last_unstale":"2023-10-09T11:07:48.987572+0000","last_undegraded":"2023-10-09T11:07:48.987572+0000","last_fullsized":"2023-10-09T11:07:48.987572+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:16.888732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":431,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.974950+0000","last_change":"2023-10-09T11:04:50.341353+0000","last_active":"2023-10-09T11:07:48.974950+0000","last_peered":"2023-10-09T11:07:48.974950+0000","last_clean":"2023-10-09T11:07:48.974950+0000","last_became_active":"2023-10-09T11:04:50.340801+0000","last_became_peered":"2023-10-09T11:04:50.340801+0000","last_unstale":"2023-10-09T11:07:48.974950+0000","last_undegraded":"2023-10-09T11:07:48.974950+0000","last_fullsized":"2023-10-09T11:07:48.974950+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:00.699069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":237,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.985946+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:07:48.985946+0000","last_peered":"2023-10-09T11:07:48.985946+0000","last_clean":"2023-10-09T11:07:48.985946+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:07:48.985946+0000","last_undegraded":"2023-10-09T11:07:48.985946+0000","last_fullsized":"2023-10-09T11:07:48.985946+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"556'8550","reported_seq":8631,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.241070+0000","last_change":"2023-10-09T11:06:43.027139+0000","last_active":"2023-10-09T11:07:58.241070+0000","last_peered":"2023-10-09T11:07:58.241070+0000","last_clean":"2023-10-09T11:07:58.241070+0000","last_became_active":"2023-10-09T11:06:43.024164+0000","last_became_peered":"2023-10-09T11:06:43.024164+0000","last_unstale":"2023-10-09T11:07:58.241070+0000","last_undegraded":"2023-10-09T11:07:58.241070+0000","last_fullsized":"2023-10-09T11:07:58.241070+0000","mapping_epoch":495,"log_start":"503'1687","ondisk_log_start":"503'1687","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6863,"log_dups_size":0,"ondisk_log_size":6863,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52625408,"num_objects":803,"num_object_clones":0,"num_object_copies":2409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5380,"num_write_kb":197856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":756,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.974808+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:07:48.974808+0000","last_peered":"2023-10-09T11:07:48.974808+0000","last_clean":"2023-10-09T11:07:48.974808+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:07:48.974808+0000","last_undegraded":"2023-10-09T11:07:48.974808+0000","last_fullsized":"2023-10-09T11:07:48.974808+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":438,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976067+0000","last_change":"2023-10-09T11:05:20.316093+0000","last_active":"2023-10-09T11:07:48.976067+0000","last_peered":"2023-10-09T11:07:48.976067+0000","last_clean":"2023-10-09T11:07:48.976067+0000","last_became_active":"2023-10-09T11:05:20.315471+0000","last_became_peered":"2023-10-09T11:05:20.315471+0000","last_unstale":"2023-10-09T11:07:48.976067+0000","last_undegraded":"2023-10-09T11:07:48.976067+0000","last_fullsized":"2023-10-09T11:07:48.976067+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:47.548763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":462,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.978532+0000","last_change":"2023-10-09T11:06:24.245282+0000","last_active":"2023-10-09T11:07:48.978532+0000","last_peered":"2023-10-09T11:07:48.978532+0000","last_clean":"2023-10-09T11:07:48.978532+0000","last_became_active":"2023-10-09T11:06:24.244981+0000","last_became_peered":"2023-10-09T11:06:24.244981+0000","last_unstale":"2023-10-09T11:07:48.978532+0000","last_undegraded":"2023-10-09T11:07:48.978532+0000","last_fullsized":"2023-10-09T11:07:48.978532+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:15.327655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":270,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.987297+0000","last_change":"2023-10-09T11:04:50.311579+0000","last_active":"2023-10-09T11:07:48.987297+0000","last_peered":"2023-10-09T11:07:48.987297+0000","last_clean":"2023-10-09T11:07:48.987297+0000","last_became_active":"2023-10-09T11:04:50.311270+0000","last_became_peered":"2023-10-09T11:04:50.311270+0000","last_unstale":"2023-10-09T11:07:48.987297+0000","last_undegraded":"2023-10-09T11:07:48.987297+0000","last_fullsized":"2023-10-09T11:07:48.987297+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:49.850302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"556'6865","reported_seq":9438,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.193761+0000","last_change":"2023-10-09T11:06:09.940480+0000","last_active":"2023-10-09T11:07:58.193761+0000","last_peered":"2023-10-09T11:07:58.193761+0000","last_clean":"2023-10-09T11:07:58.193761+0000","last_became_active":"2023-10-09T11:05:46.348515+0000","last_became_peered":"2023-10-09T11:05:46.348515+0000","last_unstale":"2023-10-09T11:07:58.193761+0000","last_undegraded":"2023-10-09T11:07:58.193761+0000","last_fullsized":"2023-10-09T11:07:58.193761+0000","mapping_epoch":510,"log_start":"498'1090","ondisk_log_start":"498'1090","created":495,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5775,"log_dups_size":0,"ondisk_log_size":5775,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51380224,"num_objects":784,"num_object_clones":0,"num_object_copies":2352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5330,"num_write_kb":195648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":654,"num_bytes_recovered":42893312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":908,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976097+0000","last_change":"2023-10-09T11:07:36.284280+0000","last_active":"2023-10-09T11:07:48.976097+0000","last_peered":"2023-10-09T11:07:48.976097+0000","last_clean":"2023-10-09T11:07:48.976097+0000","last_became_active":"2023-10-09T11:07:36.283966+0000","last_became_peered":"2023-10-09T11:07:36.283966+0000","last_unstale":"2023-10-09T11:07:48.976097+0000","last_undegraded":"2023-10-09T11:07:48.976097+0000","last_fullsized":"2023-10-09T11:07:48.976097+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:54.128331+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":334,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977478+0000","last_change":"2023-10-09T11:04:50.381744+0000","last_active":"2023-10-09T11:07:48.977478+0000","last_peered":"2023-10-09T11:07:48.977478+0000","last_clean":"2023-10-09T11:07:48.977478+0000","last_became_active":"2023-10-09T11:04:50.381253+0000","last_became_peered":"2023-10-09T11:04:50.381253+0000","last_unstale":"2023-10-09T11:07:48.977478+0000","last_undegraded":"2023-10-09T11:07:48.977478+0000","last_fullsized":"2023-10-09T11:07:48.977478+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:24.030274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":255,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.987746+0000","last_change":"2023-10-09T11:05:20.298542+0000","last_active":"2023-10-09T11:07:48.987746+0000","last_peered":"2023-10-09T11:07:48.987746+0000","last_clean":"2023-10-09T11:07:48.987746+0000","last_became_active":"2023-10-09T11:05:20.298119+0000","last_became_peered":"2023-10-09T11:05:20.298119+0000","last_unstale":"2023-10-09T11:07:48.987746+0000","last_undegraded":"2023-10-09T11:07:48.987746+0000","last_fullsized":"2023-10-09T11:07:48.987746+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:00.827214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":235,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.996648+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:07:48.996648+0000","last_peered":"2023-10-09T11:07:48.996648+0000","last_clean":"2023-10-09T11:07:48.996648+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:07:48.996648+0000","last_undegraded":"2023-10-09T11:07:48.996648+0000","last_fullsized":"2023-10-09T11:07:48.996648+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"556'8615","reported_seq":11484,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.230846+0000","last_change":"2023-10-09T11:06:54.755841+0000","last_active":"2023-10-09T11:07:58.230846+0000","last_peered":"2023-10-09T11:07:58.230846+0000","last_clean":"2023-10-09T11:07:58.230846+0000","last_became_active":"2023-10-09T11:06:42.985753+0000","last_became_peered":"2023-10-09T11:06:42.985753+0000","last_unstale":"2023-10-09T11:07:58.230846+0000","last_undegraded":"2023-10-09T11:07:58.230846+0000","last_fullsized":"2023-10-09T11:07:58.230846+0000","mapping_epoch":513,"log_start":"503'1724","ondisk_log_start":"503'1724","created":495,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6891,"log_dups_size":0,"ondisk_log_size":6891,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56295424,"num_objects":859,"num_object_clones":0,"num_object_copies":2577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5405,"num_write_kb":200448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":693,"num_bytes_recovered":45285376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":431,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975354+0000","last_change":"2023-10-09T11:07:33.227052+0000","last_active":"2023-10-09T11:07:48.975354+0000","last_peered":"2023-10-09T11:07:48.975354+0000","last_clean":"2023-10-09T11:07:48.975354+0000","last_became_active":"2023-10-09T11:07:33.226289+0000","last_became_peered":"2023-10-09T11:07:33.226289+0000","last_unstale":"2023-10-09T11:07:48.975354+0000","last_undegraded":"2023-10-09T11:07:48.975354+0000","last_fullsized":"2023-10-09T11:07:48.975354+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:15.605946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":853,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975380+0000","last_change":"2023-10-09T11:07:36.290700+0000","last_active":"2023-10-09T11:07:48.975380+0000","last_peered":"2023-10-09T11:07:48.975380+0000","last_clean":"2023-10-09T11:07:48.975380+0000","last_became_active":"2023-10-09T11:07:36.290357+0000","last_became_peered":"2023-10-09T11:07:36.290357+0000","last_unstale":"2023-10-09T11:07:48.975380+0000","last_undegraded":"2023-10-09T11:07:48.975380+0000","last_fullsized":"2023-10-09T11:07:48.975380+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:34.494777+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.963011+0000","last_change":"2023-10-09T11:05:20.310666+0000","last_active":"2023-10-09T11:07:48.963011+0000","last_peered":"2023-10-09T11:07:48.963011+0000","last_clean":"2023-10-09T11:07:48.963011+0000","last_became_active":"2023-10-09T11:05:20.310413+0000","last_became_peered":"2023-10-09T11:05:20.310413+0000","last_unstale":"2023-10-09T11:07:48.963011+0000","last_undegraded":"2023-10-09T11:07:48.963011+0000","last_fullsized":"2023-10-09T11:07:48.963011+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:34.169905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":516,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976655+0000","last_change":"2023-10-09T11:05:20.316523+0000","last_active":"2023-10-09T11:07:48.976655+0000","last_peered":"2023-10-09T11:07:48.976655+0000","last_clean":"2023-10-09T11:07:48.976655+0000","last_became_active":"2023-10-09T11:05:20.314493+0000","last_became_peered":"2023-10-09T11:05:20.314493+0000","last_unstale":"2023-10-09T11:07:48.976655+0000","last_undegraded":"2023-10-09T11:07:48.976655+0000","last_fullsized":"2023-10-09T11:07:48.976655+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:41.618701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"556'6902","reported_seq":8290,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.230461+0000","last_change":"2023-10-09T11:06:34.311475+0000","last_active":"2023-10-09T11:07:58.230461+0000","last_peered":"2023-10-09T11:07:58.230461+0000","last_clean":"2023-10-09T11:07:58.230461+0000","last_became_active":"2023-10-09T11:05:46.356286+0000","last_became_peered":"2023-10-09T11:05:46.356286+0000","last_unstale":"2023-10-09T11:07:58.230461+0000","last_undegraded":"2023-10-09T11:07:58.230461+0000","last_fullsized":"2023-10-09T11:07:58.230461+0000","mapping_epoch":510,"log_start":"498'1118","ondisk_log_start":"498'1118","created":495,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5784,"log_dups_size":0,"ondisk_log_size":5784,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54427648,"num_objects":830,"num_object_clones":0,"num_object_copies":2490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5338,"num_write_kb":197408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":650,"num_bytes_recovered":42598400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":926,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.984515+0000","last_change":"2023-10-09T11:05:23.322074+0000","last_active":"2023-10-09T11:07:48.984515+0000","last_peered":"2023-10-09T11:07:48.984515+0000","last_clean":"2023-10-09T11:07:48.984515+0000","last_became_active":"2023-10-09T11:05:23.321749+0000","last_became_peered":"2023-10-09T11:05:23.321749+0000","last_unstale":"2023-10-09T11:07:48.984515+0000","last_undegraded":"2023-10-09T11:07:48.984515+0000","last_fullsized":"2023-10-09T11:07:48.984515+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:07.010932+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":131,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975719+0000","last_change":"2023-10-09T11:02:33.198484+0000","last_active":"2023-10-09T11:07:48.975719+0000","last_peered":"2023-10-09T11:07:48.975719+0000","last_clean":"2023-10-09T11:07:48.975719+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:07:48.975719+0000","last_undegraded":"2023-10-09T11:07:48.975719+0000","last_fullsized":"2023-10-09T11:07:48.975719+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":182,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.960071+0000","last_change":"2023-10-09T11:06:24.248362+0000","last_active":"2023-10-09T11:07:48.960071+0000","last_peered":"2023-10-09T11:07:48.960071+0000","last_clean":"2023-10-09T11:07:48.960071+0000","last_became_active":"2023-10-09T11:06:24.247348+0000","last_became_peered":"2023-10-09T11:06:24.247348+0000","last_unstale":"2023-10-09T11:07:48.960071+0000","last_undegraded":"2023-10-09T11:07:48.960071+0000","last_fullsized":"2023-10-09T11:07:48.960071+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:26.728754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":381,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.986474+0000","last_change":"2023-10-09T11:06:24.253511+0000","last_active":"2023-10-09T11:07:48.986474+0000","last_peered":"2023-10-09T11:07:48.986474+0000","last_clean":"2023-10-09T11:07:48.986474+0000","last_became_active":"2023-10-09T11:06:24.252739+0000","last_became_peered":"2023-10-09T11:06:24.252739+0000","last_unstale":"2023-10-09T11:07:48.986474+0000","last_undegraded":"2023-10-09T11:07:48.986474+0000","last_fullsized":"2023-10-09T11:07:48.986474+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:50.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"556'8654","reported_seq":8735,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.232174+0000","last_change":"2023-10-09T11:06:42.983758+0000","last_active":"2023-10-09T11:07:58.232174+0000","last_peered":"2023-10-09T11:07:58.232174+0000","last_clean":"2023-10-09T11:07:58.232174+0000","last_became_active":"2023-10-09T11:06:42.981460+0000","last_became_peered":"2023-10-09T11:06:42.981460+0000","last_unstale":"2023-10-09T11:07:58.232174+0000","last_undegraded":"2023-10-09T11:07:58.232174+0000","last_fullsized":"2023-10-09T11:07:58.232174+0000","mapping_epoch":495,"log_start":"504'1919","ondisk_log_start":"504'1919","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6735,"log_dups_size":0,"ondisk_log_size":6735,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52264960,"num_objects":797,"num_object_clones":0,"num_object_copies":2391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5445,"num_write_kb":199776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":786,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.963147+0000","last_change":"2023-10-09T11:06:24.239676+0000","last_active":"2023-10-09T11:07:48.963147+0000","last_peered":"2023-10-09T11:07:48.963147+0000","last_clean":"2023-10-09T11:07:48.963147+0000","last_became_active":"2023-10-09T11:06:24.239375+0000","last_became_peered":"2023-10-09T11:06:24.239375+0000","last_unstale":"2023-10-09T11:07:48.963147+0000","last_undegraded":"2023-10-09T11:07:48.963147+0000","last_fullsized":"2023-10-09T11:07:48.963147+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:27.953053+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976253+0000","last_change":"2023-10-09T11:07:33.229415+0000","last_active":"2023-10-09T11:07:48.976253+0000","last_peered":"2023-10-09T11:07:48.976253+0000","last_clean":"2023-10-09T11:07:48.976253+0000","last_became_active":"2023-10-09T11:07:33.229061+0000","last_became_peered":"2023-10-09T11:07:33.229061+0000","last_unstale":"2023-10-09T11:07:48.976253+0000","last_undegraded":"2023-10-09T11:07:48.976253+0000","last_fullsized":"2023-10-09T11:07:48.976253+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:30.978293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":237,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.990441+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:07:48.990441+0000","last_peered":"2023-10-09T11:07:48.990441+0000","last_clean":"2023-10-09T11:07:48.990441+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:07:48.990441+0000","last_undegraded":"2023-10-09T11:07:48.990441+0000","last_fullsized":"2023-10-09T11:07:48.990441+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":515,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975281+0000","last_change":"2023-10-09T11:07:33.227271+0000","last_active":"2023-10-09T11:07:48.975281+0000","last_peered":"2023-10-09T11:07:48.975281+0000","last_clean":"2023-10-09T11:07:48.975281+0000","last_became_active":"2023-10-09T11:07:33.226760+0000","last_became_peered":"2023-10-09T11:07:33.226760+0000","last_unstale":"2023-10-09T11:07:48.975281+0000","last_undegraded":"2023-10-09T11:07:48.975281+0000","last_fullsized":"2023-10-09T11:07:48.975281+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:12.268123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"556'8690","reported_seq":11648,"reported_epoch":556,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:07:58.233466+0000","last_change":"2023-10-09T11:07:34.375236+0000","last_active":"2023-10-09T11:07:58.233466+0000","last_peered":"2023-10-09T11:07:58.233466+0000","last_clean":"2023-10-09T11:06:06.963741+0000","last_became_active":"2023-10-09T11:07:34.357802+0000","last_became_peered":"2023-10-09T11:07:34.357802+0000","last_unstale":"2023-10-09T11:07:58.233466+0000","last_undegraded":"2023-10-09T11:07:34.289955+0000","last_fullsized":"2023-10-09T11:07:34.281689+0000","mapping_epoch":550,"log_start":"504'1895","ondisk_log_start":"504'1895","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6795,"log_dups_size":0,"ondisk_log_size":6795,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55050240,"num_objects":840,"num_object_clones":0,"num_object_copies":2520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2256,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5460,"num_write_kb":201600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1434,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1,2","objects":745},{"shards":"1,2","objects":95}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":755,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.978087+0000","last_change":"2023-10-09T11:05:23.323368+0000","last_active":"2023-10-09T11:07:48.978087+0000","last_peered":"2023-10-09T11:07:48.978087+0000","last_clean":"2023-10-09T11:07:48.978087+0000","last_became_active":"2023-10-09T11:05:23.323032+0000","last_became_peered":"2023-10-09T11:05:23.323032+0000","last_unstale":"2023-10-09T11:07:48.978087+0000","last_undegraded":"2023-10-09T11:07:48.978087+0000","last_fullsized":"2023-10-09T11:07:48.978087+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:09.393584+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":112,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.963267+0000","last_change":"2023-10-09T11:07:33.246378+0000","last_active":"2023-10-09T11:07:48.963267+0000","last_peered":"2023-10-09T11:07:48.963267+0000","last_clean":"2023-10-09T11:07:48.963267+0000","last_became_active":"2023-10-09T11:07:33.228019+0000","last_became_peered":"2023-10-09T11:07:33.228019+0000","last_unstale":"2023-10-09T11:07:48.963267+0000","last_undegraded":"2023-10-09T11:07:48.963267+0000","last_fullsized":"2023-10-09T11:07:48.963267+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:18.511517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":264,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.986949+0000","last_change":"2023-10-09T11:05:20.310417+0000","last_active":"2023-10-09T11:07:48.986949+0000","last_peered":"2023-10-09T11:07:48.986949+0000","last_clean":"2023-10-09T11:07:48.986949+0000","last_became_active":"2023-10-09T11:05:20.309827+0000","last_became_peered":"2023-10-09T11:05:20.309827+0000","last_unstale":"2023-10-09T11:07:48.986949+0000","last_undegraded":"2023-10-09T11:07:48.986949+0000","last_fullsized":"2023-10-09T11:07:48.986949+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:23.934383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":100,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.959899+0000","last_change":"2023-10-09T11:07:33.217595+0000","last_active":"2023-10-09T11:07:48.959899+0000","last_peered":"2023-10-09T11:07:48.959899+0000","last_clean":"2023-10-09T11:07:48.959899+0000","last_became_active":"2023-10-09T11:07:33.217155+0000","last_became_peered":"2023-10-09T11:07:33.217155+0000","last_unstale":"2023-10-09T11:07:48.959899+0000","last_undegraded":"2023-10-09T11:07:48.959899+0000","last_fullsized":"2023-10-09T11:07:48.959899+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:55.127329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"556'8568","reported_seq":8648,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:58.316540+0000","last_change":"2023-10-09T11:06:42.990792+0000","last_active":"2023-10-09T11:07:58.316540+0000","last_peered":"2023-10-09T11:07:58.316540+0000","last_clean":"2023-10-09T11:07:58.316540+0000","last_became_active":"2023-10-09T11:06:42.988050+0000","last_became_peered":"2023-10-09T11:06:42.988050+0000","last_unstale":"2023-10-09T11:07:58.316540+0000","last_undegraded":"2023-10-09T11:07:58.316540+0000","last_fullsized":"2023-10-09T11:07:58.316540+0000","mapping_epoch":495,"log_start":"504'1803","ondisk_log_start":"504'1803","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6765,"log_dups_size":0,"ondisk_log_size":6765,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51183616,"num_objects":781,"num_object_clones":0,"num_object_copies":2343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5393,"num_write_kb":197568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":719,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.963604+0000","last_change":"2023-10-09T11:07:36.284491+0000","last_active":"2023-10-09T11:07:48.963604+0000","last_peered":"2023-10-09T11:07:48.963604+0000","last_clean":"2023-10-09T11:07:48.963604+0000","last_became_active":"2023-10-09T11:07:36.284249+0000","last_became_peered":"2023-10-09T11:07:36.284249+0000","last_unstale":"2023-10-09T11:07:48.963604+0000","last_undegraded":"2023-10-09T11:07:48.963604+0000","last_fullsized":"2023-10-09T11:07:48.963604+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:03.653371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:03.653371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:09.629417+0000","scrub_duration":0.001576771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.978134+0000","last_change":"2023-10-09T11:07:33.241346+0000","last_active":"2023-10-09T11:07:48.978134+0000","last_peered":"2023-10-09T11:07:48.978134+0000","last_clean":"2023-10-09T11:07:48.978134+0000","last_became_active":"2023-10-09T11:07:33.241013+0000","last_became_peered":"2023-10-09T11:07:33.241013+0000","last_unstale":"2023-10-09T11:07:48.978134+0000","last_undegraded":"2023-10-09T11:07:48.978134+0000","last_fullsized":"2023-10-09T11:07:48.978134+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:57.164502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":254,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.986748+0000","last_change":"2023-10-09T11:06:24.253714+0000","last_active":"2023-10-09T11:07:48.986748+0000","last_peered":"2023-10-09T11:07:48.986748+0000","last_clean":"2023-10-09T11:07:48.986748+0000","last_became_active":"2023-10-09T11:06:24.253143+0000","last_became_peered":"2023-10-09T11:06:24.253143+0000","last_unstale":"2023-10-09T11:07:48.986748+0000","last_undegraded":"2023-10-09T11:07:48.986748+0000","last_fullsized":"2023-10-09T11:07:48.986748+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:05.682288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":361,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.974720+0000","last_change":"2023-10-09T11:05:20.313560+0000","last_active":"2023-10-09T11:07:48.974720+0000","last_peered":"2023-10-09T11:07:48.974720+0000","last_clean":"2023-10-09T11:07:48.974720+0000","last_became_active":"2023-10-09T11:05:20.313214+0000","last_became_peered":"2023-10-09T11:05:20.313214+0000","last_unstale":"2023-10-09T11:07:48.974720+0000","last_undegraded":"2023-10-09T11:07:48.974720+0000","last_fullsized":"2023-10-09T11:07:48.974720+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:32.527205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"556'7062","reported_seq":8851,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:56.550373+0000","last_change":"2023-10-09T11:07:34.333312+0000","last_active":"2023-10-09T11:07:56.550373+0000","last_peered":"2023-10-09T11:07:56.550373+0000","last_clean":"2023-10-09T11:07:56.550373+0000","last_became_active":"2023-10-09T11:07:34.285534+0000","last_became_peered":"2023-10-09T11:07:34.285534+0000","last_unstale":"2023-10-09T11:07:56.550373+0000","last_undegraded":"2023-10-09T11:07:56.550373+0000","last_fullsized":"2023-10-09T11:07:56.550373+0000","mapping_epoch":550,"log_start":"498'1002","ondisk_log_start":"498'1002","created":509,"last_epoch_clean":551,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6060,"log_dups_size":0,"ondisk_log_size":6060,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59015168,"num_objects":901,"num_object_clones":0,"num_object_copies":2703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5471,"num_write_kb":203872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":838,"num_bytes_recovered":54853632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":963,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.960420+0000","last_change":"2023-10-09T11:05:23.329886+0000","last_active":"2023-10-09T11:07:48.960420+0000","last_peered":"2023-10-09T11:07:48.960420+0000","last_clean":"2023-10-09T11:07:48.960420+0000","last_became_active":"2023-10-09T11:05:23.329561+0000","last_became_peered":"2023-10-09T11:05:23.329561+0000","last_unstale":"2023-10-09T11:07:48.960420+0000","last_undegraded":"2023-10-09T11:07:48.960420+0000","last_fullsized":"2023-10-09T11:07:48.960420+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:46.933799+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":16,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976148+0000","last_change":"2023-10-09T11:07:33.229864+0000","last_active":"2023-10-09T11:07:48.976148+0000","last_peered":"2023-10-09T11:07:48.976148+0000","last_clean":"2023-10-09T11:07:48.976148+0000","last_became_active":"2023-10-09T11:07:33.229547+0000","last_became_peered":"2023-10-09T11:07:33.229547+0000","last_unstale":"2023-10-09T11:07:48.976148+0000","last_undegraded":"2023-10-09T11:07:48.976148+0000","last_fullsized":"2023-10-09T11:07:48.976148+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:34.990439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":338,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.981611+0000","last_change":"2023-10-09T11:04:50.510688+0000","last_active":"2023-10-09T11:07:48.981611+0000","last_peered":"2023-10-09T11:07:48.981611+0000","last_clean":"2023-10-09T11:07:48.981611+0000","last_became_active":"2023-10-09T11:04:50.507404+0000","last_became_peered":"2023-10-09T11:04:50.507404+0000","last_unstale":"2023-10-09T11:07:48.981611+0000","last_undegraded":"2023-10-09T11:07:48.981611+0000","last_fullsized":"2023-10-09T11:07:48.981611+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:13.663394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"556'6878","reported_seq":8322,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:56.543242+0000","last_change":"2023-10-09T11:07:26.133461+0000","last_active":"2023-10-09T11:07:56.543242+0000","last_peered":"2023-10-09T11:07:56.543242+0000","last_clean":"2023-10-09T11:07:56.543242+0000","last_became_active":"2023-10-09T11:05:45.387324+0000","last_became_peered":"2023-10-09T11:05:45.387324+0000","last_unstale":"2023-10-09T11:07:56.543242+0000","last_undegraded":"2023-10-09T11:07:56.543242+0000","last_fullsized":"2023-10-09T11:07:56.543242+0000","mapping_epoch":509,"log_start":"498'118","ondisk_log_start":"498'118","created":509,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6760,"log_dups_size":0,"ondisk_log_size":6760,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55246848,"num_objects":843,"num_object_clones":0,"num_object_copies":2529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5325,"num_write_kb":197376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":667,"num_bytes_recovered":43712512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":920,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.963546+0000","last_change":"2023-10-09T11:05:23.321040+0000","last_active":"2023-10-09T11:07:48.963546+0000","last_peered":"2023-10-09T11:07:48.963546+0000","last_clean":"2023-10-09T11:07:48.963546+0000","last_became_active":"2023-10-09T11:05:23.320674+0000","last_became_peered":"2023-10-09T11:05:23.320674+0000","last_unstale":"2023-10-09T11:07:48.963546+0000","last_undegraded":"2023-10-09T11:07:48.963546+0000","last_fullsized":"2023-10-09T11:07:48.963546+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.527660+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":383,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.975590+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:07:48.975590+0000","last_peered":"2023-10-09T11:07:48.975590+0000","last_clean":"2023-10-09T11:07:48.975590+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:07:48.975590+0000","last_undegraded":"2023-10-09T11:07:48.975590+0000","last_fullsized":"2023-10-09T11:07:48.975590+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":82,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.963681+0000","last_change":"2023-10-09T11:04:50.511984+0000","last_active":"2023-10-09T11:07:48.963681+0000","last_peered":"2023-10-09T11:07:48.963681+0000","last_clean":"2023-10-09T11:07:48.963681+0000","last_became_active":"2023-10-09T11:04:50.510820+0000","last_became_peered":"2023-10-09T11:04:50.510820+0000","last_unstale":"2023-10-09T11:07:48.963681+0000","last_undegraded":"2023-10-09T11:07:48.963681+0000","last_fullsized":"2023-10-09T11:07:48.963681+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:38.662189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.15","version":"556'6796","reported_seq":11474,"reported_epoch":556,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:07:58.211220+0000","last_change":"2023-10-09T11:07:34.356257+0000","last_active":"2023-10-09T11:07:58.211220+0000","last_peered":"2023-10-09T11:07:58.211220+0000","last_clean":"2023-10-09T11:07:32.098315+0000","last_became_active":"2023-10-09T11:07:34.338915+0000","last_became_peered":"2023-10-09T11:07:34.338915+0000","last_unstale":"2023-10-09T11:07:58.211220+0000","last_undegraded":"2023-10-09T11:07:58.211220+0000","last_fullsized":"2023-10-09T11:07:58.211220+0000","mapping_epoch":550,"log_start":"498'1066","ondisk_log_start":"498'1066","created":509,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5730,"log_dups_size":0,"ondisk_log_size":5730,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51544064,"num_objects":787,"num_object_clones":0,"num_object_copies":2361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":787,"num_objects_unfound":0,"num_objects_dirty":787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5279,"num_write_kb":194080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2280,"num_bytes_recovered":149422080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":787}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":534,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.974872+0000","last_change":"2023-10-09T11:05:20.327032+0000","last_active":"2023-10-09T11:07:48.974872+0000","last_peered":"2023-10-09T11:07:48.974872+0000","last_clean":"2023-10-09T11:07:48.974872+0000","last_became_active":"2023-10-09T11:05:20.320193+0000","last_became_peered":"2023-10-09T11:05:20.320193+0000","last_unstale":"2023-10-09T11:07:48.974872+0000","last_undegraded":"2023-10-09T11:07:48.974872+0000","last_fullsized":"2023-10-09T11:07:48.974872+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:44.862890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":82,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.961362+0000","last_change":"2023-10-09T11:04:50.384219+0000","last_active":"2023-10-09T11:07:48.961362+0000","last_peered":"2023-10-09T11:07:48.961362+0000","last_clean":"2023-10-09T11:07:48.961362+0000","last_became_active":"2023-10-09T11:04:50.384028+0000","last_became_peered":"2023-10-09T11:04:50.384028+0000","last_unstale":"2023-10-09T11:07:48.961362+0000","last_undegraded":"2023-10-09T11:07:48.961362+0000","last_fullsized":"2023-10-09T11:07:48.961362+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:04.721869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.16","version":"556'6844","reported_seq":6938,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:56.197408+0000","last_change":"2023-10-09T11:07:24.733266+0000","last_active":"2023-10-09T11:07:56.197408+0000","last_peered":"2023-10-09T11:07:56.197408+0000","last_clean":"2023-10-09T11:07:56.197408+0000","last_became_active":"2023-10-09T11:06:24.254366+0000","last_became_peered":"2023-10-09T11:06:24.254366+0000","last_unstale":"2023-10-09T11:07:56.197408+0000","last_undegraded":"2023-10-09T11:07:56.197408+0000","last_fullsized":"2023-10-09T11:07:56.197408+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6844,"log_dups_size":0,"ondisk_log_size":6844,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58720256,"num_objects":896,"num_object_clones":0,"num_object_copies":2688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5318,"num_write_kb":198848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":729,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.976550+0000","last_change":"2023-10-09T11:07:36.291401+0000","last_active":"2023-10-09T11:07:48.976550+0000","last_peered":"2023-10-09T11:07:48.976550+0000","last_clean":"2023-10-09T11:07:48.976550+0000","last_became_active":"2023-10-09T11:07:36.291077+0000","last_became_peered":"2023-10-09T11:07:36.291077+0000","last_unstale":"2023-10-09T11:07:48.976550+0000","last_undegraded":"2023-10-09T11:07:48.976550+0000","last_fullsized":"2023-10-09T11:07:48.976550+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:04.790935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.790935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":91,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.963643+0000","last_change":"2023-10-09T11:05:20.299321+0000","last_active":"2023-10-09T11:07:48.963643+0000","last_peered":"2023-10-09T11:07:48.963643+0000","last_clean":"2023-10-09T11:07:48.963643+0000","last_became_active":"2023-10-09T11:05:20.299019+0000","last_became_peered":"2023-10-09T11:05:20.299019+0000","last_unstale":"2023-10-09T11:07:48.963643+0000","last_undegraded":"2023-10-09T11:07:48.963643+0000","last_fullsized":"2023-10-09T11:07:48.963643+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:08.645909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":82,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.960493+0000","last_change":"2023-10-09T11:04:50.366890+0000","last_active":"2023-10-09T11:07:48.960493+0000","last_peered":"2023-10-09T11:07:48.960493+0000","last_clean":"2023-10-09T11:07:48.960493+0000","last_became_active":"2023-10-09T11:04:50.366558+0000","last_became_peered":"2023-10-09T11:04:50.366558+0000","last_unstale":"2023-10-09T11:07:48.960493+0000","last_undegraded":"2023-10-09T11:07:48.960493+0000","last_fullsized":"2023-10-09T11:07:48.960493+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:50.993153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"556'6896","reported_seq":6976,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:56.719723+0000","last_change":"2023-10-09T11:07:53.748581+0000","last_active":"2023-10-09T11:07:56.719723+0000","last_peered":"2023-10-09T11:07:56.719723+0000","last_clean":"2023-10-09T11:07:56.719723+0000","last_became_active":"2023-10-09T11:05:45.356981+0000","last_became_peered":"2023-10-09T11:05:45.356981+0000","last_unstale":"2023-10-09T11:07:56.719723+0000","last_undegraded":"2023-10-09T11:07:56.719723+0000","last_fullsized":"2023-10-09T11:07:56.719723+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6896,"log_dups_size":0,"ondisk_log_size":6896,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54689792,"num_objects":835,"num_object_clones":0,"num_object_copies":2505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5327,"num_write_kb":197152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":855,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.985959+0000","last_change":"2023-10-09T11:04:53.274332+0000","last_active":"2023-10-09T11:07:48.985959+0000","last_peered":"2023-10-09T11:07:48.985959+0000","last_clean":"2023-10-09T11:07:48.985959+0000","last_became_active":"2023-10-09T11:04:53.273798+0000","last_became_peered":"2023-10-09T11:04:53.273798+0000","last_unstale":"2023-10-09T11:07:48.985959+0000","last_undegraded":"2023-10-09T11:07:48.985959+0000","last_fullsized":"2023-10-09T11:07:48.985959+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:04.220970+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":103,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.977789+0000","last_change":"2023-10-09T11:06:24.249877+0000","last_active":"2023-10-09T11:07:48.977789+0000","last_peered":"2023-10-09T11:07:48.977789+0000","last_clean":"2023-10-09T11:07:48.977789+0000","last_became_active":"2023-10-09T11:06:24.249573+0000","last_became_peered":"2023-10-09T11:06:24.249573+0000","last_unstale":"2023-10-09T11:07:48.977789+0000","last_undegraded":"2023-10-09T11:07:48.977789+0000","last_fullsized":"2023-10-09T11:07:48.977789+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:11.798192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":604,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-09T11:07:48.992549+0000","last_change":"2023-10-09T11:05:20.307347+0000","last_active":"2023-10-09T11:07:48.992549+0000","last_peered":"2023-10-09T11:07:48.992549+0000","last_clean":"2023-10-09T11:07:48.992549+0000","last_became_active":"2023-10-09T11:05:20.307126+0000","last_became_peered":"2023-10-09T11:05:20.307126+0000","last_unstale":"2023-10-09T11:07:48.992549+0000","last_undegraded":"2023-10-09T11:07:48.992549+0000","last_fullsized":"2023-10-09T11:07:48.992549+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:34.408515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":36,"stat_sum":{"num_bytes":1714225176,"num_objects":26158,"num_object_clones":0,"num_object_copies":78474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6040,"num_objects_misplaced":2851,"num_objects_unfound":0,"num_objects_dirty":26158,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":172836,"num_write_kb":6367745,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23282,"num_bytes_recovered":1449328640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6047215616,"data_stored":6047203400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":231311,"ondisk_log_size":231311,"up":108,"acting":104,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":91,"num_bytes_recovered":15742,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12694,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":626148,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":31,"num_read_kb":56,"num_write":15,"num_write_kb":147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2632652,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332496,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1961484,"kb_used_data":1066180,"kb_used_omap":0,"kb_used_meta":895296,"kb_avail":91783668,"statfs":{"total":95995035648,"available":93986476032,"internally_reserved":0,"allocated":1091768320,"data_stored":1089978815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916783104},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":1.5009999999999999,"15min":1.6839999999999999},"min":{"1min":0.27300000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":5.2539999999999996,"5min":9.8040000000000003,"15min":9.8040000000000003},"last":2.246},{"interface":"front","average":{"1min":1.7989999999999999,"5min":1.7170000000000001,"15min":1.746},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.216},"max":{"1min":15.853,"5min":15.853,"15min":15.853},"last":3.5699999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:07:23 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":1.3959999999999999,"15min":1.52},"min":{"1min":0.23799999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.9130000000000003,"5min":11.619999999999999,"15min":11.619999999999999},"last":1.1579999999999999},{"interface":"front","average":{"1min":2.0019999999999998,"5min":1.8500000000000001,"15min":1.891},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":11.038,"5min":11.038,"15min":12.193},"last":13.096}]},{"osd":2,"last update":"Mon Oct 9 11:07:52 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":1.929,"15min":1.7170000000000001},"min":{"1min":0.313,"5min":0.309,"15min":0.245},"max":{"1min":3.3050000000000002,"5min":16.605,"15min":16.605},"last":3.5089999999999999},{"interface":"front","average":{"1min":1.466,"5min":1.4570000000000001,"15min":1.706},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":4.9569999999999999,"5min":9.7469999999999999,"15min":13.037000000000001},"last":2.2730000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:07:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.524,"15min":1.4399999999999999},"min":{"1min":0.30499999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.4770000000000003,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":4.0049999999999999},{"interface":"front","average":{"1min":2.1789999999999998,"5min":1.9019999999999999,"15min":1.8280000000000001},"min":{"1min":0.38200000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.968,"5min":8.2330000000000005,"15min":8.2330000000000005},"last":3.6139999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:07:02 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.0389999999999999,"15min":1.083},"min":{"1min":0.25700000000000001,"5min":0.218,"15min":0.218},"max":{"1min":3.927,"5min":7.6589999999999998,"15min":8.2940000000000005},"last":3.5979999999999999},{"interface":"front","average":{"1min":1.008,"5min":1.623,"15min":1.248},"min":{"1min":0.28299999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":9.2759999999999998,"5min":16.481000000000002,"15min":16.481000000000002},"last":3.556}]},{"osd":5,"last update":"Mon Oct 9 11:07:02 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.2589999999999999,"15min":1.262},"min":{"1min":0.28499999999999998,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":2.1829999999999998,"5min":15.414999999999999,"15min":15.414999999999999},"last":2.3479999999999999},{"interface":"front","average":{"1min":0.73599999999999999,"5min":1.3460000000000001,"15min":1.175},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.9780000000000002,"5min":15.369,"15min":15.369},"last":13.045}]},{"osd":6,"last update":"Mon Oct 9 11:07:02 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.367,"15min":1.298},"min":{"1min":0.35999999999999999,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":4.0259999999999998,"5min":14.159000000000001,"15min":14.159000000000001},"last":3.6360000000000001},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.8759999999999999,"15min":1.3959999999999999},"min":{"1min":0.35499999999999998,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":3.9630000000000001,"5min":16.538,"15min":16.538},"last":3.5830000000000002}]}]},{"osd":6,"up_from":44,"seq":188978561427,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2537856,"kb_used_data":1041208,"kb_used_omap":0,"kb_used_meta":1496640,"kb_avail":91207296,"statfs":{"total":95995035648,"available":93396271104,"internally_reserved":0,"allocated":1066196992,"data_stored":1064419109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1532559360},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.6719999999999999,"15min":1.875},"min":{"1min":0.23899999999999999,"5min":0.217,"15min":0.217},"max":{"1min":3.7389999999999999,"5min":29.379999999999999,"15min":54.743000000000002},"last":5.7590000000000003},{"interface":"front","average":{"1min":1.3120000000000001,"5min":1.6200000000000001,"15min":1.9299999999999999},"min":{"1min":0.247,"5min":0.214,"15min":0.214},"max":{"1min":7.1890000000000001,"5min":29.132999999999999,"15min":53.701000000000001},"last":3.3740000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:07:26 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":1.5369999999999999,"15min":1.1679999999999999},"min":{"1min":0.318,"5min":0.26000000000000001,"15min":0.23499999999999999},"max":{"1min":5.0110000000000001,"5min":16.225999999999999,"15min":41.241999999999997},"last":1.51},{"interface":"front","average":{"1min":2.98,"5min":2.0489999999999999,"15min":1.7350000000000001},"min":{"1min":0.31,"5min":0.26200000000000001,"15min":0.248},"max":{"1min":9.173,"5min":11.715,"15min":41.476999999999997},"last":0.55500000000000005}]},{"osd":2,"last update":"Mon Oct 9 11:07:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":2.1890000000000001,"15min":1.857},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.7089999999999996,"5min":10.481999999999999,"15min":12.003},"last":1.0580000000000001},{"interface":"front","average":{"1min":1.6990000000000001,"5min":1.764,"15min":1.8999999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.133,"5min":12.693,"15min":15.432},"last":3.9790000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:07:23 2023","interfaces":[{"interface":"back","average":{"1min":3.125,"5min":1.7629999999999999,"15min":1.554},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":28.948,"5min":28.948,"15min":28.948},"last":1.4490000000000001},{"interface":"front","average":{"1min":5.3739999999999997,"5min":2.9390000000000001,"15min":2.605},"min":{"1min":0.496,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":28.893000000000001,"5min":28.893000000000001,"15min":28.893000000000001},"last":0.307}]},{"osd":4,"last update":"Mon Oct 9 11:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":1.171,"15min":1.0129999999999999},"min":{"1min":0.26500000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.0189999999999999,"5min":17.216999999999999,"15min":17.216999999999999},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.78100000000000003,"15min":0.84299999999999997},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.24399999999999999},"max":{"1min":1.4059999999999999,"5min":10.025,"15min":10.025},"last":1.4059999999999999}]},{"osd":5,"last update":"Mon Oct 9 11:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.1419999999999999,"15min":1.2709999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":1.103,"5min":10.209,"15min":19.786999999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.90500000000000003,"15min":0.97099999999999997},"min":{"1min":0.33200000000000002,"5min":0.248,"15min":0.248},"max":{"1min":1.4810000000000001,"5min":9.9469999999999992,"15min":9.9469999999999992},"last":1.4810000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":1.419,"15min":1.3169999999999999},"min":{"1min":0.34200000000000003,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.7210000000000001,"5min":11.550000000000001,"15min":12.882},"last":2.0939999999999999},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.85599999999999998,"15min":0.94099999999999995},"min":{"1min":0.32800000000000001,"5min":0.31,"15min":0.27500000000000002},"max":{"1min":1.643,"5min":11.557,"15min":11.557},"last":3.2839999999999998}]}]},{"osd":1,"up_from":388,"seq":1666447310942,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1335560,"kb_used_data":856128,"kb_used_omap":13,"kb_used_meta":479410,"kb_avail":92409592,"statfs":{"total":95995035648,"available":94627422208,"internally_reserved":0,"allocated":876675072,"data_stored":873738120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":490916551},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":3.895,"5min":2.593,"15min":2.4369999999999998},"min":{"1min":0.26700000000000002,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":11.960000000000001,"5min":18.192,"15min":18.192},"last":2.3679999999999999},{"interface":"front","average":{"1min":3.9390000000000001,"5min":2.8260000000000001,"15min":1.9259999999999999},"min":{"1min":0.27000000000000002,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":10.099,"5min":17.957999999999998,"15min":17.957999999999998},"last":9.3219999999999992}]},{"osd":2,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8439999999999999,"5min":2.5920000000000001,"15min":2.9300000000000002},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":9.5630000000000006,"5min":47.731000000000002,"15min":47.731000000000002},"last":4.282},{"interface":"front","average":{"1min":3.8199999999999998,"5min":2.903,"15min":2.4769999999999999},"min":{"1min":0.29399999999999998,"5min":0.216,"15min":0.216},"max":{"1min":10.138,"5min":47.786999999999999,"15min":47.786999999999999},"last":3.7090000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:07:25 2023","interfaces":[{"interface":"back","average":{"1min":3.7330000000000001,"5min":3.0870000000000002,"15min":2.0209999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":20.791,"5min":92.447999999999993,"15min":92.447999999999993},"last":1.9750000000000001},{"interface":"front","average":{"1min":3.1160000000000001,"5min":3.1179999999999999,"15min":2.028},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.217},"max":{"1min":7.3630000000000004,"5min":92.066999999999993,"15min":92.066999999999993},"last":2.2639999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.7869999999999999,"15min":2.395},"min":{"1min":0.30499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.5199999999999996,"5min":11.701000000000001,"15min":11.701000000000001},"last":4.2220000000000004},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.04,"15min":1.6240000000000001},"min":{"1min":0.314,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":4.6079999999999997,"5min":11.589,"15min":11.589},"last":3.6499999999999999}]},{"osd":5,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":2.0830000000000002,"15min":2.3050000000000002},"min":{"1min":0.28999999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.0720000000000001,"5min":20.689,"15min":20.689},"last":4.1749999999999998},{"interface":"front","average":{"1min":1.764,"5min":2.0590000000000002,"15min":2.2410000000000001},"min":{"1min":0.33800000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.4900000000000002,"5min":11.65,"15min":11.65},"last":4.2569999999999997}]},{"osd":6,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.833,"5min":2.4049999999999998,"15min":2.411},"min":{"1min":0.312,"5min":0.309,"15min":0.309},"max":{"1min":6.0330000000000004,"5min":20.646999999999998,"15min":20.646999999999998},"last":2.0939999999999999},{"interface":"front","average":{"1min":1.784,"5min":1.9219999999999999,"15min":2.1720000000000002},"min":{"1min":0.35599999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":5.5380000000000003,"5min":11.75,"15min":11.75},"last":6.3810000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.869,"5min":2.1640000000000001,"15min":1.615},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.32900000000000001},"max":{"1min":4.5780000000000003,"5min":11.555999999999999,"15min":11.555999999999999},"last":3.6819999999999999},{"interface":"front","average":{"1min":1.7789999999999999,"5min":2.0219999999999998,"15min":2.3210000000000002},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.1020000000000003,"5min":20.716000000000001,"15min":20.716000000000001},"last":6.5209999999999999}]}]},{"osd":0,"up_from":80,"seq":343597384000,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1571424,"kb_used_data":630104,"kb_used_omap":28,"kb_used_meta":941283,"kb_avail":92173728,"statfs":{"total":95995035648,"available":94385897472,"internally_reserved":0,"allocated":645226496,"data_stored":642198242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":963874640},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:07:25 2023","interfaces":[{"interface":"back","average":{"1min":2.923,"5min":2.1680000000000001,"15min":2.831},"min":{"1min":0.22700000000000001,"5min":0.186,"15min":0.186},"max":{"1min":10.271000000000001,"5min":14.644,"15min":19.608000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":3.879,"5min":2.2949999999999999,"15min":2.911},"min":{"1min":0.29499999999999998,"5min":0.214,"15min":0.214},"max":{"1min":15.834,"5min":15.834,"15min":19.992999999999999},"last":0.77200000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:07:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":2.339,"15min":2.29},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.17199999999999999},"max":{"1min":11.516,"5min":12.459,"15min":24.852},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.73,"5min":2.246,"15min":2.694},"min":{"1min":0.215,"5min":0.20300000000000001,"15min":0.19800000000000001},"max":{"1min":8.4589999999999996,"5min":11.68,"15min":77.256},"last":3.1819999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":3.2730000000000001,"15min":3.4449999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":13.189,"5min":25.050000000000001,"15min":25.050000000000001},"last":3.2490000000000001},{"interface":"front","average":{"1min":2.98,"5min":2.2770000000000001,"15min":2.0249999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":14.125999999999999,"5min":14.125999999999999,"15min":14.125999999999999},"last":1.4930000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.3799999999999999,"5min":2.1749999999999998,"15min":2.4470000000000001},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":13.106,"5min":14.507,"15min":77.177999999999997},"last":3.2120000000000002},{"interface":"front","average":{"1min":1.6919999999999999,"5min":1.5640000000000001,"15min":1.681},"min":{"1min":0.39500000000000002,"5min":0.30199999999999999,"15min":0.28899999999999998},"max":{"1min":7.9249999999999998,"5min":7.9249999999999998,"15min":10.513999999999999},"last":0.40200000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.7490000000000001,"15min":1.8400000000000001},"min":{"1min":0.39300000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.9550000000000001,"5min":7.9550000000000001,"15min":10.544},"last":2.2999999999999998},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.6499999999999999,"15min":1.8240000000000001},"min":{"1min":0.38100000000000001,"5min":0.308,"15min":0.29899999999999999},"max":{"1min":8.8439999999999994,"5min":11.212999999999999,"15min":24.754999999999999},"last":0.50600000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":1.736,"15min":1.9390000000000001},"min":{"1min":0.38,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":7.9829999999999997,"5min":7.9829999999999997,"15min":18.920000000000002},"last":2.1509999999999998},{"interface":"front","average":{"1min":2.3839999999999999,"5min":1.841,"15min":1.9590000000000001},"min":{"1min":0.42499999999999999,"5min":0.34999999999999998,"15min":0.31900000000000001},"max":{"1min":8.8810000000000002,"5min":11.257999999999999,"15min":24.800000000000001},"last":0.48199999999999998}]},{"osd":7,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":2.0790000000000002,"15min":2.0289999999999999},"min":{"1min":0.496,"5min":0.35099999999999998,"15min":0.32700000000000001},"max":{"1min":8.9139999999999997,"5min":17.672000000000001,"15min":17.672000000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.4369999999999998,"5min":2.3839999999999999,"15min":2.552},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":13.154,"5min":17.539000000000001,"15min":77.228999999999999},"last":3.2320000000000002}]}]},{"osd":2,"up_from":90,"seq":386547056940,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1447656,"kb_used_data":695392,"kb_used_omap":81,"kb_used_meta":751982,"kb_avail":92297496,"statfs":{"total":95995035648,"available":94512635904,"internally_reserved":0,"allocated":712081408,"data_stored":709225657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":770029786},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7250000000000001,"5min":2.6549999999999998,"15min":2.0169999999999999},"min":{"1min":0.182,"5min":0.182,"15min":0.17499999999999999},"max":{"1min":15.609,"5min":48.933999999999997,"15min":48.933999999999997},"last":0.188},{"interface":"front","average":{"1min":2.5990000000000002,"5min":1.883,"15min":1.8440000000000001},"min":{"1min":0.26900000000000002,"5min":0.23899999999999999,"15min":0.19600000000000001},"max":{"1min":9.8010000000000002,"5min":14.515000000000001,"15min":15.595000000000001},"last":0.30299999999999999}]},{"osd":1,"last update":"Mon Oct 9 11:07:30 2023","interfaces":[{"interface":"back","average":{"1min":4.2800000000000002,"5min":2.96,"15min":2.629},"min":{"1min":0.20200000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":24.902000000000001,"5min":49.033999999999999,"15min":49.033999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":4.7300000000000004,"5min":3.0550000000000002,"15min":2.7200000000000002},"min":{"1min":0.27500000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":25.850999999999999,"5min":48.874000000000002,"15min":48.874000000000002},"last":0.253}]},{"osd":3,"last update":"Mon Oct 9 11:07:17 2023","interfaces":[{"interface":"back","average":{"1min":4.1989999999999998,"5min":2.3330000000000002,"15min":1.954},"min":{"1min":0.751,"5min":0.214,"15min":0.214},"max":{"1min":15.701000000000001,"5min":15.701000000000001,"15min":15.701000000000001},"last":0.93799999999999994},{"interface":"front","average":{"1min":3.5350000000000001,"5min":2.4329999999999998,"15min":2.2410000000000001},"min":{"1min":0.313,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":15.73,"5min":15.73,"15min":15.73},"last":0.58099999999999996}]},{"osd":4,"last update":"Mon Oct 9 11:07:00 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":1.7130000000000001,"15min":1.871},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.722,"5min":9.7989999999999995,"15min":15.672000000000001},"last":0.72399999999999998},{"interface":"front","average":{"1min":2.0110000000000001,"5min":2.3380000000000001,"15min":1.901},"min":{"1min":0.34399999999999997,"5min":0.249,"15min":0.249},"max":{"1min":15.571,"5min":48.795000000000002,"15min":48.795000000000002},"last":0.48999999999999999}]},{"osd":5,"last update":"Mon Oct 9 11:07:00 2023","interfaces":[{"interface":"back","average":{"1min":1.98,"5min":1.405,"15min":1.548},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.25900000000000001},"max":{"1min":8.5310000000000006,"5min":8.5310000000000006,"15min":12.128},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.329,"5min":2.1160000000000001,"15min":1.974},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.6859999999999999,"5min":49.061,"15min":49.061},"last":0.63}]},{"osd":6,"last update":"Mon Oct 9 11:07:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0259999999999998,"5min":2.5059999999999998,"15min":1.8400000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":15.536,"5min":48.845999999999997,"15min":48.845999999999997},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.7230000000000001,"15min":1.6100000000000001},"min":{"1min":0.36799999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.4939999999999998,"5min":8.4939999999999998,"15min":10.111000000000001},"last":0.50700000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:07:00 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":2.593,"15min":1.96},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":15.59,"5min":48.966000000000001,"15min":48.966000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":2.0270000000000001,"5min":1.746,"15min":1.7470000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":8.5549999999999997,"5min":14.94,"15min":14.94},"last":0.53700000000000003}]}]},{"osd":3,"up_from":469,"seq":2014339661870,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1152484,"kb_used_data":843676,"kb_used_omap":11,"kb_used_meta":308788,"kb_avail":92592668,"statfs":{"total":95995035648,"available":94814892032,"internally_reserved":0,"allocated":863924224,"data_stored":860775947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11309,"internal_metadata":316199891},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":3.3700000000000001,"5min":1.927,"15min":1.7889999999999999},"min":{"1min":0.27400000000000002,"5min":0.183,"15min":0.183},"max":{"1min":9.9809999999999999,"5min":14.882,"15min":14.882},"last":0.93200000000000005},{"interface":"front","average":{"1min":1.7050000000000001,"5min":1.0920000000000001,"15min":0.94199999999999995},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":5.6980000000000004,"5min":5.6980000000000004,"15min":5.6980000000000004},"last":0.20100000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":3.536,"5min":2.0859999999999999,"15min":1.6299999999999999},"min":{"1min":0.17100000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":11.757,"5min":17.93,"15min":17.93},"last":2.0089999999999999},{"interface":"front","average":{"1min":3.3050000000000002,"5min":1.9610000000000001,"15min":1.71},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":19.163,"5min":19.163,"15min":19.163},"last":6.6310000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":4.46,"5min":1.804,"15min":1.3260000000000001},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":38.840000000000003,"5min":38.840000000000003,"15min":38.840000000000003},"last":0.65900000000000003},{"interface":"front","average":{"1min":4.7619999999999996,"5min":2.0979999999999999,"15min":1.419},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":37.991999999999997,"5min":37.991999999999997,"15min":37.991999999999997},"last":0.76700000000000002}]},{"osd":4,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":2.274,"5min":1.641,"15min":1.595},"min":{"1min":0.30599999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":10.058999999999999,"5min":14.907999999999999,"15min":14.907999999999999},"last":0.878},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.873,"15min":0.755},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.294,"5min":4.1390000000000002,"15min":4.1390000000000002},"last":0.29399999999999998}]},{"osd":5,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":1.5,"15min":1.4419999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.8550000000000004,"5min":9.3300000000000001,"15min":9.3300000000000001},"last":0.80800000000000005},{"interface":"front","average":{"1min":2.2650000000000001,"5min":1.5429999999999999,"15min":1.4750000000000001},"min":{"1min":0.32800000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.029,"5min":10.029,"15min":10.029},"last":0.84499999999999997}]},{"osd":6,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.76,"15min":1.7749999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.5289999999999999,"5min":15.455,"15min":15.455},"last":0.83699999999999997},{"interface":"front","average":{"1min":2.3100000000000001,"5min":1.673,"15min":1.653},"min":{"1min":0.374,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.099,"5min":14.837,"15min":14.837},"last":0.90400000000000003}]},{"osd":7,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":1.208,"15min":1.0700000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.907,"5min":6.907,"15min":6.907},"last":0.315},{"interface":"front","average":{"1min":2.0590000000000002,"5min":1.5169999999999999,"15min":1.2769999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.9320000000000004,"5min":6.9119999999999999,"15min":6.9119999999999999},"last":4.8140000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019290,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1337420,"kb_used_data":452868,"kb_used_omap":0,"kb_used_meta":884544,"kb_avail":92407732,"statfs":{"total":95995035648,"available":94625517568,"internally_reserved":0,"allocated":463736832,"data_stored":461947060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":905773056},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:18 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.659,"15min":2.399},"min":{"1min":0.23200000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":3.6150000000000002,"5min":13.162000000000001,"15min":213.88999999999999},"last":1.1819999999999999},{"interface":"front","average":{"1min":1.569,"5min":2.161,"15min":2.6379999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":4.6699999999999999,"5min":11.128,"15min":214.09800000000001},"last":3.875}]},{"osd":1,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.54,"5min":2.508,"15min":2.6299999999999999},"min":{"1min":0.34599999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":8.1229999999999993,"5min":57.966999999999999,"15min":121.41},"last":2.0699999999999998},{"interface":"front","average":{"1min":2.117,"5min":2.3250000000000002,"15min":2.1200000000000001},"min":{"1min":0.26300000000000001,"5min":0.251,"15min":0.251},"max":{"1min":8.0960000000000001,"5min":57.832000000000001,"15min":122.21899999999999},"last":3.0350000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:06:58 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.8580000000000001,"15min":1.7509999999999999},"min":{"1min":0.29499999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":18.407,"5min":18.407,"15min":18.407},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.476,"5min":1.9810000000000001,"15min":2.0670000000000002},"min":{"1min":0.33200000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.2119999999999997,"5min":8.4250000000000007,"15min":27.818000000000001},"last":3.839}]},{"osd":3,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.837,"5min":2.4460000000000002,"15min":1.982},"min":{"1min":0.40899999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.0529999999999999,"5min":60.753,"15min":60.753},"last":2.984},{"interface":"front","average":{"1min":2.6469999999999998,"5min":2.1389999999999998,"15min":1.492},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.0220000000000002,"5min":61.473999999999997,"15min":61.473999999999997},"last":0.65900000000000003}]},{"osd":5,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":1.5049999999999999,"15min":1.4590000000000001},"min":{"1min":0.27400000000000002,"5min":0.224,"15min":0.224},"max":{"1min":3.4860000000000002,"5min":11.59,"15min":19.068999999999999},"last":2.6549999999999998},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.3620000000000001,"15min":1.272},"min":{"1min":0.27300000000000002,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":3.8999999999999999,"5min":9.8610000000000007,"15min":15.535},"last":0.83999999999999997}]},{"osd":6,"last update":"Mon Oct 9 11:06:58 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.3879999999999999,"15min":1.196},"min":{"1min":0.29899999999999999,"5min":0.28999999999999998,"15min":0.26300000000000001},"max":{"1min":8.3930000000000007,"5min":8.3930000000000007,"15min":15.739000000000001},"last":3.012},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.619,"15min":1.5189999999999999},"min":{"1min":0.35699999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.0109999999999992,"5min":12.795999999999999,"15min":14.093999999999999},"last":0.623}]},{"osd":7,"last update":"Mon Oct 9 11:06:58 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.6870000000000001,"15min":1.6399999999999999},"min":{"1min":0.318,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":10.15,"5min":12.753,"15min":19.923999999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.288,"5min":1.5109999999999999,"15min":1.5309999999999999},"min":{"1min":0.316,"5min":0.29799999999999999,"15min":0.255},"max":{"1min":10.853,"5min":10.853,"15min":19.859999999999999},"last":0.78200000000000003}]}]},{"osd":5,"up_from":37,"seq":158913790359,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1393848,"kb_used_data":494832,"kb_used_omap":0,"kb_used_meta":898624,"kb_avail":92351304,"statfs":{"total":95995035648,"available":94567735296,"internally_reserved":0,"allocated":506707968,"data_stored":504934617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920190976},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:25 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":2.3809999999999998,"15min":2.3330000000000002},"min":{"1min":0.22900000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":6.2249999999999996,"5min":49.665999999999997,"15min":58.271999999999998},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.605,"5min":2.427,"15min":2.3809999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.2649999999999997,"5min":49.701999999999998,"15min":58.25},"last":0.55700000000000005}]},{"osd":1,"last update":"Mon Oct 9 11:07:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":2.101,"15min":2.032},"min":{"1min":0.26600000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.2199999999999998,"5min":32.491999999999997,"15min":32.491999999999997},"last":0.246},{"interface":"front","average":{"1min":1.7709999999999999,"5min":2.3100000000000001,"15min":1.8680000000000001},"min":{"1min":0.39300000000000002,"5min":0.247,"15min":0.247},"max":{"1min":5.2130000000000001,"5min":32.537999999999997,"15min":32.537999999999997},"last":0.42099999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:07:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.454,"15min":1.786},"min":{"1min":0.307,"5min":0.254,"15min":0.221},"max":{"1min":5.1269999999999998,"5min":6.3929999999999998,"15min":11.257999999999999},"last":0.78800000000000003},{"interface":"front","average":{"1min":2.1869999999999998,"5min":2.0129999999999999,"15min":1.9370000000000001},"min":{"1min":0.29099999999999998,"5min":0.253,"15min":0.253},"max":{"1min":8.0359999999999996,"5min":11.673,"15min":14.621},"last":3.0089999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:07:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4449999999999998,"5min":1.6439999999999999,"15min":1.5409999999999999},"min":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":15.535,"5min":15.535,"15min":15.535},"last":0.35199999999999998},{"interface":"front","average":{"1min":2.3460000000000001,"5min":1.8560000000000001,"15min":1.8240000000000001},"min":{"1min":0.38700000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":8.8940000000000001,"5min":8.8940000000000001,"15min":8.8940000000000001},"last":0.40799999999999997}]},{"osd":4,"last update":"Mon Oct 9 11:07:30 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":1.206,"15min":0.98899999999999999},"min":{"1min":0.28000000000000003,"5min":0.25700000000000001,"15min":0.216},"max":{"1min":2.0099999999999998,"5min":12.042,"15min":12.042},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.70699999999999996,"5min":1.2150000000000001,"15min":1.165},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.5910000000000002,"5min":9.2490000000000006,"15min":9.2490000000000006},"last":0.38300000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:07:42 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.3300000000000001,"15min":1.2110000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27100000000000002},"max":{"1min":1.5669999999999999,"5min":9.1969999999999992,"15min":9.1969999999999992},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.65300000000000002,"5min":1.288,"15min":1.0629999999999999},"min":{"1min":0.33700000000000002,"5min":0.32800000000000001,"15min":0.29099999999999998},"max":{"1min":2.6059999999999999,"5min":11.983000000000001,"15min":11.983000000000001},"last":0.996}]},{"osd":7,"last update":"Mon Oct 9 11:07:06 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.421,"15min":1.3660000000000001},"min":{"1min":0.307,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.5070000000000001,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":0.439},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.4710000000000001,"15min":1.254},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30099999999999999},"max":{"1min":3.0409999999999999,"5min":11.726000000000001,"15min":11.726000000000001},"last":0.49299999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3733,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":631373824,"data_stored":631373824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":838467584,"data_stored":838467584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":680267776,"data_stored":680263704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":842792960,"data_stored":842792960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":445583360,"data_stored":445579288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":499187712,"data_stored":499187712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1032720384,"data_stored":1032716312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1076822016,"data_stored":1076822016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:08:01.815 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T11:08:01.890 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:08:01.891 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:08:02.025 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:02.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:02.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:02.139 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:02.186 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:02.210 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:02 smithi155 ceph-mon[184854]: 1.11 scrub starts 2023-10-09T11:08:02.218 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:02.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:02.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:02.292 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:02 smithi125 ceph-mon[200197]: 1.11 scrub starts 2023-10-09T11:08:02.293 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:02 smithi125 ceph-mon[195507]: 1.11 scrub starts 2023-10-09T11:08:02.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:02.327 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T11:08:02.426 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:02.459 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:02.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:02.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:02.508 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:02.565 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:02.600 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:02.633 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:02.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:02.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:02.656 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:08:02.699 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:08:02.740 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:02.759 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:08:02.762 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:08:02.792 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:02.795 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:08:02.821 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:02.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:02.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:02.855 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:08:02.892 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:08:02.929 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:02.929 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:08:02.978 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:02.981 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:08:03.012 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:08:03.012 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:08:03.012 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:08:03.012 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:08:03.012 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:08:03.012 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:08:03.013 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:08:03.013 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:08:03.013 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:03.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:03.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:03.120 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:03.165 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:03.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:03 smithi125 ceph-mon[195507]: 11.7 deep-scrub starts 2023-10-09T11:08:03.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:03 smithi125 ceph-mon[195507]: pgmap v1362: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 6040/74286 objects degraded (8.131%); 2851/74286 objects misplaced (3.838%); 0 B/s, 79 objects/s recovering 2023-10-09T11:08:03.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3938988915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:03.281 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:03 smithi125 ceph-mon[200197]: 11.7 deep-scrub starts 2023-10-09T11:08:03.281 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:03 smithi125 ceph-mon[200197]: pgmap v1362: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 6040/74286 objects degraded (8.131%); 2851/74286 objects misplaced (3.838%); 0 B/s, 79 objects/s recovering 2023-10-09T11:08:03.281 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3938988915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:03.310 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:03.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:03.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:03.417 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:03 smithi155 ceph-mon[184854]: 11.7 deep-scrub starts 2023-10-09T11:08:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:03 smithi155 ceph-mon[184854]: pgmap v1362: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 6040/74286 objects degraded (8.131%); 2851/74286 objects misplaced (3.838%); 0 B/s, 79 objects/s recovering 2023-10-09T11:08:03.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3938988915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:03.486 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:03.664 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:03.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:03.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:03.769 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:03.830 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:03.916 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:03.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:03.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:04.030 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:08:04.030 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:08:04.030 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.030 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36510.0:178955 11.11 e557/509 11:8d88e77b:::benchmark_data_smithi125_439543_object79456:head v 557'7355, mlcod=557'7353)", 2023-10-09T11:08:04.030 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:08:04.028750+0000", 2023-10-09T11:08:04.030 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.00037121499999999999, 2023-10-09T11:08:04.030 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00039845599999999999, 2023-10-09T11:08:04.030 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:04.031 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T11:08:04.031 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:04.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:04.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028750+0000", 2023-10-09T11:08:04.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:04.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:04.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:04.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028750+0000", 2023-10-09T11:08:04.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:04.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:04.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:04.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028749+0000", 2023-10-09T11:08:04.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:08:04.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:04.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:04.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028759+0000", 2023-10-09T11:08:04.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0451e-05 2023-10-09T11:08:04.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:04.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:04.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028774+0000", 2023-10-09T11:08:04.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4121000000000001e-05 2023-10-09T11:08:04.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:04.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:04.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028780+0000", 2023-10-09T11:08:04.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7229999999999998e-06 2023-10-09T11:08:04.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:04.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:04.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028829+0000", 2023-10-09T11:08:04.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8334999999999997e-05 2023-10-09T11:08:04.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:04.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:04.034 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028851+0000", 2023-10-09T11:08:04.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2197000000000001e-05 2023-10-09T11:08:04.034 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:04.034 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:04.034 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:04.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:04.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.034 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.36510.0:178956 11.21 e557/531 11:87bfdd69:::benchmark_data_smithi125_439543_object79457:head v 557'6128, mlcod=557'6125)", 2023-10-09T11:08:04.034 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:08:04.028795+0000", 2023-10-09T11:08:04.035 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.00032649300000000002, 2023-10-09T11:08:04.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00042123099999999999, 2023-10-09T11:08:04.035 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:04.035 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T11:08:04.035 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:04.035 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.035 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:04.035 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028795+0000", 2023-10-09T11:08:04.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:04.035 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:04.035 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.035 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:04.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028795+0000", 2023-10-09T11:08:04.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:04.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:04.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:04.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028794+0000", 2023-10-09T11:08:04.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:04.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:04.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:04.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028799+0000", 2023-10-09T11:08:04.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3270000000000002e-06 2023-10-09T11:08:04.037 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:04.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.037 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:04.037 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028805+0000", 2023-10-09T11:08:04.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.567e-06 2023-10-09T11:08:04.037 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:04.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.037 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:04.037 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028810+0000", 2023-10-09T11:08:04.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3030000000000001e-06 2023-10-09T11:08:04.037 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:04.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:04.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028868+0000", 2023-10-09T11:08:04.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8430000000000001e-05 2023-10-09T11:08:04.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:04.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:04.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:04.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:04.028888+0000", 2023-10-09T11:08:04.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9970999999999999e-05 2023-10-09T11:08:04.038 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:04.038 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:04.039 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:04.039 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:04.039 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T11:08:04.039 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 2 2023-10-09T11:08:04.039 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:08:04.039 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:04.058 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:08:04.091 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:04.117 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:04.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:04.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:04.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:04.306 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:04.334 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:04 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 6040/74286 objects degraded (8.131%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:04.357 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:04.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:04.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:04.363 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:04 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 6040/74286 objects degraded (8.131%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:04.364 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:04 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 6040/74286 objects degraded (8.131%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:04.399 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T11:08:04.463 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:04.504 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:04.555 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:04.559 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:04.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:04.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:04.624 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:08:04.624 INFO:teuthology.orchestra.run.smithi125.stdout:{"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 6040/63750 objects degraded (9.475%), 3 pgs degraded","count":3},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1670,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":557,"num_osds":8,"num_up_osds":8,"osd_up_since":1696849454,"num_in_osds":8,"osd_in_since":1696849519,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":130},{"state_name":"active+remapped+backfill_wait","count":5},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":3},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":139,"num_pools":5,"num_objects":21250,"data_bytes":1391567740,"bytes_used":12076634112,"bytes_avail":755883651072,"bytes_total":767960285184,"degraded_objects":6040,"degraded_total":63750,"degraded_ratio":0.094745098039215686,"misplaced_objects":2287,"misplaced_total":63750,"misplaced_ratio":0.035874509803921567,"recovering_objects_per_sec":85,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":858,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":997},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi125.satgiw","status":"up:active","gid":24575}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T11:05:13.785232+0000","services":{}},"progress_events":{"585851ac-6e70-4462-9176-ac50987fab77":{"message":"Upgrade to 18.2.0-598-gb4fa663f (3m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true},"6753946c-bfe8-4c23-ba05-7c088ea1f347":{"message":"Global Recovery Event (2m)\n [==========================..] (remaining: 9s)","progress":0.93525177240371704,"add_to_ceph_s":true}}} 2023-10-09T11:08:04.662 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:04.727 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:04.767 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:04.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:04.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:04.819 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:08:04.872 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:04.914 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:05.003 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:05.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:05.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:05.098 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T11:08:05.098 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:08:05.107 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:05.223 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:05.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:05 smithi125 ceph-mon[195507]: 11.17 scrub starts 2023-10-09T11:08:05.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:05 smithi125 ceph-mon[195507]: pgmap v1363: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 6040/63750 objects degraded (9.475%); 2287/63750 objects misplaced (3.587%); 0 B/s, 85 objects/s recovering 2023-10-09T11:08:05.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:05 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1699982704' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T11:08:05.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:05 smithi125 ceph-mon[200197]: 11.17 scrub starts 2023-10-09T11:08:05.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:05 smithi125 ceph-mon[200197]: pgmap v1363: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 6040/63750 objects degraded (9.475%); 2287/63750 objects misplaced (3.587%); 0 B/s, 85 objects/s recovering 2023-10-09T11:08:05.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:05 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1699982704' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T11:08:05.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:05 smithi155 ceph-mon[184854]: 11.17 scrub starts 2023-10-09T11:08:05.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:05 smithi155 ceph-mon[184854]: pgmap v1363: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 6040/63750 objects degraded (9.475%); 2287/63750 objects misplaced (3.587%); 0 B/s, 85 objects/s recovering 2023-10-09T11:08:05.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:05 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1699982704' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T11:08:05.504 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:08:05.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:05.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:05.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:05.661 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:05.674 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:05.819 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:05.950 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:08:05.950 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:08:05.950 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T11:08:05.950 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T11:08:05.951 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:08:06.270 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:06 smithi155 ceph-mon[184854]: 1.11 scrub starts 2023-10-09T11:08:06.351 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T11:08:06.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:06 smithi125 ceph-mon[195507]: 1.11 scrub starts 2023-10-09T11:08:06.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:06 smithi125 ceph-mon[200197]: 1.11 scrub starts 2023-10-09T11:08:06.752 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:08:06.900 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:06.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:06.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:07.004 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:07.034 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:07.210 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:07 smithi125 ceph-mon[195507]: 11.7 scrub starts 2023-10-09T11:08:07.424 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:07.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:07.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:07 smithi155 ceph-mon[184854]: 11.7 scrub starts 2023-10-09T11:08:07.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:07 smithi155 ceph-mon[184854]: pgmap v1364: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 6040/63750 objects degraded (9.475%); 2287/63750 objects misplaced (3.587%); 0 B/s, 85 objects/s recovering 2023-10-09T11:08:07.528 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:07.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:07 smithi125 ceph-mon[195507]: pgmap v1364: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 6040/63750 objects degraded (9.475%); 2287/63750 objects misplaced (3.587%); 0 B/s, 85 objects/s recovering 2023-10-09T11:08:07.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:07 smithi125 ceph-mon[200197]: 11.7 scrub starts 2023-10-09T11:08:07.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:07 smithi125 ceph-mon[200197]: pgmap v1364: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 6040/63750 objects degraded (9.475%); 2287/63750 objects misplaced (3.587%); 0 B/s, 85 objects/s recovering 2023-10-09T11:08:07.584 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:07.710 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:07.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:07.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:07.815 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:07.846 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:07.869 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:07.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:07.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:07.896 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:08:07.896 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T11:08:07.896 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T11:08:07.896 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:08:07.896 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.896 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:15613 10.a 10:5bf56187:::benchmark_data_smithi125_402869_object15612:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T11:08:07.896 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:24.488022+0000", 2023-10-09T11:08:07.897 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 403.40523074499998, 2023-10-09T11:08:07.897 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10172471299999999, 2023-10-09T11:08:07.897 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:07.897 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:07.897 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:07.897 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:07.897 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:07.897 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 15613 2023-10-09T11:08:07.897 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.897 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:07.897 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:07.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488022+0000", 2023-10-09T11:08:07.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.898 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.898 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:07.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488022+0000", 2023-10-09T11:08:07.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.898 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.898 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:07.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488022+0000", 2023-10-09T11:08:07.899 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:07.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.899 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.899 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:07.899 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488055+0000", 2023-10-09T11:08:07.899 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3102999999999998e-05 2023-10-09T11:08:07.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.899 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.899 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:07.899 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488065+0000", 2023-10-09T11:08:07.899 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0101e-05 2023-10-09T11:08:07.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.900 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.900 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:07.900 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488070+0000", 2023-10-09T11:08:07.900 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2909999999999998e-06 2023-10-09T11:08:07.900 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.900 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.900 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:07.900 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488112+0000", 2023-10-09T11:08:07.900 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1662000000000002e-05 2023-10-09T11:08:07.900 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.900 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.900 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:07.901 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488280+0000", 2023-10-09T11:08:07.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016848699999999999 2023-10-09T11:08:07.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.901 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.901 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:08:07.901 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.488418+0000", 2023-10-09T11:08:07.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013803700000000001 2023-10-09T11:08:07.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.901 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.901 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:07.901 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.491088+0000", 2023-10-09T11:08:07.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026694919999999999 2023-10-09T11:08:07.902 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.902 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.491804+0000", 2023-10-09T11:08:07.902 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00071589899999999998 2023-10-09T11:08:07.902 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.902 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.589653+0000", 2023-10-09T11:08:07.902 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.097848911999999996 2023-10-09T11:08:07.902 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:07.903 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.589707+0000", 2023-10-09T11:08:07.903 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4593999999999999e-05 2023-10-09T11:08:07.903 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.903 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.903 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:07.903 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:24.589747+0000", 2023-10-09T11:08:07.903 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9787999999999999e-05 2023-10-09T11:08:07.903 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.903 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:07.903 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.903 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.903 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.904 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:27693 10.0 10:017bad4c:::benchmark_data_smithi125_402869_object27692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:08:07.904 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:34.474346+0000", 2023-10-09T11:08:07.904 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 393.41890728099997, 2023-10-09T11:08:07.904 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14285995000000001, 2023-10-09T11:08:07.904 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:07.904 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:07.904 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:07.904 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:07.904 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:07.904 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 27693 2023-10-09T11:08:07.904 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.904 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:07.904 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.905 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:07.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474346+0000", 2023-10-09T11:08:07.905 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.905 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.905 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.905 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:07.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474346+0000", 2023-10-09T11:08:07.905 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.905 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.905 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.905 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:07.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474345+0000", 2023-10-09T11:08:07.905 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:08:07.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.906 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.906 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:07.906 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474386+0000", 2023-10-09T11:08:07.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1455000000000002e-05 2023-10-09T11:08:07.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.906 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.906 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:07.906 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474401+0000", 2023-10-09T11:08:07.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5238e-05 2023-10-09T11:08:07.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.906 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.907 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:07.907 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474408+0000", 2023-10-09T11:08:07.907 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4489999999999996e-06 2023-10-09T11:08:07.907 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.907 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:07.907 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474455+0000", 2023-10-09T11:08:07.907 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7719999999999997e-05 2023-10-09T11:08:07.907 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.907 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:07.907 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474565+0000", 2023-10-09T11:08:07.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010948200000000001 2023-10-09T11:08:07.908 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.908 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.908 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:08:07.908 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.474666+0000", 2023-10-09T11:08:07.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010145399999999999 2023-10-09T11:08:07.908 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.908 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.908 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.908 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.476207+0000", 2023-10-09T11:08:07.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015401950000000001 2023-10-09T11:08:07.908 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.908 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.909 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:07.909 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.476567+0000", 2023-10-09T11:08:07.909 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00036023899999999999 2023-10-09T11:08:07.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.909 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.909 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617122+0000", 2023-10-09T11:08:07.909 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14055489700000001 2023-10-09T11:08:07.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.909 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:07.909 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617168+0000", 2023-10-09T11:08:07.909 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6669000000000003e-05 2023-10-09T11:08:07.910 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.910 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:07.910 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617206+0000", 2023-10-09T11:08:07.910 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7505000000000001e-05 2023-10-09T11:08:07.910 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.910 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:07.910 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.910 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.910 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:27705 10.0 10:03463076:::benchmark_data_smithi125_402869_object27704:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:08:07.911 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:34.483244+0000", 2023-10-09T11:08:07.911 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 393.41000962599998, 2023-10-09T11:08:07.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13409575700000001, 2023-10-09T11:08:07.911 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:07.911 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:07.911 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:07.911 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:07.911 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:07.911 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 27705 2023-10-09T11:08:07.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.911 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:07.911 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:07.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483244+0000", 2023-10-09T11:08:07.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:07.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483244+0000", 2023-10-09T11:08:07.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:07.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483242+0000", 2023-10-09T11:08:07.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:08:07.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:07.913 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483306+0000", 2023-10-09T11:08:07.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3712000000000003e-05 2023-10-09T11:08:07.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:07.913 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483319+0000", 2023-10-09T11:08:07.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2860999999999999e-05 2023-10-09T11:08:07.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:07.914 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483325+0000", 2023-10-09T11:08:07.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0889999999999996e-06 2023-10-09T11:08:07.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.914 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.914 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:07.914 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483360+0000", 2023-10-09T11:08:07.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5259999999999998e-05 2023-10-09T11:08:07.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.914 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.914 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:07.914 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483505+0000", 2023-10-09T11:08:07.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014444300000000001 2023-10-09T11:08:07.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:08:07.915 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.483657+0000", 2023-10-09T11:08:07.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151726 2023-10-09T11:08:07.915 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:07.915 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.484799+0000", 2023-10-09T11:08:07.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001142503 2023-10-09T11:08:07.915 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.485118+0000", 2023-10-09T11:08:07.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000318676 2023-10-09T11:08:07.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.916 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617247+0000", 2023-10-09T11:08:07.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.132128987 2023-10-09T11:08:07.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.916 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:07.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617299+0000", 2023-10-09T11:08:07.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1867000000000003e-05 2023-10-09T11:08:07.917 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.917 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:07.917 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.617339+0000", 2023-10-09T11:08:07.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0726999999999998e-05 2023-10-09T11:08:07.917 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.917 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:07.917 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.917 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.917 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:27738 10.0 10:0a0f1cf2:::benchmark_data_smithi125_402869_object27737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:08:07.917 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:34.503650+0000", 2023-10-09T11:08:07.917 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 393.38960292399997, 2023-10-09T11:08:07.918 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.115650268, 2023-10-09T11:08:07.918 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:07.918 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:07.918 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:07.918 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:07.918 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:07.918 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 27738 2023-10-09T11:08:07.918 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.918 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:07.918 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.918 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:07.918 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503650+0000", 2023-10-09T11:08:07.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.919 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:07.919 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503650+0000", 2023-10-09T11:08:07.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.919 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:07.919 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503649+0000", 2023-10-09T11:08:07.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:08:07.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:07.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503716+0000", 2023-10-09T11:08:07.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7002999999999995e-05 2023-10-09T11:08:07.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:07.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503726+0000", 2023-10-09T11:08:07.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9170000000000001e-06 2023-10-09T11:08:07.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:07.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503732+0000", 2023-10-09T11:08:07.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2009999999999996e-06 2023-10-09T11:08:07.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:07.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503776+0000", 2023-10-09T11:08:07.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4100000000000001e-05 2023-10-09T11:08:07.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:07.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.503892+0000", 2023-10-09T11:08:07.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115704 2023-10-09T11:08:07.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:08:07.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.504031+0000", 2023-10-09T11:08:07.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013921300000000001 2023-10-09T11:08:07.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:07.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.505195+0000", 2023-10-09T11:08:07.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001163765 2023-10-09T11:08:07.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.505623+0000", 2023-10-09T11:08:07.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00042792200000000001 2023-10-09T11:08:07.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.619243+0000", 2023-10-09T11:08:07.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11362040900000001 2023-10-09T11:08:07.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:07.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.619272+0000", 2023-10-09T11:08:07.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8118000000000001e-05 2023-10-09T11:08:07.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:07.924 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:34.619301+0000", 2023-10-09T11:08:07.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8958e-05 2023-10-09T11:08:07.924 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.924 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:07.924 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.924 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:34496 10.e 10:7e8e0fe7:::benchmark_data_smithi125_402869_object34495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:08:07.924 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:39.324998+0000", 2023-10-09T11:08:07.925 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 388.56825483, 2023-10-09T11:08:07.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.098875269000000002, 2023-10-09T11:08:07.925 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:07.925 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:07.925 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:07.925 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:07.925 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:07.925 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 34496 2023-10-09T11:08:07.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.925 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:07.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:07.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.324998+0000", 2023-10-09T11:08:07.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:07.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.324998+0000", 2023-10-09T11:08:07.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:07.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.324998+0000", 2023-10-09T11:08:07.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:07.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:07.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325045+0000", 2023-10-09T11:08:07.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6817000000000003e-05 2023-10-09T11:08:07.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:07.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325053+0000", 2023-10-09T11:08:07.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8709999999999997e-06 2023-10-09T11:08:07.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:07.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325057+0000", 2023-10-09T11:08:07.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6120000000000001e-06 2023-10-09T11:08:07.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:07.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325108+0000", 2023-10-09T11:08:07.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1187000000000003e-05 2023-10-09T11:08:07.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:07.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325227+0000", 2023-10-09T11:08:07.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118689 2023-10-09T11:08:07.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,4", 2023-10-09T11:08:07.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325363+0000", 2023-10-09T11:08:07.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000135735 2023-10-09T11:08:07.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:07.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327640+0000", 2023-10-09T11:08:07.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022776680000000001 2023-10-09T11:08:07.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327910+0000", 2023-10-09T11:08:07.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00026939599999999998 2023-10-09T11:08:07.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.423777+0000", 2023-10-09T11:08:07.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.095867252999999999 2023-10-09T11:08:07.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:07.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.423833+0000", 2023-10-09T11:08:07.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5897999999999999e-05 2023-10-09T11:08:07.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:07.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.423874+0000", 2023-10-09T11:08:07.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0713000000000001e-05 2023-10-09T11:08:07.931 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.931 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:07.931 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.932 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:34497 10.a 10:5b894bd9:::benchmark_data_smithi125_402869_object34496:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:08:07.932 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:39.325075+0000", 2023-10-09T11:08:07.932 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 388.56817819399998, 2023-10-09T11:08:07.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.099410517000000004, 2023-10-09T11:08:07.932 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:07.932 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:07.932 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:07.932 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:07.932 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:07.932 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 34497 2023-10-09T11:08:07.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.932 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:07.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:07.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325075+0000", 2023-10-09T11:08:07.933 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:07.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325075+0000", 2023-10-09T11:08:07.933 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:07.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325074+0000", 2023-10-09T11:08:07.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:07.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:07.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325125+0000", 2023-10-09T11:08:07.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0782999999999998e-05 2023-10-09T11:08:07.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:07.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325144+0000", 2023-10-09T11:08:07.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9225e-05 2023-10-09T11:08:07.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:07.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325149+0000", 2023-10-09T11:08:07.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4429999999999996e-06 2023-10-09T11:08:07.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:07.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325342+0000", 2023-10-09T11:08:07.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00019310200000000001 2023-10-09T11:08:07.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:07.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325474+0000", 2023-10-09T11:08:07.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013155999999999999 2023-10-09T11:08:07.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:08:07.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325610+0000", 2023-10-09T11:08:07.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000136954 2023-10-09T11:08:07.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:07.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327774+0000", 2023-10-09T11:08:07.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021631739999999999 2023-10-09T11:08:07.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327960+0000", 2023-10-09T11:08:07.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018653499999999999 2023-10-09T11:08:07.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424397+0000", 2023-10-09T11:08:07.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.096436500999999994 2023-10-09T11:08:07.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:07.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424439+0000", 2023-10-09T11:08:07.938 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2216999999999999e-05 2023-10-09T11:08:07.938 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.938 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:07.938 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424486+0000", 2023-10-09T11:08:07.938 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6650000000000002e-05 2023-10-09T11:08:07.938 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.938 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:07.938 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.938 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.939 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:34498 10.0 10:05649a4c:::benchmark_data_smithi125_402869_object34497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:08:07.939 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:39.325163+0000", 2023-10-09T11:08:07.939 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 388.56809045599999, 2023-10-09T11:08:07.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.100486519, 2023-10-09T11:08:07.939 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:07.939 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:07.939 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:07.939 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:07.939 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:07.939 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 34498 2023-10-09T11:08:07.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.939 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:07.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:07.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325163+0000", 2023-10-09T11:08:07.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:07.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325163+0000", 2023-10-09T11:08:07.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:07.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325162+0000", 2023-10-09T11:08:07.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:07.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.941 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:07.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325219+0000", 2023-10-09T11:08:07.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6532e-05 2023-10-09T11:08:07.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.941 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:07.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325227+0000", 2023-10-09T11:08:07.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8269999999999998e-06 2023-10-09T11:08:07.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.941 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:07.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325233+0000", 2023-10-09T11:08:07.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1909999999999997e-06 2023-10-09T11:08:07.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:07.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.326614+0000", 2023-10-09T11:08:07.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00138123 2023-10-09T11:08:07.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:07.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.326735+0000", 2023-10-09T11:08:07.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012105800000000001 2023-10-09T11:08:07.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:08:07.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.326862+0000", 2023-10-09T11:08:07.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000126699 2023-10-09T11:08:07.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.328648+0000", 2023-10-09T11:08:07.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0017860949999999999 2023-10-09T11:08:07.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:07.944 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.329261+0000", 2023-10-09T11:08:07.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00061273000000000002 2023-10-09T11:08:07.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.944 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.944 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.330863+0000", 2023-10-09T11:08:07.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001602612 2023-10-09T11:08:07.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.944 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:07.944 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.330908+0000", 2023-10-09T11:08:07.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4719000000000003e-05 2023-10-09T11:08:07.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.945 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.945 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:07.945 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.425649+0000", 2023-10-09T11:08:07.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.094741362999999995 2023-10-09T11:08:07.945 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.945 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:07.945 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.945 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.945 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.945 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:34499 10.a 10:5b46ec36:::benchmark_data_smithi125_402869_object34498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T11:08:07.945 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:39.325241+0000", 2023-10-09T11:08:07.945 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 388.56801250299998, 2023-10-09T11:08:07.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.099333877000000001, 2023-10-09T11:08:07.946 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:07.946 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:07.946 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:07.946 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:07.946 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:07.946 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 34499 2023-10-09T11:08:07.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.946 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:07.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:07.946 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325241+0000", 2023-10-09T11:08:07.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:07.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325241+0000", 2023-10-09T11:08:07.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:07.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325240+0000", 2023-10-09T11:08:07.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:07.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:07.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325273+0000", 2023-10-09T11:08:07.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3108000000000002e-05 2023-10-09T11:08:07.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:07.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325281+0000", 2023-10-09T11:08:07.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2400000000000001e-06 2023-10-09T11:08:07.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:07.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.325285+0000", 2023-10-09T11:08:07.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.408e-06 2023-10-09T11:08:07.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:07.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.326873+0000", 2023-10-09T11:08:07.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015876180000000001 2023-10-09T11:08:07.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:07.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327003+0000", 2023-10-09T11:08:07.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013045099999999999 2023-10-09T11:08:07.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:08:07.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.327094+0000", 2023-10-09T11:08:07.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1046999999999998e-05 2023-10-09T11:08:07.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:07.950 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.329164+0000", 2023-10-09T11:08:07.950 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020697559999999999 2023-10-09T11:08:07.950 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.950 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424370+0000", 2023-10-09T11:08:07.950 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.095206683 2023-10-09T11:08:07.950 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.951 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424514+0000", 2023-10-09T11:08:07.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000143547 2023-10-09T11:08:07.951 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.951 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.951 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:07.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424547+0000", 2023-10-09T11:08:07.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3194000000000001e-05 2023-10-09T11:08:07.951 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.951 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.951 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:07.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:39.424575+0000", 2023-10-09T11:08:07.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7362999999999999e-05 2023-10-09T11:08:07.951 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.952 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:07.952 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.952 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.952 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.952 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:77753 10.f 10:f688b427:::benchmark_data_smithi125_402869_object77752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-10-09T11:08:07.952 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:02:21.288500+0000", 2023-10-09T11:08:07.952 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 346.60475290400001, 2023-10-09T11:08:07.952 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.111923807, 2023-10-09T11:08:07.952 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:07.952 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:07.952 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:07.952 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:07.953 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:07.953 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 77753 2023-10-09T11:08:07.953 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.953 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:07.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.953 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:07.953 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288500+0000", 2023-10-09T11:08:07.953 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.953 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.953 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:07.953 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288500+0000", 2023-10-09T11:08:07.953 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.953 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.954 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.954 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:07.954 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288500+0000", 2023-10-09T11:08:07.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:07.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.954 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.954 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:07.954 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288535+0000", 2023-10-09T11:08:07.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5206999999999997e-05 2023-10-09T11:08:07.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.954 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:07.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288544+0000", 2023-10-09T11:08:07.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5019999999999998e-06 2023-10-09T11:08:07.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:07.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288550+0000", 2023-10-09T11:08:07.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4079999999999997e-06 2023-10-09T11:08:07.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:07.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288589+0000", 2023-10-09T11:08:07.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9366000000000002e-05 2023-10-09T11:08:07.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:07.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288708+0000", 2023-10-09T11:08:07.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118998 2023-10-09T11:08:07.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T11:08:07.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.288884+0000", 2023-10-09T11:08:07.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017539000000000001 2023-10-09T11:08:07.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:07.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.292679+0000", 2023-10-09T11:08:07.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003795138 2023-10-09T11:08:07.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.296941+0000", 2023-10-09T11:08:07.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042627710000000003 2023-10-09T11:08:07.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.400350+0000", 2023-10-09T11:08:07.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.103408823 2023-10-09T11:08:07.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:07.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.400392+0000", 2023-10-09T11:08:07.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1792000000000002e-05 2023-10-09T11:08:07.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:07.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:02:21.400424+0000", 2023-10-09T11:08:07.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2053e-05 2023-10-09T11:08:07.958 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.958 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:07.959 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.959 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.959 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.959 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:158832 10.a 10:5433f58d:::benchmark_data_smithi125_402869_object60263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2023-10-09T11:08:07.959 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:03:55.841518+0000", 2023-10-09T11:08:07.959 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 252.05173494300001, 2023-10-09T11:08:07.959 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.116037265, 2023-10-09T11:08:07.959 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:07.959 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:07.959 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:07.959 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:07.959 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:07.960 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 158832 2023-10-09T11:08:07.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.960 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:07.960 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.960 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:07.960 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841518+0000", 2023-10-09T11:08:07.960 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.960 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.960 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:07.960 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841518+0000", 2023-10-09T11:08:07.960 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.961 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:07.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841518+0000", 2023-10-09T11:08:07.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:07.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.961 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:07.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841521+0000", 2023-10-09T11:08:07.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7020000000000001e-06 2023-10-09T11:08:07.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.961 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:07.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841527+0000", 2023-10-09T11:08:07.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1310000000000003e-06 2023-10-09T11:08:07.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:07.962 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841531+0000", 2023-10-09T11:08:07.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1049999999999997e-06 2023-10-09T11:08:07.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:07.962 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841568+0000", 2023-10-09T11:08:07.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7651000000000003e-05 2023-10-09T11:08:07.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:07.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841681+0000", 2023-10-09T11:08:07.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011254100000000001 2023-10-09T11:08:07.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:08:07.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.841795+0000", 2023-10-09T11:08:07.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000114193 2023-10-09T11:08:07.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.845131+0000", 2023-10-09T11:08:07.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003335569 2023-10-09T11:08:07.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:07.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.845137+0000", 2023-10-09T11:08:07.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9900000000000002e-06 2023-10-09T11:08:07.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.957443+0000", 2023-10-09T11:08:07.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.112306507 2023-10-09T11:08:07.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:07.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.957511+0000", 2023-10-09T11:08:07.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7582000000000004e-05 2023-10-09T11:08:07.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:07.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:03:55.957556+0000", 2023-10-09T11:08:07.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4696e-05 2023-10-09T11:08:07.965 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.965 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:07.965 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.966 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.966 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:180897 10.f 10:f1fc3a12:::benchmark_data_smithi125_402869_object82328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T11:08:07.966 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:24.612340+0000", 2023-10-09T11:08:07.966 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 223.28091327999999, 2023-10-09T11:08:07.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.107943447, 2023-10-09T11:08:07.966 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:07.966 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:07.966 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:07.966 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:07.966 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:07.967 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 180897 2023-10-09T11:08:07.967 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.967 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:07.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.967 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:07.967 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612340+0000", 2023-10-09T11:08:07.967 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.967 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.967 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:07.967 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612340+0000", 2023-10-09T11:08:07.967 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.968 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.968 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.968 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:07.968 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612339+0000", 2023-10-09T11:08:07.968 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:07.968 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.968 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.968 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:07.968 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612343+0000", 2023-10-09T11:08:07.968 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3069999999999998e-06 2023-10-09T11:08:07.968 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.968 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.968 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:07.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612353+0000", 2023-10-09T11:08:07.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9769999999999995e-06 2023-10-09T11:08:07.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.969 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:07.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612357+0000", 2023-10-09T11:08:07.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5109999999999997e-06 2023-10-09T11:08:07.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.969 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:07.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612420+0000", 2023-10-09T11:08:07.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2391999999999995e-05 2023-10-09T11:08:07.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:07.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612537+0000", 2023-10-09T11:08:07.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000117409 2023-10-09T11:08:07.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T11:08:07.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.612654+0000", 2023-10-09T11:08:07.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000117248 2023-10-09T11:08:07.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:07.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.621223+0000", 2023-10-09T11:08:07.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0085686159999999994 2023-10-09T11:08:07.971 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626399+0000", 2023-10-09T11:08:07.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051757670000000004 2023-10-09T11:08:07.971 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626899+0000", 2023-10-09T11:08:07.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00049993800000000001 2023-10-09T11:08:07.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.972 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.972 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.719838+0000", 2023-10-09T11:08:07.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.092939021999999996 2023-10-09T11:08:07.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.972 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:07.972 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.719924+0000", 2023-10-09T11:08:07.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6328000000000004e-05 2023-10-09T11:08:07.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:07.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.720283+0000", 2023-10-09T11:08:07.973 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00035945900000000002 2023-10-09T11:08:07.973 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.973 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:07.973 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.973 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.973 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:180902 10.f 10:fa4f508c:::benchmark_data_smithi125_402869_object82333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T11:08:07.973 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:24.626845+0000", 2023-10-09T11:08:07.973 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 223.266408203, 2023-10-09T11:08:07.973 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10240413199999999, 2023-10-09T11:08:07.974 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:07.974 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:07.974 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:07.974 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:07.974 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:07.974 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 180902 2023-10-09T11:08:07.974 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.974 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:07.974 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.974 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:07.974 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626845+0000", 2023-10-09T11:08:07.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.975 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.975 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.975 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:07.975 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626845+0000", 2023-10-09T11:08:07.975 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.975 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.975 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.975 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:07.975 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626844+0000", 2023-10-09T11:08:07.975 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:08:07.975 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.975 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:07.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626848+0000", 2023-10-09T11:08:07.976 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.498e-06 2023-10-09T11:08:07.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:07.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626858+0000", 2023-10-09T11:08:07.976 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2920000000000001e-06 2023-10-09T11:08:07.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:07.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626861+0000", 2023-10-09T11:08:07.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.782e-06 2023-10-09T11:08:07.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:07.977 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.626915+0000", 2023-10-09T11:08:07.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3905e-05 2023-10-09T11:08:07.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:07.977 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.627044+0000", 2023-10-09T11:08:07.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012853699999999999 2023-10-09T11:08:07.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T11:08:07.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.627159+0000", 2023-10-09T11:08:07.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000114894 2023-10-09T11:08:07.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:07.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.630494+0000", 2023-10-09T11:08:07.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003335176 2023-10-09T11:08:07.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.979 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.632097+0000", 2023-10-09T11:08:07.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001602989 2023-10-09T11:08:07.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.979 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.979 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.635044+0000", 2023-10-09T11:08:07.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002947098 2023-10-09T11:08:07.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.979 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.979 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.728793+0000", 2023-10-09T11:08:07.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.093748897999999997 2023-10-09T11:08:07.980 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.980 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.980 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:07.980 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.728842+0000", 2023-10-09T11:08:07.980 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9327999999999999e-05 2023-10-09T11:08:07.980 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.980 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.980 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:07.980 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:24.729249+0000", 2023-10-09T11:08:07.980 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00040702300000000001 2023-10-09T11:08:07.980 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.980 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:07.980 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.981 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.981 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.981 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183807 10.0 10:04af7371:::benchmark_data_smithi125_402869_object85238:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:08:07.981 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.483591+0000", 2023-10-09T11:08:07.981 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 219.409662459, 2023-10-09T11:08:07.981 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.114706303, 2023-10-09T11:08:07.981 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:07.981 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:07.981 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:07.981 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:07.981 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:07.981 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183807 2023-10-09T11:08:07.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.982 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:07.982 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.982 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:07.982 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483591+0000", 2023-10-09T11:08:07.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.982 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.982 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:07.982 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483591+0000", 2023-10-09T11:08:07.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:07.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483590+0000", 2023-10-09T11:08:07.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:08:07.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:07.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483595+0000", 2023-10-09T11:08:07.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8319999999999996e-06 2023-10-09T11:08:07.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:07.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483605+0000", 2023-10-09T11:08:07.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0108999999999999e-05 2023-10-09T11:08:07.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:07.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483610+0000", 2023-10-09T11:08:07.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7469999999999996e-06 2023-10-09T11:08:07.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:07.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483651+0000", 2023-10-09T11:08:07.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1591000000000002e-05 2023-10-09T11:08:07.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.985 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:07.985 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483743+0000", 2023-10-09T11:08:07.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1634999999999999e-05 2023-10-09T11:08:07.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.985 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:08:07.985 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.484167+0000", 2023-10-09T11:08:07.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000424702 2023-10-09T11:08:07.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.985 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:07.985 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.493854+0000", 2023-10-09T11:08:07.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0096867949999999998 2023-10-09T11:08:07.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495932+0000", 2023-10-09T11:08:07.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020782270000000002 2023-10-09T11:08:07.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598222+0000", 2023-10-09T11:08:07.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.102289846 2023-10-09T11:08:07.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:07.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598267+0000", 2023-10-09T11:08:07.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4545999999999997e-05 2023-10-09T11:08:07.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:07.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598297+0000", 2023-10-09T11:08:07.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0187000000000001e-05 2023-10-09T11:08:07.987 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.987 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:07.987 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.988 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.988 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183811 10.10 10:0ca62c6a:::benchmark_data_smithi125_402869_object85242:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:08:07.988 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.483618+0000", 2023-10-09T11:08:07.988 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 219.40963533999999, 2023-10-09T11:08:07.988 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11462522, 2023-10-09T11:08:07.988 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:07.988 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:07.988 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:07.988 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:07.988 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:07.988 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183811 2023-10-09T11:08:07.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.989 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:07.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:07.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483618+0000", 2023-10-09T11:08:07.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:07.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483618+0000", 2023-10-09T11:08:07.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:07.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483617+0000", 2023-10-09T11:08:07.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:07.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:07.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483620+0000", 2023-10-09T11:08:07.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6240000000000002e-06 2023-10-09T11:08:07.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:07.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483632+0000", 2023-10-09T11:08:07.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1652e-05 2023-10-09T11:08:07.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:07.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483636+0000", 2023-10-09T11:08:07.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1330000000000001e-06 2023-10-09T11:08:07.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:07.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483686+0000", 2023-10-09T11:08:07.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0438000000000001e-05 2023-10-09T11:08:07.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:07.992 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483793+0000", 2023-10-09T11:08:07.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106499 2023-10-09T11:08:07.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:08:07.992 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.484240+0000", 2023-10-09T11:08:07.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00044728299999999998 2023-10-09T11:08:07.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:07.992 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.493846+0000", 2023-10-09T11:08:07.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0096058700000000007 2023-10-09T11:08:07.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495948+0000", 2023-10-09T11:08:07.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021024860000000002 2023-10-09T11:08:07.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:07.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598183+0000", 2023-10-09T11:08:07.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.102234346 2023-10-09T11:08:07.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:07.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598216+0000", 2023-10-09T11:08:07.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2889999999999999e-05 2023-10-09T11:08:07.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:07.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598243+0000", 2023-10-09T11:08:07.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7449000000000001e-05 2023-10-09T11:08:07.994 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.994 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:07.994 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:07.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.995 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.995 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183812 10.10 10:0e10b5d1:::benchmark_data_smithi125_402869_object85243:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:08:07.995 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.483656+0000", 2023-10-09T11:08:07.995 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 219.40959711100001, 2023-10-09T11:08:07.995 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.114659012, 2023-10-09T11:08:07.995 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:07.995 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:07.995 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:07.995 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:07.995 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:07.995 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183812 2023-10-09T11:08:07.995 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.995 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:07.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.996 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:07.996 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483656+0000", 2023-10-09T11:08:07.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.996 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:07.996 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483656+0000", 2023-10-09T11:08:07.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:07.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.996 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:07.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483656+0000", 2023-10-09T11:08:07.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:07.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:07.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483659+0000", 2023-10-09T11:08:07.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0730000000000001e-06 2023-10-09T11:08:07.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:07.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483666+0000", 2023-10-09T11:08:07.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0539999999999997e-06 2023-10-09T11:08:07.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:07.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483670+0000", 2023-10-09T11:08:07.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8670000000000004e-06 2023-10-09T11:08:07.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:07.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486323+0000", 2023-10-09T11:08:07.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026528950000000002 2023-10-09T11:08:07.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:07.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486433+0000", 2023-10-09T11:08:07.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000110812 2023-10-09T11:08:07.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:08:07.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486801+0000", 2023-10-09T11:08:07.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00036751500000000001 2023-10-09T11:08:07.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:07.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:07.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:07.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495912+0000", 2023-10-09T11:08:08.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0091110519999999997 2023-10-09T11:08:08.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.000 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:08.000 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495984+0000", 2023-10-09T11:08:08.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1724999999999995e-05 2023-10-09T11:08:08.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.000 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:08.000 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598264+0000", 2023-10-09T11:08:08.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.102280443 2023-10-09T11:08:08.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:08.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598287+0000", 2023-10-09T11:08:08.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3285000000000002e-05 2023-10-09T11:08:08.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:08.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598315+0000", 2023-10-09T11:08:08.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.777e-05 2023-10-09T11:08:08.001 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:08.001 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:08.001 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:08.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.002 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183813 10.10 10:0c717176:::benchmark_data_smithi125_402869_object85244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:08:08.002 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.483680+0000", 2023-10-09T11:08:08.002 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 219.409572991, 2023-10-09T11:08:08.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.114707768, 2023-10-09T11:08:08.002 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:08.002 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:08.002 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:08.002 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:08.002 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:08.002 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183813 2023-10-09T11:08:08.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.002 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:08.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:08.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483680+0000", 2023-10-09T11:08:08.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:08.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:08.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483680+0000", 2023-10-09T11:08:08.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:08.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:08.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483680+0000", 2023-10-09T11:08:08.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:08.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:08.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483682+0000", 2023-10-09T11:08:08.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6599999999999999e-06 2023-10-09T11:08:08.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:08.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483688+0000", 2023-10-09T11:08:08.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6540000000000004e-06 2023-10-09T11:08:08.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:08.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.483692+0000", 2023-10-09T11:08:08.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.67e-06 2023-10-09T11:08:08.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:08.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.489061+0000", 2023-10-09T11:08:08.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0053695840000000002 2023-10-09T11:08:08.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:08.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.489172+0000", 2023-10-09T11:08:08.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011078000000000001 2023-10-09T11:08:08.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:08:08.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.489480+0000", 2023-10-09T11:08:08.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000307855 2023-10-09T11:08:08.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:08.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495919+0000", 2023-10-09T11:08:08.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064387530000000002 2023-10-09T11:08:08.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.007 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:08.007 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.496010+0000", 2023-10-09T11:08:08.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1748000000000003e-05 2023-10-09T11:08:08.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.007 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:08.007 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598341+0000", 2023-10-09T11:08:08.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.102330948 2023-10-09T11:08:08.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:08.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598368+0000", 2023-10-09T11:08:08.008 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6183e-05 2023-10-09T11:08:08.008 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:08.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598388+0000", 2023-10-09T11:08:08.008 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0367999999999999e-05 2023-10-09T11:08:08.008 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:08.008 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:08.008 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:08.008 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.009 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183815 10.10 10:0dad7a83:::benchmark_data_smithi125_402869_object85246:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:08:08.009 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.486435+0000", 2023-10-09T11:08:08.009 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 219.40681852500001, 2023-10-09T11:08:08.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11203769, 2023-10-09T11:08:08.009 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:08.009 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:08.009 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:08.009 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:08.009 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:08.009 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183815 2023-10-09T11:08:08.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.009 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:08.009 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.010 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:08.010 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486435+0000", 2023-10-09T11:08:08.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:08.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.010 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:08.010 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486435+0000", 2023-10-09T11:08:08.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:08.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.010 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:08.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486433+0000", 2023-10-09T11:08:08.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:08:08.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:08.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486450+0000", 2023-10-09T11:08:08.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6589999999999999e-05 2023-10-09T11:08:08.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:08.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486463+0000", 2023-10-09T11:08:08.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3261000000000001e-05 2023-10-09T11:08:08.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:08.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.486475+0000", 2023-10-09T11:08:08.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2328e-05 2023-10-09T11:08:08.012 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:08.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.491357+0000", 2023-10-09T11:08:08.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.004881364 2023-10-09T11:08:08.012 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:08.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.491487+0000", 2023-10-09T11:08:08.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013008299999999999 2023-10-09T11:08:08.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:08:08.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.491763+0000", 2023-10-09T11:08:08.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00027596599999999999 2023-10-09T11:08:08.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:08.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.495954+0000", 2023-10-09T11:08:08.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041909010000000003 2023-10-09T11:08:08.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:08.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.497564+0000", 2023-10-09T11:08:08.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001610081 2023-10-09T11:08:08.014 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:08.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598414+0000", 2023-10-09T11:08:08.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10085058500000001 2023-10-09T11:08:08.014 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:08.015 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598445+0000", 2023-10-09T11:08:08.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.044e-05 2023-10-09T11:08:08.015 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.015 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.015 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:08.015 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598472+0000", 2023-10-09T11:08:08.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7593000000000002e-05 2023-10-09T11:08:08.015 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:08.015 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:08.015 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:08.015 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.015 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.015 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183821 10.10 10:0e04176b:::benchmark_data_smithi125_402869_object85252:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:08:08.016 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.492183+0000", 2023-10-09T11:08:08.016 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 219.401069882, 2023-10-09T11:08:08.016 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.106356412, 2023-10-09T11:08:08.016 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:08.016 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:08.016 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:08.016 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:08.016 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:08.016 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183821 2023-10-09T11:08:08.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.016 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:08.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:08.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492183+0000", 2023-10-09T11:08:08.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:08.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:08.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492183+0000", 2023-10-09T11:08:08.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:08.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:08.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492182+0000", 2023-10-09T11:08:08.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:08:08.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.018 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:08.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492189+0000", 2023-10-09T11:08:08.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6080000000000001e-06 2023-10-09T11:08:08.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.018 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:08.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492203+0000", 2023-10-09T11:08:08.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3824e-05 2023-10-09T11:08:08.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:08.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.492209+0000", 2023-10-09T11:08:08.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6769999999999998e-06 2023-10-09T11:08:08.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:08.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.493870+0000", 2023-10-09T11:08:08.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0016607099999999999 2023-10-09T11:08:08.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:08.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.493983+0000", 2023-10-09T11:08:08.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112582 2023-10-09T11:08:08.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.020 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:08:08.020 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.494235+0000", 2023-10-09T11:08:08.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025229000000000001 2023-10-09T11:08:08.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.020 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:08.020 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.496090+0000", 2023-10-09T11:08:08.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018548270000000001 2023-10-09T11:08:08.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:08.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.497689+0000", 2023-10-09T11:08:08.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015987530000000001 2023-10-09T11:08:08.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:08.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598498+0000", 2023-10-09T11:08:08.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10080939899999999 2023-10-09T11:08:08.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:08.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598522+0000", 2023-10-09T11:08:08.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3609999999999999e-05 2023-10-09T11:08:08.022 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.022 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.022 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:08.022 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598540+0000", 2023-10-09T11:08:08.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8224e-05 2023-10-09T11:08:08.022 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:08.022 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:08.022 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:08.022 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.022 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.022 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:183823 10.10 10:0c686624:::benchmark_data_smithi125_402869_object85254:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:08:08.023 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:28.500372+0000", 2023-10-09T11:08:08.023 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 219.39288122400001, 2023-10-09T11:08:08.023 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.098236665000000001, 2023-10-09T11:08:08.023 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:08.023 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:08.023 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:08.023 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:08.023 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:08.023 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 183823 2023-10-09T11:08:08.023 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.023 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:08.023 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:08.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500372+0000", 2023-10-09T11:08:08.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:08.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.024 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:08.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500372+0000", 2023-10-09T11:08:08.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:08.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.024 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:08.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500371+0000", 2023-10-09T11:08:08.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:08:08.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.025 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:08.025 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500377+0000", 2023-10-09T11:08:08.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6710000000000004e-06 2023-10-09T11:08:08.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.025 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:08.025 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500387+0000", 2023-10-09T11:08:08.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9329999999999996e-06 2023-10-09T11:08:08.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.026 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:08.026 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500391+0000", 2023-10-09T11:08:08.026 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.887e-06 2023-10-09T11:08:08.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.026 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:08.026 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500428+0000", 2023-10-09T11:08:08.026 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6810999999999997e-05 2023-10-09T11:08:08.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.026 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:08.027 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500522+0000", 2023-10-09T11:08:08.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3919000000000007e-05 2023-10-09T11:08:08.027 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.027 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.027 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:08:08.027 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.500870+0000", 2023-10-09T11:08:08.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000347762 2023-10-09T11:08:08.027 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.027 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.027 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:08.027 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.502778+0000", 2023-10-09T11:08:08.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019076640000000001 2023-10-09T11:08:08.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:08.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.502985+0000", 2023-10-09T11:08:08.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00020777600000000001 2023-10-09T11:08:08.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:08.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598562+0000", 2023-10-09T11:08:08.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.095576784999999997 2023-10-09T11:08:08.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:08.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598590+0000", 2023-10-09T11:08:08.029 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7881999999999998e-05 2023-10-09T11:08:08.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.029 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:08.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:28.598609+0000", 2023-10-09T11:08:08.029 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8587999999999999e-05 2023-10-09T11:08:08.029 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:08.029 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:08.029 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:08.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.030 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:184330 10.a 10:57ae38ca:::benchmark_data_smithi125_402869_object85761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T11:08:08.030 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:04:29.197170+0000", 2023-10-09T11:08:08.030 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 218.69608345899999, 2023-10-09T11:08:08.030 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.120668026, 2023-10-09T11:08:08.030 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:08.030 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:08.030 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:08.030 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:08:08.030 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:08:08.030 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 184330 2023-10-09T11:08:08.030 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.030 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:08.030 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:08.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197170+0000", 2023-10-09T11:08:08.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:08.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:08.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197170+0000", 2023-10-09T11:08:08.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:08.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:08.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197169+0000", 2023-10-09T11:08:08.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:08.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:08.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197173+0000", 2023-10-09T11:08:08.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.106e-06 2023-10-09T11:08:08.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:08.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197180+0000", 2023-10-09T11:08:08.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1760000000000004e-06 2023-10-09T11:08:08.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:08.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197184+0000", 2023-10-09T11:08:08.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9639999999999997e-06 2023-10-09T11:08:08.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:08.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197223+0000", 2023-10-09T11:08:08.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9072999999999997e-05 2023-10-09T11:08:08.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:08.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197334+0000", 2023-10-09T11:08:08.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000111279 2023-10-09T11:08:08.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T11:08:08.034 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.197448+0000", 2023-10-09T11:08:08.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000113848 2023-10-09T11:08:08.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:08.034 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.200444+0000", 2023-10-09T11:08:08.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029966039999999999 2023-10-09T11:08:08.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.035 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.035 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:08.035 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.206587+0000", 2023-10-09T11:08:08.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061423409999999999 2023-10-09T11:08:08.035 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.035 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.035 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:08.035 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.317756+0000", 2023-10-09T11:08:08.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.111169485 2023-10-09T11:08:08.035 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.035 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.035 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:08:08.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.317798+0000", 2023-10-09T11:08:08.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1838000000000002e-05 2023-10-09T11:08:08.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:08.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:08.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:08:08.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:04:29.317838+0000", 2023-10-09T11:08:08.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9628000000000001e-05 2023-10-09T11:08:08.036 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:08.036 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:08.036 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:08.036 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:08.036 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:08.037 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:08:08.037 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:08.066 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:08.186 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:08.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:08.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:08.226 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:08:08.270 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T11:08:08.290 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:08.370 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:08.414 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:08.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:08.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:08 smithi155 ceph-mon[184854]: osdmap e558: 8 total, 8 up, 8 in 2023-10-09T11:08:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:08 smithi155 ceph-mon[184854]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:08 smithi155 ceph-mon[184854]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:08 smithi155 ceph-mon[184854]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:08 smithi155 ceph-mon[184854]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:08 smithi155 ceph-mon[184854]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:08 smithi155 ceph-mon[184854]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:08 smithi155 ceph-mon[184854]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:08 smithi155 ceph-mon[184854]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:08 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2588215811' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:08:08.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:08 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:08:08.522 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:08.549 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:08.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[195507]: osdmap e558: 8 total, 8 up, 8 in 2023-10-09T11:08:08.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[195507]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[195507]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[195507]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[195507]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[195507]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[195507]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[195507]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[195507]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.575 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2588215811' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:08:08.575 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:08:08.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[200197]: osdmap e558: 8 total, 8 up, 8 in 2023-10-09T11:08:08.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[200197]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[200197]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[200197]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[200197]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[200197]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[200197]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[200197]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[200197]: failed to encode map e558 with expected crc 2023-10-09T11:08:08.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2588215811' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:08:08.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:08 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:08:08.581 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:08.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:08.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:08.673 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:08:08.691 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:08.739 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:08.828 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:08.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:08.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:08.942 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:08.988 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:09.319 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:09.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:09.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:09.425 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:09.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:09 smithi155 ceph-mon[184854]: 11.17 deep-scrub starts 2023-10-09T11:08:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:09 smithi155 ceph-mon[184854]: pgmap v1366: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 6040/59904 objects degraded (10.083%); 2287/59904 objects misplaced (3.818%); 0 B/s, 54 objects/s recovering 2023-10-09T11:08:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:09 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:08:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:09 smithi155 ceph-mon[184854]: failed to encode map e559 with expected crc 2023-10-09T11:08:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:09 smithi155 ceph-mon[184854]: failed to encode map e559 with expected crc 2023-10-09T11:08:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:09 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:08:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:09 smithi155 ceph-mon[184854]: osdmap e559: 8 total, 8 up, 8 in 2023-10-09T11:08:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:09 smithi155 ceph-mon[184854]: failed to encode map e559 with expected crc 2023-10-09T11:08:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:09 smithi155 ceph-mon[184854]: failed to encode map e559 with expected crc 2023-10-09T11:08:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:09 smithi155 ceph-mon[184854]: failed to encode map e559 with expected crc 2023-10-09T11:08:09.476 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:09 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 6040/59904 objects degraded (10.083%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:09.477 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:09.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[195507]: 11.17 deep-scrub starts 2023-10-09T11:08:09.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[195507]: pgmap v1366: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 6040/59904 objects degraded (10.083%); 2287/59904 objects misplaced (3.818%); 0 B/s, 54 objects/s recovering 2023-10-09T11:08:09.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:08:09.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[195507]: failed to encode map e559 with expected crc 2023-10-09T11:08:09.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[195507]: failed to encode map e559 with expected crc 2023-10-09T11:08:09.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:08:09.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[195507]: osdmap e559: 8 total, 8 up, 8 in 2023-10-09T11:08:09.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[195507]: failed to encode map e559 with expected crc 2023-10-09T11:08:09.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[195507]: failed to encode map e559 with expected crc 2023-10-09T11:08:09.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[195507]: failed to encode map e559 with expected crc 2023-10-09T11:08:09.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 6040/59904 objects degraded (10.083%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:09.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[200197]: 11.17 deep-scrub starts 2023-10-09T11:08:09.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[200197]: pgmap v1366: 139 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 6040/59904 objects degraded (10.083%); 2287/59904 objects misplaced (3.818%); 0 B/s, 54 objects/s recovering 2023-10-09T11:08:09.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:08:09.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[200197]: failed to encode map e559 with expected crc 2023-10-09T11:08:09.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[200197]: failed to encode map e559 with expected crc 2023-10-09T11:08:09.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:08:09.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[200197]: osdmap e559: 8 total, 8 up, 8 in 2023-10-09T11:08:09.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[200197]: failed to encode map e559 with expected crc 2023-10-09T11:08:09.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[200197]: failed to encode map e559 with expected crc 2023-10-09T11:08:09.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[200197]: failed to encode map e559 with expected crc 2023-10-09T11:08:09.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:09 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 6040/59904 objects degraded (10.083%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:09.813 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:10.599 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:08:10.613 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:08:10.617 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:10.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:10.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:10.722 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:10.754 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:10.778 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:10.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:10.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:10.898 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:10.959 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:10.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:10.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:10.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:11.050 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:08:11.050 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:08:11.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.050 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:184826 11.13 11:ce40e63f:::benchmark_data_smithi125_439543_object85327:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e559)", 2023-10-09T11:08:11.051 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:08:11.042357+0000", 2023-10-09T11:08:11.051 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0077127410000000004, 2023-10-09T11:08:11.051 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00773539, 2023-10-09T11:08:11.051 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:11.051 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:08:11.051 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:11.051 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:08:11.051 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:11.051 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 184826 2023-10-09T11:08:11.051 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.051 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:11.051 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.052 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:11.052 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.042357+0000", 2023-10-09T11:08:11.052 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:11.052 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.052 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.052 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:11.052 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.042357+0000", 2023-10-09T11:08:11.052 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:11.052 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.052 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.052 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:11.052 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.042356+0000", 2023-10-09T11:08:11.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:08:11.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.053 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:11.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.042363+0000", 2023-10-09T11:08:11.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5930000000000002e-06 2023-10-09T11:08:11.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.053 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:11.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.042379+0000", 2023-10-09T11:08:11.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.63e-05 2023-10-09T11:08:11.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:11.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.042386+0000", 2023-10-09T11:08:11.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7499999999999997e-06 2023-10-09T11:08:11.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:11.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.042461+0000", 2023-10-09T11:08:11.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.538e-05 2023-10-09T11:08:11.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:11.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.042556+0000", 2023-10-09T11:08:11.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4973e-05 2023-10-09T11:08:11.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.055 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,6", 2023-10-09T11:08:11.055 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.042982+0000", 2023-10-09T11:08:11.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00042542600000000002 2023-10-09T11:08:11.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.055 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:08:11.055 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.047902+0000", 2023-10-09T11:08:11.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0049205300000000002 2023-10-09T11:08:11.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.056 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.056 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:11.056 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.048203+0000", 2023-10-09T11:08:11.056 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030070500000000002 2023-10-09T11:08:11.056 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:11.056 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:11.056 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:11.056 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.056 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.056 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:184828 11.4 11:22c83342:::benchmark_data_smithi125_439543_object85329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e559)", 2023-10-09T11:08:11.056 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:08:11.047927+0000", 2023-10-09T11:08:11.056 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0021432740000000001, 2023-10-09T11:08:11.057 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022272120000000001, 2023-10-09T11:08:11.057 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:08:11.057 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T11:08:11.057 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:08:11.057 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:08:11.057 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:11.057 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 184828 2023-10-09T11:08:11.057 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.057 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:08:11.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.057 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:08:11.058 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.047927+0000", 2023-10-09T11:08:11.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:11.058 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.058 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:08:11.058 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.047927+0000", 2023-10-09T11:08:11.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:08:11.058 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.058 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:08:11.058 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.047926+0000", 2023-10-09T11:08:11.059 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:08:11.059 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.059 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.059 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:08:11.059 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.047931+0000", 2023-10-09T11:08:11.059 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7880000000000003e-06 2023-10-09T11:08:11.059 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.059 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.059 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:08:11.059 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.047945+0000", 2023-10-09T11:08:11.059 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3815000000000001e-05 2023-10-09T11:08:11.059 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.059 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.060 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:08:11.060 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.047953+0000", 2023-10-09T11:08:11.060 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1200000000000002e-06 2023-10-09T11:08:11.060 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.060 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.060 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:08:11.060 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.048004+0000", 2023-10-09T11:08:11.060 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1147999999999999e-05 2023-10-09T11:08:11.061 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.061 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.061 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:08:11.061 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.048110+0000", 2023-10-09T11:08:11.061 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000105654 2023-10-09T11:08:11.061 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:08:11.061 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:08:11.061 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,6", 2023-10-09T11:08:11.061 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:08:11.048596+0000", 2023-10-09T11:08:11.061 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000486324 2023-10-09T11:08:11.061 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:11.062 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:08:11.062 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:11.062 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:08:11.062 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T11:08:11.062 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 2 2023-10-09T11:08:11.062 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:08:11.095 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:11.146 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:11.181 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:11.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:11.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:11.287 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:11.334 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:11.517 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:11 smithi155 ceph-mon[184854]: pgmap v1368: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 945 op/s; 5000/51567 objects degraded (9.696%); 1852/51567 objects misplaced (3.591%); 2.0 MiB/s, 86 objects/s recovering 2023-10-09T11:08:11.518 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:11.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:11.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:11.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:11 smithi125 ceph-mon[195507]: pgmap v1368: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 945 op/s; 5000/51567 objects degraded (9.696%); 1852/51567 objects misplaced (3.591%); 2.0 MiB/s, 86 objects/s recovering 2023-10-09T11:08:11.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:11 smithi125 ceph-mon[200197]: pgmap v1368: 139 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 130 active+clean; 1.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 945 op/s; 5000/51567 objects degraded (9.696%); 1852/51567 objects misplaced (3.591%); 2.0 MiB/s, 86 objects/s recovering 2023-10-09T11:08:11.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:08:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:08:11] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.43.0" 2023-10-09T11:08:11.629 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:11.687 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:08:11.708 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:11.711 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T11:08:11.755 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:11.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:11.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:11.872 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:11.933 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:11.974 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:11.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:11.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:12.078 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:12.101 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:08:12.131 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:12.169 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1368,"stamp":"2023-10-09T11:08:09.915510+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1125426044,"num_objects":17189,"num_object_clones":0,"num_object_copies":51567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5000,"num_objects_misplaced":1852,"num_objects_unfound":0,"num_objects_dirty":17189,"num_whiteouts":0,"num_read":32,"num_read_kb":57,"num_write":181906,"num_write_kb":6367921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24077,"num_bytes_recovered":1468688714,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":238363,"ondisk_log_size":238363,"up":417,"acting":413,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":428,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11182052,"kb_used_data":4212196,"kb_used_omap":134,"kb_used_meta":6969529,"kb_avail":738779164,"statfs":{"total":767960285184,"available":756509863936,"internally_reserved":0,"allocated":4313288704,"data_stored":4294119957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":137532,"internal_metadata":7136798404},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,3,5,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-496304128,"num_objects":-7573,"num_object_clones":0,"num_object_copies":-22719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1040,"num_objects_misplaced":-999,"num_objects_unfound":0,"num_objects_dirty":-7573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7573,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":694,"num_bytes_recovered":16711680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.006723"},"pg_stats":[{"pgid":"11.23","version":"558'6113","reported_seq":9136,"reported_epoch":558,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:08:07.609581+0000","last_change":"2023-10-09T11:07:34.314839+0000","last_active":"2023-10-09T11:08:07.609581+0000","last_peered":"2023-10-09T11:08:07.609581+0000","last_clean":"2023-10-09T11:07:32.177661+0000","last_became_active":"2023-10-09T11:07:34.308281+0000","last_became_peered":"2023-10-09T11:07:34.308281+0000","last_unstale":"2023-10-09T11:08:07.609581+0000","last_undegraded":"2023-10-09T11:08:07.609581+0000","last_fullsized":"2023-10-09T11:08:07.609581+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6113,"log_dups_size":0,"ondisk_log_size":6113,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17907712,"num_objects":273,"num_object_clones":0,"num_object_copies":819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":273,"num_objects_unfound":0,"num_objects_dirty":273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2897,"num_write_kb":101456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1213,"num_bytes_recovered":72024064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":273}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.22","version":"559'5870","reported_seq":5962,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.219989+0000","last_change":"2023-10-09T11:06:43.010089+0000","last_active":"2023-10-09T11:08:08.219989+0000","last_peered":"2023-10-09T11:08:08.219989+0000","last_clean":"2023-10-09T11:08:08.219989+0000","last_became_active":"2023-10-09T11:06:43.008608+0000","last_became_peered":"2023-10-09T11:06:43.008608+0000","last_unstale":"2023-10-09T11:08:08.219989+0000","last_undegraded":"2023-10-09T11:08:08.219989+0000","last_fullsized":"2023-10-09T11:08:08.219989+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5870,"log_dups_size":0,"ondisk_log_size":5870,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18874368,"num_objects":288,"num_object_clones":0,"num_object_copies":864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2767,"num_write_kb":97760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.21","version":"559'6177","reported_seq":7048,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.219807+0000","last_change":"2023-10-09T11:07:23.970291+0000","last_active":"2023-10-09T11:08:08.219807+0000","last_peered":"2023-10-09T11:08:08.219807+0000","last_clean":"2023-10-09T11:08:08.219807+0000","last_became_active":"2023-10-09T11:06:43.015074+0000","last_became_peered":"2023-10-09T11:06:43.015074+0000","last_unstale":"2023-10-09T11:08:08.219807+0000","last_undegraded":"2023-10-09T11:08:08.219807+0000","last_fullsized":"2023-10-09T11:08:08.219807+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6177,"log_dups_size":0,"ondisk_log_size":6177,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13352960,"num_objects":204,"num_object_clones":0,"num_object_copies":612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2944,"num_write_kb":100720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":344,"num_bytes_recovered":22511616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.20","version":"559'6012","reported_seq":8422,"reported_epoch":559,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:08:08.229223+0000","last_change":"2023-10-09T11:07:34.316666+0000","last_active":"2023-10-09T11:08:08.229223+0000","last_peered":"2023-10-09T11:08:08.229223+0000","last_clean":"2023-10-09T11:07:32.195530+0000","last_became_active":"2023-10-09T11:07:34.309531+0000","last_became_peered":"2023-10-09T11:07:34.309531+0000","last_unstale":"2023-10-09T11:08:08.229223+0000","last_undegraded":"2023-10-09T11:08:08.229223+0000","last_fullsized":"2023-10-09T11:08:08.229223+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6012,"log_dups_size":0,"ondisk_log_size":6012,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18235392,"num_objects":278,"num_object_clones":0,"num_object_copies":834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":278,"num_objects_unfound":0,"num_objects_dirty":278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2846,"num_write_kb":99984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1150,"num_bytes_recovered":72941568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,6],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,3,6","objects":278}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.12","version":"559'7188","reported_seq":7270,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.315459+0000","last_change":"2023-10-09T11:05:45.369342+0000","last_active":"2023-10-09T11:08:08.315459+0000","last_peered":"2023-10-09T11:08:08.315459+0000","last_clean":"2023-10-09T11:08:08.315459+0000","last_became_active":"2023-10-09T11:05:45.367721+0000","last_became_peered":"2023-10-09T11:05:45.367721+0000","last_unstale":"2023-10-09T11:08:08.315459+0000","last_undegraded":"2023-10-09T11:08:08.315459+0000","last_fullsized":"2023-10-09T11:08:08.315459+0000","mapping_epoch":509,"log_start":"504'1783","ondisk_log_start":"504'1783","created":509,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5405,"log_dups_size":0,"ondisk_log_size":5405,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33816576,"num_objects":516,"num_object_clones":0,"num_object_copies":1548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5650,"num_write_kb":197312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"76'161","reported_seq":844,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.024679+0000","last_change":"2023-10-09T11:07:36.287455+0000","last_active":"2023-10-09T11:07:51.024679+0000","last_peered":"2023-10-09T11:07:51.024679+0000","last_clean":"2023-10-09T11:07:51.024679+0000","last_became_active":"2023-10-09T11:07:36.287052+0000","last_became_peered":"2023-10-09T11:07:36.287052+0000","last_unstale":"2023-10-09T11:07:51.024679+0000","last_undegraded":"2023-10-09T11:07:51.024679+0000","last_fullsized":"2023-10-09T11:07:51.024679+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:18.864873+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":339,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.042687+0000","last_change":"2023-10-09T11:03:21.185383+0000","last_active":"2023-10-09T11:07:51.042687+0000","last_peered":"2023-10-09T11:07:51.042687+0000","last_clean":"2023-10-09T11:07:51.042687+0000","last_became_active":"2023-10-09T11:03:21.185022+0000","last_became_peered":"2023-10-09T11:03:21.185022+0000","last_unstale":"2023-10-09T11:07:51.042687+0000","last_undegraded":"2023-10-09T11:07:51.042687+0000","last_fullsized":"2023-10-09T11:07:51.042687+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:05.230434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":85,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219524+0000","last_change":"2023-10-09T11:04:50.319477+0000","last_active":"2023-10-09T11:08:07.219524+0000","last_peered":"2023-10-09T11:08:07.219524+0000","last_clean":"2023-10-09T11:08:07.219524+0000","last_became_active":"2023-10-09T11:04:50.319148+0000","last_became_peered":"2023-10-09T11:04:50.319148+0000","last_unstale":"2023-10-09T11:08:07.219524+0000","last_undegraded":"2023-10-09T11:08:07.219524+0000","last_fullsized":"2023-10-09T11:08:07.219524+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:48.725566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"559'7459","reported_seq":8872,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.284464+0000","last_change":"2023-10-09T11:06:54.752222+0000","last_active":"2023-10-09T11:08:08.284464+0000","last_peered":"2023-10-09T11:08:08.284464+0000","last_clean":"2023-10-09T11:08:08.284464+0000","last_became_active":"2023-10-09T11:05:45.395271+0000","last_became_peered":"2023-10-09T11:05:45.395271+0000","last_unstale":"2023-10-09T11:08:08.284464+0000","last_undegraded":"2023-10-09T11:08:08.284464+0000","last_fullsized":"2023-10-09T11:08:08.284464+0000","mapping_epoch":509,"log_start":"504'2080","ondisk_log_start":"504'2080","created":509,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5379,"log_dups_size":0,"ondisk_log_size":5379,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38043648,"num_objects":581,"num_object_clones":0,"num_object_copies":1743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5841,"num_write_kb":205472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":659,"num_bytes_recovered":43188224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":714,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.022446+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:07:51.022446+0000","last_peered":"2023-10-09T11:07:51.022446+0000","last_clean":"2023-10-09T11:07:51.022446+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:07:51.022446+0000","last_undegraded":"2023-10-09T11:07:51.022446+0000","last_fullsized":"2023-10-09T11:07:51.022446+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":273,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.215893+0000","last_change":"2023-10-09T11:04:50.333711+0000","last_active":"2023-10-09T11:08:07.215893+0000","last_peered":"2023-10-09T11:08:07.215893+0000","last_clean":"2023-10-09T11:08:07.215893+0000","last_became_active":"2023-10-09T11:04:50.333262+0000","last_became_peered":"2023-10-09T11:04:50.333262+0000","last_unstale":"2023-10-09T11:08:07.215893+0000","last_undegraded":"2023-10-09T11:08:07.215893+0000","last_fullsized":"2023-10-09T11:08:07.215893+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.843765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":17,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.026677+0000","last_change":"2023-10-09T11:07:33.237903+0000","last_active":"2023-10-09T11:07:51.026677+0000","last_peered":"2023-10-09T11:07:51.026677+0000","last_clean":"2023-10-09T11:07:51.026677+0000","last_became_active":"2023-10-09T11:07:33.237618+0000","last_became_peered":"2023-10-09T11:07:33.237618+0000","last_unstale":"2023-10-09T11:07:51.026677+0000","last_undegraded":"2023-10-09T11:07:51.026677+0000","last_fullsized":"2023-10-09T11:07:51.026677+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:25.600818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.10","version":"559'7279","reported_seq":7969,"reported_epoch":559,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:08:08.276326+0000","last_change":"2023-10-09T11:07:34.347309+0000","last_active":"2023-10-09T11:08:08.276326+0000","last_peered":"2023-10-09T11:08:08.276326+0000","last_clean":"2023-10-09T11:06:06.955693+0000","last_became_active":"2023-10-09T11:07:33.305104+0000","last_became_peered":"2023-10-09T11:07:33.305104+0000","last_unstale":"2023-10-09T11:08:08.276326+0000","last_undegraded":"2023-10-09T11:07:34.345913+0000","last_fullsized":"2023-10-09T11:07:33.208203+0000","mapping_epoch":549,"log_start":"504'1769","ondisk_log_start":"504'1769","created":509,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5510,"log_dups_size":0,"ondisk_log_size":5510,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34635776,"num_objects":529,"num_object_clones":0,"num_object_copies":1587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2269,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5717,"num_write_kb":199840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":292,"num_bytes_recovered":19136512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":-127},{"shards":"1,3,6","objects":562}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":732,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.026175+0000","last_change":"2023-10-09T11:07:36.284534+0000","last_active":"2023-10-09T11:07:51.026175+0000","last_peered":"2023-10-09T11:07:51.026175+0000","last_clean":"2023-10-09T11:07:51.026175+0000","last_became_active":"2023-10-09T11:07:36.284254+0000","last_became_peered":"2023-10-09T11:07:36.284254+0000","last_unstale":"2023-10-09T11:07:51.026175+0000","last_undegraded":"2023-10-09T11:07:51.026175+0000","last_fullsized":"2023-10-09T11:07:51.026175+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:19.159630+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":85,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.209764+0000","last_change":"2023-10-09T11:04:50.310477+0000","last_active":"2023-10-09T11:08:07.209764+0000","last_peered":"2023-10-09T11:08:07.209764+0000","last_clean":"2023-10-09T11:08:07.209764+0000","last_became_active":"2023-10-09T11:04:50.310067+0000","last_became_peered":"2023-10-09T11:04:50.310067+0000","last_unstale":"2023-10-09T11:08:07.209764+0000","last_undegraded":"2023-10-09T11:08:07.209764+0000","last_fullsized":"2023-10-09T11:08:07.209764+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:52.884716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":366,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.025549+0000","last_change":"2023-10-09T11:05:20.310581+0000","last_active":"2023-10-09T11:07:51.025549+0000","last_peered":"2023-10-09T11:07:51.025549+0000","last_clean":"2023-10-09T11:07:51.025549+0000","last_became_active":"2023-10-09T11:05:20.309830+0000","last_became_peered":"2023-10-09T11:05:20.309830+0000","last_unstale":"2023-10-09T11:07:51.025549+0000","last_undegraded":"2023-10-09T11:07:51.025549+0000","last_fullsized":"2023-10-09T11:07:51.025549+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:24.508211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1f","version":"557'8897","reported_seq":16582,"reported_epoch":557,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T11:08:03.338621+0000","last_change":"2023-10-09T11:07:34.394394+0000","last_active":"2023-10-09T11:08:03.338621+0000","last_peered":"2023-10-09T11:08:03.338621+0000","last_clean":"2023-10-09T11:06:06.963741+0000","last_became_active":"2023-10-09T11:07:34.331654+0000","last_became_peered":"2023-10-09T11:07:34.331654+0000","last_unstale":"2023-10-09T11:08:03.338621+0000","last_undegraded":"2023-10-09T11:08:03.338621+0000","last_fullsized":"2023-10-09T11:07:34.216155+0000","mapping_epoch":550,"log_start":"504'1895","ondisk_log_start":"504'1895","created":531,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7002,"log_dups_size":0,"ondisk_log_size":7002,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41484288,"num_objects":633,"num_object_clones":0,"num_object_copies":1899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5667,"num_write_kb":201600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3797,"num_bytes_recovered":164298752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":633}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":772,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.026787+0000","last_change":"2023-10-09T11:07:36.282502+0000","last_active":"2023-10-09T11:07:51.026787+0000","last_peered":"2023-10-09T11:07:51.026787+0000","last_clean":"2023-10-09T11:07:51.026787+0000","last_became_active":"2023-10-09T11:07:36.282133+0000","last_became_peered":"2023-10-09T11:07:36.282133+0000","last_unstale":"2023-10-09T11:07:51.026787+0000","last_undegraded":"2023-10-09T11:07:51.026787+0000","last_fullsized":"2023-10-09T11:07:51.026787+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:03.762383+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":193,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.025615+0000","last_change":"2023-10-09T11:06:24.245488+0000","last_active":"2023-10-09T11:07:51.025615+0000","last_peered":"2023-10-09T11:07:51.025615+0000","last_clean":"2023-10-09T11:07:51.025615+0000","last_became_active":"2023-10-09T11:06:24.245167+0000","last_became_peered":"2023-10-09T11:06:24.245167+0000","last_unstale":"2023-10-09T11:07:51.025615+0000","last_undegraded":"2023-10-09T11:07:51.025615+0000","last_fullsized":"2023-10-09T11:07:51.025615+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:31.702568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":94,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.209906+0000","last_change":"2023-10-09T11:05:20.299453+0000","last_active":"2023-10-09T11:08:07.209906+0000","last_peered":"2023-10-09T11:08:07.209906+0000","last_clean":"2023-10-09T11:08:07.209906+0000","last_became_active":"2023-10-09T11:05:20.299117+0000","last_became_peered":"2023-10-09T11:05:20.299117+0000","last_unstale":"2023-10-09T11:08:07.209906+0000","last_undegraded":"2023-10-09T11:08:07.209906+0000","last_fullsized":"2023-10-09T11:08:07.209906+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:30.072813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1e","version":"558'8803","reported_seq":8885,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.726721+0000","last_change":"2023-10-09T11:06:43.029410+0000","last_active":"2023-10-09T11:08:07.726721+0000","last_peered":"2023-10-09T11:08:07.726721+0000","last_clean":"2023-10-09T11:08:07.726721+0000","last_became_active":"2023-10-09T11:06:43.026551+0000","last_became_peered":"2023-10-09T11:06:43.026551+0000","last_unstale":"2023-10-09T11:08:07.726721+0000","last_undegraded":"2023-10-09T11:08:07.726721+0000","last_fullsized":"2023-10-09T11:08:07.726721+0000","mapping_epoch":531,"log_start":"504'1803","ondisk_log_start":"504'1803","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7000,"log_dups_size":0,"ondisk_log_size":7000,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35782656,"num_objects":546,"num_object_clones":0,"num_object_copies":1638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5628,"num_write_kb":197568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":886,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.056255+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:07:51.056255+0000","last_peered":"2023-10-09T11:07:51.056255+0000","last_clean":"2023-10-09T11:07:51.056255+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:07:51.056255+0000","last_undegraded":"2023-10-09T11:07:51.056255+0000","last_fullsized":"2023-10-09T11:07:51.056255+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":428,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.025153+0000","last_change":"2023-10-09T11:07:33.242665+0000","last_active":"2023-10-09T11:07:51.025153+0000","last_peered":"2023-10-09T11:07:51.025153+0000","last_clean":"2023-10-09T11:07:51.025153+0000","last_became_active":"2023-10-09T11:07:33.242313+0000","last_became_peered":"2023-10-09T11:07:33.242313+0000","last_unstale":"2023-10-09T11:07:51.025153+0000","last_undegraded":"2023-10-09T11:07:51.025153+0000","last_fullsized":"2023-10-09T11:07:51.025153+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:20.565135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":112,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.212848+0000","last_change":"2023-10-09T11:07:33.225683+0000","last_active":"2023-10-09T11:08:07.212848+0000","last_peered":"2023-10-09T11:08:07.212848+0000","last_clean":"2023-10-09T11:08:07.212848+0000","last_became_active":"2023-10-09T11:07:33.225275+0000","last_became_peered":"2023-10-09T11:07:33.225275+0000","last_unstale":"2023-10-09T11:08:07.212848+0000","last_undegraded":"2023-10-09T11:08:07.212848+0000","last_fullsized":"2023-10-09T11:08:07.212848+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:10.131060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1d","version":"557'8999","reported_seq":11917,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T11:08:06.156428+0000","last_change":"2023-10-09T11:07:23.727748+0000","last_active":"2023-10-09T11:08:06.156428+0000","last_peered":"2023-10-09T11:08:06.156428+0000","last_clean":"2023-10-09T11:08:06.156428+0000","last_became_active":"2023-10-09T11:06:43.012614+0000","last_became_peered":"2023-10-09T11:06:43.012614+0000","last_unstale":"2023-10-09T11:08:06.156428+0000","last_undegraded":"2023-10-09T11:08:06.156428+0000","last_fullsized":"2023-10-09T11:08:06.156428+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":8999,"log_dups_size":0,"ondisk_log_size":8999,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33259532,"num_objects":508,"num_object_clones":0,"num_object_copies":1524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":508,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5778,"num_write_kb":201121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":704,"num_bytes_recovered":46137344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":786,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.025232+0000","last_change":"2023-10-09T11:05:23.328981+0000","last_active":"2023-10-09T11:07:51.025232+0000","last_peered":"2023-10-09T11:07:51.025232+0000","last_clean":"2023-10-09T11:07:51.025232+0000","last_became_active":"2023-10-09T11:05:23.328616+0000","last_became_peered":"2023-10-09T11:05:23.328616+0000","last_unstale":"2023-10-09T11:07:51.025232+0000","last_undegraded":"2023-10-09T11:07:51.025232+0000","last_fullsized":"2023-10-09T11:07:51.025232+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:15.315819+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":23,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.026707+0000","last_change":"2023-10-09T11:07:33.265842+0000","last_active":"2023-10-09T11:07:51.026707+0000","last_peered":"2023-10-09T11:07:51.026707+0000","last_clean":"2023-10-09T11:07:51.026707+0000","last_became_active":"2023-10-09T11:07:33.232311+0000","last_became_peered":"2023-10-09T11:07:33.232311+0000","last_unstale":"2023-10-09T11:07:51.026707+0000","last_undegraded":"2023-10-09T11:07:51.026707+0000","last_fullsized":"2023-10-09T11:07:51.026707+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:23.402582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":335,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.022138+0000","last_change":"2023-10-09T11:05:20.316265+0000","last_active":"2023-10-09T11:07:51.022138+0000","last_peered":"2023-10-09T11:07:51.022138+0000","last_clean":"2023-10-09T11:07:51.022138+0000","last_became_active":"2023-10-09T11:05:20.315853+0000","last_became_peered":"2023-10-09T11:05:20.315853+0000","last_unstale":"2023-10-09T11:07:51.022138+0000","last_undegraded":"2023-10-09T11:07:51.022138+0000","last_fullsized":"2023-10-09T11:07:51.022138+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:35.093981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1c","version":"559'8906","reported_seq":8989,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.228877+0000","last_change":"2023-10-09T11:06:43.025845+0000","last_active":"2023-10-09T11:08:08.228877+0000","last_peered":"2023-10-09T11:08:08.228877+0000","last_clean":"2023-10-09T11:08:08.228877+0000","last_became_active":"2023-10-09T11:06:43.022156+0000","last_became_peered":"2023-10-09T11:06:43.022156+0000","last_unstale":"2023-10-09T11:08:08.228877+0000","last_undegraded":"2023-10-09T11:08:08.228877+0000","last_fullsized":"2023-10-09T11:08:08.228877+0000","mapping_epoch":531,"log_start":"504'1919","ondisk_log_start":"504'1919","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6987,"log_dups_size":0,"ondisk_log_size":6987,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35749888,"num_objects":546,"num_object_clones":0,"num_object_copies":1638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5698,"num_write_kb":199776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":951,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.217389+0000","last_change":"2023-10-09T11:04:53.274921+0000","last_active":"2023-10-09T11:08:07.217389+0000","last_peered":"2023-10-09T11:08:07.217389+0000","last_clean":"2023-10-09T11:08:07.217389+0000","last_became_active":"2023-10-09T11:04:53.274518+0000","last_became_peered":"2023-10-09T11:04:53.274518+0000","last_unstale":"2023-10-09T11:08:07.217389+0000","last_undegraded":"2023-10-09T11:08:07.217389+0000","last_fullsized":"2023-10-09T11:08:07.217389+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:14.315301+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":27,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.027685+0000","last_change":"2023-10-09T11:07:33.262559+0000","last_active":"2023-10-09T11:07:51.027685+0000","last_peered":"2023-10-09T11:07:51.027685+0000","last_clean":"2023-10-09T11:07:51.027685+0000","last_became_active":"2023-10-09T11:07:33.231695+0000","last_became_peered":"2023-10-09T11:07:33.231695+0000","last_unstale":"2023-10-09T11:07:51.027685+0000","last_undegraded":"2023-10-09T11:07:51.027685+0000","last_fullsized":"2023-10-09T11:07:51.027685+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:59.006940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":578,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.054450+0000","last_change":"2023-10-09T11:05:20.311424+0000","last_active":"2023-10-09T11:07:51.054450+0000","last_peered":"2023-10-09T11:07:51.054450+0000","last_clean":"2023-10-09T11:07:51.054450+0000","last_became_active":"2023-10-09T11:05:20.311097+0000","last_became_peered":"2023-10-09T11:05:20.311097+0000","last_unstale":"2023-10-09T11:07:51.054450+0000","last_undegraded":"2023-10-09T11:07:51.054450+0000","last_fullsized":"2023-10-09T11:07:51.054450+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:19.274582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1b","version":"559'8928","reported_seq":9011,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.302658+0000","last_change":"2023-10-09T11:06:43.035498+0000","last_active":"2023-10-09T11:08:08.302658+0000","last_peered":"2023-10-09T11:08:08.302658+0000","last_clean":"2023-10-09T11:08:08.302658+0000","last_became_active":"2023-10-09T11:06:43.031985+0000","last_became_peered":"2023-10-09T11:06:43.031985+0000","last_unstale":"2023-10-09T11:08:08.302658+0000","last_undegraded":"2023-10-09T11:08:08.302658+0000","last_fullsized":"2023-10-09T11:08:08.302658+0000","mapping_epoch":531,"log_start":"504'1979","ondisk_log_start":"504'1979","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6949,"log_dups_size":0,"ondisk_log_size":6949,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27852800,"num_objects":425,"num_object_clones":0,"num_object_copies":1275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5758,"num_write_kb":197856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":586,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T11:08:04.745102+0000","last_change":"2023-10-09T11:08:04.745102+0000","last_active":"2023-10-09T11:08:04.745102+0000","last_peered":"2023-10-09T11:08:04.745102+0000","last_clean":"2023-10-09T11:08:04.745102+0000","last_became_active":"2023-10-09T11:06:24.245076+0000","last_became_peered":"2023-10-09T11:06:24.245076+0000","last_unstale":"2023-10-09T11:08:04.745102+0000","last_undegraded":"2023-10-09T11:08:04.745102+0000","last_fullsized":"2023-10-09T11:08:04.745102+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":23,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.026619+0000","last_change":"2023-10-09T11:07:33.269200+0000","last_active":"2023-10-09T11:07:51.026619+0000","last_peered":"2023-10-09T11:07:51.026619+0000","last_clean":"2023-10-09T11:07:51.026619+0000","last_became_active":"2023-10-09T11:07:33.236205+0000","last_became_peered":"2023-10-09T11:07:33.236205+0000","last_unstale":"2023-10-09T11:07:51.026619+0000","last_undegraded":"2023-10-09T11:07:51.026619+0000","last_fullsized":"2023-10-09T11:07:51.026619+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:24.574734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":103,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.211446+0000","last_change":"2023-10-09T11:07:33.227803+0000","last_active":"2023-10-09T11:08:07.211446+0000","last_peered":"2023-10-09T11:08:07.211446+0000","last_clean":"2023-10-09T11:08:07.211446+0000","last_became_active":"2023-10-09T11:07:33.227486+0000","last_became_peered":"2023-10-09T11:07:33.227486+0000","last_unstale":"2023-10-09T11:08:07.211446+0000","last_undegraded":"2023-10-09T11:08:07.211446+0000","last_fullsized":"2023-10-09T11:08:07.211446+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:39.274875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1a","version":"559'8991","reported_seq":11854,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.264694+0000","last_change":"2023-10-09T11:06:43.001534+0000","last_active":"2023-10-09T11:08:08.264694+0000","last_peered":"2023-10-09T11:08:08.264694+0000","last_clean":"2023-10-09T11:08:08.264694+0000","last_became_active":"2023-10-09T11:06:42.998860+0000","last_became_peered":"2023-10-09T11:06:42.998860+0000","last_unstale":"2023-10-09T11:08:08.264694+0000","last_undegraded":"2023-10-09T11:08:08.264694+0000","last_fullsized":"2023-10-09T11:08:08.264694+0000","mapping_epoch":531,"log_start":"504'2132","ondisk_log_start":"504'2132","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6859,"log_dups_size":0,"ondisk_log_size":6859,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31653888,"num_objects":483,"num_object_clones":0,"num_object_copies":1449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5781,"num_write_kb":200448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":691,"num_bytes_recovered":45285376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":868,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.025037+0000","last_change":"2023-10-09T11:05:23.323856+0000","last_active":"2023-10-09T11:07:51.025037+0000","last_peered":"2023-10-09T11:07:51.025037+0000","last_clean":"2023-10-09T11:07:51.025037+0000","last_became_active":"2023-10-09T11:05:23.322763+0000","last_became_peered":"2023-10-09T11:05:23.322763+0000","last_unstale":"2023-10-09T11:07:51.025037+0000","last_undegraded":"2023-10-09T11:07:51.025037+0000","last_fullsized":"2023-10-09T11:07:51.025037+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.760820+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":17,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.026093+0000","last_change":"2023-10-09T11:07:33.233510+0000","last_active":"2023-10-09T11:07:51.026093+0000","last_peered":"2023-10-09T11:07:51.026093+0000","last_clean":"2023-10-09T11:07:51.026093+0000","last_became_active":"2023-10-09T11:07:33.232339+0000","last_became_peered":"2023-10-09T11:07:33.232339+0000","last_unstale":"2023-10-09T11:07:51.026093+0000","last_undegraded":"2023-10-09T11:07:51.026093+0000","last_fullsized":"2023-10-09T11:07:51.026093+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:55.324912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":498,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.025417+0000","last_change":"2023-10-09T11:07:33.226594+0000","last_active":"2023-10-09T11:07:51.025417+0000","last_peered":"2023-10-09T11:07:51.025417+0000","last_clean":"2023-10-09T11:07:51.025417+0000","last_became_active":"2023-10-09T11:07:33.226275+0000","last_became_peered":"2023-10-09T11:07:33.226275+0000","last_unstale":"2023-10-09T11:07:51.025417+0000","last_undegraded":"2023-10-09T11:07:51.025417+0000","last_fullsized":"2023-10-09T11:07:51.025417+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:41.180088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"559'7312","reported_seq":8769,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.283460+0000","last_change":"2023-10-09T11:06:05.166134+0000","last_active":"2023-10-09T11:08:08.283460+0000","last_peered":"2023-10-09T11:08:08.283460+0000","last_clean":"2023-10-09T11:08:08.283460+0000","last_became_active":"2023-10-09T11:05:45.388425+0000","last_became_peered":"2023-10-09T11:05:45.388425+0000","last_unstale":"2023-10-09T11:08:08.283460+0000","last_undegraded":"2023-10-09T11:08:08.283460+0000","last_fullsized":"2023-10-09T11:08:08.283460+0000","mapping_epoch":509,"log_start":"504'1966","ondisk_log_start":"504'1966","created":509,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5346,"log_dups_size":0,"ondisk_log_size":5346,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34897920,"num_objects":533,"num_object_clones":0,"num_object_copies":1599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5749,"num_write_kb":200992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":684,"num_bytes_recovered":44826624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":892,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216559+0000","last_change":"2023-10-09T11:04:53.274236+0000","last_active":"2023-10-09T11:08:07.216559+0000","last_peered":"2023-10-09T11:08:07.216559+0000","last_clean":"2023-10-09T11:08:07.216559+0000","last_became_active":"2023-10-09T11:04:53.273817+0000","last_became_peered":"2023-10-09T11:04:53.273817+0000","last_unstale":"2023-10-09T11:08:07.216559+0000","last_undegraded":"2023-10-09T11:08:07.216559+0000","last_fullsized":"2023-10-09T11:08:07.216559+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:05.096905+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":152,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.026409+0000","last_change":"2023-10-09T11:05:20.312129+0000","last_active":"2023-10-09T11:07:51.026409+0000","last_peered":"2023-10-09T11:07:51.026409+0000","last_clean":"2023-10-09T11:07:51.026409+0000","last_became_active":"2023-10-09T11:05:20.311810+0000","last_became_peered":"2023-10-09T11:05:20.311810+0000","last_unstale":"2023-10-09T11:07:51.026409+0000","last_undegraded":"2023-10-09T11:07:51.026409+0000","last_fullsized":"2023-10-09T11:07:51.026409+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:48.813391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":252,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219203+0000","last_change":"2023-10-09T11:07:33.218244+0000","last_active":"2023-10-09T11:08:07.219203+0000","last_peered":"2023-10-09T11:08:07.219203+0000","last_clean":"2023-10-09T11:08:07.219203+0000","last_became_active":"2023-10-09T11:07:33.217918+0000","last_became_peered":"2023-10-09T11:07:33.217918+0000","last_unstale":"2023-10-09T11:08:07.219203+0000","last_undegraded":"2023-10-09T11:08:07.219203+0000","last_fullsized":"2023-10-09T11:08:07.219203+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:16.150105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"559'7217","reported_seq":9771,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.219565+0000","last_change":"2023-10-09T11:06:07.154110+0000","last_active":"2023-10-09T11:08:08.219565+0000","last_peered":"2023-10-09T11:08:08.219565+0000","last_clean":"2023-10-09T11:08:08.219565+0000","last_became_active":"2023-10-09T11:05:45.388306+0000","last_became_peered":"2023-10-09T11:05:45.388306+0000","last_unstale":"2023-10-09T11:08:08.219565+0000","last_undegraded":"2023-10-09T11:08:08.219565+0000","last_fullsized":"2023-10-09T11:08:08.219565+0000","mapping_epoch":509,"log_start":"504'1839","ondisk_log_start":"504'1839","created":509,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5378,"log_dups_size":0,"ondisk_log_size":5378,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33423360,"num_objects":510,"num_object_clones":0,"num_object_copies":1530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5682,"num_write_kb":198144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":645,"num_bytes_recovered":42237952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":865,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.022682+0000","last_change":"2023-10-09T11:05:23.324537+0000","last_active":"2023-10-09T11:07:51.022682+0000","last_peered":"2023-10-09T11:07:51.022682+0000","last_clean":"2023-10-09T11:07:51.022682+0000","last_became_active":"2023-10-09T11:05:23.324187+0000","last_became_peered":"2023-10-09T11:05:23.324187+0000","last_unstale":"2023-10-09T11:07:51.022682+0000","last_undegraded":"2023-10-09T11:07:51.022682+0000","last_fullsized":"2023-10-09T11:07:51.022682+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:25.286106+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":77,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.029596+0000","last_change":"2023-10-09T11:05:20.323565+0000","last_active":"2023-10-09T11:07:51.029596+0000","last_peered":"2023-10-09T11:07:51.029596+0000","last_clean":"2023-10-09T11:07:51.029596+0000","last_became_active":"2023-10-09T11:05:20.309672+0000","last_became_peered":"2023-10-09T11:05:20.309672+0000","last_unstale":"2023-10-09T11:07:51.029596+0000","last_undegraded":"2023-10-09T11:07:51.029596+0000","last_fullsized":"2023-10-09T11:07:51.029596+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:50.211465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":103,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219691+0000","last_change":"2023-10-09T11:07:33.228001+0000","last_active":"2023-10-09T11:08:07.219691+0000","last_peered":"2023-10-09T11:08:07.219691+0000","last_clean":"2023-10-09T11:08:07.219691+0000","last_became_active":"2023-10-09T11:07:33.227691+0000","last_became_peered":"2023-10-09T11:07:33.227691+0000","last_unstale":"2023-10-09T11:08:07.219691+0000","last_undegraded":"2023-10-09T11:08:07.219691+0000","last_fullsized":"2023-10-09T11:08:07.219691+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:24:16.098138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"557'7155","reported_seq":7246,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T11:08:06.716034+0000","last_change":"2023-10-09T11:08:05.741500+0000","last_active":"2023-10-09T11:08:06.716034+0000","last_peered":"2023-10-09T11:08:06.716034+0000","last_clean":"2023-10-09T11:08:06.716034+0000","last_became_active":"2023-10-09T11:05:45.338519+0000","last_became_peered":"2023-10-09T11:05:45.338519+0000","last_unstale":"2023-10-09T11:08:06.716034+0000","last_undegraded":"2023-10-09T11:08:06.716034+0000","last_fullsized":"2023-10-09T11:08:06.716034+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7155,"log_dups_size":0,"ondisk_log_size":7155,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34045952,"num_objects":519,"num_object_clones":0,"num_object_copies":1557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5585,"num_write_kb":195360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":789,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.026865+0000","last_change":"2023-10-09T11:07:36.284927+0000","last_active":"2023-10-09T11:07:51.026865+0000","last_peered":"2023-10-09T11:07:51.026865+0000","last_clean":"2023-10-09T11:07:51.026865+0000","last_became_active":"2023-10-09T11:07:36.284631+0000","last_became_peered":"2023-10-09T11:07:36.284631+0000","last_unstale":"2023-10-09T11:07:51.026865+0000","last_undegraded":"2023-10-09T11:07:51.026865+0000","last_fullsized":"2023-10-09T11:07:51.026865+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:37.220243+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":21,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.027573+0000","last_change":"2023-10-09T11:07:33.273503+0000","last_active":"2023-10-09T11:07:51.027573+0000","last_peered":"2023-10-09T11:07:51.027573+0000","last_clean":"2023-10-09T11:07:51.027573+0000","last_became_active":"2023-10-09T11:07:33.242980+0000","last_became_peered":"2023-10-09T11:07:33.242980+0000","last_unstale":"2023-10-09T11:07:51.027573+0000","last_undegraded":"2023-10-09T11:07:51.027573+0000","last_fullsized":"2023-10-09T11:07:51.027573+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:18.934972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":519,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.024738+0000","last_change":"2023-10-09T11:07:33.304363+0000","last_active":"2023-10-09T11:07:51.024738+0000","last_peered":"2023-10-09T11:07:51.024738+0000","last_clean":"2023-10-09T11:07:51.024738+0000","last_became_active":"2023-10-09T11:07:33.304024+0000","last_became_peered":"2023-10-09T11:07:33.304024+0000","last_unstale":"2023-10-09T11:07:51.024738+0000","last_undegraded":"2023-10-09T11:07:51.024738+0000","last_fullsized":"2023-10-09T11:07:51.024738+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:21:51.522688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"557'7121","reported_seq":7221,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T11:08:06.178425+0000","last_change":"2023-10-09T11:07:28.705631+0000","last_active":"2023-10-09T11:08:06.178425+0000","last_peered":"2023-10-09T11:08:06.178425+0000","last_clean":"2023-10-09T11:08:06.178425+0000","last_became_active":"2023-10-09T11:06:24.254388+0000","last_became_peered":"2023-10-09T11:06:24.254388+0000","last_unstale":"2023-10-09T11:08:06.178425+0000","last_undegraded":"2023-10-09T11:08:06.178425+0000","last_fullsized":"2023-10-09T11:08:06.178425+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7121,"log_dups_size":0,"ondisk_log_size":7121,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36372480,"num_objects":555,"num_object_clones":0,"num_object_copies":1665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5595,"num_write_kb":196800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":940,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.211519+0000","last_change":"2023-10-09T11:05:23.316129+0000","last_active":"2023-10-09T11:08:07.211519+0000","last_peered":"2023-10-09T11:08:07.211519+0000","last_clean":"2023-10-09T11:08:07.211519+0000","last_became_active":"2023-10-09T11:05:23.315792+0000","last_became_peered":"2023-10-09T11:05:23.315792+0000","last_unstale":"2023-10-09T11:08:07.211519+0000","last_undegraded":"2023-10-09T11:08:07.211519+0000","last_fullsized":"2023-10-09T11:08:07.211519+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:51.770547+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":21,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.026766+0000","last_change":"2023-10-09T11:07:33.256691+0000","last_active":"2023-10-09T11:07:51.026766+0000","last_peered":"2023-10-09T11:07:51.026766+0000","last_clean":"2023-10-09T11:07:51.026766+0000","last_became_active":"2023-10-09T11:07:33.229554+0000","last_became_peered":"2023-10-09T11:07:33.229554+0000","last_unstale":"2023-10-09T11:07:51.026766+0000","last_undegraded":"2023-10-09T11:07:51.026766+0000","last_fullsized":"2023-10-09T11:07:51.026766+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:57.414795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":273,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.215808+0000","last_change":"2023-10-09T11:04:50.333538+0000","last_active":"2023-10-09T11:08:07.215808+0000","last_peered":"2023-10-09T11:08:07.215808+0000","last_clean":"2023-10-09T11:08:07.215808+0000","last_became_active":"2023-10-09T11:04:50.332514+0000","last_became_peered":"2023-10-09T11:04:50.332514+0000","last_unstale":"2023-10-09T11:08:07.215808+0000","last_undegraded":"2023-10-09T11:08:07.215808+0000","last_fullsized":"2023-10-09T11:08:07.215808+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:22.933839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"558'7065","reported_seq":11679,"reported_epoch":558,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:08:07.686823+0000","last_change":"2023-10-09T11:08:06.826203+0000","last_active":"2023-10-09T11:08:07.686823+0000","last_peered":"2023-10-09T11:08:07.686823+0000","last_clean":"2023-10-09T11:07:32.132824+0000","last_became_active":"2023-10-09T11:07:34.322525+0000","last_became_peered":"2023-10-09T11:07:34.322525+0000","last_unstale":"2023-10-09T11:08:07.686823+0000","last_undegraded":"2023-10-09T11:08:07.686823+0000","last_fullsized":"2023-10-09T11:08:07.686823+0000","mapping_epoch":550,"log_start":"503'1560","ondisk_log_start":"503'1560","created":495,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5505,"log_dups_size":0,"ondisk_log_size":5505,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36536320,"num_objects":557,"num_object_clones":0,"num_object_copies":1671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":557,"num_objects_unfound":0,"num_objects_dirty":557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5547,"num_write_kb":195360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2245,"num_bytes_recovered":147128320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":557}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":813,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.025306+0000","last_change":"2023-10-09T11:05:23.322251+0000","last_active":"2023-10-09T11:07:51.025306+0000","last_peered":"2023-10-09T11:07:51.025306+0000","last_clean":"2023-10-09T11:07:51.025306+0000","last_became_active":"2023-10-09T11:05:23.321826+0000","last_became_peered":"2023-10-09T11:05:23.321826+0000","last_unstale":"2023-10-09T11:07:51.025306+0000","last_undegraded":"2023-10-09T11:07:51.025306+0000","last_fullsized":"2023-10-09T11:07:51.025306+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.858013+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":21,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.027086+0000","last_change":"2023-10-09T11:07:33.271238+0000","last_active":"2023-10-09T11:07:51.027086+0000","last_peered":"2023-10-09T11:07:51.027086+0000","last_clean":"2023-10-09T11:07:51.027086+0000","last_became_active":"2023-10-09T11:07:33.241886+0000","last_became_peered":"2023-10-09T11:07:33.241886+0000","last_unstale":"2023-10-09T11:07:51.027086+0000","last_undegraded":"2023-10-09T11:07:51.027086+0000","last_fullsized":"2023-10-09T11:07:51.027086+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:48.882589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":87,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.029774+0000","last_change":"2023-10-09T11:07:33.244131+0000","last_active":"2023-10-09T11:07:51.029774+0000","last_peered":"2023-10-09T11:07:51.029774+0000","last_clean":"2023-10-09T11:07:51.029774+0000","last_became_active":"2023-10-09T11:07:33.243797+0000","last_became_peered":"2023-10-09T11:07:33.243797+0000","last_unstale":"2023-10-09T11:07:51.029774+0000","last_undegraded":"2023-10-09T11:07:51.029774+0000","last_fullsized":"2023-10-09T11:07:51.029774+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:44.851043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"557'7196","reported_seq":8664,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T11:08:06.532762+0000","last_change":"2023-10-09T11:07:29.087418+0000","last_active":"2023-10-09T11:08:06.532762+0000","last_peered":"2023-10-09T11:08:06.532762+0000","last_clean":"2023-10-09T11:08:06.532762+0000","last_became_active":"2023-10-09T11:05:46.357677+0000","last_became_peered":"2023-10-09T11:05:46.357677+0000","last_unstale":"2023-10-09T11:08:06.532762+0000","last_undegraded":"2023-10-09T11:08:06.532762+0000","last_fullsized":"2023-10-09T11:08:06.532762+0000","mapping_epoch":510,"log_start":"498'745","ondisk_log_start":"498'745","created":495,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6451,"log_dups_size":0,"ondisk_log_size":6451,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34668544,"num_objects":529,"num_object_clones":0,"num_object_copies":1587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5643,"num_write_kb":197504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":677,"num_bytes_recovered":44367872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":1067,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.026895+0000","last_change":"2023-10-09T11:07:36.286417+0000","last_active":"2023-10-09T11:07:51.026895+0000","last_peered":"2023-10-09T11:07:51.026895+0000","last_clean":"2023-10-09T11:07:51.026895+0000","last_became_active":"2023-10-09T11:07:36.286061+0000","last_became_peered":"2023-10-09T11:07:36.286061+0000","last_unstale":"2023-10-09T11:07:51.026895+0000","last_undegraded":"2023-10-09T11:07:51.026895+0000","last_fullsized":"2023-10-09T11:07:51.026895+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:00.289500+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":415,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.022763+0000","last_change":"2023-10-09T11:05:20.309385+0000","last_active":"2023-10-09T11:07:51.022763+0000","last_peered":"2023-10-09T11:07:51.022763+0000","last_clean":"2023-10-09T11:07:51.022763+0000","last_became_active":"2023-10-09T11:05:20.309034+0000","last_became_peered":"2023-10-09T11:05:20.309034+0000","last_unstale":"2023-10-09T11:07:51.022763+0000","last_undegraded":"2023-10-09T11:07:51.022763+0000","last_fullsized":"2023-10-09T11:07:51.022763+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:31.857485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":348,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216475+0000","last_change":"2023-10-09T11:06:24.245966+0000","last_active":"2023-10-09T11:08:07.216475+0000","last_peered":"2023-10-09T11:08:07.216475+0000","last_clean":"2023-10-09T11:08:07.216475+0000","last_became_active":"2023-10-09T11:06:24.245310+0000","last_became_peered":"2023-10-09T11:06:24.245310+0000","last_unstale":"2023-10-09T11:08:07.216475+0000","last_undegraded":"2023-10-09T11:08:07.216475+0000","last_fullsized":"2023-10-09T11:08:07.216475+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:23.966200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.3","version":"559'6149","reported_seq":9191,"reported_epoch":559,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:08:08.229428+0000","last_change":"2023-10-09T11:07:34.314819+0000","last_active":"2023-10-09T11:08:08.229428+0000","last_peered":"2023-10-09T11:08:08.229428+0000","last_clean":"2023-10-09T11:07:32.185795+0000","last_became_active":"2023-10-09T11:07:34.308357+0000","last_became_peered":"2023-10-09T11:07:34.308357+0000","last_unstale":"2023-10-09T11:08:08.229428+0000","last_undegraded":"2023-10-09T11:08:08.229428+0000","last_fullsized":"2023-10-09T11:08:08.229428+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6149,"log_dups_size":0,"ondisk_log_size":6149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15548416,"num_objects":237,"num_object_clones":0,"num_object_copies":711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":237,"num_objects_unfound":0,"num_objects_dirty":237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2933,"num_write_kb":101456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1223,"num_bytes_recovered":72613888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":237}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":711,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.025833+0000","last_change":"2023-10-09T11:07:36.283174+0000","last_active":"2023-10-09T11:07:51.025833+0000","last_peered":"2023-10-09T11:07:51.025833+0000","last_clean":"2023-10-09T11:07:51.025833+0000","last_became_active":"2023-10-09T11:07:36.278007+0000","last_became_peered":"2023-10-09T11:07:36.278007+0000","last_unstale":"2023-10-09T11:07:51.025833+0000","last_undegraded":"2023-10-09T11:07:51.025833+0000","last_fullsized":"2023-10-09T11:07:51.025833+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:51.693969+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":89,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.212636+0000","last_change":"2023-10-09T11:04:50.529513+0000","last_active":"2023-10-09T11:08:07.212636+0000","last_peered":"2023-10-09T11:08:07.212636+0000","last_clean":"2023-10-09T11:08:07.212636+0000","last_became_active":"2023-10-09T11:04:50.339671+0000","last_became_peered":"2023-10-09T11:04:50.339671+0000","last_unstale":"2023-10-09T11:08:07.212636+0000","last_undegraded":"2023-10-09T11:08:07.212636+0000","last_fullsized":"2023-10-09T11:08:07.212636+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:24.473000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":94,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.211374+0000","last_change":"2023-10-09T11:05:20.313855+0000","last_active":"2023-10-09T11:08:07.211374+0000","last_peered":"2023-10-09T11:08:07.211374+0000","last_clean":"2023-10-09T11:08:07.211374+0000","last_became_active":"2023-10-09T11:05:20.313535+0000","last_became_peered":"2023-10-09T11:05:20.313535+0000","last_unstale":"2023-10-09T11:08:07.211374+0000","last_undegraded":"2023-10-09T11:08:07.211374+0000","last_fullsized":"2023-10-09T11:08:07.211374+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:25.266434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"558'5948","reported_seq":6041,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.590535+0000","last_change":"2023-10-09T11:06:42.991743+0000","last_active":"2023-10-09T11:08:07.590535+0000","last_peered":"2023-10-09T11:08:07.590535+0000","last_clean":"2023-10-09T11:08:07.590535+0000","last_became_active":"2023-10-09T11:06:42.989452+0000","last_became_peered":"2023-10-09T11:06:42.989452+0000","last_unstale":"2023-10-09T11:08:07.590535+0000","last_undegraded":"2023-10-09T11:08:07.590535+0000","last_fullsized":"2023-10-09T11:08:07.590535+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5948,"log_dups_size":0,"ondisk_log_size":5948,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13762560,"num_objects":210,"num_object_clones":0,"num_object_copies":630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2845,"num_write_kb":97760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":815,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.024952+0000","last_change":"2023-10-09T11:04:53.272323+0000","last_active":"2023-10-09T11:07:51.024952+0000","last_peered":"2023-10-09T11:07:51.024952+0000","last_clean":"2023-10-09T11:07:51.024952+0000","last_became_active":"2023-10-09T11:04:53.272010+0000","last_became_peered":"2023-10-09T11:04:53.272010+0000","last_unstale":"2023-10-09T11:07:51.024952+0000","last_undegraded":"2023-10-09T11:07:51.024952+0000","last_fullsized":"2023-10-09T11:07:51.024952+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:23.913605+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":582,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216126+0000","last_change":"2023-10-09T11:05:20.307499+0000","last_active":"2023-10-09T11:08:07.216126+0000","last_peered":"2023-10-09T11:08:07.216126+0000","last_clean":"2023-10-09T11:08:07.216126+0000","last_became_active":"2023-10-09T11:05:20.307198+0000","last_became_peered":"2023-10-09T11:05:20.307198+0000","last_unstale":"2023-10-09T11:08:07.216126+0000","last_undegraded":"2023-10-09T11:08:07.216126+0000","last_fullsized":"2023-10-09T11:08:07.216126+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:53.726844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":316,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.257755+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:07:51.257755+0000","last_peered":"2023-10-09T11:07:51.257755+0000","last_clean":"2023-10-09T11:07:51.257755+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:07:51.257755+0000","last_undegraded":"2023-10-09T11:07:51.257755+0000","last_fullsized":"2023-10-09T11:07:51.257755+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"558'6120","reported_seq":6962,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.653542+0000","last_change":"2023-10-09T11:07:20.806034+0000","last_active":"2023-10-09T11:08:07.653542+0000","last_peered":"2023-10-09T11:08:07.653542+0000","last_clean":"2023-10-09T11:08:07.653542+0000","last_became_active":"2023-10-09T11:06:42.992656+0000","last_became_peered":"2023-10-09T11:06:42.992656+0000","last_unstale":"2023-10-09T11:08:07.653542+0000","last_undegraded":"2023-10-09T11:08:07.653542+0000","last_fullsized":"2023-10-09T11:08:07.653542+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6120,"log_dups_size":0,"ondisk_log_size":6120,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17088512,"num_objects":260,"num_object_clones":0,"num_object_copies":780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2886,"num_write_kb":100720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":328,"num_bytes_recovered":21528576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":909,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216648+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:08:07.216648+0000","last_peered":"2023-10-09T11:08:07.216648+0000","last_clean":"2023-10-09T11:08:07.216648+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:08:07.216648+0000","last_undegraded":"2023-10-09T11:08:07.216648+0000","last_fullsized":"2023-10-09T11:08:07.216648+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":579,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.054134+0000","last_change":"2023-10-09T11:04:57.953307+0000","last_active":"2023-10-09T11:07:51.054134+0000","last_peered":"2023-10-09T11:07:51.054134+0000","last_clean":"2023-10-09T11:07:51.054134+0000","last_became_active":"2023-10-09T11:04:57.953095+0000","last_became_peered":"2023-10-09T11:04:57.953095+0000","last_unstale":"2023-10-09T11:07:51.054134+0000","last_undegraded":"2023-10-09T11:07:51.054134+0000","last_fullsized":"2023-10-09T11:07:51.054134+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:05.562643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":267,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.217128+0000","last_change":"2023-10-09T11:05:20.310255+0000","last_active":"2023-10-09T11:08:07.217128+0000","last_peered":"2023-10-09T11:08:07.217128+0000","last_clean":"2023-10-09T11:08:07.217128+0000","last_became_active":"2023-10-09T11:05:20.309109+0000","last_became_peered":"2023-10-09T11:05:20.309109+0000","last_unstale":"2023-10-09T11:08:07.217128+0000","last_undegraded":"2023-10-09T11:08:07.217128+0000","last_fullsized":"2023-10-09T11:08:07.217128+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:07.429039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.0","version":"559'5988","reported_seq":8904,"reported_epoch":559,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T11:08:08.339961+0000","last_change":"2023-10-09T11:08:06.823204+0000","last_active":"2023-10-09T11:08:08.339961+0000","last_peered":"2023-10-09T11:08:08.339961+0000","last_clean":"2023-10-09T11:07:32.155021+0000","last_became_active":"2023-10-09T11:07:34.321010+0000","last_became_peered":"2023-10-09T11:07:34.321010+0000","last_unstale":"2023-10-09T11:08:08.339961+0000","last_undegraded":"2023-10-09T11:07:34.238729+0000","last_fullsized":"2023-10-09T11:07:34.219306+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5988,"log_dups_size":0,"ondisk_log_size":5988,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19808256,"num_objects":302,"num_object_clones":0,"num_object_copies":906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":475,"num_objects_misplaced":15,"num_objects_unfound":0,"num_objects_dirty":302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2822,"num_write_kb":99984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":88866816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,3","objects":-188},{"shards":"1,3,6","objects":15}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":799,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.022515+0000","last_change":"2023-10-09T11:05:23.329329+0000","last_active":"2023-10-09T11:07:51.022515+0000","last_peered":"2023-10-09T11:07:51.022515+0000","last_clean":"2023-10-09T11:07:51.022515+0000","last_became_active":"2023-10-09T11:05:23.328976+0000","last_became_peered":"2023-10-09T11:05:23.328976+0000","last_unstale":"2023-10-09T11:07:51.022515+0000","last_undegraded":"2023-10-09T11:07:51.022515+0000","last_fullsized":"2023-10-09T11:07:51.022515+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:41.848184+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":103,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.210097+0000","last_change":"2023-10-09T11:07:33.236133+0000","last_active":"2023-10-09T11:08:07.210097+0000","last_peered":"2023-10-09T11:08:07.210097+0000","last_clean":"2023-10-09T11:08:07.210097+0000","last_became_active":"2023-10-09T11:07:33.235818+0000","last_became_peered":"2023-10-09T11:07:33.235818+0000","last_unstale":"2023-10-09T11:08:07.210097+0000","last_undegraded":"2023-10-09T11:08:07.210097+0000","last_fullsized":"2023-10-09T11:08:07.210097+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:20.627846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":264,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216979+0000","last_change":"2023-10-09T11:04:50.332978+0000","last_active":"2023-10-09T11:08:07.216979+0000","last_peered":"2023-10-09T11:08:07.216979+0000","last_clean":"2023-10-09T11:08:07.216979+0000","last_became_active":"2023-10-09T11:04:50.332735+0000","last_became_peered":"2023-10-09T11:04:50.332735+0000","last_unstale":"2023-10-09T11:08:07.216979+0000","last_undegraded":"2023-10-09T11:08:07.216979+0000","last_fullsized":"2023-10-09T11:08:07.216979+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:39:44.659751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"557'8934","reported_seq":11857,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T11:08:06.234745+0000","last_change":"2023-10-09T11:07:21.753744+0000","last_active":"2023-10-09T11:08:06.234745+0000","last_peered":"2023-10-09T11:08:06.234745+0000","last_clean":"2023-10-09T11:08:06.234745+0000","last_became_active":"2023-10-09T11:06:42.979233+0000","last_became_peered":"2023-10-09T11:06:42.979233+0000","last_unstale":"2023-10-09T11:08:06.234745+0000","last_undegraded":"2023-10-09T11:08:06.234745+0000","last_fullsized":"2023-10-09T11:08:06.234745+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":8934,"log_dups_size":0,"ondisk_log_size":8934,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37519372,"num_objects":573,"num_object_clones":0,"num_object_copies":1719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5713,"num_write_kb":201120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":705,"num_bytes_recovered":46137344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":1085,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219776+0000","last_change":"2023-10-09T11:07:36.291716+0000","last_active":"2023-10-09T11:08:07.219776+0000","last_peered":"2023-10-09T11:08:07.219776+0000","last_clean":"2023-10-09T11:08:07.219776+0000","last_became_active":"2023-10-09T11:07:36.291424+0000","last_became_peered":"2023-10-09T11:07:36.291424+0000","last_unstale":"2023-10-09T11:08:07.219776+0000","last_undegraded":"2023-10-09T11:08:07.219776+0000","last_fullsized":"2023-10-09T11:08:07.219776+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:22.316387+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":288,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216557+0000","last_change":"2023-10-09T11:07:33.226547+0000","last_active":"2023-10-09T11:08:07.216557+0000","last_peered":"2023-10-09T11:08:07.216557+0000","last_clean":"2023-10-09T11:08:07.216557+0000","last_became_active":"2023-10-09T11:07:33.226176+0000","last_became_peered":"2023-10-09T11:07:33.226176+0000","last_unstale":"2023-10-09T11:08:07.216557+0000","last_undegraded":"2023-10-09T11:08:07.216557+0000","last_fullsized":"2023-10-09T11:08:07.216557+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:16.888732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":432,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.024988+0000","last_change":"2023-10-09T11:04:50.341353+0000","last_active":"2023-10-09T11:07:51.024988+0000","last_peered":"2023-10-09T11:07:51.024988+0000","last_clean":"2023-10-09T11:07:51.024988+0000","last_became_active":"2023-10-09T11:04:50.340801+0000","last_became_peered":"2023-10-09T11:04:50.340801+0000","last_unstale":"2023-10-09T11:07:51.024988+0000","last_undegraded":"2023-10-09T11:07:51.024988+0000","last_fullsized":"2023-10-09T11:07:51.024988+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:00.699069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":240,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216260+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:08:07.216260+0000","last_peered":"2023-10-09T11:08:07.216260+0000","last_clean":"2023-10-09T11:08:07.216260+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:08:07.216260+0000","last_undegraded":"2023-10-09T11:08:07.216260+0000","last_fullsized":"2023-10-09T11:08:07.216260+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"559'8794","reported_seq":8878,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.289248+0000","last_change":"2023-10-09T11:06:43.027139+0000","last_active":"2023-10-09T11:08:08.289248+0000","last_peered":"2023-10-09T11:08:08.289248+0000","last_clean":"2023-10-09T11:08:08.289248+0000","last_became_active":"2023-10-09T11:06:43.024164+0000","last_became_peered":"2023-10-09T11:06:43.024164+0000","last_unstale":"2023-10-09T11:08:08.289248+0000","last_undegraded":"2023-10-09T11:08:08.289248+0000","last_fullsized":"2023-10-09T11:08:08.289248+0000","mapping_epoch":495,"log_start":"504'1744","ondisk_log_start":"504'1744","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7050,"log_dups_size":0,"ondisk_log_size":7050,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36634624,"num_objects":559,"num_object_clones":0,"num_object_copies":1677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5624,"num_write_kb":197856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":757,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.022174+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:07:51.022174+0000","last_peered":"2023-10-09T11:07:51.022174+0000","last_clean":"2023-10-09T11:07:51.022174+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:07:51.022174+0000","last_undegraded":"2023-10-09T11:07:51.022174+0000","last_fullsized":"2023-10-09T11:07:51.022174+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":439,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.022264+0000","last_change":"2023-10-09T11:05:20.316093+0000","last_active":"2023-10-09T11:07:51.022264+0000","last_peered":"2023-10-09T11:07:51.022264+0000","last_clean":"2023-10-09T11:07:51.022264+0000","last_became_active":"2023-10-09T11:05:20.315471+0000","last_became_peered":"2023-10-09T11:05:20.315471+0000","last_unstale":"2023-10-09T11:07:51.022264+0000","last_undegraded":"2023-10-09T11:07:51.022264+0000","last_fullsized":"2023-10-09T11:07:51.022264+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:47.548763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":463,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.029822+0000","last_change":"2023-10-09T11:06:24.245282+0000","last_active":"2023-10-09T11:07:51.029822+0000","last_peered":"2023-10-09T11:07:51.029822+0000","last_clean":"2023-10-09T11:07:51.029822+0000","last_became_active":"2023-10-09T11:06:24.244981+0000","last_became_peered":"2023-10-09T11:06:24.244981+0000","last_unstale":"2023-10-09T11:07:51.029822+0000","last_undegraded":"2023-10-09T11:07:51.029822+0000","last_fullsized":"2023-10-09T11:07:51.029822+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:15.327655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":273,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216293+0000","last_change":"2023-10-09T11:04:50.311579+0000","last_active":"2023-10-09T11:08:07.216293+0000","last_peered":"2023-10-09T11:08:07.216293+0000","last_clean":"2023-10-09T11:08:07.216293+0000","last_became_active":"2023-10-09T11:04:50.311270+0000","last_became_peered":"2023-10-09T11:04:50.311270+0000","last_unstale":"2023-10-09T11:08:07.216293+0000","last_undegraded":"2023-10-09T11:08:07.216293+0000","last_fullsized":"2023-10-09T11:08:07.216293+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:49.850302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"559'7140","reported_seq":9716,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.277191+0000","last_change":"2023-10-09T11:06:09.940480+0000","last_active":"2023-10-09T11:08:08.277191+0000","last_peered":"2023-10-09T11:08:08.277191+0000","last_clean":"2023-10-09T11:08:08.277191+0000","last_became_active":"2023-10-09T11:05:46.348515+0000","last_became_peered":"2023-10-09T11:05:46.348515+0000","last_unstale":"2023-10-09T11:08:08.277191+0000","last_undegraded":"2023-10-09T11:08:08.277191+0000","last_fullsized":"2023-10-09T11:08:08.277191+0000","mapping_epoch":510,"log_start":"504'1670","ondisk_log_start":"504'1670","created":495,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5470,"log_dups_size":0,"ondisk_log_size":5470,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33357824,"num_objects":509,"num_object_clones":0,"num_object_copies":1527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5605,"num_write_kb":195648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":654,"num_bytes_recovered":42893312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":909,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.026260+0000","last_change":"2023-10-09T11:07:36.284280+0000","last_active":"2023-10-09T11:07:51.026260+0000","last_peered":"2023-10-09T11:07:51.026260+0000","last_clean":"2023-10-09T11:07:51.026260+0000","last_became_active":"2023-10-09T11:07:36.283966+0000","last_became_peered":"2023-10-09T11:07:36.283966+0000","last_unstale":"2023-10-09T11:07:51.026260+0000","last_undegraded":"2023-10-09T11:07:51.026260+0000","last_fullsized":"2023-10-09T11:07:51.026260+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:54.128331+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":335,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.054371+0000","last_change":"2023-10-09T11:04:50.381744+0000","last_active":"2023-10-09T11:07:51.054371+0000","last_peered":"2023-10-09T11:07:51.054371+0000","last_clean":"2023-10-09T11:07:51.054371+0000","last_became_active":"2023-10-09T11:04:50.381253+0000","last_became_peered":"2023-10-09T11:04:50.381253+0000","last_unstale":"2023-10-09T11:07:51.054371+0000","last_undegraded":"2023-10-09T11:07:51.054371+0000","last_fullsized":"2023-10-09T11:07:51.054371+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:24.030274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":258,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.218884+0000","last_change":"2023-10-09T11:05:20.298542+0000","last_active":"2023-10-09T11:08:07.218884+0000","last_peered":"2023-10-09T11:08:07.218884+0000","last_clean":"2023-10-09T11:08:07.218884+0000","last_became_active":"2023-10-09T11:05:20.298119+0000","last_became_peered":"2023-10-09T11:05:20.298119+0000","last_unstale":"2023-10-09T11:08:07.218884+0000","last_undegraded":"2023-10-09T11:08:07.218884+0000","last_fullsized":"2023-10-09T11:08:07.218884+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:00.827214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":238,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.217682+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:08:07.217682+0000","last_peered":"2023-10-09T11:08:07.217682+0000","last_clean":"2023-10-09T11:08:07.217682+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:08:07.217682+0000","last_undegraded":"2023-10-09T11:08:07.217682+0000","last_fullsized":"2023-10-09T11:08:07.217682+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"559'8885","reported_seq":11757,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.315620+0000","last_change":"2023-10-09T11:06:54.755841+0000","last_active":"2023-10-09T11:08:08.315620+0000","last_peered":"2023-10-09T11:08:08.315620+0000","last_clean":"2023-10-09T11:08:08.315620+0000","last_became_active":"2023-10-09T11:06:42.985753+0000","last_became_peered":"2023-10-09T11:06:42.985753+0000","last_unstale":"2023-10-09T11:08:08.315620+0000","last_undegraded":"2023-10-09T11:08:08.315620+0000","last_fullsized":"2023-10-09T11:08:08.315620+0000","mapping_epoch":513,"log_start":"503'1724","ondisk_log_start":"503'1724","created":495,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7161,"log_dups_size":0,"ondisk_log_size":7161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38600704,"num_objects":589,"num_object_clones":0,"num_object_copies":1767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5675,"num_write_kb":200448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":693,"num_bytes_recovered":45285376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":432,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.022697+0000","last_change":"2023-10-09T11:07:33.227052+0000","last_active":"2023-10-09T11:07:51.022697+0000","last_peered":"2023-10-09T11:07:51.022697+0000","last_clean":"2023-10-09T11:07:51.022697+0000","last_became_active":"2023-10-09T11:07:33.226289+0000","last_became_peered":"2023-10-09T11:07:33.226289+0000","last_unstale":"2023-10-09T11:07:51.022697+0000","last_undegraded":"2023-10-09T11:07:51.022697+0000","last_fullsized":"2023-10-09T11:07:51.022697+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:15.605946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":854,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.025822+0000","last_change":"2023-10-09T11:07:36.290700+0000","last_active":"2023-10-09T11:07:51.025822+0000","last_peered":"2023-10-09T11:07:51.025822+0000","last_clean":"2023-10-09T11:07:51.025822+0000","last_became_active":"2023-10-09T11:07:36.290357+0000","last_became_peered":"2023-10-09T11:07:36.290357+0000","last_unstale":"2023-10-09T11:07:51.025822+0000","last_undegraded":"2023-10-09T11:07:51.025822+0000","last_fullsized":"2023-10-09T11:07:51.025822+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:34.494777+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":94,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.213985+0000","last_change":"2023-10-09T11:05:20.310666+0000","last_active":"2023-10-09T11:08:07.213985+0000","last_peered":"2023-10-09T11:08:07.213985+0000","last_clean":"2023-10-09T11:08:07.213985+0000","last_became_active":"2023-10-09T11:05:20.310413+0000","last_became_peered":"2023-10-09T11:05:20.310413+0000","last_unstale":"2023-10-09T11:08:07.213985+0000","last_undegraded":"2023-10-09T11:08:07.213985+0000","last_fullsized":"2023-10-09T11:08:07.213985+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:34.169905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":517,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.027007+0000","last_change":"2023-10-09T11:05:20.316523+0000","last_active":"2023-10-09T11:07:51.027007+0000","last_peered":"2023-10-09T11:07:51.027007+0000","last_clean":"2023-10-09T11:07:51.027007+0000","last_became_active":"2023-10-09T11:05:20.314493+0000","last_became_peered":"2023-10-09T11:05:20.314493+0000","last_unstale":"2023-10-09T11:07:51.027007+0000","last_undegraded":"2023-10-09T11:07:51.027007+0000","last_fullsized":"2023-10-09T11:07:51.027007+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:41.618701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"559'7170","reported_seq":8561,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.220278+0000","last_change":"2023-10-09T11:06:34.311475+0000","last_active":"2023-10-09T11:08:08.220278+0000","last_peered":"2023-10-09T11:08:08.220278+0000","last_clean":"2023-10-09T11:08:08.220278+0000","last_became_active":"2023-10-09T11:05:46.356286+0000","last_became_peered":"2023-10-09T11:05:46.356286+0000","last_unstale":"2023-10-09T11:08:08.220278+0000","last_undegraded":"2023-10-09T11:08:08.220278+0000","last_fullsized":"2023-10-09T11:08:08.220278+0000","mapping_epoch":510,"log_start":"504'1671","ondisk_log_start":"504'1671","created":495,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5499,"log_dups_size":0,"ondisk_log_size":5499,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36864000,"num_objects":562,"num_object_clones":0,"num_object_copies":1686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5606,"num_write_kb":197408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":650,"num_bytes_recovered":42598400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":929,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216393+0000","last_change":"2023-10-09T11:05:23.322074+0000","last_active":"2023-10-09T11:08:07.216393+0000","last_peered":"2023-10-09T11:08:07.216393+0000","last_clean":"2023-10-09T11:08:07.216393+0000","last_became_active":"2023-10-09T11:05:23.321749+0000","last_became_peered":"2023-10-09T11:05:23.321749+0000","last_unstale":"2023-10-09T11:08:07.216393+0000","last_undegraded":"2023-10-09T11:08:07.216393+0000","last_fullsized":"2023-10-09T11:08:07.216393+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:07.010932+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":132,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.026060+0000","last_change":"2023-10-09T11:02:33.198484+0000","last_active":"2023-10-09T11:07:51.026060+0000","last_peered":"2023-10-09T11:07:51.026060+0000","last_clean":"2023-10-09T11:07:51.026060+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:07:51.026060+0000","last_undegraded":"2023-10-09T11:07:51.026060+0000","last_fullsized":"2023-10-09T11:07:51.026060+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":185,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.212624+0000","last_change":"2023-10-09T11:06:24.248362+0000","last_active":"2023-10-09T11:08:07.212624+0000","last_peered":"2023-10-09T11:08:07.212624+0000","last_clean":"2023-10-09T11:08:07.212624+0000","last_became_active":"2023-10-09T11:06:24.247348+0000","last_became_peered":"2023-10-09T11:06:24.247348+0000","last_unstale":"2023-10-09T11:08:07.212624+0000","last_undegraded":"2023-10-09T11:08:07.212624+0000","last_fullsized":"2023-10-09T11:08:07.212624+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:26.728754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":384,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216799+0000","last_change":"2023-10-09T11:06:24.253511+0000","last_active":"2023-10-09T11:08:07.216799+0000","last_peered":"2023-10-09T11:08:07.216799+0000","last_clean":"2023-10-09T11:08:07.216799+0000","last_became_active":"2023-10-09T11:06:24.252739+0000","last_became_peered":"2023-10-09T11:06:24.252739+0000","last_unstale":"2023-10-09T11:08:07.216799+0000","last_undegraded":"2023-10-09T11:08:07.216799+0000","last_fullsized":"2023-10-09T11:08:07.216799+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:50.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"559'8944","reported_seq":9028,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.337923+0000","last_change":"2023-10-09T11:06:42.983758+0000","last_active":"2023-10-09T11:08:08.337923+0000","last_peered":"2023-10-09T11:08:08.337923+0000","last_clean":"2023-10-09T11:08:08.337923+0000","last_became_active":"2023-10-09T11:06:42.981460+0000","last_became_peered":"2023-10-09T11:06:42.981460+0000","last_unstale":"2023-10-09T11:08:08.337923+0000","last_undegraded":"2023-10-09T11:08:08.337923+0000","last_fullsized":"2023-10-09T11:08:08.337923+0000","mapping_epoch":495,"log_start":"504'1919","ondisk_log_start":"504'1919","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7025,"log_dups_size":0,"ondisk_log_size":7025,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33259520,"num_objects":507,"num_object_clones":0,"num_object_copies":1521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5735,"num_write_kb":199776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":789,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219030+0000","last_change":"2023-10-09T11:06:24.239676+0000","last_active":"2023-10-09T11:08:07.219030+0000","last_peered":"2023-10-09T11:08:07.219030+0000","last_clean":"2023-10-09T11:08:07.219030+0000","last_became_active":"2023-10-09T11:06:24.239375+0000","last_became_peered":"2023-10-09T11:06:24.239375+0000","last_unstale":"2023-10-09T11:08:07.219030+0000","last_undegraded":"2023-10-09T11:08:07.219030+0000","last_fullsized":"2023-10-09T11:08:07.219030+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:27.953053+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.026619+0000","last_change":"2023-10-09T11:07:33.229415+0000","last_active":"2023-10-09T11:07:51.026619+0000","last_peered":"2023-10-09T11:07:51.026619+0000","last_clean":"2023-10-09T11:07:51.026619+0000","last_became_active":"2023-10-09T11:07:33.229061+0000","last_became_peered":"2023-10-09T11:07:33.229061+0000","last_unstale":"2023-10-09T11:07:51.026619+0000","last_undegraded":"2023-10-09T11:07:51.026619+0000","last_fullsized":"2023-10-09T11:07:51.026619+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:30.978293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":240,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216749+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:08:07.216749+0000","last_peered":"2023-10-09T11:08:07.216749+0000","last_clean":"2023-10-09T11:08:07.216749+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:08:07.216749+0000","last_undegraded":"2023-10-09T11:08:07.216749+0000","last_fullsized":"2023-10-09T11:08:07.216749+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":516,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.022614+0000","last_change":"2023-10-09T11:07:33.227271+0000","last_active":"2023-10-09T11:07:51.022614+0000","last_peered":"2023-10-09T11:07:51.022614+0000","last_clean":"2023-10-09T11:07:51.022614+0000","last_became_active":"2023-10-09T11:07:33.226760+0000","last_became_peered":"2023-10-09T11:07:33.226760+0000","last_unstale":"2023-10-09T11:07:51.022614+0000","last_undegraded":"2023-10-09T11:07:51.022614+0000","last_fullsized":"2023-10-09T11:07:51.022614+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:12.268123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"559'8950","reported_seq":11911,"reported_epoch":559,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:08:08.300381+0000","last_change":"2023-10-09T11:07:34.375236+0000","last_active":"2023-10-09T11:08:08.300381+0000","last_peered":"2023-10-09T11:08:08.300381+0000","last_clean":"2023-10-09T11:06:06.963741+0000","last_became_active":"2023-10-09T11:07:34.357802+0000","last_became_peered":"2023-10-09T11:07:34.357802+0000","last_unstale":"2023-10-09T11:08:08.300381+0000","last_undegraded":"2023-10-09T11:07:34.289955+0000","last_fullsized":"2023-10-09T11:07:34.281689+0000","mapping_epoch":550,"log_start":"504'1895","ondisk_log_start":"504'1895","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38010880,"num_objects":580,"num_object_clones":0,"num_object_copies":1740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2256,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5720,"num_write_kb":201600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1434,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1,2","objects":517},{"shards":"1,2","objects":-165}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":756,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.054958+0000","last_change":"2023-10-09T11:05:23.323368+0000","last_active":"2023-10-09T11:07:51.054958+0000","last_peered":"2023-10-09T11:07:51.054958+0000","last_clean":"2023-10-09T11:07:51.054958+0000","last_became_active":"2023-10-09T11:05:23.323032+0000","last_became_peered":"2023-10-09T11:05:23.323032+0000","last_unstale":"2023-10-09T11:07:51.054958+0000","last_undegraded":"2023-10-09T11:07:51.054958+0000","last_fullsized":"2023-10-09T11:07:51.054958+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:09.393584+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":115,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219617+0000","last_change":"2023-10-09T11:07:33.246378+0000","last_active":"2023-10-09T11:08:07.219617+0000","last_peered":"2023-10-09T11:08:07.219617+0000","last_clean":"2023-10-09T11:08:07.219617+0000","last_became_active":"2023-10-09T11:07:33.228019+0000","last_became_peered":"2023-10-09T11:07:33.228019+0000","last_unstale":"2023-10-09T11:08:07.219617+0000","last_undegraded":"2023-10-09T11:08:07.219617+0000","last_fullsized":"2023-10-09T11:08:07.219617+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:18.511517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":267,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.217271+0000","last_change":"2023-10-09T11:05:20.310417+0000","last_active":"2023-10-09T11:08:07.217271+0000","last_peered":"2023-10-09T11:08:07.217271+0000","last_clean":"2023-10-09T11:08:07.217271+0000","last_became_active":"2023-10-09T11:05:20.309827+0000","last_became_peered":"2023-10-09T11:05:20.309827+0000","last_unstale":"2023-10-09T11:08:07.217271+0000","last_undegraded":"2023-10-09T11:08:07.217271+0000","last_fullsized":"2023-10-09T11:08:07.217271+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:23.934383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":103,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.209969+0000","last_change":"2023-10-09T11:07:33.217595+0000","last_active":"2023-10-09T11:08:07.209969+0000","last_peered":"2023-10-09T11:08:07.209969+0000","last_clean":"2023-10-09T11:08:07.209969+0000","last_became_active":"2023-10-09T11:07:33.217155+0000","last_became_peered":"2023-10-09T11:07:33.217155+0000","last_unstale":"2023-10-09T11:08:07.209969+0000","last_undegraded":"2023-10-09T11:08:07.209969+0000","last_fullsized":"2023-10-09T11:08:07.209969+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:55.127329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"559'8842","reported_seq":8925,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.275715+0000","last_change":"2023-10-09T11:06:42.990792+0000","last_active":"2023-10-09T11:08:08.275715+0000","last_peered":"2023-10-09T11:08:08.275715+0000","last_clean":"2023-10-09T11:08:08.275715+0000","last_became_active":"2023-10-09T11:06:42.988050+0000","last_became_peered":"2023-10-09T11:06:42.988050+0000","last_unstale":"2023-10-09T11:08:08.275715+0000","last_undegraded":"2023-10-09T11:08:08.275715+0000","last_fullsized":"2023-10-09T11:08:08.275715+0000","mapping_epoch":495,"log_start":"504'1803","ondisk_log_start":"504'1803","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7039,"log_dups_size":0,"ondisk_log_size":7039,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33226752,"num_objects":507,"num_object_clones":0,"num_object_copies":1521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5667,"num_write_kb":197568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":722,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.211297+0000","last_change":"2023-10-09T11:07:36.284491+0000","last_active":"2023-10-09T11:08:07.211297+0000","last_peered":"2023-10-09T11:08:07.211297+0000","last_clean":"2023-10-09T11:08:07.211297+0000","last_became_active":"2023-10-09T11:07:36.284249+0000","last_became_peered":"2023-10-09T11:07:36.284249+0000","last_unstale":"2023-10-09T11:08:07.211297+0000","last_undegraded":"2023-10-09T11:08:07.211297+0000","last_fullsized":"2023-10-09T11:08:07.211297+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:03.653371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:03.653371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:09.629417+0000","scrub_duration":0.001576771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.027651+0000","last_change":"2023-10-09T11:07:33.241346+0000","last_active":"2023-10-09T11:07:51.027651+0000","last_peered":"2023-10-09T11:07:51.027651+0000","last_clean":"2023-10-09T11:07:51.027651+0000","last_became_active":"2023-10-09T11:07:33.241013+0000","last_became_peered":"2023-10-09T11:07:33.241013+0000","last_unstale":"2023-10-09T11:07:51.027651+0000","last_undegraded":"2023-10-09T11:07:51.027651+0000","last_fullsized":"2023-10-09T11:07:51.027651+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:57.164502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":257,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.217060+0000","last_change":"2023-10-09T11:06:24.253714+0000","last_active":"2023-10-09T11:08:07.217060+0000","last_peered":"2023-10-09T11:08:07.217060+0000","last_clean":"2023-10-09T11:08:07.217060+0000","last_became_active":"2023-10-09T11:06:24.253143+0000","last_became_peered":"2023-10-09T11:06:24.253143+0000","last_unstale":"2023-10-09T11:08:07.217060+0000","last_undegraded":"2023-10-09T11:08:07.217060+0000","last_fullsized":"2023-10-09T11:08:07.217060+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:05.682288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":362,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.022120+0000","last_change":"2023-10-09T11:05:20.313560+0000","last_active":"2023-10-09T11:07:51.022120+0000","last_peered":"2023-10-09T11:07:51.022120+0000","last_clean":"2023-10-09T11:07:51.022120+0000","last_became_active":"2023-10-09T11:05:20.313214+0000","last_became_peered":"2023-10-09T11:05:20.313214+0000","last_unstale":"2023-10-09T11:07:51.022120+0000","last_undegraded":"2023-10-09T11:07:51.022120+0000","last_fullsized":"2023-10-09T11:07:51.022120+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:32.527205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"557'7373","reported_seq":9163,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T11:08:06.581885+0000","last_change":"2023-10-09T11:07:34.333312+0000","last_active":"2023-10-09T11:08:06.581885+0000","last_peered":"2023-10-09T11:08:06.581885+0000","last_clean":"2023-10-09T11:08:06.581885+0000","last_became_active":"2023-10-09T11:07:34.285534+0000","last_became_peered":"2023-10-09T11:07:34.285534+0000","last_unstale":"2023-10-09T11:08:06.581885+0000","last_undegraded":"2023-10-09T11:08:06.581885+0000","last_fullsized":"2023-10-09T11:08:06.581885+0000","mapping_epoch":550,"log_start":"498'1019","ondisk_log_start":"498'1019","created":509,"last_epoch_clean":551,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6354,"log_dups_size":0,"ondisk_log_size":6354,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38633472,"num_objects":590,"num_object_clones":0,"num_object_copies":1770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5782,"num_write_kb":203872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":838,"num_bytes_recovered":54853632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":966,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.212889+0000","last_change":"2023-10-09T11:05:23.329886+0000","last_active":"2023-10-09T11:08:07.212889+0000","last_peered":"2023-10-09T11:08:07.212889+0000","last_clean":"2023-10-09T11:08:07.212889+0000","last_became_active":"2023-10-09T11:05:23.329561+0000","last_became_peered":"2023-10-09T11:05:23.329561+0000","last_unstale":"2023-10-09T11:08:07.212889+0000","last_undegraded":"2023-10-09T11:08:07.212889+0000","last_fullsized":"2023-10-09T11:08:07.212889+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:46.933799+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":17,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.026866+0000","last_change":"2023-10-09T11:07:33.229864+0000","last_active":"2023-10-09T11:07:51.026866+0000","last_peered":"2023-10-09T11:07:51.026866+0000","last_clean":"2023-10-09T11:07:51.026866+0000","last_became_active":"2023-10-09T11:07:33.229547+0000","last_became_peered":"2023-10-09T11:07:33.229547+0000","last_unstale":"2023-10-09T11:07:51.026866+0000","last_undegraded":"2023-10-09T11:07:51.026866+0000","last_fullsized":"2023-10-09T11:07:51.026866+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:34.990439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":339,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.056440+0000","last_change":"2023-10-09T11:04:50.510688+0000","last_active":"2023-10-09T11:07:51.056440+0000","last_peered":"2023-10-09T11:07:51.056440+0000","last_clean":"2023-10-09T11:07:51.056440+0000","last_became_active":"2023-10-09T11:04:50.507404+0000","last_became_peered":"2023-10-09T11:04:50.507404+0000","last_unstale":"2023-10-09T11:07:51.056440+0000","last_undegraded":"2023-10-09T11:07:51.056440+0000","last_fullsized":"2023-10-09T11:07:51.056440+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:13.663394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"557'7151","reported_seq":8596,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T11:08:06.496507+0000","last_change":"2023-10-09T11:07:26.133461+0000","last_active":"2023-10-09T11:08:06.496507+0000","last_peered":"2023-10-09T11:08:06.496507+0000","last_clean":"2023-10-09T11:08:06.496507+0000","last_became_active":"2023-10-09T11:05:45.387324+0000","last_became_peered":"2023-10-09T11:05:45.387324+0000","last_unstale":"2023-10-09T11:08:06.496507+0000","last_undegraded":"2023-10-09T11:08:06.496507+0000","last_fullsized":"2023-10-09T11:08:06.496507+0000","mapping_epoch":509,"log_start":"498'672","ondisk_log_start":"498'672","created":509,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6479,"log_dups_size":0,"ondisk_log_size":6479,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37355520,"num_objects":570,"num_object_clones":0,"num_object_copies":1710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5598,"num_write_kb":197376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":667,"num_bytes_recovered":43712512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":923,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219354+0000","last_change":"2023-10-09T11:05:23.321040+0000","last_active":"2023-10-09T11:08:07.219354+0000","last_peered":"2023-10-09T11:08:07.219354+0000","last_clean":"2023-10-09T11:08:07.219354+0000","last_became_active":"2023-10-09T11:05:23.320674+0000","last_became_peered":"2023-10-09T11:05:23.320674+0000","last_unstale":"2023-10-09T11:08:07.219354+0000","last_undegraded":"2023-10-09T11:08:07.219354+0000","last_fullsized":"2023-10-09T11:08:07.219354+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.527660+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":384,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.022685+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:07:51.022685+0000","last_peered":"2023-10-09T11:07:51.022685+0000","last_clean":"2023-10-09T11:07:51.022685+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:07:51.022685+0000","last_undegraded":"2023-10-09T11:07:51.022685+0000","last_fullsized":"2023-10-09T11:07:51.022685+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.212556+0000","last_change":"2023-10-09T11:04:50.511984+0000","last_active":"2023-10-09T11:08:07.212556+0000","last_peered":"2023-10-09T11:08:07.212556+0000","last_clean":"2023-10-09T11:08:07.212556+0000","last_became_active":"2023-10-09T11:04:50.510820+0000","last_became_peered":"2023-10-09T11:04:50.510820+0000","last_unstale":"2023-10-09T11:08:07.212556+0000","last_undegraded":"2023-10-09T11:08:07.212556+0000","last_fullsized":"2023-10-09T11:08:07.212556+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:38.662189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.15","version":"559'7091","reported_seq":11772,"reported_epoch":559,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:08:08.285980+0000","last_change":"2023-10-09T11:07:34.356257+0000","last_active":"2023-10-09T11:08:08.285980+0000","last_peered":"2023-10-09T11:08:08.285980+0000","last_clean":"2023-10-09T11:07:32.098315+0000","last_became_active":"2023-10-09T11:07:34.338915+0000","last_became_peered":"2023-10-09T11:07:34.338915+0000","last_unstale":"2023-10-09T11:08:08.285980+0000","last_undegraded":"2023-10-09T11:08:08.285980+0000","last_fullsized":"2023-10-09T11:08:08.285980+0000","mapping_epoch":550,"log_start":"504'1630","ondisk_log_start":"504'1630","created":509,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5461,"log_dups_size":0,"ondisk_log_size":5461,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32210944,"num_objects":492,"num_object_clones":0,"num_object_copies":1476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":492,"num_objects_unfound":0,"num_objects_dirty":492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5574,"num_write_kb":194080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2280,"num_bytes_recovered":149422080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":492}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":535,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.024901+0000","last_change":"2023-10-09T11:05:20.327032+0000","last_active":"2023-10-09T11:07:51.024901+0000","last_peered":"2023-10-09T11:07:51.024901+0000","last_clean":"2023-10-09T11:07:51.024901+0000","last_became_active":"2023-10-09T11:05:20.320193+0000","last_became_peered":"2023-10-09T11:05:20.320193+0000","last_unstale":"2023-10-09T11:07:51.024901+0000","last_undegraded":"2023-10-09T11:07:51.024901+0000","last_fullsized":"2023-10-09T11:07:51.024901+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:44.862890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.213812+0000","last_change":"2023-10-09T11:04:50.384219+0000","last_active":"2023-10-09T11:08:07.213812+0000","last_peered":"2023-10-09T11:08:07.213812+0000","last_clean":"2023-10-09T11:08:07.213812+0000","last_became_active":"2023-10-09T11:04:50.384028+0000","last_became_peered":"2023-10-09T11:04:50.384028+0000","last_unstale":"2023-10-09T11:08:07.213812+0000","last_undegraded":"2023-10-09T11:08:07.213812+0000","last_fullsized":"2023-10-09T11:08:07.213812+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:04.721869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.16","version":"557'7189","reported_seq":7283,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T11:08:06.244226+0000","last_change":"2023-10-09T11:07:24.733266+0000","last_active":"2023-10-09T11:08:06.244226+0000","last_peered":"2023-10-09T11:08:06.244226+0000","last_clean":"2023-10-09T11:08:06.244226+0000","last_became_active":"2023-10-09T11:06:24.254366+0000","last_became_peered":"2023-10-09T11:06:24.254366+0000","last_unstale":"2023-10-09T11:08:06.244226+0000","last_undegraded":"2023-10-09T11:08:06.244226+0000","last_fullsized":"2023-10-09T11:08:06.244226+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7189,"log_dups_size":0,"ondisk_log_size":7189,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36110336,"num_objects":551,"num_object_clones":0,"num_object_copies":1653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5663,"num_write_kb":198848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":730,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.027515+0000","last_change":"2023-10-09T11:07:36.291401+0000","last_active":"2023-10-09T11:07:51.027515+0000","last_peered":"2023-10-09T11:07:51.027515+0000","last_clean":"2023-10-09T11:07:51.027515+0000","last_became_active":"2023-10-09T11:07:36.291077+0000","last_became_peered":"2023-10-09T11:07:36.291077+0000","last_unstale":"2023-10-09T11:07:51.027515+0000","last_undegraded":"2023-10-09T11:07:51.027515+0000","last_fullsized":"2023-10-09T11:07:51.027515+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:04.790935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.790935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":94,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219433+0000","last_change":"2023-10-09T11:05:20.299321+0000","last_active":"2023-10-09T11:08:07.219433+0000","last_peered":"2023-10-09T11:08:07.219433+0000","last_clean":"2023-10-09T11:08:07.219433+0000","last_became_active":"2023-10-09T11:05:20.299019+0000","last_became_peered":"2023-10-09T11:05:20.299019+0000","last_unstale":"2023-10-09T11:08:07.219433+0000","last_undegraded":"2023-10-09T11:08:07.219433+0000","last_fullsized":"2023-10-09T11:08:07.219433+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:08.645909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":85,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.212913+0000","last_change":"2023-10-09T11:04:50.366890+0000","last_active":"2023-10-09T11:08:07.212913+0000","last_peered":"2023-10-09T11:08:07.212913+0000","last_clean":"2023-10-09T11:08:07.212913+0000","last_became_active":"2023-10-09T11:04:50.366558+0000","last_became_peered":"2023-10-09T11:04:50.366558+0000","last_unstale":"2023-10-09T11:08:07.212913+0000","last_undegraded":"2023-10-09T11:08:07.212913+0000","last_fullsized":"2023-10-09T11:08:07.212913+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:50.993153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"557'7178","reported_seq":7265,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-09T11:08:06.733797+0000","last_change":"2023-10-09T11:08:03.776503+0000","last_active":"2023-10-09T11:08:06.733797+0000","last_peered":"2023-10-09T11:08:06.733797+0000","last_clean":"2023-10-09T11:08:06.733797+0000","last_became_active":"2023-10-09T11:05:45.356981+0000","last_became_peered":"2023-10-09T11:05:45.356981+0000","last_unstale":"2023-10-09T11:08:06.733797+0000","last_undegraded":"2023-10-09T11:08:06.733797+0000","last_fullsized":"2023-10-09T11:08:06.733797+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7178,"log_dups_size":0,"ondisk_log_size":7178,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36208640,"num_objects":553,"num_object_clones":0,"num_object_copies":1659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5609,"num_write_kb":197152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":858,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216604+0000","last_change":"2023-10-09T11:04:53.274332+0000","last_active":"2023-10-09T11:08:07.216604+0000","last_peered":"2023-10-09T11:08:07.216604+0000","last_clean":"2023-10-09T11:08:07.216604+0000","last_became_active":"2023-10-09T11:04:53.273798+0000","last_became_peered":"2023-10-09T11:04:53.273798+0000","last_unstale":"2023-10-09T11:08:07.216604+0000","last_undegraded":"2023-10-09T11:08:07.216604+0000","last_fullsized":"2023-10-09T11:08:07.216604+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:04.220970+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":104,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-09T11:07:51.025474+0000","last_change":"2023-10-09T11:06:24.249877+0000","last_active":"2023-10-09T11:07:51.025474+0000","last_peered":"2023-10-09T11:07:51.025474+0000","last_clean":"2023-10-09T11:07:51.025474+0000","last_became_active":"2023-10-09T11:06:24.249573+0000","last_became_peered":"2023-10-09T11:06:24.249573+0000","last_unstale":"2023-10-09T11:07:51.025474+0000","last_undegraded":"2023-10-09T11:07:51.025474+0000","last_fullsized":"2023-10-09T11:07:51.025474+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:11.798192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":607,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219787+0000","last_change":"2023-10-09T11:05:20.307347+0000","last_active":"2023-10-09T11:08:07.219787+0000","last_peered":"2023-10-09T11:08:07.219787+0000","last_clean":"2023-10-09T11:08:07.219787+0000","last_became_active":"2023-10-09T11:05:20.307126+0000","last_became_peered":"2023-10-09T11:05:20.307126+0000","last_unstale":"2023-10-09T11:08:07.219787+0000","last_undegraded":"2023-10-09T11:08:07.219787+0000","last_fullsized":"2023-10-09T11:08:07.219787+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:34.408515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":36,"stat_sum":{"num_bytes":1124794392,"num_objects":17164,"num_object_clones":0,"num_object_copies":51492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5000,"num_objects_misplaced":1852,"num_objects_unfound":0,"num_objects_dirty":17164,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":181830,"num_write_kb":6367745,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23976,"num_bytes_recovered":1466040320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4183306240,"data_stored":4183294024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":233317,"ondisk_log_size":233317,"up":108,"acting":104,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":91,"num_bytes_recovered":15742,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12694,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":626148,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":31,"num_read_kb":56,"num_write":15,"num_write_kb":147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2632652,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332498,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1768280,"kb_used_data":868880,"kb_used_omap":0,"kb_used_meta":899392,"kb_avail":91976872,"statfs":{"total":95995035648,"available":94184316928,"internally_reserved":0,"allocated":889733120,"data_stored":887939738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920977408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":1.5009999999999999,"15min":1.6839999999999999},"min":{"1min":0.27300000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":5.2539999999999996,"5min":9.8040000000000003,"15min":9.8040000000000003},"last":0.254},{"interface":"front","average":{"1min":1.7989999999999999,"5min":1.7170000000000001,"15min":1.746},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.216},"max":{"1min":15.853,"5min":15.853,"15min":15.853},"last":0.53400000000000003}]},{"osd":1,"last update":"Mon Oct 9 11:07:23 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":1.3959999999999999,"15min":1.52},"min":{"1min":0.23799999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.9130000000000003,"5min":11.619999999999999,"15min":11.619999999999999},"last":1.698},{"interface":"front","average":{"1min":2.0019999999999998,"5min":1.8500000000000001,"15min":1.891},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":11.038,"5min":11.038,"15min":12.193},"last":0.56799999999999995}]},{"osd":2,"last update":"Mon Oct 9 11:07:52 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":1.929,"15min":1.7170000000000001},"min":{"1min":0.313,"5min":0.309,"15min":0.245},"max":{"1min":3.3050000000000002,"5min":16.605,"15min":16.605},"last":1.0029999999999999},{"interface":"front","average":{"1min":1.466,"5min":1.4570000000000001,"15min":1.706},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":4.9569999999999999,"5min":9.7469999999999999,"15min":13.037000000000001},"last":2.984}]},{"osd":3,"last update":"Mon Oct 9 11:07:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.524,"15min":1.4399999999999999},"min":{"1min":0.30499999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.4770000000000003,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":2.3919999999999999},{"interface":"front","average":{"1min":2.1789999999999998,"5min":1.9019999999999999,"15min":1.8280000000000001},"min":{"1min":0.38200000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.968,"5min":8.2330000000000005,"15min":8.2330000000000005},"last":1.1020000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":0.91800000000000004,"15min":1.0940000000000001},"min":{"1min":0.29799999999999999,"5min":0.218,"15min":0.218},"max":{"1min":7.4630000000000001,"5min":7.4630000000000001,"15min":8.2940000000000005},"last":0.499},{"interface":"front","average":{"1min":1.034,"5min":1.256,"15min":1.244},"min":{"1min":0.30599999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.7190000000000003,"5min":11.808,"15min":16.481000000000002},"last":0.80700000000000005}]},{"osd":5,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.0640000000000001,"15min":1.252},"min":{"1min":0.32800000000000001,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":5.4800000000000004,"5min":15.414999999999999,"15min":15.414999999999999},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.345,"15min":1.198},"min":{"1min":0.375,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.045,"5min":15.369,"15min":15.369},"last":1.0169999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.226,"15min":1.26},"min":{"1min":0.34599999999999997,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":3.8500000000000001,"5min":14.159000000000001,"15min":14.159000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.121,"5min":1.472,"15min":1.355},"min":{"1min":0.36099999999999999,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":3.5830000000000002,"5min":13.993,"15min":16.538},"last":1.8819999999999999}]}]},{"osd":6,"up_from":44,"seq":188978561429,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2215052,"kb_used_data":718404,"kb_used_omap":0,"kb_used_meta":1496640,"kb_avail":91530100,"statfs":{"total":95995035648,"available":93726822400,"internally_reserved":0,"allocated":735645696,"data_stored":733863936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1532559360},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.6719999999999999,"15min":1.875},"min":{"1min":0.23899999999999999,"5min":0.217,"15min":0.217},"max":{"1min":3.7389999999999999,"5min":29.379999999999999,"15min":54.743000000000002},"last":2.8820000000000001},{"interface":"front","average":{"1min":1.3120000000000001,"5min":1.6200000000000001,"15min":1.9299999999999999},"min":{"1min":0.247,"5min":0.214,"15min":0.214},"max":{"1min":7.1890000000000001,"5min":29.132999999999999,"15min":53.701000000000001},"last":0.36099999999999999}]},{"osd":1,"last update":"Mon Oct 9 11:07:26 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":1.5369999999999999,"15min":1.1679999999999999},"min":{"1min":0.318,"5min":0.26000000000000001,"15min":0.23499999999999999},"max":{"1min":5.0110000000000001,"5min":16.225999999999999,"15min":41.241999999999997},"last":0.27100000000000002},{"interface":"front","average":{"1min":2.98,"5min":2.0489999999999999,"15min":1.7350000000000001},"min":{"1min":0.31,"5min":0.26200000000000001,"15min":0.248},"max":{"1min":9.173,"5min":11.715,"15min":41.476999999999997},"last":3.5169999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:07:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":2.1890000000000001,"15min":1.857},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.7089999999999996,"5min":10.481999999999999,"15min":12.003},"last":6.2069999999999999},{"interface":"front","average":{"1min":1.6990000000000001,"5min":1.764,"15min":1.8999999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.133,"5min":12.693,"15min":15.432},"last":0.56699999999999995}]},{"osd":3,"last update":"Mon Oct 9 11:07:23 2023","interfaces":[{"interface":"back","average":{"1min":3.125,"5min":1.7629999999999999,"15min":1.554},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":28.948,"5min":28.948,"15min":28.948},"last":2.8100000000000001},{"interface":"front","average":{"1min":5.3739999999999997,"5min":2.9390000000000001,"15min":2.605},"min":{"1min":0.496,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":28.893000000000001,"5min":28.893000000000001,"15min":28.893000000000001},"last":3.4740000000000002}]},{"osd":4,"last update":"Mon Oct 9 11:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":1.171,"15min":1.0129999999999999},"min":{"1min":0.26500000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.0189999999999999,"5min":17.216999999999999,"15min":17.216999999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.78100000000000003,"15min":0.84299999999999997},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.24399999999999999},"max":{"1min":1.4059999999999999,"5min":10.025,"15min":10.025},"last":0.502}]},{"osd":5,"last update":"Mon Oct 9 11:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.1419999999999999,"15min":1.2709999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":1.103,"5min":10.209,"15min":19.786999999999999},"last":2.7000000000000002},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.90500000000000003,"15min":0.97099999999999997},"min":{"1min":0.33200000000000002,"5min":0.248,"15min":0.248},"max":{"1min":1.4810000000000001,"5min":9.9469999999999992,"15min":9.9469999999999992},"last":0.53900000000000003}]},{"osd":7,"last update":"Mon Oct 9 11:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":1.419,"15min":1.3169999999999999},"min":{"1min":0.34200000000000003,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.7210000000000001,"5min":11.550000000000001,"15min":12.882},"last":3.7250000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.85599999999999998,"15min":0.94099999999999995},"min":{"1min":0.32800000000000001,"5min":0.31,"15min":0.27500000000000002},"max":{"1min":1.643,"5min":11.557,"15min":11.557},"last":2.8479999999999999}]}]},{"osd":1,"up_from":388,"seq":1666447310944,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1027048,"kb_used_data":551712,"kb_used_omap":13,"kb_used_meta":475314,"kb_avail":92718104,"statfs":{"total":95995035648,"available":94943338496,"internally_reserved":0,"allocated":564953088,"data_stored":562000852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":486722247},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":3.895,"5min":2.593,"15min":2.4369999999999998},"min":{"1min":0.26700000000000002,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":11.960000000000001,"5min":18.192,"15min":18.192},"last":0.47299999999999998},{"interface":"front","average":{"1min":3.9390000000000001,"5min":2.8260000000000001,"15min":1.9259999999999999},"min":{"1min":0.27000000000000002,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":10.099,"5min":17.957999999999998,"15min":17.957999999999998},"last":0.65100000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8439999999999999,"5min":2.5920000000000001,"15min":2.9300000000000002},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":9.5630000000000006,"5min":47.731000000000002,"15min":47.731000000000002},"last":0.55700000000000005},{"interface":"front","average":{"1min":3.8199999999999998,"5min":2.903,"15min":2.4769999999999999},"min":{"1min":0.29399999999999998,"5min":0.216,"15min":0.216},"max":{"1min":10.138,"5min":47.786999999999999,"15min":47.786999999999999},"last":0.57299999999999995}]},{"osd":3,"last update":"Mon Oct 9 11:07:25 2023","interfaces":[{"interface":"back","average":{"1min":3.7330000000000001,"5min":3.0870000000000002,"15min":2.0209999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":20.791,"5min":92.447999999999993,"15min":92.447999999999993},"last":0.46200000000000002},{"interface":"front","average":{"1min":3.1160000000000001,"5min":3.1179999999999999,"15min":2.028},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.217},"max":{"1min":7.3630000000000004,"5min":92.066999999999993,"15min":92.066999999999993},"last":0.60799999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.7869999999999999,"15min":2.395},"min":{"1min":0.30499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.5199999999999996,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.04,"15min":1.6240000000000001},"min":{"1min":0.314,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":4.6079999999999997,"5min":11.589,"15min":11.589},"last":0.53300000000000003}]},{"osd":5,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":2.0830000000000002,"15min":2.3050000000000002},"min":{"1min":0.28999999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.0720000000000001,"5min":20.689,"15min":20.689},"last":0.622},{"interface":"front","average":{"1min":1.764,"5min":2.0590000000000002,"15min":2.2410000000000001},"min":{"1min":0.33800000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.4900000000000002,"5min":11.65,"15min":11.65},"last":0.63900000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.833,"5min":2.4049999999999998,"15min":2.411},"min":{"1min":0.312,"5min":0.309,"15min":0.309},"max":{"1min":6.0330000000000004,"5min":20.646999999999998,"15min":20.646999999999998},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.784,"5min":1.9219999999999999,"15min":2.1720000000000002},"min":{"1min":0.35599999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":5.5380000000000003,"5min":11.75,"15min":11.75},"last":0.67500000000000004}]},{"osd":7,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.869,"5min":2.1640000000000001,"15min":1.615},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.32900000000000001},"max":{"1min":4.5780000000000003,"5min":11.555999999999999,"15min":11.555999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":1.7789999999999999,"5min":2.0219999999999998,"15min":2.3210000000000002},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.1020000000000003,"5min":20.716000000000001,"15min":20.716000000000001},"last":0.69599999999999995}]}]},{"osd":0,"up_from":80,"seq":343597384002,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1401984,"kb_used_data":448376,"kb_used_omap":28,"kb_used_meta":953571,"kb_avail":92343168,"statfs":{"total":95995035648,"available":94559404032,"internally_reserved":0,"allocated":459137024,"data_stored":456093486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":976457552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:07:25 2023","interfaces":[{"interface":"back","average":{"1min":2.923,"5min":2.1680000000000001,"15min":2.831},"min":{"1min":0.22700000000000001,"5min":0.186,"15min":0.186},"max":{"1min":10.271000000000001,"5min":14.644,"15min":19.608000000000001},"last":0.191},{"interface":"front","average":{"1min":3.879,"5min":2.2949999999999999,"15min":2.911},"min":{"1min":0.29499999999999998,"5min":0.214,"15min":0.214},"max":{"1min":15.834,"5min":15.834,"15min":19.992999999999999},"last":0.223}]},{"osd":2,"last update":"Mon Oct 9 11:07:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":2.339,"15min":2.29},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.17199999999999999},"max":{"1min":11.516,"5min":12.459,"15min":24.852},"last":6.9630000000000001},{"interface":"front","average":{"1min":1.73,"5min":2.246,"15min":2.694},"min":{"1min":0.215,"5min":0.20300000000000001,"15min":0.19800000000000001},"max":{"1min":8.4589999999999996,"5min":11.68,"15min":77.256},"last":6.7910000000000004}]},{"osd":3,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":3.2730000000000001,"15min":3.4449999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":13.189,"5min":25.050000000000001,"15min":25.050000000000001},"last":1.7709999999999999},{"interface":"front","average":{"1min":2.98,"5min":2.2770000000000001,"15min":2.0249999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":14.125999999999999,"5min":14.125999999999999,"15min":14.125999999999999},"last":6.2359999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.3799999999999999,"5min":2.1749999999999998,"15min":2.4470000000000001},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":13.106,"5min":14.507,"15min":77.177999999999997},"last":1.8069999999999999},{"interface":"front","average":{"1min":1.6919999999999999,"5min":1.5640000000000001,"15min":1.681},"min":{"1min":0.39500000000000002,"5min":0.30199999999999999,"15min":0.28899999999999998},"max":{"1min":7.9249999999999998,"5min":7.9249999999999998,"15min":10.513999999999999},"last":0.98599999999999999}]},{"osd":5,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.7490000000000001,"15min":1.8400000000000001},"min":{"1min":0.39300000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.9550000000000001,"5min":7.9550000000000001,"15min":10.544},"last":1.1120000000000001},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.6499999999999999,"15min":1.8240000000000001},"min":{"1min":0.38100000000000001,"5min":0.308,"15min":0.29899999999999999},"max":{"1min":8.8439999999999994,"5min":11.212999999999999,"15min":24.754999999999999},"last":0.40100000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":1.736,"15min":1.9390000000000001},"min":{"1min":0.38,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":7.9829999999999997,"5min":7.9829999999999997,"15min":18.920000000000002},"last":1.0620000000000001},{"interface":"front","average":{"1min":2.3839999999999999,"5min":1.841,"15min":1.9590000000000001},"min":{"1min":0.42499999999999999,"5min":0.34999999999999998,"15min":0.31900000000000001},"max":{"1min":8.8810000000000002,"5min":11.257999999999999,"15min":24.800000000000001},"last":0.45800000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":2.0790000000000002,"15min":2.0289999999999999},"min":{"1min":0.496,"5min":0.35099999999999998,"15min":0.32700000000000001},"max":{"1min":8.9139999999999997,"5min":17.672000000000001,"15min":17.672000000000001},"last":6.9880000000000004},{"interface":"front","average":{"1min":2.4369999999999998,"5min":2.3839999999999999,"15min":2.552},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":13.154,"5min":17.539000000000001,"15min":77.228999999999999},"last":1.835}]}]},{"osd":2,"up_from":90,"seq":386547056942,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1203320,"kb_used_data":455280,"kb_used_omap":81,"kb_used_meta":747886,"kb_avail":92541832,"statfs":{"total":95995035648,"available":94762835968,"internally_reserved":0,"allocated":466206720,"data_stored":463343327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":765835482},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":2.2010000000000001,"15min":2.0609999999999999},"min":{"1min":0.186,"5min":0.182,"15min":0.17499999999999999},"max":{"1min":14.226000000000001,"5min":15.609,"15min":48.933999999999997},"last":0.191},{"interface":"front","average":{"1min":2.5419999999999998,"5min":2.056,"15min":1.8919999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.19600000000000001},"max":{"1min":9.5540000000000003,"5min":9.8010000000000002,"15min":15.595000000000001},"last":0.24199999999999999}]},{"osd":1,"last update":"Mon Oct 9 11:07:30 2023","interfaces":[{"interface":"back","average":{"1min":4.2800000000000002,"5min":2.96,"15min":2.629},"min":{"1min":0.20200000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":24.902000000000001,"5min":49.033999999999999,"15min":49.033999999999999},"last":0.20599999999999999},{"interface":"front","average":{"1min":4.7300000000000004,"5min":3.0550000000000002,"15min":2.7200000000000002},"min":{"1min":0.27500000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":25.850999999999999,"5min":48.874000000000002,"15min":48.874000000000002},"last":0.314}]},{"osd":3,"last update":"Mon Oct 9 11:07:17 2023","interfaces":[{"interface":"back","average":{"1min":4.1989999999999998,"5min":2.3330000000000002,"15min":1.954},"min":{"1min":0.751,"5min":0.214,"15min":0.214},"max":{"1min":15.701000000000001,"5min":15.701000000000001,"15min":15.701000000000001},"last":1.772},{"interface":"front","average":{"1min":3.5350000000000001,"5min":2.4329999999999998,"15min":2.2410000000000001},"min":{"1min":0.313,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":15.73,"5min":15.73,"15min":15.73},"last":1.8080000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":1.796,"15min":1.8959999999999999},"min":{"1min":0.27100000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":11.391999999999999,"5min":11.391999999999999,"15min":15.672000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.468,"5min":1.7909999999999999,"15min":1.873},"min":{"1min":0.34899999999999998,"5min":0.249,"15min":0.249},"max":{"1min":8.3350000000000009,"5min":15.571,"15min":48.795000000000002},"last":0.33900000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":1.597,"15min":1.5620000000000001},"min":{"1min":0.33700000000000002,"5min":0.29099999999999998,"15min":0.25900000000000001},"max":{"1min":7.4980000000000002,"5min":8.5310000000000006,"15min":12.128},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.8580000000000001,"5min":1.7869999999999999,"15min":2.0339999999999998},"min":{"1min":0.34899999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.441000000000001,"5min":11.441000000000001,"15min":49.061},"last":0.433}]},{"osd":6,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.909,"15min":1.804},"min":{"1min":0.41899999999999998,"5min":0.313,"15min":0.313},"max":{"1min":8.6039999999999992,"5min":15.536,"15min":48.845999999999997},"last":1.0249999999999999},{"interface":"front","average":{"1min":2.6499999999999999,"5min":2.012,"15min":1.657},"min":{"1min":0.42399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.157999999999999,"5min":10.157999999999999,"15min":10.157999999999999},"last":0.52200000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":2.081,"15min":1.9419999999999999},"min":{"1min":0.45100000000000001,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":8.3800000000000008,"5min":15.59,"15min":48.966000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":2.492,"5min":1.917,"15min":1.762},"min":{"1min":0.46200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.5919999999999996,"5min":8.5549999999999997,"15min":14.94},"last":0.45500000000000002}]}]},{"osd":3,"up_from":469,"seq":2014339661872,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":855364,"kb_used_data":546556,"kb_used_omap":11,"kb_used_meta":308788,"kb_avail":92889788,"statfs":{"total":95995035648,"available":95119142912,"internally_reserved":0,"allocated":559673344,"data_stored":556509783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11309,"internal_metadata":316199891},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":3.3700000000000001,"5min":1.927,"15min":1.7889999999999999},"min":{"1min":0.27400000000000002,"5min":0.183,"15min":0.183},"max":{"1min":9.9809999999999999,"5min":14.882,"15min":14.882},"last":3.2349999999999999},{"interface":"front","average":{"1min":1.7050000000000001,"5min":1.0920000000000001,"15min":0.94199999999999995},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":5.6980000000000004,"5min":5.6980000000000004,"15min":5.6980000000000004},"last":0.20300000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":3.536,"5min":2.0859999999999999,"15min":1.6299999999999999},"min":{"1min":0.17100000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":11.757,"5min":17.93,"15min":17.93},"last":1.1140000000000001},{"interface":"front","average":{"1min":3.3050000000000002,"5min":1.9610000000000001,"15min":1.71},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":19.163,"5min":19.163,"15min":19.163},"last":10.645}]},{"osd":2,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":4.46,"5min":1.804,"15min":1.3260000000000001},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":38.840000000000003,"5min":38.840000000000003,"15min":38.840000000000003},"last":0.23899999999999999},{"interface":"front","average":{"1min":4.7619999999999996,"5min":2.0979999999999999,"15min":1.419},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":37.991999999999997,"5min":37.991999999999997,"15min":37.991999999999997},"last":0.219}]},{"osd":4,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":2.274,"5min":1.641,"15min":1.595},"min":{"1min":0.30599999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":10.058999999999999,"5min":14.907999999999999,"15min":14.907999999999999},"last":3.3199999999999998},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.873,"15min":0.755},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.294,"5min":4.1390000000000002,"15min":4.1390000000000002},"last":0.33000000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":1.5,"15min":1.4419999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.8550000000000004,"5min":9.3300000000000001,"15min":9.3300000000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.2650000000000001,"5min":1.5429999999999999,"15min":1.4750000000000001},"min":{"1min":0.32800000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.029,"5min":10.029,"15min":10.029},"last":3.2869999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.76,"15min":1.7749999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.5289999999999999,"5min":15.455,"15min":15.455},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.3100000000000001,"5min":1.673,"15min":1.653},"min":{"1min":0.374,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.099,"5min":14.837,"15min":14.837},"last":3.3050000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":1.208,"15min":1.0700000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.907,"5min":6.907,"15min":6.907},"last":0.34899999999999998},{"interface":"front","average":{"1min":2.0590000000000002,"5min":1.5169999999999999,"15min":1.2769999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.9320000000000004,"5min":6.9119999999999999,"15min":6.9119999999999999},"last":0.373}]}]},{"osd":4,"up_from":30,"seq":128849019292,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1192472,"kb_used_data":299728,"kb_used_omap":0,"kb_used_meta":892736,"kb_avail":92552680,"statfs":{"total":95995035648,"available":94773944320,"internally_reserved":0,"allocated":306921472,"data_stored":305127823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914161664},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:18 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.659,"15min":2.399},"min":{"1min":0.23200000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":3.6150000000000002,"5min":13.162000000000001,"15min":213.88999999999999},"last":0.31},{"interface":"front","average":{"1min":1.569,"5min":2.161,"15min":2.6379999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":4.6699999999999999,"5min":11.128,"15min":214.09800000000001},"last":0.62}]},{"osd":1,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.54,"5min":2.508,"15min":2.6299999999999999},"min":{"1min":0.34599999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":8.1229999999999993,"5min":57.966999999999999,"15min":121.41},"last":7.718},{"interface":"front","average":{"1min":2.117,"5min":2.3250000000000002,"15min":2.1200000000000001},"min":{"1min":0.26300000000000001,"5min":0.251,"15min":0.251},"max":{"1min":8.0960000000000001,"5min":57.832000000000001,"15min":122.21899999999999},"last":2.2240000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:08:00 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":1.917,"15min":1.74},"min":{"1min":0.32100000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.2939999999999996,"5min":18.407,"15min":18.407},"last":2.3849999999999998},{"interface":"front","average":{"1min":2.149,"5min":1.9810000000000001,"15min":2.0910000000000002},"min":{"1min":0.27900000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.1479999999999997,"5min":8.4250000000000007,"15min":27.818000000000001},"last":2.5129999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.837,"5min":2.4460000000000002,"15min":1.982},"min":{"1min":0.40899999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.0529999999999999,"5min":60.753,"15min":60.753},"last":0.34799999999999998},{"interface":"front","average":{"1min":2.6469999999999998,"5min":2.1389999999999998,"15min":1.492},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.0220000000000002,"5min":61.473999999999997,"15min":61.473999999999997},"last":2.5950000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":1.5049999999999999,"15min":1.4590000000000001},"min":{"1min":0.27400000000000002,"5min":0.224,"15min":0.224},"max":{"1min":3.4860000000000002,"5min":11.59,"15min":19.068999999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.3620000000000001,"15min":1.272},"min":{"1min":0.27300000000000002,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":3.8999999999999999,"5min":9.8610000000000007,"15min":15.535},"last":0.66400000000000003}]},{"osd":6,"last update":"Mon Oct 9 11:08:00 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":1.353,"15min":1.1599999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":7.5860000000000003,"5min":8.3930000000000007,"15min":15.739000000000001},"last":6.8689999999999998},{"interface":"front","average":{"1min":1.4610000000000001,"5min":1.4890000000000001,"15min":1.4570000000000001},"min":{"1min":0.34899999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":9.5210000000000008,"5min":12.795999999999999,"15min":14.093999999999999},"last":6.8170000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:08:00 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.5600000000000001,"15min":1.5529999999999999},"min":{"1min":0.308,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.4540000000000002,"5min":12.753,"15min":19.923999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.4239999999999999,"15min":1.476},"min":{"1min":0.314,"5min":0.29799999999999999,"15min":0.28799999999999998},"max":{"1min":4.21,"5min":10.853,"15min":19.859999999999999},"last":2.3410000000000002}]}]},{"osd":5,"up_from":37,"seq":158913790361,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1518532,"kb_used_data":323260,"kb_used_omap":0,"kb_used_meta":1195200,"kb_avail":92226620,"statfs":{"total":95995035648,"available":94440058880,"internally_reserved":0,"allocated":331018240,"data_stored":329241012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1223884800},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:25 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":2.3809999999999998,"15min":2.3330000000000002},"min":{"1min":0.22900000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":6.2249999999999996,"5min":49.665999999999997,"15min":58.271999999999998},"last":2.6869999999999998},{"interface":"front","average":{"1min":1.605,"5min":2.427,"15min":2.3809999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.2649999999999997,"5min":49.701999999999998,"15min":58.25},"last":2.7200000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:07:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":2.101,"15min":2.032},"min":{"1min":0.26600000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.2199999999999998,"5min":32.491999999999997,"15min":32.491999999999997},"last":0.90700000000000003},{"interface":"front","average":{"1min":1.7709999999999999,"5min":2.3100000000000001,"15min":1.8680000000000001},"min":{"1min":0.39300000000000002,"5min":0.247,"15min":0.247},"max":{"1min":5.2130000000000001,"5min":32.537999999999997,"15min":32.537999999999997},"last":0.93000000000000005}]},{"osd":2,"last update":"Mon Oct 9 11:07:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.454,"15min":1.786},"min":{"1min":0.307,"5min":0.254,"15min":0.221},"max":{"1min":5.1269999999999998,"5min":6.3929999999999998,"15min":11.257999999999999},"last":0.308},{"interface":"front","average":{"1min":2.1869999999999998,"5min":2.0129999999999999,"15min":1.9370000000000001},"min":{"1min":0.29099999999999998,"5min":0.253,"15min":0.253},"max":{"1min":8.0359999999999996,"5min":11.673,"15min":14.621},"last":0.71699999999999997}]},{"osd":3,"last update":"Mon Oct 9 11:07:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4449999999999998,"5min":1.6439999999999999,"15min":1.5409999999999999},"min":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":15.535,"5min":15.535,"15min":15.535},"last":0.35399999999999998},{"interface":"front","average":{"1min":2.3460000000000001,"5min":1.8560000000000001,"15min":1.8240000000000001},"min":{"1min":0.38700000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":8.8940000000000001,"5min":8.8940000000000001,"15min":8.8940000000000001},"last":0.42599999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:07:30 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":1.206,"15min":0.98899999999999999},"min":{"1min":0.28000000000000003,"5min":0.25700000000000001,"15min":0.216},"max":{"1min":2.0099999999999998,"5min":12.042,"15min":12.042},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.70699999999999996,"5min":1.2150000000000001,"15min":1.165},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.5910000000000002,"5min":9.2490000000000006,"15min":9.2490000000000006},"last":0.39300000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:07:42 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.3300000000000001,"15min":1.2110000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27100000000000002},"max":{"1min":1.5669999999999999,"5min":9.1969999999999992,"15min":9.1969999999999992},"last":1.738},{"interface":"front","average":{"1min":0.65300000000000002,"5min":1.288,"15min":1.0629999999999999},"min":{"1min":0.33700000000000002,"5min":0.32800000000000001,"15min":0.29099999999999998},"max":{"1min":2.6059999999999999,"5min":11.983000000000001,"15min":11.983000000000001},"last":0.48599999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:07:06 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.421,"15min":1.3660000000000001},"min":{"1min":0.307,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.5070000000000001,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.4710000000000001,"15min":1.254},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30099999999999999},"max":{"1min":3.0409999999999999,"5min":11.726000000000001,"15min":11.726000000000001},"last":0.40799999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3733,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":447610880,"data_stored":447610880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":548143104,"data_stored":548143104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":440930304,"data_stored":440926232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":549257216,"data_stored":549257216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":290263040,"data_stored":290258968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":323289088,"data_stored":323289088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":704974848,"data_stored":704970776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":878837760,"data_stored":878837760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:08:12.247 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:12 smithi125 ceph-mon[195507]: from='client.46709 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:12.264 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:12.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:12.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:12.329 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:12.368 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:12.416 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:12.459 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:08:12.494 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:12.496 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:12 smithi155 ceph-mon[184854]: from='client.46709 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:12.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:12.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:12.506 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:08:12.530 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:12 smithi125 ceph-mon[200197]: from='client.46709 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:12.556 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:08:12.598 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:12.609 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:08:12.642 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:08:12.644 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:12.678 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:08:12.712 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:08:12.746 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:08:12.752 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:08:12.752 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:08:12.752 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:08:12.752 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:08:12.753 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:08:12.753 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:08:12.753 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:08:12.753 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:08:12.867 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:12.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:12.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:12.971 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:13.035 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:13.066 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:13.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:13.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:13.178 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:13.235 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:13.312 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:08:13.312 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:08:13.381 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:13.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:13.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:13.490 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:13.500 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:13 smithi125 ceph-mon[195507]: pgmap v1369: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 987 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 5000/47436 objects degraded (10.541%); 1852/47436 objects misplaced (3.904%); 2.0 MiB/s, 73 objects/s recovering 2023-10-09T11:08:13.500 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:13 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3346887026' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:13.500 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:13 smithi125 ceph-mon[200197]: pgmap v1369: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 987 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 5000/47436 objects degraded (10.541%); 1852/47436 objects misplaced (3.904%); 2.0 MiB/s, 73 objects/s recovering 2023-10-09T11:08:13.501 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:13 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3346887026' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:13.555 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:13 smithi155 ceph-mon[184854]: pgmap v1369: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 987 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 5000/47436 objects degraded (10.541%); 1852/47436 objects misplaced (3.904%); 2.0 MiB/s, 73 objects/s recovering 2023-10-09T11:08:13.555 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:13 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3346887026' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:13.559 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:13.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:13.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:13.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:13.746 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T11:08:13.810 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:13.890 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:13.921 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:13.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:13.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:14.028 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:14.089 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:14.124 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:14.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:14.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:14.131 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:08:14.228 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:14.278 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:14.314 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:14.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:14.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:14.424 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:14.440 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:08:14.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:14 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 5000/47436 objects degraded (10.541%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:14.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:14 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2745547327' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:14.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:14 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:14.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:14 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 5000/47436 objects degraded (10.541%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:14.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:14 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2745547327' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:14.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:14 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:14.474 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:14.623 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:14.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:14.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:14.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:14 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 5000/47436 objects degraded (10.541%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:14.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:14 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2745547327' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:14.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:14 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:14.726 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:14.803 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:14.833 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:14.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:14.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:14.940 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:15.000 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:15.031 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:15.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:15.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:15.137 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:15.177 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:15.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:08:15.212 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:15.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:15.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:15.321 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:15.333 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:08:15.333 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:08:15.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:15 smithi155 ceph-mon[184854]: pgmap v1370: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 785 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 5000/37713 objects degraded (13.258%); 1298/37713 objects misplaced (3.442%); 2.1 MiB/s, 133 objects/s recovering 2023-10-09T11:08:15.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:15 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:08:15.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:15 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:08:15.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:15 smithi155 ceph-mon[184854]: osdmap e560: 8 total, 8 up, 8 in 2023-10-09T11:08:15.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:15 smithi155 ceph-mon[184854]: failed to encode map e560 with expected crc 2023-10-09T11:08:15.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:15 smithi155 ceph-mon[184854]: failed to encode map e560 with expected crc 2023-10-09T11:08:15.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:15 smithi155 ceph-mon[184854]: failed to encode map e560 with expected crc 2023-10-09T11:08:15.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:08:15.394 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:08:15.396 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:15.428 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:15.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:15.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:15.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[195507]: pgmap v1370: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 785 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 5000/37713 objects degraded (13.258%); 1298/37713 objects misplaced (3.442%); 2.1 MiB/s, 133 objects/s recovering 2023-10-09T11:08:15.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:08:15.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:08:15.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[195507]: osdmap e560: 8 total, 8 up, 8 in 2023-10-09T11:08:15.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[195507]: failed to encode map e560 with expected crc 2023-10-09T11:08:15.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[195507]: failed to encode map e560 with expected crc 2023-10-09T11:08:15.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[195507]: failed to encode map e560 with expected crc 2023-10-09T11:08:15.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:08:15.433 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[200197]: pgmap v1370: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 785 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 5000/37713 objects degraded (13.258%); 1298/37713 objects misplaced (3.442%); 2.1 MiB/s, 133 objects/s recovering 2023-10-09T11:08:15.433 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:08:15.433 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:08:15.434 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[200197]: osdmap e560: 8 total, 8 up, 8 in 2023-10-09T11:08:15.434 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[200197]: failed to encode map e560 with expected crc 2023-10-09T11:08:15.434 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[200197]: failed to encode map e560 with expected crc 2023-10-09T11:08:15.434 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[200197]: failed to encode map e560 with expected crc 2023-10-09T11:08:15.434 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:08:15.533 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:15.576 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:15.651 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:16.340 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:16.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:16.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:16.366 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T11:08:16.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:16.516 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:16.565 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:16.565 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:16 smithi125 ceph-mon[195507]: failed to encode map e560 with expected crc 2023-10-09T11:08:16.566 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:16 smithi125 ceph-mon[195507]: failed to encode map e560 with expected crc 2023-10-09T11:08:16.566 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:16 smithi125 ceph-mon[200197]: failed to encode map e560 with expected crc 2023-10-09T11:08:16.566 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:16 smithi125 ceph-mon[200197]: failed to encode map e560 with expected crc 2023-10-09T11:08:16.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:16.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:16.677 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:16.718 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:16 smithi155 ceph-mon[184854]: failed to encode map e560 with expected crc 2023-10-09T11:08:16.718 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:16 smithi155 ceph-mon[184854]: failed to encode map e560 with expected crc 2023-10-09T11:08:16.722 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:16.747 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:16.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:16.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:16.857 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:16.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:08:16.911 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:16.949 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:16.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:17.060 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:17.114 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:17.118 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:17.118 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:17.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:17.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:17.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:17.273 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:17.326 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:17.462 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:17.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:17.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:17.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:17 smithi125 ceph-mon[195507]: pgmap v1372: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 785 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 923 op/s; 5000/37713 objects degraded (13.258%); 1298/37713 objects misplaced (3.442%); 2.1 MiB/s, 133 objects/s recovering 2023-10-09T11:08:17.470 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:17 smithi125 ceph-mon[200197]: pgmap v1372: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 785 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 923 op/s; 5000/37713 objects degraded (13.258%); 1298/37713 objects misplaced (3.442%); 2.1 MiB/s, 133 objects/s recovering 2023-10-09T11:08:17.568 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:17.632 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:17.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:17 smithi155 ceph-mon[184854]: pgmap v1372: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 785 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 923 op/s; 5000/37713 objects degraded (13.258%); 1298/37713 objects misplaced (3.442%); 2.1 MiB/s, 133 objects/s recovering 2023-10-09T11:08:17.919 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:17.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:17.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:18.023 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:18.068 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:18.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:18.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:18.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:18.209 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:18.261 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:18.972 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:18.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:18.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:19.081 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:19.120 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:19.145 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:19.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:19.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:19.250 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:19.356 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:19.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:19 smithi155 ceph-mon[184854]: pgmap v1373: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 696 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 5000/33441 objects degraded (14.952%); 1298/33441 objects misplaced (3.881%); 1.7 MiB/s, 110 objects/s recovering 2023-10-09T11:08:19.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:19 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 5000/33441 objects degraded (14.952%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:19.476 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:19.484 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:19 smithi125 ceph-mon[195507]: pgmap v1373: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 696 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 5000/33441 objects degraded (14.952%); 1298/33441 objects misplaced (3.881%); 1.7 MiB/s, 110 objects/s recovering 2023-10-09T11:08:19.484 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:19 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 5000/33441 objects degraded (14.952%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:19.484 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:19 smithi125 ceph-mon[200197]: pgmap v1373: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 696 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 5000/33441 objects degraded (14.952%); 1298/33441 objects misplaced (3.881%); 1.7 MiB/s, 110 objects/s recovering 2023-10-09T11:08:19.485 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:19 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 5000/33441 objects degraded (14.952%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:19.497 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:19.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:19.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:19.558 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:08:19.561 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:08:19.604 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:19.692 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:19.989 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:19.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:19.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:20.096 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:20.130 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:20.151 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:08:20.151 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:08:20.169 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:20.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:20.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:20.279 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:20.336 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:20.371 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:20.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:20.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:20.479 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:20.529 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:20.562 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:20.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:20.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:20.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:20.714 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:20.739 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:20.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:20.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:20.846 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:20.888 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:20.916 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:20.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:20.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:21.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:21.065 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:21.096 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:21.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:21.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:21.202 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:21.266 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:21.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[200197]: from='client.46715 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:21.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[200197]: pgmap v1374: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 497 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 4694/23892 objects degraded (19.647%); 776/23892 objects misplaced (3.248%); 96 KiB/s, 142 objects/s recovering 2023-10-09T11:08:21.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[200197]: failed to encode map e561 with expected crc 2023-10-09T11:08:21.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[200197]: osdmap e561: 8 total, 8 up, 8 in 2023-10-09T11:08:21.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[200197]: failed to encode map e561 with expected crc 2023-10-09T11:08:21.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[200197]: failed to encode map e561 with expected crc 2023-10-09T11:08:21.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[200197]: failed to encode map e561 with expected crc 2023-10-09T11:08:21.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[200197]: failed to encode map e561 with expected crc 2023-10-09T11:08:21.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3915047861' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:21.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:08:21] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-10-09T11:08:21.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[195507]: from='client.46715 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:21.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[195507]: pgmap v1374: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 497 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 4694/23892 objects degraded (19.647%); 776/23892 objects misplaced (3.248%); 96 KiB/s, 142 objects/s recovering 2023-10-09T11:08:21.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[195507]: failed to encode map e561 with expected crc 2023-10-09T11:08:21.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[195507]: osdmap e561: 8 total, 8 up, 8 in 2023-10-09T11:08:21.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[195507]: failed to encode map e561 with expected crc 2023-10-09T11:08:21.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[195507]: failed to encode map e561 with expected crc 2023-10-09T11:08:21.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[195507]: failed to encode map e561 with expected crc 2023-10-09T11:08:21.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[195507]: failed to encode map e561 with expected crc 2023-10-09T11:08:21.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:21 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3915047861' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:21 smithi155 ceph-mon[184854]: from='client.46715 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:21 smithi155 ceph-mon[184854]: pgmap v1374: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 497 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 4694/23892 objects degraded (19.647%); 776/23892 objects misplaced (3.248%); 96 KiB/s, 142 objects/s recovering 2023-10-09T11:08:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:21 smithi155 ceph-mon[184854]: failed to encode map e561 with expected crc 2023-10-09T11:08:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:21 smithi155 ceph-mon[184854]: osdmap e561: 8 total, 8 up, 8 in 2023-10-09T11:08:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:21 smithi155 ceph-mon[184854]: failed to encode map e561 with expected crc 2023-10-09T11:08:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:21 smithi155 ceph-mon[184854]: failed to encode map e561 with expected crc 2023-10-09T11:08:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:21 smithi155 ceph-mon[184854]: failed to encode map e561 with expected crc 2023-10-09T11:08:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:21 smithi155 ceph-mon[184854]: failed to encode map e561 with expected crc 2023-10-09T11:08:21.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:21 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3915047861' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:22.075 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:08:22.076 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:08:22.076 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:08:22.076 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:08:22.076 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:08:22.076 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:08:22.076 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:08:22.076 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:08:22.076 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:08:22.077 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:08:22.077 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:08:22.077 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:08:22.077 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:08:22.077 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:08:22.077 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:08:22.077 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:08:22.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:22 smithi155 ceph-mon[184854]: osdmap e562: 8 total, 8 up, 8 in 2023-10-09T11:08:22.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:22 smithi155 ceph-mon[184854]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:22 smithi155 ceph-mon[184854]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:22 smithi155 ceph-mon[184854]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:22 smithi155 ceph-mon[184854]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:22 smithi155 ceph-mon[184854]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:22 smithi155 ceph-mon[184854]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:22 smithi155 ceph-mon[184854]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:22 smithi155 ceph-mon[184854]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.723 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:22 smithi155 ceph-mon[184854]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.744 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[200197]: osdmap e562: 8 total, 8 up, 8 in 2023-10-09T11:08:22.744 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[200197]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.744 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[200197]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.745 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[200197]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.745 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[200197]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.745 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[200197]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.745 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[200197]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.745 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[200197]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.745 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[200197]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.745 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[200197]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.745 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[195507]: osdmap e562: 8 total, 8 up, 8 in 2023-10-09T11:08:22.745 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[195507]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.745 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[195507]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[195507]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[195507]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[195507]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[195507]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[195507]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[195507]: failed to encode map e562 with expected crc 2023-10-09T11:08:22.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:22 smithi125 ceph-mon[195507]: failed to encode map e562 with expected crc 2023-10-09T11:08:23.098 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1373,"stamp":"2023-10-09T11:08:17.919463+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":729457532,"num_objects":11147,"num_object_clones":0,"num_object_copies":33441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5000,"num_objects_misplaced":1298,"num_objects_unfound":0,"num_objects_dirty":11147,"num_whiteouts":0,"num_read":32,"num_read_kb":57,"num_write":187948,"num_write_kb":6367921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24892,"num_bytes_recovered":1469671754,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":238147,"ondisk_log_size":238147,"up":417,"acting":414,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":428,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9926924,"kb_used_data":2944332,"kb_used_omap":134,"kb_used_meta":6981817,"kb_avail":740034292,"statfs":{"total":767960285184,"available":757795115008,"internally_reserved":0,"allocated":3014995968,"data_stored":2995741898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":137532,"internal_metadata":7149381316},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,4,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-578093056,"num_objects":-8821,"num_object_clones":0,"num_object_copies":-26463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1040,"num_objects_misplaced":-989,"num_objects_unfound":0,"num_objects_dirty":-8821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8821,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1070,"num_bytes_recovered":17694720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.700114"},"pg_stats":[{"pgid":"11.23","version":"559'6200","reported_seq":9223,"reported_epoch":559,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:08:13.308115+0000","last_change":"2023-10-09T11:07:34.314839+0000","last_active":"2023-10-09T11:08:13.308115+0000","last_peered":"2023-10-09T11:08:13.308115+0000","last_clean":"2023-10-09T11:07:32.177661+0000","last_became_active":"2023-10-09T11:07:34.308281+0000","last_became_peered":"2023-10-09T11:07:34.308281+0000","last_unstale":"2023-10-09T11:08:13.308115+0000","last_undegraded":"2023-10-09T11:08:13.308115+0000","last_fullsized":"2023-10-09T11:08:13.308115+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6200,"log_dups_size":0,"ondisk_log_size":6200,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12206080,"num_objects":186,"num_object_clones":0,"num_object_copies":558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":186,"num_objects_unfound":0,"num_objects_dirty":186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2984,"num_write_kb":101456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1213,"num_bytes_recovered":72024064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":186}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.22","version":"559'5919","reported_seq":6011,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.133714+0000","last_change":"2023-10-09T11:06:43.010089+0000","last_active":"2023-10-09T11:08:13.133714+0000","last_peered":"2023-10-09T11:08:13.133714+0000","last_clean":"2023-10-09T11:08:13.133714+0000","last_became_active":"2023-10-09T11:06:43.008608+0000","last_became_peered":"2023-10-09T11:06:43.008608+0000","last_unstale":"2023-10-09T11:08:13.133714+0000","last_undegraded":"2023-10-09T11:08:13.133714+0000","last_fullsized":"2023-10-09T11:08:13.133714+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5919,"log_dups_size":0,"ondisk_log_size":5919,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15663104,"num_objects":239,"num_object_clones":0,"num_object_copies":717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2816,"num_write_kb":97760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.21","version":"559'6248","reported_seq":7119,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.287854+0000","last_change":"2023-10-09T11:07:23.970291+0000","last_active":"2023-10-09T11:08:13.287854+0000","last_peered":"2023-10-09T11:08:13.287854+0000","last_clean":"2023-10-09T11:08:13.287854+0000","last_became_active":"2023-10-09T11:06:43.015074+0000","last_became_peered":"2023-10-09T11:06:43.015074+0000","last_unstale":"2023-10-09T11:08:13.287854+0000","last_undegraded":"2023-10-09T11:08:13.287854+0000","last_fullsized":"2023-10-09T11:08:13.287854+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6248,"log_dups_size":0,"ondisk_log_size":6248,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8699904,"num_objects":133,"num_object_clones":0,"num_object_copies":399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3015,"num_write_kb":100720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":344,"num_bytes_recovered":22511616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.20","version":"559'6091","reported_seq":8501,"reported_epoch":559,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:08:13.302095+0000","last_change":"2023-10-09T11:07:34.316666+0000","last_active":"2023-10-09T11:08:13.302095+0000","last_peered":"2023-10-09T11:08:13.302095+0000","last_clean":"2023-10-09T11:07:32.195530+0000","last_became_active":"2023-10-09T11:07:34.309531+0000","last_became_peered":"2023-10-09T11:07:34.309531+0000","last_unstale":"2023-10-09T11:08:13.302095+0000","last_undegraded":"2023-10-09T11:08:13.302095+0000","last_fullsized":"2023-10-09T11:08:13.302095+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6091,"log_dups_size":0,"ondisk_log_size":6091,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13058048,"num_objects":199,"num_object_clones":0,"num_object_copies":597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":199,"num_objects_unfound":0,"num_objects_dirty":199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2925,"num_write_kb":99984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1150,"num_bytes_recovered":72941568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,6],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,3,6","objects":199}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.12","version":"559'7352","reported_seq":7434,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.322862+0000","last_change":"2023-10-09T11:05:45.369342+0000","last_active":"2023-10-09T11:08:13.322862+0000","last_peered":"2023-10-09T11:08:13.322862+0000","last_clean":"2023-10-09T11:08:13.322862+0000","last_became_active":"2023-10-09T11:05:45.367721+0000","last_became_peered":"2023-10-09T11:05:45.367721+0000","last_unstale":"2023-10-09T11:08:13.322862+0000","last_undegraded":"2023-10-09T11:08:13.322862+0000","last_fullsized":"2023-10-09T11:08:13.322862+0000","mapping_epoch":509,"log_start":"504'2125","ondisk_log_start":"504'2125","created":509,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5227,"log_dups_size":0,"ondisk_log_size":5227,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23068672,"num_objects":352,"num_object_clones":0,"num_object_copies":1056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5814,"num_write_kb":197312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"76'161","reported_seq":847,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260775+0000","last_change":"2023-10-09T11:07:36.287455+0000","last_active":"2023-10-09T11:08:08.260775+0000","last_peered":"2023-10-09T11:08:08.260775+0000","last_clean":"2023-10-09T11:08:08.260775+0000","last_became_active":"2023-10-09T11:07:36.287052+0000","last_became_peered":"2023-10-09T11:07:36.287052+0000","last_unstale":"2023-10-09T11:08:08.260775+0000","last_undegraded":"2023-10-09T11:08:08.260775+0000","last_fullsized":"2023-10-09T11:08:08.260775+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:18.864873+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":342,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.224610+0000","last_change":"2023-10-09T11:03:21.185383+0000","last_active":"2023-10-09T11:08:08.224610+0000","last_peered":"2023-10-09T11:08:08.224610+0000","last_clean":"2023-10-09T11:08:08.224610+0000","last_became_active":"2023-10-09T11:03:21.185022+0000","last_became_peered":"2023-10-09T11:03:21.185022+0000","last_unstale":"2023-10-09T11:08:08.224610+0000","last_undegraded":"2023-10-09T11:08:08.224610+0000","last_fullsized":"2023-10-09T11:08:08.224610+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:05.230434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":85,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219524+0000","last_change":"2023-10-09T11:04:50.319477+0000","last_active":"2023-10-09T11:08:07.219524+0000","last_peered":"2023-10-09T11:08:07.219524+0000","last_clean":"2023-10-09T11:08:07.219524+0000","last_became_active":"2023-10-09T11:04:50.319148+0000","last_became_peered":"2023-10-09T11:04:50.319148+0000","last_unstale":"2023-10-09T11:08:07.219524+0000","last_undegraded":"2023-10-09T11:08:07.219524+0000","last_fullsized":"2023-10-09T11:08:07.219524+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:48.725566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"559'7593","reported_seq":9007,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.261815+0000","last_change":"2023-10-09T11:06:54.752222+0000","last_active":"2023-10-09T11:08:13.261815+0000","last_peered":"2023-10-09T11:08:13.261815+0000","last_clean":"2023-10-09T11:08:13.261815+0000","last_became_active":"2023-10-09T11:05:45.395271+0000","last_became_peered":"2023-10-09T11:05:45.395271+0000","last_unstale":"2023-10-09T11:08:13.261815+0000","last_undegraded":"2023-10-09T11:08:13.261815+0000","last_fullsized":"2023-10-09T11:08:13.261815+0000","mapping_epoch":509,"log_start":"504'2386","ondisk_log_start":"504'2386","created":509,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5207,"log_dups_size":0,"ondisk_log_size":5207,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29261824,"num_objects":447,"num_object_clones":0,"num_object_copies":1341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5975,"num_write_kb":205472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":659,"num_bytes_recovered":43188224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":717,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260600+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:08:08.260600+0000","last_peered":"2023-10-09T11:08:08.260600+0000","last_clean":"2023-10-09T11:08:08.260600+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:08:08.260600+0000","last_undegraded":"2023-10-09T11:08:08.260600+0000","last_fullsized":"2023-10-09T11:08:08.260600+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":273,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.215893+0000","last_change":"2023-10-09T11:04:50.333711+0000","last_active":"2023-10-09T11:08:07.215893+0000","last_peered":"2023-10-09T11:08:07.215893+0000","last_clean":"2023-10-09T11:08:07.215893+0000","last_became_active":"2023-10-09T11:04:50.333262+0000","last_became_peered":"2023-10-09T11:04:50.333262+0000","last_unstale":"2023-10-09T11:08:07.215893+0000","last_undegraded":"2023-10-09T11:08:07.215893+0000","last_fullsized":"2023-10-09T11:08:07.215893+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.843765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":20,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226026+0000","last_change":"2023-10-09T11:07:33.237903+0000","last_active":"2023-10-09T11:08:08.226026+0000","last_peered":"2023-10-09T11:08:08.226026+0000","last_clean":"2023-10-09T11:08:08.226026+0000","last_became_active":"2023-10-09T11:07:33.237618+0000","last_became_peered":"2023-10-09T11:07:33.237618+0000","last_unstale":"2023-10-09T11:08:08.226026+0000","last_undegraded":"2023-10-09T11:08:08.226026+0000","last_fullsized":"2023-10-09T11:08:08.226026+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:25.600818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.10","version":"559'7443","reported_seq":8133,"reported_epoch":559,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:08:13.243097+0000","last_change":"2023-10-09T11:07:34.347309+0000","last_active":"2023-10-09T11:08:13.243097+0000","last_peered":"2023-10-09T11:08:13.243097+0000","last_clean":"2023-10-09T11:06:06.955693+0000","last_became_active":"2023-10-09T11:07:33.305104+0000","last_became_peered":"2023-10-09T11:07:33.305104+0000","last_unstale":"2023-10-09T11:08:13.243097+0000","last_undegraded":"2023-10-09T11:07:34.345913+0000","last_fullsized":"2023-10-09T11:07:33.208203+0000","mapping_epoch":549,"log_start":"504'2085","ondisk_log_start":"504'2085","created":509,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5358,"log_dups_size":0,"ondisk_log_size":5358,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23887872,"num_objects":365,"num_object_clones":0,"num_object_copies":1095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2269,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5881,"num_write_kb":199840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":292,"num_bytes_recovered":19136512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":-291},{"shards":"1,3,6","objects":398}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":735,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226022+0000","last_change":"2023-10-09T11:07:36.284534+0000","last_active":"2023-10-09T11:08:08.226022+0000","last_peered":"2023-10-09T11:08:08.226022+0000","last_clean":"2023-10-09T11:08:08.226022+0000","last_became_active":"2023-10-09T11:07:36.284254+0000","last_became_peered":"2023-10-09T11:07:36.284254+0000","last_unstale":"2023-10-09T11:08:08.226022+0000","last_undegraded":"2023-10-09T11:08:08.226022+0000","last_fullsized":"2023-10-09T11:08:08.226022+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:19.159630+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":85,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.209764+0000","last_change":"2023-10-09T11:04:50.310477+0000","last_active":"2023-10-09T11:08:07.209764+0000","last_peered":"2023-10-09T11:08:07.209764+0000","last_clean":"2023-10-09T11:08:07.209764+0000","last_became_active":"2023-10-09T11:04:50.310067+0000","last_became_peered":"2023-10-09T11:04:50.310067+0000","last_unstale":"2023-10-09T11:08:07.209764+0000","last_undegraded":"2023-10-09T11:08:07.209764+0000","last_fullsized":"2023-10-09T11:08:07.209764+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:52.884716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":369,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.262081+0000","last_change":"2023-10-09T11:05:20.310581+0000","last_active":"2023-10-09T11:08:08.262081+0000","last_peered":"2023-10-09T11:08:08.262081+0000","last_clean":"2023-10-09T11:08:08.262081+0000","last_became_active":"2023-10-09T11:05:20.309830+0000","last_became_peered":"2023-10-09T11:05:20.309830+0000","last_unstale":"2023-10-09T11:08:08.262081+0000","last_undegraded":"2023-10-09T11:08:08.262081+0000","last_fullsized":"2023-10-09T11:08:08.262081+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:24.508211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1f","version":"560'9283","reported_seq":17657,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T11:08:16.543041+0000","last_change":"2023-10-09T11:08:08.246119+0000","last_active":"2023-10-09T11:08:16.543041+0000","last_peered":"2023-10-09T11:08:16.543041+0000","last_clean":"2023-10-09T11:08:16.543041+0000","last_became_active":"2023-10-09T11:08:08.230534+0000","last_became_peered":"2023-10-09T11:08:08.230534+0000","last_unstale":"2023-10-09T11:08:16.543041+0000","last_undegraded":"2023-10-09T11:08:16.543041+0000","last_fullsized":"2023-10-09T11:08:16.543041+0000","mapping_epoch":558,"log_start":"504'1946","ondisk_log_start":"504'1946","created":531,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7337,"log_dups_size":0,"ondisk_log_size":7337,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16187392,"num_objects":247,"num_object_clones":0,"num_object_copies":741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6053,"num_write_kb":201600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4134,"num_bytes_recovered":164298752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":775,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226210+0000","last_change":"2023-10-09T11:07:36.282502+0000","last_active":"2023-10-09T11:08:08.226210+0000","last_peered":"2023-10-09T11:08:08.226210+0000","last_clean":"2023-10-09T11:08:08.226210+0000","last_became_active":"2023-10-09T11:07:36.282133+0000","last_became_peered":"2023-10-09T11:07:36.282133+0000","last_unstale":"2023-10-09T11:08:08.226210+0000","last_undegraded":"2023-10-09T11:08:08.226210+0000","last_fullsized":"2023-10-09T11:08:08.226210+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:03.762383+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":196,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260301+0000","last_change":"2023-10-09T11:06:24.245488+0000","last_active":"2023-10-09T11:08:08.260301+0000","last_peered":"2023-10-09T11:08:08.260301+0000","last_clean":"2023-10-09T11:08:08.260301+0000","last_became_active":"2023-10-09T11:06:24.245167+0000","last_became_peered":"2023-10-09T11:06:24.245167+0000","last_unstale":"2023-10-09T11:08:08.260301+0000","last_undegraded":"2023-10-09T11:08:08.260301+0000","last_fullsized":"2023-10-09T11:08:08.260301+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:31.702568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":94,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.209906+0000","last_change":"2023-10-09T11:05:20.299453+0000","last_active":"2023-10-09T11:08:07.209906+0000","last_peered":"2023-10-09T11:08:07.209906+0000","last_clean":"2023-10-09T11:08:07.209906+0000","last_became_active":"2023-10-09T11:05:20.299117+0000","last_became_peered":"2023-10-09T11:05:20.299117+0000","last_unstale":"2023-10-09T11:08:07.209906+0000","last_undegraded":"2023-10-09T11:08:07.209906+0000","last_fullsized":"2023-10-09T11:08:07.209906+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:30.072813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1e","version":"559'8944","reported_seq":9026,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.251013+0000","last_change":"2023-10-09T11:06:43.029410+0000","last_active":"2023-10-09T11:08:13.251013+0000","last_peered":"2023-10-09T11:08:13.251013+0000","last_clean":"2023-10-09T11:08:13.251013+0000","last_became_active":"2023-10-09T11:06:43.026551+0000","last_became_peered":"2023-10-09T11:06:43.026551+0000","last_unstale":"2023-10-09T11:08:13.251013+0000","last_undegraded":"2023-10-09T11:08:13.251013+0000","last_fullsized":"2023-10-09T11:08:13.251013+0000","mapping_epoch":531,"log_start":"504'1844","ondisk_log_start":"504'1844","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7100,"log_dups_size":0,"ondisk_log_size":7100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26542080,"num_objects":405,"num_object_clones":0,"num_object_copies":1215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5769,"num_write_kb":197568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":889,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.229807+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:08:08.229807+0000","last_peered":"2023-10-09T11:08:08.229807+0000","last_clean":"2023-10-09T11:08:08.229807+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:08:08.229807+0000","last_undegraded":"2023-10-09T11:08:08.229807+0000","last_fullsized":"2023-10-09T11:08:08.229807+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":431,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260802+0000","last_change":"2023-10-09T11:07:33.242665+0000","last_active":"2023-10-09T11:08:08.260802+0000","last_peered":"2023-10-09T11:08:08.260802+0000","last_clean":"2023-10-09T11:08:08.260802+0000","last_became_active":"2023-10-09T11:07:33.242313+0000","last_became_peered":"2023-10-09T11:07:33.242313+0000","last_unstale":"2023-10-09T11:08:08.260802+0000","last_undegraded":"2023-10-09T11:08:08.260802+0000","last_fullsized":"2023-10-09T11:08:08.260802+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:20.565135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":112,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.212848+0000","last_change":"2023-10-09T11:07:33.225683+0000","last_active":"2023-10-09T11:08:07.212848+0000","last_peered":"2023-10-09T11:08:07.212848+0000","last_clean":"2023-10-09T11:08:07.212848+0000","last_became_active":"2023-10-09T11:07:33.225275+0000","last_became_peered":"2023-10-09T11:07:33.225275+0000","last_unstale":"2023-10-09T11:08:07.212848+0000","last_undegraded":"2023-10-09T11:08:07.212848+0000","last_fullsized":"2023-10-09T11:08:07.212848+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:10.131060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1d","version":"560'9250","reported_seq":12171,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T11:08:16.122488+0000","last_change":"2023-10-09T11:07:23.727748+0000","last_active":"2023-10-09T11:08:16.122488+0000","last_peered":"2023-10-09T11:08:16.122488+0000","last_clean":"2023-10-09T11:08:16.122488+0000","last_became_active":"2023-10-09T11:06:43.012614+0000","last_became_peered":"2023-10-09T11:06:43.012614+0000","last_unstale":"2023-10-09T11:08:16.122488+0000","last_undegraded":"2023-10-09T11:08:16.122488+0000","last_fullsized":"2023-10-09T11:08:16.122488+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":9250,"log_dups_size":0,"ondisk_log_size":9250,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16809996,"num_objects":257,"num_object_clones":0,"num_object_copies":771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":257,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6029,"num_write_kb":201121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":704,"num_bytes_recovered":46137344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":789,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.261641+0000","last_change":"2023-10-09T11:05:23.328981+0000","last_active":"2023-10-09T11:08:08.261641+0000","last_peered":"2023-10-09T11:08:08.261641+0000","last_clean":"2023-10-09T11:08:08.261641+0000","last_became_active":"2023-10-09T11:05:23.328616+0000","last_became_peered":"2023-10-09T11:05:23.328616+0000","last_unstale":"2023-10-09T11:08:08.261641+0000","last_undegraded":"2023-10-09T11:08:08.261641+0000","last_fullsized":"2023-10-09T11:08:08.261641+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:15.315819+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":26,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226095+0000","last_change":"2023-10-09T11:07:33.265842+0000","last_active":"2023-10-09T11:08:08.226095+0000","last_peered":"2023-10-09T11:08:08.226095+0000","last_clean":"2023-10-09T11:08:08.226095+0000","last_became_active":"2023-10-09T11:07:33.232311+0000","last_became_peered":"2023-10-09T11:07:33.232311+0000","last_unstale":"2023-10-09T11:08:08.226095+0000","last_undegraded":"2023-10-09T11:08:08.226095+0000","last_fullsized":"2023-10-09T11:08:08.226095+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:23.402582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":338,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260807+0000","last_change":"2023-10-09T11:05:20.316265+0000","last_active":"2023-10-09T11:08:08.260807+0000","last_peered":"2023-10-09T11:08:08.260807+0000","last_clean":"2023-10-09T11:08:08.260807+0000","last_became_active":"2023-10-09T11:05:20.315853+0000","last_became_peered":"2023-10-09T11:05:20.315853+0000","last_unstale":"2023-10-09T11:08:08.260807+0000","last_undegraded":"2023-10-09T11:08:08.260807+0000","last_fullsized":"2023-10-09T11:08:08.260807+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:35.093981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1c","version":"559'9065","reported_seq":9148,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.315875+0000","last_change":"2023-10-09T11:06:43.025845+0000","last_active":"2023-10-09T11:08:13.315875+0000","last_peered":"2023-10-09T11:08:13.315875+0000","last_clean":"2023-10-09T11:08:13.315875+0000","last_became_active":"2023-10-09T11:06:43.022156+0000","last_became_peered":"2023-10-09T11:06:43.022156+0000","last_unstale":"2023-10-09T11:08:13.315875+0000","last_undegraded":"2023-10-09T11:08:13.315875+0000","last_fullsized":"2023-10-09T11:08:13.315875+0000","mapping_epoch":531,"log_start":"504'2080","ondisk_log_start":"504'2080","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6985,"log_dups_size":0,"ondisk_log_size":6985,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25329664,"num_objects":387,"num_object_clones":0,"num_object_copies":1161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5857,"num_write_kb":199776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":951,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.217389+0000","last_change":"2023-10-09T11:04:53.274921+0000","last_active":"2023-10-09T11:08:07.217389+0000","last_peered":"2023-10-09T11:08:07.217389+0000","last_clean":"2023-10-09T11:08:07.217389+0000","last_became_active":"2023-10-09T11:04:53.274518+0000","last_became_peered":"2023-10-09T11:04:53.274518+0000","last_unstale":"2023-10-09T11:08:07.217389+0000","last_undegraded":"2023-10-09T11:08:07.217389+0000","last_fullsized":"2023-10-09T11:08:07.217389+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:14.315301+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":30,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226133+0000","last_change":"2023-10-09T11:07:33.262559+0000","last_active":"2023-10-09T11:08:08.226133+0000","last_peered":"2023-10-09T11:08:08.226133+0000","last_clean":"2023-10-09T11:08:08.226133+0000","last_became_active":"2023-10-09T11:07:33.231695+0000","last_became_peered":"2023-10-09T11:07:33.231695+0000","last_unstale":"2023-10-09T11:08:08.226133+0000","last_undegraded":"2023-10-09T11:08:08.226133+0000","last_fullsized":"2023-10-09T11:08:08.226133+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:59.006940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":581,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.228052+0000","last_change":"2023-10-09T11:05:20.311424+0000","last_active":"2023-10-09T11:08:08.228052+0000","last_peered":"2023-10-09T11:08:08.228052+0000","last_clean":"2023-10-09T11:08:08.228052+0000","last_became_active":"2023-10-09T11:05:20.311097+0000","last_became_peered":"2023-10-09T11:05:20.311097+0000","last_unstale":"2023-10-09T11:08:08.228052+0000","last_undegraded":"2023-10-09T11:08:08.228052+0000","last_fullsized":"2023-10-09T11:08:08.228052+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:19.274582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1b","version":"559'9062","reported_seq":9145,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.311077+0000","last_change":"2023-10-09T11:06:43.035498+0000","last_active":"2023-10-09T11:08:13.311077+0000","last_peered":"2023-10-09T11:08:13.311077+0000","last_clean":"2023-10-09T11:08:13.311077+0000","last_became_active":"2023-10-09T11:06:43.031985+0000","last_became_peered":"2023-10-09T11:06:43.031985+0000","last_unstale":"2023-10-09T11:08:13.311077+0000","last_undegraded":"2023-10-09T11:08:13.311077+0000","last_fullsized":"2023-10-09T11:08:13.311077+0000","mapping_epoch":531,"log_start":"504'2226","ondisk_log_start":"504'2226","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6836,"log_dups_size":0,"ondisk_log_size":6836,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19070976,"num_objects":291,"num_object_clones":0,"num_object_copies":873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5892,"num_write_kb":197856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":589,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260234+0000","last_change":"2023-10-09T11:08:04.745102+0000","last_active":"2023-10-09T11:08:08.260234+0000","last_peered":"2023-10-09T11:08:08.260234+0000","last_clean":"2023-10-09T11:08:08.260234+0000","last_became_active":"2023-10-09T11:06:24.245076+0000","last_became_peered":"2023-10-09T11:06:24.245076+0000","last_unstale":"2023-10-09T11:08:08.260234+0000","last_undegraded":"2023-10-09T11:08:08.260234+0000","last_fullsized":"2023-10-09T11:08:08.260234+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":26,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.225998+0000","last_change":"2023-10-09T11:07:33.269200+0000","last_active":"2023-10-09T11:08:08.225998+0000","last_peered":"2023-10-09T11:08:08.225998+0000","last_clean":"2023-10-09T11:08:08.225998+0000","last_became_active":"2023-10-09T11:07:33.236205+0000","last_became_peered":"2023-10-09T11:07:33.236205+0000","last_unstale":"2023-10-09T11:08:08.225998+0000","last_undegraded":"2023-10-09T11:08:08.225998+0000","last_fullsized":"2023-10-09T11:08:08.225998+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:24.574734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":103,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.211446+0000","last_change":"2023-10-09T11:07:33.227803+0000","last_active":"2023-10-09T11:08:07.211446+0000","last_peered":"2023-10-09T11:08:07.211446+0000","last_clean":"2023-10-09T11:08:07.211446+0000","last_became_active":"2023-10-09T11:07:33.227486+0000","last_became_peered":"2023-10-09T11:07:33.227486+0000","last_unstale":"2023-10-09T11:08:07.211446+0000","last_undegraded":"2023-10-09T11:08:07.211446+0000","last_fullsized":"2023-10-09T11:08:07.211446+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:39.274875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1a","version":"559'9141","reported_seq":12004,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.327330+0000","last_change":"2023-10-09T11:06:43.001534+0000","last_active":"2023-10-09T11:08:13.327330+0000","last_peered":"2023-10-09T11:08:13.327330+0000","last_clean":"2023-10-09T11:08:13.327330+0000","last_became_active":"2023-10-09T11:06:42.998860+0000","last_became_peered":"2023-10-09T11:06:42.998860+0000","last_unstale":"2023-10-09T11:08:13.327330+0000","last_undegraded":"2023-10-09T11:08:13.327330+0000","last_fullsized":"2023-10-09T11:08:13.327330+0000","mapping_epoch":531,"log_start":"504'2435","ondisk_log_start":"504'2435","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6706,"log_dups_size":0,"ondisk_log_size":6706,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21823488,"num_objects":333,"num_object_clones":0,"num_object_copies":999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5931,"num_write_kb":200448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":691,"num_bytes_recovered":45285376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":871,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260639+0000","last_change":"2023-10-09T11:05:23.323856+0000","last_active":"2023-10-09T11:08:08.260639+0000","last_peered":"2023-10-09T11:08:08.260639+0000","last_clean":"2023-10-09T11:08:08.260639+0000","last_became_active":"2023-10-09T11:05:23.322763+0000","last_became_peered":"2023-10-09T11:05:23.322763+0000","last_unstale":"2023-10-09T11:08:08.260639+0000","last_undegraded":"2023-10-09T11:08:08.260639+0000","last_fullsized":"2023-10-09T11:08:08.260639+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.760820+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":20,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.225899+0000","last_change":"2023-10-09T11:07:33.233510+0000","last_active":"2023-10-09T11:08:08.225899+0000","last_peered":"2023-10-09T11:08:08.225899+0000","last_clean":"2023-10-09T11:08:08.225899+0000","last_became_active":"2023-10-09T11:07:33.232339+0000","last_became_peered":"2023-10-09T11:07:33.232339+0000","last_unstale":"2023-10-09T11:08:08.225899+0000","last_undegraded":"2023-10-09T11:08:08.225899+0000","last_fullsized":"2023-10-09T11:08:08.225899+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:55.324912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":501,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260909+0000","last_change":"2023-10-09T11:07:33.226594+0000","last_active":"2023-10-09T11:08:08.260909+0000","last_peered":"2023-10-09T11:08:08.260909+0000","last_clean":"2023-10-09T11:08:08.260909+0000","last_became_active":"2023-10-09T11:07:33.226275+0000","last_became_peered":"2023-10-09T11:07:33.226275+0000","last_unstale":"2023-10-09T11:08:08.260909+0000","last_undegraded":"2023-10-09T11:08:08.260909+0000","last_fullsized":"2023-10-09T11:08:08.260909+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:41.180088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"559'7473","reported_seq":8931,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.264829+0000","last_change":"2023-10-09T11:06:05.166134+0000","last_active":"2023-10-09T11:08:13.264829+0000","last_peered":"2023-10-09T11:08:13.264829+0000","last_clean":"2023-10-09T11:08:13.264829+0000","last_became_active":"2023-10-09T11:05:45.388425+0000","last_became_peered":"2023-10-09T11:05:45.388425+0000","last_unstale":"2023-10-09T11:08:13.264829+0000","last_undegraded":"2023-10-09T11:08:13.264829+0000","last_fullsized":"2023-10-09T11:08:13.264829+0000","mapping_epoch":509,"log_start":"504'2267","ondisk_log_start":"504'2267","created":509,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5206,"log_dups_size":0,"ondisk_log_size":5206,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24346624,"num_objects":372,"num_object_clones":0,"num_object_copies":1116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5910,"num_write_kb":200992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":684,"num_bytes_recovered":44826624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":892,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216559+0000","last_change":"2023-10-09T11:04:53.274236+0000","last_active":"2023-10-09T11:08:07.216559+0000","last_peered":"2023-10-09T11:08:07.216559+0000","last_clean":"2023-10-09T11:08:07.216559+0000","last_became_active":"2023-10-09T11:04:53.273817+0000","last_became_peered":"2023-10-09T11:04:53.273817+0000","last_unstale":"2023-10-09T11:08:07.216559+0000","last_undegraded":"2023-10-09T11:08:07.216559+0000","last_fullsized":"2023-10-09T11:08:07.216559+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:05.096905+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":155,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.225909+0000","last_change":"2023-10-09T11:05:20.312129+0000","last_active":"2023-10-09T11:08:08.225909+0000","last_peered":"2023-10-09T11:08:08.225909+0000","last_clean":"2023-10-09T11:08:08.225909+0000","last_became_active":"2023-10-09T11:05:20.311810+0000","last_became_peered":"2023-10-09T11:05:20.311810+0000","last_unstale":"2023-10-09T11:08:08.225909+0000","last_undegraded":"2023-10-09T11:08:08.225909+0000","last_fullsized":"2023-10-09T11:08:08.225909+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:48.813391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":252,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219203+0000","last_change":"2023-10-09T11:07:33.218244+0000","last_active":"2023-10-09T11:08:07.219203+0000","last_peered":"2023-10-09T11:08:07.219203+0000","last_clean":"2023-10-09T11:08:07.219203+0000","last_became_active":"2023-10-09T11:07:33.217918+0000","last_became_peered":"2023-10-09T11:07:33.217918+0000","last_unstale":"2023-10-09T11:08:07.219203+0000","last_undegraded":"2023-10-09T11:08:07.219203+0000","last_fullsized":"2023-10-09T11:08:07.219203+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:16.150105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.18","version":"559'7352","reported_seq":9906,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.297566+0000","last_change":"2023-10-09T11:06:07.154110+0000","last_active":"2023-10-09T11:08:13.297566+0000","last_peered":"2023-10-09T11:08:13.297566+0000","last_clean":"2023-10-09T11:08:13.297566+0000","last_became_active":"2023-10-09T11:05:45.388306+0000","last_became_peered":"2023-10-09T11:05:45.388306+0000","last_unstale":"2023-10-09T11:08:13.297566+0000","last_undegraded":"2023-10-09T11:08:13.297566+0000","last_fullsized":"2023-10-09T11:08:13.297566+0000","mapping_epoch":509,"log_start":"504'2098","ondisk_log_start":"504'2098","created":509,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5254,"log_dups_size":0,"ondisk_log_size":5254,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24576000,"num_objects":375,"num_object_clones":0,"num_object_copies":1125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5817,"num_write_kb":198144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":645,"num_bytes_recovered":42237952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":868,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.264635+0000","last_change":"2023-10-09T11:05:23.324537+0000","last_active":"2023-10-09T11:08:08.264635+0000","last_peered":"2023-10-09T11:08:08.264635+0000","last_clean":"2023-10-09T11:08:08.264635+0000","last_became_active":"2023-10-09T11:05:23.324187+0000","last_became_peered":"2023-10-09T11:05:23.324187+0000","last_unstale":"2023-10-09T11:08:08.264635+0000","last_undegraded":"2023-10-09T11:08:08.264635+0000","last_fullsized":"2023-10-09T11:08:08.264635+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:25.286106+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":80,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.262706+0000","last_change":"2023-10-09T11:05:20.323565+0000","last_active":"2023-10-09T11:08:08.262706+0000","last_peered":"2023-10-09T11:08:08.262706+0000","last_clean":"2023-10-09T11:08:08.262706+0000","last_became_active":"2023-10-09T11:05:20.309672+0000","last_became_peered":"2023-10-09T11:05:20.309672+0000","last_unstale":"2023-10-09T11:08:08.262706+0000","last_undegraded":"2023-10-09T11:08:08.262706+0000","last_fullsized":"2023-10-09T11:08:08.262706+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:50.211465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":103,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219691+0000","last_change":"2023-10-09T11:07:33.228001+0000","last_active":"2023-10-09T11:08:07.219691+0000","last_peered":"2023-10-09T11:08:07.219691+0000","last_clean":"2023-10-09T11:08:07.219691+0000","last_became_active":"2023-10-09T11:07:33.227691+0000","last_became_peered":"2023-10-09T11:07:33.227691+0000","last_unstale":"2023-10-09T11:08:07.219691+0000","last_undegraded":"2023-10-09T11:08:07.219691+0000","last_fullsized":"2023-10-09T11:08:07.219691+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:24:16.098138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"560'7416","reported_seq":7510,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T11:08:16.717824+0000","last_change":"2023-10-09T11:08:05.741500+0000","last_active":"2023-10-09T11:08:16.717824+0000","last_peered":"2023-10-09T11:08:16.717824+0000","last_clean":"2023-10-09T11:08:16.717824+0000","last_became_active":"2023-10-09T11:05:45.338519+0000","last_became_peered":"2023-10-09T11:05:45.338519+0000","last_unstale":"2023-10-09T11:08:16.717824+0000","last_undegraded":"2023-10-09T11:08:16.717824+0000","last_fullsized":"2023-10-09T11:08:16.717824+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7416,"log_dups_size":0,"ondisk_log_size":7416,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16941056,"num_objects":258,"num_object_clones":0,"num_object_copies":774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5846,"num_write_kb":195360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":792,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226451+0000","last_change":"2023-10-09T11:07:36.284927+0000","last_active":"2023-10-09T11:08:08.226451+0000","last_peered":"2023-10-09T11:08:08.226451+0000","last_clean":"2023-10-09T11:08:08.226451+0000","last_became_active":"2023-10-09T11:07:36.284631+0000","last_became_peered":"2023-10-09T11:07:36.284631+0000","last_unstale":"2023-10-09T11:08:08.226451+0000","last_undegraded":"2023-10-09T11:08:08.226451+0000","last_fullsized":"2023-10-09T11:08:08.226451+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:37.220243+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":24,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226486+0000","last_change":"2023-10-09T11:07:33.273503+0000","last_active":"2023-10-09T11:08:08.226486+0000","last_peered":"2023-10-09T11:08:08.226486+0000","last_clean":"2023-10-09T11:08:08.226486+0000","last_became_active":"2023-10-09T11:07:33.242980+0000","last_became_peered":"2023-10-09T11:07:33.242980+0000","last_unstale":"2023-10-09T11:08:08.226486+0000","last_undegraded":"2023-10-09T11:08:08.226486+0000","last_fullsized":"2023-10-09T11:08:08.226486+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:18.934972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":522,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260418+0000","last_change":"2023-10-09T11:07:33.304363+0000","last_active":"2023-10-09T11:08:08.260418+0000","last_peered":"2023-10-09T11:08:08.260418+0000","last_clean":"2023-10-09T11:08:08.260418+0000","last_became_active":"2023-10-09T11:07:33.304024+0000","last_became_peered":"2023-10-09T11:07:33.304024+0000","last_unstale":"2023-10-09T11:08:08.260418+0000","last_undegraded":"2023-10-09T11:08:08.260418+0000","last_fullsized":"2023-10-09T11:08:08.260418+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:21:51.522688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"560'7391","reported_seq":7493,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T11:08:16.239650+0000","last_change":"2023-10-09T11:07:28.705631+0000","last_active":"2023-10-09T11:08:16.239650+0000","last_peered":"2023-10-09T11:08:16.239650+0000","last_clean":"2023-10-09T11:08:16.239650+0000","last_became_active":"2023-10-09T11:06:24.254388+0000","last_became_peered":"2023-10-09T11:06:24.254388+0000","last_unstale":"2023-10-09T11:08:16.239650+0000","last_undegraded":"2023-10-09T11:08:16.239650+0000","last_fullsized":"2023-10-09T11:08:16.239650+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7391,"log_dups_size":0,"ondisk_log_size":7391,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18677760,"num_objects":285,"num_object_clones":0,"num_object_copies":855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5865,"num_write_kb":196800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":940,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.211519+0000","last_change":"2023-10-09T11:05:23.316129+0000","last_active":"2023-10-09T11:08:07.211519+0000","last_peered":"2023-10-09T11:08:07.211519+0000","last_clean":"2023-10-09T11:08:07.211519+0000","last_became_active":"2023-10-09T11:05:23.315792+0000","last_became_peered":"2023-10-09T11:05:23.315792+0000","last_unstale":"2023-10-09T11:08:07.211519+0000","last_undegraded":"2023-10-09T11:08:07.211519+0000","last_fullsized":"2023-10-09T11:08:07.211519+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:51.770547+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":24,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226272+0000","last_change":"2023-10-09T11:07:33.256691+0000","last_active":"2023-10-09T11:08:08.226272+0000","last_peered":"2023-10-09T11:08:08.226272+0000","last_clean":"2023-10-09T11:08:08.226272+0000","last_became_active":"2023-10-09T11:07:33.229554+0000","last_became_peered":"2023-10-09T11:07:33.229554+0000","last_unstale":"2023-10-09T11:08:08.226272+0000","last_undegraded":"2023-10-09T11:08:08.226272+0000","last_fullsized":"2023-10-09T11:08:08.226272+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:57.414795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":273,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.215808+0000","last_change":"2023-10-09T11:04:50.333538+0000","last_active":"2023-10-09T11:08:07.215808+0000","last_peered":"2023-10-09T11:08:07.215808+0000","last_clean":"2023-10-09T11:08:07.215808+0000","last_became_active":"2023-10-09T11:04:50.332514+0000","last_became_peered":"2023-10-09T11:04:50.332514+0000","last_unstale":"2023-10-09T11:08:07.215808+0000","last_undegraded":"2023-10-09T11:08:07.215808+0000","last_fullsized":"2023-10-09T11:08:07.215808+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:22.933839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"559'7215","reported_seq":11829,"reported_epoch":559,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:08:13.223516+0000","last_change":"2023-10-09T11:08:06.826203+0000","last_active":"2023-10-09T11:08:13.223516+0000","last_peered":"2023-10-09T11:08:13.223516+0000","last_clean":"2023-10-09T11:07:32.132824+0000","last_became_active":"2023-10-09T11:07:34.322525+0000","last_became_peered":"2023-10-09T11:07:34.322525+0000","last_unstale":"2023-10-09T11:08:13.223516+0000","last_undegraded":"2023-10-09T11:08:13.223516+0000","last_fullsized":"2023-10-09T11:08:13.223516+0000","mapping_epoch":550,"log_start":"504'1878","ondisk_log_start":"504'1878","created":495,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5337,"log_dups_size":0,"ondisk_log_size":5337,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26705920,"num_objects":407,"num_object_clones":0,"num_object_copies":1221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":407,"num_objects_unfound":0,"num_objects_dirty":407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5697,"num_write_kb":195360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2245,"num_bytes_recovered":147128320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":407}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":816,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.261722+0000","last_change":"2023-10-09T11:05:23.322251+0000","last_active":"2023-10-09T11:08:08.261722+0000","last_peered":"2023-10-09T11:08:08.261722+0000","last_clean":"2023-10-09T11:08:08.261722+0000","last_became_active":"2023-10-09T11:05:23.321826+0000","last_became_peered":"2023-10-09T11:05:23.321826+0000","last_unstale":"2023-10-09T11:08:08.261722+0000","last_undegraded":"2023-10-09T11:08:08.261722+0000","last_fullsized":"2023-10-09T11:08:08.261722+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.858013+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":24,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226496+0000","last_change":"2023-10-09T11:07:33.271238+0000","last_active":"2023-10-09T11:08:08.226496+0000","last_peered":"2023-10-09T11:08:08.226496+0000","last_clean":"2023-10-09T11:08:08.226496+0000","last_became_active":"2023-10-09T11:07:33.241886+0000","last_became_peered":"2023-10-09T11:07:33.241886+0000","last_unstale":"2023-10-09T11:08:08.226496+0000","last_undegraded":"2023-10-09T11:08:08.226496+0000","last_fullsized":"2023-10-09T11:08:08.226496+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:48.882589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":90,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.262308+0000","last_change":"2023-10-09T11:07:33.244131+0000","last_active":"2023-10-09T11:08:08.262308+0000","last_peered":"2023-10-09T11:08:08.262308+0000","last_clean":"2023-10-09T11:08:08.262308+0000","last_became_active":"2023-10-09T11:07:33.243797+0000","last_became_peered":"2023-10-09T11:07:33.243797+0000","last_unstale":"2023-10-09T11:08:08.262308+0000","last_undegraded":"2023-10-09T11:08:08.262308+0000","last_fullsized":"2023-10-09T11:08:08.262308+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:44.851043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"560'7463","reported_seq":8934,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T11:08:16.598546+0000","last_change":"2023-10-09T11:07:29.087418+0000","last_active":"2023-10-09T11:08:16.598546+0000","last_peered":"2023-10-09T11:08:16.598546+0000","last_clean":"2023-10-09T11:08:16.598546+0000","last_became_active":"2023-10-09T11:05:46.357677+0000","last_became_peered":"2023-10-09T11:05:46.357677+0000","last_unstale":"2023-10-09T11:08:16.598546+0000","last_undegraded":"2023-10-09T11:08:16.598546+0000","last_fullsized":"2023-10-09T11:08:16.598546+0000","mapping_epoch":510,"log_start":"498'1298","ondisk_log_start":"498'1298","created":495,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6165,"log_dups_size":0,"ondisk_log_size":6165,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17170432,"num_objects":262,"num_object_clones":0,"num_object_copies":786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5910,"num_write_kb":197504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":677,"num_bytes_recovered":44367872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":1070,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226320+0000","last_change":"2023-10-09T11:07:36.286417+0000","last_active":"2023-10-09T11:08:08.226320+0000","last_peered":"2023-10-09T11:08:08.226320+0000","last_clean":"2023-10-09T11:08:08.226320+0000","last_became_active":"2023-10-09T11:07:36.286061+0000","last_became_peered":"2023-10-09T11:07:36.286061+0000","last_unstale":"2023-10-09T11:08:08.226320+0000","last_undegraded":"2023-10-09T11:08:08.226320+0000","last_fullsized":"2023-10-09T11:08:08.226320+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:00.289500+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":418,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.264283+0000","last_change":"2023-10-09T11:05:20.309385+0000","last_active":"2023-10-09T11:08:08.264283+0000","last_peered":"2023-10-09T11:08:08.264283+0000","last_clean":"2023-10-09T11:08:08.264283+0000","last_became_active":"2023-10-09T11:05:20.309034+0000","last_became_peered":"2023-10-09T11:05:20.309034+0000","last_unstale":"2023-10-09T11:08:08.264283+0000","last_undegraded":"2023-10-09T11:08:08.264283+0000","last_fullsized":"2023-10-09T11:08:08.264283+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:31.857485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":348,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216475+0000","last_change":"2023-10-09T11:06:24.245966+0000","last_active":"2023-10-09T11:08:07.216475+0000","last_peered":"2023-10-09T11:08:07.216475+0000","last_clean":"2023-10-09T11:08:07.216475+0000","last_became_active":"2023-10-09T11:06:24.245310+0000","last_became_peered":"2023-10-09T11:06:24.245310+0000","last_unstale":"2023-10-09T11:08:07.216475+0000","last_undegraded":"2023-10-09T11:08:07.216475+0000","last_fullsized":"2023-10-09T11:08:07.216475+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:23.966200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.3","version":"559'6215","reported_seq":9258,"reported_epoch":559,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:08:13.334488+0000","last_change":"2023-10-09T11:07:34.314819+0000","last_active":"2023-10-09T11:08:13.334488+0000","last_peered":"2023-10-09T11:08:13.334488+0000","last_clean":"2023-10-09T11:07:32.185795+0000","last_became_active":"2023-10-09T11:07:34.308357+0000","last_became_peered":"2023-10-09T11:07:34.308357+0000","last_unstale":"2023-10-09T11:08:13.334488+0000","last_undegraded":"2023-10-09T11:08:13.334488+0000","last_fullsized":"2023-10-09T11:08:13.334488+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6215,"log_dups_size":0,"ondisk_log_size":6215,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11223040,"num_objects":171,"num_object_clones":0,"num_object_copies":513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":171,"num_objects_unfound":0,"num_objects_dirty":171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2999,"num_write_kb":101456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1223,"num_bytes_recovered":72613888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":171}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":714,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.225809+0000","last_change":"2023-10-09T11:07:36.283174+0000","last_active":"2023-10-09T11:08:08.225809+0000","last_peered":"2023-10-09T11:08:08.225809+0000","last_clean":"2023-10-09T11:08:08.225809+0000","last_became_active":"2023-10-09T11:07:36.278007+0000","last_became_peered":"2023-10-09T11:07:36.278007+0000","last_unstale":"2023-10-09T11:08:08.225809+0000","last_undegraded":"2023-10-09T11:08:08.225809+0000","last_fullsized":"2023-10-09T11:08:08.225809+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:51.693969+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":89,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.212636+0000","last_change":"2023-10-09T11:04:50.529513+0000","last_active":"2023-10-09T11:08:07.212636+0000","last_peered":"2023-10-09T11:08:07.212636+0000","last_clean":"2023-10-09T11:08:07.212636+0000","last_became_active":"2023-10-09T11:04:50.339671+0000","last_became_peered":"2023-10-09T11:04:50.339671+0000","last_unstale":"2023-10-09T11:08:07.212636+0000","last_undegraded":"2023-10-09T11:08:07.212636+0000","last_fullsized":"2023-10-09T11:08:07.212636+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:24.473000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":94,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.211374+0000","last_change":"2023-10-09T11:05:20.313855+0000","last_active":"2023-10-09T11:08:07.211374+0000","last_peered":"2023-10-09T11:08:07.211374+0000","last_clean":"2023-10-09T11:08:07.211374+0000","last_became_active":"2023-10-09T11:05:20.313535+0000","last_became_peered":"2023-10-09T11:05:20.313535+0000","last_unstale":"2023-10-09T11:08:07.211374+0000","last_undegraded":"2023-10-09T11:08:07.211374+0000","last_fullsized":"2023-10-09T11:08:07.211374+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:25.266434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"559'6003","reported_seq":6096,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.196902+0000","last_change":"2023-10-09T11:06:42.991743+0000","last_active":"2023-10-09T11:08:13.196902+0000","last_peered":"2023-10-09T11:08:13.196902+0000","last_clean":"2023-10-09T11:08:13.196902+0000","last_became_active":"2023-10-09T11:06:42.989452+0000","last_became_peered":"2023-10-09T11:06:42.989452+0000","last_unstale":"2023-10-09T11:08:13.196902+0000","last_undegraded":"2023-10-09T11:08:13.196902+0000","last_fullsized":"2023-10-09T11:08:13.196902+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6003,"log_dups_size":0,"ondisk_log_size":6003,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10158080,"num_objects":155,"num_object_clones":0,"num_object_copies":465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2900,"num_write_kb":97760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":818,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260591+0000","last_change":"2023-10-09T11:04:53.272323+0000","last_active":"2023-10-09T11:08:08.260591+0000","last_peered":"2023-10-09T11:08:08.260591+0000","last_clean":"2023-10-09T11:08:08.260591+0000","last_became_active":"2023-10-09T11:04:53.272010+0000","last_became_peered":"2023-10-09T11:04:53.272010+0000","last_unstale":"2023-10-09T11:08:08.260591+0000","last_undegraded":"2023-10-09T11:08:08.260591+0000","last_fullsized":"2023-10-09T11:08:08.260591+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:23.913605+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":582,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216126+0000","last_change":"2023-10-09T11:05:20.307499+0000","last_active":"2023-10-09T11:08:07.216126+0000","last_peered":"2023-10-09T11:08:07.216126+0000","last_clean":"2023-10-09T11:08:07.216126+0000","last_became_active":"2023-10-09T11:05:20.307198+0000","last_became_peered":"2023-10-09T11:05:20.307198+0000","last_unstale":"2023-10-09T11:08:07.216126+0000","last_undegraded":"2023-10-09T11:08:07.216126+0000","last_fullsized":"2023-10-09T11:08:07.216126+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:53.726844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":319,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.230061+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:08:08.230061+0000","last_peered":"2023-10-09T11:08:08.230061+0000","last_clean":"2023-10-09T11:08:08.230061+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:08:08.230061+0000","last_undegraded":"2023-10-09T11:08:08.230061+0000","last_fullsized":"2023-10-09T11:08:08.230061+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"559'6193","reported_seq":7035,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.253695+0000","last_change":"2023-10-09T11:07:20.806034+0000","last_active":"2023-10-09T11:08:13.253695+0000","last_peered":"2023-10-09T11:08:13.253695+0000","last_clean":"2023-10-09T11:08:13.253695+0000","last_became_active":"2023-10-09T11:06:42.992656+0000","last_became_peered":"2023-10-09T11:06:42.992656+0000","last_unstale":"2023-10-09T11:08:13.253695+0000","last_undegraded":"2023-10-09T11:08:13.253695+0000","last_fullsized":"2023-10-09T11:08:13.253695+0000","mapping_epoch":510,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6193,"log_dups_size":0,"ondisk_log_size":6193,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12304384,"num_objects":187,"num_object_clones":0,"num_object_copies":561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2959,"num_write_kb":100720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":328,"num_bytes_recovered":21528576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":909,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216648+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:08:07.216648+0000","last_peered":"2023-10-09T11:08:07.216648+0000","last_clean":"2023-10-09T11:08:07.216648+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:08:07.216648+0000","last_undegraded":"2023-10-09T11:08:07.216648+0000","last_fullsized":"2023-10-09T11:08:07.216648+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":582,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.228188+0000","last_change":"2023-10-09T11:04:57.953307+0000","last_active":"2023-10-09T11:08:08.228188+0000","last_peered":"2023-10-09T11:08:08.228188+0000","last_clean":"2023-10-09T11:08:08.228188+0000","last_became_active":"2023-10-09T11:04:57.953095+0000","last_became_peered":"2023-10-09T11:04:57.953095+0000","last_unstale":"2023-10-09T11:08:08.228188+0000","last_undegraded":"2023-10-09T11:08:08.228188+0000","last_fullsized":"2023-10-09T11:08:08.228188+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:05.562643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":267,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.217128+0000","last_change":"2023-10-09T11:05:20.310255+0000","last_active":"2023-10-09T11:08:07.217128+0000","last_peered":"2023-10-09T11:08:07.217128+0000","last_clean":"2023-10-09T11:08:07.217128+0000","last_became_active":"2023-10-09T11:05:20.309109+0000","last_became_peered":"2023-10-09T11:05:20.309109+0000","last_unstale":"2023-10-09T11:08:07.217128+0000","last_undegraded":"2023-10-09T11:08:07.217128+0000","last_fullsized":"2023-10-09T11:08:07.217128+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:07.429039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.0","version":"559'6067","reported_seq":9938,"reported_epoch":559,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T11:08:13.361526+0000","last_change":"2023-10-09T11:08:06.823204+0000","last_active":"2023-10-09T11:08:13.361526+0000","last_peered":"2023-10-09T11:08:13.361526+0000","last_clean":"2023-10-09T11:07:32.155021+0000","last_became_active":"2023-10-09T11:07:34.321010+0000","last_became_peered":"2023-10-09T11:07:34.321010+0000","last_unstale":"2023-10-09T11:08:13.361526+0000","last_undegraded":"2023-10-09T11:07:34.238729+0000","last_fullsized":"2023-10-09T11:07:34.219306+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6067,"log_dups_size":0,"ondisk_log_size":6067,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14630912,"num_objects":223,"num_object_clones":0,"num_object_copies":669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":475,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2901,"num_write_kb":99984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1880,"num_bytes_recovered":89849856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,3","objects":-252}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":802,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.264636+0000","last_change":"2023-10-09T11:05:23.329329+0000","last_active":"2023-10-09T11:08:08.264636+0000","last_peered":"2023-10-09T11:08:08.264636+0000","last_clean":"2023-10-09T11:08:08.264636+0000","last_became_active":"2023-10-09T11:05:23.328976+0000","last_became_peered":"2023-10-09T11:05:23.328976+0000","last_unstale":"2023-10-09T11:08:08.264636+0000","last_undegraded":"2023-10-09T11:08:08.264636+0000","last_fullsized":"2023-10-09T11:08:08.264636+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:41.848184+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":103,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.210097+0000","last_change":"2023-10-09T11:07:33.236133+0000","last_active":"2023-10-09T11:08:07.210097+0000","last_peered":"2023-10-09T11:08:07.210097+0000","last_clean":"2023-10-09T11:08:07.210097+0000","last_became_active":"2023-10-09T11:07:33.235818+0000","last_became_peered":"2023-10-09T11:07:33.235818+0000","last_unstale":"2023-10-09T11:08:07.210097+0000","last_undegraded":"2023-10-09T11:08:07.210097+0000","last_fullsized":"2023-10-09T11:08:07.210097+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:20.627846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":264,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216979+0000","last_change":"2023-10-09T11:04:50.332978+0000","last_active":"2023-10-09T11:08:07.216979+0000","last_peered":"2023-10-09T11:08:07.216979+0000","last_clean":"2023-10-09T11:08:07.216979+0000","last_became_active":"2023-10-09T11:04:50.332735+0000","last_became_peered":"2023-10-09T11:04:50.332735+0000","last_unstale":"2023-10-09T11:08:07.216979+0000","last_undegraded":"2023-10-09T11:08:07.216979+0000","last_fullsized":"2023-10-09T11:08:07.216979+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:39:44.659751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"560'9195","reported_seq":12121,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T11:08:16.191037+0000","last_change":"2023-10-09T11:07:21.753744+0000","last_active":"2023-10-09T11:08:16.191037+0000","last_peered":"2023-10-09T11:08:16.191037+0000","last_clean":"2023-10-09T11:08:16.191037+0000","last_became_active":"2023-10-09T11:06:42.979233+0000","last_became_peered":"2023-10-09T11:06:42.979233+0000","last_unstale":"2023-10-09T11:08:16.191037+0000","last_undegraded":"2023-10-09T11:08:16.191037+0000","last_fullsized":"2023-10-09T11:08:16.191037+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":495,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":9195,"log_dups_size":0,"ondisk_log_size":9195,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20414476,"num_objects":312,"num_object_clones":0,"num_object_copies":936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5974,"num_write_kb":201120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":705,"num_bytes_recovered":46137344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":1085,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219776+0000","last_change":"2023-10-09T11:07:36.291716+0000","last_active":"2023-10-09T11:08:07.219776+0000","last_peered":"2023-10-09T11:08:07.219776+0000","last_clean":"2023-10-09T11:08:07.219776+0000","last_became_active":"2023-10-09T11:07:36.291424+0000","last_became_peered":"2023-10-09T11:07:36.291424+0000","last_unstale":"2023-10-09T11:08:07.219776+0000","last_undegraded":"2023-10-09T11:08:07.219776+0000","last_fullsized":"2023-10-09T11:08:07.219776+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:22.316387+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":288,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216557+0000","last_change":"2023-10-09T11:07:33.226547+0000","last_active":"2023-10-09T11:08:07.216557+0000","last_peered":"2023-10-09T11:08:07.216557+0000","last_clean":"2023-10-09T11:08:07.216557+0000","last_became_active":"2023-10-09T11:07:33.226176+0000","last_became_peered":"2023-10-09T11:07:33.226176+0000","last_unstale":"2023-10-09T11:08:07.216557+0000","last_undegraded":"2023-10-09T11:08:07.216557+0000","last_fullsized":"2023-10-09T11:08:07.216557+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:16.888732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":435,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260857+0000","last_change":"2023-10-09T11:04:50.341353+0000","last_active":"2023-10-09T11:08:08.260857+0000","last_peered":"2023-10-09T11:08:08.260857+0000","last_clean":"2023-10-09T11:08:08.260857+0000","last_became_active":"2023-10-09T11:04:50.340801+0000","last_became_peered":"2023-10-09T11:04:50.340801+0000","last_unstale":"2023-10-09T11:08:08.260857+0000","last_undegraded":"2023-10-09T11:08:08.260857+0000","last_fullsized":"2023-10-09T11:08:08.260857+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:00.699069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":240,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216260+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:08:07.216260+0000","last_peered":"2023-10-09T11:08:07.216260+0000","last_clean":"2023-10-09T11:08:07.216260+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:08:07.216260+0000","last_undegraded":"2023-10-09T11:08:07.216260+0000","last_fullsized":"2023-10-09T11:08:07.216260+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"559'8945","reported_seq":9029,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.206649+0000","last_change":"2023-10-09T11:06:43.027139+0000","last_active":"2023-10-09T11:08:13.206649+0000","last_peered":"2023-10-09T11:08:13.206649+0000","last_clean":"2023-10-09T11:08:13.206649+0000","last_became_active":"2023-10-09T11:06:43.024164+0000","last_became_peered":"2023-10-09T11:06:43.024164+0000","last_unstale":"2023-10-09T11:08:13.206649+0000","last_undegraded":"2023-10-09T11:08:13.206649+0000","last_fullsized":"2023-10-09T11:08:13.206649+0000","mapping_epoch":495,"log_start":"504'2039","ondisk_log_start":"504'2039","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6906,"log_dups_size":0,"ondisk_log_size":6906,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26738688,"num_objects":408,"num_object_clones":0,"num_object_copies":1224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5775,"num_write_kb":197856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":760,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260435+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:08:08.260435+0000","last_peered":"2023-10-09T11:08:08.260435+0000","last_clean":"2023-10-09T11:08:08.260435+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:08:08.260435+0000","last_undegraded":"2023-10-09T11:08:08.260435+0000","last_fullsized":"2023-10-09T11:08:08.260435+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":442,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.264385+0000","last_change":"2023-10-09T11:05:20.316093+0000","last_active":"2023-10-09T11:08:08.264385+0000","last_peered":"2023-10-09T11:08:08.264385+0000","last_clean":"2023-10-09T11:08:08.264385+0000","last_became_active":"2023-10-09T11:05:20.315471+0000","last_became_peered":"2023-10-09T11:05:20.315471+0000","last_unstale":"2023-10-09T11:08:08.264385+0000","last_undegraded":"2023-10-09T11:08:08.264385+0000","last_fullsized":"2023-10-09T11:08:08.264385+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:47.548763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":466,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.261516+0000","last_change":"2023-10-09T11:06:24.245282+0000","last_active":"2023-10-09T11:08:08.261516+0000","last_peered":"2023-10-09T11:08:08.261516+0000","last_clean":"2023-10-09T11:08:08.261516+0000","last_became_active":"2023-10-09T11:06:24.244981+0000","last_became_peered":"2023-10-09T11:06:24.244981+0000","last_unstale":"2023-10-09T11:08:08.261516+0000","last_undegraded":"2023-10-09T11:08:08.261516+0000","last_fullsized":"2023-10-09T11:08:08.261516+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:15.327655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":273,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216293+0000","last_change":"2023-10-09T11:04:50.311579+0000","last_active":"2023-10-09T11:08:07.216293+0000","last_peered":"2023-10-09T11:08:07.216293+0000","last_clean":"2023-10-09T11:08:07.216293+0000","last_became_active":"2023-10-09T11:04:50.311270+0000","last_became_peered":"2023-10-09T11:04:50.311270+0000","last_unstale":"2023-10-09T11:08:07.216293+0000","last_undegraded":"2023-10-09T11:08:07.216293+0000","last_fullsized":"2023-10-09T11:08:07.216293+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:49.850302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"559'7272","reported_seq":9848,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.297455+0000","last_change":"2023-10-09T11:06:09.940480+0000","last_active":"2023-10-09T11:08:13.297455+0000","last_peered":"2023-10-09T11:08:13.297455+0000","last_clean":"2023-10-09T11:08:13.297455+0000","last_became_active":"2023-10-09T11:05:46.348515+0000","last_became_peered":"2023-10-09T11:05:46.348515+0000","last_unstale":"2023-10-09T11:08:13.297455+0000","last_undegraded":"2023-10-09T11:08:13.297455+0000","last_fullsized":"2023-10-09T11:08:13.297455+0000","mapping_epoch":510,"log_start":"504'1967","ondisk_log_start":"504'1967","created":495,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5305,"log_dups_size":0,"ondisk_log_size":5305,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24707072,"num_objects":377,"num_object_clones":0,"num_object_copies":1131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5737,"num_write_kb":195648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":654,"num_bytes_recovered":42893312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":912,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226121+0000","last_change":"2023-10-09T11:07:36.284280+0000","last_active":"2023-10-09T11:08:08.226121+0000","last_peered":"2023-10-09T11:08:08.226121+0000","last_clean":"2023-10-09T11:08:08.226121+0000","last_became_active":"2023-10-09T11:07:36.283966+0000","last_became_peered":"2023-10-09T11:07:36.283966+0000","last_unstale":"2023-10-09T11:08:08.226121+0000","last_undegraded":"2023-10-09T11:08:08.226121+0000","last_fullsized":"2023-10-09T11:08:08.226121+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:54.128331+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":338,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.228419+0000","last_change":"2023-10-09T11:04:50.381744+0000","last_active":"2023-10-09T11:08:08.228419+0000","last_peered":"2023-10-09T11:08:08.228419+0000","last_clean":"2023-10-09T11:08:08.228419+0000","last_became_active":"2023-10-09T11:04:50.381253+0000","last_became_peered":"2023-10-09T11:04:50.381253+0000","last_unstale":"2023-10-09T11:08:08.228419+0000","last_undegraded":"2023-10-09T11:08:08.228419+0000","last_fullsized":"2023-10-09T11:08:08.228419+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:24.030274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":259,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.230873+0000","last_change":"2023-10-09T11:05:20.298542+0000","last_active":"2023-10-09T11:08:08.230873+0000","last_peered":"2023-10-09T11:08:08.230873+0000","last_clean":"2023-10-09T11:08:08.230873+0000","last_became_active":"2023-10-09T11:05:20.298119+0000","last_became_peered":"2023-10-09T11:05:20.298119+0000","last_unstale":"2023-10-09T11:08:08.230873+0000","last_undegraded":"2023-10-09T11:08:08.230873+0000","last_fullsized":"2023-10-09T11:08:08.230873+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:00.827214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":238,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.217682+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:08:07.217682+0000","last_peered":"2023-10-09T11:08:07.217682+0000","last_clean":"2023-10-09T11:08:07.217682+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:08:07.217682+0000","last_undegraded":"2023-10-09T11:08:07.217682+0000","last_fullsized":"2023-10-09T11:08:07.217682+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"559'9042","reported_seq":11914,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.241305+0000","last_change":"2023-10-09T11:06:54.755841+0000","last_active":"2023-10-09T11:08:13.241305+0000","last_peered":"2023-10-09T11:08:13.241305+0000","last_clean":"2023-10-09T11:08:13.241305+0000","last_became_active":"2023-10-09T11:06:42.985753+0000","last_became_peered":"2023-10-09T11:06:42.985753+0000","last_unstale":"2023-10-09T11:08:13.241305+0000","last_undegraded":"2023-10-09T11:08:13.241305+0000","last_fullsized":"2023-10-09T11:08:13.241305+0000","mapping_epoch":513,"log_start":"504'2035","ondisk_log_start":"504'2035","created":495,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7007,"log_dups_size":0,"ondisk_log_size":7007,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28311552,"num_objects":432,"num_object_clones":0,"num_object_copies":1296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5832,"num_write_kb":200448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":693,"num_bytes_recovered":45285376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":435,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260675+0000","last_change":"2023-10-09T11:07:33.227052+0000","last_active":"2023-10-09T11:08:08.260675+0000","last_peered":"2023-10-09T11:08:08.260675+0000","last_clean":"2023-10-09T11:08:08.260675+0000","last_became_active":"2023-10-09T11:07:33.226289+0000","last_became_peered":"2023-10-09T11:07:33.226289+0000","last_unstale":"2023-10-09T11:08:08.260675+0000","last_undegraded":"2023-10-09T11:08:08.260675+0000","last_fullsized":"2023-10-09T11:08:08.260675+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:15.605946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":857,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226198+0000","last_change":"2023-10-09T11:07:36.290700+0000","last_active":"2023-10-09T11:08:08.226198+0000","last_peered":"2023-10-09T11:08:08.226198+0000","last_clean":"2023-10-09T11:08:08.226198+0000","last_became_active":"2023-10-09T11:07:36.290357+0000","last_became_peered":"2023-10-09T11:07:36.290357+0000","last_unstale":"2023-10-09T11:08:08.226198+0000","last_undegraded":"2023-10-09T11:08:08.226198+0000","last_fullsized":"2023-10-09T11:08:08.226198+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:34.494777+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":94,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.213985+0000","last_change":"2023-10-09T11:05:20.310666+0000","last_active":"2023-10-09T11:08:07.213985+0000","last_peered":"2023-10-09T11:08:07.213985+0000","last_clean":"2023-10-09T11:08:07.213985+0000","last_became_active":"2023-10-09T11:05:20.310413+0000","last_became_peered":"2023-10-09T11:05:20.310413+0000","last_unstale":"2023-10-09T11:08:07.213985+0000","last_undegraded":"2023-10-09T11:08:07.213985+0000","last_fullsized":"2023-10-09T11:08:07.213985+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:34.169905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":520,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226401+0000","last_change":"2023-10-09T11:05:20.316523+0000","last_active":"2023-10-09T11:08:08.226401+0000","last_peered":"2023-10-09T11:08:08.226401+0000","last_clean":"2023-10-09T11:08:08.226401+0000","last_became_active":"2023-10-09T11:05:20.314493+0000","last_became_peered":"2023-10-09T11:05:20.314493+0000","last_unstale":"2023-10-09T11:08:08.226401+0000","last_undegraded":"2023-10-09T11:08:08.226401+0000","last_fullsized":"2023-10-09T11:08:08.226401+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:41.618701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"559'7337","reported_seq":8727,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.253612+0000","last_change":"2023-10-09T11:06:34.311475+0000","last_active":"2023-10-09T11:08:13.253612+0000","last_peered":"2023-10-09T11:08:13.253612+0000","last_clean":"2023-10-09T11:08:13.253612+0000","last_became_active":"2023-10-09T11:05:46.356286+0000","last_became_peered":"2023-10-09T11:05:46.356286+0000","last_unstale":"2023-10-09T11:08:13.253612+0000","last_undegraded":"2023-10-09T11:08:13.253612+0000","last_fullsized":"2023-10-09T11:08:13.253612+0000","mapping_epoch":510,"log_start":"504'2037","ondisk_log_start":"504'2037","created":495,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5300,"log_dups_size":0,"ondisk_log_size":5300,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25919488,"num_objects":395,"num_object_clones":0,"num_object_copies":1185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5773,"num_write_kb":197408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":650,"num_bytes_recovered":42598400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":929,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216393+0000","last_change":"2023-10-09T11:05:23.322074+0000","last_active":"2023-10-09T11:08:07.216393+0000","last_peered":"2023-10-09T11:08:07.216393+0000","last_clean":"2023-10-09T11:08:07.216393+0000","last_became_active":"2023-10-09T11:05:23.321749+0000","last_became_peered":"2023-10-09T11:05:23.321749+0000","last_unstale":"2023-10-09T11:08:07.216393+0000","last_undegraded":"2023-10-09T11:08:07.216393+0000","last_fullsized":"2023-10-09T11:08:07.216393+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:07.010932+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":135,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226537+0000","last_change":"2023-10-09T11:02:33.198484+0000","last_active":"2023-10-09T11:08:08.226537+0000","last_peered":"2023-10-09T11:08:08.226537+0000","last_clean":"2023-10-09T11:08:08.226537+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:08:08.226537+0000","last_undegraded":"2023-10-09T11:08:08.226537+0000","last_fullsized":"2023-10-09T11:08:08.226537+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":185,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.212624+0000","last_change":"2023-10-09T11:06:24.248362+0000","last_active":"2023-10-09T11:08:07.212624+0000","last_peered":"2023-10-09T11:08:07.212624+0000","last_clean":"2023-10-09T11:08:07.212624+0000","last_became_active":"2023-10-09T11:06:24.247348+0000","last_became_peered":"2023-10-09T11:06:24.247348+0000","last_unstale":"2023-10-09T11:08:07.212624+0000","last_undegraded":"2023-10-09T11:08:07.212624+0000","last_fullsized":"2023-10-09T11:08:07.212624+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:26.728754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":384,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216799+0000","last_change":"2023-10-09T11:06:24.253511+0000","last_active":"2023-10-09T11:08:07.216799+0000","last_peered":"2023-10-09T11:08:07.216799+0000","last_clean":"2023-10-09T11:08:07.216799+0000","last_became_active":"2023-10-09T11:06:24.252739+0000","last_became_peered":"2023-10-09T11:06:24.252739+0000","last_unstale":"2023-10-09T11:08:07.216799+0000","last_undegraded":"2023-10-09T11:08:07.216799+0000","last_fullsized":"2023-10-09T11:08:07.216799+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:50.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"559'9068","reported_seq":9152,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.265289+0000","last_change":"2023-10-09T11:06:42.983758+0000","last_active":"2023-10-09T11:08:13.265289+0000","last_peered":"2023-10-09T11:08:13.265289+0000","last_clean":"2023-10-09T11:08:13.265289+0000","last_became_active":"2023-10-09T11:06:42.981460+0000","last_became_peered":"2023-10-09T11:06:42.981460+0000","last_unstale":"2023-10-09T11:08:13.265289+0000","last_undegraded":"2023-10-09T11:08:13.265289+0000","last_fullsized":"2023-10-09T11:08:13.265289+0000","mapping_epoch":495,"log_start":"504'1975","ondisk_log_start":"504'1975","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7093,"log_dups_size":0,"ondisk_log_size":7093,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25133056,"num_objects":383,"num_object_clones":0,"num_object_copies":1149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5859,"num_write_kb":199776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":789,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219030+0000","last_change":"2023-10-09T11:06:24.239676+0000","last_active":"2023-10-09T11:08:07.219030+0000","last_peered":"2023-10-09T11:08:07.219030+0000","last_clean":"2023-10-09T11:08:07.219030+0000","last_became_active":"2023-10-09T11:06:24.239375+0000","last_became_peered":"2023-10-09T11:06:24.239375+0000","last_unstale":"2023-10-09T11:08:07.219030+0000","last_undegraded":"2023-10-09T11:08:07.219030+0000","last_fullsized":"2023-10-09T11:08:07.219030+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:27.953053+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226008+0000","last_change":"2023-10-09T11:07:33.229415+0000","last_active":"2023-10-09T11:08:08.226008+0000","last_peered":"2023-10-09T11:08:08.226008+0000","last_clean":"2023-10-09T11:08:08.226008+0000","last_became_active":"2023-10-09T11:07:33.229061+0000","last_became_peered":"2023-10-09T11:07:33.229061+0000","last_unstale":"2023-10-09T11:08:08.226008+0000","last_undegraded":"2023-10-09T11:08:08.226008+0000","last_fullsized":"2023-10-09T11:08:08.226008+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:30.978293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":240,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216749+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:08:07.216749+0000","last_peered":"2023-10-09T11:08:07.216749+0000","last_clean":"2023-10-09T11:08:07.216749+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:08:07.216749+0000","last_undegraded":"2023-10-09T11:08:07.216749+0000","last_fullsized":"2023-10-09T11:08:07.216749+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":519,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260620+0000","last_change":"2023-10-09T11:07:33.227271+0000","last_active":"2023-10-09T11:08:08.260620+0000","last_peered":"2023-10-09T11:08:08.260620+0000","last_clean":"2023-10-09T11:08:08.260620+0000","last_became_active":"2023-10-09T11:07:33.226760+0000","last_became_peered":"2023-10-09T11:07:33.226760+0000","last_unstale":"2023-10-09T11:08:08.260620+0000","last_undegraded":"2023-10-09T11:08:08.260620+0000","last_fullsized":"2023-10-09T11:08:08.260620+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:12.268123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"559'9098","reported_seq":12059,"reported_epoch":559,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T11:08:13.294359+0000","last_change":"2023-10-09T11:07:34.375236+0000","last_active":"2023-10-09T11:08:13.294359+0000","last_peered":"2023-10-09T11:08:13.294359+0000","last_clean":"2023-10-09T11:06:06.963741+0000","last_became_active":"2023-10-09T11:07:34.357802+0000","last_became_peered":"2023-10-09T11:07:34.357802+0000","last_unstale":"2023-10-09T11:08:13.294359+0000","last_undegraded":"2023-10-09T11:07:34.289955+0000","last_fullsized":"2023-10-09T11:07:34.281689+0000","mapping_epoch":550,"log_start":"504'1990","ondisk_log_start":"504'1990","created":495,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7108,"log_dups_size":0,"ondisk_log_size":7108,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28311552,"num_objects":432,"num_object_clones":0,"num_object_copies":1296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2256,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5868,"num_write_kb":201600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1434,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1,2","objects":369},{"shards":"1,2","objects":-313}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":759,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.229991+0000","last_change":"2023-10-09T11:05:23.323368+0000","last_active":"2023-10-09T11:08:08.229991+0000","last_peered":"2023-10-09T11:08:08.229991+0000","last_clean":"2023-10-09T11:08:08.229991+0000","last_became_active":"2023-10-09T11:05:23.323032+0000","last_became_peered":"2023-10-09T11:05:23.323032+0000","last_unstale":"2023-10-09T11:08:08.229991+0000","last_undegraded":"2023-10-09T11:08:08.229991+0000","last_fullsized":"2023-10-09T11:08:08.229991+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:09.393584+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":115,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219617+0000","last_change":"2023-10-09T11:07:33.246378+0000","last_active":"2023-10-09T11:08:07.219617+0000","last_peered":"2023-10-09T11:08:07.219617+0000","last_clean":"2023-10-09T11:08:07.219617+0000","last_became_active":"2023-10-09T11:07:33.228019+0000","last_became_peered":"2023-10-09T11:07:33.228019+0000","last_unstale":"2023-10-09T11:08:07.219617+0000","last_undegraded":"2023-10-09T11:08:07.219617+0000","last_fullsized":"2023-10-09T11:08:07.219617+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:18.511517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":267,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.217271+0000","last_change":"2023-10-09T11:05:20.310417+0000","last_active":"2023-10-09T11:08:07.217271+0000","last_peered":"2023-10-09T11:08:07.217271+0000","last_clean":"2023-10-09T11:08:07.217271+0000","last_became_active":"2023-10-09T11:05:20.309827+0000","last_became_peered":"2023-10-09T11:05:20.309827+0000","last_unstale":"2023-10-09T11:08:07.217271+0000","last_undegraded":"2023-10-09T11:08:07.217271+0000","last_fullsized":"2023-10-09T11:08:07.217271+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:23.934383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":103,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.209969+0000","last_change":"2023-10-09T11:07:33.217595+0000","last_active":"2023-10-09T11:08:07.209969+0000","last_peered":"2023-10-09T11:08:07.209969+0000","last_clean":"2023-10-09T11:08:07.209969+0000","last_became_active":"2023-10-09T11:07:33.217155+0000","last_became_peered":"2023-10-09T11:07:33.217155+0000","last_unstale":"2023-10-09T11:08:07.209969+0000","last_undegraded":"2023-10-09T11:08:07.209969+0000","last_fullsized":"2023-10-09T11:08:07.209969+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:55.127329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"559'8996","reported_seq":9079,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:13.327448+0000","last_change":"2023-10-09T11:06:42.990792+0000","last_active":"2023-10-09T11:08:13.327448+0000","last_peered":"2023-10-09T11:08:13.327448+0000","last_clean":"2023-10-09T11:08:13.327448+0000","last_became_active":"2023-10-09T11:06:42.988050+0000","last_became_peered":"2023-10-09T11:06:42.988050+0000","last_unstale":"2023-10-09T11:08:13.327448+0000","last_undegraded":"2023-10-09T11:08:13.327448+0000","last_fullsized":"2023-10-09T11:08:13.327448+0000","mapping_epoch":495,"log_start":"504'1994","ondisk_log_start":"504'1994","created":495,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7002,"log_dups_size":0,"ondisk_log_size":7002,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23134208,"num_objects":353,"num_object_clones":0,"num_object_copies":1059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5821,"num_write_kb":197568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":722,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.211297+0000","last_change":"2023-10-09T11:07:36.284491+0000","last_active":"2023-10-09T11:08:07.211297+0000","last_peered":"2023-10-09T11:08:07.211297+0000","last_clean":"2023-10-09T11:08:07.211297+0000","last_became_active":"2023-10-09T11:07:36.284249+0000","last_became_peered":"2023-10-09T11:07:36.284249+0000","last_unstale":"2023-10-09T11:08:07.211297+0000","last_undegraded":"2023-10-09T11:08:07.211297+0000","last_fullsized":"2023-10-09T11:08:07.211297+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:03.653371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:03.653371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:09.629417+0000","scrub_duration":0.001576771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226497+0000","last_change":"2023-10-09T11:07:33.241346+0000","last_active":"2023-10-09T11:08:08.226497+0000","last_peered":"2023-10-09T11:08:08.226497+0000","last_clean":"2023-10-09T11:08:08.226497+0000","last_became_active":"2023-10-09T11:07:33.241013+0000","last_became_peered":"2023-10-09T11:07:33.241013+0000","last_unstale":"2023-10-09T11:08:08.226497+0000","last_undegraded":"2023-10-09T11:08:08.226497+0000","last_fullsized":"2023-10-09T11:08:08.226497+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:57.164502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":257,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.217060+0000","last_change":"2023-10-09T11:06:24.253714+0000","last_active":"2023-10-09T11:08:07.217060+0000","last_peered":"2023-10-09T11:08:07.217060+0000","last_clean":"2023-10-09T11:08:07.217060+0000","last_became_active":"2023-10-09T11:06:24.253143+0000","last_became_peered":"2023-10-09T11:06:24.253143+0000","last_unstale":"2023-10-09T11:08:07.217060+0000","last_undegraded":"2023-10-09T11:08:07.217060+0000","last_fullsized":"2023-10-09T11:08:07.217060+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:05.682288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":365,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260379+0000","last_change":"2023-10-09T11:05:20.313560+0000","last_active":"2023-10-09T11:08:08.260379+0000","last_peered":"2023-10-09T11:08:08.260379+0000","last_clean":"2023-10-09T11:08:08.260379+0000","last_became_active":"2023-10-09T11:05:20.313214+0000","last_became_peered":"2023-10-09T11:05:20.313214+0000","last_unstale":"2023-10-09T11:08:08.260379+0000","last_undegraded":"2023-10-09T11:08:08.260379+0000","last_fullsized":"2023-10-09T11:08:08.260379+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:32.527205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.13","version":"560'7667","reported_seq":9460,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T11:08:16.540785+0000","last_change":"2023-10-09T11:07:34.333312+0000","last_active":"2023-10-09T11:08:16.540785+0000","last_peered":"2023-10-09T11:08:16.540785+0000","last_clean":"2023-10-09T11:08:16.540785+0000","last_became_active":"2023-10-09T11:07:34.285534+0000","last_became_peered":"2023-10-09T11:07:34.285534+0000","last_unstale":"2023-10-09T11:08:16.540785+0000","last_undegraded":"2023-10-09T11:08:16.540785+0000","last_fullsized":"2023-10-09T11:08:16.540785+0000","mapping_epoch":550,"log_start":"503'1635","ondisk_log_start":"503'1635","created":509,"last_epoch_clean":551,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6032,"log_dups_size":0,"ondisk_log_size":6032,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19365888,"num_objects":296,"num_object_clones":0,"num_object_copies":888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6076,"num_write_kb":203872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":838,"num_bytes_recovered":54853632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":966,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.212889+0000","last_change":"2023-10-09T11:05:23.329886+0000","last_active":"2023-10-09T11:08:07.212889+0000","last_peered":"2023-10-09T11:08:07.212889+0000","last_clean":"2023-10-09T11:08:07.212889+0000","last_became_active":"2023-10-09T11:05:23.329561+0000","last_became_peered":"2023-10-09T11:05:23.329561+0000","last_unstale":"2023-10-09T11:08:07.212889+0000","last_undegraded":"2023-10-09T11:08:07.212889+0000","last_fullsized":"2023-10-09T11:08:07.212889+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:46.933799+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":20,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226294+0000","last_change":"2023-10-09T11:07:33.229864+0000","last_active":"2023-10-09T11:08:08.226294+0000","last_peered":"2023-10-09T11:08:08.226294+0000","last_clean":"2023-10-09T11:08:08.226294+0000","last_became_active":"2023-10-09T11:07:33.229547+0000","last_became_peered":"2023-10-09T11:07:33.229547+0000","last_unstale":"2023-10-09T11:08:08.226294+0000","last_undegraded":"2023-10-09T11:08:08.226294+0000","last_fullsized":"2023-10-09T11:08:08.226294+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:34.990439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":342,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.229882+0000","last_change":"2023-10-09T11:04:50.510688+0000","last_active":"2023-10-09T11:08:08.229882+0000","last_peered":"2023-10-09T11:08:08.229882+0000","last_clean":"2023-10-09T11:08:08.229882+0000","last_became_active":"2023-10-09T11:04:50.507404+0000","last_became_peered":"2023-10-09T11:04:50.507404+0000","last_unstale":"2023-10-09T11:08:08.229882+0000","last_undegraded":"2023-10-09T11:08:08.229882+0000","last_fullsized":"2023-10-09T11:08:08.229882+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:13.663394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"560'7413","reported_seq":8861,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T11:08:16.526728+0000","last_change":"2023-10-09T11:07:26.133461+0000","last_active":"2023-10-09T11:08:16.526728+0000","last_peered":"2023-10-09T11:08:16.526728+0000","last_clean":"2023-10-09T11:08:16.526728+0000","last_became_active":"2023-10-09T11:05:45.387324+0000","last_became_peered":"2023-10-09T11:05:45.387324+0000","last_unstale":"2023-10-09T11:08:16.526728+0000","last_undegraded":"2023-10-09T11:08:16.526728+0000","last_fullsized":"2023-10-09T11:08:16.526728+0000","mapping_epoch":509,"log_start":"498'1195","ondisk_log_start":"498'1195","created":509,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":6218,"log_dups_size":0,"ondisk_log_size":6218,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20185088,"num_objects":308,"num_object_clones":0,"num_object_copies":924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5860,"num_write_kb":197376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":667,"num_bytes_recovered":43712512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":923,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219354+0000","last_change":"2023-10-09T11:05:23.321040+0000","last_active":"2023-10-09T11:08:07.219354+0000","last_peered":"2023-10-09T11:08:07.219354+0000","last_clean":"2023-10-09T11:08:07.219354+0000","last_became_active":"2023-10-09T11:05:23.320674+0000","last_became_peered":"2023-10-09T11:05:23.320674+0000","last_unstale":"2023-10-09T11:08:07.219354+0000","last_undegraded":"2023-10-09T11:08:07.219354+0000","last_fullsized":"2023-10-09T11:08:07.219354+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.527660+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":387,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.264228+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:08:08.264228+0000","last_peered":"2023-10-09T11:08:08.264228+0000","last_clean":"2023-10-09T11:08:08.264228+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:08:08.264228+0000","last_undegraded":"2023-10-09T11:08:08.264228+0000","last_fullsized":"2023-10-09T11:08:08.264228+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.212556+0000","last_change":"2023-10-09T11:04:50.511984+0000","last_active":"2023-10-09T11:08:07.212556+0000","last_peered":"2023-10-09T11:08:07.212556+0000","last_clean":"2023-10-09T11:08:07.212556+0000","last_became_active":"2023-10-09T11:04:50.510820+0000","last_became_peered":"2023-10-09T11:04:50.510820+0000","last_unstale":"2023-10-09T11:08:07.212556+0000","last_undegraded":"2023-10-09T11:08:07.212556+0000","last_fullsized":"2023-10-09T11:08:07.212556+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:38.662189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.15","version":"559'7248","reported_seq":11929,"reported_epoch":559,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T11:08:13.296256+0000","last_change":"2023-10-09T11:07:34.356257+0000","last_active":"2023-10-09T11:08:13.296256+0000","last_peered":"2023-10-09T11:08:13.296256+0000","last_clean":"2023-10-09T11:07:32.098315+0000","last_became_active":"2023-10-09T11:07:34.338915+0000","last_became_peered":"2023-10-09T11:07:34.338915+0000","last_unstale":"2023-10-09T11:08:13.296256+0000","last_undegraded":"2023-10-09T11:08:13.296256+0000","last_fullsized":"2023-10-09T11:08:13.296256+0000","mapping_epoch":550,"log_start":"504'1940","ondisk_log_start":"504'1940","created":509,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":5308,"log_dups_size":0,"ondisk_log_size":5308,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21921792,"num_objects":335,"num_object_clones":0,"num_object_copies":1005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":335,"num_objects_unfound":0,"num_objects_dirty":335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5731,"num_write_kb":194080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2280,"num_bytes_recovered":149422080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":335}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":538,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260857+0000","last_change":"2023-10-09T11:05:20.327032+0000","last_active":"2023-10-09T11:08:08.260857+0000","last_peered":"2023-10-09T11:08:08.260857+0000","last_clean":"2023-10-09T11:08:08.260857+0000","last_became_active":"2023-10-09T11:05:20.320193+0000","last_became_peered":"2023-10-09T11:05:20.320193+0000","last_unstale":"2023-10-09T11:08:08.260857+0000","last_undegraded":"2023-10-09T11:08:08.260857+0000","last_fullsized":"2023-10-09T11:08:08.260857+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:44.862890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.213812+0000","last_change":"2023-10-09T11:04:50.384219+0000","last_active":"2023-10-09T11:08:07.213812+0000","last_peered":"2023-10-09T11:08:07.213812+0000","last_clean":"2023-10-09T11:08:07.213812+0000","last_became_active":"2023-10-09T11:04:50.384028+0000","last_became_peered":"2023-10-09T11:04:50.384028+0000","last_unstale":"2023-10-09T11:08:07.213812+0000","last_undegraded":"2023-10-09T11:08:07.213812+0000","last_fullsized":"2023-10-09T11:08:07.213812+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:04.721869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.16","version":"560'7478","reported_seq":7576,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T11:08:16.209594+0000","last_change":"2023-10-09T11:07:24.733266+0000","last_active":"2023-10-09T11:08:16.209594+0000","last_peered":"2023-10-09T11:08:16.209594+0000","last_clean":"2023-10-09T11:08:16.209594+0000","last_became_active":"2023-10-09T11:06:24.254366+0000","last_became_peered":"2023-10-09T11:06:24.254366+0000","last_unstale":"2023-10-09T11:08:16.209594+0000","last_undegraded":"2023-10-09T11:08:16.209594+0000","last_fullsized":"2023-10-09T11:08:16.209594+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7478,"log_dups_size":0,"ondisk_log_size":7478,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17170432,"num_objects":262,"num_object_clones":0,"num_object_copies":786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5952,"num_write_kb":198848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":733,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.226024+0000","last_change":"2023-10-09T11:07:36.291401+0000","last_active":"2023-10-09T11:08:08.226024+0000","last_peered":"2023-10-09T11:08:08.226024+0000","last_clean":"2023-10-09T11:08:08.226024+0000","last_became_active":"2023-10-09T11:07:36.291077+0000","last_became_peered":"2023-10-09T11:07:36.291077+0000","last_unstale":"2023-10-09T11:08:08.226024+0000","last_undegraded":"2023-10-09T11:08:08.226024+0000","last_fullsized":"2023-10-09T11:08:08.226024+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:04.790935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.790935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":94,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219433+0000","last_change":"2023-10-09T11:05:20.299321+0000","last_active":"2023-10-09T11:08:07.219433+0000","last_peered":"2023-10-09T11:08:07.219433+0000","last_clean":"2023-10-09T11:08:07.219433+0000","last_became_active":"2023-10-09T11:05:20.299019+0000","last_became_peered":"2023-10-09T11:05:20.299019+0000","last_unstale":"2023-10-09T11:08:07.219433+0000","last_undegraded":"2023-10-09T11:08:07.219433+0000","last_fullsized":"2023-10-09T11:08:07.219433+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:08.645909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":85,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.212913+0000","last_change":"2023-10-09T11:04:50.366890+0000","last_active":"2023-10-09T11:08:07.212913+0000","last_peered":"2023-10-09T11:08:07.212913+0000","last_clean":"2023-10-09T11:08:07.212913+0000","last_became_active":"2023-10-09T11:04:50.366558+0000","last_became_peered":"2023-10-09T11:04:50.366558+0000","last_unstale":"2023-10-09T11:08:07.212913+0000","last_undegraded":"2023-10-09T11:08:07.212913+0000","last_fullsized":"2023-10-09T11:08:07.212913+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:50.993153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"560'7438","reported_seq":7531,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T11:08:16.703910+0000","last_change":"2023-10-09T11:08:07.700054+0000","last_active":"2023-10-09T11:08:16.703910+0000","last_peered":"2023-10-09T11:08:16.703910+0000","last_clean":"2023-10-09T11:08:16.703910+0000","last_became_active":"2023-10-09T11:05:45.356981+0000","last_became_peered":"2023-10-09T11:05:45.356981+0000","last_unstale":"2023-10-09T11:08:16.703910+0000","last_undegraded":"2023-10-09T11:08:16.703910+0000","last_fullsized":"2023-10-09T11:08:16.703910+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":509,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:05:04.262580+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.262580+0000","objects_scrubbed":0,"log_size":7438,"log_dups_size":0,"ondisk_log_size":7438,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19169280,"num_objects":293,"num_object_clones":0,"num_object_copies":879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5869,"num_write_kb":197152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":858,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.216604+0000","last_change":"2023-10-09T11:04:53.274332+0000","last_active":"2023-10-09T11:08:07.216604+0000","last_peered":"2023-10-09T11:08:07.216604+0000","last_clean":"2023-10-09T11:08:07.216604+0000","last_became_active":"2023-10-09T11:04:53.273798+0000","last_became_peered":"2023-10-09T11:04:53.273798+0000","last_unstale":"2023-10-09T11:08:07.216604+0000","last_undegraded":"2023-10-09T11:08:07.216604+0000","last_fullsized":"2023-10-09T11:08:07.216604+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:04.220970+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":107,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T11:08:08.260447+0000","last_change":"2023-10-09T11:06:24.249877+0000","last_active":"2023-10-09T11:08:08.260447+0000","last_peered":"2023-10-09T11:08:08.260447+0000","last_clean":"2023-10-09T11:08:08.260447+0000","last_became_active":"2023-10-09T11:06:24.249573+0000","last_became_peered":"2023-10-09T11:06:24.249573+0000","last_unstale":"2023-10-09T11:08:08.260447+0000","last_undegraded":"2023-10-09T11:08:08.260447+0000","last_fullsized":"2023-10-09T11:08:08.260447+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:11.798192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":607,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T11:08:07.219787+0000","last_change":"2023-10-09T11:05:20.307347+0000","last_active":"2023-10-09T11:08:07.219787+0000","last_peered":"2023-10-09T11:08:07.219787+0000","last_clean":"2023-10-09T11:08:07.219787+0000","last_became_active":"2023-10-09T11:05:20.307126+0000","last_became_peered":"2023-10-09T11:05:20.307126+0000","last_unstale":"2023-10-09T11:08:07.219787+0000","last_undegraded":"2023-10-09T11:08:07.219787+0000","last_fullsized":"2023-10-09T11:08:07.219787+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:34.408515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":36,"stat_sum":{"num_bytes":728825880,"num_objects":11122,"num_object_clones":0,"num_object_copies":33366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5000,"num_objects_misplaced":1298,"num_objects_unfound":0,"num_objects_dirty":11122,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":187872,"num_write_kb":6367745,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24791,"num_bytes_recovered":1467023360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2859413504,"data_stored":2859401288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":233101,"ondisk_log_size":233101,"up":108,"acting":105,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":91,"num_bytes_recovered":15742,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12694,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":626148,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":31,"num_read_kb":56,"num_write":15,"num_write_kb":147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2632652,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332500,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1667004,"kb_used_data":767604,"kb_used_omap":0,"kb_used_meta":899392,"kb_avail":92078148,"statfs":{"total":95995035648,"available":94288023552,"internally_reserved":0,"allocated":786026496,"data_stored":784221396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920977408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":1.5009999999999999,"15min":1.6839999999999999},"min":{"1min":0.27300000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":5.2539999999999996,"5min":9.8040000000000003,"15min":9.8040000000000003},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.7989999999999999,"5min":1.7170000000000001,"15min":1.746},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.216},"max":{"1min":15.853,"5min":15.853,"15min":15.853},"last":4.7160000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:07:23 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":1.3959999999999999,"15min":1.52},"min":{"1min":0.23799999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.9130000000000003,"5min":11.619999999999999,"15min":11.619999999999999},"last":4.9770000000000003},{"interface":"front","average":{"1min":2.0019999999999998,"5min":1.8500000000000001,"15min":1.891},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":11.038,"5min":11.038,"15min":12.193},"last":4.782}]},{"osd":2,"last update":"Mon Oct 9 11:07:52 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":1.929,"15min":1.7170000000000001},"min":{"1min":0.313,"5min":0.309,"15min":0.245},"max":{"1min":3.3050000000000002,"5min":16.605,"15min":16.605},"last":5.1440000000000001},{"interface":"front","average":{"1min":1.466,"5min":1.4570000000000001,"15min":1.706},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":4.9569999999999999,"5min":9.7469999999999999,"15min":13.037000000000001},"last":4.9139999999999997}]},{"osd":3,"last update":"Mon Oct 9 11:07:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.524,"15min":1.4399999999999999},"min":{"1min":0.30499999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.4770000000000003,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":4.6689999999999996},{"interface":"front","average":{"1min":2.1789999999999998,"5min":1.9019999999999999,"15min":1.8280000000000001},"min":{"1min":0.38200000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.968,"5min":8.2330000000000005,"15min":8.2330000000000005},"last":2.3300000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":0.91800000000000004,"15min":1.0940000000000001},"min":{"1min":0.29799999999999999,"5min":0.218,"15min":0.218},"max":{"1min":7.4630000000000001,"5min":7.4630000000000001,"15min":8.2940000000000005},"last":4.7489999999999997},{"interface":"front","average":{"1min":1.034,"5min":1.256,"15min":1.244},"min":{"1min":0.30599999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.7190000000000003,"5min":11.808,"15min":16.481000000000002},"last":2.266}]},{"osd":5,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.0640000000000001,"15min":1.252},"min":{"1min":0.32800000000000001,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":5.4800000000000004,"5min":15.414999999999999,"15min":15.414999999999999},"last":4.9470000000000001},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.345,"15min":1.198},"min":{"1min":0.375,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.045,"5min":15.369,"15min":15.369},"last":4.8410000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.226,"15min":1.26},"min":{"1min":0.34599999999999997,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":3.8500000000000001,"5min":14.159000000000001,"15min":14.159000000000001},"last":4.8079999999999998},{"interface":"front","average":{"1min":1.121,"5min":1.472,"15min":1.355},"min":{"1min":0.36099999999999999,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":3.5830000000000002,"5min":13.993,"15min":16.538},"last":2.3029999999999999}]}]},{"osd":6,"up_from":44,"seq":188978561431,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1922928,"kb_used_data":426280,"kb_used_omap":0,"kb_used_meta":1496640,"kb_avail":91822224,"statfs":{"total":95995035648,"available":94025957376,"internally_reserved":0,"allocated":436510720,"data_stored":434717242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1532559360},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.6719999999999999,"15min":1.875},"min":{"1min":0.23899999999999999,"5min":0.217,"15min":0.217},"max":{"1min":3.7389999999999999,"5min":29.379999999999999,"15min":54.743000000000002},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.3120000000000001,"5min":1.6200000000000001,"15min":1.9299999999999999},"min":{"1min":0.247,"5min":0.214,"15min":0.214},"max":{"1min":7.1890000000000001,"5min":29.132999999999999,"15min":53.701000000000001},"last":0.29299999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:07:26 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":1.5369999999999999,"15min":1.1679999999999999},"min":{"1min":0.318,"5min":0.26000000000000001,"15min":0.23499999999999999},"max":{"1min":5.0110000000000001,"5min":16.225999999999999,"15min":41.241999999999997},"last":4.3710000000000004},{"interface":"front","average":{"1min":2.98,"5min":2.0489999999999999,"15min":1.7350000000000001},"min":{"1min":0.31,"5min":0.26200000000000001,"15min":0.248},"max":{"1min":9.173,"5min":11.715,"15min":41.476999999999997},"last":4.5350000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:07:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":2.1890000000000001,"15min":1.857},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.7089999999999996,"5min":10.481999999999999,"15min":12.003},"last":3.5350000000000001},{"interface":"front","average":{"1min":1.6990000000000001,"5min":1.764,"15min":1.8999999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.133,"5min":12.693,"15min":15.432},"last":3.6669999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:07:23 2023","interfaces":[{"interface":"back","average":{"1min":3.125,"5min":1.7629999999999999,"15min":1.554},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":28.948,"5min":28.948,"15min":28.948},"last":7.6840000000000002},{"interface":"front","average":{"1min":5.3739999999999997,"5min":2.9390000000000001,"15min":2.605},"min":{"1min":0.496,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":28.893000000000001,"5min":28.893000000000001,"15min":28.893000000000001},"last":3.4129999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":1.171,"15min":1.0129999999999999},"min":{"1min":0.26500000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.0189999999999999,"5min":17.216999999999999,"15min":17.216999999999999},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.78100000000000003,"15min":0.84299999999999997},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.24399999999999999},"max":{"1min":1.4059999999999999,"5min":10.025,"15min":10.025},"last":0.40600000000000003}]},{"osd":5,"last update":"Mon Oct 9 11:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.1419999999999999,"15min":1.2709999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":1.103,"5min":10.209,"15min":19.786999999999999},"last":0.753},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.90500000000000003,"15min":0.97099999999999997},"min":{"1min":0.33200000000000002,"5min":0.248,"15min":0.248},"max":{"1min":1.4810000000000001,"5min":9.9469999999999992,"15min":9.9469999999999992},"last":0.38300000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:08:14 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.081,"15min":1.268},"min":{"1min":0.32200000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.7250000000000001,"5min":10.246,"15min":12.882},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.83599999999999997,"15min":0.93300000000000005},"min":{"1min":0.35599999999999998,"5min":0.31,"15min":0.27500000000000002},"max":{"1min":3.2839999999999998,"5min":11.557,"15min":11.557},"last":0.35699999999999998}]}]},{"osd":1,"up_from":388,"seq":1666447310945,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":902068,"kb_used_data":418284,"kb_used_omap":13,"kb_used_meta":483506,"kb_avail":92843084,"statfs":{"total":95995035648,"available":95071318016,"internally_reserved":0,"allocated":428322816,"data_stored":425366674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":495110855},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":3.895,"5min":2.593,"15min":2.4369999999999998},"min":{"1min":0.26700000000000002,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":11.960000000000001,"5min":18.192,"15min":18.192},"last":2.7919999999999998},{"interface":"front","average":{"1min":3.9390000000000001,"5min":2.8260000000000001,"15min":1.9259999999999999},"min":{"1min":0.27000000000000002,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":10.099,"5min":17.957999999999998,"15min":17.957999999999998},"last":4.266}]},{"osd":2,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8439999999999999,"5min":2.5920000000000001,"15min":2.9300000000000002},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":9.5630000000000006,"5min":47.731000000000002,"15min":47.731000000000002},"last":6.7759999999999998},{"interface":"front","average":{"1min":3.8199999999999998,"5min":2.903,"15min":2.4769999999999999},"min":{"1min":0.29399999999999998,"5min":0.216,"15min":0.216},"max":{"1min":10.138,"5min":47.786999999999999,"15min":47.786999999999999},"last":4.3579999999999997}]},{"osd":3,"last update":"Mon Oct 9 11:07:25 2023","interfaces":[{"interface":"back","average":{"1min":3.7330000000000001,"5min":3.0870000000000002,"15min":2.0209999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":20.791,"5min":92.447999999999993,"15min":92.447999999999993},"last":2.0579999999999998},{"interface":"front","average":{"1min":3.1160000000000001,"5min":3.1179999999999999,"15min":2.028},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.217},"max":{"1min":7.3630000000000004,"5min":92.066999999999993,"15min":92.066999999999993},"last":6.8049999999999997}]},{"osd":4,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.7869999999999999,"15min":2.395},"min":{"1min":0.30499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.5199999999999996,"5min":11.701000000000001,"15min":11.701000000000001},"last":3.0659999999999998},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.04,"15min":1.6240000000000001},"min":{"1min":0.314,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":4.6079999999999997,"5min":11.589,"15min":11.589},"last":4.3049999999999997}]},{"osd":5,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":2.0830000000000002,"15min":2.3050000000000002},"min":{"1min":0.28999999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.0720000000000001,"5min":20.689,"15min":20.689},"last":3.948},{"interface":"front","average":{"1min":1.764,"5min":2.0590000000000002,"15min":2.2410000000000001},"min":{"1min":0.33800000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.4900000000000002,"5min":11.65,"15min":11.65},"last":6.7309999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.833,"5min":2.4049999999999998,"15min":2.411},"min":{"1min":0.312,"5min":0.309,"15min":0.309},"max":{"1min":6.0330000000000004,"5min":20.646999999999998,"15min":20.646999999999998},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.784,"5min":1.9219999999999999,"15min":2.1720000000000002},"min":{"1min":0.35599999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":5.5380000000000003,"5min":11.75,"15min":11.75},"last":3.1179999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.869,"5min":2.1640000000000001,"15min":1.615},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.32900000000000001},"max":{"1min":4.5780000000000003,"5min":11.555999999999999,"15min":11.555999999999999},"last":4.3339999999999996},{"interface":"front","average":{"1min":1.7789999999999999,"5min":2.0219999999999998,"15min":2.3210000000000002},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.1020000000000003,"5min":20.716000000000001,"15min":20.716000000000001},"last":0.55100000000000005}]}]},{"osd":0,"up_from":80,"seq":343597384003,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1318352,"kb_used_data":348168,"kb_used_omap":28,"kb_used_meta":969955,"kb_avail":92426800,"statfs":{"total":95995035648,"available":94645043200,"internally_reserved":0,"allocated":356524032,"data_stored":353472831,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":993234768},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:07:25 2023","interfaces":[{"interface":"back","average":{"1min":2.923,"5min":2.1680000000000001,"15min":2.831},"min":{"1min":0.22700000000000001,"5min":0.186,"15min":0.186},"max":{"1min":10.271000000000001,"5min":14.644,"15min":19.608000000000001},"last":4.4139999999999997},{"interface":"front","average":{"1min":3.879,"5min":2.2949999999999999,"15min":2.911},"min":{"1min":0.29499999999999998,"5min":0.214,"15min":0.214},"max":{"1min":15.834,"5min":15.834,"15min":19.992999999999999},"last":5.8609999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:07:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":2.339,"15min":2.29},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.17199999999999999},"max":{"1min":11.516,"5min":12.459,"15min":24.852},"last":0.20100000000000001},{"interface":"front","average":{"1min":1.73,"5min":2.246,"15min":2.694},"min":{"1min":0.215,"5min":0.20300000000000001,"15min":0.19800000000000001},"max":{"1min":8.4589999999999996,"5min":11.68,"15min":77.256},"last":4.1479999999999997}]},{"osd":3,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":3.2730000000000001,"15min":3.4449999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":13.189,"5min":25.050000000000001,"15min":25.050000000000001},"last":7.9050000000000002},{"interface":"front","average":{"1min":2.98,"5min":2.2770000000000001,"15min":2.0249999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":14.125999999999999,"5min":14.125999999999999,"15min":14.125999999999999},"last":3.7589999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.3799999999999999,"5min":2.1749999999999998,"15min":2.4470000000000001},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":13.106,"5min":14.507,"15min":77.177999999999997},"last":3.9089999999999998},{"interface":"front","average":{"1min":1.6919999999999999,"5min":1.5640000000000001,"15min":1.681},"min":{"1min":0.39500000000000002,"5min":0.30199999999999999,"15min":0.28899999999999998},"max":{"1min":7.9249999999999998,"5min":7.9249999999999998,"15min":10.513999999999999},"last":3.8210000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.7490000000000001,"15min":1.8400000000000001},"min":{"1min":0.39300000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.9550000000000001,"5min":7.9550000000000001,"15min":10.544},"last":3.7890000000000001},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.6499999999999999,"15min":1.8240000000000001},"min":{"1min":0.38100000000000001,"5min":0.308,"15min":0.29899999999999999},"max":{"1min":8.8439999999999994,"5min":11.212999999999999,"15min":24.754999999999999},"last":0.44700000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":1.736,"15min":1.9390000000000001},"min":{"1min":0.38,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":7.9829999999999997,"5min":7.9829999999999997,"15min":18.920000000000002},"last":3.8479999999999999},{"interface":"front","average":{"1min":2.3839999999999999,"5min":1.841,"15min":1.9590000000000001},"min":{"1min":0.42499999999999999,"5min":0.34999999999999998,"15min":0.31900000000000001},"max":{"1min":8.8810000000000002,"5min":11.257999999999999,"15min":24.800000000000001},"last":0.41899999999999998}]},{"osd":7,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":2.0790000000000002,"15min":2.0289999999999999},"min":{"1min":0.496,"5min":0.35099999999999998,"15min":0.32700000000000001},"max":{"1min":8.9139999999999997,"5min":17.672000000000001,"15min":17.672000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.4369999999999998,"5min":2.3839999999999999,"15min":2.552},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":13.154,"5min":17.539000000000001,"15min":77.228999999999999},"last":4.718}]}]},{"osd":2,"up_from":90,"seq":386547056944,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":992676,"kb_used_data":248796,"kb_used_omap":81,"kb_used_meta":743790,"kb_avail":92752476,"statfs":{"total":95995035648,"available":94978535424,"internally_reserved":0,"allocated":254767104,"data_stored":251884492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":761641178},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":2.2010000000000001,"15min":2.0609999999999999},"min":{"1min":0.186,"5min":0.182,"15min":0.17499999999999999},"max":{"1min":14.226000000000001,"5min":15.609,"15min":48.933999999999997},"last":1.8089999999999999},{"interface":"front","average":{"1min":2.5419999999999998,"5min":2.056,"15min":1.8919999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.19600000000000001},"max":{"1min":9.5540000000000003,"5min":9.8010000000000002,"15min":15.595000000000001},"last":0.45800000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:07:30 2023","interfaces":[{"interface":"back","average":{"1min":4.2800000000000002,"5min":2.96,"15min":2.629},"min":{"1min":0.20200000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":24.902000000000001,"5min":49.033999999999999,"15min":49.033999999999999},"last":1.9810000000000001},{"interface":"front","average":{"1min":4.7300000000000004,"5min":3.0550000000000002,"15min":2.7200000000000002},"min":{"1min":0.27500000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":25.850999999999999,"5min":48.874000000000002,"15min":48.874000000000002},"last":1.7869999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:07:17 2023","interfaces":[{"interface":"back","average":{"1min":4.1989999999999998,"5min":2.3330000000000002,"15min":1.954},"min":{"1min":0.751,"5min":0.214,"15min":0.214},"max":{"1min":15.701000000000001,"5min":15.701000000000001,"15min":15.701000000000001},"last":5.5730000000000004},{"interface":"front","average":{"1min":3.5350000000000001,"5min":2.4329999999999998,"15min":2.2410000000000001},"min":{"1min":0.313,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":15.73,"5min":15.73,"15min":15.73},"last":1.8879999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":1.796,"15min":1.8959999999999999},"min":{"1min":0.27100000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":11.391999999999999,"5min":11.391999999999999,"15min":15.672000000000001},"last":1.1120000000000001},{"interface":"front","average":{"1min":1.468,"5min":1.7909999999999999,"15min":1.873},"min":{"1min":0.34899999999999998,"5min":0.249,"15min":0.249},"max":{"1min":8.3350000000000009,"5min":15.571,"15min":48.795000000000002},"last":1.835}]},{"osd":5,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":1.597,"15min":1.5620000000000001},"min":{"1min":0.33700000000000002,"5min":0.29099999999999998,"15min":0.25900000000000001},"max":{"1min":7.4980000000000002,"5min":8.5310000000000006,"15min":12.128},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.8580000000000001,"5min":1.7869999999999999,"15min":2.0339999999999998},"min":{"1min":0.34899999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.441000000000001,"5min":11.441000000000001,"15min":49.061},"last":0.42099999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.909,"15min":1.804},"min":{"1min":0.41899999999999998,"5min":0.313,"15min":0.313},"max":{"1min":8.6039999999999992,"5min":15.536,"15min":48.845999999999997},"last":1.8500000000000001},{"interface":"front","average":{"1min":2.6499999999999999,"5min":2.012,"15min":1.657},"min":{"1min":0.42399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.157999999999999,"5min":10.157999999999999,"15min":10.157999999999999},"last":0.505}]},{"osd":7,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":2.081,"15min":1.9419999999999999},"min":{"1min":0.45100000000000001,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":8.3800000000000008,"5min":15.59,"15min":48.966000000000001},"last":1.9079999999999999},{"interface":"front","average":{"1min":2.492,"5min":1.917,"15min":1.762},"min":{"1min":0.46200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.5919999999999996,"5min":8.5549999999999997,"15min":14.94},"last":0.48199999999999998}]}]},{"osd":3,"up_from":469,"seq":2014339661873,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":718292,"kb_used_data":405324,"kb_used_omap":11,"kb_used_meta":312884,"kb_avail":93026860,"statfs":{"total":95995035648,"available":95259504640,"internally_reserved":0,"allocated":415051776,"data_stored":411880552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11309,"internal_metadata":320394195},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":3.3700000000000001,"5min":1.927,"15min":1.7889999999999999},"min":{"1min":0.27400000000000002,"5min":0.183,"15min":0.183},"max":{"1min":9.9809999999999999,"5min":14.882,"15min":14.882},"last":1.2450000000000001},{"interface":"front","average":{"1min":1.7050000000000001,"5min":1.0920000000000001,"15min":0.94199999999999995},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":5.6980000000000004,"5min":5.6980000000000004,"15min":5.6980000000000004},"last":1.395}]},{"osd":1,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":3.536,"5min":2.0859999999999999,"15min":1.6299999999999999},"min":{"1min":0.17100000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":11.757,"5min":17.93,"15min":17.93},"last":0.65500000000000003},{"interface":"front","average":{"1min":3.3050000000000002,"5min":1.9610000000000001,"15min":1.71},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":19.163,"5min":19.163,"15min":19.163},"last":1.1819999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":4.46,"5min":1.804,"15min":1.3260000000000001},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":38.840000000000003,"5min":38.840000000000003,"15min":38.840000000000003},"last":1.143},{"interface":"front","average":{"1min":4.7619999999999996,"5min":2.0979999999999999,"15min":1.419},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":37.991999999999997,"5min":37.991999999999997,"15min":37.991999999999997},"last":0.32900000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":2.274,"5min":1.641,"15min":1.595},"min":{"1min":0.30599999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":10.058999999999999,"5min":14.907999999999999,"15min":14.907999999999999},"last":1.2649999999999999},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.873,"15min":0.755},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.294,"5min":4.1390000000000002,"15min":4.1390000000000002},"last":0.63900000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":1.5,"15min":1.4419999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.8550000000000004,"5min":9.3300000000000001,"15min":9.3300000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.2650000000000001,"5min":1.5429999999999999,"15min":1.4750000000000001},"min":{"1min":0.32800000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.029,"5min":10.029,"15min":10.029},"last":1.2250000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.76,"15min":1.7749999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.5289999999999999,"5min":15.455,"15min":15.455},"last":0.71099999999999997},{"interface":"front","average":{"1min":2.3100000000000001,"5min":1.673,"15min":1.653},"min":{"1min":0.374,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.099,"5min":14.837,"15min":14.837},"last":1.2050000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":1.208,"15min":1.0700000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.907,"5min":6.907,"15min":6.907},"last":0.43099999999999999},{"interface":"front","average":{"1min":2.0590000000000002,"5min":1.5169999999999999,"15min":1.2769999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.9320000000000004,"5min":6.9119999999999999,"15min":6.9119999999999999},"last":0.54900000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019294,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1048188,"kb_used_data":163636,"kb_used_omap":0,"kb_used_meta":884544,"kb_avail":92696964,"statfs":{"total":95995035648,"available":94921691136,"internally_reserved":0,"allocated":167563264,"data_stored":165757897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":905773056},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:18 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.659,"15min":2.399},"min":{"1min":0.23200000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":3.6150000000000002,"5min":13.162000000000001,"15min":213.88999999999999},"last":6.1289999999999996},{"interface":"front","average":{"1min":1.569,"5min":2.161,"15min":2.6379999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":4.6699999999999999,"5min":11.128,"15min":214.09800000000001},"last":1.837}]},{"osd":1,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":2.54,"5min":2.508,"15min":2.6299999999999999},"min":{"1min":0.34599999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":8.1229999999999993,"5min":57.966999999999999,"15min":121.41},"last":7.7000000000000002},{"interface":"front","average":{"1min":2.117,"5min":2.3250000000000002,"15min":2.1200000000000001},"min":{"1min":0.26300000000000001,"5min":0.251,"15min":0.251},"max":{"1min":8.0960000000000001,"5min":57.832000000000001,"15min":122.21899999999999},"last":6.1980000000000004}]},{"osd":2,"last update":"Mon Oct 9 11:08:00 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":1.917,"15min":1.74},"min":{"1min":0.32100000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.2939999999999996,"5min":18.407,"15min":18.407},"last":1.923},{"interface":"front","average":{"1min":2.149,"5min":1.9810000000000001,"15min":2.0910000000000002},"min":{"1min":0.27900000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.1479999999999997,"5min":8.4250000000000007,"15min":27.818000000000001},"last":1.865}]},{"osd":3,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":1.837,"5min":2.4460000000000002,"15min":1.982},"min":{"1min":0.40899999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.0529999999999999,"5min":60.753,"15min":60.753},"last":6.1689999999999996},{"interface":"front","average":{"1min":2.6469999999999998,"5min":2.1389999999999998,"15min":1.492},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.0220000000000002,"5min":61.473999999999997,"15min":61.473999999999997},"last":1.7330000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:07:21 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":1.5049999999999999,"15min":1.4590000000000001},"min":{"1min":0.27400000000000002,"5min":0.224,"15min":0.224},"max":{"1min":3.4860000000000002,"5min":11.59,"15min":19.068999999999999},"last":1.79},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.3620000000000001,"15min":1.272},"min":{"1min":0.27300000000000002,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":3.8999999999999999,"5min":9.8610000000000007,"15min":15.535},"last":1.8899999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:08:00 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":1.353,"15min":1.1599999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":7.5860000000000003,"5min":8.3930000000000007,"15min":15.739000000000001},"last":6.2220000000000004},{"interface":"front","average":{"1min":1.4610000000000001,"5min":1.4890000000000001,"15min":1.4570000000000001},"min":{"1min":0.34899999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":9.5210000000000008,"5min":12.795999999999999,"15min":14.093999999999999},"last":5.1580000000000004}]},{"osd":7,"last update":"Mon Oct 9 11:08:00 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.5600000000000001,"15min":1.5529999999999999},"min":{"1min":0.308,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.4540000000000002,"5min":12.753,"15min":19.923999999999999},"last":5.181},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.4239999999999999,"15min":1.476},"min":{"1min":0.314,"5min":0.29799999999999999,"15min":0.28799999999999998},"max":{"1min":4.21,"5min":10.853,"15min":19.859999999999999},"last":3.5099999999999998}]}]},{"osd":5,"up_from":37,"seq":158913790363,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1357416,"kb_used_data":166240,"kb_used_omap":0,"kb_used_meta":1191104,"kb_avail":92387736,"statfs":{"total":95995035648,"available":94605041664,"internally_reserved":0,"allocated":170229760,"data_stored":168440814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1219690496},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:25 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":2.3809999999999998,"15min":2.3330000000000002},"min":{"1min":0.22900000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":6.2249999999999996,"5min":49.665999999999997,"15min":58.271999999999998},"last":3.1669999999999998},{"interface":"front","average":{"1min":1.605,"5min":2.427,"15min":2.3809999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.2649999999999997,"5min":49.701999999999998,"15min":58.25},"last":3.198}]},{"osd":1,"last update":"Mon Oct 9 11:07:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":2.101,"15min":2.032},"min":{"1min":0.26600000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.2199999999999998,"5min":32.491999999999997,"15min":32.491999999999997},"last":3.0209999999999999},{"interface":"front","average":{"1min":1.7709999999999999,"5min":2.3100000000000001,"15min":1.8680000000000001},"min":{"1min":0.39300000000000002,"5min":0.247,"15min":0.247},"max":{"1min":5.2130000000000001,"5min":32.537999999999997,"15min":32.537999999999997},"last":3.085}]},{"osd":2,"last update":"Mon Oct 9 11:07:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.454,"15min":1.786},"min":{"1min":0.307,"5min":0.254,"15min":0.221},"max":{"1min":5.1269999999999998,"5min":6.3929999999999998,"15min":11.257999999999999},"last":1.1419999999999999},{"interface":"front","average":{"1min":2.1869999999999998,"5min":2.0129999999999999,"15min":1.9370000000000001},"min":{"1min":0.29099999999999998,"5min":0.253,"15min":0.253},"max":{"1min":8.0359999999999996,"5min":11.673,"15min":14.621},"last":1.458}]},{"osd":3,"last update":"Mon Oct 9 11:07:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4449999999999998,"5min":1.6439999999999999,"15min":1.5409999999999999},"min":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":15.535,"5min":15.535,"15min":15.535},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.3460000000000001,"5min":1.8560000000000001,"15min":1.8240000000000001},"min":{"1min":0.38700000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":8.8940000000000001,"5min":8.8940000000000001,"15min":8.8940000000000001},"last":0.40200000000000002}]},{"osd":4,"last update":"Mon Oct 9 11:07:30 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":1.206,"15min":0.98899999999999999},"min":{"1min":0.28000000000000003,"5min":0.25700000000000001,"15min":0.216},"max":{"1min":2.0099999999999998,"5min":12.042,"15min":12.042},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.70699999999999996,"5min":1.2150000000000001,"15min":1.165},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.5910000000000002,"5min":9.2490000000000006,"15min":9.2490000000000006},"last":0.36499999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:07:42 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.3300000000000001,"15min":1.2110000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27100000000000002},"max":{"1min":1.5669999999999999,"5min":9.1969999999999992,"15min":9.1969999999999992},"last":2.1179999999999999},{"interface":"front","average":{"1min":0.65300000000000002,"5min":1.288,"15min":1.0629999999999999},"min":{"1min":0.33700000000000002,"5min":0.32800000000000001,"15min":0.29099999999999998},"max":{"1min":2.6059999999999999,"5min":11.983000000000001,"15min":11.983000000000001},"last":0.55700000000000005}]},{"osd":7,"last update":"Mon Oct 9 11:08:08 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.3160000000000001,"15min":1.302},"min":{"1min":0.35399999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.0469999999999997,"5min":8.9429999999999996,"15min":9.3539999999999992},"last":3.0569999999999999},{"interface":"front","average":{"1min":1.03,"5min":1.278,"15min":1.177},"min":{"1min":0.32800000000000001,"5min":0.30499999999999999,"15min":0.30099999999999999},"max":{"1min":4.2350000000000003,"5min":11.726000000000001,"15min":11.726000000000001},"last":0.45100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3733,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":343932928,"data_stored":343932928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":394133504,"data_stored":394133504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":229773312,"data_stored":229769240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":394657792,"data_stored":394657792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":151785472,"data_stored":151781400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":162922496,"data_stored":162922496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":404492288,"data_stored":404488216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":777715712,"data_stored":777715712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:08:23.107 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T11:08:23.633 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:23 smithi125 ceph-mon[195507]: pgmap v1377: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 410 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 4694/19707 objects degraded (23.819%); 776/19707 objects misplaced (3.938%); 0 B/s, 81 objects/s recovering 2023-10-09T11:08:23.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:23 smithi125 ceph-mon[200197]: pgmap v1377: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 410 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 4694/19707 objects degraded (23.819%); 776/19707 objects misplaced (3.938%); 0 B/s, 81 objects/s recovering 2023-10-09T11:08:23.694 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:23.695 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T11:08:23.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:23.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:23.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:23 smithi155 ceph-mon[184854]: pgmap v1377: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 131 active+clean; 410 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 4694/19707 objects degraded (23.819%); 776/19707 objects misplaced (3.938%); 0 B/s, 81 objects/s recovering 2023-10-09T11:08:23.800 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:23.834 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:23.874 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:23.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:23.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:23.978 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:24.011 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:24.034 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:24.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:24.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:24.140 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:24.215 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:24.241 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:24.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:24.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:24.283 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:24.326 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:24.346 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:24.381 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:24.403 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:24.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:24.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:24.495 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:24 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 4694/19707 objects degraded (23.819%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:24.495 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:24 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 4694/19707 objects degraded (23.819%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:24.508 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:24.544 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:24.569 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:24.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:24.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:24.677 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:24.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:24 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 4694/19707 objects degraded (23.819%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:24.736 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:24.772 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:24.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:24.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:24.879 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:24.918 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:24.940 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:24.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:24.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:25.046 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:25.088 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:25.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:25 smithi125 ceph-mon[195507]: pgmap v1378: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 132 active+clean; 214 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 2720/10317 objects degraded (26.364%); 286/10317 objects misplaced (2.772%); 1.5 MiB/s, 133 objects/s recovering 2023-10-09T11:08:25.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3153406263' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:08:25.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:25 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:08:25.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:25 smithi125 ceph-mon[200197]: pgmap v1378: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 132 active+clean; 214 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 2720/10317 objects degraded (26.364%); 286/10317 objects misplaced (2.772%); 1.5 MiB/s, 133 objects/s recovering 2023-10-09T11:08:25.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3153406263' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:08:25.452 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:08:25.673 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:25.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:25.680 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:25 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:08:25.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:25 smithi155 ceph-mon[184854]: pgmap v1378: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 132 active+clean; 214 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 2720/10317 objects degraded (26.364%); 286/10317 objects misplaced (2.772%); 1.5 MiB/s, 133 objects/s recovering 2023-10-09T11:08:25.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3153406263' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:08:25.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:25 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:08:25.779 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:25.849 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:26.087 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:08:26.420 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:08:26.420 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[195507]: failed to encode map e563 with expected crc 2023-10-09T11:08:26.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:08:26.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[195507]: osdmap e563: 8 total, 8 up, 8 in 2023-10-09T11:08:26.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[195507]: failed to encode map e563 with expected crc 2023-10-09T11:08:26.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[195507]: failed to encode map e563 with expected crc 2023-10-09T11:08:26.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[195507]: failed to encode map e563 with expected crc 2023-10-09T11:08:26.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[195507]: pgmap v1380: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 132 active+clean; 214 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 2720/10317 objects degraded (26.364%); 286/10317 objects misplaced (2.772%); 2.0 MiB/s, 75 objects/s recovering 2023-10-09T11:08:26.617 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Removed 99496 objects 2023-10-09T11:08:26.618 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Clean up completed and total clean up time :107.598 2023-10-09T11:08:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:26 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:08:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:26 smithi155 ceph-mon[184854]: failed to encode map e563 with expected crc 2023-10-09T11:08:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:26 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:08:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:26 smithi155 ceph-mon[184854]: osdmap e563: 8 total, 8 up, 8 in 2023-10-09T11:08:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:26 smithi155 ceph-mon[184854]: failed to encode map e563 with expected crc 2023-10-09T11:08:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:26 smithi155 ceph-mon[184854]: failed to encode map e563 with expected crc 2023-10-09T11:08:26.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:26 smithi155 ceph-mon[184854]: failed to encode map e563 with expected crc 2023-10-09T11:08:26.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:26 smithi155 ceph-mon[184854]: pgmap v1380: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 132 active+clean; 214 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 2720/10317 objects degraded (26.364%); 286/10317 objects misplaced (2.772%); 2.0 MiB/s, 75 objects/s recovering 2023-10-09T11:08:26.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:08:26.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[200197]: failed to encode map e563 with expected crc 2023-10-09T11:08:26.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:08:26.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[200197]: osdmap e563: 8 total, 8 up, 8 in 2023-10-09T11:08:26.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[200197]: failed to encode map e563 with expected crc 2023-10-09T11:08:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[200197]: failed to encode map e563 with expected crc 2023-10-09T11:08:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[200197]: failed to encode map e563 with expected crc 2023-10-09T11:08:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:26 smithi125 ceph-mon[200197]: pgmap v1380: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 132 active+clean; 214 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 2720/10317 objects degraded (26.364%); 286/10317 objects misplaced (2.772%); 2.0 MiB/s, 75 objects/s recovering 2023-10-09T11:08:27.293 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:27.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:27.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:27.400 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:27.430 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:27.451 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:27.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:27.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:27.554 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:27.587 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:27.606 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:27.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:27.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:27.711 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:27.743 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:27.762 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:27.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:27.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:27.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:27.897 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:27.897 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:27.916 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:27.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:27.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:28.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:28.056 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:28.083 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:28.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:28.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:28.109 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:08:28.109 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:08:28.109 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T11:08:28.109 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T11:08:28.109 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:08:28.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:28.222 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:28.293 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:28.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:28.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:28.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:28.428 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:28.664 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:08:28.665 INFO:teuthology.orchestra.run.smithi125.stdout:{"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 2720/6423 objects degraded (42.348%), 2 pgs degraded","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1694,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":563,"num_osds":8,"num_up_osds":8,"osd_up_since":1696849454,"num_in_osds":8,"osd_in_since":1696849519,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":132},{"state_name":"active+remapped+backfill_wait","count":5},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":139,"num_pools":5,"num_objects":2141,"data_bytes":139256688,"bytes_used":8428937216,"bytes_avail":759531347968,"bytes_total":767960285184,"degraded_objects":2720,"degraded_total":6423,"degraded_ratio":0.4234781254865328,"misplaced_objects":286,"misplaced_total":6423,"misplaced_ratio":0.044527479371010432,"recovering_objects_per_sec":60,"recovering_bytes_per_sec":1655882,"recovering_keys_per_sec":0,"num_objects_recovered":456,"num_bytes_recovered":12582912,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":784},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi125.satgiw","status":"up:active","gid":24575}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T11:05:13.785232+0000","services":{}},"progress_events":{"585851ac-6e70-4462-9176-ac50987fab77":{"message":"Upgrade to 18.2.0-598-gb4fa663f (3m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true},"6753946c-bfe8-4c23-ba05-7c088ea1f347":{"message":"Global Recovery Event (2m)\n [==========================..] (remaining: 8s)","progress":0.94964027404785156,"add_to_ceph_s":true}}} 2023-10-09T11:08:29.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:28 smithi125 ceph-mon[195507]: 11.16 scrub starts 2023-10-09T11:08:29.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:28 smithi125 ceph-mon[195507]: pgmap v1381: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 132 active+clean; 133 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 2720/6423 objects degraded (42.348%); 286/6423 objects misplaced (4.453%); 1.6 MiB/s, 60 objects/s recovering 2023-10-09T11:08:29.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:28 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 2720/6423 objects degraded (42.348%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:29.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:28 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/686838591' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T11:08:29.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:28 smithi125 ceph-mon[200197]: 11.16 scrub starts 2023-10-09T11:08:29.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:28 smithi125 ceph-mon[200197]: pgmap v1381: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 132 active+clean; 133 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 2720/6423 objects degraded (42.348%); 286/6423 objects misplaced (4.453%); 1.6 MiB/s, 60 objects/s recovering 2023-10-09T11:08:29.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:28 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 2720/6423 objects degraded (42.348%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:29.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:28 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/686838591' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T11:08:29.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:28 smithi155 ceph-mon[184854]: 11.16 scrub starts 2023-10-09T11:08:29.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:28 smithi155 ceph-mon[184854]: pgmap v1381: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 132 active+clean; 133 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 2720/6423 objects degraded (42.348%); 286/6423 objects misplaced (4.453%); 1.6 MiB/s, 60 objects/s recovering 2023-10-09T11:08:29.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:28 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 2720/6423 objects degraded (42.348%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:29.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:28 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/686838591' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T11:08:29.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:08:30.038 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:29 smithi125 ceph-mon[195507]: 11.6 scrub starts 2023-10-09T11:08:30.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:29 smithi125 ceph-mon[200197]: 11.6 scrub starts 2023-10-09T11:08:30.056 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T11:08:30.157 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:30.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:30.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:30.247 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-10-09T11:08:30.248 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-10-09T11:08:30.261 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:30.318 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:30.430 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T11:08:30.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:08:30.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:29 smithi155 ceph-mon[184854]: 11.6 scrub starts 2023-10-09T11:08:30.664 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:30.691 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:30.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:30.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:30.718 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:30.796 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:30.823 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:30.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:30.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:30.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:30.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:30.978 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:30.997 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:31.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:31.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:31.077 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:31.101 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:31.125 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:30 smithi155 ceph-mon[184854]: 11.7 scrub starts 2023-10-09T11:08:31.125 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:30 smithi155 ceph-mon[184854]: pgmap v1382: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 132 active+clean; 33 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 2720/1632 objects degraded (166.667%); 33/1632 objects misplaced (2.022%); 1.4 MiB/s, 133 objects/s recovering 2023-10-09T11:08:31.125 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:08:31.127 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:31.149 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:31.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:31.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:31.253 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:31.280 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:31.301 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:31.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:31.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:31.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:30 smithi125 ceph-mon[200197]: 11.7 scrub starts 2023-10-09T11:08:31.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:30 smithi125 ceph-mon[200197]: pgmap v1382: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 132 active+clean; 33 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 2720/1632 objects degraded (166.667%); 33/1632 objects misplaced (2.022%); 1.4 MiB/s, 133 objects/s recovering 2023-10-09T11:08:31.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:08:31.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:30 smithi125 ceph-mon[195507]: 11.7 scrub starts 2023-10-09T11:08:31.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:30 smithi125 ceph-mon[195507]: pgmap v1382: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 132 active+clean; 33 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 2720/1632 objects degraded (166.667%); 33/1632 objects misplaced (2.022%); 1.4 MiB/s, 133 objects/s recovering 2023-10-09T11:08:31.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:08:31.321 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:08:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:08:31] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.43.0" 2023-10-09T11:08:31.404 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:31.432 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:31.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:31.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:31.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:31.557 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:31.587 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:32.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:32 smithi125 ceph-mon[195507]: 11.17 scrub starts 2023-10-09T11:08:32.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:32 smithi125 ceph-mon[200197]: 11.17 scrub starts 2023-10-09T11:08:32.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:31 smithi155 ceph-mon[184854]: 11.17 scrub starts 2023-10-09T11:08:33.070 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:33.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:33.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:33.174 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:33.203 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:33.266 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:32 smithi125 ceph-mon[195507]: pgmap v1383: 139 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 130 active+clean; 32 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 2720/1578 objects degraded (172.370%); 33/1578 objects misplaced (2.091%); 1.2 MiB/s, 114 objects/s recovering 2023-10-09T11:08:33.266 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:32 smithi125 ceph-mon[195507]: failed to encode map e564 with expected crc 2023-10-09T11:08:33.266 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:32 smithi125 ceph-mon[195507]: failed to encode map e564 with expected crc 2023-10-09T11:08:33.266 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:32 smithi125 ceph-mon[195507]: osdmap e564: 8 total, 8 up, 8 in 2023-10-09T11:08:33.266 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:32 smithi125 ceph-mon[195507]: failed to encode map e564 with expected crc 2023-10-09T11:08:33.266 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:32 smithi125 ceph-mon[200197]: pgmap v1383: 139 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 130 active+clean; 32 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 2720/1578 objects degraded (172.370%); 33/1578 objects misplaced (2.091%); 1.2 MiB/s, 114 objects/s recovering 2023-10-09T11:08:33.267 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:32 smithi125 ceph-mon[200197]: failed to encode map e564 with expected crc 2023-10-09T11:08:33.267 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:32 smithi125 ceph-mon[200197]: failed to encode map e564 with expected crc 2023-10-09T11:08:33.267 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:32 smithi125 ceph-mon[200197]: osdmap e564: 8 total, 8 up, 8 in 2023-10-09T11:08:33.267 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:32 smithi125 ceph-mon[200197]: failed to encode map e564 with expected crc 2023-10-09T11:08:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:32 smithi155 ceph-mon[184854]: pgmap v1383: 139 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 130 active+clean; 32 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 2720/1578 objects degraded (172.370%); 33/1578 objects misplaced (2.091%); 1.2 MiB/s, 114 objects/s recovering 2023-10-09T11:08:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:32 smithi155 ceph-mon[184854]: failed to encode map e564 with expected crc 2023-10-09T11:08:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:32 smithi155 ceph-mon[184854]: failed to encode map e564 with expected crc 2023-10-09T11:08:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:32 smithi155 ceph-mon[184854]: osdmap e564: 8 total, 8 up, 8 in 2023-10-09T11:08:33.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:32 smithi155 ceph-mon[184854]: failed to encode map e564 with expected crc 2023-10-09T11:08:33.473 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:33.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:33.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:33.586 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:33.616 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:33.644 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:33.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:33.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:33.650 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:33.750 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:33.777 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:33.797 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:33.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:33.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:33.901 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:33.931 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:34.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:34 smithi155 ceph-mon[184854]: failed to encode map e565 with expected crc 2023-10-09T11:08:34.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:34 smithi155 ceph-mon[184854]: failed to encode map e565 with expected crc 2023-10-09T11:08:34.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:34 smithi155 ceph-mon[184854]: osdmap e565: 8 total, 8 up, 8 in 2023-10-09T11:08:34.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:34 smithi155 ceph-mon[184854]: failed to encode map e565 with expected crc 2023-10-09T11:08:34.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:34 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 2720/1578 objects degraded (172.370%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:34.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:34 smithi155 ceph-mon[184854]: failed to encode map e565 with expected crc 2023-10-09T11:08:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:34 smithi125 ceph-mon[195507]: failed to encode map e565 with expected crc 2023-10-09T11:08:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:34 smithi125 ceph-mon[195507]: failed to encode map e565 with expected crc 2023-10-09T11:08:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:34 smithi125 ceph-mon[195507]: osdmap e565: 8 total, 8 up, 8 in 2023-10-09T11:08:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:34 smithi125 ceph-mon[195507]: failed to encode map e565 with expected crc 2023-10-09T11:08:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:34 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 2720/1578 objects degraded (172.370%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:34.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:34 smithi125 ceph-mon[195507]: failed to encode map e565 with expected crc 2023-10-09T11:08:34.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:34 smithi125 ceph-mon[200197]: failed to encode map e565 with expected crc 2023-10-09T11:08:34.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:34 smithi125 ceph-mon[200197]: failed to encode map e565 with expected crc 2023-10-09T11:08:34.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:34 smithi125 ceph-mon[200197]: osdmap e565: 8 total, 8 up, 8 in 2023-10-09T11:08:34.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:34 smithi125 ceph-mon[200197]: failed to encode map e565 with expected crc 2023-10-09T11:08:34.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:34 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 2720/1578 objects degraded (172.370%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T11:08:34.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:34 smithi125 ceph-mon[200197]: failed to encode map e565 with expected crc 2023-10-09T11:08:34.695 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:08:34.696 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:08:34.696 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:08:34.696 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:08:34.696 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:08:34.696 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:08:34.696 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:08:34.696 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:08:34.696 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:08:34.697 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:08:34.697 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:08:34.697 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:08:34.697 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:08:34.697 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:08:34.697 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:08:34.697 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:08:35.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:35 smithi125 ceph-mon[195507]: pgmap v1386: 139 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 131 active+clean; 32 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 745/1578 objects degraded (47.212%); 33/1578 objects misplaced (2.091%); 0 B/s, 316 objects/s recovering 2023-10-09T11:08:35.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:35 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/982321110' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:35.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:35 smithi125 ceph-mon[200197]: pgmap v1386: 139 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 131 active+clean; 32 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 745/1578 objects degraded (47.212%); 33/1578 objects misplaced (2.091%); 0 B/s, 316 objects/s recovering 2023-10-09T11:08:35.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:35 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/982321110' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:35.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:35 smithi155 ceph-mon[184854]: pgmap v1386: 139 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 131 active+clean; 32 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 745/1578 objects degraded (47.212%); 33/1578 objects misplaced (2.091%); 0 B/s, 316 objects/s recovering 2023-10-09T11:08:35.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:35 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/982321110' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:35.912 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:35.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:35.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:36.016 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:36.047 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:36.269 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:36.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:36 smithi125 ceph-mon[200197]: 1.11 scrub starts 2023-10-09T11:08:36.272 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:36 smithi125 ceph-mon[195507]: 1.11 scrub starts 2023-10-09T11:08:36.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:36.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:36.376 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:36.409 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:36.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:36 smithi155 ceph-mon[184854]: 1.11 scrub starts 2023-10-09T11:08:36.484 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:08:36.484 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:08:36.966 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T11:08:37.024 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:08:37.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:37 smithi125 ceph-mon[195507]: pgmap v1387: 139 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 131 active+clean; 32 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 247 op/s; 745/1578 objects degraded (47.212%); 33/1578 objects misplaced (2.091%); 0 B/s, 316 objects/s recovering 2023-10-09T11:08:37.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:37 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4002811864' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:37.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:37 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:37.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:37 smithi125 ceph-mon[200197]: pgmap v1387: 139 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 131 active+clean; 32 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 247 op/s; 745/1578 objects degraded (47.212%); 33/1578 objects misplaced (2.091%); 0 B/s, 316 objects/s recovering 2023-10-09T11:08:37.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:37 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4002811864' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:37.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:37 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:37.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:37.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:37.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:37.191 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:37.218 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:37.238 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:37.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:37.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:37.341 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:37.369 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:37.441 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:37.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:37.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:37.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:37 smithi155 ceph-mon[184854]: pgmap v1387: 139 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 131 active+clean; 32 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 247 op/s; 745/1578 objects degraded (47.212%); 33/1578 objects misplaced (2.091%); 0 B/s, 316 objects/s recovering 2023-10-09T11:08:37.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:37 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4002811864' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:37.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:37 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:37.544 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:37.571 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:37.591 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:37.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:37.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:37.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:37.699 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:37.726 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:37.996 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:37.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:37.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:38.072 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_5' removed 2023-10-09T11:08:38.100 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:38.130 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:38.155 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:38.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:38.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:38.259 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:38.284 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:38 smithi155 ceph-mon[184854]: 11.7 scrub starts 2023-10-09T11:08:38.284 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:38 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:08:38.284 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:38 smithi155 ceph-mon[184854]: failed to encode map e566 with expected crc 2023-10-09T11:08:38.284 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:38 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:08:38.285 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:38 smithi155 ceph-mon[184854]: osdmap e566: 8 total, 8 up, 8 in 2023-10-09T11:08:38.285 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:38 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3760681080' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:08:38.285 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:38 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:08:38.285 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:38 smithi155 ceph-mon[184854]: failed to encode map e566 with expected crc 2023-10-09T11:08:38.286 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:38.304 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:38.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:38.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[195507]: 11.7 scrub starts 2023-10-09T11:08:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:08:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[195507]: failed to encode map e566 with expected crc 2023-10-09T11:08:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:08:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[195507]: osdmap e566: 8 total, 8 up, 8 in 2023-10-09T11:08:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3760681080' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:08:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:08:38.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[195507]: failed to encode map e566 with expected crc 2023-10-09T11:08:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[200197]: 11.7 scrub starts 2023-10-09T11:08:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:08:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[200197]: failed to encode map e566 with expected crc 2023-10-09T11:08:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:08:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[200197]: osdmap e566: 8 total, 8 up, 8 in 2023-10-09T11:08:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3760681080' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:08:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T11:08:38.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:38 smithi125 ceph-mon[200197]: failed to encode map e566 with expected crc 2023-10-09T11:08:38.400 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:08:38.402 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:08:38.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:38.435 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:38.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:38.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:38.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:38.557 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:38.560 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T11:08:38.560 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-09T11:08:38.560 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T11:08:38.561 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T11:08:38.561 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T11:08:38.561 DEBUG:teuthology.orchestra.run.smithi125:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T11:08:38.584 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:38.602 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:38.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:38.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:38.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T11:08:38.607 INFO:teuthology.orchestra.run.smithi125.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T11:08:38.607 DEBUG:teuthology.orchestra.run.smithi125:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T11:08:38.682 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T11:08:38.682 DEBUG:teuthology.orchestra.run.smithi125:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T11:08:38.700 INFO:tasks.workunit:timeout=3h 2023-10-09T11:08:38.700 INFO:tasks.workunit:cleanup=True 2023-10-09T11:08:38.700 DEBUG:teuthology.orchestra.run.smithi125:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T11:08:38.706 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:38.745 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:08:38.770 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:38.784 INFO:tasks.workunit.client.0.smithi125.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T11:08:38.944 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:38.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:38.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:39.056 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:39.083 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:39.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:39.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:39.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:39.205 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:39.221 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1390,"stamp":"2023-10-09T11:08:38.039254+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":631652,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":31,"num_read_kb":56,"num_write":76,"num_write_kb":176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":101,"num_bytes_recovered":2648394,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5046,"ondisk_log_size":5046,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":428,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7678128,"kb_used_data":619056,"kb_used_omap":134,"kb_used_meta":7058873,"kb_avail":742283088,"statfs":{"total":767960285184,"available":760097882112,"internally_reserved":0,"allocated":633913344,"data_stored":614446053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":137532,"internal_metadata":7228286660},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-32882700,"num_objects":-501,"num_object_clones":0,"num_object_copies":-1503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2720,"num_objects_misplaced":-33,"num_objects_unfound":0,"num_objects_dirty":-501,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-198994,"num_write_kb":-6367745,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-26553,"num_bytes_recovered":-1479606272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.039411"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":852,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004394+0000","last_change":"2023-10-09T11:07:36.287455+0000","last_active":"2023-10-09T11:08:32.004394+0000","last_peered":"2023-10-09T11:08:32.004394+0000","last_clean":"2023-10-09T11:08:32.004394+0000","last_became_active":"2023-10-09T11:07:36.287052+0000","last_became_peered":"2023-10-09T11:07:36.287052+0000","last_unstale":"2023-10-09T11:08:32.004394+0000","last_undegraded":"2023-10-09T11:08:32.004394+0000","last_fullsized":"2023-10-09T11:08:32.004394+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:18.864873+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":347,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.381010+0000","last_change":"2023-10-09T11:03:21.185383+0000","last_active":"2023-10-09T11:08:32.381010+0000","last_peered":"2023-10-09T11:08:32.381010+0000","last_clean":"2023-10-09T11:08:32.381010+0000","last_became_active":"2023-10-09T11:03:21.185022+0000","last_became_peered":"2023-10-09T11:03:21.185022+0000","last_unstale":"2023-10-09T11:08:32.381010+0000","last_undegraded":"2023-10-09T11:08:32.381010+0000","last_fullsized":"2023-10-09T11:08:32.381010+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:05.230434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":91,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.002479+0000","last_change":"2023-10-09T11:04:50.319477+0000","last_active":"2023-10-09T11:08:32.002479+0000","last_peered":"2023-10-09T11:08:32.002479+0000","last_clean":"2023-10-09T11:08:32.002479+0000","last_became_active":"2023-10-09T11:04:50.319148+0000","last_became_peered":"2023-10-09T11:04:50.319148+0000","last_unstale":"2023-10-09T11:08:32.002479+0000","last_undegraded":"2023-10-09T11:08:32.002479+0000","last_fullsized":"2023-10-09T11:08:32.002479+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:48.725566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":722,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005077+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:08:32.005077+0000","last_peered":"2023-10-09T11:08:32.005077+0000","last_clean":"2023-10-09T11:08:32.005077+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:08:32.005077+0000","last_undegraded":"2023-10-09T11:08:32.005077+0000","last_fullsized":"2023-10-09T11:08:32.005077+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":279,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004609+0000","last_change":"2023-10-09T11:04:50.333711+0000","last_active":"2023-10-09T11:08:32.004609+0000","last_peered":"2023-10-09T11:08:32.004609+0000","last_clean":"2023-10-09T11:08:32.004609+0000","last_became_active":"2023-10-09T11:04:50.333262+0000","last_became_peered":"2023-10-09T11:04:50.333262+0000","last_unstale":"2023-10-09T11:08:32.004609+0000","last_undegraded":"2023-10-09T11:08:32.004609+0000","last_fullsized":"2023-10-09T11:08:32.004609+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.843765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":25,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.010364+0000","last_change":"2023-10-09T11:07:33.237903+0000","last_active":"2023-10-09T11:08:32.010364+0000","last_peered":"2023-10-09T11:08:32.010364+0000","last_clean":"2023-10-09T11:08:32.010364+0000","last_became_active":"2023-10-09T11:07:33.237618+0000","last_became_peered":"2023-10-09T11:07:33.237618+0000","last_unstale":"2023-10-09T11:08:32.010364+0000","last_undegraded":"2023-10-09T11:08:32.010364+0000","last_fullsized":"2023-10-09T11:08:32.010364+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:25.600818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":740,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.009835+0000","last_change":"2023-10-09T11:07:36.284534+0000","last_active":"2023-10-09T11:08:32.009835+0000","last_peered":"2023-10-09T11:08:32.009835+0000","last_clean":"2023-10-09T11:08:32.009835+0000","last_became_active":"2023-10-09T11:07:36.284254+0000","last_became_peered":"2023-10-09T11:07:36.284254+0000","last_unstale":"2023-10-09T11:08:32.009835+0000","last_undegraded":"2023-10-09T11:08:32.009835+0000","last_fullsized":"2023-10-09T11:08:32.009835+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:19.159630+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.001140+0000","last_change":"2023-10-09T11:04:50.310477+0000","last_active":"2023-10-09T11:08:32.001140+0000","last_peered":"2023-10-09T11:08:32.001140+0000","last_clean":"2023-10-09T11:08:32.001140+0000","last_became_active":"2023-10-09T11:04:50.310067+0000","last_became_peered":"2023-10-09T11:04:50.310067+0000","last_unstale":"2023-10-09T11:08:32.001140+0000","last_undegraded":"2023-10-09T11:08:32.001140+0000","last_fullsized":"2023-10-09T11:08:32.001140+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:52.884716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":374,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005543+0000","last_change":"2023-10-09T11:05:20.310581+0000","last_active":"2023-10-09T11:08:32.005543+0000","last_peered":"2023-10-09T11:08:32.005543+0000","last_clean":"2023-10-09T11:08:32.005543+0000","last_became_active":"2023-10-09T11:05:20.309830+0000","last_became_peered":"2023-10-09T11:05:20.309830+0000","last_unstale":"2023-10-09T11:08:32.005543+0000","last_undegraded":"2023-10-09T11:08:32.005543+0000","last_fullsized":"2023-10-09T11:08:32.005543+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:24.508211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":780,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.010497+0000","last_change":"2023-10-09T11:07:36.282502+0000","last_active":"2023-10-09T11:08:32.010497+0000","last_peered":"2023-10-09T11:08:32.010497+0000","last_clean":"2023-10-09T11:08:32.010497+0000","last_became_active":"2023-10-09T11:07:36.282133+0000","last_became_peered":"2023-10-09T11:07:36.282133+0000","last_unstale":"2023-10-09T11:08:32.010497+0000","last_undegraded":"2023-10-09T11:08:32.010497+0000","last_fullsized":"2023-10-09T11:08:32.010497+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:03.762383+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":201,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005583+0000","last_change":"2023-10-09T11:06:24.245488+0000","last_active":"2023-10-09T11:08:32.005583+0000","last_peered":"2023-10-09T11:08:32.005583+0000","last_clean":"2023-10-09T11:08:32.005583+0000","last_became_active":"2023-10-09T11:06:24.245167+0000","last_became_peered":"2023-10-09T11:06:24.245167+0000","last_unstale":"2023-10-09T11:08:32.005583+0000","last_undegraded":"2023-10-09T11:08:32.005583+0000","last_fullsized":"2023-10-09T11:08:32.005583+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:31.702568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":100,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.001277+0000","last_change":"2023-10-09T11:05:20.299453+0000","last_active":"2023-10-09T11:08:32.001277+0000","last_peered":"2023-10-09T11:08:32.001277+0000","last_clean":"2023-10-09T11:08:32.001277+0000","last_became_active":"2023-10-09T11:05:20.299117+0000","last_became_peered":"2023-10-09T11:05:20.299117+0000","last_unstale":"2023-10-09T11:08:32.001277+0000","last_undegraded":"2023-10-09T11:08:32.001277+0000","last_fullsized":"2023-10-09T11:08:32.001277+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:30.072813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":895,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.381428+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:08:32.381428+0000","last_peered":"2023-10-09T11:08:32.381428+0000","last_clean":"2023-10-09T11:08:32.381428+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:08:32.381428+0000","last_undegraded":"2023-10-09T11:08:32.381428+0000","last_fullsized":"2023-10-09T11:08:32.381428+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":436,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004832+0000","last_change":"2023-10-09T11:07:33.242665+0000","last_active":"2023-10-09T11:08:32.004832+0000","last_peered":"2023-10-09T11:08:32.004832+0000","last_clean":"2023-10-09T11:08:32.004832+0000","last_became_active":"2023-10-09T11:07:33.242313+0000","last_became_peered":"2023-10-09T11:07:33.242313+0000","last_unstale":"2023-10-09T11:08:32.004832+0000","last_undegraded":"2023-10-09T11:08:32.004832+0000","last_fullsized":"2023-10-09T11:08:32.004832+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:20.565135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":118,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.001724+0000","last_change":"2023-10-09T11:07:33.225683+0000","last_active":"2023-10-09T11:08:32.001724+0000","last_peered":"2023-10-09T11:08:32.001724+0000","last_clean":"2023-10-09T11:08:32.001724+0000","last_became_active":"2023-10-09T11:07:33.225275+0000","last_became_peered":"2023-10-09T11:07:33.225275+0000","last_unstale":"2023-10-09T11:08:32.001724+0000","last_undegraded":"2023-10-09T11:08:32.001724+0000","last_fullsized":"2023-10-09T11:08:32.001724+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:10.131060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":794,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005251+0000","last_change":"2023-10-09T11:05:23.328981+0000","last_active":"2023-10-09T11:08:32.005251+0000","last_peered":"2023-10-09T11:08:32.005251+0000","last_clean":"2023-10-09T11:08:32.005251+0000","last_became_active":"2023-10-09T11:05:23.328616+0000","last_became_peered":"2023-10-09T11:05:23.328616+0000","last_unstale":"2023-10-09T11:08:32.005251+0000","last_undegraded":"2023-10-09T11:08:32.005251+0000","last_fullsized":"2023-10-09T11:08:32.005251+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:15.315819+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":31,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.010439+0000","last_change":"2023-10-09T11:07:33.265842+0000","last_active":"2023-10-09T11:08:32.010439+0000","last_peered":"2023-10-09T11:08:32.010439+0000","last_clean":"2023-10-09T11:08:32.010439+0000","last_became_active":"2023-10-09T11:07:33.232311+0000","last_became_peered":"2023-10-09T11:07:33.232311+0000","last_unstale":"2023-10-09T11:08:32.010439+0000","last_undegraded":"2023-10-09T11:08:32.010439+0000","last_fullsized":"2023-10-09T11:08:32.010439+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:23.402582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":343,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005944+0000","last_change":"2023-10-09T11:05:20.316265+0000","last_active":"2023-10-09T11:08:32.005944+0000","last_peered":"2023-10-09T11:08:32.005944+0000","last_clean":"2023-10-09T11:08:32.005944+0000","last_became_active":"2023-10-09T11:05:20.315853+0000","last_became_peered":"2023-10-09T11:05:20.315853+0000","last_unstale":"2023-10-09T11:08:32.005944+0000","last_undegraded":"2023-10-09T11:08:32.005944+0000","last_fullsized":"2023-10-09T11:08:32.005944+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:35.093981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":957,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005349+0000","last_change":"2023-10-09T11:04:53.274921+0000","last_active":"2023-10-09T11:08:32.005349+0000","last_peered":"2023-10-09T11:08:32.005349+0000","last_clean":"2023-10-09T11:08:32.005349+0000","last_became_active":"2023-10-09T11:04:53.274518+0000","last_became_peered":"2023-10-09T11:04:53.274518+0000","last_unstale":"2023-10-09T11:08:32.005349+0000","last_undegraded":"2023-10-09T11:08:32.005349+0000","last_fullsized":"2023-10-09T11:08:32.005349+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:14.315301+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":35,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.010315+0000","last_change":"2023-10-09T11:07:33.262559+0000","last_active":"2023-10-09T11:08:32.010315+0000","last_peered":"2023-10-09T11:08:32.010315+0000","last_clean":"2023-10-09T11:08:32.010315+0000","last_became_active":"2023-10-09T11:07:33.231695+0000","last_became_peered":"2023-10-09T11:07:33.231695+0000","last_unstale":"2023-10-09T11:08:32.010315+0000","last_undegraded":"2023-10-09T11:08:32.010315+0000","last_fullsized":"2023-10-09T11:08:32.010315+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:59.006940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":586,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.381159+0000","last_change":"2023-10-09T11:05:20.311424+0000","last_active":"2023-10-09T11:08:32.381159+0000","last_peered":"2023-10-09T11:08:32.381159+0000","last_clean":"2023-10-09T11:08:32.381159+0000","last_became_active":"2023-10-09T11:05:20.311097+0000","last_became_peered":"2023-10-09T11:05:20.311097+0000","last_unstale":"2023-10-09T11:08:32.381159+0000","last_undegraded":"2023-10-09T11:08:32.381159+0000","last_fullsized":"2023-10-09T11:08:32.381159+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:19.274582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":597,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T11:08:34.774701+0000","last_change":"2023-10-09T11:08:34.774701+0000","last_active":"2023-10-09T11:08:34.774701+0000","last_peered":"2023-10-09T11:08:34.774701+0000","last_clean":"2023-10-09T11:08:34.774701+0000","last_became_active":"2023-10-09T11:06:24.245076+0000","last_became_peered":"2023-10-09T11:06:24.245076+0000","last_unstale":"2023-10-09T11:08:34.774701+0000","last_undegraded":"2023-10-09T11:08:34.774701+0000","last_fullsized":"2023-10-09T11:08:34.774701+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T10:49:18.708281+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.708281+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":31,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.009943+0000","last_change":"2023-10-09T11:07:33.269200+0000","last_active":"2023-10-09T11:08:32.009943+0000","last_peered":"2023-10-09T11:08:32.009943+0000","last_clean":"2023-10-09T11:08:32.009943+0000","last_became_active":"2023-10-09T11:07:33.236205+0000","last_became_peered":"2023-10-09T11:07:33.236205+0000","last_unstale":"2023-10-09T11:08:32.009943+0000","last_undegraded":"2023-10-09T11:08:32.009943+0000","last_fullsized":"2023-10-09T11:08:32.009943+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:24.574734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":109,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.002319+0000","last_change":"2023-10-09T11:07:33.227803+0000","last_active":"2023-10-09T11:08:32.002319+0000","last_peered":"2023-10-09T11:08:32.002319+0000","last_clean":"2023-10-09T11:08:32.002319+0000","last_became_active":"2023-10-09T11:07:33.227486+0000","last_became_peered":"2023-10-09T11:07:33.227486+0000","last_unstale":"2023-10-09T11:08:32.002319+0000","last_undegraded":"2023-10-09T11:08:32.002319+0000","last_fullsized":"2023-10-09T11:08:32.002319+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:39.274875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":876,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004781+0000","last_change":"2023-10-09T11:05:23.323856+0000","last_active":"2023-10-09T11:08:32.004781+0000","last_peered":"2023-10-09T11:08:32.004781+0000","last_clean":"2023-10-09T11:08:32.004781+0000","last_became_active":"2023-10-09T11:05:23.322763+0000","last_became_peered":"2023-10-09T11:05:23.322763+0000","last_unstale":"2023-10-09T11:08:32.004781+0000","last_undegraded":"2023-10-09T11:08:32.004781+0000","last_fullsized":"2023-10-09T11:08:32.004781+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.760820+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":25,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.009840+0000","last_change":"2023-10-09T11:07:33.233510+0000","last_active":"2023-10-09T11:08:32.009840+0000","last_peered":"2023-10-09T11:08:32.009840+0000","last_clean":"2023-10-09T11:08:32.009840+0000","last_became_active":"2023-10-09T11:07:33.232339+0000","last_became_peered":"2023-10-09T11:07:33.232339+0000","last_unstale":"2023-10-09T11:08:32.009840+0000","last_undegraded":"2023-10-09T11:08:32.009840+0000","last_fullsized":"2023-10-09T11:08:32.009840+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:55.324912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":506,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005825+0000","last_change":"2023-10-09T11:07:33.226594+0000","last_active":"2023-10-09T11:08:32.005825+0000","last_peered":"2023-10-09T11:08:32.005825+0000","last_clean":"2023-10-09T11:08:32.005825+0000","last_became_active":"2023-10-09T11:07:33.226275+0000","last_became_peered":"2023-10-09T11:07:33.226275+0000","last_unstale":"2023-10-09T11:08:32.005825+0000","last_undegraded":"2023-10-09T11:08:32.005825+0000","last_fullsized":"2023-10-09T11:08:32.005825+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:41.180088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":898,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005815+0000","last_change":"2023-10-09T11:04:53.274236+0000","last_active":"2023-10-09T11:08:32.005815+0000","last_peered":"2023-10-09T11:08:32.005815+0000","last_clean":"2023-10-09T11:08:32.005815+0000","last_became_active":"2023-10-09T11:04:53.273817+0000","last_became_peered":"2023-10-09T11:04:53.273817+0000","last_unstale":"2023-10-09T11:08:32.005815+0000","last_undegraded":"2023-10-09T11:08:32.005815+0000","last_fullsized":"2023-10-09T11:08:32.005815+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:05.096905+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":160,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.009773+0000","last_change":"2023-10-09T11:05:20.312129+0000","last_active":"2023-10-09T11:08:32.009773+0000","last_peered":"2023-10-09T11:08:32.009773+0000","last_clean":"2023-10-09T11:08:32.009773+0000","last_became_active":"2023-10-09T11:05:20.311810+0000","last_became_peered":"2023-10-09T11:05:20.311810+0000","last_unstale":"2023-10-09T11:08:32.009773+0000","last_undegraded":"2023-10-09T11:08:32.009773+0000","last_fullsized":"2023-10-09T11:08:32.009773+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:48.813391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":258,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.002656+0000","last_change":"2023-10-09T11:07:33.218244+0000","last_active":"2023-10-09T11:08:32.002656+0000","last_peered":"2023-10-09T11:08:32.002656+0000","last_clean":"2023-10-09T11:08:32.002656+0000","last_became_active":"2023-10-09T11:07:33.217918+0000","last_became_peered":"2023-10-09T11:07:33.217918+0000","last_unstale":"2023-10-09T11:08:32.002656+0000","last_undegraded":"2023-10-09T11:08:32.002656+0000","last_fullsized":"2023-10-09T11:08:32.002656+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:16.150105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":873,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.006181+0000","last_change":"2023-10-09T11:05:23.324537+0000","last_active":"2023-10-09T11:08:32.006181+0000","last_peered":"2023-10-09T11:08:32.006181+0000","last_clean":"2023-10-09T11:08:32.006181+0000","last_became_active":"2023-10-09T11:05:23.324187+0000","last_became_peered":"2023-10-09T11:05:23.324187+0000","last_unstale":"2023-10-09T11:08:32.006181+0000","last_undegraded":"2023-10-09T11:08:32.006181+0000","last_fullsized":"2023-10-09T11:08:32.006181+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:25.286106+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":85,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004874+0000","last_change":"2023-10-09T11:05:20.323565+0000","last_active":"2023-10-09T11:08:32.004874+0000","last_peered":"2023-10-09T11:08:32.004874+0000","last_clean":"2023-10-09T11:08:32.004874+0000","last_became_active":"2023-10-09T11:05:20.309672+0000","last_became_peered":"2023-10-09T11:05:20.309672+0000","last_unstale":"2023-10-09T11:08:32.004874+0000","last_undegraded":"2023-10-09T11:08:32.004874+0000","last_fullsized":"2023-10-09T11:08:32.004874+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:50.211465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":109,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.002628+0000","last_change":"2023-10-09T11:07:33.228001+0000","last_active":"2023-10-09T11:08:32.002628+0000","last_peered":"2023-10-09T11:08:32.002628+0000","last_clean":"2023-10-09T11:08:32.002628+0000","last_became_active":"2023-10-09T11:07:33.227691+0000","last_became_peered":"2023-10-09T11:07:33.227691+0000","last_unstale":"2023-10-09T11:08:32.002628+0000","last_undegraded":"2023-10-09T11:08:32.002628+0000","last_fullsized":"2023-10-09T11:08:32.002628+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:24:16.098138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":797,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.010555+0000","last_change":"2023-10-09T11:07:36.284927+0000","last_active":"2023-10-09T11:08:32.010555+0000","last_peered":"2023-10-09T11:08:32.010555+0000","last_clean":"2023-10-09T11:08:32.010555+0000","last_became_active":"2023-10-09T11:07:36.284631+0000","last_became_peered":"2023-10-09T11:07:36.284631+0000","last_unstale":"2023-10-09T11:08:32.010555+0000","last_undegraded":"2023-10-09T11:08:32.010555+0000","last_fullsized":"2023-10-09T11:08:32.010555+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:37.220243+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":29,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.010569+0000","last_change":"2023-10-09T11:07:33.273503+0000","last_active":"2023-10-09T11:08:32.010569+0000","last_peered":"2023-10-09T11:08:32.010569+0000","last_clean":"2023-10-09T11:08:32.010569+0000","last_became_active":"2023-10-09T11:07:33.242980+0000","last_became_peered":"2023-10-09T11:07:33.242980+0000","last_unstale":"2023-10-09T11:08:32.010569+0000","last_undegraded":"2023-10-09T11:08:32.010569+0000","last_fullsized":"2023-10-09T11:08:32.010569+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:18.934972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":527,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004485+0000","last_change":"2023-10-09T11:07:33.304363+0000","last_active":"2023-10-09T11:08:32.004485+0000","last_peered":"2023-10-09T11:08:32.004485+0000","last_clean":"2023-10-09T11:08:32.004485+0000","last_became_active":"2023-10-09T11:07:33.304024+0000","last_became_peered":"2023-10-09T11:07:33.304024+0000","last_unstale":"2023-10-09T11:08:32.004485+0000","last_undegraded":"2023-10-09T11:08:32.004485+0000","last_fullsized":"2023-10-09T11:08:32.004485+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:21:51.522688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":946,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.002379+0000","last_change":"2023-10-09T11:05:23.316129+0000","last_active":"2023-10-09T11:08:32.002379+0000","last_peered":"2023-10-09T11:08:32.002379+0000","last_clean":"2023-10-09T11:08:32.002379+0000","last_became_active":"2023-10-09T11:05:23.315792+0000","last_became_peered":"2023-10-09T11:05:23.315792+0000","last_unstale":"2023-10-09T11:08:32.002379+0000","last_undegraded":"2023-10-09T11:08:32.002379+0000","last_fullsized":"2023-10-09T11:08:32.002379+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:51.770547+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":29,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.010472+0000","last_change":"2023-10-09T11:07:33.256691+0000","last_active":"2023-10-09T11:08:32.010472+0000","last_peered":"2023-10-09T11:08:32.010472+0000","last_clean":"2023-10-09T11:08:32.010472+0000","last_became_active":"2023-10-09T11:07:33.229554+0000","last_became_peered":"2023-10-09T11:07:33.229554+0000","last_unstale":"2023-10-09T11:08:32.010472+0000","last_undegraded":"2023-10-09T11:08:32.010472+0000","last_fullsized":"2023-10-09T11:08:32.010472+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:57.414795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":279,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004507+0000","last_change":"2023-10-09T11:04:50.333538+0000","last_active":"2023-10-09T11:08:32.004507+0000","last_peered":"2023-10-09T11:08:32.004507+0000","last_clean":"2023-10-09T11:08:32.004507+0000","last_became_active":"2023-10-09T11:04:50.332514+0000","last_became_peered":"2023-10-09T11:04:50.332514+0000","last_unstale":"2023-10-09T11:08:32.004507+0000","last_undegraded":"2023-10-09T11:08:32.004507+0000","last_fullsized":"2023-10-09T11:08:32.004507+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:22.933839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":821,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005330+0000","last_change":"2023-10-09T11:05:23.322251+0000","last_active":"2023-10-09T11:08:32.005330+0000","last_peered":"2023-10-09T11:08:32.005330+0000","last_clean":"2023-10-09T11:08:32.005330+0000","last_became_active":"2023-10-09T11:05:23.321826+0000","last_became_peered":"2023-10-09T11:05:23.321826+0000","last_unstale":"2023-10-09T11:08:32.005330+0000","last_undegraded":"2023-10-09T11:08:32.005330+0000","last_fullsized":"2023-10-09T11:08:32.005330+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.858013+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":29,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.010690+0000","last_change":"2023-10-09T11:07:33.271238+0000","last_active":"2023-10-09T11:08:32.010690+0000","last_peered":"2023-10-09T11:08:32.010690+0000","last_clean":"2023-10-09T11:08:32.010690+0000","last_became_active":"2023-10-09T11:07:33.241886+0000","last_became_peered":"2023-10-09T11:07:33.241886+0000","last_unstale":"2023-10-09T11:08:32.010690+0000","last_undegraded":"2023-10-09T11:08:32.010690+0000","last_fullsized":"2023-10-09T11:08:32.010690+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:48.882589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":95,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004939+0000","last_change":"2023-10-09T11:07:33.244131+0000","last_active":"2023-10-09T11:08:32.004939+0000","last_peered":"2023-10-09T11:08:32.004939+0000","last_clean":"2023-10-09T11:08:32.004939+0000","last_became_active":"2023-10-09T11:07:33.243797+0000","last_became_peered":"2023-10-09T11:07:33.243797+0000","last_unstale":"2023-10-09T11:08:32.004939+0000","last_undegraded":"2023-10-09T11:08:32.004939+0000","last_fullsized":"2023-10-09T11:08:32.004939+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:44.851043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":1075,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.010621+0000","last_change":"2023-10-09T11:07:36.286417+0000","last_active":"2023-10-09T11:08:32.010621+0000","last_peered":"2023-10-09T11:08:32.010621+0000","last_clean":"2023-10-09T11:08:32.010621+0000","last_became_active":"2023-10-09T11:07:36.286061+0000","last_became_peered":"2023-10-09T11:07:36.286061+0000","last_unstale":"2023-10-09T11:08:32.010621+0000","last_undegraded":"2023-10-09T11:08:32.010621+0000","last_fullsized":"2023-10-09T11:08:32.010621+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:00.289500+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":423,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004991+0000","last_change":"2023-10-09T11:05:20.309385+0000","last_active":"2023-10-09T11:08:32.004991+0000","last_peered":"2023-10-09T11:08:32.004991+0000","last_clean":"2023-10-09T11:08:32.004991+0000","last_became_active":"2023-10-09T11:05:20.309034+0000","last_became_peered":"2023-10-09T11:05:20.309034+0000","last_unstale":"2023-10-09T11:08:32.004991+0000","last_undegraded":"2023-10-09T11:08:32.004991+0000","last_fullsized":"2023-10-09T11:08:32.004991+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:31.857485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":354,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005739+0000","last_change":"2023-10-09T11:06:24.245966+0000","last_active":"2023-10-09T11:08:32.005739+0000","last_peered":"2023-10-09T11:08:32.005739+0000","last_clean":"2023-10-09T11:08:32.005739+0000","last_became_active":"2023-10-09T11:06:24.245310+0000","last_became_peered":"2023-10-09T11:06:24.245310+0000","last_unstale":"2023-10-09T11:08:32.005739+0000","last_undegraded":"2023-10-09T11:08:32.005739+0000","last_fullsized":"2023-10-09T11:08:32.005739+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:23.966200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":719,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.009701+0000","last_change":"2023-10-09T11:07:36.283174+0000","last_active":"2023-10-09T11:08:32.009701+0000","last_peered":"2023-10-09T11:08:32.009701+0000","last_clean":"2023-10-09T11:08:32.009701+0000","last_became_active":"2023-10-09T11:07:36.278007+0000","last_became_peered":"2023-10-09T11:07:36.278007+0000","last_unstale":"2023-10-09T11:08:32.009701+0000","last_undegraded":"2023-10-09T11:08:32.009701+0000","last_fullsized":"2023-10-09T11:08:32.009701+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:51.693969+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":95,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.002158+0000","last_change":"2023-10-09T11:04:50.529513+0000","last_active":"2023-10-09T11:08:32.002158+0000","last_peered":"2023-10-09T11:08:32.002158+0000","last_clean":"2023-10-09T11:08:32.002158+0000","last_became_active":"2023-10-09T11:04:50.339671+0000","last_became_peered":"2023-10-09T11:04:50.339671+0000","last_unstale":"2023-10-09T11:08:32.002158+0000","last_undegraded":"2023-10-09T11:08:32.002158+0000","last_fullsized":"2023-10-09T11:08:32.002158+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:24.473000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":100,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.002248+0000","last_change":"2023-10-09T11:05:20.313855+0000","last_active":"2023-10-09T11:08:32.002248+0000","last_peered":"2023-10-09T11:08:32.002248+0000","last_clean":"2023-10-09T11:08:32.002248+0000","last_became_active":"2023-10-09T11:05:20.313535+0000","last_became_peered":"2023-10-09T11:05:20.313535+0000","last_unstale":"2023-10-09T11:08:32.002248+0000","last_undegraded":"2023-10-09T11:08:32.002248+0000","last_fullsized":"2023-10-09T11:08:32.002248+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:25.266434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":823,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004695+0000","last_change":"2023-10-09T11:04:53.272323+0000","last_active":"2023-10-09T11:08:32.004695+0000","last_peered":"2023-10-09T11:08:32.004695+0000","last_clean":"2023-10-09T11:08:32.004695+0000","last_became_active":"2023-10-09T11:04:53.272010+0000","last_became_peered":"2023-10-09T11:04:53.272010+0000","last_unstale":"2023-10-09T11:08:32.004695+0000","last_undegraded":"2023-10-09T11:08:32.004695+0000","last_fullsized":"2023-10-09T11:08:32.004695+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:23.913605+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":588,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004615+0000","last_change":"2023-10-09T11:05:20.307499+0000","last_active":"2023-10-09T11:08:32.004615+0000","last_peered":"2023-10-09T11:08:32.004615+0000","last_clean":"2023-10-09T11:08:32.004615+0000","last_became_active":"2023-10-09T11:05:20.307198+0000","last_became_peered":"2023-10-09T11:05:20.307198+0000","last_unstale":"2023-10-09T11:08:32.004615+0000","last_undegraded":"2023-10-09T11:08:32.004615+0000","last_fullsized":"2023-10-09T11:08:32.004615+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:53.726844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":324,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.382591+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:08:32.382591+0000","last_peered":"2023-10-09T11:08:32.382591+0000","last_clean":"2023-10-09T11:08:32.382591+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:08:32.382591+0000","last_undegraded":"2023-10-09T11:08:32.382591+0000","last_fullsized":"2023-10-09T11:08:32.382591+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":915,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005897+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:08:32.005897+0000","last_peered":"2023-10-09T11:08:32.005897+0000","last_clean":"2023-10-09T11:08:32.005897+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:08:32.005897+0000","last_undegraded":"2023-10-09T11:08:32.005897+0000","last_fullsized":"2023-10-09T11:08:32.005897+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":587,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.380963+0000","last_change":"2023-10-09T11:04:57.953307+0000","last_active":"2023-10-09T11:08:32.380963+0000","last_peered":"2023-10-09T11:08:32.380963+0000","last_clean":"2023-10-09T11:08:32.380963+0000","last_became_active":"2023-10-09T11:04:57.953095+0000","last_became_peered":"2023-10-09T11:04:57.953095+0000","last_unstale":"2023-10-09T11:08:32.380963+0000","last_undegraded":"2023-10-09T11:08:32.380963+0000","last_fullsized":"2023-10-09T11:08:32.380963+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:05.562643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":273,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005228+0000","last_change":"2023-10-09T11:05:20.310255+0000","last_active":"2023-10-09T11:08:32.005228+0000","last_peered":"2023-10-09T11:08:32.005228+0000","last_clean":"2023-10-09T11:08:32.005228+0000","last_became_active":"2023-10-09T11:05:20.309109+0000","last_became_peered":"2023-10-09T11:05:20.309109+0000","last_unstale":"2023-10-09T11:08:32.005228+0000","last_undegraded":"2023-10-09T11:08:32.005228+0000","last_fullsized":"2023-10-09T11:08:32.005228+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:07.429039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":807,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.006139+0000","last_change":"2023-10-09T11:05:23.329329+0000","last_active":"2023-10-09T11:08:32.006139+0000","last_peered":"2023-10-09T11:08:32.006139+0000","last_clean":"2023-10-09T11:08:32.006139+0000","last_became_active":"2023-10-09T11:05:23.328976+0000","last_became_peered":"2023-10-09T11:05:23.328976+0000","last_unstale":"2023-10-09T11:08:32.006139+0000","last_undegraded":"2023-10-09T11:08:32.006139+0000","last_fullsized":"2023-10-09T11:08:32.006139+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:41.848184+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":109,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.001425+0000","last_change":"2023-10-09T11:07:33.236133+0000","last_active":"2023-10-09T11:08:32.001425+0000","last_peered":"2023-10-09T11:08:32.001425+0000","last_clean":"2023-10-09T11:08:32.001425+0000","last_became_active":"2023-10-09T11:07:33.235818+0000","last_became_peered":"2023-10-09T11:07:33.235818+0000","last_unstale":"2023-10-09T11:08:32.001425+0000","last_undegraded":"2023-10-09T11:08:32.001425+0000","last_fullsized":"2023-10-09T11:08:32.001425+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:20.627846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":270,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.008977+0000","last_change":"2023-10-09T11:04:50.332978+0000","last_active":"2023-10-09T11:08:32.008977+0000","last_peered":"2023-10-09T11:08:32.008977+0000","last_clean":"2023-10-09T11:08:32.008977+0000","last_became_active":"2023-10-09T11:04:50.332735+0000","last_became_peered":"2023-10-09T11:04:50.332735+0000","last_unstale":"2023-10-09T11:08:32.008977+0000","last_undegraded":"2023-10-09T11:08:32.008977+0000","last_fullsized":"2023-10-09T11:08:32.008977+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:39:44.659751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":1091,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.002677+0000","last_change":"2023-10-09T11:07:36.291716+0000","last_active":"2023-10-09T11:08:32.002677+0000","last_peered":"2023-10-09T11:08:32.002677+0000","last_clean":"2023-10-09T11:08:32.002677+0000","last_became_active":"2023-10-09T11:07:36.291424+0000","last_became_peered":"2023-10-09T11:07:36.291424+0000","last_unstale":"2023-10-09T11:08:32.002677+0000","last_undegraded":"2023-10-09T11:08:32.002677+0000","last_fullsized":"2023-10-09T11:08:32.002677+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:22.316387+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":295,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T11:08:33.010744+0000","last_change":"2023-10-09T11:07:33.226547+0000","last_active":"2023-10-09T11:08:33.010744+0000","last_peered":"2023-10-09T11:08:33.010744+0000","last_clean":"2023-10-09T11:08:33.010744+0000","last_became_active":"2023-10-09T11:07:33.226176+0000","last_became_peered":"2023-10-09T11:07:33.226176+0000","last_unstale":"2023-10-09T11:08:33.010744+0000","last_undegraded":"2023-10-09T11:08:33.010744+0000","last_fullsized":"2023-10-09T11:08:33.010744+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:16.888732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":440,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005757+0000","last_change":"2023-10-09T11:04:50.341353+0000","last_active":"2023-10-09T11:08:32.005757+0000","last_peered":"2023-10-09T11:08:32.005757+0000","last_clean":"2023-10-09T11:08:32.005757+0000","last_became_active":"2023-10-09T11:04:50.340801+0000","last_became_peered":"2023-10-09T11:04:50.340801+0000","last_unstale":"2023-10-09T11:08:32.005757+0000","last_undegraded":"2023-10-09T11:08:32.005757+0000","last_fullsized":"2023-10-09T11:08:32.005757+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:00.699069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":247,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T11:08:33.010973+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:08:33.010973+0000","last_peered":"2023-10-09T11:08:33.010973+0000","last_clean":"2023-10-09T11:08:33.010973+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:08:33.010973+0000","last_undegraded":"2023-10-09T11:08:33.010973+0000","last_fullsized":"2023-10-09T11:08:33.010973+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":765,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004432+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:08:32.004432+0000","last_peered":"2023-10-09T11:08:32.004432+0000","last_clean":"2023-10-09T11:08:32.004432+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:08:32.004432+0000","last_undegraded":"2023-10-09T11:08:32.004432+0000","last_fullsized":"2023-10-09T11:08:32.004432+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":447,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.006013+0000","last_change":"2023-10-09T11:05:20.316093+0000","last_active":"2023-10-09T11:08:32.006013+0000","last_peered":"2023-10-09T11:08:32.006013+0000","last_clean":"2023-10-09T11:08:32.006013+0000","last_became_active":"2023-10-09T11:05:20.315471+0000","last_became_peered":"2023-10-09T11:05:20.315471+0000","last_unstale":"2023-10-09T11:08:32.006013+0000","last_undegraded":"2023-10-09T11:08:32.006013+0000","last_fullsized":"2023-10-09T11:08:32.006013+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:47.548763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":471,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005249+0000","last_change":"2023-10-09T11:06:24.245282+0000","last_active":"2023-10-09T11:08:32.005249+0000","last_peered":"2023-10-09T11:08:32.005249+0000","last_clean":"2023-10-09T11:08:32.005249+0000","last_became_active":"2023-10-09T11:06:24.244981+0000","last_became_peered":"2023-10-09T11:06:24.244981+0000","last_unstale":"2023-10-09T11:08:32.005249+0000","last_undegraded":"2023-10-09T11:08:32.005249+0000","last_fullsized":"2023-10-09T11:08:32.005249+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:15.327655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":280,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T11:08:33.010552+0000","last_change":"2023-10-09T11:04:50.311579+0000","last_active":"2023-10-09T11:08:33.010552+0000","last_peered":"2023-10-09T11:08:33.010552+0000","last_clean":"2023-10-09T11:08:33.010552+0000","last_became_active":"2023-10-09T11:04:50.311270+0000","last_became_peered":"2023-10-09T11:04:50.311270+0000","last_unstale":"2023-10-09T11:08:33.010552+0000","last_undegraded":"2023-10-09T11:08:33.010552+0000","last_fullsized":"2023-10-09T11:08:33.010552+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:49.850302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":917,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.009921+0000","last_change":"2023-10-09T11:07:36.284280+0000","last_active":"2023-10-09T11:08:32.009921+0000","last_peered":"2023-10-09T11:08:32.009921+0000","last_clean":"2023-10-09T11:08:32.009921+0000","last_became_active":"2023-10-09T11:07:36.283966+0000","last_became_peered":"2023-10-09T11:07:36.283966+0000","last_unstale":"2023-10-09T11:08:32.009921+0000","last_undegraded":"2023-10-09T11:08:32.009921+0000","last_fullsized":"2023-10-09T11:08:32.009921+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:54.128331+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":343,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.381073+0000","last_change":"2023-10-09T11:04:50.381744+0000","last_active":"2023-10-09T11:08:32.381073+0000","last_peered":"2023-10-09T11:08:32.381073+0000","last_clean":"2023-10-09T11:08:32.381073+0000","last_became_active":"2023-10-09T11:04:50.381253+0000","last_became_peered":"2023-10-09T11:04:50.381253+0000","last_unstale":"2023-10-09T11:08:32.381073+0000","last_undegraded":"2023-10-09T11:08:32.381073+0000","last_fullsized":"2023-10-09T11:08:32.381073+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:24.030274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":265,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.007593+0000","last_change":"2023-10-09T11:05:20.298542+0000","last_active":"2023-10-09T11:08:32.007593+0000","last_peered":"2023-10-09T11:08:32.007593+0000","last_clean":"2023-10-09T11:08:32.007593+0000","last_became_active":"2023-10-09T11:05:20.298119+0000","last_became_peered":"2023-10-09T11:05:20.298119+0000","last_unstale":"2023-10-09T11:08:32.007593+0000","last_undegraded":"2023-10-09T11:08:32.007593+0000","last_fullsized":"2023-10-09T11:08:32.007593+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:00.827214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":244,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.009172+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:08:32.009172+0000","last_peered":"2023-10-09T11:08:32.009172+0000","last_clean":"2023-10-09T11:08:32.009172+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:08:32.009172+0000","last_undegraded":"2023-10-09T11:08:32.009172+0000","last_fullsized":"2023-10-09T11:08:32.009172+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":440,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004820+0000","last_change":"2023-10-09T11:07:33.227052+0000","last_active":"2023-10-09T11:08:32.004820+0000","last_peered":"2023-10-09T11:08:32.004820+0000","last_clean":"2023-10-09T11:08:32.004820+0000","last_became_active":"2023-10-09T11:07:33.226289+0000","last_became_peered":"2023-10-09T11:07:33.226289+0000","last_unstale":"2023-10-09T11:08:32.004820+0000","last_undegraded":"2023-10-09T11:08:32.004820+0000","last_fullsized":"2023-10-09T11:08:32.004820+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:15.605946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":862,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.009919+0000","last_change":"2023-10-09T11:07:36.290700+0000","last_active":"2023-10-09T11:08:32.009919+0000","last_peered":"2023-10-09T11:08:32.009919+0000","last_clean":"2023-10-09T11:08:32.009919+0000","last_became_active":"2023-10-09T11:07:36.290357+0000","last_became_peered":"2023-10-09T11:07:36.290357+0000","last_unstale":"2023-10-09T11:08:32.009919+0000","last_undegraded":"2023-10-09T11:08:32.009919+0000","last_fullsized":"2023-10-09T11:08:32.009919+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:34.494777+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":100,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.001915+0000","last_change":"2023-10-09T11:05:20.310666+0000","last_active":"2023-10-09T11:08:32.001915+0000","last_peered":"2023-10-09T11:08:32.001915+0000","last_clean":"2023-10-09T11:08:32.001915+0000","last_became_active":"2023-10-09T11:05:20.310413+0000","last_became_peered":"2023-10-09T11:05:20.310413+0000","last_unstale":"2023-10-09T11:08:32.001915+0000","last_undegraded":"2023-10-09T11:08:32.001915+0000","last_fullsized":"2023-10-09T11:08:32.001915+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:34.169905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":525,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.010641+0000","last_change":"2023-10-09T11:05:20.316523+0000","last_active":"2023-10-09T11:08:32.010641+0000","last_peered":"2023-10-09T11:08:32.010641+0000","last_clean":"2023-10-09T11:08:32.010641+0000","last_became_active":"2023-10-09T11:05:20.314493+0000","last_became_peered":"2023-10-09T11:05:20.314493+0000","last_unstale":"2023-10-09T11:08:32.010641+0000","last_undegraded":"2023-10-09T11:08:32.010641+0000","last_fullsized":"2023-10-09T11:08:32.010641+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:41.618701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":936,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T11:08:33.010298+0000","last_change":"2023-10-09T11:05:23.322074+0000","last_active":"2023-10-09T11:08:33.010298+0000","last_peered":"2023-10-09T11:08:33.010298+0000","last_clean":"2023-10-09T11:08:33.010298+0000","last_became_active":"2023-10-09T11:05:23.321749+0000","last_became_peered":"2023-10-09T11:05:23.321749+0000","last_unstale":"2023-10-09T11:08:33.010298+0000","last_undegraded":"2023-10-09T11:08:33.010298+0000","last_fullsized":"2023-10-09T11:08:33.010298+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:07.010932+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":140,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.010364+0000","last_change":"2023-10-09T11:02:33.198484+0000","last_active":"2023-10-09T11:08:32.010364+0000","last_peered":"2023-10-09T11:08:32.010364+0000","last_clean":"2023-10-09T11:08:32.010364+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:08:32.010364+0000","last_undegraded":"2023-10-09T11:08:32.010364+0000","last_fullsized":"2023-10-09T11:08:32.010364+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":192,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T11:08:33.007136+0000","last_change":"2023-10-09T11:06:24.248362+0000","last_active":"2023-10-09T11:08:33.007136+0000","last_peered":"2023-10-09T11:08:33.007136+0000","last_clean":"2023-10-09T11:08:33.007136+0000","last_became_active":"2023-10-09T11:06:24.247348+0000","last_became_peered":"2023-10-09T11:06:24.247348+0000","last_unstale":"2023-10-09T11:08:33.007136+0000","last_undegraded":"2023-10-09T11:08:33.007136+0000","last_fullsized":"2023-10-09T11:08:33.007136+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:26.728754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":390,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004836+0000","last_change":"2023-10-09T11:06:24.253511+0000","last_active":"2023-10-09T11:08:32.004836+0000","last_peered":"2023-10-09T11:08:32.004836+0000","last_clean":"2023-10-09T11:08:32.004836+0000","last_became_active":"2023-10-09T11:06:24.252739+0000","last_became_peered":"2023-10-09T11:06:24.252739+0000","last_unstale":"2023-10-09T11:08:32.004836+0000","last_undegraded":"2023-10-09T11:08:32.004836+0000","last_fullsized":"2023-10-09T11:08:32.004836+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:50.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":795,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.002515+0000","last_change":"2023-10-09T11:06:24.239676+0000","last_active":"2023-10-09T11:08:32.002515+0000","last_peered":"2023-10-09T11:08:32.002515+0000","last_clean":"2023-10-09T11:08:32.002515+0000","last_became_active":"2023-10-09T11:06:24.239375+0000","last_became_peered":"2023-10-09T11:06:24.239375+0000","last_unstale":"2023-10-09T11:08:32.002515+0000","last_undegraded":"2023-10-09T11:08:32.002515+0000","last_fullsized":"2023-10-09T11:08:32.002515+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:27.953053+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":25,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.010365+0000","last_change":"2023-10-09T11:07:33.229415+0000","last_active":"2023-10-09T11:08:32.010365+0000","last_peered":"2023-10-09T11:08:32.010365+0000","last_clean":"2023-10-09T11:08:32.010365+0000","last_became_active":"2023-10-09T11:07:33.229061+0000","last_became_peered":"2023-10-09T11:07:33.229061+0000","last_unstale":"2023-10-09T11:08:32.010365+0000","last_undegraded":"2023-10-09T11:08:32.010365+0000","last_fullsized":"2023-10-09T11:08:32.010365+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:30.978293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":247,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T11:08:33.010886+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:08:33.010886+0000","last_peered":"2023-10-09T11:08:33.010886+0000","last_clean":"2023-10-09T11:08:33.010886+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:08:33.010886+0000","last_undegraded":"2023-10-09T11:08:33.010886+0000","last_fullsized":"2023-10-09T11:08:33.010886+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":524,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004746+0000","last_change":"2023-10-09T11:07:33.227271+0000","last_active":"2023-10-09T11:08:32.004746+0000","last_peered":"2023-10-09T11:08:32.004746+0000","last_clean":"2023-10-09T11:08:32.004746+0000","last_became_active":"2023-10-09T11:07:33.226760+0000","last_became_peered":"2023-10-09T11:07:33.226760+0000","last_unstale":"2023-10-09T11:08:32.004746+0000","last_undegraded":"2023-10-09T11:08:32.004746+0000","last_fullsized":"2023-10-09T11:08:32.004746+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:12.268123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":764,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.382386+0000","last_change":"2023-10-09T11:05:23.323368+0000","last_active":"2023-10-09T11:08:32.382386+0000","last_peered":"2023-10-09T11:08:32.382386+0000","last_clean":"2023-10-09T11:08:32.382386+0000","last_became_active":"2023-10-09T11:05:23.323032+0000","last_became_peered":"2023-10-09T11:05:23.323032+0000","last_unstale":"2023-10-09T11:08:32.382386+0000","last_undegraded":"2023-10-09T11:08:32.382386+0000","last_fullsized":"2023-10-09T11:08:32.382386+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:09.393584+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":121,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.002641+0000","last_change":"2023-10-09T11:07:33.246378+0000","last_active":"2023-10-09T11:08:32.002641+0000","last_peered":"2023-10-09T11:08:32.002641+0000","last_clean":"2023-10-09T11:08:32.002641+0000","last_became_active":"2023-10-09T11:07:33.228019+0000","last_became_peered":"2023-10-09T11:07:33.228019+0000","last_unstale":"2023-10-09T11:08:32.002641+0000","last_undegraded":"2023-10-09T11:08:32.002641+0000","last_fullsized":"2023-10-09T11:08:32.002641+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:18.511517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":273,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005312+0000","last_change":"2023-10-09T11:05:20.310417+0000","last_active":"2023-10-09T11:08:32.005312+0000","last_peered":"2023-10-09T11:08:32.005312+0000","last_clean":"2023-10-09T11:08:32.005312+0000","last_became_active":"2023-10-09T11:05:20.309827+0000","last_became_peered":"2023-10-09T11:05:20.309827+0000","last_unstale":"2023-10-09T11:08:32.005312+0000","last_undegraded":"2023-10-09T11:08:32.005312+0000","last_fullsized":"2023-10-09T11:08:32.005312+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:23.934383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":109,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.001335+0000","last_change":"2023-10-09T11:07:33.217595+0000","last_active":"2023-10-09T11:08:32.001335+0000","last_peered":"2023-10-09T11:08:32.001335+0000","last_clean":"2023-10-09T11:08:32.001335+0000","last_became_active":"2023-10-09T11:07:33.217155+0000","last_became_peered":"2023-10-09T11:07:33.217155+0000","last_unstale":"2023-10-09T11:08:32.001335+0000","last_undegraded":"2023-10-09T11:08:32.001335+0000","last_fullsized":"2023-10-09T11:08:32.001335+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:55.127329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":728,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.002164+0000","last_change":"2023-10-09T11:07:36.284491+0000","last_active":"2023-10-09T11:08:32.002164+0000","last_peered":"2023-10-09T11:08:32.002164+0000","last_clean":"2023-10-09T11:08:32.002164+0000","last_became_active":"2023-10-09T11:07:36.284249+0000","last_became_peered":"2023-10-09T11:07:36.284249+0000","last_unstale":"2023-10-09T11:08:32.002164+0000","last_undegraded":"2023-10-09T11:08:32.002164+0000","last_fullsized":"2023-10-09T11:08:32.002164+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:03.653371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:03.653371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:09.629417+0000","scrub_duration":0.001576771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":25,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.010616+0000","last_change":"2023-10-09T11:07:33.241346+0000","last_active":"2023-10-09T11:08:32.010616+0000","last_peered":"2023-10-09T11:08:32.010616+0000","last_clean":"2023-10-09T11:08:32.010616+0000","last_became_active":"2023-10-09T11:07:33.241013+0000","last_became_peered":"2023-10-09T11:07:33.241013+0000","last_unstale":"2023-10-09T11:08:32.010616+0000","last_undegraded":"2023-10-09T11:08:32.010616+0000","last_fullsized":"2023-10-09T11:08:32.010616+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:57.164502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":263,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005036+0000","last_change":"2023-10-09T11:06:24.253714+0000","last_active":"2023-10-09T11:08:32.005036+0000","last_peered":"2023-10-09T11:08:32.005036+0000","last_clean":"2023-10-09T11:08:32.005036+0000","last_became_active":"2023-10-09T11:06:24.253143+0000","last_became_peered":"2023-10-09T11:06:24.253143+0000","last_unstale":"2023-10-09T11:08:32.005036+0000","last_undegraded":"2023-10-09T11:08:32.005036+0000","last_fullsized":"2023-10-09T11:08:32.005036+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:05.682288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":370,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004581+0000","last_change":"2023-10-09T11:05:20.313560+0000","last_active":"2023-10-09T11:08:32.004581+0000","last_peered":"2023-10-09T11:08:32.004581+0000","last_clean":"2023-10-09T11:08:32.004581+0000","last_became_active":"2023-10-09T11:05:20.313214+0000","last_became_peered":"2023-10-09T11:05:20.313214+0000","last_unstale":"2023-10-09T11:08:32.004581+0000","last_undegraded":"2023-10-09T11:08:32.004581+0000","last_fullsized":"2023-10-09T11:08:32.004581+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:32.527205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":972,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.001817+0000","last_change":"2023-10-09T11:05:23.329886+0000","last_active":"2023-10-09T11:08:32.001817+0000","last_peered":"2023-10-09T11:08:32.001817+0000","last_clean":"2023-10-09T11:08:32.001817+0000","last_became_active":"2023-10-09T11:05:23.329561+0000","last_became_peered":"2023-10-09T11:05:23.329561+0000","last_unstale":"2023-10-09T11:08:32.001817+0000","last_undegraded":"2023-10-09T11:08:32.001817+0000","last_fullsized":"2023-10-09T11:08:32.001817+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:46.933799+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":25,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.010101+0000","last_change":"2023-10-09T11:07:33.229864+0000","last_active":"2023-10-09T11:08:32.010101+0000","last_peered":"2023-10-09T11:08:32.010101+0000","last_clean":"2023-10-09T11:08:32.010101+0000","last_became_active":"2023-10-09T11:07:33.229547+0000","last_became_peered":"2023-10-09T11:07:33.229547+0000","last_unstale":"2023-10-09T11:08:32.010101+0000","last_undegraded":"2023-10-09T11:08:32.010101+0000","last_fullsized":"2023-10-09T11:08:32.010101+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:34.990439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":347,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.381620+0000","last_change":"2023-10-09T11:04:50.510688+0000","last_active":"2023-10-09T11:08:32.381620+0000","last_peered":"2023-10-09T11:08:32.381620+0000","last_clean":"2023-10-09T11:08:32.381620+0000","last_became_active":"2023-10-09T11:04:50.507404+0000","last_became_peered":"2023-10-09T11:04:50.507404+0000","last_unstale":"2023-10-09T11:08:32.381620+0000","last_undegraded":"2023-10-09T11:08:32.381620+0000","last_fullsized":"2023-10-09T11:08:32.381620+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:13.663394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":929,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.002737+0000","last_change":"2023-10-09T11:05:23.321040+0000","last_active":"2023-10-09T11:08:32.002737+0000","last_peered":"2023-10-09T11:08:32.002737+0000","last_clean":"2023-10-09T11:08:32.002737+0000","last_became_active":"2023-10-09T11:05:23.320674+0000","last_became_peered":"2023-10-09T11:05:23.320674+0000","last_unstale":"2023-10-09T11:08:32.002737+0000","last_undegraded":"2023-10-09T11:08:32.002737+0000","last_fullsized":"2023-10-09T11:08:32.002737+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.527660+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":392,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.004950+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:08:32.004950+0000","last_peered":"2023-10-09T11:08:32.004950+0000","last_clean":"2023-10-09T11:08:32.004950+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:08:32.004950+0000","last_undegraded":"2023-10-09T11:08:32.004950+0000","last_fullsized":"2023-10-09T11:08:32.004950+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":91,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.002053+0000","last_change":"2023-10-09T11:04:50.511984+0000","last_active":"2023-10-09T11:08:32.002053+0000","last_peered":"2023-10-09T11:08:32.002053+0000","last_clean":"2023-10-09T11:08:32.002053+0000","last_became_active":"2023-10-09T11:04:50.510820+0000","last_became_peered":"2023-10-09T11:04:50.510820+0000","last_unstale":"2023-10-09T11:08:32.002053+0000","last_undegraded":"2023-10-09T11:08:32.002053+0000","last_fullsized":"2023-10-09T11:08:32.002053+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:38.662189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":543,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005716+0000","last_change":"2023-10-09T11:05:20.327032+0000","last_active":"2023-10-09T11:08:32.005716+0000","last_peered":"2023-10-09T11:08:32.005716+0000","last_clean":"2023-10-09T11:08:32.005716+0000","last_became_active":"2023-10-09T11:05:20.320193+0000","last_became_peered":"2023-10-09T11:05:20.320193+0000","last_unstale":"2023-10-09T11:08:32.005716+0000","last_undegraded":"2023-10-09T11:08:32.005716+0000","last_fullsized":"2023-10-09T11:08:32.005716+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:44.862890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":91,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.000719+0000","last_change":"2023-10-09T11:04:50.384219+0000","last_active":"2023-10-09T11:08:32.000719+0000","last_peered":"2023-10-09T11:08:32.000719+0000","last_clean":"2023-10-09T11:08:32.000719+0000","last_became_active":"2023-10-09T11:04:50.384028+0000","last_became_peered":"2023-10-09T11:04:50.384028+0000","last_unstale":"2023-10-09T11:08:32.000719+0000","last_undegraded":"2023-10-09T11:08:32.000719+0000","last_fullsized":"2023-10-09T11:08:32.000719+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:04.721869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":738,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.010209+0000","last_change":"2023-10-09T11:07:36.291401+0000","last_active":"2023-10-09T11:08:32.010209+0000","last_peered":"2023-10-09T11:08:32.010209+0000","last_clean":"2023-10-09T11:08:32.010209+0000","last_became_active":"2023-10-09T11:07:36.291077+0000","last_became_peered":"2023-10-09T11:07:36.291077+0000","last_unstale":"2023-10-09T11:08:32.010209+0000","last_undegraded":"2023-10-09T11:08:32.010209+0000","last_fullsized":"2023-10-09T11:08:32.010209+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:04.790935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:04.790935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":190675,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":100,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.002784+0000","last_change":"2023-10-09T11:05:20.299321+0000","last_active":"2023-10-09T11:08:32.002784+0000","last_peered":"2023-10-09T11:08:32.002784+0000","last_clean":"2023-10-09T11:08:32.002784+0000","last_became_active":"2023-10-09T11:05:20.299019+0000","last_became_peered":"2023-10-09T11:05:20.299019+0000","last_unstale":"2023-10-09T11:08:32.002784+0000","last_undegraded":"2023-10-09T11:08:32.002784+0000","last_fullsized":"2023-10-09T11:08:32.002784+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:08.645909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":91,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.001827+0000","last_change":"2023-10-09T11:04:50.366890+0000","last_active":"2023-10-09T11:08:32.001827+0000","last_peered":"2023-10-09T11:08:32.001827+0000","last_clean":"2023-10-09T11:08:32.001827+0000","last_became_active":"2023-10-09T11:04:50.366558+0000","last_became_peered":"2023-10-09T11:04:50.366558+0000","last_unstale":"2023-10-09T11:08:32.001827+0000","last_undegraded":"2023-10-09T11:08:32.001827+0000","last_fullsized":"2023-10-09T11:08:32.001827+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:50.993153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":864,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005069+0000","last_change":"2023-10-09T11:04:53.274332+0000","last_active":"2023-10-09T11:08:32.005069+0000","last_peered":"2023-10-09T11:08:32.005069+0000","last_clean":"2023-10-09T11:08:32.005069+0000","last_became_active":"2023-10-09T11:04:53.273798+0000","last_became_peered":"2023-10-09T11:04:53.273798+0000","last_unstale":"2023-10-09T11:08:32.005069+0000","last_undegraded":"2023-10-09T11:08:32.005069+0000","last_fullsized":"2023-10-09T11:08:32.005069+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:04.220970+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":112,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.005500+0000","last_change":"2023-10-09T11:06:24.249877+0000","last_active":"2023-10-09T11:08:32.005500+0000","last_peered":"2023-10-09T11:08:32.005500+0000","last_clean":"2023-10-09T11:08:32.005500+0000","last_became_active":"2023-10-09T11:06:24.249573+0000","last_became_peered":"2023-10-09T11:06:24.249573+0000","last_unstale":"2023-10-09T11:08:32.005500+0000","last_undegraded":"2023-10-09T11:08:32.005500+0000","last_fullsized":"2023-10-09T11:08:32.005500+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:11.798192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":613,"reported_epoch":564,"state":"active+clean","last_fresh":"2023-10-09T11:08:32.007669+0000","last_change":"2023-10-09T11:05:20.307347+0000","last_active":"2023-10-09T11:08:32.007669+0000","last_peered":"2023-10-09T11:08:32.007669+0000","last_clean":"2023-10-09T11:08:32.007669+0000","last_became_active":"2023-10-09T11:05:20.307126+0000","last_became_peered":"2023-10-09T11:05:20.307126+0000","last_unstale":"2023-10-09T11:08:32.007669+0000","last_undegraded":"2023-10-09T11:08:32.007669+0000","last_fullsized":"2023-10-09T11:08:32.007669+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:34.408515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":91,"num_bytes_recovered":15742,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12694,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":626148,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":31,"num_read_kb":56,"num_write":15,"num_write_kb":147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2632652,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332504,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1340728,"kb_used_data":445424,"kb_used_omap":0,"kb_used_meta":895296,"kb_avail":92404424,"statfs":{"total":95995035648,"available":94622130176,"internally_reserved":0,"allocated":456114176,"data_stored":454289343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916783104},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":1.3460000000000001,"15min":1.7350000000000001},"min":{"1min":0.254,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":9.8219999999999992,"5min":9.8219999999999992,"15min":9.8219999999999992},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.9099999999999999,"5min":1.6539999999999999,"15min":1.7669999999999999},"min":{"1min":0.28999999999999998,"5min":0.24299999999999999,"15min":0.216},"max":{"1min":5.4820000000000002,"5min":15.853,"15min":15.853},"last":0.33200000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:08:26 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":1.5029999999999999,"15min":1.5469999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":5.9669999999999996,"5min":11.619999999999999,"15min":11.619999999999999},"last":0.315},{"interface":"front","average":{"1min":2.851,"5min":2.0219999999999998,"15min":1.944},"min":{"1min":0.26300000000000001,"5min":0.25,"15min":0.25},"max":{"1min":13.096,"5min":13.096,"15min":13.096},"last":0.432}]},{"osd":2,"last update":"Mon Oct 9 11:07:52 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":1.929,"15min":1.7170000000000001},"min":{"1min":0.313,"5min":0.309,"15min":0.245},"max":{"1min":3.3050000000000002,"5min":16.605,"15min":16.605},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.466,"5min":1.4570000000000001,"15min":1.706},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":4.9569999999999999,"5min":9.7469999999999999,"15min":13.037000000000001},"last":0.38800000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:08:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3050000000000002,"5min":1.7050000000000001,"15min":1.5009999999999999},"min":{"1min":0.34200000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.3970000000000002,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.7070000000000001,"5min":1.885,"15min":1.823},"min":{"1min":0.34000000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.0999999999999996,"5min":8.2330000000000005,"15min":8.2330000000000005},"last":0.5}]},{"osd":4,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":0.91800000000000004,"15min":1.0940000000000001},"min":{"1min":0.29799999999999999,"5min":0.218,"15min":0.218},"max":{"1min":7.4630000000000001,"5min":7.4630000000000001,"15min":8.2940000000000005},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.034,"5min":1.256,"15min":1.244},"min":{"1min":0.30599999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.7190000000000003,"5min":11.808,"15min":16.481000000000002},"last":0.44600000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.0640000000000001,"15min":1.252},"min":{"1min":0.32800000000000001,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":5.4800000000000004,"5min":15.414999999999999,"15min":15.414999999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.345,"15min":1.198},"min":{"1min":0.375,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.045,"5min":15.369,"15min":15.369},"last":0.40200000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.226,"15min":1.26},"min":{"1min":0.34599999999999997,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":3.8500000000000001,"5min":14.159000000000001,"15min":14.159000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.121,"5min":1.472,"15min":1.355},"min":{"1min":0.36099999999999999,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":3.5830000000000002,"5min":13.993,"15min":16.538},"last":0.51900000000000002}]}]},{"osd":6,"up_from":44,"seq":188978561435,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1565224,"kb_used_data":68576,"kb_used_omap":0,"kb_used_meta":1496640,"kb_avail":92179928,"statfs":{"total":95995035648,"available":94392246272,"internally_reserved":0,"allocated":70221824,"data_stored":68412685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1532559360},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":1.4950000000000001,"15min":1.9419999999999999},"min":{"1min":0.22600000000000001,"5min":0.223,"15min":0.217},"max":{"1min":12.209,"5min":13.867000000000001,"15min":54.743000000000002},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.081,"5min":1.181,"15min":1.8919999999999999},"min":{"1min":0.27300000000000002,"5min":0.214,"15min":0.214},"max":{"1min":3.3740000000000001,"5min":10.055999999999999,"15min":53.701000000000001},"last":0.34399999999999997}]},{"osd":1,"last update":"Mon Oct 9 11:08:26 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.514,"15min":1.1910000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.23499999999999999},"max":{"1min":4.3710000000000004,"5min":16.225999999999999,"15min":41.241999999999997},"last":0.28499999999999998},{"interface":"front","average":{"1min":2.7330000000000001,"5min":2.1139999999999999,"15min":1.8260000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.248},"max":{"1min":9.2080000000000002,"5min":11.715,"15min":41.476999999999997},"last":0.30099999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:07:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":2.1890000000000001,"15min":1.857},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.7089999999999996,"5min":10.481999999999999,"15min":12.003},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.6990000000000001,"5min":1.764,"15min":1.8999999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.133,"5min":12.693,"15min":15.432},"last":0.40799999999999997}]},{"osd":3,"last update":"Mon Oct 9 11:08:23 2023","interfaces":[{"interface":"back","average":{"1min":3.6749999999999998,"5min":2.2080000000000002,"15min":1.702},"min":{"1min":0.40200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":11.395,"5min":28.948,"15min":28.948},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.653,"5min":2.9820000000000002,"15min":2.6200000000000001},"min":{"1min":0.307,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.964,"5min":28.893000000000001,"15min":28.893000000000001},"last":0.36199999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":1.171,"15min":1.0129999999999999},"min":{"1min":0.26500000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.0189999999999999,"5min":17.216999999999999,"15min":17.216999999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.78100000000000003,"15min":0.84299999999999997},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.24399999999999999},"max":{"1min":1.4059999999999999,"5min":10.025,"15min":10.025},"last":0.39800000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.1419999999999999,"15min":1.2709999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":1.103,"5min":10.209,"15min":19.786999999999999},"last":0.375},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.90500000000000003,"15min":0.97099999999999997},"min":{"1min":0.33200000000000002,"5min":0.248,"15min":0.248},"max":{"1min":1.4810000000000001,"5min":9.9469999999999992,"15min":9.9469999999999992},"last":0.42699999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:08:14 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.081,"15min":1.268},"min":{"1min":0.32200000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.7250000000000001,"5min":10.246,"15min":12.882},"last":0.437},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.83599999999999997,"15min":0.93300000000000005},"min":{"1min":0.35599999999999998,"5min":0.31,"15min":0.27500000000000002},"max":{"1min":3.2839999999999998,"5min":11.557,"15min":11.557},"last":0.48299999999999998}]}]},{"osd":1,"up_from":388,"seq":1666447310949,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":491200,"kb_used_data":7672,"kb_used_omap":13,"kb_used_meta":483506,"kb_avail":93253952,"statfs":{"total":95995035648,"available":95492046848,"internally_reserved":0,"allocated":7856128,"data_stored":4865317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":495110855},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":3.895,"5min":2.593,"15min":2.4369999999999998},"min":{"1min":0.26700000000000002,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":11.960000000000001,"5min":18.192,"15min":18.192},"last":0.22600000000000001},{"interface":"front","average":{"1min":3.9390000000000001,"5min":2.8260000000000001,"15min":1.9259999999999999},"min":{"1min":0.27000000000000002,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":10.099,"5min":17.957999999999998,"15min":17.957999999999998},"last":0.29499999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8439999999999999,"5min":2.5920000000000001,"15min":2.9300000000000002},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":9.5630000000000006,"5min":47.731000000000002,"15min":47.731000000000002},"last":0.26700000000000002},{"interface":"front","average":{"1min":3.8199999999999998,"5min":2.903,"15min":2.4769999999999999},"min":{"1min":0.29399999999999998,"5min":0.216,"15min":0.216},"max":{"1min":10.138,"5min":47.786999999999999,"15min":47.786999999999999},"last":0.33700000000000002}]},{"osd":3,"last update":"Mon Oct 9 11:08:26 2023","interfaces":[{"interface":"back","average":{"1min":2.819,"5min":3.3530000000000002,"15min":2.1099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":9.4960000000000004,"5min":92.447999999999993,"15min":92.447999999999993},"last":0.255},{"interface":"front","average":{"1min":3.069,"5min":3.4350000000000001,"15min":2.133},"min":{"1min":0.29099999999999998,"5min":0.217,"15min":0.217},"max":{"1min":13.295999999999999,"5min":92.066999999999993,"15min":92.066999999999993},"last":0.32500000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.7869999999999999,"15min":2.395},"min":{"1min":0.30499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.5199999999999996,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.04,"15min":1.6240000000000001},"min":{"1min":0.314,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":4.6079999999999997,"5min":11.589,"15min":11.589},"last":0.42099999999999999}]},{"osd":5,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":2.0830000000000002,"15min":2.3050000000000002},"min":{"1min":0.28999999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.0720000000000001,"5min":20.689,"15min":20.689},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.764,"5min":2.0590000000000002,"15min":2.2410000000000001},"min":{"1min":0.33800000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.4900000000000002,"5min":11.65,"15min":11.65},"last":0.61299999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.833,"5min":2.4049999999999998,"15min":2.411},"min":{"1min":0.312,"5min":0.309,"15min":0.309},"max":{"1min":6.0330000000000004,"5min":20.646999999999998,"15min":20.646999999999998},"last":0.498},{"interface":"front","average":{"1min":1.784,"5min":1.9219999999999999,"15min":2.1720000000000002},"min":{"1min":0.35599999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":5.5380000000000003,"5min":11.75,"15min":11.75},"last":0.67200000000000004}]},{"osd":7,"last update":"Mon Oct 9 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.869,"5min":2.1640000000000001,"15min":1.615},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.32900000000000001},"max":{"1min":4.5780000000000003,"5min":11.555999999999999,"15min":11.555999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.7789999999999999,"5min":2.0219999999999998,"15min":2.3210000000000002},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.1020000000000003,"5min":20.716000000000001,"15min":20.716000000000001},"last":0.53600000000000003}]}]},{"osd":0,"up_from":80,"seq":343597384007,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1051612,"kb_used_data":69332,"kb_used_omap":28,"kb_used_meta":982243,"kb_avail":92693540,"statfs":{"total":95995035648,"available":94918184960,"internally_reserved":0,"allocated":70995968,"data_stored":67910069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1005817680},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:08:29 2023","interfaces":[{"interface":"back","average":{"1min":1.946,"5min":2.0249999999999999,"15min":2.742},"min":{"1min":0.191,"5min":0.186,"15min":0.186},"max":{"1min":6.4909999999999997,"5min":14.644,"15min":19.608000000000001},"last":0.20000000000000001},{"interface":"front","average":{"1min":1.718,"5min":2.1549999999999998,"15min":2.8010000000000002},"min":{"1min":0.223,"5min":0.214,"15min":0.214},"max":{"1min":5.8609999999999998,"5min":15.834,"15min":19.992999999999999},"last":0.32000000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:07:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":2.339,"15min":2.29},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.17199999999999999},"max":{"1min":11.516,"5min":12.459,"15min":24.852},"last":0.27700000000000002},{"interface":"front","average":{"1min":1.73,"5min":2.246,"15min":2.694},"min":{"1min":0.215,"5min":0.20300000000000001,"15min":0.19800000000000001},"max":{"1min":8.4589999999999996,"5min":11.68,"15min":77.256},"last":0.24299999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":3.016,"15min":3.3599999999999999},"min":{"1min":0.35099999999999998,"5min":0.22,"15min":0.22},"max":{"1min":7.9050000000000002,"5min":25.050000000000001,"15min":25.050000000000001},"last":0.307},{"interface":"front","average":{"1min":2.2890000000000001,"5min":2.355,"15min":2.0510000000000002},"min":{"1min":0.30499999999999999,"5min":0.254,"15min":0.254},"max":{"1min":6.2389999999999999,"5min":14.125999999999999,"15min":14.125999999999999},"last":0.35999999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":1.877,"5min":2.1880000000000002,"15min":2.4489999999999998},"min":{"1min":0.27100000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.8150000000000004,"5min":14.507,"15min":77.177999999999997},"last":0.372},{"interface":"front","average":{"1min":1.6910000000000001,"5min":1.675,"15min":1.6419999999999999},"min":{"1min":0.33600000000000002,"5min":0.30199999999999999,"15min":0.28899999999999998},"max":{"1min":6.194,"5min":7.9249999999999998,"15min":10.513999999999999},"last":0.39700000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.7290000000000001,"15min":1.8160000000000001},"min":{"1min":0.34699999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.2199999999999998,"5min":7.9550000000000001,"15min":10.544},"last":0.438},{"interface":"front","average":{"1min":1.7050000000000001,"5min":1.7270000000000001,"15min":1.804},"min":{"1min":0.39200000000000002,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":4.6820000000000004,"5min":11.212999999999999,"15min":24.754999999999999},"last":0.47399999999999998}]},{"osd":6,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":1.8380000000000001,"15min":1.8959999999999999},"min":{"1min":0.377,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":6.173,"5min":7.9829999999999997,"15min":18.920000000000002},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.423,"5min":1.786,"15min":1.927},"min":{"1min":0.38100000000000001,"5min":0.35299999999999998,"15min":0.31900000000000001},"max":{"1min":4.2789999999999999,"5min":11.257999999999999,"15min":24.800000000000001},"last":0.44600000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.391,"5min":2.1789999999999998,"15min":2.0339999999999998},"min":{"1min":0.38900000000000001,"5min":0.35099999999999998,"15min":0.32700000000000001},"max":{"1min":9.8710000000000004,"5min":17.672000000000001,"15min":17.672000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":2.2170000000000001,"5min":2.4500000000000002,"15min":2.569},"min":{"1min":0.33400000000000002,"5min":0.311,"15min":0.311},"max":{"1min":4.9320000000000004,"5min":17.539000000000001,"15min":77.228999999999999},"last":0.42299999999999999}]}]},{"osd":2,"up_from":90,"seq":386547056948,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":781800,"kb_used_data":7584,"kb_used_omap":81,"kb_used_meta":774126,"kb_avail":92963352,"statfs":{"total":95995035648,"available":95194472448,"internally_reserved":0,"allocated":7766016,"data_stored":4856493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":792705242},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":2.2010000000000001,"15min":2.0609999999999999},"min":{"1min":0.186,"5min":0.182,"15min":0.17499999999999999},"max":{"1min":14.226000000000001,"5min":15.609,"15min":48.933999999999997},"last":0.247},{"interface":"front","average":{"1min":2.5419999999999998,"5min":2.056,"15min":1.8919999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.19600000000000001},"max":{"1min":9.5540000000000003,"5min":9.8010000000000002,"15min":15.595000000000001},"last":0.29899999999999999}]},{"osd":1,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":2.3780000000000001,"15min":2.5339999999999998},"min":{"1min":0.193,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":3.242,"5min":24.902000000000001,"15min":49.033999999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.6600000000000001,"5min":2.5640000000000001,"15min":2.7200000000000002},"min":{"1min":0.253,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":14.26,"5min":25.850999999999999,"15min":48.874000000000002},"last":0.29099999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:08:19 2023","interfaces":[{"interface":"back","average":{"1min":4.8019999999999996,"5min":2.9399999999999999,"15min":2.1560000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":25.890999999999998,"5min":25.890999999999998,"15min":25.890999999999998},"last":0.39600000000000002},{"interface":"front","average":{"1min":4.4560000000000004,"5min":2.895,"15min":2.395},"min":{"1min":0.372,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":28.689,"5min":28.689,"15min":28.689},"last":0.318}]},{"osd":4,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":1.796,"15min":1.8959999999999999},"min":{"1min":0.27100000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":11.391999999999999,"5min":11.391999999999999,"15min":15.672000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.468,"5min":1.7909999999999999,"15min":1.873},"min":{"1min":0.34899999999999998,"5min":0.249,"15min":0.249},"max":{"1min":8.3350000000000009,"5min":15.571,"15min":48.795000000000002},"last":0.42999999999999999}]},{"osd":5,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":1.597,"15min":1.5620000000000001},"min":{"1min":0.33700000000000002,"5min":0.29099999999999998,"15min":0.25900000000000001},"max":{"1min":7.4980000000000002,"5min":8.5310000000000006,"15min":12.128},"last":0.374},{"interface":"front","average":{"1min":1.8580000000000001,"5min":1.7869999999999999,"15min":2.0339999999999998},"min":{"1min":0.34899999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.441000000000001,"5min":11.441000000000001,"15min":49.061},"last":0.58099999999999996}]},{"osd":6,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.909,"15min":1.804},"min":{"1min":0.41899999999999998,"5min":0.313,"15min":0.313},"max":{"1min":8.6039999999999992,"5min":15.536,"15min":48.845999999999997},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.6499999999999999,"5min":2.012,"15min":1.657},"min":{"1min":0.42399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.157999999999999,"5min":10.157999999999999,"15min":10.157999999999999},"last":0.46600000000000003}]},{"osd":7,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":2.081,"15min":1.9419999999999999},"min":{"1min":0.45100000000000001,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":8.3800000000000008,"5min":15.59,"15min":48.966000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.492,"5min":1.917,"15min":1.762},"min":{"1min":0.46200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.5919999999999996,"5min":8.5549999999999997,"15min":14.94},"last":0.496}]}]},{"osd":3,"up_from":469,"seq":2014339661877,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":343028,"kb_used_data":7980,"kb_used_omap":11,"kb_used_meta":335028,"kb_avail":93402124,"statfs":{"total":95995035648,"available":95643774976,"internally_reserved":0,"allocated":8171520,"data_stored":4954141,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11309,"internal_metadata":343069651},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":3.004,"5min":2.1840000000000002,"15min":1.875},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":14.521000000000001,"5min":14.882,"15min":14.882},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.3919999999999999,"5min":1.397,"15min":1.044},"min":{"1min":0.20100000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":9.4030000000000005,"5min":9.4030000000000005,"15min":9.4030000000000005},"last":0.26800000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":2.218,"15min":1.6739999999999999},"min":{"1min":0.245,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":8.8350000000000009,"5min":17.93,"15min":17.93},"last":0.28899999999999998},{"interface":"front","average":{"1min":3.0099999999999998,"5min":2.246,"15min":1.8049999999999999},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":10.645,"5min":19.163,"15min":19.163},"last":0.36599999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4860000000000002,"5min":2.0840000000000001,"15min":1.419},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":12.323,"5min":38.840000000000003,"15min":38.840000000000003},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.6539999999999999,"5min":2.2120000000000002,"15min":1.4570000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":10.172000000000001,"5min":37.991999999999997,"15min":37.991999999999997},"last":0.38700000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6419999999999999,"5min":1.655,"15min":1.6000000000000001},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.3719999999999999,"5min":14.907999999999999,"15min":14.907999999999999},"last":0.622},{"interface":"front","average":{"1min":2.0489999999999999,"5min":1.1439999999999999,"15min":0.84499999999999997},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.7889999999999997,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.39800000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.4259999999999999,"15min":1.417},"min":{"1min":0.33100000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.8499999999999996,"5min":9.3300000000000001,"15min":9.3300000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.5529999999999999,"5min":1.5649999999999999,"15min":1.482},"min":{"1min":0.34699999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.3320000000000007,"5min":10.029,"15min":10.029},"last":0.65500000000000003}]},{"osd":6,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.256,"5min":1.6539999999999999,"15min":1.74},"min":{"1min":0.32300000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.9550000000000001,"5min":15.455,"15min":15.455},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.8100000000000001,"5min":1.706,"15min":1.6639999999999999},"min":{"1min":0.36799999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.4019999999999992,"5min":14.837,"15min":14.837},"last":0.58799999999999997}]},{"osd":7,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.365,"15min":1.1220000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.6769999999999996,"5min":6.907,"15min":6.907},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.55,"5min":1.595,"15min":1.3029999999999999},"min":{"1min":0.33700000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.9320000000000004,"5min":6.9119999999999999,"15min":6.9119999999999999},"last":0.57099999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019298,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":898868,"kb_used_data":6124,"kb_used_omap":0,"kb_used_meta":892736,"kb_avail":92846284,"statfs":{"total":95995035648,"available":95074594816,"internally_reserved":0,"allocated":6270976,"data_stored":4449948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914161664},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.613,"15min":2.4129999999999998},"min":{"1min":0.22600000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":6.1289999999999996,"5min":13.162000000000001,"15min":213.88999999999999},"last":0.26900000000000002},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.742,"15min":2.5019999999999998},"min":{"1min":0.26500000000000001,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":3.875,"5min":9.0670000000000002,"15min":214.09800000000001},"last":0.34300000000000003}]},{"osd":1,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2989999999999999,"5min":2.6059999999999999,"15min":2.6440000000000001},"min":{"1min":0.28299999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.718,"5min":57.966999999999999,"15min":121.41},"last":0.313},{"interface":"front","average":{"1min":1.4810000000000001,"5min":2.2519999999999998,"15min":2.1280000000000001},"min":{"1min":0.26900000000000002,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":6.1980000000000004,"5min":57.832000000000001,"15min":122.21899999999999},"last":0.39200000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:08:00 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":1.917,"15min":1.74},"min":{"1min":0.32100000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.2939999999999996,"5min":18.407,"15min":18.407},"last":0.42399999999999999},{"interface":"front","average":{"1min":2.149,"5min":1.9810000000000001,"15min":2.0910000000000002},"min":{"1min":0.27900000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.1479999999999997,"5min":8.4250000000000007,"15min":27.818000000000001},"last":0.50900000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":2.4380000000000002,"15min":1.98},"min":{"1min":0.34799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.1689999999999996,"5min":60.753,"15min":60.753},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.5049999999999999,"5min":2.2069999999999999,"15min":1.5149999999999999},"min":{"1min":0.45300000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.4980000000000002,"5min":61.473999999999997,"15min":61.473999999999997},"last":0.46200000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":1.353,"15min":1.3600000000000001},"min":{"1min":0.30499999999999999,"5min":0.224,"15min":0.224},"max":{"1min":4.7670000000000003,"5min":11.59,"15min":19.068999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.079,"5min":1.2809999999999999,"15min":1.238},"min":{"1min":0.34200000000000003,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":3.7570000000000001,"5min":9.8610000000000007,"15min":15.535},"last":0.47499999999999998}]},{"osd":6,"last update":"Mon Oct 9 11:08:00 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":1.353,"15min":1.1599999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":7.5860000000000003,"5min":8.3930000000000007,"15min":15.739000000000001},"last":0.497},{"interface":"front","average":{"1min":1.4610000000000001,"5min":1.4890000000000001,"15min":1.4570000000000001},"min":{"1min":0.34899999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":9.5210000000000008,"5min":12.795999999999999,"15min":14.093999999999999},"last":0.48799999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:08:00 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.5600000000000001,"15min":1.5529999999999999},"min":{"1min":0.308,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.4540000000000002,"5min":12.753,"15min":19.923999999999999},"last":0.374},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.4239999999999999,"15min":1.476},"min":{"1min":0.314,"5min":0.29799999999999999,"15min":0.28799999999999998},"max":{"1min":4.21,"5min":10.853,"15min":19.859999999999999},"last":0.53700000000000003}]}]},{"osd":5,"up_from":37,"seq":158913790367,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1205668,"kb_used_data":6364,"kb_used_omap":0,"kb_used_meta":1199296,"kb_avail":92539484,"statfs":{"total":95995035648,"available":94760431616,"internally_reserved":0,"allocated":6516736,"data_stored":4708057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1228079104},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:25 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":2.3239999999999998,"15min":2.2919999999999998},"min":{"1min":0.255,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.4349999999999996,"5min":49.665999999999997,"15min":58.271999999999998},"last":0.438},{"interface":"front","average":{"1min":1.849,"5min":2.3740000000000001,"15min":2.3399999999999999},"min":{"1min":0.29699999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.4610000000000003,"5min":49.701999999999998,"15min":58.25},"last":0.40600000000000003}]},{"osd":1,"last update":"Mon Oct 9 11:08:32 2023","interfaces":[{"interface":"back","average":{"1min":2.5179999999999998,"5min":1.899,"15min":2.0670000000000002},"min":{"1min":0.246,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.1850000000000005,"5min":16.962,"15min":32.491999999999997},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.3439999999999999,"5min":2.004,"15min":1.917},"min":{"1min":0.32800000000000001,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":9.1989999999999998,"5min":17.033999999999999,"15min":32.537999999999997},"last":0.34200000000000003}]},{"osd":2,"last update":"Mon Oct 9 11:07:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.454,"15min":1.786},"min":{"1min":0.307,"5min":0.254,"15min":0.221},"max":{"1min":5.1269999999999998,"5min":6.3929999999999998,"15min":11.257999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.1869999999999998,"5min":2.0129999999999999,"15min":1.9370000000000001},"min":{"1min":0.29099999999999998,"5min":0.253,"15min":0.253},"max":{"1min":8.0359999999999996,"5min":11.673,"15min":14.621},"last":0.371}]},{"osd":3,"last update":"Mon Oct 9 11:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.962,"5min":1.738,"15min":1.5720000000000001},"min":{"1min":0.311,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.0289999999999999,"5min":15.535,"15min":15.535},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.8839999999999999,"5min":2.0710000000000002,"15min":1.8959999999999999},"min":{"1min":0.40200000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":15.757999999999999,"5min":15.757999999999999,"15min":15.757999999999999},"last":0.41599999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:08:32 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":1.0089999999999999,"15min":0.93799999999999994},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.216},"max":{"1min":3.0680000000000001,"5min":12.042,"15min":12.042},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.3480000000000001,"5min":1.1020000000000001,"15min":1.1899999999999999},"min":{"1min":0.27000000000000002,"5min":0.245,"15min":0.245},"max":{"1min":6.8639999999999999,"5min":9.2490000000000006,"15min":9.2490000000000006},"last":0.32000000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:07:42 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.3300000000000001,"15min":1.2110000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27100000000000002},"max":{"1min":1.5669999999999999,"5min":9.1969999999999992,"15min":9.1969999999999992},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.65300000000000002,"5min":1.288,"15min":1.0629999999999999},"min":{"1min":0.33700000000000002,"5min":0.32800000000000001,"15min":0.29099999999999998},"max":{"1min":2.6059999999999999,"5min":11.983000000000001,"15min":11.983000000000001},"last":0.47599999999999998}]},{"osd":7,"last update":"Mon Oct 9 11:08:08 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.3160000000000001,"15min":1.302},"min":{"1min":0.35399999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.0469999999999997,"5min":8.9429999999999996,"15min":9.3539999999999992},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.03,"5min":1.278,"15min":1.177},"min":{"1min":0.32800000000000001,"5min":0.30499999999999999,"15min":0.30099999999999999},"max":{"1min":4.2350000000000003,"5min":11.726000000000001,"15min":11.726000000000001},"last":0.46000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3733,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:08:39.232 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:39.250 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:39.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:39.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:39.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[195507]: pgmap v1389: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 133 active+clean; 32 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 745/1578 objects degraded (47.212%); 33/1578 objects misplaced (2.091%); 0 B/s, 306 objects/s recovering 2023-10-09T11:08:39.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[195507]: failed to encode map e567 with expected crc 2023-10-09T11:08:39.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T11:08:39.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[195507]: osdmap e567: 8 total, 8 up, 8 in 2023-10-09T11:08:39.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[195507]: failed to encode map e567 with expected crc 2023-10-09T11:08:39.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[195507]: from='client.56629 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[195507]: Health check update: Degraded data redundancy: 745/1578 objects degraded (47.212%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:08:39.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[195507]: failed to encode map e567 with expected crc 2023-10-09T11:08:39.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[200197]: pgmap v1389: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 133 active+clean; 32 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 745/1578 objects degraded (47.212%); 33/1578 objects misplaced (2.091%); 0 B/s, 306 objects/s recovering 2023-10-09T11:08:39.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[200197]: failed to encode map e567 with expected crc 2023-10-09T11:08:39.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T11:08:39.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[200197]: osdmap e567: 8 total, 8 up, 8 in 2023-10-09T11:08:39.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[200197]: failed to encode map e567 with expected crc 2023-10-09T11:08:39.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[200197]: from='client.56629 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:39.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[200197]: Health check update: Degraded data redundancy: 745/1578 objects degraded (47.212%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:08:39.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:39 smithi125 ceph-mon[200197]: failed to encode map e567 with expected crc 2023-10-09T11:08:39.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:39.381 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:39.385 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:39.406 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:39.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:39.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:39.441 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:08:39.441 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:08:39.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:39 smithi155 ceph-mon[184854]: pgmap v1389: 139 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 133 active+clean; 32 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 745/1578 objects degraded (47.212%); 33/1578 objects misplaced (2.091%); 0 B/s, 306 objects/s recovering 2023-10-09T11:08:39.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:39 smithi155 ceph-mon[184854]: failed to encode map e567 with expected crc 2023-10-09T11:08:39.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:39 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T11:08:39.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:39 smithi155 ceph-mon[184854]: osdmap e567: 8 total, 8 up, 8 in 2023-10-09T11:08:39.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:39 smithi155 ceph-mon[184854]: failed to encode map e567 with expected crc 2023-10-09T11:08:39.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:39 smithi155 ceph-mon[184854]: from='client.56629 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:39.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:39 smithi155 ceph-mon[184854]: Health check update: Degraded data redundancy: 745/1578 objects degraded (47.212%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T11:08:39.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:39 smithi155 ceph-mon[184854]: failed to encode map e567 with expected crc 2023-10-09T11:08:39.511 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:39.538 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:39.556 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:39.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:39.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:39.632 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:08:39.660 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:39.687 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:39.705 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:39.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:39.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:39.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:39.835 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:39.853 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:39.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:39.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:39.957 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:39.986 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:40.004 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T11:08:40.059 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:40.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:40.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:40.163 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:40.191 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:40.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:40 smithi155 ceph-mon[184854]: failed to encode map e567 with expected crc 2023-10-09T11:08:40.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:40 smithi155 ceph-mon[184854]: 1.1c scrub starts 2023-10-09T11:08:40.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:40 smithi155 ceph-mon[184854]: 1.1c scrub ok 2023-10-09T11:08:40.236 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:40 smithi125 ceph-mon[200197]: failed to encode map e567 with expected crc 2023-10-09T11:08:40.236 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:40 smithi125 ceph-mon[200197]: 1.1c scrub starts 2023-10-09T11:08:40.236 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:40 smithi125 ceph-mon[200197]: 1.1c scrub ok 2023-10-09T11:08:40.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:40 smithi125 ceph-mon[195507]: failed to encode map e567 with expected crc 2023-10-09T11:08:40.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:40 smithi125 ceph-mon[195507]: 1.1c scrub starts 2023-10-09T11:08:40.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:40 smithi125 ceph-mon[195507]: 1.1c scrub ok 2023-10-09T11:08:40.270 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:40.361 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:40.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:40.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:40.465 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:40.493 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:40.512 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:40.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:40.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:40.616 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:40.642 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:40.651 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:40.661 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:40.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:40.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:40.764 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:40.791 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:40.809 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:40.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:40.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:40.885 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:08:40.903 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:08:40.913 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:40.922 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:08:40.939 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:40.942 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:08:40.957 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:40.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:40.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:40.961 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:08:40.980 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:08:40.999 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:08:41.018 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:08:41.028 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:08:41.028 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:08:41.028 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:08:41.029 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:08:41.029 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:08:41.029 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:08:41.029 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:08:41.029 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:08:41.061 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:41.086 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:41 smithi155 ceph-mon[184854]: pgmap v1391: 103 pgs: 103 active+clean; 617 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:41.086 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:41 smithi155 ceph-mon[184854]: 1.11 deep-scrub starts 2023-10-09T11:08:41.086 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:41 smithi155 ceph-mon[184854]: 1.11 deep-scrub ok 2023-10-09T11:08:41.086 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:41 smithi155 ceph-mon[184854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 745/1578 objects degraded (47.212%), 1 pg degraded, 1 pg undersized) 2023-10-09T11:08:41.086 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:41 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T11:08:41.086 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:41 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4268146045' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:41.088 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:41.106 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:41.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:41.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:41.209 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:41.236 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:41.255 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:41.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:41.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:41.320 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:08:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:08:41] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.43.0" 2023-10-09T11:08:41.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:41 smithi125 ceph-mon[195507]: pgmap v1391: 103 pgs: 103 active+clean; 617 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:41.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:41 smithi125 ceph-mon[195507]: 1.11 deep-scrub starts 2023-10-09T11:08:41.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:41 smithi125 ceph-mon[195507]: 1.11 deep-scrub ok 2023-10-09T11:08:41.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:41 smithi125 ceph-mon[195507]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 745/1578 objects degraded (47.212%), 1 pg degraded, 1 pg undersized) 2023-10-09T11:08:41.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:41 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T11:08:41.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:41 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4268146045' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:41 smithi125 ceph-mon[200197]: pgmap v1391: 103 pgs: 103 active+clean; 617 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:41 smithi125 ceph-mon[200197]: 1.11 deep-scrub starts 2023-10-09T11:08:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:41 smithi125 ceph-mon[200197]: 1.11 deep-scrub ok 2023-10-09T11:08:41.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:41 smithi125 ceph-mon[200197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 745/1578 objects degraded (47.212%), 1 pg degraded, 1 pg undersized) 2023-10-09T11:08:41.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:41 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T11:08:41.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:41 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4268146045' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:41.358 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:41.385 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:41.404 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:41.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:41.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:41.507 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:41.540 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:41.661 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:41.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:41.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:41.765 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:41.793 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:41.863 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:41.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:41.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:41.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:41.997 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:42.069 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:42.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:42.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:42.177 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:42.209 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:42.229 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:08:42.231 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:42.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:42.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:42.255 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:08:42.255 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:08:42.255 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T11:08:42.255 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T11:08:42.255 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:08:42.334 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:42.362 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:42.383 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:42.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:42.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:42.487 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:42.520 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:42.541 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:42.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:42.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:42.645 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:42.675 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:42.695 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:42.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:42.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:42.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:42.827 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:42.847 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:42.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:42.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:42.875 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:42.914 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T11:08:42.950 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:42.984 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:43.012 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:43.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:43.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:43.048 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:08:43.116 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:43.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:43 smithi155 ceph-mon[184854]: pgmap v1392: 103 pgs: 103 active+clean; 582 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:43.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:43 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/298948369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:08:43.144 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:43.163 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:43.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:43.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:43.189 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:43 smithi125 ceph-mon[195507]: pgmap v1392: 103 pgs: 103 active+clean; 582 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:43.189 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:43 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/298948369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:08:43.189 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:43 smithi125 ceph-mon[200197]: pgmap v1392: 103 pgs: 103 active+clean; 582 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:43.189 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:43 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/298948369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:08:43.267 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:43.293 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:43.312 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:43.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:43.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:43.416 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:43.444 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:43.621 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:43.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:43.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:43.681 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:08:43.725 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:43.752 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:43.771 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:43.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:43.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:43.875 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:43.901 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:43.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:43.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:43.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:43.941 INFO:tasks.workunit.client.0.smithi125.stderr:Updating files: 98% (11312/11517) Updating files: 99% (11402/11517) Updating files: 100% (11517/11517) Updating files: 100% (11517/11517), done. 2023-10-09T11:08:43.957 DEBUG:teuthology.orchestra.run.smithi125:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T11:08:43.983 INFO:tasks.workunit.client.0.smithi125.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T11:08:43.985 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T11:08:43.986 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T11:08:44.023 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:44.049 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:44.068 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:44.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:44.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:44.171 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:44.198 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:44.216 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:44.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:44.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:44.319 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:44.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:44 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:08:44.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:44 smithi125 ceph-mon[195507]: failed to encode map e568 with expected crc 2023-10-09T11:08:44.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:44 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/298948369' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:08:44.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:44 smithi125 ceph-mon[195507]: osdmap e568: 8 total, 8 up, 8 in 2023-10-09T11:08:44.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:44 smithi125 ceph-mon[195507]: failed to encode map e568 with expected crc 2023-10-09T11:08:44.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:44 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:08:44.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:44 smithi125 ceph-mon[200197]: failed to encode map e568 with expected crc 2023-10-09T11:08:44.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:44 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/298948369' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:08:44.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:44 smithi125 ceph-mon[200197]: osdmap e568: 8 total, 8 up, 8 in 2023-10-09T11:08:44.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:44 smithi125 ceph-mon[200197]: failed to encode map e568 with expected crc 2023-10-09T11:08:44.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:44 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:08:44.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:44 smithi155 ceph-mon[184854]: failed to encode map e568 with expected crc 2023-10-09T11:08:44.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:44 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/298948369' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:08:44.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:44 smithi155 ceph-mon[184854]: osdmap e568: 8 total, 8 up, 8 in 2023-10-09T11:08:44.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:44 smithi155 ceph-mon[184854]: failed to encode map e568 with expected crc 2023-10-09T11:08:44.347 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:44.366 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:44.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:44.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:44.387 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T11:08:44.469 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:44.496 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:44.514 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:44.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:44.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:44.525 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T11:08:44.570 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T11:08:44.571 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T11:08:44.571 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T11:08:44.618 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:44.646 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:44.710 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T11:08:44.716 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:44.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:44.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:44.723 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:44.824 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:44.850 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T11:08:44.850 DEBUG:teuthology.orchestra.run.smithi125:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T11:08:44.852 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:44.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:44.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:44.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:44.978 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T11:08:44.979 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T11:08:44.980 DEBUG:teuthology.orchestra.run.smithi125:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T11:08:44.980 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:45.047 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:45.173 INFO:tasks.workunit.client.0.smithi125.stdout:Running main() from gmock_main.cc 2023-10-09T11:08:45.173 INFO:tasks.workunit.client.0.smithi125.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T11:08:45.173 INFO:tasks.workunit.client.0.smithi125.stdout:[----------] Global test environment set-up. 2023-10-09T11:08:45.173 INFO:tasks.workunit.client.0.smithi125.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T11:08:45.309 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:45.317 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:45.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:45.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:45.423 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:45.451 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:45.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:45 smithi125 ceph-mon[200197]: pgmap v1394: 103 pgs: 103 active+clean; 582 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:45.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:08:45.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:45.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:08:45.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:45 smithi125 ceph-mon[195507]: pgmap v1394: 103 pgs: 103 active+clean; 582 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:45.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:08:45.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:45.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:08:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:45 smithi155 ceph-mon[184854]: pgmap v1394: 103 pgs: 103 active+clean; 582 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:08:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:08:45.973 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:45.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:45.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:46.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:08:46.077 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:46.111 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:46.386 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:08:46.388 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:08:47.081 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T11:08:47.097 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_all_features (17 ms) 2023-10-09T11:08:47.097 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T11:08:47.187 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:47.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:47.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:47.282 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.copyup (185 ms) 2023-10-09T11:08:47.283 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T11:08:47.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:47.296 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.sparse_copyup (14 ms) 2023-10-09T11:08:47.297 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T11:08:47.319 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:47 smithi155 ceph-mon[184854]: pgmap v1395: 103 pgs: 103 active+clean; 582 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:47.319 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:47 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:47.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:47 smithi155 ceph-mon[184854]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:47 smithi155 ceph-mon[184854]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:47 smithi155 ceph-mon[184854]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:47 smithi155 ceph-mon[184854]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:47 smithi155 ceph-mon[184854]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:47 smithi155 ceph-mon[184854]: osdmap e569: 8 total, 8 up, 8 in 2023-10-09T11:08:47.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:47 smithi155 ceph-mon[184854]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:47 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3362784941' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-472182-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:08:47.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:47 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-472182-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:08:47.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:47 smithi155 ceph-mon[184854]: from='client.46769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:47.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:47 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:47.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:47 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:47.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[195507]: pgmap v1395: 103 pgs: 103 active+clean; 582 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:47.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:47.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[195507]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[195507]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[195507]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[195507]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[195507]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[195507]: osdmap e569: 8 total, 8 up, 8 in 2023-10-09T11:08:47.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[195507]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3362784941' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-472182-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:08:47.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-472182-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:08:47.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[195507]: from='client.46769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:47.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:47.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:47.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[200197]: pgmap v1395: 103 pgs: 103 active+clean; 582 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:47.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:47.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[200197]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[200197]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[200197]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[200197]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[200197]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[200197]: osdmap e569: 8 total, 8 up, 8 in 2023-10-09T11:08:47.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[200197]: failed to encode map e569 with expected crc 2023-10-09T11:08:47.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3362784941' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-472182-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:08:47.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-472182-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T11:08:47.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[200197]: from='client.46769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:47.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:47.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:47 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:47.324 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:47.342 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:47.344 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_and_set_id (48 ms) 2023-10-09T11:08:47.344 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T11:08:47.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:47.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:47.354 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.add_remove_child (10 ms) 2023-10-09T11:08:47.355 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T11:08:47.382 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.directory_methods (27 ms) 2023-10-09T11:08:47.382 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.create 2023-10-09T11:08:47.432 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.create (51 ms) 2023-10-09T11:08:47.432 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T11:08:47.433 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_features (1 ms) 2023-10-09T11:08:47.433 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T11:08:47.435 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-10-09T11:08:47.435 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T11:08:47.436 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_create_timestamp (1 ms) 2023-10-09T11:08:47.437 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T11:08:47.438 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-10-09T11:08:47.438 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T11:08:47.439 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-10-09T11:08:47.439 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T11:08:47.443 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2023-10-09T11:08:47.443 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T11:08:47.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:47.447 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2023-10-09T11:08:47.447 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T11:08:47.451 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2023-10-09T11:08:47.451 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T11:08:47.468 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.protection_status (17 ms) 2023-10-09T11:08:47.468 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T11:08:47.475 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2023-10-09T11:08:47.475 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T11:08:47.477 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:47.517 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.parents_v1 (41 ms) 2023-10-09T11:08:47.517 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T11:08:47.544 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.parents_v2 (27 ms) 2023-10-09T11:08:47.544 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T11:08:47.563 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.snapshots (20 ms) 2023-10-09T11:08:47.563 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T11:08:47.567 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.snapid_race (3 ms) 2023-10-09T11:08:47.567 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T11:08:47.575 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2023-10-09T11:08:47.576 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T11:08:47.580 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_save (5 ms) 2023-10-09T11:08:47.580 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T11:08:47.588 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_resize (7 ms) 2023-10-09T11:08:47.588 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T11:08:47.599 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_update (12 ms) 2023-10-09T11:08:47.599 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T11:08:47.599 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-10-09T11:08:47.599 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T11:08:47.609 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_snap_add (9 ms) 2023-10-09T11:08:47.609 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T11:08:47.618 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_snap_remove (10 ms) 2023-10-09T11:08:47.618 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T11:08:47.623 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-10-09T11:08:47.624 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T11:08:47.653 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.metadata (29 ms) 2023-10-09T11:08:47.653 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T11:08:47.659 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2023-10-09T11:08:47.659 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T11:08:47.694 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror (34 ms) 2023-10-09T11:08:47.694 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T11:08:47.709 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror_image (16 ms) 2023-10-09T11:08:47.709 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T11:08:48.020 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:48 smithi125 ceph-mon[195507]: failed to encode map e570 with expected crc 2023-10-09T11:08:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:48 smithi125 ceph-mon[195507]: failed to encode map e570 with expected crc 2023-10-09T11:08:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:48 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-472182-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T11:08:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:48 smithi125 ceph-mon[195507]: osdmap e570: 8 total, 8 up, 8 in 2023-10-09T11:08:48.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:48 smithi125 ceph-mon[195507]: failed to encode map e570 with expected crc 2023-10-09T11:08:48.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:48 smithi125 ceph-mon[195507]: failed to encode map e570 with expected crc 2023-10-09T11:08:48.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:48 smithi125 ceph-mon[195507]: failed to encode map e570 with expected crc 2023-10-09T11:08:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:48 smithi125 ceph-mon[200197]: failed to encode map e570 with expected crc 2023-10-09T11:08:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:48 smithi125 ceph-mon[200197]: failed to encode map e570 with expected crc 2023-10-09T11:08:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:48 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-472182-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T11:08:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:48 smithi125 ceph-mon[200197]: osdmap e570: 8 total, 8 up, 8 in 2023-10-09T11:08:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:48 smithi125 ceph-mon[200197]: failed to encode map e570 with expected crc 2023-10-09T11:08:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:48 smithi125 ceph-mon[200197]: failed to encode map e570 with expected crc 2023-10-09T11:08:48.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:48 smithi125 ceph-mon[200197]: failed to encode map e570 with expected crc 2023-10-09T11:08:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:48 smithi155 ceph-mon[184854]: failed to encode map e570 with expected crc 2023-10-09T11:08:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:48 smithi155 ceph-mon[184854]: failed to encode map e570 with expected crc 2023-10-09T11:08:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:48 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-472182-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T11:08:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:48 smithi155 ceph-mon[184854]: osdmap e570: 8 total, 8 up, 8 in 2023-10-09T11:08:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:48 smithi155 ceph-mon[184854]: failed to encode map e570 with expected crc 2023-10-09T11:08:48.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:48 smithi155 ceph-mon[184854]: failed to encode map e570 with expected crc 2023-10-09T11:08:48.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:48 smithi155 ceph-mon[184854]: failed to encode map e570 with expected crc 2023-10-09T11:08:49.022 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:49.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:49.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:49.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:49.206 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1396,"stamp":"2023-10-09T11:08:46.077466+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":2377214,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5046,"ondisk_log_size":5046,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7570192,"kb_used_data":484880,"kb_used_omap":134,"kb_used_meta":7085113,"kb_avail":742391024,"statfs":{"total":767960285184,"available":760208408576,"internally_reserved":0,"allocated":496517120,"data_stored":476944741,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":137532,"internal_metadata":7255156420},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-35780,"num_objects":-1,"num_object_clones":0,"num_object_copies":-3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1,"num_whiteouts":0,"num_read":-31,"num_read_kb":-56,"num_write":-15,"num_write_kb":-147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":-271180,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.038212"},"pg_stats":[{"pgid":"12.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.18","version":"76'161","reported_seq":855,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.083183+0000","last_change":"2023-10-09T11:07:36.287455+0000","last_active":"2023-10-09T11:08:38.083183+0000","last_peered":"2023-10-09T11:08:38.083183+0000","last_clean":"2023-10-09T11:08:38.083183+0000","last_became_active":"2023-10-09T11:07:36.287052+0000","last_became_peered":"2023-10-09T11:07:36.287052+0000","last_unstale":"2023-10-09T11:08:38.083183+0000","last_undegraded":"2023-10-09T11:08:38.083183+0000","last_fullsized":"2023-10-09T11:08:38.083183+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:18.864873+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":350,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.700236+0000","last_change":"2023-10-09T11:03:21.185383+0000","last_active":"2023-10-09T11:08:38.700236+0000","last_peered":"2023-10-09T11:08:38.700236+0000","last_clean":"2023-10-09T11:08:38.700236+0000","last_became_active":"2023-10-09T11:03:21.185022+0000","last_became_peered":"2023-10-09T11:03:21.185022+0000","last_unstale":"2023-10-09T11:08:38.700236+0000","last_undegraded":"2023-10-09T11:08:38.700236+0000","last_fullsized":"2023-10-09T11:08:38.700236+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:05.230434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":94,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.034613+0000","last_change":"2023-10-09T11:04:50.319477+0000","last_active":"2023-10-09T11:08:38.034613+0000","last_peered":"2023-10-09T11:08:38.034613+0000","last_clean":"2023-10-09T11:08:38.034613+0000","last_became_active":"2023-10-09T11:04:50.319148+0000","last_became_peered":"2023-10-09T11:04:50.319148+0000","last_unstale":"2023-10-09T11:08:38.034613+0000","last_undegraded":"2023-10-09T11:08:38.034613+0000","last_fullsized":"2023-10-09T11:08:38.034613+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:48.725566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.16","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":725,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.084971+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:08:38.084971+0000","last_peered":"2023-10-09T11:08:38.084971+0000","last_clean":"2023-10-09T11:08:38.084971+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:08:38.084971+0000","last_undegraded":"2023-10-09T11:08:38.084971+0000","last_fullsized":"2023-10-09T11:08:38.084971+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":281,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.492402+0000","last_change":"2023-10-09T11:04:50.333711+0000","last_active":"2023-10-09T11:08:37.492402+0000","last_peered":"2023-10-09T11:08:37.492402+0000","last_clean":"2023-10-09T11:08:37.492402+0000","last_became_active":"2023-10-09T11:04:50.333262+0000","last_became_peered":"2023-10-09T11:04:50.333262+0000","last_unstale":"2023-10-09T11:08:37.492402+0000","last_undegraded":"2023-10-09T11:08:37.492402+0000","last_fullsized":"2023-10-09T11:08:37.492402+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.843765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":28,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.041818+0000","last_change":"2023-10-09T11:07:33.237903+0000","last_active":"2023-10-09T11:08:38.041818+0000","last_peered":"2023-10-09T11:08:38.041818+0000","last_clean":"2023-10-09T11:08:38.041818+0000","last_became_active":"2023-10-09T11:07:33.237618+0000","last_became_peered":"2023-10-09T11:07:33.237618+0000","last_unstale":"2023-10-09T11:08:38.041818+0000","last_undegraded":"2023-10-09T11:08:38.041818+0000","last_fullsized":"2023-10-09T11:08:38.041818+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:25.600818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":743,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.040801+0000","last_change":"2023-10-09T11:07:36.284534+0000","last_active":"2023-10-09T11:08:38.040801+0000","last_peered":"2023-10-09T11:08:38.040801+0000","last_clean":"2023-10-09T11:08:38.040801+0000","last_became_active":"2023-10-09T11:07:36.284254+0000","last_became_peered":"2023-10-09T11:07:36.284254+0000","last_unstale":"2023-10-09T11:08:38.040801+0000","last_undegraded":"2023-10-09T11:08:38.040801+0000","last_fullsized":"2023-10-09T11:08:38.040801+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:19.159630+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":94,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.036669+0000","last_change":"2023-10-09T11:04:50.310477+0000","last_active":"2023-10-09T11:08:38.036669+0000","last_peered":"2023-10-09T11:08:38.036669+0000","last_clean":"2023-10-09T11:08:38.036669+0000","last_became_active":"2023-10-09T11:04:50.310067+0000","last_became_peered":"2023-10-09T11:04:50.310067+0000","last_unstale":"2023-10-09T11:08:38.036669+0000","last_undegraded":"2023-10-09T11:08:38.036669+0000","last_fullsized":"2023-10-09T11:08:38.036669+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:52.884716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":377,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.087965+0000","last_change":"2023-10-09T11:05:20.310581+0000","last_active":"2023-10-09T11:08:38.087965+0000","last_peered":"2023-10-09T11:08:38.087965+0000","last_clean":"2023-10-09T11:08:38.087965+0000","last_became_active":"2023-10-09T11:05:20.309830+0000","last_became_peered":"2023-10-09T11:05:20.309830+0000","last_unstale":"2023-10-09T11:08:38.087965+0000","last_undegraded":"2023-10-09T11:08:38.087965+0000","last_fullsized":"2023-10-09T11:08:38.087965+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:24.508211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.18","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":783,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.040990+0000","last_change":"2023-10-09T11:07:36.282502+0000","last_active":"2023-10-09T11:08:38.040990+0000","last_peered":"2023-10-09T11:08:38.040990+0000","last_clean":"2023-10-09T11:08:38.040990+0000","last_became_active":"2023-10-09T11:07:36.282133+0000","last_became_peered":"2023-10-09T11:07:36.282133+0000","last_unstale":"2023-10-09T11:08:38.040990+0000","last_undegraded":"2023-10-09T11:08:38.040990+0000","last_fullsized":"2023-10-09T11:08:38.040990+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:03.762383+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":204,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.088046+0000","last_change":"2023-10-09T11:06:24.245488+0000","last_active":"2023-10-09T11:08:38.088046+0000","last_peered":"2023-10-09T11:08:38.088046+0000","last_clean":"2023-10-09T11:08:38.088046+0000","last_became_active":"2023-10-09T11:06:24.245167+0000","last_became_peered":"2023-10-09T11:06:24.245167+0000","last_unstale":"2023-10-09T11:08:38.088046+0000","last_undegraded":"2023-10-09T11:08:38.088046+0000","last_fullsized":"2023-10-09T11:08:38.088046+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:31.702568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":103,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.036956+0000","last_change":"2023-10-09T11:05:20.299453+0000","last_active":"2023-10-09T11:08:38.036956+0000","last_peered":"2023-10-09T11:08:38.036956+0000","last_clean":"2023-10-09T11:08:38.036956+0000","last_became_active":"2023-10-09T11:05:20.299117+0000","last_became_peered":"2023-10-09T11:05:20.299117+0000","last_unstale":"2023-10-09T11:08:38.036956+0000","last_undegraded":"2023-10-09T11:08:38.036956+0000","last_fullsized":"2023-10-09T11:08:38.036956+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:30.072813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.19","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":897,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.139929+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:08:37.139929+0000","last_peered":"2023-10-09T11:08:37.139929+0000","last_clean":"2023-10-09T11:08:37.139929+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:08:37.139929+0000","last_undegraded":"2023-10-09T11:08:37.139929+0000","last_fullsized":"2023-10-09T11:08:37.139929+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":439,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.083824+0000","last_change":"2023-10-09T11:07:33.242665+0000","last_active":"2023-10-09T11:08:38.083824+0000","last_peered":"2023-10-09T11:08:38.083824+0000","last_clean":"2023-10-09T11:08:38.083824+0000","last_became_active":"2023-10-09T11:07:33.242313+0000","last_became_peered":"2023-10-09T11:07:33.242313+0000","last_unstale":"2023-10-09T11:08:38.083824+0000","last_undegraded":"2023-10-09T11:08:38.083824+0000","last_fullsized":"2023-10-09T11:08:38.083824+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:20.565135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":121,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.047880+0000","last_change":"2023-10-09T11:07:33.225683+0000","last_active":"2023-10-09T11:08:38.047880+0000","last_peered":"2023-10-09T11:08:38.047880+0000","last_clean":"2023-10-09T11:08:38.047880+0000","last_became_active":"2023-10-09T11:07:33.225275+0000","last_became_peered":"2023-10-09T11:07:33.225275+0000","last_unstale":"2023-10-09T11:08:38.047880+0000","last_undegraded":"2023-10-09T11:08:38.047880+0000","last_fullsized":"2023-10-09T11:08:38.047880+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:10.131060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":797,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.085528+0000","last_change":"2023-10-09T11:05:23.328981+0000","last_active":"2023-10-09T11:08:38.085528+0000","last_peered":"2023-10-09T11:08:38.085528+0000","last_clean":"2023-10-09T11:08:38.085528+0000","last_became_active":"2023-10-09T11:05:23.328616+0000","last_became_peered":"2023-10-09T11:05:23.328616+0000","last_unstale":"2023-10-09T11:08:38.085528+0000","last_undegraded":"2023-10-09T11:08:38.085528+0000","last_fullsized":"2023-10-09T11:08:38.085528+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:15.315819+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":34,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.040925+0000","last_change":"2023-10-09T11:07:33.265842+0000","last_active":"2023-10-09T11:08:38.040925+0000","last_peered":"2023-10-09T11:08:38.040925+0000","last_clean":"2023-10-09T11:08:38.040925+0000","last_became_active":"2023-10-09T11:07:33.232311+0000","last_became_peered":"2023-10-09T11:07:33.232311+0000","last_unstale":"2023-10-09T11:08:38.040925+0000","last_undegraded":"2023-10-09T11:08:38.040925+0000","last_fullsized":"2023-10-09T11:08:38.040925+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:23.402582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":346,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.086095+0000","last_change":"2023-10-09T11:05:20.316265+0000","last_active":"2023-10-09T11:08:38.086095+0000","last_peered":"2023-10-09T11:08:38.086095+0000","last_clean":"2023-10-09T11:08:38.086095+0000","last_became_active":"2023-10-09T11:05:20.315853+0000","last_became_peered":"2023-10-09T11:05:20.315853+0000","last_unstale":"2023-10-09T11:08:38.086095+0000","last_undegraded":"2023-10-09T11:08:38.086095+0000","last_fullsized":"2023-10-09T11:08:38.086095+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:35.093981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":959,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.492121+0000","last_change":"2023-10-09T11:04:53.274921+0000","last_active":"2023-10-09T11:08:37.492121+0000","last_peered":"2023-10-09T11:08:37.492121+0000","last_clean":"2023-10-09T11:08:37.492121+0000","last_became_active":"2023-10-09T11:04:53.274518+0000","last_became_peered":"2023-10-09T11:04:53.274518+0000","last_unstale":"2023-10-09T11:08:37.492121+0000","last_undegraded":"2023-10-09T11:08:37.492121+0000","last_fullsized":"2023-10-09T11:08:37.492121+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:14.315301+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":38,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.041015+0000","last_change":"2023-10-09T11:07:33.262559+0000","last_active":"2023-10-09T11:08:38.041015+0000","last_peered":"2023-10-09T11:08:38.041015+0000","last_clean":"2023-10-09T11:08:38.041015+0000","last_became_active":"2023-10-09T11:07:33.231695+0000","last_became_peered":"2023-10-09T11:07:33.231695+0000","last_unstale":"2023-10-09T11:08:38.041015+0000","last_undegraded":"2023-10-09T11:08:38.041015+0000","last_fullsized":"2023-10-09T11:08:38.041015+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:59.006940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":588,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.139196+0000","last_change":"2023-10-09T11:05:20.311424+0000","last_active":"2023-10-09T11:08:37.139196+0000","last_peered":"2023-10-09T11:08:37.139196+0000","last_clean":"2023-10-09T11:08:37.139196+0000","last_became_active":"2023-10-09T11:05:20.311097+0000","last_became_peered":"2023-10-09T11:05:20.311097+0000","last_unstale":"2023-10-09T11:08:37.139196+0000","last_undegraded":"2023-10-09T11:08:37.139196+0000","last_fullsized":"2023-10-09T11:08:37.139196+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:19.274582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":610,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:39.934994+0000","last_change":"2023-10-09T11:08:39.934938+0000","last_active":"2023-10-09T11:08:39.934994+0000","last_peered":"2023-10-09T11:08:39.934994+0000","last_clean":"2023-10-09T11:08:39.934994+0000","last_became_active":"2023-10-09T11:06:24.245076+0000","last_became_peered":"2023-10-09T11:06:24.245076+0000","last_unstale":"2023-10-09T11:08:39.934994+0000","last_undegraded":"2023-10-09T11:08:39.934994+0000","last_fullsized":"2023-10-09T11:08:39.934994+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.934865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:00.408009+0000","scrub_duration":0.0012831839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":34,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.040780+0000","last_change":"2023-10-09T11:07:33.269200+0000","last_active":"2023-10-09T11:08:38.040780+0000","last_peered":"2023-10-09T11:08:38.040780+0000","last_clean":"2023-10-09T11:08:38.040780+0000","last_became_active":"2023-10-09T11:07:33.236205+0000","last_became_peered":"2023-10-09T11:07:33.236205+0000","last_unstale":"2023-10-09T11:08:38.040780+0000","last_undegraded":"2023-10-09T11:08:38.040780+0000","last_fullsized":"2023-10-09T11:08:38.040780+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:24.574734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":112,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.033582+0000","last_change":"2023-10-09T11:07:33.227803+0000","last_active":"2023-10-09T11:08:38.033582+0000","last_peered":"2023-10-09T11:08:38.033582+0000","last_clean":"2023-10-09T11:08:38.033582+0000","last_became_active":"2023-10-09T11:07:33.227486+0000","last_became_peered":"2023-10-09T11:07:33.227486+0000","last_unstale":"2023-10-09T11:08:38.033582+0000","last_undegraded":"2023-10-09T11:08:38.033582+0000","last_fullsized":"2023-10-09T11:08:38.033582+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:39.274875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":879,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.084642+0000","last_change":"2023-10-09T11:05:23.323856+0000","last_active":"2023-10-09T11:08:38.084642+0000","last_peered":"2023-10-09T11:08:38.084642+0000","last_clean":"2023-10-09T11:08:38.084642+0000","last_became_active":"2023-10-09T11:05:23.322763+0000","last_became_peered":"2023-10-09T11:05:23.322763+0000","last_unstale":"2023-10-09T11:08:38.084642+0000","last_undegraded":"2023-10-09T11:08:38.084642+0000","last_fullsized":"2023-10-09T11:08:38.084642+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.760820+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":28,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.040611+0000","last_change":"2023-10-09T11:07:33.233510+0000","last_active":"2023-10-09T11:08:38.040611+0000","last_peered":"2023-10-09T11:08:38.040611+0000","last_clean":"2023-10-09T11:08:38.040611+0000","last_became_active":"2023-10-09T11:07:33.232339+0000","last_became_peered":"2023-10-09T11:07:33.232339+0000","last_unstale":"2023-10-09T11:08:38.040611+0000","last_undegraded":"2023-10-09T11:08:38.040611+0000","last_fullsized":"2023-10-09T11:08:38.040611+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:55.324912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":509,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.086015+0000","last_change":"2023-10-09T11:07:33.226594+0000","last_active":"2023-10-09T11:08:38.086015+0000","last_peered":"2023-10-09T11:08:38.086015+0000","last_clean":"2023-10-09T11:08:38.086015+0000","last_became_active":"2023-10-09T11:07:33.226275+0000","last_became_peered":"2023-10-09T11:07:33.226275+0000","last_unstale":"2023-10-09T11:08:38.086015+0000","last_undegraded":"2023-10-09T11:08:38.086015+0000","last_fullsized":"2023-10-09T11:08:38.086015+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:41.180088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":900,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.492121+0000","last_change":"2023-10-09T11:04:53.274236+0000","last_active":"2023-10-09T11:08:37.492121+0000","last_peered":"2023-10-09T11:08:37.492121+0000","last_clean":"2023-10-09T11:08:37.492121+0000","last_became_active":"2023-10-09T11:04:53.273817+0000","last_became_peered":"2023-10-09T11:04:53.273817+0000","last_unstale":"2023-10-09T11:08:37.492121+0000","last_undegraded":"2023-10-09T11:08:37.492121+0000","last_fullsized":"2023-10-09T11:08:37.492121+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:05.096905+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":163,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.040676+0000","last_change":"2023-10-09T11:05:20.312129+0000","last_active":"2023-10-09T11:08:38.040676+0000","last_peered":"2023-10-09T11:08:38.040676+0000","last_clean":"2023-10-09T11:08:38.040676+0000","last_became_active":"2023-10-09T11:05:20.311810+0000","last_became_peered":"2023-10-09T11:05:20.311810+0000","last_unstale":"2023-10-09T11:08:38.040676+0000","last_undegraded":"2023-10-09T11:08:38.040676+0000","last_fullsized":"2023-10-09T11:08:38.040676+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:48.813391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":261,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.034191+0000","last_change":"2023-10-09T11:07:33.218244+0000","last_active":"2023-10-09T11:08:38.034191+0000","last_peered":"2023-10-09T11:08:38.034191+0000","last_clean":"2023-10-09T11:08:38.034191+0000","last_became_active":"2023-10-09T11:07:33.217918+0000","last_became_peered":"2023-10-09T11:07:33.217918+0000","last_unstale":"2023-10-09T11:08:38.034191+0000","last_undegraded":"2023-10-09T11:08:38.034191+0000","last_fullsized":"2023-10-09T11:08:38.034191+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:16.150105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":876,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.087458+0000","last_change":"2023-10-09T11:05:23.324537+0000","last_active":"2023-10-09T11:08:38.087458+0000","last_peered":"2023-10-09T11:08:38.087458+0000","last_clean":"2023-10-09T11:08:38.087458+0000","last_became_active":"2023-10-09T11:05:23.324187+0000","last_became_peered":"2023-10-09T11:05:23.324187+0000","last_unstale":"2023-10-09T11:08:38.087458+0000","last_undegraded":"2023-10-09T11:08:38.087458+0000","last_fullsized":"2023-10-09T11:08:38.087458+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:25.286106+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":88,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.861159+0000","last_change":"2023-10-09T11:05:20.323565+0000","last_active":"2023-10-09T11:08:38.861159+0000","last_peered":"2023-10-09T11:08:38.861159+0000","last_clean":"2023-10-09T11:08:38.861159+0000","last_became_active":"2023-10-09T11:05:20.309672+0000","last_became_peered":"2023-10-09T11:05:20.309672+0000","last_unstale":"2023-10-09T11:08:38.861159+0000","last_undegraded":"2023-10-09T11:08:38.861159+0000","last_fullsized":"2023-10-09T11:08:38.861159+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:50.211465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":112,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.033889+0000","last_change":"2023-10-09T11:07:33.228001+0000","last_active":"2023-10-09T11:08:38.033889+0000","last_peered":"2023-10-09T11:08:38.033889+0000","last_clean":"2023-10-09T11:08:38.033889+0000","last_became_active":"2023-10-09T11:07:33.227691+0000","last_became_peered":"2023-10-09T11:07:33.227691+0000","last_unstale":"2023-10-09T11:08:38.033889+0000","last_undegraded":"2023-10-09T11:08:38.033889+0000","last_fullsized":"2023-10-09T11:08:38.033889+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:24:16.098138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":800,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.041118+0000","last_change":"2023-10-09T11:07:36.284927+0000","last_active":"2023-10-09T11:08:38.041118+0000","last_peered":"2023-10-09T11:08:38.041118+0000","last_clean":"2023-10-09T11:08:38.041118+0000","last_became_active":"2023-10-09T11:07:36.284631+0000","last_became_peered":"2023-10-09T11:07:36.284631+0000","last_unstale":"2023-10-09T11:08:38.041118+0000","last_undegraded":"2023-10-09T11:08:38.041118+0000","last_fullsized":"2023-10-09T11:08:38.041118+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:37.220243+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":32,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.041704+0000","last_change":"2023-10-09T11:07:33.273503+0000","last_active":"2023-10-09T11:08:38.041704+0000","last_peered":"2023-10-09T11:08:38.041704+0000","last_clean":"2023-10-09T11:08:38.041704+0000","last_became_active":"2023-10-09T11:07:33.242980+0000","last_became_peered":"2023-10-09T11:07:33.242980+0000","last_unstale":"2023-10-09T11:08:38.041704+0000","last_undegraded":"2023-10-09T11:08:38.041704+0000","last_fullsized":"2023-10-09T11:08:38.041704+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:18.934972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":530,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.083861+0000","last_change":"2023-10-09T11:07:33.304363+0000","last_active":"2023-10-09T11:08:38.083861+0000","last_peered":"2023-10-09T11:08:38.083861+0000","last_clean":"2023-10-09T11:08:38.083861+0000","last_became_active":"2023-10-09T11:07:33.304024+0000","last_became_peered":"2023-10-09T11:07:33.304024+0000","last_unstale":"2023-10-09T11:08:38.083861+0000","last_undegraded":"2023-10-09T11:08:38.083861+0000","last_fullsized":"2023-10-09T11:08:38.083861+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:21:51.522688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":949,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.033651+0000","last_change":"2023-10-09T11:05:23.316129+0000","last_active":"2023-10-09T11:08:38.033651+0000","last_peered":"2023-10-09T11:08:38.033651+0000","last_clean":"2023-10-09T11:08:38.033651+0000","last_became_active":"2023-10-09T11:05:23.315792+0000","last_became_peered":"2023-10-09T11:05:23.315792+0000","last_unstale":"2023-10-09T11:08:38.033651+0000","last_undegraded":"2023-10-09T11:08:38.033651+0000","last_fullsized":"2023-10-09T11:08:38.033651+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:51.770547+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":32,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.041899+0000","last_change":"2023-10-09T11:07:33.256691+0000","last_active":"2023-10-09T11:08:38.041899+0000","last_peered":"2023-10-09T11:08:38.041899+0000","last_clean":"2023-10-09T11:08:38.041899+0000","last_became_active":"2023-10-09T11:07:33.229554+0000","last_became_peered":"2023-10-09T11:07:33.229554+0000","last_unstale":"2023-10-09T11:08:38.041899+0000","last_undegraded":"2023-10-09T11:08:38.041899+0000","last_fullsized":"2023-10-09T11:08:38.041899+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:57.414795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":281,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.492317+0000","last_change":"2023-10-09T11:04:50.333538+0000","last_active":"2023-10-09T11:08:37.492317+0000","last_peered":"2023-10-09T11:08:37.492317+0000","last_clean":"2023-10-09T11:08:37.492317+0000","last_became_active":"2023-10-09T11:04:50.332514+0000","last_became_peered":"2023-10-09T11:04:50.332514+0000","last_unstale":"2023-10-09T11:08:37.492317+0000","last_undegraded":"2023-10-09T11:08:37.492317+0000","last_fullsized":"2023-10-09T11:08:37.492317+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:22.933839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":824,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.085617+0000","last_change":"2023-10-09T11:05:23.322251+0000","last_active":"2023-10-09T11:08:38.085617+0000","last_peered":"2023-10-09T11:08:38.085617+0000","last_clean":"2023-10-09T11:08:38.085617+0000","last_became_active":"2023-10-09T11:05:23.321826+0000","last_became_peered":"2023-10-09T11:05:23.321826+0000","last_unstale":"2023-10-09T11:08:38.085617+0000","last_undegraded":"2023-10-09T11:08:38.085617+0000","last_fullsized":"2023-10-09T11:08:38.085617+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.858013+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":32,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.041299+0000","last_change":"2023-10-09T11:07:33.271238+0000","last_active":"2023-10-09T11:08:38.041299+0000","last_peered":"2023-10-09T11:08:38.041299+0000","last_clean":"2023-10-09T11:08:38.041299+0000","last_became_active":"2023-10-09T11:07:33.241886+0000","last_became_peered":"2023-10-09T11:07:33.241886+0000","last_unstale":"2023-10-09T11:08:38.041299+0000","last_undegraded":"2023-10-09T11:08:38.041299+0000","last_fullsized":"2023-10-09T11:08:38.041299+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:48.882589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":98,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.859740+0000","last_change":"2023-10-09T11:07:33.244131+0000","last_active":"2023-10-09T11:08:38.859740+0000","last_peered":"2023-10-09T11:08:38.859740+0000","last_clean":"2023-10-09T11:08:38.859740+0000","last_became_active":"2023-10-09T11:07:33.243797+0000","last_became_peered":"2023-10-09T11:07:33.243797+0000","last_unstale":"2023-10-09T11:08:38.859740+0000","last_undegraded":"2023-10-09T11:08:38.859740+0000","last_fullsized":"2023-10-09T11:08:38.859740+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:44.851043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":1078,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.042019+0000","last_change":"2023-10-09T11:07:36.286417+0000","last_active":"2023-10-09T11:08:38.042019+0000","last_peered":"2023-10-09T11:08:38.042019+0000","last_clean":"2023-10-09T11:08:38.042019+0000","last_became_active":"2023-10-09T11:07:36.286061+0000","last_became_peered":"2023-10-09T11:07:36.286061+0000","last_unstale":"2023-10-09T11:08:38.042019+0000","last_undegraded":"2023-10-09T11:08:38.042019+0000","last_fullsized":"2023-10-09T11:08:38.042019+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:00.289500+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":426,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.083681+0000","last_change":"2023-10-09T11:05:20.309385+0000","last_active":"2023-10-09T11:08:38.083681+0000","last_peered":"2023-10-09T11:08:38.083681+0000","last_clean":"2023-10-09T11:08:38.083681+0000","last_became_active":"2023-10-09T11:05:20.309034+0000","last_became_peered":"2023-10-09T11:05:20.309034+0000","last_unstale":"2023-10-09T11:08:38.083681+0000","last_undegraded":"2023-10-09T11:08:38.083681+0000","last_fullsized":"2023-10-09T11:08:38.083681+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:31.857485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":357,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.066554+0000","last_change":"2023-10-09T11:06:24.245966+0000","last_active":"2023-10-09T11:08:38.066554+0000","last_peered":"2023-10-09T11:08:38.066554+0000","last_clean":"2023-10-09T11:08:38.066554+0000","last_became_active":"2023-10-09T11:06:24.245310+0000","last_became_peered":"2023-10-09T11:06:24.245310+0000","last_unstale":"2023-10-09T11:08:38.066554+0000","last_undegraded":"2023-10-09T11:08:38.066554+0000","last_fullsized":"2023-10-09T11:08:38.066554+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:23.966200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":722,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.040584+0000","last_change":"2023-10-09T11:07:36.283174+0000","last_active":"2023-10-09T11:08:38.040584+0000","last_peered":"2023-10-09T11:08:38.040584+0000","last_clean":"2023-10-09T11:08:38.040584+0000","last_became_active":"2023-10-09T11:07:36.278007+0000","last_became_peered":"2023-10-09T11:07:36.278007+0000","last_unstale":"2023-10-09T11:08:38.040584+0000","last_undegraded":"2023-10-09T11:08:38.040584+0000","last_fullsized":"2023-10-09T11:08:38.040584+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:51.693969+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":98,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.035825+0000","last_change":"2023-10-09T11:04:50.529513+0000","last_active":"2023-10-09T11:08:38.035825+0000","last_peered":"2023-10-09T11:08:38.035825+0000","last_clean":"2023-10-09T11:08:38.035825+0000","last_became_active":"2023-10-09T11:04:50.339671+0000","last_became_peered":"2023-10-09T11:04:50.339671+0000","last_unstale":"2023-10-09T11:08:38.035825+0000","last_undegraded":"2023-10-09T11:08:38.035825+0000","last_fullsized":"2023-10-09T11:08:38.035825+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:24.473000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":103,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.033521+0000","last_change":"2023-10-09T11:05:20.313855+0000","last_active":"2023-10-09T11:08:38.033521+0000","last_peered":"2023-10-09T11:08:38.033521+0000","last_clean":"2023-10-09T11:08:38.033521+0000","last_became_active":"2023-10-09T11:05:20.313535+0000","last_became_peered":"2023-10-09T11:05:20.313535+0000","last_unstale":"2023-10-09T11:08:38.033521+0000","last_undegraded":"2023-10-09T11:08:38.033521+0000","last_fullsized":"2023-10-09T11:08:38.033521+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:25.266434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":826,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.084560+0000","last_change":"2023-10-09T11:04:53.272323+0000","last_active":"2023-10-09T11:08:38.084560+0000","last_peered":"2023-10-09T11:08:38.084560+0000","last_clean":"2023-10-09T11:08:38.084560+0000","last_became_active":"2023-10-09T11:04:53.272010+0000","last_became_peered":"2023-10-09T11:04:53.272010+0000","last_unstale":"2023-10-09T11:08:38.084560+0000","last_undegraded":"2023-10-09T11:08:38.084560+0000","last_fullsized":"2023-10-09T11:08:38.084560+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:23.913605+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":590,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.492510+0000","last_change":"2023-10-09T11:05:20.307499+0000","last_active":"2023-10-09T11:08:37.492510+0000","last_peered":"2023-10-09T11:08:37.492510+0000","last_clean":"2023-10-09T11:08:37.492510+0000","last_became_active":"2023-10-09T11:05:20.307198+0000","last_became_peered":"2023-10-09T11:05:20.307198+0000","last_unstale":"2023-10-09T11:08:37.492510+0000","last_undegraded":"2023-10-09T11:08:37.492510+0000","last_fullsized":"2023-10-09T11:08:37.492510+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:53.726844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":326,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.141426+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:08:37.141426+0000","last_peered":"2023-10-09T11:08:37.141426+0000","last_clean":"2023-10-09T11:08:37.141426+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:08:37.141426+0000","last_undegraded":"2023-10-09T11:08:37.141426+0000","last_fullsized":"2023-10-09T11:08:37.141426+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":917,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.492948+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:08:37.492948+0000","last_peered":"2023-10-09T11:08:37.492948+0000","last_clean":"2023-10-09T11:08:37.492948+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:08:37.492948+0000","last_undegraded":"2023-10-09T11:08:37.492948+0000","last_fullsized":"2023-10-09T11:08:37.492948+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":589,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.139198+0000","last_change":"2023-10-09T11:04:57.953307+0000","last_active":"2023-10-09T11:08:37.139198+0000","last_peered":"2023-10-09T11:08:37.139198+0000","last_clean":"2023-10-09T11:08:37.139198+0000","last_became_active":"2023-10-09T11:04:57.953095+0000","last_became_peered":"2023-10-09T11:04:57.953095+0000","last_unstale":"2023-10-09T11:08:37.139198+0000","last_undegraded":"2023-10-09T11:08:37.139198+0000","last_fullsized":"2023-10-09T11:08:37.139198+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:05.562643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":275,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.493164+0000","last_change":"2023-10-09T11:05:20.310255+0000","last_active":"2023-10-09T11:08:37.493164+0000","last_peered":"2023-10-09T11:08:37.493164+0000","last_clean":"2023-10-09T11:08:37.493164+0000","last_became_active":"2023-10-09T11:05:20.309109+0000","last_became_peered":"2023-10-09T11:05:20.309109+0000","last_unstale":"2023-10-09T11:08:37.493164+0000","last_undegraded":"2023-10-09T11:08:37.493164+0000","last_fullsized":"2023-10-09T11:08:37.493164+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:07.429039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":810,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.087381+0000","last_change":"2023-10-09T11:05:23.329329+0000","last_active":"2023-10-09T11:08:38.087381+0000","last_peered":"2023-10-09T11:08:38.087381+0000","last_clean":"2023-10-09T11:08:38.087381+0000","last_became_active":"2023-10-09T11:05:23.328976+0000","last_became_peered":"2023-10-09T11:05:23.328976+0000","last_unstale":"2023-10-09T11:08:38.087381+0000","last_undegraded":"2023-10-09T11:08:38.087381+0000","last_fullsized":"2023-10-09T11:08:38.087381+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:41.848184+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":112,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.037525+0000","last_change":"2023-10-09T11:07:33.236133+0000","last_active":"2023-10-09T11:08:38.037525+0000","last_peered":"2023-10-09T11:08:38.037525+0000","last_clean":"2023-10-09T11:08:38.037525+0000","last_became_active":"2023-10-09T11:07:33.235818+0000","last_became_peered":"2023-10-09T11:07:33.235818+0000","last_unstale":"2023-10-09T11:08:38.037525+0000","last_undegraded":"2023-10-09T11:08:38.037525+0000","last_fullsized":"2023-10-09T11:08:38.037525+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:20.627846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":272,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.493601+0000","last_change":"2023-10-09T11:04:50.332978+0000","last_active":"2023-10-09T11:08:37.493601+0000","last_peered":"2023-10-09T11:08:37.493601+0000","last_clean":"2023-10-09T11:08:37.493601+0000","last_became_active":"2023-10-09T11:04:50.332735+0000","last_became_peered":"2023-10-09T11:04:50.332735+0000","last_unstale":"2023-10-09T11:08:37.493601+0000","last_undegraded":"2023-10-09T11:08:37.493601+0000","last_fullsized":"2023-10-09T11:08:37.493601+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:39:44.659751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":1094,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.033957+0000","last_change":"2023-10-09T11:07:36.291716+0000","last_active":"2023-10-09T11:08:38.033957+0000","last_peered":"2023-10-09T11:08:38.033957+0000","last_clean":"2023-10-09T11:08:38.033957+0000","last_became_active":"2023-10-09T11:07:36.291424+0000","last_became_peered":"2023-10-09T11:07:36.291424+0000","last_unstale":"2023-10-09T11:08:38.033957+0000","last_undegraded":"2023-10-09T11:08:38.033957+0000","last_fullsized":"2023-10-09T11:08:38.033957+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:22.316387+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":297,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.491993+0000","last_change":"2023-10-09T11:07:33.226547+0000","last_active":"2023-10-09T11:08:37.491993+0000","last_peered":"2023-10-09T11:08:37.491993+0000","last_clean":"2023-10-09T11:08:37.491993+0000","last_became_active":"2023-10-09T11:07:33.226176+0000","last_became_peered":"2023-10-09T11:07:33.226176+0000","last_unstale":"2023-10-09T11:08:37.491993+0000","last_undegraded":"2023-10-09T11:08:37.491993+0000","last_fullsized":"2023-10-09T11:08:37.491993+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:16.888732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":443,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.085885+0000","last_change":"2023-10-09T11:04:50.341353+0000","last_active":"2023-10-09T11:08:38.085885+0000","last_peered":"2023-10-09T11:08:38.085885+0000","last_clean":"2023-10-09T11:08:38.085885+0000","last_became_active":"2023-10-09T11:04:50.340801+0000","last_became_peered":"2023-10-09T11:04:50.340801+0000","last_unstale":"2023-10-09T11:08:38.085885+0000","last_undegraded":"2023-10-09T11:08:38.085885+0000","last_fullsized":"2023-10-09T11:08:38.085885+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:00.699069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":249,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.492651+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:08:37.492651+0000","last_peered":"2023-10-09T11:08:37.492651+0000","last_clean":"2023-10-09T11:08:37.492651+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:08:37.492651+0000","last_undegraded":"2023-10-09T11:08:37.492651+0000","last_fullsized":"2023-10-09T11:08:37.492651+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":768,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.084006+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:08:38.084006+0000","last_peered":"2023-10-09T11:08:38.084006+0000","last_clean":"2023-10-09T11:08:38.084006+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:08:38.084006+0000","last_undegraded":"2023-10-09T11:08:38.084006+0000","last_fullsized":"2023-10-09T11:08:38.084006+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":450,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.086680+0000","last_change":"2023-10-09T11:05:20.316093+0000","last_active":"2023-10-09T11:08:38.086680+0000","last_peered":"2023-10-09T11:08:38.086680+0000","last_clean":"2023-10-09T11:08:38.086680+0000","last_became_active":"2023-10-09T11:05:20.315471+0000","last_became_peered":"2023-10-09T11:05:20.315471+0000","last_unstale":"2023-10-09T11:08:38.086680+0000","last_undegraded":"2023-10-09T11:08:38.086680+0000","last_fullsized":"2023-10-09T11:08:38.086680+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:47.548763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":474,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.860162+0000","last_change":"2023-10-09T11:06:24.245282+0000","last_active":"2023-10-09T11:08:38.860162+0000","last_peered":"2023-10-09T11:08:38.860162+0000","last_clean":"2023-10-09T11:08:38.860162+0000","last_became_active":"2023-10-09T11:06:24.244981+0000","last_became_peered":"2023-10-09T11:06:24.244981+0000","last_unstale":"2023-10-09T11:08:38.860162+0000","last_undegraded":"2023-10-09T11:08:38.860162+0000","last_fullsized":"2023-10-09T11:08:38.860162+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:15.327655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":282,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.492645+0000","last_change":"2023-10-09T11:04:50.311579+0000","last_active":"2023-10-09T11:08:37.492645+0000","last_peered":"2023-10-09T11:08:37.492645+0000","last_clean":"2023-10-09T11:08:37.492645+0000","last_became_active":"2023-10-09T11:04:50.311270+0000","last_became_peered":"2023-10-09T11:04:50.311270+0000","last_unstale":"2023-10-09T11:08:37.492645+0000","last_undegraded":"2023-10-09T11:08:37.492645+0000","last_fullsized":"2023-10-09T11:08:37.492645+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:49.850302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":920,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.040889+0000","last_change":"2023-10-09T11:07:36.284280+0000","last_active":"2023-10-09T11:08:38.040889+0000","last_peered":"2023-10-09T11:08:38.040889+0000","last_clean":"2023-10-09T11:08:38.040889+0000","last_became_active":"2023-10-09T11:07:36.283966+0000","last_became_peered":"2023-10-09T11:07:36.283966+0000","last_unstale":"2023-10-09T11:08:38.040889+0000","last_undegraded":"2023-10-09T11:08:38.040889+0000","last_fullsized":"2023-10-09T11:08:38.040889+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:54.128331+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":345,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.139399+0000","last_change":"2023-10-09T11:04:50.381744+0000","last_active":"2023-10-09T11:08:37.139399+0000","last_peered":"2023-10-09T11:08:37.139399+0000","last_clean":"2023-10-09T11:08:37.139399+0000","last_became_active":"2023-10-09T11:04:50.381253+0000","last_became_peered":"2023-10-09T11:04:50.381253+0000","last_unstale":"2023-10-09T11:08:37.139399+0000","last_undegraded":"2023-10-09T11:08:37.139399+0000","last_fullsized":"2023-10-09T11:08:37.139399+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:24.030274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":267,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.492510+0000","last_change":"2023-10-09T11:05:20.298542+0000","last_active":"2023-10-09T11:08:37.492510+0000","last_peered":"2023-10-09T11:08:37.492510+0000","last_clean":"2023-10-09T11:08:37.492510+0000","last_became_active":"2023-10-09T11:05:20.298119+0000","last_became_peered":"2023-10-09T11:05:20.298119+0000","last_unstale":"2023-10-09T11:08:37.492510+0000","last_undegraded":"2023-10-09T11:08:37.492510+0000","last_fullsized":"2023-10-09T11:08:37.492510+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:00.827214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":246,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.493709+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:08:37.493709+0000","last_peered":"2023-10-09T11:08:37.493709+0000","last_clean":"2023-10-09T11:08:37.493709+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:08:37.493709+0000","last_undegraded":"2023-10-09T11:08:37.493709+0000","last_fullsized":"2023-10-09T11:08:37.493709+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":443,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.085117+0000","last_change":"2023-10-09T11:07:33.227052+0000","last_active":"2023-10-09T11:08:38.085117+0000","last_peered":"2023-10-09T11:08:38.085117+0000","last_clean":"2023-10-09T11:08:38.085117+0000","last_became_active":"2023-10-09T11:07:33.226289+0000","last_became_peered":"2023-10-09T11:07:33.226289+0000","last_unstale":"2023-10-09T11:08:38.085117+0000","last_undegraded":"2023-10-09T11:08:38.085117+0000","last_fullsized":"2023-10-09T11:08:38.085117+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:15.605946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":865,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.040640+0000","last_change":"2023-10-09T11:07:36.290700+0000","last_active":"2023-10-09T11:08:38.040640+0000","last_peered":"2023-10-09T11:08:38.040640+0000","last_clean":"2023-10-09T11:08:38.040640+0000","last_became_active":"2023-10-09T11:07:36.290357+0000","last_became_peered":"2023-10-09T11:07:36.290357+0000","last_unstale":"2023-10-09T11:08:38.040640+0000","last_undegraded":"2023-10-09T11:08:38.040640+0000","last_fullsized":"2023-10-09T11:08:38.040640+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:34.494777+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":103,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.042003+0000","last_change":"2023-10-09T11:05:20.310666+0000","last_active":"2023-10-09T11:08:38.042003+0000","last_peered":"2023-10-09T11:08:38.042003+0000","last_clean":"2023-10-09T11:08:38.042003+0000","last_became_active":"2023-10-09T11:05:20.310413+0000","last_became_peered":"2023-10-09T11:05:20.310413+0000","last_unstale":"2023-10-09T11:08:38.042003+0000","last_undegraded":"2023-10-09T11:08:38.042003+0000","last_fullsized":"2023-10-09T11:08:38.042003+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:34.169905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":528,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.041201+0000","last_change":"2023-10-09T11:05:20.316523+0000","last_active":"2023-10-09T11:08:38.041201+0000","last_peered":"2023-10-09T11:08:38.041201+0000","last_clean":"2023-10-09T11:08:38.041201+0000","last_became_active":"2023-10-09T11:05:20.314493+0000","last_became_peered":"2023-10-09T11:05:20.314493+0000","last_unstale":"2023-10-09T11:08:38.041201+0000","last_undegraded":"2023-10-09T11:08:38.041201+0000","last_fullsized":"2023-10-09T11:08:38.041201+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:41.618701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":938,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.492843+0000","last_change":"2023-10-09T11:05:23.322074+0000","last_active":"2023-10-09T11:08:37.492843+0000","last_peered":"2023-10-09T11:08:37.492843+0000","last_clean":"2023-10-09T11:08:37.492843+0000","last_became_active":"2023-10-09T11:05:23.321749+0000","last_became_peered":"2023-10-09T11:05:23.321749+0000","last_unstale":"2023-10-09T11:08:37.492843+0000","last_undegraded":"2023-10-09T11:08:37.492843+0000","last_fullsized":"2023-10-09T11:08:37.492843+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:07.010932+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":143,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.041691+0000","last_change":"2023-10-09T11:02:33.198484+0000","last_active":"2023-10-09T11:08:38.041691+0000","last_peered":"2023-10-09T11:08:38.041691+0000","last_clean":"2023-10-09T11:08:38.041691+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:08:38.041691+0000","last_undegraded":"2023-10-09T11:08:38.041691+0000","last_fullsized":"2023-10-09T11:08:38.041691+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":195,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.046679+0000","last_change":"2023-10-09T11:06:24.248362+0000","last_active":"2023-10-09T11:08:38.046679+0000","last_peered":"2023-10-09T11:08:38.046679+0000","last_clean":"2023-10-09T11:08:38.046679+0000","last_became_active":"2023-10-09T11:06:24.247348+0000","last_became_peered":"2023-10-09T11:06:24.247348+0000","last_unstale":"2023-10-09T11:08:38.046679+0000","last_undegraded":"2023-10-09T11:08:38.046679+0000","last_fullsized":"2023-10-09T11:08:38.046679+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:26.728754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":393,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.070075+0000","last_change":"2023-10-09T11:06:24.253511+0000","last_active":"2023-10-09T11:08:38.070075+0000","last_peered":"2023-10-09T11:08:38.070075+0000","last_clean":"2023-10-09T11:08:38.070075+0000","last_became_active":"2023-10-09T11:06:24.252739+0000","last_became_peered":"2023-10-09T11:06:24.252739+0000","last_unstale":"2023-10-09T11:08:38.070075+0000","last_undegraded":"2023-10-09T11:08:38.070075+0000","last_fullsized":"2023-10-09T11:08:38.070075+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:50.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":799,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.345624+0000","last_change":"2023-10-09T11:06:24.239676+0000","last_active":"2023-10-09T11:08:38.345624+0000","last_peered":"2023-10-09T11:08:38.345624+0000","last_clean":"2023-10-09T11:08:38.345624+0000","last_became_active":"2023-10-09T11:06:24.239375+0000","last_became_peered":"2023-10-09T11:06:24.239375+0000","last_unstale":"2023-10-09T11:08:38.345624+0000","last_undegraded":"2023-10-09T11:08:38.345624+0000","last_fullsized":"2023-10-09T11:08:38.345624+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:27.953053+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.040899+0000","last_change":"2023-10-09T11:07:33.229415+0000","last_active":"2023-10-09T11:08:38.040899+0000","last_peered":"2023-10-09T11:08:38.040899+0000","last_clean":"2023-10-09T11:08:38.040899+0000","last_became_active":"2023-10-09T11:07:33.229061+0000","last_became_peered":"2023-10-09T11:07:33.229061+0000","last_unstale":"2023-10-09T11:08:38.040899+0000","last_undegraded":"2023-10-09T11:08:38.040899+0000","last_fullsized":"2023-10-09T11:08:38.040899+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:30.978293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":249,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.493013+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:08:37.493013+0000","last_peered":"2023-10-09T11:08:37.493013+0000","last_clean":"2023-10-09T11:08:37.493013+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:08:37.493013+0000","last_undegraded":"2023-10-09T11:08:37.493013+0000","last_fullsized":"2023-10-09T11:08:37.493013+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":527,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.085047+0000","last_change":"2023-10-09T11:07:33.227271+0000","last_active":"2023-10-09T11:08:38.085047+0000","last_peered":"2023-10-09T11:08:38.085047+0000","last_clean":"2023-10-09T11:08:38.085047+0000","last_became_active":"2023-10-09T11:07:33.226760+0000","last_became_peered":"2023-10-09T11:07:33.226760+0000","last_unstale":"2023-10-09T11:08:38.085047+0000","last_undegraded":"2023-10-09T11:08:38.085047+0000","last_fullsized":"2023-10-09T11:08:38.085047+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:12.268123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":766,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.139932+0000","last_change":"2023-10-09T11:05:23.323368+0000","last_active":"2023-10-09T11:08:37.139932+0000","last_peered":"2023-10-09T11:08:37.139932+0000","last_clean":"2023-10-09T11:08:37.139932+0000","last_became_active":"2023-10-09T11:05:23.323032+0000","last_became_peered":"2023-10-09T11:05:23.323032+0000","last_unstale":"2023-10-09T11:08:37.139932+0000","last_undegraded":"2023-10-09T11:08:37.139932+0000","last_fullsized":"2023-10-09T11:08:37.139932+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:09.393584+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":124,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.033827+0000","last_change":"2023-10-09T11:07:33.246378+0000","last_active":"2023-10-09T11:08:38.033827+0000","last_peered":"2023-10-09T11:08:38.033827+0000","last_clean":"2023-10-09T11:08:38.033827+0000","last_became_active":"2023-10-09T11:07:33.228019+0000","last_became_peered":"2023-10-09T11:07:33.228019+0000","last_unstale":"2023-10-09T11:08:38.033827+0000","last_undegraded":"2023-10-09T11:08:38.033827+0000","last_fullsized":"2023-10-09T11:08:38.033827+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:18.511517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":275,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.493268+0000","last_change":"2023-10-09T11:05:20.310417+0000","last_active":"2023-10-09T11:08:37.493268+0000","last_peered":"2023-10-09T11:08:37.493268+0000","last_clean":"2023-10-09T11:08:37.493268+0000","last_became_active":"2023-10-09T11:05:20.309827+0000","last_became_peered":"2023-10-09T11:05:20.309827+0000","last_unstale":"2023-10-09T11:08:37.493268+0000","last_undegraded":"2023-10-09T11:08:37.493268+0000","last_fullsized":"2023-10-09T11:08:37.493268+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:23.934383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":112,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.037023+0000","last_change":"2023-10-09T11:07:33.217595+0000","last_active":"2023-10-09T11:08:38.037023+0000","last_peered":"2023-10-09T11:08:38.037023+0000","last_clean":"2023-10-09T11:08:38.037023+0000","last_became_active":"2023-10-09T11:07:33.217155+0000","last_became_peered":"2023-10-09T11:07:33.217155+0000","last_unstale":"2023-10-09T11:08:38.037023+0000","last_undegraded":"2023-10-09T11:08:38.037023+0000","last_fullsized":"2023-10-09T11:08:38.037023+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:55.127329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":731,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.033452+0000","last_change":"2023-10-09T11:07:36.284491+0000","last_active":"2023-10-09T11:08:38.033452+0000","last_peered":"2023-10-09T11:08:38.033452+0000","last_clean":"2023-10-09T11:08:38.033452+0000","last_became_active":"2023-10-09T11:07:36.284249+0000","last_became_peered":"2023-10-09T11:07:36.284249+0000","last_unstale":"2023-10-09T11:08:38.033452+0000","last_undegraded":"2023-10-09T11:08:38.033452+0000","last_fullsized":"2023-10-09T11:08:38.033452+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:03.653371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:03.653371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:09.629417+0000","scrub_duration":0.001576771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.041837+0000","last_change":"2023-10-09T11:07:33.241346+0000","last_active":"2023-10-09T11:08:38.041837+0000","last_peered":"2023-10-09T11:08:38.041837+0000","last_clean":"2023-10-09T11:08:38.041837+0000","last_became_active":"2023-10-09T11:07:33.241013+0000","last_became_peered":"2023-10-09T11:07:33.241013+0000","last_unstale":"2023-10-09T11:08:38.041837+0000","last_undegraded":"2023-10-09T11:08:38.041837+0000","last_fullsized":"2023-10-09T11:08:38.041837+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:57.164502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":266,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.077199+0000","last_change":"2023-10-09T11:06:24.253714+0000","last_active":"2023-10-09T11:08:38.077199+0000","last_peered":"2023-10-09T11:08:38.077199+0000","last_clean":"2023-10-09T11:08:38.077199+0000","last_became_active":"2023-10-09T11:06:24.253143+0000","last_became_peered":"2023-10-09T11:06:24.253143+0000","last_unstale":"2023-10-09T11:08:38.077199+0000","last_undegraded":"2023-10-09T11:08:38.077199+0000","last_fullsized":"2023-10-09T11:08:38.077199+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:05.682288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":373,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.083922+0000","last_change":"2023-10-09T11:05:20.313560+0000","last_active":"2023-10-09T11:08:38.083922+0000","last_peered":"2023-10-09T11:08:38.083922+0000","last_clean":"2023-10-09T11:08:38.083922+0000","last_became_active":"2023-10-09T11:05:20.313214+0000","last_became_peered":"2023-10-09T11:05:20.313214+0000","last_unstale":"2023-10-09T11:08:38.083922+0000","last_undegraded":"2023-10-09T11:08:38.083922+0000","last_fullsized":"2023-10-09T11:08:38.083922+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:32.527205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":975,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.047972+0000","last_change":"2023-10-09T11:05:23.329886+0000","last_active":"2023-10-09T11:08:38.047972+0000","last_peered":"2023-10-09T11:08:38.047972+0000","last_clean":"2023-10-09T11:08:38.047972+0000","last_became_active":"2023-10-09T11:05:23.329561+0000","last_became_peered":"2023-10-09T11:05:23.329561+0000","last_unstale":"2023-10-09T11:08:38.047972+0000","last_undegraded":"2023-10-09T11:08:38.047972+0000","last_fullsized":"2023-10-09T11:08:38.047972+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:46.933799+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":28,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.041499+0000","last_change":"2023-10-09T11:07:33.229864+0000","last_active":"2023-10-09T11:08:38.041499+0000","last_peered":"2023-10-09T11:08:38.041499+0000","last_clean":"2023-10-09T11:08:38.041499+0000","last_became_active":"2023-10-09T11:07:33.229547+0000","last_became_peered":"2023-10-09T11:07:33.229547+0000","last_unstale":"2023-10-09T11:08:38.041499+0000","last_undegraded":"2023-10-09T11:08:38.041499+0000","last_fullsized":"2023-10-09T11:08:38.041499+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:34.990439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":349,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.140038+0000","last_change":"2023-10-09T11:04:50.510688+0000","last_active":"2023-10-09T11:08:37.140038+0000","last_peered":"2023-10-09T11:08:37.140038+0000","last_clean":"2023-10-09T11:08:37.140038+0000","last_became_active":"2023-10-09T11:04:50.507404+0000","last_became_peered":"2023-10-09T11:04:50.507404+0000","last_unstale":"2023-10-09T11:08:37.140038+0000","last_undegraded":"2023-10-09T11:08:37.140038+0000","last_fullsized":"2023-10-09T11:08:37.140038+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:13.663394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":932,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.034328+0000","last_change":"2023-10-09T11:05:23.321040+0000","last_active":"2023-10-09T11:08:38.034328+0000","last_peered":"2023-10-09T11:08:38.034328+0000","last_clean":"2023-10-09T11:08:38.034328+0000","last_became_active":"2023-10-09T11:05:23.320674+0000","last_became_peered":"2023-10-09T11:05:23.320674+0000","last_unstale":"2023-10-09T11:08:38.034328+0000","last_undegraded":"2023-10-09T11:08:38.034328+0000","last_fullsized":"2023-10-09T11:08:38.034328+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.527660+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":395,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.083600+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:08:38.083600+0000","last_peered":"2023-10-09T11:08:38.083600+0000","last_clean":"2023-10-09T11:08:38.083600+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:08:38.083600+0000","last_undegraded":"2023-10-09T11:08:38.083600+0000","last_fullsized":"2023-10-09T11:08:38.083600+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":94,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.035747+0000","last_change":"2023-10-09T11:04:50.511984+0000","last_active":"2023-10-09T11:08:38.035747+0000","last_peered":"2023-10-09T11:08:38.035747+0000","last_clean":"2023-10-09T11:08:38.035747+0000","last_became_active":"2023-10-09T11:04:50.510820+0000","last_became_peered":"2023-10-09T11:04:50.510820+0000","last_unstale":"2023-10-09T11:08:38.035747+0000","last_undegraded":"2023-10-09T11:08:38.035747+0000","last_fullsized":"2023-10-09T11:08:38.035747+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:38.662189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":546,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.085808+0000","last_change":"2023-10-09T11:05:20.327032+0000","last_active":"2023-10-09T11:08:38.085808+0000","last_peered":"2023-10-09T11:08:38.085808+0000","last_clean":"2023-10-09T11:08:38.085808+0000","last_became_active":"2023-10-09T11:05:20.320193+0000","last_became_peered":"2023-10-09T11:05:20.320193+0000","last_unstale":"2023-10-09T11:08:38.085808+0000","last_undegraded":"2023-10-09T11:08:38.085808+0000","last_fullsized":"2023-10-09T11:08:38.085808+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:44.862890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":94,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.035315+0000","last_change":"2023-10-09T11:04:50.384219+0000","last_active":"2023-10-09T11:08:38.035315+0000","last_peered":"2023-10-09T11:08:38.035315+0000","last_clean":"2023-10-09T11:08:38.035315+0000","last_became_active":"2023-10-09T11:04:50.384028+0000","last_became_peered":"2023-10-09T11:04:50.384028+0000","last_unstale":"2023-10-09T11:08:38.035315+0000","last_undegraded":"2023-10-09T11:08:38.035315+0000","last_fullsized":"2023-10-09T11:08:38.035315+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:04.721869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":751,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:39.540078+0000","last_change":"2023-10-09T11:08:39.540029+0000","last_active":"2023-10-09T11:08:39.540078+0000","last_peered":"2023-10-09T11:08:39.540078+0000","last_clean":"2023-10-09T11:08:39.540078+0000","last_became_active":"2023-10-09T11:07:36.291077+0000","last_became_peered":"2023-10-09T11:07:36.291077+0000","last_unstale":"2023-10-09T11:08:39.540078+0000","last_undegraded":"2023-10-09T11:08:39.540078+0000","last_fullsized":"2023-10-09T11:08:39.540078+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.539966+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.539966+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:20.767812+0000","scrub_duration":0.0016245929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":103,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.034316+0000","last_change":"2023-10-09T11:05:20.299321+0000","last_active":"2023-10-09T11:08:38.034316+0000","last_peered":"2023-10-09T11:08:38.034316+0000","last_clean":"2023-10-09T11:08:38.034316+0000","last_became_active":"2023-10-09T11:05:20.299019+0000","last_became_peered":"2023-10-09T11:05:20.299019+0000","last_unstale":"2023-10-09T11:08:38.034316+0000","last_undegraded":"2023-10-09T11:08:38.034316+0000","last_fullsized":"2023-10-09T11:08:38.034316+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:08.645909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.048046+0000","last_change":"2023-10-09T11:04:50.366890+0000","last_active":"2023-10-09T11:08:38.048046+0000","last_peered":"2023-10-09T11:08:38.048046+0000","last_clean":"2023-10-09T11:08:38.048046+0000","last_became_active":"2023-10-09T11:04:50.366558+0000","last_became_peered":"2023-10-09T11:04:50.366558+0000","last_unstale":"2023-10-09T11:08:38.048046+0000","last_undegraded":"2023-10-09T11:08:38.048046+0000","last_fullsized":"2023-10-09T11:08:38.048046+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:50.993153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T11:08:46.070598+0000","last_change":"2023-10-09T11:08:46.070598+0000","last_active":"2023-10-09T11:08:46.070598+0000","last_peered":"2023-10-09T11:08:46.070598+0000","last_clean":"2023-10-09T11:08:46.070598+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T11:08:46.070598+0000","last_undegraded":"2023-10-09T11:08:46.070598+0000","last_fullsized":"2023-10-09T11:08:46.070598+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T11:08:46.070598+0000","last_clean_scrub_stamp":"2023-10-09T11:08:46.070598+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":866,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.492089+0000","last_change":"2023-10-09T11:04:53.274332+0000","last_active":"2023-10-09T11:08:37.492089+0000","last_peered":"2023-10-09T11:08:37.492089+0000","last_clean":"2023-10-09T11:08:37.492089+0000","last_became_active":"2023-10-09T11:04:53.273798+0000","last_became_peered":"2023-10-09T11:04:53.273798+0000","last_unstale":"2023-10-09T11:08:37.492089+0000","last_undegraded":"2023-10-09T11:08:37.492089+0000","last_fullsized":"2023-10-09T11:08:37.492089+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:04.220970+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":115,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T11:08:38.087883+0000","last_change":"2023-10-09T11:06:24.249877+0000","last_active":"2023-10-09T11:08:38.087883+0000","last_peered":"2023-10-09T11:08:38.087883+0000","last_clean":"2023-10-09T11:08:38.087883+0000","last_became_active":"2023-10-09T11:06:24.249573+0000","last_became_peered":"2023-10-09T11:06:24.249573+0000","last_unstale":"2023-10-09T11:08:38.087883+0000","last_undegraded":"2023-10-09T11:08:38.087883+0000","last_fullsized":"2023-10-09T11:08:38.087883+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:11.798192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":615,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T11:08:37.492601+0000","last_change":"2023-10-09T11:05:20.307347+0000","last_active":"2023-10-09T11:08:37.492601+0000","last_peered":"2023-10-09T11:08:37.492601+0000","last_clean":"2023-10-09T11:08:37.492601+0000","last_became_active":"2023-10-09T11:05:20.307126+0000","last_became_peered":"2023-10-09T11:05:20.307126+0000","last_unstale":"2023-10-09T11:08:37.492601+0000","last_undegraded":"2023-10-09T11:08:37.492601+0000","last_fullsized":"2023-10-09T11:08:37.492601+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:34.408515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":91,"num_bytes_recovered":15742,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12694,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332506,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1272472,"kb_used_data":368976,"kb_used_omap":0,"kb_used_meta":903488,"kb_avail":92472680,"statfs":{"total":95995035648,"available":94692024320,"internally_reserved":0,"allocated":377831424,"data_stored":375998042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925171712},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":1.3460000000000001,"15min":1.7350000000000001},"min":{"1min":0.254,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":9.8219999999999992,"5min":9.8219999999999992,"15min":9.8219999999999992},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.9099999999999999,"5min":1.6539999999999999,"15min":1.7669999999999999},"min":{"1min":0.28999999999999998,"5min":0.24299999999999999,"15min":0.216},"max":{"1min":5.4820000000000002,"5min":15.853,"15min":15.853},"last":0.68500000000000005}]},{"osd":1,"last update":"Mon Oct 9 11:08:26 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":1.5029999999999999,"15min":1.5469999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":5.9669999999999996,"5min":11.619999999999999,"15min":11.619999999999999},"last":0.502},{"interface":"front","average":{"1min":2.851,"5min":2.0219999999999998,"15min":1.944},"min":{"1min":0.26300000000000001,"5min":0.25,"15min":0.25},"max":{"1min":13.096,"5min":13.096,"15min":13.096},"last":0.55400000000000005}]},{"osd":2,"last update":"Mon Oct 9 11:07:52 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":1.929,"15min":1.7170000000000001},"min":{"1min":0.313,"5min":0.309,"15min":0.245},"max":{"1min":3.3050000000000002,"5min":16.605,"15min":16.605},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.466,"5min":1.4570000000000001,"15min":1.706},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":4.9569999999999999,"5min":9.7469999999999999,"15min":13.037000000000001},"last":0.73099999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:08:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3050000000000002,"5min":1.7050000000000001,"15min":1.5009999999999999},"min":{"1min":0.34200000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.3970000000000002,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.7070000000000001,"5min":1.885,"15min":1.823},"min":{"1min":0.34000000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.0999999999999996,"5min":8.2330000000000005,"15min":8.2330000000000005},"last":0.71199999999999997}]},{"osd":4,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":0.91800000000000004,"15min":1.0940000000000001},"min":{"1min":0.29799999999999999,"5min":0.218,"15min":0.218},"max":{"1min":7.4630000000000001,"5min":7.4630000000000001,"15min":8.2940000000000005},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.034,"5min":1.256,"15min":1.244},"min":{"1min":0.30599999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.7190000000000003,"5min":11.808,"15min":16.481000000000002},"last":0.66700000000000004}]},{"osd":5,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.0640000000000001,"15min":1.252},"min":{"1min":0.32800000000000001,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":5.4800000000000004,"5min":15.414999999999999,"15min":15.414999999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.345,"15min":1.198},"min":{"1min":0.375,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.045,"5min":15.369,"15min":15.369},"last":0.63300000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.226,"15min":1.26},"min":{"1min":0.34599999999999997,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":3.8500000000000001,"5min":14.159000000000001,"15min":14.159000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.121,"5min":1.472,"15min":1.355},"min":{"1min":0.36099999999999999,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":3.5830000000000002,"5min":13.993,"15min":16.538},"last":0.751}]}]},{"osd":6,"up_from":44,"seq":188978561437,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1532744,"kb_used_data":36096,"kb_used_omap":0,"kb_used_meta":1496640,"kb_avail":92212408,"statfs":{"total":95995035648,"available":94425505792,"internally_reserved":0,"allocated":36962304,"data_stored":35144616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1532559360},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":1.4950000000000001,"15min":1.9419999999999999},"min":{"1min":0.22600000000000001,"5min":0.223,"15min":0.217},"max":{"1min":12.209,"5min":13.867000000000001,"15min":54.743000000000002},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.081,"5min":1.181,"15min":1.8919999999999999},"min":{"1min":0.27300000000000002,"5min":0.214,"15min":0.214},"max":{"1min":3.3740000000000001,"5min":10.055999999999999,"15min":53.701000000000001},"last":0.45400000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:08:26 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.514,"15min":1.1910000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.23499999999999999},"max":{"1min":4.3710000000000004,"5min":16.225999999999999,"15min":41.241999999999997},"last":0.52700000000000002},{"interface":"front","average":{"1min":2.7330000000000001,"5min":2.1139999999999999,"15min":1.8260000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.248},"max":{"1min":9.2080000000000002,"5min":11.715,"15min":41.476999999999997},"last":0.42199999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:07:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":2.1890000000000001,"15min":1.857},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.7089999999999996,"5min":10.481999999999999,"15min":12.003},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.6990000000000001,"5min":1.764,"15min":1.8999999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.133,"5min":12.693,"15min":15.432},"last":0.48199999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:08:23 2023","interfaces":[{"interface":"back","average":{"1min":3.6749999999999998,"5min":2.2080000000000002,"15min":1.702},"min":{"1min":0.40200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":11.395,"5min":28.948,"15min":28.948},"last":0.49099999999999999},{"interface":"front","average":{"1min":2.653,"5min":2.9820000000000002,"15min":2.6200000000000001},"min":{"1min":0.307,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.964,"5min":28.893000000000001,"15min":28.893000000000001},"last":0.499}]},{"osd":4,"last update":"Mon Oct 9 11:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":1.171,"15min":1.0129999999999999},"min":{"1min":0.26500000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.0189999999999999,"5min":17.216999999999999,"15min":17.216999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.78100000000000003,"15min":0.84299999999999997},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.24399999999999999},"max":{"1min":1.4059999999999999,"5min":10.025,"15min":10.025},"last":0.38}]},{"osd":5,"last update":"Mon Oct 9 11:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.1419999999999999,"15min":1.2709999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":1.103,"5min":10.209,"15min":19.786999999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.90500000000000003,"15min":0.97099999999999997},"min":{"1min":0.33200000000000002,"5min":0.248,"15min":0.248},"max":{"1min":1.4810000000000001,"5min":9.9469999999999992,"15min":9.9469999999999992},"last":0.40899999999999997}]},{"osd":7,"last update":"Mon Oct 9 11:08:14 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.081,"15min":1.268},"min":{"1min":0.32200000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.7250000000000001,"5min":10.246,"15min":12.882},"last":0.441},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.83599999999999997,"15min":0.93300000000000005},"min":{"1min":0.35599999999999998,"5min":0.31,"15min":0.27500000000000002},"max":{"1min":3.2839999999999998,"5min":11.557,"15min":11.557},"last":0.35399999999999998}]}]},{"osd":1,"up_from":388,"seq":1666447310951,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":509032,"kb_used_data":7456,"kb_used_omap":13,"kb_used_meta":501554,"kb_avail":93236120,"statfs":{"total":95995035648,"available":95473786880,"internally_reserved":0,"allocated":7634944,"data_stored":4628077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":513592007},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":2.6960000000000002,"15min":2.4430000000000001},"min":{"1min":0.22600000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":7.0570000000000004,"5min":18.192,"15min":18.192},"last":0.379},{"interface":"front","average":{"1min":3.7949999999999999,"5min":3.0840000000000001,"15min":2.0920000000000001},"min":{"1min":0.26700000000000002,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":16.895,"5min":17.957999999999998,"15min":17.957999999999998},"last":0.44600000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.9380000000000002,"5min":2.5339999999999998,"15min":2.911},"min":{"1min":0.216,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":8.798,"5min":12.436999999999999,"15min":47.731000000000002},"last":0.46000000000000002},{"interface":"front","average":{"1min":3.895,"5min":2.8690000000000002,"15min":2.5880000000000001},"min":{"1min":0.26000000000000001,"5min":0.216,"15min":0.216},"max":{"1min":16.870000000000001,"5min":16.870000000000001,"15min":47.786999999999999},"last":0.47399999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:08:26 2023","interfaces":[{"interface":"back","average":{"1min":2.819,"5min":3.3530000000000002,"15min":2.1099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":9.4960000000000004,"5min":92.447999999999993,"15min":92.447999999999993},"last":0.40100000000000002},{"interface":"front","average":{"1min":3.069,"5min":3.4350000000000001,"15min":2.133},"min":{"1min":0.29099999999999998,"5min":0.217,"15min":0.217},"max":{"1min":13.295999999999999,"5min":92.066999999999993,"15min":92.066999999999993},"last":0.41999999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0059999999999998,"5min":1.879,"15min":2.3380000000000001},"min":{"1min":0.30299999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.9790000000000001,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":2.9670000000000001,"5min":2.1480000000000001,"15min":1.73},"min":{"1min":0.34799999999999998,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":16.841000000000001,"5min":16.841000000000001,"15min":16.841000000000001},"last":0.56000000000000005}]},{"osd":5,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2559999999999998,"5min":1.968,"15min":2.286},"min":{"1min":0.311,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":9.7010000000000005,"5min":20.689,"15min":20.689},"last":0.61199999999999999},{"interface":"front","average":{"1min":2.427,"5min":2.113,"15min":2.242},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.0199999999999996,"5min":11.65,"15min":11.65},"last":0.56999999999999995}]},{"osd":6,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0819999999999999,"5min":2.1819999999999999,"15min":2.3799999999999999},"min":{"1min":0.32900000000000001,"5min":0.312,"15min":0.309},"max":{"1min":9.9459999999999997,"5min":20.646999999999998,"15min":20.646999999999998},"last":0.54600000000000004},{"interface":"front","average":{"1min":2.339,"5min":2.0339999999999998,"15min":2.1680000000000001},"min":{"1min":0.34399999999999997,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":7.0410000000000004,"5min":11.75,"15min":11.75},"last":0.59499999999999997}]},{"osd":7,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.2530000000000001,"15min":1.7310000000000001},"min":{"1min":0.378,"5min":0.33000000000000002,"15min":0.32900000000000001},"max":{"1min":16.797999999999998,"5min":16.797999999999998,"15min":16.797999999999998},"last":0.51600000000000001},{"interface":"front","average":{"1min":2.2029999999999998,"5min":1.996,"15min":2.294},"min":{"1min":0.38100000000000001,"5min":0.35599999999999998,"15min":0.33200000000000002},"max":{"1min":8.4469999999999992,"5min":20.716000000000001,"15min":20.716000000000001},"last":0.58099999999999996}]}]},{"osd":0,"up_from":80,"seq":343597384009,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1026444,"kb_used_data":44164,"kb_used_omap":28,"kb_used_meta":982243,"kb_avail":92718708,"statfs":{"total":95995035648,"available":94943956992,"internally_reserved":0,"allocated":45223936,"data_stored":42114798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1005817680},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:08:29 2023","interfaces":[{"interface":"back","average":{"1min":1.946,"5min":2.0249999999999999,"15min":2.742},"min":{"1min":0.191,"5min":0.186,"15min":0.186},"max":{"1min":6.4909999999999997,"5min":14.644,"15min":19.608000000000001},"last":0.31},{"interface":"front","average":{"1min":1.718,"5min":2.1549999999999998,"15min":2.8010000000000002},"min":{"1min":0.223,"5min":0.214,"15min":0.214},"max":{"1min":5.8609999999999998,"5min":15.834,"15min":19.992999999999999},"last":0.32700000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:07:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":2.339,"15min":2.29},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.17199999999999999},"max":{"1min":11.516,"5min":12.459,"15min":24.852},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.73,"5min":2.246,"15min":2.694},"min":{"1min":0.215,"5min":0.20300000000000001,"15min":0.19800000000000001},"max":{"1min":8.4589999999999996,"5min":11.68,"15min":77.256},"last":0.432}]},{"osd":3,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":3.016,"15min":3.3599999999999999},"min":{"1min":0.35099999999999998,"5min":0.22,"15min":0.22},"max":{"1min":7.9050000000000002,"5min":25.050000000000001,"15min":25.050000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.2890000000000001,"5min":2.355,"15min":2.0510000000000002},"min":{"1min":0.30499999999999999,"5min":0.254,"15min":0.254},"max":{"1min":6.2389999999999999,"5min":14.125999999999999,"15min":14.125999999999999},"last":0.36199999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":1.877,"5min":2.1880000000000002,"15min":2.4489999999999998},"min":{"1min":0.27100000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.8150000000000004,"5min":14.507,"15min":77.177999999999997},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.6910000000000001,"5min":1.675,"15min":1.6419999999999999},"min":{"1min":0.33600000000000002,"5min":0.30199999999999999,"15min":0.28899999999999998},"max":{"1min":6.194,"5min":7.9249999999999998,"15min":10.513999999999999},"last":0.44600000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.7290000000000001,"15min":1.8160000000000001},"min":{"1min":0.34699999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.2199999999999998,"5min":7.9550000000000001,"15min":10.544},"last":0.499},{"interface":"front","average":{"1min":1.7050000000000001,"5min":1.7270000000000001,"15min":1.804},"min":{"1min":0.39200000000000002,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":4.6820000000000004,"5min":11.212999999999999,"15min":24.754999999999999},"last":0.60499999999999998}]},{"osd":6,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":1.8380000000000001,"15min":1.8959999999999999},"min":{"1min":0.377,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":6.173,"5min":7.9829999999999997,"15min":18.920000000000002},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.423,"5min":1.786,"15min":1.927},"min":{"1min":0.38100000000000001,"5min":0.35299999999999998,"15min":0.31900000000000001},"max":{"1min":4.2789999999999999,"5min":11.257999999999999,"15min":24.800000000000001},"last":0.58699999999999997}]},{"osd":7,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.391,"5min":2.1789999999999998,"15min":2.0339999999999998},"min":{"1min":0.38900000000000001,"5min":0.35099999999999998,"15min":0.32700000000000001},"max":{"1min":9.8710000000000004,"5min":17.672000000000001,"15min":17.672000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.2170000000000001,"5min":2.4500000000000002,"15min":2.569},"min":{"1min":0.33400000000000002,"5min":0.311,"15min":0.311},"max":{"1min":4.9320000000000004,"5min":17.539000000000001,"15min":77.228999999999999},"last":0.46899999999999997}]}]},{"osd":2,"up_from":90,"seq":386547056949,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":781828,"kb_used_data":7612,"kb_used_omap":81,"kb_used_meta":774126,"kb_avail":92963324,"statfs":{"total":95995035648,"available":95194443776,"internally_reserved":0,"allocated":7794688,"data_stored":4873278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":792705242},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":2.2010000000000001,"15min":2.0609999999999999},"min":{"1min":0.186,"5min":0.182,"15min":0.17499999999999999},"max":{"1min":14.226000000000001,"5min":15.609,"15min":48.933999999999997},"last":0.41199999999999998},{"interface":"front","average":{"1min":2.5419999999999998,"5min":2.056,"15min":1.8919999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.19600000000000001},"max":{"1min":9.5540000000000003,"5min":9.8010000000000002,"15min":15.595000000000001},"last":1.0820000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":2.3780000000000001,"15min":2.5339999999999998},"min":{"1min":0.193,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":3.242,"5min":24.902000000000001,"15min":49.033999999999999},"last":1.1060000000000001},{"interface":"front","average":{"1min":2.6600000000000001,"5min":2.5640000000000001,"15min":2.7200000000000002},"min":{"1min":0.253,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":14.26,"5min":25.850999999999999,"15min":48.874000000000002},"last":1.2210000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:08:19 2023","interfaces":[{"interface":"back","average":{"1min":4.8019999999999996,"5min":2.9399999999999999,"15min":2.1560000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":25.890999999999998,"5min":25.890999999999998,"15min":25.890999999999998},"last":0.80400000000000005},{"interface":"front","average":{"1min":4.4560000000000004,"5min":2.895,"15min":2.395},"min":{"1min":0.372,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":28.689,"5min":28.689,"15min":28.689},"last":0.92800000000000005}]},{"osd":4,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":1.796,"15min":1.8959999999999999},"min":{"1min":0.27100000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":11.391999999999999,"5min":11.391999999999999,"15min":15.672000000000001},"last":1.2090000000000001},{"interface":"front","average":{"1min":1.468,"5min":1.7909999999999999,"15min":1.873},"min":{"1min":0.34899999999999998,"5min":0.249,"15min":0.249},"max":{"1min":8.3350000000000009,"5min":15.571,"15min":48.795000000000002},"last":0.878}]},{"osd":5,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":1.597,"15min":1.5620000000000001},"min":{"1min":0.33700000000000002,"5min":0.29099999999999998,"15min":0.25900000000000001},"max":{"1min":7.4980000000000002,"5min":8.5310000000000006,"15min":12.128},"last":1.3859999999999999},{"interface":"front","average":{"1min":1.8580000000000001,"5min":1.7869999999999999,"15min":2.0339999999999998},"min":{"1min":0.34899999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.441000000000001,"5min":11.441000000000001,"15min":49.061},"last":1.175}]},{"osd":6,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.909,"15min":1.804},"min":{"1min":0.41899999999999998,"5min":0.313,"15min":0.313},"max":{"1min":8.6039999999999992,"5min":15.536,"15min":48.845999999999997},"last":0.84199999999999997},{"interface":"front","average":{"1min":2.6499999999999999,"5min":2.012,"15min":1.657},"min":{"1min":0.42399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.157999999999999,"5min":10.157999999999999,"15min":10.157999999999999},"last":1.458}]},{"osd":7,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":2.081,"15min":1.9419999999999999},"min":{"1min":0.45100000000000001,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":8.3800000000000008,"5min":15.59,"15min":48.966000000000001},"last":0.90400000000000003},{"interface":"front","average":{"1min":2.492,"5min":1.917,"15min":1.762},"min":{"1min":0.46200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.5919999999999996,"5min":8.5549999999999997,"15min":14.94},"last":1.427}]}]},{"osd":3,"up_from":469,"seq":2014339661879,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":343072,"kb_used_data":8024,"kb_used_omap":11,"kb_used_meta":335028,"kb_avail":93402080,"statfs":{"total":95995035648,"available":95643729920,"internally_reserved":0,"allocated":8216576,"data_stored":4979487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11309,"internal_metadata":343069651},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":3.004,"5min":2.1840000000000002,"15min":1.875},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":14.521000000000001,"5min":14.882,"15min":14.882},"last":0.33600000000000002},{"interface":"front","average":{"1min":2.3919999999999999,"5min":1.397,"15min":1.044},"min":{"1min":0.20100000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":9.4030000000000005,"5min":9.4030000000000005,"15min":9.4030000000000005},"last":0.27700000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":2.218,"15min":1.6739999999999999},"min":{"1min":0.245,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":8.8350000000000009,"5min":17.93,"15min":17.93},"last":0.28699999999999998},{"interface":"front","average":{"1min":3.0099999999999998,"5min":2.246,"15min":1.8049999999999999},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":10.645,"5min":19.163,"15min":19.163},"last":0.65700000000000003}]},{"osd":2,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4860000000000002,"5min":2.0840000000000001,"15min":1.419},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":12.323,"5min":38.840000000000003,"15min":38.840000000000003},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.6539999999999999,"5min":2.2120000000000002,"15min":1.4570000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":10.172000000000001,"5min":37.991999999999997,"15min":37.991999999999997},"last":0.504}]},{"osd":4,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6419999999999999,"5min":1.655,"15min":1.6000000000000001},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.3719999999999999,"5min":14.907999999999999,"15min":14.907999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":2.0489999999999999,"5min":1.1439999999999999,"15min":0.84499999999999997},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.7889999999999997,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.45000000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.4259999999999999,"15min":1.417},"min":{"1min":0.33100000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.8499999999999996,"5min":9.3300000000000001,"15min":9.3300000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.5529999999999999,"5min":1.5649999999999999,"15min":1.482},"min":{"1min":0.34699999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.3320000000000007,"5min":10.029,"15min":10.029},"last":0.57099999999999995}]},{"osd":6,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.256,"5min":1.6539999999999999,"15min":1.74},"min":{"1min":0.32300000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.9550000000000001,"5min":15.455,"15min":15.455},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.8100000000000001,"5min":1.706,"15min":1.6639999999999999},"min":{"1min":0.36799999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.4019999999999992,"5min":14.837,"15min":14.837},"last":0.63300000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.365,"15min":1.1220000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.6769999999999996,"5min":6.907,"15min":6.907},"last":0.58799999999999997},{"interface":"front","average":{"1min":1.55,"5min":1.595,"15min":1.3029999999999999},"min":{"1min":0.33700000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.9320000000000004,"5min":6.9119999999999999,"15min":6.9119999999999999},"last":0.59799999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019300,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":898900,"kb_used_data":6156,"kb_used_omap":0,"kb_used_meta":892736,"kb_avail":92846252,"statfs":{"total":95995035648,"available":95074562048,"internally_reserved":0,"allocated":6303744,"data_stored":4474167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914161664},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.613,"15min":2.4129999999999998},"min":{"1min":0.22600000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":6.1289999999999996,"5min":13.162000000000001,"15min":213.88999999999999},"last":0.376},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.742,"15min":2.5019999999999998},"min":{"1min":0.26500000000000001,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":3.875,"5min":9.0670000000000002,"15min":214.09800000000001},"last":0.57299999999999995}]},{"osd":1,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2989999999999999,"5min":2.6059999999999999,"15min":2.6440000000000001},"min":{"1min":0.28299999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.718,"5min":57.966999999999999,"15min":121.41},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.4810000000000001,"5min":2.2519999999999998,"15min":2.1280000000000001},"min":{"1min":0.26900000000000002,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":6.1980000000000004,"5min":57.832000000000001,"15min":122.21899999999999},"last":0.53000000000000003}]},{"osd":2,"last update":"Mon Oct 9 11:08:00 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":1.917,"15min":1.74},"min":{"1min":0.32100000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.2939999999999996,"5min":18.407,"15min":18.407},"last":0.59399999999999997},{"interface":"front","average":{"1min":2.149,"5min":1.9810000000000001,"15min":2.0910000000000002},"min":{"1min":0.27900000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.1479999999999997,"5min":8.4250000000000007,"15min":27.818000000000001},"last":0.51200000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":2.4380000000000002,"15min":1.98},"min":{"1min":0.34799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.1689999999999996,"5min":60.753,"15min":60.753},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.5049999999999999,"5min":2.2069999999999999,"15min":1.5149999999999999},"min":{"1min":0.45300000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.4980000000000002,"5min":61.473999999999997,"15min":61.473999999999997},"last":0.48599999999999999}]},{"osd":5,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":1.353,"15min":1.3600000000000001},"min":{"1min":0.30499999999999999,"5min":0.224,"15min":0.224},"max":{"1min":4.7670000000000003,"5min":11.59,"15min":19.068999999999999},"last":0.44},{"interface":"front","average":{"1min":1.079,"5min":1.2809999999999999,"15min":1.238},"min":{"1min":0.34200000000000003,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":3.7570000000000001,"5min":9.8610000000000007,"15min":15.535},"last":0.46800000000000003}]},{"osd":6,"last update":"Mon Oct 9 11:08:00 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":1.353,"15min":1.1599999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":7.5860000000000003,"5min":8.3930000000000007,"15min":15.739000000000001},"last":0.498},{"interface":"front","average":{"1min":1.4610000000000001,"5min":1.4890000000000001,"15min":1.4570000000000001},"min":{"1min":0.34899999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":9.5210000000000008,"5min":12.795999999999999,"15min":14.093999999999999},"last":0.52200000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:08:00 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.5600000000000001,"15min":1.5529999999999999},"min":{"1min":0.308,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.4540000000000002,"5min":12.753,"15min":19.923999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.4239999999999999,"15min":1.476},"min":{"1min":0.314,"5min":0.29799999999999999,"15min":0.28799999999999998},"max":{"1min":4.21,"5min":10.853,"15min":19.859999999999999},"last":0.55000000000000004}]}]},{"osd":5,"up_from":37,"seq":158913790369,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1205700,"kb_used_data":6396,"kb_used_omap":0,"kb_used_meta":1199296,"kb_avail":92539452,"statfs":{"total":95995035648,"available":94760398848,"internally_reserved":0,"allocated":6549504,"data_stored":4732276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1228079104},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:25 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":2.3239999999999998,"15min":2.2919999999999998},"min":{"1min":0.255,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.4349999999999996,"5min":49.665999999999997,"15min":58.271999999999998},"last":0.435},{"interface":"front","average":{"1min":1.849,"5min":2.3740000000000001,"15min":2.3399999999999999},"min":{"1min":0.29699999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.4610000000000003,"5min":49.701999999999998,"15min":58.25},"last":0.46400000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:08:32 2023","interfaces":[{"interface":"back","average":{"1min":2.5179999999999998,"5min":1.899,"15min":2.0670000000000002},"min":{"1min":0.246,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.1850000000000005,"5min":16.962,"15min":32.491999999999997},"last":0.47399999999999998},{"interface":"front","average":{"1min":2.3439999999999999,"5min":2.004,"15min":1.917},"min":{"1min":0.32800000000000001,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":9.1989999999999998,"5min":17.033999999999999,"15min":32.537999999999997},"last":0.48599999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:07:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.454,"15min":1.786},"min":{"1min":0.307,"5min":0.254,"15min":0.221},"max":{"1min":5.1269999999999998,"5min":6.3929999999999998,"15min":11.257999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.1869999999999998,"5min":2.0129999999999999,"15min":1.9370000000000001},"min":{"1min":0.29099999999999998,"5min":0.253,"15min":0.253},"max":{"1min":8.0359999999999996,"5min":11.673,"15min":14.621},"last":0.51100000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.962,"5min":1.738,"15min":1.5720000000000001},"min":{"1min":0.311,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.0289999999999999,"5min":15.535,"15min":15.535},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.8839999999999999,"5min":2.0710000000000002,"15min":1.8959999999999999},"min":{"1min":0.40200000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":15.757999999999999,"5min":15.757999999999999,"15min":15.757999999999999},"last":0.56699999999999995}]},{"osd":4,"last update":"Mon Oct 9 11:08:32 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":1.0089999999999999,"15min":0.93799999999999994},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.216},"max":{"1min":3.0680000000000001,"5min":12.042,"15min":12.042},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.3480000000000001,"5min":1.1020000000000001,"15min":1.1899999999999999},"min":{"1min":0.27000000000000002,"5min":0.245,"15min":0.245},"max":{"1min":6.8639999999999999,"5min":9.2490000000000006,"15min":9.2490000000000006},"last":0.44700000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:08:45 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.2090000000000001,"15min":1.2450000000000001},"min":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.27100000000000002},"max":{"1min":5.944,"5min":7.1020000000000003,"15min":9.1969999999999992},"last":0.502},{"interface":"front","average":{"1min":1.214,"5min":1.204,"15min":1.0700000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29099999999999998},"max":{"1min":9.298,"5min":11.983000000000001,"15min":11.983000000000001},"last":0.53000000000000003}]},{"osd":7,"last update":"Mon Oct 9 11:08:08 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.3160000000000001,"15min":1.302},"min":{"1min":0.35399999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.0469999999999997,"5min":8.9429999999999996,"15min":9.3539999999999992},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.03,"5min":1.278,"15min":1.177},"min":{"1min":0.32800000000000001,"5min":0.30499999999999999,"15min":0.30099999999999999},"max":{"1min":4.2350000000000003,"5min":11.726000000000001,"15min":11.726000000000001},"last":0.54800000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3733,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:08:49.215 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T11:08:49.228 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:49.406 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:49.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:49 smithi125 ceph-mon[195507]: pgmap v1398: 135 pgs: 6 creating+peering, 26 unknown, 103 active+clean; 582 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:49.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:49 smithi125 ceph-mon[195507]: failed to encode map e571 with expected crc 2023-10-09T11:08:49.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:49 smithi125 ceph-mon[195507]: osdmap e571: 8 total, 8 up, 8 in 2023-10-09T11:08:49.411 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:49 smithi125 ceph-mon[195507]: failed to encode map e571 with expected crc 2023-10-09T11:08:49.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:49 smithi125 ceph-mon[200197]: pgmap v1398: 135 pgs: 6 creating+peering, 26 unknown, 103 active+clean; 582 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:49.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:49 smithi125 ceph-mon[200197]: failed to encode map e571 with expected crc 2023-10-09T11:08:49.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:49 smithi125 ceph-mon[200197]: osdmap e571: 8 total, 8 up, 8 in 2023-10-09T11:08:49.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:49 smithi125 ceph-mon[200197]: failed to encode map e571 with expected crc 2023-10-09T11:08:49.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:49.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:49.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:49 smithi155 ceph-mon[184854]: pgmap v1398: 135 pgs: 6 creating+peering, 26 unknown, 103 active+clean; 582 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:49.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:49 smithi155 ceph-mon[184854]: failed to encode map e571 with expected crc 2023-10-09T11:08:49.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:49 smithi155 ceph-mon[184854]: osdmap e571: 8 total, 8 up, 8 in 2023-10-09T11:08:49.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:49 smithi155 ceph-mon[184854]: failed to encode map e571 with expected crc 2023-10-09T11:08:49.513 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:49.557 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:50.135 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:50.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:50.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:50.239 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:50.276 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:50.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:50 smithi125 ceph-mon[195507]: failed to encode map e571 with expected crc 2023-10-09T11:08:50.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:50 smithi125 ceph-mon[195507]: failed to encode map e571 with expected crc 2023-10-09T11:08:50.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:50 smithi125 ceph-mon[200197]: failed to encode map e571 with expected crc 2023-10-09T11:08:50.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:50 smithi125 ceph-mon[200197]: failed to encode map e571 with expected crc 2023-10-09T11:08:50.405 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:08:50.405 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T11:08:50.405 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T11:08:50.405 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:08:50.405 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.406 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:3548 11.1 11:8c8e4577:::benchmark_data_smithi125_439543_object3547:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e498)", 2023-10-09T11:08:50.406 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:10.492309+0000", 2023-10-09T11:08:50.406 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 219.91029280399999, 2023-10-09T11:08:50.406 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.127281062, 2023-10-09T11:08:50.406 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.406 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.406 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:08:50.406 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:08:50.406 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:50.406 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 3548 2023-10-09T11:08:50.406 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.406 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.407 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.407 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.407 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492309+0000", 2023-10-09T11:08:50.407 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.407 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.407 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.407 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.407 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492309+0000", 2023-10-09T11:08:50.407 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.407 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.407 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.407 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.408 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492308+0000", 2023-10-09T11:08:50.408 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:50.408 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.408 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.408 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.408 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492331+0000", 2023-10-09T11:08:50.408 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2861e-05 2023-10-09T11:08:50.408 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.408 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.408 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.408 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492340+0000", 2023-10-09T11:08:50.408 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4779999999999997e-06 2023-10-09T11:08:50.408 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.409 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.409 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.409 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492352+0000", 2023-10-09T11:08:50.409 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2148e-05 2023-10-09T11:08:50.409 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.409 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.409 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.409 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492405+0000", 2023-10-09T11:08:50.409 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2435999999999997e-05 2023-10-09T11:08:50.409 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.409 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.409 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.410 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492536+0000", 2023-10-09T11:08:50.410 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013124299999999999 2023-10-09T11:08:50.410 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.410 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.410 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,2", 2023-10-09T11:08:50.410 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.492663+0000", 2023-10-09T11:08:50.410 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000126938 2023-10-09T11:08:50.410 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.410 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.410 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:08:50.410 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.494589+0000", 2023-10-09T11:08:50.410 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001926292 2023-10-09T11:08:50.411 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.411 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.411 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.411 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.494715+0000", 2023-10-09T11:08:50.411 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000126098 2023-10-09T11:08:50.411 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.411 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.411 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.411 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.497672+0000", 2023-10-09T11:08:50.411 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029564629999999999 2023-10-09T11:08:50.411 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.411 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.412 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.412 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.619543+0000", 2023-10-09T11:08:50.412 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12187139 2023-10-09T11:08:50.412 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.412 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.412 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.412 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:10.619590+0000", 2023-10-09T11:08:50.412 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6424999999999997e-05 2023-10-09T11:08:50.412 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.412 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.412 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.412 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.413 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.413 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:59720 11.19 11:9a472153:::benchmark_data_smithi125_439543_object59719:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-10-09T11:08:50.413 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.203684+0000", 2023-10-09T11:08:50.413 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 177.198917228, 2023-10-09T11:08:50.413 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18449432900000001, 2023-10-09T11:08:50.413 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.413 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.413 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:08:50.413 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:08:50.413 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:50.413 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 59720 2023-10-09T11:08:50.413 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.414 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.414 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.414 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.414 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.203684+0000", 2023-10-09T11:08:50.414 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.414 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.414 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.414 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.414 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.203684+0000", 2023-10-09T11:08:50.414 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.414 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.414 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.415 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.203684+0000", 2023-10-09T11:08:50.415 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:50.415 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.415 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.203712+0000", 2023-10-09T11:08:50.415 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8048999999999999e-05 2023-10-09T11:08:50.415 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.415 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.203723+0000", 2023-10-09T11:08:50.415 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1450999999999999e-05 2023-10-09T11:08:50.416 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.416 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.416 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.203730+0000", 2023-10-09T11:08:50.416 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5300000000000002e-06 2023-10-09T11:08:50.416 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.416 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.416 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.205127+0000", 2023-10-09T11:08:50.416 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013968769999999999 2023-10-09T11:08:50.416 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.416 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.417 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.417 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.205251+0000", 2023-10-09T11:08:50.417 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012439100000000001 2023-10-09T11:08:50.417 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.417 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.417 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:08:50.417 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.205387+0000", 2023-10-09T11:08:50.417 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013618899999999999 2023-10-09T11:08:50.417 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.417 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.417 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:08:50.417 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.207083+0000", 2023-10-09T11:08:50.418 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001695682 2023-10-09T11:08:50.418 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.418 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.418 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.418 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210270+0000", 2023-10-09T11:08:50.418 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0031871040000000001 2023-10-09T11:08:50.418 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.418 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.418 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.418 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210989+0000", 2023-10-09T11:08:50.418 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00071919799999999999 2023-10-09T11:08:50.418 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.419 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.419 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.419 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.211040+0000", 2023-10-09T11:08:50.419 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0840000000000001e-05 2023-10-09T11:08:50.419 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.419 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.419 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.419 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388178+0000", 2023-10-09T11:08:50.419 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17713855100000001 2023-10-09T11:08:50.419 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.419 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.419 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.420 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.420 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.420 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:59727 11.a 11:58601085:::benchmark_data_smithi125_439543_object59726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-10-09T11:08:50.420 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.208530+0000", 2023-10-09T11:08:50.420 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 177.19407094600001, 2023-10-09T11:08:50.420 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19139910500000001, 2023-10-09T11:08:50.420 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.420 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.420 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:08:50.420 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:08:50.420 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:50.420 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 59727 2023-10-09T11:08:50.421 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.421 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.421 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.421 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.421 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208530+0000", 2023-10-09T11:08:50.421 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.421 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.421 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.421 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.421 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208530+0000", 2023-10-09T11:08:50.421 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.421 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.421 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.422 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.422 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208529+0000", 2023-10-09T11:08:50.422 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:08:50.422 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.422 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.422 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.422 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208567+0000", 2023-10-09T11:08:50.422 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.786e-05 2023-10-09T11:08:50.422 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.422 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.422 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.423 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208579+0000", 2023-10-09T11:08:50.423 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2223e-05 2023-10-09T11:08:50.423 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.423 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.423 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.423 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208587+0000", 2023-10-09T11:08:50.423 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2109999999999998e-06 2023-10-09T11:08:50.423 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.423 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.423 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.423 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210748+0000", 2023-10-09T11:08:50.423 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002160861 2023-10-09T11:08:50.424 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.424 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.424 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.424 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210882+0000", 2023-10-09T11:08:50.424 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013406099999999999 2023-10-09T11:08:50.424 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.424 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.424 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T11:08:50.424 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.211049+0000", 2023-10-09T11:08:50.424 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016722899999999999 2023-10-09T11:08:50.424 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.424 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.425 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.392637+0000", 2023-10-09T11:08:50.425 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18158798400000001 2023-10-09T11:08:50.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.425 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.392674+0000", 2023-10-09T11:08:50.425 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6511999999999999e-05 2023-10-09T11:08:50.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:08:50.425 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.399528+0000", 2023-10-09T11:08:50.426 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0068543329999999998 2023-10-09T11:08:50.426 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.426 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.426 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.399590+0000", 2023-10-09T11:08:50.426 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1787999999999999e-05 2023-10-09T11:08:50.426 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.426 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.426 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.399930+0000", 2023-10-09T11:08:50.426 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00033950300000000001 2023-10-09T11:08:50.426 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.426 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.427 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.427 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.427 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.427 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:59730 11.b 11:d344def5:::benchmark_data_smithi125_439543_object59729:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-10-09T11:08:50.427 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.208596+0000", 2023-10-09T11:08:50.427 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 177.194005796, 2023-10-09T11:08:50.427 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18725814099999999, 2023-10-09T11:08:50.427 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.427 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.427 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:08:50.427 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:08:50.427 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:50.428 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 59730 2023-10-09T11:08:50.428 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.428 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.428 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.428 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.428 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208596+0000", 2023-10-09T11:08:50.428 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.428 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.428 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.428 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.428 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208596+0000", 2023-10-09T11:08:50.428 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.429 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.429 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.429 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.429 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208595+0000", 2023-10-09T11:08:50.429 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:50.429 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.429 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.429 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.429 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208617+0000", 2023-10-09T11:08:50.429 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1835e-05 2023-10-09T11:08:50.429 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.429 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.430 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.430 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208624+0000", 2023-10-09T11:08:50.430 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0700000000000001e-06 2023-10-09T11:08:50.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.430 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.430 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.430 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.208629+0000", 2023-10-09T11:08:50.430 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3539999999999999e-06 2023-10-09T11:08:50.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.430 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.430 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.430 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.387822+0000", 2023-10-09T11:08:50.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.179192617 2023-10-09T11:08:50.431 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.387993+0000", 2023-10-09T11:08:50.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000171397 2023-10-09T11:08:50.431 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,1", 2023-10-09T11:08:50.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388182+0000", 2023-10-09T11:08:50.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018824799999999999 2023-10-09T11:08:50.431 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.432 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.432 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.394988+0000", 2023-10-09T11:08:50.432 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.006806449 2023-10-09T11:08:50.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.432 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:08:50.432 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.394995+0000", 2023-10-09T11:08:50.432 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1430000000000001e-06 2023-10-09T11:08:50.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.432 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.432 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.395753+0000", 2023-10-09T11:08:50.433 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00075767900000000003 2023-10-09T11:08:50.433 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.395797+0000", 2023-10-09T11:08:50.433 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4195e-05 2023-10-09T11:08:50.433 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.395854+0000", 2023-10-09T11:08:50.433 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6684999999999997e-05 2023-10-09T11:08:50.433 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.433 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.434 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.434 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.434 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.434 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:59725 11.0 e511/509 11:06360e61:::benchmark_data_smithi125_439543_object59724:head v 511'3409, mlcod=511'3406)", 2023-10-09T11:08:50.434 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.209430+0000", 2023-10-09T11:08:50.434 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 177.19317185099999, 2023-10-09T11:08:50.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17842855799999999, 2023-10-09T11:08:50.434 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.434 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.434 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.434 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.435 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209430+0000", 2023-10-09T11:08:50.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.435 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209430+0000", 2023-10-09T11:08:50.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.435 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209429+0000", 2023-10-09T11:08:50.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:50.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209460+0000", 2023-10-09T11:08:50.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1449999999999999e-05 2023-10-09T11:08:50.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209469+0000", 2023-10-09T11:08:50.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6710000000000005e-06 2023-10-09T11:08:50.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.437 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.437 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209475+0000", 2023-10-09T11:08:50.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4019999999999999e-06 2023-10-09T11:08:50.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.437 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.437 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209562+0000", 2023-10-09T11:08:50.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6658999999999998e-05 2023-10-09T11:08:50.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.437 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.438 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.209575+0000", 2023-10-09T11:08:50.438 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2787e-05 2023-10-09T11:08:50.438 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.438 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.438 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.438 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.387811+0000", 2023-10-09T11:08:50.438 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17823647000000001 2023-10-09T11:08:50.438 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.438 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.438 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.438 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.387858+0000", 2023-10-09T11:08:50.439 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6767000000000001e-05 2023-10-09T11:08:50.439 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.439 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.439 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.439 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.439 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.439 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.36510.0:59720 11.19 e511/509 ondisk, result = 0)", 2023-10-09T11:08:50.439 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.210916+0000", 2023-10-09T11:08:50.439 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 177.191685049, 2023-10-09T11:08:50.440 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17726972299999999, 2023-10-09T11:08:50.440 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.440 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:08:50.440 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.440 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.440 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.440 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210916+0000", 2023-10-09T11:08:50.440 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.441 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210916+0000", 2023-10-09T11:08:50.441 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.441 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210916+0000", 2023-10-09T11:08:50.441 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:50.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.442 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210920+0000", 2023-10-09T11:08:50.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5900000000000001e-06 2023-10-09T11:08:50.442 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.442 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210927+0000", 2023-10-09T11:08:50.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6669999999999999e-06 2023-10-09T11:08:50.442 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.442 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210932+0000", 2023-10-09T11:08:50.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1590000000000001e-06 2023-10-09T11:08:50.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210975+0000", 2023-10-09T11:08:50.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2398999999999998e-05 2023-10-09T11:08:50.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.210983+0000", 2023-10-09T11:08:50.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4430000000000003e-06 2023-10-09T11:08:50.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.444 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388186+0000", 2023-10-09T11:08:50.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.177203103 2023-10-09T11:08:50.444 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.444 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.444 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.444 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:59736 11.12 11:49022463:::benchmark_data_smithi125_439543_object59735:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-10-09T11:08:50.444 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.211151+0000", 2023-10-09T11:08:50.444 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 177.19145083399999, 2023-10-09T11:08:50.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19015921999999999, 2023-10-09T11:08:50.444 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.445 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.445 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:08:50.445 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:08:50.445 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:50.445 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 59736 2023-10-09T11:08:50.445 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.445 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.445 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.445 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.445 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.211151+0000", 2023-10-09T11:08:50.445 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.445 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.446 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.446 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.211151+0000", 2023-10-09T11:08:50.446 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.446 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.446 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.211149+0000", 2023-10-09T11:08:50.446 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:08:50.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.446 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.447 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.211171+0000", 2023-10-09T11:08:50.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1407e-05 2023-10-09T11:08:50.447 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.447 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.447 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.447 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.387780+0000", 2023-10-09T11:08:50.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17660874300000001 2023-10-09T11:08:50.447 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.447 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.447 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.447 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.387795+0000", 2023-10-09T11:08:50.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5179e-05 2023-10-09T11:08:50.447 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.387987+0000", 2023-10-09T11:08:50.448 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000192127 2023-10-09T11:08:50.448 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388106+0000", 2023-10-09T11:08:50.448 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119504 2023-10-09T11:08:50.448 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,2", 2023-10-09T11:08:50.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388384+0000", 2023-10-09T11:08:50.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027804799999999998 2023-10-09T11:08:50.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.449 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.449 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.391619+0000", 2023-10-09T11:08:50.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032347769999999999 2023-10-09T11:08:50.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.449 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:08:50.449 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.399516+0000", 2023-10-09T11:08:50.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0078969420000000005 2023-10-09T11:08:50.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.401227+0000", 2023-10-09T11:08:50.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001710528 2023-10-09T11:08:50.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.401272+0000", 2023-10-09T11:08:50.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5578999999999998e-05 2023-10-09T11:08:50.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.401310+0000", 2023-10-09T11:08:50.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7546000000000003e-05 2023-10-09T11:08:50.451 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.451 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.451 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.451 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:59728 11.0 e511/509 11:02f73ca9:::benchmark_data_smithi125_439543_object59727:head v 511'3410, mlcod=511'3406)", 2023-10-09T11:08:50.451 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.214456+0000", 2023-10-09T11:08:50.451 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 177.18814552500001, 2023-10-09T11:08:50.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18448844, 2023-10-09T11:08:50.451 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.451 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.451 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.214456+0000", 2023-10-09T11:08:50.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.214456+0000", 2023-10-09T11:08:50.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.214455+0000", 2023-10-09T11:08:50.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:08:50.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.214482+0000", 2023-10-09T11:08:50.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7591999999999999e-05 2023-10-09T11:08:50.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388348+0000", 2023-10-09T11:08:50.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.173865874 2023-10-09T11:08:50.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388356+0000", 2023-10-09T11:08:50.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0649999999999999e-06 2023-10-09T11:08:50.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.392408+0000", 2023-10-09T11:08:50.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040511490000000004 2023-10-09T11:08:50.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.392428+0000", 2023-10-09T11:08:50.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0920999999999998e-05 2023-10-09T11:08:50.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.398906+0000", 2023-10-09T11:08:50.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0064776460000000001 2023-10-09T11:08:50.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.398944+0000", 2023-10-09T11:08:50.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8172e-05 2023-10-09T11:08:50.456 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.456 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.456 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.456 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPushReply(11.a 511/500 [PushReplyOp(11:546b761d:::benchmark_data_smithi125_439543_object15921:head)])", 2023-10-09T11:08:50.456 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.220273+0000", 2023-10-09T11:08:50.456 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 177.18232828399999, 2023-10-09T11:08:50.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18383527099999999, 2023-10-09T11:08:50.456 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.456 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:08:50.456 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.220273+0000", 2023-10-09T11:08:50.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.220273+0000", 2023-10-09T11:08:50.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.220272+0000", 2023-10-09T11:08:50.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:08:50.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.220282+0000", 2023-10-09T11:08:50.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0423e-05 2023-10-09T11:08:50.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388222+0000", 2023-10-09T11:08:50.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.167939901 2023-10-09T11:08:50.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.388230+0000", 2023-10-09T11:08:50.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5320000000000006e-06 2023-10-09T11:08:50.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.403669+0000", 2023-10-09T11:08:50.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015438881999999999 2023-10-09T11:08:50.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.404108+0000", 2023-10-09T11:08:50.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00043913399999999999 2023-10-09T11:08:50.460 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.460 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.460 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.460 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:60183 11.19 11:9e2099d9:::benchmark_data_smithi125_439543_object60182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T11:08:50.460 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.701324+0000", 2023-10-09T11:08:50.460 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 176.701277514, 2023-10-09T11:08:50.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.13270354000000001, 2023-10-09T11:08:50.460 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.460 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.461 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:08:50.461 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:08:50.461 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:50.461 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 60183 2023-10-09T11:08:50.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.461 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.461 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.701324+0000", 2023-10-09T11:08:50.461 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.701324+0000", 2023-10-09T11:08:50.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.462 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.462 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.462 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.701323+0000", 2023-10-09T11:08:50.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:50.462 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.462 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.462 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.701345+0000", 2023-10-09T11:08:50.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1982999999999999e-05 2023-10-09T11:08:50.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.463 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.701352+0000", 2023-10-09T11:08:50.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7719999999999997e-06 2023-10-09T11:08:50.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.463 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.701366+0000", 2023-10-09T11:08:50.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3806e-05 2023-10-09T11:08:50.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.464 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.702792+0000", 2023-10-09T11:08:50.464 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014258389999999999 2023-10-09T11:08:50.464 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.464 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.464 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.464 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.702935+0000", 2023-10-09T11:08:50.464 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014319800000000001 2023-10-09T11:08:50.464 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.464 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.464 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:08:50.464 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.703081+0000", 2023-10-09T11:08:50.464 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014630900000000001 2023-10-09T11:08:50.465 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.465 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.465 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830385+0000", 2023-10-09T11:08:50.465 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12730330400000001 2023-10-09T11:08:50.465 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.465 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.465 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830422+0000", 2023-10-09T11:08:50.465 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.752e-05 2023-10-09T11:08:50.465 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:08:50.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.833365+0000", 2023-10-09T11:08:50.466 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029424059999999998 2023-10-09T11:08:50.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.466 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.833415+0000", 2023-10-09T11:08:50.466 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0049999999999997e-05 2023-10-09T11:08:50.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.466 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.834027+0000", 2023-10-09T11:08:50.467 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00061278999999999999 2023-10-09T11:08:50.467 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.467 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.467 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.467 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.467 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:60185 11.a 11:5f9a8cdf:::benchmark_data_smithi125_439543_object60184:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T11:08:50.467 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.705137+0000", 2023-10-09T11:08:50.467 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 176.69746452300001, 2023-10-09T11:08:50.467 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.13415111399999999, 2023-10-09T11:08:50.467 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.467 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.468 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:08:50.468 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:08:50.468 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:50.468 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 60185 2023-10-09T11:08:50.468 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.468 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.468 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.468 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.468 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705137+0000", 2023-10-09T11:08:50.468 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.468 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.468 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705137+0000", 2023-10-09T11:08:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705135+0000", 2023-10-09T11:08:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:08:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705170+0000", 2023-10-09T11:08:50.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4335999999999998e-05 2023-10-09T11:08:50.470 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.470 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705178+0000", 2023-10-09T11:08:50.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4999999999999999e-06 2023-10-09T11:08:50.470 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.470 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705184+0000", 2023-10-09T11:08:50.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1819999999999997e-06 2023-10-09T11:08:50.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.471 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.471 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705293+0000", 2023-10-09T11:08:50.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000109036 2023-10-09T11:08:50.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.471 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.471 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705435+0000", 2023-10-09T11:08:50.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000142564 2023-10-09T11:08:50.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.471 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.472 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T11:08:50.472 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705599+0000", 2023-10-09T11:08:50.472 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016396499999999999 2023-10-09T11:08:50.472 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.472 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.472 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.472 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.833012+0000", 2023-10-09T11:08:50.472 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12741256100000001 2023-10-09T11:08:50.472 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.472 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.473 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.473 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.833049+0000", 2023-10-09T11:08:50.473 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7626999999999997e-05 2023-10-09T11:08:50.473 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.473 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.473 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:08:50.473 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.838831+0000", 2023-10-09T11:08:50.473 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0057820240000000002 2023-10-09T11:08:50.473 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.474 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.474 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.474 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.838880+0000", 2023-10-09T11:08:50.474 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8893000000000001e-05 2023-10-09T11:08:50.474 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.474 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.474 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.474 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.839288+0000", 2023-10-09T11:08:50.474 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00040779899999999998 2023-10-09T11:08:50.474 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.474 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.474 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.475 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.475 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.475 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:60186 11.b 11:d8ac8572:::benchmark_data_smithi125_439543_object60185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T11:08:50.475 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.705197+0000", 2023-10-09T11:08:50.475 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 176.69740451000001, 2023-10-09T11:08:50.475 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.13208626500000001, 2023-10-09T11:08:50.475 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.475 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.475 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:08:50.475 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:08:50.475 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:50.475 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 60186 2023-10-09T11:08:50.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.476 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.476 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.476 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.476 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705197+0000", 2023-10-09T11:08:50.476 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.476 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.476 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.476 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705197+0000", 2023-10-09T11:08:50.476 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.476 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.477 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.477 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705196+0000", 2023-10-09T11:08:50.477 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:50.477 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.477 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.477 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.477 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705237+0000", 2023-10-09T11:08:50.477 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0639000000000002e-05 2023-10-09T11:08:50.477 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.477 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.477 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.477 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705243+0000", 2023-10-09T11:08:50.478 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4809999999999997e-06 2023-10-09T11:08:50.478 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.478 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.478 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.478 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705246+0000", 2023-10-09T11:08:50.478 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3390000000000001e-06 2023-10-09T11:08:50.478 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.478 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.478 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.478 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705307+0000", 2023-10-09T11:08:50.478 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1023999999999997e-05 2023-10-09T11:08:50.478 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.479 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.479 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.479 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705419+0000", 2023-10-09T11:08:50.479 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011229499999999999 2023-10-09T11:08:50.479 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.479 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.479 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,1", 2023-10-09T11:08:50.479 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705572+0000", 2023-10-09T11:08:50.479 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000152349 2023-10-09T11:08:50.479 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.479 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.480 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.480 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.832363+0000", 2023-10-09T11:08:50.480 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12679153100000001 2023-10-09T11:08:50.480 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.480 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.480 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.480 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.832401+0000", 2023-10-09T11:08:50.480 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7738999999999999e-05 2023-10-09T11:08:50.480 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.480 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.481 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:08:50.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.837189+0000", 2023-10-09T11:08:50.481 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0047882649999999999 2023-10-09T11:08:50.481 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.481 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.481 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.837244+0000", 2023-10-09T11:08:50.481 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4719999999999998e-05 2023-10-09T11:08:50.481 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.481 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.481 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.482 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.837283+0000", 2023-10-09T11:08:50.482 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9283000000000003e-05 2023-10-09T11:08:50.482 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.482 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.482 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.482 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.482 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.482 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:60187 11.11 11:8970ccab:::benchmark_data_smithi125_439543_object60186:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T11:08:50.482 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.705256+0000", 2023-10-09T11:08:50.482 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 176.69734494599999, 2023-10-09T11:08:50.482 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.13120647499999999, 2023-10-09T11:08:50.482 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.483 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.483 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:08:50.483 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:08:50.483 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:50.483 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 60187 2023-10-09T11:08:50.483 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.483 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.483 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.483 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.483 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705256+0000", 2023-10-09T11:08:50.483 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.483 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.484 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.484 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.484 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705256+0000", 2023-10-09T11:08:50.484 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.484 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.484 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.484 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.484 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705256+0000", 2023-10-09T11:08:50.484 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:50.484 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.484 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.484 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.484 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705273+0000", 2023-10-09T11:08:50.485 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6568000000000001e-05 2023-10-09T11:08:50.485 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.485 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.485 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.485 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705277+0000", 2023-10-09T11:08:50.485 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9439999999999998e-06 2023-10-09T11:08:50.485 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.485 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.485 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.485 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705281+0000", 2023-10-09T11:08:50.485 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2069999999999997e-06 2023-10-09T11:08:50.485 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.486 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.486 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.486 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830449+0000", 2023-10-09T11:08:50.486 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12516856500000001 2023-10-09T11:08:50.486 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.486 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.486 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.486 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830612+0000", 2023-10-09T11:08:50.486 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016255299999999999 2023-10-09T11:08:50.486 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.486 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.486 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:08:50.487 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830791+0000", 2023-10-09T11:08:50.487 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017918900000000001 2023-10-09T11:08:50.487 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.487 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.487 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.487 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.833018+0000", 2023-10-09T11:08:50.487 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022270580000000001 2023-10-09T11:08:50.487 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.487 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.487 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:08:50.487 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.835358+0000", 2023-10-09T11:08:50.487 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023397209999999999 2023-10-09T11:08:50.488 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.488 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.488 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.488 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.835764+0000", 2023-10-09T11:08:50.488 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00040589199999999998 2023-10-09T11:08:50.488 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.488 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.488 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.488 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.835811+0000", 2023-10-09T11:08:50.488 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7602000000000003e-05 2023-10-09T11:08:50.488 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.488 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.488 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.489 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.836463+0000", 2023-10-09T11:08:50.489 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00065151399999999996 2023-10-09T11:08:50.489 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.489 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.489 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.489 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.489 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.489 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "MOSDPGPushReply(11.a 512/500 [PushReplyOp(11:52aba3e0:::benchmark_data_smithi125_439543_object16408:head)])", 2023-10-09T11:08:50.489 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.705514+0000", 2023-10-09T11:08:50.489 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 176.69708740900001, 2023-10-09T11:08:50.489 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12745868099999999, 2023-10-09T11:08:50.489 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.490 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:08:50.490 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.490 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.490 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.490 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705514+0000", 2023-10-09T11:08:50.490 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.490 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.490 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.490 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.490 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705514+0000", 2023-10-09T11:08:50.490 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.490 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.491 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.491 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.491 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705514+0000", 2023-10-09T11:08:50.491 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:50.491 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.491 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.491 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.491 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705518+0000", 2023-10-09T11:08:50.491 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8509999999999998e-06 2023-10-09T11:08:50.491 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.491 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.491 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.491 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705530+0000", 2023-10-09T11:08:50.492 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1951e-05 2023-10-09T11:08:50.492 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.492 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.492 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.492 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705538+0000", 2023-10-09T11:08:50.492 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4900000000000003e-06 2023-10-09T11:08:50.492 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.492 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.492 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.492 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.832353+0000", 2023-10-09T11:08:50.492 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.126815452 2023-10-09T11:08:50.492 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.493 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.493 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.493 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.832973+0000", 2023-10-09T11:08:50.493 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00061930299999999998 2023-10-09T11:08:50.493 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.493 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.493 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.493 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.493 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.493 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:60188 11.1 11:80f1a15d:::benchmark_data_smithi125_439543_object60187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T11:08:50.493 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:53.705811+0000", 2023-10-09T11:08:50.493 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 176.69679067600001, 2023-10-09T11:08:50.493 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.13158576999999999, 2023-10-09T11:08:50.494 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.494 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.494 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:08:50.494 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:08:50.494 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:50.494 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 60188 2023-10-09T11:08:50.494 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.494 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.494 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.494 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.494 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705811+0000", 2023-10-09T11:08:50.494 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.495 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.495 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.495 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.495 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705811+0000", 2023-10-09T11:08:50.495 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.495 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.495 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.495 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.495 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705810+0000", 2023-10-09T11:08:50.495 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:50.495 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.495 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.496 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.496 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.705830+0000", 2023-10-09T11:08:50.496 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9949000000000002e-05 2023-10-09T11:08:50.496 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.496 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.496 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.496 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830332+0000", 2023-10-09T11:08:50.496 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.124501739 2023-10-09T11:08:50.496 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.496 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.496 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.496 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830341+0000", 2023-10-09T11:08:50.496 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5899999999999997e-06 2023-10-09T11:08:50.497 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.497 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.497 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.497 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830700+0000", 2023-10-09T11:08:50.497 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000359062 2023-10-09T11:08:50.497 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.497 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.497 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.497 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830837+0000", 2023-10-09T11:08:50.497 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000136861 2023-10-09T11:08:50.497 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.497 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.497 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:08:50.498 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.830983+0000", 2023-10-09T11:08:50.498 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000145616 2023-10-09T11:08:50.498 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.498 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.498 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.498 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.833055+0000", 2023-10-09T11:08:50.498 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002071603 2023-10-09T11:08:50.498 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.498 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.498 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.498 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.835734+0000", 2023-10-09T11:08:50.498 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026798730000000002 2023-10-09T11:08:50.499 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.499 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.499 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:08:50.499 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.836720+0000", 2023-10-09T11:08:50.499 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0009858200000000001 2023-10-09T11:08:50.499 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.499 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.499 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.499 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.836765+0000", 2023-10-09T11:08:50.499 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4527000000000002e-05 2023-10-09T11:08:50.499 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.499 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.500 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.500 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:53.837396+0000", 2023-10-09T11:08:50.500 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00063176300000000003 2023-10-09T11:08:50.500 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.500 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.500 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.500 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.500 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.500 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:84218 11.2 11:44986b72:::benchmark_data_smithi125_439543_object84217:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T11:08:50.500 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:06:20.502344+0000", 2023-10-09T11:08:50.500 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 149.90025708300001, 2023-10-09T11:08:50.501 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12896063699999999, 2023-10-09T11:08:50.501 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.501 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.501 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:08:50.501 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:08:50.501 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:50.501 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 84218 2023-10-09T11:08:50.501 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.501 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.501 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.501 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.501 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502344+0000", 2023-10-09T11:08:50.502 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.502 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.502 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.502 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.502 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502344+0000", 2023-10-09T11:08:50.502 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.502 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.502 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.502 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.502 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502343+0000", 2023-10-09T11:08:50.502 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:08:50.502 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.503 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.503 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.503 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502367+0000", 2023-10-09T11:08:50.503 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3883000000000002e-05 2023-10-09T11:08:50.503 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.503 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.503 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.503 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502377+0000", 2023-10-09T11:08:50.503 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8770000000000005e-06 2023-10-09T11:08:50.503 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.503 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.503 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.504 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502383+0000", 2023-10-09T11:08:50.504 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6320000000000002e-06 2023-10-09T11:08:50.504 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.504 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.504 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.504 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502491+0000", 2023-10-09T11:08:50.504 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108058 2023-10-09T11:08:50.504 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.504 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.504 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.504 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502611+0000", 2023-10-09T11:08:50.504 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119146 2023-10-09T11:08:50.505 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.505 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.505 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,2", 2023-10-09T11:08:50.505 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.502746+0000", 2023-10-09T11:08:50.505 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000135814 2023-10-09T11:08:50.505 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.505 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.505 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.505 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.504710+0000", 2023-10-09T11:08:50.505 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019633710000000002 2023-10-09T11:08:50.505 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.505 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.505 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:08:50.506 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505713+0000", 2023-10-09T11:08:50.506 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00100332 2023-10-09T11:08:50.506 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.506 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.506 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.506 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.631217+0000", 2023-10-09T11:08:50.506 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12550412499999999 2023-10-09T11:08:50.506 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.506 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.506 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.506 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.631267+0000", 2023-10-09T11:08:50.507 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9614999999999998e-05 2023-10-09T11:08:50.507 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.507 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.507 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.507 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.631305+0000", 2023-10-09T11:08:50.507 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8152999999999999e-05 2023-10-09T11:08:50.507 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.507 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.507 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.507 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.508 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.508 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:84219 11.b 11:d750bcf9:::benchmark_data_smithi125_439543_object84218:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T11:08:50.508 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:06:20.504974+0000", 2023-10-09T11:08:50.508 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 149.89762751800001, 2023-10-09T11:08:50.508 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12751848499999999, 2023-10-09T11:08:50.508 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.508 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.508 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:08:50.508 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:08:50.508 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:50.508 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 84219 2023-10-09T11:08:50.509 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.509 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.509 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.509 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.509 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.504974+0000", 2023-10-09T11:08:50.509 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.509 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.509 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.509 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.509 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.504974+0000", 2023-10-09T11:08:50.509 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.509 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.509 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.510 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.510 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.504972+0000", 2023-10-09T11:08:50.510 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:08:50.510 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.510 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.510 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.510 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.504994+0000", 2023-10-09T11:08:50.510 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2175999999999999e-05 2023-10-09T11:08:50.510 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.510 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.510 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.510 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505004+0000", 2023-10-09T11:08:50.511 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0035e-05 2023-10-09T11:08:50.511 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.511 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.511 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.511 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505010+0000", 2023-10-09T11:08:50.511 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0580000000000003e-06 2023-10-09T11:08:50.511 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.511 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.511 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.511 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.506161+0000", 2023-10-09T11:08:50.511 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011510800000000001 2023-10-09T11:08:50.511 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.511 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.512 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.512 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.506300+0000", 2023-10-09T11:08:50.512 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013864699999999999 2023-10-09T11:08:50.512 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.512 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.512 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,1", 2023-10-09T11:08:50.512 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.506447+0000", 2023-10-09T11:08:50.512 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014706699999999999 2023-10-09T11:08:50.512 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.512 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.512 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:08:50.512 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.630287+0000", 2023-10-09T11:08:50.513 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.123840197 2023-10-09T11:08:50.513 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.513 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.513 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.513 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.631531+0000", 2023-10-09T11:08:50.513 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012432350000000001 2023-10-09T11:08:50.513 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.513 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.513 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.513 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632417+0000", 2023-10-09T11:08:50.513 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00088668199999999996 2023-10-09T11:08:50.513 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.514 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.514 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.514 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632460+0000", 2023-10-09T11:08:50.514 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3158999999999998e-05 2023-10-09T11:08:50.514 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.514 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.514 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.514 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632492+0000", 2023-10-09T11:08:50.514 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1985e-05 2023-10-09T11:08:50.514 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.514 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.514 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.515 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.515 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.515 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:84223 11.11 11:89bbe302:::benchmark_data_smithi125_439543_object84222:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T11:08:50.515 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:06:20.505039+0000", 2023-10-09T11:08:50.515 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 149.897562064, 2023-10-09T11:08:50.515 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12827670299999999, 2023-10-09T11:08:50.515 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.515 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.515 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:08:50.515 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:08:50.515 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:50.516 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 84223 2023-10-09T11:08:50.516 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.516 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.516 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.516 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.516 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505039+0000", 2023-10-09T11:08:50.516 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.516 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.516 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.516 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.516 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505039+0000", 2023-10-09T11:08:50.516 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.516 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.517 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.517 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.517 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505039+0000", 2023-10-09T11:08:50.517 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:50.517 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.517 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.517 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.517 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505065+0000", 2023-10-09T11:08:50.517 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6179000000000001e-05 2023-10-09T11:08:50.517 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.517 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.517 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.518 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505071+0000", 2023-10-09T11:08:50.518 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9349999999999999e-06 2023-10-09T11:08:50.518 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.518 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.518 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.518 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505075+0000", 2023-10-09T11:08:50.518 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9639999999999997e-06 2023-10-09T11:08:50.518 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.518 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.518 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.518 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505119+0000", 2023-10-09T11:08:50.518 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4150999999999998e-05 2023-10-09T11:08:50.519 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.519 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.519 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.519 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505291+0000", 2023-10-09T11:08:50.519 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017220500000000001 2023-10-09T11:08:50.519 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.519 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.519 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:08:50.519 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505402+0000", 2023-10-09T11:08:50.519 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011069399999999999 2023-10-09T11:08:50.519 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.519 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.519 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:08:50.520 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632096+0000", 2023-10-09T11:08:50.520 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.126693951 2023-10-09T11:08:50.520 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.520 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.520 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.520 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632163+0000", 2023-10-09T11:08:50.520 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7118000000000004e-05 2023-10-09T11:08:50.520 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.520 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.520 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.520 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632252+0000", 2023-10-09T11:08:50.520 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9019999999999998e-05 2023-10-09T11:08:50.521 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.521 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.521 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.521 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632614+0000", 2023-10-09T11:08:50.521 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00036246999999999999 2023-10-09T11:08:50.521 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.521 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.521 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.521 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.633316+0000", 2023-10-09T11:08:50.521 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00070155499999999997 2023-10-09T11:08:50.521 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.521 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.522 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.522 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.522 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.522 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:84226 11.11 11:8d2cd1ab:::benchmark_data_smithi125_439543_object84225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T11:08:50.522 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:06:20.505181+0000", 2023-10-09T11:08:50.522 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 149.89742006200001, 2023-10-09T11:08:50.522 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.129217903, 2023-10-09T11:08:50.522 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.522 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.522 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:08:50.522 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:08:50.522 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:50.522 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 84226 2023-10-09T11:08:50.523 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.523 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.523 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.523 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.523 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505181+0000", 2023-10-09T11:08:50.523 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.523 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.523 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.523 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.523 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505181+0000", 2023-10-09T11:08:50.523 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.523 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.524 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.524 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.524 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505181+0000", 2023-10-09T11:08:50.524 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:50.524 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.524 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.524 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.524 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505200+0000", 2023-10-09T11:08:50.524 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9017000000000001e-05 2023-10-09T11:08:50.524 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.524 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.524 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.524 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505220+0000", 2023-10-09T11:08:50.525 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9729e-05 2023-10-09T11:08:50.525 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.525 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.525 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.525 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505225+0000", 2023-10-09T11:08:50.525 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7969999999999997e-06 2023-10-09T11:08:50.525 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.525 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.525 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.525 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.506648+0000", 2023-10-09T11:08:50.525 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014222219999999999 2023-10-09T11:08:50.525 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.525 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.526 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.526 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.506725+0000", 2023-10-09T11:08:50.526 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7541999999999997e-05 2023-10-09T11:08:50.526 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.526 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.526 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T11:08:50.526 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.506815+0000", 2023-10-09T11:08:50.526 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0074999999999994e-05 2023-10-09T11:08:50.526 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.526 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.526 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:08:50.526 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632116+0000", 2023-10-09T11:08:50.527 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.125300404 2023-10-09T11:08:50.527 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.527 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.527 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.527 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632204+0000", 2023-10-09T11:08:50.527 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7865e-05 2023-10-09T11:08:50.527 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.527 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.527 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.527 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.633770+0000", 2023-10-09T11:08:50.527 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015663960000000001 2023-10-09T11:08:50.527 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.528 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.528 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.528 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.633826+0000", 2023-10-09T11:08:50.528 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5597000000000003e-05 2023-10-09T11:08:50.528 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.528 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.528 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.528 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.634399+0000", 2023-10-09T11:08:50.528 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00057372100000000002 2023-10-09T11:08:50.528 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.528 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.528 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.528 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.529 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.529 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:84230 11.b 11:d191ae1b:::benchmark_data_smithi125_439543_object84229:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T11:08:50.529 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:06:20.505234+0000", 2023-10-09T11:08:50.529 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 149.89736691100001, 2023-10-09T11:08:50.529 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.13026402200000001, 2023-10-09T11:08:50.529 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:08:50.529 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:08:50.529 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:08:50.529 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:08:50.529 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:08:50.529 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 84230 2023-10-09T11:08:50.529 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.530 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:08:50.530 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.530 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:08:50.530 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505234+0000", 2023-10-09T11:08:50.530 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.530 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.530 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.530 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:08:50.530 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505234+0000", 2023-10-09T11:08:50.530 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:08:50.530 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.530 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.531 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:08:50.531 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505234+0000", 2023-10-09T11:08:50.531 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:08:50.531 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.531 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.531 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:08:50.531 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505256+0000", 2023-10-09T11:08:50.531 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2177000000000001e-05 2023-10-09T11:08:50.531 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.531 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.531 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:08:50.531 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505263+0000", 2023-10-09T11:08:50.532 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7530000000000004e-06 2023-10-09T11:08:50.532 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.532 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.532 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:08:50.532 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.505269+0000", 2023-10-09T11:08:50.532 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7359999999999998e-06 2023-10-09T11:08:50.532 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.532 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.532 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:08:50.532 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.508763+0000", 2023-10-09T11:08:50.532 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0034948259999999999 2023-10-09T11:08:50.532 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.533 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.533 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:08:50.533 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.508857+0000", 2023-10-09T11:08:50.533 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3800000000000003e-05 2023-10-09T11:08:50.533 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.533 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.533 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,1", 2023-10-09T11:08:50.533 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.508957+0000", 2023-10-09T11:08:50.533 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9436000000000001e-05 2023-10-09T11:08:50.533 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.533 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.533 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.534 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.631561+0000", 2023-10-09T11:08:50.534 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.122603957 2023-10-09T11:08:50.534 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.534 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.534 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:08:50.534 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.632510+0000", 2023-10-09T11:08:50.534 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00094982900000000002 2023-10-09T11:08:50.534 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.534 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.534 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:08:50.534 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.635417+0000", 2023-10-09T11:08:50.534 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029067490000000001 2023-10-09T11:08:50.534 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.535 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.535 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:08:50.535 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.635461+0000", 2023-10-09T11:08:50.535 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3571e-05 2023-10-09T11:08:50.535 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:08:50.535 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:08:50.535 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:08:50.535 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:06:20.635499+0000", 2023-10-09T11:08:50.535 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7709000000000001e-05 2023-10-09T11:08:50.535 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.535 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.535 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.536 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:08:50.536 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:08:50.536 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:08:50.536 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:50 smithi155 ceph-mon[184854]: failed to encode map e571 with expected crc 2023-10-09T11:08:50.536 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:50 smithi155 ceph-mon[184854]: failed to encode map e571 with expected crc 2023-10-09T11:08:50.537 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror_image_status (2726 ms) 2023-10-09T11:08:50.537 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T11:08:50.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:50.538 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:50.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:50.538 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:08:50.539 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:08:50.539 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:08:50.539 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:08:50.539 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:08:50.539 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:08:50.539 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:08:50.540 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror_image_map (105 ms) 2023-10-09T11:08:50.541 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T11:08:50.547 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror_instances (7 ms) 2023-10-09T11:08:50.548 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T11:08:50.554 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:08:50.555 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:08:50.555 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:08:50.556 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:08:50.556 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:08:50.556 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:08:50.556 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:08:50.556 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:08:50.556 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:08:50.558 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:50.560 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror_snapshot (13 ms) 2023-10-09T11:08:50.560 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T11:08:50.562 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-10-09T11:08:50.563 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T11:08:50.564 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-10-09T11:08:50.564 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T11:08:50.567 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-10-09T11:08:50.567 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T11:08:50.572 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_dir_rename (5 ms) 2023-10-09T11:08:50.573 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T11:08:50.575 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-10-09T11:08:50.576 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T11:08:50.580 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-10-09T11:08:50.580 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T11:08:50.582 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2023-10-09T11:08:50.582 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T11:08:50.586 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-10-09T11:08:50.586 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T11:08:50.588 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-10-09T11:08:50.588 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T11:08:50.593 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2023-10-09T11:08:50.593 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T11:08:50.596 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2023-10-09T11:08:50.596 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T11:08:50.599 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.image_group_remove (4 ms) 2023-10-09T11:08:50.599 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T11:08:50.602 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-10-09T11:08:50.602 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T11:08:50.605 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-10-09T11:08:50.605 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T11:08:50.606 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-10-09T11:08:50.606 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T11:08:50.608 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-10-09T11:08:50.608 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T11:08:50.611 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-10-09T11:08:50.611 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T11:08:50.614 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_set (3 ms) 2023-10-09T11:08:50.614 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T11:08:50.621 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_list (6 ms) 2023-10-09T11:08:50.621 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T11:08:50.639 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:50.650 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (30 ms) 2023-10-09T11:08:50.650 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T11:08:50.675 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:50.874 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (224 ms) 2023-10-09T11:08:50.875 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T11:08:50.877 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2023-10-09T11:08:50.877 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T11:08:50.879 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-10-09T11:08:50.879 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T11:08:50.887 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.trash_methods (8 ms) 2023-10-09T11:08:50.887 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T11:08:50.893 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.op_features (6 ms) 2023-10-09T11:08:50.893 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T11:08:50.913 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.clone_parent (20 ms) 2023-10-09T11:08:50.914 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T11:08:50.921 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.clone_parent_ns (7 ms) 2023-10-09T11:08:50.921 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T11:08:50.932 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.clone_child (11 ms) 2023-10-09T11:08:50.932 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T11:08:50.938 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.namespace_methods (6 ms) 2023-10-09T11:08:50.938 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T11:08:50.947 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.migration (9 ms) 2023-10-09T11:08:50.947 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T11:08:50.955 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.migration_v1 (8 ms) 2023-10-09T11:08:50.955 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T11:08:51.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:51 smithi155 ceph-mon[184854]: pgmap v1400: 135 pgs: 6 creating+peering, 11 unknown, 118 active+clean; 582 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 309 KiB/s rd, 180 KiB/s wr, 512 op/s 2023-10-09T11:08:51.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/94359600' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:51.489 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:51 smithi125 ceph-mon[195507]: pgmap v1400: 135 pgs: 6 creating+peering, 11 unknown, 118 active+clean; 582 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 309 KiB/s rd, 180 KiB/s wr, 512 op/s 2023-10-09T11:08:51.489 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/94359600' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:51.489 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:08:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:08:51] "GET /metrics HTTP/1.1" 200 34109 "" "Prometheus/2.43.0" 2023-10-09T11:08:51.490 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:51 smithi125 ceph-mon[200197]: pgmap v1400: 135 pgs: 6 creating+peering, 11 unknown, 118 active+clean; 582 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 309 KiB/s rd, 180 KiB/s wr, 512 op/s 2023-10-09T11:08:51.490 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/94359600' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:52.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T11:08:52.104 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:52 smithi125 ceph-mon[195507]: failed to encode map e572 with expected crc 2023-10-09T11:08:52.104 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:52 smithi125 ceph-mon[195507]: osdmap e572: 8 total, 8 up, 8 in 2023-10-09T11:08:52.104 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:52 smithi125 ceph-mon[195507]: failed to encode map e572 with expected crc 2023-10-09T11:08:52.104 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:52 smithi125 ceph-mon[200197]: failed to encode map e572 with expected crc 2023-10-09T11:08:52.104 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:52 smithi125 ceph-mon[200197]: osdmap e572: 8 total, 8 up, 8 in 2023-10-09T11:08:52.104 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:52 smithi125 ceph-mon[200197]: failed to encode map e572 with expected crc 2023-10-09T11:08:52.107 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1153 ms) 2023-10-09T11:08:52.107 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T11:08:52.146 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.sparsify (39 ms) 2023-10-09T11:08:52.202 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:52.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:52.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:52.307 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:52.338 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:52 smithi155 ceph-mon[184854]: failed to encode map e572 with expected crc 2023-10-09T11:08:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:52 smithi155 ceph-mon[184854]: osdmap e572: 8 total, 8 up, 8 in 2023-10-09T11:08:52.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:52 smithi155 ceph-mon[184854]: failed to encode map e572 with expected crc 2023-10-09T11:08:52.708 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:52.711 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:52.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:52.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:52.815 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:52.844 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:52.863 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:52.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:52.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:52.967 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:53.002 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:53.023 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:53.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:53.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:53.125 INFO:tasks.workunit.client.0.smithi125.stdout:[----------] 70 tests from TestClsRbd (5066 ms total) 2023-10-09T11:08:53.125 INFO:tasks.workunit.client.0.smithi125.stdout: 2023-10-09T11:08:53.125 INFO:tasks.workunit.client.0.smithi125.stdout:[----------] Global test environment tear-down 2023-10-09T11:08:53.125 INFO:tasks.workunit.client.0.smithi125.stdout:[==========] 70 tests from 1 test suite ran. (7950 ms total) 2023-10-09T11:08:53.125 INFO:tasks.workunit.client.0.smithi125.stdout:[ PASSED ] 70 tests. 2023-10-09T11:08:53.125 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T11:08:53.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T11:08:53.128 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:53.131 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:08:53.157 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:53.161 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T11:08:53.161 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T11:08:53.176 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:53.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:53.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:53.280 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:53.306 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:53 smithi155 ceph-mon[184854]: pgmap v1402: 135 pgs: 135 active+clean; 582 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.0 MiB/s rd, 936 KiB/s wr, 717 op/s 2023-10-09T11:08:53.307 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:53 smithi155 ceph-mon[184854]: failed to encode map e573 with expected crc 2023-10-09T11:08:53.307 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:53 smithi155 ceph-mon[184854]: failed to encode map e573 with expected crc 2023-10-09T11:08:53.307 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:53 smithi155 ceph-mon[184854]: osdmap e573: 8 total, 8 up, 8 in 2023-10-09T11:08:53.307 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:53 smithi155 ceph-mon[184854]: failed to encode map e573 with expected crc 2023-10-09T11:08:53.307 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:53 smithi155 ceph-mon[184854]: failed to encode map e573 with expected crc 2023-10-09T11:08:53.307 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:53 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/448570434' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:53.308 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:53.328 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:53.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:53.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:53.398 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:08:53.399 INFO:teuthology.orchestra.run.smithi125.stdout:{"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1719,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":574,"num_osds":8,"num_up_osds":8,"osd_up_since":1696849454,"num_in_osds":8,"osd_in_since":1696849519,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":135}],"num_pgs":135,"num_pools":5,"num_objects":70,"data_bytes":596246,"bytes_used":7669063680,"bytes_avail":760291221504,"bytes_total":767960285184,"read_bytes_sec":2045373,"write_bytes_sec":958063,"read_op_per_sec":471,"write_op_per_sec":245},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi125.satgiw","status":"up:active","gid":24575}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T11:05:13.785232+0000","services":{}},"progress_events":{"585851ac-6e70-4462-9176-ac50987fab77":{"message":"Upgrade to 18.2.0-598-gb4fa663f (3m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true},"df0fb881-12cb-49d8-8731-ace6d4046a03":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true}}} 2023-10-09T11:08:53.423 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:53 smithi125 ceph-mon[195507]: pgmap v1402: 135 pgs: 135 active+clean; 582 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.0 MiB/s rd, 936 KiB/s wr, 717 op/s 2023-10-09T11:08:53.423 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:53 smithi125 ceph-mon[195507]: failed to encode map e573 with expected crc 2023-10-09T11:08:53.424 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:53 smithi125 ceph-mon[195507]: failed to encode map e573 with expected crc 2023-10-09T11:08:53.424 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:53 smithi125 ceph-mon[195507]: osdmap e573: 8 total, 8 up, 8 in 2023-10-09T11:08:53.424 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:53 smithi125 ceph-mon[195507]: failed to encode map e573 with expected crc 2023-10-09T11:08:53.424 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:53 smithi125 ceph-mon[195507]: failed to encode map e573 with expected crc 2023-10-09T11:08:53.424 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:53 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/448570434' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:53.424 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:53 smithi125 ceph-mon[200197]: pgmap v1402: 135 pgs: 135 active+clean; 582 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2.0 MiB/s rd, 936 KiB/s wr, 717 op/s 2023-10-09T11:08:53.424 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:53 smithi125 ceph-mon[200197]: failed to encode map e573 with expected crc 2023-10-09T11:08:53.424 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:53 smithi125 ceph-mon[200197]: failed to encode map e573 with expected crc 2023-10-09T11:08:53.424 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:53 smithi125 ceph-mon[200197]: osdmap e573: 8 total, 8 up, 8 in 2023-10-09T11:08:53.424 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:53 smithi125 ceph-mon[200197]: failed to encode map e573 with expected crc 2023-10-09T11:08:53.425 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:53 smithi125 ceph-mon[200197]: failed to encode map e573 with expected crc 2023-10-09T11:08:53.425 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:53 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/448570434' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:08:53.431 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:53.458 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:53.476 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:53.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:53.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:53.580 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:53.608 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:53.678 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:53.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:53.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:53.752 DEBUG:teuthology.parallel:result is None 2023-10-09T11:08:53.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T11:08:53.781 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T11:08:53.781 DEBUG:teuthology.orchestra.run.smithi125:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T11:08:53.782 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:53.839 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T11:08:53.839 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T11:08:53.839 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T11:08:53.839 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T11:08:53.840 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T11:08:53.840 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T11:08:53.840 DEBUG:teuthology.orchestra.run.smithi125:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T11:08:53.848 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:53.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T11:08:53.867 INFO:teuthology.orchestra.run.smithi125.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T11:08:53.868 DEBUG:teuthology.orchestra.run.smithi125:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T11:08:53.926 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T11:08:53.927 DEBUG:teuthology.orchestra.run.smithi125:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T11:08:53.985 INFO:tasks.workunit:timeout=3h 2023-10-09T11:08:53.985 INFO:tasks.workunit:cleanup=True 2023-10-09T11:08:53.985 DEBUG:teuthology.orchestra.run.smithi125:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T11:08:54.043 INFO:tasks.workunit.client.0.smithi125.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T11:08:54.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:08:54.252 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:54 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:08:54.252 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:54 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/448570434' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:08:54.252 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:54 smithi125 ceph-mon[195507]: osdmap e574: 8 total, 8 up, 8 in 2023-10-09T11:08:54.253 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:54 smithi125 ceph-mon[195507]: failed to encode map e574 with expected crc 2023-10-09T11:08:54.253 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:54 smithi125 ceph-mon[195507]: failed to encode map e574 with expected crc 2023-10-09T11:08:54.253 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:54 smithi125 ceph-mon[195507]: failed to encode map e574 with expected crc 2023-10-09T11:08:54.253 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:54 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/854285082' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T11:08:54.253 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:54 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:08:54.253 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:54 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/448570434' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:08:54.253 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:54 smithi125 ceph-mon[200197]: osdmap e574: 8 total, 8 up, 8 in 2023-10-09T11:08:54.253 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:54 smithi125 ceph-mon[200197]: failed to encode map e574 with expected crc 2023-10-09T11:08:54.253 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:54 smithi125 ceph-mon[200197]: failed to encode map e574 with expected crc 2023-10-09T11:08:54.253 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:54 smithi125 ceph-mon[200197]: failed to encode map e574 with expected crc 2023-10-09T11:08:54.254 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:54 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/854285082' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T11:08:54.321 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:54.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:54.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:54.336 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-10-09T11:08:54.336 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:08:54.428 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:54.461 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:54 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:08:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:54 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/448570434' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:08:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:54 smithi155 ceph-mon[184854]: osdmap e574: 8 total, 8 up, 8 in 2023-10-09T11:08:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:54 smithi155 ceph-mon[184854]: failed to encode map e574 with expected crc 2023-10-09T11:08:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:54 smithi155 ceph-mon[184854]: failed to encode map e574 with expected crc 2023-10-09T11:08:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:54 smithi155 ceph-mon[184854]: failed to encode map e574 with expected crc 2023-10-09T11:08:54.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:54 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/854285082' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T11:08:54.530 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:54.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:54.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:54.637 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:54.671 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:54.691 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:54.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:54.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:54.737 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:08:54.737 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:08:54.796 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:54.825 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:54.847 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:54.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:54.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:54.951 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:54.978 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:55.048 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:55.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:55.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:55.074 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:55.108 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:55 smithi125 ceph-mon[200197]: failed to encode map e574 with expected crc 2023-10-09T11:08:55.108 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:55 smithi125 ceph-mon[200197]: pgmap v1405: 103 pgs: 103 active+clean; 582 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:55.109 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:55 smithi125 ceph-mon[195507]: failed to encode map e574 with expected crc 2023-10-09T11:08:55.109 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:55 smithi125 ceph-mon[195507]: pgmap v1405: 103 pgs: 103 active+clean; 582 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:55.152 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:55.185 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:55.206 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:55.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:55.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:55.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:55.312 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T11:08:55.337 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:55 smithi155 ceph-mon[184854]: failed to encode map e574 with expected crc 2023-10-09T11:08:55.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:55 smithi155 ceph-mon[184854]: pgmap v1405: 103 pgs: 103 active+clean; 582 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:55.608 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:55.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:55.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:55.712 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:55.744 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:55.765 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:55.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:55.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:55.869 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:55.899 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:55.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:55.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:55.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:56.023 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:08:56.052 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:56.071 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:08:56.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:56.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:08:56.102 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:56.175 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:08:56.202 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:56.203 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:56.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:08:56.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:56.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:08:56.317 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:08:56.325 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:56.351 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:56.369 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:56.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:56.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:56.473 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:08:56.515 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:56.588 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:08:56.590 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:08:56.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:08:56.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:56.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:08:56.959 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:56.989 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:57.012 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:57.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:57.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:57.116 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:08:57.143 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:57.205 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:57.205 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:56 smithi125 ceph-mon[195507]: pgmap v1406: 103 pgs: 103 active+clean; 582 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:57.205 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:57.205 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:08:57.205 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:08:57.205 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:56 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:57.205 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:56 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:57.205 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:56 smithi125 ceph-mon[200197]: pgmap v1406: 103 pgs: 103 active+clean; 582 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:57.206 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:56 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:57.206 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:56 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:08:57.206 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:56 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:08:57.206 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:56 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:56 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:56 smithi155 ceph-mon[184854]: pgmap v1406: 103 pgs: 103 active+clean; 582 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:56 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:56 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:08:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:56 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:08:57.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:56 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:08:57.314 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:08:57.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:57.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:08:57.434 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:57.468 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:57.647 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:08:57.647 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:08:57.648 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:08:57.648 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:08:57.648 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:08:57.648 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:08:57.648 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:08:57.648 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:08:57.648 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:08:57.648 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:08:57.648 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:08:57.649 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:08:57.649 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:08:57.649 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:08:57.649 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:08:57.649 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:08:57.978 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:57.994 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:57.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:57.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:58.098 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:58.122 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:57 smithi125 ceph-mon[195507]: from='client.56677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:58.122 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:57 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3361177964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:58.122 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:57 smithi125 ceph-mon[200197]: from='client.56677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:58.122 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:57 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3361177964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:58.131 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:58.154 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:58.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:58.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:58.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:57 smithi155 ceph-mon[184854]: from='client.56677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:08:58.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:57 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3361177964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:08:58.258 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:58.261 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1406,"stamp":"2023-10-09T11:08:55.933522+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":2377214,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5046,"ondisk_log_size":5046,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":377,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7405928,"kb_used_data":349288,"kb_used_omap":134,"kb_used_meta":7056441,"kb_avail":742555288,"statfs":{"total":767960285184,"available":760376614912,"internally_reserved":0,"allocated":357670912,"data_stored":337929396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":137532,"internal_metadata":7225796292},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-58,"num_objects":-14,"num_object_clones":0,"num_object_copies":-42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-14,"num_whiteouts":0,"num_read":-1935,"num_read_kb":-1853,"num_write":-1139,"num_write_kb":-1081,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-11,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001630"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":861,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.108383+0000","last_change":"2023-10-09T11:07:36.287455+0000","last_active":"2023-10-09T11:08:52.108383+0000","last_peered":"2023-10-09T11:08:52.108383+0000","last_clean":"2023-10-09T11:08:52.108383+0000","last_became_active":"2023-10-09T11:07:36.287052+0000","last_became_peered":"2023-10-09T11:07:36.287052+0000","last_unstale":"2023-10-09T11:08:52.108383+0000","last_undegraded":"2023-10-09T11:08:52.108383+0000","last_fullsized":"2023-10-09T11:08:52.108383+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:18.864873+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":356,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.369626+0000","last_change":"2023-10-09T11:03:21.185383+0000","last_active":"2023-10-09T11:08:52.369626+0000","last_peered":"2023-10-09T11:08:52.369626+0000","last_clean":"2023-10-09T11:08:52.369626+0000","last_became_active":"2023-10-09T11:03:21.185022+0000","last_became_peered":"2023-10-09T11:03:21.185022+0000","last_unstale":"2023-10-09T11:08:52.369626+0000","last_undegraded":"2023-10-09T11:08:52.369626+0000","last_fullsized":"2023-10-09T11:08:52.369626+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:05.230434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":100,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.107605+0000","last_change":"2023-10-09T11:04:50.319477+0000","last_active":"2023-10-09T11:08:52.107605+0000","last_peered":"2023-10-09T11:08:52.107605+0000","last_clean":"2023-10-09T11:08:52.107605+0000","last_became_active":"2023-10-09T11:04:50.319148+0000","last_became_peered":"2023-10-09T11:04:50.319148+0000","last_unstale":"2023-10-09T11:08:52.107605+0000","last_undegraded":"2023-10-09T11:08:52.107605+0000","last_fullsized":"2023-10-09T11:08:52.107605+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:48.725566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":731,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109164+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:08:52.109164+0000","last_peered":"2023-10-09T11:08:52.109164+0000","last_clean":"2023-10-09T11:08:52.109164+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:08:52.109164+0000","last_undegraded":"2023-10-09T11:08:52.109164+0000","last_fullsized":"2023-10-09T11:08:52.109164+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":287,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.662674+0000","last_change":"2023-10-09T11:04:50.333711+0000","last_active":"2023-10-09T11:08:51.662674+0000","last_peered":"2023-10-09T11:08:51.662674+0000","last_clean":"2023-10-09T11:08:51.662674+0000","last_became_active":"2023-10-09T11:04:50.333262+0000","last_became_peered":"2023-10-09T11:04:50.333262+0000","last_unstale":"2023-10-09T11:08:51.662674+0000","last_undegraded":"2023-10-09T11:08:51.662674+0000","last_fullsized":"2023-10-09T11:08:51.662674+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.843765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":34,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366724+0000","last_change":"2023-10-09T11:07:33.237903+0000","last_active":"2023-10-09T11:08:52.366724+0000","last_peered":"2023-10-09T11:08:52.366724+0000","last_clean":"2023-10-09T11:08:52.366724+0000","last_became_active":"2023-10-09T11:07:33.237618+0000","last_became_peered":"2023-10-09T11:07:33.237618+0000","last_unstale":"2023-10-09T11:08:52.366724+0000","last_undegraded":"2023-10-09T11:08:52.366724+0000","last_fullsized":"2023-10-09T11:08:52.366724+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:25.600818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":749,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.365872+0000","last_change":"2023-10-09T11:07:36.284534+0000","last_active":"2023-10-09T11:08:52.365872+0000","last_peered":"2023-10-09T11:08:52.365872+0000","last_clean":"2023-10-09T11:08:52.365872+0000","last_became_active":"2023-10-09T11:07:36.284254+0000","last_became_peered":"2023-10-09T11:07:36.284254+0000","last_unstale":"2023-10-09T11:08:52.365872+0000","last_undegraded":"2023-10-09T11:08:52.365872+0000","last_fullsized":"2023-10-09T11:08:52.365872+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:19.159630+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":100,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.106558+0000","last_change":"2023-10-09T11:04:50.310477+0000","last_active":"2023-10-09T11:08:52.106558+0000","last_peered":"2023-10-09T11:08:52.106558+0000","last_clean":"2023-10-09T11:08:52.106558+0000","last_became_active":"2023-10-09T11:04:50.310067+0000","last_became_peered":"2023-10-09T11:04:50.310067+0000","last_unstale":"2023-10-09T11:08:52.106558+0000","last_undegraded":"2023-10-09T11:08:52.106558+0000","last_fullsized":"2023-10-09T11:08:52.106558+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:52.884716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":383,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109922+0000","last_change":"2023-10-09T11:05:20.310581+0000","last_active":"2023-10-09T11:08:52.109922+0000","last_peered":"2023-10-09T11:08:52.109922+0000","last_clean":"2023-10-09T11:08:52.109922+0000","last_became_active":"2023-10-09T11:05:20.309830+0000","last_became_peered":"2023-10-09T11:05:20.309830+0000","last_unstale":"2023-10-09T11:08:52.109922+0000","last_undegraded":"2023-10-09T11:08:52.109922+0000","last_fullsized":"2023-10-09T11:08:52.109922+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:24.508211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":789,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366454+0000","last_change":"2023-10-09T11:07:36.282502+0000","last_active":"2023-10-09T11:08:52.366454+0000","last_peered":"2023-10-09T11:08:52.366454+0000","last_clean":"2023-10-09T11:08:52.366454+0000","last_became_active":"2023-10-09T11:07:36.282133+0000","last_became_peered":"2023-10-09T11:07:36.282133+0000","last_unstale":"2023-10-09T11:08:52.366454+0000","last_undegraded":"2023-10-09T11:08:52.366454+0000","last_fullsized":"2023-10-09T11:08:52.366454+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:03.762383+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":210,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.110417+0000","last_change":"2023-10-09T11:06:24.245488+0000","last_active":"2023-10-09T11:08:52.110417+0000","last_peered":"2023-10-09T11:08:52.110417+0000","last_clean":"2023-10-09T11:08:52.110417+0000","last_became_active":"2023-10-09T11:06:24.245167+0000","last_became_peered":"2023-10-09T11:06:24.245167+0000","last_unstale":"2023-10-09T11:08:52.110417+0000","last_undegraded":"2023-10-09T11:08:52.110417+0000","last_fullsized":"2023-10-09T11:08:52.110417+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:31.702568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":109,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.106657+0000","last_change":"2023-10-09T11:05:20.299453+0000","last_active":"2023-10-09T11:08:52.106657+0000","last_peered":"2023-10-09T11:08:52.106657+0000","last_clean":"2023-10-09T11:08:52.106657+0000","last_became_active":"2023-10-09T11:05:20.299117+0000","last_became_peered":"2023-10-09T11:05:20.299117+0000","last_unstale":"2023-10-09T11:08:52.106657+0000","last_undegraded":"2023-10-09T11:08:52.106657+0000","last_fullsized":"2023-10-09T11:08:52.106657+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:30.072813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":901,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T11:08:47.087956+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:08:47.087956+0000","last_peered":"2023-10-09T11:08:47.087956+0000","last_clean":"2023-10-09T11:08:47.087956+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:08:47.087956+0000","last_undegraded":"2023-10-09T11:08:47.087956+0000","last_fullsized":"2023-10-09T11:08:47.087956+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":445,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109259+0000","last_change":"2023-10-09T11:07:33.242665+0000","last_active":"2023-10-09T11:08:52.109259+0000","last_peered":"2023-10-09T11:08:52.109259+0000","last_clean":"2023-10-09T11:08:52.109259+0000","last_became_active":"2023-10-09T11:07:33.242313+0000","last_became_peered":"2023-10-09T11:07:33.242313+0000","last_unstale":"2023-10-09T11:08:52.109259+0000","last_undegraded":"2023-10-09T11:08:52.109259+0000","last_fullsized":"2023-10-09T11:08:52.109259+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:20.565135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":127,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.106844+0000","last_change":"2023-10-09T11:07:33.225683+0000","last_active":"2023-10-09T11:08:52.106844+0000","last_peered":"2023-10-09T11:08:52.106844+0000","last_clean":"2023-10-09T11:08:52.106844+0000","last_became_active":"2023-10-09T11:07:33.225275+0000","last_became_peered":"2023-10-09T11:07:33.225275+0000","last_unstale":"2023-10-09T11:08:52.106844+0000","last_undegraded":"2023-10-09T11:08:52.106844+0000","last_fullsized":"2023-10-09T11:08:52.106844+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:10.131060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":803,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109437+0000","last_change":"2023-10-09T11:05:23.328981+0000","last_active":"2023-10-09T11:08:52.109437+0000","last_peered":"2023-10-09T11:08:52.109437+0000","last_clean":"2023-10-09T11:08:52.109437+0000","last_became_active":"2023-10-09T11:05:23.328616+0000","last_became_peered":"2023-10-09T11:05:23.328616+0000","last_unstale":"2023-10-09T11:08:52.109437+0000","last_undegraded":"2023-10-09T11:08:52.109437+0000","last_fullsized":"2023-10-09T11:08:52.109437+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:15.315819+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":40,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366409+0000","last_change":"2023-10-09T11:07:33.265842+0000","last_active":"2023-10-09T11:08:52.366409+0000","last_peered":"2023-10-09T11:08:52.366409+0000","last_clean":"2023-10-09T11:08:52.366409+0000","last_became_active":"2023-10-09T11:07:33.232311+0000","last_became_peered":"2023-10-09T11:07:33.232311+0000","last_unstale":"2023-10-09T11:08:52.366409+0000","last_undegraded":"2023-10-09T11:08:52.366409+0000","last_fullsized":"2023-10-09T11:08:52.366409+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:23.402582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":352,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109835+0000","last_change":"2023-10-09T11:05:20.316265+0000","last_active":"2023-10-09T11:08:52.109835+0000","last_peered":"2023-10-09T11:08:52.109835+0000","last_clean":"2023-10-09T11:08:52.109835+0000","last_became_active":"2023-10-09T11:05:20.315853+0000","last_became_peered":"2023-10-09T11:05:20.315853+0000","last_unstale":"2023-10-09T11:08:52.109835+0000","last_undegraded":"2023-10-09T11:08:52.109835+0000","last_fullsized":"2023-10-09T11:08:52.109835+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:35.093981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":965,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.663402+0000","last_change":"2023-10-09T11:04:53.274921+0000","last_active":"2023-10-09T11:08:51.663402+0000","last_peered":"2023-10-09T11:08:51.663402+0000","last_clean":"2023-10-09T11:08:51.663402+0000","last_became_active":"2023-10-09T11:04:53.274518+0000","last_became_peered":"2023-10-09T11:04:53.274518+0000","last_unstale":"2023-10-09T11:08:51.663402+0000","last_undegraded":"2023-10-09T11:08:51.663402+0000","last_fullsized":"2023-10-09T11:08:51.663402+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:14.315301+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":44,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366387+0000","last_change":"2023-10-09T11:07:33.262559+0000","last_active":"2023-10-09T11:08:52.366387+0000","last_peered":"2023-10-09T11:08:52.366387+0000","last_clean":"2023-10-09T11:08:52.366387+0000","last_became_active":"2023-10-09T11:07:33.231695+0000","last_became_peered":"2023-10-09T11:07:33.231695+0000","last_unstale":"2023-10-09T11:08:52.366387+0000","last_undegraded":"2023-10-09T11:08:52.366387+0000","last_fullsized":"2023-10-09T11:08:52.366387+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:59.006940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":592,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T11:08:47.087819+0000","last_change":"2023-10-09T11:05:20.311424+0000","last_active":"2023-10-09T11:08:47.087819+0000","last_peered":"2023-10-09T11:08:47.087819+0000","last_clean":"2023-10-09T11:08:47.087819+0000","last_became_active":"2023-10-09T11:05:20.311097+0000","last_became_peered":"2023-10-09T11:05:20.311097+0000","last_unstale":"2023-10-09T11:08:47.087819+0000","last_undegraded":"2023-10-09T11:08:47.087819+0000","last_fullsized":"2023-10-09T11:08:47.087819+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:19.274582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":616,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109969+0000","last_change":"2023-10-09T11:08:39.934938+0000","last_active":"2023-10-09T11:08:52.109969+0000","last_peered":"2023-10-09T11:08:52.109969+0000","last_clean":"2023-10-09T11:08:52.109969+0000","last_became_active":"2023-10-09T11:06:24.245076+0000","last_became_peered":"2023-10-09T11:06:24.245076+0000","last_unstale":"2023-10-09T11:08:52.109969+0000","last_undegraded":"2023-10-09T11:08:52.109969+0000","last_fullsized":"2023-10-09T11:08:52.109969+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.934865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:00.408009+0000","scrub_duration":0.0012831839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":40,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366128+0000","last_change":"2023-10-09T11:07:33.269200+0000","last_active":"2023-10-09T11:08:52.366128+0000","last_peered":"2023-10-09T11:08:52.366128+0000","last_clean":"2023-10-09T11:08:52.366128+0000","last_became_active":"2023-10-09T11:07:33.236205+0000","last_became_peered":"2023-10-09T11:07:33.236205+0000","last_unstale":"2023-10-09T11:08:52.366128+0000","last_undegraded":"2023-10-09T11:08:52.366128+0000","last_fullsized":"2023-10-09T11:08:52.366128+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:24.574734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":118,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.107530+0000","last_change":"2023-10-09T11:07:33.227803+0000","last_active":"2023-10-09T11:08:52.107530+0000","last_peered":"2023-10-09T11:08:52.107530+0000","last_clean":"2023-10-09T11:08:52.107530+0000","last_became_active":"2023-10-09T11:07:33.227486+0000","last_became_peered":"2023-10-09T11:07:33.227486+0000","last_unstale":"2023-10-09T11:08:52.107530+0000","last_undegraded":"2023-10-09T11:08:52.107530+0000","last_fullsized":"2023-10-09T11:08:52.107530+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:39.274875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":885,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109524+0000","last_change":"2023-10-09T11:05:23.323856+0000","last_active":"2023-10-09T11:08:52.109524+0000","last_peered":"2023-10-09T11:08:52.109524+0000","last_clean":"2023-10-09T11:08:52.109524+0000","last_became_active":"2023-10-09T11:05:23.322763+0000","last_became_peered":"2023-10-09T11:05:23.322763+0000","last_unstale":"2023-10-09T11:08:52.109524+0000","last_undegraded":"2023-10-09T11:08:52.109524+0000","last_fullsized":"2023-10-09T11:08:52.109524+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.760820+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":34,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.365853+0000","last_change":"2023-10-09T11:07:33.233510+0000","last_active":"2023-10-09T11:08:52.365853+0000","last_peered":"2023-10-09T11:08:52.365853+0000","last_clean":"2023-10-09T11:08:52.365853+0000","last_became_active":"2023-10-09T11:07:33.232339+0000","last_became_peered":"2023-10-09T11:07:33.232339+0000","last_unstale":"2023-10-09T11:08:52.365853+0000","last_undegraded":"2023-10-09T11:08:52.365853+0000","last_fullsized":"2023-10-09T11:08:52.365853+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:55.324912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":515,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109498+0000","last_change":"2023-10-09T11:07:33.226594+0000","last_active":"2023-10-09T11:08:52.109498+0000","last_peered":"2023-10-09T11:08:52.109498+0000","last_clean":"2023-10-09T11:08:52.109498+0000","last_became_active":"2023-10-09T11:07:33.226275+0000","last_became_peered":"2023-10-09T11:07:33.226275+0000","last_unstale":"2023-10-09T11:08:52.109498+0000","last_undegraded":"2023-10-09T11:08:52.109498+0000","last_fullsized":"2023-10-09T11:08:52.109498+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:41.180088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":906,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.662960+0000","last_change":"2023-10-09T11:04:53.274236+0000","last_active":"2023-10-09T11:08:51.662960+0000","last_peered":"2023-10-09T11:08:51.662960+0000","last_clean":"2023-10-09T11:08:51.662960+0000","last_became_active":"2023-10-09T11:04:53.273817+0000","last_became_peered":"2023-10-09T11:04:53.273817+0000","last_unstale":"2023-10-09T11:08:51.662960+0000","last_undegraded":"2023-10-09T11:08:51.662960+0000","last_fullsized":"2023-10-09T11:08:51.662960+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:05.096905+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":169,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.365810+0000","last_change":"2023-10-09T11:05:20.312129+0000","last_active":"2023-10-09T11:08:52.365810+0000","last_peered":"2023-10-09T11:08:52.365810+0000","last_clean":"2023-10-09T11:08:52.365810+0000","last_became_active":"2023-10-09T11:05:20.311810+0000","last_became_peered":"2023-10-09T11:05:20.311810+0000","last_unstale":"2023-10-09T11:08:52.365810+0000","last_undegraded":"2023-10-09T11:08:52.365810+0000","last_fullsized":"2023-10-09T11:08:52.365810+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:48.813391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":268,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-10-09T11:08:53.123000+0000","last_change":"2023-10-09T11:07:33.218244+0000","last_active":"2023-10-09T11:08:53.123000+0000","last_peered":"2023-10-09T11:08:53.123000+0000","last_clean":"2023-10-09T11:08:53.123000+0000","last_became_active":"2023-10-09T11:07:33.217918+0000","last_became_peered":"2023-10-09T11:07:33.217918+0000","last_unstale":"2023-10-09T11:08:53.123000+0000","last_undegraded":"2023-10-09T11:08:53.123000+0000","last_fullsized":"2023-10-09T11:08:53.123000+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:16.150105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":882,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.110267+0000","last_change":"2023-10-09T11:05:23.324537+0000","last_active":"2023-10-09T11:08:52.110267+0000","last_peered":"2023-10-09T11:08:52.110267+0000","last_clean":"2023-10-09T11:08:52.110267+0000","last_became_active":"2023-10-09T11:05:23.324187+0000","last_became_peered":"2023-10-09T11:05:23.324187+0000","last_unstale":"2023-10-09T11:08:52.110267+0000","last_undegraded":"2023-10-09T11:08:52.110267+0000","last_fullsized":"2023-10-09T11:08:52.110267+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:25.286106+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":94,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.110575+0000","last_change":"2023-10-09T11:05:20.323565+0000","last_active":"2023-10-09T11:08:52.110575+0000","last_peered":"2023-10-09T11:08:52.110575+0000","last_clean":"2023-10-09T11:08:52.110575+0000","last_became_active":"2023-10-09T11:05:20.309672+0000","last_became_peered":"2023-10-09T11:05:20.309672+0000","last_unstale":"2023-10-09T11:08:52.110575+0000","last_undegraded":"2023-10-09T11:08:52.110575+0000","last_fullsized":"2023-10-09T11:08:52.110575+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:50.211465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":118,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.107273+0000","last_change":"2023-10-09T11:07:33.228001+0000","last_active":"2023-10-09T11:08:52.107273+0000","last_peered":"2023-10-09T11:08:52.107273+0000","last_clean":"2023-10-09T11:08:52.107273+0000","last_became_active":"2023-10-09T11:07:33.227691+0000","last_became_peered":"2023-10-09T11:07:33.227691+0000","last_unstale":"2023-10-09T11:08:52.107273+0000","last_undegraded":"2023-10-09T11:08:52.107273+0000","last_fullsized":"2023-10-09T11:08:52.107273+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:24:16.098138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":806,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366565+0000","last_change":"2023-10-09T11:07:36.284927+0000","last_active":"2023-10-09T11:08:52.366565+0000","last_peered":"2023-10-09T11:08:52.366565+0000","last_clean":"2023-10-09T11:08:52.366565+0000","last_became_active":"2023-10-09T11:07:36.284631+0000","last_became_peered":"2023-10-09T11:07:36.284631+0000","last_unstale":"2023-10-09T11:08:52.366565+0000","last_undegraded":"2023-10-09T11:08:52.366565+0000","last_fullsized":"2023-10-09T11:08:52.366565+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:37.220243+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":38,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366592+0000","last_change":"2023-10-09T11:07:33.273503+0000","last_active":"2023-10-09T11:08:52.366592+0000","last_peered":"2023-10-09T11:08:52.366592+0000","last_clean":"2023-10-09T11:08:52.366592+0000","last_became_active":"2023-10-09T11:07:33.242980+0000","last_became_peered":"2023-10-09T11:07:33.242980+0000","last_unstale":"2023-10-09T11:08:52.366592+0000","last_undegraded":"2023-10-09T11:08:52.366592+0000","last_fullsized":"2023-10-09T11:08:52.366592+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:18.934972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":536,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.108500+0000","last_change":"2023-10-09T11:07:33.304363+0000","last_active":"2023-10-09T11:08:52.108500+0000","last_peered":"2023-10-09T11:08:52.108500+0000","last_clean":"2023-10-09T11:08:52.108500+0000","last_became_active":"2023-10-09T11:07:33.304024+0000","last_became_peered":"2023-10-09T11:07:33.304024+0000","last_unstale":"2023-10-09T11:08:52.108500+0000","last_undegraded":"2023-10-09T11:08:52.108500+0000","last_fullsized":"2023-10-09T11:08:52.108500+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:21:51.522688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":955,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.107618+0000","last_change":"2023-10-09T11:05:23.316129+0000","last_active":"2023-10-09T11:08:52.107618+0000","last_peered":"2023-10-09T11:08:52.107618+0000","last_clean":"2023-10-09T11:08:52.107618+0000","last_became_active":"2023-10-09T11:05:23.315792+0000","last_became_peered":"2023-10-09T11:05:23.315792+0000","last_unstale":"2023-10-09T11:08:52.107618+0000","last_undegraded":"2023-10-09T11:08:52.107618+0000","last_fullsized":"2023-10-09T11:08:52.107618+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:51.770547+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":38,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366828+0000","last_change":"2023-10-09T11:07:33.256691+0000","last_active":"2023-10-09T11:08:52.366828+0000","last_peered":"2023-10-09T11:08:52.366828+0000","last_clean":"2023-10-09T11:08:52.366828+0000","last_became_active":"2023-10-09T11:07:33.229554+0000","last_became_peered":"2023-10-09T11:07:33.229554+0000","last_unstale":"2023-10-09T11:08:52.366828+0000","last_undegraded":"2023-10-09T11:08:52.366828+0000","last_fullsized":"2023-10-09T11:08:52.366828+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:57.414795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":288,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.110255+0000","last_change":"2023-10-09T11:04:50.333538+0000","last_active":"2023-10-09T11:08:52.110255+0000","last_peered":"2023-10-09T11:08:52.110255+0000","last_clean":"2023-10-09T11:08:52.110255+0000","last_became_active":"2023-10-09T11:04:50.332514+0000","last_became_peered":"2023-10-09T11:04:50.332514+0000","last_unstale":"2023-10-09T11:08:52.110255+0000","last_undegraded":"2023-10-09T11:08:52.110255+0000","last_fullsized":"2023-10-09T11:08:52.110255+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:22.933839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":830,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109706+0000","last_change":"2023-10-09T11:05:23.322251+0000","last_active":"2023-10-09T11:08:52.109706+0000","last_peered":"2023-10-09T11:08:52.109706+0000","last_clean":"2023-10-09T11:08:52.109706+0000","last_became_active":"2023-10-09T11:05:23.321826+0000","last_became_peered":"2023-10-09T11:05:23.321826+0000","last_unstale":"2023-10-09T11:08:52.109706+0000","last_undegraded":"2023-10-09T11:08:52.109706+0000","last_fullsized":"2023-10-09T11:08:52.109706+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.858013+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":38,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366822+0000","last_change":"2023-10-09T11:07:33.271238+0000","last_active":"2023-10-09T11:08:52.366822+0000","last_peered":"2023-10-09T11:08:52.366822+0000","last_clean":"2023-10-09T11:08:52.366822+0000","last_became_active":"2023-10-09T11:07:33.241886+0000","last_became_peered":"2023-10-09T11:07:33.241886+0000","last_unstale":"2023-10-09T11:08:52.366822+0000","last_undegraded":"2023-10-09T11:08:52.366822+0000","last_fullsized":"2023-10-09T11:08:52.366822+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:48.882589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":104,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109222+0000","last_change":"2023-10-09T11:07:33.244131+0000","last_active":"2023-10-09T11:08:52.109222+0000","last_peered":"2023-10-09T11:08:52.109222+0000","last_clean":"2023-10-09T11:08:52.109222+0000","last_became_active":"2023-10-09T11:07:33.243797+0000","last_became_peered":"2023-10-09T11:07:33.243797+0000","last_unstale":"2023-10-09T11:08:52.109222+0000","last_undegraded":"2023-10-09T11:08:52.109222+0000","last_fullsized":"2023-10-09T11:08:52.109222+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:44.851043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":1084,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366973+0000","last_change":"2023-10-09T11:07:36.286417+0000","last_active":"2023-10-09T11:08:52.366973+0000","last_peered":"2023-10-09T11:08:52.366973+0000","last_clean":"2023-10-09T11:08:52.366973+0000","last_became_active":"2023-10-09T11:07:36.286061+0000","last_became_peered":"2023-10-09T11:07:36.286061+0000","last_unstale":"2023-10-09T11:08:52.366973+0000","last_undegraded":"2023-10-09T11:08:52.366973+0000","last_fullsized":"2023-10-09T11:08:52.366973+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:00.289500+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":432,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.110432+0000","last_change":"2023-10-09T11:05:20.309385+0000","last_active":"2023-10-09T11:08:52.110432+0000","last_peered":"2023-10-09T11:08:52.110432+0000","last_clean":"2023-10-09T11:08:52.110432+0000","last_became_active":"2023-10-09T11:05:20.309034+0000","last_became_peered":"2023-10-09T11:05:20.309034+0000","last_unstale":"2023-10-09T11:08:52.110432+0000","last_undegraded":"2023-10-09T11:08:52.110432+0000","last_fullsized":"2023-10-09T11:08:52.110432+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:31.857485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":363,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.662921+0000","last_change":"2023-10-09T11:06:24.245966+0000","last_active":"2023-10-09T11:08:51.662921+0000","last_peered":"2023-10-09T11:08:51.662921+0000","last_clean":"2023-10-09T11:08:51.662921+0000","last_became_active":"2023-10-09T11:06:24.245310+0000","last_became_peered":"2023-10-09T11:06:24.245310+0000","last_unstale":"2023-10-09T11:08:51.662921+0000","last_undegraded":"2023-10-09T11:08:51.662921+0000","last_fullsized":"2023-10-09T11:08:51.662921+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:23.966200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":728,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.365711+0000","last_change":"2023-10-09T11:07:36.283174+0000","last_active":"2023-10-09T11:08:52.365711+0000","last_peered":"2023-10-09T11:08:52.365711+0000","last_clean":"2023-10-09T11:08:52.365711+0000","last_became_active":"2023-10-09T11:07:36.278007+0000","last_became_peered":"2023-10-09T11:07:36.278007+0000","last_unstale":"2023-10-09T11:08:52.365711+0000","last_undegraded":"2023-10-09T11:08:52.365711+0000","last_fullsized":"2023-10-09T11:08:52.365711+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:51.693969+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":104,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.108265+0000","last_change":"2023-10-09T11:04:50.529513+0000","last_active":"2023-10-09T11:08:52.108265+0000","last_peered":"2023-10-09T11:08:52.108265+0000","last_clean":"2023-10-09T11:08:52.108265+0000","last_became_active":"2023-10-09T11:04:50.339671+0000","last_became_peered":"2023-10-09T11:04:50.339671+0000","last_unstale":"2023-10-09T11:08:52.108265+0000","last_undegraded":"2023-10-09T11:08:52.108265+0000","last_fullsized":"2023-10-09T11:08:52.108265+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:24.473000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":109,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.107444+0000","last_change":"2023-10-09T11:05:20.313855+0000","last_active":"2023-10-09T11:08:52.107444+0000","last_peered":"2023-10-09T11:08:52.107444+0000","last_clean":"2023-10-09T11:08:52.107444+0000","last_became_active":"2023-10-09T11:05:20.313535+0000","last_became_peered":"2023-10-09T11:05:20.313535+0000","last_unstale":"2023-10-09T11:08:52.107444+0000","last_undegraded":"2023-10-09T11:08:52.107444+0000","last_fullsized":"2023-10-09T11:08:52.107444+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:25.266434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":832,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109408+0000","last_change":"2023-10-09T11:04:53.272323+0000","last_active":"2023-10-09T11:08:52.109408+0000","last_peered":"2023-10-09T11:08:52.109408+0000","last_clean":"2023-10-09T11:08:52.109408+0000","last_became_active":"2023-10-09T11:04:53.272010+0000","last_became_peered":"2023-10-09T11:04:53.272010+0000","last_unstale":"2023-10-09T11:08:52.109408+0000","last_undegraded":"2023-10-09T11:08:52.109408+0000","last_fullsized":"2023-10-09T11:08:52.109408+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:23.913605+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":596,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.662763+0000","last_change":"2023-10-09T11:05:20.307499+0000","last_active":"2023-10-09T11:08:51.662763+0000","last_peered":"2023-10-09T11:08:51.662763+0000","last_clean":"2023-10-09T11:08:51.662763+0000","last_became_active":"2023-10-09T11:05:20.307198+0000","last_became_peered":"2023-10-09T11:05:20.307198+0000","last_unstale":"2023-10-09T11:08:51.662763+0000","last_undegraded":"2023-10-09T11:08:51.662763+0000","last_fullsized":"2023-10-09T11:08:51.662763+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:53.726844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":330,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T11:08:47.087874+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:08:47.087874+0000","last_peered":"2023-10-09T11:08:47.087874+0000","last_clean":"2023-10-09T11:08:47.087874+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:08:47.087874+0000","last_undegraded":"2023-10-09T11:08:47.087874+0000","last_fullsized":"2023-10-09T11:08:47.087874+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":923,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.663019+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:08:51.663019+0000","last_peered":"2023-10-09T11:08:51.663019+0000","last_clean":"2023-10-09T11:08:51.663019+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:08:51.663019+0000","last_undegraded":"2023-10-09T11:08:51.663019+0000","last_fullsized":"2023-10-09T11:08:51.663019+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":593,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T11:08:47.087544+0000","last_change":"2023-10-09T11:04:57.953307+0000","last_active":"2023-10-09T11:08:47.087544+0000","last_peered":"2023-10-09T11:08:47.087544+0000","last_clean":"2023-10-09T11:08:47.087544+0000","last_became_active":"2023-10-09T11:04:57.953095+0000","last_became_peered":"2023-10-09T11:04:57.953095+0000","last_unstale":"2023-10-09T11:08:47.087544+0000","last_undegraded":"2023-10-09T11:08:47.087544+0000","last_fullsized":"2023-10-09T11:08:47.087544+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:05.562643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":281,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.663285+0000","last_change":"2023-10-09T11:05:20.310255+0000","last_active":"2023-10-09T11:08:51.663285+0000","last_peered":"2023-10-09T11:08:51.663285+0000","last_clean":"2023-10-09T11:08:51.663285+0000","last_became_active":"2023-10-09T11:05:20.309109+0000","last_became_peered":"2023-10-09T11:05:20.309109+0000","last_unstale":"2023-10-09T11:08:51.663285+0000","last_undegraded":"2023-10-09T11:08:51.663285+0000","last_fullsized":"2023-10-09T11:08:51.663285+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:07.429039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":816,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.110167+0000","last_change":"2023-10-09T11:05:23.329329+0000","last_active":"2023-10-09T11:08:52.110167+0000","last_peered":"2023-10-09T11:08:52.110167+0000","last_clean":"2023-10-09T11:08:52.110167+0000","last_became_active":"2023-10-09T11:05:23.328976+0000","last_became_peered":"2023-10-09T11:05:23.328976+0000","last_unstale":"2023-10-09T11:08:52.110167+0000","last_undegraded":"2023-10-09T11:08:52.110167+0000","last_fullsized":"2023-10-09T11:08:52.110167+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:41.848184+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":118,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.107500+0000","last_change":"2023-10-09T11:07:33.236133+0000","last_active":"2023-10-09T11:08:52.107500+0000","last_peered":"2023-10-09T11:08:52.107500+0000","last_clean":"2023-10-09T11:08:52.107500+0000","last_became_active":"2023-10-09T11:07:33.235818+0000","last_became_peered":"2023-10-09T11:07:33.235818+0000","last_unstale":"2023-10-09T11:08:52.107500+0000","last_undegraded":"2023-10-09T11:08:52.107500+0000","last_fullsized":"2023-10-09T11:08:52.107500+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:20.627846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":278,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.663321+0000","last_change":"2023-10-09T11:04:50.332978+0000","last_active":"2023-10-09T11:08:51.663321+0000","last_peered":"2023-10-09T11:08:51.663321+0000","last_clean":"2023-10-09T11:08:51.663321+0000","last_became_active":"2023-10-09T11:04:50.332735+0000","last_became_peered":"2023-10-09T11:04:50.332735+0000","last_unstale":"2023-10-09T11:08:51.663321+0000","last_undegraded":"2023-10-09T11:08:51.663321+0000","last_fullsized":"2023-10-09T11:08:51.663321+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:39:44.659751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":1100,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.107327+0000","last_change":"2023-10-09T11:07:36.291716+0000","last_active":"2023-10-09T11:08:52.107327+0000","last_peered":"2023-10-09T11:08:52.107327+0000","last_clean":"2023-10-09T11:08:52.107327+0000","last_became_active":"2023-10-09T11:07:36.291424+0000","last_became_peered":"2023-10-09T11:07:36.291424+0000","last_unstale":"2023-10-09T11:08:52.107327+0000","last_undegraded":"2023-10-09T11:08:52.107327+0000","last_fullsized":"2023-10-09T11:08:52.107327+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:22.316387+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":303,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.663038+0000","last_change":"2023-10-09T11:07:33.226547+0000","last_active":"2023-10-09T11:08:51.663038+0000","last_peered":"2023-10-09T11:08:51.663038+0000","last_clean":"2023-10-09T11:08:51.663038+0000","last_became_active":"2023-10-09T11:07:33.226176+0000","last_became_peered":"2023-10-09T11:07:33.226176+0000","last_unstale":"2023-10-09T11:08:51.663038+0000","last_undegraded":"2023-10-09T11:08:51.663038+0000","last_fullsized":"2023-10-09T11:08:51.663038+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:16.888732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":449,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109333+0000","last_change":"2023-10-09T11:04:50.341353+0000","last_active":"2023-10-09T11:08:52.109333+0000","last_peered":"2023-10-09T11:08:52.109333+0000","last_clean":"2023-10-09T11:08:52.109333+0000","last_became_active":"2023-10-09T11:04:50.340801+0000","last_became_peered":"2023-10-09T11:04:50.340801+0000","last_unstale":"2023-10-09T11:08:52.109333+0000","last_undegraded":"2023-10-09T11:08:52.109333+0000","last_fullsized":"2023-10-09T11:08:52.109333+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:00.699069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":255,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.662833+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:08:51.662833+0000","last_peered":"2023-10-09T11:08:51.662833+0000","last_clean":"2023-10-09T11:08:51.662833+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:08:51.662833+0000","last_undegraded":"2023-10-09T11:08:51.662833+0000","last_fullsized":"2023-10-09T11:08:51.662833+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":774,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109877+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:08:52.109877+0000","last_peered":"2023-10-09T11:08:52.109877+0000","last_clean":"2023-10-09T11:08:52.109877+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:08:52.109877+0000","last_undegraded":"2023-10-09T11:08:52.109877+0000","last_fullsized":"2023-10-09T11:08:52.109877+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":456,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109929+0000","last_change":"2023-10-09T11:05:20.316093+0000","last_active":"2023-10-09T11:08:52.109929+0000","last_peered":"2023-10-09T11:08:52.109929+0000","last_clean":"2023-10-09T11:08:52.109929+0000","last_became_active":"2023-10-09T11:05:20.315471+0000","last_became_peered":"2023-10-09T11:05:20.315471+0000","last_unstale":"2023-10-09T11:08:52.109929+0000","last_undegraded":"2023-10-09T11:08:52.109929+0000","last_fullsized":"2023-10-09T11:08:52.109929+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:47.548763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":480,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.108899+0000","last_change":"2023-10-09T11:06:24.245282+0000","last_active":"2023-10-09T11:08:52.108899+0000","last_peered":"2023-10-09T11:08:52.108899+0000","last_clean":"2023-10-09T11:08:52.108899+0000","last_became_active":"2023-10-09T11:06:24.244981+0000","last_became_peered":"2023-10-09T11:06:24.244981+0000","last_unstale":"2023-10-09T11:08:52.108899+0000","last_undegraded":"2023-10-09T11:08:52.108899+0000","last_fullsized":"2023-10-09T11:08:52.108899+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:15.327655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":288,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.662956+0000","last_change":"2023-10-09T11:04:50.311579+0000","last_active":"2023-10-09T11:08:51.662956+0000","last_peered":"2023-10-09T11:08:51.662956+0000","last_clean":"2023-10-09T11:08:51.662956+0000","last_became_active":"2023-10-09T11:04:50.311270+0000","last_became_peered":"2023-10-09T11:04:50.311270+0000","last_unstale":"2023-10-09T11:08:51.662956+0000","last_undegraded":"2023-10-09T11:08:51.662956+0000","last_fullsized":"2023-10-09T11:08:51.662956+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:49.850302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":926,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366390+0000","last_change":"2023-10-09T11:07:36.284280+0000","last_active":"2023-10-09T11:08:52.366390+0000","last_peered":"2023-10-09T11:08:52.366390+0000","last_clean":"2023-10-09T11:08:52.366390+0000","last_became_active":"2023-10-09T11:07:36.283966+0000","last_became_peered":"2023-10-09T11:07:36.283966+0000","last_unstale":"2023-10-09T11:08:52.366390+0000","last_undegraded":"2023-10-09T11:08:52.366390+0000","last_fullsized":"2023-10-09T11:08:52.366390+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:54.128331+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":350,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-09T11:08:48.083390+0000","last_change":"2023-10-09T11:04:50.381744+0000","last_active":"2023-10-09T11:08:48.083390+0000","last_peered":"2023-10-09T11:08:48.083390+0000","last_clean":"2023-10-09T11:08:48.083390+0000","last_became_active":"2023-10-09T11:04:50.381253+0000","last_became_peered":"2023-10-09T11:04:50.381253+0000","last_unstale":"2023-10-09T11:08:48.083390+0000","last_undegraded":"2023-10-09T11:08:48.083390+0000","last_fullsized":"2023-10-09T11:08:48.083390+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:24.030274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":273,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.662683+0000","last_change":"2023-10-09T11:05:20.298542+0000","last_active":"2023-10-09T11:08:51.662683+0000","last_peered":"2023-10-09T11:08:51.662683+0000","last_clean":"2023-10-09T11:08:51.662683+0000","last_became_active":"2023-10-09T11:05:20.298119+0000","last_became_peered":"2023-10-09T11:05:20.298119+0000","last_unstale":"2023-10-09T11:08:51.662683+0000","last_undegraded":"2023-10-09T11:08:51.662683+0000","last_fullsized":"2023-10-09T11:08:51.662683+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:00.827214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":253,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.112436+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:08:52.112436+0000","last_peered":"2023-10-09T11:08:52.112436+0000","last_clean":"2023-10-09T11:08:52.112436+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:08:52.112436+0000","last_undegraded":"2023-10-09T11:08:52.112436+0000","last_fullsized":"2023-10-09T11:08:52.112436+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":449,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.110295+0000","last_change":"2023-10-09T11:07:33.227052+0000","last_active":"2023-10-09T11:08:52.110295+0000","last_peered":"2023-10-09T11:08:52.110295+0000","last_clean":"2023-10-09T11:08:52.110295+0000","last_became_active":"2023-10-09T11:07:33.226289+0000","last_became_peered":"2023-10-09T11:07:33.226289+0000","last_unstale":"2023-10-09T11:08:52.110295+0000","last_undegraded":"2023-10-09T11:08:52.110295+0000","last_fullsized":"2023-10-09T11:08:52.110295+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:15.605946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":871,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.365755+0000","last_change":"2023-10-09T11:07:36.290700+0000","last_active":"2023-10-09T11:08:52.365755+0000","last_peered":"2023-10-09T11:08:52.365755+0000","last_clean":"2023-10-09T11:08:52.365755+0000","last_became_active":"2023-10-09T11:07:36.290357+0000","last_became_peered":"2023-10-09T11:07:36.290357+0000","last_unstale":"2023-10-09T11:08:52.365755+0000","last_undegraded":"2023-10-09T11:08:52.365755+0000","last_fullsized":"2023-10-09T11:08:52.365755+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:34.494777+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":109,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.107046+0000","last_change":"2023-10-09T11:05:20.310666+0000","last_active":"2023-10-09T11:08:52.107046+0000","last_peered":"2023-10-09T11:08:52.107046+0000","last_clean":"2023-10-09T11:08:52.107046+0000","last_became_active":"2023-10-09T11:05:20.310413+0000","last_became_peered":"2023-10-09T11:05:20.310413+0000","last_unstale":"2023-10-09T11:08:52.107046+0000","last_undegraded":"2023-10-09T11:08:52.107046+0000","last_fullsized":"2023-10-09T11:08:52.107046+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:34.169905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":534,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366666+0000","last_change":"2023-10-09T11:05:20.316523+0000","last_active":"2023-10-09T11:08:52.366666+0000","last_peered":"2023-10-09T11:08:52.366666+0000","last_clean":"2023-10-09T11:08:52.366666+0000","last_became_active":"2023-10-09T11:05:20.314493+0000","last_became_peered":"2023-10-09T11:05:20.314493+0000","last_unstale":"2023-10-09T11:08:52.366666+0000","last_undegraded":"2023-10-09T11:08:52.366666+0000","last_fullsized":"2023-10-09T11:08:52.366666+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:41.618701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":944,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.662874+0000","last_change":"2023-10-09T11:05:23.322074+0000","last_active":"2023-10-09T11:08:51.662874+0000","last_peered":"2023-10-09T11:08:51.662874+0000","last_clean":"2023-10-09T11:08:51.662874+0000","last_became_active":"2023-10-09T11:05:23.321749+0000","last_became_peered":"2023-10-09T11:05:23.321749+0000","last_unstale":"2023-10-09T11:08:51.662874+0000","last_undegraded":"2023-10-09T11:08:51.662874+0000","last_fullsized":"2023-10-09T11:08:51.662874+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:07.010932+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":149,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366005+0000","last_change":"2023-10-09T11:02:33.198484+0000","last_active":"2023-10-09T11:08:52.366005+0000","last_peered":"2023-10-09T11:08:52.366005+0000","last_clean":"2023-10-09T11:08:52.366005+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:08:52.366005+0000","last_undegraded":"2023-10-09T11:08:52.366005+0000","last_fullsized":"2023-10-09T11:08:52.366005+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":201,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.106622+0000","last_change":"2023-10-09T11:06:24.248362+0000","last_active":"2023-10-09T11:08:52.106622+0000","last_peered":"2023-10-09T11:08:52.106622+0000","last_clean":"2023-10-09T11:08:52.106622+0000","last_became_active":"2023-10-09T11:06:24.247348+0000","last_became_peered":"2023-10-09T11:06:24.247348+0000","last_unstale":"2023-10-09T11:08:52.106622+0000","last_undegraded":"2023-10-09T11:08:52.106622+0000","last_fullsized":"2023-10-09T11:08:52.106622+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:26.728754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":399,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.663128+0000","last_change":"2023-10-09T11:06:24.253511+0000","last_active":"2023-10-09T11:08:51.663128+0000","last_peered":"2023-10-09T11:08:51.663128+0000","last_clean":"2023-10-09T11:08:51.663128+0000","last_became_active":"2023-10-09T11:06:24.252739+0000","last_became_peered":"2023-10-09T11:06:24.252739+0000","last_unstale":"2023-10-09T11:08:51.663128+0000","last_undegraded":"2023-10-09T11:08:51.663128+0000","last_fullsized":"2023-10-09T11:08:51.663128+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:50.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":805,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.106921+0000","last_change":"2023-10-09T11:06:24.239676+0000","last_active":"2023-10-09T11:08:52.106921+0000","last_peered":"2023-10-09T11:08:52.106921+0000","last_clean":"2023-10-09T11:08:52.106921+0000","last_became_active":"2023-10-09T11:06:24.239375+0000","last_became_peered":"2023-10-09T11:06:24.239375+0000","last_unstale":"2023-10-09T11:08:52.106921+0000","last_undegraded":"2023-10-09T11:08:52.106921+0000","last_fullsized":"2023-10-09T11:08:52.106921+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:27.953053+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":34,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366285+0000","last_change":"2023-10-09T11:07:33.229415+0000","last_active":"2023-10-09T11:08:52.366285+0000","last_peered":"2023-10-09T11:08:52.366285+0000","last_clean":"2023-10-09T11:08:52.366285+0000","last_became_active":"2023-10-09T11:07:33.229061+0000","last_became_peered":"2023-10-09T11:07:33.229061+0000","last_unstale":"2023-10-09T11:08:52.366285+0000","last_undegraded":"2023-10-09T11:08:52.366285+0000","last_fullsized":"2023-10-09T11:08:52.366285+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:30.978293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":255,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.663190+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:08:51.663190+0000","last_peered":"2023-10-09T11:08:51.663190+0000","last_clean":"2023-10-09T11:08:51.663190+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:08:51.663190+0000","last_undegraded":"2023-10-09T11:08:51.663190+0000","last_fullsized":"2023-10-09T11:08:51.663190+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":533,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109335+0000","last_change":"2023-10-09T11:07:33.227271+0000","last_active":"2023-10-09T11:08:52.109335+0000","last_peered":"2023-10-09T11:08:52.109335+0000","last_clean":"2023-10-09T11:08:52.109335+0000","last_became_active":"2023-10-09T11:07:33.226760+0000","last_became_peered":"2023-10-09T11:07:33.226760+0000","last_unstale":"2023-10-09T11:08:52.109335+0000","last_undegraded":"2023-10-09T11:08:52.109335+0000","last_fullsized":"2023-10-09T11:08:52.109335+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:12.268123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":770,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T11:08:47.088039+0000","last_change":"2023-10-09T11:05:23.323368+0000","last_active":"2023-10-09T11:08:47.088039+0000","last_peered":"2023-10-09T11:08:47.088039+0000","last_clean":"2023-10-09T11:08:47.088039+0000","last_became_active":"2023-10-09T11:05:23.323032+0000","last_became_peered":"2023-10-09T11:05:23.323032+0000","last_unstale":"2023-10-09T11:08:47.088039+0000","last_undegraded":"2023-10-09T11:08:47.088039+0000","last_fullsized":"2023-10-09T11:08:47.088039+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:09.393584+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":130,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.107215+0000","last_change":"2023-10-09T11:07:33.246378+0000","last_active":"2023-10-09T11:08:52.107215+0000","last_peered":"2023-10-09T11:08:52.107215+0000","last_clean":"2023-10-09T11:08:52.107215+0000","last_became_active":"2023-10-09T11:07:33.228019+0000","last_became_peered":"2023-10-09T11:07:33.228019+0000","last_unstale":"2023-10-09T11:08:52.107215+0000","last_undegraded":"2023-10-09T11:08:52.107215+0000","last_fullsized":"2023-10-09T11:08:52.107215+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:18.511517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":282,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.111720+0000","last_change":"2023-10-09T11:05:20.310417+0000","last_active":"2023-10-09T11:08:52.111720+0000","last_peered":"2023-10-09T11:08:52.111720+0000","last_clean":"2023-10-09T11:08:52.111720+0000","last_became_active":"2023-10-09T11:05:20.309827+0000","last_became_peered":"2023-10-09T11:05:20.309827+0000","last_unstale":"2023-10-09T11:08:52.111720+0000","last_undegraded":"2023-10-09T11:08:52.111720+0000","last_fullsized":"2023-10-09T11:08:52.111720+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:23.934383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":119,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-10-09T11:08:53.120627+0000","last_change":"2023-10-09T11:07:33.217595+0000","last_active":"2023-10-09T11:08:53.120627+0000","last_peered":"2023-10-09T11:08:53.120627+0000","last_clean":"2023-10-09T11:08:53.120627+0000","last_became_active":"2023-10-09T11:07:33.217155+0000","last_became_peered":"2023-10-09T11:07:33.217155+0000","last_unstale":"2023-10-09T11:08:53.120627+0000","last_undegraded":"2023-10-09T11:08:53.120627+0000","last_fullsized":"2023-10-09T11:08:53.120627+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:55.127329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":737,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.107215+0000","last_change":"2023-10-09T11:07:36.284491+0000","last_active":"2023-10-09T11:08:52.107215+0000","last_peered":"2023-10-09T11:08:52.107215+0000","last_clean":"2023-10-09T11:08:52.107215+0000","last_became_active":"2023-10-09T11:07:36.284249+0000","last_became_peered":"2023-10-09T11:07:36.284249+0000","last_unstale":"2023-10-09T11:08:52.107215+0000","last_undegraded":"2023-10-09T11:08:52.107215+0000","last_fullsized":"2023-10-09T11:08:52.107215+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:03.653371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:03.653371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:09.629417+0000","scrub_duration":0.001576771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":34,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366820+0000","last_change":"2023-10-09T11:07:33.241346+0000","last_active":"2023-10-09T11:08:52.366820+0000","last_peered":"2023-10-09T11:08:52.366820+0000","last_clean":"2023-10-09T11:08:52.366820+0000","last_became_active":"2023-10-09T11:07:33.241013+0000","last_became_peered":"2023-10-09T11:07:33.241013+0000","last_unstale":"2023-10-09T11:08:52.366820+0000","last_undegraded":"2023-10-09T11:08:52.366820+0000","last_fullsized":"2023-10-09T11:08:52.366820+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:57.164502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":272,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.663235+0000","last_change":"2023-10-09T11:06:24.253714+0000","last_active":"2023-10-09T11:08:51.663235+0000","last_peered":"2023-10-09T11:08:51.663235+0000","last_clean":"2023-10-09T11:08:51.663235+0000","last_became_active":"2023-10-09T11:06:24.253143+0000","last_became_peered":"2023-10-09T11:06:24.253143+0000","last_unstale":"2023-10-09T11:08:51.663235+0000","last_undegraded":"2023-10-09T11:08:51.663235+0000","last_fullsized":"2023-10-09T11:08:51.663235+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:05.682288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":379,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109781+0000","last_change":"2023-10-09T11:05:20.313560+0000","last_active":"2023-10-09T11:08:52.109781+0000","last_peered":"2023-10-09T11:08:52.109781+0000","last_clean":"2023-10-09T11:08:52.109781+0000","last_became_active":"2023-10-09T11:05:20.313214+0000","last_became_peered":"2023-10-09T11:05:20.313214+0000","last_unstale":"2023-10-09T11:08:52.109781+0000","last_undegraded":"2023-10-09T11:08:52.109781+0000","last_fullsized":"2023-10-09T11:08:52.109781+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:32.527205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":981,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.106882+0000","last_change":"2023-10-09T11:05:23.329886+0000","last_active":"2023-10-09T11:08:52.106882+0000","last_peered":"2023-10-09T11:08:52.106882+0000","last_clean":"2023-10-09T11:08:52.106882+0000","last_became_active":"2023-10-09T11:05:23.329561+0000","last_became_peered":"2023-10-09T11:05:23.329561+0000","last_unstale":"2023-10-09T11:08:52.106882+0000","last_undegraded":"2023-10-09T11:08:52.106882+0000","last_fullsized":"2023-10-09T11:08:52.106882+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:46.933799+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":34,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366341+0000","last_change":"2023-10-09T11:07:33.229864+0000","last_active":"2023-10-09T11:08:52.366341+0000","last_peered":"2023-10-09T11:08:52.366341+0000","last_clean":"2023-10-09T11:08:52.366341+0000","last_became_active":"2023-10-09T11:07:33.229547+0000","last_became_peered":"2023-10-09T11:07:33.229547+0000","last_unstale":"2023-10-09T11:08:52.366341+0000","last_undegraded":"2023-10-09T11:08:52.366341+0000","last_fullsized":"2023-10-09T11:08:52.366341+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:34.990439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":353,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T11:08:47.088100+0000","last_change":"2023-10-09T11:04:50.510688+0000","last_active":"2023-10-09T11:08:47.088100+0000","last_peered":"2023-10-09T11:08:47.088100+0000","last_clean":"2023-10-09T11:08:47.088100+0000","last_became_active":"2023-10-09T11:04:50.507404+0000","last_became_peered":"2023-10-09T11:04:50.507404+0000","last_unstale":"2023-10-09T11:08:47.088100+0000","last_undegraded":"2023-10-09T11:08:47.088100+0000","last_fullsized":"2023-10-09T11:08:47.088100+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:13.663394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":939,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-10-09T11:08:53.123155+0000","last_change":"2023-10-09T11:05:23.321040+0000","last_active":"2023-10-09T11:08:53.123155+0000","last_peered":"2023-10-09T11:08:53.123155+0000","last_clean":"2023-10-09T11:08:53.123155+0000","last_became_active":"2023-10-09T11:05:23.320674+0000","last_became_peered":"2023-10-09T11:05:23.320674+0000","last_unstale":"2023-10-09T11:08:53.123155+0000","last_undegraded":"2023-10-09T11:08:53.123155+0000","last_fullsized":"2023-10-09T11:08:53.123155+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.527660+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":401,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.110337+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:08:52.110337+0000","last_peered":"2023-10-09T11:08:52.110337+0000","last_clean":"2023-10-09T11:08:52.110337+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:08:52.110337+0000","last_undegraded":"2023-10-09T11:08:52.110337+0000","last_fullsized":"2023-10-09T11:08:52.110337+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":100,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.108175+0000","last_change":"2023-10-09T11:04:50.511984+0000","last_active":"2023-10-09T11:08:52.108175+0000","last_peered":"2023-10-09T11:08:52.108175+0000","last_clean":"2023-10-09T11:08:52.108175+0000","last_became_active":"2023-10-09T11:04:50.510820+0000","last_became_peered":"2023-10-09T11:04:50.510820+0000","last_unstale":"2023-10-09T11:08:52.108175+0000","last_undegraded":"2023-10-09T11:08:52.108175+0000","last_fullsized":"2023-10-09T11:08:52.108175+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:38.662189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":552,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109236+0000","last_change":"2023-10-09T11:05:20.327032+0000","last_active":"2023-10-09T11:08:52.109236+0000","last_peered":"2023-10-09T11:08:52.109236+0000","last_clean":"2023-10-09T11:08:52.109236+0000","last_became_active":"2023-10-09T11:05:20.320193+0000","last_became_peered":"2023-10-09T11:05:20.320193+0000","last_unstale":"2023-10-09T11:08:52.109236+0000","last_undegraded":"2023-10-09T11:08:52.109236+0000","last_fullsized":"2023-10-09T11:08:52.109236+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:44.862890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":100,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.106968+0000","last_change":"2023-10-09T11:04:50.384219+0000","last_active":"2023-10-09T11:08:52.106968+0000","last_peered":"2023-10-09T11:08:52.106968+0000","last_clean":"2023-10-09T11:08:52.106968+0000","last_became_active":"2023-10-09T11:04:50.384028+0000","last_became_peered":"2023-10-09T11:04:50.384028+0000","last_unstale":"2023-10-09T11:08:52.106968+0000","last_undegraded":"2023-10-09T11:08:52.106968+0000","last_fullsized":"2023-10-09T11:08:52.106968+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:04.721869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":757,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.366146+0000","last_change":"2023-10-09T11:08:39.540029+0000","last_active":"2023-10-09T11:08:52.366146+0000","last_peered":"2023-10-09T11:08:52.366146+0000","last_clean":"2023-10-09T11:08:52.366146+0000","last_became_active":"2023-10-09T11:07:36.291077+0000","last_became_peered":"2023-10-09T11:07:36.291077+0000","last_unstale":"2023-10-09T11:08:52.366146+0000","last_undegraded":"2023-10-09T11:08:52.366146+0000","last_fullsized":"2023-10-09T11:08:52.366146+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.539966+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.539966+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:20.767812+0000","scrub_duration":0.0016245929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":110,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-10-09T11:08:53.123261+0000","last_change":"2023-10-09T11:05:20.299321+0000","last_active":"2023-10-09T11:08:53.123261+0000","last_peered":"2023-10-09T11:08:53.123261+0000","last_clean":"2023-10-09T11:08:53.123261+0000","last_became_active":"2023-10-09T11:05:20.299019+0000","last_became_peered":"2023-10-09T11:05:20.299019+0000","last_unstale":"2023-10-09T11:08:53.123261+0000","last_undegraded":"2023-10-09T11:08:53.123261+0000","last_fullsized":"2023-10-09T11:08:53.123261+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:08.645909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":100,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.106980+0000","last_change":"2023-10-09T11:04:50.366890+0000","last_active":"2023-10-09T11:08:52.106980+0000","last_peered":"2023-10-09T11:08:52.106980+0000","last_clean":"2023-10-09T11:08:52.106980+0000","last_became_active":"2023-10-09T11:04:50.366558+0000","last_became_peered":"2023-10-09T11:04:50.366558+0000","last_unstale":"2023-10-09T11:08:52.106980+0000","last_undegraded":"2023-10-09T11:08:52.106980+0000","last_fullsized":"2023-10-09T11:08:52.106980+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:50.993153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":872,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.663340+0000","last_change":"2023-10-09T11:04:53.274332+0000","last_active":"2023-10-09T11:08:51.663340+0000","last_peered":"2023-10-09T11:08:51.663340+0000","last_clean":"2023-10-09T11:08:51.663340+0000","last_became_active":"2023-10-09T11:04:53.273798+0000","last_became_peered":"2023-10-09T11:04:53.273798+0000","last_unstale":"2023-10-09T11:08:51.663340+0000","last_undegraded":"2023-10-09T11:08:51.663340+0000","last_fullsized":"2023-10-09T11:08:51.663340+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:04.220970+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":121,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-10-09T11:08:52.109822+0000","last_change":"2023-10-09T11:06:24.249877+0000","last_active":"2023-10-09T11:08:52.109822+0000","last_peered":"2023-10-09T11:08:52.109822+0000","last_clean":"2023-10-09T11:08:52.109822+0000","last_became_active":"2023-10-09T11:06:24.249573+0000","last_became_peered":"2023-10-09T11:06:24.249573+0000","last_unstale":"2023-10-09T11:08:52.109822+0000","last_undegraded":"2023-10-09T11:08:52.109822+0000","last_fullsized":"2023-10-09T11:08:52.109822+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:11.798192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":621,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-09T11:08:51.662769+0000","last_change":"2023-10-09T11:05:20.307347+0000","last_active":"2023-10-09T11:08:51.662769+0000","last_peered":"2023-10-09T11:08:51.662769+0000","last_clean":"2023-10-09T11:08:51.662769+0000","last_became_active":"2023-10-09T11:05:20.307126+0000","last_became_peered":"2023-10-09T11:05:20.307126+0000","last_unstale":"2023-10-09T11:08:51.662769+0000","last_undegraded":"2023-10-09T11:08:51.662769+0000","last_fullsized":"2023-10-09T11:08:51.662769+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:34.408515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":91,"num_bytes_recovered":15742,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12694,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332508,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1168336,"kb_used_data":273032,"kb_used_omap":0,"kb_used_meta":895296,"kb_avail":92576816,"statfs":{"total":95995035648,"available":94798659584,"internally_reserved":0,"allocated":279584768,"data_stored":277743033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916783104},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":1.3460000000000001,"15min":1.7350000000000001},"min":{"1min":0.254,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":9.8219999999999992,"5min":9.8219999999999992,"15min":9.8219999999999992},"last":0.63300000000000001},{"interface":"front","average":{"1min":1.9099999999999999,"5min":1.6539999999999999,"15min":1.7669999999999999},"min":{"1min":0.28999999999999998,"5min":0.24299999999999999,"15min":0.216},"max":{"1min":5.4820000000000002,"5min":15.853,"15min":15.853},"last":0.75}]},{"osd":1,"last update":"Mon Oct 9 11:08:26 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":1.5029999999999999,"15min":1.5469999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":5.9669999999999996,"5min":11.619999999999999,"15min":11.619999999999999},"last":0.83499999999999996},{"interface":"front","average":{"1min":2.851,"5min":2.0219999999999998,"15min":1.944},"min":{"1min":0.26300000000000001,"5min":0.25,"15min":0.25},"max":{"1min":13.096,"5min":13.096,"15min":13.096},"last":0.54700000000000004}]},{"osd":2,"last update":"Mon Oct 9 11:08:54 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":1.573,"15min":1.77},"min":{"1min":0.309,"5min":0.309,"15min":0.245},"max":{"1min":5.1440000000000001,"5min":12.226000000000001,"15min":16.605},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.2770000000000001,"5min":1.6220000000000001,"15min":1.802},"min":{"1min":0.379,"5min":0.223,"15min":0.223},"max":{"1min":16.157,"5min":16.157,"15min":16.157},"last":0.79100000000000004}]},{"osd":3,"last update":"Mon Oct 9 11:08:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3050000000000002,"5min":1.7050000000000001,"15min":1.5009999999999999},"min":{"1min":0.34200000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.3970000000000002,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.7070000000000001,"5min":1.885,"15min":1.823},"min":{"1min":0.34000000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.0999999999999996,"5min":8.2330000000000005,"15min":8.2330000000000005},"last":0.53600000000000003}]},{"osd":4,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":0.91800000000000004,"15min":1.0940000000000001},"min":{"1min":0.29799999999999999,"5min":0.218,"15min":0.218},"max":{"1min":7.4630000000000001,"5min":7.4630000000000001,"15min":8.2940000000000005},"last":0.77900000000000003},{"interface":"front","average":{"1min":1.034,"5min":1.256,"15min":1.244},"min":{"1min":0.30599999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.7190000000000003,"5min":11.808,"15min":16.481000000000002},"last":0.49299999999999999}]},{"osd":5,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.0640000000000001,"15min":1.252},"min":{"1min":0.32800000000000001,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":5.4800000000000004,"5min":15.414999999999999,"15min":15.414999999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.345,"15min":1.198},"min":{"1min":0.375,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.045,"5min":15.369,"15min":15.369},"last":0.70799999999999996}]},{"osd":6,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.226,"15min":1.26},"min":{"1min":0.34599999999999997,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":3.8500000000000001,"5min":14.159000000000001,"15min":14.159000000000001},"last":0.67800000000000005},{"interface":"front","average":{"1min":1.121,"5min":1.472,"15min":1.355},"min":{"1min":0.36099999999999999,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":3.5830000000000002,"5min":13.993,"15min":16.538},"last":0.59099999999999997}]}]},{"osd":6,"up_from":44,"seq":188978561439,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1515968,"kb_used_data":15224,"kb_used_omap":0,"kb_used_meta":1500736,"kb_avail":92229184,"statfs":{"total":95995035648,"available":94442684416,"internally_reserved":0,"allocated":15589376,"data_stored":13763335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1536753664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":1.4950000000000001,"15min":1.9419999999999999},"min":{"1min":0.22600000000000001,"5min":0.223,"15min":0.217},"max":{"1min":12.209,"5min":13.867000000000001,"15min":54.743000000000002},"last":0.72799999999999998},{"interface":"front","average":{"1min":1.081,"5min":1.181,"15min":1.8919999999999999},"min":{"1min":0.27300000000000002,"5min":0.214,"15min":0.214},"max":{"1min":3.3740000000000001,"5min":10.055999999999999,"15min":53.701000000000001},"last":0.64600000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:08:26 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.514,"15min":1.1910000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.23499999999999999},"max":{"1min":4.3710000000000004,"5min":16.225999999999999,"15min":41.241999999999997},"last":0.754},{"interface":"front","average":{"1min":2.7330000000000001,"5min":2.1139999999999999,"15min":1.8260000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.248},"max":{"1min":9.2080000000000002,"5min":11.715,"15min":41.476999999999997},"last":0.504}]},{"osd":2,"last update":"Mon Oct 9 11:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.867,"15min":1.9019999999999999},"min":{"1min":0.32200000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.2069999999999999,"5min":10.382999999999999,"15min":12.003},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.85599999999999998,"5min":1.2909999999999999,"15min":1.877},"min":{"1min":0.33800000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.6669999999999998,"5min":12.693,"15min":15.432},"last":0.58299999999999996}]},{"osd":3,"last update":"Mon Oct 9 11:08:23 2023","interfaces":[{"interface":"back","average":{"1min":3.6749999999999998,"5min":2.2080000000000002,"15min":1.702},"min":{"1min":0.40200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":11.395,"5min":28.948,"15min":28.948},"last":0.68899999999999995},{"interface":"front","average":{"1min":2.653,"5min":2.9820000000000002,"15min":2.6200000000000001},"min":{"1min":0.307,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.964,"5min":28.893000000000001,"15min":28.893000000000001},"last":0.61299999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.66900000000000004,"15min":0.998},"min":{"1min":0.28100000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.67100000000000004,"5min":6.6609999999999996,"15min":17.216999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.53700000000000003,"15min":0.83299999999999996},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.24399999999999999},"max":{"1min":0.80100000000000005,"5min":2.3610000000000002,"15min":10.025},"last":0.80100000000000005}]},{"osd":5,"last update":"Mon Oct 9 11:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.78300000000000003,"15min":1.228},"min":{"1min":0.34000000000000002,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":2.7000000000000002,"5min":3.7389999999999999,"15min":19.786999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.68999999999999995,"15min":0.96499999999999997},"min":{"1min":0.377,"5min":0.248,"15min":0.248},"max":{"1min":1.22,"5min":4.085,"15min":9.9469999999999992},"last":0.77700000000000002}]},{"osd":7,"last update":"Mon Oct 9 11:08:14 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.081,"15min":1.268},"min":{"1min":0.32200000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.7250000000000001,"5min":10.246,"15min":12.882},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.83599999999999997,"15min":0.93300000000000005},"min":{"1min":0.35599999999999998,"5min":0.31,"15min":0.27500000000000002},"max":{"1min":3.2839999999999998,"5min":11.557,"15min":11.557},"last":0.82299999999999995}]}]},{"osd":1,"up_from":388,"seq":1666447310953,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":505024,"kb_used_data":7544,"kb_used_omap":13,"kb_used_meta":497458,"kb_avail":93240128,"statfs":{"total":95995035648,"available":95477891072,"internally_reserved":0,"allocated":7725056,"data_stored":4680404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":509397703},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":2.6960000000000002,"15min":2.4430000000000001},"min":{"1min":0.22600000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":7.0570000000000004,"5min":18.192,"15min":18.192},"last":0.48699999999999999},{"interface":"front","average":{"1min":3.7949999999999999,"5min":3.0840000000000001,"15min":2.0920000000000001},"min":{"1min":0.26700000000000002,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":16.895,"5min":17.957999999999998,"15min":17.957999999999998},"last":0.503}]},{"osd":2,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.9380000000000002,"5min":2.5339999999999998,"15min":2.911},"min":{"1min":0.216,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":8.798,"5min":12.436999999999999,"15min":47.731000000000002},"last":0.60099999999999998},{"interface":"front","average":{"1min":3.895,"5min":2.8690000000000002,"15min":2.5880000000000001},"min":{"1min":0.26000000000000001,"5min":0.216,"15min":0.216},"max":{"1min":16.870000000000001,"5min":16.870000000000001,"15min":47.786999999999999},"last":0.56699999999999995}]},{"osd":3,"last update":"Mon Oct 9 11:08:26 2023","interfaces":[{"interface":"back","average":{"1min":2.819,"5min":3.3530000000000002,"15min":2.1099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":9.4960000000000004,"5min":92.447999999999993,"15min":92.447999999999993},"last":0.77700000000000002},{"interface":"front","average":{"1min":3.069,"5min":3.4350000000000001,"15min":2.133},"min":{"1min":0.29099999999999998,"5min":0.217,"15min":0.217},"max":{"1min":13.295999999999999,"5min":92.066999999999993,"15min":92.066999999999993},"last":0.30499999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0059999999999998,"5min":1.879,"15min":2.3380000000000001},"min":{"1min":0.30299999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.9790000000000001,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":2.9670000000000001,"5min":2.1480000000000001,"15min":1.73},"min":{"1min":0.34799999999999998,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":16.841000000000001,"5min":16.841000000000001,"15min":16.841000000000001},"last":0.39200000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2559999999999998,"5min":1.968,"15min":2.286},"min":{"1min":0.311,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":9.7010000000000005,"5min":20.689,"15min":20.689},"last":0.73499999999999999},{"interface":"front","average":{"1min":2.427,"5min":2.113,"15min":2.242},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.0199999999999996,"5min":11.65,"15min":11.65},"last":0.40799999999999997}]},{"osd":6,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0819999999999999,"5min":2.1819999999999999,"15min":2.3799999999999999},"min":{"1min":0.32900000000000001,"5min":0.312,"15min":0.309},"max":{"1min":9.9459999999999997,"5min":20.646999999999998,"15min":20.646999999999998},"last":0.69199999999999995},{"interface":"front","average":{"1min":2.339,"5min":2.0339999999999998,"15min":2.1680000000000001},"min":{"1min":0.34399999999999997,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":7.0410000000000004,"5min":11.75,"15min":11.75},"last":0.375}]},{"osd":7,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.2530000000000001,"15min":1.7310000000000001},"min":{"1min":0.378,"5min":0.33000000000000002,"15min":0.32900000000000001},"max":{"1min":16.797999999999998,"5min":16.797999999999998,"15min":16.797999999999998},"last":0.36199999999999999},{"interface":"front","average":{"1min":2.2029999999999998,"5min":1.996,"15min":2.294},"min":{"1min":0.38100000000000001,"5min":0.35599999999999998,"15min":0.33200000000000002},"max":{"1min":8.4469999999999992,"5min":20.716000000000001,"15min":20.716000000000001},"last":0.76200000000000001}]}]},{"osd":0,"up_from":80,"seq":343597384011,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":999132,"kb_used_data":25044,"kb_used_omap":28,"kb_used_meta":974051,"kb_avail":92746020,"statfs":{"total":95995035648,"available":94971924480,"internally_reserved":0,"allocated":25645056,"data_stored":22506206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":997429072},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:08:29 2023","interfaces":[{"interface":"back","average":{"1min":1.946,"5min":2.0249999999999999,"15min":2.742},"min":{"1min":0.191,"5min":0.186,"15min":0.186},"max":{"1min":6.4909999999999997,"5min":14.644,"15min":19.608000000000001},"last":0.23400000000000001},{"interface":"front","average":{"1min":1.718,"5min":2.1549999999999998,"15min":2.8010000000000002},"min":{"1min":0.223,"5min":0.214,"15min":0.214},"max":{"1min":5.8609999999999998,"5min":15.834,"15min":19.992999999999999},"last":0.29799999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:07:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":2.339,"15min":2.29},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.17199999999999999},"max":{"1min":11.516,"5min":12.459,"15min":24.852},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.73,"5min":2.246,"15min":2.694},"min":{"1min":0.215,"5min":0.20300000000000001,"15min":0.19800000000000001},"max":{"1min":8.4589999999999996,"5min":11.68,"15min":77.256},"last":0.21299999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":3.016,"15min":3.3599999999999999},"min":{"1min":0.35099999999999998,"5min":0.22,"15min":0.22},"max":{"1min":7.9050000000000002,"5min":25.050000000000001,"15min":25.050000000000001},"last":0.28499999999999998},{"interface":"front","average":{"1min":2.2890000000000001,"5min":2.355,"15min":2.0510000000000002},"min":{"1min":0.30499999999999999,"5min":0.254,"15min":0.254},"max":{"1min":6.2389999999999999,"5min":14.125999999999999,"15min":14.125999999999999},"last":0.33500000000000002}]},{"osd":4,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":1.877,"5min":2.1880000000000002,"15min":2.4489999999999998},"min":{"1min":0.27100000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.8150000000000004,"5min":14.507,"15min":77.177999999999997},"last":0.312},{"interface":"front","average":{"1min":1.6910000000000001,"5min":1.675,"15min":1.6419999999999999},"min":{"1min":0.33600000000000002,"5min":0.30199999999999999,"15min":0.28899999999999998},"max":{"1min":6.194,"5min":7.9249999999999998,"15min":10.513999999999999},"last":0.40300000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.7290000000000001,"15min":1.8160000000000001},"min":{"1min":0.34699999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.2199999999999998,"5min":7.9550000000000001,"15min":10.544},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.7050000000000001,"5min":1.7270000000000001,"15min":1.804},"min":{"1min":0.39200000000000002,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":4.6820000000000004,"5min":11.212999999999999,"15min":24.754999999999999},"last":0.46500000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":1.8380000000000001,"15min":1.8959999999999999},"min":{"1min":0.377,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":6.173,"5min":7.9829999999999997,"15min":18.920000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.423,"5min":1.786,"15min":1.927},"min":{"1min":0.38100000000000001,"5min":0.35299999999999998,"15min":0.31900000000000001},"max":{"1min":4.2789999999999999,"5min":11.257999999999999,"15min":24.800000000000001},"last":0.42899999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.391,"5min":2.1789999999999998,"15min":2.0339999999999998},"min":{"1min":0.38900000000000001,"5min":0.35099999999999998,"15min":0.32700000000000001},"max":{"1min":9.8710000000000004,"5min":17.672000000000001,"15min":17.672000000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":2.2170000000000001,"5min":2.4500000000000002,"15min":2.569},"min":{"1min":0.33400000000000002,"5min":0.311,"15min":0.311},"max":{"1min":4.9320000000000004,"5min":17.539000000000001,"15min":77.228999999999999},"last":0.41299999999999998}]}]},{"osd":2,"up_from":90,"seq":386547056951,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":765512,"kb_used_data":7680,"kb_used_omap":81,"kb_used_meta":757742,"kb_avail":92979640,"statfs":{"total":95995035648,"available":95211151360,"internally_reserved":0,"allocated":7864320,"data_stored":4908130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":775928026},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":2.2010000000000001,"15min":2.0609999999999999},"min":{"1min":0.186,"5min":0.182,"15min":0.17499999999999999},"max":{"1min":14.226000000000001,"5min":15.609,"15min":48.933999999999997},"last":0.28199999999999997},{"interface":"front","average":{"1min":2.5419999999999998,"5min":2.056,"15min":1.8919999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.19600000000000001},"max":{"1min":9.5540000000000003,"5min":9.8010000000000002,"15min":15.595000000000001},"last":0.31}]},{"osd":1,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":2.3780000000000001,"15min":2.5339999999999998},"min":{"1min":0.193,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":3.242,"5min":24.902000000000001,"15min":49.033999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.6600000000000001,"5min":2.5640000000000001,"15min":2.7200000000000002},"min":{"1min":0.253,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":14.26,"5min":25.850999999999999,"15min":48.874000000000002},"last":0.40999999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:08:19 2023","interfaces":[{"interface":"back","average":{"1min":4.8019999999999996,"5min":2.9399999999999999,"15min":2.1560000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":25.890999999999998,"5min":25.890999999999998,"15min":25.890999999999998},"last":0.51500000000000001},{"interface":"front","average":{"1min":4.4560000000000004,"5min":2.895,"15min":2.395},"min":{"1min":0.372,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":28.689,"5min":28.689,"15min":28.689},"last":0.378}]},{"osd":4,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":1.796,"15min":1.8959999999999999},"min":{"1min":0.27100000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":11.391999999999999,"5min":11.391999999999999,"15min":15.672000000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.468,"5min":1.7909999999999999,"15min":1.873},"min":{"1min":0.34899999999999998,"5min":0.249,"15min":0.249},"max":{"1min":8.3350000000000009,"5min":15.571,"15min":48.795000000000002},"last":0.45800000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":1.597,"15min":1.5620000000000001},"min":{"1min":0.33700000000000002,"5min":0.29099999999999998,"15min":0.25900000000000001},"max":{"1min":7.4980000000000002,"5min":8.5310000000000006,"15min":12.128},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.8580000000000001,"5min":1.7869999999999999,"15min":2.0339999999999998},"min":{"1min":0.34899999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.441000000000001,"5min":11.441000000000001,"15min":49.061},"last":0.44400000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.909,"15min":1.804},"min":{"1min":0.41899999999999998,"5min":0.313,"15min":0.313},"max":{"1min":8.6039999999999992,"5min":15.536,"15min":48.845999999999997},"last":0.48799999999999999},{"interface":"front","average":{"1min":2.6499999999999999,"5min":2.012,"15min":1.657},"min":{"1min":0.42399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.157999999999999,"5min":10.157999999999999,"15min":10.157999999999999},"last":0.54600000000000004}]},{"osd":7,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":2.081,"15min":1.9419999999999999},"min":{"1min":0.45100000000000001,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":8.3800000000000008,"5min":15.59,"15min":48.966000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":2.492,"5min":1.917,"15min":1.762},"min":{"1min":0.46200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.5919999999999996,"5min":8.5549999999999997,"15min":14.94},"last":0.58099999999999996}]}]},{"osd":3,"up_from":469,"seq":2014339661881,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":343148,"kb_used_data":8100,"kb_used_omap":11,"kb_used_meta":335028,"kb_avail":93402004,"statfs":{"total":95995035648,"available":95643652096,"internally_reserved":0,"allocated":8294400,"data_stored":5023863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11309,"internal_metadata":343069651},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":3.004,"5min":2.1840000000000002,"15min":1.875},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":14.521000000000001,"5min":14.882,"15min":14.882},"last":1.0740000000000001},{"interface":"front","average":{"1min":2.3919999999999999,"5min":1.397,"15min":1.044},"min":{"1min":0.20100000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":9.4030000000000005,"5min":9.4030000000000005,"15min":9.4030000000000005},"last":0.19900000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":2.218,"15min":1.6739999999999999},"min":{"1min":0.245,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":8.8350000000000009,"5min":17.93,"15min":17.93},"last":0.56599999999999995},{"interface":"front","average":{"1min":3.0099999999999998,"5min":2.246,"15min":1.8049999999999999},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":10.645,"5min":19.163,"15min":19.163},"last":1.0349999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4860000000000002,"5min":2.0840000000000001,"15min":1.419},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":12.323,"5min":38.840000000000003,"15min":38.840000000000003},"last":4.0270000000000001},{"interface":"front","average":{"1min":1.6539999999999999,"5min":2.2120000000000002,"15min":1.4570000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":10.172000000000001,"5min":37.991999999999997,"15min":37.991999999999997},"last":0.47499999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6419999999999999,"5min":1.655,"15min":1.6000000000000001},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.3719999999999999,"5min":14.907999999999999,"15min":14.907999999999999},"last":1.131},{"interface":"front","average":{"1min":2.0489999999999999,"5min":1.1439999999999999,"15min":0.84499999999999997},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.7889999999999997,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.307}]},{"osd":5,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.4259999999999999,"15min":1.417},"min":{"1min":0.33100000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.8499999999999996,"5min":9.3300000000000001,"15min":9.3300000000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.5529999999999999,"5min":1.5649999999999999,"15min":1.482},"min":{"1min":0.34699999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.3320000000000007,"5min":10.029,"15min":10.029},"last":1.101}]},{"osd":6,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.256,"5min":1.6539999999999999,"15min":1.74},"min":{"1min":0.32300000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.9550000000000001,"5min":15.455,"15min":15.455},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.8100000000000001,"5min":1.706,"15min":1.6639999999999999},"min":{"1min":0.36799999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.4019999999999992,"5min":14.837,"15min":14.837},"last":1.1559999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.365,"15min":1.1220000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.6769999999999996,"5min":6.907,"15min":6.907},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.55,"5min":1.595,"15min":1.3029999999999999},"min":{"1min":0.33700000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.9320000000000004,"5min":6.9119999999999999,"15min":6.9119999999999999},"last":0.63800000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019302,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903052,"kb_used_data":6212,"kb_used_omap":0,"kb_used_meta":896832,"kb_avail":92842100,"statfs":{"total":95995035648,"available":95070310400,"internally_reserved":0,"allocated":6361088,"data_stored":4523158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918355968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.613,"15min":2.4129999999999998},"min":{"1min":0.22600000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":6.1289999999999996,"5min":13.162000000000001,"15min":213.88999999999999},"last":0.441},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.742,"15min":2.5019999999999998},"min":{"1min":0.26500000000000001,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":3.875,"5min":9.0670000000000002,"15min":214.09800000000001},"last":0.36299999999999999}]},{"osd":1,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2989999999999999,"5min":2.6059999999999999,"15min":2.6440000000000001},"min":{"1min":0.28299999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.718,"5min":57.966999999999999,"15min":121.41},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.4810000000000001,"5min":2.2519999999999998,"15min":2.1280000000000001},"min":{"1min":0.26900000000000002,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":6.1980000000000004,"5min":57.832000000000001,"15min":122.21899999999999},"last":3.589}]},{"osd":2,"last update":"Mon Oct 9 11:08:00 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":1.917,"15min":1.74},"min":{"1min":0.32100000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.2939999999999996,"5min":18.407,"15min":18.407},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.149,"5min":1.9810000000000001,"15min":2.0910000000000002},"min":{"1min":0.27900000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.1479999999999997,"5min":8.4250000000000007,"15min":27.818000000000001},"last":0.55200000000000005}]},{"osd":3,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":2.4380000000000002,"15min":1.98},"min":{"1min":0.34799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.1689999999999996,"5min":60.753,"15min":60.753},"last":3.6520000000000001},{"interface":"front","average":{"1min":1.5049999999999999,"5min":2.2069999999999999,"15min":1.5149999999999999},"min":{"1min":0.45300000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.4980000000000002,"5min":61.473999999999997,"15min":61.473999999999997},"last":9.5199999999999996}]},{"osd":5,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":1.353,"15min":1.3600000000000001},"min":{"1min":0.30499999999999999,"5min":0.224,"15min":0.224},"max":{"1min":4.7670000000000003,"5min":11.59,"15min":19.068999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.079,"5min":1.2809999999999999,"15min":1.238},"min":{"1min":0.34200000000000003,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":3.7570000000000001,"5min":9.8610000000000007,"15min":15.535},"last":0.61099999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:08:00 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":1.353,"15min":1.1599999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":7.5860000000000003,"5min":8.3930000000000007,"15min":15.739000000000001},"last":0.98399999999999999},{"interface":"front","average":{"1min":1.4610000000000001,"5min":1.4890000000000001,"15min":1.4570000000000001},"min":{"1min":0.34899999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":9.5210000000000008,"5min":12.795999999999999,"15min":14.093999999999999},"last":0.72999999999999998}]},{"osd":7,"last update":"Mon Oct 9 11:08:00 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.5600000000000001,"15min":1.5529999999999999},"min":{"1min":0.308,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.4540000000000002,"5min":12.753,"15min":19.923999999999999},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.4239999999999999,"15min":1.476},"min":{"1min":0.314,"5min":0.29799999999999999,"15min":0.28799999999999998},"max":{"1min":4.21,"5min":10.853,"15min":19.859999999999999},"last":0.70399999999999996}]}]},{"osd":5,"up_from":37,"seq":158913790371,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1205756,"kb_used_data":6452,"kb_used_omap":0,"kb_used_meta":1199296,"kb_avail":92539396,"statfs":{"total":95995035648,"available":94760341504,"internally_reserved":0,"allocated":6606848,"data_stored":4781267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1228079104},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:25 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":2.3239999999999998,"15min":2.2919999999999998},"min":{"1min":0.255,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.4349999999999996,"5min":49.665999999999997,"15min":58.271999999999998},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.849,"5min":2.3740000000000001,"15min":2.3399999999999999},"min":{"1min":0.29699999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.4610000000000003,"5min":49.701999999999998,"15min":58.25},"last":0.57999999999999996}]},{"osd":1,"last update":"Mon Oct 9 11:08:32 2023","interfaces":[{"interface":"back","average":{"1min":2.5179999999999998,"5min":1.899,"15min":2.0670000000000002},"min":{"1min":0.246,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.1850000000000005,"5min":16.962,"15min":32.491999999999997},"last":0.54700000000000004},{"interface":"front","average":{"1min":2.3439999999999999,"5min":2.004,"15min":1.917},"min":{"1min":0.32800000000000001,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":9.1989999999999998,"5min":17.033999999999999,"15min":32.537999999999997},"last":0.72599999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:08:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.321,"15min":1.8240000000000001},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.221},"max":{"1min":7.3109999999999999,"5min":7.3109999999999999,"15min":11.257999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.8919999999999999,"15min":1.988},"min":{"1min":0.317,"5min":0.253,"15min":0.253},"max":{"1min":4.085,"5min":11.673,"15min":14.621},"last":0.78100000000000003}]},{"osd":3,"last update":"Mon Oct 9 11:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.962,"5min":1.738,"15min":1.5720000000000001},"min":{"1min":0.311,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.0289999999999999,"5min":15.535,"15min":15.535},"last":0.56299999999999994},{"interface":"front","average":{"1min":2.8839999999999999,"5min":2.0710000000000002,"15min":1.8959999999999999},"min":{"1min":0.40200000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":15.757999999999999,"5min":15.757999999999999,"15min":15.757999999999999},"last":0.70299999999999996}]},{"osd":4,"last update":"Mon Oct 9 11:08:32 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":1.0089999999999999,"15min":0.93799999999999994},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.216},"max":{"1min":3.0680000000000001,"5min":12.042,"15min":12.042},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.3480000000000001,"5min":1.1020000000000001,"15min":1.1899999999999999},"min":{"1min":0.27000000000000002,"5min":0.245,"15min":0.245},"max":{"1min":6.8639999999999999,"5min":9.2490000000000006,"15min":9.2490000000000006},"last":0.64700000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:08:45 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.2090000000000001,"15min":1.2450000000000001},"min":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.27100000000000002},"max":{"1min":5.944,"5min":7.1020000000000003,"15min":9.1969999999999992},"last":0.67200000000000004},{"interface":"front","average":{"1min":1.214,"5min":1.204,"15min":1.0700000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29099999999999998},"max":{"1min":9.298,"5min":11.983000000000001,"15min":11.983000000000001},"last":0.68899999999999995}]},{"osd":7,"last update":"Mon Oct 9 11:08:08 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.3160000000000001,"15min":1.302},"min":{"1min":0.35399999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.0469999999999997,"5min":8.9429999999999996,"15min":9.3539999999999992},"last":0.622},{"interface":"front","average":{"1min":1.03,"5min":1.278,"15min":1.177},"min":{"1min":0.32800000000000001,"5min":0.30499999999999999,"15min":0.30099999999999999},"max":{"1min":4.2350000000000003,"5min":11.726000000000001,"15min":11.726000000000001},"last":0.75700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3733,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:08:58.331 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:08:58.331 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T11:08:58.331 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T11:08:58.331 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T11:08:58.331 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:08:58.335 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:58.710 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:58.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:58.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:08:58.814 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:58.847 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:58.848 DEBUG:teuthology.orchestra.run.smithi125:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T11:08:58.866 INFO:tasks.workunit.client.0.smithi125.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T11:08:58.868 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:58.868 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T11:08:58.868 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T11:08:58.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:58.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:58.912 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T11:08:58.917 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T11:08:58.966 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T11:08:58.972 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:59.002 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:59.005 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T11:08:59.007 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T11:08:59.007 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T11:08:59.022 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:59.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:59.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:59.046 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T11:08:59.049 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T11:08:59.050 DEBUG:teuthology.orchestra.run.smithi125:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T11:08:59.067 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-09T11:08:59.068 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-09T11:08:59.068 DEBUG:teuthology.orchestra.run.smithi125:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-09T11:08:59.126 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:08:59.131 INFO:tasks.workunit.client.0.smithi125.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-09T11:08:59.131 INFO:tasks.workunit.client.0.smithi125.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-10-09T11:08:59.131 INFO:tasks.workunit.client.0.smithi125.stderr:+ TMPDIR=/tmp/rbd_import_export_475522 2023-10-09T11:08:59.131 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm -rf /tmp/rbd_import_export_475522 2023-10-09T11:08:59.132 INFO:tasks.workunit.client.0.smithi125.stderr:+ mkdir /tmp/rbd_import_export_475522 2023-10-09T11:08:59.133 INFO:tasks.workunit.client.0.smithi125.stderr:+ trap 'rm -rf /tmp/rbd_import_export_475522' INT TERM EXIT 2023-10-09T11:08:59.133 INFO:tasks.workunit.client.0.smithi125.stderr:+ mkdir foo.475522 2023-10-09T11:08:59.133 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import foo.475522 foo.dir 2023-10-09T11:08:59.155 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:59.174 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:08:59.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:59.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:08:59.189 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:08:59.277 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:59.308 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:59.342 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: cannot import a directory 2023-10-09T11:08:59.342 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 0% complete...failed. 2023-10-09T11:08:59.342 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: import failed: (21) Is a directory 2023-10-09T11:08:59.344 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T11:08:59.344 INFO:tasks.workunit.client.0.smithi125.stderr:+ rmdir foo.475522 2023-10-09T11:08:59.345 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_475522/img bs=1k count=1 seek=10 2023-10-09T11:08:59.346 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records in 2023-10-09T11:08:59.346 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records out 2023-10-09T11:08:59.346 INFO:tasks.workunit.client.0.smithi125.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.317e-05 s, 23.7 MB/s 2023-10-09T11:08:59.346 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_475522/img bs=1k count=10 seek=100 2023-10-09T11:08:59.347 INFO:tasks.workunit.client.0.smithi125.stderr:10+0 records in 2023-10-09T11:08:59.347 INFO:tasks.workunit.client.0.smithi125.stderr:10+0 records out 2023-10-09T11:08:59.347 INFO:tasks.workunit.client.0.smithi125.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.0513e-05 s, 203 MB/s 2023-10-09T11:08:59.347 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_475522/img bs=1k count=100 seek=1000 2023-10-09T11:08:59.348 INFO:tasks.workunit.client.0.smithi125.stderr:70+1 records in 2023-10-09T11:08:59.348 INFO:tasks.workunit.client.0.smithi125.stderr:70+1 records out 2023-10-09T11:08:59.348 INFO:tasks.workunit.client.0.smithi125.stderr:72208 bytes (72 kB, 71 KiB) copied, 0.000164071 s, 440 MB/s 2023-10-09T11:08:59.348 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_475522/img bs=1k seek=10000 2023-10-09T11:08:59.349 INFO:tasks.workunit.client.0.smithi125.stderr:140+1 records in 2023-10-09T11:08:59.349 INFO:tasks.workunit.client.0.smithi125.stderr:140+1 records out 2023-10-09T11:08:59.349 INFO:tasks.workunit.client.0.smithi125.stderr:143376 bytes (143 kB, 140 KiB) copied, 0.000297419 s, 482 MB/s 2023-10-09T11:08:59.349 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_475522/img bs=1k seek=100000 2023-10-09T11:08:59.361 INFO:tasks.workunit.client.0.smithi125.stderr:70+1 records in 2023-10-09T11:08:59.361 INFO:tasks.workunit.client.0.smithi125.stderr:70+1 records out 2023-10-09T11:08:59.361 INFO:tasks.workunit.client.0.smithi125.stderr:72312 bytes (72 kB, 71 KiB) copied, 0.0112378 s, 6.4 MB/s 2023-10-09T11:08:59.361 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_475522/img bs=1k seek=1000000 2023-10-09T11:08:59.362 INFO:tasks.workunit.client.0.smithi125.stderr:193+1 records in 2023-10-09T11:08:59.362 INFO:tasks.workunit.client.0.smithi125.stderr:193+1 records out 2023-10-09T11:08:59.362 INFO:tasks.workunit.client.0.smithi125.stderr:198280 bytes (198 kB, 194 KiB) copied, 0.000463697 s, 428 MB/s 2023-10-09T11:08:59.362 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm testimg 2023-10-09T11:08:59.377 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:59.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:59.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:59.404 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 0% complete...failed. 2023-10-09T11:08:59.405 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T11:08:59.409 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T11:08:59.409 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format /tmp/rbd_import_export_475522/img testimg 2023-10-09T11:08:59.481 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:08:59.508 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:59.527 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:08:59.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:59.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:08:59.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:08:59 smithi125 ceph-mon[195507]: pgmap v1407: 103 pgs: 103 active+clean; 582 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:59.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:08:59 smithi125 ceph-mon[200197]: pgmap v1407: 103 pgs: 103 active+clean; 582 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:59.631 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:08:59.659 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:59.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:08:59 smithi155 ceph-mon[184854]: pgmap v1407: 103 pgs: 103 active+clean; 582 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:08:59.729 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:08:59.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:59.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:08:59.833 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:08:59.861 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:08:59.980 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:08:59.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:08:59.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:09:00.084 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:00.112 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:00.131 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:00.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:00.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:00.237 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:09:00.267 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:00.287 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:09:00.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:00.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:09:00.391 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:00.422 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:09:00.424 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:00.433 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:09:00.433 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T11:09:00.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T11:09:00.434 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:09:00.434 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.434 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157305 9.9 9:92e816c0:::benchmark_data_smithi125_363789_object47006:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T11:09:00.434 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:53.128382+0000", 2023-10-09T11:09:00.434 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 547.30155780500002, 2023-10-09T11:09:00.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0278687440000001, 2023-10-09T11:09:00.434 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.434 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:09:00.434 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.434 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:09:00.435 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:09:00.435 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157305 2023-10-09T11:09:00.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.435 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.435 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128382+0000", 2023-10-09T11:09:00.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.435 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128382+0000", 2023-10-09T11:09:00.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128381+0000", 2023-10-09T11:09:00.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:00.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.437 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.437 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128386+0000", 2023-10-09T11:09:00.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1699999999999996e-06 2023-10-09T11:09:00.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.437 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.437 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128396+0000", 2023-10-09T11:09:00.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0259e-05 2023-10-09T11:09:00.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.437 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.437 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128404+0000", 2023-10-09T11:09:00.438 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.554e-06 2023-10-09T11:09:00.438 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.438 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.438 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.438 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.129279+0000", 2023-10-09T11:09:00.438 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00087499400000000003 2023-10-09T11:09:00.438 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.438 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.438 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.438 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.129287+0000", 2023-10-09T11:09:00.438 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4479999999999994e-06 2023-10-09T11:09:00.439 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.439 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.439 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.439 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.156164+0000", 2023-10-09T11:09:00.439 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0268760690000001 2023-10-09T11:09:00.439 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.439 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.439 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.439 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.156251+0000", 2023-10-09T11:09:00.440 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7682000000000005e-05 2023-10-09T11:09:00.440 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.440 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.440 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.440 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.440 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.440 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:157332 9.9 9:949c85f3:::benchmark_data_smithi125_363789_object47033:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-10-09T11:09:00.440 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:53.128409+0000", 2023-10-09T11:09:00.440 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 547.30153083200003, 2023-10-09T11:09:00.440 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.041208887, 2023-10-09T11:09:00.440 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.440 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.441 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.441 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:09:00.441 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:09:00.441 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 157332 2023-10-09T11:09:00.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.441 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.441 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128409+0000", 2023-10-09T11:09:00.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.442 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.442 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128409+0000", 2023-10-09T11:09:00.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.442 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.442 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128409+0000", 2023-10-09T11:09:00.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:00.442 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128411+0000", 2023-10-09T11:09:00.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0779999999999998e-06 2023-10-09T11:09:00.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128416+0000", 2023-10-09T11:09:00.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2399999999999998e-06 2023-10-09T11:09:00.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.444 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.128421+0000", 2023-10-09T11:09:00.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4909999999999999e-06 2023-10-09T11:09:00.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.444 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.129306+0000", 2023-10-09T11:09:00.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00088470200000000001 2023-10-09T11:09:00.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.445 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.445 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.445 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:53.129311+0000", 2023-10-09T11:09:00.445 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3970000000000002e-06 2023-10-09T11:09:00.445 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.445 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.445 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.445 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.156606+0000", 2023-10-09T11:09:00.445 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.027294844 2023-10-09T11:09:00.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.446 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.446 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.156717+0000", 2023-10-09T11:09:00.446 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111334 2023-10-09T11:09:00.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.446 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,7", 2023-10-09T11:09:00.447 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.156823+0000", 2023-10-09T11:09:00.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000105351 2023-10-09T11:09:00.447 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.447 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.447 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.447 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.164715+0000", 2023-10-09T11:09:00.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0078925379999999993 2023-10-09T11:09:00.447 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.447 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.166304+0000", 2023-10-09T11:09:00.448 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015885669999999999 2023-10-09T11:09:00.448 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.169523+0000", 2023-10-09T11:09:00.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032196880000000001 2023-10-09T11:09:00.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.449 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.449 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.169578+0000", 2023-10-09T11:09:00.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4339999999999998e-05 2023-10-09T11:09:00.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.449 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:54.169618+0000", 2023-10-09T11:09:00.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0618999999999999e-05 2023-10-09T11:09:00.450 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.450 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.450 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.450 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:51295 10.c 10:349d53bf:::benchmark_data_smithi125_402869_object51294:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T11:09:00.450 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:54.224908+0000", 2023-10-09T11:09:00.450 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 426.205032136, 2023-10-09T11:09:00.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0131283310000001, 2023-10-09T11:09:00.450 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.451 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:09:00.451 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.451 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.451 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.451 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 51295 2023-10-09T11:09:00.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.451 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.224908+0000", 2023-10-09T11:09:00.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.224908+0000", 2023-10-09T11:09:00.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.224907+0000", 2023-10-09T11:09:00.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:00.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.224943+0000", 2023-10-09T11:09:00.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6658e-05 2023-10-09T11:09:00.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.224954+0000", 2023-10-09T11:09:00.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0533000000000001e-05 2023-10-09T11:09:00.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.224962+0000", 2023-10-09T11:09:00.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2069999999999995e-06 2023-10-09T11:09:00.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.225016+0000", 2023-10-09T11:09:00.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4055999999999999e-05 2023-10-09T11:09:00.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.225130+0000", 2023-10-09T11:09:00.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000113702 2023-10-09T11:09:00.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T11:09:00.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.225260+0000", 2023-10-09T11:09:00.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000129834 2023-10-09T11:09:00.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.229508+0000", 2023-10-09T11:09:00.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042478300000000002 2023-10-09T11:09:00.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.239287+0000", 2023-10-09T11:09:00.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0097793959999999992 2023-10-09T11:09:00.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.239301+0000", 2023-10-09T11:09:00.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3771000000000001e-05 2023-10-09T11:09:00.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.237967+0000", 2023-10-09T11:09:00.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99866647399999997 2023-10-09T11:09:00.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.238036+0000", 2023-10-09T11:09:00.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9280000000000001e-05 2023-10-09T11:09:00.457 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.457 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.457 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.457 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:51299 10.c 10:39b4c66a:::benchmark_data_smithi125_402869_object51298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T11:09:00.457 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:54.232859+0000", 2023-10-09T11:09:00.457 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 426.19708151200001, 2023-10-09T11:09:00.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0181773119999999, 2023-10-09T11:09:00.458 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.458 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.458 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.458 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.458 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.458 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 51299 2023-10-09T11:09:00.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.458 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.232859+0000", 2023-10-09T11:09:00.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.232859+0000", 2023-10-09T11:09:00.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.232858+0000", 2023-10-09T11:09:00.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:00.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.232880+0000", 2023-10-09T11:09:00.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1630000000000001e-05 2023-10-09T11:09:00.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.232888+0000", 2023-10-09T11:09:00.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9459999999999998e-06 2023-10-09T11:09:00.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.232894+0000", 2023-10-09T11:09:00.461 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0240000000000003e-06 2023-10-09T11:09:00.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.461 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.239349+0000", 2023-10-09T11:09:00.461 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0064550709999999997 2023-10-09T11:09:00.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.461 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.239360+0000", 2023-10-09T11:09:00.461 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0786e-05 2023-10-09T11:09:00.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.462 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.238058+0000", 2023-10-09T11:09:00.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99869864600000002 2023-10-09T11:09:00.462 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.462 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.462 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.238183+0000", 2023-10-09T11:09:00.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000124444 2023-10-09T11:09:00.462 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.462 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.462 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T11:09:00.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.238332+0000", 2023-10-09T11:09:00.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000148734 2023-10-09T11:09:00.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.463 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.242271+0000", 2023-10-09T11:09:00.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039395869999999996 2023-10-09T11:09:00.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.463 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.247098+0000", 2023-10-09T11:09:00.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0048267379999999997 2023-10-09T11:09:00.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.464 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.464 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.464 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.250972+0000", 2023-10-09T11:09:00.464 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003874204 2023-10-09T11:09:00.464 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.464 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.464 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.464 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.251011+0000", 2023-10-09T11:09:00.464 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9406000000000002e-05 2023-10-09T11:09:00.464 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.464 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.464 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.465 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.251036+0000", 2023-10-09T11:09:00.465 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4553999999999999e-05 2023-10-09T11:09:00.465 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.465 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.465 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.465 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.465 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:51302 10.c 10:3c52ba0a:::benchmark_data_smithi125_402869_object51301:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T11:09:00.465 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:01:54.233473+0000", 2023-10-09T11:09:00.465 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 426.19646737900001, 2023-10-09T11:09:00.465 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0177018390000001, 2023-10-09T11:09:00.465 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.465 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.466 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.466 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.466 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.466 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 51302 2023-10-09T11:09:00.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.466 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.466 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.233473+0000", 2023-10-09T11:09:00.466 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.466 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.467 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.467 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.233473+0000", 2023-10-09T11:09:00.467 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.467 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.467 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.467 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.233472+0000", 2023-10-09T11:09:00.467 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:00.467 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.467 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.467 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.233495+0000", 2023-10-09T11:09:00.468 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2628000000000001e-05 2023-10-09T11:09:00.468 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.468 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.468 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.468 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.233502+0000", 2023-10-09T11:09:00.468 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2690000000000002e-06 2023-10-09T11:09:00.468 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.468 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.468 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.468 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.233509+0000", 2023-10-09T11:09:00.468 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2049999999999996e-06 2023-10-09T11:09:00.468 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.468 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.469 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.239417+0000", 2023-10-09T11:09:00.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.005907537 2023-10-09T11:09:00.469 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.469 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.469 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:54.239425+0000", 2023-10-09T11:09:00.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4460000000000001e-06 2023-10-09T11:09:00.470 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.470 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.240118+0000", 2023-10-09T11:09:00.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0006926060000001 2023-10-09T11:09:00.470 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.470 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.240257+0000", 2023-10-09T11:09:00.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013881399999999999 2023-10-09T11:09:00.470 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.471 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.471 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T11:09:00.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.240390+0000", 2023-10-09T11:09:00.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013303999999999999 2023-10-09T11:09:00.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.471 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.471 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.242304+0000", 2023-10-09T11:09:00.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019137710000000001 2023-10-09T11:09:00.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.471 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.472 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.472 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.251044+0000", 2023-10-09T11:09:00.472 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0087401330000000006 2023-10-09T11:09:00.472 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.472 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.472 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.472 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.251095+0000", 2023-10-09T11:09:00.472 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0915000000000003e-05 2023-10-09T11:09:00.473 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.473 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.473 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.473 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.251137+0000", 2023-10-09T11:09:00.473 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2345000000000002e-05 2023-10-09T11:09:00.473 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.473 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.473 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.473 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:01:55.251175+0000", 2023-10-09T11:09:00.473 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7639999999999999e-05 2023-10-09T11:09:00.473 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.473 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.474 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.474 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.474 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.474 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:64757 10.1 10:823ba87e:::benchmark_data_smithi125_402869_object64756:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-10-09T11:09:00.474 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:07.554401+0000", 2023-10-09T11:09:00.474 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 412.87553919499999, 2023-10-09T11:09:00.474 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.020742526, 2023-10-09T11:09:00.474 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.474 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:09:00.474 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.474 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.474 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.475 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 64757 2023-10-09T11:09:00.475 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.475 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.475 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.475 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.475 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554401+0000", 2023-10-09T11:09:00.475 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.475 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.475 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.475 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.475 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554401+0000", 2023-10-09T11:09:00.475 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.476 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.476 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.476 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554400+0000", 2023-10-09T11:09:00.476 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:00.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.476 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.476 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.476 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554441+0000", 2023-10-09T11:09:00.476 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0886999999999997e-05 2023-10-09T11:09:00.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.476 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.477 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.477 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554452+0000", 2023-10-09T11:09:00.477 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1292e-05 2023-10-09T11:09:00.477 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.477 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.477 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.477 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554459+0000", 2023-10-09T11:09:00.477 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9199999999999998e-06 2023-10-09T11:09:00.477 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.478 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.478 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.478 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.558760+0000", 2023-10-09T11:09:00.478 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043012119999999996 2023-10-09T11:09:00.478 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.478 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.478 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.478 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.558901+0000", 2023-10-09T11:09:00.478 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014095199999999999 2023-10-09T11:09:00.478 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.479 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.479 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:09:00.479 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.559060+0000", 2023-10-09T11:09:00.479 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015870399999999999 2023-10-09T11:09:00.479 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.479 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.479 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.479 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.564296+0000", 2023-10-09T11:09:00.480 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0052360820000000004 2023-10-09T11:09:00.480 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.480 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.480 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.480 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.564325+0000", 2023-10-09T11:09:00.480 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9091999999999999e-05 2023-10-09T11:09:00.480 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.480 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.481 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.568712+0000", 2023-10-09T11:09:00.481 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043873230000000003 2023-10-09T11:09:00.481 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.481 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.481 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.568726+0000", 2023-10-09T11:09:00.481 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4224e-05 2023-10-09T11:09:00.481 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.481 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.482 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.482 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:08.575077+0000", 2023-10-09T11:09:00.482 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.006350866 2023-10-09T11:09:00.482 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.482 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.482 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.482 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:08.575144+0000", 2023-10-09T11:09:00.482 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6302000000000003e-05 2023-10-09T11:09:00.483 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.483 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.483 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.483 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.483 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.483 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:64759 10.1 10:8e3eeb45:::benchmark_data_smithi125_402869_object64758:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-10-09T11:09:00.483 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:07.554465+0000", 2023-10-09T11:09:00.483 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 412.87547525799999, 2023-10-09T11:09:00.484 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0210216519999999, 2023-10-09T11:09:00.484 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.484 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:09:00.484 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.484 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.484 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.484 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 64759 2023-10-09T11:09:00.484 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.484 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.485 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.485 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.485 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554465+0000", 2023-10-09T11:09:00.485 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.485 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.485 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.485 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.485 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554465+0000", 2023-10-09T11:09:00.485 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.486 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.486 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.486 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.486 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554465+0000", 2023-10-09T11:09:00.486 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:00.486 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.486 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.486 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.486 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554520+0000", 2023-10-09T11:09:00.486 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5247000000000001e-05 2023-10-09T11:09:00.486 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.486 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.487 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.487 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554528+0000", 2023-10-09T11:09:00.487 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8480000000000001e-06 2023-10-09T11:09:00.487 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.487 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.487 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.487 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.554534+0000", 2023-10-09T11:09:00.487 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9399999999999999e-06 2023-10-09T11:09:00.487 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.487 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.487 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.487 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.561310+0000", 2023-10-09T11:09:00.488 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0067761460000000003 2023-10-09T11:09:00.488 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.488 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.488 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.488 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.561472+0000", 2023-10-09T11:09:00.488 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016197600000000001 2023-10-09T11:09:00.488 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.488 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.488 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:09:00.488 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.561638+0000", 2023-10-09T11:09:00.488 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000166341 2023-10-09T11:09:00.488 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.488 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.489 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.489 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.564331+0000", 2023-10-09T11:09:00.489 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026929190000000002 2023-10-09T11:09:00.489 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.489 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.489 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.489 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.569105+0000", 2023-10-09T11:09:00.489 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0047744650000000003 2023-10-09T11:09:00.489 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.489 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.489 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.489 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:07.569117+0000", 2023-10-09T11:09:00.490 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1493e-05 2023-10-09T11:09:00.490 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.490 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.490 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.490 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:08.575439+0000", 2023-10-09T11:09:00.490 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0063218190000001 2023-10-09T11:09:00.490 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.490 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.490 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.490 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:08.575487+0000", 2023-10-09T11:09:00.490 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7911999999999998e-05 2023-10-09T11:09:00.490 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.491 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.491 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.491 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.491 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.491 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87922 10.c 10:3a18da39:::benchmark_data_smithi125_402869_object87921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:09:00.491 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178226+0000", 2023-10-09T11:09:00.491 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 387.25171406499999, 2023-10-09T11:09:00.491 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1423559400000001, 2023-10-09T11:09:00.491 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.491 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.491 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.491 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.492 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.492 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87922 2023-10-09T11:09:00.492 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.492 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.492 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.492 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.492 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178226+0000", 2023-10-09T11:09:00.492 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.492 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.492 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.492 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.492 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178226+0000", 2023-10-09T11:09:00.493 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.493 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.493 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.493 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.493 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178225+0000", 2023-10-09T11:09:00.493 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:00.493 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.493 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.493 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.493 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178265+0000", 2023-10-09T11:09:00.493 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0617000000000001e-05 2023-10-09T11:09:00.494 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.494 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.494 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.494 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178278+0000", 2023-10-09T11:09:00.494 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2721999999999999e-05 2023-10-09T11:09:00.494 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.494 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.494 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.494 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178289+0000", 2023-10-09T11:09:00.494 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0787e-05 2023-10-09T11:09:00.494 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.494 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.495 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.495 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178908+0000", 2023-10-09T11:09:00.495 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00061957899999999998 2023-10-09T11:09:00.495 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.495 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.495 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.495 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178919+0000", 2023-10-09T11:09:00.495 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0808999999999999e-05 2023-10-09T11:09:00.495 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.495 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.495 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.495 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.306310+0000", 2023-10-09T11:09:00.495 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.12739133 2023-10-09T11:09:00.496 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.496 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.496 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.496 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.306501+0000", 2023-10-09T11:09:00.496 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000190779 2023-10-09T11:09:00.496 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.496 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.496 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,7", 2023-10-09T11:09:00.496 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.306691+0000", 2023-10-09T11:09:00.496 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000189328 2023-10-09T11:09:00.496 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.496 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.497 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.497 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316145+0000", 2023-10-09T11:09:00.497 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0094546479999999995 2023-10-09T11:09:00.497 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.497 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.497 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.497 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316181+0000", 2023-10-09T11:09:00.497 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5404000000000002e-05 2023-10-09T11:09:00.497 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.497 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.497 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.497 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316319+0000", 2023-10-09T11:09:00.498 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013836499999999999 2023-10-09T11:09:00.498 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.498 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.498 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.498 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320404+0000", 2023-10-09T11:09:00.498 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040849550000000004 2023-10-09T11:09:00.498 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.498 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.498 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.498 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320459+0000", 2023-10-09T11:09:00.498 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5013000000000003e-05 2023-10-09T11:09:00.498 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.498 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.499 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.499 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320582+0000", 2023-10-09T11:09:00.499 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012316000000000001 2023-10-09T11:09:00.499 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.499 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.499 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.499 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.499 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.499 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87923 10.1 10:84c65cae:::benchmark_data_smithi125_402869_object87922:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:09:00.499 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178299+0000", 2023-10-09T11:09:00.499 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 387.25164090700002, 2023-10-09T11:09:00.499 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.208577113, 2023-10-09T11:09:00.500 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.500 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.500 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.500 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.500 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.500 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87923 2023-10-09T11:09:00.500 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.500 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.500 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.500 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.500 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178299+0000", 2023-10-09T11:09:00.500 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.501 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.501 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.501 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.501 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178299+0000", 2023-10-09T11:09:00.501 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.501 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.501 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.501 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.501 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178299+0000", 2023-10-09T11:09:00.501 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:00.501 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.501 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.501 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.502 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178336+0000", 2023-10-09T11:09:00.502 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7317999999999997e-05 2023-10-09T11:09:00.502 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.502 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.502 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.502 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178344+0000", 2023-10-09T11:09:00.502 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3289999999999994e-06 2023-10-09T11:09:00.502 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.502 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.502 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.503 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178351+0000", 2023-10-09T11:09:00.503 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4570000000000002e-06 2023-10-09T11:09:00.503 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.503 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.503 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.503 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178912+0000", 2023-10-09T11:09:00.503 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00056114399999999997 2023-10-09T11:09:00.503 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.503 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.503 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.503 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178923+0000", 2023-10-09T11:09:00.503 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0679000000000001e-05 2023-10-09T11:09:00.503 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.504 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.504 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.504 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.378720+0000", 2023-10-09T11:09:00.504 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1997972379999999 2023-10-09T11:09:00.504 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.504 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.504 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.504 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.378826+0000", 2023-10-09T11:09:00.504 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010621200000000001 2023-10-09T11:09:00.504 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.504 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.504 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T11:09:00.505 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.378928+0000", 2023-10-09T11:09:00.505 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000101439 2023-10-09T11:09:00.505 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.505 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.505 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.505 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386610+0000", 2023-10-09T11:09:00.505 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076823810000000003 2023-10-09T11:09:00.505 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.505 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.505 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.505 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386650+0000", 2023-10-09T11:09:00.506 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0089999999999997e-05 2023-10-09T11:09:00.506 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.506 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.506 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.506 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386697+0000", 2023-10-09T11:09:00.506 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7126999999999998e-05 2023-10-09T11:09:00.506 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.507 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.507 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.507 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386761+0000", 2023-10-09T11:09:00.507 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3379000000000004e-05 2023-10-09T11:09:00.507 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.507 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.507 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.507 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386796+0000", 2023-10-09T11:09:00.507 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5688000000000001e-05 2023-10-09T11:09:00.507 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.507 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.507 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.508 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386876+0000", 2023-10-09T11:09:00.508 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0235999999999995e-05 2023-10-09T11:09:00.508 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.508 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.508 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.508 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.508 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.508 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87926 10.1 10:895d4d32:::benchmark_data_smithi125_402869_object87925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:09:00.509 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178357+0000", 2023-10-09T11:09:00.509 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 387.25158299499998, 2023-10-09T11:09:00.509 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.20874682, 2023-10-09T11:09:00.509 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.509 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.509 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.509 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.509 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.509 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87926 2023-10-09T11:09:00.509 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.509 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.510 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.510 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.510 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178357+0000", 2023-10-09T11:09:00.510 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.510 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.510 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.510 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.510 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178357+0000", 2023-10-09T11:09:00.510 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.511 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.511 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.511 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.511 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178357+0000", 2023-10-09T11:09:00.511 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:00.511 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.511 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.511 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.511 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178411+0000", 2023-10-09T11:09:00.511 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4347e-05 2023-10-09T11:09:00.511 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.511 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.512 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.512 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178419+0000", 2023-10-09T11:09:00.512 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4209999999999995e-06 2023-10-09T11:09:00.512 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.512 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.512 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.512 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178426+0000", 2023-10-09T11:09:00.512 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7150000000000001e-06 2023-10-09T11:09:00.512 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.513 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.513 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.513 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178946+0000", 2023-10-09T11:09:00.513 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000519769 2023-10-09T11:09:00.513 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.513 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.513 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.513 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178954+0000", 2023-10-09T11:09:00.513 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1110000000000008e-06 2023-10-09T11:09:00.513 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.513 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.514 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.514 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.381425+0000", 2023-10-09T11:09:00.514 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.20247094 2023-10-09T11:09:00.514 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.514 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.514 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.514 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.381541+0000", 2023-10-09T11:09:00.514 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000116335 2023-10-09T11:09:00.514 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.515 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.515 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T11:09:00.515 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.381643+0000", 2023-10-09T11:09:00.515 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010179900000000001 2023-10-09T11:09:00.515 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.515 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.515 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.515 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386707+0000", 2023-10-09T11:09:00.516 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0050636739999999998 2023-10-09T11:09:00.516 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.516 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.516 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.516 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386726+0000", 2023-10-09T11:09:00.516 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9184000000000001e-05 2023-10-09T11:09:00.516 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.516 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.516 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.516 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386917+0000", 2023-10-09T11:09:00.517 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019101099999999999 2023-10-09T11:09:00.517 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.517 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.517 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.517 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.386955+0000", 2023-10-09T11:09:00.517 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8059000000000003e-05 2023-10-09T11:09:00.517 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.517 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.518 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.518 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387002+0000", 2023-10-09T11:09:00.518 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7280999999999997e-05 2023-10-09T11:09:00.518 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.518 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.518 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.518 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387104+0000", 2023-10-09T11:09:00.518 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000101885 2023-10-09T11:09:00.519 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.519 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.519 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.519 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.519 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.519 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87932 10.c 10:32331c82:::benchmark_data_smithi125_402869_object87931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:09:00.519 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178432+0000", 2023-10-09T11:09:00.519 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 387.25150868600002, 2023-10-09T11:09:00.519 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1426129899999999, 2023-10-09T11:09:00.519 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.519 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.519 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.519 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.520 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.520 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87932 2023-10-09T11:09:00.520 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.520 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.520 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.520 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.520 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178432+0000", 2023-10-09T11:09:00.520 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.520 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.520 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.520 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.521 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178432+0000", 2023-10-09T11:09:00.521 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.521 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.521 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.521 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.521 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178431+0000", 2023-10-09T11:09:00.521 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:00.521 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.521 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.521 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.521 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178498+0000", 2023-10-09T11:09:00.521 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6989000000000005e-05 2023-10-09T11:09:00.521 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.522 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.522 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.522 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178507+0000", 2023-10-09T11:09:00.522 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6000000000000007e-06 2023-10-09T11:09:00.522 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.522 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.522 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.522 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178514+0000", 2023-10-09T11:09:00.522 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9029999999999998e-06 2023-10-09T11:09:00.522 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.522 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.523 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.523 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178941+0000", 2023-10-09T11:09:00.523 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000427873 2023-10-09T11:09:00.523 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.523 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.523 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.523 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178949+0000", 2023-10-09T11:09:00.523 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8080000000000005e-06 2023-10-09T11:09:00.523 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.523 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.523 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.523 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.309831+0000", 2023-10-09T11:09:00.523 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1308817790000001 2023-10-09T11:09:00.524 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.524 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.524 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.524 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.310021+0000", 2023-10-09T11:09:00.524 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019044099999999999 2023-10-09T11:09:00.524 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.524 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.524 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,7", 2023-10-09T11:09:00.524 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.310197+0000", 2023-10-09T11:09:00.524 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000175231 2023-10-09T11:09:00.524 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.524 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.525 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.525 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316241+0000", 2023-10-09T11:09:00.525 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.006044715 2023-10-09T11:09:00.525 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.525 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.525 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.525 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316249+0000", 2023-10-09T11:09:00.525 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7030000000000006e-06 2023-10-09T11:09:00.525 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.525 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.525 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.525 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320823+0000", 2023-10-09T11:09:00.526 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004573433 2023-10-09T11:09:00.526 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.526 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.526 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.526 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320877+0000", 2023-10-09T11:09:00.526 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4082000000000002e-05 2023-10-09T11:09:00.526 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.526 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.526 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.526 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320939+0000", 2023-10-09T11:09:00.526 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2507000000000003e-05 2023-10-09T11:09:00.526 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.527 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.527 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.527 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.321045+0000", 2023-10-09T11:09:00.527 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010545 2023-10-09T11:09:00.527 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.527 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.527 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.527 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.527 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.527 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87933 10.1 10:8333223a:::benchmark_data_smithi125_402869_object87932:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:09:00.527 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178522+0000", 2023-10-09T11:09:00.527 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 387.25141850699998, 2023-10-09T11:09:00.528 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2145040599999999, 2023-10-09T11:09:00.528 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.528 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.528 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.528 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.528 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.528 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87933 2023-10-09T11:09:00.528 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.528 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.528 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.528 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.528 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178522+0000", 2023-10-09T11:09:00.529 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.529 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.529 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.529 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.529 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178522+0000", 2023-10-09T11:09:00.529 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.529 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.529 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.529 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.529 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178521+0000", 2023-10-09T11:09:00.529 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:00.529 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.529 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.530 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.530 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178569+0000", 2023-10-09T11:09:00.530 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7472999999999998e-05 2023-10-09T11:09:00.530 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.530 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.530 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.530 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178577+0000", 2023-10-09T11:09:00.530 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1839999999999999e-06 2023-10-09T11:09:00.530 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.530 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.530 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.530 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178583+0000", 2023-10-09T11:09:00.531 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1e-06 2023-10-09T11:09:00.531 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.531 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.531 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.531 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178976+0000", 2023-10-09T11:09:00.531 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000393216 2023-10-09T11:09:00.531 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.531 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.531 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.531 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178984+0000", 2023-10-09T11:09:00.531 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0199999999999994e-06 2023-10-09T11:09:00.531 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.532 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.532 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.532 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.383925+0000", 2023-10-09T11:09:00.532 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2049411139999999 2023-10-09T11:09:00.532 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.532 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.532 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.532 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.384095+0000", 2023-10-09T11:09:00.532 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016973699999999999 2023-10-09T11:09:00.532 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.532 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.532 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T11:09:00.532 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.384262+0000", 2023-10-09T11:09:00.533 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000166964 2023-10-09T11:09:00.533 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.533 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.533 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.533 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387118+0000", 2023-10-09T11:09:00.533 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028561709999999998 2023-10-09T11:09:00.533 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.533 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.533 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.533 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387158+0000", 2023-10-09T11:09:00.533 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9368e-05 2023-10-09T11:09:00.533 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.534 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.534 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.534 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387200+0000", 2023-10-09T11:09:00.534 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2500999999999998e-05 2023-10-09T11:09:00.534 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.534 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.534 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.534 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.392880+0000", 2023-10-09T11:09:00.534 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0056800849999999996 2023-10-09T11:09:00.534 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.534 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.534 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.535 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.392916+0000", 2023-10-09T11:09:00.535 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6239999999999999e-05 2023-10-09T11:09:00.535 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.535 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.535 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.535 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393026+0000", 2023-10-09T11:09:00.535 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010939500000000001 2023-10-09T11:09:00.535 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.535 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.535 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.535 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.535 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.536 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87943 10.1 10:802b3a31:::benchmark_data_smithi125_402869_object87942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:09:00.536 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178589+0000", 2023-10-09T11:09:00.536 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 387.25135175100002, 2023-10-09T11:09:00.536 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.214764771, 2023-10-09T11:09:00.536 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.536 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.536 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.536 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.536 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.536 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87943 2023-10-09T11:09:00.536 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.537 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.537 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.537 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.537 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178589+0000", 2023-10-09T11:09:00.537 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.537 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.537 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.537 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.537 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178589+0000", 2023-10-09T11:09:00.537 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.537 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.537 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.537 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.538 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178588+0000", 2023-10-09T11:09:00.538 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:00.538 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.538 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.538 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.538 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178633+0000", 2023-10-09T11:09:00.538 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4960000000000003e-05 2023-10-09T11:09:00.538 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.538 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.538 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.539 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178641+0000", 2023-10-09T11:09:00.539 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8359999999999992e-06 2023-10-09T11:09:00.539 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.539 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.539 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.539 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178648+0000", 2023-10-09T11:09:00.539 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6750000000000002e-06 2023-10-09T11:09:00.539 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.539 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.540 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.540 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.179006+0000", 2023-10-09T11:09:00.540 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035727099999999999 2023-10-09T11:09:00.540 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.540 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.540 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.540 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.179014+0000", 2023-10-09T11:09:00.540 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9580000000000007e-06 2023-10-09T11:09:00.540 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.540 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.540 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.540 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387221+0000", 2023-10-09T11:09:00.541 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2082074030000001 2023-10-09T11:09:00.541 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.541 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.541 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.541 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387349+0000", 2023-10-09T11:09:00.541 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012751599999999999 2023-10-09T11:09:00.541 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.541 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.541 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T11:09:00.542 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.387477+0000", 2023-10-09T11:09:00.542 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012794799999999999 2023-10-09T11:09:00.542 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.542 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.542 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.542 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393068+0000", 2023-10-09T11:09:00.542 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0055912039999999998 2023-10-09T11:09:00.542 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.542 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.542 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.542 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393078+0000", 2023-10-09T11:09:00.543 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0084e-05 2023-10-09T11:09:00.543 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.543 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.543 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.543 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393133+0000", 2023-10-09T11:09:00.543 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4762999999999997e-05 2023-10-09T11:09:00.543 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.543 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.544 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.544 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393179+0000", 2023-10-09T11:09:00.544 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6622999999999997e-05 2023-10-09T11:09:00.544 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.544 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.544 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.544 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393248+0000", 2023-10-09T11:09:00.544 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9108000000000003e-05 2023-10-09T11:09:00.544 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.544 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.544 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.544 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393353+0000", 2023-10-09T11:09:00.545 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000104959 2023-10-09T11:09:00.545 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.545 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.545 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.545 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.545 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.545 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87970 10.c 10:397744e6:::benchmark_data_smithi125_402869_object87969:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:09:00.545 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178654+0000", 2023-10-09T11:09:00.545 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 387.25128614200003, 2023-10-09T11:09:00.546 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2299234779999999, 2023-10-09T11:09:00.546 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.546 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.546 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.546 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.546 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.546 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87970 2023-10-09T11:09:00.546 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.546 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.546 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.546 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.546 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178654+0000", 2023-10-09T11:09:00.547 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.547 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.547 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.547 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.547 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178654+0000", 2023-10-09T11:09:00.547 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.547 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.547 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.547 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.547 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178654+0000", 2023-10-09T11:09:00.547 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:00.547 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.548 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.548 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.548 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178698+0000", 2023-10-09T11:09:00.548 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4047e-05 2023-10-09T11:09:00.548 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.548 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.548 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.548 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178706+0000", 2023-10-09T11:09:00.548 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8619999999999992e-06 2023-10-09T11:09:00.548 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.548 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.548 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.548 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178713+0000", 2023-10-09T11:09:00.549 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.46e-06 2023-10-09T11:09:00.549 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.549 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.549 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.549 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178969+0000", 2023-10-09T11:09:00.549 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000256562 2023-10-09T11:09:00.549 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.549 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.549 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.549 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178977+0000", 2023-10-09T11:09:00.549 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.204e-06 2023-10-09T11:09:00.549 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.550 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.550 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.550 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.313409+0000", 2023-10-09T11:09:00.550 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1344324960000001 2023-10-09T11:09:00.550 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.550 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.550 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.550 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.313530+0000", 2023-10-09T11:09:00.551 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000121099 2023-10-09T11:09:00.551 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.551 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.551 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,7", 2023-10-09T11:09:00.551 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.313651+0000", 2023-10-09T11:09:00.551 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120791 2023-10-09T11:09:00.551 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.551 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.551 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.551 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320352+0000", 2023-10-09T11:09:00.552 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0067007380000000004 2023-10-09T11:09:00.552 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.552 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.552 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.552 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.320631+0000", 2023-10-09T11:09:00.552 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027918300000000001 2023-10-09T11:09:00.552 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.552 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.552 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.553 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.321145+0000", 2023-10-09T11:09:00.553 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051379300000000002 2023-10-09T11:09:00.553 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.553 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.553 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.553 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.408399+0000", 2023-10-09T11:09:00.553 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.087254527999999998 2023-10-09T11:09:00.553 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.554 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.554 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.554 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.408459+0000", 2023-10-09T11:09:00.554 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9611000000000002e-05 2023-10-09T11:09:00.554 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.554 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.554 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.554 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.408578+0000", 2023-10-09T11:09:00.555 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118651 2023-10-09T11:09:00.555 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.555 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.555 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.555 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.555 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.555 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87980 10.c 10:372347b9:::benchmark_data_smithi125_402869_object87979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:09:00.555 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178720+0000", 2023-10-09T11:09:00.555 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 387.25122048999998, 2023-10-09T11:09:00.555 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2300607610000001, 2023-10-09T11:09:00.555 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.556 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.556 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.556 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.556 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.556 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87980 2023-10-09T11:09:00.556 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.556 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.556 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.556 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.556 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178720+0000", 2023-10-09T11:09:00.556 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.556 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.557 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.557 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.557 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178720+0000", 2023-10-09T11:09:00.557 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.557 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.557 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.557 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.557 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178719+0000", 2023-10-09T11:09:00.557 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:00.557 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.557 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.557 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.557 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178763+0000", 2023-10-09T11:09:00.558 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3375999999999999e-05 2023-10-09T11:09:00.558 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.558 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.558 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.558 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178771+0000", 2023-10-09T11:09:00.558 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0530000000000007e-06 2023-10-09T11:09:00.558 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.558 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.558 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.558 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178777+0000", 2023-10-09T11:09:00.558 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3620000000000002e-06 2023-10-09T11:09:00.559 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.559 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.559 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.559 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178996+0000", 2023-10-09T11:09:00.559 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00021923799999999999 2023-10-09T11:09:00.559 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.559 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.559 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.559 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.179003+0000", 2023-10-09T11:09:00.559 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9890000000000003e-06 2023-10-09T11:09:00.559 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.559 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.559 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.560 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316530+0000", 2023-10-09T11:09:00.560 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.137527073 2023-10-09T11:09:00.560 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.560 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.560 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.560 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316665+0000", 2023-10-09T11:09:00.560 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013503899999999999 2023-10-09T11:09:00.560 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.560 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.560 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,7", 2023-10-09T11:09:00.560 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.316850+0000", 2023-10-09T11:09:00.560 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000185062 2023-10-09T11:09:00.561 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.561 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.561 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.561 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.321062+0000", 2023-10-09T11:09:00.561 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042115169999999997 2023-10-09T11:09:00.561 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.561 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.561 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.561 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.321100+0000", 2023-10-09T11:09:00.561 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7601000000000001e-05 2023-10-09T11:09:00.561 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.561 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.561 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.562 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.321193+0000", 2023-10-09T11:09:00.562 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3624999999999999e-05 2023-10-09T11:09:00.562 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.562 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.562 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.562 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.408637+0000", 2023-10-09T11:09:00.562 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.087443641000000003 2023-10-09T11:09:00.562 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.562 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.562 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.562 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.408684+0000", 2023-10-09T11:09:00.563 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7571999999999998e-05 2023-10-09T11:09:00.563 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.563 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.563 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.563 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.408781+0000", 2023-10-09T11:09:00.563 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6157999999999997e-05 2023-10-09T11:09:00.563 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.563 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.563 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.563 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.563 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.563 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:87987 10.1 10:834eb049:::benchmark_data_smithi125_402869_object87986:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e435)", 2023-10-09T11:09:00.563 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:33.178783+0000", 2023-10-09T11:09:00.564 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 387.25115741100001, 2023-10-09T11:09:00.564 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2178343309999999, 2023-10-09T11:09:00.564 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.564 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.564 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.564 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.564 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.564 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87987 2023-10-09T11:09:00.564 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.564 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.564 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.564 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.565 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178783+0000", 2023-10-09T11:09:00.565 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.565 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.565 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.565 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.565 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178783+0000", 2023-10-09T11:09:00.565 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.565 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.565 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.565 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.565 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178782+0000", 2023-10-09T11:09:00.565 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:00.565 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.566 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.566 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.566 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178824+0000", 2023-10-09T11:09:00.566 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2045999999999997e-05 2023-10-09T11:09:00.566 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.566 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.566 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.566 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178832+0000", 2023-10-09T11:09:00.566 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6930000000000007e-06 2023-10-09T11:09:00.566 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.566 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.567 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.567 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.178838+0000", 2023-10-09T11:09:00.567 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0929999999999999e-06 2023-10-09T11:09:00.567 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.567 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.567 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.567 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.179035+0000", 2023-10-09T11:09:00.567 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000197152 2023-10-09T11:09:00.567 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.567 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.567 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.567 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:33.179043+0000", 2023-10-09T11:09:00.567 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9300000000000003e-06 2023-10-09T11:09:00.568 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.568 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.568 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.568 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.390120+0000", 2023-10-09T11:09:00.568 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.211076751 2023-10-09T11:09:00.568 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.568 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.568 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.568 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.390256+0000", 2023-10-09T11:09:00.568 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013596300000000001 2023-10-09T11:09:00.568 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.568 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.569 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T11:09:00.569 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.390361+0000", 2023-10-09T11:09:00.569 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000104608 2023-10-09T11:09:00.569 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.569 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.569 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.569 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393371+0000", 2023-10-09T11:09:00.569 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030100359999999998 2023-10-09T11:09:00.569 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.569 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.569 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.570 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.393424+0000", 2023-10-09T11:09:00.570 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3146000000000002e-05 2023-10-09T11:09:00.570 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.570 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.570 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.570 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.396331+0000", 2023-10-09T11:09:00.570 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029073200000000001 2023-10-09T11:09:00.570 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.570 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.570 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.570 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.396445+0000", 2023-10-09T11:09:00.570 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011435 2023-10-09T11:09:00.571 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.571 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.571 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.571 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.396501+0000", 2023-10-09T11:09:00.571 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5992000000000002e-05 2023-10-09T11:09:00.571 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.571 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.571 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.571 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:34.396617+0000", 2023-10-09T11:09:00.571 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000115742 2023-10-09T11:09:00.571 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.571 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.571 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.572 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.572 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.572 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:98555 10.11 10:8b1dad8d:::benchmark_data_smithi125_402869_object98554:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e443)", 2023-10-09T11:09:00.572 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:50.335541+0000", 2023-10-09T11:09:00.572 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 370.09439881100002, 2023-10-09T11:09:00.572 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0107502129999999, 2023-10-09T11:09:00.572 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.572 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.572 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.572 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.572 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.573 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 98555 2023-10-09T11:09:00.573 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.573 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.573 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.573 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.573 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335541+0000", 2023-10-09T11:09:00.573 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.573 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.573 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.573 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.573 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335541+0000", 2023-10-09T11:09:00.573 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.574 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.574 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.574 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.574 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335540+0000", 2023-10-09T11:09:00.574 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:00.574 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.574 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.574 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.574 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335565+0000", 2023-10-09T11:09:00.574 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4861000000000001e-05 2023-10-09T11:09:00.574 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.574 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.574 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.575 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335578+0000", 2023-10-09T11:09:00.575 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3101e-05 2023-10-09T11:09:00.575 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.575 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.575 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.575 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.335588+0000", 2023-10-09T11:09:00.575 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0098000000000001e-05 2023-10-09T11:09:00.575 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.575 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.575 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.575 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336134+0000", 2023-10-09T11:09:00.576 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00054624999999999997 2023-10-09T11:09:00.576 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.576 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.576 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.576 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336146+0000", 2023-10-09T11:09:00.576 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1460999999999999e-05 2023-10-09T11:09:00.576 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.576 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.576 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.576 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.341862+0000", 2023-10-09T11:09:00.576 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.005716211 2023-10-09T11:09:00.576 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.576 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.577 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.577 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.341988+0000", 2023-10-09T11:09:00.577 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012611400000000001 2023-10-09T11:09:00.577 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.577 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.577 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T11:09:00.577 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.342191+0000", 2023-10-09T11:09:00.577 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020272700000000001 2023-10-09T11:09:00.577 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.577 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.578 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.578 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.344401+0000", 2023-10-09T11:09:00.578 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022105839999999998 2023-10-09T11:09:00.578 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.578 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.578 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.578 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.344426+0000", 2023-10-09T11:09:00.578 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5215000000000002e-05 2023-10-09T11:09:00.578 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.579 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.579 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.579 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.345867+0000", 2023-10-09T11:09:00.579 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001441035 2023-10-09T11:09:00.579 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.579 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.579 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.579 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.346178+0000", 2023-10-09T11:09:00.579 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000310475 2023-10-09T11:09:00.579 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.579 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.579 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.580 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.346219+0000", 2023-10-09T11:09:00.580 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1458000000000002e-05 2023-10-09T11:09:00.580 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.580 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.580 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.580 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.346292+0000", 2023-10-09T11:09:00.580 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2249000000000006e-05 2023-10-09T11:09:00.580 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.581 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.581 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.581 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.581 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.581 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:98558 10.11 10:89be634c:::benchmark_data_smithi125_402869_object98557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e443)", 2023-10-09T11:09:00.581 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:02:50.336104+0000", 2023-10-09T11:09:00.581 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 370.09383629799999, 2023-10-09T11:09:00.581 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0103175630000001, 2023-10-09T11:09:00.581 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.581 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.581 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.581 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.582 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.582 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 98558 2023-10-09T11:09:00.582 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.582 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.582 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.582 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.582 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336104+0000", 2023-10-09T11:09:00.582 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.582 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.583 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.583 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.583 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336104+0000", 2023-10-09T11:09:00.583 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.583 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.583 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.583 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.583 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336103+0000", 2023-10-09T11:09:00.583 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:00.584 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.584 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.584 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.584 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336129+0000", 2023-10-09T11:09:00.584 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5403000000000001e-05 2023-10-09T11:09:00.584 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.584 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.584 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.584 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336140+0000", 2023-10-09T11:09:00.585 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1060000000000001e-05 2023-10-09T11:09:00.585 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.585 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.585 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.585 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336149+0000", 2023-10-09T11:09:00.585 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8810000000000002e-06 2023-10-09T11:09:00.585 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.585 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.585 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.585 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336186+0000", 2023-10-09T11:09:00.586 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6980999999999997e-05 2023-10-09T11:09:00.586 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.586 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.586 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.586 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:50.336192+0000", 2023-10-09T11:09:00.586 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7560000000000002e-06 2023-10-09T11:09:00.586 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.586 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.587 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.587 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.343404+0000", 2023-10-09T11:09:00.587 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0072118370000001 2023-10-09T11:09:00.587 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.587 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.587 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.588 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.343498+0000", 2023-10-09T11:09:00.588 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3412000000000006e-05 2023-10-09T11:09:00.588 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.588 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.588 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4,7", 2023-10-09T11:09:00.588 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.343638+0000", 2023-10-09T11:09:00.588 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014086200000000001 2023-10-09T11:09:00.588 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.588 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.588 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.588 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.344550+0000", 2023-10-09T11:09:00.589 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00091130400000000004 2023-10-09T11:09:00.589 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.589 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.589 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.589 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.344753+0000", 2023-10-09T11:09:00.589 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000203329 2023-10-09T11:09:00.589 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.589 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.589 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.589 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.346008+0000", 2023-10-09T11:09:00.589 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012553180000000001 2023-10-09T11:09:00.589 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.590 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.590 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.590 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.346322+0000", 2023-10-09T11:09:00.590 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00031327699999999999 2023-10-09T11:09:00.590 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.590 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.590 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.590 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.346356+0000", 2023-10-09T11:09:00.590 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3915999999999999e-05 2023-10-09T11:09:00.590 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.590 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.590 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.591 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:02:51.346422+0000", 2023-10-09T11:09:00.591 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5908999999999995e-05 2023-10-09T11:09:00.591 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.591 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.591 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.591 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.591 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.591 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36123.0:179298 10.4 10:22bc55cd:::benchmark_data_smithi125_402869_object80729:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-10-09T11:09:00.591 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:04:22.139061+0000", 2023-10-09T11:09:00.591 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 278.29087910099997, 2023-10-09T11:09:00.591 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96191459000000001, 2023-10-09T11:09:00.591 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.592 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.592 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:00.592 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36123", 2023-10-09T11:09:00.592 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:00.592 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 179298 2023-10-09T11:09:00.592 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.592 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.592 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.592 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.592 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139061+0000", 2023-10-09T11:09:00.592 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.592 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.593 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.593 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.593 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139061+0000", 2023-10-09T11:09:00.593 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.593 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.593 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.593 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.593 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139061+0000", 2023-10-09T11:09:00.593 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:00.593 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.593 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.593 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.593 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139064+0000", 2023-10-09T11:09:00.594 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5810000000000001e-06 2023-10-09T11:09:00.594 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.594 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.594 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.594 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139072+0000", 2023-10-09T11:09:00.594 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0299999999999994e-06 2023-10-09T11:09:00.594 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.594 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.594 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.594 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139079+0000", 2023-10-09T11:09:00.594 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3550000000000001e-06 2023-10-09T11:09:00.595 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.595 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.595 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.595 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139322+0000", 2023-10-09T11:09:00.595 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00024333699999999999 2023-10-09T11:09:00.595 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.595 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.595 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:00.595 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:22.139330+0000", 2023-10-09T11:09:00.595 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1629999999999996e-06 2023-10-09T11:09:00.595 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.595 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.595 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.596 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.094275+0000", 2023-10-09T11:09:00.596 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95494484800000001 2023-10-09T11:09:00.596 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.596 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.596 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.596 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.094367+0000", 2023-10-09T11:09:00.596 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1500000000000001e-05 2023-10-09T11:09:00.596 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.596 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.596 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T11:09:00.596 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.094649+0000", 2023-10-09T11:09:00.596 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00028209399999999999 2023-10-09T11:09:00.597 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.597 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.597 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:00.597 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.100786+0000", 2023-10-09T11:09:00.597 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0061369880000000003 2023-10-09T11:09:00.597 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.597 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.597 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.597 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.100840+0000", 2023-10-09T11:09:00.597 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4830000000000002e-05 2023-10-09T11:09:00.597 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.597 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.598 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:00.598 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.100888+0000", 2023-10-09T11:09:00.598 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7370999999999998e-05 2023-10-09T11:09:00.598 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.598 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.598 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.598 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.100936+0000", 2023-10-09T11:09:00.598 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8653999999999999e-05 2023-10-09T11:09:00.598 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.598 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.598 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.598 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:04:23.100976+0000", 2023-10-09T11:09:00.598 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9283000000000003e-05 2023-10-09T11:09:00.599 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.599 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.599 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.599 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.599 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.599 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.36510.0:142420 11.1f e545/531 11:fb0ad690:::benchmark_data_smithi125_439543_object42921:head v 545'7745, mlcod=507'2640)", 2023-10-09T11:09:00.599 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:07:23.911276+0000", 2023-10-09T11:09:00.599 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 96.518663896999996, 2023-10-09T11:09:00.599 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.110437917, 2023-10-09T11:09:00.599 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:00.599 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:00.599 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:00.600 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.600 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:00.600 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:23.911276+0000", 2023-10-09T11:09:00.600 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.600 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.600 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.600 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:00.600 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:23.911276+0000", 2023-10-09T11:09:00.600 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:00.600 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.600 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.600 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:00.600 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:23.911276+0000", 2023-10-09T11:09:00.601 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:00.601 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.601 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.601 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:00.601 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:23.911281+0000", 2023-10-09T11:09:00.601 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5419999999999996e-06 2023-10-09T11:09:00.601 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.601 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.601 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:00.601 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:24.015537+0000", 2023-10-09T11:09:00.601 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.104255523 2023-10-09T11:09:00.601 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.602 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.602 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:00.602 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:24.015549+0000", 2023-10-09T11:09:00.602 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2184000000000001e-05 2023-10-09T11:09:00.602 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.602 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.602 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:00.602 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:24.018417+0000", 2023-10-09T11:09:00.602 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028677540000000001 2023-10-09T11:09:00.602 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.602 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.602 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:00.603 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:24.018437+0000", 2023-10-09T11:09:00.603 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9831999999999999e-05 2023-10-09T11:09:00.603 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.603 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.603 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:00.603 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:24.021682+0000", 2023-10-09T11:09:00.603 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032458330000000001 2023-10-09T11:09:00.603 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:00.603 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:00.603 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:00.603 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:07:24.021714+0000", 2023-10-09T11:09:00.603 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1776000000000002e-05 2023-10-09T11:09:00.604 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.604 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.604 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.604 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:00.604 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:00.604 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:09:00.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:00.605 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:00.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:00.704 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:09:00.706 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:09:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:00 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3074686345' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:09:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:09:00.729 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:00 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3074686345' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:09:00.730 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:09:00.730 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:00 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3074686345' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:09:00.730 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:09:00.737 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:00.758 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:09:00.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:00.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:09:00.862 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:00.890 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:00.943 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T11:09:00.959 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:00.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:00.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:01.034 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:01.063 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:09:01.093 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:01.112 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:09:01.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:01.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:09:01.216 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:09:01.244 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:09:01] "GET /metrics HTTP/1.1" 200 34108 "" "Prometheus/2.43.0" 2023-10-09T11:09:01.246 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:01.266 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:09:01.269 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:09:01.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:01.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:09:01.371 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:01.402 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:01.425 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:01.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:01.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:01.457 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:01.530 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:09:01.557 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:01.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[200197]: pgmap v1408: 103 pgs: 103 active+clean; 582 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:01.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:09:01.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[200197]: failed to encode map e575 with expected crc 2023-10-09T11:09:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3074686345' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:09:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[200197]: osdmap e575: 8 total, 8 up, 8 in 2023-10-09T11:09:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[200197]: failed to encode map e575 with expected crc 2023-10-09T11:09:01.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[200197]: failed to encode map e575 with expected crc 2023-10-09T11:09:01.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:01.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[195507]: pgmap v1408: 103 pgs: 103 active+clean; 582 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:01.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:09:01.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[195507]: failed to encode map e575 with expected crc 2023-10-09T11:09:01.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3074686345' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:09:01.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[195507]: osdmap e575: 8 total, 8 up, 8 in 2023-10-09T11:09:01.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[195507]: failed to encode map e575 with expected crc 2023-10-09T11:09:01.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[195507]: failed to encode map e575 with expected crc 2023-10-09T11:09:01.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:01 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:01.575 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:09:01.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:01.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:09:01.680 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:09:01.711 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:01.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:01 smithi155 ceph-mon[184854]: pgmap v1408: 103 pgs: 103 active+clean; 582 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:01.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:01 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:09:01.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:01 smithi155 ceph-mon[184854]: failed to encode map e575 with expected crc 2023-10-09T11:09:01.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3074686345' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:09:01.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:01 smithi155 ceph-mon[184854]: osdmap e575: 8 total, 8 up, 8 in 2023-10-09T11:09:01.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:01 smithi155 ceph-mon[184854]: failed to encode map e575 with expected crc 2023-10-09T11:09:01.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:01 smithi155 ceph-mon[184854]: failed to encode map e575 with expected crc 2023-10-09T11:09:01.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:01 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:01.832 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:09:01.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:01.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:09:01.937 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:09:01.970 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:02.003 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-09T11:09:02.004 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T11:09:02.007 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg /tmp/rbd_import_export_475522/img2 2023-10-09T11:09:02.042 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:09:02.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:02.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:09:02.148 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:09:02.176 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:02.196 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:09:02.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:02.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:09:02.300 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:09:02.301 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T11:09:02.304 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg - 2023-10-09T11:09:02.327 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:02.346 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:09:02.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:02.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:09:02.449 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:09:02.476 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:02.495 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:09:02.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:02.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:09:02.598 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:09:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:02 smithi155 ceph-mon[184854]: osdmap e576: 8 total, 8 up, 8 in 2023-10-09T11:09:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:02 smithi155 ceph-mon[184854]: failed to encode map e576 with expected crc 2023-10-09T11:09:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:02 smithi155 ceph-mon[184854]: failed to encode map e576 with expected crc 2023-10-09T11:09:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:02 smithi155 ceph-mon[184854]: failed to encode map e576 with expected crc 2023-10-09T11:09:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:02 smithi155 ceph-mon[184854]: pgmap v1411: 103 pgs: 103 active+clean; 582 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-10-09T11:09:02.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:02 smithi125 ceph-mon[195507]: osdmap e576: 8 total, 8 up, 8 in 2023-10-09T11:09:02.747 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:02 smithi125 ceph-mon[195507]: failed to encode map e576 with expected crc 2023-10-09T11:09:02.747 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:02 smithi125 ceph-mon[195507]: failed to encode map e576 with expected crc 2023-10-09T11:09:02.747 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:02 smithi125 ceph-mon[195507]: failed to encode map e576 with expected crc 2023-10-09T11:09:02.747 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:02 smithi125 ceph-mon[195507]: pgmap v1411: 103 pgs: 103 active+clean; 582 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-10-09T11:09:02.747 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:02 smithi125 ceph-mon[200197]: osdmap e576: 8 total, 8 up, 8 in 2023-10-09T11:09:02.747 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:02 smithi125 ceph-mon[200197]: failed to encode map e576 with expected crc 2023-10-09T11:09:02.747 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:02 smithi125 ceph-mon[200197]: failed to encode map e576 with expected crc 2023-10-09T11:09:02.747 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:02 smithi125 ceph-mon[200197]: failed to encode map e576 with expected crc 2023-10-09T11:09:02.747 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:02 smithi125 ceph-mon[200197]: pgmap v1411: 103 pgs: 103 active+clean; 582 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-10-09T11:09:02.749 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:02.818 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:09:03.068 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T11:09:03.170 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm testimg 2023-10-09T11:09:03.170 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T11:09:03.170 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp /tmp/rbd_import_export_475522/img /tmp/rbd_import_export_475522/img2 2023-10-09T11:09:03.175 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:09:03.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:03.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:09:03.284 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:09:03.311 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:03.330 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:09:03.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:03.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:09:03.435 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:09:03.462 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:03.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:09:03.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:03.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:09:03.584 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:09:03.619 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:03.840 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:09:03.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:03.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:09:03.939 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp /tmp/rbd_import_export_475522/img /tmp/rbd_import_export_475522/img3 2023-10-09T11:09:03.998 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:09:04.025 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:04.044 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:09:04.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:04.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:09:04.147 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:09:04.182 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:04.252 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:09:04.273 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:09:04.295 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:09:04.300 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:04.309 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:09:04.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:04.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:09:04.334 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:09:04.362 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:09:04.386 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:09:04.411 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:09:04.413 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:04.423 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/img2 /tmp/rbd_import_export_475522/img3 2023-10-09T11:09:04.436 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:09:04.437 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:04 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2244700466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:09:04.437 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:04 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2244700466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:09:04.443 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:09:04.443 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:09:04.443 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:09:04.444 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:09:04.444 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:09:04.444 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:09:04.444 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:09:04.444 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:09:04.451 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:04.670 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format - testimg 2023-10-09T11:09:04.675 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:04.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:04.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:04.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:04 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2244700466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:09:04.783 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:09:04.811 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:04.829 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:09:04.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:04.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:09:04.874 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:04.874 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:04.932 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:09:04.959 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:04.978 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:09:04.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:04.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:09:05.081 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:09:05.096 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T11:09:05.096 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T11:09:05.099 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg /tmp/rbd_import_export_475522/img2 2023-10-09T11:09:05.109 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:05.128 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:09:05.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:05.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:09:05.234 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:05.271 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:05.398 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:05.399 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T11:09:05.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:05.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:05.403 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg - 2023-10-09T11:09:05.503 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:09:05.530 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:05.541 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:05 smithi125 ceph-mon[195507]: pgmap v1412: 103 pgs: 103 active+clean; 582 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 895 B/s rd, 255 B/s wr, 1 op/s 2023-10-09T11:09:05.541 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:05 smithi125 ceph-mon[200197]: pgmap v1412: 103 pgs: 103 active+clean; 582 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 895 B/s rd, 255 B/s wr, 1 op/s 2023-10-09T11:09:05.548 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:09:05.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:05.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:09:05.592 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:05.602 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T11:09:05.652 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:09:05.677 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:05 smithi155 ceph-mon[184854]: pgmap v1412: 103 pgs: 103 active+clean; 582 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 895 B/s rd, 255 B/s wr, 1 op/s 2023-10-09T11:09:05.679 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:05.697 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:09:05.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:05.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:09:05.801 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:09:05.828 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:05.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:09:05.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:05.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:09:05.951 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:09:05.981 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:06.001 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:09:06.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:06.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:09:06.108 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:09:06.139 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:06.215 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T11:09:06.218 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm testimg 2023-10-09T11:09:06.239 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:06.268 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:09:06.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:06.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:09:06.308 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T11:09:06.313 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp /tmp/rbd_import_export_475522/img /tmp/rbd_import_export_475522/img2 2023-10-09T11:09:06.369 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:09:06.371 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:09:06.373 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:09:06.447 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:06.627 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:09:06.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:06.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:09:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:06 smithi155 ceph-mon[184854]: pgmap v1413: 103 pgs: 103 active+clean; 3.5 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 215 KiB/s rd, 96 KiB/s wr, 48 op/s 2023-10-09T11:09:06.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:06 smithi155 ceph-mon[184854]: from='client.46871 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:09:06.730 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:09:06.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:06 smithi125 ceph-mon[195507]: pgmap v1413: 103 pgs: 103 active+clean; 3.5 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 215 KiB/s rd, 96 KiB/s wr, 48 op/s 2023-10-09T11:09:06.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:06 smithi125 ceph-mon[195507]: from='client.46871 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:09:06.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:06 smithi125 ceph-mon[200197]: pgmap v1413: 103 pgs: 103 active+clean; 3.5 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 215 KiB/s rd, 96 KiB/s wr, 48 op/s 2023-10-09T11:09:06.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:06 smithi125 ceph-mon[200197]: from='client.46871 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:09:06.758 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:06.777 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:09:06.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:06.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:09:06.880 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:09:06.912 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:06.930 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:09:06.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:06.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:09:06.971 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp /tmp/rbd_import_export_475522/img /tmp/rbd_import_export_475522/img3 2023-10-09T11:09:07.033 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:07.067 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:07.194 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:07.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:07.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:07.298 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:09:07.330 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:07.406 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:09:07.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:07.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:09:07.413 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1413,"stamp":"2023-10-09T11:09:05.935661+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3692563,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":315,"num_read_kb":1718,"num_write":136,"num_write_kb":797,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":2377214,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5126,"ondisk_log_size":5126,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":316,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7285384,"kb_used_data":228740,"kb_used_omap":134,"kb_used_meta":7056441,"kb_avail":742675832,"statfs":{"total":767960285184,"available":760500051968,"internally_reserved":0,"allocated":234229760,"data_stored":214462834,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":137532,"internal_metadata":7225796292},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":3096691,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":315,"num_read_kb":1718,"num_write":75,"num_write_kb":768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001600"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":863,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.422700+0000","last_change":"2023-10-09T11:07:36.287455+0000","last_active":"2023-10-09T11:09:00.422700+0000","last_peered":"2023-10-09T11:09:00.422700+0000","last_clean":"2023-10-09T11:09:00.422700+0000","last_became_active":"2023-10-09T11:07:36.287052+0000","last_became_peered":"2023-10-09T11:07:36.287052+0000","last_unstale":"2023-10-09T11:09:00.422700+0000","last_undegraded":"2023-10-09T11:09:00.422700+0000","last_fullsized":"2023-10-09T11:09:00.422700+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:18.864873+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":358,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.420272+0000","last_change":"2023-10-09T11:03:21.185383+0000","last_active":"2023-10-09T11:09:00.420272+0000","last_peered":"2023-10-09T11:09:00.420272+0000","last_clean":"2023-10-09T11:09:00.420272+0000","last_became_active":"2023-10-09T11:03:21.185022+0000","last_became_peered":"2023-10-09T11:03:21.185022+0000","last_unstale":"2023-10-09T11:09:00.420272+0000","last_undegraded":"2023-10-09T11:09:00.420272+0000","last_fullsized":"2023-10-09T11:09:00.420272+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:05.230434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":102,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414567+0000","last_change":"2023-10-09T11:04:50.319477+0000","last_active":"2023-10-09T11:09:00.414567+0000","last_peered":"2023-10-09T11:09:00.414567+0000","last_clean":"2023-10-09T11:09:00.414567+0000","last_became_active":"2023-10-09T11:04:50.319148+0000","last_became_peered":"2023-10-09T11:04:50.319148+0000","last_unstale":"2023-10-09T11:09:00.414567+0000","last_undegraded":"2023-10-09T11:09:00.414567+0000","last_fullsized":"2023-10-09T11:09:00.414567+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:48.725566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":733,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.422813+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:09:00.422813+0000","last_peered":"2023-10-09T11:09:00.422813+0000","last_clean":"2023-10-09T11:09:00.422813+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:09:00.422813+0000","last_undegraded":"2023-10-09T11:09:00.422813+0000","last_fullsized":"2023-10-09T11:09:00.422813+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":290,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.424673+0000","last_change":"2023-10-09T11:04:50.333711+0000","last_active":"2023-10-09T11:09:00.424673+0000","last_peered":"2023-10-09T11:09:00.424673+0000","last_clean":"2023-10-09T11:09:00.424673+0000","last_became_active":"2023-10-09T11:04:50.333262+0000","last_became_peered":"2023-10-09T11:04:50.333262+0000","last_unstale":"2023-10-09T11:09:00.424673+0000","last_undegraded":"2023-10-09T11:09:00.424673+0000","last_fullsized":"2023-10-09T11:09:00.424673+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.843765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":36,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.543956+0000","last_change":"2023-10-09T11:07:33.237903+0000","last_active":"2023-10-09T11:09:00.543956+0000","last_peered":"2023-10-09T11:09:00.543956+0000","last_clean":"2023-10-09T11:09:00.543956+0000","last_became_active":"2023-10-09T11:07:33.237618+0000","last_became_peered":"2023-10-09T11:07:33.237618+0000","last_unstale":"2023-10-09T11:09:00.543956+0000","last_undegraded":"2023-10-09T11:09:00.543956+0000","last_fullsized":"2023-10-09T11:09:00.543956+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:25.600818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":751,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.543649+0000","last_change":"2023-10-09T11:07:36.284534+0000","last_active":"2023-10-09T11:09:00.543649+0000","last_peered":"2023-10-09T11:09:00.543649+0000","last_clean":"2023-10-09T11:09:00.543649+0000","last_became_active":"2023-10-09T11:07:36.284254+0000","last_became_peered":"2023-10-09T11:07:36.284254+0000","last_unstale":"2023-10-09T11:09:00.543649+0000","last_undegraded":"2023-10-09T11:09:00.543649+0000","last_fullsized":"2023-10-09T11:09:00.543649+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:19.159630+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":102,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414040+0000","last_change":"2023-10-09T11:04:50.310477+0000","last_active":"2023-10-09T11:09:00.414040+0000","last_peered":"2023-10-09T11:09:00.414040+0000","last_clean":"2023-10-09T11:09:00.414040+0000","last_became_active":"2023-10-09T11:04:50.310067+0000","last_became_peered":"2023-10-09T11:04:50.310067+0000","last_unstale":"2023-10-09T11:09:00.414040+0000","last_undegraded":"2023-10-09T11:09:00.414040+0000","last_fullsized":"2023-10-09T11:09:00.414040+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:52.884716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":385,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.423283+0000","last_change":"2023-10-09T11:05:20.310581+0000","last_active":"2023-10-09T11:09:00.423283+0000","last_peered":"2023-10-09T11:09:00.423283+0000","last_clean":"2023-10-09T11:09:00.423283+0000","last_became_active":"2023-10-09T11:05:20.309830+0000","last_became_peered":"2023-10-09T11:05:20.309830+0000","last_unstale":"2023-10-09T11:09:00.423283+0000","last_undegraded":"2023-10-09T11:09:00.423283+0000","last_fullsized":"2023-10-09T11:09:00.423283+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:24.508211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":791,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.543952+0000","last_change":"2023-10-09T11:07:36.282502+0000","last_active":"2023-10-09T11:09:00.543952+0000","last_peered":"2023-10-09T11:09:00.543952+0000","last_clean":"2023-10-09T11:09:00.543952+0000","last_became_active":"2023-10-09T11:07:36.282133+0000","last_became_peered":"2023-10-09T11:07:36.282133+0000","last_unstale":"2023-10-09T11:09:00.543952+0000","last_undegraded":"2023-10-09T11:09:00.543952+0000","last_fullsized":"2023-10-09T11:09:00.543952+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:03.762383+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":212,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.423352+0000","last_change":"2023-10-09T11:06:24.245488+0000","last_active":"2023-10-09T11:09:00.423352+0000","last_peered":"2023-10-09T11:09:00.423352+0000","last_clean":"2023-10-09T11:09:00.423352+0000","last_became_active":"2023-10-09T11:06:24.245167+0000","last_became_peered":"2023-10-09T11:06:24.245167+0000","last_unstale":"2023-10-09T11:09:00.423352+0000","last_undegraded":"2023-10-09T11:09:00.423352+0000","last_fullsized":"2023-10-09T11:09:00.423352+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:31.702568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":111,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414103+0000","last_change":"2023-10-09T11:05:20.299453+0000","last_active":"2023-10-09T11:09:00.414103+0000","last_peered":"2023-10-09T11:09:00.414103+0000","last_clean":"2023-10-09T11:09:00.414103+0000","last_became_active":"2023-10-09T11:05:20.299117+0000","last_became_peered":"2023-10-09T11:05:20.299117+0000","last_unstale":"2023-10-09T11:09:00.414103+0000","last_undegraded":"2023-10-09T11:09:00.414103+0000","last_fullsized":"2023-10-09T11:09:00.414103+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:30.072813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":905,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-10-09T11:08:53.114383+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:08:53.114383+0000","last_peered":"2023-10-09T11:08:53.114383+0000","last_clean":"2023-10-09T11:08:53.114383+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:08:53.114383+0000","last_undegraded":"2023-10-09T11:08:53.114383+0000","last_fullsized":"2023-10-09T11:08:53.114383+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":447,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.422983+0000","last_change":"2023-10-09T11:07:33.242665+0000","last_active":"2023-10-09T11:09:00.422983+0000","last_peered":"2023-10-09T11:09:00.422983+0000","last_clean":"2023-10-09T11:09:00.422983+0000","last_became_active":"2023-10-09T11:07:33.242313+0000","last_became_peered":"2023-10-09T11:07:33.242313+0000","last_unstale":"2023-10-09T11:09:00.422983+0000","last_undegraded":"2023-10-09T11:09:00.422983+0000","last_fullsized":"2023-10-09T11:09:00.422983+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:20.565135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":129,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414517+0000","last_change":"2023-10-09T11:07:33.225683+0000","last_active":"2023-10-09T11:09:00.414517+0000","last_peered":"2023-10-09T11:09:00.414517+0000","last_clean":"2023-10-09T11:09:00.414517+0000","last_became_active":"2023-10-09T11:07:33.225275+0000","last_became_peered":"2023-10-09T11:07:33.225275+0000","last_unstale":"2023-10-09T11:09:00.414517+0000","last_undegraded":"2023-10-09T11:09:00.414517+0000","last_fullsized":"2023-10-09T11:09:00.414517+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:10.131060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":805,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.422996+0000","last_change":"2023-10-09T11:05:23.328981+0000","last_active":"2023-10-09T11:09:00.422996+0000","last_peered":"2023-10-09T11:09:00.422996+0000","last_clean":"2023-10-09T11:09:00.422996+0000","last_became_active":"2023-10-09T11:05:23.328616+0000","last_became_peered":"2023-10-09T11:05:23.328616+0000","last_unstale":"2023-10-09T11:09:00.422996+0000","last_undegraded":"2023-10-09T11:09:00.422996+0000","last_fullsized":"2023-10-09T11:09:00.422996+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:15.315819+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":42,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.543844+0000","last_change":"2023-10-09T11:07:33.265842+0000","last_active":"2023-10-09T11:09:00.543844+0000","last_peered":"2023-10-09T11:09:00.543844+0000","last_clean":"2023-10-09T11:09:00.543844+0000","last_became_active":"2023-10-09T11:07:33.232311+0000","last_became_peered":"2023-10-09T11:07:33.232311+0000","last_unstale":"2023-10-09T11:09:00.543844+0000","last_undegraded":"2023-10-09T11:09:00.543844+0000","last_fullsized":"2023-10-09T11:09:00.543844+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:23.402582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":354,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.422677+0000","last_change":"2023-10-09T11:05:20.316265+0000","last_active":"2023-10-09T11:09:00.422677+0000","last_peered":"2023-10-09T11:09:00.422677+0000","last_clean":"2023-10-09T11:09:00.422677+0000","last_became_active":"2023-10-09T11:05:20.315853+0000","last_became_peered":"2023-10-09T11:05:20.315853+0000","last_unstale":"2023-10-09T11:09:00.422677+0000","last_undegraded":"2023-10-09T11:09:00.422677+0000","last_fullsized":"2023-10-09T11:09:00.422677+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:35.093981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":968,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.426552+0000","last_change":"2023-10-09T11:04:53.274921+0000","last_active":"2023-10-09T11:09:00.426552+0000","last_peered":"2023-10-09T11:09:00.426552+0000","last_clean":"2023-10-09T11:09:00.426552+0000","last_became_active":"2023-10-09T11:04:53.274518+0000","last_became_peered":"2023-10-09T11:04:53.274518+0000","last_unstale":"2023-10-09T11:09:00.426552+0000","last_undegraded":"2023-10-09T11:09:00.426552+0000","last_fullsized":"2023-10-09T11:09:00.426552+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:14.315301+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":46,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.543848+0000","last_change":"2023-10-09T11:07:33.262559+0000","last_active":"2023-10-09T11:09:00.543848+0000","last_peered":"2023-10-09T11:09:00.543848+0000","last_clean":"2023-10-09T11:09:00.543848+0000","last_became_active":"2023-10-09T11:07:33.231695+0000","last_became_peered":"2023-10-09T11:07:33.231695+0000","last_unstale":"2023-10-09T11:09:00.543848+0000","last_undegraded":"2023-10-09T11:09:00.543848+0000","last_fullsized":"2023-10-09T11:09:00.543848+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:59.006940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":596,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-10-09T11:08:53.114057+0000","last_change":"2023-10-09T11:05:20.311424+0000","last_active":"2023-10-09T11:08:53.114057+0000","last_peered":"2023-10-09T11:08:53.114057+0000","last_clean":"2023-10-09T11:08:53.114057+0000","last_became_active":"2023-10-09T11:05:20.311097+0000","last_became_peered":"2023-10-09T11:05:20.311097+0000","last_unstale":"2023-10-09T11:08:53.114057+0000","last_undegraded":"2023-10-09T11:08:53.114057+0000","last_fullsized":"2023-10-09T11:08:53.114057+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:19.274582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":618,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.422785+0000","last_change":"2023-10-09T11:08:39.934938+0000","last_active":"2023-10-09T11:09:00.422785+0000","last_peered":"2023-10-09T11:09:00.422785+0000","last_clean":"2023-10-09T11:09:00.422785+0000","last_became_active":"2023-10-09T11:06:24.245076+0000","last_became_peered":"2023-10-09T11:06:24.245076+0000","last_unstale":"2023-10-09T11:09:00.422785+0000","last_undegraded":"2023-10-09T11:09:00.422785+0000","last_fullsized":"2023-10-09T11:09:00.422785+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.934865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:00.408009+0000","scrub_duration":0.0012831839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":42,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.543824+0000","last_change":"2023-10-09T11:07:33.269200+0000","last_active":"2023-10-09T11:09:00.543824+0000","last_peered":"2023-10-09T11:09:00.543824+0000","last_clean":"2023-10-09T11:09:00.543824+0000","last_became_active":"2023-10-09T11:07:33.236205+0000","last_became_peered":"2023-10-09T11:07:33.236205+0000","last_unstale":"2023-10-09T11:09:00.543824+0000","last_undegraded":"2023-10-09T11:09:00.543824+0000","last_fullsized":"2023-10-09T11:09:00.543824+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:24.574734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":120,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414642+0000","last_change":"2023-10-09T11:07:33.227803+0000","last_active":"2023-10-09T11:09:00.414642+0000","last_peered":"2023-10-09T11:09:00.414642+0000","last_clean":"2023-10-09T11:09:00.414642+0000","last_became_active":"2023-10-09T11:07:33.227486+0000","last_became_peered":"2023-10-09T11:07:33.227486+0000","last_unstale":"2023-10-09T11:09:00.414642+0000","last_undegraded":"2023-10-09T11:09:00.414642+0000","last_fullsized":"2023-10-09T11:09:00.414642+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:39.274875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":887,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.422921+0000","last_change":"2023-10-09T11:05:23.323856+0000","last_active":"2023-10-09T11:09:00.422921+0000","last_peered":"2023-10-09T11:09:00.422921+0000","last_clean":"2023-10-09T11:09:00.422921+0000","last_became_active":"2023-10-09T11:05:23.322763+0000","last_became_peered":"2023-10-09T11:05:23.322763+0000","last_unstale":"2023-10-09T11:09:00.422921+0000","last_undegraded":"2023-10-09T11:09:00.422921+0000","last_fullsized":"2023-10-09T11:09:00.422921+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.760820+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":36,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.543543+0000","last_change":"2023-10-09T11:07:33.233510+0000","last_active":"2023-10-09T11:09:00.543543+0000","last_peered":"2023-10-09T11:09:00.543543+0000","last_clean":"2023-10-09T11:09:00.543543+0000","last_became_active":"2023-10-09T11:07:33.232339+0000","last_became_peered":"2023-10-09T11:07:33.232339+0000","last_unstale":"2023-10-09T11:09:00.543543+0000","last_undegraded":"2023-10-09T11:09:00.543543+0000","last_fullsized":"2023-10-09T11:09:00.543543+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:55.324912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":517,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.423195+0000","last_change":"2023-10-09T11:07:33.226594+0000","last_active":"2023-10-09T11:09:00.423195+0000","last_peered":"2023-10-09T11:09:00.423195+0000","last_clean":"2023-10-09T11:09:00.423195+0000","last_became_active":"2023-10-09T11:07:33.226275+0000","last_became_peered":"2023-10-09T11:07:33.226275+0000","last_unstale":"2023-10-09T11:09:00.423195+0000","last_undegraded":"2023-10-09T11:09:00.423195+0000","last_fullsized":"2023-10-09T11:09:00.423195+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:41.180088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":909,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.426007+0000","last_change":"2023-10-09T11:04:53.274236+0000","last_active":"2023-10-09T11:09:00.426007+0000","last_peered":"2023-10-09T11:09:00.426007+0000","last_clean":"2023-10-09T11:09:00.426007+0000","last_became_active":"2023-10-09T11:04:53.273817+0000","last_became_peered":"2023-10-09T11:04:53.273817+0000","last_unstale":"2023-10-09T11:09:00.426007+0000","last_undegraded":"2023-10-09T11:09:00.426007+0000","last_fullsized":"2023-10-09T11:09:00.426007+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:05.096905+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":171,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.543566+0000","last_change":"2023-10-09T11:05:20.312129+0000","last_active":"2023-10-09T11:09:00.543566+0000","last_peered":"2023-10-09T11:09:00.543566+0000","last_clean":"2023-10-09T11:09:00.543566+0000","last_became_active":"2023-10-09T11:05:20.311810+0000","last_became_peered":"2023-10-09T11:05:20.311810+0000","last_unstale":"2023-10-09T11:09:00.543566+0000","last_undegraded":"2023-10-09T11:09:00.543566+0000","last_fullsized":"2023-10-09T11:09:00.543566+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:48.813391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":270,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414807+0000","last_change":"2023-10-09T11:07:33.218244+0000","last_active":"2023-10-09T11:09:00.414807+0000","last_peered":"2023-10-09T11:09:00.414807+0000","last_clean":"2023-10-09T11:09:00.414807+0000","last_became_active":"2023-10-09T11:07:33.217918+0000","last_became_peered":"2023-10-09T11:07:33.217918+0000","last_unstale":"2023-10-09T11:09:00.414807+0000","last_undegraded":"2023-10-09T11:09:00.414807+0000","last_fullsized":"2023-10-09T11:09:00.414807+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:16.150105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":884,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.423078+0000","last_change":"2023-10-09T11:05:23.324537+0000","last_active":"2023-10-09T11:09:00.423078+0000","last_peered":"2023-10-09T11:09:00.423078+0000","last_clean":"2023-10-09T11:09:00.423078+0000","last_became_active":"2023-10-09T11:05:23.324187+0000","last_became_peered":"2023-10-09T11:05:23.324187+0000","last_unstale":"2023-10-09T11:09:00.423078+0000","last_undegraded":"2023-10-09T11:09:00.423078+0000","last_fullsized":"2023-10-09T11:09:00.423078+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:25.286106+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":95,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-10-09T11:08:53.397006+0000","last_change":"2023-10-09T11:05:20.323565+0000","last_active":"2023-10-09T11:08:53.397006+0000","last_peered":"2023-10-09T11:08:53.397006+0000","last_clean":"2023-10-09T11:08:53.397006+0000","last_became_active":"2023-10-09T11:05:20.309672+0000","last_became_peered":"2023-10-09T11:05:20.309672+0000","last_unstale":"2023-10-09T11:08:53.397006+0000","last_undegraded":"2023-10-09T11:08:53.397006+0000","last_fullsized":"2023-10-09T11:08:53.397006+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:50.211465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":120,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414880+0000","last_change":"2023-10-09T11:07:33.228001+0000","last_active":"2023-10-09T11:09:00.414880+0000","last_peered":"2023-10-09T11:09:00.414880+0000","last_clean":"2023-10-09T11:09:00.414880+0000","last_became_active":"2023-10-09T11:07:33.227691+0000","last_became_peered":"2023-10-09T11:07:33.227691+0000","last_unstale":"2023-10-09T11:09:00.414880+0000","last_undegraded":"2023-10-09T11:09:00.414880+0000","last_fullsized":"2023-10-09T11:09:00.414880+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:24:16.098138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":808,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.543968+0000","last_change":"2023-10-09T11:07:36.284927+0000","last_active":"2023-10-09T11:09:00.543968+0000","last_peered":"2023-10-09T11:09:00.543968+0000","last_clean":"2023-10-09T11:09:00.543968+0000","last_became_active":"2023-10-09T11:07:36.284631+0000","last_became_peered":"2023-10-09T11:07:36.284631+0000","last_unstale":"2023-10-09T11:09:00.543968+0000","last_undegraded":"2023-10-09T11:09:00.543968+0000","last_fullsized":"2023-10-09T11:09:00.543968+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:37.220243+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":40,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.544036+0000","last_change":"2023-10-09T11:07:33.273503+0000","last_active":"2023-10-09T11:09:00.544036+0000","last_peered":"2023-10-09T11:09:00.544036+0000","last_clean":"2023-10-09T11:09:00.544036+0000","last_became_active":"2023-10-09T11:07:33.242980+0000","last_became_peered":"2023-10-09T11:07:33.242980+0000","last_unstale":"2023-10-09T11:09:00.544036+0000","last_undegraded":"2023-10-09T11:09:00.544036+0000","last_fullsized":"2023-10-09T11:09:00.544036+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:18.934972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":538,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.422701+0000","last_change":"2023-10-09T11:07:33.304363+0000","last_active":"2023-10-09T11:09:00.422701+0000","last_peered":"2023-10-09T11:09:00.422701+0000","last_clean":"2023-10-09T11:09:00.422701+0000","last_became_active":"2023-10-09T11:07:33.304024+0000","last_became_peered":"2023-10-09T11:07:33.304024+0000","last_unstale":"2023-10-09T11:09:00.422701+0000","last_undegraded":"2023-10-09T11:09:00.422701+0000","last_fullsized":"2023-10-09T11:09:00.422701+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:21:51.522688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":957,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414734+0000","last_change":"2023-10-09T11:05:23.316129+0000","last_active":"2023-10-09T11:09:00.414734+0000","last_peered":"2023-10-09T11:09:00.414734+0000","last_clean":"2023-10-09T11:09:00.414734+0000","last_became_active":"2023-10-09T11:05:23.315792+0000","last_became_peered":"2023-10-09T11:05:23.315792+0000","last_unstale":"2023-10-09T11:09:00.414734+0000","last_undegraded":"2023-10-09T11:09:00.414734+0000","last_fullsized":"2023-10-09T11:09:00.414734+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:51.770547+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":40,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.544027+0000","last_change":"2023-10-09T11:07:33.256691+0000","last_active":"2023-10-09T11:09:00.544027+0000","last_peered":"2023-10-09T11:09:00.544027+0000","last_clean":"2023-10-09T11:09:00.544027+0000","last_became_active":"2023-10-09T11:07:33.229554+0000","last_became_peered":"2023-10-09T11:07:33.229554+0000","last_unstale":"2023-10-09T11:09:00.544027+0000","last_undegraded":"2023-10-09T11:09:00.544027+0000","last_fullsized":"2023-10-09T11:09:00.544027+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:57.414795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":291,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.424592+0000","last_change":"2023-10-09T11:04:50.333538+0000","last_active":"2023-10-09T11:09:00.424592+0000","last_peered":"2023-10-09T11:09:00.424592+0000","last_clean":"2023-10-09T11:09:00.424592+0000","last_became_active":"2023-10-09T11:04:50.332514+0000","last_became_peered":"2023-10-09T11:04:50.332514+0000","last_unstale":"2023-10-09T11:09:00.424592+0000","last_undegraded":"2023-10-09T11:09:00.424592+0000","last_fullsized":"2023-10-09T11:09:00.424592+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:22.933839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":832,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.423195+0000","last_change":"2023-10-09T11:05:23.322251+0000","last_active":"2023-10-09T11:09:00.423195+0000","last_peered":"2023-10-09T11:09:00.423195+0000","last_clean":"2023-10-09T11:09:00.423195+0000","last_became_active":"2023-10-09T11:05:23.321826+0000","last_became_peered":"2023-10-09T11:05:23.321826+0000","last_unstale":"2023-10-09T11:09:00.423195+0000","last_undegraded":"2023-10-09T11:09:00.423195+0000","last_fullsized":"2023-10-09T11:09:00.423195+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.858013+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":40,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.544154+0000","last_change":"2023-10-09T11:07:33.271238+0000","last_active":"2023-10-09T11:09:00.544154+0000","last_peered":"2023-10-09T11:09:00.544154+0000","last_clean":"2023-10-09T11:09:00.544154+0000","last_became_active":"2023-10-09T11:07:33.241886+0000","last_became_peered":"2023-10-09T11:07:33.241886+0000","last_unstale":"2023-10-09T11:09:00.544154+0000","last_undegraded":"2023-10-09T11:09:00.544154+0000","last_fullsized":"2023-10-09T11:09:00.544154+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:48.882589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":105,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-10-09T11:08:53.397111+0000","last_change":"2023-10-09T11:07:33.244131+0000","last_active":"2023-10-09T11:08:53.397111+0000","last_peered":"2023-10-09T11:08:53.397111+0000","last_clean":"2023-10-09T11:08:53.397111+0000","last_became_active":"2023-10-09T11:07:33.243797+0000","last_became_peered":"2023-10-09T11:07:33.243797+0000","last_unstale":"2023-10-09T11:08:53.397111+0000","last_undegraded":"2023-10-09T11:08:53.397111+0000","last_fullsized":"2023-10-09T11:08:53.397111+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:44.851043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":1086,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.544074+0000","last_change":"2023-10-09T11:07:36.286417+0000","last_active":"2023-10-09T11:09:00.544074+0000","last_peered":"2023-10-09T11:09:00.544074+0000","last_clean":"2023-10-09T11:09:00.544074+0000","last_became_active":"2023-10-09T11:07:36.286061+0000","last_became_peered":"2023-10-09T11:07:36.286061+0000","last_unstale":"2023-10-09T11:09:00.544074+0000","last_undegraded":"2023-10-09T11:09:00.544074+0000","last_fullsized":"2023-10-09T11:09:00.544074+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:00.289500+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":434,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.423032+0000","last_change":"2023-10-09T11:05:20.309385+0000","last_active":"2023-10-09T11:09:00.423032+0000","last_peered":"2023-10-09T11:09:00.423032+0000","last_clean":"2023-10-09T11:09:00.423032+0000","last_became_active":"2023-10-09T11:05:20.309034+0000","last_became_peered":"2023-10-09T11:05:20.309034+0000","last_unstale":"2023-10-09T11:09:00.423032+0000","last_undegraded":"2023-10-09T11:09:00.423032+0000","last_fullsized":"2023-10-09T11:09:00.423032+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:31.857485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":366,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.425927+0000","last_change":"2023-10-09T11:06:24.245966+0000","last_active":"2023-10-09T11:09:00.425927+0000","last_peered":"2023-10-09T11:09:00.425927+0000","last_clean":"2023-10-09T11:09:00.425927+0000","last_became_active":"2023-10-09T11:06:24.245310+0000","last_became_peered":"2023-10-09T11:06:24.245310+0000","last_unstale":"2023-10-09T11:09:00.425927+0000","last_undegraded":"2023-10-09T11:09:00.425927+0000","last_fullsized":"2023-10-09T11:09:00.425927+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:23.966200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":730,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.543456+0000","last_change":"2023-10-09T11:07:36.283174+0000","last_active":"2023-10-09T11:09:00.543456+0000","last_peered":"2023-10-09T11:09:00.543456+0000","last_clean":"2023-10-09T11:09:00.543456+0000","last_became_active":"2023-10-09T11:07:36.278007+0000","last_became_peered":"2023-10-09T11:07:36.278007+0000","last_unstale":"2023-10-09T11:09:00.543456+0000","last_undegraded":"2023-10-09T11:09:00.543456+0000","last_fullsized":"2023-10-09T11:09:00.543456+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:51.693969+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":106,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414381+0000","last_change":"2023-10-09T11:04:50.529513+0000","last_active":"2023-10-09T11:09:00.414381+0000","last_peered":"2023-10-09T11:09:00.414381+0000","last_clean":"2023-10-09T11:09:00.414381+0000","last_became_active":"2023-10-09T11:04:50.339671+0000","last_became_peered":"2023-10-09T11:04:50.339671+0000","last_unstale":"2023-10-09T11:09:00.414381+0000","last_undegraded":"2023-10-09T11:09:00.414381+0000","last_fullsized":"2023-10-09T11:09:00.414381+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:24.473000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":111,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414567+0000","last_change":"2023-10-09T11:05:20.313855+0000","last_active":"2023-10-09T11:09:00.414567+0000","last_peered":"2023-10-09T11:09:00.414567+0000","last_clean":"2023-10-09T11:09:00.414567+0000","last_became_active":"2023-10-09T11:05:20.313535+0000","last_became_peered":"2023-10-09T11:05:20.313535+0000","last_unstale":"2023-10-09T11:09:00.414567+0000","last_undegraded":"2023-10-09T11:09:00.414567+0000","last_fullsized":"2023-10-09T11:09:00.414567+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:25.266434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":834,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.422854+0000","last_change":"2023-10-09T11:04:53.272323+0000","last_active":"2023-10-09T11:09:00.422854+0000","last_peered":"2023-10-09T11:09:00.422854+0000","last_clean":"2023-10-09T11:09:00.422854+0000","last_became_active":"2023-10-09T11:04:53.272010+0000","last_became_peered":"2023-10-09T11:04:53.272010+0000","last_unstale":"2023-10-09T11:09:00.422854+0000","last_undegraded":"2023-10-09T11:09:00.422854+0000","last_fullsized":"2023-10-09T11:09:00.422854+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:23.913605+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":599,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.424944+0000","last_change":"2023-10-09T11:05:20.307499+0000","last_active":"2023-10-09T11:09:00.424944+0000","last_peered":"2023-10-09T11:09:00.424944+0000","last_clean":"2023-10-09T11:09:00.424944+0000","last_became_active":"2023-10-09T11:05:20.307198+0000","last_became_peered":"2023-10-09T11:05:20.307198+0000","last_unstale":"2023-10-09T11:09:00.424944+0000","last_undegraded":"2023-10-09T11:09:00.424944+0000","last_fullsized":"2023-10-09T11:09:00.424944+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:53.726844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":334,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-10-09T11:08:53.118308+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:08:53.118308+0000","last_peered":"2023-10-09T11:08:53.118308+0000","last_clean":"2023-10-09T11:08:53.118308+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:08:53.118308+0000","last_undegraded":"2023-10-09T11:08:53.118308+0000","last_fullsized":"2023-10-09T11:08:53.118308+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":926,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.426090+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:09:00.426090+0000","last_peered":"2023-10-09T11:09:00.426090+0000","last_clean":"2023-10-09T11:09:00.426090+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:09:00.426090+0000","last_undegraded":"2023-10-09T11:09:00.426090+0000","last_fullsized":"2023-10-09T11:09:00.426090+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":597,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-10-09T11:08:53.112222+0000","last_change":"2023-10-09T11:04:57.953307+0000","last_active":"2023-10-09T11:08:53.112222+0000","last_peered":"2023-10-09T11:08:53.112222+0000","last_clean":"2023-10-09T11:08:53.112222+0000","last_became_active":"2023-10-09T11:04:57.953095+0000","last_became_peered":"2023-10-09T11:04:57.953095+0000","last_unstale":"2023-10-09T11:08:53.112222+0000","last_undegraded":"2023-10-09T11:08:53.112222+0000","last_fullsized":"2023-10-09T11:08:53.112222+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:05.562643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":284,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.426455+0000","last_change":"2023-10-09T11:05:20.310255+0000","last_active":"2023-10-09T11:09:00.426455+0000","last_peered":"2023-10-09T11:09:00.426455+0000","last_clean":"2023-10-09T11:09:00.426455+0000","last_became_active":"2023-10-09T11:05:20.309109+0000","last_became_peered":"2023-10-09T11:05:20.309109+0000","last_unstale":"2023-10-09T11:09:00.426455+0000","last_undegraded":"2023-10-09T11:09:00.426455+0000","last_fullsized":"2023-10-09T11:09:00.426455+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:07.429039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":818,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.422963+0000","last_change":"2023-10-09T11:05:23.329329+0000","last_active":"2023-10-09T11:09:00.422963+0000","last_peered":"2023-10-09T11:09:00.422963+0000","last_clean":"2023-10-09T11:09:00.422963+0000","last_became_active":"2023-10-09T11:05:23.328976+0000","last_became_peered":"2023-10-09T11:05:23.328976+0000","last_unstale":"2023-10-09T11:09:00.422963+0000","last_undegraded":"2023-10-09T11:09:00.422963+0000","last_fullsized":"2023-10-09T11:09:00.422963+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:41.848184+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":120,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414192+0000","last_change":"2023-10-09T11:07:33.236133+0000","last_active":"2023-10-09T11:09:00.414192+0000","last_peered":"2023-10-09T11:09:00.414192+0000","last_clean":"2023-10-09T11:09:00.414192+0000","last_became_active":"2023-10-09T11:07:33.235818+0000","last_became_peered":"2023-10-09T11:07:33.235818+0000","last_unstale":"2023-10-09T11:09:00.414192+0000","last_undegraded":"2023-10-09T11:09:00.414192+0000","last_fullsized":"2023-10-09T11:09:00.414192+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:20.627846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":281,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.426491+0000","last_change":"2023-10-09T11:04:50.332978+0000","last_active":"2023-10-09T11:09:00.426491+0000","last_peered":"2023-10-09T11:09:00.426491+0000","last_clean":"2023-10-09T11:09:00.426491+0000","last_became_active":"2023-10-09T11:04:50.332735+0000","last_became_peered":"2023-10-09T11:04:50.332735+0000","last_unstale":"2023-10-09T11:09:00.426491+0000","last_undegraded":"2023-10-09T11:09:00.426491+0000","last_fullsized":"2023-10-09T11:09:00.426491+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:39:44.659751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":1102,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414953+0000","last_change":"2023-10-09T11:07:36.291716+0000","last_active":"2023-10-09T11:09:00.414953+0000","last_peered":"2023-10-09T11:09:00.414953+0000","last_clean":"2023-10-09T11:09:00.414953+0000","last_became_active":"2023-10-09T11:07:36.291424+0000","last_became_peered":"2023-10-09T11:07:36.291424+0000","last_unstale":"2023-10-09T11:09:00.414953+0000","last_undegraded":"2023-10-09T11:09:00.414953+0000","last_fullsized":"2023-10-09T11:09:00.414953+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:22.316387+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"576'4","reported_seq":318,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-10-09T11:09:03.116510+0000","last_change":"2023-10-09T11:09:01.430580+0000","last_active":"2023-10-09T11:09:03.116510+0000","last_peered":"2023-10-09T11:09:03.116510+0000","last_clean":"2023-10-09T11:09:03.116510+0000","last_became_active":"2023-10-09T11:07:33.226176+0000","last_became_peered":"2023-10-09T11:07:33.226176+0000","last_unstale":"2023-10-09T11:09:03.116510+0000","last_undegraded":"2023-10-09T11:09:03.116510+0000","last_fullsized":"2023-10-09T11:09:03.116510+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:16.888732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001104589,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":3,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"3.5","version":"0'0","reported_seq":451,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.423100+0000","last_change":"2023-10-09T11:04:50.341353+0000","last_active":"2023-10-09T11:09:00.423100+0000","last_peered":"2023-10-09T11:09:00.423100+0000","last_clean":"2023-10-09T11:09:00.423100+0000","last_became_active":"2023-10-09T11:04:50.340801+0000","last_became_peered":"2023-10-09T11:04:50.340801+0000","last_unstale":"2023-10-09T11:09:00.423100+0000","last_undegraded":"2023-10-09T11:09:00.423100+0000","last_fullsized":"2023-10-09T11:09:00.423100+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:00.699069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":258,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.425081+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:09:00.425081+0000","last_peered":"2023-10-09T11:09:00.425081+0000","last_clean":"2023-10-09T11:09:00.425081+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:09:00.425081+0000","last_undegraded":"2023-10-09T11:09:00.425081+0000","last_fullsized":"2023-10-09T11:09:00.425081+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":776,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.422707+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:09:00.422707+0000","last_peered":"2023-10-09T11:09:00.422707+0000","last_clean":"2023-10-09T11:09:00.422707+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:09:00.422707+0000","last_undegraded":"2023-10-09T11:09:00.422707+0000","last_fullsized":"2023-10-09T11:09:00.422707+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"576'20","reported_seq":505,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-10-09T11:09:05.134261+0000","last_change":"2023-10-09T11:09:01.427844+0000","last_active":"2023-10-09T11:09:05.134261+0000","last_peered":"2023-10-09T11:09:05.134261+0000","last_clean":"2023-10-09T11:09:05.134261+0000","last_became_active":"2023-10-09T11:05:20.315471+0000","last_became_peered":"2023-10-09T11:05:20.315471+0000","last_unstale":"2023-10-09T11:09:05.134261+0000","last_undegraded":"2023-10-09T11:09:05.134261+0000","last_fullsized":"2023-10-09T11:09:05.134261+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":20,"log_dups_size":0,"ondisk_log_size":20,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:47.548763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085674299999999998,"stat_sum":{"num_bytes":115,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":78,"num_read_kb":448,"num_write":24,"num_write_kb":214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"3.3","version":"0'0","reported_seq":481,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-10-09T11:08:53.397989+0000","last_change":"2023-10-09T11:06:24.245282+0000","last_active":"2023-10-09T11:08:53.397989+0000","last_peered":"2023-10-09T11:08:53.397989+0000","last_clean":"2023-10-09T11:08:53.397989+0000","last_became_active":"2023-10-09T11:06:24.244981+0000","last_became_peered":"2023-10-09T11:06:24.244981+0000","last_unstale":"2023-10-09T11:08:53.397989+0000","last_undegraded":"2023-10-09T11:08:53.397989+0000","last_fullsized":"2023-10-09T11:08:53.397989+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:15.327655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":291,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.425285+0000","last_change":"2023-10-09T11:04:50.311579+0000","last_active":"2023-10-09T11:09:00.425285+0000","last_peered":"2023-10-09T11:09:00.425285+0000","last_clean":"2023-10-09T11:09:00.425285+0000","last_became_active":"2023-10-09T11:04:50.311270+0000","last_became_peered":"2023-10-09T11:04:50.311270+0000","last_unstale":"2023-10-09T11:09:00.425285+0000","last_undegraded":"2023-10-09T11:09:00.425285+0000","last_fullsized":"2023-10-09T11:09:00.425285+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:49.850302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":928,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.543729+0000","last_change":"2023-10-09T11:07:36.284280+0000","last_active":"2023-10-09T11:09:00.543729+0000","last_peered":"2023-10-09T11:09:00.543729+0000","last_clean":"2023-10-09T11:09:00.543729+0000","last_became_active":"2023-10-09T11:07:36.283966+0000","last_became_peered":"2023-10-09T11:07:36.283966+0000","last_unstale":"2023-10-09T11:09:00.543729+0000","last_undegraded":"2023-10-09T11:09:00.543729+0000","last_fullsized":"2023-10-09T11:09:00.543729+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:54.128331+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"574'1","reported_seq":355,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-10-09T11:08:59.404357+0000","last_change":"2023-10-09T11:04:50.381744+0000","last_active":"2023-10-09T11:08:59.404357+0000","last_peered":"2023-10-09T11:08:59.404357+0000","last_clean":"2023-10-09T11:08:59.404357+0000","last_became_active":"2023-10-09T11:04:50.381253+0000","last_became_peered":"2023-10-09T11:04:50.381253+0000","last_unstale":"2023-10-09T11:08:59.404357+0000","last_undegraded":"2023-10-09T11:08:59.404357+0000","last_fullsized":"2023-10-09T11:08:59.404357+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:24.030274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":276,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.424457+0000","last_change":"2023-10-09T11:05:20.298542+0000","last_active":"2023-10-09T11:09:00.424457+0000","last_peered":"2023-10-09T11:09:00.424457+0000","last_clean":"2023-10-09T11:09:00.424457+0000","last_became_active":"2023-10-09T11:05:20.298119+0000","last_became_peered":"2023-10-09T11:05:20.298119+0000","last_unstale":"2023-10-09T11:09:00.424457+0000","last_undegraded":"2023-10-09T11:09:00.424457+0000","last_fullsized":"2023-10-09T11:09:00.424457+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:00.827214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":256,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.426669+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:09:00.426669+0000","last_peered":"2023-10-09T11:09:00.426669+0000","last_clean":"2023-10-09T11:09:00.426669+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:09:00.426669+0000","last_undegraded":"2023-10-09T11:09:00.426669+0000","last_fullsized":"2023-10-09T11:09:00.426669+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"576'33","reported_seq":570,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-10-09T11:09:05.135564+0000","last_change":"2023-10-09T11:09:01.427903+0000","last_active":"2023-10-09T11:09:05.135564+0000","last_peered":"2023-10-09T11:09:05.135564+0000","last_clean":"2023-10-09T11:09:05.135564+0000","last_became_active":"2023-10-09T11:07:33.226289+0000","last_became_peered":"2023-10-09T11:07:33.226289+0000","last_unstale":"2023-10-09T11:09:05.135564+0000","last_undegraded":"2023-10-09T11:09:05.135564+0000","last_fullsized":"2023-10-09T11:09:05.135564+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":33,"log_dups_size":0,"ondisk_log_size":33,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:15.605946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083326699999999995,"stat_sum":{"num_bytes":1097728,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":173,"num_read_kb":646,"num_write":23,"num_write_kb":252,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"76'161","reported_seq":873,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.543487+0000","last_change":"2023-10-09T11:07:36.290700+0000","last_active":"2023-10-09T11:09:00.543487+0000","last_peered":"2023-10-09T11:09:00.543487+0000","last_clean":"2023-10-09T11:09:00.543487+0000","last_became_active":"2023-10-09T11:07:36.290357+0000","last_became_peered":"2023-10-09T11:07:36.290357+0000","last_unstale":"2023-10-09T11:09:00.543487+0000","last_undegraded":"2023-10-09T11:09:00.543487+0000","last_fullsized":"2023-10-09T11:09:00.543487+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:34.494777+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":112,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-10-09T11:09:01.421686+0000","last_change":"2023-10-09T11:05:20.310666+0000","last_active":"2023-10-09T11:09:01.421686+0000","last_peered":"2023-10-09T11:09:01.421686+0000","last_clean":"2023-10-09T11:09:01.421686+0000","last_became_active":"2023-10-09T11:05:20.310413+0000","last_became_peered":"2023-10-09T11:05:20.310413+0000","last_unstale":"2023-10-09T11:09:01.421686+0000","last_undegraded":"2023-10-09T11:09:01.421686+0000","last_fullsized":"2023-10-09T11:09:01.421686+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:34.169905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":536,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.544087+0000","last_change":"2023-10-09T11:05:20.316523+0000","last_active":"2023-10-09T11:09:00.544087+0000","last_peered":"2023-10-09T11:09:00.544087+0000","last_clean":"2023-10-09T11:09:00.544087+0000","last_became_active":"2023-10-09T11:05:20.314493+0000","last_became_peered":"2023-10-09T11:05:20.314493+0000","last_unstale":"2023-10-09T11:09:00.544087+0000","last_undegraded":"2023-10-09T11:09:00.544087+0000","last_fullsized":"2023-10-09T11:09:00.544087+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:41.618701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":947,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.425844+0000","last_change":"2023-10-09T11:05:23.322074+0000","last_active":"2023-10-09T11:09:00.425844+0000","last_peered":"2023-10-09T11:09:00.425844+0000","last_clean":"2023-10-09T11:09:00.425844+0000","last_became_active":"2023-10-09T11:05:23.321749+0000","last_became_peered":"2023-10-09T11:05:23.321749+0000","last_unstale":"2023-10-09T11:09:00.425844+0000","last_undegraded":"2023-10-09T11:09:00.425844+0000","last_fullsized":"2023-10-09T11:09:00.425844+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:07.010932+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"576'3","reported_seq":160,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-10-09T11:09:02.339964+0000","last_change":"2023-10-09T11:09:01.431862+0000","last_active":"2023-10-09T11:09:02.339964+0000","last_peered":"2023-10-09T11:09:02.339964+0000","last_clean":"2023-10-09T11:09:02.339964+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:09:02.339964+0000","last_undegraded":"2023-10-09T11:09:02.339964+0000","last_fullsized":"2023-10-09T11:09:02.339964+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027964599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2,"num_read_kb":288,"num_write":2,"num_write_kb":144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"3.1","version":"0'0","reported_seq":203,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414307+0000","last_change":"2023-10-09T11:06:24.248362+0000","last_active":"2023-10-09T11:09:00.414307+0000","last_peered":"2023-10-09T11:09:00.414307+0000","last_clean":"2023-10-09T11:09:00.414307+0000","last_became_active":"2023-10-09T11:06:24.247348+0000","last_became_peered":"2023-10-09T11:06:24.247348+0000","last_unstale":"2023-10-09T11:09:00.414307+0000","last_undegraded":"2023-10-09T11:09:00.414307+0000","last_fullsized":"2023-10-09T11:09:00.414307+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:26.728754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":402,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.426314+0000","last_change":"2023-10-09T11:06:24.253511+0000","last_active":"2023-10-09T11:09:00.426314+0000","last_peered":"2023-10-09T11:09:00.426314+0000","last_clean":"2023-10-09T11:09:00.426314+0000","last_became_active":"2023-10-09T11:06:24.252739+0000","last_became_peered":"2023-10-09T11:06:24.252739+0000","last_unstale":"2023-10-09T11:09:00.426314+0000","last_undegraded":"2023-10-09T11:09:00.426314+0000","last_fullsized":"2023-10-09T11:09:00.426314+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:50.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":807,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414625+0000","last_change":"2023-10-09T11:06:24.239676+0000","last_active":"2023-10-09T11:09:00.414625+0000","last_peered":"2023-10-09T11:09:00.414625+0000","last_clean":"2023-10-09T11:09:00.414625+0000","last_became_active":"2023-10-09T11:06:24.239375+0000","last_became_peered":"2023-10-09T11:06:24.239375+0000","last_unstale":"2023-10-09T11:09:00.414625+0000","last_undegraded":"2023-10-09T11:09:00.414625+0000","last_fullsized":"2023-10-09T11:09:00.414625+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:27.953053+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":40,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-10-09T11:09:01.432128+0000","last_change":"2023-10-09T11:09:01.432451+0000","last_active":"2023-10-09T11:09:01.432128+0000","last_peered":"2023-10-09T11:09:01.432128+0000","last_clean":"2023-10-09T11:09:01.432128+0000","last_became_active":"2023-10-09T11:07:33.229061+0000","last_became_peered":"2023-10-09T11:07:33.229061+0000","last_unstale":"2023-10-09T11:09:01.432128+0000","last_undegraded":"2023-10-09T11:09:01.432128+0000","last_fullsized":"2023-10-09T11:09:01.432128+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:30.978293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039999499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"3.4","version":"0'0","reported_seq":258,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.425733+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:09:00.425733+0000","last_peered":"2023-10-09T11:09:00.425733+0000","last_clean":"2023-10-09T11:09:00.425733+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:09:00.425733+0000","last_undegraded":"2023-10-09T11:09:00.425733+0000","last_fullsized":"2023-10-09T11:09:00.425733+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":535,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.422901+0000","last_change":"2023-10-09T11:07:33.227271+0000","last_active":"2023-10-09T11:09:00.422901+0000","last_peered":"2023-10-09T11:09:00.422901+0000","last_clean":"2023-10-09T11:09:00.422901+0000","last_became_active":"2023-10-09T11:07:33.226760+0000","last_became_peered":"2023-10-09T11:07:33.226760+0000","last_unstale":"2023-10-09T11:09:00.422901+0000","last_undegraded":"2023-10-09T11:09:00.422901+0000","last_fullsized":"2023-10-09T11:09:00.422901+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:12.268123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":774,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-10-09T11:08:53.118458+0000","last_change":"2023-10-09T11:05:23.323368+0000","last_active":"2023-10-09T11:08:53.118458+0000","last_peered":"2023-10-09T11:08:53.118458+0000","last_clean":"2023-10-09T11:08:53.118458+0000","last_became_active":"2023-10-09T11:05:23.323032+0000","last_became_peered":"2023-10-09T11:05:23.323032+0000","last_unstale":"2023-10-09T11:08:53.118458+0000","last_undegraded":"2023-10-09T11:08:53.118458+0000","last_fullsized":"2023-10-09T11:08:53.118458+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:09.393584+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":132,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414806+0000","last_change":"2023-10-09T11:07:33.246378+0000","last_active":"2023-10-09T11:09:00.414806+0000","last_peered":"2023-10-09T11:09:00.414806+0000","last_clean":"2023-10-09T11:09:00.414806+0000","last_became_active":"2023-10-09T11:07:33.228019+0000","last_became_peered":"2023-10-09T11:07:33.228019+0000","last_unstale":"2023-10-09T11:09:00.414806+0000","last_undegraded":"2023-10-09T11:09:00.414806+0000","last_fullsized":"2023-10-09T11:09:00.414806+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:18.511517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"576'2","reported_seq":291,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-10-09T11:09:01.429977+0000","last_change":"2023-10-09T11:09:01.429977+0000","last_active":"2023-10-09T11:09:01.429977+0000","last_peered":"2023-10-09T11:09:01.429977+0000","last_clean":"2023-10-09T11:09:01.429977+0000","last_became_active":"2023-10-09T11:05:20.309827+0000","last_became_peered":"2023-10-09T11:05:20.309827+0000","last_unstale":"2023-10-09T11:09:01.429977+0000","last_undegraded":"2023-10-09T11:09:01.429977+0000","last_fullsized":"2023-10-09T11:09:01.429977+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:23.934383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00020781599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"4.0","version":"0'0","reported_seq":121,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414135+0000","last_change":"2023-10-09T11:07:33.217595+0000","last_active":"2023-10-09T11:09:00.414135+0000","last_peered":"2023-10-09T11:09:00.414135+0000","last_clean":"2023-10-09T11:09:00.414135+0000","last_became_active":"2023-10-09T11:07:33.217155+0000","last_became_peered":"2023-10-09T11:07:33.217155+0000","last_unstale":"2023-10-09T11:09:00.414135+0000","last_undegraded":"2023-10-09T11:09:00.414135+0000","last_fullsized":"2023-10-09T11:09:00.414135+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:55.127329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":740,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-10-09T11:09:01.422829+0000","last_change":"2023-10-09T11:07:36.284491+0000","last_active":"2023-10-09T11:09:01.422829+0000","last_peered":"2023-10-09T11:09:01.422829+0000","last_clean":"2023-10-09T11:09:01.422829+0000","last_became_active":"2023-10-09T11:07:36.284249+0000","last_became_peered":"2023-10-09T11:07:36.284249+0000","last_unstale":"2023-10-09T11:09:01.422829+0000","last_undegraded":"2023-10-09T11:09:01.422829+0000","last_fullsized":"2023-10-09T11:09:01.422829+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:03.653371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:03.653371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:09.629417+0000","scrub_duration":0.001576771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"576'17","reported_seq":63,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-10-09T11:09:05.136328+0000","last_change":"2023-10-09T11:09:01.432169+0000","last_active":"2023-10-09T11:09:05.136328+0000","last_peered":"2023-10-09T11:09:05.136328+0000","last_clean":"2023-10-09T11:09:05.136328+0000","last_became_active":"2023-10-09T11:07:33.241013+0000","last_became_peered":"2023-10-09T11:07:33.241013+0000","last_unstale":"2023-10-09T11:09:05.136328+0000","last_undegraded":"2023-10-09T11:09:05.136328+0000","last_fullsized":"2023-10-09T11:09:05.136328+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:57.164502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0002341,"stat_sum":{"num_bytes":1998848,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":55,"num_read_kb":329,"num_write":23,"num_write_kb":157,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"3.6","version":"64'1","reported_seq":275,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.426401+0000","last_change":"2023-10-09T11:06:24.253714+0000","last_active":"2023-10-09T11:09:00.426401+0000","last_peered":"2023-10-09T11:09:00.426401+0000","last_clean":"2023-10-09T11:09:00.426401+0000","last_became_active":"2023-10-09T11:06:24.253143+0000","last_became_peered":"2023-10-09T11:06:24.253143+0000","last_unstale":"2023-10-09T11:09:00.426401+0000","last_undegraded":"2023-10-09T11:09:00.426401+0000","last_fullsized":"2023-10-09T11:09:00.426401+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:05.682288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":381,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.422677+0000","last_change":"2023-10-09T11:05:20.313560+0000","last_active":"2023-10-09T11:09:00.422677+0000","last_peered":"2023-10-09T11:09:00.422677+0000","last_clean":"2023-10-09T11:09:00.422677+0000","last_became_active":"2023-10-09T11:05:20.313214+0000","last_became_peered":"2023-10-09T11:05:20.313214+0000","last_unstale":"2023-10-09T11:09:00.422677+0000","last_undegraded":"2023-10-09T11:09:00.422677+0000","last_fullsized":"2023-10-09T11:09:00.422677+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:32.527205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":983,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414600+0000","last_change":"2023-10-09T11:05:23.329886+0000","last_active":"2023-10-09T11:09:00.414600+0000","last_peered":"2023-10-09T11:09:00.414600+0000","last_clean":"2023-10-09T11:09:00.414600+0000","last_became_active":"2023-10-09T11:05:23.329561+0000","last_became_peered":"2023-10-09T11:05:23.329561+0000","last_unstale":"2023-10-09T11:09:00.414600+0000","last_undegraded":"2023-10-09T11:09:00.414600+0000","last_fullsized":"2023-10-09T11:09:00.414600+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:46.933799+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":36,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.543961+0000","last_change":"2023-10-09T11:07:33.229864+0000","last_active":"2023-10-09T11:09:00.543961+0000","last_peered":"2023-10-09T11:09:00.543961+0000","last_clean":"2023-10-09T11:09:00.543961+0000","last_became_active":"2023-10-09T11:07:33.229547+0000","last_became_peered":"2023-10-09T11:07:33.229547+0000","last_unstale":"2023-10-09T11:09:00.543961+0000","last_undegraded":"2023-10-09T11:09:00.543961+0000","last_fullsized":"2023-10-09T11:09:00.543961+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:34.990439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":357,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-10-09T11:08:53.114469+0000","last_change":"2023-10-09T11:04:50.510688+0000","last_active":"2023-10-09T11:08:53.114469+0000","last_peered":"2023-10-09T11:08:53.114469+0000","last_clean":"2023-10-09T11:08:53.114469+0000","last_became_active":"2023-10-09T11:04:50.507404+0000","last_became_peered":"2023-10-09T11:04:50.507404+0000","last_unstale":"2023-10-09T11:08:53.114469+0000","last_undegraded":"2023-10-09T11:08:53.114469+0000","last_fullsized":"2023-10-09T11:08:53.114469+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:13.663394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":941,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414934+0000","last_change":"2023-10-09T11:05:23.321040+0000","last_active":"2023-10-09T11:09:00.414934+0000","last_peered":"2023-10-09T11:09:00.414934+0000","last_clean":"2023-10-09T11:09:00.414934+0000","last_became_active":"2023-10-09T11:05:23.320674+0000","last_became_peered":"2023-10-09T11:05:23.320674+0000","last_unstale":"2023-10-09T11:09:00.414934+0000","last_undegraded":"2023-10-09T11:09:00.414934+0000","last_fullsized":"2023-10-09T11:09:00.414934+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.527660+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":403,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.422975+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:09:00.422975+0000","last_peered":"2023-10-09T11:09:00.422975+0000","last_clean":"2023-10-09T11:09:00.422975+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:09:00.422975+0000","last_undegraded":"2023-10-09T11:09:00.422975+0000","last_fullsized":"2023-10-09T11:09:00.422975+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":102,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414276+0000","last_change":"2023-10-09T11:04:50.511984+0000","last_active":"2023-10-09T11:09:00.414276+0000","last_peered":"2023-10-09T11:09:00.414276+0000","last_clean":"2023-10-09T11:09:00.414276+0000","last_became_active":"2023-10-09T11:04:50.510820+0000","last_became_peered":"2023-10-09T11:04:50.510820+0000","last_unstale":"2023-10-09T11:09:00.414276+0000","last_undegraded":"2023-10-09T11:09:00.414276+0000","last_fullsized":"2023-10-09T11:09:00.414276+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:38.662189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":554,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.422820+0000","last_change":"2023-10-09T11:05:20.327032+0000","last_active":"2023-10-09T11:09:00.422820+0000","last_peered":"2023-10-09T11:09:00.422820+0000","last_clean":"2023-10-09T11:09:00.422820+0000","last_became_active":"2023-10-09T11:05:20.320193+0000","last_became_peered":"2023-10-09T11:05:20.320193+0000","last_unstale":"2023-10-09T11:09:00.422820+0000","last_undegraded":"2023-10-09T11:09:00.422820+0000","last_fullsized":"2023-10-09T11:09:00.422820+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:44.862890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":102,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414166+0000","last_change":"2023-10-09T11:04:50.384219+0000","last_active":"2023-10-09T11:09:00.414166+0000","last_peered":"2023-10-09T11:09:00.414166+0000","last_clean":"2023-10-09T11:09:00.414166+0000","last_became_active":"2023-10-09T11:04:50.384028+0000","last_became_peered":"2023-10-09T11:04:50.384028+0000","last_unstale":"2023-10-09T11:09:00.414166+0000","last_undegraded":"2023-10-09T11:09:00.414166+0000","last_fullsized":"2023-10-09T11:09:00.414166+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:04.721869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":759,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.543691+0000","last_change":"2023-10-09T11:08:39.540029+0000","last_active":"2023-10-09T11:09:00.543691+0000","last_peered":"2023-10-09T11:09:00.543691+0000","last_clean":"2023-10-09T11:09:00.543691+0000","last_became_active":"2023-10-09T11:07:36.291077+0000","last_became_peered":"2023-10-09T11:07:36.291077+0000","last_unstale":"2023-10-09T11:09:00.543691+0000","last_undegraded":"2023-10-09T11:09:00.543691+0000","last_fullsized":"2023-10-09T11:09:00.543691+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.539966+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.539966+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:20.767812+0000","scrub_duration":0.0016245929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":112,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.415017+0000","last_change":"2023-10-09T11:05:20.299321+0000","last_active":"2023-10-09T11:09:00.415017+0000","last_peered":"2023-10-09T11:09:00.415017+0000","last_clean":"2023-10-09T11:09:00.415017+0000","last_became_active":"2023-10-09T11:05:20.299019+0000","last_became_peered":"2023-10-09T11:05:20.299019+0000","last_unstale":"2023-10-09T11:09:00.415017+0000","last_undegraded":"2023-10-09T11:09:00.415017+0000","last_fullsized":"2023-10-09T11:09:00.415017+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:08.645909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":102,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.414698+0000","last_change":"2023-10-09T11:04:50.366890+0000","last_active":"2023-10-09T11:09:00.414698+0000","last_peered":"2023-10-09T11:09:00.414698+0000","last_clean":"2023-10-09T11:09:00.414698+0000","last_became_active":"2023-10-09T11:04:50.366558+0000","last_became_peered":"2023-10-09T11:04:50.366558+0000","last_unstale":"2023-10-09T11:09:00.414698+0000","last_undegraded":"2023-10-09T11:09:00.414698+0000","last_fullsized":"2023-10-09T11:09:00.414698+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:50.993153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":875,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.426294+0000","last_change":"2023-10-09T11:04:53.274332+0000","last_active":"2023-10-09T11:09:00.426294+0000","last_peered":"2023-10-09T11:09:00.426294+0000","last_clean":"2023-10-09T11:09:00.426294+0000","last_became_active":"2023-10-09T11:04:53.273798+0000","last_became_peered":"2023-10-09T11:04:53.273798+0000","last_unstale":"2023-10-09T11:09:00.426294+0000","last_undegraded":"2023-10-09T11:09:00.426294+0000","last_fullsized":"2023-10-09T11:09:00.426294+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:04.220970+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":123,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.423308+0000","last_change":"2023-10-09T11:06:24.249877+0000","last_active":"2023-10-09T11:09:00.423308+0000","last_peered":"2023-10-09T11:09:00.423308+0000","last_clean":"2023-10-09T11:09:00.423308+0000","last_became_active":"2023-10-09T11:06:24.249573+0000","last_became_peered":"2023-10-09T11:06:24.249573+0000","last_unstale":"2023-10-09T11:09:00.423308+0000","last_undegraded":"2023-10-09T11:09:00.423308+0000","last_fullsized":"2023-10-09T11:09:00.423308+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:11.798192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":624,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-10-09T11:09:00.424493+0000","last_change":"2023-10-09T11:05:20.307347+0000","last_active":"2023-10-09T11:09:00.424493+0000","last_peered":"2023-10-09T11:09:00.424493+0000","last_clean":"2023-10-09T11:09:00.424493+0000","last_became_active":"2023-10-09T11:05:20.307126+0000","last_became_peered":"2023-10-09T11:05:20.307126+0000","last_unstale":"2023-10-09T11:09:00.424493+0000","last_undegraded":"2023-10-09T11:09:00.424493+0000","last_fullsized":"2023-10-09T11:09:00.424493+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:34.408515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":91,"num_bytes_recovered":15742,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12694,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":3096691,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":315,"num_read_kb":1718,"num_write":75,"num_write_kb":768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":618496,"data_stored":599830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":80,"ondisk_log_size":80,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332510,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1072660,"kb_used_data":177352,"kb_used_omap":0,"kb_used_meta":895296,"kb_avail":92672492,"statfs":{"total":95995035648,"available":94896631808,"internally_reserved":0,"allocated":181608448,"data_stored":179765971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916783104},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":1.3460000000000001,"15min":1.7350000000000001},"min":{"1min":0.254,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":9.8219999999999992,"5min":9.8219999999999992,"15min":9.8219999999999992},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.9099999999999999,"5min":1.6539999999999999,"15min":1.7669999999999999},"min":{"1min":0.28999999999999998,"5min":0.24299999999999999,"15min":0.216},"max":{"1min":5.4820000000000002,"5min":15.853,"15min":15.853},"last":0.47599999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:08:26 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":1.5029999999999999,"15min":1.5469999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":5.9669999999999996,"5min":11.619999999999999,"15min":11.619999999999999},"last":0.378},{"interface":"front","average":{"1min":2.851,"5min":2.0219999999999998,"15min":1.944},"min":{"1min":0.26300000000000001,"5min":0.25,"15min":0.25},"max":{"1min":13.096,"5min":13.096,"15min":13.096},"last":0.50700000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:08:54 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":1.573,"15min":1.77},"min":{"1min":0.309,"5min":0.309,"15min":0.245},"max":{"1min":5.1440000000000001,"5min":12.226000000000001,"15min":16.605},"last":0.497},{"interface":"front","average":{"1min":2.2770000000000001,"5min":1.6220000000000001,"15min":1.802},"min":{"1min":0.379,"5min":0.223,"15min":0.223},"max":{"1min":16.157,"5min":16.157,"15min":16.157},"last":0.45600000000000002}]},{"osd":3,"last update":"Mon Oct 9 11:08:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3050000000000002,"5min":1.7050000000000001,"15min":1.5009999999999999},"min":{"1min":0.34200000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.3970000000000002,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.7070000000000001,"5min":1.885,"15min":1.823},"min":{"1min":0.34000000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.0999999999999996,"5min":8.2330000000000005,"15min":8.2330000000000005},"last":0.39300000000000002}]},{"osd":4,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":0.91800000000000004,"15min":1.0940000000000001},"min":{"1min":0.29799999999999999,"5min":0.218,"15min":0.218},"max":{"1min":7.4630000000000001,"5min":7.4630000000000001,"15min":8.2940000000000005},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.034,"5min":1.256,"15min":1.244},"min":{"1min":0.30599999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.7190000000000003,"5min":11.808,"15min":16.481000000000002},"last":0.46500000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.0640000000000001,"15min":1.252},"min":{"1min":0.32800000000000001,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":5.4800000000000004,"5min":15.414999999999999,"15min":15.414999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.345,"15min":1.198},"min":{"1min":0.375,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.045,"5min":15.369,"15min":15.369},"last":0.44700000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:08:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":1.226,"15min":1.26},"min":{"1min":0.34599999999999997,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":3.8500000000000001,"5min":14.159000000000001,"15min":14.159000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.121,"5min":1.472,"15min":1.355},"min":{"1min":0.36099999999999999,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":3.5830000000000002,"5min":13.993,"15min":16.538},"last":0.437}]}]},{"osd":6,"up_from":44,"seq":188978561440,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1507264,"kb_used_data":6520,"kb_used_omap":0,"kb_used_meta":1500736,"kb_avail":92237888,"statfs":{"total":95995035648,"available":94451597312,"internally_reserved":0,"allocated":6676480,"data_stored":4850439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1536753664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":1.4950000000000001,"15min":1.9419999999999999},"min":{"1min":0.22600000000000001,"5min":0.223,"15min":0.217},"max":{"1min":12.209,"5min":13.867000000000001,"15min":54.743000000000002},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.081,"5min":1.181,"15min":1.8919999999999999},"min":{"1min":0.27300000000000002,"5min":0.214,"15min":0.214},"max":{"1min":3.3740000000000001,"5min":10.055999999999999,"15min":53.701000000000001},"last":0.60399999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:08:26 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.514,"15min":1.1910000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.23499999999999999},"max":{"1min":4.3710000000000004,"5min":16.225999999999999,"15min":41.241999999999997},"last":0.58299999999999996},{"interface":"front","average":{"1min":2.7330000000000001,"5min":2.1139999999999999,"15min":1.8260000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.248},"max":{"1min":9.2080000000000002,"5min":11.715,"15min":41.476999999999997},"last":0.34399999999999997}]},{"osd":2,"last update":"Mon Oct 9 11:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.867,"15min":1.9019999999999999},"min":{"1min":0.32200000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.2069999999999999,"5min":10.382999999999999,"15min":12.003},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.85599999999999998,"5min":1.2909999999999999,"15min":1.877},"min":{"1min":0.33800000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.6669999999999998,"5min":12.693,"15min":15.432},"last":0.436}]},{"osd":3,"last update":"Mon Oct 9 11:08:23 2023","interfaces":[{"interface":"back","average":{"1min":3.6749999999999998,"5min":2.2080000000000002,"15min":1.702},"min":{"1min":0.40200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":11.395,"5min":28.948,"15min":28.948},"last":0.502},{"interface":"front","average":{"1min":2.653,"5min":2.9820000000000002,"15min":2.6200000000000001},"min":{"1min":0.307,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.964,"5min":28.893000000000001,"15min":28.893000000000001},"last":0.38600000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.66900000000000004,"15min":0.998},"min":{"1min":0.28100000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.67100000000000004,"5min":6.6609999999999996,"15min":17.216999999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.53700000000000003,"15min":0.83299999999999996},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.24399999999999999},"max":{"1min":0.80100000000000005,"5min":2.3610000000000002,"15min":10.025},"last":0.53300000000000003}]},{"osd":5,"last update":"Mon Oct 9 11:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.78300000000000003,"15min":1.228},"min":{"1min":0.34000000000000002,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":2.7000000000000002,"5min":3.7389999999999999,"15min":19.786999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.68999999999999995,"15min":0.96499999999999997},"min":{"1min":0.377,"5min":0.248,"15min":0.248},"max":{"1min":1.22,"5min":4.085,"15min":9.9469999999999992},"last":0.56200000000000006}]},{"osd":7,"last update":"Mon Oct 9 11:08:14 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.081,"15min":1.268},"min":{"1min":0.32200000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.7250000000000001,"5min":10.246,"15min":12.882},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.83599999999999997,"15min":0.93300000000000005},"min":{"1min":0.35599999999999998,"5min":0.31,"15min":0.27500000000000002},"max":{"1min":3.2839999999999998,"5min":11.557,"15min":11.557},"last":0.627}]}]},{"osd":1,"up_from":388,"seq":1666447310955,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":505384,"kb_used_data":7904,"kb_used_omap":13,"kb_used_meta":497458,"kb_avail":93239768,"statfs":{"total":95995035648,"available":95477522432,"internally_reserved":0,"allocated":8093696,"data_stored":5050040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":509397703},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":2.6960000000000002,"15min":2.4430000000000001},"min":{"1min":0.22600000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":7.0570000000000004,"5min":18.192,"15min":18.192},"last":0.53300000000000003},{"interface":"front","average":{"1min":3.7949999999999999,"5min":3.0840000000000001,"15min":2.0920000000000001},"min":{"1min":0.26700000000000002,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":16.895,"5min":17.957999999999998,"15min":17.957999999999998},"last":0.45200000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.9380000000000002,"5min":2.5339999999999998,"15min":2.911},"min":{"1min":0.216,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":8.798,"5min":12.436999999999999,"15min":47.731000000000002},"last":0.60299999999999998},{"interface":"front","average":{"1min":3.895,"5min":2.8690000000000002,"15min":2.5880000000000001},"min":{"1min":0.26000000000000001,"5min":0.216,"15min":0.216},"max":{"1min":16.870000000000001,"5min":16.870000000000001,"15min":47.786999999999999},"last":0.56000000000000005}]},{"osd":3,"last update":"Mon Oct 9 11:08:26 2023","interfaces":[{"interface":"back","average":{"1min":2.819,"5min":3.3530000000000002,"15min":2.1099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":9.4960000000000004,"5min":92.447999999999993,"15min":92.447999999999993},"last":0.49399999999999999},{"interface":"front","average":{"1min":3.069,"5min":3.4350000000000001,"15min":2.133},"min":{"1min":0.29099999999999998,"5min":0.217,"15min":0.217},"max":{"1min":13.295999999999999,"5min":92.066999999999993,"15min":92.066999999999993},"last":0.47199999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0059999999999998,"5min":1.879,"15min":2.3380000000000001},"min":{"1min":0.30299999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.9790000000000001,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.68200000000000005},{"interface":"front","average":{"1min":2.9670000000000001,"5min":2.1480000000000001,"15min":1.73},"min":{"1min":0.34799999999999998,"5min":0.312,"15min":0.29999999999999999},"max":{"1min":16.841000000000001,"5min":16.841000000000001,"15min":16.841000000000001},"last":0.61599999999999999}]},{"osd":5,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2559999999999998,"5min":1.968,"15min":2.286},"min":{"1min":0.311,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":9.7010000000000005,"5min":20.689,"15min":20.689},"last":0.63100000000000001},{"interface":"front","average":{"1min":2.427,"5min":2.113,"15min":2.242},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.0199999999999996,"5min":11.65,"15min":11.65},"last":0.65400000000000003}]},{"osd":6,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0819999999999999,"5min":2.1819999999999999,"15min":2.3799999999999999},"min":{"1min":0.32900000000000001,"5min":0.312,"15min":0.309},"max":{"1min":9.9459999999999997,"5min":20.646999999999998,"15min":20.646999999999998},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.339,"5min":2.0339999999999998,"15min":2.1680000000000001},"min":{"1min":0.34399999999999997,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":7.0410000000000004,"5min":11.75,"15min":11.75},"last":0.752}]},{"osd":7,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.2530000000000001,"15min":1.7310000000000001},"min":{"1min":0.378,"5min":0.33000000000000002,"15min":0.32900000000000001},"max":{"1min":16.797999999999998,"5min":16.797999999999998,"15min":16.797999999999998},"last":0.64500000000000002},{"interface":"front","average":{"1min":2.2029999999999998,"5min":1.996,"15min":2.294},"min":{"1min":0.38100000000000001,"5min":0.35599999999999998,"15min":0.33200000000000002},"max":{"1min":8.4469999999999992,"5min":20.716000000000001,"15min":20.716000000000001},"last":0.66900000000000004}]}]},{"osd":0,"up_from":80,"seq":343597384013,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":982216,"kb_used_data":8128,"kb_used_omap":28,"kb_used_meta":974051,"kb_avail":92762936,"statfs":{"total":95995035648,"available":94989246464,"internally_reserved":0,"allocated":8323072,"data_stored":5180219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":997429072},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:08:29 2023","interfaces":[{"interface":"back","average":{"1min":1.946,"5min":2.0249999999999999,"15min":2.742},"min":{"1min":0.191,"5min":0.186,"15min":0.186},"max":{"1min":6.4909999999999997,"5min":14.644,"15min":19.608000000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.718,"5min":2.1549999999999998,"15min":2.8010000000000002},"min":{"1min":0.223,"5min":0.214,"15min":0.214},"max":{"1min":5.8609999999999998,"5min":15.834,"15min":19.992999999999999},"last":0.36299999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:08:57 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":2.0579999999999998,"15min":2.2959999999999998},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":7.9329999999999998,"5min":12.459,"15min":24.852},"last":0.38500000000000001},{"interface":"front","average":{"1min":6.4219999999999997,"5min":3.0819999999999999,"15min":3.0009999999999999},"min":{"1min":0.21299999999999999,"5min":0.20399999999999999,"15min":0.19800000000000001},"max":{"1min":99.679000000000002,"5min":99.679000000000002,"15min":99.679000000000002},"last":0.42799999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":3.016,"15min":3.3599999999999999},"min":{"1min":0.35099999999999998,"5min":0.22,"15min":0.22},"max":{"1min":7.9050000000000002,"5min":25.050000000000001,"15min":25.050000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.2890000000000001,"5min":2.355,"15min":2.0510000000000002},"min":{"1min":0.30499999999999999,"5min":0.254,"15min":0.254},"max":{"1min":6.2389999999999999,"5min":14.125999999999999,"15min":14.125999999999999},"last":0.39900000000000002}]},{"osd":4,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":1.877,"5min":2.1880000000000002,"15min":2.4489999999999998},"min":{"1min":0.27100000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.8150000000000004,"5min":14.507,"15min":77.177999999999997},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.6910000000000001,"5min":1.675,"15min":1.6419999999999999},"min":{"1min":0.33600000000000002,"5min":0.30199999999999999,"15min":0.28899999999999998},"max":{"1min":6.194,"5min":7.9249999999999998,"15min":10.513999999999999},"last":0.64600000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.7290000000000001,"15min":1.8160000000000001},"min":{"1min":0.34699999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.2199999999999998,"5min":7.9550000000000001,"15min":10.544},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.7050000000000001,"5min":1.7270000000000001,"15min":1.804},"min":{"1min":0.39200000000000002,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":4.6820000000000004,"5min":11.212999999999999,"15min":24.754999999999999},"last":0.63}]},{"osd":6,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":1.8380000000000001,"15min":1.8959999999999999},"min":{"1min":0.377,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":6.173,"5min":7.9829999999999997,"15min":18.920000000000002},"last":0.622},{"interface":"front","average":{"1min":1.423,"5min":1.786,"15min":1.927},"min":{"1min":0.38100000000000001,"5min":0.35299999999999998,"15min":0.31900000000000001},"max":{"1min":4.2789999999999999,"5min":11.257999999999999,"15min":24.800000000000001},"last":0.66800000000000004}]},{"osd":7,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.391,"5min":2.1789999999999998,"15min":2.0339999999999998},"min":{"1min":0.38900000000000001,"5min":0.35099999999999998,"15min":0.32700000000000001},"max":{"1min":9.8710000000000004,"5min":17.672000000000001,"15min":17.672000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":2.2170000000000001,"5min":2.4500000000000002,"15min":2.569},"min":{"1min":0.33400000000000002,"5min":0.311,"15min":0.311},"max":{"1min":4.9320000000000004,"5min":17.539000000000001,"15min":77.228999999999999},"last":0.65500000000000003}]}]},{"osd":2,"up_from":90,"seq":386547056953,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":765548,"kb_used_data":7716,"kb_used_omap":81,"kb_used_meta":757742,"kb_avail":92979604,"statfs":{"total":95995035648,"available":95211114496,"internally_reserved":0,"allocated":7901184,"data_stored":4942637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":775928026},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":2.2010000000000001,"15min":2.0609999999999999},"min":{"1min":0.186,"5min":0.182,"15min":0.17499999999999999},"max":{"1min":14.226000000000001,"5min":15.609,"15min":48.933999999999997},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.5419999999999998,"5min":2.056,"15min":1.8919999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.19600000000000001},"max":{"1min":9.5540000000000003,"5min":9.8010000000000002,"15min":15.595000000000001},"last":0.46899999999999997}]},{"osd":1,"last update":"Mon Oct 9 11:08:34 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":2.3780000000000001,"15min":2.5339999999999998},"min":{"1min":0.193,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":3.242,"5min":24.902000000000001,"15min":49.033999999999999},"last":0.438},{"interface":"front","average":{"1min":2.6600000000000001,"5min":2.5640000000000001,"15min":2.7200000000000002},"min":{"1min":0.253,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":14.26,"5min":25.850999999999999,"15min":48.874000000000002},"last":0.39900000000000002}]},{"osd":3,"last update":"Mon Oct 9 11:08:19 2023","interfaces":[{"interface":"back","average":{"1min":4.8019999999999996,"5min":2.9399999999999999,"15min":2.1560000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":25.890999999999998,"5min":25.890999999999998,"15min":25.890999999999998},"last":0.51500000000000001},{"interface":"front","average":{"1min":4.4560000000000004,"5min":2.895,"15min":2.395},"min":{"1min":0.372,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":28.689,"5min":28.689,"15min":28.689},"last":0.56299999999999994}]},{"osd":4,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":1.796,"15min":1.8959999999999999},"min":{"1min":0.27100000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":11.391999999999999,"5min":11.391999999999999,"15min":15.672000000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.468,"5min":1.7909999999999999,"15min":1.873},"min":{"1min":0.34899999999999998,"5min":0.249,"15min":0.249},"max":{"1min":8.3350000000000009,"5min":15.571,"15min":48.795000000000002},"last":0.68100000000000005}]},{"osd":5,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":1.597,"15min":1.5620000000000001},"min":{"1min":0.33700000000000002,"5min":0.29099999999999998,"15min":0.25900000000000001},"max":{"1min":7.4980000000000002,"5min":8.5310000000000006,"15min":12.128},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.8580000000000001,"5min":1.7869999999999999,"15min":2.0339999999999998},"min":{"1min":0.34899999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.441000000000001,"5min":11.441000000000001,"15min":49.061},"last":0.60699999999999998}]},{"osd":6,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.909,"15min":1.804},"min":{"1min":0.41899999999999998,"5min":0.313,"15min":0.313},"max":{"1min":8.6039999999999992,"5min":15.536,"15min":48.845999999999997},"last":0.70599999999999996},{"interface":"front","average":{"1min":2.6499999999999999,"5min":2.012,"15min":1.657},"min":{"1min":0.42399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.157999999999999,"5min":10.157999999999999,"15min":10.157999999999999},"last":0.59299999999999997}]},{"osd":7,"last update":"Mon Oct 9 11:08:02 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":2.081,"15min":1.9419999999999999},"min":{"1min":0.45100000000000001,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":8.3800000000000008,"5min":15.59,"15min":48.966000000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":2.492,"5min":1.917,"15min":1.762},"min":{"1min":0.46200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.5919999999999996,"5min":8.5549999999999997,"15min":14.94},"last":0.621}]}]},{"osd":3,"up_from":469,"seq":2014339661883,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":343180,"kb_used_data":8132,"kb_used_omap":11,"kb_used_meta":335028,"kb_avail":93401972,"statfs":{"total":95995035648,"available":95643619328,"internally_reserved":0,"allocated":8327168,"data_stored":5049312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11309,"internal_metadata":343069651},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":3.004,"5min":2.1840000000000002,"15min":1.875},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":14.521000000000001,"5min":14.882,"15min":14.882},"last":0.35299999999999998},{"interface":"front","average":{"1min":2.3919999999999999,"5min":1.397,"15min":1.044},"min":{"1min":0.20100000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":9.4030000000000005,"5min":9.4030000000000005,"15min":9.4030000000000005},"last":0.33700000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":2.218,"15min":1.6739999999999999},"min":{"1min":0.245,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":8.8350000000000009,"5min":17.93,"15min":17.93},"last":0.48799999999999999},{"interface":"front","average":{"1min":3.0099999999999998,"5min":2.246,"15min":1.8049999999999999},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":10.645,"5min":19.163,"15min":19.163},"last":0.41899999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4860000000000002,"5min":2.0840000000000001,"15min":1.419},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":12.323,"5min":38.840000000000003,"15min":38.840000000000003},"last":0.37},{"interface":"front","average":{"1min":1.6539999999999999,"5min":2.2120000000000002,"15min":1.4570000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":10.172000000000001,"5min":37.991999999999997,"15min":37.991999999999997},"last":0.38400000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6419999999999999,"5min":1.655,"15min":1.6000000000000001},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.3719999999999999,"5min":14.907999999999999,"15min":14.907999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.0489999999999999,"5min":1.1439999999999999,"15min":0.84499999999999997},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.7889999999999997,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.433}]},{"osd":5,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.4259999999999999,"15min":1.417},"min":{"1min":0.33100000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.8499999999999996,"5min":9.3300000000000001,"15min":9.3300000000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.5529999999999999,"5min":1.5649999999999999,"15min":1.482},"min":{"1min":0.34699999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.3320000000000007,"5min":10.029,"15min":10.029},"last":0.50800000000000001}]},{"osd":6,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.256,"5min":1.6539999999999999,"15min":1.74},"min":{"1min":0.32300000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.9550000000000001,"5min":15.455,"15min":15.455},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.8100000000000001,"5min":1.706,"15min":1.6639999999999999},"min":{"1min":0.36799999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.4019999999999992,"5min":14.837,"15min":14.837},"last":0.53300000000000003}]},{"osd":7,"last update":"Mon Oct 9 11:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.365,"15min":1.1220000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.6769999999999996,"5min":6.907,"15min":6.907},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.55,"5min":1.595,"15min":1.3029999999999999},"min":{"1min":0.33700000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.9320000000000004,"5min":6.9119999999999999,"15min":6.9119999999999999},"last":0.52400000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019304,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903264,"kb_used_data":6424,"kb_used_omap":0,"kb_used_meta":896832,"kb_avail":92841888,"statfs":{"total":95995035648,"available":95070093312,"internally_reserved":0,"allocated":6578176,"data_stored":4737080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918355968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.613,"15min":2.4129999999999998},"min":{"1min":0.22600000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":6.1289999999999996,"5min":13.162000000000001,"15min":213.88999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.742,"15min":2.5019999999999998},"min":{"1min":0.26500000000000001,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":3.875,"5min":9.0670000000000002,"15min":214.09800000000001},"last":0.5}]},{"osd":1,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2989999999999999,"5min":2.6059999999999999,"15min":2.6440000000000001},"min":{"1min":0.28299999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.718,"5min":57.966999999999999,"15min":121.41},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.4810000000000001,"5min":2.2519999999999998,"15min":2.1280000000000001},"min":{"1min":0.26900000000000002,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":6.1980000000000004,"5min":57.832000000000001,"15min":122.21899999999999},"last":0.47299999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:09:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.748,"15min":1.738},"min":{"1min":0.30499999999999999,"5min":0.246,"15min":0.23799999999999999},"max":{"1min":3.0779999999999998,"5min":18.407,"15min":18.407},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.7430000000000001,"15min":2.0739999999999998},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":2.5129999999999999,"5min":8.4250000000000007,"15min":27.818000000000001},"last":0.441}]},{"osd":3,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":2.4380000000000002,"15min":1.98},"min":{"1min":0.34799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.1689999999999996,"5min":60.753,"15min":60.753},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.5049999999999999,"5min":2.2069999999999999,"15min":1.5149999999999999},"min":{"1min":0.45300000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.4980000000000002,"5min":61.473999999999997,"15min":61.473999999999997},"last":0.40200000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:08:21 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":1.353,"15min":1.3600000000000001},"min":{"1min":0.30499999999999999,"5min":0.224,"15min":0.224},"max":{"1min":4.7670000000000003,"5min":11.59,"15min":19.068999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.079,"5min":1.2809999999999999,"15min":1.238},"min":{"1min":0.34200000000000003,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":3.7570000000000001,"5min":9.8610000000000007,"15min":15.535},"last":0.317}]},{"osd":6,"last update":"Mon Oct 9 11:09:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.2969999999999999,"15min":1.1910000000000001},"min":{"1min":0.29299999999999998,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":6.8689999999999998,"5min":8.3930000000000007,"15min":15.739000000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.3500000000000001,"5min":1.4179999999999999,"15min":1.4730000000000001},"min":{"1min":0.33700000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.8170000000000002,"5min":12.795999999999999,"15min":14.093999999999999},"last":0.48699999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:09:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.294,"15min":1.5629999999999999},"min":{"1min":0.374,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.181,"5min":10.15,"15min":19.923999999999999},"last":0.432},{"interface":"front","average":{"1min":1.155,"5min":1.091,"15min":1.488},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.0439999999999996,"5min":10.853,"15min":19.859999999999999},"last":0.378}]}]},{"osd":5,"up_from":37,"seq":158913790373,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1205868,"kb_used_data":6564,"kb_used_omap":0,"kb_used_meta":1199296,"kb_avail":92539284,"statfs":{"total":95995035648,"available":94760226816,"internally_reserved":0,"allocated":6721536,"data_stored":4887136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1228079104},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:08:25 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":2.3239999999999998,"15min":2.2919999999999998},"min":{"1min":0.255,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.4349999999999996,"5min":49.665999999999997,"15min":58.271999999999998},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.849,"5min":2.3740000000000001,"15min":2.3399999999999999},"min":{"1min":0.29699999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.4610000000000003,"5min":49.701999999999998,"15min":58.25},"last":0.47199999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:08:32 2023","interfaces":[{"interface":"back","average":{"1min":2.5179999999999998,"5min":1.899,"15min":2.0670000000000002},"min":{"1min":0.246,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.1850000000000005,"5min":16.962,"15min":32.491999999999997},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.3439999999999999,"5min":2.004,"15min":1.917},"min":{"1min":0.32800000000000001,"5min":0.27300000000000002,"15min":0.247},"max":{"1min":9.1989999999999998,"5min":17.033999999999999,"15min":32.537999999999997},"last":0.52200000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:08:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.321,"15min":1.8240000000000001},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.221},"max":{"1min":7.3109999999999999,"5min":7.3109999999999999,"15min":11.257999999999999},"last":0.434},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.8919999999999999,"15min":1.988},"min":{"1min":0.317,"5min":0.253,"15min":0.253},"max":{"1min":4.085,"5min":11.673,"15min":14.621},"last":0.32600000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.962,"5min":1.738,"15min":1.5720000000000001},"min":{"1min":0.311,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.0289999999999999,"5min":15.535,"15min":15.535},"last":0.377},{"interface":"front","average":{"1min":2.8839999999999999,"5min":2.0710000000000002,"15min":1.8959999999999999},"min":{"1min":0.40200000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":15.757999999999999,"5min":15.757999999999999,"15min":15.757999999999999},"last":0.45500000000000002}]},{"osd":4,"last update":"Mon Oct 9 11:08:32 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":1.0089999999999999,"15min":0.93799999999999994},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.216},"max":{"1min":3.0680000000000001,"5min":12.042,"15min":12.042},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.3480000000000001,"5min":1.1020000000000001,"15min":1.1899999999999999},"min":{"1min":0.27000000000000002,"5min":0.245,"15min":0.245},"max":{"1min":6.8639999999999999,"5min":9.2490000000000006,"15min":9.2490000000000006},"last":0.41999999999999998}]},{"osd":6,"last update":"Mon Oct 9 11:08:45 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.2090000000000001,"15min":1.2450000000000001},"min":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.27100000000000002},"max":{"1min":5.944,"5min":7.1020000000000003,"15min":9.1969999999999992},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.214,"5min":1.204,"15min":1.0700000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29099999999999998},"max":{"1min":9.298,"5min":11.983000000000001,"15min":11.983000000000001},"last":0.499}]},{"osd":7,"last update":"Mon Oct 9 11:08:08 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.3160000000000001,"15min":1.302},"min":{"1min":0.35399999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.0469999999999997,"5min":8.9429999999999996,"15min":9.3539999999999992},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.03,"5min":1.278,"15min":1.177},"min":{"1min":0.32800000000000001,"5min":0.30499999999999999,"15min":0.30099999999999999},"max":{"1min":4.2350000000000003,"5min":11.726000000000001,"15min":11.726000000000001},"last":0.54500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":200704,"data_stored":198280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":90227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":8,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":311296,"data_stored":311296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3733,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:09:07.420 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T11:09:07.420 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:09:07.420 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T11:09:07.421 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T11:09:07.421 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T11:09:07.421 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T11:09:07.456 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/img /tmp/rbd_import_export_475522/img2 /tmp/rbd_import_export_475522/img3 2023-10-09T11:09:07.492 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:09:07.511 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:09:07.542 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:07.565 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:09:07.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:07.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:09:07.617 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:09:07.646 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:07 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/577587733' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:09:07.646 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:07 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/577587733' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:09:07.669 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:09:07.706 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:07.789 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd help export 2023-10-09T11:09:07.789 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep -q export-format 2023-10-09T11:09:07.808 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_475522/img bs=1k count=10 seek=100 2023-10-09T11:09:07.809 INFO:tasks.workunit.client.0.smithi125.stderr:10+0 records in 2023-10-09T11:09:07.809 INFO:tasks.workunit.client.0.smithi125.stderr:10+0 records out 2023-10-09T11:09:07.809 INFO:tasks.workunit.client.0.smithi125.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.305e-05 s, 162 MB/s 2023-10-09T11:09:07.810 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format /tmp/rbd_import_export_475522/img testimg 2023-10-09T11:09:07.870 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T11:09:07.871 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T11:09:07.873 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap create testimg@snap 2023-10-09T11:09:07.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:07 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/577587733' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:09:07.971 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:07.971 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:08.182 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:09:08.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:08.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:09:08.285 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:09:08.315 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:08.487 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:09:08.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:08.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:09:08.500 INFO:tasks.workunit.client.0.smithi125.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T11:09:08.510 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:08.510 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd image-meta set testimg key1 value1 2023-10-09T11:09:08.535 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T11:09:08.560 INFO:tasks.workunit.client.0.smithi125.stderr:++ rbd image-meta list testimg 2023-10-09T11:09:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:09:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T11:09:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/577587733' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:09:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[195507]: osdmap e577: 8 total, 8 up, 8 in 2023-10-09T11:09:08.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[195507]: failed to encode map e577 with expected crc 2023-10-09T11:09:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[195507]: failed to encode map e577 with expected crc 2023-10-09T11:09:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[195507]: failed to encode map e577 with expected crc 2023-10-09T11:09:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[195507]: failed to encode map e577 with expected crc 2023-10-09T11:09:08.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[195507]: pgmap v1415: 103 pgs: 103 active+clean; 3.5 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 229 KiB/s rd, 102 KiB/s wr, 55 op/s 2023-10-09T11:09:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:09:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T11:09:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/577587733' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:09:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[200197]: osdmap e577: 8 total, 8 up, 8 in 2023-10-09T11:09:08.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[200197]: failed to encode map e577 with expected crc 2023-10-09T11:09:08.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[200197]: failed to encode map e577 with expected crc 2023-10-09T11:09:08.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[200197]: failed to encode map e577 with expected crc 2023-10-09T11:09:08.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[200197]: failed to encode map e577 with expected crc 2023-10-09T11:09:08.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:08 smithi125 ceph-mon[200197]: pgmap v1415: 103 pgs: 103 active+clean; 3.5 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 229 KiB/s rd, 102 KiB/s wr, 55 op/s 2023-10-09T11:09:08.591 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:09:08.602 INFO:tasks.workunit.client.0.smithi125.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-10-09T11:09:08.602 INFO:tasks.workunit.client.0.smithi125.stderr: 2023-10-09T11:09:08.602 INFO:tasks.workunit.client.0.smithi125.stderr:Key Value 2023-10-09T11:09:08.602 INFO:tasks.workunit.client.0.smithi125.stderr:key1 value1' 2023-10-09T11:09:08.602 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_475522/img_v2 2023-10-09T11:09:08.623 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:08.640 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T11:09:08.644 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:09:08.645 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_475522/img_v2 testimg_import 2023-10-09T11:09:08.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:08.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:09:08.748 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:09:08.777 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:08.847 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:09:08.848 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:08.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:08.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:09:08.951 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:09:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:08 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:09:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:08 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T11:09:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:08 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/577587733' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:09:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:08 smithi155 ceph-mon[184854]: osdmap e577: 8 total, 8 up, 8 in 2023-10-09T11:09:08.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:08 smithi155 ceph-mon[184854]: failed to encode map e577 with expected crc 2023-10-09T11:09:08.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:08 smithi155 ceph-mon[184854]: failed to encode map e577 with expected crc 2023-10-09T11:09:08.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:08 smithi155 ceph-mon[184854]: failed to encode map e577 with expected crc 2023-10-09T11:09:08.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:08 smithi155 ceph-mon[184854]: failed to encode map e577 with expected crc 2023-10-09T11:09:08.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:08 smithi155 ceph-mon[184854]: pgmap v1415: 103 pgs: 103 active+clean; 3.5 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 229 KiB/s rd, 102 KiB/s wr, 55 op/s 2023-10-09T11:09:08.980 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:09.199 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:09:09.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:09.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:09:09.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:09:09.337 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:09.358 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:09:09.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:09.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:09:09.462 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:09:09.499 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:09.520 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T11:09:09.523 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import 2023-10-09T11:09:09.556 INFO:tasks.workunit.client.0.smithi125.stdout:rbd image 'testimg_import': 2023-10-09T11:09:09.556 INFO:tasks.workunit.client.0.smithi125.stdout: size 110 KiB in 1 objects 2023-10-09T11:09:09.556 INFO:tasks.workunit.client.0.smithi125.stdout: order 22 (4 MiB objects) 2023-10-09T11:09:09.557 INFO:tasks.workunit.client.0.smithi125.stdout: snapshot_count: 1 2023-10-09T11:09:09.557 INFO:tasks.workunit.client.0.smithi125.stdout: id: b7418ac709c1 2023-10-09T11:09:09.557 INFO:tasks.workunit.client.0.smithi125.stdout: block_name_prefix: rbd_data.b7418ac709c1 2023-10-09T11:09:09.557 INFO:tasks.workunit.client.0.smithi125.stdout: format: 2 2023-10-09T11:09:09.557 INFO:tasks.workunit.client.0.smithi125.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T11:09:09.557 INFO:tasks.workunit.client.0.smithi125.stdout: op_features: 2023-10-09T11:09:09.557 INFO:tasks.workunit.client.0.smithi125.stdout: flags: 2023-10-09T11:09:09.557 INFO:tasks.workunit.client.0.smithi125.stdout: create_timestamp: Mon Oct 9 11:09:08 2023 2023-10-09T11:09:09.558 INFO:tasks.workunit.client.0.smithi125.stdout: access_timestamp: Mon Oct 9 11:09:08 2023 2023-10-09T11:09:09.558 INFO:tasks.workunit.client.0.smithi125.stdout: modify_timestamp: Mon Oct 9 11:09:08 2023 2023-10-09T11:09:09.559 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import@snap 2023-10-09T11:09:09.593 INFO:tasks.workunit.client.0.smithi125.stdout:rbd image 'testimg_import': 2023-10-09T11:09:09.593 INFO:tasks.workunit.client.0.smithi125.stdout: size 110 KiB in 1 objects 2023-10-09T11:09:09.593 INFO:tasks.workunit.client.0.smithi125.stdout: order 22 (4 MiB objects) 2023-10-09T11:09:09.593 INFO:tasks.workunit.client.0.smithi125.stdout: snapshot_count: 1 2023-10-09T11:09:09.594 INFO:tasks.workunit.client.0.smithi125.stdout: id: b7418ac709c1 2023-10-09T11:09:09.594 INFO:tasks.workunit.client.0.smithi125.stdout: block_name_prefix: rbd_data.b7418ac709c1 2023-10-09T11:09:09.594 INFO:tasks.workunit.client.0.smithi125.stdout: format: 2 2023-10-09T11:09:09.594 INFO:tasks.workunit.client.0.smithi125.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T11:09:09.594 INFO:tasks.workunit.client.0.smithi125.stdout: op_features: 2023-10-09T11:09:09.594 INFO:tasks.workunit.client.0.smithi125.stdout: flags: 2023-10-09T11:09:09.594 INFO:tasks.workunit.client.0.smithi125.stdout: create_timestamp: Mon Oct 9 11:09:08 2023 2023-10-09T11:09:09.594 INFO:tasks.workunit.client.0.smithi125.stdout: access_timestamp: Mon Oct 9 11:09:08 2023 2023-10-09T11:09:09.595 INFO:tasks.workunit.client.0.smithi125.stdout: modify_timestamp: Mon Oct 9 11:09:08 2023 2023-10-09T11:09:09.595 INFO:tasks.workunit.client.0.smithi125.stdout: protected: False 2023-10-09T11:09:09.596 INFO:tasks.workunit.client.0.smithi125.stderr:++ rbd image-meta list testimg_import 2023-10-09T11:09:09.636 INFO:tasks.workunit.client.0.smithi125.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-10-09T11:09:09.636 INFO:tasks.workunit.client.0.smithi125.stderr: 2023-10-09T11:09:09.636 INFO:tasks.workunit.client.0.smithi125.stderr:Key Value 2023-10-09T11:09:09.636 INFO:tasks.workunit.client.0.smithi125.stderr:key1 value1' 2023-10-09T11:09:09.636 INFO:tasks.workunit.client.0.smithi125.stderr:+ '[' 'There is 1 metadatum on this image: 2023-10-09T11:09:09.636 INFO:tasks.workunit.client.0.smithi125.stderr: 2023-10-09T11:09:09.637 INFO:tasks.workunit.client.0.smithi125.stderr:Key Value 2023-10-09T11:09:09.637 INFO:tasks.workunit.client.0.smithi125.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-10-09T11:09:09.637 INFO:tasks.workunit.client.0.smithi125.stderr: 2023-10-09T11:09:09.637 INFO:tasks.workunit.client.0.smithi125.stderr:Key Value 2023-10-09T11:09:09.637 INFO:tasks.workunit.client.0.smithi125.stderr:key1 value1' ']' 2023-10-09T11:09:09.637 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg_import /tmp/rbd_import_export_475522/img_import 2023-10-09T11:09:09.669 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete...done. 2023-10-09T11:09:09.672 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_475522/img /tmp/rbd_import_export_475522/img_import 2023-10-09T11:09:09.672 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_475522/img /tmp/rbd_import_export_475522/img_import 2023-10-09T11:09:09.673 INFO:tasks.workunit.client.0.smithi125.stderr:++ stat /tmp/rbd_import_export_475522/img --format %b 2023-10-09T11:09:09.674 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=24 2023-10-09T11:09:09.674 INFO:tasks.workunit.client.0.smithi125.stderr:++ stat /tmp/rbd_import_export_475522/img_import --format %b 2023-10-09T11:09:09.675 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=224 2023-10-09T11:09:09.676 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=200 2023-10-09T11:09:09.676 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=200 2023-10-09T11:09:09.676 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-09T11:09:09.676 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_475522/img_snap 2023-10-09T11:09:09.713 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete...done. 2023-10-09T11:09:09.716 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_475522/img_snap_import 2023-10-09T11:09:09.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:09 smithi155 ceph-mon[184854]: failed to encode map e578 with expected crc 2023-10-09T11:09:09.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:09 smithi155 ceph-mon[184854]: osdmap e578: 8 total, 8 up, 8 in 2023-10-09T11:09:09.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:09 smithi155 ceph-mon[184854]: failed to encode map e578 with expected crc 2023-10-09T11:09:09.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:09 smithi155 ceph-mon[184854]: failed to encode map e578 with expected crc 2023-10-09T11:09:09.754 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete...done. 2023-10-09T11:09:09.757 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_475522/img_snap /tmp/rbd_import_export_475522/img_snap_import 2023-10-09T11:09:09.757 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_475522/img_snap /tmp/rbd_import_export_475522/img_snap_import 2023-10-09T11:09:09.758 INFO:tasks.workunit.client.0.smithi125.stderr:++ stat /tmp/rbd_import_export_475522/img_snap --format %b 2023-10-09T11:09:09.759 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=224 2023-10-09T11:09:09.759 INFO:tasks.workunit.client.0.smithi125.stderr:++ stat /tmp/rbd_import_export_475522/img_snap_import --format %b 2023-10-09T11:09:09.760 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=224 2023-10-09T11:09:09.761 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T11:09:09.761 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T11:09:09.761 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T11:09:09.761 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/img_v2 2023-10-09T11:09:09.761 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/img_import 2023-10-09T11:09:09.762 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/img_snap 2023-10-09T11:09:09.762 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/img_snap_import 2023-10-09T11:09:09.763 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap rm testimg_import@snap 2023-10-09T11:09:09.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:09 smithi125 ceph-mon[195507]: failed to encode map e578 with expected crc 2023-10-09T11:09:09.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:09 smithi125 ceph-mon[195507]: osdmap e578: 8 total, 8 up, 8 in 2023-10-09T11:09:09.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:09 smithi125 ceph-mon[195507]: failed to encode map e578 with expected crc 2023-10-09T11:09:09.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:09 smithi125 ceph-mon[195507]: failed to encode map e578 with expected crc 2023-10-09T11:09:09.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:09 smithi125 ceph-mon[200197]: failed to encode map e578 with expected crc 2023-10-09T11:09:09.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:09 smithi125 ceph-mon[200197]: osdmap e578: 8 total, 8 up, 8 in 2023-10-09T11:09:09.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:09 smithi125 ceph-mon[200197]: failed to encode map e578 with expected crc 2023-10-09T11:09:09.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:09 smithi125 ceph-mon[200197]: failed to encode map e578 with expected crc 2023-10-09T11:09:09.838 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:09:09.838 INFO:teuthology.orchestra.run.smithi125.stdout: "success": "" 2023-10-09T11:09:09.838 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:09:09.872 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:09:09.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:09.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:09:09.976 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:09:10.012 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:10.033 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:09:10.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:10.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:09:10.052 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:10.103 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:09:10.122 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:09:10.136 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:10.140 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:09:10.160 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:09:10.166 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:10.175 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T11:09:10.181 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:09:10.201 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:09:10.220 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:09:10.251 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:09:10.252 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:09:10.252 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:09:10.252 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:09:10.252 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:09:10.252 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:09:10.252 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:09:10.252 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:09:10.253 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:09:10.508 INFO:tasks.workunit.client.0.smithi125.stderr: Removing snap: 100% complete...done. 2023-10-09T11:09:10.515 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg_import 2023-10-09T11:09:10.593 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:10.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:10.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:10.603 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T11:09:10.606 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap rm testimg@snap 2023-10-09T11:09:10.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:09:10.727 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:10.746 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:09:10.748 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:10 smithi155 ceph-mon[184854]: failed to encode map e578 with expected crc 2023-10-09T11:09:10.748 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:10 smithi155 ceph-mon[184854]: failed to encode map e578 with expected crc 2023-10-09T11:09:10.748 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:10 smithi155 ceph-mon[184854]: osdmap e579: 8 total, 8 up, 8 in 2023-10-09T11:09:10.748 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:10 smithi155 ceph-mon[184854]: failed to encode map e579 with expected crc 2023-10-09T11:09:10.748 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:10 smithi155 ceph-mon[184854]: failed to encode map e579 with expected crc 2023-10-09T11:09:10.748 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:10 smithi155 ceph-mon[184854]: pgmap v1418: 103 pgs: 103 active+clean; 3.5 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 428 KiB/s rd, 174 KiB/s wr, 133 op/s 2023-10-09T11:09:10.748 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:10 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2941230769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:09:10.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:10.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:09:10.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:10 smithi125 ceph-mon[195507]: failed to encode map e578 with expected crc 2023-10-09T11:09:10.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:10 smithi125 ceph-mon[195507]: failed to encode map e578 with expected crc 2023-10-09T11:09:10.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:10 smithi125 ceph-mon[195507]: osdmap e579: 8 total, 8 up, 8 in 2023-10-09T11:09:10.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:10 smithi125 ceph-mon[195507]: failed to encode map e579 with expected crc 2023-10-09T11:09:10.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:10 smithi125 ceph-mon[195507]: failed to encode map e579 with expected crc 2023-10-09T11:09:10.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:10 smithi125 ceph-mon[195507]: pgmap v1418: 103 pgs: 103 active+clean; 3.5 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 428 KiB/s rd, 174 KiB/s wr, 133 op/s 2023-10-09T11:09:10.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:10 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2941230769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:09:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:10 smithi125 ceph-mon[200197]: failed to encode map e578 with expected crc 2023-10-09T11:09:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:10 smithi125 ceph-mon[200197]: failed to encode map e578 with expected crc 2023-10-09T11:09:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:10 smithi125 ceph-mon[200197]: osdmap e579: 8 total, 8 up, 8 in 2023-10-09T11:09:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:10 smithi125 ceph-mon[200197]: failed to encode map e579 with expected crc 2023-10-09T11:09:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:10 smithi125 ceph-mon[200197]: failed to encode map e579 with expected crc 2023-10-09T11:09:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:10 smithi125 ceph-mon[200197]: pgmap v1418: 103 pgs: 103 active+clean; 3.5 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 428 KiB/s rd, 174 KiB/s wr, 133 op/s 2023-10-09T11:09:10.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:10 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2941230769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:09:10.850 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:09:10.878 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:10.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:10.901 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:09:10.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:09:10.957 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:11.002 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:09:11.030 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:11.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:11.053 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:09:11.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:09:11.154 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:09:11.181 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:11.199 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:09:11.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:11.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:09:11.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:09:11.330 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:11.349 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:09:11.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:11.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:09:11.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:09:11.480 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:11.488 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:09:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:09:11] "GET /metrics HTTP/1.1" 200 34108 "" "Prometheus/2.43.0" 2023-10-09T11:09:11.499 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:09:11.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:11.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:09:11.511 INFO:tasks.workunit.client.0.smithi125.stderr: Removing snap: 100% complete...done. 2023-10-09T11:09:11.520 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm testimg 2023-10-09T11:09:11.603 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:09:11.606 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T11:09:11.609 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --order 20 /tmp/rbd_import_export_475522/img testimg 2023-10-09T11:09:11.632 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:11.670 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T11:09:11.670 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T11:09:11.673 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_475522/img_v2 2023-10-09T11:09:11.706 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:09:11.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:11.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:09:11.714 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T11:09:11.718 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_475522/img_v2 testimg_import 2023-10-09T11:09:11.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:11 smithi155 ceph-mon[184854]: failed to encode map e579 with expected crc 2023-10-09T11:09:11.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:11 smithi155 ceph-mon[184854]: failed to encode map e580 with expected crc 2023-10-09T11:09:11.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:11 smithi155 ceph-mon[184854]: osdmap e580: 8 total, 8 up, 8 in 2023-10-09T11:09:11.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:11 smithi155 ceph-mon[184854]: failed to encode map e580 with expected crc 2023-10-09T11:09:11.788 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T11:09:11.792 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import 2023-10-09T11:09:11.792 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep order 2023-10-09T11:09:11.792 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk '{print $2}' 2023-10-09T11:09:11.792 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep 20 2023-10-09T11:09:11.812 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:09:11.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:11 smithi125 ceph-mon[200197]: failed to encode map e579 with expected crc 2023-10-09T11:09:11.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:11 smithi125 ceph-mon[200197]: failed to encode map e580 with expected crc 2023-10-09T11:09:11.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:11 smithi125 ceph-mon[200197]: osdmap e580: 8 total, 8 up, 8 in 2023-10-09T11:09:11.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:11 smithi125 ceph-mon[200197]: failed to encode map e580 with expected crc 2023-10-09T11:09:11.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:11 smithi125 ceph-mon[195507]: failed to encode map e579 with expected crc 2023-10-09T11:09:11.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:11 smithi125 ceph-mon[195507]: failed to encode map e580 with expected crc 2023-10-09T11:09:11.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:11 smithi125 ceph-mon[195507]: osdmap e580: 8 total, 8 up, 8 in 2023-10-09T11:09:11.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:11 smithi125 ceph-mon[195507]: failed to encode map e580 with expected crc 2023-10-09T11:09:11.830 INFO:tasks.workunit.client.0.smithi125.stdout:20 2023-10-09T11:09:11.830 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/img_v2 2023-10-09T11:09:11.831 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg_import 2023-10-09T11:09:11.838 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:11.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:09:11.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:11.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:09:11.902 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T11:09:11.902 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T11:09:11.902 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T11:09:11.903 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T11:09:11.903 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.903 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:133037 9.b 9.da5dd31b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e362)", 2023-10-09T11:09:11.903 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:20.474469+0000", 2023-10-09T11:09:11.903 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 591.42490582000005, 2023-10-09T11:09:11.903 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97365184000000005, 2023-10-09T11:09:11.903 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:11.903 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:09:11.903 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:11.903 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:09:11.904 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:09:11.904 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133037 2023-10-09T11:09:11.904 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.904 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:11.904 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.904 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:11.904 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474469+0000", 2023-10-09T11:09:11.904 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.905 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.905 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.905 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:11.905 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474469+0000", 2023-10-09T11:09:11.905 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.905 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.905 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.906 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:11.906 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474467+0000", 2023-10-09T11:09:11.906 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:11.906 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.906 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.906 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:11.906 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474475+0000", 2023-10-09T11:09:11.906 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6009999999999997e-06 2023-10-09T11:09:11.906 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.906 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.906 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:11.906 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474493+0000", 2023-10-09T11:09:11.907 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8150000000000001e-05 2023-10-09T11:09:11.907 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.907 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.907 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:11.907 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474502+0000", 2023-10-09T11:09:11.907 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1400000000000006e-06 2023-10-09T11:09:11.907 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.908 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.908 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.908 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474569+0000", 2023-10-09T11:09:11.908 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6849999999999999e-05 2023-10-09T11:09:11.908 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.908 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.908 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:11.908 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.474579+0000", 2023-10-09T11:09:11.908 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9369999999999999e-06 2023-10-09T11:09:11.908 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.908 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.909 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.909 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448111+0000", 2023-10-09T11:09:11.909 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97353240399999996 2023-10-09T11:09:11.909 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.909 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.909 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:11.909 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448121+0000", 2023-10-09T11:09:11.910 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1679999999999993e-06 2023-10-09T11:09:11.910 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.910 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:11.910 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.910 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.910 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.910 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45683.0:133044 9.b 9.5473ef4b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e362)", 2023-10-09T11:09:11.910 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T10:59:20.482185+0000", 2023-10-09T11:09:11.910 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 591.41718944599995, 2023-10-09T11:09:11.910 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96595614900000004, 2023-10-09T11:09:11.910 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:11.911 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T11:09:11.911 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:11.911 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45683", 2023-10-09T11:09:11.911 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3816099015", 2023-10-09T11:09:11.911 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 133044 2023-10-09T11:09:11.911 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.911 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:11.911 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.912 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:11.912 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482185+0000", 2023-10-09T11:09:11.912 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.912 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.912 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.912 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:11.912 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482185+0000", 2023-10-09T11:09:11.913 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.913 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.913 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.913 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:11.913 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482184+0000", 2023-10-09T11:09:11.913 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:11.913 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.913 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.913 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:11.913 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482193+0000", 2023-10-09T11:09:11.914 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0310000000000004e-06 2023-10-09T11:09:11.914 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.914 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.914 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:11.914 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482217+0000", 2023-10-09T11:09:11.915 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4677000000000001e-05 2023-10-09T11:09:11.915 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.915 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.915 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:11.915 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482228+0000", 2023-10-09T11:09:11.915 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.043e-05 2023-10-09T11:09:11.915 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.916 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.916 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.916 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482326+0000", 2023-10-09T11:09:11.916 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8241999999999999e-05 2023-10-09T11:09:11.916 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.916 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.917 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:11.917 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:20.482334+0000", 2023-10-09T11:09:11.917 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7910000000000005e-06 2023-10-09T11:09:11.917 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.917 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.917 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.917 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448135+0000", 2023-10-09T11:09:11.917 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96580099799999997 2023-10-09T11:09:11.917 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.917 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.918 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:11.918 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T10:59:21.448141+0000", 2023-10-09T11:09:11.918 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5719999999999999e-06 2023-10-09T11:09:11.918 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.918 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:11.918 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.918 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.918 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.918 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39381 11.d 11:b5913833:::benchmark_data_smithi125_439543_object39380:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:09:11.918 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.310294+0000", 2023-10-09T11:09:11.918 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 217.589080475, 2023-10-09T11:09:11.918 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0048304729999999, 2023-10-09T11:09:11.919 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:11.919 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:11.919 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:11.919 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:09:11.919 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:11.919 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39381 2023-10-09T11:09:11.919 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.919 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:11.919 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.919 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:11.919 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310294+0000", 2023-10-09T11:09:11.919 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.920 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.920 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.920 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:11.920 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310294+0000", 2023-10-09T11:09:11.920 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.920 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.920 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.920 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:11.920 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310292+0000", 2023-10-09T11:09:11.920 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:09:11.920 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.921 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.921 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:11.921 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310321+0000", 2023-10-09T11:09:11.921 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8965000000000001e-05 2023-10-09T11:09:11.921 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.921 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.921 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:11.921 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310335+0000", 2023-10-09T11:09:11.921 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3407999999999999e-05 2023-10-09T11:09:11.922 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.922 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.922 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:11.922 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310343+0000", 2023-10-09T11:09:11.922 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1030000000000002e-06 2023-10-09T11:09:11.922 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.922 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.922 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.922 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310413+0000", 2023-10-09T11:09:11.922 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9822999999999999e-05 2023-10-09T11:09:11.922 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.923 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.923 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:11.923 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.310428+0000", 2023-10-09T11:09:11.923 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5021999999999999e-05 2023-10-09T11:09:11.923 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.923 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.923 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.923 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.311384+0000", 2023-10-09T11:09:11.923 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0009558540000001 2023-10-09T11:09:11.923 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.923 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.923 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:11.924 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.311492+0000", 2023-10-09T11:09:11.924 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000108618 2023-10-09T11:09:11.924 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.924 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.924 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:09:11.924 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.311595+0000", 2023-10-09T11:09:11.924 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000102431 2023-10-09T11:09:11.924 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.924 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.924 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.924 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315040+0000", 2023-10-09T11:09:11.924 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0034454049999999999 2023-10-09T11:09:11.925 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.925 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.925 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:11.925 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315049+0000", 2023-10-09T11:09:11.925 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7749999999999998e-06 2023-10-09T11:09:11.925 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.925 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.925 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.925 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315067+0000", 2023-10-09T11:09:11.925 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8595999999999999e-05 2023-10-09T11:09:11.925 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.925 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.926 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:11.926 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315098+0000", 2023-10-09T11:09:11.926 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0185999999999999e-05 2023-10-09T11:09:11.926 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.926 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.926 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:11.926 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315125+0000", 2023-10-09T11:09:11.926 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6982999999999999e-05 2023-10-09T11:09:11.926 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.926 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:11.926 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.926 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.927 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.927 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39392 11.d 11:b103d758:::benchmark_data_smithi125_439543_object39391:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:09:11.927 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.311770+0000", 2023-10-09T11:09:11.927 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 217.58760447399999, 2023-10-09T11:09:11.927 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0081084490000001, 2023-10-09T11:09:11.927 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:11.927 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:11.927 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:11.927 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:09:11.927 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:11.927 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39392 2023-10-09T11:09:11.928 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.928 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:11.928 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.928 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:11.928 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311770+0000", 2023-10-09T11:09:11.928 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.928 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.928 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.928 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:11.928 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311770+0000", 2023-10-09T11:09:11.928 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.929 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.929 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.929 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:11.929 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311769+0000", 2023-10-09T11:09:11.929 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:11.929 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.929 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.929 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:11.929 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311790+0000", 2023-10-09T11:09:11.929 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1759999999999998e-05 2023-10-09T11:09:11.929 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.929 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.930 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:11.930 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311800+0000", 2023-10-09T11:09:11.930 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6430000000000007e-06 2023-10-09T11:09:11.930 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.930 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.930 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:11.930 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311806+0000", 2023-10-09T11:09:11.930 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0619999999999997e-06 2023-10-09T11:09:11.930 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.930 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.930 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.930 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311962+0000", 2023-10-09T11:09:11.931 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000156312 2023-10-09T11:09:11.931 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.931 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.931 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:11.931 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.311972+0000", 2023-10-09T11:09:11.931 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0098000000000001e-05 2023-10-09T11:09:11.931 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.931 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.931 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.931 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.312904+0000", 2023-10-09T11:09:11.931 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0009318810000001 2023-10-09T11:09:11.931 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.932 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.932 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:11.932 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.312992+0000", 2023-10-09T11:09:11.932 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7459000000000004e-05 2023-10-09T11:09:11.932 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.932 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.932 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:09:11.932 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.313085+0000", 2023-10-09T11:09:11.932 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2730000000000002e-05 2023-10-09T11:09:11.932 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.932 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.933 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.933 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315270+0000", 2023-10-09T11:09:11.933 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021854069999999999 2023-10-09T11:09:11.933 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.933 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.933 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.933 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318204+0000", 2023-10-09T11:09:11.933 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002933719 2023-10-09T11:09:11.933 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.933 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.933 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:11.933 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319781+0000", 2023-10-09T11:09:11.933 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001577427 2023-10-09T11:09:11.934 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.934 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.934 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:11.934 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319838+0000", 2023-10-09T11:09:11.934 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6882000000000002e-05 2023-10-09T11:09:11.934 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.934 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.934 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:11.934 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319879+0000", 2023-10-09T11:09:11.934 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0627999999999998e-05 2023-10-09T11:09:11.934 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.935 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:11.935 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.935 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.935 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.935 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39394 11.d 11:ba71174e:::benchmark_data_smithi125_439543_object39393:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:09:11.935 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.316112+0000", 2023-10-09T11:09:11.935 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 217.58326262599999, 2023-10-09T11:09:11.935 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.003833969, 2023-10-09T11:09:11.935 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:11.935 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:11.935 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:11.935 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:09:11.936 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:11.936 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39394 2023-10-09T11:09:11.936 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.936 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:11.936 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.936 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:11.936 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316112+0000", 2023-10-09T11:09:11.936 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.936 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.936 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.936 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:11.936 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316112+0000", 2023-10-09T11:09:11.936 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.937 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.937 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.937 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:11.937 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316110+0000", 2023-10-09T11:09:11.937 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:09:11.937 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.937 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.937 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:11.937 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316139+0000", 2023-10-09T11:09:11.937 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9023e-05 2023-10-09T11:09:11.937 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.937 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.938 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:11.938 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316160+0000", 2023-10-09T11:09:11.938 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0852999999999998e-05 2023-10-09T11:09:11.938 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.938 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.938 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:11.939 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316170+0000", 2023-10-09T11:09:11.939 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8970000000000003e-06 2023-10-09T11:09:11.939 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.939 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.939 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.939 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316244+0000", 2023-10-09T11:09:11.939 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4184000000000007e-05 2023-10-09T11:09:11.939 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.939 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.939 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:11.940 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.316256+0000", 2023-10-09T11:09:11.940 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1447000000000001e-05 2023-10-09T11:09:11.940 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.940 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.940 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.940 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.313863+0000", 2023-10-09T11:09:11.940 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.99760760800000003 2023-10-09T11:09:11.941 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.941 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.941 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:11.941 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.313943+0000", 2023-10-09T11:09:11.941 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9276999999999993e-05 2023-10-09T11:09:11.941 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.941 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.941 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:09:11.941 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.314050+0000", 2023-10-09T11:09:11.941 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000107831 2023-10-09T11:09:11.942 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.942 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.942 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.942 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315326+0000", 2023-10-09T11:09:11.942 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001276103 2023-10-09T11:09:11.942 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.943 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.943 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.943 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318251+0000", 2023-10-09T11:09:11.943 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002924683 2023-10-09T11:09:11.943 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.943 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.943 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:11.943 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319894+0000", 2023-10-09T11:09:11.943 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016428599999999999 2023-10-09T11:09:11.943 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.943 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.944 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:11.944 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319928+0000", 2023-10-09T11:09:11.944 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3674999999999999e-05 2023-10-09T11:09:11.944 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.944 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.944 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:11.944 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319946+0000", 2023-10-09T11:09:11.945 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8297000000000001e-05 2023-10-09T11:09:11.945 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.945 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:11.945 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.945 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.945 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.945 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39401 11.d 11:b5dcd9bc:::benchmark_data_smithi125_439543_object39400:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:09:11.945 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.322614+0000", 2023-10-09T11:09:11.945 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 217.576760788, 2023-10-09T11:09:11.945 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.003544776, 2023-10-09T11:09:11.945 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:11.946 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:11.946 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:11.946 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:09:11.946 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:11.946 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39401 2023-10-09T11:09:11.946 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.947 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:11.947 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.947 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:11.947 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322614+0000", 2023-10-09T11:09:11.947 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.947 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.947 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.948 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:11.948 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322614+0000", 2023-10-09T11:09:11.948 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.948 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.948 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.948 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:11.948 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322612+0000", 2023-10-09T11:09:11.948 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:11.949 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.949 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.949 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:11.949 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322636+0000", 2023-10-09T11:09:11.949 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4015e-05 2023-10-09T11:09:11.949 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.950 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.950 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:11.950 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322650+0000", 2023-10-09T11:09:11.950 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.345e-05 2023-10-09T11:09:11.950 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.950 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.950 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:11.950 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322657+0000", 2023-10-09T11:09:11.951 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3490000000000003e-06 2023-10-09T11:09:11.951 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.951 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.951 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.951 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322718+0000", 2023-10-09T11:09:11.951 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0520999999999999e-05 2023-10-09T11:09:11.951 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.951 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.951 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:11.951 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.322728+0000", 2023-10-09T11:09:11.951 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0216e-05 2023-10-09T11:09:11.951 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.952 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.952 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.952 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315358+0000", 2023-10-09T11:09:11.952 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.992629808 2023-10-09T11:09:11.952 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.952 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.952 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:11.952 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315493+0000", 2023-10-09T11:09:11.952 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013509899999999999 2023-10-09T11:09:11.952 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.952 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.952 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:09:11.953 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.315627+0000", 2023-10-09T11:09:11.953 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000134077 2023-10-09T11:09:11.953 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.953 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.953 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.953 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318286+0000", 2023-10-09T11:09:11.953 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00265898 2023-10-09T11:09:11.953 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.953 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.953 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.953 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319980+0000", 2023-10-09T11:09:11.953 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016941 2023-10-09T11:09:11.953 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.954 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.954 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:11.954 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326073+0000", 2023-10-09T11:09:11.954 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0060933239999999998 2023-10-09T11:09:11.954 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.954 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.954 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:11.954 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326124+0000", 2023-10-09T11:09:11.954 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1088000000000003e-05 2023-10-09T11:09:11.954 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.954 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.954 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:11.955 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326159+0000", 2023-10-09T11:09:11.955 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4353000000000001e-05 2023-10-09T11:09:11.955 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.955 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:11.955 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.955 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.955 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.955 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39439 11.d 11:b59e136a:::benchmark_data_smithi125_439543_object39438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:09:11.955 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.352200+0000", 2023-10-09T11:09:11.955 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 217.54717427599999, 2023-10-09T11:09:11.955 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97404727300000005, 2023-10-09T11:09:11.955 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:11.955 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:11.956 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:11.956 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:09:11.956 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:11.956 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39439 2023-10-09T11:09:11.956 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.956 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:11.956 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.956 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:11.956 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352200+0000", 2023-10-09T11:09:11.956 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.956 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.956 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.956 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:11.957 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352200+0000", 2023-10-09T11:09:11.957 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.957 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.957 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.957 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:11.957 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352199+0000", 2023-10-09T11:09:11.957 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:11.957 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.957 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.957 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:11.957 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352248+0000", 2023-10-09T11:09:11.957 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9008000000000002e-05 2023-10-09T11:09:11.957 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.958 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.958 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:11.958 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352264+0000", 2023-10-09T11:09:11.958 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5722e-05 2023-10-09T11:09:11.958 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.958 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.958 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:11.958 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352272+0000", 2023-10-09T11:09:11.958 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2199999999999992e-06 2023-10-09T11:09:11.958 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.958 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.958 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.959 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352340+0000", 2023-10-09T11:09:11.959 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8590000000000006e-05 2023-10-09T11:09:11.959 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.959 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.959 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:11.959 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352353+0000", 2023-10-09T11:09:11.959 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2648e-05 2023-10-09T11:09:11.959 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.959 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.959 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.959 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.316802+0000", 2023-10-09T11:09:11.959 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96444901500000002 2023-10-09T11:09:11.960 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.960 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.960 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:11.960 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.316917+0000", 2023-10-09T11:09:11.960 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011529800000000001 2023-10-09T11:09:11.960 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.960 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.960 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:09:11.960 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.317050+0000", 2023-10-09T11:09:11.960 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000132442 2023-10-09T11:09:11.960 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.960 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.960 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.961 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.319767+0000", 2023-10-09T11:09:11.961 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027173449999999999 2023-10-09T11:09:11.961 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.961 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.961 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.961 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326058+0000", 2023-10-09T11:09:11.961 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062911210000000002 2023-10-09T11:09:11.961 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.961 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.961 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:11.961 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326171+0000", 2023-10-09T11:09:11.961 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000112565 2023-10-09T11:09:11.962 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.962 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.962 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:11.962 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326218+0000", 2023-10-09T11:09:11.962 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7377999999999999e-05 2023-10-09T11:09:11.962 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.962 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.962 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:11.962 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326248+0000", 2023-10-09T11:09:11.962 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9482999999999999e-05 2023-10-09T11:09:11.962 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.962 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:11.962 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.963 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.963 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.963 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39440 11.d 11:bb23716e:::benchmark_data_smithi125_439543_object39439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:09:11.963 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.352285+0000", 2023-10-09T11:09:11.963 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 217.547090059, 2023-10-09T11:09:11.963 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97412860000000001, 2023-10-09T11:09:11.963 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:11.963 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:11.963 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:11.963 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:09:11.963 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:11.963 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39440 2023-10-09T11:09:11.964 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.964 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:11.964 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.964 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:11.964 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352285+0000", 2023-10-09T11:09:11.964 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.964 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.964 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.964 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:11.964 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352285+0000", 2023-10-09T11:09:11.964 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.964 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.964 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.965 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:11.965 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352284+0000", 2023-10-09T11:09:11.965 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:11.965 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.965 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.965 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:11.965 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352305+0000", 2023-10-09T11:09:11.965 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1382e-05 2023-10-09T11:09:11.965 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.965 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.965 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:11.965 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352314+0000", 2023-10-09T11:09:11.965 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5220000000000007e-06 2023-10-09T11:09:11.966 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.966 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.966 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:11.966 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352320+0000", 2023-10-09T11:09:11.966 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0850000000000002e-06 2023-10-09T11:09:11.966 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.966 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.966 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.966 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352375+0000", 2023-10-09T11:09:11.966 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5411999999999997e-05 2023-10-09T11:09:11.966 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.966 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.966 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:11.967 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.352382+0000", 2023-10-09T11:09:11.967 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.849e-06 2023-10-09T11:09:11.967 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.967 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.967 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.967 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318309+0000", 2023-10-09T11:09:11.967 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96592686800000005 2023-10-09T11:09:11.967 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.967 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.967 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:11.967 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318427+0000", 2023-10-09T11:09:11.967 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118267 2023-10-09T11:09:11.968 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.968 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.968 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:09:11.968 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.318567+0000", 2023-10-09T11:09:11.968 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013931800000000001 2023-10-09T11:09:11.968 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.968 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.968 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.968 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.320026+0000", 2023-10-09T11:09:11.968 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014593410000000001 2023-10-09T11:09:11.968 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.968 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.968 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:11.969 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326259+0000", 2023-10-09T11:09:11.969 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062325130000000003 2023-10-09T11:09:11.969 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.969 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.969 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.969 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326327+0000", 2023-10-09T11:09:11.971 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8077999999999994e-05 2023-10-09T11:09:11.971 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.971 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.971 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:11.971 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326376+0000", 2023-10-09T11:09:11.971 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8949000000000002e-05 2023-10-09T11:09:11.971 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.971 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.971 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:11.971 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326413+0000", 2023-10-09T11:09:11.971 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7545000000000001e-05 2023-10-09T11:09:11.972 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.972 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:11.972 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.972 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.972 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.972 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39464 11.d 11:b03a9b94:::benchmark_data_smithi125_439543_object39463:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:09:11.972 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.367860+0000", 2023-10-09T11:09:11.972 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 217.53151418100001, 2023-10-09T11:09:11.972 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96193788300000005, 2023-10-09T11:09:11.972 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:11.972 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:11.972 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:11.972 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:09:11.973 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:11.973 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39464 2023-10-09T11:09:11.973 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.973 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:11.973 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.973 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:11.973 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367860+0000", 2023-10-09T11:09:11.973 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.973 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.974 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.974 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:11.974 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367860+0000", 2023-10-09T11:09:11.974 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.974 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.974 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.974 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:11.974 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367859+0000", 2023-10-09T11:09:11.974 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:09:11.974 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.975 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:11.975 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367885+0000", 2023-10-09T11:09:11.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5820999999999999e-05 2023-10-09T11:09:11.975 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.975 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:11.975 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367899+0000", 2023-10-09T11:09:11.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4803e-05 2023-10-09T11:09:11.975 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.975 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:11.975 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367907+0000", 2023-10-09T11:09:11.976 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2010000000000002e-06 2023-10-09T11:09:11.976 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.976 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367967+0000", 2023-10-09T11:09:11.976 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0299e-05 2023-10-09T11:09:11.976 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.976 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:11.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.367978+0000", 2023-10-09T11:09:11.976 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1616e-05 2023-10-09T11:09:11.976 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.977 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.977 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.320095+0000", 2023-10-09T11:09:11.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95211647399999999 2023-10-09T11:09:11.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.977 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:11.977 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.320237+0000", 2023-10-09T11:09:11.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014207400000000001 2023-10-09T11:09:11.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.977 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:09:11.978 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.320378+0000", 2023-10-09T11:09:11.978 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014121199999999999 2023-10-09T11:09:11.978 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.978 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.978 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:11.978 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326264+0000", 2023-10-09T11:09:11.978 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0058858579999999999 2023-10-09T11:09:11.978 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.978 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.978 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.978 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.329650+0000", 2023-10-09T11:09:11.978 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0033856849999999998 2023-10-09T11:09:11.979 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.979 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.979 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.979 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.329705+0000", 2023-10-09T11:09:11.979 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5633000000000001e-05 2023-10-09T11:09:11.979 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.979 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.979 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:11.979 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.329758+0000", 2023-10-09T11:09:11.980 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2395999999999998e-05 2023-10-09T11:09:11.980 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.980 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.980 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:11.980 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.329798+0000", 2023-10-09T11:09:11.980 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0491000000000002e-05 2023-10-09T11:09:11.980 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.980 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:11.980 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.980 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.980 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.980 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39465 11.d 11:baf45870:::benchmark_data_smithi125_439543_object39464:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:09:11.981 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.368960+0000", 2023-10-09T11:09:11.981 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 217.530414898, 2023-10-09T11:09:11.981 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96470808799999996, 2023-10-09T11:09:11.981 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:11.981 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:11.981 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:11.981 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:09:11.981 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:11.981 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39465 2023-10-09T11:09:11.982 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.982 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:11.982 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.982 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:11.982 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368960+0000", 2023-10-09T11:09:11.982 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.982 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.982 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.982 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:11.982 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368960+0000", 2023-10-09T11:09:11.982 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.982 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.983 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.983 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:11.983 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368959+0000", 2023-10-09T11:09:11.983 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:09:11.983 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.983 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.983 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:11.984 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368977+0000", 2023-10-09T11:09:11.984 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8229000000000001e-05 2023-10-09T11:09:11.984 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.984 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.984 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:11.984 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368987+0000", 2023-10-09T11:09:11.984 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0139999999999999e-05 2023-10-09T11:09:11.984 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.984 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.985 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:11.985 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.368993+0000", 2023-10-09T11:09:11.985 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1700000000000002e-06 2023-10-09T11:09:11.985 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.985 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.985 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.985 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.369036+0000", 2023-10-09T11:09:11.985 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2355999999999999e-05 2023-10-09T11:09:11.986 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.986 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.986 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:11.986 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.369042+0000", 2023-10-09T11:09:11.986 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8070000000000002e-06 2023-10-09T11:09:11.986 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.986 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.986 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.987 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.321501+0000", 2023-10-09T11:09:11.987 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95245864800000002 2023-10-09T11:09:11.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.987 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.987 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:11.987 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.321617+0000", 2023-10-09T11:09:11.987 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011572 2023-10-09T11:09:11.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.987 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:09:11.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.321752+0000", 2023-10-09T11:09:11.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013548 2023-10-09T11:09:11.988 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.988 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:11.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326270+0000", 2023-10-09T11:09:11.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045178120000000004 2023-10-09T11:09:11.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.989 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.989 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.329989+0000", 2023-10-09T11:09:11.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037189179999999999 2023-10-09T11:09:11.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.989 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.989 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333591+0000", 2023-10-09T11:09:11.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036021080000000001 2023-10-09T11:09:11.990 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.990 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.990 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:11.990 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333626+0000", 2023-10-09T11:09:11.990 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5120000000000003e-05 2023-10-09T11:09:11.990 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.990 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.990 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:11.990 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333668+0000", 2023-10-09T11:09:11.990 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1616000000000003e-05 2023-10-09T11:09:11.990 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.990 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:11.991 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.991 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.991 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.991 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39467 11.d 11:b2ddd126:::benchmark_data_smithi125_439543_object39466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:09:11.991 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.370134+0000", 2023-10-09T11:09:11.991 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 217.529240144, 2023-10-09T11:09:11.991 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96363200699999996, 2023-10-09T11:09:11.991 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:11.991 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:11.991 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:11.991 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:09:11.991 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:11.991 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39467 2023-10-09T11:09:11.992 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.992 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:11.992 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.992 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:11.992 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370134+0000", 2023-10-09T11:09:11.992 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.992 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.992 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.992 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:11.992 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370134+0000", 2023-10-09T11:09:11.992 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:11.992 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.992 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.993 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:11.993 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370133+0000", 2023-10-09T11:09:11.993 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:09:11.993 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.993 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.993 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:11.993 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370183+0000", 2023-10-09T11:09:11.993 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9186e-05 2023-10-09T11:09:11.993 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.993 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.993 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:11.993 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370200+0000", 2023-10-09T11:09:11.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6937999999999999e-05 2023-10-09T11:09:11.994 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.994 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.994 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:11.994 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370213+0000", 2023-10-09T11:09:11.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3599e-05 2023-10-09T11:09:11.994 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.994 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.994 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.994 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370255+0000", 2023-10-09T11:09:11.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1913999999999999e-05 2023-10-09T11:09:11.994 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.995 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:11.995 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.370262+0000", 2023-10-09T11:09:11.995 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6089999999999997e-06 2023-10-09T11:09:11.995 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.995 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:11.995 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.323037+0000", 2023-10-09T11:09:11.995 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95277573000000004 2023-10-09T11:09:11.995 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.995 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:11.995 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.323131+0000", 2023-10-09T11:09:11.996 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3330000000000003e-05 2023-10-09T11:09:11.996 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.996 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.996 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:09:11.996 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.323244+0000", 2023-10-09T11:09:11.996 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000113033 2023-10-09T11:09:11.996 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.996 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.996 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:11.996 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.328073+0000", 2023-10-09T11:09:11.996 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0048294849999999997 2023-10-09T11:09:11.996 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.996 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.997 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.997 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330038+0000", 2023-10-09T11:09:11.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019642829999999998 2023-10-09T11:09:11.997 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.997 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:11.997 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333706+0000", 2023-10-09T11:09:11.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003667942 2023-10-09T11:09:11.997 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.997 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:11.997 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333732+0000", 2023-10-09T11:09:11.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6774000000000001e-05 2023-10-09T11:09:11.998 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.998 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.998 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:11.998 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333766+0000", 2023-10-09T11:09:11.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4178999999999999e-05 2023-10-09T11:09:11.998 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.998 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:11.998 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:11.998 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.998 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.998 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39473 11.d 11:b96957ab:::benchmark_data_smithi125_439543_object39472:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:09:11.998 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.372932+0000", 2023-10-09T11:09:11.999 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 217.52644222200001, 2023-10-09T11:09:11.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96091150700000005, 2023-10-09T11:09:11.999 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:11.999 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:11.999 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:11.999 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:09:11.999 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:11.999 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39473 2023-10-09T11:09:11.999 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:11.999 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:11.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:11.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:11.999 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.372932+0000", 2023-10-09T11:09:12.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.000 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:12.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.372932+0000", 2023-10-09T11:09:12.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.000 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:12.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.372931+0000", 2023-10-09T11:09:12.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:12.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:12.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.372988+0000", 2023-10-09T11:09:12.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6953999999999997e-05 2023-10-09T11:09:12.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:12.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.373007+0000", 2023-10-09T11:09:12.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8711000000000001e-05 2023-10-09T11:09:12.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:12.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.373017+0000", 2023-10-09T11:09:12.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0400999999999999e-05 2023-10-09T11:09:12.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.002 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:12.002 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.373082+0000", 2023-10-09T11:09:12.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4999000000000006e-05 2023-10-09T11:09:12.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.002 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:12.002 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.373094+0000", 2023-10-09T11:09:12.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1766000000000001e-05 2023-10-09T11:09:12.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:12.003 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.324554+0000", 2023-10-09T11:09:12.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95145981199999996 2023-10-09T11:09:12.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.003 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:12.003 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.324645+0000", 2023-10-09T11:09:12.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1250999999999998e-05 2023-10-09T11:09:12.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.003 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:09:12.003 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.324763+0000", 2023-10-09T11:09:12.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011804899999999999 2023-10-09T11:09:12.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:12.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.328078+0000", 2023-10-09T11:09:12.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0033156180000000002 2023-10-09T11:09:12.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330081+0000", 2023-10-09T11:09:12.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020020369999999999 2023-10-09T11:09:12.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333788+0000", 2023-10-09T11:09:12.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037078150000000002 2023-10-09T11:09:12.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:12.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333813+0000", 2023-10-09T11:09:12.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4239e-05 2023-10-09T11:09:12.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:12.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333844+0000", 2023-10-09T11:09:12.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1325999999999998e-05 2023-10-09T11:09:12.006 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.006 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:12.006 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.006 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.006 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.006 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39476 11.d 11:bbb64df7:::benchmark_data_smithi125_439543_object39475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:09:12.006 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.377052+0000", 2023-10-09T11:09:12.006 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 217.522322452, 2023-10-09T11:09:12.006 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96144097100000003, 2023-10-09T11:09:12.006 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:12.006 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:12.006 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:12.007 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:09:12.007 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:12.007 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39476 2023-10-09T11:09:12.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.007 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:12.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.007 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:12.007 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377052+0000", 2023-10-09T11:09:12.007 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.007 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:12.007 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377052+0000", 2023-10-09T11:09:12.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:12.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377050+0000", 2023-10-09T11:09:12.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:09:12.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:12.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377075+0000", 2023-10-09T11:09:12.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4941000000000001e-05 2023-10-09T11:09:12.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:12.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377090+0000", 2023-10-09T11:09:12.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.428e-05 2023-10-09T11:09:12.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:12.009 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377098+0000", 2023-10-09T11:09:12.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7110000000000001e-06 2023-10-09T11:09:12.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.009 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:12.010 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377155+0000", 2023-10-09T11:09:12.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6977e-05 2023-10-09T11:09:12.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.010 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.010 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:12.010 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.377164+0000", 2023-10-09T11:09:12.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1920000000000005e-06 2023-10-09T11:09:12.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.010 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.010 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:12.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326460+0000", 2023-10-09T11:09:12.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.94929656600000001 2023-10-09T11:09:12.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:12.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326573+0000", 2023-10-09T11:09:12.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011275100000000001 2023-10-09T11:09:12.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:09:12.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.326705+0000", 2023-10-09T11:09:12.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013251199999999999 2023-10-09T11:09:12.012 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.012 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.012 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:12.012 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330000+0000", 2023-10-09T11:09:12.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032948890000000001 2023-10-09T11:09:12.012 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.012 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.012 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.013 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333869+0000", 2023-10-09T11:09:12.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038684290000000001 2023-10-09T11:09:12.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.013 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.013 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338415+0000", 2023-10-09T11:09:12.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045463700000000001 2023-10-09T11:09:12.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.013 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:12.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338458+0000", 2023-10-09T11:09:12.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2852999999999998e-05 2023-10-09T11:09:12.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:12.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338493+0000", 2023-10-09T11:09:12.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5231000000000002e-05 2023-10-09T11:09:12.014 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.014 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:12.014 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.015 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39490 11.d 11:b79c85dd:::benchmark_data_smithi125_439543_object39489:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:09:12.015 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.388748+0000", 2023-10-09T11:09:12.015 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 217.51062662999999, 2023-10-09T11:09:12.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.94984969600000002, 2023-10-09T11:09:12.015 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:12.015 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:12.015 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:12.015 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:09:12.015 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:12.016 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39490 2023-10-09T11:09:12.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.016 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:12.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:12.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388748+0000", 2023-10-09T11:09:12.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.017 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:12.017 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388748+0000", 2023-10-09T11:09:12.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.017 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.017 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:12.017 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388746+0000", 2023-10-09T11:09:12.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:09:12.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.018 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:12.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388771+0000", 2023-10-09T11:09:12.018 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4868e-05 2023-10-09T11:09:12.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.018 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:12.019 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388789+0000", 2023-10-09T11:09:12.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8077e-05 2023-10-09T11:09:12.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:12.019 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388801+0000", 2023-10-09T11:09:12.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1698999999999999e-05 2023-10-09T11:09:12.020 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.020 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:12.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388915+0000", 2023-10-09T11:09:12.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000114003 2023-10-09T11:09:12.020 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.021 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:12.021 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.388926+0000", 2023-10-09T11:09:12.021 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0729e-05 2023-10-09T11:09:12.021 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.021 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.021 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:12.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.328103+0000", 2023-10-09T11:09:12.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.93917761600000005 2023-10-09T11:09:12.022 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:12.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.328228+0000", 2023-10-09T11:09:12.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012434699999999999 2023-10-09T11:09:12.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:09:12.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.328355+0000", 2023-10-09T11:09:12.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012757299999999999 2023-10-09T11:09:12.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:12.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330045+0000", 2023-10-09T11:09:12.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001689968 2023-10-09T11:09:12.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.337060+0000", 2023-10-09T11:09:12.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0070151830000000004 2023-10-09T11:09:12.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338523+0000", 2023-10-09T11:09:12.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001462514 2023-10-09T11:09:12.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:12.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338557+0000", 2023-10-09T11:09:12.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3948000000000001e-05 2023-10-09T11:09:12.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:12.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338598+0000", 2023-10-09T11:09:12.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0923000000000001e-05 2023-10-09T11:09:12.025 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.025 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:12.025 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.026 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.36510.0:39501 11.d 11:b409f08d:::benchmark_data_smithi125_439543_object39500:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e505)", 2023-10-09T11:09:12.026 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:05:34.397575+0000", 2023-10-09T11:09:12.026 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 217.50179999400001, 2023-10-09T11:09:12.026 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.94120543800000001, 2023-10-09T11:09:12.026 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:12.026 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:12.026 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:12.026 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.36510", 2023-10-09T11:09:12.026 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:12.027 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39501 2023-10-09T11:09:12.027 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.027 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:12.027 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.027 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:12.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397575+0000", 2023-10-09T11:09:12.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.027 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.027 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.027 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:12.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397575+0000", 2023-10-09T11:09:12.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.028 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:12.028 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397573+0000", 2023-10-09T11:09:12.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:09:12.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.028 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:12.028 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397613+0000", 2023-10-09T11:09:12.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0145000000000003e-05 2023-10-09T11:09:12.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.029 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:12.029 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397626+0000", 2023-10-09T11:09:12.029 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3421000000000001e-05 2023-10-09T11:09:12.029 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.029 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.029 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:12.029 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397634+0000", 2023-10-09T11:09:12.029 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6210000000000004e-06 2023-10-09T11:09:12.029 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.029 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.029 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:12.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397700+0000", 2023-10-09T11:09:12.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5512000000000006e-05 2023-10-09T11:09:12.030 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T11:09:12.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:34.397709+0000", 2023-10-09T11:09:12.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3360000000000001e-06 2023-10-09T11:09:12.030 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:12.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330105+0000", 2023-10-09T11:09:12.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.93239575399999997 2023-10-09T11:09:12.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.031 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.031 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:12.031 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330264+0000", 2023-10-09T11:09:12.031 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015971200000000001 2023-10-09T11:09:12.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.031 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.031 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,6", 2023-10-09T11:09:12.031 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.330365+0000", 2023-10-09T11:09:12.031 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000100578 2023-10-09T11:09:12.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.032 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.032 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:12.032 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.333554+0000", 2023-10-09T11:09:12.032 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003188897 2023-10-09T11:09:12.032 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.032 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.032 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.032 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.337092+0000", 2023-10-09T11:09:12.032 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003537756 2023-10-09T11:09:12.032 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.032 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338698+0000", 2023-10-09T11:09:12.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001606648 2023-10-09T11:09:12.033 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:12.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338750+0000", 2023-10-09T11:09:12.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1326999999999998e-05 2023-10-09T11:09:12.033 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:12.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:05:35.338780+0000", 2023-10-09T11:09:12.034 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0420999999999999e-05 2023-10-09T11:09:12.034 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.034 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:12.034 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.034 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.034 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.034 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.46997.0:7 2.1 2:97f8e5dc:::rbd_header.b79535dab9c8:head [create,call rbd.create in=9b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e581)", 2023-10-09T11:09:12.034 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:09:11.763177+0000", 2023-10-09T11:09:12.034 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.13619809899999999, 2023-10-09T11:09:12.034 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001483381, 2023-10-09T11:09:12.035 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:12.035 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:12.035 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:12.035 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.46997", 2023-10-09T11:09:12.035 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2067202369", 2023-10-09T11:09:12.035 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 7 2023-10-09T11:09:12.035 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.035 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:12.035 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.035 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:12.035 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.763177+0000", 2023-10-09T11:09:12.035 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.036 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:12.036 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.763177+0000", 2023-10-09T11:09:12.036 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.036 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:12.036 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.763175+0000", 2023-10-09T11:09:12.036 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T11:09:12.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.037 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.037 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:12.037 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.763182+0000", 2023-10-09T11:09:12.037 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4680000000000003e-06 2023-10-09T11:09:12.037 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.037 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.037 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:12.037 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.763195+0000", 2023-10-09T11:09:12.037 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2435e-05 2023-10-09T11:09:12.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:12.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.763204+0000", 2023-10-09T11:09:12.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5149999999999995e-06 2023-10-09T11:09:12.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:12.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.763277+0000", 2023-10-09T11:09:12.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2807999999999998e-05 2023-10-09T11:09:12.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.039 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:12.039 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.763436+0000", 2023-10-09T11:09:12.039 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015907000000000001 2023-10-09T11:09:12.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.039 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:09:12.039 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.763689+0000", 2023-10-09T11:09:12.039 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025265299999999998 2023-10-09T11:09:12.040 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.040 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.040 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:12.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.764346+0000", 2023-10-09T11:09:12.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00065752599999999997 2023-10-09T11:09:12.040 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.040 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.040 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.764575+0000", 2023-10-09T11:09:12.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000229232 2023-10-09T11:09:12.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.764602+0000", 2023-10-09T11:09:12.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6631e-05 2023-10-09T11:09:12.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:12.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.764632+0000", 2023-10-09T11:09:12.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0196e-05 2023-10-09T11:09:12.042 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.042 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.042 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:12.042 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.764660+0000", 2023-10-09T11:09:12.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7674e-05 2023-10-09T11:09:12.042 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.042 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:12.042 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.042 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.043 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.043 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.46997.0:27 2.1 2:97f8e5dc:::rbd_header.b79535dab9c8:head [notify-ack] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e581)", 2023-10-09T11:09:12.043 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:09:11.778032+0000", 2023-10-09T11:09:12.043 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.121342424, 2023-10-09T11:09:12.043 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012813620000000001, 2023-10-09T11:09:12.043 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:12.044 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T11:09:12.044 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:12.044 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.46997", 2023-10-09T11:09:12.044 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2067202369", 2023-10-09T11:09:12.044 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 27 2023-10-09T11:09:12.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.044 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:12.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:12.044 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.778032+0000", 2023-10-09T11:09:12.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.045 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.045 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:12.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.778032+0000", 2023-10-09T11:09:12.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.045 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.046 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.046 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:12.046 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.778032+0000", 2023-10-09T11:09:12.046 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:12.046 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.046 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.046 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:12.046 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.778035+0000", 2023-10-09T11:09:12.046 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7560000000000001e-06 2023-10-09T11:09:12.047 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.047 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:12.047 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.778041+0000", 2023-10-09T11:09:12.047 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5670000000000002e-06 2023-10-09T11:09:12.047 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.048 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:12.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.778045+0000", 2023-10-09T11:09:12.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2189999999999998e-06 2023-10-09T11:09:12.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.048 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:12.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.778579+0000", 2023-10-09T11:09:12.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00053404500000000003 2023-10-09T11:09:12.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for rw locks", 2023-10-09T11:09:12.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.778644+0000", 2023-10-09T11:09:12.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5198999999999997e-05 2023-10-09T11:09:12.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:12.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.779111+0000", 2023-10-09T11:09:12.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00046654800000000001 2023-10-09T11:09:12.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:12.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.779191+0000", 2023-10-09T11:09:12.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0341999999999998e-05 2023-10-09T11:09:12.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:12.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.779314+0000", 2023-10-09T11:09:12.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012229999999999999 2023-10-09T11:09:12.050 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.050 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:12.050 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.051 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.56866.0:13 2.1 2:97f8e5dc:::rbd_header.b79535dab9c8:head [watch watch cookie 140313159020960] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e581)", 2023-10-09T11:09:12.051 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:09:11.871838+0000", 2023-10-09T11:09:12.051 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.027537104, 2023-10-09T11:09:12.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016745200000000001, 2023-10-09T11:09:12.051 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:12.051 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:12.051 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:12.052 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.56866", 2023-10-09T11:09:12.052 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/979354288", 2023-10-09T11:09:12.052 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 13 2023-10-09T11:09:12.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.052 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:12.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.053 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:12.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.871838+0000", 2023-10-09T11:09:12.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.053 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.053 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:12.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.871838+0000", 2023-10-09T11:09:12.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.053 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.053 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:12.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.871837+0000", 2023-10-09T11:09:12.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:12.054 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.054 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.054 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:12.054 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.871841+0000", 2023-10-09T11:09:12.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.754e-06 2023-10-09T11:09:12.054 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.054 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.054 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:12.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.871850+0000", 2023-10-09T11:09:12.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6800000000000005e-06 2023-10-09T11:09:12.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:12.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.871855+0000", 2023-10-09T11:09:12.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9660000000000004e-06 2023-10-09T11:09:12.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:12.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.871878+0000", 2023-10-09T11:09:12.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2424000000000001e-05 2023-10-09T11:09:12.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.056 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:12.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.871924+0000", 2023-10-09T11:09:12.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6285999999999997e-05 2023-10-09T11:09:12.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:09:12.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.872031+0000", 2023-10-09T11:09:12.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010657100000000001 2023-10-09T11:09:12.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:12.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.872523+0000", 2023-10-09T11:09:12.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00049217300000000005 2023-10-09T11:09:12.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.872780+0000", 2023-10-09T11:09:12.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025736599999999997 2023-10-09T11:09:12.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.059 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.873381+0000", 2023-10-09T11:09:12.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00060065699999999999 2023-10-09T11:09:12.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.059 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.059 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:12.059 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.873434+0000", 2023-10-09T11:09:12.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3708999999999997e-05 2023-10-09T11:09:12.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.059 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.059 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:12.059 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.873512+0000", 2023-10-09T11:09:12.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7569000000000002e-05 2023-10-09T11:09:12.060 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.060 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:12.060 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.060 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.56866.0:15 2.1 2:97f8e5dc:::rbd_header.b79535dab9c8:head [call lock.lock in=8b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e581)", 2023-10-09T11:09:12.060 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:09:11.874314+0000", 2023-10-09T11:09:12.060 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.025060598, 2023-10-09T11:09:12.060 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002160262, 2023-10-09T11:09:12.060 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:12.060 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:12.060 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:12.061 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.56866", 2023-10-09T11:09:12.061 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/979354288", 2023-10-09T11:09:12.061 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15 2023-10-09T11:09:12.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.061 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:12.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:12.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.874314+0000", 2023-10-09T11:09:12.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:12.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.874314+0000", 2023-10-09T11:09:12.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:12.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.874313+0000", 2023-10-09T11:09:12.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T11:09:12.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:12.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.874318+0000", 2023-10-09T11:09:12.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8670000000000002e-06 2023-10-09T11:09:12.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:12.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.874328+0000", 2023-10-09T11:09:12.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4679999999999998e-06 2023-10-09T11:09:12.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:12.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.874335+0000", 2023-10-09T11:09:12.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9240000000000001e-06 2023-10-09T11:09:12.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:12.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.874378+0000", 2023-10-09T11:09:12.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3727999999999999e-05 2023-10-09T11:09:12.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.064 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:12.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.874468+0000", 2023-10-09T11:09:12.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9713e-05 2023-10-09T11:09:12.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.064 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:09:12.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.874676+0000", 2023-10-09T11:09:12.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020838000000000001 2023-10-09T11:09:12.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:12.065 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.875174+0000", 2023-10-09T11:09:12.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00049723200000000001 2023-10-09T11:09:12.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.065 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.875593+0000", 2023-10-09T11:09:12.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00041951199999999998 2023-10-09T11:09:12.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.065 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.876376+0000", 2023-10-09T11:09:12.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00078334900000000002 2023-10-09T11:09:12.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:12.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.876433+0000", 2023-10-09T11:09:12.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6127e-05 2023-10-09T11:09:12.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:12.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.876474+0000", 2023-10-09T11:09:12.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1684999999999998e-05 2023-10-09T11:09:12.066 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.067 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:12.067 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.067 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.56866.0:26 2.1 2:97f8e5dc:::rbd_header.b79535dab9c8:head [watch unwatch cookie 140313159020960] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e581)", 2023-10-09T11:09:12.067 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T11:09:11.883486+0000", 2023-10-09T11:09:12.069 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.015888954, 2023-10-09T11:09:12.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014886120000000001, 2023-10-09T11:09:12.070 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T11:09:12.070 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:12.070 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T11:09:12.070 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.56866", 2023-10-09T11:09:12.070 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/979354288", 2023-10-09T11:09:12.070 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 26 2023-10-09T11:09:12.070 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.070 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T11:09:12.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.070 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T11:09:12.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.883486+0000", 2023-10-09T11:09:12.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.071 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T11:09:12.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.883486+0000", 2023-10-09T11:09:12.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T11:09:12.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.071 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T11:09:12.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.883485+0000", 2023-10-09T11:09:12.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:12.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T11:09:12.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.883487+0000", 2023-10-09T11:09:12.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9549999999999999e-06 2023-10-09T11:09:12.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T11:09:12.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.883491+0000", 2023-10-09T11:09:12.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7670000000000001e-06 2023-10-09T11:09:12.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T11:09:12.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.883494+0000", 2023-10-09T11:09:12.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7489999999999999e-06 2023-10-09T11:09:12.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T11:09:12.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.883760+0000", 2023-10-09T11:09:12.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00026649699999999998 2023-10-09T11:09:12.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T11:09:12.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.883807+0000", 2023-10-09T11:09:12.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7114000000000003e-05 2023-10-09T11:09:12.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,3", 2023-10-09T11:09:12.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.883891+0000", 2023-10-09T11:09:12.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3996e-05 2023-10-09T11:09:12.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T11:09:12.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.884242+0000", 2023-10-09T11:09:12.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035098699999999998 2023-10-09T11:09:12.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.075 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.884740+0000", 2023-10-09T11:09:12.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000497982 2023-10-09T11:09:12.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.075 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.075 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:12.076 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.884829+0000", 2023-10-09T11:09:12.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8975999999999996e-05 2023-10-09T11:09:12.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.076 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T11:09:12.076 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.884881+0000", 2023-10-09T11:09:12.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1666999999999998e-05 2023-10-09T11:09:12.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T11:09:12.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T11:09:12.076 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T11:09:12.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T11:09:11.884974+0000", 2023-10-09T11:09:12.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3257999999999994e-05 2023-10-09T11:09:12.077 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.077 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:12.077 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.077 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T11:09:12.077 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T11:09:12.077 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T11:09:12.078 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:09:12.079 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T11:09:12.079 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg 2023-10-09T11:09:12.079 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T11:09:12.080 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_475522/img testimg 2023-10-09T11:09:12.084 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T11:09:12.089 INFO:tasks.workunit.client.0.smithi125.stderr:++ rbd info testimg 2023-10-09T11:09:12.089 INFO:tasks.workunit.client.0.smithi125.stderr:++ grep features 2023-10-09T11:09:12.105 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:12.123 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:09:12.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:12.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:09:12.127 INFO:tasks.workunit.client.0.smithi125.stderr:+ FEATURES_BEFORE=' features: layering 2023-10-09T11:09:12.127 INFO:tasks.workunit.client.0.smithi125.stderr: op_features: ' 2023-10-09T11:09:12.128 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_475522/img_v2 2023-10-09T11:09:12.165 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T11:09:12.168 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_475522/img_v2 testimg_import 2023-10-09T11:09:12.217 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T11:09:12.221 INFO:tasks.workunit.client.0.smithi125.stderr:++ rbd info testimg_import 2023-10-09T11:09:12.221 INFO:tasks.workunit.client.0.smithi125.stderr:++ grep features 2023-10-09T11:09:12.227 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:09:12.254 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:12.257 INFO:tasks.workunit.client.0.smithi125.stderr:+ FEATURES_AFTER=' features: layering 2023-10-09T11:09:12.258 INFO:tasks.workunit.client.0.smithi125.stderr: op_features: ' 2023-10-09T11:09:12.258 INFO:tasks.workunit.client.0.smithi125.stderr:+ '[' ' features: layering 2023-10-09T11:09:12.258 INFO:tasks.workunit.client.0.smithi125.stderr: op_features: ' '!=' ' features: layering 2023-10-09T11:09:12.258 INFO:tasks.workunit.client.0.smithi125.stderr: op_features: ' ']' 2023-10-09T11:09:12.258 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/img_v2 2023-10-09T11:09:12.258 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg_import 2023-10-09T11:09:12.272 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:09:12.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:12.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:09:12.325 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T11:09:12.328 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg 2023-10-09T11:09:12.343 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T11:09:12.376 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:09:12.394 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T11:09:12.397 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_475522/img testimg 2023-10-09T11:09:12.413 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:12.432 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:09:12.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:12.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:09:12.458 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T11:09:12.461 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_475522/img_v2 2023-10-09T11:09:12.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:12 smithi125 ceph-mon[200197]: failed to encode map e580 with expected crc 2023-10-09T11:09:12.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:12 smithi125 ceph-mon[200197]: failed to encode map e581 with expected crc 2023-10-09T11:09:12.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:12 smithi125 ceph-mon[200197]: failed to encode map e581 with expected crc 2023-10-09T11:09:12.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:12 smithi125 ceph-mon[200197]: osdmap e581: 8 total, 8 up, 8 in 2023-10-09T11:09:12.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:12 smithi125 ceph-mon[200197]: failed to encode map e581 with expected crc 2023-10-09T11:09:12.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:12 smithi125 ceph-mon[200197]: pgmap v1421: 103 pgs: 103 active+clean; 692 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 242 KiB/s rd, 77 KiB/s wr, 142 op/s 2023-10-09T11:09:12.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:12 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/213246377' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:09:12.499 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T11:09:12.502 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_475522/img_v2 testimg_import 2023-10-09T11:09:12.514 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T11:09:12.537 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:09:12.568 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:12.583 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T11:09:12.587 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import 2023-10-09T11:09:12.587 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep 'stripe unit' 2023-10-09T11:09:12.587 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep -Ei '(4 KiB|4096)' 2023-10-09T11:09:12.590 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:09:12.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:12.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:09:12.622 INFO:tasks.workunit.client.0.smithi125.stdout: stripe unit: 4 KiB 2023-10-09T11:09:12.623 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import 2023-10-09T11:09:12.623 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep 'stripe count' 2023-10-09T11:09:12.623 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk '{print $3}' 2023-10-09T11:09:12.623 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep 1000 2023-10-09T11:09:12.630 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:09:12.659 INFO:tasks.workunit.client.0.smithi125.stdout:1000 2023-10-09T11:09:12.659 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/img_v2 2023-10-09T11:09:12.660 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg_import 2023-10-09T11:09:12.689 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:09:12.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:09:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:12 smithi155 ceph-mon[184854]: failed to encode map e580 with expected crc 2023-10-09T11:09:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:12 smithi155 ceph-mon[184854]: failed to encode map e581 with expected crc 2023-10-09T11:09:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:12 smithi155 ceph-mon[184854]: failed to encode map e581 with expected crc 2023-10-09T11:09:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:12 smithi155 ceph-mon[184854]: osdmap e581: 8 total, 8 up, 8 in 2023-10-09T11:09:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:12 smithi155 ceph-mon[184854]: failed to encode map e581 with expected crc 2023-10-09T11:09:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:12 smithi155 ceph-mon[184854]: pgmap v1421: 103 pgs: 103 active+clean; 692 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 242 KiB/s rd, 77 KiB/s wr, 142 op/s 2023-10-09T11:09:12.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:12 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/213246377' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:09:12.745 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T11:09:12.748 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg 2023-10-09T11:09:12.759 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:12 smithi125 ceph-mon[195507]: failed to encode map e580 with expected crc 2023-10-09T11:09:12.760 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:12 smithi125 ceph-mon[195507]: failed to encode map e581 with expected crc 2023-10-09T11:09:12.760 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:12 smithi125 ceph-mon[195507]: failed to encode map e581 with expected crc 2023-10-09T11:09:12.760 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:12 smithi125 ceph-mon[195507]: osdmap e581: 8 total, 8 up, 8 in 2023-10-09T11:09:12.760 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:12 smithi125 ceph-mon[195507]: failed to encode map e581 with expected crc 2023-10-09T11:09:12.760 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:12 smithi125 ceph-mon[195507]: pgmap v1421: 103 pgs: 103 active+clean; 692 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 242 KiB/s rd, 77 KiB/s wr, 142 op/s 2023-10-09T11:09:12.760 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:12 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/213246377' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T11:09:12.763 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:12.838 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:09:12.840 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T11:09:12.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:12.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:09:12.844 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_475522/img testimg 2023-10-09T11:09:12.901 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T11:09:12.904 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap create testimg@snap1 2023-10-09T11:09:12.942 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:09:12.970 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:12.990 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:09:12.990 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:12.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:12.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:09:13.094 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:13.123 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:13.142 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:13.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:13.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:13.246 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:09:13.274 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:13.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:09:13.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:13.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:09:13.424 INFO:tasks.workunit.client.0.smithi125.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T11:09:13.431 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap create testimg@snap2 2023-10-09T11:09:13.447 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:09:13.476 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:13.546 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:09:13.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:13.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:09:13.650 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:09:13.683 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:13.702 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:09:13.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:13.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:09:13.728 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:09:13.729 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:09:13.729 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T11:09:13.729 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:09:13.735 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[195507]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:09:13.735 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[195507]: failed to encode map e582 with expected crc 2023-10-09T11:09:13.735 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/213246377' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:09:13.735 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[195507]: osdmap e582: 8 total, 8 up, 8 in 2023-10-09T11:09:13.735 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[195507]: failed to encode map e582 with expected crc 2023-10-09T11:09:13.735 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[195507]: failed to encode map e582 with expected crc 2023-10-09T11:09:13.735 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[195507]: failed to encode map e582 with expected crc 2023-10-09T11:09:13.735 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[195507]: osdmap e583: 8 total, 8 up, 8 in 2023-10-09T11:09:13.735 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[200197]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:09:13.736 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[200197]: failed to encode map e582 with expected crc 2023-10-09T11:09:13.736 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/213246377' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:09:13.736 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[200197]: osdmap e582: 8 total, 8 up, 8 in 2023-10-09T11:09:13.736 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[200197]: failed to encode map e582 with expected crc 2023-10-09T11:09:13.736 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[200197]: failed to encode map e582 with expected crc 2023-10-09T11:09:13.736 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[200197]: failed to encode map e582 with expected crc 2023-10-09T11:09:13.736 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:13 smithi125 ceph-mon[200197]: osdmap e583: 8 total, 8 up, 8 in 2023-10-09T11:09:13.755 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:13 smithi155 ceph-mon[184854]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:09:13.755 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:13 smithi155 ceph-mon[184854]: failed to encode map e582 with expected crc 2023-10-09T11:09:13.755 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:13 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/213246377' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T11:09:13.755 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:13 smithi155 ceph-mon[184854]: osdmap e582: 8 total, 8 up, 8 in 2023-10-09T11:09:13.755 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:13 smithi155 ceph-mon[184854]: failed to encode map e582 with expected crc 2023-10-09T11:09:13.755 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:13 smithi155 ceph-mon[184854]: failed to encode map e582 with expected crc 2023-10-09T11:09:13.755 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:13 smithi155 ceph-mon[184854]: failed to encode map e582 with expected crc 2023-10-09T11:09:13.755 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:13 smithi155 ceph-mon[184854]: osdmap e583: 8 total, 8 up, 8 in 2023-10-09T11:09:13.807 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:09:13.834 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:13.854 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:09:13.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:13.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:09:13.929 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:09:13.948 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:09:13.960 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:13.967 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:09:13.986 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:09:13.990 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:14.006 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:09:14.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:14.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:14.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:14.024 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:09:14.043 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:09:14.061 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:09:14.062 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:09:14.062 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:09:14.062 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:09:14.062 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:09:14.062 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:09:14.062 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:09:14.063 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:09:14.063 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:09:14.100 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T11:09:14.115 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:09:14.170 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:14.189 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:09:14.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:14.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:09:14.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:09:14.321 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:14.342 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:09:14.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:14.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:09:14.368 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:09:14.428 INFO:tasks.workunit.client.0.smithi125.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T11:09:14.435 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap protect testimg@snap2 2023-10-09T11:09:14.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:09:14.476 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:14.477 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_475522/snap_protect 2023-10-09T11:09:14.496 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:09:14.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:14.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:09:14.520 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T11:09:14.523 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_475522/snap_protect testimg_import 2023-10-09T11:09:14.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:14 smithi125 ceph-mon[195507]: failed to encode map e583 with expected crc 2023-10-09T11:09:14.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:14 smithi125 ceph-mon[195507]: failed to encode map e583 with expected crc 2023-10-09T11:09:14.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:14 smithi125 ceph-mon[195507]: failed to encode map e583 with expected crc 2023-10-09T11:09:14.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:14 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2161556236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:09:14.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:14 smithi125 ceph-mon[195507]: pgmap v1424: 103 pgs: 103 active+clean; 692 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 302 KiB/s rd, 30 KiB/s wr, 398 op/s 2023-10-09T11:09:14.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:14 smithi125 ceph-mon[195507]: failed to encode map e584 with expected crc 2023-10-09T11:09:14.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:14 smithi125 ceph-mon[195507]: osdmap e584: 8 total, 8 up, 8 in 2023-10-09T11:09:14.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:14 smithi125 ceph-mon[200197]: failed to encode map e583 with expected crc 2023-10-09T11:09:14.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:14 smithi125 ceph-mon[200197]: failed to encode map e583 with expected crc 2023-10-09T11:09:14.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:14 smithi125 ceph-mon[200197]: failed to encode map e583 with expected crc 2023-10-09T11:09:14.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:14 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2161556236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:09:14.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:14 smithi125 ceph-mon[200197]: pgmap v1424: 103 pgs: 103 active+clean; 692 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 302 KiB/s rd, 30 KiB/s wr, 398 op/s 2023-10-09T11:09:14.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:14 smithi125 ceph-mon[200197]: failed to encode map e584 with expected crc 2023-10-09T11:09:14.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:14 smithi125 ceph-mon[200197]: osdmap e584: 8 total, 8 up, 8 in 2023-10-09T11:09:14.601 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:09:14.628 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:14.646 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:09:14.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:14.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:09:14.661 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:14 smithi155 ceph-mon[184854]: failed to encode map e583 with expected crc 2023-10-09T11:09:14.661 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:14 smithi155 ceph-mon[184854]: failed to encode map e583 with expected crc 2023-10-09T11:09:14.661 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:14 smithi155 ceph-mon[184854]: failed to encode map e583 with expected crc 2023-10-09T11:09:14.661 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:14 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2161556236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:09:14.662 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:14 smithi155 ceph-mon[184854]: pgmap v1424: 103 pgs: 103 active+clean; 692 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 302 KiB/s rd, 30 KiB/s wr, 398 op/s 2023-10-09T11:09:14.662 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:14 smithi155 ceph-mon[184854]: failed to encode map e584 with expected crc 2023-10-09T11:09:14.662 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:14 smithi155 ceph-mon[184854]: osdmap e584: 8 total, 8 up, 8 in 2023-10-09T11:09:14.750 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:09:14.780 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:14.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T11:09:14.950 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:09:14.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:14.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:09:15.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:09:15.083 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:15.104 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:09:15.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:15.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:09:15.208 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:09:15.215 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:15.238 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:15.287 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 2, 1, 0, 7, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 4, 5, 6, 7, 1, 3] 2023-10-09T11:09:15.287 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T11:09:15.306 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:09:15.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:15.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:09:15.359 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:15.359 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:15.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:15.438 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:15.456 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:15.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:15.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:15.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:09:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:15 smithi125 ceph-mon[195507]: failed to encode map e584 with expected crc 2023-10-09T11:09:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:15 smithi125 ceph-mon[195507]: failed to encode map e584 with expected crc 2023-10-09T11:09:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:15 smithi125 ceph-mon[195507]: failed to encode map e584 with expected crc 2023-10-09T11:09:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:09:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:15 smithi125 ceph-mon[200197]: failed to encode map e584 with expected crc 2023-10-09T11:09:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:15 smithi125 ceph-mon[200197]: failed to encode map e584 with expected crc 2023-10-09T11:09:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:15 smithi125 ceph-mon[200197]: failed to encode map e584 with expected crc 2023-10-09T11:09:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:09:15.590 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:15.660 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:09:15.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:15.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:09:15.732 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T11:09:15.757 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:15 smithi155 ceph-mon[184854]: failed to encode map e584 with expected crc 2023-10-09T11:09:15.757 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:15 smithi155 ceph-mon[184854]: failed to encode map e584 with expected crc 2023-10-09T11:09:15.757 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:15 smithi155 ceph-mon[184854]: failed to encode map e584 with expected crc 2023-10-09T11:09:15.757 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:09:15.763 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:09:15.791 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:15.860 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:09:15.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:15.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:09:15.871 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:15.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:09:15.991 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:15.994 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:09:16.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:09:16.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:16.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:09:16.115 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:16.142 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:16.400 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T11:09:16.548 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T11:09:16.553 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import@snap1 2023-10-09T11:09:16.553 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep 'protected: False' 2023-10-09T11:09:16.591 INFO:tasks.workunit.client.0.smithi125.stdout: protected: False 2023-10-09T11:09:16.591 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import@snap2 2023-10-09T11:09:16.591 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep 'protected: True' 2023-10-09T11:09:16.631 INFO:tasks.workunit.client.0.smithi125.stdout: protected: True 2023-10-09T11:09:16.632 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/snap_protect 2023-10-09T11:09:16.632 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap unprotect testimg@snap2 2023-10-09T11:09:16.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:16 smithi125 ceph-mon[200197]: failed to encode map e585 with expected crc 2023-10-09T11:09:16.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:16 smithi125 ceph-mon[200197]: failed to encode map e585 with expected crc 2023-10-09T11:09:16.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:16 smithi125 ceph-mon[200197]: osdmap e585: 8 total, 8 up, 8 in 2023-10-09T11:09:16.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:16 smithi125 ceph-mon[200197]: failed to encode map e585 with expected crc 2023-10-09T11:09:16.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:16 smithi125 ceph-mon[200197]: failed to encode map e585 with expected crc 2023-10-09T11:09:16.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:16 smithi125 ceph-mon[200197]: pgmap v1427: 103 pgs: 103 active+clean; 802 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 443 KiB/s rd, 53 KiB/s wr, 579 op/s 2023-10-09T11:09:16.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:16 smithi125 ceph-mon[195507]: failed to encode map e585 with expected crc 2023-10-09T11:09:16.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:16 smithi125 ceph-mon[195507]: failed to encode map e585 with expected crc 2023-10-09T11:09:16.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:16 smithi125 ceph-mon[195507]: osdmap e585: 8 total, 8 up, 8 in 2023-10-09T11:09:16.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:16 smithi125 ceph-mon[195507]: failed to encode map e585 with expected crc 2023-10-09T11:09:16.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:16 smithi125 ceph-mon[195507]: failed to encode map e585 with expected crc 2023-10-09T11:09:16.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:16 smithi125 ceph-mon[195507]: pgmap v1427: 103 pgs: 103 active+clean; 802 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 443 KiB/s rd, 53 KiB/s wr, 579 op/s 2023-10-09T11:09:16.662 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:16.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:16.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:16.677 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-09T11:09:16.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:16 smithi155 ceph-mon[184854]: failed to encode map e585 with expected crc 2023-10-09T11:09:16.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:16 smithi155 ceph-mon[184854]: failed to encode map e585 with expected crc 2023-10-09T11:09:16.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:16 smithi155 ceph-mon[184854]: osdmap e585: 8 total, 8 up, 8 in 2023-10-09T11:09:16.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:16 smithi155 ceph-mon[184854]: failed to encode map e585 with expected crc 2023-10-09T11:09:16.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:16 smithi155 ceph-mon[184854]: failed to encode map e585 with expected crc 2023-10-09T11:09:16.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:16 smithi155 ceph-mon[184854]: pgmap v1427: 103 pgs: 103 active+clean; 802 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 443 KiB/s rd, 53 KiB/s wr, 579 op/s 2023-10-09T11:09:16.733 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap purge testimg 2023-10-09T11:09:16.755 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:16.772 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:09:16.801 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:16.820 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:09:16.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:16.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:09:16.924 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:09:16.939 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:16.939 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:16.952 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:17.023 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:09:17.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:17.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:09:17.127 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:09:17.155 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:17.225 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:09:17.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:17.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:09:17.306 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T11:09:17.329 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:09:17.377 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:17.395 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:09:17.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:17.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:09:17.499 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:09:17.529 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:17.531 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T11:09:17.532 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T11:09:17.583 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:09:17.602 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:09:17.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:17.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:09:17.701 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:17 smithi155 ceph-mon[184854]: osdmap e586: 8 total, 8 up, 8 in 2023-10-09T11:09:17.701 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:17 smithi155 ceph-mon[184854]: failed to encode map e586 with expected crc 2023-10-09T11:09:17.701 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:17 smithi155 ceph-mon[184854]: failed to encode map e586 with expected crc 2023-10-09T11:09:17.701 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:17 smithi155 ceph-mon[184854]: failed to encode map e586 with expected crc 2023-10-09T11:09:17.701 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:17 smithi155 ceph-mon[184854]: failed to encode map e586 with expected crc 2023-10-09T11:09:17.702 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2536865482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:09:17.702 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:17 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:09:17.702 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:17 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2364851560' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:09:17.706 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:09:17.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[195507]: osdmap e586: 8 total, 8 up, 8 in 2023-10-09T11:09:17.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[195507]: failed to encode map e586 with expected crc 2023-10-09T11:09:17.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[195507]: failed to encode map e586 with expected crc 2023-10-09T11:09:17.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[195507]: failed to encode map e586 with expected crc 2023-10-09T11:09:17.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[195507]: failed to encode map e586 with expected crc 2023-10-09T11:09:17.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2536865482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:09:17.709 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:09:17.710 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2364851560' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:09:17.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[200197]: osdmap e586: 8 total, 8 up, 8 in 2023-10-09T11:09:17.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[200197]: failed to encode map e586 with expected crc 2023-10-09T11:09:17.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[200197]: failed to encode map e586 with expected crc 2023-10-09T11:09:17.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[200197]: failed to encode map e586 with expected crc 2023-10-09T11:09:17.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[200197]: failed to encode map e586 with expected crc 2023-10-09T11:09:17.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2536865482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:09:17.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:09:17.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:17 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2364851560' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:09:17.736 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:17.756 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:09:17.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:17.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:09:17.860 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:17.895 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:17.910 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:09:17.915 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:17.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:17.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:17.930 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:09:17.950 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:09:17.972 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:09:17.990 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:09:18.009 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:09:18.019 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:18.028 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:09:18.046 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:18.048 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:09:18.049 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T11:09:18.049 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T11:09:18.049 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T11:09:18.049 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T11:09:18.050 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T11:09:18.050 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T11:09:18.050 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T11:09:18.050 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T11:09:18.217 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:18.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:18.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:18.320 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:09:18.347 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:18.419 INFO:tasks.workunit.client.0.smithi125.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T11:09:18.426 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap purge testimg_import 2023-10-09T11:09:18.466 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:09:18.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:18.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:09:18.541 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T11:09:18.551 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:09:18.551 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:09:18.551 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T11:09:18.551 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:09:18.572 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:09:18.603 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:18.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:18 smithi155 ceph-mon[184854]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:09:18.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:18 smithi155 ceph-mon[184854]: failed to encode map e587 with expected crc 2023-10-09T11:09:18.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:18 smithi155 ceph-mon[184854]: failed to encode map e587 with expected crc 2023-10-09T11:09:18.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:18 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:09:18.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2364851560' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T11:09:18.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:18 smithi155 ceph-mon[184854]: osdmap e587: 8 total, 8 up, 8 in 2023-10-09T11:09:18.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:18 smithi155 ceph-mon[184854]: failed to encode map e587 with expected crc 2023-10-09T11:09:18.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:18 smithi155 ceph-mon[184854]: failed to encode map e587 with expected crc 2023-10-09T11:09:18.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/134318398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:09:18.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:18 smithi155 ceph-mon[184854]: pgmap v1430: 103 pgs: 103 active+clean; 802 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 207 KiB/s rd, 32 KiB/s wr, 279 op/s 2023-10-09T11:09:18.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:18 smithi155 ceph-mon[184854]: osdmap e588: 8 total, 8 up, 8 in 2023-10-09T11:09:18.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:18 smithi155 ceph-mon[184854]: failed to encode map e588 with expected crc 2023-10-09T11:09:18.723 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:09:18.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:18.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:09:18.802 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:18.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[195507]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:09:18.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[195507]: failed to encode map e587 with expected crc 2023-10-09T11:09:18.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[195507]: failed to encode map e587 with expected crc 2023-10-09T11:09:18.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:09:18.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2364851560' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T11:09:18.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[195507]: osdmap e587: 8 total, 8 up, 8 in 2023-10-09T11:09:18.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[195507]: failed to encode map e587 with expected crc 2023-10-09T11:09:18.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[195507]: failed to encode map e587 with expected crc 2023-10-09T11:09:18.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/134318398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:09:18.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[195507]: pgmap v1430: 103 pgs: 103 active+clean; 802 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 207 KiB/s rd, 32 KiB/s wr, 279 op/s 2023-10-09T11:09:18.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[195507]: osdmap e588: 8 total, 8 up, 8 in 2023-10-09T11:09:18.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[195507]: failed to encode map e588 with expected crc 2023-10-09T11:09:18.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[200197]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:09:18.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[200197]: failed to encode map e587 with expected crc 2023-10-09T11:09:18.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[200197]: failed to encode map e587 with expected crc 2023-10-09T11:09:18.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T11:09:18.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2364851560' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T11:09:18.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[200197]: osdmap e587: 8 total, 8 up, 8 in 2023-10-09T11:09:18.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[200197]: failed to encode map e587 with expected crc 2023-10-09T11:09:18.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[200197]: failed to encode map e587 with expected crc 2023-10-09T11:09:18.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/134318398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:09:18.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[200197]: pgmap v1430: 103 pgs: 103 active+clean; 802 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 207 KiB/s rd, 32 KiB/s wr, 279 op/s 2023-10-09T11:09:18.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[200197]: osdmap e588: 8 total, 8 up, 8 in 2023-10-09T11:09:18.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:18 smithi125 ceph-mon[200197]: failed to encode map e588 with expected crc 2023-10-09T11:09:18.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:09:18.854 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:18.924 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:09:18.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:18.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:09:18.979 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T11:09:19.028 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:09:19.055 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:19.074 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:09:19.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:19.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:09:19.178 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:19.207 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:19.248 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:09:19.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:19.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:19.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:19.581 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:09:19.606 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:19 smithi155 ceph-mon[184854]: failed to encode map e588 with expected crc 2023-10-09T11:09:19.606 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:19 smithi155 ceph-mon[184854]: failed to encode map e588 with expected crc 2023-10-09T11:09:19.607 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:19 smithi155 ceph-mon[184854]: failed to encode map e588 with expected crc 2023-10-09T11:09:19.607 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:19 smithi155 ceph-mon[184854]: failed to encode map e589 with expected crc 2023-10-09T11:09:19.607 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:19 smithi155 ceph-mon[184854]: osdmap e589: 8 total, 8 up, 8 in 2023-10-09T11:09:19.609 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:19.629 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:09:19.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:19.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:09:19.689 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:19 smithi125 ceph-mon[200197]: failed to encode map e588 with expected crc 2023-10-09T11:09:19.689 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:19 smithi125 ceph-mon[200197]: failed to encode map e588 with expected crc 2023-10-09T11:09:19.689 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:19 smithi125 ceph-mon[200197]: failed to encode map e588 with expected crc 2023-10-09T11:09:19.689 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:19 smithi125 ceph-mon[200197]: failed to encode map e589 with expected crc 2023-10-09T11:09:19.689 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:19 smithi125 ceph-mon[200197]: osdmap e589: 8 total, 8 up, 8 in 2023-10-09T11:09:19.690 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:19 smithi125 ceph-mon[195507]: failed to encode map e588 with expected crc 2023-10-09T11:09:19.690 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:19 smithi125 ceph-mon[195507]: failed to encode map e588 with expected crc 2023-10-09T11:09:19.690 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:19 smithi125 ceph-mon[195507]: failed to encode map e588 with expected crc 2023-10-09T11:09:19.690 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:19 smithi125 ceph-mon[195507]: failed to encode map e589 with expected crc 2023-10-09T11:09:19.690 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:19 smithi125 ceph-mon[195507]: osdmap e589: 8 total, 8 up, 8 in 2023-10-09T11:09:19.733 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:09:19.761 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:19.780 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:09:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:19.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:09:19.884 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:09:19.912 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:19.933 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:09:19.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:19.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:09:20.037 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:09:20.067 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:20.174 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:09:20.186 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:09:20.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:20.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:09:20.262 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:09:20.263 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T11:09:20.263 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T11:09:20.263 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T11:09:20.263 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:09:20.290 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:09:20.317 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:20.338 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:09:20.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:20.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:09:20.427 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T11:09:20.428 INFO:tasks.workunit.client.0.smithi125.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T11:09:20.439 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg 2023-10-09T11:09:20.442 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:09:20.458 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:20.472 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:20.491 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T11:09:20.493 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:09:20.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:20.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:09:20.554 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T11:09:20.558 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg_import 2023-10-09T11:09:20.601 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:09:20.632 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:20.642 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T11:09:20.670 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T11:09:20.674 INFO:tasks.workunit.client.0.smithi125.stderr:+ tiered=0 2023-10-09T11:09:20.674 INFO:tasks.workunit.client.0.smithi125.stderr:+ ceph osd dump 2023-10-09T11:09:20.674 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep '^pool' 2023-10-09T11:09:20.674 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep ''\''rbd'\''' 2023-10-09T11:09:20.674 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep tier 2023-10-09T11:09:20.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[195507]: failed to encode map e589 with expected crc 2023-10-09T11:09:20.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[195507]: failed to encode map e589 with expected crc 2023-10-09T11:09:20.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[195507]: failed to encode map e589 with expected crc 2023-10-09T11:09:20.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[195507]: pgmap v1433: 103 pgs: 103 active+clean; 802 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 148 KiB/s rd, 8.0 KiB/s wr, 195 op/s 2023-10-09T11:09:20.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2135975863' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:09:20.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2135975863' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:09:20.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[195507]: osdmap e590: 8 total, 8 up, 8 in 2023-10-09T11:09:20.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[195507]: failed to encode map e590 with expected crc 2023-10-09T11:09:20.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[200197]: failed to encode map e589 with expected crc 2023-10-09T11:09:20.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[200197]: failed to encode map e589 with expected crc 2023-10-09T11:09:20.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[200197]: failed to encode map e589 with expected crc 2023-10-09T11:09:20.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[200197]: pgmap v1433: 103 pgs: 103 active+clean; 802 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 148 KiB/s rd, 8.0 KiB/s wr, 195 op/s 2023-10-09T11:09:20.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2135975863' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:09:20.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2135975863' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:09:20.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[200197]: osdmap e590: 8 total, 8 up, 8 in 2023-10-09T11:09:20.823 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:20 smithi125 ceph-mon[200197]: failed to encode map e590 with expected crc 2023-10-09T11:09:20.906 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:09:20.907 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:09:20.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:20.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:09:20.921 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T11:09:20.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:20 smithi155 ceph-mon[184854]: failed to encode map e589 with expected crc 2023-10-09T11:09:20.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:20 smithi155 ceph-mon[184854]: failed to encode map e589 with expected crc 2023-10-09T11:09:20.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:20 smithi155 ceph-mon[184854]: failed to encode map e589 with expected crc 2023-10-09T11:09:20.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:20 smithi155 ceph-mon[184854]: pgmap v1433: 103 pgs: 103 active+clean; 802 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 148 KiB/s rd, 8.0 KiB/s wr, 195 op/s 2023-10-09T11:09:20.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:20 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2135975863' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:09:20.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:20 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2135975863' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:09:20.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:20 smithi155 ceph-mon[184854]: osdmap e590: 8 total, 8 up, 8 in 2023-10-09T11:09:20.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:20 smithi155 ceph-mon[184854]: failed to encode map e590 with expected crc 2023-10-09T11:09:21.009 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T11:09:21.038 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:21.059 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6" found: no such container 2023-10-09T11:09:21.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:21.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T11:09:21.163 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:09:21.193 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:21.364 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:21.366 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:09:21.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:21.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:09:21.419 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:21.429 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_475522/sparse1 2023-10-09T11:09:21.436 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records in 2023-10-09T11:09:21.436 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records out 2023-10-09T11:09:21.436 INFO:tasks.workunit.client.0.smithi125.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00524004 s, 200 MB/s 2023-10-09T11:09:21.436 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_475522/sparse2 2023-10-09T11:09:21.443 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records in 2023-10-09T11:09:21.443 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records out 2023-10-09T11:09:21.443 INFO:tasks.workunit.client.0.smithi125.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00640081 s, 164 MB/s 2023-10-09T11:09:21.444 INFO:tasks.workunit.client.0.smithi125.stderr:+ truncate /tmp/rbd_import_export_475522/sparse2 -s 2M 2023-10-09T11:09:21.453 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse1 2023-10-09T11:09:21.471 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T11:09:21.494 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 0% complete...failed. 2023-10-09T11:09:21.495 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T11:09:21.499 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:09:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:09:21] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.43.0" 2023-10-09T11:09:21.499 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T11:09:21.499 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_475522/sparse1 2023-10-09T11:09:21.500 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:21.573 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T11:09:21.573 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T11:09:21.573 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T11:09:21.577 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd ls -l 2023-10-09T11:09:21.577 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep sparse1 2023-10-09T11:09:21.577 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T11:09:21.614 INFO:tasks.workunit.client.0.smithi125.stdout:sparse1 2 MiB 2 2023-10-09T11:09:21.615 INFO:tasks.workunit.client.0.smithi125.stderr:++ objects sparse1 2023-10-09T11:09:21.615 INFO:tasks.workunit.client.0.smithi125.stderr:++ image=sparse1 2023-10-09T11:09:21.615 INFO:tasks.workunit.client.0.smithi125.stderr:+++ rbd info sparse1 2023-10-09T11:09:21.615 INFO:tasks.workunit.client.0.smithi125.stderr:+++ grep block_name_prefix 2023-10-09T11:09:21.616 INFO:tasks.workunit.client.0.smithi125.stderr:+++ awk '{print $NF;}' 2023-10-09T11:09:21.621 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0" found: no such container 2023-10-09T11:09:21.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:21.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T11:09:21.663 INFO:tasks.workunit.client.0.smithi125.stderr:++ prefix=rbd_data.deee174c2417 2023-10-09T11:09:21.663 INFO:tasks.workunit.client.0.smithi125.stderr:+++ grep rbd_data.deee174c2417 2023-10-09T11:09:21.664 INFO:tasks.workunit.client.0.smithi125.stderr:++++ get_image_data_pool sparse1 2023-10-09T11:09:21.664 INFO:tasks.workunit.client.0.smithi125.stderr:++++ image=sparse1 2023-10-09T11:09:21.664 INFO:tasks.workunit.client.0.smithi125.stderr:+++ sed -e 's/rbd_data.deee174c2417\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T11:09:21.664 INFO:tasks.workunit.client.0.smithi125.stderr:+++ sort -u 2023-10-09T11:09:21.665 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ rbd info sparse1 2023-10-09T11:09:21.665 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ grep 'data_pool: ' 2023-10-09T11:09:21.665 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T11:09:21.701 INFO:tasks.workunit.client.0.smithi125.stderr:++++ data_pool= 2023-10-09T11:09:21.701 INFO:tasks.workunit.client.0.smithi125.stderr:++++ '[' -z ']' 2023-10-09T11:09:21.701 INFO:tasks.workunit.client.0.smithi125.stderr:++++ data_pool=rbd 2023-10-09T11:09:21.701 INFO:tasks.workunit.client.0.smithi125.stderr:++++ echo rbd 2023-10-09T11:09:21.701 INFO:tasks.workunit.client.0.smithi125.stderr:+++ rados ls -p rbd 2023-10-09T11:09:21.712 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:21 smithi155 ceph-mon[184854]: failed to encode map e590 with expected crc 2023-10-09T11:09:21.712 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:21 smithi155 ceph-mon[184854]: failed to encode map e590 with expected crc 2023-10-09T11:09:21.712 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:21 smithi155 ceph-mon[184854]: failed to encode map e590 with expected crc 2023-10-09T11:09:21.712 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:21 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1649639813' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T11:09:21.712 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:21 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1649639813' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T11:09:21.730 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:21.743 INFO:tasks.workunit.client.0.smithi125.stderr:++ objects=1 2023-10-09T11:09:21.743 INFO:tasks.workunit.client.0.smithi125.stderr:++ echo 1 2023-10-09T11:09:21.744 INFO:tasks.workunit.client.0.smithi125.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T11:09:21.744 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export sparse1 /tmp/rbd_import_export_475522/sparse1.out 2023-10-09T11:09:21.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:21 smithi125 ceph-mon[195507]: failed to encode map e590 with expected crc 2023-10-09T11:09:21.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:21 smithi125 ceph-mon[195507]: failed to encode map e590 with expected crc 2023-10-09T11:09:21.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:21 smithi125 ceph-mon[195507]: failed to encode map e590 with expected crc 2023-10-09T11:09:21.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:21 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1649639813' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T11:09:21.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:21 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1649639813' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T11:09:21.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:21 smithi125 ceph-mon[200197]: failed to encode map e590 with expected crc 2023-10-09T11:09:21.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:21 smithi125 ceph-mon[200197]: failed to encode map e590 with expected crc 2023-10-09T11:09:21.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:21 smithi125 ceph-mon[200197]: failed to encode map e590 with expected crc 2023-10-09T11:09:21.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:21 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1649639813' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T11:09:21.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:21 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1649639813' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T11:09:21.763 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:21.784 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T11:09:21.788 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_475522/sparse1 /tmp/rbd_import_export_475522/sparse1.out 2023-10-09T11:09:21.788 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_475522/sparse1 /tmp/rbd_import_export_475522/sparse1.out 2023-10-09T11:09:21.790 INFO:tasks.workunit.client.0.smithi125.stderr:++ stat /tmp/rbd_import_export_475522/sparse1 --format %b 2023-10-09T11:09:21.791 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=2048 2023-10-09T11:09:21.791 INFO:tasks.workunit.client.0.smithi125.stderr:++ stat /tmp/rbd_import_export_475522/sparse1.out --format %b 2023-10-09T11:09:21.792 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=2048 2023-10-09T11:09:21.792 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T11:09:21.792 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T11:09:21.792 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T11:09:21.792 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/sparse1.out 2023-10-09T11:09:21.793 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse1 2023-10-09T11:09:21.888 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T11:09:21.891 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse2 2023-10-09T11:09:21.922 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T11:09:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T11:09:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T11:09:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T11:09:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:17842 10.9 10:9e537702:::benchmark_data_smithi125_402869_object17841:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T11:09:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:25.625511+0000", 2023-10-09T11:09:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 476.29419525499998, 2023-10-09T11:09:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0429926150000002, 2023-10-09T11:09:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:09:21.923 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:09:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 17842 2023-10-09T11:09:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625511+0000", 2023-10-09T11:09:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625511+0000", 2023-10-09T11:09:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625510+0000", 2023-10-09T11:09:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625561+0000", 2023-10-09T11:09:21.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1332999999999998e-05 2023-10-09T11:09:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625575+0000", 2023-10-09T11:09:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3494e-05 2023-10-09T11:09:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625581+0000", 2023-10-09T11:09:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1689999999999997e-06 2023-10-09T11:09:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625627+0000", 2023-10-09T11:09:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5561e-05 2023-10-09T11:09:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625730+0000", 2023-10-09T11:09:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103268 2023-10-09T11:09:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,7", 2023-10-09T11:09:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.625821+0000", 2023-10-09T11:09:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1324999999999997e-05 2023-10-09T11:09:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.626893+0000", 2023-10-09T11:09:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001071734 2023-10-09T11:09:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.627060+0000", 2023-10-09T11:09:21.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000166935 2023-10-09T11:09:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.630997+0000", 2023-10-09T11:09:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039370630000000002 2023-10-09T11:09:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.631007+0000", 2023-10-09T11:09:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0301e-05 2023-10-09T11:09:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:26.629171+0000", 2023-10-09T11:09:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.99816343500000004 2023-10-09T11:09:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:26.629184+0000", 2023-10-09T11:09:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3192e-05 2023-10-09T11:09:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.668416+0000", 2023-10-09T11:09:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.039231926 2023-10-09T11:09:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.668504+0000", 2023-10-09T11:09:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8197999999999998e-05 2023-10-09T11:09:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.931 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:17873 10.9 10:95cd1191:::benchmark_data_smithi125_402869_object17872:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T11:09:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:25.660643+0000", 2023-10-09T11:09:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 476.25906356000002, 2023-10-09T11:09:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0183232979999999, 2023-10-09T11:09:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:09:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 17873 2023-10-09T11:09:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:21.932 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660643+0000", 2023-10-09T11:09:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660643+0000", 2023-10-09T11:09:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:21.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660643+0000", 2023-10-09T11:09:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T11:09:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660712+0000", 2023-10-09T11:09:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9728999999999996e-05 2023-10-09T11:09:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660721+0000", 2023-10-09T11:09:21.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8689999999999993e-06 2023-10-09T11:09:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660727+0000", 2023-10-09T11:09:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4430000000000002e-06 2023-10-09T11:09:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660785+0000", 2023-10-09T11:09:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7958999999999999e-05 2023-10-09T11:09:21.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.660793+0000", 2023-10-09T11:09:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1980000000000001e-06 2023-10-09T11:09:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:26.629206+0000", 2023-10-09T11:09:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.96841274899999996 2023-10-09T11:09:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:26.629212+0000", 2023-10-09T11:09:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.81e-06 2023-10-09T11:09:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.668525+0000", 2023-10-09T11:09:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0393124840000001 2023-10-09T11:09:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.668619+0000", 2023-10-09T11:09:21.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3868000000000003e-05 2023-10-09T11:09:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,4", 2023-10-09T11:09:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.668736+0000", 2023-10-09T11:09:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000117102 2023-10-09T11:09:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.670900+0000", 2023-10-09T11:09:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021641379999999999 2023-10-09T11:09:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.671389+0000", 2023-10-09T11:09:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00048903200000000003 2023-10-09T11:09:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.678582+0000", 2023-10-09T11:09:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0071925959999999999 2023-10-09T11:09:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:21.939 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.678617+0000", 2023-10-09T11:09:21.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5842e-05 2023-10-09T11:09:21.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:21.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.678966+0000", 2023-10-09T11:09:21.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00034906200000000003 2023-10-09T11:09:21.940 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.940 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:21.940 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.940 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36123.0:17893 10.9 10:964b1fe0:::benchmark_data_smithi125_402869_object17892:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T11:09:21.941 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:01:25.752542+0000", 2023-10-09T11:09:21.941 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 476.16716461099998, 2023-10-09T11:09:21.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9269636349999999, 2023-10-09T11:09:21.941 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:21.941 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:21.941 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:21.941 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36123", 2023-10-09T11:09:21.941 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2797100415", 2023-10-09T11:09:21.941 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 17893 2023-10-09T11:09:21.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.941 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:21.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:21.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752542+0000", 2023-10-09T11:09:21.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:21.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752542+0000", 2023-10-09T11:09:21.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:21.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752541+0000", 2023-10-09T11:09:21.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:09:21.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:21.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752612+0000", 2023-10-09T11:09:21.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0902000000000007e-05 2023-10-09T11:09:21.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:21.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752626+0000", 2023-10-09T11:09:21.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3498e-05 2023-10-09T11:09:21.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.944 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:21.944 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752634+0000", 2023-10-09T11:09:21.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2910000000000007e-06 2023-10-09T11:09:21.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.944 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.944 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752687+0000", 2023-10-09T11:09:21.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2960000000000001e-05 2023-10-09T11:09:21.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.944 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.945 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:25.752695+0000", 2023-10-09T11:09:21.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.072e-06 2023-10-09T11:09:21.945 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.945 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.945 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.945 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:26.629232+0000", 2023-10-09T11:09:21.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.87653732900000003 2023-10-09T11:09:21.945 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.945 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.945 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.945 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:26.629239+0000", 2023-10-09T11:09:21.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4590000000000003e-06 2023-10-09T11:09:21.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.946 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.669565+0000", 2023-10-09T11:09:21.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.040326184 2023-10-09T11:09:21.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:21.946 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.669680+0000", 2023-10-09T11:09:21.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000114832 2023-10-09T11:09:21.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,4", 2023-10-09T11:09:21.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.669797+0000", 2023-10-09T11:09:21.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000116857 2023-10-09T11:09:21.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:21.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.670905+0000", 2023-10-09T11:09:21.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011081909999999999 2023-10-09T11:09:21.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:21.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.671446+0000", 2023-10-09T11:09:21.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00054133399999999998 2023-10-09T11:09:21.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:21.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.679019+0000", 2023-10-09T11:09:21.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0075722669999999997 2023-10-09T11:09:21.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:21.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.679076+0000", 2023-10-09T11:09:21.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7707999999999997e-05 2023-10-09T11:09:21.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:21.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:01:27.679506+0000", 2023-10-09T11:09:21.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00042955000000000001 2023-10-09T11:09:21.949 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.949 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:21.949 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.949 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:75122 11.15 11:ac8cd55c:::benchmark_data_smithi125_439543_object75121:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e516)", 2023-10-09T11:09:21.949 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:07.030637+0000", 2023-10-09T11:09:21.950 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 194.889069761, 2023-10-09T11:09:21.950 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0992256789999999, 2023-10-09T11:09:21.950 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:21.950 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:21.950 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:21.950 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:21.950 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:21.950 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 75122 2023-10-09T11:09:21.950 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.950 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:21.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.951 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:21.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.030637+0000", 2023-10-09T11:09:21.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.951 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.951 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.951 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:21.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.030637+0000", 2023-10-09T11:09:21.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.951 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.951 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.951 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:21.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.030636+0000", 2023-10-09T11:09:21.952 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:21.952 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.952 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.952 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:21.952 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.030690+0000", 2023-10-09T11:09:21.952 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3973e-05 2023-10-09T11:09:21.952 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.952 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.952 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:21.952 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.030701+0000", 2023-10-09T11:09:21.952 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1185999999999999e-05 2023-10-09T11:09:21.952 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.953 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:21.953 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.030706+0000", 2023-10-09T11:09:21.953 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9949999999999996e-06 2023-10-09T11:09:21.953 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.953 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.953 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.030743+0000", 2023-10-09T11:09:21.953 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7049999999999999e-05 2023-10-09T11:09:21.953 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.954 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.954 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.030750+0000", 2023-10-09T11:09:21.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5089999999999996e-06 2023-10-09T11:09:21.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.954 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.954 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.954 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:08.006545+0000", 2023-10-09T11:09:21.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.975794571 2023-10-09T11:09:21.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.954 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.954 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:08.006554+0000", 2023-10-09T11:09:21.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4649999999999994e-06 2023-10-09T11:09:21.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.088916+0000", 2023-10-09T11:09:21.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.082362907 2023-10-09T11:09:21.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:21.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.089075+0000", 2023-10-09T11:09:21.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015856900000000001 2023-10-09T11:09:21.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:21.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.089236+0000", 2023-10-09T11:09:21.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016059000000000001 2023-10-09T11:09:21.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:21.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.094578+0000", 2023-10-09T11:09:21.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.005342528 2023-10-09T11:09:21.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:21.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.128380+0000", 2023-10-09T11:09:21.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.033801497999999999 2023-10-09T11:09:21.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:21.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.128451+0000", 2023-10-09T11:09:21.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1355999999999999e-05 2023-10-09T11:09:21.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:21.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.128516+0000", 2023-10-09T11:09:21.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4654999999999997e-05 2023-10-09T11:09:21.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:21.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.129863+0000", 2023-10-09T11:09:21.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0013470509999999999 2023-10-09T11:09:21.958 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.958 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:21.958 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.959 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:75124 11.15 11:ac4b0df1:::benchmark_data_smithi125_439543_object75123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e516)", 2023-10-09T11:09:21.959 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:07.037312+0000", 2023-10-09T11:09:21.959 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 194.88239465500001, 2023-10-09T11:09:21.959 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0935586540000002, 2023-10-09T11:09:21.959 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:21.959 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:21.959 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:21.959 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:21.959 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:21.959 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 75124 2023-10-09T11:09:21.959 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.959 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:21.960 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.960 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:21.960 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.037312+0000", 2023-10-09T11:09:21.960 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.960 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.960 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:21.960 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.037312+0000", 2023-10-09T11:09:21.960 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.960 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.960 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:21.960 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.037311+0000", 2023-10-09T11:09:21.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:21.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.961 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:21.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.037371+0000", 2023-10-09T11:09:21.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0452000000000003e-05 2023-10-09T11:09:21.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.961 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:21.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.037387+0000", 2023-10-09T11:09:21.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6181999999999999e-05 2023-10-09T11:09:21.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:21.962 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.037394+0000", 2023-10-09T11:09:21.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6050000000000003e-06 2023-10-09T11:09:21.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.962 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.037450+0000", 2023-10-09T11:09:21.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6439e-05 2023-10-09T11:09:21.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.037461+0000", 2023-10-09T11:09:21.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0788000000000001e-05 2023-10-09T11:09:21.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:08.006575+0000", 2023-10-09T11:09:21.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.96911384899999997 2023-10-09T11:09:21.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:08.006582+0000", 2023-10-09T11:09:21.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6599999999999998e-06 2023-10-09T11:09:21.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.090779+0000", 2023-10-09T11:09:21.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0841977110000001 2023-10-09T11:09:21.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:21.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.090933+0000", 2023-10-09T11:09:21.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000153077 2023-10-09T11:09:21.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:21.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.091109+0000", 2023-10-09T11:09:21.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017667599999999999 2023-10-09T11:09:21.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:21.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.096528+0000", 2023-10-09T11:09:21.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0054191079999999997 2023-10-09T11:09:21.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:21.966 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.128398+0000", 2023-10-09T11:09:21.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.031869887999999999 2023-10-09T11:09:21.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.966 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.966 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:21.966 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.129904+0000", 2023-10-09T11:09:21.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001505929 2023-10-09T11:09:21.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.966 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.966 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:21.966 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.129959+0000", 2023-10-09T11:09:21.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4874999999999999e-05 2023-10-09T11:09:21.967 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.967 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:21.967 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.130871+0000", 2023-10-09T11:09:21.967 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00091170900000000002 2023-10-09T11:09:21.967 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.967 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:21.967 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.967 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.967 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:75153 11.5 11:a11920e6:::benchmark_data_smithi125_439543_object75152:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e516)", 2023-10-09T11:09:21.967 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:07.064999+0000", 2023-10-09T11:09:21.968 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 194.85470745699999, 2023-10-09T11:09:21.968 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0883036549999998, 2023-10-09T11:09:21.968 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:21.968 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:21.968 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:21.968 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:21.968 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:21.968 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 75153 2023-10-09T11:09:21.968 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.968 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:21.968 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.968 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:21.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.064999+0000", 2023-10-09T11:09:21.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.969 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:21.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.064999+0000", 2023-10-09T11:09:21.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.969 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:21.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.064998+0000", 2023-10-09T11:09:21.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:09:21.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:21.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.065049+0000", 2023-10-09T11:09:21.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0751000000000002e-05 2023-10-09T11:09:21.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:21.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.065060+0000", 2023-10-09T11:09:21.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1004e-05 2023-10-09T11:09:21.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:21.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.065065+0000", 2023-10-09T11:09:21.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.186e-06 2023-10-09T11:09:21.971 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.065109+0000", 2023-10-09T11:09:21.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4127e-05 2023-10-09T11:09:21.971 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.065116+0000", 2023-10-09T11:09:21.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7460000000000003e-06 2023-10-09T11:09:21.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.972 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.972 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:08.005008+0000", 2023-10-09T11:09:21.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93989228199999997 2023-10-09T11:09:21.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.972 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.972 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:08.005019+0000", 2023-10-09T11:09:21.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1121000000000001e-05 2023-10-09T11:09:21.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.125607+0000", 2023-10-09T11:09:21.973 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1205879940000001 2023-10-09T11:09:21.973 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:21.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.125731+0000", 2023-10-09T11:09:21.973 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000123786 2023-10-09T11:09:21.973 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:21.974 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.125882+0000", 2023-10-09T11:09:21.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015042500000000001 2023-10-09T11:09:21.974 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.974 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.974 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:21.974 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.150285+0000", 2023-10-09T11:09:21.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024403347999999998 2023-10-09T11:09:21.974 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.974 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.974 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:21.974 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.152048+0000", 2023-10-09T11:09:21.975 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0017630790000000001 2023-10-09T11:09:21.975 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.975 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.975 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:21.975 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.152129+0000", 2023-10-09T11:09:21.975 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1346000000000003e-05 2023-10-09T11:09:21.975 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.975 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.975 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:21.975 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.152168+0000", 2023-10-09T11:09:21.975 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8278999999999997e-05 2023-10-09T11:09:21.975 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:21.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.153303+0000", 2023-10-09T11:09:21.976 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001135346 2023-10-09T11:09:21.976 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.976 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:21.976 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.976 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:75165 11.5 11:a195c022:::benchmark_data_smithi125_439543_object75164:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e516)", 2023-10-09T11:09:21.976 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:07.077016+0000", 2023-10-09T11:09:21.976 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 194.84269067700001, 2023-10-09T11:09:21.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0770512339999998, 2023-10-09T11:09:21.977 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:21.977 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:21.977 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:21.977 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:21.977 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:21.977 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 75165 2023-10-09T11:09:21.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.977 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:21.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:21.977 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.077016+0000", 2023-10-09T11:09:21.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:21.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.077016+0000", 2023-10-09T11:09:21.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:21.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.077015+0000", 2023-10-09T11:09:21.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:21.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.979 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:21.979 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.077067+0000", 2023-10-09T11:09:21.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1996000000000002e-05 2023-10-09T11:09:21.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.979 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:21.979 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.077078+0000", 2023-10-09T11:09:21.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1833e-05 2023-10-09T11:09:21.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.980 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:21.980 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.077084+0000", 2023-10-09T11:09:21.980 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6659999999999996e-06 2023-10-09T11:09:21.980 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.980 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.980 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.980 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.077134+0000", 2023-10-09T11:09:21.980 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9793999999999998e-05 2023-10-09T11:09:21.980 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.980 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.980 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.980 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.077149+0000", 2023-10-09T11:09:21.981 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4691999999999999e-05 2023-10-09T11:09:21.981 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.981 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.981 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.981 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:08.007499+0000", 2023-10-09T11:09:21.981 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93035056699999996 2023-10-09T11:09:21.981 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.981 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.981 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.981 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:08.007508+0000", 2023-10-09T11:09:21.981 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2210000000000003e-06 2023-10-09T11:09:21.981 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.982 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.982 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.982 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.127100+0000", 2023-10-09T11:09:21.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1195914710000001 2023-10-09T11:09:21.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.982 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.982 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:21.982 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.127208+0000", 2023-10-09T11:09:21.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000108099 2023-10-09T11:09:21.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.982 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.982 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:21.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.127371+0000", 2023-10-09T11:09:21.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000163306 2023-10-09T11:09:21.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:21.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.150295+0000", 2023-10-09T11:09:21.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022923743 2023-10-09T11:09:21.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:21.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.152094+0000", 2023-10-09T11:09:21.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00179868 2023-10-09T11:09:21.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:21.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.153344+0000", 2023-10-09T11:09:21.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012506800000000001 2023-10-09T11:09:21.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:21.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.153377+0000", 2023-10-09T11:09:21.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2444999999999999e-05 2023-10-09T11:09:21.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.985 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:21.985 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.154067+0000", 2023-10-09T11:09:21.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00069052499999999999 2023-10-09T11:09:21.985 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.985 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:21.985 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.985 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:75168 11.15 11:af472354:::benchmark_data_smithi125_439543_object75167:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e516)", 2023-10-09T11:09:21.985 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:07.085933+0000", 2023-10-09T11:09:21.985 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 194.83377328099999, 2023-10-09T11:09:21.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0659854950000001, 2023-10-09T11:09:21.986 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:21.986 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:21.986 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:21.986 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:21.986 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:21.986 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 75168 2023-10-09T11:09:21.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.986 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:21.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:21.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.085933+0000", 2023-10-09T11:09:21.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:21.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.085933+0000", 2023-10-09T11:09:21.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.988 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:21.988 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.085932+0000", 2023-10-09T11:09:21.988 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:21.988 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.988 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:21.988 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.085989+0000", 2023-10-09T11:09:21.988 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6635000000000002e-05 2023-10-09T11:09:21.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:21.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.086002+0000", 2023-10-09T11:09:21.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2918999999999999e-05 2023-10-09T11:09:21.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:21.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.086007+0000", 2023-10-09T11:09:21.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1039999999999998e-06 2023-10-09T11:09:21.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.086048+0000", 2023-10-09T11:09:21.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1505999999999999e-05 2023-10-09T11:09:21.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:07.086055+0000", 2023-10-09T11:09:21.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2999999999999998e-06 2023-10-09T11:09:21.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:08.006601+0000", 2023-10-09T11:09:21.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92054682600000004 2023-10-09T11:09:21.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:21.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:08.006609+0000", 2023-10-09T11:09:21.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2049999999999996e-06 2023-10-09T11:09:21.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:21.992 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.092728+0000", 2023-10-09T11:09:21.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0861194359999999 2023-10-09T11:09:21.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:21.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.092890+0000", 2023-10-09T11:09:21.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000161912 2023-10-09T11:09:21.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:21.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.093064+0000", 2023-10-09T11:09:21.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017412600000000001 2023-10-09T11:09:21.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:21.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.120736+0000", 2023-10-09T11:09:21.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027672236999999999 2023-10-09T11:09:21.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:21.995 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.128422+0000", 2023-10-09T11:09:21.995 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0076854949999999997 2023-10-09T11:09:21.995 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.995 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.995 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:21.995 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.150347+0000", 2023-10-09T11:09:21.995 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021924842 2023-10-09T11:09:21.995 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.995 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.996 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:21.996 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.150414+0000", 2023-10-09T11:09:21.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7088999999999994e-05 2023-10-09T11:09:21.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.996 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:21.996 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:09.151919+0000", 2023-10-09T11:09:21.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015052290000000001 2023-10-09T11:09:21.996 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.997 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:21.997 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:21.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.997 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:91560 11.15 11:ad26c48c:::benchmark_data_smithi125_439543_object91559:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T11:09:21.997 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:29.339269+0000", 2023-10-09T11:09:21.997 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 172.580437969, 2023-10-09T11:09:21.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.050562719, 2023-10-09T11:09:21.998 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:21.998 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:09:21.998 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:21.998 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:21.998 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:21.998 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 91560 2023-10-09T11:09:21.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:21.999 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:21.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:21.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:21.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.339269+0000", 2023-10-09T11:09:21.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:21.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.000 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:22.000 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.339269+0000", 2023-10-09T11:09:22.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.000 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:22.000 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.339267+0000", 2023-10-09T11:09:22.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:22.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:22.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.339302+0000", 2023-10-09T11:09:22.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5250999999999998e-05 2023-10-09T11:09:22.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:22.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.339321+0000", 2023-10-09T11:09:22.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8519999999999999e-05 2023-10-09T11:09:22.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:22.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.339328+0000", 2023-10-09T11:09:22.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.764e-06 2023-10-09T11:09:22.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.339384+0000", 2023-10-09T11:09:22.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6439e-05 2023-10-09T11:09:22.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:22.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.339551+0000", 2023-10-09T11:09:22.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016660899999999999 2023-10-09T11:09:22.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:22.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.339721+0000", 2023-10-09T11:09:22.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017018400000000001 2023-10-09T11:09:22.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:22.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.343407+0000", 2023-10-09T11:09:22.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003685893 2023-10-09T11:09:22.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.347982+0000", 2023-10-09T11:09:22.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.004574809 2023-10-09T11:09:22.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:22.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.347991+0000", 2023-10-09T11:09:22.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0799999999999995e-06 2023-10-09T11:09:22.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.361117+0000", 2023-10-09T11:09:22.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0131260900000001 2023-10-09T11:09:22.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T11:09:22.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.361801+0000", 2023-10-09T11:09:22.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00068387199999999997 2023-10-09T11:09:22.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.389715+0000", 2023-10-09T11:09:22.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027914392 2023-10-09T11:09:22.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:22.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.389831+0000", 2023-10-09T11:09:22.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011646600000000001 2023-10-09T11:09:22.006 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.006 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:22.006 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.006 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:91561 11.15 11:aad1daa3:::benchmark_data_smithi125_439543_object91560:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T11:09:22.007 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:29.341334+0000", 2023-10-09T11:09:22.007 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 172.578372586, 2023-10-09T11:09:22.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0489422530000001, 2023-10-09T11:09:22.007 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:22.007 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T11:09:22.007 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:22.007 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:22.007 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:22.007 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 91561 2023-10-09T11:09:22.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.008 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:22.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:22.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.341334+0000", 2023-10-09T11:09:22.008 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.008 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:22.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.341334+0000", 2023-10-09T11:09:22.008 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.008 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.009 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:22.009 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.341333+0000", 2023-10-09T11:09:22.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:09:22.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.009 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.009 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:22.009 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.341364+0000", 2023-10-09T11:09:22.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0639000000000003e-05 2023-10-09T11:09:22.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.009 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.009 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:22.009 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.341374+0000", 2023-10-09T11:09:22.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0302999999999999e-05 2023-10-09T11:09:22.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.010 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:22.010 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.341380+0000", 2023-10-09T11:09:22.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8969999999999996e-06 2023-10-09T11:09:22.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.010 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.010 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.341489+0000", 2023-10-09T11:09:22.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000108639 2023-10-09T11:09:22.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:22.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.341641+0000", 2023-10-09T11:09:22.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015202799999999999 2023-10-09T11:09:22.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:22.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.341786+0000", 2023-10-09T11:09:22.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000145012 2023-10-09T11:09:22.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:22.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.346149+0000", 2023-10-09T11:09:22.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043635690000000003 2023-10-09T11:09:22.012 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.348021+0000", 2023-10-09T11:09:22.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018716850000000001 2023-10-09T11:09:22.012 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:22.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.348027+0000", 2023-10-09T11:09:22.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1850000000000001e-06 2023-10-09T11:09:22.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.361848+0000", 2023-10-09T11:09:22.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.013821203 2023-10-09T11:09:22.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for degraded object", 2023-10-09T11:09:22.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.362307+0000", 2023-10-09T11:09:22.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00045905499999999999 2023-10-09T11:09:22.014 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.390182+0000", 2023-10-09T11:09:22.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.02787485 2023-10-09T11:09:22.014 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:22.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.390276+0000", 2023-10-09T11:09:22.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4073999999999994e-05 2023-10-09T11:09:22.015 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.015 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:22.015 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.015 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.015 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.015 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:91600 11.15 11:ae3dbb56:::benchmark_data_smithi125_439543_object91599:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-10-09T11:09:22.015 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:29.390362+0000", 2023-10-09T11:09:22.015 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 172.529344732, 2023-10-09T11:09:22.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.000194802, 2023-10-09T11:09:22.015 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:22.015 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:22.015 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:22.016 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:22.016 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:22.016 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 91600 2023-10-09T11:09:22.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.016 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:22.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.016 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:22.016 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.390362+0000", 2023-10-09T11:09:22.016 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:22.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.390362+0000", 2023-10-09T11:09:22.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:22.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.390360+0000", 2023-10-09T11:09:22.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T11:09:22.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:22.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.390398+0000", 2023-10-09T11:09:22.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7546000000000003e-05 2023-10-09T11:09:22.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.018 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:22.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.390413+0000", 2023-10-09T11:09:22.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4729000000000001e-05 2023-10-09T11:09:22.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.018 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:22.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.390419+0000", 2023-10-09T11:09:22.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7769999999999997e-06 2023-10-09T11:09:22.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.390478+0000", 2023-10-09T11:09:22.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8779000000000001e-05 2023-10-09T11:09:22.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:22.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.390488+0000", 2023-10-09T11:09:22.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.008e-05 2023-10-09T11:09:22.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.020 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.020 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.362358+0000", 2023-10-09T11:09:22.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97186956700000005 2023-10-09T11:09:22.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.020 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:22.020 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.362506+0000", 2023-10-09T11:09:22.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000148355 2023-10-09T11:09:22.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.020 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:22.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.362689+0000", 2023-10-09T11:09:22.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018245800000000001 2023-10-09T11:09:22.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:22.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.369454+0000", 2023-10-09T11:09:22.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0067655969999999999 2023-10-09T11:09:22.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.390320+0000", 2023-10-09T11:09:22.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020865489000000001 2023-10-09T11:09:22.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.022 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.022 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.022 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.390389+0000", 2023-10-09T11:09:22.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9258999999999995e-05 2023-10-09T11:09:22.022 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.022 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.022 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:22.022 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.390443+0000", 2023-10-09T11:09:22.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4079000000000002e-05 2023-10-09T11:09:22.022 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.022 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.023 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:22.023 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.390557+0000", 2023-10-09T11:09:22.023 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000113798 2023-10-09T11:09:22.023 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.023 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:22.023 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.023 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.023 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.023 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:91690 11.15 11:af2bb28a:::benchmark_data_smithi125_439543_object91689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-10-09T11:09:22.023 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:29.498362+0000", 2023-10-09T11:09:22.023 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 172.42134436200001, 2023-10-09T11:09:22.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.8923875, 2023-10-09T11:09:22.024 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:22.024 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:22.024 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:22.024 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:22.024 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:22.024 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 91690 2023-10-09T11:09:22.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.025 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:22.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.025 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:22.025 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.498362+0000", 2023-10-09T11:09:22.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.025 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:22.025 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.498362+0000", 2023-10-09T11:09:22.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.026 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:22.026 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.498361+0000", 2023-10-09T11:09:22.026 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:22.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.027 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:22.027 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.498436+0000", 2023-10-09T11:09:22.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5573000000000003e-05 2023-10-09T11:09:22.027 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.027 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.027 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:22.027 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.498452+0000", 2023-10-09T11:09:22.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6027000000000001e-05 2023-10-09T11:09:22.027 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.027 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:22.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.498459+0000", 2023-10-09T11:09:22.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.409e-06 2023-10-09T11:09:22.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.498529+0000", 2023-10-09T11:09:22.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0233000000000003e-05 2023-10-09T11:09:22.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:22.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.498543+0000", 2023-10-09T11:09:22.029 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4290999999999999e-05 2023-10-09T11:09:22.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.029 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.364494+0000", 2023-10-09T11:09:22.029 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.86595035300000001 2023-10-09T11:09:22.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.030 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.030 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:22.030 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.364650+0000", 2023-10-09T11:09:22.030 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015589400000000001 2023-10-09T11:09:22.030 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.030 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.030 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:22.030 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.364838+0000", 2023-10-09T11:09:22.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018853699999999999 2023-10-09T11:09:22.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:22.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.371366+0000", 2023-10-09T11:09:22.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0065275530000000002 2023-10-09T11:09:22.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.390354+0000", 2023-10-09T11:09:22.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018988204000000002 2023-10-09T11:09:22.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.390597+0000", 2023-10-09T11:09:22.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00024327300000000001 2023-10-09T11:09:22.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:22.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.390652+0000", 2023-10-09T11:09:22.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4611999999999998e-05 2023-10-09T11:09:22.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:22.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.390750+0000", 2023-10-09T11:09:22.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8099999999999999e-05 2023-10-09T11:09:22.033 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.033 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:22.033 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.034 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:91718 11.15 11:ab9b964f:::benchmark_data_smithi125_439543_object91717:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-10-09T11:09:22.034 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:29.527960+0000", 2023-10-09T11:09:22.034 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 172.39174668300001, 2023-10-09T11:09:22.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.86304709599999996, 2023-10-09T11:09:22.034 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:22.035 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:22.035 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:22.035 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:22.035 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:22.035 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 91718 2023-10-09T11:09:22.035 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.035 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:22.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:22.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.527960+0000", 2023-10-09T11:09:22.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:22.037 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.527960+0000", 2023-10-09T11:09:22.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.037 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.037 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:22.037 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.527958+0000", 2023-10-09T11:09:22.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T11:09:22.037 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:22.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.527991+0000", 2023-10-09T11:09:22.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3149999999999999e-05 2023-10-09T11:09:22.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:22.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.528009+0000", 2023-10-09T11:09:22.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7139e-05 2023-10-09T11:09:22.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:22.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.528017+0000", 2023-10-09T11:09:22.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2789999999999998e-06 2023-10-09T11:09:22.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.528073+0000", 2023-10-09T11:09:22.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6616999999999997e-05 2023-10-09T11:09:22.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:22.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.528082+0000", 2023-10-09T11:09:22.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4670000000000004e-06 2023-10-09T11:09:22.040 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.040 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.040 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.040 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.366522+0000", 2023-10-09T11:09:22.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.83844000500000004 2023-10-09T11:09:22.040 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.040 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.040 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:22.040 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.366665+0000", 2023-10-09T11:09:22.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000143447 2023-10-09T11:09:22.040 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.041 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.041 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:22.041 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.366846+0000", 2023-10-09T11:09:22.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000180156 2023-10-09T11:09:22.041 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.041 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.041 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:22.041 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.371373+0000", 2023-10-09T11:09:22.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045271449999999998 2023-10-09T11:09:22.041 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.041 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.041 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.042 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.390793+0000", 2023-10-09T11:09:22.042 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019420014999999999 2023-10-09T11:09:22.042 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.042 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.390858+0000", 2023-10-09T11:09:22.042 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5543000000000006e-05 2023-10-09T11:09:22.042 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:22.042 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.390913+0000", 2023-10-09T11:09:22.042 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5086000000000001e-05 2023-10-09T11:09:22.043 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:22.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391007+0000", 2023-10-09T11:09:22.043 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3842e-05 2023-10-09T11:09:22.043 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.043 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:22.043 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.043 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.043 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:91745 11.15 11:ab4b4ae4:::benchmark_data_smithi125_439543_object91744:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-10-09T11:09:22.043 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:29.579981+0000", 2023-10-09T11:09:22.043 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 172.33972602200001, 2023-10-09T11:09:22.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.81130343999999999, 2023-10-09T11:09:22.044 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:22.044 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:22.044 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:22.044 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:22.044 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:22.044 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 91745 2023-10-09T11:09:22.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.044 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:22.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.044 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:22.044 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.579981+0000", 2023-10-09T11:09:22.045 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.045 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.045 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:22.045 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.579981+0000", 2023-10-09T11:09:22.045 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.045 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.045 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:22.045 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.579980+0000", 2023-10-09T11:09:22.045 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:09:22.046 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.046 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.046 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:22.046 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.580018+0000", 2023-10-09T11:09:22.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8609999999999998e-05 2023-10-09T11:09:22.046 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.046 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.046 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:22.046 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.580028+0000", 2023-10-09T11:09:22.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3589999999999997e-06 2023-10-09T11:09:22.046 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.046 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.047 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:22.047 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.580034+0000", 2023-10-09T11:09:22.047 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.998e-06 2023-10-09T11:09:22.047 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.047 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.047 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.047 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.580576+0000", 2023-10-09T11:09:22.047 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00054218599999999995 2023-10-09T11:09:22.047 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.047 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.047 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:22.047 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.580582+0000", 2023-10-09T11:09:22.048 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2369999999999997e-06 2023-10-09T11:09:22.048 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.048 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.048 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.049 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.368002+0000", 2023-10-09T11:09:22.049 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.78742019100000005 2023-10-09T11:09:22.049 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.049 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.049 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:22.049 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.368083+0000", 2023-10-09T11:09:22.049 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0724999999999997e-05 2023-10-09T11:09:22.049 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:22.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.368196+0000", 2023-10-09T11:09:22.050 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011305400000000001 2023-10-09T11:09:22.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:22.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.373317+0000", 2023-10-09T11:09:22.050 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.005120711 2023-10-09T11:09:22.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.051 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.390826+0000", 2023-10-09T11:09:22.051 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017509423999999999 2023-10-09T11:09:22.051 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.051 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.051 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.051 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391076+0000", 2023-10-09T11:09:22.051 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00024970299999999999 2023-10-09T11:09:22.051 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.051 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.051 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:22.051 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391180+0000", 2023-10-09T11:09:22.051 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103912 2023-10-09T11:09:22.052 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.052 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.052 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:22.052 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391284+0000", 2023-10-09T11:09:22.052 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104409 2023-10-09T11:09:22.052 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.052 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:22.052 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.052 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.052 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.052 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:91752 11.15 11:aa793d73:::benchmark_data_smithi125_439543_object91751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-10-09T11:09:22.052 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:29.590492+0000", 2023-10-09T11:09:22.053 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 172.32921487900001, 2023-10-09T11:09:22.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.80094553999999996, 2023-10-09T11:09:22.053 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:22.053 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:22.053 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:22.053 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:22.053 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:22.053 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 91752 2023-10-09T11:09:22.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.053 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:22.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.053 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:22.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.590492+0000", 2023-10-09T11:09:22.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:22.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.590492+0000", 2023-10-09T11:09:22.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:22.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.590490+0000", 2023-10-09T11:09:22.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:22.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.055 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:22.055 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.590524+0000", 2023-10-09T11:09:22.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3809000000000001e-05 2023-10-09T11:09:22.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.055 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:22.055 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.590545+0000", 2023-10-09T11:09:22.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1321000000000001e-05 2023-10-09T11:09:22.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.055 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:22.055 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.590555+0000", 2023-10-09T11:09:22.056 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0999999999999993e-06 2023-10-09T11:09:22.056 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.056 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.056 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.056 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.592601+0000", 2023-10-09T11:09:22.056 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020466999999999998 2023-10-09T11:09:22.056 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.056 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.056 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:22.056 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.592607+0000", 2023-10-09T11:09:22.056 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0800000000000002e-06 2023-10-09T11:09:22.056 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.057 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.057 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.369491+0000", 2023-10-09T11:09:22.057 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.77688346699999999 2023-10-09T11:09:22.057 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.057 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:22.057 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.369625+0000", 2023-10-09T11:09:22.057 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000133912 2023-10-09T11:09:22.057 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.057 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:22.057 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.369788+0000", 2023-10-09T11:09:22.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016285900000000001 2023-10-09T11:09:22.058 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.058 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:22.058 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.381898+0000", 2023-10-09T11:09:22.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012110173 2023-10-09T11:09:22.058 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.058 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.058 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391046+0000", 2023-10-09T11:09:22.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0091483959999999996 2023-10-09T11:09:22.058 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.059 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.059 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.059 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391319+0000", 2023-10-09T11:09:22.059 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00027267300000000002 2023-10-09T11:09:22.059 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.059 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.059 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:22.059 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391357+0000", 2023-10-09T11:09:22.059 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8084000000000003e-05 2023-10-09T11:09:22.059 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.059 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.059 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:22.060 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391437+0000", 2023-10-09T11:09:22.060 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0484999999999999e-05 2023-10-09T11:09:22.060 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.060 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:22.060 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.060 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.060 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.060 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:91766 11.15 11:af01a442:::benchmark_data_smithi125_439543_object91765:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-10-09T11:09:22.060 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:29.617046+0000", 2023-10-09T11:09:22.060 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 172.30266062999999, 2023-10-09T11:09:22.060 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.77460204600000004, 2023-10-09T11:09:22.060 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:22.061 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:22.061 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:22.061 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:22.061 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:22.061 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 91766 2023-10-09T11:09:22.061 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.061 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:22.061 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.061 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:22.061 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.617046+0000", 2023-10-09T11:09:22.061 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.061 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.061 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.062 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:22.062 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.617046+0000", 2023-10-09T11:09:22.062 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.062 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.062 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.062 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:22.062 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.617045+0000", 2023-10-09T11:09:22.062 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:09:22.062 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.062 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.062 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:22.062 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.617077+0000", 2023-10-09T11:09:22.063 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1671999999999998e-05 2023-10-09T11:09:22.063 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.063 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.063 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:22.063 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.617089+0000", 2023-10-09T11:09:22.063 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2615e-05 2023-10-09T11:09:22.063 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.063 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.063 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:22.063 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.617095+0000", 2023-10-09T11:09:22.063 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2599999999999996e-06 2023-10-09T11:09:22.063 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.064 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.064 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.064 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.617164+0000", 2023-10-09T11:09:22.064 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9424000000000005e-05 2023-10-09T11:09:22.064 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.064 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.064 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:22.064 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.617176+0000", 2023-10-09T11:09:22.064 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2257e-05 2023-10-09T11:09:22.064 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.064 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.064 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.064 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.371403+0000", 2023-10-09T11:09:22.065 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.75422663999999995 2023-10-09T11:09:22.065 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.065 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.065 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:22.065 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.371537+0000", 2023-10-09T11:09:22.065 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013418999999999999 2023-10-09T11:09:22.065 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.065 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.065 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:22.065 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.371656+0000", 2023-10-09T11:09:22.065 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011877799999999999 2023-10-09T11:09:22.065 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.066 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.066 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:22.066 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.381908+0000", 2023-10-09T11:09:22.066 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010252219 2023-10-09T11:09:22.066 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.066 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.066 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.066 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391499+0000", 2023-10-09T11:09:22.066 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0095908090000000005 2023-10-09T11:09:22.066 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.066 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.066 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.067 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391528+0000", 2023-10-09T11:09:22.067 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9420999999999998e-05 2023-10-09T11:09:22.067 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.067 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.067 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:22.067 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391567+0000", 2023-10-09T11:09:22.067 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8414999999999997e-05 2023-10-09T11:09:22.067 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.067 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.067 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:22.067 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391648+0000", 2023-10-09T11:09:22.067 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1398999999999998e-05 2023-10-09T11:09:22.067 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.068 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:22.068 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.068 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.068 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.068 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:91806 11.15 11:ab425412:::benchmark_data_smithi125_439543_object91805:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-10-09T11:09:22.068 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:29.875234+0000", 2023-10-09T11:09:22.068 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 172.04447227899999, 2023-10-09T11:09:22.068 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.51671341999999998, 2023-10-09T11:09:22.068 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:22.068 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:22.068 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:22.068 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:22.069 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:22.069 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 91806 2023-10-09T11:09:22.069 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.069 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:22.069 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.069 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:22.069 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.875234+0000", 2023-10-09T11:09:22.069 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.069 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.069 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.069 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:22.069 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.875234+0000", 2023-10-09T11:09:22.069 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.070 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.070 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.070 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:22.070 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.875234+0000", 2023-10-09T11:09:22.070 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T11:09:22.070 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.070 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.070 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:22.070 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.875314+0000", 2023-10-09T11:09:22.070 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0222999999999994e-05 2023-10-09T11:09:22.070 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.070 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.071 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:22.071 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.875324+0000", 2023-10-09T11:09:22.071 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0213e-05 2023-10-09T11:09:22.071 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.071 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.071 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:22.071 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.875329+0000", 2023-10-09T11:09:22.071 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3260000000000003e-06 2023-10-09T11:09:22.071 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.071 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.071 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.071 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.875399+0000", 2023-10-09T11:09:22.072 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9439999999999999e-05 2023-10-09T11:09:22.072 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.072 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.072 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:22.072 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.875412+0000", 2023-10-09T11:09:22.072 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3125999999999999e-05 2023-10-09T11:09:22.072 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.072 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.072 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.072 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.373371+0000", 2023-10-09T11:09:22.072 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.497958554 2023-10-09T11:09:22.072 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.072 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.073 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:22.073 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.373531+0000", 2023-10-09T11:09:22.073 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016065799999999999 2023-10-09T11:09:22.073 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.073 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.073 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:22.073 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.373701+0000", 2023-10-09T11:09:22.073 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016947199999999999 2023-10-09T11:09:22.073 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.073 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.073 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:22.073 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.381914+0000", 2023-10-09T11:09:22.073 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0082136269999999994 2023-10-09T11:09:22.074 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.074 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.074 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.074 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391685+0000", 2023-10-09T11:09:22.074 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0097704420000000007 2023-10-09T11:09:22.074 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.074 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.074 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.074 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391830+0000", 2023-10-09T11:09:22.074 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000145406 2023-10-09T11:09:22.074 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.074 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.075 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:22.075 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391867+0000", 2023-10-09T11:09:22.075 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6729e-05 2023-10-09T11:09:22.075 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.075 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.075 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:22.075 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391948+0000", 2023-10-09T11:09:22.075 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1037000000000004e-05 2023-10-09T11:09:22.075 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.075 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:22.075 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.075 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.075 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.076 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:91831 11.15 11:ae08dc9b:::benchmark_data_smithi125_439543_object91830:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-10-09T11:09:22.076 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:06:29.909549+0000", 2023-10-09T11:09:22.076 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 172.01015729100001, 2023-10-09T11:09:22.076 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.48254506200000002, 2023-10-09T11:09:22.076 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:22.076 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:22.076 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:22.076 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:22.076 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:22.076 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 91831 2023-10-09T11:09:22.076 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.077 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:22.077 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.077 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:22.077 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.909549+0000", 2023-10-09T11:09:22.077 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.077 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.077 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.077 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:22.077 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.909549+0000", 2023-10-09T11:09:22.077 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.077 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.077 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.077 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:22.078 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.909548+0000", 2023-10-09T11:09:22.078 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:22.078 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.078 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.078 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:22.078 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.909619+0000", 2023-10-09T11:09:22.078 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1011000000000002e-05 2023-10-09T11:09:22.078 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.078 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.078 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:22.078 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.909635+0000", 2023-10-09T11:09:22.078 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5908000000000001e-05 2023-10-09T11:09:22.079 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.079 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.079 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:22.079 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.909641+0000", 2023-10-09T11:09:22.079 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6189999999999996e-06 2023-10-09T11:09:22.079 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.079 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.079 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.079 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.909704+0000", 2023-10-09T11:09:22.079 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2549e-05 2023-10-09T11:09:22.079 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.080 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.080 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:22.080 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:29.909716+0000", 2023-10-09T11:09:22.080 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1773999999999999e-05 2023-10-09T11:09:22.080 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.080 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.080 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.080 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.375323+0000", 2023-10-09T11:09:22.080 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.46560713199999998 2023-10-09T11:09:22.080 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.080 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.081 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:22.081 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.375448+0000", 2023-10-09T11:09:22.081 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012506999999999999 2023-10-09T11:09:22.081 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.081 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.081 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:22.081 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.375641+0000", 2023-10-09T11:09:22.081 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000192962 2023-10-09T11:09:22.081 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.081 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.081 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:22.081 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.381920+0000", 2023-10-09T11:09:22.082 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062792530000000003 2023-10-09T11:09:22.082 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.082 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.082 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.082 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391716+0000", 2023-10-09T11:09:22.082 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.009795382 2023-10-09T11:09:22.082 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.082 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.082 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.082 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.391979+0000", 2023-10-09T11:09:22.082 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00026356200000000001 2023-10-09T11:09:22.082 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.083 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.083 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:22.083 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.392015+0000", 2023-10-09T11:09:22.083 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5982999999999998e-05 2023-10-09T11:09:22.083 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.083 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.083 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:22.083 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:06:30.392095+0000", 2023-10-09T11:09:22.083 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9407999999999996e-05 2023-10-09T11:09:22.083 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.083 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:22.083 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.084 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.084 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.084 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:135664 11.5 11:a5e2a1fd:::benchmark_data_smithi125_439543_object36165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e541)", 2023-10-09T11:09:22.084 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:07:16.499582+0000", 2023-10-09T11:09:22.084 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 125.42012429499999, 2023-10-09T11:09:22.084 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.54005183899999998, 2023-10-09T11:09:22.084 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:22.084 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:22.084 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:22.084 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:22.085 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:22.085 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135664 2023-10-09T11:09:22.085 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.085 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:22.085 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.085 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:22.085 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.499582+0000", 2023-10-09T11:09:22.085 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.085 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.085 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.085 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:22.085 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.499582+0000", 2023-10-09T11:09:22.086 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.086 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.086 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.086 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:22.086 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.499581+0000", 2023-10-09T11:09:22.086 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T11:09:22.086 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.086 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.086 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:22.086 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.499587+0000", 2023-10-09T11:09:22.086 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9800000000000003e-06 2023-10-09T11:09:22.087 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.087 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.087 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:22.087 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.499599+0000", 2023-10-09T11:09:22.087 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.183e-05 2023-10-09T11:09:22.087 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.087 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.087 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:22.087 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.499604+0000", 2023-10-09T11:09:22.087 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3009999999999997e-06 2023-10-09T11:09:22.088 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.088 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.088 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.088 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.499668+0000", 2023-10-09T11:09:22.088 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3447000000000004e-05 2023-10-09T11:09:22.088 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.088 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.088 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:22.088 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.499679+0000", 2023-10-09T11:09:22.088 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1499e-05 2023-10-09T11:09:22.088 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.088 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.089 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.089 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.030417+0000", 2023-10-09T11:09:22.089 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.53073825299999999 2023-10-09T11:09:22.089 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.089 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.089 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:22.089 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.030491+0000", 2023-10-09T11:09:22.089 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3986000000000006e-05 2023-10-09T11:09:22.089 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.089 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.089 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:22.089 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.030582+0000", 2023-10-09T11:09:22.090 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0943e-05 2023-10-09T11:09:22.090 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.090 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.090 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:22.090 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.036856+0000", 2023-10-09T11:09:22.090 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062736019999999997 2023-10-09T11:09:22.090 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.090 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.090 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.090 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.039432+0000", 2023-10-09T11:09:22.090 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0025765559999999998 2023-10-09T11:09:22.091 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.091 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.091 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.091 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.039588+0000", 2023-10-09T11:09:22.091 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000155997 2023-10-09T11:09:22.091 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.091 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.091 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:22.091 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.039614+0000", 2023-10-09T11:09:22.091 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5493000000000001e-05 2023-10-09T11:09:22.092 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.092 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.092 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:22.092 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.039634+0000", 2023-10-09T11:09:22.092 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0420000000000001e-05 2023-10-09T11:09:22.092 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.092 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:22.092 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.092 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.092 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.092 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36510.0:135710 11.5 11:a2864040:::benchmark_data_smithi125_439543_object36211:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e541)", 2023-10-09T11:09:22.092 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T11:07:16.552772+0000", 2023-10-09T11:09:22.093 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 125.36693445900001, 2023-10-09T11:09:22.093 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.48941067199999999, 2023-10-09T11:09:22.093 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T11:09:22.093 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T11:09:22.093 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T11:09:22.093 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36510", 2023-10-09T11:09:22.093 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3592884926", 2023-10-09T11:09:22.093 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135710 2023-10-09T11:09:22.093 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.093 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T11:09:22.093 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.093 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T11:09:22.094 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.552772+0000", 2023-10-09T11:09:22.094 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.094 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.094 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.094 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T11:09:22.094 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.552772+0000", 2023-10-09T11:09:22.094 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T11:09:22.094 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.094 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.094 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T11:09:22.094 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.552770+0000", 2023-10-09T11:09:22.094 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T11:09:22.095 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.095 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.095 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T11:09:22.095 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.552777+0000", 2023-10-09T11:09:22.095 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.601e-06 2023-10-09T11:09:22.095 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.095 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.095 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T11:09:22.095 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.552789+0000", 2023-10-09T11:09:22.095 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2277e-05 2023-10-09T11:09:22.095 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.095 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.096 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T11:09:22.096 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.552795+0000", 2023-10-09T11:09:22.096 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8409999999999996e-06 2023-10-09T11:09:22.096 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.096 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.096 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.096 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.552853+0000", 2023-10-09T11:09:22.096 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7738000000000002e-05 2023-10-09T11:09:22.096 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.096 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.096 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T11:09:22.096 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:16.552864+0000", 2023-10-09T11:09:22.097 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0849999999999999e-05 2023-10-09T11:09:22.097 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.097 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.097 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T11:09:22.097 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.031698+0000", 2023-10-09T11:09:22.097 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.47883416299999998 2023-10-09T11:09:22.097 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.097 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.097 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T11:09:22.097 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.031768+0000", 2023-10-09T11:09:22.098 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0487000000000005e-05 2023-10-09T11:09:22.098 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.098 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.098 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T11:09:22.098 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.031833+0000", 2023-10-09T11:09:22.098 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4193e-05 2023-10-09T11:09:22.098 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.098 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.098 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T11:09:22.098 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.036862+0000", 2023-10-09T11:09:22.098 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.005029525 2023-10-09T11:09:22.098 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.099 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.099 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.099 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.039473+0000", 2023-10-09T11:09:22.099 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026103680000000001 2023-10-09T11:09:22.099 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.099 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.099 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T11:09:22.099 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.042105+0000", 2023-10-09T11:09:22.099 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026322110000000002 2023-10-09T11:09:22.099 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.099 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.100 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T11:09:22.100 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.042146+0000", 2023-10-09T11:09:22.100 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0951000000000001e-05 2023-10-09T11:09:22.100 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T11:09:22.100 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T11:09:22.100 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T11:09:22.100 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T11:07:17.042183+0000", 2023-10-09T11:09:22.100 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7255000000000002e-05 2023-10-09T11:09:22.100 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.101 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:22.101 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.101 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T11:09:22.101 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T11:09:22.101 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T11:09:22.102 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 0% complete...failed. 2023-10-09T11:09:22.102 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T11:09:22.103 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T11:09:22.103 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_475522/sparse2 2023-10-09T11:09:22.103 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T11:09:22.103 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T11:09:22.103 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T11:09:22.103 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd ls -l 2023-10-09T11:09:22.103 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep sparse2 2023-10-09T11:09:22.103 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T11:09:22.104 INFO:tasks.workunit.client.0.smithi125.stdout:sparse2 2 MiB 2 2023-10-09T11:09:22.104 INFO:tasks.workunit.client.0.smithi125.stderr:++ objects sparse2 2023-10-09T11:09:22.104 INFO:tasks.workunit.client.0.smithi125.stderr:++ image=sparse2 2023-10-09T11:09:22.104 INFO:tasks.workunit.client.0.smithi125.stderr:+++ rbd info sparse2 2023-10-09T11:09:22.104 INFO:tasks.workunit.client.0.smithi125.stderr:+++ grep block_name_prefix 2023-10-09T11:09:22.105 INFO:tasks.workunit.client.0.smithi125.stderr:+++ awk '{print $NF;}' 2023-10-09T11:09:22.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:22.105 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:22.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:22.107 INFO:tasks.workunit.client.0.smithi125.stderr:++ prefix=rbd_data.91d1e6709307 2023-10-09T11:09:22.107 INFO:tasks.workunit.client.0.smithi125.stderr:+++ grep rbd_data.91d1e6709307 2023-10-09T11:09:22.107 INFO:tasks.workunit.client.0.smithi125.stderr:++++ get_image_data_pool sparse2 2023-10-09T11:09:22.107 INFO:tasks.workunit.client.0.smithi125.stderr:+++ sed -e 's/rbd_data.91d1e6709307\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T11:09:22.107 INFO:tasks.workunit.client.0.smithi125.stderr:++++ image=sparse2 2023-10-09T11:09:22.107 INFO:tasks.workunit.client.0.smithi125.stderr:+++ sort -u 2023-10-09T11:09:22.107 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ rbd info sparse2 2023-10-09T11:09:22.107 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ grep 'data_pool: ' 2023-10-09T11:09:22.107 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T11:09:22.112 INFO:tasks.workunit.client.0.smithi125.stderr:++++ data_pool= 2023-10-09T11:09:22.112 INFO:tasks.workunit.client.0.smithi125.stderr:++++ '[' -z ']' 2023-10-09T11:09:22.112 INFO:tasks.workunit.client.0.smithi125.stderr:++++ data_pool=rbd 2023-10-09T11:09:22.112 INFO:tasks.workunit.client.0.smithi125.stderr:++++ echo rbd 2023-10-09T11:09:22.112 INFO:tasks.workunit.client.0.smithi125.stderr:+++ rados ls -p rbd 2023-10-09T11:09:22.145 INFO:tasks.workunit.client.0.smithi125.stderr:++ objects=0 2023-10-09T11:09:22.145 INFO:tasks.workunit.client.0.smithi125.stderr:++ echo 0 2023-10-09T11:09:22.145 INFO:tasks.workunit.client.0.smithi125.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T11:09:22.145 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export sparse2 /tmp/rbd_import_export_475522/sparse2.out 2023-10-09T11:09:22.180 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T11:09:22.183 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_475522/sparse2 /tmp/rbd_import_export_475522/sparse2.out 2023-10-09T11:09:22.183 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_475522/sparse2 /tmp/rbd_import_export_475522/sparse2.out 2023-10-09T11:09:22.185 INFO:tasks.workunit.client.0.smithi125.stderr:++ stat /tmp/rbd_import_export_475522/sparse2 --format %b 2023-10-09T11:09:22.186 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=2048 2023-10-09T11:09:22.186 INFO:tasks.workunit.client.0.smithi125.stderr:++ stat /tmp/rbd_import_export_475522/sparse2.out --format %b 2023-10-09T11:09:22.187 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=2048 2023-10-09T11:09:22.187 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T11:09:22.188 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T11:09:22.188 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T11:09:22.188 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/sparse2.out 2023-10-09T11:09:22.188 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse2 2023-10-09T11:09:22.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T11:09:22.236 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:22.274 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T11:09:22.277 INFO:tasks.workunit.client.0.smithi125.stderr:+ truncate /tmp/rbd_import_export_475522/sparse1 -s 10M 2023-10-09T11:09:22.278 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-09T11:09:22.290 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T11:09:22.369 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T11:09:22.369 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T11:09:22.369 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T11:09:22.372 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd ls -l 2023-10-09T11:09:22.372 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep sparse1 2023-10-09T11:09:22.373 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-10-09T11:09:22.415 INFO:tasks.workunit.client.0.smithi125.stdout:sparse1 10 MiB 2 2023-10-09T11:09:22.416 INFO:tasks.workunit.client.0.smithi125.stderr:++ objects sparse1 2023-10-09T11:09:22.416 INFO:tasks.workunit.client.0.smithi125.stderr:++ image=sparse1 2023-10-09T11:09:22.416 INFO:tasks.workunit.client.0.smithi125.stderr:+++ rbd info sparse1 2023-10-09T11:09:22.416 INFO:tasks.workunit.client.0.smithi125.stderr:+++ grep block_name_prefix 2023-10-09T11:09:22.416 INFO:tasks.workunit.client.0.smithi125.stderr:+++ awk '{print $NF;}' 2023-10-09T11:09:22.451 INFO:tasks.workunit.client.0.smithi125.stderr:++ prefix=rbd_data.91fe3e550e4 2023-10-09T11:09:22.452 INFO:tasks.workunit.client.0.smithi125.stderr:+++ grep rbd_data.91fe3e550e4 2023-10-09T11:09:22.452 INFO:tasks.workunit.client.0.smithi125.stderr:++++ get_image_data_pool sparse1 2023-10-09T11:09:22.453 INFO:tasks.workunit.client.0.smithi125.stderr:++++ image=sparse1 2023-10-09T11:09:22.453 INFO:tasks.workunit.client.0.smithi125.stderr:+++ sed -e 's/rbd_data.91fe3e550e4\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T11:09:22.453 INFO:tasks.workunit.client.0.smithi125.stderr:+++ sort -u 2023-10-09T11:09:22.453 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ rbd info sparse1 2023-10-09T11:09:22.453 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ grep 'data_pool: ' 2023-10-09T11:09:22.453 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T11:09:22.491 INFO:tasks.workunit.client.0.smithi125.stderr:++++ data_pool= 2023-10-09T11:09:22.491 INFO:tasks.workunit.client.0.smithi125.stderr:++++ '[' -z ']' 2023-10-09T11:09:22.491 INFO:tasks.workunit.client.0.smithi125.stderr:++++ data_pool=rbd 2023-10-09T11:09:22.491 INFO:tasks.workunit.client.0.smithi125.stderr:++++ echo rbd 2023-10-09T11:09:22.491 INFO:tasks.workunit.client.0.smithi125.stderr:+++ rados ls -p rbd 2023-10-09T11:09:22.509 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2" found: no such container 2023-10-09T11:09:22.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:22.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T11:09:22.528 INFO:tasks.workunit.client.0.smithi125.stderr:++ objects=1 2023-10-09T11:09:22.529 INFO:tasks.workunit.client.0.smithi125.stderr:++ echo 1 2023-10-09T11:09:22.529 INFO:tasks.workunit.client.0.smithi125.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T11:09:22.529 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export sparse1 /tmp/rbd_import_export_475522/sparse1.out 2023-10-09T11:09:22.556 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:09:22.566 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:09:22.576 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:22 smithi125 ceph-mon[195507]: pgmap v1435: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 121 KiB/s rd, 10 KiB/s wr, 164 op/s 2023-10-09T11:09:22.576 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:22 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3762106114' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:09:22.576 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:22 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:09:22.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:22 smithi125 ceph-mon[200197]: pgmap v1435: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 121 KiB/s rd, 10 KiB/s wr, 164 op/s 2023-10-09T11:09:22.576 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:22 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3762106114' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:09:22.577 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:22 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:09:22.590 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-09T11:09:22.594 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_475522/sparse1 /tmp/rbd_import_export_475522/sparse1.out 2023-10-09T11:09:22.594 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_475522/sparse1 /tmp/rbd_import_export_475522/sparse1.out 2023-10-09T11:09:22.602 INFO:tasks.workunit.client.0.smithi125.stderr:++ stat /tmp/rbd_import_export_475522/sparse1 --format %b 2023-10-09T11:09:22.603 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=2048 2023-10-09T11:09:22.603 INFO:tasks.workunit.client.0.smithi125.stderr:++ stat /tmp/rbd_import_export_475522/sparse1.out --format %b 2023-10-09T11:09:22.604 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=2048 2023-10-09T11:09:22.604 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T11:09:22.605 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T11:09:22.605 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T11:09:22.605 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/sparse1.out 2023-10-09T11:09:22.606 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse1 2023-10-09T11:09:22.612 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:22.645 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:22.668 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:22.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:22.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:22.701 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T11:09:22.704 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_475522/sparse2 oflag=append conv=notrunc 2023-10-09T11:09:22.715 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records in 2023-10-09T11:09:22.716 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records out 2023-10-09T11:09:22.716 INFO:tasks.workunit.client.0.smithi125.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0102554 s, 204 MB/s 2023-10-09T11:09:22.716 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-09T11:09:22.773 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T11:09:22.802 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T11:09:22.803 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T11:09:22.803 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T11:09:22.806 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd ls -l 2023-10-09T11:09:22.807 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep sparse2 2023-10-09T11:09:22.807 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:22.807 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-10-09T11:09:22.848 INFO:tasks.workunit.client.0.smithi125.stdout:sparse2 4 MiB 2 2023-10-09T11:09:22.850 INFO:tasks.workunit.client.0.smithi125.stderr:++ objects sparse2 2023-10-09T11:09:22.850 INFO:tasks.workunit.client.0.smithi125.stderr:++ image=sparse2 2023-10-09T11:09:22.850 INFO:tasks.workunit.client.0.smithi125.stderr:+++ rbd info sparse2 2023-10-09T11:09:22.850 INFO:tasks.workunit.client.0.smithi125.stderr:+++ grep block_name_prefix 2023-10-09T11:09:22.850 INFO:tasks.workunit.client.0.smithi125.stderr:+++ awk '{print $NF;}' 2023-10-09T11:09:22.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:22 smithi155 ceph-mon[184854]: pgmap v1435: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 121 KiB/s rd, 10 KiB/s wr, 164 op/s 2023-10-09T11:09:22.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:22 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3762106114' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:09:22.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:22 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:09:22.898 INFO:tasks.workunit.client.0.smithi125.stderr:++ prefix=rbd_data.922b73c1665c 2023-10-09T11:09:22.899 INFO:tasks.workunit.client.0.smithi125.stderr:+++ grep rbd_data.922b73c1665c 2023-10-09T11:09:22.899 INFO:tasks.workunit.client.0.smithi125.stderr:+++ sed -e 's/rbd_data.922b73c1665c\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T11:09:22.899 INFO:tasks.workunit.client.0.smithi125.stderr:++++ get_image_data_pool sparse2 2023-10-09T11:09:22.899 INFO:tasks.workunit.client.0.smithi125.stderr:++++ image=sparse2 2023-10-09T11:09:22.900 INFO:tasks.workunit.client.0.smithi125.stderr:+++ sort -u 2023-10-09T11:09:22.900 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ rbd info sparse2 2023-10-09T11:09:22.900 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ grep 'data_pool: ' 2023-10-09T11:09:22.900 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T11:09:22.907 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T11:09:22.931 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T11:09:22.942 INFO:tasks.workunit.client.0.smithi125.stderr:++++ data_pool= 2023-10-09T11:09:22.942 INFO:tasks.workunit.client.0.smithi125.stderr:++++ '[' -z ']' 2023-10-09T11:09:22.942 INFO:tasks.workunit.client.0.smithi125.stderr:++++ data_pool=rbd 2023-10-09T11:09:22.943 INFO:tasks.workunit.client.0.smithi125.stderr:++++ echo rbd 2023-10-09T11:09:22.943 INFO:tasks.workunit.client.0.smithi125.stderr:+++ rados ls -p rbd 2023-10-09T11:09:22.953 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T11:09:22.974 INFO:tasks.workunit.client.0.smithi125.stderr:++ objects='0 2023-10-09T11:09:22.975 INFO:tasks.workunit.client.0.smithi125.stderr:2 2023-10-09T11:09:22.975 INFO:tasks.workunit.client.0.smithi125.stderr:3' 2023-10-09T11:09:22.975 INFO:tasks.workunit.client.0.smithi125.stderr:++ echo 0 2 3 2023-10-09T11:09:22.975 INFO:tasks.workunit.client.0.smithi125.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-10-09T11:09:22.975 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export sparse2 /tmp/rbd_import_export_475522/sparse2.out 2023-10-09T11:09:22.977 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T11:09:22.997 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T11:09:23.019 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T11:09:23.027 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-09T11:09:23.031 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_475522/sparse2 /tmp/rbd_import_export_475522/sparse2.out 2023-10-09T11:09:23.031 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_475522/sparse2 /tmp/rbd_import_export_475522/sparse2.out 2023-10-09T11:09:23.034 INFO:tasks.workunit.client.0.smithi125.stderr:++ stat /tmp/rbd_import_export_475522/sparse2 --format %b 2023-10-09T11:09:23.035 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=6144 2023-10-09T11:09:23.035 INFO:tasks.workunit.client.0.smithi125.stderr:++ stat /tmp/rbd_import_export_475522/sparse2.out --format %b 2023-10-09T11:09:23.036 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=6144 2023-10-09T11:09:23.036 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T11:09:23.036 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T11:09:23.037 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T11:09:23.037 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/sparse2.out 2023-10-09T11:09:23.037 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse2 2023-10-09T11:09:23.042 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T11:09:23.074 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T11:09:23.076 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T11:09:23.076 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T11:09:23.076 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T11:09:23.076 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T11:09:23.076 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T11:09:23.076 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T11:09:23.076 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T11:09:23.076 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T11:09:23.081 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1" found: no such container 2023-10-09T11:09:23.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:23.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T11:09:23.139 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-09T11:09:23.143 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-10-09T11:09:23.143 INFO:tasks.workunit.client.0.smithi125.stdout:partially-sparse file imports to partially-sparse image 2023-10-09T11:09:23.143 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_475522/sparse1 sparse 2023-10-09T11:09:23.189 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:09:23.204 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-09T11:09:23.205 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T11:09:23.205 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T11:09:23.208 INFO:tasks.workunit.client.0.smithi125.stderr:++ objects sparse 2023-10-09T11:09:23.208 INFO:tasks.workunit.client.0.smithi125.stderr:++ image=sparse 2023-10-09T11:09:23.208 INFO:tasks.workunit.client.0.smithi125.stderr:+++ rbd info sparse 2023-10-09T11:09:23.208 INFO:tasks.workunit.client.0.smithi125.stderr:+++ grep block_name_prefix 2023-10-09T11:09:23.209 INFO:tasks.workunit.client.0.smithi125.stderr:+++ awk '{print $NF;}' 2023-10-09T11:09:23.220 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:23.238 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:09:23.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:23.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:09:23.248 INFO:tasks.workunit.client.0.smithi125.stderr:++ prefix=rbd_data.b948b7461b07 2023-10-09T11:09:23.248 INFO:tasks.workunit.client.0.smithi125.stderr:+++ grep rbd_data.b948b7461b07 2023-10-09T11:09:23.249 INFO:tasks.workunit.client.0.smithi125.stderr:++++ get_image_data_pool sparse 2023-10-09T11:09:23.249 INFO:tasks.workunit.client.0.smithi125.stderr:++++ image=sparse 2023-10-09T11:09:23.249 INFO:tasks.workunit.client.0.smithi125.stderr:+++ sed -e 's/rbd_data.b948b7461b07\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T11:09:23.249 INFO:tasks.workunit.client.0.smithi125.stderr:+++ sort -u 2023-10-09T11:09:23.250 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ rbd info sparse 2023-10-09T11:09:23.250 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ grep 'data_pool: ' 2023-10-09T11:09:23.250 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T11:09:23.286 INFO:tasks.workunit.client.0.smithi125.stderr:++++ data_pool= 2023-10-09T11:09:23.286 INFO:tasks.workunit.client.0.smithi125.stderr:++++ '[' -z ']' 2023-10-09T11:09:23.286 INFO:tasks.workunit.client.0.smithi125.stderr:++++ data_pool=rbd 2023-10-09T11:09:23.287 INFO:tasks.workunit.client.0.smithi125.stderr:++++ echo rbd 2023-10-09T11:09:23.287 INFO:tasks.workunit.client.0.smithi125.stderr:+++ rados ls -p rbd 2023-10-09T11:09:23.319 INFO:tasks.workunit.client.0.smithi125.stderr:++ objects=1 2023-10-09T11:09:23.320 INFO:tasks.workunit.client.0.smithi125.stderr:++ echo 1 2023-10-09T11:09:23.320 INFO:tasks.workunit.client.0.smithi125.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T11:09:23.320 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse 2023-10-09T11:09:23.342 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T11:09:23.372 INFO:teuthology.orchestra.run.smithi125.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:23.393 INFO:teuthology.orchestra.run.smithi125.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3" found: no such container 2023-10-09T11:09:23.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:23.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T11:09:23.416 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T11:09:23.421 INFO:tasks.workunit.client.0.smithi125.stdout:zeros import through stdin to sparse image 2023-10-09T11:09:23.421 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 'zeros import through stdin to sparse image' 2023-10-09T11:09:23.422 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-09T11:09:23.422 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format - sparse 2023-10-09T11:09:23.422 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T11:09:23.481 INFO:tasks.workunit.client.0.smithi125.stderr:4+0 records in 2023-10-09T11:09:23.481 INFO:tasks.workunit.client.0.smithi125.stderr:4+0 records out 2023-10-09T11:09:23.481 INFO:tasks.workunit.client.0.smithi125.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0587565 s, 71.4 MB/s 2023-10-09T11:09:23.491 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T11:09:23.491 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T11:09:23.494 INFO:tasks.workunit.client.0.smithi125.stderr:++ objects sparse 2023-10-09T11:09:23.494 INFO:tasks.workunit.client.0.smithi125.stderr:++ image=sparse 2023-10-09T11:09:23.495 INFO:tasks.workunit.client.0.smithi125.stderr:+++ rbd info sparse 2023-10-09T11:09:23.495 INFO:tasks.workunit.client.0.smithi125.stderr:+++ grep block_name_prefix 2023-10-09T11:09:23.495 INFO:tasks.workunit.client.0.smithi125.stderr:+++ awk '{print $NF;}' 2023-10-09T11:09:23.497 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T11:09:23.527 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:23.531 INFO:tasks.workunit.client.0.smithi125.stderr:++ prefix=rbd_data.b96925a42d11 2023-10-09T11:09:23.532 INFO:tasks.workunit.client.0.smithi125.stderr:+++ grep rbd_data.b96925a42d11 2023-10-09T11:09:23.532 INFO:tasks.workunit.client.0.smithi125.stderr:++++ get_image_data_pool sparse 2023-10-09T11:09:23.532 INFO:tasks.workunit.client.0.smithi125.stderr:++++ image=sparse 2023-10-09T11:09:23.532 INFO:tasks.workunit.client.0.smithi125.stderr:+++ sed -e 's/rbd_data.b96925a42d11\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T11:09:23.532 INFO:tasks.workunit.client.0.smithi125.stderr:+++ sort -u 2023-10-09T11:09:23.532 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ rbd info sparse 2023-10-09T11:09:23.532 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ grep 'data_pool: ' 2023-10-09T11:09:23.532 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T11:09:23.569 INFO:tasks.workunit.client.0.smithi125.stderr:++++ data_pool= 2023-10-09T11:09:23.569 INFO:tasks.workunit.client.0.smithi125.stderr:++++ '[' -z ']' 2023-10-09T11:09:23.569 INFO:tasks.workunit.client.0.smithi125.stderr:++++ data_pool=rbd 2023-10-09T11:09:23.569 INFO:tasks.workunit.client.0.smithi125.stderr:++++ echo rbd 2023-10-09T11:09:23.570 INFO:tasks.workunit.client.0.smithi125.stderr:+++ rados ls -p rbd 2023-10-09T11:09:23.597 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5" found: no such container 2023-10-09T11:09:23.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:23.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T11:09:23.601 INFO:tasks.workunit.client.0.smithi125.stderr:++ objects= 2023-10-09T11:09:23.601 INFO:tasks.workunit.client.0.smithi125.stderr:++ echo 2023-10-09T11:09:23.601 INFO:tasks.workunit.client.0.smithi125.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-10-09T11:09:23.601 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse 2023-10-09T11:09:23.610 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[195507]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:09:23.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[195507]: failed to encode map e591 with expected crc 2023-10-09T11:09:23.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:09:23.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[195507]: osdmap e591: 8 total, 8 up, 8 in 2023-10-09T11:09:23.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[195507]: failed to encode map e591 with expected crc 2023-10-09T11:09:23.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[195507]: failed to encode map e591 with expected crc 2023-10-09T11:09:23.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[195507]: failed to encode map e591 with expected crc 2023-10-09T11:09:23.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1402528886' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:09:23.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4191640805' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:09:23.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:09:23.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[195507]: failed to encode map e592 with expected crc 2023-10-09T11:09:23.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T11:09:23.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[195507]: osdmap e592: 8 total, 8 up, 8 in 2023-10-09T11:09:23.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[200197]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:09:23.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[200197]: failed to encode map e591 with expected crc 2023-10-09T11:09:23.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:09:23.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[200197]: osdmap e591: 8 total, 8 up, 8 in 2023-10-09T11:09:23.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[200197]: failed to encode map e591 with expected crc 2023-10-09T11:09:23.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[200197]: failed to encode map e591 with expected crc 2023-10-09T11:09:23.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[200197]: failed to encode map e591 with expected crc 2023-10-09T11:09:23.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1402528886' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:09:23.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4191640805' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:09:23.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:09:23.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[200197]: failed to encode map e592 with expected crc 2023-10-09T11:09:23.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T11:09:23.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:23 smithi125 ceph-mon[200197]: osdmap e592: 8 total, 8 up, 8 in 2023-10-09T11:09:23.642 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:23.642 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:23.680 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T11:09:23.683 INFO:tasks.workunit.client.0.smithi125.stdout:zeros export to sparse file 2023-10-09T11:09:23.684 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 'zeros export to sparse file' 2023-10-09T11:09:23.684 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd create --new-format sparse --size 4 2023-10-09T11:09:23.701 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T11:09:23.722 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T11:09:23.724 INFO:tasks.workunit.client.0.smithi125.stderr:++ rbd info sparse 2023-10-09T11:09:23.725 INFO:tasks.workunit.client.0.smithi125.stderr:++ grep block_name_prefix 2023-10-09T11:09:23.725 INFO:tasks.workunit.client.0.smithi125.stderr:++ awk '{print $NF;}' 2023-10-09T11:09:23.730 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:23.750 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7" found: no such container 2023-10-09T11:09:23.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:23.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T11:09:23.762 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.92a0981e3095 2023-10-09T11:09:23.762 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-09T11:09:23.763 INFO:tasks.workunit.client.0.smithi125.stderr:++ get_image_data_pool sparse 2023-10-09T11:09:23.763 INFO:tasks.workunit.client.0.smithi125.stderr:++ image=sparse 2023-10-09T11:09:23.763 INFO:tasks.workunit.client.0.smithi125.stderr:+++ rbd info sparse 2023-10-09T11:09:23.763 INFO:tasks.workunit.client.0.smithi125.stderr:+++ grep 'data_pool: ' 2023-10-09T11:09:23.763 INFO:tasks.workunit.client.0.smithi125.stderr:+++ awk -F: '{ print $NF }' 2023-10-09T11:09:23.798 INFO:tasks.workunit.client.0.smithi125.stderr:++ data_pool= 2023-10-09T11:09:23.798 INFO:tasks.workunit.client.0.smithi125.stderr:++ '[' -z ']' 2023-10-09T11:09:23.798 INFO:tasks.workunit.client.0.smithi125.stderr:++ data_pool=rbd 2023-10-09T11:09:23.798 INFO:tasks.workunit.client.0.smithi125.stderr:++ echo rbd 2023-10-09T11:09:23.798 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados -p rbd put rbd_data.92a0981e3095.000000000000 - 2023-10-09T11:09:23.823 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T11:09:23.829 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records in 2023-10-09T11:09:23.829 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records out 2023-10-09T11:09:23.829 INFO:tasks.workunit.client.0.smithi125.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0666524 s, 62.9 MB/s 2023-10-09T11:09:23.848 INFO:tasks.workunit.client.0.smithi125.stderr:++ objects sparse 2023-10-09T11:09:23.848 INFO:tasks.workunit.client.0.smithi125.stderr:++ image=sparse 2023-10-09T11:09:23.849 INFO:tasks.workunit.client.0.smithi125.stderr:+++ rbd info sparse 2023-10-09T11:09:23.849 INFO:tasks.workunit.client.0.smithi125.stderr:+++ grep block_name_prefix 2023-10-09T11:09:23.849 INFO:tasks.workunit.client.0.smithi125.stderr:+++ awk '{print $NF;}' 2023-10-09T11:09:23.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:09:23.882 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:23 smithi155 ceph-mon[184854]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T11:09:23.882 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:23 smithi155 ceph-mon[184854]: failed to encode map e591 with expected crc 2023-10-09T11:09:23.882 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:23 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:09:23.882 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:23 smithi155 ceph-mon[184854]: osdmap e591: 8 total, 8 up, 8 in 2023-10-09T11:09:23.882 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:23 smithi155 ceph-mon[184854]: failed to encode map e591 with expected crc 2023-10-09T11:09:23.882 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:23 smithi155 ceph-mon[184854]: failed to encode map e591 with expected crc 2023-10-09T11:09:23.882 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:23 smithi155 ceph-mon[184854]: failed to encode map e591 with expected crc 2023-10-09T11:09:23.882 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:23 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1402528886' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T11:09:23.883 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:23 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4191640805' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:09:23.883 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:23 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:09:23.883 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:23 smithi155 ceph-mon[184854]: failed to encode map e592 with expected crc 2023-10-09T11:09:23.883 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:23 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T11:09:23.883 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:23 smithi155 ceph-mon[184854]: osdmap e592: 8 total, 8 up, 8 in 2023-10-09T11:09:23.885 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:23.885 INFO:tasks.workunit.client.0.smithi125.stderr:++ prefix=rbd_data.92a0981e3095 2023-10-09T11:09:23.886 INFO:tasks.workunit.client.0.smithi125.stderr:+++ grep rbd_data.92a0981e3095 2023-10-09T11:09:23.886 INFO:tasks.workunit.client.0.smithi125.stderr:++++ get_image_data_pool sparse 2023-10-09T11:09:23.887 INFO:tasks.workunit.client.0.smithi125.stderr:++++ image=sparse 2023-10-09T11:09:23.887 INFO:tasks.workunit.client.0.smithi125.stderr:+++ sed -e 's/rbd_data.92a0981e3095\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T11:09:23.887 INFO:tasks.workunit.client.0.smithi125.stderr:+++ sort -u 2023-10-09T11:09:23.890 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ rbd info sparse 2023-10-09T11:09:23.890 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ grep 'data_pool: ' 2023-10-09T11:09:23.890 INFO:tasks.workunit.client.0.smithi125.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T11:09:23.923 INFO:tasks.workunit.client.0.smithi125.stderr:++++ data_pool= 2023-10-09T11:09:23.923 INFO:tasks.workunit.client.0.smithi125.stderr:++++ '[' -z ']' 2023-10-09T11:09:23.923 INFO:tasks.workunit.client.0.smithi125.stderr:++++ data_pool=rbd 2023-10-09T11:09:23.924 INFO:tasks.workunit.client.0.smithi125.stderr:++++ echo rbd 2023-10-09T11:09:23.924 INFO:tasks.workunit.client.0.smithi125.stderr:+++ rados ls -p rbd 2023-10-09T11:09:23.957 INFO:tasks.workunit.client.0.smithi125.stderr:++ objects=0 2023-10-09T11:09:23.957 INFO:tasks.workunit.client.0.smithi125.stderr:++ echo 0 2023-10-09T11:09:23.957 INFO:tasks.workunit.client.0.smithi125.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T11:09:23.957 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/sparse 2023-10-09T11:09:23.957 INFO:tasks.workunit.client.0.smithi125.stderr:rm: cannot remove '/tmp/rbd_import_export_475522/sparse': No such file or directory 2023-10-09T11:09:23.958 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T11:09:23.958 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export sparse /tmp/rbd_import_export_475522/sparse 2023-10-09T11:09:23.993 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete...done. 2023-10-09T11:09:23.996 INFO:tasks.workunit.client.0.smithi125.stderr:++ stat /tmp/rbd_import_export_475522/sparse --format=%b 2023-10-09T11:09:23.997 INFO:tasks.workunit.client.0.smithi125.stderr:+ '[' 0 = 0 ']' 2023-10-09T11:09:23.997 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse 2023-10-09T11:09:24.080 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T11:09:24.083 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_475522/sparse /tmp/rbd_import_export_475522/sparse1 /tmp/rbd_import_export_475522/sparse2 /tmp/rbd_import_export_475522/sparse3 2023-10-09T11:09:24.085 INFO:tasks.workunit.client.0.smithi125.stderr:rm: cannot remove '/tmp/rbd_import_export_475522/sparse3': No such file or directory 2023-10-09T11:09:24.085 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T11:09:24.085 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo OK 2023-10-09T11:09:24.086 INFO:tasks.workunit.client.0.smithi125.stdout:OK 2023-10-09T11:09:24.086 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm -rf /tmp/rbd_import_export_475522 2023-10-09T11:09:24.087 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T11:09:24.087 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T11:09:24.114 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:24.143 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-09T11:09:24.143 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T11:09:24.406 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:09:24.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:24.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:09:24.468 DEBUG:teuthology.parallel:result is None 2023-10-09T11:09:24.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T11:09:24.498 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T11:09:24.498 DEBUG:teuthology.orchestra.run.smithi125:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T11:09:24.511 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T11:09:24.539 INFO:teuthology.orchestra.run.smithi155.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T11:09:24.558 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T11:09:24.558 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T11:09:24.558 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-09T11:09:24.558 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T11:09:24.559 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T11:09:24.559 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T11:09:24.559 DEBUG:teuthology.orchestra.run.smithi125:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T11:09:24.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T11:09:24.616 INFO:teuthology.orchestra.run.smithi125.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T11:09:24.617 DEBUG:teuthology.orchestra.run.smithi125:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T11:09:24.659 INFO:teuthology.orchestra.run.smithi155.stderr:Error: no container with name or ID "ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4" found: no such container 2023-10-09T11:09:24.662 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:24 smithi155 ceph-mon[184854]: failed to encode map e592 with expected crc 2023-10-09T11:09:24.663 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:24 smithi155 ceph-mon[184854]: failed to encode map e592 with expected crc 2023-10-09T11:09:24.663 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:24 smithi155 ceph-mon[184854]: failed to encode map e592 with expected crc 2023-10-09T11:09:24.663 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:24 smithi155 ceph-mon[184854]: pgmap v1438: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 660 KiB/s rd, 468 KiB/s wr, 282 op/s 2023-10-09T11:09:24.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T11:09:24.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 125: 'sudo docker kill -s 1 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T11:09:24.674 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T11:09:24.675 DEBUG:teuthology.orchestra.run.smithi125:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T11:09:24.676 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T11:09:24.732 INFO:tasks.thrashosds:joining thrashosds 2023-10-09T11:09:24.741 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:24 smithi125 ceph-mon[195507]: failed to encode map e592 with expected crc 2023-10-09T11:09:24.741 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:24 smithi125 ceph-mon[195507]: failed to encode map e592 with expected crc 2023-10-09T11:09:24.741 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:24 smithi125 ceph-mon[195507]: failed to encode map e592 with expected crc 2023-10-09T11:09:24.742 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:24 smithi125 ceph-mon[195507]: pgmap v1438: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 660 KiB/s rd, 468 KiB/s wr, 282 op/s 2023-10-09T11:09:24.742 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:24 smithi125 ceph-mon[200197]: failed to encode map e592 with expected crc 2023-10-09T11:09:24.742 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:24 smithi125 ceph-mon[200197]: failed to encode map e592 with expected crc 2023-10-09T11:09:24.742 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:24 smithi125 ceph-mon[200197]: failed to encode map e592 with expected crc 2023-10-09T11:09:24.742 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:24 smithi125 ceph-mon[200197]: pgmap v1438: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 660 KiB/s rd, 468 KiB/s wr, 282 op/s 2023-10-09T11:09:24.978 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:09:25.012 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T11:09:25.293 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:25.555 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T11:09:25.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:25 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3593500011' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:09:25.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:25 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3593500011' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:09:25.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:25 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3593500011' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:09:25.926 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:25.927 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:26.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T11:09:26.289 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T11:09:26.457 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:26.555 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:09:26.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:26 smithi125 ceph-mon[195507]: failed to encode map e593 with expected crc 2023-10-09T11:09:26.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:26 smithi125 ceph-mon[195507]: failed to encode map e593 with expected crc 2023-10-09T11:09:26.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:26 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3593500011' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T11:09:26.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:26 smithi125 ceph-mon[195507]: osdmap e593: 8 total, 8 up, 8 in 2023-10-09T11:09:26.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:26 smithi125 ceph-mon[195507]: failed to encode map e593 with expected crc 2023-10-09T11:09:26.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:26 smithi125 ceph-mon[195507]: pgmap v1440: 103 pgs: 103 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.7 MiB/s wr, 445 op/s 2023-10-09T11:09:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:26 smithi125 ceph-mon[200197]: failed to encode map e593 with expected crc 2023-10-09T11:09:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:26 smithi125 ceph-mon[200197]: failed to encode map e593 with expected crc 2023-10-09T11:09:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:26 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3593500011' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T11:09:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:26 smithi125 ceph-mon[200197]: osdmap e593: 8 total, 8 up, 8 in 2023-10-09T11:09:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:26 smithi125 ceph-mon[200197]: failed to encode map e593 with expected crc 2023-10-09T11:09:26.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:26 smithi125 ceph-mon[200197]: pgmap v1440: 103 pgs: 103 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.7 MiB/s wr, 445 op/s 2023-10-09T11:09:26.832 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:26 smithi155 ceph-mon[184854]: failed to encode map e593 with expected crc 2023-10-09T11:09:26.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:26 smithi155 ceph-mon[184854]: failed to encode map e593 with expected crc 2023-10-09T11:09:26.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:26 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3593500011' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T11:09:26.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:26 smithi155 ceph-mon[184854]: osdmap e593: 8 total, 8 up, 8 in 2023-10-09T11:09:26.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:26 smithi155 ceph-mon[184854]: failed to encode map e593 with expected crc 2023-10-09T11:09:26.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:26 smithi155 ceph-mon[184854]: pgmap v1440: 103 pgs: 103 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.7 MiB/s wr, 445 op/s 2023-10-09T11:09:27.566 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T11:09:27.570 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:27.570 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:27.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:27 smithi155 ceph-mon[184854]: failed to encode map e593 with expected crc 2023-10-09T11:09:27.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:27 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4222715430' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:09:27.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:27 smithi125 ceph-mon[200197]: failed to encode map e593 with expected crc 2023-10-09T11:09:27.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:27 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4222715430' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:09:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:27 smithi125 ceph-mon[195507]: failed to encode map e593 with expected crc 2023-10-09T11:09:27.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:27 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4222715430' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T11:09:27.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T11:09:27.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T11:09:28.229 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:28.245 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:09:28.420 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T11:09:28.626 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:28 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4222715430' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:09:28.626 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:28 smithi155 ceph-mon[184854]: osdmap e594: 8 total, 8 up, 8 in 2023-10-09T11:09:28.627 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:28 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/482654303' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:09:28.627 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:28 smithi155 ceph-mon[184854]: pgmap v1442: 103 pgs: 103 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 1.4 MiB/s rd, 2.1 MiB/s wr, 433 op/s 2023-10-09T11:09:28.627 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:28 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/482654303' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T11:09:28.627 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:28 smithi155 ceph-mon[184854]: osdmap e595: 8 total, 8 up, 8 in 2023-10-09T11:09:28.733 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:28 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4222715430' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:09:28.733 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:28 smithi125 ceph-mon[195507]: osdmap e594: 8 total, 8 up, 8 in 2023-10-09T11:09:28.733 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:28 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/482654303' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:09:28.733 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:28 smithi125 ceph-mon[195507]: pgmap v1442: 103 pgs: 103 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 1.4 MiB/s rd, 2.1 MiB/s wr, 433 op/s 2023-10-09T11:09:28.734 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:28 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/482654303' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T11:09:28.734 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:28 smithi125 ceph-mon[195507]: osdmap e595: 8 total, 8 up, 8 in 2023-10-09T11:09:28.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:28 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4222715430' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T11:09:28.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:28 smithi125 ceph-mon[200197]: osdmap e594: 8 total, 8 up, 8 in 2023-10-09T11:09:28.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:28 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/482654303' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:09:28.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:28 smithi125 ceph-mon[200197]: pgmap v1442: 103 pgs: 103 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 1.4 MiB/s rd, 2.1 MiB/s wr, 433 op/s 2023-10-09T11:09:28.735 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:28 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/482654303' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T11:09:28.735 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:28 smithi125 ceph-mon[200197]: osdmap e595: 8 total, 8 up, 8 in 2023-10-09T11:09:28.912 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T11:09:29.147 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:29.147 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:29.407 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:29.498 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T11:09:29.766 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:09:29.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:29 smithi125 ceph-mon[195507]: failed to encode map e594 with expected crc 2023-10-09T11:09:29.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:29 smithi125 ceph-mon[195507]: failed to encode map e594 with expected crc 2023-10-09T11:09:29.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:29 smithi125 ceph-mon[195507]: failed to encode map e594 with expected crc 2023-10-09T11:09:29.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:29 smithi125 ceph-mon[195507]: failed to encode map e595 with expected crc 2023-10-09T11:09:29.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:29 smithi125 ceph-mon[195507]: failed to encode map e595 with expected crc 2023-10-09T11:09:29.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:29 smithi125 ceph-mon[195507]: failed to encode map e594 with expected crc 2023-10-09T11:09:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:29 smithi125 ceph-mon[200197]: failed to encode map e594 with expected crc 2023-10-09T11:09:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:29 smithi125 ceph-mon[200197]: failed to encode map e594 with expected crc 2023-10-09T11:09:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:29 smithi125 ceph-mon[200197]: failed to encode map e594 with expected crc 2023-10-09T11:09:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:29 smithi125 ceph-mon[200197]: failed to encode map e595 with expected crc 2023-10-09T11:09:29.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:29 smithi125 ceph-mon[200197]: failed to encode map e595 with expected crc 2023-10-09T11:09:29.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:29 smithi125 ceph-mon[200197]: failed to encode map e594 with expected crc 2023-10-09T11:09:29.965 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:29 smithi155 ceph-mon[184854]: failed to encode map e594 with expected crc 2023-10-09T11:09:29.965 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:29 smithi155 ceph-mon[184854]: failed to encode map e594 with expected crc 2023-10-09T11:09:29.965 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:29 smithi155 ceph-mon[184854]: failed to encode map e594 with expected crc 2023-10-09T11:09:29.965 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:29 smithi155 ceph-mon[184854]: failed to encode map e595 with expected crc 2023-10-09T11:09:29.965 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:29 smithi155 ceph-mon[184854]: failed to encode map e595 with expected crc 2023-10-09T11:09:29.965 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:29 smithi155 ceph-mon[184854]: failed to encode map e594 with expected crc 2023-10-09T11:09:30.582 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T11:09:30.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:30 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/486305247' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:09:30.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:30 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:09:30.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:30 smithi155 ceph-mon[184854]: pgmap v1444: 103 pgs: 103 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 867 KiB/s rd, 1.5 MiB/s wr, 246 op/s 2023-10-09T11:09:30.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:09:30.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:30 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/486305247' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:09:30.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:30 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:09:30.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:30 smithi125 ceph-mon[195507]: pgmap v1444: 103 pgs: 103 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 867 KiB/s rd, 1.5 MiB/s wr, 246 op/s 2023-10-09T11:09:30.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:09:30.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:30 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/486305247' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:09:30.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:30 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T11:09:30.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:30 smithi125 ceph-mon[200197]: pgmap v1444: 103 pgs: 103 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 867 KiB/s rd, 1.5 MiB/s wr, 246 op/s 2023-10-09T11:09:30.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:09:30.831 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:30.831 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:31.190 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T11:09:31.455 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T11:09:31.566 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:09:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:09:31] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.43.0" 2023-10-09T11:09:31.578 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T11:09:31.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:31 smithi125 ceph-mon[200197]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:09:31.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:31 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T11:09:31.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:31 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:09:31.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:31 smithi125 ceph-mon[200197]: osdmap e596: 8 total, 8 up, 8 in 2023-10-09T11:09:31.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3077279524' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:09:31.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:31 smithi125 ceph-mon[195507]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:09:31.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:31 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T11:09:31.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:31 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:09:31.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:31 smithi125 ceph-mon[195507]: osdmap e596: 8 total, 8 up, 8 in 2023-10-09T11:09:31.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3077279524' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:09:31.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:31 smithi155 ceph-mon[184854]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T11:09:31.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:31 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T11:09:31.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:31 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T11:09:31.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:31 smithi155 ceph-mon[184854]: osdmap e596: 8 total, 8 up, 8 in 2023-10-09T11:09:31.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3077279524' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:09:31.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T11:09:32.235 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:32.439 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:32.439 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:32.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:32 smithi125 ceph-mon[195507]: failed to encode map e596 with expected crc 2023-10-09T11:09:32.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:32 smithi125 ceph-mon[195507]: failed to encode map e596 with expected crc 2023-10-09T11:09:32.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:32 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3077279524' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T11:09:32.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:32 smithi125 ceph-mon[195507]: osdmap e597: 8 total, 8 up, 8 in 2023-10-09T11:09:32.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:32 smithi125 ceph-mon[195507]: pgmap v1447: 103 pgs: 103 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 41 KiB/s rd, 1.4 KiB/s wr, 57 op/s 2023-10-09T11:09:32.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:32 smithi125 ceph-mon[195507]: failed to encode map e596 with expected crc 2023-10-09T11:09:32.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:32 smithi125 ceph-mon[200197]: failed to encode map e596 with expected crc 2023-10-09T11:09:32.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:32 smithi125 ceph-mon[200197]: failed to encode map e596 with expected crc 2023-10-09T11:09:32.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:32 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3077279524' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T11:09:32.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:32 smithi125 ceph-mon[200197]: osdmap e597: 8 total, 8 up, 8 in 2023-10-09T11:09:32.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:32 smithi125 ceph-mon[200197]: pgmap v1447: 103 pgs: 103 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 41 KiB/s rd, 1.4 KiB/s wr, 57 op/s 2023-10-09T11:09:32.666 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:32 smithi125 ceph-mon[200197]: failed to encode map e596 with expected crc 2023-10-09T11:09:32.814 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T11:09:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:32 smithi155 ceph-mon[184854]: failed to encode map e596 with expected crc 2023-10-09T11:09:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:32 smithi155 ceph-mon[184854]: failed to encode map e596 with expected crc 2023-10-09T11:09:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:32 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3077279524' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T11:09:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:32 smithi155 ceph-mon[184854]: osdmap e597: 8 total, 8 up, 8 in 2023-10-09T11:09:32.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:32 smithi155 ceph-mon[184854]: pgmap v1447: 103 pgs: 103 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 41 KiB/s rd, 1.4 KiB/s wr, 57 op/s 2023-10-09T11:09:32.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:32 smithi155 ceph-mon[184854]: failed to encode map e596 with expected crc 2023-10-09T11:09:33.195 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:33 smithi155 ceph-mon[184854]: failed to encode map e597 with expected crc 2023-10-09T11:09:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:33 smithi155 ceph-mon[184854]: failed to encode map e596 with expected crc 2023-10-09T11:09:33.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:33 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2538744181' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:09:33.989 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:33 smithi125 ceph-mon[195507]: failed to encode map e597 with expected crc 2023-10-09T11:09:33.990 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:33 smithi125 ceph-mon[195507]: failed to encode map e596 with expected crc 2023-10-09T11:09:33.990 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:33 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2538744181' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:09:33.990 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:33 smithi125 ceph-mon[200197]: failed to encode map e597 with expected crc 2023-10-09T11:09:33.990 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:33 smithi125 ceph-mon[200197]: failed to encode map e596 with expected crc 2023-10-09T11:09:33.990 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:33 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2538744181' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:09:34.188 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:34.188 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:34.423 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T11:09:34.813 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T11:09:34.870 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T11:09:35.092 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:35.704 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:35 smithi125 ceph-mon[195507]: pgmap v1448: 103 pgs: 103 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 30 KiB/s rd, 1023 B/s wr, 41 op/s 2023-10-09T11:09:35.704 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:35 smithi125 ceph-mon[195507]: failed to encode map e598 with expected crc 2023-10-09T11:09:35.705 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:35 smithi125 ceph-mon[195507]: failed to encode map e598 with expected crc 2023-10-09T11:09:35.705 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:35 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2538744181' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T11:09:35.705 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:35 smithi125 ceph-mon[195507]: osdmap e598: 8 total, 8 up, 8 in 2023-10-09T11:09:35.705 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:35 smithi125 ceph-mon[200197]: pgmap v1448: 103 pgs: 103 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 30 KiB/s rd, 1023 B/s wr, 41 op/s 2023-10-09T11:09:35.705 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:35 smithi125 ceph-mon[200197]: failed to encode map e598 with expected crc 2023-10-09T11:09:35.705 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:35 smithi125 ceph-mon[200197]: failed to encode map e598 with expected crc 2023-10-09T11:09:35.706 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:35 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2538744181' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T11:09:35.706 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:35 smithi125 ceph-mon[200197]: osdmap e598: 8 total, 8 up, 8 in 2023-10-09T11:09:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:35 smithi155 ceph-mon[184854]: pgmap v1448: 103 pgs: 103 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 30 KiB/s rd, 1023 B/s wr, 41 op/s 2023-10-09T11:09:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:35 smithi155 ceph-mon[184854]: failed to encode map e598 with expected crc 2023-10-09T11:09:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:35 smithi155 ceph-mon[184854]: failed to encode map e598 with expected crc 2023-10-09T11:09:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:35 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2538744181' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T11:09:35.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:35 smithi155 ceph-mon[184854]: osdmap e598: 8 total, 8 up, 8 in 2023-10-09T11:09:35.987 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:36.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:36 smithi125 ceph-mon[195507]: failed to encode map e598 with expected crc 2023-10-09T11:09:36.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:36 smithi125 ceph-mon[195507]: failed to encode map e598 with expected crc 2023-10-09T11:09:36.667 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:36 smithi125 ceph-mon[200197]: failed to encode map e598 with expected crc 2023-10-09T11:09:36.667 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:36 smithi125 ceph-mon[200197]: failed to encode map e598 with expected crc 2023-10-09T11:09:36.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:36 smithi155 ceph-mon[184854]: failed to encode map e598 with expected crc 2023-10-09T11:09:36.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:36 smithi155 ceph-mon[184854]: failed to encode map e598 with expected crc 2023-10-09T11:09:37.004 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:37.004 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:37.386 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T11:09:37.652 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:37.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:37 smithi125 ceph-mon[195507]: pgmap v1450: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:37.667 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:37 smithi125 ceph-mon[200197]: pgmap v1450: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:37.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:37 smithi155 ceph-mon[184854]: pgmap v1450: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:38.439 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T11:09:38.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:38 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3494167187' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:09:38.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:38 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:09:38.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:38 smithi125 ceph-mon[195507]: pgmap v1451: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:38.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:38 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3494167187' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:09:38.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:38 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:09:38.488 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:38 smithi125 ceph-mon[200197]: pgmap v1451: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:38.690 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:38.690 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:38.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:38 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3494167187' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:09:38.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:38 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:09:38.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:38 smithi155 ceph-mon[184854]: pgmap v1451: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:38.819 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T11:09:38.819 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:09:39.132 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T11:09:39.140 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:39.606 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:39.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:39 smithi155 ceph-mon[184854]: failed to encode map e599 with expected crc 2023-10-09T11:09:39.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:39 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T11:09:39.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:39 smithi155 ceph-mon[184854]: osdmap e599: 8 total, 8 up, 8 in 2023-10-09T11:09:39.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:39 smithi125 ceph-mon[195507]: failed to encode map e599 with expected crc 2023-10-09T11:09:39.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:39 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T11:09:39.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:39 smithi125 ceph-mon[195507]: osdmap e599: 8 total, 8 up, 8 in 2023-10-09T11:09:39.729 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:39 smithi125 ceph-mon[200197]: failed to encode map e599 with expected crc 2023-10-09T11:09:39.729 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:39 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T11:09:39.729 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:39 smithi125 ceph-mon[200197]: osdmap e599: 8 total, 8 up, 8 in 2023-10-09T11:09:40.531 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:40 smithi125 ceph-mon[195507]: failed to encode map e599 with expected crc 2023-10-09T11:09:40.531 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:40 smithi125 ceph-mon[195507]: failed to encode map e599 with expected crc 2023-10-09T11:09:40.531 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:40 smithi125 ceph-mon[195507]: failed to encode map e599 with expected crc 2023-10-09T11:09:40.531 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:40 smithi125 ceph-mon[195507]: failed to encode map e599 with expected crc 2023-10-09T11:09:40.531 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:40 smithi125 ceph-mon[195507]: pgmap v1453: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:40.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:40 smithi125 ceph-mon[200197]: failed to encode map e599 with expected crc 2023-10-09T11:09:40.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:40 smithi125 ceph-mon[200197]: failed to encode map e599 with expected crc 2023-10-09T11:09:40.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:40 smithi125 ceph-mon[200197]: failed to encode map e599 with expected crc 2023-10-09T11:09:40.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:40 smithi125 ceph-mon[200197]: failed to encode map e599 with expected crc 2023-10-09T11:09:40.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:40 smithi125 ceph-mon[200197]: pgmap v1453: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:40 smithi155 ceph-mon[184854]: failed to encode map e599 with expected crc 2023-10-09T11:09:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:40 smithi155 ceph-mon[184854]: failed to encode map e599 with expected crc 2023-10-09T11:09:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:40 smithi155 ceph-mon[184854]: failed to encode map e599 with expected crc 2023-10-09T11:09:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:40 smithi155 ceph-mon[184854]: failed to encode map e599 with expected crc 2023-10-09T11:09:40.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:40 smithi155 ceph-mon[184854]: pgmap v1453: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:40.727 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:40.728 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:40.745 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:09:40.746 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:09:41.281 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1453,"stamp":"2023-10-09T11:09:39.942889+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5995,"num_read_kb":13244,"num_write":907,"num_write_kb":12741,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":2377214,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":311,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7164940,"kb_used_data":100108,"kb_used_omap":134,"kb_used_meta":7064633,"kb_avail":742796276,"statfs":{"total":767960285184,"available":760623386624,"internally_reserved":0,"allocated":102510592,"data_stored":82427297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":137532,"internal_metadata":7234184900},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002107"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":883,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576461+0000","last_change":"2023-10-09T11:07:36.287455+0000","last_active":"2023-10-09T11:09:31.576461+0000","last_peered":"2023-10-09T11:09:31.576461+0000","last_clean":"2023-10-09T11:09:31.576461+0000","last_became_active":"2023-10-09T11:07:36.287052+0000","last_became_peered":"2023-10-09T11:07:36.287052+0000","last_unstale":"2023-10-09T11:09:31.576461+0000","last_undegraded":"2023-10-09T11:09:31.576461+0000","last_fullsized":"2023-10-09T11:09:31.576461+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:18.864873+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":379,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.104334+0000","last_change":"2023-10-09T11:03:21.185383+0000","last_active":"2023-10-09T11:09:32.104334+0000","last_peered":"2023-10-09T11:09:32.104334+0000","last_clean":"2023-10-09T11:09:32.104334+0000","last_became_active":"2023-10-09T11:03:21.185022+0000","last_became_peered":"2023-10-09T11:03:21.185022+0000","last_unstale":"2023-10-09T11:09:32.104334+0000","last_undegraded":"2023-10-09T11:09:32.104334+0000","last_fullsized":"2023-10-09T11:09:32.104334+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:05.230434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":124,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575492+0000","last_change":"2023-10-09T11:04:50.319477+0000","last_active":"2023-10-09T11:09:31.575492+0000","last_peered":"2023-10-09T11:09:31.575492+0000","last_clean":"2023-10-09T11:09:31.575492+0000","last_became_active":"2023-10-09T11:04:50.319148+0000","last_became_peered":"2023-10-09T11:04:50.319148+0000","last_unstale":"2023-10-09T11:09:31.575492+0000","last_undegraded":"2023-10-09T11:09:31.575492+0000","last_fullsized":"2023-10-09T11:09:31.575492+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:48.725566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":753,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576528+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:09:31.576528+0000","last_peered":"2023-10-09T11:09:31.576528+0000","last_clean":"2023-10-09T11:09:31.576528+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:09:31.576528+0000","last_undegraded":"2023-10-09T11:09:31.576528+0000","last_fullsized":"2023-10-09T11:09:31.576528+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":312,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.099852+0000","last_change":"2023-10-09T11:04:50.333711+0000","last_active":"2023-10-09T11:09:32.099852+0000","last_peered":"2023-10-09T11:09:32.099852+0000","last_clean":"2023-10-09T11:09:32.099852+0000","last_became_active":"2023-10-09T11:04:50.333262+0000","last_became_peered":"2023-10-09T11:04:50.333262+0000","last_unstale":"2023-10-09T11:09:32.099852+0000","last_undegraded":"2023-10-09T11:09:32.099852+0000","last_fullsized":"2023-10-09T11:09:32.099852+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.843765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":58,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.103236+0000","last_change":"2023-10-09T11:07:33.237903+0000","last_active":"2023-10-09T11:09:32.103236+0000","last_peered":"2023-10-09T11:09:32.103236+0000","last_clean":"2023-10-09T11:09:32.103236+0000","last_became_active":"2023-10-09T11:07:33.237618+0000","last_became_peered":"2023-10-09T11:07:33.237618+0000","last_unstale":"2023-10-09T11:09:32.103236+0000","last_undegraded":"2023-10-09T11:09:32.103236+0000","last_fullsized":"2023-10-09T11:09:32.103236+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:25.600818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":773,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.102509+0000","last_change":"2023-10-09T11:07:36.284534+0000","last_active":"2023-10-09T11:09:32.102509+0000","last_peered":"2023-10-09T11:09:32.102509+0000","last_clean":"2023-10-09T11:09:32.102509+0000","last_became_active":"2023-10-09T11:07:36.284254+0000","last_became_peered":"2023-10-09T11:07:36.284254+0000","last_unstale":"2023-10-09T11:09:32.102509+0000","last_undegraded":"2023-10-09T11:09:32.102509+0000","last_fullsized":"2023-10-09T11:09:32.102509+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:19.159630+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":124,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.574900+0000","last_change":"2023-10-09T11:04:50.310477+0000","last_active":"2023-10-09T11:09:31.574900+0000","last_peered":"2023-10-09T11:09:31.574900+0000","last_clean":"2023-10-09T11:09:31.574900+0000","last_became_active":"2023-10-09T11:04:50.310067+0000","last_became_peered":"2023-10-09T11:04:50.310067+0000","last_unstale":"2023-10-09T11:09:31.574900+0000","last_undegraded":"2023-10-09T11:09:31.574900+0000","last_fullsized":"2023-10-09T11:09:31.574900+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:52.884716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":405,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.577141+0000","last_change":"2023-10-09T11:05:20.310581+0000","last_active":"2023-10-09T11:09:31.577141+0000","last_peered":"2023-10-09T11:09:31.577141+0000","last_clean":"2023-10-09T11:09:31.577141+0000","last_became_active":"2023-10-09T11:05:20.309830+0000","last_became_peered":"2023-10-09T11:05:20.309830+0000","last_unstale":"2023-10-09T11:09:31.577141+0000","last_undegraded":"2023-10-09T11:09:31.577141+0000","last_fullsized":"2023-10-09T11:09:31.577141+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:24.508211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":813,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.103300+0000","last_change":"2023-10-09T11:07:36.282502+0000","last_active":"2023-10-09T11:09:32.103300+0000","last_peered":"2023-10-09T11:09:32.103300+0000","last_clean":"2023-10-09T11:09:32.103300+0000","last_became_active":"2023-10-09T11:07:36.282133+0000","last_became_peered":"2023-10-09T11:07:36.282133+0000","last_unstale":"2023-10-09T11:09:32.103300+0000","last_undegraded":"2023-10-09T11:09:32.103300+0000","last_fullsized":"2023-10-09T11:09:32.103300+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:03.762383+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":232,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.577249+0000","last_change":"2023-10-09T11:06:24.245488+0000","last_active":"2023-10-09T11:09:31.577249+0000","last_peered":"2023-10-09T11:09:31.577249+0000","last_clean":"2023-10-09T11:09:31.577249+0000","last_became_active":"2023-10-09T11:06:24.245167+0000","last_became_peered":"2023-10-09T11:06:24.245167+0000","last_unstale":"2023-10-09T11:09:31.577249+0000","last_undegraded":"2023-10-09T11:09:31.577249+0000","last_fullsized":"2023-10-09T11:09:31.577249+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:31.702568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":134,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:33.352991+0000","last_change":"2023-10-09T11:05:20.299453+0000","last_active":"2023-10-09T11:09:33.352991+0000","last_peered":"2023-10-09T11:09:33.352991+0000","last_clean":"2023-10-09T11:09:33.352991+0000","last_became_active":"2023-10-09T11:05:20.299117+0000","last_became_peered":"2023-10-09T11:05:20.299117+0000","last_unstale":"2023-10-09T11:09:33.352991+0000","last_undegraded":"2023-10-09T11:09:33.352991+0000","last_fullsized":"2023-10-09T11:09:33.352991+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:30.072813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":928,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100195+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:09:32.100195+0000","last_peered":"2023-10-09T11:09:32.100195+0000","last_clean":"2023-10-09T11:09:32.100195+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:09:32.100195+0000","last_undegraded":"2023-10-09T11:09:32.100195+0000","last_fullsized":"2023-10-09T11:09:32.100195+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":467,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576995+0000","last_change":"2023-10-09T11:07:33.242665+0000","last_active":"2023-10-09T11:09:31.576995+0000","last_peered":"2023-10-09T11:09:31.576995+0000","last_clean":"2023-10-09T11:09:31.576995+0000","last_became_active":"2023-10-09T11:07:33.242313+0000","last_became_peered":"2023-10-09T11:07:33.242313+0000","last_unstale":"2023-10-09T11:09:31.576995+0000","last_undegraded":"2023-10-09T11:09:31.576995+0000","last_fullsized":"2023-10-09T11:09:31.576995+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:20.565135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":152,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:33.353046+0000","last_change":"2023-10-09T11:07:33.225683+0000","last_active":"2023-10-09T11:09:33.353046+0000","last_peered":"2023-10-09T11:09:33.353046+0000","last_clean":"2023-10-09T11:09:33.353046+0000","last_became_active":"2023-10-09T11:07:33.225275+0000","last_became_peered":"2023-10-09T11:07:33.225275+0000","last_unstale":"2023-10-09T11:09:33.353046+0000","last_undegraded":"2023-10-09T11:09:33.353046+0000","last_fullsized":"2023-10-09T11:09:33.353046+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:10.131060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":825,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.577031+0000","last_change":"2023-10-09T11:05:23.328981+0000","last_active":"2023-10-09T11:09:31.577031+0000","last_peered":"2023-10-09T11:09:31.577031+0000","last_clean":"2023-10-09T11:09:31.577031+0000","last_became_active":"2023-10-09T11:05:23.328616+0000","last_became_peered":"2023-10-09T11:05:23.328616+0000","last_unstale":"2023-10-09T11:09:31.577031+0000","last_undegraded":"2023-10-09T11:09:31.577031+0000","last_fullsized":"2023-10-09T11:09:31.577031+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:15.315819+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":64,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.103139+0000","last_change":"2023-10-09T11:07:33.265842+0000","last_active":"2023-10-09T11:09:32.103139+0000","last_peered":"2023-10-09T11:09:32.103139+0000","last_clean":"2023-10-09T11:09:32.103139+0000","last_became_active":"2023-10-09T11:07:33.232311+0000","last_became_peered":"2023-10-09T11:07:33.232311+0000","last_unstale":"2023-10-09T11:09:32.103139+0000","last_undegraded":"2023-10-09T11:09:32.103139+0000","last_fullsized":"2023-10-09T11:09:32.103139+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:23.402582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":374,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576529+0000","last_change":"2023-10-09T11:05:20.316265+0000","last_active":"2023-10-09T11:09:31.576529+0000","last_peered":"2023-10-09T11:09:31.576529+0000","last_clean":"2023-10-09T11:09:31.576529+0000","last_became_active":"2023-10-09T11:05:20.315853+0000","last_became_peered":"2023-10-09T11:05:20.315853+0000","last_unstale":"2023-10-09T11:09:31.576529+0000","last_undegraded":"2023-10-09T11:09:31.576529+0000","last_fullsized":"2023-10-09T11:09:31.576529+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:35.093981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":990,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100956+0000","last_change":"2023-10-09T11:04:53.274921+0000","last_active":"2023-10-09T11:09:32.100956+0000","last_peered":"2023-10-09T11:09:32.100956+0000","last_clean":"2023-10-09T11:09:32.100956+0000","last_became_active":"2023-10-09T11:04:53.274518+0000","last_became_peered":"2023-10-09T11:04:53.274518+0000","last_unstale":"2023-10-09T11:09:32.100956+0000","last_undegraded":"2023-10-09T11:09:32.100956+0000","last_fullsized":"2023-10-09T11:09:32.100956+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:14.315301+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":68,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.102853+0000","last_change":"2023-10-09T11:07:33.262559+0000","last_active":"2023-10-09T11:09:32.102853+0000","last_peered":"2023-10-09T11:09:32.102853+0000","last_clean":"2023-10-09T11:09:32.102853+0000","last_became_active":"2023-10-09T11:07:33.231695+0000","last_became_peered":"2023-10-09T11:07:33.231695+0000","last_unstale":"2023-10-09T11:09:32.102853+0000","last_undegraded":"2023-10-09T11:09:32.102853+0000","last_fullsized":"2023-10-09T11:09:32.102853+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:59.006940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":619,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100620+0000","last_change":"2023-10-09T11:05:20.311424+0000","last_active":"2023-10-09T11:09:32.100620+0000","last_peered":"2023-10-09T11:09:32.100620+0000","last_clean":"2023-10-09T11:09:32.100620+0000","last_became_active":"2023-10-09T11:05:20.311097+0000","last_became_peered":"2023-10-09T11:05:20.311097+0000","last_unstale":"2023-10-09T11:09:32.100620+0000","last_undegraded":"2023-10-09T11:09:32.100620+0000","last_fullsized":"2023-10-09T11:09:32.100620+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:19.274582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":638,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576600+0000","last_change":"2023-10-09T11:08:39.934938+0000","last_active":"2023-10-09T11:09:31.576600+0000","last_peered":"2023-10-09T11:09:31.576600+0000","last_clean":"2023-10-09T11:09:31.576600+0000","last_became_active":"2023-10-09T11:06:24.245076+0000","last_became_peered":"2023-10-09T11:06:24.245076+0000","last_unstale":"2023-10-09T11:09:31.576600+0000","last_undegraded":"2023-10-09T11:09:31.576600+0000","last_fullsized":"2023-10-09T11:09:31.576600+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.934865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:00.408009+0000","scrub_duration":0.0012831839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":64,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.102796+0000","last_change":"2023-10-09T11:07:33.269200+0000","last_active":"2023-10-09T11:09:32.102796+0000","last_peered":"2023-10-09T11:09:32.102796+0000","last_clean":"2023-10-09T11:09:32.102796+0000","last_became_active":"2023-10-09T11:07:33.236205+0000","last_became_peered":"2023-10-09T11:07:33.236205+0000","last_unstale":"2023-10-09T11:09:32.102796+0000","last_undegraded":"2023-10-09T11:09:32.102796+0000","last_fullsized":"2023-10-09T11:09:32.102796+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:24.574734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":143,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:33.353156+0000","last_change":"2023-10-09T11:07:33.227803+0000","last_active":"2023-10-09T11:09:33.353156+0000","last_peered":"2023-10-09T11:09:33.353156+0000","last_clean":"2023-10-09T11:09:33.353156+0000","last_became_active":"2023-10-09T11:07:33.227486+0000","last_became_peered":"2023-10-09T11:07:33.227486+0000","last_unstale":"2023-10-09T11:09:33.353156+0000","last_undegraded":"2023-10-09T11:09:33.353156+0000","last_fullsized":"2023-10-09T11:09:33.353156+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:39.274875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":907,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576702+0000","last_change":"2023-10-09T11:05:23.323856+0000","last_active":"2023-10-09T11:09:31.576702+0000","last_peered":"2023-10-09T11:09:31.576702+0000","last_clean":"2023-10-09T11:09:31.576702+0000","last_became_active":"2023-10-09T11:05:23.322763+0000","last_became_peered":"2023-10-09T11:05:23.322763+0000","last_unstale":"2023-10-09T11:09:31.576702+0000","last_undegraded":"2023-10-09T11:09:31.576702+0000","last_fullsized":"2023-10-09T11:09:31.576702+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.760820+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":58,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.102422+0000","last_change":"2023-10-09T11:07:33.233510+0000","last_active":"2023-10-09T11:09:32.102422+0000","last_peered":"2023-10-09T11:09:32.102422+0000","last_clean":"2023-10-09T11:09:32.102422+0000","last_became_active":"2023-10-09T11:07:33.232339+0000","last_became_peered":"2023-10-09T11:07:33.232339+0000","last_unstale":"2023-10-09T11:09:32.102422+0000","last_undegraded":"2023-10-09T11:09:32.102422+0000","last_fullsized":"2023-10-09T11:09:32.102422+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:55.324912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":537,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576982+0000","last_change":"2023-10-09T11:07:33.226594+0000","last_active":"2023-10-09T11:09:31.576982+0000","last_peered":"2023-10-09T11:09:31.576982+0000","last_clean":"2023-10-09T11:09:31.576982+0000","last_became_active":"2023-10-09T11:07:33.226275+0000","last_became_peered":"2023-10-09T11:07:33.226275+0000","last_unstale":"2023-10-09T11:09:31.576982+0000","last_undegraded":"2023-10-09T11:09:31.576982+0000","last_fullsized":"2023-10-09T11:09:31.576982+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:41.180088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":932,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:33.388720+0000","last_change":"2023-10-09T11:04:53.274236+0000","last_active":"2023-10-09T11:09:33.388720+0000","last_peered":"2023-10-09T11:09:33.388720+0000","last_clean":"2023-10-09T11:09:33.388720+0000","last_became_active":"2023-10-09T11:04:53.273817+0000","last_became_peered":"2023-10-09T11:04:53.273817+0000","last_unstale":"2023-10-09T11:09:33.388720+0000","last_undegraded":"2023-10-09T11:09:33.388720+0000","last_fullsized":"2023-10-09T11:09:33.388720+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:05.096905+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":193,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.102442+0000","last_change":"2023-10-09T11:05:20.312129+0000","last_active":"2023-10-09T11:09:32.102442+0000","last_peered":"2023-10-09T11:09:32.102442+0000","last_clean":"2023-10-09T11:09:32.102442+0000","last_became_active":"2023-10-09T11:05:20.311810+0000","last_became_peered":"2023-10-09T11:05:20.311810+0000","last_unstale":"2023-10-09T11:09:32.102442+0000","last_undegraded":"2023-10-09T11:09:32.102442+0000","last_fullsized":"2023-10-09T11:09:32.102442+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:48.813391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":292,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575800+0000","last_change":"2023-10-09T11:07:33.218244+0000","last_active":"2023-10-09T11:09:31.575800+0000","last_peered":"2023-10-09T11:09:31.575800+0000","last_clean":"2023-10-09T11:09:31.575800+0000","last_became_active":"2023-10-09T11:07:33.217918+0000","last_became_peered":"2023-10-09T11:07:33.217918+0000","last_unstale":"2023-10-09T11:09:31.575800+0000","last_undegraded":"2023-10-09T11:09:31.575800+0000","last_fullsized":"2023-10-09T11:09:31.575800+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:16.150105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":904,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576371+0000","last_change":"2023-10-09T11:05:23.324537+0000","last_active":"2023-10-09T11:09:31.576371+0000","last_peered":"2023-10-09T11:09:31.576371+0000","last_clean":"2023-10-09T11:09:31.576371+0000","last_became_active":"2023-10-09T11:05:23.324187+0000","last_became_peered":"2023-10-09T11:05:23.324187+0000","last_unstale":"2023-10-09T11:09:31.576371+0000","last_undegraded":"2023-10-09T11:09:31.576371+0000","last_fullsized":"2023-10-09T11:09:31.576371+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:25.286106+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":116,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.102839+0000","last_change":"2023-10-09T11:05:20.323565+0000","last_active":"2023-10-09T11:09:32.102839+0000","last_peered":"2023-10-09T11:09:32.102839+0000","last_clean":"2023-10-09T11:09:32.102839+0000","last_became_active":"2023-10-09T11:05:20.309672+0000","last_became_peered":"2023-10-09T11:05:20.309672+0000","last_unstale":"2023-10-09T11:09:32.102839+0000","last_undegraded":"2023-10-09T11:09:32.102839+0000","last_fullsized":"2023-10-09T11:09:32.102839+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:50.211465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":142,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576662+0000","last_change":"2023-10-09T11:07:33.228001+0000","last_active":"2023-10-09T11:09:31.576662+0000","last_peered":"2023-10-09T11:09:31.576662+0000","last_clean":"2023-10-09T11:09:31.576662+0000","last_became_active":"2023-10-09T11:07:33.227691+0000","last_became_peered":"2023-10-09T11:07:33.227691+0000","last_unstale":"2023-10-09T11:09:31.576662+0000","last_undegraded":"2023-10-09T11:09:31.576662+0000","last_fullsized":"2023-10-09T11:09:31.576662+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:24:16.098138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":830,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.103235+0000","last_change":"2023-10-09T11:07:36.284927+0000","last_active":"2023-10-09T11:09:32.103235+0000","last_peered":"2023-10-09T11:09:32.103235+0000","last_clean":"2023-10-09T11:09:32.103235+0000","last_became_active":"2023-10-09T11:07:36.284631+0000","last_became_peered":"2023-10-09T11:07:36.284631+0000","last_unstale":"2023-10-09T11:09:32.103235+0000","last_undegraded":"2023-10-09T11:09:32.103235+0000","last_fullsized":"2023-10-09T11:09:32.103235+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:37.220243+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":62,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.103418+0000","last_change":"2023-10-09T11:07:33.273503+0000","last_active":"2023-10-09T11:09:32.103418+0000","last_peered":"2023-10-09T11:09:32.103418+0000","last_clean":"2023-10-09T11:09:32.103418+0000","last_became_active":"2023-10-09T11:07:33.242980+0000","last_became_peered":"2023-10-09T11:07:33.242980+0000","last_unstale":"2023-10-09T11:09:32.103418+0000","last_undegraded":"2023-10-09T11:09:32.103418+0000","last_fullsized":"2023-10-09T11:09:32.103418+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:18.934972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":558,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576436+0000","last_change":"2023-10-09T11:07:33.304363+0000","last_active":"2023-10-09T11:09:31.576436+0000","last_peered":"2023-10-09T11:09:31.576436+0000","last_clean":"2023-10-09T11:09:31.576436+0000","last_became_active":"2023-10-09T11:07:33.304024+0000","last_became_peered":"2023-10-09T11:07:33.304024+0000","last_unstale":"2023-10-09T11:09:31.576436+0000","last_undegraded":"2023-10-09T11:09:31.576436+0000","last_fullsized":"2023-10-09T11:09:31.576436+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:21:51.522688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":980,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575743+0000","last_change":"2023-10-09T11:05:23.316129+0000","last_active":"2023-10-09T11:09:31.575743+0000","last_peered":"2023-10-09T11:09:31.575743+0000","last_clean":"2023-10-09T11:09:31.575743+0000","last_became_active":"2023-10-09T11:05:23.315792+0000","last_became_peered":"2023-10-09T11:05:23.315792+0000","last_unstale":"2023-10-09T11:09:31.575743+0000","last_undegraded":"2023-10-09T11:09:31.575743+0000","last_fullsized":"2023-10-09T11:09:31.575743+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:51.770547+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":62,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.103432+0000","last_change":"2023-10-09T11:07:33.256691+0000","last_active":"2023-10-09T11:09:32.103432+0000","last_peered":"2023-10-09T11:09:32.103432+0000","last_clean":"2023-10-09T11:09:32.103432+0000","last_became_active":"2023-10-09T11:07:33.229554+0000","last_became_peered":"2023-10-09T11:07:33.229554+0000","last_unstale":"2023-10-09T11:09:32.103432+0000","last_undegraded":"2023-10-09T11:09:32.103432+0000","last_fullsized":"2023-10-09T11:09:32.103432+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:57.414795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":313,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.099814+0000","last_change":"2023-10-09T11:04:50.333538+0000","last_active":"2023-10-09T11:09:32.099814+0000","last_peered":"2023-10-09T11:09:32.099814+0000","last_clean":"2023-10-09T11:09:32.099814+0000","last_became_active":"2023-10-09T11:04:50.332514+0000","last_became_peered":"2023-10-09T11:04:50.332514+0000","last_unstale":"2023-10-09T11:09:32.099814+0000","last_undegraded":"2023-10-09T11:09:32.099814+0000","last_fullsized":"2023-10-09T11:09:32.099814+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:22.933839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":852,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.577038+0000","last_change":"2023-10-09T11:05:23.322251+0000","last_active":"2023-10-09T11:09:31.577038+0000","last_peered":"2023-10-09T11:09:31.577038+0000","last_clean":"2023-10-09T11:09:31.577038+0000","last_became_active":"2023-10-09T11:05:23.321826+0000","last_became_peered":"2023-10-09T11:05:23.321826+0000","last_unstale":"2023-10-09T11:09:31.577038+0000","last_undegraded":"2023-10-09T11:09:31.577038+0000","last_fullsized":"2023-10-09T11:09:31.577038+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.858013+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":62,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.103395+0000","last_change":"2023-10-09T11:07:33.271238+0000","last_active":"2023-10-09T11:09:32.103395+0000","last_peered":"2023-10-09T11:09:32.103395+0000","last_clean":"2023-10-09T11:09:32.103395+0000","last_became_active":"2023-10-09T11:07:33.241886+0000","last_became_peered":"2023-10-09T11:07:33.241886+0000","last_unstale":"2023-10-09T11:09:32.103395+0000","last_undegraded":"2023-10-09T11:09:32.103395+0000","last_fullsized":"2023-10-09T11:09:32.103395+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:48.882589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":126,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.102352+0000","last_change":"2023-10-09T11:07:33.244131+0000","last_active":"2023-10-09T11:09:32.102352+0000","last_peered":"2023-10-09T11:09:32.102352+0000","last_clean":"2023-10-09T11:09:32.102352+0000","last_became_active":"2023-10-09T11:07:33.243797+0000","last_became_peered":"2023-10-09T11:07:33.243797+0000","last_unstale":"2023-10-09T11:09:32.102352+0000","last_undegraded":"2023-10-09T11:09:32.102352+0000","last_fullsized":"2023-10-09T11:09:32.102352+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:44.851043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":1108,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.103396+0000","last_change":"2023-10-09T11:07:36.286417+0000","last_active":"2023-10-09T11:09:32.103396+0000","last_peered":"2023-10-09T11:09:32.103396+0000","last_clean":"2023-10-09T11:09:32.103396+0000","last_became_active":"2023-10-09T11:07:36.286061+0000","last_became_peered":"2023-10-09T11:07:36.286061+0000","last_unstale":"2023-10-09T11:09:32.103396+0000","last_undegraded":"2023-10-09T11:09:32.103396+0000","last_fullsized":"2023-10-09T11:09:32.103396+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:00.289500+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":454,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576880+0000","last_change":"2023-10-09T11:05:20.309385+0000","last_active":"2023-10-09T11:09:31.576880+0000","last_peered":"2023-10-09T11:09:31.576880+0000","last_clean":"2023-10-09T11:09:31.576880+0000","last_became_active":"2023-10-09T11:05:20.309034+0000","last_became_peered":"2023-10-09T11:05:20.309034+0000","last_unstale":"2023-10-09T11:09:31.576880+0000","last_undegraded":"2023-10-09T11:09:31.576880+0000","last_fullsized":"2023-10-09T11:09:31.576880+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:31.857485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":388,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100071+0000","last_change":"2023-10-09T11:06:24.245966+0000","last_active":"2023-10-09T11:09:32.100071+0000","last_peered":"2023-10-09T11:09:32.100071+0000","last_clean":"2023-10-09T11:09:32.100071+0000","last_became_active":"2023-10-09T11:06:24.245310+0000","last_became_peered":"2023-10-09T11:06:24.245310+0000","last_unstale":"2023-10-09T11:09:32.100071+0000","last_undegraded":"2023-10-09T11:09:32.100071+0000","last_fullsized":"2023-10-09T11:09:32.100071+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:23.966200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":752,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.102310+0000","last_change":"2023-10-09T11:07:36.283174+0000","last_active":"2023-10-09T11:09:32.102310+0000","last_peered":"2023-10-09T11:09:32.102310+0000","last_clean":"2023-10-09T11:09:32.102310+0000","last_became_active":"2023-10-09T11:07:36.278007+0000","last_became_peered":"2023-10-09T11:07:36.278007+0000","last_unstale":"2023-10-09T11:09:32.102310+0000","last_undegraded":"2023-10-09T11:09:32.102310+0000","last_fullsized":"2023-10-09T11:09:32.102310+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:51.693969+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":129,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575547+0000","last_change":"2023-10-09T11:04:50.529513+0000","last_active":"2023-10-09T11:09:31.575547+0000","last_peered":"2023-10-09T11:09:31.575547+0000","last_clean":"2023-10-09T11:09:31.575547+0000","last_became_active":"2023-10-09T11:04:50.339671+0000","last_became_peered":"2023-10-09T11:04:50.339671+0000","last_unstale":"2023-10-09T11:09:31.575547+0000","last_undegraded":"2023-10-09T11:09:31.575547+0000","last_fullsized":"2023-10-09T11:09:31.575547+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:24.473000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":134,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575656+0000","last_change":"2023-10-09T11:05:20.313855+0000","last_active":"2023-10-09T11:09:31.575656+0000","last_peered":"2023-10-09T11:09:31.575656+0000","last_clean":"2023-10-09T11:09:31.575656+0000","last_became_active":"2023-10-09T11:05:20.313535+0000","last_became_peered":"2023-10-09T11:05:20.313535+0000","last_unstale":"2023-10-09T11:09:31.575656+0000","last_undegraded":"2023-10-09T11:09:31.575656+0000","last_fullsized":"2023-10-09T11:09:31.575656+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:25.266434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":854,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576585+0000","last_change":"2023-10-09T11:04:53.272323+0000","last_active":"2023-10-09T11:09:31.576585+0000","last_peered":"2023-10-09T11:09:31.576585+0000","last_clean":"2023-10-09T11:09:31.576585+0000","last_became_active":"2023-10-09T11:04:53.272010+0000","last_became_peered":"2023-10-09T11:04:53.272010+0000","last_unstale":"2023-10-09T11:09:31.576585+0000","last_undegraded":"2023-10-09T11:09:31.576585+0000","last_fullsized":"2023-10-09T11:09:31.576585+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:23.913605+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":622,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100051+0000","last_change":"2023-10-09T11:05:20.307499+0000","last_active":"2023-10-09T11:09:32.100051+0000","last_peered":"2023-10-09T11:09:32.100051+0000","last_clean":"2023-10-09T11:09:32.100051+0000","last_became_active":"2023-10-09T11:05:20.307198+0000","last_became_peered":"2023-10-09T11:05:20.307198+0000","last_unstale":"2023-10-09T11:09:32.100051+0000","last_undegraded":"2023-10-09T11:09:32.100051+0000","last_fullsized":"2023-10-09T11:09:32.100051+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:53.726844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":357,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100036+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:09:32.100036+0000","last_peered":"2023-10-09T11:09:32.100036+0000","last_clean":"2023-10-09T11:09:32.100036+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:09:32.100036+0000","last_undegraded":"2023-10-09T11:09:32.100036+0000","last_fullsized":"2023-10-09T11:09:32.100036+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":949,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100248+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:09:32.100248+0000","last_peered":"2023-10-09T11:09:32.100248+0000","last_clean":"2023-10-09T11:09:32.100248+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:09:32.100248+0000","last_undegraded":"2023-10-09T11:09:32.100248+0000","last_fullsized":"2023-10-09T11:09:32.100248+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":621,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100787+0000","last_change":"2023-10-09T11:04:57.953307+0000","last_active":"2023-10-09T11:09:32.100787+0000","last_peered":"2023-10-09T11:09:32.100787+0000","last_clean":"2023-10-09T11:09:32.100787+0000","last_became_active":"2023-10-09T11:04:57.953095+0000","last_became_peered":"2023-10-09T11:04:57.953095+0000","last_unstale":"2023-10-09T11:09:32.100787+0000","last_undegraded":"2023-10-09T11:09:32.100787+0000","last_fullsized":"2023-10-09T11:09:32.100787+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:05.562643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":307,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100741+0000","last_change":"2023-10-09T11:05:20.310255+0000","last_active":"2023-10-09T11:09:32.100741+0000","last_peered":"2023-10-09T11:09:32.100741+0000","last_clean":"2023-10-09T11:09:32.100741+0000","last_became_active":"2023-10-09T11:05:20.309109+0000","last_became_peered":"2023-10-09T11:05:20.309109+0000","last_unstale":"2023-10-09T11:09:32.100741+0000","last_undegraded":"2023-10-09T11:09:32.100741+0000","last_fullsized":"2023-10-09T11:09:32.100741+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:07.429039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":838,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576398+0000","last_change":"2023-10-09T11:05:23.329329+0000","last_active":"2023-10-09T11:09:31.576398+0000","last_peered":"2023-10-09T11:09:31.576398+0000","last_clean":"2023-10-09T11:09:31.576398+0000","last_became_active":"2023-10-09T11:05:23.328976+0000","last_became_peered":"2023-10-09T11:05:23.328976+0000","last_unstale":"2023-10-09T11:09:31.576398+0000","last_undegraded":"2023-10-09T11:09:31.576398+0000","last_fullsized":"2023-10-09T11:09:31.576398+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:41.848184+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":143,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575601+0000","last_change":"2023-10-09T11:07:33.236133+0000","last_active":"2023-10-09T11:09:31.575601+0000","last_peered":"2023-10-09T11:09:31.575601+0000","last_clean":"2023-10-09T11:09:31.575601+0000","last_became_active":"2023-10-09T11:07:33.235818+0000","last_became_peered":"2023-10-09T11:07:33.235818+0000","last_unstale":"2023-10-09T11:09:31.575601+0000","last_undegraded":"2023-10-09T11:09:31.575601+0000","last_fullsized":"2023-10-09T11:09:31.575601+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:20.627846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":304,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100156+0000","last_change":"2023-10-09T11:04:50.332978+0000","last_active":"2023-10-09T11:09:32.100156+0000","last_peered":"2023-10-09T11:09:32.100156+0000","last_clean":"2023-10-09T11:09:32.100156+0000","last_became_active":"2023-10-09T11:04:50.332735+0000","last_became_peered":"2023-10-09T11:04:50.332735+0000","last_unstale":"2023-10-09T11:09:32.100156+0000","last_undegraded":"2023-10-09T11:09:32.100156+0000","last_fullsized":"2023-10-09T11:09:32.100156+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:39:44.659751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":1125,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576746+0000","last_change":"2023-10-09T11:07:36.291716+0000","last_active":"2023-10-09T11:09:31.576746+0000","last_peered":"2023-10-09T11:09:31.576746+0000","last_clean":"2023-10-09T11:09:31.576746+0000","last_became_active":"2023-10-09T11:07:36.291424+0000","last_became_peered":"2023-10-09T11:07:36.291424+0000","last_unstale":"2023-10-09T11:09:31.576746+0000","last_undegraded":"2023-10-09T11:09:31.576746+0000","last_fullsized":"2023-10-09T11:09:31.576746+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:22.316387+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"592'150","reported_seq":883,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100251+0000","last_change":"2023-10-09T11:09:20.424914+0000","last_active":"2023-10-09T11:09:32.100251+0000","last_peered":"2023-10-09T11:09:32.100251+0000","last_clean":"2023-10-09T11:09:32.100251+0000","last_became_active":"2023-10-09T11:07:33.226176+0000","last_became_peered":"2023-10-09T11:07:33.226176+0000","last_unstale":"2023-10-09T11:09:32.100251+0000","last_undegraded":"2023-10-09T11:09:32.100251+0000","last_fullsized":"2023-10-09T11:09:32.100251+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:16.888732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034113299999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":876,"num_read_kb":2947,"num_write":148,"num_write_kb":2179,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":471,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576974+0000","last_change":"2023-10-09T11:04:50.341353+0000","last_active":"2023-10-09T11:09:31.576974+0000","last_peered":"2023-10-09T11:09:31.576974+0000","last_clean":"2023-10-09T11:09:31.576974+0000","last_became_active":"2023-10-09T11:04:50.340801+0000","last_became_peered":"2023-10-09T11:04:50.340801+0000","last_unstale":"2023-10-09T11:09:31.576974+0000","last_undegraded":"2023-10-09T11:09:31.576974+0000","last_fullsized":"2023-10-09T11:09:31.576974+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:00.699069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":280,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100219+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:09:32.100219+0000","last_peered":"2023-10-09T11:09:32.100219+0000","last_clean":"2023-10-09T11:09:32.100219+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:09:32.100219+0000","last_undegraded":"2023-10-09T11:09:32.100219+0000","last_fullsized":"2023-10-09T11:09:32.100219+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":796,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576376+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:09:31.576376+0000","last_peered":"2023-10-09T11:09:31.576376+0000","last_clean":"2023-10-09T11:09:31.576376+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:09:31.576376+0000","last_undegraded":"2023-10-09T11:09:31.576376+0000","last_fullsized":"2023-10-09T11:09:31.576376+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"592'149","reported_seq":1213,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576592+0000","last_change":"2023-10-09T11:09:20.477238+0000","last_active":"2023-10-09T11:09:31.576592+0000","last_peered":"2023-10-09T11:09:31.576592+0000","last_clean":"2023-10-09T11:09:31.576592+0000","last_became_active":"2023-10-09T11:05:20.315471+0000","last_became_peered":"2023-10-09T11:05:20.315471+0000","last_unstale":"2023-10-09T11:09:31.576592+0000","last_undegraded":"2023-10-09T11:09:31.576592+0000","last_fullsized":"2023-10-09T11:09:31.576592+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:47.548763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051877200000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1181,"num_read_kb":1421,"num_write":125,"num_write_kb":1278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":502,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.102206+0000","last_change":"2023-10-09T11:06:24.245282+0000","last_active":"2023-10-09T11:09:32.102206+0000","last_peered":"2023-10-09T11:09:32.102206+0000","last_clean":"2023-10-09T11:09:32.102206+0000","last_became_active":"2023-10-09T11:06:24.244981+0000","last_became_peered":"2023-10-09T11:06:24.244981+0000","last_unstale":"2023-10-09T11:09:32.102206+0000","last_undegraded":"2023-10-09T11:09:32.102206+0000","last_fullsized":"2023-10-09T11:09:32.102206+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:15.327655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":313,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100020+0000","last_change":"2023-10-09T11:04:50.311579+0000","last_active":"2023-10-09T11:09:32.100020+0000","last_peered":"2023-10-09T11:09:32.100020+0000","last_clean":"2023-10-09T11:09:32.100020+0000","last_became_active":"2023-10-09T11:04:50.311270+0000","last_became_peered":"2023-10-09T11:04:50.311270+0000","last_unstale":"2023-10-09T11:09:32.100020+0000","last_undegraded":"2023-10-09T11:09:32.100020+0000","last_fullsized":"2023-10-09T11:09:32.100020+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:49.850302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":950,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.102616+0000","last_change":"2023-10-09T11:07:36.284280+0000","last_active":"2023-10-09T11:09:32.102616+0000","last_peered":"2023-10-09T11:09:32.102616+0000","last_clean":"2023-10-09T11:09:32.102616+0000","last_became_active":"2023-10-09T11:07:36.283966+0000","last_became_peered":"2023-10-09T11:07:36.283966+0000","last_unstale":"2023-10-09T11:09:32.102616+0000","last_undegraded":"2023-10-09T11:09:32.102616+0000","last_fullsized":"2023-10-09T11:09:32.102616+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:54.128331+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"591'48","reported_seq":609,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100919+0000","last_change":"2023-10-09T11:09:20.421917+0000","last_active":"2023-10-09T11:09:32.100919+0000","last_peered":"2023-10-09T11:09:32.100919+0000","last_clean":"2023-10-09T11:09:32.100919+0000","last_became_active":"2023-10-09T11:04:50.381253+0000","last_became_peered":"2023-10-09T11:04:50.381253+0000","last_unstale":"2023-10-09T11:09:32.100919+0000","last_undegraded":"2023-10-09T11:09:32.100919+0000","last_fullsized":"2023-10-09T11:09:32.100919+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":48,"log_dups_size":0,"ondisk_log_size":48,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:24.030274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001144136,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":297,"num_read_kb":2229,"num_write":37,"num_write_kb":2061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":298,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.099847+0000","last_change":"2023-10-09T11:05:20.298542+0000","last_active":"2023-10-09T11:09:32.099847+0000","last_peered":"2023-10-09T11:09:32.099847+0000","last_clean":"2023-10-09T11:09:32.099847+0000","last_became_active":"2023-10-09T11:05:20.298119+0000","last_became_peered":"2023-10-09T11:05:20.298119+0000","last_unstale":"2023-10-09T11:09:32.099847+0000","last_undegraded":"2023-10-09T11:09:32.099847+0000","last_fullsized":"2023-10-09T11:09:32.099847+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:00.827214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":278,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100321+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:09:32.100321+0000","last_peered":"2023-10-09T11:09:32.100321+0000","last_clean":"2023-10-09T11:09:32.100321+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:09:32.100321+0000","last_undegraded":"2023-10-09T11:09:32.100321+0000","last_fullsized":"2023-10-09T11:09:32.100321+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"592'125","reported_seq":792,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576795+0000","last_change":"2023-10-09T11:09:20.477093+0000","last_active":"2023-10-09T11:09:31.576795+0000","last_peered":"2023-10-09T11:09:31.576795+0000","last_clean":"2023-10-09T11:09:31.576795+0000","last_became_active":"2023-10-09T11:07:33.226289+0000","last_became_peered":"2023-10-09T11:07:33.226289+0000","last_unstale":"2023-10-09T11:09:31.576795+0000","last_undegraded":"2023-10-09T11:09:31.576795+0000","last_fullsized":"2023-10-09T11:09:31.576795+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:15.605946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026939299999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":322,"num_read_kb":786,"num_write":116,"num_write_kb":327,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":895,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.101944+0000","last_change":"2023-10-09T11:07:36.290700+0000","last_active":"2023-10-09T11:09:32.101944+0000","last_peered":"2023-10-09T11:09:32.101944+0000","last_clean":"2023-10-09T11:09:32.101944+0000","last_became_active":"2023-10-09T11:07:36.290357+0000","last_became_peered":"2023-10-09T11:07:36.290357+0000","last_unstale":"2023-10-09T11:09:32.101944+0000","last_undegraded":"2023-10-09T11:09:32.101944+0000","last_fullsized":"2023-10-09T11:09:32.101944+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:34.494777+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":134,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575245+0000","last_change":"2023-10-09T11:05:20.310666+0000","last_active":"2023-10-09T11:09:31.575245+0000","last_peered":"2023-10-09T11:09:31.575245+0000","last_clean":"2023-10-09T11:09:31.575245+0000","last_became_active":"2023-10-09T11:05:20.310413+0000","last_became_peered":"2023-10-09T11:05:20.310413+0000","last_unstale":"2023-10-09T11:09:31.575245+0000","last_undegraded":"2023-10-09T11:09:31.575245+0000","last_fullsized":"2023-10-09T11:09:31.575245+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:34.169905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":558,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.103372+0000","last_change":"2023-10-09T11:05:20.316523+0000","last_active":"2023-10-09T11:09:32.103372+0000","last_peered":"2023-10-09T11:09:32.103372+0000","last_clean":"2023-10-09T11:09:32.103372+0000","last_became_active":"2023-10-09T11:05:20.314493+0000","last_became_peered":"2023-10-09T11:05:20.314493+0000","last_unstale":"2023-10-09T11:09:32.103372+0000","last_undegraded":"2023-10-09T11:09:32.103372+0000","last_fullsized":"2023-10-09T11:09:32.103372+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:41.618701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":969,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.099991+0000","last_change":"2023-10-09T11:05:23.322074+0000","last_active":"2023-10-09T11:09:32.099991+0000","last_peered":"2023-10-09T11:09:32.099991+0000","last_clean":"2023-10-09T11:09:32.099991+0000","last_became_active":"2023-10-09T11:05:23.321749+0000","last_became_peered":"2023-10-09T11:05:23.321749+0000","last_unstale":"2023-10-09T11:09:32.099991+0000","last_undegraded":"2023-10-09T11:09:32.099991+0000","last_fullsized":"2023-10-09T11:09:32.099991+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:07.010932+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"591'46","reported_seq":352,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.102155+0000","last_change":"2023-10-09T11:09:20.432327+0000","last_active":"2023-10-09T11:09:32.102155+0000","last_peered":"2023-10-09T11:09:32.102155+0000","last_clean":"2023-10-09T11:09:32.102155+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:09:32.102155+0000","last_undegraded":"2023-10-09T11:09:32.102155+0000","last_fullsized":"2023-10-09T11:09:32.102155+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":46,"log_dups_size":0,"ondisk_log_size":46,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000350947,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":271,"num_read_kb":1543,"num_write":39,"num_write_kb":1183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":225,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575963+0000","last_change":"2023-10-09T11:06:24.248362+0000","last_active":"2023-10-09T11:09:31.575963+0000","last_peered":"2023-10-09T11:09:31.575963+0000","last_clean":"2023-10-09T11:09:31.575963+0000","last_became_active":"2023-10-09T11:06:24.247348+0000","last_became_peered":"2023-10-09T11:06:24.247348+0000","last_unstale":"2023-10-09T11:09:31.575963+0000","last_undegraded":"2023-10-09T11:09:31.575963+0000","last_fullsized":"2023-10-09T11:09:31.575963+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:26.728754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":424,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100468+0000","last_change":"2023-10-09T11:06:24.253511+0000","last_active":"2023-10-09T11:09:32.100468+0000","last_peered":"2023-10-09T11:09:32.100468+0000","last_clean":"2023-10-09T11:09:32.100468+0000","last_became_active":"2023-10-09T11:06:24.252739+0000","last_became_peered":"2023-10-09T11:06:24.252739+0000","last_unstale":"2023-10-09T11:09:32.100468+0000","last_undegraded":"2023-10-09T11:09:32.100468+0000","last_fullsized":"2023-10-09T11:09:32.100468+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:50.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":829,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575249+0000","last_change":"2023-10-09T11:06:24.239676+0000","last_active":"2023-10-09T11:09:31.575249+0000","last_peered":"2023-10-09T11:09:31.575249+0000","last_clean":"2023-10-09T11:09:31.575249+0000","last_became_active":"2023-10-09T11:06:24.239375+0000","last_became_peered":"2023-10-09T11:06:24.239375+0000","last_unstale":"2023-10-09T11:09:31.575249+0000","last_undegraded":"2023-10-09T11:09:31.575249+0000","last_fullsized":"2023-10-09T11:09:31.575249+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:27.953053+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"591'57","reported_seq":256,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.103125+0000","last_change":"2023-10-09T11:09:20.432685+0000","last_active":"2023-10-09T11:09:32.103125+0000","last_peered":"2023-10-09T11:09:32.103125+0000","last_clean":"2023-10-09T11:09:32.103125+0000","last_became_active":"2023-10-09T11:07:33.229061+0000","last_became_peered":"2023-10-09T11:07:33.229061+0000","last_unstale":"2023-10-09T11:09:32.103125+0000","last_undegraded":"2023-10-09T11:09:32.103125+0000","last_fullsized":"2023-10-09T11:09:32.103125+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":57,"log_dups_size":0,"ondisk_log_size":57,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:30.978293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043334800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":262,"num_read_kb":1253,"num_write":50,"num_write_kb":1077,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":280,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100454+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:09:32.100454+0000","last_peered":"2023-10-09T11:09:32.100454+0000","last_clean":"2023-10-09T11:09:32.100454+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:09:32.100454+0000","last_undegraded":"2023-10-09T11:09:32.100454+0000","last_fullsized":"2023-10-09T11:09:32.100454+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":555,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576713+0000","last_change":"2023-10-09T11:07:33.227271+0000","last_active":"2023-10-09T11:09:31.576713+0000","last_peered":"2023-10-09T11:09:31.576713+0000","last_clean":"2023-10-09T11:09:31.576713+0000","last_became_active":"2023-10-09T11:07:33.226760+0000","last_became_peered":"2023-10-09T11:07:33.226760+0000","last_unstale":"2023-10-09T11:09:31.576713+0000","last_undegraded":"2023-10-09T11:09:31.576713+0000","last_fullsized":"2023-10-09T11:09:31.576713+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:12.268123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":798,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100186+0000","last_change":"2023-10-09T11:05:23.323368+0000","last_active":"2023-10-09T11:09:32.100186+0000","last_peered":"2023-10-09T11:09:32.100186+0000","last_clean":"2023-10-09T11:09:32.100186+0000","last_became_active":"2023-10-09T11:05:23.323032+0000","last_became_peered":"2023-10-09T11:05:23.323032+0000","last_unstale":"2023-10-09T11:09:32.100186+0000","last_undegraded":"2023-10-09T11:09:32.100186+0000","last_fullsized":"2023-10-09T11:09:32.100186+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:09.393584+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":155,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576669+0000","last_change":"2023-10-09T11:07:33.246378+0000","last_active":"2023-10-09T11:09:31.576669+0000","last_peered":"2023-10-09T11:09:31.576669+0000","last_clean":"2023-10-09T11:09:31.576669+0000","last_became_active":"2023-10-09T11:07:33.228019+0000","last_became_peered":"2023-10-09T11:07:33.228019+0000","last_unstale":"2023-10-09T11:09:31.576669+0000","last_undegraded":"2023-10-09T11:09:31.576669+0000","last_fullsized":"2023-10-09T11:09:31.576669+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:18.511517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"592'319","reported_seq":1588,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100818+0000","last_change":"2023-10-09T11:09:20.427659+0000","last_active":"2023-10-09T11:09:32.100818+0000","last_peered":"2023-10-09T11:09:32.100818+0000","last_clean":"2023-10-09T11:09:32.100818+0000","last_became_active":"2023-10-09T11:05:20.309827+0000","last_became_peered":"2023-10-09T11:05:20.309827+0000","last_unstale":"2023-10-09T11:09:32.100818+0000","last_undegraded":"2023-10-09T11:09:32.100818+0000","last_fullsized":"2023-10-09T11:09:32.100818+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":319,"log_dups_size":0,"ondisk_log_size":319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:23.934383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001122743,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2478,"num_read_kb":2521,"num_write":249,"num_write_kb":324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":143,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575553+0000","last_change":"2023-10-09T11:07:33.217595+0000","last_active":"2023-10-09T11:09:31.575553+0000","last_peered":"2023-10-09T11:09:31.575553+0000","last_clean":"2023-10-09T11:09:31.575553+0000","last_became_active":"2023-10-09T11:07:33.217155+0000","last_became_peered":"2023-10-09T11:07:33.217155+0000","last_unstale":"2023-10-09T11:09:31.575553+0000","last_undegraded":"2023-10-09T11:09:31.575553+0000","last_fullsized":"2023-10-09T11:09:31.575553+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:55.127329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":762,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575658+0000","last_change":"2023-10-09T11:07:36.284491+0000","last_active":"2023-10-09T11:09:31.575658+0000","last_peered":"2023-10-09T11:09:31.575658+0000","last_clean":"2023-10-09T11:09:31.575658+0000","last_became_active":"2023-10-09T11:07:36.284249+0000","last_became_peered":"2023-10-09T11:07:36.284249+0000","last_unstale":"2023-10-09T11:09:31.575658+0000","last_undegraded":"2023-10-09T11:09:31.575658+0000","last_fullsized":"2023-10-09T11:09:31.575658+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:03.653371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:03.653371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:09.629417+0000","scrub_duration":0.001576771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"592'82","reported_seq":263,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.103496+0000","last_change":"2023-10-09T11:09:20.433178+0000","last_active":"2023-10-09T11:09:32.103496+0000","last_peered":"2023-10-09T11:09:32.103496+0000","last_clean":"2023-10-09T11:09:32.103496+0000","last_became_active":"2023-10-09T11:07:33.241013+0000","last_became_peered":"2023-10-09T11:07:33.241013+0000","last_unstale":"2023-10-09T11:09:32.103496+0000","last_undegraded":"2023-10-09T11:09:32.103496+0000","last_fullsized":"2023-10-09T11:09:32.103496+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":82,"log_dups_size":0,"ondisk_log_size":82,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:57.164502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075188800000000004,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":544,"num_write":82,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":297,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100651+0000","last_change":"2023-10-09T11:06:24.253714+0000","last_active":"2023-10-09T11:09:32.100651+0000","last_peered":"2023-10-09T11:09:32.100651+0000","last_clean":"2023-10-09T11:09:32.100651+0000","last_became_active":"2023-10-09T11:06:24.253143+0000","last_became_peered":"2023-10-09T11:06:24.253143+0000","last_unstale":"2023-10-09T11:09:32.100651+0000","last_undegraded":"2023-10-09T11:09:32.100651+0000","last_fullsized":"2023-10-09T11:09:32.100651+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:05.682288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":401,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576556+0000","last_change":"2023-10-09T11:05:20.313560+0000","last_active":"2023-10-09T11:09:31.576556+0000","last_peered":"2023-10-09T11:09:31.576556+0000","last_clean":"2023-10-09T11:09:31.576556+0000","last_became_active":"2023-10-09T11:05:20.313214+0000","last_became_peered":"2023-10-09T11:05:20.313214+0000","last_unstale":"2023-10-09T11:09:31.576556+0000","last_undegraded":"2023-10-09T11:09:31.576556+0000","last_fullsized":"2023-10-09T11:09:31.576556+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:32.527205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":1005,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576235+0000","last_change":"2023-10-09T11:05:23.329886+0000","last_active":"2023-10-09T11:09:31.576235+0000","last_peered":"2023-10-09T11:09:31.576235+0000","last_clean":"2023-10-09T11:09:31.576235+0000","last_became_active":"2023-10-09T11:05:23.329561+0000","last_became_peered":"2023-10-09T11:05:23.329561+0000","last_unstale":"2023-10-09T11:09:31.576235+0000","last_undegraded":"2023-10-09T11:09:31.576235+0000","last_fullsized":"2023-10-09T11:09:31.576235+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:46.933799+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":58,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.102949+0000","last_change":"2023-10-09T11:07:33.229864+0000","last_active":"2023-10-09T11:09:32.102949+0000","last_peered":"2023-10-09T11:09:32.102949+0000","last_clean":"2023-10-09T11:09:32.102949+0000","last_became_active":"2023-10-09T11:07:33.229547+0000","last_became_peered":"2023-10-09T11:07:33.229547+0000","last_unstale":"2023-10-09T11:09:32.102949+0000","last_undegraded":"2023-10-09T11:09:32.102949+0000","last_fullsized":"2023-10-09T11:09:32.102949+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:34.990439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":381,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100344+0000","last_change":"2023-10-09T11:04:50.510688+0000","last_active":"2023-10-09T11:09:32.100344+0000","last_peered":"2023-10-09T11:09:32.100344+0000","last_clean":"2023-10-09T11:09:32.100344+0000","last_became_active":"2023-10-09T11:04:50.507404+0000","last_became_peered":"2023-10-09T11:04:50.507404+0000","last_unstale":"2023-10-09T11:09:32.100344+0000","last_undegraded":"2023-10-09T11:09:32.100344+0000","last_fullsized":"2023-10-09T11:09:32.100344+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:13.663394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":963,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575897+0000","last_change":"2023-10-09T11:05:23.321040+0000","last_active":"2023-10-09T11:09:31.575897+0000","last_peered":"2023-10-09T11:09:31.575897+0000","last_clean":"2023-10-09T11:09:31.575897+0000","last_became_active":"2023-10-09T11:05:23.320674+0000","last_became_peered":"2023-10-09T11:05:23.320674+0000","last_unstale":"2023-10-09T11:09:31.575897+0000","last_undegraded":"2023-10-09T11:09:31.575897+0000","last_fullsized":"2023-10-09T11:09:31.575897+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.527660+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":423,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576384+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:09:31.576384+0000","last_peered":"2023-10-09T11:09:31.576384+0000","last_clean":"2023-10-09T11:09:31.576384+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:09:31.576384+0000","last_undegraded":"2023-10-09T11:09:31.576384+0000","last_fullsized":"2023-10-09T11:09:31.576384+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":124,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575451+0000","last_change":"2023-10-09T11:04:50.511984+0000","last_active":"2023-10-09T11:09:31.575451+0000","last_peered":"2023-10-09T11:09:31.575451+0000","last_clean":"2023-10-09T11:09:31.575451+0000","last_became_active":"2023-10-09T11:04:50.510820+0000","last_became_peered":"2023-10-09T11:04:50.510820+0000","last_unstale":"2023-10-09T11:09:31.575451+0000","last_undegraded":"2023-10-09T11:09:31.575451+0000","last_fullsized":"2023-10-09T11:09:31.575451+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:38.662189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":574,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576784+0000","last_change":"2023-10-09T11:05:20.327032+0000","last_active":"2023-10-09T11:09:31.576784+0000","last_peered":"2023-10-09T11:09:31.576784+0000","last_clean":"2023-10-09T11:09:31.576784+0000","last_became_active":"2023-10-09T11:05:20.320193+0000","last_became_peered":"2023-10-09T11:05:20.320193+0000","last_unstale":"2023-10-09T11:09:31.576784+0000","last_undegraded":"2023-10-09T11:09:31.576784+0000","last_fullsized":"2023-10-09T11:09:31.576784+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:44.862890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":124,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575160+0000","last_change":"2023-10-09T11:04:50.384219+0000","last_active":"2023-10-09T11:09:31.575160+0000","last_peered":"2023-10-09T11:09:31.575160+0000","last_clean":"2023-10-09T11:09:31.575160+0000","last_became_active":"2023-10-09T11:04:50.384028+0000","last_became_peered":"2023-10-09T11:04:50.384028+0000","last_unstale":"2023-10-09T11:09:31.575160+0000","last_undegraded":"2023-10-09T11:09:31.575160+0000","last_fullsized":"2023-10-09T11:09:31.575160+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:04.721869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":781,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.102660+0000","last_change":"2023-10-09T11:08:39.540029+0000","last_active":"2023-10-09T11:09:32.102660+0000","last_peered":"2023-10-09T11:09:32.102660+0000","last_clean":"2023-10-09T11:09:32.102660+0000","last_became_active":"2023-10-09T11:07:36.291077+0000","last_became_peered":"2023-10-09T11:07:36.291077+0000","last_unstale":"2023-10-09T11:09:32.102660+0000","last_undegraded":"2023-10-09T11:09:32.102660+0000","last_fullsized":"2023-10-09T11:09:32.102660+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.539966+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.539966+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:20.767812+0000","scrub_duration":0.0016245929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":134,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575951+0000","last_change":"2023-10-09T11:05:20.299321+0000","last_active":"2023-10-09T11:09:31.575951+0000","last_peered":"2023-10-09T11:09:31.575951+0000","last_clean":"2023-10-09T11:09:31.575951+0000","last_became_active":"2023-10-09T11:05:20.299019+0000","last_became_peered":"2023-10-09T11:05:20.299019+0000","last_unstale":"2023-10-09T11:09:31.575951+0000","last_undegraded":"2023-10-09T11:09:31.575951+0000","last_fullsized":"2023-10-09T11:09:31.575951+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:08.645909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":124,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576310+0000","last_change":"2023-10-09T11:04:50.366890+0000","last_active":"2023-10-09T11:09:31.576310+0000","last_peered":"2023-10-09T11:09:31.576310+0000","last_clean":"2023-10-09T11:09:31.576310+0000","last_became_active":"2023-10-09T11:04:50.366558+0000","last_became_peered":"2023-10-09T11:04:50.366558+0000","last_unstale":"2023-10-09T11:09:31.576310+0000","last_undegraded":"2023-10-09T11:09:31.576310+0000","last_fullsized":"2023-10-09T11:09:31.576310+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:50.993153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":897,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100718+0000","last_change":"2023-10-09T11:04:53.274332+0000","last_active":"2023-10-09T11:09:32.100718+0000","last_peered":"2023-10-09T11:09:32.100718+0000","last_clean":"2023-10-09T11:09:32.100718+0000","last_became_active":"2023-10-09T11:04:53.273798+0000","last_became_peered":"2023-10-09T11:04:53.273798+0000","last_unstale":"2023-10-09T11:09:32.100718+0000","last_undegraded":"2023-10-09T11:09:32.100718+0000","last_fullsized":"2023-10-09T11:09:32.100718+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:04.220970+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":143,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.577151+0000","last_change":"2023-10-09T11:06:24.249877+0000","last_active":"2023-10-09T11:09:31.577151+0000","last_peered":"2023-10-09T11:09:31.577151+0000","last_clean":"2023-10-09T11:09:31.577151+0000","last_became_active":"2023-10-09T11:06:24.249573+0000","last_became_peered":"2023-10-09T11:06:24.249573+0000","last_unstale":"2023-10-09T11:09:31.577151+0000","last_undegraded":"2023-10-09T11:09:31.577151+0000","last_fullsized":"2023-10-09T11:09:31.577151+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:11.798192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":646,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.099883+0000","last_change":"2023-10-09T11:05:20.307347+0000","last_active":"2023-10-09T11:09:32.099883+0000","last_peered":"2023-10-09T11:09:32.099883+0000","last_clean":"2023-10-09T11:09:32.099883+0000","last_became_active":"2023-10-09T11:05:20.307126+0000","last_became_peered":"2023-10-09T11:05:20.307126+0000","last_unstale":"2023-10-09T11:09:32.099883+0000","last_undegraded":"2023-10-09T11:09:32.099883+0000","last_fullsized":"2023-10-09T11:09:32.099883+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:34.408515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":91,"num_bytes_recovered":15742,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12694,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13244,"num_write":846,"num_write_kb":12712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332516,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":939152,"kb_used_data":39752,"kb_used_omap":0,"kb_used_meta":899392,"kb_avail":92806000,"statfs":{"total":95995035648,"available":95033344000,"internally_reserved":0,"allocated":40706048,"data_stored":38855721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920977408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:21 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.04,"15min":1.6990000000000001},"min":{"1min":0.27100000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.2949999999999999,"5min":9.8219999999999992,"15min":9.8219999999999992},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.389,"15min":1.694},"min":{"1min":0.33200000000000002,"5min":0.24299999999999999,"15min":0.216},"max":{"1min":3.1640000000000001,"5min":15.853,"15min":15.853},"last":0.64900000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":1.194,"15min":1.47},"min":{"1min":0.315,"5min":0.222,"15min":0.222},"max":{"1min":0.83499999999999996,"5min":11.619999999999999,"15min":11.619999999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":1.8,"15min":1.843},"min":{"1min":0.31900000000000001,"5min":0.25,"15min":0.25},"max":{"1min":0.91000000000000003,"5min":13.096,"15min":13.096},"last":0.69699999999999995}]},{"osd":2,"last update":"Mon Oct 9 11:08:54 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":1.573,"15min":1.77},"min":{"1min":0.309,"5min":0.309,"15min":0.245},"max":{"1min":5.1440000000000001,"5min":12.226000000000001,"15min":16.605},"last":0.57499999999999996},{"interface":"front","average":{"1min":2.2770000000000001,"5min":1.6220000000000001,"15min":1.802},"min":{"1min":0.379,"5min":0.223,"15min":0.223},"max":{"1min":16.157,"5min":16.157,"15min":16.157},"last":0.55800000000000005}]},{"osd":3,"last update":"Mon Oct 9 11:09:21 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":1.554,"15min":1.45},"min":{"1min":0.35499999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.194,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.68899999999999995,"5min":1.665,"15min":1.7490000000000001},"min":{"1min":0.39300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.8879999999999999,"5min":8.2330000000000005,"15min":8.2330000000000005},"last":0.54200000000000004}]},{"osd":4,"last update":"Mon Oct 9 11:09:09 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.85799999999999998,"15min":1.0900000000000001},"min":{"1min":0.248,"5min":0.218,"15min":0.218},"max":{"1min":4.7489999999999997,"5min":7.4630000000000001,"15min":8.2940000000000005},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.91000000000000003,"5min":0.92800000000000005,"15min":1.26},"min":{"1min":0.35399999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.7619999999999996,"5min":9.2759999999999998,"15min":16.481000000000002},"last":0.46200000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:09:09 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.85199999999999998,"15min":1.2549999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.247},"max":{"1min":4.9470000000000001,"5min":6.9379999999999997,"15min":15.414999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.095,"15min":1.228},"min":{"1min":0.34300000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.9980000000000002,"5min":13.045,"15min":15.369},"last":0.51700000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:09:09 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.98799999999999999,"15min":1.242},"min":{"1min":0.39800000000000002,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":4.8079999999999998,"5min":5.1120000000000001,"15min":14.159000000000001},"last":0.71999999999999997},{"interface":"front","average":{"1min":1.0049999999999999,"5min":0.96599999999999997,"15min":1.369},"min":{"1min":0.39000000000000001,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":3.1179999999999999,"5min":4.3659999999999997,"15min":16.538},"last":0.624}]}]},{"osd":6,"up_from":44,"seq":188978561447,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1511556,"kb_used_data":10812,"kb_used_omap":0,"kb_used_meta":1500736,"kb_avail":92233596,"statfs":{"total":95995035648,"available":94447202304,"internally_reserved":0,"allocated":11071488,"data_stored":9232778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1536753664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":1.0900000000000001,"15min":1.9179999999999999},"min":{"1min":0.26900000000000002,"5min":0.223,"15min":0.217},"max":{"1min":0.72799999999999998,"5min":12.209,"15min":54.743000000000002},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.94099999999999995,"15min":1.8580000000000001},"min":{"1min":0.33700000000000002,"5min":0.247,"15min":0.214},"max":{"1min":0.753,"5min":8.4789999999999992,"15min":53.701000000000001},"last":0.48199999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":1.2390000000000001,"15min":1.1739999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.23499999999999999},"max":{"1min":0.754,"5min":16.225999999999999,"15min":41.241999999999997},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.45900000000000002,"5min":1.7749999999999999,"15min":1.766},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.248},"max":{"1min":0.62,"5min":11.715,"15min":41.476999999999997},"last":0.39400000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.867,"15min":1.9019999999999999},"min":{"1min":0.32200000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.2069999999999999,"5min":10.382999999999999,"15min":12.003},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.85599999999999998,"5min":1.2909999999999999,"15min":1.877},"min":{"1min":0.33800000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.6669999999999998,"5min":12.693,"15min":15.432},"last":0.433}]},{"osd":3,"last update":"Mon Oct 9 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":2.0310000000000001,"15min":1.643},"min":{"1min":0.372,"5min":0.313,"15min":0.313},"max":{"1min":0.96999999999999997,"5min":28.948,"15min":28.948},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.51400000000000001,"5min":2.597,"15min":2.4910000000000001},"min":{"1min":0.36199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.65100000000000002,"5min":28.893000000000001,"15min":28.893000000000001},"last":0.44600000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.66900000000000004,"15min":0.998},"min":{"1min":0.28100000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.67100000000000004,"5min":6.6609999999999996,"15min":17.216999999999999},"last":0.309},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.53700000000000003,"15min":0.83299999999999996},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.24399999999999999},"max":{"1min":0.80100000000000005,"5min":2.3610000000000002,"15min":10.025},"last":0.504}]},{"osd":5,"last update":"Mon Oct 9 11:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.78300000000000003,"15min":1.228},"min":{"1min":0.34000000000000002,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":2.7000000000000002,"5min":3.7389999999999999,"15min":19.786999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.68999999999999995,"15min":0.96499999999999997},"min":{"1min":0.377,"5min":0.248,"15min":0.248},"max":{"1min":1.22,"5min":4.085,"15min":9.9469999999999992},"last":0.54100000000000004}]},{"osd":7,"last update":"Mon Oct 9 11:09:16 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.71899999999999997,"15min":1.238},"min":{"1min":0.32400000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.73999999999999999,"5min":3.7250000000000001,"15min":12.882},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.68500000000000005,"15min":0.91700000000000004},"min":{"1min":0.35399999999999998,"5min":0.32800000000000001,"15min":0.27500000000000002},"max":{"1min":0.84399999999999997,"5min":3.2839999999999998,"15min":11.557},"last":0.52300000000000002}]}]},{"osd":1,"up_from":388,"seq":1666447310962,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":505304,"kb_used_data":7824,"kb_used_omap":13,"kb_used_meta":497458,"kb_avail":93239848,"statfs":{"total":95995035648,"available":95477604352,"internally_reserved":0,"allocated":8011776,"data_stored":4888001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":509397703},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":2.347,"15min":2.3250000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.17699999999999999},"max":{"1min":1.9590000000000001,"5min":18.192,"15min":18.192},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.434,"5min":2.6749999999999998,"15min":2.0329999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.19900000000000001},"max":{"1min":1.2470000000000001,"5min":17.957999999999998,"15min":17.957999999999998},"last":0.317}]},{"osd":2,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":2.0939999999999999,"15min":2.73},"min":{"1min":0.29399999999999998,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":2.032,"5min":12.436999999999999,"15min":47.731000000000002},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.44400000000000001,"5min":2.5019999999999998,"15min":2.4689999999999999},"min":{"1min":0.28399999999999997,"5min":0.216,"15min":0.216},"max":{"1min":0.81200000000000006,"5min":16.870000000000001,"15min":47.786999999999999},"last":0.38200000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":3.141,"15min":2.0390000000000001},"min":{"1min":0.254,"5min":0.251,"15min":0.251},"max":{"1min":0.86299999999999999,"5min":92.447999999999993,"15min":92.447999999999993},"last":0.497},{"interface":"front","average":{"1min":0.52800000000000002,"5min":3.2440000000000002,"15min":2.0699999999999998},"min":{"1min":0.252,"5min":0.217,"15min":0.217},"max":{"1min":2.0009999999999999,"5min":92.066999999999993,"15min":92.066999999999993},"last":0.30299999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.6020000000000001,"15min":2.1880000000000002},"min":{"1min":0.34699999999999998,"5min":0.27300000000000002,"15min":0.26000000000000001},"max":{"1min":2.0739999999999998,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.905,"15min":1.673},"min":{"1min":0.33300000000000002,"5min":0.314,"15min":0.29999999999999999},"max":{"1min":0.88,"5min":16.841000000000001,"15min":16.841000000000001},"last":0.46100000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.6879999999999999,"15min":2.1549999999999998},"min":{"1min":0.40400000000000003,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":0.86899999999999999,"5min":20.689,"15min":20.689},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.66400000000000003,"5min":1.7549999999999999,"15min":2.1240000000000001},"min":{"1min":0.39900000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.133,"5min":11.65,"15min":11.65},"last":0.61899999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.7749999999999999,"15min":2.2480000000000002},"min":{"1min":0.44,"5min":0.312,"15min":0.309},"max":{"1min":0.89900000000000002,"5min":20.646999999999998,"15min":20.646999999999998},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.766,"15min":2.052},"min":{"1min":0.375,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.1040000000000001,"5min":11.75,"15min":11.75},"last":0.57799999999999996}]},{"osd":7,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.9330000000000001,"15min":1.6830000000000001},"min":{"1min":0.36199999999999999,"5min":0.33000000000000002,"15min":0.32900000000000001},"max":{"1min":0.78300000000000003,"5min":16.797999999999998,"15min":16.797999999999998},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.766,"15min":2.1589999999999998},"min":{"1min":0.42699999999999999,"5min":0.36899999999999999,"15min":0.33200000000000002},"max":{"1min":0.91900000000000004,"5min":20.716000000000001,"15min":20.716000000000001},"last":0.54200000000000004}]}]},{"osd":0,"up_from":80,"seq":343597384020,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":986400,"kb_used_data":8216,"kb_used_omap":28,"kb_used_meta":978147,"kb_avail":92758752,"statfs":{"total":95995035648,"available":94984962048,"internally_reserved":0,"allocated":8413184,"data_stored":5211967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1001623376},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":1.8700000000000001,"15min":2.5489999999999999},"min":{"1min":0.23300000000000001,"5min":0.186,"15min":0.186},"max":{"1min":1.6220000000000001,"5min":14.644,"15min":19.608000000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.96,"15min":2.613},"min":{"1min":0.23400000000000001,"5min":0.214,"15min":0.214},"max":{"1min":3.7240000000000002,"5min":15.834,"15min":19.992999999999999},"last":0.29499999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:08:57 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":2.0579999999999998,"15min":2.2959999999999998},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":7.9329999999999998,"5min":12.459,"15min":24.852},"last":0.27700000000000002},{"interface":"front","average":{"1min":6.4219999999999997,"5min":3.0819999999999999,"15min":3.0009999999999999},"min":{"1min":0.21299999999999999,"5min":0.20399999999999999,"15min":0.19800000000000001},"max":{"1min":99.679000000000002,"5min":99.679000000000002,"15min":99.679000000000002},"last":0.373}]},{"osd":3,"last update":"Mon Oct 9 11:09:25 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":2.52,"15min":3.194},"min":{"1min":0.28499999999999998,"5min":0.22,"15min":0.22},"max":{"1min":8.0180000000000007,"5min":25.050000000000001,"15min":25.050000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.51100000000000001,"5min":2.077,"15min":1.958},"min":{"1min":0.314,"5min":0.254,"15min":0.254},"max":{"1min":2.0419999999999998,"5min":14.125999999999999,"15min":14.125999999999999},"last":0.35899999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:09:25 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.905,"15min":2.4540000000000002},"min":{"1min":0.312,"5min":0.27100000000000002,"15min":0.25700000000000001},"max":{"1min":7.9829999999999997,"5min":14.507,"15min":77.177999999999997},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.3660000000000001,"15min":1.6120000000000001},"min":{"1min":0.34100000000000003,"5min":0.33600000000000002,"15min":0.29199999999999998},"max":{"1min":2.1160000000000001,"5min":7.9249999999999998,"15min":10.513999999999999},"last":0.50600000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:09:25 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.361,"15min":1.7909999999999999},"min":{"1min":0.36699999999999999,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":2.1440000000000001,"5min":7.9550000000000001,"15min":10.544},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.629,"5min":1.516,"15min":1.7829999999999999},"min":{"1min":0.439,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":1.7010000000000001,"5min":11.212999999999999,"15min":24.754999999999999},"last":0.46200000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:09:25 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":1.4450000000000001,"15min":1.8740000000000001},"min":{"1min":0.38600000000000001,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":2.1859999999999999,"5min":7.9829999999999997,"15min":18.920000000000002},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.63,"5min":1.542,"15min":1.899},"min":{"1min":0.41699999999999998,"5min":0.378,"15min":0.31900000000000001},"max":{"1min":1.728,"5min":11.257999999999999,"15min":24.800000000000001},"last":0.55500000000000005}]},{"osd":7,"last update":"Mon Oct 9 11:09:25 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":1.9219999999999999,"15min":2.0049999999999999},"min":{"1min":0.39300000000000002,"5min":0.35099999999999998,"15min":0.32700000000000001},"max":{"1min":1.6679999999999999,"5min":17.672000000000001,"15min":17.672000000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":2.0779999999999998,"15min":2.573},"min":{"1min":0.41299999999999998,"5min":0.311,"15min":0.311},"max":{"1min":7.9390000000000001,"5min":17.539000000000001,"15min":77.228999999999999},"last":0.59099999999999997}]}]},{"osd":2,"up_from":90,"seq":386547056960,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":769892,"kb_used_data":12060,"kb_used_omap":81,"kb_used_meta":757742,"kb_avail":92975260,"statfs":{"total":95995035648,"available":95206666240,"internally_reserved":0,"allocated":12349440,"data_stored":9324994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":775928026},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.363,"5min":1.9750000000000001,"15min":2.052},"min":{"1min":0.191,"5min":0.182,"15min":0.17499999999999999},"max":{"1min":8.3710000000000004,"5min":15.609,"15min":48.933999999999997},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.998,"15min":1.851},"min":{"1min":0.24199999999999999,"5min":0.23100000000000001,"15min":0.19600000000000001},"max":{"1min":7.7859999999999996,"5min":9.8010000000000002,"15min":14.515000000000001},"last":0.38700000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:09:36 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.897,"15min":2.3940000000000001},"min":{"1min":0.26900000000000002,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":1.1060000000000001,"5min":24.902000000000001,"15min":49.033999999999999},"last":0.37},{"interface":"front","average":{"1min":0.437,"5min":2.1600000000000001,"15min":2.5720000000000001},"min":{"1min":0.255,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":1.2210000000000001,"5min":25.850999999999999,"15min":48.874000000000002},"last":0.40799999999999997}]},{"osd":3,"last update":"Mon Oct 9 11:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":2.8719999999999999,"15min":2.1339999999999999},"min":{"1min":0.27900000000000003,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":14.801,"5min":25.890999999999998,"15min":25.890999999999998},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.2609999999999999,"5min":2.7189999999999999,"15min":2.3359999999999999},"min":{"1min":0.29399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":11.724,"5min":28.689,"15min":28.689},"last":0.437}]},{"osd":4,"last update":"Mon Oct 9 11:09:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.575,"15min":1.8720000000000001},"min":{"1min":0.34699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.6949999999999998,"5min":11.391999999999999,"15min":15.672000000000001},"last":0.76600000000000001},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.6200000000000001,"15min":1.8959999999999999},"min":{"1min":0.33900000000000002,"5min":0.32900000000000001,"15min":0.249},"max":{"1min":8.1609999999999996,"5min":15.571,"15min":48.795000000000002},"last":0.77600000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.6659999999999999,"15min":1.5900000000000001},"min":{"1min":0.35699999999999998,"5min":0.32100000000000001,"15min":0.25900000000000001},"max":{"1min":7.7279999999999998,"5min":8.5310000000000006,"15min":12.128},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.84399999999999997,"5min":1.5509999999999999,"15min":2.0030000000000001},"min":{"1min":0.39900000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.7189999999999999,"5min":11.441000000000001,"15min":49.061},"last":0.79000000000000004}]},{"osd":6,"last update":"Mon Oct 9 11:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5940000000000001,"5min":1.661,"15min":1.841},"min":{"1min":0.33000000000000002,"5min":0.313,"15min":0.313},"max":{"1min":8.1289999999999996,"5min":15.536,"15min":48.845999999999997},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.383,"5min":1.897,"15min":1.6759999999999999},"min":{"1min":0.38,"5min":0.36799999999999999,"15min":0.29199999999999998},"max":{"1min":7.8129999999999997,"5min":10.157999999999999,"15min":10.157999999999999},"last":0.75}]},{"osd":7,"last update":"Mon Oct 9 11:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":1.8660000000000001,"15min":2.0009999999999999},"min":{"1min":0.39700000000000002,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":8.343,"5min":15.59,"15min":48.966000000000001},"last":0.82099999999999995},{"interface":"front","average":{"1min":1.3220000000000001,"5min":1.8720000000000001,"15min":1.7849999999999999},"min":{"1min":0.36099999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.8390000000000004,"5min":8.5549999999999997,"15min":14.94},"last":0.70099999999999996}]}]},{"osd":3,"up_from":469,"seq":2014339661890,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":343440,"kb_used_data":8392,"kb_used_omap":11,"kb_used_meta":335028,"kb_avail":93401712,"statfs":{"total":95995035648,"available":95643353088,"internally_reserved":0,"allocated":8593408,"data_stored":5233360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11309,"internal_metadata":343069651},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":1.915,"15min":1.7849999999999999},"min":{"1min":0.22900000000000001,"5min":0.183,"15min":0.183},"max":{"1min":1.0740000000000001,"5min":14.882,"15min":14.882},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.337,"15min":1.024},"min":{"1min":0.19900000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":2.5939999999999999,"5min":9.4030000000000005,"15min":9.4030000000000005},"last":0.246}]},{"osd":1,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":2.0710000000000002,"15min":1.625},"min":{"1min":0.22,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":4.1799999999999997,"5min":17.93,"15min":17.93},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.4590000000000001,"5min":2.2210000000000001,"15min":1.7969999999999999},"min":{"1min":0.28100000000000003,"5min":0.224,"15min":0.224},"max":{"1min":13.236000000000001,"5min":19.163,"15min":19.163},"last":0.38100000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":2.0310000000000001,"15min":1.401},"min":{"1min":0.27400000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":4.0270000000000001,"5min":38.840000000000003,"15min":38.840000000000003},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.69999999999999996,"5min":2.1360000000000001,"15min":1.4319999999999999},"min":{"1min":0.27700000000000002,"5min":0.219,"15min":0.219},"max":{"1min":4.016,"5min":37.991999999999997,"15min":37.991999999999997},"last":0.35999999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.46,"15min":1.5349999999999999},"min":{"1min":0.34000000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.131,"5min":14.907999999999999,"15min":14.907999999999999},"last":0.44},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.123,"15min":0.83799999999999997},"min":{"1min":0.307,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.0739999999999998,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.47999999999999998}]},{"osd":5,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.2729999999999999,"15min":1.3660000000000001},"min":{"1min":0.41699999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.4450000000000001,"5min":9.3300000000000001,"15min":9.3300000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.3999999999999999,"15min":1.427},"min":{"1min":0.32000000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.101,"5min":10.029,"15min":10.029},"last":0.54600000000000004}]},{"osd":6,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":1.4410000000000001,"15min":1.669},"min":{"1min":0.44900000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.2120000000000002,"5min":15.455,"15min":15.455},"last":0.501},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.516,"15min":1.6000000000000001},"min":{"1min":0.39900000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.865,"5min":14.837,"15min":14.837},"last":0.51200000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":1.3240000000000001,"15min":1.1080000000000001},"min":{"1min":0.39500000000000002,"5min":0.315,"15min":0.315},"max":{"1min":3.8029999999999999,"5min":6.907,"15min":6.907},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.86299999999999999,"5min":1.536,"15min":1.2829999999999999},"min":{"1min":0.42299999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.8370000000000002,"5min":6.9119999999999999,"15min":6.9119999999999999},"last":0.47199999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019310,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903244,"kb_used_data":6404,"kb_used_omap":0,"kb_used_meta":896832,"kb_avail":92841908,"statfs":{"total":95995035648,"available":95070113792,"internally_reserved":0,"allocated":6557696,"data_stored":4711174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918355968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:19 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":1.494,"15min":2.4129999999999998},"min":{"1min":0.26900000000000002,"5min":0.223,"15min":0.20300000000000001},"max":{"1min":3.1259999999999999,"5min":13.162000000000001,"15min":213.88999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.329,"15min":2.5},"min":{"1min":0.30099999999999999,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":2.399,"5min":9.0670000000000002,"15min":214.09800000000001},"last":0.77100000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":2.331,"15min":2.5350000000000001},"min":{"1min":0.28299999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.68200000000000005,"5min":57.966999999999999,"15min":121.41},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.65800000000000003,"5min":1.9810000000000001,"15min":2.081},"min":{"1min":0.28199999999999997,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":3.589,"5min":57.832000000000001,"15min":122.21899999999999},"last":0.51300000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:09:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.748,"15min":1.738},"min":{"1min":0.30499999999999999,"5min":0.246,"15min":0.23799999999999999},"max":{"1min":3.0779999999999998,"5min":18.407,"15min":18.407},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.7430000000000001,"15min":2.0739999999999998},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":2.5129999999999999,"5min":8.4250000000000007,"15min":27.818000000000001},"last":0.73299999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":2.2629999999999999,"15min":1.921},"min":{"1min":0.35599999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.1189999999999998,"5min":60.753,"15min":60.753},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.0329999999999999,"5min":2.1789999999999998,"15min":1.5049999999999999},"min":{"1min":0.39800000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.5199999999999996,"5min":61.473999999999997,"15min":61.473999999999997},"last":0.63800000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.89600000000000002,"15min":1.337},"min":{"1min":0.30299999999999999,"5min":0.27400000000000002,"15min":0.224},"max":{"1min":0.68200000000000005,"5min":4.9409999999999998,"15min":19.068999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.499,"5min":0.83799999999999997,"15min":1.22},"min":{"1min":0.29899999999999999,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":1.052,"5min":8.3230000000000004,"15min":15.535},"last":0.66000000000000003}]},{"osd":6,"last update":"Mon Oct 9 11:09:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.2969999999999999,"15min":1.1910000000000001},"min":{"1min":0.29299999999999998,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":6.8689999999999998,"5min":8.3930000000000007,"15min":15.739000000000001},"last":0.753},{"interface":"front","average":{"1min":1.3500000000000001,"5min":1.4179999999999999,"15min":1.4730000000000001},"min":{"1min":0.33700000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.8170000000000002,"5min":12.795999999999999,"15min":14.093999999999999},"last":0.71299999999999997}]},{"osd":7,"last update":"Mon Oct 9 11:09:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.294,"15min":1.5629999999999999},"min":{"1min":0.374,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.181,"5min":10.15,"15min":19.923999999999999},"last":0.78800000000000003},{"interface":"front","average":{"1min":1.155,"5min":1.091,"15min":1.488},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.0439999999999996,"5min":10.853,"15min":19.859999999999999},"last":0.84899999999999998}]}]},{"osd":5,"up_from":37,"seq":158913790379,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1205952,"kb_used_data":6648,"kb_used_omap":0,"kb_used_meta":1199296,"kb_avail":92539200,"statfs":{"total":95995035648,"available":94760140800,"internally_reserved":0,"allocated":6807552,"data_stored":4969302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1228079104},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":1.321,"15min":2.2389999999999999},"min":{"1min":0.23000000000000001,"5min":0.219,"15min":0.21299999999999999},"max":{"1min":0.77900000000000003,"5min":6.2249999999999996,"15min":58.271999999999998},"last":0.505},{"interface":"front","average":{"1min":0.52000000000000002,"5min":1.3740000000000001,"15min":2.2890000000000001},"min":{"1min":0.38200000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.752,"5min":6.2649999999999997,"15min":58.25},"last":0.54800000000000004}]},{"osd":1,"last update":"Mon Oct 9 11:09:32 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":1.5569999999999999,"15min":1.964},"min":{"1min":0.26100000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.64800000000000002,"5min":16.962,"15min":32.491999999999997},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":1.675,"15min":1.845},"min":{"1min":0.34200000000000003,"5min":0.30499999999999999,"15min":0.247},"max":{"1min":0.80800000000000005,"5min":17.033999999999999,"15min":32.537999999999997},"last":0.58299999999999996}]},{"osd":2,"last update":"Mon Oct 9 11:08:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.321,"15min":1.8240000000000001},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.221},"max":{"1min":7.3109999999999999,"5min":7.3109999999999999,"15min":11.257999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.8919999999999999,"15min":1.988},"min":{"1min":0.317,"5min":0.253,"15min":0.253},"max":{"1min":4.085,"5min":11.673,"15min":14.621},"last":0.64500000000000002}]},{"osd":3,"last update":"Mon Oct 9 11:09:19 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":1.607,"15min":1.5289999999999999},"min":{"1min":0.35899999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.7880000000000003,"5min":15.535,"15min":15.535},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.9059999999999999,"15min":1.841},"min":{"1min":0.40699999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.8090000000000002,"5min":15.757999999999999,"15min":15.757999999999999},"last":0.59799999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:09:32 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.80900000000000005,"15min":0.90400000000000003},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.216},"max":{"1min":0.65900000000000003,"5min":7.5640000000000001,"15min":12.042},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.92800000000000005,"15min":1.151},"min":{"1min":0.32000000000000001,"5min":0.245,"15min":0.245},"max":{"1min":1.0840000000000001,"5min":6.8639999999999999,"15min":9.2490000000000006},"last":0.52800000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:08:45 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.2090000000000001,"15min":1.2450000000000001},"min":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.27100000000000002},"max":{"1min":5.944,"5min":7.1020000000000003,"15min":9.1969999999999992},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.214,"5min":1.204,"15min":1.0700000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29099999999999998},"max":{"1min":9.298,"5min":11.983000000000001,"15min":11.983000000000001},"last":0.622}]},{"osd":7,"last update":"Mon Oct 9 11:09:09 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.026,"15min":1.298},"min":{"1min":0.35399999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.9699999999999998,"5min":5.9699999999999998,"15min":9.3539999999999992},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.94199999999999995,"15min":1.1739999999999999},"min":{"1min":0.38700000000000001,"5min":0.30499999999999999,"15min":0.30099999999999999},"max":{"1min":2.2549999999999999,"5min":8.0009999999999994,"15min":11.726000000000001},"last":0.66400000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3733,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:09:41.290 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:09:41.291 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T11:09:41.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:41 smithi125 ceph-mon[195507]: from='client.37638 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:09:41.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:41 smithi125 ceph-mon[200197]: from='client.37638 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:09:41.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:09:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:09:41] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.43.0" 2023-10-09T11:09:41.612 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:41.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:41 smithi155 ceph-mon[184854]: from='client.37638 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:09:41.900 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:42.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:42 smithi125 ceph-mon[195507]: pgmap v1454: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:42.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:42 smithi125 ceph-mon[200197]: pgmap v1454: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:42.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:42 smithi155 ceph-mon[184854]: pgmap v1454: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:42.912 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:09:42.914 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:09:43.292 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:43.293 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:43.452 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1454,"stamp":"2023-10-09T11:09:41.943301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5995,"num_read_kb":13244,"num_write":907,"num_write_kb":12741,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":2377214,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":310,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7153656,"kb_used_data":88824,"kb_used_omap":134,"kb_used_meta":7064633,"kb_avail":742807560,"statfs":{"total":767960285184,"available":760634941440,"internally_reserved":0,"allocated":90955776,"data_stored":70866729,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":137532,"internal_metadata":7234184900},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002138"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":884,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114685+0000","last_change":"2023-10-09T11:07:36.287455+0000","last_active":"2023-10-09T11:09:35.114685+0000","last_peered":"2023-10-09T11:09:35.114685+0000","last_clean":"2023-10-09T11:09:35.114685+0000","last_became_active":"2023-10-09T11:07:36.287052+0000","last_became_peered":"2023-10-09T11:07:36.287052+0000","last_unstale":"2023-10-09T11:09:35.114685+0000","last_undegraded":"2023-10-09T11:09:35.114685+0000","last_fullsized":"2023-10-09T11:09:35.114685+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:18.864873+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":380,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.009484+0000","last_change":"2023-10-09T11:03:21.185383+0000","last_active":"2023-10-09T11:09:35.009484+0000","last_peered":"2023-10-09T11:09:35.009484+0000","last_clean":"2023-10-09T11:09:35.009484+0000","last_became_active":"2023-10-09T11:03:21.185022+0000","last_became_peered":"2023-10-09T11:03:21.185022+0000","last_unstale":"2023-10-09T11:09:35.009484+0000","last_undegraded":"2023-10-09T11:09:35.009484+0000","last_fullsized":"2023-10-09T11:09:35.009484+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:05.230434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":124,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575492+0000","last_change":"2023-10-09T11:04:50.319477+0000","last_active":"2023-10-09T11:09:31.575492+0000","last_peered":"2023-10-09T11:09:31.575492+0000","last_clean":"2023-10-09T11:09:31.575492+0000","last_became_active":"2023-10-09T11:04:50.319148+0000","last_became_peered":"2023-10-09T11:04:50.319148+0000","last_unstale":"2023-10-09T11:09:31.575492+0000","last_undegraded":"2023-10-09T11:09:31.575492+0000","last_fullsized":"2023-10-09T11:09:31.575492+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:48.725566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":754,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114999+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:09:35.114999+0000","last_peered":"2023-10-09T11:09:35.114999+0000","last_clean":"2023-10-09T11:09:35.114999+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:09:35.114999+0000","last_undegraded":"2023-10-09T11:09:35.114999+0000","last_fullsized":"2023-10-09T11:09:35.114999+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":312,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.099852+0000","last_change":"2023-10-09T11:04:50.333711+0000","last_active":"2023-10-09T11:09:32.099852+0000","last_peered":"2023-10-09T11:09:32.099852+0000","last_clean":"2023-10-09T11:09:32.099852+0000","last_became_active":"2023-10-09T11:04:50.333262+0000","last_became_peered":"2023-10-09T11:04:50.333262+0000","last_unstale":"2023-10-09T11:09:32.099852+0000","last_undegraded":"2023-10-09T11:09:32.099852+0000","last_fullsized":"2023-10-09T11:09:32.099852+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.843765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":59,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422023+0000","last_change":"2023-10-09T11:07:33.237903+0000","last_active":"2023-10-09T11:09:34.422023+0000","last_peered":"2023-10-09T11:09:34.422023+0000","last_clean":"2023-10-09T11:09:34.422023+0000","last_became_active":"2023-10-09T11:07:33.237618+0000","last_became_peered":"2023-10-09T11:07:33.237618+0000","last_unstale":"2023-10-09T11:09:34.422023+0000","last_undegraded":"2023-10-09T11:09:34.422023+0000","last_fullsized":"2023-10-09T11:09:34.422023+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:25.600818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":774,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421682+0000","last_change":"2023-10-09T11:07:36.284534+0000","last_active":"2023-10-09T11:09:34.421682+0000","last_peered":"2023-10-09T11:09:34.421682+0000","last_clean":"2023-10-09T11:09:34.421682+0000","last_became_active":"2023-10-09T11:07:36.284254+0000","last_became_peered":"2023-10-09T11:07:36.284254+0000","last_unstale":"2023-10-09T11:09:34.421682+0000","last_undegraded":"2023-10-09T11:09:34.421682+0000","last_fullsized":"2023-10-09T11:09:34.421682+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:19.159630+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":124,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.574900+0000","last_change":"2023-10-09T11:04:50.310477+0000","last_active":"2023-10-09T11:09:31.574900+0000","last_peered":"2023-10-09T11:09:31.574900+0000","last_clean":"2023-10-09T11:09:31.574900+0000","last_became_active":"2023-10-09T11:04:50.310067+0000","last_became_peered":"2023-10-09T11:04:50.310067+0000","last_unstale":"2023-10-09T11:09:31.574900+0000","last_undegraded":"2023-10-09T11:09:31.574900+0000","last_fullsized":"2023-10-09T11:09:31.574900+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:52.884716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":406,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115388+0000","last_change":"2023-10-09T11:05:20.310581+0000","last_active":"2023-10-09T11:09:35.115388+0000","last_peered":"2023-10-09T11:09:35.115388+0000","last_clean":"2023-10-09T11:09:35.115388+0000","last_became_active":"2023-10-09T11:05:20.309830+0000","last_became_peered":"2023-10-09T11:05:20.309830+0000","last_unstale":"2023-10-09T11:09:35.115388+0000","last_undegraded":"2023-10-09T11:09:35.115388+0000","last_fullsized":"2023-10-09T11:09:35.115388+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:24.508211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":814,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422374+0000","last_change":"2023-10-09T11:07:36.282502+0000","last_active":"2023-10-09T11:09:34.422374+0000","last_peered":"2023-10-09T11:09:34.422374+0000","last_clean":"2023-10-09T11:09:34.422374+0000","last_became_active":"2023-10-09T11:07:36.282133+0000","last_became_peered":"2023-10-09T11:07:36.282133+0000","last_unstale":"2023-10-09T11:09:34.422374+0000","last_undegraded":"2023-10-09T11:09:34.422374+0000","last_fullsized":"2023-10-09T11:09:34.422374+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:03.762383+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":233,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115443+0000","last_change":"2023-10-09T11:06:24.245488+0000","last_active":"2023-10-09T11:09:35.115443+0000","last_peered":"2023-10-09T11:09:35.115443+0000","last_clean":"2023-10-09T11:09:35.115443+0000","last_became_active":"2023-10-09T11:06:24.245167+0000","last_became_peered":"2023-10-09T11:06:24.245167+0000","last_unstale":"2023-10-09T11:09:35.115443+0000","last_undegraded":"2023-10-09T11:09:35.115443+0000","last_fullsized":"2023-10-09T11:09:35.115443+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:31.702568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":134,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:33.352991+0000","last_change":"2023-10-09T11:05:20.299453+0000","last_active":"2023-10-09T11:09:33.352991+0000","last_peered":"2023-10-09T11:09:33.352991+0000","last_clean":"2023-10-09T11:09:33.352991+0000","last_became_active":"2023-10-09T11:05:20.299117+0000","last_became_peered":"2023-10-09T11:05:20.299117+0000","last_unstale":"2023-10-09T11:09:33.352991+0000","last_undegraded":"2023-10-09T11:09:33.352991+0000","last_fullsized":"2023-10-09T11:09:33.352991+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:30.072813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":929,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.112627+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:09:35.112627+0000","last_peered":"2023-10-09T11:09:35.112627+0000","last_clean":"2023-10-09T11:09:35.112627+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:09:35.112627+0000","last_undegraded":"2023-10-09T11:09:35.112627+0000","last_fullsized":"2023-10-09T11:09:35.112627+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":468,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115221+0000","last_change":"2023-10-09T11:07:33.242665+0000","last_active":"2023-10-09T11:09:35.115221+0000","last_peered":"2023-10-09T11:09:35.115221+0000","last_clean":"2023-10-09T11:09:35.115221+0000","last_became_active":"2023-10-09T11:07:33.242313+0000","last_became_peered":"2023-10-09T11:07:33.242313+0000","last_unstale":"2023-10-09T11:09:35.115221+0000","last_undegraded":"2023-10-09T11:09:35.115221+0000","last_fullsized":"2023-10-09T11:09:35.115221+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:20.565135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":152,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:33.353046+0000","last_change":"2023-10-09T11:07:33.225683+0000","last_active":"2023-10-09T11:09:33.353046+0000","last_peered":"2023-10-09T11:09:33.353046+0000","last_clean":"2023-10-09T11:09:33.353046+0000","last_became_active":"2023-10-09T11:07:33.225275+0000","last_became_peered":"2023-10-09T11:07:33.225275+0000","last_unstale":"2023-10-09T11:09:33.353046+0000","last_undegraded":"2023-10-09T11:09:33.353046+0000","last_fullsized":"2023-10-09T11:09:33.353046+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:10.131060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":826,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115215+0000","last_change":"2023-10-09T11:05:23.328981+0000","last_active":"2023-10-09T11:09:35.115215+0000","last_peered":"2023-10-09T11:09:35.115215+0000","last_clean":"2023-10-09T11:09:35.115215+0000","last_became_active":"2023-10-09T11:05:23.328616+0000","last_became_peered":"2023-10-09T11:05:23.328616+0000","last_unstale":"2023-10-09T11:09:35.115215+0000","last_undegraded":"2023-10-09T11:09:35.115215+0000","last_fullsized":"2023-10-09T11:09:35.115215+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:15.315819+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":65,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422309+0000","last_change":"2023-10-09T11:07:33.265842+0000","last_active":"2023-10-09T11:09:34.422309+0000","last_peered":"2023-10-09T11:09:34.422309+0000","last_clean":"2023-10-09T11:09:34.422309+0000","last_became_active":"2023-10-09T11:07:33.232311+0000","last_became_peered":"2023-10-09T11:07:33.232311+0000","last_unstale":"2023-10-09T11:09:34.422309+0000","last_undegraded":"2023-10-09T11:09:34.422309+0000","last_fullsized":"2023-10-09T11:09:34.422309+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:23.402582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":375,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114846+0000","last_change":"2023-10-09T11:05:20.316265+0000","last_active":"2023-10-09T11:09:35.114846+0000","last_peered":"2023-10-09T11:09:35.114846+0000","last_clean":"2023-10-09T11:09:35.114846+0000","last_became_active":"2023-10-09T11:05:20.315853+0000","last_became_peered":"2023-10-09T11:05:20.315853+0000","last_unstale":"2023-10-09T11:09:35.114846+0000","last_undegraded":"2023-10-09T11:09:35.114846+0000","last_fullsized":"2023-10-09T11:09:35.114846+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:35.093981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":990,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100956+0000","last_change":"2023-10-09T11:04:53.274921+0000","last_active":"2023-10-09T11:09:32.100956+0000","last_peered":"2023-10-09T11:09:32.100956+0000","last_clean":"2023-10-09T11:09:32.100956+0000","last_became_active":"2023-10-09T11:04:53.274518+0000","last_became_peered":"2023-10-09T11:04:53.274518+0000","last_unstale":"2023-10-09T11:09:32.100956+0000","last_undegraded":"2023-10-09T11:09:32.100956+0000","last_fullsized":"2023-10-09T11:09:32.100956+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:14.315301+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":69,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422034+0000","last_change":"2023-10-09T11:07:33.262559+0000","last_active":"2023-10-09T11:09:34.422034+0000","last_peered":"2023-10-09T11:09:34.422034+0000","last_clean":"2023-10-09T11:09:34.422034+0000","last_became_active":"2023-10-09T11:07:33.231695+0000","last_became_peered":"2023-10-09T11:07:33.231695+0000","last_unstale":"2023-10-09T11:09:34.422034+0000","last_undegraded":"2023-10-09T11:09:34.422034+0000","last_fullsized":"2023-10-09T11:09:34.422034+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:59.006940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":621,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-10-09T11:09:38.434817+0000","last_change":"2023-10-09T11:05:20.311424+0000","last_active":"2023-10-09T11:09:38.434817+0000","last_peered":"2023-10-09T11:09:38.434817+0000","last_clean":"2023-10-09T11:09:38.434817+0000","last_became_active":"2023-10-09T11:05:20.311097+0000","last_became_peered":"2023-10-09T11:05:20.311097+0000","last_unstale":"2023-10-09T11:09:38.434817+0000","last_undegraded":"2023-10-09T11:09:38.434817+0000","last_fullsized":"2023-10-09T11:09:38.434817+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:19.274582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":639,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114885+0000","last_change":"2023-10-09T11:08:39.934938+0000","last_active":"2023-10-09T11:09:35.114885+0000","last_peered":"2023-10-09T11:09:35.114885+0000","last_clean":"2023-10-09T11:09:35.114885+0000","last_became_active":"2023-10-09T11:06:24.245076+0000","last_became_peered":"2023-10-09T11:06:24.245076+0000","last_unstale":"2023-10-09T11:09:35.114885+0000","last_undegraded":"2023-10-09T11:09:35.114885+0000","last_fullsized":"2023-10-09T11:09:35.114885+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.934865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:00.408009+0000","scrub_duration":0.0012831839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":65,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421699+0000","last_change":"2023-10-09T11:07:33.269200+0000","last_active":"2023-10-09T11:09:34.421699+0000","last_peered":"2023-10-09T11:09:34.421699+0000","last_clean":"2023-10-09T11:09:34.421699+0000","last_became_active":"2023-10-09T11:07:33.236205+0000","last_became_peered":"2023-10-09T11:07:33.236205+0000","last_unstale":"2023-10-09T11:09:34.421699+0000","last_undegraded":"2023-10-09T11:09:34.421699+0000","last_fullsized":"2023-10-09T11:09:34.421699+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:24.574734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":143,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:33.353156+0000","last_change":"2023-10-09T11:07:33.227803+0000","last_active":"2023-10-09T11:09:33.353156+0000","last_peered":"2023-10-09T11:09:33.353156+0000","last_clean":"2023-10-09T11:09:33.353156+0000","last_became_active":"2023-10-09T11:07:33.227486+0000","last_became_peered":"2023-10-09T11:07:33.227486+0000","last_unstale":"2023-10-09T11:09:33.353156+0000","last_undegraded":"2023-10-09T11:09:33.353156+0000","last_fullsized":"2023-10-09T11:09:33.353156+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:39.274875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":908,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114935+0000","last_change":"2023-10-09T11:05:23.323856+0000","last_active":"2023-10-09T11:09:35.114935+0000","last_peered":"2023-10-09T11:09:35.114935+0000","last_clean":"2023-10-09T11:09:35.114935+0000","last_became_active":"2023-10-09T11:05:23.322763+0000","last_became_peered":"2023-10-09T11:05:23.322763+0000","last_unstale":"2023-10-09T11:09:35.114935+0000","last_undegraded":"2023-10-09T11:09:35.114935+0000","last_fullsized":"2023-10-09T11:09:35.114935+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.760820+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":59,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421588+0000","last_change":"2023-10-09T11:07:33.233510+0000","last_active":"2023-10-09T11:09:34.421588+0000","last_peered":"2023-10-09T11:09:34.421588+0000","last_clean":"2023-10-09T11:09:34.421588+0000","last_became_active":"2023-10-09T11:07:33.232339+0000","last_became_peered":"2023-10-09T11:07:33.232339+0000","last_unstale":"2023-10-09T11:09:34.421588+0000","last_undegraded":"2023-10-09T11:09:34.421588+0000","last_fullsized":"2023-10-09T11:09:34.421588+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:55.324912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":538,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115298+0000","last_change":"2023-10-09T11:07:33.226594+0000","last_active":"2023-10-09T11:09:35.115298+0000","last_peered":"2023-10-09T11:09:35.115298+0000","last_clean":"2023-10-09T11:09:35.115298+0000","last_became_active":"2023-10-09T11:07:33.226275+0000","last_became_peered":"2023-10-09T11:07:33.226275+0000","last_unstale":"2023-10-09T11:09:35.115298+0000","last_undegraded":"2023-10-09T11:09:35.115298+0000","last_fullsized":"2023-10-09T11:09:35.115298+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:41.180088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":932,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:33.388720+0000","last_change":"2023-10-09T11:04:53.274236+0000","last_active":"2023-10-09T11:09:33.388720+0000","last_peered":"2023-10-09T11:09:33.388720+0000","last_clean":"2023-10-09T11:09:33.388720+0000","last_became_active":"2023-10-09T11:04:53.273817+0000","last_became_peered":"2023-10-09T11:04:53.273817+0000","last_unstale":"2023-10-09T11:09:33.388720+0000","last_undegraded":"2023-10-09T11:09:33.388720+0000","last_fullsized":"2023-10-09T11:09:33.388720+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:05.096905+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":194,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421899+0000","last_change":"2023-10-09T11:05:20.312129+0000","last_active":"2023-10-09T11:09:34.421899+0000","last_peered":"2023-10-09T11:09:34.421899+0000","last_clean":"2023-10-09T11:09:34.421899+0000","last_became_active":"2023-10-09T11:05:20.311810+0000","last_became_peered":"2023-10-09T11:05:20.311810+0000","last_unstale":"2023-10-09T11:09:34.421899+0000","last_undegraded":"2023-10-09T11:09:34.421899+0000","last_fullsized":"2023-10-09T11:09:34.421899+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:48.813391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":292,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575800+0000","last_change":"2023-10-09T11:07:33.218244+0000","last_active":"2023-10-09T11:09:31.575800+0000","last_peered":"2023-10-09T11:09:31.575800+0000","last_clean":"2023-10-09T11:09:31.575800+0000","last_became_active":"2023-10-09T11:07:33.217918+0000","last_became_peered":"2023-10-09T11:07:33.217918+0000","last_unstale":"2023-10-09T11:09:31.575800+0000","last_undegraded":"2023-10-09T11:09:31.575800+0000","last_fullsized":"2023-10-09T11:09:31.575800+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:16.150105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":905,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115096+0000","last_change":"2023-10-09T11:05:23.324537+0000","last_active":"2023-10-09T11:09:35.115096+0000","last_peered":"2023-10-09T11:09:35.115096+0000","last_clean":"2023-10-09T11:09:35.115096+0000","last_became_active":"2023-10-09T11:05:23.324187+0000","last_became_peered":"2023-10-09T11:05:23.324187+0000","last_unstale":"2023-10-09T11:09:35.115096+0000","last_undegraded":"2023-10-09T11:09:35.115096+0000","last_fullsized":"2023-10-09T11:09:35.115096+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:25.286106+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":117,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421726+0000","last_change":"2023-10-09T11:05:20.323565+0000","last_active":"2023-10-09T11:09:34.421726+0000","last_peered":"2023-10-09T11:09:34.421726+0000","last_clean":"2023-10-09T11:09:34.421726+0000","last_became_active":"2023-10-09T11:05:20.309672+0000","last_became_peered":"2023-10-09T11:05:20.309672+0000","last_unstale":"2023-10-09T11:09:34.421726+0000","last_undegraded":"2023-10-09T11:09:34.421726+0000","last_fullsized":"2023-10-09T11:09:34.421726+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:50.211465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":142,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576662+0000","last_change":"2023-10-09T11:07:33.228001+0000","last_active":"2023-10-09T11:09:31.576662+0000","last_peered":"2023-10-09T11:09:31.576662+0000","last_clean":"2023-10-09T11:09:31.576662+0000","last_became_active":"2023-10-09T11:07:33.227691+0000","last_became_peered":"2023-10-09T11:07:33.227691+0000","last_unstale":"2023-10-09T11:09:31.576662+0000","last_undegraded":"2023-10-09T11:09:31.576662+0000","last_fullsized":"2023-10-09T11:09:31.576662+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:24:16.098138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":831,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422390+0000","last_change":"2023-10-09T11:07:36.284927+0000","last_active":"2023-10-09T11:09:34.422390+0000","last_peered":"2023-10-09T11:09:34.422390+0000","last_clean":"2023-10-09T11:09:34.422390+0000","last_became_active":"2023-10-09T11:07:36.284631+0000","last_became_peered":"2023-10-09T11:07:36.284631+0000","last_unstale":"2023-10-09T11:09:34.422390+0000","last_undegraded":"2023-10-09T11:09:34.422390+0000","last_fullsized":"2023-10-09T11:09:34.422390+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:37.220243+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":63,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422056+0000","last_change":"2023-10-09T11:07:33.273503+0000","last_active":"2023-10-09T11:09:34.422056+0000","last_peered":"2023-10-09T11:09:34.422056+0000","last_clean":"2023-10-09T11:09:34.422056+0000","last_became_active":"2023-10-09T11:07:33.242980+0000","last_became_peered":"2023-10-09T11:07:33.242980+0000","last_unstale":"2023-10-09T11:09:34.422056+0000","last_undegraded":"2023-10-09T11:09:34.422056+0000","last_fullsized":"2023-10-09T11:09:34.422056+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:18.934972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":559,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114700+0000","last_change":"2023-10-09T11:07:33.304363+0000","last_active":"2023-10-09T11:09:35.114700+0000","last_peered":"2023-10-09T11:09:35.114700+0000","last_clean":"2023-10-09T11:09:35.114700+0000","last_became_active":"2023-10-09T11:07:33.304024+0000","last_became_peered":"2023-10-09T11:07:33.304024+0000","last_unstale":"2023-10-09T11:09:35.114700+0000","last_undegraded":"2023-10-09T11:09:35.114700+0000","last_fullsized":"2023-10-09T11:09:35.114700+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:21:51.522688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":980,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575743+0000","last_change":"2023-10-09T11:05:23.316129+0000","last_active":"2023-10-09T11:09:31.575743+0000","last_peered":"2023-10-09T11:09:31.575743+0000","last_clean":"2023-10-09T11:09:31.575743+0000","last_became_active":"2023-10-09T11:05:23.315792+0000","last_became_peered":"2023-10-09T11:05:23.315792+0000","last_unstale":"2023-10-09T11:09:31.575743+0000","last_undegraded":"2023-10-09T11:09:31.575743+0000","last_fullsized":"2023-10-09T11:09:31.575743+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:51.770547+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":63,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422114+0000","last_change":"2023-10-09T11:07:33.256691+0000","last_active":"2023-10-09T11:09:34.422114+0000","last_peered":"2023-10-09T11:09:34.422114+0000","last_clean":"2023-10-09T11:09:34.422114+0000","last_became_active":"2023-10-09T11:07:33.229554+0000","last_became_peered":"2023-10-09T11:07:33.229554+0000","last_unstale":"2023-10-09T11:09:34.422114+0000","last_undegraded":"2023-10-09T11:09:34.422114+0000","last_fullsized":"2023-10-09T11:09:34.422114+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:57.414795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":313,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.099814+0000","last_change":"2023-10-09T11:04:50.333538+0000","last_active":"2023-10-09T11:09:32.099814+0000","last_peered":"2023-10-09T11:09:32.099814+0000","last_clean":"2023-10-09T11:09:32.099814+0000","last_became_active":"2023-10-09T11:04:50.332514+0000","last_became_peered":"2023-10-09T11:04:50.332514+0000","last_unstale":"2023-10-09T11:09:32.099814+0000","last_undegraded":"2023-10-09T11:09:32.099814+0000","last_fullsized":"2023-10-09T11:09:32.099814+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:22.933839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":853,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115321+0000","last_change":"2023-10-09T11:05:23.322251+0000","last_active":"2023-10-09T11:09:35.115321+0000","last_peered":"2023-10-09T11:09:35.115321+0000","last_clean":"2023-10-09T11:09:35.115321+0000","last_became_active":"2023-10-09T11:05:23.321826+0000","last_became_peered":"2023-10-09T11:05:23.321826+0000","last_unstale":"2023-10-09T11:09:35.115321+0000","last_undegraded":"2023-10-09T11:09:35.115321+0000","last_fullsized":"2023-10-09T11:09:35.115321+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.858013+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":63,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422551+0000","last_change":"2023-10-09T11:07:33.271238+0000","last_active":"2023-10-09T11:09:34.422551+0000","last_peered":"2023-10-09T11:09:34.422551+0000","last_clean":"2023-10-09T11:09:34.422551+0000","last_became_active":"2023-10-09T11:07:33.241886+0000","last_became_peered":"2023-10-09T11:07:33.241886+0000","last_unstale":"2023-10-09T11:09:34.422551+0000","last_undegraded":"2023-10-09T11:09:34.422551+0000","last_fullsized":"2023-10-09T11:09:34.422551+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:48.882589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":127,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422757+0000","last_change":"2023-10-09T11:07:33.244131+0000","last_active":"2023-10-09T11:09:34.422757+0000","last_peered":"2023-10-09T11:09:34.422757+0000","last_clean":"2023-10-09T11:09:34.422757+0000","last_became_active":"2023-10-09T11:07:33.243797+0000","last_became_peered":"2023-10-09T11:07:33.243797+0000","last_unstale":"2023-10-09T11:09:34.422757+0000","last_undegraded":"2023-10-09T11:09:34.422757+0000","last_fullsized":"2023-10-09T11:09:34.422757+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:44.851043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":1109,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422231+0000","last_change":"2023-10-09T11:07:36.286417+0000","last_active":"2023-10-09T11:09:34.422231+0000","last_peered":"2023-10-09T11:09:34.422231+0000","last_clean":"2023-10-09T11:09:34.422231+0000","last_became_active":"2023-10-09T11:07:36.286061+0000","last_became_peered":"2023-10-09T11:07:36.286061+0000","last_unstale":"2023-10-09T11:09:34.422231+0000","last_undegraded":"2023-10-09T11:09:34.422231+0000","last_fullsized":"2023-10-09T11:09:34.422231+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:00.289500+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":455,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115174+0000","last_change":"2023-10-09T11:05:20.309385+0000","last_active":"2023-10-09T11:09:35.115174+0000","last_peered":"2023-10-09T11:09:35.115174+0000","last_clean":"2023-10-09T11:09:35.115174+0000","last_became_active":"2023-10-09T11:05:20.309034+0000","last_became_peered":"2023-10-09T11:05:20.309034+0000","last_unstale":"2023-10-09T11:09:35.115174+0000","last_undegraded":"2023-10-09T11:09:35.115174+0000","last_fullsized":"2023-10-09T11:09:35.115174+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:31.857485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":388,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100071+0000","last_change":"2023-10-09T11:06:24.245966+0000","last_active":"2023-10-09T11:09:32.100071+0000","last_peered":"2023-10-09T11:09:32.100071+0000","last_clean":"2023-10-09T11:09:32.100071+0000","last_became_active":"2023-10-09T11:06:24.245310+0000","last_became_peered":"2023-10-09T11:06:24.245310+0000","last_unstale":"2023-10-09T11:09:32.100071+0000","last_undegraded":"2023-10-09T11:09:32.100071+0000","last_fullsized":"2023-10-09T11:09:32.100071+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:23.966200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":753,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421779+0000","last_change":"2023-10-09T11:07:36.283174+0000","last_active":"2023-10-09T11:09:34.421779+0000","last_peered":"2023-10-09T11:09:34.421779+0000","last_clean":"2023-10-09T11:09:34.421779+0000","last_became_active":"2023-10-09T11:07:36.278007+0000","last_became_peered":"2023-10-09T11:07:36.278007+0000","last_unstale":"2023-10-09T11:09:34.421779+0000","last_undegraded":"2023-10-09T11:09:34.421779+0000","last_fullsized":"2023-10-09T11:09:34.421779+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:51.693969+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":129,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575547+0000","last_change":"2023-10-09T11:04:50.529513+0000","last_active":"2023-10-09T11:09:31.575547+0000","last_peered":"2023-10-09T11:09:31.575547+0000","last_clean":"2023-10-09T11:09:31.575547+0000","last_became_active":"2023-10-09T11:04:50.339671+0000","last_became_peered":"2023-10-09T11:04:50.339671+0000","last_unstale":"2023-10-09T11:09:31.575547+0000","last_undegraded":"2023-10-09T11:09:31.575547+0000","last_fullsized":"2023-10-09T11:09:31.575547+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:24.473000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":134,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575656+0000","last_change":"2023-10-09T11:05:20.313855+0000","last_active":"2023-10-09T11:09:31.575656+0000","last_peered":"2023-10-09T11:09:31.575656+0000","last_clean":"2023-10-09T11:09:31.575656+0000","last_became_active":"2023-10-09T11:05:20.313535+0000","last_became_peered":"2023-10-09T11:05:20.313535+0000","last_unstale":"2023-10-09T11:09:31.575656+0000","last_undegraded":"2023-10-09T11:09:31.575656+0000","last_fullsized":"2023-10-09T11:09:31.575656+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:25.266434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":855,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114836+0000","last_change":"2023-10-09T11:04:53.272323+0000","last_active":"2023-10-09T11:09:35.114836+0000","last_peered":"2023-10-09T11:09:35.114836+0000","last_clean":"2023-10-09T11:09:35.114836+0000","last_became_active":"2023-10-09T11:04:53.272010+0000","last_became_peered":"2023-10-09T11:04:53.272010+0000","last_unstale":"2023-10-09T11:09:35.114836+0000","last_undegraded":"2023-10-09T11:09:35.114836+0000","last_fullsized":"2023-10-09T11:09:35.114836+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:23.913605+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":622,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100051+0000","last_change":"2023-10-09T11:05:20.307499+0000","last_active":"2023-10-09T11:09:32.100051+0000","last_peered":"2023-10-09T11:09:32.100051+0000","last_clean":"2023-10-09T11:09:32.100051+0000","last_became_active":"2023-10-09T11:05:20.307198+0000","last_became_peered":"2023-10-09T11:05:20.307198+0000","last_unstale":"2023-10-09T11:09:32.100051+0000","last_undegraded":"2023-10-09T11:09:32.100051+0000","last_fullsized":"2023-10-09T11:09:32.100051+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:53.726844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":358,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.111904+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:09:35.111904+0000","last_peered":"2023-10-09T11:09:35.111904+0000","last_clean":"2023-10-09T11:09:35.111904+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:09:35.111904+0000","last_undegraded":"2023-10-09T11:09:35.111904+0000","last_fullsized":"2023-10-09T11:09:35.111904+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":949,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100248+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:09:32.100248+0000","last_peered":"2023-10-09T11:09:32.100248+0000","last_clean":"2023-10-09T11:09:32.100248+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:09:32.100248+0000","last_undegraded":"2023-10-09T11:09:32.100248+0000","last_fullsized":"2023-10-09T11:09:32.100248+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":622,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.111960+0000","last_change":"2023-10-09T11:04:57.953307+0000","last_active":"2023-10-09T11:09:35.111960+0000","last_peered":"2023-10-09T11:09:35.111960+0000","last_clean":"2023-10-09T11:09:35.111960+0000","last_became_active":"2023-10-09T11:04:57.953095+0000","last_became_peered":"2023-10-09T11:04:57.953095+0000","last_unstale":"2023-10-09T11:09:35.111960+0000","last_undegraded":"2023-10-09T11:09:35.111960+0000","last_fullsized":"2023-10-09T11:09:35.111960+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:05.562643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":307,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100741+0000","last_change":"2023-10-09T11:05:20.310255+0000","last_active":"2023-10-09T11:09:32.100741+0000","last_peered":"2023-10-09T11:09:32.100741+0000","last_clean":"2023-10-09T11:09:32.100741+0000","last_became_active":"2023-10-09T11:05:20.309109+0000","last_became_peered":"2023-10-09T11:05:20.309109+0000","last_unstale":"2023-10-09T11:09:32.100741+0000","last_undegraded":"2023-10-09T11:09:32.100741+0000","last_fullsized":"2023-10-09T11:09:32.100741+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:07.429039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":839,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115026+0000","last_change":"2023-10-09T11:05:23.329329+0000","last_active":"2023-10-09T11:09:35.115026+0000","last_peered":"2023-10-09T11:09:35.115026+0000","last_clean":"2023-10-09T11:09:35.115026+0000","last_became_active":"2023-10-09T11:05:23.328976+0000","last_became_peered":"2023-10-09T11:05:23.328976+0000","last_unstale":"2023-10-09T11:09:35.115026+0000","last_undegraded":"2023-10-09T11:09:35.115026+0000","last_fullsized":"2023-10-09T11:09:35.115026+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:41.848184+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":143,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575601+0000","last_change":"2023-10-09T11:07:33.236133+0000","last_active":"2023-10-09T11:09:31.575601+0000","last_peered":"2023-10-09T11:09:31.575601+0000","last_clean":"2023-10-09T11:09:31.575601+0000","last_became_active":"2023-10-09T11:07:33.235818+0000","last_became_peered":"2023-10-09T11:07:33.235818+0000","last_unstale":"2023-10-09T11:09:31.575601+0000","last_undegraded":"2023-10-09T11:09:31.575601+0000","last_fullsized":"2023-10-09T11:09:31.575601+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:20.627846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":304,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100156+0000","last_change":"2023-10-09T11:04:50.332978+0000","last_active":"2023-10-09T11:09:32.100156+0000","last_peered":"2023-10-09T11:09:32.100156+0000","last_clean":"2023-10-09T11:09:32.100156+0000","last_became_active":"2023-10-09T11:04:50.332735+0000","last_became_peered":"2023-10-09T11:04:50.332735+0000","last_unstale":"2023-10-09T11:09:32.100156+0000","last_undegraded":"2023-10-09T11:09:32.100156+0000","last_fullsized":"2023-10-09T11:09:32.100156+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:39:44.659751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":1125,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576746+0000","last_change":"2023-10-09T11:07:36.291716+0000","last_active":"2023-10-09T11:09:31.576746+0000","last_peered":"2023-10-09T11:09:31.576746+0000","last_clean":"2023-10-09T11:09:31.576746+0000","last_became_active":"2023-10-09T11:07:36.291424+0000","last_became_peered":"2023-10-09T11:07:36.291424+0000","last_unstale":"2023-10-09T11:09:31.576746+0000","last_undegraded":"2023-10-09T11:09:31.576746+0000","last_fullsized":"2023-10-09T11:09:31.576746+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:22.316387+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"592'150","reported_seq":883,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100251+0000","last_change":"2023-10-09T11:09:20.424914+0000","last_active":"2023-10-09T11:09:32.100251+0000","last_peered":"2023-10-09T11:09:32.100251+0000","last_clean":"2023-10-09T11:09:32.100251+0000","last_became_active":"2023-10-09T11:07:33.226176+0000","last_became_peered":"2023-10-09T11:07:33.226176+0000","last_unstale":"2023-10-09T11:09:32.100251+0000","last_undegraded":"2023-10-09T11:09:32.100251+0000","last_fullsized":"2023-10-09T11:09:32.100251+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:16.888732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034113299999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":876,"num_read_kb":2947,"num_write":148,"num_write_kb":2179,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":472,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115215+0000","last_change":"2023-10-09T11:04:50.341353+0000","last_active":"2023-10-09T11:09:35.115215+0000","last_peered":"2023-10-09T11:09:35.115215+0000","last_clean":"2023-10-09T11:09:35.115215+0000","last_became_active":"2023-10-09T11:04:50.340801+0000","last_became_peered":"2023-10-09T11:04:50.340801+0000","last_unstale":"2023-10-09T11:09:35.115215+0000","last_undegraded":"2023-10-09T11:09:35.115215+0000","last_fullsized":"2023-10-09T11:09:35.115215+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:00.699069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":280,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100219+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:09:32.100219+0000","last_peered":"2023-10-09T11:09:32.100219+0000","last_clean":"2023-10-09T11:09:32.100219+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:09:32.100219+0000","last_undegraded":"2023-10-09T11:09:32.100219+0000","last_fullsized":"2023-10-09T11:09:32.100219+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":797,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114791+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:09:35.114791+0000","last_peered":"2023-10-09T11:09:35.114791+0000","last_clean":"2023-10-09T11:09:35.114791+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:09:35.114791+0000","last_undegraded":"2023-10-09T11:09:35.114791+0000","last_fullsized":"2023-10-09T11:09:35.114791+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"592'149","reported_seq":1214,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114904+0000","last_change":"2023-10-09T11:09:20.477238+0000","last_active":"2023-10-09T11:09:35.114904+0000","last_peered":"2023-10-09T11:09:35.114904+0000","last_clean":"2023-10-09T11:09:35.114904+0000","last_became_active":"2023-10-09T11:05:20.315471+0000","last_became_peered":"2023-10-09T11:05:20.315471+0000","last_unstale":"2023-10-09T11:09:35.114904+0000","last_undegraded":"2023-10-09T11:09:35.114904+0000","last_fullsized":"2023-10-09T11:09:35.114904+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:47.548763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051877200000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1181,"num_read_kb":1421,"num_write":125,"num_write_kb":1278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":503,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422022+0000","last_change":"2023-10-09T11:06:24.245282+0000","last_active":"2023-10-09T11:09:34.422022+0000","last_peered":"2023-10-09T11:09:34.422022+0000","last_clean":"2023-10-09T11:09:34.422022+0000","last_became_active":"2023-10-09T11:06:24.244981+0000","last_became_peered":"2023-10-09T11:06:24.244981+0000","last_unstale":"2023-10-09T11:09:34.422022+0000","last_undegraded":"2023-10-09T11:09:34.422022+0000","last_fullsized":"2023-10-09T11:09:34.422022+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:15.327655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":313,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100020+0000","last_change":"2023-10-09T11:04:50.311579+0000","last_active":"2023-10-09T11:09:32.100020+0000","last_peered":"2023-10-09T11:09:32.100020+0000","last_clean":"2023-10-09T11:09:32.100020+0000","last_became_active":"2023-10-09T11:04:50.311270+0000","last_became_peered":"2023-10-09T11:04:50.311270+0000","last_unstale":"2023-10-09T11:09:32.100020+0000","last_undegraded":"2023-10-09T11:09:32.100020+0000","last_fullsized":"2023-10-09T11:09:32.100020+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:49.850302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":951,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421888+0000","last_change":"2023-10-09T11:07:36.284280+0000","last_active":"2023-10-09T11:09:34.421888+0000","last_peered":"2023-10-09T11:09:34.421888+0000","last_clean":"2023-10-09T11:09:34.421888+0000","last_became_active":"2023-10-09T11:07:36.283966+0000","last_became_peered":"2023-10-09T11:07:36.283966+0000","last_unstale":"2023-10-09T11:09:34.421888+0000","last_undegraded":"2023-10-09T11:09:34.421888+0000","last_fullsized":"2023-10-09T11:09:34.421888+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:54.128331+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"591'48","reported_seq":610,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.112595+0000","last_change":"2023-10-09T11:09:20.421917+0000","last_active":"2023-10-09T11:09:35.112595+0000","last_peered":"2023-10-09T11:09:35.112595+0000","last_clean":"2023-10-09T11:09:35.112595+0000","last_became_active":"2023-10-09T11:04:50.381253+0000","last_became_peered":"2023-10-09T11:04:50.381253+0000","last_unstale":"2023-10-09T11:09:35.112595+0000","last_undegraded":"2023-10-09T11:09:35.112595+0000","last_fullsized":"2023-10-09T11:09:35.112595+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":48,"log_dups_size":0,"ondisk_log_size":48,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:24.030274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001144136,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":297,"num_read_kb":2229,"num_write":37,"num_write_kb":2061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":298,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.099847+0000","last_change":"2023-10-09T11:05:20.298542+0000","last_active":"2023-10-09T11:09:32.099847+0000","last_peered":"2023-10-09T11:09:32.099847+0000","last_clean":"2023-10-09T11:09:32.099847+0000","last_became_active":"2023-10-09T11:05:20.298119+0000","last_became_peered":"2023-10-09T11:05:20.298119+0000","last_unstale":"2023-10-09T11:09:32.099847+0000","last_undegraded":"2023-10-09T11:09:32.099847+0000","last_fullsized":"2023-10-09T11:09:32.099847+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:00.827214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":278,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100321+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:09:32.100321+0000","last_peered":"2023-10-09T11:09:32.100321+0000","last_clean":"2023-10-09T11:09:32.100321+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:09:32.100321+0000","last_undegraded":"2023-10-09T11:09:32.100321+0000","last_fullsized":"2023-10-09T11:09:32.100321+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"592'125","reported_seq":793,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115065+0000","last_change":"2023-10-09T11:09:20.477093+0000","last_active":"2023-10-09T11:09:35.115065+0000","last_peered":"2023-10-09T11:09:35.115065+0000","last_clean":"2023-10-09T11:09:35.115065+0000","last_became_active":"2023-10-09T11:07:33.226289+0000","last_became_peered":"2023-10-09T11:07:33.226289+0000","last_unstale":"2023-10-09T11:09:35.115065+0000","last_undegraded":"2023-10-09T11:09:35.115065+0000","last_fullsized":"2023-10-09T11:09:35.115065+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:15.605946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026939299999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":322,"num_read_kb":786,"num_write":116,"num_write_kb":327,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":896,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421964+0000","last_change":"2023-10-09T11:07:36.290700+0000","last_active":"2023-10-09T11:09:34.421964+0000","last_peered":"2023-10-09T11:09:34.421964+0000","last_clean":"2023-10-09T11:09:34.421964+0000","last_became_active":"2023-10-09T11:07:36.290357+0000","last_became_peered":"2023-10-09T11:07:36.290357+0000","last_unstale":"2023-10-09T11:09:34.421964+0000","last_undegraded":"2023-10-09T11:09:34.421964+0000","last_fullsized":"2023-10-09T11:09:34.421964+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:34.494777+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":134,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575245+0000","last_change":"2023-10-09T11:05:20.310666+0000","last_active":"2023-10-09T11:09:31.575245+0000","last_peered":"2023-10-09T11:09:31.575245+0000","last_clean":"2023-10-09T11:09:31.575245+0000","last_became_active":"2023-10-09T11:05:20.310413+0000","last_became_peered":"2023-10-09T11:05:20.310413+0000","last_unstale":"2023-10-09T11:09:31.575245+0000","last_undegraded":"2023-10-09T11:09:31.575245+0000","last_fullsized":"2023-10-09T11:09:31.575245+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:34.169905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":559,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422486+0000","last_change":"2023-10-09T11:05:20.316523+0000","last_active":"2023-10-09T11:09:34.422486+0000","last_peered":"2023-10-09T11:09:34.422486+0000","last_clean":"2023-10-09T11:09:34.422486+0000","last_became_active":"2023-10-09T11:05:20.314493+0000","last_became_peered":"2023-10-09T11:05:20.314493+0000","last_unstale":"2023-10-09T11:09:34.422486+0000","last_undegraded":"2023-10-09T11:09:34.422486+0000","last_fullsized":"2023-10-09T11:09:34.422486+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:41.618701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":969,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.099991+0000","last_change":"2023-10-09T11:05:23.322074+0000","last_active":"2023-10-09T11:09:32.099991+0000","last_peered":"2023-10-09T11:09:32.099991+0000","last_clean":"2023-10-09T11:09:32.099991+0000","last_became_active":"2023-10-09T11:05:23.321749+0000","last_became_peered":"2023-10-09T11:05:23.321749+0000","last_unstale":"2023-10-09T11:09:32.099991+0000","last_undegraded":"2023-10-09T11:09:32.099991+0000","last_fullsized":"2023-10-09T11:09:32.099991+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:07.010932+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"591'46","reported_seq":353,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421703+0000","last_change":"2023-10-09T11:09:20.432327+0000","last_active":"2023-10-09T11:09:34.421703+0000","last_peered":"2023-10-09T11:09:34.421703+0000","last_clean":"2023-10-09T11:09:34.421703+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:09:34.421703+0000","last_undegraded":"2023-10-09T11:09:34.421703+0000","last_fullsized":"2023-10-09T11:09:34.421703+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":46,"log_dups_size":0,"ondisk_log_size":46,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000350947,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":271,"num_read_kb":1543,"num_write":39,"num_write_kb":1183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":225,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575963+0000","last_change":"2023-10-09T11:06:24.248362+0000","last_active":"2023-10-09T11:09:31.575963+0000","last_peered":"2023-10-09T11:09:31.575963+0000","last_clean":"2023-10-09T11:09:31.575963+0000","last_became_active":"2023-10-09T11:06:24.247348+0000","last_became_peered":"2023-10-09T11:06:24.247348+0000","last_unstale":"2023-10-09T11:09:31.575963+0000","last_undegraded":"2023-10-09T11:09:31.575963+0000","last_fullsized":"2023-10-09T11:09:31.575963+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:26.728754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":424,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100468+0000","last_change":"2023-10-09T11:06:24.253511+0000","last_active":"2023-10-09T11:09:32.100468+0000","last_peered":"2023-10-09T11:09:32.100468+0000","last_clean":"2023-10-09T11:09:32.100468+0000","last_became_active":"2023-10-09T11:06:24.252739+0000","last_became_peered":"2023-10-09T11:06:24.252739+0000","last_unstale":"2023-10-09T11:09:32.100468+0000","last_undegraded":"2023-10-09T11:09:32.100468+0000","last_fullsized":"2023-10-09T11:09:32.100468+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:50.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":829,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575249+0000","last_change":"2023-10-09T11:06:24.239676+0000","last_active":"2023-10-09T11:09:31.575249+0000","last_peered":"2023-10-09T11:09:31.575249+0000","last_clean":"2023-10-09T11:09:31.575249+0000","last_became_active":"2023-10-09T11:06:24.239375+0000","last_became_peered":"2023-10-09T11:06:24.239375+0000","last_unstale":"2023-10-09T11:09:31.575249+0000","last_undegraded":"2023-10-09T11:09:31.575249+0000","last_fullsized":"2023-10-09T11:09:31.575249+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:27.953053+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"591'57","reported_seq":257,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422283+0000","last_change":"2023-10-09T11:09:20.432685+0000","last_active":"2023-10-09T11:09:34.422283+0000","last_peered":"2023-10-09T11:09:34.422283+0000","last_clean":"2023-10-09T11:09:34.422283+0000","last_became_active":"2023-10-09T11:07:33.229061+0000","last_became_peered":"2023-10-09T11:07:33.229061+0000","last_unstale":"2023-10-09T11:09:34.422283+0000","last_undegraded":"2023-10-09T11:09:34.422283+0000","last_fullsized":"2023-10-09T11:09:34.422283+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":57,"log_dups_size":0,"ondisk_log_size":57,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:30.978293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043334800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":262,"num_read_kb":1253,"num_write":50,"num_write_kb":1077,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":280,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100454+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:09:32.100454+0000","last_peered":"2023-10-09T11:09:32.100454+0000","last_clean":"2023-10-09T11:09:32.100454+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:09:32.100454+0000","last_undegraded":"2023-10-09T11:09:32.100454+0000","last_fullsized":"2023-10-09T11:09:32.100454+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":556,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114975+0000","last_change":"2023-10-09T11:07:33.227271+0000","last_active":"2023-10-09T11:09:35.114975+0000","last_peered":"2023-10-09T11:09:35.114975+0000","last_clean":"2023-10-09T11:09:35.114975+0000","last_became_active":"2023-10-09T11:07:33.226760+0000","last_became_peered":"2023-10-09T11:07:33.226760+0000","last_unstale":"2023-10-09T11:09:35.114975+0000","last_undegraded":"2023-10-09T11:09:35.114975+0000","last_fullsized":"2023-10-09T11:09:35.114975+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:12.268123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":799,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.112784+0000","last_change":"2023-10-09T11:05:23.323368+0000","last_active":"2023-10-09T11:09:35.112784+0000","last_peered":"2023-10-09T11:09:35.112784+0000","last_clean":"2023-10-09T11:09:35.112784+0000","last_became_active":"2023-10-09T11:05:23.323032+0000","last_became_peered":"2023-10-09T11:05:23.323032+0000","last_unstale":"2023-10-09T11:09:35.112784+0000","last_undegraded":"2023-10-09T11:09:35.112784+0000","last_fullsized":"2023-10-09T11:09:35.112784+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:09.393584+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":155,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576669+0000","last_change":"2023-10-09T11:07:33.246378+0000","last_active":"2023-10-09T11:09:31.576669+0000","last_peered":"2023-10-09T11:09:31.576669+0000","last_clean":"2023-10-09T11:09:31.576669+0000","last_became_active":"2023-10-09T11:07:33.228019+0000","last_became_peered":"2023-10-09T11:07:33.228019+0000","last_unstale":"2023-10-09T11:09:31.576669+0000","last_undegraded":"2023-10-09T11:09:31.576669+0000","last_fullsized":"2023-10-09T11:09:31.576669+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:18.511517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"592'319","reported_seq":1588,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100818+0000","last_change":"2023-10-09T11:09:20.427659+0000","last_active":"2023-10-09T11:09:32.100818+0000","last_peered":"2023-10-09T11:09:32.100818+0000","last_clean":"2023-10-09T11:09:32.100818+0000","last_became_active":"2023-10-09T11:05:20.309827+0000","last_became_peered":"2023-10-09T11:05:20.309827+0000","last_unstale":"2023-10-09T11:09:32.100818+0000","last_undegraded":"2023-10-09T11:09:32.100818+0000","last_fullsized":"2023-10-09T11:09:32.100818+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":319,"log_dups_size":0,"ondisk_log_size":319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:23.934383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001122743,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2478,"num_read_kb":2521,"num_write":249,"num_write_kb":324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":143,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575553+0000","last_change":"2023-10-09T11:07:33.217595+0000","last_active":"2023-10-09T11:09:31.575553+0000","last_peered":"2023-10-09T11:09:31.575553+0000","last_clean":"2023-10-09T11:09:31.575553+0000","last_became_active":"2023-10-09T11:07:33.217155+0000","last_became_peered":"2023-10-09T11:07:33.217155+0000","last_unstale":"2023-10-09T11:09:31.575553+0000","last_undegraded":"2023-10-09T11:09:31.575553+0000","last_fullsized":"2023-10-09T11:09:31.575553+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:55.127329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":762,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575658+0000","last_change":"2023-10-09T11:07:36.284491+0000","last_active":"2023-10-09T11:09:31.575658+0000","last_peered":"2023-10-09T11:09:31.575658+0000","last_clean":"2023-10-09T11:09:31.575658+0000","last_became_active":"2023-10-09T11:07:36.284249+0000","last_became_peered":"2023-10-09T11:07:36.284249+0000","last_unstale":"2023-10-09T11:09:31.575658+0000","last_undegraded":"2023-10-09T11:09:31.575658+0000","last_fullsized":"2023-10-09T11:09:31.575658+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:03.653371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:03.653371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:09.629417+0000","scrub_duration":0.001576771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"592'82","reported_seq":264,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422176+0000","last_change":"2023-10-09T11:09:20.433178+0000","last_active":"2023-10-09T11:09:34.422176+0000","last_peered":"2023-10-09T11:09:34.422176+0000","last_clean":"2023-10-09T11:09:34.422176+0000","last_became_active":"2023-10-09T11:07:33.241013+0000","last_became_peered":"2023-10-09T11:07:33.241013+0000","last_unstale":"2023-10-09T11:09:34.422176+0000","last_undegraded":"2023-10-09T11:09:34.422176+0000","last_fullsized":"2023-10-09T11:09:34.422176+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":82,"log_dups_size":0,"ondisk_log_size":82,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:57.164502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075188800000000004,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":544,"num_write":82,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":297,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100651+0000","last_change":"2023-10-09T11:06:24.253714+0000","last_active":"2023-10-09T11:09:32.100651+0000","last_peered":"2023-10-09T11:09:32.100651+0000","last_clean":"2023-10-09T11:09:32.100651+0000","last_became_active":"2023-10-09T11:06:24.253143+0000","last_became_peered":"2023-10-09T11:06:24.253143+0000","last_unstale":"2023-10-09T11:09:32.100651+0000","last_undegraded":"2023-10-09T11:09:32.100651+0000","last_fullsized":"2023-10-09T11:09:32.100651+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:05.682288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":402,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114775+0000","last_change":"2023-10-09T11:05:20.313560+0000","last_active":"2023-10-09T11:09:35.114775+0000","last_peered":"2023-10-09T11:09:35.114775+0000","last_clean":"2023-10-09T11:09:35.114775+0000","last_became_active":"2023-10-09T11:05:20.313214+0000","last_became_peered":"2023-10-09T11:05:20.313214+0000","last_unstale":"2023-10-09T11:09:35.114775+0000","last_undegraded":"2023-10-09T11:09:35.114775+0000","last_fullsized":"2023-10-09T11:09:35.114775+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:32.527205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":1005,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576235+0000","last_change":"2023-10-09T11:05:23.329886+0000","last_active":"2023-10-09T11:09:31.576235+0000","last_peered":"2023-10-09T11:09:31.576235+0000","last_clean":"2023-10-09T11:09:31.576235+0000","last_became_active":"2023-10-09T11:05:23.329561+0000","last_became_peered":"2023-10-09T11:05:23.329561+0000","last_unstale":"2023-10-09T11:09:31.576235+0000","last_undegraded":"2023-10-09T11:09:31.576235+0000","last_fullsized":"2023-10-09T11:09:31.576235+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:46.933799+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":59,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422089+0000","last_change":"2023-10-09T11:07:33.229864+0000","last_active":"2023-10-09T11:09:34.422089+0000","last_peered":"2023-10-09T11:09:34.422089+0000","last_clean":"2023-10-09T11:09:34.422089+0000","last_became_active":"2023-10-09T11:07:33.229547+0000","last_became_peered":"2023-10-09T11:07:33.229547+0000","last_unstale":"2023-10-09T11:09:34.422089+0000","last_undegraded":"2023-10-09T11:09:34.422089+0000","last_fullsized":"2023-10-09T11:09:34.422089+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:34.990439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":382,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.112943+0000","last_change":"2023-10-09T11:04:50.510688+0000","last_active":"2023-10-09T11:09:35.112943+0000","last_peered":"2023-10-09T11:09:35.112943+0000","last_clean":"2023-10-09T11:09:35.112943+0000","last_became_active":"2023-10-09T11:04:50.507404+0000","last_became_peered":"2023-10-09T11:04:50.507404+0000","last_unstale":"2023-10-09T11:09:35.112943+0000","last_undegraded":"2023-10-09T11:09:35.112943+0000","last_fullsized":"2023-10-09T11:09:35.112943+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:13.663394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":963,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575897+0000","last_change":"2023-10-09T11:05:23.321040+0000","last_active":"2023-10-09T11:09:31.575897+0000","last_peered":"2023-10-09T11:09:31.575897+0000","last_clean":"2023-10-09T11:09:31.575897+0000","last_became_active":"2023-10-09T11:05:23.320674+0000","last_became_peered":"2023-10-09T11:05:23.320674+0000","last_unstale":"2023-10-09T11:09:31.575897+0000","last_undegraded":"2023-10-09T11:09:31.575897+0000","last_fullsized":"2023-10-09T11:09:31.575897+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.527660+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":424,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115106+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:09:35.115106+0000","last_peered":"2023-10-09T11:09:35.115106+0000","last_clean":"2023-10-09T11:09:35.115106+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:09:35.115106+0000","last_undegraded":"2023-10-09T11:09:35.115106+0000","last_fullsized":"2023-10-09T11:09:35.115106+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":124,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575451+0000","last_change":"2023-10-09T11:04:50.511984+0000","last_active":"2023-10-09T11:09:31.575451+0000","last_peered":"2023-10-09T11:09:31.575451+0000","last_clean":"2023-10-09T11:09:31.575451+0000","last_became_active":"2023-10-09T11:04:50.510820+0000","last_became_peered":"2023-10-09T11:04:50.510820+0000","last_unstale":"2023-10-09T11:09:31.575451+0000","last_undegraded":"2023-10-09T11:09:31.575451+0000","last_fullsized":"2023-10-09T11:09:31.575451+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:38.662189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":575,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115201+0000","last_change":"2023-10-09T11:05:20.327032+0000","last_active":"2023-10-09T11:09:35.115201+0000","last_peered":"2023-10-09T11:09:35.115201+0000","last_clean":"2023-10-09T11:09:35.115201+0000","last_became_active":"2023-10-09T11:05:20.320193+0000","last_became_peered":"2023-10-09T11:05:20.320193+0000","last_unstale":"2023-10-09T11:09:35.115201+0000","last_undegraded":"2023-10-09T11:09:35.115201+0000","last_fullsized":"2023-10-09T11:09:35.115201+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:44.862890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":124,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575160+0000","last_change":"2023-10-09T11:04:50.384219+0000","last_active":"2023-10-09T11:09:31.575160+0000","last_peered":"2023-10-09T11:09:31.575160+0000","last_clean":"2023-10-09T11:09:31.575160+0000","last_became_active":"2023-10-09T11:04:50.384028+0000","last_became_peered":"2023-10-09T11:04:50.384028+0000","last_unstale":"2023-10-09T11:09:31.575160+0000","last_undegraded":"2023-10-09T11:09:31.575160+0000","last_fullsized":"2023-10-09T11:09:31.575160+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:04.721869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":782,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421841+0000","last_change":"2023-10-09T11:08:39.540029+0000","last_active":"2023-10-09T11:09:34.421841+0000","last_peered":"2023-10-09T11:09:34.421841+0000","last_clean":"2023-10-09T11:09:34.421841+0000","last_became_active":"2023-10-09T11:07:36.291077+0000","last_became_peered":"2023-10-09T11:07:36.291077+0000","last_unstale":"2023-10-09T11:09:34.421841+0000","last_undegraded":"2023-10-09T11:09:34.421841+0000","last_fullsized":"2023-10-09T11:09:34.421841+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.539966+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.539966+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:20.767812+0000","scrub_duration":0.0016245929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":134,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.575951+0000","last_change":"2023-10-09T11:05:20.299321+0000","last_active":"2023-10-09T11:09:31.575951+0000","last_peered":"2023-10-09T11:09:31.575951+0000","last_clean":"2023-10-09T11:09:31.575951+0000","last_became_active":"2023-10-09T11:05:20.299019+0000","last_became_peered":"2023-10-09T11:05:20.299019+0000","last_unstale":"2023-10-09T11:09:31.575951+0000","last_undegraded":"2023-10-09T11:09:31.575951+0000","last_fullsized":"2023-10-09T11:09:31.575951+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:08.645909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":124,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:31.576310+0000","last_change":"2023-10-09T11:04:50.366890+0000","last_active":"2023-10-09T11:09:31.576310+0000","last_peered":"2023-10-09T11:09:31.576310+0000","last_clean":"2023-10-09T11:09:31.576310+0000","last_became_active":"2023-10-09T11:04:50.366558+0000","last_became_peered":"2023-10-09T11:04:50.366558+0000","last_unstale":"2023-10-09T11:09:31.576310+0000","last_undegraded":"2023-10-09T11:09:31.576310+0000","last_fullsized":"2023-10-09T11:09:31.576310+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:50.993153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":897,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.100718+0000","last_change":"2023-10-09T11:04:53.274332+0000","last_active":"2023-10-09T11:09:32.100718+0000","last_peered":"2023-10-09T11:09:32.100718+0000","last_clean":"2023-10-09T11:09:32.100718+0000","last_became_active":"2023-10-09T11:04:53.273798+0000","last_became_peered":"2023-10-09T11:04:53.273798+0000","last_unstale":"2023-10-09T11:09:32.100718+0000","last_undegraded":"2023-10-09T11:09:32.100718+0000","last_fullsized":"2023-10-09T11:09:32.100718+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:04.220970+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":144,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115365+0000","last_change":"2023-10-09T11:06:24.249877+0000","last_active":"2023-10-09T11:09:35.115365+0000","last_peered":"2023-10-09T11:09:35.115365+0000","last_clean":"2023-10-09T11:09:35.115365+0000","last_became_active":"2023-10-09T11:06:24.249573+0000","last_became_peered":"2023-10-09T11:06:24.249573+0000","last_unstale":"2023-10-09T11:09:35.115365+0000","last_undegraded":"2023-10-09T11:09:35.115365+0000","last_fullsized":"2023-10-09T11:09:35.115365+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:11.798192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":646,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T11:09:32.099883+0000","last_change":"2023-10-09T11:05:20.307347+0000","last_active":"2023-10-09T11:09:32.099883+0000","last_peered":"2023-10-09T11:09:32.099883+0000","last_clean":"2023-10-09T11:09:32.099883+0000","last_became_active":"2023-10-09T11:05:20.307126+0000","last_became_peered":"2023-10-09T11:05:20.307126+0000","last_unstale":"2023-10-09T11:09:32.099883+0000","last_undegraded":"2023-10-09T11:09:32.099883+0000","last_fullsized":"2023-10-09T11:09:32.099883+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:34.408515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":91,"num_bytes_recovered":15742,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12694,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13244,"num_write":846,"num_write_kb":12712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332517,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":927832,"kb_used_data":28432,"kb_used_omap":0,"kb_used_meta":899392,"kb_avail":92817320,"statfs":{"total":95995035648,"available":95044935680,"internally_reserved":0,"allocated":29114368,"data_stored":27263642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920977408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:21 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.04,"15min":1.6990000000000001},"min":{"1min":0.27100000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.2949999999999999,"5min":9.8219999999999992,"15min":9.8219999999999992},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.389,"15min":1.694},"min":{"1min":0.33200000000000002,"5min":0.24299999999999999,"15min":0.216},"max":{"1min":3.1640000000000001,"5min":15.853,"15min":15.853},"last":0.61199999999999999}]},{"osd":1,"last update":"Mon Oct 9 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":1.194,"15min":1.47},"min":{"1min":0.315,"5min":0.222,"15min":0.222},"max":{"1min":0.83499999999999996,"5min":11.619999999999999,"15min":11.619999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":1.8,"15min":1.843},"min":{"1min":0.31900000000000001,"5min":0.25,"15min":0.25},"max":{"1min":0.91000000000000003,"5min":13.096,"15min":13.096},"last":0.51400000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:08:54 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":1.573,"15min":1.77},"min":{"1min":0.309,"5min":0.309,"15min":0.245},"max":{"1min":5.1440000000000001,"5min":12.226000000000001,"15min":16.605},"last":0.499},{"interface":"front","average":{"1min":2.2770000000000001,"5min":1.6220000000000001,"15min":1.802},"min":{"1min":0.379,"5min":0.223,"15min":0.223},"max":{"1min":16.157,"5min":16.157,"15min":16.157},"last":0.65800000000000003}]},{"osd":3,"last update":"Mon Oct 9 11:09:21 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":1.554,"15min":1.45},"min":{"1min":0.35499999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.194,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.68899999999999995,"5min":1.665,"15min":1.7490000000000001},"min":{"1min":0.39300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.8879999999999999,"5min":8.2330000000000005,"15min":8.2330000000000005},"last":0.54200000000000004}]},{"osd":4,"last update":"Mon Oct 9 11:09:09 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.85799999999999998,"15min":1.0900000000000001},"min":{"1min":0.248,"5min":0.218,"15min":0.218},"max":{"1min":4.7489999999999997,"5min":7.4630000000000001,"15min":8.2940000000000005},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.91000000000000003,"5min":0.92800000000000005,"15min":1.26},"min":{"1min":0.35399999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.7619999999999996,"5min":9.2759999999999998,"15min":16.481000000000002},"last":0.44800000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:09:09 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.85199999999999998,"15min":1.2549999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.247},"max":{"1min":4.9470000000000001,"5min":6.9379999999999997,"15min":15.414999999999999},"last":0.623},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.095,"15min":1.228},"min":{"1min":0.34300000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.9980000000000002,"5min":13.045,"15min":15.369},"last":0.70799999999999996}]},{"osd":6,"last update":"Mon Oct 9 11:09:09 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.98799999999999999,"15min":1.242},"min":{"1min":0.39800000000000002,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":4.8079999999999998,"5min":5.1120000000000001,"15min":14.159000000000001},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.0049999999999999,"5min":0.96599999999999997,"15min":1.369},"min":{"1min":0.39000000000000001,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":3.1179999999999999,"5min":4.3659999999999997,"15min":16.538},"last":0.59699999999999998}]}]},{"osd":6,"up_from":44,"seq":188978561448,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1511564,"kb_used_data":10820,"kb_used_omap":0,"kb_used_meta":1500736,"kb_avail":92233588,"statfs":{"total":95995035648,"available":94447194112,"internally_reserved":0,"allocated":11079680,"data_stored":9240571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1536753664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":1.0900000000000001,"15min":1.9179999999999999},"min":{"1min":0.26900000000000002,"5min":0.223,"15min":0.217},"max":{"1min":0.72799999999999998,"5min":12.209,"15min":54.743000000000002},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.94099999999999995,"15min":1.8580000000000001},"min":{"1min":0.33700000000000002,"5min":0.247,"15min":0.214},"max":{"1min":0.753,"5min":8.4789999999999992,"15min":53.701000000000001},"last":0.66600000000000004}]},{"osd":1,"last update":"Mon Oct 9 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":1.2390000000000001,"15min":1.1739999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.23499999999999999},"max":{"1min":0.754,"5min":16.225999999999999,"15min":41.241999999999997},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.45900000000000002,"5min":1.7749999999999999,"15min":1.766},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.248},"max":{"1min":0.62,"5min":11.715,"15min":41.476999999999997},"last":0.56399999999999995}]},{"osd":2,"last update":"Mon Oct 9 11:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.867,"15min":1.9019999999999999},"min":{"1min":0.32200000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.2069999999999999,"5min":10.382999999999999,"15min":12.003},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.85599999999999998,"5min":1.2909999999999999,"15min":1.877},"min":{"1min":0.33800000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.6669999999999998,"5min":12.693,"15min":15.432},"last":0.68999999999999995}]},{"osd":3,"last update":"Mon Oct 9 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":2.0310000000000001,"15min":1.643},"min":{"1min":0.372,"5min":0.313,"15min":0.313},"max":{"1min":0.96999999999999997,"5min":28.948,"15min":28.948},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.51400000000000001,"5min":2.597,"15min":2.4910000000000001},"min":{"1min":0.36199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.65100000000000002,"5min":28.893000000000001,"15min":28.893000000000001},"last":0.63700000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.66900000000000004,"15min":0.998},"min":{"1min":0.28100000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.67100000000000004,"5min":6.6609999999999996,"15min":17.216999999999999},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.53700000000000003,"15min":0.83299999999999996},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.24399999999999999},"max":{"1min":0.80100000000000005,"5min":2.3610000000000002,"15min":10.025},"last":0.78400000000000003}]},{"osd":5,"last update":"Mon Oct 9 11:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.78300000000000003,"15min":1.228},"min":{"1min":0.34000000000000002,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":2.7000000000000002,"5min":3.7389999999999999,"15min":19.786999999999999},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.68999999999999995,"15min":0.96499999999999997},"min":{"1min":0.377,"5min":0.248,"15min":0.248},"max":{"1min":1.22,"5min":4.085,"15min":9.9469999999999992},"last":0.81599999999999995}]},{"osd":7,"last update":"Mon Oct 9 11:09:16 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.71899999999999997,"15min":1.238},"min":{"1min":0.32400000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.73999999999999999,"5min":3.7250000000000001,"15min":12.882},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.68500000000000005,"15min":0.91700000000000004},"min":{"1min":0.35399999999999998,"5min":0.32800000000000001,"15min":0.27500000000000002},"max":{"1min":0.84399999999999997,"5min":3.2839999999999998,"15min":11.557},"last":0.74399999999999999}]}]},{"osd":1,"up_from":388,"seq":1666447310962,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":505304,"kb_used_data":7824,"kb_used_omap":13,"kb_used_meta":497458,"kb_avail":93239848,"statfs":{"total":95995035648,"available":95477604352,"internally_reserved":0,"allocated":8011776,"data_stored":4888001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":509397703},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":2.347,"15min":2.3250000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.17699999999999999},"max":{"1min":1.9590000000000001,"5min":18.192,"15min":18.192},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.434,"5min":2.6749999999999998,"15min":2.0329999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.19900000000000001},"max":{"1min":1.2470000000000001,"5min":17.957999999999998,"15min":17.957999999999998},"last":0.317}]},{"osd":2,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":2.0939999999999999,"15min":2.73},"min":{"1min":0.29399999999999998,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":2.032,"5min":12.436999999999999,"15min":47.731000000000002},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.44400000000000001,"5min":2.5019999999999998,"15min":2.4689999999999999},"min":{"1min":0.28399999999999997,"5min":0.216,"15min":0.216},"max":{"1min":0.81200000000000006,"5min":16.870000000000001,"15min":47.786999999999999},"last":0.38200000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":3.141,"15min":2.0390000000000001},"min":{"1min":0.254,"5min":0.251,"15min":0.251},"max":{"1min":0.86299999999999999,"5min":92.447999999999993,"15min":92.447999999999993},"last":0.497},{"interface":"front","average":{"1min":0.52800000000000002,"5min":3.2440000000000002,"15min":2.0699999999999998},"min":{"1min":0.252,"5min":0.217,"15min":0.217},"max":{"1min":2.0009999999999999,"5min":92.066999999999993,"15min":92.066999999999993},"last":0.30299999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.6020000000000001,"15min":2.1880000000000002},"min":{"1min":0.34699999999999998,"5min":0.27300000000000002,"15min":0.26000000000000001},"max":{"1min":2.0739999999999998,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.905,"15min":1.673},"min":{"1min":0.33300000000000002,"5min":0.314,"15min":0.29999999999999999},"max":{"1min":0.88,"5min":16.841000000000001,"15min":16.841000000000001},"last":0.46100000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.6879999999999999,"15min":2.1549999999999998},"min":{"1min":0.40400000000000003,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":0.86899999999999999,"5min":20.689,"15min":20.689},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.66400000000000003,"5min":1.7549999999999999,"15min":2.1240000000000001},"min":{"1min":0.39900000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.133,"5min":11.65,"15min":11.65},"last":0.61899999999999999}]},{"osd":6,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.7749999999999999,"15min":2.2480000000000002},"min":{"1min":0.44,"5min":0.312,"15min":0.309},"max":{"1min":0.89900000000000002,"5min":20.646999999999998,"15min":20.646999999999998},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.766,"15min":2.052},"min":{"1min":0.375,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.1040000000000001,"5min":11.75,"15min":11.75},"last":0.57799999999999996}]},{"osd":7,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.9330000000000001,"15min":1.6830000000000001},"min":{"1min":0.36199999999999999,"5min":0.33000000000000002,"15min":0.32900000000000001},"max":{"1min":0.78300000000000003,"5min":16.797999999999998,"15min":16.797999999999998},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.766,"15min":2.1589999999999998},"min":{"1min":0.42699999999999999,"5min":0.36899999999999999,"15min":0.33200000000000002},"max":{"1min":0.91900000000000004,"5min":20.716000000000001,"15min":20.716000000000001},"last":0.54200000000000004}]}]},{"osd":0,"up_from":80,"seq":343597384020,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":986400,"kb_used_data":8216,"kb_used_omap":28,"kb_used_meta":978147,"kb_avail":92758752,"statfs":{"total":95995035648,"available":94984962048,"internally_reserved":0,"allocated":8413184,"data_stored":5211967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1001623376},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":1.8700000000000001,"15min":2.5489999999999999},"min":{"1min":0.23300000000000001,"5min":0.186,"15min":0.186},"max":{"1min":1.6220000000000001,"5min":14.644,"15min":19.608000000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.96,"15min":2.613},"min":{"1min":0.23400000000000001,"5min":0.214,"15min":0.214},"max":{"1min":3.7240000000000002,"5min":15.834,"15min":19.992999999999999},"last":0.29499999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:08:57 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":2.0579999999999998,"15min":2.2959999999999998},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":7.9329999999999998,"5min":12.459,"15min":24.852},"last":0.27700000000000002},{"interface":"front","average":{"1min":6.4219999999999997,"5min":3.0819999999999999,"15min":3.0009999999999999},"min":{"1min":0.21299999999999999,"5min":0.20399999999999999,"15min":0.19800000000000001},"max":{"1min":99.679000000000002,"5min":99.679000000000002,"15min":99.679000000000002},"last":0.373}]},{"osd":3,"last update":"Mon Oct 9 11:09:25 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":2.52,"15min":3.194},"min":{"1min":0.28499999999999998,"5min":0.22,"15min":0.22},"max":{"1min":8.0180000000000007,"5min":25.050000000000001,"15min":25.050000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.51100000000000001,"5min":2.077,"15min":1.958},"min":{"1min":0.314,"5min":0.254,"15min":0.254},"max":{"1min":2.0419999999999998,"5min":14.125999999999999,"15min":14.125999999999999},"last":0.35899999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:09:25 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.905,"15min":2.4540000000000002},"min":{"1min":0.312,"5min":0.27100000000000002,"15min":0.25700000000000001},"max":{"1min":7.9829999999999997,"5min":14.507,"15min":77.177999999999997},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.3660000000000001,"15min":1.6120000000000001},"min":{"1min":0.34100000000000003,"5min":0.33600000000000002,"15min":0.29199999999999998},"max":{"1min":2.1160000000000001,"5min":7.9249999999999998,"15min":10.513999999999999},"last":0.50600000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:09:25 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.361,"15min":1.7909999999999999},"min":{"1min":0.36699999999999999,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":2.1440000000000001,"5min":7.9550000000000001,"15min":10.544},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.629,"5min":1.516,"15min":1.7829999999999999},"min":{"1min":0.439,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":1.7010000000000001,"5min":11.212999999999999,"15min":24.754999999999999},"last":0.46200000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:09:25 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":1.4450000000000001,"15min":1.8740000000000001},"min":{"1min":0.38600000000000001,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":2.1859999999999999,"5min":7.9829999999999997,"15min":18.920000000000002},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.63,"5min":1.542,"15min":1.899},"min":{"1min":0.41699999999999998,"5min":0.378,"15min":0.31900000000000001},"max":{"1min":1.728,"5min":11.257999999999999,"15min":24.800000000000001},"last":0.55500000000000005}]},{"osd":7,"last update":"Mon Oct 9 11:09:25 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":1.9219999999999999,"15min":2.0049999999999999},"min":{"1min":0.39300000000000002,"5min":0.35099999999999998,"15min":0.32700000000000001},"max":{"1min":1.6679999999999999,"5min":17.672000000000001,"15min":17.672000000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":2.0779999999999998,"15min":2.573},"min":{"1min":0.41299999999999998,"5min":0.311,"15min":0.311},"max":{"1min":7.9390000000000001,"5min":17.539000000000001,"15min":77.228999999999999},"last":0.59099999999999997}]}]},{"osd":2,"up_from":90,"seq":386547056961,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":769904,"kb_used_data":12072,"kb_used_omap":81,"kb_used_meta":757742,"kb_avail":92975248,"statfs":{"total":95995035648,"available":95206653952,"internally_reserved":0,"allocated":12361728,"data_stored":9333126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":775928026},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.363,"5min":1.9750000000000001,"15min":2.052},"min":{"1min":0.191,"5min":0.182,"15min":0.17499999999999999},"max":{"1min":8.3710000000000004,"5min":15.609,"15min":48.933999999999997},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.998,"15min":1.851},"min":{"1min":0.24199999999999999,"5min":0.23100000000000001,"15min":0.19600000000000001},"max":{"1min":7.7859999999999996,"5min":9.8010000000000002,"15min":14.515000000000001},"last":0.32800000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:09:36 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.897,"15min":2.3940000000000001},"min":{"1min":0.26900000000000002,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":1.1060000000000001,"5min":24.902000000000001,"15min":49.033999999999999},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.437,"5min":2.1600000000000001,"15min":2.5720000000000001},"min":{"1min":0.255,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":1.2210000000000001,"5min":25.850999999999999,"15min":48.874000000000002},"last":0.44800000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":2.8719999999999999,"15min":2.1339999999999999},"min":{"1min":0.27900000000000003,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":14.801,"5min":25.890999999999998,"15min":25.890999999999998},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.2609999999999999,"5min":2.7189999999999999,"15min":2.3359999999999999},"min":{"1min":0.29399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":11.724,"5min":28.689,"15min":28.689},"last":0.47499999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:09:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.575,"15min":1.8720000000000001},"min":{"1min":0.34699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.6949999999999998,"5min":11.391999999999999,"15min":15.672000000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.6200000000000001,"15min":1.8959999999999999},"min":{"1min":0.33900000000000002,"5min":0.32900000000000001,"15min":0.249},"max":{"1min":8.1609999999999996,"5min":15.571,"15min":48.795000000000002},"last":0.67800000000000005}]},{"osd":5,"last update":"Mon Oct 9 11:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.6659999999999999,"15min":1.5900000000000001},"min":{"1min":0.35699999999999998,"5min":0.32100000000000001,"15min":0.25900000000000001},"max":{"1min":7.7279999999999998,"5min":8.5310000000000006,"15min":12.128},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.84399999999999997,"5min":1.5509999999999999,"15min":2.0030000000000001},"min":{"1min":0.39900000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.7189999999999999,"5min":11.441000000000001,"15min":49.061},"last":0.70899999999999996}]},{"osd":6,"last update":"Mon Oct 9 11:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5940000000000001,"5min":1.661,"15min":1.841},"min":{"1min":0.33000000000000002,"5min":0.313,"15min":0.313},"max":{"1min":8.1289999999999996,"5min":15.536,"15min":48.845999999999997},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.383,"5min":1.897,"15min":1.6759999999999999},"min":{"1min":0.38,"5min":0.36799999999999999,"15min":0.29199999999999998},"max":{"1min":7.8129999999999997,"5min":10.157999999999999,"15min":10.157999999999999},"last":0.69499999999999995}]},{"osd":7,"last update":"Mon Oct 9 11:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":1.8660000000000001,"15min":2.0009999999999999},"min":{"1min":0.39700000000000002,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":8.343,"5min":15.59,"15min":48.966000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.3220000000000001,"5min":1.8720000000000001,"15min":1.7849999999999999},"min":{"1min":0.36099999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.8390000000000004,"5min":8.5549999999999997,"15min":14.94},"last":0.73999999999999999}]}]},{"osd":3,"up_from":469,"seq":2014339661890,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":343440,"kb_used_data":8392,"kb_used_omap":11,"kb_used_meta":335028,"kb_avail":93401712,"statfs":{"total":95995035648,"available":95643353088,"internally_reserved":0,"allocated":8593408,"data_stored":5233360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11309,"internal_metadata":343069651},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":1.915,"15min":1.7849999999999999},"min":{"1min":0.22900000000000001,"5min":0.183,"15min":0.183},"max":{"1min":1.0740000000000001,"5min":14.882,"15min":14.882},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.337,"15min":1.024},"min":{"1min":0.19900000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":2.5939999999999999,"5min":9.4030000000000005,"15min":9.4030000000000005},"last":0.246}]},{"osd":1,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":2.0710000000000002,"15min":1.625},"min":{"1min":0.22,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":4.1799999999999997,"5min":17.93,"15min":17.93},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.4590000000000001,"5min":2.2210000000000001,"15min":1.7969999999999999},"min":{"1min":0.28100000000000003,"5min":0.224,"15min":0.224},"max":{"1min":13.236000000000001,"5min":19.163,"15min":19.163},"last":0.38100000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":2.0310000000000001,"15min":1.401},"min":{"1min":0.27400000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":4.0270000000000001,"5min":38.840000000000003,"15min":38.840000000000003},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.69999999999999996,"5min":2.1360000000000001,"15min":1.4319999999999999},"min":{"1min":0.27700000000000002,"5min":0.219,"15min":0.219},"max":{"1min":4.016,"5min":37.991999999999997,"15min":37.991999999999997},"last":0.35999999999999999}]},{"osd":4,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.46,"15min":1.5349999999999999},"min":{"1min":0.34000000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.131,"5min":14.907999999999999,"15min":14.907999999999999},"last":0.44},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.123,"15min":0.83799999999999997},"min":{"1min":0.307,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.0739999999999998,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.47999999999999998}]},{"osd":5,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.2729999999999999,"15min":1.3660000000000001},"min":{"1min":0.41699999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.4450000000000001,"5min":9.3300000000000001,"15min":9.3300000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.3999999999999999,"15min":1.427},"min":{"1min":0.32000000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.101,"5min":10.029,"15min":10.029},"last":0.54600000000000004}]},{"osd":6,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":1.4410000000000001,"15min":1.669},"min":{"1min":0.44900000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.2120000000000002,"5min":15.455,"15min":15.455},"last":0.501},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.516,"15min":1.6000000000000001},"min":{"1min":0.39900000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.865,"5min":14.837,"15min":14.837},"last":0.51200000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":1.3240000000000001,"15min":1.1080000000000001},"min":{"1min":0.39500000000000002,"5min":0.315,"15min":0.315},"max":{"1min":3.8029999999999999,"5min":6.907,"15min":6.907},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.86299999999999999,"5min":1.536,"15min":1.2829999999999999},"min":{"1min":0.42299999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.8370000000000002,"5min":6.9119999999999999,"15min":6.9119999999999999},"last":0.47199999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019311,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903252,"kb_used_data":6412,"kb_used_omap":0,"kb_used_meta":896832,"kb_avail":92841900,"statfs":{"total":95995035648,"available":95070105600,"internally_reserved":0,"allocated":6565888,"data_stored":4718967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918355968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:19 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":1.494,"15min":2.4129999999999998},"min":{"1min":0.26900000000000002,"5min":0.223,"15min":0.20300000000000001},"max":{"1min":3.1259999999999999,"5min":13.162000000000001,"15min":213.88999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.329,"15min":2.5},"min":{"1min":0.30099999999999999,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":2.399,"5min":9.0670000000000002,"15min":214.09800000000001},"last":0.58499999999999996}]},{"osd":1,"last update":"Mon Oct 9 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":2.331,"15min":2.5350000000000001},"min":{"1min":0.28299999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.68200000000000005,"5min":57.966999999999999,"15min":121.41},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.65800000000000003,"5min":1.9810000000000001,"15min":2.081},"min":{"1min":0.28199999999999997,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":3.589,"5min":57.832000000000001,"15min":122.21899999999999},"last":0.55300000000000005}]},{"osd":2,"last update":"Mon Oct 9 11:09:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.748,"15min":1.738},"min":{"1min":0.30499999999999999,"5min":0.246,"15min":0.23799999999999999},"max":{"1min":3.0779999999999998,"5min":18.407,"15min":18.407},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.7430000000000001,"15min":2.0739999999999998},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":2.5129999999999999,"5min":8.4250000000000007,"15min":27.818000000000001},"last":0.48799999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":2.2629999999999999,"15min":1.921},"min":{"1min":0.35599999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.1189999999999998,"5min":60.753,"15min":60.753},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.0329999999999999,"5min":2.1789999999999998,"15min":1.5049999999999999},"min":{"1min":0.39800000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.5199999999999996,"5min":61.473999999999997,"15min":61.473999999999997},"last":0.52300000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.89600000000000002,"15min":1.337},"min":{"1min":0.30299999999999999,"5min":0.27400000000000002,"15min":0.224},"max":{"1min":0.68200000000000005,"5min":4.9409999999999998,"15min":19.068999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.499,"5min":0.83799999999999997,"15min":1.22},"min":{"1min":0.29899999999999999,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":1.052,"5min":8.3230000000000004,"15min":15.535},"last":0.67600000000000005}]},{"osd":6,"last update":"Mon Oct 9 11:09:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.2969999999999999,"15min":1.1910000000000001},"min":{"1min":0.29299999999999998,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":6.8689999999999998,"5min":8.3930000000000007,"15min":15.739000000000001},"last":0.62},{"interface":"front","average":{"1min":1.3500000000000001,"5min":1.4179999999999999,"15min":1.4730000000000001},"min":{"1min":0.33700000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.8170000000000002,"5min":12.795999999999999,"15min":14.093999999999999},"last":0.65500000000000003}]},{"osd":7,"last update":"Mon Oct 9 11:09:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.294,"15min":1.5629999999999999},"min":{"1min":0.374,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.181,"5min":10.15,"15min":19.923999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.155,"5min":1.091,"15min":1.488},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.0439999999999996,"5min":10.853,"15min":19.859999999999999},"last":0.70799999999999996}]}]},{"osd":5,"up_from":37,"seq":158913790380,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1205960,"kb_used_data":6656,"kb_used_omap":0,"kb_used_meta":1199296,"kb_avail":92539192,"statfs":{"total":95995035648,"available":94760132608,"internally_reserved":0,"allocated":6815744,"data_stored":4977095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1228079104},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":1.321,"15min":2.2389999999999999},"min":{"1min":0.23000000000000001,"5min":0.219,"15min":0.21299999999999999},"max":{"1min":0.77900000000000003,"5min":6.2249999999999996,"15min":58.271999999999998},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":1.3740000000000001,"15min":2.2890000000000001},"min":{"1min":0.38200000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.752,"5min":6.2649999999999997,"15min":58.25},"last":0.50600000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:09:32 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":1.5569999999999999,"15min":1.964},"min":{"1min":0.26100000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.64800000000000002,"5min":16.962,"15min":32.491999999999997},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":1.675,"15min":1.845},"min":{"1min":0.34200000000000003,"5min":0.30499999999999999,"15min":0.247},"max":{"1min":0.80800000000000005,"5min":17.033999999999999,"15min":32.537999999999997},"last":0.69799999999999995}]},{"osd":2,"last update":"Mon Oct 9 11:08:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.321,"15min":1.8240000000000001},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.221},"max":{"1min":7.3109999999999999,"5min":7.3109999999999999,"15min":11.257999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.8919999999999999,"15min":1.988},"min":{"1min":0.317,"5min":0.253,"15min":0.253},"max":{"1min":4.085,"5min":11.673,"15min":14.621},"last":0.58399999999999996}]},{"osd":3,"last update":"Mon Oct 9 11:09:19 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":1.607,"15min":1.5289999999999999},"min":{"1min":0.35899999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.7880000000000003,"5min":15.535,"15min":15.535},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.9059999999999999,"15min":1.841},"min":{"1min":0.40699999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.8090000000000002,"5min":15.757999999999999,"15min":15.757999999999999},"last":0.47299999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:09:32 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.80900000000000005,"15min":0.90400000000000003},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.216},"max":{"1min":0.65900000000000003,"5min":7.5640000000000001,"15min":12.042},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.92800000000000005,"15min":1.151},"min":{"1min":0.32000000000000001,"5min":0.245,"15min":0.245},"max":{"1min":1.0840000000000001,"5min":6.8639999999999999,"15min":9.2490000000000006},"last":0.55600000000000005}]},{"osd":6,"last update":"Mon Oct 9 11:08:45 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.2090000000000001,"15min":1.2450000000000001},"min":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.27100000000000002},"max":{"1min":5.944,"5min":7.1020000000000003,"15min":9.1969999999999992},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.214,"5min":1.204,"15min":1.0700000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29099999999999998},"max":{"1min":9.298,"5min":11.983000000000001,"15min":11.983000000000001},"last":0.81599999999999995}]},{"osd":7,"last update":"Mon Oct 9 11:09:09 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.026,"15min":1.298},"min":{"1min":0.35399999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.9699999999999998,"5min":5.9699999999999998,"15min":9.3539999999999992},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.94199999999999995,"15min":1.1739999999999999},"min":{"1min":0.38700000000000001,"5min":0.30499999999999999,"15min":0.30099999999999999},"max":{"1min":2.2549999999999999,"5min":8.0009999999999994,"15min":11.726000000000001},"last":0.746}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3733,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:09:43.461 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T11:09:43.461 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T11:09:43.461 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T11:09:43.461 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:09:43.734 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:43.771 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T11:09:44.350 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:44.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:44 smithi125 ceph-mon[195507]: from='client.37644 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:09:44.703 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:44 smithi125 ceph-mon[200197]: from='client.37644 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:09:44.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:44 smithi155 ceph-mon[184854]: from='client.37644 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:09:45.171 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:09:45.173 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:09:45.645 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1455,"stamp":"2023-10-09T11:09:43.943753+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5995,"num_read_kb":13244,"num_write":907,"num_write_kb":12741,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":2377214,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":310,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7153692,"kb_used_data":88860,"kb_used_omap":134,"kb_used_meta":7064633,"kb_avail":742807524,"statfs":{"total":767960285184,"available":760634904576,"internally_reserved":0,"allocated":90992640,"data_stored":70891125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":137532,"internal_metadata":7234184900},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.526086"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":884,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114685+0000","last_change":"2023-10-09T11:07:36.287455+0000","last_active":"2023-10-09T11:09:35.114685+0000","last_peered":"2023-10-09T11:09:35.114685+0000","last_clean":"2023-10-09T11:09:35.114685+0000","last_became_active":"2023-10-09T11:07:36.287052+0000","last_became_peered":"2023-10-09T11:07:36.287052+0000","last_unstale":"2023-10-09T11:09:35.114685+0000","last_undegraded":"2023-10-09T11:09:35.114685+0000","last_fullsized":"2023-10-09T11:09:35.114685+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:18.864873+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":380,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.009484+0000","last_change":"2023-10-09T11:03:21.185383+0000","last_active":"2023-10-09T11:09:35.009484+0000","last_peered":"2023-10-09T11:09:35.009484+0000","last_clean":"2023-10-09T11:09:35.009484+0000","last_became_active":"2023-10-09T11:03:21.185022+0000","last_became_peered":"2023-10-09T11:03:21.185022+0000","last_unstale":"2023-10-09T11:09:35.009484+0000","last_undegraded":"2023-10-09T11:09:35.009484+0000","last_fullsized":"2023-10-09T11:09:35.009484+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:05.230434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":125,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419880+0000","last_change":"2023-10-09T11:04:50.319477+0000","last_active":"2023-10-09T11:09:34.419880+0000","last_peered":"2023-10-09T11:09:34.419880+0000","last_clean":"2023-10-09T11:09:34.419880+0000","last_became_active":"2023-10-09T11:04:50.319148+0000","last_became_peered":"2023-10-09T11:04:50.319148+0000","last_unstale":"2023-10-09T11:09:34.419880+0000","last_undegraded":"2023-10-09T11:09:34.419880+0000","last_fullsized":"2023-10-09T11:09:34.419880+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:48.725566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":754,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114999+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:09:35.114999+0000","last_peered":"2023-10-09T11:09:35.114999+0000","last_clean":"2023-10-09T11:09:35.114999+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:09:35.114999+0000","last_undegraded":"2023-10-09T11:09:35.114999+0000","last_fullsized":"2023-10-09T11:09:35.114999+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":313,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.611319+0000","last_change":"2023-10-09T11:04:50.333711+0000","last_active":"2023-10-09T11:09:34.611319+0000","last_peered":"2023-10-09T11:09:34.611319+0000","last_clean":"2023-10-09T11:09:34.611319+0000","last_became_active":"2023-10-09T11:04:50.333262+0000","last_became_peered":"2023-10-09T11:04:50.333262+0000","last_unstale":"2023-10-09T11:09:34.611319+0000","last_undegraded":"2023-10-09T11:09:34.611319+0000","last_fullsized":"2023-10-09T11:09:34.611319+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.843765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":59,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422023+0000","last_change":"2023-10-09T11:07:33.237903+0000","last_active":"2023-10-09T11:09:34.422023+0000","last_peered":"2023-10-09T11:09:34.422023+0000","last_clean":"2023-10-09T11:09:34.422023+0000","last_became_active":"2023-10-09T11:07:33.237618+0000","last_became_peered":"2023-10-09T11:07:33.237618+0000","last_unstale":"2023-10-09T11:09:34.422023+0000","last_undegraded":"2023-10-09T11:09:34.422023+0000","last_fullsized":"2023-10-09T11:09:34.422023+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:25.600818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":774,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421682+0000","last_change":"2023-10-09T11:07:36.284534+0000","last_active":"2023-10-09T11:09:34.421682+0000","last_peered":"2023-10-09T11:09:34.421682+0000","last_clean":"2023-10-09T11:09:34.421682+0000","last_became_active":"2023-10-09T11:07:36.284254+0000","last_became_peered":"2023-10-09T11:07:36.284254+0000","last_unstale":"2023-10-09T11:09:34.421682+0000","last_undegraded":"2023-10-09T11:09:34.421682+0000","last_fullsized":"2023-10-09T11:09:34.421682+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:19.159630+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":126,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-10-09T11:09:38.431577+0000","last_change":"2023-10-09T11:04:50.310477+0000","last_active":"2023-10-09T11:09:38.431577+0000","last_peered":"2023-10-09T11:09:38.431577+0000","last_clean":"2023-10-09T11:09:38.431577+0000","last_became_active":"2023-10-09T11:04:50.310067+0000","last_became_peered":"2023-10-09T11:04:50.310067+0000","last_unstale":"2023-10-09T11:09:38.431577+0000","last_undegraded":"2023-10-09T11:09:38.431577+0000","last_fullsized":"2023-10-09T11:09:38.431577+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:52.884716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":406,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115388+0000","last_change":"2023-10-09T11:05:20.310581+0000","last_active":"2023-10-09T11:09:35.115388+0000","last_peered":"2023-10-09T11:09:35.115388+0000","last_clean":"2023-10-09T11:09:35.115388+0000","last_became_active":"2023-10-09T11:05:20.309830+0000","last_became_peered":"2023-10-09T11:05:20.309830+0000","last_unstale":"2023-10-09T11:09:35.115388+0000","last_undegraded":"2023-10-09T11:09:35.115388+0000","last_fullsized":"2023-10-09T11:09:35.115388+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:24.508211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":814,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422374+0000","last_change":"2023-10-09T11:07:36.282502+0000","last_active":"2023-10-09T11:09:34.422374+0000","last_peered":"2023-10-09T11:09:34.422374+0000","last_clean":"2023-10-09T11:09:34.422374+0000","last_became_active":"2023-10-09T11:07:36.282133+0000","last_became_peered":"2023-10-09T11:07:36.282133+0000","last_unstale":"2023-10-09T11:09:34.422374+0000","last_undegraded":"2023-10-09T11:09:34.422374+0000","last_fullsized":"2023-10-09T11:09:34.422374+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:03.762383+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":233,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115443+0000","last_change":"2023-10-09T11:06:24.245488+0000","last_active":"2023-10-09T11:09:35.115443+0000","last_peered":"2023-10-09T11:09:35.115443+0000","last_clean":"2023-10-09T11:09:35.115443+0000","last_became_active":"2023-10-09T11:06:24.245167+0000","last_became_peered":"2023-10-09T11:06:24.245167+0000","last_unstale":"2023-10-09T11:09:35.115443+0000","last_undegraded":"2023-10-09T11:09:35.115443+0000","last_fullsized":"2023-10-09T11:09:35.115443+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:31.702568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":135,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419097+0000","last_change":"2023-10-09T11:05:20.299453+0000","last_active":"2023-10-09T11:09:34.419097+0000","last_peered":"2023-10-09T11:09:34.419097+0000","last_clean":"2023-10-09T11:09:34.419097+0000","last_became_active":"2023-10-09T11:05:20.299117+0000","last_became_peered":"2023-10-09T11:05:20.299117+0000","last_unstale":"2023-10-09T11:09:34.419097+0000","last_undegraded":"2023-10-09T11:09:34.419097+0000","last_fullsized":"2023-10-09T11:09:34.419097+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:30.072813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":929,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.112627+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:09:35.112627+0000","last_peered":"2023-10-09T11:09:35.112627+0000","last_clean":"2023-10-09T11:09:35.112627+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:09:35.112627+0000","last_undegraded":"2023-10-09T11:09:35.112627+0000","last_fullsized":"2023-10-09T11:09:35.112627+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":468,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115221+0000","last_change":"2023-10-09T11:07:33.242665+0000","last_active":"2023-10-09T11:09:35.115221+0000","last_peered":"2023-10-09T11:09:35.115221+0000","last_clean":"2023-10-09T11:09:35.115221+0000","last_became_active":"2023-10-09T11:07:33.242313+0000","last_became_peered":"2023-10-09T11:07:33.242313+0000","last_unstale":"2023-10-09T11:09:35.115221+0000","last_undegraded":"2023-10-09T11:09:35.115221+0000","last_fullsized":"2023-10-09T11:09:35.115221+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:20.565135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":153,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419342+0000","last_change":"2023-10-09T11:07:33.225683+0000","last_active":"2023-10-09T11:09:34.419342+0000","last_peered":"2023-10-09T11:09:34.419342+0000","last_clean":"2023-10-09T11:09:34.419342+0000","last_became_active":"2023-10-09T11:07:33.225275+0000","last_became_peered":"2023-10-09T11:07:33.225275+0000","last_unstale":"2023-10-09T11:09:34.419342+0000","last_undegraded":"2023-10-09T11:09:34.419342+0000","last_fullsized":"2023-10-09T11:09:34.419342+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:10.131060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":826,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115215+0000","last_change":"2023-10-09T11:05:23.328981+0000","last_active":"2023-10-09T11:09:35.115215+0000","last_peered":"2023-10-09T11:09:35.115215+0000","last_clean":"2023-10-09T11:09:35.115215+0000","last_became_active":"2023-10-09T11:05:23.328616+0000","last_became_peered":"2023-10-09T11:05:23.328616+0000","last_unstale":"2023-10-09T11:09:35.115215+0000","last_undegraded":"2023-10-09T11:09:35.115215+0000","last_fullsized":"2023-10-09T11:09:35.115215+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:15.315819+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":65,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422309+0000","last_change":"2023-10-09T11:07:33.265842+0000","last_active":"2023-10-09T11:09:34.422309+0000","last_peered":"2023-10-09T11:09:34.422309+0000","last_clean":"2023-10-09T11:09:34.422309+0000","last_became_active":"2023-10-09T11:07:33.232311+0000","last_became_peered":"2023-10-09T11:07:33.232311+0000","last_unstale":"2023-10-09T11:09:34.422309+0000","last_undegraded":"2023-10-09T11:09:34.422309+0000","last_fullsized":"2023-10-09T11:09:34.422309+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:23.402582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":375,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114846+0000","last_change":"2023-10-09T11:05:20.316265+0000","last_active":"2023-10-09T11:09:35.114846+0000","last_peered":"2023-10-09T11:09:35.114846+0000","last_clean":"2023-10-09T11:09:35.114846+0000","last_became_active":"2023-10-09T11:05:20.315853+0000","last_became_peered":"2023-10-09T11:05:20.315853+0000","last_unstale":"2023-10-09T11:09:35.114846+0000","last_undegraded":"2023-10-09T11:09:35.114846+0000","last_fullsized":"2023-10-09T11:09:35.114846+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:35.093981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":991,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.612442+0000","last_change":"2023-10-09T11:04:53.274921+0000","last_active":"2023-10-09T11:09:34.612442+0000","last_peered":"2023-10-09T11:09:34.612442+0000","last_clean":"2023-10-09T11:09:34.612442+0000","last_became_active":"2023-10-09T11:04:53.274518+0000","last_became_peered":"2023-10-09T11:04:53.274518+0000","last_unstale":"2023-10-09T11:09:34.612442+0000","last_undegraded":"2023-10-09T11:09:34.612442+0000","last_fullsized":"2023-10-09T11:09:34.612442+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:14.315301+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":69,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422034+0000","last_change":"2023-10-09T11:07:33.262559+0000","last_active":"2023-10-09T11:09:34.422034+0000","last_peered":"2023-10-09T11:09:34.422034+0000","last_clean":"2023-10-09T11:09:34.422034+0000","last_became_active":"2023-10-09T11:07:33.231695+0000","last_became_peered":"2023-10-09T11:07:33.231695+0000","last_unstale":"2023-10-09T11:09:34.422034+0000","last_undegraded":"2023-10-09T11:09:34.422034+0000","last_fullsized":"2023-10-09T11:09:34.422034+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:59.006940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":621,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-10-09T11:09:38.434817+0000","last_change":"2023-10-09T11:05:20.311424+0000","last_active":"2023-10-09T11:09:38.434817+0000","last_peered":"2023-10-09T11:09:38.434817+0000","last_clean":"2023-10-09T11:09:38.434817+0000","last_became_active":"2023-10-09T11:05:20.311097+0000","last_became_peered":"2023-10-09T11:05:20.311097+0000","last_unstale":"2023-10-09T11:09:38.434817+0000","last_undegraded":"2023-10-09T11:09:38.434817+0000","last_fullsized":"2023-10-09T11:09:38.434817+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:19.274582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":639,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114885+0000","last_change":"2023-10-09T11:08:39.934938+0000","last_active":"2023-10-09T11:09:35.114885+0000","last_peered":"2023-10-09T11:09:35.114885+0000","last_clean":"2023-10-09T11:09:35.114885+0000","last_became_active":"2023-10-09T11:06:24.245076+0000","last_became_peered":"2023-10-09T11:06:24.245076+0000","last_unstale":"2023-10-09T11:09:35.114885+0000","last_undegraded":"2023-10-09T11:09:35.114885+0000","last_fullsized":"2023-10-09T11:09:35.114885+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.934865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:00.408009+0000","scrub_duration":0.0012831839999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":65,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421699+0000","last_change":"2023-10-09T11:07:33.269200+0000","last_active":"2023-10-09T11:09:34.421699+0000","last_peered":"2023-10-09T11:09:34.421699+0000","last_clean":"2023-10-09T11:09:34.421699+0000","last_became_active":"2023-10-09T11:07:33.236205+0000","last_became_peered":"2023-10-09T11:07:33.236205+0000","last_unstale":"2023-10-09T11:09:34.421699+0000","last_undegraded":"2023-10-09T11:09:34.421699+0000","last_fullsized":"2023-10-09T11:09:34.421699+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:24.574734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":144,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419672+0000","last_change":"2023-10-09T11:07:33.227803+0000","last_active":"2023-10-09T11:09:34.419672+0000","last_peered":"2023-10-09T11:09:34.419672+0000","last_clean":"2023-10-09T11:09:34.419672+0000","last_became_active":"2023-10-09T11:07:33.227486+0000","last_became_peered":"2023-10-09T11:07:33.227486+0000","last_unstale":"2023-10-09T11:09:34.419672+0000","last_undegraded":"2023-10-09T11:09:34.419672+0000","last_fullsized":"2023-10-09T11:09:34.419672+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:39.274875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":908,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114935+0000","last_change":"2023-10-09T11:05:23.323856+0000","last_active":"2023-10-09T11:09:35.114935+0000","last_peered":"2023-10-09T11:09:35.114935+0000","last_clean":"2023-10-09T11:09:35.114935+0000","last_became_active":"2023-10-09T11:05:23.322763+0000","last_became_peered":"2023-10-09T11:05:23.322763+0000","last_unstale":"2023-10-09T11:09:35.114935+0000","last_undegraded":"2023-10-09T11:09:35.114935+0000","last_fullsized":"2023-10-09T11:09:35.114935+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.760820+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":59,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421588+0000","last_change":"2023-10-09T11:07:33.233510+0000","last_active":"2023-10-09T11:09:34.421588+0000","last_peered":"2023-10-09T11:09:34.421588+0000","last_clean":"2023-10-09T11:09:34.421588+0000","last_became_active":"2023-10-09T11:07:33.232339+0000","last_became_peered":"2023-10-09T11:07:33.232339+0000","last_unstale":"2023-10-09T11:09:34.421588+0000","last_undegraded":"2023-10-09T11:09:34.421588+0000","last_fullsized":"2023-10-09T11:09:34.421588+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:20:55.324912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":538,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115298+0000","last_change":"2023-10-09T11:07:33.226594+0000","last_active":"2023-10-09T11:09:35.115298+0000","last_peered":"2023-10-09T11:09:35.115298+0000","last_clean":"2023-10-09T11:09:35.115298+0000","last_became_active":"2023-10-09T11:07:33.226275+0000","last_became_peered":"2023-10-09T11:07:33.226275+0000","last_unstale":"2023-10-09T11:09:35.115298+0000","last_undegraded":"2023-10-09T11:09:35.115298+0000","last_fullsized":"2023-10-09T11:09:35.115298+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:41.180088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":933,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.611510+0000","last_change":"2023-10-09T11:04:53.274236+0000","last_active":"2023-10-09T11:09:34.611510+0000","last_peered":"2023-10-09T11:09:34.611510+0000","last_clean":"2023-10-09T11:09:34.611510+0000","last_became_active":"2023-10-09T11:04:53.273817+0000","last_became_peered":"2023-10-09T11:04:53.273817+0000","last_unstale":"2023-10-09T11:09:34.611510+0000","last_undegraded":"2023-10-09T11:09:34.611510+0000","last_fullsized":"2023-10-09T11:09:34.611510+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:05.096905+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":194,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421899+0000","last_change":"2023-10-09T11:05:20.312129+0000","last_active":"2023-10-09T11:09:34.421899+0000","last_peered":"2023-10-09T11:09:34.421899+0000","last_clean":"2023-10-09T11:09:34.421899+0000","last_became_active":"2023-10-09T11:05:20.311810+0000","last_became_peered":"2023-10-09T11:05:20.311810+0000","last_unstale":"2023-10-09T11:09:34.421899+0000","last_undegraded":"2023-10-09T11:09:34.421899+0000","last_fullsized":"2023-10-09T11:09:34.421899+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:48.813391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":293,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.420147+0000","last_change":"2023-10-09T11:07:33.218244+0000","last_active":"2023-10-09T11:09:34.420147+0000","last_peered":"2023-10-09T11:09:34.420147+0000","last_clean":"2023-10-09T11:09:34.420147+0000","last_became_active":"2023-10-09T11:07:33.217918+0000","last_became_peered":"2023-10-09T11:07:33.217918+0000","last_unstale":"2023-10-09T11:09:34.420147+0000","last_undegraded":"2023-10-09T11:09:34.420147+0000","last_fullsized":"2023-10-09T11:09:34.420147+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:16.150105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":905,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115096+0000","last_change":"2023-10-09T11:05:23.324537+0000","last_active":"2023-10-09T11:09:35.115096+0000","last_peered":"2023-10-09T11:09:35.115096+0000","last_clean":"2023-10-09T11:09:35.115096+0000","last_became_active":"2023-10-09T11:05:23.324187+0000","last_became_peered":"2023-10-09T11:05:23.324187+0000","last_unstale":"2023-10-09T11:09:35.115096+0000","last_undegraded":"2023-10-09T11:09:35.115096+0000","last_fullsized":"2023-10-09T11:09:35.115096+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:25.286106+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":117,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421726+0000","last_change":"2023-10-09T11:05:20.323565+0000","last_active":"2023-10-09T11:09:34.421726+0000","last_peered":"2023-10-09T11:09:34.421726+0000","last_clean":"2023-10-09T11:09:34.421726+0000","last_became_active":"2023-10-09T11:05:20.309672+0000","last_became_peered":"2023-10-09T11:05:20.309672+0000","last_unstale":"2023-10-09T11:09:34.421726+0000","last_undegraded":"2023-10-09T11:09:34.421726+0000","last_fullsized":"2023-10-09T11:09:34.421726+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:50.211465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":144,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-10-09T11:09:38.432253+0000","last_change":"2023-10-09T11:07:33.228001+0000","last_active":"2023-10-09T11:09:38.432253+0000","last_peered":"2023-10-09T11:09:38.432253+0000","last_clean":"2023-10-09T11:09:38.432253+0000","last_became_active":"2023-10-09T11:07:33.227691+0000","last_became_peered":"2023-10-09T11:07:33.227691+0000","last_unstale":"2023-10-09T11:09:38.432253+0000","last_undegraded":"2023-10-09T11:09:38.432253+0000","last_fullsized":"2023-10-09T11:09:38.432253+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:24:16.098138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":831,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422390+0000","last_change":"2023-10-09T11:07:36.284927+0000","last_active":"2023-10-09T11:09:34.422390+0000","last_peered":"2023-10-09T11:09:34.422390+0000","last_clean":"2023-10-09T11:09:34.422390+0000","last_became_active":"2023-10-09T11:07:36.284631+0000","last_became_peered":"2023-10-09T11:07:36.284631+0000","last_unstale":"2023-10-09T11:09:34.422390+0000","last_undegraded":"2023-10-09T11:09:34.422390+0000","last_fullsized":"2023-10-09T11:09:34.422390+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:37.220243+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":63,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422056+0000","last_change":"2023-10-09T11:07:33.273503+0000","last_active":"2023-10-09T11:09:34.422056+0000","last_peered":"2023-10-09T11:09:34.422056+0000","last_clean":"2023-10-09T11:09:34.422056+0000","last_became_active":"2023-10-09T11:07:33.242980+0000","last_became_peered":"2023-10-09T11:07:33.242980+0000","last_unstale":"2023-10-09T11:09:34.422056+0000","last_undegraded":"2023-10-09T11:09:34.422056+0000","last_fullsized":"2023-10-09T11:09:34.422056+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:18.934972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":559,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114700+0000","last_change":"2023-10-09T11:07:33.304363+0000","last_active":"2023-10-09T11:09:35.114700+0000","last_peered":"2023-10-09T11:09:35.114700+0000","last_clean":"2023-10-09T11:09:35.114700+0000","last_became_active":"2023-10-09T11:07:33.304024+0000","last_became_peered":"2023-10-09T11:07:33.304024+0000","last_unstale":"2023-10-09T11:09:35.114700+0000","last_undegraded":"2023-10-09T11:09:35.114700+0000","last_fullsized":"2023-10-09T11:09:35.114700+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:21:51.522688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":981,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419713+0000","last_change":"2023-10-09T11:05:23.316129+0000","last_active":"2023-10-09T11:09:34.419713+0000","last_peered":"2023-10-09T11:09:34.419713+0000","last_clean":"2023-10-09T11:09:34.419713+0000","last_became_active":"2023-10-09T11:05:23.315792+0000","last_became_peered":"2023-10-09T11:05:23.315792+0000","last_unstale":"2023-10-09T11:09:34.419713+0000","last_undegraded":"2023-10-09T11:09:34.419713+0000","last_fullsized":"2023-10-09T11:09:34.419713+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:51.770547+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":63,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422114+0000","last_change":"2023-10-09T11:07:33.256691+0000","last_active":"2023-10-09T11:09:34.422114+0000","last_peered":"2023-10-09T11:09:34.422114+0000","last_clean":"2023-10-09T11:09:34.422114+0000","last_became_active":"2023-10-09T11:07:33.229554+0000","last_became_peered":"2023-10-09T11:07:33.229554+0000","last_unstale":"2023-10-09T11:09:34.422114+0000","last_undegraded":"2023-10-09T11:09:34.422114+0000","last_fullsized":"2023-10-09T11:09:34.422114+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:57.414795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":314,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.611301+0000","last_change":"2023-10-09T11:04:50.333538+0000","last_active":"2023-10-09T11:09:34.611301+0000","last_peered":"2023-10-09T11:09:34.611301+0000","last_clean":"2023-10-09T11:09:34.611301+0000","last_became_active":"2023-10-09T11:04:50.332514+0000","last_became_peered":"2023-10-09T11:04:50.332514+0000","last_unstale":"2023-10-09T11:09:34.611301+0000","last_undegraded":"2023-10-09T11:09:34.611301+0000","last_fullsized":"2023-10-09T11:09:34.611301+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:22.933839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":853,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115321+0000","last_change":"2023-10-09T11:05:23.322251+0000","last_active":"2023-10-09T11:09:35.115321+0000","last_peered":"2023-10-09T11:09:35.115321+0000","last_clean":"2023-10-09T11:09:35.115321+0000","last_became_active":"2023-10-09T11:05:23.321826+0000","last_became_peered":"2023-10-09T11:05:23.321826+0000","last_unstale":"2023-10-09T11:09:35.115321+0000","last_undegraded":"2023-10-09T11:09:35.115321+0000","last_fullsized":"2023-10-09T11:09:35.115321+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.858013+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":63,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422551+0000","last_change":"2023-10-09T11:07:33.271238+0000","last_active":"2023-10-09T11:09:34.422551+0000","last_peered":"2023-10-09T11:09:34.422551+0000","last_clean":"2023-10-09T11:09:34.422551+0000","last_became_active":"2023-10-09T11:07:33.241886+0000","last_became_peered":"2023-10-09T11:07:33.241886+0000","last_unstale":"2023-10-09T11:09:34.422551+0000","last_undegraded":"2023-10-09T11:09:34.422551+0000","last_fullsized":"2023-10-09T11:09:34.422551+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:48.882589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":127,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422757+0000","last_change":"2023-10-09T11:07:33.244131+0000","last_active":"2023-10-09T11:09:34.422757+0000","last_peered":"2023-10-09T11:09:34.422757+0000","last_clean":"2023-10-09T11:09:34.422757+0000","last_became_active":"2023-10-09T11:07:33.243797+0000","last_became_peered":"2023-10-09T11:07:33.243797+0000","last_unstale":"2023-10-09T11:09:34.422757+0000","last_undegraded":"2023-10-09T11:09:34.422757+0000","last_fullsized":"2023-10-09T11:09:34.422757+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:44.851043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":1109,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422231+0000","last_change":"2023-10-09T11:07:36.286417+0000","last_active":"2023-10-09T11:09:34.422231+0000","last_peered":"2023-10-09T11:09:34.422231+0000","last_clean":"2023-10-09T11:09:34.422231+0000","last_became_active":"2023-10-09T11:07:36.286061+0000","last_became_peered":"2023-10-09T11:07:36.286061+0000","last_unstale":"2023-10-09T11:09:34.422231+0000","last_undegraded":"2023-10-09T11:09:34.422231+0000","last_fullsized":"2023-10-09T11:09:34.422231+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:00.289500+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":455,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115174+0000","last_change":"2023-10-09T11:05:20.309385+0000","last_active":"2023-10-09T11:09:35.115174+0000","last_peered":"2023-10-09T11:09:35.115174+0000","last_clean":"2023-10-09T11:09:35.115174+0000","last_became_active":"2023-10-09T11:05:20.309034+0000","last_became_peered":"2023-10-09T11:05:20.309034+0000","last_unstale":"2023-10-09T11:09:35.115174+0000","last_undegraded":"2023-10-09T11:09:35.115174+0000","last_fullsized":"2023-10-09T11:09:35.115174+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:31.857485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":389,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.611471+0000","last_change":"2023-10-09T11:06:24.245966+0000","last_active":"2023-10-09T11:09:34.611471+0000","last_peered":"2023-10-09T11:09:34.611471+0000","last_clean":"2023-10-09T11:09:34.611471+0000","last_became_active":"2023-10-09T11:06:24.245310+0000","last_became_peered":"2023-10-09T11:06:24.245310+0000","last_unstale":"2023-10-09T11:09:34.611471+0000","last_undegraded":"2023-10-09T11:09:34.611471+0000","last_fullsized":"2023-10-09T11:09:34.611471+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:23.966200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":753,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421779+0000","last_change":"2023-10-09T11:07:36.283174+0000","last_active":"2023-10-09T11:09:34.421779+0000","last_peered":"2023-10-09T11:09:34.421779+0000","last_clean":"2023-10-09T11:09:34.421779+0000","last_became_active":"2023-10-09T11:07:36.278007+0000","last_became_peered":"2023-10-09T11:07:36.278007+0000","last_unstale":"2023-10-09T11:09:34.421779+0000","last_undegraded":"2023-10-09T11:09:34.421779+0000","last_fullsized":"2023-10-09T11:09:34.421779+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:51.693969+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":130,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419348+0000","last_change":"2023-10-09T11:04:50.529513+0000","last_active":"2023-10-09T11:09:34.419348+0000","last_peered":"2023-10-09T11:09:34.419348+0000","last_clean":"2023-10-09T11:09:34.419348+0000","last_became_active":"2023-10-09T11:04:50.339671+0000","last_became_peered":"2023-10-09T11:04:50.339671+0000","last_unstale":"2023-10-09T11:09:34.419348+0000","last_undegraded":"2023-10-09T11:09:34.419348+0000","last_fullsized":"2023-10-09T11:09:34.419348+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:24.473000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":135,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419635+0000","last_change":"2023-10-09T11:05:20.313855+0000","last_active":"2023-10-09T11:09:34.419635+0000","last_peered":"2023-10-09T11:09:34.419635+0000","last_clean":"2023-10-09T11:09:34.419635+0000","last_became_active":"2023-10-09T11:05:20.313535+0000","last_became_peered":"2023-10-09T11:05:20.313535+0000","last_unstale":"2023-10-09T11:09:34.419635+0000","last_undegraded":"2023-10-09T11:09:34.419635+0000","last_fullsized":"2023-10-09T11:09:34.419635+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:18:25.266434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":855,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114836+0000","last_change":"2023-10-09T11:04:53.272323+0000","last_active":"2023-10-09T11:09:35.114836+0000","last_peered":"2023-10-09T11:09:35.114836+0000","last_clean":"2023-10-09T11:09:35.114836+0000","last_became_active":"2023-10-09T11:04:53.272010+0000","last_became_peered":"2023-10-09T11:04:53.272010+0000","last_unstale":"2023-10-09T11:09:35.114836+0000","last_undegraded":"2023-10-09T11:09:35.114836+0000","last_fullsized":"2023-10-09T11:09:35.114836+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:23.913605+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":623,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.611516+0000","last_change":"2023-10-09T11:05:20.307499+0000","last_active":"2023-10-09T11:09:34.611516+0000","last_peered":"2023-10-09T11:09:34.611516+0000","last_clean":"2023-10-09T11:09:34.611516+0000","last_became_active":"2023-10-09T11:05:20.307198+0000","last_became_peered":"2023-10-09T11:05:20.307198+0000","last_unstale":"2023-10-09T11:09:34.611516+0000","last_undegraded":"2023-10-09T11:09:34.611516+0000","last_fullsized":"2023-10-09T11:09:34.611516+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:53.726844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":358,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.111904+0000","last_change":"2023-10-09T11:00:14.876266+0000","last_active":"2023-10-09T11:09:35.111904+0000","last_peered":"2023-10-09T11:09:35.111904+0000","last_clean":"2023-10-09T11:09:35.111904+0000","last_became_active":"2023-10-09T11:00:14.876003+0000","last_became_peered":"2023-10-09T11:00:14.876003+0000","last_unstale":"2023-10-09T11:09:35.111904+0000","last_undegraded":"2023-10-09T11:09:35.111904+0000","last_fullsized":"2023-10-09T11:09:35.111904+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:59.396936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":950,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.611556+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:09:34.611556+0000","last_peered":"2023-10-09T11:09:34.611556+0000","last_clean":"2023-10-09T11:09:34.611556+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:09:34.611556+0000","last_undegraded":"2023-10-09T11:09:34.611556+0000","last_fullsized":"2023-10-09T11:09:34.611556+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":622,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.111960+0000","last_change":"2023-10-09T11:04:57.953307+0000","last_active":"2023-10-09T11:09:35.111960+0000","last_peered":"2023-10-09T11:09:35.111960+0000","last_clean":"2023-10-09T11:09:35.111960+0000","last_became_active":"2023-10-09T11:04:57.953095+0000","last_became_peered":"2023-10-09T11:04:57.953095+0000","last_unstale":"2023-10-09T11:09:35.111960+0000","last_undegraded":"2023-10-09T11:09:35.111960+0000","last_fullsized":"2023-10-09T11:09:35.111960+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:05.562643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":308,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.612332+0000","last_change":"2023-10-09T11:05:20.310255+0000","last_active":"2023-10-09T11:09:34.612332+0000","last_peered":"2023-10-09T11:09:34.612332+0000","last_clean":"2023-10-09T11:09:34.612332+0000","last_became_active":"2023-10-09T11:05:20.309109+0000","last_became_peered":"2023-10-09T11:05:20.309109+0000","last_unstale":"2023-10-09T11:09:34.612332+0000","last_undegraded":"2023-10-09T11:09:34.612332+0000","last_fullsized":"2023-10-09T11:09:34.612332+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:07.429039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":839,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115026+0000","last_change":"2023-10-09T11:05:23.329329+0000","last_active":"2023-10-09T11:09:35.115026+0000","last_peered":"2023-10-09T11:09:35.115026+0000","last_clean":"2023-10-09T11:09:35.115026+0000","last_became_active":"2023-10-09T11:05:23.328976+0000","last_became_peered":"2023-10-09T11:05:23.328976+0000","last_unstale":"2023-10-09T11:09:35.115026+0000","last_undegraded":"2023-10-09T11:09:35.115026+0000","last_fullsized":"2023-10-09T11:09:35.115026+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:41.848184+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":144,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419212+0000","last_change":"2023-10-09T11:07:33.236133+0000","last_active":"2023-10-09T11:09:34.419212+0000","last_peered":"2023-10-09T11:09:34.419212+0000","last_clean":"2023-10-09T11:09:34.419212+0000","last_became_active":"2023-10-09T11:07:33.235818+0000","last_became_peered":"2023-10-09T11:07:33.235818+0000","last_unstale":"2023-10-09T11:09:34.419212+0000","last_undegraded":"2023-10-09T11:09:34.419212+0000","last_fullsized":"2023-10-09T11:09:34.419212+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:20.627846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":305,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.612410+0000","last_change":"2023-10-09T11:04:50.332978+0000","last_active":"2023-10-09T11:09:34.612410+0000","last_peered":"2023-10-09T11:09:34.612410+0000","last_clean":"2023-10-09T11:09:34.612410+0000","last_became_active":"2023-10-09T11:04:50.332735+0000","last_became_peered":"2023-10-09T11:04:50.332735+0000","last_unstale":"2023-10-09T11:09:34.612410+0000","last_undegraded":"2023-10-09T11:09:34.612410+0000","last_fullsized":"2023-10-09T11:09:34.612410+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:39:44.659751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":1126,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419588+0000","last_change":"2023-10-09T11:07:36.291716+0000","last_active":"2023-10-09T11:09:34.419588+0000","last_peered":"2023-10-09T11:09:34.419588+0000","last_clean":"2023-10-09T11:09:34.419588+0000","last_became_active":"2023-10-09T11:07:36.291424+0000","last_became_peered":"2023-10-09T11:07:36.291424+0000","last_unstale":"2023-10-09T11:09:34.419588+0000","last_undegraded":"2023-10-09T11:09:34.419588+0000","last_fullsized":"2023-10-09T11:09:34.419588+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:36:22.316387+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"592'150","reported_seq":884,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.611866+0000","last_change":"2023-10-09T11:09:20.424914+0000","last_active":"2023-10-09T11:09:34.611866+0000","last_peered":"2023-10-09T11:09:34.611866+0000","last_clean":"2023-10-09T11:09:34.611866+0000","last_became_active":"2023-10-09T11:07:33.226176+0000","last_became_peered":"2023-10-09T11:07:33.226176+0000","last_unstale":"2023-10-09T11:09:34.611866+0000","last_undegraded":"2023-10-09T11:09:34.611866+0000","last_fullsized":"2023-10-09T11:09:34.611866+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:16.888732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034113299999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":876,"num_read_kb":2947,"num_write":148,"num_write_kb":2179,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":472,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115215+0000","last_change":"2023-10-09T11:04:50.341353+0000","last_active":"2023-10-09T11:09:35.115215+0000","last_peered":"2023-10-09T11:09:35.115215+0000","last_clean":"2023-10-09T11:09:35.115215+0000","last_became_active":"2023-10-09T11:04:50.340801+0000","last_became_peered":"2023-10-09T11:04:50.340801+0000","last_unstale":"2023-10-09T11:09:35.115215+0000","last_undegraded":"2023-10-09T11:09:35.115215+0000","last_fullsized":"2023-10-09T11:09:35.115215+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:00.699069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":281,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.611667+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:09:34.611667+0000","last_peered":"2023-10-09T11:09:34.611667+0000","last_clean":"2023-10-09T11:09:34.611667+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:09:34.611667+0000","last_undegraded":"2023-10-09T11:09:34.611667+0000","last_fullsized":"2023-10-09T11:09:34.611667+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":797,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114791+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:09:35.114791+0000","last_peered":"2023-10-09T11:09:35.114791+0000","last_clean":"2023-10-09T11:09:35.114791+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:09:35.114791+0000","last_undegraded":"2023-10-09T11:09:35.114791+0000","last_fullsized":"2023-10-09T11:09:35.114791+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"592'149","reported_seq":1214,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114904+0000","last_change":"2023-10-09T11:09:20.477238+0000","last_active":"2023-10-09T11:09:35.114904+0000","last_peered":"2023-10-09T11:09:35.114904+0000","last_clean":"2023-10-09T11:09:35.114904+0000","last_became_active":"2023-10-09T11:05:20.315471+0000","last_became_peered":"2023-10-09T11:05:20.315471+0000","last_unstale":"2023-10-09T11:09:35.114904+0000","last_undegraded":"2023-10-09T11:09:35.114904+0000","last_fullsized":"2023-10-09T11:09:35.114904+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:47.548763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051877200000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1181,"num_read_kb":1421,"num_write":125,"num_write_kb":1278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":503,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422022+0000","last_change":"2023-10-09T11:06:24.245282+0000","last_active":"2023-10-09T11:09:34.422022+0000","last_peered":"2023-10-09T11:09:34.422022+0000","last_clean":"2023-10-09T11:09:34.422022+0000","last_became_active":"2023-10-09T11:06:24.244981+0000","last_became_peered":"2023-10-09T11:06:24.244981+0000","last_unstale":"2023-10-09T11:09:34.422022+0000","last_undegraded":"2023-10-09T11:09:34.422022+0000","last_fullsized":"2023-10-09T11:09:34.422022+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:22:15.327655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":314,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.611756+0000","last_change":"2023-10-09T11:04:50.311579+0000","last_active":"2023-10-09T11:09:34.611756+0000","last_peered":"2023-10-09T11:09:34.611756+0000","last_clean":"2023-10-09T11:09:34.611756+0000","last_became_active":"2023-10-09T11:04:50.311270+0000","last_became_peered":"2023-10-09T11:04:50.311270+0000","last_unstale":"2023-10-09T11:09:34.611756+0000","last_undegraded":"2023-10-09T11:09:34.611756+0000","last_fullsized":"2023-10-09T11:09:34.611756+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:49.850302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":951,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421888+0000","last_change":"2023-10-09T11:07:36.284280+0000","last_active":"2023-10-09T11:09:34.421888+0000","last_peered":"2023-10-09T11:09:34.421888+0000","last_clean":"2023-10-09T11:09:34.421888+0000","last_became_active":"2023-10-09T11:07:36.283966+0000","last_became_peered":"2023-10-09T11:07:36.283966+0000","last_unstale":"2023-10-09T11:09:34.421888+0000","last_undegraded":"2023-10-09T11:09:34.421888+0000","last_fullsized":"2023-10-09T11:09:34.421888+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:54.128331+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"591'48","reported_seq":610,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.112595+0000","last_change":"2023-10-09T11:09:20.421917+0000","last_active":"2023-10-09T11:09:35.112595+0000","last_peered":"2023-10-09T11:09:35.112595+0000","last_clean":"2023-10-09T11:09:35.112595+0000","last_became_active":"2023-10-09T11:04:50.381253+0000","last_became_peered":"2023-10-09T11:04:50.381253+0000","last_unstale":"2023-10-09T11:09:35.112595+0000","last_undegraded":"2023-10-09T11:09:35.112595+0000","last_fullsized":"2023-10-09T11:09:35.112595+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":48,"log_dups_size":0,"ondisk_log_size":48,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:24.030274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001144136,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":297,"num_read_kb":2229,"num_write":37,"num_write_kb":2061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":299,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.611346+0000","last_change":"2023-10-09T11:05:20.298542+0000","last_active":"2023-10-09T11:09:34.611346+0000","last_peered":"2023-10-09T11:09:34.611346+0000","last_clean":"2023-10-09T11:09:34.611346+0000","last_became_active":"2023-10-09T11:05:20.298119+0000","last_became_peered":"2023-10-09T11:05:20.298119+0000","last_unstale":"2023-10-09T11:09:34.611346+0000","last_undegraded":"2023-10-09T11:09:34.611346+0000","last_fullsized":"2023-10-09T11:09:34.611346+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:00.827214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":279,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.612482+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:09:34.612482+0000","last_peered":"2023-10-09T11:09:34.612482+0000","last_clean":"2023-10-09T11:09:34.612482+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:09:34.612482+0000","last_undegraded":"2023-10-09T11:09:34.612482+0000","last_fullsized":"2023-10-09T11:09:34.612482+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"592'125","reported_seq":793,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115065+0000","last_change":"2023-10-09T11:09:20.477093+0000","last_active":"2023-10-09T11:09:35.115065+0000","last_peered":"2023-10-09T11:09:35.115065+0000","last_clean":"2023-10-09T11:09:35.115065+0000","last_became_active":"2023-10-09T11:07:33.226289+0000","last_became_peered":"2023-10-09T11:07:33.226289+0000","last_unstale":"2023-10-09T11:09:35.115065+0000","last_undegraded":"2023-10-09T11:09:35.115065+0000","last_fullsized":"2023-10-09T11:09:35.115065+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:15.605946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026939299999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":322,"num_read_kb":786,"num_write":116,"num_write_kb":327,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":896,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421964+0000","last_change":"2023-10-09T11:07:36.290700+0000","last_active":"2023-10-09T11:09:34.421964+0000","last_peered":"2023-10-09T11:09:34.421964+0000","last_clean":"2023-10-09T11:09:34.421964+0000","last_became_active":"2023-10-09T11:07:36.290357+0000","last_became_peered":"2023-10-09T11:07:36.290357+0000","last_unstale":"2023-10-09T11:09:34.421964+0000","last_undegraded":"2023-10-09T11:09:34.421964+0000","last_fullsized":"2023-10-09T11:09:34.421964+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:34.494777+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":135,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419402+0000","last_change":"2023-10-09T11:05:20.310666+0000","last_active":"2023-10-09T11:09:34.419402+0000","last_peered":"2023-10-09T11:09:34.419402+0000","last_clean":"2023-10-09T11:09:34.419402+0000","last_became_active":"2023-10-09T11:05:20.310413+0000","last_became_peered":"2023-10-09T11:05:20.310413+0000","last_unstale":"2023-10-09T11:09:34.419402+0000","last_undegraded":"2023-10-09T11:09:34.419402+0000","last_fullsized":"2023-10-09T11:09:34.419402+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:34.169905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":559,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422486+0000","last_change":"2023-10-09T11:05:20.316523+0000","last_active":"2023-10-09T11:09:34.422486+0000","last_peered":"2023-10-09T11:09:34.422486+0000","last_clean":"2023-10-09T11:09:34.422486+0000","last_became_active":"2023-10-09T11:05:20.314493+0000","last_became_peered":"2023-10-09T11:05:20.314493+0000","last_unstale":"2023-10-09T11:09:34.422486+0000","last_undegraded":"2023-10-09T11:09:34.422486+0000","last_fullsized":"2023-10-09T11:09:34.422486+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:41.618701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":970,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.611425+0000","last_change":"2023-10-09T11:05:23.322074+0000","last_active":"2023-10-09T11:09:34.611425+0000","last_peered":"2023-10-09T11:09:34.611425+0000","last_clean":"2023-10-09T11:09:34.611425+0000","last_became_active":"2023-10-09T11:05:23.321749+0000","last_became_peered":"2023-10-09T11:05:23.321749+0000","last_unstale":"2023-10-09T11:09:34.611425+0000","last_undegraded":"2023-10-09T11:09:34.611425+0000","last_fullsized":"2023-10-09T11:09:34.611425+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:07.010932+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"591'46","reported_seq":353,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421703+0000","last_change":"2023-10-09T11:09:20.432327+0000","last_active":"2023-10-09T11:09:34.421703+0000","last_peered":"2023-10-09T11:09:34.421703+0000","last_clean":"2023-10-09T11:09:34.421703+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:09:34.421703+0000","last_undegraded":"2023-10-09T11:09:34.421703+0000","last_fullsized":"2023-10-09T11:09:34.421703+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":46,"log_dups_size":0,"ondisk_log_size":46,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000350947,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":271,"num_read_kb":1543,"num_write":39,"num_write_kb":1183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":226,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419185+0000","last_change":"2023-10-09T11:06:24.248362+0000","last_active":"2023-10-09T11:09:34.419185+0000","last_peered":"2023-10-09T11:09:34.419185+0000","last_clean":"2023-10-09T11:09:34.419185+0000","last_became_active":"2023-10-09T11:06:24.247348+0000","last_became_peered":"2023-10-09T11:06:24.247348+0000","last_unstale":"2023-10-09T11:09:34.419185+0000","last_undegraded":"2023-10-09T11:09:34.419185+0000","last_fullsized":"2023-10-09T11:09:34.419185+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:26.728754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":425,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.611487+0000","last_change":"2023-10-09T11:06:24.253511+0000","last_active":"2023-10-09T11:09:34.611487+0000","last_peered":"2023-10-09T11:09:34.611487+0000","last_clean":"2023-10-09T11:09:34.611487+0000","last_became_active":"2023-10-09T11:06:24.252739+0000","last_became_peered":"2023-10-09T11:06:24.252739+0000","last_unstale":"2023-10-09T11:09:34.611487+0000","last_undegraded":"2023-10-09T11:09:34.611487+0000","last_fullsized":"2023-10-09T11:09:34.611487+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:50.241907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":830,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419964+0000","last_change":"2023-10-09T11:06:24.239676+0000","last_active":"2023-10-09T11:09:34.419964+0000","last_peered":"2023-10-09T11:09:34.419964+0000","last_clean":"2023-10-09T11:09:34.419964+0000","last_became_active":"2023-10-09T11:06:24.239375+0000","last_became_peered":"2023-10-09T11:06:24.239375+0000","last_unstale":"2023-10-09T11:09:34.419964+0000","last_undegraded":"2023-10-09T11:09:34.419964+0000","last_fullsized":"2023-10-09T11:09:34.419964+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:27.953053+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"591'57","reported_seq":257,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422283+0000","last_change":"2023-10-09T11:09:20.432685+0000","last_active":"2023-10-09T11:09:34.422283+0000","last_peered":"2023-10-09T11:09:34.422283+0000","last_clean":"2023-10-09T11:09:34.422283+0000","last_became_active":"2023-10-09T11:07:33.229061+0000","last_became_peered":"2023-10-09T11:07:33.229061+0000","last_unstale":"2023-10-09T11:09:34.422283+0000","last_undegraded":"2023-10-09T11:09:34.422283+0000","last_fullsized":"2023-10-09T11:09:34.422283+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":57,"log_dups_size":0,"ondisk_log_size":57,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:30.978293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043334800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":262,"num_read_kb":1253,"num_write":50,"num_write_kb":1077,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":281,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.611974+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:09:34.611974+0000","last_peered":"2023-10-09T11:09:34.611974+0000","last_clean":"2023-10-09T11:09:34.611974+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:09:34.611974+0000","last_undegraded":"2023-10-09T11:09:34.611974+0000","last_fullsized":"2023-10-09T11:09:34.611974+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":556,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114975+0000","last_change":"2023-10-09T11:07:33.227271+0000","last_active":"2023-10-09T11:09:35.114975+0000","last_peered":"2023-10-09T11:09:35.114975+0000","last_clean":"2023-10-09T11:09:35.114975+0000","last_became_active":"2023-10-09T11:07:33.226760+0000","last_became_peered":"2023-10-09T11:07:33.226760+0000","last_unstale":"2023-10-09T11:09:35.114975+0000","last_undegraded":"2023-10-09T11:09:35.114975+0000","last_fullsized":"2023-10-09T11:09:35.114975+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:32:12.268123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":799,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.112784+0000","last_change":"2023-10-09T11:05:23.323368+0000","last_active":"2023-10-09T11:09:35.112784+0000","last_peered":"2023-10-09T11:09:35.112784+0000","last_clean":"2023-10-09T11:09:35.112784+0000","last_became_active":"2023-10-09T11:05:23.323032+0000","last_became_peered":"2023-10-09T11:05:23.323032+0000","last_unstale":"2023-10-09T11:09:35.112784+0000","last_undegraded":"2023-10-09T11:09:35.112784+0000","last_fullsized":"2023-10-09T11:09:35.112784+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:09.393584+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":156,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419438+0000","last_change":"2023-10-09T11:07:33.246378+0000","last_active":"2023-10-09T11:09:34.419438+0000","last_peered":"2023-10-09T11:09:34.419438+0000","last_clean":"2023-10-09T11:09:34.419438+0000","last_became_active":"2023-10-09T11:07:33.228019+0000","last_became_peered":"2023-10-09T11:07:33.228019+0000","last_unstale":"2023-10-09T11:09:34.419438+0000","last_undegraded":"2023-10-09T11:09:34.419438+0000","last_fullsized":"2023-10-09T11:09:34.419438+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:18.511517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"592'319","reported_seq":1589,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.612380+0000","last_change":"2023-10-09T11:09:20.427659+0000","last_active":"2023-10-09T11:09:34.612380+0000","last_peered":"2023-10-09T11:09:34.612380+0000","last_clean":"2023-10-09T11:09:34.612380+0000","last_became_active":"2023-10-09T11:05:20.309827+0000","last_became_peered":"2023-10-09T11:05:20.309827+0000","last_unstale":"2023-10-09T11:09:34.612380+0000","last_undegraded":"2023-10-09T11:09:34.612380+0000","last_fullsized":"2023-10-09T11:09:34.612380+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":319,"log_dups_size":0,"ondisk_log_size":319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:23.934383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001122743,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2478,"num_read_kb":2521,"num_write":249,"num_write_kb":324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":144,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419123+0000","last_change":"2023-10-09T11:07:33.217595+0000","last_active":"2023-10-09T11:09:34.419123+0000","last_peered":"2023-10-09T11:09:34.419123+0000","last_clean":"2023-10-09T11:09:34.419123+0000","last_became_active":"2023-10-09T11:07:33.217155+0000","last_became_peered":"2023-10-09T11:07:33.217155+0000","last_unstale":"2023-10-09T11:09:34.419123+0000","last_undegraded":"2023-10-09T11:09:34.419123+0000","last_fullsized":"2023-10-09T11:09:34.419123+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:55.127329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":763,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419592+0000","last_change":"2023-10-09T11:07:36.284491+0000","last_active":"2023-10-09T11:09:34.419592+0000","last_peered":"2023-10-09T11:09:34.419592+0000","last_clean":"2023-10-09T11:09:34.419592+0000","last_became_active":"2023-10-09T11:07:36.284249+0000","last_became_peered":"2023-10-09T11:07:36.284249+0000","last_unstale":"2023-10-09T11:09:34.419592+0000","last_undegraded":"2023-10-09T11:09:34.419592+0000","last_fullsized":"2023-10-09T11:09:34.419592+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:03.653371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:03.653371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:09.629417+0000","scrub_duration":0.001576771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"592'82","reported_seq":264,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422176+0000","last_change":"2023-10-09T11:09:20.433178+0000","last_active":"2023-10-09T11:09:34.422176+0000","last_peered":"2023-10-09T11:09:34.422176+0000","last_clean":"2023-10-09T11:09:34.422176+0000","last_became_active":"2023-10-09T11:07:33.241013+0000","last_became_peered":"2023-10-09T11:07:33.241013+0000","last_unstale":"2023-10-09T11:09:34.422176+0000","last_undegraded":"2023-10-09T11:09:34.422176+0000","last_fullsized":"2023-10-09T11:09:34.422176+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":82,"log_dups_size":0,"ondisk_log_size":82,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:26:57.164502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075188800000000004,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":544,"num_write":82,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":298,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.612357+0000","last_change":"2023-10-09T11:06:24.253714+0000","last_active":"2023-10-09T11:09:34.612357+0000","last_peered":"2023-10-09T11:09:34.612357+0000","last_clean":"2023-10-09T11:09:34.612357+0000","last_became_active":"2023-10-09T11:06:24.253143+0000","last_became_peered":"2023-10-09T11:06:24.253143+0000","last_unstale":"2023-10-09T11:09:34.612357+0000","last_undegraded":"2023-10-09T11:09:34.612357+0000","last_fullsized":"2023-10-09T11:09:34.612357+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:05.682288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":402,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.114775+0000","last_change":"2023-10-09T11:05:20.313560+0000","last_active":"2023-10-09T11:09:35.114775+0000","last_peered":"2023-10-09T11:09:35.114775+0000","last_clean":"2023-10-09T11:09:35.114775+0000","last_became_active":"2023-10-09T11:05:20.313214+0000","last_became_peered":"2023-10-09T11:05:20.313214+0000","last_unstale":"2023-10-09T11:09:35.114775+0000","last_undegraded":"2023-10-09T11:09:35.114775+0000","last_fullsized":"2023-10-09T11:09:35.114775+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:23:32.527205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":1007,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-10-09T11:09:38.432364+0000","last_change":"2023-10-09T11:05:23.329886+0000","last_active":"2023-10-09T11:09:38.432364+0000","last_peered":"2023-10-09T11:09:38.432364+0000","last_clean":"2023-10-09T11:09:38.432364+0000","last_became_active":"2023-10-09T11:05:23.329561+0000","last_became_peered":"2023-10-09T11:05:23.329561+0000","last_unstale":"2023-10-09T11:09:38.432364+0000","last_undegraded":"2023-10-09T11:09:38.432364+0000","last_fullsized":"2023-10-09T11:09:38.432364+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:46.933799+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":59,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.422089+0000","last_change":"2023-10-09T11:07:33.229864+0000","last_active":"2023-10-09T11:09:34.422089+0000","last_peered":"2023-10-09T11:09:34.422089+0000","last_clean":"2023-10-09T11:09:34.422089+0000","last_became_active":"2023-10-09T11:07:33.229547+0000","last_became_peered":"2023-10-09T11:07:33.229547+0000","last_unstale":"2023-10-09T11:09:34.422089+0000","last_undegraded":"2023-10-09T11:09:34.422089+0000","last_fullsized":"2023-10-09T11:09:34.422089+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:16:34.990439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":382,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.112943+0000","last_change":"2023-10-09T11:04:50.510688+0000","last_active":"2023-10-09T11:09:35.112943+0000","last_peered":"2023-10-09T11:09:35.112943+0000","last_clean":"2023-10-09T11:09:35.112943+0000","last_became_active":"2023-10-09T11:04:50.507404+0000","last_became_peered":"2023-10-09T11:04:50.507404+0000","last_unstale":"2023-10-09T11:09:35.112943+0000","last_undegraded":"2023-10-09T11:09:35.112943+0000","last_fullsized":"2023-10-09T11:09:35.112943+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:13.663394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":964,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.420235+0000","last_change":"2023-10-09T11:05:23.321040+0000","last_active":"2023-10-09T11:09:34.420235+0000","last_peered":"2023-10-09T11:09:34.420235+0000","last_clean":"2023-10-09T11:09:34.420235+0000","last_became_active":"2023-10-09T11:05:23.320674+0000","last_became_peered":"2023-10-09T11:05:23.320674+0000","last_unstale":"2023-10-09T11:09:34.420235+0000","last_undegraded":"2023-10-09T11:09:34.420235+0000","last_fullsized":"2023-10-09T11:09:34.420235+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.527660+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":424,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115106+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:09:35.115106+0000","last_peered":"2023-10-09T11:09:35.115106+0000","last_clean":"2023-10-09T11:09:35.115106+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:09:35.115106+0000","last_undegraded":"2023-10-09T11:09:35.115106+0000","last_fullsized":"2023-10-09T11:09:35.115106+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":125,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419257+0000","last_change":"2023-10-09T11:04:50.511984+0000","last_active":"2023-10-09T11:09:34.419257+0000","last_peered":"2023-10-09T11:09:34.419257+0000","last_clean":"2023-10-09T11:09:34.419257+0000","last_became_active":"2023-10-09T11:04:50.510820+0000","last_became_peered":"2023-10-09T11:04:50.510820+0000","last_unstale":"2023-10-09T11:09:34.419257+0000","last_undegraded":"2023-10-09T11:09:34.419257+0000","last_fullsized":"2023-10-09T11:09:34.419257+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:38.662189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":575,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115201+0000","last_change":"2023-10-09T11:05:20.327032+0000","last_active":"2023-10-09T11:09:35.115201+0000","last_peered":"2023-10-09T11:09:35.115201+0000","last_clean":"2023-10-09T11:09:35.115201+0000","last_became_active":"2023-10-09T11:05:20.320193+0000","last_became_peered":"2023-10-09T11:05:20.320193+0000","last_unstale":"2023-10-09T11:09:35.115201+0000","last_undegraded":"2023-10-09T11:09:35.115201+0000","last_fullsized":"2023-10-09T11:09:35.115201+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:44.862890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":125,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.419327+0000","last_change":"2023-10-09T11:04:50.384219+0000","last_active":"2023-10-09T11:09:34.419327+0000","last_peered":"2023-10-09T11:09:34.419327+0000","last_clean":"2023-10-09T11:09:34.419327+0000","last_became_active":"2023-10-09T11:04:50.384028+0000","last_became_peered":"2023-10-09T11:04:50.384028+0000","last_unstale":"2023-10-09T11:09:34.419327+0000","last_undegraded":"2023-10-09T11:09:34.419327+0000","last_fullsized":"2023-10-09T11:09:34.419327+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:04.721869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":782,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.421841+0000","last_change":"2023-10-09T11:08:39.540029+0000","last_active":"2023-10-09T11:09:34.421841+0000","last_peered":"2023-10-09T11:09:34.421841+0000","last_clean":"2023-10-09T11:09:34.421841+0000","last_became_active":"2023-10-09T11:07:36.291077+0000","last_became_peered":"2023-10-09T11:07:36.291077+0000","last_unstale":"2023-10-09T11:09:34.421841+0000","last_undegraded":"2023-10-09T11:09:34.421841+0000","last_fullsized":"2023-10-09T11:09:34.421841+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.539966+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.539966+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:20.767812+0000","scrub_duration":0.0016245929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":135,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.420316+0000","last_change":"2023-10-09T11:05:20.299321+0000","last_active":"2023-10-09T11:09:34.420316+0000","last_peered":"2023-10-09T11:09:34.420316+0000","last_clean":"2023-10-09T11:09:34.420316+0000","last_became_active":"2023-10-09T11:05:20.299019+0000","last_became_peered":"2023-10-09T11:05:20.299019+0000","last_unstale":"2023-10-09T11:09:34.420316+0000","last_undegraded":"2023-10-09T11:09:34.420316+0000","last_fullsized":"2023-10-09T11:09:34.420316+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:08.645909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":126,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-10-09T11:09:38.433480+0000","last_change":"2023-10-09T11:04:50.366890+0000","last_active":"2023-10-09T11:09:38.433480+0000","last_peered":"2023-10-09T11:09:38.433480+0000","last_clean":"2023-10-09T11:09:38.433480+0000","last_became_active":"2023-10-09T11:04:50.366558+0000","last_became_peered":"2023-10-09T11:04:50.366558+0000","last_unstale":"2023-10-09T11:09:38.433480+0000","last_undegraded":"2023-10-09T11:09:38.433480+0000","last_fullsized":"2023-10-09T11:09:38.433480+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:50.993153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":898,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.611659+0000","last_change":"2023-10-09T11:04:53.274332+0000","last_active":"2023-10-09T11:09:34.611659+0000","last_peered":"2023-10-09T11:09:34.611659+0000","last_clean":"2023-10-09T11:09:34.611659+0000","last_became_active":"2023-10-09T11:04:53.273798+0000","last_became_peered":"2023-10-09T11:04:53.273798+0000","last_unstale":"2023-10-09T11:09:34.611659+0000","last_undegraded":"2023-10-09T11:09:34.611659+0000","last_fullsized":"2023-10-09T11:09:34.611659+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:04.220970+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":144,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:35.115365+0000","last_change":"2023-10-09T11:06:24.249877+0000","last_active":"2023-10-09T11:09:35.115365+0000","last_peered":"2023-10-09T11:09:35.115365+0000","last_clean":"2023-10-09T11:09:35.115365+0000","last_became_active":"2023-10-09T11:06:24.249573+0000","last_became_peered":"2023-10-09T11:06:24.249573+0000","last_unstale":"2023-10-09T11:09:35.115365+0000","last_undegraded":"2023-10-09T11:09:35.115365+0000","last_fullsized":"2023-10-09T11:09:35.115365+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:11.798192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":647,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-10-09T11:09:34.611385+0000","last_change":"2023-10-09T11:05:20.307347+0000","last_active":"2023-10-09T11:09:34.611385+0000","last_peered":"2023-10-09T11:09:34.611385+0000","last_clean":"2023-10-09T11:09:34.611385+0000","last_became_active":"2023-10-09T11:05:20.307126+0000","last_became_peered":"2023-10-09T11:05:20.307126+0000","last_unstale":"2023-10-09T11:09:34.611385+0000","last_undegraded":"2023-10-09T11:09:34.611385+0000","last_fullsized":"2023-10-09T11:09:34.611385+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:34.408515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":91,"num_bytes_recovered":15742,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12694,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13244,"num_write":846,"num_write_kb":12712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332517,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":927832,"kb_used_data":28432,"kb_used_omap":0,"kb_used_meta":899392,"kb_avail":92817320,"statfs":{"total":95995035648,"available":95044935680,"internally_reserved":0,"allocated":29114368,"data_stored":27263642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920977408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:21 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.04,"15min":1.6990000000000001},"min":{"1min":0.27100000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.2949999999999999,"5min":9.8219999999999992,"15min":9.8219999999999992},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.389,"15min":1.694},"min":{"1min":0.33200000000000002,"5min":0.24299999999999999,"15min":0.216},"max":{"1min":3.1640000000000001,"5min":15.853,"15min":15.853},"last":0.61199999999999999}]},{"osd":1,"last update":"Mon Oct 9 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":1.194,"15min":1.47},"min":{"1min":0.315,"5min":0.222,"15min":0.222},"max":{"1min":0.83499999999999996,"5min":11.619999999999999,"15min":11.619999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":1.8,"15min":1.843},"min":{"1min":0.31900000000000001,"5min":0.25,"15min":0.25},"max":{"1min":0.91000000000000003,"5min":13.096,"15min":13.096},"last":0.51400000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:08:54 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":1.573,"15min":1.77},"min":{"1min":0.309,"5min":0.309,"15min":0.245},"max":{"1min":5.1440000000000001,"5min":12.226000000000001,"15min":16.605},"last":0.499},{"interface":"front","average":{"1min":2.2770000000000001,"5min":1.6220000000000001,"15min":1.802},"min":{"1min":0.379,"5min":0.223,"15min":0.223},"max":{"1min":16.157,"5min":16.157,"15min":16.157},"last":0.65800000000000003}]},{"osd":3,"last update":"Mon Oct 9 11:09:21 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":1.554,"15min":1.45},"min":{"1min":0.35499999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.194,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.68899999999999995,"5min":1.665,"15min":1.7490000000000001},"min":{"1min":0.39300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.8879999999999999,"5min":8.2330000000000005,"15min":8.2330000000000005},"last":0.54200000000000004}]},{"osd":4,"last update":"Mon Oct 9 11:09:09 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.85799999999999998,"15min":1.0900000000000001},"min":{"1min":0.248,"5min":0.218,"15min":0.218},"max":{"1min":4.7489999999999997,"5min":7.4630000000000001,"15min":8.2940000000000005},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.91000000000000003,"5min":0.92800000000000005,"15min":1.26},"min":{"1min":0.35399999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.7619999999999996,"5min":9.2759999999999998,"15min":16.481000000000002},"last":0.44800000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:09:09 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.85199999999999998,"15min":1.2549999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.247},"max":{"1min":4.9470000000000001,"5min":6.9379999999999997,"15min":15.414999999999999},"last":0.623},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.095,"15min":1.228},"min":{"1min":0.34300000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.9980000000000002,"5min":13.045,"15min":15.369},"last":0.70799999999999996}]},{"osd":6,"last update":"Mon Oct 9 11:09:09 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.98799999999999999,"15min":1.242},"min":{"1min":0.39800000000000002,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":4.8079999999999998,"5min":5.1120000000000001,"15min":14.159000000000001},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.0049999999999999,"5min":0.96599999999999997,"15min":1.369},"min":{"1min":0.39000000000000001,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":3.1179999999999999,"5min":4.3659999999999997,"15min":16.538},"last":0.59699999999999998}]}]},{"osd":6,"up_from":44,"seq":188978561448,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1511564,"kb_used_data":10820,"kb_used_omap":0,"kb_used_meta":1500736,"kb_avail":92233588,"statfs":{"total":95995035648,"available":94447194112,"internally_reserved":0,"allocated":11079680,"data_stored":9240571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1536753664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":1.0900000000000001,"15min":1.9179999999999999},"min":{"1min":0.26900000000000002,"5min":0.223,"15min":0.217},"max":{"1min":0.72799999999999998,"5min":12.209,"15min":54.743000000000002},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.94099999999999995,"15min":1.8580000000000001},"min":{"1min":0.33700000000000002,"5min":0.247,"15min":0.214},"max":{"1min":0.753,"5min":8.4789999999999992,"15min":53.701000000000001},"last":0.66600000000000004}]},{"osd":1,"last update":"Mon Oct 9 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":1.2390000000000001,"15min":1.1739999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.23499999999999999},"max":{"1min":0.754,"5min":16.225999999999999,"15min":41.241999999999997},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.45900000000000002,"5min":1.7749999999999999,"15min":1.766},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.248},"max":{"1min":0.62,"5min":11.715,"15min":41.476999999999997},"last":0.56399999999999995}]},{"osd":2,"last update":"Mon Oct 9 11:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.867,"15min":1.9019999999999999},"min":{"1min":0.32200000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.2069999999999999,"5min":10.382999999999999,"15min":12.003},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.85599999999999998,"5min":1.2909999999999999,"15min":1.877},"min":{"1min":0.33800000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.6669999999999998,"5min":12.693,"15min":15.432},"last":0.68999999999999995}]},{"osd":3,"last update":"Mon Oct 9 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":2.0310000000000001,"15min":1.643},"min":{"1min":0.372,"5min":0.313,"15min":0.313},"max":{"1min":0.96999999999999997,"5min":28.948,"15min":28.948},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.51400000000000001,"5min":2.597,"15min":2.4910000000000001},"min":{"1min":0.36199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.65100000000000002,"5min":28.893000000000001,"15min":28.893000000000001},"last":0.63700000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.66900000000000004,"15min":0.998},"min":{"1min":0.28100000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.67100000000000004,"5min":6.6609999999999996,"15min":17.216999999999999},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.53700000000000003,"15min":0.83299999999999996},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.24399999999999999},"max":{"1min":0.80100000000000005,"5min":2.3610000000000002,"15min":10.025},"last":0.78400000000000003}]},{"osd":5,"last update":"Mon Oct 9 11:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.78300000000000003,"15min":1.228},"min":{"1min":0.34000000000000002,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":2.7000000000000002,"5min":3.7389999999999999,"15min":19.786999999999999},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.68999999999999995,"15min":0.96499999999999997},"min":{"1min":0.377,"5min":0.248,"15min":0.248},"max":{"1min":1.22,"5min":4.085,"15min":9.9469999999999992},"last":0.81599999999999995}]},{"osd":7,"last update":"Mon Oct 9 11:09:16 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.71899999999999997,"15min":1.238},"min":{"1min":0.32400000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.73999999999999999,"5min":3.7250000000000001,"15min":12.882},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.68500000000000005,"15min":0.91700000000000004},"min":{"1min":0.35399999999999998,"5min":0.32800000000000001,"15min":0.27500000000000002},"max":{"1min":0.84399999999999997,"5min":3.2839999999999998,"15min":11.557},"last":0.74399999999999999}]}]},{"osd":1,"up_from":388,"seq":1666447310963,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":505316,"kb_used_data":7836,"kb_used_omap":13,"kb_used_meta":497458,"kb_avail":93239836,"statfs":{"total":95995035648,"available":95477592064,"internally_reserved":0,"allocated":8024064,"data_stored":4896133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":509397703},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":2.347,"15min":2.3250000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.17699999999999999},"max":{"1min":1.9590000000000001,"5min":18.192,"15min":18.192},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.434,"5min":2.6749999999999998,"15min":2.0329999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.19900000000000001},"max":{"1min":1.2470000000000001,"5min":17.957999999999998,"15min":17.957999999999998},"last":0.309}]},{"osd":2,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":2.0939999999999999,"15min":2.73},"min":{"1min":0.29399999999999998,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":2.032,"5min":12.436999999999999,"15min":47.731000000000002},"last":0.371},{"interface":"front","average":{"1min":0.44400000000000001,"5min":2.5019999999999998,"15min":2.4689999999999999},"min":{"1min":0.28399999999999997,"5min":0.216,"15min":0.216},"max":{"1min":0.81200000000000006,"5min":16.870000000000001,"15min":47.786999999999999},"last":0.33400000000000002}]},{"osd":3,"last update":"Mon Oct 9 11:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":3.141,"15min":2.0390000000000001},"min":{"1min":0.254,"5min":0.251,"15min":0.251},"max":{"1min":0.86299999999999999,"5min":92.447999999999993,"15min":92.447999999999993},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.52800000000000002,"5min":3.2440000000000002,"15min":2.0699999999999998},"min":{"1min":0.252,"5min":0.217,"15min":0.217},"max":{"1min":2.0009999999999999,"5min":92.066999999999993,"15min":92.066999999999993},"last":0.38800000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.6020000000000001,"15min":2.1880000000000002},"min":{"1min":0.34699999999999998,"5min":0.27300000000000002,"15min":0.26000000000000001},"max":{"1min":2.0739999999999998,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.905,"15min":1.673},"min":{"1min":0.33300000000000002,"5min":0.314,"15min":0.29999999999999999},"max":{"1min":0.88,"5min":16.841000000000001,"15min":16.841000000000001},"last":0.67800000000000005}]},{"osd":5,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.6879999999999999,"15min":2.1549999999999998},"min":{"1min":0.40400000000000003,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":0.86899999999999999,"5min":20.689,"15min":20.689},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.66400000000000003,"5min":1.7549999999999999,"15min":2.1240000000000001},"min":{"1min":0.39900000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.133,"5min":11.65,"15min":11.65},"last":0.628}]},{"osd":6,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.7749999999999999,"15min":2.2480000000000002},"min":{"1min":0.44,"5min":0.312,"15min":0.309},"max":{"1min":0.89900000000000002,"5min":20.646999999999998,"15min":20.646999999999998},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.766,"15min":2.052},"min":{"1min":0.375,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.1040000000000001,"5min":11.75,"15min":11.75},"last":0.61199999999999999}]},{"osd":7,"last update":"Mon Oct 9 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.9330000000000001,"15min":1.6830000000000001},"min":{"1min":0.36199999999999999,"5min":0.33000000000000002,"15min":0.32900000000000001},"max":{"1min":0.78300000000000003,"5min":16.797999999999998,"15min":16.797999999999998},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.766,"15min":2.1589999999999998},"min":{"1min":0.42699999999999999,"5min":0.36899999999999999,"15min":0.33200000000000002},"max":{"1min":0.91900000000000004,"5min":20.716000000000001,"15min":20.716000000000001},"last":0.58399999999999996}]}]},{"osd":0,"up_from":80,"seq":343597384021,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":986412,"kb_used_data":8228,"kb_used_omap":28,"kb_used_meta":978147,"kb_avail":92758740,"statfs":{"total":95995035648,"available":94984949760,"internally_reserved":0,"allocated":8425472,"data_stored":5220099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1001623376},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":1.8700000000000001,"15min":2.5489999999999999},"min":{"1min":0.23300000000000001,"5min":0.186,"15min":0.186},"max":{"1min":1.6220000000000001,"5min":14.644,"15min":19.608000000000001},"last":0.503},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.96,"15min":2.613},"min":{"1min":0.23400000000000001,"5min":0.214,"15min":0.214},"max":{"1min":3.7240000000000002,"5min":15.834,"15min":19.992999999999999},"last":0.53800000000000003}]},{"osd":2,"last update":"Mon Oct 9 11:08:57 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":2.0579999999999998,"15min":2.2959999999999998},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":7.9329999999999998,"5min":12.459,"15min":24.852},"last":0.42299999999999999},{"interface":"front","average":{"1min":6.4219999999999997,"5min":3.0819999999999999,"15min":3.0009999999999999},"min":{"1min":0.21299999999999999,"5min":0.20399999999999999,"15min":0.19800000000000001},"max":{"1min":99.679000000000002,"5min":99.679000000000002,"15min":99.679000000000002},"last":0.47299999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:09:25 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":2.52,"15min":3.194},"min":{"1min":0.28499999999999998,"5min":0.22,"15min":0.22},"max":{"1min":8.0180000000000007,"5min":25.050000000000001,"15min":25.050000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.51100000000000001,"5min":2.077,"15min":1.958},"min":{"1min":0.314,"5min":0.254,"15min":0.254},"max":{"1min":2.0419999999999998,"5min":14.125999999999999,"15min":14.125999999999999},"last":0.58599999999999997}]},{"osd":4,"last update":"Mon Oct 9 11:09:25 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.905,"15min":2.4540000000000002},"min":{"1min":0.312,"5min":0.27100000000000002,"15min":0.25700000000000001},"max":{"1min":7.9829999999999997,"5min":14.507,"15min":77.177999999999997},"last":0.628},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.3660000000000001,"15min":1.6120000000000001},"min":{"1min":0.34100000000000003,"5min":0.33600000000000002,"15min":0.29199999999999998},"max":{"1min":2.1160000000000001,"5min":7.9249999999999998,"15min":10.513999999999999},"last":0.66100000000000003}]},{"osd":5,"last update":"Mon Oct 9 11:09:25 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.361,"15min":1.7909999999999999},"min":{"1min":0.36699999999999999,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":2.1440000000000001,"5min":7.9550000000000001,"15min":10.544},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.629,"5min":1.516,"15min":1.7829999999999999},"min":{"1min":0.439,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":1.7010000000000001,"5min":11.212999999999999,"15min":24.754999999999999},"last":0.64400000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:09:25 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":1.4450000000000001,"15min":1.8740000000000001},"min":{"1min":0.38600000000000001,"5min":0.34699999999999998,"15min":0.314},"max":{"1min":2.1859999999999999,"5min":7.9829999999999997,"15min":18.920000000000002},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.63,"5min":1.542,"15min":1.899},"min":{"1min":0.41699999999999998,"5min":0.378,"15min":0.31900000000000001},"max":{"1min":1.728,"5min":11.257999999999999,"15min":24.800000000000001},"last":0.68899999999999995}]},{"osd":7,"last update":"Mon Oct 9 11:09:25 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":1.9219999999999999,"15min":2.0049999999999999},"min":{"1min":0.39300000000000002,"5min":0.35099999999999998,"15min":0.32700000000000001},"max":{"1min":1.6679999999999999,"5min":17.672000000000001,"15min":17.672000000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":2.0779999999999998,"15min":2.573},"min":{"1min":0.41299999999999998,"5min":0.311,"15min":0.311},"max":{"1min":7.9390000000000001,"5min":17.539000000000001,"15min":77.228999999999999},"last":0.67300000000000004}]}]},{"osd":2,"up_from":90,"seq":386547056961,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":769904,"kb_used_data":12072,"kb_used_omap":81,"kb_used_meta":757742,"kb_avail":92975248,"statfs":{"total":95995035648,"available":95206653952,"internally_reserved":0,"allocated":12361728,"data_stored":9333126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":775928026},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.363,"5min":1.9750000000000001,"15min":2.052},"min":{"1min":0.191,"5min":0.182,"15min":0.17499999999999999},"max":{"1min":8.3710000000000004,"5min":15.609,"15min":48.933999999999997},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.998,"15min":1.851},"min":{"1min":0.24199999999999999,"5min":0.23100000000000001,"15min":0.19600000000000001},"max":{"1min":7.7859999999999996,"5min":9.8010000000000002,"15min":14.515000000000001},"last":0.32800000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:09:36 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.897,"15min":2.3940000000000001},"min":{"1min":0.26900000000000002,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":1.1060000000000001,"5min":24.902000000000001,"15min":49.033999999999999},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.437,"5min":2.1600000000000001,"15min":2.5720000000000001},"min":{"1min":0.255,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":1.2210000000000001,"5min":25.850999999999999,"15min":48.874000000000002},"last":0.44800000000000001}]},{"osd":3,"last update":"Mon Oct 9 11:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":2.8719999999999999,"15min":2.1339999999999999},"min":{"1min":0.27900000000000003,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":14.801,"5min":25.890999999999998,"15min":25.890999999999998},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.2609999999999999,"5min":2.7189999999999999,"15min":2.3359999999999999},"min":{"1min":0.29399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":11.724,"5min":28.689,"15min":28.689},"last":0.47499999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:09:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.575,"15min":1.8720000000000001},"min":{"1min":0.34699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.6949999999999998,"5min":11.391999999999999,"15min":15.672000000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.6200000000000001,"15min":1.8959999999999999},"min":{"1min":0.33900000000000002,"5min":0.32900000000000001,"15min":0.249},"max":{"1min":8.1609999999999996,"5min":15.571,"15min":48.795000000000002},"last":0.67800000000000005}]},{"osd":5,"last update":"Mon Oct 9 11:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.6659999999999999,"15min":1.5900000000000001},"min":{"1min":0.35699999999999998,"5min":0.32100000000000001,"15min":0.25900000000000001},"max":{"1min":7.7279999999999998,"5min":8.5310000000000006,"15min":12.128},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.84399999999999997,"5min":1.5509999999999999,"15min":2.0030000000000001},"min":{"1min":0.39900000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.7189999999999999,"5min":11.441000000000001,"15min":49.061},"last":0.70899999999999996}]},{"osd":6,"last update":"Mon Oct 9 11:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5940000000000001,"5min":1.661,"15min":1.841},"min":{"1min":0.33000000000000002,"5min":0.313,"15min":0.313},"max":{"1min":8.1289999999999996,"5min":15.536,"15min":48.845999999999997},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.383,"5min":1.897,"15min":1.6759999999999999},"min":{"1min":0.38,"5min":0.36799999999999999,"15min":0.29199999999999998},"max":{"1min":7.8129999999999997,"5min":10.157999999999999,"15min":10.157999999999999},"last":0.69499999999999995}]},{"osd":7,"last update":"Mon Oct 9 11:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":1.8660000000000001,"15min":2.0009999999999999},"min":{"1min":0.39700000000000002,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":8.343,"5min":15.59,"15min":48.966000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.3220000000000001,"5min":1.8720000000000001,"15min":1.7849999999999999},"min":{"1min":0.36099999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.8390000000000004,"5min":8.5549999999999997,"15min":14.94},"last":0.73999999999999999}]}]},{"osd":3,"up_from":469,"seq":2014339661891,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":343452,"kb_used_data":8404,"kb_used_omap":11,"kb_used_meta":335028,"kb_avail":93401700,"statfs":{"total":95995035648,"available":95643340800,"internally_reserved":0,"allocated":8605696,"data_stored":5241492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11309,"internal_metadata":343069651},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":1.915,"15min":1.7849999999999999},"min":{"1min":0.22900000000000001,"5min":0.183,"15min":0.183},"max":{"1min":1.0740000000000001,"5min":14.882,"15min":14.882},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.337,"15min":1.024},"min":{"1min":0.19900000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":2.5939999999999999,"5min":9.4030000000000005,"15min":9.4030000000000005},"last":0.41399999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":2.0710000000000002,"15min":1.625},"min":{"1min":0.22,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":4.1799999999999997,"5min":17.93,"15min":17.93},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.4590000000000001,"5min":2.2210000000000001,"15min":1.7969999999999999},"min":{"1min":0.28100000000000003,"5min":0.224,"15min":0.224},"max":{"1min":13.236000000000001,"5min":19.163,"15min":19.163},"last":0.46899999999999997}]},{"osd":2,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":2.0310000000000001,"15min":1.401},"min":{"1min":0.27400000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":4.0270000000000001,"5min":38.840000000000003,"15min":38.840000000000003},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.69999999999999996,"5min":2.1360000000000001,"15min":1.4319999999999999},"min":{"1min":0.27700000000000002,"5min":0.219,"15min":0.219},"max":{"1min":4.016,"5min":37.991999999999997,"15min":37.991999999999997},"last":0.45300000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.46,"15min":1.5349999999999999},"min":{"1min":0.34000000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.131,"5min":14.907999999999999,"15min":14.907999999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.123,"15min":0.83799999999999997},"min":{"1min":0.307,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.0739999999999998,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.501}]},{"osd":5,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.2729999999999999,"15min":1.3660000000000001},"min":{"1min":0.41699999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.4450000000000001,"5min":9.3300000000000001,"15min":9.3300000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.3999999999999999,"15min":1.427},"min":{"1min":0.32000000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.101,"5min":10.029,"15min":10.029},"last":0.65000000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":1.4410000000000001,"15min":1.669},"min":{"1min":0.44900000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.2120000000000002,"5min":15.455,"15min":15.455},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.516,"15min":1.6000000000000001},"min":{"1min":0.39900000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.865,"5min":14.837,"15min":14.837},"last":0.67000000000000004}]},{"osd":7,"last update":"Mon Oct 9 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":1.3240000000000001,"15min":1.1080000000000001},"min":{"1min":0.39500000000000002,"5min":0.315,"15min":0.315},"max":{"1min":3.8029999999999999,"5min":6.907,"15min":6.907},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.86299999999999999,"5min":1.536,"15min":1.2829999999999999},"min":{"1min":0.42299999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.8370000000000002,"5min":6.9119999999999999,"15min":6.9119999999999999},"last":0.64000000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019311,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903252,"kb_used_data":6412,"kb_used_omap":0,"kb_used_meta":896832,"kb_avail":92841900,"statfs":{"total":95995035648,"available":95070105600,"internally_reserved":0,"allocated":6565888,"data_stored":4718967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918355968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:19 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":1.494,"15min":2.4129999999999998},"min":{"1min":0.26900000000000002,"5min":0.223,"15min":0.20300000000000001},"max":{"1min":3.1259999999999999,"5min":13.162000000000001,"15min":213.88999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.329,"15min":2.5},"min":{"1min":0.30099999999999999,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":2.399,"5min":9.0670000000000002,"15min":214.09800000000001},"last":0.58499999999999996}]},{"osd":1,"last update":"Mon Oct 9 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":2.331,"15min":2.5350000000000001},"min":{"1min":0.28299999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.68200000000000005,"5min":57.966999999999999,"15min":121.41},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.65800000000000003,"5min":1.9810000000000001,"15min":2.081},"min":{"1min":0.28199999999999997,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":3.589,"5min":57.832000000000001,"15min":122.21899999999999},"last":0.55300000000000005}]},{"osd":2,"last update":"Mon Oct 9 11:09:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.748,"15min":1.738},"min":{"1min":0.30499999999999999,"5min":0.246,"15min":0.23799999999999999},"max":{"1min":3.0779999999999998,"5min":18.407,"15min":18.407},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.7430000000000001,"15min":2.0739999999999998},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":2.5129999999999999,"5min":8.4250000000000007,"15min":27.818000000000001},"last":0.48799999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":2.2629999999999999,"15min":1.921},"min":{"1min":0.35599999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.1189999999999998,"5min":60.753,"15min":60.753},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.0329999999999999,"5min":2.1789999999999998,"15min":1.5049999999999999},"min":{"1min":0.39800000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.5199999999999996,"5min":61.473999999999997,"15min":61.473999999999997},"last":0.52300000000000002}]},{"osd":5,"last update":"Mon Oct 9 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.89600000000000002,"15min":1.337},"min":{"1min":0.30299999999999999,"5min":0.27400000000000002,"15min":0.224},"max":{"1min":0.68200000000000005,"5min":4.9409999999999998,"15min":19.068999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.499,"5min":0.83799999999999997,"15min":1.22},"min":{"1min":0.29899999999999999,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":1.052,"5min":8.3230000000000004,"15min":15.535},"last":0.67600000000000005}]},{"osd":6,"last update":"Mon Oct 9 11:09:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.2969999999999999,"15min":1.1910000000000001},"min":{"1min":0.29299999999999998,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":6.8689999999999998,"5min":8.3930000000000007,"15min":15.739000000000001},"last":0.62},{"interface":"front","average":{"1min":1.3500000000000001,"5min":1.4179999999999999,"15min":1.4730000000000001},"min":{"1min":0.33700000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.8170000000000002,"5min":12.795999999999999,"15min":14.093999999999999},"last":0.65500000000000003}]},{"osd":7,"last update":"Mon Oct 9 11:09:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.294,"15min":1.5629999999999999},"min":{"1min":0.374,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.181,"5min":10.15,"15min":19.923999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.155,"5min":1.091,"15min":1.488},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.0439999999999996,"5min":10.853,"15min":19.859999999999999},"last":0.70799999999999996}]}]},{"osd":5,"up_from":37,"seq":158913790380,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1205960,"kb_used_data":6656,"kb_used_omap":0,"kb_used_meta":1199296,"kb_avail":92539192,"statfs":{"total":95995035648,"available":94760132608,"internally_reserved":0,"allocated":6815744,"data_stored":4977095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1228079104},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":1.321,"15min":2.2389999999999999},"min":{"1min":0.23000000000000001,"5min":0.219,"15min":0.21299999999999999},"max":{"1min":0.77900000000000003,"5min":6.2249999999999996,"15min":58.271999999999998},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":1.3740000000000001,"15min":2.2890000000000001},"min":{"1min":0.38200000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.752,"5min":6.2649999999999997,"15min":58.25},"last":0.50600000000000001}]},{"osd":1,"last update":"Mon Oct 9 11:09:32 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":1.5569999999999999,"15min":1.964},"min":{"1min":0.26100000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.64800000000000002,"5min":16.962,"15min":32.491999999999997},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":1.675,"15min":1.845},"min":{"1min":0.34200000000000003,"5min":0.30499999999999999,"15min":0.247},"max":{"1min":0.80800000000000005,"5min":17.033999999999999,"15min":32.537999999999997},"last":0.69799999999999995}]},{"osd":2,"last update":"Mon Oct 9 11:08:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.321,"15min":1.8240000000000001},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.221},"max":{"1min":7.3109999999999999,"5min":7.3109999999999999,"15min":11.257999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.8919999999999999,"15min":1.988},"min":{"1min":0.317,"5min":0.253,"15min":0.253},"max":{"1min":4.085,"5min":11.673,"15min":14.621},"last":0.58399999999999996}]},{"osd":3,"last update":"Mon Oct 9 11:09:19 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":1.607,"15min":1.5289999999999999},"min":{"1min":0.35899999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.7880000000000003,"5min":15.535,"15min":15.535},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.9059999999999999,"15min":1.841},"min":{"1min":0.40699999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.8090000000000002,"5min":15.757999999999999,"15min":15.757999999999999},"last":0.47299999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:09:32 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.80900000000000005,"15min":0.90400000000000003},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.216},"max":{"1min":0.65900000000000003,"5min":7.5640000000000001,"15min":12.042},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.92800000000000005,"15min":1.151},"min":{"1min":0.32000000000000001,"5min":0.245,"15min":0.245},"max":{"1min":1.0840000000000001,"5min":6.8639999999999999,"15min":9.2490000000000006},"last":0.55600000000000005}]},{"osd":6,"last update":"Mon Oct 9 11:08:45 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.2090000000000001,"15min":1.2450000000000001},"min":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.27100000000000002},"max":{"1min":5.944,"5min":7.1020000000000003,"15min":9.1969999999999992},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.214,"5min":1.204,"15min":1.0700000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29099999999999998},"max":{"1min":9.298,"5min":11.983000000000001,"15min":11.983000000000001},"last":0.81599999999999995}]},{"osd":7,"last update":"Mon Oct 9 11:09:09 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.026,"15min":1.298},"min":{"1min":0.35399999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.9699999999999998,"5min":5.9699999999999998,"15min":9.3539999999999992},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.94199999999999995,"15min":1.1739999999999999},"min":{"1min":0.38700000000000001,"5min":0.30499999999999999,"15min":0.30099999999999999},"max":{"1min":2.2549999999999999,"5min":8.0009999999999994,"15min":11.726000000000001},"last":0.746}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3733,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:09:45.654 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T11:09:45.654 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 91 2023-10-09T11:09:45.681 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:45 smithi125 ceph-mon[195507]: pgmap v1455: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:45.682 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:45.682 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:09:45.682 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:45 smithi125 ceph-mon[195507]: from='client.47612 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:09:45.682 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:45 smithi125 ceph-mon[200197]: pgmap v1455: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:45.682 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:45.682 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:09:45.683 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:45 smithi125 ceph-mon[200197]: from='client.47612 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:09:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:45 smithi155 ceph-mon[184854]: pgmap v1455: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:09:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:45 smithi155 ceph-mon[184854]: from='client.47612 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:09:45.749 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T11:09:45.750 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T11:09:46.087 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:46.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:46 smithi125 ceph-mon[195507]: pgmap v1456: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:46.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:46 smithi125 ceph-mon[200197]: pgmap v1456: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:46 smithi155 ceph-mon[184854]: pgmap v1456: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:47.263 INFO:teuthology.orchestra.run.smithi125.stderr:Error EINVAL: specified pg_num 91 < pg_num_max 32 2023-10-09T11:09:47.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2696420410' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "91"}]: dispatch 2023-10-09T11:09:47.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:47 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "91"}]: dispatch 2023-10-09T11:09:47.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2696420410' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "91"}]: dispatch 2023-10-09T11:09:47.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:47 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "91"}]: dispatch 2023-10-09T11:09:47.683 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T11:09:47.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:47 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2696420410' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "91"}]: dispatch 2023-10-09T11:09:47.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:47 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "91"}]: dispatch 2023-10-09T11:09:49.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:49 smithi155 ceph-mon[184854]: pgmap v1457: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:49.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:49 smithi125 ceph-mon[195507]: pgmap v1457: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:49.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:49 smithi125 ceph-mon[200197]: pgmap v1457: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:51.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:51 smithi125 ceph-mon[195507]: pgmap v1458: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:51.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:51 smithi125 ceph-mon[200197]: pgmap v1458: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:51.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:09:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:09:51] "GET /metrics HTTP/1.1" 200 34165 "" "Prometheus/2.43.0" 2023-10-09T11:09:51.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:51 smithi155 ceph-mon[184854]: pgmap v1458: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:52.684 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T11:09:52.684 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 91 2023-10-09T11:09:52.719 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:52 smithi125 ceph-mon[195507]: pgmap v1459: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:52.719 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:52 smithi125 ceph-mon[200197]: pgmap v1459: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:52.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:52 smithi155 ceph-mon[184854]: pgmap v1459: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:52.969 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:54.162 INFO:teuthology.orchestra.run.smithi125.stderr:Error EINVAL: specified pgp_num 91 > pg_num 31 2023-10-09T11:09:54.559 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T11:09:54.560 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-10-09T11:09:54.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:54 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3277025092' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "91"}]: dispatch 2023-10-09T11:09:54.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:54 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3277025092' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "91"}]: dispatch 2023-10-09T11:09:54.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:54 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3277025092' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "91"}]: dispatch 2023-10-09T11:09:54.867 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:55.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:55 smithi125 ceph-mon[195507]: pgmap v1460: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:55.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:55 smithi125 ceph-mon[200197]: pgmap v1460: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:55.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:55 smithi155 ceph-mon[184854]: pgmap v1460: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:55.864 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a: {} 2023-10-09T11:09:55.874 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b: {} 2023-10-09T11:09:55.883 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c: {} 2023-10-09T11:09:55.884 INFO:teuthology.orchestra.run.smithi125.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T11:09:55.884 INFO:teuthology.orchestra.run.smithi125.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T11:09:55.884 INFO:teuthology.orchestra.run.smithi125.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T11:09:56.257 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T11:09:56.523 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:56.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:56 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4203588694' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T11:09:56.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:56 smithi125 ceph-mon[195507]: pgmap v1461: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:56 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4203588694' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T11:09:56.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:56 smithi125 ceph-mon[200197]: pgmap v1461: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:56 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4203588694' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T11:09:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:56 smithi155 ceph-mon[184854]: pgmap v1461: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:57.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:57 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:09:57.581 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:57 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:09:57.581 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:57 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T11:09:58.442 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T11:09:58.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:58 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4177159908' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:09:58.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:58 smithi125 ceph-mon[195507]: pgmap v1462: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:58.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:58 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4177159908' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:09:58.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:58 smithi125 ceph-mon[200197]: pgmap v1462: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:58.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:58 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4177159908' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:09:58.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:58 smithi155 ceph-mon[184854]: pgmap v1462: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:09:58.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-10-09T11:09:59.252 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:09:59.535 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:59 smithi125 ceph-mon[195507]: failed to encode map e600 with expected crc 2023-10-09T11:09:59.535 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:59 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4177159908' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T11:09:59.535 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:59 smithi125 ceph-mon[195507]: osdmap e600: 8 total, 8 up, 8 in 2023-10-09T11:09:59.535 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:59 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:59.535 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:59 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:59.535 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:59 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:59.536 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:09:59 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:59.536 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:59 smithi125 ceph-mon[200197]: failed to encode map e600 with expected crc 2023-10-09T11:09:59.536 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:59 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4177159908' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T11:09:59.536 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:59 smithi125 ceph-mon[200197]: osdmap e600: 8 total, 8 up, 8 in 2023-10-09T11:09:59.536 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:59 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:59.536 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:59 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:59.536 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:59 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:59.536 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:09:59 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:59.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:59 smithi155 ceph-mon[184854]: failed to encode map e600 with expected crc 2023-10-09T11:09:59.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:59 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4177159908' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T11:09:59.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:59 smithi155 ceph-mon[184854]: osdmap e600: 8 total, 8 up, 8 in 2023-10-09T11:09:59.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:59 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:59.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:59 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:59.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:59 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:09:59.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:09:59 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:10:00.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:10:00.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:10:00.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:10:00.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:00 smithi125 ceph-mon[200197]: failed to encode map e600 with expected crc 2023-10-09T11:10:00.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:00 smithi125 ceph-mon[200197]: pgmap v1464: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:00.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:00 smithi125 ceph-mon[200197]: overall HEALTH_OK 2023-10-09T11:10:00.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:00 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:10:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:10:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:10:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:10:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:00 smithi125 ceph-mon[195507]: failed to encode map e600 with expected crc 2023-10-09T11:10:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:00 smithi125 ceph-mon[195507]: pgmap v1464: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:00 smithi125 ceph-mon[195507]: overall HEALTH_OK 2023-10-09T11:10:00.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:00 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:10:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T11:10:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T11:10:00.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:10:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:00 smithi155 ceph-mon[184854]: failed to encode map e600 with expected crc 2023-10-09T11:10:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:00 smithi155 ceph-mon[184854]: pgmap v1464: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:00 smithi155 ceph-mon[184854]: overall HEALTH_OK 2023-10-09T11:10:00.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:00 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:10:01.461 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-10-09T11:10:01.489 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:01 smithi125 ceph-mon[195507]: failed to encode map e600 with expected crc 2023-10-09T11:10:01.490 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:01 smithi125 ceph-mon[195507]: failed to encode map e600 with expected crc 2023-10-09T11:10:01.490 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:01 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2027357457' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:10:01.490 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:01 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:10:01.490 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:01 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:10:01] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.43.0" 2023-10-09T11:10:01.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:01 smithi125 ceph-mon[200197]: failed to encode map e600 with expected crc 2023-10-09T11:10:01.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:01 smithi125 ceph-mon[200197]: failed to encode map e600 with expected crc 2023-10-09T11:10:01.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:01 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2027357457' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:10:01.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:01 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:10:01.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:01 smithi155 ceph-mon[184854]: failed to encode map e600 with expected crc 2023-10-09T11:10:01.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:01 smithi155 ceph-mon[184854]: failed to encode map e600 with expected crc 2023-10-09T11:10:01.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:01 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2027357457' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:10:01.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:01 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T11:10:01.852 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T11:10:02.125 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:02.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:02 smithi125 ceph-mon[195507]: failed to encode map e601 with expected crc 2023-10-09T11:10:02.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:02 smithi125 ceph-mon[195507]: failed to encode map e601 with expected crc 2023-10-09T11:10:02.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:02 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T11:10:02.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:02 smithi125 ceph-mon[195507]: osdmap e601: 8 total, 8 up, 8 in 2023-10-09T11:10:02.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:02 smithi125 ceph-mon[195507]: failed to encode map e601 with expected crc 2023-10-09T11:10:02.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:02 smithi125 ceph-mon[195507]: failed to encode map e601 with expected crc 2023-10-09T11:10:02.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:02 smithi125 ceph-mon[195507]: pgmap v1466: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:02 smithi125 ceph-mon[200197]: failed to encode map e601 with expected crc 2023-10-09T11:10:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:02 smithi125 ceph-mon[200197]: failed to encode map e601 with expected crc 2023-10-09T11:10:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:02 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T11:10:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:02 smithi125 ceph-mon[200197]: osdmap e601: 8 total, 8 up, 8 in 2023-10-09T11:10:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:02 smithi125 ceph-mon[200197]: failed to encode map e601 with expected crc 2023-10-09T11:10:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:02 smithi125 ceph-mon[200197]: failed to encode map e601 with expected crc 2023-10-09T11:10:02.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:02 smithi125 ceph-mon[200197]: pgmap v1466: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:02 smithi155 ceph-mon[184854]: failed to encode map e601 with expected crc 2023-10-09T11:10:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:02 smithi155 ceph-mon[184854]: failed to encode map e601 with expected crc 2023-10-09T11:10:02.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:02 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T11:10:02.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:02 smithi155 ceph-mon[184854]: osdmap e601: 8 total, 8 up, 8 in 2023-10-09T11:10:02.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:02 smithi155 ceph-mon[184854]: failed to encode map e601 with expected crc 2023-10-09T11:10:02.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:02 smithi155 ceph-mon[184854]: failed to encode map e601 with expected crc 2023-10-09T11:10:02.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:02 smithi155 ceph-mon[184854]: pgmap v1466: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:03.470 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T11:10:03.780 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:03 smithi125 ceph-mon[195507]: osdmap e602: 8 total, 8 up, 8 in 2023-10-09T11:10:03.780 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:03 smithi125 ceph-mon[195507]: failed to encode map e602 with expected crc 2023-10-09T11:10:03.780 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:03 smithi125 ceph-mon[195507]: failed to encode map e602 with expected crc 2023-10-09T11:10:03.780 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:03 smithi125 ceph-mon[195507]: failed to encode map e602 with expected crc 2023-10-09T11:10:03.780 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:03 smithi125 ceph-mon[195507]: failed to encode map e602 with expected crc 2023-10-09T11:10:03.780 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:03 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4257470383' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:10:03.781 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:03 smithi125 ceph-mon[200197]: osdmap e602: 8 total, 8 up, 8 in 2023-10-09T11:10:03.781 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:03 smithi125 ceph-mon[200197]: failed to encode map e602 with expected crc 2023-10-09T11:10:03.781 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:03 smithi125 ceph-mon[200197]: failed to encode map e602 with expected crc 2023-10-09T11:10:03.781 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:03 smithi125 ceph-mon[200197]: failed to encode map e602 with expected crc 2023-10-09T11:10:03.781 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:03 smithi125 ceph-mon[200197]: failed to encode map e602 with expected crc 2023-10-09T11:10:03.782 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:03 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4257470383' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:10:03.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-09T11:10:03.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:03 smithi155 ceph-mon[184854]: osdmap e602: 8 total, 8 up, 8 in 2023-10-09T11:10:03.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:03 smithi155 ceph-mon[184854]: failed to encode map e602 with expected crc 2023-10-09T11:10:03.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:03 smithi155 ceph-mon[184854]: failed to encode map e602 with expected crc 2023-10-09T11:10:03.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:03 smithi155 ceph-mon[184854]: failed to encode map e602 with expected crc 2023-10-09T11:10:03.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:03 smithi155 ceph-mon[184854]: failed to encode map e602 with expected crc 2023-10-09T11:10:03.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:03 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4257470383' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:10:04.122 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:04.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:04 smithi125 ceph-mon[200197]: failed to encode map e603 with expected crc 2023-10-09T11:10:04.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:04 smithi125 ceph-mon[200197]: failed to encode map e603 with expected crc 2023-10-09T11:10:04.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:04 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4257470383' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T11:10:04.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:04 smithi125 ceph-mon[200197]: osdmap e603: 8 total, 8 up, 8 in 2023-10-09T11:10:04.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:04 smithi125 ceph-mon[200197]: pgmap v1469: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 301 B/s, 1 objects/s recovering 2023-10-09T11:10:04.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:04 smithi125 ceph-mon[200197]: failed to encode map e603 with expected crc 2023-10-09T11:10:04.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:04 smithi125 ceph-mon[200197]: failed to encode map e603 with expected crc 2023-10-09T11:10:04.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:04 smithi125 ceph-mon[195507]: failed to encode map e603 with expected crc 2023-10-09T11:10:04.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:04 smithi125 ceph-mon[195507]: failed to encode map e603 with expected crc 2023-10-09T11:10:04.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:04 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4257470383' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T11:10:04.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:04 smithi125 ceph-mon[195507]: osdmap e603: 8 total, 8 up, 8 in 2023-10-09T11:10:04.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:04 smithi125 ceph-mon[195507]: pgmap v1469: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 301 B/s, 1 objects/s recovering 2023-10-09T11:10:04.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:04 smithi125 ceph-mon[195507]: failed to encode map e603 with expected crc 2023-10-09T11:10:04.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:04 smithi125 ceph-mon[195507]: failed to encode map e603 with expected crc 2023-10-09T11:10:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:04 smithi155 ceph-mon[184854]: failed to encode map e603 with expected crc 2023-10-09T11:10:04.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:04 smithi155 ceph-mon[184854]: failed to encode map e603 with expected crc 2023-10-09T11:10:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:04 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4257470383' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T11:10:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:04 smithi155 ceph-mon[184854]: osdmap e603: 8 total, 8 up, 8 in 2023-10-09T11:10:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:04 smithi155 ceph-mon[184854]: pgmap v1469: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 301 B/s, 1 objects/s recovering 2023-10-09T11:10:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:04 smithi155 ceph-mon[184854]: failed to encode map e603 with expected crc 2023-10-09T11:10:04.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:04 smithi155 ceph-mon[184854]: failed to encode map e603 with expected crc 2023-10-09T11:10:05.489 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-09T11:10:05.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:05 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2883117393' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:10:05.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:05 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:10:05.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:05 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2883117393' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:10:05.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:05 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:10:05.875 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T11:10:05.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:05 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2883117393' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:10:05.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:05 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T11:10:06.141 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:06.751 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:06 smithi125 ceph-mon[195507]: failed to encode map e604 with expected crc 2023-10-09T11:10:06.751 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:06 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T11:10:06.751 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:06 smithi125 ceph-mon[195507]: osdmap e604: 8 total, 8 up, 8 in 2023-10-09T11:10:06.751 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:06 smithi125 ceph-mon[195507]: pgmap v1471: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 369 B/s, 1 objects/s recovering 2023-10-09T11:10:06.751 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:06 smithi125 ceph-mon[195507]: failed to encode map e604 with expected crc 2023-10-09T11:10:06.751 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:06 smithi125 ceph-mon[200197]: failed to encode map e604 with expected crc 2023-10-09T11:10:06.751 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:06 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T11:10:06.752 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:06 smithi125 ceph-mon[200197]: osdmap e604: 8 total, 8 up, 8 in 2023-10-09T11:10:06.752 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:06 smithi125 ceph-mon[200197]: pgmap v1471: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 369 B/s, 1 objects/s recovering 2023-10-09T11:10:06.752 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:06 smithi125 ceph-mon[200197]: failed to encode map e604 with expected crc 2023-10-09T11:10:06.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:06 smithi155 ceph-mon[184854]: failed to encode map e604 with expected crc 2023-10-09T11:10:06.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:06 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T11:10:06.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:06 smithi155 ceph-mon[184854]: osdmap e604: 8 total, 8 up, 8 in 2023-10-09T11:10:06.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:06 smithi155 ceph-mon[184854]: pgmap v1471: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 369 B/s, 1 objects/s recovering 2023-10-09T11:10:06.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:06 smithi155 ceph-mon[184854]: failed to encode map e604 with expected crc 2023-10-09T11:10:07.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:07 smithi125 ceph-mon[195507]: failed to encode map e604 with expected crc 2023-10-09T11:10:07.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:07 smithi125 ceph-mon[195507]: failed to encode map e605 with expected crc 2023-10-09T11:10:07.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:07 smithi125 ceph-mon[195507]: failed to encode map e605 with expected crc 2023-10-09T11:10:07.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:07 smithi125 ceph-mon[195507]: osdmap e605: 8 total, 8 up, 8 in 2023-10-09T11:10:07.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:07 smithi125 ceph-mon[200197]: failed to encode map e604 with expected crc 2023-10-09T11:10:07.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:07 smithi125 ceph-mon[200197]: failed to encode map e605 with expected crc 2023-10-09T11:10:07.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:07 smithi125 ceph-mon[200197]: failed to encode map e605 with expected crc 2023-10-09T11:10:07.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:07 smithi125 ceph-mon[200197]: osdmap e605: 8 total, 8 up, 8 in 2023-10-09T11:10:07.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:07 smithi155 ceph-mon[184854]: failed to encode map e604 with expected crc 2023-10-09T11:10:07.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:07 smithi155 ceph-mon[184854]: failed to encode map e605 with expected crc 2023-10-09T11:10:07.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:07 smithi155 ceph-mon[184854]: failed to encode map e605 with expected crc 2023-10-09T11:10:07.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:07 smithi155 ceph-mon[184854]: osdmap e605: 8 total, 8 up, 8 in 2023-10-09T11:10:08.510 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T11:10:08.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:08 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4202465659' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:10:08.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:08 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:10:08.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:08 smithi125 ceph-mon[195507]: pgmap v1473: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 302 B/s, 1 objects/s recovering 2023-10-09T11:10:08.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:08 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4202465659' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:10:08.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:08 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:10:08.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:08 smithi125 ceph-mon[200197]: pgmap v1473: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 302 B/s, 1 objects/s recovering 2023-10-09T11:10:08.900 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-10-09T11:10:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:08 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4202465659' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:10:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:08 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:10:08.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:08 smithi155 ceph-mon[184854]: pgmap v1473: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 302 B/s, 1 objects/s recovering 2023-10-09T11:10:09.168 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:09.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:09 smithi125 ceph-mon[200197]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T11:10:09.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:09 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T11:10:09.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:09 smithi125 ceph-mon[200197]: osdmap e606: 8 total, 8 up, 8 in 2023-10-09T11:10:09.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:09 smithi125 ceph-mon[200197]: failed to encode map e606 with expected crc 2023-10-09T11:10:09.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:09 smithi125 ceph-mon[195507]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T11:10:09.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:09 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T11:10:09.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:09 smithi125 ceph-mon[195507]: osdmap e606: 8 total, 8 up, 8 in 2023-10-09T11:10:09.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:09 smithi125 ceph-mon[195507]: failed to encode map e606 with expected crc 2023-10-09T11:10:09.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:09 smithi155 ceph-mon[184854]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T11:10:09.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:09 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T11:10:09.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:09 smithi155 ceph-mon[184854]: osdmap e606: 8 total, 8 up, 8 in 2023-10-09T11:10:09.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:09 smithi155 ceph-mon[184854]: failed to encode map e606 with expected crc 2023-10-09T11:10:10.509 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-10-09T11:10:10.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:10 smithi125 ceph-mon[195507]: failed to encode map e606 with expected crc 2023-10-09T11:10:10.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:10 smithi125 ceph-mon[195507]: failed to encode map e606 with expected crc 2023-10-09T11:10:10.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:10 smithi125 ceph-mon[195507]: pgmap v1475: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:10.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:10 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/315747539' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:10:10.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:10 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:10:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:10 smithi125 ceph-mon[200197]: failed to encode map e606 with expected crc 2023-10-09T11:10:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:10 smithi125 ceph-mon[200197]: failed to encode map e606 with expected crc 2023-10-09T11:10:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:10 smithi125 ceph-mon[200197]: pgmap v1475: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:10 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/315747539' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:10:10.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:10 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:10:10.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T11:10:10.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:10 smithi155 ceph-mon[184854]: failed to encode map e606 with expected crc 2023-10-09T11:10:10.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:10 smithi155 ceph-mon[184854]: failed to encode map e606 with expected crc 2023-10-09T11:10:10.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:10 smithi155 ceph-mon[184854]: pgmap v1475: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:10.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:10 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/315747539' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:10:10.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:10 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T11:10:11.169 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:11.502 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:11 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:10:11] "GET /metrics HTTP/1.1" 200 34168 "" "Prometheus/2.43.0" 2023-10-09T11:10:11.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:11 smithi125 ceph-mon[195507]: failed to encode map e607 with expected crc 2023-10-09T11:10:11.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:11 smithi125 ceph-mon[195507]: failed to encode map e607 with expected crc 2023-10-09T11:10:11.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:11 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T11:10:11.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:11 smithi125 ceph-mon[195507]: osdmap e607: 8 total, 8 up, 8 in 2023-10-09T11:10:11.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:11 smithi125 ceph-mon[200197]: failed to encode map e607 with expected crc 2023-10-09T11:10:11.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:11 smithi125 ceph-mon[200197]: failed to encode map e607 with expected crc 2023-10-09T11:10:11.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:11 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T11:10:11.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:11 smithi125 ceph-mon[200197]: osdmap e607: 8 total, 8 up, 8 in 2023-10-09T11:10:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:11 smithi155 ceph-mon[184854]: failed to encode map e607 with expected crc 2023-10-09T11:10:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:11 smithi155 ceph-mon[184854]: failed to encode map e607 with expected crc 2023-10-09T11:10:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:11 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T11:10:11.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:11 smithi155 ceph-mon[184854]: osdmap e607: 8 total, 8 up, 8 in 2023-10-09T11:10:12.517 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T11:10:12.786 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[195507]: failed to encode map e608 with expected crc 2023-10-09T11:10:12.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[195507]: failed to encode map e608 with expected crc 2023-10-09T11:10:12.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[195507]: osdmap e608: 8 total, 8 up, 8 in 2023-10-09T11:10:12.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[195507]: failed to encode map e608 with expected crc 2023-10-09T11:10:12.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[195507]: failed to encode map e608 with expected crc 2023-10-09T11:10:12.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[195507]: pgmap v1478: 103 pgs: 18 peering, 85 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:12.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/420797215' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:10:12.787 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:10:12.787 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[200197]: failed to encode map e608 with expected crc 2023-10-09T11:10:12.787 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[200197]: failed to encode map e608 with expected crc 2023-10-09T11:10:12.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[200197]: osdmap e608: 8 total, 8 up, 8 in 2023-10-09T11:10:12.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[200197]: failed to encode map e608 with expected crc 2023-10-09T11:10:12.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[200197]: failed to encode map e608 with expected crc 2023-10-09T11:10:12.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[200197]: pgmap v1478: 103 pgs: 18 peering, 85 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:12.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/420797215' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:10:12.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:12 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:10:12.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T11:10:12.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:12 smithi155 ceph-mon[184854]: failed to encode map e608 with expected crc 2023-10-09T11:10:12.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:12 smithi155 ceph-mon[184854]: failed to encode map e608 with expected crc 2023-10-09T11:10:12.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:12 smithi155 ceph-mon[184854]: osdmap e608: 8 total, 8 up, 8 in 2023-10-09T11:10:12.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:12 smithi155 ceph-mon[184854]: failed to encode map e608 with expected crc 2023-10-09T11:10:12.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:12 smithi155 ceph-mon[184854]: failed to encode map e608 with expected crc 2023-10-09T11:10:12.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:12 smithi155 ceph-mon[184854]: pgmap v1478: 103 pgs: 18 peering, 85 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:12.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:12 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/420797215' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:10:12.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:12 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:10:13.181 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:13.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:13 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:10:13.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:13 smithi125 ceph-mon[195507]: osdmap e609: 8 total, 8 up, 8 in 2023-10-09T11:10:13.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:13 smithi125 ceph-mon[195507]: failed to encode map e609 with expected crc 2023-10-09T11:10:13.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:13 smithi125 ceph-mon[195507]: failed to encode map e609 with expected crc 2023-10-09T11:10:13.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:13 smithi125 ceph-mon[195507]: failed to encode map e609 with expected crc 2023-10-09T11:10:13.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:13 smithi125 ceph-mon[195507]: failed to encode map e609 with expected crc 2023-10-09T11:10:13.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:13 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:10:13.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:13 smithi125 ceph-mon[200197]: osdmap e609: 8 total, 8 up, 8 in 2023-10-09T11:10:13.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:13 smithi125 ceph-mon[200197]: failed to encode map e609 with expected crc 2023-10-09T11:10:13.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:13 smithi125 ceph-mon[200197]: failed to encode map e609 with expected crc 2023-10-09T11:10:13.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:13 smithi125 ceph-mon[200197]: failed to encode map e609 with expected crc 2023-10-09T11:10:13.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:13 smithi125 ceph-mon[200197]: failed to encode map e609 with expected crc 2023-10-09T11:10:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:13 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:10:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:13 smithi155 ceph-mon[184854]: osdmap e609: 8 total, 8 up, 8 in 2023-10-09T11:10:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:13 smithi155 ceph-mon[184854]: failed to encode map e609 with expected crc 2023-10-09T11:10:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:13 smithi155 ceph-mon[184854]: failed to encode map e609 with expected crc 2023-10-09T11:10:13.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:13 smithi155 ceph-mon[184854]: failed to encode map e609 with expected crc 2023-10-09T11:10:13.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:13 smithi155 ceph-mon[184854]: failed to encode map e609 with expected crc 2023-10-09T11:10:14.529 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T11:10:14.819 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:14 smithi125 ceph-mon[195507]: failed to encode map e609 with expected crc 2023-10-09T11:10:14.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:14 smithi125 ceph-mon[195507]: pgmap v1480: 103 pgs: 18 peering, 85 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:14.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:14 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3313569102' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:10:14.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:14 smithi125 ceph-mon[200197]: failed to encode map e609 with expected crc 2023-10-09T11:10:14.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:14 smithi125 ceph-mon[200197]: pgmap v1480: 103 pgs: 18 peering, 85 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:14.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:14 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3313569102' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:10:14.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T11:10:14.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:14 smithi155 ceph-mon[184854]: failed to encode map e609 with expected crc 2023-10-09T11:10:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:14 smithi155 ceph-mon[184854]: pgmap v1480: 103 pgs: 18 peering, 85 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:14.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:14 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3313569102' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T11:10:15.171 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[195507]: failed to encode map e610 with expected crc 2023-10-09T11:10:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3313569102' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:10:15.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[195507]: osdmap e610: 8 total, 8 up, 8 in 2023-10-09T11:10:15.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[195507]: failed to encode map e610 with expected crc 2023-10-09T11:10:15.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[195507]: failed to encode map e610 with expected crc 2023-10-09T11:10:15.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[195507]: failed to encode map e610 with expected crc 2023-10-09T11:10:15.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:10:15.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:10:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[200197]: failed to encode map e610 with expected crc 2023-10-09T11:10:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3313569102' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:10:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[200197]: osdmap e610: 8 total, 8 up, 8 in 2023-10-09T11:10:15.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[200197]: failed to encode map e610 with expected crc 2023-10-09T11:10:15.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[200197]: failed to encode map e610 with expected crc 2023-10-09T11:10:15.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[200197]: failed to encode map e610 with expected crc 2023-10-09T11:10:15.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:10:15.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:15 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:10:15.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:15 smithi155 ceph-mon[184854]: failed to encode map e610 with expected crc 2023-10-09T11:10:15.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:15 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3313569102' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T11:10:15.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:15 smithi155 ceph-mon[184854]: osdmap e610: 8 total, 8 up, 8 in 2023-10-09T11:10:15.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:15 smithi155 ceph-mon[184854]: failed to encode map e610 with expected crc 2023-10-09T11:10:15.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:15 smithi155 ceph-mon[184854]: failed to encode map e610 with expected crc 2023-10-09T11:10:15.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:15 smithi155 ceph-mon[184854]: failed to encode map e610 with expected crc 2023-10-09T11:10:15.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:10:15.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:15 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:10:16.534 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T11:10:16.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:16 smithi125 ceph-mon[195507]: pgmap v1482: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:16.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:16 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2148764271' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:10:16.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:16 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:10:16.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:16 smithi125 ceph-mon[200197]: pgmap v1482: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:16.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:16 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2148764271' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:10:16.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:16 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:10:16.929 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-09T11:10:16.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:16 smithi155 ceph-mon[184854]: pgmap v1482: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:16.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:16 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2148764271' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:10:16.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:16 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:10:17.192 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:17.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:17 smithi125 ceph-mon[195507]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 13 pgs inactive, 17 pgs peering) 2023-10-09T11:10:17.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:17 smithi125 ceph-mon[195507]: Cluster is now healthy 2023-10-09T11:10:17.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:17 smithi125 ceph-mon[195507]: failed to encode map e611 with expected crc 2023-10-09T11:10:17.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:17 smithi125 ceph-mon[195507]: failed to encode map e611 with expected crc 2023-10-09T11:10:17.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:17 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T11:10:17.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:17 smithi125 ceph-mon[195507]: osdmap e611: 8 total, 8 up, 8 in 2023-10-09T11:10:17.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:17 smithi125 ceph-mon[200197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 13 pgs inactive, 17 pgs peering) 2023-10-09T11:10:17.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:17 smithi125 ceph-mon[200197]: Cluster is now healthy 2023-10-09T11:10:17.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:17 smithi125 ceph-mon[200197]: failed to encode map e611 with expected crc 2023-10-09T11:10:17.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:17 smithi125 ceph-mon[200197]: failed to encode map e611 with expected crc 2023-10-09T11:10:17.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:17 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T11:10:17.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:17 smithi125 ceph-mon[200197]: osdmap e611: 8 total, 8 up, 8 in 2023-10-09T11:10:17.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:17 smithi155 ceph-mon[184854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 13 pgs inactive, 17 pgs peering) 2023-10-09T11:10:17.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:17 smithi155 ceph-mon[184854]: Cluster is now healthy 2023-10-09T11:10:17.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:17 smithi155 ceph-mon[184854]: failed to encode map e611 with expected crc 2023-10-09T11:10:17.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:17 smithi155 ceph-mon[184854]: failed to encode map e611 with expected crc 2023-10-09T11:10:17.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:17 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T11:10:17.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:17 smithi155 ceph-mon[184854]: osdmap e611: 8 total, 8 up, 8 in 2023-10-09T11:10:18.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:18 smithi125 ceph-mon[200197]: failed to encode map e611 with expected crc 2023-10-09T11:10:18.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:18 smithi125 ceph-mon[200197]: failed to encode map e611 with expected crc 2023-10-09T11:10:18.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:18 smithi125 ceph-mon[200197]: failed to encode map e611 with expected crc 2023-10-09T11:10:18.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:18 smithi125 ceph-mon[200197]: pgmap v1484: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:18.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:18 smithi125 ceph-mon[200197]: failed to encode map e611 with expected crc 2023-10-09T11:10:18.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:18 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1667769407' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:10:18.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:18 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:10:18.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:18 smithi125 ceph-mon[195507]: failed to encode map e611 with expected crc 2023-10-09T11:10:18.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:18 smithi125 ceph-mon[195507]: failed to encode map e611 with expected crc 2023-10-09T11:10:18.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:18 smithi125 ceph-mon[195507]: failed to encode map e611 with expected crc 2023-10-09T11:10:18.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:18 smithi125 ceph-mon[195507]: pgmap v1484: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:18.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:18 smithi125 ceph-mon[195507]: failed to encode map e611 with expected crc 2023-10-09T11:10:18.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:18 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1667769407' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:10:18.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:18 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:10:18.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:18 smithi155 ceph-mon[184854]: failed to encode map e611 with expected crc 2023-10-09T11:10:18.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:18 smithi155 ceph-mon[184854]: failed to encode map e611 with expected crc 2023-10-09T11:10:18.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:18 smithi155 ceph-mon[184854]: failed to encode map e611 with expected crc 2023-10-09T11:10:18.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:18 smithi155 ceph-mon[184854]: pgmap v1484: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:18.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:18 smithi155 ceph-mon[184854]: failed to encode map e611 with expected crc 2023-10-09T11:10:18.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:18 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1667769407' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:10:18.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:18 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T11:10:19.437 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-09T11:10:19.821 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T11:10:20.088 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:20.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:20 smithi125 ceph-mon[195507]: failed to encode map e612 with expected crc 2023-10-09T11:10:20.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:20 smithi125 ceph-mon[195507]: failed to encode map e612 with expected crc 2023-10-09T11:10:20.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:20 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T11:10:20.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:20 smithi125 ceph-mon[195507]: osdmap e612: 8 total, 8 up, 8 in 2023-10-09T11:10:20.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:20 smithi125 ceph-mon[195507]: failed to encode map e612 with expected crc 2023-10-09T11:10:20.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:20 smithi125 ceph-mon[195507]: pgmap v1486: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:20.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:20 smithi125 ceph-mon[200197]: failed to encode map e612 with expected crc 2023-10-09T11:10:20.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:20 smithi125 ceph-mon[200197]: failed to encode map e612 with expected crc 2023-10-09T11:10:20.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:20 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T11:10:20.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:20 smithi125 ceph-mon[200197]: osdmap e612: 8 total, 8 up, 8 in 2023-10-09T11:10:20.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:20 smithi125 ceph-mon[200197]: failed to encode map e612 with expected crc 2023-10-09T11:10:20.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:20 smithi125 ceph-mon[200197]: pgmap v1486: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:20 smithi155 ceph-mon[184854]: failed to encode map e612 with expected crc 2023-10-09T11:10:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:20 smithi155 ceph-mon[184854]: failed to encode map e612 with expected crc 2023-10-09T11:10:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:20 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T11:10:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:20 smithi155 ceph-mon[184854]: osdmap e612: 8 total, 8 up, 8 in 2023-10-09T11:10:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:20 smithi155 ceph-mon[184854]: failed to encode map e612 with expected crc 2023-10-09T11:10:20.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:20 smithi155 ceph-mon[184854]: pgmap v1486: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:21.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:21 smithi125 ceph-mon[195507]: failed to encode map e613 with expected crc 2023-10-09T11:10:21.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:21 smithi125 ceph-mon[195507]: failed to encode map e613 with expected crc 2023-10-09T11:10:21.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:21 smithi125 ceph-mon[195507]: failed to encode map e613 with expected crc 2023-10-09T11:10:21.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:21 smithi125 ceph-mon[195507]: osdmap e613: 8 total, 8 up, 8 in 2023-10-09T11:10:21.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:21 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:10:21.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:21 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3293619304' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:10:21.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:21 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:10:21.571 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:21 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:10:21] "GET /metrics HTTP/1.1" 200 34167 "" "Prometheus/2.43.0" 2023-10-09T11:10:21.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:21 smithi125 ceph-mon[200197]: failed to encode map e613 with expected crc 2023-10-09T11:10:21.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:21 smithi125 ceph-mon[200197]: failed to encode map e613 with expected crc 2023-10-09T11:10:21.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:21 smithi125 ceph-mon[200197]: failed to encode map e613 with expected crc 2023-10-09T11:10:21.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:21 smithi125 ceph-mon[200197]: osdmap e613: 8 total, 8 up, 8 in 2023-10-09T11:10:21.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:21 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:10:21.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:21 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3293619304' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:10:21.572 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:21 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:10:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:21 smithi155 ceph-mon[184854]: failed to encode map e613 with expected crc 2023-10-09T11:10:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:21 smithi155 ceph-mon[184854]: failed to encode map e613 with expected crc 2023-10-09T11:10:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:21 smithi155 ceph-mon[184854]: failed to encode map e613 with expected crc 2023-10-09T11:10:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:21 smithi155 ceph-mon[184854]: osdmap e613: 8 total, 8 up, 8 in 2023-10-09T11:10:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:21 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:10:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:21 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3293619304' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:10:21.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:21 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:10:22.077 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T11:10:22.471 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-10-09T11:10:22.739 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:23.156 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:23 smithi125 ceph-mon[195507]: pgmap v1488: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:23.156 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:23 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T11:10:23.156 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:23 smithi125 ceph-mon[195507]: osdmap e614: 8 total, 8 up, 8 in 2023-10-09T11:10:23.157 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:23 smithi125 ceph-mon[200197]: pgmap v1488: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:23.157 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:23 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T11:10:23.157 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:23 smithi125 ceph-mon[200197]: osdmap e614: 8 total, 8 up, 8 in 2023-10-09T11:10:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:23 smithi155 ceph-mon[184854]: pgmap v1488: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:23 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T11:10:23.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:23 smithi155 ceph-mon[184854]: osdmap e614: 8 total, 8 up, 8 in 2023-10-09T11:10:24.440 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-10-09T11:10:24.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[195507]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[195507]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[195507]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[195507]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[195507]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[195507]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.469 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/142339430' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:10:24.469 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:10:24.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[200197]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[200197]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[200197]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[200197]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[200197]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[200197]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/142339430' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:10:24.470 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:24 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:10:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:24 smithi155 ceph-mon[184854]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:24 smithi155 ceph-mon[184854]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:24 smithi155 ceph-mon[184854]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:24 smithi155 ceph-mon[184854]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:24 smithi155 ceph-mon[184854]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:24 smithi155 ceph-mon[184854]: failed to encode map e614 with expected crc 2023-10-09T11:10:24.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:24 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/142339430' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:10:24.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:24 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T11:10:24.778 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T11:10:25.045 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:25.380 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:25 smithi125 ceph-mon[195507]: pgmap v1490: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:25.380 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:25 smithi125 ceph-mon[195507]: failed to encode map e615 with expected crc 2023-10-09T11:10:25.380 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:25 smithi125 ceph-mon[195507]: failed to encode map e615 with expected crc 2023-10-09T11:10:25.380 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:25 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T11:10:25.380 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:25 smithi125 ceph-mon[195507]: osdmap e615: 8 total, 8 up, 8 in 2023-10-09T11:10:25.381 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:25 smithi125 ceph-mon[200197]: pgmap v1490: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:25.381 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:25 smithi125 ceph-mon[200197]: failed to encode map e615 with expected crc 2023-10-09T11:10:25.381 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:25 smithi125 ceph-mon[200197]: failed to encode map e615 with expected crc 2023-10-09T11:10:25.381 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:25 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T11:10:25.381 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:25 smithi125 ceph-mon[200197]: osdmap e615: 8 total, 8 up, 8 in 2023-10-09T11:10:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:25 smithi155 ceph-mon[184854]: pgmap v1490: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:25 smithi155 ceph-mon[184854]: failed to encode map e615 with expected crc 2023-10-09T11:10:25.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:25 smithi155 ceph-mon[184854]: failed to encode map e615 with expected crc 2023-10-09T11:10:25.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:25 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T11:10:25.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:25 smithi155 ceph-mon[184854]: osdmap e615: 8 total, 8 up, 8 in 2023-10-09T11:10:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:26 smithi155 ceph-mon[184854]: failed to encode map e615 with expected crc 2023-10-09T11:10:26.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:26 smithi155 ceph-mon[184854]: failed to encode map e615 with expected crc 2023-10-09T11:10:26.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:26 smithi125 ceph-mon[195507]: failed to encode map e615 with expected crc 2023-10-09T11:10:26.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:26 smithi125 ceph-mon[195507]: failed to encode map e615 with expected crc 2023-10-09T11:10:26.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:26 smithi125 ceph-mon[200197]: failed to encode map e615 with expected crc 2023-10-09T11:10:26.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:26 smithi125 ceph-mon[200197]: failed to encode map e615 with expected crc 2023-10-09T11:10:27.088 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T11:10:27.419 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:27 smithi125 ceph-mon[195507]: pgmap v1492: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:27.419 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:27 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4265517774' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:10:27.420 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:27 smithi125 ceph-mon[200197]: pgmap v1492: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:27.420 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:27 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4265517774' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:10:27.443 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T11:10:27.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:27 smithi155 ceph-mon[184854]: pgmap v1492: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:27.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:27 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4265517774' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:10:27.715 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:28.359 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:28 smithi125 ceph-mon[195507]: failed to encode map e616 with expected crc 2023-10-09T11:10:28.359 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:28 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4265517774' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T11:10:28.359 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:28 smithi125 ceph-mon[195507]: osdmap e616: 8 total, 8 up, 8 in 2023-10-09T11:10:28.359 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:28 smithi125 ceph-mon[195507]: failed to encode map e616 with expected crc 2023-10-09T11:10:28.359 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:28 smithi125 ceph-mon[200197]: failed to encode map e616 with expected crc 2023-10-09T11:10:28.360 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:28 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4265517774' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T11:10:28.360 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:28 smithi125 ceph-mon[200197]: osdmap e616: 8 total, 8 up, 8 in 2023-10-09T11:10:28.360 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:28 smithi125 ceph-mon[200197]: failed to encode map e616 with expected crc 2023-10-09T11:10:28.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:28 smithi155 ceph-mon[184854]: failed to encode map e616 with expected crc 2023-10-09T11:10:28.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:28 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4265517774' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T11:10:28.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:28 smithi155 ceph-mon[184854]: osdmap e616: 8 total, 8 up, 8 in 2023-10-09T11:10:28.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:28 smithi155 ceph-mon[184854]: failed to encode map e616 with expected crc 2023-10-09T11:10:29.439 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T11:10:29.463 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:29 smithi125 ceph-mon[195507]: failed to encode map e616 with expected crc 2023-10-09T11:10:29.463 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:29 smithi125 ceph-mon[195507]: failed to encode map e616 with expected crc 2023-10-09T11:10:29.463 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:29 smithi125 ceph-mon[195507]: pgmap v1494: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:29.463 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:29 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/117756911' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:10:29.464 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:29 smithi125 ceph-mon[200197]: failed to encode map e616 with expected crc 2023-10-09T11:10:29.464 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:29 smithi125 ceph-mon[200197]: failed to encode map e616 with expected crc 2023-10-09T11:10:29.464 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:29 smithi125 ceph-mon[200197]: pgmap v1494: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:29.464 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:29 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/117756911' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:10:29.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:29 smithi155 ceph-mon[184854]: failed to encode map e616 with expected crc 2023-10-09T11:10:29.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:29 smithi155 ceph-mon[184854]: failed to encode map e616 with expected crc 2023-10-09T11:10:29.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:29 smithi155 ceph-mon[184854]: pgmap v1494: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:29.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:29 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/117756911' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T11:10:29.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T11:10:30.088 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:30.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/117756911' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T11:10:30.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[195507]: osdmap e617: 8 total, 8 up, 8 in 2023-10-09T11:10:30.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[195507]: failed to encode map e617 with expected crc 2023-10-09T11:10:30.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[195507]: failed to encode map e617 with expected crc 2023-10-09T11:10:30.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[195507]: failed to encode map e617 with expected crc 2023-10-09T11:10:30.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[195507]: failed to encode map e617 with expected crc 2023-10-09T11:10:30.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[195507]: failed to encode map e617 with expected crc 2023-10-09T11:10:30.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[195507]: pgmap v1496: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:30.565 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:10:30.565 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:10:30.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/117756911' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T11:10:30.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[200197]: osdmap e617: 8 total, 8 up, 8 in 2023-10-09T11:10:30.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[200197]: failed to encode map e617 with expected crc 2023-10-09T11:10:30.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[200197]: failed to encode map e617 with expected crc 2023-10-09T11:10:30.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[200197]: failed to encode map e617 with expected crc 2023-10-09T11:10:30.566 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[200197]: failed to encode map e617 with expected crc 2023-10-09T11:10:30.566 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[200197]: failed to encode map e617 with expected crc 2023-10-09T11:10:30.566 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[200197]: pgmap v1496: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:30.566 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:10:30.566 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:30 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:10:30.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:30 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/117756911' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T11:10:30.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:30 smithi155 ceph-mon[184854]: osdmap e617: 8 total, 8 up, 8 in 2023-10-09T11:10:30.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:30 smithi155 ceph-mon[184854]: failed to encode map e617 with expected crc 2023-10-09T11:10:30.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:30 smithi155 ceph-mon[184854]: failed to encode map e617 with expected crc 2023-10-09T11:10:30.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:30 smithi155 ceph-mon[184854]: failed to encode map e617 with expected crc 2023-10-09T11:10:30.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:30 smithi155 ceph-mon[184854]: failed to encode map e617 with expected crc 2023-10-09T11:10:30.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:30 smithi155 ceph-mon[184854]: failed to encode map e617 with expected crc 2023-10-09T11:10:30.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:30 smithi155 ceph-mon[184854]: pgmap v1496: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:30.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' 2023-10-09T11:10:30.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:30 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:10:31.447 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T11:10:31.472 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:31 smithi125 ceph-mon[195507]: failed to encode map e617 with expected crc 2023-10-09T11:10:31.472 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:31 smithi125 ceph-mon[195507]: failed to encode map e617 with expected crc 2023-10-09T11:10:31.472 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:31 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2822229075' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:10:31.472 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:31 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:10:31.473 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:31 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:10:31] "GET /metrics HTTP/1.1" 200 34166 "" "Prometheus/2.43.0" 2023-10-09T11:10:31.473 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:31 smithi125 ceph-mon[200197]: failed to encode map e617 with expected crc 2023-10-09T11:10:31.473 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:31 smithi125 ceph-mon[200197]: failed to encode map e617 with expected crc 2023-10-09T11:10:31.473 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:31 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2822229075' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:10:31.473 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:31 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:10:31.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:31 smithi155 ceph-mon[184854]: failed to encode map e617 with expected crc 2023-10-09T11:10:31.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:31 smithi155 ceph-mon[184854]: failed to encode map e617 with expected crc 2023-10-09T11:10:31.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:31 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2822229075' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:10:31.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:31 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:10:31.851 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-09T11:10:32.117 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:32.563 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:32 smithi125 ceph-mon[200197]: failed to encode map e618 with expected crc 2023-10-09T11:10:32.563 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:32 smithi125 ceph-mon[200197]: failed to encode map e618 with expected crc 2023-10-09T11:10:32.563 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:32 smithi125 ceph-mon[200197]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T11:10:32.564 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:32 smithi125 ceph-mon[200197]: osdmap e618: 8 total, 8 up, 8 in 2023-10-09T11:10:32.564 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:32 smithi125 ceph-mon[200197]: pgmap v1498: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:32.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:32 smithi125 ceph-mon[195507]: failed to encode map e618 with expected crc 2023-10-09T11:10:32.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:32 smithi125 ceph-mon[195507]: failed to encode map e618 with expected crc 2023-10-09T11:10:32.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:32 smithi125 ceph-mon[195507]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T11:10:32.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:32 smithi125 ceph-mon[195507]: osdmap e618: 8 total, 8 up, 8 in 2023-10-09T11:10:32.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:32 smithi125 ceph-mon[195507]: pgmap v1498: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:32.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:32 smithi155 ceph-mon[184854]: failed to encode map e618 with expected crc 2023-10-09T11:10:32.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:32 smithi155 ceph-mon[184854]: failed to encode map e618 with expected crc 2023-10-09T11:10:32.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:32 smithi155 ceph-mon[184854]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T11:10:32.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:32 smithi155 ceph-mon[184854]: osdmap e618: 8 total, 8 up, 8 in 2023-10-09T11:10:32.721 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:32 smithi155 ceph-mon[184854]: pgmap v1498: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:33.466 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-09T11:10:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:33 smithi155 ceph-mon[184854]: failed to encode map e618 with expected crc 2023-10-09T11:10:33.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:33 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1768018576' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:10:33.766 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:33 smithi125 ceph-mon[195507]: failed to encode map e618 with expected crc 2023-10-09T11:10:33.766 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:33 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1768018576' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:10:33.767 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:33 smithi125 ceph-mon[200197]: failed to encode map e618 with expected crc 2023-10-09T11:10:33.767 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:33 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1768018576' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T11:10:33.851 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-10-09T11:10:33.851 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-10-09T11:10:33.851 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-10-09T11:10:33.852 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-10-09T11:10:33.852 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T11:10:33.852 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T11:10:34.117 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:34 smithi125 ceph-mon[195507]: failed to encode map e619 with expected crc 2023-10-09T11:10:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:34 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1768018576' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T11:10:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:34 smithi125 ceph-mon[195507]: osdmap e619: 8 total, 8 up, 8 in 2023-10-09T11:10:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:34 smithi125 ceph-mon[195507]: failed to encode map e619 with expected crc 2023-10-09T11:10:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:34 smithi125 ceph-mon[195507]: failed to encode map e619 with expected crc 2023-10-09T11:10:34.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:34 smithi125 ceph-mon[195507]: pgmap v1500: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:34.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:34 smithi125 ceph-mon[200197]: failed to encode map e619 with expected crc 2023-10-09T11:10:34.821 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:34 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1768018576' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T11:10:34.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:34 smithi125 ceph-mon[200197]: osdmap e619: 8 total, 8 up, 8 in 2023-10-09T11:10:34.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:34 smithi125 ceph-mon[200197]: failed to encode map e619 with expected crc 2023-10-09T11:10:34.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:34 smithi125 ceph-mon[200197]: failed to encode map e619 with expected crc 2023-10-09T11:10:34.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:34 smithi125 ceph-mon[200197]: pgmap v1500: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:34 smithi155 ceph-mon[184854]: failed to encode map e619 with expected crc 2023-10-09T11:10:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:34 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1768018576' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T11:10:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:34 smithi155 ceph-mon[184854]: osdmap e619: 8 total, 8 up, 8 in 2023-10-09T11:10:34.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:34 smithi155 ceph-mon[184854]: failed to encode map e619 with expected crc 2023-10-09T11:10:34.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:34 smithi155 ceph-mon[184854]: failed to encode map e619 with expected crc 2023-10-09T11:10:34.971 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:34 smithi155 ceph-mon[184854]: pgmap v1500: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:35.324 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:10:35.325 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":619,"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","created":"2023-10-09T10:31:07.767709+0000","modified":"2023-10-09T11:10:33.454127+0000","last_up_change":"2023-10-09T11:04:14.243961+0000","last_in_change":"2023-10-09T11:05:19.269738+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T10:33:34.342993+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"256","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"228","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T10:35:19.807378+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"590","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":590,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T10:35:55.201745+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T10:35:56.049529+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"874cd96b-ddd8-4164-931e-06f2388dd41b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":76,"up_from":80,"up_thru":601,"down_at":77,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6803","nonce":2302743228}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6805","nonce":2302743228}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6809","nonce":2302743228}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6807","nonce":2302743228}]},"public_addr":"172.21.15.125:6803/2302743228","cluster_addr":"172.21.15.125:6805/2302743228","heartbeat_back_addr":"172.21.15.125:6809/2302743228","heartbeat_front_addr":"172.21.15.125:6807/2302743228","state":["exists","up"]},{"osd":1,"uuid":"0e517dd5-6d0b-4841-bb81-1eb04377ba5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":85,"last_clean_end":357,"up_from":388,"up_thru":569,"down_at":358,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":2306109326},{"type":"v1","addr":"172.21.15.125:6811","nonce":2306109326}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":2306109326},{"type":"v1","addr":"172.21.15.125:6813","nonce":2306109326}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":2306109326},{"type":"v1","addr":"172.21.15.125:6817","nonce":2306109326}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":2306109326},{"type":"v1","addr":"172.21.15.125:6815","nonce":2306109326}]},"public_addr":"172.21.15.125:6811/2306109326","cluster_addr":"172.21.15.125:6813/2306109326","heartbeat_back_addr":"172.21.15.125:6817/2306109326","heartbeat_front_addr":"172.21.15.125:6815/2306109326","state":["exists","up"]},{"osd":2,"uuid":"643c1f3c-ecd9-464a-ad53-bf35a4deee5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":86,"up_from":90,"up_thru":604,"down_at":87,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6819","nonce":3577506432}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6821","nonce":3577506432}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6825","nonce":3577506432}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6823","nonce":3577506432}]},"public_addr":"172.21.15.125:6819/3577506432","cluster_addr":"172.21.15.125:6821/3577506432","heartbeat_back_addr":"172.21.15.125:6825/3577506432","heartbeat_front_addr":"172.21.15.125:6823/3577506432","state":["exists","up"]},{"osd":3,"uuid":"3015cd6e-b270-44c3-8e6e-6bcd979bcd21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":95,"last_clean_end":448,"up_from":469,"up_thru":569,"down_at":449,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":152767826},{"type":"v1","addr":"172.21.15.125:6827","nonce":152767826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":152767826},{"type":"v1","addr":"172.21.15.125:6829","nonce":152767826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":152767826},{"type":"v1","addr":"172.21.15.125:6833","nonce":152767826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":152767826},{"type":"v1","addr":"172.21.15.125:6831","nonce":152767826}]},"public_addr":"172.21.15.125:6827/152767826","cluster_addr":"172.21.15.125:6829/152767826","heartbeat_back_addr":"172.21.15.125:6833/152767826","heartbeat_front_addr":"172.21.15.125:6831/152767826","state":["exists","up"]},{"osd":4,"uuid":"272c7b95-f89c-4fe1-a299-a861e0ec9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":607,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6801","nonce":1086065767}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6803","nonce":1086065767}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6807","nonce":1086065767}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6805","nonce":1086065767}]},"public_addr":"172.21.15.155:6801/1086065767","cluster_addr":"172.21.15.155:6803/1086065767","heartbeat_back_addr":"172.21.15.155:6807/1086065767","heartbeat_front_addr":"172.21.15.155:6805/1086065767","state":["exists","up"]},{"osd":5,"uuid":"201d51f2-233c-40fc-a42d-649d225954fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":569,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6809","nonce":104536199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6811","nonce":104536199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6815","nonce":104536199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6813","nonce":104536199}]},"public_addr":"172.21.15.155:6809/104536199","cluster_addr":"172.21.15.155:6811/104536199","heartbeat_back_addr":"172.21.15.155:6815/104536199","heartbeat_front_addr":"172.21.15.155:6813/104536199","state":["exists","up"]},{"osd":6,"uuid":"4ec66642-108e-401f-aff3-e7ab258bc083","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":612,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6817","nonce":1730141492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6819","nonce":1730141492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6823","nonce":1730141492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6821","nonce":1730141492}]},"public_addr":"172.21.15.155:6817/1730141492","cluster_addr":"172.21.15.155:6819/1730141492","heartbeat_back_addr":"172.21.15.155:6823/1730141492","heartbeat_front_addr":"172.21.15.155:6821/1730141492","state":["exists","up"]},{"osd":7,"uuid":"f8eccbf5-4ad0-43e7-be9a-00c05a362e57","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":569,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6825","nonce":3094552858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6827","nonce":3094552858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6831","nonce":3094552858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6829","nonce":3094552858}]},"public_addr":"172.21.15.155:6825/3094552858","cluster_addr":"172.21.15.155:6827/3094552858","heartbeat_back_addr":"172.21.15.155:6831/3094552858","heartbeat_front_addr":"172.21.15.155:6829/3094552858","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:32:58.904066+0000","dead_epoch":76},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:15.883014+0000","dead_epoch":357},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:32.639145+0000","dead_epoch":86},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:49.937002+0000","dead_epoch":448},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:06.212675+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:22.755647+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:39.598906+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:56.143619+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.12","mappings":[{"from":1,"to":5}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:6801/1533297380":"2023-10-10T10:38:44.908167+0000","172.21.15.125:6800/1533297380":"2023-10-10T10:38:44.908167+0000","172.21.15.125:0/2429796720":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/2611643565":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/178379618":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/1316191185":"2023-10-10T10:35:20.607863+0000","172.21.15.155:0/1923144979":"2023-10-10T10:38:13.935349+0000","172.21.15.125:6800/1128053553":"2023-10-10T10:37:37.647752+0000","172.21.15.125:0/1483568239":"2023-10-10T10:37:37.647752+0000","172.21.15.125:6801/1128053553":"2023-10-10T10:37:37.647752+0000","172.21.15.125:6800/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.155:6835/2184630814":"2023-10-10T10:38:13.935349+0000","172.21.15.155:0/1920197956":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/2789680395":"2023-10-10T10:35:20.607863+0000","172.21.15.155:0/1738428098":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/1507781272":"2023-10-10T10:38:44.908167+0000","172.21.15.155:6834/2184630814":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/725980307":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6800/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/86510625":"2023-10-10T10:37:37.647752+0000","172.21.15.125:0/1623791415":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6801/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/461781620":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6801/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.155:0/2043466629":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/3709143923":"2023-10-10T10:37:37.647752+0000","172.21.15.125:0/1709045642":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/2757467822":"2023-10-10T10:37:37.647752+0000","172.21.15.125:6800/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/3693269321":"2023-10-10T10:38:44.908167+0000","172.21.15.155:0/48952571":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/3028032775":"2023-10-10T10:38:44.908167+0000","172.21.15.125:0/1101683240":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/653697639":"2023-10-10T10:38:44.908167+0000","172.21.15.125:6801/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/3810210930":"2023-10-10T10:38:44.908167+0000","172.21.15.155:0/565975522":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/1633295829":"2023-10-10T10:38:44.908167+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T11:10:35.608 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:35 smithi125 ceph-mon[195507]: failed to encode map e619 with expected crc 2023-10-09T11:10:35.608 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:35 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3933526996' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T11:10:35.608 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:35 smithi125 ceph-mon[200197]: failed to encode map e619 with expected crc 2023-10-09T11:10:35.608 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:35 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3933526996' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T11:10:35.701 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T11:10:35.702 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T11:10:35.967 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:35 smithi155 ceph-mon[184854]: failed to encode map e619 with expected crc 2023-10-09T11:10:35.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:35 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3933526996' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T11:10:36.704 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:36 smithi125 ceph-mon[195507]: pgmap v1501: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:36.704 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:36 smithi125 ceph-mon[200197]: pgmap v1501: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:36.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:36 smithi155 ceph-mon[184854]: pgmap v1501: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:37.097 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:10:37.097 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":619,"fsid":"daf690ae-668e-11ee-8db5-212e2dc638e7","created":"2023-10-09T10:31:07.767709+0000","modified":"2023-10-09T11:10:33.454127+0000","last_up_change":"2023-10-09T11:04:14.243961+0000","last_in_change":"2023-10-09T11:05:19.269738+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T10:33:34.342993+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"256","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"228","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T10:35:19.807378+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"590","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":590,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T10:35:55.201745+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T10:35:56.049529+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"874cd96b-ddd8-4164-931e-06f2388dd41b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":76,"up_from":80,"up_thru":601,"down_at":77,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6803","nonce":2302743228}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6805","nonce":2302743228}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6809","nonce":2302743228}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":2302743228},{"type":"v1","addr":"172.21.15.125:6807","nonce":2302743228}]},"public_addr":"172.21.15.125:6803/2302743228","cluster_addr":"172.21.15.125:6805/2302743228","heartbeat_back_addr":"172.21.15.125:6809/2302743228","heartbeat_front_addr":"172.21.15.125:6807/2302743228","state":["exists","up"]},{"osd":1,"uuid":"0e517dd5-6d0b-4841-bb81-1eb04377ba5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":85,"last_clean_end":357,"up_from":388,"up_thru":569,"down_at":358,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":2306109326},{"type":"v1","addr":"172.21.15.125:6811","nonce":2306109326}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":2306109326},{"type":"v1","addr":"172.21.15.125:6813","nonce":2306109326}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":2306109326},{"type":"v1","addr":"172.21.15.125:6817","nonce":2306109326}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":2306109326},{"type":"v1","addr":"172.21.15.125:6815","nonce":2306109326}]},"public_addr":"172.21.15.125:6811/2306109326","cluster_addr":"172.21.15.125:6813/2306109326","heartbeat_back_addr":"172.21.15.125:6817/2306109326","heartbeat_front_addr":"172.21.15.125:6815/2306109326","state":["exists","up"]},{"osd":2,"uuid":"643c1f3c-ecd9-464a-ad53-bf35a4deee5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":86,"up_from":90,"up_thru":604,"down_at":87,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6819","nonce":3577506432}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6821","nonce":3577506432}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6825","nonce":3577506432}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":3577506432},{"type":"v1","addr":"172.21.15.125:6823","nonce":3577506432}]},"public_addr":"172.21.15.125:6819/3577506432","cluster_addr":"172.21.15.125:6821/3577506432","heartbeat_back_addr":"172.21.15.125:6825/3577506432","heartbeat_front_addr":"172.21.15.125:6823/3577506432","state":["exists","up"]},{"osd":3,"uuid":"3015cd6e-b270-44c3-8e6e-6bcd979bcd21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":95,"last_clean_end":448,"up_from":469,"up_thru":569,"down_at":449,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":152767826},{"type":"v1","addr":"172.21.15.125:6827","nonce":152767826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":152767826},{"type":"v1","addr":"172.21.15.125:6829","nonce":152767826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":152767826},{"type":"v1","addr":"172.21.15.125:6833","nonce":152767826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":152767826},{"type":"v1","addr":"172.21.15.125:6831","nonce":152767826}]},"public_addr":"172.21.15.125:6827/152767826","cluster_addr":"172.21.15.125:6829/152767826","heartbeat_back_addr":"172.21.15.125:6833/152767826","heartbeat_front_addr":"172.21.15.125:6831/152767826","state":["exists","up"]},{"osd":4,"uuid":"272c7b95-f89c-4fe1-a299-a861e0ec9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":607,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6801","nonce":1086065767}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6803","nonce":1086065767}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6807","nonce":1086065767}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":1086065767},{"type":"v1","addr":"172.21.15.155:6805","nonce":1086065767}]},"public_addr":"172.21.15.155:6801/1086065767","cluster_addr":"172.21.15.155:6803/1086065767","heartbeat_back_addr":"172.21.15.155:6807/1086065767","heartbeat_front_addr":"172.21.15.155:6805/1086065767","state":["exists","up"]},{"osd":5,"uuid":"201d51f2-233c-40fc-a42d-649d225954fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":569,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6809","nonce":104536199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6811","nonce":104536199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6815","nonce":104536199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":104536199},{"type":"v1","addr":"172.21.15.155:6813","nonce":104536199}]},"public_addr":"172.21.15.155:6809/104536199","cluster_addr":"172.21.15.155:6811/104536199","heartbeat_back_addr":"172.21.15.155:6815/104536199","heartbeat_front_addr":"172.21.15.155:6813/104536199","state":["exists","up"]},{"osd":6,"uuid":"4ec66642-108e-401f-aff3-e7ab258bc083","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":612,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6817","nonce":1730141492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6819","nonce":1730141492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6823","nonce":1730141492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":1730141492},{"type":"v1","addr":"172.21.15.155:6821","nonce":1730141492}]},"public_addr":"172.21.15.155:6817/1730141492","cluster_addr":"172.21.15.155:6819/1730141492","heartbeat_back_addr":"172.21.15.155:6823/1730141492","heartbeat_front_addr":"172.21.15.155:6821/1730141492","state":["exists","up"]},{"osd":7,"uuid":"f8eccbf5-4ad0-43e7-be9a-00c05a362e57","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":569,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6825","nonce":3094552858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6827","nonce":3094552858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6831","nonce":3094552858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":3094552858},{"type":"v1","addr":"172.21.15.155:6829","nonce":3094552858}]},"public_addr":"172.21.15.155:6825/3094552858","cluster_addr":"172.21.15.155:6827/3094552858","heartbeat_back_addr":"172.21.15.155:6831/3094552858","heartbeat_front_addr":"172.21.15.155:6829/3094552858","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:32:58.904066+0000","dead_epoch":76},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:15.883014+0000","dead_epoch":357},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:32.639145+0000","dead_epoch":86},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:33:49.937002+0000","dead_epoch":448},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:06.212675+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:22.755647+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:39.598906+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T10:34:56.143619+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.12","mappings":[{"from":1,"to":5}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:6801/1533297380":"2023-10-10T10:38:44.908167+0000","172.21.15.125:6800/1533297380":"2023-10-10T10:38:44.908167+0000","172.21.15.125:0/2429796720":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/2611643565":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/178379618":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/1316191185":"2023-10-10T10:35:20.607863+0000","172.21.15.155:0/1923144979":"2023-10-10T10:38:13.935349+0000","172.21.15.125:6800/1128053553":"2023-10-10T10:37:37.647752+0000","172.21.15.125:0/1483568239":"2023-10-10T10:37:37.647752+0000","172.21.15.125:6801/1128053553":"2023-10-10T10:37:37.647752+0000","172.21.15.125:6800/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.155:6835/2184630814":"2023-10-10T10:38:13.935349+0000","172.21.15.155:0/1920197956":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/2789680395":"2023-10-10T10:35:20.607863+0000","172.21.15.155:0/1738428098":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/1507781272":"2023-10-10T10:38:44.908167+0000","172.21.15.155:6834/2184630814":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/725980307":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6800/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.125:0/86510625":"2023-10-10T10:37:37.647752+0000","172.21.15.125:0/1623791415":"2023-10-10T10:31:56.030149+0000","172.21.15.125:6801/1633421367":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/461781620":"2023-10-10T10:35:20.607863+0000","172.21.15.125:6801/3950444489":"2023-10-10T10:31:56.030149+0000","172.21.15.155:0/2043466629":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/3709143923":"2023-10-10T10:37:37.647752+0000","172.21.15.125:0/1709045642":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/2757467822":"2023-10-10T10:37:37.647752+0000","172.21.15.125:6800/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/3693269321":"2023-10-10T10:38:44.908167+0000","172.21.15.155:0/48952571":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/3028032775":"2023-10-10T10:38:44.908167+0000","172.21.15.125:0/1101683240":"2023-10-10T10:31:33.872122+0000","172.21.15.125:0/653697639":"2023-10-10T10:38:44.908167+0000","172.21.15.125:6801/459376588":"2023-10-10T10:35:20.607863+0000","172.21.15.125:0/3810210930":"2023-10-10T10:38:44.908167+0000","172.21.15.155:0/565975522":"2023-10-10T10:38:13.935349+0000","172.21.15.125:0/1633295829":"2023-10-10T10:38:44.908167+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T11:10:37.476 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T11:10:37.477 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T11:10:37.477 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T11:10:37.477 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T11:10:37.478 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T11:10:37.478 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T11:10:37.479 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T11:10:37.479 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T11:10:37.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:37 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/1686514807' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T11:10:37.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:37 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/1686514807' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T11:10:37.955 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:37.969 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:37.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:37 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/1686514807' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T11:10:37.979 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:38.384 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:38.757 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:38.769 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:38 smithi125 ceph-mon[195507]: pgmap v1502: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:38.769 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:38 smithi125 ceph-mon[200197]: pgmap v1502: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:38.807 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:38.890 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:38.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:38 smithi155 ceph-mon[184854]: pgmap v1502: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:41.113 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:41 smithi125 ceph-mon[195507]: pgmap v1503: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:41.113 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:41 smithi125 ceph-mon[200197]: pgmap v1503: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:41.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:41 smithi155 ceph-mon[184854]: pgmap v1503: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:41.540 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:41 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:10:41] "GET /metrics HTTP/1.1" 200 34166 "" "Prometheus/2.43.0" 2023-10-09T11:10:41.559 INFO:teuthology.orchestra.run.smithi125.stdout:386547056973 2023-10-09T11:10:41.559 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T11:10:42.000 INFO:teuthology.orchestra.run.smithi125.stdout:1666447310975 2023-10-09T11:10:42.000 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T11:10:42.231 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:43.028 INFO:teuthology.orchestra.run.smithi125.stdout:158913790393 2023-10-09T11:10:43.029 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T11:10:43.044 INFO:teuthology.orchestra.run.smithi125.stdout:128849019324 2023-10-09T11:10:43.045 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T11:10:43.272 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:43 smithi125 ceph-mon[195507]: pgmap v1504: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:43.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:43 smithi125 ceph-mon[200197]: pgmap v1504: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:43.470 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:43 smithi155 ceph-mon[184854]: pgmap v1504: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:43.610 INFO:teuthology.orchestra.run.smithi125.stdout:2014339661903 2023-10-09T11:10:43.610 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T11:10:43.907 INFO:teuthology.orchestra.run.smithi125.stdout:343597384033 2023-10-09T11:10:43.908 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T11:10:44.565 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:44.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:44 smithi125 ceph-mon[195507]: pgmap v1505: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:44.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:44 smithi125 ceph-mon[200197]: pgmap v1505: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:44.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:44 smithi155 ceph-mon[184854]: pgmap v1505: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:44.917 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:45.341 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:45.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:45 smithi155 ceph-mon[184854]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:10:45.790 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:45 smithi125 ceph-mon[195507]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:10:45.790 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:45 smithi125 ceph-mon[200197]: from='mgr.34107 172.21.15.125:0/653693151' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T11:10:45.996 INFO:teuthology.orchestra.run.smithi125.stdout:386547056974 2023-10-09T11:10:46.595 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:46.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:46 smithi125 ceph-mon[195507]: pgmap v1506: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:46.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:46 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2488783873' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T11:10:46.644 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:46 smithi125 ceph-mon[200197]: pgmap v1506: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:46.645 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:46 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2488783873' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T11:10:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:46 smithi155 ceph-mon[184854]: pgmap v1506: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:46.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:46 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2488783873' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T11:10:46.810 INFO:tasks.cephadm.ceph_manager.ceph:need seq 386547056973 got 386547056974 for osd.2 2023-10-09T11:10:46.811 DEBUG:teuthology.parallel:result is None 2023-10-09T11:10:46.941 INFO:teuthology.orchestra.run.smithi125.stdout:219043332530 2023-10-09T11:10:46.941 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T11:10:47.244 INFO:teuthology.orchestra.run.smithi125.stdout:1666447310976 2023-10-09T11:10:47.398 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:47.527 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:47 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/4196877119' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T11:10:47.527 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:47 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/4196877119' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T11:10:47.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:47 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/4196877119' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T11:10:47.823 INFO:teuthology.orchestra.run.smithi125.stdout:188978561461 2023-10-09T11:10:47.824 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T11:10:47.850 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:47.883 INFO:teuthology.orchestra.run.smithi125.stdout:158913790394 2023-10-09T11:10:47.943 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1666447310975 got 1666447310976 for osd.1 2023-10-09T11:10:47.944 DEBUG:teuthology.parallel:result is None 2023-10-09T11:10:48.608 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913790393 got 158913790394 for osd.5 2023-10-09T11:10:48.608 DEBUG:teuthology.parallel:result is None 2023-10-09T11:10:48.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:48 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2190841121' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T11:10:48.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:48 smithi155 ceph-mon[184854]: pgmap v1507: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:48.756 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:48 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2190841121' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T11:10:48.756 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:48 smithi125 ceph-mon[195507]: pgmap v1507: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:48.756 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:48 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2190841121' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T11:10:48.756 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:48 smithi125 ceph-mon[200197]: pgmap v1507: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:49.343 INFO:teuthology.orchestra.run.smithi125.stdout:128849019325 2023-10-09T11:10:49.529 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:49.680 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:49.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:49 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2635569871' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T11:10:49.820 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:49 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2635569871' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T11:10:49.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:49 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2635569871' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T11:10:49.857 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019324 got 128849019325 for osd.4 2023-10-09T11:10:49.857 DEBUG:teuthology.parallel:result is None 2023-10-09T11:10:49.963 INFO:teuthology.orchestra.run.smithi125.stdout:2014339661904 2023-10-09T11:10:50.495 INFO:teuthology.orchestra.run.smithi125.stdout:343597384035 2023-10-09T11:10:50.664 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2014339661903 got 2014339661904 for osd.3 2023-10-09T11:10:50.664 DEBUG:teuthology.parallel:result is None 2023-10-09T11:10:50.695 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:50 smithi125 ceph-mon[195507]: pgmap v1508: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:50.695 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:50 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2051335045' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T11:10:50.696 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:50 smithi125 ceph-mon[200197]: pgmap v1508: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:50.696 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:50 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2051335045' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T11:10:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:50 smithi155 ceph-mon[184854]: pgmap v1508: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:50.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:50 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2051335045' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T11:10:51.276 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:51 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: ::ffff:172.21.15.155 - - [09/Oct/2023:11:10:51] "GET /metrics HTTP/1.1" 200 34163 "" "Prometheus/2.43.0" 2023-10-09T11:10:51.552 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:51 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/2060206523' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T11:10:51.552 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:51 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/2060206523' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T11:10:51.570 INFO:tasks.cephadm.ceph_manager.ceph:need seq 343597384033 got 343597384035 for osd.0 2023-10-09T11:10:51.570 DEBUG:teuthology.parallel:result is None 2023-10-09T11:10:51.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:51 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/2060206523' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T11:10:51.967 INFO:teuthology.orchestra.run.smithi125.stdout:219043332531 2023-10-09T11:10:52.077 INFO:teuthology.orchestra.run.smithi125.stdout:188978561462 2023-10-09T11:10:52.382 INFO:tasks.cephadm.ceph_manager.ceph:need seq 219043332530 got 219043332531 for osd.7 2023-10-09T11:10:52.382 DEBUG:teuthology.parallel:result is None 2023-10-09T11:10:52.547 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:52 smithi125 ceph-mon[200197]: pgmap v1509: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:52.547 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:52 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/656402302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T11:10:52.547 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:52 smithi125 ceph-mon[200197]: from='client.? 172.21.15.125:0/3288308475' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T11:10:52.547 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:52 smithi125 ceph-mon[195507]: pgmap v1509: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:52.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:52 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/656402302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T11:10:52.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:52 smithi125 ceph-mon[195507]: from='client.? 172.21.15.125:0/3288308475' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T11:10:52.566 INFO:tasks.cephadm.ceph_manager.ceph:need seq 188978561461 got 188978561462 for osd.6 2023-10-09T11:10:52.566 DEBUG:teuthology.parallel:result is None 2023-10-09T11:10:52.566 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T11:10:52.566 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:10:52.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:52 smithi155 ceph-mon[184854]: pgmap v1509: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:52.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:52 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/656402302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T11:10:52.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:52 smithi155 ceph-mon[184854]: from='client.? 172.21.15.125:0/3288308475' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T11:10:52.833 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:53.983 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:10:53.991 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:10:54.355 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1510,"stamp":"2023-10-09T11:10:53.960110+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5995,"num_read_kb":13244,"num_write":907,"num_write_kb":12741,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":98,"num_bytes_recovered":2369408,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7137584,"kb_used_data":72752,"kb_used_omap":134,"kb_used_meta":7064633,"kb_avail":742823632,"statfs":{"total":767960285184,"available":760651399168,"internally_reserved":0,"allocated":74498048,"data_stored":54073460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":137532,"internal_metadata":7234184900},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002848"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":904,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445356+0000","last_change":"2023-10-09T11:07:36.287455+0000","last_active":"2023-10-09T11:10:31.445356+0000","last_peered":"2023-10-09T11:10:31.445356+0000","last_clean":"2023-10-09T11:10:31.445356+0000","last_became_active":"2023-10-09T11:07:36.287052+0000","last_became_peered":"2023-10-09T11:07:36.287052+0000","last_unstale":"2023-10-09T11:10:31.445356+0000","last_undegraded":"2023-10-09T11:10:31.445356+0000","last_fullsized":"2023-10-09T11:10:31.445356+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:18.864873+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":407,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578694+0000","last_change":"2023-10-09T11:10:11.520572+0000","last_active":"2023-10-09T11:10:32.578694+0000","last_peered":"2023-10-09T11:10:32.578694+0000","last_clean":"2023-10-09T11:10:32.578694+0000","last_became_active":"2023-10-09T11:10:11.520214+0000","last_became_peered":"2023-10-09T11:10:11.520214+0000","last_unstale":"2023-10-09T11:10:32.578694+0000","last_undegraded":"2023-10-09T11:10:32.578694+0000","last_fullsized":"2023-10-09T11:10:32.578694+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:50:00.039165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":146,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443437+0000","last_change":"2023-10-09T11:04:50.319477+0000","last_active":"2023-10-09T11:10:31.443437+0000","last_peered":"2023-10-09T11:10:31.443437+0000","last_clean":"2023-10-09T11:10:31.443437+0000","last_became_active":"2023-10-09T11:04:50.319148+0000","last_became_peered":"2023-10-09T11:04:50.319148+0000","last_unstale":"2023-10-09T11:10:31.443437+0000","last_undegraded":"2023-10-09T11:10:31.443437+0000","last_fullsized":"2023-10-09T11:10:31.443437+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:48.725566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":774,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446233+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:10:31.446233+0000","last_peered":"2023-10-09T11:10:31.446233+0000","last_clean":"2023-10-09T11:10:31.446233+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:10:31.446233+0000","last_undegraded":"2023-10-09T11:10:31.446233+0000","last_fullsized":"2023-10-09T11:10:31.446233+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":334,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.226630+0000","last_change":"2023-10-09T11:04:50.333711+0000","last_active":"2023-10-09T11:10:32.226630+0000","last_peered":"2023-10-09T11:10:32.226630+0000","last_clean":"2023-10-09T11:10:32.226630+0000","last_became_active":"2023-10-09T11:04:50.333262+0000","last_became_peered":"2023-10-09T11:04:50.333262+0000","last_unstale":"2023-10-09T11:10:32.226630+0000","last_undegraded":"2023-10-09T11:10:32.226630+0000","last_fullsized":"2023-10-09T11:10:32.226630+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.843765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":527,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578897+0000","last_change":"2023-10-09T11:10:02.479621+0000","last_active":"2023-10-09T11:10:32.578897+0000","last_peered":"2023-10-09T11:10:32.578897+0000","last_clean":"2023-10-09T11:10:32.578897+0000","last_became_active":"2023-10-09T11:10:02.479324+0000","last_became_peered":"2023-10-09T11:10:02.479324+0000","last_unstale":"2023-10-09T11:10:32.578897+0000","last_undegraded":"2023-10-09T11:10:32.578897+0000","last_fullsized":"2023-10-09T11:10:32.578897+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:22.686515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":993,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578425+0000","last_change":"2023-10-09T11:10:11.521202+0000","last_active":"2023-10-09T11:10:32.578425+0000","last_peered":"2023-10-09T11:10:32.578425+0000","last_clean":"2023-10-09T11:10:32.578425+0000","last_became_active":"2023-10-09T11:10:11.520809+0000","last_became_peered":"2023-10-09T11:10:11.520809+0000","last_unstale":"2023-10-09T11:10:32.578425+0000","last_undegraded":"2023-10-09T11:10:32.578425+0000","last_fullsized":"2023-10-09T11:10:32.578425+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:24.092815+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":146,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.442843+0000","last_change":"2023-10-09T11:04:50.310477+0000","last_active":"2023-10-09T11:10:31.442843+0000","last_peered":"2023-10-09T11:10:31.442843+0000","last_clean":"2023-10-09T11:10:31.442843+0000","last_became_active":"2023-10-09T11:04:50.310067+0000","last_became_peered":"2023-10-09T11:04:50.310067+0000","last_unstale":"2023-10-09T11:10:31.442843+0000","last_undegraded":"2023-10-09T11:10:31.442843+0000","last_fullsized":"2023-10-09T11:10:31.442843+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:52.884716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":319,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445888+0000","last_change":"2023-10-09T11:10:20.507397+0000","last_active":"2023-10-09T11:10:31.445888+0000","last_peered":"2023-10-09T11:10:31.445888+0000","last_clean":"2023-10-09T11:10:31.445888+0000","last_became_active":"2023-10-09T11:10:20.507062+0000","last_became_peered":"2023-10-09T11:10:20.507062+0000","last_unstale":"2023-10-09T11:10:31.445888+0000","last_undegraded":"2023-10-09T11:10:31.445888+0000","last_fullsized":"2023-10-09T11:10:31.445888+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:53.085834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":834,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.363850+0000","last_change":"2023-10-09T11:07:36.282502+0000","last_active":"2023-10-09T11:10:32.363850+0000","last_peered":"2023-10-09T11:10:32.363850+0000","last_clean":"2023-10-09T11:10:32.363850+0000","last_became_active":"2023-10-09T11:07:36.282133+0000","last_became_peered":"2023-10-09T11:07:36.282133+0000","last_unstale":"2023-10-09T11:10:32.363850+0000","last_undegraded":"2023-10-09T11:10:32.363850+0000","last_fullsized":"2023-10-09T11:10:32.363850+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:03.762383+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":220,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578969+0000","last_change":"2023-10-09T11:10:02.475444+0000","last_active":"2023-10-09T11:10:32.578969+0000","last_peered":"2023-10-09T11:10:32.578969+0000","last_clean":"2023-10-09T11:10:32.578969+0000","last_became_active":"2023-10-09T11:10:02.475066+0000","last_became_peered":"2023-10-09T11:10:02.475066+0000","last_unstale":"2023-10-09T11:10:32.578969+0000","last_undegraded":"2023-10-09T11:10:32.578969+0000","last_fullsized":"2023-10-09T11:10:32.578969+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:12.203653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":155,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.442901+0000","last_change":"2023-10-09T11:05:20.299453+0000","last_active":"2023-10-09T11:10:31.442901+0000","last_peered":"2023-10-09T11:10:31.442901+0000","last_clean":"2023-10-09T11:10:31.442901+0000","last_became_active":"2023-10-09T11:05:20.299117+0000","last_became_peered":"2023-10-09T11:05:20.299117+0000","last_unstale":"2023-10-09T11:10:31.442901+0000","last_undegraded":"2023-10-09T11:10:31.442901+0000","last_fullsized":"2023-10-09T11:10:31.442901+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:30.072813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":950,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.461802+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:10:33.461802+0000","last_peered":"2023-10-09T11:10:33.461802+0000","last_clean":"2023-10-09T11:10:33.461802+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:10:33.461802+0000","last_undegraded":"2023-10-09T11:10:33.461802+0000","last_fullsized":"2023-10-09T11:10:33.461802+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":488,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446547+0000","last_change":"2023-10-09T11:07:33.242665+0000","last_active":"2023-10-09T11:10:31.446547+0000","last_peered":"2023-10-09T11:10:31.446547+0000","last_clean":"2023-10-09T11:10:31.446547+0000","last_became_active":"2023-10-09T11:07:33.242313+0000","last_became_peered":"2023-10-09T11:07:33.242313+0000","last_unstale":"2023-10-09T11:10:31.446547+0000","last_undegraded":"2023-10-09T11:10:31.446547+0000","last_fullsized":"2023-10-09T11:10:31.446547+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:20.565135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":173,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443180+0000","last_change":"2023-10-09T11:07:33.225683+0000","last_active":"2023-10-09T11:10:31.443180+0000","last_peered":"2023-10-09T11:10:31.443180+0000","last_clean":"2023-10-09T11:10:31.443180+0000","last_became_active":"2023-10-09T11:07:33.225275+0000","last_became_peered":"2023-10-09T11:07:33.225275+0000","last_unstale":"2023-10-09T11:10:31.443180+0000","last_undegraded":"2023-10-09T11:10:31.443180+0000","last_fullsized":"2023-10-09T11:10:31.443180+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:10.131060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":846,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446606+0000","last_change":"2023-10-09T11:05:23.328981+0000","last_active":"2023-10-09T11:10:31.446606+0000","last_peered":"2023-10-09T11:10:31.446606+0000","last_clean":"2023-10-09T11:10:31.446606+0000","last_became_active":"2023-10-09T11:05:23.328616+0000","last_became_peered":"2023-10-09T11:05:23.328616+0000","last_unstale":"2023-10-09T11:10:31.446606+0000","last_undegraded":"2023-10-09T11:10:31.446606+0000","last_fullsized":"2023-10-09T11:10:31.446606+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:15.315819+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":85,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364593+0000","last_change":"2023-10-09T11:07:33.265842+0000","last_active":"2023-10-09T11:10:32.364593+0000","last_peered":"2023-10-09T11:10:32.364593+0000","last_clean":"2023-10-09T11:10:32.364593+0000","last_became_active":"2023-10-09T11:07:33.232311+0000","last_became_peered":"2023-10-09T11:07:33.232311+0000","last_unstale":"2023-10-09T11:10:32.364593+0000","last_undegraded":"2023-10-09T11:10:32.364593+0000","last_fullsized":"2023-10-09T11:10:32.364593+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:23.402582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":395,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445952+0000","last_change":"2023-10-09T11:05:20.316265+0000","last_active":"2023-10-09T11:10:31.445952+0000","last_peered":"2023-10-09T11:10:31.445952+0000","last_clean":"2023-10-09T11:10:31.445952+0000","last_became_active":"2023-10-09T11:05:20.315853+0000","last_became_peered":"2023-10-09T11:05:20.315853+0000","last_unstale":"2023-10-09T11:10:31.445952+0000","last_undegraded":"2023-10-09T11:10:31.445952+0000","last_fullsized":"2023-10-09T11:10:31.445952+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:35.093981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":1012,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.227156+0000","last_change":"2023-10-09T11:04:53.274921+0000","last_active":"2023-10-09T11:10:32.227156+0000","last_peered":"2023-10-09T11:10:32.227156+0000","last_clean":"2023-10-09T11:10:32.227156+0000","last_became_active":"2023-10-09T11:04:53.274518+0000","last_became_peered":"2023-10-09T11:04:53.274518+0000","last_unstale":"2023-10-09T11:10:32.227156+0000","last_undegraded":"2023-10-09T11:10:32.227156+0000","last_fullsized":"2023-10-09T11:10:32.227156+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:14.315301+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":235,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578771+0000","last_change":"2023-10-09T11:10:11.522075+0000","last_active":"2023-10-09T11:10:32.578771+0000","last_peered":"2023-10-09T11:10:32.578771+0000","last_clean":"2023-10-09T11:10:32.578771+0000","last_became_active":"2023-10-09T11:10:11.521860+0000","last_became_peered":"2023-10-09T11:10:11.521860+0000","last_unstale":"2023-10-09T11:10:32.578771+0000","last_undegraded":"2023-10-09T11:10:32.578771+0000","last_fullsized":"2023-10-09T11:10:32.578771+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:22.783526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":537,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578831+0000","last_change":"2023-10-09T11:10:11.523601+0000","last_active":"2023-10-09T11:10:32.578831+0000","last_peered":"2023-10-09T11:10:32.578831+0000","last_clean":"2023-10-09T11:10:32.578831+0000","last_became_active":"2023-10-09T11:10:11.523065+0000","last_became_peered":"2023-10-09T11:10:11.523065+0000","last_unstale":"2023-10-09T11:10:32.578831+0000","last_undegraded":"2023-10-09T11:10:32.578831+0000","last_fullsized":"2023-10-09T11:10:32.578831+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:18:08.763192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":954,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578511+0000","last_change":"2023-10-09T11:10:02.473069+0000","last_active":"2023-10-09T11:10:32.578511+0000","last_peered":"2023-10-09T11:10:32.578511+0000","last_clean":"2023-10-09T11:10:32.578511+0000","last_became_active":"2023-10-09T11:10:02.472741+0000","last_became_peered":"2023-10-09T11:10:02.472741+0000","last_unstale":"2023-10-09T11:10:32.578511+0000","last_undegraded":"2023-10-09T11:10:32.578511+0000","last_fullsized":"2023-10-09T11:10:32.578511+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.934865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:04.611884+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":85,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364223+0000","last_change":"2023-10-09T11:07:33.269200+0000","last_active":"2023-10-09T11:10:32.364223+0000","last_peered":"2023-10-09T11:10:32.364223+0000","last_clean":"2023-10-09T11:10:32.364223+0000","last_became_active":"2023-10-09T11:07:33.236205+0000","last_became_peered":"2023-10-09T11:07:33.236205+0000","last_unstale":"2023-10-09T11:10:32.364223+0000","last_undegraded":"2023-10-09T11:10:32.364223+0000","last_fullsized":"2023-10-09T11:10:32.364223+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:24.574734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":164,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443290+0000","last_change":"2023-10-09T11:07:33.227803+0000","last_active":"2023-10-09T11:10:31.443290+0000","last_peered":"2023-10-09T11:10:31.443290+0000","last_clean":"2023-10-09T11:10:31.443290+0000","last_became_active":"2023-10-09T11:07:33.227486+0000","last_became_peered":"2023-10-09T11:07:33.227486+0000","last_unstale":"2023-10-09T11:10:31.443290+0000","last_undegraded":"2023-10-09T11:10:31.443290+0000","last_fullsized":"2023-10-09T11:10:31.443290+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:39.274875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":928,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445760+0000","last_change":"2023-10-09T11:05:23.323856+0000","last_active":"2023-10-09T11:10:31.445760+0000","last_peered":"2023-10-09T11:10:31.445760+0000","last_clean":"2023-10-09T11:10:31.445760+0000","last_became_active":"2023-10-09T11:05:23.322763+0000","last_became_peered":"2023-10-09T11:05:23.322763+0000","last_unstale":"2023-10-09T11:10:31.445760+0000","last_undegraded":"2023-10-09T11:10:31.445760+0000","last_fullsized":"2023-10-09T11:10:31.445760+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.760820+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":525,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578417+0000","last_change":"2023-10-09T11:10:02.479550+0000","last_active":"2023-10-09T11:10:32.578417+0000","last_peered":"2023-10-09T11:10:32.578417+0000","last_clean":"2023-10-09T11:10:32.578417+0000","last_became_active":"2023-10-09T11:10:02.479243+0000","last_became_peered":"2023-10-09T11:10:02.479243+0000","last_unstale":"2023-10-09T11:10:32.578417+0000","last_undegraded":"2023-10-09T11:10:32.578417+0000","last_fullsized":"2023-10-09T11:10:32.578417+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:28:59.098743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":558,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446628+0000","last_change":"2023-10-09T11:07:33.226594+0000","last_active":"2023-10-09T11:10:31.446628+0000","last_peered":"2023-10-09T11:10:31.446628+0000","last_clean":"2023-10-09T11:10:31.446628+0000","last_became_active":"2023-10-09T11:07:33.226275+0000","last_became_peered":"2023-10-09T11:07:33.226275+0000","last_unstale":"2023-10-09T11:10:31.446628+0000","last_undegraded":"2023-10-09T11:10:31.446628+0000","last_fullsized":"2023-10-09T11:10:31.446628+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:41.180088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":953,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.226853+0000","last_change":"2023-10-09T11:04:53.274236+0000","last_active":"2023-10-09T11:10:32.226853+0000","last_peered":"2023-10-09T11:10:32.226853+0000","last_clean":"2023-10-09T11:10:32.226853+0000","last_became_active":"2023-10-09T11:04:53.273817+0000","last_became_peered":"2023-10-09T11:04:53.273817+0000","last_unstale":"2023-10-09T11:10:32.226853+0000","last_undegraded":"2023-10-09T11:10:32.226853+0000","last_fullsized":"2023-10-09T11:10:32.226853+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:05.096905+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":214,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.363757+0000","last_change":"2023-10-09T11:05:20.312129+0000","last_active":"2023-10-09T11:10:32.363757+0000","last_peered":"2023-10-09T11:10:32.363757+0000","last_clean":"2023-10-09T11:10:32.363757+0000","last_became_active":"2023-10-09T11:05:20.311810+0000","last_became_peered":"2023-10-09T11:05:20.311810+0000","last_unstale":"2023-10-09T11:10:32.363757+0000","last_undegraded":"2023-10-09T11:10:32.363757+0000","last_fullsized":"2023-10-09T11:10:32.363757+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:48.813391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":263,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578265+0000","last_change":"2023-10-09T11:10:02.479630+0000","last_active":"2023-10-09T11:10:32.578265+0000","last_peered":"2023-10-09T11:10:32.578265+0000","last_clean":"2023-10-09T11:10:32.578265+0000","last_became_active":"2023-10-09T11:10:02.479320+0000","last_became_peered":"2023-10-09T11:10:02.479320+0000","last_unstale":"2023-10-09T11:10:32.578265+0000","last_undegraded":"2023-10-09T11:10:32.578265+0000","last_fullsized":"2023-10-09T11:10:32.578265+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:43:27.560586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":925,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446295+0000","last_change":"2023-10-09T11:05:23.324537+0000","last_active":"2023-10-09T11:10:31.446295+0000","last_peered":"2023-10-09T11:10:31.446295+0000","last_clean":"2023-10-09T11:10:31.446295+0000","last_became_active":"2023-10-09T11:05:23.324187+0000","last_became_peered":"2023-10-09T11:05:23.324187+0000","last_unstale":"2023-10-09T11:10:31.446295+0000","last_undegraded":"2023-10-09T11:10:31.446295+0000","last_fullsized":"2023-10-09T11:10:31.446295+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:25.286106+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":137,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445428+0000","last_change":"2023-10-09T11:05:20.323565+0000","last_active":"2023-10-09T11:10:31.445428+0000","last_peered":"2023-10-09T11:10:31.445428+0000","last_clean":"2023-10-09T11:10:31.445428+0000","last_became_active":"2023-10-09T11:05:20.309672+0000","last_became_peered":"2023-10-09T11:05:20.309672+0000","last_unstale":"2023-10-09T11:10:31.445428+0000","last_undegraded":"2023-10-09T11:10:31.445428+0000","last_fullsized":"2023-10-09T11:10:31.445428+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:50.211465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":164,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443653+0000","last_change":"2023-10-09T11:07:33.228001+0000","last_active":"2023-10-09T11:10:31.443653+0000","last_peered":"2023-10-09T11:10:31.443653+0000","last_clean":"2023-10-09T11:10:31.443653+0000","last_became_active":"2023-10-09T11:07:33.227691+0000","last_became_peered":"2023-10-09T11:07:33.227691+0000","last_unstale":"2023-10-09T11:10:31.443653+0000","last_undegraded":"2023-10-09T11:10:31.443653+0000","last_fullsized":"2023-10-09T11:10:31.443653+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:24:16.098138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":851,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.363813+0000","last_change":"2023-10-09T11:07:36.284927+0000","last_active":"2023-10-09T11:10:32.363813+0000","last_peered":"2023-10-09T11:10:32.363813+0000","last_clean":"2023-10-09T11:10:32.363813+0000","last_became_active":"2023-10-09T11:07:36.284631+0000","last_became_peered":"2023-10-09T11:07:36.284631+0000","last_unstale":"2023-10-09T11:10:32.363813+0000","last_undegraded":"2023-10-09T11:10:32.363813+0000","last_fullsized":"2023-10-09T11:10:32.363813+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:37.220243+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":83,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364596+0000","last_change":"2023-10-09T11:07:33.273503+0000","last_active":"2023-10-09T11:10:32.364596+0000","last_peered":"2023-10-09T11:10:32.364596+0000","last_clean":"2023-10-09T11:10:32.364596+0000","last_became_active":"2023-10-09T11:07:33.242980+0000","last_became_peered":"2023-10-09T11:07:33.242980+0000","last_unstale":"2023-10-09T11:10:32.364596+0000","last_undegraded":"2023-10-09T11:10:32.364596+0000","last_fullsized":"2023-10-09T11:10:32.364596+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:18.934972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":579,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445459+0000","last_change":"2023-10-09T11:07:33.304363+0000","last_active":"2023-10-09T11:10:31.445459+0000","last_peered":"2023-10-09T11:10:31.445459+0000","last_clean":"2023-10-09T11:10:31.445459+0000","last_became_active":"2023-10-09T11:07:33.304024+0000","last_became_peered":"2023-10-09T11:07:33.304024+0000","last_unstale":"2023-10-09T11:10:31.445459+0000","last_undegraded":"2023-10-09T11:10:31.445459+0000","last_fullsized":"2023-10-09T11:10:31.445459+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:21:51.522688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":1001,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443366+0000","last_change":"2023-10-09T11:05:23.316129+0000","last_active":"2023-10-09T11:10:31.443366+0000","last_peered":"2023-10-09T11:10:31.443366+0000","last_clean":"2023-10-09T11:10:31.443366+0000","last_became_active":"2023-10-09T11:05:23.315792+0000","last_became_peered":"2023-10-09T11:05:23.315792+0000","last_unstale":"2023-10-09T11:10:31.443366+0000","last_undegraded":"2023-10-09T11:10:31.443366+0000","last_fullsized":"2023-10-09T11:10:31.443366+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:51.770547+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":83,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364420+0000","last_change":"2023-10-09T11:07:33.256691+0000","last_active":"2023-10-09T11:10:32.364420+0000","last_peered":"2023-10-09T11:10:32.364420+0000","last_clean":"2023-10-09T11:10:32.364420+0000","last_became_active":"2023-10-09T11:07:33.229554+0000","last_became_peered":"2023-10-09T11:07:33.229554+0000","last_unstale":"2023-10-09T11:10:32.364420+0000","last_undegraded":"2023-10-09T11:10:32.364420+0000","last_fullsized":"2023-10-09T11:10:32.364420+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:57.414795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":294,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578539+0000","last_change":"2023-10-09T11:10:11.520169+0000","last_active":"2023-10-09T11:10:32.578539+0000","last_peered":"2023-10-09T11:10:32.578539+0000","last_clean":"2023-10-09T11:10:32.578539+0000","last_became_active":"2023-10-09T11:10:11.519857+0000","last_became_peered":"2023-10-09T11:10:11.519857+0000","last_unstale":"2023-10-09T11:10:32.578539+0000","last_undegraded":"2023-10-09T11:10:32.578539+0000","last_fullsized":"2023-10-09T11:10:32.578539+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:48.203519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":759,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445841+0000","last_change":"2023-10-09T11:10:20.449427+0000","last_active":"2023-10-09T11:10:31.445841+0000","last_peered":"2023-10-09T11:10:31.445841+0000","last_clean":"2023-10-09T11:10:31.445841+0000","last_became_active":"2023-10-09T11:10:20.448905+0000","last_became_peered":"2023-10-09T11:10:20.448905+0000","last_unstale":"2023-10-09T11:10:31.445841+0000","last_undegraded":"2023-10-09T11:10:31.445841+0000","last_fullsized":"2023-10-09T11:10:31.445841+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:49:21.622527+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":83,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364690+0000","last_change":"2023-10-09T11:07:33.271238+0000","last_active":"2023-10-09T11:10:32.364690+0000","last_peered":"2023-10-09T11:10:32.364690+0000","last_clean":"2023-10-09T11:10:32.364690+0000","last_became_active":"2023-10-09T11:07:33.241886+0000","last_became_peered":"2023-10-09T11:07:33.241886+0000","last_unstale":"2023-10-09T11:10:32.364690+0000","last_undegraded":"2023-10-09T11:10:32.364690+0000","last_fullsized":"2023-10-09T11:10:32.364690+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:48.882589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":147,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445946+0000","last_change":"2023-10-09T11:07:33.244131+0000","last_active":"2023-10-09T11:10:31.445946+0000","last_peered":"2023-10-09T11:10:31.445946+0000","last_clean":"2023-10-09T11:10:31.445946+0000","last_became_active":"2023-10-09T11:07:33.243797+0000","last_became_peered":"2023-10-09T11:07:33.243797+0000","last_unstale":"2023-10-09T11:10:31.445946+0000","last_undegraded":"2023-10-09T11:10:31.445946+0000","last_fullsized":"2023-10-09T11:10:31.445946+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:44.851043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":1085,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.579133+0000","last_change":"2023-10-09T11:10:11.520963+0000","last_active":"2023-10-09T11:10:32.579133+0000","last_peered":"2023-10-09T11:10:32.579133+0000","last_clean":"2023-10-09T11:10:32.579133+0000","last_became_active":"2023-10-09T11:10:11.520593+0000","last_became_peered":"2023-10-09T11:10:11.520593+0000","last_unstale":"2023-10-09T11:10:32.579133+0000","last_undegraded":"2023-10-09T11:10:32.579133+0000","last_fullsized":"2023-10-09T11:10:32.579133+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:52.567104+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":475,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446426+0000","last_change":"2023-10-09T11:05:20.309385+0000","last_active":"2023-10-09T11:10:31.446426+0000","last_peered":"2023-10-09T11:10:31.446426+0000","last_clean":"2023-10-09T11:10:31.446426+0000","last_became_active":"2023-10-09T11:05:20.309034+0000","last_became_peered":"2023-10-09T11:05:20.309034+0000","last_unstale":"2023-10-09T11:10:31.446426+0000","last_undegraded":"2023-10-09T11:10:31.446426+0000","last_fullsized":"2023-10-09T11:10:31.446426+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:31.857485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":472,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578583+0000","last_change":"2023-10-09T11:10:02.477873+0000","last_active":"2023-10-09T11:10:32.578583+0000","last_peered":"2023-10-09T11:10:32.578583+0000","last_clean":"2023-10-09T11:10:32.578583+0000","last_became_active":"2023-10-09T11:10:02.477370+0000","last_became_peered":"2023-10-09T11:10:02.477370+0000","last_unstale":"2023-10-09T11:10:32.578583+0000","last_undegraded":"2023-10-09T11:10:32.578583+0000","last_fullsized":"2023-10-09T11:10:32.578583+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.390703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":773,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.363761+0000","last_change":"2023-10-09T11:07:36.283174+0000","last_active":"2023-10-09T11:10:32.363761+0000","last_peered":"2023-10-09T11:10:32.363761+0000","last_clean":"2023-10-09T11:10:32.363761+0000","last_became_active":"2023-10-09T11:07:36.278007+0000","last_became_peered":"2023-10-09T11:07:36.278007+0000","last_unstale":"2023-10-09T11:10:32.363761+0000","last_undegraded":"2023-10-09T11:10:32.363761+0000","last_fullsized":"2023-10-09T11:10:32.363761+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:51.693969+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":150,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443165+0000","last_change":"2023-10-09T11:04:50.529513+0000","last_active":"2023-10-09T11:10:31.443165+0000","last_peered":"2023-10-09T11:10:31.443165+0000","last_clean":"2023-10-09T11:10:31.443165+0000","last_became_active":"2023-10-09T11:04:50.339671+0000","last_became_peered":"2023-10-09T11:04:50.339671+0000","last_unstale":"2023-10-09T11:10:31.443165+0000","last_undegraded":"2023-10-09T11:10:31.443165+0000","last_fullsized":"2023-10-09T11:10:31.443165+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:24.473000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":226,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578822+0000","last_change":"2023-10-09T11:10:11.522176+0000","last_active":"2023-10-09T11:10:32.578822+0000","last_peered":"2023-10-09T11:10:32.578822+0000","last_clean":"2023-10-09T11:10:32.578822+0000","last_became_active":"2023-10-09T11:10:11.521859+0000","last_became_peered":"2023-10-09T11:10:11.521859+0000","last_unstale":"2023-10-09T11:10:32.578822+0000","last_undegraded":"2023-10-09T11:10:32.578822+0000","last_fullsized":"2023-10-09T11:10:32.578822+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:56:53.974041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":875,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445655+0000","last_change":"2023-10-09T11:04:53.272323+0000","last_active":"2023-10-09T11:10:31.445655+0000","last_peered":"2023-10-09T11:10:31.445655+0000","last_clean":"2023-10-09T11:10:31.445655+0000","last_became_active":"2023-10-09T11:04:53.272010+0000","last_became_peered":"2023-10-09T11:04:53.272010+0000","last_unstale":"2023-10-09T11:10:31.445655+0000","last_undegraded":"2023-10-09T11:10:31.445655+0000","last_fullsized":"2023-10-09T11:10:31.445655+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:23.913605+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":536,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446036+0000","last_change":"2023-10-09T11:10:20.506780+0000","last_active":"2023-10-09T11:10:31.446036+0000","last_peered":"2023-10-09T11:10:31.446036+0000","last_clean":"2023-10-09T11:10:31.446036+0000","last_became_active":"2023-10-09T11:10:20.506482+0000","last_became_peered":"2023-10-09T11:10:20.506482+0000","last_unstale":"2023-10-09T11:10:31.446036+0000","last_undegraded":"2023-10-09T11:10:31.446036+0000","last_fullsized":"2023-10-09T11:10:31.446036+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:20.610043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":246,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.579245+0000","last_change":"2023-10-09T11:10:11.521500+0000","last_active":"2023-10-09T11:10:32.579245+0000","last_peered":"2023-10-09T11:10:32.579245+0000","last_clean":"2023-10-09T11:10:32.579245+0000","last_became_active":"2023-10-09T11:10:11.519785+0000","last_became_peered":"2023-10-09T11:10:11.519785+0000","last_unstale":"2023-10-09T11:10:32.579245+0000","last_undegraded":"2023-10-09T11:10:32.579245+0000","last_fullsized":"2023-10-09T11:10:32.579245+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:59:20.899382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":970,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.226516+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:10:32.226516+0000","last_peered":"2023-10-09T11:10:32.226516+0000","last_clean":"2023-10-09T11:10:32.226516+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:10:32.226516+0000","last_undegraded":"2023-10-09T11:10:32.226516+0000","last_fullsized":"2023-10-09T11:10:32.226516+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":588,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578422+0000","last_change":"2023-10-09T11:10:11.520888+0000","last_active":"2023-10-09T11:10:32.578422+0000","last_peered":"2023-10-09T11:10:32.578422+0000","last_clean":"2023-10-09T11:10:32.578422+0000","last_became_active":"2023-10-09T11:10:11.520494+0000","last_became_peered":"2023-10-09T11:10:11.520494+0000","last_unstale":"2023-10-09T11:10:32.578422+0000","last_undegraded":"2023-10-09T11:10:32.578422+0000","last_fullsized":"2023-10-09T11:10:32.578422+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:12.005423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":256,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.579236+0000","last_change":"2023-10-09T11:10:11.523653+0000","last_active":"2023-10-09T11:10:32.579236+0000","last_peered":"2023-10-09T11:10:32.579236+0000","last_clean":"2023-10-09T11:10:32.579236+0000","last_became_active":"2023-10-09T11:10:11.523317+0000","last_became_peered":"2023-10-09T11:10:11.523317+0000","last_unstale":"2023-10-09T11:10:32.579236+0000","last_undegraded":"2023-10-09T11:10:32.579236+0000","last_fullsized":"2023-10-09T11:10:32.579236+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:31.551257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":859,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446290+0000","last_change":"2023-10-09T11:05:23.329329+0000","last_active":"2023-10-09T11:10:31.446290+0000","last_peered":"2023-10-09T11:10:31.446290+0000","last_clean":"2023-10-09T11:10:31.446290+0000","last_became_active":"2023-10-09T11:05:23.328976+0000","last_became_peered":"2023-10-09T11:05:23.328976+0000","last_unstale":"2023-10-09T11:10:31.446290+0000","last_undegraded":"2023-10-09T11:10:31.446290+0000","last_fullsized":"2023-10-09T11:10:31.446290+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:41.848184+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":164,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.442989+0000","last_change":"2023-10-09T11:07:33.236133+0000","last_active":"2023-10-09T11:10:31.442989+0000","last_peered":"2023-10-09T11:10:31.442989+0000","last_clean":"2023-10-09T11:10:31.442989+0000","last_became_active":"2023-10-09T11:07:33.235818+0000","last_became_peered":"2023-10-09T11:07:33.235818+0000","last_unstale":"2023-10-09T11:10:31.442989+0000","last_undegraded":"2023-10-09T11:10:31.442989+0000","last_fullsized":"2023-10-09T11:10:31.442989+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:20.627846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":325,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.227071+0000","last_change":"2023-10-09T11:04:50.332978+0000","last_active":"2023-10-09T11:10:32.227071+0000","last_peered":"2023-10-09T11:10:32.227071+0000","last_clean":"2023-10-09T11:10:32.227071+0000","last_became_active":"2023-10-09T11:04:50.332735+0000","last_became_peered":"2023-10-09T11:04:50.332735+0000","last_unstale":"2023-10-09T11:10:32.227071+0000","last_undegraded":"2023-10-09T11:10:32.227071+0000","last_fullsized":"2023-10-09T11:10:32.227071+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:39:44.659751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":995,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445434+0000","last_change":"2023-10-09T11:10:20.447863+0000","last_active":"2023-10-09T11:10:31.445434+0000","last_peered":"2023-10-09T11:10:31.445434+0000","last_clean":"2023-10-09T11:10:31.445434+0000","last_became_active":"2023-10-09T11:10:20.446919+0000","last_became_peered":"2023-10-09T11:10:20.446919+0000","last_unstale":"2023-10-09T11:10:31.445434+0000","last_undegraded":"2023-10-09T11:10:31.445434+0000","last_fullsized":"2023-10-09T11:10:31.445434+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:43.348375+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"592'150","reported_seq":905,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.555247+0000","last_change":"2023-10-09T11:09:20.424914+0000","last_active":"2023-10-09T11:10:33.555247+0000","last_peered":"2023-10-09T11:10:33.555247+0000","last_clean":"2023-10-09T11:10:33.555247+0000","last_became_active":"2023-10-09T11:07:33.226176+0000","last_became_peered":"2023-10-09T11:07:33.226176+0000","last_unstale":"2023-10-09T11:10:33.555247+0000","last_undegraded":"2023-10-09T11:10:33.555247+0000","last_fullsized":"2023-10-09T11:10:33.555247+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:16.888732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034113299999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":876,"num_read_kb":2947,"num_write":148,"num_write_kb":2179,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":492,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446488+0000","last_change":"2023-10-09T11:04:50.341353+0000","last_active":"2023-10-09T11:10:31.446488+0000","last_peered":"2023-10-09T11:10:31.446488+0000","last_clean":"2023-10-09T11:10:31.446488+0000","last_became_active":"2023-10-09T11:04:50.340801+0000","last_became_peered":"2023-10-09T11:04:50.340801+0000","last_unstale":"2023-10-09T11:10:31.446488+0000","last_undegraded":"2023-10-09T11:10:31.446488+0000","last_fullsized":"2023-10-09T11:10:31.446488+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:00.699069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":302,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.554933+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:10:33.554933+0000","last_peered":"2023-10-09T11:10:33.554933+0000","last_clean":"2023-10-09T11:10:33.554933+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:10:33.554933+0000","last_undegraded":"2023-10-09T11:10:33.554933+0000","last_fullsized":"2023-10-09T11:10:33.554933+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":817,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445421+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:10:31.445421+0000","last_peered":"2023-10-09T11:10:31.445421+0000","last_clean":"2023-10-09T11:10:31.445421+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:10:31.445421+0000","last_undegraded":"2023-10-09T11:10:31.445421+0000","last_fullsized":"2023-10-09T11:10:31.445421+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"592'149","reported_seq":1234,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446059+0000","last_change":"2023-10-09T11:09:20.477238+0000","last_active":"2023-10-09T11:10:31.446059+0000","last_peered":"2023-10-09T11:10:31.446059+0000","last_clean":"2023-10-09T11:10:31.446059+0000","last_became_active":"2023-10-09T11:05:20.315471+0000","last_became_peered":"2023-10-09T11:05:20.315471+0000","last_unstale":"2023-10-09T11:10:31.446059+0000","last_undegraded":"2023-10-09T11:10:31.446059+0000","last_fullsized":"2023-10-09T11:10:31.446059+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:47.548763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051877200000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1181,"num_read_kb":1421,"num_write":125,"num_write_kb":1278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":473,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578918+0000","last_change":"2023-10-09T11:10:11.705748+0000","last_active":"2023-10-09T11:10:32.578918+0000","last_peered":"2023-10-09T11:10:32.578918+0000","last_clean":"2023-10-09T11:10:32.578918+0000","last_became_active":"2023-10-09T11:10:11.705401+0000","last_became_peered":"2023-10-09T11:10:11.705401+0000","last_unstale":"2023-10-09T11:10:32.578918+0000","last_undegraded":"2023-10-09T11:10:32.578918+0000","last_fullsized":"2023-10-09T11:10:32.578918+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:48.364690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":335,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.555091+0000","last_change":"2023-10-09T11:04:50.311579+0000","last_active":"2023-10-09T11:10:33.555091+0000","last_peered":"2023-10-09T11:10:33.555091+0000","last_clean":"2023-10-09T11:10:33.555091+0000","last_became_active":"2023-10-09T11:04:50.311270+0000","last_became_peered":"2023-10-09T11:04:50.311270+0000","last_unstale":"2023-10-09T11:10:33.555091+0000","last_undegraded":"2023-10-09T11:10:33.555091+0000","last_fullsized":"2023-10-09T11:10:33.555091+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:49.850302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":971,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364221+0000","last_change":"2023-10-09T11:07:36.284280+0000","last_active":"2023-10-09T11:10:32.364221+0000","last_peered":"2023-10-09T11:10:32.364221+0000","last_clean":"2023-10-09T11:10:32.364221+0000","last_became_active":"2023-10-09T11:07:36.283966+0000","last_became_peered":"2023-10-09T11:07:36.283966+0000","last_unstale":"2023-10-09T11:10:32.364221+0000","last_undegraded":"2023-10-09T11:10:32.364221+0000","last_fullsized":"2023-10-09T11:10:32.364221+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:54.128331+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"591'48","reported_seq":630,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578199+0000","last_change":"2023-10-09T11:09:20.421917+0000","last_active":"2023-10-09T11:10:32.578199+0000","last_peered":"2023-10-09T11:10:32.578199+0000","last_clean":"2023-10-09T11:10:32.578199+0000","last_became_active":"2023-10-09T11:04:50.381253+0000","last_became_peered":"2023-10-09T11:04:50.381253+0000","last_unstale":"2023-10-09T11:10:32.578199+0000","last_undegraded":"2023-10-09T11:10:32.578199+0000","last_fullsized":"2023-10-09T11:10:32.578199+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":48,"log_dups_size":0,"ondisk_log_size":48,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:24.030274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001144136,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":297,"num_read_kb":2229,"num_write":37,"num_write_kb":2061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":320,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.226660+0000","last_change":"2023-10-09T11:05:20.298542+0000","last_active":"2023-10-09T11:10:32.226660+0000","last_peered":"2023-10-09T11:10:32.226660+0000","last_clean":"2023-10-09T11:10:32.226660+0000","last_became_active":"2023-10-09T11:05:20.298119+0000","last_became_peered":"2023-10-09T11:05:20.298119+0000","last_unstale":"2023-10-09T11:10:32.226660+0000","last_undegraded":"2023-10-09T11:10:32.226660+0000","last_fullsized":"2023-10-09T11:10:32.226660+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:00.827214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":299,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.226516+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:10:32.226516+0000","last_peered":"2023-10-09T11:10:32.226516+0000","last_clean":"2023-10-09T11:10:32.226516+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:10:32.226516+0000","last_undegraded":"2023-10-09T11:10:32.226516+0000","last_fullsized":"2023-10-09T11:10:32.226516+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"592'125","reported_seq":813,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446343+0000","last_change":"2023-10-09T11:09:20.477093+0000","last_active":"2023-10-09T11:10:31.446343+0000","last_peered":"2023-10-09T11:10:31.446343+0000","last_clean":"2023-10-09T11:10:31.446343+0000","last_became_active":"2023-10-09T11:07:33.226289+0000","last_became_peered":"2023-10-09T11:07:33.226289+0000","last_unstale":"2023-10-09T11:10:31.446343+0000","last_undegraded":"2023-10-09T11:10:31.446343+0000","last_fullsized":"2023-10-09T11:10:31.446343+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:15.605946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026939299999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":322,"num_read_kb":786,"num_write":116,"num_write_kb":327,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":916,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.363759+0000","last_change":"2023-10-09T11:07:36.290700+0000","last_active":"2023-10-09T11:10:32.363759+0000","last_peered":"2023-10-09T11:10:32.363759+0000","last_clean":"2023-10-09T11:10:32.363759+0000","last_became_active":"2023-10-09T11:07:36.290357+0000","last_became_peered":"2023-10-09T11:07:36.290357+0000","last_unstale":"2023-10-09T11:10:32.363759+0000","last_undegraded":"2023-10-09T11:10:32.363759+0000","last_fullsized":"2023-10-09T11:10:32.363759+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:34.494777+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":155,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443030+0000","last_change":"2023-10-09T11:05:20.310666+0000","last_active":"2023-10-09T11:10:31.443030+0000","last_peered":"2023-10-09T11:10:31.443030+0000","last_clean":"2023-10-09T11:10:31.443030+0000","last_became_active":"2023-10-09T11:05:20.310413+0000","last_became_peered":"2023-10-09T11:05:20.310413+0000","last_unstale":"2023-10-09T11:10:31.443030+0000","last_undegraded":"2023-10-09T11:10:31.443030+0000","last_fullsized":"2023-10-09T11:10:31.443030+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:34.169905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":474,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445593+0000","last_change":"2023-10-09T11:10:20.449809+0000","last_active":"2023-10-09T11:10:31.445593+0000","last_peered":"2023-10-09T11:10:31.445593+0000","last_clean":"2023-10-09T11:10:31.445593+0000","last_became_active":"2023-10-09T11:10:20.449420+0000","last_became_peered":"2023-10-09T11:10:20.449420+0000","last_unstale":"2023-10-09T11:10:31.445593+0000","last_undegraded":"2023-10-09T11:10:31.445593+0000","last_fullsized":"2023-10-09T11:10:31.445593+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:01.442053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":991,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.555599+0000","last_change":"2023-10-09T11:05:23.322074+0000","last_active":"2023-10-09T11:10:33.555599+0000","last_peered":"2023-10-09T11:10:33.555599+0000","last_clean":"2023-10-09T11:10:33.555599+0000","last_became_active":"2023-10-09T11:05:23.321749+0000","last_became_peered":"2023-10-09T11:05:23.321749+0000","last_unstale":"2023-10-09T11:10:33.555599+0000","last_undegraded":"2023-10-09T11:10:33.555599+0000","last_fullsized":"2023-10-09T11:10:33.555599+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:07.010932+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"591'46","reported_seq":373,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364428+0000","last_change":"2023-10-09T11:09:20.432327+0000","last_active":"2023-10-09T11:10:32.364428+0000","last_peered":"2023-10-09T11:10:32.364428+0000","last_clean":"2023-10-09T11:10:32.364428+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:10:32.364428+0000","last_undegraded":"2023-10-09T11:10:32.364428+0000","last_fullsized":"2023-10-09T11:10:32.364428+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":46,"log_dups_size":0,"ondisk_log_size":46,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000350947,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":271,"num_read_kb":1543,"num_write":39,"num_write_kb":1183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":212,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578385+0000","last_change":"2023-10-09T11:10:02.477481+0000","last_active":"2023-10-09T11:10:32.578385+0000","last_peered":"2023-10-09T11:10:32.578385+0000","last_clean":"2023-10-09T11:10:32.578385+0000","last_became_active":"2023-10-09T11:10:02.477147+0000","last_became_peered":"2023-10-09T11:10:02.477147+0000","last_unstale":"2023-10-09T11:10:32.578385+0000","last_undegraded":"2023-10-09T11:10:32.578385+0000","last_fullsized":"2023-10-09T11:10:32.578385+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:47:05.471856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":509,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578799+0000","last_change":"2023-10-09T11:10:02.478037+0000","last_active":"2023-10-09T11:10:32.578799+0000","last_peered":"2023-10-09T11:10:32.578799+0000","last_clean":"2023-10-09T11:10:32.578799+0000","last_became_active":"2023-10-09T11:10:02.477517+0000","last_became_peered":"2023-10-09T11:10:02.477517+0000","last_unstale":"2023-10-09T11:10:32.578799+0000","last_undegraded":"2023-10-09T11:10:32.578799+0000","last_fullsized":"2023-10-09T11:10:32.578799+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:54.404295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":792,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445763+0000","last_change":"2023-10-09T11:10:20.447023+0000","last_active":"2023-10-09T11:10:31.445763+0000","last_peered":"2023-10-09T11:10:31.445763+0000","last_clean":"2023-10-09T11:10:31.445763+0000","last_became_active":"2023-10-09T11:10:20.446654+0000","last_became_peered":"2023-10-09T11:10:20.446654+0000","last_unstale":"2023-10-09T11:10:31.445763+0000","last_undegraded":"2023-10-09T11:10:31.445763+0000","last_fullsized":"2023-10-09T11:10:31.445763+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:51.438822+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"591'57","reported_seq":277,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364494+0000","last_change":"2023-10-09T11:09:20.432685+0000","last_active":"2023-10-09T11:10:32.364494+0000","last_peered":"2023-10-09T11:10:32.364494+0000","last_clean":"2023-10-09T11:10:32.364494+0000","last_became_active":"2023-10-09T11:07:33.229061+0000","last_became_peered":"2023-10-09T11:07:33.229061+0000","last_unstale":"2023-10-09T11:10:32.364494+0000","last_undegraded":"2023-10-09T11:10:32.364494+0000","last_fullsized":"2023-10-09T11:10:32.364494+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":57,"log_dups_size":0,"ondisk_log_size":57,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:30.978293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043334800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":262,"num_read_kb":1253,"num_write":50,"num_write_kb":1077,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":302,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.555330+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:10:33.555330+0000","last_peered":"2023-10-09T11:10:33.555330+0000","last_clean":"2023-10-09T11:10:33.555330+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:10:33.555330+0000","last_undegraded":"2023-10-09T11:10:33.555330+0000","last_fullsized":"2023-10-09T11:10:33.555330+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":529,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578429+0000","last_change":"2023-10-09T11:10:02.479721+0000","last_active":"2023-10-09T11:10:32.578429+0000","last_peered":"2023-10-09T11:10:32.578429+0000","last_clean":"2023-10-09T11:10:32.578429+0000","last_became_active":"2023-10-09T11:10:02.479427+0000","last_became_peered":"2023-10-09T11:10:02.479427+0000","last_unstale":"2023-10-09T11:10:32.578429+0000","last_undegraded":"2023-10-09T11:10:32.578429+0000","last_fullsized":"2023-10-09T11:10:32.578429+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:31.125346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":928,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.579132+0000","last_change":"2023-10-09T11:10:11.523340+0000","last_active":"2023-10-09T11:10:32.579132+0000","last_peered":"2023-10-09T11:10:32.579132+0000","last_clean":"2023-10-09T11:10:32.579132+0000","last_became_active":"2023-10-09T11:10:11.522841+0000","last_became_peered":"2023-10-09T11:10:11.522841+0000","last_unstale":"2023-10-09T11:10:32.579132+0000","last_undegraded":"2023-10-09T11:10:32.579132+0000","last_fullsized":"2023-10-09T11:10:32.579132+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:41.203109+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":176,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443616+0000","last_change":"2023-10-09T11:07:33.246378+0000","last_active":"2023-10-09T11:10:31.443616+0000","last_peered":"2023-10-09T11:10:31.443616+0000","last_clean":"2023-10-09T11:10:31.443616+0000","last_became_active":"2023-10-09T11:07:33.228019+0000","last_became_peered":"2023-10-09T11:07:33.228019+0000","last_unstale":"2023-10-09T11:10:31.443616+0000","last_undegraded":"2023-10-09T11:10:31.443616+0000","last_fullsized":"2023-10-09T11:10:31.443616+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:18.511517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"592'319","reported_seq":1609,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.227139+0000","last_change":"2023-10-09T11:09:20.427659+0000","last_active":"2023-10-09T11:10:32.227139+0000","last_peered":"2023-10-09T11:10:32.227139+0000","last_clean":"2023-10-09T11:10:32.227139+0000","last_became_active":"2023-10-09T11:05:20.309827+0000","last_became_peered":"2023-10-09T11:05:20.309827+0000","last_unstale":"2023-10-09T11:10:32.227139+0000","last_undegraded":"2023-10-09T11:10:32.227139+0000","last_fullsized":"2023-10-09T11:10:32.227139+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":319,"log_dups_size":0,"ondisk_log_size":319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:23.934383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001122743,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2478,"num_read_kb":2521,"num_write":249,"num_write_kb":324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":165,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.459452+0000","last_change":"2023-10-09T11:07:33.217595+0000","last_active":"2023-10-09T11:10:33.459452+0000","last_peered":"2023-10-09T11:10:33.459452+0000","last_clean":"2023-10-09T11:10:33.459452+0000","last_became_active":"2023-10-09T11:07:33.217155+0000","last_became_peered":"2023-10-09T11:07:33.217155+0000","last_unstale":"2023-10-09T11:10:33.459452+0000","last_undegraded":"2023-10-09T11:10:33.459452+0000","last_fullsized":"2023-10-09T11:10:33.459452+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:55.127329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":783,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443134+0000","last_change":"2023-10-09T11:07:36.284491+0000","last_active":"2023-10-09T11:10:31.443134+0000","last_peered":"2023-10-09T11:10:31.443134+0000","last_clean":"2023-10-09T11:10:31.443134+0000","last_became_active":"2023-10-09T11:07:36.284249+0000","last_became_peered":"2023-10-09T11:07:36.284249+0000","last_unstale":"2023-10-09T11:10:31.443134+0000","last_undegraded":"2023-10-09T11:10:31.443134+0000","last_fullsized":"2023-10-09T11:10:31.443134+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:03.653371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:03.653371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:09.629417+0000","scrub_duration":0.001576771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"592'82","reported_seq":273,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445963+0000","last_change":"2023-10-09T11:10:20.448154+0000","last_active":"2023-10-09T11:10:31.445963+0000","last_peered":"2023-10-09T11:10:31.445963+0000","last_clean":"2023-10-09T11:10:31.445963+0000","last_became_active":"2023-10-09T11:10:20.447384+0000","last_became_peered":"2023-10-09T11:10:20.447384+0000","last_unstale":"2023-10-09T11:10:31.445963+0000","last_undegraded":"2023-10-09T11:10:31.445963+0000","last_fullsized":"2023-10-09T11:10:31.445963+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":82,"log_dups_size":0,"ondisk_log_size":82,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:03:31.024074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075188800000000004,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":544,"num_write":82,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":493,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578852+0000","last_change":"2023-10-09T11:10:02.477655+0000","last_active":"2023-10-09T11:10:32.578852+0000","last_peered":"2023-10-09T11:10:32.578852+0000","last_clean":"2023-10-09T11:10:32.578852+0000","last_became_active":"2023-10-09T11:10:02.477293+0000","last_became_peered":"2023-10-09T11:10:02.477293+0000","last_unstale":"2023-10-09T11:10:32.578852+0000","last_undegraded":"2023-10-09T11:10:32.578852+0000","last_fullsized":"2023-10-09T11:10:32.578852+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:24.604828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":358,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578414+0000","last_change":"2023-10-09T11:10:11.671501+0000","last_active":"2023-10-09T11:10:32.578414+0000","last_peered":"2023-10-09T11:10:32.578414+0000","last_clean":"2023-10-09T11:10:32.578414+0000","last_became_active":"2023-10-09T11:10:11.671154+0000","last_became_peered":"2023-10-09T11:10:11.671154+0000","last_unstale":"2023-10-09T11:10:32.578414+0000","last_undegraded":"2023-10-09T11:10:32.578414+0000","last_fullsized":"2023-10-09T11:10:32.578414+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:35.689307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":1027,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443242+0000","last_change":"2023-10-09T11:05:23.329886+0000","last_active":"2023-10-09T11:10:31.443242+0000","last_peered":"2023-10-09T11:10:31.443242+0000","last_clean":"2023-10-09T11:10:31.443242+0000","last_became_active":"2023-10-09T11:05:23.329561+0000","last_became_peered":"2023-10-09T11:05:23.329561+0000","last_unstale":"2023-10-09T11:10:31.443242+0000","last_undegraded":"2023-10-09T11:10:31.443242+0000","last_fullsized":"2023-10-09T11:10:31.443242+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:46.933799+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":501,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578508+0000","last_change":"2023-10-09T11:10:02.479983+0000","last_active":"2023-10-09T11:10:32.578508+0000","last_peered":"2023-10-09T11:10:32.578508+0000","last_clean":"2023-10-09T11:10:32.578508+0000","last_became_active":"2023-10-09T11:10:02.479627+0000","last_became_peered":"2023-10-09T11:10:02.479627+0000","last_unstale":"2023-10-09T11:10:32.578508+0000","last_undegraded":"2023-10-09T11:10:32.578508+0000","last_fullsized":"2023-10-09T11:10:32.578508+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:14.632490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":402,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578391+0000","last_change":"2023-10-09T11:04:50.510688+0000","last_active":"2023-10-09T11:10:32.578391+0000","last_peered":"2023-10-09T11:10:32.578391+0000","last_clean":"2023-10-09T11:10:32.578391+0000","last_became_active":"2023-10-09T11:04:50.507404+0000","last_became_peered":"2023-10-09T11:04:50.507404+0000","last_unstale":"2023-10-09T11:10:32.578391+0000","last_undegraded":"2023-10-09T11:10:32.578391+0000","last_fullsized":"2023-10-09T11:10:32.578391+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:13.663394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":985,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.459948+0000","last_change":"2023-10-09T11:05:23.321040+0000","last_active":"2023-10-09T11:10:33.459948+0000","last_peered":"2023-10-09T11:10:33.459948+0000","last_clean":"2023-10-09T11:10:33.459948+0000","last_became_active":"2023-10-09T11:05:23.320674+0000","last_became_peered":"2023-10-09T11:05:23.320674+0000","last_unstale":"2023-10-09T11:10:33.459948+0000","last_undegraded":"2023-10-09T11:10:33.459948+0000","last_fullsized":"2023-10-09T11:10:33.459948+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.527660+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":444,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446353+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:10:31.446353+0000","last_peered":"2023-10-09T11:10:31.446353+0000","last_clean":"2023-10-09T11:10:31.446353+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:10:31.446353+0000","last_undegraded":"2023-10-09T11:10:31.446353+0000","last_fullsized":"2023-10-09T11:10:31.446353+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":383,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.579008+0000","last_change":"2023-10-09T11:10:11.516290+0000","last_active":"2023-10-09T11:10:32.579008+0000","last_peered":"2023-10-09T11:10:32.579008+0000","last_clean":"2023-10-09T11:10:32.579008+0000","last_became_active":"2023-10-09T11:10:11.515955+0000","last_became_peered":"2023-10-09T11:10:11.515955+0000","last_unstale":"2023-10-09T11:10:32.579008+0000","last_undegraded":"2023-10-09T11:10:32.579008+0000","last_fullsized":"2023-10-09T11:10:32.579008+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:59.774818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":595,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446480+0000","last_change":"2023-10-09T11:05:20.327032+0000","last_active":"2023-10-09T11:10:31.446480+0000","last_peered":"2023-10-09T11:10:31.446480+0000","last_clean":"2023-10-09T11:10:31.446480+0000","last_became_active":"2023-10-09T11:05:20.320193+0000","last_became_peered":"2023-10-09T11:05:20.320193+0000","last_unstale":"2023-10-09T11:10:31.446480+0000","last_undegraded":"2023-10-09T11:10:31.446480+0000","last_fullsized":"2023-10-09T11:10:31.446480+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:44.862890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":373,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578539+0000","last_change":"2023-10-09T11:10:11.705871+0000","last_active":"2023-10-09T11:10:32.578539+0000","last_peered":"2023-10-09T11:10:32.578539+0000","last_clean":"2023-10-09T11:10:32.578539+0000","last_became_active":"2023-10-09T11:10:11.705528+0000","last_became_peered":"2023-10-09T11:10:11.705528+0000","last_unstale":"2023-10-09T11:10:32.578539+0000","last_undegraded":"2023-10-09T11:10:32.578539+0000","last_fullsized":"2023-10-09T11:10:32.578539+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:19.337219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":802,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.363769+0000","last_change":"2023-10-09T11:08:39.540029+0000","last_active":"2023-10-09T11:10:32.363769+0000","last_peered":"2023-10-09T11:10:32.363769+0000","last_clean":"2023-10-09T11:10:32.363769+0000","last_became_active":"2023-10-09T11:07:36.291077+0000","last_became_peered":"2023-10-09T11:07:36.291077+0000","last_unstale":"2023-10-09T11:10:32.363769+0000","last_undegraded":"2023-10-09T11:10:32.363769+0000","last_fullsized":"2023-10-09T11:10:32.363769+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.539966+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.539966+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:20.767812+0000","scrub_duration":0.0016245929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":156,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.459994+0000","last_change":"2023-10-09T11:05:20.299321+0000","last_active":"2023-10-09T11:10:33.459994+0000","last_peered":"2023-10-09T11:10:33.459994+0000","last_clean":"2023-10-09T11:10:33.459994+0000","last_became_active":"2023-10-09T11:05:20.299019+0000","last_became_peered":"2023-10-09T11:05:20.299019+0000","last_unstale":"2023-10-09T11:10:33.459994+0000","last_undegraded":"2023-10-09T11:10:33.459994+0000","last_fullsized":"2023-10-09T11:10:33.459994+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:08.645909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":146,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443275+0000","last_change":"2023-10-09T11:04:50.366890+0000","last_active":"2023-10-09T11:10:31.443275+0000","last_peered":"2023-10-09T11:10:31.443275+0000","last_clean":"2023-10-09T11:10:31.443275+0000","last_became_active":"2023-10-09T11:04:50.366558+0000","last_became_peered":"2023-10-09T11:04:50.366558+0000","last_unstale":"2023-10-09T11:10:31.443275+0000","last_undegraded":"2023-10-09T11:10:31.443275+0000","last_fullsized":"2023-10-09T11:10:31.443275+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:50.993153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":919,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.227064+0000","last_change":"2023-10-09T11:04:53.274332+0000","last_active":"2023-10-09T11:10:32.227064+0000","last_peered":"2023-10-09T11:10:32.227064+0000","last_clean":"2023-10-09T11:10:32.227064+0000","last_became_active":"2023-10-09T11:04:53.273798+0000","last_became_peered":"2023-10-09T11:04:53.273798+0000","last_unstale":"2023-10-09T11:10:32.227064+0000","last_undegraded":"2023-10-09T11:10:32.227064+0000","last_fullsized":"2023-10-09T11:10:32.227064+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:04.220970+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":194,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578859+0000","last_change":"2023-10-09T11:10:02.473453+0000","last_active":"2023-10-09T11:10:32.578859+0000","last_peered":"2023-10-09T11:10:32.578859+0000","last_clean":"2023-10-09T11:10:32.578859+0000","last_became_active":"2023-10-09T11:10:02.473105+0000","last_became_peered":"2023-10-09T11:10:02.473105+0000","last_unstale":"2023-10-09T11:10:32.578859+0000","last_undegraded":"2023-10-09T11:10:32.578859+0000","last_fullsized":"2023-10-09T11:10:32.578859+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:11.409980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":562,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578304+0000","last_change":"2023-10-09T11:10:11.523400+0000","last_active":"2023-10-09T11:10:32.578304+0000","last_peered":"2023-10-09T11:10:32.578304+0000","last_clean":"2023-10-09T11:10:32.578304+0000","last_became_active":"2023-10-09T11:10:11.523063+0000","last_became_peered":"2023-10-09T11:10:11.523063+0000","last_unstale":"2023-10-09T11:10:32.578304+0000","last_undegraded":"2023-10-09T11:10:32.578304+0000","last_fullsized":"2023-10-09T11:10:32.578304+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:41:18.121353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":7936,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12694,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13244,"num_write":846,"num_write_kb":12712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332532,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":910328,"kb_used_data":10928,"kb_used_omap":0,"kb_used_meta":899392,"kb_avail":92834824,"statfs":{"total":95995035648,"available":95062859776,"internally_reserved":0,"allocated":11190272,"data_stored":9331566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920977408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:10:23 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":1.022,"15min":1.5409999999999999},"min":{"1min":0.27700000000000002,"5min":0.20599999999999999,"15min":0.20100000000000001},"max":{"1min":0.65100000000000002,"5min":9.8219999999999992,"15min":9.8219999999999992},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.55600000000000005,"5min":1.242,"15min":1.577},"min":{"1min":0.313,"5min":0.24299999999999999,"15min":0.216},"max":{"1min":0.81200000000000006,"5min":15.853,"15min":15.853},"last":0.55600000000000005}]},{"osd":1,"last update":"Mon Oct 9 11:10:28 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.1379999999999999,"15min":1.395},"min":{"1min":0.34300000000000003,"5min":0.222,"15min":0.222},"max":{"1min":0.89000000000000001,"5min":11.619999999999999,"15min":11.619999999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.5880000000000001,"15min":1.7430000000000001},"min":{"1min":0.34899999999999998,"5min":0.25,"15min":0.25},"max":{"1min":0.73299999999999998,"5min":13.096,"15min":13.096},"last":0.88300000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":1.1439999999999999,"15min":1.722},"min":{"1min":0.35099999999999998,"5min":0.309,"15min":0.245},"max":{"1min":0.59499999999999997,"5min":8.5340000000000007,"15min":16.605},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":1.2749999999999999,"15min":1.6830000000000001},"min":{"1min":0.34100000000000003,"5min":0.223,"15min":0.223},"max":{"1min":0.65800000000000003,"5min":16.157,"15min":16.157},"last":0.54100000000000004}]},{"osd":3,"last update":"Mon Oct 9 11:10:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.389,"15min":1.395},"min":{"1min":0.39800000000000002,"5min":0.30499999999999999,"15min":0.25600000000000001},"max":{"1min":0.72999999999999998,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":1.4159999999999999,"15min":1.6659999999999999},"min":{"1min":0.40999999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.85999999999999999,"5min":6.3920000000000003,"15min":8.2330000000000005},"last":0.52200000000000002}]},{"osd":4,"last update":"Mon Oct 9 11:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.752,"15min":1.032},"min":{"1min":0.40600000000000003,"5min":0.247,"15min":0.218},"max":{"1min":0.72599999999999998,"5min":7.4630000000000001,"15min":7.96},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.81699999999999995,"15min":1.22},"min":{"1min":0.36499999999999999,"5min":0.28299999999999997,"15min":0.27900000000000003},"max":{"1min":0.64900000000000002,"5min":9.2759999999999998,"15min":16.481000000000002},"last":0.45300000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.67200000000000004,"15min":1.2070000000000001},"min":{"1min":0.32800000000000001,"5min":0.249,"15min":0.247},"max":{"1min":0.68500000000000005,"5min":5.4800000000000004,"15min":15.414999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.004,"15min":1.1679999999999999},"min":{"1min":0.372,"5min":0.317,"15min":0.26400000000000001},"max":{"1min":0.75,"5min":13.045,"15min":15.369},"last":0.47399999999999998}]},{"osd":6,"last update":"Mon Oct 9 11:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.81699999999999995,"15min":1.1950000000000001},"min":{"1min":0.41999999999999998,"5min":0.34599999999999997,"15min":0.315},"max":{"1min":0.77400000000000002,"5min":4.8079999999999998,"15min":14.159000000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.87,"15min":1.3280000000000001},"min":{"1min":0.39700000000000002,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":0.71399999999999997,"5min":3.9630000000000001,"15min":16.538},"last":0.59399999999999997}]}]},{"osd":6,"up_from":44,"seq":188978561463,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1511724,"kb_used_data":10980,"kb_used_omap":0,"kb_used_meta":1500736,"kb_avail":92233428,"statfs":{"total":95995035648,"available":94447030272,"internally_reserved":0,"allocated":11243520,"data_stored":9396431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1536753664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:10:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.99199999999999999,"15min":1.8480000000000001},"min":{"1min":0.33500000000000002,"5min":0.223,"15min":0.217},"max":{"1min":0.69899999999999995,"5min":12.209,"15min":54.743000000000002},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.84599999999999997,"15min":1.8009999999999999},"min":{"1min":0.39300000000000002,"5min":0.247,"15min":0.214},"max":{"1min":0.746,"5min":7.1890000000000001,"15min":53.701000000000001},"last":0.66100000000000003}]},{"osd":1,"last update":"Mon Oct 9 11:10:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":1.0089999999999999,"15min":1.1619999999999999},"min":{"1min":0.35899999999999999,"5min":0.26900000000000002,"15min":0.23499999999999999},"max":{"1min":0.746,"5min":5.0110000000000001,"15min":41.241999999999997},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.52000000000000002,"5min":1.5209999999999999,"15min":1.71},"min":{"1min":0.38200000000000001,"5min":0.28999999999999998,"15min":0.248},"max":{"1min":0.73499999999999999,"5min":9.2080000000000002,"15min":41.476999999999997},"last":0.51000000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.4399999999999999,"15min":1.8220000000000001},"min":{"1min":0.32500000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.70499999999999996,"5min":10.382999999999999,"15min":12.003},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.93600000000000005,"15min":1.77},"min":{"1min":0.433,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.68999999999999995,"5min":7.133,"15min":15.432},"last":0.40899999999999997}]},{"osd":3,"last update":"Mon Oct 9 11:10:31 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.8540000000000001,"15min":1.5840000000000001},"min":{"1min":0.42699999999999999,"5min":0.313,"15min":0.313},"max":{"1min":0.77400000000000002,"5min":28.948,"15min":28.948},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.58799999999999997,"5min":2.2269999999999999,"15min":2.3679999999999999},"min":{"1min":0.44600000000000001,"5min":0.307,"15min":0.30199999999999999},"max":{"1min":0.71999999999999997,"5min":28.893000000000001,"15min":28.893000000000001},"last":0.58299999999999996}]},{"osd":4,"last update":"Mon Oct 9 11:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.48599999999999999,"15min":0.96599999999999997},"min":{"1min":0.309,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.73099999999999998,"5min":1.6519999999999999,"15min":17.216999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.51000000000000001,"15min":0.77100000000000002},"min":{"1min":0.39000000000000001,"5min":0.26900000000000002,"15min":0.24399999999999999},"max":{"1min":0.78400000000000003,"5min":1.5089999999999999,"15min":10.025},"last":0.71999999999999997}]},{"osd":5,"last update":"Mon Oct 9 11:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.60999999999999999,"15min":1.1379999999999999},"min":{"1min":0.34699999999999998,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":0.80100000000000005,"5min":2.7000000000000002,"15min":19.786999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.60999999999999999,"15min":0.90100000000000002},"min":{"1min":0.44400000000000001,"5min":0.30499999999999999,"15min":0.248},"max":{"1min":0.81599999999999995,"5min":3.488,"15min":9.9469999999999992},"last":0.55600000000000005}]},{"osd":7,"last update":"Mon Oct 9 11:10:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.66200000000000003,"15min":1.119},"min":{"1min":0.38100000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.79200000000000004,"5min":3.7250000000000001,"15min":11.550000000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.69899999999999995,"15min":0.84799999999999998},"min":{"1min":0.42199999999999999,"5min":0.32800000000000001,"15min":0.27500000000000002},"max":{"1min":0.84899999999999998,"5min":3.2839999999999998,"15min":11.557},"last":0.75700000000000001}]}]},{"osd":1,"up_from":388,"seq":1666447310978,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":505544,"kb_used_data":8064,"kb_used_omap":13,"kb_used_meta":497458,"kb_avail":93239608,"statfs":{"total":95995035648,"available":95477358592,"internally_reserved":0,"allocated":8257536,"data_stored":5057756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":509397703},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":2.1299999999999999,"15min":2.2109999999999999},"min":{"1min":0.20499999999999999,"5min":0.182,"15min":0.17699999999999999},"max":{"1min":5.4720000000000004,"5min":18.192,"15min":18.192},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.40200000000000002,"5min":2.3650000000000002,"15min":1.9730000000000001},"min":{"1min":0.245,"5min":0.22900000000000001,"15min":0.19900000000000001},"max":{"1min":1.3180000000000001,"5min":17.957999999999998,"15min":17.957999999999998},"last":0.33000000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.8660000000000001,"15min":2.5550000000000002},"min":{"1min":0.249,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":5.798,"5min":11.875,"15min":47.731000000000002},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.41199999999999998,"5min":2.1850000000000001,"15min":2.347},"min":{"1min":0.27700000000000002,"5min":0.216,"15min":0.216},"max":{"1min":1.2769999999999999,"5min":16.870000000000001,"15min":47.786999999999999},"last":0.30199999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:10:31 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":2.9140000000000001,"15min":1.964},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.497,"5min":92.447999999999993,"15min":92.447999999999993},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.34599999999999997,"5min":3.016,"15min":1.994},"min":{"1min":0.23999999999999999,"5min":0.217,"15min":0.217},"max":{"1min":0.41599999999999998,"5min":92.066999999999993,"15min":92.066999999999993},"last":0.34200000000000003}]},{"osd":4,"last update":"Mon Oct 9 11:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":1.5269999999999999,"15min":2.048},"min":{"1min":0.35099999999999998,"5min":0.27300000000000002,"15min":0.26000000000000001},"max":{"1min":5.407,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.7230000000000001,"15min":1.621},"min":{"1min":0.38200000000000001,"5min":0.314,"15min":0.29999999999999999},"max":{"1min":1.3759999999999999,"5min":16.841000000000001,"15min":16.841000000000001},"last":0.57299999999999995}]},{"osd":5,"last update":"Mon Oct 9 11:10:37 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":1.792,"15min":2.089},"min":{"1min":0.374,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":23.989000000000001,"5min":23.989000000000001,"15min":23.989000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.79900000000000004,"5min":1.615,"15min":2.016},"min":{"1min":0.45700000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.4459999999999997,"5min":11.65,"15min":11.65},"last":0.54000000000000004}]},{"osd":6,"last update":"Mon Oct 9 11:10:37 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":1.853,"15min":2.1819999999999999},"min":{"1min":0.42299999999999999,"5min":0.312,"15min":0.309},"max":{"1min":24.024999999999999,"5min":24.024999999999999,"15min":24.024999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.77500000000000002,"5min":1.6259999999999999,"15min":1.9430000000000001},"min":{"1min":0.42299999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":5.375,"5min":11.75,"15min":11.75},"last":0.499}]},{"osd":7,"last update":"Mon Oct 9 11:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":1.7509999999999999,"15min":1.6379999999999999},"min":{"1min":0.40899999999999997,"5min":0.33000000000000002,"15min":0.32900000000000001},"max":{"1min":1.3480000000000001,"5min":16.797999999999998,"15min":16.797999999999998},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.8260000000000001,"15min":2.0910000000000002},"min":{"1min":0.44700000000000001,"5min":0.36899999999999999,"15min":0.33200000000000002},"max":{"1min":23.923999999999999,"5min":23.923999999999999,"15min":23.923999999999999},"last":0.48399999999999999}]}]},{"osd":0,"up_from":80,"seq":343597384036,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":986628,"kb_used_data":8444,"kb_used_omap":28,"kb_used_meta":978147,"kb_avail":92758524,"statfs":{"total":95995035648,"available":94984728576,"internally_reserved":0,"allocated":8646656,"data_stored":5380705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1001623376},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":1.502,"15min":2.3540000000000001},"min":{"1min":0.22700000000000001,"5min":0.186,"15min":0.186},"max":{"1min":0.503,"5min":10.271000000000001,"15min":19.608000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.35999999999999999,"5min":1.7729999999999999,"15min":2.4129999999999998},"min":{"1min":0.25,"5min":0.214,"15min":0.214},"max":{"1min":0.53800000000000003,"5min":15.834,"15min":19.992999999999999},"last":0.36799999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:10:00 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":1.629,"15min":2.2109999999999999},"min":{"1min":0.26400000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":1.6479999999999999,"5min":11.516,"15min":24.852},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.442,"5min":2.5659999999999998,"15min":2.9060000000000001},"min":{"1min":0.30499999999999999,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":0.81899999999999995,"5min":99.679000000000002,"15min":99.679000000000002},"last":0.34899999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":1.8939999999999999,"15min":2.9860000000000002},"min":{"1min":0.29999999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.54500000000000004,"5min":13.189,"15min":25.050000000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.42699999999999999,"5min":1.7829999999999999,"15min":1.8600000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.60899999999999999,"5min":14.125999999999999,"15min":14.125999999999999},"last":0.32200000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.4610000000000001,"15min":2.0710000000000002},"min":{"1min":0.41299999999999998,"5min":0.27100000000000002,"15min":0.25700000000000001},"max":{"1min":0.879,"5min":13.106,"15min":14.507},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.23,"15min":1.538},"min":{"1min":0.432,"5min":0.33600000000000002,"15min":0.29199999999999998},"max":{"1min":0.88,"5min":7.9249999999999998,"15min":10.513999999999999},"last":0.67800000000000005}]},{"osd":5,"last update":"Mon Oct 9 11:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":1.276,"15min":1.6679999999999999},"min":{"1min":0.39200000000000002,"5min":0.33200000000000002,"15min":0.30399999999999999},"max":{"1min":0.79500000000000004,"5min":7.9550000000000001,"15min":10.544},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.63600000000000001,"5min":1.276,"15min":1.6539999999999999},"min":{"1min":0.46200000000000002,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":0.97099999999999997,"5min":8.8439999999999994,"15min":11.212999999999999},"last":0.66800000000000004}]},{"osd":6,"last update":"Mon Oct 9 11:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":1.3979999999999999,"15min":1.74},"min":{"1min":0.47499999999999998,"5min":0.377,"15min":0.314},"max":{"1min":0.81200000000000006,"5min":7.9829999999999997,"15min":18.920000000000002},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.2969999999999999,"15min":1.7210000000000001},"min":{"1min":0.47599999999999998,"5min":0.378,"15min":0.31900000000000001},"max":{"1min":0.93200000000000005,"5min":8.8810000000000002,"15min":11.257999999999999},"last":0.59599999999999997}]},{"osd":7,"last update":"Mon Oct 9 11:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.5940000000000001,"15min":1.8879999999999999},"min":{"1min":0.46500000000000002,"5min":0.38900000000000001,"15min":0.32700000000000001},"max":{"1min":0.94899999999999995,"5min":9.8710000000000004,"15min":17.672000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.6259999999999999,"15min":2.2010000000000001},"min":{"1min":0.435,"5min":0.311,"15min":0.311},"max":{"1min":0.95899999999999996,"5min":13.154,"15min":17.539000000000001},"last":0.60699999999999998}]}]},{"osd":2,"up_from":90,"seq":386547056976,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":770136,"kb_used_data":12304,"kb_used_omap":81,"kb_used_meta":757742,"kb_avail":92975016,"statfs":{"total":95995035648,"available":95206416384,"internally_reserved":0,"allocated":12599296,"data_stored":9495088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":775928026},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":1.627,"15min":2.0070000000000001},"min":{"1min":0.25600000000000001,"5min":0.182,"15min":0.17699999999999999},"max":{"1min":0.51300000000000001,"5min":15.609,"15min":48.933999999999997},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.35999999999999999,"5min":1.7050000000000001,"15min":1.776},"min":{"1min":0.29399999999999998,"5min":0.23100000000000001,"15min":0.19600000000000001},"max":{"1min":0.47699999999999998,"5min":9.8010000000000002,"15min":14.515000000000001},"last":0.27500000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.31900000000000001,"5min":1.609,"15min":2.2469999999999999},"min":{"1min":0.23000000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":0.55800000000000005,"5min":24.902000000000001,"15min":49.033999999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.38200000000000001,"5min":1.9670000000000001,"15min":2.4199999999999999},"min":{"1min":0.255,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":0.54000000000000004,"5min":25.850999999999999,"15min":48.874000000000002},"last":0.318}]},{"osd":3,"last update":"Mon Oct 9 11:10:24 2023","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":2.601,"15min":2.0430000000000001},"min":{"1min":0.24099999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.57099999999999995,"5min":25.890999999999998,"15min":25.890999999999998},"last":0.371},{"interface":"front","average":{"1min":0.434,"5min":2.3759999999999999,"15min":2.222},"min":{"1min":0.28599999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.59099999999999997,"5min":28.689,"15min":28.689},"last":0.29399999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.149,"15min":1.728},"min":{"1min":0.40300000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.84399999999999997,"5min":11.391999999999999,"15min":12.452999999999999},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.63700000000000001,"5min":1.385,"15min":1.78},"min":{"1min":0.44900000000000001,"5min":0.33900000000000002,"15min":0.249},"max":{"1min":0.83399999999999996,"5min":15.571,"15min":48.795000000000002},"last":0.68200000000000005}]},{"osd":5,"last update":"Mon Oct 9 11:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.538,"15min":1.4910000000000001},"min":{"1min":0.39900000000000002,"5min":0.32400000000000001,"15min":0.25900000000000001},"max":{"1min":0.90900000000000003,"5min":8.5310000000000006,"15min":10.135999999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.629,"5min":1.2090000000000001,"15min":1.8420000000000001},"min":{"1min":0.38800000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.91600000000000004,"5min":11.441000000000001,"15min":49.061},"last":0.65800000000000003}]},{"osd":6,"last update":"Mon Oct 9 11:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":1.411,"15min":1.7909999999999999},"min":{"1min":0.46200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":0.93899999999999995,"5min":15.536,"15min":48.845999999999997},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.7350000000000001,"15min":1.587},"min":{"1min":0.45000000000000001,"5min":0.36799999999999999,"15min":0.29199999999999998},"max":{"1min":0.81000000000000005,"5min":10.157999999999999,"15min":10.157999999999999},"last":0.56899999999999995}]},{"osd":7,"last update":"Mon Oct 9 11:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.5800000000000001,"15min":1.9530000000000001},"min":{"1min":0.44,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":0.88,"5min":15.59,"15min":48.966000000000001},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.64200000000000002,"5min":1.6830000000000001,"15min":1.681},"min":{"1min":0.46300000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.86199999999999999,"5min":8.5549999999999997,"15min":14.94},"last":0.59499999999999997}]}]},{"osd":3,"up_from":469,"seq":2014339661906,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":343692,"kb_used_data":8644,"kb_used_omap":11,"kb_used_meta":335028,"kb_avail":93401460,"statfs":{"total":95995035648,"available":95643095040,"internally_reserved":0,"allocated":8851456,"data_stored":5404132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11309,"internal_metadata":343069651},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":1.643,"15min":1.6950000000000001},"min":{"1min":0.252,"5min":0.183,"15min":0.183},"max":{"1min":0.52200000000000002,"5min":14.521000000000001,"15min":14.882},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.374,"5min":1.238,"15min":0.99099999999999999},"min":{"1min":0.246,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.55800000000000005,"5min":9.4030000000000005,"15min":9.4030000000000005},"last":0.35299999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":1.863,"15min":1.5549999999999999},"min":{"1min":0.248,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":0.51600000000000001,"5min":17.93,"15min":17.93},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.39100000000000001,"5min":1.982,"15min":1.7170000000000001},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.55400000000000005,"5min":19.163,"15min":19.163},"last":0.29099999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":1.8879999999999999,"15min":1.3540000000000001},"min":{"1min":0.217,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.55800000000000005,"5min":38.840000000000003,"15min":38.840000000000003},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.41799999999999998,"5min":2.004,"15min":1.3879999999999999},"min":{"1min":0.28599999999999998,"5min":0.219,"15min":0.219},"max":{"1min":0.66300000000000003,"5min":37.991999999999997,"15min":37.991999999999997},"last":0.309}]},{"osd":4,"last update":"Mon Oct 9 11:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.264,"15min":1.47},"min":{"1min":0.35999999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.98699999999999999,"5min":10.058999999999999,"15min":14.907999999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.58299999999999996,"5min":1.101,"15min":0.83099999999999996},"min":{"1min":0.42799999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.95699999999999996,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.65400000000000003}]},{"osd":5,"last update":"Mon Oct 9 11:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":1.1160000000000001,"15min":1.3140000000000001},"min":{"1min":0.47399999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.02,"5min":5.8499999999999996,"15min":9.3300000000000001},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.64200000000000002,"5min":1.24,"15min":1.3740000000000001},"min":{"1min":0.46700000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.069,"5min":10.029,"15min":10.029},"last":0.64900000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.208,"15min":1.591},"min":{"1min":0.41699999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.97599999999999998,"5min":6.8479999999999999,"15min":15.455},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.63,"5min":1.3129999999999999,"15min":1.5329999999999999},"min":{"1min":0.46600000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.0309999999999999,"5min":10.099,"15min":14.837},"last":0.625}]},{"osd":7,"last update":"Mon Oct 9 11:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.252,"15min":1.0840000000000001},"min":{"1min":0.36299999999999999,"5min":0.315,"15min":0.315},"max":{"1min":1.0029999999999999,"5min":6.907,"15min":6.907},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.4339999999999999,"15min":1.2490000000000001},"min":{"1min":0.44400000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.0429999999999999,"5min":6.9119999999999999,"15min":6.9119999999999999},"last":0.57599999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019326,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903412,"kb_used_data":6572,"kb_used_omap":0,"kb_used_meta":896832,"kb_avail":92841740,"statfs":{"total":95995035648,"available":95069941760,"internally_reserved":0,"allocated":6729728,"data_stored":4874827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918355968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:10:21 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":1.042,"15min":1.5640000000000001},"min":{"1min":0.34300000000000003,"5min":0.224,"15min":0.20300000000000001},"max":{"1min":0.69999999999999996,"5min":8.3460000000000001,"15min":15.128},"last":0.622},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.0700000000000001,"15min":1.6659999999999999},"min":{"1min":0.32100000000000001,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":1.5,"5min":4.6699999999999999,"15min":14.359999999999999},"last":0.53800000000000003}]},{"osd":1,"last update":"Mon Oct 9 11:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":2.1509999999999998,"15min":2.4340000000000002},"min":{"1min":0.32300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.82799999999999996,"5min":57.966999999999999,"15min":121.41},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":1.79,"15min":2.0270000000000001},"min":{"1min":0.35499999999999998,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":0.78100000000000003,"5min":57.832000000000001,"15min":122.21899999999999},"last":0.51400000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:10:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.3180000000000001,"15min":1.6279999999999999},"min":{"1min":0.33700000000000002,"5min":0.246,"15min":0.23799999999999999},"max":{"1min":1.5249999999999999,"5min":18.407,"15min":18.407},"last":0.378},{"interface":"front","average":{"1min":0.57799999999999996,"5min":1.28,"15min":1.9410000000000001},"min":{"1min":0.307,"5min":0.27900000000000003,"15min":0.26400000000000001},"max":{"1min":1.5600000000000001,"5min":7.2119999999999997,"15min":27.818000000000001},"last":0.433}]},{"osd":3,"last update":"Mon Oct 9 11:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":2.0339999999999998,"15min":1.845},"min":{"1min":0.38500000000000001,"5min":0.34799999999999998,"15min":0.27700000000000002},"max":{"1min":0.89900000000000002,"5min":60.753,"15min":60.753},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.61799999999999999,"5min":2.069,"15min":1.4690000000000001},"min":{"1min":0.40899999999999997,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.413,"5min":61.473999999999997,"15min":61.473999999999997},"last":0.47799999999999998}]},{"osd":5,"last update":"Mon Oct 9 11:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.83399999999999996,"15min":1.171},"min":{"1min":0.28499999999999998,"5min":0.27400000000000002,"15min":0.224},"max":{"1min":1.5109999999999999,"5min":4.9409999999999998,"15min":11.59},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.751,"15min":1.0720000000000001},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":1.542,"5min":8.3230000000000004,"15min":12.459},"last":0.58099999999999996}]},{"osd":6,"last update":"Mon Oct 9 11:10:01 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":1.0089999999999999,"15min":1.0800000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":0.86499999999999999,"5min":8.3930000000000007,"15min":8.5739999999999998},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.1339999999999999,"15min":1.355},"min":{"1min":0.372,"5min":0.33700000000000002,"15min":0.29999999999999999},"max":{"1min":1.45,"5min":9.5210000000000008,"15min":12.795999999999999},"last":0.63900000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:10:01 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.1930000000000001,"15min":1.4610000000000001},"min":{"1min":0.32800000000000001,"5min":0.308,"15min":0.27700000000000002},"max":{"1min":1.4790000000000001,"5min":10.15,"15min":19.923999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.63200000000000001,"5min":1.0049999999999999,"15min":1.351},"min":{"1min":0.37,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.5760000000000001,"5min":10.853,"15min":19.859999999999999},"last":0.60799999999999998}]}]},{"osd":5,"up_from":37,"seq":158913790395,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1206120,"kb_used_data":6816,"kb_used_omap":0,"kb_used_meta":1199296,"kb_avail":92539032,"statfs":{"total":95995035648,"available":94759968768,"internally_reserved":0,"allocated":6979584,"data_stored":5132955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1228079104},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":1.1259999999999999,"15min":2.137},"min":{"1min":0.35499999999999998,"5min":0.219,"15min":0.21299999999999999},"max":{"1min":0.91500000000000004,"5min":6.2249999999999996,"15min":58.271999999999998},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.624,"5min":1.177,"15min":2.1840000000000002},"min":{"1min":0.49099999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.96699999999999997,"5min":6.2649999999999997,"15min":58.25},"last":0.60099999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.4099999999999999,"15min":1.8700000000000001},"min":{"1min":0.38100000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.88300000000000001,"5min":16.962,"15min":32.491999999999997},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.70099999999999996,"5min":1.4099999999999999,"15min":1.784},"min":{"1min":0.45000000000000001,"5min":0.30499999999999999,"15min":0.247},"max":{"1min":0.98899999999999999,"5min":17.033999999999999,"15min":32.537999999999997},"last":0.57699999999999996}]},{"osd":2,"last update":"Mon Oct 9 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":1.0489999999999999,"15min":1.716},"min":{"1min":0.32800000000000001,"5min":0.26800000000000002,"15min":0.221},"max":{"1min":0.69599999999999995,"5min":7.3109999999999999,"15min":11.257999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":1.407,"15min":1.8340000000000001},"min":{"1min":0.32600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.78100000000000003,"5min":8.0359999999999996,"15min":14.621},"last":0.46000000000000002}]},{"osd":3,"last update":"Mon Oct 9 11:10:23 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.417,"15min":1.4650000000000001},"min":{"1min":0.377,"5min":0.29799999999999999,"15min":0.28799999999999998},"max":{"1min":0.747,"5min":15.535,"15min":15.535},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.661,"15min":1.7589999999999999},"min":{"1min":0.41599999999999998,"5min":0.38700000000000001,"15min":0.33600000000000002},"max":{"1min":0.77700000000000002,"5min":15.757999999999999,"15min":15.757999999999999},"last":0.435}]},{"osd":4,"last update":"Mon Oct 9 11:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.69099999999999995,"15min":0.89000000000000001},"min":{"1min":0.34000000000000002,"5min":0.26000000000000001,"15min":0.216},"max":{"1min":0.84299999999999997,"5min":3.0680000000000001,"15min":12.042},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.626,"5min":0.85699999999999998,"15min":1.117},"min":{"1min":0.35499999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.78900000000000003,"5min":6.8639999999999999,"15min":9.2490000000000006},"last":0.48099999999999998}]},{"osd":6,"last update":"Mon Oct 9 11:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.81200000000000006,"15min":1.1200000000000001},"min":{"1min":0.40899999999999997,"5min":0.27200000000000002,"15min":0.27100000000000002},"max":{"1min":0.79300000000000004,"5min":5.944,"15min":9.1969999999999992},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.89600000000000002,"15min":1.0169999999999999},"min":{"1min":0.41499999999999998,"5min":0.29699999999999999,"15min":0.29099999999999998},"max":{"1min":0.878,"5min":9.298,"15min":11.983000000000001},"last":0.628}]},{"osd":7,"last update":"Mon Oct 9 11:10:12 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.85299999999999998,"15min":1.2050000000000001},"min":{"1min":0.39400000000000002,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":0.82799999999999996,"5min":5.9699999999999998,"15min":9.3539999999999992},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.85399999999999998,"15min":1.1220000000000001},"min":{"1min":0.439,"5min":0.30499999999999999,"15min":0.30099999999999999},"max":{"1min":0.91500000000000004,"5min":5.6920000000000002,"15min":11.726000000000001},"last":0.64900000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3733,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:10:54.362 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T11:10:54.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:54 smithi125 ceph-mon[200197]: pgmap v1510: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:54.570 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:54 smithi125 ceph-mon[200197]: from='client.37881 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:10:54.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:54 smithi125 ceph-mon[195507]: pgmap v1510: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:54.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:54 smithi125 ceph-mon[195507]: from='client.37881 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:10:54.632 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T11:10:54.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:54 smithi155 ceph-mon[184854]: pgmap v1510: 103 pgs: 103 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T11:10:54.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:54 smithi155 ceph-mon[184854]: from='client.37881 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T11:10:55.761 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T11:10:55.762 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T11:10:56.146 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1510,"stamp":"2023-10-09T11:10:53.960110+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5995,"num_read_kb":13244,"num_write":907,"num_write_kb":12741,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":98,"num_bytes_recovered":2369408,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7137584,"kb_used_data":72752,"kb_used_omap":134,"kb_used_meta":7064633,"kb_avail":742823632,"statfs":{"total":767960285184,"available":760651399168,"internally_reserved":0,"allocated":74498048,"data_stored":54073460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":137532,"internal_metadata":7234184900},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002848"},"pg_stats":[{"pgid":"1.18","version":"76'161","reported_seq":904,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445356+0000","last_change":"2023-10-09T11:07:36.287455+0000","last_active":"2023-10-09T11:10:31.445356+0000","last_peered":"2023-10-09T11:10:31.445356+0000","last_clean":"2023-10-09T11:10:31.445356+0000","last_became_active":"2023-10-09T11:07:36.287052+0000","last_became_peered":"2023-10-09T11:07:36.287052+0000","last_unstale":"2023-10-09T11:10:31.445356+0000","last_undegraded":"2023-10-09T11:10:31.445356+0000","last_fullsized":"2023-10-09T11:10:31.445356+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:18.646580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:18.646580+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:18.864873+0000","scrub_duration":0.001602247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":407,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578694+0000","last_change":"2023-10-09T11:10:11.520572+0000","last_active":"2023-10-09T11:10:32.578694+0000","last_peered":"2023-10-09T11:10:32.578694+0000","last_clean":"2023-10-09T11:10:32.578694+0000","last_became_active":"2023-10-09T11:10:11.520214+0000","last_became_peered":"2023-10-09T11:10:11.520214+0000","last_unstale":"2023-10-09T11:10:32.578694+0000","last_undegraded":"2023-10-09T11:10:32.578694+0000","last_fullsized":"2023-10-09T11:10:32.578694+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:50:00.039165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":146,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443437+0000","last_change":"2023-10-09T11:04:50.319477+0000","last_active":"2023-10-09T11:10:31.443437+0000","last_peered":"2023-10-09T11:10:31.443437+0000","last_clean":"2023-10-09T11:10:31.443437+0000","last_became_active":"2023-10-09T11:04:50.319148+0000","last_became_peered":"2023-10-09T11:04:50.319148+0000","last_unstale":"2023-10-09T11:10:31.443437+0000","last_undegraded":"2023-10-09T11:10:31.443437+0000","last_fullsized":"2023-10-09T11:10:31.443437+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:38:48.725566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"76'161","reported_seq":774,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446233+0000","last_change":"2023-10-09T11:02:27.187180+0000","last_active":"2023-10-09T11:10:31.446233+0000","last_peered":"2023-10-09T11:10:31.446233+0000","last_clean":"2023-10-09T11:10:31.446233+0000","last_became_active":"2023-10-09T11:01:15.403759+0000","last_became_peered":"2023-10-09T11:01:15.403759+0000","last_unstale":"2023-10-09T11:10:31.446233+0000","last_undegraded":"2023-10-09T11:10:31.446233+0000","last_fullsized":"2023-10-09T11:10:31.446233+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:27.187120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:27.187120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:53:25.115486+0000","scrub_duration":0.012003906,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":334,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.226630+0000","last_change":"2023-10-09T11:04:50.333711+0000","last_active":"2023-10-09T11:10:32.226630+0000","last_peered":"2023-10-09T11:10:32.226630+0000","last_clean":"2023-10-09T11:10:32.226630+0000","last_became_active":"2023-10-09T11:04:50.333262+0000","last_became_peered":"2023-10-09T11:04:50.333262+0000","last_unstale":"2023-10-09T11:10:32.226630+0000","last_undegraded":"2023-10-09T11:10:32.226630+0000","last_fullsized":"2023-10-09T11:10:32.226630+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:27.843765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":527,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578897+0000","last_change":"2023-10-09T11:10:02.479621+0000","last_active":"2023-10-09T11:10:32.578897+0000","last_peered":"2023-10-09T11:10:32.578897+0000","last_clean":"2023-10-09T11:10:32.578897+0000","last_became_active":"2023-10-09T11:10:02.479324+0000","last_became_peered":"2023-10-09T11:10:02.479324+0000","last_unstale":"2023-10-09T11:10:32.578897+0000","last_undegraded":"2023-10-09T11:10:32.578897+0000","last_fullsized":"2023-10-09T11:10:32.578897+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:22.686515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"76'161","reported_seq":993,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578425+0000","last_change":"2023-10-09T11:10:11.521202+0000","last_active":"2023-10-09T11:10:32.578425+0000","last_peered":"2023-10-09T11:10:32.578425+0000","last_clean":"2023-10-09T11:10:32.578425+0000","last_became_active":"2023-10-09T11:10:11.520809+0000","last_became_peered":"2023-10-09T11:10:11.520809+0000","last_unstale":"2023-10-09T11:10:32.578425+0000","last_undegraded":"2023-10-09T11:10:32.578425+0000","last_fullsized":"2023-10-09T11:10:32.578425+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:37.730075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:37.730075+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:54:24.092815+0000","scrub_duration":0.0083091669999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":146,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.442843+0000","last_change":"2023-10-09T11:04:50.310477+0000","last_active":"2023-10-09T11:10:31.442843+0000","last_peered":"2023-10-09T11:10:31.442843+0000","last_clean":"2023-10-09T11:10:31.442843+0000","last_became_active":"2023-10-09T11:04:50.310067+0000","last_became_peered":"2023-10-09T11:04:50.310067+0000","last_unstale":"2023-10-09T11:10:31.442843+0000","last_undegraded":"2023-10-09T11:10:31.442843+0000","last_fullsized":"2023-10-09T11:10:31.442843+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:58:52.884716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":319,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445888+0000","last_change":"2023-10-09T11:10:20.507397+0000","last_active":"2023-10-09T11:10:31.445888+0000","last_peered":"2023-10-09T11:10:31.445888+0000","last_clean":"2023-10-09T11:10:31.445888+0000","last_became_active":"2023-10-09T11:10:20.507062+0000","last_became_peered":"2023-10-09T11:10:20.507062+0000","last_unstale":"2023-10-09T11:10:31.445888+0000","last_undegraded":"2023-10-09T11:10:31.445888+0000","last_fullsized":"2023-10-09T11:10:31.445888+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:53.085834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"76'161","reported_seq":834,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.363850+0000","last_change":"2023-10-09T11:07:36.282502+0000","last_active":"2023-10-09T11:10:32.363850+0000","last_peered":"2023-10-09T11:10:32.363850+0000","last_clean":"2023-10-09T11:10:32.363850+0000","last_became_active":"2023-10-09T11:07:36.282133+0000","last_became_peered":"2023-10-09T11:07:36.282133+0000","last_unstale":"2023-10-09T11:10:32.363850+0000","last_undegraded":"2023-10-09T11:10:32.363850+0000","last_fullsized":"2023-10-09T11:10:32.363850+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:28.162970+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:28.162970+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:23:03.762383+0000","scrub_duration":0.0060669280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":220,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578969+0000","last_change":"2023-10-09T11:10:02.475444+0000","last_active":"2023-10-09T11:10:32.578969+0000","last_peered":"2023-10-09T11:10:32.578969+0000","last_clean":"2023-10-09T11:10:32.578969+0000","last_became_active":"2023-10-09T11:10:02.475066+0000","last_became_peered":"2023-10-09T11:10:02.475066+0000","last_unstale":"2023-10-09T11:10:32.578969+0000","last_undegraded":"2023-10-09T11:10:32.578969+0000","last_fullsized":"2023-10-09T11:10:32.578969+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:12.203653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":155,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.442901+0000","last_change":"2023-10-09T11:05:20.299453+0000","last_active":"2023-10-09T11:10:31.442901+0000","last_peered":"2023-10-09T11:10:31.442901+0000","last_clean":"2023-10-09T11:10:31.442901+0000","last_became_active":"2023-10-09T11:05:20.299117+0000","last_became_peered":"2023-10-09T11:05:20.299117+0000","last_unstale":"2023-10-09T11:10:31.442901+0000","last_undegraded":"2023-10-09T11:10:31.442901+0000","last_fullsized":"2023-10-09T11:10:31.442901+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:30.072813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"76'161","reported_seq":950,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.461802+0000","last_change":"2023-10-09T11:02:36.216635+0000","last_active":"2023-10-09T11:10:33.461802+0000","last_peered":"2023-10-09T11:10:33.461802+0000","last_clean":"2023-10-09T11:10:33.461802+0000","last_became_active":"2023-10-09T11:02:36.216315+0000","last_became_peered":"2023-10-09T11:02:36.216315+0000","last_unstale":"2023-10-09T11:10:33.461802+0000","last_undegraded":"2023-10-09T11:10:33.461802+0000","last_fullsized":"2023-10-09T11:10:33.461802+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:57:17.772327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:57:17.772327+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:21.198531+0000","scrub_duration":0.001375598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":488,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446547+0000","last_change":"2023-10-09T11:07:33.242665+0000","last_active":"2023-10-09T11:10:31.446547+0000","last_peered":"2023-10-09T11:10:31.446547+0000","last_clean":"2023-10-09T11:10:31.446547+0000","last_became_active":"2023-10-09T11:07:33.242313+0000","last_became_peered":"2023-10-09T11:07:33.242313+0000","last_unstale":"2023-10-09T11:10:31.446547+0000","last_undegraded":"2023-10-09T11:10:31.446547+0000","last_fullsized":"2023-10-09T11:10:31.446547+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:20.565135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":173,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443180+0000","last_change":"2023-10-09T11:07:33.225683+0000","last_active":"2023-10-09T11:10:31.443180+0000","last_peered":"2023-10-09T11:10:31.443180+0000","last_clean":"2023-10-09T11:10:31.443180+0000","last_became_active":"2023-10-09T11:07:33.225275+0000","last_became_peered":"2023-10-09T11:07:33.225275+0000","last_unstale":"2023-10-09T11:10:31.443180+0000","last_undegraded":"2023-10-09T11:10:31.443180+0000","last_fullsized":"2023-10-09T11:10:31.443180+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:10.131060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"76'161","reported_seq":846,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446606+0000","last_change":"2023-10-09T11:05:23.328981+0000","last_active":"2023-10-09T11:10:31.446606+0000","last_peered":"2023-10-09T11:10:31.446606+0000","last_clean":"2023-10-09T11:10:31.446606+0000","last_became_active":"2023-10-09T11:05:23.328616+0000","last_became_peered":"2023-10-09T11:05:23.328616+0000","last_unstale":"2023-10-09T11:10:31.446606+0000","last_undegraded":"2023-10-09T11:10:31.446606+0000","last_fullsized":"2023-10-09T11:10:31.446606+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.673062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.673062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:43:15.315819+0000","scrub_duration":0.0018420559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":85,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364593+0000","last_change":"2023-10-09T11:07:33.265842+0000","last_active":"2023-10-09T11:10:32.364593+0000","last_peered":"2023-10-09T11:10:32.364593+0000","last_clean":"2023-10-09T11:10:32.364593+0000","last_became_active":"2023-10-09T11:07:33.232311+0000","last_became_peered":"2023-10-09T11:07:33.232311+0000","last_unstale":"2023-10-09T11:10:32.364593+0000","last_undegraded":"2023-10-09T11:10:32.364593+0000","last_fullsized":"2023-10-09T11:10:32.364593+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:23.402582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":395,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445952+0000","last_change":"2023-10-09T11:05:20.316265+0000","last_active":"2023-10-09T11:10:31.445952+0000","last_peered":"2023-10-09T11:10:31.445952+0000","last_clean":"2023-10-09T11:10:31.445952+0000","last_became_active":"2023-10-09T11:05:20.315853+0000","last_became_peered":"2023-10-09T11:05:20.315853+0000","last_unstale":"2023-10-09T11:10:31.445952+0000","last_undegraded":"2023-10-09T11:10:31.445952+0000","last_fullsized":"2023-10-09T11:10:31.445952+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:35.093981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"76'161","reported_seq":1012,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.227156+0000","last_change":"2023-10-09T11:04:53.274921+0000","last_active":"2023-10-09T11:10:32.227156+0000","last_peered":"2023-10-09T11:10:32.227156+0000","last_clean":"2023-10-09T11:10:32.227156+0000","last_became_active":"2023-10-09T11:04:53.274518+0000","last_became_peered":"2023-10-09T11:04:53.274518+0000","last_unstale":"2023-10-09T11:10:32.227156+0000","last_undegraded":"2023-10-09T11:10:32.227156+0000","last_fullsized":"2023-10-09T11:10:32.227156+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:09.381429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:09.381429+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:00:14.315301+0000","scrub_duration":0.018121557,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"76'22","reported_seq":235,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578771+0000","last_change":"2023-10-09T11:10:11.522075+0000","last_active":"2023-10-09T11:10:32.578771+0000","last_peered":"2023-10-09T11:10:32.578771+0000","last_clean":"2023-10-09T11:10:32.578771+0000","last_became_active":"2023-10-09T11:10:11.521860+0000","last_became_peered":"2023-10-09T11:10:11.521860+0000","last_unstale":"2023-10-09T11:10:32.578771+0000","last_undegraded":"2023-10-09T11:10:32.578771+0000","last_fullsized":"2023-10-09T11:10:32.578771+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:22.783526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":537,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578831+0000","last_change":"2023-10-09T11:10:11.523601+0000","last_active":"2023-10-09T11:10:32.578831+0000","last_peered":"2023-10-09T11:10:32.578831+0000","last_clean":"2023-10-09T11:10:32.578831+0000","last_became_active":"2023-10-09T11:10:11.523065+0000","last_became_peered":"2023-10-09T11:10:11.523065+0000","last_unstale":"2023-10-09T11:10:32.578831+0000","last_undegraded":"2023-10-09T11:10:32.578831+0000","last_fullsized":"2023-10-09T11:10:32.578831+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:18:08.763192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"76'161","reported_seq":954,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578511+0000","last_change":"2023-10-09T11:10:02.473069+0000","last_active":"2023-10-09T11:10:32.578511+0000","last_peered":"2023-10-09T11:10:32.578511+0000","last_clean":"2023-10-09T11:10:32.578511+0000","last_became_active":"2023-10-09T11:10:02.472741+0000","last_became_peered":"2023-10-09T11:10:02.472741+0000","last_unstale":"2023-10-09T11:10:32.578511+0000","last_undegraded":"2023-10-09T11:10:32.578511+0000","last_fullsized":"2023-10-09T11:10:32.578511+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_deep_scrub":"76'161","last_deep_scrub_stamp":"2023-10-09T11:08:39.934865+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.934865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:04.611884+0000","scrub_duration":0.00143524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":85,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364223+0000","last_change":"2023-10-09T11:07:33.269200+0000","last_active":"2023-10-09T11:10:32.364223+0000","last_peered":"2023-10-09T11:10:32.364223+0000","last_clean":"2023-10-09T11:10:32.364223+0000","last_became_active":"2023-10-09T11:07:33.236205+0000","last_became_peered":"2023-10-09T11:07:33.236205+0000","last_unstale":"2023-10-09T11:10:32.364223+0000","last_undegraded":"2023-10-09T11:10:32.364223+0000","last_fullsized":"2023-10-09T11:10:32.364223+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:24.574734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":164,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443290+0000","last_change":"2023-10-09T11:07:33.227803+0000","last_active":"2023-10-09T11:10:31.443290+0000","last_peered":"2023-10-09T11:10:31.443290+0000","last_clean":"2023-10-09T11:10:31.443290+0000","last_became_active":"2023-10-09T11:07:33.227486+0000","last_became_peered":"2023-10-09T11:07:33.227486+0000","last_unstale":"2023-10-09T11:10:31.443290+0000","last_undegraded":"2023-10-09T11:10:31.443290+0000","last_fullsized":"2023-10-09T11:10:31.443290+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:53:39.274875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"76'161","reported_seq":928,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445760+0000","last_change":"2023-10-09T11:05:23.323856+0000","last_active":"2023-10-09T11:10:31.445760+0000","last_peered":"2023-10-09T11:10:31.445760+0000","last_clean":"2023-10-09T11:10:31.445760+0000","last_became_active":"2023-10-09T11:05:23.322763+0000","last_became_peered":"2023-10-09T11:05:23.322763+0000","last_unstale":"2023-10-09T11:10:31.445760+0000","last_undegraded":"2023-10-09T11:10:31.445760+0000","last_fullsized":"2023-10-09T11:10:31.445760+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:39.776301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:39.776301+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:28.760820+0000","scrub_duration":0.001605198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":525,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578417+0000","last_change":"2023-10-09T11:10:02.479550+0000","last_active":"2023-10-09T11:10:32.578417+0000","last_peered":"2023-10-09T11:10:32.578417+0000","last_clean":"2023-10-09T11:10:32.578417+0000","last_became_active":"2023-10-09T11:10:02.479243+0000","last_became_peered":"2023-10-09T11:10:02.479243+0000","last_unstale":"2023-10-09T11:10:32.578417+0000","last_undegraded":"2023-10-09T11:10:32.578417+0000","last_fullsized":"2023-10-09T11:10:32.578417+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:28:59.098743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":558,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446628+0000","last_change":"2023-10-09T11:07:33.226594+0000","last_active":"2023-10-09T11:10:31.446628+0000","last_peered":"2023-10-09T11:10:31.446628+0000","last_clean":"2023-10-09T11:10:31.446628+0000","last_became_active":"2023-10-09T11:07:33.226275+0000","last_became_peered":"2023-10-09T11:07:33.226275+0000","last_unstale":"2023-10-09T11:10:31.446628+0000","last_undegraded":"2023-10-09T11:10:31.446628+0000","last_fullsized":"2023-10-09T11:10:31.446628+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:31:41.180088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"76'161","reported_seq":953,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.226853+0000","last_change":"2023-10-09T11:04:53.274236+0000","last_active":"2023-10-09T11:10:32.226853+0000","last_peered":"2023-10-09T11:10:32.226853+0000","last_clean":"2023-10-09T11:10:32.226853+0000","last_became_active":"2023-10-09T11:04:53.273817+0000","last_became_peered":"2023-10-09T11:04:53.273817+0000","last_unstale":"2023-10-09T11:10:32.226853+0000","last_undegraded":"2023-10-09T11:10:32.226853+0000","last_fullsized":"2023-10-09T11:10:32.226853+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:25.620099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:25.620099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:05.096905+0000","scrub_duration":0.068593214999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":214,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.363757+0000","last_change":"2023-10-09T11:05:20.312129+0000","last_active":"2023-10-09T11:10:32.363757+0000","last_peered":"2023-10-09T11:10:32.363757+0000","last_clean":"2023-10-09T11:10:32.363757+0000","last_became_active":"2023-10-09T11:05:20.311810+0000","last_became_peered":"2023-10-09T11:05:20.311810+0000","last_unstale":"2023-10-09T11:10:32.363757+0000","last_undegraded":"2023-10-09T11:10:32.363757+0000","last_fullsized":"2023-10-09T11:10:32.363757+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:48.813391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":263,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578265+0000","last_change":"2023-10-09T11:10:02.479630+0000","last_active":"2023-10-09T11:10:32.578265+0000","last_peered":"2023-10-09T11:10:32.578265+0000","last_clean":"2023-10-09T11:10:32.578265+0000","last_became_active":"2023-10-09T11:10:02.479320+0000","last_became_peered":"2023-10-09T11:10:02.479320+0000","last_unstale":"2023-10-09T11:10:32.578265+0000","last_undegraded":"2023-10-09T11:10:32.578265+0000","last_fullsized":"2023-10-09T11:10:32.578265+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:43:27.560586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"76'161","reported_seq":925,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446295+0000","last_change":"2023-10-09T11:05:23.324537+0000","last_active":"2023-10-09T11:10:31.446295+0000","last_peered":"2023-10-09T11:10:31.446295+0000","last_clean":"2023-10-09T11:10:31.446295+0000","last_became_active":"2023-10-09T11:05:23.324187+0000","last_became_peered":"2023-10-09T11:05:23.324187+0000","last_unstale":"2023-10-09T11:10:31.446295+0000","last_undegraded":"2023-10-09T11:10:31.446295+0000","last_fullsized":"2023-10-09T11:10:31.446295+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:18.715504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:18.715504+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:25.286106+0000","scrub_duration":0.0012777070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":137,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445428+0000","last_change":"2023-10-09T11:05:20.323565+0000","last_active":"2023-10-09T11:10:31.445428+0000","last_peered":"2023-10-09T11:10:31.445428+0000","last_clean":"2023-10-09T11:10:31.445428+0000","last_became_active":"2023-10-09T11:05:20.309672+0000","last_became_peered":"2023-10-09T11:05:20.309672+0000","last_unstale":"2023-10-09T11:10:31.445428+0000","last_undegraded":"2023-10-09T11:10:31.445428+0000","last_fullsized":"2023-10-09T11:10:31.445428+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:21:50.211465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":164,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443653+0000","last_change":"2023-10-09T11:07:33.228001+0000","last_active":"2023-10-09T11:10:31.443653+0000","last_peered":"2023-10-09T11:10:31.443653+0000","last_clean":"2023-10-09T11:10:31.443653+0000","last_became_active":"2023-10-09T11:07:33.227691+0000","last_became_peered":"2023-10-09T11:07:33.227691+0000","last_unstale":"2023-10-09T11:10:31.443653+0000","last_undegraded":"2023-10-09T11:10:31.443653+0000","last_fullsized":"2023-10-09T11:10:31.443653+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:24:16.098138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"76'161","reported_seq":851,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.363813+0000","last_change":"2023-10-09T11:07:36.284927+0000","last_active":"2023-10-09T11:10:32.363813+0000","last_peered":"2023-10-09T11:10:32.363813+0000","last_clean":"2023-10-09T11:10:32.363813+0000","last_became_active":"2023-10-09T11:07:36.284631+0000","last_became_peered":"2023-10-09T11:07:36.284631+0000","last_unstale":"2023-10-09T11:10:32.363813+0000","last_undegraded":"2023-10-09T11:10:32.363813+0000","last_fullsized":"2023-10-09T11:10:32.363813+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:23.805511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:23.805511+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:37.220243+0000","scrub_duration":0.007550462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":83,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364596+0000","last_change":"2023-10-09T11:07:33.273503+0000","last_active":"2023-10-09T11:10:32.364596+0000","last_peered":"2023-10-09T11:10:32.364596+0000","last_clean":"2023-10-09T11:10:32.364596+0000","last_became_active":"2023-10-09T11:07:33.242980+0000","last_became_peered":"2023-10-09T11:07:33.242980+0000","last_unstale":"2023-10-09T11:10:32.364596+0000","last_undegraded":"2023-10-09T11:10:32.364596+0000","last_fullsized":"2023-10-09T11:10:32.364596+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:25:18.934972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":579,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445459+0000","last_change":"2023-10-09T11:07:33.304363+0000","last_active":"2023-10-09T11:10:31.445459+0000","last_peered":"2023-10-09T11:10:31.445459+0000","last_clean":"2023-10-09T11:10:31.445459+0000","last_became_active":"2023-10-09T11:07:33.304024+0000","last_became_peered":"2023-10-09T11:07:33.304024+0000","last_unstale":"2023-10-09T11:10:31.445459+0000","last_undegraded":"2023-10-09T11:10:31.445459+0000","last_fullsized":"2023-10-09T11:10:31.445459+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:21:51.522688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"76'161","reported_seq":1001,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443366+0000","last_change":"2023-10-09T11:05:23.316129+0000","last_active":"2023-10-09T11:10:31.443366+0000","last_peered":"2023-10-09T11:10:31.443366+0000","last_clean":"2023-10-09T11:10:31.443366+0000","last_became_active":"2023-10-09T11:05:23.315792+0000","last_became_peered":"2023-10-09T11:05:23.315792+0000","last_unstale":"2023-10-09T11:10:31.443366+0000","last_undegraded":"2023-10-09T11:10:31.443366+0000","last_fullsized":"2023-10-09T11:10:31.443366+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:25.540320+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:25.540320+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:51.770547+0000","scrub_duration":0.0032053709999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":83,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364420+0000","last_change":"2023-10-09T11:07:33.256691+0000","last_active":"2023-10-09T11:10:32.364420+0000","last_peered":"2023-10-09T11:10:32.364420+0000","last_clean":"2023-10-09T11:10:32.364420+0000","last_became_active":"2023-10-09T11:07:33.229554+0000","last_became_peered":"2023-10-09T11:07:33.229554+0000","last_unstale":"2023-10-09T11:10:32.364420+0000","last_undegraded":"2023-10-09T11:10:32.364420+0000","last_fullsized":"2023-10-09T11:10:32.364420+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:06:57.414795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":294,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578539+0000","last_change":"2023-10-09T11:10:11.520169+0000","last_active":"2023-10-09T11:10:32.578539+0000","last_peered":"2023-10-09T11:10:32.578539+0000","last_clean":"2023-10-09T11:10:32.578539+0000","last_became_active":"2023-10-09T11:10:11.519857+0000","last_became_peered":"2023-10-09T11:10:11.519857+0000","last_unstale":"2023-10-09T11:10:32.578539+0000","last_undegraded":"2023-10-09T11:10:32.578539+0000","last_fullsized":"2023-10-09T11:10:32.578539+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:48.203519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"76'161","reported_seq":759,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445841+0000","last_change":"2023-10-09T11:10:20.449427+0000","last_active":"2023-10-09T11:10:31.445841+0000","last_peered":"2023-10-09T11:10:31.445841+0000","last_clean":"2023-10-09T11:10:31.445841+0000","last_became_active":"2023-10-09T11:10:20.448905+0000","last_became_peered":"2023-10-09T11:10:20.448905+0000","last_unstale":"2023-10-09T11:10:31.445841+0000","last_undegraded":"2023-10-09T11:10:31.445841+0000","last_fullsized":"2023-10-09T11:10:31.445841+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:22.599060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:22.599060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:49:21.622527+0000","scrub_duration":0.001975868,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":83,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364690+0000","last_change":"2023-10-09T11:07:33.271238+0000","last_active":"2023-10-09T11:10:32.364690+0000","last_peered":"2023-10-09T11:10:32.364690+0000","last_clean":"2023-10-09T11:10:32.364690+0000","last_became_active":"2023-10-09T11:07:33.241886+0000","last_became_peered":"2023-10-09T11:07:33.241886+0000","last_unstale":"2023-10-09T11:10:32.364690+0000","last_undegraded":"2023-10-09T11:10:32.364690+0000","last_fullsized":"2023-10-09T11:10:32.364690+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:48.882589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":147,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445946+0000","last_change":"2023-10-09T11:07:33.244131+0000","last_active":"2023-10-09T11:10:31.445946+0000","last_peered":"2023-10-09T11:10:31.445946+0000","last_clean":"2023-10-09T11:10:31.445946+0000","last_became_active":"2023-10-09T11:07:33.243797+0000","last_became_peered":"2023-10-09T11:07:33.243797+0000","last_unstale":"2023-10-09T11:10:31.445946+0000","last_undegraded":"2023-10-09T11:10:31.445946+0000","last_fullsized":"2023-10-09T11:10:31.445946+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:44.851043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"76'161","reported_seq":1085,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.579133+0000","last_change":"2023-10-09T11:10:11.520963+0000","last_active":"2023-10-09T11:10:32.579133+0000","last_peered":"2023-10-09T11:10:32.579133+0000","last_clean":"2023-10-09T11:10:32.579133+0000","last_became_active":"2023-10-09T11:10:11.520593+0000","last_became_peered":"2023-10-09T11:10:11.520593+0000","last_unstale":"2023-10-09T11:10:32.579133+0000","last_undegraded":"2023-10-09T11:10:32.579133+0000","last_fullsized":"2023-10-09T11:10:32.579133+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:38.681804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:38.681804+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:19:52.567104+0000","scrub_duration":0.0041222209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":475,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446426+0000","last_change":"2023-10-09T11:05:20.309385+0000","last_active":"2023-10-09T11:10:31.446426+0000","last_peered":"2023-10-09T11:10:31.446426+0000","last_clean":"2023-10-09T11:10:31.446426+0000","last_became_active":"2023-10-09T11:05:20.309034+0000","last_became_peered":"2023-10-09T11:05:20.309034+0000","last_unstale":"2023-10-09T11:10:31.446426+0000","last_undegraded":"2023-10-09T11:10:31.446426+0000","last_fullsized":"2023-10-09T11:10:31.446426+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:31.857485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":472,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578583+0000","last_change":"2023-10-09T11:10:02.477873+0000","last_active":"2023-10-09T11:10:32.578583+0000","last_peered":"2023-10-09T11:10:32.578583+0000","last_clean":"2023-10-09T11:10:32.578583+0000","last_became_active":"2023-10-09T11:10:02.477370+0000","last_became_peered":"2023-10-09T11:10:02.477370+0000","last_unstale":"2023-10-09T11:10:32.578583+0000","last_undegraded":"2023-10-09T11:10:32.578583+0000","last_fullsized":"2023-10-09T11:10:32.578583+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:43.390703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"76'161","reported_seq":773,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.363761+0000","last_change":"2023-10-09T11:07:36.283174+0000","last_active":"2023-10-09T11:10:32.363761+0000","last_peered":"2023-10-09T11:10:32.363761+0000","last_clean":"2023-10-09T11:10:32.363761+0000","last_became_active":"2023-10-09T11:07:36.278007+0000","last_became_peered":"2023-10-09T11:07:36.278007+0000","last_unstale":"2023-10-09T11:10:32.363761+0000","last_undegraded":"2023-10-09T11:10:32.363761+0000","last_fullsized":"2023-10-09T11:10:32.363761+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:31.553275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:31.553275+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:51.693969+0000","scrub_duration":0.0078636220000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":150,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443165+0000","last_change":"2023-10-09T11:04:50.529513+0000","last_active":"2023-10-09T11:10:31.443165+0000","last_peered":"2023-10-09T11:10:31.443165+0000","last_clean":"2023-10-09T11:10:31.443165+0000","last_became_active":"2023-10-09T11:04:50.339671+0000","last_became_peered":"2023-10-09T11:04:50.339671+0000","last_unstale":"2023-10-09T11:10:31.443165+0000","last_undegraded":"2023-10-09T11:10:31.443165+0000","last_fullsized":"2023-10-09T11:10:31.443165+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:38:24.473000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":226,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578822+0000","last_change":"2023-10-09T11:10:11.522176+0000","last_active":"2023-10-09T11:10:32.578822+0000","last_peered":"2023-10-09T11:10:32.578822+0000","last_clean":"2023-10-09T11:10:32.578822+0000","last_became_active":"2023-10-09T11:10:11.521859+0000","last_became_peered":"2023-10-09T11:10:11.521859+0000","last_unstale":"2023-10-09T11:10:32.578822+0000","last_undegraded":"2023-10-09T11:10:32.578822+0000","last_fullsized":"2023-10-09T11:10:32.578822+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:56:53.974041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"76'161","reported_seq":875,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445655+0000","last_change":"2023-10-09T11:04:53.272323+0000","last_active":"2023-10-09T11:10:31.445655+0000","last_peered":"2023-10-09T11:10:31.445655+0000","last_clean":"2023-10-09T11:10:31.445655+0000","last_became_active":"2023-10-09T11:04:53.272010+0000","last_became_peered":"2023-10-09T11:04:53.272010+0000","last_unstale":"2023-10-09T11:10:31.445655+0000","last_undegraded":"2023-10-09T11:10:31.445655+0000","last_fullsized":"2023-10-09T11:10:31.445655+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:20.667399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:20.667399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:23.913605+0000","scrub_duration":0.0077670070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":536,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446036+0000","last_change":"2023-10-09T11:10:20.506780+0000","last_active":"2023-10-09T11:10:31.446036+0000","last_peered":"2023-10-09T11:10:31.446036+0000","last_clean":"2023-10-09T11:10:31.446036+0000","last_became_active":"2023-10-09T11:10:20.506482+0000","last_became_peered":"2023-10-09T11:10:20.506482+0000","last_unstale":"2023-10-09T11:10:31.446036+0000","last_undegraded":"2023-10-09T11:10:31.446036+0000","last_fullsized":"2023-10-09T11:10:31.446036+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:10:20.610043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":246,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.579245+0000","last_change":"2023-10-09T11:10:11.521500+0000","last_active":"2023-10-09T11:10:32.579245+0000","last_peered":"2023-10-09T11:10:32.579245+0000","last_clean":"2023-10-09T11:10:32.579245+0000","last_became_active":"2023-10-09T11:10:11.519785+0000","last_became_peered":"2023-10-09T11:10:11.519785+0000","last_unstale":"2023-10-09T11:10:32.579245+0000","last_undegraded":"2023-10-09T11:10:32.579245+0000","last_fullsized":"2023-10-09T11:10:32.579245+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:59:20.899382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"76'161","reported_seq":970,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.226516+0000","last_change":"2023-10-09T11:00:15.866625+0000","last_active":"2023-10-09T11:10:32.226516+0000","last_peered":"2023-10-09T11:10:32.226516+0000","last_clean":"2023-10-09T11:10:32.226516+0000","last_became_active":"2023-10-09T11:00:15.866255+0000","last_became_peered":"2023-10-09T11:00:15.866255+0000","last_unstale":"2023-10-09T11:10:32.226516+0000","last_undegraded":"2023-10-09T11:10:32.226516+0000","last_fullsized":"2023-10-09T11:10:32.226516+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:00:03.913971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:00:03.913971+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:25.031224+0000","scrub_duration":0.003385313,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":588,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578422+0000","last_change":"2023-10-09T11:10:11.520888+0000","last_active":"2023-10-09T11:10:32.578422+0000","last_peered":"2023-10-09T11:10:32.578422+0000","last_clean":"2023-10-09T11:10:32.578422+0000","last_became_active":"2023-10-09T11:10:11.520494+0000","last_became_peered":"2023-10-09T11:10:11.520494+0000","last_unstale":"2023-10-09T11:10:32.578422+0000","last_undegraded":"2023-10-09T11:10:32.578422+0000","last_fullsized":"2023-10-09T11:10:32.578422+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:45:12.005423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":256,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.579236+0000","last_change":"2023-10-09T11:10:11.523653+0000","last_active":"2023-10-09T11:10:32.579236+0000","last_peered":"2023-10-09T11:10:32.579236+0000","last_clean":"2023-10-09T11:10:32.579236+0000","last_became_active":"2023-10-09T11:10:11.523317+0000","last_became_peered":"2023-10-09T11:10:11.523317+0000","last_unstale":"2023-10-09T11:10:32.579236+0000","last_undegraded":"2023-10-09T11:10:32.579236+0000","last_fullsized":"2023-10-09T11:10:32.579236+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:31.551257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"76'161","reported_seq":859,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446290+0000","last_change":"2023-10-09T11:05:23.329329+0000","last_active":"2023-10-09T11:10:31.446290+0000","last_peered":"2023-10-09T11:10:31.446290+0000","last_clean":"2023-10-09T11:10:31.446290+0000","last_became_active":"2023-10-09T11:05:23.328976+0000","last_became_peered":"2023-10-09T11:05:23.328976+0000","last_unstale":"2023-10-09T11:10:31.446290+0000","last_undegraded":"2023-10-09T11:10:31.446290+0000","last_fullsized":"2023-10-09T11:10:31.446290+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:51.463238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:51.463238+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:07:41.848184+0000","scrub_duration":0.0080046249999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":164,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.442989+0000","last_change":"2023-10-09T11:07:33.236133+0000","last_active":"2023-10-09T11:10:31.442989+0000","last_peered":"2023-10-09T11:10:31.442989+0000","last_clean":"2023-10-09T11:10:31.442989+0000","last_became_active":"2023-10-09T11:07:33.235818+0000","last_became_peered":"2023-10-09T11:07:33.235818+0000","last_unstale":"2023-10-09T11:10:31.442989+0000","last_undegraded":"2023-10-09T11:10:31.442989+0000","last_fullsized":"2023-10-09T11:10:31.442989+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:20.627846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":325,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.227071+0000","last_change":"2023-10-09T11:04:50.332978+0000","last_active":"2023-10-09T11:10:32.227071+0000","last_peered":"2023-10-09T11:10:32.227071+0000","last_clean":"2023-10-09T11:10:32.227071+0000","last_became_active":"2023-10-09T11:04:50.332735+0000","last_became_peered":"2023-10-09T11:04:50.332735+0000","last_unstale":"2023-10-09T11:10:32.227071+0000","last_undegraded":"2023-10-09T11:10:32.227071+0000","last_fullsized":"2023-10-09T11:10:32.227071+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:39:44.659751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"76'161","reported_seq":995,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445434+0000","last_change":"2023-10-09T11:10:20.447863+0000","last_active":"2023-10-09T11:10:31.445434+0000","last_peered":"2023-10-09T11:10:31.445434+0000","last_clean":"2023-10-09T11:10:31.445434+0000","last_became_active":"2023-10-09T11:10:20.446919+0000","last_became_peered":"2023-10-09T11:10:20.446919+0000","last_unstale":"2023-10-09T11:10:31.445434+0000","last_undegraded":"2023-10-09T11:10:31.445434+0000","last_fullsized":"2023-10-09T11:10:31.445434+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:34.602284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:34.602284+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:43.348375+0000","scrub_duration":0.0058989569999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"592'150","reported_seq":905,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.555247+0000","last_change":"2023-10-09T11:09:20.424914+0000","last_active":"2023-10-09T11:10:33.555247+0000","last_peered":"2023-10-09T11:10:33.555247+0000","last_clean":"2023-10-09T11:10:33.555247+0000","last_became_active":"2023-10-09T11:07:33.226176+0000","last_became_peered":"2023-10-09T11:07:33.226176+0000","last_unstale":"2023-10-09T11:10:33.555247+0000","last_undegraded":"2023-10-09T11:10:33.555247+0000","last_fullsized":"2023-10-09T11:10:33.555247+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:16.888732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034113299999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":876,"num_read_kb":2947,"num_write":148,"num_write_kb":2179,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":492,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446488+0000","last_change":"2023-10-09T11:04:50.341353+0000","last_active":"2023-10-09T11:10:31.446488+0000","last_peered":"2023-10-09T11:10:31.446488+0000","last_clean":"2023-10-09T11:10:31.446488+0000","last_became_active":"2023-10-09T11:04:50.340801+0000","last_became_peered":"2023-10-09T11:04:50.340801+0000","last_unstale":"2023-10-09T11:10:31.446488+0000","last_undegraded":"2023-10-09T11:10:31.446488+0000","last_fullsized":"2023-10-09T11:10:31.446488+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:00.699069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":302,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.554933+0000","last_change":"2023-10-09T11:00:14.889964+0000","last_active":"2023-10-09T11:10:33.554933+0000","last_peered":"2023-10-09T11:10:33.554933+0000","last_clean":"2023-10-09T11:10:33.554933+0000","last_became_active":"2023-10-09T11:00:14.889183+0000","last_became_peered":"2023-10-09T11:00:14.889183+0000","last_unstale":"2023-10-09T11:10:33.554933+0000","last_undegraded":"2023-10-09T11:10:33.554933+0000","last_fullsized":"2023-10-09T11:10:33.554933+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:42:29.789079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"76'161","reported_seq":817,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445421+0000","last_change":"2023-10-09T11:01:15.400417+0000","last_active":"2023-10-09T11:10:31.445421+0000","last_peered":"2023-10-09T11:10:31.445421+0000","last_clean":"2023-10-09T11:10:31.445421+0000","last_became_active":"2023-10-09T11:01:15.400083+0000","last_became_peered":"2023-10-09T11:01:15.400083+0000","last_unstale":"2023-10-09T11:10:31.445421+0000","last_undegraded":"2023-10-09T11:10:31.445421+0000","last_fullsized":"2023-10-09T11:10:31.445421+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:30.549807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:30.549807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:20:50.718380+0000","scrub_duration":0.0081463089999999991,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"592'149","reported_seq":1234,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446059+0000","last_change":"2023-10-09T11:09:20.477238+0000","last_active":"2023-10-09T11:10:31.446059+0000","last_peered":"2023-10-09T11:10:31.446059+0000","last_clean":"2023-10-09T11:10:31.446059+0000","last_became_active":"2023-10-09T11:05:20.315471+0000","last_became_peered":"2023-10-09T11:05:20.315471+0000","last_unstale":"2023-10-09T11:10:31.446059+0000","last_undegraded":"2023-10-09T11:10:31.446059+0000","last_fullsized":"2023-10-09T11:10:31.446059+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:47.548763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051877200000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1181,"num_read_kb":1421,"num_write":125,"num_write_kb":1278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":473,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578918+0000","last_change":"2023-10-09T11:10:11.705748+0000","last_active":"2023-10-09T11:10:32.578918+0000","last_peered":"2023-10-09T11:10:32.578918+0000","last_clean":"2023-10-09T11:10:32.578918+0000","last_became_active":"2023-10-09T11:10:11.705401+0000","last_became_peered":"2023-10-09T11:10:11.705401+0000","last_unstale":"2023-10-09T11:10:32.578918+0000","last_undegraded":"2023-10-09T11:10:32.578918+0000","last_fullsized":"2023-10-09T11:10:32.578918+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:48.364690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":335,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.555091+0000","last_change":"2023-10-09T11:04:50.311579+0000","last_active":"2023-10-09T11:10:33.555091+0000","last_peered":"2023-10-09T11:10:33.555091+0000","last_clean":"2023-10-09T11:10:33.555091+0000","last_became_active":"2023-10-09T11:04:50.311270+0000","last_became_peered":"2023-10-09T11:04:50.311270+0000","last_unstale":"2023-10-09T11:10:33.555091+0000","last_undegraded":"2023-10-09T11:10:33.555091+0000","last_fullsized":"2023-10-09T11:10:33.555091+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:49.850302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"76'161","reported_seq":971,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364221+0000","last_change":"2023-10-09T11:07:36.284280+0000","last_active":"2023-10-09T11:10:32.364221+0000","last_peered":"2023-10-09T11:10:32.364221+0000","last_clean":"2023-10-09T11:10:32.364221+0000","last_became_active":"2023-10-09T11:07:36.283966+0000","last_became_peered":"2023-10-09T11:07:36.283966+0000","last_unstale":"2023-10-09T11:10:32.364221+0000","last_undegraded":"2023-10-09T11:10:32.364221+0000","last_fullsized":"2023-10-09T11:10:32.364221+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":4,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:35.501459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:35.501459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:54.128331+0000","scrub_duration":0.011167761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"591'48","reported_seq":630,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578199+0000","last_change":"2023-10-09T11:09:20.421917+0000","last_active":"2023-10-09T11:10:32.578199+0000","last_peered":"2023-10-09T11:10:32.578199+0000","last_clean":"2023-10-09T11:10:32.578199+0000","last_became_active":"2023-10-09T11:04:50.381253+0000","last_became_peered":"2023-10-09T11:04:50.381253+0000","last_unstale":"2023-10-09T11:10:32.578199+0000","last_undegraded":"2023-10-09T11:10:32.578199+0000","last_fullsized":"2023-10-09T11:10:32.578199+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":48,"log_dups_size":0,"ondisk_log_size":48,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:59:24.030274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001144136,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":297,"num_read_kb":2229,"num_write":37,"num_write_kb":2061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":320,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.226660+0000","last_change":"2023-10-09T11:05:20.298542+0000","last_active":"2023-10-09T11:10:32.226660+0000","last_peered":"2023-10-09T11:10:32.226660+0000","last_clean":"2023-10-09T11:10:32.226660+0000","last_became_active":"2023-10-09T11:05:20.298119+0000","last_became_peered":"2023-10-09T11:05:20.298119+0000","last_unstale":"2023-10-09T11:10:32.226660+0000","last_undegraded":"2023-10-09T11:10:32.226660+0000","last_fullsized":"2023-10-09T11:10:32.226660+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:00.827214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":299,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.226516+0000","last_change":"2023-10-09T11:00:15.865937+0000","last_active":"2023-10-09T11:10:32.226516+0000","last_peered":"2023-10-09T11:10:32.226516+0000","last_clean":"2023-10-09T11:10:32.226516+0000","last_became_active":"2023-10-09T11:00:15.865576+0000","last_became_peered":"2023-10-09T11:00:15.865576+0000","last_unstale":"2023-10-09T11:10:32.226516+0000","last_undegraded":"2023-10-09T11:10:32.226516+0000","last_fullsized":"2023-10-09T11:10:32.226516+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:26:45.097506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"592'125","reported_seq":813,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446343+0000","last_change":"2023-10-09T11:09:20.477093+0000","last_active":"2023-10-09T11:10:31.446343+0000","last_peered":"2023-10-09T11:10:31.446343+0000","last_clean":"2023-10-09T11:10:31.446343+0000","last_became_active":"2023-10-09T11:07:33.226289+0000","last_became_peered":"2023-10-09T11:07:33.226289+0000","last_unstale":"2023-10-09T11:10:31.446343+0000","last_undegraded":"2023-10-09T11:10:31.446343+0000","last_fullsized":"2023-10-09T11:10:31.446343+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:15.605946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026939299999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":322,"num_read_kb":786,"num_write":116,"num_write_kb":327,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'161","reported_seq":916,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.363759+0000","last_change":"2023-10-09T11:07:36.290700+0000","last_active":"2023-10-09T11:10:32.363759+0000","last_peered":"2023-10-09T11:10:32.363759+0000","last_clean":"2023-10-09T11:10:32.363759+0000","last_became_active":"2023-10-09T11:07:36.290357+0000","last_became_peered":"2023-10-09T11:07:36.290357+0000","last_unstale":"2023-10-09T11:10:32.363759+0000","last_undegraded":"2023-10-09T11:10:32.363759+0000","last_fullsized":"2023-10-09T11:10:32.363759+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:49:16.685286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:49:16.685286+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:34.494777+0000","scrub_duration":0.0019529339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":155,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443030+0000","last_change":"2023-10-09T11:05:20.310666+0000","last_active":"2023-10-09T11:10:31.443030+0000","last_peered":"2023-10-09T11:10:31.443030+0000","last_clean":"2023-10-09T11:10:31.443030+0000","last_became_active":"2023-10-09T11:05:20.310413+0000","last_became_peered":"2023-10-09T11:05:20.310413+0000","last_unstale":"2023-10-09T11:10:31.443030+0000","last_undegraded":"2023-10-09T11:10:31.443030+0000","last_fullsized":"2023-10-09T11:10:31.443030+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:32:34.169905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":474,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445593+0000","last_change":"2023-10-09T11:10:20.449809+0000","last_active":"2023-10-09T11:10:31.445593+0000","last_peered":"2023-10-09T11:10:31.445593+0000","last_clean":"2023-10-09T11:10:31.445593+0000","last_became_active":"2023-10-09T11:10:20.449420+0000","last_became_peered":"2023-10-09T11:10:20.449420+0000","last_unstale":"2023-10-09T11:10:31.445593+0000","last_undegraded":"2023-10-09T11:10:31.445593+0000","last_fullsized":"2023-10-09T11:10:31.445593+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:01.442053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"76'161","reported_seq":991,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.555599+0000","last_change":"2023-10-09T11:05:23.322074+0000","last_active":"2023-10-09T11:10:33.555599+0000","last_peered":"2023-10-09T11:10:33.555599+0000","last_clean":"2023-10-09T11:10:33.555599+0000","last_became_active":"2023-10-09T11:05:23.321749+0000","last_became_peered":"2023-10-09T11:05:23.321749+0000","last_unstale":"2023-10-09T11:10:33.555599+0000","last_undegraded":"2023-10-09T11:10:33.555599+0000","last_fullsized":"2023-10-09T11:10:33.555599+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:17.369694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:17.369694+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:07.010932+0000","scrub_duration":0.0016615200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"591'46","reported_seq":373,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364428+0000","last_change":"2023-10-09T11:09:20.432327+0000","last_active":"2023-10-09T11:10:32.364428+0000","last_peered":"2023-10-09T11:10:32.364428+0000","last_clean":"2023-10-09T11:10:32.364428+0000","last_became_active":"2023-10-09T11:02:33.198144+0000","last_became_peered":"2023-10-09T11:02:33.198144+0000","last_unstale":"2023-10-09T11:10:32.364428+0000","last_undegraded":"2023-10-09T11:10:32.364428+0000","last_fullsized":"2023-10-09T11:10:32.364428+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":46,"log_dups_size":0,"ondisk_log_size":46,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:37:37.313591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000350947,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":271,"num_read_kb":1543,"num_write":39,"num_write_kb":1183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":212,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578385+0000","last_change":"2023-10-09T11:10:02.477481+0000","last_active":"2023-10-09T11:10:32.578385+0000","last_peered":"2023-10-09T11:10:32.578385+0000","last_clean":"2023-10-09T11:10:32.578385+0000","last_became_active":"2023-10-09T11:10:02.477147+0000","last_became_peered":"2023-10-09T11:10:02.477147+0000","last_unstale":"2023-10-09T11:10:32.578385+0000","last_undegraded":"2023-10-09T11:10:32.578385+0000","last_fullsized":"2023-10-09T11:10:32.578385+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:47:05.471856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":509,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578799+0000","last_change":"2023-10-09T11:10:02.478037+0000","last_active":"2023-10-09T11:10:32.578799+0000","last_peered":"2023-10-09T11:10:32.578799+0000","last_clean":"2023-10-09T11:10:32.578799+0000","last_became_active":"2023-10-09T11:10:02.477517+0000","last_became_peered":"2023-10-09T11:10:02.477517+0000","last_unstale":"2023-10-09T11:10:32.578799+0000","last_undegraded":"2023-10-09T11:10:32.578799+0000","last_fullsized":"2023-10-09T11:10:32.578799+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:54.404295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"76'161","reported_seq":792,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445763+0000","last_change":"2023-10-09T11:10:20.447023+0000","last_active":"2023-10-09T11:10:31.445763+0000","last_peered":"2023-10-09T11:10:31.445763+0000","last_clean":"2023-10-09T11:10:31.445763+0000","last_became_active":"2023-10-09T11:10:20.446654+0000","last_became_peered":"2023-10-09T11:10:20.446654+0000","last_unstale":"2023-10-09T11:10:31.445763+0000","last_undegraded":"2023-10-09T11:10:31.445763+0000","last_fullsized":"2023-10-09T11:10:31.445763+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:01:16.782451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:01:16.782451+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:51.438822+0000","scrub_duration":0.001715678,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"591'57","reported_seq":277,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.364494+0000","last_change":"2023-10-09T11:09:20.432685+0000","last_active":"2023-10-09T11:10:32.364494+0000","last_peered":"2023-10-09T11:10:32.364494+0000","last_clean":"2023-10-09T11:10:32.364494+0000","last_became_active":"2023-10-09T11:07:33.229061+0000","last_became_peered":"2023-10-09T11:07:33.229061+0000","last_unstale":"2023-10-09T11:10:32.364494+0000","last_undegraded":"2023-10-09T11:10:32.364494+0000","last_fullsized":"2023-10-09T11:10:32.364494+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":57,"log_dups_size":0,"ondisk_log_size":57,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:30.978293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043334800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":262,"num_read_kb":1253,"num_write":50,"num_write_kb":1077,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":302,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.555330+0000","last_change":"2023-10-09T11:00:14.868219+0000","last_active":"2023-10-09T11:10:33.555330+0000","last_peered":"2023-10-09T11:10:33.555330+0000","last_clean":"2023-10-09T11:10:33.555330+0000","last_became_active":"2023-10-09T11:00:14.864488+0000","last_became_peered":"2023-10-09T11:00:14.864488+0000","last_unstale":"2023-10-09T11:10:33.555330+0000","last_undegraded":"2023-10-09T11:10:33.555330+0000","last_fullsized":"2023-10-09T11:10:33.555330+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:54.302738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":529,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578429+0000","last_change":"2023-10-09T11:10:02.479721+0000","last_active":"2023-10-09T11:10:32.578429+0000","last_peered":"2023-10-09T11:10:32.578429+0000","last_clean":"2023-10-09T11:10:32.578429+0000","last_became_active":"2023-10-09T11:10:02.479427+0000","last_became_peered":"2023-10-09T11:10:02.479427+0000","last_unstale":"2023-10-09T11:10:32.578429+0000","last_undegraded":"2023-10-09T11:10:32.578429+0000","last_fullsized":"2023-10-09T11:10:32.578429+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:31.125346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"76'161","reported_seq":928,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.579132+0000","last_change":"2023-10-09T11:10:11.523340+0000","last_active":"2023-10-09T11:10:32.579132+0000","last_peered":"2023-10-09T11:10:32.579132+0000","last_clean":"2023-10-09T11:10:32.579132+0000","last_became_active":"2023-10-09T11:10:11.522841+0000","last_became_peered":"2023-10-09T11:10:11.522841+0000","last_unstale":"2023-10-09T11:10:32.579132+0000","last_undegraded":"2023-10-09T11:10:32.579132+0000","last_fullsized":"2023-10-09T11:10:32.579132+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:22.817272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:22.817272+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:41.203109+0000","scrub_duration":0.0037682520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":176,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443616+0000","last_change":"2023-10-09T11:07:33.246378+0000","last_active":"2023-10-09T11:10:31.443616+0000","last_peered":"2023-10-09T11:10:31.443616+0000","last_clean":"2023-10-09T11:10:31.443616+0000","last_became_active":"2023-10-09T11:07:33.228019+0000","last_became_peered":"2023-10-09T11:07:33.228019+0000","last_unstale":"2023-10-09T11:10:31.443616+0000","last_undegraded":"2023-10-09T11:10:31.443616+0000","last_fullsized":"2023-10-09T11:10:31.443616+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T19:14:18.511517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"592'319","reported_seq":1609,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.227139+0000","last_change":"2023-10-09T11:09:20.427659+0000","last_active":"2023-10-09T11:10:32.227139+0000","last_peered":"2023-10-09T11:10:32.227139+0000","last_clean":"2023-10-09T11:10:32.227139+0000","last_became_active":"2023-10-09T11:05:20.309827+0000","last_became_peered":"2023-10-09T11:05:20.309827+0000","last_unstale":"2023-10-09T11:10:32.227139+0000","last_undegraded":"2023-10-09T11:10:32.227139+0000","last_fullsized":"2023-10-09T11:10:32.227139+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":319,"log_dups_size":0,"ondisk_log_size":319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:23.934383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001122743,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2478,"num_read_kb":2521,"num_write":249,"num_write_kb":324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":165,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.459452+0000","last_change":"2023-10-09T11:07:33.217595+0000","last_active":"2023-10-09T11:10:33.459452+0000","last_peered":"2023-10-09T11:10:33.459452+0000","last_clean":"2023-10-09T11:10:33.459452+0000","last_became_active":"2023-10-09T11:07:33.217155+0000","last_became_peered":"2023-10-09T11:07:33.217155+0000","last_unstale":"2023-10-09T11:10:33.459452+0000","last_undegraded":"2023-10-09T11:10:33.459452+0000","last_fullsized":"2023-10-09T11:10:33.459452+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:55.127329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"76'161","reported_seq":783,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443134+0000","last_change":"2023-10-09T11:07:36.284491+0000","last_active":"2023-10-09T11:10:31.443134+0000","last_peered":"2023-10-09T11:10:31.443134+0000","last_clean":"2023-10-09T11:10:31.443134+0000","last_became_active":"2023-10-09T11:07:36.284249+0000","last_became_peered":"2023-10-09T11:07:36.284249+0000","last_unstale":"2023-10-09T11:10:31.443134+0000","last_undegraded":"2023-10-09T11:10:31.443134+0000","last_fullsized":"2023-10-09T11:10:31.443134+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":3,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:05:03.653371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:05:03.653371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T22:32:09.629417+0000","scrub_duration":0.001576771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"592'82","reported_seq":273,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.445963+0000","last_change":"2023-10-09T11:10:20.448154+0000","last_active":"2023-10-09T11:10:31.445963+0000","last_peered":"2023-10-09T11:10:31.445963+0000","last_clean":"2023-10-09T11:10:31.445963+0000","last_became_active":"2023-10-09T11:10:20.447384+0000","last_became_peered":"2023-10-09T11:10:20.447384+0000","last_unstale":"2023-10-09T11:10:31.445963+0000","last_undegraded":"2023-10-09T11:10:31.445963+0000","last_fullsized":"2023-10-09T11:10:31.445963+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:19.857573+0000","last_clean_scrub_stamp":"2023-10-09T10:35:19.857573+0000","objects_scrubbed":0,"log_size":82,"log_dups_size":0,"ondisk_log_size":82,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:03:31.024074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075188800000000004,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":544,"num_write":82,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":493,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578852+0000","last_change":"2023-10-09T11:10:02.477655+0000","last_active":"2023-10-09T11:10:32.578852+0000","last_peered":"2023-10-09T11:10:32.578852+0000","last_clean":"2023-10-09T11:10:32.578852+0000","last_became_active":"2023-10-09T11:10:02.477293+0000","last_became_peered":"2023-10-09T11:10:02.477293+0000","last_unstale":"2023-10-09T11:10:32.578852+0000","last_undegraded":"2023-10-09T11:10:32.578852+0000","last_fullsized":"2023-10-09T11:10:32.578852+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:24.604828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":358,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578414+0000","last_change":"2023-10-09T11:10:11.671501+0000","last_active":"2023-10-09T11:10:32.578414+0000","last_peered":"2023-10-09T11:10:32.578414+0000","last_clean":"2023-10-09T11:10:32.578414+0000","last_became_active":"2023-10-09T11:10:11.671154+0000","last_became_peered":"2023-10-09T11:10:11.671154+0000","last_unstale":"2023-10-09T11:10:32.578414+0000","last_undegraded":"2023-10-09T11:10:32.578414+0000","last_fullsized":"2023-10-09T11:10:32.578414+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:28:35.689307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"76'161","reported_seq":1027,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443242+0000","last_change":"2023-10-09T11:05:23.329886+0000","last_active":"2023-10-09T11:10:31.443242+0000","last_peered":"2023-10-09T11:10:31.443242+0000","last_clean":"2023-10-09T11:10:31.443242+0000","last_became_active":"2023-10-09T11:05:23.329561+0000","last_became_peered":"2023-10-09T11:05:23.329561+0000","last_unstale":"2023-10-09T11:10:31.443242+0000","last_undegraded":"2023-10-09T11:10:31.443242+0000","last_fullsized":"2023-10-09T11:10:31.443242+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:29.568072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:29.568072+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:46.933799+0000","scrub_duration":0.003792592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":501,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578508+0000","last_change":"2023-10-09T11:10:02.479983+0000","last_active":"2023-10-09T11:10:32.578508+0000","last_peered":"2023-10-09T11:10:32.578508+0000","last_clean":"2023-10-09T11:10:32.578508+0000","last_became_active":"2023-10-09T11:10:02.479627+0000","last_became_peered":"2023-10-09T11:10:02.479627+0000","last_unstale":"2023-10-09T11:10:32.578508+0000","last_undegraded":"2023-10-09T11:10:32.578508+0000","last_fullsized":"2023-10-09T11:10:32.578508+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:36:14.632490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":402,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578391+0000","last_change":"2023-10-09T11:04:50.510688+0000","last_active":"2023-10-09T11:10:32.578391+0000","last_peered":"2023-10-09T11:10:32.578391+0000","last_clean":"2023-10-09T11:10:32.578391+0000","last_became_active":"2023-10-09T11:04:50.507404+0000","last_became_peered":"2023-10-09T11:04:50.507404+0000","last_unstale":"2023-10-09T11:10:32.578391+0000","last_undegraded":"2023-10-09T11:10:32.578391+0000","last_fullsized":"2023-10-09T11:10:32.578391+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:26:13.663394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"76'161","reported_seq":985,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.459948+0000","last_change":"2023-10-09T11:05:23.321040+0000","last_active":"2023-10-09T11:10:33.459948+0000","last_peered":"2023-10-09T11:10:33.459948+0000","last_clean":"2023-10-09T11:10:33.459948+0000","last_became_active":"2023-10-09T11:05:23.320674+0000","last_became_peered":"2023-10-09T11:05:23.320674+0000","last_unstale":"2023-10-09T11:10:33.459948+0000","last_undegraded":"2023-10-09T11:10:33.459948+0000","last_fullsized":"2023-10-09T11:10:33.459948+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T10:53:36.563780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T10:53:36.563780+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:01.527660+0000","scrub_duration":0.0047028759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":444,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446353+0000","last_change":"2023-10-09T11:01:15.403552+0000","last_active":"2023-10-09T11:10:31.446353+0000","last_peered":"2023-10-09T11:10:31.446353+0000","last_clean":"2023-10-09T11:10:31.446353+0000","last_became_active":"2023-10-09T11:01:15.403192+0000","last_became_peered":"2023-10-09T11:01:15.403192+0000","last_unstale":"2023-10-09T11:10:31.446353+0000","last_undegraded":"2023-10-09T11:10:31.446353+0000","last_fullsized":"2023-10-09T11:10:31.446353+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:47.750908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":383,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.579008+0000","last_change":"2023-10-09T11:10:11.516290+0000","last_active":"2023-10-09T11:10:32.579008+0000","last_peered":"2023-10-09T11:10:32.579008+0000","last_clean":"2023-10-09T11:10:32.579008+0000","last_became_active":"2023-10-09T11:10:11.515955+0000","last_became_peered":"2023-10-09T11:10:11.515955+0000","last_unstale":"2023-10-09T11:10:32.579008+0000","last_undegraded":"2023-10-09T11:10:32.579008+0000","last_fullsized":"2023-10-09T11:10:32.579008+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T21:38:59.774818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":595,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.446480+0000","last_change":"2023-10-09T11:05:20.327032+0000","last_active":"2023-10-09T11:10:31.446480+0000","last_peered":"2023-10-09T11:10:31.446480+0000","last_clean":"2023-10-09T11:10:31.446480+0000","last_became_active":"2023-10-09T11:05:20.320193+0000","last_became_peered":"2023-10-09T11:05:20.320193+0000","last_unstale":"2023-10-09T11:10:31.446480+0000","last_undegraded":"2023-10-09T11:10:31.446480+0000","last_fullsized":"2023-10-09T11:10:31.446480+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:44.862890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":373,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578539+0000","last_change":"2023-10-09T11:10:11.705871+0000","last_active":"2023-10-09T11:10:32.578539+0000","last_peered":"2023-10-09T11:10:32.578539+0000","last_clean":"2023-10-09T11:10:32.578539+0000","last_became_active":"2023-10-09T11:10:11.705528+0000","last_became_peered":"2023-10-09T11:10:11.705528+0000","last_unstale":"2023-10-09T11:10:32.578539+0000","last_undegraded":"2023-10-09T11:10:32.578539+0000","last_fullsized":"2023-10-09T11:10:32.578539+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:19.337219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"76'161","reported_seq":802,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.363769+0000","last_change":"2023-10-09T11:08:39.540029+0000","last_active":"2023-10-09T11:10:32.363769+0000","last_peered":"2023-10-09T11:10:32.363769+0000","last_clean":"2023-10-09T11:10:32.363769+0000","last_became_active":"2023-10-09T11:07:36.291077+0000","last_became_peered":"2023-10-09T11:07:36.291077+0000","last_unstale":"2023-10-09T11:10:32.363769+0000","last_undegraded":"2023-10-09T11:10:32.363769+0000","last_fullsized":"2023-10-09T11:10:32.363769+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:08:39.539966+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:08:39.539966+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:20.767812+0000","scrub_duration":0.0016245929999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":156,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-10-09T11:10:33.459994+0000","last_change":"2023-10-09T11:05:20.299321+0000","last_active":"2023-10-09T11:10:33.459994+0000","last_peered":"2023-10-09T11:10:33.459994+0000","last_clean":"2023-10-09T11:10:33.459994+0000","last_became_active":"2023-10-09T11:05:20.299019+0000","last_became_peered":"2023-10-09T11:05:20.299019+0000","last_unstale":"2023-10-09T11:10:33.459994+0000","last_undegraded":"2023-10-09T11:10:33.459994+0000","last_fullsized":"2023-10-09T11:10:33.459994+0000","mapping_epoch":499,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:08.645909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":146,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:31.443275+0000","last_change":"2023-10-09T11:04:50.366890+0000","last_active":"2023-10-09T11:10:31.443275+0000","last_peered":"2023-10-09T11:10:31.443275+0000","last_clean":"2023-10-09T11:10:31.443275+0000","last_became_active":"2023-10-09T11:04:50.366558+0000","last_became_peered":"2023-10-09T11:04:50.366558+0000","last_unstale":"2023-10-09T11:10:31.443275+0000","last_undegraded":"2023-10-09T11:10:31.443275+0000","last_fullsized":"2023-10-09T11:10:31.443275+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:50.993153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"76'161","reported_seq":919,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.227064+0000","last_change":"2023-10-09T11:04:53.274332+0000","last_active":"2023-10-09T11:10:32.227064+0000","last_peered":"2023-10-09T11:10:32.227064+0000","last_clean":"2023-10-09T11:10:32.227064+0000","last_became_active":"2023-10-09T11:04:53.273798+0000","last_became_peered":"2023-10-09T11:04:53.273798+0000","last_unstale":"2023-10-09T11:10:32.227064+0000","last_undegraded":"2023-10-09T11:10:32.227064+0000","last_fullsized":"2023-10-09T11:10:32.227064+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":228,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":5,"last_scrub":"76'161","last_scrub_stamp":"2023-10-09T11:02:25.273107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:33:35.022165+0000","last_clean_scrub_stamp":"2023-10-09T11:02:25.273107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:04.220970+0000","scrub_duration":0.0067745720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":194,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578859+0000","last_change":"2023-10-09T11:10:02.473453+0000","last_active":"2023-10-09T11:10:32.578859+0000","last_peered":"2023-10-09T11:10:32.578859+0000","last_clean":"2023-10-09T11:10:32.578859+0000","last_became_active":"2023-10-09T11:10:02.473105+0000","last_became_peered":"2023-10-09T11:10:02.473105+0000","last_unstale":"2023-10-09T11:10:32.578859+0000","last_undegraded":"2023-10-09T11:10:32.578859+0000","last_fullsized":"2023-10-09T11:10:32.578859+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:55.799457+0000","last_clean_scrub_stamp":"2023-10-09T10:35:55.799457+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:54:11.409980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":562,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-10-09T11:10:32.578304+0000","last_change":"2023-10-09T11:10:11.523400+0000","last_active":"2023-10-09T11:10:32.578304+0000","last_peered":"2023-10-09T11:10:32.578304+0000","last_clean":"2023-10-09T11:10:32.578304+0000","last_became_active":"2023-10-09T11:10:11.523063+0000","last_became_peered":"2023-10-09T11:10:11.523063+0000","last_unstale":"2023-10-09T11:10:32.578304+0000","last_undegraded":"2023-10-09T11:10:32.578304+0000","last_fullsized":"2023-10-09T11:10:32.578304+0000","mapping_epoch":607,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T10:35:56.801286+0000","last_clean_scrub_stamp":"2023-10-09T10:35:56.801286+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T20:41:18.121353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":7936,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12694,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13244,"num_write":846,"num_write_kb":12712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332532,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":910328,"kb_used_data":10928,"kb_used_omap":0,"kb_used_meta":899392,"kb_avail":92834824,"statfs":{"total":95995035648,"available":95062859776,"internally_reserved":0,"allocated":11190272,"data_stored":9331566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920977408},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:10:23 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":1.022,"15min":1.5409999999999999},"min":{"1min":0.27700000000000002,"5min":0.20599999999999999,"15min":0.20100000000000001},"max":{"1min":0.65100000000000002,"5min":9.8219999999999992,"15min":9.8219999999999992},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.55600000000000005,"5min":1.242,"15min":1.577},"min":{"1min":0.313,"5min":0.24299999999999999,"15min":0.216},"max":{"1min":0.81200000000000006,"5min":15.853,"15min":15.853},"last":0.55600000000000005}]},{"osd":1,"last update":"Mon Oct 9 11:10:28 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.1379999999999999,"15min":1.395},"min":{"1min":0.34300000000000003,"5min":0.222,"15min":0.222},"max":{"1min":0.89000000000000001,"5min":11.619999999999999,"15min":11.619999999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.5880000000000001,"15min":1.7430000000000001},"min":{"1min":0.34899999999999998,"5min":0.25,"15min":0.25},"max":{"1min":0.73299999999999998,"5min":13.096,"15min":13.096},"last":0.88300000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":1.1439999999999999,"15min":1.722},"min":{"1min":0.35099999999999998,"5min":0.309,"15min":0.245},"max":{"1min":0.59499999999999997,"5min":8.5340000000000007,"15min":16.605},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":1.2749999999999999,"15min":1.6830000000000001},"min":{"1min":0.34100000000000003,"5min":0.223,"15min":0.223},"max":{"1min":0.65800000000000003,"5min":16.157,"15min":16.157},"last":0.54100000000000004}]},{"osd":3,"last update":"Mon Oct 9 11:10:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.389,"15min":1.395},"min":{"1min":0.39800000000000002,"5min":0.30499999999999999,"15min":0.25600000000000001},"max":{"1min":0.72999999999999998,"5min":7.4770000000000003,"15min":7.4770000000000003},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":1.4159999999999999,"15min":1.6659999999999999},"min":{"1min":0.40999999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.85999999999999999,"5min":6.3920000000000003,"15min":8.2330000000000005},"last":0.52200000000000002}]},{"osd":4,"last update":"Mon Oct 9 11:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.752,"15min":1.032},"min":{"1min":0.40600000000000003,"5min":0.247,"15min":0.218},"max":{"1min":0.72599999999999998,"5min":7.4630000000000001,"15min":7.96},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.81699999999999995,"15min":1.22},"min":{"1min":0.36499999999999999,"5min":0.28299999999999997,"15min":0.27900000000000003},"max":{"1min":0.64900000000000002,"5min":9.2759999999999998,"15min":16.481000000000002},"last":0.45300000000000001}]},{"osd":5,"last update":"Mon Oct 9 11:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.67200000000000004,"15min":1.2070000000000001},"min":{"1min":0.32800000000000001,"5min":0.249,"15min":0.247},"max":{"1min":0.68500000000000005,"5min":5.4800000000000004,"15min":15.414999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.004,"15min":1.1679999999999999},"min":{"1min":0.372,"5min":0.317,"15min":0.26400000000000001},"max":{"1min":0.75,"5min":13.045,"15min":15.369},"last":0.47399999999999998}]},{"osd":6,"last update":"Mon Oct 9 11:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.81699999999999995,"15min":1.1950000000000001},"min":{"1min":0.41999999999999998,"5min":0.34599999999999997,"15min":0.315},"max":{"1min":0.77400000000000002,"5min":4.8079999999999998,"15min":14.159000000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.87,"15min":1.3280000000000001},"min":{"1min":0.39700000000000002,"5min":0.32900000000000001,"15min":0.29399999999999998},"max":{"1min":0.71399999999999997,"5min":3.9630000000000001,"15min":16.538},"last":0.59399999999999997}]}]},{"osd":6,"up_from":44,"seq":188978561463,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1511724,"kb_used_data":10980,"kb_used_omap":0,"kb_used_meta":1500736,"kb_avail":92233428,"statfs":{"total":95995035648,"available":94447030272,"internally_reserved":0,"allocated":11243520,"data_stored":9396431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1536753664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:10:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.99199999999999999,"15min":1.8480000000000001},"min":{"1min":0.33500000000000002,"5min":0.223,"15min":0.217},"max":{"1min":0.69899999999999995,"5min":12.209,"15min":54.743000000000002},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.84599999999999997,"15min":1.8009999999999999},"min":{"1min":0.39300000000000002,"5min":0.247,"15min":0.214},"max":{"1min":0.746,"5min":7.1890000000000001,"15min":53.701000000000001},"last":0.66100000000000003}]},{"osd":1,"last update":"Mon Oct 9 11:10:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":1.0089999999999999,"15min":1.1619999999999999},"min":{"1min":0.35899999999999999,"5min":0.26900000000000002,"15min":0.23499999999999999},"max":{"1min":0.746,"5min":5.0110000000000001,"15min":41.241999999999997},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.52000000000000002,"5min":1.5209999999999999,"15min":1.71},"min":{"1min":0.38200000000000001,"5min":0.28999999999999998,"15min":0.248},"max":{"1min":0.73499999999999999,"5min":9.2080000000000002,"15min":41.476999999999997},"last":0.51000000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.4399999999999999,"15min":1.8220000000000001},"min":{"1min":0.32500000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.70499999999999996,"5min":10.382999999999999,"15min":12.003},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.93600000000000005,"15min":1.77},"min":{"1min":0.433,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.68999999999999995,"5min":7.133,"15min":15.432},"last":0.40899999999999997}]},{"osd":3,"last update":"Mon Oct 9 11:10:31 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.8540000000000001,"15min":1.5840000000000001},"min":{"1min":0.42699999999999999,"5min":0.313,"15min":0.313},"max":{"1min":0.77400000000000002,"5min":28.948,"15min":28.948},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.58799999999999997,"5min":2.2269999999999999,"15min":2.3679999999999999},"min":{"1min":0.44600000000000001,"5min":0.307,"15min":0.30199999999999999},"max":{"1min":0.71999999999999997,"5min":28.893000000000001,"15min":28.893000000000001},"last":0.58299999999999996}]},{"osd":4,"last update":"Mon Oct 9 11:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.48599999999999999,"15min":0.96599999999999997},"min":{"1min":0.309,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.73099999999999998,"5min":1.6519999999999999,"15min":17.216999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.51000000000000001,"15min":0.77100000000000002},"min":{"1min":0.39000000000000001,"5min":0.26900000000000002,"15min":0.24399999999999999},"max":{"1min":0.78400000000000003,"5min":1.5089999999999999,"15min":10.025},"last":0.71999999999999997}]},{"osd":5,"last update":"Mon Oct 9 11:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.60999999999999999,"15min":1.1379999999999999},"min":{"1min":0.34699999999999998,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":0.80100000000000005,"5min":2.7000000000000002,"15min":19.786999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.60999999999999999,"15min":0.90100000000000002},"min":{"1min":0.44400000000000001,"5min":0.30499999999999999,"15min":0.248},"max":{"1min":0.81599999999999995,"5min":3.488,"15min":9.9469999999999992},"last":0.55600000000000005}]},{"osd":7,"last update":"Mon Oct 9 11:10:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.66200000000000003,"15min":1.119},"min":{"1min":0.38100000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.79200000000000004,"5min":3.7250000000000001,"15min":11.550000000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.69899999999999995,"15min":0.84799999999999998},"min":{"1min":0.42199999999999999,"5min":0.32800000000000001,"15min":0.27500000000000002},"max":{"1min":0.84899999999999998,"5min":3.2839999999999998,"15min":11.557},"last":0.75700000000000001}]}]},{"osd":1,"up_from":388,"seq":1666447310978,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":505544,"kb_used_data":8064,"kb_used_omap":13,"kb_used_meta":497458,"kb_avail":93239608,"statfs":{"total":95995035648,"available":95477358592,"internally_reserved":0,"allocated":8257536,"data_stored":5057756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13625,"internal_metadata":509397703},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":2.1299999999999999,"15min":2.2109999999999999},"min":{"1min":0.20499999999999999,"5min":0.182,"15min":0.17699999999999999},"max":{"1min":5.4720000000000004,"5min":18.192,"15min":18.192},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.40200000000000002,"5min":2.3650000000000002,"15min":1.9730000000000001},"min":{"1min":0.245,"5min":0.22900000000000001,"15min":0.19900000000000001},"max":{"1min":1.3180000000000001,"5min":17.957999999999998,"15min":17.957999999999998},"last":0.33000000000000002}]},{"osd":2,"last update":"Mon Oct 9 11:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.8660000000000001,"15min":2.5550000000000002},"min":{"1min":0.249,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":5.798,"5min":11.875,"15min":47.731000000000002},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.41199999999999998,"5min":2.1850000000000001,"15min":2.347},"min":{"1min":0.27700000000000002,"5min":0.216,"15min":0.216},"max":{"1min":1.2769999999999999,"5min":16.870000000000001,"15min":47.786999999999999},"last":0.30199999999999999}]},{"osd":3,"last update":"Mon Oct 9 11:10:31 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":2.9140000000000001,"15min":1.964},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.497,"5min":92.447999999999993,"15min":92.447999999999993},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.34599999999999997,"5min":3.016,"15min":1.994},"min":{"1min":0.23999999999999999,"5min":0.217,"15min":0.217},"max":{"1min":0.41599999999999998,"5min":92.066999999999993,"15min":92.066999999999993},"last":0.34200000000000003}]},{"osd":4,"last update":"Mon Oct 9 11:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":1.5269999999999999,"15min":2.048},"min":{"1min":0.35099999999999998,"5min":0.27300000000000002,"15min":0.26000000000000001},"max":{"1min":5.407,"5min":11.701000000000001,"15min":11.701000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.7230000000000001,"15min":1.621},"min":{"1min":0.38200000000000001,"5min":0.314,"15min":0.29999999999999999},"max":{"1min":1.3759999999999999,"5min":16.841000000000001,"15min":16.841000000000001},"last":0.57299999999999995}]},{"osd":5,"last update":"Mon Oct 9 11:10:37 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":1.792,"15min":2.089},"min":{"1min":0.374,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":23.989000000000001,"5min":23.989000000000001,"15min":23.989000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.79900000000000004,"5min":1.615,"15min":2.016},"min":{"1min":0.45700000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.4459999999999997,"5min":11.65,"15min":11.65},"last":0.54000000000000004}]},{"osd":6,"last update":"Mon Oct 9 11:10:37 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":1.853,"15min":2.1819999999999999},"min":{"1min":0.42299999999999999,"5min":0.312,"15min":0.309},"max":{"1min":24.024999999999999,"5min":24.024999999999999,"15min":24.024999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.77500000000000002,"5min":1.6259999999999999,"15min":1.9430000000000001},"min":{"1min":0.42299999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":5.375,"5min":11.75,"15min":11.75},"last":0.499}]},{"osd":7,"last update":"Mon Oct 9 11:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":1.7509999999999999,"15min":1.6379999999999999},"min":{"1min":0.40899999999999997,"5min":0.33000000000000002,"15min":0.32900000000000001},"max":{"1min":1.3480000000000001,"5min":16.797999999999998,"15min":16.797999999999998},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.8260000000000001,"15min":2.0910000000000002},"min":{"1min":0.44700000000000001,"5min":0.36899999999999999,"15min":0.33200000000000002},"max":{"1min":23.923999999999999,"5min":23.923999999999999,"15min":23.923999999999999},"last":0.48399999999999999}]}]},{"osd":0,"up_from":80,"seq":343597384036,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":986628,"kb_used_data":8444,"kb_used_omap":28,"kb_used_meta":978147,"kb_avail":92758524,"statfs":{"total":95995035648,"available":94984728576,"internally_reserved":0,"allocated":8646656,"data_stored":5380705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":1001623376},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 11:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":1.502,"15min":2.3540000000000001},"min":{"1min":0.22700000000000001,"5min":0.186,"15min":0.186},"max":{"1min":0.503,"5min":10.271000000000001,"15min":19.608000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.35999999999999999,"5min":1.7729999999999999,"15min":2.4129999999999998},"min":{"1min":0.25,"5min":0.214,"15min":0.214},"max":{"1min":0.53800000000000003,"5min":15.834,"15min":19.992999999999999},"last":0.36799999999999999}]},{"osd":2,"last update":"Mon Oct 9 11:10:00 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":1.629,"15min":2.2109999999999999},"min":{"1min":0.26400000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":1.6479999999999999,"5min":11.516,"15min":24.852},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.442,"5min":2.5659999999999998,"15min":2.9060000000000001},"min":{"1min":0.30499999999999999,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":0.81899999999999995,"5min":99.679000000000002,"15min":99.679000000000002},"last":0.34899999999999998}]},{"osd":3,"last update":"Mon Oct 9 11:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":1.8939999999999999,"15min":2.9860000000000002},"min":{"1min":0.29999999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.54500000000000004,"5min":13.189,"15min":25.050000000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.42699999999999999,"5min":1.7829999999999999,"15min":1.8600000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.60899999999999999,"5min":14.125999999999999,"15min":14.125999999999999},"last":0.32200000000000001}]},{"osd":4,"last update":"Mon Oct 9 11:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.4610000000000001,"15min":2.0710000000000002},"min":{"1min":0.41299999999999998,"5min":0.27100000000000002,"15min":0.25700000000000001},"max":{"1min":0.879,"5min":13.106,"15min":14.507},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.23,"15min":1.538},"min":{"1min":0.432,"5min":0.33600000000000002,"15min":0.29199999999999998},"max":{"1min":0.88,"5min":7.9249999999999998,"15min":10.513999999999999},"last":0.67800000000000005}]},{"osd":5,"last update":"Mon Oct 9 11:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":1.276,"15min":1.6679999999999999},"min":{"1min":0.39200000000000002,"5min":0.33200000000000002,"15min":0.30399999999999999},"max":{"1min":0.79500000000000004,"5min":7.9550000000000001,"15min":10.544},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.63600000000000001,"5min":1.276,"15min":1.6539999999999999},"min":{"1min":0.46200000000000002,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":0.97099999999999997,"5min":8.8439999999999994,"15min":11.212999999999999},"last":0.66800000000000004}]},{"osd":6,"last update":"Mon Oct 9 11:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":1.3979999999999999,"15min":1.74},"min":{"1min":0.47499999999999998,"5min":0.377,"15min":0.314},"max":{"1min":0.81200000000000006,"5min":7.9829999999999997,"15min":18.920000000000002},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.2969999999999999,"15min":1.7210000000000001},"min":{"1min":0.47599999999999998,"5min":0.378,"15min":0.31900000000000001},"max":{"1min":0.93200000000000005,"5min":8.8810000000000002,"15min":11.257999999999999},"last":0.59599999999999997}]},{"osd":7,"last update":"Mon Oct 9 11:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.5940000000000001,"15min":1.8879999999999999},"min":{"1min":0.46500000000000002,"5min":0.38900000000000001,"15min":0.32700000000000001},"max":{"1min":0.94899999999999995,"5min":9.8710000000000004,"15min":17.672000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.6259999999999999,"15min":2.2010000000000001},"min":{"1min":0.435,"5min":0.311,"15min":0.311},"max":{"1min":0.95899999999999996,"5min":13.154,"15min":17.539000000000001},"last":0.60699999999999998}]}]},{"osd":2,"up_from":90,"seq":386547056976,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":770136,"kb_used_data":12304,"kb_used_omap":81,"kb_used_meta":757742,"kb_avail":92975016,"statfs":{"total":95995035648,"available":95206416384,"internally_reserved":0,"allocated":12599296,"data_stored":9495088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":775928026},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":1.627,"15min":2.0070000000000001},"min":{"1min":0.25600000000000001,"5min":0.182,"15min":0.17699999999999999},"max":{"1min":0.51300000000000001,"5min":15.609,"15min":48.933999999999997},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.35999999999999999,"5min":1.7050000000000001,"15min":1.776},"min":{"1min":0.29399999999999998,"5min":0.23100000000000001,"15min":0.19600000000000001},"max":{"1min":0.47699999999999998,"5min":9.8010000000000002,"15min":14.515000000000001},"last":0.27500000000000002}]},{"osd":1,"last update":"Mon Oct 9 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.31900000000000001,"5min":1.609,"15min":2.2469999999999999},"min":{"1min":0.23000000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":0.55800000000000005,"5min":24.902000000000001,"15min":49.033999999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.38200000000000001,"5min":1.9670000000000001,"15min":2.4199999999999999},"min":{"1min":0.255,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":0.54000000000000004,"5min":25.850999999999999,"15min":48.874000000000002},"last":0.318}]},{"osd":3,"last update":"Mon Oct 9 11:10:24 2023","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":2.601,"15min":2.0430000000000001},"min":{"1min":0.24099999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.57099999999999995,"5min":25.890999999999998,"15min":25.890999999999998},"last":0.371},{"interface":"front","average":{"1min":0.434,"5min":2.3759999999999999,"15min":2.222},"min":{"1min":0.28599999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.59099999999999997,"5min":28.689,"15min":28.689},"last":0.29399999999999998}]},{"osd":4,"last update":"Mon Oct 9 11:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.149,"15min":1.728},"min":{"1min":0.40300000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.84399999999999997,"5min":11.391999999999999,"15min":12.452999999999999},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.63700000000000001,"5min":1.385,"15min":1.78},"min":{"1min":0.44900000000000001,"5min":0.33900000000000002,"15min":0.249},"max":{"1min":0.83399999999999996,"5min":15.571,"15min":48.795000000000002},"last":0.68200000000000005}]},{"osd":5,"last update":"Mon Oct 9 11:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.538,"15min":1.4910000000000001},"min":{"1min":0.39900000000000002,"5min":0.32400000000000001,"15min":0.25900000000000001},"max":{"1min":0.90900000000000003,"5min":8.5310000000000006,"15min":10.135999999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.629,"5min":1.2090000000000001,"15min":1.8420000000000001},"min":{"1min":0.38800000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.91600000000000004,"5min":11.441000000000001,"15min":49.061},"last":0.65800000000000003}]},{"osd":6,"last update":"Mon Oct 9 11:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":1.411,"15min":1.7909999999999999},"min":{"1min":0.46200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":0.93899999999999995,"5min":15.536,"15min":48.845999999999997},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.7350000000000001,"15min":1.587},"min":{"1min":0.45000000000000001,"5min":0.36799999999999999,"15min":0.29199999999999998},"max":{"1min":0.81000000000000005,"5min":10.157999999999999,"15min":10.157999999999999},"last":0.56899999999999995}]},{"osd":7,"last update":"Mon Oct 9 11:10:06 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.5800000000000001,"15min":1.9530000000000001},"min":{"1min":0.44,"5min":0.34000000000000002,"15min":0.31900000000000001},"max":{"1min":0.88,"5min":15.59,"15min":48.966000000000001},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.64200000000000002,"5min":1.6830000000000001,"15min":1.681},"min":{"1min":0.46300000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.86199999999999999,"5min":8.5549999999999997,"15min":14.94},"last":0.59499999999999997}]}]},{"osd":3,"up_from":469,"seq":2014339661906,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":343692,"kb_used_data":8644,"kb_used_omap":11,"kb_used_meta":335028,"kb_avail":93401460,"statfs":{"total":95995035648,"available":95643095040,"internally_reserved":0,"allocated":8851456,"data_stored":5404132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11309,"internal_metadata":343069651},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":1.643,"15min":1.6950000000000001},"min":{"1min":0.252,"5min":0.183,"15min":0.183},"max":{"1min":0.52200000000000002,"5min":14.521000000000001,"15min":14.882},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.374,"5min":1.238,"15min":0.99099999999999999},"min":{"1min":0.246,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.55800000000000005,"5min":9.4030000000000005,"15min":9.4030000000000005},"last":0.35299999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":1.863,"15min":1.5549999999999999},"min":{"1min":0.248,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":0.51600000000000001,"5min":17.93,"15min":17.93},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.39100000000000001,"5min":1.982,"15min":1.7170000000000001},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.55400000000000005,"5min":19.163,"15min":19.163},"last":0.29099999999999998}]},{"osd":2,"last update":"Mon Oct 9 11:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":1.8879999999999999,"15min":1.3540000000000001},"min":{"1min":0.217,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.55800000000000005,"5min":38.840000000000003,"15min":38.840000000000003},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.41799999999999998,"5min":2.004,"15min":1.3879999999999999},"min":{"1min":0.28599999999999998,"5min":0.219,"15min":0.219},"max":{"1min":0.66300000000000003,"5min":37.991999999999997,"15min":37.991999999999997},"last":0.309}]},{"osd":4,"last update":"Mon Oct 9 11:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.264,"15min":1.47},"min":{"1min":0.35999999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.98699999999999999,"5min":10.058999999999999,"15min":14.907999999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.58299999999999996,"5min":1.101,"15min":0.83099999999999996},"min":{"1min":0.42799999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.95699999999999996,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.65400000000000003}]},{"osd":5,"last update":"Mon Oct 9 11:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":1.1160000000000001,"15min":1.3140000000000001},"min":{"1min":0.47399999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.02,"5min":5.8499999999999996,"15min":9.3300000000000001},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.64200000000000002,"5min":1.24,"15min":1.3740000000000001},"min":{"1min":0.46700000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.069,"5min":10.029,"15min":10.029},"last":0.64900000000000002}]},{"osd":6,"last update":"Mon Oct 9 11:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.208,"15min":1.591},"min":{"1min":0.41699999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.97599999999999998,"5min":6.8479999999999999,"15min":15.455},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.63,"5min":1.3129999999999999,"15min":1.5329999999999999},"min":{"1min":0.46600000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.0309999999999999,"5min":10.099,"15min":14.837},"last":0.625}]},{"osd":7,"last update":"Mon Oct 9 11:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.252,"15min":1.0840000000000001},"min":{"1min":0.36299999999999999,"5min":0.315,"15min":0.315},"max":{"1min":1.0029999999999999,"5min":6.907,"15min":6.907},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.4339999999999999,"15min":1.2490000000000001},"min":{"1min":0.44400000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.0429999999999999,"5min":6.9119999999999999,"15min":6.9119999999999999},"last":0.57599999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019326,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903412,"kb_used_data":6572,"kb_used_omap":0,"kb_used_meta":896832,"kb_avail":92841740,"statfs":{"total":95995035648,"available":95069941760,"internally_reserved":0,"allocated":6729728,"data_stored":4874827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918355968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:10:21 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":1.042,"15min":1.5640000000000001},"min":{"1min":0.34300000000000003,"5min":0.224,"15min":0.20300000000000001},"max":{"1min":0.69999999999999996,"5min":8.3460000000000001,"15min":15.128},"last":0.622},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.0700000000000001,"15min":1.6659999999999999},"min":{"1min":0.32100000000000001,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":1.5,"5min":4.6699999999999999,"15min":14.359999999999999},"last":0.53800000000000003}]},{"osd":1,"last update":"Mon Oct 9 11:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":2.1509999999999998,"15min":2.4340000000000002},"min":{"1min":0.32300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.82799999999999996,"5min":57.966999999999999,"15min":121.41},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":1.79,"15min":2.0270000000000001},"min":{"1min":0.35499999999999998,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":0.78100000000000003,"5min":57.832000000000001,"15min":122.21899999999999},"last":0.51400000000000001}]},{"osd":2,"last update":"Mon Oct 9 11:10:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.3180000000000001,"15min":1.6279999999999999},"min":{"1min":0.33700000000000002,"5min":0.246,"15min":0.23799999999999999},"max":{"1min":1.5249999999999999,"5min":18.407,"15min":18.407},"last":0.378},{"interface":"front","average":{"1min":0.57799999999999996,"5min":1.28,"15min":1.9410000000000001},"min":{"1min":0.307,"5min":0.27900000000000003,"15min":0.26400000000000001},"max":{"1min":1.5600000000000001,"5min":7.2119999999999997,"15min":27.818000000000001},"last":0.433}]},{"osd":3,"last update":"Mon Oct 9 11:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":2.0339999999999998,"15min":1.845},"min":{"1min":0.38500000000000001,"5min":0.34799999999999998,"15min":0.27700000000000002},"max":{"1min":0.89900000000000002,"5min":60.753,"15min":60.753},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.61799999999999999,"5min":2.069,"15min":1.4690000000000001},"min":{"1min":0.40899999999999997,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.413,"5min":61.473999999999997,"15min":61.473999999999997},"last":0.47799999999999998}]},{"osd":5,"last update":"Mon Oct 9 11:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.83399999999999996,"15min":1.171},"min":{"1min":0.28499999999999998,"5min":0.27400000000000002,"15min":0.224},"max":{"1min":1.5109999999999999,"5min":4.9409999999999998,"15min":11.59},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.751,"15min":1.0720000000000001},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":1.542,"5min":8.3230000000000004,"15min":12.459},"last":0.58099999999999996}]},{"osd":6,"last update":"Mon Oct 9 11:10:01 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":1.0089999999999999,"15min":1.0800000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":0.86499999999999999,"5min":8.3930000000000007,"15min":8.5739999999999998},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.1339999999999999,"15min":1.355},"min":{"1min":0.372,"5min":0.33700000000000002,"15min":0.29999999999999999},"max":{"1min":1.45,"5min":9.5210000000000008,"15min":12.795999999999999},"last":0.63900000000000001}]},{"osd":7,"last update":"Mon Oct 9 11:10:01 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.1930000000000001,"15min":1.4610000000000001},"min":{"1min":0.32800000000000001,"5min":0.308,"15min":0.27700000000000002},"max":{"1min":1.4790000000000001,"5min":10.15,"15min":19.923999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.63200000000000001,"5min":1.0049999999999999,"15min":1.351},"min":{"1min":0.37,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.5760000000000001,"5min":10.853,"15min":19.859999999999999},"last":0.60799999999999998}]}]},{"osd":5,"up_from":37,"seq":158913790395,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1206120,"kb_used_data":6816,"kb_used_omap":0,"kb_used_meta":1199296,"kb_avail":92539032,"statfs":{"total":95995035648,"available":94759968768,"internally_reserved":0,"allocated":6979584,"data_stored":5132955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1228079104},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 11:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":1.1259999999999999,"15min":2.137},"min":{"1min":0.35499999999999998,"5min":0.219,"15min":0.21299999999999999},"max":{"1min":0.91500000000000004,"5min":6.2249999999999996,"15min":58.271999999999998},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.624,"5min":1.177,"15min":2.1840000000000002},"min":{"1min":0.49099999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.96699999999999997,"5min":6.2649999999999997,"15min":58.25},"last":0.60099999999999998}]},{"osd":1,"last update":"Mon Oct 9 11:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.4099999999999999,"15min":1.8700000000000001},"min":{"1min":0.38100000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.88300000000000001,"5min":16.962,"15min":32.491999999999997},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.70099999999999996,"5min":1.4099999999999999,"15min":1.784},"min":{"1min":0.45000000000000001,"5min":0.30499999999999999,"15min":0.247},"max":{"1min":0.98899999999999999,"5min":17.033999999999999,"15min":32.537999999999997},"last":0.57699999999999996}]},{"osd":2,"last update":"Mon Oct 9 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":1.0489999999999999,"15min":1.716},"min":{"1min":0.32800000000000001,"5min":0.26800000000000002,"15min":0.221},"max":{"1min":0.69599999999999995,"5min":7.3109999999999999,"15min":11.257999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":1.407,"15min":1.8340000000000001},"min":{"1min":0.32600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.78100000000000003,"5min":8.0359999999999996,"15min":14.621},"last":0.46000000000000002}]},{"osd":3,"last update":"Mon Oct 9 11:10:23 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.417,"15min":1.4650000000000001},"min":{"1min":0.377,"5min":0.29799999999999999,"15min":0.28799999999999998},"max":{"1min":0.747,"5min":15.535,"15min":15.535},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.661,"15min":1.7589999999999999},"min":{"1min":0.41599999999999998,"5min":0.38700000000000001,"15min":0.33600000000000002},"max":{"1min":0.77700000000000002,"5min":15.757999999999999,"15min":15.757999999999999},"last":0.435}]},{"osd":4,"last update":"Mon Oct 9 11:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.69099999999999995,"15min":0.89000000000000001},"min":{"1min":0.34000000000000002,"5min":0.26000000000000001,"15min":0.216},"max":{"1min":0.84299999999999997,"5min":3.0680000000000001,"15min":12.042},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.626,"5min":0.85699999999999998,"15min":1.117},"min":{"1min":0.35499999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.78900000000000003,"5min":6.8639999999999999,"15min":9.2490000000000006},"last":0.48099999999999998}]},{"osd":6,"last update":"Mon Oct 9 11:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.81200000000000006,"15min":1.1200000000000001},"min":{"1min":0.40899999999999997,"5min":0.27200000000000002,"15min":0.27100000000000002},"max":{"1min":0.79300000000000004,"5min":5.944,"15min":9.1969999999999992},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.89600000000000002,"15min":1.0169999999999999},"min":{"1min":0.41499999999999998,"5min":0.29699999999999999,"15min":0.29099999999999998},"max":{"1min":0.878,"5min":9.298,"15min":11.983000000000001},"last":0.628}]},{"osd":7,"last update":"Mon Oct 9 11:10:12 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.85299999999999998,"15min":1.2050000000000001},"min":{"1min":0.39400000000000002,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":0.82799999999999996,"5min":5.9699999999999998,"15min":9.3539999999999992},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.85399999999999998,"15min":1.1220000000000001},"min":{"1min":0.439,"5min":0.30499999999999999,"15min":0.30099999999999999},"max":{"1min":0.91500000000000004,"5min":5.6920000000000002,"15min":11.726000000000001},"last":0.64900000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4386,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3733,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T11:10:56.153 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T11:10:56.154 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T11:10:56.216 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=6dcb33211f0f474db9102b2ab94aa932 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T11:10:56.217 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-09T11:10:56.227 INFO:tasks.cephadm:Teardown begin 2023-10-09T11:10:56.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T11:10:56.290 DEBUG:teuthology.orchestra.run.smithi155:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T11:10:56.351 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-09T11:10:56.351 DEBUG:teuthology.orchestra.run.smithi125:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T11:10:56.369 DEBUG:teuthology.orchestra.run.smithi155:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T11:10:56.390 INFO:tasks.cephadm:Stopping all daemons... 2023-10-09T11:10:56.390 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-09T11:10:56.390 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T11:10:56.747 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:56 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:11:10:56] ENGINE Bus STOPPING 2023-10-09T11:10:56.747 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:56 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:11:10:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T11:10:56.747 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:56 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:11:10:56] ENGINE Bus STOPPED 2023-10-09T11:10:56.748 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:56 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:11:10:56] ENGINE Bus STARTING 2023-10-09T11:10:56.748 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:56 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:11:10:56] ENGINE Serving on http://:::9283 2023-10-09T11:10:56.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:56 smithi125 systemd[1]: Stopping Ceph mon.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:10:56.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:56 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T11:10:56.624+0000 7fbf36728700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T11:10:56.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:56 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a[195483]: 2023-10-09T11:10:56.624+0000 7fbf36728700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-10-09T11:10:57.070 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 11:10:56 smithi125 bash[496377]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T11:10:57.070 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:56 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:11:10:56] ENGINE Bus STARTED 2023-10-09T11:10:57.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-09T11:10:57.213 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T11:10:57.214 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-09T11:10:57.214 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-10-09T11:10:57.214 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.c 2023-10-09T11:10:57.466 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:57 smithi125 systemd[1]: Stopping Ceph mon.c for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:10:57.767 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-c[200171]: 2023-10-09T11:10:57.465+0000 7feb2f9ae700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T11:10:57.768 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-c[200171]: 2023-10-09T11:10:57.465+0000 7feb2f9ae700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-10-09T11:10:57.768 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:11:10:57] ENGINE Bus STOPPING 2023-10-09T11:10:57.768 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:11:10:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T11:10:57.768 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:11:10:57] ENGINE Bus STOPPED 2023-10-09T11:10:57.768 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:11:10:57] ENGINE Bus STARTING 2023-10-09T11:10:57.769 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:11:10:57] ENGINE Serving on http://:::9283 2023-10-09T11:10:57.769 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:57 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: [09/Oct/2023:11:10:57] ENGINE Bus STARTED 2023-10-09T11:10:58.069 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 11:10:57 smithi125 bash[496532]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T11:10:58.074 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.c.service' 2023-10-09T11:10:58.110 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T11:10:58.111 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-10-09T11:10:58.111 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-10-09T11:10:58.111 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.b 2023-10-09T11:10:58.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:58 smithi155 systemd[1]: Stopping Ceph mon.b for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:10:58.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:58 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-b[184831]: 2023-10-09T11:10:58.367+0000 7f4e5e962700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T11:10:58.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:58 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-b[184831]: 2023-10-09T11:10:58.367+0000 7f4e5e962700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-10-09T11:10:58.872 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 11:10:58 smithi155 bash[328085]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T11:10:58.879 DEBUG:teuthology.orchestra.run.smithi155:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mon.b.service' 2023-10-09T11:10:58.928 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T11:10:58.928 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-10-09T11:10:58.929 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-10-09T11:10:58.929 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T11:10:59.309 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:58 smithi125 systemd[1]: Stopping Ceph mgr.y for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:10:59.310 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:59 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y[183351]: 2023-10-09T11:10:59.147+0000 7fa569ecd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T11:10:59.570 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 11:10:59 smithi125 bash[496687]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T11:10:59.656 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.y.service' 2023-10-09T11:10:59.691 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T11:10:59.692 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-10-09T11:10:59.692 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-09T11:10:59.692 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.x 2023-10-09T11:11:00.029 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 11:10:59 smithi155 systemd[1]: Stopping Ceph mgr.x for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:11:00.302 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 11:11:00 smithi155 bash[328229]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T11:11:00.302 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 11:11:00 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T11:11:00.307 DEBUG:teuthology.orchestra.run.smithi155:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-09T11:11:00.344 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T11:11:00.344 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-09T11:11:00.344 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T11:11:00.344 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T11:11:00.820 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 11:11:00 smithi125 systemd[1]: Stopping Ceph osd.0 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:11:00.821 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 11:11:00 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-0[209418]: 2023-10-09T11:11:00.563+0000 7f4eeff5d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T11:11:00.821 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 11:11:00 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-0[209418]: 2023-10-09T11:11:00.563+0000 7f4eeff5d700 -1 osd.0 619 *** Got signal Terminated *** 2023-10-09T11:11:00.821 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 11:11:00 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-0[209418]: 2023-10-09T11:11:00.563+0000 7f4eeff5d700 -1 osd.0 619 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T11:11:06.820 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 11:11:06 smithi125 bash[496834]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T11:11:07.807 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T11:11:07.849 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T11:11:07.850 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T11:11:07.850 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-09T11:11:07.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T11:11:08.070 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:11:07 smithi125 systemd[1]: Stopping Ceph osd.1 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:11:08.570 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:11:08 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[391396]: 2023-10-09T11:11:08.088+0000 7f8d37be9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T11:11:08.570 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:11:08 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[391396]: 2023-10-09T11:11:08.088+0000 7f8d37be9700 -1 osd.1 619 *** Got signal Terminated *** 2023-10-09T11:11:08.571 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:11:08 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1[391396]: 2023-10-09T11:11:08.088+0000 7f8d37be9700 -1 osd.1 619 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T11:11:14.070 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 11:11:13 smithi125 bash[497100]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T11:11:15.006 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-09T11:11:15.047 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T11:11:15.047 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-09T11:11:15.048 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-09T11:11:15.048 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T11:11:15.296 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 11:11:15 smithi125 systemd[1]: Stopping Ceph osd.2 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:11:15.570 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 11:11:15 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-2[222806]: 2023-10-09T11:11:15.295+0000 7f7804a5b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T11:11:15.571 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 11:11:15 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-2[222806]: 2023-10-09T11:11:15.295+0000 7f7804a5b700 -1 osd.2 619 *** Got signal Terminated *** 2023-10-09T11:11:15.571 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 11:11:15 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-2[222806]: 2023-10-09T11:11:15.295+0000 7f7804a5b700 -1 osd.2 619 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T11:11:21.570 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 11:11:21 smithi125 bash[497367]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T11:11:22.388 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-09T11:11:22.428 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T11:11:22.428 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-09T11:11:22.428 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-09T11:11:22.428 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T11:11:22.676 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:11:22 smithi125 systemd[1]: Stopping Ceph osd.3 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:11:23.070 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:11:22 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[428685]: 2023-10-09T11:11:22.674+0000 7f4e51094700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T11:11:23.071 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:11:22 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[428685]: 2023-10-09T11:11:22.674+0000 7f4e51094700 -1 osd.3 619 *** Got signal Terminated *** 2023-10-09T11:11:23.071 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:11:22 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3[428685]: 2023-10-09T11:11:22.674+0000 7f4e51094700 -1 osd.3 619 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T11:11:24.720 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:24.392Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.125:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.125:8765: connect: connection refused" 2023-10-09T11:11:24.721 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:24.393Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.125:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.125:8765: connect: connection refused" 2023-10-09T11:11:24.721 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:24.396Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.125:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.125:8765: connect: connection refused" 2023-10-09T11:11:24.721 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:24 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:24.396Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.125:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.125:8765: connect: connection refused" 2023-10-09T11:11:27.470 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:27 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:27.209+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:28.171 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:27 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:27.802+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:00.302035+0000 front 2023-10-09T11:11:00.301976+0000 (oldest deadline 2023-10-09T11:11:27.302226+0000) 2023-10-09T11:11:28.470 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:28 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:28.171+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:28.571 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 11:11:28 smithi125 bash[497628]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T11:11:28.832 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:28 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:28.568+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:29.197 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:28 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:28.832+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:00.302035+0000 front 2023-10-09T11:11:00.301976+0000 (oldest deadline 2023-10-09T11:11:27.302226+0000) 2023-10-09T11:11:29.453 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-09T11:11:29.470 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:29 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:29.196+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:29.492 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T11:11:29.492 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-09T11:11:29.493 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T11:11:29.493 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T11:11:29.785 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:29 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:29.573+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:29.785 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 11:11:29 smithi155 systemd[1]: Stopping Ceph osd.4 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:11:29.786 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 11:11:29 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-4[123099]: 2023-10-09T11:11:29.715+0000 7f0a81247700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T11:11:29.786 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 11:11:29 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-4[123099]: 2023-10-09T11:11:29.715+0000 7f0a81247700 -1 osd.4 619 *** Got signal Terminated *** 2023-10-09T11:11:29.786 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 11:11:29 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-4[123099]: 2023-10-09T11:11:29.715+0000 7f0a81247700 -1 osd.4 619 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T11:11:30.188 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:29 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:29.785+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:00.302035+0000 front 2023-10-09T11:11:00.301976+0000 (oldest deadline 2023-10-09T11:11:27.302226+0000) 2023-10-09T11:11:30.470 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:30 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:30.188+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:30.970 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:30 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:30.785+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:00.302035+0000 front 2023-10-09T11:11:00.301976+0000 (oldest deadline 2023-10-09T11:11:27.302226+0000) 2023-10-09T11:11:30.971 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:30 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:30.576+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:31.523 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:31 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:31.222+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:31.970 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 11:11:31 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-4[123099]: 2023-10-09T11:11:31.523+0000 7f0a7c042700 -1 osd.4 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:05.520535+0000 front 2023-10-09T11:11:05.520298+0000 (oldest deadline 2023-10-09T11:11:30.820375+0000) 2023-10-09T11:11:31.970 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:31 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:31.741+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:00.302035+0000 front 2023-10-09T11:11:00.301976+0000 (oldest deadline 2023-10-09T11:11:27.302226+0000) 2023-10-09T11:11:31.971 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:31 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:31.590+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:32.507 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:32 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:32.237+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:32.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:32 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:32.573+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:32.971 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 11:11:32 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-4[123099]: 2023-10-09T11:11:32.507+0000 7f0a7c042700 -1 osd.4 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:05.520535+0000 front 2023-10-09T11:11:05.520298+0000 (oldest deadline 2023-10-09T11:11:30.820375+0000) 2023-10-09T11:11:32.971 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:32 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:32.754+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:00.302035+0000 front 2023-10-09T11:11:00.301976+0000 (oldest deadline 2023-10-09T11:11:27.302226+0000) 2023-10-09T11:11:33.571 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 11:11:33 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-4[123099]: 2023-10-09T11:11:33.515+0000 7f0a7c042700 -1 osd.4 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:05.520535+0000 front 2023-10-09T11:11:05.520298+0000 (oldest deadline 2023-10-09T11:11:30.820375+0000) 2023-10-09T11:11:33.571 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:33 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:33.271+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:33.970 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:33 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:33.750+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:00.302035+0000 front 2023-10-09T11:11:00.301976+0000 (oldest deadline 2023-10-09T11:11:27.302226+0000) 2023-10-09T11:11:33.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:33 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:33.570+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:34.587 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 11:11:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-4[123099]: 2023-10-09T11:11:34.528+0000 7f0a7c042700 -1 osd.4 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:05.520535+0000 front 2023-10-09T11:11:05.520298+0000 (oldest deadline 2023-10-09T11:11:30.820375+0000) 2023-10-09T11:11:34.587 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:34.317+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:34.587 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:34.317+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:34.970 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:34.792+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:00.302035+0000 front 2023-10-09T11:11:00.301976+0000 (oldest deadline 2023-10-09T11:11:27.302226+0000) 2023-10-09T11:11:34.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:34.586+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:34.971 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:34 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:34.586+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:35.720 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 11:11:35 smithi155 bash[328385]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-4 2023-10-09T11:11:35.721 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:35 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:35.342+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:35.721 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:35 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:35.342+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:35.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:35 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:35.566+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:35.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:35 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:35.566+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:36.220 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:35 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:35.806+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:00.302035+0000 front 2023-10-09T11:11:00.301976+0000 (oldest deadline 2023-10-09T11:11:27.302226+0000) 2023-10-09T11:11:36.720 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:36 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:36.558+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:36.720 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:36 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:36.558+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:36.721 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:36 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:36.348+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:36.721 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:36 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:36.348+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:36.890 DEBUG:teuthology.orchestra.run.smithi155:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T11:11:36.930 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T11:11:36.931 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T11:11:36.931 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-09T11:11:36.931 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T11:11:36.996 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:36 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:36.843+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:00.302035+0000 front 2023-10-09T11:11:00.301976+0000 (oldest deadline 2023-10-09T11:11:27.302226+0000) 2023-10-09T11:11:37.356 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:37 smithi155 systemd[1]: Stopping Ceph osd.5 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:11:37.356 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:37.179+0000 7fbf17a8e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T11:11:37.356 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:37.179+0000 7fbf17a8e700 -1 osd.5 619 *** Got signal Terminated *** 2023-10-09T11:11:37.356 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:37.179+0000 7fbf17a8e700 -1 osd.5 619 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T11:11:37.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:37.355+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:37.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:37.355+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:37.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:37.554+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:37.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:37.554+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:38.220 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:37.839+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:00.302035+0000 front 2023-10-09T11:11:00.301976+0000 (oldest deadline 2023-10-09T11:11:27.302226+0000) 2023-10-09T11:11:38.221 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:37 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:37.839+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:09.003420+0000 front 2023-10-09T11:11:09.003433+0000 (oldest deadline 2023-10-09T11:11:37.203985+0000) 2023-10-09T11:11:38.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:38.383+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:38.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:38.383+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:38.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:38.589+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:38.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:38.589+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:39.220 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:38.856+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:00.302035+0000 front 2023-10-09T11:11:00.301976+0000 (oldest deadline 2023-10-09T11:11:27.302226+0000) 2023-10-09T11:11:39.220 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:38 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:38.856+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:09.003420+0000 front 2023-10-09T11:11:09.003433+0000 (oldest deadline 2023-10-09T11:11:37.203985+0000) 2023-10-09T11:11:39.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:39 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:39.426+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:39.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:39 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:39.426+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:39.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:39 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:39.564+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:39.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:39 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:39.564+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:40.220 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:39 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:39.832+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:00.302035+0000 front 2023-10-09T11:11:00.301976+0000 (oldest deadline 2023-10-09T11:11:27.302226+0000) 2023-10-09T11:11:40.220 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:39 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:39.832+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:09.003420+0000 front 2023-10-09T11:11:09.003433+0000 (oldest deadline 2023-10-09T11:11:37.203985+0000) 2023-10-09T11:11:40.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:40 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:40.469+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:40.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:40 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:40.469+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:40.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:40 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:40.566+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:40.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:40 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:40.566+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:41.220 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:40 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:40.864+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:00.302035+0000 front 2023-10-09T11:11:00.301976+0000 (oldest deadline 2023-10-09T11:11:27.302226+0000) 2023-10-09T11:11:41.220 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:40 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:40.864+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:09.003420+0000 front 2023-10-09T11:11:09.003433+0000 (oldest deadline 2023-10-09T11:11:37.203985+0000) 2023-10-09T11:11:41.838 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:41.613+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:41.838 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:41.613+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:41.839 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:41.507+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:41.839 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:41.507+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:42.220 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:41.837+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:00.302035+0000 front 2023-10-09T11:11:00.301976+0000 (oldest deadline 2023-10-09T11:11:27.302226+0000) 2023-10-09T11:11:42.220 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:41 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5[130071]: 2023-10-09T11:11:41.837+0000 7fbf11086700 -1 osd.5 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:09.003420+0000 front 2023-10-09T11:11:09.003433+0000 (oldest deadline 2023-10-09T11:11:37.203985+0000) 2023-10-09T11:11:42.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:42 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:42.468+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:42.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:42 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:42.468+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:42.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:42 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:42.593+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:42.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:42 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:42.593+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:43.220 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 11:11:42 smithi155 bash[328648]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-5 2023-10-09T11:11:43.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:43.433+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:43.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:43.433+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:43.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:43.609+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:43.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:43 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:43.609+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:44.164 DEBUG:teuthology.orchestra.run.smithi155:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-09T11:11:44.247 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T11:11:44.248 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-09T11:11:44.248 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-09T11:11:44.248 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T11:11:44.462 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:44 smithi155 systemd[1]: Stopping Ceph osd.6 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:11:44.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:44.460+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:44.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:44.460+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:44.721 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:44.520+0000 7fecc58af700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T11:11:44.721 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:44.520+0000 7fecc58af700 -1 osd.6 619 *** Got signal Terminated *** 2023-10-09T11:11:44.721 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:44.520+0000 7fecc58af700 -1 osd.6 619 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T11:11:44.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:44.617+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:44.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:44.617+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:44.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:44 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:44.617+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.784185+0000 front 2023-10-09T11:11:19.784102+0000 (oldest deadline 2023-10-09T11:11:43.884098+0000) 2023-10-09T11:11:45.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:45 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:45.430+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:45.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:45 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:45.430+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:45.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:45 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:45.430+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.482501+0000 front 2023-10-09T11:11:19.482550+0000 (oldest deadline 2023-10-09T11:11:44.782325+0000) 2023-10-09T11:11:45.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:45 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:45.575+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:45.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:45 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:45.576+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:45.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:45 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:45.576+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.784185+0000 front 2023-10-09T11:11:19.784102+0000 (oldest deadline 2023-10-09T11:11:43.884098+0000) 2023-10-09T11:11:46.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:46 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:46.426+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:46.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:46 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:46.426+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:46.721 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:46 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:46.426+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.482501+0000 front 2023-10-09T11:11:19.482550+0000 (oldest deadline 2023-10-09T11:11:44.782325+0000) 2023-10-09T11:11:46.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:46 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:46.567+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:46.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:46 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:46.567+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:46.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:46 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:46.567+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.784185+0000 front 2023-10-09T11:11:19.784102+0000 (oldest deadline 2023-10-09T11:11:43.884098+0000) 2023-10-09T11:11:47.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:47 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:47.421+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:47.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:47 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:47.421+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:47.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:47 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:47.421+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.482501+0000 front 2023-10-09T11:11:19.482550+0000 (oldest deadline 2023-10-09T11:11:44.782325+0000) 2023-10-09T11:11:47.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:47 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:47.534+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:47.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:47 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:47.534+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:47.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:47 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:47.534+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.784185+0000 front 2023-10-09T11:11:19.784102+0000 (oldest deadline 2023-10-09T11:11:43.884098+0000) 2023-10-09T11:11:48.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:48 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:48.397+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:48.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:48 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:48.397+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:48.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:48 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:48.397+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.482501+0000 front 2023-10-09T11:11:19.482550+0000 (oldest deadline 2023-10-09T11:11:44.782325+0000) 2023-10-09T11:11:48.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:48 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:48.584+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:48.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:48 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:48.584+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:48.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:48 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:48.584+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.784185+0000 front 2023-10-09T11:11:19.784102+0000 (oldest deadline 2023-10-09T11:11:43.884098+0000) 2023-10-09T11:11:49.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:49 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:49.349+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:04.579081+0000 front 2023-10-09T11:11:04.579124+0000 (oldest deadline 2023-10-09T11:11:26.278930+0000) 2023-10-09T11:11:49.720 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:49 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:49.349+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:10.780946+0000 front 2023-10-09T11:11:10.780959+0000 (oldest deadline 2023-10-09T11:11:34.280880+0000) 2023-10-09T11:11:49.721 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:49 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6[137190]: 2023-10-09T11:11:49.349+0000 7fecc16ac700 -1 osd.6 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.482501+0000 front 2023-10-09T11:11:19.482550+0000 (oldest deadline 2023-10-09T11:11:44.782325+0000) 2023-10-09T11:11:49.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:49 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:49.552+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:49.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:49 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:49.552+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:49.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:49 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:49.552+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.784185+0000 front 2023-10-09T11:11:19.784102+0000 (oldest deadline 2023-10-09T11:11:43.884098+0000) 2023-10-09T11:11:49.721 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:49 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:49.552+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T11:11:27.385185+0000 front 2023-10-09T11:11:27.385250+0000 (oldest deadline 2023-10-09T11:11:49.085234+0000) 2023-10-09T11:11:50.570 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 11:11:50 smithi155 bash[328911]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-6 2023-10-09T11:11:50.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:50.568+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:50.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:50.568+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:50.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:50.568+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.784185+0000 front 2023-10-09T11:11:19.784102+0000 (oldest deadline 2023-10-09T11:11:43.884098+0000) 2023-10-09T11:11:50.971 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:50 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:50.568+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T11:11:27.385185+0000 front 2023-10-09T11:11:27.385250+0000 (oldest deadline 2023-10-09T11:11:49.085234+0000) 2023-10-09T11:11:51.512 DEBUG:teuthology.orchestra.run.smithi155:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-09T11:11:51.553 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T11:11:51.554 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-09T11:11:51.554 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-09T11:11:51.554 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T11:11:51.805 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:51 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:51.561+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:51.805 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:51 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:51.561+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:51.805 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:51 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:51.561+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.784185+0000 front 2023-10-09T11:11:19.784102+0000 (oldest deadline 2023-10-09T11:11:43.884098+0000) 2023-10-09T11:11:51.805 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:51 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:51.561+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T11:11:27.385185+0000 front 2023-10-09T11:11:27.385250+0000 (oldest deadline 2023-10-09T11:11:49.085234+0000) 2023-10-09T11:11:51.805 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:51 smithi155 systemd[1]: Stopping Ceph osd.7 for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:11:52.220 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:51 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:51.803+0000 7f4fdfb0b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T11:11:52.220 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:51 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:51.803+0000 7f4fdfb0b700 -1 osd.7 619 *** Got signal Terminated *** 2023-10-09T11:11:52.220 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:51 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:51.803+0000 7f4fdfb0b700 -1 osd.7 619 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T11:11:52.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:52.523+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:52.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:52.523+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:52.971 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:52.523+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.784185+0000 front 2023-10-09T11:11:19.784102+0000 (oldest deadline 2023-10-09T11:11:43.884098+0000) 2023-10-09T11:11:52.971 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:52 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:52.523+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T11:11:27.385185+0000 front 2023-10-09T11:11:27.385250+0000 (oldest deadline 2023-10-09T11:11:49.085234+0000) 2023-10-09T11:11:53.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:53.490+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:53.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:53.490+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:53.971 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:53.490+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.784185+0000 front 2023-10-09T11:11:19.784102+0000 (oldest deadline 2023-10-09T11:11:43.884098+0000) 2023-10-09T11:11:53.971 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:53 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:53.490+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T11:11:27.385185+0000 front 2023-10-09T11:11:27.385250+0000 (oldest deadline 2023-10-09T11:11:49.085234+0000) 2023-10-09T11:11:54.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:54 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:54.497+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:54.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:54 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:54.498+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:54.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:54 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:54.498+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.784185+0000 front 2023-10-09T11:11:19.784102+0000 (oldest deadline 2023-10-09T11:11:43.884098+0000) 2023-10-09T11:11:54.971 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:54 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:54.498+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T11:11:27.385185+0000 front 2023-10-09T11:11:27.385250+0000 (oldest deadline 2023-10-09T11:11:49.085234+0000) 2023-10-09T11:11:55.720 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:55 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:55.451+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:55.720 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:55 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:55.451+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:55.720 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:55 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:55.451+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.784185+0000 front 2023-10-09T11:11:19.784102+0000 (oldest deadline 2023-10-09T11:11:43.884098+0000) 2023-10-09T11:11:55.720 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:55 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:55.451+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T11:11:27.385185+0000 front 2023-10-09T11:11:27.385250+0000 (oldest deadline 2023-10-09T11:11:49.085234+0000) 2023-10-09T11:11:56.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:56 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:56.488+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T11:11:02.782126+0000 front 2023-10-09T11:11:02.782038+0000 (oldest deadline 2023-10-09T11:11:28.081961+0000) 2023-10-09T11:11:56.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:56 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:56.488+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T11:11:11.583243+0000 front 2023-10-09T11:11:11.583103+0000 (oldest deadline 2023-10-09T11:11:33.883109+0000) 2023-10-09T11:11:56.971 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:56 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:56.488+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T11:11:19.784185+0000 front 2023-10-09T11:11:19.784102+0000 (oldest deadline 2023-10-09T11:11:43.884098+0000) 2023-10-09T11:11:56.971 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:56 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:56.488+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T11:11:27.385185+0000 front 2023-10-09T11:11:27.385250+0000 (oldest deadline 2023-10-09T11:11:49.085234+0000) 2023-10-09T11:11:56.971 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:56 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7[145089]: 2023-10-09T11:11:56.488+0000 7f4fdd90c700 -1 osd.7 619 heartbeat_check: no reply from 172.21.15.155:6804 osd.4 since back 2023-10-09T11:11:34.386077+0000 front 2023-10-09T11:11:34.386101+0000 (oldest deadline 2023-10-09T11:11:55.486046+0000) 2023-10-09T11:11:57.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 11:11:57 smithi155 bash[329174]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-osd-7 2023-10-09T11:11:58.737 DEBUG:teuthology.orchestra.run.smithi155:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-09T11:11:58.777 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T11:11:58.778 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-09T11:11:58.778 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-10-09T11:11:58.778 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@prometheus.a 2023-10-09T11:11:59.015 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:58 smithi155 systemd[1]: Stopping Ceph prometheus.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:11:59.426 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:59.015Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T11:11:59.427 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:59.015Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T11:11:59.427 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:59.015Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T11:11:59.427 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:59.015Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T11:11:59.427 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:59.015Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T11:11:59.427 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:59.015Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T11:11:59.427 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:59.015Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T11:11:59.428 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:59.015Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T11:11:59.428 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:59.015Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T11:11:59.428 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:59.016Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T11:11:59.428 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:59.016Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T11:11:59.428 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:59 smithi155 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a[172426]: ts=2023-10-09T11:11:59.016Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T11:11:59.428 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 11:11:59 smithi155 bash[329435]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T11:11:59.432 DEBUG:teuthology.orchestra.run.smithi155:> sudo pkill -f 'journalctl -f -n 0 -u ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@prometheus.a.service' 2023-10-09T11:11:59.466 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T11:11:59.467 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-10-09T11:11:59.467 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T11:12:00.820 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 11:12:00 smithi125 systemd[1]: Stopping Ceph alertmanager.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:12:01.082 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 11:12:00 smithi125 ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a[182292]: ts=2023-10-09T11:12:00.831Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T11:12:01.083 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 11:12:01 smithi125 bash[498164]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T11:12:01.765 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 11:12:01 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T11:12:01.765 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 11:12:01 smithi125 systemd[1]: Stopped Ceph alertmanager.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T11:12:01.765 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 11:12:01 smithi125 systemd[1]: Stopping Ceph node-exporter.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:12:02.070 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 11:12:01 smithi125 bash[498293]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T11:12:02.070 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 11:12:01 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T11:12:02.372 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 11:12:02 smithi125 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T11:12:02.372 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 11:12:02 smithi125 systemd[1]: Stopped Ceph node-exporter.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T11:12:13.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T11:12:14.470 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 11:12:14 smithi155 systemd[1]: Stopping Ceph grafana.a for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:12:14.885 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 11:12:14 smithi155 bash[329845]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T11:12:14.885 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 11:12:14 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T11:12:15.220 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 11:12:14 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-10-09T11:12:15.220 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 11:12:14 smithi155 systemd[1]: Stopped Ceph grafana.a for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T11:12:15.221 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 11:12:14 smithi155 systemd[1]: Stopping Ceph node-exporter.b for daf690ae-668e-11ee-8db5-212e2dc638e7... 2023-10-09T11:12:15.556 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 11:12:15 smithi155 bash[329975]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T11:12:15.556 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 11:12:15 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T11:12:15.872 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 11:12:15 smithi155 systemd[1]: ceph-daf690ae-668e-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T11:12:15.872 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 11:12:15 smithi155 systemd[1]: Stopped Ceph node-exporter.b for daf690ae-668e-11ee-8db5-212e2dc638e7. 2023-10-09T11:12:20.069 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T11:12:20.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T11:12:20.140 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-09T11:12:20.142 DEBUG:teuthology.misc:Transferring archived files from smithi125:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417662/remote/smithi125/crash 2023-10-09T11:12:20.143 DEBUG:teuthology.orchestra.run.smithi125:> sudo tar c -f - -C /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T11:12:20.203 INFO:teuthology.orchestra.run.smithi125.stderr:tar: /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T11:12:20.203 INFO:teuthology.orchestra.run.smithi125.stderr:tar: Error is not recoverable: exiting now 2023-10-09T11:12:20.205 DEBUG:teuthology.misc:Transferring archived files from smithi155:/var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417662/remote/smithi155/crash 2023-10-09T11:12:20.206 DEBUG:teuthology.orchestra.run.smithi155:> sudo tar c -f - -C /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T11:12:20.274 INFO:teuthology.orchestra.run.smithi155.stderr:tar: /var/lib/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T11:12:20.274 INFO:teuthology.orchestra.run.smithi155.stderr:tar: Error is not recoverable: exiting now 2023-10-09T11:12:20.275 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-09T11:12:20.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-09T11:12:20.310 INFO:teuthology.orchestra.run.smithi125.stderr:grep: /var/log/ceph/daf690ae-668e-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-09T11:12:20.312 INFO:tasks.cephadm:Compressing logs... 2023-10-09T11:12:20.312 DEBUG:teuthology.orchestra.run.smithi125:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T11:12:20.355 DEBUG:teuthology.orchestra.run.smithi155:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T11:12:20.397 INFO:teuthology.orchestra.run.smithi155.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-09T11:12:20.443 INFO:teuthology.orchestra.run.smithi125.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-09T11:22:45.459 INFO:tasks.cephadm:Archiving logs... 2023-10-09T11:22:45.491 DEBUG:teuthology.misc:Transferring archived files from smithi125:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417662/remote/smithi125/log 2023-10-09T11:22:45.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T11:24:04.538 DEBUG:teuthology.misc:Transferring archived files from smithi155:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417662/remote/smithi155/log 2023-10-09T11:24:04.539 DEBUG:teuthology.orchestra.run.smithi155:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T11:24:52.586 INFO:tasks.cephadm:Removing cluster... 2023-10-09T11:24:52.587 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 --force 2023-10-09T11:24:53.760 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid daf690ae-668e-11ee-8db5-212e2dc638e7 --force 2023-10-09T11:24:54.620 INFO:tasks.cephadm:Removing cephadm ... 2023-10-09T11:24:54.621 DEBUG:teuthology.orchestra.run.smithi125:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T11:24:54.643 DEBUG:teuthology.orchestra.run.smithi155:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T11:24:54.662 INFO:tasks.cephadm:Teardown complete 2023-10-09T11:24:54.663 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-10-09T11:24:54.677 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-10-09T11:24:54.677 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T11:24:54.686 DEBUG:teuthology.orchestra.run.smithi155:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T11:24:54.834 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-09T11:24:54.846 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-09T11:24:54.847 DEBUG:teuthology.orchestra.run.smithi125:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T11:24:54.849 DEBUG:teuthology.orchestra.run.smithi155:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T11:24:54.866 INFO:teuthology.orchestra.run.smithi155.stderr:bash: ntpq: command not found 2023-10-09T11:24:54.867 INFO:teuthology.orchestra.run.smithi125.stderr:bash: ntpq: command not found 2023-10-09T11:24:54.886 INFO:teuthology.orchestra.run.smithi155.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T11:24:54.886 INFO:teuthology.orchestra.run.smithi155.stdout:=============================================================================== 2023-10-09T11:24:54.886 INFO:teuthology.orchestra.run.smithi155.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 62 -3227ns[-3227ns] +/- 94ms 2023-10-09T11:24:54.887 INFO:teuthology.orchestra.run.smithi155.stdout:^* hv02.front.sepia.ceph.com 3 6 377 63 +217us[ +233us] +/- 68ms 2023-10-09T11:24:54.887 INFO:teuthology.orchestra.run.smithi155.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 62 -97us[ -97us] +/- 78ms 2023-10-09T11:24:54.887 INFO:teuthology.orchestra.run.smithi155.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T11:24:54.888 INFO:teuthology.orchestra.run.smithi125.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T11:24:54.888 INFO:teuthology.orchestra.run.smithi125.stdout:=============================================================================== 2023-10-09T11:24:54.888 INFO:teuthology.orchestra.run.smithi125.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 61 -29us[ -32us] +/- 94ms 2023-10-09T11:24:54.888 INFO:teuthology.orchestra.run.smithi125.stdout:^* hv02.front.sepia.ceph.com 3 6 377 59 +188us[ +185us] +/- 68ms 2023-10-09T11:24:54.888 INFO:teuthology.orchestra.run.smithi125.stdout:^+ hv03.front.sepia.ceph.com 3 7 377 59 -110us[ -113us] +/- 78ms 2023-10-09T11:24:54.888 INFO:teuthology.orchestra.run.smithi125.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T11:24:54.889 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-09T11:24:54.899 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-09T11:24:54.899 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-09T11:24:54.927 DEBUG:teuthology.orchestra.run.smithi125:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-09T11:24:55.062 DEBUG:teuthology.orchestra.run.smithi155:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-09T11:24:55.180 DEBUG:teuthology.orchestra.run.smithi125:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T11:24:55.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T11:24:55.213 DEBUG:teuthology.orchestra.run.smithi155:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T11:24:55.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T11:24:55.245 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-09T11:24:55.256 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-09T11:24:55.301 INFO:teuthology.task.internal:Duration was 3942.029478 seconds 2023-10-09T11:24:55.301 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-09T11:24:55.334 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-09T11:24:55.334 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T11:24:55.336 DEBUG:teuthology.orchestra.run.smithi155:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T11:24:55.426 INFO:teuthology.orchestra.run.smithi125.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T11:24:55.454 INFO:teuthology.orchestra.run.smithi155.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T11:24:55.865 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-09T11:24:55.865 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi125.front.sepia.ceph.com 2023-10-09T11:24:55.866 DEBUG:teuthology.orchestra.run.smithi125:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T11:24:55.892 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi155.front.sepia.ceph.com 2023-10-09T11:24:55.892 DEBUG:teuthology.orchestra.run.smithi155:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T11:24:55.938 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-09T11:24:55.938 DEBUG:teuthology.orchestra.run.smithi125:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T11:24:55.940 DEBUG:teuthology.orchestra.run.smithi155:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T11:24:56.089 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-09T11:24:56.090 DEBUG:teuthology.orchestra.run.smithi125:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T11:24:56.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T11:24:56.424 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-09T11:24:56.440 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-09T11:24:56.440 DEBUG:teuthology.orchestra.run.smithi125:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T11:24:56.489 DEBUG:teuthology.orchestra.run.smithi155:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T11:24:56.562 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-09T11:24:56.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T11:24:56.579 DEBUG:teuthology.orchestra.run.smithi155:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T11:24:56.615 INFO:teuthology.orchestra.run.smithi125.stdout:kernel.core_pattern = core 2023-10-09T11:24:56.648 INFO:teuthology.orchestra.run.smithi155.stdout:kernel.core_pattern = core 2023-10-09T11:24:56.663 DEBUG:teuthology.orchestra.run.smithi125:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T11:24:56.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T11:24:56.687 DEBUG:teuthology.orchestra.run.smithi155:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T11:24:56.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T11:24:56.705 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-09T11:24:56.716 INFO:teuthology.task.internal:Transferring archived files... 2023-10-09T11:24:56.717 DEBUG:teuthology.misc:Transferring archived files from smithi125:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417662/remote/smithi125 2023-10-09T11:24:56.717 DEBUG:teuthology.orchestra.run.smithi125:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T11:24:56.789 DEBUG:teuthology.misc:Transferring archived files from smithi155:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417662/remote/smithi155 2023-10-09T11:24:56.789 DEBUG:teuthology.orchestra.run.smithi155:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T11:24:56.837 INFO:teuthology.task.internal:Removing archive directory... 2023-10-09T11:24:56.838 DEBUG:teuthology.orchestra.run.smithi125:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T11:24:56.840 DEBUG:teuthology.orchestra.run.smithi155:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T11:24:56.886 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-09T11:24:56.899 INFO:teuthology.task.internal:Not uploading archives. 2023-10-09T11:24:56.900 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-09T11:24:56.937 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-09T11:24:56.937 DEBUG:teuthology.orchestra.run.smithi125:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T11:24:56.939 DEBUG:teuthology.orchestra.run.smithi155:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T11:24:56.955 INFO:teuthology.orchestra.run.smithi125.stdout: 526080 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 9 11:24 /home/ubuntu/cephtest 2023-10-09T11:24:56.955 INFO:teuthology.orchestra.run.smithi125.stdout: 4456449 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 9 11:09 /home/ubuntu/cephtest/mnt.0 2023-10-09T11:24:56.955 INFO:teuthology.orchestra.run.smithi125.stdout: 4456450 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Oct 9 11:09 /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T11:24:56.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T11:24:56.956 INFO:teuthology.orchestra.run.smithi125.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-10-09T11:24:56.957 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi125 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T11:24:56.963 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-09T11:24:56.977 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-09T11:24:57.038 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi125 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T11:24:57.038 INFO:teuthology.nuke:Checking targets against current locks 2023-10-09T11:24:57.046 INFO:teuthology.orchestra.run.smithi155.stdout: 525728 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 9 11:24 /home/ubuntu/cephtest 2023-10-09T11:24:57.076 DEBUG:teuthology.nuke:shortname: smithi125 2023-10-09T11:24:57.077 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T11:24:57.101 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi125.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417662', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 10:11:03.970578', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLvfSjrBnafMJIFS7H9oHfjQHewGNMax+GQ3sxvUIztX+jkFCA9WPSbSLRc3J7q7WQPVpNnSwcAtA2o/q4iewss='} 2023-10-09T11:24:57.105 DEBUG:teuthology.nuke:shortname: smithi155 2023-10-09T11:24:57.105 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T11:24:57.129 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi155.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417662', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 10:11:03.972598', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP7/b5hy+Pcu9QddzM6XYEy8fuE7d73rSIMRFCY9EzEPXJxbWWrxGt2DebD8swkrGVC9PJaBejVh7lZlsmOEnrM='} 2023-10-09T11:24:57.157 INFO:teuthology.orchestra.console.smithi125:Power off 2023-10-09T11:24:57.157 DEBUG:teuthology.orchestra.console.smithi125:pexpect command: ipmitool -H smithi125.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T11:24:57.181 DEBUG:teuthology.orchestra.console.smithi125:power off output: Chassis Power Control: Down/Off 2023-10-09T11:24:57.182 DEBUG:teuthology.orchestra.console.smithi125:pexpect command: ipmitool -H smithi125.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T11:24:57.201 INFO:teuthology.orchestra.console.smithi155:Power off 2023-10-09T11:24:57.202 DEBUG:teuthology.orchestra.console.smithi155:pexpect command: ipmitool -H smithi155.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T11:24:57.212 DEBUG:teuthology.orchestra.console.smithi125:check power output: Chassis Power is on 2023-10-09T11:24:57.226 DEBUG:teuthology.orchestra.console.smithi155:power off output: Chassis Power Control: Down/Off 2023-10-09T11:24:57.227 DEBUG:teuthology.orchestra.console.smithi155:pexpect command: ipmitool -H smithi155.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T11:24:57.252 DEBUG:teuthology.orchestra.console.smithi155:check power output: Chassis Power is on 2023-10-09T11:25:01.213 DEBUG:teuthology.orchestra.console.smithi125:pexpect command: ipmitool -H smithi125.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T11:25:01.253 DEBUG:teuthology.orchestra.console.smithi155:pexpect command: ipmitool -H smithi155.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T11:25:01.323 DEBUG:teuthology.orchestra.console.smithi125:check power output: Chassis Power is on 2023-10-09T11:25:01.363 DEBUG:teuthology.orchestra.console.smithi155:check power output: Chassis Power is on 2023-10-09T11:25:05.329 DEBUG:teuthology.orchestra.console.smithi125:pexpect command: ipmitool -H smithi125.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T11:25:05.364 DEBUG:teuthology.orchestra.console.smithi155:pexpect command: ipmitool -H smithi155.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T11:25:05.440 DEBUG:teuthology.orchestra.console.smithi125:check power output: Chassis Power is on 2023-10-09T11:25:05.477 DEBUG:teuthology.orchestra.console.smithi155:check power output: Chassis Power is off 2023-10-09T11:25:05.577 INFO:teuthology.orchestra.console.smithi155:Power off completed 2023-10-09T11:25:09.442 DEBUG:teuthology.orchestra.console.smithi125:pexpect command: ipmitool -H smithi125.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T11:25:09.552 DEBUG:teuthology.orchestra.console.smithi125:check power output: Chassis Power is off 2023-10-09T11:25:09.653 INFO:teuthology.orchestra.console.smithi125:Power off completed 2023-10-09T11:25:09.757 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/centos_8.stream_container_tools 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 3942.0294783115387 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=6dcb33211f0f474db9102b2ab94aa932 status: fail success: false 2023-10-09T11:25:09.758 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T11:25:09.835 INFO:teuthology.run:FAIL